diff --git a/.ci/metrics/metrics.py b/.ci/metrics/metrics.py
index bd2b51154768d..e1712bd015151 100644
--- a/.ci/metrics/metrics.py
+++ b/.ci/metrics/metrics.py
@@ -1,21 +1,57 @@
-import requests
-import time
+import collections
+import datetime
+import github
+import logging
 import os
-from dataclasses import dataclass
+import requests
 import sys
-import logging
+import time
 
-import github
-from github import Github
+from dataclasses import dataclass
 from github import Auth
+from github import Github
 
 GRAFANA_URL = (
     "https://influx-prod-13-prod-us-east-0.grafana.net/api/v1/push/influx/write"
 )
-GITHUB_PROJECT = "llvm/llvm-project"
-WORKFLOWS_TO_TRACK = ["LLVM Premerge Checks"]
 SCRAPE_INTERVAL_SECONDS = 5 * 60
 
+# Lists the Github workflows we want to track. Maps the Github job name to
+# the metric name prefix in grafana.
+# This metric name is also used as a key in the job->name map.
+GITHUB_WORKFLOW_TO_TRACK = {"LLVM Premerge Checks": "github_llvm_premerge_checks"}
+
+# Lists the Github jobs to track for a given workflow. The key is the stable
+# name (metric name) of the workflow (see GITHUB_WORKFLOW_TO_TRACK).
+# Each value is a map to link the github job name to the corresponding metric
+# name.
+GITHUB_JOB_TO_TRACK = {
+    "github_llvm_premerge_checks": {
+        "Linux Premerge Checks (Test Only - Please Ignore Results)": "premerge_linux",
+        "Windows Premerge Checks (Test Only - Please Ignore Results)": "premerge_windows",
+    }
+}
+
+# The number of workflows to pull when sampling Github workflows.
+# - Github API filtering is broken: we cannot apply any filtering:
+# - See https://github.com/orgs/community/discussions/86766
+# - A workflow can complete before another workflow, even when starting later.
+# - We don't want to sample the same workflow twice.
+#
+# This means we essentially have a list of workflows sorted by creation date,
+# and that's all we can deduce from it. So for each iteration, we'll blindly
+# process the last N workflows.
+GITHUB_WORKFLOWS_MAX_PROCESS_COUNT = 1000
+# Second reason for the cut: reaching a workflow older than X.
+# This means we will miss long-tails (exceptional jobs running for more than
+# X hours), but that's also the case with the count cutoff above.
+# Only solution to avoid missing any workflow would be to process the complete
+# list, which is not possible.
+GITHUB_WORKFLOW_MAX_CREATED_AGE_HOURS = 8
+
+# Grafana will fail to insert any metric older than ~2 hours (value determined
+# by trial and error).
+GRAFANA_METRIC_MAX_AGE_MN = 120
 
 @dataclass
 class JobMetrics:
@@ -23,11 +59,10 @@ class JobMetrics:
     queue_time: int
     run_time: int
     status: int
-    created_at_ns: int
+    completed_at_ns: int
     workflow_id: int
     workflow_name: str
 
-
 @dataclass
 class GaugeMetric:
     name: str
@@ -35,141 +70,88 @@ class GaugeMetric:
     time_ns: int
 
 
-def get_sampled_workflow_metrics(github_repo: github.Repository):
-    """Gets global statistics about the Github workflow queue
-
-    Args:
-      github_repo: A github repo object to use to query the relevant information.
-
-    Returns:
-      Returns a list of GaugeMetric objects, containing the relevant metrics about
-      the workflow
-    """
-    queued_job_counts = {}
-    running_job_counts = {}
-
-    # Other states are available (pending, waiting, etc), but the meaning
-    # is not documented (See #70540).
-    # "queued" seems to be the info we want.
-    for queued_workflow in github_repo.get_workflow_runs(status="queued"):
-        if queued_workflow.name not in WORKFLOWS_TO_TRACK:
-            continue
-        for queued_workflow_job in queued_workflow.jobs():
-            job_name = queued_workflow_job.name
-            # Workflows marked as queued can potentially only have some jobs
-            # queued, so make sure to also count jobs currently in progress.
-            if queued_workflow_job.status == "queued":
-                if job_name not in queued_job_counts:
-                    queued_job_counts[job_name] = 1
-                else:
-                    queued_job_counts[job_name] += 1
-            elif queued_workflow_job.status == "in_progress":
-                if job_name not in running_job_counts:
-                    running_job_counts[job_name] = 1
-                else:
-                    running_job_counts[job_name] += 1
-
-    for running_workflow in github_repo.get_workflow_runs(status="in_progress"):
-        if running_workflow.name not in WORKFLOWS_TO_TRACK:
-            continue
-        for running_workflow_job in running_workflow.jobs():
-            job_name = running_workflow_job.name
-            if running_workflow_job.status != "in_progress":
-                continue
-
-            if job_name not in running_job_counts:
-                running_job_counts[job_name] = 1
-            else:
-                running_job_counts[job_name] += 1
-
-    workflow_metrics = []
-    for queued_job in queued_job_counts:
-        workflow_metrics.append(
-            GaugeMetric(
-                f"workflow_queue_size_{queued_job}",
-                queued_job_counts[queued_job],
-                time.time_ns(),
-            )
-        )
-    for running_job in running_job_counts:
-        workflow_metrics.append(
-            GaugeMetric(
-                f"running_workflow_count_{running_job}",
-                running_job_counts[running_job],
-                time.time_ns(),
-            )
-        )
-    # Always send a hearbeat metric so we can monitor is this container is still able to log to Grafana.
-    workflow_metrics.append(
-        GaugeMetric("metrics_container_heartbeat", 1, time.time_ns())
-    )
-    return workflow_metrics
-
-
-def get_per_workflow_metrics(
-    github_repo: github.Repository, workflows_to_track: dict[str, int]
-):
+def github_get_metrics(
+    github_repo: github.Repository, last_workflows_seen_as_completed: set[int]
+) -> tuple[list[JobMetrics], int]:
     """Gets the metrics for specified Github workflows.
 
     This function takes in a list of workflows to track, and optionally the
     workflow ID of the last tracked invocation. It grabs the relevant data
     from Github, returning it to the caller.
+    If the last_seen_workflow parameter is None, this returns no metrics, but
+    returns the id of the most recent workflow.
 
     Args:
       github_repo: A github repo object to use to query the relevant information.
-      workflows_to_track: A dictionary mapping workflow names to the last
-        invocation ID where metrics have been collected, or None to collect the
-        last five results.
+      last_seen_workflow: the last workflow this function processed.
 
     Returns:
-      Returns a list of JobMetrics objects, containing the relevant metrics about
-      the workflow.
+      Returns a tuple with 2 elements:
+        - a list of JobMetrics objects, one per processed job.
+        - the ID of the most recent processed workflow run.
     """
     workflow_metrics = []
-
-    workflows_to_include = set(workflows_to_track.keys())
-
-    for workflow_run in iter(github_repo.get_workflow_runs()):
-        if len(workflows_to_include) == 0:
+    queued_count = collections.Counter()
+    running_count = collections.Counter()
+
+    # The list of workflows this iteration will process.
+    # MaxSize = GITHUB_WORKFLOWS_MAX_PROCESS_COUNT
+    workflow_seen_as_completed = set()
+
+    # Since we process a fixed count of workflows, we want to know when
+    # the depth is too small and if we miss workflows.
+    # E.g.: is there was more than N workflows int last 2 hours.
+    # To monitor this, we'll log the age of the oldest workflow processed,
+    # and setup alterting in Grafana to help us adjust this depth.
+    oldest_seen_workflow_age_mn = None
+
+    # Do not apply any filters to this query.
+    # See https://github.com/orgs/community/discussions/86766
+    # Applying filters like `status=completed` will break pagination, and
+    # return a non-sorted and incomplete list of workflows.
+    i = 0
+    for task in iter(github_repo.get_workflow_runs()):
+        # Max depth reached, stopping.
+        if i >= GITHUB_WORKFLOWS_MAX_PROCESS_COUNT:
+            break
+        i += 1
+
+        workflow_age_mn = (
+            datetime.datetime.now(datetime.timezone.utc) - task.created_at
+        ).total_seconds() / 60
+        oldest_seen_workflow_age_mn = workflow_age_mn
+        # If we reach a workflow older than X, stop.
+        if workflow_age_mn > GITHUB_WORKFLOW_MAX_CREATED_AGE_HOURS * 60:
             break
 
-        if workflow_run.status != "completed":
+        # This workflow is not interesting to us.
+        if task.name not in GITHUB_WORKFLOW_TO_TRACK:
             continue
 
-        # This workflow was already sampled for this run, or is not tracked at
-        # all. Ignoring.
-        if workflow_run.name not in workflows_to_include:
-            continue
+        if task.status == "completed":
+            workflow_seen_as_completed.add(task.id)
 
-        # There were no new workflow invocations since the previous scrape.
-        # The API returns a sorted list with the most recent invocations first,
-        # so we can stop looking for this particular workflow. Continue to grab
-        # information on the other workflows of interest, if present.
-        if workflows_to_track[workflow_run.name] == workflow_run.id:
-            workflows_to_include.remove(workflow_run.name)
+        # This workflow has already been seen completed in the previous run.
+        if task.id in last_workflows_seen_as_completed:
             continue
 
-        workflow_jobs = workflow_run.jobs()
-        if workflow_jobs.totalCount == 0:
-            continue
+        name_prefix = GITHUB_WORKFLOW_TO_TRACK[task.name]
+        for job in task.jobs():
+            # This job is not interesting to us.
+            if job.name not in GITHUB_JOB_TO_TRACK[name_prefix]:
+                continue
 
-        if (
-            workflows_to_track[workflow_run.name] is None
-            or workflows_to_track[workflow_run.name] == workflow_run.id
-        ):
-            workflows_to_include.remove(workflow_run.name)
-        if (
-            workflows_to_track[workflow_run.name] is not None
-            and len(workflows_to_include) == 0
-        ):
-            break
+            name_suffix = GITHUB_JOB_TO_TRACK[name_prefix][job.name]
+            metric_name = name_prefix + "_" + name_suffix
 
-        for workflow_job in workflow_jobs:
-            created_at = workflow_job.created_at
-            started_at = workflow_job.started_at
-            completed_at = workflow_job.completed_at
+            if task.status != "completed":
+                if job.status == "queued":
+                    queued_count[metric_name] += 1
+                elif job.status == "in_progress":
+                    running_count[metric_name] += 1
+                continue
 
-            job_result = int(workflow_job.conclusion == "success")
+            job_result = int(job.conclusion == "success")
             if job_result:
                 # We still might want to mark the job as a failure if one of the steps
                 # failed. This is required due to use setting continue-on-error in
@@ -178,38 +160,72 @@ def get_per_workflow_metrics(
                 # TODO(boomanaiden154): Remove this once the premerge pipeline is no
                 # longer in a testing state and we can directly assert the workflow
                 # result.
-                for step in workflow_job.steps:
+                for step in job.steps:
                     if step.conclusion != "success" and step.conclusion != "skipped":
                         job_result = 0
                         break
 
+            created_at = job.created_at
+            started_at = job.started_at
+            completed_at = job.completed_at
             queue_time = started_at - created_at
             run_time = completed_at - started_at
-
             if run_time.seconds == 0:
                 continue
 
+            # Grafana will refuse to ingest metrics older than ~2 hours, so we
+            # should avoid sending historical data.
+            metric_age_mn = (
+                datetime.datetime.now(datetime.timezone.utc) - completed_at
+            ).total_seconds() / 60
+            if metric_age_mn > GRAFANA_METRIC_MAX_AGE_MN:
+                logging.info(
+                    f"Job {job.id} from workflow {task.id} dropped due"
+                    + f" to staleness: {metric_age_mn}mn old."
+                )
+                continue
+
+            logging.info(f"Adding a job metric for job {job.id} in workflow {task.id}")
             # The timestamp associated with the event is expected by Grafana to be
             # in nanoseconds.
-            created_at_ns = int(created_at.timestamp()) * 10**9
-
-            logging.info(
-                f"Adding a job metric for job {workflow_job.id} in workflow {workflow_run.id}"
-            )
-
+            completed_at_ns = int(completed_at.timestamp()) * 10**9
             workflow_metrics.append(
                 JobMetrics(
-                    workflow_run.name + "-" + workflow_job.name,
+                    metric_name,
                     queue_time.seconds,
                     run_time.seconds,
                     job_result,
-                    created_at_ns,
-                    workflow_run.id,
-                    workflow_run.name,
+                    completed_at_ns,
+                    task.id,
+                    task.name,
                 )
             )
 
-    return workflow_metrics
+    for name, value in queued_count.items():
+        workflow_metrics.append(
+            GaugeMetric(f"workflow_queue_size_{name}", value, time.time_ns())
+        )
+    for name, value in running_count.items():
+        workflow_metrics.append(
+            GaugeMetric(f"running_workflow_count_{name}", value, time.time_ns())
+        )
+
+    # Always send a hearbeat metric so we can monitor is this container is still able to log to Grafana.
+    workflow_metrics.append(
+        GaugeMetric("metrics_container_heartbeat", 1, time.time_ns())
+    )
+
+    # Log the oldest workflow we saw, allowing us to monitor if the processing
+    # depth is correctly set-up.
+    if oldest_seen_workflow_age_mn is not None:
+        workflow_metrics.append(
+            GaugeMetric(
+                "github_oldest_processed_workflow_mn",
+                oldest_seen_workflow_age_mn,
+                time.time_ns(),
+            )
+        )
+    return workflow_metrics, workflow_seen_as_completed
 
 
 def upload_metrics(workflow_metrics, metrics_userid, api_key):
@@ -238,7 +254,7 @@ def upload_metrics(workflow_metrics, metrics_userid, api_key):
         elif isinstance(workflow_metric, JobMetrics):
             name = workflow_metric.job_name.lower().replace(" ", "_")
             metrics_batch.append(
-                f"{name} queue_time={workflow_metric.queue_time},run_time={workflow_metric.run_time},status={workflow_metric.status} {workflow_metric.created_at_ns}"
+                f"{name} queue_time={workflow_metric.queue_time},run_time={workflow_metric.run_time},status={workflow_metric.status} {workflow_metric.completed_at_ns}"
             )
         else:
             raise ValueError(
@@ -259,32 +275,26 @@ def upload_metrics(workflow_metrics, metrics_userid, api_key):
 
 def main():
     # Authenticate with Github
-    auth = Auth.Token(os.environ["GITHUB_TOKEN"])
-
+    github_auth = Auth.Token(os.environ["GITHUB_TOKEN"])
     grafana_api_key = os.environ["GRAFANA_API_KEY"]
     grafana_metrics_userid = os.environ["GRAFANA_METRICS_USERID"]
 
-    workflows_to_track = {}
-    for workflow_to_track in WORKFLOWS_TO_TRACK:
-        workflows_to_track[workflow_to_track] = None
+    # The last workflow this script processed.
+    # Because the Github queries are broken, we'll simply log a 'processed'
+    # bit for the last COUNT_TO_PROCESS workflows.
+    gh_last_workflows_seen_as_completed = set()
 
     # Enter the main loop. Every five minutes we wake up and dump metrics for
     # the relevant jobs.
     while True:
-        github_object = Github(auth=auth)
+        github_object = Github(auth=github_auth)
         github_repo = github_object.get_repo("llvm/llvm-project")
 
-        current_metrics = get_per_workflow_metrics(github_repo, workflows_to_track)
-        current_metrics += get_sampled_workflow_metrics(github_repo)
-
-        upload_metrics(current_metrics, grafana_metrics_userid, grafana_api_key)
-        logging.info(f"Uploaded {len(current_metrics)} metrics")
-
-        for workflow_metric in reversed(current_metrics):
-            if isinstance(workflow_metric, JobMetrics):
-                workflows_to_track[
-                    workflow_metric.workflow_name
-                ] = workflow_metric.workflow_id
+        metrics, gh_last_workflows_seen_as_completed = github_get_metrics(
+            github_repo, gh_last_workflows_seen_as_completed
+        )
+        upload_metrics(metrics, grafana_metrics_userid, grafana_api_key)
+        logging.info(f"Uploaded {len(metrics)} metrics")
 
         time.sleep(SCRAPE_INTERVAL_SECONDS)
 
diff --git a/.github/workflows/issue-write.yml b/.github/workflows/issue-write.yml
index f5c60f51ca213..ad17d9a33ddb7 100644
--- a/.github/workflows/issue-write.yml
+++ b/.github/workflows/issue-write.yml
@@ -39,7 +39,7 @@ jobs:
 
       - name: 'Comment on PR'
         if: steps.download-artifact.outputs.artifact-id != ''
-        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        uses: actions/github-script@ffc2c79a5b2490bd33e0a41c1de74b877714d736 # v3.2.0
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
diff --git a/.github/workflows/libclang-python-tests.yml b/.github/workflows/libclang-python-tests.yml
index d8f58c5b8d1ce..cbfb3e5cbabfe 100644
--- a/.github/workflows/libclang-python-tests.yml
+++ b/.github/workflows/libclang-python-tests.yml
@@ -30,7 +30,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.8", "3.13"]
     uses: ./.github/workflows/llvm-project-tests.yml
     with:
       build_target: check-clang-python
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index 5d4394435890a..8c011425abfcf 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -55,7 +55,7 @@ jobs:
             cc: 'gcc-14'
             cxx: 'g++-14'
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: ${{ matrix.config }}.${{ matrix.cxx }}
         run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
         env:
@@ -105,7 +105,7 @@ jobs:
             cc: 'clang-18'
             cxx: 'clang++-18'
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: ${{ matrix.config }}
         run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
         env:
@@ -169,7 +169,7 @@ jobs:
     runs-on: ${{ matrix.machine }}
     container: ghcr.io/llvm/libcxx-linux-builder:b319dfef21f6c7b0bc6a356d6b9f41a3b3b98ae9
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: ${{ matrix.config }}
         run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
         env:
@@ -207,11 +207,11 @@ jobs:
           os: macos-13
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v4
-      - uses: maxim-lobanov/setup-xcode@v1
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: maxim-lobanov/setup-xcode@60606e260d2fc5762a71e64e74b2174e8ea3c8bd # v1.6.0
         with:
           xcode-version: 'latest'
-      - uses: seanmiddleditch/gha-setup-ninja@master
+      - uses: seanmiddleditch/gha-setup-ninja@3b1f8f94a2f8254bd26914c4ab9474d4f0015f67 # v6
       - name: Build and test
         run: |
           python3 -m venv .venv
@@ -247,7 +247,7 @@ jobs:
         - { config: mingw-dll-i686, mingw: true }
         - { config: mingw-incomplete-sysroot, mingw: true }
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Install dependencies
         run: |
           choco install -y ninja
@@ -275,7 +275,7 @@ jobs:
           echo "c:\Program Files\Git\usr\bin" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf8 -Append
       - name: Set up the MSVC dev environment
         if: ${{ matrix.mingw != true }}
-        uses: ilammy/msvc-dev-cmd@v1
+        uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0
       - name: Build and test
         run: |
           bash libcxx/utils/ci/run-buildbot ${{ matrix.config }}
diff --git a/.github/workflows/libcxx-build-containers.yml b/.github/workflows/libcxx-build-containers.yml
index b499af943c8e7..564a79341edb1 100644
--- a/.github/workflows/libcxx-build-containers.yml
+++ b/.github/workflows/libcxx-build-containers.yml
@@ -32,7 +32,7 @@ jobs:
       packages: write
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     - name: Build the Linux builder image
       working-directory: libcxx/utils/ci
@@ -47,7 +47,7 @@ jobs:
     #     TAG: ${{ github.sha }}
 
     - name: Log in to GitHub Container Registry
-      uses: docker/login-action@v3
+      uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
       with:
         registry: ghcr.io
         username: ${{ github.actor }}
diff --git a/.github/workflows/libcxx-check-generated-files.yml b/.github/workflows/libcxx-check-generated-files.yml
index 1f8103a114b9f..0226edd7aa17a 100644
--- a/.github/workflows/libcxx-check-generated-files.yml
+++ b/.github/workflows/libcxx-check-generated-files.yml
@@ -12,10 +12,10 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - name: Fetch LLVM sources
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install dependencies
-        uses: aminya/setup-cpp@v1
+        uses: aminya/setup-cpp@17c11551771948abc5752bbf3183482567c7caf0 # v1.1.1
         with:
           clangformat: 17.0.1
           ninja: true
diff --git a/.github/workflows/release-asset-audit.py b/.github/workflows/release-asset-audit.py
index 0ff1ce81b28dd..23b901a476dc0 100644
--- a/.github/workflows/release-asset-audit.py
+++ b/.github/workflows/release-asset-audit.py
@@ -24,7 +24,7 @@ def _get_uploaders(release_version):
         return set(
             [
                 "DimitryAndric",
-                "stefanp-ibm",
+                "stefanp-synopsys",
                 "lei137",
                 "omjavaid",
                 "nicolerabjohn",
diff --git a/clang-tools-extra/clang-tidy/bugprone/OptionalValueConversionCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/OptionalValueConversionCheck.cpp
index 55ca4809f058a..cda9288c0531a 100644
--- a/clang-tools-extra/clang-tidy/bugprone/OptionalValueConversionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/OptionalValueConversionCheck.cpp
@@ -12,6 +12,7 @@
 #include "../utils/OptionsUtils.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
 #include <array>
 
 using namespace clang::ast_matchers;
@@ -27,27 +28,11 @@ AST_MATCHER_P(QualType, hasCleanType, Matcher<QualType>, InnerMatcher) {
       Finder, Builder);
 }
 
-constexpr std::array<StringRef, 2> NameList{
+constexpr std::array<StringRef, 2> MakeSmartPtrList{
     "::std::make_unique",
     "::std::make_shared",
 };
-
-Matcher<Expr> constructFrom(Matcher<QualType> TypeMatcher,
-                            Matcher<Expr> ArgumentMatcher) {
-  return expr(
-      anyOf(
-          // construct optional
-          cxxConstructExpr(argumentCountIs(1U), hasType(TypeMatcher),
-                           hasArgument(0U, ArgumentMatcher)),
-          // known template methods in std
-          callExpr(argumentCountIs(1),
-                   callee(functionDecl(
-                       matchers::matchesAnyListedName(NameList),
-                       hasTemplateArgument(0, refersToType(TypeMatcher)))),
-                   hasArgument(0, ArgumentMatcher))),
-      unless(anyOf(hasAncestor(typeLoc()),
-                   hasAncestor(expr(matchers::hasUnevaluatedContext())))));
-}
+constexpr StringRef MakeOptional = "::std::make_optional";
 
 } // namespace
 
@@ -74,7 +59,7 @@ void OptionalValueConversionCheck::registerMatchers(MatchFinder *Finder) {
   auto EqualsBoundOptionalType =
       qualType(hasCleanType(equalsBoundNode("optional-type")));
 
-  auto OptionalDereferenceMatcher = callExpr(
+  auto OptionalDerefMatcherImpl = callExpr(
       anyOf(
           cxxOperatorCallExpr(hasOverloadedOperatorName("*"),
                               hasUnaryOperand(hasType(EqualsBoundOptionalType)))
@@ -88,11 +73,41 @@ void OptionalValueConversionCheck::registerMatchers(MatchFinder *Finder) {
 
   auto StdMoveCallMatcher =
       callExpr(argumentCountIs(1), callee(functionDecl(hasName("::std::move"))),
-               hasArgument(0, ignoringImpCasts(OptionalDereferenceMatcher)));
+               hasArgument(0, ignoringImpCasts(OptionalDerefMatcherImpl)));
+  auto OptionalDerefMatcher =
+      ignoringImpCasts(anyOf(OptionalDerefMatcherImpl, StdMoveCallMatcher));
+
   Finder->addMatcher(
-      expr(constructFrom(BindOptionalType,
-                         ignoringImpCasts(anyOf(OptionalDereferenceMatcher,
-                                                StdMoveCallMatcher))))
+      expr(anyOf(
+               // construct optional
+               cxxConstructExpr(argumentCountIs(1), hasType(BindOptionalType),
+                                hasArgument(0, OptionalDerefMatcher)),
+               // known template methods in std
+               callExpr(
+                   argumentCountIs(1),
+                   anyOf(
+                       // match std::make_unique std::make_shared
+                       callee(functionDecl(
+                           matchers::matchesAnyListedName(MakeSmartPtrList),
+                           hasTemplateArgument(
+                               0, refersToType(BindOptionalType)))),
+                       // match first std::make_optional by limit argument count
+                       // (1) and template count (1).
+                       // 1. template< class T > constexpr
+                       //    std::optional<decay_t<T>> make_optional(T&& value);
+                       // 2. template< class T, class... Args > constexpr
+                       //    std::optional<T> make_optional(Args&&... args);
+                       callee(functionDecl(templateArgumentCountIs(1),
+                                           hasName(MakeOptional),
+                                           returns(BindOptionalType)))),
+                   hasArgument(0, OptionalDerefMatcher)),
+               callExpr(
+
+                   argumentCountIs(1),
+
+                   hasArgument(0, OptionalDerefMatcher))),
+           unless(anyOf(hasAncestor(typeLoc()),
+                        hasAncestor(expr(matchers::hasUnevaluatedContext())))))
           .bind("expr"),
       this);
 }
diff --git a/clang-tools-extra/clang-tidy/misc/ConstCorrectnessCheck.cpp b/clang-tools-extra/clang-tidy/misc/ConstCorrectnessCheck.cpp
index 6e412e576e5f9..dbe59233df699 100644
--- a/clang-tools-extra/clang-tidy/misc/ConstCorrectnessCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/ConstCorrectnessCheck.cpp
@@ -136,16 +136,14 @@ void ConstCorrectnessCheck::check(const MatchFinder::MatchResult &Result) {
     return;
 
   VariableCategory VC = VariableCategory::Value;
-  if (Variable->getType()->isReferenceType())
+  const QualType VT = Variable->getType();
+  if (VT->isReferenceType())
     VC = VariableCategory::Reference;
-  if (Variable->getType()->isPointerType())
+  else if (VT->isPointerType())
     VC = VariableCategory::Pointer;
-  if (Variable->getType()->isArrayType()) {
-    if (const auto *ArrayT = dyn_cast<ArrayType>(Variable->getType())) {
-      if (ArrayT->getElementType()->isPointerType())
-        VC = VariableCategory::Pointer;
-    }
-  }
+  else if (const auto *ArrayT = dyn_cast<ArrayType>(VT))
+    if (ArrayT->getElementType()->isPointerType())
+      VC = VariableCategory::Pointer;
 
   // Each variable can only be in one category: Value, Pointer, Reference.
   // Analysis can be controlled for every category.
diff --git a/clang-tools-extra/clang-tidy/misc/UseInternalLinkageCheck.cpp b/clang-tools-extra/clang-tidy/misc/UseInternalLinkageCheck.cpp
index 4778182944abd..a1a20c0782230 100644
--- a/clang-tools-extra/clang-tidy/misc/UseInternalLinkageCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/UseInternalLinkageCheck.cpp
@@ -52,7 +52,7 @@ AST_MATCHER(FunctionDecl, hasBody) { return Node.hasBody(); }
 static bool isInMainFile(SourceLocation L, SourceManager &SM,
                          const FileExtensionsSet &HeaderFileExtensions) {
   for (;;) {
-    if (utils::isSpellingLocInHeaderFile(L, SM, HeaderFileExtensions))
+    if (utils::isExpansionLocInHeaderFile(L, SM, HeaderFileExtensions))
       return false;
     if (SM.isInMainFile(L))
       return true;
diff --git a/clang-tools-extra/clang-tidy/modernize/UseDefaultMemberInitCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseDefaultMemberInitCheck.cpp
index 6c06b0af342f6..5b0b9b59d4e3b 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseDefaultMemberInitCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseDefaultMemberInitCheck.cpp
@@ -194,6 +194,8 @@ void UseDefaultMemberInitCheck::storeOptions(
 }
 
 void UseDefaultMemberInitCheck::registerMatchers(MatchFinder *Finder) {
+  auto ConstExpRef = varDecl(anyOf(isConstexpr(), isStaticStorageClass()));
+
   auto InitBase =
       anyOf(stringLiteral(), characterLiteral(), integerLiteral(),
             unaryOperator(hasAnyOperatorName("+", "-"),
@@ -202,7 +204,7 @@ void UseDefaultMemberInitCheck::registerMatchers(MatchFinder *Finder) {
             unaryOperator(hasAnyOperatorName("+", "-"),
                           hasUnaryOperand(floatLiteral())),
             cxxBoolLiteral(), cxxNullPtrLiteralExpr(), implicitValueInitExpr(),
-            declRefExpr(to(enumConstantDecl())));
+            declRefExpr(to(anyOf(enumConstantDecl(), ConstExpRef))));
 
   auto Init =
       anyOf(initListExpr(anyOf(allOf(initCountIs(1), hasInit(0, InitBase)),
diff --git a/clang-tools-extra/clang-tidy/modernize/UseStdNumbersCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseStdNumbersCheck.cpp
index 1548fc454cfb3..38ef7712aa4ef 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseStdNumbersCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseStdNumbersCheck.cpp
@@ -91,8 +91,11 @@ struct MatchBuilder {
 
   auto matchMathCall(const StringRef FunctionName,
                      const Matcher<clang::Expr> ArgumentMatcher) const {
+    auto HasAnyPrecisionName = hasAnyName(
+        FunctionName, (FunctionName + "l").str(),
+        (FunctionName + "f").str()); // Support long double(l) and float(f).
     return expr(ignoreParenAndFloatingCasting(
-        callExpr(callee(functionDecl(hasName(FunctionName),
+        callExpr(callee(functionDecl(HasAnyPrecisionName,
                                      hasParameter(0, hasType(isArithmetic())))),
                  hasArgument(0, ArgumentMatcher))));
   }
diff --git a/clang-tools-extra/clang-tidy/readability/AmbiguousSmartptrResetCallCheck.cpp b/clang-tools-extra/clang-tidy/readability/AmbiguousSmartptrResetCallCheck.cpp
new file mode 100644
index 0000000000000..5f36c3976fc69
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/readability/AmbiguousSmartptrResetCallCheck.cpp
@@ -0,0 +1,126 @@
+//===--- AmbiguousSmartptrResetCallCheck.cpp - clang-tidy -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AmbiguousSmartptrResetCallCheck.h"
+#include "../utils/OptionsUtils.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Lex/Lexer.h"
+
+using namespace clang::ast_matchers;
+
+namespace clang::tidy::readability {
+
+namespace {
+
+AST_MATCHER(CXXMethodDecl, hasOnlyDefaultParameters) {
+  for (const auto *Param : Node.parameters()) {
+    if (!Param->hasDefaultArg())
+      return false;
+  }
+
+  return true;
+}
+
+const auto DefaultSmartPointers = "::std::shared_ptr;::std::unique_ptr;"
+                                  "::boost::shared_ptr";
+} // namespace
+
+AmbiguousSmartptrResetCallCheck::AmbiguousSmartptrResetCallCheck(
+    StringRef Name, ClangTidyContext *Context)
+    : ClangTidyCheck(Name, Context),
+      SmartPointers(utils::options::parseStringList(
+          Options.get("SmartPointers", DefaultSmartPointers))) {}
+
+void AmbiguousSmartptrResetCallCheck::storeOptions(
+    ClangTidyOptions::OptionMap &Opts) {
+  Options.store(Opts, "SmartPointers",
+                utils::options::serializeStringList(SmartPointers));
+}
+
+void AmbiguousSmartptrResetCallCheck::registerMatchers(MatchFinder *Finder) {
+  const auto IsSmartptr = hasAnyName(SmartPointers);
+
+  const auto ResetMethod =
+      cxxMethodDecl(hasName("reset"), hasOnlyDefaultParameters());
+
+  const auto TypeWithReset =
+      anyOf(cxxRecordDecl(
+                anyOf(hasMethod(ResetMethod),
+                      isDerivedFrom(cxxRecordDecl(hasMethod(ResetMethod))))),
+            classTemplateSpecializationDecl(
+                hasSpecializedTemplate(classTemplateDecl(has(ResetMethod)))));
+
+  const auto SmartptrWithReset = expr(hasType(hasUnqualifiedDesugaredType(
+      recordType(hasDeclaration(classTemplateSpecializationDecl(
+          IsSmartptr,
+          hasTemplateArgument(
+              0, templateArgument(refersToType(hasUnqualifiedDesugaredType(
+                     recordType(hasDeclaration(TypeWithReset))))))))))));
+
+  Finder->addMatcher(
+      cxxMemberCallExpr(
+          callee(ResetMethod),
+          unless(hasAnyArgument(expr(unless(cxxDefaultArgExpr())))),
+          anyOf(on(cxxOperatorCallExpr(hasOverloadedOperatorName("->"),
+                                       hasArgument(0, SmartptrWithReset))
+                       .bind("ArrowOp")),
+                on(SmartptrWithReset)))
+          .bind("MemberCall"),
+      this);
+}
+
+void AmbiguousSmartptrResetCallCheck::check(
+    const MatchFinder::MatchResult &Result) {
+  const auto *MemberCall =
+      Result.Nodes.getNodeAs<CXXMemberCallExpr>("MemberCall");
+  assert(MemberCall);
+
+  if (const auto *Arrow =
+          Result.Nodes.getNodeAs<CXXOperatorCallExpr>("ArrowOp")) {
+    const CharSourceRange SmartptrSourceRange =
+        Lexer::getAsCharRange(Arrow->getArg(0)->getSourceRange(),
+                              *Result.SourceManager, getLangOpts());
+
+    diag(MemberCall->getBeginLoc(),
+         "ambiguous call to 'reset()' on a pointee of a smart pointer, prefer "
+         "more explicit approach");
+
+    diag(MemberCall->getBeginLoc(),
+         "consider dereferencing smart pointer to call 'reset' method "
+         "of the pointee here",
+         DiagnosticIDs::Note)
+        << FixItHint::CreateInsertion(SmartptrSourceRange.getBegin(), "(*")
+        << FixItHint::CreateInsertion(SmartptrSourceRange.getEnd(), ")")
+        << FixItHint::CreateReplacement(
+               CharSourceRange::getCharRange(
+                   Arrow->getOperatorLoc(),
+                   Arrow->getOperatorLoc().getLocWithOffset(2)),
+               ".");
+  } else {
+    const auto *Member = cast<MemberExpr>(MemberCall->getCallee());
+    assert(Member);
+
+    diag(MemberCall->getBeginLoc(),
+         "ambiguous call to 'reset()' on a smart pointer with pointee that "
+         "also has a 'reset()' method, prefer more explicit approach");
+
+    diag(MemberCall->getBeginLoc(),
+         "consider assigning the pointer to 'nullptr' here",
+         DiagnosticIDs::Note)
+        << FixItHint::CreateReplacement(
+               SourceRange(Member->getOperatorLoc(), Member->getOperatorLoc()),
+               " =")
+        << FixItHint::CreateReplacement(
+               SourceRange(Member->getMemberLoc(), MemberCall->getEndLoc()),
+               " nullptr");
+  }
+}
+
+} // namespace clang::tidy::readability
diff --git a/clang-tools-extra/clang-tidy/readability/AmbiguousSmartptrResetCallCheck.h b/clang-tools-extra/clang-tidy/readability/AmbiguousSmartptrResetCallCheck.h
new file mode 100644
index 0000000000000..05932e59e7928
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/readability/AmbiguousSmartptrResetCallCheck.h
@@ -0,0 +1,37 @@
+//===--- AmbiguousSmartptrResetCallCheck.h - clang-tidy ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_AMBIGUOUSSMARTPTRRESETCALLCHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_AMBIGUOUSSMARTPTRRESETCALLCHECK_H
+
+#include "../ClangTidyCheck.h"
+
+namespace clang::tidy::readability {
+
+/// Finds potentially erroneous calls to 'reset' method on smart pointers when
+/// the pointee type also has a 'reset' method
+///
+/// For the user-facing documentation see:
+/// http://clang.llvm.org/extra/clang-tidy/checks/readability/ambiguous-smartptr-reset-call.html
+class AmbiguousSmartptrResetCallCheck : public ClangTidyCheck {
+public:
+  AmbiguousSmartptrResetCallCheck(StringRef Name, ClangTidyContext *Context);
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+  void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
+
+private:
+  const std::vector<StringRef> SmartPointers;
+};
+
+} // namespace clang::tidy::readability
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_AMBIGUOUSSMARTPTRRESETCALLCHECK_H
diff --git a/clang-tools-extra/clang-tidy/readability/CMakeLists.txt b/clang-tools-extra/clang-tidy/readability/CMakeLists.txt
index 8f303c51e1b0d..4be1a8f831339 100644
--- a/clang-tools-extra/clang-tidy/readability/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/readability/CMakeLists.txt
@@ -4,6 +4,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 add_clang_library(clangTidyReadabilityModule STATIC
+  AmbiguousSmartptrResetCallCheck.cpp
   AvoidConstParamsInDecls.cpp
   AvoidNestedConditionalOperatorCheck.cpp
   AvoidReturnWithVoidValueCheck.cpp
diff --git a/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp b/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp
index d61c0ba39658e..4c0812f0e6793 100644
--- a/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp
@@ -9,6 +9,7 @@
 #include "../ClangTidy.h"
 #include "../ClangTidyModule.h"
 #include "../ClangTidyModuleRegistry.h"
+#include "AmbiguousSmartptrResetCallCheck.h"
 #include "AvoidConstParamsInDecls.h"
 #include "AvoidNestedConditionalOperatorCheck.h"
 #include "AvoidReturnWithVoidValueCheck.h"
@@ -68,6 +69,8 @@ namespace readability {
 class ReadabilityModule : public ClangTidyModule {
 public:
   void addCheckFactories(ClangTidyCheckFactories &CheckFactories) override {
+    CheckFactories.registerCheck<AmbiguousSmartptrResetCallCheck>(
+        "readability-ambiguous-smartptr-reset-call");
     CheckFactories.registerCheck<AvoidConstParamsInDecls>(
         "readability-avoid-const-params-in-decls");
     CheckFactories.registerCheck<AvoidNestedConditionalOperatorCheck>(
diff --git a/clang-tools-extra/clang-tidy/utils/UseRangesCheck.cpp b/clang-tools-extra/clang-tidy/utils/UseRangesCheck.cpp
index aba4d17ccd035..f7a19cd72d500 100644
--- a/clang-tools-extra/clang-tidy/utils/UseRangesCheck.cpp
+++ b/clang-tools-extra/clang-tidy/utils/UseRangesCheck.cpp
@@ -215,6 +215,19 @@ void UseRangesCheck::check(const MatchFinder::MatchResult &Result) {
     const auto *Call = Result.Nodes.getNodeAs<CallExpr>(Buffer);
     if (!Call)
       continue;
+
+    // FIXME: This check specifically handles `CXXNullPtrLiteralExpr`, but
+    // a more general solution might be needed.
+    if (Function->getName() == "find") {
+      const unsigned ValueArgIndex = 2;
+      if (Call->getNumArgs() <= ValueArgIndex)
+        continue;
+      const Expr *ValueExpr =
+          Call->getArg(ValueArgIndex)->IgnoreParenImpCasts();
+      if (isa<CXXNullPtrLiteralExpr>(ValueExpr))
+        return;
+    }
+
     auto Diag = createDiag(*Call);
     if (auto ReplaceName = Replacer->getReplaceName(*Function))
       Diag << FixItHint::CreateReplacement(Call->getCallee()->getSourceRange(),
diff --git a/clang-tools-extra/clangd/Config.h b/clang-tools-extra/clangd/Config.h
index 586d031d58481..3f8a3c9b060f6 100644
--- a/clang-tools-extra/clangd/Config.h
+++ b/clang-tools-extra/clangd/Config.h
@@ -59,6 +59,7 @@ struct Config {
     std::optional<std::string> FixedCDBPath;
   };
 
+  enum class BuiltinHeaderPolicy { Clangd, QueryDriver };
   /// Controls how the compile command for the current file is determined.
   struct {
     /// Edits to apply to the compile command, in sequence.
@@ -66,6 +67,10 @@ struct Config {
         Edits;
     /// Where to search for compilation databases for this file's flags.
     CDBSearchSpec CDBSearch = {CDBSearchSpec::Ancestors, std::nullopt};
+
+    /// Whether to use clangd's own builtin headers, or ones from the system
+    /// include extractor, if available.
+    BuiltinHeaderPolicy BuiltinHeaders = BuiltinHeaderPolicy::Clangd;
   } CompileFlags;
 
   enum class BackgroundPolicy { Build, Skip };
diff --git a/clang-tools-extra/clangd/ConfigCompile.cpp b/clang-tools-extra/clangd/ConfigCompile.cpp
index aa2561e081047..31530c206acd7 100644
--- a/clang-tools-extra/clangd/ConfigCompile.cpp
+++ b/clang-tools-extra/clangd/ConfigCompile.cpp
@@ -290,6 +290,18 @@ struct FragmentCompiler {
       });
     }
 
+    if (F.BuiltinHeaders) {
+      if (auto Val =
+              compileEnum<Config::BuiltinHeaderPolicy>("BuiltinHeaders",
+                                                       *F.BuiltinHeaders)
+                  .map("Clangd", Config::BuiltinHeaderPolicy::Clangd)
+                  .map("QueryDriver", Config::BuiltinHeaderPolicy::QueryDriver)
+                  .value())
+        Out.Apply.push_back([Val](const Params &, Config &C) {
+          C.CompileFlags.BuiltinHeaders = *Val;
+        });
+    }
+
     if (F.CompilationDatabase) {
       std::optional<Config::CDBSearchSpec> Spec;
       if (**F.CompilationDatabase == "Ancestors") {
diff --git a/clang-tools-extra/clangd/ConfigFragment.h b/clang-tools-extra/clangd/ConfigFragment.h
index 9535b20253b13..6f95474b9c008 100644
--- a/clang-tools-extra/clangd/ConfigFragment.h
+++ b/clang-tools-extra/clangd/ConfigFragment.h
@@ -170,6 +170,14 @@ struct Fragment {
     /// - Ancestors: search all parent directories (the default)
     /// - std::nullopt: do not use a compilation database, just default flags.
     std::optional<Located<std::string>> CompilationDatabase;
+
+    /// Controls whether Clangd should use its own built-in system headers (like
+    /// stddef.h), or use the system headers from the query driver. Use the
+    /// option value 'Clangd' (default) to indicate Clangd's headers, and use
+    /// 'QueryDriver' to indicate QueryDriver's headers. `Clangd` is the
+    /// fallback if no query driver is supplied or if the query driver regex
+    /// string fails to match the compiler used in the CDB.
+    std::optional<Located<std::string>> BuiltinHeaders;
   };
   CompileFlagsBlock CompileFlags;
 
diff --git a/clang-tools-extra/clangd/ConfigYAML.cpp b/clang-tools-extra/clangd/ConfigYAML.cpp
index 95cc5c1f9f1cf..a46d45df0d319 100644
--- a/clang-tools-extra/clangd/ConfigYAML.cpp
+++ b/clang-tools-extra/clangd/ConfigYAML.cpp
@@ -104,6 +104,10 @@ class Parser {
       if (auto Values = scalarValues(N))
         F.Remove = std::move(*Values);
     });
+    Dict.handle("BuiltinHeaders", [&](Node &N) {
+      if (auto BuiltinHeaders = scalarValue(N, "BuiltinHeaders"))
+        F.BuiltinHeaders = *BuiltinHeaders;
+    });
     Dict.handle("CompilationDatabase", [&](Node &N) {
       F.CompilationDatabase = scalarValue(N, "CompilationDatabase");
     });
diff --git a/clang-tools-extra/clangd/IncludeFixer.cpp b/clang-tools-extra/clangd/IncludeFixer.cpp
index fadd1105691fc..8b74c761d23ca 100644
--- a/clang-tools-extra/clangd/IncludeFixer.cpp
+++ b/clang-tools-extra/clangd/IncludeFixer.cpp
@@ -84,7 +84,6 @@ std::vector<Fix> IncludeFixer::fix(DiagnosticsEngine::Level DiagLevel,
   case diag::err_array_incomplete_or_sizeless_type:
   case diag::err_array_size_incomplete_type:
   case diag::err_asm_incomplete_type:
-  case diag::err_assoc_type_incomplete:
   case diag::err_bad_cast_incomplete:
   case diag::err_call_function_incomplete_return:
   case diag::err_call_incomplete_argument:
diff --git a/clang-tools-extra/clangd/Preamble.cpp b/clang-tools-extra/clangd/Preamble.cpp
index b247e608eece3..b634981bb46bd 100644
--- a/clang-tools-extra/clangd/Preamble.cpp
+++ b/clang-tools-extra/clangd/Preamble.cpp
@@ -673,7 +673,6 @@ buildPreamble(PathRef FileName, CompilerInvocation CI,
   // Reset references to ref-counted-ptrs before executing the callbacks, to
   // prevent resetting them concurrently.
   PreambleDiagsEngine.reset();
-  CI.DiagnosticOpts.reset();
 
   // When building the AST for the main file, we do want the function
   // bodies.
diff --git a/clang-tools-extra/clangd/SystemIncludeExtractor.cpp b/clang-tools-extra/clangd/SystemIncludeExtractor.cpp
index c1c2e9fab9664..9399b910025b6 100644
--- a/clang-tools-extra/clangd/SystemIncludeExtractor.cpp
+++ b/clang-tools-extra/clangd/SystemIncludeExtractor.cpp
@@ -30,6 +30,7 @@
 // in the paths that are explicitly included by the user.
 
 #include "CompileCommands.h"
+#include "Config.h"
 #include "GlobalCompilationDatabase.h"
 #include "support/Logger.h"
 #include "support/Threading.h"
@@ -401,22 +402,30 @@ extractSystemIncludesAndTarget(const DriverArgs &InputArgs,
   if (!Info)
     return std::nullopt;
 
-  // The built-in headers are tightly coupled to parser builtins.
-  // (These are clang's "resource dir", GCC's GCC_INCLUDE_DIR.)
-  // We should keep using clangd's versions, so exclude the queried builtins.
-  // They're not specially marked in the -v output, but we can get the path
-  // with `$DRIVER -print-file-name=include`.
-  if (auto BuiltinHeaders =
-          run({Driver, "-print-file-name=include"}, /*OutputIsStderr=*/false)) {
-    auto Path = llvm::StringRef(*BuiltinHeaders).trim();
-    if (!Path.empty() && llvm::sys::path::is_absolute(Path)) {
-      auto Size = Info->SystemIncludes.size();
-      llvm::erase(Info->SystemIncludes, Path);
-      vlog("System includes extractor: builtin headers {0} {1}", Path,
-           (Info->SystemIncludes.size() != Size)
-               ? "excluded"
-               : "not found in driver's response");
+  switch (Config::current().CompileFlags.BuiltinHeaders) {
+  case Config::BuiltinHeaderPolicy::Clangd: {
+    // The built-in headers are tightly coupled to parser builtins.
+    // (These are clang's "resource dir", GCC's GCC_INCLUDE_DIR.)
+    // We should keep using clangd's versions, so exclude the queried
+    // builtins. They're not specially marked in the -v output, but we can
+    // get the path with `$DRIVER -print-file-name=include`.
+    if (auto BuiltinHeaders = run({Driver, "-print-file-name=include"},
+                                  /*OutputIsStderr=*/false)) {
+      auto Path = llvm::StringRef(*BuiltinHeaders).trim();
+      if (!Path.empty() && llvm::sys::path::is_absolute(Path)) {
+        auto Size = Info->SystemIncludes.size();
+        llvm::erase(Info->SystemIncludes, Path);
+        vlog("System includes extractor: builtin headers {0} {1}", Path,
+             (Info->SystemIncludes.size() != Size)
+                 ? "excluded"
+                 : "not found in driver's response");
+      }
     }
+    break;
+  }
+  case Config::BuiltinHeaderPolicy::QueryDriver:
+    vlog("System includes extractor: Using builtin headers from query driver.");
+    break;
   }
 
   log("System includes extractor: successfully executed {0}\n\tgot includes: "
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index ce1418a2a7d58..110f949741c3f 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -58,6 +58,9 @@ Semantic Highlighting
 Compile flags
 ^^^^^^^^^^^^^
 
+- Added `BuiltinHeaders` config key which controls whether clangd's built-in
+  headers are used or ones extracted from the driver.
+
 Hover
 ^^^^^
 
@@ -97,12 +100,22 @@ New checks
   Finds unintended character output from ``unsigned char`` and ``signed char`` to an
   ``ostream``.
 
+- New :doc:`readability-ambiguous-smartptr-reset-call
+  <clang-tidy/checks/readability/ambiguous-smartptr-reset-call>` check.
+
+  Finds potentially erroneous calls to ``reset`` method on smart pointers when
+  the pointee type also has a ``reset`` method.
+
 New check aliases
 ^^^^^^^^^^^^^^^^^
 
 Changes in existing checks
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+- Improved :doc:`bugprone-optional-value-conversion
+  <clang-tidy/checks/bugprone/optional-value-conversion>` check to detect
+  conversion in argument of ``std::make_optional``.
+
 - Improved :doc:`bugprone-string-constructor
   <clang-tidy/checks/bugprone/string-constructor>` check to find suspicious
   calls of ``std::string`` constructor with char pointer, start position and
@@ -112,7 +125,7 @@ Changes in existing checks
   <clang-tidy/checks/bugprone/unchecked-optional-access>` fixing false
   positives from smart pointer accessors repeated in checking ``has_value``
   and accessing ``value``. The option `IgnoreSmartPointerDereference` should
-  no longer be needed and will be removed. Also fixing false positive from 
+  no longer be needed and will be removed. Also fixing false positive from
   const reference accessors to objects containing optional member.
 
 - Improved :doc:`bugprone-unsafe-functions
@@ -128,10 +141,26 @@ Changes in existing checks
 - Improved :doc:`misc-redundant-expression
   <clang-tidy/checks/misc/redundant-expression>` check by providing additional
   examples and fixing some macro related false positives.
+  
+- Improved :doc:`modernize-use-ranges
+  <clang-tidy/checks/modernize/use-ranges>` check by updating suppress 
+  warnings logic for ``nullptr`` in ``std::find``.
+
+- Improved :doc:`modernize-use-std-numbers
+  <clang-tidy/checks/modernize/use-std-numbers>` check to support math
+  functions of different precisions.
+
+- Improved :doc:`misc-use-internal-linkage
+  <clang-tidy/checks/misc/use-internal-linkage>` check by fix false positives
+  for function or variable in header file which contains macro expansion.
+
+- Improved :doc:`modernize-use-default-member-init
+  <clang-tidy/checks/modernize/use-default-member-init>` check by matching
+  ``constexpr`` and ``static`` values on member initialization.
 
 - Improved :doc:`performance/unnecessary-value-param
   <clang-tidy/checks/performance/unnecessary-value-param>` check performance by
-  tolerating fix-it breaking compilation when functions is used as pointers 
+  tolerating fix-it breaking compilation when functions is used as pointers
   to avoid matching usage of functions within the current compilation unit.
 
 - Improved :doc:`performance-move-const-arg
diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst
index 5f03ef72cc603..c73bc8bff3539 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/list.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst
@@ -353,6 +353,7 @@ Clang-Tidy Checks
    :doc:`portability-simd-intrinsics <portability/simd-intrinsics>`,
    :doc:`portability-std-allocator-const <portability/std-allocator-const>`,
    :doc:`portability-template-virtual-member-function <portability/template-virtual-member-function>`,
+   :doc:`readability-ambiguous-smartptr-reset-call <readability/ambiguous-smartptr-reset-call>`, "Yes"
    :doc:`readability-avoid-const-params-in-decls <readability/avoid-const-params-in-decls>`, "Yes"
    :doc:`readability-avoid-nested-conditional-operator <readability/avoid-nested-conditional-operator>`,
    :doc:`readability-avoid-return-with-void-value <readability/avoid-return-with-void-value>`, "Yes"
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability/ambiguous-smartptr-reset-call.rst b/clang-tools-extra/docs/clang-tidy/checks/readability/ambiguous-smartptr-reset-call.rst
new file mode 100644
index 0000000000000..cf73839a46cfb
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability/ambiguous-smartptr-reset-call.rst
@@ -0,0 +1,62 @@
+.. title:: clang-tidy - readability-ambiguous-smartptr-reset-call
+
+readability-ambiguous-smartptr-reset-call
+=========================================
+
+Finds potentially erroneous calls to ``reset`` method on smart pointers when
+the pointee type also has a ``reset`` method. Having a ``reset`` method in
+both classes makes it easy to accidentally make the pointer null when
+intending to reset the underlying object.
+
+.. code-block:: c++
+
+  struct Resettable {
+    void reset() { /* Own reset logic */ }
+  };
+
+  auto ptr = std::make_unique<Resettable>();
+
+  ptr->reset();  // Calls underlying reset method
+  ptr.reset();   // Makes the pointer null
+
+Both calls are valid C++ code, but the second one might not be what the
+developer intended, as it destroys the pointed-to object rather than resetting
+its state. It's easy to make such a typo because the difference between
+``.`` and ``->`` is really small.
+
+The recommended approach is to make the intent explicit by using either member
+access or direct assignment:
+
+.. code-block:: c++
+
+  std::unique_ptr<Resettable> ptr = std::make_unique<Resettable>();
+
+  (*ptr).reset();  // Clearly calls underlying reset method
+  ptr = nullptr;   // Clearly makes the pointer null
+
+The default smart pointers and classes that are considered are
+``std::unique_ptr``, ``std::shared_ptr``, ``boost::shared_ptr``. To specify
+other smart pointers or other classes use the :option:`SmartPointers` option.
+
+
+.. note::
+    
+  The check may emit invalid fix-its and misleading warning messages when
+  specifying custom smart pointers or other classes in the
+  :option:`SmartPointers` option. For example, ``boost::scoped_ptr`` does not
+  have an ``operator=`` which makes fix-its invalid.
+
+.. note::
+    
+  Automatic fix-its are enabled only if :program:`clang-tidy` is invoked with
+  the `--fix-notes` option.
+
+
+Options
+-------
+
+.. option:: SmartPointers
+
+    Semicolon-separated list of fully qualified class names of custom smart
+    pointers. Default value is `::std::unique_ptr;::std::shared_ptr;
+    ::boost::shared_ptr`.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/optional-value-conversion-construct-from-std.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/optional-value-conversion-construct-from-std.cpp
index 768ab1ce014ce..305fd6890710d 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/optional-value-conversion-construct-from-std.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/optional-value-conversion-construct-from-std.cpp
@@ -27,9 +27,19 @@ class unique_ptr {};
 template <typename type>
 class shared_ptr {};
 
+template <typename T>
+class initializer_list {};
+
 template <class T, class... Args> unique_ptr<T> make_unique(Args &&...args);
 template <class T, class... Args> shared_ptr<T> make_shared(Args &&...args);
 
+template <class T>
+constexpr std::optional<__decay(T)> make_optional(T &&value);
+template <class T, class... Args>
+constexpr std::optional<T> make_optional(Args &&...args);
+template <class T, class U, class... Args>
+constexpr std::optional<T> make_optional(std::initializer_list<U> il, Args &&...args);
+
 } // namespace std
 
 struct A {
@@ -45,9 +55,12 @@ void invalid() {
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: conversion from 'std::optional<int>' into 'int' and back into 'std::optional<int>', remove potentially error-prone optional dereference [bugprone-optional-value-conversion]
   std::make_shared<std::optional<int>>(opt.value());
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: conversion from 'std::optional<int>' into 'int' and back into 'std::optional<int>', remove potentially error-prone optional dereference [bugprone-optional-value-conversion]
+  std::make_optional(opt.value());
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: conversion from 'std::optional<int>' into 'int' and back into 'std::optional<int>', remove potentially error-prone optional dereference [bugprone-optional-value-conversion]
 }
 
 void valid() {
   std::make_unique<A>(opt.value());
   std::make_shared<A>(opt.value());
+  std::make_optional<int>(opt.value());
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/use-internal-linkage-macro.hpp b/clang-tools-extra/test/clang-tidy/checkers/misc/use-internal-linkage-macro.hpp
new file mode 100644
index 0000000000000..209bd56028c3d
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/use-internal-linkage-macro.hpp
@@ -0,0 +1,5 @@
+// RUN: %check_clang_tidy %s misc-use-internal-linkage %t -- -- -I%S/Inputs/use-internal-linkage
+
+#define B A##C
+
+inline void B() {}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-default-member-init.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-default-member-init.cpp
index 81c980e0217e6..8b9bfaf0df34f 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-default-member-init.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-default-member-init.cpp
@@ -518,3 +518,22 @@ class ArrayBraceInitMultipleValues {
 };
 
 } // namespace PR63285
+
+namespace PR122480 {
+
+  static int STATIC_VAL = 23;
+  constexpr const char* CONSTEXPR_REF = "Const";
+
+  class StaticConstExprInit {
+
+    StaticConstExprInit() : a{CONSTEXPR_REF}, b{STATIC_VAL}{}
+    // CHECK-FIXES: StaticConstExprInit()  {}
+    const char* a;
+    // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: use default member initializer for 'a' [modernize-use-default-member-init]
+    // CHECK-FIXES: const char* a{CONSTEXPR_REF};
+    int b;
+    // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: use default member initializer for 'b' [modernize-use-default-member-init]
+    // CHECK-FIXES: int b{STATIC_VAL};
+  };
+
+} //namespace PR122480
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-ranges.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-ranges.cpp
index b022efebfdf4d..5aa026038b1cd 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-ranges.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-ranges.cpp
@@ -1,14 +1,25 @@
-// RUN: %check_clang_tidy -std=c++20 %s modernize-use-ranges %t -- -- -I %S/Inputs/use-ranges/
-// RUN: %check_clang_tidy -std=c++23 %s modernize-use-ranges %t -check-suffixes=,CPP23 -- -I %S/Inputs/use-ranges/
+// RUN: %check_clang_tidy -std=c++20 %s modernize-use-ranges %t -- -- -I %S/Inputs/
+// RUN: %check_clang_tidy -std=c++23 %s modernize-use-ranges %t -check-suffixes=,CPP23 -- -I %S/Inputs/
+// Example: ./check_clang_tidy.py -std=c++20 checkers/modernize/use-ranges.cpp modernize-use-ranges temp.txt -- -- -I ~/llvm-project/clang-tools-extra/test/clang-tidy/checkers/modernize/Inputs/
 
 // CHECK-FIXES: #include <algorithm>
 // CHECK-FIXES-CPP23: #include <numeric>
 // CHECK-FIXES: #include <ranges>
 
-#include "fake_std.h"
+#include "use-ranges/fake_std.h"
+#include "smart-ptr/unique_ptr.h"
 
 void Positives() {
   std::vector<int> I, J;
+  std::vector<std::unique_ptr<int>> K;
+
+  // Expect to have no check messages
+  std::find(K.begin(), K.end(), nullptr);
+
+  std::find(K.begin(), K.end(), std::unique_ptr<int>());
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use a ranges version of this algorithm
+  // CHECK-FIXES: std::ranges::find(K, std::unique_ptr<int>());
+
   std::find(I.begin(), I.end(), 0);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use a ranges version of this algorithm
   // CHECK-FIXES: std::ranges::find(I, 0);
@@ -76,6 +87,15 @@ void Positives() {
 
 void Reverse(){
   std::vector<int> I, J;
+  std::vector<std::unique_ptr<int>> K;
+  
+  // Expect to have no check messages
+  std::find(K.rbegin(), K.rend(), nullptr);
+
+  std::find(K.rbegin(), K.rend(), std::unique_ptr<int>());
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use a ranges version of this algorithm
+  // CHECK-FIXES: std::ranges::find(std::ranges::reverse_view(K), std::unique_ptr<int>());
+
   std::find(I.rbegin(), I.rend(), 0);
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use a ranges version of this algorithm
   // CHECK-FIXES: std::ranges::find(std::ranges::reverse_view(I), 0);
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-numbers.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-numbers.cpp
index 6c5762da5e2e8..11121ae6d8e93 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-numbers.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-numbers.cpp
@@ -9,12 +9,17 @@ namespace bar {
     template <typename T>
     auto sqrt(T val) { return sqrt(static_cast<double>(val)); }
 
+    float sqrtf(float Arg);
+    long double sqrtl(long double Arg);
+
     static constexpr double e = 2.718281828459045235360287471352662497757247093;
     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:33: warning: prefer 'std::numbers::e' to this literal, differs by '0.00e+00' [modernize-use-std-numbers]
     // CHECK-FIXES-ALL: static constexpr double e = std::numbers::e;
 }
 
+float expf(float Arg);
 double exp(double Arg);
+long double expl(long double Arg);
 double log(double Arg);
 
 double log2(double Arg);
@@ -110,6 +115,14 @@ void foo(){
     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:5: warning: prefer 'std::numbers::sqrt2_v<float>'  to this formula [modernize-use-std-numbers]
     // CHECK-FIXES-ALL: std::numbers::sqrt2_v<float>;
 
+    bar::sqrtf(2.0F);
+    // CHECK-MESSAGES-ALL: :[[@LINE-1]]:5: warning: prefer 'std::numbers::sqrt2_v<float>'  to this formula [modernize-use-std-numbers]
+    // CHECK-FIXES-ALL: std::numbers::sqrt2_v<float>;
+
+    bar::sqrtl(2.0);
+    // CHECK-MESSAGES-ALL: :[[@LINE-1]]:5: warning: prefer 'std::numbers::sqrt2_v<long double>'  to this formula [modernize-use-std-numbers]
+    // CHECK-FIXES-ALL: std::numbers::sqrt2_v<long double>;
+
     sink(MY_PI);
     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:10: warning: prefer 'std::numbers::pi' to this macro, differs by '5.36e-08' [modernize-use-std-numbers]
     // CHECK-FIXES-ALL: sink(std::numbers::pi);
@@ -155,6 +168,14 @@ void foo(){
     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:5: warning: prefer 'std::numbers::e' to this formula [modernize-use-std-numbers]
     // CHECK-FIXES-ALL: std::numbers::e;
 
+    expf(1);
+    // CHECK-MESSAGES-ALL: :[[@LINE-1]]:5: warning: prefer 'std::numbers::e_v<float>' to this formula [modernize-use-std-numbers]
+    // CHECK-FIXES-ALL: std::numbers::e_v<float>;
+
+    expl(1);
+    // CHECK-MESSAGES-ALL: :[[@LINE-1]]:5: warning: prefer 'std::numbers::e_v<long double>' to this formula [modernize-use-std-numbers]
+    // CHECK-FIXES-ALL: std::numbers::e_v<long double>;
+
     log2(exp(1));
     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:5: warning: prefer 'std::numbers::log2e' to this formula [modernize-use-std-numbers]
     // CHECK-MESSAGES-ALL: :[[@LINE-2]]:10: warning: prefer 'std::numbers::e' to this formula [modernize-use-std-numbers]
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/ambiguous-smartptr-reset-call-custom-pointers.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/ambiguous-smartptr-reset-call-custom-pointers.cpp
new file mode 100644
index 0000000000000..df3f16a9cf9ec
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/ambiguous-smartptr-reset-call-custom-pointers.cpp
@@ -0,0 +1,52 @@
+// RUN: %check_clang_tidy %s readability-ambiguous-smartptr-reset-call %t -- \
+// RUN: -config='{CheckOptions: \
+// RUN:  {readability-ambiguous-smartptr-reset-call.SmartPointers: "::std::unique_ptr;::other_ptr"}}' \
+// RUN: --fix-notes -- -I %S/../modernize/Inputs/smart-ptr
+
+#include "unique_ptr.h"
+#include "shared_ptr.h"
+
+template <typename T>
+struct other_ptr {
+  T& operator*() const;
+  T* operator->() const;
+  void reset();
+};
+
+struct Resettable {
+  void reset();
+  void doSomething();
+};
+
+void Positive() {
+  std::unique_ptr<Resettable> u;
+  u.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: u = nullptr;
+  u->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*u).reset();
+
+  other_ptr<Resettable> s;
+  s.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: s = nullptr;
+  s->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*s).reset();
+}
+
+void Negative() {
+  std::shared_ptr<Resettable> s_ptr;
+  s_ptr.reset();
+  s_ptr->reset();
+  s_ptr->doSomething();
+
+  std::unique_ptr<Resettable> u_ptr;
+  u_ptr.reset(nullptr);
+  u_ptr->doSomething();
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/ambiguous-smartptr-reset-call.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/ambiguous-smartptr-reset-call.cpp
new file mode 100644
index 0000000000000..e6e7eb9231ec2
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/ambiguous-smartptr-reset-call.cpp
@@ -0,0 +1,397 @@
+// RUN: %check_clang_tidy %s readability-ambiguous-smartptr-reset-call %t --fix-notes -- -I %S/../modernize/Inputs/smart-ptr
+
+#include "unique_ptr.h"
+#include "shared_ptr.h"
+
+template <typename T>
+struct non_default_reset_ptr {
+  T& operator*() const;
+  T* operator->() const;
+  void reset(T* p);
+};
+
+struct Resettable {
+  void reset();
+  void doSomething();
+};
+
+struct ResettableWithParam {
+  void reset(int a);
+  void doSomething();
+};
+
+struct ResettableWithDefaultParams {
+  void reset(int a = 0, double b = 0.0);
+  void doSomething();
+};
+
+struct NonResettable {
+  void doSomething();
+};
+
+void Positive() {
+  std::unique_ptr<Resettable> u;
+  u.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: u = nullptr;
+  u->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*u).reset();
+
+  std::shared_ptr<Resettable> s;
+  s.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: s = nullptr;
+  s->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*s).reset();
+
+  std::unique_ptr<std::unique_ptr<int>> uu_ptr;
+  uu_ptr.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: uu_ptr = nullptr;
+  uu_ptr->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*uu_ptr).reset();
+
+  std::unique_ptr<std::shared_ptr<int>> su_ptr;
+  su_ptr.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: su_ptr = nullptr;
+  su_ptr->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*su_ptr).reset();
+
+  std::unique_ptr<ResettableWithDefaultParams> rd;
+  rd.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: rd = nullptr;
+  rd->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*rd).reset();
+
+  std::unique_ptr<std::shared_ptr<std::unique_ptr<Resettable>>> nested_ptr;
+  nested_ptr.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: nested_ptr = nullptr;
+  nested_ptr->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*nested_ptr).reset();
+  (*nested_ptr).reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: (*nested_ptr) = nullptr;
+  (*nested_ptr)->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*(*nested_ptr)).reset();
+  (**nested_ptr).reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: (**nested_ptr) = nullptr;
+  (**nested_ptr)->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*(**nested_ptr)).reset();
+}
+
+void Negative() {
+  std::unique_ptr<Resettable> u_ptr;
+  u_ptr.reset(nullptr);
+  u_ptr->doSomething();
+
+  std::shared_ptr<Resettable> s_ptr;
+  s_ptr.reset(nullptr);
+  s_ptr->doSomething();
+  
+  Resettable* raw_ptr;
+  raw_ptr->reset();
+  raw_ptr->doSomething();
+
+  Resettable resettable;
+  resettable.reset();
+  resettable.doSomething();
+  
+  std::unique_ptr<ResettableWithParam> u_ptr_param;
+  u_ptr_param.reset();
+  u_ptr_param.reset(nullptr);
+  u_ptr_param->reset(0);
+
+  std::unique_ptr<NonResettable> u_ptr_no_reset;
+  u_ptr_no_reset.reset();
+
+  std::shared_ptr<ResettableWithParam> s_ptr_param;
+  s_ptr_param.reset();
+  s_ptr_param->reset(0);
+  s_ptr_param->doSomething();
+
+  std::shared_ptr<NonResettable> s_ptr_no_reset;
+  s_ptr_no_reset.reset();
+
+  std::unique_ptr<ResettableWithDefaultParams> u_ptr_default_params;
+  u_ptr_default_params.reset(nullptr);
+  u_ptr_default_params->reset(1);
+  u_ptr_default_params->reset(1, 2.0);
+
+  non_default_reset_ptr<Resettable> non_default_reset_ptr;
+  non_default_reset_ptr.reset(new Resettable);
+  non_default_reset_ptr->reset();
+}
+
+template <typename T>
+void TemplatePositiveTest() {
+  std::unique_ptr<T> u_ptr;
+
+  u_ptr.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: u_ptr = nullptr;
+  u_ptr->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*u_ptr).reset();
+
+  std::shared_ptr<T> s_ptr;
+  s_ptr.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: s_ptr = nullptr;
+  s_ptr->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*s_ptr).reset();
+}
+
+template <typename T>
+void TemplatNegativeTestTypeWithReset() {
+  std::unique_ptr<T> u_ptr;
+  u_ptr.reset();
+  u_ptr->reset(0);
+
+  std::shared_ptr<T> s_ptr;
+  s_ptr.reset();
+  s_ptr->reset(0);
+}
+
+template <typename T>
+void TemplatNegativeTestTypeWithoutReset() {
+  std::unique_ptr<T> u_ptr;
+  u_ptr.reset();
+
+  std::unique_ptr<T> s_ptr;
+  s_ptr.reset();
+}
+
+void instantiate() {
+  TemplatePositiveTest<Resettable>();
+  TemplatePositiveTest<std::unique_ptr<int>>();
+  TemplatePositiveTest<std::shared_ptr<int>>();
+  TemplatNegativeTestTypeWithReset<ResettableWithParam>();
+  TemplatNegativeTestTypeWithoutReset<NonResettable>();
+}
+
+struct S {
+  void foo() {
+    u_ptr.reset();
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: note: consider assigning the pointer to 'nullptr' here
+    // CHECK-FIXES: u_ptr = nullptr;
+    u_ptr->reset();
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+    // CHECK-FIXES: (*u_ptr).reset();
+    u_ptr.reset(nullptr);
+    u_ptr->doSomething();
+
+    s_ptr.reset();
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: note: consider assigning the pointer to 'nullptr' here
+    // CHECK-FIXES: s_ptr = nullptr;
+    s_ptr->reset();
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+    // CHECK-FIXES: (*s_ptr).reset();
+    s_ptr.reset(nullptr);
+
+    ptr.reset();
+  }
+
+  std::unique_ptr<Resettable> u_ptr;
+  std::unique_ptr<std::shared_ptr<int>> s_ptr;
+  std::unique_ptr<NonResettable> ptr;
+};
+
+
+typedef std::unique_ptr<Resettable> TypedefResettableUniquePtr;
+typedef std::shared_ptr<Resettable> TypedefResettableSharedPtr;
+
+void TypedefPositive() {
+  TypedefResettableUniquePtr u_ptr;
+  u_ptr.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: u_ptr = nullptr;
+  u_ptr->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*u_ptr).reset();
+
+  TypedefResettableSharedPtr s_ptr;
+  s_ptr.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: s_ptr = nullptr;
+
+  s_ptr->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*s_ptr).reset();
+}
+
+using UsingResettableUniquePtr = std::unique_ptr<Resettable>;
+using UsingResettableSharedPtr = std::shared_ptr<Resettable>;
+
+void UsingPositive() {
+  UsingResettableUniquePtr u_ptr;
+  u_ptr.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: u_ptr = nullptr;
+  u_ptr->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*u_ptr).reset();
+
+  UsingResettableSharedPtr s_ptr;
+  s_ptr.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: s_ptr = nullptr;
+
+  s_ptr->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*s_ptr).reset();
+}
+
+template<typename T>
+using UsingUniquePtr = std::unique_ptr<T>;
+template<typename T>
+using UsingSharedPtr = std::shared_ptr<T>;
+
+void UsingTemplatePositive() {
+  UsingUniquePtr<Resettable> u_ptr;
+  u_ptr.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: u_ptr = nullptr;
+  u_ptr->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*u_ptr).reset();
+
+  UsingSharedPtr<Resettable> s_ptr;
+  s_ptr.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: s_ptr = nullptr;
+
+  s_ptr->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*s_ptr).reset();
+}
+
+template<typename T>
+void UsingByTemplatePositive() {
+  UsingUniquePtr<T> u_ptr;
+  u_ptr.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: u_ptr = nullptr;
+  u_ptr->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*u_ptr).reset();
+
+  UsingSharedPtr<T> s_ptr;
+  s_ptr.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: s_ptr = nullptr;
+
+  s_ptr->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*s_ptr).reset();
+}
+
+void instantiate2() {
+  UsingByTemplatePositive<Resettable>();
+}
+
+void NestedUsingPositive() {
+  UsingUniquePtr<UsingSharedPtr<TypedefResettableUniquePtr>> nested_ptr;
+  nested_ptr.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: nested_ptr = nullptr;
+  nested_ptr->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*nested_ptr).reset();
+  (*nested_ptr).reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: (*nested_ptr) = nullptr;
+  (*nested_ptr)->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*(*nested_ptr)).reset();
+  (**nested_ptr).reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: (**nested_ptr) = nullptr;
+  (**nested_ptr)->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*(**nested_ptr)).reset();
+}
+
+// Check other default pointers and classes.
+namespace boost {
+
+template <typename T>
+struct shared_ptr {
+  T& operator*() const;
+  T* operator->() const;
+  void reset();
+  void reset(T*);
+};
+
+} // namespace boost
+
+void PositiveOtherClasses() {
+  boost::shared_ptr<Resettable> sh;
+  sh.reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a smart pointer with pointee that also has a 'reset()' method, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider assigning the pointer to 'nullptr' here
+  // CHECK-FIXES: sh = nullptr;
+  sh->reset();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: ambiguous call to 'reset()' on a pointee of a smart pointer, prefer more explicit approach
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: note: consider dereferencing smart pointer to call 'reset' method of the pointee here
+  // CHECK-FIXES: (*sh).reset();
+}
+
diff --git a/clang/AreaTeamMembers.txt b/clang/AreaTeamMembers.txt
new file mode 100644
index 0000000000000..964d11e79f694
--- /dev/null
+++ b/clang/AreaTeamMembers.txt
@@ -0,0 +1,17 @@
+This is a list of the current Clang Area Team members.
+
+Chair
+-----
+Aaron Ballman
+aaron@aaronballman.com (email), AaronBallman (Discourse), AaronBallman (GitHub), AaronBallman (Discord)
+
+Secretary
+---------
+Reid Kleckner
+rnk@google.com (email), rnk (Discourse), rnk (GitHub), rnk (Discord)
+
+Other Members
+-------------
+Eli Friedman
+efriedma@quicinc.com> (email), efriedma-quic (Discourse), efriedma-quic (GitHub)
+
diff --git a/clang/Maintainers.rst b/clang/Maintainers.rst
index 9eaa1ec19f0d3..ef41628c0e75d 100644
--- a/clang/Maintainers.rst
+++ b/clang/Maintainers.rst
@@ -4,7 +4,8 @@ Clang Maintainers
 
 This file is a list of the
 `maintainers <https://llvm.org/docs/DeveloperPolicy.html#maintainers>`_ for
-Clang.
+Clang. The list of current Clang Area Team members can be found
+`here <https://github.com/llvm/llvm-project/blob/main/clang/AreaTeamMembers.txt>`_.
 
 .. contents::
    :depth: 2
diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index 578ee02f09b9b..387251804fa76 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -786,6 +786,7 @@ of different sizes and signs is forbidden in binary and ternary builtins.
  T __builtin_elementwise_bitreverse(T x)        return the integer represented after reversing the bits of x           integer types
  T __builtin_elementwise_exp(T x)               returns the base-e exponential, e^x, of the specified value            floating point types
  T __builtin_elementwise_exp2(T x)              returns the base-2 exponential, 2^x, of the specified value            floating point types
+ T __builtin_elementwise_exp10(T x)             returns the base-10 exponential, 10^x, of the specified value          floating point types
 
  T __builtin_elementwise_sqrt(T x)              return the square root of a floating-point number                      floating point types
  T __builtin_elementwise_roundeven(T x)         round x to the nearest integer value in floating point format,         floating point types
@@ -1604,53 +1605,54 @@ More information could be found `here <https://clang.llvm.org/docs/Modules.html>
 Language Extensions Back-ported to Previous Standards
 =====================================================
 
-============================================ ================================ ============= =============
-Feature                                      Feature Test Macro               Introduced In Backported To
-============================================ ================================ ============= =============
-variadic templates                           __cpp_variadic_templates         C++11         C++03
-Alias templates                              __cpp_alias_templates            C++11         C++03
-Non-static data member initializers          __cpp_nsdmi                      C++11         C++03
-Range-based ``for`` loop                     __cpp_range_based_for            C++11         C++03
-RValue references                            __cpp_rvalue_references          C++11         C++03
-Attributes                                   __cpp_attributes                 C++11         C++03
-Lambdas                                      __cpp_lambdas                    C++11         C++03
-Generalized lambda captures                  __cpp_init_captures              C++14         C++03
-Generic lambda expressions                   __cpp_generic_lambdas            C++14         C++03
-variable templates                           __cpp_variable_templates         C++14         C++03
-Binary literals                              __cpp_binary_literals            C++14         C++03
-Relaxed constexpr                            __cpp_constexpr                  C++14         C++11
-Static assert with no message                __cpp_static_assert >= 201411L   C++17         C++11
-Pack expansion in generalized lambda-capture __cpp_init_captures              C++17         C++03
-``if constexpr``                             __cpp_if_constexpr               C++17         C++11
-fold expressions                             __cpp_fold_expressions           C++17         C++03
-Lambda capture of \*this by value            __cpp_capture_star_this          C++17         C++03
-Attributes on enums                          __cpp_enumerator_attributes      C++17         C++03
-Guaranteed copy elision                      __cpp_guaranteed_copy_elision    C++17         C++03
-Hexadecimal floating literals                __cpp_hex_float                  C++17         C++03
-``inline`` variables                         __cpp_inline_variables           C++17         C++03
-Attributes on namespaces                     __cpp_namespace_attributes       C++17         C++11
-Structured bindings                          __cpp_structured_bindings        C++17         C++03
-template template arguments                  __cpp_template_template_args     C++17         C++03
-Familiar template syntax for generic lambdas __cpp_generic_lambdas            C++20         C++03
-``static operator[]``                        __cpp_multidimensional_subscript C++20         C++03
-Designated initializers                      __cpp_designated_initializers    C++20         C++03
-Conditional ``explicit``                     __cpp_conditional_explicit       C++20         C++03
-``using enum``                               __cpp_using_enum                 C++20         C++03
-``if consteval``                             __cpp_if_consteval               C++23         C++20
-``static operator()``                        __cpp_static_call_operator       C++23         C++03
-Attributes on Lambda-Expressions                                              C++23         C++11
-Attributes on Structured Bindings            __cpp_structured_bindings        C++26         C++03
-Packs in Structured Bindings                 __cpp_structured_bindings        C++26         C++03
-Static assert with user-generated message    __cpp_static_assert >= 202306L   C++26         C++11
-Pack Indexing                                __cpp_pack_indexing              C++26         C++03
-``= delete ("should have a reason");``       __cpp_deleted_function           C++26         C++03
-Variadic Friends                             __cpp_variadic_friend            C++26         C++03
--------------------------------------------- -------------------------------- ------------- -------------
-Designated initializers (N494)                                                C99           C89
-Array & element qualification (N2607)                                         C23           C89
-Attributes (N2335)                                                            C23           C89
-``#embed`` (N3017)                                                            C23           C89, C++
-============================================ ================================ ============= =============
+============================================= ================================ ============= =============
+Feature                                       Feature Test Macro               Introduced In Backported To
+============================================= ================================ ============= =============
+variadic templates                            __cpp_variadic_templates         C++11         C++03
+Alias templates                               __cpp_alias_templates            C++11         C++03
+Non-static data member initializers           __cpp_nsdmi                      C++11         C++03
+Range-based ``for`` loop                      __cpp_range_based_for            C++11         C++03
+RValue references                             __cpp_rvalue_references          C++11         C++03
+Attributes                                    __cpp_attributes                 C++11         C++03
+Lambdas                                       __cpp_lambdas                    C++11         C++03
+Generalized lambda captures                   __cpp_init_captures              C++14         C++03
+Generic lambda expressions                    __cpp_generic_lambdas            C++14         C++03
+variable templates                            __cpp_variable_templates         C++14         C++03
+Binary literals                               __cpp_binary_literals            C++14         C++03
+Relaxed constexpr                             __cpp_constexpr                  C++14         C++11
+Static assert with no message                 __cpp_static_assert >= 201411L   C++17         C++11
+Pack expansion in generalized lambda-capture  __cpp_init_captures              C++17         C++03
+``if constexpr``                              __cpp_if_constexpr               C++17         C++11
+fold expressions                              __cpp_fold_expressions           C++17         C++03
+Lambda capture of \*this by value             __cpp_capture_star_this          C++17         C++03
+Attributes on enums                           __cpp_enumerator_attributes      C++17         C++03
+Guaranteed copy elision                       __cpp_guaranteed_copy_elision    C++17         C++03
+Hexadecimal floating literals                 __cpp_hex_float                  C++17         C++03
+``inline`` variables                          __cpp_inline_variables           C++17         C++03
+Attributes on namespaces                      __cpp_namespace_attributes       C++17         C++11
+Structured bindings                           __cpp_structured_bindings        C++17         C++03
+template template arguments                   __cpp_template_template_args     C++17         C++03
+Familiar template syntax for generic lambdas  __cpp_generic_lambdas            C++20         C++03
+``static operator[]``                         __cpp_multidimensional_subscript C++20         C++03
+Designated initializers                       __cpp_designated_initializers    C++20         C++03
+Conditional ``explicit``                      __cpp_conditional_explicit       C++20         C++03
+``using enum``                                __cpp_using_enum                 C++20         C++03
+``if consteval``                              __cpp_if_consteval               C++23         C++20
+``static operator()``                         __cpp_static_call_operator       C++23         C++03
+Attributes on Lambda-Expressions                                               C++23         C++11
+Attributes on Structured Bindings             __cpp_structured_bindings        C++26         C++03
+Packs in Structured Bindings                  __cpp_structured_bindings        C++26         C++03
+Structured binding declaration as a condition __cpp_structured_bindings        C++26         C++98
+Static assert with user-generated message     __cpp_static_assert >= 202306L   C++26         C++11
+Pack Indexing                                 __cpp_pack_indexing              C++26         C++03
+``= delete ("should have a reason");``        __cpp_deleted_function           C++26         C++03
+Variadic Friends                              __cpp_variadic_friend            C++26         C++03
+--------------------------------------------- -------------------------------- ------------- -------------
+Designated initializers (N494)                                                 C99           C89
+Array & element qualification (N2607)                                          C23           C89
+Attributes (N2335)                                                             C23           C89
+``#embed`` (N3017)                                                             C23           C89, C++
+============================================= ================================ ============= =============
 
 Builtin type aliases
 ====================
diff --git a/clang/docs/LibASTMatchersReference.html b/clang/docs/LibASTMatchersReference.html
index 48dfd9cac0033..9b30057b5257f 100644
--- a/clang/docs/LibASTMatchersReference.html
+++ b/clang/docs/LibASTMatchersReference.html
@@ -1842,12 +1842,6 @@ <h2 id="decl-matchers">Node Matchers</h2>
   if (x) {}
 </pre></td></tr>
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('dependentScopeDeclRefExpr0')"><a name="dependentScopeDeclRefExpr0Anchor">dependentScopeDeclRefExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DependentScopeDeclRefExpr.html">DependentScopeDeclRefExpr</a>&gt;...</td></tr>
-<tr><td colspan="4" class="doc" id="dependentScopeDeclRefExpr0"><pre>Matches expressions that refer to dependent scope declarations.
-
-Example matches T::v
-   template <class T> class X : T { void f() { T::v; } };
-</pre></td></tr>
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('declStmt0')"><a name="declStmt0Anchor">declStmt</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclStmt.html">DeclStmt</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="declStmt0"><pre>Matches declaration statements.
@@ -1874,6 +1868,14 @@ <h2 id="decl-matchers">Node Matchers</h2>
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('dependentScopeDeclRefExpr0')"><a name="dependentScopeDeclRefExpr0Anchor">dependentScopeDeclRefExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DependentScopeDeclRefExpr.html">DependentScopeDeclRefExpr</a>&gt;...</td></tr>
+<tr><td colspan="4" class="doc" id="dependentScopeDeclRefExpr0"><pre>Matches expressions that refer to dependent scope declarations.
+
+example matches T::v;
+ template &lt;class T&gt; class X : T { void f() { T::v; } };
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('designatedInitExpr0')"><a name="designatedInitExpr0Anchor">designatedInitExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DesignatedInitExpr.html">DesignatedInitExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="designatedInitExpr0"><pre>Matches C99 designated initializer expressions [C99 6.7.8].
 
@@ -1931,7 +1933,24 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('fixedPointLiteral0')"><a name="fixedPointLiteral0Anchor">fixedPointLiteral</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FixedPointLiteral.html">FixedPointLiteral</a>&gt;...</td></tr>
-<tr><td colspan="4" class="doc" id="fixedPointLiteral0"><pre>Matches fixed point literals
+<tr><td colspan="4" class="doc" id="fixedPointLiteral0"><pre>Matches fixed-point literals eg.
+0.5r, 0.5hr, 0.5lr, 0.5uhr, 0.5ur, 0.5ulr
+1.0k, 1.0hk, 1.0lk, 1.0uhk, 1.0uk, 1.0ulk
+Exponents 1.0e10k
+Hexadecimal numbers 0x0.2p2r
+
+Does not match implicit conversions such as first two lines:
+   short _Accum sa = 2;
+   _Accum a = 12.5;
+   _Accum b = 1.25hk;
+   _Fract c = 0.25hr;
+   _Fract v = 0.35uhr;
+   _Accum g = 1.45uhk;
+   _Accum decexp1 = 1.575e1k;
+
+The matcher matches
+but does not
+match and from the code block.
 </pre></td></tr>
 
 
@@ -2536,26 +2555,6 @@ <h2 id="decl-matchers">Node Matchers</h2>
   matches "decltype(i + j)"
 </pre></td></tr>
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('dependentNameType0')"><a name="dependentNameType0Anchor">dependentNameType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DependentNameType.html">DependentNameType</a>&gt;...</td></tr>
-<tr><td colspan="4" class="doc" id="dependentNameType0"><pre>Matches a dependent name type.
-
-Example matches T::type
-
-  template <typename T> struct declToImport {
-    typedef typename T::type dependent_name;
-  };
-</pre></td></tr>
-
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('dependentTemplateSpecializationType0')"><a name="dependentTemplateSpecializationType0Anchor">dependentTemplateSpecializationType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DependentTemplateSpecializationType.html">DependentTemplateSpecializationType</a>&gt;...</td></tr>
-<tr><td colspan="4" class="doc" id="dependentTemplateSpecializationType0"><pre>Matches a dependent template specialization type.
-
-Example matches A<T>::template B<T>
-
-  template<typename T> struct A;
-  template<typename T> struct declToImport {
-    typename A<T>::template B<T> a;
-  };
-</pre></td></tr>
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('deducedTemplateSpecializationType0')"><a name="deducedTemplateSpecializationType0Anchor">deducedTemplateSpecializationType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeducedTemplateSpecializationType.html">DeducedTemplateSpecializationType</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="deducedTemplateSpecializationType0"><pre>Matches C++17 deduced template specialization types, e.g. deduced class
@@ -2571,6 +2570,16 @@ <h2 id="decl-matchers">Node Matchers</h2>
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('dependentNameType0')"><a name="dependentNameType0Anchor">dependentNameType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DependentNameType.html">DependentNameType</a>&gt;...</td></tr>
+<tr><td colspan="4" class="doc" id="dependentNameType0"><pre>Matches a dependent name type
+
+Example matches T::type
+ template &lt;typename T&gt; struct declToImport {
+   typedef typename T::type dependent_name;
+ };
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('dependentSizedArrayType0')"><a name="dependentSizedArrayType0Anchor">dependentSizedArrayType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DependentSizedArrayType.html">DependentSizedArrayType</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="dependentSizedArrayType0"><pre>Matches C++ arrays whose size is a value-dependent expression.
 
@@ -2598,6 +2607,17 @@ <h2 id="decl-matchers">Node Matchers</h2>
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('dependentTemplateSpecializationType0')"><a name="dependentTemplateSpecializationType0Anchor">dependentTemplateSpecializationType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DependentTemplateSpecializationType.html">DependentTemplateSpecializationType</a>&gt;...</td></tr>
+<tr><td colspan="4" class="doc" id="dependentTemplateSpecializationType0"><pre>Matches a dependent template specialization type
+
+Example matches A&lt;T&gt;::template B&lt;T&gt;
+  template&lt;typename T&gt; struct A;
+  template&lt;typename T&gt; struct declToImport {
+    typename A&lt;T&gt;::template B&lt;T&gt; a;
+  };
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('elaboratedType0')"><a name="elaboratedType0Anchor">elaboratedType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ElaboratedType.html">ElaboratedType</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="elaboratedType0"><pre>Matches types specified with an elaborated type keyword or with a
 qualified name.
@@ -3449,34 +3469,6 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DependentScopeDeclRefExpr.html">DependentScopeDeclRefExpr</a>&gt;</td><td class="name" onclick="toggle('hasDependentName0')"><a name="hasDependentName0Anchor">hasDependentName</a></td><td>std::string N</td></tr>
-<tr><td colspan="4" class="doc" id="hasDependentName0"><pre>Matches the dependent name of a DependentScopeDeclRefExpr.
-
-Matches the dependent name of a DependentScopeDeclRefExpr
-
-Given:
-
-  template &lt;class T&lt; class X : T { void f() { T::v; } };
-
-dependentScopeDeclRefExpr(hasDependentName("v")) matches `T::v`
-</pre></td></tr>
-
-
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DependentNameType.html">DependentNameType</a>&gt;</td><td class="name" onclick="toggle('hasDependentName1')"><a name="hasDependentName1Anchor">hasDependentName</a></td><td>std::string N</td></tr>
-<tr><td colspan="4" class="doc" id="hasDependentName1"><pre>Matches the dependent name of a DependentNameType.
-
-Matches the dependent name of a DependentNameType
-
-Given:
-
-  template &lt;typename T&lt; struct declToImport {
-    typedef typename T::type dependent_name;
-  };
-
-dependentNameType(hasDependentName("type")) matches `T::type`
-</pre></td></tr>
-
-
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXDependentScopeMemberExpr.html">CXXDependentScopeMemberExpr</a>&gt;</td><td class="name" onclick="toggle('memberHasSameNameAsBoundNode0')"><a name="memberHasSameNameAsBoundNode0Anchor">memberHasSameNameAsBoundNode</a></td><td>std::string BindingID</td></tr>
 <tr><td colspan="4" class="doc" id="memberHasSameNameAsBoundNode0"><pre>Matches template-dependent, but known, member names against an already-bound
 node
@@ -4116,8 +4108,14 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 Given
   template&lt;typename T&gt; struct C {};
   C&lt;int&gt; c;
+  template&lt;typename T&gt; void f() {}
+  void func() { f&lt;int&gt;(); };
+
 classTemplateSpecializationDecl(templateArgumentCountIs(1))
   matches C&lt;int&gt;.
+
+functionDecl(templateArgumentCountIs(1))
+  matches f&lt;int&gt;();
 </pre></td></tr>
 
 
@@ -4373,6 +4371,38 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DependentNameType.html">DependentNameType</a>&gt;</td><td class="name" onclick="toggle('hasDependentName1')"><a name="hasDependentName1Anchor">hasDependentName</a></td><td>std::string N</td></tr>
+<tr><td colspan="4" class="doc" id="hasDependentName1"><pre>Matches the dependent name of a DependentScopeDeclRefExpr or
+DependentNameType
+
+Given:
+ template &lt;class T&gt; class X : T { void f() { T::v; } };
+dependentScopeDeclRefExpr(hasDependentName("v")) matches `T::v`
+
+Given:
+ template &lt;typename T&gt; struct declToImport {
+   typedef typename T::type dependent_name;
+ };
+dependentNameType(hasDependentName("type")) matches `T::type`
+</pre></td></tr>
+
+
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DependentScopeDeclRefExpr.html">DependentScopeDeclRefExpr</a>&gt;</td><td class="name" onclick="toggle('hasDependentName0')"><a name="hasDependentName0Anchor">hasDependentName</a></td><td>std::string N</td></tr>
+<tr><td colspan="4" class="doc" id="hasDependentName0"><pre>Matches the dependent name of a DependentScopeDeclRefExpr or
+DependentNameType
+
+Given:
+ template &lt;class T&gt; class X : T { void f() { T::v; } };
+dependentScopeDeclRefExpr(hasDependentName("v")) matches `T::v`
+
+Given:
+ template &lt;typename T&gt; struct declToImport {
+   typedef typename T::type dependent_name;
+ };
+dependentNameType(hasDependentName("type")) matches `T::type`
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DesignatedInitExpr.html">DesignatedInitExpr</a>&gt;</td><td class="name" onclick="toggle('designatorCountIs0')"><a name="designatorCountIs0Anchor">designatorCountIs</a></td><td>unsigned N</td></tr>
 <tr><td colspan="4" class="doc" id="designatorCountIs0"><pre>Matches designated initializer expressions that contain
 a specific number of designators.
@@ -4809,6 +4839,23 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('templateArgumentCountIs2')"><a name="templateArgumentCountIs2Anchor">templateArgumentCountIs</a></td><td>unsigned N</td></tr>
+<tr><td colspan="4" class="doc" id="templateArgumentCountIs2"><pre>Matches if the number of template arguments equals N.
+
+Given
+  template&lt;typename T&gt; struct C {};
+  C&lt;int&gt; c;
+  template&lt;typename T&gt; void f() {}
+  void func() { f&lt;int&gt;(); };
+
+classTemplateSpecializationDecl(templateArgumentCountIs(1))
+  matches C&lt;int&gt;.
+
+functionDecl(templateArgumentCountIs(1))
+  matches f&lt;int&gt;();
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionProtoType.html">FunctionProtoType</a>&gt;</td><td class="name" onclick="toggle('hasDynamicExceptionSpec1')"><a name="hasDynamicExceptionSpec1Anchor">hasDynamicExceptionSpec</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="hasDynamicExceptionSpec1"><pre>Matches functions that have a dynamic exception specification.
 
@@ -5759,14 +5806,20 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('templateArgumentCountIs1')"><a name="templateArgumentCountIs1Anchor">templateArgumentCountIs</a></td><td>unsigned N</td></tr>
-<tr><td colspan="4" class="doc" id="templateArgumentCountIs1"><pre>Matches if the number of template arguments equals N.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('templateArgumentCountIs3')"><a name="templateArgumentCountIs3Anchor">templateArgumentCountIs</a></td><td>unsigned N</td></tr>
+<tr><td colspan="4" class="doc" id="templateArgumentCountIs3"><pre>Matches if the number of template arguments equals N.
 
 Given
   template&lt;typename T&gt; struct C {};
   C&lt;int&gt; c;
+  template&lt;typename T&gt; void f() {}
+  void func() { f&lt;int&gt;(); };
+
 classTemplateSpecializationDecl(templateArgumentCountIs(1))
   matches C&lt;int&gt;.
+
+functionDecl(templateArgumentCountIs(1))
+  matches f&lt;int&gt;();
 </pre></td></tr>
 
 
@@ -6195,6 +6248,23 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;
 </pre></td></tr>
 
+
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarTemplateSpecializationDecl.html">VarTemplateSpecializationDecl</a>&gt;</td><td class="name" onclick="toggle('templateArgumentCountIs1')"><a name="templateArgumentCountIs1Anchor">templateArgumentCountIs</a></td><td>unsigned N</td></tr>
+<tr><td colspan="4" class="doc" id="templateArgumentCountIs1"><pre>Matches if the number of template arguments equals N.
+
+Given
+  template&lt;typename T&gt; struct C {};
+  C&lt;int&gt; c;
+  template&lt;typename T&gt; void f() {}
+  void func() { f&lt;int&gt;(); };
+
+classTemplateSpecializationDecl(templateArgumentCountIs(1))
+  matches C&lt;int&gt;.
+
+functionDecl(templateArgumentCountIs(1))
+  matches f&lt;int&gt;();
+</pre></td></tr>
+
 <!--END_NARROWING_MATCHERS -->
 </table>
 
@@ -6799,7 +6869,8 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   matches "int const *b"
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockPointerType.html">BlockPointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1MemberPointerType.html">MemberPointerType</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1PointerType.html">PointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ReferenceType.html">ReferenceType</a>&gt;
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1PointerType.html">PointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ReferenceType.html">ReferenceType</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCObjectPointerType.html">ObjCObjectPointerType</a>&gt;
 </pre></td></tr>
 
 
@@ -9300,7 +9371,8 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   matches "int const *b"
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockPointerType.html">BlockPointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1MemberPointerType.html">MemberPointerType</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1PointerType.html">PointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ReferenceType.html">ReferenceType</a>&gt;
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1PointerType.html">PointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ReferenceType.html">ReferenceType</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCObjectPointerType.html">ObjCObjectPointerType</a>&gt;
 </pre></td></tr>
 
 
@@ -9414,6 +9486,36 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCInterfaceDecl.html">ObjCInterfaceDecl</a>&gt;</td><td class="name" onclick="toggle('hasType9')"><a name="hasType9Anchor">hasType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="hasType9"><pre>Overloaded to match the declaration of the expression's or value
+declaration's type.
+
+In case of a value declaration (for example a variable declaration),
+this resolves one layer of indirection. For example, in the value
+declaration "X x;", cxxRecordDecl(hasName("X")) matches the declaration of
+X, while varDecl(hasType(cxxRecordDecl(hasName("X")))) matches the
+declaration of x.
+
+Example matches x (matcher = expr(hasType(cxxRecordDecl(hasName("X")))))
+            and z (matcher = varDecl(hasType(cxxRecordDecl(hasName("X")))))
+            and friend class X (matcher = friendDecl(hasType("X"))
+            and public virtual X (matcher = cxxBaseSpecifier(hasType(
+                                              cxxRecordDecl(hasName("X"))))
+ class X {};
+ void y(X &amp;x) { x; X z; }
+ class Y { friend class X; };
+ class Z : public virtual X {};
+
+Example matches class Derived
+(matcher = cxxRecordDecl(hasAnyBase(hasType(cxxRecordDecl(hasName("Base"))))))
+class Base {};
+class Derived : Base {};
+
+Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FriendDecl.html">FriendDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ValueDecl.html">ValueDecl</a>&gt;,
+Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCInterfaceDecl.html">ObjCInterfaceDecl</a>&gt;</td><td class="name" onclick="toggle('isDerivedFrom1')"><a name="isDerivedFrom1Anchor">isDerivedFrom</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1NamedDecl.html">NamedDecl</a>&gt; Base</td></tr>
 <tr><td colspan="4" class="doc" id="isDerivedFrom1"><pre>Matches C++ classes that are directly or indirectly derived from a class
 matching Base, or Objective-C classes that directly or indirectly
@@ -9592,6 +9694,23 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 </pre></td></tr>
 
 
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCObjectPointerType.html">ObjCObjectPointerType</a>&gt;</td><td class="name" onclick="toggle('pointee4')"><a name="pointee4Anchor">pointee</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td></tr>
+<tr><td colspan="4" class="doc" id="pointee4"><pre>Narrows PointerType (and similar) matchers to those where the
+pointee matches a given matcher.
+
+Given
+  int *a;
+  int const *b;
+  float const *f;
+pointerType(pointee(isConstQualified(), isInteger()))
+  matches "int const *b"
+
+Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockPointerType.html">BlockPointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1MemberPointerType.html">MemberPointerType</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1PointerType.html">PointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ReferenceType.html">ReferenceType</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCObjectPointerType.html">ObjCObjectPointerType</a>&gt;
+</pre></td></tr>
+
+
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc10')"><a name="hasTypeLoc10Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
 <tr><td colspan="4" class="doc" id="hasTypeLoc10"><pre>Matches if the type location of a node matches the inner matcher.
 
@@ -9689,7 +9808,8 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   matches "int const *b"
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockPointerType.html">BlockPointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1MemberPointerType.html">MemberPointerType</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1PointerType.html">PointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ReferenceType.html">ReferenceType</a>&gt;
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1PointerType.html">PointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ReferenceType.html">ReferenceType</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCObjectPointerType.html">ObjCObjectPointerType</a>&gt;
 </pre></td></tr>
 
 
@@ -9858,7 +9978,8 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   matches "int const *b"
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockPointerType.html">BlockPointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1MemberPointerType.html">MemberPointerType</a>&gt;,
-  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1PointerType.html">PointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ReferenceType.html">ReferenceType</a>&gt;
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1PointerType.html">PointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ReferenceType.html">ReferenceType</a>&gt;,
+  Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCObjectPointerType.html">ObjCObjectPointerType</a>&gt;
 </pre></td></tr>
 
 
diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index 88af120d06edb..dcee1db2a18e0 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -408,7 +408,7 @@ implementation.
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | Private reductions                                          | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| Self maps                                                   | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
+| Self maps                                                   | :part:`partial`           | :none:`unclaimed`         | parsing/sema done: https://github.com/llvm/llvm-project/pull/129888      |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | Release map type for declare mapper                         | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index e1a8dad87ad29..f6e4d6a34e5dc 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -62,6 +62,9 @@ AST Dumping Potentially Breaking Changes
 Clang Frontend Potentially Breaking Changes
 -------------------------------------------
 
+- The ``-Wglobal-constructors`` flag now applies to ``[[gnu::constructor]]`` and
+  ``[[gnu::destructor]]`` attributes.
+
 Clang Python Bindings Potentially Breaking Changes
 --------------------------------------------------
 
@@ -76,6 +79,8 @@ C++2c Feature Support
 
 - Implemented `P1061R10 Structured Bindings can introduce a Pack <https://wg21.link/P1061R10>`_.
 
+- Implemented `P0963R3 Structured binding declaration as a condition <https://wg21.link/P0963R3>`_.
+
 C++23 Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
 
@@ -108,6 +113,13 @@ C Language Changes
 
 C2y Feature Support
 ^^^^^^^^^^^^^^^^^^^
+- Implement `WG14 N3409 <https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3409.pdf>`_
+  which removes UB around use of ``void`` expressions. In practice, this means
+  that ``_Generic`` selection associations may now have ``void`` type, but it
+  also removes UB with code like ``(void)(void)1;``.
+- Implemented `WG14 N3411 <https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3411.pdf>`_
+  which allows a source file to not end with a newline character. This is still
+  reported as a conforming extension in earlier language modes.
 
 C23 Feature Support
 ^^^^^^^^^^^^^^^^^^^
@@ -116,6 +128,7 @@ Non-comprehensive list of changes in this release
 -------------------------------------------------
 
 - Support parsing the `cc` operand modifier and alias it to the `c` modifier (#GH127719).
+- Added `__builtin_elementwise_exp10`.
 
 New Compiler Flags
 ------------------
@@ -145,13 +158,14 @@ Adding [[clang::unsafe_buffer_usage]] attribute to a method definition now turns
 related warnings within the method body.
 
 - The ``no_sanitize`` attribute now accepts both ``gnu`` and ``clang`` names.
+- The ``ext_vector_type(n)`` attribute can now be used as a generic type attribute.
 - Clang now diagnoses use of declaration attributes on void parameters. (#GH108819)
 - Clang now allows ``__attribute__((model("small")))`` and
   ``__attribute__((model("large")))`` on non-TLS globals in x86-64 compilations.
   This forces the global to be considered small or large in regards to the
   x86-64 code model, regardless of the code model specified for the compilation.
-- Clang now emits a warning ``-Wreserved-init-priority`` instead of a hard error 
-  when ``__attribute__((init_priority(n)))`` is used with values of n in the 
+- Clang now emits a warning ``-Wreserved-init-priority`` instead of a hard error
+  when ``__attribute__((init_priority(n)))`` is used with values of n in the
   reserved range [0, 100]. The warning will be treated as an error by default.
 
 - There is a new ``format_matches`` attribute to complement the existing
@@ -223,12 +237,16 @@ Improvements to Clang's diagnostics
 - Diagnostics on chained comparisons (``a < b < c``) are now an error by default. This can be disabled with
   ``-Wno-error=parentheses``.
 - The ``-Wshift-bool`` warning has been added to warn about shifting a boolean. (#GH28334)
-
+- Fixed diagnostics adding a trailing ``::`` when printing some source code
+  constructs, like base classes.
 - The :doc:`ThreadSafetyAnalysis` now supports ``-Wthread-safety-pointer``,
   which enables warning on passing or returning pointers to guarded variables
   as function arguments or return value respectively. Note that
   :doc:`ThreadSafetyAnalysis` still does not perform alias analysis. The
   feature will be default-enabled with ``-Wthread-safety`` in a future release.
+- The ``-Wsign-compare`` warning now treats expressions with bitwise not(~) and minus(-) as signed integers 
+  except for the case where the operand is an unsigned integer
+  and throws warning if they are compared with unsigned integers (##18878).
 
 - Improve the diagnostics for chained comparisons to report actual expressions and operators (#GH129069).
 
@@ -245,6 +263,7 @@ Bug Fixes in This Version
 
 - Clang now outputs correct values when #embed data contains bytes with negative
   signed char values (#GH102798).
+- Fixed a crash when merging named enumerations in modules (#GH114240).
 - Fixed rejects-valid problem when #embed appears in std::initializer_list or
   when it can affect template argument deduction (#GH122306).
 - Fix crash on code completion of function calls involving partial order of function templates
@@ -282,20 +301,19 @@ Bug Fixes to C++ Support
   direct-list-initialized from an array is corrected to direct-initialization.
 - Clang no longer crashes when a coroutine is declared ``[[noreturn]]``. (#GH127327)
 - Clang now uses the parameter location for abbreviated function templates in ``extern "C"``. (#GH46386)
-
-Improvements to C++ diagnostics
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-- Clang now more consistently adds a note pointing to the relevant template
-  parameter. Some diagnostics are reworded to better take advantage of this.
-- Template Template Parameter diagnostics now stop referring to template
-  parameters as template arguments, in some circumstances, better hiding
-  from the users template template parameter partial ordering arcana.
-
+- Clang will emit an error instead of crash when use co_await or co_yield in
+  C++26 braced-init-list template parameter initialization. (#GH78426)
+- Fixes matching of nested template template parameters. (#GH130362)
+- Correctly diagnoses template template paramters which have a pack parameter
+  not in the last position.
+- Clang now correctly parses ``if constexpr`` expressions in immediate function context. (#GH123524)
+- Fixed an assertion failure affecting code that uses C++23 "deducing this". (#GH130272)
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 - Fixed type checking when a statement expression ends in an l-value of atomic type. (#GH106576)
+- Fixed uninitialized use check in a lambda within CXXOperatorCallExpr. (#GH129198)
+- Fixed a malformed printout of ``CXXParenListInitExpr`` in certain contexts.
 
 Miscellaneous Bug Fixes
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -380,6 +398,8 @@ AST Matchers
 ------------
 
 - Ensure ``isDerivedFrom`` matches the correct base in case more than one alias exists.
+- Extend ``templateArgumentCountIs`` to support function and variable template
+  specialization.
 
 clang-format
 ------------
@@ -443,6 +463,7 @@ Python Binding Changes
 OpenMP Support
 --------------
 - Added support 'no_openmp_constructs' assumption clause.
+- Added support for 'self_maps' in map and requirement clause.
 - Added support for 'omp stripe' directive.
 
 Improvements
diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst
index 8213334b61c22..b4a994444d5e8 100644
--- a/clang/docs/UsersManual.rst
+++ b/clang/docs/UsersManual.rst
@@ -4681,25 +4681,7 @@ Clang supports generation of SPIR-V conformant to `the OpenCL Environment
 Specification
 <https://www.khronos.org/registry/OpenCL/specs/3.0-unified/html/OpenCL_Env.html>`_.
 
-To generate SPIR-V binaries, Clang uses the external ``llvm-spirv`` tool from the
-`SPIRV-LLVM-Translator repo
-<https://github.com/KhronosGroup/SPIRV-LLVM-Translator>`_.
-
-Prior to the generation of SPIR-V binary with Clang, ``llvm-spirv``
-should be built or installed. Please refer to `the following instructions
-<https://github.com/KhronosGroup/SPIRV-LLVM-Translator#build-instructions>`_
-for more details. Clang will look for ``llvm-spirv-<LLVM-major-version>`` and
-``llvm-spirv`` executables, in this order, in the ``PATH`` environment variable.
-Clang uses ``llvm-spirv`` with `the widely adopted assembly syntax package
-<https://github.com/KhronosGroup/SPIRV-LLVM-Translator/#build-with-spirv-tools>`_.
-
-`The versioning
-<https://github.com/KhronosGroup/SPIRV-LLVM-Translator/releases>`_ of
-``llvm-spirv`` is aligned with Clang major releases. The same applies to the
-main development branch. It is therefore important to ensure the ``llvm-spirv``
-version is in alignment with the Clang version. For troubleshooting purposes
-``llvm-spirv`` can be `tested in isolation
-<https://github.com/KhronosGroup/SPIRV-LLVM-Translator#test-instructions>`_.
+To generate SPIR-V binaries, Clang uses the in-tree LLVM SPIR-V backend.
 
 Example usage for OpenCL kernel compilation:
 
@@ -4717,18 +4699,6 @@ Converting to SPIR-V produced with the optimization levels other than `-O0` is
 currently available as an experimental feature and it is not guaranteed to work
 in all cases.
 
-Clang also supports integrated generation of SPIR-V without use of ``llvm-spirv``
-tool as an experimental feature when ``-fintegrated-objemitter`` flag is passed in
-the command line.
-
-   .. code-block:: console
-
-     $ clang --target=spirv32 -fintegrated-objemitter -c test.cl
-
-Note that only very basic functionality is supported at this point and therefore
-it is not suitable for arbitrary use cases. This feature is only enabled when clang
-build is configured with ``-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=SPIRV`` option.
-
 Linking is done using ``spirv-link`` from `the SPIRV-Tools project
 <https://github.com/KhronosGroup/SPIRV-Tools#linker>`_. Similar to other external
 linkers, Clang will expect ``spirv-link`` to be installed separately and to be
diff --git a/clang/docs/analyzer/checkers.rst b/clang/docs/analyzer/checkers.rst
index b817a99a1c56f..c947f26204159 100644
--- a/clang/docs/analyzer/checkers.rst
+++ b/clang/docs/analyzer/checkers.rst
@@ -3522,6 +3522,31 @@ Raw pointers and references to an object which supports CheckedPtr or CheckedRef
 
 See `WebKit Guidelines for Safer C++ Programming <https://github.com/WebKit/WebKit/wiki/Safer-CPP-Guidelines>`_ for details.
 
+alpha.webkit.NoUnretainedMemberChecker
+""""""""""""""""""""""""""""""""""""""""
+Raw pointers and references to a NS or CF object can't be used as class members or ivars. Only RetainPtr is allowed for CF types regardless of whether ARC is enabled or disabled. Only RetainPtr is allowed for NS types when ARC is disabled.
+
+.. code-block:: cpp
+
+ struct Foo {
+   NSObject *ptr; // warn
+   // ...
+ };
+
+See `WebKit Guidelines for Safer C++ Programming <https://github.com/WebKit/WebKit/wiki/Safer-CPP-Guidelines>`_ for details.
+
+alpha.webkit.UnretainedLambdaCapturesChecker
+""""""""""""""""""""""""""""""""""""""""""""
+Raw pointers and references to NS or CF types can't be captured in lambdas. Only RetainPtr is allowed for CF types regardless of whether ARC is enabled or disabled, and only RetainPtr is allowed for NS types when ARC is disabled.
+
+.. code-block:: cpp
+
+ void foo(NSObject *a, NSObject *b) {
+   [&, a](){ // warn about 'a'
+     do_something(b); // warn about 'b'
+   };
+ };
+
 .. _alpha-webkit-UncountedCallArgsChecker:
 
 alpha.webkit.UncountedCallArgsChecker
diff --git a/clang/docs/tools/dump_ast_matchers.py b/clang/docs/tools/dump_ast_matchers.py
index b6f00657ec914..46b7bb718ba08 100755
--- a/clang/docs/tools/dump_ast_matchers.py
+++ b/clang/docs/tools/dump_ast_matchers.py
@@ -106,7 +106,7 @@ def extract_result_types(comment):
 
 
 def strip_doxygen(comment):
-    """Returns the given comment without \-escaped words."""
+    """Returns the given comment without -escaped words."""
     # If there is only a doxygen keyword in the line, delete the whole line.
     comment = re.sub(r"^\\[^\s]+\n", r"", comment, flags=re.M)
 
@@ -241,7 +241,7 @@ def act_on_decl(declaration, comment, allowed_types):
 
         # Parse the various matcher definition macros.
         m = re.match(
-            """.*AST_TYPE(LOC)?_TRAVERSE_MATCHER(?:_DECL)?\(
+            r""".*AST_TYPE(LOC)?_TRAVERSE_MATCHER(?:_DECL)?\(
                        \s*([^\s,]+\s*),
                        \s*(?:[^\s,]+\s*),
                        \s*AST_POLYMORPHIC_SUPPORTED_TYPES\(([^)]*)\)
@@ -615,5 +615,5 @@ def sort_table(matcher_type, matcher_map):
     flags=re.S,
 )
 
-with open("../LibASTMatchersReference.html", "w", newline="\n") as output:
+with open(HTML_FILE, "w", newline="\n") as output:
     output.write(reference)
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index cfe49acf20b77..b81f3a403baf6 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -3005,18 +3005,6 @@ class CallExpr : public Expr {
                           FPOptionsOverride FPFeatures, unsigned MinNumArgs = 0,
                           ADLCallKind UsesADL = NotADL);
 
-  /// Create a temporary call expression with no arguments in the memory
-  /// pointed to by Mem. Mem must points to at least sizeof(CallExpr)
-  /// + sizeof(Stmt *) bytes of storage, aligned to alignof(CallExpr):
-  ///
-  /// \code{.cpp}
-  ///   alignas(CallExpr) char Buffer[sizeof(CallExpr) + sizeof(Stmt *)];
-  ///   CallExpr *TheCall = CallExpr::CreateTemporary(Buffer, etc);
-  /// \endcode
-  static CallExpr *CreateTemporary(void *Mem, Expr *Fn, QualType Ty,
-                                   ExprValueKind VK, SourceLocation RParenLoc,
-                                   ADLCallKind UsesADL = NotADL);
-
   /// Create an empty call expression, for deserialization.
   static CallExpr *CreateEmpty(const ASTContext &Ctx, unsigned NumArgs,
                                bool HasFPFeatures, EmptyShell Empty);
diff --git a/clang/include/clang/AST/Mangle.h b/clang/include/clang/AST/Mangle.h
index 6134a70c04bd3..9ed8895cbfff1 100644
--- a/clang/include/clang/AST/Mangle.h
+++ b/clang/include/clang/AST/Mangle.h
@@ -140,7 +140,7 @@ class MangleContext {
   virtual void mangleCXXRTTIName(QualType T, raw_ostream &,
                                  bool NormalizeIntegers = false) = 0;
   virtual void mangleStringLiteral(const StringLiteral *SL, raw_ostream &) = 0;
-  virtual void mangleMSGuidDecl(const MSGuidDecl *GD, raw_ostream &);
+  virtual void mangleMSGuidDecl(const MSGuidDecl *GD, raw_ostream &) const;
 
   void mangleGlobalBlock(const BlockDecl *BD, const NamedDecl *ID,
                          raw_ostream &Out);
@@ -153,9 +153,9 @@ class MangleContext {
 
   void mangleObjCMethodName(const ObjCMethodDecl *MD, raw_ostream &OS,
                             bool includePrefixByte = true,
-                            bool includeCategoryNamespace = true);
+                            bool includeCategoryNamespace = true) const;
   void mangleObjCMethodNameAsSourceName(const ObjCMethodDecl *MD,
-                                        raw_ostream &);
+                                        raw_ostream &) const;
 
   virtual void mangleStaticGuardVariable(const VarDecl *D, raw_ostream &) = 0;
 
diff --git a/clang/include/clang/AST/NestedNameSpecifier.h b/clang/include/clang/AST/NestedNameSpecifier.h
index a1d9e30e660d1..051d632f1cdf9 100644
--- a/clang/include/clang/AST/NestedNameSpecifier.h
+++ b/clang/include/clang/AST/NestedNameSpecifier.h
@@ -223,7 +223,8 @@ class NestedNameSpecifier : public llvm::FoldingSetNode {
   /// `ns::SomeTemplate<int, MyClass>` instead of
   /// `ns::SomeTemplate<Container::value_type, T>`.
   void print(raw_ostream &OS, const PrintingPolicy &Policy,
-             bool ResolveTemplateArguments = false) const;
+             bool ResolveTemplateArguments = false,
+             bool PrintFinalScopeResOp = true) const;
 
   void Profile(llvm::FoldingSetNodeID &ID) const {
     ID.AddPointer(Prefix.getOpaqueValue());
diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index b2cf621bc0a78..049389229b8d5 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -18,6 +18,7 @@
 #include "clang/Basic/OpenACCKinds.h"
 
 #include <utility>
+#include <variant>
 
 namespace clang {
 /// This is the base type for all OpenACC Clauses.
@@ -206,6 +207,50 @@ class OpenACCClauseWithParams : public OpenACCClause {
   }
 };
 
+class OpenACCBindClause final : public OpenACCClauseWithParams {
+  std::variant<const StringLiteral *, const IdentifierInfo *> Argument;
+
+  OpenACCBindClause(SourceLocation BeginLoc, SourceLocation LParenLoc,
+                    const clang::StringLiteral *SL, SourceLocation EndLoc)
+      : OpenACCClauseWithParams(OpenACCClauseKind::Bind, BeginLoc, LParenLoc,
+                                EndLoc),
+        Argument(SL) {}
+  OpenACCBindClause(SourceLocation BeginLoc, SourceLocation LParenLoc,
+                    const IdentifierInfo *ID, SourceLocation EndLoc)
+      : OpenACCClauseWithParams(OpenACCClauseKind::Bind, BeginLoc, LParenLoc,
+                                EndLoc),
+        Argument(ID) {}
+
+public:
+  static bool classof(const OpenACCClause *C) {
+    return C->getClauseKind() == OpenACCClauseKind::Bind;
+  }
+  static OpenACCBindClause *Create(const ASTContext &C, SourceLocation BeginLoc,
+                                   SourceLocation LParenLoc,
+                                   const IdentifierInfo *ID,
+                                   SourceLocation EndLoc);
+  static OpenACCBindClause *Create(const ASTContext &C, SourceLocation BeginLoc,
+                                   SourceLocation LParenLoc,
+                                   const StringLiteral *SL,
+                                   SourceLocation EndLoc);
+
+  bool isStringArgument() const {
+    return std::holds_alternative<const StringLiteral *>(Argument);
+  }
+
+  const StringLiteral *getStringArgument() const {
+    return std::get<const StringLiteral *>(Argument);
+  }
+
+  bool isIdentifierArgument() const {
+    return std::holds_alternative<const IdentifierInfo *>(Argument);
+  }
+
+  const IdentifierInfo *getIdentifierArgument() const {
+    return std::get<const IdentifierInfo *>(Argument);
+  }
+};
+
 using DeviceTypeArgument = std::pair<IdentifierInfo *, SourceLocation>;
 /// A 'device_type' or 'dtype' clause, takes a list of either an 'asterisk' or
 /// an identifier. The 'asterisk' means 'the rest'.
diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 154ecfbaa4418..46f34a7302847 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1656,6 +1656,49 @@ class OMPAtomicDefaultMemOrderClause final : public OMPClause {
   }
 };
 
+/// This represents 'self_maps' clause in the '#pragma omp requires'
+/// directive.
+///
+/// \code
+/// #pragma omp requires self_maps
+/// \endcode
+/// In this example directive '#pragma omp requires' has 'self_maps'
+/// clause.
+class OMPSelfMapsClause final : public OMPClause {
+public:
+  friend class OMPClauseReader;
+  /// Build 'self_maps' clause.
+  ///
+  /// \param StartLoc Starting location of the clause.
+  /// \param EndLoc Ending location of the clause.
+  OMPSelfMapsClause(SourceLocation StartLoc, SourceLocation EndLoc)
+      : OMPClause(llvm::omp::OMPC_self_maps, StartLoc, EndLoc) {}
+
+  /// Build an empty clause.
+  OMPSelfMapsClause()
+      : OMPClause(llvm::omp::OMPC_self_maps, SourceLocation(),
+                  SourceLocation()) {}
+
+  child_range children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+
+  const_child_range children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  child_range used_children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+  const_child_range used_children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  static bool classof(const OMPClause *T) {
+    return T->getClauseKind() == llvm::omp::OMPC_self_maps;
+  }
+};
+
 /// This represents 'at' clause in the '#pragma omp error' directive
 ///
 /// \code
@@ -6349,7 +6392,8 @@ class OMPMapClause final : public OMPMappableExprListClause<OMPMapClause>,
   OpenMPMapModifierKind MapTypeModifiers[NumberOfOMPMapClauseModifiers] = {
       OMPC_MAP_MODIFIER_unknown, OMPC_MAP_MODIFIER_unknown,
       OMPC_MAP_MODIFIER_unknown, OMPC_MAP_MODIFIER_unknown,
-      OMPC_MAP_MODIFIER_unknown, OMPC_MAP_MODIFIER_unknown};
+      OMPC_MAP_MODIFIER_unknown, OMPC_MAP_MODIFIER_unknown,
+      OMPC_MAP_MODIFIER_unknown};
 
   /// Location of map-type-modifiers for the 'map' clause.
   SourceLocation MapTypeModifiersLoc[NumberOfOMPMapClauseModifiers];
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 5ca3e435f033b..fac4c10987157 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3443,6 +3443,11 @@ bool RecursiveASTVisitor<Derived>::VisitOMPAtomicDefaultMemOrderClause(
   return true;
 }
 
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPSelfMapsClause(OMPSelfMapsClause *) {
+  return true;
+}
+
 template <typename Derived>
 bool RecursiveASTVisitor<Derived>::VisitOMPAtClause(OMPAtClause *) {
   return true;
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index ef59bd1621fb8..3c942f2ed7486 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -2568,6 +2568,9 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
   bool isVectorType() const;                    // GCC vector type.
   bool isExtVectorType() const;                 // Extended vector type.
   bool isExtVectorBoolType() const;             // Extended vector type with bool element.
+  // Extended vector type with bool element that is packed. HLSL doesn't pack
+  // its bool vectors.
+  bool isPackedVectorBoolType(const ASTContext &ctx) const;
   bool isSubscriptableVectorType() const;
   bool isMatrixType() const;                    // Matrix type.
   bool isConstantMatrixType() const;            // Constant matrix type.
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index 0f7e3a8a01762..738617759eb29 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -1084,12 +1084,19 @@ AST_POLYMORPHIC_MATCHER_P2(
 /// \code
 ///   template<typename T> struct C {};
 ///   C<int> c;
+///   template<typename T> void f() {}
+///   void func() { f<int>(); };
 /// \endcode
+///
 /// classTemplateSpecializationDecl(templateArgumentCountIs(1))
 ///   matches C<int>.
+///
+/// functionDecl(templateArgumentCountIs(1))
+///   matches f<int>();
 AST_POLYMORPHIC_MATCHER_P(
     templateArgumentCountIs,
     AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
+                                    VarTemplateSpecializationDecl, FunctionDecl,
                                     TemplateSpecializationType),
     unsigned, N) {
   return internal::getTemplateSpecializationArgs(Node).size() == N;
@@ -7494,7 +7501,8 @@ extern const AstTypeMatcher<RValueReferenceType> rValueReferenceType;
 ///   matches "int const *b"
 ///
 /// Usable as: Matcher<BlockPointerType>, Matcher<MemberPointerType>,
-///   Matcher<PointerType>, Matcher<ReferenceType>
+///   Matcher<PointerType>, Matcher<ReferenceType>,
+///   Matcher<ObjCObjectPointerType>
 AST_TYPELOC_TRAVERSE_MATCHER_DECL(
     pointee, getPointee,
     AST_POLYMORPHIC_SUPPORTED_TYPES(BlockPointerType, MemberPointerType,
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index ccfe69d32e0a6..4d34346460561 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -1721,17 +1721,10 @@ def EnableIf : InheritableAttr {
   let Documentation = [EnableIfDocs];
 }
 
-def ExtVectorType : Attr {
-  // This is an OpenCL-related attribute and does not receive a [[]] spelling.
-  let Spellings = [GNU<"ext_vector_type">];
-  // FIXME: This subject list is wrong; this is a type attribute.
-  let Subjects = SubjectList<[TypedefName], ErrorDiag>;
+def ExtVectorType : TypeAttr {
+  let Spellings = [Clang<"ext_vector_type">];
   let Args = [ExprArgument<"NumElements">];
-  let ASTNode = 0;
-  let Documentation = [Undocumented];
-  // This is a type attribute with an incorrect subject list, so should not be
-  // permitted by #pragma clang attribute.
-  let PragmaAttributeSupport = 0;
+  let Documentation = [ExtVectorTypeDocs];
 }
 
 def FallThrough : StmtAttr {
@@ -4724,21 +4717,21 @@ def HLSLNumThreads: InheritableAttr {
 }
 
 def HLSLSV_GroupThreadID: HLSLAnnotationAttr {
-  let Spellings = [HLSLAnnotation<"SV_GroupThreadID">];
+  let Spellings = [HLSLAnnotation<"sv_groupthreadid">];
   let Subjects = SubjectList<[ParmVar, Field]>;
   let LangOpts = [HLSL];
   let Documentation = [HLSLSV_GroupThreadIDDocs];
 }
 
 def HLSLSV_GroupID: HLSLAnnotationAttr {
-  let Spellings = [HLSLAnnotation<"SV_GroupID">];
+  let Spellings = [HLSLAnnotation<"sv_groupid">];
   let Subjects = SubjectList<[ParmVar, Field]>;
   let LangOpts = [HLSL];
   let Documentation = [HLSLSV_GroupIDDocs];
 }
 
 def HLSLSV_GroupIndex: HLSLAnnotationAttr {
-  let Spellings = [HLSLAnnotation<"SV_GroupIndex">];
+  let Spellings = [HLSLAnnotation<"sv_groupindex">];
   let Subjects = SubjectList<[ParmVar, GlobalVar]>;
   let LangOpts = [HLSL];
   let Documentation = [HLSLSV_GroupIndexDocs];
@@ -4790,7 +4783,7 @@ def HLSLPackOffset: HLSLAnnotationAttr {
 }
 
 def HLSLSV_DispatchThreadID: HLSLAnnotationAttr {
-  let Spellings = [HLSLAnnotation<"SV_DispatchThreadID">];
+  let Spellings = [HLSLAnnotation<"sv_dispatchthreadid">];
   let Subjects = SubjectList<[ParmVar, Field]>;
   let LangOpts = [HLSL];
   let Documentation = [HLSLSV_DispatchThreadIDDocs];
@@ -5033,4 +5026,7 @@ def OpenACCRoutineAnnot : InheritableAttr {
   let Spellings = [];
   let Subjects = SubjectList<[Function]>;
   let Documentation = [InternalOnly];
+  let AdditionalMembers = [{
+    SourceLocation BindClause;
+  }];
 }
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index fdc58d1c92c0d..34e7ff9612859 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -1113,6 +1113,41 @@ template instantiation, so the value for ``T::number`` is known.
   }];
 }
 
+def ExtVectorTypeDocs : Documentation {
+  let Category = DocCatFunction;
+  let Content = [{
+The ``ext_vector_type(N)`` attribute specifies that a type is a vector with N
+elements, directly mapping to an LLVM vector type. Originally from OpenCL, it
+allows element access the array subscript operator ``[]``, ``sN`` where N is
+a hexadecimal value, or ``x, y, z, w`` for graphics-style indexing.
+This attribute enables efficient SIMD operations and is usable in
+general-purpose code.
+
+.. code-block:: c++
+
+  template <typename T, uint32_t N>
+  constexpr T simd_reduce(T [[clang::ext_vector_type(N)]] v) {
+    static_assert((N & (N - 1)) == 0, "N must be a power of two");
+    if constexpr (N == 1)
+      return v[0];
+    else
+      return simd_reduce<T, N / 2>(v.hi + v.lo);
+  }
+
+The vector type also supports swizzling up to sixteen elements. This can be done
+using the object accessors. The OpenCL documentation lists all of the accepted
+values.
+
+.. code-block:: c++
+
+  using f16_x16 = _Float16 __attribute__((ext_vector_type(16)));
+
+  f16_x16 reverse(f16_x16 v) { return v.sfedcba9876543210; }
+
+See the OpenCL documentation for some more complete examples.
+  }];
+}
+
 def DiagnoseIfDocs : Documentation {
   let Category = DocCatFunction;
   let Content = [{
@@ -2848,7 +2883,7 @@ https://gcc.gnu.org/onlinedocs/gcc/RISC-V-Function-Attributes.html
 https://riscv.org/specifications/privileged-isa/
 The RISC-V Instruction Set Manual Volume II: Privileged Architecture
 Version 1.10.
-https://github.com/quic/riscv-unified-db/releases/tag/Xqci-0.6
+https://github.com/quic/riscv-unified-db/releases/tag/Xqci-0.7
   }];
 }
 
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 2268df70927a7..2fbdfaea57ccd 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -1340,6 +1340,12 @@ def ElementwiseExp2 : Builtin {
   let Prototype = "void(...)";
 }
 
+def ElementwiseExp10 : Builtin {
+  let Spellings = ["__builtin_elementwise_exp10"];
+  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Prototype = "void(...)";
+}
+
 def ElementwiseFloor : Builtin {
   let Spellings = ["__builtin_elementwise_floor"];
   let Attributes = [NoThrow, Const, CustomTypeChecking];
diff --git a/clang/include/clang/Basic/DiagnosticOptions.h b/clang/include/clang/Basic/DiagnosticOptions.h
index e2faf3d0df95c..29146532f9524 100644
--- a/clang/include/clang/Basic/DiagnosticOptions.h
+++ b/clang/include/clang/Basic/DiagnosticOptions.h
@@ -67,7 +67,8 @@ inline DiagnosticLevelMask operator&(DiagnosticLevelMask LHS,
 raw_ostream& operator<<(raw_ostream& Out, DiagnosticLevelMask M);
 
 /// Options for controlling the compiler diagnostics engine.
-class DiagnosticOptions : public RefCountedBase<DiagnosticOptions>{
+class DiagnosticOptions
+    : public llvm::ThreadSafeRefCountedBase<DiagnosticOptions> {
   friend bool ParseDiagnosticArgs(DiagnosticOptions &, llvm::opt::ArgList &,
                                   clang::DiagnosticsEngine *, bool);
 
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index c513dab810d1f..092a55f9e9e0c 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1495,7 +1495,7 @@ def note_previous_map_type_specified_here
     : Note<"map type '%0' is previous specified here">;
 def err_omp_unknown_map_type_modifier : Error<
   "incorrect map type modifier, expected one of: 'always', 'close', 'mapper'"
-  "%select{|, 'present'|, 'present', 'iterator'}0%select{|, 'ompx_hold'}1">;
+  "%select{|, 'present'|, 'present', 'iterator'}0%select{|, 'ompx_hold'}1%select{|, 'self'}2">;
 def err_omp_map_type_missing : Error<
   "missing map type">;
 def err_omp_map_type_modifier_missing : Error<
@@ -1817,5 +1817,9 @@ def ext_hlsl_access_specifiers : ExtWarn<
   InGroup<HLSLExtension>;
 def err_hlsl_unsupported_component : Error<"invalid component '%0' used; expected 'x', 'y', 'z', or 'w'">;
 def err_hlsl_packoffset_invalid_reg : Error<"invalid resource class specifier '%0' for packoffset, expected 'c'">;
+def err_hlsl_virtual_function
+    : Error<"virtual functions are unsupported in HLSL">;
+def err_hlsl_virtual_inheritance
+    : Error<"virtual inheritance is unsupported in HLSL">;
 
 } // end of Parser diagnostics
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index cf0e9846d4259..8e6e6e892cdd7 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -529,8 +529,12 @@ def warn_cxx14_compat_decomp_decl : Warning<
 def ext_decomp_decl : ExtWarn<
   "decomposition declarations are a C++17 extension">, InGroup<CXX17>;
 def ext_decomp_decl_cond : ExtWarn<
-  "ISO C++17 does not permit structured binding declaration in a condition">,
-  InGroup<DiagGroup<"binding-in-condition">>;
+  "structured binding declaration in a condition is a C++2c extenstion">,
+  InGroup<CXX26>;
+def warn_cxx26_decomp_decl_cond : Warning<
+  "structured binding declaration in a condition is incompatible with "
+  "C++ standards before C++2c">,
+  InGroup<CXXPre26Compat>, DefaultIgnore;
 def err_decomp_decl_spec : Error<
   "decomposition declaration cannot be declared "
   "%plural{1:'%1'|:with '%1' specifiers}0">;
@@ -551,7 +555,7 @@ def err_decomp_decl_constraint : Error<
 def err_decomp_decl_parens : Error<
   "decomposition declaration cannot be declared with parentheses">;
 def err_decomp_decl_template : Error<
-  "decomposition declaration template not supported">;
+  "decomposition declaration cannot be a template">;
 def err_decomp_decl_not_alone : Error<
   "decomposition declaration must be the only declaration in its group">;
 def err_decomp_decl_requires_init : Error<
@@ -599,16 +603,17 @@ def err_using_typename_non_type : Error<
   "'typename' keyword used on a non-type">;
 def err_using_dependent_value_is_type : Error<
   "dependent using declaration resolved to type without 'typename'">;
-def err_using_decl_nested_name_specifier_is_not_class : Error<
-  "using declaration in class refers into '%0', which is not a class">;
+def err_using_decl_nested_name_specifier_is_not_class
+    : Error<"using declaration in class refers into %0, which is not a class">;
 def warn_cxx17_compat_using_decl_non_member_enumerator : Warning<
   "member using declaration naming non-class '%0' enumerator is "
   "incompatible with C++ standards before C++20">, InGroup<CXXPre20Compat>,
   DefaultIgnore;
 def err_using_decl_nested_name_specifier_is_current_class : Error<
   "using declaration refers to its own class">;
-def err_using_decl_nested_name_specifier_is_not_base_class : Error<
-  "using declaration refers into '%0', which is not a base class of %1">;
+def err_using_decl_nested_name_specifier_is_not_base_class
+    : Error<
+          "using declaration refers into %0, which is not a base class of %1">;
 def err_using_decl_constructor_not_in_direct_base : Error<
   "%0 is not a direct base of %1, cannot inherit constructors">;
 def err_using_decl_can_not_refer_to_class_member : Error<
@@ -1733,8 +1738,8 @@ def err_no_matching_local_friend_suggest : Error<
   "cannot define friend function %0 in a local class definition; did you mean %3?">;
 def err_partial_specialization_friend : Error<
   "partial specialization cannot be declared as a friend">;
-def err_qualified_friend_def : Error<
-  "friend function definition cannot be qualified with '%0'">;
+def err_qualified_friend_def
+    : Error<"friend function definition cannot be qualified with %0">;
 def err_friend_def_in_local_class : Error<
   "friend function cannot be defined in a local class">;
 def err_friend_specialization_def : Error<
@@ -1743,14 +1748,16 @@ def err_friend_not_first_in_declaration : Error<
   "'friend' must appear first in a non-function declaration">;
 def err_using_decl_friend : Error<
   "cannot befriend target of using declaration">;
-def warn_template_qualified_friend_unsupported : Warning<
-  "dependent nested name specifier '%0' for friend class declaration is "
-  "not supported; turning off access control for %1">,
-  InGroup<UnsupportedFriend>;
-def warn_template_qualified_friend_ignored : Warning<
-  "dependent nested name specifier '%0' for friend template declaration is "
-  "not supported; ignoring this friend declaration">,
-  InGroup<UnsupportedFriend>;
+def warn_template_qualified_friend_unsupported
+    : Warning<
+          "dependent nested name specifier %0 for friend class declaration is "
+          "not supported; turning off access control for %1">,
+      InGroup<UnsupportedFriend>;
+def warn_template_qualified_friend_ignored
+    : Warning<"dependent nested name specifier %0 for friend template "
+              "declaration is "
+              "not supported; ignoring this friend declaration">,
+      InGroup<UnsupportedFriend>;
 def ext_friend_tag_redecl_outside_namespace : ExtWarn<
   "unqualified friend declaration referring to type outside of the nearest "
   "enclosing namespace is a Microsoft extension; add a nested name specifier">,
@@ -5210,11 +5217,16 @@ def err_template_unnamed_class : Error<
 def err_template_param_list_different_arity : Error<
   "%select{too few|too many}0 template parameters in template "
   "%select{|template parameter }1redeclaration">;
+def note_template_param_list_different_arity : Note<
+  "%select{too few|too many}0 template parameters in template template "
+  "argument">;
 def note_template_prev_declaration : Note<
   "previous template %select{declaration|template parameter}0 is here">;
 def err_template_param_different_kind : Error<
   "template parameter has a different kind in template "
   "%select{|template parameter }0redeclaration">;
+def note_template_param_different_kind : Note<
+  "template parameter has a different kind in template argument">;
 
 def err_invalid_decl_specifier_in_nontype_parm : Error<
   "invalid declaration specifier in template non-type parameter">;
@@ -5223,6 +5235,8 @@ def err_template_nontype_parm_different_type : Error<
   "template non-type parameter has a different type %0 in template "
   "%select{|template parameter }1redeclaration">;
 
+def note_template_nontype_parm_different_type : Note<
+  "template non-type parameter has a different type %0 in template argument">;
 def note_template_nontype_parm_prev_declaration : Note<
   "previous non-type template parameter with type %0 is here">;
 def err_template_nontype_parm_bad_type : Error<
@@ -5313,15 +5327,10 @@ def err_template_missing_args : Error<
   "%select{class template|function template|variable template|alias template|"
   "template template parameter|concept|template}0 %1 requires template "
   "arguments">;
-def err_template_param_missing_arg : Error<
-  "missing template argument for template parameter">;
-def err_template_template_param_missing_param : Error<
-  "no template parameter in this template template parameter "
-  "corresponds to non-defaulted template parameter of argument template">;
-def err_template_too_many_args : Error<
-  "too many template arguments for "
+def err_template_arg_list_different_arity : Error<
+  "%select{too few|too many}0 template arguments for "
   "%select{class template|function template|variable template|alias template|"
-  "template template parameter|concept|template}0 %1">;
+  "template template parameter|concept|template}1 %2">;
 def note_template_decl_here : Note<"template is declared here">;
 def note_template_decl_external : Note<
   "template declaration from hidden source: %0">;
@@ -5359,8 +5368,11 @@ def err_template_arg_not_valid_template : Error<
   "template parameter">;
 def note_template_arg_refers_here_func : Note<
   "template argument refers to function template %0, here">;
+def err_template_arg_template_params_mismatch : Error<
+  "template template argument has different template parameters than its "
+  "corresponding template template parameter">;
 def note_template_arg_template_params_mismatch : Note<
-  "template template argument is incompatible with its "
+  "template template argument has different template parameters than its "
   "corresponding template template parameter">;
 def err_non_deduced_mismatch : Error<
   "could not match %diff{$ against $|types}0,1">;
@@ -5551,9 +5563,10 @@ def ext_template_spec_extra_headers : ExtWarn<
 def note_explicit_template_spec_does_not_need_header : Note<
   "'template<>' header not required for explicitly-specialized class %0 "
   "declared here">;
-def err_template_qualified_declarator_no_match : Error<
-  "nested name specifier '%0' for declaration does not refer into a class, "
-  "class template or class template partial specialization">;
+def err_template_qualified_declarator_no_match
+    : Error<"nested name specifier %0 for declaration does not refer into a "
+            "class, "
+            "class template or class template partial specialization">;
 def err_specialize_member_of_template : Error<
   "cannot specialize %select{|(with 'template<>') }0a member of an "
   "unspecialized template">;
@@ -5853,13 +5866,13 @@ def note_typename_member_refers_here : Note<
     "referenced member %0 is declared here">;
 def note_typename_refers_here : Note<
     "referenced %0 is declared here">;
-def err_typename_missing : Error<
-  "missing 'typename' prior to dependent type name '%0%1'">;
-def err_typename_missing_template : Error<
-  "missing 'typename' prior to dependent type template name '%0%1'">;
-def ext_typename_missing : ExtWarn<
-  "missing 'typename' prior to dependent type name '%0%1'">,
-  InGroup<DiagGroup<"typename-missing">>;
+def err_typename_missing
+    : Error<"missing 'typename' prior to dependent type name %0">;
+def err_typename_missing_template
+    : Error<"missing 'typename' prior to dependent type template name %0">;
+def ext_typename_missing
+    : ExtWarn<"missing 'typename' prior to dependent type name %0">,
+      InGroup<DiagGroup<"typename-missing">>;
 def ext_typename_outside_of_template : ExtWarn<
   "'typename' occurs outside of a template">, InGroup<CXX11>;
 def warn_cxx98_compat_typename_outside_of_template : Warning<
@@ -5873,9 +5886,10 @@ def note_using_value_decl_missing_typename : Note<
 def warn_cxx17_compat_implicit_typename : Warning<"use of implicit 'typename' is "
   "incompatible with C++ standards before C++20">, InGroup<CXX20Compat>,
   DefaultIgnore;
-def ext_implicit_typename : ExtWarn<"missing 'typename' prior to dependent "
-  "type name %0%1; implicit 'typename' is a C++20 extension">,
-  InGroup<CXX20>;
+def ext_implicit_typename
+    : ExtWarn<"missing 'typename' prior to dependent "
+              "type name %0; implicit 'typename' is a C++20 extension">,
+      InGroup<CXX20>;
 
 def err_template_kw_refers_to_non_template : Error<
   "%0%select{| following the 'template' keyword}1 "
@@ -5885,12 +5899,13 @@ def note_template_kw_refers_to_non_template : Note<
 def err_template_kw_refers_to_dependent_non_template : Error<
   "%0%select{| following the 'template' keyword}1 "
   "cannot refer to a dependent template">;
-def err_template_kw_refers_to_type_template : Error<
-  "'%0%1' is expected to be a non-type template, but instantiated to a %select{class|type alias}2 template">;
+def err_template_kw_refers_to_type_template
+    : Error<"%0 is expected to be a non-type template, but instantiated to a "
+            "%select{class|type alias}1 template">;
 def note_referenced_type_template : Note<
   "%select{class|type alias}0 template declared here">;
-def err_template_kw_missing : Error<
-  "missing 'template' keyword prior to dependent template name '%0%1'">;
+def err_template_kw_missing
+    : Error<"missing 'template' keyword prior to dependent template name %0">;
 def ext_template_outside_of_template : ExtWarn<
   "'template' keyword outside of a template">, InGroup<CXX11>;
 def warn_cxx98_compat_template_outside_of_template : Warning<
@@ -5922,6 +5937,10 @@ def err_template_parameter_pack_non_pack : Error<
   "%select{template type|non-type template|template template}0 parameter"
   "%select{| pack}1 conflicts with previous %select{template type|"
   "non-type template|template template}0 parameter%select{ pack|}1">;
+def note_template_parameter_pack_non_pack : Note<
+  "%select{template type|non-type template|template template}0 parameter"
+  "%select{| pack}1 does not match %select{template type|non-type template"
+  "|template template}0 parameter%select{ pack|}1 in template argument">;
 def note_template_parameter_pack_here : Note<
   "previous %select{template type|non-type template|template template}0 "
   "parameter%select{| pack}1 declared here">;
@@ -7879,8 +7898,8 @@ def err_nogetter_property_incdec : Error<
   "no getter method %1 for %select{increment|decrement}0 of property">;
 def err_no_subobject_property_setting : Error<
   "expression is not assignable">;
-def err_qualified_objc_access : Error<
-  "%select{property|instance variable}0 access cannot be qualified with '%1'">;
+def err_qualified_objc_access : Error<"%select{property|instance variable}0 "
+                                      "access cannot be qualified with %1">;
 
 def ext_freestanding_complex : Extension<
   "complex numbers are an extension in a freestanding C99 implementation">;
@@ -9830,8 +9849,8 @@ def note_non_usual_function_declared_here : Note<
 // C++ literal operators
 def err_literal_operator_outside_namespace : Error<
   "literal operator %0 must be in a namespace or global scope">;
-def err_literal_operator_id_outside_namespace : Error<
-  "non-namespace scope '%0' cannot have a literal operator member">;
+def err_literal_operator_id_outside_namespace
+    : Error<"non-namespace scope %0 cannot have a literal operator member">;
 def err_literal_operator_default_argument : Error<
   "literal operator cannot have a default argument">;
 def err_literal_operator_bad_param_count : Error<
@@ -10425,8 +10444,13 @@ def warn_type_safety_null_pointer_required : Warning<
   "specified %0 type tag requires a null pointer">, InGroup<TypeSafety>;
 
 // Generic selections.
-def err_assoc_type_incomplete : Error<
-  "type %0 in generic association incomplete">;
+def ext_assoc_type_incomplete : Extension<
+  "incomplete type %0 in a '_Generic' association is a C2y extension">,
+  InGroup<C2y>;
+def warn_c2y_compat_assoc_type_incomplete : Warning<
+  "use of incomplete type %0 in a '_Generic' association is incompatible with "
+  "C standards before C2y">,
+  InGroup<CPre2yCompat>, DefaultIgnore;
 def err_assoc_type_nonobject : Error<
   "type %0 in generic association not an object type">;
 def err_assoc_type_variably_modified : Error<
@@ -12702,6 +12726,10 @@ def err_hlsl_param_qualifier_mismatch :
 def err_hlsl_vector_compound_assignment_truncation : Error<
   "left hand operand of type %0 to compound assignment cannot be truncated "
   "when used with right hand operand of type %1">;
+def err_hlsl_builtin_scalar_vector_mismatch
+    : Error<
+          "%select{all|second and third}0 arguments to %1 must be of scalar or "
+          "vector type with matching scalar element type%diff{: $ vs $|}2,3">;
 
 def warn_hlsl_impcast_vector_truncation : Warning<
   "implicit conversion truncates vector: %0 to %1">, InGroup<Conversion>;
@@ -12820,9 +12848,6 @@ def warn_acc_routine_unimplemented
     : Warning<"OpenACC construct 'routine' with implicit function not yet "
               "implemented, pragma ignored">,
       InGroup<SourceUsesOpenACC>;
-def warn_acc_clause_unimplemented
-    : Warning<"OpenACC clause '%0' not yet implemented, clause ignored">,
-      InGroup<SourceUsesOpenACC>;
 def err_acc_construct_appertainment
     : Error<"OpenACC construct '%0' cannot be used here; it can only "
             "be used in a statement context">;
@@ -13063,6 +13088,9 @@ def err_acc_routine_overload_set
 def err_acc_magic_static_in_routine
     : Error<"function static variables are not permitted in functions to which "
             "an OpenACC 'routine' directive applies">;
+def err_acc_duplicate_bind
+    : Error<"multiple 'routine' directives with 'bind' clauses are not "
+            "permitted to refer to the same function">;
 
 // AMDGCN builtins diagnostics
 def err_amdgcn_global_load_lds_size_invalid_value : Error<"invalid size value">;
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index 383440ddbc0ea..3879cc7942877 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -515,7 +515,7 @@ VALUE_LANGOPT(FuchsiaAPILevel, 32, 0, "Fuchsia API level")
 // on large _BitInts.
 BENIGN_VALUE_LANGOPT(MaxBitIntWidth, 32, 128, "Maximum width of a _BitInt")
 
-COMPATIBLE_LANGOPT(IncrementalExtensions, 1, 0, " True if we want to process statements"
+COMPATIBLE_LANGOPT(IncrementalExtensions, 1, 0, "True if we want to process statements "
         "on the global scope, ignore EOF token and continue later on (thus "
         "avoid tearing the Lexer and etc. down). Controlled by "
         "-fincremental-extensions.")
diff --git a/clang/include/clang/Basic/OpenACCClauses.def b/clang/include/clang/Basic/OpenACCClauses.def
index f04965363f25e..a19ee69d3d190 100644
--- a/clang/include/clang/Basic/OpenACCClauses.def
+++ b/clang/include/clang/Basic/OpenACCClauses.def
@@ -24,6 +24,7 @@
 VISIT_CLAUSE(Auto)
 VISIT_CLAUSE(Async)
 VISIT_CLAUSE(Attach)
+VISIT_CLAUSE(Bind)
 VISIT_CLAUSE(Collapse)
 VISIT_CLAUSE(Copy)
 CLAUSE_ALIAS(PCopy, Copy, true)
diff --git a/clang/include/clang/Basic/OpenMPKinds.def b/clang/include/clang/Basic/OpenMPKinds.def
index 76a861f416fd5..c4a2cca731607 100644
--- a/clang/include/clang/Basic/OpenMPKinds.def
+++ b/clang/include/clang/Basic/OpenMPKinds.def
@@ -172,6 +172,7 @@ OPENMP_MAP_MODIFIER_KIND(close)
 OPENMP_MAP_MODIFIER_KIND(mapper)
 OPENMP_MAP_MODIFIER_KIND(iterator)
 OPENMP_MAP_MODIFIER_KIND(present)
+OPENMP_MAP_MODIFIER_KIND(self)
 // This is an OpenMP extension for the sake of OpenACC support.
 OPENMP_MAP_MODIFIER_KIND(ompx_hold)
 
diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
index b65797e40d5f9..017ae0c53a984 100644
--- a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
+++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
@@ -26,6 +26,10 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
   CIRBaseBuilderTy(mlir::MLIRContext &mlirContext)
       : mlir::OpBuilder(&mlirContext) {}
 
+  cir::ConstantOp getConstant(mlir::Location loc, mlir::TypedAttr attr) {
+    return create<cir::ConstantOp>(loc, attr.getType(), attr);
+  }
+
   cir::ConstantOp getBool(bool state, mlir::Location loc) {
     return create<cir::ConstantOp>(loc, getBoolTy(), getCIRBoolAttr(state));
   }
diff --git a/clang/include/clang/CIR/Dialect/CMakeLists.txt b/clang/include/clang/CIR/Dialect/CMakeLists.txt
index f33061b2d87cf..3d4e6586e1c62 100644
--- a/clang/include/clang/CIR/Dialect/CMakeLists.txt
+++ b/clang/include/clang/CIR/Dialect/CMakeLists.txt
@@ -1 +1,7 @@
 add_subdirectory(IR)
+
+set(LLVM_TARGET_DEFINITIONS Passes.td)
+mlir_tablegen(Passes.h.inc -gen-pass-decls -name CIR)
+mlir_tablegen(Passes.capi.h.inc -gen-pass-capi-header --prefix CIR)
+mlir_tablegen(Passes.capi.cpp.inc -gen-pass-capi-impl --prefix CIR)
+add_public_tablegen_target(MLIRCIRPassIncGen)
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
index ece04c225e322..7b3741de29075 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
@@ -54,6 +54,20 @@ def CIR_BoolAttr : CIR_Attr<"Bool", "bool", [TypedAttrInterface]> {
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// ZeroAttr
+//===----------------------------------------------------------------------===//
+
+def ZeroAttr : CIR_Attr<"Zero", "zero", [TypedAttrInterface]> {
+  let summary = "Attribute to represent zero initialization";
+  let description = [{
+    The ZeroAttr is used to indicate zero initialization on structs.
+  }];
+
+  let parameters = (ins AttributeSelfTypeParameter<"">:$type);
+  let assemblyFormat = [{}];
+}
+
 //===----------------------------------------------------------------------===//
 // UndefAttr
 //===----------------------------------------------------------------------===//
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index e2ab50c78ec2d..77c43e5ace64a 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -428,6 +428,46 @@ def ScopeOp : CIR_Op<"scope", [
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// BrOp
+//===----------------------------------------------------------------------===//
+
+def BrOp : CIR_Op<"br",
+      [DeclareOpInterfaceMethods<BranchOpInterface, ["getSuccessorForOperands"]>,
+     Pure, Terminator]> {
+  let summary = "Unconditional branch";
+  let description = [{
+    The `cir.br` branches unconditionally to a block. Used to represent C/C++
+    goto's and general block branching.
+
+    Note that for source level `goto`'s crossing scope boundaries, those are
+    usually represented with the "symbolic" `cir.goto` operation.
+
+    Example:
+
+    ```mlir
+      ...
+        cir.br ^bb3
+      ^bb3:
+        cir.return
+    ```
+  }];
+
+  let builders = [
+    OpBuilder<(ins "mlir::Block *":$dest,
+              CArg<"mlir::ValueRange", "{}">:$destOperands), [{
+      $_state.addSuccessors(dest);
+      $_state.addOperands(destOperands);
+    }]>
+  ];
+
+  let arguments = (ins Variadic<CIR_AnyType>:$destOperands);
+  let successors = (successor AnySuccessor:$dest);
+  let assemblyFormat = [{
+    $dest (`(` $destOperands^ `:` type($destOperands) `)`)? attr-dict
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // GlobalOp
 //===----------------------------------------------------------------------===//
diff --git a/clang/include/clang/CIR/Dialect/Passes.h b/clang/include/clang/CIR/Dialect/Passes.h
new file mode 100644
index 0000000000000..b691849dfc563
--- /dev/null
+++ b/clang/include/clang/CIR/Dialect/Passes.h
@@ -0,0 +1,39 @@
+//===- Passes.h - CIR pass entry points -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This header file defines prototypes that expose pass constructors.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_CIR_DIALECT_PASSES_H
+#define CLANG_CIR_DIALECT_PASSES_H
+
+#include "mlir/Pass/Pass.h"
+
+namespace clang {
+class ASTContext;
+}
+namespace mlir {
+
+std::unique_ptr<Pass> createCIRFlattenCFGPass();
+
+void populateCIRPreLoweringPasses(mlir::OpPassManager &pm);
+
+//===----------------------------------------------------------------------===//
+// Registration
+//===----------------------------------------------------------------------===//
+
+void registerCIRDialectTranslation(mlir::MLIRContext &context);
+
+/// Generate the code for registering passes.
+#define GEN_PASS_REGISTRATION
+#include "clang/CIR/Dialect/Passes.h.inc"
+
+} // namespace mlir
+
+#endif // CLANG_CIR_DIALECT_PASSES_H
diff --git a/clang/include/clang/CIR/Dialect/Passes.td b/clang/include/clang/CIR/Dialect/Passes.td
new file mode 100644
index 0000000000000..84b7ecba2630a
--- /dev/null
+++ b/clang/include/clang/CIR/Dialect/Passes.td
@@ -0,0 +1,28 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_CIR_DIALECT_PASSES_TD
+#define CLANG_CIR_DIALECT_PASSES_TD
+
+include "mlir/Pass/PassBase.td"
+
+def CIRFlattenCFG : Pass<"cir-flatten-cfg"> {
+  let summary = "Produces flatten CFG";
+  let description = [{
+    This pass transforms CIR by inlining all the nested regions. Thus,
+    the following conditions are true after the pass applied:
+    - there are no nested regions in any function body
+    - all the blocks in a function belong to the parent region
+    In other words, this pass removes such CIR operations like IfOp, LoopOp,
+    ScopeOp and etc. and produces a flat CIR.
+  }];
+  let constructor = "mlir::createCIRFlattenCFGPass()";
+  let dependentDialects = ["cir::CIRDialect"];
+}
+
+#endif // CLANG_CIR_DIALECT_PASSES_TD
diff --git a/clang/include/clang/CIR/MissingFeatures.h b/clang/include/clang/CIR/MissingFeatures.h
index 6f845b7689e51..ddfe654009644 100644
--- a/clang/include/clang/CIR/MissingFeatures.h
+++ b/clang/include/clang/CIR/MissingFeatures.h
@@ -41,16 +41,17 @@ struct MissingFeatures {
   static bool supportComdat() { return false; }
 
   // Load/store attributes
-  static bool opLoadThreadLocal() { return false; }
+  static bool opLoadStoreThreadLocal() { return false; }
   static bool opLoadEmitScalarRangeCheck() { return false; }
   static bool opLoadBooleanRepresentation() { return false; }
   static bool opLoadStoreTbaa() { return false; }
   static bool opLoadStoreMemOrder() { return false; }
   static bool opLoadStoreVolatile() { return false; }
   static bool opLoadStoreAlignment() { return false; }
+  static bool opLoadStoreAtomic() { return false; }
+  static bool opLoadStoreObjC() { return false; }
 
   // AllocaOp handling
-  static bool opAllocaVarDeclContext() { return false; }
   static bool opAllocaStaticLocal() { return false; }
   static bool opAllocaNonGC() { return false; }
   static bool opAllocaImpreciseLifetime() { return false; }
@@ -61,6 +62,7 @@ struct MissingFeatures {
   static bool opAllocaReference() { return false; }
   static bool opAllocaAnnotations() { return false; }
   static bool opAllocaDynAllocSize() { return false; }
+  static bool opAllocaCaptureByInit() { return false; }
 
   // FuncOp handling
   static bool opFuncOpenCLKernelMetadata() { return false; }
@@ -76,6 +78,11 @@ struct MissingFeatures {
   static bool constructABIArgDirectExtend() { return false; }
   static bool opGlobalViewAttr() { return false; }
   static bool lowerModeOptLevel() { return false; }
+  static bool opTBAA() { return false; }
+  static bool objCLifetime() { return false; }
+  static bool emitNullabilityCheck() { return false; }
+  static bool astVarDeclInterface() { return false; }
+  static bool stackSaveOp() { return false; }
 };
 
 } // namespace cir
diff --git a/clang/include/clang/Driver/Action.h b/clang/include/clang/Driver/Action.h
index e5307b0fcedd5..92bb19314e3d6 100644
--- a/clang/include/clang/Driver/Action.h
+++ b/clang/include/clang/Driver/Action.h
@@ -75,9 +75,10 @@ class Action {
     LinkerWrapperJobClass,
     StaticLibJobClass,
     BinaryAnalyzeJobClass,
+    BinaryTranslatorJobClass,
 
     JobClassFirst = PreprocessJobClass,
-    JobClassLast = BinaryAnalyzeJobClass
+    JobClassLast = BinaryTranslatorJobClass
   };
 
   // The offloading kind determines if this action is binded to a particular
@@ -675,6 +676,17 @@ class BinaryAnalyzeJobAction : public JobAction {
   }
 };
 
+class BinaryTranslatorJobAction : public JobAction {
+  void anchor() override;
+
+public:
+  BinaryTranslatorJobAction(Action *Input, types::ID Type);
+
+  static bool classof(const Action *A) {
+    return A->getKind() == BinaryTranslatorJobClass;
+  }
+};
+
 } // namespace driver
 } // namespace clang
 
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index d0414aba35209..e69cd6b833c3a 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -876,7 +876,7 @@ def Wa_COMMA : CommaJoined<["-"], "Wa,">,
   MetaVarName<"<arg>">;
 def warning_suppression_mappings_EQ : Joined<["--"],
   "warning-suppression-mappings=">, Group<Diag_Group>,
-  HelpText<"File containing diagnostic suppresion mappings. See user manual "
+  HelpText<"File containing diagnostic suppression mappings. See user manual "
   "for file format.">, Visibility<[ClangOption, CC1Option]>;
 def Wall : Flag<["-"], "Wall">, Group<W_Group>, Flags<[HelpHidden]>,
   Visibility<[ClangOption, CC1Option, FlangOption]>;
@@ -941,7 +941,7 @@ def Xarch__
   target matches the specified architecture. This can be used with the target
   CPU, triple architecture, or offloading host and device. It is most useful
   for separating behavior undesirable on one of the targets when combining many
-  compilation jobs, as is commong with offloading. For example, -Xarch_x86_64,
+  compilation jobs, as is common with offloading. For example, -Xarch_x86_64,
   -Xarch_gfx90a, and -Xarch_device are all valid selectors. -Xarch_device will
   forward the argument to the offloading device while -Xarch_host will target
   the host system, which can be used to suppress incompatible GPU arguments.}]>,
@@ -1678,7 +1678,7 @@ def fsample_profile_use_profi : Flag<["-"], "fsample-profile-use-profi">,
     HelpText<"Use profi to infer block and edge counts">,
     DocBrief<[{Infer block and edge counts. If the profiles have errors or missing
                blocks caused by sampling, profile inference (profi) can convert
-               basic block counts to branch probabilites to fix them by extended
+               basic block counts to branch probabilities to fix them by extended
                and re-engineered classic MCMF (min-cost max-flow) approach.}]>;
 def fno_profile_sample_accurate : Flag<["-"], "fno-profile-sample-accurate">, Group<f_Group>;
 def fno_auto_profile : Flag<["-"], "fno-auto-profile">, Group<f_Group>,
@@ -4725,9 +4725,9 @@ def mno_long_calls : Flag<["-"], "mno-long-calls">, Group<m_Group>,
   HelpText<"Restore the default behaviour of not generating long calls">;
 } // let Flags = [TargetSpecific]
 def mexecute_only : Flag<["-"], "mexecute-only">, Group<m_arm_Features_Group>,
-  HelpText<"Disallow generation of data access to code sections (ARM only)">;
+  HelpText<"Disallow generation of data access to code sections (AArch64/ARM only)">;
 def mno_execute_only : Flag<["-"], "mno-execute-only">, Group<m_arm_Features_Group>,
-  HelpText<"Allow generation of data access to code sections (ARM only)">;
+  HelpText<"Allow generation of data access to code sections (AArch64/ARM only)">;
 let Flags = [TargetSpecific] in {
 def mtp_mode_EQ : Joined<["-"], "mtp=">, Group<m_arm_Features_Group>, Values<"soft,cp15,tpidrurw,tpidruro,tpidrprw,el0,el1,el2,el3,tpidr_el0,tpidr_el1,tpidr_el2,tpidr_el3,tpidrro_el0,auto">,
   HelpText<"Thread pointer access method. "
@@ -9085,6 +9085,7 @@ def : Option<["/", "-"], "Qembed_debug", KIND_FLAG>, Group<dxc_Group>,
   HelpText<"Embed PDB in shader container (ignored)">;
 def spirv : DXCFlag<"spirv">,
   HelpText<"Generate SPIR-V code">;
+def metal : DXCFlag<"metal">, HelpText<"Generate Metal library">;
 def fspv_target_env_EQ : Joined<["-"], "fspv-target-env=">, Group<dxc_Group>,
   HelpText<"Specify the target environment">,
   Values<"vulkan1.2, vulkan1.3">;
diff --git a/clang/include/clang/Frontend/CompilerInvocation.h b/clang/include/clang/Frontend/CompilerInvocation.h
index 9daa0a1ecf948..1e4d2da86c2be 100644
--- a/clang/include/clang/Frontend/CompilerInvocation.h
+++ b/clang/include/clang/Frontend/CompilerInvocation.h
@@ -269,7 +269,6 @@ class CompilerInvocation : public CompilerInvocationBase {
   /// @{
   using CompilerInvocationBase::LangOpts;
   using CompilerInvocationBase::TargetOpts;
-  using CompilerInvocationBase::DiagnosticOpts;
   std::shared_ptr<HeaderSearchOptions> getHeaderSearchOptsPtr() {
     return HSOpts;
   }
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 049156e266c70..1c8caf55a546d 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3783,8 +3783,9 @@ class Parser : public CodeCompletionHandler {
   OpenACCWaitParseInfo ParseOpenACCWaitArgument(SourceLocation Loc,
                                                 bool IsDirective);
   /// Parses the clause of the 'bind' argument, which can be a string literal or
-  /// an ID expression.
-  ExprResult ParseOpenACCBindClauseArgument();
+  /// an identifier.
+  std::variant<std::monostate, StringLiteral *, IdentifierInfo *>
+  ParseOpenACCBindClauseArgument();
 
   /// A type to represent the state of parsing after an attempt to parse an
   /// OpenACC int-expr. This is useful to determine whether an int-expr list can
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 08621e633b55c..9ac26d8728446 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -4008,7 +4008,7 @@ class Sema final : public SemaBase {
   /// Perform ODR-like check for C/ObjC when merging tag types from modules.
   /// Differently from C++, actually parse the body and reject / error out
   /// in case of a structural mismatch.
-  bool ActOnDuplicateDefinition(Decl *Prev, SkipBodyInfo &SkipBody);
+  bool ActOnDuplicateDefinition(Scope *S, Decl *Prev, SkipBodyInfo &SkipBody);
 
   typedef void *SkippedDefinitionContext;
 
@@ -4132,6 +4132,12 @@ class Sema final : public SemaBase {
   void MergeTypedefNameDecl(Scope *S, TypedefNameDecl *New,
                             LookupResult &OldDecls);
 
+  /// CleanupMergedEnum - We have just merged the decl 'New' by making another
+  /// definition visible.
+  /// This method performs any necessary cleanup on the parser state to discard
+  /// child nodes from newly parsed decl we are retiring.
+  void CleanupMergedEnum(Scope *S, Decl *New);
+
   /// MergeFunctionDecl - We just parsed a function 'New' from
   /// declarator D which has the same name and scope as a previous
   /// declaration 'Old'.  Figure out how to resolve this situation,
@@ -11357,14 +11363,16 @@ class Sema final : public SemaBase {
 
   /// The context in which we are checking a template parameter list.
   enum TemplateParamListContext {
-    TPC_ClassTemplate,
-    TPC_VarTemplate,
+    // For this context, Class, Variable, TypeAlias, and non-pack Template
+    // Template Parameters are treated uniformly.
+    TPC_Other,
+
     TPC_FunctionTemplate,
     TPC_ClassTemplateMember,
     TPC_FriendClassTemplate,
     TPC_FriendFunctionTemplate,
     TPC_FriendFunctionTemplateDefinition,
-    TPC_TypeAliasTemplate
+    TPC_TemplateTemplateParameterPack,
   };
 
   /// Checks the validity of a template parameter list, possibly
@@ -11827,7 +11835,7 @@ class Sema final : public SemaBase {
                                  bool *ConstraintsNotSatisfied = nullptr);
 
   bool CheckTemplateTypeArgument(
-      TemplateArgumentLoc &Arg,
+      TemplateTypeParmDecl *Param, TemplateArgumentLoc &Arg,
       SmallVectorImpl<TemplateArgument> &SugaredConverted,
       SmallVectorImpl<TemplateArgument> &CanonicalConverted);
 
@@ -11863,13 +11871,9 @@ class Sema final : public SemaBase {
                                      bool PartialOrdering,
                                      bool *StrictPackMatch);
 
-  /// Print the given named declaration to a string,
-  /// using the current PrintingPolicy, except that
-  /// TerseOutput will always be set.
-  SmallString<128> toTerseString(const NamedDecl &D) const;
-
   void NoteTemplateLocation(const NamedDecl &Decl,
                             std::optional<SourceRange> ParamRange = {});
+  void NoteTemplateParameterLocation(const NamedDecl &Decl);
 
   /// Given a non-type template argument that refers to a
   /// declaration and the type of its corresponding non-type template
@@ -11984,13 +11988,15 @@ class Sema final : public SemaBase {
   bool TemplateParameterListsAreEqual(
       const TemplateCompareNewDeclInfo &NewInstFrom, TemplateParameterList *New,
       const NamedDecl *OldInstFrom, TemplateParameterList *Old, bool Complain,
-      TemplateParameterListEqualKind Kind);
+      TemplateParameterListEqualKind Kind,
+      SourceLocation TemplateArgLoc = SourceLocation());
 
-  bool TemplateParameterListsAreEqual(TemplateParameterList *New,
-                                      TemplateParameterList *Old, bool Complain,
-                                      TemplateParameterListEqualKind Kind) {
+  bool TemplateParameterListsAreEqual(
+      TemplateParameterList *New, TemplateParameterList *Old, bool Complain,
+      TemplateParameterListEqualKind Kind,
+      SourceLocation TemplateArgLoc = SourceLocation()) {
     return TemplateParameterListsAreEqual(nullptr, New, nullptr, Old, Complain,
-                                          Kind);
+                                          Kind, TemplateArgLoc);
   }
 
   /// Check whether a template can be declared within this scope.
@@ -12870,11 +12876,6 @@ class Sema final : public SemaBase {
 
       /// We are performing partial ordering for template template parameters.
       PartialOrderingTTP,
-
-      /// We are Checking a Template Parameter, so for any diagnostics which
-      /// occur in this scope, we will add a context note which points to this
-      /// template parameter.
-      CheckTemplateParameter,
     } Kind;
 
     /// Was the enclosing context a non-instantiation SFINAE context?
@@ -13102,11 +13103,6 @@ class Sema final : public SemaBase {
                           PartialOrderingTTP, TemplateDecl *PArg,
                           SourceRange InstantiationRange = SourceRange());
 
-    struct CheckTemplateParameter {};
-    /// \brief Note that we are checking a template parameter.
-    InstantiatingTemplate(Sema &SemaRef, CheckTemplateParameter,
-                          NamedDecl *Param);
-
     /// Note that we have finished instantiating this template.
     void Clear();
 
@@ -13140,13 +13136,6 @@ class Sema final : public SemaBase {
     InstantiatingTemplate &operator=(const InstantiatingTemplate &) = delete;
   };
 
-  /// For any diagnostics which occur within its scope, adds a context note
-  /// pointing to the declaration of the template parameter.
-  struct CheckTemplateParameterRAII : InstantiatingTemplate {
-    CheckTemplateParameterRAII(Sema &S, NamedDecl *Param)
-        : InstantiatingTemplate(S, CheckTemplateParameter(), Param) {}
-  };
-
   bool SubstTemplateArgument(const TemplateArgumentLoc &Input,
                              const MultiLevelTemplateArgumentList &TemplateArgs,
                              TemplateArgumentLoc &Output,
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index 748dcdd251a92..358dec4cadb72 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -261,10 +261,14 @@ class SemaOpenACC : public SemaBase {
       SmallVector<OpenACCGangKind> GangKinds;
       SmallVector<Expr *> IntExprs;
     };
+    struct BindDetails {
+      std::variant<std::monostate, clang::StringLiteral *, IdentifierInfo *>
+          Argument;
+    };
 
     std::variant<std::monostate, DefaultDetails, ConditionDetails,
                  IntExprDetails, VarListDetails, WaitDetails, DeviceTypeDetails,
-                 ReductionDetails, CollapseDetails, GangDetails>
+                 ReductionDetails, CollapseDetails, GangDetails, BindDetails>
         Details = std::monostate{};
 
   public:
@@ -468,6 +472,13 @@ class SemaOpenACC : public SemaBase {
       return std::get<DeviceTypeDetails>(Details).Archs;
     }
 
+    std::variant<std::monostate, clang::StringLiteral *, IdentifierInfo *>
+    getBindDetails() const {
+      assert(ClauseKind == OpenACCClauseKind::Bind &&
+             "Only 'bind' has bind details");
+      return std::get<BindDetails>(Details).Argument;
+    }
+
     void setLParenLoc(SourceLocation EndLoc) { LParenLoc = EndLoc; }
     void setEndLoc(SourceLocation EndLoc) { ClauseRange.setEnd(EndLoc); }
 
@@ -652,6 +663,14 @@ class SemaOpenACC : public SemaBase {
              "Only 'collapse' has collapse details");
       Details = CollapseDetails{IsForce, LoopCount};
     }
+
+    void setBindDetails(
+        std::variant<std::monostate, clang::StringLiteral *, IdentifierInfo *>
+            Arg) {
+      assert(ClauseKind == OpenACCClauseKind::Bind &&
+             "Only 'bind' has bind details");
+      Details = BindDetails{Arg};
+    }
   };
 
   SemaOpenACC(Sema &S);
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 64f0cfa0676af..7fc9b64f0ae65 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -1112,6 +1112,10 @@ class SemaOpenMP : public SemaBase {
       OpenMPAtomicDefaultMemOrderClauseKind Kind, SourceLocation KindLoc,
       SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation EndLoc);
 
+  /// Called on well-formed 'self_maps' clause.
+  OMPClause *ActOnOpenMPSelfMapsClause(SourceLocation StartLoc,
+                                       SourceLocation EndLoc);
+
   /// Called on well-formed 'at' clause.
   OMPClause *ActOnOpenMPAtClause(OpenMPAtClauseKind Kind,
                                  SourceLocation KindLoc,
diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h
index ad291d0948b57..7371a679988e9 100644
--- a/clang/include/clang/Serialization/ASTWriter.h
+++ b/clang/include/clang/Serialization/ASTWriter.h
@@ -571,7 +571,7 @@ class ASTWriter : public ASTDeserializationListener,
   std::pair<ASTFileSignature, ASTFileSignature> createSignature() const;
   ASTFileSignature createSignatureForNamedModule() const;
 
-  void WriteInputFiles(SourceManager &SourceMgr, HeaderSearchOptions &HSOpts);
+  void WriteInputFiles(SourceManager &SourceMgr);
   void WriteSourceManagerBlock(SourceManager &SourceMgr);
   void WritePreprocessor(const Preprocessor &PP, bool IsModule);
   void WriteHeaderSearch(const HeaderSearch &HS);
diff --git a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
index c8895db914d13..7ee2ec1aaf09f 100644
--- a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
+++ b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
@@ -385,7 +385,7 @@ def TrustReturnsNonnullChecker : Checker<"TrustReturnsNonnull">,
 } // end "apiModeling"
 
 //===----------------------------------------------------------------------===//
-// Evaluate "builtin" functions.
+// Evaluate "builtin" functions and assumptions.
 //===----------------------------------------------------------------------===//
 
 let ParentPackage = CoreBuiltin in {
@@ -399,6 +399,10 @@ def BuiltinFunctionChecker : Checker<"BuiltinFunctions">,
   HelpText<"Evaluate compiler builtin functions (e.g., alloca())">,
   Documentation<NotDocumented>;
 
+def AssumeModeling : Checker<"AssumeModeling">,
+  HelpText<"Model compiler builtin assume functions and the assume attribute">,
+  Documentation<NotDocumented>;
+
 } // end "core.builtin"
 
 //===----------------------------------------------------------------------===//
@@ -1765,6 +1769,14 @@ def NoUncheckedPtrMemberChecker : Checker<"NoUncheckedPtrMemberChecker">,
   HelpText<"Check for no unchecked member variables.">,
   Documentation<HasDocumentation>;
 
+def NoUnretainedMemberChecker : Checker<"NoUnretainedMemberChecker">,
+  HelpText<"Check for no unretained member variables.">,
+  Documentation<HasDocumentation>;
+
+def UnretainedLambdaCapturesChecker : Checker<"UnretainedLambdaCapturesChecker">,
+  HelpText<"Check unretained lambda captures.">,
+  Documentation<HasDocumentation>;
+
 def UncountedCallArgsChecker : Checker<"UncountedCallArgsChecker">,
   HelpText<"Check uncounted call arguments.">,
   Documentation<HasDocumentation>;
diff --git a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugType.h b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugType.h
index e50afd6d0da7e..97cf7eda0ae2f 100644
--- a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugType.h
+++ b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugType.h
@@ -41,19 +41,19 @@ class BugType {
         Checker(nullptr), SuppressOnSink(SuppressOnSink) {}
   BugType(const CheckerBase *Checker, StringRef Desc,
           StringRef Cat = categories::LogicError, bool SuppressOnSink = false)
-      : CheckerName(Checker->getCheckerName()), Description(Desc),
-        Category(Cat), Checker(Checker), SuppressOnSink(SuppressOnSink) {}
+      : CheckerName(), Description(Desc), Category(Cat), Checker(Checker),
+        SuppressOnSink(SuppressOnSink) {}
   virtual ~BugType() = default;
 
   StringRef getDescription() const { return Description; }
   StringRef getCategory() const { return Category; }
   StringRef getCheckerName() const {
-    // FIXME: This is a workaround to ensure that the correct checerk name is
+    // FIXME: This is a workaround to ensure that the correct checker name is
     // used. The checker names are set after the constructors are run.
     // In case the BugType object is initialized in the checker's ctor
     // the CheckerName field will be empty. To circumvent this problem we use
     // CheckerBase whenever it is possible.
-    StringRef Ret = Checker ? Checker->getCheckerName() : CheckerName;
+    StringRef Ret = Checker ? Checker->getName() : CheckerName;
     assert(!Ret.empty() && "Checker name is not set properly.");
     return Ret;
   }
diff --git a/clang/include/clang/StaticAnalyzer/Core/Checker.h b/clang/include/clang/StaticAnalyzer/Core/Checker.h
index 2ec54a837c42c..4d9b33cc559b8 100644
--- a/clang/include/clang/StaticAnalyzer/Core/Checker.h
+++ b/clang/include/clang/StaticAnalyzer/Core/Checker.h
@@ -490,7 +490,7 @@ class CheckerBase : public ProgramPointTag {
 
 public:
   StringRef getTagDescription() const override;
-  CheckerNameRef getCheckerName() const;
+  CheckerNameRef getName() const;
 
   /// See CheckerManager::runCheckersForPrintState.
   virtual void printState(raw_ostream &Out, ProgramStateRef State,
diff --git a/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h b/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h
index 7d3120b5395c4..929ce96824e95 100644
--- a/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h
+++ b/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h
@@ -113,7 +113,6 @@ class CheckerNameRef {
 public:
   CheckerNameRef() = default;
 
-  StringRef getName() const { return Name; }
   operator StringRef() const { return Name; }
 };
 
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h
index 168983fd5cb68..bb33a6912bec7 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 //
 //  This file defines CheckerContext that provides contextual info for
-// path-sensitive checkers.
+//  path-sensitive checkers.
 //
 //===----------------------------------------------------------------------===//
 
@@ -152,7 +152,7 @@ class CheckerContext {
   }
 
   /// Returns true if the value of \p E is greater than or equal to \p
-  /// Val under unsigned comparison
+  /// Val under unsigned comparison.
   bool isGreaterOrEqual(const Expr *E, unsigned long long Val);
 
   /// Returns true if the value of \p E is negative.
@@ -392,7 +392,7 @@ class CheckerContext {
   /// hardened variant that's not yet covered by it.
   static bool isHardenedVariantOf(const FunctionDecl *FD, StringRef Name);
 
-  /// Depending on wither the location corresponds to a macro, return
+  /// Depending on whether the location corresponds to a macro, return
   /// either the macro name or the token spelling.
   ///
   /// This could be useful when checkers' logic depends on whether a function
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h
index 80b79fd4e928f..b2b4e8729af25 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h
@@ -156,7 +156,7 @@ class CoreEngine {
   void dispatchWorkItem(ExplodedNode* Pred, ProgramPoint Loc,
                         const WorkListUnit& WU);
 
-  // Functions for external checking of whether we have unfinished work
+  // Functions for external checking of whether we have unfinished work.
   bool wasBlockAborted() const { return !blocksAborted.empty(); }
   bool wasBlocksExhausted() const { return !blocksExhausted.empty(); }
   bool hasWorkRemaining() const { return wasBlocksExhausted() ||
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h
index 2fb05ac46e8fa..967b25e5696f6 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 //
 //  This file defines the template classes ExplodedNode and ExplodedGraph,
-//  which represent a path-sensitive, intra-procedural "exploded graph."
+//  which represent a path-sensitive, intra-procedural "exploded graph".
 //  See "Precise interprocedural dataflow analysis via graph reachability"
 //  by Reps, Horwitz, and Sagiv
 //  (http://portal.acm.org/citation.cfm?id=199462) for the definition of an
@@ -426,8 +426,8 @@ class ExplodedGraph {
   ///
   /// \param Nodes The nodes which must appear in the final graph. Presumably
   ///              these are end-of-path nodes (i.e. they have no successors).
-  /// \param[out] ForwardMap A optional map from nodes in this graph to nodes in
-  ///                        the returned graph.
+  /// \param[out] ForwardMap An optional map from nodes in this graph to nodes
+  ///                        in the returned graph.
   /// \param[out] InverseMap An optional map from nodes in the returned graph to
   ///                        nodes in this graph.
   /// \returns The trimmed graph
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
index 804fc74b009df..5f855251b3cde 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
@@ -429,7 +429,7 @@ class ExprEngine {
 
   DataTag::Factory &getDataTags() { return Engine.getDataTags(); }
 
-  // Functions for external checking of whether we have unfinished work
+  // Functions for external checking of whether we have unfinished work.
   bool wasBlocksExhausted() const { return Engine.wasBlocksExhausted(); }
   bool hasEmptyWorkList() const { return !Engine.getWorkList()->hasWork(); }
   bool hasWorkRemaining() const { return Engine.hasWorkRemaining(); }
@@ -498,11 +498,11 @@ class ExprEngine {
   void VisitInitListExpr(const InitListExpr *E, ExplodedNode *Pred,
                          ExplodedNodeSet &Dst);
 
-  /// VisitAttributedStmt - Transfer function logic for AttributedStmt
+  /// VisitAttributedStmt - Transfer function logic for AttributedStmt.
   void VisitAttributedStmt(const AttributedStmt *A, ExplodedNode *Pred,
                            ExplodedNodeSet &Dst);
 
-  /// VisitLogicalExpr - Transfer function logic for '&&', '||'
+  /// VisitLogicalExpr - Transfer function logic for '&&', '||'.
   void VisitLogicalExpr(const BinaryOperator* B, ExplodedNode *Pred,
                         ExplodedNodeSet &Dst);
 
@@ -510,7 +510,7 @@ class ExprEngine {
   void VisitMemberExpr(const MemberExpr *M, ExplodedNode *Pred,
                        ExplodedNodeSet &Dst);
 
-  /// VisitAtomicExpr - Transfer function for builtin atomic expressions
+  /// VisitAtomicExpr - Transfer function for builtin atomic expressions.
   void VisitAtomicExpr(const AtomicExpr *E, ExplodedNode *Pred,
                        ExplodedNodeSet &Dst);
 
@@ -725,7 +725,7 @@ class ExprEngine {
   /// For `int arr[4]` this index can be 0,1,2,3.
   /// For `int arr2[3][3]` this index can be 0,1,...,7,8.
   /// A multi-dimensional array is also a continuous memory location in a
-  /// row major order, so for arr[0][0] Idx is 0 and for arr[2][2] Idx is 8.
+  /// row major order, so for arr[0][0] Idx is 0 and for arr[3][3] Idx is 8.
   SVal computeObjectUnderConstruction(const Expr *E, ProgramStateRef State,
                                       const NodeBuilderContext *BldrCtx,
                                       const LocationContext *LCtx,
@@ -806,7 +806,7 @@ class ExprEngine {
   /// should inline, just by looking at the declaration of the function.
   bool mayInlineDecl(AnalysisDeclContext *ADC) const;
 
-  /// Checks our policies and decides weither the given call should be inlined.
+  /// Checks our policies and decides whether the given call should be inlined.
   bool shouldInlineCall(const CallEvent &Call, const Decl *D,
                         const ExplodedNode *Pred,
                         const EvalCallOptions &CallOpts = {});
@@ -902,8 +902,8 @@ class ExprEngine {
                          ExplodedNodeSet &Dst);
 
 public:
-  /// Note whether this loop has any more iteratios to model. These methods are
-  /// essentially an interface for a GDM trait. Further reading in
+  /// Note whether this loop has any more iterations to model. These methods
+  //  are essentially an interface for a GDM trait. Further reading in
   /// ExprEngine::VisitObjCForCollectionStmt().
   [[nodiscard]] static ProgramStateRef
   setWhetherHasMoreIteration(ProgramStateRef State,
@@ -967,7 +967,7 @@ class ExprEngine {
                              const ConstructionContextItem &Item,
                              const LocationContext *LC, SVal V);
 
-  /// Mark the object sa fully constructed, cleaning up the state trait
+  /// Mark the object as fully constructed, cleaning up the state trait
   /// that tracks objects under construction.
   static ProgramStateRef
   finishObjectConstruction(ProgramStateRef State,
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h
index 208b32e376883..4d66e086a2c2c 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file defines the state of the program along the analysisa path.
+// This file defines the state of the program along the analysis path.
 //
 //===----------------------------------------------------------------------===//
 
@@ -161,7 +161,6 @@ class ProgramState : public llvm::FoldingSetNode {
   ///  is a mapping from locations to values.
   Store getStore() const { return store; }
 
-
   /// getGDM - Return the generic data map associated with this state.
   GenericDataMap getGDM() const { return GDM; }
 
@@ -382,7 +381,7 @@ class ProgramState : public llvm::FoldingSetNode {
   /// Returns UnknownVal() if none found.
   SVal getSVal(Loc LV, QualType T = QualType()) const;
 
-  /// Returns the "raw" SVal bound to LV before any value simplfication.
+  /// Returns the "raw" SVal bound to LV before any value simplification.
   SVal getRawSVal(Loc LV, QualType T= QualType()) const;
 
   /// Return the value bound to the specified location.
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConv.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConv.h
index fcc9c02999b3b..580b49a38dc72 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConv.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConv.h
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-//  This file defines a set of functions to create SMT expressions
+//  This file defines a set of functions to create SMT expressions.
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index fce1c34897da7..55022a4bc000a 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -2005,8 +2005,9 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
   case Type::Vector: {
     const auto *VT = cast<VectorType>(T);
     TypeInfo EltInfo = getTypeInfo(VT->getElementType());
-    Width = VT->isExtVectorBoolType() ? VT->getNumElements()
-                                      : EltInfo.Width * VT->getNumElements();
+    Width = VT->isPackedVectorBoolType(*this)
+                ? VT->getNumElements()
+                : EltInfo.Width * VT->getNumElements();
     // Enforce at least byte size and alignment.
     Width = std::max<unsigned>(8, Width);
     Align = std::max<unsigned>(8, Width);
diff --git a/clang/lib/AST/ASTDiagnostic.cpp b/clang/lib/AST/ASTDiagnostic.cpp
index 7b873ee9833b3..b4e7360e126fb 100644
--- a/clang/lib/AST/ASTDiagnostic.cpp
+++ b/clang/lib/AST/ASTDiagnostic.cpp
@@ -461,8 +461,9 @@ void clang::FormatASTNodeDiagnosticArgument(
     }
     case DiagnosticsEngine::ak_nestednamespec: {
       NestedNameSpecifier *NNS = reinterpret_cast<NestedNameSpecifier*>(Val);
-      NNS->print(OS, Context.getPrintingPolicy());
-      NeedQuotes = false;
+      NNS->print(OS, Context.getPrintingPolicy(),
+                 /*ResolveTemplateArguments=*/false,
+                 /*PrintFinalScopeResOp=*/false);
       break;
     }
     case DiagnosticsEngine::ak_declcontext: {
diff --git a/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp b/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
index 5bd1b73133d65..4162b55070da9 100644
--- a/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
+++ b/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
@@ -367,6 +367,16 @@ bool ByteCodeEmitter::fallthrough(const LabelTy &Label) {
   return true;
 }
 
+bool ByteCodeEmitter::speculate(const CallExpr *E, const LabelTy &EndLabel) {
+  const Expr *Arg = E->getArg(0);
+  PrimType T = Ctx.classify(Arg->getType()).value_or(PT_Ptr);
+  if (!this->emitBCP(getOffset(EndLabel), T, E))
+    return false;
+  if (!this->visit(Arg))
+    return false;
+  return true;
+}
+
 //===----------------------------------------------------------------------===//
 // Opcode emitters
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/AST/ByteCode/ByteCodeEmitter.h b/clang/lib/AST/ByteCode/ByteCodeEmitter.h
index ac728830527a2..64670c32cbcf6 100644
--- a/clang/lib/AST/ByteCode/ByteCodeEmitter.h
+++ b/clang/lib/AST/ByteCode/ByteCodeEmitter.h
@@ -48,12 +48,16 @@ class ByteCodeEmitter {
   virtual bool visitFunc(const FunctionDecl *E) = 0;
   virtual bool visitExpr(const Expr *E, bool DestroyToplevelScope) = 0;
   virtual bool visitDeclAndReturn(const VarDecl *E, bool ConstantContext) = 0;
+  virtual bool visit(const Expr *E) = 0;
+  virtual bool emitBool(bool V, const Expr *E) = 0;
 
   /// Emits jumps.
   bool jumpTrue(const LabelTy &Label);
   bool jumpFalse(const LabelTy &Label);
   bool jump(const LabelTy &Label);
   bool fallthrough(const LabelTy &Label);
+  /// Speculative execution.
+  bool speculate(const CallExpr *E, const LabelTy &EndLabel);
 
   /// We're always emitting bytecode.
   bool isActive() const { return true; }
diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index 281fb7e14a57d..b9f88230007b5 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -25,6 +25,16 @@ using APSInt = llvm::APSInt;
 namespace clang {
 namespace interp {
 
+static std::optional<bool> getBoolValue(const Expr *E) {
+  if (const auto *CE = dyn_cast_if_present<ConstantExpr>(E);
+      CE && CE->hasAPValueResult() &&
+      CE->getResultAPValueKind() == APValue::ValueKind::Int) {
+    return CE->getResultAsAPSInt().getBoolValue();
+  }
+
+  return std::nullopt;
+}
+
 /// Scope used to handle temporaries in toplevel variable declarations.
 template <class Emitter> class DeclScope final : public LocalScope<Emitter> {
 public:
@@ -2286,39 +2296,49 @@ bool Compiler<Emitter>::VisitAbstractConditionalOperator(
   const Expr *TrueExpr = E->getTrueExpr();
   const Expr *FalseExpr = E->getFalseExpr();
 
+  auto visitChildExpr = [&](const Expr *E) -> bool {
+    LocalScope<Emitter> S(this);
+    if (!this->delegate(E))
+      return false;
+    return S.destroyLocals();
+  };
+
+  if (std::optional<bool> BoolValue = getBoolValue(Condition)) {
+    if (BoolValue)
+      return visitChildExpr(TrueExpr);
+    return visitChildExpr(FalseExpr);
+  }
+
+  bool IsBcpCall = false;
+  if (const auto *CE = dyn_cast<CallExpr>(Condition->IgnoreParenCasts());
+      CE && CE->getBuiltinCallee() == Builtin::BI__builtin_constant_p) {
+    IsBcpCall = true;
+  }
+
   LabelTy LabelEnd = this->getLabel();   // Label after the operator.
   LabelTy LabelFalse = this->getLabel(); // Label for the false expr.
 
+  if (IsBcpCall) {
+    if (!this->emitStartSpeculation(E))
+      return false;
+  }
+
   if (!this->visitBool(Condition))
     return false;
-
   if (!this->jumpFalse(LabelFalse))
     return false;
-
-  {
-    LocalScope<Emitter> S(this);
-    if (!this->delegate(TrueExpr))
-      return false;
-    if (!S.destroyLocals())
-      return false;
-  }
-
+  if (!visitChildExpr(TrueExpr))
+    return false;
   if (!this->jump(LabelEnd))
     return false;
-
   this->emitLabel(LabelFalse);
-
-  {
-    LocalScope<Emitter> S(this);
-    if (!this->delegate(FalseExpr))
-      return false;
-    if (!S.destroyLocals())
-      return false;
-  }
-
+  if (!visitChildExpr(FalseExpr))
+    return false;
   this->fallthrough(LabelEnd);
   this->emitLabel(LabelEnd);
 
+  if (IsBcpCall)
+    return this->emitEndSpeculation(E);
   return true;
 }
 
@@ -4681,6 +4701,28 @@ bool Compiler<Emitter>::visitAPValueInitializer(const APValue &Val,
 template <class Emitter>
 bool Compiler<Emitter>::VisitBuiltinCallExpr(const CallExpr *E,
                                              unsigned BuiltinID) {
+
+  if (BuiltinID == Builtin::BI__builtin_constant_p) {
+    // Void argument is always invalid and harder to handle later.
+    if (E->getArg(0)->getType()->isVoidType()) {
+      if (DiscardResult)
+        return true;
+      return this->emitConst(0, E);
+    }
+
+    if (!this->emitStartSpeculation(E))
+      return false;
+    LabelTy EndLabel = this->getLabel();
+    if (!this->speculate(E, EndLabel))
+      return false;
+    this->fallthrough(EndLabel);
+    if (!this->emitEndSpeculation(E))
+      return false;
+    if (DiscardResult)
+      return this->emitPop(classifyPrim(E), E);
+    return true;
+  }
+
   const Function *Func = getFunction(E->getDirectCallee());
   if (!Func)
     return false;
@@ -5051,7 +5093,7 @@ template <class Emitter> bool Compiler<Emitter>::visitStmt(const Stmt *S) {
   case Stmt::CompoundStmtClass:
     return visitCompoundStmt(cast<CompoundStmt>(S));
   case Stmt::DeclStmtClass:
-    return visitDeclStmt(cast<DeclStmt>(S));
+    return visitDeclStmt(cast<DeclStmt>(S), /*EvaluateConditionDecl=*/true);
   case Stmt::ReturnStmtClass:
     return visitReturnStmt(cast<ReturnStmt>(S));
   case Stmt::IfStmtClass:
@@ -5105,7 +5147,18 @@ bool Compiler<Emitter>::visitCompoundStmt(const CompoundStmt *S) {
 }
 
 template <class Emitter>
-bool Compiler<Emitter>::visitDeclStmt(const DeclStmt *DS) {
+bool Compiler<Emitter>::maybeEmitDeferredVarInit(const VarDecl *VD) {
+  if (auto *DD = dyn_cast_if_present<DecompositionDecl>(VD)) {
+    for (auto *BD : DD->bindings())
+      if (auto *KD = BD->getHoldingVar(); KD && !this->visitVarDecl(KD))
+        return false;
+  }
+  return true;
+}
+
+template <class Emitter>
+bool Compiler<Emitter>::visitDeclStmt(const DeclStmt *DS,
+                                      bool EvaluateConditionDecl) {
   for (const auto *D : DS->decls()) {
     if (isa<StaticAssertDecl, TagDecl, TypedefNameDecl, BaseUsingDecl,
             FunctionDecl, NamespaceAliasDecl, UsingDirectiveDecl>(D))
@@ -5118,13 +5171,8 @@ bool Compiler<Emitter>::visitDeclStmt(const DeclStmt *DS) {
       return false;
 
     // Register decomposition decl holding vars.
-    if (const auto *DD = dyn_cast<DecompositionDecl>(VD)) {
-      for (auto *BD : DD->bindings())
-        if (auto *KD = BD->getHoldingVar()) {
-          if (!this->visitVarDecl(KD))
-            return false;
-        }
-    }
+    if (EvaluateConditionDecl && !this->maybeEmitDeferredVarInit(VD))
+      return false;
   }
 
   return true;
@@ -5167,6 +5215,12 @@ bool Compiler<Emitter>::visitReturnStmt(const ReturnStmt *RS) {
 }
 
 template <class Emitter> bool Compiler<Emitter>::visitIfStmt(const IfStmt *IS) {
+  auto visitChildStmt = [&](const Stmt *S) -> bool {
+    LocalScope<Emitter> SScope(this);
+    if (!visitStmt(S))
+      return false;
+    return SScope.destroyLocals();
+  };
   if (auto *CondInit = IS->getInit())
     if (!visitStmt(CondInit))
       return false;
@@ -5175,7 +5229,19 @@ template <class Emitter> bool Compiler<Emitter>::visitIfStmt(const IfStmt *IS) {
     if (!visitDeclStmt(CondDecl))
       return false;
 
-  // Compile condition.
+  // Save ourselves compiling some code and the jumps, etc. if the condition is
+  // stataically known to be either true or false. We could look at more cases
+  // here, but I think all the ones that actually happen are using a
+  // ConstantExpr.
+  if (std::optional<bool> BoolValue = getBoolValue(IS->getCond())) {
+    if (*BoolValue)
+      return visitChildStmt(IS->getThen());
+    else if (const Stmt *Else = IS->getElse())
+      return visitChildStmt(Else);
+    return true;
+  }
+
+  // Otherwise, compile the condition.
   if (IS->isNonNegatedConsteval()) {
     if (!this->emitIsConstantContext(IS))
       return false;
@@ -5189,40 +5255,28 @@ template <class Emitter> bool Compiler<Emitter>::visitIfStmt(const IfStmt *IS) {
       return false;
   }
 
+  if (!this->maybeEmitDeferredVarInit(IS->getConditionVariable()))
+    return false;
+
   if (const Stmt *Else = IS->getElse()) {
     LabelTy LabelElse = this->getLabel();
     LabelTy LabelEnd = this->getLabel();
     if (!this->jumpFalse(LabelElse))
       return false;
-    {
-      LocalScope<Emitter> ThenScope(this);
-      if (!visitStmt(IS->getThen()))
-        return false;
-      if (!ThenScope.destroyLocals())
-        return false;
-    }
+    if (!visitChildStmt(IS->getThen()))
+      return false;
     if (!this->jump(LabelEnd))
       return false;
     this->emitLabel(LabelElse);
-    {
-      LocalScope<Emitter> ElseScope(this);
-      if (!visitStmt(Else))
-        return false;
-      if (!ElseScope.destroyLocals())
-        return false;
-    }
+    if (!visitChildStmt(Else))
+      return false;
     this->emitLabel(LabelEnd);
   } else {
     LabelTy LabelEnd = this->getLabel();
     if (!this->jumpFalse(LabelEnd))
       return false;
-    {
-      LocalScope<Emitter> ThenScope(this);
-      if (!visitStmt(IS->getThen()))
-        return false;
-      if (!ThenScope.destroyLocals())
-        return false;
-    }
+    if (!visitChildStmt(IS->getThen()))
+      return false;
     this->emitLabel(LabelEnd);
   }
 
@@ -5249,6 +5303,10 @@ bool Compiler<Emitter>::visitWhileStmt(const WhileStmt *S) {
 
     if (!this->visitBool(Cond))
       return false;
+
+    if (!this->maybeEmitDeferredVarInit(S->getConditionVariable()))
+      return false;
+
     if (!this->jumpFalse(EndLabel))
       return false;
 
@@ -5330,6 +5388,9 @@ bool Compiler<Emitter>::visitForStmt(const ForStmt *S) {
         return false;
     }
 
+    if (!this->maybeEmitDeferredVarInit(S->getConditionVariable()))
+      return false;
+
     if (Body && !this->visitStmt(Body))
       return false;
 
@@ -5452,6 +5513,9 @@ bool Compiler<Emitter>::visitSwitchStmt(const SwitchStmt *S) {
   if (!this->emitSetLocal(CondT, CondVar, S))
     return false;
 
+  if (!this->maybeEmitDeferredVarInit(S->getConditionVariable()))
+    return false;
+
   CaseMap CaseLabels;
   // Create labels and comparison ops for all case statements.
   for (const SwitchCase *SC = S->getSwitchCaseList(); SC;
diff --git a/clang/lib/AST/ByteCode/Compiler.h b/clang/lib/AST/ByteCode/Compiler.h
index 77fcc3d1b41ce..902da8ffdd330 100644
--- a/clang/lib/AST/ByteCode/Compiler.h
+++ b/clang/lib/AST/ByteCode/Compiler.h
@@ -213,7 +213,7 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
 
   // Statements.
   bool visitCompoundStmt(const CompoundStmt *S);
-  bool visitDeclStmt(const DeclStmt *DS);
+  bool visitDeclStmt(const DeclStmt *DS, bool EvaluateConditionDecl = false);
   bool visitReturnStmt(const ReturnStmt *RS);
   bool visitIfStmt(const IfStmt *IS);
   bool visitWhileStmt(const WhileStmt *S);
@@ -274,7 +274,7 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
   /// Evaluates an expression and places the result on the stack. If the
   /// expression is of composite type, a local variable will be created
   /// and a pointer to said variable will be placed on the stack.
-  bool visit(const Expr *E);
+  bool visit(const Expr *E) override;
   /// Compiles an initializer. This is like visit() but it will never
   /// create a variable and instead rely on a variable already having
   /// been created. visitInitializer() then relies on a pointer to this
@@ -342,6 +342,9 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
   /// Emits an integer constant.
   template <typename T> bool emitConst(T Value, PrimType Ty, const Expr *E);
   template <typename T> bool emitConst(T Value, const Expr *E);
+  bool emitBool(bool V, const Expr *E) override {
+    return this->emitConst(V, E);
+  }
 
   llvm::RoundingMode getRoundingMode(const Expr *E) const {
     FPOptions FPO = E->getFPFeaturesInEffect(Ctx.getLangOpts());
@@ -386,6 +389,7 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
   bool compileUnionAssignmentOperator(const CXXMethodDecl *MD);
 
   bool checkLiteralType(const Expr *E);
+  bool maybeEmitDeferredVarInit(const VarDecl *VD);
 
 protected:
   /// Variable to storage mapping.
diff --git a/clang/lib/AST/ByteCode/EvalEmitter.cpp b/clang/lib/AST/ByteCode/EvalEmitter.cpp
index 95149efbea992..5326862c506d1 100644
--- a/clang/lib/AST/ByteCode/EvalEmitter.cpp
+++ b/clang/lib/AST/ByteCode/EvalEmitter.cpp
@@ -127,6 +127,33 @@ bool EvalEmitter::fallthrough(const LabelTy &Label) {
   return true;
 }
 
+bool EvalEmitter::speculate(const CallExpr *E, const LabelTy &EndLabel) {
+  size_t StackSizeBefore = S.Stk.size();
+  const Expr *Arg = E->getArg(0);
+  if (!this->visit(Arg)) {
+    S.Stk.clearTo(StackSizeBefore);
+
+    if (S.inConstantContext() || Arg->HasSideEffects(S.getASTContext()))
+      return this->emitBool(false, E);
+    return Invalid(S, OpPC);
+  }
+
+  PrimType T = Ctx.classify(Arg->getType()).value_or(PT_Ptr);
+  if (T == PT_Ptr) {
+    const auto &Ptr = S.Stk.pop<Pointer>();
+    return this->emitBool(CheckBCPResult(S, Ptr), E);
+  } else if (T == PT_FnPtr) {
+    S.Stk.discard<FunctionPointer>();
+    // Never accepted
+    return this->emitBool(false, E);
+  }
+
+  // Otherwise, this is fine!
+  if (!this->emitPop(T, E))
+    return false;
+  return this->emitBool(true, E);
+}
+
 template <PrimType OpType> bool EvalEmitter::emitRet(const SourceInfo &Info) {
   if (!isActive())
     return true;
diff --git a/clang/lib/AST/ByteCode/EvalEmitter.h b/clang/lib/AST/ByteCode/EvalEmitter.h
index 2cac2ba2ef221..f53f86c31ec1e 100644
--- a/clang/lib/AST/ByteCode/EvalEmitter.h
+++ b/clang/lib/AST/ByteCode/EvalEmitter.h
@@ -55,12 +55,16 @@ class EvalEmitter : public SourceMapper {
   virtual bool visitExpr(const Expr *E, bool DestroyToplevelScope) = 0;
   virtual bool visitDeclAndReturn(const VarDecl *VD, bool ConstantContext) = 0;
   virtual bool visitFunc(const FunctionDecl *F) = 0;
+  virtual bool visit(const Expr *E) = 0;
+  virtual bool emitBool(bool V, const Expr *E) = 0;
 
   /// Emits jumps.
   bool jumpTrue(const LabelTy &Label);
   bool jumpFalse(const LabelTy &Label);
   bool jump(const LabelTy &Label);
   bool fallthrough(const LabelTy &Label);
+  /// Speculative execution.
+  bool speculate(const CallExpr *E, const LabelTy &EndLabel);
 
   /// Since expressions can only jump forward, predicated execution is
   /// used to deal with if-else statements.
diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index 1107c0c32792f..201fec3e864d5 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -54,6 +54,79 @@ static bool Jf(InterpState &S, CodePtr &PC, int32_t Offset) {
   return true;
 }
 
+// https://github.com/llvm/llvm-project/issues/102513
+#if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG)
+#pragma optimize("", off)
+#endif
+// FIXME: We have the large switch over all opcodes here again, and in
+// Interpret().
+static bool BCP(InterpState &S, CodePtr &RealPC, int32_t Offset, PrimType PT) {
+  [[maybe_unused]] CodePtr PCBefore = RealPC;
+  size_t StackSizeBefore = S.Stk.size();
+
+  auto SpeculativeInterp = [&S, RealPC]() -> bool {
+    const InterpFrame *StartFrame = S.Current;
+    CodePtr PC = RealPC;
+
+    for (;;) {
+      auto Op = PC.read<Opcode>();
+      if (Op == OP_EndSpeculation)
+        return true;
+      CodePtr OpPC = PC;
+
+      switch (Op) {
+#define GET_INTERP
+#include "Opcodes.inc"
+#undef GET_INTERP
+      }
+    }
+    llvm_unreachable("We didn't see an EndSpeculation op?");
+  };
+
+  if (SpeculativeInterp()) {
+    if (PT == PT_Ptr) {
+      const auto &Ptr = S.Stk.pop<Pointer>();
+      assert(S.Stk.size() == StackSizeBefore);
+      S.Stk.push<Integral<32, true>>(
+          Integral<32, true>::from(CheckBCPResult(S, Ptr)));
+    } else if (PT == PT_FnPtr) {
+      S.Stk.discard<FunctionPointer>();
+      S.Stk.push<Integral<32, true>>(Integral<32, true>::from(0));
+    } else {
+      // Pop the result from the stack and return success.
+      TYPE_SWITCH(PT, S.Stk.pop<T>(););
+      assert(S.Stk.size() == StackSizeBefore);
+      S.Stk.push<Integral<32, true>>(Integral<32, true>::from(1));
+    }
+  } else {
+    if (!S.inConstantContext())
+      return Invalid(S, RealPC);
+
+    S.Stk.clearTo(StackSizeBefore);
+    S.Stk.push<Integral<32, true>>(Integral<32, true>::from(0));
+  }
+
+  // RealPC should not have been modified.
+  assert(*RealPC == *PCBefore);
+
+  // Jump to end label. This is a little tricker than just RealPC += Offset
+  // because our usual jump instructions don't have any arguments, to the offset
+  // we get is a little too much and we need to subtract the size of the
+  // bool and PrimType arguments again.
+  int32_t ParamSize = align(sizeof(PrimType));
+  assert(Offset >= ParamSize);
+  RealPC += Offset - ParamSize;
+
+  [[maybe_unused]] CodePtr PCCopy = RealPC;
+  assert(PCCopy.read<Opcode>() == OP_EndSpeculation);
+
+  return true;
+}
+// https://github.com/llvm/llvm-project/issues/102513
+#if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG)
+#pragma optimize("", on)
+#endif
+
 static void diagnoseMissingInitializer(InterpState &S, CodePtr OpPC,
                                        const ValueDecl *VD) {
   const SourceInfo &E = S.Current->getSource(OpPC);
@@ -290,6 +363,22 @@ void cleanupAfterFunctionCall(InterpState &S, CodePtr OpPC,
     TYPE_SWITCH(Ty, S.Stk.discard<T>());
 }
 
+bool CheckBCPResult(InterpState &S, const Pointer &Ptr) {
+  if (Ptr.isDummy())
+    return false;
+  if (Ptr.isZero())
+    return true;
+  if (Ptr.isIntegralPointer())
+    return true;
+  if (Ptr.isTypeidPointer())
+    return true;
+
+  if (const Expr *Base = Ptr.getDeclDesc()->asExpr())
+    return isa<StringLiteral>(Base);
+
+  return false;
+}
+
 bool CheckExtern(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
   if (!Ptr.isExtern())
     return true;
diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index d8f90e45b0ced..f2ddeac99cd7e 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -159,6 +159,7 @@ bool CheckLiteralType(InterpState &S, CodePtr OpPC, const Type *T);
 bool InvalidShuffleVectorIndex(InterpState &S, CodePtr OpPC, uint32_t Index);
 bool CheckBitCast(InterpState &S, CodePtr OpPC, bool HasIndeterminateBits,
                   bool TargetIsUCharOrByte);
+bool CheckBCPResult(InterpState &S, const Pointer &Ptr);
 
 template <typename T>
 static bool handleOverflow(InterpState &S, CodePtr OpPC, const T &SrcValue) {
@@ -2431,9 +2432,12 @@ inline bool This(InterpState &S, CodePtr OpPC) {
   // Ensure the This pointer has been cast to the correct base.
   if (!This.isDummy()) {
     assert(isa<CXXMethodDecl>(S.Current->getFunction()->getDecl()));
-    assert(This.getRecord());
+    [[maybe_unused]] const Record *R = This.getRecord();
+    if (!R)
+      R = This.narrow().getRecord();
+    assert(R);
     assert(
-        This.getRecord()->getDecl() ==
+        R->getDecl() ==
         cast<CXXMethodDecl>(S.Current->getFunction()->getDecl())->getParent());
   }
 
@@ -2776,8 +2780,29 @@ inline bool Unsupported(InterpState &S, CodePtr OpPC) {
   return false;
 }
 
+inline bool StartSpeculation(InterpState &S, CodePtr OpPC) {
+  ++S.SpeculationDepth;
+  if (S.SpeculationDepth != 1)
+    return true;
+
+  assert(S.PrevDiags == nullptr);
+  S.PrevDiags = S.getEvalStatus().Diag;
+  S.getEvalStatus().Diag = nullptr;
+  return true;
+}
+inline bool EndSpeculation(InterpState &S, CodePtr OpPC) {
+  assert(S.SpeculationDepth != 0);
+  --S.SpeculationDepth;
+  if (S.SpeculationDepth == 0) {
+    S.getEvalStatus().Diag = S.PrevDiags;
+    S.PrevDiags = nullptr;
+  }
+  return true;
+}
+
 /// Do nothing and just abort execution.
 inline bool Error(InterpState &S, CodePtr OpPC) { return false; }
+
 inline bool SideEffect(InterpState &S, CodePtr OpPC) {
   return S.noteSideEffect();
 }
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index df9c2bc24b15f..4660c80fc90db 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -1483,80 +1483,6 @@ static bool interp__builtin_ptrauth_string_discriminator(
   return true;
 }
 
-// FIXME: This implementation is not complete.
-// The Compiler instance we create cannot access the current stack frame, local
-// variables, function parameters, etc. We also need protection from
-// side-effects, fatal errors, etc.
-static bool interp__builtin_constant_p(InterpState &S, CodePtr OpPC,
-                                       const InterpFrame *Frame,
-                                       const Function *Func,
-                                       const CallExpr *Call) {
-  const Expr *Arg = Call->getArg(0);
-  QualType ArgType = Arg->getType();
-
-  auto returnInt = [&S, Call](bool Value) -> bool {
-    pushInteger(S, Value, Call->getType());
-    return true;
-  };
-
-  // __builtin_constant_p always has one operand. The rules which gcc follows
-  // are not precisely documented, but are as follows:
-  //
-  //  - If the operand is of integral, floating, complex or enumeration type,
-  //    and can be folded to a known value of that type, it returns 1.
-  //  - If the operand can be folded to a pointer to the first character
-  //    of a string literal (or such a pointer cast to an integral type)
-  //    or to a null pointer or an integer cast to a pointer, it returns 1.
-  //
-  // Otherwise, it returns 0.
-  //
-  // FIXME: GCC also intends to return 1 for literals of aggregate types, but
-  // its support for this did not work prior to GCC 9 and is not yet well
-  // understood.
-  if (ArgType->isIntegralOrEnumerationType() || ArgType->isFloatingType() ||
-      ArgType->isAnyComplexType() || ArgType->isPointerType() ||
-      ArgType->isNullPtrType()) {
-    auto PrevDiags = S.getEvalStatus().Diag;
-    S.getEvalStatus().Diag = nullptr;
-    InterpStack Stk;
-    Compiler<EvalEmitter> C(S.Ctx, S.P, S, Stk);
-    auto Res = C.interpretExpr(Arg, /*ConvertResultToRValue=*/Arg->isGLValue());
-    S.getEvalStatus().Diag = PrevDiags;
-    if (Res.isInvalid()) {
-      C.cleanup();
-      Stk.clear();
-      return returnInt(false);
-    }
-
-    if (!Res.empty()) {
-      const APValue &LV = Res.toAPValue();
-      if (LV.isLValue()) {
-        APValue::LValueBase Base = LV.getLValueBase();
-        if (Base.isNull()) {
-          // A null base is acceptable.
-          return returnInt(true);
-        } else if (const auto *E = Base.dyn_cast<const Expr *>()) {
-          if (!isa<StringLiteral>(E))
-            return returnInt(false);
-          return returnInt(LV.getLValueOffset().isZero());
-        } else if (Base.is<TypeInfoLValue>()) {
-          // Surprisingly, GCC considers __builtin_constant_p(&typeid(int)) to
-          // evaluate to true.
-          return returnInt(true);
-        } else {
-          // Any other base is not constant enough for GCC.
-          return returnInt(false);
-        }
-      }
-    }
-
-    // Otherwise, any constant value is good enough.
-    return returnInt(true);
-  }
-
-  return returnInt(false);
-}
-
 static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC,
                                          const InterpFrame *Frame,
                                          const Function *Func,
@@ -1955,11 +1881,23 @@ static bool interp__builtin_memcmp(InterpState &S, CodePtr OpPC,
   bool IsWide =
       (ID == Builtin::BIwmemcmp || ID == Builtin::BI__builtin_wmemcmp);
 
+  auto getElemType = [](const Pointer &P) -> QualType {
+    const Descriptor *Desc = P.getFieldDesc();
+    QualType T = Desc->getType();
+    if (T->isPointerType())
+      return T->getAs<PointerType>()->getPointeeType();
+    if (Desc->isArray())
+      return Desc->getElemQualType();
+    return T;
+  };
+
   const ASTContext &ASTCtx = S.getASTContext();
+  QualType ElemTypeA = getElemType(PtrA);
+  QualType ElemTypeB = getElemType(PtrB);
   // FIXME: This is an arbitrary limitation the current constant interpreter
   // had. We could remove this.
-  if (!IsWide && (!isOneByteCharacterType(PtrA.getType()) ||
-                  !isOneByteCharacterType(PtrB.getType()))) {
+  if (!IsWide && (!isOneByteCharacterType(ElemTypeA) ||
+                  !isOneByteCharacterType(ElemTypeB))) {
     S.FFDiag(S.Current->getSource(OpPC),
              diag::note_constexpr_memcmp_unsupported)
         << ASTCtx.BuiltinInfo.getQuotedName(ID) << PtrA.getType()
@@ -1972,7 +1910,7 @@ static bool interp__builtin_memcmp(InterpState &S, CodePtr OpPC,
 
   // Now, read both pointers to a buffer and compare those.
   BitcastBuffer BufferA(
-      Bits(ASTCtx.getTypeSize(PtrA.getFieldDesc()->getType())));
+      Bits(ASTCtx.getTypeSize(ElemTypeA) * PtrA.getNumElems()));
   readPointerToBuffer(S.getContext(), PtrA, BufferA, false);
   // FIXME: The swapping here is UNDOING something we do when reading the
   // data into the buffer.
@@ -1980,7 +1918,7 @@ static bool interp__builtin_memcmp(InterpState &S, CodePtr OpPC,
     swapBytes(BufferA.Data.get(), BufferA.byteSize().getQuantity());
 
   BitcastBuffer BufferB(
-      Bits(ASTCtx.getTypeSize(PtrB.getFieldDesc()->getType())));
+      Bits(ASTCtx.getTypeSize(ElemTypeB) * PtrB.getNumElems()));
   readPointerToBuffer(S.getContext(), PtrB, BufferB, false);
   // FIXME: The swapping here is UNDOING something we do when reading the
   // data into the buffer.
@@ -2034,15 +1972,108 @@ static bool interp__builtin_memcmp(InterpState &S, CodePtr OpPC,
 
   // However, if we read all the available bytes but were instructed to read
   // even more, diagnose this as a "read of dereferenced one-past-the-end
-  // pointer". This is what would happen if we called CheckRead() on every array
+  // pointer". This is what would happen if we called CheckLoad() on every array
   // element.
   S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_access_past_end)
       << AK_Read << S.Current->getRange(OpPC);
   return false;
 }
 
+static bool interp__builtin_memchr(InterpState &S, CodePtr OpPC,
+                                   const InterpFrame *Frame,
+                                   const Function *Func, const CallExpr *Call) {
+  unsigned ID = Func->getBuiltinID();
+  if (ID == Builtin::BImemchr || ID == Builtin::BIwcschr ||
+      ID == Builtin::BIstrchr || ID == Builtin::BIwmemchr)
+    diagnoseNonConstexprBuiltin(S, OpPC, ID);
+
+  const Pointer &Ptr = getParam<Pointer>(Frame, 0);
+  APSInt Desired;
+  std::optional<APSInt> MaxLength;
+  if (Call->getNumArgs() == 3) {
+    MaxLength =
+        peekToAPSInt(S.Stk, *S.getContext().classify(Call->getArg(2)), 0);
+    Desired = peekToAPSInt(
+        S.Stk, *S.getContext().classify(Call->getArg(1)),
+        align(primSize(*S.getContext().classify(Call->getArg(2)))) +
+            align(primSize(*S.getContext().classify(Call->getArg(1)))));
+  } else {
+    Desired = peekToAPSInt(S.Stk, *S.getContext().classify(Call->getArg(1)));
+  }
+
+  if (MaxLength && MaxLength->isZero()) {
+    S.Stk.push<Pointer>();
+    return true;
+  }
+
+  if (Ptr.isDummy())
+    return false;
+
+  // Null is only okay if the given size is 0.
+  if (Ptr.isZero()) {
+    S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_access_null)
+        << AK_Read;
+    return false;
+  }
+
+  QualType ElemTy = Ptr.getFieldDesc()->isArray()
+                        ? Ptr.getFieldDesc()->getElemQualType()
+                        : Ptr.getFieldDesc()->getType();
+  bool IsRawByte = ID == Builtin::BImemchr || ID == Builtin::BI__builtin_memchr;
+
+  // Give up on byte-oriented matching against multibyte elements.
+  if (IsRawByte && !isOneByteCharacterType(ElemTy)) {
+    S.FFDiag(S.Current->getSource(OpPC),
+             diag::note_constexpr_memchr_unsupported)
+        << S.getASTContext().BuiltinInfo.getQuotedName(ID) << ElemTy;
+    return false;
+  }
+
+  if (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr) {
+    // strchr compares directly to the passed integer, and therefore
+    // always fails if given an int that is not a char.
+    if (Desired !=
+        Desired.trunc(S.getASTContext().getCharWidth()).getSExtValue()) {
+      S.Stk.push<Pointer>();
+      return true;
+    }
+  }
+
+  uint64_t DesiredVal =
+      Desired.trunc(S.getASTContext().getCharWidth()).getZExtValue();
+  bool StopAtZero =
+      (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr);
+
+  size_t Index = Ptr.getIndex();
+  for (;;) {
+    const Pointer &ElemPtr = Index > 0 ? Ptr.atIndex(Index) : Ptr;
+
+    if (!CheckLoad(S, OpPC, ElemPtr))
+      return false;
+
+    unsigned char V = static_cast<unsigned char>(ElemPtr.deref<char>());
+    if (V == DesiredVal) {
+      S.Stk.push<Pointer>(ElemPtr);
+      return true;
+    }
+
+    if (StopAtZero && V == 0)
+      break;
+
+    ++Index;
+    if (MaxLength && Index == MaxLength->getZExtValue())
+      break;
+  }
+
+  S.Stk.push<Pointer>();
+  return true;
+}
+
 bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
                       const CallExpr *Call, uint32_t BuiltinID) {
+  if (!S.getASTContext().BuiltinInfo.isConstantEvaluated(BuiltinID))
+    return false;
+
   const InterpFrame *Frame = S.Current;
 
   std::optional<PrimType> ReturnT = S.getContext().classify(Call);
@@ -2468,11 +2499,6 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
       return false;
     break;
 
-  case Builtin::BI__builtin_constant_p:
-    if (!interp__builtin_constant_p(S, OpPC, Frame, F, Call))
-      return false;
-    break;
-
   case Builtin::BI__noop:
     pushInteger(S, 0, Call->getType());
     break;
@@ -2524,6 +2550,21 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
       return false;
     break;
 
+  case Builtin::BImemchr:
+  case Builtin::BI__builtin_memchr:
+  case Builtin::BIstrchr:
+  case Builtin::BI__builtin_strchr:
+#if 0
+  case Builtin::BIwcschr:
+  case Builtin::BI__builtin_wcschr:
+  case Builtin::BImemchr:
+  case Builtin::BI__builtin_wmemchr:
+#endif
+  case Builtin::BI__builtin_char_memchr:
+    if (!interp__builtin_memchr(S, OpPC, Frame, F, Call))
+      return false;
+    break;
+
   default:
     S.FFDiag(S.Current->getLocation(OpPC),
              diag::note_invalid_subexpr_in_const_expr)
diff --git a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
index f4c54551a9a60..6b8860c09167c 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
@@ -94,7 +94,8 @@ static bool enumerateData(const Pointer &P, const Context &Ctx, Bits Offset,
     Bits ElemSize = Bits(Ctx.getASTContext().getTypeSize(ElemType));
     PrimType ElemT = *Ctx.classify(ElemType);
     // Special case, since the bools here are packed.
-    bool PackedBools = FieldDesc->getType()->isExtVectorBoolType();
+    bool PackedBools =
+        FieldDesc->getType()->isPackedVectorBoolType(Ctx.getASTContext());
     unsigned NumElems = FieldDesc->getNumElems();
     bool Ok = true;
     for (unsigned I = P.getIndex(); I != NumElems; ++I) {
@@ -227,7 +228,7 @@ static bool CheckBitcastType(InterpState &S, CodePtr OpPC, QualType T,
     QualType EltTy = VT->getElementType();
     unsigned NElts = VT->getNumElements();
     unsigned EltSize =
-        VT->isExtVectorBoolType() ? 1 : ASTCtx.getTypeSize(EltTy);
+        VT->isPackedVectorBoolType(ASTCtx) ? 1 : ASTCtx.getTypeSize(EltTy);
 
     if ((NElts * EltSize) % ASTCtx.getCharWidth() != 0) {
       // The vector's size in bits is not a multiple of the target's byte size,
diff --git a/clang/lib/AST/ByteCode/InterpState.h b/clang/lib/AST/ByteCode/InterpState.h
index d6adfff1a713a..74001b80d9c00 100644
--- a/clang/lib/AST/ByteCode/InterpState.h
+++ b/clang/lib/AST/ByteCode/InterpState.h
@@ -144,6 +144,9 @@ class InterpState final : public State, public SourceMapper {
   SourceLocation EvalLocation;
   /// Declaration we're initializing/evaluting, if any.
   const VarDecl *EvaluatingDecl = nullptr;
+  /// Things needed to do speculative execution.
+  SmallVectorImpl<PartialDiagnosticAt> *PrevDiags = nullptr;
+  unsigned SpeculationDepth = 0;
 
   llvm::SmallVector<
       std::pair<const Expr *, const LifetimeExtendedTemporaryDecl *>>
diff --git a/clang/lib/AST/ByteCode/Opcodes.td b/clang/lib/AST/ByteCode/Opcodes.td
index 41e4bae65c195..98f9818cb5ffb 100644
--- a/clang/lib/AST/ByteCode/Opcodes.td
+++ b/clang/lib/AST/ByteCode/Opcodes.td
@@ -58,7 +58,7 @@ def ArgRecordField : ArgType { let Name = "const Record::Field *"; }
 def ArgFltSemantics : ArgType { let Name = "const llvm::fltSemantics *"; }
 def ArgRoundingMode : ArgType { let Name = "llvm::RoundingMode"; }
 def ArgLETD: ArgType { let Name = "const LifetimeExtendedTemporaryDecl *"; }
-def ArgCastKind : ArgType { let Name = "CastKind"; }
+def ArgCastKind : ArgType { let Name = "interp::CastKind"; }
 def ArgCallExpr : ArgType { let Name = "const CallExpr *"; }
 def ArgExpr : ArgType { let Name = "const Expr *"; }
 def ArgOffsetOfExpr : ArgType { let Name = "const OffsetOfExpr *"; }
@@ -172,6 +172,14 @@ def Jt : JumpOpcode;
 // [Bool] -> [], jumps if false.
 def Jf : JumpOpcode;
 
+def StartSpeculation : Opcode;
+def EndSpeculation : Opcode;
+def BCP : Opcode {
+  let ChangesPC = 1;
+  let HasCustomEval = 1;
+  let Args = [ArgSint32, ArgPrimType];
+}
+
 //===----------------------------------------------------------------------===//
 // Returns
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index ccfec7fda0cbc..901ebf9592680 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -1509,16 +1509,6 @@ CallExpr *CallExpr::Create(const ASTContext &Ctx, Expr *Fn,
                             RParenLoc, FPFeatures, MinNumArgs, UsesADL);
 }
 
-CallExpr *CallExpr::CreateTemporary(void *Mem, Expr *Fn, QualType Ty,
-                                    ExprValueKind VK, SourceLocation RParenLoc,
-                                    ADLCallKind UsesADL) {
-  assert(!(reinterpret_cast<uintptr_t>(Mem) % alignof(CallExpr)) &&
-         "Misaligned memory in CallExpr::CreateTemporary!");
-  return new (Mem) CallExpr(CallExprClass, Fn, /*PreArgs=*/{}, /*Args=*/{}, Ty,
-                            VK, RParenLoc, FPOptionsOverride(),
-                            /*MinNumArgs=*/0, UsesADL);
-}
-
 CallExpr *CallExpr::CreateEmpty(const ASTContext &Ctx, unsigned NumArgs,
                                 bool HasFPFeatures, EmptyShell Empty) {
   unsigned SizeOfTrailingObjects =
@@ -1655,12 +1645,8 @@ SourceLocation CallExpr::getBeginLoc() const {
   if (!isTypeDependent()) {
     if (const auto *Method =
             dyn_cast_if_present<const CXXMethodDecl>(getCalleeDecl());
-        Method && Method->isExplicitObjectMemberFunction()) {
-      bool HasFirstArg = getNumArgs() > 0 && getArg(0);
-      assert(HasFirstArg);
-      if (HasFirstArg)
-        return getArg(0)->getBeginLoc();
-    }
+        Method && Method->isExplicitObjectMemberFunction())
+      return getArg(0)->getBeginLoc();
   }
 
   SourceLocation begin = getCallee()->getBeginLoc();
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index d9a1e5bb42343..f8e8aaddbfdbd 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -2419,6 +2419,16 @@ static bool CheckLValueConstantExpression(EvalInfo &Info, SourceLocation Loc,
           LVal.getLValueCallIndex() == 0) &&
          "have call index for global lvalue");
 
+  if (LVal.allowConstexprUnknown()) {
+    if (BaseVD) {
+      Info.FFDiag(Loc, diag::note_constexpr_var_init_non_constant, 1) << BaseVD;
+      NoteLValueLocation(Info, Base);
+    } else {
+      Info.FFDiag(Loc);
+    }
+    return false;
+  }
+
   if (Base.is<DynamicAllocLValue>()) {
     Info.FFDiag(Loc, diag::note_constexpr_dynamic_alloc)
         << IsReferenceType << !Designator.Entries.empty();
@@ -3597,7 +3607,8 @@ static bool evaluateVarDeclInit(EvalInfo &Info, const Expr *E,
   // expressions here; doing so would regress diagnostics for things like
   // reading from a volatile constexpr variable.
   if ((Info.getLangOpts().CPlusPlus && !VD->hasConstantInitialization() &&
-       VD->mightBeUsableInConstantExpressions(Info.Ctx)) ||
+       VD->mightBeUsableInConstantExpressions(Info.Ctx) &&
+       !AllowConstexprUnknown) ||
       ((Info.getLangOpts().CPlusPlus || Info.getLangOpts().OpenCL) &&
        !Info.getLangOpts().CPlusPlus11 && !VD->hasICEInitializer(Info.Ctx))) {
     if (Init) {
@@ -5216,20 +5227,41 @@ static bool EvaluateVarDecl(EvalInfo &Info, const VarDecl *VD) {
   return true;
 }
 
-static bool EvaluateDecl(EvalInfo &Info, const Decl *D) {
-  bool OK = true;
+static bool EvaluateDecompositionDeclInit(EvalInfo &Info,
+                                          const DecompositionDecl *DD);
 
+static bool EvaluateDecl(EvalInfo &Info, const Decl *D,
+                         bool EvaluateConditionDecl = false) {
+  bool OK = true;
   if (const VarDecl *VD = dyn_cast<VarDecl>(D))
     OK &= EvaluateVarDecl(Info, VD);
 
-  if (const DecompositionDecl *DD = dyn_cast<DecompositionDecl>(D))
-    for (auto *BD : DD->flat_bindings())
-      if (auto *VD = BD->getHoldingVar())
-        OK &= EvaluateDecl(Info, VD);
+  if (const DecompositionDecl *DD = dyn_cast<DecompositionDecl>(D);
+      EvaluateConditionDecl && DD)
+    OK &= EvaluateDecompositionDeclInit(Info, DD);
 
   return OK;
 }
 
+static bool EvaluateDecompositionDeclInit(EvalInfo &Info,
+                                          const DecompositionDecl *DD) {
+  bool OK = true;
+  for (auto *BD : DD->flat_bindings())
+    if (auto *VD = BD->getHoldingVar())
+      OK &= EvaluateDecl(Info, VD, /*EvaluateConditionDecl=*/true);
+
+  return OK;
+}
+
+static bool MaybeEvaluateDeferredVarDeclInit(EvalInfo &Info,
+                                             const VarDecl *VD) {
+  if (auto *DD = dyn_cast_if_present<DecompositionDecl>(VD)) {
+    if (!EvaluateDecompositionDeclInit(Info, DD))
+      return false;
+  }
+  return true;
+}
+
 static bool EvaluateDependentExpr(const Expr *E, EvalInfo &Info) {
   assert(E->isValueDependent());
   if (Info.noteSideEffect())
@@ -5249,6 +5281,8 @@ static bool EvaluateCond(EvalInfo &Info, const VarDecl *CondDecl,
     return false;
   if (!EvaluateAsBooleanCondition(Cond, Result, Info))
     return false;
+  if (!MaybeEvaluateDeferredVarDeclInit(Info, CondDecl))
+    return false;
   return Scope.destroy();
 }
 
@@ -5333,6 +5367,9 @@ static EvalStmtResult EvaluateSwitch(StmtResult &Result, EvalInfo &Info,
     if (!EvaluateInteger(SS->getCond(), Value, Info))
       return ESR_Failed;
 
+    if (!MaybeEvaluateDeferredVarDeclInit(Info, SS->getConditionVariable()))
+      return ESR_Failed;
+
     if (!CondScope.destroy())
       return ESR_Failed;
   }
@@ -5557,7 +5594,8 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info,
         return ESR_Failed;
       // Each declaration initialization is its own full-expression.
       FullExpressionRAII Scope(Info);
-      if (!EvaluateDecl(Info, D) && !Info.noteFailure())
+      if (!EvaluateDecl(Info, D, /*EvaluateConditionDecl=*/true) &&
+          !Info.noteFailure())
         return ESR_Failed;
       if (!Scope.destroy())
         return ESR_Failed;
@@ -7436,7 +7474,7 @@ class APValueToBufferConverter {
     QualType EltTy = VTy->getElementType();
     unsigned NElts = VTy->getNumElements();
 
-    if (VTy->isExtVectorBoolType()) {
+    if (VTy->isPackedVectorBoolType(Info.Ctx)) {
       // Special handling for OpenCL bool vectors:
       // Since these vectors are stored as packed bits, but we can't write
       // individual bits to the BitCastBuffer, we'll buffer all of the elements
@@ -7699,11 +7737,11 @@ class BufferToAPValueConverter {
     QualType EltTy = VTy->getElementType();
     unsigned NElts = VTy->getNumElements();
     unsigned EltSize =
-        VTy->isExtVectorBoolType() ? 1 : Info.Ctx.getTypeSize(EltTy);
+        VTy->isPackedVectorBoolType(Info.Ctx) ? 1 : Info.Ctx.getTypeSize(EltTy);
 
     SmallVector<APValue, 4> Elts;
     Elts.reserve(NElts);
-    if (VTy->isExtVectorBoolType()) {
+    if (VTy->isPackedVectorBoolType(Info.Ctx)) {
       // Special handling for OpenCL bool vectors:
       // Since these vectors are stored as packed bits, but we can't read
       // individual bits from the BitCastBuffer, we'll buffer all of the
@@ -7832,7 +7870,8 @@ static bool checkBitCastConstexprEligibilityType(SourceLocation Loc,
   if (const auto *VTy = Ty->getAs<VectorType>()) {
     QualType EltTy = VTy->getElementType();
     unsigned NElts = VTy->getNumElements();
-    unsigned EltSize = VTy->isExtVectorBoolType() ? 1 : Ctx.getTypeSize(EltTy);
+    unsigned EltSize =
+        VTy->isPackedVectorBoolType(Ctx) ? 1 : Ctx.getTypeSize(EltTy);
 
     if ((NElts * EltSize) % Ctx.getCharWidth() != 0) {
       // The vector's size in bits is not a multiple of the target's byte size,
@@ -16892,24 +16931,19 @@ bool Expr::EvaluateAsConstantExpr(EvalResult &Result, const ASTContext &Ctx,
   APValue::LValueBase Base(&BaseMTE);
   Info.setEvaluatingDecl(Base, Result.Val);
 
-  if (Info.EnableNewConstInterp) {
-    if (!Info.Ctx.getInterpContext().evaluateAsRValue(Info, this, Result.Val))
-      return false;
-  } else {
-    LValue LVal;
-    LVal.set(Base);
-    // C++23 [intro.execution]/p5
-    // A full-expression is [...] a constant-expression
-    // So we need to make sure temporary objects are destroyed after having
-    // evaluating the expression (per C++23 [class.temporary]/p4).
-    FullExpressionRAII Scope(Info);
-    if (!::EvaluateInPlace(Result.Val, Info, LVal, this) ||
-        Result.HasSideEffects || !Scope.destroy())
-      return false;
+  LValue LVal;
+  LVal.set(Base);
+  // C++23 [intro.execution]/p5
+  // A full-expression is [...] a constant-expression
+  // So we need to make sure temporary objects are destroyed after having
+  // evaluating the expression (per C++23 [class.temporary]/p4).
+  FullExpressionRAII Scope(Info);
+  if (!::EvaluateInPlace(Result.Val, Info, LVal, this) ||
+      Result.HasSideEffects || !Scope.destroy())
+    return false;
 
-    if (!Info.discardCleanups())
-      llvm_unreachable("Unhandled cleanup; missing full expression marker?");
-  }
+  if (!Info.discardCleanups())
+    llvm_unreachable("Unhandled cleanup; missing full expression marker?");
 
   if (!CheckConstantExpression(Info, getExprLoc(), getStorageType(Ctx, this),
                                Result.Val, Kind))
@@ -16993,18 +17027,6 @@ bool Expr::EvaluateAsInitializer(APValue &Value, const ASTContext &Ctx,
 
     if (!Info.discardCleanups())
       llvm_unreachable("Unhandled cleanup; missing full expression marker?");
-
-    if (Value.allowConstexprUnknown()) {
-      assert(Value.isLValue() && "Expected an lvalue");
-      auto Base = Value.getLValueBase();
-      const auto *NewVD = Base.dyn_cast<const ValueDecl *>();
-      if (!NewVD)
-        NewVD = VD;
-      Info.FFDiag(getExprLoc(), diag::note_constexpr_var_init_non_constant, 1)
-          << NewVD;
-      NoteLValueLocation(Info, Base);
-      return false;
-    }
   }
 
   return CheckConstantExpression(Info, DeclLoc, DeclTy, Value,
diff --git a/clang/lib/AST/Mangle.cpp b/clang/lib/AST/Mangle.cpp
index 4c4f2038c51e6..b44ab23f1d0e1 100644
--- a/clang/lib/AST/Mangle.cpp
+++ b/clang/lib/AST/Mangle.cpp
@@ -240,7 +240,8 @@ void MangleContext::mangleName(GlobalDecl GD, raw_ostream &Out) {
   Out << ((DefaultPtrWidth / 8) * ArgWords);
 }
 
-void MangleContext::mangleMSGuidDecl(const MSGuidDecl *GD, raw_ostream &Out) {
+void MangleContext::mangleMSGuidDecl(const MSGuidDecl *GD,
+                                     raw_ostream &Out) const {
   // For now, follow the MSVC naming convention for GUID objects on all
   // targets.
   MSGuidDecl::Parts P = GD->getParts();
@@ -327,7 +328,7 @@ void MangleContext::mangleBlock(const DeclContext *DC, const BlockDecl *BD,
 void MangleContext::mangleObjCMethodName(const ObjCMethodDecl *MD,
                                          raw_ostream &OS,
                                          bool includePrefixByte,
-                                         bool includeCategoryNamespace) {
+                                         bool includeCategoryNamespace) const {
   if (getASTContext().getLangOpts().ObjCRuntime.isGNUFamily()) {
     // This is the mangling we've always used on the GNU runtimes, but it
     // has obvious collisions in the face of underscores within class
@@ -382,7 +383,7 @@ void MangleContext::mangleObjCMethodName(const ObjCMethodDecl *MD,
 }
 
 void MangleContext::mangleObjCMethodNameAsSourceName(const ObjCMethodDecl *MD,
-                                                     raw_ostream &Out) {
+                                                     raw_ostream &Out) const {
   SmallString<64> Name;
   llvm::raw_svector_ostream OS(Name);
 
diff --git a/clang/lib/AST/NestedNameSpecifier.cpp b/clang/lib/AST/NestedNameSpecifier.cpp
index 76c77569da9fd..593f2fcc0159c 100644
--- a/clang/lib/AST/NestedNameSpecifier.cpp
+++ b/clang/lib/AST/NestedNameSpecifier.cpp
@@ -248,7 +248,8 @@ bool NestedNameSpecifier::containsErrors() const {
 /// Print this nested name specifier to the given output
 /// stream.
 void NestedNameSpecifier::print(raw_ostream &OS, const PrintingPolicy &Policy,
-                                bool ResolveTemplateArguments) const {
+                                bool ResolveTemplateArguments,
+                                bool PrintFinalScopeResOp) const {
   if (getPrefix())
     getPrefix()->print(OS, Policy);
 
@@ -269,7 +270,8 @@ void NestedNameSpecifier::print(raw_ostream &OS, const PrintingPolicy &Policy,
     break;
 
   case Global:
-    break;
+    OS << "::";
+    return;
 
   case Super:
     OS << "__super";
@@ -331,7 +333,8 @@ void NestedNameSpecifier::print(raw_ostream &OS, const PrintingPolicy &Policy,
   }
   }
 
-  OS << "::";
+  if (PrintFinalScopeResOp)
+    OS << "::";
 }
 
 LLVM_DUMP_METHOD void NestedNameSpecifier::dump(const LangOptions &LO) const {
diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index fd2c38a0e64e7..579ee91b7ec99 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -20,7 +20,8 @@ using namespace clang;
 bool OpenACCClauseWithParams::classof(const OpenACCClause *C) {
   return OpenACCDeviceTypeClause::classof(C) ||
          OpenACCClauseWithCondition::classof(C) ||
-         OpenACCClauseWithExprs::classof(C) || OpenACCSelfClause::classof(C);
+         OpenACCBindClause::classof(C) || OpenACCClauseWithExprs::classof(C) ||
+         OpenACCSelfClause::classof(C);
 }
 bool OpenACCClauseWithExprs::classof(const OpenACCClause *C) {
   return OpenACCWaitClause::classof(C) || OpenACCNumGangsClause::classof(C) ||
@@ -609,6 +610,24 @@ OpenACCIfPresentClause *OpenACCIfPresentClause::Create(const ASTContext &C,
   return new (Mem) OpenACCIfPresentClause(BeginLoc, EndLoc);
 }
 
+OpenACCBindClause *OpenACCBindClause::Create(const ASTContext &C,
+                                             SourceLocation BeginLoc,
+                                             SourceLocation LParenLoc,
+                                             const StringLiteral *SL,
+                                             SourceLocation EndLoc) {
+  void *Mem = C.Allocate(sizeof(OpenACCBindClause), alignof(OpenACCBindClause));
+  return new (Mem) OpenACCBindClause(BeginLoc, LParenLoc, SL, EndLoc);
+}
+
+OpenACCBindClause *OpenACCBindClause::Create(const ASTContext &C,
+                                             SourceLocation BeginLoc,
+                                             SourceLocation LParenLoc,
+                                             const IdentifierInfo *ID,
+                                             SourceLocation EndLoc) {
+  void *Mem = C.Allocate(sizeof(OpenACCBindClause), alignof(OpenACCBindClause));
+  return new (Mem) OpenACCBindClause(BeginLoc, LParenLoc, ID, EndLoc);
+}
+
 //===----------------------------------------------------------------------===//
 //  OpenACC clauses printing methods
 //===----------------------------------------------------------------------===//
@@ -936,3 +955,12 @@ void OpenACCClausePrinter::VisitIfPresentClause(
     const OpenACCIfPresentClause &C) {
   OS << "if_present";
 }
+
+void OpenACCClausePrinter::VisitBindClause(const OpenACCBindClause &C) {
+  OS << "bind(";
+  if (C.isStringArgument())
+    OS << '"' << C.getStringArgument()->getString() << '"';
+  else
+    OS << C.getIdentifierArgument()->getName();
+  OS << ")";
+}
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 424cab3a7de35..648a273c4a2c6 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -155,6 +155,7 @@ const OMPClauseWithPreInit *OMPClauseWithPreInit::get(const OMPClause *C) {
   case OMPC_reverse_offload:
   case OMPC_dynamic_allocators:
   case OMPC_atomic_default_mem_order:
+  case OMPC_self_maps:
   case OMPC_at:
   case OMPC_severity:
   case OMPC_message:
@@ -259,6 +260,7 @@ const OMPClauseWithPostUpdate *OMPClauseWithPostUpdate::get(const OMPClause *C)
   case OMPC_reverse_offload:
   case OMPC_dynamic_allocators:
   case OMPC_atomic_default_mem_order:
+  case OMPC_self_maps:
   case OMPC_at:
   case OMPC_severity:
   case OMPC_message:
@@ -1941,6 +1943,10 @@ void OMPClausePrinter::VisitOMPAtomicDefaultMemOrderClause(
      << ")";
 }
 
+void OMPClausePrinter::VisitOMPSelfMapsClause(OMPSelfMapsClause *) {
+  OS << "self_maps";
+}
+
 void OMPClausePrinter::VisitOMPAtClause(OMPAtClause *Node) {
   OS << "at(" << getOpenMPSimpleClauseTypeName(OMPC_at, Node->getAtKind())
      << ")";
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index facdc4104c374..e0063ec5f25eb 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -2659,10 +2659,8 @@ void StmtPrinter::VisitCXXFoldExpr(CXXFoldExpr *E) {
 }
 
 void StmtPrinter::VisitCXXParenListInitExpr(CXXParenListInitExpr *Node) {
-  OS << "(";
-  llvm::interleaveComma(Node->getInitExprs(), OS,
+  llvm::interleaveComma(Node->getUserSpecifiedInitExprs(), OS,
                         [&](Expr *E) { PrintExpr(E); });
-  OS << ")";
 }
 
 void StmtPrinter::VisitConceptSpecializationExpr(ConceptSpecializationExpr *E) {
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 574f67f4274e7..bf72de3854f4b 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -557,6 +557,8 @@ void OMPClauseProfiler::VisitOMPDynamicAllocatorsClause(
 void OMPClauseProfiler::VisitOMPAtomicDefaultMemOrderClause(
     const OMPAtomicDefaultMemOrderClause *C) {}
 
+void OMPClauseProfiler::VisitOMPSelfMapsClause(const OMPSelfMapsClause *C) {}
+
 void OMPClauseProfiler::VisitOMPAtClause(const OMPAtClause *C) {}
 
 void OMPClauseProfiler::VisitOMPSeverityClause(const OMPSeverityClause *C) {}
@@ -2732,6 +2734,10 @@ void OpenACCClauseProfiler::VisitReductionClause(
     const OpenACCReductionClause &Clause) {
   VisitClauseWithVarList(Clause);
 }
+
+void OpenACCClauseProfiler::VisitBindClause(const OpenACCBindClause &Clause) {
+  assert(false && "not implemented... what can we do about our expr?");
+}
 } // namespace
 
 void StmtProfiler::VisitOpenACCComputeConstruct(
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 91f3f14c6b454..7600b40a7482e 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -435,6 +435,7 @@ void TextNodeDumper::Visit(const OpenACCClause *C) {
     case OpenACCClauseKind::UseDevice:
     case OpenACCClauseKind::Vector:
     case OpenACCClauseKind::VectorLength:
+    case OpenACCClauseKind::Invalid:
       // The condition expression will be printed as a part of the 'children',
       // but print 'clause' here so it is clear what is happening from the dump.
       OS << " clause";
@@ -501,9 +502,15 @@ void TextNodeDumper::Visit(const OpenACCClause *C) {
       OS << " clause Operator: "
          << cast<OpenACCReductionClause>(C)->getReductionOp();
       break;
-    default:
-      // Nothing to do here.
-      break;
+    case OpenACCClauseKind::Bind:
+      OS << " clause";
+      if (cast<OpenACCBindClause>(C)->isIdentifierArgument())
+        OS << " identifier '"
+           << cast<OpenACCBindClause>(C)->getIdentifierArgument()->getName()
+           << "'";
+      else
+        AddChild(
+            [=] { Visit(cast<OpenACCBindClause>(C)->getStringArgument()); });
     }
   }
   dumpPointer(C);
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 2fd7f5800594a..72161c06a88d4 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -410,6 +410,12 @@ VectorType::VectorType(TypeClass tc, QualType vecType, unsigned nElements,
   VectorTypeBits.NumElements = nElements;
 }
 
+bool Type::isPackedVectorBoolType(const ASTContext &ctx) const {
+  if (ctx.getLangOpts().HLSL)
+    return false;
+  return isExtVectorBoolType();
+}
+
 BitIntType::BitIntType(bool IsUnsigned, unsigned NumBits)
     : Type(BitInt, QualType{}, TypeDependence::None), IsUnsigned(IsUnsigned),
       NumBits(NumBits) {}
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 8762cc7b1e4e1..dfd1959bbdcb0 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -2093,6 +2093,9 @@ void TypePrinter::printAttributedAfter(const AttributedType *T,
   case attr::ArmMveStrictPolymorphism:
     OS << "__clang_arm_mve_strict_polymorphism";
     break;
+  case attr::ExtVectorType:
+    OS << "ext_vector_type";
+    break;
   }
   OS << "))";
 }
diff --git a/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp b/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp
index 9381c5c42e566..c28424fac8fef 100644
--- a/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp
+++ b/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp
@@ -551,83 +551,92 @@ void transferCallReturningOptional(const CallExpr *E,
   setHasValue(*Loc, State.Env.makeAtomicBoolValue(), State.Env);
 }
 
-void handleConstMemberCall(const CallExpr *CE,
+// Returns true if the const accessor is handled by caching.
+// Returns false if we could not cache. We should perform default handling
+// in that case.
+bool handleConstMemberCall(const CallExpr *CE,
                            dataflow::RecordStorageLocation *RecordLoc,
                            const MatchFinder::MatchResult &Result,
                            LatticeTransferState &State) {
-  // If the const method returns an optional or reference to an optional.
-  if (RecordLoc != nullptr && isSupportedOptionalType(CE->getType())) {
-    const FunctionDecl *DirectCallee = CE->getDirectCallee();
-    if (DirectCallee == nullptr)
-      return;
-    StorageLocation &Loc =
-        State.Lattice.getOrCreateConstMethodReturnStorageLocation(
-            *RecordLoc, DirectCallee, State.Env, [&](StorageLocation &Loc) {
-              setHasValue(cast<RecordStorageLocation>(Loc),
-                          State.Env.makeAtomicBoolValue(), State.Env);
-            });
-    if (CE->isGLValue()) {
-      // If the call to the const method returns a reference to an optional,
-      // link the call expression to the cached StorageLocation.
-      State.Env.setStorageLocation(*CE, Loc);
-    } else {
-      // If the call to the const method returns an optional by value, we
-      // need to use CopyRecord to link the optional to the result object
-      // of the call expression.
-      auto &ResultLoc = State.Env.getResultObjectLocation(*CE);
-      copyRecord(cast<RecordStorageLocation>(Loc), ResultLoc, State.Env);
-    }
-    return;
-  }
+  if (RecordLoc == nullptr)
+    return false;
 
-  // Cache if the const method returns a reference
-  if (RecordLoc != nullptr && CE->isGLValue()) {
+  // Cache if the const method returns a reference.
+  if (CE->isGLValue()) {
     const FunctionDecl *DirectCallee = CE->getDirectCallee();
     if (DirectCallee == nullptr)
-      return;
+      return false;
 
+    // Initialize the optional's "has_value" property to true if the type is
+    // optional, otherwise no-op. If we want to support const ref to pointers or
+    // bools we should initialize their values here too.
+    auto Init = [&](StorageLocation &Loc) {
+      if (isSupportedOptionalType(CE->getType()))
+        setHasValue(cast<RecordStorageLocation>(Loc),
+                    State.Env.makeAtomicBoolValue(), State.Env);
+    };
     StorageLocation &Loc =
         State.Lattice.getOrCreateConstMethodReturnStorageLocation(
-            *RecordLoc, DirectCallee, State.Env, [&](StorageLocation &Loc) {
-              // no-op
-            });
+            *RecordLoc, DirectCallee, State.Env, Init);
 
     State.Env.setStorageLocation(*CE, Loc);
-    return;
+    return true;
   }
-
-  // Cache if the const method returns a boolean or pointer type.
-  // We may decide to cache other return types in the future.
-  if (RecordLoc != nullptr &&
-      (CE->getType()->isBooleanType() || CE->getType()->isPointerType())) {
+  // PRValue cases:
+  if (CE->getType()->isBooleanType() || CE->getType()->isPointerType()) {
+    // If the const method returns a boolean or pointer type.
     Value *Val = State.Lattice.getOrCreateConstMethodReturnValue(*RecordLoc, CE,
                                                                  State.Env);
     if (Val == nullptr)
-      return;
+      return false;
     State.Env.setValue(*CE, *Val);
-    return;
+    return true;
   }
-
-  // Perform default handling if the call returns an optional
-  // but wasn't handled above (if RecordLoc is nullptr).
   if (isSupportedOptionalType(CE->getType())) {
-    transferCallReturningOptional(CE, Result, State);
+    // If the const method returns an optional by value.
+    const FunctionDecl *DirectCallee = CE->getDirectCallee();
+    if (DirectCallee == nullptr)
+      return false;
+    StorageLocation &Loc =
+        State.Lattice.getOrCreateConstMethodReturnStorageLocation(
+            *RecordLoc, DirectCallee, State.Env, [&](StorageLocation &Loc) {
+              setHasValue(cast<RecordStorageLocation>(Loc),
+                          State.Env.makeAtomicBoolValue(), State.Env);
+            });
+    // Use copyRecord to link the optional to the result object of the call
+    // expression.
+    auto &ResultLoc = State.Env.getResultObjectLocation(*CE);
+    copyRecord(cast<RecordStorageLocation>(Loc), ResultLoc, State.Env);
+    return true;
   }
+
+  return false;
 }
 
-void transferValue_ConstMemberCall(const CXXMemberCallExpr *MCE,
-                                   const MatchFinder::MatchResult &Result,
-                                   LatticeTransferState &State) {
-  handleConstMemberCall(
+void handleConstMemberCallWithFallbacks(
+    const CallExpr *CE, dataflow::RecordStorageLocation *RecordLoc,
+    const MatchFinder::MatchResult &Result, LatticeTransferState &State) {
+  if (handleConstMemberCall(CE, RecordLoc, Result, State))
+    return;
+  // Perform default handling if the call returns an optional, but wasn't
+  // handled by caching.
+  if (isSupportedOptionalType(CE->getType()))
+    transferCallReturningOptional(CE, Result, State);
+}
+
+void transferConstMemberCall(const CXXMemberCallExpr *MCE,
+                             const MatchFinder::MatchResult &Result,
+                             LatticeTransferState &State) {
+  handleConstMemberCallWithFallbacks(
       MCE, dataflow::getImplicitObjectLocation(*MCE, State.Env), Result, State);
 }
 
-void transferValue_ConstMemberOperatorCall(
-    const CXXOperatorCallExpr *OCE, const MatchFinder::MatchResult &Result,
-    LatticeTransferState &State) {
+void transferConstMemberOperatorCall(const CXXOperatorCallExpr *OCE,
+                                     const MatchFinder::MatchResult &Result,
+                                     LatticeTransferState &State) {
   auto *RecordLoc = cast_or_null<dataflow::RecordStorageLocation>(
       State.Env.getStorageLocation(*OCE->getArg(0)));
-  handleConstMemberCall(OCE, RecordLoc, Result, State);
+  handleConstMemberCallWithFallbacks(OCE, RecordLoc, Result, State);
 }
 
 void handleNonConstMemberCall(const CallExpr *CE,
@@ -1094,9 +1103,9 @@ auto buildTransferMatchSwitch() {
 
       // const accessor calls
       .CaseOfCFGStmt<CXXMemberCallExpr>(isZeroParamConstMemberCall(),
-                                        transferValue_ConstMemberCall)
+                                        transferConstMemberCall)
       .CaseOfCFGStmt<CXXOperatorCallExpr>(isZeroParamConstMemberOperatorCall(),
-                                          transferValue_ConstMemberOperatorCall)
+                                          transferConstMemberOperatorCall)
       // non-const member calls that may modify the state of an object.
       .CaseOfCFGStmt<CXXMemberCallExpr>(isNonConstMemberCall(),
                                         transferValue_NonConstMemberCall)
diff --git a/clang/lib/Analysis/UnsafeBufferUsage.cpp b/clang/lib/Analysis/UnsafeBufferUsage.cpp
index 12e99143cb148..89b16c9c3179a 100644
--- a/clang/lib/Analysis/UnsafeBufferUsage.cpp
+++ b/clang/lib/Analysis/UnsafeBufferUsage.cpp
@@ -7,7 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Analysis/Analyses/UnsafeBufferUsage.h"
+#include "clang/AST/APValue.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DynamicRecursiveASTVisitor.h"
 #include "clang/AST/Expr.h"
@@ -353,23 +355,90 @@ isInUnspecifiedUntypedContext(internal::Matcher<Stmt> InnerMatcher) {
   return stmt(anyOf(CompStmt, IfStmtThen, IfStmtElse));
 }
 
+// Returns true iff integer E1 is equivalent to integer E2.
+//
+// For now we only support such expressions:
+//    expr := DRE | const-value | expr BO expr
+//    BO   := '*' | '+'
+//
+// FIXME: We can reuse the expression comparator of the interop analysis after
+// it has been upstreamed.
+static bool areEqualIntegers(const Expr *E1, const Expr *E2, ASTContext &Ctx);
+static bool areEqualIntegralBinaryOperators(const BinaryOperator *E1,
+                                            const Expr *E2_LHS,
+                                            BinaryOperatorKind BOP,
+                                            const Expr *E2_RHS,
+                                            ASTContext &Ctx) {
+  if (E1->getOpcode() == BOP) {
+    switch (BOP) {
+      // Commutative operators:
+    case BO_Mul:
+    case BO_Add:
+      return (areEqualIntegers(E1->getLHS(), E2_LHS, Ctx) &&
+              areEqualIntegers(E1->getRHS(), E2_RHS, Ctx)) ||
+             (areEqualIntegers(E1->getLHS(), E2_RHS, Ctx) &&
+              areEqualIntegers(E1->getRHS(), E2_LHS, Ctx));
+    default:
+      return false;
+    }
+  }
+  return false;
+}
+
+static bool areEqualIntegers(const Expr *E1, const Expr *E2, ASTContext &Ctx) {
+  E1 = E1->IgnoreParenImpCasts();
+  E2 = E2->IgnoreParenImpCasts();
+  if (!E1->getType()->isIntegerType() || E1->getType() != E2->getType())
+    return false;
+
+  Expr::EvalResult ER1, ER2;
+
+  // If both are constants:
+  if (E1->EvaluateAsInt(ER1, Ctx) &&
+      E2->EvaluateAsInt(ER2, Ctx))
+    return ER1.Val.getInt() == ER2.Val.getInt();
+
+  // Otherwise, they should have identical stmt kind:
+  if (E1->getStmtClass() != E2->getStmtClass())
+    return false;
+  switch (E1->getStmtClass()) {
+  case Stmt::DeclRefExprClass:
+    return cast<DeclRefExpr>(E1)->getDecl() == cast<DeclRefExpr>(E2)->getDecl();
+  case Stmt::BinaryOperatorClass: {
+    auto BO2 = cast<BinaryOperator>(E2);
+    return areEqualIntegralBinaryOperators(cast<BinaryOperator>(E1),
+                                           BO2->getLHS(), BO2->getOpcode(),
+                                           BO2->getRHS(), Ctx);
+  }
+  default:
+    return false;
+  }
+}
+
 // Given a two-param std::span construct call, matches iff the call has the
 // following forms:
 //   1. `std::span<T>{new T[n], n}`, where `n` is a literal or a DRE
 //   2. `std::span<T>{new T, 1}`
-//   3. `std::span<T>{&var, 1}`
+//   3. `std::span<T>{&var, 1}` or `std::span<T>{std::addressof(...), 1}`
 //   4. `std::span<T>{a, n}`, where `a` is of an array-of-T with constant size
 //   `n`
 //   5. `std::span<T>{any, 0}`
-//   6. `std::span<T>{std::addressof(...), 1}`
+//   6. `std::span<T>{ (char *)f(args), args[N] * arg*[M]}`, where
+//       `f` is a function with attribute `alloc_size(N, M)`;
+//       `args` represents the list of arguments;
+//       `N, M` are parameter indexes to the allocating element number and size.
+//        Sometimes, there is only one parameter index representing the total
+//        size.
 AST_MATCHER(CXXConstructExpr, isSafeSpanTwoParamConstruct) {
   assert(Node.getNumArgs() == 2 &&
          "expecting a two-parameter std::span constructor");
-  const Expr *Arg0 = Node.getArg(0)->IgnoreImplicit();
-  const Expr *Arg1 = Node.getArg(1)->IgnoreImplicit();
-  auto HaveEqualConstantValues = [&Finder](const Expr *E0, const Expr *E1) {
-    if (auto E0CV = E0->getIntegerConstantExpr(Finder->getASTContext()))
-      if (auto E1CV = E1->getIntegerConstantExpr(Finder->getASTContext())) {
+  const Expr *Arg0 = Node.getArg(0)->IgnoreParenImpCasts();
+  const Expr *Arg1 = Node.getArg(1)->IgnoreParenImpCasts();
+  ASTContext &Ctx = Finder->getASTContext();
+
+  auto HaveEqualConstantValues = [&Ctx](const Expr *E0, const Expr *E1) {
+    if (auto E0CV = E0->getIntegerConstantExpr(Ctx))
+      if (auto E1CV = E1->getIntegerConstantExpr(Ctx)) {
         return APSInt::compareValues(*E0CV, *E1CV) == 0;
       }
     return false;
@@ -381,13 +450,14 @@ AST_MATCHER(CXXConstructExpr, isSafeSpanTwoParamConstruct) {
       }
     return false;
   };
-  std::optional<APSInt> Arg1CV =
-      Arg1->getIntegerConstantExpr(Finder->getASTContext());
+  std::optional<APSInt> Arg1CV = Arg1->getIntegerConstantExpr(Ctx);
 
   if (Arg1CV && Arg1CV->isZero())
     // Check form 5:
     return true;
-  switch (Arg0->IgnoreImplicit()->getStmtClass()) {
+
+  // Check forms 1-3:
+  switch (Arg0->getStmtClass()) {
   case Stmt::CXXNewExprClass:
     if (auto Size = cast<CXXNewExpr>(Arg0)->getArraySize()) {
       // Check form 1:
@@ -407,6 +477,7 @@ AST_MATCHER(CXXConstructExpr, isSafeSpanTwoParamConstruct) {
       return Arg1CV && Arg1CV->isOne();
     break;
   case Stmt::CallExprClass:
+    // Check form 3:
     if (const auto *CE = dyn_cast<CallExpr>(Arg0)) {
       const auto FnDecl = CE->getDirectCallee();
       if (FnDecl && FnDecl->getNameAsString() == "addressof" &&
@@ -421,13 +492,41 @@ AST_MATCHER(CXXConstructExpr, isSafeSpanTwoParamConstruct) {
 
   QualType Arg0Ty = Arg0->IgnoreImplicit()->getType();
 
-  if (auto *ConstArrTy =
-          Finder->getASTContext().getAsConstantArrayType(Arg0Ty)) {
+  if (auto *ConstArrTy = Ctx.getAsConstantArrayType(Arg0Ty)) {
     const APSInt ConstArrSize = APSInt(ConstArrTy->getSize());
 
     // Check form 4:
     return Arg1CV && APSInt::compareValues(ConstArrSize, *Arg1CV) == 0;
   }
+  // Check form 6:
+  if (auto CCast = dyn_cast<CStyleCastExpr>(Arg0)) {
+    if (!CCast->getType()->isPointerType())
+      return false;
+
+    QualType PteTy = CCast->getType()->getPointeeType();
+
+    if (!(PteTy->isConstantSizeType() && Ctx.getTypeSizeInChars(PteTy).isOne()))
+      return false;
+
+    if (const auto *Call = dyn_cast<CallExpr>(CCast->getSubExpr())) {
+      if (const FunctionDecl *FD = Call->getDirectCallee())
+        if (auto *AllocAttr = FD->getAttr<AllocSizeAttr>()) {
+          const Expr *EleSizeExpr =
+              Call->getArg(AllocAttr->getElemSizeParam().getASTIndex());
+          // NumElemIdx is invalid if AllocSizeAttr has 1 argument:
+          ParamIdx NumElemIdx = AllocAttr->getNumElemsParam();
+
+          if (!NumElemIdx.isValid())
+            return areEqualIntegers(Arg1, EleSizeExpr, Ctx);
+
+          const Expr *NumElesExpr = Call->getArg(NumElemIdx.getASTIndex());
+
+          if (auto BO = dyn_cast<BinaryOperator>(Arg1))
+            return areEqualIntegralBinaryOperators(BO, NumElesExpr, BO_Mul,
+                                                   EleSizeExpr, Ctx);
+        }
+    }
+  }
   return false;
 }
 
diff --git a/clang/lib/Basic/Attributes.cpp b/clang/lib/Basic/Attributes.cpp
index 2035d4c0a5768..6a070a99c8d96 100644
--- a/clang/lib/Basic/Attributes.cpp
+++ b/clang/lib/Basic/Attributes.cpp
@@ -143,13 +143,17 @@ static SmallString<64> normalizeName(const IdentifierInfo *Name,
   StringRef ScopeName = normalizeAttrScopeName(Scope, SyntaxUsed);
   StringRef AttrName = normalizeAttrName(Name, ScopeName, SyntaxUsed);
 
+  std::string StrAttrName = AttrName.str();
+  if (SyntaxUsed == AttributeCommonInfo::AS_HLSLAnnotation)
+    StrAttrName = AttrName.lower();
+
   SmallString<64> FullName = ScopeName;
   if (!ScopeName.empty()) {
     assert(SyntaxUsed == AttributeCommonInfo::AS_CXX11 ||
            SyntaxUsed == AttributeCommonInfo::AS_C23);
     FullName += "::";
   }
-  FullName += AttrName;
+  FullName += StrAttrName;
 
   return FullName;
 }
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 1ff342cb22a03..09921e3b1edfc 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -235,6 +235,7 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind, StringRef Str,
   case OMPC_unified_shared_memory:
   case OMPC_reverse_offload:
   case OMPC_dynamic_allocators:
+  case OMPC_self_maps:
   case OMPC_match:
   case OMPC_nontemporal:
   case OMPC_destroy:
@@ -569,6 +570,7 @@ const char *clang::getOpenMPSimpleClauseTypeName(OpenMPClauseKind Kind,
   case OMPC_unified_shared_memory:
   case OMPC_reverse_offload:
   case OMPC_dynamic_allocators:
+  case OMPC_self_maps:
   case OMPC_match:
   case OMPC_nontemporal:
   case OMPC_destroy:
diff --git a/clang/lib/CIR/CMakeLists.txt b/clang/lib/CIR/CMakeLists.txt
index 4a99ecb33dfb2..7bdf3fcc59035 100644
--- a/clang/lib/CIR/CMakeLists.txt
+++ b/clang/lib/CIR/CMakeLists.txt
@@ -1,6 +1,17 @@
 include_directories(${LLVM_MAIN_SRC_DIR}/../mlir/include)
 include_directories(${CMAKE_BINARY_DIR}/tools/mlir/include)
 
+# GCC, unlike clang, issues a warning when one virtual function is overridden
+# in a derived class but one or more other virtual functions with the same
+# name and different signature from a base class are not overridden. This
+# leads to many warnings in the MLIR and ClangIR code when using the
+# OpenConversionPattern<>::matchAndRewrite() function in the ordinary way.
+# The "hiding" behavior is what we want, so we're just disabling the warning
+# here.
+if (LLVM_COMPILER_IS_GCC_COMPATIBLE AND (NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang"))
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-overloaded-virtual")
+endif()
+
 add_subdirectory(Dialect)
 add_subdirectory(CodeGen)
 add_subdirectory(FrontendAction)
diff --git a/clang/lib/CIR/CodeGen/CIRGenConstantEmitter.h b/clang/lib/CIR/CodeGen/CIRGenConstantEmitter.h
new file mode 100644
index 0000000000000..5b22a8e59908d
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenConstantEmitter.h
@@ -0,0 +1,100 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A helper class for emitting expressions and values as cir::ConstantOp
+// and as initializers for global variables.
+//
+// Note: this is based on clang's LLVM IR codegen in ConstantEmitter.h, reusing
+// this class interface makes it easier move forward with bringing CIR codegen
+// to completion.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_LIB_CIR_CODEGEN_CIRGENCONSTANTEMITTER_H
+#define CLANG_LIB_CIR_CODEGEN_CIRGENCONSTANTEMITTER_H
+
+#include "CIRGenFunction.h"
+#include "CIRGenModule.h"
+#include "llvm/ADT/SmallVector.h"
+
+namespace clang::CIRGen {
+
+class ConstantEmitter {
+public:
+  CIRGenModule &cgm;
+  const CIRGenFunction *cgf;
+
+private:
+  bool abstract = false;
+
+  /// Whether we're in a constant context.
+  bool inConstantContext = false;
+
+public:
+  /// Initialize this emission in the context of the given function.
+  /// Use this if the expression might contain contextual references like
+  /// block addresses or PredefinedExprs.
+  ConstantEmitter(CIRGenFunction &cgf) : cgm(cgf.cgm), cgf(&cgf) {}
+
+  ConstantEmitter(const ConstantEmitter &other) = delete;
+  ConstantEmitter &operator=(const ConstantEmitter &other) = delete;
+
+  // All of the "abstract" emission methods below permit the emission to
+  // be immediately discarded without finalizing anything.  Therefore, they
+  // must also promise not to do anything that will, in the future, require
+  // finalization:
+  //
+  //   - using the CGF (if present) for anything other than establishing
+  //     semantic context; for example, an expression with ignored
+  //     side-effects must not be emitted as an abstract expression
+  //
+  //   - doing anything that would not be safe to duplicate within an
+  //     initializer or to propagate to another context; for example,
+  //     side effects, or emitting an initialization that requires a
+  //     reference to its current location.
+  mlir::Attribute emitForMemory(mlir::Attribute c, QualType t);
+
+  /// Emit the result of the given expression as an abstract constant,
+  /// asserting that it succeeded.  This is only safe to do when the
+  /// expression is known to be a constant expression with either a fairly
+  /// simple type or a known simple form.
+  mlir::Attribute emitAbstract(SourceLocation loc, const APValue &value,
+                               QualType t);
+
+  mlir::Attribute tryEmitConstantExpr(const ConstantExpr *CE);
+
+  // These are private helper routines of the constant emitter that
+  // can't actually be private because things are split out into helper
+  // functions and classes.
+
+  mlir::Attribute tryEmitPrivateForVarInit(const VarDecl &d);
+
+  mlir::Attribute tryEmitPrivate(const APValue &value, QualType destType);
+  mlir::Attribute tryEmitPrivateForMemory(const APValue &value, QualType t);
+
+  /// Try to emit the initializer of the given declaration as an abstract
+  /// constant.
+  mlir::Attribute tryEmitAbstractForInitializer(const VarDecl &d);
+
+private:
+  class AbstractStateRAII {
+    ConstantEmitter &emitter;
+    bool oldValue;
+
+  public:
+    AbstractStateRAII(ConstantEmitter &emitter, bool value)
+        : emitter(emitter), oldValue(emitter.abstract) {
+      emitter.abstract = value;
+    }
+    ~AbstractStateRAII() { emitter.abstract = oldValue; }
+  };
+};
+
+} // namespace clang::CIRGen
+
+#endif // CLANG_LIB_CIR_CODEGEN_CIRGENCONSTANTEMITTER_H
diff --git a/clang/lib/CIR/CodeGen/CIRGenDecl.cpp b/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
index 406026b0b9f27..27ed0113a4f55 100644
--- a/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
@@ -10,23 +10,29 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "CIRGenConstantEmitter.h"
 #include "CIRGenFunction.h"
+#include "mlir/IR/Location.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/Expr.h"
+#include "clang/AST/ExprCXX.h"
 #include "clang/CIR/MissingFeatures.h"
 
 using namespace clang;
 using namespace clang::CIRGen;
 
-void CIRGenFunction::emitAutoVarAlloca(const VarDecl &d) {
+CIRGenFunction::AutoVarEmission
+CIRGenFunction::emitAutoVarAlloca(const VarDecl &d) {
   QualType ty = d.getType();
   if (ty.getAddressSpace() != LangAS::Default)
     cgm.errorNYI(d.getSourceRange(), "emitAutoVarAlloca: address space");
 
-  auto loc = getLoc(d.getSourceRange());
+  mlir::Location loc = getLoc(d.getSourceRange());
 
-  if (d.isEscapingByref())
+  CIRGenFunction::AutoVarEmission emission(d);
+  emission.IsEscapingByRef = d.isEscapingByref();
+  if (emission.IsEscapingByRef)
     cgm.errorNYI(d.getSourceRange(),
                  "emitAutoVarDecl: decl escaping by reference");
 
@@ -46,21 +52,130 @@ void CIRGenFunction::emitAutoVarAlloca(const VarDecl &d) {
   address = createTempAlloca(allocaTy, alignment, loc, d.getName());
   declare(address.getPointer(), &d, ty, getLoc(d.getSourceRange()), alignment);
 
+  emission.Addr = address;
   setAddrOfLocalVar(&d, address);
+
+  return emission;
+}
+
+/// Determine whether the given initializer is trivial in the sense
+/// that it requires no code to be generated.
+bool CIRGenFunction::isTrivialInitializer(const Expr *init) {
+  if (!init)
+    return true;
+
+  if (const CXXConstructExpr *construct = dyn_cast<CXXConstructExpr>(init))
+    if (CXXConstructorDecl *constructor = construct->getConstructor())
+      if (constructor->isTrivial() && constructor->isDefaultConstructor() &&
+          !construct->requiresZeroInitialization())
+        return true;
+
+  return false;
 }
 
-void CIRGenFunction::emitAutoVarInit(const clang::VarDecl &d) {
+void CIRGenFunction::emitAutoVarInit(
+    const CIRGenFunction::AutoVarEmission &emission) {
+  assert(emission.Variable && "emission was not valid!");
+
+  // If this was emitted as a global constant, we're done.
+  if (emission.wasEmittedAsGlobal())
+    return;
+
+  const VarDecl &d = *emission.Variable;
+
   QualType type = d.getType();
 
   // If this local has an initializer, emit it now.
   const Expr *init = d.getInit();
 
-  if (init || !type.isPODType(getContext())) {
-    cgm.errorNYI(d.getSourceRange(), "emitAutoVarInit");
+  if (!type.isPODType(getContext())) {
+    cgm.errorNYI(d.getSourceRange(), "emitAutoVarInit: non-POD type");
+    return;
+  }
+
+  const Address addr = emission.Addr;
+
+  // Check whether this is a byref variable that's potentially
+  // captured and moved by its own initializer.  If so, we'll need to
+  // emit the initializer first, then copy into the variable.
+  assert(!cir::MissingFeatures::opAllocaCaptureByInit());
+
+  // Note: constexpr already initializes everything correctly.
+  LangOptions::TrivialAutoVarInitKind trivialAutoVarInit =
+      (d.isConstexpr()
+           ? LangOptions::TrivialAutoVarInitKind::Uninitialized
+           : (d.getAttr<UninitializedAttr>()
+                  ? LangOptions::TrivialAutoVarInitKind::Uninitialized
+                  : getContext().getLangOpts().getTrivialAutoVarInit()));
+
+  auto initializeWhatIsTechnicallyUninitialized = [&](Address addr) {
+    if (trivialAutoVarInit ==
+        LangOptions::TrivialAutoVarInitKind::Uninitialized)
+      return;
+
+    cgm.errorNYI(d.getSourceRange(), "emitAutoVarInit: trivial initialization");
+  };
+
+  if (isTrivialInitializer(init)) {
+    initializeWhatIsTechnicallyUninitialized(addr);
+    return;
+  }
+
+  mlir::Attribute constant;
+  if (emission.IsConstantAggregate ||
+      d.mightBeUsableInConstantExpressions(getContext())) {
+    // FIXME: Differently from LLVM we try not to emit / lower too much
+    // here for CIR since we are interested in seeing the ctor in some
+    // analysis later on. So CIR's implementation of ConstantEmitter will
+    // frequently return an empty Attribute, to signal we want to codegen
+    // some trivial ctor calls and whatnots.
+    constant = ConstantEmitter(*this).tryEmitAbstractForInitializer(d);
+    if (constant && !mlir::isa<cir::ZeroAttr>(constant) &&
+        (trivialAutoVarInit !=
+         LangOptions::TrivialAutoVarInitKind::Uninitialized)) {
+      cgm.errorNYI(d.getSourceRange(), "emitAutoVarInit: constant aggregate");
+      return;
+    }
+  }
+
+  // NOTE(cir): In case we have a constant initializer, we can just emit a
+  // store. But, in CIR, we wish to retain any ctor calls, so if it is a
+  // CXX temporary object creation, we ensure the ctor call is used deferring
+  // its removal/optimization to the CIR lowering.
+  if (!constant || isa<CXXTemporaryObjectExpr>(init)) {
+    initializeWhatIsTechnicallyUninitialized(addr);
+    LValue lv = LValue::makeAddr(addr, type);
+    emitExprAsInit(init, &d, lv);
+    // In case lv has uses it means we indeed initialized something
+    // out of it while trying to build the expression, mark it as such.
+    mlir::Value val = lv.getAddress().getPointer();
+    assert(val && "Should have an address");
+    auto allocaOp = dyn_cast_or_null<cir::AllocaOp>(val.getDefiningOp());
+    assert(allocaOp && "Address should come straight out of the alloca");
+
+    if (!allocaOp.use_empty())
+      allocaOp.setInitAttr(mlir::UnitAttr::get(&getMLIRContext()));
+    return;
+  }
+
+  // FIXME(cir): migrate most of this file to use mlir::TypedAttr directly.
+  auto typedConstant = mlir::dyn_cast<mlir::TypedAttr>(constant);
+  assert(typedConstant && "expected typed attribute");
+  if (!emission.IsConstantAggregate) {
+    // For simple scalar/complex initialization, store the value directly.
+    LValue lv = LValue::makeAddr(addr, type);
+    assert(init && "expected initializer");
+    mlir::Location initLoc = getLoc(init->getSourceRange());
+    // lv.setNonGC(true);
+    return emitStoreThroughLValue(
+        RValue::get(builder.getConstant(initLoc, typedConstant)), lv);
   }
 }
 
-void CIRGenFunction::emitAutoVarCleanups(const clang::VarDecl &d) {
+void CIRGenFunction::emitAutoVarCleanups(
+    const CIRGenFunction::AutoVarEmission &emission) {
+  const VarDecl &d = *emission.Variable;
+
   // Check the type for a cleanup.
   if (d.needsDestruction(getContext()))
     cgm.errorNYI(d.getSourceRange(), "emitAutoVarCleanups: type cleanup");
@@ -76,9 +191,9 @@ void CIRGenFunction::emitAutoVarCleanups(const clang::VarDecl &d) {
 /// register, or no storage class specifier. These turn into simple stack
 /// objects, globals depending on target.
 void CIRGenFunction::emitAutoVarDecl(const VarDecl &d) {
-  emitAutoVarAlloca(d);
-  emitAutoVarInit(d);
-  emitAutoVarCleanups(d);
+  CIRGenFunction::AutoVarEmission emission = emitAutoVarAlloca(d);
+  emitAutoVarInit(emission);
+  emitAutoVarCleanups(emission);
 }
 
 void CIRGenFunction::emitVarDecl(const VarDecl &d) {
@@ -94,10 +209,54 @@ void CIRGenFunction::emitVarDecl(const VarDecl &d) {
 
   assert(d.hasLocalStorage());
 
-  assert(!cir::MissingFeatures::opAllocaVarDeclContext());
+  CIRGenFunction::VarDeclContext varDeclCtx{*this, &d};
   return emitAutoVarDecl(d);
 }
 
+void CIRGenFunction::emitScalarInit(const Expr *init, mlir::Location loc,
+                                    LValue lvalue, bool capturedByInit) {
+  assert(!cir::MissingFeatures::objCLifetime());
+
+  SourceLocRAIIObject locRAII{*this, loc};
+  mlir::Value value = emitScalarExpr(init);
+  if (capturedByInit) {
+    cgm.errorNYI(init->getSourceRange(), "emitScalarInit: captured by init");
+    return;
+  }
+  assert(!cir::MissingFeatures::emitNullabilityCheck());
+  emitStoreThroughLValue(RValue::get(value), lvalue, true);
+  return;
+}
+
+void CIRGenFunction::emitExprAsInit(const Expr *init, const ValueDecl *d,
+                                    LValue lvalue, bool capturedByInit) {
+  SourceLocRAIIObject loc{*this, getLoc(init->getSourceRange())};
+  if (capturedByInit) {
+    cgm.errorNYI(init->getSourceRange(), "emitExprAsInit: captured by init");
+    return;
+  }
+
+  QualType type = d->getType();
+
+  if (type->isReferenceType()) {
+    cgm.errorNYI(init->getSourceRange(), "emitExprAsInit: reference type");
+    return;
+  }
+  switch (CIRGenFunction::getEvaluationKind(type)) {
+  case cir::TEK_Scalar:
+    emitScalarInit(init, getLoc(d->getSourceRange()), lvalue);
+    return;
+  case cir::TEK_Complex: {
+    cgm.errorNYI(init->getSourceRange(), "emitExprAsInit: complex type");
+    return;
+  }
+  case cir::TEK_Aggregate:
+    cgm.errorNYI(init->getSourceRange(), "emitExprAsInit: aggregate type");
+    return;
+  }
+  llvm_unreachable("bad evaluation kind");
+}
+
 void CIRGenFunction::emitDecl(const Decl &d) {
   switch (d.getKind()) {
   case Decl::Var: {
diff --git a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
index ccc3e20875263..07fb4cf8f1513 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
@@ -25,9 +25,77 @@ using namespace clang;
 using namespace clang::CIRGen;
 using namespace cir;
 
+void CIRGenFunction::emitStoreThroughLValue(RValue src, LValue dst,
+                                            bool isInit) {
+  if (!dst.isSimple()) {
+    cgm.errorNYI(dst.getPointer().getLoc(),
+                 "emitStoreThroughLValue: non-simple lvalue");
+    return;
+  }
+
+  assert(!cir::MissingFeatures::opLoadStoreObjC());
+
+  assert(src.isScalar() && "Can't emit an aggregate store with this method");
+  emitStoreOfScalar(src.getScalarVal(), dst, isInit);
+}
+
+void CIRGenFunction::emitStoreOfScalar(mlir::Value value, Address addr,
+                                       bool isVolatile, QualType ty,
+                                       bool isInit, bool isNontemporal) {
+  assert(!cir::MissingFeatures::opLoadStoreThreadLocal());
+
+  if (ty->getAs<clang::VectorType>()) {
+    cgm.errorNYI(addr.getPointer().getLoc(), "emitStoreOfScalar vector type");
+    return;
+  }
+
+  value = emitToMemory(value, ty);
+
+  assert(!cir::MissingFeatures::opLoadStoreAtomic());
+
+  // Update the alloca with more info on initialization.
+  assert(addr.getPointer() && "expected pointer to exist");
+  auto srcAlloca =
+      dyn_cast_or_null<cir::AllocaOp>(addr.getPointer().getDefiningOp());
+  if (currVarDecl && srcAlloca) {
+    const VarDecl *vd = currVarDecl;
+    assert(vd && "VarDecl expected");
+    if (vd->hasInit())
+      srcAlloca.setInitAttr(mlir::UnitAttr::get(&getMLIRContext()));
+  }
+
+  assert(currSrcLoc && "must pass in source location");
+  builder.createStore(*currSrcLoc, value, addr.getPointer() /*, isVolatile*/);
+
+  if (isNontemporal) {
+    cgm.errorNYI(addr.getPointer().getLoc(), "emitStoreOfScalar nontemporal");
+    return;
+  }
+
+  assert(!cir::MissingFeatures::opTBAA());
+}
+
+mlir::Value CIRGenFunction::emitToMemory(mlir::Value value, QualType ty) {
+  // Bool has a different representation in memory than in registers,
+  // but in ClangIR, it is simply represented as a cir.bool value.
+  // This function is here as a placeholder for possible future changes.
+  return value;
+}
+
+void CIRGenFunction::emitStoreOfScalar(mlir::Value value, LValue lvalue,
+                                       bool isInit) {
+  if (lvalue.getType()->isConstantMatrixType()) {
+    assert(0 && "NYI: emitStoreOfScalar constant matrix type");
+    return;
+  }
+
+  emitStoreOfScalar(value, lvalue.getAddress(), lvalue.isVolatile(),
+                    lvalue.getType(), isInit, /*isNontemporal=*/false);
+}
+
 mlir::Value CIRGenFunction::emitLoadOfScalar(LValue lvalue,
                                              SourceLocation loc) {
-  assert(!cir::MissingFeatures::opLoadThreadLocal());
+  assert(!cir::MissingFeatures::opLoadStoreThreadLocal());
   assert(!cir::MissingFeatures::opLoadEmitScalarRangeCheck());
   assert(!cir::MissingFeatures::opLoadBooleanRepresentation());
 
@@ -115,7 +183,7 @@ mlir::Value CIRGenFunction::emitAlloca(StringRef name, mlir::Type ty,
     builder.restoreInsertionPoint(builder.getBestAllocaInsertPoint(entryBlock));
     addr = builder.createAlloca(loc, /*addr type*/ localVarPtrTy,
                                 /*var type*/ ty, name, alignIntAttr);
-    assert(!cir::MissingFeatures::opAllocaVarDeclContext());
+    assert(!cir::MissingFeatures::astVarDeclInterface());
   }
   return addr;
 }
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
new file mode 100644
index 0000000000000..1ea7f6212766c
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
@@ -0,0 +1,309 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This contains code to emit Constant Expr nodes as LLVM code.
+//
+//===----------------------------------------------------------------------===//
+
+#include "Address.h"
+#include "CIRGenConstantEmitter.h"
+#include "CIRGenFunction.h"
+#include "CIRGenModule.h"
+#include "mlir/IR/Attributes.h"
+#include "mlir/IR/BuiltinAttributeInterfaces.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "clang/AST/APValue.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Attr.h"
+#include "clang/AST/OperationKinds.h"
+#include "clang/AST/RecordLayout.h"
+#include "clang/AST/StmtVisitor.h"
+#include "clang/Basic/Builtins.h"
+#include "clang/Basic/Specifiers.h"
+#include "clang/CIR/Dialect/IR/CIRAttrs.h"
+#include "clang/CIR/Dialect/IR/CIRTypes.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Sequence.h"
+#include "llvm/Support/ErrorHandling.h"
+
+using namespace clang;
+using namespace clang::CIRGen;
+
+//===----------------------------------------------------------------------===//
+//                             ConstExprEmitter
+//===----------------------------------------------------------------------===//
+
+// This class only needs to handle arrays, structs and unions.
+//
+// In LLVM codegen, when outside C++11 mode, those types are not constant
+// folded, while all other types are handled by constant folding.
+//
+// In CIR codegen, instead of folding things here, we should defer that work
+// to MLIR: do not attempt to do much here.
+class ConstExprEmitter
+    : public StmtVisitor<ConstExprEmitter, mlir::Attribute, QualType> {
+  CIRGenModule &cgm;
+  LLVM_ATTRIBUTE_UNUSED ConstantEmitter &emitter;
+
+public:
+  ConstExprEmitter(ConstantEmitter &emitter)
+      : cgm(emitter.cgm), emitter(emitter) {}
+
+  //===--------------------------------------------------------------------===//
+  //                            Visitor Methods
+  //===--------------------------------------------------------------------===//
+
+  mlir::Attribute VisitStmt(Stmt *S, QualType T) { return {}; }
+
+  mlir::Attribute VisitConstantExpr(ConstantExpr *ce, QualType t) {
+    if (mlir::Attribute result = emitter.tryEmitConstantExpr(ce))
+      return result;
+    return Visit(ce->getSubExpr(), t);
+  }
+
+  mlir::Attribute VisitParenExpr(ParenExpr *pe, QualType t) {
+    return Visit(pe->getSubExpr(), t);
+  }
+
+  mlir::Attribute
+  VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *pe,
+                                    QualType t) {
+    return Visit(pe->getReplacement(), t);
+  }
+
+  mlir::Attribute VisitGenericSelectionExpr(GenericSelectionExpr *ge,
+                                            QualType t) {
+    return Visit(ge->getResultExpr(), t);
+  }
+
+  mlir::Attribute VisitChooseExpr(ChooseExpr *ce, QualType t) {
+    return Visit(ce->getChosenSubExpr(), t);
+  }
+
+  mlir::Attribute VisitCompoundLiteralExpr(CompoundLiteralExpr *e, QualType t) {
+    return Visit(e->getInitializer(), t);
+  }
+
+  mlir::Attribute VisitCastExpr(CastExpr *e, QualType destType) {
+    cgm.errorNYI(e->getBeginLoc(), "ConstExprEmitter::VisitCastExpr");
+    return {};
+  }
+
+  mlir::Attribute VisitCXXDefaultInitExpr(CXXDefaultInitExpr *die, QualType t) {
+    cgm.errorNYI(die->getBeginLoc(),
+                 "ConstExprEmitter::VisitCXXDefaultInitExpr");
+    return {};
+  }
+
+  mlir::Attribute VisitExprWithCleanups(ExprWithCleanups *e, QualType t) {
+    // Since this about constant emission no need to wrap this under a scope.
+    return Visit(e->getSubExpr(), t);
+  }
+
+  mlir::Attribute VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *e,
+                                                QualType t) {
+    return Visit(e->getSubExpr(), t);
+  }
+
+  mlir::Attribute VisitImplicitValueInitExpr(ImplicitValueInitExpr *E,
+                                             QualType T) {
+    cgm.errorNYI(E->getBeginLoc(),
+                 "ConstExprEmitter::VisitImplicitValueInitExpr");
+    return {};
+  }
+
+  mlir::Attribute VisitInitListExpr(InitListExpr *ile, QualType t) {
+    cgm.errorNYI(ile->getBeginLoc(), "ConstExprEmitter::VisitInitListExpr");
+    return {};
+  }
+
+  mlir::Attribute VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *e,
+                                                QualType destType) {
+    mlir::Attribute c = Visit(e->getBase(), destType);
+    if (!c)
+      return {};
+
+    cgm.errorNYI(e->getBeginLoc(),
+                 "ConstExprEmitter::VisitDesignatedInitUpdateExpr");
+    return {};
+  }
+
+  mlir::Attribute VisitCXXConstructExpr(CXXConstructExpr *e, QualType ty) {
+    cgm.errorNYI(e->getBeginLoc(), "ConstExprEmitter::VisitCXXConstructExpr");
+    return {};
+  }
+
+  mlir::Attribute VisitStringLiteral(StringLiteral *e, QualType t) {
+    cgm.errorNYI(e->getBeginLoc(), "ConstExprEmitter::VisitStringLiteral");
+    return {};
+  }
+
+  mlir::Attribute VisitObjCEncodeExpr(ObjCEncodeExpr *e, QualType t) {
+    cgm.errorNYI(e->getBeginLoc(), "ConstExprEmitter::VisitObjCEncodeExpr");
+    return {};
+  }
+
+  mlir::Attribute VisitUnaryExtension(const UnaryOperator *e, QualType t) {
+    return Visit(e->getSubExpr(), t);
+  }
+
+  // Utility methods
+  mlir::Type convertType(QualType t) { return cgm.convertType(t); }
+};
+
+// TODO(cir): this can be shared with LLVM's codegen
+static QualType getNonMemoryType(CIRGenModule &cgm, QualType type) {
+  if (auto at = type->getAs<AtomicType>()) {
+    return cgm.getASTContext().getQualifiedType(at->getValueType(),
+                                                type.getQualifiers());
+  }
+  return type;
+}
+
+//===----------------------------------------------------------------------===//
+//                             ConstantEmitter
+//===----------------------------------------------------------------------===//
+
+mlir::Attribute
+ConstantEmitter::tryEmitAbstractForInitializer(const VarDecl &d) {
+  AbstractStateRAII state(*this, true);
+  return tryEmitPrivateForVarInit(d);
+}
+
+mlir::Attribute ConstantEmitter::tryEmitPrivateForVarInit(const VarDecl &d) {
+  // Make a quick check if variable can be default NULL initialized
+  // and avoid going through rest of code which may do, for c++11,
+  // initialization of memory to all NULLs.
+  if (!d.hasLocalStorage()) {
+    QualType ty = cgm.getASTContext().getBaseElementType(d.getType());
+    if (ty->isRecordType())
+      if (d.getInit() && isa<CXXConstructExpr>(d.getInit())) {
+        cgm.errorNYI(d.getInit()->getBeginLoc(),
+                     "tryEmitPrivateForVarInit CXXConstructExpr");
+        return {};
+      }
+  }
+  inConstantContext = d.hasConstantInitialization();
+
+  const Expr *e = d.getInit();
+  assert(e && "No initializer to emit");
+
+  QualType destType = d.getType();
+
+  if (!destType->isReferenceType()) {
+    QualType nonMemoryDestType = getNonMemoryType(cgm, destType);
+    if (mlir::Attribute c = ConstExprEmitter(*this).Visit(const_cast<Expr *>(e),
+                                                          nonMemoryDestType))
+      return emitForMemory(c, destType);
+  }
+
+  // Try to emit the initializer.  Note that this can allow some things that
+  // are not allowed by tryEmitPrivateForMemory alone.
+  if (APValue *value = d.evaluateValue())
+    return tryEmitPrivateForMemory(*value, destType);
+
+  return {};
+}
+
+mlir::Attribute ConstantEmitter::tryEmitConstantExpr(const ConstantExpr *ce) {
+  if (!ce->hasAPValueResult())
+    return {};
+
+  QualType retType = ce->getType();
+  if (ce->isGLValue())
+    retType = cgm.getASTContext().getLValueReferenceType(retType);
+
+  return emitAbstract(ce->getBeginLoc(), ce->getAPValueResult(), retType);
+}
+
+mlir::Attribute ConstantEmitter::tryEmitPrivateForMemory(const APValue &value,
+                                                         QualType destType) {
+  QualType nonMemoryDestType = getNonMemoryType(cgm, destType);
+  mlir::Attribute c = tryEmitPrivate(value, nonMemoryDestType);
+  return (c ? emitForMemory(c, destType) : nullptr);
+}
+
+mlir::Attribute ConstantEmitter::emitAbstract(SourceLocation loc,
+                                              const APValue &value,
+                                              QualType destType) {
+  AbstractStateRAII state(*this, true);
+  mlir::Attribute c = tryEmitPrivate(value, destType);
+  if (!c)
+    cgm.errorNYI(loc, "emitAbstract failed, emit null constaant");
+  return c;
+}
+
+mlir::Attribute ConstantEmitter::emitForMemory(mlir::Attribute c,
+                                               QualType destType) {
+  // For an _Atomic-qualified constant, we may need to add tail padding.
+  if (destType->getAs<AtomicType>()) {
+    cgm.errorNYI("emitForMemory: atomic type");
+    return {};
+  }
+
+  return c;
+}
+
+mlir::Attribute ConstantEmitter::tryEmitPrivate(const APValue &value,
+                                                QualType destType) {
+  auto &builder = cgm.getBuilder();
+  switch (value.getKind()) {
+  case APValue::None:
+  case APValue::Indeterminate:
+    cgm.errorNYI("ConstExprEmitter::tryEmitPrivate none or indeterminate");
+    return {};
+  case APValue::Int: {
+    mlir::Type ty = cgm.convertType(destType);
+    if (mlir::isa<cir::BoolType>(ty))
+      return builder.getCIRBoolAttr(value.getInt().getZExtValue());
+    assert(mlir::isa<cir::IntType>(ty) && "expected integral type");
+    return cgm.getBuilder().getAttr<cir::IntAttr>(ty, value.getInt());
+  }
+  case APValue::Float: {
+    const llvm::APFloat &init = value.getFloat();
+    if (&init.getSemantics() == &llvm::APFloat::IEEEhalf() &&
+        !cgm.getASTContext().getLangOpts().NativeHalfType &&
+        cgm.getASTContext().getTargetInfo().useFP16ConversionIntrinsics()) {
+      cgm.errorNYI("ConstExprEmitter::tryEmitPrivate half");
+      return {};
+    } else {
+      mlir::Type ty = cgm.convertType(destType);
+      assert(mlir::isa<cir::CIRFPTypeInterface>(ty) &&
+             "expected floating-point type");
+      return cgm.getBuilder().getAttr<cir::FPAttr>(ty, init);
+    }
+  }
+  case APValue::Array: {
+    cgm.errorNYI("ConstExprEmitter::tryEmitPrivate array");
+    return {};
+  }
+  case APValue::Vector: {
+    cgm.errorNYI("ConstExprEmitter::tryEmitPrivate vector");
+    return {};
+  }
+  case APValue::MemberPointer: {
+    cgm.errorNYI("ConstExprEmitter::tryEmitPrivate member pointer");
+    return {};
+  }
+  case APValue::LValue:
+    cgm.errorNYI("ConstExprEmitter::tryEmitPrivate lvalue");
+    return {};
+  case APValue::Struct:
+  case APValue::Union:
+    cgm.errorNYI("ConstExprEmitter::tryEmitPrivate struct or union");
+    return {};
+  case APValue::FixedPoint:
+  case APValue::ComplexInt:
+  case APValue::ComplexFloat:
+  case APValue::AddrLabelDiff:
+    cgm.errorNYI("ConstExprEmitter::tryEmitPrivate fixed point, complex int, "
+                 "complex float, addr label diff");
+    return {};
+  }
+  llvm_unreachable("Unknown APValue kind");
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index cf896d3c0a946..72445f62232a4 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -26,6 +26,7 @@
 #include "clang/AST/Decl.h"
 #include "clang/AST/Type.h"
 #include "clang/CIR/Dialect/IR/CIRDialect.h"
+#include "clang/CIR/MissingFeatures.h"
 #include "clang/CIR/TypeEvaluationKind.h"
 
 #include "llvm/ADT/ScopedHashTable.h"
@@ -105,6 +106,27 @@ class CIRGenFunction : public CIRGenTypeCache {
   mlir::Value emitAlloca(llvm::StringRef name, mlir::Type ty,
                          mlir::Location loc, clang::CharUnits alignment);
 
+private:
+  // Track current variable initialization (if there's one)
+  const clang::VarDecl *currVarDecl = nullptr;
+  class VarDeclContext {
+    CIRGenFunction &p;
+    const clang::VarDecl *oldVal = nullptr;
+
+  public:
+    VarDeclContext(CIRGenFunction &p, const VarDecl *value) : p(p) {
+      if (p.currVarDecl)
+        oldVal = p.currVarDecl;
+      p.currVarDecl = value;
+    }
+
+    /// Can be used to restore the state early, before the dtor
+    /// is run.
+    void restore() { p.currVarDecl = oldVal; }
+    ~VarDeclContext() { restore(); }
+  };
+
+public:
   /// Use to track source locations across nested visitor traversals.
   /// Always use a `SourceLocRAIIObject` to change currSrcLoc.
   std::optional<mlir::Location> currSrcLoc;
@@ -170,16 +192,97 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   void emitDecl(const clang::Decl &d);
 
+  void emitScalarInit(const clang::Expr *init, mlir::Location loc,
+                      LValue lvalue, bool capturedByInit = false);
+
   LValue emitDeclRefLValue(const clang::DeclRefExpr *e);
 
+  /// Determine whether the given initializer is trivial in the sense
+  /// that it requires no code to be generated.
+  bool isTrivialInitializer(const Expr *init);
+
+  /// Emit an expression as an initializer for an object (variable, field, etc.)
+  /// at the given location.  The expression is not necessarily the normal
+  /// initializer for the object, and the address is not necessarily
+  /// its normal location.
+  ///
+  /// \param init the initializing expression
+  /// \param d the object to act as if we're initializing
+  /// \param lvalue the lvalue to initialize
+  /// \param capturedByInit true if \p d is a __block variable whose address is
+  /// potentially changed by the initializer
+  void emitExprAsInit(const clang::Expr *init, const clang::ValueDecl *d,
+                      LValue lvalue, bool capturedByInit = false);
+
   /// Emit code and set up symbol table for a variable declaration with auto,
   /// register, or no storage class specifier. These turn into simple stack
   /// objects, globals depending on target.
   void emitAutoVarDecl(const clang::VarDecl &d);
 
-  void emitAutoVarAlloca(const clang::VarDecl &d);
-  void emitAutoVarInit(const clang::VarDecl &d);
-  void emitAutoVarCleanups(const clang::VarDecl &d);
+  struct AutoVarEmission {
+    const clang::VarDecl *Variable;
+    /// The address of the alloca for languages with explicit address space
+    /// (e.g. OpenCL) or alloca casted to generic pointer for address space
+    /// agnostic languages (e.g. C++). Invalid if the variable was emitted
+    /// as a global constant.
+    Address Addr;
+
+    /// True if the variable is of aggregate type and has a constant
+    /// initializer.
+    bool IsConstantAggregate = false;
+
+    /// True if the variable is a __block variable that is captured by an
+    /// escaping block.
+    bool IsEscapingByRef = false;
+
+    mlir::Value NRVOFlag{};
+
+    struct Invalid {};
+    AutoVarEmission(Invalid) : Variable(nullptr), Addr(Address::invalid()) {}
+
+    AutoVarEmission(const clang::VarDecl &variable)
+        : Variable(&variable), Addr(Address::invalid()) {}
+
+    static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); }
+
+    bool wasEmittedAsGlobal() const { return !Addr.isValid(); }
+
+    /// Returns the raw, allocated address, which is not necessarily
+    /// the address of the object itself. It is casted to default
+    /// address space for address space agnostic languages.
+    Address getAllocatedAddress() const { return Addr; }
+
+    /// Returns the address of the object within this declaration.
+    /// Note that this does not chase the forwarding pointer for
+    /// __block decls.
+    Address getObjectAddress(CIRGenFunction &CGF) const {
+      if (!IsEscapingByRef)
+        return Addr;
+
+      assert(!cir::MissingFeatures::opAllocaEscapeByReference());
+      return Address::invalid();
+    }
+  };
+
+  AutoVarEmission emitAutoVarAlloca(const clang::VarDecl &d);
+  void emitAutoVarInit(const AutoVarEmission &emission);
+  void emitAutoVarCleanups(const AutoVarEmission &emission);
+
+  void emitStoreOfScalar(mlir::Value value, Address addr, bool isVolatile,
+                         clang::QualType ty, bool isInit = false,
+                         bool isNontemporal = false);
+  void emitStoreOfScalar(mlir::Value value, LValue lvalue, bool isInit);
+
+  /// Given a value and its clang type, returns the value casted to its memory
+  /// representation.
+  /// Note: CIR defers most of the special casting to the final lowering passes
+  /// to conserve the high level information.
+  mlir::Value emitToMemory(mlir::Value Value, clang::QualType Ty);
+
+  /// Store the specified rvalue into the specified
+  /// lvalue, where both are guaranteed to the have the same type, and that type
+  /// is 'Ty'.
+  void emitStoreThroughLValue(RValue Src, LValue Dst, bool isInit = false);
 
   /// This method handles emission of any variable declaration
   /// inside a function, including static vars etc.
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.h b/clang/lib/CIR/CodeGen/CIRGenModule.h
index 5fb5ef505a8c1..2a798f4cd56a9 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.h
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.h
@@ -147,6 +147,20 @@ class CIRGenModule : public CIRGenTypeCache {
     return diags.Report(loc, diagID) << feature << name;
   }
 
+  DiagnosticBuilder errorNYI(mlir::Location loc, llvm::StringRef feature) {
+    // TODO: Convert the location to a SourceLocation
+    unsigned diagID = diags.getCustomDiagID(
+        DiagnosticsEngine::Error, "ClangIR code gen Not Yet Implemented: %0");
+    return diags.Report(diagID) << feature;
+  }
+
+  DiagnosticBuilder errorNYI(llvm::StringRef feature) {
+    // TODO: Make a default location? currSrcLoc?
+    unsigned diagID = diags.getCustomDiagID(
+        DiagnosticsEngine::Error, "ClangIR code gen Not Yet Implemented: %0");
+    return diags.Report(diagID) << feature;
+  }
+
   DiagnosticBuilder errorNYI(SourceRange, llvm::StringRef);
 
   template <typename T>
diff --git a/clang/lib/CIR/CodeGen/CMakeLists.txt b/clang/lib/CIR/CodeGen/CMakeLists.txt
index e6d3cbabd853b..8ee65c2763e70 100644
--- a/clang/lib/CIR/CodeGen/CMakeLists.txt
+++ b/clang/lib/CIR/CodeGen/CMakeLists.txt
@@ -10,6 +10,7 @@ add_clang_library(clangCIR
   CIRGenerator.cpp
   CIRGenDecl.cpp
   CIRGenExpr.cpp
+  CIRGenExprConstant.cpp
   CIRGenExprScalar.cpp
   CIRGenFunction.cpp
   CIRGenModule.cpp
diff --git a/clang/lib/CIR/Dialect/CMakeLists.txt b/clang/lib/CIR/Dialect/CMakeLists.txt
index f33061b2d87cf..9f57627c321fb 100644
--- a/clang/lib/CIR/Dialect/CMakeLists.txt
+++ b/clang/lib/CIR/Dialect/CMakeLists.txt
@@ -1 +1,2 @@
 add_subdirectory(IR)
+add_subdirectory(Transforms)
diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
index 5ad369b40cda1..d041791770d82 100644
--- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
@@ -268,6 +268,19 @@ LogicalResult cir::ScopeOp::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// BrOp
+//===----------------------------------------------------------------------===//
+
+mlir::SuccessorOperands cir::BrOp::getSuccessorOperands(unsigned index) {
+  assert(index == 0 && "invalid successor index");
+  return mlir::SuccessorOperands(getDestOperandsMutable());
+}
+
+Block *cir::BrOp::getSuccessorForOperands(ArrayRef<Attribute>) {
+  return getDest();
+}
+
 //===----------------------------------------------------------------------===//
 // GlobalOp
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/CIR/Dialect/IR/CMakeLists.txt b/clang/lib/CIR/Dialect/IR/CMakeLists.txt
index e3a6fc6e80ecc..aa5ea52a5e93f 100644
--- a/clang/lib/CIR/Dialect/IR/CMakeLists.txt
+++ b/clang/lib/CIR/Dialect/IR/CMakeLists.txt
@@ -11,6 +11,7 @@ add_clang_library(MLIRCIR
 
   LINK_LIBS PUBLIC
   MLIRIR
+  MLIRCIRInterfaces
   MLIRDLTIDialect
   MLIRDataLayoutInterfaces
   MLIRFuncDialect
diff --git a/clang/lib/CIR/Dialect/Transforms/CMakeLists.txt b/clang/lib/CIR/Dialect/Transforms/CMakeLists.txt
new file mode 100644
index 0000000000000..aa27074cc6131
--- /dev/null
+++ b/clang/lib/CIR/Dialect/Transforms/CMakeLists.txt
@@ -0,0 +1,18 @@
+add_clang_library(MLIRCIRTransforms
+  FlattenCFG.cpp
+
+  DEPENDS
+  MLIRCIRPassIncGen
+
+  LINK_LIBS PUBLIC
+  clangAST
+  clangBasic
+
+  MLIRAnalysis
+  MLIRIR
+  MLIRPass
+  MLIRTransformUtils
+
+  MLIRCIR
+  MLIRCIRInterfaces
+)
diff --git a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
new file mode 100644
index 0000000000000..b01d3d4d43e65
--- /dev/null
+++ b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp
@@ -0,0 +1,117 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements pass that inlines CIR operations regions into the parent
+// function region.
+//
+//===----------------------------------------------------------------------===//
+
+#include "PassDetail.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Support/LogicalResult.h"
+#include "mlir/Transforms/DialectConversion.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include "clang/CIR/Dialect/IR/CIRDialect.h"
+#include "clang/CIR/Dialect/Passes.h"
+#include "clang/CIR/MissingFeatures.h"
+
+using namespace mlir;
+using namespace cir;
+
+namespace {
+
+struct CIRFlattenCFGPass : public CIRFlattenCFGBase<CIRFlattenCFGPass> {
+
+  CIRFlattenCFGPass() = default;
+  void runOnOperation() override;
+};
+
+class CIRScopeOpFlattening : public mlir::OpRewritePattern<cir::ScopeOp> {
+public:
+  using OpRewritePattern<cir::ScopeOp>::OpRewritePattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::ScopeOp scopeOp,
+                  mlir::PatternRewriter &rewriter) const override {
+    mlir::OpBuilder::InsertionGuard guard(rewriter);
+    mlir::Location loc = scopeOp.getLoc();
+
+    // Empty scope: just remove it.
+    // TODO: Remove this logic once CIR uses MLIR infrastructure to remove
+    // trivially dead operations. MLIR canonicalizer is too aggressive and we
+    // need to either (a) make sure all our ops model all side-effects and/or
+    // (b) have more options in the canonicalizer in MLIR to temper
+    // aggressiveness level.
+    if (scopeOp.isEmpty()) {
+      rewriter.eraseOp(scopeOp);
+      return mlir::success();
+    }
+
+    // Split the current block before the ScopeOp to create the inlining
+    // point.
+    mlir::Block *currentBlock = rewriter.getInsertionBlock();
+    mlir::Block *continueBlock =
+        rewriter.splitBlock(currentBlock, rewriter.getInsertionPoint());
+    if (scopeOp.getNumResults() > 0)
+      continueBlock->addArguments(scopeOp.getResultTypes(), loc);
+
+    // Inline body region.
+    mlir::Block *beforeBody = &scopeOp.getScopeRegion().front();
+    mlir::Block *afterBody = &scopeOp.getScopeRegion().back();
+    rewriter.inlineRegionBefore(scopeOp.getScopeRegion(), continueBlock);
+
+    // Save stack and then branch into the body of the region.
+    rewriter.setInsertionPointToEnd(currentBlock);
+    assert(!cir::MissingFeatures::stackSaveOp());
+    rewriter.create<cir::BrOp>(loc, mlir::ValueRange(), beforeBody);
+
+    // Replace the scopeop return with a branch that jumps out of the body.
+    // Stack restore before leaving the body region.
+    rewriter.setInsertionPointToEnd(afterBody);
+    if (auto yieldOp = dyn_cast<cir::YieldOp>(afterBody->getTerminator())) {
+      rewriter.replaceOpWithNewOp<cir::BrOp>(yieldOp, yieldOp.getArgs(),
+                                             continueBlock);
+    }
+
+    // Replace the op with values return from the body region.
+    rewriter.replaceOp(scopeOp, continueBlock->getArguments());
+
+    return mlir::success();
+  }
+};
+
+void populateFlattenCFGPatterns(RewritePatternSet &patterns) {
+  patterns.add<CIRScopeOpFlattening>(patterns.getContext());
+}
+
+void CIRFlattenCFGPass::runOnOperation() {
+  RewritePatternSet patterns(&getContext());
+  populateFlattenCFGPatterns(patterns);
+
+  // Collect operations to apply patterns.
+  llvm::SmallVector<Operation *, 16> ops;
+  getOperation()->walk<mlir::WalkOrder::PostOrder>([&](Operation *op) {
+    if (isa<ScopeOp>(op))
+      ops.push_back(op);
+  });
+
+  // Apply patterns.
+  if (applyOpPatternsGreedily(ops, std::move(patterns)).failed())
+    signalPassFailure();
+}
+
+} // namespace
+
+namespace mlir {
+
+std::unique_ptr<Pass> createCIRFlattenCFGPass() {
+  return std::make_unique<CIRFlattenCFGPass>();
+}
+
+} // namespace mlir
diff --git a/clang/lib/CIR/Dialect/Transforms/PassDetail.h b/clang/lib/CIR/Dialect/Transforms/PassDetail.h
new file mode 100644
index 0000000000000..600dde56d679f
--- /dev/null
+++ b/clang/lib/CIR/Dialect/Transforms/PassDetail.h
@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CIR_DIALECT_TRANSFORMS_PASSDETAIL_H
+#define CIR_DIALECT_TRANSFORMS_PASSDETAIL_H
+
+#include "mlir/IR/Dialect.h"
+#include "mlir/Pass/Pass.h"
+
+namespace cir {
+class CIRDialect;
+} // namespace cir
+
+namespace mlir {
+// Forward declaration from Dialect.h
+template <typename ConcreteDialect>
+void registerDialect(DialectRegistry &registry);
+
+#define GEN_PASS_CLASSES
+#include "clang/CIR/Dialect/Passes.h.inc"
+
+} // namespace mlir
+
+#endif // CIR_DIALECT_TRANSFORMS_PASSDETAIL_H
diff --git a/clang/lib/CIR/Lowering/CIRPasses.cpp b/clang/lib/CIR/Lowering/CIRPasses.cpp
new file mode 100644
index 0000000000000..235acbfee8967
--- /dev/null
+++ b/clang/lib/CIR/Lowering/CIRPasses.cpp
@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements machinery for any CIR <-> CIR passes used by clang.
+//
+//===----------------------------------------------------------------------===//
+
+// #include "clang/AST/ASTContext.h"
+#include "clang/CIR/Dialect/Passes.h"
+
+#include "mlir/Pass/PassManager.h"
+
+namespace mlir {
+
+void populateCIRPreLoweringPasses(OpPassManager &pm) {
+  pm.addPass(createCIRFlattenCFGPass());
+}
+
+} // namespace mlir
diff --git a/clang/lib/CIR/Lowering/CMakeLists.txt b/clang/lib/CIR/Lowering/CMakeLists.txt
index 95c304ded9183..09e48862df63c 100644
--- a/clang/lib/CIR/Lowering/CMakeLists.txt
+++ b/clang/lib/CIR/Lowering/CMakeLists.txt
@@ -1 +1,20 @@
+set(LLVM_LINK_COMPONENTS
+  Core
+  Support
+  )
+
+get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
+
+add_clang_library(clangCIRLoweringCommon
+  CIRPasses.cpp
+
+  LINK_LIBS
+  clangCIR
+  ${dialect_libs}
+  MLIRCIR
+  MLIRCIRTransforms
+  MLIRTransforms
+  MLIRSupport
+  )
+
 add_subdirectory(DirectToLLVM)
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/CMakeLists.txt b/clang/lib/CIR/Lowering/DirectToLLVM/CMakeLists.txt
index c11ecb82183d0..7baff3412a84e 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/CMakeLists.txt
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/CMakeLists.txt
@@ -7,6 +7,7 @@ get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
 
 add_clang_library(clangCIRLoweringDirectToLLVM
   LowerToLLVM.cpp
+  LowerToLLVMIR.cpp
 
   DEPENDS
   MLIRCIREnumsGen
@@ -14,9 +15,10 @@ add_clang_library(clangCIRLoweringDirectToLLVM
   MLIRCIROpInterfacesIncGen
 
   LINK_LIBS
-  MLIRIR
+  clangCIRLoweringCommon
   ${dialect_libs}
   MLIRCIR
   MLIRBuiltinToLLVMIRTranslation
   MLIRLLVMToLLVMIRTranslation
+  MLIRIR
   )
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index 3200527bd03af..79ec0696eb180 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -12,6 +12,8 @@
 
 #include "LowerToLLVM.h"
 
+#include <deque>
+
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
 #include "mlir/Dialect/DLTI/DLTI.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
@@ -25,11 +27,13 @@
 #include "mlir/Target/LLVMIR/Export.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "clang/CIR/Dialect/IR/CIRDialect.h"
+#include "clang/CIR/Dialect/Passes.h"
 #include "clang/CIR/MissingFeatures.h"
 #include "clang/CIR/Passes.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/TimeProfiler.h"
 
 using namespace cir;
@@ -135,6 +139,7 @@ mlir::LLVM::Linkage convertLinkage(cir::GlobalLinkageKind linkage) {
   case CIR::WeakODRLinkage:
     return LLVM::WeakODR;
   };
+  llvm_unreachable("Unknown CIR linkage type");
 }
 
 class CIRAttrToValue {
@@ -603,6 +608,69 @@ static void prepareTypeConverter(mlir::LLVMTypeConverter &converter,
   });
 }
 
+// The applyPartialConversion function traverses blocks in the dominance order,
+// so it does not lower and operations that are not reachachable from the
+// operations passed in as arguments. Since we do need to lower such code in
+// order to avoid verification errors occur, we cannot just pass the module op
+// to applyPartialConversion. We must build a set of unreachable ops and
+// explicitly add them, along with the module, to the vector we pass to
+// applyPartialConversion.
+//
+// For instance, this CIR code:
+//
+//    cir.func @foo(%arg0: !s32i) -> !s32i {
+//      %4 = cir.cast(int_to_bool, %arg0 : !s32i), !cir.bool
+//      cir.if %4 {
+//        %5 = cir.const #cir.int<1> : !s32i
+//        cir.return %5 : !s32i
+//      } else {
+//        %5 = cir.const #cir.int<0> : !s32i
+//       cir.return %5 : !s32i
+//      }
+//      cir.return %arg0 : !s32i
+//    }
+//
+// contains an unreachable return operation (the last one). After the flattening
+// pass it will be placed into the unreachable block. The possible error
+// after the lowering pass is: error: 'cir.return' op expects parent op to be
+// one of 'cir.func, cir.scope, cir.if ... The reason that this operation was
+// not lowered and the new parent is llvm.func.
+//
+// In the future we may want to get rid of this function and use a DCE pass or
+// something similar. But for now we need to guarantee the absence of the
+// dialect verification errors.
+static void collectUnreachable(mlir::Operation *parent,
+                               llvm::SmallVector<mlir::Operation *> &ops) {
+
+  llvm::SmallVector<mlir::Block *> unreachableBlocks;
+  parent->walk([&](mlir::Block *blk) { // check
+    if (blk->hasNoPredecessors() && !blk->isEntryBlock())
+      unreachableBlocks.push_back(blk);
+  });
+
+  std::set<mlir::Block *> visited;
+  for (mlir::Block *root : unreachableBlocks) {
+    // We create a work list for each unreachable block.
+    // Thus we traverse operations in some order.
+    std::deque<mlir::Block *> workList;
+    workList.push_back(root);
+
+    while (!workList.empty()) {
+      mlir::Block *blk = workList.back();
+      workList.pop_back();
+      if (visited.count(blk))
+        continue;
+      visited.emplace(blk);
+
+      for (mlir::Operation &op : *blk)
+        ops.push_back(&op);
+
+      for (mlir::Block *succ : blk->getSuccessors())
+        workList.push_back(succ);
+    }
+  }
+}
+
 void ConvertCIRToLLVMPass::processCIRAttrs(mlir::ModuleOp module) {
   // Lower the module attributes to LLVM equivalents.
   if (mlir::Attribute tripleAttr =
@@ -630,7 +698,13 @@ void ConvertCIRToLLVMPass::runOnOperation() {
   patterns.add<CIRToLLVMGlobalOpLowering>(converter, patterns.getContext(), dl);
   patterns.add<CIRToLLVMConstantOpLowering>(converter, patterns.getContext(),
                                             dl);
-  patterns.add<CIRToLLVMFuncOpLowering>(converter, patterns.getContext());
+  patterns.add<
+      // clang-format off
+               CIRToLLVMBrOpLowering,
+               CIRToLLVMFuncOpLowering,
+               CIRToLLVMTrapOpLowering
+      // clang-format on
+      >(converter, patterns.getContext());
 
   processCIRAttrs(module);
 
@@ -640,9 +714,36 @@ void ConvertCIRToLLVMPass::runOnOperation() {
   target.addIllegalDialect<mlir::BuiltinDialect, cir::CIRDialect,
                            mlir::func::FuncDialect>();
 
-  if (failed(applyPartialConversion(module, target, std::move(patterns)))) {
+  llvm::SmallVector<mlir::Operation *> ops;
+  ops.push_back(module);
+  collectUnreachable(module, ops);
+
+  if (failed(applyPartialConversion(ops, target, std::move(patterns))))
     signalPassFailure();
-  }
+}
+
+mlir::LogicalResult CIRToLLVMBrOpLowering::matchAndRewrite(
+    cir::BrOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(op, adaptor.getOperands(),
+                                                op.getDest());
+  return mlir::LogicalResult::success();
+}
+
+mlir::LogicalResult CIRToLLVMTrapOpLowering::matchAndRewrite(
+    cir::TrapOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  mlir::Location loc = op->getLoc();
+  rewriter.eraseOp(op);
+
+  rewriter.create<mlir::LLVM::Trap>(loc);
+
+  // Note that the call to llvm.trap is not a terminator in LLVM dialect.
+  // So we must emit an additional llvm.unreachable to terminate the current
+  // block.
+  rewriter.create<mlir::LLVM::UnreachableOp>(loc);
+
+  return mlir::success();
 }
 
 std::unique_ptr<mlir::Pass> createConvertCIRToLLVMPass() {
@@ -650,6 +751,7 @@ std::unique_ptr<mlir::Pass> createConvertCIRToLLVMPass() {
 }
 
 void populateCIRToLLVMPasses(mlir::OpPassManager &pm) {
+  mlir::populateCIRPreLoweringPasses(pm);
   pm.addPass(createConvertCIRToLLVMPass());
 }
 
@@ -670,6 +772,7 @@ lowerDirectlyFromCIRToLLVMIR(mlir::ModuleOp mlirModule, LLVMContext &llvmCtx) {
 
   mlir::registerBuiltinDialectTranslation(*mlirCtx);
   mlir::registerLLVMDialectTranslation(*mlirCtx);
+  mlir::registerCIRDialectTranslation(*mlirCtx);
 
   llvm::TimeTraceScope translateScope("translateModuleToLLVMIR");
 
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
index a694047e3616b..d090bbe4f2e10 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
@@ -136,6 +136,24 @@ class CIRToLLVMGlobalOpLowering
       cir::GlobalOp op, mlir::ConversionPatternRewriter &rewriter) const;
 };
 
+class CIRToLLVMBrOpLowering : public mlir::OpConversionPattern<cir::BrOp> {
+public:
+  using mlir::OpConversionPattern<cir::BrOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::BrOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMTrapOpLowering : public mlir::OpConversionPattern<cir::TrapOp> {
+public:
+  using mlir::OpConversionPattern<cir::TrapOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::TrapOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
 } // namespace direct
 } // namespace cir
 
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVMIR.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVMIR.cpp
new file mode 100644
index 0000000000000..30b9eaaca2d37
--- /dev/null
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVMIR.cpp
@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements lowering of CIR attributes and operations directly to
+// LLVMIR.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/IR/DialectRegistry.h"
+#include "mlir/Target/LLVMIR/LLVMTranslationInterface.h"
+#include "mlir/Target/LLVMIR/ModuleTranslation.h"
+#include "clang/CIR/Dialect/IR/CIRAttrs.h"
+#include "clang/CIR/Dialect/IR/CIRDialect.h"
+#include "clang/CIR/MissingFeatures.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/IR/Constant.h"
+#include "llvm/IR/GlobalVariable.h"
+
+using namespace llvm;
+
+namespace cir {
+namespace direct {
+
+/// Implementation of the dialect interface that converts CIR attributes to LLVM
+/// IR metadata.
+class CIRDialectLLVMIRTranslationInterface
+    : public mlir::LLVMTranslationDialectInterface {
+public:
+  using LLVMTranslationDialectInterface::LLVMTranslationDialectInterface;
+
+  /// Translates the given operation to LLVM IR using the provided IR builder
+  /// and saving the state in `moduleTranslation`.
+  mlir::LogicalResult convertOperation(
+      mlir::Operation *op, llvm::IRBuilderBase &builder,
+      mlir::LLVM::ModuleTranslation &moduleTranslation) const final {
+
+    if (auto cirOp = llvm::dyn_cast<mlir::LLVM::ZeroOp>(op))
+      moduleTranslation.mapValue(cirOp.getResult()) =
+          llvm::Constant::getNullValue(
+              moduleTranslation.convertType(cirOp.getType()));
+
+    return mlir::success();
+  }
+};
+
+void registerCIRDialectTranslation(mlir::DialectRegistry &registry) {
+  registry.insert<cir::CIRDialect>();
+  registry.addExtension(+[](mlir::MLIRContext *ctx, cir::CIRDialect *dialect) {
+    dialect->addInterfaces<CIRDialectLLVMIRTranslationInterface>();
+  });
+}
+
+} // namespace direct
+} // namespace cir
+
+namespace mlir {
+void registerCIRDialectTranslation(mlir::MLIRContext &context) {
+  mlir::DialectRegistry registry;
+  cir::direct::registerCIRDialectTranslation(registry);
+  context.appendDialectRegistry(registry);
+}
+} // namespace mlir
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index b86bb242755be..a5ed2595bad4d 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -4309,6 +4309,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI__builtin_elementwise_exp2:
     return RValue::get(emitBuiltinWithOneOverloadedType<1>(
         *this, E, llvm::Intrinsic::exp2, "elt.exp2"));
+  case Builtin::BI__builtin_elementwise_exp10:
+    return RValue::get(emitBuiltinWithOneOverloadedType<1>(
+        *this, E, llvm::Intrinsic::exp10, "elt.exp10"));
   case Builtin::BI__builtin_elementwise_log:
     return RValue::get(emitBuiltinWithOneOverloadedType<1>(
         *this, E, llvm::Intrinsic::log, "elt.log"));
@@ -19836,6 +19839,14 @@ case Builtin::BI__builtin_hlsl_elementwise_isinf: {
         RValFalse.isScalar()
             ? RValFalse.getScalarVal()
             : RValFalse.getAggregatePointer(E->getArg(2)->getType(), *this);
+    if (auto *VTy = E->getType()->getAs<VectorType>()) {
+      if (!OpTrue->getType()->isVectorTy())
+        OpTrue =
+            Builder.CreateVectorSplat(VTy->getNumElements(), OpTrue, "splat");
+      if (!OpFalse->getType()->isVectorTy())
+        OpFalse =
+            Builder.CreateVectorSplat(VTy->getNumElements(), OpFalse, "splat");
+    }
 
     Value *SelectVal =
         Builder.CreateSelect(OpCond, OpTrue, OpFalse, "hlsl.select");
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index bfcbc273dbda7..7aa77e55dbfcc 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -2705,7 +2705,7 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
         llvm::AttributeSet::get(getLLVMContext(), Attrs);
   }
 
-  // Apply `nonnull`, `dereferencable(N)` and `align N` to the `this` argument,
+  // Apply `nonnull`, `dereferenceable(N)` and `align N` to the `this` argument,
   // unless this is a thunk function.
   // FIXME: fix this properly, https://reviews.llvm.org/D100388
   if (FI.isInstanceMethod() && !IRFunctionArgs.hasInallocaArg() &&
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 0e6daa42ee7bf..7020cef875170 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -3308,7 +3308,7 @@ llvm::DIType *CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty,
 
 llvm::DIType *CGDebugInfo::CreateType(const VectorType *Ty,
                                       llvm::DIFile *Unit) {
-  if (Ty->isExtVectorBoolType()) {
+  if (Ty->isPackedVectorBoolType(CGM.getContext())) {
     // Boolean ext_vector_type(N) are special because their real element type
     // (bits of bit size) is not their Clang element type (_Bool of size byte).
     // For now, we pretend the boolean vector were actually a vector of bytes
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index 3ad9ebf624143..eab1ebfb2369b 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -48,7 +48,7 @@ using namespace CodeGen;
 static_assert(clang::Sema::MaximumAlignment <= llvm::Value::MaximumAlignment,
               "Clang max alignment greater than what LLVM supports?");
 
-void CodeGenFunction::EmitDecl(const Decl &D) {
+void CodeGenFunction::EmitDecl(const Decl &D, bool EvaluateConditionDecl) {
   switch (D.getKind()) {
   case Decl::BuiltinTemplate:
   case Decl::TranslationUnit:
@@ -152,7 +152,7 @@ void CodeGenFunction::EmitDecl(const Decl &D) {
     return;
   case Decl::UsingPack:
     for (auto *Using : cast<UsingPackDecl>(D).expansions())
-      EmitDecl(*Using);
+      EmitDecl(*Using, /*EvaluateConditionDecl=*/EvaluateConditionDecl);
     return;
   case Decl::UsingDirective: // using namespace X; [C++]
     if (CGDebugInfo *DI = getDebugInfo())
@@ -164,10 +164,8 @@ void CodeGenFunction::EmitDecl(const Decl &D) {
     assert(VD.isLocalVarDecl() &&
            "Should not see file-scope variables inside a function!");
     EmitVarDecl(VD);
-    if (auto *DD = dyn_cast<DecompositionDecl>(&VD))
-      for (auto *B : DD->flat_bindings())
-        if (auto *HD = B->getHoldingVar())
-          EmitVarDecl(*HD);
+    if (EvaluateConditionDecl)
+      MaybeEmitDeferredVarDeclInit(&VD);
 
     return;
   }
@@ -2059,6 +2057,14 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) {
                         /*IsAutoInit=*/false);
 }
 
+void CodeGenFunction::MaybeEmitDeferredVarDeclInit(const VarDecl *VD) {
+  if (auto *DD = dyn_cast_if_present<DecompositionDecl>(VD)) {
+    for (auto *B : DD->flat_bindings())
+      if (auto *HD = B->getHoldingVar())
+        EmitVarDecl(*HD);
+  }
+}
+
 /// Emit an expression as an initializer for an object (variable, field, etc.)
 /// at the given location.  The expression is not necessarily the normal
 /// initializer for the object, and the address is not necessarily
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 191912ca7d800..5943ff9294e1a 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -1985,7 +1985,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
 
   if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
     // Boolean vectors use `iN` as storage type.
-    if (ClangVecTy->isExtVectorBoolType()) {
+    if (ClangVecTy->isPackedVectorBoolType(getContext())) {
       llvm::Type *ValTy = ConvertType(Ty);
       unsigned ValNumElems =
           cast<llvm::FixedVectorType>(ValTy)->getNumElements();
@@ -2064,6 +2064,10 @@ llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
 
   if (Ty->isExtVectorBoolType()) {
     llvm::Type *StoreTy = convertTypeForLoadStore(Ty, Value->getType());
+    if (StoreTy->isVectorTy() && StoreTy->getScalarSizeInBits() >
+                                     Value->getType()->getScalarSizeInBits())
+      return Builder.CreateZExt(Value, StoreTy);
+
     // Expand to the memory bit width.
     unsigned MemNumElems = StoreTy->getPrimitiveSizeInBits();
     // <N x i1> --> <P x i1>.
@@ -2079,8 +2083,9 @@ llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
 /// by convertTypeForLoadStore) to its primary IR type (as returned
 /// by ConvertType).
 llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
-  if (Ty->isExtVectorBoolType()) {
+  if (Ty->isPackedVectorBoolType(getContext())) {
     const auto *RawIntTy = Value->getType();
+
     // Bitcast iP --> <P x i1>.
     auto *PaddedVecTy = llvm::FixedVectorType::get(
         Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
@@ -2091,10 +2096,10 @@ llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
     return emitBoolVecConversion(V, ValNumElems, "extractvec");
   }
 
-  if (hasBooleanRepresentation(Ty) || Ty->isBitIntType()) {
-    llvm::Type *ResTy = ConvertType(Ty);
+  llvm::Type *ResTy = ConvertType(Ty);
+  if (hasBooleanRepresentation(Ty) || Ty->isBitIntType() ||
+      Ty->isExtVectorBoolType())
     return Builder.CreateTrunc(Value, ResTy, "loadedv");
-  }
 
   return Value;
 }
@@ -2152,7 +2157,8 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr,
     if (auto *VecTy = dyn_cast<llvm::FixedVectorType>(SrcTy)) {
       auto *NewVecTy =
           CGM.getABIInfo().getOptimalVectorMemoryType(VecTy, getLangOpts());
-      if (!ClangVecTy->isExtVectorBoolType() && VecTy != NewVecTy) {
+      if (!ClangVecTy->isPackedVectorBoolType(getContext()) &&
+          VecTy != NewVecTy) {
         SmallVector<int, 16> Mask(NewVecTy->getNumElements(), -1);
         std::iota(Mask.begin(), Mask.begin() + VecTy->getNumElements(), 0);
         Value = Builder.CreateShuffleVector(Value, Mask, "extractVec");
@@ -2343,7 +2349,15 @@ RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
   if (!ExprVT) {
     unsigned InIdx = getAccessedFieldNo(0, Elts);
     llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx);
-    return RValue::get(Builder.CreateExtractElement(Vec, Elt));
+
+    llvm::Value *Element = Builder.CreateExtractElement(Vec, Elt);
+
+    llvm::Type *LVTy = ConvertType(LV.getType());
+    if (Element->getType()->getPrimitiveSizeInBits() >
+        LVTy->getPrimitiveSizeInBits())
+      Element = Builder.CreateTrunc(Element, LVTy);
+
+    return RValue::get(Element);
   }
 
   // Always use shuffle vector to try to retain the original program structure
@@ -2354,6 +2368,10 @@ RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
     Mask.push_back(getAccessedFieldNo(i, Elts));
 
   Vec = Builder.CreateShuffleVector(Vec, Mask);
+
+  if (LV.getType()->isExtVectorBoolType())
+    Vec = Builder.CreateTrunc(Vec, ConvertType(LV.getType()), "truncv");
+
   return RValue::get(Vec);
 }
 
@@ -2407,6 +2425,13 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
       // Read/modify/write the vector, inserting the new element.
       llvm::Value *Vec = Builder.CreateLoad(Dst.getVectorAddress(),
                                             Dst.isVolatileQualified());
+      llvm::Type *VecTy = Vec->getType();
+      llvm::Value *SrcVal = Src.getScalarVal();
+
+      if (SrcVal->getType()->getPrimitiveSizeInBits() <
+          VecTy->getScalarSizeInBits())
+        SrcVal = Builder.CreateZExt(SrcVal, VecTy->getScalarType());
+
       auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Vec->getType());
       if (IRStoreTy) {
         auto *IRVecTy = llvm::FixedVectorType::get(
@@ -2414,19 +2439,21 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
         Vec = Builder.CreateBitCast(Vec, IRVecTy);
         // iN --> <N x i1>.
       }
-      llvm::Value *SrcVal = Src.getScalarVal();
+
       // Allow inserting `<1 x T>` into an `<N x T>`. It can happen with scalar
       // types which are mapped to vector LLVM IR types (e.g. for implementing
       // an ABI).
       if (auto *EltTy = dyn_cast<llvm::FixedVectorType>(SrcVal->getType());
           EltTy && EltTy->getNumElements() == 1)
         SrcVal = Builder.CreateBitCast(SrcVal, EltTy->getElementType());
+
       Vec = Builder.CreateInsertElement(Vec, SrcVal, Dst.getVectorIdx(),
                                         "vecins");
       if (IRStoreTy) {
         // <N x i1> --> <iN>.
         Vec = Builder.CreateBitCast(Vec, IRStoreTy);
       }
+
       Builder.CreateStore(Vec, Dst.getVectorAddress(),
                           Dst.isVolatileQualified());
       return;
@@ -2623,14 +2650,12 @@ void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
   // This access turns into a read/modify/write of the vector.  Load the input
   // value now.
   llvm::Value *Vec = Builder.CreateLoad(DstAddr, Dst.isVolatileQualified());
+  llvm::Type *VecTy = Vec->getType();
   const llvm::Constant *Elts = Dst.getExtVectorElts();
 
-  llvm::Value *SrcVal = Src.getScalarVal();
-
   if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) {
     unsigned NumSrcElts = VTy->getNumElements();
-    unsigned NumDstElts =
-        cast<llvm::FixedVectorType>(Vec->getType())->getNumElements();
+    unsigned NumDstElts = cast<llvm::FixedVectorType>(VecTy)->getNumElements();
     if (NumDstElts == NumSrcElts) {
       // Use shuffle vector is the src and destination are the same number of
       // elements and restore the vector mask since it is on the side it will be
@@ -2639,6 +2664,11 @@ void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
       for (unsigned i = 0; i != NumSrcElts; ++i)
         Mask[getAccessedFieldNo(i, Elts)] = i;
 
+      llvm::Value *SrcVal = Src.getScalarVal();
+      if (VecTy->getScalarSizeInBits() >
+          SrcVal->getType()->getScalarSizeInBits())
+        SrcVal = Builder.CreateZExt(SrcVal, VecTy);
+
       Vec = Builder.CreateShuffleVector(SrcVal, Mask);
     } else if (NumDstElts > NumSrcElts) {
       // Extended the source vector to the same length and then shuffle it
@@ -2649,7 +2679,8 @@ void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
       for (unsigned i = 0; i != NumSrcElts; ++i)
         ExtMask.push_back(i);
       ExtMask.resize(NumDstElts, -1);
-      llvm::Value *ExtSrcVal = Builder.CreateShuffleVector(SrcVal, ExtMask);
+      llvm::Value *ExtSrcVal =
+          Builder.CreateShuffleVector(Src.getScalarVal(), ExtMask);
       // build identity
       SmallVector<int, 4> Mask;
       for (unsigned i = 0; i != NumDstElts; ++i)
@@ -2674,6 +2705,11 @@ void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
     // be updating one element.
     unsigned InIdx = getAccessedFieldNo(0, Elts);
     llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx);
+
+    llvm::Value *SrcVal = Src.getScalarVal();
+    if (VecTy->getScalarSizeInBits() > SrcVal->getType()->getScalarSizeInBits())
+      SrcVal = Builder.CreateZExt(SrcVal, VecTy->getScalarType());
+
     Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt);
   }
 
@@ -4701,9 +4737,13 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
 
     // Store the vector to memory (because LValue wants an address).
     Address VecMem = CreateMemTemp(E->getBase()->getType());
+    // need to zero extend an hlsl boolean vector to store it back to memory
+    QualType Ty = E->getBase()->getType();
+    llvm::Type *LTy = convertTypeForLoadStore(Ty, Vec->getType());
+    if (LTy->getScalarSizeInBits() > Vec->getType()->getScalarSizeInBits())
+      Vec = Builder.CreateZExt(Vec, LTy);
     Builder.CreateStore(Vec, VecMem);
-    Base = MakeAddrLValue(VecMem, E->getBase()->getType(),
-                          AlignmentSource::Decl);
+    Base = MakeAddrLValue(VecMem, Ty, AlignmentSource::Decl);
   }
 
   QualType type =
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index 08e42a9e1dcf3..e90881a9743bf 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -1981,7 +1981,10 @@ llvm::Constant *ConstantEmitter::emitForMemory(CodeGenModule &CGM,
   }
 
   // Zero-extend bool.
-  if (C->getType()->isIntegerTy(1) && !destType->isBitIntType()) {
+  // In HLSL bool vectors are stored in memory as a vector of i32
+  if ((C->getType()->isIntegerTy(1) && !destType->isBitIntType()) ||
+      (destType->isExtVectorBoolType() &&
+       !destType->isPackedVectorBoolType(CGM.getContext()))) {
     llvm::Type *boolTy = CGM.getTypes().ConvertTypeForMem(destType);
     llvm::Constant *Res = llvm::ConstantFoldCastOperand(
         llvm::Instruction::ZExt, C, boolTy, CGM.getDataLayout());
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index abe799af32c6e..99b6f563d7c82 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -913,6 +913,7 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
       if (CondConstant)
         incrementProfileCounter(&S);
       if (Executed) {
+        MaybeEmitDeferredVarDeclInit(S.getConditionVariable());
         RunCleanupsScope ExecutedScope(*this);
         EmitStmt(Executed);
       }
@@ -952,10 +953,13 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
   // there is a 'return' within the body, but this is particularly beneficial
   // when one if-stmt is nested within another if-stmt so that all of the MC/DC
   // updates are kept linear and consistent.
-  if (!CGM.getCodeGenOpts().MCDCCoverage)
-    EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock, ThenCount, LH);
-  else {
+  if (!CGM.getCodeGenOpts().MCDCCoverage) {
+    EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock, ThenCount, LH,
+                         /*ConditionalOp=*/nullptr,
+                         /*ConditionalDecl=*/S.getConditionVariable());
+  } else {
     llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
+    MaybeEmitDeferredVarDeclInit(S.getConditionVariable());
     Builder.CreateCondBr(BoolCondVal, ThenBlock, ElseBlock);
   }
 
@@ -1099,6 +1103,8 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
   // execution of the loop body.
   llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
 
+  MaybeEmitDeferredVarDeclInit(S.getConditionVariable());
+
   // while(1) is common, avoid extra exit blocks.  Be sure
   // to correctly handle break/continue though.
   llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal);
@@ -1332,6 +1338,9 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S,
     // C99 6.8.5p2/p4: The first substatement is executed if the expression
     // compares unequal to 0.  The condition must be a scalar type.
     llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
+
+    MaybeEmitDeferredVarDeclInit(S.getConditionVariable());
+
     llvm::MDNode *Weights =
         createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
     if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
@@ -1662,7 +1671,7 @@ void CodeGenFunction::EmitDeclStmt(const DeclStmt &S) {
     EmitStopPoint(&S);
 
   for (const auto *I : S.decls())
-    EmitDecl(*I);
+    EmitDecl(*I, /*EvaluateConditionDecl=*/true);
 }
 
 void CodeGenFunction::EmitBreakStmt(const BreakStmt &S) {
@@ -2227,7 +2236,7 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
       // Emit the condition variable if needed inside the entire cleanup scope
       // used by this special case for constant folded switches.
       if (S.getConditionVariable())
-        EmitDecl(*S.getConditionVariable());
+        EmitDecl(*S.getConditionVariable(), /*EvaluateConditionDecl=*/true);
 
       // At this point, we are no longer "within" a switch instance, so
       // we can temporarily enforce this to ensure that any embedded case
@@ -2259,6 +2268,7 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
   if (S.getConditionVariable())
     EmitDecl(*S.getConditionVariable());
   llvm::Value *CondV = EmitScalarExpr(S.getCond());
+  MaybeEmitDeferredVarDeclInit(S.getConditionVariable());
 
   // Create basic block to hold stuff that comes after switch
   // statement. We also need to create a default block now so that
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 08165e0b28406..447192bc7f60c 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -1534,7 +1534,7 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
     // Initialize helper which will detect jumps which can cause invalid
     // lifetime markers.
     if (ShouldEmitLifetimeMarkers)
-      Bypasses.Init(Body);
+      Bypasses.Init(CGM, Body);
   }
 
   // Emit the standard function prologue.
@@ -1846,7 +1846,8 @@ void CodeGenFunction::EmitBranchToCounterBlock(
 /// LHS and RHS nodes.
 void CodeGenFunction::EmitBranchOnBoolExpr(
     const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock,
-    uint64_t TrueCount, Stmt::Likelihood LH, const Expr *ConditionalOp) {
+    uint64_t TrueCount, Stmt::Likelihood LH, const Expr *ConditionalOp,
+    const VarDecl *ConditionalDecl) {
   Cond = Cond->IgnoreParens();
 
   if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Cond)) {
@@ -2047,6 +2048,8 @@ void CodeGenFunction::EmitBranchOnBoolExpr(
     CondV = EvaluateExprAsBool(Cond);
   }
 
+  MaybeEmitDeferredVarDeclInit(ConditionalDecl);
+
   // If not at the top of the logical operator nest, update MCDC temp with the
   // boolean result of the evaluated condition.
   if (!MCDCLogOpStack.empty()) {
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 018fc66b72a1e..ca00a0e8c6cf4 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -3377,7 +3377,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   /// EmitDecl - Emit a declaration.
   ///
   /// This function can be called with a null (unreachable) insert point.
-  void EmitDecl(const Decl &D);
+  void EmitDecl(const Decl &D, bool EvaluateConditionDecl = false);
 
   /// EmitVarDecl - Emit a local variable declaration.
   ///
@@ -3474,6 +3474,8 @@ class CodeGenFunction : public CodeGenTypeCache {
   void emitAutoVarTypeCleanup(const AutoVarEmission &emission,
                               QualType::DestructionKind dtorKind);
 
+  void MaybeEmitDeferredVarDeclInit(const VarDecl *var);
+
   /// Emits the alloca and debug information for the size expressions for each
   /// dimension of an array. It registers the association of its (1-dimensional)
   /// QualTypes and size expression's debug node, so that CGDebugInfo can
@@ -5180,7 +5182,8 @@ class CodeGenFunction : public CodeGenTypeCache {
   void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock,
                             llvm::BasicBlock *FalseBlock, uint64_t TrueCount,
                             Stmt::Likelihood LH = Stmt::LH_None,
-                            const Expr *ConditionalOp = nullptr);
+                            const Expr *ConditionalOp = nullptr,
+                            const VarDecl *ConditionalDecl = nullptr);
 
   /// Given an assignment `*LHS = RHS`, emit a test that checks if \p RHS is
   /// nonnull, if \p LHS is marked _Nonnull.
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index bca0a932b3495..1f6321a5908e6 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -2068,9 +2068,10 @@ StringRef CodeGenModule::getMangledName(GlobalDecl GD) {
   // Prior work:
   // https://discourse.llvm.org/t/rfc-clang-diagnostic-for-demangling-failures/82835/8
   // https://github.com/llvm/llvm-project/issues/111345
-  // assert((MangledName.startswith("_Z") || MangledName.startswith("?")) &&
-  //        !GD->hasAttr<AsmLabelAttr>() &&
-  //        llvm::demangle(MangledName) != MangledName &&
+  // assert(!((StringRef(MangledName).starts_with("_Z") ||
+  //           StringRef(MangledName).starts_with("?")) &&
+  //          !GD.getDecl()->hasAttr<AsmLabelAttr>() &&
+  //          llvm::demangle(MangledName) == MangledName) &&
   //        "LLVM demangler must demangle clang-generated names");
 
   auto Result = Manglings.insert(std::make_pair(MangledName, GD));
@@ -7300,6 +7301,13 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) {
     getHLSLRuntime().addBuffer(cast<HLSLBufferDecl>(D));
     break;
 
+  case Decl::OpenACCDeclare:
+    EmitOpenACCDeclare(cast<OpenACCDeclareDecl>(D));
+    break;
+  case Decl::OpenACCRoutine:
+    EmitOpenACCRoutine(cast<OpenACCRoutineDecl>(D));
+    break;
+
   default:
     // Make sure we handled everything we should, every other kind is a
     // non-top-level decl.  FIXME: Would be nice to have an isTopLevelDeclKind
diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h
index 83bb5bc54d077..e7c923834238f 100644
--- a/clang/lib/CodeGen/CodeGenModule.h
+++ b/clang/lib/CodeGen/CodeGenModule.h
@@ -1574,9 +1574,11 @@ class CodeGenModule : public CodeGenTypeCache {
                             CodeGenFunction *CGF = nullptr);
 
   // Emit code for the OpenACC Declare declaration.
-  void EmitOpenACCDeclare(const OpenACCDeclareDecl *D, CodeGenFunction *CGF);
+  void EmitOpenACCDeclare(const OpenACCDeclareDecl *D,
+                          CodeGenFunction *CGF = nullptr);
   // Emit code for the OpenACC Routine declaration.
-  void EmitOpenACCRoutine(const OpenACCRoutineDecl *D, CodeGenFunction *CGF);
+  void EmitOpenACCRoutine(const OpenACCRoutineDecl *D,
+                          CodeGenFunction *CGF = nullptr);
 
   /// Emit a code for requires directive.
   /// \param D Requires declaration
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index bd625052cb5ed..dfbd444a850a5 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -115,6 +115,12 @@ llvm::Type *CodeGenTypes::ConvertTypeForMem(QualType T) {
   // Check for the boolean vector case.
   if (T->isExtVectorBoolType()) {
     auto *FixedVT = cast<llvm::FixedVectorType>(R);
+
+    if (Context.getLangOpts().HLSL) {
+      llvm::Type *IRElemTy = ConvertTypeForMem(Context.BoolTy);
+      return llvm::FixedVectorType::get(IRElemTy, FixedVT->getNumElements());
+    }
+
     // Pad to at least one byte.
     uint64_t BytePadded = std::max<uint64_t>(FixedVT->getNumElements(), 8);
     return llvm::IntegerType::get(FixedVT->getContext(), BytePadded);
@@ -657,7 +663,7 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
   case Type::Vector: {
     const auto *VT = cast<VectorType>(Ty);
     // An ext_vector_type of Bool is really a vector of bits.
-    llvm::Type *IRElemTy = VT->isExtVectorBoolType()
+    llvm::Type *IRElemTy = VT->isPackedVectorBoolType(Context)
                                ? llvm::Type::getInt1Ty(getLLVMContext())
                            : VT->getElementType()->isMFloat8Type()
                                ? llvm::Type::getInt8Ty(getLLVMContext())
diff --git a/clang/lib/CodeGen/Targets/AMDGPU.cpp b/clang/lib/CodeGen/Targets/AMDGPU.cpp
index 9d29f31c77881..7d84310ba0bf5 100644
--- a/clang/lib/CodeGen/Targets/AMDGPU.cpp
+++ b/clang/lib/CodeGen/Targets/AMDGPU.cpp
@@ -614,6 +614,20 @@ void AMDGPUTargetCodeGenInfo::setCUDAKernelCallingConvention(
       FT, FT->getExtInfo().withCallingConv(CC_OpenCLKernel));
 }
 
+/// Return IR struct type for rtinfo struct in rocm-device-libs used for device
+/// enqueue.
+///
+/// ptr addrspace(1) kernel_object, i32 private_segment_size,
+/// i32 group_segment_size
+
+static llvm::StructType *
+getAMDGPURuntimeHandleType(llvm::LLVMContext &C,
+                           llvm::Type *KernelDescriptorPtrTy) {
+  llvm::Type *Int32 = llvm::Type::getInt32Ty(C);
+  return llvm::StructType::create(C, {KernelDescriptorPtrTy, Int32, Int32},
+                                  "block.runtime.handle.t");
+}
+
 /// Create an OpenCL kernel for an enqueued block.
 ///
 /// The type of the first argument (the block literal) is the struct type
@@ -653,23 +667,29 @@ llvm::Value *AMDGPUTargetCodeGenInfo::createEnqueuedBlockKernel(
     ArgNames.push_back(
         llvm::MDString::get(C, (Twine("local_arg") + Twine(I)).str()));
   }
-  std::string Name = Invoke->getName().str() + "_kernel";
+
+  llvm::Module &Mod = CGF.CGM.getModule();
+  const llvm::DataLayout &DL = Mod.getDataLayout();
+
+  llvm::Twine Name = Invoke->getName() + "_kernel";
   auto *FT = llvm::FunctionType::get(llvm::Type::getVoidTy(C), ArgTys, false);
+
+  // The kernel itself can be internal, the runtime does not directly access the
+  // kernel address (only the kernel descriptor).
   auto *F = llvm::Function::Create(FT, llvm::GlobalValue::InternalLinkage, Name,
-                                   &CGF.CGM.getModule());
+                                   &Mod);
   F->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
 
   llvm::AttrBuilder KernelAttrs(C);
   // FIXME: The invoke isn't applying the right attributes either
   // FIXME: This is missing setTargetAttributes
   CGF.CGM.addDefaultFunctionDefinitionAttributes(KernelAttrs);
-  KernelAttrs.addAttribute("enqueued-block");
   F->addFnAttrs(KernelAttrs);
 
   auto IP = CGF.Builder.saveIP();
   auto *BB = llvm::BasicBlock::Create(C, "entry", F);
   Builder.SetInsertPoint(BB);
-  const auto BlockAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(BlockTy);
+  const auto BlockAlign = DL.getPrefTypeAlign(BlockTy);
   auto *BlockPtr = Builder.CreateAlloca(BlockTy, nullptr);
   BlockPtr->setAlignment(BlockAlign);
   Builder.CreateAlignedStore(F->arg_begin(), BlockPtr, BlockAlign);
@@ -692,7 +712,39 @@ llvm::Value *AMDGPUTargetCodeGenInfo::createEnqueuedBlockKernel(
   if (CGF.CGM.getCodeGenOpts().EmitOpenCLArgMetadata)
     F->setMetadata("kernel_arg_name", llvm::MDNode::get(C, ArgNames));
 
-  return F;
+  llvm::StructType *HandleTy = getAMDGPURuntimeHandleType(
+      C, llvm::PointerType::get(C, DL.getDefaultGlobalsAddressSpace()));
+  llvm::Constant *RuntimeHandleInitializer =
+      llvm::ConstantAggregateZero::get(HandleTy);
+
+  llvm::Twine RuntimeHandleName = F->getName() + ".runtime.handle";
+
+  // The runtime needs access to the runtime handle as an external symbol. The
+  // runtime handle will need to be made external later, in
+  // AMDGPUExportOpenCLEnqueuedBlocks. The kernel itself has a hidden reference
+  // inside the runtime handle, and is not directly referenced.
+
+  // TODO: We would initialize the first field by declaring F->getName() + ".kd"
+  // to reference the kernel descriptor. The runtime wouldn't need to bother
+  // setting it. We would need to have a final symbol name though.
+  // TODO: Can we directly use an external symbol with getGlobalIdentifier?
+  auto *RuntimeHandle = new llvm::GlobalVariable(
+      Mod, HandleTy,
+      /*isConstant=*/true, llvm::GlobalValue::InternalLinkage,
+      /*Initializer=*/RuntimeHandleInitializer, RuntimeHandleName,
+      /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal,
+      DL.getDefaultGlobalsAddressSpace(),
+      /*isExternallyInitialized=*/true);
+
+  llvm::MDNode *HandleAsMD =
+      llvm::MDNode::get(C, llvm::ValueAsMetadata::get(RuntimeHandle));
+  F->setMetadata(llvm::LLVMContext::MD_associated, HandleAsMD);
+
+  RuntimeHandle->setSection(".amdgpu.kernel.runtime.handle");
+
+  CGF.CGM.addUsedGlobal(F);
+  CGF.CGM.addUsedGlobal(RuntimeHandle);
+  return RuntimeHandle;
 }
 
 void CodeGenModule::handleAMDGPUFlatWorkGroupSizeAttr(
diff --git a/clang/lib/CodeGen/Targets/RISCV.cpp b/clang/lib/CodeGen/Targets/RISCV.cpp
index 081ae8a403111..5aa10ba41f5ed 100644
--- a/clang/lib/CodeGen/Targets/RISCV.cpp
+++ b/clang/lib/CodeGen/Targets/RISCV.cpp
@@ -389,7 +389,7 @@ ABIArgInfo RISCVABIInfo::coerceAndExpandFPCCEligibleStruct(
 bool RISCVABIInfo::detectVLSCCEligibleStruct(QualType Ty, unsigned ABIVLen,
                                              llvm::Type *&VLSType) const {
   // No riscv_vls_cc attribute.
-  if (ABIVLen == 1)
+  if (ABIVLen == 0)
     return false;
 
   // Legal struct for VLS calling convention should fulfill following rules:
diff --git a/clang/lib/CodeGen/VarBypassDetector.cpp b/clang/lib/CodeGen/VarBypassDetector.cpp
index 6eda83dfdef2f..7b2b3542928ad 100644
--- a/clang/lib/CodeGen/VarBypassDetector.cpp
+++ b/clang/lib/CodeGen/VarBypassDetector.cpp
@@ -8,6 +8,7 @@
 
 #include "VarBypassDetector.h"
 
+#include "CodeGenModule.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/Stmt.h"
@@ -17,13 +18,13 @@ using namespace CodeGen;
 
 /// Clear the object and pre-process for the given statement, usually function
 /// body statement.
-void VarBypassDetector::Init(const Stmt *Body) {
+void VarBypassDetector::Init(CodeGenModule &CGM, const Stmt *Body) {
   FromScopes.clear();
   ToScopes.clear();
   Bypasses.clear();
   Scopes = {{~0U, nullptr}};
   unsigned ParentScope = 0;
-  AlwaysBypassed = !BuildScopeInformation(Body, ParentScope);
+  AlwaysBypassed = !BuildScopeInformation(CGM, Body, ParentScope);
   if (!AlwaysBypassed)
     Detect();
 }
@@ -31,7 +32,7 @@ void VarBypassDetector::Init(const Stmt *Body) {
 /// Build scope information for a declaration that is part of a DeclStmt.
 /// Returns false if we failed to build scope information and can't tell for
 /// which vars are being bypassed.
-bool VarBypassDetector::BuildScopeInformation(const Decl *D,
+bool VarBypassDetector::BuildScopeInformation(CodeGenModule &CGM, const Decl *D,
                                               unsigned &ParentScope) {
   const VarDecl *VD = dyn_cast<VarDecl>(D);
   if (VD && VD->hasLocalStorage()) {
@@ -41,7 +42,7 @@ bool VarBypassDetector::BuildScopeInformation(const Decl *D,
 
   if (const VarDecl *VD = dyn_cast<VarDecl>(D))
     if (const Expr *Init = VD->getInit())
-      return BuildScopeInformation(Init, ParentScope);
+      return BuildScopeInformation(CGM, Init, ParentScope);
 
   return true;
 }
@@ -50,7 +51,7 @@ bool VarBypassDetector::BuildScopeInformation(const Decl *D,
 /// LabelAndGotoScopes and recursively walking the AST as needed.
 /// Returns false if we failed to build scope information and can't tell for
 /// which vars are being bypassed.
-bool VarBypassDetector::BuildScopeInformation(const Stmt *S,
+bool VarBypassDetector::BuildScopeInformation(CodeGenModule &CGM, const Stmt *S,
                                               unsigned &origParentScope) {
   // If this is a statement, rather than an expression, scopes within it don't
   // propagate out into the enclosing scope. Otherwise we have to worry about
@@ -68,12 +69,12 @@ bool VarBypassDetector::BuildScopeInformation(const Stmt *S,
 
   case Stmt::SwitchStmtClass:
     if (const Stmt *Init = cast<SwitchStmt>(S)->getInit()) {
-      if (!BuildScopeInformation(Init, ParentScope))
+      if (!BuildScopeInformation(CGM, Init, ParentScope))
         return false;
       ++StmtsToSkip;
     }
     if (const VarDecl *Var = cast<SwitchStmt>(S)->getConditionVariable()) {
-      if (!BuildScopeInformation(Var, ParentScope))
+      if (!BuildScopeInformation(CGM, Var, ParentScope))
         return false;
       ++StmtsToSkip;
     }
@@ -86,7 +87,7 @@ bool VarBypassDetector::BuildScopeInformation(const Stmt *S,
   case Stmt::DeclStmtClass: {
     const DeclStmt *DS = cast<DeclStmt>(S);
     for (auto *I : DS->decls())
-      if (!BuildScopeInformation(I, origParentScope))
+      if (!BuildScopeInformation(CGM, I, origParentScope))
         return false;
     return true;
   }
@@ -126,7 +127,11 @@ bool VarBypassDetector::BuildScopeInformation(const Stmt *S,
     }
 
     // Recursively walk the AST.
-    if (!BuildScopeInformation(SubStmt, ParentScope))
+    bool Result;
+    CGM.runWithSufficientStackSpace(S->getEndLoc(), [&] {
+      Result = BuildScopeInformation(CGM, SubStmt, ParentScope);
+    });
+    if (!Result)
       return false;
   }
   return true;
diff --git a/clang/lib/CodeGen/VarBypassDetector.h b/clang/lib/CodeGen/VarBypassDetector.h
index 164e88c0b2f1b..cc4d387aeaa5b 100644
--- a/clang/lib/CodeGen/VarBypassDetector.h
+++ b/clang/lib/CodeGen/VarBypassDetector.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_LIB_CODEGEN_VARBYPASSDETECTOR_H
 #define LLVM_CLANG_LIB_CODEGEN_VARBYPASSDETECTOR_H
 
+#include "CodeGenModule.h"
 #include "clang/AST/Decl.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
@@ -50,7 +51,7 @@ class VarBypassDetector {
   bool AlwaysBypassed = false;
 
 public:
-  void Init(const Stmt *Body);
+  void Init(CodeGenModule &CGM, const Stmt *Body);
 
   /// Returns true if the variable declaration was by bypassed by any goto or
   /// switch statement.
@@ -59,8 +60,10 @@ class VarBypassDetector {
   }
 
 private:
-  bool BuildScopeInformation(const Decl *D, unsigned &ParentScope);
-  bool BuildScopeInformation(const Stmt *S, unsigned &origParentScope);
+  bool BuildScopeInformation(CodeGenModule &CGM, const Decl *D,
+                             unsigned &ParentScope);
+  bool BuildScopeInformation(CodeGenModule &CGM, const Stmt *S,
+                             unsigned &origParentScope);
   void Detect();
   void Detect(unsigned From, unsigned To);
 };
diff --git a/clang/lib/Driver/Action.cpp b/clang/lib/Driver/Action.cpp
index 0899b8ef00152..ec09726044812 100644
--- a/clang/lib/Driver/Action.cpp
+++ b/clang/lib/Driver/Action.cpp
@@ -50,6 +50,8 @@ const char *Action::getClassName(ActionClass AC) {
     return "static-lib-linker";
   case BinaryAnalyzeJobClass:
     return "binary-analyzer";
+  case BinaryTranslatorJobClass:
+    return "binary-translator";
   }
 
   llvm_unreachable("invalid class");
@@ -459,3 +461,9 @@ void BinaryAnalyzeJobAction::anchor() {}
 
 BinaryAnalyzeJobAction::BinaryAnalyzeJobAction(Action *Input, types::ID Type)
     : JobAction(BinaryAnalyzeJobClass, Input, Type) {}
+
+void BinaryTranslatorJobAction::anchor() {}
+
+BinaryTranslatorJobAction::BinaryTranslatorJobAction(Action *Input,
+                                                     types::ID Type)
+    : JobAction(BinaryTranslatorJobClass, Input, Type) {}
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index eca96c1cce7f7..9457a19255f21 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -226,10 +226,7 @@ std::string CUIDOptions::getCUID(StringRef InputFile,
     else if (UseCUID == Kind::Hash) {
       llvm::MD5 Hasher;
       llvm::MD5::MD5Result Hash;
-      SmallString<256> RealPath;
-      llvm::sys::fs::real_path(InputFile, RealPath,
-                               /*expand_tilde=*/true);
-      Hasher.update(RealPath);
+      Hasher.update(InputFile);
       for (auto *A : Args) {
         if (A->getOption().matches(options::OPT_INPUT))
           continue;
@@ -4672,6 +4669,16 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
       Actions.push_back(C.MakeAction<BinaryAnalyzeJobAction>(
           LastAction, types::TY_DX_CONTAINER));
     }
+    if (TC.requiresBinaryTranslation(Args)) {
+      Action *LastAction = Actions.back();
+      // Metal shader converter runs on DXIL containers, which can either be
+      // validated (in which case they are TY_DX_CONTAINER), or unvalidated
+      // (TY_OBJECT).
+      if (LastAction->getType() == types::TY_DX_CONTAINER ||
+          LastAction->getType() == types::TY_Object)
+        Actions.push_back(C.MakeAction<BinaryTranslatorJobAction>(
+            LastAction, types::TY_DX_CONTAINER));
+    }
   }
 
   // Claim ignored clang-cl options.
@@ -5212,8 +5219,14 @@ void Driver::BuildJobs(Compilation &C) const {
     unsigned NumOutputs = 0;
     unsigned NumIfsOutputs = 0;
     for (const Action *A : C.getActions()) {
+      // The actions below do not increase the number of outputs, when operating
+      // on DX containers.
+      if (A->getType() == types::TY_DX_CONTAINER &&
+          (A->getKind() == clang::driver::Action::BinaryAnalyzeJobClass ||
+           A->getKind() == clang::driver::Action::BinaryTranslatorJobClass))
+        continue;
+
       if (A->getType() != types::TY_Nothing &&
-          A->getType() != types::TY_DX_CONTAINER &&
           !(A->getKind() == Action::IfsMergeJobClass ||
             (A->getType() == clang::driver::types::TY_IFS_CPP &&
              A->getKind() == clang::driver::Action::CompileJobClass &&
@@ -6214,11 +6227,27 @@ const char *Driver::GetNamedOutputPath(Compilation &C, const JobAction &JA,
     return C.addResultFile(C.getArgs().MakeArgString(FcValue.str()), &JA);
   }
 
-  if (JA.getType() == types::TY_Object &&
-      C.getArgs().hasArg(options::OPT_dxc_Fo)) {
+  if ((JA.getType() == types::TY_Object &&
+       C.getArgs().hasArg(options::OPT_dxc_Fo)) ||
+      JA.getType() == types::TY_DX_CONTAINER) {
     StringRef FoValue = C.getArgs().getLastArgValue(options::OPT_dxc_Fo);
-    // TODO: Should we use `MakeCLOutputFilename` here? If so, we can probably
-    // handle this as part of the SLASH_Fo handling below.
+    // If we are targeting DXIL and not validating or translating, we should set
+    // the final result file. Otherwise we should emit to a temporary.
+    if (C.getDefaultToolChain().getTriple().isDXIL()) {
+      const auto &TC = static_cast<const toolchains::HLSLToolChain &>(
+          C.getDefaultToolChain());
+      // Fo can be empty here if the validator is running for a compiler flow
+      // that is using Fc or just printing disassembly.
+      if (TC.isLastJob(C.getArgs(), JA.getKind()) && !FoValue.empty())
+        return C.addResultFile(C.getArgs().MakeArgString(FoValue.str()), &JA);
+      StringRef Name = llvm::sys::path::filename(BaseInput);
+      std::pair<StringRef, StringRef> Split = Name.split('.');
+      const char *Suffix = types::getTypeTempSuffix(JA.getType(), true);
+      return CreateTempFile(C, Split.first, Suffix, false);
+    }
+    // We don't have SPIRV-val integrated (yet), so for now we can assume this
+    // is the final output.
+    assert(C.getDefaultToolChain().getTriple().isSPIRV());
     return C.addResultFile(C.getArgs().MakeArgString(FoValue.str()), &JA);
   }
 
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 65acbe8a9dbea..f3bafbd01c5af 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -210,7 +210,9 @@ static void getAArch64MultilibFlags(const Driver &D,
                                           const llvm::opt::ArgList &Args,
                                           Multilib::flags_list &Result) {
   std::vector<StringRef> Features;
-  tools::aarch64::getAArch64TargetFeatures(D, Triple, Args, Features, false);
+  tools::aarch64::getAArch64TargetFeatures(D, Triple, Args, Features,
+                                           /*ForAS=*/false,
+                                           /*ForMultilib=*/true);
   const auto UnifiedFeatures = tools::unifyTargetFeatures(Features);
   llvm::DenseSet<StringRef> FeatureSet(UnifiedFeatures.begin(),
                                        UnifiedFeatures.end());
@@ -639,6 +641,7 @@ Tool *ToolChain::getTool(Action::ActionClass AC) const {
   case Action::DsymutilJobClass:
   case Action::VerifyDebugInfoJobClass:
   case Action::BinaryAnalyzeJobClass:
+  case Action::BinaryTranslatorJobClass:
     llvm_unreachable("Invalid tool kind.");
 
   case Action::CompileJobClass:
diff --git a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
index 1e2ac4e501baf..6e70effe9e325 100644
--- a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
@@ -196,7 +196,7 @@ void aarch64::getAArch64TargetFeatures(const Driver &D,
                                        const llvm::Triple &Triple,
                                        const ArgList &Args,
                                        std::vector<StringRef> &Features,
-                                       bool ForAS) {
+                                       bool ForAS, bool ForMultilib) {
   Arg *A;
   bool success = true;
   llvm::StringRef WaMArch;
@@ -332,6 +332,19 @@ void aarch64::getAArch64TargetFeatures(const Driver &D,
   } else if (Triple.isOSOpenBSD())
     Features.push_back("+strict-align");
 
+  // Generate execute-only output (no data access to code sections).
+  // This only makes sense for the compiler, not for the assembler.
+  // It's not needed for multilib selection and may hide an unused
+  // argument diagnostic if the code is always run.
+  if (!ForAS && !ForMultilib) {
+    if (Arg *A = Args.getLastArg(options::OPT_mexecute_only,
+                                 options::OPT_mno_execute_only)) {
+      if (A->getOption().matches(options::OPT_mexecute_only)) {
+        Features.push_back("+execute-only");
+      }
+    }
+  }
+
   if (Args.hasArg(options::OPT_ffixed_x1))
     Features.push_back("+reserve-x1");
 
diff --git a/clang/lib/Driver/ToolChains/Arch/AArch64.h b/clang/lib/Driver/ToolChains/Arch/AArch64.h
index 6d071167bd392..2057272867a17 100644
--- a/clang/lib/Driver/ToolChains/Arch/AArch64.h
+++ b/clang/lib/Driver/ToolChains/Arch/AArch64.h
@@ -23,7 +23,7 @@ namespace aarch64 {
 void getAArch64TargetFeatures(const Driver &D, const llvm::Triple &Triple,
                               const llvm::opt::ArgList &Args,
                               std::vector<llvm::StringRef> &Features,
-                              bool ForAS);
+                              bool ForAS, bool ForMultilib = false);
 
 std::string getAArch64TargetCPU(const llvm::opt::ArgList &Args,
                                 const llvm::Triple &Triple, llvm::opt::Arg *&A);
diff --git a/clang/lib/Driver/ToolChains/Arch/ARM.cpp b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
index 51454de1b9dcc..e50cb3836f2c9 100644
--- a/clang/lib/Driver/ToolChains/Arch/ARM.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
@@ -202,13 +202,25 @@ bool arm::useAAPCSForMachO(const llvm::Triple &T) {
          T.getOS() == llvm::Triple::UnknownOS || isARMMProfile(T);
 }
 
-// We follow GCC and support when the backend has support for the MRC/MCR
+// Check whether the architecture backend has support for the MRC/MCR
 // instructions that are used to set the hard thread pointer ("CP15 C13
 // Thread id").
+// This is not identical to ability to use the instruction, as the ARMV6K
+// variants can only use it in Arm mode since they don't support Thumb2
+// encoding.
 bool arm::isHardTPSupported(const llvm::Triple &Triple) {
   int Ver = getARMSubArchVersionNumber(Triple);
   llvm::ARM::ArchKind AK = llvm::ARM::parseArch(Triple.getArchName());
-  return Triple.isARM() || AK == llvm::ARM::ArchKind::ARMV6T2 ||
+  return AK == llvm::ARM::ArchKind::ARMV6K ||
+         AK == llvm::ARM::ArchKind::ARMV6KZ ||
+         (Ver >= 7 && !isARMMProfile(Triple));
+}
+
+// Checks whether the architecture is capable of supporting the Thumb2 encoding
+static bool supportsThumb2Encoding(const llvm::Triple &Triple) {
+  int Ver = arm::getARMSubArchVersionNumber(Triple);
+  llvm::ARM::ArchKind AK = llvm::ARM::parseArch(Triple.getArchName());
+  return AK == llvm::ARM::ArchKind::ARMV6T2 ||
          (Ver >= 7 && AK != llvm::ARM::ArchKind::ARMV8MBaseline);
 }
 
@@ -240,7 +252,13 @@ arm::ReadTPMode arm::getReadTPMode(const Driver &D, const ArgList &Args,
       D.Diag(diag::err_drv_invalid_mtp) << A->getAsString(Args);
     return ReadTPMode::Invalid;
   }
-  return (isHardTPSupported(Triple) ? ReadTPMode::TPIDRURO : ReadTPMode::Soft);
+  // In auto mode we enable HW mode only if both the hardware supports it and
+  // the thumb2 encoding. For example ARMV6T2 supports thumb2, but not hardware.
+  // ARMV6K has HW suport, but not thumb2. Otherwise we could enable it for
+  // ARMV6K in thumb mode.
+  bool autoUseHWTPMode =
+      isHardTPSupported(Triple) && supportsThumb2Encoding(Triple);
+  return autoUseHWTPMode ? ReadTPMode::TPIDRURO : ReadTPMode::Soft;
 }
 
 void arm::setArchNameInTriple(const Driver &D, const ArgList &Args,
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 4ebbd241d2f0b..a6a96286e9857 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -762,8 +762,7 @@ static void addPGOAndCoverageFlags(const ToolChain &TC, Compilation &C,
     else {
       CmdArgs.push_back("-fprofile-continuous");
       // Platforms that require a bias variable:
-      if (T.isOSBinFormatELF() || T.isOSAIX() ||
-          T.isKnownWindowsMSVCEnvironment()) {
+      if (T.isOSBinFormatELF() || T.isOSAIX() || T.isOSWindows()) {
         CmdArgs.push_back("-mllvm");
         CmdArgs.push_back("-runtime-counter-relocation");
       }
diff --git a/clang/lib/Driver/ToolChains/HLSL.cpp b/clang/lib/Driver/ToolChains/HLSL.cpp
index ad44c2cfcd811..22498bff1f251 100644
--- a/clang/lib/Driver/ToolChains/HLSL.cpp
+++ b/clang/lib/Driver/ToolChains/HLSL.cpp
@@ -186,18 +186,32 @@ void tools::hlsl::Validator::ConstructJob(Compilation &C, const JobAction &JA,
   ArgStringList CmdArgs;
   assert(Inputs.size() == 1 && "Unable to handle multiple inputs.");
   const InputInfo &Input = Inputs[0];
-  assert(Input.isFilename() && "Unexpected verify input");
-  // Grabbing the output of the earlier cc1 run.
   CmdArgs.push_back(Input.getFilename());
-  // Use the same name as output.
   CmdArgs.push_back("-o");
-  CmdArgs.push_back(Input.getFilename());
+  CmdArgs.push_back(Output.getFilename());
 
   const char *Exec = Args.MakeArgString(DxvPath);
   C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
                                          Exec, CmdArgs, Inputs, Input));
 }
 
+void tools::hlsl::MetalConverter::ConstructJob(
+    Compilation &C, const JobAction &JA, const InputInfo &Output,
+    const InputInfoList &Inputs, const ArgList &Args,
+    const char *LinkingOutput) const {
+  std::string MSCPath = getToolChain().GetProgramPath("metal-shaderconverter");
+  ArgStringList CmdArgs;
+  assert(Inputs.size() == 1 && "Unable to handle multiple inputs.");
+  const InputInfo &Input = Inputs[0];
+  CmdArgs.push_back(Input.getFilename());
+  CmdArgs.push_back("-o");
+  CmdArgs.push_back(Output.getFilename());
+
+  const char *Exec = Args.MakeArgString(MSCPath);
+  C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
+                                         Exec, CmdArgs, Inputs, Input));
+}
+
 /// DirectX Toolchain
 HLSLToolChain::HLSLToolChain(const Driver &D, const llvm::Triple &Triple,
                              const ArgList &Args)
@@ -214,6 +228,10 @@ Tool *clang::driver::toolchains::HLSLToolChain::getTool(
     if (!Validator)
       Validator.reset(new tools::hlsl::Validator(*this));
     return Validator.get();
+  case Action::BinaryTranslatorJobClass:
+    if (!MetalConverter)
+      MetalConverter.reset(new tools::hlsl::MetalConverter(*this));
+    return MetalConverter.get();
   default:
     return ToolChain::getTool(AC);
   }
@@ -301,3 +319,21 @@ bool HLSLToolChain::requiresValidation(DerivedArgList &Args) const {
   getDriver().Diag(diag::warn_drv_dxc_missing_dxv);
   return false;
 }
+
+bool HLSLToolChain::requiresBinaryTranslation(DerivedArgList &Args) const {
+  return Args.hasArg(options::OPT_metal) && Args.hasArg(options::OPT_dxc_Fo);
+}
+
+bool HLSLToolChain::isLastJob(DerivedArgList &Args,
+                              Action::ActionClass AC) const {
+  bool HasTranslation = requiresBinaryTranslation(Args);
+  bool HasValidation = requiresValidation(Args);
+  // If translation and validation are not required, we should only have one
+  // action.
+  if (!HasTranslation && !HasValidation)
+    return true;
+  if ((HasTranslation && AC == Action::BinaryTranslatorJobClass) ||
+      (!HasTranslation && HasValidation && AC == Action::BinaryAnalyzeJobClass))
+    return true;
+  return false;
+}
diff --git a/clang/lib/Driver/ToolChains/HLSL.h b/clang/lib/Driver/ToolChains/HLSL.h
index b2a31aabab7dc..3824b4252324b 100644
--- a/clang/lib/Driver/ToolChains/HLSL.h
+++ b/clang/lib/Driver/ToolChains/HLSL.h
@@ -29,6 +29,19 @@ class LLVM_LIBRARY_VISIBILITY Validator : public Tool {
                     const llvm::opt::ArgList &TCArgs,
                     const char *LinkingOutput) const override;
 };
+
+class LLVM_LIBRARY_VISIBILITY MetalConverter : public Tool {
+public:
+  MetalConverter(const ToolChain &TC)
+      : Tool("hlsl::MetalConverter", "metal-shaderconverter", TC) {}
+
+  bool hasIntegratedCPP() const override { return false; }
+
+  void ConstructJob(Compilation &C, const JobAction &JA,
+                    const InputInfo &Output, const InputInfoList &Inputs,
+                    const llvm::opt::ArgList &TCArgs,
+                    const char *LinkingOutput) const override;
+};
 } // namespace hlsl
 } // namespace tools
 
@@ -51,12 +64,15 @@ class LLVM_LIBRARY_VISIBILITY HLSLToolChain : public ToolChain {
                 Action::OffloadKind DeviceOffloadKind) const override;
   static std::optional<std::string> parseTargetProfile(StringRef TargetProfile);
   bool requiresValidation(llvm::opt::DerivedArgList &Args) const;
+  bool requiresBinaryTranslation(llvm::opt::DerivedArgList &Args) const;
+  bool isLastJob(llvm::opt::DerivedArgList &Args, Action::ActionClass AC) const;
 
   // Set default DWARF version to 4 for DXIL uses version 4.
   unsigned GetDefaultDwarfVersion() const override { return 4; }
 
 private:
   mutable std::unique_ptr<tools::hlsl::Validator> Validator;
+  mutable std::unique_ptr<tools::hlsl::MetalConverter> MetalConverter;
 };
 
 } // end namespace toolchains
diff --git a/clang/lib/Driver/ToolChains/Hexagon.cpp b/clang/lib/Driver/ToolChains/Hexagon.cpp
index 7ca5ab9af8810..6ea701a7882d1 100644
--- a/clang/lib/Driver/ToolChains/Hexagon.cpp
+++ b/clang/lib/Driver/ToolChains/Hexagon.cpp
@@ -313,6 +313,7 @@ constructHexagonLinkArgs(Compilation &C, const JobAction &JA,
                                      // handled somewhere else.
   Args.ClaimAllArgs(options::OPT_static_libgcc);
 
+  CmdArgs.push_back("--eh-frame-hdr");
   //----------------------------------------------------------------------------
   //
   //----------------------------------------------------------------------------
diff --git a/clang/lib/Driver/ToolChains/SPIRV.cpp b/clang/lib/Driver/ToolChains/SPIRV.cpp
index 5a7894f5435fc..8ff39fe42f3aa 100644
--- a/clang/lib/Driver/ToolChains/SPIRV.cpp
+++ b/clang/lib/Driver/ToolChains/SPIRV.cpp
@@ -93,12 +93,6 @@ void SPIRV::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
   constructAssembleCommand(C, *this, JA, Output, Inputs[0], {});
 }
 
-clang::driver::Tool *SPIRVToolChain::getTranslator() const {
-  if (!Translator)
-    Translator = std::make_unique<SPIRV::Translator>(*this);
-  return Translator.get();
-}
-
 clang::driver::Tool *SPIRVToolChain::getAssembler() const {
   if (!Assembler)
     Assembler = std::make_unique<SPIRV::Assembler>(*this);
@@ -114,8 +108,6 @@ clang::driver::Tool *SPIRVToolChain::getTool(Action::ActionClass AC) const {
   switch (AC) {
   default:
     break;
-  case Action::BackendJobClass:
-    return SPIRVToolChain::getTranslator();
   case Action::AssembleJobClass:
     return SPIRVToolChain::getAssembler();
   }
diff --git a/clang/lib/Driver/ToolChains/SPIRV.h b/clang/lib/Driver/ToolChains/SPIRV.h
index 6223d55eca643..924eb01adcbbf 100644
--- a/clang/lib/Driver/ToolChains/SPIRV.h
+++ b/clang/lib/Driver/ToolChains/SPIRV.h
@@ -69,7 +69,6 @@ class LLVM_LIBRARY_VISIBILITY Assembler final : public Tool {
 namespace toolchains {
 
 class LLVM_LIBRARY_VISIBILITY SPIRVToolChain : public ToolChain {
-  mutable std::unique_ptr<Tool> Translator;
   mutable std::unique_ptr<Tool> Assembler;
 
 public:
@@ -78,7 +77,7 @@ class LLVM_LIBRARY_VISIBILITY SPIRVToolChain : public ToolChain {
 
   bool useIntegratedAs() const override { return true; }
 
-  bool IsIntegratedBackendDefault() const override { return false; }
+  bool IsIntegratedBackendDefault() const override { return true; }
   bool IsNonIntegratedBackendSupported() const override { return true; }
   bool IsMathErrnoDefault() const override { return false; }
   bool isCrossCompiling() const override { return true; }
@@ -97,7 +96,6 @@ class LLVM_LIBRARY_VISIBILITY SPIRVToolChain : public ToolChain {
   Tool *buildLinker() const override;
 
 private:
-  clang::driver::Tool *getTranslator() const;
   clang::driver::Tool *getAssembler() const;
 
   bool NativeLLVMSupport;
diff --git a/clang/lib/Format/UnwrappedLineFormatter.cpp b/clang/lib/Format/UnwrappedLineFormatter.cpp
index 14e984529d640..000a5105ca407 100644
--- a/clang/lib/Format/UnwrappedLineFormatter.cpp
+++ b/clang/lib/Format/UnwrappedLineFormatter.cpp
@@ -116,36 +116,18 @@ class LevelIndentTracker {
         Style.isCSharp()) {
       return 0;
     }
-
-    auto IsAccessModifier = [&](const FormatToken &RootToken) {
-      if (Line.Type == LT_AccessModifier || RootToken.isObjCAccessSpecifier())
-        return true;
-
-      const auto *Next = RootToken.Next;
-
-      // Handle Qt signals.
-      if (RootToken.isOneOf(Keywords.kw_signals, Keywords.kw_qsignals) &&
-          Next && Next->is(tok::colon)) {
-        return true;
-      }
-
-      if (Next && Next->isOneOf(Keywords.kw_slots, Keywords.kw_qslots) &&
-          Next->Next && Next->Next->is(tok::colon)) {
-        return true;
-      }
-
-      // Handle malformed access specifier e.g. 'private' without trailing ':'.
-      return !Next && RootToken.isAccessSpecifier(false);
-    };
-
-    if (IsAccessModifier(*Line.First)) {
+    const auto &RootToken = *Line.First;
+    if (Line.Type == LT_AccessModifier ||
+        RootToken.isAccessSpecifier(/*ColonRequired=*/false) ||
+        RootToken.isObjCAccessSpecifier() ||
+        (RootToken.isOneOf(Keywords.kw_signals, Keywords.kw_qsignals) &&
+         RootToken.Next && RootToken.Next->is(tok::colon))) {
       // The AccessModifierOffset may be overridden by IndentAccessModifiers,
       // in which case we take a negative value of the IndentWidth to simulate
       // the upper indent level.
       return Style.IndentAccessModifiers ? -Style.IndentWidth
                                          : Style.AccessModifierOffset;
     }
-
     return 0;
   }
 
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index efb22bcdbe53f..a6e0596add5d2 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -2562,12 +2562,12 @@ bool UnwrappedLineParser::parseBracedList(bool IsAngleBracket, bool IsEnum) {
 /// Returns whether there is a `=` token between the parentheses.
 bool UnwrappedLineParser::parseParens(TokenType AmpAmpTokenType) {
   assert(FormatTok->is(tok::l_paren) && "'(' expected.");
-  auto *LeftParen = FormatTok;
+  auto *LParen = FormatTok;
   bool SeenComma = false;
   bool SeenEqual = false;
   bool MightBeFoldExpr = false;
-  const bool MightBeStmtExpr = Tokens->peekNextToken()->is(tok::l_brace);
   nextToken();
+  const bool MightBeStmtExpr = FormatTok->is(tok::l_brace);
   do {
     switch (FormatTok->Tok.getKind()) {
     case tok::l_paren:
@@ -2577,44 +2577,60 @@ bool UnwrappedLineParser::parseParens(TokenType AmpAmpTokenType) {
         parseChildBlock();
       break;
     case tok::r_paren: {
-      auto *Prev = LeftParen->Previous;
-      if (!MightBeStmtExpr && !MightBeFoldExpr && !Line->InMacroBody &&
-          Style.RemoveParentheses > FormatStyle::RPS_Leave) {
-        const auto *Next = Tokens->peekNextToken();
-        const bool DoubleParens =
-            Prev && Prev->is(tok::l_paren) && Next && Next->is(tok::r_paren);
-        const bool CommaSeparated =
-            !DoubleParens && Prev && Prev->isOneOf(tok::l_paren, tok::comma) &&
-            Next && Next->isOneOf(tok::comma, tok::r_paren);
-        const auto *PrevPrev = Prev ? Prev->getPreviousNonComment() : nullptr;
-        const bool Excluded =
-            PrevPrev &&
-            (PrevPrev->isOneOf(tok::kw___attribute, tok::kw_decltype) ||
-             SeenComma ||
-             (SeenEqual &&
-              (PrevPrev->isOneOf(tok::kw_if, tok::kw_while) ||
-               PrevPrev->endsSequence(tok::kw_constexpr, tok::kw_if))));
-        const bool ReturnParens =
-            Style.RemoveParentheses == FormatStyle::RPS_ReturnStatement &&
-            ((NestedLambdas.empty() && !IsDecltypeAutoFunction) ||
-             (!NestedLambdas.empty() && !NestedLambdas.back())) &&
-            Prev && Prev->isOneOf(tok::kw_return, tok::kw_co_return) && Next &&
-            Next->is(tok::semi);
-        if ((DoubleParens && !Excluded) || (CommaSeparated && !SeenComma) ||
-            ReturnParens) {
-          LeftParen->Optional = true;
-          FormatTok->Optional = true;
-        }
-      }
+      auto *Prev = LParen->Previous;
+      auto *RParen = FormatTok;
+      nextToken();
       if (Prev) {
+        auto OptionalParens = [&] {
+          if (MightBeStmtExpr || MightBeFoldExpr || Line->InMacroBody ||
+              SeenComma || Style.RemoveParentheses == FormatStyle::RPS_Leave) {
+            return false;
+          }
+          const bool DoubleParens =
+              Prev->is(tok::l_paren) && FormatTok->is(tok::r_paren);
+          if (DoubleParens) {
+            const auto *PrevPrev = Prev->getPreviousNonComment();
+            const bool Excluded =
+                PrevPrev &&
+                (PrevPrev->isOneOf(tok::kw___attribute, tok::kw_decltype) ||
+                 (SeenEqual &&
+                  (PrevPrev->isOneOf(tok::kw_if, tok::kw_while) ||
+                   PrevPrev->endsSequence(tok::kw_constexpr, tok::kw_if))));
+            if (!Excluded)
+              return true;
+          } else {
+            const bool CommaSeparated =
+                Prev->isOneOf(tok::l_paren, tok::comma) &&
+                FormatTok->isOneOf(tok::comma, tok::r_paren);
+            if (CommaSeparated &&
+                // LParen is not preceded by ellipsis, comma.
+                !Prev->endsSequence(tok::comma, tok::ellipsis) &&
+                // RParen is not followed by comma, ellipsis.
+                !(FormatTok->is(tok::comma) &&
+                  Tokens->peekNextToken()->is(tok::ellipsis))) {
+              return true;
+            }
+            const bool ReturnParens =
+                Style.RemoveParentheses == FormatStyle::RPS_ReturnStatement &&
+                ((NestedLambdas.empty() && !IsDecltypeAutoFunction) ||
+                 (!NestedLambdas.empty() && !NestedLambdas.back())) &&
+                Prev->isOneOf(tok::kw_return, tok::kw_co_return) &&
+                FormatTok->is(tok::semi);
+            if (ReturnParens)
+              return true;
+          }
+          return false;
+        };
         if (Prev->is(TT_TypenameMacro)) {
-          LeftParen->setFinalizedType(TT_TypeDeclarationParen);
-          FormatTok->setFinalizedType(TT_TypeDeclarationParen);
-        } else if (Prev->is(tok::greater) && FormatTok->Previous == LeftParen) {
+          LParen->setFinalizedType(TT_TypeDeclarationParen);
+          RParen->setFinalizedType(TT_TypeDeclarationParen);
+        } else if (Prev->is(tok::greater) && RParen->Previous == LParen) {
           Prev->setFinalizedType(TT_TemplateCloser);
+        } else if (OptionalParens()) {
+          LParen->Optional = true;
+          RParen->Optional = true;
         }
       }
-      nextToken();
       return SeenEqual;
     }
     case tok::r_brace:
@@ -3386,75 +3402,15 @@ void UnwrappedLineParser::parseSwitch(bool IsExpr) {
     NestedTooDeep.pop_back();
 }
 
-// Operators that can follow a C variable.
-static bool isCOperatorFollowingVar(tok::TokenKind Kind) {
-  switch (Kind) {
-  case tok::ampamp:
-  case tok::ampequal:
-  case tok::arrow:
-  case tok::caret:
-  case tok::caretequal:
-  case tok::comma:
-  case tok::ellipsis:
-  case tok::equal:
-  case tok::equalequal:
-  case tok::exclaim:
-  case tok::exclaimequal:
-  case tok::greater:
-  case tok::greaterequal:
-  case tok::greatergreater:
-  case tok::greatergreaterequal:
-  case tok::l_paren:
-  case tok::l_square:
-  case tok::less:
-  case tok::lessequal:
-  case tok::lessless:
-  case tok::lesslessequal:
-  case tok::minus:
-  case tok::minusequal:
-  case tok::minusminus:
-  case tok::percent:
-  case tok::percentequal:
-  case tok::period:
-  case tok::pipe:
-  case tok::pipeequal:
-  case tok::pipepipe:
-  case tok::plus:
-  case tok::plusequal:
-  case tok::plusplus:
-  case tok::question:
-  case tok::r_brace:
-  case tok::r_paren:
-  case tok::r_square:
-  case tok::semi:
-  case tok::slash:
-  case tok::slashequal:
-  case tok::star:
-  case tok::starequal:
-    return true;
-  default:
-    return false;
-  }
-}
-
 void UnwrappedLineParser::parseAccessSpecifier() {
-  FormatToken *AccessSpecifierCandidate = FormatTok;
   nextToken();
   // Understand Qt's slots.
   if (FormatTok->isOneOf(Keywords.kw_slots, Keywords.kw_qslots))
     nextToken();
   // Otherwise, we don't know what it is, and we'd better keep the next token.
-  if (FormatTok->is(tok::colon)) {
+  if (FormatTok->is(tok::colon))
     nextToken();
-    addUnwrappedLine();
-  } else if (FormatTok->isNot(tok::coloncolon) &&
-             !isCOperatorFollowingVar(FormatTok->Tok.getKind())) {
-    // Not a variable name nor namespace name.
-    addUnwrappedLine();
-  } else if (AccessSpecifierCandidate) {
-    // Consider the access specifier to be a C identifier.
-    AccessSpecifierCandidate->Tok.setKind(tok::identifier);
-  }
+  addUnwrappedLine();
 }
 
 /// \brief Parses a requires, decides if it is a clause or an expression.
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index c11c857ea0606..e9e96683ca51f 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -1406,7 +1406,7 @@ static bool readASTAfterCompileModule(CompilerInstance &ImportingInstance,
                                       SourceLocation ImportLoc,
                                       SourceLocation ModuleNameLoc,
                                       Module *Module, StringRef ModuleFileName,
-                                      bool *OutOfDate) {
+                                      bool *OutOfDate, bool *Missing) {
   DiagnosticsEngine &Diags = ImportingInstance.getDiagnostics();
 
   unsigned ModuleLoadCapabilities = ASTReader::ARR_Missing;
@@ -1427,6 +1427,12 @@ static bool readASTAfterCompileModule(CompilerInstance &ImportingInstance,
     return false;
   }
 
+  // The caller wants to handle missing module files.
+  if (Missing && ReadResult == ASTReader::Missing) {
+    *Missing = true;
+    return false;
+  }
+
   // The ASTReader didn't diagnose the error, so conservatively report it.
   if (ReadResult == ASTReader::Missing || !Diags.hasErrorOccurred())
     Diags.Report(ModuleNameLoc, diag::err_module_not_built)
@@ -1452,7 +1458,7 @@ static bool compileModuleAndReadASTImpl(CompilerInstance &ImportingInstance,
 
   return readASTAfterCompileModule(ImportingInstance, ImportLoc, ModuleNameLoc,
                                    Module, ModuleFileName,
-                                   /*OutOfDate=*/nullptr);
+                                   /*OutOfDate=*/nullptr, /*Missing=*/nullptr);
 }
 
 /// Compile a module in a separate compiler instance and read the AST,
@@ -1477,29 +1483,28 @@ static bool compileModuleAndReadASTBehindLock(
   llvm::sys::fs::create_directories(Dir);
 
   while (true) {
-    llvm::LockFileManager Locked(ModuleFileName);
-    switch (Locked) {
-    case llvm::LockFileManager::LFS_Error:
+    llvm::LockFileManager Lock(ModuleFileName);
+    bool Owned;
+    if (llvm::Error Err = Lock.tryLock().moveInto(Owned)) {
       // ModuleCache takes care of correctness and locks are only necessary for
       // performance. Fallback to building the module in case of any lock
       // related errors.
       Diags.Report(ModuleNameLoc, diag::remark_module_lock_failure)
-          << Module->Name << Locked.getErrorMessage();
+          << Module->Name << toString(std::move(Err));
       // Clear out any potential leftover.
-      Locked.unsafeRemoveLockFile();
-      [[fallthrough]];
-    case llvm::LockFileManager::LFS_Owned:
+      Lock.unsafeRemoveLockFile();
+      return compileModuleAndReadASTImpl(ImportingInstance, ImportLoc,
+                                         ModuleNameLoc, Module, ModuleFileName);
+    }
+    if (Owned) {
       // We're responsible for building the module ourselves.
       return compileModuleAndReadASTImpl(ImportingInstance, ImportLoc,
                                          ModuleNameLoc, Module, ModuleFileName);
-
-    case llvm::LockFileManager::LFS_Shared:
-      break; // The interesting case.
     }
 
     // Someone else is responsible for building the module. Wait for them to
     // finish.
-    switch (Locked.waitForUnlock()) {
+    switch (Lock.waitForUnlock()) {
     case llvm::LockFileManager::Res_Success:
       break; // The interesting case.
     case llvm::LockFileManager::Res_OwnerDied:
@@ -1511,21 +1516,23 @@ static bool compileModuleAndReadASTBehindLock(
       Diags.Report(ModuleNameLoc, diag::remark_module_lock_timeout)
           << Module->Name;
       // Clear the lock file so that future invocations can make progress.
-      Locked.unsafeRemoveLockFile();
+      Lock.unsafeRemoveLockFile();
       continue;
     }
 
     // Read the module that was just written by someone else.
     bool OutOfDate = false;
+    bool Missing = false;
     if (readASTAfterCompileModule(ImportingInstance, ImportLoc, ModuleNameLoc,
-                                  Module, ModuleFileName, &OutOfDate))
+                                  Module, ModuleFileName, &OutOfDate, &Missing))
       return true;
-    if (!OutOfDate)
+    if (!OutOfDate && !Missing)
       return false;
 
-    // The module may be out of date in the presence of file system races,
-    // or if one of its imports depends on header search paths that are not
-    // consistent with this ImportingInstance.  Try again...
+    // The module may be missing or out of date in the presence of file system
+    // races. It may also be out of date if one of its imports depends on header
+    // search paths that are not consistent with this ImportingInstance.
+    // Try again...
   }
 }
 
diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp
index 60e103e643e27..1ea4a2e9e88cf 100644
--- a/clang/lib/Frontend/FrontendActions.cpp
+++ b/clang/lib/Frontend/FrontendActions.cpp
@@ -403,8 +403,7 @@ class DefaultTemplateInstCallback : public TemplateInstantiationCallback {
   }
 
 private:
-  static std::optional<std::string>
-  toString(CodeSynthesisContext::SynthesisKind Kind) {
+  static std::string toString(CodeSynthesisContext::SynthesisKind Kind) {
     switch (Kind) {
     case CodeSynthesisContext::TemplateInstantiation:
       return "TemplateInstantiation";
@@ -462,10 +461,8 @@ class DefaultTemplateInstCallback : public TemplateInstantiationCallback {
       return "TypeAliasTemplateInstantiation";
     case CodeSynthesisContext::PartialOrderingTTP:
       return "PartialOrderingTTP";
-    case CodeSynthesisContext::CheckTemplateParameter:
-      return std::nullopt;
     }
-    return std::nullopt;
+    return "";
   }
 
   template <bool BeginInstantiation>
@@ -473,14 +470,12 @@ class DefaultTemplateInstCallback : public TemplateInstantiationCallback {
                                     const CodeSynthesisContext &Inst) {
     std::string YAML;
     {
-      std::optional<TemplightEntry> Entry =
-          getTemplightEntry<BeginInstantiation>(TheSema, Inst);
-      if (!Entry)
-        return;
       llvm::raw_string_ostream OS(YAML);
       llvm::yaml::Output YO(OS);
+      TemplightEntry Entry =
+          getTemplightEntry<BeginInstantiation>(TheSema, Inst);
       llvm::yaml::EmptyContext Context;
-      llvm::yaml::yamlize(YO, *Entry, true, Context);
+      llvm::yaml::yamlize(YO, Entry, true, Context);
     }
     Out << "---" << YAML << "\n";
   }
@@ -560,13 +555,10 @@ class DefaultTemplateInstCallback : public TemplateInstantiationCallback {
   }
 
   template <bool BeginInstantiation>
-  static std::optional<TemplightEntry>
-  getTemplightEntry(const Sema &TheSema, const CodeSynthesisContext &Inst) {
+  static TemplightEntry getTemplightEntry(const Sema &TheSema,
+                                          const CodeSynthesisContext &Inst) {
     TemplightEntry Entry;
-    std::optional<std::string> Kind = toString(Inst.Kind);
-    if (!Kind)
-      return std::nullopt;
-    Entry.Kind = *Kind;
+    Entry.Kind = toString(Inst.Kind);
     Entry.Event = BeginInstantiation ? "Begin" : "End";
     llvm::raw_string_ostream OS(Entry.Name);
     printEntryName(TheSema, Inst.Entity, OS);
diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt
index e5bf8f35f7d52..d26de236998ca 100644
--- a/clang/lib/Headers/CMakeLists.txt
+++ b/clang/lib/Headers/CMakeLists.txt
@@ -87,6 +87,7 @@ set(hlsl_h
 set(hlsl_subdir_files
   hlsl/hlsl_basic_types.h
   hlsl/hlsl_alias_intrinsics.h
+  hlsl/hlsl_intrinsic_helpers.h
   hlsl/hlsl_intrinsics.h
   hlsl/hlsl_detail.h
   )
diff --git a/clang/lib/Headers/hlsl.h b/clang/lib/Headers/hlsl.h
index 6edfd949f2b97..4be68eb84a34a 100644
--- a/clang/lib/Headers/hlsl.h
+++ b/clang/lib/Headers/hlsl.h
@@ -16,7 +16,12 @@
 #pragma clang diagnostic ignored "-Whlsl-dxc-compatability"
 #endif
 
+// Basic types, type traits and type-independent templates.
 #include "hlsl/hlsl_basic_types.h"
+#include "hlsl/hlsl_detail.h"
+
+// HLSL standard library function declarations/definitions.
+#include "hlsl/hlsl_alias_intrinsics.h"
 #include "hlsl/hlsl_intrinsics.h"
 
 #if defined(__clang__)
diff --git a/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h
index 7573f6e024167..89dfeb475488e 100644
--- a/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h
@@ -2123,6 +2123,41 @@ template <typename T, int Sz>
 _HLSL_BUILTIN_ALIAS(__builtin_hlsl_select)
 vector<T, Sz> select(vector<bool, Sz>, vector<T, Sz>, vector<T, Sz>);
 
+/// \fn vector<T,Sz> select(vector<bool,Sz> Conds, T TrueVal,
+///                         vector<T,Sz> FalseVals)
+/// \brief ternary operator for vectors. All vectors must be the same size.
+/// \param Conds The Condition input values.
+/// \param TrueVal The scalar value to splat from when conditions are true.
+/// \param FalseVals The vector values are chosen from when conditions are
+/// false.
+
+template <typename T, int Sz>
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_select)
+vector<T, Sz> select(vector<bool, Sz>, T, vector<T, Sz>);
+
+/// \fn vector<T,Sz> select(vector<bool,Sz> Conds, vector<T,Sz> TrueVals,
+///                         T FalseVal)
+/// \brief ternary operator for vectors. All vectors must be the same size.
+/// \param Conds The Condition input values.
+/// \param TrueVals The vector values are chosen from when conditions are true.
+/// \param FalseVal The scalar value to splat from when conditions are false.
+
+template <typename T, int Sz>
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_select)
+vector<T, Sz> select(vector<bool, Sz>, vector<T, Sz>, T);
+
+/// \fn vector<T,Sz> select(vector<bool,Sz> Conds, vector<T,Sz> TrueVals,
+///                         T FalseVal)
+/// \brief ternary operator for vectors. All vectors must be the same size.
+/// \param Conds The Condition input values.
+/// \param TrueVal The scalar value to splat from when conditions are true.
+/// \param FalseVal The scalar value to splat from when conditions are false.
+
+template <typename T, int Sz>
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_select)
+__detail::enable_if_t<__detail::is_arithmetic<T>::Value, vector<T, Sz>> select(
+    vector<bool, Sz>, T, T);
+
 //===----------------------------------------------------------------------===//
 // sin builtins
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/Headers/hlsl/hlsl_detail.h b/clang/lib/Headers/hlsl/hlsl_detail.h
index 39254a3cc3a0a..80c4900121dfb 100644
--- a/clang/lib/Headers/hlsl/hlsl_detail.h
+++ b/clang/lib/Headers/hlsl/hlsl_detail.h
@@ -1,4 +1,4 @@
-//===----- detail.h - HLSL definitions for intrinsics ----------===//
+//===----- hlsl_detail.h - HLSL definitions for intrinsics ----------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -9,8 +9,6 @@
 #ifndef _HLSL_HLSL_DETAILS_H_
 #define _HLSL_HLSL_DETAILS_H_
 
-#include "hlsl_alias_intrinsics.h"
-
 namespace hlsl {
 
 namespace __detail {
@@ -43,59 +41,13 @@ constexpr enable_if_t<sizeof(U) == sizeof(T), U> bit_cast(T F) {
   return __builtin_bit_cast(U, F);
 }
 
-constexpr vector<uint, 4> d3d_color_to_ubyte4_impl(vector<float, 4> V) {
-  // Use the same scaling factor used by FXC, and DXC for DXIL
-  // (i.e., 255.001953)
-  // https://github.com/microsoft/DirectXShaderCompiler/blob/070d0d5a2beacef9eeb51037a9b04665716fd6f3/lib/HLSL/HLOperationLower.cpp#L666C1-L697C2
-  // The DXC implementation refers to a comment on the following stackoverflow
-  // discussion to justify the scaling factor: "Built-in rounding, necessary
-  // because of truncation. 0.001953 * 256 = 0.5"
-  // https://stackoverflow.com/questions/52103720/why-does-d3dcolortoubyte4-multiplies-components-by-255-001953f
-  return V.zyxw * 255.001953f;
-}
-
-template <typename T>
-constexpr enable_if_t<is_same<float, T>::value || is_same<half, T>::value, T>
-length_impl(T X) {
-  return abs(X);
-}
-
-template <typename T, int N>
-constexpr enable_if_t<is_same<float, T>::value || is_same<half, T>::value, T>
-length_vec_impl(vector<T, N> X) {
-#if (__has_builtin(__builtin_spirv_length))
-  return __builtin_spirv_length(X);
-#else
-  return sqrt(dot(X, X));
-#endif
-}
-
-template <typename T>
-constexpr enable_if_t<is_same<float, T>::value || is_same<half, T>::value, T>
-distance_impl(T X, T Y) {
-  return length_impl(X - Y);
-}
+template <typename T> struct is_arithmetic {
+  static const bool Value = __is_arithmetic(T);
+};
 
 template <typename T, int N>
-constexpr enable_if_t<is_same<float, T>::value || is_same<half, T>::value, T>
-distance_vec_impl(vector<T, N> X, vector<T, N> Y) {
-  return length_vec_impl(X - Y);
-}
-
-template <typename T>
-constexpr enable_if_t<is_same<float, T>::value || is_same<half, T>::value, T>
-reflect_impl(T I, T N) {
-  return I - 2 * N * I * N;
-}
-
-template <typename T, int L>
-constexpr vector<T, L> reflect_vec_impl(vector<T, L> I, vector<T, L> N) {
-#if (__has_builtin(__builtin_spirv_reflect))
-  return __builtin_spirv_reflect(I, N);
-#else
-  return I - 2 * N * dot(I, N);
-#endif
-}
+using HLSL_FIXED_VECTOR =
+    vector<__detail::enable_if_t<(N > 1 && N <= 4), T>, N>;
 
 } // namespace __detail
 } // namespace hlsl
diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h b/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h
new file mode 100644
index 0000000000000..87b52792447f6
--- /dev/null
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h
@@ -0,0 +1,63 @@
+//===----- hlsl_intrinsic_helpers.h - HLSL helpers intrinsics -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _HLSL_HLSL_INTRINSIC_HELPERS_H_
+#define _HLSL_HLSL_INTRINSIC_HELPERS_H_
+
+namespace hlsl {
+namespace __detail {
+
+constexpr vector<uint, 4> d3d_color_to_ubyte4_impl(vector<float, 4> V) {
+  // Use the same scaling factor used by FXC, and DXC for DXIL
+  // (i.e., 255.001953)
+  // https://github.com/microsoft/DirectXShaderCompiler/blob/070d0d5a2beacef9eeb51037a9b04665716fd6f3/lib/HLSL/HLOperationLower.cpp#L666C1-L697C2
+  // The DXC implementation refers to a comment on the following stackoverflow
+  // discussion to justify the scaling factor: "Built-in rounding, necessary
+  // because of truncation. 0.001953 * 256 = 0.5"
+  // https://stackoverflow.com/questions/52103720/why-does-d3dcolortoubyte4-multiplies-components-by-255-001953f
+  return V.zyxw * 255.001953f;
+}
+
+template <typename T> constexpr T length_impl(T X) { return abs(X); }
+
+template <typename T, int N>
+constexpr enable_if_t<is_same<float, T>::value || is_same<half, T>::value, T>
+length_vec_impl(vector<T, N> X) {
+#if (__has_builtin(__builtin_spirv_length))
+  return __builtin_spirv_length(X);
+#else
+  return sqrt(dot(X, X));
+#endif
+}
+
+template <typename T> constexpr T distance_impl(T X, T Y) {
+  return length_impl(X - Y);
+}
+
+template <typename T, int N>
+constexpr enable_if_t<is_same<float, T>::value || is_same<half, T>::value, T>
+distance_vec_impl(vector<T, N> X, vector<T, N> Y) {
+  return length_vec_impl(X - Y);
+}
+
+template <typename T> constexpr T reflect_impl(T I, T N) {
+  return I - 2 * N * I * N;
+}
+
+template <typename T, int L>
+constexpr vector<T, L> reflect_vec_impl(vector<T, L> I, vector<T, L> N) {
+#if (__has_builtin(__builtin_spirv_reflect))
+  return __builtin_spirv_reflect(I, N);
+#else
+  return I - 2 * N * dot(I, N);
+#endif
+}
+} // namespace __detail
+} // namespace hlsl
+
+#endif // _HLSL_HLSL_INTRINSIC_HELPERS_H_
diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
index fe9441080433d..47a7066b2b3e1 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
@@ -9,7 +9,7 @@
 #ifndef _HLSL_HLSL_INTRINSICS_H_
 #define _HLSL_HLSL_INTRINSICS_H_
 
-#include "hlsl_detail.h"
+#include "hlsl/hlsl_intrinsic_helpers.h"
 
 namespace hlsl {
 
@@ -89,23 +89,31 @@ void asuint(double4, out uint4, out uint4);
 /// \param X The X input value.
 /// \param Y The Y input value.
 
+template <typename T>
 _HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
-const inline half distance(half X, half Y) {
+const inline __detail::enable_if_t<__detail::is_arithmetic<T>::Value &&
+                                       __detail::is_same<half, T>::value,
+                                   T> distance(T X, T Y) {
   return __detail::distance_impl(X, Y);
 }
 
-const inline float distance(float X, float Y) {
+template <typename T>
+const inline __detail::enable_if_t<
+    __detail::is_arithmetic<T>::Value && __detail::is_same<float, T>::value, T>
+distance(T X, T Y) {
   return __detail::distance_impl(X, Y);
 }
 
 template <int N>
 _HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
-const inline half distance(vector<half, N> X, vector<half, N> Y) {
+const inline half distance(__detail::HLSL_FIXED_VECTOR<half, N> X,
+                           __detail::HLSL_FIXED_VECTOR<half, N> Y) {
   return __detail::distance_vec_impl(X, Y);
 }
 
 template <int N>
-const inline float distance(vector<float, N> X, vector<float, N> Y) {
+const inline float distance(__detail::HLSL_FIXED_VECTOR<float, N> X,
+                            __detail::HLSL_FIXED_VECTOR<float, N> Y) {
   return __detail::distance_vec_impl(X, Y);
 }
 
@@ -119,17 +127,29 @@ const inline float distance(vector<float, N> X, vector<float, N> Y) {
 ///
 /// Length is based on the following formula: sqrt(x[0]^2 + x[1]^2 + ...).
 
+template <typename T>
 _HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
-const inline half length(half X) { return __detail::length_impl(X); }
-const inline float length(float X) { return __detail::length_impl(X); }
+const inline __detail::enable_if_t<__detail::is_arithmetic<T>::Value &&
+                                       __detail::is_same<half, T>::value,
+                                   T> length(T X) {
+  return __detail::length_impl(X);
+}
+
+template <typename T>
+const inline __detail::enable_if_t<
+    __detail::is_arithmetic<T>::Value && __detail::is_same<float, T>::value, T>
+length(T X) {
+  return __detail::length_impl(X);
+}
 
 template <int N>
 _HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
-const inline half length(vector<half, N> X) {
+const inline half length(__detail::HLSL_FIXED_VECTOR<half, N> X) {
   return __detail::length_vec_impl(X);
 }
 
-template <int N> const inline float length(vector<float, N> X) {
+template <int N>
+const inline float length(__detail::HLSL_FIXED_VECTOR<float, N> X) {
   return __detail::length_vec_impl(X);
 }
 
@@ -173,23 +193,33 @@ constexpr vector<uint, 4> D3DCOLORtoUBYTE4(vector<float, 4> V) {
 ///
 /// Result type and the type of all operands must be the same type.
 
+template <typename T>
 _HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
-const inline half reflect(half I, half N) {
+const inline __detail::enable_if_t<__detail::is_arithmetic<T>::Value &&
+                                       __detail::is_same<half, T>::value,
+                                   T> reflect(T I, T N) {
   return __detail::reflect_impl(I, N);
 }
 
-const inline float reflect(float I, float N) {
+template <typename T>
+const inline __detail::enable_if_t<
+    __detail::is_arithmetic<T>::Value && __detail::is_same<float, T>::value, T>
+reflect(T I, T N) {
   return __detail::reflect_impl(I, N);
 }
 
 template <int L>
 _HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
-const inline vector<half, L> reflect(vector<half, L> I, vector<half, L> N) {
+const inline __detail::HLSL_FIXED_VECTOR<half, L> reflect(
+    __detail::HLSL_FIXED_VECTOR<half, L> I,
+    __detail::HLSL_FIXED_VECTOR<half, L> N) {
   return __detail::reflect_vec_impl(I, N);
 }
 
 template <int L>
-const inline vector<float, L> reflect(vector<float, L> I, vector<float, L> N) {
+const inline __detail::HLSL_FIXED_VECTOR<float, L>
+reflect(__detail::HLSL_FIXED_VECTOR<float, L> I,
+        __detail::HLSL_FIXED_VECTOR<float, L> N) {
   return __detail::reflect_vec_impl(I, N);
 }
 } // namespace hlsl
diff --git a/clang/lib/Headers/nvptxintrin.h b/clang/lib/Headers/nvptxintrin.h
index 7af22baccb511..73eb0af8b5926 100644
--- a/clang/lib/Headers/nvptxintrin.h
+++ b/clang/lib/Headers/nvptxintrin.h
@@ -180,8 +180,9 @@ __gpu_shuffle_idx_u64(uint64_t __lane_mask, uint32_t __idx, uint64_t __x,
 _DEFAULT_FN_ATTRS static __inline__ uint64_t
 __gpu_match_any_u32(uint64_t __lane_mask, uint32_t __x) {
   // Newer targets can use the dedicated CUDA support.
-  if (__CUDA_ARCH__ >= 700 || __nvvm_reflect("__CUDA_ARCH") >= 700)
-    return __nvvm_match_any_sync_i32(__lane_mask, __x);
+#if __CUDA_ARCH__ >= 700
+  return __nvvm_match_any_sync_i32(__lane_mask, __x);
+#endif
 
   uint32_t __match_mask = 0;
   bool __done = 0;
@@ -201,8 +202,9 @@ __gpu_match_any_u32(uint64_t __lane_mask, uint32_t __x) {
 _DEFAULT_FN_ATTRS static __inline__ uint64_t
 __gpu_match_any_u64(uint64_t __lane_mask, uint64_t __x) {
   // Newer targets can use the dedicated CUDA support.
-  if (__CUDA_ARCH__ >= 700 || __nvvm_reflect("__CUDA_ARCH") >= 700)
-    return __nvvm_match_any_sync_i64(__lane_mask, __x);
+#if __CUDA_ARCH__ >= 700
+  return __nvvm_match_any_sync_i64(__lane_mask, __x);
+#endif
 
   uint64_t __match_mask = 0;
 
@@ -224,9 +226,10 @@ __gpu_match_any_u64(uint64_t __lane_mask, uint64_t __x) {
 _DEFAULT_FN_ATTRS static __inline__ uint64_t
 __gpu_match_all_u32(uint64_t __lane_mask, uint32_t __x) {
   // Newer targets can use the dedicated CUDA support.
+#if __CUDA_ARCH__ >= 700
   int predicate;
-  if (__CUDA_ARCH__ >= 700 || __nvvm_reflect("__CUDA_ARCH") >= 700)
-    return __nvvm_match_all_sync_i32p(__lane_mask, __x, &predicate);
+  return __nvvm_match_all_sync_i32p(__lane_mask, __x, &predicate);
+#endif
 
   uint32_t __first = __gpu_read_first_lane_u64(__lane_mask, __x);
   uint64_t __ballot = __gpu_ballot(__lane_mask, __x == __first);
@@ -237,9 +240,10 @@ __gpu_match_all_u32(uint64_t __lane_mask, uint32_t __x) {
 _DEFAULT_FN_ATTRS static __inline__ uint64_t
 __gpu_match_all_u64(uint64_t __lane_mask, uint64_t __x) {
   // Newer targets can use the dedicated CUDA support.
+#if __CUDA_ARCH__ >= 700
   int predicate;
-  if (__CUDA_ARCH__ >= 700 || __nvvm_reflect("__CUDA_ARCH") >= 700)
-    return __nvvm_match_all_sync_i64p(__lane_mask, __x, &predicate);
+  return __nvvm_match_all_sync_i64p(__lane_mask, __x, &predicate);
+#endif
 
   uint64_t __first = __gpu_read_first_lane_u64(__lane_mask, __x);
   uint64_t __ballot = __gpu_ballot(__lane_mask, __x == __first);
diff --git a/clang/lib/Interpreter/Wasm.cpp b/clang/lib/Interpreter/Wasm.cpp
index aa10b160ccf84..6f584fab52ba9 100644
--- a/clang/lib/Interpreter/Wasm.cpp
+++ b/clang/lib/Interpreter/Wasm.cpp
@@ -73,8 +73,9 @@ llvm::Error WasmIncrementalExecutor::addModule(PartialTranslationUnit &PTU) {
   }
 
   llvm::TargetOptions TO = llvm::TargetOptions();
-  llvm::TargetMachine *TargetMachine = Target->createTargetMachine(
-      PTU.TheModule->getTargetTriple(), "", "", TO, llvm::Reloc::Model::PIC_);
+  llvm::TargetMachine *TargetMachine =
+      Target->createTargetMachine(PTU.TheModule->getTargetTriple().str(), "",
+                                  "", TO, llvm::Reloc::Model::PIC_);
   PTU.TheModule->setDataLayout(TargetMachine->createDataLayout());
   std::string ObjectFileName = PTU.TheModule->getName().str() + ".o";
   std::string BinaryFileName = PTU.TheModule->getName().str() + ".wasm";
@@ -146,4 +147,4 @@ llvm::Error WasmIncrementalExecutor::cleanUp() {
 
 WasmIncrementalExecutor::~WasmIncrementalExecutor() = default;
 
-} // namespace clang
\ No newline at end of file
+} // namespace clang
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 087c6f13aea66..c62a9f5041183 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -3192,23 +3192,22 @@ bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
   if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')) {
     DiagnosticsEngine &Diags = PP->getDiagnostics();
     SourceLocation EndLoc = getSourceLocation(BufferEnd);
-    unsigned DiagID;
+    unsigned DiagID = diag::warn_no_newline_eof;
 
     if (LangOpts.CPlusPlus11) {
       // C++11 [lex.phases] 2.2 p2
       // Prefer the C++98 pedantic compatibility warning over the generic,
       // non-extension, user-requested "missing newline at EOF" warning.
-      if (!Diags.isIgnored(diag::warn_cxx98_compat_no_newline_eof, EndLoc)) {
+      if (!Diags.isIgnored(diag::warn_cxx98_compat_no_newline_eof, EndLoc))
         DiagID = diag::warn_cxx98_compat_no_newline_eof;
-      } else {
-        DiagID = diag::warn_no_newline_eof;
-      }
     } else {
-      DiagID = diag::ext_no_newline_eof;
+      // This is conforming in C2y, but is an extension in earlier language
+      // modes.
+      if (!LangOpts.C2y)
+        DiagID = diag::ext_no_newline_eof;
     }
 
-    Diag(BufferEnd, DiagID)
-      << FixItHint::CreateInsertion(EndLoc, "\n");
+    Diag(BufferEnd, DiagID) << FixItHint::CreateInsertion(EndLoc, "\n");
   }
 
   BufferPtr = CurPtr;
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 7ae136af47391..82b394d5b4ca6 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -4411,6 +4411,10 @@ void Parser::ParseDeclarationSpecifiers(
         DiagID = diag::err_openclcxx_virtual_function;
         PrevSpec = Tok.getIdentifierInfo()->getNameStart();
         isInvalid = true;
+      } else if (getLangOpts().HLSL) {
+        DiagID = diag::err_hlsl_virtual_function;
+        PrevSpec = Tok.getIdentifierInfo()->getNameStart();
+        isInvalid = true;
       } else {
         isInvalid = DS.setFunctionSpecVirtual(Loc, PrevSpec, DiagID);
       }
@@ -5652,7 +5656,7 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
     Decl *D = SkipBody.CheckSameAsPrevious ? SkipBody.New : TagDecl;
     ParseEnumBody(StartLoc, D);
     if (SkipBody.CheckSameAsPrevious &&
-        !Actions.ActOnDuplicateDefinition(TagDecl, SkipBody)) {
+        !Actions.ActOnDuplicateDefinition(getCurScope(), TagDecl, SkipBody)) {
       DS.SetTypeSpecError();
       return;
     }
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index 43db715ac6d70..9384f9ab10af0 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -2350,7 +2350,8 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
       // Parse the definition body.
       ParseStructUnionBody(StartLoc, TagType, cast<RecordDecl>(D));
       if (SkipBody.CheckSameAsPrevious &&
-          !Actions.ActOnDuplicateDefinition(TagOrTempResult.get(), SkipBody)) {
+          !Actions.ActOnDuplicateDefinition(getCurScope(),
+                                            TagOrTempResult.get(), SkipBody)) {
         DS.SetTypeSpecError();
         return;
       }
@@ -2491,6 +2492,9 @@ BaseResult Parser::ParseBaseSpecifier(Decl *ClassDecl) {
     IsVirtual = true;
   }
 
+  if (getLangOpts().HLSL && IsVirtual)
+    Diag(Tok.getLocation(), diag::err_hlsl_virtual_inheritance);
+
   CheckMisplacedCXX11Attribute(Attributes, StartLoc);
 
   // Parse the class-name.
diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp
index 33a90e0cb8a42..26be78ee8ca15 100644
--- a/clang/lib/Parse/ParseExprCXX.cpp
+++ b/clang/lib/Parse/ParseExprCXX.cpp
@@ -2203,8 +2203,16 @@ Parser::ParseCXXCondition(StmtResult *InitStmt, SourceLocation Loc,
       return ParseCXXCondition(nullptr, Loc, CK, MissingOK);
     }
 
-    // Parse the expression.
-    ExprResult Expr = ParseExpression(); // expression
+    ExprResult Expr = [&] {
+      EnterExpressionEvaluationContext Eval(
+          Actions, Sema::ExpressionEvaluationContext::ConstantEvaluated,
+          /*LambdaContextDecl=*/nullptr,
+          /*ExprContext=*/Sema::ExpressionEvaluationContextRecord::EK_Other,
+          /*ShouldEnter=*/CK == Sema::ConditionKind::ConstexprIf);
+      // Parse the expression.
+      return ParseExpression(); // expression
+    }();
+
     if (Expr.isInvalid())
       return Sema::ConditionError();
 
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index 6ea17c97d6345..8d3501082cc27 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -1059,8 +1059,11 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
       break;
     }
     case OpenACCClauseKind::Bind: {
-      ExprResult BindArg = ParseOpenACCBindClauseArgument();
-      if (BindArg.isInvalid()) {
+      ParsedClause.setBindDetails(ParseOpenACCBindClauseArgument());
+
+      // We can create an 'empty' bind clause in the event of an error
+      if (std::holds_alternative<std::monostate>(
+              ParsedClause.getBindDetails())) {
         Parens.skipToEnd();
         return OpenACCCanContinue();
       }
@@ -1334,7 +1337,8 @@ ExprResult Parser::ParseOpenACCIDExpression() {
   return getActions().CorrectDelayedTyposInExpr(Res);
 }
 
-ExprResult Parser::ParseOpenACCBindClauseArgument() {
+std::variant<std::monostate, clang::StringLiteral *, IdentifierInfo *>
+Parser::ParseOpenACCBindClauseArgument() {
   // OpenACC 3.3 section 2.15:
   // The bind clause specifies the name to use when calling the procedure on a
   // device other than the host. If the name is specified as an identifier, it
@@ -1343,14 +1347,21 @@ ExprResult Parser::ParseOpenACCBindClauseArgument() {
   // name unmodified.
   if (getCurToken().is(tok::r_paren)) {
     Diag(getCurToken(), diag::err_acc_incorrect_bind_arg);
-    return ExprError();
+    return std::monostate{};
   }
 
-  if (tok::isStringLiteral(getCurToken().getKind()))
-    return getActions().CorrectDelayedTyposInExpr(ParseStringLiteralExpression(
-        /*AllowUserDefinedLiteral=*/false, /*Unevaluated=*/true));
+  if (getCurToken().is(tok::identifier)) {
+    IdentifierInfo *II = getCurToken().getIdentifierInfo();
+    ConsumeToken();
+    return II;
+  }
 
-  return ParseOpenACCIDExpression();
+  ExprResult Res =
+      getActions().CorrectDelayedTyposInExpr(ParseStringLiteralExpression(
+          /*AllowUserDefinedLiteral=*/false, /*Unevaluated=*/true));
+  if (!Res.isUsable())
+    return std::monostate{};
+  return cast<StringLiteral>(Res.get());
 }
 
 /// OpenACC 3.3, section 1.6:
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index b791c5d5e3019..28386d2260a83 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3353,6 +3353,20 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
       ErrorFound = true;
     }
 
+    Clause = ParseOpenMPClause(CKind, WrongDirective);
+    break;
+  case OMPC_self_maps:
+    // OpenMP [6.0, self_maps clause]
+    if (getLangOpts().OpenMP < 60) {
+      Diag(Tok, diag::err_omp_expected_clause)
+          << getOpenMPDirectiveName(OMPD_requires);
+      ErrorFound = true;
+    }
+    if (!FirstClause) {
+      Diag(Tok, diag::err_omp_more_one_clause)
+          << getOpenMPDirectiveName(DKind) << getOpenMPClauseName(CKind) << 0;
+      ErrorFound = true;
+    }
     Clause = ParseOpenMPClause(CKind, WrongDirective);
     break;
   case OMPC_update:
@@ -4333,6 +4347,20 @@ bool Parser::parseMapTypeModifiers(SemaOpenMP::OpenMPVarListDataTy &Data) {
             << PreMapName;
       }
       ConsumeToken();
+    } else if (TypeModifier == OMPC_MAP_MODIFIER_self) {
+      Data.MapTypeModifiers.push_back(TypeModifier);
+      Data.MapTypeModifiersLoc.push_back(Tok.getLocation());
+      if (PP.LookAhead(0).isNot(tok::comma) &&
+          PP.LookAhead(0).isNot(tok::colon))
+        Diag(Tok.getLocation(), diag::err_omp_missing_comma)
+            << "map type modifier";
+      if (getLangOpts().OpenMP < 60)
+        Diag(Tok, diag::err_omp_unknown_map_type_modifier)
+            << (getLangOpts().OpenMP >= 51
+                    ? (getLangOpts().OpenMP >= 52 ? 2 : 1)
+                    : 0)
+            << getLangOpts().OpenMPExtensions << 0;
+      ConsumeToken();
     } else {
       // For the case of unknown map-type-modifier or a map-type.
       // Map-type is followed by a colon; the function returns when it
@@ -4354,7 +4382,8 @@ bool Parser::parseMapTypeModifiers(SemaOpenMP::OpenMPVarListDataTy &Data) {
       Diag(Tok, diag::err_omp_unknown_map_type_modifier)
           << (getLangOpts().OpenMP >= 51 ? (getLangOpts().OpenMP >= 52 ? 2 : 1)
                                          : 0)
-          << getLangOpts().OpenMPExtensions;
+          << getLangOpts().OpenMPExtensions
+          << (getLangOpts().OpenMP >= 60 ? 1 : 0);
       ConsumeToken();
     }
     if (getCurToken().is(tok::comma))
@@ -4829,7 +4858,7 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
       if (getLangOpts().OpenMP < 52) {
         Diag(Tok, diag::err_omp_unknown_map_type_modifier)
             << (getLangOpts().OpenMP >= 51 ? 1 : 0)
-            << getLangOpts().OpenMPExtensions;
+            << getLangOpts().OpenMPExtensions << 0;
         InvalidIterator = true;
       }
     }
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index b3fba097999f5..93a2d797679d4 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -1105,9 +1105,13 @@ void Sema::ActOnStartOfTranslationUnit() {
 }
 
 void Sema::ActOnEndOfTranslationUnitFragment(TUFragmentKind Kind) {
-  // No explicit actions are required at the end of the global module fragment.
-  if (Kind == TUFragmentKind::Global)
+  if (Kind == TUFragmentKind::Global) {
+    // Perform Pending Instantiations at the end of global module fragment so
+    // that the module ownership of TU-level decls won't get messed.
+    llvm::TimeTraceScope TimeScope("PerformPendingInstantiations");
+    PerformPendingInstantiations();
     return;
+  }
 
   // Transfer late parsed template instantiations over to the pending template
   // instantiation list. During normal compilation, the late template parser
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index b4b40f293b28e..3f53fb200a93d 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -249,16 +249,16 @@ bool SemaARM::BuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall,
       }
     }
 
-    SmallVector<int, 5> Ranges;
+    SmallVector<int, 5> FieldBitWidths;
     if (FiveFields)
-      Ranges.append({IsAArch64Builtin ? 1 : 15, 7, 15, 15, 7});
+      FieldBitWidths.append({IsAArch64Builtin ? 2 : 4, 3, 4, 4, 3});
     else
-      Ranges.append({15, 7, 15});
+      FieldBitWidths.append({4, 3, 4});
 
     for (unsigned i = 0; i < Fields.size(); ++i) {
       int IntField;
       ValidString &= !Fields[i].getAsInteger(10, IntField);
-      ValidString &= (IntField >= 0 && IntField <= Ranges[i]);
+      ValidString &= (IntField >= 0 && IntField < (1 << FieldBitWidths[i]));
     }
 
     if (!ValidString)
diff --git a/clang/lib/Sema/SemaAvailability.cpp b/clang/lib/Sema/SemaAvailability.cpp
index c806b832dec7a..96aa65412906c 100644
--- a/clang/lib/Sema/SemaAvailability.cpp
+++ b/clang/lib/Sema/SemaAvailability.cpp
@@ -99,16 +99,29 @@ ShouldDiagnoseAvailabilityOfDecl(Sema &S, const NamedDecl *D,
   // For typedefs, if the typedef declaration appears available look
   // to the underlying type to see if it is more restrictive.
   while (const auto *TD = dyn_cast<TypedefNameDecl>(D)) {
-    if (Result == AR_Available) {
-      if (const auto *TT = TD->getUnderlyingType()->getAs<TagType>()) {
+    if (Result != AR_Available)
+      break;
+    for (const Type *T = TD->getUnderlyingType().getTypePtr(); /**/; /**/) {
+      if (auto *TT = dyn_cast<TagType>(T)) {
         D = TT->getDecl();
-        Result = D->getAvailability(Message);
+      } else if (isa<SubstTemplateTypeParmType>(T)) {
+        // A Subst* node represents a use through a template.
+        // Any uses of the underlying declaration happened through it's template
+        // specialization.
+        goto done;
+      } else {
+        const Type *NextT =
+            T->getLocallyUnqualifiedSingleStepDesugaredType().getTypePtr();
+        if (NextT == T)
+          goto done;
+        T = NextT;
         continue;
       }
+      Result = D->getAvailability(Message);
+      break;
     }
-    break;
   }
-
+done:
   // For alias templates, get the underlying declaration.
   if (const auto *ADecl = dyn_cast<TypeAliasTemplateDecl>(D)) {
     D = ADecl->getTemplatedDecl();
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 9cac9cf5c4df7..9bcb014ab9bfc 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -2723,6 +2723,7 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
   case Builtin::BI__builtin_elementwise_cosh:
   case Builtin::BI__builtin_elementwise_exp:
   case Builtin::BI__builtin_elementwise_exp2:
+  case Builtin::BI__builtin_elementwise_exp10:
   case Builtin::BI__builtin_elementwise_floor:
   case Builtin::BI__builtin_elementwise_log:
   case Builtin::BI__builtin_elementwise_log2:
@@ -10620,6 +10621,42 @@ static std::optional<IntRange> TryGetExprRange(ASTContext &C, const Expr *E,
     case UO_AddrOf: // should be impossible
       return IntRange::forValueOfType(C, GetExprType(E));
 
+    case UO_Minus: {
+      if (E->getType()->isUnsignedIntegerType()) {
+        return TryGetExprRange(C, UO->getSubExpr(), MaxWidth, InConstantContext,
+                               Approximate);
+      }
+
+      std::optional<IntRange> SubRange = TryGetExprRange(
+          C, UO->getSubExpr(), MaxWidth, InConstantContext, Approximate);
+
+      if (!SubRange)
+        return std::nullopt;
+
+      // If the range was previously non-negative, we need an extra bit for the
+      // sign bit. If the range was not non-negative, we need an extra bit
+      // because the negation of the most-negative value is one bit wider than
+      // that value.
+      return IntRange(SubRange->Width + 1, false);
+    }
+
+    case UO_Not: {
+      if (E->getType()->isUnsignedIntegerType()) {
+        return TryGetExprRange(C, UO->getSubExpr(), MaxWidth, InConstantContext,
+                               Approximate);
+      }
+
+      std::optional<IntRange> SubRange = TryGetExprRange(
+          C, UO->getSubExpr(), MaxWidth, InConstantContext, Approximate);
+
+      if (!SubRange)
+        return std::nullopt;
+
+      // The width increments by 1 if the sub-expression cannot be negative
+      // since it now can be.
+      return IntRange(SubRange->Width + (int)SubRange->NonNegative, false);
+    }
+
     default:
       return TryGetExprRange(C, UO->getSubExpr(), MaxWidth, InConstantContext,
                              Approximate);
diff --git a/clang/lib/Sema/SemaCoroutine.cpp b/clang/lib/Sema/SemaCoroutine.cpp
index 0e4f3b20c78cd..53536b0d14037 100644
--- a/clang/lib/Sema/SemaCoroutine.cpp
+++ b/clang/lib/Sema/SemaCoroutine.cpp
@@ -788,7 +788,11 @@ static bool checkSuspensionContext(Sema &S, SourceLocation Loc,
   // First emphasis of [expr.await]p2: must be a potentially evaluated context.
   // That is, 'co_await' and 'co_yield' cannot appear in subexpressions of
   // \c sizeof.
-  if (S.isUnevaluatedContext()) {
+  const auto ExprContext = S.currentEvaluationContext().ExprContext;
+  const bool BadContext =
+      S.isUnevaluatedContext() ||
+      ExprContext != Sema::ExpressionEvaluationContextRecord::EK_Other;
+  if (BadContext) {
     S.Diag(Loc, diag::err_coroutine_unevaluated_context) << Keyword;
     return false;
   }
@@ -798,7 +802,6 @@ static bool checkSuspensionContext(Sema &S, SourceLocation Loc,
     S.Diag(Loc, diag::err_coroutine_within_handler) << Keyword;
     return false;
   }
-
   return true;
 }
 
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 8c5125b8eb6d9..9c67fbd40ac71 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -61,6 +61,7 @@
 #include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/SaveAndRestore.h"
 #include "llvm/TargetParser/Triple.h"
 #include <algorithm>
 #include <cstring>
@@ -350,7 +351,7 @@ ParsedType Sema::getTypeName(const IdentifierInfo &II, SourceLocation NameLoc,
             Diag(QualifiedLoc, diag::warn_cxx17_compat_implicit_typename);
           else
             Diag(QualifiedLoc, diag::ext_implicit_typename)
-                << SS->getScopeRep() << II.getName()
+                << NestedNameSpecifier::Create(Context, SS->getScopeRep(), &II)
                 << FixItHint::CreateInsertion(QualifiedLoc, "typename ");
         }
 
@@ -794,9 +795,9 @@ void Sema::DiagnoseUnknownTypeName(IdentifierInfo *&II,
       DiagID = diag::ext_typename_missing;
 
     Diag(SS->getRange().getBegin(), DiagID)
-      << SS->getScopeRep() << II->getName()
-      << SourceRange(SS->getRange().getBegin(), IILoc)
-      << FixItHint::CreateInsertion(SS->getRange().getBegin(), "typename ");
+        << NestedNameSpecifier::Create(Context, SS->getScopeRep(), II)
+        << SourceRange(SS->getRange().getBegin(), IILoc)
+        << FixItHint::CreateInsertion(SS->getRange().getBegin(), "typename ");
     SuggestedType = ActOnTypenameType(S, SourceLocation(),
                                       *SS, *II, IILoc).get();
   } else {
@@ -2563,18 +2564,7 @@ void Sema::MergeTypedefNameDecl(Scope *S, TypedefNameDecl *New,
       // Make the old tag definition visible.
       makeMergedDefinitionVisible(Hidden);
 
-      // If this was an unscoped enumeration, yank all of its enumerators
-      // out of the scope.
-      if (isa<EnumDecl>(NewTag)) {
-        Scope *EnumScope = getNonFieldDeclScope(S);
-        for (auto *D : NewTag->decls()) {
-          auto *ED = cast<EnumConstantDecl>(D);
-          assert(EnumScope->isDeclScope(ED));
-          EnumScope->RemoveDecl(ED);
-          IdResolver.RemoveDecl(ED);
-          ED->getLexicalDeclContext()->removeDecl(ED);
-        }
-      }
+      CleanupMergedEnum(S, NewTag);
     }
   }
 
@@ -2651,6 +2641,19 @@ void Sema::MergeTypedefNameDecl(Scope *S, TypedefNameDecl *New,
   notePreviousDefinition(Old, New->getLocation());
 }
 
+void Sema::CleanupMergedEnum(Scope *S, Decl *New) {
+  // If this was an unscoped enumeration, yank all of its enumerators
+  // out of the scope.
+  if (auto *ED = dyn_cast<EnumDecl>(New); ED && !ED->isScoped()) {
+    Scope *EnumScope = getNonFieldDeclScope(S);
+    for (auto *ECD : ED->enumerators()) {
+      assert(EnumScope->isDeclScope(ECD));
+      EnumScope->RemoveDecl(ECD);
+      IdResolver.RemoveDecl(ECD);
+    }
+  }
+}
+
 /// DeclhasAttr - returns true if decl Declaration already has the target
 /// attribute.
 static bool DeclHasAttr(const Decl *D, const Attr *A) {
@@ -8162,7 +8165,7 @@ NamedDecl *Sema::ActOnVariableDeclarator(
               (D.getCXXScopeSpec().isSet() && DC && DC->isRecord() &&
                DC->isDependentContext())
                   ? TPC_ClassTemplateMember
-                  : TPC_VarTemplate))
+                  : TPC_Other))
         NewVD->setInvalidDecl();
 
       // If we are providing an explicit specialization of a static variable
@@ -12611,6 +12614,7 @@ namespace {
     bool isRecordType;
     bool isPODType;
     bool isReferenceType;
+    bool isInCXXOperatorCall;
 
     bool isInitList;
     llvm::SmallVector<unsigned, 4> InitFieldIndex;
@@ -12623,6 +12627,7 @@ namespace {
       isPODType = false;
       isRecordType = false;
       isReferenceType = false;
+      isInCXXOperatorCall = false;
       isInitList = false;
       if (ValueDecl *VD = dyn_cast<ValueDecl>(OrigDecl)) {
         isPODType = VD->getType().isPODType(S.Context);
@@ -12810,6 +12815,7 @@ namespace {
     }
 
     void VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
+      llvm::SaveAndRestore CxxOpCallScope(isInCXXOperatorCall, true);
       Expr *Callee = E->getCallee();
 
       if (isa<UnresolvedLookupExpr>(Callee))
@@ -12820,6 +12826,19 @@ namespace {
         HandleValue(Arg->IgnoreParenImpCasts());
     }
 
+    void VisitLambdaExpr(LambdaExpr *E) {
+      if (!isInCXXOperatorCall) {
+        Inherited::VisitLambdaExpr(E);
+        return;
+      }
+
+      for (Expr *Init : E->capture_inits())
+        if (DeclRefExpr *DRE = dyn_cast_if_present<DeclRefExpr>(Init))
+          HandleDeclRefExpr(DRE);
+        else if (Init)
+          Visit(Init);
+    }
+
     void VisitUnaryOperator(UnaryOperator *E) {
       // For POD record types, addresses of its own members are well-defined.
       if (E->getOpcode() == UO_AddrOf && isRecordType &&
@@ -18330,12 +18349,14 @@ void Sema::ActOnTagStartDefinition(Scope *S, Decl *TagD) {
   AddPushedVisibilityAttribute(Tag);
 }
 
-bool Sema::ActOnDuplicateDefinition(Decl *Prev, SkipBodyInfo &SkipBody) {
+bool Sema::ActOnDuplicateDefinition(Scope *S, Decl *Prev,
+                                    SkipBodyInfo &SkipBody) {
   if (!hasStructuralCompatLayout(Prev, SkipBody.New))
     return false;
 
   // Make the previous decl visible.
   makeMergedDefinitionVisible(SkipBody.Previous);
+  CleanupMergedEnum(S, SkipBody.New);
   return true;
 }
 
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 1405ee5341dcf..de2696ea62e3b 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -1191,7 +1191,8 @@ static void handleTestTypestateAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 
 static void handleExtVectorTypeAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   // Remember this typedef decl, we will need it later for diagnostics.
-  S.ExtVectorDecls.push_back(cast<TypedefNameDecl>(D));
+  if (isa<TypedefNameDecl>(D))
+    S.ExtVectorDecls.push_back(cast<TypedefNameDecl>(D));
 }
 
 static void handlePackedAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
@@ -2138,6 +2139,8 @@ static void handleConstructorAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   if (AL.getNumArgs() &&
       !S.checkUInt32Argument(AL, AL.getArgAsExpr(0), priority))
     return;
+  S.Diag(D->getLocation(), diag::warn_global_constructor)
+      << D->getSourceRange();
 
   D->addAttr(::new (S.Context) ConstructorAttr(S.Context, AL, priority));
 }
@@ -2147,6 +2150,7 @@ static void handleDestructorAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   if (AL.getNumArgs() &&
       !S.checkUInt32Argument(AL, AL.getArgAsExpr(0), priority))
     return;
+  S.Diag(D->getLocation(), diag::warn_global_destructor) << D->getSourceRange();
 
   D->addAttr(::new (S.Context) DestructorAttr(S.Context, AL, priority));
 }
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index a3a028b9485d6..96aac7871db1e 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -733,20 +733,26 @@ Sema::ActOnDecompositionDeclarator(Scope *S, Declarator &D,
   }
 
   if (!TemplateParamLists.empty()) {
-    // FIXME: There's no rule against this, but there are also no rules that
-    // would actually make it usable, so we reject it for now.
+    // C++17 [temp]/1:
+    //   A template defines a family of class, functions, or variables, or an
+    //   alias for a family of types.
+    //
+    // Structured bindings are not included.
     Diag(TemplateParamLists.front()->getTemplateLoc(),
          diag::err_decomp_decl_template);
     return nullptr;
   }
 
-  Diag(Decomp.getLSquareLoc(),
-       !getLangOpts().CPlusPlus17
-           ? diag::ext_decomp_decl
-           : D.getContext() == DeclaratorContext::Condition
-                 ? diag::ext_decomp_decl_cond
-                 : diag::warn_cxx14_compat_decomp_decl)
-      << Decomp.getSourceRange();
+  unsigned DiagID;
+  if (!getLangOpts().CPlusPlus17)
+    DiagID = diag::ext_decomp_decl;
+  else if (D.getContext() == DeclaratorContext::Condition)
+    DiagID = getLangOpts().CPlusPlus26 ? diag::warn_cxx26_decomp_decl_cond
+                                       : diag::ext_decomp_decl_cond;
+  else
+    DiagID = diag::warn_cxx14_compat_decomp_decl;
+
+  Diag(Decomp.getLSquareLoc(), DiagID) << Decomp.getSourceRange();
 
   // The semantic context is always just the current context.
   DeclContext *const DC = CurContext;
@@ -13583,7 +13589,7 @@ Decl *Sema::ActOnAliasDeclaration(Scope *S, AccessSpecifier AS,
     // Merge any previous default template arguments into our parameters,
     // and check the parameter list.
     if (CheckTemplateParameterList(TemplateParams, OldTemplateParams,
-                                   TPC_TypeAliasTemplate))
+                                   TPC_Other))
       return nullptr;
 
     TypeAliasTemplateDecl *NewDecl =
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index f896ccab53a54..a36a2f563739e 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -1748,9 +1748,14 @@ ExprResult Sema::CreateGenericSelectionExpr(
         //
         // C11 6.5.1.1p2 "The type name in a generic association shall specify a
         // complete object type other than a variably modified type."
+        // C2y removed the requirement that an expression form must
+        // use a complete type, though it's still as-if the type has undergone
+        // lvalue conversion. We support this as an extension in C23 and
+        // earlier because GCC does so.
         unsigned D = 0;
         if (ControllingExpr && Types[i]->getType()->isIncompleteType())
-          D = diag::err_assoc_type_incomplete;
+          D = LangOpts.C2y ? diag::warn_c2y_compat_assoc_type_incomplete
+                           : diag::ext_assoc_type_incomplete;
         else if (ControllingExpr && !Types[i]->getType()->isObjectType())
           D = diag::err_assoc_type_nonobject;
         else if (Types[i]->getType()->isVariablyModifiedType())
@@ -2935,6 +2940,9 @@ ExprResult Sema::BuildQualifiedDeclarationNameExpr(
   }
 
   if (const TypeDecl *TD = R.getAsSingle<TypeDecl>()) {
+    QualType Ty = Context.getTypeDeclType(TD);
+    QualType ET = getElaboratedType(ElaboratedTypeKeyword::None, SS, Ty);
+
     // Diagnose a missing typename if this resolved unambiguously to a type in
     // a dependent context.  If we can recover with a type, downgrade this to
     // a warning in Microsoft compatibility mode.
@@ -2943,8 +2951,7 @@ ExprResult Sema::BuildQualifiedDeclarationNameExpr(
       DiagID = diag::ext_typename_missing;
     SourceLocation Loc = SS.getBeginLoc();
     auto D = Diag(Loc, DiagID);
-    D << SS.getScopeRep() << NameInfo.getName().getAsString()
-      << SourceRange(Loc, NameInfo.getEndLoc());
+    D << ET << SourceRange(Loc, NameInfo.getEndLoc());
 
     // Don't recover if the caller isn't expecting us to or if we're in a SFINAE
     // context.
@@ -2955,11 +2962,9 @@ ExprResult Sema::BuildQualifiedDeclarationNameExpr(
     D << FixItHint::CreateInsertion(Loc, "typename ");
 
     // Recover by pretending this was an elaborated type.
-    QualType Ty = Context.getTypeDeclType(TD);
     TypeLocBuilder TLB;
     TLB.pushTypeSpec(Ty).setNameLoc(NameInfo.getLoc());
 
-    QualType ET = getElaboratedType(ElaboratedTypeKeyword::None, SS, Ty);
     ElaboratedTypeLoc QTL = TLB.push<ElaboratedTypeLoc>(ET);
     QTL.setElaboratedKeywordLoc(SourceLocation());
     QTL.setQualifierLoc(SS.getWithLocInContext(Context));
@@ -15428,7 +15433,7 @@ ExprResult Sema::BuildBinOp(Scope *S, SourceLocation OpLoc,
         Diag(OE->getQualifier() ? OE->getQualifierLoc().getBeginLoc()
                                 : OE->getNameLoc(),
              diag::err_template_kw_missing)
-          << OE->getName().getAsString() << "";
+            << OE->getName().getAsIdentifierInfo();
         return ExprError();
       }
     }
@@ -21020,18 +21025,24 @@ ExprResult Sema::CheckPlaceholderExpr(Expr *E) {
     NamedDecl *Temp = *ULE->decls_begin();
     const bool IsTypeAliasTemplateDecl = isa<TypeAliasTemplateDecl>(Temp);
 
-    if (NestedNameSpecifierLoc Loc = ULE->getQualifierLoc(); Loc.hasQualifier())
-      Diag(NameInfo.getLoc(), diag::err_template_kw_refers_to_type_template)
-          << Loc.getNestedNameSpecifier() << NameInfo.getName().getAsString()
-          << Loc.getSourceRange() << IsTypeAliasTemplateDecl;
-    else
-      Diag(NameInfo.getLoc(), diag::err_template_kw_refers_to_type_template)
-          << "" << NameInfo.getName().getAsString() << ULE->getSourceRange()
-          << IsTypeAliasTemplateDecl;
+    NestedNameSpecifier *NNS = ULE->getQualifierLoc().getNestedNameSpecifier();
+    TemplateName TN(dyn_cast<TemplateDecl>(Temp));
+    if (TN.isNull())
+      TN = Context.getAssumedTemplateName(NameInfo.getName());
+    TN = Context.getQualifiedTemplateName(NNS,
+                                          /*TemplateKeyword=*/true, TN);
+
+    Diag(NameInfo.getLoc(), diag::err_template_kw_refers_to_type_template)
+        << TN << ULE->getSourceRange() << IsTypeAliasTemplateDecl;
     Diag(Temp->getLocation(), diag::note_referenced_type_template)
         << IsTypeAliasTemplateDecl;
 
-    return CreateRecoveryExpr(NameInfo.getBeginLoc(), NameInfo.getEndLoc(), {});
+    QualType TST =
+        Context.getTemplateSpecializationType(TN, ULE->template_arguments());
+    QualType ET =
+        Context.getElaboratedType(ElaboratedTypeKeyword::None, NNS, TST);
+    return CreateRecoveryExpr(NameInfo.getBeginLoc(), NameInfo.getEndLoc(), {},
+                              ET);
   }
 
   // Overloaded expressions.
diff --git a/clang/lib/Sema/SemaExprMember.cpp b/clang/lib/Sema/SemaExprMember.cpp
index d130e8b86bc56..1d9efbeb5ccb5 100644
--- a/clang/lib/Sema/SemaExprMember.cpp
+++ b/clang/lib/Sema/SemaExprMember.cpp
@@ -1697,7 +1697,7 @@ static ExprResult LookupMemberExpr(Sema &S, LookupResult &R,
         QualType(), false);
   }
 
-  if (BaseType->isExtVectorBoolType()) {
+  if (BaseType->isPackedVectorBoolType(S.Context)) {
     // We disallow element access for ext_vector_type bool.  There is no way to
     // materialize a reference to a vector element as a pointer (each element is
     // one bit in the vector).
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index aff349a932eec..d10acd1b7807c 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -752,7 +752,7 @@ void SemaHLSL::DiagnoseAttrStageMismatch(
                         HLSLShaderAttr::ConvertEnvironmentTypeToStr(ST));
                   });
   Diag(A->getLoc(), diag::err_hlsl_attr_unsupported_in_stage)
-      << A << llvm::Triple::getEnvironmentTypeName(Stage)
+      << A->getAttrName() << llvm::Triple::getEnvironmentTypeName(Stage)
       << (AllowedStages.size() != 1) << join(StageStrings, ", ");
 }
 
@@ -2225,40 +2225,48 @@ static bool CheckBoolSelect(Sema *S, CallExpr *TheCall) {
 static bool CheckVectorSelect(Sema *S, CallExpr *TheCall) {
   assert(TheCall->getNumArgs() == 3);
   Expr *Arg1 = TheCall->getArg(1);
+  QualType Arg1Ty = Arg1->getType();
   Expr *Arg2 = TheCall->getArg(2);
-  if (!Arg1->getType()->isVectorType()) {
-    S->Diag(Arg1->getBeginLoc(), diag::err_builtin_non_vector_type)
-        << "Second" << TheCall->getDirectCallee() << Arg1->getType()
+  QualType Arg2Ty = Arg2->getType();
+
+  QualType Arg1ScalarTy = Arg1Ty;
+  if (auto VTy = Arg1ScalarTy->getAs<VectorType>())
+    Arg1ScalarTy = VTy->getElementType();
+
+  QualType Arg2ScalarTy = Arg2Ty;
+  if (auto VTy = Arg2ScalarTy->getAs<VectorType>())
+    Arg2ScalarTy = VTy->getElementType();
+
+  if (!S->Context.hasSameUnqualifiedType(Arg1ScalarTy, Arg2ScalarTy))
+    S->Diag(Arg1->getBeginLoc(), diag::err_hlsl_builtin_scalar_vector_mismatch)
+        << /* second and third */ 1 << TheCall->getCallee() << Arg1Ty << Arg2Ty;
+
+  QualType Arg0Ty = TheCall->getArg(0)->getType();
+  unsigned Arg0Length = Arg0Ty->getAs<VectorType>()->getNumElements();
+  unsigned Arg1Length = Arg1Ty->isVectorType()
+                            ? Arg1Ty->getAs<VectorType>()->getNumElements()
+                            : 0;
+  unsigned Arg2Length = Arg2Ty->isVectorType()
+                            ? Arg2Ty->getAs<VectorType>()->getNumElements()
+                            : 0;
+  if (Arg1Length > 0 && Arg0Length != Arg1Length) {
+    S->Diag(TheCall->getBeginLoc(),
+            diag::err_typecheck_vector_lengths_not_equal)
+        << Arg0Ty << Arg1Ty << TheCall->getArg(0)->getSourceRange()
         << Arg1->getSourceRange();
     return true;
   }
 
-  if (!Arg2->getType()->isVectorType()) {
-    S->Diag(Arg2->getBeginLoc(), diag::err_builtin_non_vector_type)
-        << "Third" << TheCall->getDirectCallee() << Arg2->getType()
-        << Arg2->getSourceRange();
-    return true;
-  }
-
-  if (!S->Context.hasSameUnqualifiedType(Arg1->getType(), Arg2->getType())) {
+  if (Arg2Length > 0 && Arg0Length != Arg2Length) {
     S->Diag(TheCall->getBeginLoc(),
-            diag::err_typecheck_call_different_arg_types)
-        << Arg1->getType() << Arg2->getType() << Arg1->getSourceRange()
+            diag::err_typecheck_vector_lengths_not_equal)
+        << Arg0Ty << Arg2Ty << TheCall->getArg(0)->getSourceRange()
         << Arg2->getSourceRange();
     return true;
   }
 
-  // caller has checked that Arg0 is a vector.
-  // check all three args have the same length.
-  if (TheCall->getArg(0)->getType()->getAs<VectorType>()->getNumElements() !=
-      Arg1->getType()->getAs<VectorType>()->getNumElements()) {
-    S->Diag(TheCall->getBeginLoc(),
-            diag::err_typecheck_vector_lengths_not_equal)
-        << TheCall->getArg(0)->getType() << Arg1->getType()
-        << TheCall->getArg(0)->getSourceRange() << Arg1->getSourceRange();
-    return true;
-  }
-  TheCall->setType(Arg1->getType());
+  TheCall->setType(
+      S->getASTContext().getExtVectorType(Arg1ScalarTy, Arg0Length));
   return false;
 }
 
@@ -2641,6 +2649,7 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
   case Builtin::BI__builtin_elementwise_cosh:
   case Builtin::BI__builtin_elementwise_exp:
   case Builtin::BI__builtin_elementwise_exp2:
+  case Builtin::BI__builtin_elementwise_exp10:
   case Builtin::BI__builtin_elementwise_floor:
   case Builtin::BI__builtin_elementwise_fmod:
   case Builtin::BI__builtin_elementwise_log:
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index edbd4c071b563..56ec33fe37bf3 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -7271,7 +7271,7 @@ static void CheckCXX98CompatAccessibleCopy(Sema &S,
 
 void InitializationSequence::PrintInitLocationNote(Sema &S,
                                               const InitializedEntity &Entity) {
-  if (Entity.isParameterKind() && Entity.getDecl()) {
+  if (Entity.isParamOrTemplateParamKind() && Entity.getDecl()) {
     if (Entity.getDecl()->getLocation().isInvalid())
       return;
 
@@ -7280,8 +7280,9 @@ void InitializationSequence::PrintInitLocationNote(Sema &S,
         << Entity.getDecl()->getDeclName();
     else
       S.Diag(Entity.getDecl()->getLocation(), diag::note_parameter_here);
-  } else if (Entity.getKind() == InitializedEntity::EK_RelatedResult &&
-             Entity.getMethodDecl())
+  }
+  else if (Entity.getKind() == InitializedEntity::EK_RelatedResult &&
+           Entity.getMethodDecl())
     S.Diag(Entity.getMethodDecl()->getLocation(),
            diag::note_method_return_type_change)
       << Entity.getMethodDecl()->getDeclName();
diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp
index 4d278bbc67d28..ceb32ee15dfa3 100644
--- a/clang/lib/Sema/SemaLambda.cpp
+++ b/clang/lib/Sema/SemaLambda.cpp
@@ -1506,13 +1506,14 @@ void Sema::ActOnStartOfLambdaDefinition(LambdaIntroducer &Intro,
   TemplateParameterList *TemplateParams =
       getGenericLambdaTemplateParameterList(LSI, *this);
   if (TemplateParams) {
-    for (auto *TP : TemplateParams->asArray()) {
+    for (const auto *TP : TemplateParams->asArray()) {
       if (!TP->getIdentifier())
         continue;
-      CheckTemplateParameterRAII CTP(*this, TP);
       for (const auto &Capture : Intro.Captures) {
-        if (Capture.Id == TP->getIdentifier())
+        if (Capture.Id == TP->getIdentifier()) {
           Diag(Capture.Loc, diag::err_template_param_shadow) << Capture.Id;
+          NoteTemplateParameterLocation(*TP);
+        }
       }
     }
   }
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index f3af514596a7e..aecf8ed1b4e4d 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -1580,13 +1580,9 @@ llvm::DenseSet<Module*> &Sema::getLookupModules() {
   unsigned N = CodeSynthesisContexts.size();
   for (unsigned I = CodeSynthesisContextLookupModules.size();
        I != N; ++I) {
-    auto &Ctx = CodeSynthesisContexts[I];
-    // FIXME: Are there any other context kinds that shouldn't be looked at
-    // here?
-    if (Ctx.Kind == CodeSynthesisContext::PartialOrderingTTP ||
-        Ctx.Kind == CodeSynthesisContext::CheckTemplateParameter)
-      continue;
-    Module *M = Ctx.Entity ? getDefiningModule(*this, Ctx.Entity) : nullptr;
+    Module *M = CodeSynthesisContexts[I].Entity ?
+                getDefiningModule(*this, CodeSynthesisContexts[I].Entity) :
+                nullptr;
     if (M && !LookupModulesCache.insert(M).second)
       M = nullptr;
     CodeSynthesisContextLookupModules.push_back(M);
@@ -3707,8 +3703,7 @@ Sema::LookupLiteralOperator(Scope *S, LookupResult &R,
       TemplateParameterList *Params = FD->getTemplateParameters();
       if (Params->size() == 1) {
         IsTemplate = true;
-        NamedDecl *Param = Params->getParam(0);
-        if (!Param->isTemplateParameterPack() && !StringLit) {
+        if (!Params->getParam(0)->isTemplateParameterPack() && !StringLit) {
           // Implied but not stated: user-defined integer and floating literals
           // only ever use numeric literal operator templates, not templates
           // taking a parameter of class type.
@@ -3721,7 +3716,6 @@ Sema::LookupLiteralOperator(Scope *S, LookupResult &R,
         if (StringLit) {
           SFINAETrap Trap(*this);
           CheckTemplateArgumentInfo CTAI;
-          CheckTemplateParameterRAII CTP(*this, Param);
           TemplateArgumentLoc Arg(TemplateArgument(StringLit), StringLit);
           if (CheckTemplateArgument(
                   Params->getParam(0), Arg, FD, R.getNameLoc(), R.getNameLoc(),
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index 4fe6bf5099a64..ec9e9527dca6f 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -1904,7 +1904,31 @@ DeclGroupRef SemaOpenACC::ActOnEndDeclDirective(
         Diag(DirLoc, diag::note_acc_construct_here)
             << OpenACCDirectiveKind::Routine;
       }
-      FD->addAttr(OpenACCRoutineAnnotAttr::Create(getASTContext(), DirLoc));
+
+      // OpenACC 3.3 2.15:
+      // A bind clause may not bind to a routine name that has a visible bind
+      // clause.
+      // TODO OpenACC: There is an exception to this rule that if these are the
+      // implicit function style (that is, without a name), they may have
+      // duplicates as long as they have the same name.
+      auto BindItr = llvm::find_if(Clauses, llvm::IsaPred<OpenACCBindClause>);
+      if (auto *A = FD->getAttr<OpenACCRoutineAnnotAttr>()) {
+        if (BindItr != Clauses.end()) {
+          if (A->BindClause.isInvalid()) {
+            // If we have a bind clause, and the function doesn't have one
+            // annotated yet, set it.
+            A->BindClause = (*BindItr)->getBeginLoc();
+          } else {
+            Diag((*BindItr)->getBeginLoc(), diag::err_acc_duplicate_bind);
+            Diag(A->BindClause, diag::note_acc_previous_clause_here);
+          }
+        }
+      } else {
+        auto *RAA = OpenACCRoutineAnnotAttr::Create(getASTContext(), DirLoc);
+        FD->addAttr(RAA);
+        if (BindItr != Clauses.end())
+          RAA->BindClause = (*BindItr)->getBeginLoc();
+      }
     }
 
     return DeclGroupRef{RoutineDecl};
diff --git a/clang/lib/Sema/SemaOpenACCClause.cpp b/clang/lib/Sema/SemaOpenACCClause.cpp
index 582681f247b31..ad54e2bbe9495 100644
--- a/clang/lib/Sema/SemaOpenACCClause.cpp
+++ b/clang/lib/Sema/SemaOpenACCClause.cpp
@@ -483,6 +483,14 @@ bool doesClauseApplyToDirective(OpenACCDirectiveKind DirectiveKind,
       return false;
     }
   }
+  case OpenACCClauseKind::Bind: {
+    switch (DirectiveKind) {
+    case OpenACCDirectiveKind::Routine:
+      return true;
+    default:
+      return false;
+    }
+  }
   }
 
   default:
@@ -509,11 +517,6 @@ bool checkAlreadyHasClauseOfKind(
 bool checkValidAfterDeviceType(
     SemaOpenACC &S, const OpenACCDeviceTypeClause &DeviceTypeClause,
     const SemaOpenACC::OpenACCParsedClause &NewClause) {
-  // This is implemented for everything but 'routine', so treat as 'fine' for
-  // that.
-  if (NewClause.getDirectiveKind() == OpenACCDirectiveKind::Routine)
-    return false;
-
   // OpenACC3.3: Section 2.4: Clauses that precede any device_type clause are
   // default clauses.  Clauses that follow a device_type clause up to the end of
   // the directive or up to the next device_type clause are device-specific
@@ -601,6 +604,19 @@ bool checkValidAfterDeviceType(
     default:
       break;
     }
+  } else if (NewClause.getDirectiveKind() == OpenACCDirectiveKind::Routine) {
+    // OpenACC 3.3 section 2.15: Only the 'gang', 'worker', 'vector', 'seq', and
+    // 'bind' clauses may follow a device_type clause.
+    switch (NewClause.getClauseKind()) {
+    case OpenACCClauseKind::Gang:
+    case OpenACCClauseKind::Worker:
+    case OpenACCClauseKind::Vector:
+    case OpenACCClauseKind::Seq:
+    case OpenACCClauseKind::Bind:
+      return false;
+    default:
+      break;
+    }
   }
   S.Diag(NewClause.getBeginLoc(), diag::err_acc_clause_after_device_type)
       << NewClause.getClauseKind() << DeviceTypeClause.getClauseKind()
@@ -669,10 +685,6 @@ class SemaOpenACCClauseVisitor {
   SemaOpenACCClauseVisitor(SemaOpenACC &S,
                            ArrayRef<const OpenACCClause *> ExistingClauses)
       : SemaRef(S), Ctx(S.getASTContext()), ExistingClauses(ExistingClauses) {}
-  // Once we've implemented everything, we shouldn't need this infrastructure.
-  // But in the meantime, we use this to help decide whether the clause was
-  // handled for this directive.
-  bool diagNotImplemented() { return NotImplemented; }
 
   OpenACCClause *Visit(SemaOpenACC::OpenACCParsedClause &Clause) {
     switch (Clause.getClauseKind()) {
@@ -1256,10 +1268,6 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitWaitClause(
 
 OpenACCClause *SemaOpenACCClauseVisitor::VisitDeviceTypeClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
-  // Restrictions implemented properly on everything except 'routine'.
-  if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Routine)
-    return isNotImplemented();
-
   // OpenACC 3.3 2.14.3: Two instances of the same clause may not appear on the
   // same directive.
   if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Set &&
@@ -1981,6 +1989,17 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitCollapseClause(
                                        LoopCount.get(), Clause.getEndLoc());
 }
 
+OpenACCClause *SemaOpenACCClauseVisitor::VisitBindClause(
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  if (std::holds_alternative<StringLiteral *>(Clause.getBindDetails()))
+    return OpenACCBindClause::Create(
+        Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(),
+        std::get<StringLiteral *>(Clause.getBindDetails()), Clause.getEndLoc());
+  return OpenACCBindClause::Create(
+      Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(),
+      std::get<IdentifierInfo *>(Clause.getBindDetails()), Clause.getEndLoc());
+}
+
 // Return true if the two vars refer to the same variable, for the purposes of
 // equality checking.
 bool areVarsEqual(Expr *VarExpr1, Expr *VarExpr2) {
@@ -2056,11 +2075,8 @@ SemaOpenACC::ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
     return nullptr;
   }
 
-  if (const auto *DevTypeClause =
-          llvm::find_if(ExistingClauses,
-                        [&](const OpenACCClause *C) {
-                          return isa<OpenACCDeviceTypeClause>(C);
-                        });
+  if (const auto *DevTypeClause = llvm::find_if(
+          ExistingClauses, llvm::IsaPred<OpenACCDeviceTypeClause>);
       DevTypeClause != ExistingClauses.end()) {
     if (checkValidAfterDeviceType(
             *this, *cast<OpenACCDeviceTypeClause>(*DevTypeClause), Clause))
@@ -2072,12 +2088,7 @@ SemaOpenACC::ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
   assert((!Result || Result->getClauseKind() == Clause.getClauseKind()) &&
          "Created wrong clause?");
 
-  if (Visitor.diagNotImplemented())
-    Diag(Clause.getBeginLoc(), diag::warn_acc_clause_unimplemented)
-        << Clause.getClauseKind();
-
   return Result;
-
 }
 
 /// OpenACC 3.3 section 2.5.15:
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 616296027d811..e2c1b2aa81ce5 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -6626,6 +6626,7 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
       case OMPC_reverse_offload:
       case OMPC_dynamic_allocators:
       case OMPC_atomic_default_mem_order:
+      case OMPC_self_maps:
       case OMPC_device_type:
       case OMPC_match:
       case OMPC_when:
@@ -15537,6 +15538,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
   case OMPC_reverse_offload:
   case OMPC_dynamic_allocators:
   case OMPC_atomic_default_mem_order:
+  case OMPC_self_maps:
   case OMPC_device_type:
   case OMPC_match:
   case OMPC_nontemporal:
@@ -16192,6 +16194,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPSimpleClause(
   case OMPC_unified_shared_memory:
   case OMPC_reverse_offload:
   case OMPC_dynamic_allocators:
+  case OMPC_self_maps:
   case OMPC_device_type:
   case OMPC_match:
   case OMPC_nontemporal:
@@ -16664,6 +16667,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprWithArgClause(
   case OMPC_reverse_offload:
   case OMPC_dynamic_allocators:
   case OMPC_atomic_default_mem_order:
+  case OMPC_self_maps:
   case OMPC_device_type:
   case OMPC_match:
   case OMPC_nontemporal:
@@ -16871,6 +16875,9 @@ OMPClause *SemaOpenMP::ActOnOpenMPClause(OpenMPClauseKind Kind,
   case OMPC_dynamic_allocators:
     Res = ActOnOpenMPDynamicAllocatorsClause(StartLoc, EndLoc);
     break;
+  case OMPC_self_maps:
+    Res = ActOnOpenMPSelfMapsClause(StartLoc, EndLoc);
+    break;
   case OMPC_destroy:
     Res = ActOnOpenMPDestroyClause(/*InteropVar=*/nullptr, StartLoc,
                                    /*LParenLoc=*/SourceLocation(),
@@ -17081,6 +17088,11 @@ SemaOpenMP::ActOnOpenMPDynamicAllocatorsClause(SourceLocation StartLoc,
   return new (getASTContext()) OMPDynamicAllocatorsClause(StartLoc, EndLoc);
 }
 
+OMPClause *SemaOpenMP::ActOnOpenMPSelfMapsClause(SourceLocation StartLoc,
+                                                 SourceLocation EndLoc) {
+  return new (getASTContext()) OMPSelfMapsClause(StartLoc, EndLoc);
+}
+
 StmtResult
 SemaOpenMP::ActOnOpenMPInteropDirective(ArrayRef<OMPClause *> Clauses,
                                         SourceLocation StartLoc,
@@ -17553,6 +17565,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPVarListClause(OpenMPClauseKind Kind,
   case OMPC_reverse_offload:
   case OMPC_dynamic_allocators:
   case OMPC_atomic_default_mem_order:
+  case OMPC_self_maps:
   case OMPC_device_type:
   case OMPC_match:
   case OMPC_order:
@@ -21961,7 +21974,8 @@ OMPClause *SemaOpenMP::ActOnOpenMPMapClause(
   OpenMPMapModifierKind Modifiers[] = {
       OMPC_MAP_MODIFIER_unknown, OMPC_MAP_MODIFIER_unknown,
       OMPC_MAP_MODIFIER_unknown, OMPC_MAP_MODIFIER_unknown,
-      OMPC_MAP_MODIFIER_unknown, OMPC_MAP_MODIFIER_unknown};
+      OMPC_MAP_MODIFIER_unknown, OMPC_MAP_MODIFIER_unknown,
+      OMPC_MAP_MODIFIER_unknown};
   SourceLocation ModifiersLoc[NumberOfOMPMapClauseModifiers];
 
   if (IteratorModifier && !IteratorModifier->getType()->isSpecificBuiltinType(
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index d3c0534b4dd0b..990f2659eb8fa 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -8147,17 +8147,14 @@ void Sema::AddConversionCandidate(
 
   ExprValueKind VK = Expr::getValueKindForType(ConversionType);
 
-  // Note that it is safe to allocate CallExpr on the stack here because
-  // there are 0 arguments (i.e., nothing is allocated using ASTContext's
-  // allocator).
   QualType CallResultType = ConversionType.getNonLValueExprType(Context);
 
-  alignas(CallExpr) char Buffer[sizeof(CallExpr) + sizeof(Stmt *)];
-  CallExpr *TheTemporaryCall = CallExpr::CreateTemporary(
-      Buffer, &ConversionFn, CallResultType, VK, From->getBeginLoc());
+  // Introduce a temporary expression with the right type and value category
+  // that we can use for deduction purposes.
+  OpaqueValueExpr FakeCall(From->getBeginLoc(), CallResultType, VK);
 
   ImplicitConversionSequence ICS =
-      TryCopyInitialization(*this, TheTemporaryCall, ToType,
+      TryCopyInitialization(*this, &FakeCall, ToType,
                             /*SuppressUserConversions=*/true,
                             /*InOverloadResolution=*/false,
                             /*AllowObjCWritebackConversion=*/false);
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index 0a193b5299bcc..944e3300c83fc 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -4052,9 +4052,9 @@ StmtResult Sema::BuildReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp,
           Diag(ReturnLoc, D) << CurDecl << isa<CXXDestructorDecl>(CurDecl)
                              << RetValExp->getSourceRange();
         }
-        // return (some void expression); is legal in C++.
+        // return (some void expression); is legal in C++ and C2y.
         else if (D != diag::ext_return_has_void_expr ||
-                 !getLangOpts().CPlusPlus) {
+                 (!getLangOpts().CPlusPlus && !getLangOpts().C2y)) {
           NamedDecl *CurDecl = getCurFunctionOrMethodDecl();
 
           int FunctionKind = 0;
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 0caabc6573361..64aabb1fcdc35 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -363,12 +363,12 @@ bool Sema::DiagnoseUnknownTemplateName(const IdentifierInfo &II,
 
   // The code is missing a 'template' keyword prior to the dependent template
   // name.
-  NestedNameSpecifier *Qualifier = (NestedNameSpecifier*)SS->getScopeRep();
-  Diag(IILoc, diag::err_template_kw_missing)
-    << Qualifier << II.getName()
-    << FixItHint::CreateInsertion(IILoc, "template ");
+  NestedNameSpecifier *Qualifier = (NestedNameSpecifier *)SS->getScopeRep();
   SuggestedTemplate
     = TemplateTy::make(Context.getDependentTemplateName(Qualifier, &II));
+  Diag(IILoc, diag::err_template_kw_missing)
+      << SuggestedTemplate.get()
+      << FixItHint::CreateInsertion(IILoc, "template ");
   SuggestedKind = TNK_Dependent_template_name;
   return true;
 }
@@ -660,7 +660,7 @@ void Sema::diagnoseExprIntendedAsTemplateName(Scope *S, ExprResult TemplateName,
   // was missing.
   if (MissingTemplateKeyword) {
     Diag(NameInfo.getBeginLoc(), diag::err_template_kw_missing)
-        << "" << NameInfo.getName().getAsString() << SourceRange(Less, Greater);
+        << NameInfo.getName() << SourceRange(Less, Greater);
     return;
   }
 
@@ -869,11 +869,9 @@ void Sema::DiagnoseTemplateParameterShadow(SourceLocation Loc, Decl *PrevDecl,
           ? diag::ext_template_param_shadow
           : (SupportedForCompatibility ? diag::ext_compat_template_param_shadow
                                        : diag::err_template_param_shadow);
-  auto *ND = cast<NamedDecl>(PrevDecl);
-  CheckTemplateParameterRAII CTP(*this, ND);
-  // FIXME: Don't put the name in the diagnostic, unless there is no source
-  // location.
+  const auto *ND = cast<NamedDecl>(PrevDecl);
   Diag(Loc, DiagId) << ND->getDeclName();
+  NoteTemplateParameterLocation(*ND);
 }
 
 TemplateDecl *Sema::AdjustDeclIfTemplate(Decl *&D) {
@@ -1593,8 +1591,16 @@ NamedDecl *Sema::ActOnTemplateTemplateParameter(
   assert(S->isTemplateParamScope() &&
          "Template template parameter not in template parameter scope!");
 
-  // Construct the parameter object.
   bool IsParameterPack = EllipsisLoc.isValid();
+
+  bool Invalid = false;
+  if (CheckTemplateParameterList(
+          Params,
+          /*OldParams=*/nullptr,
+          IsParameterPack ? TPC_TemplateTemplateParameterPack : TPC_Other))
+    Invalid = true;
+
+  // Construct the parameter object.
   TemplateTemplateParmDecl *Param = TemplateTemplateParmDecl::Create(
       Context, Context.getTranslationUnitDecl(),
       NameLoc.isInvalid() ? TmpLoc : NameLoc, Depth, Position, IsParameterPack,
@@ -1617,9 +1623,12 @@ NamedDecl *Sema::ActOnTemplateTemplateParameter(
   if (Params->size() == 0) {
     Diag(Param->getLocation(), diag::err_template_template_parm_no_parms)
     << SourceRange(Params->getLAngleLoc(), Params->getRAngleLoc());
-    Param->setInvalidDecl();
+    Invalid = true;
   }
 
+  if (Invalid)
+    Param->setInvalidDecl();
+
   // C++0x [temp.param]p9:
   //   A default template-argument may be specified for any kind of
   //   template-parameter that is not a template parameter pack.
@@ -2068,7 +2077,7 @@ DeclResult Sema::CheckClassTemplate(
            SemanticContext->isDependentContext())
               ? TPC_ClassTemplateMember
           : TUK == TagUseKind::Friend ? TPC_FriendClassTemplate
-                                      : TPC_ClassTemplate,
+                                      : TPC_Other,
           SkipBody))
     Invalid = true;
 
@@ -2210,9 +2219,8 @@ static bool DiagnoseDefaultTemplateArgument(Sema &S,
                                             SourceLocation ParamLoc,
                                             SourceRange DefArgRange) {
   switch (TPC) {
-  case Sema::TPC_ClassTemplate:
-  case Sema::TPC_VarTemplate:
-  case Sema::TPC_TypeAliasTemplate:
+  case Sema::TPC_Other:
+  case Sema::TPC_TemplateTemplateParameterPack:
     return false;
 
   case Sema::TPC_FunctionTemplate:
@@ -2385,8 +2393,11 @@ bool Sema::CheckTemplateParameterList(TemplateParameterList *NewParams,
         MissingDefaultArg = true;
     } else if (NonTypeTemplateParmDecl *NewNonTypeParm
                = dyn_cast<NonTypeTemplateParmDecl>(*NewParam)) {
-      // Check for unexpanded parameter packs.
-      if (!NewNonTypeParm->isParameterPack() &&
+      // Check for unexpanded parameter packs, except in a template template
+      // parameter pack, as in those any unexpanded packs should be expanded
+      // along with the parameter itself.
+      if (TPC != TPC_TemplateTemplateParameterPack &&
+          !NewNonTypeParm->isParameterPack() &&
           DiagnoseUnexpandedParameterPack(NewNonTypeParm->getLocation(),
                                           NewNonTypeParm->getTypeSourceInfo(),
                                           UPPC_NonTypeTemplateParameterType)) {
@@ -2494,8 +2505,7 @@ bool Sema::CheckTemplateParameterList(TemplateParameterList *NewParams,
     //   If a template parameter of a primary class template or alias template
     //   is a template parameter pack, it shall be the last template parameter.
     if (SawParameterPack && (NewParam + 1) != NewParamEnd &&
-        (TPC == TPC_ClassTemplate || TPC == TPC_VarTemplate ||
-         TPC == TPC_TypeAliasTemplate)) {
+        (TPC == TPC_Other || TPC == TPC_TemplateTemplateParameterPack)) {
       Diag((*NewParam)->getLocation(),
            diag::err_template_param_pack_must_be_last_template_parameter);
       Invalid = true;
@@ -2528,8 +2538,8 @@ bool Sema::CheckTemplateParameterList(TemplateParameterList *NewParams,
           << PrevModuleName;
       Invalid = true;
     } else if (MissingDefaultArg &&
-               (TPC == TPC_ClassTemplate || TPC == TPC_FriendClassTemplate ||
-                TPC == TPC_VarTemplate || TPC == TPC_TypeAliasTemplate)) {
+               (TPC == TPC_Other || TPC == TPC_TemplateTemplateParameterPack ||
+                TPC == TPC_FriendClassTemplate)) {
       // C++ 23[temp.param]p14:
       // If a template-parameter of a class template, variable template, or
       // alias template has a default template argument, each subsequent
@@ -3764,16 +3774,17 @@ TypeResult Sema::ActOnTemplateIdType(
     //   elaborated-type-specifier (7.1.5.3).
     if (!LookupCtx && isDependentScopeSpecifier(SS)) {
       // C++2a relaxes some of those restrictions in [temp.res]p5.
+      NestedNameSpecifier *NNS =
+          NestedNameSpecifier::Create(Context, SS.getScopeRep(), TemplateII);
       if (AllowImplicitTypename == ImplicitTypenameContext::Yes) {
         if (getLangOpts().CPlusPlus20)
           Diag(SS.getBeginLoc(), diag::warn_cxx17_compat_implicit_typename);
         else
           Diag(SS.getBeginLoc(), diag::ext_implicit_typename)
-              << SS.getScopeRep() << TemplateII->getName()
+              << NNS
               << FixItHint::CreateInsertion(SS.getBeginLoc(), "typename ");
       } else
-        Diag(SS.getBeginLoc(), diag::err_typename_missing_template)
-            << SS.getScopeRep() << TemplateII->getName();
+        Diag(SS.getBeginLoc(), diag::err_typename_missing_template) << NNS;
 
       // FIXME: This is not quite correct recovery as we don't transform SS
       // into the corresponding dependent form (and we don't diagnose missing
@@ -4826,7 +4837,7 @@ TemplateNameKind Sema::ActOnTemplateName(Scope *S,
 }
 
 bool Sema::CheckTemplateTypeArgument(
-    TemplateArgumentLoc &AL,
+    TemplateTypeParmDecl *Param, TemplateArgumentLoc &AL,
     SmallVectorImpl<TemplateArgument> &SugaredConverted,
     SmallVectorImpl<TemplateArgument> &CanonicalConverted) {
   const TemplateArgument &Arg = AL.getArgument();
@@ -4882,6 +4893,7 @@ bool Sema::CheckTemplateTypeArgument(
                       ? diag::ext_ms_template_type_arg_missing_typename
                       : diag::err_template_arg_must_be_type_suggest)
             << FixItHint::CreateInsertion(Loc, "typename ");
+        NoteTemplateParameterLocation(*Param);
 
         // Recover by synthesizing a type using the location information that we
         // already have.
@@ -4919,6 +4931,7 @@ bool Sema::CheckTemplateTypeArgument(
     // is not a type.
     SourceRange SR = AL.getSourceRange();
     Diag(SR.getBegin(), diag::err_template_arg_must_be_type) << SR;
+    NoteTemplateParameterLocation(*Param);
 
     return true;
   }
@@ -5208,8 +5221,8 @@ bool Sema::CheckTemplateArgument(NamedDecl *Param, TemplateArgumentLoc &ArgLoc,
                                  CheckTemplateArgumentInfo &CTAI,
                                  CheckTemplateArgumentKind CTAK) {
   // Check template type parameters.
-  if (isa<TemplateTypeParmDecl>(Param))
-    return CheckTemplateTypeArgument(ArgLoc, CTAI.SugaredConverted,
+  if (TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(Param))
+    return CheckTemplateTypeArgument(TTP, ArgLoc, CTAI.SugaredConverted,
                                      CTAI.CanonicalConverted);
 
   const TemplateArgument &Arg = ArgLoc.getArgument();
@@ -5354,6 +5367,8 @@ bool Sema::CheckTemplateArgument(NamedDecl *Param, TemplateArgumentLoc &ArgLoc,
       // therefore cannot be a non-type template argument.
       Diag(ArgLoc.getLocation(), diag::err_template_arg_must_be_expr)
           << ArgLoc.getSourceRange();
+      NoteTemplateParameterLocation(*Param);
+
       return true;
 
     case TemplateArgument::Type: {
@@ -5373,6 +5388,7 @@ bool Sema::CheckTemplateArgument(NamedDecl *Param, TemplateArgumentLoc &ArgLoc,
         Diag(SR.getBegin(), diag::err_template_arg_nontype_ambig) << SR << T;
       else
         Diag(SR.getBegin(), diag::err_template_arg_must_be_expr) << SR;
+      NoteTemplateParameterLocation(*Param);
       return true;
     }
 
@@ -5463,11 +5479,11 @@ bool Sema::CheckTemplateArgument(NamedDecl *Param, TemplateArgumentLoc &ArgLoc,
 }
 
 /// Diagnose a missing template argument.
-template <typename TemplateParmDecl>
+template<typename TemplateParmDecl>
 static bool diagnoseMissingArgument(Sema &S, SourceLocation Loc,
-                                    TemplateDecl *TD, const TemplateParmDecl *D,
-                                    TemplateArgumentListInfo &Args,
-                                    bool MatchingTTP) {
+                                    TemplateDecl *TD,
+                                    const TemplateParmDecl *D,
+                                    TemplateArgumentListInfo &Args) {
   // Dig out the most recent declaration of the template parameter; there may be
   // declarations of the template that are more recent than TD.
   D = cast<TemplateParmDecl>(cast<TemplateDecl>(TD->getMostRecentDecl())
@@ -5485,12 +5501,16 @@ static bool diagnoseMissingArgument(Sema &S, SourceLocation Loc,
     return true;
   }
 
-  SourceLocation DiagLoc = Args.getRAngleLoc();
   // FIXME: If there's a more recent default argument that *is* visible,
   // diagnose that it was declared too late.
-  S.Diag(DiagLoc.isValid() ? DiagLoc : Loc,
-         MatchingTTP ? diag::err_template_template_param_missing_param
-                     : diag::err_template_param_missing_arg);
+
+  TemplateParameterList *Params = TD->getTemplateParameters();
+
+  S.Diag(Loc, diag::err_template_arg_list_different_arity)
+    << /*not enough args*/0
+    << (int)S.getTemplateNameKindForDiagnostics(TemplateName(TD))
+    << TD;
+  S.NoteTemplateLocation(*TD, Params->getSourceRange());
   return true;
 }
 
@@ -5529,8 +5549,6 @@ bool Sema::CheckTemplateArgumentList(
                                        Param = ParamBegin;
        Param != ParamEnd;
        /* increment in loop */) {
-    CheckTemplateParameterRAII CTP1(*this, *Param);
-
     if (size_t ParamIdx = Param - ParamBegin;
         DefaultArgs && ParamIdx >= DefaultArgs.StartPos) {
       // All written arguments should have been consumed by this point.
@@ -5567,9 +5585,11 @@ bool Sema::CheckTemplateArgumentList(
         continue;
       } else if (ArgIdx == NumArgs && !PartialTemplateArgs) {
         // Not enough arguments for this parameter pack.
-        Diag(RAngleLoc, CTAI.MatchingTTP
-                            ? diag::err_template_template_param_missing_param
-                            : diag::err_template_param_missing_arg);
+        Diag(TemplateLoc, diag::err_template_arg_list_different_arity)
+          << /*not enough args*/0
+          << (int)getTemplateNameKindForDiagnostics(TemplateName(Template))
+          << Template;
+        NoteTemplateLocation(*Template, Params->getSourceRange());
         return true;
       }
     }
@@ -5582,10 +5602,8 @@ bool Sema::CheckTemplateArgumentList(
 
       if (ArgIsExpansion && CTAI.MatchingTTP) {
         SmallVector<TemplateArgument, 4> Args(ParamEnd - Param);
-        CTP1.Clear(); // Will continue processing parameters below.
         for (TemplateParameterList::iterator First = Param; Param != ParamEnd;
              ++Param) {
-          CheckTemplateParameterRAII CTP2(*this, *Param);
           TemplateArgument &Arg = Args[Param - First];
           Arg = ArgLoc.getArgument();
           if (!(*Param)->isTemplateParameterPack() ||
@@ -5626,6 +5644,7 @@ bool Sema::CheckTemplateArgumentList(
                  diag::err_template_expansion_into_fixed_list)
                 << (isa<ConceptDecl>(Template) ? 1 : 0)
                 << ArgLoc.getSourceRange();
+            NoteTemplateParameterLocation(**Param);
             return true;
           }
         }
@@ -5732,14 +5751,14 @@ bool Sema::CheckTemplateArgumentList(
       if (!HasDefaultArg) {
         if (TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(*Param))
           return diagnoseMissingArgument(*this, TemplateLoc, Template, TTP,
-                                         NewArgs, CTAI.MatchingTTP);
+                                         NewArgs);
         if (NonTypeTemplateParmDecl *NTTP =
                 dyn_cast<NonTypeTemplateParmDecl>(*Param))
           return diagnoseMissingArgument(*this, TemplateLoc, Template, NTTP,
-                                         NewArgs, CTAI.MatchingTTP);
+                                         NewArgs);
         return diagnoseMissingArgument(*this, TemplateLoc, Template,
                                        cast<TemplateTemplateParmDecl>(*Param),
-                                       NewArgs, CTAI.MatchingTTP);
+                                       NewArgs);
       }
       return true;
     }
@@ -5795,7 +5814,8 @@ bool Sema::CheckTemplateArgumentList(
   // If we have any leftover arguments, then there were too many arguments.
   // Complain and fail.
   if (ArgIdx < NumArgs) {
-    Diag(TemplateLoc, diag::err_template_too_many_args)
+    Diag(TemplateLoc, diag::err_template_arg_list_different_arity)
+        << /*too many args*/1
         << (int)getTemplateNameKindForDiagnostics(TemplateName(Template))
         << Template
         << SourceRange(NewArgs[ArgIdx].getLocation(), NewArgs.getRAngleLoc());
@@ -6220,6 +6240,8 @@ isNullPointerValueTemplateArgument(Sema &S, NonTypeTemplateParmDecl *Param,
       << Arg->getType() << Arg->getSourceRange();
     for (unsigned I = 0, N = Notes.size(); I != N; ++I)
       S.Diag(Notes[I].first, Notes[I].second);
+
+    S.NoteTemplateParameterLocation(*Param);
     return NPV_Error;
   }
 
@@ -6244,7 +6266,8 @@ isNullPointerValueTemplateArgument(Sema &S, NonTypeTemplateParmDecl *Param,
     // The types didn't match, but we know we got a null pointer; complain,
     // then recover as if the types were correct.
     S.Diag(Arg->getExprLoc(), diag::err_template_arg_wrongtype_null_constant)
-        << Arg->getType() << ParamType << Arg->getSourceRange();
+      << Arg->getType() << ParamType << Arg->getSourceRange();
+    S.NoteTemplateParameterLocation(*Param);
     return NPV_NullPointer;
   }
 
@@ -6253,7 +6276,8 @@ isNullPointerValueTemplateArgument(Sema &S, NonTypeTemplateParmDecl *Param,
     // We could just return NPV_NotNullPointer, but we can print a better
     // message with the information we have here.
     S.Diag(Arg->getExprLoc(), diag::err_template_arg_invalid)
-        << EvalResult.Val.getAsString(S.Context, ParamType);
+      << EvalResult.Val.getAsString(S.Context, ParamType);
+    S.NoteTemplateParameterLocation(*Param);
     return NPV_Error;
   }
 
@@ -6265,6 +6289,7 @@ isNullPointerValueTemplateArgument(Sema &S, NonTypeTemplateParmDecl *Param,
         << ParamType << FixItHint::CreateInsertion(Arg->getBeginLoc(), Code)
         << FixItHint::CreateInsertion(S.getLocForEndOfToken(Arg->getEndLoc()),
                                       ")");
+    S.NoteTemplateParameterLocation(*Param);
     return NPV_NullPointer;
   }
 
@@ -6305,6 +6330,7 @@ static bool CheckTemplateArgumentIsCompatibleWithParameter(
           S.Diag(Arg->getBeginLoc(),
                  diag::err_template_arg_ref_bind_ignores_quals)
               << ParamType << Arg->getType() << Arg->getSourceRange();
+          S.NoteTemplateParameterLocation(*Param);
           return true;
         }
       }
@@ -6322,6 +6348,7 @@ static bool CheckTemplateArgumentIsCompatibleWithParameter(
       else
         S.Diag(Arg->getBeginLoc(), diag::err_template_arg_not_convertible)
             << ArgIn->getType() << ParamType << Arg->getSourceRange();
+      S.NoteTemplateParameterLocation(*Param);
       return true;
     }
   }
@@ -6464,6 +6491,7 @@ static bool CheckTemplateArgumentAddressOfObjectOrFunction(
   if (!Entity) {
     S.Diag(Arg->getBeginLoc(), diag::err_template_arg_not_decl_ref)
         << Arg->getSourceRange();
+    S.NoteTemplateParameterLocation(*Param);
     return true;
   }
 
@@ -6471,6 +6499,7 @@ static bool CheckTemplateArgumentAddressOfObjectOrFunction(
   if (isa<FieldDecl>(Entity) || isa<IndirectFieldDecl>(Entity)) {
     S.Diag(Arg->getBeginLoc(), diag::err_template_arg_field)
         << Entity << Arg->getSourceRange();
+    S.NoteTemplateParameterLocation(*Param);
     return true;
   }
 
@@ -6479,6 +6508,7 @@ static bool CheckTemplateArgumentAddressOfObjectOrFunction(
     if (!Method->isStatic()) {
       S.Diag(Arg->getBeginLoc(), diag::err_template_arg_method)
           << Method << Arg->getSourceRange();
+      S.NoteTemplateParameterLocation(*Param);
       return true;
     }
   }
@@ -6518,6 +6548,7 @@ static bool CheckTemplateArgumentAddressOfObjectOrFunction(
     if (Var->getType()->isReferenceType()) {
       S.Diag(Arg->getBeginLoc(), diag::err_template_arg_reference_var)
           << Var->getType() << Arg->getSourceRange();
+      S.NoteTemplateParameterLocation(*Param);
       return true;
     }
 
@@ -6537,12 +6568,15 @@ static bool CheckTemplateArgumentAddressOfObjectOrFunction(
     if (!S.Context.hasSameUnqualifiedType(Entity->getType(),
                                           ParamType.getNonReferenceType())) {
       S.Diag(AddrOpLoc, diag::err_template_arg_address_of_non_pointer)
-          << ParamType;
+        << ParamType;
+      S.NoteTemplateParameterLocation(*Param);
       return true;
     }
 
     S.Diag(AddrOpLoc, diag::err_template_arg_address_of_non_pointer)
-        << ParamType << FixItHint::CreateRemoval(AddrOpLoc);
+      << ParamType
+      << FixItHint::CreateRemoval(AddrOpLoc);
+    S.NoteTemplateParameterLocation(*Param);
 
     ArgType = Entity->getType();
   }
@@ -6563,12 +6597,15 @@ static bool CheckTemplateArgumentAddressOfObjectOrFunction(
       ArgType = S.Context.getPointerType(Entity->getType());
       if (!S.Context.hasSameUnqualifiedType(ArgType, ParamType)) {
         S.Diag(Arg->getBeginLoc(), diag::err_template_arg_not_address_of)
-            << ParamType;
+          << ParamType;
+        S.NoteTemplateParameterLocation(*Param);
         return true;
       }
 
       S.Diag(Arg->getBeginLoc(), diag::err_template_arg_not_address_of)
-          << ParamType << FixItHint::CreateInsertion(Arg->getBeginLoc(), "&");
+        << ParamType << FixItHint::CreateInsertion(Arg->getBeginLoc(), "&");
+
+      S.NoteTemplateParameterLocation(*Param);
     }
   }
 
@@ -6684,6 +6721,7 @@ CheckTemplateArgumentPointerToMember(Sema &S, NonTypeTemplateParmDecl *Param,
     // We can't perform this conversion.
     S.Diag(ResultArg->getBeginLoc(), diag::err_template_arg_not_convertible)
         << ResultArg->getType() << ParamType << ResultArg->getSourceRange();
+    S.NoteTemplateParameterLocation(*Param);
     return true;
   }
 
@@ -6789,6 +6827,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
              diag::err_non_type_template_parm_type_deduction_failure)
             << Param->getDeclName() << Param->getType() << Arg->getType()
             << Arg->getSourceRange();
+        NoteTemplateParameterLocation(*Param);
         return ExprError();
       }
     }
@@ -6797,8 +6836,10 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
     // declaration, but here we'll pass the argument location because that's
     // where the parameter type is deduced.
     ParamType = CheckNonTypeTemplateParameterType(ParamType, Arg->getExprLoc());
-    if (ParamType.isNull())
+    if (ParamType.isNull()) {
+      NoteTemplateParameterLocation(*Param);
       return ExprError();
+    }
   }
 
   // We should have already dropped all cv-qualifiers by now.
@@ -6830,7 +6871,9 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
     // not the type of the template argument deduced from A, against the
     // template parameter type.
     Diag(StartLoc, diag::err_deduced_non_type_template_arg_type_mismatch)
-        << Arg->getType() << ParamType.getUnqualifiedType();
+      << Arg->getType()
+      << ParamType.getUnqualifiedType();
+    NoteTemplateParameterLocation(*Param);
     return ExprError();
   }
 
@@ -6925,8 +6968,10 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
           Arg, ParamType,
           PartialOrderingTTP ? CCEK_InjectedTTP : CCEK_TemplateArg, Param);
       assert(!ArgResult.isUnset());
-      if (ArgResult.isInvalid())
+      if (ArgResult.isInvalid()) {
+        NoteTemplateParameterLocation(*Param);
         return ExprError();
+      }
     } else {
       ArgResult = Arg;
     }
@@ -7073,6 +7118,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
     if (!ArgType->isIntegralOrEnumerationType()) {
       Diag(Arg->getBeginLoc(), diag::err_template_arg_not_integral_or_enumeral)
           << ArgType << Arg->getSourceRange();
+      NoteTemplateParameterLocation(*Param);
       return ExprError();
     } else if (!Arg->isValueDependent()) {
       class TmplArgICEDiagnoser : public VerifyICEDiagnoser {
@@ -7110,6 +7156,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
       // We can't perform this conversion.
       Diag(Arg->getBeginLoc(), diag::err_template_arg_not_convertible)
           << Arg->getType() << ParamType << Arg->getSourceRange();
+      NoteTemplateParameterLocation(*Param);
       return ExprError();
     }
 
@@ -7155,6 +7202,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
         Diag(Arg->getBeginLoc(), diag::warn_template_arg_negative)
             << toString(OldValue, 10) << toString(Value, 10) << Param->getType()
             << Arg->getSourceRange();
+        NoteTemplateParameterLocation(*Param);
       }
 
       // Complain if we overflowed the template parameter's type.
@@ -7165,10 +7213,12 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
         RequiredBits = OldValue.getActiveBits() + 1;
       else
         RequiredBits = OldValue.getSignificantBits();
-      if (RequiredBits > AllowedBits)
+      if (RequiredBits > AllowedBits) {
         Diag(Arg->getBeginLoc(), diag::warn_template_arg_too_large)
             << toString(OldValue, 10) << toString(Value, 10) << Param->getType()
             << Arg->getSourceRange();
+        NoteTemplateParameterLocation(*Param);
+      }
     }
 
     QualType T = ParamType->isEnumeralType() ? ParamType : IntegerType;
@@ -7293,7 +7343,8 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
     switch (isNullPointerValueTemplateArgument(*this, Param, ParamType, Arg)) {
     case NPV_NotNullPointer:
       Diag(Arg->getExprLoc(), diag::err_template_arg_not_convertible)
-          << Arg->getType() << ParamType;
+        << Arg->getType() << ParamType;
+      NoteTemplateParameterLocation(*Param);
       return ExprError();
 
     case NPV_Error:
@@ -7321,7 +7372,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
 
 static void DiagnoseTemplateParameterListArityMismatch(
     Sema &S, TemplateParameterList *New, TemplateParameterList *Old,
-    Sema::TemplateParameterListEqualKind Kind);
+    Sema::TemplateParameterListEqualKind Kind, SourceLocation TemplateArgLoc);
 
 bool Sema::CheckTemplateTemplateArgument(TemplateTemplateParmDecl *Param,
                                          TemplateParameterList *Params,
@@ -7391,6 +7442,7 @@ bool Sema::CheckTemplateTemplateArgument(TemplateTemplateParmDecl *Param,
     Diag(Arg.getLocation(),
          diag::err_template_template_parameter_not_at_least_as_constrained)
         << Template << Param << Arg.getSourceRange();
+    Diag(Param->getLocation(), diag::note_entity_declared_at) << Param;
     Diag(Template->getLocation(), diag::note_entity_declared_at) << Template;
     MaybeEmitAmbiguousAtomicConstraintsDiagnostic(Param, ParamsAC, Template,
                                                   TemplateAC);
@@ -7399,24 +7451,25 @@ bool Sema::CheckTemplateTemplateArgument(TemplateTemplateParmDecl *Param,
   return false;
 }
 
-SmallString<128> Sema::toTerseString(const NamedDecl &D) const {
+static Sema::SemaDiagnosticBuilder noteLocation(Sema &S, const NamedDecl &Decl,
+                                                unsigned HereDiagID,
+                                                unsigned ExternalDiagID) {
+  if (Decl.getLocation().isValid())
+    return S.Diag(Decl.getLocation(), HereDiagID);
+
   SmallString<128> Str;
   llvm::raw_svector_ostream Out(Str);
-  PrintingPolicy PP = getPrintingPolicy();
+  PrintingPolicy PP = S.getPrintingPolicy();
   PP.TerseOutput = 1;
-  D.print(Out, PP);
-  return Str;
+  Decl.print(Out, PP);
+  return S.Diag(Decl.getLocation(), ExternalDiagID) << Out.str();
 }
 
-// FIXME: Transform this into a context note.
 void Sema::NoteTemplateLocation(const NamedDecl &Decl,
                                 std::optional<SourceRange> ParamRange) {
-  bool HasLoc = Decl.getLocation().isValid();
   SemaDiagnosticBuilder DB =
-      Diag(Decl.getLocation(), HasLoc ? diag::note_template_decl_here
-                                      : diag::note_template_decl_external);
-  if (!HasLoc)
-    DB << toTerseString(Decl).str();
+      noteLocation(*this, Decl, diag::note_template_decl_here,
+                   diag::note_template_decl_external);
   if (ParamRange && ParamRange->isValid()) {
     assert(Decl.getLocation().isValid() &&
            "Parameter range has location when Decl does not");
@@ -7424,6 +7477,11 @@ void Sema::NoteTemplateLocation(const NamedDecl &Decl,
   }
 }
 
+void Sema::NoteTemplateParameterLocation(const NamedDecl &Decl) {
+  noteLocation(*this, Decl, diag::note_template_param_here,
+               diag::note_template_param_external);
+}
+
 ExprResult Sema::BuildExpressionFromDeclTemplateArgument(
     const TemplateArgument &Arg, QualType ParamType, SourceLocation Loc,
     NamedDecl *TemplateParam) {
@@ -7700,17 +7758,21 @@ Sema::BuildExpressionFromNonTypeTemplateArgument(const TemplateArgument &Arg,
 }
 
 /// Match two template parameters within template parameter lists.
-static bool
-MatchTemplateParameterKind(Sema &S, NamedDecl *New,
-                           const Sema::TemplateCompareNewDeclInfo &NewInstFrom,
-                           NamedDecl *Old, const NamedDecl *OldInstFrom,
-                           bool Complain,
-                           Sema::TemplateParameterListEqualKind Kind) {
+static bool MatchTemplateParameterKind(
+    Sema &S, NamedDecl *New,
+    const Sema::TemplateCompareNewDeclInfo &NewInstFrom, NamedDecl *Old,
+    const NamedDecl *OldInstFrom, bool Complain,
+    Sema::TemplateParameterListEqualKind Kind, SourceLocation TemplateArgLoc) {
   // Check the actual kind (type, non-type, template).
   if (Old->getKind() != New->getKind()) {
     if (Complain) {
-      S.Diag(New->getLocation(), diag::err_template_param_different_kind)
-          << (Kind != Sema::TPL_TemplateMatch);
+      unsigned NextDiag = diag::err_template_param_different_kind;
+      if (TemplateArgLoc.isValid()) {
+        S.Diag(TemplateArgLoc, diag::err_template_arg_template_params_mismatch);
+        NextDiag = diag::note_template_param_different_kind;
+      }
+      S.Diag(New->getLocation(), NextDiag)
+        << (Kind != Sema::TPL_TemplateMatch);
       S.Diag(Old->getLocation(), diag::note_template_prev_declaration)
         << (Kind != Sema::TPL_TemplateMatch);
     }
@@ -7724,11 +7786,18 @@ MatchTemplateParameterKind(Sema &S, NamedDecl *New,
   // a parameter pack where the template template argument does not.
   if (Old->isTemplateParameterPack() != New->isTemplateParameterPack()) {
     if (Complain) {
+      unsigned NextDiag = diag::err_template_parameter_pack_non_pack;
+      if (TemplateArgLoc.isValid()) {
+        S.Diag(TemplateArgLoc,
+             diag::err_template_arg_template_params_mismatch);
+        NextDiag = diag::note_template_parameter_pack_non_pack;
+      }
+
       unsigned ParamKind = isa<TemplateTypeParmDecl>(New)? 0
                       : isa<NonTypeTemplateParmDecl>(New)? 1
                       : 2;
-      S.Diag(New->getLocation(), diag::err_template_parameter_pack_non_pack)
-          << ParamKind << New->isParameterPack();
+      S.Diag(New->getLocation(), NextDiag)
+        << ParamKind << New->isParameterPack();
       S.Diag(Old->getLocation(), diag::note_template_parameter_pack_here)
         << ParamKind << Old->isParameterPack();
     }
@@ -7749,8 +7818,13 @@ MatchTemplateParameterKind(Sema &S, NamedDecl *New,
     QualType NewType = S.Context.getUnconstrainedType(NewNTTP->getType());
     if (!S.Context.hasSameType(OldType, NewType)) {
       if (Complain) {
-        S.Diag(NewNTTP->getLocation(),
-               diag::err_template_nontype_parm_different_type)
+        unsigned NextDiag = diag::err_template_nontype_parm_different_type;
+        if (TemplateArgLoc.isValid()) {
+          S.Diag(TemplateArgLoc,
+                 diag::err_template_arg_template_params_mismatch);
+          NextDiag = diag::note_template_nontype_parm_different_type;
+        }
+        S.Diag(NewNTTP->getLocation(), NextDiag)
             << NewNTTP->getType() << (Kind != Sema::TPL_TemplateMatch);
         S.Diag(OldNTTP->getLocation(),
                diag::note_template_nontype_parm_prev_declaration)
@@ -7771,7 +7845,8 @@ MatchTemplateParameterKind(Sema &S, NamedDecl *New,
             OldTTP->getTemplateParameters(), Complain,
             (Kind == Sema::TPL_TemplateMatch
                  ? Sema::TPL_TemplateTemplateParmMatch
-                 : Kind)))
+                 : Kind),
+            TemplateArgLoc))
       return false;
   }
 
@@ -7822,12 +7897,21 @@ MatchTemplateParameterKind(Sema &S, NamedDecl *New,
 
 /// Diagnose a known arity mismatch when comparing template argument
 /// lists.
-static void DiagnoseTemplateParameterListArityMismatch(
-    Sema &S, TemplateParameterList *New, TemplateParameterList *Old,
-    Sema::TemplateParameterListEqualKind Kind) {
-  S.Diag(New->getTemplateLoc(), diag::err_template_param_list_different_arity)
-      << (New->size() > Old->size()) << (Kind != Sema::TPL_TemplateMatch)
-      << SourceRange(New->getTemplateLoc(), New->getRAngleLoc());
+static
+void DiagnoseTemplateParameterListArityMismatch(Sema &S,
+                                                TemplateParameterList *New,
+                                                TemplateParameterList *Old,
+                                      Sema::TemplateParameterListEqualKind Kind,
+                                                SourceLocation TemplateArgLoc) {
+  unsigned NextDiag = diag::err_template_param_list_different_arity;
+  if (TemplateArgLoc.isValid()) {
+    S.Diag(TemplateArgLoc, diag::err_template_arg_template_params_mismatch);
+    NextDiag = diag::note_template_param_list_different_arity;
+  }
+  S.Diag(New->getTemplateLoc(), NextDiag)
+    << (New->size() > Old->size())
+    << (Kind != Sema::TPL_TemplateMatch)
+    << SourceRange(New->getTemplateLoc(), New->getRAngleLoc());
   S.Diag(Old->getTemplateLoc(), diag::note_template_prev_declaration)
     << (Kind != Sema::TPL_TemplateMatch)
     << SourceRange(Old->getTemplateLoc(), Old->getRAngleLoc());
@@ -7836,10 +7920,11 @@ static void DiagnoseTemplateParameterListArityMismatch(
 bool Sema::TemplateParameterListsAreEqual(
     const TemplateCompareNewDeclInfo &NewInstFrom, TemplateParameterList *New,
     const NamedDecl *OldInstFrom, TemplateParameterList *Old, bool Complain,
-    TemplateParameterListEqualKind Kind) {
+    TemplateParameterListEqualKind Kind, SourceLocation TemplateArgLoc) {
   if (Old->size() != New->size()) {
     if (Complain)
-      DiagnoseTemplateParameterListArityMismatch(*this, New, Old, Kind);
+      DiagnoseTemplateParameterListArityMismatch(*this, New, Old, Kind,
+                                                 TemplateArgLoc);
 
     return false;
   }
@@ -7857,18 +7942,21 @@ bool Sema::TemplateParameterListsAreEqual(
        OldParm != OldParmEnd; ++OldParm, ++NewParm) {
     if (NewParm == NewParmEnd) {
       if (Complain)
-        DiagnoseTemplateParameterListArityMismatch(*this, New, Old, Kind);
+        DiagnoseTemplateParameterListArityMismatch(*this, New, Old, Kind,
+                                                   TemplateArgLoc);
       return false;
     }
     if (!MatchTemplateParameterKind(*this, *NewParm, NewInstFrom, *OldParm,
-                                    OldInstFrom, Complain, Kind))
+                                    OldInstFrom, Complain, Kind,
+                                    TemplateArgLoc))
       return false;
   }
 
   // Make sure we exhausted all of the arguments.
   if (NewParm != NewParmEnd) {
     if (Complain)
-      DiagnoseTemplateParameterListArityMismatch(*this, New, Old, Kind);
+      DiagnoseTemplateParameterListArityMismatch(*this, New, Old, Kind,
+                                                 TemplateArgLoc);
 
     return false;
   }
@@ -8166,6 +8254,7 @@ static bool CheckNonTypeTemplatePartialSpecializationArgs(
       S.Diag(IsDefaultArgument ? TemplateNameLoc : ArgExpr->getBeginLoc(),
              diag::err_dependent_typed_non_type_arg_in_partial_spec)
           << Param->getType();
+      S.NoteTemplateParameterLocation(*Param);
       return true;
     }
   }
@@ -8189,7 +8278,6 @@ bool Sema::CheckTemplatePartialSpecializationArgs(
     if (!Param)
       continue;
 
-    CheckTemplateParameterRAII CTP(*this, Param);
     if (CheckNonTypeTemplatePartialSpecializationArgs(*this, TemplateNameLoc,
                                                       Param, &TemplateArgs[I],
                                                       1, I >= NumExplicit))
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index b1a1f86c5cfe1..e6ec4a7178e81 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -3006,7 +3006,7 @@ ConvertDeducedTemplateArgument(Sema &S, NamedDecl *Param,
         // arguments).
         S.Diag(Param->getLocation(),
                diag::err_template_arg_deduced_incomplete_pack)
-            << Arg << Param;
+          << Arg << Param;
         return true;
       }
       if (ConvertArg(InnerArg, SugaredPackedArgsBuilder.size()))
@@ -3072,7 +3072,7 @@ static TemplateDeductionResult ConvertDeducedTemplateArguments(
 
   for (unsigned I = 0, N = TemplateParams->size(); I != N; ++I) {
     NamedDecl *Param = TemplateParams->getParam(I);
-    Sema::CheckTemplateParameterRAII CTP(S, Param);
+
     // C++0x [temp.arg.explicit]p3:
     //    A trailing template parameter pack (14.5.3) not otherwise deduced will
     //    be deduced to an empty sequence of template arguments.
@@ -3427,9 +3427,9 @@ static TemplateDeductionResult FinishTemplateArgumentDeduction(
       if (!P.isPackExpansion() && !A.isPackExpansion()) {
         Info.Param =
             makeTemplateParameter(Template->getTemplateParameters()->getParam(
-                (PsStack.empty() ? TemplateArgs.end()
-                                 : PsStack.front().begin()) -
-                TemplateArgs.begin()));
+                (AsStack.empty() ? CTAI.CanonicalConverted.end()
+                                 : AsStack.front().begin()) -
+                1 - CTAI.CanonicalConverted.begin()));
         Info.FirstArg = P;
         Info.SecondArg = A;
         return TemplateDeductionResult::NonDeducedMismatch;
@@ -6642,17 +6642,19 @@ bool Sema::isTemplateTemplateParameterAtLeastAsSpecializedAs(
 
   TemplateDeductionResult TDK;
   runWithSufficientStackSpace(Info.getLocation(), [&] {
-    TDK = ::FinishTemplateArgumentDeduction(
-        *this, AArg, /*IsPartialOrdering=*/true, PArgs, Deduced, Info);
+    TDK = ::FinishTemplateArgumentDeduction(*this, AArg, PartialOrdering, PArgs,
+                                            Deduced, Info);
   });
   switch (TDK) {
   case TemplateDeductionResult::Success:
     return true;
 
   // It doesn't seem possible to get a non-deduced mismatch when partial
-  // ordering TTPs.
+  // ordering TTPs, except with an invalid template parameter list which has
+  // a parameter after a pack.
   case TemplateDeductionResult::NonDeducedMismatch:
-    llvm_unreachable("Unexpected NonDeducedMismatch");
+    assert(PArg->isInvalidDecl() && "Unexpected NonDeducedMismatch");
+    return false;
 
   // Substitution failures should have already been diagnosed.
   case TemplateDeductionResult::AlreadyDiagnosed:
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 847057a3f70ea..19c27a76b182c 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -575,7 +575,6 @@ bool Sema::CodeSynthesisContext::isInstantiationRecord() const {
   case BuildingDeductionGuides:
   case TypeAliasTemplateInstantiation:
   case PartialOrderingTTP:
-  case CheckTemplateParameter:
     return false;
 
   // This function should never be called when Kind's value is Memoization.
@@ -810,16 +809,7 @@ Sema::InstantiatingTemplate::InstantiatingTemplate(
     Sema &SemaRef, SourceLocation ArgLoc, PartialOrderingTTP,
     TemplateDecl *PArg, SourceRange InstantiationRange)
     : InstantiatingTemplate(SemaRef, CodeSynthesisContext::PartialOrderingTTP,
-                            ArgLoc, SourceRange(), PArg) {}
-
-Sema::InstantiatingTemplate::InstantiatingTemplate(Sema &SemaRef,
-                                                   CheckTemplateParameter,
-                                                   NamedDecl *Param)
-    : InstantiatingTemplate(
-          SemaRef, CodeSynthesisContext::CheckTemplateParameter,
-          Param->getLocation(), Param->getSourceRange(), Param) {
-  assert(Param->isTemplateParameter());
-}
+                            ArgLoc, InstantiationRange, PArg) {}
 
 void Sema::pushCodeSynthesisContext(CodeSynthesisContext Ctx) {
   Ctx.SavedInNonInstantiationSFINAEContext = InNonInstantiationSFINAEContext;
@@ -1261,18 +1251,12 @@ void Sema::PrintInstantiationStack(InstantiationContextDiagFuncRef DiagFunc) {
     case CodeSynthesisContext::PartialOrderingTTP:
       DiagFunc(Active->PointOfInstantiation,
                PDiag(diag::note_template_arg_template_params_mismatch));
+      if (SourceLocation ParamLoc = Active->Entity->getLocation();
+          ParamLoc.isValid())
+        DiagFunc(ParamLoc, PDiag(diag::note_template_prev_declaration)
+                               << /*isTemplateTemplateParam=*/true
+                               << Active->InstantiationRange);
       break;
-    case CodeSynthesisContext::CheckTemplateParameter: {
-      const auto &ND = *cast<NamedDecl>(Active->Entity);
-      if (SourceLocation Loc = ND.getLocation(); Loc.isValid()) {
-        DiagFunc(Loc, PDiag(diag::note_template_param_here)
-                          << ND.getSourceRange());
-        break;
-      }
-      DiagFunc(SourceLocation(), PDiag(diag::note_template_param_external)
-                                     << toTerseString(ND).str());
-      break;
-    }
     }
   }
 }
@@ -1316,7 +1300,6 @@ std::optional<TemplateDeductionInfo *> Sema::isSFINAEContext() const {
     case CodeSynthesisContext::DefaultTemplateArgumentChecking:
     case CodeSynthesisContext::RewritingOperatorAsSpaceship:
     case CodeSynthesisContext::PartialOrderingTTP:
-    case CodeSynthesisContext::CheckTemplateParameter:
       // A default template argument instantiation and substitution into
       // template parameters with arguments for prior parameters may or may
       // not be a SFINAE context; look further up the stack.
@@ -2365,7 +2348,6 @@ TemplateInstantiator::TransformSubstNonTypeTemplateParmPackExpr(
 ExprResult
 TemplateInstantiator::TransformSubstNonTypeTemplateParmExpr(
                                           SubstNonTypeTemplateParmExpr *E) {
-  Sema::CheckTemplateParameterRAII CTP(SemaRef, E->getParameter());
   ExprResult SubstReplacement = E->getReplacement();
   if (!isa<ConstantExpr>(SubstReplacement.get()))
     SubstReplacement = TransformExpr(E->getReplacement());
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index f3e33078c55c7..170c0b0d39f86 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -1056,7 +1056,6 @@ CLAUSE_NOT_ON_DECLS(Delete)
 CLAUSE_NOT_ON_DECLS(Detach)
 CLAUSE_NOT_ON_DECLS(Device)
 CLAUSE_NOT_ON_DECLS(DeviceNum)
-CLAUSE_NOT_ON_DECLS(DeviceType)
 CLAUSE_NOT_ON_DECLS(Finalize)
 CLAUSE_NOT_ON_DECLS(FirstPrivate)
 CLAUSE_NOT_ON_DECLS(Host)
@@ -1116,6 +1115,15 @@ void OpenACCDeclClauseInstantiator::VisitNoHostClause(
                                           ParsedClause.getEndLoc());
 }
 
+void OpenACCDeclClauseInstantiator::VisitDeviceTypeClause(
+    const OpenACCDeviceTypeClause &C) {
+  // Nothing to transform here, just create a new version of 'C'.
+  NewClause = OpenACCDeviceTypeClause::Create(
+      SemaRef.getASTContext(), C.getClauseKind(), ParsedClause.getBeginLoc(),
+      ParsedClause.getLParenLoc(), C.getArchitectures(),
+      ParsedClause.getEndLoc());
+}
+
 void OpenACCDeclClauseInstantiator::VisitWorkerClause(
     const OpenACCWorkerClause &C) {
   assert(!C.hasIntExpr() && "Int Expr not allowed on routine 'worker' clause");
@@ -1241,6 +1249,21 @@ void OpenACCDeclClauseInstantiator::VisitDevicePtrClause(
       ParsedClause.getEndLoc());
 }
 
+void OpenACCDeclClauseInstantiator::VisitBindClause(
+    const OpenACCBindClause &C) {
+  // Nothing to instantiate, we support only string literal or identifier.
+  if (C.isStringArgument())
+    NewClause = OpenACCBindClause::Create(
+        SemaRef.getASTContext(), ParsedClause.getBeginLoc(),
+        ParsedClause.getLParenLoc(), C.getStringArgument(),
+        ParsedClause.getEndLoc());
+  else
+    NewClause = OpenACCBindClause::Create(
+        SemaRef.getASTContext(), ParsedClause.getBeginLoc(),
+        ParsedClause.getLParenLoc(), C.getIdentifierArgument(),
+        ParsedClause.getEndLoc());
+}
+
 llvm::SmallVector<OpenACCClause *> InstantiateOpenACCClauseList(
     Sema &S, const MultiLevelTemplateArgumentList &MLTAL,
     OpenACCDirectiveKind DK, ArrayRef<const OpenACCClause *> ClauseList) {
@@ -2168,7 +2191,7 @@ Decl *TemplateDeclInstantiator::VisitClassTemplateDecl(ClassTemplateDecl *D) {
       // Do some additional validation, then merge default arguments
       // from the existing declarations.
       if (SemaRef.CheckTemplateParameterList(InstParams, PrevParams,
-                                             Sema::TPC_ClassTemplate))
+                                             Sema::TPC_Other))
         return nullptr;
 
       Inst->setAccess(PrevClassTemplate->getAccess());
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 9591fd4cfcc1c..a09f623ff8fbf 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -10835,6 +10835,12 @@ OMPClause *TreeTransform<Derived>::TransformOMPAtomicDefaultMemOrderClause(
       "atomic_default_mem_order clause cannot appear in dependent context");
 }
 
+template <typename Derived>
+OMPClause *
+TreeTransform<Derived>::TransformOMPSelfMapsClause(OMPSelfMapsClause *C) {
+  llvm_unreachable("self_maps clause cannot appear in dependent context");
+}
+
 template <typename Derived>
 OMPClause *TreeTransform<Derived>::TransformOMPAtClause(OMPAtClause *C) {
   return getDerived().RebuildOMPAtClause(C->getAtKind(), C->getAtKindKwLoc(),
@@ -11878,7 +11884,12 @@ void OpenACCClauseTransform<Derived>::VisitDeviceResidentClause(
 template <typename Derived>
 void OpenACCClauseTransform<Derived>::VisitNoHostClause(
     const OpenACCNoHostClause &C) {
-  llvm_unreachable("device_resident clause not valid unless a decl transform");
+  llvm_unreachable("nohost  clause not valid unless a decl transform");
+}
+template <typename Derived>
+void OpenACCClauseTransform<Derived>::VisitBindClause(
+    const OpenACCBindClause &C) {
+  llvm_unreachable("bind clause not valid unless a decl transform");
 }
 
 template <typename Derived>
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 2ac9754f02eed..d0a1a9221ce6f 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11094,6 +11094,9 @@ OMPClause *OMPClauseReader::readClause() {
   case llvm::omp::OMPC_atomic_default_mem_order:
     C = new (Context) OMPAtomicDefaultMemOrderClause();
     break;
+  case llvm::omp::OMPC_self_maps:
+    C = new (Context) OMPSelfMapsClause();
+    break;
   case llvm::omp::OMPC_at:
     C = new (Context) OMPAtClause();
     break;
@@ -11575,6 +11578,8 @@ void OMPClauseReader::VisitOMPAtomicDefaultMemOrderClause(
   C->setAtomicDefaultMemOrderKindKwLoc(Record.readSourceLocation());
 }
 
+void OMPClauseReader::VisitOMPSelfMapsClause(OMPSelfMapsClause *) {}
+
 void OMPClauseReader::VisitOMPAtClause(OMPAtClause *C) {
   C->setAtKind(static_cast<OpenMPAtClauseKind>(Record.readInt()));
   C->setLParenLoc(Record.readSourceLocation());
@@ -12797,7 +12802,15 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
                                                LParenLoc, VarList, EndLoc);
   }
 
-  case OpenACCClauseKind::Bind:
+  case OpenACCClauseKind::Bind: {
+    SourceLocation LParenLoc = readSourceLocation();
+    bool IsString = readBool();
+    if (IsString)
+      return OpenACCBindClause::Create(getContext(), BeginLoc, LParenLoc,
+                                       cast<StringLiteral>(readExpr()), EndLoc);
+    return OpenACCBindClause::Create(getContext(), BeginLoc, LParenLoc,
+                                     readIdentifier(), EndLoc);
+  }
   case OpenACCClauseKind::Invalid:
     llvm_unreachable("Clause serialization not yet implemented");
   }
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 0aa115ecadf8e..0f53d9e137910 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -1753,7 +1753,7 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, StringRef isysroot) {
   AddFileID(SourceMgr.getMainFileID(), Record);
   Stream.EmitRecord(ORIGINAL_FILE_ID, Record);
 
-  WriteInputFiles(SourceMgr, PP.getHeaderSearchInfo().getHeaderSearchOpts());
+  WriteInputFiles(SourceMgr);
   Stream.ExitBlock();
 }
 
@@ -1786,8 +1786,7 @@ SourceLocation ASTWriter::getAffectingIncludeLoc(const SourceManager &SourceMgr,
   return IncludeLoc;
 }
 
-void ASTWriter::WriteInputFiles(SourceManager &SourceMgr,
-                                HeaderSearchOptions &HSOpts) {
+void ASTWriter::WriteInputFiles(SourceManager &SourceMgr) {
   using namespace llvm;
 
   Stream.EnterSubblock(INPUT_FILES_BLOCK_ID, 4);
@@ -8430,6 +8429,8 @@ void OMPClauseWriter::VisitOMPAtomicDefaultMemOrderClause(
   Record.AddSourceLocation(C->getAtomicDefaultMemOrderKindKwLoc());
 }
 
+void OMPClauseWriter::VisitOMPSelfMapsClause(OMPSelfMapsClause *) {}
+
 void OMPClauseWriter::VisitOMPAtClause(OMPAtClause *C) {
   Record.push_back(C->getAtKind());
   Record.AddSourceLocation(C->getLParenLoc());
@@ -8844,7 +8845,17 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
     return;
   }
 
-  case OpenACCClauseKind::Bind:
+  case OpenACCClauseKind::Bind: {
+    const auto *BC = cast<OpenACCBindClause>(C);
+    writeSourceLocation(BC->getLParenLoc());
+    writeBool(BC->isStringArgument());
+    if (BC->isStringArgument())
+      AddStmt(const_cast<StringLiteral *>(BC->getStringArgument()));
+    else
+      AddIdentifierRef(BC->getIdentifierArgument());
+
+    return;
+  }
   case OpenACCClauseKind::Invalid:
     llvm_unreachable("Clause serialization not yet implemented");
   }
diff --git a/clang/lib/Serialization/GlobalModuleIndex.cpp b/clang/lib/Serialization/GlobalModuleIndex.cpp
index 4b920fccecac3..1e2272c48bd04 100644
--- a/clang/lib/Serialization/GlobalModuleIndex.cpp
+++ b/clang/lib/Serialization/GlobalModuleIndex.cpp
@@ -849,22 +849,21 @@ GlobalModuleIndex::writeIndex(FileManager &FileMgr,
 
   // Coordinate building the global index file with other processes that might
   // try to do the same.
-  llvm::LockFileManager Locked(IndexPath);
-  switch (Locked) {
-  case llvm::LockFileManager::LFS_Error:
+  llvm::LockFileManager Lock(IndexPath);
+  bool Owned;
+  if (llvm::Error Err = Lock.tryLock().moveInto(Owned)) {
+    llvm::consumeError(std::move(Err));
     return llvm::createStringError(std::errc::io_error, "LFS error");
-
-  case llvm::LockFileManager::LFS_Owned:
-    // We're responsible for building the index ourselves. Do so below.
-    break;
-
-  case llvm::LockFileManager::LFS_Shared:
+  }
+  if (!Owned) {
     // Someone else is responsible for building the index. We don't care
     // when they finish, so we're done.
     return llvm::createStringError(std::errc::device_or_resource_busy,
                                    "someone else is building the index");
   }
 
+  // We're responsible for building the index ourselves.
+
   // The module index builder.
   GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr);
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/AssumeModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/AssumeModeling.cpp
new file mode 100644
index 0000000000000..1e3adb4f266ca
--- /dev/null
+++ b/clang/lib/StaticAnalyzer/Checkers/AssumeModeling.cpp
@@ -0,0 +1,86 @@
+//=== AssumeModeling.cpp --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This checker evaluates the builting assume functions.
+// This checker also sinks execution paths leaving [[assume]] attributes with
+// false assumptions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/AST/AttrIterator.h"
+#include "clang/Basic/Builtins.h"
+#include "clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h"
+#include "clang/StaticAnalyzer/Core/Checker.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
+#include "llvm/ADT/STLExtras.h"
+
+using namespace clang;
+using namespace ento;
+
+namespace {
+class AssumeModelingChecker
+    : public Checker<eval::Call, check::PostStmt<AttributedStmt>> {
+public:
+  void checkPostStmt(const AttributedStmt *A, CheckerContext &C) const;
+  bool evalCall(const CallEvent &Call, CheckerContext &C) const;
+};
+} // namespace
+
+void AssumeModelingChecker::checkPostStmt(const AttributedStmt *A,
+                                          CheckerContext &C) const {
+  if (!hasSpecificAttr<CXXAssumeAttr>(A->getAttrs()))
+    return;
+
+  for (const auto *Attr : getSpecificAttrs<CXXAssumeAttr>(A->getAttrs())) {
+    SVal AssumptionVal = C.getSVal(Attr->getAssumption());
+
+    // The assumption is not evaluated at all if it had sideffects; skip them.
+    if (AssumptionVal.isUnknown())
+      continue;
+
+    const auto *Assumption = AssumptionVal.getAsInteger();
+    assert(Assumption && "We should know the exact outcome of an assume expr");
+    if (Assumption && Assumption->isZero()) {
+      C.addSink();
+    }
+  }
+}
+
+bool AssumeModelingChecker::evalCall(const CallEvent &Call,
+                                     CheckerContext &C) const {
+  ProgramStateRef State = C.getState();
+  const auto *FD = dyn_cast_or_null<FunctionDecl>(Call.getDecl());
+  if (!FD)
+    return false;
+
+  if (!llvm::is_contained({Builtin::BI__builtin_assume, Builtin::BI__assume},
+                          FD->getBuiltinID())) {
+    return false;
+  }
+
+  assert(Call.getNumArgs() > 0);
+  SVal Arg = Call.getArgSVal(0);
+  if (Arg.isUndef())
+    return true; // Return true to model purity.
+
+  State = State->assume(Arg.castAs<DefinedOrUnknownSVal>(), true);
+  if (!State) {
+    C.addSink();
+    return true;
+  }
+
+  C.addTransition(State);
+  return true;
+}
+
+void ento::registerAssumeModeling(CheckerManager &Mgr) {
+  Mgr.registerChecker<AssumeModelingChecker>();
+}
+
+bool ento::shouldRegisterAssumeModeling(const CheckerManager &) { return true; }
diff --git a/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
index cfdd3c9faa360..b1a11442dec53 100644
--- a/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
@@ -23,7 +23,6 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/DynamicExtent.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/SVals.h"
 
 using namespace clang;
@@ -288,25 +287,6 @@ bool BuiltinFunctionChecker::evalCall(const CallEvent &Call,
     handleOverflowBuiltin(Call, C, BO_Add,
                           getOverflowBuiltinResultType(Call, C, BI));
     return true;
-  case Builtin::BI__builtin_assume:
-  case Builtin::BI__assume: {
-    assert (Call.getNumArgs() > 0);
-    SVal Arg = Call.getArgSVal(0);
-    if (Arg.isUndef())
-      return true; // Return true to model purity.
-
-    state = state->assume(Arg.castAs<DefinedOrUnknownSVal>(), true);
-    // FIXME: do we want to warn here? Not right now. The most reports might
-    // come from infeasible paths, thus being false positives.
-    if (!state) {
-      C.generateSink(C.getState(), C.getPredecessor());
-      return true;
-    }
-
-    C.addTransition(state);
-    return true;
-  }
-
   case Builtin::BI__builtin_unpredictable:
   case Builtin::BI__builtin_expect:
   case Builtin::BI__builtin_expect_with_probability:
diff --git a/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt b/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
index 5910043440987..878b48197cd58 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
+++ b/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
@@ -8,6 +8,7 @@ add_clang_library(clangStaticAnalyzerCheckers
   AnalysisOrderChecker.cpp
   AnalyzerStatsChecker.cpp
   ArrayBoundChecker.cpp
+  AssumeModeling.cpp
   BasicObjCFoundationChecks.cpp
   BitwiseShiftChecker.cpp
   BlockInCriticalSectionChecker.cpp
@@ -128,14 +129,14 @@ add_clang_library(clangStaticAnalyzerCheckers
   VLASizeChecker.cpp
   ValistChecker.cpp
   VirtualCallChecker.cpp
-  WebKit/RawPtrRefMemberChecker.cpp
   WebKit/ASTUtils.cpp
   WebKit/MemoryUnsafeCastChecker.cpp
   WebKit/PtrTypesSemantics.cpp
   WebKit/RefCntblBaseVirtualDtorChecker.cpp
   WebKit/RawPtrRefCallArgsChecker.cpp
-  WebKit/UncountedLambdaCapturesChecker.cpp
+  WebKit/RawPtrRefLambdaCapturesChecker.cpp
   WebKit/RawPtrRefLocalVarsChecker.cpp
+  WebKit/RawPtrRefMemberChecker.cpp
 
   LINK_LIBS
   clangAST
diff --git a/clang/lib/StaticAnalyzer/Checkers/ExprInspectionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ExprInspectionChecker.cpp
index 0b6c5163a1637..6035e2d34c2b3 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ExprInspectionChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ExprInspectionChecker.cpp
@@ -430,9 +430,8 @@ void ExprInspectionChecker::analyzerHashDump(const CallExpr *CE,
   const LangOptions &Opts = C.getLangOpts();
   const SourceManager &SM = C.getSourceManager();
   FullSourceLoc FL(CE->getArg(0)->getBeginLoc(), SM);
-  std::string HashContent =
-      getIssueString(FL, getCheckerName().getName(), "Category",
-                     C.getLocationContext()->getDecl(), Opts);
+  std::string HashContent = getIssueString(
+      FL, getName(), "Category", C.getLocationContext()->getDecl(), Opts);
 
   reportBug(HashContent, C);
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
index 0d1213ddf8b01..1c4293c30abdb 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
@@ -3885,7 +3885,7 @@ void MallocChecker::printState(raw_ostream &Out, ProgramStateRef State,
       Out << " : ";
       Data.dump(Out);
       if (CheckKind)
-        Out << " (" << CheckNames[*CheckKind].getName() << ")";
+        Out << " (" << CheckNames[*CheckKind] << ")";
       Out << NL;
     }
   }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ReturnValueChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ReturnValueChecker.cpp
index 3da571adfa44c..1e114149debf5 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ReturnValueChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ReturnValueChecker.cpp
@@ -60,7 +60,7 @@ class ReturnValueChecker : public Checker<check::PostCall> {
 };
 } // namespace
 
-static std::string getName(const CallEvent &Call) {
+static std::string getFunctionName(const CallEvent &Call) {
   std::string Name;
   if (const auto *MD = dyn_cast<CXXMethodDecl>(Call.getDecl()))
     if (const CXXRecordDecl *RD = MD->getParent())
@@ -84,7 +84,7 @@ void ReturnValueChecker::checkPostCall(const CallEvent &Call,
   if (ProgramStateRef StTrue = State->assume(*ReturnV, true)) {
     // The return value can be true, so transition to a state where it's true.
     std::string Msg =
-        formatv("'{0}' returns true (by convention)", getName(Call));
+        formatv("'{0}' returns true (by convention)", getFunctionName(Call));
     C.addTransition(StTrue, C.getNoteTag(Msg, /*IsPrunable=*/true));
     return;
   }
@@ -94,7 +94,7 @@ void ReturnValueChecker::checkPostCall(const CallEvent &Call,
   // Note that this checker is 'hidden' so it cannot produce a bug report.
   std::string Msg = formatv("'{0}' returned false, breaking the convention "
                             "that it always returns true",
-                            getName(Call));
+                            getFunctionName(Call));
   C.addTransition(State, C.getNoteTag(Msg, /*IsPrunable=*/true));
 }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
index 356d63e3e8b80..fef19b4547555 100644
--- a/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
@@ -1797,7 +1797,8 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
   auto IsNull = [&](ArgNo ArgN) {
     return std::make_shared<NotNullConstraint>(ArgN, false);
   };
-  auto NotNullBuffer = [&](ArgNo ArgN, ArgNo SizeArg1N, ArgNo SizeArg2N) {
+  auto NotNullBuffer = [&](ArgNo ArgN, ArgNo SizeArg1N,
+                           std::optional<ArgNo> SizeArg2N = std::nullopt) {
     return std::make_shared<NotNullBufferConstraint>(ArgN, SizeArg1N,
                                                      SizeArg2N);
   };
@@ -3365,7 +3366,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         Summary(NoEvalCall)
             .Case(ReturnsZero, ErrnoMustNotBeChecked, GenericSuccessMsg)
             .Case(ReturnsMinusOne, ErrnoNEZeroIrrelevant, GenericFailureMsg)
-            .ArgConstraint(NotNull(ArgNo(3)))
+            .ArgConstraint(NotNullBuffer(ArgNo(3), ArgNo(4)))
             .ArgConstraint(
                 BufferSize(/*Buffer=*/ArgNo(3), /*BufSize=*/ArgNo(4)))
             .ArgConstraint(
diff --git a/clang/lib/StaticAnalyzer/Checkers/ValistChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ValistChecker.cpp
index 28320f46f237a..bd2f88c7b1bcc 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ValistChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ValistChecker.cpp
@@ -272,12 +272,12 @@ void ValistChecker::reportLeakedVALists(const RegionVector &LeakedVALists,
     if (!BT_leakedvalist) {
       // FIXME: maybe creating a new check name for this type of bug is a better
       // solution.
-      BT_leakedvalist.reset(
-          new BugType(CheckNames[CK_Unterminated].getName().empty()
-                          ? CheckNames[CK_Uninitialized]
-                          : CheckNames[CK_Unterminated],
-                      "Leaked va_list", categories::MemoryError,
-                      /*SuppressOnSink=*/true));
+      BT_leakedvalist.reset(new BugType(
+          static_cast<StringRef>(CheckNames[CK_Unterminated]).empty()
+              ? CheckNames[CK_Uninitialized]
+              : CheckNames[CK_Unterminated],
+          "Leaked va_list", categories::MemoryError,
+          /*SuppressOnSink=*/true));
     }
 
     const ExplodedNode *StartNode = getStartCallSite(N, Reg);
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
index 58020ec4e084d..5e67cb29d08e4 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/ASTUtils.cpp
@@ -43,6 +43,10 @@ bool tryToFindPtrOrigin(
         break;
       }
     }
+    if (auto *TempExpr = dyn_cast<CXXUnresolvedConstructExpr>(E)) {
+      if (isSafePtrType(TempExpr->getTypeAsWritten()))
+        return callback(TempExpr, true);
+    }
     if (auto *POE = dyn_cast<PseudoObjectExpr>(E)) {
       if (auto *RF = POE->getResultExpr()) {
         E = RF;
@@ -87,9 +91,17 @@ bool tryToFindPtrOrigin(
       }
 
       if (auto *operatorCall = dyn_cast<CXXOperatorCallExpr>(E)) {
-        if (operatorCall->getNumArgs() == 1) {
-          E = operatorCall->getArg(0);
-          continue;
+        if (auto *Callee = operatorCall->getDirectCallee()) {
+          auto ClsName = safeGetName(Callee->getParent());
+          if (isRefType(ClsName) || isCheckedPtr(ClsName) ||
+              isRetainPtr(ClsName) || ClsName == "unique_ptr" ||
+              ClsName == "UniqueRef" || ClsName == "WeakPtr" ||
+              ClsName == "WeakRef") {
+            if (operatorCall->getNumArgs() == 1) {
+              E = operatorCall->getArg(0);
+              continue;
+            }
+          }
         }
       }
 
@@ -211,7 +223,7 @@ bool EnsureFunctionAnalysis::isACallToEnsureFn(const clang::Expr *E) const {
   if (!Callee)
     return false;
   auto *Body = Callee->getBody();
-  if (!Body)
+  if (!Body || Callee->isVirtualAsWritten())
     return false;
   auto [CacheIt, IsNew] = Cache.insert(std::make_pair(Callee, false));
   if (IsNew)
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
index 7899b19854806..419d9c2325412 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
@@ -162,13 +162,14 @@ static bool isPtrOfType(const clang::QualType T, Predicate Pred) {
       type = elaboratedT->desugar();
       continue;
     }
-    auto *SpecialT = type->getAs<TemplateSpecializationType>();
-    if (!SpecialT)
-      return false;
-    auto *Decl = SpecialT->getTemplateName().getAsTemplateDecl();
-    if (!Decl)
-      return false;
-    return Pred(Decl->getNameAsString());
+    if (auto *SpecialT = type->getAs<TemplateSpecializationType>()) {
+      auto *Decl = SpecialT->getTemplateName().getAsTemplateDecl();
+      return Decl && Pred(Decl->getNameAsString());
+    } else if (auto *DTS = type->getAs<DeducedTemplateSpecializationType>()) {
+      auto *Decl = DTS->getTemplateName().getAsTemplateDecl();
+      return Decl && Pred(Decl->getNameAsString());
+    } else
+      break;
   }
   return false;
 }
@@ -479,6 +480,10 @@ class TrivialFunctionAnalysisVisitor
   TrivialFunctionAnalysisVisitor(CacheTy &Cache) : Cache(Cache) {}
 
   bool IsFunctionTrivial(const Decl *D) {
+    if (auto *FnDecl = dyn_cast<FunctionDecl>(D)) {
+      if (FnDecl->isVirtualAsWritten())
+        return false;
+    }
     return WithCachedResult(D, [&]() {
       if (auto *CtorDecl = dyn_cast<CXXConstructorDecl>(D)) {
         for (auto *CtorInit : CtorDecl->inits()) {
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h
index fcc1a41dba78b..323d473665888 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h
@@ -131,6 +131,9 @@ bool isRefType(const std::string &Name);
 /// \returns true if \p Name is CheckedRef or CheckedPtr, false if not.
 bool isCheckedPtr(const std::string &Name);
 
+/// \returns true if \p Name is RetainPtr or its variant, false if not.
+bool isRetainPtr(const std::string &Name);
+
 /// \returns true if \p M is getter of a ref-counted class, false if not.
 std::optional<bool> isGetterOfSafePtr(const clang::CXXMethodDecl *Method);
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefCallArgsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefCallArgsChecker.cpp
index d633fbcbd798b..9d07c65da88af 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefCallArgsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefCallArgsChecker.cpp
@@ -172,7 +172,7 @@ class RawPtrRefCallArgsChecker
     if (BR->getSourceManager().isInSystemHeader(CE->getExprLoc()))
       return true;
 
-    if (Callee && TFA.isTrivial(Callee))
+    if (Callee && TFA.isTrivial(Callee) && !Callee->isVirtualAsWritten())
       return true;
 
     if (CE->getNumArgs() == 0)
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLambdaCapturesChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefLambdaCapturesChecker.cpp
similarity index 76%
rename from clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLambdaCapturesChecker.cpp
rename to clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefLambdaCapturesChecker.cpp
index 506442f352288..4a297a6c0ab91 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLambdaCapturesChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefLambdaCapturesChecker.cpp
@@ -21,15 +21,23 @@ using namespace clang;
 using namespace ento;
 
 namespace {
-class UncountedLambdaCapturesChecker
+class RawPtrRefLambdaCapturesChecker
     : public Checker<check::ASTDecl<TranslationUnitDecl>> {
 private:
-  BugType Bug{this, "Lambda capture of uncounted variable",
-              "WebKit coding guidelines"};
+  BugType Bug;
   mutable BugReporter *BR = nullptr;
   TrivialFunctionAnalysis TFA;
 
+protected:
+  mutable std::optional<RetainTypeChecker> RTC;
+
 public:
+  RawPtrRefLambdaCapturesChecker(const char *description)
+      : Bug(this, description, "WebKit coding guidelines") {}
+
+  virtual std::optional<bool> isUnsafePtr(QualType) const = 0;
+  virtual const char *ptrKind(QualType QT) const = 0;
+
   void checkASTDecl(const TranslationUnitDecl *TUD, AnalysisManager &MGR,
                     BugReporter &BRArg) const {
     BR = &BRArg;
@@ -38,7 +46,7 @@ class UncountedLambdaCapturesChecker
     // visit template instantiations or lambda classes. We
     // want to visit those, so we make our own RecursiveASTVisitor.
     struct LocalVisitor : DynamicRecursiveASTVisitor {
-      const UncountedLambdaCapturesChecker *Checker;
+      const RawPtrRefLambdaCapturesChecker *Checker;
       llvm::DenseSet<const DeclRefExpr *> DeclRefExprsToIgnore;
       llvm::DenseSet<const LambdaExpr *> LambdasToIgnore;
       llvm::DenseSet<const ValueDecl *> ProtectedThisDecls;
@@ -46,7 +54,7 @@ class UncountedLambdaCapturesChecker
 
       QualType ClsType;
 
-      explicit LocalVisitor(const UncountedLambdaCapturesChecker *Checker)
+      explicit LocalVisitor(const RawPtrRefLambdaCapturesChecker *Checker)
           : Checker(Checker) {
         assert(Checker);
         ShouldVisitTemplateInstantiations = true;
@@ -60,16 +68,23 @@ class UncountedLambdaCapturesChecker
         return DynamicRecursiveASTVisitor::TraverseCXXMethodDecl(CXXMD);
       }
 
+      bool VisitTypedefDecl(TypedefDecl *TD) override {
+        if (Checker->RTC)
+          Checker->RTC->visitTypedef(TD);
+        return true;
+      }
+
       bool shouldCheckThis() {
         auto result =
-            !ClsType.isNull() ? isUnsafePtr(ClsType, false) : std::nullopt;
+            !ClsType.isNull() ? Checker->isUnsafePtr(ClsType) : std::nullopt;
         return result && *result;
       }
 
       bool VisitLambdaExpr(LambdaExpr *L) override {
         if (LambdasToIgnore.contains(L))
           return true;
-        Checker->visitLambdaExpr(L, shouldCheckThis() && !hasProtectedThis(L));
+        Checker->visitLambdaExpr(L, shouldCheckThis() && !hasProtectedThis(L),
+                                 ClsType);
         return true;
       }
 
@@ -97,7 +112,8 @@ class UncountedLambdaCapturesChecker
         if (!L)
           return true;
         LambdasToIgnore.insert(L);
-        Checker->visitLambdaExpr(L, shouldCheckThis() && !hasProtectedThis(L));
+        Checker->visitLambdaExpr(L, shouldCheckThis() && !hasProtectedThis(L),
+                                 ClsType);
         return true;
       }
 
@@ -122,8 +138,8 @@ class UncountedLambdaCapturesChecker
             if (auto *L = findLambdaInArg(Arg)) {
               LambdasToIgnore.insert(L);
               if (!Param->hasAttr<NoEscapeAttr>())
-                Checker->visitLambdaExpr(L, shouldCheckThis() &&
-                                                !hasProtectedThis(L));
+                Checker->visitLambdaExpr(
+                    L, shouldCheckThis() && !hasProtectedThis(L), ClsType);
             }
             ++ArgIndex;
           }
@@ -143,8 +159,8 @@ class UncountedLambdaCapturesChecker
             if (auto *L = findLambdaInArg(Arg)) {
               LambdasToIgnore.insert(L);
               if (!Param->hasAttr<NoEscapeAttr>() && !TreatAllArgsAsNoEscape)
-                Checker->visitLambdaExpr(L, shouldCheckThis() &&
-                                                !hasProtectedThis(L));
+                Checker->visitLambdaExpr(
+                    L, shouldCheckThis() && !hasProtectedThis(L), ClsType);
             }
             ++ArgIndex;
           }
@@ -169,14 +185,22 @@ class UncountedLambdaCapturesChecker
         auto *CtorArg = CE->getArg(0)->IgnoreParenCasts();
         if (!CtorArg)
           return nullptr;
-        if (auto *Lambda = dyn_cast<LambdaExpr>(CtorArg)) {
+        auto *InnerCE = dyn_cast_or_null<CXXConstructExpr>(CtorArg);
+        if (InnerCE && InnerCE->getNumArgs())
+          CtorArg = InnerCE->getArg(0)->IgnoreParenCasts();
+        auto updateIgnoreList = [&] {
           ConstructToIgnore.insert(CE);
+          if (InnerCE)
+            ConstructToIgnore.insert(InnerCE);
+        };
+        if (auto *Lambda = dyn_cast<LambdaExpr>(CtorArg)) {
+          updateIgnoreList();
           return Lambda;
         }
         if (auto *TempExpr = dyn_cast<CXXBindTemporaryExpr>(CtorArg)) {
           E = TempExpr->getSubExpr()->IgnoreParenCasts();
           if (auto *Lambda = dyn_cast<LambdaExpr>(E)) {
-            ConstructToIgnore.insert(CE);
+            updateIgnoreList();
             return Lambda;
           }
         }
@@ -189,10 +213,14 @@ class UncountedLambdaCapturesChecker
         auto *Init = VD->getInit();
         if (!Init)
           return nullptr;
+        if (auto *Lambda = dyn_cast<LambdaExpr>(Init)) {
+          updateIgnoreList();
+          return Lambda;
+        }
         TempExpr = dyn_cast<CXXBindTemporaryExpr>(Init->IgnoreParenCasts());
         if (!TempExpr)
           return nullptr;
-        ConstructToIgnore.insert(CE);
+        updateIgnoreList();
         return dyn_cast_or_null<LambdaExpr>(TempExpr->getSubExpr());
       }
 
@@ -226,7 +254,7 @@ class UncountedLambdaCapturesChecker
         DeclRefExprsToIgnore.insert(ArgRef);
         LambdasToIgnore.insert(L);
         Checker->visitLambdaExpr(L, shouldCheckThis() && !hasProtectedThis(L),
-                                 /* ignoreParamVarDecl */ true);
+                                 ClsType, /* ignoreParamVarDecl */ true);
       }
 
       bool hasProtectedThis(LambdaExpr *L) {
@@ -293,10 +321,12 @@ class UncountedLambdaCapturesChecker
     };
 
     LocalVisitor visitor(this);
+    if (RTC)
+      RTC->visitTranslationUnitDecl(TUD);
     visitor.TraverseDecl(const_cast<TranslationUnitDecl *>(TUD));
   }
 
-  void visitLambdaExpr(LambdaExpr *L, bool shouldCheckThis,
+  void visitLambdaExpr(LambdaExpr *L, bool shouldCheckThis, const QualType T,
                        bool ignoreParamVarDecl = false) const {
     if (TFA.isTrivial(L->getBody()))
       return;
@@ -306,13 +336,13 @@ class UncountedLambdaCapturesChecker
         if (ignoreParamVarDecl && isa<ParmVarDecl>(CapturedVar))
           continue;
         QualType CapturedVarQualType = CapturedVar->getType();
-        auto IsUncountedPtr = isUnsafePtr(CapturedVar->getType(), false);
+        auto IsUncountedPtr = isUnsafePtr(CapturedVar->getType());
         if (IsUncountedPtr && *IsUncountedPtr)
           reportBug(C, CapturedVar, CapturedVarQualType);
       } else if (C.capturesThis() && shouldCheckThis) {
         if (ignoreParamVarDecl) // this is always a parameter to this function.
           continue;
-        reportBugOnThisPtr(C);
+        reportBugOnThisPtr(C, T);
       }
     }
   }
@@ -321,6 +351,9 @@ class UncountedLambdaCapturesChecker
                  const QualType T) const {
     assert(CapturedVar);
 
+    if (isa<ImplicitParamDecl>(CapturedVar) && !Capture.getLocation().isValid())
+      return; // Ignore implicit captruing of self.
+
     SmallString<100> Buf;
     llvm::raw_svector_ostream Os(Buf);
 
@@ -329,22 +362,22 @@ class UncountedLambdaCapturesChecker
     } else {
       Os << "Implicitly captured ";
     }
-    if (T->isPointerType()) {
+    if (isa<PointerType>(T) || isa<ObjCObjectPointerType>(T)) {
       Os << "raw-pointer ";
     } else {
-      assert(T->isReferenceType());
       Os << "reference ";
     }
 
-    printQuotedQualifiedName(Os, Capture.getCapturedVar());
-    Os << " to ref-counted type or CheckedPtr-capable type is unsafe.";
+    printQuotedQualifiedName(Os, CapturedVar);
+    Os << " to " << ptrKind(T) << " type is unsafe.";
 
     PathDiagnosticLocation BSLoc(Capture.getLocation(), BR->getSourceManager());
     auto Report = std::make_unique<BasicBugReport>(Bug, Os.str(), BSLoc);
     BR->emitReport(std::move(Report));
   }
 
-  void reportBugOnThisPtr(const LambdaCapture &Capture) const {
+  void reportBugOnThisPtr(const LambdaCapture &Capture,
+                          const QualType T) const {
     SmallString<100> Buf;
     llvm::raw_svector_ostream Os(Buf);
 
@@ -354,14 +387,54 @@ class UncountedLambdaCapturesChecker
       Os << "Implicitly captured ";
     }
 
-    Os << "raw-pointer 'this' to ref-counted type or CheckedPtr-capable type "
-          "is unsafe.";
+    Os << "raw-pointer 'this' to " << ptrKind(T) << " type is unsafe.";
 
     PathDiagnosticLocation BSLoc(Capture.getLocation(), BR->getSourceManager());
     auto Report = std::make_unique<BasicBugReport>(Bug, Os.str(), BSLoc);
     BR->emitReport(std::move(Report));
   }
 };
+
+class UncountedLambdaCapturesChecker : public RawPtrRefLambdaCapturesChecker {
+public:
+  UncountedLambdaCapturesChecker()
+      : RawPtrRefLambdaCapturesChecker("Lambda capture of uncounted or "
+                                       "unchecked variable") {}
+
+  std::optional<bool> isUnsafePtr(QualType QT) const final {
+    auto result1 = isUncountedPtr(QT);
+    auto result2 = isUncheckedPtr(QT);
+    if (result1 && *result1)
+      return true;
+    if (result2 && *result2)
+      return true;
+    if (result1)
+      return *result1;
+    return result2;
+  }
+
+  const char *ptrKind(QualType QT) const final {
+    if (isUncounted(QT))
+      return "uncounted";
+    return "unchecked";
+  }
+};
+
+class UnretainedLambdaCapturesChecker : public RawPtrRefLambdaCapturesChecker {
+public:
+  UnretainedLambdaCapturesChecker()
+      : RawPtrRefLambdaCapturesChecker("Lambda capture of unretained "
+                                       "variables") {
+    RTC = RetainTypeChecker();
+  }
+
+  std::optional<bool> isUnsafePtr(QualType QT) const final {
+    return RTC->isUnretained(QT);
+  }
+
+  const char *ptrKind(QualType QT) const final { return "unretained"; }
+};
+
 } // namespace
 
 void ento::registerUncountedLambdaCapturesChecker(CheckerManager &Mgr) {
@@ -372,3 +445,12 @@ bool ento::shouldRegisterUncountedLambdaCapturesChecker(
     const CheckerManager &mgr) {
   return true;
 }
+
+void ento::registerUnretainedLambdaCapturesChecker(CheckerManager &Mgr) {
+  Mgr.registerChecker<UnretainedLambdaCapturesChecker>();
+}
+
+bool ento::shouldRegisterUnretainedLambdaCapturesChecker(
+    const CheckerManager &mgr) {
+  return true;
+}
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefMemberChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefMemberChecker.cpp
index 963f59831c8ed..593e6e3663639 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefMemberChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefMemberChecker.cpp
@@ -31,12 +31,16 @@ class RawPtrRefMemberChecker
   BugType Bug;
   mutable BugReporter *BR;
 
+protected:
+  mutable std::optional<RetainTypeChecker> RTC;
+
 public:
   RawPtrRefMemberChecker(const char *description)
       : Bug(this, description, "WebKit coding guidelines") {}
 
   virtual std::optional<bool>
-  isPtrCompatible(const clang::CXXRecordDecl *) const = 0;
+  isPtrCompatible(const clang::QualType,
+                  const clang::CXXRecordDecl *R) const = 0;
   virtual bool isPtrCls(const clang::CXXRecordDecl *) const = 0;
   virtual const char *typeName() const = 0;
   virtual const char *invariant() const = 0;
@@ -57,6 +61,12 @@ class RawPtrRefMemberChecker
         ShouldVisitImplicitCode = false;
       }
 
+      bool VisitTypedefDecl(const TypedefDecl *TD) override {
+        if (Checker->RTC)
+          Checker->RTC->visitTypedef(TD);
+        return true;
+      }
+
       bool VisitRecordDecl(const RecordDecl *RD) override {
         Checker->visitRecordDecl(RD);
         return true;
@@ -69,6 +79,8 @@ class RawPtrRefMemberChecker
     };
 
     LocalVisitor visitor(this);
+    if (RTC)
+      RTC->visitTranslationUnitDecl(TUD);
     visitor.TraverseDecl(TUD);
   }
 
@@ -77,16 +89,22 @@ class RawPtrRefMemberChecker
       return;
 
     for (auto *Member : RD->fields()) {
-      const Type *MemberType = Member->getType().getTypePtrOrNull();
+      auto QT = Member->getType();
+      const Type *MemberType = QT.getTypePtrOrNull();
       if (!MemberType)
         continue;
 
       if (auto *MemberCXXRD = MemberType->getPointeeCXXRecordDecl()) {
-        // If we don't see the definition we just don't know.
-        if (MemberCXXRD->hasDefinition()) {
-          std::optional<bool> isRCAble = isPtrCompatible(MemberCXXRD);
-          if (isRCAble && *isRCAble)
-            reportBug(Member, MemberType, MemberCXXRD, RD);
+        std::optional<bool> IsCompatible = isPtrCompatible(QT, MemberCXXRD);
+        if (IsCompatible && *IsCompatible)
+          reportBug(Member, MemberType, MemberCXXRD, RD);
+      } else {
+        std::optional<bool> IsCompatible = isPtrCompatible(QT, nullptr);
+        auto *PointeeType = MemberType->getPointeeType().getTypePtrOrNull();
+        if (IsCompatible && *IsCompatible) {
+          auto *Desugared = PointeeType->getUnqualifiedDesugaredType();
+          if (auto *ObjCType = dyn_cast_or_null<ObjCInterfaceType>(Desugared))
+            reportBug(Member, MemberType, ObjCType->getDecl(), RD);
         }
       }
     }
@@ -107,11 +125,12 @@ class RawPtrRefMemberChecker
 
   void visitIvarDecl(const ObjCContainerDecl *CD,
                      const ObjCIvarDecl *Ivar) const {
-    const Type *IvarType = Ivar->getType().getTypePtrOrNull();
+    auto QT = Ivar->getType();
+    const Type *IvarType = QT.getTypePtrOrNull();
     if (!IvarType)
       return;
     if (auto *IvarCXXRD = IvarType->getPointeeCXXRecordDecl()) {
-      std::optional<bool> IsCompatible = isPtrCompatible(IvarCXXRD);
+      std::optional<bool> IsCompatible = isPtrCompatible(QT, IvarCXXRD);
       if (IsCompatible && *IsCompatible)
         reportBug(Ivar, IvarType, IvarCXXRD, CD);
     }
@@ -151,13 +170,13 @@ class RawPtrRefMemberChecker
     return false;
   }
 
-  template <typename DeclType, typename ParentDeclType>
+  template <typename DeclType, typename PointeeType, typename ParentDeclType>
   void reportBug(const DeclType *Member, const Type *MemberType,
-                 const CXXRecordDecl *MemberCXXRD,
+                 const PointeeType *Pointee,
                  const ParentDeclType *ClassCXXRD) const {
     assert(Member);
     assert(MemberType);
-    assert(MemberCXXRD);
+    assert(Pointee);
 
     SmallString<100> Buf;
     llvm::raw_svector_ostream Os(Buf);
@@ -169,10 +188,13 @@ class RawPtrRefMemberChecker
     printQuotedName(Os, Member);
     Os << " in ";
     printQuotedQualifiedName(Os, ClassCXXRD);
-    Os << " is a "
-       << (isa<PointerType>(MemberType) ? "raw pointer" : "reference") << " to "
-       << typeName() << " ";
-    printQuotedQualifiedName(Os, MemberCXXRD);
+    Os << " is a ";
+    if (printPointer(Os, MemberType) == PrintDeclKind::Pointer) {
+      auto Typedef = MemberType->getAs<TypedefType>();
+      assert(Typedef);
+      printQuotedQualifiedName(Os, Typedef->getDecl());
+    } else
+      printQuotedQualifiedName(Os, Pointee);
     Os << "; " << invariant() << ".";
 
     PathDiagnosticLocation BSLoc(Member->getSourceRange().getBegin(),
@@ -181,6 +203,15 @@ class RawPtrRefMemberChecker
     Report->addRange(Member->getSourceRange());
     BR->emitReport(std::move(Report));
   }
+
+  enum class PrintDeclKind { Pointee, Pointer };
+  virtual PrintDeclKind printPointer(llvm::raw_svector_ostream &Os,
+                                     const Type *T) const {
+    T = T->getUnqualifiedDesugaredType();
+    bool IsPtr = isa<PointerType>(T) || isa<ObjCObjectPointerType>(T);
+    Os << (IsPtr ? "raw pointer" : "reference") << " to " << typeName() << " ";
+    return PrintDeclKind::Pointee;
+  }
 };
 
 class NoUncountedMemberChecker final : public RawPtrRefMemberChecker {
@@ -190,8 +221,9 @@ class NoUncountedMemberChecker final : public RawPtrRefMemberChecker {
                                "reference-countable type") {}
 
   std::optional<bool>
-  isPtrCompatible(const clang::CXXRecordDecl *R) const final {
-    return isRefCountable(R);
+  isPtrCompatible(const clang::QualType,
+                  const clang::CXXRecordDecl *R) const final {
+    return R && isRefCountable(R);
   }
 
   bool isPtrCls(const clang::CXXRecordDecl *R) const final {
@@ -212,8 +244,9 @@ class NoUncheckedPtrMemberChecker final : public RawPtrRefMemberChecker {
                                "checked-pointer capable type") {}
 
   std::optional<bool>
-  isPtrCompatible(const clang::CXXRecordDecl *R) const final {
-    return isCheckedPtrCapable(R);
+  isPtrCompatible(const clang::QualType,
+                  const clang::CXXRecordDecl *R) const final {
+    return R && isCheckedPtrCapable(R);
   }
 
   bool isPtrCls(const clang::CXXRecordDecl *R) const final {
@@ -228,6 +261,40 @@ class NoUncheckedPtrMemberChecker final : public RawPtrRefMemberChecker {
   }
 };
 
+class NoUnretainedMemberChecker final : public RawPtrRefMemberChecker {
+public:
+  NoUnretainedMemberChecker()
+      : RawPtrRefMemberChecker("Member variable is a raw-pointer/reference to "
+                               "retainable type") {
+    RTC = RetainTypeChecker();
+  }
+
+  std::optional<bool>
+  isPtrCompatible(const clang::QualType QT,
+                  const clang::CXXRecordDecl *) const final {
+    return RTC->isUnretained(QT);
+  }
+
+  bool isPtrCls(const clang::CXXRecordDecl *R) const final {
+    return isRetainPtr(R);
+  }
+
+  const char *typeName() const final { return "retainable type"; }
+
+  const char *invariant() const final {
+    return "member variables must be a RetainPtr";
+  }
+
+  PrintDeclKind printPointer(llvm::raw_svector_ostream &Os,
+                             const Type *T) const final {
+    if (!isa<ObjCObjectPointerType>(T) && T->getAs<TypedefType>()) {
+      Os << typeName() << " ";
+      return PrintDeclKind::Pointer;
+    }
+    return RawPtrRefMemberChecker::printPointer(Os, T);
+  }
+};
+
 } // namespace
 
 void ento::registerNoUncountedMemberChecker(CheckerManager &Mgr) {
@@ -246,3 +313,11 @@ bool ento::shouldRegisterNoUncheckedPtrMemberChecker(
     const CheckerManager &Mgr) {
   return true;
 }
+
+void ento::registerNoUnretainedMemberChecker(CheckerManager &Mgr) {
+  Mgr.registerChecker<NoUnretainedMemberChecker>();
+}
+
+bool ento::shouldRegisterNoUnretainedMemberChecker(const CheckerManager &Mgr) {
+  return true;
+}
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
index 7101b1eb2c7f5..a4f9e092e8205 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
@@ -3418,8 +3418,8 @@ void BugReporter::EmitBasicReport(const Decl *DeclWithIssue,
                                   PathDiagnosticLocation Loc,
                                   ArrayRef<SourceRange> Ranges,
                                   ArrayRef<FixItHint> Fixits) {
-  EmitBasicReport(DeclWithIssue, Checker->getCheckerName(), Name, Category, Str,
-                  Loc, Ranges, Fixits);
+  EmitBasicReport(DeclWithIssue, Checker->getName(), Name, Category, Str, Loc,
+                  Ranges, Fixits);
 }
 
 void BugReporter::EmitBasicReport(const Decl *DeclWithIssue,
@@ -3442,8 +3442,8 @@ void BugReporter::EmitBasicReport(const Decl *DeclWithIssue,
 BugType *BugReporter::getBugTypeForName(CheckerNameRef CheckName,
                                         StringRef name, StringRef category) {
   SmallString<136> fullDesc;
-  llvm::raw_svector_ostream(fullDesc) << CheckName.getName() << ":" << name
-                                      << ":" << category;
+  llvm::raw_svector_ostream(fullDesc)
+      << CheckName << ":" << name << ":" << category;
   std::unique_ptr<BugType> &BT = StrBugTypes[fullDesc];
   if (!BT)
     BT = std::make_unique<BugType>(CheckName, name, category);
diff --git a/clang/lib/StaticAnalyzer/Core/Checker.cpp b/clang/lib/StaticAnalyzer/Core/Checker.cpp
index bc1c8964b3ee4..d93db6ddfc3bf 100644
--- a/clang/lib/StaticAnalyzer/Core/Checker.cpp
+++ b/clang/lib/StaticAnalyzer/Core/Checker.cpp
@@ -18,11 +18,9 @@ using namespace ento;
 
 int ImplicitNullDerefEvent::Tag;
 
-StringRef CheckerBase::getTagDescription() const {
-  return getCheckerName().getName();
-}
+StringRef CheckerBase::getTagDescription() const { return getName(); }
 
-CheckerNameRef CheckerBase::getCheckerName() const { return Name; }
+CheckerNameRef CheckerBase::getName() const { return Name; }
 
 CheckerProgramPointTag::CheckerProgramPointTag(StringRef CheckerName,
                                                StringRef Msg)
@@ -30,10 +28,10 @@ CheckerProgramPointTag::CheckerProgramPointTag(StringRef CheckerName,
 
 CheckerProgramPointTag::CheckerProgramPointTag(const CheckerBase *Checker,
                                                StringRef Msg)
-    : SimpleProgramPointTag(Checker->getCheckerName().getName(), Msg) {}
+    : SimpleProgramPointTag(Checker->getName(), Msg) {}
 
 raw_ostream& clang::ento::operator<<(raw_ostream &Out,
                                      const CheckerBase &Checker) {
-  Out << Checker.getCheckerName().getName();
+  Out << Checker.getName();
   return Out;
 }
diff --git a/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp b/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
index 53929d370e2fe..56b1e44acc1fd 100644
--- a/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
@@ -139,7 +139,7 @@ std::string checkerScopeName(StringRef Name, const CheckerBase *Checker) {
   if (!llvm::timeTraceProfilerEnabled())
     return "";
   StringRef CheckerName =
-      Checker ? Checker->getCheckerName().getName() : "<unknown>";
+      Checker ? static_cast<StringRef>(Checker->getName()) : "<unknown>";
   return (Name + ":" + CheckerName).str();
 }
 
@@ -721,13 +721,13 @@ void CheckerManager::runCheckersForEvalCall(ExplodedNodeSet &Dst,
             "The '{0}' call has been already evaluated by the {1} checker, "
             "while the {2} checker also tried to evaluate the same call. At "
             "most one checker supposed to evaluate a call.",
-            toString(Call), evaluatorChecker->getName(),
-            EvalCallChecker.Checker->getCheckerName());
+            toString(Call), evaluatorChecker,
+            EvalCallChecker.Checker->getName());
         llvm_unreachable(AssertionMessage.c_str());
       }
 #endif
       if (evaluated) {
-        evaluatorChecker = EvalCallChecker.Checker->getCheckerName();
+        evaluatorChecker = EvalCallChecker.Checker->getName();
         Dst.insert(checkDst);
 #ifdef NDEBUG
         break; // on release don't check that no other checker also evals.
@@ -798,9 +798,8 @@ void CheckerManager::runCheckersForPrintStateJson(raw_ostream &Out,
     if (TempBuf.empty())
       continue;
 
-    Indent(Out, Space, IsDot)
-        << "{ \"checker\": \"" << CT.second->getCheckerName().getName()
-        << "\", \"messages\": [" << NL;
+    Indent(Out, Space, IsDot) << "{ \"checker\": \"" << CT.second->getName()
+                              << "\", \"messages\": [" << NL;
     Indent(Out, InnerSpace, IsDot)
         << '\"' << TempBuf.str().trim() << '\"' << NL;
     Indent(Out, Space, IsDot) << "]}";
diff --git a/clang/test/AST/ByteCode/builtin-constant-p.cpp b/clang/test/AST/ByteCode/builtin-constant-p.cpp
index 62899b60064c2..ed9e606ed16aa 100644
--- a/clang/test/AST/ByteCode/builtin-constant-p.cpp
+++ b/clang/test/AST/ByteCode/builtin-constant-p.cpp
@@ -1,6 +1,7 @@
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -verify=expected,both %s
-// RUN: %clang_cc1 -verify=ref,both %s
+// RUN: %clang_cc1 -std=c++20 -fexperimental-new-constant-interpreter -verify=expected,both %s
+// RUN: %clang_cc1 -std=c++20 -verify=ref,both %s
 
+using intptr_t = __INTPTR_TYPE__;
 
 static_assert(__builtin_constant_p(12), "");
 static_assert(__builtin_constant_p(1.0), "");
@@ -18,3 +19,105 @@ constexpr int foo(int &a) {
   return __builtin_constant_p(a);
 }
 static_assert(!foo(z));
+
+static_assert(__builtin_constant_p(__builtin_constant_p(1)));
+
+constexpr bool nested(int& a) {
+  return __builtin_constant_p(__builtin_constant_p(a));
+}
+static_assert(nested(z));
+
+constexpr bool Local() {
+  int z = 10;
+  return __builtin_constant_p(z);
+}
+static_assert(Local());
+
+constexpr bool Local2() {
+  int z = 10;
+  return __builtin_constant_p(&z);
+}
+static_assert(!Local2());
+
+constexpr bool Parameter(int a) {
+  return __builtin_constant_p(a);
+}
+static_assert(Parameter(10));
+
+constexpr bool InvalidLocal() {
+  int *z;
+  {
+    int b = 10;
+    z = &b;
+  }
+  return __builtin_constant_p(z);
+}
+static_assert(!InvalidLocal());
+
+template<typename T> constexpr bool bcp(T t) {
+  return __builtin_constant_p(t);
+}
+
+constexpr intptr_t ptr_to_int(const void *p) {
+  return __builtin_constant_p(1) ? (intptr_t)p : (intptr_t)p;
+}
+
+static_assert(bcp(ptr_to_int("foo")));
+
+constexpr bool AndFold(const int &a, const int &b) {
+  return __builtin_constant_p(a && b);
+}
+
+static_assert(AndFold(10, 20));
+static_assert(!AndFold(z, 10));
+static_assert(!AndFold(10, z));
+
+
+struct F {
+  int a;
+};
+
+constexpr F f{12};
+static_assert(__builtin_constant_p(f.a));
+
+constexpr bool Member() {
+  F f;
+  return __builtin_constant_p(f.a);
+}
+static_assert(!Member());
+
+constexpr bool Discard() {
+  (void)__builtin_constant_p(10);
+  return true;
+}
+static_assert(Discard());
+
+static_assert(__builtin_constant_p((int*)123));
+
+constexpr void func() {}
+static_assert(!__builtin_constant_p(func));
+
+/// This is from SemaCXX/builtin-constant-p and GCC agrees with the bytecode interpreter.
+constexpr int mutate1() {
+  int n = 1;
+  int m = __builtin_constant_p(++n);
+  return n * 10 + m;
+}
+static_assert(mutate1() == 21); // ref-error {{static assertion failed}} \
+                                // ref-note {{evaluates to '10 == 21'}}
+
+/// Similar for this. GCC agrees with the bytecode interpreter.
+constexpr int mutate_param(bool mutate, int &param) {
+  mutate = mutate; // Mutation of internal state is OK
+  if (mutate)
+    ++param;
+  return param;
+}
+constexpr int mutate6(bool mutate) {
+  int n = 1;
+  int m = __builtin_constant_p(mutate_param(mutate, n));
+  return n * 10 + m;
+}
+static_assert(mutate6(false) == 11);
+static_assert(mutate6(true) == 21); // ref-error {{static assertion failed}} \
+                                    // ref-note {{evaluates to '10 == 21'}}
diff --git a/clang/test/AST/ByteCode/builtin-functions.cpp b/clang/test/AST/ByteCode/builtin-functions.cpp
index 75380f99901a2..29812553af9f0 100644
--- a/clang/test/AST/ByteCode/builtin-functions.cpp
+++ b/clang/test/AST/ByteCode/builtin-functions.cpp
@@ -18,6 +18,8 @@
 extern "C" {
   typedef decltype(sizeof(int)) size_t;
   extern size_t wcslen(const wchar_t *p);
+  extern void *memchr(const void *s, int c, size_t n);
+  extern char *strchr(const char *s, int c);
 }
 
 namespace strcmp {
@@ -1350,4 +1352,135 @@ namespace Memcmp {
   static_assert(__builtin_wmemcmp(L"abab\0banana", L"abab\0canada", 7) == -1);
   static_assert(__builtin_wmemcmp(L"abab\0banana", L"abab\0canada", 6) == -1);
   static_assert(__builtin_wmemcmp(L"abab\0banana", L"abab\0canada", 5) == 0);
+
+#if __cplusplus >= 202002L
+  constexpr bool f() {
+    char *c = new char[12];
+    c[0] = 'b';
+
+    char n = 'a';
+    bool b = __builtin_memcmp(c, &n, 1) == 0;
+
+    delete[] c;
+    return !b;
+  }
+  static_assert(f());
+#endif
+
+}
+
+namespace Memchr {
+  constexpr const char *kStr = "abca\xff\0d";
+  constexpr char kFoo[] = {'f', 'o', 'o'};
+
+  static_assert(__builtin_memchr(kStr, 'a', 0) == nullptr);
+  static_assert(__builtin_memchr(kStr, 'a', 1) == kStr);
+  static_assert(__builtin_memchr(kStr, '\0', 5) == nullptr);
+  static_assert(__builtin_memchr(kStr, '\0', 6) == kStr + 5);
+  static_assert(__builtin_memchr(kStr, '\xff', 8) == kStr + 4);
+  static_assert(__builtin_memchr(kStr, '\xff' + 256, 8) == kStr + 4);
+  static_assert(__builtin_memchr(kStr, '\xff' - 256, 8) == kStr + 4);
+  static_assert(__builtin_memchr(kFoo, 'x', 3) == nullptr);
+  static_assert(__builtin_memchr(kFoo, 'x', 4) == nullptr); // both-error {{not an integral constant}} \
+                                                            // both-note {{dereferenced one-past-the-end}}
+  static_assert(__builtin_memchr(nullptr, 'x', 3) == nullptr); // both-error {{not an integral constant}} \
+                                                               // both-note {{dereferenced null}}
+  static_assert(__builtin_memchr(nullptr, 'x', 0) == nullptr);
+
+
+#if defined(CHAR8_T)
+  constexpr const char8_t *kU8Str = u8"abca\xff\0d";
+  constexpr char8_t kU8Foo[] = {u8'f', u8'o', u8'o'};
+  static_assert(__builtin_memchr(kU8Str, u8'a', 0) == nullptr);
+  static_assert(__builtin_memchr(kU8Str, u8'a', 1) == kU8Str);
+  static_assert(__builtin_memchr(kU8Str, u8'\0', 5) == nullptr);
+  static_assert(__builtin_memchr(kU8Str, u8'\0', 6) == kU8Str + 5);
+  static_assert(__builtin_memchr(kU8Str, u8'\xff', 8) == kU8Str + 4);
+  static_assert(__builtin_memchr(kU8Str, u8'\xff' + 256, 8) == kU8Str + 4);
+  static_assert(__builtin_memchr(kU8Str, u8'\xff' - 256, 8) == kU8Str + 4);
+  static_assert(__builtin_memchr(kU8Foo, u8'x', 3) == nullptr);
+  static_assert(__builtin_memchr(kU8Foo, u8'x', 4) == nullptr); // both-error {{not an integral constant}} \
+                                                                // both-note {{dereferenced one-past-the-end}}
+  static_assert(__builtin_memchr(nullptr, u8'x', 3) == nullptr); // both-error {{not an integral constant}} \
+                                                                 // both-note {{dereferenced null}}
+  static_assert(__builtin_memchr(nullptr, u8'x', 0) == nullptr);
+#endif
+
+  extern struct Incomplete incomplete;
+  static_assert(__builtin_memchr(&incomplete, 0, 0u) == nullptr);
+  static_assert(__builtin_memchr(&incomplete, 0, 1u) == nullptr); // both-error {{not an integral constant}} \
+                                                                  // ref-note {{read of incomplete type 'struct Incomplete'}}
+
+  const unsigned char &u1 = 0xf0;
+  auto &&i1 = (const signed char []){-128};
+  static_assert(__builtin_memchr(&u1, -(0x0f + 1), 1) == &u1);
+  static_assert(__builtin_memchr(i1, 0x80, 1) == i1);
+
+  enum class E : unsigned char {};
+  struct EPair { E e, f; };
+  constexpr EPair ee{E{240}};
+  static_assert(__builtin_memchr(&ee.e, 240, 1) == &ee.e); // both-error {{constant}} \
+                                                           // both-note {{not supported}}
+
+  constexpr bool kBool[] = {false, true, false};
+  constexpr const bool *const kBoolPastTheEndPtr = kBool + 3;
+  static_assert(sizeof(bool) != 1u || __builtin_memchr(kBoolPastTheEndPtr - 3, 1, 99) == kBool + 1); // both-error {{constant}} \
+                                                                                                     // both-note {{not supported}}
+  static_assert(sizeof(bool) != 1u || __builtin_memchr(kBool + 1, 0, 99) == kBoolPastTheEndPtr - 1); // both-error {{constant}} \
+                                                                                                     // both-note {{not supported}}
+  static_assert(sizeof(bool) != 1u || __builtin_memchr(kBoolPastTheEndPtr - 3, -1, 3) == nullptr); // both-error {{constant}} \
+                                                                                                   // both-note {{not supported}}
+  static_assert(sizeof(bool) != 1u || __builtin_memchr(kBoolPastTheEndPtr, 0, 1) == nullptr); // both-error {{constant}} \
+                                                                                              // both-note {{not supported}}
+
+  static_assert(__builtin_char_memchr(kStr, 'a', 0) == nullptr);
+  static_assert(__builtin_char_memchr(kStr, 'a', 1) == kStr);
+  static_assert(__builtin_char_memchr(kStr, '\0', 5) == nullptr);
+  static_assert(__builtin_char_memchr(kStr, '\0', 6) == kStr + 5);
+  static_assert(__builtin_char_memchr(kStr, '\xff', 8) == kStr + 4);
+  static_assert(__builtin_char_memchr(kStr, '\xff' + 256, 8) == kStr + 4);
+  static_assert(__builtin_char_memchr(kStr, '\xff' - 256, 8) == kStr + 4);
+  static_assert(__builtin_char_memchr(kFoo, 'x', 3) == nullptr);
+  static_assert(__builtin_char_memchr(kFoo, 'x', 4) == nullptr); // both-error {{not an integral constant}} \
+                                                                 // both-note {{dereferenced one-past-the-end}}
+  static_assert(__builtin_char_memchr(nullptr, 'x', 3) == nullptr); // both-error {{not an integral constant}} \
+                                                                    // both-note {{dereferenced null}}
+  static_assert(__builtin_char_memchr(nullptr, 'x', 0) == nullptr);
+
+  static_assert(*__builtin_char_memchr(kStr, '\xff', 8) == '\xff');
+  constexpr bool char_memchr_mutable() {
+    char buffer[] = "mutable";
+    *__builtin_char_memchr(buffer, 't', 8) = 'r';
+    *__builtin_char_memchr(buffer, 'm', 8) = 'd';
+    return __builtin_strcmp(buffer, "durable") == 0;
+  }
+  static_assert(char_memchr_mutable());
+
+  constexpr bool b = !memchr("hello", 'h', 3); // both-error {{constant expression}} \
+                                               // both-note {{non-constexpr function 'memchr' cannot be used in a constant expression}}
+
+}
+
+namespace Strchr {
+  constexpr const char *kStr = "abca\xff\0d";
+  constexpr char kFoo[] = {'f', 'o', 'o'};
+  static_assert(__builtin_strchr(kStr, 'a') == kStr);
+  static_assert(__builtin_strchr(kStr, 'b') == kStr + 1);
+  static_assert(__builtin_strchr(kStr, 'c') == kStr + 2);
+  static_assert(__builtin_strchr(kStr, 'd') == nullptr);
+  static_assert(__builtin_strchr(kStr, 'e') == nullptr);
+  static_assert(__builtin_strchr(kStr, '\0') == kStr + 5);
+  static_assert(__builtin_strchr(kStr, 'a' + 256) == nullptr);
+  static_assert(__builtin_strchr(kStr, 'a' - 256) == nullptr);
+  static_assert(__builtin_strchr(kStr, '\xff') == kStr + 4);
+  static_assert(__builtin_strchr(kStr, '\xff' + 256) == nullptr);
+  static_assert(__builtin_strchr(kStr, '\xff' - 256) == nullptr);
+  static_assert(__builtin_strchr(kFoo, 'o') == kFoo + 1);
+  static_assert(__builtin_strchr(kFoo, 'x') == nullptr); // both-error {{not an integral constant}} \
+                                                         // both-note {{dereferenced one-past-the-end}}
+  static_assert(__builtin_strchr(nullptr, 'x') == nullptr); // both-error {{not an integral constant}} \
+                                                            // both-note {{dereferenced null}}
+
+  constexpr bool a = !strchr("hello", 'h'); // both-error {{constant expression}} \
+                                            // both-note {{non-constexpr function 'strchr' cannot be used in a constant expression}}
 }
diff --git a/clang/test/AST/ByteCode/cxx1z.cpp b/clang/test/AST/ByteCode/cxx1z.cpp
index ca5f10f6567b4..57f99235a2b20 100644
--- a/clang/test/AST/ByteCode/cxx1z.cpp
+++ b/clang/test/AST/ByteCode/cxx1z.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -fexperimental-new-constant-interpreter -std=c++17 -verify=expected,both %s
 // RUN: %clang_cc1 -std=c++17 -verify=ref,both %s
 
-template<typename T, T val> struct A {}; // both-note 6{{template parameter is declared here}}
+template<typename T, T val> struct A {};
 namespace Temp {
   struct S { int n; };
   constexpr S &addr(S &&s) { return s; }
diff --git a/clang/test/AST/ByteCode/cxx20.cpp b/clang/test/AST/ByteCode/cxx20.cpp
index d39c2281ec146..06501de64916a 100644
--- a/clang/test/AST/ByteCode/cxx20.cpp
+++ b/clang/test/AST/ByteCode/cxx20.cpp
@@ -897,7 +897,7 @@ namespace VirtDtor {
 }
 
 namespace TemporaryInNTTP {
-  template<auto n> struct B { /* ... */ }; // both-note {{template parameter is declared here}}
+  template<auto n> struct B { /* ... */ };
   struct J1 {
     J1 *self=this;
   };
diff --git a/clang/test/AST/ByteCode/cxx98.cpp b/clang/test/AST/ByteCode/cxx98.cpp
index 9af668029f8a3..c17049b01c1da 100644
--- a/clang/test/AST/ByteCode/cxx98.cpp
+++ b/clang/test/AST/ByteCode/cxx98.cpp
@@ -6,7 +6,7 @@
 namespace IntOrEnum {
   const int k = 0;
   const int &p = k; // both-note {{declared here}}
-  template<int n> struct S {}; // both-note {{template parameter is declared here}}
+  template<int n> struct S {};
   S<p> s; // both-error {{not an integral constant expression}} \
           // both-note {{read of variable 'p' of non-integral, non-enumeration type 'const int &'}}
 }
diff --git a/clang/test/AST/ByteCode/if.cpp b/clang/test/AST/ByteCode/if.cpp
index c48b2b8d378c8..909e08a22a283 100644
--- a/clang/test/AST/ByteCode/if.cpp
+++ b/clang/test/AST/ByteCode/if.cpp
@@ -54,7 +54,7 @@ namespace InitDecl {
 constexpr char g(char const (&x)[2]) {
     return 'x';
   if (auto [a, b] = x) // both-error {{an array type is not allowed here}} \
-                       // both-warning {{ISO C++17 does not permit structured binding declaration in a condition}}
+                       // both-warning {{structured binding declaration in a condition is a C++2c extenstion}}
     ;
 }
 static_assert(g("x") == 'x');
diff --git a/clang/test/AST/ByteCode/libcxx/memcmp-pointer.cpp b/clang/test/AST/ByteCode/libcxx/memcmp-pointer.cpp
new file mode 100644
index 0000000000000..ec4caefd4af85
--- /dev/null
+++ b/clang/test/AST/ByteCode/libcxx/memcmp-pointer.cpp
@@ -0,0 +1,120 @@
+// RUN: %clang_cc1 -std=c++2c -fexperimental-new-constant-interpreter -verify=expected,both %s
+// RUN: %clang_cc1 -std=c++2c  -verify=ref,both %s
+
+// both-no-diagnostics
+
+namespace std {
+inline namespace {
+template <int __v> struct integral_constant {
+  static const int value = __v;
+};
+template <bool, class> using __enable_if_t = int;
+char *addressof(char &);
+struct pointer_traits {
+  template <class _Up> using rebind = _Up *;
+};
+} // namespace
+} // namespace std
+void *operator new(decltype(sizeof(int)), void *);
+namespace std {
+inline namespace {
+template <class _Tp> using __make_unsigned_t = __make_unsigned(_Tp);
+template <class _Default, template <class> class>
+using __detected_or_t = _Default;
+template <class _Tp> using __pointer_member = _Tp;
+template <class _Tp, class>
+using __pointer = __detected_or_t<_Tp *, __pointer_member>;
+template <class _Tp> using __size_type_member = _Tp;
+template <class, class _DiffType>
+using __size_type =
+    __detected_or_t<__make_unsigned_t<_DiffType>, __size_type_member>;
+struct allocation_result {
+  char *ptr;
+  unsigned long count;
+};
+template <class _Alloc> struct allocator_traits {
+  using allocator_type = _Alloc;
+  using pointer =
+      __pointer<typename allocator_type::value_type, allocator_type>;
+  using const_pointer = pointer_traits::rebind<char>;
+  using size_type =
+      __size_type<allocator_type, decltype(static_cast<int *>(nullptr) -
+                                           static_cast<int *>(nullptr))>;
+  template <class _Ap>
+  static constexpr allocation_result allocate_at_least(_Ap __alloc,
+                                                       size_type __n) {
+    return {__alloc.allocate(__n), (unsigned long)__n};
+  }
+};
+template <class _Alloc>
+constexpr auto __allocate_at_least(_Alloc __alloc, decltype(sizeof(int)) __n) {
+  return allocator_traits<_Alloc>::allocate_at_least(__alloc, __n);
+}
+template <class> struct allocator {
+  typedef char value_type;
+  constexpr char *allocate(decltype(sizeof(int)) __n) {
+    return static_cast<char *>(operator new(__n));
+  }
+  constexpr void deallocate(char *__p) { operator delete(__p); }
+};
+struct __long {
+  allocator_traits<allocator<char>>::size_type __is_long_;
+  allocator_traits<allocator<char>>::size_type __size_;
+  allocator_traits<allocator<char>>::pointer __data_;
+};
+allocator<char> __alloc_;
+struct basic_string {
+  __long __l;
+  constexpr basic_string(basic_string &__str) {
+    allocator_traits<allocator<char>>::size_type __trans_tmp_1 =
+        __str.__get_long_size();
+    auto __allocation = __allocate_at_least(__alloc_, __trans_tmp_1);
+    for (allocator_traits<allocator<char>>::size_type __i = 0;
+         __i != __allocation.count; ++__i) {
+      char *__trans_tmp_9 = addressof(__allocation.ptr[__i]);
+      new (__trans_tmp_9) char();
+    }
+    __l.__data_ = __allocation.ptr;
+    __l.__is_long_ = __l.__size_ = __trans_tmp_1;
+  }
+  template <__enable_if_t<integral_constant<false>::value, int> = 0>
+  constexpr basic_string(const char *__s, allocator<char>) {
+    decltype(sizeof(int)) __trans_tmp_11, __i = 0;
+    for (; __s[__i]; ++__i)
+      __trans_tmp_11 = __i;
+    auto __allocation = __allocate_at_least(__alloc_, 1);
+    __l.__data_ = __allocation.ptr;
+    __l.__size_ = __trans_tmp_11;
+  }
+  constexpr ~basic_string() {
+    allocator<char> __a;
+    __a.deallocate(__l.__data_);
+  }
+  constexpr allocator_traits<allocator<char>>::size_type size() {
+    return __l.__is_long_;
+  }
+  constexpr char *data() {
+    allocator_traits<allocator<char>>::const_pointer __trans_tmp_6 =
+        __l.__is_long_ ? __l.__data_ : 0;
+    return __trans_tmp_6;
+  }
+  constexpr allocator_traits<allocator<char>>::size_type __get_long_size() {
+    return __l.__size_;
+  }
+};
+constexpr void operator==(basic_string __lhs, basic_string __rhs) {
+  decltype(sizeof(int)) __lhs_sz = __lhs.size();
+  char *__trans_tmp_10 = __rhs.data(), *__trans_tmp_15 = __lhs.data();
+  __builtin_memcmp(__trans_tmp_15, __trans_tmp_10, __lhs_sz);
+}
+} // namespace
+} // namespace std
+constexpr void test(std::basic_string s0) {
+  std::basic_string s1 = s0, s2(s0);
+  s2 == s1;
+}
+constexpr bool test() {
+  test(std::basic_string("2345678901234567890", std::allocator<char>()));
+  return true;
+}
+static_assert(test());
diff --git a/clang/test/AST/ByteCode/placement-new.cpp b/clang/test/AST/ByteCode/placement-new.cpp
index 7a4fc89a27dac..c353162a7aab0 100644
--- a/clang/test/AST/ByteCode/placement-new.cpp
+++ b/clang/test/AST/ByteCode/placement-new.cpp
@@ -339,6 +339,30 @@ namespace PR48606 {
   static_assert(f());
 }
 
+/// This used to crash because of an assertion in the implementation
+/// of the This instruction.
+namespace ExplicitThisOnArrayElement {
+  struct S {
+    int a = 12;
+    constexpr S(int a) {
+      this->a = a;
+    }
+  };
+
+  template <class _Tp, class... _Args>
+  constexpr void construct_at(_Tp *__location, _Args &&...__args) {
+    new (__location) _Tp(__args...);
+  }
+
+  constexpr bool foo() {
+    auto *M = std::allocator<S>().allocate(13); // both-note {{allocation performed here was not deallocated}}
+    construct_at(M, 12);
+    return true;
+  }
+
+  static_assert(foo()); // both-error {{not an integral constant expression}}
+}
+
 #ifdef BYTECODE
 constexpr int N = [] // expected-error {{must be initialized by a constant expression}} \
                      // expected-note {{assignment to dereferenced one-past-the-end pointer is not allowed in a constant expression}} \
diff --git a/clang/test/AST/ByteCode/records.cpp b/clang/test/AST/ByteCode/records.cpp
index d125fb0bcf1b4..42b6d82d7190b 100644
--- a/clang/test/AST/ByteCode/records.cpp
+++ b/clang/test/AST/ByteCode/records.cpp
@@ -413,7 +413,7 @@ namespace DeriveFailures {
 
     constexpr Derived(int i) : OtherVal(i) {} // ref-error {{never produces a constant expression}} \
                                               // both-note {{non-constexpr constructor 'Base' cannot be used in a constant expression}} \
-                                              // ref-note {{non-constexpr constructor 'Base' cannot be used in a constant expression}}
+                                              // ref-note {{non-constexpr constructor 'Base' cannot be used in a constant expression}} 
   };
 
   constexpr Derived D(12); // both-error {{must be initialized by a constant expression}} \
@@ -1740,9 +1740,9 @@ namespace CtorOfInvalidClass {
                                                // both-error {{must be initialized by a constant expression}}
 
 #if __cplusplus >= 202002L
-  template <typename T, auto Q>// both-note {{template parameter is declared here}}
+  template <typename T, auto Q>
   concept ReferenceOf = Q;
-  /// This calls a valid and constexpr copy constructor of InvalidCtor,
+  /// This calls a valid and constexpr copy constructor of InvalidCtor, 
   /// but should still be rejected.
   template<ReferenceOf<InvalidCtor> auto R, typename Rep> int F; // both-error {{non-type template argument is not a constant expression}}
 #endif
diff --git a/clang/test/AST/ast-dump-cxx2b-deducing-this.cpp b/clang/test/AST/ast-dump-cxx2b-deducing-this.cpp
index abe9d6a5b5bc6..fc86aeb3e5ec3 100644
--- a/clang/test/AST/ast-dump-cxx2b-deducing-this.cpp
+++ b/clang/test/AST/ast-dump-cxx2b-deducing-this.cpp
@@ -26,3 +26,12 @@ struct S {
   // CHECK-NEXT:   `-DeclRefExpr 0x{{[^ ]*}} <col:5> 'S<T>' lvalue ParmVar 0x{{[^ ]*}} 's' 'S<T>'
 };
 }
+
+namespace GH130272 {
+struct A {};
+struct B {
+  operator A(this B);
+};
+A a = A(B{});
+// CHECK: CallExpr 0x{{[^ ]*}} <col:9, col:11> 'A':'GH130272::A'
+}
diff --git a/clang/test/AST/ast-print-openacc-routine-construct.cpp b/clang/test/AST/ast-print-openacc-routine-construct.cpp
index 739718724f179..1d21e2ddee9a1 100644
--- a/clang/test/AST/ast-print-openacc-routine-construct.cpp
+++ b/clang/test/AST/ast-print-openacc-routine-construct.cpp
@@ -1,11 +1,16 @@
 // RUN: %clang_cc1 -fopenacc -ast-print %s -o - | FileCheck %s
 
 auto Lambda = [](){};
-// CHECK: #pragma acc routine(Lambda) worker
-#pragma acc routine(Lambda) worker
+// CHECK: #pragma acc routine(Lambda) worker bind(identifier)
+#pragma acc routine(Lambda) worker bind(identifier)
 int function();
-// CHECK: #pragma acc routine(function) vector nohost
-#pragma acc routine (function) vector nohost
+// CHECK: #pragma acc routine(function) vector nohost bind("string")
+#pragma acc routine (function) vector nohost bind("string")
+
+// CHECK: #pragma acc routine(function) device_type(Something) seq
+#pragma acc routine(function) device_type(Something) seq
+// CHECK: #pragma acc routine(function) dtype(Something) seq
+#pragma acc routine(function) dtype(Something) seq
 
 namespace NS {
   int NSFunc();
@@ -70,22 +75,27 @@ struct DepS {
 #pragma acc routine(DepS<T>::MemFunc) seq nohost
 // CHECK: #pragma acc routine(DepS<T>::StaticMemFunc) nohost worker
 #pragma acc routine(DepS<T>::StaticMemFunc) nohost worker
+
+// CHECK: #pragma acc routine(MemFunc) worker dtype(*)
+#pragma acc routine (MemFunc) worker dtype(*)
+// CHECK: #pragma acc routine(MemFunc) device_type(Lambda) vector
+#pragma acc routine (MemFunc) device_type(Lambda) vector
 };
 
-// CHECK: #pragma acc routine(DepS<int>::Lambda) gang
-#pragma acc routine(DepS<int>::Lambda) gang
+// CHECK: #pragma acc routine(DepS<int>::Lambda) gang bind("string")
+#pragma acc routine(DepS<int>::Lambda) gang bind("string")
 // CHECK: #pragma acc routine(DepS<int>::MemFunc) gang(dim: 1)
 #pragma acc routine(DepS<int>::MemFunc) gang(dim:1)
-// CHECK: #pragma acc routine(DepS<int>::StaticMemFunc) vector
-#pragma acc routine(DepS<int>::StaticMemFunc) vector
+// CHECK: #pragma acc routine(DepS<int>::StaticMemFunc) vector bind(identifier)
+#pragma acc routine(DepS<int>::StaticMemFunc) vector bind(identifier)
 
 
 template<typename T>
 void TemplFunc() {
 // CHECK: #pragma acc routine(T::MemFunc) gang(dim: T::SomethingElse())
 #pragma acc routine(T::MemFunc) gang(dim:T::SomethingElse())
-// CHECK: #pragma acc routine(T::StaticMemFunc) worker nohost
-#pragma acc routine(T::StaticMemFunc) worker nohost
-// CHECK: #pragma acc routine(T::Lambda) nohost seq
-#pragma acc routine(T::Lambda) nohost seq
+// CHECK: #pragma acc routine(T::StaticMemFunc) worker nohost bind(identifier)
+#pragma acc routine(T::StaticMemFunc) worker nohost bind(identifier)
+// CHECK: #pragma acc routine(T::Lambda) nohost seq bind("string")
+#pragma acc routine(T::Lambda) nohost seq bind("string")
 }
diff --git a/clang/test/Analysis/out-of-bounds-notes.c b/clang/test/Analysis/ArrayBound/assumption-reporting.c
similarity index 88%
rename from clang/test/Analysis/out-of-bounds-notes.c
rename to clang/test/Analysis/ArrayBound/assumption-reporting.c
index 7256beb7e893e..d687886ada1ae 100644
--- a/clang/test/Analysis/out-of-bounds-notes.c
+++ b/clang/test/Analysis/ArrayBound/assumption-reporting.c
@@ -1,6 +1,16 @@
 // RUN: %clang_analyze_cc1 -Wno-array-bounds -analyzer-output=text        \
 // RUN:     -analyzer-checker=core,security.ArrayBound,unix.Malloc,optin.taint -verify %s
 
+// When the checker security.ArrayBound encounters an array subscript operation
+// that _may be_ in bounds, it assumes that indexing _is_ in bound. These
+// assumptions will be reported to the user if the execution path leads to a
+// bug report (made by any checker) and the symbol which was constrainted by
+// the assumption is marked as interesting (with `markInteresting` or
+// indirectly via `trackExpressionValue`) in that bug report.
+//
+// This test file validates the content of these note tags which describe the
+// assumptions for the user.
+
 int TenElements[10];
 
 int irrelevantAssumptions(int arg) {
@@ -197,3 +207,14 @@ int *extentInterestingness(int arg) {
   // expected-warning@-1 {{Out of bound access to memory after the end of the heap area}}
   // expected-note@-2 {{Access of 'int' element in the heap area at index 12}}
 }
+
+int triggeredByAnyReport(int arg) {
+  // Verify that note tags explaining the assumptions made by ArrayBound are
+  // not limited to ArrayBound reports but will appear on any bug report (that
+  // marks the relevant symbol as interesting).
+  TenElements[arg + 10] = 8;
+  // expected-note@-1 {{Assuming index is non-negative and less than 10, the number of 'int' elements in 'TenElements'}}
+  return 1024 >> arg;
+  // expected-warning@-1 {{Right operand is negative in right shift}}
+  // expected-note@-2 {{The result of right shift is undefined because the right operand is negative}}
+}
diff --git a/clang/test/Analysis/out-of-bounds-constraint-check.c b/clang/test/Analysis/ArrayBound/assumptions.c
similarity index 100%
rename from clang/test/Analysis/out-of-bounds-constraint-check.c
rename to clang/test/Analysis/ArrayBound/assumptions.c
diff --git a/clang/test/Analysis/out-of-bounds.c b/clang/test/Analysis/ArrayBound/brief-tests.c
similarity index 96%
rename from clang/test/Analysis/out-of-bounds.c
rename to clang/test/Analysis/ArrayBound/brief-tests.c
index 2174dafc0021b..f4811efd8d8b6 100644
--- a/clang/test/Analysis/out-of-bounds.c
+++ b/clang/test/Analysis/ArrayBound/brief-tests.c
@@ -1,5 +1,10 @@
 // RUN: %clang_analyze_cc1 -Wno-array-bounds -analyzer-checker=core,security.ArrayBound,debug.ExprInspection -verify %s
 
+// Miscellaneous tests for `security.ArrayBound` where we only test the
+// presence or absence of a bug report. If a test doesn't fit in a more
+// specific file and doesn't need to verify the details of 'note' diagnostics,
+// then it should be placed here.
+
 void clang_analyzer_value(int);
 
 // Tests doing an out-of-bounds access after the end of an array using:
diff --git a/clang/test/Analysis/out-of-bounds-new.cpp b/clang/test/Analysis/ArrayBound/cplusplus.cpp
similarity index 58%
rename from clang/test/Analysis/out-of-bounds-new.cpp
rename to clang/test/Analysis/ArrayBound/cplusplus.cpp
index 9ae371819714d..680fbf4817c30 100644
--- a/clang/test/Analysis/out-of-bounds-new.cpp
+++ b/clang/test/Analysis/ArrayBound/cplusplus.cpp
@@ -1,10 +1,6 @@
-// RUN: %clang_analyze_cc1 -std=c++11 -Wno-array-bounds -verify %s \
-// RUN:   -triple=x86_64-unknown-linux-gnu \
-// RUN:   -analyzer-checker=unix,core,security.ArrayBound,debug.ExprInspection
+// RUN: %clang_analyze_cc1 -std=c++11 -Wno-array-bounds -analyzer-checker=unix,core,security.ArrayBound -verify %s
 
-void clang_analyzer_eval(bool);
-void clang_analyzer_value(int);
-void clang_analyzer_dump(int);
+// Test the interactions of `security.ArrayBound` with C++ features.
 
 // Tests doing an out-of-bounds access after the end of an array using:
 // - constant integer index
@@ -156,15 +152,13 @@ void test_dynamic_size(int s) {
 }
 //Tests complex arithmetic
 //in new expression
-void test_dynamic_size2(unsigned m,unsigned n){
+void test_dynamic_size2(unsigned m, unsigned n){
   unsigned *U = nullptr;
   U = new unsigned[m + n + 1];
 }
 
 //Test creating invalid references, which break the invariant that a reference
 //is always holding a value, and could lead to nasty runtime errors.
-//(This is not related to operator new, but placed in this file because the
-//other test files are not C++.)
 int array[10] = {0};
 
 void test_after_the_end_reference() {
@@ -185,60 +179,3 @@ int test_reference_that_might_be_after_the_end(int idx) {
     return -1;
   return ref;
 }
-
-// From: https://github.com/llvm/llvm-project/issues/100762
-extern int arrOf10[10];
-void using_builtin(int x) {
-  __builtin_assume(x > 101); // CallExpr
-  arrOf10[x] = 404; // expected-warning {{Out of bound access to memory}}
-}
-
-void using_assume_attr(int ax) {
-  [[assume(ax > 100)]]; // NullStmt with an "assume" attribute.
-  arrOf10[ax] = 405; // expected-warning {{Out of bound access to memory}}
-}
-
-void using_many_assume_attr(int yx) {
-  [[assume(yx > 104), assume(yx > 200), assume(yx < 300)]]; // NullStmt with an attribute
-  arrOf10[yx] = 406; // expected-warning{{Out of bound access to memory}}
-}
-
-int using_assume_attr_has_no_sideeffects(int y) {
-  int orig_y = y;
-  clang_analyzer_value(y);      // expected-warning {{32s:{ [-2147483648, 2147483647] }}}
-  clang_analyzer_value(orig_y); // expected-warning {{32s:{ [-2147483648, 2147483647] }}}
-  clang_analyzer_dump(y);       // expected-warning-re {{{{^}}reg_${{[0-9]+}}<int y> [debug.ExprInspection]{{$}}}}
-  clang_analyzer_dump(orig_y);  // expected-warning-re {{{{^}}reg_${{[0-9]+}}<int y> [debug.ExprInspection]{{$}}}}
-
-  // We should not apply sideeffects of the argument of [[assume(...)]].
-  // "y" should not get incremented;
-  [[assume(++y == 43)]]; // expected-warning {{assumption is ignored because it contains (potential) side-effects}}
- 
-  clang_analyzer_dump(y);       // expected-warning-re {{{{^}}reg_${{[0-9]+}}<int y> [debug.ExprInspection]{{$}}}}
-  clang_analyzer_dump(orig_y);  // expected-warning-re {{{{^}}reg_${{[0-9]+}}<int y> [debug.ExprInspection]{{$}}}}
-  clang_analyzer_value(y);      // expected-warning {{32s:{ [-2147483648, 2147483647] }}}
-  clang_analyzer_value(orig_y); // expected-warning {{32s:{ [-2147483648, 2147483647] }}}
-  clang_analyzer_eval(y == orig_y); // expected-warning {{TRUE}} Good.
-
-  return y;
-}
-
-int using_builtin_assume_has_no_sideeffects(int y) {
-  int orig_y = y;
-  clang_analyzer_value(y);      // expected-warning {{32s:{ [-2147483648, 2147483647] }}}
-  clang_analyzer_value(orig_y); // expected-warning {{32s:{ [-2147483648, 2147483647] }}}
-  clang_analyzer_dump(y);       // expected-warning-re {{{{^}}reg_${{[0-9]+}}<int y> [debug.ExprInspection]{{$}}}}
-  clang_analyzer_dump(orig_y);  // expected-warning-re {{{{^}}reg_${{[0-9]+}}<int y> [debug.ExprInspection]{{$}}}}
-
-  // We should not apply sideeffects of the argument of __builtin_assume(...)
-  // "u" should not get incremented;
-  __builtin_assume(++y == 43); // expected-warning {{assumption is ignored because it contains (potential) side-effects}}
- 
-  clang_analyzer_dump(y);       // expected-warning-re {{{{^}}reg_${{[0-9]+}}<int y> [debug.ExprInspection]{{$}}}}
-  clang_analyzer_dump(orig_y);  // expected-warning-re {{{{^}}reg_${{[0-9]+}}<int y> [debug.ExprInspection]{{$}}}}
-  clang_analyzer_value(y);      // expected-warning {{32s:{ [-2147483648, 2147483647] }}}
-  clang_analyzer_value(orig_y); // expected-warning {{32s:{ [-2147483648, 2147483647] }}}
-  clang_analyzer_eval(y == orig_y); // expected-warning {{TRUE}} Good.
-
-  return y;
-}
diff --git a/clang/test/Analysis/out-of-bounds-diagnostics.c b/clang/test/Analysis/ArrayBound/verbose-tests.c
similarity index 98%
rename from clang/test/Analysis/out-of-bounds-diagnostics.c
rename to clang/test/Analysis/ArrayBound/verbose-tests.c
index 524fa4e2aaaf7..84d238ed1a2a4 100644
--- a/clang/test/Analysis/out-of-bounds-diagnostics.c
+++ b/clang/test/Analysis/ArrayBound/verbose-tests.c
@@ -1,6 +1,11 @@
 // RUN: %clang_analyze_cc1 -Wno-array-bounds -analyzer-output=text        \
 // RUN:     -analyzer-checker=core,security.ArrayBound,unix.Malloc,optin.taint -verify %s
 
+// Miscellaneous tests for `security.ArrayBound` where we also verify the
+// content of the 'note' diagnostics. This makes the tests sensitive to textual
+// changes in the diagnostics, so prefer adding new tests to `brief-tests.c`
+// unless they need to verify the correctness of 'note' diagnostics.
+
 int TenElements[10];
 
 void arrayUnderflow(void) {
diff --git a/clang/test/Analysis/Checkers/WebKit/call-args.cpp b/clang/test/Analysis/Checkers/WebKit/call-args.cpp
index b4613d5090f29..e7afd9798da3e 100644
--- a/clang/test/Analysis/Checkers/WebKit/call-args.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/call-args.cpp
@@ -359,6 +359,41 @@ namespace call_with_ptr_on_ref {
   }
 }
 
+namespace call_with_explicit_construct_from_auto {
+
+  struct Impl {
+    void ref() const;
+    void deref() const;
+
+    static Ref<Impl> create();
+  };
+
+  template <typename T>
+  struct ArgObj {
+    T* t;
+  };
+
+  struct Object {
+    Object();
+    Object(Ref<Impl>&&);
+
+    Impl* impl() const { return m_impl.get(); }
+
+    static Object create(ArgObj<char>&) { return Impl::create(); }
+    static void bar(Impl&);
+
+  private:
+    RefPtr<Impl> m_impl;
+  };
+
+  template<typename CharacterType> void foo()
+  {
+      auto result = Object::create(ArgObj<CharacterType> { });
+      Object::bar(Ref { *result.impl() });
+  }
+
+}
+
 namespace call_with_explicit_temporary_obj {
   void foo() {
     Ref { *provide() }->method();
diff --git a/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h b/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h
index 7bb33bcb6cf44..9b13810d0c5c9 100644
--- a/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h
+++ b/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h
@@ -1,30 +1,94 @@
 @class NSString;
 @class NSArray;
 @class NSMutableArray;
+#define nil ((id)0)
 #define CF_BRIDGED_TYPE(T) __attribute__((objc_bridge(T)))
+#define CF_BRIDGED_MUTABLE_TYPE(T) __attribute__((objc_bridge_mutable(T)))
 typedef CF_BRIDGED_TYPE(id) void * CFTypeRef;
+typedef signed char BOOL;
 typedef signed long CFIndex;
-typedef const struct __CFAllocator * CFAllocatorRef;
+typedef unsigned long NSUInteger;
+typedef const struct CF_BRIDGED_TYPE(id) __CFAllocator * CFAllocatorRef;
 typedef const struct CF_BRIDGED_TYPE(NSString) __CFString * CFStringRef;
 typedef const struct CF_BRIDGED_TYPE(NSArray) __CFArray * CFArrayRef;
-typedef struct CF_BRIDGED_TYPE(NSMutableArray) __CFArray * CFMutableArrayRef;
+typedef struct CF_BRIDGED_MUTABLE_TYPE(NSMutableArray) __CFArray * CFMutableArrayRef;
+typedef struct CF_BRIDGED_MUTABLE_TYPE(CFRunLoopRef) __CFRunLoop * CFRunLoopRef;
 extern const CFAllocatorRef kCFAllocatorDefault;
+typedef struct _NSZone NSZone;
 CFMutableArrayRef CFArrayCreateMutable(CFAllocatorRef allocator, CFIndex capacity);
 extern void CFArrayAppendValue(CFMutableArrayRef theArray, const void *value);
 CFArrayRef CFArrayCreate(CFAllocatorRef allocator, const void **values, CFIndex numValues);
 CFIndex CFArrayGetCount(CFArrayRef theArray);
+CFRunLoopRef CFRunLoopGetCurrent(void);
+CFRunLoopRef CFRunLoopGetMain(void);
 extern CFTypeRef CFRetain(CFTypeRef cf);
 extern void CFRelease(CFTypeRef cf);
+#define CFSTR(cStr) ((CFStringRef) __builtin___CFStringMakeConstantString ("" cStr ""))
+extern Class NSClassFromString(NSString *aClassName);
 
 __attribute__((objc_root_class))
 @interface NSObject
 + (instancetype) alloc;
++ (Class) class;
++ (Class) superclass;
 - (instancetype) init;
 - (instancetype)retain;
 - (void)release;
+- (BOOL)isKindOfClass:(Class)aClass;
+@end
+
+@protocol NSCopying
+- (id)copyWithZone:(NSZone *)zone;
+@end
+
+@protocol NSFastEnumeration
+- (int)countByEnumeratingWithState:(void *)state objects:(id *)objects count:(unsigned)count;
+- (void)protocolMethod;
+@end
+
+@interface NSEnumerator <NSFastEnumeration>
+@end
+
+@interface NSDictionary : NSObject <NSCopying>
+- (NSUInteger)count;
+- (id)objectForKey:(id)aKey;
+- (id)objectForKeyedSubscript:(id)aKey;
+- (NSEnumerator *)keyEnumerator;
++ (id)dictionary;
++ (id)dictionaryWithObject:(id)object forKey:(id <NSCopying>)key;
++ (instancetype)dictionaryWithObjects:(const id [])objects forKeys:(const id <NSCopying> [])keys count:(NSUInteger)cnt;
+@end
+
+@interface NSArray : NSObject <NSCopying, NSFastEnumeration>
+- (NSUInteger)count;
+- (NSEnumerator *)objectEnumerator;
++ (NSArray *)arrayWithObjects:(const id [])objects count:(NSUInteger)count;
+@end
+
+@interface NSString : NSObject <NSCopying>
+- (NSUInteger)length;
+- (NSString *)stringByAppendingString:(NSString *)aString;
+- ( const char *)UTF8String;
+- (id)initWithUTF8String:(const char *)nullTerminatedCString;
++ (id)stringWithUTF8String:(const char *)nullTerminatedCString;
+@end
+
+@interface NSMutableString : NSString
+@end
+
+@interface NSValue : NSObject <NSCopying>
+- (void)getValue:(void *)value;
+@end
+
+@interface NSNumber : NSValue
+- (char)charValue;
+- (id)initWithInt:(int)value;
++ (NSNumber *)numberWithInt:(int)value;
 @end
 
 @interface SomeObj : NSObject
+- (SomeObj *)mutableCopy;
+- (SomeObj *)copyWithValue:(int)value;
 - (void)doWork;
 - (SomeObj *)other;
 - (SomeObj *)next;
@@ -57,28 +121,34 @@ template <typename T> struct RetainPtr {
   PtrType t;
 
   RetainPtr() : t(nullptr) { }
-
   RetainPtr(PtrType t)
     : t(t) {
     if (t)
-      CFRetain(t);
+      CFRetain(toCFTypeRef(t));
   }
-  RetainPtr(RetainPtr&& o)
-    : RetainPtr(o.t)
-  {
-    o.t = nullptr;
-  }
-  RetainPtr(const RetainPtr& o)
+  RetainPtr(RetainPtr&&);
+  RetainPtr(const RetainPtr&);
+  template <typename U>
+  RetainPtr(const RetainPtr<U>& o)
     : RetainPtr(o.t)
+  {}
+  RetainPtr operator=(const RetainPtr& o)
   {
+    if (t)
+      CFRelease(toCFTypeRef(t));
+    t = o.t;
+    if (t)
+      CFRetain(toCFTypeRef(t));
+    return *this;
   }
-  RetainPtr operator=(const RetainPtr& o)
+  template <typename U>
+  RetainPtr operator=(const RetainPtr<U>& o)
   {
     if (t)
-      CFRelease(t);
+      CFRelease(toCFTypeRef(t));
     t = o.t;
     if (t)
-      CFRetain(t);
+      CFRetain(toCFTypeRef(t));
     return *this;
   }
   ~RetainPtr() {
@@ -86,7 +156,7 @@ template <typename T> struct RetainPtr {
   }
   void clear() {
     if (t)
-      CFRelease(t);
+      CFRelease(toCFTypeRef(t));
     t = nullptr;
   }
   void swap(RetainPtr& o) {
@@ -102,10 +172,19 @@ template <typename T> struct RetainPtr {
     swap(o);
     return *this;
   }
+  PtrType leakRef()
+  {
+    PtrType s = t;
+    t = nullptr;
+    return s;
+  }
   operator PtrType() const { return t; }
   operator bool() const { return t; }
 
 private:
+  CFTypeRef toCFTypeRef(id ptr) { return (__bridge CFTypeRef)ptr; }
+  CFTypeRef toCFTypeRef(const void* ptr) { return (CFTypeRef)ptr; }
+
   template <typename U> friend RetainPtr<U> adoptNS(U*);
   template <typename U> friend RetainPtr<U> adoptCF(U);
 
@@ -113,9 +192,26 @@ template <typename T> struct RetainPtr {
   RetainPtr(PtrType t, AdoptTag) : t(t) { }
 };
 
+template <typename T>
+RetainPtr<T>::RetainPtr(RetainPtr<T>&& o)
+  : RetainPtr(o.t)
+{
+  o.t = nullptr;
+}
+
+template <typename T>
+RetainPtr<T>::RetainPtr(const RetainPtr<T>& o)
+  : RetainPtr(o.t)
+{
+}
+
 template <typename T>
 RetainPtr<T> adoptNS(T* t) {
+#if __has_feature(objc_arc)
+  return t;
+#else
   return RetainPtr<T>(t, RetainPtr<T>::Adopt);
+#endif
 }
 
 template <typename T>
@@ -123,9 +219,31 @@ RetainPtr<T> adoptCF(T t) {
   return RetainPtr<T>(t, RetainPtr<T>::Adopt);
 }
 
+template<typename T> inline RetainPtr<T> retainPtr(T ptr)
+{
+  return ptr;
+}
+
+template<typename T> inline RetainPtr<T> retainPtr(T* ptr)
+{
+  return ptr;
+}
+
+inline NSObject *bridge_cast(CFTypeRef object)
+{
+    return (__bridge NSObject *)object;
+}
+
+inline CFTypeRef bridge_cast(NSObject *object)
+{
+    return (__bridge CFTypeRef)object;
+}
+
 }
 
 using WTF::RetainPtr;
 using WTF::adoptNS;
 using WTF::adoptCF;
+using WTF::retainPtr;
 using WTF::downcast;
+using WTF::bridge_cast;
\ No newline at end of file
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-lambda-captures-decl-protects-this-crash.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-lambda-captures-decl-protects-this-crash.cpp
index 840433db5133a..0c10c69a97eda 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-lambda-captures-decl-protects-this-crash.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-lambda-captures-decl-protects-this-crash.cpp
@@ -28,9 +28,9 @@ struct Obj {
 
   void foo(Foo foo) {
     bar([this](auto baz) {
-      // expected-warning@-1{{Captured raw-pointer 'this' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+      // expected-warning@-1{{Captured raw-pointer 'this' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
       bar([this, foo = *baz, foo2 = !baz](auto&&) {
-        // expected-warning@-1{{Captured raw-pointer 'this' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+        // expected-warning@-1{{Captured raw-pointer 'this' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
         someFunction();
       });
     });
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-lambda-captures.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-lambda-captures.cpp
index 82058bf13d137..1746d2b93d469 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-lambda-captures.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-lambda-captures.cpp
@@ -94,22 +94,22 @@ void callAsync(const WTF::Function<void()>&);
 void raw_ptr() {
   RefCountable* ref_countable = make_obj();
   auto foo1 = [ref_countable](){
-    // expected-warning@-1{{Captured raw-pointer 'ref_countable' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+    // expected-warning@-1{{Captured raw-pointer 'ref_countable' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
     ref_countable->method();
   };
   auto foo2 = [&ref_countable](){
-    // expected-warning@-1{{Captured raw-pointer 'ref_countable' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+    // expected-warning@-1{{Captured raw-pointer 'ref_countable' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
     ref_countable->method();
   };
   auto foo3 = [&](){
     ref_countable->method();
-    // expected-warning@-1{{Implicitly captured raw-pointer 'ref_countable' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+    // expected-warning@-1{{Implicitly captured raw-pointer 'ref_countable' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
     ref_countable = nullptr;
   };
 
   auto foo4 = [=](){
     ref_countable->method();
-    // expected-warning@-1{{Implicitly captured raw-pointer 'ref_countable' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+    // expected-warning@-1{{Implicitly captured raw-pointer 'ref_countable' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
   };
 
   call(foo1);
@@ -128,13 +128,13 @@ void references() {
   RefCountable automatic;
   RefCountable& ref_countable_ref = automatic;
   auto foo1 = [ref_countable_ref](){ ref_countable_ref.constMethod(); };
-  // expected-warning@-1{{Captured reference 'ref_countable_ref' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+  // expected-warning@-1{{Captured reference 'ref_countable_ref' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
   auto foo2 = [&ref_countable_ref](){ ref_countable_ref.method(); };
-  // expected-warning@-1{{Captured reference 'ref_countable_ref' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+  // expected-warning@-1{{Captured reference 'ref_countable_ref' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
   auto foo3 = [&](){ ref_countable_ref.method(); };
-  // expected-warning@-1{{Implicitly captured reference 'ref_countable_ref' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+  // expected-warning@-1{{Implicitly captured reference 'ref_countable_ref' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
   auto foo4 = [=](){ ref_countable_ref.constMethod(); };
-  // expected-warning@-1{{Implicitly captured reference 'ref_countable_ref' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+  // expected-warning@-1{{Implicitly captured reference 'ref_countable_ref' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
 
   call(foo1);
   call(foo2);
@@ -187,7 +187,7 @@ void noescape_lambda() {
     otherObj->method();
   }, [&](RefCountable& obj) {
     otherObj->method();
-    // expected-warning@-1{{Implicitly captured raw-pointer 'otherObj' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+    // expected-warning@-1{{Implicitly captured raw-pointer 'otherObj' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
   });
   ([&] {
     someObj->method();
@@ -217,7 +217,7 @@ struct RefCountableWithLambdaCapturingThis {
   void method_captures_this_unsafe() {
     auto lambda = [&]() {
       nonTrivial();
-      // expected-warning@-1{{Implicitly captured raw-pointer 'this' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+      // expected-warning@-1{{Implicitly captured raw-pointer 'this' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
     };
     call(lambda);
   }
@@ -225,7 +225,7 @@ struct RefCountableWithLambdaCapturingThis {
   void method_captures_this_unsafe_capture_local_var_explicitly() {
     RefCountable* x = make_obj();
     call([this, protectedThis = RefPtr { this }, x]() {
-      // expected-warning@-1{{Captured raw-pointer 'x' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+      // expected-warning@-1{{Captured raw-pointer 'x' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
       nonTrivial();
       x->method();
     });
@@ -234,7 +234,7 @@ struct RefCountableWithLambdaCapturingThis {
   void method_captures_this_with_other_protected_var() {
     RefCountable* x = make_obj();
     call([this, protectedX = RefPtr { x }]() {
-      // expected-warning@-1{{Captured raw-pointer 'this' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+      // expected-warning@-1{{Captured raw-pointer 'this' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
       nonTrivial();
       protectedX->method();
     });
@@ -243,7 +243,7 @@ struct RefCountableWithLambdaCapturingThis {
   void method_captures_this_unsafe_capture_local_var_explicitly_with_deref() {
     RefCountable* x = make_obj();
     call([this, protectedThis = Ref { *this }, x]() {
-      // expected-warning@-1{{Captured raw-pointer 'x' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+      // expected-warning@-1{{Captured raw-pointer 'x' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
       nonTrivial();
       x->method();
     });
@@ -252,7 +252,7 @@ struct RefCountableWithLambdaCapturingThis {
   void method_captures_this_unsafe_local_var_via_vardecl() {
     RefCountable* x = make_obj();
     auto lambda = [this, protectedThis = Ref { *this }, x]() {
-      // expected-warning@-1{{Captured raw-pointer 'x' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+      // expected-warning@-1{{Captured raw-pointer 'x' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
       nonTrivial();
       x->method();
     };
@@ -330,7 +330,7 @@ struct RefCountableWithLambdaCapturingThis {
   void method_nested_lambda3() {
     callAsync([this, protectedThis = RefPtr { this }] {
       callAsync([this] {
-        // expected-warning@-1{{Captured raw-pointer 'this' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+        // expected-warning@-1{{Captured raw-pointer 'this' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
         nonTrivial();
       });
     });
@@ -380,10 +380,10 @@ void lambda_converted_to_function(RefCountable* obj)
 {
   callFunction([&]() {
     obj->method();
-    // expected-warning@-1{{Implicitly captured raw-pointer 'obj' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+    // expected-warning@-1{{Implicitly captured raw-pointer 'obj' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
   });
   callFunctionOpaque([&]() {
     obj->method();
-    // expected-warning@-1{{Implicitly captured raw-pointer 'obj' to ref-counted type or CheckedPtr-capable type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
+    // expected-warning@-1{{Implicitly captured raw-pointer 'obj' to uncounted type is unsafe [webkit.UncountedLambdaCapturesChecker]}}
   });
 }
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp
index 52854cd10f68c..07b6de21df80f 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-local-vars.cpp
@@ -464,4 +464,17 @@ namespace local_var_for_singleton {
     RefCountable* bar = singleton();
     RefCountable* baz = otherSingleton();
   }
+}
+
+namespace virtual_function {
+  struct SomeObject {
+    virtual RefCountable* provide() { return nullptr; }
+    virtual RefCountable* operator&() { return nullptr; }
+  };
+  void foo(SomeObject* obj) {
+    auto* bar = obj->provide();
+    // expected-warning@-1{{Local variable 'bar' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}}
+    auto* baz = &*obj;
+    // expected-warning@-1{{Local variable 'baz' is uncounted and unsafe [alpha.webkit.UncountedLocalVarsChecker]}}
+  }
 }
\ No newline at end of file
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
index fe7ce158eb8ba..0279e2c68ec6d 100644
--- a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-arg.cpp
@@ -196,6 +196,10 @@ class ComplexNumber {
   ComplexNumber& operator+();
 
   const Number& real() const { return realPart; }
+  const Number& complex() const;
+
+  void ref() const;
+  void deref() const;
 
 private:
   Number realPart;
@@ -240,6 +244,11 @@ class SomeType : public BaseType {
   using BaseType::BaseType;
 };
 
+struct OtherObj {
+  unsigned v { 0 };
+  OtherObj* children[4] { nullptr };
+};
+
 void __libcpp_verbose_abort(const char *__format, ...);
 
 class RefCounted {
@@ -375,7 +384,7 @@ class RefCounted {
     double y;
   };
   void trivial68() { point pt = { 1.0 }; }
-  unsigned trivial69() { return offsetof(RefCounted, children); }
+  unsigned trivial69() { return offsetof(OtherObj, children); }
   DerivedNumber* trivial70() { [[clang::suppress]] return static_cast<DerivedNumber*>(number); }
 
   static RefCounted& singleton() {
@@ -467,6 +476,8 @@ class RefCounted {
   unsigned nonTrivial22() { return ComplexNumber(123, "456").real().value(); }
   unsigned nonTrivial23() { return DerivedNumber("123").value(); }
   SomeType nonTrivial24() { return SomeType("123"); }
+  virtual void nonTrivial25() { }
+  virtual ComplexNumber* operator->() { return nullptr; }
 
   static unsigned s_v;
   unsigned v { 0 };
@@ -642,6 +653,10 @@ class UnrelatedClass {
     // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
     getFieldTrivial().nonTrivial24();
     // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
+    getFieldTrivial().nonTrivial25();
+    // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
+    getFieldTrivial()->complex();
+    // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
   }
 
   void setField(RefCounted*);
diff --git a/clang/test/Analysis/Checkers/WebKit/unretained-lambda-captures-arc.mm b/clang/test/Analysis/Checkers/WebKit/unretained-lambda-captures-arc.mm
new file mode 100644
index 0000000000000..4e3d9c2708d96
--- /dev/null
+++ b/clang/test/Analysis/Checkers/WebKit/unretained-lambda-captures-arc.mm
@@ -0,0 +1,273 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.webkit.UnretainedLambdaCapturesChecker -fobjc-arc -verify %s
+
+#include "objc-mock-types.h"
+
+namespace std {
+
+template <typename T>
+class unique_ptr {
+private:
+  T *t;
+
+public:
+  unique_ptr() : t(nullptr) { }
+  unique_ptr(T *t) : t(t) { }
+  ~unique_ptr() {
+    if (t)
+      delete t;
+  }
+  template <typename U> unique_ptr(unique_ptr<U>&& u)
+    : t(u.t)
+  {
+    u.t = nullptr;
+  }
+  T *get() const { return t; }
+  T *operator->() const { return t; }
+  T &operator*() const { return *t; }
+  unique_ptr &operator=(T *) { return *this; }
+  explicit operator bool() const { return !!t; }
+};
+
+};
+
+namespace WTF {
+
+namespace Detail {
+
+template<typename Out, typename... In>
+class CallableWrapperBase {
+public:
+    virtual ~CallableWrapperBase() { }
+    virtual Out call(In...) = 0;
+};
+
+template<typename, typename, typename...> class CallableWrapper;
+
+template<typename CallableType, typename Out, typename... In>
+class CallableWrapper : public CallableWrapperBase<Out, In...> {
+public:
+    explicit CallableWrapper(CallableType& callable)
+        : m_callable(callable) { }
+    Out call(In... in) final { return m_callable(in...); }
+
+private:
+    CallableType m_callable;
+};
+
+} // namespace Detail
+
+template<typename> class Function;
+
+template<typename Out, typename... In> Function<Out(In...)> adopt(Detail::CallableWrapperBase<Out, In...>*);
+
+template <typename Out, typename... In>
+class Function<Out(In...)> {
+public:
+    using Impl = Detail::CallableWrapperBase<Out, In...>;
+
+    Function() = default;
+
+    template<typename FunctionType>
+    Function(FunctionType f)
+        : m_callableWrapper(new Detail::CallableWrapper<FunctionType, Out, In...>(f)) { }
+
+    Out operator()(In... in) const { return m_callableWrapper->call(in...); }
+    explicit operator bool() const { return !!m_callableWrapper; }
+
+private:
+    enum AdoptTag { Adopt };
+    Function(Impl* impl, AdoptTag)
+        : m_callableWrapper(impl)
+    {
+    }
+
+    friend Function adopt<Out, In...>(Impl*);
+
+    std::unique_ptr<Impl> m_callableWrapper;
+};
+
+template<typename Out, typename... In> Function<Out(In...)> adopt(Detail::CallableWrapperBase<Out, In...>* impl)
+{
+    return Function<Out(In...)>(impl, Function<Out(In...)>::Adopt);
+}
+
+template <typename KeyType, typename ValueType>
+class HashMap {
+public:
+  HashMap();
+  HashMap([[clang::noescape]] const Function<ValueType()>&);
+  void ensure(const KeyType&, [[clang::noescape]] const Function<ValueType()>&);
+  bool operator+([[clang::noescape]] const Function<ValueType()>&) const;
+  static void ifAny(HashMap, [[clang::noescape]] const Function<bool(ValueType)>&);
+
+private:
+  ValueType* m_table { nullptr };
+};
+
+} // namespace WTF
+
+struct A {
+  static void b();
+};
+
+SomeObj* make_obj();
+CFMutableArrayRef make_cf();
+
+void someFunction();
+template <typename Callback> void call(Callback callback) {
+  someFunction();
+  callback();
+}
+void callAsync(const WTF::Function<void()>&);
+
+void raw_ptr() {
+  SomeObj* obj = make_obj();
+  auto foo1 = [obj](){
+    [obj doWork];
+  };
+  call(foo1);
+
+  auto foo2 = [&obj](){
+    [obj doWork];
+  };
+  auto foo3 = [&](){
+    [obj doWork];
+    obj = nullptr;
+  };
+  auto foo4 = [=](){
+    [obj doWork];
+  };
+  
+  auto cf = make_cf();
+  auto bar1 = [cf](){
+    // expected-warning@-1{{Captured reference 'cf' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+    CFArrayAppendValue(cf, nullptr);
+  };
+  auto bar2 = [&cf](){
+    // expected-warning@-1{{Captured reference 'cf' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+    CFArrayAppendValue(cf, nullptr);
+  };
+  auto bar3 = [&](){
+    CFArrayAppendValue(cf, nullptr);
+    // expected-warning@-1{{Implicitly captured reference 'cf' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+    cf = nullptr;
+  };
+  auto bar4 = [=](){
+    CFArrayAppendValue(cf, nullptr);
+    // expected-warning@-1{{Implicitly captured reference 'cf' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+  };
+
+  call(foo1);
+  call(foo2);
+  call(foo3);
+  call(foo4);
+
+  call(bar1);
+  call(bar2);
+  call(bar3);
+  call(bar4);
+}
+
+void quiet() {
+// This code is not expected to trigger any warnings.
+  SomeObj *obj;
+
+  auto foo3 = [&]() {};
+  auto foo4 = [=]() {};
+
+  call(foo3);
+  call(foo4);
+
+  obj = nullptr;
+}
+
+template <typename Callback>
+void map(SomeObj* start, [[clang::noescape]] Callback&& callback)
+{
+  while (start) {
+    callback(start);
+    start = [start next];
+  }
+}
+
+template <typename Callback1, typename Callback2>
+void doubleMap(SomeObj* start, [[clang::noescape]] Callback1&& callback1, Callback2&& callback2)
+{
+  while (start) {
+    callback1(start);
+    callback2(start);
+    start = [start next];
+  }
+}
+
+template <typename Callback1, typename Callback2>
+void get_count_cf(CFArrayRef array, [[clang::noescape]] Callback1&& callback1, Callback2&& callback2)
+{
+  auto count = CFArrayGetCount(array);
+  callback1(count);
+  callback2(count);
+}
+
+void noescape_lambda() {
+  SomeObj* someObj = make_obj();
+  SomeObj* otherObj = make_obj();
+  map(make_obj(), [&](SomeObj *obj) {
+    [otherObj doWork];
+  });
+  doubleMap(make_obj(), [&](SomeObj *obj) {
+    [otherObj doWork];
+  }, [&](SomeObj *obj) {
+    [otherObj doWork];
+  });
+  ([&] {
+    [someObj doWork];
+  })();
+
+  CFMutableArrayRef someCF = make_cf();
+  get_count_cf(make_cf(), [&](CFIndex count) {
+    CFArrayAppendValue(someCF, nullptr);
+  }, [&](CFIndex count) {
+    CFArrayAppendValue(someCF, nullptr);
+    // expected-warning@-1{{Implicitly captured reference 'someCF' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+  });
+}
+
+void callFunctionOpaque(WTF::Function<void()>&&);
+void callFunction(WTF::Function<void()>&& function) {
+  someFunction();
+  function();
+}
+
+void lambda_converted_to_function(SomeObj* obj, CFMutableArrayRef cf)
+{
+  callFunction([&]() {
+    [obj doWork];
+    CFArrayAppendValue(cf, nullptr);
+    // expected-warning@-1{{Implicitly captured reference 'cf' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+  });
+  callFunctionOpaque([&]() {
+    [obj doWork];
+    CFArrayAppendValue(cf, nullptr);
+    // expected-warning@-1{{Implicitly captured reference 'cf' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+  });
+}
+
+@interface ObjWithSelf : NSObject {
+  RetainPtr<id> delegate;
+}
+-(void)doWork;
+-(void)run;
+@end
+
+@implementation ObjWithSelf
+-(void)doWork {
+  auto doWork = [&] {
+    someFunction();
+    [delegate doWork];
+  };
+  callFunctionOpaque(doWork);
+}
+-(void)run {
+  someFunction();
+}
+@end
\ No newline at end of file
diff --git a/clang/test/Analysis/Checkers/WebKit/unretained-lambda-captures.mm b/clang/test/Analysis/Checkers/WebKit/unretained-lambda-captures.mm
new file mode 100644
index 0000000000000..073eff9386baa
--- /dev/null
+++ b/clang/test/Analysis/Checkers/WebKit/unretained-lambda-captures.mm
@@ -0,0 +1,296 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.webkit.UnretainedLambdaCapturesChecker -verify %s
+
+#include "objc-mock-types.h"
+
+namespace std {
+
+template <typename T>
+class unique_ptr {
+private:
+  T *t;
+
+public:
+  unique_ptr() : t(nullptr) { }
+  unique_ptr(T *t) : t(t) { }
+  ~unique_ptr() {
+    if (t)
+      delete t;
+  }
+  template <typename U> unique_ptr(unique_ptr<U>&& u)
+    : t(u.t)
+  {
+    u.t = nullptr;
+  }
+  T *get() const { return t; }
+  T *operator->() const { return t; }
+  T &operator*() const { return *t; }
+  unique_ptr &operator=(T *) { return *this; }
+  explicit operator bool() const { return !!t; }
+};
+
+};
+
+namespace WTF {
+
+namespace Detail {
+
+template<typename Out, typename... In>
+class CallableWrapperBase {
+public:
+    virtual ~CallableWrapperBase() { }
+    virtual Out call(In...) = 0;
+};
+
+template<typename, typename, typename...> class CallableWrapper;
+
+template<typename CallableType, typename Out, typename... In>
+class CallableWrapper : public CallableWrapperBase<Out, In...> {
+public:
+    explicit CallableWrapper(CallableType& callable)
+        : m_callable(callable) { }
+    Out call(In... in) final { return m_callable(in...); }
+
+private:
+    CallableType m_callable;
+};
+
+} // namespace Detail
+
+template<typename> class Function;
+
+template<typename Out, typename... In> Function<Out(In...)> adopt(Detail::CallableWrapperBase<Out, In...>*);
+
+template <typename Out, typename... In>
+class Function<Out(In...)> {
+public:
+    using Impl = Detail::CallableWrapperBase<Out, In...>;
+
+    Function() = default;
+
+    template<typename FunctionType>
+    Function(FunctionType f)
+        : m_callableWrapper(new Detail::CallableWrapper<FunctionType, Out, In...>(f)) { }
+
+    Out operator()(In... in) const { return m_callableWrapper->call(in...); }
+    explicit operator bool() const { return !!m_callableWrapper; }
+
+private:
+    enum AdoptTag { Adopt };
+    Function(Impl* impl, AdoptTag)
+        : m_callableWrapper(impl)
+    {
+    }
+
+    friend Function adopt<Out, In...>(Impl*);
+
+    std::unique_ptr<Impl> m_callableWrapper;
+};
+
+template<typename Out, typename... In> Function<Out(In...)> adopt(Detail::CallableWrapperBase<Out, In...>* impl)
+{
+    return Function<Out(In...)>(impl, Function<Out(In...)>::Adopt);
+}
+
+template <typename KeyType, typename ValueType>
+class HashMap {
+public:
+  HashMap();
+  HashMap([[clang::noescape]] const Function<ValueType()>&);
+  void ensure(const KeyType&, [[clang::noescape]] const Function<ValueType()>&);
+  bool operator+([[clang::noescape]] const Function<ValueType()>&) const;
+  static void ifAny(HashMap, [[clang::noescape]] const Function<bool(ValueType)>&);
+
+private:
+  ValueType* m_table { nullptr };
+};
+
+} // namespace WTF
+
+struct A {
+  static void b();
+};
+
+SomeObj* make_obj();
+CFMutableArrayRef make_cf();
+
+void someFunction();
+template <typename Callback> void call(Callback callback) {
+  someFunction();
+  callback();
+}
+void callAsync(const WTF::Function<void()>&);
+
+void raw_ptr() {
+  SomeObj* obj = make_obj();
+  auto foo1 = [obj](){
+    // expected-warning@-1{{Captured raw-pointer 'obj' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+    [obj doWork];
+  };
+  call(foo1);
+
+  auto foo2 = [&obj](){
+    // expected-warning@-1{{Captured raw-pointer 'obj' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+    [obj doWork];
+  };
+  auto foo3 = [&](){
+    [obj doWork];
+    // expected-warning@-1{{Implicitly captured raw-pointer 'obj' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+    obj = nullptr;
+  };
+  auto foo4 = [=](){
+    [obj doWork];
+    // expected-warning@-1{{Implicitly captured raw-pointer 'obj' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+  };
+  
+  auto cf = make_cf();
+  auto bar1 = [cf](){
+    // expected-warning@-1{{Captured reference 'cf' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+    CFArrayAppendValue(cf, nullptr);
+  };
+  auto bar2 = [&cf](){
+    // expected-warning@-1{{Captured reference 'cf' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+    CFArrayAppendValue(cf, nullptr);
+  };
+  auto bar3 = [&](){
+    CFArrayAppendValue(cf, nullptr);
+    // expected-warning@-1{{Implicitly captured reference 'cf' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+    cf = nullptr;
+  };
+  auto bar4 = [=](){
+    CFArrayAppendValue(cf, nullptr);
+    // expected-warning@-1{{Implicitly captured reference 'cf' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+  };
+
+  call(foo1);
+  call(foo2);
+  call(foo3);
+  call(foo4);
+
+  call(bar1);
+  call(bar2);
+  call(bar3);
+  call(bar4);
+
+  // Confirm that the checker respects [[clang::suppress]].
+  SomeObj* suppressed_obj = nullptr;
+  [[clang::suppress]] auto foo5 = [suppressed_obj](){
+    [suppressed_obj doWork];
+  };
+  // no warning.
+  call(foo5);
+
+  // Confirm that the checker respects [[clang::suppress]].
+  CFMutableArrayRef suppressed_cf = nullptr;
+  [[clang::suppress]] auto bar5 = [suppressed_cf](){
+    CFArrayAppendValue(suppressed_cf, nullptr);
+  };
+  // no warning.
+  call(bar5);
+}
+
+void quiet() {
+// This code is not expected to trigger any warnings.
+  SomeObj *obj;
+
+  auto foo3 = [&]() {};
+  auto foo4 = [=]() {};
+
+  call(foo3);
+  call(foo4);
+
+  obj = nullptr;
+}
+
+template <typename Callback>
+void map(SomeObj* start, [[clang::noescape]] Callback&& callback)
+{
+  while (start) {
+    callback(start);
+    start = [start next];
+  }
+}
+
+template <typename Callback1, typename Callback2>
+void doubleMap(SomeObj* start, [[clang::noescape]] Callback1&& callback1, Callback2&& callback2)
+{
+  while (start) {
+    callback1(start);
+    callback2(start);
+    start = [start next];
+  }
+}
+
+template <typename Callback1, typename Callback2>
+void get_count_cf(CFArrayRef array, [[clang::noescape]] Callback1&& callback1, Callback2&& callback2)
+{
+  auto count = CFArrayGetCount(array);
+  callback1(count);
+  callback2(count);
+}
+
+void noescape_lambda() {
+  SomeObj* someObj = make_obj();
+  SomeObj* otherObj = make_obj();
+  map(make_obj(), [&](SomeObj *obj) {
+    [otherObj doWork];
+  });
+  doubleMap(make_obj(), [&](SomeObj *obj) {
+    [otherObj doWork];
+  }, [&](SomeObj *obj) {
+    [otherObj doWork];
+    // expected-warning@-1{{Implicitly captured raw-pointer 'otherObj' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+  });
+  ([&] {
+    [someObj doWork];
+  })();
+
+  CFMutableArrayRef someCF = make_cf();
+  get_count_cf(make_cf(), [&](CFIndex count) {
+    CFArrayAppendValue(someCF, nullptr);
+  }, [&](CFIndex count) {
+    CFArrayAppendValue(someCF, nullptr);
+    // expected-warning@-1{{Implicitly captured reference 'someCF' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+  });
+}
+
+void callFunctionOpaque(WTF::Function<void()>&&);
+void callFunction(WTF::Function<void()>&& function) {
+  someFunction();
+  function();
+}
+
+void lambda_converted_to_function(SomeObj* obj, CFMutableArrayRef cf)
+{
+  callFunction([&]() {
+    [obj doWork];
+    // expected-warning@-1{{Implicitly captured raw-pointer 'obj' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+    CFArrayAppendValue(cf, nullptr);
+    // expected-warning@-1{{Implicitly captured reference 'cf' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+  });
+  callFunctionOpaque([&]() {
+    [obj doWork];
+    // expected-warning@-1{{Implicitly captured raw-pointer 'obj' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+    CFArrayAppendValue(cf, nullptr);
+    // expected-warning@-1{{Implicitly captured reference 'cf' to unretained type is unsafe [alpha.webkit.UnretainedLambdaCapturesChecker]}}
+  });
+}
+
+@interface ObjWithSelf : NSObject {
+  RetainPtr<id> delegate;
+}
+-(void)doWork;
+-(void)run;
+@end
+
+@implementation ObjWithSelf
+-(void)doWork {
+  auto doWork = [&] {
+    someFunction();
+    [delegate doWork];
+  };
+  callFunctionOpaque(doWork);
+}
+-(void)run {
+  someFunction();
+}
+@end
\ No newline at end of file
diff --git a/clang/test/Analysis/Checkers/WebKit/unretained-members-arc.mm b/clang/test/Analysis/Checkers/WebKit/unretained-members-arc.mm
new file mode 100644
index 0000000000000..9820c875b87c0
--- /dev/null
+++ b/clang/test/Analysis/Checkers/WebKit/unretained-members-arc.mm
@@ -0,0 +1,39 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.webkit.NoUnretainedMemberChecker -fobjc-arc -verify %s
+
+#include "objc-mock-types.h"
+
+namespace members {
+
+  struct Foo {
+  private:
+    SomeObj* a = nullptr;
+
+    [[clang::suppress]]
+    SomeObj* a_suppressed = nullptr;
+
+  protected:
+    RetainPtr<SomeObj> b;
+
+  public:
+    SomeObj* c = nullptr;
+    RetainPtr<SomeObj> d;
+
+    CFMutableArrayRef e = nullptr;
+// expected-warning@-1{{Member variable 'e' in 'members::Foo' is a retainable type 'CFMutableArrayRef'}}
+  };
+
+  template<class T, class S>
+  struct FooTmpl {
+    T* x;
+    S y;
+// expected-warning@-1{{Member variable 'y' in 'members::FooTmpl<SomeObj, __CFArray *>' is a raw pointer to retainable type}}
+  };
+
+  void forceTmplToInstantiate(FooTmpl<SomeObj, CFMutableArrayRef>) {}
+
+  struct [[clang::suppress]] FooSuppressed {
+  private:
+    SomeObj* a = nullptr;
+  };
+
+}
diff --git a/clang/test/Analysis/Checkers/WebKit/unretained-members.mm b/clang/test/Analysis/Checkers/WebKit/unretained-members.mm
new file mode 100644
index 0000000000000..e068a583c18c5
--- /dev/null
+++ b/clang/test/Analysis/Checkers/WebKit/unretained-members.mm
@@ -0,0 +1,60 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.webkit.NoUnretainedMemberChecker -verify %s
+
+#include "objc-mock-types.h"
+
+namespace members {
+
+  struct Foo {
+  private:
+    SomeObj* a = nullptr;
+// expected-warning@-1{{Member variable 'a' in 'members::Foo' is a raw pointer to retainable type}}
+
+    [[clang::suppress]]
+    SomeObj* a_suppressed = nullptr;
+// No warning.
+
+  protected:
+    RetainPtr<SomeObj> b;
+// No warning.
+
+  public:
+    SomeObj* c = nullptr;
+// expected-warning@-1{{Member variable 'c' in 'members::Foo' is a raw pointer to retainable type}}
+    RetainPtr<SomeObj> d;
+
+    CFMutableArrayRef e = nullptr;
+// expected-warning@-1{{Member variable 'e' in 'members::Foo' is a retainable type 'CFMutableArrayRef'}}
+  };
+
+  template<class T, class S>
+  struct FooTmpl {
+    T* a;
+// expected-warning@-1{{Member variable 'a' in 'members::FooTmpl<SomeObj, __CFArray *>' is a raw pointer to retainable type}}
+    S b;
+// expected-warning@-1{{Member variable 'b' in 'members::FooTmpl<SomeObj, __CFArray *>' is a raw pointer to retainable type}}
+  };
+
+  void forceTmplToInstantiate(FooTmpl<SomeObj, CFMutableArrayRef>) {}
+
+  struct [[clang::suppress]] FooSuppressed {
+  private:
+    SomeObj* a = nullptr;
+// No warning.
+  };
+
+}
+
+namespace ignore_unions {
+  union Foo {
+    SomeObj* a;
+    RetainPtr<SomeObj> b;
+    CFMutableArrayRef c;
+  };
+
+  template<class T>
+  union RefPtr {
+    T* a;
+  };
+
+  void forceTmplToInstantiate(RefPtr<SomeObj>) {}
+}
diff --git a/clang/test/Analysis/analyzer-enabled-checkers.c b/clang/test/Analysis/analyzer-enabled-checkers.c
index c70aeb21ab045..e5d0acb84a039 100644
--- a/clang/test/Analysis/analyzer-enabled-checkers.c
+++ b/clang/test/Analysis/analyzer-enabled-checkers.c
@@ -24,6 +24,7 @@
 // CHECK-NEXT: core.StackAddressEscape
 // CHECK-NEXT: core.UndefinedBinaryOperatorResult
 // CHECK-NEXT: core.VLASize
+// CHECK-NEXT: core.builtin.AssumeModeling
 // CHECK-NEXT: core.builtin.BuiltinFunctions
 // CHECK-NEXT: core.builtin.NoReturnFunctions
 // CHECK-NEXT: core.uninitialized.ArraySubscript
diff --git a/clang/test/Analysis/builtin_assume.cpp b/clang/test/Analysis/builtin_assume.cpp
new file mode 100644
index 0000000000000..7158306be2b82
--- /dev/null
+++ b/clang/test/Analysis/builtin_assume.cpp
@@ -0,0 +1,64 @@
+// RUN: %clang_analyze_cc1 -std=c++11 -verify %s \
+// RUN:   -triple=x86_64-unknown-linux-gnu \
+// RUN:   -analyzer-checker=core,security.ArrayBound,debug.ExprInspection
+
+void clang_analyzer_eval(bool);
+void clang_analyzer_value(int);
+void clang_analyzer_dump(int);
+
+// From: https://github.com/llvm/llvm-project/issues/100762
+extern int arrOf10[10];
+void using_builtin(int x) {
+  __builtin_assume(x > 101); // CallExpr
+  arrOf10[x] = 404; // expected-warning {{Out of bound access to memory}}
+}
+
+void using_assume_attr(int ax) {
+  [[assume(ax > 100)]]; // NullStmt with an "assume" attribute.
+  arrOf10[ax] = 405; // expected-warning {{Out of bound access to memory}}
+}
+
+void using_many_assume_attr(int yx) {
+  [[assume(yx > 104), assume(yx > 200), assume(yx < 300)]]; // NullStmt with an attribute
+  arrOf10[yx] = 406; // expected-warning{{Out of bound access to memory}}
+}
+
+int using_assume_attr_has_no_sideeffects(int y) {
+  int orig_y = y;
+  clang_analyzer_value(y);      // expected-warning {{32s:{ [-2147483648, 2147483647] }}}
+  clang_analyzer_value(orig_y); // expected-warning {{32s:{ [-2147483648, 2147483647] }}}
+  clang_analyzer_dump(y);       // expected-warning-re {{{{^}}reg_${{[0-9]+}}<int y> [debug.ExprInspection]{{$}}}}
+  clang_analyzer_dump(orig_y);  // expected-warning-re {{{{^}}reg_${{[0-9]+}}<int y> [debug.ExprInspection]{{$}}}}
+
+  // We should not apply sideeffects of the argument of [[assume(...)]].
+  // "y" should not get incremented;
+  [[assume(++y == 43)]]; // expected-warning {{assumption is ignored because it contains (potential) side-effects}}
+
+  clang_analyzer_dump(y);       // expected-warning-re {{{{^}}reg_${{[0-9]+}}<int y> [debug.ExprInspection]{{$}}}}
+  clang_analyzer_dump(orig_y);  // expected-warning-re {{{{^}}reg_${{[0-9]+}}<int y> [debug.ExprInspection]{{$}}}}
+  clang_analyzer_value(y);      // expected-warning {{32s:{ [-2147483648, 2147483647] }}}
+  clang_analyzer_value(orig_y); // expected-warning {{32s:{ [-2147483648, 2147483647] }}}
+  clang_analyzer_eval(y == orig_y); // expected-warning {{TRUE}} Good.
+
+  return y;
+}
+
+int using_builtin_assume_has_no_sideeffects(int y) {
+  int orig_y = y;
+  clang_analyzer_value(y);      // expected-warning {{32s:{ [-2147483648, 2147483647] }}}
+  clang_analyzer_value(orig_y); // expected-warning {{32s:{ [-2147483648, 2147483647] }}}
+  clang_analyzer_dump(y);       // expected-warning-re {{{{^}}reg_${{[0-9]+}}<int y> [debug.ExprInspection]{{$}}}}
+  clang_analyzer_dump(orig_y);  // expected-warning-re {{{{^}}reg_${{[0-9]+}}<int y> [debug.ExprInspection]{{$}}}}
+
+  // We should not apply sideeffects of the argument of __builtin_assume(...)
+  // "u" should not get incremented;
+  __builtin_assume(++y == 43); // expected-warning {{assumption is ignored because it contains (potential) side-effects}}
+
+  clang_analyzer_dump(y);       // expected-warning-re {{{{^}}reg_${{[0-9]+}}<int y> [debug.ExprInspection]{{$}}}}
+  clang_analyzer_dump(orig_y);  // expected-warning-re {{{{^}}reg_${{[0-9]+}}<int y> [debug.ExprInspection]{{$}}}}
+  clang_analyzer_value(y);      // expected-warning {{32s:{ [-2147483648, 2147483647] }}}
+  clang_analyzer_value(orig_y); // expected-warning {{32s:{ [-2147483648, 2147483647] }}}
+  clang_analyzer_eval(y == orig_y); // expected-warning {{TRUE}} Good.
+
+  return y;
+}
diff --git a/clang/test/Analysis/cxx23-assume-attribute.cpp b/clang/test/Analysis/cxx23-assume-attribute.cpp
index ee049af9f13aa..86e7662cd2af9 100644
--- a/clang/test/Analysis/cxx23-assume-attribute.cpp
+++ b/clang/test/Analysis/cxx23-assume-attribute.cpp
@@ -1,6 +1,7 @@
 // RUN: %clang_analyze_cc1 -std=c++23 -triple x86_64-pc-linux-gnu \
 // RUN:   -analyzer-checker=core,debug.ExprInspection -verify %s
 
+void clang_analyzer_warnIfReached();
 template <typename T> void clang_analyzer_dump(T);
 template <typename T> void clang_analyzer_value(T);
 
@@ -27,33 +28,22 @@ int ternary_in_builtin_assume(int a, int b) {
 
 // From: https://github.com/llvm/llvm-project/pull/116462#issuecomment-2517853226
 int ternary_in_assume(int a, int b) {
-  // FIXME notes
-  // Currently, if this test is run without the core.builtin.Builtin checker, the above function with the __builtin_assume behaves identically to the following test
-  // i.e. calls to `clang_analyzer_dump` result in "extraneous"  prints of the SVal(s) `reg<int b> ...`
-  // as opposed to 4 or 10
-  // which likely implies the Program State(s) did not get narrowed.
-  // A new checker is likely needed to be implemented to properly handle the expressions within `[[assume]]` to eliminate the states where `b` is not narrowed.
-
   [[assume(a > 10 ? b == 4 : b == 10)]];
   clang_analyzer_value(a);
   // expected-warning@-1 {{[-2147483648, 10]}}
   // expected-warning@-2 {{[11, 2147483647]}}
 
   clang_analyzer_dump(b); // expected-warning {{4}} expected-warning {{10}}
-  // expected-warning-re@-1 {{reg_${{[0-9]+}}<int b>}} FIXME: We shouldn't have this dump.
 
   if (a > 20) {
     clang_analyzer_dump(b + 100); // expected-warning {{104}}
-    // expected-warning-re@-1 {{(reg_${{[0-9]+}}<int b>) + 100}} FIXME: We shouldn't have this dump.
     return 2;
   }
   if (a > 10) {
     clang_analyzer_dump(b + 200); // expected-warning {{204}}
-    // expected-warning-re@-1 {{(reg_${{[0-9]+}}<int b>) + 200}} FIXME: We shouldn't have this dump.
     return 1;
   }
   clang_analyzer_dump(b + 300); // expected-warning {{310}}
-  // expected-warning-re@-1 {{(reg_${{[0-9]+}}<int b>) + 300}} FIXME: We shouldn't have this dump.
   return 0;
 }
 
@@ -70,8 +60,12 @@ int assume_and_fallthrough_at_the_same_attrstmt(int a, int b) {
       return b;
     }
   }
+
   // This code should be unreachable.
+  clang_analyzer_warnIfReached(); // expected-warning {{REACHABLE}}
+
   [[assume(false)]]; // This should definitely make it so.
-  clang_analyzer_dump(33); // expected-warning {{33 S32b}}
+  clang_analyzer_warnIfReached(); // no-warning
+
   return 0;
 }
diff --git a/clang/test/Analysis/std-c-library-functions-POSIX.c b/clang/test/Analysis/std-c-library-functions-POSIX.c
index b53f3132b8687..f6d88e6c1502d 100644
--- a/clang/test/Analysis/std-c-library-functions-POSIX.c
+++ b/clang/test/Analysis/std-c-library-functions-POSIX.c
@@ -237,3 +237,14 @@ void test_readlinkat_bufsize_zero(int fd, char *Buf, size_t Bufsize) {
   else
     clang_analyzer_eval(Bufsize == 0); // expected-warning{{UNKNOWN}}
 }
+
+void test_setsockopt_bufptr_null(int x) {
+  char buf[10] = {0};
+
+  setsockopt(1, 2, 3, 0, 0);
+  setsockopt(1, 2, 3, buf, 10);
+  if (x)
+    setsockopt(1, 2, 3, buf, 11); // expected-warning{{The 4th argument to 'setsockopt' is a buffer with size 10 but should be a buffer with size equal to or greater than the value of the 5th argument (which is 11)}}
+  else
+    setsockopt(1, 2, 3, 0, 10);  // expected-warning{{The 4th argument to 'setsockopt' is NULL but should not be NULL}}
+}
diff --git a/clang/test/Analysis/std-c-library-functions-arg-enabled-checkers.c b/clang/test/Analysis/std-c-library-functions-arg-enabled-checkers.c
index faf0a8f19d919..d2900c6a42fff 100644
--- a/clang/test/Analysis/std-c-library-functions-arg-enabled-checkers.c
+++ b/clang/test/Analysis/std-c-library-functions-arg-enabled-checkers.c
@@ -32,6 +32,7 @@
 // CHECK-NEXT: core.StackAddressEscape
 // CHECK-NEXT: core.UndefinedBinaryOperatorResult
 // CHECK-NEXT: core.VLASize
+// CHECK-NEXT: core.builtin.AssumeModeling
 // CHECK-NEXT: core.builtin.BuiltinFunctions
 // CHECK-NEXT: core.builtin.NoReturnFunctions
 // CHECK-NEXT: core.uninitialized.ArraySubscript
diff --git a/clang/test/C/C2y/n3409.c b/clang/test/C/C2y/n3409.c
new file mode 100644
index 0000000000000..01be716132b11
--- /dev/null
+++ b/clang/test/C/C2y/n3409.c
@@ -0,0 +1,36 @@
+// RUN: %clang_cc1 -verify -std=c2y -pedantic %s
+// RUN: %clang_cc1 -verify=pre-c2y -std=c2y -Wpre-c2y-compat %s
+// RUN: %clang_cc1 -verify=ext -std=c23 -pedantic %s
+// expected-no-diagnostics
+
+/* WG14 N3409: Clang 21
+ * Slay Some Earthly Demons X
+ *
+ * Removes the requirement that an expression with type void cannot be used in
+ * any way. This was making it UB to use a void expression in a _Generic
+ * selection expression for no good reason, as well as making it UB to cast a
+ * void expression to void, etc.
+ */
+
+extern void x;
+void foo() {
+  // FIXME: this is technically an extension before C2y and should be diagnosed
+  // under -pedantic.
+  (void)(void)1;
+  // FIXME: same with this.
+  x;
+  _Generic(x, void: 1);      /* pre-c2y-warning {{use of incomplete type 'void' in a '_Generic' association is incompatible with C standards before C2y}}
+                                ext-warning {{incomplete type 'void' in a '_Generic' association is a C2y extension}}
+                              */
+  _Generic(x, typeof(x): 1); /* pre-c2y-warning {{use of incomplete type 'typeof (x)' (aka 'void') in a '_Generic' association is incompatible with C standards before C2y}}
+                                ext-warning {{incomplete type 'typeof (x)' (aka 'void') in a '_Generic' association is a C2y extension}}
+                              */
+  (void)_Generic(void, default : 1); /* pre-c2y-warning {{passing a type argument as the first operand to '_Generic' is incompatible with C standards before C2y}}
+                                        ext-warning {{passing a type argument as the first operand to '_Generic' is a C2y extension}}
+                                      */
+
+  // This is not sufficiently important of an extension to warrant a "not
+  // compatible with standards before C2y" warning, but it is an extension in
+  // C23 and earlier.
+  return x; // ext-warning {{void function 'foo' should not return void expression}}
+}
diff --git a/clang/test/C/C2y/n3410.c b/clang/test/C/C2y/n3410.c
new file mode 100644
index 0000000000000..e1cb41f375b82
--- /dev/null
+++ b/clang/test/C/C2y/n3410.c
@@ -0,0 +1,45 @@
+// RUN: %clang_cc1 -verify -std=c2y -Wall -pedantic -Wno-unused %s
+
+/* WG14 N3410: No
+ * Slay Some Earthly Demons XI
+ *
+ * It is now ill-formed for the same identifier within a TU to have both
+ * internal and external linkage.
+ */
+
+void func1() {
+  extern int a; // #a
+}
+
+// This 'a' is the same as the one declared extern above.
+static int a; /* expected-error {{static declaration of 'a' follows non-static declaration}}
+                 expected-note@#a {{previous declaration is here}}
+               */
+
+static int b;
+void func2() {
+  // This 'b' is the same as the one declaraed static above, but this is not
+  // ill-formed because of C2y 6.2.2p4, which gives this variable internal
+  // linkage because the previous declaration had internal linkage.
+  extern int b; // Ok
+}
+
+static int c, d;
+void func3() {
+  int c; // no linkage, different object from the one declared above.
+  for (int d;;) {
+    // This 'c' is the same as the one declared at file scope, but because of
+    // the local scope 'c', the file scope 'c' is not visible.
+    // FIXME: This should be diagnosed under N3410.
+    extern int c;
+    // This 'd' is the same as the one declared at file scope as well, but
+    // because of the 'd' declared within the for loop, the file scope 'd' is
+    // also not visible, same as with 'c'.
+    // FIXME: This should be diagnosed under N3410.
+    extern int d;
+  }
+  for (static int e;;) {
+    extern int e; // Ok for the same reason as 'b' above.
+  }
+}
+
diff --git a/clang/test/C/C2y/n3411.c b/clang/test/C/C2y/n3411.c
new file mode 100644
index 0000000000000..934a7c70fa67e
--- /dev/null
+++ b/clang/test/C/C2y/n3411.c
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -verify=good -std=c2y -Wall -pedantic %s
+// RUN: %clang_cc1 -verify -Wnewline-eof -std=c2y -Wall -pedantic %s
+// RUN: %clang_cc1 -verify -std=c23 -Wall -pedantic %s
+// RUN: %clang_cc1 -verify=good -std=c23 %s
+
+/* WG14 N3411: Yes
+ * Slay Some Earthly Demons XII
+ *
+ * Allow a non-empty source file to end without a final newline character. Note
+ * that this file intentionally does not end with a trailing newline.
+ */
+// good-no-diagnostics
+
+int x; // Ensure the file contains at least one declaration.
+// expected-warning {{no newline at end of file}}
\ No newline at end of file
diff --git a/clang/test/C/C2y/n3451.c b/clang/test/C/C2y/n3451.c
new file mode 100644
index 0000000000000..e6727f6cbe591
--- /dev/null
+++ b/clang/test/C/C2y/n3451.c
@@ -0,0 +1,43 @@
+// RUN: %clang_cc1 -verify -std=c2y -Wall -pedantic -Wno-gnu-folding-constant %s
+// RUN: %clang_cc1 -verify -std=c23 -Wall -pedantic -Wno-gnu-folding-constant %s
+
+/* WG14 N3451: Yes
+ * Initialization of anonymous structures and unions
+ *
+ * This paper allows initialization of anonymous structure and union members
+ * within the containing object.
+ */
+// expected-no-diagnostics
+
+constexpr struct {
+  int a : 10;
+  int : 12;
+  long b;
+} s = { 1, 2 };
+static_assert(s.a == 1 && s.b == 2);
+
+constexpr union {
+  int : 16;
+  char c;
+} u = {3};
+static_assert(u.c == 3);
+
+constexpr struct {
+  union {
+    float a;
+    int b;
+    void *p;
+  };
+  char c;
+} t = {{.b = 1}, 2};
+static_assert(t.b == 1 && t.c == 2);
+
+constexpr struct {
+  union {
+    float a;
+    int b;
+    void *p;
+  };
+  char c;
+} v = {.b = 1, 2};
+static_assert(v.b == 1 && v.c == 2);
diff --git a/clang/test/C/C2y/n3478.c b/clang/test/C/C2y/n3478.c
new file mode 100644
index 0000000000000..9d1d8f97fa962
--- /dev/null
+++ b/clang/test/C/C2y/n3478.c
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -verify -std=c2y %s
+// RUN: %clang_cc1 -verify -std=c23 %s
+
+/* WG14 N3478: Yes
+ * Slay Some Earthly Demons XIII
+ *
+ * It was previously UB to end a source file with a partial preprocessing token
+ * or a partial comment. Clang has always diagnosed these.
+ */
+
+// expected-error@+1 {{unterminated /* comment}}
+/*
\ No newline at end of file
diff --git a/clang/test/C/C2y/n3481.c b/clang/test/C/C2y/n3481.c
new file mode 100644
index 0000000000000..1088b62dd8085
--- /dev/null
+++ b/clang/test/C/C2y/n3481.c
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 -verify -std=c2y %s
+// RUN: %clang_cc1 -verify -std=c23 %s
+
+/* WG14 N3481: Yes
+ * Slay Some Earthly Demons XVI
+ *
+ * It was previously UB to use a non-array lvalue with an incomplete type in a
+ * context which required the value of the object. Clang has always diagnosed
+ * this as an error, except when the incomplete type is void. Then we allow the
+ * dereference, but not for a value computation.
+ */
+
+struct f *p;  // expected-note {{forward declaration of 'struct f'}}
+void g(void) {
+  (void)*p; // expected-error {{incomplete type 'struct f' where a complete type is required}}
+}
+
+void h(void *ptr) {
+  (void)*ptr; // expected-warning {{ISO C does not allow indirection on operand of type 'void *'}}
+  (*ptr)++;   /* expected-warning {{ISO C does not allow indirection on operand of type 'void *'}}
+                 expected-error {{cannot increment value of type 'void'}}
+               */
+}
diff --git a/clang/test/C/C2y/n3496.c b/clang/test/C/C2y/n3496.c
new file mode 100644
index 0000000000000..74b8880b8ca19
--- /dev/null
+++ b/clang/test/C/C2y/n3496.c
@@ -0,0 +1,17 @@
+// RUN: %clang_cc1 -verify -std=c2y -ffreestanding %s
+// RUN: %clang_cc1 -verify -std=c23 -ffreestanding %s
+
+/* WG14 N3496: Clang 20
+ * Clarify the specification of the width macros
+ *
+ * C23 N2412 mandated a two's complement sign representation for integers, and
+ * added *_WIDTH macros for all of the various integer types. There was
+ * confusion as to whether BOOL_WIDTH specified the minimum number of bits or
+ * an exact number of bits. N3496 clarified that it's an exact number of bits.
+ */
+// expected-no-diagnostics
+
+#include <limits.h>
+
+static_assert(BOOL_WIDTH == 1);
+
diff --git a/clang/test/C/C2y/n3505.c b/clang/test/C/C2y/n3505.c
new file mode 100644
index 0000000000000..7f307390e09ca
--- /dev/null
+++ b/clang/test/C/C2y/n3505.c
@@ -0,0 +1,32 @@
+// RUN: %clang_cc1 -verify -std=c2y -Wall -pedantic %s
+// RUN: %clang_cc1 -verify -std=c23 -Wall -pedantic %s
+// RUN: %clang_cc1 -verify -std=c11 -Wall -pedantic %s
+// RUN: %clang_cc1 -verify -std=c99 -Wall -pedantic %s
+
+/* WG14 N3505: Yes
+ * Preprocessor integer expressions, v. 2
+ *
+ * This introduces a constraint that preprocessing tokens must be an integer
+ * literal, character literal, punctuator, or some other implementation-defined
+ * sequence of tokens (to support builtins that insert odd tokens into the
+ * parsing stream).
+ */
+
+// This is technically an integer constant expression, but it does not match
+// the new constraints and thus needs to be diagnosed.
+#if 1 ? 1 : (""[0] += 5) // expected-error {{invalid token at start of a preprocessor expression}}
+#endif
+
+// But with a character literal, it is fine.
+#if 1 ? 1 : ('a' + 5) // Ok
+#endif
+
+// This doesn't mean that all punctuators are fine, however.
+#if 1 ? 1 : ('a' += 5) // expected-error {{token is not a valid binary operator in a preprocessor subexpression}}
+#endif
+
+// But some are.
+#if 1 ? 1 : ~('a') // Ok
+#endif
+
+int x; // Needs a declaration to avoid a pedantic warning
diff --git a/clang/test/CIR/CodeGen/basic.cpp b/clang/test/CIR/CodeGen/basic.cpp
index 6a2faa725a34d..ef922cc2b46fc 100644
--- a/clang/test/CIR/CodeGen/basic.cpp
+++ b/clang/test/CIR/CodeGen/basic.cpp
@@ -1,9 +1,4 @@
-// RUN: not %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o - 2>&1 | FileCheck %s
-
-// This error is caused by the "const int i = 2" line in f2(). When
-// initaliziers are implemented, the checks there should be updated
-// and the "not" should be removed from the run line.
-// CHECK: error: ClangIR code gen Not Yet Implemented: emitAutoVarInit
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o - 2>&1 | FileCheck %s
 
 int f1() {
   int i;
@@ -22,13 +17,15 @@ int f2() {
 }
 
 // CHECK: cir.func @f2() -> !cir.int<s, 32>
-// CHECK:    %[[I_PTR:.*]] = cir.alloca !cir.int<s, 32>, !cir.ptr<!cir.int<s, 32>>, ["i", const] {alignment = 4 : i64}
+// CHECK:    %[[I_PTR:.*]] = cir.alloca !cir.int<s, 32>, !cir.ptr<!cir.int<s, 32>>, ["i", init, const] {alignment = 4 : i64}
+// CHECK:    %[[TWO:.*]] = cir.const #cir.int<2> : !cir.int<s, 32>
+// CHECK:    cir.store %[[TWO]], %[[I_PTR]] : !cir.int<s, 32>, !cir.ptr<!cir.int<s, 32>>
 // CHECK:    %[[I:.*]] = cir.load %[[I_PTR]] : !cir.ptr<!cir.int<s, 32>>, !cir.int<s, 32>
 // CHECK:    cir.return %[[I]] : !cir.int<s, 32>
 
 int f3(int i) {
-    return i;
-  }
+  return i;
+}
 
 // CHECK: cir.func @f3(%[[ARG:.*]]: !cir.int<s, 32> loc({{.*}})) -> !cir.int<s, 32>
 // CHECK:   %[[ARG_ALLOCA:.*]] = cir.alloca !cir.int<s, 32>, !cir.ptr<!cir.int<s, 32>>, ["i", init] {alignment = 4 : i64}
diff --git a/clang/test/CIR/CodeGen/local-vars.cpp b/clang/test/CIR/CodeGen/local-vars.cpp
new file mode 100644
index 0000000000000..14be8f4da902b
--- /dev/null
+++ b/clang/test/CIR/CodeGen/local-vars.cpp
@@ -0,0 +1,65 @@
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o - 2>&1 | FileCheck %s
+
+void test() {
+  int i = 1;
+  long l = 2l;
+  float f = 3.0f;
+  double d = 4.0;
+  bool b1 = true;
+  bool b2 = false;
+  const int ci = 1;
+  const long cl = 2l;
+  const float cf = 3.0f;
+  const double cd = 4.0;
+  const bool cb1 = true;
+  const bool cb2 = false;
+  int uii;
+  long uil;
+  float uif;
+  double uid;
+  bool uib;
+}
+
+// CHECK: module
+// CHECK: cir.func @test()
+// CHECK:    %[[I_PTR:.*]] = cir.alloca !cir.int<s, 32>, !cir.ptr<!cir.int<s, 32>>, ["i", init] {alignment = 4 : i64}
+// CHECK:    %[[L_PTR:.*]] = cir.alloca !cir.int<s, 64>, !cir.ptr<!cir.int<s, 64>>, ["l", init] {alignment = 8 : i64}
+// CHECK:    %[[F_PTR:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["f", init] {alignment = 4 : i64}
+// CHECK:    %[[D_PTR:.*]] = cir.alloca !cir.double, !cir.ptr<!cir.double>, ["d", init] {alignment = 8 : i64}
+// CHECK:    %[[B1_PTR:.*]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["b1", init] {alignment = 1 : i64}
+// CHECK:    %[[B2_PTR:.*]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["b2", init] {alignment = 1 : i64}
+// CHECK:    %[[CI_PTR:.*]] = cir.alloca !cir.int<s, 32>, !cir.ptr<!cir.int<s, 32>>, ["ci", init, const] {alignment = 4 : i64}
+// CHECK:    %[[CL_PTR:.*]] = cir.alloca !cir.int<s, 64>, !cir.ptr<!cir.int<s, 64>>, ["cl", init, const] {alignment = 8 : i64}
+// CHECK:    %[[CF_PTR:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["cf", init, const] {alignment = 4 : i64}
+// CHECK:    %[[CD_PTR:.*]] = cir.alloca !cir.double, !cir.ptr<!cir.double>, ["cd", init, const] {alignment = 8 : i64}
+// CHECK:    %[[CB1_PTR:.*]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["cb1", init, const] {alignment = 1 : i64}
+// CHECK:    %[[CB2_PTR:.*]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["cb2", init, const] {alignment = 1 : i64}
+// CHECK:    %[[UII_PTR:.*]] = cir.alloca !cir.int<s, 32>, !cir.ptr<!cir.int<s, 32>>, ["uii"] {alignment = 4 : i64}
+// CHECK:    %[[UIL_PTR:.*]] = cir.alloca !cir.int<s, 64>, !cir.ptr<!cir.int<s, 64>>, ["uil"] {alignment = 8 : i64}
+// CHECK:    %[[UIF_PTR:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["uif"] {alignment = 4 : i64}
+// CHECK:    %[[UID_PTR:.*]] = cir.alloca !cir.double, !cir.ptr<!cir.double>, ["uid"] {alignment = 8 : i64}
+// CHECK:    %[[UIB_PTR:.*]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["uib"] {alignment = 1 : i64}
+// CHECK:    %[[ONE:.*]] = cir.const #cir.int<1> : !cir.int<s, 32>
+// CHECK:    cir.store %[[ONE]], %[[I_PTR]] : !cir.int<s, 32>, !cir.ptr<!cir.int<s, 32>>
+// CHECK:    %[[TWO:.*]] = cir.const #cir.int<2> : !cir.int<s, 64>
+// CHECK:    cir.store %[[TWO]], %[[L_PTR]] : !cir.int<s, 64>, !cir.ptr<!cir.int<s, 64>>
+// CHECK:    %[[THREE:.*]] = cir.const #cir.fp<3.0{{.*}}> : !cir.float
+// CHECK:    cir.store %[[THREE]], %[[F_PTR]] : !cir.float, !cir.ptr<!cir.float>
+// CHECK:    %[[FOUR:.*]] = cir.const #cir.fp<4.0{{.*}}> : !cir.double
+// CHECK:    cir.store %[[FOUR]], %[[D_PTR]] : !cir.double, !cir.ptr<!cir.double>
+// CHECK:    %[[TRUE:.*]] = cir.const #true
+// CHECK:    cir.store %[[TRUE]], %[[B1_PTR]] : !cir.bool, !cir.ptr<!cir.bool>
+// CHECK:    %[[FALSE:.*]] = cir.const #false
+// CHECK:    cir.store %[[FALSE]], %[[B2_PTR]] : !cir.bool, !cir.ptr<!cir.bool>
+// CHECK:    %[[ONEC:.*]] = cir.const #cir.int<1> : !cir.int<s, 32>
+// CHECK:    cir.store %[[ONEC]], %[[CI_PTR]] : !cir.int<s, 32>, !cir.ptr<!cir.int<s, 32>>
+// CHECK:    %[[TWOC:.*]] = cir.const #cir.int<2> : !cir.int<s, 64>
+// CHECK:    cir.store %[[TWOC]], %[[CL_PTR]] : !cir.int<s, 64>, !cir.ptr<!cir.int<s, 64>>
+// CHECK:    %[[THREEC:.*]] = cir.const #cir.fp<3.0{{.*}}> : !cir.float
+// CHECK:    cir.store %[[THREEC]], %[[CF_PTR]] : !cir.float, !cir.ptr<!cir.float>
+// CHECK:    %[[FOURC:.*]] = cir.const #cir.fp<4.0{{.*}}> : !cir.double
+// CHECK:    cir.store %[[FOURC]], %[[CD_PTR]] : !cir.double, !cir.ptr<!cir.double>
+// CHECK:    %[[TRUEC:.*]] = cir.const #true
+// CHECK:    cir.store %[[TRUEC]], %[[CB1_PTR]] : !cir.bool, !cir.ptr<!cir.bool>
+// CHECK:    %[[FALSEC:.*]] = cir.const #false
+// CHECK:    cir.store %[[FALSEC]], %[[CB2_PTR]] : !cir.bool, !cir.ptr<!cir.bool>
diff --git a/clang/test/CIR/Lowering/basic.cpp b/clang/test/CIR/Lowering/basic.cpp
index 2c29368bd5835..d1dc343a068a8 100644
--- a/clang/test/CIR/Lowering/basic.cpp
+++ b/clang/test/CIR/Lowering/basic.cpp
@@ -1,9 +1,4 @@
-// RUN: not %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o - 2>&1 | FileCheck %s
-
-// This error is caused by the "const int i = 2" line in f2(). When
-// initaliziers are implemented, the checks there should be updated
-// and the "not" should be removed from the run line.
-// CHECK: error: ClangIR code gen Not Yet Implemented: emitAutoVarInit
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o - 2>&1 | FileCheck %s
 
 int f1() {
   int i;
@@ -22,6 +17,7 @@ int f2() {
 
 // CHECK: define{{.*}} i32 @f2() {
 // CHECK:    %[[I_PTR:.*]] = alloca i32, i64 1, align 4
+// CHECK:    store i32 2, ptr %[[I_PTR]], align 4
 // CHECK:    %[[I:.*]] = load i32, ptr %[[I_PTR]], align 4
 // CHECK:    ret i32 %[[I]]
 
diff --git a/clang/test/CIR/Lowering/func-simple.cpp b/clang/test/CIR/Lowering/func-simple.cpp
index 1429ec270774b..32d75cdd2c15d 100644
--- a/clang/test/CIR/Lowering/func-simple.cpp
+++ b/clang/test/CIR/Lowering/func-simple.cpp
@@ -13,6 +13,26 @@ int intfunc() { return 42; }
 // CHECK: define{{.*}} i32 @intfunc()
 // CHECK:   ret i32 42
 
+int scopes() {
+  {
+    {
+      return 99;
+    }
+  }
+}
+// CHECK: define{{.*}} i32 @scopes() {
+// CHECK:     br label %[[LABEL1:.*]]
+// CHECK:     [[LABEL1]]:
+// CHECK:       br label %[[LABEL2:.*]]
+// CHECK:     [[LABEL2]]:
+// CHECK:       ret i32 99
+// CHECK:     [[LABEL3:.*]]:
+// CHECK:       br label %[[LABEL4:.*]]
+// CHECK:     [[LABEL4]]:
+// CHECK:       call void @llvm.trap()
+// CHECK:       unreachable
+// CHECK: }
+
 long longfunc() { return 42l; }
 // CHECK: define{{.*}} i64 @longfunc() {
 // CHECK:   ret i64 42
diff --git a/clang/test/CIR/Lowering/local-vars.cpp b/clang/test/CIR/Lowering/local-vars.cpp
new file mode 100644
index 0000000000000..bd47ed14065c6
--- /dev/null
+++ b/clang/test/CIR/Lowering/local-vars.cpp
@@ -0,0 +1,55 @@
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o - 2>&1 | FileCheck %s
+
+void test() {
+  int i = 1;
+  long l = 2l;
+  float f = 3.0f;
+  double d = 4.0;
+  bool b1 = true;
+  bool b2 = false;
+  const int ci = 1;
+  const long cl = 2l;
+  const float cf = 3.0f;
+  const double cd = 4.0;
+  const bool cb1 = true;
+  const bool cb2 = false;
+  int uii;
+  long uil;
+  float uif;
+  double uid;
+  bool uib;
+}
+
+// Note: The alignment of i64 stores below is wrong. That should be fixed
+//       when we add alignment attributes to the load/store ops.
+
+// CHECK: define{{.*}} void @test()
+// CHECK:    %[[I_PTR:.*]] = alloca i32, i64 1, align 4
+// CHECK:    %[[L_PTR:.*]] = alloca i64, i64 1, align 8
+// CHECK:    %[[F_PTR:.*]] = alloca float, i64 1, align 4
+// CHECK:    %[[D_PTR:.*]] = alloca double, i64 1, align 8
+// CHECK:    %[[B1_PTR:.*]] = alloca i8, i64 1, align 1
+// CHECK:    %[[B2_PTR:.*]] = alloca i8, i64 1, align 1
+// CHECK:    %[[CI_PTR:.*]] = alloca i32, i64 1, align 4
+// CHECK:    %[[CL_PTR:.*]] = alloca i64, i64 1, align 8
+// CHECK:    %[[CF_PTR:.*]] = alloca float, i64 1, align 4
+// CHECK:    %[[CD_PTR:.*]] = alloca double, i64 1, align 8
+// CHECK:    %[[CB1_PTR:.*]] = alloca i8, i64 1, align 1
+// CHECK:    %[[CB2_PTR:.*]] = alloca i8, i64 1, align 1
+// CHECK:    %[[UII_PTR:.*]] = alloca i32, i64 1, align 4
+// CHECK:    %[[UIL_PTR:.*]] = alloca i64, i64 1, align 8
+// CHECK:    %[[UIF_PTR:.*]] = alloca float, i64 1, align 4
+// CHECK:    %[[UID_PTR:.*]] = alloca double, i64 1, align 8
+// CHECK:    %[[UIB_PTR:.*]] = alloca i8, i64 1, align 1
+// CHECK:    store i32 1, ptr %[[I_PTR]], align 4
+// CHECK:    store i64 2, ptr %[[L_PTR]], align 4
+// CHECK:    store float 3.000000e+00, ptr %[[F_PTR]], align 4
+// CHECK:    store double 4.000000e+00, ptr %[[D_PTR]], align 8
+// CHECK:    store i8 1, ptr %[[B1_PTR]], align 1
+// CHECK:    store i8 0, ptr %[[B2_PTR]], align 1
+// CHECK:    store i32 1, ptr %[[CI_PTR]], align 4
+// CHECK:    store i64 2, ptr %[[CL_PTR]], align 4
+// CHECK:    store float 3.000000e+00, ptr %[[CF_PTR]], align 4
+// CHECK:    store double 4.000000e+00, ptr %[[CD_PTR]], align 8
+// CHECK:    store i8 1, ptr %[[CB1_PTR]], align 1
+// CHECK:    store i8 0, ptr %[[CB2_PTR]], align 1
diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt
index 1c93e2b0d9844..51fd4820d7edb 100644
--- a/clang/test/CMakeLists.txt
+++ b/clang/test/CMakeLists.txt
@@ -136,6 +136,7 @@ if( NOT CLANG_BUILT_STANDALONE )
     llvm-dis
     llvm-dwarfdump
     llvm-ifs
+    llvm-link
     llvm-lto2
     llvm-mc
     llvm-modextract
diff --git a/clang/test/CXX/basic/basic.lookup/basic.lookup.qual/class.qual/p2.cpp b/clang/test/CXX/basic/basic.lookup/basic.lookup.qual/class.qual/p2.cpp
index 0fa98ad101f6c..5c281ac806836 100644
--- a/clang/test/CXX/basic/basic.lookup/basic.lookup.qual/class.qual/p2.cpp
+++ b/clang/test/CXX/basic/basic.lookup/basic.lookup.qual/class.qual/p2.cpp
@@ -190,7 +190,7 @@ namespace InhCtor {
   }
   struct DerivedFromNS : NS::NS {
     // No special case unless the NNS names a class.
-    using InhCtor::NS::NS; // expected-error {{using declaration in class refers into 'InhCtor::NS::', which is not a class}}
+    using InhCtor::NS::NS; // expected-error {{using declaration in class refers into 'InhCtor::NS', which is not a class}}
 
   };
 
diff --git a/clang/test/CXX/basic/basic.lookup/basic.lookup.unqual/p7.cpp b/clang/test/CXX/basic/basic.lookup/basic.lookup.unqual/p7.cpp
index fc4e359666ba9..9632fda296aa1 100644
--- a/clang/test/CXX/basic/basic.lookup/basic.lookup.unqual/p7.cpp
+++ b/clang/test/CXX/basic/basic.lookup/basic.lookup.unqual/p7.cpp
@@ -33,6 +33,5 @@ namespace test1 {
 // specifiers.
 namespace test2 {
   template <class T> struct bar {};
-  // expected-note@-1 {{template parameter is declared here}}
   template <class T> struct foo : bar<foo> {}; // expected-error {{use of class template 'foo' requires template arguments}} expected-note {{template is declared here}}
 }
diff --git a/clang/test/CXX/class.access/class.access.dcl/p1.cpp b/clang/test/CXX/class.access/class.access.dcl/p1.cpp
index 118ab9e52d0a1..fdb1373dd9b12 100644
--- a/clang/test/CXX/class.access/class.access.dcl/p1.cpp
+++ b/clang/test/CXX/class.access/class.access.dcl/p1.cpp
@@ -331,7 +331,7 @@ namespace test4 {
     // expected-warning@-2 {{access declarations are deprecated; use using declarations instead}}
 #else
     // expected-error@-4 {{ISO C++11 does not allow access declarations; use using declarations instead}}
-    // expected-error@-5 {{using declaration refers into 'Subclass::', which is not a base class of 'C'}}
+    // expected-error@-5 {{using declaration refers into 'Subclass', which is not a base class of 'C'}}
 #endif
 
     int bar();
diff --git a/clang/test/CXX/class.access/class.friend/p3-cxx0x.cpp b/clang/test/CXX/class.access/class.friend/p3-cxx0x.cpp
index 9aabdbe540a66..f7216ea7eb7b0 100644
--- a/clang/test/CXX/class.access/class.friend/p3-cxx0x.cpp
+++ b/clang/test/CXX/class.access/class.friend/p3-cxx0x.cpp
@@ -36,7 +36,7 @@ class A {
 public:
   class foo {};
   static int y;
-  template <typename S> friend class B<S>::ty; // expected-warning {{dependent nested name specifier 'B<S>::' for friend class declaration is not supported}}
+  template <typename S> friend class B<S>::ty; // expected-warning {{dependent nested name specifier 'B<S>' for friend class declaration is not supported}}
 };
 
 template<typename T> class B { typedef int ty; };
diff --git a/clang/test/CXX/class.access/class.friend/p6.cpp b/clang/test/CXX/class.access/class.friend/p6.cpp
index 47104e29dc6b3..da6d212b78868 100644
--- a/clang/test/CXX/class.access/class.friend/p6.cpp
+++ b/clang/test/CXX/class.access/class.friend/p6.cpp
@@ -8,11 +8,11 @@ struct X {
 
 struct Y {
   friend void ::f1() { } // expected-error{{friend function definition cannot be qualified with '::'}}
-  friend void X::f2() { } // expected-error{{friend function definition cannot be qualified with 'X::'}}
+  friend void X::f2() { } // expected-error{{friend function definition cannot be qualified with 'X'}}
 };
 
 template <typename T> struct Z {
-  friend void T::f() {} // expected-error{{friend function definition cannot be qualified with 'T::'}}
+  friend void T::f() {} // expected-error{{friend function definition cannot be qualified with 'T'}}
 };
 
 void local() {
@@ -32,6 +32,6 @@ namespace N {
 template<typename T> struct A {
   friend void f3(T) {}
   friend void f3<T>(T) {} // expected-error{{friend function specialization cannot be defined}}
-  friend void N::f4(T) {} // expected-error{{friend function definition cannot be qualified with 'N::'}}
-  friend void N::f4<T>(T) {} // expected-error{{friend function definition cannot be qualified with 'N::'}}
+  friend void N::f4(T) {} // expected-error{{friend function definition cannot be qualified with 'N'}}
+  friend void N::f4<T>(T) {} // expected-error{{friend function definition cannot be qualified with 'N'}}
 };
diff --git a/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p3.cpp b/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p3.cpp
index 3351666525374..657695657e0f7 100644
--- a/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p3.cpp
+++ b/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p3.cpp
@@ -28,7 +28,7 @@ class D2 : public B {
   using B::E;
   using B::e;
   using B::x;
-  using C::g; // expected-error{{using declaration refers into 'C::', which is not a base class of 'D2'}}
+  using C::g; // expected-error{{using declaration refers into 'C', which is not a base class of 'D2'}}
 
   // These are valid in C++98 but not in C++11.
   using D::f2;
@@ -36,10 +36,10 @@ class D2 : public B {
   using D::e2;
   using D::x2;
 #if __cplusplus >= 201103L
-  // expected-error@-5 {{using declaration refers into 'D::', which is not a base class of 'D2'}}
-  // expected-error@-5 {{using declaration refers into 'D::', which is not a base class of 'D2'}}
-  // expected-error@-5 {{using declaration refers into 'D::', which is not a base class of 'D2'}}
-  // expected-error@-5 {{using declaration refers into 'D::', which is not a base class of 'D2'}}
+  // expected-error@-5 {{using declaration refers into 'D', which is not a base class of 'D2'}}
+  // expected-error@-5 {{using declaration refers into 'D', which is not a base class of 'D2'}}
+  // expected-error@-5 {{using declaration refers into 'D', which is not a base class of 'D2'}}
+  // expected-error@-5 {{using declaration refers into 'D', which is not a base class of 'D2'}}
 #endif
 
   using B::EC;
diff --git a/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p4.cpp b/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p4.cpp
index 65ccb751b9aa5..973cd77279f19 100644
--- a/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p4.cpp
+++ b/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p4.cpp
@@ -210,7 +210,7 @@ namespace test4 {
     using Subclass::foo; // legal in C++03
 #if __cplusplus >= 201103L
     // expected-error@-3 {{refers to its own class}}
-    // expected-error@-3 {{refers into 'Subclass::', which is not a base class}}
+    // expected-error@-3 {{refers into 'Subclass', which is not a base class}}
 #endif
 
     int bar();
diff --git a/clang/test/CXX/dcl.decl/dcl.init/dcl.init.list/p7-0x.cpp b/clang/test/CXX/dcl.decl/dcl.init/dcl.init.list/p7-0x.cpp
index 57c99212f4c69..2bceb3e267790 100644
--- a/clang/test/CXX/dcl.decl/dcl.init/dcl.init.list/p7-0x.cpp
+++ b/clang/test/CXX/dcl.decl/dcl.init/dcl.init.list/p7-0x.cpp
@@ -251,5 +251,5 @@ void P1957R2(void *a, int *b, Agg<int> *c, int Agg<int>::*d) {
   Agg<bool> tc = {c}; // expected-error {{cannot be narrowed}} expected-note {{}}
   Agg<bool> td = {d}; // expected-error {{cannot be narrowed}} expected-note {{}}
 }
-template<bool> struct BoolParam {}; // expected-note {{template parameter is declared here}}
+template<bool> struct BoolParam {};
 BoolParam<&P1957R2> bp; // expected-error {{not allowed in a converted constant expression}}
diff --git a/clang/test/CXX/dcl.decl/dcl.meaning/p1-0x.cpp b/clang/test/CXX/dcl.decl/dcl.meaning/p1-0x.cpp
index 13be079a40bc3..4475bd787c74d 100644
--- a/clang/test/CXX/dcl.decl/dcl.meaning/p1-0x.cpp
+++ b/clang/test/CXX/dcl.decl/dcl.meaning/p1-0x.cpp
@@ -18,9 +18,9 @@ class tfoo {
 };
 
 template<typename T>
-int decltype(tfoo<T>())::i; // expected-error{{nested name specifier 'decltype(tfoo<T>())::' for declaration does not refer into a class, class template or class template partial specialization}}
+int decltype(tfoo<T>())::i; // expected-error{{nested name specifier 'decltype(tfoo<T>())' for declaration does not refer into a class, class template or class template partial specialization}}
 template<typename T>
-void decltype(tfoo<T>())::func() { // expected-error{{nested name specifier 'decltype(tfoo<T>())::' for declaration does not refer into a class, class template or class template partial specialization}}
+void decltype(tfoo<T>())::func() { // expected-error{{nested name specifier 'decltype(tfoo<T>())' for declaration does not refer into a class, class template or class template partial specialization}}
 }
 
 // An init-declarator named with a qualified-id can refer to an element of the
diff --git a/clang/test/CXX/drs/cwg0xx.cpp b/clang/test/CXX/drs/cwg0xx.cpp
index c035aa3f4ec61..282e71bbf3bda 100644
--- a/clang/test/CXX/drs/cwg0xx.cpp
+++ b/clang/test/CXX/drs/cwg0xx.cpp
@@ -806,7 +806,6 @@ namespace cwg49 { // cwg49: 2.8
   // since-cxx17-error@#cwg49-c {{non-type template argument is not a constant expression}}
   //   since-cxx17-note@#cwg49-c {{read of non-constexpr variable 'q' is not allowed in a constant expression}}
   //   since-cxx17-note@#cwg49-q {{declared here}}
-  //   since-cxx17-note@#cwg49-A {{template parameter is declared here}}
 } // namespace cwg49
 
 namespace cwg50 { // cwg50: 2.7
@@ -1019,9 +1018,9 @@ namespace cwg62 { // cwg62: 2.9
   struct A {
     struct { int n; } b;
   };
-  template<typename T> struct X {}; // #cwg62-X
-  template<typename T> T get() { return get<T>(); } // #cwg62-get
-  template<typename T> int take(T) { return 0; } // #cwg62-take
+  template<typename T> struct X {};
+  template<typename T> T get() { return get<T>(); }
+  template<typename T> int take(T) { return 0; }
 
   X<A> x1;
   A a = get<A>();
@@ -1035,27 +1034,22 @@ namespace cwg62 { // cwg62: 2.9
 
   X<NoNameForLinkagePtr> x2;
   // cxx98-error@-1 {{template argument uses unnamed type}}
-  //   cxx98-note@#cwg62-X {{template parameter is declared here}}
   //   cxx98-note@#cwg62-unnamed {{unnamed type used in template argument was declared here}}
   X<const NoNameForLinkagePtr> x3;
   // cxx98-error@-1 {{template argument uses unnamed type}}
-  //   cxx98-note@#cwg62-X {{template parameter is declared here}}
   //   cxx98-note@#cwg62-unnamed {{unnamed type used in template argument was declared here}}
   NoNameForLinkagePtr p1 = get<NoNameForLinkagePtr>();
   // cxx98-error@-1 {{template argument uses unnamed type}}
-  //   cxx98-note@#cwg62-get {{template parameter is declared here}}
   //   cxx98-note@#cwg62-unnamed {{unnamed type used in template argument was declared here}}
-  //   cxx98-note@-4 {{while substituting explicitly-specified template arguments}}
+  //   cxx98-note@-3 {{while substituting explicitly-specified template arguments}}
   NoNameForLinkagePtr p2 = get<const NoNameForLinkagePtr>();
   // cxx98-error@-1 {{template argument uses unnamed type}}
-  //   cxx98-note@#cwg62-get {{template parameter is declared here}}
   //   cxx98-note@#cwg62-unnamed {{unnamed type used in template argument was declared here}}
-  //   cxx98-note@-4 {{while substituting explicitly-specified template arguments}}
+  //   cxx98-note@-3 {{while substituting explicitly-specified template arguments}}
   int n1 = take(noNameForLinkagePtr);
   // cxx98-error@-1 {{template argument uses unnamed type}}
-  //   cxx98-note@#cwg62-take {{template parameter is declared here}}
   //   cxx98-note@#cwg62-unnamed {{unnamed type used in template argument was declared here}}
-  //   cxx98-note@-4 {{while substituting deduced template arguments}}
+  //   cxx98-note@-3 {{while substituting deduced template arguments}}
 
   X<Danger> x4;
 
@@ -1063,24 +1057,18 @@ namespace cwg62 { // cwg62: 2.9
     struct NoLinkage {};
     X<NoLinkage> a;
     // cxx98-error@-1 {{template argument uses local type }}
-    //   cxx98-note@#cwg62-X {{template parameter is declared here}}
     X<const NoLinkage> b;
     // cxx98-error@-1 {{template argument uses local type }}
-    //   cxx98-note@#cwg62-X {{template parameter is declared here}}
     get<NoLinkage>();
     // cxx98-error@-1 {{template argument uses local type }}
-    //   cxx98-note@#cwg62-get {{template parameter is declared here}}
-    //   cxx98-note@-3 {{while substituting explicitly-specified template arguments}}
+    //   cxx98-note@-2 {{while substituting explicitly-specified template arguments}}
     get<const NoLinkage>();
     // cxx98-error@-1 {{template argument uses local type }}
-    //   cxx98-note@#cwg62-get {{template parameter is declared here}}
-    //   cxx98-note@-3 {{while substituting explicitly-specified template arguments}}
+    //   cxx98-note@-2 {{while substituting explicitly-specified template arguments}}
     X<void (*)(NoLinkage A::*)> c;
     // cxx98-error@-1 {{template argument uses local type }}
-    //   cxx98-note@#cwg62-X {{template parameter is declared here}}
     X<int NoLinkage::*> d;
     // cxx98-error@-1 {{template argument uses local type }}
-    //   cxx98-note@#cwg62-X {{template parameter is declared here}}
   }
 } // namespace cwg62
 
@@ -1147,11 +1135,10 @@ namespace cwg69 { // cwg69: 9
   extern template void f<char>();
   // cxx98-error@-1 {{extern templates are a C++11 extension}}
   // expected-error@-2 {{explicit instantiation declaration of 'f' with internal linkage}}
-  template<void(*)()> struct Q {}; // #cwg69-Q
+  template<void(*)()> struct Q {};
   Q<&f<int> > q;
   // cxx98-error@-1 {{non-type template argument referring to function 'f<int>' with internal linkage is a C++11 extension}}
   //   cxx98-note@#cwg69-f {{non-type template argument refers to function here}}
-  //   cxx98-note@#cwg69-Q {{template parameter is declared here}}
 } // namespace cwg69
 
 namespace cwg70 { // cwg70: 2.7
diff --git a/clang/test/CXX/drs/cwg10xx.cpp b/clang/test/CXX/drs/cwg10xx.cpp
index 10ee50558c7e8..c5b96c4ab8ffc 100644
--- a/clang/test/CXX/drs/cwg10xx.cpp
+++ b/clang/test/CXX/drs/cwg10xx.cpp
@@ -43,7 +43,6 @@ namespace cwg1004 { // cwg1004: 5
   template<class T, template<class> class U = T::template A> struct Third { };
   // expected-error@-1 {{is a constructor name}}
   //   expected-note@#cwg1004-t {{in instantiation of default argument}}
-  //   expected-note@-3 {{template parameter is declared here}}
   Third<A<int> > t; // #cwg1004-t
 } // namespace cwg1004
 
diff --git a/clang/test/CXX/drs/cwg13xx.cpp b/clang/test/CXX/drs/cwg13xx.cpp
index 8a9f285673699..9c72fefb5b65c 100644
--- a/clang/test/CXX/drs/cwg13xx.cpp
+++ b/clang/test/CXX/drs/cwg13xx.cpp
@@ -180,7 +180,6 @@ namespace cwg1315 { // cwg1315: partial
   // dependent type of T::value is not the same as 'int'.
   // A core issue will be opened to decide what is supposed to happen here.
   template <typename T, int I> struct C;
-  // expected-note@-1 {{template parameter is declared here}}
   template <typename T> struct C<T, T::value>;
   // expected-error@-1 {{type of specialized non-type template argument depends on a template parameter of the partial specialization}}
 } // namespace cwg1315
diff --git a/clang/test/CXX/drs/cwg14xx.cpp b/clang/test/CXX/drs/cwg14xx.cpp
index 759a402b5f9d6..26febd9715184 100644
--- a/clang/test/CXX/drs/cwg14xx.cpp
+++ b/clang/test/CXX/drs/cwg14xx.cpp
@@ -660,14 +660,14 @@ namespace cwg1487 { // cwg1487: 3.3
 struct A { // #cwg1482-A
   struct B {
     using A::A;
-    // since-cxx11-error@-1 {{using declaration refers into 'A::', which is not a base class of 'B'}}
+    // since-cxx11-error@-1 {{using declaration refers into 'A', which is not a base class of 'B'}}
   };
 
   struct C : A {
   // since-cxx11-error@-1 {{base class has incomplete type}}
   //   since-cxx11-note@#cwg1482-A {{definition of 'cwg1487::A' is not complete until the closing '}'}}
     using A::A;
-    // since-cxx11-error@-1 {{using declaration refers into 'A::', which is not a base class of 'C'}}
+    // since-cxx11-error@-1 {{using declaration refers into 'A', which is not a base class of 'C'}}
   };
 
   struct D;
diff --git a/clang/test/CXX/drs/cwg18xx.cpp b/clang/test/CXX/drs/cwg18xx.cpp
index 5f40fe469b01b..5b4551ba0143b 100644
--- a/clang/test/CXX/drs/cwg18xx.cpp
+++ b/clang/test/CXX/drs/cwg18xx.cpp
@@ -26,7 +26,6 @@ S<i> V; // #cwg1801-S-i
 // cxx98-14-error@-1 {{non-type template argument does not refer to any declaration}}
 //   cxx98-14-note@#cwg1801-S {{template parameter is declared here}}
 // cxx17-error@#cwg1801-S-i {{non-type template argument refers to subobject '.i'}}
-//   cxx17-note@#cwg1801-S {{template parameter is declared here}}
 } // namespace cwg1801
 
 namespace cwg1802 { // cwg1802: 3.1
@@ -421,25 +420,25 @@ class C {
 
   template<class T>
   friend struct A<T>::B;
-  // expected-warning@-1 {{dependent nested name specifier 'A<T>::' for friend class declaration is not supported; turning off access control for 'C'}}
+  // expected-warning@-1 {{dependent nested name specifier 'A<T>' for friend class declaration is not supported; turning off access control for 'C'}}
 
   template<class T>
   friend void A<T>::f();
-  // expected-warning@-1 {{dependent nested name specifier 'A<T>::' for friend class declaration is not supported; turning off access control for 'C'}}
+  // expected-warning@-1 {{dependent nested name specifier 'A<T>' for friend class declaration is not supported; turning off access control for 'C'}}
 
   // FIXME: this is ill-formed, because A<T>​::​D does not end with a simple-template-id
   template<class T>
   friend void A<T>::D::g();
-  // expected-warning@-1 {{dependent nested name specifier 'A<T>::D::' for friend class declaration is not supported; turning off access control for 'C'}}
+  // expected-warning@-1 {{dependent nested name specifier 'A<T>::D' for friend class declaration is not supported; turning off access control for 'C'}}
 
   template<class T>
   friend int *A<T*>::h();
-  // expected-warning@-1 {{dependent nested name specifier 'A<T *>::' for friend class declaration is not supported; turning off access control for 'C'}}
+  // expected-warning@-1 {{dependent nested name specifier 'A<T *>' for friend class declaration is not supported; turning off access control for 'C'}}
 
   template<class T>
   template<T U>
   friend T A<T>::i();
-  // expected-warning@-1 {{dependent nested name specifier 'A<T>::' for friend class declaration is not supported; turning off access control for 'C'}}
+  // expected-warning@-1 {{dependent nested name specifier 'A<T>' for friend class declaration is not supported; turning off access control for 'C'}}
 };
 
 C c;
diff --git a/clang/test/CXX/drs/cwg19xx.cpp b/clang/test/CXX/drs/cwg19xx.cpp
index 55a7c7cbba66f..15ad3921cb1be 100644
--- a/clang/test/CXX/drs/cwg19xx.cpp
+++ b/clang/test/CXX/drs/cwg19xx.cpp
@@ -18,7 +18,7 @@ struct A {
 };
 }
 int N::f() { return 0; }
-int N::g() { return 0; } 
+int N::g() { return 0; }
 // expected-error@-1 {{out-of-line definition of 'g' does not match any declaration in namespace 'cwg1900::N'}}
 } // namespace cwg1900
 
@@ -84,7 +84,7 @@ namespace cwg1909 { // cwg1909: 3.7
   };
   struct C {
     template<typename T> static int C;
-    // expected-error@-1 {{member 'C' has the same name as its class}} 
+    // expected-error@-1 {{member 'C' has the same name as its class}}
     // cxx98-11-error@-2 {{variable templates are a C++14 extension}}
   };
   struct D {
@@ -105,7 +105,7 @@ class X {
   // FIXME: this is ill-formed, because A<T>::B::C does not end with a simple-template-id
   template <typename T>
   friend class A<T>::B::C;
-  // expected-warning@-1 {{dependent nested name specifier 'A<T>::B::' for friend class declaration is not supported; turning off access control for 'X'}}
+  // expected-warning@-1 {{dependent nested name specifier 'A<T>::B' for friend class declaration is not supported; turning off access control for 'X'}}
 };
 template<> struct A<int> {
   typedef struct Q B;
@@ -170,7 +170,7 @@ class X {
   // FIXME: this is ill-formed, because A<T>::B::C does not end with a simple-template-id
   template <typename T>
   friend class A<T>::B::C;
-  // expected-warning@-1 {{dependent nested name specifier 'A<T>::B::' for friend class declaration is not supported; turning off access control for 'X'}}
+  // expected-warning@-1 {{dependent nested name specifier 'A<T>::B' for friend class declaration is not supported; turning off access control for 'X'}}
 };
 } // namespace cwg1945
 
diff --git a/clang/test/CXX/drs/cwg1xx.cpp b/clang/test/CXX/drs/cwg1xx.cpp
index 51f320fc43cfa..4a5394be146bc 100644
--- a/clang/test/CXX/drs/cwg1xx.cpp
+++ b/clang/test/CXX/drs/cwg1xx.cpp
@@ -26,22 +26,18 @@ namespace cwg100 { // cwg100: 2.7
   // cxx98-14-error@#cwg100-a {{non-type template argument does not refer to any declaration}}
   //   cxx98-14-note@#cwg100-A {{template parameter is declared here}}
   // since-cxx17-error@#cwg100-a {{pointer to string literal is not allowed in a template argument}}
-  //   since-cxx17-note@#cwg100-A {{template parameter is declared here}}
   B<"bar"> b; // #cwg100-b
   // cxx98-14-error@#cwg100-b {{non-type template argument does not refer to any declaration}}
   //   cxx98-14-note@#cwg100-B {{template parameter is declared here}}
   // since-cxx17-error@#cwg100-b {{reference to string literal is not allowed in a template argument}}
-  //   since-cxx17-note@#cwg100-B {{template parameter is declared here}}
   C<"baz"> c; // #cwg100-c
   // cxx98-14-error@#cwg100-c {{non-type template argument does not refer to any declaration}}
   //   cxx98-14-note@#cwg100-C {{template parameter is declared here}}
   // since-cxx17-error@#cwg100-c {{pointer to subobject of string literal is not allowed in a template argument}}
-  //   since-cxx17-note@#cwg100-C {{template parameter is declared here}}
   D<*"quux"> d; // #cwg100-d
   // cxx98-14-error@#cwg100-d {{non-type template argument does not refer to any declaration}}
   //   cxx98-14-note@#cwg100-D {{template parameter is declared here}}
   // since-cxx17-error@#cwg100-d {{reference to subobject of string literal is not allowed in a template argument}}
-  //   since-cxx17-note@#cwg100-D {{template parameter is declared here}}
 } // namespace cwg100
 
 namespace cwg101 { // cwg101: 3.5
@@ -100,7 +96,7 @@ namespace cwg108 { // cwg108: 2.9
   template<typename T> struct A {
     struct B { typedef int X; };
     B::X x;
-    // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name B::X; implicit 'typename' is a C++20 extension}}
+    // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name 'B::X'; implicit 'typename' is a C++20 extension}}
     struct C : B { X x; };
     // expected-error@-1 {{unknown type name 'X'}}
   };
@@ -156,17 +152,15 @@ namespace cwg112 { // cwg112: 3.1
   volatile T a2[1] = {};
   const Arr a3 = {}; // #cwg112-a3
   volatile Arr a4 = {};
-  template<const volatile T*> struct X {}; // #cwg112-X
+  template<const volatile T*> struct X {};
   // FIXME: Test this somehow in C++11 and on.
   X<a1> x1;
   // cxx98-error@-1 {{non-type template argument referring to object 'a1' with internal linkage is a C++11 extension}}
   //   cxx98-note@#cwg112-a1 {{non-type template argument refers to object here}}
-  //   cxx98-note@#cwg112-X {{template parameter is declared here}}
   X<a2> x2;
   X<a3> x3;
   // cxx98-error@-1 {{non-type template argument referring to object 'a3' with internal linkage is a C++11 extension}}
   //   cxx98-note@#cwg112-a3 {{non-type template argument refers to object here}}
-  //   cxx98-note@#cwg112-X {{template parameter is declared here}}
   X<a4> x4;
 } // namespace cwg112
 
@@ -327,7 +321,7 @@ namespace cwg121 { // cwg121: 2.7
     X::Y<T> x;
     T::Y<T> y;
     // expected-error@-1 {{use 'template' keyword to treat 'Y' as a dependent template name}}
-    // cxx98-17-error@-2 {{missing 'typename' prior to dependent type name T::Y; implicit 'typename' is a C++20 extension}}
+    // cxx98-17-error@-2 {{missing 'typename' prior to dependent type name 'T::Y'; implicit 'typename' is a C++20 extension}}
   };
   Z<X> z;
 } // namespace cwg121
@@ -640,7 +634,7 @@ namespace example3 {
 struct Base {
 private:
   static const int i = 10; // #cwg138-ex3-Base-i
-
+  
 public:
   struct Data;
   // Elaborated type specifier is not the sole constituent of declaration,
@@ -654,7 +648,7 @@ struct Base {
   };
 };
 struct Data {
-  void f() {
+  void f() {  
     int i2 = Base::i;
     // expected-error@-1 {{'i' is a private member of 'cwg138::example3::Base'}}
     //   expected-note@#cwg138-ex3-Base-i {{declared private here}}
@@ -1315,8 +1309,8 @@ namespace cwg184 { // cwg184: 2.7
 
   template<template<typename TT> class T> void A<T>::f() { // #cwg184-T
     T<> t;
-    // expected-error@-1 {{missing template argument for template parameter}}
-    //   expected-note@#cwg184-T {{template parameter is declared here}}
+    // expected-error@-1 {{too few template arguments for template template parameter 'T'}}
+    //   expected-note@#cwg184-T {{template is declared here}}
   }
 
   template<template<typename TT = char> class T> void A<T>::g() {
diff --git a/clang/test/CXX/drs/cwg20xx.cpp b/clang/test/CXX/drs/cwg20xx.cpp
index fd31a51f79ec5..141a1012aef93 100644
--- a/clang/test/CXX/drs/cwg20xx.cpp
+++ b/clang/test/CXX/drs/cwg20xx.cpp
@@ -27,7 +27,7 @@ int b = __builtin_addressof(b2)->foo;
 // cwg2009: na
 
 namespace cwg2026 { // cwg2026: 11
-  template<int> struct X {}; // #cwg2026-X
+  template<int> struct X {};
 
   const int a = a + 1; // #cwg2026-a
   // expected-warning@-1 {{variable 'a' is uninitialized when used within its own initialization}}
@@ -35,11 +35,9 @@ namespace cwg2026 { // cwg2026: 11
   // cxx98-error@-1 {{non-type template argument of type 'int' is not an integral constant expression}}
   //   cxx98-note@-2 {{initializer of 'a' is not a constant expression}}
   //   cxx98-note@#cwg2026-a {{declared here}}
-  //   cxx98-note@#cwg2026-X {{template parameter is declared here}}
   // since-cxx11-error@#cwg2026-xa {{non-type template argument is not a constant expression}}
   //   since-cxx11-note@#cwg2026-xa {{initializer of 'a' is not a constant expression}}
   //   since-cxx11-note@#cwg2026-a {{declared here}}
-  //   since-cxx11-note@#cwg2026-X {{template parameter is declared here}}
 
 #if __cplusplus >= 201103L
   constexpr int b = b;
@@ -67,11 +65,9 @@ namespace cwg2026 { // cwg2026: 11
     // cxx98-error@-1 {{non-type template argument of type 'int' is not an integral constant expression}}
     //   cxx98-note@-2 {{initializer of 'e' is not a constant expression}}
     //   cxx98-note@#cwg2026-e {{declared here}}
-    //   cxx98-note@#cwg2026-X {{template parameter is declared here}}
     // since-cxx11-error@#cwg2026-xe {{non-type template argument is not a constant expression}}
     //   since-cxx11-note@#cwg2026-xe {{initializer of 'e' is not a constant expression}}
     //   since-cxx11-note@#cwg2026-e {{declared here}}
-    //   since-cxx11-note@#cwg2026-X {{template parameter is declared here}}
 
 #if __cplusplus >= 201103L
     static constexpr int f = f;
@@ -153,7 +149,7 @@ namespace cwg2076 { // cwg2076: 13
     operator string_view() const;
   };
 
-  void foo(const string &); // #cwg2076-foo
+  void foo(const string &); // #cwg2076-foo 
   void bar(string_view); // #cwg2076-bar
 
   void func(const string &arg) {
@@ -433,7 +429,7 @@ int f()
   return 0;
 }
 } // namespace GH42233
-} // namespace cwg2091
+} // namespace cwg2091 
 
 namespace cwg2094 { // cwg2094: 5
   struct A { int n; };
diff --git a/clang/test/CXX/drs/cwg21xx.cpp b/clang/test/CXX/drs/cwg21xx.cpp
index 97bf320658759..42a7c4d7bbded 100644
--- a/clang/test/CXX/drs/cwg21xx.cpp
+++ b/clang/test/CXX/drs/cwg21xx.cpp
@@ -24,7 +24,7 @@ namespace std {
 }
 
 namespace cwg2100 { // cwg2100: 12
-  template<const int *P, bool = true> struct X {}; // #cwg2100-X
+  template<const int *P, bool = true> struct X {};
   template<typename T> struct A {
     static const int n = 1;
     int f() {
@@ -35,7 +35,6 @@ namespace cwg2100 { // cwg2100: 12
       return X<&n>::n; // ok, value-dependent
       // cxx98-14-error@-1 {{non-type template argument refers to object 'n' that does not have linkage}}
       //   cxx98-14-note@#cwg2100-n {{non-type template argument refers to object here}}
-      //   cxx98-14-note@#cwg2100-X {{template parameter is declared here}}
     }
   };
   template<const int *P> struct X<P> {
diff --git a/clang/test/CXX/drs/cwg22xx.cpp b/clang/test/CXX/drs/cwg22xx.cpp
index d93070ef3804d..8c8ad9f7f74ee 100644
--- a/clang/test/CXX/drs/cwg22xx.cpp
+++ b/clang/test/CXX/drs/cwg22xx.cpp
@@ -1,10 +1,10 @@
 // RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
-// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
-// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
-// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
-// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors
 
 
 namespace cwg2211 { // cwg2211: 8
@@ -196,6 +196,17 @@ void g() {
 #endif
 } // namespace cwg2277
 
+namespace cwg2285 { // cwg2285: 4
+// Note: Clang 4 implements this DR but it set a wrong value of `__cplusplus`
+#if __cplusplus >= 201703L
+  void test() {
+    using T = int[1];
+    auto [a] = T{a};
+    // since-cxx17-error@-1 {{binding 'a' cannot appear in the initializer of its own decomposition declaration}}
+  }
+#endif
+} // namespace cwg2285
+
 namespace cwg2292 { // cwg2292: 9
 #if __cplusplus >= 201103L
   template<typename T> using id = T;
diff --git a/clang/test/CXX/drs/cwg28xx.cpp b/clang/test/CXX/drs/cwg28xx.cpp
index caa9b0f1f5058..b32e649374893 100644
--- a/clang/test/CXX/drs/cwg28xx.cpp
+++ b/clang/test/CXX/drs/cwg28xx.cpp
@@ -150,7 +150,7 @@ struct A {
   // FIXME: The index of the pack-index-specifier is printed as a memory address in the diagnostic.
   template<typename U>
   friend struct Ts...[0]::C;
-  // since-cxx26-warning@-1 {{dependent nested name specifier 'Ts...[0]::' for friend template declaration is not supported; ignoring this friend declaration}}
+  // since-cxx26-warning@-1 {{dependent nested name specifier 'Ts...[0]' for friend template declaration is not supported; ignoring this friend declaration}}
 };
 
 #endif
diff --git a/clang/test/CXX/drs/cwg2xx.cpp b/clang/test/CXX/drs/cwg2xx.cpp
index 0d644bae78382..4f383aacd4532 100644
--- a/clang/test/CXX/drs/cwg2xx.cpp
+++ b/clang/test/CXX/drs/cwg2xx.cpp
@@ -426,7 +426,7 @@ namespace cwg224 { // cwg224: 16
       A::type a;
       A<T>::type b;
       A<T*>::type c;
-      // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name A<T *>::type; implicit 'typename' is a C++20 extension}}
+      // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name 'A<T *>::type'; implicit 'typename' is a C++20 extension}}
       ::cwg224::example1::A<T>::type d;
 
       class B {
@@ -435,13 +435,13 @@ namespace cwg224 { // cwg224: 16
         A::type a;
         A<T>::type b;
         A<T*>::type c;
-        // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name A<T *>::type; implicit 'typename' is a C++20 extension}}
+        // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name 'A<T *>::type'; implicit 'typename' is a C++20 extension}}
         ::cwg224::example1::A<T>::type d;
 
         B::type e;
         A<T>::B::type f;
         A<T*>::B::type g;
-        // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name A<T *>::B::type; implicit 'typename' is a C++20 extension}}
+        // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name 'A<T *>::B::type'; implicit 'typename' is a C++20 extension}}
         typename A<T*>::B::type h;
       };
     };
@@ -450,25 +450,25 @@ namespace cwg224 { // cwg224: 16
       typedef int type;
       A<T*>::type a;
       A<T>::type b;
-      // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name A<T>::type; implicit 'typename' is a C++20 extension}}
+      // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name 'A<T>::type'; implicit 'typename' is a C++20 extension}}
     };
 
     template <class T1, class T2, int I> struct B {
       typedef int type;
       B<T1, T2, I>::type b1;
       B<T2, T1, I>::type b2;
-      // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name B<T2, T1, I>::type; implicit 'typename' is a C++20 extension}}
+      // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name 'B<T2, T1, I>::type'; implicit 'typename' is a C++20 extension}}
 
       typedef T1 my_T1;
       static const int my_I = I;
       static const int my_I2 = I+0;
       static const int my_I3 = my_I;
       B<my_T1, T2, my_I>::type b3;
-      // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name B<my_T1, T2, my_I>::type; implicit 'typename' is a C++20 extension}}
+      // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name 'B<my_T1, T2, my_I>::type'; implicit 'typename' is a C++20 extension}}
       B<my_T1, T2, my_I2>::type b4;
-      // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name B<my_T1, T2, my_I2>::type; implicit 'typename' is a C++20 extension}}
+      // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name 'B<my_T1, T2, my_I2>::type'; implicit 'typename' is a C++20 extension}}
       B<my_T1, T2, my_I3>::type b5;
-      // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name B<my_T1, T2, my_I3>::type; implicit 'typename' is a C++20 extension}}
+      // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name 'B<my_T1, T2, my_I3>::type'; implicit 'typename' is a C++20 extension}}
     };
   }
 
@@ -480,7 +480,7 @@ namespace cwg224 { // cwg224: 16
       X<A::i, char>::type x;
       X<A<T>::i, double>::type y;
       X<A<T*>::i, long>::type z;
-      // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name X<A<T *>::i, long>::type; implicit 'typename' is a C++20 extension}}
+      // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name 'X<A<T *>::i, long>::type'; implicit 'typename' is a C++20 extension}}
       int f();
     };
     template <class T> int A<T>::f() {
@@ -867,7 +867,7 @@ namespace cwg252 { // cwg252: 3.1
   struct E {
     void operator delete(void*, int);
     void operator delete(void*) = delete; // #cwg252-E
-    // cxx98-error@-1 {{deleted function definitions are a C++11 extension}} 
+    // cxx98-error@-1 {{deleted function definitions are a C++11 extension}}
     virtual ~E();
   };
   E::~E() {}
diff --git a/clang/test/CXX/drs/cwg3xx.cpp b/clang/test/CXX/drs/cwg3xx.cpp
index a616120b82063..164cc26ae585a 100644
--- a/clang/test/CXX/drs/cwg3xx.cpp
+++ b/clang/test/CXX/drs/cwg3xx.cpp
@@ -317,20 +317,18 @@ namespace cwg319 { // cwg319: no
   typedef int (*pa)[n1];
   pa parr; // ok, type has linkage despite using 'n1'
 
-  template<typename> struct X {}; // #cwg319-X
+  template<typename> struct X {};
 
   void f() {
     struct A { int n; };
     extern A a; // FIXME: ill-formed
     X<A> xa;
     // cxx98-error@-1 {{template argument uses local type 'A'}}
-    //   cxx98-note@#cwg319-X {{template parameter is declared here}}
 
     typedef A B;
     extern B b; // FIXME: ill-formed
     X<B> xb;
     // cxx98-error@-1 {{template argument uses local type 'A'}}
-    //   cxx98-note@#cwg319-X {{template parameter is declared here}}
 
     const int n = 1;
     typedef int (*C)[n];
@@ -605,7 +603,7 @@ namespace cwg336 { // cwg336: 2.7
     // expected-error@-1 {{out-of-line definition of 'mf1' does not match any declaration in 'cwg336::Pre::A<int>::B<double>'}}
     //   expected-note@#cwg336-B {{defined here}}
     template<class Y> template<> void A<Y>::B<double>::mf2() {}
-    // expected-error@-1 {{nested name specifier 'A<Y>::B<double>::' for declaration does not refer into a class, class template or class template partial specialization}}
+    // expected-error@-1 {{nested name specifier 'A<Y>::B<double>' for declaration does not refer into a class, class template or class template partial specialization}}
   }
   namespace Post {
     template<class T1> class A {
@@ -620,7 +618,7 @@ namespace cwg336 { // cwg336: 2.7
     template<> template<> template<class T> void A<int>::B<double>::mf1(T t) {}
     // FIXME: This diagnostic isn't very good.
     template<class Y> template<> void A<Y>::B<double>::mf2() {}
-    // expected-error@-1 {{nested name specifier 'A<Y>::B<double>::' for declaration does not refer into a class, class template or class template partial specialization}}
+    // expected-error@-1 {{nested name specifier 'A<Y>::B<double>' for declaration does not refer into a class, class template or class template partial specialization}}
   }
 } // namespace cwg336
 
@@ -999,7 +997,6 @@ namespace cwg354 { // cwg354: 3.1 c++11
   //   since-cxx17-note@#cwg354-ptr_mem {{template parameter is declared here}}
   ptr_mem<(int S::*)0> m1;
   // cxx98-error@-1 {{non-type template argument is not a pointer to member constant}}
-  //   cxx98-note@#cwg354-ptr_mem {{template parameter is declared here}}
   ptr_mem<(float S::*)0> m2; // #cwg354-m2
   // cxx98-error@#cwg354-m2 {{non-type template argument of type 'float S::*' cannot be converted to a value of type 'int S::*'}}
   //   cxx98-note@#cwg354-ptr_mem {{template parameter is declared here}}
@@ -1505,7 +1502,7 @@ namespace cwg389 { // cwg389: no
     typedef struct {} const C; // #cwg389-C
     typedef enum {} const D; // #cwg389-D
   };
-  template<typename> struct T {}; // #cwg389-T
+  template<typename> struct T {};
 
   struct WithLinkage1 {};
   enum WithLinkage2 {};
@@ -1546,23 +1543,18 @@ namespace cwg389 { // cwg389: no
 
   typedef T<WithoutLinkage1> BadArg1;
   // expected-error@-1 {{template argument uses unnamed type}}
-  //   expected-note@#cwg389-T {{template parameter is declared here}}
   //   expected-note@#cwg389-no-link-1 {{unnamed type used in template argument was declared here}}
   typedef T<WithoutLinkage2> BadArg2;
   // expected-error@-1 {{template argument uses unnamed type}}
-  //   expected-note@#cwg389-T {{template parameter is declared here}}
   //   expected-note@#cwg389-no-link-2 {{unnamed type used in template argument was declared here}}
   typedef T<WithoutLinkage3> BadArg3;
   // expected-error@-1 {{template argument uses unnamed type}}
-  //   expected-note@#cwg389-T {{template parameter is declared here}}
   //   expected-note@#cwg389-C {{unnamed type used in template argument was declared here}}
   typedef T<WithoutLinkage4> BadArg4;
   // expected-error@-1 {{template argument uses unnamed type}}
-  //   expected-note@#cwg389-T {{template parameter is declared here}}
   //   expected-note@#cwg389-D {{unnamed type used in template argument was declared here}}
   typedef T<WithoutLinkage5> BadArg5;
   // expected-error@-1 {{template argument uses unnamed type}}
-  //   expected-note@#cwg389-T {{template parameter is declared here}}
   //   expected-note@#cwg389-C {{unnamed type used in template argument was declared here}}
 #endif
 
diff --git a/clang/test/CXX/drs/cwg4xx.cpp b/clang/test/CXX/drs/cwg4xx.cpp
index 78b03b3df2323..d76c6012b9195 100644
--- a/clang/test/CXX/drs/cwg4xx.cpp
+++ b/clang/test/CXX/drs/cwg4xx.cpp
@@ -43,15 +43,12 @@ namespace cwg401 { // cwg401: 2.8
   // expected-error@#cwg401-A {{'type' is a private member of 'cwg401::C'}}
   //   expected-note@#cwg402-friend-A-C {{in instantiation of default argument for 'A<C>' required here}}
   //   expected-note@#cwg402-C-type {{implicitly declared private here}}
-  //   expected-note@#cwg401-A {{template parameter is declared here}}
   // expected-error@#cwg401-A {{'type' is a protected member of 'cwg401::B'}}
   //   expected-note@#cwg402-b {{in instantiation of default argument for 'A<B>' required here}}
   //   expected-note@#cwg402-B-type {{declared protected here}}
-  //   expected-note@#cwg401-A {{template parameter is declared here}}
   // expected-error@#cwg401-A {{'type' is a private member of 'cwg401::D'}}
   //   expected-note@#cwg402-d {{in instantiation of default argument for 'A<D>' required here}}
   //   expected-note@#cwg402-D-type {{implicitly declared private here}}
-  //   expected-note@#cwg401-A {{template parameter is declared here}}
   class B {
   protected:
     typedef int type; // #cwg402-B-type
@@ -83,9 +80,8 @@ namespace cwg401 { // cwg401: 2.8
   // to not treat the default template argument as a SFINAE context in C++98.
   template<class T, class U = typename T::type> void f(T) {} // #cwg402-f
   // cxx98-error@-1 {{default template arguments for a function template are a C++11 extension}}
-  //   cxx98-note@-2 {{template parameter is declared here}}
-  // cxx98-error@-3 {{'type' is a protected member of 'cwg401::B'}}
-  //   cxx98-note@-4 {{in instantiation of default argument for 'f<B>' required here}}
+  // cxx98-error@-2 {{'type' is a protected member of 'cwg401::B'}}
+  //   cxx98-note@-3 {{in instantiation of default argument for 'f<B>' required here}}
   //   cxx98-note@#cwg402-f-b {{while substituting deduced template arguments into function template 'f' [with T = B, U = (no value)]}}
   //   cxx98-note@#cwg402-B-type {{declared protected here}}
   void g(B b) { f(b); } // #cwg402-f-b
@@ -261,7 +257,7 @@ namespace cwg409 { // cwg409: 2.7
     A::B b2;
     A<T>::B b3;
     A<T*>::B b4;
-    // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name A<T *>::B; implicit 'typename' is a C++20 extension}}
+    // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name 'A<T *>::B'; implicit 'typename' is a C++20 extension}}
   };
 } // namespace cwg409
 
@@ -640,17 +636,15 @@ namespace cwg431 { // cwg431: 2.8
 } // namespace cwg431
 
 namespace cwg432 { // cwg432: 3.0
-  template<typename T> struct A {}; // #cwg432-A
+  template<typename T> struct A {};
   template<typename T> struct B : A<B> {};
   // expected-error@-1 {{use of class template 'B' requires template arguments}}
-  //   expected-note@#cwg432-A {{template parameter is declared here}}
-  //   expected-note@-3 {{template is declared here}}
+  //   expected-note@-2 {{template is declared here}}
   template<typename T> struct C : A<C<T> > {};
 #if __cplusplus >= 201103L
   template<typename T> struct D : decltype(A<D>()) {};
   // since-cxx11-error@-1 {{use of class template 'D' requires template arguments}}
-  //   since-cxx11-note@#cwg432-A {{template parameter is declared here}}
-  //   since-cxx11-note@-3 {{template is declared here}}
+  //   since-cxx11-note@-2 {{template is declared here}}
 #endif
 } // namespace cwg432
 
@@ -1383,7 +1377,6 @@ namespace cwg487 { // cwg487: 2.7
 
 namespace cwg488 { // cwg488: 2.9 c++11
   template <typename T> void f(T);
-  // cxx98-note@-1 {{template parameter is declared here}}
   void f(int);
   void g() {
     // FIXME: It seems CWG thought this should be a SFINAE failure prior to
diff --git a/clang/test/CXX/drs/cwg5xx.cpp b/clang/test/CXX/drs/cwg5xx.cpp
index 1fdfe5785c5c4..0825b52653b4d 100644
--- a/clang/test/CXX/drs/cwg5xx.cpp
+++ b/clang/test/CXX/drs/cwg5xx.cpp
@@ -254,9 +254,9 @@ namespace cwg526 { // cwg526: 2.7
     typedef int type;
     X<N>::type v1;
     X<(N)>::type v2;
-    // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name X<(N)>::type; implicit 'typename' is a C++20 extension}}
+    // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name 'X<(N)>::type'; implicit 'typename' is a C++20 extension}}
     X<+N>::type v3;
-    // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name X<+N>::type; implicit 'typename' is a C++20 extension}}
+    // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name 'X<+N>::type'; implicit 'typename' is a C++20 extension}}
   };
 } // namespace cwg526
 
@@ -783,7 +783,7 @@ struct Outer {
 };
 template <class T>
 Outer<T>::Inner* Outer<T>::Inner::self() { return this; }
-// cxx98-17-error@-1 {{missing 'typename' prior to dependent type name Outer<T>::Inner; implicit 'typename' is a C++20 extension}}
+// cxx98-17-error@-1 {{missing 'typename' prior to dependent type name 'Outer<T>::Inner'; implicit 'typename' is a C++20 extension}}
 
 } // namespace cwg560
 
diff --git a/clang/test/CXX/drs/cwg6xx.cpp b/clang/test/CXX/drs/cwg6xx.cpp
index 11a77575c70c1..e2eb009508b52 100644
--- a/clang/test/CXX/drs/cwg6xx.cpp
+++ b/clang/test/CXX/drs/cwg6xx.cpp
@@ -79,11 +79,10 @@ namespace cwg602 { // cwg602: 2.7
 } // namespace cwg602
 
 namespace cwg603 { // cwg603: 3.1
-  template<unsigned char> struct S {}; // #cwg603-S
+  template<unsigned char> struct S {};
   typedef S<'\001'> S1;
   typedef S<(1ul << __CHAR_BIT__) + 1> S1;
   // since-cxx11-error@-1 {{non-type template argument evaluates to 257, which cannot be narrowed to type 'unsigned char'}}
-  //   since-cxx11-note@#cwg603-S {{template parameter is declared here}}
 } // namespace cwg603
 
 // cwg604: na
@@ -409,13 +408,13 @@ namespace cwg638 { // cwg638: no
   class X {
     typedef int type;
     template<class T> friend struct A<T>::B;
-    // expected-warning@-1 {{dependent nested name specifier 'A<T>::' for friend class declaration is not supported; turning off access control for 'X'}}
+    // expected-warning@-1 {{dependent nested name specifier 'A<T>' for friend class declaration is not supported; turning off access control for 'X'}}
     template<class T> friend void A<T>::f();
-    // expected-warning@-1 {{dependent nested name specifier 'A<T>::' for friend class declaration is not supported; turning off access control for 'X'}}
+    // expected-warning@-1 {{dependent nested name specifier 'A<T>' for friend class declaration is not supported; turning off access control for 'X'}}
     template<class T> friend void A<T>::g();
-    // expected-warning@-1 {{dependent nested name specifier 'A<T>::' for friend class declaration is not supported; turning off access control for 'X'}}
+    // expected-warning@-1 {{dependent nested name specifier 'A<T>' for friend class declaration is not supported; turning off access control for 'X'}}
     template<class T> friend void A<T>::C::h();
-    // expected-warning@-1 {{dependent nested name specifier 'A<T>::C::' for friend class declaration is not supported; turning off access control for 'X'}}
+    // expected-warning@-1 {{dependent nested name specifier 'A<T>::C' for friend class declaration is not supported; turning off access control for 'X'}}
   };
 
   template<> struct A<int> {
diff --git a/clang/test/CXX/expr/expr.const/p3-0x.cpp b/clang/test/CXX/expr/expr.const/p3-0x.cpp
index f40e1af14d111..3eedef3cf7712 100644
--- a/clang/test/CXX/expr/expr.const/p3-0x.cpp
+++ b/clang/test/CXX/expr/expr.const/p3-0x.cpp
@@ -4,7 +4,7 @@
 // A converted constant expression of type T is a core constant expression,
 int nonconst = 8; // expected-note 3 {{here}}
 enum NonConstE : unsigned char { NCE = nonconst }; // expected-error {{enumerator value is not a constant expression}} expected-note {{read of non-const}}
-template<int = nonconst> struct NonConstT {}; // expected-error {{non-type template argument is not a constant expression}} expected-note {{read of non-const}} expected-note {{template parameter is declared here}}
+template<int = nonconst> struct NonConstT {}; // expected-error {{non-type template argument is not a constant expression}} expected-note {{read of non-const}}
 void NonConstF() {
   switch (nonconst) {
     case nonconst: // expected-error {{case value is not a constant expression}} expected-note {{read of non-const}}
@@ -66,7 +66,7 @@ enum class EEE : unsigned short {
   e = 123456, // expected-error {{enumerator value evaluates to 123456, which cannot be narrowed to type 'unsigned short'}}
   f = -3 // expected-error {{enumerator value evaluates to -3, which cannot be narrowed to type 'unsigned short'}}
 };
-template<unsigned char> using A = int; // expected-note 4{{template parameter is declared here}}
+template<unsigned char> using A = int; // cxx17-note 2{{template parameter is declared here}}
 
 using Int = A<E6>;
 using Int = A<EE::EE32>; // expected-error {{not implicitly convertible}}
@@ -79,8 +79,7 @@ using Int = A<-3>; // expected-error {{template argument evaluates to -3, which
 // integral conversions as well as boolean conversions.
 // FIXME: Per core issue 1407, this is not correct.
 template<typename T, T v> struct Val { static constexpr T value = v; };
-// cxx17-note@-1 1{{template parameter is declared here}}
-// expected-note@-2 2{{template parameter is declared here}}
+// cxx17-note@-1 2{{template parameter is declared here}}
 static_assert(Val<bool, E1>::value == 1, ""); // ok
 static_assert(Val<bool, '\0'>::value == 0, ""); // ok
 static_assert(Val<bool, U'\1'>::value == 1, ""); // ok
diff --git a/clang/test/CXX/expr/expr.prim/expr.prim.req/type-requirement.cpp b/clang/test/CXX/expr/expr.prim/expr.prim.req/type-requirement.cpp
index 2572e766cb263..5433cfb21955d 100644
--- a/clang/test/CXX/expr/expr.prim/expr.prim.req/type-requirement.cpp
+++ b/clang/test/CXX/expr/expr.prim/expr.prim.req/type-requirement.cpp
@@ -77,7 +77,7 @@ using r2i3 = r2<int, int>; // expected-error{{constraints not satisfied for clas
 namespace ns2 {
   template<typename T, typename U> struct identity {};
 
-  template<typename... Ts> requires requires { typename identity<Ts...>; } // expected-note 2{{because 'typename identity<Ts...>' would be invalid: missing template argument for template parameter}}
+  template<typename... Ts> requires requires { typename identity<Ts...>; } // expected-note 2{{because 'typename identity<Ts...>' would be invalid: too few template arguments for class template 'identity'}}
   struct r4 {};
 
   using r4i1 = r4<int>; // expected-error{{constraints not satisfied for class template 'r4' [with Ts = <int>]}}
diff --git a/clang/test/CXX/special/class.inhctor/elsewhere.cpp b/clang/test/CXX/special/class.inhctor/elsewhere.cpp
index f86f4b86faa7d..9aa310bb7f6ab 100644
--- a/clang/test/CXX/special/class.inhctor/elsewhere.cpp
+++ b/clang/test/CXX/special/class.inhctor/elsewhere.cpp
@@ -33,14 +33,14 @@ struct D1 : I1 {
 template<typename T> struct A {};
 
 template<typename T> struct B : A<bool>, A<char> {
-  using A<T>::A; // expected-error {{'A<double>::', which is not a base class of 'B<double>'}}
+  using A<T>::A; // expected-error {{'A<double>', which is not a base class of 'B<double>'}}
 };
 B<bool> bb;
 B<char> bc;
 B<double> bd; // expected-note {{here}}
 
 template<typename T> struct C : A<T> {
-  using A<bool>::A; // expected-error {{'A<bool>::', which is not a base class of 'C<char>'}}
+  using A<bool>::A; // expected-error {{'A<bool>', which is not a base class of 'C<char>'}}
 };
 C<bool> cb;
 C<char> cc; // expected-note {{here}}
diff --git a/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p1-11.cpp b/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p1-11.cpp
index 332f69bacb69e..692958ef565cf 100644
--- a/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p1-11.cpp
+++ b/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p1-11.cpp
@@ -9,7 +9,7 @@ template<int *ip> struct IP {  // expected-note 6 {{template parameter is declar
   IP<ip> *ip2;
 };
 
-template<int &ip> struct IR {}; // expected-note {{template parameter is declared here}}
+template<int &ip> struct IR {};
 
 constexpr std::nullptr_t get_nullptr() { return nullptr; }
 
diff --git a/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p1.cpp b/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p1.cpp
index e979051e23419..629000d88acc3 100644
--- a/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p1.cpp
+++ b/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p1.cpp
@@ -31,16 +31,16 @@ namespace non_type_tmpl_param {
 //      omitted if the name refers to a function or array and shall be omitted
 //      if the corresopnding template-parameter is a reference; or
 namespace addr_of_obj_or_func {
-  template <int* p> struct X0 { }; // expected-note 5{{here}} cxx17-note 2{{here}}
+  template <int* p> struct X0 { }; // expected-note 5{{here}}
 #if __cplusplus >= 201103L
   // precxx17-note@-2 2{{template parameter is declared here}}
 #endif
 
-  template <int (*fp)(int)> struct X1 { }; // cxx17-note {{here}} precxx17-note{{here}}
+  template <int (*fp)(int)> struct X1 { }; // cxx17-note {{here}}
 #if __cplusplus <= 199711L
   // precxx17-note@-2 {{here}}
 #endif
-  template <int &p> struct X2 { }; // expected-note 5{{here}}
+  template <int &p> struct X2 { }; // expected-note 4{{here}}
   template <const int &p> struct X2k { }; // expected-note {{here}}
   template <int (&fp)(int)> struct X3 { }; // expected-note 4{{here}}
 
@@ -180,7 +180,6 @@ namespace addr_of_obj_or_func {
 
 namespace bad_args {
   template <int* N> struct X0 { }; // precxx17-note 4{{template parameter is declared here}}
-                                   // cxx17-note@-1 3{{template parameter is declared here}}
   int i = 42;
   X0<&i + 2> x0a; // precxx17-error{{non-type template argument does not refer to any declaration}} \
                      cxx17-error {{non-type template argument is not a constant expression}} \
diff --git a/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p5.cpp b/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p5.cpp
index bb7512a95ed3b..034ad49d0715c 100644
--- a/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p5.cpp
+++ b/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p5.cpp
@@ -48,9 +48,9 @@ namespace pointer_to_object_parameters {
     X(int, int);
     operator int() const;
   };
-
-  template<X const *Ptr> struct A2; // expected-note 1-2{{template parameter is declared here}}
-
+  
+  template<X const *Ptr> struct A2; // expected-note 0-1{{template parameter is declared here}}
+  
   X *X_ptr; // expected-note 0-1{{declared here}}
   X an_X;
   X array_of_Xs[10];
@@ -131,16 +131,16 @@ namespace reference_parameters {
     S3<vi> s3v;
     S3<cvi> s3cv;
   }
-
+  
   namespace PR6250 {
     template <typename T, const T &ref> void inc() {
       ref++; // expected-error{{read-only variable is not assignable}}
     }
-
+  
     template<typename T, const T &ref> void bind() {
       T &ref2 = ref; // expected-error{{drops 'const' qualifier}}
     }
-
+    
     int counter;
     void test() {
       inc<int, counter>(); // expected-note{{instantiation of}}
@@ -213,7 +213,7 @@ namespace reference_to_function {
 //        (13.4).
 namespace pointer_to_member_function {
   struct X { };
-  struct Y : X {
+  struct Y : X { 
     int f(int);
     int g(int);
     int g(float);
diff --git a/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp b/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp
index 2638bef8f20a0..3caed045c6688 100644
--- a/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp
+++ b/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp
@@ -20,39 +20,39 @@ eval<E<int, float>> eE; // expected-error{{implicit instantiation of undefined t
 template<
   template <int ...N> // expected-error {{cannot be narrowed from type 'int' to 'short'}}
                       // expected-error@-1 {{conversion from 'int' to 'void *' is not allowed in a converted constant expression}}
-  class TT // expected-note 2{{template parameter is declared here}}
+  class TT // expected-note 2{{previous template template parameter is here}}
 > struct X0 { };
 
 template<int I, int J, int ...Rest> struct X0a;
 template<int ...Rest> struct X0b;
 template<int I, long J> struct X0c;
-template<int I, short J> struct X0d; // expected-note {{template parameter is declared here}}
-template<int I, void *J> struct X0e; // expected-note {{template parameter is declared here}}
+template<int I, short J> struct X0d;
+template<int I, void *J> struct X0e; // expected-note{{template parameter is declared here}}
 
 X0<X0a> inst_x0a;
 X0<X0b> inst_x0b;
 X0<X0c> inst_x0c;
-X0<X0d> inst_x0d; // expected-note {{template template argument is incompatible}}
-X0<X0e> inst_x0e; // expected-note {{template template argument is incompatible}}
+X0<X0d> inst_x0d; // expected-note {{has different template parameters}}
+X0<X0e> inst_x0e; // expected-note{{template template argument has different template parameters than its corresponding template template parameter}}
 
 template<typename T,
          template <T ...N> // expected-error {{conversion from 'short' to 'void *' is not allowed in a converted constant expression}}
                            // expected-error@-1 {{cannot be narrowed from type 'int' to 'short'}}
-         class TT // expected-note 2{{template parameter is declared here}}
+         class TT // expected-note 2{{previous template template parameter is here}}
 > struct X1 { };
 
 template<int I, int J, int ...Rest> struct X1a;
 template<long I, long ...Rest> struct X1b;
 template<short I, short J> struct X1c;
-template<short I, long J> struct X1d;  // expected-note {{template parameter is declared here}}
-template<short I, void *J> struct X1e; // expected-note {{template parameter is declared here}}
+template<short I, long J> struct X1d;
+template<short I, void *J> struct X1e; // expected-note{{template parameter is declared here}}
 
 X1<int, X1a> inst_x1a;
 X1<long, X1b> inst_x1b;
 X1<short, X1c> inst_x1c;
 X1<short, X1d> inst_sx1d;
-X1<int, X1d> inst_ix1d;  // expected-note {{template template argument is incompatible}}
-X1<short, X1e> inst_x1e; // expected-note {{template template argument is incompatible}}
+X1<int, X1d> inst_ix1d;  // expected-note {{has different template parameters}}
+X1<short, X1e> inst_x1e; // expected-note {{has different template parameters}}
 
 template <int> class X2; // expected-note{{template is declared here}} \
                          // expected-note{{template is declared here}}
diff --git a/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-2a.cpp b/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-2a.cpp
index 5570a7f9d75de..342ffba53dbfa 100644
--- a/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-2a.cpp
+++ b/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-2a.cpp
@@ -13,12 +13,12 @@ template<F> struct W { }; // #W
 S1<X> s11;
 S1<Y> s12;
 // expected-error@-1 {{template template argument 'Y' is more constrained than template template parameter 'P'}}
-// expected-note@#S1 {{template parameter is declared here}}
+// expected-note@#S1 {{'P' declared here}}
 // expected-note@#Y {{'Y' declared here}}
 S1<Z> s13;
 S1<W> s14;
 // expected-error@-1 {{template template argument 'W' is more constrained than template template parameter 'P'}}
-// expected-note@#S1 {{template parameter is declared here}}
+// expected-note@#S1 {{'P' declared here}}
 // expected-note@#W {{'W' declared here}}
 // expected-note@#F 1-2{{similar constraint expressions not considered equivalent}}
 // expected-note@#C 1-2{{similar constraint}}
@@ -43,12 +43,12 @@ template<template<typename T> requires C<T> class P> struct S4 { }; // #S4
 S4<X> s41;
 S4<Y> s42;
 // expected-error@-1 {{template template argument 'Y' is more constrained than template template parameter 'P'}}
-// expected-note@#S4 {{template parameter is declared here}}
+// expected-note@#S4 {{'P' declared here}}
 // expected-note@#Y {{'Y' declared here}}
 S4<Z> s43;
 S4<W> s44;
 // expected-error@-1 {{template template argument 'W' is more constrained than template template parameter 'P'}}
-// expected-note@#S4 {{template parameter is declared here}}
+// expected-note@#S4 {{'P' declared here}}
 // expected-note@#W {{'W' declared here}}
 
 template<template<typename T> requires C<T> typename U> struct S5 {
diff --git a/clang/test/CXX/temp/temp.arg/temp.arg.type/p2.cpp b/clang/test/CXX/temp/temp.arg/temp.arg.type/p2.cpp
index 65f3e178ac963..650f8585b115a 100644
--- a/clang/test/CXX/temp/temp.arg/temp.arg.type/p2.cpp
+++ b/clang/test/CXX/temp/temp.arg/temp.arg.type/p2.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -Wvla %s
-// RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx98 -std=c++98 %s
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++98 %s
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 %s
 
 template<class T> struct A {
@@ -13,8 +13,7 @@ template<typename T> struct B {
 };
 B<function> b; // expected-note{{instantiation of}}
 
-template <typename T> // #f0-temphead
-int f0(void *, const T&); // expected-note{{candidate template ignored: substitution failure}}
+template <typename T> int f0(void *, const T&); // expected-note{{candidate template ignored: substitution failure}}
 enum {e};
 // expected-note@-1 {{unnamed type used in template argument was declared here}}
 
@@ -23,7 +22,6 @@ void test_f0(int n) { // #here
 #if __cplusplus <= 199711L
   // expected-warning@-2 {{template argument uses unnamed type}}
   // expected-note@-3 {{while substituting deduced template arguments}}
-  // expected-note@#f0-temphead {{template parameter is declared here}}
 #endif
 
   int vla[n]; // expected-warning {{variable length arrays in C++ are a Clang extension}}
@@ -35,9 +33,9 @@ void test_f0(int n) { // #here
 }
 
 namespace N0 {
-  template <typename R, typename A1> void f0(R (*)(A1)); // #f0
-  template <typename T> int f1(T);                       // #f1-1
-  template <typename T, typename U> int f1(T, U);        // #f1-2
+  template <typename R, typename A1> void f0(R (*)(A1));
+  template <typename T> int f1(T);
+  template <typename T, typename U> int f1(T, U);
   enum {e1};
 #if __cplusplus <= 199711L
   // expected-note@-2 2{{unnamed type used in template argument was declared here}}
@@ -53,7 +51,7 @@ namespace N0 {
  // expected-note@-2 {{unnamed type used in template argument was declared here}}
 #endif
 
-  template<typename T> struct X; // cxx98-note {{template parameter is declared here}}
+  template<typename T> struct X;
   template<typename T> struct X<T*> { };
 
   void f() {
@@ -61,28 +59,24 @@ namespace N0 {
 #if __cplusplus <= 199711L
     // expected-warning@-2 {{template argument uses unnamed type}}
     // expected-note@-3 {{while substituting deduced template arguments}}
-    // expected-note@#f0 {{template parameter is declared here}}
 #endif
 
        &f1<__typeof__(e1)>);
 #if __cplusplus <= 199711L
  // expected-warning@-2 {{template argument uses unnamed type}}
  // expected-note@-3 {{while substituting explicitly-specified template arguments}}
- // expected-note@#f1-1 {{template parameter is declared here}}
 #endif
 
     int (*fp1)(int, __typeof__(e2)) = f1;
 #if __cplusplus <= 199711L
     // expected-warning@-2 {{template argument uses unnamed type}}
     // expected-note@-3 {{while substituting deduced template arguments}}
-    // expected-note@#f1-2 {{template parameter is declared here}}
 #endif
 
     f1(e2);
 #if __cplusplus <= 199711L
     // expected-warning@-2 {{template argument uses unnamed type}}
     // expected-note@-3 {{while substituting deduced template arguments}}
-    // expected-note@#f1-1 {{template parameter is declared here}}
 #endif
 
     f1(e2);
diff --git a/clang/test/CXX/temp/temp.decls/temp.class.spec/p8-1y.cpp b/clang/test/CXX/temp/temp.decls/temp.class.spec/p8-1y.cpp
index cf01e406c14ad..388a80ee765c8 100644
--- a/clang/test/CXX/temp/temp.decls/temp.class.spec/p8-1y.cpp
+++ b/clang/test/CXX/temp/temp.decls/temp.class.spec/p8-1y.cpp
@@ -25,7 +25,7 @@ template<typename Outer> struct X {
   template<typename Inner> static int y<Outer>; // expected-error 3{{cannot be deduced}} expected-note 3{{'Inner'}}
   template<typename Inner> static int y<Inner>; // expected-error {{does not specialize}}
 
-  template<typename, int> static int z; // expected-note {{template parameter is declared here}}
+  template<typename, int> static int z;
   template<Outer N> static int z<int, N>; // expected-error {{not implicitly convertible}}
 };
 template<typename Outer> template<typename Inner> int X<Outer>::y<Outer>; // expected-error {{cannot be deduced}} expected-note {{'Inner'}}
@@ -33,4 +33,4 @@ template<typename Outer> template<typename Inner> int X<Outer>::y<Inner>; // exp
 template<> template<typename Inner> int X<int>::y<Inner>; // expected-error {{does not specialize}} expected-note {{instantiation of}}
 
 X<int> xi;
-X<int*> xf; // expected-note 2{{instantiation of}}
+X<int*> xf; // expected-note {{instantiation of}}
diff --git a/clang/test/CXX/temp/temp.decls/temp.variadic/fixed-expansion.cpp b/clang/test/CXX/temp/temp.decls/temp.variadic/fixed-expansion.cpp
index cab5c967b0c4f..ab4c663d24c7d 100644
--- a/clang/test/CXX/temp/temp.decls/temp.variadic/fixed-expansion.cpp
+++ b/clang/test/CXX/temp/temp.decls/temp.variadic/fixed-expansion.cpp
@@ -14,10 +14,10 @@ struct is_same<T, T> {
 };
 
 namespace ExpandIntoFixed {
-  template<typename T,
-           typename U,
-           typename V = pair<T, U>,
-           typename W = V*>
+  template<typename T, 
+           typename U, 
+           typename V = pair<T, U>, 
+           typename W = V*> 
   class X0 { };
 
   template<typename ...Ts>
@@ -26,24 +26,24 @@ namespace ExpandIntoFixed {
     typedef X0<Ts...> type;
   };
 
-  static_assert(is_same<X1<int, int>::type,
+  static_assert(is_same<X1<int, int>::type, 
                         X0<int, int, pair<int, int>, pair<int, int>*>>::value,
                 "fails with two default arguments");
 
-  static_assert(is_same<X1<int, int, float>::type,
+  static_assert(is_same<X1<int, int, float>::type, 
                         X0<int, int, float, float*>>::value,
                 "fails with one default argument");
 
-  static_assert(is_same<X1<int, int, float, double>::type,
+  static_assert(is_same<X1<int, int, float, double>::type, 
                         X0<int, int, float, double>>::value,
                 "fails with no default arguments");
 }
 
 namespace ExpandIntoFixedShifted {
-  template<typename T,
-           typename U,
-           typename V = pair<T, U>,
-           typename W = V*>
+  template<typename T, 
+           typename U, 
+           typename V = pair<T, U>, 
+           typename W = V*> 
   class X0 { };
 
   template<typename ...Ts>
@@ -52,15 +52,15 @@ namespace ExpandIntoFixedShifted {
     typedef X0<char, Ts...> type;
   };
 
-  static_assert(is_same<X1<int>::type,
+  static_assert(is_same<X1<int>::type, 
                         X0<char, int, pair<char, int>, pair<char, int>*>>::value,
                 "fails with two default arguments");
 
-  static_assert(is_same<X1<int, float>::type,
+  static_assert(is_same<X1<int, float>::type, 
                         X0<char, int, float, float*>>::value,
                 "fails with one default argument");
 
-  static_assert(is_same<X1<int, float, double>::type,
+  static_assert(is_same<X1<int, float, double>::type, 
                         X0<char, int, float, double>>::value,
                 "fails with no default arguments");
 }
@@ -76,11 +76,11 @@ namespace Deduction {
 }
 
 namespace PR9021a {
-  template<typename, typename>
+  template<typename, typename> 
   struct A { };
 
   template<typename ...T>
-  struct B {
+  struct B { 
     A<T...> a1;
   };
 
@@ -93,9 +93,9 @@ namespace PR9021b {
   template<class, class>
   struct t2
   {
-
+    
   };
-
+  
   template<template<class...> class M>
   struct m
   {
@@ -107,14 +107,14 @@ namespace PR9021b {
 }
 
 namespace PartialSpecialization {
-  template<typename T, typename U, typename V = U> // expected-note {{template parameter is declared here}}
-  struct X0; // expected-note {{template is declared here}}
+  template<typename T, typename U, typename V = U>
+  struct X0; // expected-note 2{{template is declared here}}
 
   template<typename ...Ts>
   struct X0<Ts...> { // expected-error {{class template partial specialization is not more specialized than the primary template}}
   };
 
-  X0<int> x0i; // expected-error{{missing template argument for template parameter}}
+  X0<int> x0i; // expected-error{{too few template arguments for class template 'X0'}}
   X0<int, float> x0if;
   X0<int, float, double> x0ifd;
 }
diff --git a/clang/test/CXX/temp/temp.decls/temp.variadic/multi-level-substitution.cpp b/clang/test/CXX/temp/temp.decls/temp.variadic/multi-level-substitution.cpp
index c6e6038fb6ddf..30e7c65dac91c 100644
--- a/clang/test/CXX/temp/temp.decls/temp.variadic/multi-level-substitution.cpp
+++ b/clang/test/CXX/temp/temp.decls/temp.variadic/multi-level-substitution.cpp
@@ -48,7 +48,7 @@ namespace PacksAtDifferentLevels {
   int check1[X<short, int>::Inner<tuple<pair<short, unsigned short>,
                                         pair<int, unsigned int>,
                                         pair<long, unsigned long>>
-                                       >::value == 1? 1 : -1];
+                                       >::value == 1? 1 : -1]; 
 
   template<unsigned ...Values> struct unsigned_tuple { };
   template<typename ...Types>
@@ -99,7 +99,7 @@ namespace PacksAtDifferentLevels {
   int check5[X2<short, int>::Inner<int(pair<short, unsigned short>,
                                        pair<int, unsigned int>,
                                        pair<long, unsigned long>)
-                                     >::value == 1? 1 : -1];
+                                     >::value == 1? 1 : -1]; 
 
   template<typename T, typename U>
   struct some_function_object {
@@ -217,8 +217,8 @@ namespace ExpandingNonTypeTemplateParameters {
   template<typename ...Types>
   struct tuple_of_values {
     template<Types ...Values> // expected-error{{a non-type template parameter cannot have type 'float'}} \
-    // expected-note 2{{template parameter is declared here}}
-    struct apply { // expected-note {{template is declared here}}
+    // expected-note{{template parameter is declared here}}
+    struct apply { // expected-note 2{{template is declared here}}
       typedef tuple<value_c<Types, Values>...> type;
     };
   };
@@ -236,7 +236,7 @@ namespace ExpandingNonTypeTemplateParameters {
 
   tuple_of_values<int&, float&>::apply<i, i>::type tv2; // expected-error{{non-type template parameter of reference type 'float &' cannot bind to template argument of type 'int'}}
 
-  tuple_of_values<int&, float&>::apply<i>::type tv3; // expected-error{{missing template argument for template parameter}}
+  tuple_of_values<int&, float&>::apply<i>::type tv3; // expected-error{{too few template arguments for class template 'apply'}}
 
   tuple_of_values<int&, float&>::apply<i, f, i>::type tv4; // expected-error{{too many template arguments for class template 'apply'}}
 }
diff --git a/clang/test/CXX/temp/temp.deduct/p9.cpp b/clang/test/CXX/temp/temp.deduct/p9.cpp
index 5f9ea27234c81..7b661c275211b 100644
--- a/clang/test/CXX/temp/temp.deduct/p9.cpp
+++ b/clang/test/CXX/temp/temp.deduct/p9.cpp
@@ -15,14 +15,13 @@ void test_f() {
 }
 
 template <class T, unsigned = sizeof([]() { T::invalid; })>
-// expected-note@-1 {{template parameter is declared here}}
 void g(T);
 void g(...);
 void test_g() {
-  g(0); // expected-error@-5 {{type 'int' cannot be used prior to '::'}}
+  g(0); // expected-error@-4 {{type 'int' cannot be used prior to '::'}}
         // expected-note@-4 {{in instantiation of default argument}}
         // expected-note@-2 {{while substituting deduced template arguments}}
-        // expected-note@-8 {{while substituting into a lambda expression here}}
+        // expected-note@-7 {{while substituting into a lambda expression here}}
 }
 
 template <class T>
diff --git a/clang/test/CXX/temp/temp.param/p1.cpp b/clang/test/CXX/temp/temp.param/p1.cpp
index e2eecdf0d4554..e9a978961769c 100644
--- a/clang/test/CXX/temp/temp.param/p1.cpp
+++ b/clang/test/CXX/temp/temp.param/p1.cpp
@@ -5,9 +5,8 @@ template<template<> class C> class D; // expected-error{{template template param
 
 
 struct A {};
-template<class M,
-         class T // expected-note {{template parameter is declared here}}
-           = A,  // expected-note{{previous default template argument defined here}}
+template<class M, 
+         class T = A,  // expected-note{{previous default template argument defined here}}
          class C> // expected-error{{template parameter missing a default argument}}
-class X0 {};
-X0<int> x0; // expected-error{{missing template argument for template parameter}}
+class X0 {}; // expected-note{{template is declared here}}
+X0<int> x0; // expected-error{{too few template arguments for class template 'X0'}}
diff --git a/clang/test/CXX/temp/temp.param/p12.cpp b/clang/test/CXX/temp/temp.param/p12.cpp
index e3dfb65d29461..8317e7f24152c 100644
--- a/clang/test/CXX/temp/temp.param/p12.cpp
+++ b/clang/test/CXX/temp/temp.param/p12.cpp
@@ -32,9 +32,9 @@ template<int N,
   class B3n;
 
 // Check validity of default arguments
-template<template<class, int> class =// expected-note {{template parameter is declared here}}
+template<template<class, int> class =// expected-note {{previous template template parameter is here}}
            Y1> // expected-error{{too many template arguments for class template 'Y1'}}
-               // expected-note@-1 {{template template argument is incompatible}}
+               // expected-note@-1 {{template template argument has different template parameters than its corresponding template template parameter}}
   class C1 {};
 
 C1<> c1; // expected-note{{while checking a default template argument}}
diff --git a/clang/test/CXX/temp/temp.param/p15-cxx0x.cpp b/clang/test/CXX/temp/temp.param/p15-cxx0x.cpp
index 402a205f1ea63..83144a494937b 100644
--- a/clang/test/CXX/temp/temp.param/p15-cxx0x.cpp
+++ b/clang/test/CXX/temp/temp.param/p15-cxx0x.cpp
@@ -77,9 +77,9 @@ template<typename T> struct wrap {
 
 template<typename T> struct takedrop_impl;
 template<place...X> struct takedrop_impl<places<X...>> {
-  template<template<decltype(X)> class ...Take, // expected-note 2{{template parameter is declared here}}
+  template<template<decltype(X)> class ...Take,
            template<place      > class ...Drop>
-  struct inner {
+  struct inner { // expected-note 2{{declared}}
     typedef types<typename Take<_>::type...> take;
     typedef types<typename Drop<_>::type...> drop;
   };
@@ -87,11 +87,11 @@ template<place...X> struct takedrop_impl<places<X...>> {
 
 template<unsigned N, typename...Ts> struct take {
   using type = typename takedrop_impl<typename make_places<N>::type>::
-    template inner<wrap<Ts>::template inner...>::take; // expected-error {{missing template argument}}
+    template inner<wrap<Ts>::template inner...>::take; // expected-error {{too few template arguments}}
 };
 template<unsigned N, typename...Ts> struct drop {
   using type = typename takedrop_impl<typename make_places<N>::type>::
-    template inner<wrap<Ts>::template inner...>::drop; // expected-error {{missing template argument}}
+    template inner<wrap<Ts>::template inner...>::drop; // expected-error {{too few template arguments}}
 };
 
 using T1 = take<3, int, char, double, long>::type; // expected-note {{previous}}
@@ -118,7 +118,7 @@ using D3 = drop<5, int, char, double, long>::type; // expected-note {{in instant
 // implicitly a pack expansion.
 template<typename ...Default> struct DefArg {
   template<template<typename T = Default> class ...Classes> struct Inner { // expected-error {{default argument contains unexpanded parameter pack}} expected-note {{here}}
-    Inner(Classes<>...); // expected-error {{missing template argument}}
+    Inner(Classes<>...); // expected-error {{too few}}
   };
 };
 template<typename T> struct vector {};
diff --git a/clang/test/CXX/temp/temp.param/p8-cxx20.cpp b/clang/test/CXX/temp/temp.param/p8-cxx20.cpp
index aa1666f6306d0..a3478c0669661 100644
--- a/clang/test/CXX/temp/temp.param/p8-cxx20.cpp
+++ b/clang/test/CXX/temp/temp.param/p8-cxx20.cpp
@@ -59,7 +59,7 @@ namespace ConstDestruction {
     f<D{1, true}>();
   }
 
-  template<D d> struct Z {}; // expected-note {{template parameter is declared here}}
+  template<D d> struct Z {};
   Z<D{2, true}> z1;
   Z<D{2, false}> z2; // expected-error {{non-type template argument is not a constant expression}} expected-note-re {{in call to '{{.*}}.~D()'}}
 }
diff --git a/clang/test/CXX/temp/temp.res/temp.dep/p3.cpp b/clang/test/CXX/temp/temp.res/temp.dep/p3.cpp
index 583fb4b3cc05d..a5cbcfc0fcafa 100644
--- a/clang/test/CXX/temp/temp.res/temp.dep/p3.cpp
+++ b/clang/test/CXX/temp/temp.res/temp.dep/p3.cpp
@@ -10,13 +10,13 @@ template <typename T> struct B0: A0 {
 };
 
 namespace E1 {
-  typedef double A; 
+  typedef double A;
 
   template<class T> class B {
-    typedef int A; 
+    typedef int A;
   };
 
-  template<class T> 
+  template<class T>
   struct X : B<T> {
     A* blarg(double *dp) {
       return dp;
@@ -25,20 +25,20 @@ namespace E1 {
 }
 
 namespace E2 {
-  struct A { 
+  struct A {
     struct B;
     int *a;
     int Y;
   };
-    
+
   int a;
-  template<class T> struct Y : T { 
+  template<class T> struct Y : T {
     struct B { /* ... */ };
-    B b; 
-    void f(int i) { a = i; } 
+    B b;
+    void f(int i) { a = i; }
     Y* p;
-  }; 
-  
+  };
+
   Y<A> ya;
 }
 
@@ -56,7 +56,7 @@ namespace PR14402 {
     };
 
     struct D {
-      using A::n; // expected-error {{using declaration refers into 'A<T>::', which is not a base class of 'D'}}
+      using A::n; // expected-error {{using declaration refers into 'A<T>', which is not a base class of 'D'}}
       int g() { return f(); } // expected-error {{call to non-static member function 'f' of 'A' from nested type 'D'}}
     };
 
diff --git a/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.constexpr/p2.cpp b/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.constexpr/p2.cpp
index cb048ecdf7001..ecb82372bcb47 100644
--- a/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.constexpr/p2.cpp
+++ b/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.constexpr/p2.cpp
@@ -2,7 +2,7 @@
 // RUN: %clang_cc1 -std=c++11 -verify=cxx11 %s
 // cxx11-no-diagnostics
 
-template<int n> struct S; // cxx98-note {{template parameter is declared here}}
+template<int n> struct S;
 
 template<int n> struct T {
   T() {
diff --git a/clang/test/CXX/temp/temp.spec/cxx1y-variable-template-no-body.cpp b/clang/test/CXX/temp/temp.spec/cxx1y-variable-template-no-body.cpp
index 749b45524f9e6..741ebc5de41fc 100644
--- a/clang/test/CXX/temp/temp.spec/cxx1y-variable-template-no-body.cpp
+++ b/clang/test/CXX/temp/temp.spec/cxx1y-variable-template-no-body.cpp
@@ -3,27 +3,27 @@
 // RUN: not %clang_cc1 --std=c++1y -x c++ -fixit %t -DFIXING
 // RUN: %clang_cc1 --std=c++1y -x c++ %t -DFIXING
 
-template<typename T> // expected-note {{template parameter is declared here}}
-T pi = T(3.1415926535897932385);
+template<typename T> 
+T pi = T(3.1415926535897932385); // expected-note {{template is declared here}}
 
 template int pi<int>;
 
 #ifndef FIXING
-template float pi<>; // expected-error {{missing template argument for template parameter}}
+template float pi<>; // expected-error {{too few template arguments for variable template 'pi'}}
 template double pi_var0; // expected-error {{explicit instantiation of 'pi_var0' does not refer to a function template, variable template, member function, member class, or static data member}}
 #endif
 
 // Should recover as if definition
 template double pi_var = 5; // expected-error {{variable cannot be defined in an explicit instantiation; if this declaration is meant to be a variable definition, remove the 'template' keyword}}
 #ifndef FIXING
-template<typename T>
+template<typename T> 
 T pi0 = T(3.1415926535897932385); // expected-note {{previous definition is here}}
 
 template int pi0 = 10; // expected-error {{variable cannot be defined in an explicit instantiation; if this declaration is meant to be a variable definition, remove the 'template' keyword}} \
                           expected-error{{redefinition of 'pi0' as different kind of symbol}}
 #endif
 
-template<typename T>
+template<typename T> 
 T pi1 = T(3.1415926535897932385); // expected-note 0-2 {{here}}
 
 // Should recover as if specialization
diff --git a/clang/test/CXX/temp/temp.spec/part.spec.cpp b/clang/test/CXX/temp/temp.spec/part.spec.cpp
index 3923d160ef5cf..4b0fdb902633a 100644
--- a/clang/test/CXX/temp/temp.spec/part.spec.cpp
+++ b/clang/test/CXX/temp/temp.spec/part.spec.cpp
@@ -250,7 +250,7 @@ template <typename T> class PCT1 {};
 template <typename T1, typename T2> class PCT2 {};
 template <int X> class PCT3 {};
 template <void (TestClass::*)()> class PCT4 {};
-template <void (*)()> class PCT5 {}; // expected-note {{template parameter is declared here}}
+template <void (*)()> class PCT5 {};
 template <typename T> class PCT6 {
   // expected-note@+1 3{{implicitly declared private here}}
   template <typename NT> class NPCT1 {};
@@ -416,7 +416,7 @@ template <typename T1, typename T2> class PCTT1 {};
 template <typename T1, typename T2, typename T3> class PCTT2 {};
 template <typename T, int X> class PCTT3 {};
 template <typename T, void (TestClass::*)()> class PCTT4 {};
-template <typename T, void (*)()> class PCTT5 {}; // expected-note {{template parameter is declared here}}
+template <typename T, void (*)()> class PCTT5 {};
 template <typename T1, typename T2> class PCTT6 {
   template <typename NT> class NCT1 {};
   template <typename NT> class NCT2; // forward declaration
diff --git a/clang/test/CXX/temp/temp.spec/temp.expl.spec/p20.cpp b/clang/test/CXX/temp/temp.spec/temp.expl.spec/p20.cpp
index 63fd997add4ec..0283dba63b110 100644
--- a/clang/test/CXX/temp/temp.spec/temp.expl.spec/p20.cpp
+++ b/clang/test/CXX/temp/temp.spec/temp.expl.spec/p20.cpp
@@ -2,13 +2,13 @@
 template<typename T>
 void f(T);
 
-template<typename T> // expected-note {{template parameter is declared here}}
-struct A { };
+template<typename T>
+struct A { }; // expected-note{{template is declared here}}
 
 struct X {
   template<> friend void f<int>(int); // expected-error{{in a friend}}
   template<> friend class A<int>; // expected-error{{cannot be a friend}}
-
+  
   friend void f<float>(float); // okay
   friend class A<float>; // okay
 };
@@ -18,6 +18,6 @@ struct PR41792 {
   template <> friend void f<>(int);
 
   // expected-error@+2{{template specialization declaration cannot be a friend}}
-  // expected-error@+1{{missing template argument for template parameter}}
+  // expected-error@+1{{too few template arguments for class template 'A'}}
   template <> friend class A<>;
 };
diff --git a/clang/test/CodeGen/AArch64/sve-vector-bits-codegen.c b/clang/test/CodeGen/AArch64/sve-vector-bits-codegen.c
index 1391a1b09fbd1..0ed14b4b3b793 100644
--- a/clang/test/CodeGen/AArch64/sve-vector-bits-codegen.c
+++ b/clang/test/CodeGen/AArch64/sve-vector-bits-codegen.c
@@ -13,9 +13,12 @@
 
 void func(int *restrict a, int *restrict b) {
 // CHECK-LABEL: func
-// CHECK256-COUNT-8: str
-// CHECK512-COUNT-4: str
-// CHECK1024-COUNT-2: str
+// CHECK256-COUNT-1: str
+// CHECK256-COUNT-7: st1w
+// CHECK512-COUNT-1: str
+// CHECK512-COUNT-3: st1w
+// CHECK1024-COUNT-1: str
+// CHECK1024-COUNT-1: st1w
 // CHECK2048-COUNT-1: st1w
 #pragma clang loop vectorize(enable)
   for (int i = 0; i < 64; ++i)
diff --git a/clang/test/CodeGen/PowerPC/builtins-ppc-p9vector.c b/clang/test/CodeGen/PowerPC/builtins-ppc-p9vector.c
index b55a522ed2608..68d32ee14c8fa 100644
--- a/clang/test/CodeGen/PowerPC/builtins-ppc-p9vector.c
+++ b/clang/test/CodeGen/PowerPC/builtins-ppc-p9vector.c
@@ -853,10 +853,16 @@ vector unsigned char test73(void) {
 vector unsigned short test74(void) {
 // CHECK-BE: @llvm.ppc.vsx.xvcvsphp(<4 x float>
 // CHECK-BE: @llvm.ppc.vsx.xvcvsphp(<4 x float>
-// CHECK-BE: @llvm.ppc.altivec.vperm
+// CHECK-BE: [[REG0:%[0-9]+]] = call <4 x i32> @llvm.ppc.altivec.vperm
+// CHECK-BE-NEXT: [[REG1:%[0-9]+]] = bitcast <4 x i32> [[REG0]] to <4 x float>
+// CHECK-BE-NEXT: [[REG2:%[0-9]+]] = bitcast <4 x float> [[REG1]] to <8 x i16>
+// CHECK-BE-NEXT: ret <8 x i16> [[REG2]]
 // CHECK: @llvm.ppc.vsx.xvcvsphp(<4 x float>
 // CHECK: @llvm.ppc.vsx.xvcvsphp(<4 x float>
-// CHECK: @llvm.ppc.altivec.vperm
+// CHECK: [[REG0:%[0-9]+]] = call <4 x i32> @llvm.ppc.altivec.vperm
+// CHECK-NEXT: [[REG1:%[0-9]+]] = bitcast <4 x i32> [[REG0]] to <4 x float>
+// CHECK-NEXT: [[REG2:%[0-9]+]] = bitcast <4 x float> [[REG1]] to <8 x i16>
+// CHECK-NEXT: ret <8 x i16> [[REG2]]
   return vec_pack_to_short_fp32(vfa, vfb);
 }
 vector unsigned int test75(void) {
diff --git a/clang/test/CodeGen/RISCV/pr129995.cc b/clang/test/CodeGen/RISCV/pr129995.cc
new file mode 100644
index 0000000000000..590c6b9fbdf96
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/pr129995.cc
@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 triple riscv64 -emit-llvm -target-feature +m -target-feature +v -target-abi lp64d -o /dev/null %s
+
+struct a {
+  using b = char __attribute__((vector_size(sizeof(char))));
+};
+class c {
+  using d = a::b;
+  d e;
+
+public:
+  static c f();
+};
+class g {
+public:
+  template <class h> g(h);
+  friend g operator^(g, g) { c::f; }
+  friend g operator^=(g i, g j) { i ^ j; }
+};
+template <typename, int> using k = g;
+template <typename l> using m = k<l, sizeof(l)>;
+void n() {
+  void o();
+  m<char> p = o ^= p;
+}
diff --git a/clang/test/CodeGen/aarch64-execute-only.c b/clang/test/CodeGen/aarch64-execute-only.c
new file mode 100644
index 0000000000000..19fb29fcfcf21
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-execute-only.c
@@ -0,0 +1,19 @@
+// RUN: %clang -target aarch64 -### %s 2>&1 | \
+// RUN:   FileCheck %s -check-prefix CHECK-NO-EXECUTE-ONLY
+// RUN: %clang -target aarch64 -### -mexecute-only %s 2>&1 | \
+// RUN:   FileCheck %s -check-prefix CHECK-EXECUTE-ONLY
+// RUN: %clang -target aarch64 -### -mexecute-only -mno-execute-only %s 2>&1 | \
+// RUN:   FileCheck %s -check-prefix CHECK-NO-EXECUTE-ONLY
+
+/// -mpure-code flag for GCC compatibility
+// RUN: %clang -target aarch64 -### %s 2>&1 | \
+// RUN:   FileCheck %s -check-prefix CHECK-NO-EXECUTE-ONLY
+// RUN: %clang -target aarch64 -### -mpure-code %s 2>&1 | \
+// RUN:   FileCheck %s -check-prefix CHECK-EXECUTE-ONLY
+// RUN: %clang -target aarch64 -### -mpure-code -mno-pure-code %s 2>&1 | \
+// RUN:   FileCheck %s -check-prefix CHECK-NO-EXECUTE-ONLY
+
+// CHECK-NO-EXECUTE-ONLY-NOT: "+execute-only"
+// CHECK-EXECUTE-ONLY: "+execute-only"
+
+void a() {}
diff --git a/clang/test/CodeGen/attr-counted-by.c b/clang/test/CodeGen/attr-counted-by.c
index e85f3db1121af..dfdf06587f0e2 100644
--- a/clang/test/CodeGen/attr-counted-by.c
+++ b/clang/test/CodeGen/attr-counted-by.c
@@ -333,7 +333,7 @@ size_t test3_bdos(struct annotated *p) {
 // SANITIZE-WITH-ATTR-NEXT:    [[COUNT50:%.*]] = sext i32 [[DOTCOUNTED_BY_LOAD44]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP10:%.*]] = sub nsw i64 [[COUNT50]], [[IDXPROM42]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP11:%.*]] = tail call i64 @llvm.smax.i64(i64 [[TMP10]], i64 0)
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTTR:%.*]] = trunc nuw i64 [[TMP11]] to i32
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTTR:%.*]] = trunc nuw nsw i64 [[TMP11]] to i32
 // SANITIZE-WITH-ATTR-NEXT:    [[CONV54:%.*]] = shl i32 [[DOTTR]], 2
 // SANITIZE-WITH-ATTR-NEXT:    [[CONV55:%.*]] = and i32 [[CONV54]], 252
 // SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV55]], ptr [[ARRAYIDX65]], align 4, !tbaa [[TBAA4]]
diff --git a/clang/test/CodeGen/builtin-constant-p.c b/clang/test/CodeGen/builtin-constant-p.c
index 8e35cb2b2ef83..07288069a5e3a 100644
--- a/clang/test/CodeGen/builtin-constant-p.c
+++ b/clang/test/CodeGen/builtin-constant-p.c
@@ -1,5 +1,8 @@
 // RUN: %clang_cc1 -triple x86_64-unknown-unknown -emit-llvm -disable-llvm-optzns -o - %s -O2 | FileCheck %s
 // RUN: %clang_cc1 -triple x86_64-unknown-unknown -emit-llvm -disable-llvm-optzns -o - %s -O0 | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -emit-llvm -disable-llvm-optzns -o - %s -O2 -fexperimental-new-constant-interpreter | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -emit-llvm -disable-llvm-optzns -o - %s -O0 -fexperimental-new-constant-interpreter | FileCheck %s
+
 
 int a = 42;
 
diff --git a/clang/test/CodeGen/builtins-elementwise-math.c b/clang/test/CodeGen/builtins-elementwise-math.c
index 5661bbc6008a0..ee8345ff51e5e 100644
--- a/clang/test/CodeGen/builtins-elementwise-math.c
+++ b/clang/test/CodeGen/builtins-elementwise-math.c
@@ -710,6 +710,21 @@ void test_builtin_elementwise_exp2(float f1, float f2, double d1, double d2,
   vf2 = __builtin_elementwise_exp2(vf1);
 }
 
+void test_builtin_elementwise_exp10(float f1, float f2, double d1, double d2,
+                                  float4 vf1, float4 vf2) {
+  // CHECK-LABEL: define void @test_builtin_elementwise_exp10(
+  // CHECK:      [[F1:%.+]] = load float, ptr %f1.addr, align 4
+  // CHECK-NEXT:  call float @llvm.exp10.f32(float [[F1]])
+  f2 = __builtin_elementwise_exp10(f1);
+
+  // CHECK:      [[D1:%.+]] = load double, ptr %d1.addr, align 8
+  // CHECK-NEXT: call double @llvm.exp10.f64(double [[D1]])
+  d2 = __builtin_elementwise_exp10(d1);
+
+  // CHECK:      [[VF1:%.+]] = load <4 x float>, ptr %vf1.addr, align 16
+  // CHECK-NEXT: call <4 x float> @llvm.exp10.v4f32(<4 x float> [[VF1]])
+  vf2 = __builtin_elementwise_exp10(vf1);
+}
 
 void test_builtin_elementwise_floor(float f1, float f2, double d1, double d2,
                                     float4 vf1, float4 vf2) {
diff --git a/clang/test/CodeGen/openacc-noop-decl.c b/clang/test/CodeGen/openacc-noop-decl.c
new file mode 100644
index 0000000000000..41f756d835aa5
--- /dev/null
+++ b/clang/test/CodeGen/openacc-noop-decl.c
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 %s -emit-llvm -o - | FileCheck %s
+
+// Check to ensure these are no-ops/don't really do anything.
+void name();
+
+void foo() {
+#pragma acc declare
+  // CHECK-NOT: declare
+#pragma acc routine(name) worker
+  // CHECK-NOT: routine
+}
+#pragma acc declare
+  // CHECK-NOT: declare
+#pragma acc routine(name) worker
+  // CHECK-NOT: routine
diff --git a/clang/test/CodeGen/p0963r3.cpp b/clang/test/CodeGen/p0963r3.cpp
new file mode 100644
index 0000000000000..4a5e6c3f5d751
--- /dev/null
+++ b/clang/test/CodeGen/p0963r3.cpp
@@ -0,0 +1,209 @@
+// RUN: %clang_cc1 -std=c++2c -verify -emit-llvm -triple=x86_64-pc-linux-gnu %s -o - | FileCheck %s
+// RUN: %clang_cc1 -std=c++2c -verify -fsyntax-only -fexperimental-new-constant-interpreter -triple=x86_64-pc-linux-gnu %s
+// expected-no-diagnostics
+
+namespace std {
+
+template <typename T> struct tuple_size;
+
+template <int, typename> struct tuple_element;
+
+} // namespace std
+
+namespace Case1 {
+
+struct S {
+  int a, b;
+  bool flag = false;
+
+  constexpr explicit operator bool() {
+    flag = true;
+    return a != b;
+  }
+
+  constexpr operator int() {
+    flag = true;
+    return a * b;
+  }
+
+  constexpr bool operator==(S rhs) const {
+    return a == rhs.a && b == rhs.b;
+  }
+
+  template <int I>
+  constexpr int& get() {
+    if (!flag)
+      return a = a + b;
+    return I == 0 ? a : b;
+  }
+};
+
+} // namespace Case1
+
+template <> struct std::tuple_size<Case1::S> {
+  static const int value = 2;
+};
+
+template <int I> struct std::tuple_element<I, Case1::S> {
+  using type = int;
+};
+
+namespace Case1 {
+
+void foo() {
+  if (S s(1, 2); auto [a, b] = s) {
+    __builtin_assume(a == 1);
+    __builtin_assume(b == 2);
+  }
+// CHECK: %[[call:.+]] = call {{.*}} i1 @_ZN5Case11ScvbEv
+// CHECK: %{{.*}} = call {{.*}} ptr @_ZN5Case11S3getILi0EEERiv
+// CHECK: %{{.*}} = call {{.*}} ptr @_ZN5Case11S3getILi1EEERiv
+// CHECK: br i1 %[[call]], label {{.*}}, label {{.*}}
+
+  if (auto [a, b] = S(1, 2)) {
+    __builtin_assume(a == 1);
+    __builtin_assume(b == 2);
+  }
+// CHECK: %[[call2:.+]] = call {{.*}} i1 @_ZN5Case11ScvbEv
+// CHECK: %{{.*}} = call {{.*}} ptr @_ZN5Case11S3getILi0EEERiv
+// CHECK: %{{.*}} = call {{.*}} ptr @_ZN5Case11S3getILi1EEERiv
+// CHECK: br i1 %[[call2]], label {{.*}}, label {{.*}}
+
+  if (S s(3, 4); auto& [a, b] = s) {
+    __builtin_assume(a == 3);
+    __builtin_assume(b == 4);
+  }
+// CHECK: %[[call3:.+]] = call {{.*}} i1 @_ZN5Case11ScvbEv
+// CHECK: %{{.*}} = call {{.*}} ptr @_ZN5Case11S3getILi0EEERiv
+// CHECK: %{{.*}} = call {{.*}} ptr @_ZN5Case11S3getILi1EEERiv
+// CHECK: br i1 %[[call3]], label {{.*}}, label {{.*}}
+
+  while (auto [i, j] = S(5, 6))
+    break;
+
+// CHECK: while.cond{{.*}}:
+// CHECK: %[[call4:.+]] = call {{.*}} i1 @_ZN5Case11ScvbEv
+// CHECK: %{{.*}} = call {{.*}} ptr @_ZN5Case11S3getILi0EEERiv
+// CHECK: %{{.*}} = call {{.*}} ptr @_ZN5Case11S3getILi1EEERiv
+// CHECK: br i1 %[[call4]], label {{.*}}, label {{.*}}
+
+  S s(7, 8);
+  while (auto& [i, j] = s)
+    break;
+
+// CHECK: while.cond{{.*}}:
+// CHECK: %[[call5:.+]] = call {{.*}} i1 @_ZN5Case11ScvbEv
+// CHECK: %{{.*}} = call {{.*}} ptr @_ZN5Case11S3getILi0EEERiv
+// CHECK: %{{.*}} = call {{.*}} ptr @_ZN5Case11S3getILi1EEERiv
+// CHECK: br i1 %[[call5]], label {{.*}}, label {{.*}}
+
+  for (int k = 0; auto [i, j] = S(24, 42); ++k)
+    break;
+
+// CHECK: for.cond{{.*}}:
+// CHECK: %[[call6:.+]] = call {{.*}} i1 @_ZN5Case11ScvbEv
+// CHECK: %{{.*}} = call {{.*}} ptr @_ZN5Case11S3getILi0EEERiv
+// CHECK: %{{.*}} = call {{.*}} ptr @_ZN5Case11S3getILi1EEERiv
+// CHECK: br i1 %[[call6]], label {{.*}}, label {{.*}}
+
+  for (S s(114, 514); auto& [i, j] = s; ++i)
+    break;
+
+// CHECK: for.cond{{.*}}:
+// CHECK: %[[call7:.+]] = call {{.*}} i1 @_ZN5Case11ScvbEv
+// CHECK: %{{.*}} = call {{.*}} ptr @_ZN5Case11S3getILi0EEERiv
+// CHECK: %{{.*}} = call {{.*}} ptr @_ZN5Case11S3getILi1EEERiv
+// CHECK: br i1 %[[call7]], label {{.*}}, label {{.*}}
+
+  switch (S s(10, 11); auto& [i, j] = s) {
+    case 10 * 11:
+      __builtin_assume(i == 10);
+      __builtin_assume(j == 11);
+      break;
+    default:
+      break;
+  }
+
+// CHECK: %[[call8:.+]] = call {{.*}} i32 @_ZN5Case11ScviEv
+// CHECK: %{{.*}} = call {{.*}} ptr @_ZN5Case11S3getILi0EEERiv
+// CHECK: %{{.*}} = call {{.*}} ptr @_ZN5Case11S3getILi1EEERiv
+// CHECK: switch i32 %[[call8]], label {{.*}}
+
+}
+
+constexpr int bar(auto) {
+  constexpr auto value = [] {
+    if (S s(1, 2); auto [i, j] = s)
+      return S(i, j);
+    return S(0, 0);
+  }();
+  static_assert(value == S(1, 2));
+
+  constexpr auto value2 = [] {
+    if (auto [a, b] = S(1, 2))
+      return S(a, b);
+    return S(0, 0);
+  }();
+  static_assert(value2 == S(1, 2));
+
+  constexpr auto value3 = [] {
+    if (auto&& [a, b] = S(3, 4))
+      return S(a, b);
+    return S(0, 0);
+  }();
+  static_assert(value3 == S(3, 4));
+
+  constexpr auto value4 = [] {
+    S s(7, 8);
+    int cnt = 0;
+    while (auto& [i, j] = s) {
+      s.flag = false;
+      ++i, ++j;
+      if (++cnt == 10)
+        break;
+    }
+    return s;
+  }();
+  static_assert(value4 == S(17, 18));
+
+  constexpr auto value5 = [] {
+    S s(3, 4);
+    for (int cnt = 0; auto& [x, y] = s; s.flag = false, ++cnt) {
+      if (cnt == 3)
+        break;
+      ++x, ++y;
+    }
+    return s;
+  }();
+  static_assert(value5 == S(6, 7));
+
+  constexpr auto value6 = [] {
+    switch (auto [x, y] = S(3, 4)) {
+      case 3 * 4:
+        return S(x, y);
+      default:
+        return S(y, x);
+    }
+  }();
+  static_assert(value6 == S(3, 4));
+
+  return 42;
+}
+
+constexpr int value = bar(1);
+
+#if 0
+
+// FIXME: This causes clang to ICE, though this is not a regression.
+constexpr int ice(auto) {
+  if constexpr (S s(1, 2); auto [i, j] = s) {
+    static_assert(i == 1);
+  }
+  return 42;
+}
+
+constexpr int value2 = ice(1);
+
+#endif
+
+} // namespace Case1
diff --git a/clang/test/CodeGen/strictfp-elementwise-bulitins.cpp b/clang/test/CodeGen/strictfp-elementwise-builtins.cpp
similarity index 96%
rename from clang/test/CodeGen/strictfp-elementwise-bulitins.cpp
rename to clang/test/CodeGen/strictfp-elementwise-builtins.cpp
index 175ad22601839..b250512efc5c7 100644
--- a/clang/test/CodeGen/strictfp-elementwise-bulitins.cpp
+++ b/clang/test/CodeGen/strictfp-elementwise-builtins.cpp
@@ -127,6 +127,16 @@ float4 strict_elementwise_exp2(float4 a) {
   return __builtin_elementwise_exp2(a);
 }
 
+// CHECK-LABEL: define dso_local noundef <4 x float> @_Z24strict_elementwise_exp10Dv4_f
+// CHECK-SAME: (<4 x float> noundef [[A:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[ELT_EXP10:%.*]] = tail call <4 x float> @llvm.exp10.v4f32(<4 x float> [[A]]) #[[ATTR4]]
+// CHECK-NEXT:    ret <4 x float> [[ELT_EXP10]]
+//
+float4 strict_elementwise_exp10(float4 a) {
+  return __builtin_elementwise_exp10(a);
+}
+
 // CHECK-LABEL: define dso_local noundef <4 x float> @_Z24strict_elementwise_floorDv4_f
 // CHECK-SAME: (<4 x float> noundef [[A:%.*]]) local_unnamed_addr #[[ATTR2]] {
 // CHECK-NEXT:  entry:
@@ -340,9 +350,8 @@ float4 strict_elementwise_pow(float4 a, float4 b) {
 // CHECK-LABEL: define dso_local noundef <4 x float> @_Z23strict_elementwise_fmodDv4_fS_
 // CHECK-SAME: (<4 x float> noundef [[A:%.*]], <4 x float> noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <4 x float> @llvm.experimental.constrained.frem.v4f32(<4 x float> [[A]], <4 x float> [[B]],
-// CHECK-SAME:    metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR4]]
-// CHECK-NEXT:    ret <4 x float> [[TMP0]]
+// CHECK-NEXT:    [[FMOD:%.*]] = tail call <4 x float> @llvm.experimental.constrained.frem.v4f32(<4 x float> [[A]], <4 x float> [[B]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR4]]
+// CHECK-NEXT:    ret <4 x float> [[FMOD]]
 //
 float4 strict_elementwise_fmod(float4 a, float4 b) {
   return __builtin_elementwise_fmod(a, b);
diff --git a/clang/test/CodeGen/vector.c b/clang/test/CodeGen/vector.c
index c20e320463056..5d677aaf16948 100644
--- a/clang/test/CodeGen/vector.c
+++ b/clang/test/CodeGen/vector.c
@@ -8,6 +8,10 @@ void test1(void) {
 __v4hi x = {1,2,3};
 __v4hi y = {1,2,3,4};
 
+
+// CHECK: @z = local_unnamed_addr global <8 x float> zeroinitializer
+float z __attribute__((ext_vector_type(8)));
+
 typedef int vty __attribute((vector_size(16)));
 int test2(void) { vty b; return b[2LL]; }
 
@@ -18,9 +22,6 @@ void test3 ( vec4* a, char b, float c ) {
   (*a)[b] = c;
 }
 
-
-
-
 #include <mmintrin.h>
 
 int test4(int argc, char *argv[]) {
diff --git a/clang/test/CodeGenCUDA/amdgpu-bf16.cu b/clang/test/CodeGenCUDA/amdgpu-bf16.cu
index 4610b4ae3cbe5..f9b067d3fe0d3 100644
--- a/clang/test/CodeGenCUDA/amdgpu-bf16.cu
+++ b/clang/test/CodeGenCUDA/amdgpu-bf16.cu
@@ -111,19 +111,15 @@ __device__ __bf16 test_call( __bf16 in) {
 // CHECK-NEXT:    ret void
 //
 __device__ void test_vec_assign() {
-  typedef __attribute__((ext_vector_type(2))) __bf16 bf16_x2;
-  bf16_x2 vec2_a, vec2_b;
+  __bf16 [[clang::ext_vector_type(2)]] vec2_a, vec2_b;
   vec2_a = vec2_b;
 
-  typedef __attribute__((ext_vector_type(4))) __bf16 bf16_x4;
-  bf16_x4 vec4_a, vec4_b;
+  __bf16 __attribute__((ext_vector_type(4))) vec4_a, vec4_b;
   vec4_a = vec4_b;
 
-  typedef __attribute__((ext_vector_type(8))) __bf16 bf16_x8;
-  bf16_x8 vec8_a, vec8_b;
+  __bf16 [[clang::ext_vector_type(8)]] vec8_a, vec8_b;
   vec8_a = vec8_b;
 
-  typedef __attribute__((ext_vector_type(16))) __bf16 bf16_x16;
-  bf16_x16 vec16_a, vec16_b;
+  __bf16 __attribute__((ext_vector_type(16))) vec16_a, vec16_b;
   vec16_a = vec16_b;
 }
diff --git a/clang/test/CodeGenCXX/builtin-constant-p.cpp b/clang/test/CodeGenCXX/builtin-constant-p.cpp
index 866faa5ec9761..39416b94375fe 100644
--- a/clang/test/CodeGenCXX/builtin-constant-p.cpp
+++ b/clang/test/CodeGenCXX/builtin-constant-p.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -triple=x86_64-linux-gnu -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -emit-llvm -o - %s -fexperimental-new-constant-interpreter | FileCheck %s
 
 // Don't crash if the argument to __builtin_constant_p isn't scalar.
 template <typename T>
diff --git a/clang/test/CodeGenCXX/cxx23-p2280r4.cpp b/clang/test/CodeGenCXX/cxx23-p2280r4.cpp
index d5409be451df0..53b00695d9d6d 100644
--- a/clang/test/CodeGenCXX/cxx23-p2280r4.cpp
+++ b/clang/test/CodeGenCXX/cxx23-p2280r4.cpp
@@ -4,7 +4,7 @@
 
 extern int& s;
 
-// CHECK: @_Z4testv()
+// CHECK-LABEL: @_Z4testv()
 // CHECK-NEXT: entry:
 // CHECK-NEXT: [[I:%.*]] = alloca ptr, align {{.*}}
 // CHECK-NEXT: [[X:%.*]] = load ptr, ptr @s, align {{.*}}
@@ -13,3 +13,16 @@ int& test() {
   auto &i = s;
   return i;
 }
+
+// CHECK-LABEL: @_Z1fv(
+// CHECK: [[X1:%.*]] = load ptr, ptr @x, align {{.*}}
+// CHECK-NEXT: store ptr [[X1]]
+// CHECK: [[X2:%.*]] = load ptr, ptr @x, align {{.*}}
+// CHECK-NEXT: store ptr [[X2]]
+// CHECK: [[X3:%.*]] = load ptr, ptr @x, align {{.*}}
+// CHECK-NEXT: store ptr [[X3]]
+int &ff();
+int &x = ff();
+struct A { int& x; };
+struct B { A x[20]; };
+B f() { return {x,x,x,x,x,x,x,x,x,x,x,x,x,x,x,x,x,x,x,x}; }
diff --git a/clang/test/CodeGenCXX/wasm-eh.cpp b/clang/test/CodeGenCXX/wasm-eh.cpp
index e8797794e7c1e..faff764878f5d 100644
--- a/clang/test/CodeGenCXX/wasm-eh.cpp
+++ b/clang/test/CodeGenCXX/wasm-eh.cpp
@@ -17,7 +17,7 @@ struct Cleanup {
 };
 
 // Multiple catch clauses w/o catch-all
-void test0() {
+void multiple_catches_wo_catch_all() {
   try {
     may_throw();
   } catch (int) {
@@ -27,7 +27,7 @@ void test0() {
   }
 }
 
-// CHECK-LABEL: define void @_Z5test0v() {{.*}} personality ptr @__gxx_wasm_personality_v0
+// CHECK-LABEL: define void @_Z29multiple_catches_wo_catch_allv() {{.*}} personality ptr @__gxx_wasm_personality_v0
 
 // CHECK:   %[[INT_ALLOCA:.*]] = alloca i32
 // CHECK:   invoke void @_Z9may_throwv()
@@ -73,7 +73,7 @@ void test0() {
 // CHECK-NEXT:   unreachable
 
 // Single catch-all
-void test1() {
+void single_catch_all() {
   try {
     may_throw();
   } catch (...) {
@@ -81,7 +81,7 @@ void test1() {
   }
 }
 
-// CATCH-LABEL: @_Z5test1v()
+// CATCH-LABEL: @_Z16single_catch_allv()
 
 // CHECK:   %[[CATCHSWITCH:.*]] = catchswitch within none [label %[[CATCHSTART_BB:.*]]] unwind to caller
 
@@ -93,7 +93,7 @@ void test1() {
 // CHECK:   catchret from %[[CATCHPAD]] to label
 
 // Multiple catch clauses w/ catch-all
-void test2() {
+void multiple_catches_w_catch_all() {
   try {
     may_throw();
   } catch (int) {
@@ -103,7 +103,7 @@ void test2() {
   }
 }
 
-// CHECK-LABEL: @_Z5test2v()
+// CHECK-LABEL: @_Z28multiple_catches_w_catch_allv()
 
 // CHECK:   %[[CATCHSWITCH:.*]] = catchswitch within none [label %[[CATCHSTART_BB:.*]]] unwind to caller
 
@@ -118,12 +118,12 @@ void test2() {
 // CHECK:   catchret from %[[CATCHPAD]] to label
 
 // Cleanup
-void test3() {
+void cleanup() {
   Cleanup c;
   may_throw();
 }
 
-// CHECK-LABEL: @_Z5test3v()
+// CHECK-LABEL: @_Z7cleanupv()
 
 // CHECK:   invoke void @_Z9may_throwv()
 // CHECK-NEXT:           to label {{.*}} unwind label %[[EHCLEANUP_BB:.*]]
@@ -134,7 +134,7 @@ void test3() {
 // CHECK-NEXT:   cleanupret from %[[CLEANUPPAD]] unwind to caller
 
 // Possibly throwing function call within a catch
-void test4() {
+void catch_int() {
   try {
     may_throw();
   } catch (int) {
@@ -142,7 +142,7 @@ void test4() {
   }
 }
 
-// CHECK-LABEL: @_Z5test4v()
+// CHECK-LABEL: @_Z9catch_intv()
 
 // CHECK:   %[[CATCHSWITCH]] = catchswitch within none [label %[[CATCHSTART_BB]]] unwind to caller
 
@@ -162,7 +162,7 @@ void test4() {
 // CHECK-NEXT:   cleanupret from %[[CLEANUPPAD]] unwind to caller
 
 // Possibly throwing function call within a catch-all
-void test5() {
+void catch_all() {
   try {
     may_throw();
   } catch (...) {
@@ -170,7 +170,7 @@ void test5() {
   }
 }
 
-// CHECK-LABEL: @_Z5test5v()
+// CHECK-LABEL: @_Z9catch_allv()
 
 // CHECK:   %[[CATCHSWITCH:.*]] = catchswitch within none [label %[[CATCHSTART_BB]]] unwind to caller
 
@@ -198,7 +198,7 @@ void test5() {
 // CHECK-NEXT:   unreachable
 
 // Try-catch with cleanups
-void test6() {
+void try_catch_w_cleanups() {
   Cleanup c1;
   try {
     Cleanup c2;
@@ -209,7 +209,7 @@ void test6() {
   }
 }
 
-// CHECK-LABEL: @_Z5test6v()
+// CHECK-LABEL: @_Z20try_catch_w_cleanupsv()
 // CHECK:   invoke void @_Z9may_throwv()
 // CHECK-NEXT:           to label %{{.*}} unwind label %[[EHCLEANUP_BB0:.*]]
 
@@ -254,7 +254,7 @@ void test6() {
 // CHECK-NEXT:   unreachable
 
 // Nested try-catches within a try with cleanups
-void test7() {
+void nested_try_catches_with_cleanups() {
   Cleanup c1;
   may_throw();
   try {
@@ -275,7 +275,7 @@ void test7() {
   }
 }
 
-// CHECK-LABEL: @_Z5test7v()
+// CHECK-LABEL: @_Z32nested_try_catches_with_cleanupsv()
 // CHECK:   invoke void @_Z9may_throwv()
 
 // CHECK:   invoke void @_Z9may_throwv()
@@ -340,7 +340,7 @@ void test7() {
 // CHECK:   unreachable
 
 // Nested try-catches within a catch
-void test8() {
+void nested_try_catch_within_catch() {
   try {
     may_throw();
   } catch (int) {
@@ -352,7 +352,7 @@ void test8() {
   }
 }
 
-// CHECK-LABEL: @_Z5test8v()
+// CHECK-LABEL: @_Z29nested_try_catch_within_catchv()
 // CHECK:   invoke void @_Z9may_throwv()
 
 // CHECK:   %[[CATCHSWITCH0:.*]] = catchswitch within none
@@ -402,19 +402,19 @@ void noexcept_throw() noexcept {
 // This is controlled by -Wwasm-exception-spec, which is on by default. This
 // warning can be suppressed with -Wno-wasm-exception-spec. Checks if a warning
 // message is correctly printed or not printed depending on the options.
-void test9() throw(int) {
+void exception_spec_warning() throw(int) {
 }
 // WARNING-DEFAULT: warning: dynamic exception specifications with types are currently ignored in wasm
 // WARNING-ON: warning: dynamic exception specifications with types are currently ignored in wasm
 // WARNING-OFF-NOT: warning: dynamic exception specifications with types are currently ignored in wasm
 // EM-EH-WARNING: warning: dynamic exception specifications with types are currently ignored in wasm
 
-// Wasm curremtly treats 'throw()' in the same way as 'noexept'. Check if the
+// Wasm currently treats 'throw()' in the same way as 'noexcept'. Check if the
 // same warning message is printed as if when a 'noexcept' function throws.
-void test10() throw() {
+void exception_spec_throw_empty() throw() {
   throw 3;
 }
-// WARNING-DEFAULT: warning: 'test10' has a non-throwing exception specification but can still throw
+// WARNING-DEFAULT: warning: 'exception_spec_throw_empty' has a non-throwing exception specification but can still throw
 // WARNING-DEFAULT: function declared non-throwing here
 
 // Here we only check if the command enables wasm exception handling in the
diff --git a/clang/test/CodeGenHLSL/BasicFeatures/standard_conversion_sequences.hlsl b/clang/test/CodeGenHLSL/BasicFeatures/standard_conversion_sequences.hlsl
index 1665a0260ab05..6770efefe94fe 100644
--- a/clang/test/CodeGenHLSL/BasicFeatures/standard_conversion_sequences.hlsl
+++ b/clang/test/CodeGenHLSL/BasicFeatures/standard_conversion_sequences.hlsl
@@ -91,13 +91,12 @@ void l4_to_i2() {
 
 // CHECK-LABEL: i2_to_b2
 // CHECK: [[l2:%.*]] = alloca <2 x i32>
-// CHECK: [[b2:%.*]] = alloca i8
+// CHECK: [[b2:%.*]] = alloca <2 x i32>
 // CHECK: store <2 x i32> splat (i32 8), ptr [[i2]]
 // CHECK: [[veci2:%.*]] = load <2 x i32>, ptr [[i2]]
 // CHECK: [[vecb2:%.*]] = icmp ne <2 x i32> [[veci2]], zeroinitializer
-// CHECK: [[vecb8:%.*]] = shufflevector <2 x i1> [[vecb2]], <2 x i1> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-// CHECK: [[i8:%.*]] = bitcast <8 x i1> [[vecb8]] to i8
-// CHECK: store i8 [[i8]], ptr [[b2]]
+// CHECK: [[vecb8:%.*]] = zext <2 x i1> [[vecb2]] to <2 x i32>
+// CHECK: store <2 x i32> [[vecb8]], ptr [[b2]]
 void i2_to_b2() {
   vector<int, 2> i2 = 8;
   vector<bool, 2> b2 = i2;
@@ -105,14 +104,13 @@ void i2_to_b2() {
 
 // CHECK-LABEL: d4_to_b2
 // CHECK: [[d4:%.*]] = alloca <4 x double>
-// CHECK: [[b2:%.*]] = alloca i8
+// CHECK: [[b2:%.*]] = alloca <2 x i32>
 // CHECK: store <4 x double> splat (double 9.000000e+00), ptr [[d4]]
 // CHECK: [[vecd4:%.*]] = load <4 x double>, ptr [[d4]]
 // CHECK: [[vecb4:%.*]] = fcmp reassoc nnan ninf nsz arcp afn une <4 x double> [[vecd4]], zeroinitializer
 // CHECK: [[vecd2:%.*]] = shufflevector <4 x i1> [[vecb4]], <4 x i1> poison, <2 x i32> <i32 0, i32 1>
-// CHECK: [[vecb8:%.*]] = shufflevector <2 x i1> [[vecd2]], <2 x i1> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-// CHECK: [[i8:%.*]] = bitcast <8 x i1> [[vecb8]] to i8
-// CHECK: store i8 [[i8]], ptr [[b2]]
+// CHECK: [[vecb8:%.*]] = zext <2 x i1> [[vecd2]] to <2 x i32>
+// CHECK: store <2 x i32> [[vecb8]], ptr [[b2]]
 void d4_to_b2() {
   vector<double,4> d4 = 9.0;
   vector<bool, 2> b2 = d4;
diff --git a/clang/test/CodeGenHLSL/BoolVector.hlsl b/clang/test/CodeGenHLSL/BoolVector.hlsl
new file mode 100644
index 0000000000000..5e889d50be98d
--- /dev/null
+++ b/clang/test/CodeGenHLSL/BoolVector.hlsl
@@ -0,0 +1,111 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.3-library -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
+
+// CHECK: %struct.S = type { <2 x i32>, float }
+// CHECK: [[ConstS:@.*]] = private unnamed_addr constant %struct.S { <2 x i32> splat (i32 1), float 1.000000e+00 }, align 8
+// CHECK: [[ConstArr:.*]] = private unnamed_addr constant [2 x <2 x i32>] [<2 x i32> splat (i32 1), <2 x i32> zeroinitializer], align 8
+
+struct S {
+    bool2 bv;
+    float f;
+};
+
+// CHECK-LABEL: define noundef i1 {{.*}}fn1{{.*}}
+// CHECK: [[B:%.*]] = alloca <2 x i32>, align 8
+// CHECK-NEXT: store <2 x i32> splat (i32 1), ptr [[B]], align 8
+// CHECK-NEXT: [[BoolVec:%.*]] = load <2 x i32>, ptr [[B]], align 8
+// CHECK-NEXT: [[L:%.*]] = trunc <2 x i32> [[BoolVec:%.*]] to <2 x i1>
+// CHECK-NEXT: [[VecExt:%.*]] = extractelement <2 x i1> [[L]], i32 0
+// CHECK-NEXT: ret i1 [[VecExt]]
+bool fn1() {
+  bool2 B = {true,true};
+  return B[0];
+}
+
+// CHECK-LABEL: define noundef <2 x i1> {{.*}}fn2{{.*}}
+// CHECK: [[VAddr:%.*]] = alloca i32, align 4
+// CHECK-NEXT: [[A:%.*]] = alloca <2 x i32>, align 8
+// CHECK-NEXT: [[StoreV:%.*]] = zext i1 {{.*}} to i32
+// CHECK-NEXT: store i32 [[StoreV]], ptr [[VAddr]], align 4
+// CHECK-NEXT: [[L:%.*]] = load i32, ptr [[VAddr]], align 4
+// CHECK-NEXT: [[LoadV:%.*]] = trunc i32 [[L]] to i1
+// CHECK-NEXT: [[Vec:%.*]] = insertelement <2 x i1> poison, i1 [[LoadV]], i32 0
+// CHECK-NEXT: [[Vec1:%.*]] = insertelement <2 x i1> [[Vec]], i1 true, i32 1
+// CHECK-NEXT: [[Z:%.*]] = zext <2 x i1> [[Vec1]] to <2 x i32>
+// CHECK-NEXT: store <2 x i32> [[Z]], ptr [[A]], align 8
+// CHECK-NEXT: [[LoadBV:%.*]] = load <2 x i32>, ptr [[A]], align 8
+// CHECK-NEXT: [[LoadV2:%.*]] = trunc <2 x i32> [[LoadBV]] to <2 x i1>
+// CHECK-NEXT: ret <2 x i1> [[LoadV2]]
+bool2 fn2(bool V) {
+  bool2 A = {V,true};
+  return A;
+}
+
+// CHECK-LABEL: define noundef i1 {{.*}}fn3{{.*}}
+// CHECK: [[s:%.*]] = alloca %struct.S, align 8
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 8 [[s]], ptr align 8 [[ConstS]], i32 16, i1 false)
+// CHECK-NEXT: [[BV:%.*]] = getelementptr inbounds nuw %struct.S, ptr [[s]], i32 0, i32 0
+// CHECK-NEXT: [[LBV:%.*]] = load <2 x i32>, ptr [[BV]], align 8
+// CHECK-NEXT: [[LV:%.*]] = trunc <2 x i32> [[LBV]] to <2 x i1>
+// CHECK-NEXT: [[VX:%.*]] = extractelement <2 x i1> [[LV]], i32 0
+// CHECK-NEXT: ret i1 [[VX]]
+bool fn3() {
+  S s = {{true,true}, 1.0};
+  return s.bv[0];
+}
+
+// CHECK-LABEL: define noundef i1 {{.*}}fn4{{.*}}
+// CHECK: [[Arr:%.*]] = alloca [2 x <2 x i32>], align 8
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 8 [[Arr]], ptr align 8 [[ConstArr]], i32 16, i1 false)
+// CHECK-NEXT: [[Idx:%.*]] = getelementptr inbounds [2 x <2 x i32>], ptr [[Arr]], i32 0, i32 0
+// CHECK-NEXT: [[L:%.*]] = load <2 x i32>, ptr [[Idx]], align 8
+// CHECK-NEXT: [[LV:%.*]] = trunc <2 x i32> [[L]] to <2 x i1>
+// CHECK-NEXT: [[VX:%.*]] = extractelement <2 x i1> [[LV]], i32 1
+// CHECK-NEXT: ret i1 [[VX]]
+bool fn4() {
+  bool2 Arr[2] = {{true,true}, {false,false}};
+  return Arr[0][1];
+}
+
+// CHECK-LABEL: define void {{.*}}fn5{{.*}}
+// CHECK: [[Arr:%.*]] = alloca <2 x i32>, align 8
+// CHECK-NEXT: store <2 x i32> splat (i32 1), ptr [[Arr]], align 8
+// CHECK-NEXT: [[L:%.*]] = load <2 x i32>, ptr [[Arr]], align 8
+// CHECK-NEXT: [[V:%.*]] = insertelement <2 x i32> [[L]], i32 0, i32 1
+// CHECK-NEXT: store <2 x i32> [[V]], ptr [[Arr]], align 8
+// CHECK-NEXT: ret void
+void fn5() {
+  bool2 Arr = {true,true};
+  Arr[1] = false;
+}
+
+// CHECK-LABEL: define void {{.*}}fn6{{.*}}
+// CHECK: [[V:%.*]] = alloca i32, align 4
+// CHECK-NEXT: [[S:%.*]] = alloca %struct.S, align 8
+// CHECK-NEXT: store i32 0, ptr [[V]], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 8 [[S]], ptr align 8 {{.*}}, i32 16, i1 false)
+// CHECK-NEXT: [[Y:%.*]] = load i32, ptr [[V]], align 4
+// CHECK-NEXT: [[LV:%.*]] = trunc i32 [[Y]] to i1
+// CHECK-NEXT: [[BV:%.*]] = getelementptr inbounds nuw %struct.S, ptr [[S]], i32 0, i32 0
+// CHECK-NEXT: [[X:%.*]] = load <2 x i32>, ptr [[BV]], align 8
+// CHECK-NEXT: [[Z:%.*]] = zext i1 [[LV]] to i32
+// CHECK-NEXT: [[VI:%.*]] = insertelement <2 x i32> [[X]], i32 [[Z]], i32 1
+// CHECK-NEXT: store <2 x i32> [[VI]], ptr [[BV]], align 8
+// CHECK-NEXT: ret void
+void fn6() {
+  bool V = false;
+  S s = {{true,true}, 1.0};
+  s.bv[1] = V;
+}
+
+// CHECK-LABEL: define void {{.*}}fn7{{.*}}
+// CHECK: [[Arr:%.*]] = alloca [2 x <2 x i32>], align 8
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 8 [[Arr]], ptr align 8 {{.*}}, i32 16, i1 false)
+// CHECK-NEXT: [[Idx:%.*]] = getelementptr inbounds [2 x <2 x i32>], ptr [[Arr]], i32 0, i32 0
+// CHECK-NEXT: [[X:%.*]] = load <2 x i32>, ptr [[Idx]], align 8
+// CHECK-NEXT: [[VI:%.*]] = insertelement <2 x i32> [[X]], i32 0, i32 1
+// CHECK-NEXT: store <2 x i32> [[VI]], ptr [[Idx]], align 8
+// CHECK-NEXT: ret void
+void fn7() {
+  bool2 Arr[2] = {{true,true}, {false,false}};
+  Arr[0][1] = false;
+}
diff --git a/clang/test/CodeGenHLSL/builtins/ScalarSwizzles.hlsl b/clang/test/CodeGenHLSL/builtins/ScalarSwizzles.hlsl
index 97711c9ee25a1..96e17046ee934 100644
--- a/clang/test/CodeGenHLSL/builtins/ScalarSwizzles.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/ScalarSwizzles.hlsl
@@ -11,13 +11,23 @@ int2 ToTwoInts(int V){
 }
 
 // CHECK-LABEL: ToFourFloats
-// [[splat:%.*]] = insertelement <1 x float> poison, float {{.*}}, i64 0
-// [[vec4:%.*]] = shufflevector <1 x float> [[splat]], <1 x float> poison, <4 x i32> zeroinitializer
+// CHECK: [[splat:%.*]] = insertelement <1 x float> poison, float {{.*}}, i64 0
+// CHECK: [[vec4:%.*]] = shufflevector <1 x float> [[splat]], <1 x float> poison, <4 x i32> zeroinitializer
 // ret <4 x float> [[vec4]]
 float4 ToFourFloats(float V){
   return V.rrrr;
 }
 
+// CHECK-LABEL: ToFourBools
+// CHECK: {{%.*}} = zext i1 {{.*}} to i32
+// CHECK: [[splat:%.*]] = insertelement <1 x i32> poison, i32 {{.*}}, i64 0
+// CHECK-NEXT: [[vec4:%.*]] = shufflevector <1 x i32> [[splat]], <1 x i32> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT: [[vec2Ret:%.*]] = trunc <4 x i32> [[vec4]] to <4 x i1>
+// CHECK-NEXT: ret <4 x i1> [[vec2Ret]]
+bool4 ToFourBools(bool V) {
+  return V.rrrr;
+}
+
 // CHECK-LABEL: FillOne
 // CHECK: [[vec1Ptr:%.*]] = alloca <1 x i32>, align 4
 // CHECK: store <1 x i32> splat (i32 1), ptr [[vec1Ptr]], align 4
@@ -93,6 +103,17 @@ vector<float, 1> FillOneHalfFloat(){
   return .5f.r;
 }
 
+// CHECK-LABEL: FillTrue
+// CHECK: [[Tmp:%.*]] = alloca <1 x i32>, align 4
+// CHECK-NEXT: store <1 x i32> splat (i32 1), ptr [[Tmp]], align 4
+// CHECK-NEXT: [[Vec1:%.*]] = load <1 x i32>, ptr [[Tmp]], align 4
+// CHECK-NEXT: [[Vec2:%.*]] = shufflevector <1 x i32> [[Vec1]], <1 x i32> poison, <2 x i32> zeroinitializer
+// CHECK-NEXT: [[Vec2Ret:%.*]] = trunc <2 x i32> [[Vec2]] to <2 x i1>
+// CHECK-NEXT: ret <2 x i1> [[Vec2Ret]]
+bool2 FillTrue() {
+  return true.xx;
+}
+
 // The initial codegen for this case is correct but a bit odd. The IR optimizer
 // cleans this up very nicely.
 
@@ -110,6 +131,25 @@ float2 HowManyFloats(float V) {
   return V.rr.rr;
 }
 
+// CHECK-LABEL: HowManyBools
+// CHECK: [[VAddr:%.*]] = alloca i32, align 4
+// CHECK-NEXT: [[Vec2Ptr:%.*]] = alloca <2 x i32>, align 8
+// CHECK-NEXT: [[Tmp:%.*]] = zext i1 {{.*}} to i32
+// CHECK-NEXT: store i32 [[Tmp]], ptr [[VAddr]], align 4
+// CHECK-NEXT: [[VVal:%.*]] = load i32, ptr [[VAddr]], align 4
+// CHECK-NEXT: [[Splat:%.*]] = insertelement <1 x i32> poison, i32 [[VVal]], i64 0
+// CHECK-NEXT: [[Vec2:%.*]] = shufflevector <1 x i32> [[Splat]], <1 x i32> poison, <2 x i32> zeroinitializer
+// CHECK-NEXT: [[Trunc:%.*]] = trunc <2 x i32> [[Vec2]] to <2 x i1>
+// CHECK-NEXT: [[Ext:%.*]] = zext <2 x i1> [[Trunc]] to <2 x i32>
+// CHECK-NEXT: store <2 x i32> [[Ext]], ptr [[Vec2Ptr]], align 8
+// CHECK-NEXT: [[V2:%.*]] = load <2 x i32>, ptr [[Vec2Ptr]], align 8
+// CHECK-NEXT: [[V3:%.*]] = shufflevector <2 x i32> [[V2]], <2 x i32> poison, <2 x i32> zeroinitializer
+// CHECK-NEXT: [[LV1:%.*]] = trunc <2 x i32> [[V3]] to <2 x i1>
+// CHECK-NEXT: ret <2 x i1> [[LV1]]
+bool2 HowManyBools(bool V) {
+  return V.rr.rr;
+}
+
 // This codegen is gnarly because `1.l` is a double, so this creates double
 // vectors that need to be truncated down to floats. The optimizer cleans this
 // up nicely too.
@@ -166,3 +206,99 @@ int AssignInt(int V){
   X.x = V.x + V.x;
   return X;
 }
+
+// CHECK-LABEL: AssignBool
+// CHECK: [[VAddr:%.*]] = alloca i32, align 4
+// CHECK-NEXT: [[XAddr:%.*]] = alloca i32, align 4
+// CHECK-NEXT: [[Zext:%.*]] = zext i1 %V to i32
+// CHECK-NEXT: store i32 [[Zext]], ptr [[VAddr]], align 4
+// CHECK-NEXT: [[X:%.*]] = load i32, ptr [[VAddr]], align 4
+// CHECK-NEXT: [[Splat:%.*]] = insertelement <1 x i32> poison, i32 [[X]], i64 0
+// CHECK-NEXT: [[Y:%.*]] = extractelement <1 x i32> [[Splat]], i32 0
+// CHECK-NEXT: [[Z:%.*]] = trunc i32 [[Y]] to i1
+// CHECK-NEXT: [[A:%.*]] = zext i1 [[Z]] to i32
+// CHECK-NEXT: store i32 [[A]], ptr [[XAddr]], align 4
+// CHECK-NEXT: [[B:%.*]] = load i32, ptr [[VAddr]], align 4
+// CHECK-NEXT: [[Splat2:%.*]] = insertelement <1 x i32> poison, i32 [[B]], i64 0
+// CHECK-NEXT: [[C:%.*]] = extractelement <1 x i32> [[Splat2]], i32 0
+// CHECK-NEXT: [[D:%.*]] = trunc i32 [[C]] to i1
+// CHECK-NEXT: br i1 [[D]], label %lor.end, label %lor.rhs
+
+// CHECK: lor.rhs:
+// CHECK-NEXT: [[E:%.*]] = load i32, ptr [[VAddr]], align 4
+// CHECK-NEXT: [[Splat3:%.*]] = insertelement <1 x i32> poison, i32 [[E]], i64 0
+// CHECK-NEXT: [[F:%.*]] = extractelement <1 x i32> [[Splat3]], i32 0
+// CHECK-NEXT: [[G:%.*]] = trunc i32 [[F]] to i1
+// CHECK-NEXT: br label %lor.end
+
+// CHECK: lor.end:
+// CHECK-NEXT: [[H:%.*]] = phi i1 [ true, %entry ], [ [[G]], %lor.rhs ]
+// CHECK-NEXT: store i1 [[H]], ptr [[XAddr]], align 4
+// CHECK-NEXT: [[I:%.*]] = load i32, ptr [[XAddr]], align 4
+// CHECK-NEXT: [[LoadV:%.*]] = trunc i32 [[I]] to i1
+// CHECK-NEXT: ret i1 [[LoadV]]
+bool AssignBool(bool V) {
+  bool X = V.x;
+  X.x = V.x || V.x;
+  return X;
+}
+
+// CHECK-LABEL: AssignBool2
+// CHECK: [[VAdddr:%.*]] = alloca i32, align 4
+// CHECK-NEXT: [[X:%.*]] = alloca <2 x i32>, align 8
+// CHECK-NEXT: [[Tmp:%.*]] = alloca <1 x i32>, align 4
+// CHECK-NEXT: [[SV:%.*]] = zext i1 %V to i32
+// CHECK-NEXT: store i32 [[SV]], ptr [[VAddr]], align 4
+// CHECK-NEXT: store <1 x i32> splat (i32 1), ptr [[Tmp]], align 4
+// CHECK-NEXT: [[Y:%.*]] = load <1 x i32>, ptr [[Tmp]], align 4
+// CHECK-NEXT: [[Z:%.*]] = shufflevector <1 x i32> [[Y]], <1 x i32> poison, <2 x i32> zeroinitializer
+// CHECK-NEXT: [[LV:%.*]] = trunc <2 x i32> [[Z]] to <2 x i1>
+// CHECK-NEXT: [[A:%.*]] = zext <2 x i1> [[LV]] to <2 x i32>
+// CHECK-NEXT: store <2 x i32> [[A]], ptr [[X]], align 8
+// CHECK-NEXT: [[B:%.*]] = load i32, ptr [[VAddr]], align 4
+// CHECK-NEXT: [[LV1:%.*]] = trunc i32 [[B]] to i1
+// CHECK-NEXT: [[C:%.*]] = load <2 x i32>, ptr [[X]], align 8
+// CHECK-NEXT: [[D:%.*]] = zext i1 [[LV1]] to i32
+// CHECK-NEXT: [[E:%.*]] = insertelement <2 x i32> [[C]], i32 [[D]], i32 1
+// CHECK-NEXT: store <2 x i32> [[E]], ptr [[X]], align 8
+// CHECK-NEXT: ret void
+void AssignBool2(bool V) {
+  bool2 X = true.xx;
+  X.y = V;
+}
+
+// CHECK-LABEL: AssignBool3
+// CHECK: [[VAddr:%.*]] = alloca <2 x i32>, align 8
+// CHECK-NEXT: [[X:%.*]] = alloca <2 x i32>, align 8
+// CHECK-NEXT: [[Y:%.*]] = zext <2 x i1> %V to <2 x i32>
+// CHECK-NEXT: store <2 x i32> [[Y]], ptr [[VAddr]], align 8
+// CHECK-NEXT: store <2 x i32> splat (i32 1), ptr [[X]], align 8
+// CHECK-NEXT: [[Z:%.*]] = load <2 x i32>, ptr [[VAddr]], align 8
+// CHECK-NEXT: [[LV:%.*]] = trunc <2 x i32> [[Z]] to <2 x i1>
+// CHECK-NEXT: [[A:%.*]] = load <2 x i32>, ptr [[X]], align 8
+// CHECK-NEXT: [[B:%.*]] = zext <2 x i1> [[LV]] to <2 x i32>
+// CHECK-NEXT: [[C:%.*]] = shufflevector <2 x i32> [[B]], <2 x i32> poison, <2 x i32> <i32 0, i32 1>
+// CHECK-NEXT: store <2 x i32> [[C]], ptr [[X]], align 8
+// CHECK-NEXT: ret void
+void AssignBool3(bool2 V) {
+  bool2 X = {true,true};
+  X.xy = V;
+}
+
+// CHECK-LABEL: AccessBools
+// CHECK: [[X:%.*]] = alloca <4 x i32>, align 16
+// CHECK-NEXT: [[Tmp:%.*]] = alloca <1 x i32>, align 4
+// CHECK-NEXT: store <1 x i32> splat (i32 1), ptr [[Tmp]], align 4
+// CHECK-NEXT: [[Y:%.*]] = load <1 x i32>, ptr [[Tmp]], align 4
+// CHECK-NEXT: [[Z:%.*]] = shufflevector <1 x i32> [[Y]], <1 x i32> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT: [[LV:%.*]] = trunc <4 x i32> [[Z]] to <4 x i1>
+// CHECK-NEXT: [[A:%.*]] = zext <4 x i1> [[LV]] to <4 x i32>
+// CHECK-NEXT: store <4 x i32> [[A]], ptr [[X]], align 16
+// CHECK-NEXT: [[B:%.*]] = load <4 x i32>, ptr [[X]], align 16
+// CHECK-NEXT: [[C:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
+// CHECK-NEXT: [[LV1:%.*]] = trunc <2 x i32> [[C]] to <2 x i1>
+// CHECK-NEXT: ret <2 x i1> [[LV1]]
+bool2 AccessBools() {
+  bool4 X = true.xxxx;
+  return X.zw;
+}
diff --git a/clang/test/CodeGenHLSL/builtins/select.hlsl b/clang/test/CodeGenHLSL/builtins/select.hlsl
index cade938b71a2b..196b8a90cd877 100644
--- a/clang/test/CodeGenHLSL/builtins/select.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/select.hlsl
@@ -52,3 +52,32 @@ int3 test_select_vector_3(bool3 cond0, int3 tVals, int3 fVals) {
 int4 test_select_vector_4(bool4 cond0, int4 tVals, int4 fVals) {
   return select(cond0, tVals, fVals);
 }
+
+// CHECK-LABEL: test_select_vector_scalar_vector
+// CHECK: [[SPLAT_SRC1:%.*]] = insertelement <4 x i32> poison, i32 {{%.*}}, i64 0
+// CHECK: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[SPLAT_SRC1]], <4 x i32> poison, <4 x i32> zeroinitializer
+// CHECK: [[SELECT:%.*]] = select <4 x i1> {{%.*}}, <4 x i32> [[SPLAT1]], <4 x i32> {{%.*}}
+// CHECK: ret <4 x i32> [[SELECT]]
+int4 test_select_vector_scalar_vector(bool4 cond0, int tVal, int4 fVals) {
+  return select(cond0, tVal, fVals);
+}
+
+// CHECK-LABEL: test_select_vector_vector_scalar
+// CHECK: [[SPLAT_SRC1:%.*]] = insertelement <4 x i32> poison, i32 {{%.*}}, i64 0
+// CHECK: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[SPLAT_SRC1]], <4 x i32> poison, <4 x i32> zeroinitializer
+// CHECK: [[SELECT:%.*]] = select <4 x i1> {{%.*}}, <4 x i32> {{%.*}}, <4 x i32> [[SPLAT1]]
+// CHECK: ret <4 x i32> [[SELECT]]
+int4 test_select_vector_vector_scalar(bool4 cond0, int4 tVals, int fVal) {
+  return select(cond0, tVals, fVal);
+}
+
+// CHECK-LABEL: test_select_vector_scalar_scalar
+// CHECK: [[SPLAT_SRC1:%.*]] = insertelement <4 x i32> poison, i32 {{%.*}}, i64 0
+// CHECK: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[SPLAT_SRC1]], <4 x i32> poison, <4 x i32> zeroinitializer
+// CHECK: [[SPLAT_SRC2:%.*]] = insertelement <4 x i32> poison, i32 %3, i64 0
+// CHECK: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[SPLAT_SRC2]], <4 x i32> poison, <4 x i32> zeroinitializer
+// CHECK: [[SELECT:%.*]] = select <4 x i1> {{%.*}}, <4 x i32> [[SPLAT1]], <4 x i32> [[SPLAT2]]
+// CHECK: ret <4 x i32> [[SELECT]]
+int4 test_select_vector_scalar_scalar(bool4 cond0, int tVal, int fVal) {
+  return select(cond0, tVal, fVal);
+}
diff --git a/clang/test/CodeGenHLSL/cbuffer.hlsl b/clang/test/CodeGenHLSL/cbuffer.hlsl
index 38093c6dfacd7..b5e435619438f 100644
--- a/clang/test/CodeGenHLSL/cbuffer.hlsl
+++ b/clang/test/CodeGenHLSL/cbuffer.hlsl
@@ -20,6 +20,14 @@
 // CHECK: %anon = type <{ float }>
 // CHECK: %anon.0 = type <{ <2 x i32> }>
 
+// CHECK: %__cblayout_CB_A = type <{ [2 x double], [3 x <3 x float>], float, [3 x double], half, [1 x <2 x double>], float, [2 x <3 x half>], <3 x half> }>
+// CHECK: %__cblayout_CB_B = type <{ [3 x <3 x double>], <3 x half> }>
+// CHECK: %__cblayout_CB_C = type <{ i32, target("dx.Layout", %F, 96, 0, 16, 28, 32, 56, 64, 80, 84, 90), half, target("dx.Layout", %G, 258, 0, 48, 64, 256), double }>
+
+// CHECK: %F = type <{ double, <3 x float>, float, <3 x double>, half, <2 x double>, float, <3 x half>, <3 x half> }>
+// CHECK: %G = type <{ target("dx.Layout", %E, 36, 0, 8, 16, 20, 22, 24, 32), [1 x float], [2 x target("dx.Layout", %F, 96, 0, 16, 28, 32, 56, 64, 80, 84, 90)], half }>
+// CHECK: %E = type <{ float, double, float, half, i16, i64, i32 }>
+
 cbuffer CBScalars : register(b1, space5) {
   float a1;
   double a2;
@@ -155,6 +163,64 @@ cbuffer CBMix {
     uint16_t f9;
 };  
 
+// CHECK: @CB_A.cb = external constant target("dx.CBuffer", target("dx.Layout", %__cblayout_CB_A, 188, 0, 32, 76, 80, 120, 128, 144, 160, 182))
+
+cbuffer CB_A {
+  double B0[2];
+  float3 B1[3];
+  float B2;
+  double B3[3];
+  half B4;
+  double2 B5[1];
+  float B6;
+  half3 B7[2];
+  half3 B8;
+}
+
+// CHECK: @CB_B.cb = external constant target("dx.CBuffer", target("dx.Layout", %__cblayout_CB_B, 94, 0, 88))
+cbuffer CB_B {
+  double3 B9[3];
+  half3 B10;
+}
+
+struct E {
+  float A0;
+  double A1;
+  float A2;
+  half A3;
+  int16_t A4;
+  int64_t A5;
+  int A6;
+};
+
+struct F {
+  double B0;
+  float3 B1;
+  float B2;
+  double3 B3;
+  half B4;
+  double2 B5;
+  float B6;
+  half3 B7;
+  half3 B8;
+};
+
+struct G {
+  E C0;
+  float C1[1];
+  F C2[2];
+  half C3;
+};
+
+// CHECK: @CB_C.cb = external constant target("dx.CBuffer", target("dx.Layout", %__cblayout_CB_C, 400, 0, 16, 112, 128, 392))
+cbuffer CB_C {
+  int D0;
+  F D1;
+  half D2;
+  G D3;
+  double D4;
+}
+
 // CHECK: define internal void @_init_resource_CBScalars.cb()
 // CHECK-NEXT: entry:
 // CHECK-NEXT: %[[HANDLE1:.*]] = call target("dx.CBuffer", target("dx.Layout", %__cblayout_CBScalars, 56, 0, 8, 16, 24, 32, 36, 40, 48))
@@ -171,7 +237,7 @@ RWBuffer<float> Buf;
 
 [numthreads(4,1,1)]
 void main() {
-  Buf[0] = a1 + b1.z + c1[2] + a.f1.y + f1;
+  Buf[0] = a1 + b1.z + c1[2] + a.f1.y + f1 + B1[0].x + B10.z + D1.B2;
 }
 
 // CHECK: define internal void @_GLOBAL__sub_I_cbuffer.hlsl()
@@ -179,7 +245,8 @@ void main() {
 // CHECK-NEXT: call void @_init_resource_CBScalars.cb()
 // CHECK-NEXT: call void @_init_resource_CBArrays.cb()
 
-// CHECK: !hlsl.cbs = !{![[CBSCALARS:[0-9]+]], ![[CBVECTORS:[0-9]+]], ![[CBARRAYS:[0-9]+]], ![[CBSTRUCTS:[0-9]+]], ![[CBMIX:[0-9]+]]}
+// CHECK: !hlsl.cbs = !{![[CBSCALARS:[0-9]+]], ![[CBVECTORS:[0-9]+]], ![[CBARRAYS:[0-9]+]], ![[CBSTRUCTS:[0-9]+]], ![[CBMIX:[0-9]+]],
+// CHECK-SAME: ![[CB_A:[0-9]+]], ![[CB_B:[0-9]+]], ![[CB_C:[0-9]+]]}
 
 // CHECK: ![[CBSCALARS]] = !{ptr @CBScalars.cb, ptr addrspace(2) @a1, ptr addrspace(2) @a2, ptr addrspace(2) @a3, ptr addrspace(2) @a4,
 // CHECK-SAME: ptr addrspace(2) @a5, ptr addrspace(2) @a6, ptr addrspace(2) @a7, ptr addrspace(2) @a8}
@@ -195,3 +262,10 @@ void main() {
 
 // CHECK: ![[CBMIX]] = !{ptr @CBMix.cb, ptr addrspace(2) @test, ptr addrspace(2) @f1, ptr addrspace(2) @f2, ptr addrspace(2) @f3,
 // CHECK-SAME: ptr addrspace(2) @f4, ptr addrspace(2) @f5, ptr addrspace(2) @f6, ptr addrspace(2) @f7, ptr addrspace(2) @f8, ptr addrspace(2) @f9}
+
+// CHECK: ![[CB_A]] = !{ptr @CB_A.cb, ptr addrspace(2) @B0, ptr addrspace(2) @B1, ptr addrspace(2) @B2, ptr addrspace(2) @B3, ptr addrspace(2) @B4,
+// CHECK-SAME: ptr addrspace(2) @B5, ptr addrspace(2) @B6, ptr addrspace(2) @B7, ptr addrspace(2) @B8}
+
+// CHECK: ![[CB_B]] = !{ptr @CB_B.cb, ptr addrspace(2) @B9, ptr addrspace(2) @B10}
+
+// CHECK: ![[CB_C]] = !{ptr @CB_C.cb, ptr addrspace(2) @D0, ptr addrspace(2) @D1, ptr addrspace(2) @D2, ptr addrspace(2) @D3, ptr addrspace(2) @D4}
diff --git a/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel-linking.cl b/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel-linking.cl
new file mode 100644
index 0000000000000..a61c5530c8a73
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel-linking.cl
@@ -0,0 +1,80 @@
+// Make sure that invoking blocks in static functions with the same name in
+// different modules are linked together.
+
+// RUN: %clang_cc1 -O0 -cl-std=CL2.0 -triple amdgcn-amd-amdhsa -fno-ident -DKERNEL_NAME=test_kernel_first -DTYPE=float -DCONST=256.0f -emit-llvm-bc -o %t.0.bc %s
+// RUN: %clang_cc1 -O0 -cl-std=CL2.0 -triple amdgcn-amd-amdhsa -fno-ident -DKERNEL_NAME=test_kernel_second -DTYPE=int -DCONST=128.0f -emit-llvm-bc -o %t.1.bc %s
+
+// Make sure nothing strange happens with the linkage choices.
+// RUN: opt -passes=globalopt -o %t.opt.0.bc %t.0.bc
+// RUN: opt -passes=globalopt -o %t.opt.1.bc %t.1.bc
+
+// Check the result of linking
+// RUN: llvm-link -S %t.opt.0.bc %t.opt.1.bc -o - | FileCheck %s
+
+// Make sure that a block invoke used with the same name works in multiple
+// translation units
+
+// CHECK: @llvm.used = appending addrspace(1) global [4 x ptr] [ptr @__static_invoker_block_invoke_kernel, ptr addrspacecast (ptr addrspace(1) @__static_invoker_block_invoke_kernel.runtime.handle to ptr), ptr @__static_invoker_block_invoke_kernel.2, ptr addrspacecast (ptr addrspace(1) @__static_invoker_block_invoke_kernel.runtime.handle.3 to ptr)], section "llvm.metadata"
+
+
+// CHECK: @__static_invoker_block_invoke_kernel.runtime.handle = internal addrspace(1) externally_initialized constant %block.runtime.handle.t zeroinitializer, section ".amdgpu.kernel.runtime.handle"
+// CHECK: @__static_invoker_block_invoke_kernel.runtime.handle.3 = internal addrspace(1) externally_initialized constant %block.runtime.handle.t zeroinitializer, section ".amdgpu.kernel.runtime.handle"
+
+// CHECK: define internal amdgpu_kernel void @__static_invoker_block_invoke_kernel(<{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1) }> %0) #{{[0-9]+}} !associated ![[ASSOC_FIRST_MD:[0-9]+]]
+
+
+// CHECK-LABEL: define internal void @__static_invoker_block_invoke(ptr noundef %.block_descriptor)
+// CHECK: call float @llvm.fmuladd.f32
+
+
+// CHECK-LABEL: define dso_local amdgpu_kernel void @test_kernel_first(
+
+
+// CHECK-LABEL: define internal fastcc void @static_invoker(ptr addrspace(1) noundef %outptr, ptr addrspace(1) noundef %argptr)
+// CHECK:  call i32 @__enqueue_kernel_basic(ptr addrspace(1) %{{[0-9]+}}, i32 %{{[0-9]+}}, ptr addrspace(5) %tmp, ptr addrspacecast (ptr addrspace(1) @__static_invoker_block_invoke_kernel.runtime.handle to ptr), ptr %{{.+}})
+
+// CHECK: declare i32 @__enqueue_kernel_basic(ptr addrspace(1), i32, ptr addrspace(5), ptr, ptr) local_unnamed_addr
+
+
+// CHECK: define internal amdgpu_kernel void @__static_invoker_block_invoke_kernel.2(<{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1) }> %0) #{{[0-9]+}} !associated ![[ASSOC_SECOND_MD:[0-9]+]]
+// CHECK: call void @__static_invoker_block_invoke.4(ptr %
+
+
+// CHECK-LABEL: define internal void @__static_invoker_block_invoke.4(ptr noundef %.block_descriptor)
+// CHECK: mul nsw i32
+// CHECK: sitofp
+// CHECK: fadd
+// CHECK: fptosi
+
+// CHECK-LABEL: define dso_local amdgpu_kernel void @test_kernel_second(ptr addrspace(1) noundef align 4 %outptr, ptr addrspace(1) noundef align 4 %argptr, ptr addrspace(1) noundef align 4 %difference)
+
+// CHECK-LABEL: define internal fastcc void @static_invoker.5(ptr addrspace(1) noundef %outptr, ptr addrspace(1) noundef %argptr) unnamed_addr #{{[0-9]+}} {
+// CHECK: call i32 @__enqueue_kernel_basic(ptr addrspace(1) %{{[0-9]+}}, i32 %{{[0-9]+}}, ptr addrspace(5) %tmp, ptr addrspacecast (ptr addrspace(1) @__static_invoker_block_invoke_kernel.runtime.handle.3 to ptr), ptr %{{.+}})
+
+
+typedef struct {int a;} ndrange_t;
+
+static void static_invoker(global TYPE* outptr, global TYPE* argptr) {
+  queue_t default_queue;
+  unsigned flags = 0;
+  ndrange_t ndrange;
+
+  enqueue_kernel(default_queue, flags, ndrange,
+  ^(void) {
+  global TYPE* f = argptr;
+  outptr[0] = f[1] * f[2] + CONST;
+  });
+}
+
+kernel void KERNEL_NAME(global TYPE *outptr, global TYPE *argptr, global TYPE *difference) {
+  queue_t default_queue;
+  unsigned flags = 0;
+  ndrange_t ndrange;
+
+  static_invoker(outptr, argptr);
+
+  *difference = CONST;
+}
+
+// CHECK: ![[ASSOC_FIRST_MD]] = !{ptr addrspace(1) @__static_invoker_block_invoke_kernel.runtime.handle}
+// CHECK: ![[ASSOC_SECOND_MD]] = !{ptr addrspace(1) @__static_invoker_block_invoke_kernel.runtime.handle.3}
diff --git a/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl b/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl
index 9ba36643c4b9e..367217579e765 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl
@@ -61,7 +61,13 @@ kernel void test_target_features_kernel(global int *i) {
 }
 
 //.
+// CHECK: @__test_block_invoke_kernel.runtime.handle = internal addrspace(1) externally_initialized constant %block.runtime.handle.t zeroinitializer, section ".amdgpu.kernel.runtime.handle"
+// CHECK: @__test_block_invoke_2_kernel.runtime.handle = internal addrspace(1) externally_initialized constant %block.runtime.handle.t.0 zeroinitializer, section ".amdgpu.kernel.runtime.handle"
+// CHECK: @__test_block_invoke_3_kernel.runtime.handle = internal addrspace(1) externally_initialized constant %block.runtime.handle.t.1 zeroinitializer, section ".amdgpu.kernel.runtime.handle"
+// CHECK: @__test_block_invoke_4_kernel.runtime.handle = internal addrspace(1) externally_initialized constant %block.runtime.handle.t.2 zeroinitializer, section ".amdgpu.kernel.runtime.handle"
 // CHECK: @__block_literal_global = internal addrspace(1) constant { i32, i32, ptr } { i32 16, i32 8, ptr @__test_target_features_kernel_block_invoke }, align 8 #0
+// CHECK: @__test_target_features_kernel_block_invoke_kernel.runtime.handle = internal addrspace(1) externally_initialized constant %block.runtime.handle.t.3 zeroinitializer, section ".amdgpu.kernel.runtime.handle"
+// CHECK: @llvm.used = appending addrspace(1) global [10 x ptr] [ptr @__test_block_invoke_kernel, ptr addrspacecast (ptr addrspace(1) @__test_block_invoke_kernel.runtime.handle to ptr), ptr @__test_block_invoke_2_kernel, ptr addrspacecast (ptr addrspace(1) @__test_block_invoke_2_kernel.runtime.handle to ptr), ptr @__test_block_invoke_3_kernel, ptr addrspacecast (ptr addrspace(1) @__test_block_invoke_3_kernel.runtime.handle to ptr), ptr @__test_block_invoke_4_kernel, ptr addrspacecast (ptr addrspace(1) @__test_block_invoke_4_kernel.runtime.handle to ptr), ptr @__test_target_features_kernel_block_invoke_kernel, ptr addrspacecast (ptr addrspace(1) @__test_target_features_kernel_block_invoke_kernel.runtime.handle to ptr)], section "llvm.metadata"
 // CHECK: @__oclc_ABI_version = weak_odr hidden local_unnamed_addr addrspace(4) constant i32 500
 //.
 // NOCPU: Function Attrs: convergent noinline norecurse nounwind optnone
@@ -140,7 +146,7 @@ kernel void test_target_features_kernel(global int *i) {
 // NOCPU-NEXT:    [[BLOCK_CAPTURED1:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), i8 }>, ptr [[BLOCK_ASCAST]], i32 0, i32 4
 // NOCPU-NEXT:    [[TMP3:%.*]] = load i8, ptr [[B_ADDR_ASCAST]], align 1
 // NOCPU-NEXT:    store i8 [[TMP3]], ptr [[BLOCK_CAPTURED1]], align 8
-// NOCPU-NEXT:    [[TMP4:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP0]], i32 [[TMP1]], ptr addrspace(5) [[TMP]], ptr @__test_block_invoke_kernel, ptr [[BLOCK_ASCAST]])
+// NOCPU-NEXT:    [[TMP4:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP0]], i32 [[TMP1]], ptr addrspace(5) [[TMP]], ptr addrspacecast (ptr addrspace(1) @__test_block_invoke_kernel.runtime.handle to ptr), ptr [[BLOCK_ASCAST]])
 // NOCPU-NEXT:    [[TMP5:%.*]] = load ptr addrspace(1), ptr [[DEFAULT_QUEUE_ASCAST]], align 8
 // NOCPU-NEXT:    [[TMP6:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4
 // NOCPU-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP2_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false)
@@ -162,7 +168,7 @@ kernel void test_target_features_kernel(global int *i) {
 // NOCPU-NEXT:    [[BLOCK_CAPTURED10:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[BLOCK3_ASCAST]], i32 0, i32 5
 // NOCPU-NEXT:    [[TMP10:%.*]] = load i64, ptr [[D_ADDR_ASCAST]], align 8
 // NOCPU-NEXT:    store i64 [[TMP10]], ptr [[BLOCK_CAPTURED10]], align 8
-// NOCPU-NEXT:    [[TMP11:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP5]], i32 [[TMP6]], ptr addrspace(5) [[VARTMP2]], ptr @__test_block_invoke_2_kernel, ptr [[BLOCK3_ASCAST]])
+// NOCPU-NEXT:    [[TMP11:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP5]], i32 [[TMP6]], ptr addrspace(5) [[VARTMP2]], ptr addrspacecast (ptr addrspace(1) @__test_block_invoke_2_kernel.runtime.handle to ptr), ptr [[BLOCK3_ASCAST]])
 // NOCPU-NEXT:    [[TMP12:%.*]] = load ptr addrspace(1), ptr [[DEFAULT_QUEUE_ASCAST]], align 8
 // NOCPU-NEXT:    [[TMP13:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4
 // NOCPU-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP11_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false)
@@ -186,7 +192,7 @@ kernel void test_target_features_kernel(global int *i) {
 // NOCPU-NEXT:    store i64 [[TMP17]], ptr [[BLOCK_CAPTURED19]], align 8
 // NOCPU-NEXT:    [[TMP18:%.*]] = getelementptr [1 x i64], ptr [[BLOCK_SIZES_ASCAST]], i32 0, i32 0
 // NOCPU-NEXT:    store i64 100, ptr [[TMP18]], align 8
-// NOCPU-NEXT:    [[TMP19:%.*]] = call i32 @__enqueue_kernel_varargs(ptr addrspace(1) [[TMP12]], i32 [[TMP13]], ptr addrspace(5) [[VARTMP11]], ptr @__test_block_invoke_3_kernel, ptr [[BLOCK12_ASCAST]], i32 1, ptr [[TMP18]])
+// NOCPU-NEXT:    [[TMP19:%.*]] = call i32 @__enqueue_kernel_varargs(ptr addrspace(1) [[TMP12]], i32 [[TMP13]], ptr addrspace(5) [[VARTMP11]], ptr addrspacecast (ptr addrspace(1) @__test_block_invoke_3_kernel.runtime.handle to ptr), ptr [[BLOCK12_ASCAST]], i32 1, ptr [[TMP18]])
 // NOCPU-NEXT:    [[BLOCK_SIZE22:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, i64, ptr addrspace(1) }>, ptr [[BLOCK21_ASCAST]], i32 0, i32 0
 // NOCPU-NEXT:    store i32 32, ptr [[BLOCK_SIZE22]], align 8
 // NOCPU-NEXT:    [[BLOCK_ALIGN23:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, i64, ptr addrspace(1) }>, ptr [[BLOCK21_ASCAST]], i32 0, i32 1
@@ -204,7 +210,7 @@ kernel void test_target_features_kernel(global int *i) {
 // NOCPU-NEXT:    [[TMP23:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4
 // NOCPU-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP27_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false)
 // NOCPU-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[BLOCK20_ASCAST]], align 8
-// NOCPU-NEXT:    [[TMP25:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP22]], i32 [[TMP23]], ptr addrspace(5) [[VARTMP27]], ptr @__test_block_invoke_4_kernel, ptr [[BLOCK21_ASCAST]])
+// NOCPU-NEXT:    [[TMP25:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP22]], i32 [[TMP23]], ptr addrspace(5) [[VARTMP27]], ptr addrspacecast (ptr addrspace(1) @__test_block_invoke_4_kernel.runtime.handle to ptr), ptr [[BLOCK21_ASCAST]])
 // NOCPU-NEXT:    ret void
 //
 //
@@ -229,7 +235,7 @@ kernel void test_target_features_kernel(global int *i) {
 //
 // NOCPU: Function Attrs: convergent nounwind
 // NOCPU-LABEL: define {{[^@]+}}@__test_block_invoke_kernel
-// NOCPU-SAME: (<{ i32, i32, ptr, ptr addrspace(1), i8 }> [[TMP0:%.*]]) #[[ATTR5:[0-9]+]] !kernel_arg_addr_space [[META7:![0-9]+]] !kernel_arg_access_qual [[META8:![0-9]+]] !kernel_arg_type [[META9:![0-9]+]] !kernel_arg_base_type [[META9]] !kernel_arg_type_qual [[META10:![0-9]+]] {
+// NOCPU-SAME: (<{ i32, i32, ptr, ptr addrspace(1), i8 }> [[TMP0:%.*]]) #[[ATTR5:[0-9]+]] !associated [[META7:![0-9]+]] !kernel_arg_addr_space [[META8:![0-9]+]] !kernel_arg_access_qual [[META9:![0-9]+]] !kernel_arg_type [[META10:![0-9]+]] !kernel_arg_base_type [[META10]] !kernel_arg_type_qual [[META11:![0-9]+]] {
 // NOCPU-NEXT:  entry:
 // NOCPU-NEXT:    [[TMP1:%.*]] = alloca <{ i32, i32, ptr, ptr addrspace(1), i8 }>, align 8, addrspace(5)
 // NOCPU-NEXT:    store <{ i32, i32, ptr, ptr addrspace(1), i8 }> [[TMP0]], ptr addrspace(5) [[TMP1]], align 8
@@ -265,7 +271,7 @@ kernel void test_target_features_kernel(global int *i) {
 //
 // NOCPU: Function Attrs: convergent nounwind
 // NOCPU-LABEL: define {{[^@]+}}@__test_block_invoke_2_kernel
-// NOCPU-SAME: (<{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0:%.*]]) #[[ATTR5]] !kernel_arg_addr_space [[META7]] !kernel_arg_access_qual [[META8]] !kernel_arg_type [[META9]] !kernel_arg_base_type [[META9]] !kernel_arg_type_qual [[META10]] {
+// NOCPU-SAME: (<{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0:%.*]]) #[[ATTR5]] !associated [[META12:![0-9]+]] !kernel_arg_addr_space [[META8]] !kernel_arg_access_qual [[META9]] !kernel_arg_type [[META10]] !kernel_arg_base_type [[META10]] !kernel_arg_type_qual [[META11]] {
 // NOCPU-NEXT:  entry:
 // NOCPU-NEXT:    [[TMP1:%.*]] = alloca <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, align 8, addrspace(5)
 // NOCPU-NEXT:    store <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0]], ptr addrspace(5) [[TMP1]], align 8
@@ -307,7 +313,7 @@ kernel void test_target_features_kernel(global int *i) {
 //
 // NOCPU: Function Attrs: convergent nounwind
 // NOCPU-LABEL: define {{[^@]+}}@__test_block_invoke_3_kernel
-// NOCPU-SAME: (<{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) #[[ATTR5]] !kernel_arg_addr_space [[META11:![0-9]+]] !kernel_arg_access_qual [[META12:![0-9]+]] !kernel_arg_type [[META13:![0-9]+]] !kernel_arg_base_type [[META13]] !kernel_arg_type_qual [[META14:![0-9]+]] {
+// NOCPU-SAME: (<{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) #[[ATTR5]] !associated [[META13:![0-9]+]] !kernel_arg_addr_space [[META14:![0-9]+]] !kernel_arg_access_qual [[META15:![0-9]+]] !kernel_arg_type [[META16:![0-9]+]] !kernel_arg_base_type [[META16]] !kernel_arg_type_qual [[META17:![0-9]+]] {
 // NOCPU-NEXT:  entry:
 // NOCPU-NEXT:    [[TMP2:%.*]] = alloca <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, align 8, addrspace(5)
 // NOCPU-NEXT:    store <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0]], ptr addrspace(5) [[TMP2]], align 8
@@ -336,7 +342,7 @@ kernel void test_target_features_kernel(global int *i) {
 //
 // NOCPU: Function Attrs: convergent nounwind
 // NOCPU-LABEL: define {{[^@]+}}@__test_block_invoke_4_kernel
-// NOCPU-SAME: (<{ i32, i32, ptr, i64, ptr addrspace(1) }> [[TMP0:%.*]]) #[[ATTR5]] !kernel_arg_addr_space [[META7]] !kernel_arg_access_qual [[META8]] !kernel_arg_type [[META9]] !kernel_arg_base_type [[META9]] !kernel_arg_type_qual [[META10]] {
+// NOCPU-SAME: (<{ i32, i32, ptr, i64, ptr addrspace(1) }> [[TMP0:%.*]]) #[[ATTR5]] !associated [[META18:![0-9]+]] !kernel_arg_addr_space [[META8]] !kernel_arg_access_qual [[META9]] !kernel_arg_type [[META10]] !kernel_arg_base_type [[META10]] !kernel_arg_type_qual [[META11]] {
 // NOCPU-NEXT:  entry:
 // NOCPU-NEXT:    [[TMP1:%.*]] = alloca <{ i32, i32, ptr, i64, ptr addrspace(1) }>, align 8, addrspace(5)
 // NOCPU-NEXT:    store <{ i32, i32, ptr, i64, ptr addrspace(1) }> [[TMP0]], ptr addrspace(5) [[TMP1]], align 8
@@ -347,7 +353,7 @@ kernel void test_target_features_kernel(global int *i) {
 //
 // NOCPU: Function Attrs: convergent noinline norecurse nounwind optnone
 // NOCPU-LABEL: define {{[^@]+}}@test_target_features_kernel
-// NOCPU-SAME: (ptr addrspace(1) noundef align 4 [[I:%.*]]) #[[ATTR6:[0-9]+]] !kernel_arg_addr_space [[META15:![0-9]+]] !kernel_arg_access_qual [[META8]] !kernel_arg_type [[META16:![0-9]+]] !kernel_arg_base_type [[META16]] !kernel_arg_type_qual [[META10]] {
+// NOCPU-SAME: (ptr addrspace(1) noundef align 4 [[I:%.*]]) #[[ATTR6:[0-9]+]] !kernel_arg_addr_space [[META19:![0-9]+]] !kernel_arg_access_qual [[META9]] !kernel_arg_type [[META20:![0-9]+]] !kernel_arg_base_type [[META20]] !kernel_arg_type_qual [[META11]] {
 // NOCPU-NEXT:  entry:
 // NOCPU-NEXT:    [[I_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
 // NOCPU-NEXT:    [[DEFAULT_QUEUE:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
@@ -365,7 +371,7 @@ kernel void test_target_features_kernel(global int *i) {
 // NOCPU-NEXT:    [[TMP1:%.*]] = load ptr addrspace(1), ptr [[DEFAULT_QUEUE_ASCAST]], align 8
 // NOCPU-NEXT:    [[TMP2:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4
 // NOCPU-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false)
-// NOCPU-NEXT:    [[TMP3:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP1]], i32 [[TMP2]], ptr addrspace(5) [[TMP]], ptr @__test_target_features_kernel_block_invoke_kernel, ptr addrspacecast (ptr addrspace(1) @__block_literal_global to ptr))
+// NOCPU-NEXT:    [[TMP3:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP1]], i32 [[TMP2]], ptr addrspace(5) [[TMP]], ptr addrspacecast (ptr addrspace(1) @__test_target_features_kernel_block_invoke_kernel.runtime.handle to ptr), ptr addrspacecast (ptr addrspace(1) @__block_literal_global to ptr))
 // NOCPU-NEXT:    ret void
 //
 //
@@ -385,7 +391,7 @@ kernel void test_target_features_kernel(global int *i) {
 //
 // NOCPU: Function Attrs: convergent nounwind
 // NOCPU-LABEL: define {{[^@]+}}@__test_target_features_kernel_block_invoke_kernel
-// NOCPU-SAME: ({ i32, i32, ptr } [[TMP0:%.*]]) #[[ATTR5]] !kernel_arg_addr_space [[META7]] !kernel_arg_access_qual [[META8]] !kernel_arg_type [[META9]] !kernel_arg_base_type [[META9]] !kernel_arg_type_qual [[META10]] {
+// NOCPU-SAME: ({ i32, i32, ptr } [[TMP0:%.*]]) #[[ATTR5]] !associated [[META21:![0-9]+]] !kernel_arg_addr_space [[META8]] !kernel_arg_access_qual [[META9]] !kernel_arg_type [[META10]] !kernel_arg_base_type [[META10]] !kernel_arg_type_qual [[META11]] {
 // NOCPU-NEXT:  entry:
 // NOCPU-NEXT:    [[TMP1:%.*]] = alloca { i32, i32, ptr }, align 8, addrspace(5)
 // NOCPU-NEXT:    store { i32, i32, ptr } [[TMP0]], ptr addrspace(5) [[TMP1]], align 8
@@ -467,10 +473,10 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    store i8 [[B]], ptr [[B_ADDR_ASCAST]], align 1, !tbaa [[TBAA16:![0-9]+]]
 // GFX900-NEXT:    store ptr addrspace(1) [[C]], ptr [[C_ADDR_ASCAST]], align 8, !tbaa [[TBAA7]]
 // GFX900-NEXT:    store i64 [[D]], ptr [[D_ADDR_ASCAST]], align 8, !tbaa [[TBAA3]]
-// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR8:[0-9]+]]
-// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[FLAGS]]) #[[ATTR8]]
+// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR7:[0-9]+]]
+// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[FLAGS]]) #[[ATTR7]]
 // GFX900-NEXT:    store i32 0, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA17:![0-9]+]]
-// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[NDRANGE]]) #[[ATTR8]]
+// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[NDRANGE]]) #[[ATTR7]]
 // GFX900-NEXT:    [[TMP0:%.*]] = load ptr addrspace(1), ptr [[DEFAULT_QUEUE_ASCAST]], align 8, !tbaa [[TBAA19:![0-9]+]]
 // GFX900-NEXT:    [[TMP1:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA17]]
 // GFX900-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT21:![0-9]+]]
@@ -486,7 +492,7 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[BLOCK_CAPTURED1:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), i8 }>, ptr [[BLOCK_ASCAST]], i32 0, i32 4
 // GFX900-NEXT:    [[TMP3:%.*]] = load i8, ptr [[B_ADDR_ASCAST]], align 1, !tbaa [[TBAA16]]
 // GFX900-NEXT:    store i8 [[TMP3]], ptr [[BLOCK_CAPTURED1]], align 8, !tbaa [[TBAA16]]
-// GFX900-NEXT:    [[TMP4:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP0]], i32 [[TMP1]], ptr addrspace(5) [[TMP]], ptr @__test_block_invoke_kernel, ptr [[BLOCK_ASCAST]])
+// GFX900-NEXT:    [[TMP4:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP0]], i32 [[TMP1]], ptr addrspace(5) [[TMP]], ptr addrspacecast (ptr addrspace(1) @__test_block_invoke_kernel.runtime.handle to ptr), ptr [[BLOCK_ASCAST]])
 // GFX900-NEXT:    [[TMP5:%.*]] = load ptr addrspace(1), ptr [[DEFAULT_QUEUE_ASCAST]], align 8, !tbaa [[TBAA19]]
 // GFX900-NEXT:    [[TMP6:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA17]]
 // GFX900-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP2_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT21]]
@@ -508,7 +514,7 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[BLOCK_CAPTURED10:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[BLOCK3_ASCAST]], i32 0, i32 5
 // GFX900-NEXT:    [[TMP10:%.*]] = load i64, ptr [[D_ADDR_ASCAST]], align 8, !tbaa [[TBAA3]]
 // GFX900-NEXT:    store i64 [[TMP10]], ptr [[BLOCK_CAPTURED10]], align 8, !tbaa [[TBAA3]]
-// GFX900-NEXT:    [[TMP11:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP5]], i32 [[TMP6]], ptr addrspace(5) [[VARTMP2]], ptr @__test_block_invoke_2_kernel, ptr [[BLOCK3_ASCAST]])
+// GFX900-NEXT:    [[TMP11:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP5]], i32 [[TMP6]], ptr addrspace(5) [[VARTMP2]], ptr addrspacecast (ptr addrspace(1) @__test_block_invoke_2_kernel.runtime.handle to ptr), ptr [[BLOCK3_ASCAST]])
 // GFX900-NEXT:    [[TMP12:%.*]] = load ptr addrspace(1), ptr [[DEFAULT_QUEUE_ASCAST]], align 8, !tbaa [[TBAA19]]
 // GFX900-NEXT:    [[TMP13:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA17]]
 // GFX900-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP11_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT21]]
@@ -530,12 +536,12 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[BLOCK_CAPTURED19:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[BLOCK12_ASCAST]], i32 0, i32 5
 // GFX900-NEXT:    [[TMP17:%.*]] = load i64, ptr [[D_ADDR_ASCAST]], align 8, !tbaa [[TBAA3]]
 // GFX900-NEXT:    store i64 [[TMP17]], ptr [[BLOCK_CAPTURED19]], align 8, !tbaa [[TBAA3]]
-// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[BLOCK_SIZES]]) #[[ATTR8]]
+// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[BLOCK_SIZES]]) #[[ATTR7]]
 // GFX900-NEXT:    [[TMP18:%.*]] = getelementptr [1 x i64], ptr [[BLOCK_SIZES_ASCAST]], i32 0, i32 0
 // GFX900-NEXT:    store i64 100, ptr [[TMP18]], align 8
-// GFX900-NEXT:    [[TMP19:%.*]] = call i32 @__enqueue_kernel_varargs(ptr addrspace(1) [[TMP12]], i32 [[TMP13]], ptr addrspace(5) [[VARTMP11]], ptr @__test_block_invoke_3_kernel, ptr [[BLOCK12_ASCAST]], i32 1, ptr [[TMP18]])
-// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[BLOCK_SIZES]]) #[[ATTR8]]
-// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[BLOCK20]]) #[[ATTR8]]
+// GFX900-NEXT:    [[TMP19:%.*]] = call i32 @__enqueue_kernel_varargs(ptr addrspace(1) [[TMP12]], i32 [[TMP13]], ptr addrspace(5) [[VARTMP11]], ptr addrspacecast (ptr addrspace(1) @__test_block_invoke_3_kernel.runtime.handle to ptr), ptr [[BLOCK12_ASCAST]], i32 1, ptr [[TMP18]])
+// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[BLOCK_SIZES]]) #[[ATTR7]]
+// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[BLOCK20]]) #[[ATTR7]]
 // GFX900-NEXT:    [[BLOCK_SIZE22:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, i64, ptr addrspace(1) }>, ptr [[BLOCK21_ASCAST]], i32 0, i32 0
 // GFX900-NEXT:    store i32 32, ptr [[BLOCK_SIZE22]], align 8
 // GFX900-NEXT:    [[BLOCK_ALIGN23:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, i64, ptr addrspace(1) }>, ptr [[BLOCK21_ASCAST]], i32 0, i32 1
@@ -553,11 +559,11 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[TMP23:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA17]]
 // GFX900-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP27_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT21]]
 // GFX900-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[BLOCK20_ASCAST]], align 8, !tbaa [[TBAA16]]
-// GFX900-NEXT:    [[TMP25:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP22]], i32 [[TMP23]], ptr addrspace(5) [[VARTMP27]], ptr @__test_block_invoke_4_kernel, ptr [[BLOCK21_ASCAST]])
-// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[BLOCK20]]) #[[ATTR8]]
-// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[NDRANGE]]) #[[ATTR8]]
-// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[FLAGS]]) #[[ATTR8]]
-// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR8]]
+// GFX900-NEXT:    [[TMP25:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP22]], i32 [[TMP23]], ptr addrspace(5) [[VARTMP27]], ptr addrspacecast (ptr addrspace(1) @__test_block_invoke_4_kernel.runtime.handle to ptr), ptr [[BLOCK21_ASCAST]])
+// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[BLOCK20]]) #[[ATTR7]]
+// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[NDRANGE]]) #[[ATTR7]]
+// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[FLAGS]]) #[[ATTR7]]
+// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR7]]
 // GFX900-NEXT:    ret void
 //
 //
@@ -579,7 +585,7 @@ kernel void test_target_features_kernel(global int *i) {
 //
 // GFX900: Function Attrs: convergent nounwind
 // GFX900-LABEL: define {{[^@]+}}@__test_block_invoke_kernel
-// GFX900-SAME: (<{ i32, i32, ptr, ptr addrspace(1), i8 }> [[TMP0:%.*]]) #[[ATTR6:[0-9]+]] !kernel_arg_addr_space [[META22:![0-9]+]] !kernel_arg_access_qual [[META23:![0-9]+]] !kernel_arg_type [[META24:![0-9]+]] !kernel_arg_base_type [[META24]] !kernel_arg_type_qual [[META25:![0-9]+]] {
+// GFX900-SAME: (<{ i32, i32, ptr, ptr addrspace(1), i8 }> [[TMP0:%.*]]) #[[ATTR5]] !associated [[META22:![0-9]+]] !kernel_arg_addr_space [[META23:![0-9]+]] !kernel_arg_access_qual [[META24:![0-9]+]] !kernel_arg_type [[META25:![0-9]+]] !kernel_arg_base_type [[META25]] !kernel_arg_type_qual [[META26:![0-9]+]] {
 // GFX900-NEXT:  entry:
 // GFX900-NEXT:    [[TMP1:%.*]] = alloca <{ i32, i32, ptr, ptr addrspace(1), i8 }>, align 8, addrspace(5)
 // GFX900-NEXT:    store <{ i32, i32, ptr, ptr addrspace(1), i8 }> [[TMP0]], ptr addrspace(5) [[TMP1]], align 8
@@ -612,7 +618,7 @@ kernel void test_target_features_kernel(global int *i) {
 //
 // GFX900: Function Attrs: convergent nounwind
 // GFX900-LABEL: define {{[^@]+}}@__test_block_invoke_2_kernel
-// GFX900-SAME: (<{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0:%.*]]) #[[ATTR6]] !kernel_arg_addr_space [[META22]] !kernel_arg_access_qual [[META23]] !kernel_arg_type [[META24]] !kernel_arg_base_type [[META24]] !kernel_arg_type_qual [[META25]] {
+// GFX900-SAME: (<{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0:%.*]]) #[[ATTR5]] !associated [[META27:![0-9]+]] !kernel_arg_addr_space [[META23]] !kernel_arg_access_qual [[META24]] !kernel_arg_type [[META25]] !kernel_arg_base_type [[META25]] !kernel_arg_type_qual [[META26]] {
 // GFX900-NEXT:  entry:
 // GFX900-NEXT:    [[TMP1:%.*]] = alloca <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, align 8, addrspace(5)
 // GFX900-NEXT:    store <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0]], ptr addrspace(5) [[TMP1]], align 8
@@ -630,7 +636,7 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[DOTBLOCK_DESCRIPTOR_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DOTBLOCK_DESCRIPTOR_ADDR]] to ptr
 // GFX900-NEXT:    [[LP_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[LP_ADDR]] to ptr
 // GFX900-NEXT:    store ptr [[DOTBLOCK_DESCRIPTOR]], ptr [[DOTBLOCK_DESCRIPTOR_ADDR_ASCAST]], align 8
-// GFX900-NEXT:    store ptr addrspace(3) [[LP]], ptr [[LP_ADDR_ASCAST]], align 4, !tbaa [[TBAA26:![0-9]+]]
+// GFX900-NEXT:    store ptr addrspace(3) [[LP]], ptr [[LP_ADDR_ASCAST]], align 4, !tbaa [[TBAA28:![0-9]+]]
 // GFX900-NEXT:    [[BLOCK_CAPTURE_ADDR:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
 // GFX900-NEXT:    [[TMP0:%.*]] = load i8, ptr [[BLOCK_CAPTURE_ADDR]], align 8, !tbaa [[TBAA16]]
 // GFX900-NEXT:    [[BLOCK_CAPTURE_ADDR1:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 3
@@ -643,7 +649,7 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[TMP3:%.*]] = load ptr addrspace(1), ptr [[BLOCK_CAPTURE_ADDR3]], align 8, !tbaa [[TBAA7]]
 // GFX900-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[TMP3]], i64 0
 // GFX900-NEXT:    store i64 [[TMP2]], ptr addrspace(1) [[ARRAYIDX4]], align 8, !tbaa [[TBAA3]]
-// GFX900-NEXT:    [[TMP4:%.*]] = load ptr addrspace(3), ptr [[LP_ADDR_ASCAST]], align 4, !tbaa [[TBAA26]]
+// GFX900-NEXT:    [[TMP4:%.*]] = load ptr addrspace(3), ptr [[LP_ADDR_ASCAST]], align 4, !tbaa [[TBAA28]]
 // GFX900-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[TMP4]], i64 0
 // GFX900-NEXT:    store i32 1, ptr addrspace(3) [[ARRAYIDX5]], align 4, !tbaa [[TBAA17]]
 // GFX900-NEXT:    ret void
@@ -651,7 +657,7 @@ kernel void test_target_features_kernel(global int *i) {
 //
 // GFX900: Function Attrs: convergent nounwind
 // GFX900-LABEL: define {{[^@]+}}@__test_block_invoke_3_kernel
-// GFX900-SAME: (<{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) #[[ATTR6]] !kernel_arg_addr_space [[META27:![0-9]+]] !kernel_arg_access_qual [[META28:![0-9]+]] !kernel_arg_type [[META29:![0-9]+]] !kernel_arg_base_type [[META29]] !kernel_arg_type_qual [[META30:![0-9]+]] {
+// GFX900-SAME: (<{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) #[[ATTR5]] !associated [[META29:![0-9]+]] !kernel_arg_addr_space [[META30:![0-9]+]] !kernel_arg_access_qual [[META31:![0-9]+]] !kernel_arg_type [[META32:![0-9]+]] !kernel_arg_base_type [[META32]] !kernel_arg_type_qual [[META33:![0-9]+]] {
 // GFX900-NEXT:  entry:
 // GFX900-NEXT:    [[TMP2:%.*]] = alloca <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, align 8, addrspace(5)
 // GFX900-NEXT:    store <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[TMP0]], ptr addrspace(5) [[TMP2]], align 8
@@ -671,13 +677,13 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[TMP0:%.*]] = load i64, ptr [[BLOCK_CAPTURE_ADDR]], align 8, !tbaa [[TBAA3]]
 // GFX900-NEXT:    [[BLOCK_CAPTURE_ADDR1:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, i64, ptr addrspace(1) }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 4
 // GFX900-NEXT:    [[TMP1:%.*]] = load ptr addrspace(1), ptr [[BLOCK_CAPTURE_ADDR1]], align 8, !tbaa [[TBAA7]]
-// GFX900-NEXT:    call void @callee(i64 noundef [[TMP0]], ptr addrspace(1) noundef [[TMP1]]) #[[ATTR9:[0-9]+]]
+// GFX900-NEXT:    call void @callee(i64 noundef [[TMP0]], ptr addrspace(1) noundef [[TMP1]]) #[[ATTR8:[0-9]+]]
 // GFX900-NEXT:    ret void
 //
 //
 // GFX900: Function Attrs: convergent nounwind
 // GFX900-LABEL: define {{[^@]+}}@__test_block_invoke_4_kernel
-// GFX900-SAME: (<{ i32, i32, ptr, i64, ptr addrspace(1) }> [[TMP0:%.*]]) #[[ATTR6]] !kernel_arg_addr_space [[META22]] !kernel_arg_access_qual [[META23]] !kernel_arg_type [[META24]] !kernel_arg_base_type [[META24]] !kernel_arg_type_qual [[META25]] {
+// GFX900-SAME: (<{ i32, i32, ptr, i64, ptr addrspace(1) }> [[TMP0:%.*]]) #[[ATTR5]] !associated [[META34:![0-9]+]] !kernel_arg_addr_space [[META23]] !kernel_arg_access_qual [[META24]] !kernel_arg_type [[META25]] !kernel_arg_base_type [[META25]] !kernel_arg_type_qual [[META26]] {
 // GFX900-NEXT:  entry:
 // GFX900-NEXT:    [[TMP1:%.*]] = alloca <{ i32, i32, ptr, i64, ptr addrspace(1) }>, align 8, addrspace(5)
 // GFX900-NEXT:    store <{ i32, i32, ptr, i64, ptr addrspace(1) }> [[TMP0]], ptr addrspace(5) [[TMP1]], align 8
@@ -688,7 +694,7 @@ kernel void test_target_features_kernel(global int *i) {
 //
 // GFX900: Function Attrs: convergent norecurse nounwind
 // GFX900-LABEL: define {{[^@]+}}@test_target_features_kernel
-// GFX900-SAME: (ptr addrspace(1) noundef align 4 [[I:%.*]]) #[[ATTR2]] !kernel_arg_addr_space [[META31:![0-9]+]] !kernel_arg_access_qual [[META23]] !kernel_arg_type [[META32:![0-9]+]] !kernel_arg_base_type [[META32]] !kernel_arg_type_qual [[META25]] {
+// GFX900-SAME: (ptr addrspace(1) noundef align 4 [[I:%.*]]) #[[ATTR2]] !kernel_arg_addr_space [[META35:![0-9]+]] !kernel_arg_access_qual [[META24]] !kernel_arg_type [[META36:![0-9]+]] !kernel_arg_base_type [[META36]] !kernel_arg_type_qual [[META26]] {
 // GFX900-NEXT:  entry:
 // GFX900-NEXT:    [[I_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
 // GFX900-NEXT:    [[DEFAULT_QUEUE:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
@@ -700,19 +706,19 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[FLAGS_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[FLAGS]] to ptr
 // GFX900-NEXT:    [[NDRANGE_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[NDRANGE]] to ptr
 // GFX900-NEXT:    [[TMP_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[TMP]] to ptr
-// GFX900-NEXT:    store ptr addrspace(1) [[I]], ptr [[I_ADDR_ASCAST]], align 8, !tbaa [[TBAA33:![0-9]+]]
-// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR8]]
-// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[FLAGS]]) #[[ATTR8]]
+// GFX900-NEXT:    store ptr addrspace(1) [[I]], ptr [[I_ADDR_ASCAST]], align 8, !tbaa [[TBAA37:![0-9]+]]
+// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR7]]
+// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[FLAGS]]) #[[ATTR7]]
 // GFX900-NEXT:    store i32 0, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA17]]
-// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[NDRANGE]]) #[[ATTR8]]
+// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[NDRANGE]]) #[[ATTR7]]
 // GFX900-NEXT:    [[TMP0:%.*]] = call i64 @llvm.amdgcn.s.memtime()
 // GFX900-NEXT:    [[TMP1:%.*]] = load ptr addrspace(1), ptr [[DEFAULT_QUEUE_ASCAST]], align 8, !tbaa [[TBAA19]]
 // GFX900-NEXT:    [[TMP2:%.*]] = load i32, ptr [[FLAGS_ASCAST]], align 4, !tbaa [[TBAA17]]
 // GFX900-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP_ASCAST]], ptr align 4 [[NDRANGE_ASCAST]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT21]]
-// GFX900-NEXT:    [[TMP3:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP1]], i32 [[TMP2]], ptr addrspace(5) [[TMP]], ptr @__test_target_features_kernel_block_invoke_kernel, ptr addrspacecast (ptr addrspace(1) @__block_literal_global to ptr))
-// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[NDRANGE]]) #[[ATTR8]]
-// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[FLAGS]]) #[[ATTR8]]
-// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR8]]
+// GFX900-NEXT:    [[TMP3:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP1]], i32 [[TMP2]], ptr addrspace(5) [[TMP]], ptr addrspacecast (ptr addrspace(1) @__test_target_features_kernel_block_invoke_kernel.runtime.handle to ptr), ptr addrspacecast (ptr addrspace(1) @__block_literal_global to ptr))
+// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[NDRANGE]]) #[[ATTR7]]
+// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[FLAGS]]) #[[ATTR7]]
+// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR7]]
 // GFX900-NEXT:    ret void
 //
 //
@@ -729,7 +735,7 @@ kernel void test_target_features_kernel(global int *i) {
 //
 // GFX900: Function Attrs: convergent nounwind
 // GFX900-LABEL: define {{[^@]+}}@__test_target_features_kernel_block_invoke_kernel
-// GFX900-SAME: ({ i32, i32, ptr } [[TMP0:%.*]]) #[[ATTR6]] !kernel_arg_addr_space [[META22]] !kernel_arg_access_qual [[META23]] !kernel_arg_type [[META24]] !kernel_arg_base_type [[META24]] !kernel_arg_type_qual [[META25]] {
+// GFX900-SAME: ({ i32, i32, ptr } [[TMP0:%.*]]) #[[ATTR5]] !associated [[META39:![0-9]+]] !kernel_arg_addr_space [[META23]] !kernel_arg_access_qual [[META24]] !kernel_arg_type [[META25]] !kernel_arg_base_type [[META25]] !kernel_arg_type_qual [[META26]] {
 // GFX900-NEXT:  entry:
 // GFX900-NEXT:    [[TMP1:%.*]] = alloca { i32, i32, ptr }, align 8, addrspace(5)
 // GFX900-NEXT:    store { i32, i32, ptr } [[TMP0]], ptr addrspace(5) [[TMP1]], align 8
@@ -743,7 +749,7 @@ kernel void test_target_features_kernel(global int *i) {
 // NOCPU: attributes #[[ATTR2]] = { convergent noinline norecurse nounwind optnone "amdgpu-flat-work-group-size"="1,256" "denormal-fp-math-f32"="preserve-sign,preserve-sign" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "uniform-work-group-size"="false" }
 // NOCPU: attributes #[[ATTR3:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
 // NOCPU: attributes #[[ATTR4]] = { convergent noinline nounwind optnone "denormal-fp-math-f32"="preserve-sign,preserve-sign" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
-// NOCPU: attributes #[[ATTR5]] = { convergent nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" "enqueued-block" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+// NOCPU: attributes #[[ATTR5]] = { convergent nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
 // NOCPU: attributes #[[ATTR6]] = { convergent noinline norecurse nounwind optnone "amdgpu-flat-work-group-size"="1,256" "denormal-fp-math-f32"="preserve-sign,preserve-sign" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+s-memtime-inst" "uniform-work-group-size"="false" }
 // NOCPU: attributes #[[ATTR7:[0-9]+]] = { nocallback nofree nosync nounwind willreturn }
 // NOCPU: attributes #[[ATTR8]] = { convergent nounwind }
@@ -754,10 +760,9 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900: attributes #[[ATTR3:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
 // GFX900: attributes #[[ATTR4:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
 // GFX900: attributes #[[ATTR5]] = { convergent nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx900" "target-features"="+16-bit-insts,+ci-insts,+dpp,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst,+wavefrontsize64,-sram-ecc" }
-// GFX900: attributes #[[ATTR6]] = { convergent nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" "enqueued-block" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx900" "target-features"="+16-bit-insts,+ci-insts,+dpp,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst,+wavefrontsize64,-sram-ecc" }
-// GFX900: attributes #[[ATTR7:[0-9]+]] = { nocallback nofree nosync nounwind willreturn }
-// GFX900: attributes #[[ATTR8]] = { nounwind }
-// GFX900: attributes #[[ATTR9]] = { convergent nounwind }
+// GFX900: attributes #[[ATTR6:[0-9]+]] = { nocallback nofree nosync nounwind willreturn }
+// GFX900: attributes #[[ATTR7]] = { nounwind }
+// GFX900: attributes #[[ATTR8]] = { convergent nounwind }
 //.
 // NOCPU: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500}
 // NOCPU: [[META1:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
@@ -766,16 +771,21 @@ kernel void test_target_features_kernel(global int *i) {
 // NOCPU: [[META4]] = !{!"none", !"none", !"none", !"none"}
 // NOCPU: [[META5]] = !{!"char*", !"char", !"long*", !"long"}
 // NOCPU: [[META6]] = !{!"", !"", !"", !""}
-// NOCPU: [[META7]] = !{i32 0}
-// NOCPU: [[META8]] = !{!"none"}
-// NOCPU: [[META9]] = !{!"__block_literal"}
-// NOCPU: [[META10]] = !{!""}
-// NOCPU: [[META11]] = !{i32 0, i32 3}
-// NOCPU: [[META12]] = !{!"none", !"none"}
-// NOCPU: [[META13]] = !{!"__block_literal", !"void*"}
-// NOCPU: [[META14]] = !{!"", !""}
-// NOCPU: [[META15]] = !{i32 1}
-// NOCPU: [[META16]] = !{!"int*"}
+// NOCPU: [[META7]] = !{ptr addrspace(1) @__test_block_invoke_kernel.runtime.handle}
+// NOCPU: [[META8]] = !{i32 0}
+// NOCPU: [[META9]] = !{!"none"}
+// NOCPU: [[META10]] = !{!"__block_literal"}
+// NOCPU: [[META11]] = !{!""}
+// NOCPU: [[META12]] = !{ptr addrspace(1) @__test_block_invoke_2_kernel.runtime.handle}
+// NOCPU: [[META13]] = !{ptr addrspace(1) @__test_block_invoke_3_kernel.runtime.handle}
+// NOCPU: [[META14]] = !{i32 0, i32 3}
+// NOCPU: [[META15]] = !{!"none", !"none"}
+// NOCPU: [[META16]] = !{!"__block_literal", !"void*"}
+// NOCPU: [[META17]] = !{!"", !""}
+// NOCPU: [[META18]] = !{ptr addrspace(1) @__test_block_invoke_4_kernel.runtime.handle}
+// NOCPU: [[META19]] = !{i32 1}
+// NOCPU: [[META20]] = !{!"int*"}
+// NOCPU: [[META21]] = !{ptr addrspace(1) @__test_target_features_kernel_block_invoke_kernel.runtime.handle}
 //.
 // GFX900: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500}
 // GFX900: [[META1:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
@@ -799,19 +809,24 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900: [[TBAA19]] = !{[[META20:![0-9]+]], [[META20]], i64 0}
 // GFX900: [[META20]] = !{!"queue_t", [[META5]], i64 0}
 // GFX900: [[TBAA_STRUCT21]] = !{i64 0, i64 4, [[TBAA17]]}
-// GFX900: [[META22]] = !{i32 0}
-// GFX900: [[META23]] = !{!"none"}
-// GFX900: [[META24]] = !{!"__block_literal"}
-// GFX900: [[META25]] = !{!""}
-// GFX900: [[TBAA26]] = !{[[META9]], [[META9]], i64 0}
-// GFX900: [[META27]] = !{i32 0, i32 3}
-// GFX900: [[META28]] = !{!"none", !"none"}
-// GFX900: [[META29]] = !{!"__block_literal", !"void*"}
-// GFX900: [[META30]] = !{!"", !""}
-// GFX900: [[META31]] = !{i32 1}
-// GFX900: [[META32]] = !{!"int*"}
-// GFX900: [[TBAA33]] = !{[[META34:![0-9]+]], [[META34]], i64 0}
-// GFX900: [[META34]] = !{!"p1 int", [[META9]], i64 0}
+// GFX900: [[META22]] = !{ptr addrspace(1) @__test_block_invoke_kernel.runtime.handle}
+// GFX900: [[META23]] = !{i32 0}
+// GFX900: [[META24]] = !{!"none"}
+// GFX900: [[META25]] = !{!"__block_literal"}
+// GFX900: [[META26]] = !{!""}
+// GFX900: [[META27]] = !{ptr addrspace(1) @__test_block_invoke_2_kernel.runtime.handle}
+// GFX900: [[TBAA28]] = !{[[META9]], [[META9]], i64 0}
+// GFX900: [[META29]] = !{ptr addrspace(1) @__test_block_invoke_3_kernel.runtime.handle}
+// GFX900: [[META30]] = !{i32 0, i32 3}
+// GFX900: [[META31]] = !{!"none", !"none"}
+// GFX900: [[META32]] = !{!"__block_literal", !"void*"}
+// GFX900: [[META33]] = !{!"", !""}
+// GFX900: [[META34]] = !{ptr addrspace(1) @__test_block_invoke_4_kernel.runtime.handle}
+// GFX900: [[META35]] = !{i32 1}
+// GFX900: [[META36]] = !{!"int*"}
+// GFX900: [[TBAA37]] = !{[[META38:![0-9]+]], [[META38]], i64 0}
+// GFX900: [[META38]] = !{!"p1 int", [[META9]], i64 0}
+// GFX900: [[META39]] = !{ptr addrspace(1) @__test_target_features_kernel_block_invoke_kernel.runtime.handle}
 //.
 //// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 // CHECK: {{.*}}
diff --git a/clang/test/Driver/HLSL/metal-converter.hlsl b/clang/test/Driver/HLSL/metal-converter.hlsl
new file mode 100644
index 0000000000000..536f24be6e73b
--- /dev/null
+++ b/clang/test/Driver/HLSL/metal-converter.hlsl
@@ -0,0 +1,19 @@
+// RUN: %clang_dxc -T cs_6_0 %s -metal -Fo %t.mtl -### 2>&1 | FileCheck %s
+// RUN: %clang_dxc -T cs_6_0 %s -metal -Vd -Fo %t.mtl -### 2>&1 | FileCheck %s
+// CHECK: "{{.*}}metal-shaderconverter{{(.exe)?}}" "{{.*}}.obj" "-o" "{{.*}}.mtl"
+
+// RUN: %clang_dxc -T cs_6_0 %s -metal -### 2>&1 | FileCheck --check-prefix=NO_MTL %s
+// NO_MTL-NOT: metal-shaderconverter
+
+// RUN: echo "dxv" > %T/dxv && chmod 754 %T/dxv
+// RUN: %clang_dxc -T cs_6_0 %s --dxv-path=%T -metal -Fo %t.mtl -### 2>&1 | FileCheck --check-prefix=DXV %s
+// DXV: "{{.*}}dxv{{(.exe)?}}" "{{.*}}.obj" "-o" "{{.*}}.dxo"
+// DXV: "{{.*}}metal-shaderconverter{{(.exe)?}}" "{{.*}}.dxo" "-o" "{{.*}}.mtl"
+
+RWBuffer<float4> In : register(u0, space0);
+RWBuffer<float4> Out : register(u1, space4);
+
+[numthreads(1,1,1)]
+void main(uint GI : SV_GroupIndex) {
+  Out[GI] = In[GI] * In[GI];
+}
diff --git a/clang/test/Driver/aarch64-execute-only.c b/clang/test/Driver/aarch64-execute-only.c
new file mode 100644
index 0000000000000..04370064a61f3
--- /dev/null
+++ b/clang/test/Driver/aarch64-execute-only.c
@@ -0,0 +1,9 @@
+// RUN: %clang --sysroot=%S/Inputs -c -fdriver-only -Werror --target=aarch64-unknown-linux-gnu \
+// RUN:     -mexecute-only %s 2>&1 | count 0
+
+// RUN: %clang -### --target=aarch64-unknown-linux-gnu -x assembler -mexecute-only %s -c -### 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=CHECK-NO-EXECUTE-ONLY-ASM
+// RUN: %clang -### --multi-lib-config=%S/Inputs/multilib/empty.yaml --sysroot= \
+// RUN:     --target=aarch64-none-elf -x assembler -mexecute-only %s -c -### 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=CHECK-NO-EXECUTE-ONLY-ASM
+// CHECK-NO-EXECUTE-ONLY-ASM: warning: argument unused during compilation: '-mexecute-only'
diff --git a/clang/test/Driver/arm-thread-pointer.c b/clang/test/Driver/arm-thread-pointer.c
index 985c5046f6d26..37d1a3e4d7e89 100644
--- a/clang/test/Driver/arm-thread-pointer.c
+++ b/clang/test/Driver/arm-thread-pointer.c
@@ -14,18 +14,26 @@
 // RUN: FileCheck -check-prefix=ARMv7_THREAD_POINTER-TPIDRPRW %s
 // ARMv7_THREAD_POINTER-TPIDRPRW: "-target-feature" "+read-tp-tpidrprw"
 
-// RUN: %clang --target=armv6t2-linux -mtp=cp15 -### -S %s 2>&1 | \
-// RUN: FileCheck -check-prefix=ARM_THREAD_POINTER-HARD %s
-// RUN: %clang --target=thumbv6t2-linux -mtp=cp15 -### -S %s 2>&1 | \
-// RUN: FileCheck -check-prefix=ARM_THREAD_POINTER-HARD %s
 // RUN: %clang --target=armv6k-linux -mtp=cp15 -### -S %s 2>&1 | \
 // RUN: FileCheck -check-prefix=ARM_THREAD_POINTER-HARD %s
-// RUN: %clang --target=armv6-linux -mtp=cp15 -### -S %s 2>&1 | \
-// RUN: FileCheck -check-prefix=ARM_THREAD_POINTER-HARD %s
-// RUN: %clang --target=armv5t-linux -mtp=cp15 -### -S %s 2>&1 | \
-// RUN: FileCheck -check-prefix=ARM_THREAD_POINTER-HARD %s
 // ARM_THREAD_POINTER-HARD: "-target-feature" "+read-tp-tpidruro"
 
+// RUN: %clang --target=armv6k-linux -mtp=auto -### -S %s 2>&1 | \
+// RUN: FileCheck -check-prefix=ARM_THREAD_POINTER_AUTO %s
+// ARM_THREAD_POINTER_AUTO-NOT: "-target-feature" "+read-tp-tpidruro"
+
+// RUN: %clang --target=thumbv6k-apple-darwin -### -S %s 2>&1 | \
+// RUN: FileCheck -check-prefix=THUMBv6_THREAD_POINTER_NO_AUTO %s
+// THUMBv6_THREAD_POINTER_NO_AUTO-NOT: "-target-feature" "+read-tp-tpidruro"
+
+// RUN: not %clang --target=thumbv6k-apple-darwin -mtp=cp15 -### -S %s 2>&1 | \
+// RUN: FileCheck -check-prefix=THUMBv6_THREAD_POINTER_NO_HARD %s
+// THUMBv6_THREAD_POINTER_NO_HARD: unsupported option '-mtp=' for target 'thumbv6k-apple-darwin'
+
+// RUN: not %clang --target=thumbv6t2-linux -mtp=cp15 -### -S %s 2>&1 | \
+// RUN: FileCheck -check-prefix=ARM_THREAD_POINTER_NO_HARD %s
+// ARM_THREAD_POINTER_NO_HARD: hardware TLS register is not supported for the armv6t2 sub-architecture
+
 // RUN: %clang --target=armv5t-linux -mtp=cp15 -x assembler -### %s 2>&1 | \
 // RUN: FileCheck -check-prefix=ARMv5_THREAD_POINTER_ASSEMBLER %s
 // ARMv5_THREAD_POINTER_ASSEMBLER-NOT: hardware TLS register is not supported for the armv5 sub-architecture
@@ -47,3 +55,39 @@
 // RUN: %clang --target=armv7-linux -mtp=auto -### -S %s 2>&1 | \
 // RUN: FileCheck -check-prefix=ARMv7_THREAD_POINTER_Auto %s
 // ARMv7_THREAD_POINTER_Auto: "-target-feature" "+read-tp-tpidruro"
+
+// RUN: %clang --target=armv7-linux -mtp=cp15 -### -S %s 2>&1 | \
+// RUN: FileCheck -check-prefix=ARMv7_THREAD_POINTER_HARD %s
+// ARMv7_THREAD_POINTER_HARD: "-target-feature" "+read-tp-tpidruro"
+
+// RUN: %clang --target=armv7m-linux -mtp=auto -### -S %s 2>&1 | \
+// RUN: FileCheck -check-prefix=ARMv7m_THREAD_POINTER_Auto %s
+// ARMv7m_THREAD_POINTER_Auto-NOT: "-target-feature" "+read-tp-tpidruro"
+
+// RUN: not %clang --target=armv7m-linux -mtp=cp15 -### -S %s 2>&1 | \
+// RUN: FileCheck -check-prefix=ARMv7m_THREAD_POINTER_HARD %s
+// ARMv7m_THREAD_POINTER_HARD: hardware TLS register is not supported for the thumbv7m sub-architecture
+
+// RUN: %clang --target=armv5t-linux -mtp=auto -### -S %s 2>&1 | \
+// RUN: FileCheck -check-prefix=ARMv5t_THREAD_POINTER_Auto %s
+// ARMv5t_THREAD_POINTER_Auto-NOT: "-target-feature" "+read-tp-tpidruro"
+
+// RUN: %clang --target=armv6k-linux -mtp=cp15 -### -S %s 2>&1 | \
+// RUN: FileCheck -check-prefix=ARMv6k_THREAD_POINTER_Auto %s
+// ARMv6k_THREAD_POINTER_Auto: "-target-feature" "+read-tp-tpidruro"
+
+// RUN: not %clang --target=armv6t2-linux -mtp=cp15 -### -S %s 2>&1 | \
+// RUN: FileCheck -check-prefix=ARMv6t2_THREAD_POINTER_HARD %s
+// ARMv6t2_THREAD_POINTER_HARD: hardware TLS register is not supported for the armv6t2 sub-architecture
+
+// RUN: %clang --target=armv6t2-linux -mtp=auto -### -S %s 2>&1 | \
+// RUN: FileCheck -check-prefix=ARMV6t2_THREAD_POINTER_AUTO %s
+// ARMV6t2_THREAD_POINTER_AUTO-NOT: "-target-feature" "+read-tp-tpidruro"
+
+// RUN: %clang --target=armv6kz-linux -mtp=cp15 -### -S %s 2>&1 | \
+// RUN: FileCheck -check-prefix=ARMv6kz_THREAD_POINTER_HARD %s
+// ARMv6kz_THREAD_POINTER_HARD: "-target-feature" "+read-tp-tpidruro"
+
+// RUN: %clang --target=armv6kz-linux -mtp=auto -### -S %s 2>&1 | \
+// RUN: FileCheck -check-prefix=ARMV6KZ_THREAD_POINTER_AUTO %s
+// ARMV6KZ_THREAD_POINTER_AUTO-NOT: "-target-feature" "+read-tp-tpidruro"
\ No newline at end of file
diff --git a/clang/test/Driver/dxc_dxv_path.hlsl b/clang/test/Driver/dxc_dxv_path.hlsl
index db2c87063ac31..55a07f34a648e 100644
--- a/clang/test/Driver/dxc_dxv_path.hlsl
+++ b/clang/test/Driver/dxc_dxv_path.hlsl
@@ -4,15 +4,15 @@
 // CHECK:dxv not found
 
 // RUN: echo "dxv" > %T/dxv && chmod 754 %T/dxv && %clang_dxc --dxv-path=%T %s -Tlib_6_3 -### 2>&1 | FileCheck %s --check-prefix=DXV_PATH
-// DXV_PATH:dxv{{(.exe)?}}" "-" "-o" "-"
+// DXV_PATH:dxv{{(.exe)?}}" "-" "-o" "{{.*}}.dxo"
 
 // RUN: %clang_dxc -I test -Vd -Tlib_6_3  -### %s 2>&1 | FileCheck %s --check-prefix=VD
 // VD:"-cc1"{{.*}}"-triple" "dxilv1.3-unknown-shadermodel6.3-library"
 // VD-NOT:dxv not found
 
 // RUN: %clang_dxc -Tlib_6_3 -ccc-print-bindings --dxv-path=%T -Fo %t.dxo  %s 2>&1 | FileCheck %s --check-prefix=BINDINGS
-// BINDINGS: "dxilv1.3-unknown-shadermodel6.3-library" - "clang", inputs: ["[[INPUT:.+]]"], output: "[[DXC:.+]].dxo"
-// BINDINGS-NEXT: "dxilv1.3-unknown-shadermodel6.3-library" - "hlsl::Validator", inputs: ["[[DXC]].dxo"]
+// BINDINGS: "dxilv1.3-unknown-shadermodel6.3-library" - "clang", inputs: ["[[INPUT:.+]]"], output: "[[obj:.+]].obj"
+// BINDINGS-NEXT: "dxilv1.3-unknown-shadermodel6.3-library" - "hlsl::Validator", inputs: ["[[obj]].obj"], output: "{{.+}}.dxo"
 
 // RUN: %clang_dxc -Tlib_6_3 -ccc-print-phases --dxv-path=%T -Fo %t.dxc  %s 2>&1 | FileCheck %s --check-prefix=PHASES
 
diff --git a/clang/test/Driver/fprofile-continuous.c b/clang/test/Driver/fprofile-continuous.c
index cc8e56c8f9fe0..b76f2cd2fe3b8 100644
--- a/clang/test/Driver/fprofile-continuous.c
+++ b/clang/test/Driver/fprofile-continuous.c
@@ -7,6 +7,7 @@
 // RUN: %clang --target=powerpc64-ibm-aix -fprofile-generate -fprofile-continuous -### -c %s 2>&1 | FileCheck %s --check-prefix=RELOC
 // RUN: %clang --target=x86_64-unknown-fuchsia -fprofile-generate -fprofile-continuous -### -c %s 2>&1 | FileCheck %s --check-prefix=RELOC
 // RUN: %clang --target=x86_64-windows-msvc -fprofile-generate -fprofile-continuous -### -c %s 2>&1 | FileCheck %s --check-prefix=RELOC
+// RUN: %clang --target=x86_64-windows-gnu -fprofile-generate -fprofile-continuous -### -c %s 2>&1 | FileCheck %s --check-prefix=RELOC
 // RELOC: "-cc1" {{.*}} "-fprofile-continuous" "-mllvm" "-runtime-counter-relocation"
 
 // 2) test -fprofile-continuous with cs-profile-generate and -fprofile-instr-generate
diff --git a/clang/test/Driver/fsanitize.c b/clang/test/Driver/fsanitize.c
index 79c936a79ffd2..8652f1d875e68 100644
--- a/clang/test/Driver/fsanitize.c
+++ b/clang/test/Driver/fsanitize.c
@@ -1053,13 +1053,18 @@
 // RUN: not %clang --target=x86_64-sie-ps5 -fsanitize=kcfi %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-KCFI-TARGET
 // RUN: not %clang --target=x86_64-sie-ps5 -fsanitize=function -fsanitize=kcfi %s -### 2>&1 | FileCheck %s  --check-prefix=CHECK-UBSAN-KCFI-TARGET --check-prefix=CHECK-UBSAN-FUNCTION-TARGET
 // RUN: %clang --target=x86_64-sie-ps5 -fsanitize=undefined %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-UNDEFINED
-// CHECK-UBSAN-UNDEFINED: "-fsanitize={{((alignment|array-bounds|bool|builtin|enum|float-cast-overflow|integer-divide-by-zero|nonnull-attribute|null|pointer-overflow|return|returns-nonnull-attribute|shift-base|shift-exponent|signed-integer-overflow|unreachable|vla-bound),?){17}"}}
 
 // RUN: not %clang --target=armv6t2-eabi -mexecute-only -fsanitize=function %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-FUNCTION-MEXECUTE-ONLY
 // RUN: not %clang --target=armv6t2-eabi -mpure-code -fsanitize=function %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-FUNCTION-MPURE-CODE
 // RUN: not %clang --target=armv6t2-eabi -mexecute-only -fsanitize=kcfi %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-KCFI-MEXECUTE-ONLY
 // RUN: not %clang --target=armv6t2-eabi -mpure-code -fsanitize=kcfi %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-KCFI-MPURE-CODE
-// RUN: %clang --target=armv6t2-eabi -mexecute-only -fsanitize=undefined %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-UNDEFINED-VPTR
+// RUN: %clang --target=armv6t2-eabi -mexecute-only -fsanitize=undefined %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-UNDEFINED
+
+// RUN: not %clang --target=aarch64 -mexecute-only -fsanitize=function %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-FUNCTION-MEXECUTE-ONLY
+// RUN: not %clang --target=aarch64 -mpure-code -fsanitize=function %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-FUNCTION-MPURE-CODE
+// RUN: not %clang --target=aarch64 -mexecute-only -fsanitize=kcfi %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-KCFI-MEXECUTE-ONLY
+// RUN: not %clang --target=aarch64 -mpure-code -fsanitize=kcfi %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-KCFI-MPURE-CODE
+// RUN: %clang --target=aarch64 -mexecute-only -fsanitize=undefined %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-UBSAN-UNDEFINED
 
 // CHECK-UBSAN-KCFI-TARGET-DAG: error: unsupported option '-fsanitize=kcfi' for target 'x86_64-sie-ps5'
 // CHECK-UBSAN-KCFI-MEXECUTE-ONLY-DAG: error: invalid argument '-fsanitize=kcfi' not allowed with '-mexecute-only'
@@ -1067,7 +1072,7 @@
 // CHECK-UBSAN-FUNCTION-TARGET-DAG: error: unsupported option '-fsanitize=function' for target 'x86_64-sie-ps5'
 // CHECK-UBSAN-FUNCTION-MEXECUTE-ONLY-DAG: error: invalid argument '-fsanitize=function' not allowed with '-mexecute-only'
 // CHECK-UBSAN-FUNCTION-MPURE-CODE-DAG: error: invalid argument '-fsanitize=function' not allowed with '-mpure-code'
-// CHECK-UBSAN-UNDEFINED-VPTR: "-fsanitize={{((alignment|array-bounds|bool|builtin|enum|float-cast-overflow|integer-divide-by-zero|nonnull-attribute|null|pointer-overflow|return|returns-nonnull-attribute|shift-base|shift-exponent|signed-integer-overflow|unreachable|vla-bound),?){17}"}}
+// CHECK-UBSAN-UNDEFINED: "-fsanitize={{((alignment|array-bounds|bool|builtin|enum|float-cast-overflow|integer-divide-by-zero|nonnull-attribute|null|pointer-overflow|return|returns-nonnull-attribute|shift-base|shift-exponent|signed-integer-overflow|unreachable|vla-bound),?){17}"}}
 
 // * Test BareMetal toolchain sanitizer support *
 
diff --git a/clang/test/Driver/hexagon-toolchain-elf.c b/clang/test/Driver/hexagon-toolchain-elf.c
index be812dda40d57..de2ebfeeda26c 100644
--- a/clang/test/Driver/hexagon-toolchain-elf.c
+++ b/clang/test/Driver/hexagon-toolchain-elf.c
@@ -555,6 +555,7 @@
 // RUN:   -ccc-install-dir %S/Inputs/hexagon_tree/Tools/bin \
 // RUN:   -mcpu=hexagonv60 \
 // RUN:   -fuse-ld=lld %s 2>&1 | FileCheck -check-prefix=CHECK382 %s
+// CHECK382:          "--eh-frame-hdr
 // CHECK382-NOT:      "-march=
 // CHECK382-NOT:      "-mcpu=
 // -----------------------------------------------------------------------------
diff --git a/clang/test/Driver/hexagon-toolchain-linux.c b/clang/test/Driver/hexagon-toolchain-linux.c
index 6f7f3b20f9141..e791353cca07f 100644
--- a/clang/test/Driver/hexagon-toolchain-linux.c
+++ b/clang/test/Driver/hexagon-toolchain-linux.c
@@ -127,6 +127,7 @@
 // RUN:    --target=hexagon-unknown-linux-musl %s -### 2>&1 \
 // RUN:    | FileCheck -check-prefix=CHECK011 %s
 // CHECK011:   InstalledDir: [[INSTALLED_DIR:.+]]
+// CHECK011:   "--eh-frame-hdr"
 // CHECK011:   crt1.o
 // CHECK011-NOT:  "-lunwind"
 // CHECK011-NOT:  "-lgcc_eh"
diff --git a/clang/test/Driver/hip-cuid-hash.hip b/clang/test/Driver/hip-cuid-hash.hip
index 103a1cbf26d50..a4167d664537e 100644
--- a/clang/test/Driver/hip-cuid-hash.hip
+++ b/clang/test/Driver/hip-cuid-hash.hip
@@ -1,13 +1,20 @@
 // Check CUID generated by hash.
 // The same CUID is generated for the same file with the same options.
 
+// This test requires relative paths for input files. Since the test may be
+// done out of source tree, create the local directory structure and copy the
+// input file from the source tree into that directory.
+// RUN: mkdir -p %t/Inputs/hip_multiple_inputs
+// RUN: cp %S/Inputs/hip_multiple_inputs/a.cu %t/Inputs/hip_multiple_inputs/a.cu
+// RUN: cd %t
+
 // RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nogpuinc -nogpulib -fuse-cuid=hash \
-// RUN:   %S/Inputs/hip_multiple_inputs/a.cu >%t.out 2>&1
+// RUN:   Inputs/hip_multiple_inputs/a.cu >%t.out 2>&1
 
 // RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nogpuinc -nogpulib -fuse-cuid=hash \
-// RUN:   %S/Inputs/hip_multiple_inputs/a.cu >>%t.out 2>&1
+// RUN:   Inputs/hip_multiple_inputs/a.cu >>%t.out 2>&1
 
 // RUN: FileCheck %s -check-prefixes=SAME -input-file %t.out
 
@@ -16,11 +23,11 @@
 
 // RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu -DX=1 --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nogpuinc -nogpulib -fuse-cuid=hash \
-// RUN:   %S/Inputs/hip_multiple_inputs/a.cu >%t.out 2>&1
+// RUN:   Inputs/hip_multiple_inputs/a.cu >%t.out 2>&1
 
 // RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu -DX=2 --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nogpuinc -nogpulib -fuse-cuid=hash \
-// RUN:   %S/Inputs/../Inputs/hip_multiple_inputs/a.cu >>%t.out 2>&1
+// RUN:   Inputs/../Inputs/hip_multiple_inputs/a.cu >>%t.out 2>&1
 
 // RUN: FileCheck %s -check-prefixes=DIFF -input-file %t.out
 
diff --git a/clang/test/Driver/hip-partial-link.hip b/clang/test/Driver/hip-partial-link.hip
index 8b27f78f3bdd1..d1a57bbe1d75c 100644
--- a/clang/test/Driver/hip-partial-link.hip
+++ b/clang/test/Driver/hip-partial-link.hip
@@ -1,4 +1,4 @@
-// REQUIRES: x86-registered-target, amdgpu-registered-target, lld, system-linux
+// REQUIRES: x86-registered-target, amdgpu-registered-target, lld
 
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nostdinc -nogpuinc -nohipwrapperinc \
@@ -32,11 +32,11 @@
 // LD-R: Found undefined HIP fatbin symbol: __hip_fatbin_[[ID2:[0-9a-f]+]]
 // LD-R: Found undefined HIP gpubin handle symbol: __hip_gpubin_handle_[[ID1]]
 // LD-R: Found undefined HIP gpubin handle symbol: __hip_gpubin_handle_[[ID2]]
-// LD-R: "{{.*}}/clang-offload-bundler" {{.*}}-unbundle
-// LD-R: "{{.*}}/lld" -flavor gnu -m elf64_amdgpu
-// LD-R: "{{.*}}/clang-offload-bundler"
-// LD-R: "{{.*}}/clang{{.*}}" -target x86_64-unknown-linux-gnu
-// LD-R: "{{.*}}/ld.lld" {{.*}} -r
+// LD-R: "{{.*[/\\]}}clang-offload-bundler" {{.*}}-unbundle
+// LD-R: "{{.*[/\\]}}lld" -flavor gnu -m elf64_amdgpu
+// LD-R: "{{.*[/\\]}}clang-offload-bundler"
+// LD-R: "{{.*[/\\]}}clang{{.*}}" -target x86_64-unknown-linux-gnu
+// LD-R: "{{.*[/\\]}}ld.lld" {{.*}} -r
 
 // RUN: llvm-nm  %t.lib.o | FileCheck -check-prefix=OBJ %s
 // OBJ:  B __hip_cuid_[[ID1:[0-9a-f]+]]
@@ -62,11 +62,11 @@
 // STATIC: Found undefined HIP fatbin symbol: __hip_fatbin_[[ID2:[0-9a-f]+]]
 // STATIC: Found undefined HIP gpubin handle symbol: __hip_gpubin_handle_[[ID1]]
 // STATIC: Found undefined HIP gpubin handle symbol: __hip_gpubin_handle_[[ID2]]
-// STATIC: "{{.*}}/clang-offload-bundler" {{.*}}-unbundle
-// STATIC: "{{.*}}/lld" -flavor gnu -m elf64_amdgpu
-// STATIC: "{{.*}}/clang-offload-bundler"
-// STATIC: "{{.*}}/clang{{.*}}" -target x86_64-unknown-linux-gnu
-// STATIC: "{{.*}}/llvm-ar"
+// STATIC: "{{.*[/\\]}}clang-offload-bundler" {{.*}}-unbundle
+// STATIC: "{{.*[/\\]}}lld" -flavor gnu -m elf64_amdgpu
+// STATIC: "{{.*[/\\]}}clang-offload-bundler"
+// STATIC: "{{.*[/\\]}}clang{{.*}}" -target x86_64-unknown-linux-gnu
+// STATIC: "{{.*[/\\]}}llvm-ar"
 
 // RUN: %clang -v --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN:   --hip-link -no-hip-rt -fgpu-rdc --offload-arch=gfx906 \
diff --git a/clang/test/Driver/linker-wrapper.c b/clang/test/Driver/linker-wrapper.c
index 7586b87743bf5..0c77c2b34216a 100644
--- a/clang/test/Driver/linker-wrapper.c
+++ b/clang/test/Driver/linker-wrapper.c
@@ -1,9 +1,8 @@
+// UNSUPPORTED: system-windows
 // REQUIRES: x86-registered-target
 // REQUIRES: nvptx-registered-target
 // REQUIRES: amdgpu-registered-target
 
-// REQUIRES: system-linux
-
 // An externally visible variable so static libraries extract.
 __attribute__((visibility("protected"), used)) int x;
 
@@ -39,7 +38,7 @@ __attribute__((visibility("protected"), used)) int x;
 // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \
 // RUN:   --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=AMDGPU-LINK
 
-// AMDGPU-LINK: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx908 -Wl,--no-undefined {{.*}}.o {{.*}}.o
+// AMDGPU-LINK: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx908 -flto -Wl,--no-undefined {{.*}}.o {{.*}}.o
 
 // RUN: clang-offload-packager -o %t.out \
 // RUN:   --image=file=%t.amdgpu.bc,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030 \
@@ -48,7 +47,7 @@ __attribute__((visibility("protected"), used)) int x;
 // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run --device-compiler=--save-temps \
 // RUN:   --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=AMDGPU-LTO-TEMPS
 
-// AMDGPU-LTO-TEMPS: clang{{.*}} --target=amdgcn-amd-amdhsa -mcpu=gfx1030 {{.*}}-save-temps
+// AMDGPU-LTO-TEMPS: clang{{.*}} --target=amdgcn-amd-amdhsa -mcpu=gfx1030 -flto {{.*}}-save-temps
 
 // RUN: clang-offload-packager -o %t.out \
 // RUN:   --image=file=%t.elf.o,kind=openmp,triple=x86_64-unknown-linux-gnu \
@@ -120,7 +119,7 @@ __attribute__((visibility("protected"), used)) int x;
 
 // HIP: clang{{.*}} -o [[IMG_GFX90A:.+]] --target=amdgcn-amd-amdhsa -mcpu=gfx90a
 // HIP: clang{{.*}} -o [[IMG_GFX908:.+]] --target=amdgcn-amd-amdhsa -mcpu=gfx908
-// HIP: clang-offload-bundler{{.*}}-type=o -bundle-align=4096 -compress -compression-level=6 -targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa--gfx90a,hip-amdgcn-amd-amdhsa--gfx908 -input=/dev/null -input=[[IMG_GFX90A]] -input=[[IMG_GFX908]] -output={{.*}}.hipfb
+// HIP: clang-offload-bundler{{.*}}-type=o -bundle-align=4096 -compress -compression-level=6 -targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa--gfx90a,hip-amdgcn-amd-amdhsa--gfx908 -input={{/dev/null|NUL}} -input=[[IMG_GFX90A]] -input=[[IMG_GFX908]] -output={{.*}}.hipfb
 
 // RUN: clang-offload-packager -o %t.out \
 // RUN:   --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx908 \
@@ -148,7 +147,7 @@ __attribute__((visibility("protected"), used)) int x;
 // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run --clang-backend \
 // RUN:   --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=CLANG-BACKEND
 
-// CLANG-BACKEND: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx908 -Wl,--no-undefined {{.*}}.o
+// CLANG-BACKEND: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx908 -flto -Wl,--no-undefined {{.*}}.o
 
 // RUN: clang-offload-packager -o %t.out \
 // RUN:   --image=file=%t.elf.o,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70
@@ -171,8 +170,8 @@ __attribute__((visibility("protected"), used)) int x;
 // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \
 // RUN:   --linker-path=/usr/bin/ld %t-on.o %t-off.o %t.a -o a.out 2>&1 | FileCheck %s --check-prefix=AMD-TARGET-ID
 
-// AMD-TARGET-ID: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx90a:xnack+ -Wl,--no-undefined {{.*}}.o {{.*}}.o
-// AMD-TARGET-ID: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx90a:xnack- -Wl,--no-undefined {{.*}}.o {{.*}}.o
+// AMD-TARGET-ID: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx90a:xnack+ -flto -Wl,--no-undefined {{.*}}.o {{.*}}.o
+// AMD-TARGET-ID: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx90a:xnack- -flto -Wl,--no-undefined {{.*}}.o {{.*}}.o
 
 // RUN: clang-offload-packager -o %t-lib.out \
 // RUN:   --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=generic
@@ -187,8 +186,8 @@ __attribute__((visibility("protected"), used)) int x;
 // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \
 // RUN:   --linker-path=/usr/bin/ld %t1.o %t2.o %t.a -o a.out 2>&1 | FileCheck %s --check-prefix=ARCH-ALL
 
-// ARCH-ALL: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx90a -Wl,--no-undefined {{.*}}.o {{.*}}.o
-// ARCH-ALL: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx908 -Wl,--no-undefined {{.*}}.o {{.*}}.o
+// ARCH-ALL: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx90a -flto -Wl,--no-undefined {{.*}}.o {{.*}}.o
+// ARCH-ALL: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx908 -flto -Wl,--no-undefined {{.*}}.o {{.*}}.o
 
 // RUN: clang-offload-packager -o %t.out \
 // RUN:   --image=file=%t.elf.o,kind=openmp,triple=x86_64-unknown-linux-gnu \
@@ -211,7 +210,7 @@ __attribute__((visibility("protected"), used)) int x;
 // RUN:   %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=RELOCATABLE-LINK-HIP
 
 // RELOCATABLE-LINK-HIP: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa
-// RELOCATABLE-LINK-HIP: clang-offload-bundler{{.*}} -type=o -bundle-align=4096 -targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa--gfx90a -input=/dev/null -input={{.*}} -output={{.*}}
+// RELOCATABLE-LINK-HIP: clang-offload-bundler{{.*}} -type=o -bundle-align=4096 -targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa--gfx90a -input={{/dev/null|NUL}} -input={{.*}} -output={{.*}}
 // RELOCATABLE-LINK-HIP: /usr/bin/ld.lld{{.*}}-r
 // RELOCATABLE-LINK-HIP: llvm-objcopy{{.*}}a.out --remove-section .llvm.offloading
 
diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c
index 69b76f0c4c4cd..d9335fe502bb6 100644
--- a/clang/test/Driver/print-supported-extensions-riscv.c
+++ b/clang/test/Driver/print-supported-extensions-riscv.c
@@ -201,7 +201,7 @@
 // CHECK-NEXT:     xqcicm               0.2       'Xqcicm' (Qualcomm uC Conditional Move Extension)
 // CHECK-NEXT:     xqcics               0.2       'Xqcics' (Qualcomm uC Conditional Select Extension)
 // CHECK-NEXT:     xqcicsr              0.2       'Xqcicsr' (Qualcomm uC CSR Extension)
-// CHECK-NEXT:     xqciint              0.2       'Xqciint' (Qualcomm uC Interrupts Extension)
+// CHECK-NEXT:     xqciint              0.4       'Xqciint' (Qualcomm uC Interrupts Extension)
 // CHECK-NEXT:     xqcilia              0.2       'Xqcilia' (Qualcomm uC Large Immediate Arithmetic Extension)
 // CHECK-NEXT:     xqcilo               0.2       'Xqcilo' (Qualcomm uC Large Offset Load Store Extension)
 // CHECK-NEXT:     xqcilsm              0.2       'Xqcilsm' (Qualcomm uC Load Store Multiple Extension)
diff --git a/clang/test/Driver/spirv-openmp-toolchain.c b/clang/test/Driver/spirv-openmp-toolchain.c
index 5a76bf70e7ed3..3fd6d94a1222b 100644
--- a/clang/test/Driver/spirv-openmp-toolchain.c
+++ b/clang/test/Driver/spirv-openmp-toolchain.c
@@ -4,8 +4,7 @@
 
 // verify the tools invocations
 // CHECK: "-cc1" "-triple" "x86_64-unknown-linux-gnu"{{.*}}"-emit-llvm-bc"{{.*}}"-x" "c"
-// CHECK: "-cc1" "-triple" "spirv64-intel" "-aux-triple" "x86_64-unknown-linux-gnu"
-// CHECK: llvm-spirv{{.*}}
+// CHECK: "-cc1" "-triple" "spirv64-intel" "-aux-triple" "x86_64-unknown-linux-gnu"{{.*}} "-o" "{{.*}}.o"
 // CHECK: "-cc1" "-triple" "x86_64-unknown-linux-gnu"{{.*}}"-emit-obj"
 // CHECK: clang-linker-wrapper{{.*}} "-o" "a.out"
 
@@ -32,8 +31,7 @@
 // RUN: %clang -### --target=x86_64-unknown-linux-gnu -ccc-print-bindings -fopenmp=libomp -fopenmp-targets=spirv64-intel -nogpulib %s 2>&1 | FileCheck %s --check-prefix=CHECK-BINDINGS
 
 // CHECK-BINDINGS: "x86_64-unknown-linux-gnu" - "clang", inputs: ["[[INPUT:.+]]"], output: "[[HOST_BC:.+]]"
-// CHECK-BINDINGS: "spirv64-intel" - "clang", inputs: ["[[INPUT]]", "[[HOST_BC]]"], output: "[[DEVICE_TEMP_BC:.+]]"
-// CHECK-BINDINGS: "spirv64-intel" - "SPIR-V::Translator", inputs: ["[[DEVICE_TEMP_BC]]"], output: "[[DEVICE_SPV:.+]]"
+// CHECK-BINDINGS: "spirv64-intel" - "clang", inputs: ["[[INPUT]]", "[[HOST_BC]]"], output: "[[DEVICE_SPV:.+]]"
 // CHECK-BINDINGS: "x86_64-unknown-linux-gnu" - "Offload::Packager", inputs: ["[[DEVICE_SPV]]"], output: "[[DEVICE_IMAGE:.+]]"
 // CHECK-BINDINGS: "x86_64-unknown-linux-gnu" - "clang", inputs: ["[[HOST_BC]]", "[[DEVICE_IMAGE]]"], output: "[[HOST_OBJ:.+]]"
 // CHECK-BINDINGS: "x86_64-unknown-linux-gnu" - "Offload::Linker", inputs: ["[[HOST_OBJ]]"], output: "a.out"
@@ -43,8 +41,8 @@
 // CHECK-BINDINGS-TEMPS: "x86_64-unknown-linux-gnu" - "clang", inputs: ["[[INPUT:.+]]"], output: "[[HOST_PP:.+]]"
 // CHECK-BINDINGS-TEMPS: "x86_64-unknown-linux-gnu" - "clang", inputs: ["[[HOST_PP]]"], output: "[[HOST_BC:.+]]"
 // CHECK-BINDINGS-TEMPS: "spirv64-intel" - "clang", inputs: ["[[INPUT]]"], output: "[[DEVICE_PP:.+]]"
-// CHECK-BINDINGS-TEMPS: "spirv64-intel" - "clang", inputs: ["[[DEVICE_PP]]", "[[HOST_BC]]"], output: "[[DEVICE_TEMP_BC:.+]]"
-// CHECK-BINDINGS-TEMPS: "spirv64-intel" - "SPIR-V::Translator", inputs: ["[[DEVICE_TEMP_BC]]"], output: "[[DEVICE_ASM:.+]]"
+// CHECK-BINDINGS-TEMPS: "spirv64-intel" - "clang", inputs: ["[[DEVICE_PP]]", "[[HOST_BC]]"], output: "[[DEVICE_BC:.+]]"
+// CHECK-BINDINGS-TEMPS: "spirv64-intel" - "clang", inputs: ["[[DEVICE_BC]]"], output: "[[DEVICE_ASM:.+]]"
 // CHECK-BINDINGS-TEMPS: "spirv64-intel" - "SPIR-V::Assembler", inputs: ["[[DEVICE_ASM]]"], output: "[[DEVICE_SPV:.+]]"
 // CHECK-BINDINGS-TEMPS: "x86_64-unknown-linux-gnu" - "Offload::Packager", inputs: ["[[DEVICE_SPV]]"], output: "[[DEVICE_IMAGE:.+]]"
 // CHECK-BINDINGS-TEMPS: "x86_64-unknown-linux-gnu" - "clang", inputs: ["[[HOST_BC]]", "[[DEVICE_IMAGE]]"], output: "[[HOST_ASM:.+]]"
diff --git a/clang/test/Driver/spirv-toolchain.cl b/clang/test/Driver/spirv-toolchain.cl
index fd29dbe71e910..53e8455e3d3dd 100644
--- a/clang/test/Driver/spirv-toolchain.cl
+++ b/clang/test/Driver/spirv-toolchain.cl
@@ -6,8 +6,7 @@
 // RUN: %clang -### --target=spirv64 -x c -c %s 2>&1 | FileCheck --check-prefix=SPV64 %s
 
 // SPV64: "-cc1" "-triple" "spirv64"
-// SPV64-SAME: "-o" [[BC:".*bc"]]
-// SPV64: {{llvm-spirv.*"}} [[BC]] "-o" {{".*o"}}
+// SPV64-SAME: "-o" {{".*o"}}
 
 // RUN: %clang -### --target=spirv32 -x cl -c %s 2>&1 | FileCheck --check-prefix=SPV32 %s
 // RUN: %clang -### --target=spirv32 %s 2>&1 | FileCheck --check-prefix=SPV32 %s
@@ -16,8 +15,7 @@
 // RUN: %clang -### --target=spirv32 -x c -c %s 2>&1 | FileCheck --check-prefix=SPV32 %s
 
 // SPV32: "-cc1" "-triple" "spirv32"
-// SPV32-SAME: "-o" [[BC:".*bc"]]
-// SPV32: {{llvm-spirv.*"}} [[BC]] "-o" {{".*o"}}
+// SPV32-SAME: "-o" {{".*o"}}
 
 //-----------------------------------------------------------------------------
 // Check Assembly emission.
@@ -27,8 +25,7 @@
 // RUN: %clang -### --target=spirv64 -x c -S %s 2>&1 | FileCheck --check-prefix=SPT64 %s
 
 // SPT64: "-cc1" "-triple" "spirv64"
-// SPT64-SAME: "-o" [[BC:".*bc"]]
-// SPT64: {{llvm-spirv.*"}} [[BC]] "--spirv-tools-dis" "-o" {{".*s"}}
+// SPT64-SAME: "-o" {{".*s"}}
 
 // RUN: %clang -### --target=spirv32 -x cl -S %s 2>&1 | FileCheck --check-prefix=SPT32 %s
 // RUN: %clang -### --target=spirv32 -x ir -S %s 2>&1 | FileCheck --check-prefix=SPT32 %s
@@ -36,8 +33,7 @@
 // RUN: %clang -### --target=spirv32 -x c -S %s 2>&1 | FileCheck --check-prefix=SPT32 %s
 
 // SPT32: "-cc1" "-triple" "spirv32"
-// SPT32-SAME: "-o" [[BC:".*bc"]]
-// SPT32: {{llvm-spirv.*"}} [[BC]] "--spirv-tools-dis" "-o" {{".*s"}}
+// SPT32-SAME: "-o" {{".*s"}}
 
 //-----------------------------------------------------------------------------
 // Check assembly input -> object output
@@ -55,7 +51,9 @@
 // TMP: "-cc1" "-triple" "spirv64"
 // TMP-SAME: "-o" [[BC:".*bc"]]
 // TMP-SAME: [[I]]
-// TMP: {{llvm-spirv.*"}} [[BC]] "--spirv-tools-dis" "-o" [[S:".*s"]]
+// TMP: "-cc1"
+// TMP-SAME: "-o" [[S:".*s"]]
+// TMP-SAME: [[BC]]
 // TMP: {{spirv-as.*"}} [[S]] "-o" {{".*o"}}
 
 //-----------------------------------------------------------------------------
@@ -63,21 +61,17 @@
 // RUN: %clang -### -target spirv64 %s %s 2>&1 | FileCheck --check-prefix=SPLINK %s
 
 // SPLINK: "-cc1" "-triple" "spirv64"
-// SPLINK-SAME: "-o" [[BC:".*bc"]]
-// SPLINK: {{llvm-spirv.*"}} [[BC]] "-o" [[SPV1:".*o"]]
+// SPLINK-SAME: "-o" [[SPV1:".*o"]]
 // SPLINK: "-cc1" "-triple" "spirv64"
-// SPLINK-SAME: "-o" [[BC:".*bc"]]
-// SPLINK: {{llvm-spirv.*"}} [[BC]] "-o" [[SPV2:".*o"]]
+// SPLINK-SAME: "-o" [[SPV2:".*o"]]
 // SPLINK: {{spirv-link.*"}} [[SPV1]] [[SPV2]] "-o" "a.out"
 
 //-----------------------------------------------------------------------------
 // Check bindings when linking when multiple input files are passed.
 // RUN: %clang -### -target spirv64 -ccc-print-bindings %s %s 2>&1 | FileCheck --check-prefix=SPLINK-BINDINGS %s
 
-// SPLINK-BINDINGS: "clang", inputs: [[[CL:".*cl"]]], output: [[BC1:".*bc"]]
-// SPLINK-BINDINGS: "SPIR-V::Translator", inputs: [[[BC1]]], output: [[OBJ1:".*o"]]
-// SPLINK-BINDINGS: "clang", inputs: [[[CL]]], output: [[BC2:".*bc"]]
-// SPLINK-BINDINGS: "SPIR-V::Translator", inputs: [[[BC2]]], output: [[OBJ2:".*o"]]
+// SPLINK-BINDINGS: "clang", inputs: [[[CL:".*cl"]]], output: [[OBJ1:".*o"]]
+// SPLINK-BINDINGS: "clang", inputs: [[[CL]]], output: [[OBJ2:".*o"]]
 // SPLINK-BINDINGS: "SPIR-V::Linker", inputs: [[[OBJ1]], [[OBJ2]]], output: "a.out"
 
 //-----------------------------------------------------------------------------
@@ -85,20 +79,20 @@
 // RUN: %clang -### --target=spirv64 -fno-integrated-objemitter %s 2>&1 | FileCheck --check-prefix=XTOR %s
 // RUN: %clang -### --target=spirv64 -fintegrated-objemitter %s 2>&1 | FileCheck --check-prefix=BACKEND %s
 
-// XTOR: {{llvm-spirv.*"}}
-// BACKEND-NOT: {{llvm-spirv.*"}}
+// XTOR-NOT: "llvm-spirv.*"
+// BACKEND-NOT: "llvm-spirv.*"
 
 //-----------------------------------------------------------------------------
-// Check llvm-spirv-<LLVM_VERSION_MAJOR> is used if it is found in PATH.
+// Check spirv-as-<LLVM_VERSION_MAJOR> is used if it is found in PATH.
 //
 // This test uses the PATH environment variable; on Windows, we may need to retain
 // the original path for the built Clang binary to be able to execute (as it is
 // used for locating dependent DLLs). Therefore, skip this test on system-windows.
 //
 // RUN: mkdir -p %t/versioned
-// RUN: touch %t/versioned/llvm-spirv-%llvm-version-major \
-// RUN:   && chmod +x %t/versioned/llvm-spirv-%llvm-version-major
-// RUN: %if !system-windows %{ env "PATH=%t/versioned" %clang -### --target=spirv64 -x cl -c %s 2>&1 \
+// RUN: touch %t/versioned/spirv-as-%llvm-version-major \
+// RUN:   && chmod +x %t/versioned/spirv-as-%llvm-version-major
+// RUN: %if !system-windows %{ env "PATH=%t/versioned" %clang -### --target=spirv64 -x cl -c --save-temps %s 2>&1 \
 // RUN:   | FileCheck -DVERSION=%llvm-version-major --check-prefix=VERSIONED %s %}
 
-// VERSIONED: {{.*}}llvm-spirv-[[VERSION]]
+// VERSIONED: {{.*}}spirv-as-[[VERSION]]
diff --git a/clang/test/Headers/gpuintrin.c b/clang/test/Headers/gpuintrin.c
index 89efe12ee8def..30aa6f147ba03 100644
--- a/clang/test/Headers/gpuintrin.c
+++ b/clang/test/Headers/gpuintrin.c
@@ -1,10 +1,10 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
-// RUN: %clang_cc1 -internal-isystem %S/Inputs/include \
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --include-generated-funcs --version 5
+// RUN: %clang_cc1 -internal-isystem %S/Inputs/include  \
 // RUN:   -internal-isystem %S/../../lib/Headers/ \
 // RUN:   -triple amdgcn-amd-amdhsa -emit-llvm %s -o - \
 // RUN: | FileCheck %s --check-prefix=AMDGPU
 //
-// RUN: %clang_cc1 -internal-isystem %S/Inputs/include \
+// RUN: %clang_cc1 -internal-isystem %S/Inputs/include  \
 // RUN:   -internal-isystem %S/../../lib/Headers/ \
 // RUN:   -target-feature +ptx62 \
 // RUN:   -triple nvptx64-nvidia-cuda -emit-llvm %s -o - \
@@ -12,6 +12,35 @@
 
 #include <gpuintrin.h>
 
+__gpu_kernel void foo() {
+  __gpu_num_blocks_x();
+  __gpu_num_blocks_y();
+  __gpu_num_blocks_z();
+  __gpu_num_blocks(0);
+  __gpu_block_id_x();
+  __gpu_block_id_y();
+  __gpu_block_id_z();
+  __gpu_block_id(0);
+  __gpu_num_threads_x();
+  __gpu_num_threads_y();
+  __gpu_num_threads_z();
+  __gpu_num_threads(0);
+  __gpu_thread_id_x();
+  __gpu_thread_id_y();
+  __gpu_thread_id_z();
+  __gpu_thread_id(0);
+  __gpu_num_lanes();
+  __gpu_lane_id();
+  __gpu_lane_mask();
+  __gpu_read_first_lane_u32(-1, -1);
+  __gpu_ballot(-1, 1);
+  __gpu_sync_threads();
+  __gpu_sync_lane(-1);
+  __gpu_shuffle_idx_u32(-1, -1, -1, 0);
+  __gpu_first_lane_id(-1);
+  __gpu_is_first_in_lane(-1);
+  __gpu_exit();
+}
 // AMDGPU-LABEL: define protected amdgpu_kernel void @foo(
 // AMDGPU-SAME: ) #[[ATTR0:[0-9]+]] {
 // AMDGPU-NEXT:  [[ENTRY:.*:]]
@@ -44,52 +73,244 @@
 // AMDGPU-NEXT:    call void @__gpu_exit() #[[ATTR8:[0-9]+]]
 // AMDGPU-NEXT:    unreachable
 //
-// NVPTX-LABEL: define protected ptx_kernel void @foo(
-// NVPTX-SAME: ) #[[ATTR0:[0-9]+]] {
-// NVPTX-NEXT:  [[ENTRY:.*:]]
-// NVPTX-NEXT:    [[CALL:%.*]] = call i32 @__gpu_num_blocks_x() #[[ATTR6:[0-9]+]]
-// NVPTX-NEXT:    [[CALL1:%.*]] = call i32 @__gpu_num_blocks_y() #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL2:%.*]] = call i32 @__gpu_num_blocks_z() #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL3:%.*]] = call i32 @__gpu_num_blocks(i32 noundef 0) #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL4:%.*]] = call i32 @__gpu_block_id_x() #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL5:%.*]] = call i32 @__gpu_block_id_y() #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL6:%.*]] = call i32 @__gpu_block_id_z() #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL7:%.*]] = call i32 @__gpu_block_id(i32 noundef 0) #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL8:%.*]] = call i32 @__gpu_num_threads_x() #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL9:%.*]] = call i32 @__gpu_num_threads_y() #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL10:%.*]] = call i32 @__gpu_num_threads_z() #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL11:%.*]] = call i32 @__gpu_num_threads(i32 noundef 0) #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL12:%.*]] = call i32 @__gpu_thread_id_x() #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL13:%.*]] = call i32 @__gpu_thread_id_y() #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL14:%.*]] = call i32 @__gpu_thread_id_z() #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL15:%.*]] = call i32 @__gpu_thread_id(i32 noundef 0) #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL16:%.*]] = call i32 @__gpu_num_lanes() #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL17:%.*]] = call i32 @__gpu_lane_id() #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL18:%.*]] = call i64 @__gpu_lane_mask() #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL19:%.*]] = call i32 @__gpu_read_first_lane_u32(i64 noundef -1, i32 noundef -1) #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL20:%.*]] = call i64 @__gpu_ballot(i64 noundef -1, i1 noundef zeroext true) #[[ATTR6]]
-// NVPTX-NEXT:    call void @__gpu_sync_threads() #[[ATTR6]]
-// NVPTX-NEXT:    call void @__gpu_sync_lane(i64 noundef -1) #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL21:%.*]] = call i32 @__gpu_shuffle_idx_u32(i64 noundef -1, i32 noundef -1, i32 noundef -1, i32 noundef 0) #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL22:%.*]] = call i64 @__gpu_first_lane_id(i64 noundef -1) #[[ATTR6]]
-// NVPTX-NEXT:    [[CALL23:%.*]] = call zeroext i1 @__gpu_is_first_in_lane(i64 noundef -1) #[[ATTR6]]
-// NVPTX-NEXT:    call void @__gpu_exit() #[[ATTR7:[0-9]+]]
-// NVPTX-NEXT:    unreachable
 //
-__gpu_kernel void foo() {
-  __gpu_num_blocks_x();
-  __gpu_num_blocks_y();
-  __gpu_num_blocks_z();
-  __gpu_num_blocks(0);
-  __gpu_block_id_x();
-  __gpu_block_id_y();
-  __gpu_block_id_z();
-  __gpu_block_id(0);
-  __gpu_num_threads_x();
-  __gpu_num_threads_y();
-  __gpu_num_threads_z();
-  __gpu_num_threads(0);
-  __gpu_thread_id_x();
+// AMDGPU-LABEL: define internal i32 @__gpu_num_blocks_x(
+// AMDGPU-SAME: ) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[TMP0:%.*]] = call align 4 dereferenceable(64) ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
+// AMDGPU-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr addrspace(4) [[TMP0]], i32 12
+// AMDGPU-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(4) [[TMP1]], align 4, !range [[RNG3:![0-9]+]], !invariant.load [[META4:![0-9]+]]
+// AMDGPU-NEXT:    [[TMP3:%.*]] = call align 8 dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+// AMDGPU-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr addrspace(4) [[TMP3]], i32 12
+// AMDGPU-NEXT:    [[TMP5:%.*]] = load i16, ptr addrspace(4) [[TMP4]], align 2, !range [[RNG5:![0-9]+]], !invariant.load [[META4]], !noundef [[META4]]
+// AMDGPU-NEXT:    [[CONV:%.*]] = zext i16 [[TMP5]] to i32
+// AMDGPU-NEXT:    [[DIV:%.*]] = udiv i32 [[TMP2]], [[CONV]]
+// AMDGPU-NEXT:    ret i32 [[DIV]]
+//
+//
+// AMDGPU-LABEL: define internal i32 @__gpu_num_blocks_y(
+// AMDGPU-SAME: ) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[TMP0:%.*]] = call align 4 dereferenceable(64) ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
+// AMDGPU-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr addrspace(4) [[TMP0]], i32 16
+// AMDGPU-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(4) [[TMP1]], align 4, !range [[RNG3]], !invariant.load [[META4]]
+// AMDGPU-NEXT:    [[TMP3:%.*]] = call align 8 dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+// AMDGPU-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr addrspace(4) [[TMP3]], i32 14
+// AMDGPU-NEXT:    [[TMP5:%.*]] = load i16, ptr addrspace(4) [[TMP4]], align 2, !range [[RNG5]], !invariant.load [[META4]], !noundef [[META4]]
+// AMDGPU-NEXT:    [[CONV:%.*]] = zext i16 [[TMP5]] to i32
+// AMDGPU-NEXT:    [[DIV:%.*]] = udiv i32 [[TMP2]], [[CONV]]
+// AMDGPU-NEXT:    ret i32 [[DIV]]
+//
+//
+// AMDGPU-LABEL: define internal i32 @__gpu_num_blocks_z(
+// AMDGPU-SAME: ) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[TMP0:%.*]] = call align 4 dereferenceable(64) ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
+// AMDGPU-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr addrspace(4) [[TMP0]], i32 20
+// AMDGPU-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(4) [[TMP1]], align 4, !range [[RNG3]], !invariant.load [[META4]]
+// AMDGPU-NEXT:    [[TMP3:%.*]] = call align 8 dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+// AMDGPU-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr addrspace(4) [[TMP3]], i32 16
+// AMDGPU-NEXT:    [[TMP5:%.*]] = load i16, ptr addrspace(4) [[TMP4]], align 2, !range [[RNG5]], !invariant.load [[META4]], !noundef [[META4]]
+// AMDGPU-NEXT:    [[CONV:%.*]] = zext i16 [[TMP5]] to i32
+// AMDGPU-NEXT:    [[DIV:%.*]] = udiv i32 [[TMP2]], [[CONV]]
+// AMDGPU-NEXT:    ret i32 [[DIV]]
+//
+//
+// AMDGPU-LABEL: define internal i32 @__gpu_num_blocks(
+// AMDGPU-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[__DIM_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[__DIM_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__DIM_ADDR]] to ptr
+// AMDGPU-NEXT:    store i32 [[__DIM]], ptr [[__DIM_ADDR_ASCAST]], align 4
+// AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR_ASCAST]], align 4
+// AMDGPU-NEXT:    switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
+// AMDGPU-NEXT:      i32 0, label %[[SW_BB:.*]]
+// AMDGPU-NEXT:      i32 1, label %[[SW_BB1:.*]]
+// AMDGPU-NEXT:      i32 2, label %[[SW_BB3:.*]]
+// AMDGPU-NEXT:    ]
+// AMDGPU:       [[SW_BB]]:
+// AMDGPU-NEXT:    [[CALL:%.*]] = call i32 @__gpu_num_blocks_x() #[[ATTR7]]
+// AMDGPU-NEXT:    store i32 [[CALL]], ptr [[RETVAL_ASCAST]], align 4
+// AMDGPU-NEXT:    br label %[[RETURN:.*]]
+// AMDGPU:       [[SW_BB1]]:
+// AMDGPU-NEXT:    [[CALL2:%.*]] = call i32 @__gpu_num_blocks_y() #[[ATTR7]]
+// AMDGPU-NEXT:    store i32 [[CALL2]], ptr [[RETVAL_ASCAST]], align 4
+// AMDGPU-NEXT:    br label %[[RETURN]]
+// AMDGPU:       [[SW_BB3]]:
+// AMDGPU-NEXT:    [[CALL4:%.*]] = call i32 @__gpu_num_blocks_z() #[[ATTR7]]
+// AMDGPU-NEXT:    store i32 [[CALL4]], ptr [[RETVAL_ASCAST]], align 4
+// AMDGPU-NEXT:    br label %[[RETURN]]
+// AMDGPU:       [[SW_DEFAULT]]:
+// AMDGPU-NEXT:    unreachable
+// AMDGPU:       [[RETURN]]:
+// AMDGPU-NEXT:    [[TMP1:%.*]] = load i32, ptr [[RETVAL_ASCAST]], align 4
+// AMDGPU-NEXT:    ret i32 [[TMP1]]
+//
+//
+// AMDGPU-LABEL: define internal i32 @__gpu_block_id_x(
+// AMDGPU-SAME: ) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[TMP0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
+// AMDGPU-NEXT:    ret i32 [[TMP0]]
+//
+//
+// AMDGPU-LABEL: define internal i32 @__gpu_block_id_y(
+// AMDGPU-SAME: ) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[TMP0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
+// AMDGPU-NEXT:    ret i32 [[TMP0]]
+//
+//
+// AMDGPU-LABEL: define internal i32 @__gpu_block_id_z(
+// AMDGPU-SAME: ) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[TMP0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
+// AMDGPU-NEXT:    ret i32 [[TMP0]]
+//
+//
+// AMDGPU-LABEL: define internal i32 @__gpu_block_id(
+// AMDGPU-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[__DIM_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[__DIM_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__DIM_ADDR]] to ptr
+// AMDGPU-NEXT:    store i32 [[__DIM]], ptr [[__DIM_ADDR_ASCAST]], align 4
+// AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR_ASCAST]], align 4
+// AMDGPU-NEXT:    switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
+// AMDGPU-NEXT:      i32 0, label %[[SW_BB:.*]]
+// AMDGPU-NEXT:      i32 1, label %[[SW_BB1:.*]]
+// AMDGPU-NEXT:      i32 2, label %[[SW_BB3:.*]]
+// AMDGPU-NEXT:    ]
+// AMDGPU:       [[SW_BB]]:
+// AMDGPU-NEXT:    [[CALL:%.*]] = call i32 @__gpu_block_id_x() #[[ATTR7]]
+// AMDGPU-NEXT:    store i32 [[CALL]], ptr [[RETVAL_ASCAST]], align 4
+// AMDGPU-NEXT:    br label %[[RETURN:.*]]
+// AMDGPU:       [[SW_BB1]]:
+// AMDGPU-NEXT:    [[CALL2:%.*]] = call i32 @__gpu_block_id_y() #[[ATTR7]]
+// AMDGPU-NEXT:    store i32 [[CALL2]], ptr [[RETVAL_ASCAST]], align 4
+// AMDGPU-NEXT:    br label %[[RETURN]]
+// AMDGPU:       [[SW_BB3]]:
+// AMDGPU-NEXT:    [[CALL4:%.*]] = call i32 @__gpu_block_id_z() #[[ATTR7]]
+// AMDGPU-NEXT:    store i32 [[CALL4]], ptr [[RETVAL_ASCAST]], align 4
+// AMDGPU-NEXT:    br label %[[RETURN]]
+// AMDGPU:       [[SW_DEFAULT]]:
+// AMDGPU-NEXT:    unreachable
+// AMDGPU:       [[RETURN]]:
+// AMDGPU-NEXT:    [[TMP1:%.*]] = load i32, ptr [[RETVAL_ASCAST]], align 4
+// AMDGPU-NEXT:    ret i32 [[TMP1]]
+//
+//
+// AMDGPU-LABEL: define internal i32 @__gpu_num_threads_x(
+// AMDGPU-SAME: ) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[TMP0:%.*]] = call align 8 dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+// AMDGPU-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr addrspace(4) [[TMP0]], i32 12
+// AMDGPU-NEXT:    [[TMP2:%.*]] = load i16, ptr addrspace(4) [[TMP1]], align 2, !range [[RNG5]], !invariant.load [[META4]], !noundef [[META4]]
+// AMDGPU-NEXT:    [[CONV:%.*]] = zext i16 [[TMP2]] to i32
+// AMDGPU-NEXT:    ret i32 [[CONV]]
+//
+//
+// AMDGPU-LABEL: define internal i32 @__gpu_num_threads_y(
+// AMDGPU-SAME: ) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[TMP0:%.*]] = call align 8 dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+// AMDGPU-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr addrspace(4) [[TMP0]], i32 14
+// AMDGPU-NEXT:    [[TMP2:%.*]] = load i16, ptr addrspace(4) [[TMP1]], align 2, !range [[RNG5]], !invariant.load [[META4]], !noundef [[META4]]
+// AMDGPU-NEXT:    [[CONV:%.*]] = zext i16 [[TMP2]] to i32
+// AMDGPU-NEXT:    ret i32 [[CONV]]
+//
+//
+// AMDGPU-LABEL: define internal i32 @__gpu_num_threads_z(
+// AMDGPU-SAME: ) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[TMP0:%.*]] = call align 8 dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+// AMDGPU-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr addrspace(4) [[TMP0]], i32 16
+// AMDGPU-NEXT:    [[TMP2:%.*]] = load i16, ptr addrspace(4) [[TMP1]], align 2, !range [[RNG5]], !invariant.load [[META4]], !noundef [[META4]]
+// AMDGPU-NEXT:    [[CONV:%.*]] = zext i16 [[TMP2]] to i32
+// AMDGPU-NEXT:    ret i32 [[CONV]]
+//
+//
+// AMDGPU-LABEL: define internal i32 @__gpu_num_threads(
+// AMDGPU-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[__DIM_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[__DIM_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__DIM_ADDR]] to ptr
+// AMDGPU-NEXT:    store i32 [[__DIM]], ptr [[__DIM_ADDR_ASCAST]], align 4
+// AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR_ASCAST]], align 4
+// AMDGPU-NEXT:    switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
+// AMDGPU-NEXT:      i32 0, label %[[SW_BB:.*]]
+// AMDGPU-NEXT:      i32 1, label %[[SW_BB1:.*]]
+// AMDGPU-NEXT:      i32 2, label %[[SW_BB3:.*]]
+// AMDGPU-NEXT:    ]
+// AMDGPU:       [[SW_BB]]:
+// AMDGPU-NEXT:    [[CALL:%.*]] = call i32 @__gpu_num_threads_x() #[[ATTR7]]
+// AMDGPU-NEXT:    store i32 [[CALL]], ptr [[RETVAL_ASCAST]], align 4
+// AMDGPU-NEXT:    br label %[[RETURN:.*]]
+// AMDGPU:       [[SW_BB1]]:
+// AMDGPU-NEXT:    [[CALL2:%.*]] = call i32 @__gpu_num_threads_y() #[[ATTR7]]
+// AMDGPU-NEXT:    store i32 [[CALL2]], ptr [[RETVAL_ASCAST]], align 4
+// AMDGPU-NEXT:    br label %[[RETURN]]
+// AMDGPU:       [[SW_BB3]]:
+// AMDGPU-NEXT:    [[CALL4:%.*]] = call i32 @__gpu_num_threads_z() #[[ATTR7]]
+// AMDGPU-NEXT:    store i32 [[CALL4]], ptr [[RETVAL_ASCAST]], align 4
+// AMDGPU-NEXT:    br label %[[RETURN]]
+// AMDGPU:       [[SW_DEFAULT]]:
+// AMDGPU-NEXT:    unreachable
+// AMDGPU:       [[RETURN]]:
+// AMDGPU-NEXT:    [[TMP1:%.*]] = load i32, ptr [[RETVAL_ASCAST]], align 4
+// AMDGPU-NEXT:    ret i32 [[TMP1]]
+//
+//
+// AMDGPU-LABEL: define internal i32 @__gpu_thread_id_x(
+// AMDGPU-SAME: ) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[TMP0:%.*]] = call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+// AMDGPU-NEXT:    ret i32 [[TMP0]]
+//
+//
+// AMDGPU-LABEL: define internal i32 @__gpu_thread_id_y(
+// AMDGPU-SAME: ) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[TMP0:%.*]] = call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
+// AMDGPU-NEXT:    ret i32 [[TMP0]]
+//
+//
+// AMDGPU-LABEL: define internal i32 @__gpu_thread_id_z(
+// AMDGPU-SAME: ) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[TMP0:%.*]] = call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.z()
+// AMDGPU-NEXT:    ret i32 [[TMP0]]
+//
+//
 // AMDGPU-LABEL: define internal i32 @__gpu_thread_id(
 // AMDGPU-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
 // AMDGPU-NEXT:  [[ENTRY:.*:]]
@@ -122,6 +343,375 @@ __gpu_kernel void foo() {
 // AMDGPU-NEXT:    [[TMP1:%.*]] = load i32, ptr [[RETVAL_ASCAST]], align 4
 // AMDGPU-NEXT:    ret i32 [[TMP1]]
 //
+//
+// AMDGPU-LABEL: define internal i32 @__gpu_num_lanes(
+// AMDGPU-SAME: ) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[TMP0:%.*]] = call i32 @llvm.amdgcn.wavefrontsize()
+// AMDGPU-NEXT:    ret i32 [[TMP0]]
+//
+//
+// AMDGPU-LABEL: define internal i32 @__gpu_lane_id(
+// AMDGPU-SAME: ) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[TMP0:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+// AMDGPU-NEXT:    [[TMP1:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[TMP0]])
+// AMDGPU-NEXT:    ret i32 [[TMP1]]
+//
+//
+// AMDGPU-LABEL: define internal i64 @__gpu_lane_mask(
+// AMDGPU-SAME: ) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i64, align 8, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+// AMDGPU-NEXT:    ret i64 [[TMP0]]
+//
+//
+// AMDGPU-LABEL: define internal i32 @__gpu_read_first_lane_u32(
+// AMDGPU-SAME: i64 noundef [[__LANE_MASK:%.*]], i32 noundef [[__X:%.*]]) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8, addrspace(5)
+// AMDGPU-NEXT:    [[__X_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[__LANE_MASK_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__LANE_MASK_ADDR]] to ptr
+// AMDGPU-NEXT:    [[__X_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__X_ADDR]] to ptr
+// AMDGPU-NEXT:    store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
+// AMDGPU-NEXT:    store i32 [[__X]], ptr [[__X_ADDR_ASCAST]], align 4
+// AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[__X_ADDR_ASCAST]], align 4
+// AMDGPU-NEXT:    [[TMP1:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP0]])
+// AMDGPU-NEXT:    ret i32 [[TMP1]]
+//
+//
+// AMDGPU-LABEL: define internal i64 @__gpu_ballot(
+// AMDGPU-SAME: i64 noundef [[__LANE_MASK:%.*]], i1 noundef zeroext [[__X:%.*]]) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i64, align 8, addrspace(5)
+// AMDGPU-NEXT:    [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8, addrspace(5)
+// AMDGPU-NEXT:    [[__X_ADDR:%.*]] = alloca i8, align 1, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[__LANE_MASK_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__LANE_MASK_ADDR]] to ptr
+// AMDGPU-NEXT:    [[__X_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__X_ADDR]] to ptr
+// AMDGPU-NEXT:    store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
+// AMDGPU-NEXT:    [[STOREDV:%.*]] = zext i1 [[__X]] to i8
+// AMDGPU-NEXT:    store i8 [[STOREDV]], ptr [[__X_ADDR_ASCAST]], align 1
+// AMDGPU-NEXT:    [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
+// AMDGPU-NEXT:    [[TMP1:%.*]] = load i8, ptr [[__X_ADDR_ASCAST]], align 1
+// AMDGPU-NEXT:    [[LOADEDV:%.*]] = trunc i8 [[TMP1]] to i1
+// AMDGPU-NEXT:    [[TMP2:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[LOADEDV]])
+// AMDGPU-NEXT:    [[AND:%.*]] = and i64 [[TMP0]], [[TMP2]]
+// AMDGPU-NEXT:    ret i64 [[AND]]
+//
+//
+// AMDGPU-LABEL: define internal void @__gpu_sync_threads(
+// AMDGPU-SAME: ) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    call void @llvm.amdgcn.s.barrier()
+// AMDGPU-NEXT:    fence syncscope("workgroup") seq_cst
+// AMDGPU-NEXT:    ret void
+//
+//
+// AMDGPU-LABEL: define internal void @__gpu_sync_lane(
+// AMDGPU-SAME: i64 noundef [[__LANE_MASK:%.*]]) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8, addrspace(5)
+// AMDGPU-NEXT:    [[__LANE_MASK_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__LANE_MASK_ADDR]] to ptr
+// AMDGPU-NEXT:    store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
+// AMDGPU-NEXT:    call void @llvm.amdgcn.wave.barrier()
+// AMDGPU-NEXT:    ret void
+//
+//
+// AMDGPU-LABEL: define internal i32 @__gpu_shuffle_idx_u32(
+// AMDGPU-SAME: i64 noundef [[__LANE_MASK:%.*]], i32 noundef [[__IDX:%.*]], i32 noundef [[__X:%.*]], i32 noundef [[__WIDTH:%.*]]) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8, addrspace(5)
+// AMDGPU-NEXT:    [[__IDX_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[__X_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[__WIDTH_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[__LANE:%.*]] = alloca i32, align 4, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[__LANE_MASK_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__LANE_MASK_ADDR]] to ptr
+// AMDGPU-NEXT:    [[__IDX_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__IDX_ADDR]] to ptr
+// AMDGPU-NEXT:    [[__X_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__X_ADDR]] to ptr
+// AMDGPU-NEXT:    [[__WIDTH_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__WIDTH_ADDR]] to ptr
+// AMDGPU-NEXT:    [[__LANE_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__LANE]] to ptr
+// AMDGPU-NEXT:    store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
+// AMDGPU-NEXT:    store i32 [[__IDX]], ptr [[__IDX_ADDR_ASCAST]], align 4
+// AMDGPU-NEXT:    store i32 [[__X]], ptr [[__X_ADDR_ASCAST]], align 4
+// AMDGPU-NEXT:    store i32 [[__WIDTH]], ptr [[__WIDTH_ADDR_ASCAST]], align 4
+// AMDGPU-NEXT:    [[TMP0:%.*]] = load i32, ptr [[__IDX_ADDR_ASCAST]], align 4
+// AMDGPU-NEXT:    [[CALL:%.*]] = call i32 @__gpu_lane_id() #[[ATTR7]]
+// AMDGPU-NEXT:    [[TMP1:%.*]] = load i32, ptr [[__WIDTH_ADDR_ASCAST]], align 4
+// AMDGPU-NEXT:    [[SUB:%.*]] = sub i32 [[TMP1]], 1
+// AMDGPU-NEXT:    [[NOT:%.*]] = xor i32 [[SUB]], -1
+// AMDGPU-NEXT:    [[AND:%.*]] = and i32 [[CALL]], [[NOT]]
+// AMDGPU-NEXT:    [[ADD:%.*]] = add i32 [[TMP0]], [[AND]]
+// AMDGPU-NEXT:    store i32 [[ADD]], ptr [[__LANE_ASCAST]], align 4
+// AMDGPU-NEXT:    [[TMP2:%.*]] = load i32, ptr [[__LANE_ASCAST]], align 4
+// AMDGPU-NEXT:    [[SHL:%.*]] = shl i32 [[TMP2]], 2
+// AMDGPU-NEXT:    [[TMP3:%.*]] = load i32, ptr [[__X_ADDR_ASCAST]], align 4
+// AMDGPU-NEXT:    [[TMP4:%.*]] = call i32 @llvm.amdgcn.ds.bpermute(i32 [[SHL]], i32 [[TMP3]])
+// AMDGPU-NEXT:    ret i32 [[TMP4]]
+//
+//
+// AMDGPU-LABEL: define internal i64 @__gpu_first_lane_id(
+// AMDGPU-SAME: i64 noundef [[__LANE_MASK:%.*]]) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i64, align 8, addrspace(5)
+// AMDGPU-NEXT:    [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[__LANE_MASK_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__LANE_MASK_ADDR]] to ptr
+// AMDGPU-NEXT:    store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
+// AMDGPU-NEXT:    [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
+// AMDGPU-NEXT:    [[TMP1:%.*]] = call i64 @llvm.cttz.i64(i64 [[TMP0]], i1 true)
+// AMDGPU-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 1
+// AMDGPU-NEXT:    [[ISZERO:%.*]] = icmp eq i64 [[TMP0]], 0
+// AMDGPU-NEXT:    [[FFS:%.*]] = select i1 [[ISZERO]], i64 0, i64 [[TMP2]]
+// AMDGPU-NEXT:    [[CAST:%.*]] = trunc i64 [[FFS]] to i32
+// AMDGPU-NEXT:    [[SUB:%.*]] = sub nsw i32 [[CAST]], 1
+// AMDGPU-NEXT:    [[CONV:%.*]] = sext i32 [[SUB]] to i64
+// AMDGPU-NEXT:    ret i64 [[CONV]]
+//
+//
+// AMDGPU-LABEL: define internal zeroext i1 @__gpu_is_first_in_lane(
+// AMDGPU-SAME: i64 noundef [[__LANE_MASK:%.*]]) #[[ATTR0]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    [[RETVAL:%.*]] = alloca i1, align 1, addrspace(5)
+// AMDGPU-NEXT:    [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8, addrspace(5)
+// AMDGPU-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// AMDGPU-NEXT:    [[__LANE_MASK_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__LANE_MASK_ADDR]] to ptr
+// AMDGPU-NEXT:    store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
+// AMDGPU-NEXT:    [[CALL:%.*]] = call i32 @__gpu_lane_id() #[[ATTR7]]
+// AMDGPU-NEXT:    [[CONV:%.*]] = zext i32 [[CALL]] to i64
+// AMDGPU-NEXT:    [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
+// AMDGPU-NEXT:    [[CALL1:%.*]] = call i64 @__gpu_first_lane_id(i64 noundef [[TMP0]]) #[[ATTR7]]
+// AMDGPU-NEXT:    [[CMP:%.*]] = icmp eq i64 [[CONV]], [[CALL1]]
+// AMDGPU-NEXT:    ret i1 [[CMP]]
+//
+//
+// AMDGPU-LABEL: define internal void @__gpu_exit(
+// AMDGPU-SAME: ) #[[ATTR1:[0-9]+]] {
+// AMDGPU-NEXT:  [[ENTRY:.*:]]
+// AMDGPU-NEXT:    call void @llvm.amdgcn.endpgm()
+// AMDGPU-NEXT:    ret void
+//
+//
+// NVPTX-LABEL: define protected ptx_kernel void @foo(
+// NVPTX-SAME: ) #[[ATTR0:[0-9]+]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[CALL:%.*]] = call i32 @__gpu_num_blocks_x() #[[ATTR6:[0-9]+]]
+// NVPTX-NEXT:    [[CALL1:%.*]] = call i32 @__gpu_num_blocks_y() #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL2:%.*]] = call i32 @__gpu_num_blocks_z() #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL3:%.*]] = call i32 @__gpu_num_blocks(i32 noundef 0) #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL4:%.*]] = call i32 @__gpu_block_id_x() #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL5:%.*]] = call i32 @__gpu_block_id_y() #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL6:%.*]] = call i32 @__gpu_block_id_z() #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL7:%.*]] = call i32 @__gpu_block_id(i32 noundef 0) #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL8:%.*]] = call i32 @__gpu_num_threads_x() #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL9:%.*]] = call i32 @__gpu_num_threads_y() #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL10:%.*]] = call i32 @__gpu_num_threads_z() #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL11:%.*]] = call i32 @__gpu_num_threads(i32 noundef 0) #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL12:%.*]] = call i32 @__gpu_thread_id_x() #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL13:%.*]] = call i32 @__gpu_thread_id_y() #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL14:%.*]] = call i32 @__gpu_thread_id_z() #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL15:%.*]] = call i32 @__gpu_thread_id(i32 noundef 0) #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL16:%.*]] = call i32 @__gpu_num_lanes() #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL17:%.*]] = call i32 @__gpu_lane_id() #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL18:%.*]] = call i64 @__gpu_lane_mask() #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL19:%.*]] = call i32 @__gpu_read_first_lane_u32(i64 noundef -1, i32 noundef -1) #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL20:%.*]] = call i64 @__gpu_ballot(i64 noundef -1, i1 noundef zeroext true) #[[ATTR6]]
+// NVPTX-NEXT:    call void @__gpu_sync_threads() #[[ATTR6]]
+// NVPTX-NEXT:    call void @__gpu_sync_lane(i64 noundef -1) #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL21:%.*]] = call i32 @__gpu_shuffle_idx_u32(i64 noundef -1, i32 noundef -1, i32 noundef -1, i32 noundef 0) #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL22:%.*]] = call i64 @__gpu_first_lane_id(i64 noundef -1) #[[ATTR6]]
+// NVPTX-NEXT:    [[CALL23:%.*]] = call zeroext i1 @__gpu_is_first_in_lane(i64 noundef -1) #[[ATTR6]]
+// NVPTX-NEXT:    call void @__gpu_exit() #[[ATTR7:[0-9]+]]
+// NVPTX-NEXT:    unreachable
+//
+//
+// NVPTX-LABEL: define internal i32 @__gpu_num_blocks_x(
+// NVPTX-SAME: ) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.nctaid.x()
+// NVPTX-NEXT:    ret i32 [[TMP0]]
+//
+//
+// NVPTX-LABEL: define internal i32 @__gpu_num_blocks_y(
+// NVPTX-SAME: ) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.nctaid.y()
+// NVPTX-NEXT:    ret i32 [[TMP0]]
+//
+//
+// NVPTX-LABEL: define internal i32 @__gpu_num_blocks_z(
+// NVPTX-SAME: ) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.nctaid.z()
+// NVPTX-NEXT:    ret i32 [[TMP0]]
+//
+//
+// NVPTX-LABEL: define internal i32 @__gpu_num_blocks(
+// NVPTX-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+// NVPTX-NEXT:    [[__DIM_ADDR:%.*]] = alloca i32, align 4
+// NVPTX-NEXT:    store i32 [[__DIM]], ptr [[__DIM_ADDR]], align 4
+// NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR]], align 4
+// NVPTX-NEXT:    switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
+// NVPTX-NEXT:      i32 0, label %[[SW_BB:.*]]
+// NVPTX-NEXT:      i32 1, label %[[SW_BB1:.*]]
+// NVPTX-NEXT:      i32 2, label %[[SW_BB3:.*]]
+// NVPTX-NEXT:    ]
+// NVPTX:       [[SW_BB]]:
+// NVPTX-NEXT:    [[CALL:%.*]] = call i32 @__gpu_num_blocks_x() #[[ATTR6]]
+// NVPTX-NEXT:    store i32 [[CALL]], ptr [[RETVAL]], align 4
+// NVPTX-NEXT:    br label %[[RETURN:.*]]
+// NVPTX:       [[SW_BB1]]:
+// NVPTX-NEXT:    [[CALL2:%.*]] = call i32 @__gpu_num_blocks_y() #[[ATTR6]]
+// NVPTX-NEXT:    store i32 [[CALL2]], ptr [[RETVAL]], align 4
+// NVPTX-NEXT:    br label %[[RETURN]]
+// NVPTX:       [[SW_BB3]]:
+// NVPTX-NEXT:    [[CALL4:%.*]] = call i32 @__gpu_num_blocks_z() #[[ATTR6]]
+// NVPTX-NEXT:    store i32 [[CALL4]], ptr [[RETVAL]], align 4
+// NVPTX-NEXT:    br label %[[RETURN]]
+// NVPTX:       [[SW_DEFAULT]]:
+// NVPTX-NEXT:    unreachable
+// NVPTX:       [[RETURN]]:
+// NVPTX-NEXT:    [[TMP1:%.*]] = load i32, ptr [[RETVAL]], align 4
+// NVPTX-NEXT:    ret i32 [[TMP1]]
+//
+//
+// NVPTX-LABEL: define internal i32 @__gpu_block_id_x(
+// NVPTX-SAME: ) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x()
+// NVPTX-NEXT:    ret i32 [[TMP0]]
+//
+//
+// NVPTX-LABEL: define internal i32 @__gpu_block_id_y(
+// NVPTX-SAME: ) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.y()
+// NVPTX-NEXT:    ret i32 [[TMP0]]
+//
+//
+// NVPTX-LABEL: define internal i32 @__gpu_block_id_z(
+// NVPTX-SAME: ) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.z()
+// NVPTX-NEXT:    ret i32 [[TMP0]]
+//
+//
+// NVPTX-LABEL: define internal i32 @__gpu_block_id(
+// NVPTX-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+// NVPTX-NEXT:    [[__DIM_ADDR:%.*]] = alloca i32, align 4
+// NVPTX-NEXT:    store i32 [[__DIM]], ptr [[__DIM_ADDR]], align 4
+// NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR]], align 4
+// NVPTX-NEXT:    switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
+// NVPTX-NEXT:      i32 0, label %[[SW_BB:.*]]
+// NVPTX-NEXT:      i32 1, label %[[SW_BB1:.*]]
+// NVPTX-NEXT:      i32 2, label %[[SW_BB3:.*]]
+// NVPTX-NEXT:    ]
+// NVPTX:       [[SW_BB]]:
+// NVPTX-NEXT:    [[CALL:%.*]] = call i32 @__gpu_block_id_x() #[[ATTR6]]
+// NVPTX-NEXT:    store i32 [[CALL]], ptr [[RETVAL]], align 4
+// NVPTX-NEXT:    br label %[[RETURN:.*]]
+// NVPTX:       [[SW_BB1]]:
+// NVPTX-NEXT:    [[CALL2:%.*]] = call i32 @__gpu_block_id_y() #[[ATTR6]]
+// NVPTX-NEXT:    store i32 [[CALL2]], ptr [[RETVAL]], align 4
+// NVPTX-NEXT:    br label %[[RETURN]]
+// NVPTX:       [[SW_BB3]]:
+// NVPTX-NEXT:    [[CALL4:%.*]] = call i32 @__gpu_block_id_z() #[[ATTR6]]
+// NVPTX-NEXT:    store i32 [[CALL4]], ptr [[RETVAL]], align 4
+// NVPTX-NEXT:    br label %[[RETURN]]
+// NVPTX:       [[SW_DEFAULT]]:
+// NVPTX-NEXT:    unreachable
+// NVPTX:       [[RETURN]]:
+// NVPTX-NEXT:    [[TMP1:%.*]] = load i32, ptr [[RETVAL]], align 4
+// NVPTX-NEXT:    ret i32 [[TMP1]]
+//
+//
+// NVPTX-LABEL: define internal i32 @__gpu_num_threads_x(
+// NVPTX-SAME: ) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ntid.x()
+// NVPTX-NEXT:    ret i32 [[TMP0]]
+//
+//
+// NVPTX-LABEL: define internal i32 @__gpu_num_threads_y(
+// NVPTX-SAME: ) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ntid.y()
+// NVPTX-NEXT:    ret i32 [[TMP0]]
+//
+//
+// NVPTX-LABEL: define internal i32 @__gpu_num_threads_z(
+// NVPTX-SAME: ) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ntid.z()
+// NVPTX-NEXT:    ret i32 [[TMP0]]
+//
+//
+// NVPTX-LABEL: define internal i32 @__gpu_num_threads(
+// NVPTX-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+// NVPTX-NEXT:    [[__DIM_ADDR:%.*]] = alloca i32, align 4
+// NVPTX-NEXT:    store i32 [[__DIM]], ptr [[__DIM_ADDR]], align 4
+// NVPTX-NEXT:    [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR]], align 4
+// NVPTX-NEXT:    switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
+// NVPTX-NEXT:      i32 0, label %[[SW_BB:.*]]
+// NVPTX-NEXT:      i32 1, label %[[SW_BB1:.*]]
+// NVPTX-NEXT:      i32 2, label %[[SW_BB3:.*]]
+// NVPTX-NEXT:    ]
+// NVPTX:       [[SW_BB]]:
+// NVPTX-NEXT:    [[CALL:%.*]] = call i32 @__gpu_num_threads_x() #[[ATTR6]]
+// NVPTX-NEXT:    store i32 [[CALL]], ptr [[RETVAL]], align 4
+// NVPTX-NEXT:    br label %[[RETURN:.*]]
+// NVPTX:       [[SW_BB1]]:
+// NVPTX-NEXT:    [[CALL2:%.*]] = call i32 @__gpu_num_threads_y() #[[ATTR6]]
+// NVPTX-NEXT:    store i32 [[CALL2]], ptr [[RETVAL]], align 4
+// NVPTX-NEXT:    br label %[[RETURN]]
+// NVPTX:       [[SW_BB3]]:
+// NVPTX-NEXT:    [[CALL4:%.*]] = call i32 @__gpu_num_threads_z() #[[ATTR6]]
+// NVPTX-NEXT:    store i32 [[CALL4]], ptr [[RETVAL]], align 4
+// NVPTX-NEXT:    br label %[[RETURN]]
+// NVPTX:       [[SW_DEFAULT]]:
+// NVPTX-NEXT:    unreachable
+// NVPTX:       [[RETURN]]:
+// NVPTX-NEXT:    [[TMP1:%.*]] = load i32, ptr [[RETVAL]], align 4
+// NVPTX-NEXT:    ret i32 [[TMP1]]
+//
+//
+// NVPTX-LABEL: define internal i32 @__gpu_thread_id_x(
+// NVPTX-SAME: ) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+// NVPTX-NEXT:    ret i32 [[TMP0]]
+//
+//
+// NVPTX-LABEL: define internal i32 @__gpu_thread_id_y(
+// NVPTX-SAME: ) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.y()
+// NVPTX-NEXT:    ret i32 [[TMP0]]
+//
+//
+// NVPTX-LABEL: define internal i32 @__gpu_thread_id_z(
+// NVPTX-SAME: ) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.z()
+// NVPTX-NEXT:    ret i32 [[TMP0]]
+//
+//
 // NVPTX-LABEL: define internal i32 @__gpu_thread_id(
 // NVPTX-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
 // NVPTX-NEXT:  [[ENTRY:.*:]]
@@ -152,18 +742,173 @@ __gpu_kernel void foo() {
 // NVPTX-NEXT:    [[TMP1:%.*]] = load i32, ptr [[RETVAL]], align 4
 // NVPTX-NEXT:    ret i32 [[TMP1]]
 //
-  __gpu_thread_id_y();
-  __gpu_thread_id_z();
-  __gpu_thread_id(0);
-  __gpu_num_lanes();
-  __gpu_lane_id();
-  __gpu_lane_mask();
-  __gpu_read_first_lane_u32(-1, -1);
-  __gpu_ballot(-1, 1);
-  __gpu_sync_threads();
-  __gpu_sync_lane(-1);
-  __gpu_shuffle_idx_u32(-1, -1, -1, 0);
-  __gpu_first_lane_id(-1);
-  __gpu_is_first_in_lane(-1);
-  __gpu_exit();
-}
+//
+// NVPTX-LABEL: define internal i32 @__gpu_num_lanes(
+// NVPTX-SAME: ) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.warpsize()
+// NVPTX-NEXT:    ret i32 [[TMP0]]
+//
+//
+// NVPTX-LABEL: define internal i32 @__gpu_lane_id(
+// NVPTX-SAME: ) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.laneid()
+// NVPTX-NEXT:    ret i32 [[TMP0]]
+//
+//
+// NVPTX-LABEL: define internal i64 @__gpu_lane_mask(
+// NVPTX-SAME: ) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[TMP0:%.*]] = call i32 @llvm.nvvm.activemask()
+// NVPTX-NEXT:    [[CONV:%.*]] = zext i32 [[TMP0]] to i64
+// NVPTX-NEXT:    ret i64 [[CONV]]
+//
+//
+// NVPTX-LABEL: define internal i32 @__gpu_read_first_lane_u32(
+// NVPTX-SAME: i64 noundef [[__LANE_MASK:%.*]], i32 noundef [[__X:%.*]]) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
+// NVPTX-NEXT:    [[__X_ADDR:%.*]] = alloca i32, align 4
+// NVPTX-NEXT:    [[__MASK:%.*]] = alloca i32, align 4
+// NVPTX-NEXT:    [[__ID:%.*]] = alloca i32, align 4
+// NVPTX-NEXT:    store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
+// NVPTX-NEXT:    store i32 [[__X]], ptr [[__X_ADDR]], align 4
+// NVPTX-NEXT:    [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
+// NVPTX-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP0]] to i32
+// NVPTX-NEXT:    store i32 [[CONV]], ptr [[__MASK]], align 4
+// NVPTX-NEXT:    [[TMP1:%.*]] = load i32, ptr [[__MASK]], align 4
+// NVPTX-NEXT:    [[TMP2:%.*]] = call i32 @llvm.cttz.i32(i32 [[TMP1]], i1 true)
+// NVPTX-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 1
+// NVPTX-NEXT:    [[ISZERO:%.*]] = icmp eq i32 [[TMP1]], 0
+// NVPTX-NEXT:    [[FFS:%.*]] = select i1 [[ISZERO]], i32 0, i32 [[TMP3]]
+// NVPTX-NEXT:    [[SUB:%.*]] = sub nsw i32 [[FFS]], 1
+// NVPTX-NEXT:    store i32 [[SUB]], ptr [[__ID]], align 4
+// NVPTX-NEXT:    [[TMP4:%.*]] = load i32, ptr [[__MASK]], align 4
+// NVPTX-NEXT:    [[TMP5:%.*]] = load i32, ptr [[__X_ADDR]], align 4
+// NVPTX-NEXT:    [[TMP6:%.*]] = load i32, ptr [[__ID]], align 4
+// NVPTX-NEXT:    [[CALL:%.*]] = call i32 @__gpu_num_lanes() #[[ATTR6]]
+// NVPTX-NEXT:    [[SUB1:%.*]] = sub i32 [[CALL]], 1
+// NVPTX-NEXT:    [[TMP7:%.*]] = call i32 @llvm.nvvm.shfl.sync.idx.i32(i32 [[TMP4]], i32 [[TMP5]], i32 [[TMP6]], i32 [[SUB1]])
+// NVPTX-NEXT:    ret i32 [[TMP7]]
+//
+//
+// NVPTX-LABEL: define internal i64 @__gpu_ballot(
+// NVPTX-SAME: i64 noundef [[__LANE_MASK:%.*]], i1 noundef zeroext [[__X:%.*]]) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
+// NVPTX-NEXT:    [[__X_ADDR:%.*]] = alloca i8, align 1
+// NVPTX-NEXT:    [[__MASK:%.*]] = alloca i32, align 4
+// NVPTX-NEXT:    store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
+// NVPTX-NEXT:    [[STOREDV:%.*]] = zext i1 [[__X]] to i8
+// NVPTX-NEXT:    store i8 [[STOREDV]], ptr [[__X_ADDR]], align 1
+// NVPTX-NEXT:    [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
+// NVPTX-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP0]] to i32
+// NVPTX-NEXT:    store i32 [[CONV]], ptr [[__MASK]], align 4
+// NVPTX-NEXT:    [[TMP1:%.*]] = load i32, ptr [[__MASK]], align 4
+// NVPTX-NEXT:    [[TMP2:%.*]] = load i8, ptr [[__X_ADDR]], align 1
+// NVPTX-NEXT:    [[LOADEDV:%.*]] = trunc i8 [[TMP2]] to i1
+// NVPTX-NEXT:    [[TMP3:%.*]] = call i32 @llvm.nvvm.vote.ballot.sync(i32 [[TMP1]], i1 [[LOADEDV]])
+// NVPTX-NEXT:    [[CONV1:%.*]] = zext i32 [[TMP3]] to i64
+// NVPTX-NEXT:    ret i64 [[CONV1]]
+//
+//
+// NVPTX-LABEL: define internal void @__gpu_sync_threads(
+// NVPTX-SAME: ) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    call void @llvm.nvvm.barrier0()
+// NVPTX-NEXT:    ret void
+//
+//
+// NVPTX-LABEL: define internal void @__gpu_sync_lane(
+// NVPTX-SAME: i64 noundef [[__LANE_MASK:%.*]]) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
+// NVPTX-NEXT:    store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
+// NVPTX-NEXT:    [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
+// NVPTX-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP0]] to i32
+// NVPTX-NEXT:    call void @llvm.nvvm.bar.warp.sync(i32 [[CONV]])
+// NVPTX-NEXT:    ret void
+//
+//
+// NVPTX-LABEL: define internal i32 @__gpu_shuffle_idx_u32(
+// NVPTX-SAME: i64 noundef [[__LANE_MASK:%.*]], i32 noundef [[__IDX:%.*]], i32 noundef [[__X:%.*]], i32 noundef [[__WIDTH:%.*]]) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
+// NVPTX-NEXT:    [[__IDX_ADDR:%.*]] = alloca i32, align 4
+// NVPTX-NEXT:    [[__X_ADDR:%.*]] = alloca i32, align 4
+// NVPTX-NEXT:    [[__WIDTH_ADDR:%.*]] = alloca i32, align 4
+// NVPTX-NEXT:    [[__MASK:%.*]] = alloca i32, align 4
+// NVPTX-NEXT:    [[__BITMASK:%.*]] = alloca i8, align 1
+// NVPTX-NEXT:    store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
+// NVPTX-NEXT:    store i32 [[__IDX]], ptr [[__IDX_ADDR]], align 4
+// NVPTX-NEXT:    store i32 [[__X]], ptr [[__X_ADDR]], align 4
+// NVPTX-NEXT:    store i32 [[__WIDTH]], ptr [[__WIDTH_ADDR]], align 4
+// NVPTX-NEXT:    [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
+// NVPTX-NEXT:    [[CONV:%.*]] = trunc i64 [[TMP0]] to i32
+// NVPTX-NEXT:    store i32 [[CONV]], ptr [[__MASK]], align 4
+// NVPTX-NEXT:    [[TMP1:%.*]] = load i32, ptr [[__IDX_ADDR]], align 4
+// NVPTX-NEXT:    [[SH_PROM:%.*]] = zext i32 [[TMP1]] to i64
+// NVPTX-NEXT:    [[SHL:%.*]] = shl i64 1, [[SH_PROM]]
+// NVPTX-NEXT:    [[TMP2:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
+// NVPTX-NEXT:    [[AND:%.*]] = and i64 [[SHL]], [[TMP2]]
+// NVPTX-NEXT:    [[TOBOOL:%.*]] = icmp ne i64 [[AND]], 0
+// NVPTX-NEXT:    [[STOREDV:%.*]] = zext i1 [[TOBOOL]] to i8
+// NVPTX-NEXT:    store i8 [[STOREDV]], ptr [[__BITMASK]], align 1
+// NVPTX-NEXT:    [[TMP3:%.*]] = load i8, ptr [[__BITMASK]], align 1
+// NVPTX-NEXT:    [[LOADEDV:%.*]] = trunc i8 [[TMP3]] to i1
+// NVPTX-NEXT:    [[CONV1:%.*]] = zext i1 [[LOADEDV]] to i32
+// NVPTX-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[CONV1]]
+// NVPTX-NEXT:    [[TMP4:%.*]] = load i32, ptr [[__MASK]], align 4
+// NVPTX-NEXT:    [[TMP5:%.*]] = load i32, ptr [[__X_ADDR]], align 4
+// NVPTX-NEXT:    [[TMP6:%.*]] = load i32, ptr [[__IDX_ADDR]], align 4
+// NVPTX-NEXT:    [[CALL:%.*]] = call i32 @__gpu_num_lanes() #[[ATTR6]]
+// NVPTX-NEXT:    [[TMP7:%.*]] = load i32, ptr [[__WIDTH_ADDR]], align 4
+// NVPTX-NEXT:    [[SUB2:%.*]] = sub i32 [[CALL]], [[TMP7]]
+// NVPTX-NEXT:    [[SHL3:%.*]] = shl i32 [[SUB2]], 8
+// NVPTX-NEXT:    [[OR:%.*]] = or i32 [[SHL3]], 31
+// NVPTX-NEXT:    [[TMP8:%.*]] = call i32 @llvm.nvvm.shfl.sync.idx.i32(i32 [[TMP4]], i32 [[TMP5]], i32 [[TMP6]], i32 [[OR]])
+// NVPTX-NEXT:    [[AND4:%.*]] = and i32 [[SUB]], [[TMP8]]
+// NVPTX-NEXT:    ret i32 [[AND4]]
+//
+//
+// NVPTX-LABEL: define internal i64 @__gpu_first_lane_id(
+// NVPTX-SAME: i64 noundef [[__LANE_MASK:%.*]]) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
+// NVPTX-NEXT:    store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
+// NVPTX-NEXT:    [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
+// NVPTX-NEXT:    [[TMP1:%.*]] = call i64 @llvm.cttz.i64(i64 [[TMP0]], i1 true)
+// NVPTX-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 1
+// NVPTX-NEXT:    [[ISZERO:%.*]] = icmp eq i64 [[TMP0]], 0
+// NVPTX-NEXT:    [[FFS:%.*]] = select i1 [[ISZERO]], i64 0, i64 [[TMP2]]
+// NVPTX-NEXT:    [[CAST:%.*]] = trunc i64 [[FFS]] to i32
+// NVPTX-NEXT:    [[SUB:%.*]] = sub nsw i32 [[CAST]], 1
+// NVPTX-NEXT:    [[CONV:%.*]] = sext i32 [[SUB]] to i64
+// NVPTX-NEXT:    ret i64 [[CONV]]
+//
+//
+// NVPTX-LABEL: define internal zeroext i1 @__gpu_is_first_in_lane(
+// NVPTX-SAME: i64 noundef [[__LANE_MASK:%.*]]) #[[ATTR0]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
+// NVPTX-NEXT:    store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
+// NVPTX-NEXT:    [[CALL:%.*]] = call i32 @__gpu_lane_id() #[[ATTR6]]
+// NVPTX-NEXT:    [[CONV:%.*]] = zext i32 [[CALL]] to i64
+// NVPTX-NEXT:    [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
+// NVPTX-NEXT:    [[CALL1:%.*]] = call i64 @__gpu_first_lane_id(i64 noundef [[TMP0]]) #[[ATTR6]]
+// NVPTX-NEXT:    [[CMP:%.*]] = icmp eq i64 [[CONV]], [[CALL1]]
+// NVPTX-NEXT:    ret i1 [[CMP]]
+//
+//
+// NVPTX-LABEL: define internal void @__gpu_exit(
+// NVPTX-SAME: ) #[[ATTR1:[0-9]+]] {
+// NVPTX-NEXT:  [[ENTRY:.*:]]
+// NVPTX-NEXT:    call void @llvm.nvvm.exit()
+// NVPTX-NEXT:    ret void
+//
+//.
+// AMDGPU: [[RNG3]] = !{i32 1, i32 0}
+// AMDGPU: [[META4]] = !{}
+// AMDGPU: [[RNG5]] = !{i16 1, i16 1025}
+//.
diff --git a/clang/test/Misc/integer-literal-printing.cpp b/clang/test/Misc/integer-literal-printing.cpp
index bc52b3f044679..bd231a368fb70 100644
--- a/clang/test/Misc/integer-literal-printing.cpp
+++ b/clang/test/Misc/integer-literal-printing.cpp
@@ -14,7 +14,6 @@ enum class boolTy : bool {
 template <boolTy T> struct Type3Helper;
 template <> struct Type3Helper<boolTy::b> { typedef boolTy Ty; };
 template <boolTy T, typename Type3Helper<T>::Ty U> struct Type3 {};
-// expected-note@-1 {{template parameter is declared here}}
 
 // PR14386
 enum class charTy : char {
@@ -24,7 +23,6 @@ enum class charTy : char {
 template <charTy T> struct Type4Helper;
 template <> struct Type4Helper<charTy::c> { typedef charTy Ty; };
 template <charTy T, typename Type4Helper<T>::Ty U> struct Type4 {};
-// expected-note@-1 {{template parameter is declared here}}
 
 enum class scharTy : signed char {
   c = 0,
@@ -33,7 +31,6 @@ enum class scharTy : signed char {
 template <scharTy T> struct Type5Helper;
 template <> struct Type5Helper<scharTy::c> { typedef scharTy Ty; };
 template <scharTy T, typename Type5Helper<T>::Ty U> struct Type5 {};
-// expected-note@-1 {{template parameter is declared here}}
 
 enum class ucharTy : unsigned char {
   c = 0,
@@ -42,7 +39,6 @@ enum class ucharTy : unsigned char {
 template <ucharTy T> struct Type6Helper;
 template <> struct Type6Helper<ucharTy::c> { typedef ucharTy Ty; };
 template <ucharTy T, typename Type6Helper<T>::Ty U> struct Type6 {};
-// expected-note@-1 {{template parameter is declared here}}
 
 enum class wcharTy : wchar_t {
   c = 0,
@@ -51,7 +47,6 @@ enum class wcharTy : wchar_t {
 template <wcharTy T> struct Type7Helper;
 template <> struct Type7Helper<wcharTy::c> { typedef wcharTy Ty; };
 template <wcharTy T, typename Type7Helper<T>::Ty U> struct Type7 {};
-// expected-note@-1 {{template parameter is declared here}}
 
 enum class char16Ty : char16_t {
   c = 0,
@@ -60,7 +55,6 @@ enum class char16Ty : char16_t {
 template <char16Ty T> struct Type8Helper;
 template <> struct Type8Helper<char16Ty::c> { typedef char16Ty Ty; };
 template <char16Ty T, typename Type8Helper<T>::Ty U> struct Type8 {};
-// expected-note@-1 {{template parameter is declared here}}
 
 enum class char32Ty : char16_t {
   c = 0,
@@ -69,7 +63,6 @@ enum class char32Ty : char16_t {
 template <char32Ty T> struct Type9Helper;
 template <> struct Type9Helper<char32Ty::c> { typedef char32Ty Ty; };
 template <char32Ty T, typename Type9Helper<T>::Ty U> struct Type9 {};
-// expected-note@-1 {{template parameter is declared here}}
 
 void Function() {
   Function1(Type1<-42>()); // expected-error{{no matching function for call to 'Function1'}}
diff --git a/clang/test/Modules/malformed-constraint-template-non-type-parm-decl.cpp b/clang/test/Modules/malformed-constraint-template-non-type-parm-decl.cpp
index 250b54ade0ecb..73dff88e506b4 100644
--- a/clang/test/Modules/malformed-constraint-template-non-type-parm-decl.cpp
+++ b/clang/test/Modules/malformed-constraint-template-non-type-parm-decl.cpp
@@ -11,7 +11,7 @@
 //--- mod.cppm
 export module mod;
 
-template <typename T, auto Q> // expected-note 2{{template parameter is declared here}}
+template <typename T, auto Q>
 concept ReferenceOf = Q;
 
 // expected-error@+2 {{unknown type name 'AngleIsInvalidNow'}}
diff --git a/clang/test/Modules/missing-body-in-import.cpp b/clang/test/Modules/missing-body-in-import.cpp
index e25f7b5921301..b52ebba15087a 100644
--- a/clang/test/Modules/missing-body-in-import.cpp
+++ b/clang/test/Modules/missing-body-in-import.cpp
@@ -29,7 +29,6 @@ export module mod2;
 import mod1;
 
 struct C: B <A{"a", "b"}> { // expected-error {{non-type template argument is not a constant expression}}
-                            // expected-note@mod1.cppm:11 {{template parameter is declared here}}
   constexpr C(int a) { }
 };
 
diff --git a/clang/test/Modules/modules-merge-enum.m b/clang/test/Modules/modules-merge-enum.m
new file mode 100644
index 0000000000000..f1010c1decc24
--- /dev/null
+++ b/clang/test/Modules/modules-merge-enum.m
@@ -0,0 +1,111 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+
+
+// Expect no crash
+// RUN: %clang_cc1 -fmodules -fmodules-cache-path=%t/modcache -fmodule-map-file=%t/module.modulemap %t/source.m
+
+// Add -ast-dump-all to check that the AST nodes are merged correctly.
+// RUN: %clang_cc1 -fmodules -fmodules-cache-path=%t/modcache -fmodule-map-file=%t/module.modulemap %t/source.m -ast-dump-all 2>&1 | FileCheck %s
+
+
+//--- shared.h
+// This header is shared between two modules, but it's not a module itself.
+// The enums defined here are parsed in both modules, and merged while building ModB.
+
+typedef enum MyEnum1 { MyVal_A } MyEnum1;
+// CHECK:      |-EnumDecl 0x{{.*}} imported in ModA.ModAFile1 <undeserialized declarations> MyEnum1
+// CHECK-NEXT: | |-also in ModB
+// CHECK-NEXT: | `-EnumConstantDecl 0x{{.*}} imported in ModA.ModAFile1 referenced MyVal_A 'int'
+// CHECK-NEXT: |-TypedefDecl 0x{{.*}} imported in ModA.ModAFile1 hidden MyEnum1 'enum MyEnum1'
+// CHECK-NEXT: | `-ElaboratedType 0x{{.*}} 'enum MyEnum1' sugar imported
+// CHECK-NEXT: |   `-EnumType 0x{{.*}} 'enum MyEnum1' imported
+// CHECK-NEXT: |     `-Enum 0x{{.*}} 'MyEnum1'
+
+
+enum MyEnum2 { MyVal_B };
+// CHECK:      |-EnumDecl 0x{{.*}} imported in ModA.ModAFile1 <undeserialized declarations> MyEnum2
+// CHECK-NEXT: | |-also in ModB
+// CHECK-NEXT: | `-EnumConstantDecl 0x{{.*}} imported in ModA.ModAFile1 referenced MyVal_B 'int'
+
+
+typedef enum { MyVal_C } MyEnum3;
+// CHECK:      |-EnumDecl 0x{{.*}} imported in ModA.ModAFile1 <undeserialized declarations>
+// CHECK-NEXT: | |-also in ModB
+// CHECK-NEXT: | `-EnumConstantDecl 0x{{.*}} imported in ModA.ModAFile1 referenced MyVal_C 'int'
+// CHECK-NEXT: |-TypedefDecl 0x{{.*}} imported in ModA.ModAFile1 hidden MyEnum3 'enum MyEnum3':'MyEnum3'
+// CHECK-NEXT: | `-ElaboratedType 0x{{.*}} 'enum MyEnum3' sugar imported
+// CHECK-NEXT: |   `-EnumType 0x{{.*}} 'MyEnum3' imported
+// CHECK-NEXT: |     `-Enum 0x{{.*}}
+
+struct MyStruct {
+  enum MyEnum5 { MyVal_D } Field;
+};
+
+// CHECK:      |-RecordDecl 0x{{.*}} imported in ModA.ModAFile1 <undeserialized declarations> struct MyStruct definition
+// CHECK-NEXT: | |-also in ModB
+// CHECK-NEXT: | |-EnumDecl 0x{{.*}} imported in ModA.ModAFile1 <undeserialized declarations> MyEnum5
+// CHECK-NEXT: | | |-also in ModB
+// CHECK-NEXT: | | `-EnumConstantDecl 0x{{.*}} imported in ModA.ModAFile1 referenced MyVal_D 'int'
+// CHECK-NEXT: | `-FieldDecl 0x{{.*}} imported in ModA.ModAFile1 hidden Field 'enum MyEnum5'
+
+// In this case, no merging happens on the EnumDecl in Objective-C, and ASTWriter writes both EnumConstantDecls when building ModB.
+enum { MyVal_E };
+// CHECK:      |-EnumDecl 0x{{.*}} imported in ModA.ModAFile1 hidden <undeserialized declarations>
+// CHECK-NEXT: | `-EnumConstantDecl 0x{{.*}} imported in ModA.ModAFile1 hidden MyVal_E 'int'
+
+
+// Redeclarations coming from ModB.
+// CHECK:      |-TypedefDecl 0x{{.*}} prev 0x{{.*}} imported in ModB MyEnum1 'enum MyEnum1'
+// CHECK-NEXT: | `-ElaboratedType 0x{{.*}} 'enum MyEnum1' sugar imported
+// CHECK-NEXT: |   `-EnumType 0x{{.*}} 'enum MyEnum1' imported
+// CHECK-NEXT: |     `-Enum 0x{{.*}} 'MyEnum1'
+
+// CHECK:      |-EnumDecl 0x{{.*}} prev 0x{{.*}} imported in ModB <undeserialized declarations>
+// CHECK-NEXT: | |-also in ModB
+// CHECK-NEXT: | `-EnumConstantDecl 0x{{.*}} imported in ModB MyVal_C 'int'
+// CHECK-NEXT: |-TypedefDecl 0x{{.*}} prev 0x{{.*}} imported in ModB MyEnum3 'enum MyEnum3':'MyEnum3'
+// CHECK-NEXT: | `-ElaboratedType 0x{{.*}} 'enum MyEnum3' sugar imported
+// CHECK-NEXT: |   `-EnumType 0x{{.*}} 'MyEnum3' imported
+// CHECK-NEXT: |     `-Enum 0x{{.*}}
+
+// CHECK:      |-EnumDecl 0x{{.*}} imported in ModB <undeserialized declarations>
+// CHECK-NEXT: | `-EnumConstantDecl 0x{{.*}} first 0x{{.*}} imported in ModB referenced MyVal_E 'int'
+
+
+
+//--- module.modulemap
+module ModA {
+  module ModAFile1 {
+    header "ModAFile1.h"
+    export *
+  }
+  module ModAFile2 {
+    header "ModAFile2.h"
+    export *
+  }
+}
+// The goal of writing ModB is to test that ASTWriter can handle the merged AST nodes correctly.
+// For example, a stale declaration in IdResolver can cause an assertion failure while writing the identifier table.
+module ModB {
+  header "ModBFile.h"
+  export *
+}
+
+//--- ModAFile1.h
+#include "shared.h"
+
+//--- ModAFile2.h
+// Including this file, triggers loading of the module ModA with nodes coming ModAFile1.h being hidden.
+
+//--- ModBFile.h
+// ModBFile depends on ModAFile2.h only.
+#include "ModAFile2.h"
+// Including shared.h here causes Sema to merge the AST nodes from shared.h with the hidden ones from ModA.
+#include "shared.h"
+
+
+//--- source.m
+#include "ModBFile.h"
+
+int main() { return MyVal_A + MyVal_B + MyVal_C + MyVal_D + MyVal_E; }
diff --git a/clang/test/Modules/pr125999.cppm b/clang/test/Modules/pr125999.cppm
new file mode 100644
index 0000000000000..a859ce7592604
--- /dev/null
+++ b/clang/test/Modules/pr125999.cppm
@@ -0,0 +1,33 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/foo.cppm -verify -fsyntax-only
+
+//--- bar.h
+template <typename T>
+struct Singleton {
+    static T* instance_;
+    static T* get() {
+        static bool init = false;
+        if (!init) {
+            init = true;
+            instance_ = ::new T();
+        }
+        return instance_;
+    }
+};
+
+template <typename T>
+T* Singleton<T>::instance_ = nullptr;
+
+struct s{};
+inline void* foo() {
+    return Singleton<s>::get();
+}
+
+//--- foo.cppm
+// expected-no-diagnostics
+module;
+#include "bar.h"
+export module foo;
diff --git a/clang/test/Modules/template-default-args.cpp b/clang/test/Modules/template-default-args.cpp
index 1d8de709fd598..85b2a18d9e506 100644
--- a/clang/test/Modules/template-default-args.cpp
+++ b/clang/test/Modules/template-default-args.cpp
@@ -22,7 +22,7 @@ template<typename T = int> struct B;
 template<typename T = int> struct C;
 template<typename T> struct D {};
 template<typename T> struct F {};
-template<typename T> struct G {}; // #G
+template<typename T> struct G {};
 template<typename T> struct J {};
 template<typename T = int> struct J;
 struct K : J<> {};
@@ -39,10 +39,8 @@ E<> e;
 F<> f;
 G<> g; // expected-error {{missing '#include "a.h"'; default argument of 'G' must be defined before it is used}}
 // expected-note@a.h:7 {{default argument declared here is not reachable}}
-// expected-note@#G {{template parameter is declared here}}
 H<> h; // expected-error {{missing '#include "a.h"'; default argument of 'H' must be defined before it is used}}
 // expected-note@a.h:8 {{default argument declared here is not reachable}}
-// expected-note@a.h:8 {{template parameter is declared here}}
 I<> i;
 L<> *l;
 END
diff --git a/clang/test/OpenMP/cancel_codegen.cpp b/clang/test/OpenMP/cancel_codegen.cpp
index 7a95c0138b3a1..16e7542a8e826 100644
--- a/clang/test/OpenMP/cancel_codegen.cpp
+++ b/clang/test/OpenMP/cancel_codegen.cpp
@@ -728,9 +728,7 @@ for (int i = 0; i < argc; ++i) {
 // CHECK3-NEXT:    store ptr [[ARGV_ADDR]], ptr [[GEP_ARGV_ADDR]], align 8
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 1, ptr @main..omp_par, ptr [[STRUCTARG]])
 // CHECK3-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-// CHECK3:       omp.par.outlined.exit:
-// CHECK3-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// CHECK3:       omp.par.exit.split:
+// CHECK3:       omp.par.exit:
 // CHECK3-NEXT:    br label [[OMP_SECTION_LOOP_PREHEADER:%.*]]
 // CHECK3:       omp_section_loop.preheader:
 // CHECK3-NEXT:    store i32 0, ptr [[P_LOWERBOUND]], align 4
@@ -998,7 +996,7 @@ for (int i = 0; i < argc; ++i) {
 // CHECK3-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT_EXITSTUB]]
 // CHECK3:       .split:
 // CHECK3-NEXT:    br label [[TMP4]]
-// CHECK3:       omp.par.outlined.exit.exitStub:
+// CHECK3:       omp.par.exit.exitStub:
 // CHECK3-NEXT:    ret void
 //
 //
diff --git a/clang/test/OpenMP/irbuilder_nested_openmp_parallel_empty.c b/clang/test/OpenMP/irbuilder_nested_openmp_parallel_empty.c
index 7bdb9749757cc..96962f71c709d 100644
--- a/clang/test/OpenMP/irbuilder_nested_openmp_parallel_empty.c
+++ b/clang/test/OpenMP/irbuilder_nested_openmp_parallel_empty.c
@@ -16,10 +16,10 @@
 // ALL-NEXT:    br label [[OMP_PARALLEL:%.*]]
 // ALL:       omp_parallel:
 // ALL-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 0, ptr @_Z17nested_parallel_0v..omp_par.1)
-// ALL-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT12:%.*]]
-// ALL:       omp.par.outlined.exit12:
+// ALL-NEXT:    br label [[OMP_PAR_EXIT:%.*]]
+// ALL:       omp.par.exit7:
 // ALL-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// ALL:       omp.par.exit.split:
+// ALL:       omp.par.exit.exitStub:
 // ALL-NEXT:    ret void
 //
 void nested_parallel_0(void) {
@@ -50,10 +50,8 @@ void nested_parallel_0(void) {
 // ALL-NEXT:    [[GEP_R_ADDR17:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG14]], i32 0, i32 2
 // ALL-NEXT:    store ptr [[R_ADDR]], ptr [[GEP_R_ADDR17]], align 8
 // ALL-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 1, ptr @_Z17nested_parallel_1Pfid..omp_par.2, ptr [[STRUCTARG14]])
-// ALL-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT13:%.*]]
-// ALL:       omp.par.outlined.exit13:
-// ALL-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// ALL:       omp.par.exit.split:
+// ALL-NEXT:    br label [[OMP_PAR_EXIT:%.*]]
+// ALL:       omp.par.exit:
 // ALL-NEXT:    ret void
 //
 void nested_parallel_1(float *r, int a, double b) {
@@ -85,10 +83,8 @@ void nested_parallel_1(float *r, int a, double b) {
 // ALL-NEXT:    [[GEP_R_ADDR:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 2
 // ALL-NEXT:    store ptr [[R_ADDR]], ptr [[GEP_R_ADDR]], align 8
 // ALL-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 1, ptr @_Z17nested_parallel_2Pfid..omp_par.5, ptr [[STRUCTARG]])
-// ALL-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT55:%.*]]
-// ALL:       omp.par.outlined.exit55:
-// ALL-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// ALL:       omp.par.exit.split:
+// ALL-NEXT:    br label [[OMP_PAR_EXIT:%.*]]
+// ALL:       omp.par.exit:
 // ALL-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // ALL-NEXT:    [[CONV56:%.*]] = sitofp i32 [[TMP0]] to double
 // ALL-NEXT:    [[TMP1:%.*]] = load double, ptr [[B_ADDR]], align 8
diff --git a/clang/test/OpenMP/irbuilder_nested_parallel_for.c b/clang/test/OpenMP/irbuilder_nested_parallel_for.c
index 68727e0503ede..ae3570fda412d 100644
--- a/clang/test/OpenMP/irbuilder_nested_parallel_for.c
+++ b/clang/test/OpenMP/irbuilder_nested_parallel_for.c
@@ -78,10 +78,8 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-NEXT:    br label [[OMP_PARALLEL:%.*]]
 // CHECK:       omp_parallel:
 // CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 0, ptr @_Z14parallel_for_0v..omp_par)
-// CHECK-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-// CHECK:       omp.par.outlined.exit:
-// CHECK-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// CHECK:       omp.par.exit.split:
+// CHECK-NEXT:    br label [[OMP_PAR_EXIT:%.*]]
+// CHECK:       omp.par.exit:
 // CHECK-NEXT:    ret void
 //
 //
@@ -147,7 +145,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK:       omp_loop.inc:
 // CHECK-NEXT:    [[OMP_LOOP_NEXT]] = add nuw i32 [[OMP_LOOP_IV]], 1
 // CHECK-NEXT:    br label [[OMP_LOOP_HEADER]]
-// CHECK:       omp.par.outlined.exit.exitStub:
+// CHECK:       omp.par.exit.exitStub:
 // CHECK-NEXT:    ret void
 //
 //
@@ -231,10 +229,8 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-NEXT:    [[GEP_R_ADDR20:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG17]], i32 0, i32 2
 // CHECK-NEXT:    store ptr [[R_ADDR]], ptr [[GEP_R_ADDR20]], align 8
 // CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 1, ptr @_Z14parallel_for_1Pfid..omp_par.4, ptr [[STRUCTARG17]])
-// CHECK-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT16:%.*]]
-// CHECK:       omp.par.outlined.exit16:
-// CHECK-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// CHECK:       omp.par.exit.split:
+// CHECK-NEXT:    br label [[OMP_PAR_EXIT:%.*]]
+// CHECK:       omp.par.exit:
 // CHECK-NEXT:    ret void
 //
 //
@@ -264,16 +260,14 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-NEXT:    [[GEP_R_ADDR3:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 2
 // CHECK-NEXT:    store ptr [[LOADGEP_R_ADDR]], ptr [[GEP_R_ADDR3]], align 8
 // CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 1, ptr @_Z14parallel_for_1Pfid..omp_par, ptr [[STRUCTARG]])
-// CHECK-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-// CHECK:       omp.par.outlined.exit:
-// CHECK-NEXT:    br label [[OMP_PAR_EXIT7_SPLIT:%.*]]
-// CHECK:       omp.par.exit7.split:
+// CHECK-NEXT:    br label [[OMP_PAR_EXIT:%.*]]
+// CHECK:       omp.par.exit7:
 // CHECK-NEXT:    br label [[OMP_PAR_REGION_PARALLEL_AFTER:%.*]]
 // CHECK:       omp.par.region.parallel.after:
 // CHECK-NEXT:    br label [[OMP_PAR_PRE_FINALIZE:%.*]]
 // CHECK:       omp.par.pre_finalize:
-// CHECK-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT16_EXITSTUB:%.*]]
-// CHECK:       omp.par.outlined.exit16.exitStub:
+// CHECK-NEXT:    br label [[OMP_PAR_EXIT16_EXITSTUB:%.*]]
+// CHECK:       omp.par.exit.exitStub:
 // CHECK-NEXT:    ret void
 //
 //
@@ -352,7 +346,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK:       omp_loop.inc:
 // CHECK-NEXT:    [[OMP_LOOP_NEXT]] = add nuw i32 [[OMP_LOOP_IV]], 1
 // CHECK-NEXT:    br label [[OMP_LOOP_HEADER]]
-// CHECK:       omp.par.outlined.exit.exitStub:
+// CHECK:       omp.par.exit7.exitStub:
 // CHECK-NEXT:    ret void
 //
 //
@@ -444,10 +438,8 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-NEXT:    [[GEP_R_ADDR:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 2
 // CHECK-NEXT:    store ptr [[R_ADDR]], ptr [[GEP_R_ADDR]], align 8
 // CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 1, ptr @_Z14parallel_for_2Pfid..omp_par.23, ptr [[STRUCTARG]])
-// CHECK-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT184:%.*]]
-// CHECK:       omp.par.outlined.exit184:
-// CHECK-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// CHECK:       omp.par.exit.split:
+// CHECK-NEXT:    br label [[OMP_PAR_EXIT184:%.*]]
+// CHECK:       omp.par.exit:
 // CHECK-NEXT:    store i32 0, ptr [[I185]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[STRUCT_ANON_17]], ptr [[AGG_CAPTURED186]], i32 0, i32 0
 // CHECK-NEXT:    store ptr [[I185]], ptr [[TMP0]], align 8
@@ -457,7 +449,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-NEXT:    call void @__captured_stmt.19(ptr [[DOTCOUNT_ADDR188]], ptr [[AGG_CAPTURED186]])
 // CHECK-NEXT:    [[DOTCOUNT189:%.*]] = load i32, ptr [[DOTCOUNT_ADDR188]], align 4
 // CHECK-NEXT:    br label [[OMP_LOOP_PREHEADER190:%.*]]
-// CHECK:       omp_loop.preheader190:
+// CHECK:       omp_loop.preheader187:
 // CHECK-NEXT:    store i32 0, ptr [[P_LOWERBOUND204]], align 4
 // CHECK-NEXT:    [[TMP3:%.*]] = sub i32 [[DOTCOUNT189]], 1
 // CHECK-NEXT:    store i32 [[TMP3]], ptr [[P_UPPERBOUND205]], align 4
@@ -469,13 +461,13 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-NEXT:    [[TMP6:%.*]] = sub i32 [[TMP5]], [[TMP4]]
 // CHECK-NEXT:    [[TMP7:%.*]] = add i32 [[TMP6]], 1
 // CHECK-NEXT:    br label [[OMP_LOOP_HEADER191:%.*]]
-// CHECK:       omp_loop.header191:
+// CHECK:       omp_loop.header188:
 // CHECK-NEXT:    [[OMP_LOOP_IV197:%.*]] = phi i32 [ 0, [[OMP_LOOP_PREHEADER190]] ], [ [[OMP_LOOP_NEXT199:%.*]], [[OMP_LOOP_INC194:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_LOOP_COND192:%.*]]
-// CHECK:       omp_loop.cond192:
+// CHECK:       omp_loop.cond189:
 // CHECK-NEXT:    [[OMP_LOOP_CMP198:%.*]] = icmp ult i32 [[OMP_LOOP_IV197]], [[TMP7]]
 // CHECK-NEXT:    br i1 [[OMP_LOOP_CMP198]], label [[OMP_LOOP_BODY193:%.*]], label [[OMP_LOOP_EXIT195:%.*]]
-// CHECK:       omp_loop.body193:
+// CHECK:       omp_loop.body190:
 // CHECK-NEXT:    [[TMP8:%.*]] = add i32 [[OMP_LOOP_IV197]], [[TMP4]]
 // CHECK-NEXT:    call void @__captured_stmt.20(ptr [[I185]], i32 [[TMP8]], ptr [[AGG_CAPTURED187]])
 // CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[A_ADDR]], align 4
@@ -486,15 +478,15 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[R_ADDR]], align 8
 // CHECK-NEXT:    store float [[CONV202]], ptr [[TMP11]], align 4
 // CHECK-NEXT:    br label [[OMP_LOOP_INC194]]
-// CHECK:       omp_loop.inc194:
+// CHECK:       omp_loop.inc191:
 // CHECK-NEXT:    [[OMP_LOOP_NEXT199]] = add nuw i32 [[OMP_LOOP_IV197]], 1
 // CHECK-NEXT:    br label [[OMP_LOOP_HEADER191]]
-// CHECK:       omp_loop.exit195:
+// CHECK:       omp_loop.exit192:
 // CHECK-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[OMP_GLOBAL_THREAD_NUM207]])
 // CHECK-NEXT:    [[OMP_GLOBAL_THREAD_NUM208:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
 // CHECK-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM208]])
 // CHECK-NEXT:    br label [[OMP_LOOP_AFTER196:%.*]]
-// CHECK:       omp_loop.after196:
+// CHECK:       omp_loop.after193:
 // CHECK-NEXT:    ret void
 //
 //
@@ -573,10 +565,8 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-NEXT:    [[GEP_R_ADDR217:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG214]], i32 0, i32 2
 // CHECK-NEXT:    store ptr [[LOADGEP_R_ADDR]], ptr [[GEP_R_ADDR217]], align 8
 // CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 1, ptr @_Z14parallel_for_2Pfid..omp_par.22, ptr [[STRUCTARG214]])
-// CHECK-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT159:%.*]]
-// CHECK:       omp.par.outlined.exit159:
-// CHECK-NEXT:    br label [[OMP_PAR_EXIT11_SPLIT:%.*]]
-// CHECK:       omp.par.exit11.split:
+// CHECK-NEXT:    br label [[OMP_PAR_EXIT159:%.*]]
+// CHECK:       omp.par.exit11:
 // CHECK-NEXT:    store i32 0, ptr [[I160]], align 4
 // CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [[STRUCT_ANON_15]], ptr [[AGG_CAPTURED161]], i32 0, i32 0
 // CHECK-NEXT:    store ptr [[I160]], ptr [[TMP10]], align 8
@@ -586,7 +576,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-NEXT:    call void @__captured_stmt.17(ptr [[DOTCOUNT_ADDR163]], ptr [[AGG_CAPTURED161]])
 // CHECK-NEXT:    [[DOTCOUNT164:%.*]] = load i32, ptr [[DOTCOUNT_ADDR163]], align 4
 // CHECK-NEXT:    br label [[OMP_LOOP_PREHEADER165:%.*]]
-// CHECK:       omp_loop.preheader165:
+// CHECK:       omp_loop.preheader163:
 // CHECK-NEXT:    store i32 0, ptr [[P_LOWERBOUND179]], align 4
 // CHECK-NEXT:    [[TMP13:%.*]] = sub i32 [[DOTCOUNT164]], 1
 // CHECK-NEXT:    store i32 [[TMP13]], ptr [[P_UPPERBOUND180]], align 4
@@ -598,24 +588,24 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], [[TMP14]]
 // CHECK-NEXT:    [[TMP17:%.*]] = add i32 [[TMP16]], 1
 // CHECK-NEXT:    br label [[OMP_LOOP_HEADER166:%.*]]
-// CHECK:       omp_loop.header166:
+// CHECK:       omp_loop.header164:
 // CHECK-NEXT:    [[OMP_LOOP_IV172:%.*]] = phi i32 [ 0, [[OMP_LOOP_PREHEADER165]] ], [ [[OMP_LOOP_NEXT174:%.*]], [[OMP_LOOP_INC169:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_LOOP_COND167:%.*]]
-// CHECK:       omp_loop.cond167:
+// CHECK:       omp_loop.cond165:
 // CHECK-NEXT:    [[OMP_LOOP_CMP173:%.*]] = icmp ult i32 [[OMP_LOOP_IV172]], [[TMP17]]
 // CHECK-NEXT:    br i1 [[OMP_LOOP_CMP173]], label [[OMP_LOOP_BODY168:%.*]], label [[OMP_LOOP_EXIT170:%.*]]
-// CHECK:       omp_loop.exit170:
+// CHECK:       omp_loop.exit168:
 // CHECK-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[OMP_GLOBAL_THREAD_NUM182]])
 // CHECK-NEXT:    [[OMP_GLOBAL_THREAD_NUM183:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
 // CHECK-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM183]])
 // CHECK-NEXT:    br label [[OMP_LOOP_AFTER171:%.*]]
-// CHECK:       omp_loop.after171:
+// CHECK:       omp_loop.after169:
 // CHECK-NEXT:    br label [[OMP_PAR_REGION_PARALLEL_AFTER:%.*]]
 // CHECK:       omp.par.region.parallel.after:
 // CHECK-NEXT:    br label [[OMP_PAR_PRE_FINALIZE:%.*]]
 // CHECK:       omp.par.pre_finalize:
 // CHECK-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT184_EXITSTUB:%.*]]
-// CHECK:       omp_loop.body168:
+// CHECK:       omp_loop.body166:
 // CHECK-NEXT:    [[TMP18:%.*]] = add i32 [[OMP_LOOP_IV172]], [[TMP14]]
 // CHECK-NEXT:    call void @__captured_stmt.18(ptr [[I160]], i32 [[TMP18]], ptr [[AGG_CAPTURED162]])
 // CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[LOADGEP_A_ADDR]], align 4
@@ -626,7 +616,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[LOADGEP_R_ADDR]], align 8
 // CHECK-NEXT:    store float [[CONV177]], ptr [[TMP21]], align 4
 // CHECK-NEXT:    br label [[OMP_LOOP_INC169]]
-// CHECK:       omp_loop.inc169:
+// CHECK:       omp_loop.inc167:
 // CHECK-NEXT:    [[OMP_LOOP_NEXT174]] = add nuw i32 [[OMP_LOOP_IV172]], 1
 // CHECK-NEXT:    br label [[OMP_LOOP_HEADER166]]
 // CHECK:       omp_loop.body:
@@ -643,7 +633,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK:       omp_loop.inc:
 // CHECK-NEXT:    [[OMP_LOOP_NEXT]] = add nuw i32 [[OMP_LOOP_IV]], 1
 // CHECK-NEXT:    br label [[OMP_LOOP_HEADER]]
-// CHECK:       omp.par.outlined.exit184.exitStub:
+// CHECK:       omp.par.exit.exitStub:
 // CHECK-NEXT:    ret void
 //
 //
@@ -731,10 +721,8 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-NEXT:    [[GEP_R_ADDR3:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 2
 // CHECK-NEXT:    store ptr [[LOADGEP_R_ADDR]], ptr [[GEP_R_ADDR3]], align 8
 // CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 1, ptr @_Z14parallel_for_2Pfid..omp_par, ptr [[STRUCTARG]])
-// CHECK-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-// CHECK:       omp.par.outlined.exit:
-// CHECK-NEXT:    br label [[OMP_PAR_EXIT46_SPLIT:%.*]]
-// CHECK:       omp.par.exit46.split:
+// CHECK-NEXT:    br label [[OMP_PAR_EXIT:%.*]]
+// CHECK:       omp.par.exit46:
 // CHECK-NEXT:    store i32 0, ptr [[I75]], align 4
 // CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [[STRUCT_ANON_9]], ptr [[AGG_CAPTURED76]], i32 0, i32 0
 // CHECK-NEXT:    store ptr [[I75]], ptr [[TMP10]], align 8
@@ -770,7 +758,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK:       omp_loop.after86:
 // CHECK-NEXT:    [[OMP_GLOBAL_THREAD_NUM99:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
 // CHECK-NEXT:    br label [[OMP_PARALLEL213:%.*]]
-// CHECK:       omp_parallel213:
+// CHECK:       omp_parallel210:
 // CHECK-NEXT:    [[GEP_A_ADDR210:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG209]], i32 0, i32 0
 // CHECK-NEXT:    store ptr [[LOADGEP_A_ADDR]], ptr [[GEP_A_ADDR210]], align 8
 // CHECK-NEXT:    [[GEP_B_ADDR211:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG209]], i32 0, i32 1
@@ -779,9 +767,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-NEXT:    store ptr [[LOADGEP_R_ADDR]], ptr [[GEP_R_ADDR212]], align 8
 // CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 1, ptr @_Z14parallel_for_2Pfid..omp_par.21, ptr [[STRUCTARG209]])
 // CHECK-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT134:%.*]]
-// CHECK:       omp.par.outlined.exit134:
-// CHECK-NEXT:    br label [[OMP_PAR_EXIT105_SPLIT:%.*]]
-// CHECK:       omp.par.exit105.split:
+// CHECK:       omp.par.exit105:
 // CHECK-NEXT:    store i32 0, ptr [[I135]], align 4
 // CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw [[STRUCT_ANON_13]], ptr [[AGG_CAPTURED136]], i32 0, i32 0
 // CHECK-NEXT:    store ptr [[I135]], ptr [[TMP18]], align 8
@@ -791,7 +777,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-NEXT:    call void @__captured_stmt.15(ptr [[DOTCOUNT_ADDR138]], ptr [[AGG_CAPTURED136]])
 // CHECK-NEXT:    [[DOTCOUNT139:%.*]] = load i32, ptr [[DOTCOUNT_ADDR138]], align 4
 // CHECK-NEXT:    br label [[OMP_LOOP_PREHEADER140:%.*]]
-// CHECK:       omp_loop.preheader140:
+// CHECK:       omp_loop.preheader139:
 // CHECK-NEXT:    store i32 0, ptr [[P_LOWERBOUND154]], align 4
 // CHECK-NEXT:    [[TMP21:%.*]] = sub i32 [[DOTCOUNT139]], 1
 // CHECK-NEXT:    store i32 [[TMP21]], ptr [[P_UPPERBOUND155]], align 4
@@ -803,24 +789,24 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP23]], [[TMP22]]
 // CHECK-NEXT:    [[TMP25:%.*]] = add i32 [[TMP24]], 1
 // CHECK-NEXT:    br label [[OMP_LOOP_HEADER141:%.*]]
-// CHECK:       omp_loop.header141:
+// CHECK:       omp_loop.header140:
 // CHECK-NEXT:    [[OMP_LOOP_IV147:%.*]] = phi i32 [ 0, [[OMP_LOOP_PREHEADER140]] ], [ [[OMP_LOOP_NEXT149:%.*]], [[OMP_LOOP_INC144:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_LOOP_COND142:%.*]]
-// CHECK:       omp_loop.cond142:
+// CHECK:       omp_loop.cond141:
 // CHECK-NEXT:    [[OMP_LOOP_CMP148:%.*]] = icmp ult i32 [[OMP_LOOP_IV147]], [[TMP25]]
 // CHECK-NEXT:    br i1 [[OMP_LOOP_CMP148]], label [[OMP_LOOP_BODY143:%.*]], label [[OMP_LOOP_EXIT145:%.*]]
-// CHECK:       omp_loop.exit145:
+// CHECK:       omp_loop.exit144:
 // CHECK-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[OMP_GLOBAL_THREAD_NUM157]])
 // CHECK-NEXT:    [[OMP_GLOBAL_THREAD_NUM158:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
 // CHECK-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM158]])
 // CHECK-NEXT:    br label [[OMP_LOOP_AFTER146:%.*]]
-// CHECK:       omp_loop.after146:
+// CHECK:       omp_loop.after145:
 // CHECK-NEXT:    br label [[OMP_PAR_REGION9_PARALLEL_AFTER:%.*]]
 // CHECK:       omp.par.region9.parallel.after:
 // CHECK-NEXT:    br label [[OMP_PAR_PRE_FINALIZE10:%.*]]
 // CHECK:       omp.par.pre_finalize10:
 // CHECK-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT159_EXITSTUB:%.*]]
-// CHECK:       omp_loop.body143:
+// CHECK:       omp_loop.body142:
 // CHECK-NEXT:    [[TMP26:%.*]] = add i32 [[OMP_LOOP_IV147]], [[TMP22]]
 // CHECK-NEXT:    call void @__captured_stmt.16(ptr [[I135]], i32 [[TMP26]], ptr [[AGG_CAPTURED137]])
 // CHECK-NEXT:    [[TMP27:%.*]] = load i32, ptr [[LOADGEP_A_ADDR]], align 4
@@ -831,7 +817,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-NEXT:    [[TMP29:%.*]] = load ptr, ptr [[LOADGEP_R_ADDR]], align 8
 // CHECK-NEXT:    store float [[CONV152]], ptr [[TMP29]], align 4
 // CHECK-NEXT:    br label [[OMP_LOOP_INC144]]
-// CHECK:       omp_loop.inc144:
+// CHECK:       omp_loop.inc143:
 // CHECK-NEXT:    [[OMP_LOOP_NEXT149]] = add nuw i32 [[OMP_LOOP_IV147]], 1
 // CHECK-NEXT:    br label [[OMP_LOOP_HEADER141]]
 // CHECK:       omp_loop.body83:
@@ -862,7 +848,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK:       omp_loop.inc25:
 // CHECK-NEXT:    [[OMP_LOOP_NEXT30]] = add nuw i32 [[OMP_LOOP_IV28]], 1
 // CHECK-NEXT:    br label [[OMP_LOOP_HEADER22]]
-// CHECK:       omp.par.outlined.exit159.exitStub:
+// CHECK:       omp.par.exit11.exitStub:
 // CHECK-NEXT:    ret void
 //
 //
@@ -941,7 +927,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK:       omp_loop.inc119:
 // CHECK-NEXT:    [[OMP_LOOP_NEXT124]] = add nuw i32 [[OMP_LOOP_IV122]], 1
 // CHECK-NEXT:    br label [[OMP_LOOP_HEADER116]]
-// CHECK:       omp.par.outlined.exit134.exitStub:
+// CHECK:       omp.par.exit105.exitStub:
 // CHECK-NEXT:    ret void
 //
 //
@@ -1020,7 +1006,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK:       omp_loop.inc60:
 // CHECK-NEXT:    [[OMP_LOOP_NEXT65]] = add nuw i32 [[OMP_LOOP_IV63]], 1
 // CHECK-NEXT:    br label [[OMP_LOOP_HEADER57]]
-// CHECK:       omp.par.outlined.exit.exitStub:
+// CHECK:       omp.par.exit46.exitStub:
 // CHECK-NEXT:    ret void
 //
 //
@@ -1512,9 +1498,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG:       omp_parallel:
 // CHECK-DEBUG-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 0, ptr @_Z14parallel_for_0v..omp_par), !dbg [[DBG14:![0-9]+]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-// CHECK-DEBUG:       omp.par.outlined.exit:
-// CHECK-DEBUG-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// CHECK-DEBUG:       omp.par.exit.split:
+// CHECK-DEBUG:       omp.par.exit:
 // CHECK-DEBUG-NEXT:    ret void, !dbg [[DBG18:![0-9]+]]
 //
 //
@@ -1581,7 +1565,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG:       omp_loop.inc:
 // CHECK-DEBUG-NEXT:    [[OMP_LOOP_NEXT]] = add nuw i32 [[OMP_LOOP_IV]], 1, !dbg [[DBG27]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_HEADER]], !dbg [[DBG27]]
-// CHECK-DEBUG:       omp.par.outlined.exit.exitStub:
+// CHECK-DEBUG:       omp.par.exit.exitStub:
 // CHECK-DEBUG-NEXT:    ret void
 //
 //
@@ -1677,9 +1661,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG-NEXT:    store ptr [[R_ADDR]], ptr [[GEP_R_ADDR20]], align 8
 // CHECK-DEBUG-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB6]], i32 1, ptr @_Z14parallel_for_1Pfid..omp_par.4, ptr [[STRUCTARG17]]), !dbg [[DBG82:![0-9]+]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT16:%.*]]
-// CHECK-DEBUG:       omp.par.outlined.exit16:
-// CHECK-DEBUG-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// CHECK-DEBUG:       omp.par.exit.split:
+// CHECK-DEBUG:       omp.par.exit:
 // CHECK-DEBUG-NEXT:    ret void, !dbg [[DBG84:![0-9]+]]
 //
 //
@@ -1710,15 +1692,13 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG-NEXT:    store ptr [[LOADGEP_R_ADDR]], ptr [[GEP_R_ADDR3]], align 8
 // CHECK-DEBUG-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB8]], i32 1, ptr @_Z14parallel_for_1Pfid..omp_par, ptr [[STRUCTARG]]), !dbg [[DBG88:![0-9]+]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-// CHECK-DEBUG:       omp.par.outlined.exit:
-// CHECK-DEBUG-NEXT:    br label [[OMP_PAR_EXIT7_SPLIT:%.*]]
-// CHECK-DEBUG:       omp.par.exit7.split:
+// CHECK-DEBUG:       omp.par.exit7:
 // CHECK-DEBUG-NEXT:    br label [[OMP_PAR_REGION_PARALLEL_AFTER:%.*]], !dbg [[DBG92:![0-9]+]]
 // CHECK-DEBUG:       omp.par.region.parallel.after:
 // CHECK-DEBUG-NEXT:    br label [[OMP_PAR_PRE_FINALIZE:%.*]]
 // CHECK-DEBUG:       omp.par.pre_finalize:
 // CHECK-DEBUG-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT16_EXITSTUB:%.*]], !dbg [[DBG92]]
-// CHECK-DEBUG:       omp.par.outlined.exit16.exitStub:
+// CHECK-DEBUG:       omp.par.exit.exitStub:
 // CHECK-DEBUG-NEXT:    ret void
 //
 //
@@ -1798,7 +1778,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG:       omp_loop.inc:
 // CHECK-DEBUG-NEXT:    [[OMP_LOOP_NEXT]] = add nuw i32 [[OMP_LOOP_IV]], 1, !dbg [[DBG100]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_HEADER]], !dbg [[DBG100]]
-// CHECK-DEBUG:       omp.par.outlined.exit.exitStub:
+// CHECK-DEBUG:       omp.par.exit7.exitStub:
 // CHECK-DEBUG-NEXT:    ret void
 //
 //
@@ -1901,10 +1881,8 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG-NEXT:    [[GEP_R_ADDR:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 2
 // CHECK-DEBUG-NEXT:    store ptr [[R_ADDR]], ptr [[GEP_R_ADDR]], align 8
 // CHECK-DEBUG-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB13]], i32 1, ptr @_Z14parallel_for_2Pfid..omp_par.23, ptr [[STRUCTARG]]), !dbg [[DBG140:![0-9]+]]
-// CHECK-DEBUG-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT184:%.*]]
-// CHECK-DEBUG:       omp.par.outlined.exit184:
-// CHECK-DEBUG-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// CHECK-DEBUG:       omp.par.exit.split:
+// CHECK-DEBUG-NEXT:    br label [[OMP_PAR_EXIT184:%.*]]
+// CHECK-DEBUG:       omp.par.exit:
 // CHECK-DEBUG-NEXT:      #dbg_declare(ptr [[I185]], [[META144:![0-9]+]], !DIExpression(), [[META147:![0-9]+]])
 // CHECK-DEBUG-NEXT:    store i32 0, ptr [[I185]], align 4, !dbg [[META147]]
 // CHECK-DEBUG-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[STRUCT_ANON_17]], ptr [[AGG_CAPTURED186]], i32 0, i32 0, !dbg [[DBG148:![0-9]+]]
@@ -1915,7 +1893,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG-NEXT:    call void @__captured_stmt.19(ptr [[DOTCOUNT_ADDR188]], ptr [[AGG_CAPTURED186]]), !dbg [[DBG148]]
 // CHECK-DEBUG-NEXT:    [[DOTCOUNT189:%.*]] = load i32, ptr [[DOTCOUNT_ADDR188]], align 4, !dbg [[DBG148]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_PREHEADER190:%.*]], !dbg [[DBG148]]
-// CHECK-DEBUG:       omp_loop.preheader190:
+// CHECK-DEBUG:       omp_loop.preheader187:
 // CHECK-DEBUG-NEXT:    store i32 0, ptr [[P_LOWERBOUND204]], align 4, !dbg [[DBG148]]
 // CHECK-DEBUG-NEXT:    [[TMP3:%.*]] = sub i32 [[DOTCOUNT189]], 1, !dbg [[DBG148]]
 // CHECK-DEBUG-NEXT:    store i32 [[TMP3]], ptr [[P_UPPERBOUND205]], align 4, !dbg [[DBG148]]
@@ -1927,13 +1905,13 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG-NEXT:    [[TMP6:%.*]] = sub i32 [[TMP5]], [[TMP4]], !dbg [[DBG148]]
 // CHECK-DEBUG-NEXT:    [[TMP7:%.*]] = add i32 [[TMP6]], 1, !dbg [[DBG148]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_HEADER191:%.*]], !dbg [[DBG148]]
-// CHECK-DEBUG:       omp_loop.header191:
+// CHECK-DEBUG:       omp_loop.header188:
 // CHECK-DEBUG-NEXT:    [[OMP_LOOP_IV197:%.*]] = phi i32 [ 0, [[OMP_LOOP_PREHEADER190]] ], [ [[OMP_LOOP_NEXT199:%.*]], [[OMP_LOOP_INC194:%.*]] ], !dbg [[DBG148]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_COND192:%.*]], !dbg [[DBG148]]
-// CHECK-DEBUG:       omp_loop.cond192:
+// CHECK-DEBUG:       omp_loop.cond189:
 // CHECK-DEBUG-NEXT:    [[OMP_LOOP_CMP198:%.*]] = icmp ult i32 [[OMP_LOOP_IV197]], [[TMP7]], !dbg [[DBG148]]
 // CHECK-DEBUG-NEXT:    br i1 [[OMP_LOOP_CMP198]], label [[OMP_LOOP_BODY193:%.*]], label [[OMP_LOOP_EXIT195:%.*]], !dbg [[DBG148]]
-// CHECK-DEBUG:       omp_loop.body193:
+// CHECK-DEBUG:       omp_loop.body190:
 // CHECK-DEBUG-NEXT:    [[TMP8:%.*]] = add i32 [[OMP_LOOP_IV197]], [[TMP4]], !dbg [[DBG150:![0-9]+]]
 // CHECK-DEBUG-NEXT:    call void @__captured_stmt.20(ptr [[I185]], i32 [[TMP8]], ptr [[AGG_CAPTURED187]]), !dbg [[DBG148]]
 // CHECK-DEBUG-NEXT:    [[TMP9:%.*]] = load i32, ptr [[A_ADDR]], align 4, !dbg [[DBG151:![0-9]+]]
@@ -1944,15 +1922,15 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[R_ADDR]], align 8, !dbg [[DBG153:![0-9]+]]
 // CHECK-DEBUG-NEXT:    store float [[CONV202]], ptr [[TMP11]], align 4, !dbg [[DBG154:![0-9]+]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_INC194]], !dbg [[DBG148]]
-// CHECK-DEBUG:       omp_loop.inc194:
+// CHECK-DEBUG:       omp_loop.inc191:
 // CHECK-DEBUG-NEXT:    [[OMP_LOOP_NEXT199]] = add nuw i32 [[OMP_LOOP_IV197]], 1, !dbg [[DBG148]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_HEADER191]], !dbg [[DBG148]]
-// CHECK-DEBUG:       omp_loop.exit195:
+// CHECK-DEBUG:       omp_loop.exit192:
 // CHECK-DEBUG-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB42]], i32 [[OMP_GLOBAL_THREAD_NUM207]]), !dbg [[DBG148]]
 // CHECK-DEBUG-NEXT:    [[OMP_GLOBAL_THREAD_NUM208:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB42]]), !dbg [[DBG150]]
 // CHECK-DEBUG-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB43:[0-9]+]], i32 [[OMP_GLOBAL_THREAD_NUM208]]), !dbg [[DBG150]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_AFTER196:%.*]], !dbg [[DBG148]]
-// CHECK-DEBUG:       omp_loop.after196:
+// CHECK-DEBUG:       omp_loop.after193:
 // CHECK-DEBUG-NEXT:    ret void, !dbg [[DBG155:![0-9]+]]
 //
 //
@@ -2033,9 +2011,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG-NEXT:    store ptr [[LOADGEP_R_ADDR]], ptr [[GEP_R_ADDR217]], align 8
 // CHECK-DEBUG-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB18]], i32 1, ptr @_Z14parallel_for_2Pfid..omp_par.22, ptr [[STRUCTARG214]]), !dbg [[DBG166:![0-9]+]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT159:%.*]]
-// CHECK-DEBUG:       omp.par.outlined.exit159:
-// CHECK-DEBUG-NEXT:    br label [[OMP_PAR_EXIT11_SPLIT:%.*]]
-// CHECK-DEBUG:       omp.par.exit11.split:
+// CHECK-DEBUG:       omp.par.exit11:
 // CHECK-DEBUG-NEXT:      #dbg_declare(ptr [[I160]], [[META170:![0-9]+]], !DIExpression(), [[META173:![0-9]+]])
 // CHECK-DEBUG-NEXT:    store i32 0, ptr [[I160]], align 4, !dbg [[META173]]
 // CHECK-DEBUG-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [[STRUCT_ANON_15]], ptr [[AGG_CAPTURED161]], i32 0, i32 0, !dbg [[DBG174:![0-9]+]]
@@ -2046,7 +2022,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG-NEXT:    call void @__captured_stmt.17(ptr [[DOTCOUNT_ADDR163]], ptr [[AGG_CAPTURED161]]), !dbg [[DBG174]]
 // CHECK-DEBUG-NEXT:    [[DOTCOUNT164:%.*]] = load i32, ptr [[DOTCOUNT_ADDR163]], align 4, !dbg [[DBG174]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_PREHEADER165:%.*]], !dbg [[DBG174]]
-// CHECK-DEBUG:       omp_loop.preheader165:
+// CHECK-DEBUG:       omp_loop.preheader163:
 // CHECK-DEBUG-NEXT:    store i32 0, ptr [[P_LOWERBOUND179]], align 4, !dbg [[DBG174]]
 // CHECK-DEBUG-NEXT:    [[TMP13:%.*]] = sub i32 [[DOTCOUNT164]], 1, !dbg [[DBG174]]
 // CHECK-DEBUG-NEXT:    store i32 [[TMP13]], ptr [[P_UPPERBOUND180]], align 4, !dbg [[DBG174]]
@@ -2058,24 +2034,24 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], [[TMP14]], !dbg [[DBG174]]
 // CHECK-DEBUG-NEXT:    [[TMP17:%.*]] = add i32 [[TMP16]], 1, !dbg [[DBG174]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_HEADER166:%.*]], !dbg [[DBG174]]
-// CHECK-DEBUG:       omp_loop.header166:
+// CHECK-DEBUG:       omp_loop.header164:
 // CHECK-DEBUG-NEXT:    [[OMP_LOOP_IV172:%.*]] = phi i32 [ 0, [[OMP_LOOP_PREHEADER165]] ], [ [[OMP_LOOP_NEXT174:%.*]], [[OMP_LOOP_INC169:%.*]] ], !dbg [[DBG174]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_COND167:%.*]], !dbg [[DBG174]]
-// CHECK-DEBUG:       omp_loop.cond167:
+// CHECK-DEBUG:       omp_loop.cond165:
 // CHECK-DEBUG-NEXT:    [[OMP_LOOP_CMP173:%.*]] = icmp ult i32 [[OMP_LOOP_IV172]], [[TMP17]], !dbg [[DBG174]]
 // CHECK-DEBUG-NEXT:    br i1 [[OMP_LOOP_CMP173]], label [[OMP_LOOP_BODY168:%.*]], label [[OMP_LOOP_EXIT170:%.*]], !dbg [[DBG174]]
-// CHECK-DEBUG:       omp_loop.exit170:
+// CHECK-DEBUG:       omp_loop.exit168:
 // CHECK-DEBUG-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB39]], i32 [[OMP_GLOBAL_THREAD_NUM182]]), !dbg [[DBG174]]
 // CHECK-DEBUG-NEXT:    [[OMP_GLOBAL_THREAD_NUM183:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB39]]), !dbg [[DBG176:![0-9]+]]
 // CHECK-DEBUG-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB40:[0-9]+]], i32 [[OMP_GLOBAL_THREAD_NUM183]]), !dbg [[DBG176]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_AFTER171:%.*]], !dbg [[DBG174]]
-// CHECK-DEBUG:       omp_loop.after171:
+// CHECK-DEBUG:       omp_loop.after169:
 // CHECK-DEBUG-NEXT:    br label [[OMP_PAR_REGION_PARALLEL_AFTER:%.*]], !dbg [[DBG177:![0-9]+]]
 // CHECK-DEBUG:       omp.par.region.parallel.after:
 // CHECK-DEBUG-NEXT:    br label [[OMP_PAR_PRE_FINALIZE:%.*]]
 // CHECK-DEBUG:       omp.par.pre_finalize:
 // CHECK-DEBUG-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT184_EXITSTUB:%.*]], !dbg [[DBG177]]
-// CHECK-DEBUG:       omp_loop.body168:
+// CHECK-DEBUG:       omp_loop.body166:
 // CHECK-DEBUG-NEXT:    [[TMP18:%.*]] = add i32 [[OMP_LOOP_IV172]], [[TMP14]], !dbg [[DBG176]]
 // CHECK-DEBUG-NEXT:    call void @__captured_stmt.18(ptr [[I160]], i32 [[TMP18]], ptr [[AGG_CAPTURED162]]), !dbg [[DBG174]]
 // CHECK-DEBUG-NEXT:    [[TMP19:%.*]] = load i32, ptr [[LOADGEP_A_ADDR]], align 4, !dbg [[DBG178:![0-9]+]]
@@ -2086,7 +2062,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[LOADGEP_R_ADDR]], align 8, !dbg [[DBG180:![0-9]+]]
 // CHECK-DEBUG-NEXT:    store float [[CONV177]], ptr [[TMP21]], align 4, !dbg [[DBG181:![0-9]+]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_INC169]], !dbg [[DBG174]]
-// CHECK-DEBUG:       omp_loop.inc169:
+// CHECK-DEBUG:       omp_loop.inc167:
 // CHECK-DEBUG-NEXT:    [[OMP_LOOP_NEXT174]] = add nuw i32 [[OMP_LOOP_IV172]], 1, !dbg [[DBG174]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_HEADER166]], !dbg [[DBG174]]
 // CHECK-DEBUG:       omp_loop.body:
@@ -2103,7 +2079,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG:       omp_loop.inc:
 // CHECK-DEBUG-NEXT:    [[OMP_LOOP_NEXT]] = add nuw i32 [[OMP_LOOP_IV]], 1, !dbg [[DBG162]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_HEADER]], !dbg [[DBG162]]
-// CHECK-DEBUG:       omp.par.outlined.exit184.exitStub:
+// CHECK-DEBUG:       omp.par.exit.exitStub:
 // CHECK-DEBUG-NEXT:    ret void
 //
 //
@@ -2193,9 +2169,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG-NEXT:    store ptr [[LOADGEP_R_ADDR]], ptr [[GEP_R_ADDR3]], align 8
 // CHECK-DEBUG-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB23]], i32 1, ptr @_Z14parallel_for_2Pfid..omp_par, ptr [[STRUCTARG]]), !dbg [[DBG197:![0-9]+]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-// CHECK-DEBUG:       omp.par.outlined.exit:
-// CHECK-DEBUG-NEXT:    br label [[OMP_PAR_EXIT46_SPLIT:%.*]]
-// CHECK-DEBUG:       omp.par.exit46.split:
+// CHECK-DEBUG:       omp.par.exit46:
 // CHECK-DEBUG-NEXT:      #dbg_declare(ptr [[I75]], [[META201:![0-9]+]], !DIExpression(), [[META204:![0-9]+]])
 // CHECK-DEBUG-NEXT:    store i32 0, ptr [[I75]], align 4, !dbg [[META204]]
 // CHECK-DEBUG-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [[STRUCT_ANON_9]], ptr [[AGG_CAPTURED76]], i32 0, i32 0, !dbg [[DBG205:![0-9]+]]
@@ -2232,7 +2206,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG:       omp_loop.after86:
 // CHECK-DEBUG-NEXT:    [[OMP_GLOBAL_THREAD_NUM99:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB31:[0-9]+]]), !dbg [[DBG208:![0-9]+]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_PARALLEL213:%.*]]
-// CHECK-DEBUG:       omp_parallel213:
+// CHECK-DEBUG:       omp_parallel210:
 // CHECK-DEBUG-NEXT:    [[GEP_A_ADDR210:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG209]], i32 0, i32 0
 // CHECK-DEBUG-NEXT:    store ptr [[LOADGEP_A_ADDR]], ptr [[GEP_A_ADDR210]], align 8
 // CHECK-DEBUG-NEXT:    [[GEP_B_ADDR211:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG209]], i32 0, i32 1
@@ -2241,9 +2215,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG-NEXT:    store ptr [[LOADGEP_R_ADDR]], ptr [[GEP_R_ADDR212]], align 8
 // CHECK-DEBUG-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB31]], i32 1, ptr @_Z14parallel_for_2Pfid..omp_par.21, ptr [[STRUCTARG209]]), !dbg [[DBG209:![0-9]+]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT134:%.*]]
-// CHECK-DEBUG:       omp.par.outlined.exit134:
-// CHECK-DEBUG-NEXT:    br label [[OMP_PAR_EXIT105_SPLIT:%.*]]
-// CHECK-DEBUG:       omp.par.exit105.split:
+// CHECK-DEBUG:       omp.par.exit105:
 // CHECK-DEBUG-NEXT:      #dbg_declare(ptr [[I135]], [[META213:![0-9]+]], !DIExpression(), [[META216:![0-9]+]])
 // CHECK-DEBUG-NEXT:    store i32 0, ptr [[I135]], align 4, !dbg [[META216]]
 // CHECK-DEBUG-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw [[STRUCT_ANON_13]], ptr [[AGG_CAPTURED136]], i32 0, i32 0, !dbg [[DBG217:![0-9]+]]
@@ -2254,7 +2226,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG-NEXT:    call void @__captured_stmt.15(ptr [[DOTCOUNT_ADDR138]], ptr [[AGG_CAPTURED136]]), !dbg [[DBG217]]
 // CHECK-DEBUG-NEXT:    [[DOTCOUNT139:%.*]] = load i32, ptr [[DOTCOUNT_ADDR138]], align 4, !dbg [[DBG217]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_PREHEADER140:%.*]], !dbg [[DBG217]]
-// CHECK-DEBUG:       omp_loop.preheader140:
+// CHECK-DEBUG:       omp_loop.preheader139:
 // CHECK-DEBUG-NEXT:    store i32 0, ptr [[P_LOWERBOUND154]], align 4, !dbg [[DBG217]]
 // CHECK-DEBUG-NEXT:    [[TMP21:%.*]] = sub i32 [[DOTCOUNT139]], 1, !dbg [[DBG217]]
 // CHECK-DEBUG-NEXT:    store i32 [[TMP21]], ptr [[P_UPPERBOUND155]], align 4, !dbg [[DBG217]]
@@ -2266,24 +2238,24 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP23]], [[TMP22]], !dbg [[DBG217]]
 // CHECK-DEBUG-NEXT:    [[TMP25:%.*]] = add i32 [[TMP24]], 1, !dbg [[DBG217]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_HEADER141:%.*]], !dbg [[DBG217]]
-// CHECK-DEBUG:       omp_loop.header141:
+// CHECK-DEBUG:       omp_loop.header140:
 // CHECK-DEBUG-NEXT:    [[OMP_LOOP_IV147:%.*]] = phi i32 [ 0, [[OMP_LOOP_PREHEADER140]] ], [ [[OMP_LOOP_NEXT149:%.*]], [[OMP_LOOP_INC144:%.*]] ], !dbg [[DBG217]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_COND142:%.*]], !dbg [[DBG217]]
-// CHECK-DEBUG:       omp_loop.cond142:
+// CHECK-DEBUG:       omp_loop.cond141:
 // CHECK-DEBUG-NEXT:    [[OMP_LOOP_CMP148:%.*]] = icmp ult i32 [[OMP_LOOP_IV147]], [[TMP25]], !dbg [[DBG217]]
 // CHECK-DEBUG-NEXT:    br i1 [[OMP_LOOP_CMP148]], label [[OMP_LOOP_BODY143:%.*]], label [[OMP_LOOP_EXIT145:%.*]], !dbg [[DBG217]]
-// CHECK-DEBUG:       omp_loop.exit145:
+// CHECK-DEBUG:       omp_loop.exit144:
 // CHECK-DEBUG-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB36]], i32 [[OMP_GLOBAL_THREAD_NUM157]]), !dbg [[DBG217]]
 // CHECK-DEBUG-NEXT:    [[OMP_GLOBAL_THREAD_NUM158:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB36]]), !dbg [[DBG219:![0-9]+]]
 // CHECK-DEBUG-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB37:[0-9]+]], i32 [[OMP_GLOBAL_THREAD_NUM158]]), !dbg [[DBG219]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_AFTER146:%.*]], !dbg [[DBG217]]
-// CHECK-DEBUG:       omp_loop.after146:
+// CHECK-DEBUG:       omp_loop.after145:
 // CHECK-DEBUG-NEXT:    br label [[OMP_PAR_REGION9_PARALLEL_AFTER:%.*]], !dbg [[DBG220:![0-9]+]]
 // CHECK-DEBUG:       omp.par.region9.parallel.after:
 // CHECK-DEBUG-NEXT:    br label [[OMP_PAR_PRE_FINALIZE10:%.*]]
 // CHECK-DEBUG:       omp.par.pre_finalize10:
 // CHECK-DEBUG-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT159_EXITSTUB:%.*]], !dbg [[DBG220]]
-// CHECK-DEBUG:       omp_loop.body143:
+// CHECK-DEBUG:       omp_loop.body142:
 // CHECK-DEBUG-NEXT:    [[TMP26:%.*]] = add i32 [[OMP_LOOP_IV147]], [[TMP22]], !dbg [[DBG219]]
 // CHECK-DEBUG-NEXT:    call void @__captured_stmt.16(ptr [[I135]], i32 [[TMP26]], ptr [[AGG_CAPTURED137]]), !dbg [[DBG217]]
 // CHECK-DEBUG-NEXT:    [[TMP27:%.*]] = load i32, ptr [[LOADGEP_A_ADDR]], align 4, !dbg [[DBG221:![0-9]+]]
@@ -2294,7 +2266,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG-NEXT:    [[TMP29:%.*]] = load ptr, ptr [[LOADGEP_R_ADDR]], align 8, !dbg [[DBG223:![0-9]+]]
 // CHECK-DEBUG-NEXT:    store float [[CONV152]], ptr [[TMP29]], align 4, !dbg [[DBG224:![0-9]+]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_INC144]], !dbg [[DBG217]]
-// CHECK-DEBUG:       omp_loop.inc144:
+// CHECK-DEBUG:       omp_loop.inc143:
 // CHECK-DEBUG-NEXT:    [[OMP_LOOP_NEXT149]] = add nuw i32 [[OMP_LOOP_IV147]], 1, !dbg [[DBG217]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_HEADER141]], !dbg [[DBG217]]
 // CHECK-DEBUG:       omp_loop.body83:
@@ -2325,7 +2297,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG:       omp_loop.inc25:
 // CHECK-DEBUG-NEXT:    [[OMP_LOOP_NEXT30]] = add nuw i32 [[OMP_LOOP_IV28]], 1, !dbg [[DBG193]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_HEADER22]], !dbg [[DBG193]]
-// CHECK-DEBUG:       omp.par.outlined.exit159.exitStub:
+// CHECK-DEBUG:       omp.par.exit11.exitStub:
 // CHECK-DEBUG-NEXT:    ret void
 //
 //
@@ -2405,7 +2377,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG:       omp_loop.inc119:
 // CHECK-DEBUG-NEXT:    [[OMP_LOOP_NEXT124]] = add nuw i32 [[OMP_LOOP_IV122]], 1, !dbg [[DBG241]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_HEADER116]], !dbg [[DBG241]]
-// CHECK-DEBUG:       omp.par.outlined.exit134.exitStub:
+// CHECK-DEBUG:       omp.par.exit105.exitStub:
 // CHECK-DEBUG-NEXT:    ret void
 //
 //
@@ -2485,7 +2457,7 @@ void parallel_for_2(float *r, int a, double b) {
 // CHECK-DEBUG:       omp_loop.inc60:
 // CHECK-DEBUG-NEXT:    [[OMP_LOOP_NEXT65]] = add nuw i32 [[OMP_LOOP_IV63]], 1, !dbg [[DBG257]]
 // CHECK-DEBUG-NEXT:    br label [[OMP_LOOP_HEADER57]], !dbg [[DBG257]]
-// CHECK-DEBUG:       omp.par.outlined.exit.exitStub:
+// CHECK-DEBUG:       omp.par.exit46.exitStub:
 // CHECK-DEBUG-NEXT:    ret void
 //
 //
diff --git a/clang/test/OpenMP/nested_loop_codegen.cpp b/clang/test/OpenMP/nested_loop_codegen.cpp
index a08950e2e55c7..d8fab26bf1e7f 100644
--- a/clang/test/OpenMP/nested_loop_codegen.cpp
+++ b/clang/test/OpenMP/nested_loop_codegen.cpp
@@ -530,10 +530,8 @@ int inline_decl() {
 // CHECK3-NEXT:    [[GEP_K:%.*]] = getelementptr { ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 1
 // CHECK3-NEXT:    store ptr [[K]], ptr [[GEP_K]], align 8
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 1, ptr @_Z12outline_declv..omp_par, ptr [[STRUCTARG]])
-// CHECK3-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-// CHECK3:       omp.par.outlined.exit:
-// CHECK3-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// CHECK3:       omp.par.exit.split:
+// CHECK3-NEXT:    br label [[OMP_PAR_EXIT:%.*]]
+// CHECK3:       omp.par.exit:
 // CHECK3-NEXT:    [[TMP0:%.*]] = load i32, ptr [[K]], align 4
 // CHECK3-NEXT:    ret i32 [[TMP0]]
 //
@@ -620,7 +618,7 @@ int inline_decl() {
 // CHECK3:       omp_loop.inc:
 // CHECK3-NEXT:    [[OMP_LOOP_NEXT]] = add nuw i32 [[OMP_LOOP_IV]], 1
 // CHECK3-NEXT:    br label [[OMP_LOOP_HEADER]]
-// CHECK3:       omp.par.outlined.exit.exitStub:
+// CHECK3:       omp.par.exit.exitStub:
 // CHECK3-NEXT:    ret void
 //
 //
@@ -699,9 +697,7 @@ int inline_decl() {
 // CHECK3-NEXT:    store ptr [[RES]], ptr [[GEP_RES]], align 8
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 1, ptr @_Z11inline_declv..omp_par, ptr [[STRUCTARG]])
 // CHECK3-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-// CHECK3:       omp.par.outlined.exit:
-// CHECK3-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// CHECK3:       omp.par.exit.split:
+// CHECK3:       omp.par.exit:
 // CHECK3-NEXT:    [[TMP0:%.*]] = load i32, ptr [[RES]], align 4
 // CHECK3-NEXT:    ret i32 [[TMP0]]
 //
@@ -789,7 +785,7 @@ int inline_decl() {
 // CHECK3:       omp_loop.inc:
 // CHECK3-NEXT:    [[OMP_LOOP_NEXT]] = add nuw i32 [[OMP_LOOP_IV]], 1
 // CHECK3-NEXT:    br label [[OMP_LOOP_HEADER]]
-// CHECK3:       omp.par.outlined.exit.exitStub:
+// CHECK3:       omp.par.exit.exitStub:
 // CHECK3-NEXT:    ret void
 //
 //
@@ -870,9 +866,7 @@ int inline_decl() {
 // CHECK4-NEXT:    store ptr [[K]], ptr [[GEP_K]], align 8
 // CHECK4-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 1, ptr @_Z12outline_declv..omp_par, ptr [[STRUCTARG]]), !dbg [[DBG18:![0-9]+]]
 // CHECK4-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-// CHECK4:       omp.par.outlined.exit:
-// CHECK4-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// CHECK4:       omp.par.exit.split:
+// CHECK4:       omp.par.exit:
 // CHECK4-NEXT:    [[TMP0:%.*]] = load i32, ptr [[K]], align 4, !dbg [[DBG20:![0-9]+]]
 // CHECK4-NEXT:    ret i32 [[TMP0]], !dbg [[DBG20]]
 //
@@ -959,7 +953,7 @@ int inline_decl() {
 // CHECK4:       omp_loop.inc:
 // CHECK4-NEXT:    [[OMP_LOOP_NEXT]] = add nuw i32 [[OMP_LOOP_IV]], 1, !dbg [[DBG28]]
 // CHECK4-NEXT:    br label [[OMP_LOOP_HEADER]], !dbg [[DBG28]]
-// CHECK4:       omp.par.outlined.exit.exitStub:
+// CHECK4:       omp.par.exit.exitStub:
 // CHECK4-NEXT:    ret void
 //
 //
@@ -1048,9 +1042,7 @@ int inline_decl() {
 // CHECK4-NEXT:    store ptr [[RES]], ptr [[GEP_RES]], align 8
 // CHECK4-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB6]], i32 1, ptr @_Z11inline_declv..omp_par, ptr [[STRUCTARG]]), !dbg [[DBG82:![0-9]+]]
 // CHECK4-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-// CHECK4:       omp.par.outlined.exit:
-// CHECK4-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// CHECK4:       omp.par.exit.split:
+// CHECK4:       omp.par.exit:
 // CHECK4-NEXT:    [[TMP0:%.*]] = load i32, ptr [[RES]], align 4, !dbg [[DBG84:![0-9]+]]
 // CHECK4-NEXT:    ret i32 [[TMP0]], !dbg [[DBG84]]
 //
@@ -1139,7 +1131,7 @@ int inline_decl() {
 // CHECK4:       omp_loop.inc:
 // CHECK4-NEXT:    [[OMP_LOOP_NEXT]] = add nuw i32 [[OMP_LOOP_IV]], 1, !dbg [[META95]]
 // CHECK4-NEXT:    br label [[OMP_LOOP_HEADER]], !dbg [[META95]]
-// CHECK4:       omp.par.outlined.exit.exitStub:
+// CHECK4:       omp.par.exit.exitStub:
 // CHECK4-NEXT:    ret void
 //
 //
diff --git a/clang/test/OpenMP/parallel_codegen.cpp b/clang/test/OpenMP/parallel_codegen.cpp
index 2a0a881b109be..c63c6f554f4ae 100644
--- a/clang/test/OpenMP/parallel_codegen.cpp
+++ b/clang/test/OpenMP/parallel_codegen.cpp
@@ -736,9 +736,7 @@ int main (int argc, char **argv) {
 // CHECK3-NEXT:    store ptr [[VLA]], ptr [[GEP_VLA]], align 8
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 1, ptr @main..omp_par, ptr [[STRUCTARG]])
 // CHECK3-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-// CHECK3:       omp.par.outlined.exit:
-// CHECK3-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// CHECK3:       omp.par.exit.split:
+// CHECK3:       omp.par.exit:
 // CHECK3-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[ARGV_ADDR]], align 8
 // CHECK3-NEXT:    [[CALL:%.*]] = call noundef i32 @_Z5tmainIPPcEiT_(ptr noundef [[TMP3]])
 // CHECK3-NEXT:    store i32 [[CALL]], ptr [[RETVAL]], align 4
@@ -770,7 +768,7 @@ int main (int argc, char **argv) {
 // CHECK3-NEXT:    br label [[OMP_PAR_PRE_FINALIZE:%.*]]
 // CHECK3:       omp.par.pre_finalize:
 // CHECK3-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT_EXITSTUB:%.*]]
-// CHECK3:       omp.par.outlined.exit.exitStub:
+// CHECK3:       omp.par.exit.exitStub:
 // CHECK3-NEXT:    ret void
 //
 //
@@ -805,9 +803,7 @@ int main (int argc, char **argv) {
 // CHECK3-NEXT:    store ptr [[ARGC_ADDR]], ptr [[GEP_ARGC_ADDR]], align 8
 // CHECK3-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 1, ptr @_Z5tmainIPPcEiT_..omp_par, ptr [[STRUCTARG]])
 // CHECK3-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-// CHECK3:       omp.par.outlined.exit:
-// CHECK3-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// CHECK3:       omp.par.exit.split:
+// CHECK3:       omp.par.exit:
 // CHECK3-NEXT:    ret i32 0
 //
 //
@@ -837,7 +833,7 @@ int main (int argc, char **argv) {
 // CHECK3-NEXT:    br label [[OMP_PAR_PRE_FINALIZE:%.*]]
 // CHECK3:       omp.par.pre_finalize:
 // CHECK3-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT_EXITSTUB:%.*]]
-// CHECK3:       omp.par.outlined.exit.exitStub:
+// CHECK3:       omp.par.exit.exitStub:
 // CHECK3-NEXT:    ret void
 //
 //
@@ -878,9 +874,7 @@ int main (int argc, char **argv) {
 // CHECK4-NEXT:    store ptr [[VLA]], ptr [[GEP_VLA]], align 8
 // CHECK4-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 1, ptr @main..omp_par, ptr [[STRUCTARG]]), !dbg [[DBG30:![0-9]+]]
 // CHECK4-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-// CHECK4:       omp.par.outlined.exit:
-// CHECK4-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// CHECK4:       omp.par.exit.split:
+// CHECK4:       omp.par.exit:
 // CHECK4-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[ARGV_ADDR]], align 8, !dbg [[DBG31:![0-9]+]]
 // CHECK4-NEXT:    [[CALL:%.*]] = call noundef i32 @_Z5tmainIPPcEiT_(ptr noundef [[TMP3]]), !dbg [[DBG31]]
 // CHECK4-NEXT:    store i32 [[CALL]], ptr [[RETVAL]], align 4, !dbg [[DBG31]]
@@ -912,7 +906,7 @@ int main (int argc, char **argv) {
 // CHECK4-NEXT:    br label [[OMP_PAR_PRE_FINALIZE:%.*]]
 // CHECK4:       omp.par.pre_finalize:
 // CHECK4-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT_EXITSTUB:%.*]], !dbg [[DBG35]]
-// CHECK4:       omp.par.outlined.exit.exitStub:
+// CHECK4:       omp.par.exit.exitStub:
 // CHECK4-NEXT:    ret void
 //
 //
@@ -949,9 +943,7 @@ int main (int argc, char **argv) {
 // CHECK4-NEXT:    store ptr [[ARGC_ADDR]], ptr [[GEP_ARGC_ADDR]], align 8
 // CHECK4-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 1, ptr @_Z5tmainIPPcEiT_..omp_par, ptr [[STRUCTARG]]), !dbg [[DBG52:![0-9]+]]
 // CHECK4-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-// CHECK4:       omp.par.outlined.exit:
-// CHECK4-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// CHECK4:       omp.par.exit.split:
+// CHECK4:       omp.par.exit:
 // CHECK4-NEXT:    ret i32 0, !dbg [[DBG54:![0-9]+]]
 //
 //
@@ -982,7 +974,7 @@ int main (int argc, char **argv) {
 // CHECK4-NEXT:    br label [[OMP_PAR_PRE_FINALIZE:%.*]]
 // CHECK4:       omp.par.pre_finalize:
 // CHECK4-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT_EXITSTUB:%.*]], !dbg [[DBG66]]
-// CHECK4:       omp.par.outlined.exit.exitStub:
+// CHECK4:       omp.par.exit.exitStub:
 // CHECK4-NEXT:    ret void
 //
 //
diff --git a/clang/test/OpenMP/requires_ast_print.cpp b/clang/test/OpenMP/requires_ast_print.cpp
index efa1551e75898..a602a1c41e950 100644
--- a/clang/test/OpenMP/requires_ast_print.cpp
+++ b/clang/test/OpenMP/requires_ast_print.cpp
@@ -13,6 +13,10 @@
 // RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=99 -DOMP99 -ast-print %s | FileCheck --check-prefixes=CHECK,REV %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=99 -DOMP99 -x c++ -std=c++11 -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=99 -DOMP99 -std=c++11 -include-pch %t -verify %s -ast-print | FileCheck --check-prefixes=CHECK,REV %s
+
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -DOMP60 -ast-print %s | FileCheck --check-prefixes=CHECK,CHECK_SELF %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -DOMP60 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -DOMP60 -std=c++11 -include-pch %t -verify %s -ast-print | FileCheck --check-prefixes=CHECK,CHECK_SELF %s
 // expected-no-diagnostics
 
 #ifndef HEADER
@@ -24,6 +28,11 @@
 #pragma omp requires unified_shared_memory
 // CHECK:#pragma omp requires unified_shared_memory
 
+#ifdef OMP60
+#pragma omp requires self_maps
+// CHECK_SELF:#pragma omp requires self_maps
+#endif
+
 #ifdef OMP99
 #pragma omp requires reverse_offload
 // REV:#pragma omp requires reverse_offload
diff --git a/clang/test/OpenMP/requires_messages.cpp b/clang/test/OpenMP/requires_messages.cpp
index dbb2b317067b7..db1805443e458 100644
--- a/clang/test/OpenMP/requires_messages.cpp
+++ b/clang/test/OpenMP/requires_messages.cpp
@@ -1,5 +1,6 @@
 // RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100  %s -Wuninitialized
 // RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=99 -DOMP99 -verify=expected,rev -ferror-limit 100  %s -Wuninitialized
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -DOMP60 -verify=self -ferror-limit 100  %s -Wuninitialized
 
 int a;
 #pragma omp requires unified_address allocate(a) // rev-note {{unified_address clause previously used here}} expected-note {{unified_address clause previously used here}} expected-note {{unified_address clause previously used here}} expected-note {{unified_address clause previously used here}} expected-note {{unified_address clause previously used here}} expected-note{{unified_address clause previously used here}} expected-error {{unexpected OpenMP clause 'allocate' in directive '#pragma omp requires'}}
@@ -22,6 +23,12 @@ int a;
 
 #pragma omp requires dynamic_allocators, dynamic_allocators // expected-error {{only one dynamic_allocators clause can appear on a requires directive in a single translation unit}} expected-error {{directive '#pragma omp requires' cannot contain more than one 'dynamic_allocators' clause}}
 
+#ifdef OMP60
+#pragma omp requires self_maps // self-note {{self_maps clause previously used here}}
+
+#pragma omp requires self_maps, self_maps // self-error {{only one self_maps clause can appear on a requires directive in a single translation unit}} self-error {{directive '#pragma omp requires' cannot contain more than one 'self_maps' clause}}
+#endif
+
 #pragma omp requires atomic_default_mem_order(seq_cst) // rev-note {{atomic_default_mem_order clause previously used here}} expected-note {{atomic_default_mem_order clause previously used here}} expected-note {{atomic_default_mem_order clause previously used here}} expected-note {{atomic_default_mem_order clause previously used here}} expected-note {{atomic_default_mem_order clause previously used here}}
 
 #pragma omp requires atomic_default_mem_order(acq_rel) // expected-error {{only one atomic_default_mem_order clause can appear on a requires directive in a single translation unit}}
diff --git a/clang/test/OpenMP/target_data_ast_print.cpp b/clang/test/OpenMP/target_data_ast_print.cpp
index 9e883b25f11d4..a41c7f1a0da53 100644
--- a/clang/test/OpenMP/target_data_ast_print.cpp
+++ b/clang/test/OpenMP/target_data_ast_print.cpp
@@ -13,6 +13,14 @@
 // RUN: %clang_cc1 -DOMP51 -DOMPX -verify -fopenmp-simd -fopenmp-extensions -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51,OMPX %s
 // RUN: %clang_cc1 -DOMP51 -DOMPX -fopenmp-simd -fopenmp-extensions -x c++ -std=c++11 -emit-pch -o %t %s
 // RUN: %clang_cc1 -DOMP51 -DOMPX -fopenmp-simd -fopenmp-extensions -std=c++11 -include-pch %t -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51,OMPX %s
+
+// RUN: %clang_cc1 -DOMP60 -verify -fopenmp -fopenmp-version=60 -fopenmp-extensions -ast-print %s | FileCheck -check-prefixes=CHECK,OMP60 %s
+// RUN: %clang_cc1 -DOMP60 -fopenmp -fopenmp-version=60 -fopenmp-extensions -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -DOMP60 -fopenmp -fopenmp-version=60 -fopenmp-extensions -std=c++11 -include-pch %t -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP60 %s
+
+// RUN: %clang_cc1 -DOMP60 -verify -fopenmp-simd -fopenmp -fopenmp-version=60 -fopenmp-extensions -ast-print %s | FileCheck -check-prefixes=CHECK,OMP60 %s
+// RUN: %clang_cc1 -DOMP60 -fopenmp-simd -fopenmp -fopenmp-version=60 -fopenmp-extensions -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -DOMP60 -fopenmp-simd -fopenmp -fopenmp-version=60 -fopenmp-extensions -std=c++11 -include-pch %t -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP60 %s
 // expected-no-diagnostics
 
 #ifndef HEADER
@@ -51,6 +59,11 @@ T tmain(T argc, T *argv) {
 #pragma omp target data map(close,alloc: e)
   foo();
 
+#ifdef OMP60
+#pragma omp target data map(self,alloc: e)
+  foo();
+#endif
+
 #ifdef OMP51
 #pragma omp target data map(present,alloc: e)
   foo();
@@ -68,6 +81,10 @@ T tmain(T argc, T *argv) {
     foo();
   #pragma omp target map(close, alloc: e)
     foo();
+#ifdef OMP60
+  #pragma omp target map(self,alloc: e)
+    foo();
+#endif
 #ifdef OMP51
   #pragma omp target map(present, alloc: e)
     foo();
@@ -101,6 +118,8 @@ T tmain(T argc, T *argv) {
 // CHECK-NEXT: foo();
 // CHECK-NEXT: #pragma omp target data map(close,alloc: e)
 // CHECK-NEXT: foo();
+// OMP60-NEXT: #pragma omp target data map(self,alloc: e)
+// OMP60-NEXT: foo();
 // OMP51-NEXT: #pragma omp target data map(present,alloc: e)
 // OMP51-NEXT: foo();
 //  OMPX-NEXT: #pragma omp target data map(ompx_hold,alloc: e)
@@ -111,6 +130,8 @@ T tmain(T argc, T *argv) {
 // CHECK-NEXT: foo();
 // CHECK-NEXT: #pragma omp target map(close,alloc: e)
 // CHECK-NEXT: foo();
+// OMP60-NEXT: #pragma omp target map(self,alloc: e)
+// OMP60-NEXT: foo();
 // OMP51-NEXT: #pragma omp target map(present,alloc: e)
 // OMP51-NEXT: foo();
 //  OMPX-NEXT: #pragma omp target map(ompx_hold,alloc: e)
@@ -135,6 +156,8 @@ T tmain(T argc, T *argv) {
 // CHECK-NEXT: foo();
 // CHECK-NEXT: #pragma omp target data map(close,alloc: e)
 // CHECK-NEXT: foo();
+// OMP60-NEXT: #pragma omp target data map(self,alloc: e)
+// OMP60-NEXT: foo();
 // OMP51-NEXT: #pragma omp target data map(present,alloc: e)
 // OMP51-NEXT: foo();
 //  OMPX-NEXT: #pragma omp target data map(ompx_hold,alloc: e)
@@ -145,6 +168,8 @@ T tmain(T argc, T *argv) {
 // CHECK-NEXT: foo();
 // CHECK-NEXT: #pragma omp target map(close,alloc: e)
 // CHECK-NEXT: foo();
+// OMP60-NEXT: #pragma omp target map(self,alloc: e)
+// OMP60-NEXT: foo();
 // OMP51-NEXT: #pragma omp target map(present,alloc: e)
 // OMP51-NEXT: foo();
 //  OMPX-NEXT: #pragma omp target map(ompx_hold,alloc: e)
@@ -169,6 +194,8 @@ T tmain(T argc, T *argv) {
 // CHECK-NEXT: foo();
 // CHECK-NEXT: #pragma omp target data map(close,alloc: e)
 // CHECK-NEXT: foo();
+// OMP60-NEXT: #pragma omp target data map(self,alloc: e)
+// OMP60-NEXT: foo();
 // OMP51-NEXT: #pragma omp target data map(present,alloc: e)
 // OMP51-NEXT: foo();
 //  OMPX-NEXT: #pragma omp target data map(ompx_hold,alloc: e)
@@ -179,6 +206,8 @@ T tmain(T argc, T *argv) {
 // CHECK-NEXT: foo();
 // CHECK-NEXT: #pragma omp target map(close,alloc: e)
 // CHECK-NEXT: foo();
+// OMP60-NEXT: #pragma omp target map(self,alloc: e)
+// OMP60-NEXT: foo();
 // OMP51-NEXT: #pragma omp target map(present,alloc: e)
 // OMP51-NEXT: foo();
 //  OMPX-NEXT: #pragma omp target map(ompx_hold,alloc: e)
@@ -235,6 +264,13 @@ int main (int argc, char **argv) {
   foo();
 // CHECK-NEXT: foo();
 
+// OMP60-NEXT: #pragma omp target data map(self,alloc: e)
+// OMP60-NEXT: foo();
+#ifdef OMP60
+#pragma omp target data map(self,alloc: e)
+  foo();
+#endif
+
 // OMP51-NEXT: #pragma omp target data map(present,alloc: e)
 // OMP51-NEXT: foo();
 #ifdef OMP51
diff --git a/clang/test/OpenMP/target_map_messages.cpp b/clang/test/OpenMP/target_map_messages.cpp
index 10f46687d6379..911031d5412a9 100644
--- a/clang/test/OpenMP/target_map_messages.cpp
+++ b/clang/test/OpenMP/target_map_messages.cpp
@@ -128,6 +128,16 @@ struct SA {
     {}
     #pragma omp target map(always)   // expected-error {{use of undeclared identifier 'always'}}
     {}
+    #pragma omp target map(self, tofrom: c,f)   // lt60-error {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
+    {}
+    #pragma omp target map(self, tofrom: c[1:2],f)    // lt60-error {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
+    {}
+    #pragma omp target map(self, tofrom: c,f[1:2])   // lt60-error {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
+    {}
+    #pragma omp target map(self, tofrom: c[:],f)   // lt60-error {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}} // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
+    {}
+    #pragma omp target map(self, tofrom: c,f[:])   // lt60-error {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}} // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}} 
+    {}
     #pragma omp target map(close, tofrom: c,f)
     {}
     #pragma omp target map(close, tofrom: c[1:2],f)
@@ -194,6 +204,8 @@ struct SA {
     {}
     #pragma omp target map(always, close, always, close, tofrom: a)   // expected-error 2 {{same map type modifier has been specified more than once}}
     {}
+    #pragma omp target map(self, self, tofrom: a)   // lt60-error {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}  // lt60-error {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}} // lt60-error {{same map type modifier has been specified more than once}} // ge60-error {{same map type modifier has been specified more than once}}
+    {}
     // ge60-error@+3 {{same map type modifier has been specified more than once}}
     // ge51-error@+2 {{same map type modifier has been specified more than once}}
     // lt51-error@+1 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
@@ -725,6 +737,10 @@ T tmain(T argc) {
 #pragma omp target data map(close, tofrom: close, tofrom, x)
   foo();
 
+#pragma omp target data map(self, tofrom: x) // lt60-error {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
+#pragma omp target data map(self: x) // lt60-error {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}} // lt60-error {{missing map type}}
+  foo();
+
 // lt51-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
 #pragma omp target data map(present, tofrom: x)
 // ge51-error@+2 {{missing map type}}
diff --git a/clang/test/OpenMP/taskgroup_codegen.cpp b/clang/test/OpenMP/taskgroup_codegen.cpp
index 61105a6472e37..72653144d08dd 100644
--- a/clang/test/OpenMP/taskgroup_codegen.cpp
+++ b/clang/test/OpenMP/taskgroup_codegen.cpp
@@ -224,9 +224,7 @@ void parallel_taskgroup() {
 // CHECK2:       omp_parallel:
 // CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 0, ptr @_Z18parallel_taskgroupv..omp_par)
 // CHECK2-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-// CHECK2:       omp.par.outlined.exit:
-// CHECK2-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-// CHECK2:       omp.par.exit.split:
+// CHECK2:       omp.par.exit:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -250,6 +248,6 @@ void parallel_taskgroup() {
 // CHECK2-NEXT:    br label [[OMP_PAR_PRE_FINALIZE:%.*]]
 // CHECK2:       omp.par.pre_finalize:
 // CHECK2-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT_EXITSTUB:%.*]]
-// CHECK2:       omp.par.outlined.exit.exitStub:
+// CHECK2:       omp.par.exit.exitStub:
 // CHECK2-NEXT:    ret void
 //
diff --git a/clang/test/Parser/MicrosoftExtensions.cpp b/clang/test/Parser/MicrosoftExtensions.cpp
index e47b4ef78afdf..9102bca8f6bb2 100644
--- a/clang/test/Parser/MicrosoftExtensions.cpp
+++ b/clang/test/Parser/MicrosoftExtensions.cpp
@@ -126,7 +126,7 @@ void template_uuid()
 }
 
 
-template <class T, const GUID* g = &__uuidof(T)> // expected-note 2{{template parameter is declared here}}
+template <class T, const GUID* g = &__uuidof(T)> // expected-note {{template parameter is declared here}}
 class COM_CLASS_TEMPLATE  { };
 
 typedef COM_CLASS_TEMPLATE<struct_with_uuid, &*&__uuidof(struct_with_uuid)> COM_TYPE_1; // expected-warning {{non-type template argument containing a dereference operation is a Microsoft extension}}
diff --git a/clang/test/Parser/cxx-attributes.cpp b/clang/test/Parser/cxx-attributes.cpp
index 51d1f9c822812..cd09f8d11f1a6 100644
--- a/clang/test/Parser/cxx-attributes.cpp
+++ b/clang/test/Parser/cxx-attributes.cpp
@@ -14,7 +14,7 @@ class c {
 };
 
 template <typename T> class X {
-  template <typename S> void X<S>::f() __attribute__((locks_excluded())); // expected-error{{nested name specifier 'X<S>::' for declaration does not refer into a class, class template or class template partial specialization}} \
+  template <typename S> void X<S>::f() __attribute__((locks_excluded())); // expected-error{{nested name specifier 'X<S>' for declaration does not refer into a class, class template or class template partial specialization}} \
                                                                           // expected-warning{{attribute locks_excluded ignored, because it is not attached to a declaration}}
 };
 
diff --git a/clang/test/Parser/cxx-template-argument.cpp b/clang/test/Parser/cxx-template-argument.cpp
index ffe53e7c6f77e..3c2169f86d6e7 100644
--- a/clang/test/Parser/cxx-template-argument.cpp
+++ b/clang/test/Parser/cxx-template-argument.cpp
@@ -57,9 +57,9 @@ namespace PR13210 {
 // Don't emit spurious messages
 namespace pr16225add {
 
-  template<class T1, typename T2> struct Known { }; // expected-note 3{{template parameter is declared here}}
+  template<class T1, typename T2> struct Known { }; // expected-note 3 {{template is declared here}}
   template<class T1, typename T2> struct X;
-  template<class T1, typename T2> struct ABC; // expected-note {{template parameter is declared here}}
+  template<class T1, typename T2> struct ABC; // expected-note {{template is declared here}}
   template<int N1, int N2> struct ABC2 {};
 
   template<class T1, typename T2> struct foo :
@@ -68,7 +68,7 @@ namespace pr16225add {
 
   template<class T1, typename T2> struct foo2 :
     UnknownBase<T1,T2>, // expected-error {{no template named 'UnknownBase'}}
-    Known<T1>  // expected-error {{missing template argument for template parameter}}
+    Known<T1>  // expected-error {{too few template arguments for class template 'Known'}}
   { };
 
   template<class T1, typename T2> struct foo3 :
@@ -76,8 +76,8 @@ namespace pr16225add {
   { };
 
   template<class T1, typename T2> struct foo4 :
-    UnknownBase<T1,ABC<T2> >, // expected-error {{missing template argument for template parameter}}
-    Known<T1>  // expected-error {{missing template argument for template parameter}}
+    UnknownBase<T1,ABC<T2> >, // expected-error {{too few template arguments for class template 'ABC'}}
+    Known<T1>  // expected-error {{too few template arguments for class template 'Known'}}
   { };
 
   template<class T1, typename T2> struct foo5 :
@@ -92,7 +92,7 @@ namespace pr16225add {
 #if __cplusplus <= 199711L
     // expected-error@-2 {{use '> >'}}
 #endif
-    Known<T1>  // expected-error {{missing template argument for template parameter}}
+    Known<T1>  // expected-error {{too few template arguments for class template 'Known'}}
   { };
 
   template<class T1, typename T2, int N> struct foo7 :
diff --git a/clang/test/Parser/cxx-template-template-recovery.cpp b/clang/test/Parser/cxx-template-template-recovery.cpp
index 2ece5a8ccdbea..5700b160cd364 100644
--- a/clang/test/Parser/cxx-template-template-recovery.cpp
+++ b/clang/test/Parser/cxx-template-template-recovery.cpp
@@ -1,33 +1,33 @@
 // RUN: %clang_cc1 -std=c++20 -verify -fsyntax-only %s
 
 namespace a {
-  template <typename T> // #C1-T
-  concept C1 = true;
+  template <typename T>
+  concept C1 = true; // #C1
 
   template <typename T>
   auto V1 = true; // #V1
 
   namespace b {
-    template <typename T> // #C2-T
-    concept C2 = true;
+    template <typename T>
+    concept C2 = true; // #C2
     template <typename T>
     auto V2 = true; // #V2
   }
 }
 
-template <typename T> // #C3-T
-concept C3 = true;
+template <typename T>
+concept C3 = true; // #C3
 template <typename T>
 auto V3 = true; // #V3
 template <template <typename T> typename C>
 constexpr bool test = true;
 
-static_assert(test<a::C1>); // expected-error {{missing template argument for template parameter}} \
-                            // expected-note@#C1-T {{here}}
-static_assert(test<a::b::C2>); // expected-error {{missing template argument for template parameter}} \
-                               // expected-note@#C2-T {{here}}
-static_assert(test<C3>); // expected-error {{missing template argument for template parameter}} \
-                         // expected-note@#C3-T {{here}}
+static_assert(test<a::C1>); // expected-error {{too few template arguments for concept 'C1'}} \
+                            // expected-note@#C1 {{here}}
+static_assert(test<a::b::C2>); // expected-error {{too few template arguments for concept 'C2'}} \
+                               // expected-note@#C2 {{here}}
+static_assert(test<C3>); // expected-error {{too few template arguments for concept 'C3'}} \
+                         // expected-note@#C3 {{here}}
 
 static_assert(test<a::V1>); // expected-error {{use of variable template 'a::V1' requires template arguments}} \
                             // expected-note@#V1 {{here}}
diff --git a/clang/test/Parser/cxx1z-class-template-argument-deduction.cpp b/clang/test/Parser/cxx1z-class-template-argument-deduction.cpp
index eda8cb739f0af..a1594333abae7 100644
--- a/clang/test/Parser/cxx1z-class-template-argument-deduction.cpp
+++ b/clang/test/Parser/cxx1z-class-template-argument-deduction.cpp
@@ -11,7 +11,7 @@ A(int) -> A<int>;
 // Make sure we still correctly parse cases where a template can appear without arguments.
 namespace template_template_arg {
   template<template<typename> typename> struct X {};
-  template<typename> struct Y {}; // expected-note 2{{template parameter is declared here}}
+  template<typename> struct Y {};
 
   X<A> xa;
   Y<A> ya; // expected-error {{requires template arguments}}
@@ -36,10 +36,10 @@ namespace template_template_arg {
 
 namespace template_template_arg_pack {
   template<template<typename> typename...> struct XP {};
-  template<typename...> struct YP {}; // expected-note 2{{template parameter is declared here}}
+  template<typename...> struct YP {};
 
   struct Z { template<typename T> struct Q {}; }; // expected-note 2{{here}}
-
+  
   template<typename T> using ZId = Z;
 
   template<typename ...Ts> struct A {
@@ -116,7 +116,7 @@ namespace stmt {
 }
 
 namespace expr {
-  template<typename T> struct U {}; // expected-note {{template parameter is declared here}}
+  template<typename T> struct U {};
   void j() {
     (void)typeid(A); // expected-error{{requires template arguments; argument deduction not allowed here}}
     (void)sizeof(A); // expected-error{{requires template arguments; argument deduction not allowed here}}
@@ -217,7 +217,7 @@ namespace typename_specifier {
 }
 
 namespace parenthesized {
-  template<typename T> struct X { X(T); };
+  template<typename T> struct X { X(T); };                    
   auto n = (X([]{}));
 }
 
diff --git a/clang/test/Parser/cxx1z-decomposition.cpp b/clang/test/Parser/cxx1z-decomposition.cpp
index acf3f99069185..a6ca642269eee 100644
--- a/clang/test/Parser/cxx1z-decomposition.cpp
+++ b/clang/test/Parser/cxx1z-decomposition.cpp
@@ -37,12 +37,12 @@ namespace OtherDecl {
   void g() {
     // A condition is allowed as a Clang extension.
     // See commentary in test/Parser/decomposed-condition.cpp
-    for (; auto [a, b, c] = S(); ) {} // expected-warning {{ISO C++17 does not permit structured binding declaration in a condition}} expected-error {{value of type 'S' is not contextually convertible to 'bool'}}
-    if (auto [a, b, c] = S()) {} // expected-warning {{ISO C++17 does not permit structured binding declaration in a condition}} expected-error {{value of type 'S' is not contextually convertible to 'bool'}}
-    if (int n; auto [a, b, c] = S()) {} // expected-warning {{ISO C++17 does not permit structured binding declaration in a condition}} expected-error {{value of type 'S' is not contextually convertible to 'bool'}}
-    switch (auto [a, b, c] = S()) {} // expected-warning {{ISO C++17 does not permit structured binding declaration in a condition}} expected-error {{statement requires expression of integer type ('S' invalid)}}
-    switch (int n; auto [a, b, c] = S()) {} // expected-warning {{ISO C++17 does not permit structured binding declaration in a condition}} expected-error {{statement requires expression of integer type ('S' invalid)}}
-    while (auto [a, b, c] = S()) {} // expected-warning {{ISO C++17 does not permit structured binding declaration in a condition}} expected-error {{value of type 'S' is not contextually convertible to 'bool'}}
+    for (; auto [a, b, c] = S(); ) {} // pre2c-warning {{structured binding declaration in a condition is a C++2c extenstion}} expected-error {{value of type 'S' is not contextually convertible to 'bool'}}
+    if (auto [a, b, c] = S()) {} // pre2c-warning {{structured binding declaration in a condition is a C++2c extenstion}} expected-error {{value of type 'S' is not contextually convertible to 'bool'}}
+    if (int n; auto [a, b, c] = S()) {} // pre2c-warning {{structured binding declaration in a condition is a C++2c extenstion}} expected-error {{value of type 'S' is not contextually convertible to 'bool'}}
+    switch (auto [a, b, c] = S()) {} // pre2c-warning {{structured binding declaration in a condition is a C++2c extenstion}} expected-error {{statement requires expression of integer type ('S' invalid)}}
+    switch (int n; auto [a, b, c] = S()) {} // pre2c-warning {{structured binding declaration in a condition is a C++2c extenstion}} expected-error {{statement requires expression of integer type ('S' invalid)}}
+    while (auto [a, b, c] = S()) {} // pre2c-warning {{structured binding declaration in a condition is a C++2c extenstion}} expected-error {{value of type 'S' is not contextually convertible to 'bool'}}
 
     // An exception-declaration is not a simple-declaration.
     try {}
@@ -136,8 +136,8 @@ namespace MultiDeclarator {
 
 namespace Template {
   int n[3];
-  // FIXME: There's no actual rule against this...
-  template<typename T> auto [a, b, c] = n; // expected-error {{decomposition declaration template not supported}}
+  // Structured binding template is not allowed.
+  template<typename T> auto [a, b, c] = n; // expected-error {{decomposition declaration cannot be a template}}
 }
 
 namespace Init {
diff --git a/clang/test/Parser/cxx2c-variadic-friends.cpp b/clang/test/Parser/cxx2c-variadic-friends.cpp
index b7da3e6110481..621ae912c1ac9 100644
--- a/clang/test/Parser/cxx2c-variadic-friends.cpp
+++ b/clang/test/Parser/cxx2c-variadic-friends.cpp
@@ -59,10 +59,10 @@ struct VS {
   // FIXME: Both of these should be valid, but we can't handle these at
   // the moment because the NNS is dependent.
   template<class ...T>
-  friend class TS<Ts>::Nested...; // expected-warning {{dependent nested name specifier 'TS<Ts>::' for friend template declaration is not supported; ignoring this friend declaration}}
+  friend class TS<Ts>::Nested...; // expected-warning {{dependent nested name specifier 'TS<Ts>' for friend template declaration is not supported; ignoring this friend declaration}}
 
   template<class T>
-  friend class D<T, Ts>::Nested...; // expected-warning {{dependent nested name specifier 'D<T, Ts>::' for friend class declaration is not supported; turning off access control for 'VS'}}
+  friend class D<T, Ts>::Nested...; // expected-warning {{dependent nested name specifier 'D<T, Ts>' for friend class declaration is not supported; turning off access control for 'VS'}}
 };
 
 namespace length_mismatch {
diff --git a/clang/test/Parser/decomposed-condition.cpp b/clang/test/Parser/decomposed-condition.cpp
index 4c635c4735db8..37a24f8f95093 100644
--- a/clang/test/Parser/decomposed-condition.cpp
+++ b/clang/test/Parser/decomposed-condition.cpp
@@ -33,33 +33,33 @@ Na g();
 
 namespace CondInIf {
 int h() {
-  if (auto [ok, d] = f()) // expected-warning {{ISO C++17 does not permit structured binding declaration in a condition}}
+  if (auto [ok, d] = f()) // expected-warning {{structured binding declaration in a condition is a C++2c extenstion}}
     ;
-  if (auto [ok, d] = g()) // expected-warning {{ISO C++17 does not permit structured binding declaration in a condition}} expected-error {{value of type 'Na' is not contextually convertible to 'bool'}}
+  if (auto [ok, d] = g()) // expected-warning {{structured binding declaration in a condition is a C++2c extenstion}} expected-error {{value of type 'Na' is not contextually convertible to 'bool'}}
     ;
-  if (auto [value] = Get()) // expected-warning {{ISO C++17 does not permit structured binding declaration in a condition}}
+  if (auto [value] = Get()) // expected-warning {{structured binding declaration in a condition is a C++2c extenstion}}
     return value;
 }
 } // namespace CondInIf
 
 namespace CondInWhile {
 int h() {
-  while (auto [ok, d] = f()) // expected-warning {{ISO C++17 does not permit structured binding declaration in a condition}}
+  while (auto [ok, d] = f()) // expected-warning {{structured binding declaration in a condition is a C++2c extenstion}}
     ;
-  while (auto [ok, d] = g()) // expected-warning {{ISO C++17 does not permit structured binding declaration in a condition}} expected-error {{value of type 'Na' is not contextually convertible to 'bool'}}
+  while (auto [ok, d] = g()) // expected-warning {{structured binding declaration in a condition is a C++2c extenstion}} expected-error {{value of type 'Na' is not contextually convertible to 'bool'}}
     ;
-  while (auto [value] = Get()) // expected-warning{{ISO C++17 does not permit structured binding declaration in a condition}}
+  while (auto [value] = Get()) // expected-warning{{structured binding declaration in a condition is a C++2c extenstion}}
     return value;
 }
 } // namespace CondInWhile
 
 namespace CondInFor {
 int h() {
-  for (; auto [ok, d] = f();) // expected-warning {{ISO C++17 does not permit structured binding declaration in a condition}}
+  for (; auto [ok, d] = f();) // expected-warning {{structured binding declaration in a condition is a C++2c extenstion}}
     ;
-  for (; auto [ok, d] = g();) // expected-warning {{ISO C++17 does not permit structured binding declaration in a condition}} expected-error {{value of type 'Na' is not contextually convertible to 'bool'}}
+  for (; auto [ok, d] = g();) // expected-warning {{structured binding declaration in a condition is a C++2c extenstion}} expected-error {{value of type 'Na' is not contextually convertible to 'bool'}}
     ;
-  for (; auto [value] = Get();) // expected-warning {{ISO C++17 does not permit structured binding declaration in a condition}}
+  for (; auto [value] = Get();) // expected-warning {{structured binding declaration in a condition is a C++2c extenstion}}
     return value;
 }
 } // namespace CondInFor
@@ -74,11 +74,11 @@ struct IntegerLike {
 
 namespace CondInSwitch {
 int h(IntegerLike x) {
-  switch (auto [ok, d] = x) // expected-warning {{ISO C++17 does not permit structured binding declaration in a condition}}
+  switch (auto [ok, d] = x) // expected-warning {{structured binding declaration in a condition is a C++2c extenstion}}
     ;
-  switch (auto [ok, d] = g()) // expected-warning {{ISO C++17 does not permit structured binding declaration in a condition}} expected-error {{statement requires expression of integer type ('Na' invalid)}}
+  switch (auto [ok, d] = g()) // expected-warning {{structured binding declaration in a condition is a C++2c extenstion}} expected-error {{statement requires expression of integer type ('Na' invalid)}}
     ;
-  switch (auto [value] = Get()) {// expected-warning {{ISO C++17 does not permit structured binding declaration in a condition}}
+  switch (auto [value] = Get()) {// expected-warning {{structured binding declaration in a condition is a C++2c extenstion}}
   // expected-warning@-1{{switch condition has boolean value}}
   case 1:
     return value;
diff --git a/clang/test/ParserOpenACC/parse-clauses.c b/clang/test/ParserOpenACC/parse-clauses.c
index 0964ae78216dd..5178138216773 100644
--- a/clang/test/ParserOpenACC/parse-clauses.c
+++ b/clang/test/ParserOpenACC/parse-clauses.c
@@ -1286,13 +1286,7 @@ void BCP1();
   // expected-error@+1{{expected identifier or string literal}}
 #pragma acc routine(BCP1) seq bind()
 
-  // expected-warning@+2{{OpenACC clause 'bind' not yet implemented, clause ignored}}
   // expected-warning@+1{{OpenACC construct 'routine' with implicit function not yet implemented, pragma ignored}}
 #pragma acc routine seq bind("ReductionClauseParsing")
-void BCP2();
 
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented, clause ignored}}
-#pragma acc routine(BCP1) seq bind(BCP2)
-
-  // expected-error@+1{{use of undeclared identifier 'unknown_thing'}}
 #pragma acc routine(BCP1) seq bind(unknown_thing)
diff --git a/clang/test/ParserOpenACC/parse-clauses.cpp b/clang/test/ParserOpenACC/parse-clauses.cpp
index 03a0fd945e0b9..6c84018060943 100644
--- a/clang/test/ParserOpenACC/parse-clauses.cpp
+++ b/clang/test/ParserOpenACC/parse-clauses.cpp
@@ -49,52 +49,15 @@ void use() {
   templ<7, S>();
 }
 
-namespace NS {
-void NSFunc();
-
-class RecordTy { // #RecTy
-  static constexpr bool Value = false; // #VAL
-  void priv_mem_function(); // #PrivMemFun
-  public:
-  static constexpr bool ValuePub = true;
-  void mem_function();
-};
-template<typename T>
-class TemplTy{};
-void function();
-}
-
-
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented, clause ignored}}
+// expected-error@+2{{expected ')'}}
+// expected-note@+1{{to match this '('}}
 #pragma acc routine(use) seq bind(NS::NSFunc)
-  // expected-error@+2{{'RecordTy' does not refer to a value}}
-  // expected-note@#RecTy{{declared here}}
-#pragma acc routine(use) seq bind(NS::RecordTy)
-  // expected-error@+3{{'Value' is a private member of 'NS::RecordTy'}}
-  // expected-note@#VAL{{implicitly declared private here}}
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented, clause ignored}}
-#pragma acc routine(use) seq bind(NS::RecordTy::Value)
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented, clause ignored}}
-#pragma acc routine(use) seq bind(NS::RecordTy::ValuePub)
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented, clause ignored}}
-#pragma acc routine(use) seq bind(NS::TemplTy<int>)
-  // expected-error@+1{{no member named 'unknown' in namespace 'NS'}}
-#pragma acc routine(use) seq bind(NS::unknown<int>)
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented, clause ignored}}
-#pragma acc routine(use) seq bind(NS::function)
-  // expected-error@+3{{'priv_mem_function' is a private member of 'NS::RecordTy'}}
-  // expected-note@#PrivMemFun{{implicitly declared private here}}
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented, clause ignored}}
-#pragma acc routine(use) seq bind(NS::RecordTy::priv_mem_function)
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented, clause ignored}}
-#pragma acc routine(use) seq bind(NS::RecordTy::mem_function)
 
   // expected-error@+1{{string literal with user-defined suffix cannot be used here}}
 #pragma acc routine(use) seq bind("unknown udl"_UDL)
 
-  // expected-warning@+2{{encoding prefix 'u' on an unevaluated string literal has no effect}}
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented, clause ignored}}
+  // expected-warning@+1{{encoding prefix 'u' on an unevaluated string literal has no effect}}
 #pragma acc routine(use) seq bind(u"16 bits")
-  // expected-warning@+2{{encoding prefix 'U' on an unevaluated string literal has no effect}}
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented, clause ignored}}
-#pragma acc routine(use) seq bind(U"32 bits")
+void another_func();
+  // expected-warning@+1{{encoding prefix 'U' on an unevaluated string literal has no effect}}
+#pragma acc routine(another_func) seq bind(U"32 bits")
diff --git a/clang/test/Sema/aarch64-special-register.c b/clang/test/Sema/aarch64-special-register.c
index 4d2cfd8b37c84..937c1eaf6b656 100644
--- a/clang/test/Sema/aarch64-special-register.c
+++ b/clang/test/Sema/aarch64-special-register.c
@@ -116,6 +116,13 @@ unsigned long rsr64_6(void) {
   return __builtin_arm_rsr64("0:1:16:16:2"); //expected-error {{invalid special register for builtin}}
 }
 
+void rsr64_7(unsigned long *r) {
+  // The following three instructions should produce the same assembly.
+  r[0] = __builtin_arm_rsr64("ICC_CTLR_EL3");
+  r[1] = __builtin_arm_rsr64("s3_6_c12_c12_4");
+  r[2] = __builtin_arm_rsr64("3:6:12:12:4");
+}
+
 __uint128_t rsr128_3(void) {
   return __builtin_arm_rsr128("0:1:2"); //expected-error {{invalid special register for builtin}}
 }
diff --git a/clang/test/Sema/attr-availability-macosx.cpp b/clang/test/Sema/attr-availability-macosx.cpp
index 52f320d409281..41ac3e7495c43 100644
--- a/clang/test/Sema/attr-availability-macosx.cpp
+++ b/clang/test/Sema/attr-availability-macosx.cpp
@@ -14,4 +14,17 @@ bool try_acquire_for(T duration) { // expected-note{{'try_acquire_for<int>' has
 int main() {
   try_acquire_for(1); // expected-warning{{'try_acquire_for<int>' is only available on macOS 11 or newer}}
   // expected-note@-1{{enclose 'try_acquire_for<int>' in a __builtin_available check to silence this warning}}
-}
\ No newline at end of file
+}
+
+namespace typename_template {
+  struct [[clang::availability(macos, introduced = 16)]] A {};
+
+  template<class T> struct B { using type = T; };
+  template <class T> struct C {
+    typename B<T>::type v;
+  };
+
+  struct [[clang::availability(macos, introduced = 16)]] D {
+    C<A> c;
+  };
+} // namespace typename_template
diff --git a/clang/test/Sema/builtins-elementwise-math.c b/clang/test/Sema/builtins-elementwise-math.c
index d6785f78340ca..86fec7632af83 100644
--- a/clang/test/Sema/builtins-elementwise-math.c
+++ b/clang/test/Sema/builtins-elementwise-math.c
@@ -526,6 +526,26 @@ void test_builtin_elementwise_exp2(int i, float f, double d, float4 v, int3 iv,
   // expected-error@-1 {{1st argument must be a floating point type (was 'unsigned4' (vector of 4 'unsigned int' values))}}
 }
 
+void test_builtin_elementwise_exp10(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) {
+
+  struct Foo s = __builtin_elementwise_exp10(f);
+  // expected-error@-1 {{initializing 'struct Foo' with an expression of incompatible type 'float'}}
+
+  i = __builtin_elementwise_exp10();
+  // expected-error@-1 {{too few arguments to function call, expected 1, have 0}}
+
+  i = __builtin_elementwise_exp10(i);
+  // expected-error@-1 {{1st argument must be a floating point type (was 'int')}}
+
+  i = __builtin_elementwise_exp10(f, f);
+  // expected-error@-1 {{too many arguments to function call, expected 1, have 2}}
+
+  u = __builtin_elementwise_exp10(u);
+  // expected-error@-1 {{1st argument must be a floating point type (was 'unsigned int')}}
+
+  uv = __builtin_elementwise_exp10(uv);
+  // expected-error@-1 {{1st argument must be a floating point type (was 'unsigned4' (vector of 4 'unsigned int' values))}}
+}
 
 void test_builtin_elementwise_floor(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) {
 
diff --git a/clang/test/Sema/compare.c b/clang/test/Sema/compare.c
index 17cf0351ef4f5..fdae3bc19841e 100644
--- a/clang/test/Sema/compare.c
+++ b/clang/test/Sema/compare.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin -fsyntax-only -pedantic -verify -Wsign-compare -Wtautological-constant-in-range-compare %s -Wno-unreachable-code
-// RUN: %clang_cc1 -triple x86_64-apple-darwin -fsyntax-only -pedantic -verify -Wsign-compare -Wtype-limits %s -Wno-unreachable-code
+// RUN: %clang_cc1 -triple x86_64-apple-darwin -fsyntax-only -pedantic -verify -Wsign-compare -Wtautological-constant-in-range-compare %s -Wno-unreachable-code -DTEST=1
+// RUN: %clang_cc1 -triple x86_64-apple-darwin -fsyntax-only -pedantic -verify -Wsign-compare -Wtype-limits %s -Wno-unreachable-code -DTEST=2
 
 int test(char *C) { // nothing here should warn.
   return C != ((void*)0);
@@ -419,3 +419,65 @@ void pr36008(enum PR36008EnumTest lhs) {
   if (x == y) x = y; // no warning
   if (y == x) y = x; // no warning
 }
+
+int test13(unsigned a, int b) {
+  return a > ~(95 != b); // expected-warning {{comparison of integers of different signs}}
+}
+
+int test14(unsigned a, unsigned b) {
+  return a > ~b; // no-warning
+}
+
+int test15(unsigned a, int b) {
+  return a > -(95 != b); // expected-warning {{comparison of integers of different signs}}
+}
+
+int test16(unsigned a, unsigned b) {
+  return a > -b; // no-warning
+}
+
+int test17(int a, unsigned b) {
+  return a > -(-b); // expected-warning {{comparison of integers of different signs}}
+}
+
+int test18(int a) {
+  return a == -(-2147483648); // expected-warning {{result of comparison of constant 2147483648 with expression of type 'int' is always false}}
+}
+
+int test19(int n) {
+  return -(n & 15) <= -15; // no-warning
+}
+
+#if TEST == 1
+int test20(int n) {
+  return -(n & 15) <= -17; // expected-warning {{result of comparison of 5-bit signed value <= -17 is always false}}
+}
+#endif
+
+int test21(short n) {
+  return -n == 32768; // no-warning
+}
+
+#if TEST == 1
+int test22(short n) {
+  return -n == 65536; // expected-warning {{result of comparison of 17-bit signed value == 65536 is always false}}
+}
+#endif
+
+int test23(unsigned short n) {
+  return ~n == 32768; // no-warning
+}
+
+int test24(short n) {
+  return ~n == 32767; // no-warning
+}
+
+#if TEST == 1
+int test25(unsigned short n) {
+  return ~n == 65536; // expected-warning {{result of comparison of 17-bit signed value == 65536 is always false}}
+}
+
+int test26(short n) {
+  return ~n == 32768; // expected-warning {{result of comparison of 16-bit signed value == 32768 is always false}}
+}
+#endif
diff --git a/clang/test/Sema/generic-selection-type-extension.c b/clang/test/Sema/generic-selection-type-extension.c
index 89ac3235807da..1efd045acba2d 100644
--- a/clang/test/Sema/generic-selection-type-extension.c
+++ b/clang/test/Sema/generic-selection-type-extension.c
@@ -37,7 +37,7 @@ static_assert(_Generic(ci, int : 1, const int : 0) == 1); // expected-warning {{
 // but the expression operand form still rejects them.
 static_assert(_Generic(struct incomplete, struct incomplete : 1, default : 0) == 1);
 static_assert(_Generic(struct another_incomplete, struct incomplete : 1, default : 0) == 0);
-static_assert(_Generic(1, struct also_incomplete : 1, default : 0) == 0); // expected-error {{type 'struct also_incomplete' in generic association incomplete}}
+static_assert(_Generic(1, struct also_incomplete : 1, default : 0) == 0);
 
 void foo(int);
 static_assert(_Generic(__typeof__(foo), void(int) : 1, default : 0) == 1);
diff --git a/clang/test/Sema/generic-selection.c b/clang/test/Sema/generic-selection.c
index 1f17896ca4cda..8e7b6ed00dd50 100644
--- a/clang/test/Sema/generic-selection.c
+++ b/clang/test/Sema/generic-selection.c
@@ -5,7 +5,7 @@ void g(void);
 
 void foo(int n) {
   (void) _Generic(0, // ext-warning {{'_Generic' is a C11 extension}}
-      struct A: 0, // expected-error {{type 'struct A' in generic association incomplete}}
+      struct A: 0, // ext-warning {{incomplete type 'struct A' in a '_Generic' association is a C2y extension}}
       void(): 0,   // expected-error {{type 'void ()' in generic association not an object type}}
       int[n]: 0);  // expected-error {{type 'int[n]' in generic association is a variably modified type}}
 
diff --git a/clang/test/Sema/types.c b/clang/test/Sema/types.c
index 2a5f530740e9a..2be0e6544f3d7 100644
--- a/clang/test/Sema/types.c
+++ b/clang/test/Sema/types.c
@@ -78,7 +78,7 @@ typedef int __attribute__((ext_vector_type(0))) e4;                // expected-e
 // no support for vector enum type
 enum { e_2 } x3 __attribute__((vector_size(64))); // expected-error {{invalid vector element type}}
 
-int x4 __attribute__((ext_vector_type(64)));  // expected-error {{'ext_vector_type' attribute only applies to typedefs}}
+int x4 __attribute__((ext_vector_type(64)));
 
 typedef __attribute__ ((ext_vector_type(32),__aligned__(32))) unsigned char uchar32;
 
diff --git a/clang/test/Sema/vector-ast.cpp b/clang/test/Sema/vector-ast.cpp
new file mode 100644
index 0000000000000..c9823719feabe
--- /dev/null
+++ b/clang/test/Sema/vector-ast.cpp
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 %s -verify -ast-dump | FileCheck %s
+
+// expected-no-diagnostics
+
+//      CHECK: ExtVectorType {{.*}} 'int __attribute__((ext_vector_type(4)))' 4
+// CHECK-NEXT: BuiltinType {{.*}} 'int'
+int x __attribute__((ext_vector_type(4)));
+using ExtVecType = decltype(x);
+
+// CHECK: FunctionDecl {{.*}} 'int () __attribute__((ext_vector_type(4)))'
+int __attribute__((ext_vector_type(4))) foo() { return x; }
+//      CHECK:  CompoundStmt
+// CHECK-NEXT: ReturnStmt
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'int __attribute__((ext_vector_type(4)))' <LValueToRValue>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'int __attribute__((ext_vector_type(4)))' lvalue Var {{.*}} 'x' 'int __attribute__((ext_vector_type(4)))'
diff --git a/clang/test/SemaCXX/PR62533.cpp b/clang/test/SemaCXX/PR62533.cpp
index 0753156813f8e..2015161e7946f 100644
--- a/clang/test/SemaCXX/PR62533.cpp
+++ b/clang/test/SemaCXX/PR62533.cpp
@@ -7,11 +7,11 @@ struct test {
 
 template<typename T, typename V>
 decltype(T::template fun_diff<V>) foo1() {}
-// expected-note@-1 {{candidate template ignored: substitution failure [with T = test<int>, V = int]: 'test<int>::fun_diff' is expected to be a non-type template, but instantiated to a type alias template}}
+// expected-note@-1 {{candidate template ignored: substitution failure [with T = test<int>, V = int]: 'test<int>::template fun_diff' is expected to be a non-type template, but instantiated to a type alias template}}
 
 template<typename T>
 void foo2() {
-  // expected-error@+1 {{test<int>::fun_diff' is expected to be a non-type template, but instantiated to a type alias template}}
+  // expected-error@+1 {{test<int>::template fun_diff' is expected to be a non-type template, but instantiated to a type alias template}}
   int a = test<T>::template fun_diff<int>;
 }
 
@@ -22,7 +22,7 @@ struct has_fun_diff {
 
 template<typename T>
 struct has_fun_diff<T, int> {
-  // expected-error@+1 {{'test<int>::fun_diff' is expected to be a non-type template, but instantiated to a type alias template}}
+  // expected-error@+1 {{'test<int>::template fun_diff' is expected to be a non-type template, but instantiated to a type alias template}}
   using type = decltype(T::template fun_diff<int>);
 };
 
diff --git a/clang/test/SemaCXX/access-base-class.cpp b/clang/test/SemaCXX/access-base-class.cpp
index 7d5fe38848f95..47d0f02ec545c 100644
--- a/clang/test/SemaCXX/access-base-class.cpp
+++ b/clang/test/SemaCXX/access-base-class.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s
 namespace T1 {
-
+  
 class A { };
 class B : private A { }; // expected-note {{declared private here}}
 
@@ -10,7 +10,7 @@ void f(B* b) {
 
 }
 
-namespace T2 {
+namespace T2 { 
 
 class A { };
 class B : A { }; // expected-note {{implicitly declared private here}}
@@ -24,7 +24,7 @@ void f(B* b) {
 namespace T3 {
 
 class A { };
-class B : public A { };
+class B : public A { }; 
 
 void f(B* b) {
   A *a = b;
@@ -50,30 +50,30 @@ void f(D *d) {
 
 namespace T5 {
   class A {};
-
+    
   class B : private A {
     void f(B *b) {
       A *a = b;
     }
-  };
+  };    
 }
 
 namespace T6 {
   class C;
-
+  
   class A {}; // expected-note{{member is declared here}}
-
+  
   class B : private A { // expected-note {{declared private here}} expected-note {{constrained by private inheritance here}}
     void f(C* c);
   };
-
-  class C : public B {
+  
+  class C : public B { 
     void f(C *c) {
       A* a = c; // expected-error {{cannot cast 'C' to its private base class 'A'}} \
                 // expected-error {{'A' is a private member of 'T6::A'}}
     }
   };
-
+  
   void B::f(C *c) {
     A *a = c;
   }
@@ -82,7 +82,7 @@ namespace T6 {
 namespace T7 {
   class A {};
   class B : public A {};
-  class C : private B {
+  class C : private B { 
     void f(C *c) {
       A* a = c; // okay
     }
@@ -98,7 +98,7 @@ struct flag {
 template <class T>
 struct trait : flag<sizeof(T)> {}; // expected-note 2{{here}}
 
-template <class T, bool Inferred = trait<T>::value> // expected-note 2{{here}}
+template <class T, bool Inferred = trait<T>::value> // expected-note {{here}}
 struct a {};
 
 template <class T>
diff --git a/clang/test/SemaCXX/alias-template.cpp b/clang/test/SemaCXX/alias-template.cpp
index 90095b35a5774..b49d36a6267e6 100644
--- a/clang/test/SemaCXX/alias-template.cpp
+++ b/clang/test/SemaCXX/alias-template.cpp
@@ -167,10 +167,7 @@ namespace SFINAE {
     f<E>();
   }
 
-  template<typename T,
-    typename U = // expected-note {{template parameter is declared here}}
-      EnableIf<is_enum<T>>> // expected-note {{in instantiation of template type alias 'EnableIf' requested here}}
-  struct fail1 {};
+  template<typename T, typename U = EnableIf<is_enum<T>>> struct fail1 {}; // expected-note {{here}}
   template<typename T> struct fail2 : DisableIf<is_enum<T>> {}; // expected-note {{here}}
 
   fail1<int> f1; // expected-note {{here}}
diff --git a/clang/test/SemaCXX/anonymous-struct.cpp b/clang/test/SemaCXX/anonymous-struct.cpp
index 51066a4516873..75309821998eb 100644
--- a/clang/test/SemaCXX/anonymous-struct.cpp
+++ b/clang/test/SemaCXX/anonymous-struct.cpp
@@ -27,7 +27,7 @@ struct E {
   };
 };
 
-template <class T> void foo(T); // #foo
+template <class T> void foo(T);
 typedef struct { // expected-error {{anonymous non-C-compatible type given name for linkage purposes by typedef declaration after its linkage was computed; add a tag name here to establish linkage prior to definition}}
 // expected-note@-1 {{unnamed type used in template argument was declared here}}
 
@@ -36,7 +36,6 @@ typedef struct { // expected-error {{anonymous non-C-compatible type given name
 #if __cplusplus <= 199711L
     // expected-warning@-2 {{template argument uses unnamed type}}
     // expected-note@-3 {{while substituting deduced template arguments}}
-    // expected-note@#foo {{template parameter is declared here}}
 #endif
   }
 } A; // expected-note {{type is given name 'A' for linkage purposes by this typedef declaration}}
diff --git a/clang/test/SemaCXX/attr-require-constant-initialization.cpp b/clang/test/SemaCXX/attr-require-constant-initialization.cpp
index 4c0a834551715..b1736ea7a3ebe 100644
--- a/clang/test/SemaCXX/attr-require-constant-initialization.cpp
+++ b/clang/test/SemaCXX/attr-require-constant-initialization.cpp
@@ -337,6 +337,11 @@ constexpr TestCtor<NotC> inval_constexpr(42); // expected-error {{must be initia
 ATTR constexpr TestCtor<NotC> inval_constexpr2(42); // expected-error {{must be initialized by a constant expression}}
 // expected-note@-1 {{in call to 'TestCtor(42)'}}
 
+[[gnu::constructor]] void ctor() {}
+// expected-warning@-1 {{declaration requires a global constructor}}
+[[gnu::destructor]] void dtor() {}
+// expected-warning@-1 {{declaration requires a global destructor}}
+
 #elif defined(TEST_THREE)
 #if defined(__cplusplus)
 #error This test requires C
diff --git a/clang/test/SemaCXX/builtins-elementwise-math.cpp b/clang/test/SemaCXX/builtins-elementwise-math.cpp
index 0d891bdd661f1..7482e5043aa35 100644
--- a/clang/test/SemaCXX/builtins-elementwise-math.cpp
+++ b/clang/test/SemaCXX/builtins-elementwise-math.cpp
@@ -134,6 +134,13 @@ void test_builtin_elementwise_exp2() {
   static_assert(!is_const<decltype(__builtin_elementwise_exp2(b))>::value);
 }
 
+void test_builtin_elementwise_exp10() {
+  const float a = 42.0;
+  float b = 42.3;
+  static_assert(!is_const<decltype(__builtin_elementwise_exp10(a))>::value);
+  static_assert(!is_const<decltype(__builtin_elementwise_exp10(b))>::value);
+}
+
 void test_builtin_elementwise_asin() {
   const float a = 42.0;
   float b = 42.3;
diff --git a/clang/test/SemaCXX/constant-expression-cxx11.cpp b/clang/test/SemaCXX/constant-expression-cxx11.cpp
index 4a7f4fd9c7e50..c35f3a5632a05 100644
--- a/clang/test/SemaCXX/constant-expression-cxx11.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx11.cpp
@@ -102,7 +102,7 @@ static_assert(n9 == 123, "");
 }
 
 namespace TemplateArgumentConversion {
-  template<int n> struct IntParam {}; // expected-note {{template parameter is declared here}}
+  template<int n> struct IntParam {};
 
   using IntParam0 = IntParam<0>;
   using IntParam0 = IntParam<id(0)>;
@@ -1472,8 +1472,8 @@ namespace ConvertedConstantExpr {
   enum class E {
     em = m,
     en = n, // expected-error {{enumerator value is not a constant expression}} cxx11_20-note {{initializer of 'n' is unknown}}
-    eo = (m + // pre-cxx23-error {{not a constant expression}}
-          n // cxx11_20-note {{initializer of 'n' is unknown}} cxx23-error {{not a constant expression}}
+    eo = (m + // expected-error {{not a constant expression}}
+          n // cxx11_20-note {{initializer of 'n' is unknown}}
           ),
     eq = reinterpret_cast<long>((int*)0) // expected-error {{not a constant expression}} expected-note {{reinterpret_cast}}
   };
@@ -1529,7 +1529,7 @@ namespace MutableMembers {
   constexpr int mmn2 = mm.n; // expected-error {{constant expression}} expected-note {{read of mutable member 'n' is not allowed in a constant expression}}
 
   // Here's one reason why allowing this would be a disaster...
-  template<int n> struct Id { int k = n; }; // expected-note {{template parameter is declared here}}
+  template<int n> struct Id { int k = n; };
   int f() {
     constexpr MM m = { 0 };
     ++m.n;
diff --git a/clang/test/SemaCXX/constant-expression-p2280r4.cpp b/clang/test/SemaCXX/constant-expression-p2280r4.cpp
index 65e5e6b34b48f..f22430a0e88a2 100644
--- a/clang/test/SemaCXX/constant-expression-p2280r4.cpp
+++ b/clang/test/SemaCXX/constant-expression-p2280r4.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++23 -verify %s
+// RUN: %clang_cc1 -std=c++23 -verify=expected,nointerpreter %s
 // RUN: %clang_cc1 -std=c++23 -verify %s -fexperimental-new-constant-interpreter
 
 using size_t = decltype(sizeof(0));
@@ -39,8 +39,8 @@ void splash(Swim& swam) {
   static_assert(swam.phelps() == 28);     // ok
   static_assert((&swam)->phelps() == 28); // ok
   Swim* pswam = &swam;                    // expected-note {{declared here}}
-  static_assert(pswam->phelps() == 28);   // expected-error {{static assertion expression is not an integral constant expression}}
-                                          // expected-note@-1 {{read of non-constexpr variable 'pswam' is not allowed in a constant expression}}
+  static_assert(pswam->phelps() == 28);   // expected-error {{static assertion expression is not an integral constant expression}} \
+                                          // expected-note {{read of non-constexpr variable 'pswam' is not allowed in a constant expression}}
   static_assert(how_many(swam) == 28);    // ok
   static_assert(Swim().lochte() == 12);   // ok
   static_assert(swam.lochte() == 12);     // expected-error {{static assertion expression is not an integral constant expression}}
@@ -154,3 +154,26 @@ int g() {
     static_assert(f(arr) == 5);
 }
 }
+
+namespace GH128409 {
+  int &ff();
+  int &x = ff(); // nointerpreter-note {{declared here}}
+  constinit int &z = x; // expected-error {{variable does not have a constant initializer}} \
+                        // expected-note {{required by 'constinit' specifier here}} \
+                        // nointerpreter-note {{initializer of 'x' is not a constant expression}}
+}
+
+namespace GH129845 {
+  int &ff();
+  int &x = ff(); // nointerpreter-note {{declared here}}
+  struct A { int& x; };
+  constexpr A g = {x}; // expected-error {{constexpr variable 'g' must be initialized by a constant expression}} \
+                       // nointerpreter-note {{initializer of 'x' is not a constant expression}}
+  const A* gg = &g;
+}
+
+namespace extern_reference_used_as_unknown {
+  extern int &x;
+  int y;
+  constinit int& g = (x,y); // expected-warning {{left operand of comma operator has no effect}}
+}
diff --git a/clang/test/SemaCXX/constant-expression.cpp b/clang/test/SemaCXX/constant-expression.cpp
index d984ced4eae3b..cc041a4acd18c 100644
--- a/clang/test/SemaCXX/constant-expression.cpp
+++ b/clang/test/SemaCXX/constant-expression.cpp
@@ -99,7 +99,7 @@ void diags(int n) {
 namespace IntOrEnum {
   const int k = 0;
   const int &p = k; // expected-note {{declared here}}
-  template<int n> struct S {}; // expected-note {{template parameter is declared here}}
+  template<int n> struct S {};
   S<p> s; // expected-error {{not an integral constant expression}} expected-note {{read of variable 'p' of non-integral, non-enumeration type 'const int &'}}
 }
 
diff --git a/clang/test/SemaCXX/constexpr-if.cpp b/clang/test/SemaCXX/constexpr-if.cpp
new file mode 100644
index 0000000000000..494fc45c55c4e
--- /dev/null
+++ b/clang/test/SemaCXX/constexpr-if.cpp
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -std=c++20 -verify=cxx20,expected %s
+// RUN: %clang_cc1 -std=c++23 -verify=cxx23,expected %s
+// RUN: %clang_cc1 -std=c++26 -verify=cxx26,expected %s
+
+// expected-no-diagnostics
+
+namespace GH123524 {
+consteval void fn1() {}
+void fn2() {
+  if constexpr (&fn1 != nullptr) { }
+}
+}
diff --git a/clang/test/SemaCXX/coroutine-decltype.cpp b/clang/test/SemaCXX/coroutine-unevaluate.cpp
similarity index 78%
rename from clang/test/SemaCXX/coroutine-decltype.cpp
rename to clang/test/SemaCXX/coroutine-unevaluate.cpp
index 7cbe6688d4155..164caed2836a1 100644
--- a/clang/test/SemaCXX/coroutine-decltype.cpp
+++ b/clang/test/SemaCXX/coroutine-unevaluate.cpp
@@ -32,3 +32,11 @@ MyTask DoAnotherthing() {
   static_assert(__is_same(void, decltype(co_yield 0))); // expected-error {{'co_yield' cannot be used in an unevaluated context}}
   co_return;
 }
+
+template<class>
+struct Task {};
+
+void BracedInitListCXX26() {
+  []() -> Task<{ co_await 1 }> {}; // expected-error {{'co_await' cannot be used in an unevaluated context}}
+  []() -> Task<{ co_yield 1 }> {}; // expected-error {{'co_yield' cannot be used in an unevaluated context}}
+}
diff --git a/clang/test/SemaCXX/cxx0x-defaulted-functions.cpp b/clang/test/SemaCXX/cxx0x-defaulted-functions.cpp
index 0c3dd1ea7aa27..340e9037b0f42 100644
--- a/clang/test/SemaCXX/cxx0x-defaulted-functions.cpp
+++ b/clang/test/SemaCXX/cxx0x-defaulted-functions.cpp
@@ -174,10 +174,10 @@ namespace PR14577 {
   };
 
   template<typename T>
-  Outer<T>::Inner1<T>::~Inner1() = delete; // expected-error {{nested name specifier 'Outer<T>::Inner1<T>::' for declaration does not refer into a class, class template or class template partial specialization}}  expected-error {{only functions can have deleted definitions}}
+  Outer<T>::Inner1<T>::~Inner1() = delete; // expected-error {{nested name specifier 'Outer<T>::Inner1<T>' for declaration does not refer into a class, class template or class template partial specialization}}  expected-error {{only functions can have deleted definitions}}
 
   template<typename T>
-  Outer<T>::Inner2<T>::~Inner2() = default; // expected-error {{nested name specifier 'Outer<T>::Inner2<T>::' for declaration does not refer into a class, class template or class template partial specialization}}
+  Outer<T>::Inner2<T>::~Inner2() = default; // expected-error {{nested name specifier 'Outer<T>::Inner2<T>' for declaration does not refer into a class, class template or class template partial specialization}}
 }
 
 extern "C" { // expected-note {{extern "C" language linkage specification begins here}}
diff --git a/clang/test/SemaCXX/cxx11-user-defined-literals.cpp b/clang/test/SemaCXX/cxx11-user-defined-literals.cpp
index bdf88ee01c3a1..c31ea4e6bc45e 100644
--- a/clang/test/SemaCXX/cxx11-user-defined-literals.cpp
+++ b/clang/test/SemaCXX/cxx11-user-defined-literals.cpp
@@ -150,18 +150,18 @@ namespace bad_names {
         operator // expected-error {{expected identifier}}
             ""_q<'a'>;
 
-    T::template operator""_q<'a'>(); // expected-error {{non-namespace scope 'T::' cannot have a literal operator member}} expected-error +{{}}
-    T::template operator""_q<'a'>::X; // expected-error {{non-namespace scope 'T::' cannot have a literal operator member}} expected-error +{{}}
-    T::operator""_q<'a'>(); // expected-error {{non-namespace scope 'T::' cannot have a literal operator member}} expected-error +{{}}
-    typename T::template operator""_q<'a'> a; // expected-error {{non-namespace scope 'T::' cannot have a literal operator member}} expected-error +{{}}
+    T::template operator""_q<'a'>(); // expected-error {{non-namespace scope 'T' cannot have a literal operator member}} expected-error +{{}}
+    T::template operator""_q<'a'>::X; // expected-error {{non-namespace scope 'T' cannot have a literal operator member}} expected-error +{{}}
+    T::operator""_q<'a'>(); // expected-error {{non-namespace scope 'T' cannot have a literal operator member}} expected-error +{{}}
+    typename T::template operator""_q<'a'> a; // expected-error {{non-namespace scope 'T' cannot have a literal operator member}} expected-error +{{}}
     typename T::operator""_q(""); // expected-error +{{}} expected-note {{to match}}
-    T::operator""_q(""); // expected-error {{non-namespace scope 'T::' cannot have a literal operator member}}
+    T::operator""_q(""); // expected-error {{non-namespace scope 'T' cannot have a literal operator member}}
 
     bad_names::operator""_x<'a', 'b', 'c'>();
   };
 
   struct S {};
   void g() {
-    S::operator""_q(); // expected-error {{non-namespace scope 'S::' cannot have a literal operator member}}
+    S::operator""_q(); // expected-error {{non-namespace scope 'S' cannot have a literal operator member}}
   }
 }
diff --git a/clang/test/SemaCXX/cxx1z-noexcept-function-type.cpp b/clang/test/SemaCXX/cxx1z-noexcept-function-type.cpp
index 6b8495e8c9d95..c8204c21523a3 100644
--- a/clang/test/SemaCXX/cxx1z-noexcept-function-type.cpp
+++ b/clang/test/SemaCXX/cxx1z-noexcept-function-type.cpp
@@ -74,7 +74,6 @@ namespace DependentDefaultCtorExceptionSpec {
   };
   struct InstantiateFromAnotherClass {
     template <class B, class T = decltype(static_cast<bool (B::*)(int)>(&B::foo))> // expected-note {{in instantiation of function template specialization}}
-    // expected-note@-1 {{template parameter is declared here}}
     InstantiateFromAnotherClass(B *) {} // expected-note {{in instantiation of default argument}}
   };
   NoexceptWithThis<int> f{};
diff --git a/clang/test/SemaCXX/cxx2a-consteval.cpp b/clang/test/SemaCXX/cxx2a-consteval.cpp
index 931acfd86f808..fef4674d17841 100644
--- a/clang/test/SemaCXX/cxx2a-consteval.cpp
+++ b/clang/test/SemaCXX/cxx2a-consteval.cpp
@@ -201,7 +201,7 @@ struct A {
 
 using mem_ptr_type = int (A::*)(int);
 
-template<mem_ptr_type ptr> // expected-note 2{{template parameter is declared here}}
+template<mem_ptr_type ptr>
 struct C {};
 
 C<&A::f> c;
diff --git a/clang/test/SemaCXX/cxx98-compat-flags.cpp b/clang/test/SemaCXX/cxx98-compat-flags.cpp
index 3986251357aca..6ffb3a5884d17 100644
--- a/clang/test/SemaCXX/cxx98-compat-flags.cpp
+++ b/clang/test/SemaCXX/cxx98-compat-flags.cpp
@@ -1,8 +1,7 @@
 // RUN: %clang_cc1 -fsyntax-only -std=c++17 -Wc++98-compat-pedantic -verify %s
 // RUN: %clang_cc1 -fsyntax-only -std=c++17 -Wc++98-compat-pedantic -Wno-bind-to-temporary-copy -Wno-unnamed-type-template-args -Wno-local-type-template-args -Wno-binary-literal -Werror %s
 
-template<typename T> // expected-note 2{{template parameter is declared here}}
-int TemplateFn(T) { return 0; }
+template<typename T> int TemplateFn(T) { return 0; }
 void LocalTemplateArg() {
   struct S {};
   TemplateFn(S()); // expected-warning {{local type 'S' as template argument is incompatible with C++98}}
diff --git a/clang/test/SemaCXX/cxx98-compat.cpp b/clang/test/SemaCXX/cxx98-compat.cpp
index 99ed49481614b..d31d95a9995f1 100644
--- a/clang/test/SemaCXX/cxx98-compat.cpp
+++ b/clang/test/SemaCXX/cxx98-compat.cpp
@@ -173,8 +173,7 @@ struct DelegCtor {
 
 template<int n = 0> void DefaultFuncTemplateArg(); // expected-warning {{default template arguments for a function template are incompatible with C++98}}
 
-template<typename T> // expected-note 2{{template parameter is declared here}}
-int TemplateFn(T) { return 0; }
+template<typename T> int TemplateFn(T) { return 0; }
 void LocalTemplateArg() {
   struct S {};
   TemplateFn(S()); // expected-warning {{local type 'S' as template argument is incompatible with C++98}}
@@ -189,7 +188,7 @@ int UnnamedTemplateArg = TemplateFn(obj_of_unnamed_type); // expected-warning {{
 #ifndef CXX17COMPAT
 namespace RedundantParensInAddressTemplateParam {
   int n;
-  template<int*p> struct S {}; // expected-note 2{{template parameter is declared here}}
+  template<int*p> struct S {};
   S<(&n)> s; // expected-warning {{redundant parentheses surrounding address non-type template argument are incompatible with C++98}}
   S<(((&n)))> t; // expected-warning {{redundant parentheses surrounding address non-type template argument are incompatible with C++98}}
 }
@@ -312,7 +311,7 @@ namespace LiteralUCNs {
 // template argument evaluation rules.
 #ifndef CXX17COMPAT
 namespace NonTypeTemplateArgs {
-  template<typename T, T v> struct S {}; // expected-note 2{{template parameter is declared here}}
+  template<typename T, T v> struct S {};
   const int k = 5; // expected-note {{here}}
   static void f() {} // expected-note {{here}}
   S<const int&, k> s1; // expected-warning {{non-type template argument referring to object 'k' with internal linkage is incompatible with C++98}}
@@ -321,8 +320,8 @@ namespace NonTypeTemplateArgs {
 
 namespace NullPointerTemplateArg {
   struct A {};
-  template<int*> struct X {}; // expected-note {{template parameter is declared here}}
-  template<int A::*> struct Y {}; // expected-note {{template parameter is declared here}}
+  template<int*> struct X {};
+  template<int A::*> struct Y {};
   X<(int*)0> x; // expected-warning {{use of null pointer as non-type template argument is incompatible with C++98}}
   Y<(int A::*)0> y; // expected-warning {{use of null pointer as non-type template argument is incompatible with C++98}}
 }
diff --git a/clang/test/SemaCXX/decomposed-condition.cpp b/clang/test/SemaCXX/decomposed-condition.cpp
index e55bbee3134ca..14a23af3f5658 100644
--- a/clang/test/SemaCXX/decomposed-condition.cpp
+++ b/clang/test/SemaCXX/decomposed-condition.cpp
@@ -1,5 +1,7 @@
-// RUN: %clang_cc1 -std=c++1z -Wno-binding-in-condition -verify %s
-// RUN: %clang_cc1 -std=c++1z -Wno-binding-in-condition -verify %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++17 -Wno-c++26-extensions -verify %s
+// RUN: %clang_cc1 -std=c++17 -Wno-c++26-extensions -verify %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++2c -Wpre-c++26-compat -verify=cxx26,expected %s
+// RUN: %clang_cc1 -std=c++2c -Wpre-c++26-compat -verify=cxx26,expected %s -fexperimental-new-constant-interpreter
 
 struct X {
   bool flag;
@@ -14,7 +16,7 @@ struct X {
 
 namespace CondInIf {
 constexpr int f(X x) {
-  if (auto [ok, d] = x)
+  if (auto [ok, d] = x) // cxx26-warning {{structured binding declaration in a condition is incompatible with C++ standards before C++2c}}
     return d + int(ok);
   else
     return d * int(ok);
@@ -26,12 +28,13 @@ static_assert(f({true, 2}) == 3);
 static_assert(f({false, 2}) == 0);
 
 constexpr char g(char const (&x)[2]) {
-  if (auto &[a, b] = x)
+  if (auto &[a, b] = x) // cxx26-warning {{structured binding declaration in a condition is incompatible with C++ standards before C++2c}}
     return a;
   else
     return b;
 
-  if (auto [a, b] = x) // expected-error {{an array type is not allowed here}}
+  if (auto [a, b] = x) // expected-error {{an array type is not allowed here}} \
+                       // cxx26-warning {{structured binding declaration in a condition is incompatible with C++ standards before C++2c}}
     ;
 }
 
@@ -41,6 +44,7 @@ static_assert(g("x") == 'x');
 namespace CondInSwitch {
 constexpr int f(int n) {
   switch (X s = {true, n}; auto [ok, d] = s) {
+    // cxx26-warning@-1 {{structured binding declaration in a condition is incompatible with C++ standards before C++2c}}
     s = {};
   case 0:
     return int(ok);
@@ -65,6 +69,7 @@ namespace CondInWhile {
 constexpr int f(int n) {
   int m = 1;
   while (auto [ok, d] = X{n > 1, n}) {
+    // cxx26-warning@-1 {{structured binding declaration in a condition is incompatible with C++ standards before C++2c}}
     m *= d;
     --n;
   }
@@ -81,6 +86,7 @@ namespace CondInFor {
 constexpr int f(int n) {
   int a = 1, b = 1;
   for (X x = {true, n}; auto &[ok, d] = x; --d) {
+    // cxx26-warning@-1 {{structured binding declaration in a condition is incompatible with C++ standards before C++2c}}
     if (d < 2)
       ok = false;
     else {
diff --git a/clang/test/SemaCXX/generic-selection.cpp b/clang/test/SemaCXX/generic-selection.cpp
index 9e47dde3ea444..aa4a4c435adec 100644
--- a/clang/test/SemaCXX/generic-selection.cpp
+++ b/clang/test/SemaCXX/generic-selection.cpp
@@ -81,10 +81,10 @@ void func(struct S s) {
   // is an elaborated type specifier followed by the association's value and
   // it should work the same as in C.
   (void)_Generic(s, struct S : 1);
+  (void)_Generic(s, struct T : 1);
 
   // The rest of these cases test that we still produce a reasonable diagnostic
   // when referencing an unknown type or trying to define a type in other ways.
-  (void)_Generic(s, struct T : 1);            // expected-error {{type 'struct T' in generic association incomplete}}
   (void)_Generic(s, struct U { int a; } : 1); // expected-error {{'U' cannot be defined in a type specifier}}
   (void)_Generic(s, struct V : S);            // expected-error {{'S' does not refer to a value}}
   (void)_Generic(s, struct W : S { int b; } : 1); // expected-error {{expected '(' for function-style cast or type construction}}
diff --git a/clang/test/SemaCXX/implicit-member-functions.cpp b/clang/test/SemaCXX/implicit-member-functions.cpp
index b48f93778ff81..1554b1af5d59a 100644
--- a/clang/test/SemaCXX/implicit-member-functions.cpp
+++ b/clang/test/SemaCXX/implicit-member-functions.cpp
@@ -63,7 +63,6 @@ namespace Recursion {
     template<typename T,
              typename = typename InvokeCopyConstructor<typename T::type>::type>
     // expected-note@-1 {{in instantiation of template class}}
-    // expected-note@-2 {{template parameter is declared here}}
     A(const T &);
     // expected-note@-1 {{in instantiation of default argument}}
   };
diff --git a/clang/test/SemaCXX/lambda-expressions.cpp b/clang/test/SemaCXX/lambda-expressions.cpp
index f411ef581f216..2d2dde82a28e6 100644
--- a/clang/test/SemaCXX/lambda-expressions.cpp
+++ b/clang/test/SemaCXX/lambda-expressions.cpp
@@ -442,7 +442,7 @@ struct A {
   // expected-error@-1 {{field has incomplete type 'void'}}
 };
 
-template <typename F> // cxx03-note {{template parameter is declared here}}
+template <typename F>
 void g(F f) {
   auto a = A<decltype(f())>();
   // expected-note@-1 {{in instantiation of template class 'PR20731::A<void>' requested here}}
@@ -499,14 +499,12 @@ namespace error_in_transform_prototype {
 
 namespace PR21857 {
   template<typename Fn> struct fun : Fn {
-    // cxx03-note@-1 {{template parameter is declared here}}
     fun() = default;
     using Fn::operator();
   };
   template<typename Fn> fun<Fn> wrap(Fn fn); // cxx03-warning {{template argument uses unnamed type}}
-                                             // cxx03-note@-1 {{template parameter is declared here}}
   auto x = wrap([](){}); // cxx03-warning {{template argument uses unnamed type}} cxx03-note 2 {{unnamed type used in template argument was declared here}}
-                         // cxx03-note@-1 2{{while substituting deduced template arguments into function template}}
+                         // cxx03-note@-1 {{while substituting deduced template arguments into function template}}
 }
 
 namespace PR13987 {
diff --git a/clang/test/SemaCXX/make_integer_seq.cpp b/clang/test/SemaCXX/make_integer_seq.cpp
index 7fca9ed8ea35d..71b7b8260d4ab 100644
--- a/clang/test/SemaCXX/make_integer_seq.cpp
+++ b/clang/test/SemaCXX/make_integer_seq.cpp
@@ -48,6 +48,5 @@ using illformed2 = ErrorSeq<int, -5>; // expected-note{{in instantiation}}
 template <typename T, T N> void f() {}
 __make_integer_seq<f, int, 0> x; // expected-error{{template template parameter must be a class template or type alias template}}
 
-__make_integer_seq<__make_integer_seq, int, 10> PR28494; // expected-note{{template template argument is incompatible}}
+__make_integer_seq<__make_integer_seq, int, 10> PR28494; // expected-note{{different template parameters}}
 // expected-error@make_integer_seq.cpp:* {{template argument for template template parameter must be a class template or type alias template}}
-// expected-note@*:* 3{{template parameter from hidden source: template <class, type-parameter-1-0 ...> class}}
diff --git a/clang/test/SemaCXX/paren-list-agg-init.cpp b/clang/test/SemaCXX/paren-list-agg-init.cpp
index 61afba85e1dff..b2d0f2564bb59 100644
--- a/clang/test/SemaCXX/paren-list-agg-init.cpp
+++ b/clang/test/SemaCXX/paren-list-agg-init.cpp
@@ -357,3 +357,22 @@ ThroughAlias<int, 1> e(42);
 // beforecxx20-warning@-1 {{aggregate initialization of type 'ThroughAlias<int, 1>' (aka 'int[1]') from a parenthesized list of values is a C++20 extension}} 
 
 }
+
+namespace CXXParenListInitExpr {
+
+struct S {
+  int a, b;
+  bool flag = false;
+
+  constexpr bool operator==(S rhs) {
+    return a == rhs.a && b == rhs.b;
+  }
+};
+
+static_assert(S(1, 2) == S(1, 2)); // beforecxx20-warning 2{{C++20 extension}}
+
+static_assert(S(1, 2) == S(3, 4));
+// expected-error@-1 {{failed due to requirement 'CXXParenListInitExpr::S(1, 2) == CXXParenListInitExpr::S(3, 4)'}} \
+// beforecxx20-warning@-1 2{{C++20 extension}}
+
+}
diff --git a/clang/test/SemaCXX/pr25181-crash-on-invalid.cpp b/clang/test/SemaCXX/pr25181-crash-on-invalid.cpp
index 41178c95e8ad4..e0ece54270fec 100644
--- a/clang/test/SemaCXX/pr25181-crash-on-invalid.cpp
+++ b/clang/test/SemaCXX/pr25181-crash-on-invalid.cpp
@@ -3,5 +3,5 @@
 
 template <typename T> class Foo { // expected-note {{template parameter is declared here}}
   template <typename T> // expected-error {{declaration of 'T' shadows template parameter}}
-  void Foo<T>::method(T *) const throw() {} // expected-error {{nested name specifier 'Foo<T>::' for declaration does not refer into a class, class template or class template partial specialization}}
+  void Foo<T>::method(T *) const throw() {} // expected-error {{nested name specifier 'Foo<T>' for declaration does not refer into a class, class template or class template partial specialization}}
 };
diff --git a/clang/test/SemaCXX/pr36536.cpp b/clang/test/SemaCXX/pr36536.cpp
index 702b9ccb707e8..a923d79e25070 100644
--- a/clang/test/SemaCXX/pr36536.cpp
+++ b/clang/test/SemaCXX/pr36536.cpp
@@ -23,7 +23,7 @@ void Foo::f() {
 
 struct ImplicitDefaultCtor2 {};
 template <typename T> class TFoo { void f(); };
-// expected-error@+1 {{nested name specifier 'decltype(TFoo<T>())::'}}
+// expected-error@+1 {{nested name specifier 'decltype(TFoo<T>())'}}
 template <typename T> void decltype(TFoo<T>())::f() {
   switch (0) { case 0: ImplicitDefaultCtor1 o; }
 }
@@ -37,7 +37,7 @@ template <class T1> class A {
 };
 template <class Y>
 template <>
-// expected-error@+1 {{nested name specifier 'A<Y>::B<double>::'}}
+// expected-error@+1 {{nested name specifier 'A<Y>::B<double>'}}
 void A<Y>::B<double>::mf2() {
   switch (0) { case 0: ImplicitDefaultCtor3 o; }
 }
diff --git a/clang/test/SemaCXX/type-trait-common-type.cpp b/clang/test/SemaCXX/type-trait-common-type.cpp
index a59e0320e5306..7190dcad76f1a 100644
--- a/clang/test/SemaCXX/type-trait-common-type.cpp
+++ b/clang/test/SemaCXX/type-trait-common-type.cpp
@@ -5,12 +5,10 @@
 #  error
 #endif
 
-// expected-note@*:* {{template parameter from hidden source: template <class ...> class}}
-// expected-note@*:* {{template parameter from hidden source: class ...}}
-// expected-note@*:* 2{{template parameter from hidden source: template <class ...> class}}
+// expected-note@*:* {{template declaration from hidden source: template <template <class ...> class, template <class> class, class, class ...>}}
 
 void test() {
-  __builtin_common_type<> a; // expected-error {{missing template argument for template parameter}}
+  __builtin_common_type<> a; // expected-error {{too few template arguments for template '__builtin_common_type'}}
   __builtin_common_type<1> b; // expected-error {{template argument for template template parameter must be a class template or type alias template}}
   __builtin_common_type<int, 1> c; // expected-error {{template argument for template template parameter must be a class template or type alias template}}
 }
diff --git a/clang/test/SemaCXX/undefined-internal.cpp b/clang/test/SemaCXX/undefined-internal.cpp
index 7f466237370e6..9745f097c76b7 100644
--- a/clang/test/SemaCXX/undefined-internal.cpp
+++ b/clang/test/SemaCXX/undefined-internal.cpp
@@ -209,11 +209,9 @@ namespace OverloadUse {
     t<f>(&n, &n); // expected-note {{used here}}
 #if __cplusplus < 201103L
     // expected-warning@-3 {{non-type template argument referring to function 'f' with internal linkage}}
-    // expected-note@-7 {{template parameter is declared here}}
+    // expected-note@-4 {{while substituting explicitly-specified template arguments}}
+    // expected-warning@-4 {{non-type template argument referring to function 'f' with internal linkage}}
     // expected-note@-5 {{while substituting explicitly-specified template arguments}}
-    // expected-warning@-5 {{non-type template argument referring to function 'f' with internal linkage}}
-    // expected-note@-9 {{template parameter is declared here}}
-    // expected-note@-7 {{while substituting explicitly-specified template arguments}}
 #endif
   }
 }
diff --git a/clang/test/SemaCXX/uninitialized.cpp b/clang/test/SemaCXX/uninitialized.cpp
index 4af2c998f082e..c7b987e2172e6 100644
--- a/clang/test/SemaCXX/uninitialized.cpp
+++ b/clang/test/SemaCXX/uninitialized.cpp
@@ -892,6 +892,11 @@ namespace lambdas {
       return a1.x;
     });
     A a2([&] { return a2.x; }); // ok
+    A a3([=] { return a3.x; }()); // expected-warning{{variable 'a3' is uninitialized when used within its own initialization}}
+    A a4([&] { return a4.x; }()); // expected-warning{{variable 'a4' is uninitialized when used within its own initialization}}
+    A a5([&] { return a5; }()); // expected-warning{{variable 'a5' is uninitialized when used within its own initialization}}
+    A a6([&] { return a5.x; }()); // ok
+    A a7 = [&a7] { return a7; }(); // expected-warning{{variable 'a7' is uninitialized when used within its own initialization}}
   }
 }
 
diff --git a/clang/test/SemaCXX/using-decl-templates.cpp b/clang/test/SemaCXX/using-decl-templates.cpp
index 1cf4caee1c0db..c96c4879cb68c 100644
--- a/clang/test/SemaCXX/using-decl-templates.cpp
+++ b/clang/test/SemaCXX/using-decl-templates.cpp
@@ -8,22 +8,22 @@ template<typename T> struct A {
 template<typename T> struct B : A<T> {
   using A<T>::f;
   using A<T>::N; // expected-error{{dependent using declaration resolved to type without 'typename'}}
-  
+
   using A<T>::foo; // expected-error{{no member named 'foo'}}
-  using A<double>::f; // expected-error{{using declaration refers into 'A<double>::', which is not a base class of 'B<int>'}}
+  using A<double>::f; // expected-error{{using declaration refers into 'A<double>', which is not a base class of 'B<int>'}}
 };
 
 B<int> a; // expected-note{{in instantiation of template class 'B<int>' requested here}}
 
 template<typename T> struct C : A<T> {
   using A<T>::f;
-  
+
   void f() { };
 };
 
 template <typename T> struct D : A<T> {
   using A<T>::f;
-  
+
   void f();
 };
 
diff --git a/clang/test/SemaCXX/warn-deprecated-specializations-in-system-headers.cpp b/clang/test/SemaCXX/warn-deprecated-specializations-in-system-headers.cpp
index 1fe0916b3226c..79c9d339b6f20 100644
--- a/clang/test/SemaCXX/warn-deprecated-specializations-in-system-headers.cpp
+++ b/clang/test/SemaCXX/warn-deprecated-specializations-in-system-headers.cpp
@@ -10,7 +10,6 @@ template <>
 struct [[deprecated]] traits<int> {}; // expected-note {{'traits<int>' has been explicitly marked deprecated here}}
 
 template<typename T, typename Trait = traits<T>>  // expected-warning {{'traits<int>' is deprecated}}
-                                                  // expected-note@-1 {{template parameter is declared here}}
 struct basic_string {};
 
 // should not warn, defined and used in system headers
diff --git a/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp b/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp
index 30b6d4ba9fb90..e287f00a5453b 100644
--- a/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp
+++ b/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp
@@ -26,7 +26,7 @@ namespace std {
   _Tp* addressof(_Tp& __x) {
     return &__x;
   }
- 
+
 }
 
 namespace irrelevant_constructors {
@@ -154,6 +154,44 @@ namespace construct_wt_begin_end {
   }
 } // namespace construct_wt_begin_end
 
+namespace test_alloc_size_attr {
+  void * my_alloc(unsigned size) __attribute__((alloc_size(1)));
+  void * my_alloc2(unsigned count, unsigned size) __attribute__((alloc_size(1,2)));
+
+  void safe(int x, unsigned y) {
+    std::span<char>{(char *)my_alloc(10), 10};
+    std::span<char>{(char *)my_alloc(x), x};
+    std::span<char>{(char *)my_alloc(x * y), x * y};
+    std::span<char>{(char *)my_alloc(x * y), y * x};
+    std::span<char>{(char *)my_alloc(x * y + x), x * y + x};
+    std::span<char>{(char *)my_alloc(x * y + x), x + y * x};
+
+    std::span<char>{(char *)my_alloc2(x, y), x * y};
+    std::span<char>{(char *)my_alloc2(x, y), y * x};
+    //foo(std::span<char>{(char *)my_alloc2(x, sizeof(char)), x}); // lets not worry about this case for now
+    std::span<char>{(char *)my_alloc2(x, sizeof(char)), x * sizeof(char)};
+    //foo(std::span<char>{(char *)my_alloc2(10, sizeof(char)), 10});
+    std::span<char>{(char *)my_alloc2(10, sizeof(char)), 10 * sizeof(char)};
+  }
+
+  void unsafe(int x, int y) {
+    std::span<char>{(char *)my_alloc(10), 11};       // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
+    std::span<char>{(char *)my_alloc(x * y), x + y}; // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
+    std::span<int>{(int *)my_alloc(x), x};           // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
+    std::span<char>{(char *)my_alloc2(x, y), x + y}; // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
+  }
+
+  void unsupport(int x, int y, int z) {
+    // Casting to `T*` where sizeof(T) > 1 is not supported yet:
+    std::span<int>{(int *)my_alloc2(x, y),  x * y};  // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
+    std::span<long>{(long *)my_alloc(10 * sizeof(long)), 10}; // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
+    std::span<long>{(long *)my_alloc2(x, sizeof(long)), x};   // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
+    std::span<long>{(long *)my_alloc2(x, sizeof(long)), x};   // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
+    // The expression is too complicated:
+    std::span<char>{(char *)my_alloc(x + y + z), z + y + x};   // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
+  }
+}
+
 namespace test_flag {
   void f(int *p) {
 #pragma clang diagnostic push
diff --git a/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl b/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl
index e62cc8d24e3a7..941e0a975d5f4 100644
--- a/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl
@@ -8,7 +8,7 @@ typedef vector<double, 3> double3;
 // expected-error@+1 {{class template 'RWBuffer' requires template arguments}}
 RWBuffer BufferErr1;
 
-// expected-error@+1 {{missing template argument for template parameter}}
+// expected-error@+1 {{too few template arguments for class template 'RWBuffer'}}
 RWBuffer<> BufferErr2;
 
 // test implicit RWBuffer concept
@@ -39,7 +39,7 @@ template<typename T> struct TemplatedVector {
 
 // structs not allowed
 // expected-error@+4 {{constraints not satisfied for class template 'RWBuffer'}}
-// expected-note@*:* {{template parameter from hidden source: typename element_type}}
+// expected-note@*:* {{template declaration from hidden source: template <typename element_type> requires __is_typed_resource_element_compatible<element_type> class RWBuffer}}
 // expected-note@*:* {{because 's' does not satisfy '__is_typed_resource_element_compatible'}}
 // expected-note@*:* {{because '__builtin_hlsl_is_typed_resource_element_compatible(s)' evaluated to false}}
 RWBuffer<s> r6;
diff --git a/clang/test/SemaHLSL/BuiltIns/StructuredBuffers.hlsl b/clang/test/SemaHLSL/BuiltIns/StructuredBuffers.hlsl
index d5e2142cc55d2..fb14429025d5a 100644
--- a/clang/test/SemaHLSL/BuiltIns/StructuredBuffers.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/StructuredBuffers.hlsl
@@ -1,31 +1,31 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -fsyntax-only -verify %s
-
-typedef vector<float, 3> float3;
-
-StructuredBuffer<float3> Buffer;
-
-// expected-error@+2 {{class template 'StructuredBuffer' requires template arguments}}
-// expected-note@*:* {{template declaration from hidden source: template <typename element_type> requires __is_structured_resource_element_compatible<element_type> class StructuredBuffer {}}}
-StructuredBuffer BufferErr1;
-
-// expected-error@+2 {{missing template argument for template parameter}}
-// expected-note@*:* {{template parameter from hidden source: typename element_type}}
-StructuredBuffer<> BufferErr2;
-
-// test elements of 0 size
-// expected-error@+3{{constraints not satisfied for class template 'StructuredBuffer' [with element_type = int[0]]}}
-// expected-note@*:*{{because 'int[0]' does not satisfy '__is_structured_resource_element_compatible'}}
-// expected-note@*:*{{because 'sizeof(int[0]) >= 1UL' (0 >= 1) evaluated to false}}
-StructuredBuffer<int[0]> BufferErr3;
-
-// In C++, empty structs do have a size of 1. So should HLSL.
-// The concept will accept empty structs as element types, despite it being unintuitive.
-struct Empty {};
-StructuredBuffer<Empty> BufferErr4;
-
-
-[numthreads(1,1,1)]
-void main() {
-  (void)Buffer.__handle; // expected-error {{'__handle' is a private member of 'hlsl::StructuredBuffer<vector<float, 3>>'}}
-  // expected-note@* {{implicitly declared private here}}
-}
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -fsyntax-only -verify %s
+
+typedef vector<float, 3> float3;
+
+StructuredBuffer<float3> Buffer;
+
+// expected-error@+2 {{class template 'StructuredBuffer' requires template arguments}}
+// expected-note@*:* {{template declaration from hidden source: template <typename element_type> requires __is_structured_resource_element_compatible<element_type> class StructuredBuffer {}}}
+StructuredBuffer BufferErr1;
+
+// expected-error@+2 {{too few template arguments for class template 'StructuredBuffer'}}
+// expected-note@*:* {{template declaration from hidden source: template <typename element_type> requires __is_structured_resource_element_compatible<element_type> class StructuredBuffer {}}}
+StructuredBuffer<> BufferErr2;
+
+// test elements of 0 size
+// expected-error@+3{{constraints not satisfied for class template 'StructuredBuffer' [with element_type = int[0]]}}
+// expected-note@*:*{{because 'int[0]' does not satisfy '__is_structured_resource_element_compatible'}}
+// expected-note@*:*{{because 'sizeof(int[0]) >= 1UL' (0 >= 1) evaluated to false}}
+StructuredBuffer<int[0]> BufferErr3;
+
+// In C++, empty structs do have a size of 1. So should HLSL.
+// The concept will accept empty structs as element types, despite it being unintuitive.
+struct Empty {};
+StructuredBuffer<Empty> BufferErr4;
+
+
+[numthreads(1,1,1)]
+void main() {
+  (void)Buffer.__handle; // expected-error {{'__handle' is a private member of 'hlsl::StructuredBuffer<vector<float, 3>>'}}
+  // expected-note@* {{implicitly declared private here}}
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/distance-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/distance-errors.hlsl
index e996bf5d2cb7c..e7521c7251432 100644
--- a/clang/test/SemaHLSL/BuiltIns/distance-errors.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/distance-errors.hlsl
@@ -3,8 +3,8 @@
 float test_no_second_arg(float2 p0) {
   return distance(p0);
   // expected-error@-1 {{no matching function for call to 'distance'}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires 2 arguments, but 1 was provided}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires 2 arguments, but 1 was provided}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 1 was provided}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 1 was provided}}
   // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 1 was provided}}
   // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 1 was provided}}
 }
@@ -12,22 +12,46 @@ float test_no_second_arg(float2 p0) {
 float test_too_many_arg(float2 p0) {
   return distance(p0, p0, p0);
   // expected-error@-1 {{no matching function for call to 'distance'}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires 2 arguments, but 3 were provided}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires 2 arguments, but 3 were provided}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 3 were provided}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 3 were provided}}
   // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 3 were provided}}
   // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 3 were provided}}
 }
 
 float test_double_inputs(double p0, double p1) {
   return distance(p0, p1);
-  // expected-error@-1  {{call to 'distance' is ambiguous}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function}}
+  // expected-error@-1 {{no matching function for call to 'distance'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
 }
 
 float test_int_inputs(int p0, int p1) {
   return distance(p0, p1);
-  // expected-error@-1  {{call to 'distance' is ambiguous}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function}}
+  // expected-error@-1 {{no matching function for call to 'distance'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+}
+
+float1 test_vec1_inputs(float1 p0, float1 p1) {
+  return distance(p0, p1);
+  // expected-error@-1  {{no matching function for call to 'distance'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = float1]: no type named 'Type' in 'hlsl::__detail::enable_if<false, vector<float, 1>>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = float1]: no type named 'Type' in 'hlsl::__detail::enable_if<false, vector<float, 1>>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with N = 1]: no type named 'Type' in 'hlsl::__detail::enable_if<false, half>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with N = 1]: no type named 'Type' in 'hlsl::__detail::enable_if<false, float>'}}
+}
+
+typedef float float5 __attribute__((ext_vector_type(5)));
+
+float5 test_vec5_inputs(float5 p0, float5 p1) {
+  return distance(p0, p1);
+  // expected-error@-1  {{no matching function for call to 'distance'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = float5]: no type named 'Type' in 'hlsl::__detail::enable_if<false, vector<float, 5>>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = float5]: no type named 'Type' in 'hlsl::__detail::enable_if<false, vector<float, 5>>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with N = 5]: no type named 'Type' in 'hlsl::__detail::enable_if<false, half>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with N = 5]: no type named 'Type' in 'hlsl::__detail::enable_if<false, float>'}}
 }
diff --git a/clang/test/SemaHLSL/BuiltIns/exp-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/exp-errors.hlsl
index 321fc915ec01f..b21fefc372c3f 100644
--- a/clang/test/SemaHLSL/BuiltIns/exp-errors.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/exp-errors.hlsl
@@ -1,6 +1,7 @@
 
 // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -verify-ignore-unexpected -DTEST_FUNC=__builtin_elementwise_exp
 // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -verify-ignore-unexpected -DTEST_FUNC=__builtin_elementwise_exp2
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -verify-ignore-unexpected -DTEST_FUNC=__builtin_elementwise_exp10
 float test_too_few_arg() {
   return TEST_FUNC();
   // expected-error@-1 {{too few arguments to function call, expected 1, have 0}}
diff --git a/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl
index cdd130052b6a6..763368153e38f 100644
--- a/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl
@@ -6,6 +6,7 @@
 // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_cosh
 // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_exp
 // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_exp2
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_exp10
 // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_floor
 // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_log
 // RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm-only -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_log2
diff --git a/clang/test/SemaHLSL/BuiltIns/length-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/length-errors.hlsl
index a191f0419fbba..92eca64e66bd2 100644
--- a/clang/test/SemaHLSL/BuiltIns/length-errors.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/length-errors.hlsl
@@ -4,8 +4,8 @@ void test_too_few_arg()
 {
   return length();
   // expected-error@-1 {{no matching function for call to 'length'}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires single argument 'X', but no arguments were provided}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires single argument 'X', but no arguments were provided}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires single argument 'X', but no arguments were provided}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires single argument 'X', but no arguments were provided}}
   // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires single argument 'X', but no arguments were provided}}
   // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires single argument 'X', but no arguments were provided}}
 }
@@ -14,40 +14,68 @@ void test_too_many_arg(float2 p0)
 {
   return length(p0, p0);
   // expected-error@-1 {{no matching function for call to 'length'}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires single argument 'X', but 2 arguments were provided}}
   // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires single argument 'X', but 2 arguments were provided}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires single argument 'X', but 2 arguments were provided}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires single argument 'X', but 2 arguments were provided}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires single argument 'X', but 2 arguments were provided}}
   // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires single argument 'X', but 2 arguments were provided}}
 }
 
 float double_to_float_type(double p0) {
   return length(p0);
-  // expected-error@-1  {{call to 'length' is ambiguous}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function}}
+  // expected-error@-1  {{no matching function for call to 'length'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = double]: no type named 'Type' in 'hlsl::__detail::enable_if<false, double>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = double]: no type named 'Type' in 'hlsl::__detail::enable_if<false, double>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: could not match '__detail::HLSL_FIXED_VECTOR<half, N>' (aka 'vector<__detail::enable_if_t<(N > 1 && N <= 4), half>, N>') against 'double'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: could not match '__detail::HLSL_FIXED_VECTOR<float, N>' (aka 'vector<__detail::enable_if_t<(N > 1 && N <= 4), float>, N>') against 'double'}}
 }
 
 
 float bool_to_float_type_promotion(bool p1)
 {
   return length(p1);
-  // expected-error@-1  {{call to 'length' is ambiguous}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function}}
+  // expected-error@-1  {{no matching function for call to 'length'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{andidate template ignored: substitution failure [with T = bool]: no type named 'Type' in 'hlsl::__detail::enable_if<false, bool>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{andidate template ignored: substitution failure [with T = bool]: no type named 'Type' in 'hlsl::__detail::enable_if<false, bool>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: could not match '__detail::HLSL_FIXED_VECTOR<half, N>' (aka 'vector<__detail::enable_if_t<(N > 1 && N <= 4), half>, N>') against 'bool'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: could not match '__detail::HLSL_FIXED_VECTOR<float, N>' (aka 'vector<__detail::enable_if_t<(N > 1 && N <= 4), float>, N>') against 'bool'}}
 }
 
 float length_int_to_float_promotion(int p1)
 {
   return length(p1);
-  // expected-error@-1  {{call to 'length' is ambiguous}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function}}
+  // expected-error@-1  {{no matching function for call to 'length'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = int]: no type named 'Type' in 'hlsl::__detail::enable_if<false, int>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = int]: no type named 'Type' in 'hlsl::__detail::enable_if<false, int>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: could not match '__detail::HLSL_FIXED_VECTOR<half, N>' (aka 'vector<__detail::enable_if_t<(N > 1 && N <= 4), half>, N>') against 'int'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: could not match '__detail::HLSL_FIXED_VECTOR<float, N>' (aka 'vector<__detail::enable_if_t<(N > 1 && N <= 4), float>, N>') against 'int'}}
 }
 
 float2 length_int2_to_float2_promotion(int2 p1)
 {
   return length(p1);
-  // expected-error@-1  {{call to 'length' is ambiguous}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function}}
+  // expected-error@-1  {{no matching function for call to 'length'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = int2]}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = int2]}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{1st parameter does not match adjusted type 'vector<int, [...]>' of argument [with N = 2]}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{1st parameter does not match adjusted type 'vector<int, [...]>' of argument [with N = 2]}}
+}
+
+float1 test_vec1_inputs(float1 p0) {
+  return length(p0);
+  // expected-error@-1  {{no matching function for call to 'length'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = float1]: no type named 'Type' in 'hlsl::__detail::enable_if<false, vector<float, 1>>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = float1]: no type named 'Type' in 'hlsl::__detail::enable_if<false, vector<float, 1>>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with N = 1]: no type named 'Type' in 'hlsl::__detail::enable_if<false, half>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with N = 1]: no type named 'Type' in 'hlsl::__detail::enable_if<false, float>'}}
+}
+
+typedef float float5 __attribute__((ext_vector_type(5)));
+
+float5 test_vec5_inputs(float5 p0) {
+  return length(p0);
+  // expected-error@-1  {{no matching function for call to 'length'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = float5]: no type named 'Type' in 'hlsl::__detail::enable_if<false, vector<float, 5>>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = float5]: no type named 'Type' in 'hlsl::__detail::enable_if<false, vector<float, 5>>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with N = 5]: no type named 'Type' in 'hlsl::__detail::enable_if<false, half>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with N = 5]: no type named 'Type' in 'hlsl::__detail::enable_if<false, float>'}}
 }
diff --git a/clang/test/SemaHLSL/BuiltIns/reflect-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/reflect-errors.hlsl
index 28cf992ed602b..9934a3e525d38 100644
--- a/clang/test/SemaHLSL/BuiltIns/reflect-errors.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/reflect-errors.hlsl
@@ -3,8 +3,8 @@
 float test_no_second_arg(float2 p0) {
   return reflect(p0);
   // expected-error@-1 {{no matching function for call to 'reflect'}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires 2 arguments, but 1 was provided}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires 2 arguments, but 1 was provided}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 1 was provided}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 1 was provided}}
   // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 1 was provided}}
   // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 1 was provided}}
 }
@@ -12,22 +12,46 @@ float test_no_second_arg(float2 p0) {
 float test_too_many_arg(float2 p0) {
   return reflect(p0, p0, p0);
   // expected-error@-1 {{no matching function for call to 'reflect'}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires 2 arguments, but 3 were provided}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires 2 arguments, but 3 were provided}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 3 were provided}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 3 were provided}}
   // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 3 were provided}}
   // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 3 were provided}}
 }
 
 float test_double_inputs(double p0, double p1) {
   return reflect(p0, p1);
-  // expected-error@-1  {{call to 'reflect' is ambiguous}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function}}
+  // expected-error@-1  {{no matching function for call to 'reflect'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
 }
 
 float test_int_inputs(int p0, int p1) {
   return reflect(p0, p1);
-  // expected-error@-1  {{call to 'reflect' is ambiguous}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function}}
+  // expected-error@-1  {{no matching function for call to 'reflect'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+}
+
+float1 test_vec1_inputs(float1 p0, float1 p1) {
+  return reflect(p0, p1);
+  // expected-error@-1  {{no matching function for call to 'reflect'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = float1]: no type named 'Type' in 'hlsl::__detail::enable_if<false, vector<float, 1>>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = float1]: no type named 'Type' in 'hlsl::__detail::enable_if<false, vector<float, 1>>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with L = 1]: no type named 'Type' in 'hlsl::__detail::enable_if<false, half>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with L = 1]: no type named 'Type' in 'hlsl::__detail::enable_if<false, float>'}}
+}
+
+typedef float float5 __attribute__((ext_vector_type(5)));
+
+float5 test_vec5_inputs(float5 p0, float5 p1) {
+  return reflect(p0, p1);
+  // expected-error@-1  {{no matching function for call to 'reflect'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = float5]: no type named 'Type' in 'hlsl::__detail::enable_if<false, vector<float, 5>>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = float5]: no type named 'Type' in 'hlsl::__detail::enable_if<false, vector<float, 5>>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with L = 5]: no type named 'Type' in 'hlsl::__detail::enable_if<false, half>'}}
+  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with L = 5]: no type named 'Type' in 'hlsl::__detail::enable_if<false, float>'}}
 }
diff --git a/clang/test/SemaHLSL/BuiltIns/select-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/select-errors.hlsl
index 34b5fb6d54cd5..7000affaef77f 100644
--- a/clang/test/SemaHLSL/BuiltIns/select-errors.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/select-errors.hlsl
@@ -1,119 +1,65 @@
-// RUN: %clang_cc1 -finclude-default-header
-// -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only
-// -disable-llvm-passes -verify -verify-ignore-unexpected
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only -disable-llvm-passes -verify
 
-int test_no_arg() {
-  return select();
-  // expected-error@-1 {{no matching function for call to 'select'}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template
-  // not viable: requires 3 arguments, but 0 were provided}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not
-  // viable: requires 3 arguments, but 0 were provided}}
-}
-
-int test_too_few_args(bool p0) {
-  return select(p0);
-  // expected-error@-1 {{no matching function for call to 'select'}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not
-  // viable: requires 3 arguments, but 1 was provided}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not
-  // viable: requires 3 arguments, but 1 was provided}}
-}
-
-int test_too_many_args(bool p0, int t0, int f0, int g0) {
-  return select<int>(p0, t0, f0, g0);
-  // expected-error@-1 {{no matching function for call to 'select'}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not
-  // viable: requires 3 arguments, but 4 were provided}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not
-  // viable: requires 3 arguments, but 4 were provided}}
-}
 
 int test_select_first_arg_wrong_type(int1 p0, int t0, int f0) {
-  return select(p0, t0, f0);
-  // expected-error@-1 {{no matching function for call to 'select'}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not
-  // viable: no known conversion from 'vector<int, 1>' (vector of 1 'int' value)
-  // to 'bool' for 1st argument}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored: could
-  // not match 'vector<T, Sz>' against 'int'}}
+  return select(p0, t0, f0); // No diagnostic expected.
 }
 
 int1 test_select_bool_vals_diff_vecs(bool p0, int1 t0, int1 f0) {
-  return select<int1>(p0, t0, f0);
-  // expected-warning@-1 {{implicit conversion truncates vector:
-  // 'vector<int, 2>' (vector of 2 'int' values) to 'vector<int, 1>'
-  // (vector of 1 'int' value)}}
+  return select<int1>(p0, t0, f0); // No diagnostic expected.
 }
 
 int2 test_select_vector_vals_not_vecs(bool2 p0, int t0,
                                                int f0) {
-  return select(p0, t0, f0);
-  // expected-error@-1 {{no matching function for call to 'select'}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate template ignored:
-  // could not match 'vector<T, Sz>' against 'int'}}
-  // expected-note@hlsl/hlsl_intrinsics.h:* {{candidate function template not
-  // viable: no known conversion from 'vector<bool, 2>'
-  // (vector of 2 'bool' values) to 'bool' for 1st argument}}
+  return select(p0, t0, f0); // No diagnostic expected.
 }
 
 int1 test_select_vector_vals_wrong_size(bool2 p0, int1 t0, int1 f0) {
-  return select<int,1>(p0, t0, f0); // produce warnings
-  // expected-warning@-1 {{implicit conversion truncates vector:
-  // 'vector<bool, 2>' (vector of 2 'bool' values) to 'vector<bool, 1>'
-  // (vector of 1 'bool' value)}}
-  // expected-warning@-2 {{implicit conversion truncates vector:
-  // 'vector<int, 2>' (vector of 2 'int' values) to 'vector<int, 1>'
-  // (vector of 1 'int' value)}}
+  return select<int,1>(p0, t0, f0); // expected-warning{{implicit conversion truncates vector: 'bool2' (aka 'vector<bool, 2>') to 'vector<bool, 1>' (vector of 1 'bool' value)}}
+}
+
+int test_select_no_args() {
+  return __builtin_hlsl_select(); // expected-error{{too few arguments to function call, expected 3, have 0}}
+}
+
+int test_select_builtin_wrong_arg_count(bool p0) {
+  return __builtin_hlsl_select(p0); // expected-error{{too few arguments to function call, expected 3, have 1}}
 }
 
 // __builtin_hlsl_select tests
-int test_select_builtin_wrong_arg_count(bool p0, int t0) {
-  return __builtin_hlsl_select(p0, t0);
-  // expected-error@-1 {{too few arguments to function call, expected 3,
-  // have 2}}
+int test_select_builtin_wrong_arg_count2(bool p0, int t0) {
+  return __builtin_hlsl_select(p0, t0); // expected-error{{too few arguments to function call, expected 3, have 2}}
+}
+
+int test_too_many_args(bool p0, int t0, int f0, int g0) {
+  return __builtin_hlsl_select(p0, t0, f0, g0); // expected-error{{too many arguments to function call, expected 3, have 4}}
 }
 
 // not a bool or a vector of bool. should be 2 errors.
 int test_select_builtin_first_arg_wrong_type1(int p0, int t0, int f0) {
-  return __builtin_hlsl_select(p0, t0, f0);
-  // expected-error@-1 {{passing 'int' to parameter of incompatible type
-  // 'bool'}}
-  // expected-error@-2 {{First argument to __builtin_hlsl_select must be of
-  // vector type}}
-  }
+  return __builtin_hlsl_select(p0, t0, f0); // expected-error{{invalid operand of type 'int' where 'bool' or a vector of such type is required}}
+}
 
 int test_select_builtin_first_arg_wrong_type2(int1 p0, int t0, int f0) {
-  return __builtin_hlsl_select(p0, t0, f0);
-  // expected-error@-1 {{passing 'vector<int, 1>' (vector of 1 'int' value) to
-  // parameter of incompatible type 'bool'}}
-  // expected-error@-2 {{First argument to __builtin_hlsl_select must be of
-  // vector type}}
+  return __builtin_hlsl_select(p0, t0, f0); // expected-error{{invalid operand of type 'int1' (aka 'vector<int, 1>') where 'bool' or a vector of such type is required}}
 }
 
 // if a bool last 2 args are of same type
 int test_select_builtin_bool_incompatible_args(bool p0, int t0, double f0) {
-  return __builtin_hlsl_select(p0, t0, f0);
-  // expected-error@-1 {{arguments are of different types ('int' vs 'double')}}
+  return __builtin_hlsl_select(p0, t0, f0); // expected-error{{arguments are of different types ('int' vs 'double')}}
 }
 
 // if a vector second arg isnt a vector
 int2 test_select_builtin_second_arg_not_vector(bool2 p0, int t0, int2 f0) {
-  return __builtin_hlsl_select(p0, t0, f0);
-  // expected-error@-1 {{Second argument to __builtin_hlsl_select must be of
-  // vector type}}
+  return __builtin_hlsl_select(p0, t0, f0); // No diagnostic expected.
 }
 
 // if a vector third arg isn't a vector
 int2 test_select_builtin_second_arg_not_vector(bool2 p0, int2 t0, int f0) {
-  return __builtin_hlsl_select(p0, t0, f0);
-  // expected-error@-1 {{Third argument to __builtin_hlsl_select must be of
-  // vector type}}
+  return __builtin_hlsl_select(p0, t0, f0); // No diagnostic expected.
 }
 
 // if vector last 2 aren't same type (so both are vectors but wrong type)
-int2 test_select_builtin_diff_types(bool1 p0, int1 t0, float1 f0) {
-  return __builtin_hlsl_select(p0, t0, f0);
-  // expected-error@-1 {{arguments are of different types ('vector<int, [...]>'
-  // vs 'vector<float, [...]>')}}
+int1 test_select_builtin_diff_types(bool1 p0, int1 t0, float1 f0) {
+  return __builtin_hlsl_select(p0, t0, f0); // expected-error{{second and third arguments to __builtin_hlsl_select must be of scalar or vector type with matching scalar element type: 'vector<int, [...]>' vs 'vector<float, [...]>'}}
 }
diff --git a/clang/test/SemaHLSL/Language/NoVirtual.hlsl b/clang/test/SemaHLSL/Language/NoVirtual.hlsl
new file mode 100644
index 0000000000000..8d61bde7d836e
--- /dev/null
+++ b/clang/test/SemaHLSL/Language/NoVirtual.hlsl
@@ -0,0 +1,14 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -verify %s
+
+struct Base {
+  int X;
+  void MemberFunction();  // valid
+  virtual void MemberFunction2(); // expected-error{{virtual functions are unsupported in HLSL}}
+};
+
+struct Derived : virtual Base { // expected-error{{virtual inheritance is unsupported in HLSL}}
+  int Y;
+
+  void MemberFunction2() override; // expected-error{{only virtual member functions can be marked 'override'}}
+};
+
diff --git a/clang/test/SemaHLSL/Semantics/valid_entry_parameter.hlsl b/clang/test/SemaHLSL/Semantics/valid_entry_parameter.hlsl
index 6781f9241df24..a2203692b582b 100644
--- a/clang/test/SemaHLSL/Semantics/valid_entry_parameter.hlsl
+++ b/clang/test/SemaHLSL/Semantics/valid_entry_parameter.hlsl
@@ -24,6 +24,12 @@ void CSMain3(uint3 : SV_DispatchThreadID) {
 // CHECK-NEXT: ParmVarDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> col:20 'uint3'
 // CHECK-NEXT: HLSLSV_DispatchThreadIDAttr
 }
+[numthreads(8,8,1)]
+void CSMain4(uint3 : SV_DispatchThreadId) {
+// CHECK: FunctionDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> line:[[@LINE-1]]:6 CSMain4 'void (uint3)'
+// CHECK-NEXT: ParmVarDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> col:20 'uint3'
+// CHECK-NEXT: HLSLSV_DispatchThreadIDAttr
+}
 
 [numthreads(8,8,1)]
 void CSMain_GID(uint ID : SV_GroupID) {
@@ -49,6 +55,12 @@ void CSMain3_GID(uint3 : SV_GroupID) {
 // CHECK-NEXT: ParmVarDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> col:24 'uint3'
 // CHECK-NEXT: HLSLSV_GroupIDAttr
 }
+[numthreads(8,8,1)]
+void CSMain4_GID(uint3 : Sv_GroupId) {
+// CHECK: FunctionDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> line:[[@LINE-1]]:6 CSMain4_GID 'void (uint3)'
+// CHECK-NEXT: ParmVarDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> col:24 'uint3'
+// CHECK-NEXT: HLSLSV_GroupIDAttr
+}
 
 [numthreads(8,8,1)]
 void CSMain_GThreadID(uint ID : SV_GroupThreadID) {
@@ -74,3 +86,9 @@ void CSMain3_GThreadID(uint3 : SV_GroupThreadID) {
 // CHECK-NEXT: ParmVarDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> col:30 'uint3'
 // CHECK-NEXT: HLSLSV_GroupThreadIDAttr
 }
+[numthreads(8,8,1)]
+void CSMain4_GThreadID(uint3 : sv_GroupThreadid) {
+// CHECK: FunctionDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> line:[[@LINE-1]]:6 CSMain4_GThreadID 'void (uint3)'
+// CHECK-NEXT: ParmVarDecl 0x{{[0-9a-fA-F]+}} <{{.*}}> col:30 'uint3'
+// CHECK-NEXT: HLSLSV_GroupThreadIDAttr
+}
diff --git a/clang/test/SemaHLSL/Types/BuiltinVector/BooleanVectorConstantExpr.hlsl b/clang/test/SemaHLSL/Types/BuiltinVector/BooleanVectorConstantExpr.hlsl
new file mode 100644
index 0000000000000..1d368befc839a
--- /dev/null
+++ b/clang/test/SemaHLSL/Types/BuiltinVector/BooleanVectorConstantExpr.hlsl
@@ -0,0 +1,9 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.6-library -finclude-default-header -std=hlsl202x -verify %s
+
+// expected-no-diagnostics
+
+export void fn() {
+  _Static_assert((true.xxxx).y == true, "Woo!");
+
+  _Static_assert((true.xx).x && false == false, "Woo!");
+}
diff --git a/clang/test/SemaObjCXX/parameterized_classes_subst.mm b/clang/test/SemaObjCXX/parameterized_classes_subst.mm
index e113bea22f8a8..8aacf21faf091 100644
--- a/clang/test/SemaObjCXX/parameterized_classes_subst.mm
+++ b/clang/test/SemaObjCXX/parameterized_classes_subst.mm
@@ -408,7 +408,7 @@ void testVariadicInstantiation() {
 // --------------------------------------------------------------------------
 // Parameterized classes are not templates
 // --------------------------------------------------------------------------
-template<template<typename T, typename U> class TT> // expected-note {{template parameter is declared here}}
+template<template<typename T, typename U> class TT>
 struct AcceptsTemplateTemplate { };
 
 typedef AcceptsTemplateTemplate<NSMutableDictionary> TemplateTemplateFail1; // expected-error{{template argument for template template parameter must be a class template or type alias template}}
diff --git a/clang/test/SemaObjCXX/propert-dot-error.mm b/clang/test/SemaObjCXX/propert-dot-error.mm
index 31ee43a3c27d8..377f50de68d11 100644
--- a/clang/test/SemaObjCXX/propert-dot-error.mm
+++ b/clang/test/SemaObjCXX/propert-dot-error.mm
@@ -60,8 +60,8 @@ @interface D { // expected-note 2 {{'D' declared here}}
 @end
 
 void testD(D *d) {
-  d.Forward::property = 17; // expected-error{{property access cannot be qualified with 'Forward::'}}
-  d->Forward::ivar = 12; // expected-error{{instance variable access cannot be qualified with 'Forward::'}}
+  d.Forward::property = 17; // expected-error{{property access cannot be qualified with 'Forward'}}
+  d->Forward::ivar = 12; // expected-error{{instance variable access cannot be qualified with 'Forward'}}
   d.D::property = 17; // expected-error{{'D' is not a class, namespace, or enumeration}}
   d->D::ivar = 12; // expected-error{{'D' is not a class, namespace, or enumeration}}
 }
diff --git a/clang/test/SemaOpenACC/combined-construct-auto_seq_independent-clauses.c b/clang/test/SemaOpenACC/combined-construct-auto_seq_independent-clauses.c
index 1cdc657df0460..dbba86a1ffc61 100644
--- a/clang/test/SemaOpenACC/combined-construct-auto_seq_independent-clauses.c
+++ b/clang/test/SemaOpenACC/combined-construct-auto_seq_independent-clauses.c
@@ -125,7 +125,7 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop auto collapse(1)
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented}}
+  // expected-error@+1{{OpenACC 'bind' clause is not valid on 'parallel loop' directive}}
 #pragma acc parallel loop auto bind(Var)
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop auto vector_length(1)
@@ -242,7 +242,7 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop collapse(1) auto
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented}}
+  // expected-error@+1{{OpenACC 'bind' clause is not valid on 'parallel loop' directive}}
 #pragma acc parallel loop bind(Var) auto
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop vector_length(1) auto
@@ -360,7 +360,7 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop independent collapse(1)
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented}}
+  // expected-error@+1{{OpenACC 'bind' clause is not valid on 'parallel loop' directive}}
 #pragma acc parallel loop independent bind(Var)
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop independent vector_length(1)
@@ -477,7 +477,7 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop collapse(1) independent
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented}}
+  // expected-error@+1{{OpenACC 'bind' clause is not valid on 'parallel loop' directive}}
 #pragma acc parallel loop bind(Var) independent
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop vector_length(1) independent
@@ -603,7 +603,7 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop seq collapse(1)
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented}}
+  // expected-error@+1{{OpenACC 'bind' clause is not valid on 'parallel loop' directive}}
 #pragma acc parallel loop seq bind(Var)
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop seq vector_length(1)
@@ -726,7 +726,7 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop collapse(1) seq
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented}}
+  // expected-error@+1{{OpenACC 'bind' clause is not valid on 'parallel loop' directive}}
 #pragma acc parallel loop bind(Var) seq
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc parallel loop vector_length(1) seq
diff --git a/clang/test/SemaOpenACC/combined-construct-device_type-clause.c b/clang/test/SemaOpenACC/combined-construct-device_type-clause.c
index 49987663c79e1..38c72551997b8 100644
--- a/clang/test/SemaOpenACC/combined-construct-device_type-clause.c
+++ b/clang/test/SemaOpenACC/combined-construct-device_type-clause.c
@@ -173,8 +173,7 @@ void uses() {
   for(int i = 0; i < 5; ++i);
 #pragma acc serial loop device_type(*) collapse(1)
   for(int i = 0; i < 5; ++i);
-  // expected-error@+2{{OpenACC clause 'bind' may not follow a 'device_type' clause in a 'parallel loop' construct}}
-  // expected-note@+1{{previous clause is here}}
+  // expected-error@+1{{OpenACC 'bind' clause is not valid on 'parallel loop' directive}}
 #pragma acc parallel loop device_type(*) bind(Var)
   for(int i = 0; i < 5; ++i);
   // expected-error@+1{{OpenACC 'vector_length' clause is not valid on 'serial loop' directive}}
diff --git a/clang/test/SemaOpenACC/compute-construct-device_type-clause.c b/clang/test/SemaOpenACC/compute-construct-device_type-clause.c
index a41936b57f09a..1e74e5ac23377 100644
--- a/clang/test/SemaOpenACC/compute-construct-device_type-clause.c
+++ b/clang/test/SemaOpenACC/compute-construct-device_type-clause.c
@@ -180,8 +180,7 @@ void uses() {
   // expected-error@+1{{OpenACC 'collapse' clause is not valid on 'kernels' directive}}
 #pragma acc kernels device_type(*) collapse(1)
   while(1);
-  // expected-error@+2{{OpenACC clause 'bind' may not follow a 'device_type' clause in a 'kernels' construct}}
-  // expected-note@+1{{previous clause is here}}
+  // expected-error@+1{{OpenACC 'bind' clause is not valid on 'kernels' directive}}
 #pragma acc kernels device_type(*) bind(Var)
   while(1);
 #pragma acc kernels device_type(*) vector_length(1)
diff --git a/clang/test/SemaOpenACC/loop-construct-auto_seq_independent-clauses.c b/clang/test/SemaOpenACC/loop-construct-auto_seq_independent-clauses.c
index 9dde1a96720d0..2f2bd15935856 100644
--- a/clang/test/SemaOpenACC/loop-construct-auto_seq_independent-clauses.c
+++ b/clang/test/SemaOpenACC/loop-construct-auto_seq_independent-clauses.c
@@ -150,7 +150,7 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc loop auto collapse(1)
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented}}
+  // expected-error@+1{{OpenACC 'bind' clause is not valid on 'loop' directive}}
 #pragma acc loop auto bind(Var)
   for(unsigned i = 0; i < 5; ++i);
   // expected-error@+1{{OpenACC 'vector_length' clause is not valid on 'loop' directive}}
@@ -284,7 +284,7 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc loop collapse(1) auto
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented}}
+  // expected-error@+1{{OpenACC 'bind' clause is not valid on 'loop' directive}}
 #pragma acc loop bind(Var) auto
   for(unsigned i = 0; i < 5; ++i);
   // expected-error@+1{{OpenACC 'vector_length' clause is not valid on 'loop' directive}}
@@ -419,7 +419,7 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc loop independent collapse(1)
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented}}
+  // expected-error@+1{{OpenACC 'bind' clause is not valid on 'loop' directive}}
 #pragma acc loop independent bind(Var)
   for(unsigned i = 0; i < 5; ++i);
   // expected-error@+1{{OpenACC 'vector_length' clause is not valid on 'loop' directive}}
@@ -553,7 +553,7 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc loop collapse(1) independent
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented}}
+  // expected-error@+1{{OpenACC 'bind' clause is not valid on 'loop' directive}}
 #pragma acc loop bind(Var) independent
   for(unsigned i = 0; i < 5; ++i);
   // expected-error@+1{{OpenACC 'vector_length' clause is not valid on 'loop' directive}}
@@ -696,7 +696,7 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc loop seq collapse(1)
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented}}
+  // expected-error@+1{{OpenACC 'bind' clause is not valid on 'loop' directive}}
 #pragma acc loop seq bind(Var)
   for(unsigned i = 0; i < 5; ++i);
   // expected-error@+1{{OpenACC 'vector_length' clause is not valid on 'loop' directive}}
@@ -836,7 +836,7 @@ void uses() {
   for(unsigned i = 0; i < 5; ++i);
 #pragma acc loop collapse(1) seq
   for(unsigned i = 0; i < 5; ++i);
-  // expected-warning@+1{{OpenACC clause 'bind' not yet implemented}}
+  // expected-error@+1{{OpenACC 'bind' clause is not valid on 'loop' directive}}
 #pragma acc loop bind(Var) seq
   for(unsigned i = 0; i < 5; ++i);
   // expected-error@+1{{OpenACC 'vector_length' clause is not valid on 'loop' directive}}
diff --git a/clang/test/SemaOpenACC/loop-construct-device_type-clause.c b/clang/test/SemaOpenACC/loop-construct-device_type-clause.c
index 83100b80bff91..5c5e76881a889 100644
--- a/clang/test/SemaOpenACC/loop-construct-device_type-clause.c
+++ b/clang/test/SemaOpenACC/loop-construct-device_type-clause.c
@@ -152,8 +152,7 @@ void uses() {
   for(int i = 0; i < 5; ++i);
 #pragma acc loop device_type(*) collapse(1)
   for(int i = 0; i < 5; ++i);
-  // expected-error@+2{{OpenACC clause 'bind' may not follow a 'device_type' clause in a 'loop' construct}}
-  // expected-note@+1{{previous clause is here}}
+  // expected-error@+1{{OpenACC 'bind' clause is not valid on 'loop' directive}}
 #pragma acc loop device_type(*) bind(Var)
   for(int i = 0; i < 5; ++i);
   // expected-error@+1{{OpenACC 'vector_length' clause is not valid on 'loop' directive}}
diff --git a/clang/test/SemaOpenACC/routine-construct-ast.cpp b/clang/test/SemaOpenACC/routine-construct-ast.cpp
index 8fe2c3695826f..7715ec62e7d4e 100644
--- a/clang/test/SemaOpenACC/routine-construct-ast.cpp
+++ b/clang/test/SemaOpenACC/routine-construct-ast.cpp
@@ -7,17 +7,32 @@
 #ifndef PCH_HELPER
 #define PCH_HELPER
 auto Lambda = [](){};
-#pragma acc routine(Lambda) worker nohost
+#pragma acc routine(Lambda) worker nohost bind("string")
 // CHECK: OpenACCRoutineDecl{{.*}} routine name_specified
 // CHECK-NEXT: DeclRefExpr{{.*}} 'Lambda' '(lambda at
 // CHECK-NEXT: worker clause
 // CHECK-NEXT: nohost clause
+// CHECK-NEXT: bind clause
+// CHECK-NEXT: StringLiteral{{.*}} "string"
 int function();
-#pragma acc routine (function) nohost vector
+#pragma acc routine (function) nohost vector bind(identifier)
 // CHECK: OpenACCRoutineDecl{{.*}} routine name_specified
 // CHECK-NEXT: DeclRefExpr{{.*}} 'function' 'int ()'
 // CHECK-NEXT: nohost clause
 // CHECK-NEXT: vector clause
+// CHECK-NEXT: bind clause identifier 'identifier'
+
+#pragma acc routine(function) device_type(Something) seq
+// CHECK-NEXT: OpenACCRoutineDecl{{.*}} routine name_specified
+// CHECK-NEXT: DeclRefExpr{{.*}} 'function' 'int ()'
+// CHECK-NEXT: device_type(Something)
+// CHECK-NEXT: seq clause
+#pragma acc routine(function) nohost dtype(Something) vector
+// CHECK-NEXT: OpenACCRoutineDecl{{.*}} routine name_specified
+// CHECK-NEXT: DeclRefExpr{{.*}} 'function' 'int ()'
+// CHECK-NEXT: nohost clause
+// CHECK-NEXT: dtype(Something)
+// CHECK-NEXT: vector clause
 
 namespace NS {
   // CHECK-NEXT: NamespaceDecl
@@ -77,6 +92,16 @@ struct S {
 // CHECK-NEXT: OpenACCRoutineDecl{{.*}} routine name_specified
 // CHECK-NEXT: DeclRefExpr{{.*}} 'Lambda' 'const S::(lambda at
 // CHECK-NEXT: worker clause
+#pragma acc routine(Lambda) worker device_type(Lambda)
+// CHECK-NEXT: OpenACCRoutineDecl{{.*}} routine name_specified
+// CHECK-NEXT: DeclRefExpr{{.*}} 'Lambda' 'const S::(lambda at
+// CHECK-NEXT: worker clause
+// CHECK-NEXT: device_type(Lambda)
+#pragma acc routine(Lambda) dtype(Lambda) vector
+// CHECK-NEXT: OpenACCRoutineDecl{{.*}} routine name_specified
+// CHECK-NEXT: DeclRefExpr{{.*}} 'Lambda' 'const S::(lambda at
+// CHECK-NEXT: dtype(Lambda)
+// CHECK-NEXT: vector clause
 };
 
 #pragma acc routine(S::MemFunc) gang(dim: 1)
@@ -150,6 +175,12 @@ struct DepS {
 // CHECK-NEXT: DeclRefExpr{{.*}} 'StaticMemFunc' 'T ()'
 // CHECK-NEXT: NestedNameSpecifier{{.*}} 'DepS<T>'
 // CHECK-NEXT: worker clause
+#pragma acc routine(DepS<T>::StaticMemFunc) worker device_type(T)
+// CHECK-NEXT: OpenACCRoutineDecl{{.*}} routine name_specified
+// CHECK-NEXT: DeclRefExpr{{.*}} 'StaticMemFunc' 'T ()'
+// CHECK-NEXT: NestedNameSpecifier{{.*}} 'DepS<T>'
+// CHECK-NEXT: worker clause
+// CHECK-NEXT: device_type(T)
 
 // Instantiation:
 // CHECK: ClassTemplateSpecializationDecl{{.*}}struct DepS
@@ -200,6 +231,12 @@ struct DepS {
 // CHECK-NEXT: DeclRefExpr{{.*}} 'StaticMemFunc' 'int ()'
 // CHECK-NEXT: NestedNameSpecifier{{.*}} 'DepS<int>'
 // CHECK-NEXT: worker clause
+
+// CHECK-NEXT: OpenACCRoutineDecl{{.*}} routine name_specified
+// CHECK-NEXT: DeclRefExpr{{.*}} 'StaticMemFunc' 'int ()'
+// CHECK-NEXT: NestedNameSpecifier{{.*}} 'DepS<int>'
+// CHECK-NEXT: worker clause
+// CHECK-NEXT: device_type(T)
 };
 
 #pragma acc routine(DepS<int>::Lambda) gang(dim:1)
@@ -229,20 +266,23 @@ void TemplFunc() {
 // CHECK-NEXT: NestedNameSpecifier{{.*}} 'T'
 // CHECK-NEXT: gang clause
 // CHECK-NEXT: CallExpr{{.*}}'<dependent type>'
-#pragma acc routine(T::StaticMemFunc) nohost worker
+#pragma acc routine(T::StaticMemFunc) nohost worker bind("string")
 // CHECK-NEXT: DeclStmt
 // CHECK-NEXT: OpenACCRoutineDecl{{.*}} routine name_specified
 // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}}'<dependent type>'
 // CHECK-NEXT: NestedNameSpecifier{{.*}} 'T'
 // CHECK-NEXT: nohost clause
 // CHECK-NEXT: worker clause
-#pragma acc routine(T::Lambda) seq nohost
+// CHECK-NEXT: bind clause
+// CHECK-NEXT: StringLiteral{{.*}} "string"
+#pragma acc routine(T::Lambda) seq nohost bind(identifier)
 // CHECK-NEXT: DeclStmt
 // CHECK-NEXT: OpenACCRoutineDecl{{.*}} routine name_specified
 // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}}'<dependent type>'
 // CHECK-NEXT: NestedNameSpecifier{{.*}} 'T'
 // CHECK-NEXT: seq clause
 // CHECK-NEXT: nohost clause
+// CHECK-NEXT: bind clause identifier 'identifier'
 
 // Instantiation:
 // CHECK: FunctionDecl{{.*}} TemplFunc 'void ()' implicit_instantiation
@@ -260,6 +300,8 @@ void TemplFunc() {
 // CHECK-NEXT: NestedNameSpecifier{{.*}} 'S'
 // CHECK-NEXT: nohost clause
 // CHECK-NEXT: worker clause
+// CHECK-NEXT: bind clause
+// CHECK-NEXT: StringLiteral{{.*}} "string"
 
 // CHECK-NEXT: DeclStmt
 // CHECK-NEXT: OpenACCRoutineDecl{{.*}} routine name_specified
@@ -267,6 +309,7 @@ void TemplFunc() {
 // CHECK-NEXT: NestedNameSpecifier{{.*}} 'S'
 // CHECK-NEXT: seq clause
 // CHECK-NEXT: nohost clause
+// CHECK-NEXT: bind clause identifier 'identifier'
 }
 
 void usage() {
diff --git a/clang/test/SemaOpenACC/routine-construct-clauses.cpp b/clang/test/SemaOpenACC/routine-construct-clauses.cpp
index ac9289b798a3e..da8ed382c4d63 100644
--- a/clang/test/SemaOpenACC/routine-construct-clauses.cpp
+++ b/clang/test/SemaOpenACC/routine-construct-clauses.cpp
@@ -1,6 +1,7 @@
 // RUN: %clang_cc1 %s -fopenacc -verify
 
 void Func();
+void Func2();
 
 #pragma acc routine(Func) worker
 #pragma acc routine(Func) vector nohost
@@ -121,3 +122,46 @@ struct DependentT {
 void Inst() {
   DependentT<HasFuncs> T;// expected-note{{in instantiation of}}
 }
+
+#pragma acc routine(Func) gang device_type(Inst)
+#pragma acc routine(Func) gang dtype(Inst)
+#pragma acc routine(Func) device_type(*) worker
+#pragma acc routine(Func) dtype(*) worker
+
+#pragma acc routine(Func) dtype(*) gang
+#pragma acc routine(Func) device_type(*) worker
+#pragma acc routine(Func) device_type(*) vector
+#pragma acc routine(Func) dtype(*) seq
+#pragma acc routine(Func) seq device_type(*) bind("asdf")
+#pragma acc routine(Func2) seq device_type(*) bind(WhateverElse)
+#pragma acc routine(Func) seq dtype(*) device_type(*)
+// expected-error@+2{{OpenACC clause 'nohost' may not follow a 'dtype' clause in a 'routine' construct}}
+// expected-note@+1{{previous clause is here}}
+#pragma acc routine(Func) seq dtype(*) nohost
+// expected-error@+2{{OpenACC clause 'nohost' may not follow a 'device_type' clause in a 'routine' construct}}
+// expected-note@+1{{previous clause is here}}
+#pragma acc routine(Func) seq device_type(*) nohost
+
+// 2.15: a bind clause may not bind to a routine name that has a visible bind clause.
+void Func3();
+#pragma acc routine(Func3) seq bind("asdf")
+// OK: Doesn't have a bind
+#pragma acc routine(Func3) seq
+// expected-error@+2{{multiple 'routine' directives with 'bind' clauses are not permitted to refer to the same function}}
+// expected-note@-4{{previous clause is here}}
+#pragma acc routine(Func3) seq bind("asdf")
+
+void Func4();
+#pragma acc routine(Func4) seq
+// OK: Doesn't have a bind
+#pragma acc routine(Func4) seq bind("asdf")
+// expected-error@+2{{multiple 'routine' directives with 'bind' clauses are not permitted to refer to the same function}}
+// expected-note@-2{{previous clause is here}}
+#pragma acc routine(Func4) seq bind("asdf")
+void Func5();
+#pragma acc routine(Func5) seq bind("asdf")
+// expected-error@+2{{multiple 'routine' directives with 'bind' clauses are not permitted to refer to the same function}}
+// expected-note@-2{{previous clause is here}}
+#pragma acc routine(Func5) seq bind("asdf")
+// OK: Doesn't have a bind
+#pragma acc routine(Func5) seq
diff --git a/clang/test/SemaTemplate/alias-templates.cpp b/clang/test/SemaTemplate/alias-templates.cpp
index c27b193361654..ab5cad72faf1b 100644
--- a/clang/test/SemaTemplate/alias-templates.cpp
+++ b/clang/test/SemaTemplate/alias-templates.cpp
@@ -278,12 +278,12 @@ namespace PR31514 {
 namespace an_alias_template_is_not_a_class_template {
   template<typename T> using Foo = int; // expected-note 3{{here}}
   Foo x; // expected-error {{use of alias template 'Foo' requires template arguments}}
-  Foo<> y; // expected-error {{missing template argument for template parameter}}
+  Foo<> y; // expected-error {{too few template arguments for alias template 'Foo'}}
   int z = Foo(); // expected-error {{use of alias template 'Foo' requires template arguments}}
 
   template<template<typename> class Bar> void f() { // expected-note 3{{here}}
     Bar x; // expected-error {{use of template template parameter 'Bar' requires template arguments}}
-    Bar<> y; // expected-error {{missing template argument for template parameter}}
+    Bar<> y; // expected-error {{too few template arguments for template template parameter 'Bar'}}
     int z = Bar(); // expected-error {{use of template template parameter 'Bar' requires template arguments}}
   }
 }
@@ -304,12 +304,11 @@ namespace resolved_nttp {
   using TB = int (*)(int (*)[1], int (*)[2], int (*)[3]);
 
   template <typename T, int ...M> struct C {
-    template <T... N> // expected-note 3{{template parameter is declared here}}
-      using Fn = T(int(*...A)[N]);
+    template <T... N> using Fn = T(int(*...A)[N]);
     Fn<1, M..., 4> *p; // expected-error-re 3{{evaluates to {{[234]}}, which cannot be narrowed to type 'bool'}}
   };
   using TC = decltype(C<int, 2, 3>::p);
   using TC = int (*)(int (*)[1], int (*)[2], int (*)[3], int (*)[4]);
 
-  using TC2 = decltype(C<bool, 2, 3>::p); // expected-note 3{{instantiation of}}
+  using TC2 = decltype(C<bool, 2, 3>::p); // expected-note {{instantiation of}}
 }
diff --git a/clang/test/SemaTemplate/cwg2398.cpp b/clang/test/SemaTemplate/cwg2398.cpp
index 3fb5eb6d1c3ac..06333c81a603e 100644
--- a/clang/test/SemaTemplate/cwg2398.cpp
+++ b/clang/test/SemaTemplate/cwg2398.cpp
@@ -264,7 +264,7 @@ namespace classes {
     template<class T, class U> struct A {};
 
     template<template<class> class TT> auto f(TT<int> a) { return a; }
-    // expected-note@-1 2{{no template parameter}}
+    // expected-note@-1 2{{substitution failure: too few template arguments}}
 
     A<int, float> v1;
     A<int, double> v2;
@@ -280,7 +280,7 @@ namespace classes {
       static constexpr auto val = E1;
     };
     template <template <class T3> class TT> void f(TT<int> v) {
-      // expected-note@-1 {{no template parameter}}
+      // expected-note@-1 {{substitution failure: too few template arguments}}
       static_assert(v.val == 3);
     };
     void test() {
@@ -313,7 +313,7 @@ namespace classes {
     }
 
     template <template <class T2, int V3> class TT2> auto g(TT2<double, 1>) {
-      // expected-note@-1 {{no template parameter}}
+      // expected-note@-1 {{too few template arguments for class template 'A'}}
       return f(TT2<int, 2>());
     }
 
@@ -347,11 +347,11 @@ namespace packs {
   namespace t1 {
     template<template<int, int...> class> struct A {};
     // expected-error@-1 {{non-type parameter of template template parameter cannot be narrowed from type 'int' to 'char'}}
-    // expected-note@-2 {{template parameter is declared here}}
+    // expected-note@-2 {{previous template template parameter is here}}
 
-    template<char> struct B; // expected-note {{template parameter is declared here}}
+    template<char> struct B;
     template struct A<B>;
-    // expected-note@-1 {{template template argument is incompatible}}
+    // expected-note@-1 {{has different template parameters}}
   } // namespace t1
   namespace t2 {
     template<template<char, int...> class> struct A {};
@@ -361,11 +361,11 @@ namespace packs {
   namespace t3 {
     template<template<int...> class> struct A {};
     // expected-error@-1 {{non-type parameter of template template parameter cannot be narrowed from type 'int' to 'char'}}
-    // expected-note@-2 {{template parameter is declared here}}
+    // expected-note@-2 {{previous template template parameter is here}}
 
-    template<char> struct B; // expected-note {{template parameter is declared here}}
+    template<char> struct B;
     template struct A<B>;
-    // expected-note@-1 {{template template argument is incompatible}}
+    // expected-note@-1 {{has different template parameters}}
   } // namespace t3
   namespace t4 {
     template<template<char...> class> struct A {};
@@ -501,7 +501,7 @@ namespace constraints {
   // expected-note@-1 {{similar constraint expressions not considered equivalent}}
 
   namespace t1 {
-    template<template<C1, class... T1s> class TT1> // expected-note {{template parameter is declared here}}
+    template<template<C1, class... T1s> class TT1> // expected-note {{TT1' declared here}}
     struct A {};
     template<D1, class T2> struct B {}; // expected-note {{'B' declared here}}
     template struct A<B>;
@@ -513,7 +513,7 @@ namespace constraints {
     template struct A<B>;
   } // namespace t2
   namespace t3 {
-    template<template<C1, class... T1s> class TT1> // expected-note {{template parameter is declared here}}
+    template<template<C1, class... T1s> class TT1> // expected-note {{'TT1' declared here}}
     struct A {};
     template<C2, class T2> struct B {}; // expected-note {{'B' declared here}}
     template struct A<B>;
@@ -521,7 +521,7 @@ namespace constraints {
   } // namespace t2
   namespace t4 {
     // FIXME: This should be accepted.
-    template<template<C1... T1s> class TT1> // expected-note {{template parameter is declared here}}
+    template<template<C1... T1s> class TT1> // expected-note {{'TT1' declared here}}
     struct A {};
     template<C1 T2> struct B {}; // expected-note {{'B' declared here}}
     template struct A<B>;
@@ -529,14 +529,14 @@ namespace constraints {
   } // namespace t4
   namespace t5 {
     // FIXME: This should be accepted
-    template<template<C2... T1s> class TT1> // expected-note {{template parameter is declared here}}
+    template<template<C2... T1s> class TT1> // expected-note {{'TT1' declared here}}
     struct A {};
     template<C1 T2> struct B {}; // expected-note {{'B' declared here}}
     template struct A<B>;
     // expected-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
   } // namespace t5
   namespace t6 {
-    template<template<C1... T1s> class TT1> // expected-note {{template parameter is declared here}}
+    template<template<C1... T1s> class TT1> // expected-note {{'TT1' declared here}}
     struct A {};
     template<C2 T2> struct B {}; // expected-note {{'B' declared here}}
     template struct A<B>;
@@ -555,14 +555,14 @@ namespace constraints {
     template struct A<B>;
   } // namespace t8
   namespace t9 {
-    template<template<C1... T1s> class TT1> // expected-note {{template parameter is declared here}}
+    template<template<C1... T1s> class TT1> // expected-note {{'TT1' declared here}}
     struct A {};
     template<D1 T2> struct B {}; // expected-note {{'B' declared here}}
     template struct A<B>;
     // expected-error@-1 {{'B' is more constrained than template template parameter 'TT1'}}
   } // namespace t9
   namespace t10 {
-    template<template<class...> requires C1<int> class TT1> // expected-note {{template parameter is declared here}}
+    template<template<class...> requires C1<int> class TT1> // expected-note {{'TT1' declared here}}
     struct A {};
 
     template<class> requires C2<int> struct B {}; // expected-note {{'B' declared here}}
@@ -602,6 +602,11 @@ namespace regression3 {
   template <class...> class B {};
   template struct A<B, Node<None>>;
 } // namespace regression3
+namespace GH130362 {
+  template <template <template <class... T1> class TT1> class TT2> struct A {};
+  template <template <class U1> class UU1> struct B {};
+  template struct A<B>;
+} // namespace GH130362
 
 namespace nttp_auto {
   namespace t1 {
@@ -610,24 +615,16 @@ namespace nttp_auto {
     template struct A<B>;
   } // namespace t1
   namespace t2 {
-    // FIXME: Shouldn't accept parameters after a parameter pack.
     template<template<auto... Va1, auto Va2> class> struct A {};
-    // expected-error@-1 {{deduced non-type template argument does not have the same type as the corresponding template parameter ('auto' vs 'int')}}
-    // expected-note@-2 {{template parameter is declared here}}
+    // expected-error@-1 {{template parameter pack must be the last template parameter}}
     template<int... Vi> struct B;
-    // expected-note@-1 {{template parameter is declared here}}
     template struct A<B>;
-    // expected-note@-1 {{template template argument is incompatible}}
   } // namespace t2
   namespace t3 {
-    // FIXME: Shouldn't accept parameters after a parameter pack.
     template<template<auto... Va1, auto... Va2> class> struct A {};
-    // expected-error@-1 {{deduced non-type template argument does not have the same type as the corresponding template parameter ('auto' vs 'int')}}
-    // expected-note@-2 {{template parameter is declared here}}
+    // expected-error@-1 {{template parameter pack must be the last template parameter}}
     template<int... Vi> struct B;
-    // expected-note@-1 {{template parameter is declared here}}
     template struct A<B>;
-    // expected-note@-1 {{template template argument is incompatible}}
   } // namespace t3
 } // namespace nttp_auto
 
diff --git a/clang/test/SemaTemplate/default-arguments.cpp b/clang/test/SemaTemplate/default-arguments.cpp
index fdf666baf01bb..5ea34c0254ec1 100644
--- a/clang/test/SemaTemplate/default-arguments.cpp
+++ b/clang/test/SemaTemplate/default-arguments.cpp
@@ -1,12 +1,12 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++98 %s
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 %s
-template<typename T, int N = 2> struct X; // expected-note {{template parameter is declared here}}
+template<typename T, int N = 2> struct X; // expected-note{{template is declared here}}
 
 X<int, 1> *x1;
 X<int> *x2;
 
-X<> *x3; // expected-error{{missing template argument for template parameter}}
+X<> *x3; // expected-error{{too few template arguments for class template 'X'}}
 
 template<typename U = float, int M> struct X;
 
@@ -20,7 +20,6 @@ template<class T> struct a { };
 template<> struct a<int> { static const bool v = true; };
 
 template<class T, bool = a<T>::v> struct p { }; // expected-error {{no member named 'v'}}
-                                                // expected-note@-1 {{template parameter is declared here}}
 
 template struct p<bool>; // expected-note {{in instantiation of default argument for 'p<bool>' required here}}
 template struct p<int>;
@@ -51,14 +50,12 @@ template<typename T> struct X1 { };
 template<typename T>
 struct X2 {
   template<typename U = typename X1<T>::type> // expected-error{{no type named 'type' in 'X1<int>'}} \
-                                              // expected-error{{no type named 'type' in 'X1<char>'}} \
-                                              // expected-note {{template parameter is declared here}}
-  struct Inner1 { };
+                                              // expected-error{{no type named 'type' in 'X1<char>'}}
+  struct Inner1 { }; // expected-note{{template is declared here}}
 
   template<T Value = X1<T>::value> // expected-error{{no member named 'value' in 'X1<int>'}} \
-                                   // expected-error{{no member named 'value' in 'X1<char>'}} \
-                                   // expected-note {{template parameter is declared here}}
-  struct NonType1 { };
+                                   // expected-error{{no member named 'value' in 'X1<char>'}}
+  struct NonType1 { }; // expected-note{{template is declared here}}
 
   template<T Value>
   struct Inner2 { };
@@ -77,10 +74,10 @@ struct X2 {
 X2<int> x2i; // expected-note{{in instantiation of template class 'X2<int>' requested here}}
 X2<int>::Inner1<float> x2iif;
 
-X2<int>::Inner1<> x2bad; // expected-error{{missing template argument for template parameter}}
+X2<int>::Inner1<> x2bad; // expected-error{{too few template arguments for class template 'Inner1'}}
 
 X2<int>::NonType1<'a'> x2_nontype1;
-X2<int>::NonType1<> x2_nontype1_bad; // expected-error{{missing template argument for template parameter}}
+X2<int>::NonType1<> x2_nontype1_bad; // expected-error{{too few template arguments for class template 'NonType1'}}
 
 // Check multi-level substitution into template type arguments
 X2<int>::Inner3<float>::VeryInner<> vi;
@@ -115,12 +112,12 @@ template<typename T, template<typename> class X = T::template apply>
 int array4[is_same<X4<add_pointer>,
                    X4<add_pointer, add_pointer::apply> >::value? 1 : -1];
 
-template<int> struct X5 {}; // expected-note {{template parameter is declared here}}
+template<int> struct X5 {};
 template<long long> struct X5b {};
 template<typename T,
          template<T> class B = X5> // expected-error {{cannot be narrowed from type 'long long' to 'int'}}
-                                   // expected-note@-1 {{template template argument is incompatible}}
-                                   // expected-note@-2 {{template parameter is declared here}}
+                                   // expected-note@-1 {{has different template parameters}}
+                                   // expected-note@-2 {{previous template template parameter is here}}
   struct X6 {};
 
 X6<int> x6a;
diff --git a/clang/test/SemaTemplate/friend-template.cpp b/clang/test/SemaTemplate/friend-template.cpp
index 2dcee6c76da7d..2b5a226c3b33c 100644
--- a/clang/test/SemaTemplate/friend-template.cpp
+++ b/clang/test/SemaTemplate/friend-template.cpp
@@ -30,7 +30,7 @@ namespace test2 {
   template<typename T> struct X0 {
     template<typename U> friend struct X0;
   };
-  
+
   template<typename T> struct X0<T*> {
     template<typename U> friend struct X0;
   };
@@ -116,13 +116,13 @@ namespace PR6022 {
   template <class T1, class T2 , class T3  > class A;
 
   namespace inner {
-    template<class T1, class T2, class T3, class T> 
+    template<class T1, class T2, class T3, class T>
     A<T1, T2, T3>& f0(A<T1, T2, T3>&, T);
-  } 
+  }
 
   template<class T1, class T2, class T3>
   class A {
-    template<class U1, class U2, class U3, class T>  
+    template<class U1, class U2, class U3, class T>
     friend A<U1, U2, U3>& inner::f0(A<U1, U2, U3>&, T);
   };
 }
@@ -235,7 +235,7 @@ namespace rdar11147355 {
   template <class T>
   struct A {
     template <class U> class B;
-    template <class S> template <class U> friend class A<S>::B; // expected-warning {{dependent nested name specifier 'A<S>::' for friend template declaration is not supported; ignoring this friend declaration}}
+    template <class S> template <class U> friend class A<S>::B; // expected-warning {{dependent nested name specifier 'A<S>' for friend template declaration is not supported; ignoring this friend declaration}}
   private:
     int n; // expected-note {{here}}
   };
diff --git a/clang/test/SemaTemplate/instantiate-member-pointers.cpp b/clang/test/SemaTemplate/instantiate-member-pointers.cpp
index 4eeaa3db6e77c..4757870d13b70 100644
--- a/clang/test/SemaTemplate/instantiate-member-pointers.cpp
+++ b/clang/test/SemaTemplate/instantiate-member-pointers.cpp
@@ -17,7 +17,7 @@ template<typename T, typename Class>
 struct X2 {
   T f(Class &obj, T Class::*pm) { // expected-error{{to a reference}} \
                       // expected-error{{member pointer to void}}
-    return obj.*pm;
+    return obj.*pm; 
   }
 };
 
@@ -39,12 +39,12 @@ typedef int Y::*IntMember;
 template<IntMember Member>
 struct X4 {
   X3<int, Y, Member> member;
-
+  
   int &getMember(Y& y) { return y.*Member; }
 };
 
-int &get_X4(X4<&Y::x> x4, Y& y) {
-  return x4.getMember(y);
+int &get_X4(X4<&Y::x> x4, Y& y) { 
+  return x4.getMember(y); 
 }
 
 template<IntMember Member>
@@ -63,12 +63,12 @@ namespace ValueDepMemberPointer {
   template <typename T> void S<T>::instantiate() {
     int a[(int)sizeof(T)-42]; // expected-error{{array with a negative size}}
   }
-  S<int> s;
+  S<int> s; 
 }
 
 namespace PR18192 {
   struct A { struct { int n; }; };
-  template<int A::*> struct X {}; // expected-note {{template parameter is declared here}}
+  template<int A::*> struct X {};
   constexpr int A::*p = &A::n;
   X<p> x; // expected-error{{not a pointer to member constant}}
 }
diff --git a/clang/test/SemaTemplate/instantiate-template-template-parm.cpp b/clang/test/SemaTemplate/instantiate-template-template-parm.cpp
index 601049ae93c22..dce30aa2af4ee 100644
--- a/clang/test/SemaTemplate/instantiate-template-template-parm.cpp
+++ b/clang/test/SemaTemplate/instantiate-template-template-parm.cpp
@@ -20,17 +20,17 @@ apply<add_reference, int>::type ir = i;
 apply<add_reference, float>::type fr = i; // expected-error{{non-const lvalue reference to type 'float' cannot bind to a value of unrelated type 'int'}}
 
 // Template template parameters
-template<int> struct B; // expected-note {{template parameter is declared here}}
+template<int> struct B;
 
 template<typename T,
          template<T Value> class X> // expected-error{{cannot have type 'float'}}
                                     // expected-error@-1 {{cannot be narrowed from type 'long long' to 'int'}}
-                                    // expected-note@-2 2{{template parameter is declared here}}
+                                    // expected-note@-2 {{previous template template parameter is here}}
 struct X0 { };
 
 X0<int, B> x0b1;
 X0<float, B> x0b2; // expected-note{{while substituting}}
-X0<long long, B> x0b3; // expected-note {{template template argument is incompatible}}
+X0<long long, B> x0b3; // expected-note {{has different template parameters}}
 
 template<template<int V> class TT>
 struct X1 { };
diff --git a/clang/test/SemaTemplate/instantiation-default-1.cpp b/clang/test/SemaTemplate/instantiation-default-1.cpp
index 8ef9f123e9c1c..3e70a2148ca1a 100644
--- a/clang/test/SemaTemplate/instantiation-default-1.cpp
+++ b/clang/test/SemaTemplate/instantiation-default-1.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s
 template<typename T, typename U = const T> struct Def1;
 
-template<> struct Def1<int> {
+template<> struct Def1<int> { 
   void foo();
 };
 
@@ -22,10 +22,9 @@ void test_Def1(Def1<int, const int> *d1, Def1<const int, const int> *d2,
 
 template<typename T,  // FIXME: bad error message below, needs better location info
          typename T2 = const T*>  // expected-error{{'T2' declared as a pointer to a reference}}
-                                  // expected-note@-1 {{template parameter is declared here}}
   struct Def2;
 
-template<> struct Def2<int> {
+template<> struct Def2<int> { 
   void foo();
 };
 
@@ -41,11 +40,11 @@ template<> struct Def1<const int> { }; // expected-error{{redefinition of 'Def1<
 
 template<typename T, typename T2 = T&> struct Def3;
 
-template<> struct Def3<int> {
+template<> struct Def3<int> { 
   void foo();
 };
 
-template<> struct Def3<int&> {
+template<> struct Def3<int&> { 
   void bar();
 };
 
@@ -84,7 +83,7 @@ template<typename R, typename Arg1, typename Arg2 = Arg1,
          typename FuncType = R (*)(Arg1, Arg2)>
   struct Def6;
 
-template<> struct Def6<int, float> {
+template<> struct Def6<int, float> { 
   void foo();
 };
 
@@ -92,7 +91,7 @@ template<> struct Def6<bool, int[5], float(double, double)> {
   void bar();
 };
 
-bool test_Def6(Def6<int, float, float> *d6a,
+bool test_Def6(Def6<int, float, float> *d6a, 
                Def6<int, float, float, int (*)(float, float)> *d6b,
                Def6<bool, int[5], float(double, double),
                     bool(*)(int*, float(*)(double, double))> *d6c) {
diff --git a/clang/test/SemaTemplate/instantiation-default-2.cpp b/clang/test/SemaTemplate/instantiation-default-2.cpp
index e5db31000a419..ffa77cf1ee567 100644
--- a/clang/test/SemaTemplate/instantiation-default-2.cpp
+++ b/clang/test/SemaTemplate/instantiation-default-2.cpp
@@ -4,7 +4,8 @@
 
 template<typename T, T Value> struct Constant;
 // FIXME: bad location precxx20-error@-1 {{a non-type template parameter cannot have type 'float'}}
-// expected-note@-2 2{{template parameter is declared here}}
+// expected-note@-2 {{template parameter is declared here}}
+// cxx20-note@-3 {{template parameter is declared here}}
 
 Constant<int, 5> *c1;
 
diff --git a/clang/test/SemaTemplate/instantiation-dependence.cpp b/clang/test/SemaTemplate/instantiation-dependence.cpp
index d98d98240db7c..1a97dbf769ec6 100644
--- a/clang/test/SemaTemplate/instantiation-dependence.cpp
+++ b/clang/test/SemaTemplate/instantiation-dependence.cpp
@@ -17,8 +17,7 @@ namespace PR24076 {
   }
 
   template<class T,
-           class = // expected-note {{template parameter is declared here}}
-             void_t<decltype(declval<T>() + 1)>> // expected-error {{invalid operands to binary expression}}
+           class = void_t<decltype(declval<T>() + 1)>> // expected-error {{invalid operands to binary expression}}
   struct bar {};
 
   bar<s> bar; // expected-note {{in instantiation of}}
diff --git a/clang/test/SemaTemplate/instantiation-depth-defarg.cpp b/clang/test/SemaTemplate/instantiation-depth-defarg.cpp
index 82bba72929553..24cd941042917 100644
--- a/clang/test/SemaTemplate/instantiation-depth-defarg.cpp
+++ b/clang/test/SemaTemplate/instantiation-depth-defarg.cpp
@@ -1,12 +1,12 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -ftemplate-depth=128 -ftemplate-backtrace-limit=4 %s
 
-template<int N> struct S { // \
-// expected-error{{recursive template instantiation exceeded maximum depth of 128}} \
-// expected-note {{use -ftemplate-depth=N to increase recursive template instantiation depth}}
+template<int N> struct S {
   typedef typename S<N-1>::type type;
   static int f(int n = S<N-1>::f()); // \
+// expected-error{{recursive template instantiation exceeded maximum depth of 128}} \
 // expected-note 3 {{instantiation of default function argument}} \
-// expected-note {{skipping 125 contexts in backtrace}}
+// expected-note {{skipping 125 contexts in backtrace}} \
+// expected-note {{use -ftemplate-depth=N to increase recursive template instantiation depth}}
 
 };
 template<> struct S<0> {
diff --git a/clang/test/SemaTemplate/instantiation-depth-exception-spec.cpp b/clang/test/SemaTemplate/instantiation-depth-exception-spec.cpp
index a0d5a1458ce5d..4464fbb31b383 100644
--- a/clang/test/SemaTemplate/instantiation-depth-exception-spec.cpp
+++ b/clang/test/SemaTemplate/instantiation-depth-exception-spec.cpp
@@ -1,11 +1,11 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 -ftemplate-depth=16 -fcxx-exceptions -fexceptions %s
 
-template<int N> struct X { // \
-// expected-error {{recursive template instantiation exceeded maximum depth of 16}} \
-// expected-note {{use -ftemplate-depth}}
+template<int N> struct X {
   static int go(int a) noexcept(noexcept(X<N+1>::go(a))); // \
+// expected-error {{recursive template instantiation exceeded maximum depth of 16}} \
 // expected-note 9{{in instantiation of exception specification}} \
-// expected-note {{skipping 7 context}}
+// expected-note {{skipping 7 context}} \
+// expected-note {{use -ftemplate-depth}}
 };
 
 void f() {
diff --git a/clang/test/SemaTemplate/instantiation-depth.cpp b/clang/test/SemaTemplate/instantiation-depth.cpp
index 35cf42a2724bc..15e2c2ba7c8bd 100644
--- a/clang/test/SemaTemplate/instantiation-depth.cpp
+++ b/clang/test/SemaTemplate/instantiation-depth.cpp
@@ -19,11 +19,11 @@ void test() {
 // RUN: %clang_cc1 -fsyntax-only -verify -ftemplate-depth=5 -ftemplate-backtrace-limit=4 -std=c++11 -DNOEXCEPT %s
 
 template<typename T> struct S {
-// expected-error@-1 {{recursive template instantiation exceeded maximum depth of 5}}
-// expected-note@-2  {{use -ftemplate-depth=N to increase recursive template instantiation depth}}
   S() noexcept(noexcept(S<S>())); \
+// expected-error{{recursive template instantiation exceeded maximum depth of 5}} \
 // expected-note 3 {{in instantiation of exception spec}} \
-// expected-note {{skipping 2 contexts in backtrace}}
+// expected-note {{skipping 2 contexts in backtrace}} \
+// expected-note {{use -ftemplate-depth=N to increase recursive template instantiation depth}}
 };
 S<void> t; // expected-note {{in instantiation of exception spec}}
 
diff --git a/clang/test/SemaTemplate/ms-sizeof-missing-typename.cpp b/clang/test/SemaTemplate/ms-sizeof-missing-typename.cpp
index fcba385be5ab4..463d86f0d3b94 100644
--- a/clang/test/SemaTemplate/ms-sizeof-missing-typename.cpp
+++ b/clang/test/SemaTemplate/ms-sizeof-missing-typename.cpp
@@ -50,7 +50,7 @@ template struct Foo<Bar>; // expected-note-re {{in instantiation {{.*}} requeste
 }
 
 namespace ambiguous_missing_parens {
-// expected-error@+1 {{'Q::U' is expected to be a non-type template, but instantiated to a class template}}
+// expected-error@+1 {{'Q::template U' is expected to be a non-type template, but instantiated to a class template}}
 template <typename T> void f() { int a = sizeof T::template U<0> + 4; }
 struct Q {
   // expected-note@+1 {{class template declared here}}
diff --git a/clang/test/SemaTemplate/ms-unqualified-base-class.cpp b/clang/test/SemaTemplate/ms-unqualified-base-class.cpp
index d2b02e511e2ed..5e926c5087f93 100644
--- a/clang/test/SemaTemplate/ms-unqualified-base-class.cpp
+++ b/clang/test/SemaTemplate/ms-unqualified-base-class.cpp
@@ -84,7 +84,7 @@ int main() {
   return I;
 }
 
-template <typename Type, int TSize> class Vec {}; // expected-note {{template parameter is declared here}}
+template <typename Type, int TSize> class Vec {}; // expected-note {{template is declared here}}
 
 template <int TDim> class Index : public Vec<int, TDim> {
   // after-error@+1 {{member initializer 'Vec' does not name a non-static data member or base class}}
@@ -107,7 +107,7 @@ template <typename T> class Wrong : public Vec<T, 4> {
 template class Wrong<double>;
 
 template <typename T> class Wrong2 : public Vec<T, 4> {
-  Wrong2() : Vec<T>() {} // expected-error {{missing template argument for template parameter}}
+  Wrong2() : Vec<T>() {} // expected-error {{too few template arguments for class template 'Vec'}}
 };
 
 template class Wrong2<double>;
diff --git a/clang/test/SemaTemplate/nested-template.cpp b/clang/test/SemaTemplate/nested-template.cpp
index 4e82371ef13f7..7911cf555f4b4 100644
--- a/clang/test/SemaTemplate/nested-template.cpp
+++ b/clang/test/SemaTemplate/nested-template.cpp
@@ -90,7 +90,7 @@ Y Outer<X>::Inner1<Y>::ReallyInner::value3 = Y();
 
 template<typename X>
 template<typename Y>
-Y Outer<X>::Inner1<Y*>::ReallyInner::value4; // expected-error{{Outer<X>::Inner1<Y *>::ReallyInner::}}
+Y Outer<X>::Inner1<Y*>::ReallyInner::value4; // expected-error{{'Outer<X>::Inner1<Y *>::ReallyInner'}}
 
 
 template<typename T>
@@ -114,16 +114,16 @@ template<typename T>
 struct X2 {
   template<template<class U, T Value> class>  // expected-error {{cannot have type 'float'}}
                                               // expected-error@-1 {{cannot be narrowed from type 'long long' to 'int'}}
-                                              // expected-note@-2 {{template parameter is declared here}}
+                                              // expected-note@-2 {{previous template template parameter is here}}
     struct Inner { };
 };
 
-template<typename T, int Value> // expected-note {{template parameter is declared here}}
+template<typename T, int Value>
   struct X2_arg;
 
 X2<int>::Inner<X2_arg> x2i1;
 X2<float> x2a; // expected-note{{instantiation}}
-X2<long long>::Inner<X2_arg> x2i3; // expected-note {{template template argument is incompatible}}
+X2<long long>::Inner<X2_arg> x2i3; // expected-note {{has different template parameters}}
 
 namespace PR10896 {
   template<typename TN>
diff --git a/clang/test/SemaTemplate/partial-spec-instantiate.cpp b/clang/test/SemaTemplate/partial-spec-instantiate.cpp
index f171bf4c712b8..0b84df69562e2 100644
--- a/clang/test/SemaTemplate/partial-spec-instantiate.cpp
+++ b/clang/test/SemaTemplate/partial-spec-instantiate.cpp
@@ -42,7 +42,7 @@ namespace WonkyAccess {
 }
 
 namespace rdar9169404 {
-  template<typename T, T N> struct X { }; // #rdar9169404-X
+  template<typename T, T N> struct X { };
   template<bool C> struct X<bool, C> {
     typedef int type;
   };
@@ -50,7 +50,6 @@ namespace rdar9169404 {
   X<bool, -1>::type value;
 #if __cplusplus >= 201103L
   // expected-error@-2 {{non-type template argument evaluates to -1, which cannot be narrowed to type 'bool'}}
-  // expected-note@#rdar9169404-X {{template parameter is declared here}}
 #endif
 }
 
diff --git a/clang/test/SemaTemplate/qualified-id.cpp b/clang/test/SemaTemplate/qualified-id.cpp
index 883d3d330d148..b15f9826b62e8 100644
--- a/clang/test/SemaTemplate/qualified-id.cpp
+++ b/clang/test/SemaTemplate/qualified-id.cpp
@@ -33,12 +33,12 @@ namespace test2 {
 
 namespace PR6063 {
   template <typename T> void f(T, T);
-  
-  namespace detail 
+
+  namespace detail
   {
     using PR6063::f;
   }
-  
+
   template <typename T>
   void g(T a, T b)
   {
@@ -51,7 +51,7 @@ namespace PR12291 {
   class Outer2 {
     template <typename V>
     template <typename W>
-    class Outer2<V>::Inner; // expected-error{{nested name specifier 'Outer2<V>::' for declaration does not refer into a class, class template or class template partial specialization}}
+    class Outer2<V>::Inner; // expected-error{{nested name specifier 'Outer2<V>' for declaration does not refer into a class, class template or class template partial specialization}}
                             // expected-error@-1{{forward declaration of class cannot have a nested name specifier}}
   };
 }
diff --git a/clang/test/SemaTemplate/recovery-crash.cpp b/clang/test/SemaTemplate/recovery-crash.cpp
index 26683b11a1256..ac8053da101ab 100644
--- a/clang/test/SemaTemplate/recovery-crash.cpp
+++ b/clang/test/SemaTemplate/recovery-crash.cpp
@@ -26,14 +26,13 @@ namespace PR16134 {
 }
 
 namespace PR16225 {
-  template <typename T> void f(); // #PR16225-f
+  template <typename T> void f();
   template <typename C> void g(C*) {
     struct LocalStruct : UnknownBase<Mumble, C> { };  // expected-error {{use of undeclared identifier 'Mumble'}}
     f<LocalStruct>();
 #if __cplusplus <= 199711L
     // expected-warning@-2 {{template argument uses local type 'LocalStruct'}}
     // expected-note@-3 {{while substituting explicitly-specified template arguments}}
-    // expected-note@#PR16225-f {{template parameter is declared here}}
 #endif
     struct LocalStruct2 : UnknownBase<C> { };  // expected-error {{no template named 'UnknownBase'}}
   }
diff --git a/clang/test/SemaTemplate/stack-exhaustion.cpp b/clang/test/SemaTemplate/stack-exhaustion.cpp
index b6e2fc9d35999..c7bfea4132d5e 100644
--- a/clang/test/SemaTemplate/stack-exhaustion.cpp
+++ b/clang/test/SemaTemplate/stack-exhaustion.cpp
@@ -33,11 +33,11 @@ int k = f(X<1000>());
 
 namespace template_argument_recursion {
   struct ostream;
-  template<typename T> T &&declval(); // expected-error {{exceeded maximum depth}}
+  template<typename T> T &&declval();
 
   namespace mlir {
     template<typename T, typename = decltype(declval<ostream&>() << declval<T&>())>
-    ostream &operator<<(ostream& os, const T& obj);
+    ostream &operator<<(ostream& os, const T& obj); // expected-error {{exceeded maximum depth}}
     struct Value;
   }
 
@@ -48,12 +48,12 @@ namespace template_argument_recursion {
 
 namespace template_parameter_type_recursion {
   struct ostream;
-  template<typename T> T &&declval(); // expected-error {{exceeded maximum depth}}
+  template<typename T> T &&declval();
   template<bool B, typename T> struct enable_if { using type = T; };
 
   namespace mlir {
     template<typename T, typename enable_if<declval<ostream&>() << declval<T&>(), void*>::type = nullptr>
-    ostream &operator<<(ostream& os, const T& obj);
+    ostream &operator<<(ostream& os, const T& obj); // expected-error {{exceeded maximum depth}}
     struct Value;
   }
 
diff --git a/clang/test/SemaTemplate/temp_arg.cpp b/clang/test/SemaTemplate/temp_arg.cpp
index 9048744f3095a..538056a4e44c7 100644
--- a/clang/test/SemaTemplate/temp_arg.cpp
+++ b/clang/test/SemaTemplate/temp_arg.cpp
@@ -1,10 +1,10 @@
 // RUN: %clang_cc1 -fsyntax-only -verify=expected,precxx17 %std_cxx98-14 %s
 // RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx17 %std_cxx17- %s
-template<typename T,
-         int I,
-         template<typename> class TT> // expected-note {{template parameter is declared here}}
-  class A; // precxx17-note 2 {{template is declared here}} \
-              cxx17-note {{template is declared here}} \
+template<typename T, 
+         int I, 
+         template<typename> class TT>
+  class A; // precxx17-note 3 {{template is declared here}} \
+              cxx17-note 2 {{template is declared here}} \
               cxx17-note {{candidate template ignored: couldn't infer template argument 'T'}} \
               cxx17-note {{implicit deduction guide declared as 'template <typename T, int I, template <typename> class TT> A(A<T, I, TT>) -> A<T, I, TT>'}} \
               cxx17-note {{candidate function template not viable: requires 1 argument, but 0 were provided}} \
@@ -15,7 +15,7 @@ template<typename> class X;
 A<int, 0, X> * a1;
 
 A<float, 1, X, double> *a2; // expected-error{{too many template arguments for class template 'A'}}
-A<float, 1> *a3; // expected-error{{missing template argument for template parameter}}
+A<float, 1> *a3; // expected-error{{too few template arguments for class template 'A'}}
 A a4; // precxx17-error{{use of class template 'A' requires template arguments}} \
          cxx17-error{{no viable constructor or deduction guide for deduction of template arguments of 'A'}}
 
diff --git a/clang/test/SemaTemplate/temp_arg_nontype.cpp b/clang/test/SemaTemplate/temp_arg_nontype.cpp
index 36d7d9cb6f519..2a1c059df002e 100644
--- a/clang/test/SemaTemplate/temp_arg_nontype.cpp
+++ b/clang/test/SemaTemplate/temp_arg_nontype.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -std=c++98 -Wconversion -verify %s
-template<int N> struct A; // expected-note 6{{template parameter is declared here}}
+template<int N> struct A; // expected-note 5{{template parameter is declared here}}
 
 A<0> *a0;
 
@@ -42,7 +42,7 @@ double g(double); // expected-note 2{{candidate function}}
 int h(int);
 float h2(float);
 
-template<int fp(int)> struct A3; // expected-note 2{{template parameter is declared here}}
+template<int fp(int)> struct A3; // expected-note 1{{template parameter is declared here}}
 A3<h> *a14_1;
 A3<&h> *a14_2;
 A3<f> *a14_3;
@@ -61,7 +61,7 @@ A4<*X_volatile_ptr> *a15_2; // expected-error{{non-type template argument does n
 A4<y> *15_3; //  expected-error{{non-type template parameter of reference type 'const X &' cannot bind to template argument of type 'struct Y'}} \
             // FIXME: expected-error{{expected unqualified-id}}
 
-template<int (&fr)(int)> struct A5; // expected-note 2{{template parameter is declared here}}
+template<int (&fr)(int)> struct A5; // expected-note{{template parameter is declared here}}
 A5<h> *a16_1;
 A5<f> *a16_3;
 A5<h2> *a16_6;  // expected-error{{non-type template parameter of reference type 'int (&)(int)' cannot bind to template argument of type 'float (float)'}}
@@ -86,7 +86,7 @@ A6<&Z::baz> *a17_3; // expected-error-re{{non-type template argument of type 'do
 
 
 template<int Z::*pm> struct A7;  // expected-note{{template parameter is declared here}}
-template<int Z::*pm> struct A7c; // expected-note{{template parameter is declared here}}
+template<int Z::*pm> struct A7c;
 A7<&Z::int_member> *a18_1;
 A7c<&Z::int_member> *a18_2;
 A7<&Z::float_member> *a18_3; // expected-error{{non-type template argument of type 'float Z::*' cannot be converted to a value of type 'int Z::*'}}
@@ -442,10 +442,7 @@ namespace dependent_nested_partial_specialization {
   A<Y>::B<int, &n> ay; // expected-error {{undefined}} expected-note {{instantiation of}}
 
   template<template<typename> class X> struct C {
-    template<typename T, int N,
-      int M // expected-note {{template parameter is declared here}}
-    > struct D;
-    // expected-note@-1 {{template is declared here}}
+    template<typename T, int N, int M> struct D; // expected-note {{here}}
     template<typename T, X<T> N> struct D<T*, N, N + 1> {}; // expected-error {{type of specialized non-type template argument depends on}}
   };
   C<X>::D<int*, 0, 1> cx;
@@ -495,12 +492,7 @@ namespace dependent_backreference {
   Incomplete f(int); // expected-note 2{{here}}
   int f(short);
 
-  template<typename T, T Value,
-    int(*)[ // expected-note 2{{template parameter is declared here}}
-      sizeof(f(Value)) // expected-error 2{{incomplete}}
-    ]
-  > struct X {};
-
+  template<typename T, T Value, int(*)[sizeof(f(Value))]> struct X {}; // expected-error 2{{incomplete}}
   int arr[sizeof(int)];
   // When checking this template-id, we must not treat 'Value' as having type
   // 'int'; its type is the dependent type 'T'.
diff --git a/clang/test/SemaTemplate/temp_arg_nontype_cxx11.cpp b/clang/test/SemaTemplate/temp_arg_nontype_cxx11.cpp
index 32bb678c4bf4e..5752cbac0291d 100644
--- a/clang/test/SemaTemplate/temp_arg_nontype_cxx11.cpp
+++ b/clang/test/SemaTemplate/temp_arg_nontype_cxx11.cpp
@@ -30,7 +30,7 @@ namespace Auto {
 
 namespace check_conversion_early {
   struct X {};
-  template<int> struct A {}; // expected-note {{template parameter is declared here}}
+  template<int> struct A {};
   template<X &x> struct A<x> {}; // expected-error {{not implicitly convertible}}
 
   struct Y { constexpr operator int() const { return 0; } };
diff --git a/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp b/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp
index 724ac85b5f73d..c35743b87adbc 100644
--- a/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp
+++ b/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++1z %s
 
-template<typename T, T val> struct A {}; // expected-note 15{{template parameter is declared here}}
+template<typename T, T val> struct A {}; // expected-note 3{{template parameter is declared here}}
 
 template<typename T, typename U> constexpr bool is_same = false;
 template<typename T> constexpr bool is_same<T, T> = true;
@@ -254,8 +254,8 @@ namespace Auto {
   }
 
   namespace DecltypeAuto {
-    template<auto v> struct A { }; // expected-note {{template parameter is declared here}}
-    template<decltype(auto) v> struct DA { }; // expected-note {{template parameter is declared here}}
+    template<auto v> struct A { };
+    template<decltype(auto) v> struct DA { };
     template<auto&> struct R { };
 
     auto n = 0; // expected-note + {{declared here}}
@@ -448,7 +448,7 @@ namespace PR42108 {
   struct R {};
   struct S { constexpr S() {} constexpr S(R) {} };
   struct T { constexpr operator S() { return {}; } };
-  template <const S &> struct A {}; // expected-note 3{{template parameter is declared here}}
+  template <const S &> struct A {}; // expected-note {{template parameter is declared here}}
   void f() {
     A<R{}>(); // expected-error {{would bind reference to a temporary}}
     A<S{}>(); // expected-error {{reference to temporary object}}
diff --git a/clang/test/SemaTemplate/temp_arg_nontype_cxx20.cpp b/clang/test/SemaTemplate/temp_arg_nontype_cxx20.cpp
index 221f8595f43d5..56ceb7af4ccd9 100644
--- a/clang/test/SemaTemplate/temp_arg_nontype_cxx20.cpp
+++ b/clang/test/SemaTemplate/temp_arg_nontype_cxx20.cpp
@@ -16,7 +16,7 @@ union U { int a, b; } u;
 int n; // expected-note 1+{{here}}
 
 // pointers to subobjects
-template<int *> struct IntPtr {}; // expected-note 2{{template parameter is declared here}}
+template<int *> struct IntPtr {};
 using IPn = IntPtr<&n + 1>;
 using IPn = IntPtr<&n + 1>;
 
@@ -30,7 +30,7 @@ using IP3 = IntPtr<s.n + 3>;
 
 using IP5 = IntPtr<&s.n[5]>; // expected-error {{not a constant expression}} expected-note {{cannot refer to element 5 of array of 3 elements}}
 
-template<int &> struct IntRef {}; // expected-note 4{{template parameter is declared here}}
+template<int &> struct IntRef {};
 using IRn = IntRef<*(&n + 1)>; // expected-error {{not a constant expression}} expected-note {{dereferenced pointer past the end of 'n'}}
 using IRn = IntRef<*(&n + 1)>; // expected-error {{not a constant expression}} expected-note {{dereferenced pointer past the end of 'n'}}
 
@@ -72,20 +72,20 @@ namespace ClassNTTP {
   static_assert(&id<A{1,3}> != &id<a>);
 
   int k = id<1>; // expected-error {{no viable conversion from 'int' to 'A'}}
-                 // expected-note@#ClassNTTP1 {{template parameter is declared here}}
+                 // expected-note@#ClassNTTP1 {{passing argument to parameter 'a' here}}
 
   struct B {
     constexpr B() {}
     constexpr B(int) = delete; // expected-note {{here}}
   };
-  template<B> struct Q {}; // expected-note {{template parameter is declared here}}
+  template<B> struct Q {}; // expected-note {{passing argument to parameter here}}
   Q<1> q; // expected-error {{conversion function from 'int' to 'B' invokes a deleted function}}
 
   struct C {
     constexpr C() {}
     C(const C&) = delete; // expected-note {{here}}
   };
-  template<C> struct R {}; // expected-note {{template parameter is declared here}}
+  template<C> struct R {}; // expected-note {{passing argument to parameter here}}
   constexpr C c;
   R<c> r; // expected-error {{call to deleted constructor}}
 }
@@ -228,7 +228,7 @@ namespace UnnamedBitfield {
   //
   // FIXME: We shouldn't track a value for unnamed bit-fields, nor number
   // them when computing field indexes.
-  template <A> struct X {}; // expected-note {{template parameter is declared here}}
+  template <A> struct X {};
   constexpr A a;
   using T = X<a>;
   using T = X<A{}>;
@@ -238,7 +238,7 @@ namespace UnnamedBitfield {
 }
 
 namespace Temporary {
-  template<const int &> struct A {}; // expected-note 6{{template parameter is declared here}}
+  template<const int &> struct A {}; // expected-note {{template parameter is declared here}}
   A<0> a0; // expected-error {{conversion from 'int' to 'const int &' in converted constant expression would bind reference to a temporary}}
 
   A<(const int&)1> a1; // expected-error {{reference to temporary object is not allowed in a template argument}}
@@ -254,18 +254,18 @@ namespace Temporary {
   X &&x = X{};
   A<x.a[3]> a5; // expected-error {{reference to subobject of temporary object}}
 
-  template<const int*> struct B {}; // expected-note 3{{template parameter is declared here}}
+  template<const int*> struct B {};
   B<&(int&)(int&&)0> b0; // expected-error {{pointer to temporary object}}
   B<&r3> b3; // expected-error {{pointer to temporary object}}
   B<&x.a[3]> b5; // expected-error {{pointer to subobject of temporary object}}
 
   struct C { const int *p[2]; };
-  template<C> struct D {}; // expected-note {{template parameter is declared here}}
+  template<C> struct D {};
   D<C{nullptr, &r3}> d; // expected-error {{pointer to temporary object}}
 }
 
 namespace StringLiteral {
-  template<decltype(auto)> struct Y {}; // expected-note 6{{template parameter is declared here}}
+  template<decltype(auto)> struct Y {};
   Y<&"hello"> y1; // expected-error {{pointer to string literal}}
   Y<"hello"> y2; // expected-error {{reference to string literal}}
   Y<+"hello"> y3; // expected-error {{pointer to subobject of string literal}}
@@ -278,7 +278,7 @@ namespace StringLiteral {
 }
 
 namespace TypeInfo {
-  template<decltype(auto)> struct Y {}; // expected-note 4{{template parameter is declared here}}
+  template<decltype(auto)> struct Y {};
   Y<&typeid(int)> y1; // expected-error {{pointer to type_info object}}
   Y<typeid(int)> y2; // expected-error {{reference to type_info object}}
 
@@ -289,7 +289,7 @@ namespace TypeInfo {
 }
 
 namespace Predefined {
-  template<decltype(auto)> struct Y {}; // expected-note 7{{template parameter is declared here}}
+  template<decltype(auto)> struct Y {};
 
   struct A { const char *p; };
   struct B { const char &r; };
diff --git a/clang/test/SemaTemplate/temp_arg_nontype_cxx2c.cpp b/clang/test/SemaTemplate/temp_arg_nontype_cxx2c.cpp
index a87ffcbc66659..e74c031eba4c1 100644
--- a/clang/test/SemaTemplate/temp_arg_nontype_cxx2c.cpp
+++ b/clang/test/SemaTemplate/temp_arg_nontype_cxx2c.cpp
@@ -5,16 +5,16 @@ struct Test {
     int b = 42;
 };
 
-template <Test t> // expected-note {{template parameter is declared here}}
+template <Test t>
 struct A {
     static constexpr auto a = t.a;
     static constexpr auto b = t.b;
 };
 
-template <auto N> // expected-note {{template parameter is declared here}}
+template <auto N>
 struct Auto {};
 
-template <typename T, T elem> // expected-note {{template parameter is declared here}}
+template <typename T, T elem>
 struct Explicit{};
 
 struct L {};
@@ -57,8 +57,8 @@ void test() {
     DefaultParam3<> d3;
 }
 
-template<auto n> struct B { /* ... */ }; // expected-note 2{{template parameter is declared here}}
-template<int i> struct C { /* ... */ }; // expected-note {{template parameter is declared here}}
+template<auto n> struct B { /* ... */ };
+template<int i> struct C { /* ... */ };
 C<{ 42 }> c1;  // expected-warning {{braces around scalar initializer}}
 
 struct J1 {
diff --git a/clang/test/SemaTemplate/temp_arg_template.cpp b/clang/test/SemaTemplate/temp_arg_template.cpp
index c10b594c7c2b4..aa53dba652050 100644
--- a/clang/test/SemaTemplate/temp_arg_template.cpp
+++ b/clang/test/SemaTemplate/temp_arg_template.cpp
@@ -2,13 +2,13 @@
 // RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx17 -std=c++17 %s
 
 template<template<typename T> class X> struct A; // #A
-// expected-note@-1 3{{template parameter is declared here}}
+// expected-note@-1 2{{previous template template parameter is here}}
 
-template<template<typename T, int I> class X> struct B; // expected-note{{template parameter is declared here}}
+template<template<typename T, int I> class X> struct B; // expected-note{{previous template template parameter is here}}
 
 template<template<int I> class X> struct C;
 // expected-error@-1 {{conversion from 'int' to 'const int &' in converted constant expression would bind reference to a temporary}}
-// expected-note@-2 {{template parameter is declared here}}
+// expected-note@-2 {{previous template template parameter is here}}
 
 template<class> struct X; // expected-note {{template is declared here}}
 template<int N> struct Y; // expected-note {{template parameter is declared here}}
@@ -18,10 +18,7 @@ template<const int &N> struct Yref; // expected-note {{template parameter is dec
 namespace N {
   template<class> struct Z;
 }
-template<
-  class,
-  class> // expected-note {{template parameter is declared here}}
-struct TooMany;
+template<class, class> struct TooMany; // expected-note{{template is declared here}}
 
 
 A<X> *a1;
@@ -29,14 +26,14 @@ A<N::Z> *a2;
 A< ::N::Z> *a3;
 
 A<Y> *a4; // expected-error@#A {{template argument for non-type template parameter must be an expression}}
-          // expected-note@-1 {{template template argument is incompatible}}
-A<TooMany> *a5; // expected-error@#A {{no template parameter in this template template parameter}}
-                // expected-note@-1 {{template template argument is incompatible}}
+          // expected-note@-1 {{different template parameters}}
+A<TooMany> *a5; // expected-error {{too few template arguments for class template 'TooMany'}}
+                // expected-note@-1 {{different template parameters}}
 B<X> *a6; // expected-error {{too many template arguments for class template 'X'}}
-          // expected-note@-1 {{template template argument is incompatible}}
+          // expected-note@-1 {{different template parameters}}
 C<Y> *a7;
 C<Ylong> *a8;
-C<Yref> *a9; // expected-note {{template template argument is incompatible}}
+C<Yref> *a9; // expected-note {{different template parameters}}
 
 template<typename T> void f(int);
 
@@ -112,7 +109,7 @@ void foo() {
 
 namespace CheckDependentNonTypeParamTypes {
   template<template<typename T, typename U, T v> class X> struct A {
-    // expected-note@-1 {{template parameter is declared here}}
+    // expected-note@-1 {{previous template template parameter is here}}
     void f() {
       X<int, void*, 3> x;
     }
@@ -139,7 +136,7 @@ namespace CheckDependentNonTypeParamTypes {
   };
 
   // FIXME: This should probably be rejected, but the rules are at best unclear.
-  A<B> ab; // expected-note {{template template argument is incompatible}}
+  A<B> ab; // expected-note {{different template parameters}}
 
   void use() {
     ab.f();
diff --git a/clang/test/SemaTemplate/temp_arg_template_p0522.cpp b/clang/test/SemaTemplate/temp_arg_template_p0522.cpp
index e9084d1c27413..d8a81bb363112 100644
--- a/clang/test/SemaTemplate/temp_arg_template_p0522.cpp
+++ b/clang/test/SemaTemplate/temp_arg_template_p0522.cpp
@@ -1,11 +1,14 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++20 %s
 
+// expected-note@temp_arg_template_p0522.cpp:* 1+{{template is declared here}}
 // expected-note@temp_arg_template_p0522.cpp:* 1+{{template parameter is declared here}}
+// expected-note@temp_arg_template_p0522.cpp:* 1+{{previous template template parameter is here}}
 
 template<template<int> typename> struct Ti; // #Ti
 template<template<int...> typename> struct TPi; // #TPi
 template<template<int, int...> typename> struct TiPi;
-template<template<int..., int...> typename> struct TPiPi; // FIXME: Why is this not ill-formed?
+template<template<int..., int...> typename> struct TPiPi;
+// expected-error@-1 {{template parameter pack must be the last template parameter}}
 
 template<typename T, template<T> typename> struct tT0; // #tT0
 template<template<typename T, T> typename> struct Tt0; // #Tt0
@@ -31,14 +34,14 @@ namespace IntParam {
         Ti<iDi>,
         Ti<Pi>,
         Ti<iDt>>;
-  using err1 = Ti<ii>; // expected-error@#Ti {{no template parameter in this template template parameter}}
-                       // expected-note@-1 {{template template argument is incompatible}}
-  using err2 = Ti<iiPi>; // expected-error@#Ti {{no template parameter in this template template parameter}}
-                         // expected-note@-1 {{template template argument is incompatible}}
+  using err1 = Ti<ii>; // expected-error {{too few template arguments for class template 'ii'}}
+                       // expected-note@-1 {{different template parameters}}
+  using err2 = Ti<iiPi>; // expected-error {{too few template arguments for class template 'iiPi'}}
+                         // expected-note@-1 {{different template parameters}}
   using err3 = Ti<t0>; // expected-error@#Ti {{template argument for template type parameter must be a type}}
-                       // expected-note@-1 {{template template argument is incompatible}}
-  using err4 = Ti<it>; // expected-error@#Ti {{no template parameter in this template template parameter}}
-                       // expected-note@-1 {{template template argument is incompatible}}
+                       // expected-note@-1 {{different template parameters}}
+  using err4 = Ti<it>; // expected-error {{too few template arguments for class template 'it'}}
+                       // expected-note@-1 {{different template parameters}}
 }
 
 // These are accepted by the backwards-compatibility "parameter pack in
@@ -47,11 +50,11 @@ namespace IntPackParam {
   using ok = TPi<Pi>;
   using ok_compat = Pt<TPi<i>, TPi<iDi>, TPi<ii>, TPi<iiPi>>;
   using err1 = TPi<t0>; // expected-error@#TPi {{template argument for template type parameter must be a type}}
-                        // expected-note@-1 {{template template argument is incompatible}}
+                        // expected-note@-1 {{different template parameters}}
   using err2 = TPi<iDt>; // expected-error@#TPi {{template argument for template type parameter must be a type}}
-                         // expected-note@-1 {{template template argument is incompatible}}
+                         // expected-note@-1 {{different template parameters}}
   using err3 = TPi<it>; // expected-error@#TPi {{template argument for template type parameter must be a type}}
-                        // expected-note@-1 {{template template argument is incompatible}}
+                        // expected-note@-1 {{different template parameters}}
 }
 
 namespace IntAndPackParam {
@@ -62,19 +65,19 @@ namespace IntAndPackParam {
 
 namespace DependentType {
   using ok = Pt<tT0<int, i>, tT0<int, iDi>>;
-  using err1 = tT0<int, ii>; // expected-error@#tT0 {{no template parameter in this template template parameter}}
-                             // expected-note@-1 {{template template argument is incompatible}}
+  using err1 = tT0<int, ii>; // expected-error {{too few template arguments for class template 'ii'}}
+                             // expected-note@-1 {{different template parameters}}
   using err2 = tT0<short, i>;
   using err2a = tT0<long long, i>; // expected-error@#tT0 {{cannot be narrowed from type 'long long' to 'int'}}
-                                   // expected-note@-1 {{template template argument is incompatible}}
+                                   // expected-note@-1 {{different template parameters}}
   using err2b = tT0<void*, i>; // expected-error@#tT0 {{value of type 'void *' is not implicitly convertible to 'int'}}
-                               // expected-note@-1 {{template template argument is incompatible}}
+                               // expected-note@-1 {{different template parameters}}
   using err3 = tT0<short, t0>; // expected-error@#tT0 {{template argument for template type parameter must be a type}}
-                               // expected-note@-1 {{template template argument is incompatible}}
+                               // expected-note@-1 {{different template parameters}}
 
   using ok2 = Tt0<t0>;
   using err4 = Tt0<it>; // expected-error@#Tt0 {{template argument for non-type template parameter must be an expression}}
-                        // expected-note@-1 {{template template argument is incompatible}}
+                        // expected-note@-1 {{different template parameters}}
 }
 
 namespace Auto {
@@ -91,22 +94,22 @@ namespace Auto {
 
   TInt<Auto> ia;
   TInt<AutoPtr> iap; // expected-error@#TInt {{non-type template parameter '' with type 'auto *' has incompatible initializer of type 'int'}}
-                     // expected-note@-1 {{template template argument is incompatible}}
+                     // expected-note@-1 {{different template parameters}}
   TInt<DecltypeAuto> ida;
   TInt<Int> ii;
   TInt<IntPtr> iip; // expected-error@#TInt {{conversion from 'int' to 'int *' is not allowed in a converted constant expression}}
-                    // expected-note@-1 {{template template argument is incompatible}}
+                    // expected-note@-1 {{different template parameters}}
 
   TIntPtr<Auto> ipa;
   TIntPtr<AutoPtr> ipap;
   TIntPtr<DecltypeAuto> ipda;
   TIntPtr<Int> ipi; // expected-error@#TIntPtr {{value of type 'int *' is not implicitly convertible to 'int'}}
-                    // expected-note@-1 {{template template argument is incompatible}}
+                    // expected-note@-1 {{different template parameters}}
   TIntPtr<IntPtr> ipip;
 
   TAuto<Auto> aa;
   TAuto<AutoPtr> aap; // expected-error@#AutoPtr {{could not match 'auto *' against 'auto'}}
-                      // expected-note@-1 {{template template argument is incompatible}}
+                      // expected-note@-1 {{different template parameters}}
   TAuto<Int> ai; // FIXME: ill-formed (?)
   TAuto<IntPtr> aip; // FIXME: ill-formed (?)
 
@@ -128,7 +131,7 @@ namespace Auto {
   // parameters.
   TDecltypeAuto<Auto> daa;
   TDecltypeAuto<AutoPtr> daap; // expected-error@#AutoPtr {{could not match 'auto *' against 'decltype(auto)'}}
-                               // expected-note@-1 {{template template argument is incompatible}}
+                               // expected-note@-1 {{different template parameters}}
 
   int n;
   template<auto A, decltype(A) B = &n> struct SubstFailure;
@@ -157,10 +160,11 @@ namespace GH101394 {
   } // namespace t1
   namespace t2 {
     template<template<Y> class> struct A {}; // #A
-    template<X> struct B;
+    template<X> struct B; // #B
     template struct A<B>;
     // expected-error@#A {{no viable conversion from 'const Y' to 'X'}}
-    // expected-note@-2  {{template template argument is incompatible}}
+    // expected-note@-2  {{different template parameters}}
     // expected-note@#X 2{{not viable}}
+    // expected-note@#B  {{passing argument to parameter here}}
   } // namespace t2
 } // namespace GH101394
diff --git a/clang/test/SemaTemplate/temp_arg_type.cpp b/clang/test/SemaTemplate/temp_arg_type.cpp
index a0d2bc383d917..392d2573d3d0e 100644
--- a/clang/test/SemaTemplate/temp_arg_type.cpp
+++ b/clang/test/SemaTemplate/temp_arg_type.cpp
@@ -1,12 +1,9 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s
-// RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx98 -std=c++98 %s
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++98 %s
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 %s
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++1z %s
 
-template<typename T> class A;
-// expected-note@-1 4{{template parameter is declared here}}
-// cxx98-note@-2    2{{template parameter is declared here}}
-// expected-note@-3  {{template is declared here}}
+template<typename T> class A; // expected-note 2 {{template parameter is declared here}} expected-note{{template is declared here}}
 
 // [temp.arg.type]p1
 A<0> *a1; // expected-error{{template argument for template type parameter must be a type}}
diff --git a/clang/test/SemaTemplate/template-id-expr.cpp b/clang/test/SemaTemplate/template-id-expr.cpp
index dc12823ae307f..a13013a1ffd47 100644
--- a/clang/test/SemaTemplate/template-id-expr.cpp
+++ b/clang/test/SemaTemplate/template-id-expr.cpp
@@ -19,7 +19,7 @@ template<typename T>
 struct X0 {
   template<typename U>
   void f1();
-  
+
   template<typename U>
   void f2(U) {
     f1<U>();
@@ -39,9 +39,9 @@ struct Y {
 template<int I>
 struct X {
   X(int, int);
-  void f() { 
-    Y<X<I> >(X<I>(0, 0)); 
-    Y<X<I> >(::X<I>(0, 0)); 
+  void f() {
+    Y<X<I> >(X<I>(0, 0));
+    Y<X<I> >(::X<I>(0, 0));
   }
 };
 
@@ -184,7 +184,7 @@ class E {
 #if __cplusplus <= 199711L
 // expected-warning@+2 {{extension}}
 #endif
-template<typename T> using D = int; // expected-note {{declared here}} 
+template<typename T> using D = int; // expected-note {{declared here}}
 E<D> ed; // expected-note {{instantiation of}}
 
 namespace non_functions {
@@ -202,7 +202,7 @@ struct P {
 struct Q {
   template <typename T> int foo() {
     return T::template I<int>;
-    // expected-error@-1 {{'P::I' is expected to be a non-type template, but instantiated to a type alias template}}
+    // expected-error@-1 {{'P::template I' is expected to be a non-type template, but instantiated to a type alias template}}
     // expected-note@#TypeAlias {{type alias template declared here}}
   }
 };
@@ -251,18 +251,18 @@ template <typename T1> struct A {
 
   void foo() {
     C<T1>::template Type<2>;
-    // expected-error@-1 {{'C<float>::Type' is expected to be a non-type template, but instantiated to a class template}}}
+    // expected-error@-1 {{'C<float>::template Type' is expected to be a non-type template, but instantiated to a class template}}}
     // expected-note@#ClassTemplate {{class template declared here}}
 
     foo(C<T1>::Type<2>); // expected-error {{expected expression}}
 
     foo(C<T1>::template Type<2>);
-    // expected-error@-1 {{'C<float>::Type' is expected to be a non-type template, but instantiated to a class template}}
+    // expected-error@-1 {{'C<float>::template Type' is expected to be a non-type template, but instantiated to a class template}}
     // expected-note@#ClassTemplate {{class template declared here}}
 
     foo(C<T1>::template Type<2>());
-    // expected-error@-1 {{'C<float>::Type' is expected to be a non-type template, but instantiated to a class template}}
-    // expected-error@-2 {{called object type '<dependent type>' is not a function or function pointer}}
+    // expected-error@-1 {{'C<float>::template Type' is expected to be a non-type template, but instantiated to a class template}}
+    // expected-error@-2 {{called object type 'C<float>::template Type<2>' is not a function or function pointer}}
     // expected-note@#ClassTemplate {{class template declared here}}
 
     foo(typename C<T1>::template Type<2>());
diff --git a/clang/test/SemaTemplate/typename-specifier-3.cpp b/clang/test/SemaTemplate/typename-specifier-3.cpp
index 714830f0032d2..0140b1a479c2d 100644
--- a/clang/test/SemaTemplate/typename-specifier-3.cpp
+++ b/clang/test/SemaTemplate/typename-specifier-3.cpp
@@ -28,7 +28,7 @@ namespace PR12884_original {
       typedef int arg;
     };
     struct C {
-      typedef B::X<typename B::arg> x; // precxx17-warning{{missing 'typename' prior to dependent type name B::X; implicit 'typename' is a C++20 extension}}
+      typedef B::X<typename B::arg> x; // precxx17-warning{{missing 'typename' prior to dependent type name 'B::X'; implicit 'typename' is a C++20 extension}}
     };
   };
 
diff --git a/clang/test/TableGen/HLSLAttribute-errors.td b/clang/test/TableGen/HLSLAttribute-errors.td
new file mode 100644
index 0000000000000..fc9473dcc1fb4
--- /dev/null
+++ b/clang/test/TableGen/HLSLAttribute-errors.td
@@ -0,0 +1,11 @@
+// RUN: not clang-tblgen -gen-clang-attr-parsed-attr-kinds -I%p/../../include %s -o - 2>&1 | FileCheck %s
+
+include "clang/Basic/Attr.td"
+
+// CHECK: error: HLSLAnnotation Attribute must be lower case.
+def HLSLSV_FAKE: HLSLAnnotationAttr {
+  let Spellings = [HLSLAnnotation<"SV_Fake">];
+  let Subjects = SubjectList<[ParmVar, Field]>;
+  let LangOpts = [HLSL];
+  let Documentation = [HLSLSV_GroupThreadIDDocs];
+}
diff --git a/clang/tools/amdgpu-arch/AMDGPUArch.cpp b/clang/tools/amdgpu-arch/AMDGPUArch.cpp
index 6c10cbc5c46a8..86f3e31f47bbc 100644
--- a/clang/tools/amdgpu-arch/AMDGPUArch.cpp
+++ b/clang/tools/amdgpu-arch/AMDGPUArch.cpp
@@ -21,6 +21,9 @@ static cl::opt<bool> Help("h", cl::desc("Alias for -help"), cl::Hidden);
 // Mark all our options with this category.
 static cl::OptionCategory AMDGPUArchCategory("amdgpu-arch options");
 
+cl::opt<bool> Verbose("verbose", cl::desc("Enable verbose output"),
+                      cl::init(false), cl::cat(AMDGPUArchCategory));
+
 static void PrintVersion(raw_ostream &OS) {
   OS << clang::getClangToolFullVersion("amdgpu-arch") << '\n';
 }
diff --git a/clang/tools/amdgpu-arch/AMDGPUArchByHIP.cpp b/clang/tools/amdgpu-arch/AMDGPUArchByHIP.cpp
index 7338872dbf32f..4803f83f55ac7 100644
--- a/clang/tools/amdgpu-arch/AMDGPUArchByHIP.cpp
+++ b/clang/tools/amdgpu-arch/AMDGPUArchByHIP.cpp
@@ -11,9 +11,24 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/DynamicLibrary.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Process.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/VersionTuple.h"
 #include "llvm/Support/raw_ostream.h"
+#include <algorithm>
+#include <string>
+#include <vector>
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
 
 using namespace llvm;
 
@@ -31,16 +46,124 @@ typedef hipError_t (*hipGetDeviceCount_t)(int *);
 typedef hipError_t (*hipDeviceGet_t)(int *, int);
 typedef hipError_t (*hipGetDeviceProperties_t)(hipDeviceProp_t *, int);
 
-int printGPUsByHIP() {
+extern cl::opt<bool> Verbose;
+
 #ifdef _WIN32
-  constexpr const char *DynamicHIPPath = "amdhip64.dll";
+static std::vector<std::string> getSearchPaths() {
+  std::vector<std::string> Paths;
+
+  // Get the directory of the current executable
+  if (auto MainExe = sys::fs::getMainExecutable(nullptr, nullptr);
+      !MainExe.empty())
+    Paths.push_back(sys::path::parent_path(MainExe).str());
+
+  // Get the system directory
+  wchar_t SystemDirectory[MAX_PATH];
+  if (GetSystemDirectoryW(SystemDirectory, MAX_PATH) > 0) {
+    std::string Utf8SystemDir;
+    if (convertUTF16ToUTF8String(
+            ArrayRef<UTF16>(reinterpret_cast<const UTF16 *>(SystemDirectory),
+                            wcslen(SystemDirectory)),
+            Utf8SystemDir))
+      Paths.push_back(Utf8SystemDir);
+  }
+
+  // Get the Windows directory
+  wchar_t WindowsDirectory[MAX_PATH];
+  if (GetWindowsDirectoryW(WindowsDirectory, MAX_PATH) > 0) {
+    std::string Utf8WindowsDir;
+    if (convertUTF16ToUTF8String(
+            ArrayRef<UTF16>(reinterpret_cast<const UTF16 *>(WindowsDirectory),
+                            wcslen(WindowsDirectory)),
+            Utf8WindowsDir))
+      Paths.push_back(Utf8WindowsDir);
+  }
+
+  // Get the current working directory
+  SmallVector<char, 256> CWD;
+  if (sys::fs::current_path(CWD))
+    Paths.push_back(std::string(CWD.begin(), CWD.end()));
+
+  // Get the PATH environment variable
+  if (std::optional<std::string> PathEnv = sys::Process::GetEnv("PATH")) {
+    SmallVector<StringRef, 16> PathList;
+    StringRef(*PathEnv).split(PathList, sys::EnvPathSeparator);
+    for (auto &Path : PathList)
+      Paths.push_back(Path.str());
+  }
+
+  return Paths;
+}
+
+// Custom comparison function for dll name
+static bool compareVersions(StringRef A, StringRef B) {
+  auto ParseVersion = [](StringRef S) -> VersionTuple {
+    unsigned Pos = S.find_last_of('_');
+    StringRef VerStr = (Pos == StringRef::npos) ? S : S.substr(Pos + 1);
+    VersionTuple Vt;
+    (void)Vt.tryParse(VerStr);
+    return Vt;
+  };
+
+  VersionTuple VtA = ParseVersion(A);
+  VersionTuple VtB = ParseVersion(B);
+  return VtA > VtB;
+}
+#endif
+
+// On Windows, prefer amdhip64_n.dll where n is ROCm major version and greater
+// value of n takes precedence. If amdhip64_n.dll is not found, fall back to
+// amdhip64.dll. The reason is that a normal driver installation only has
+// amdhip64_n.dll but we do not know what n is since this program may be used
+// with a future version of HIP runtime.
+//
+// On Linux, always use default libamdhip64.so.
+static std::pair<std::string, bool> findNewestHIPDLL() {
+#ifdef _WIN32
+  StringRef HipDLLPrefix = "amdhip64_";
+  StringRef HipDLLSuffix = ".dll";
+
+  std::vector<std::string> SearchPaths = getSearchPaths();
+  std::vector<std::string> DLLNames;
+
+  for (const auto &Dir : SearchPaths) {
+    std::error_code EC;
+    for (sys::fs::directory_iterator DirIt(Dir, EC), DirEnd;
+         DirIt != DirEnd && !EC; DirIt.increment(EC)) {
+      StringRef Filename = sys::path::filename(DirIt->path());
+      if (Filename.starts_with(HipDLLPrefix) &&
+          Filename.ends_with(HipDLLSuffix))
+        DLLNames.push_back(sys::path::convert_to_slash(DirIt->path()));
+    }
+    if (!DLLNames.empty())
+      break;
+  }
+
+  if (DLLNames.empty())
+    return {"amdhip64.dll", true};
+
+  llvm::sort(DLLNames, compareVersions);
+  return {DLLNames[0], false};
 #else
-  constexpr const char *DynamicHIPPath = "libamdhip64.so";
+  // On Linux, fallback to default shared object
+  return {"libamdhip64.so", true};
 #endif
+}
+
+int printGPUsByHIP() {
+  auto [DynamicHIPPath, IsFallback] = findNewestHIPDLL();
+
+  if (Verbose) {
+    if (IsFallback)
+      outs() << "Using default HIP runtime: " << DynamicHIPPath << '\n';
+    else
+      outs() << "Found HIP runtime: " << DynamicHIPPath << '\n';
+  }
 
   std::string ErrMsg;
   auto DynlibHandle = std::make_unique<llvm::sys::DynamicLibrary>(
-      llvm::sys::DynamicLibrary::getPermanentLibrary(DynamicHIPPath, &ErrMsg));
+      llvm::sys::DynamicLibrary::getPermanentLibrary(DynamicHIPPath.c_str(),
+                                                     &ErrMsg));
   if (!DynlibHandle->isValid()) {
     llvm::errs() << "Failed to load " << DynamicHIPPath << ": " << ErrMsg
                  << '\n';
diff --git a/clang/tools/cir-opt/CMakeLists.txt b/clang/tools/cir-opt/CMakeLists.txt
index 75bec5f4e1b0b..ca7ee44f6fd75 100644
--- a/clang/tools/cir-opt/CMakeLists.txt
+++ b/clang/tools/cir-opt/CMakeLists.txt
@@ -4,6 +4,17 @@ get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
 include_directories(${LLVM_MAIN_SRC_DIR}/../mlir/include)
 include_directories(${CMAKE_BINARY_DIR}/tools/mlir/include)
 
+# GCC, unlike clang, issues a warning when one virtual function is overridden
+# in a derived class but one or more other virtual functions with the same
+# name and different signature from a base class are not overridden. This
+# leads to many warnings in the MLIR and ClangIR code when using the
+# OpenConversionPattern<>::matchAndRewrite() function in the ordinary way.
+# The "hiding" behavior is what we want, so we're just disabling the warning
+# here.
+if (LLVM_COMPILER_IS_GCC_COMPATIBLE AND (NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang"))
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-overloaded-virtual")
+endif()
+
 add_clang_tool(cir-opt
   cir-opt.cpp
 )
diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index 10dfb008cbec2..dcfed14746200 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -495,6 +495,10 @@ Expected<StringRef> clang(ArrayRef<StringRef> InputFiles, const ArgList &Args) {
     Triple.isAMDGPU() ? CmdArgs.push_back(Args.MakeArgString("-mcpu=" + Arch))
                       : CmdArgs.push_back(Args.MakeArgString("-march=" + Arch));
 
+  // AMDGPU is always in LTO mode currently.
+  if (Triple.isAMDGPU())
+    CmdArgs.push_back("-flto");
+
   // Forward all of the `--offload-opt` and similar options to the device.
   for (auto &Arg : Args.filtered(OPT_offload_opt_eq_minus, OPT_mllvm))
     CmdArgs.append(
@@ -599,9 +603,11 @@ Expected<StringRef> writeOffloadFile(const OffloadFile &File) {
 
   StringRef Prefix =
       sys::path::stem(Binary.getMemoryBufferRef().getBufferIdentifier());
-
-  auto TempFileOrErr = createOutputFile(
-      Prefix + "-" + Binary.getTriple() + "-" + Binary.getArch(), "o");
+  SmallString<128> Filename;
+  (Prefix + "-" + Binary.getTriple() + "-" + Binary.getArch())
+      .toVector(Filename);
+  llvm::replace(Filename, ':', '-');
+  auto TempFileOrErr = createOutputFile(Filename, "o");
   if (!TempFileOrErr)
     return TempFileOrErr.takeError();
 
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index f412a38a92337..e498a875bbbe8 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2536,6 +2536,8 @@ void OMPClauseEnqueue::VisitOMPDynamicAllocatorsClause(
 void OMPClauseEnqueue::VisitOMPAtomicDefaultMemOrderClause(
     const OMPAtomicDefaultMemOrderClause *) {}
 
+void OMPClauseEnqueue::VisitOMPSelfMapsClause(const OMPSelfMapsClause *) {}
+
 void OMPClauseEnqueue::VisitOMPAtClause(const OMPAtClause *) {}
 
 void OMPClauseEnqueue::VisitOMPSeverityClause(const OMPSeverityClause *) {}
@@ -2974,6 +2976,9 @@ void OpenACCClauseEnqueue::VisitIndependentClause(
     const OpenACCIndependentClause &C) {}
 void OpenACCClauseEnqueue::VisitSeqClause(const OpenACCSeqClause &C) {}
 void OpenACCClauseEnqueue::VisitNoHostClause(const OpenACCNoHostClause &C) {}
+void OpenACCClauseEnqueue::VisitBindClause(const OpenACCBindClause &C) {
+  assert(false && "TODO ERICH");
+}
 void OpenACCClauseEnqueue::VisitFinalizeClause(const OpenACCFinalizeClause &C) {
 }
 void OpenACCClauseEnqueue::VisitIfPresentClause(
diff --git a/clang/unittests/AST/DeclPrinterTest.cpp b/clang/unittests/AST/DeclPrinterTest.cpp
index 6945dff537cae..124b1a166cb18 100644
--- a/clang/unittests/AST/DeclPrinterTest.cpp
+++ b/clang/unittests/AST/DeclPrinterTest.cpp
@@ -1196,21 +1196,21 @@ TEST(DeclPrinter, TestUnnamedTemplateParameters) {
 }
 
 TEST(DeclPrinter, TestUnnamedTemplateParametersPacks) {
-  ASSERT_TRUE(PrintedDeclCXX17Matches(
-      "template <typename ..., int ...,"
-      " template <typename ..., bool ...> class ...> void A();",
-      functionTemplateDecl(hasName("A")).bind("id"),
-      "template <typename ..., int ...,"
-      " template <typename ..., bool ...> class ...> void A()"));
+  ASSERT_TRUE(
+      PrintedDeclCXX17Matches("template <typename ..., int ...,"
+                              " template <typename ...> class ...> void A();",
+                              functionTemplateDecl(hasName("A")).bind("id"),
+                              "template <typename ..., int ...,"
+                              " template <typename ...> class ...> void A()"));
 }
 
 TEST(DeclPrinter, TestNamedTemplateParametersPacks) {
   ASSERT_TRUE(PrintedDeclCXX17Matches(
       "template <typename ...T, int ...I,"
-      " template <typename ...X, bool ...B> class ...Z> void A();",
+      " template <typename ...X> class ...Z> void A();",
       functionTemplateDecl(hasName("A")).bind("id"),
       "template <typename ...T, int ...I,"
-      " template <typename ...X, bool ...B> class ...Z> void A()"));
+      " template <typename ...X> class ...Z> void A()"));
 }
 
 TEST(DeclPrinter, TestTemplateTemplateParameterWrittenWithTypename) {
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
index 4e6baedae2be5..49abe881eeabb 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -2028,6 +2028,13 @@ TEST_P(ASTMatchersTest, TemplateArgumentCountIs) {
   EXPECT_TRUE(
       notMatches("template<typename T> struct C {}; C<int> c;",
                  templateSpecializationType(templateArgumentCountIs(2))));
+
+  const char *FuncTemplateCode =
+      "template<typename T> T f(); auto v = f<int>();";
+  EXPECT_TRUE(
+      matches(FuncTemplateCode, functionDecl(templateArgumentCountIs(1))));
+  EXPECT_TRUE(
+      notMatches(FuncTemplateCode, functionDecl(templateArgumentCountIs(2))));
 }
 
 TEST_P(ASTMatchersTest, IsIntegral) {
diff --git a/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp b/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp
index 5031e17188e17..46fad6b655c4d 100644
--- a/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp
@@ -3829,7 +3829,7 @@ TEST_P(UncheckedOptionalAccessTest, ConstBoolAccessor) {
     };
 
     void target(A& a) {
-      std::optional<int> opt;
+      $ns::$optional<int> opt;
       if (a.isFoo()) {
         opt = 1;
       }
@@ -3851,7 +3851,7 @@ TEST_P(UncheckedOptionalAccessTest, ConstBoolAccessorWithModInBetween) {
     };
 
     void target(A& a) {
-      std::optional<int> opt;
+      $ns::$optional<int> opt;
       if (a.isFoo()) {
         opt = 1;
       }
@@ -3870,13 +3870,13 @@ TEST_P(UncheckedOptionalAccessTest,
 
     struct A {
       const $ns::$optional<int>& get() const { return x; }
-      
+
       $ns::$optional<int> x;
     };
 
     struct B {
       const A& getA() const { return a; }
-    
+
       A a;
     };
 
@@ -3896,13 +3896,13 @@ TEST_P(
 
     struct A {
       const $ns::$optional<int>& get() const { return x; }
-      
+
       $ns::$optional<int> x;
     };
 
     struct B {
       const A& getA() const { return a; }
-    
+
       A a;
     };
 
@@ -3919,13 +3919,13 @@ TEST_P(UncheckedOptionalAccessTest,
 
     struct A {
       const $ns::$optional<int>& get() const { return x; }
-      
+
       $ns::$optional<int> x;
     };
 
     struct B {
       const A& getA() const { return a; }
-    
+
       A a;
     };
 
@@ -3945,13 +3945,13 @@ TEST_P(UncheckedOptionalAccessTest,
 
     struct A {
       const $ns::$optional<int>& get() const { return x; }
-      
+
       $ns::$optional<int> x;
     };
 
     struct B {
       const A copyA() const { return a; }
-    
+
       A a;
     };
 
@@ -3970,13 +3970,13 @@ TEST_P(UncheckedOptionalAccessTest,
 
     struct A {
       const $ns::$optional<int>& get() const { return x; }
-      
+
       $ns::$optional<int> x;
     };
 
     struct B {
       A& getA() { return a; }
-    
+
       A a;
     };
 
@@ -4031,23 +4031,23 @@ TEST_P(
     ConstRefAccessorToOptionalViaConstRefAccessorToHoldingObjectWithAnotherConstCallAfterCheck) {
   ExpectDiagnosticsFor(R"cc(
       #include "unchecked_optional_access_test.h"
-  
+
       struct A {
         const $ns::$optional<int>& get() const { return x; }
-      
+
         $ns::$optional<int> x;
       };
-  
+
       struct B {
         const A& getA() const { return a; }
-  
+
         void callWithoutChanges() const { 
           // no-op 
         }
-  
+
         A a;
       };
-  
+
       void target(B& b) {  
         if (b.getA().get().has_value()) {
           b.callWithoutChanges(); // calling const method which cannot change A
@@ -4057,6 +4057,34 @@ TEST_P(
     )cc");
 }
 
+TEST_P(UncheckedOptionalAccessTest, ConstPointerRefAccessor) {
+  // A crash reproducer for https://github.com/llvm/llvm-project/issues/125589
+  // NOTE: we currently cache const ref accessors's locations.
+  // If we want to support const ref to pointers or bools, we should initialize
+  // their values.
+  ExpectDiagnosticsFor(R"cc(
+    #include "unchecked_optional_access_test.h"
+
+    struct A {
+      $ns::$optional<int> x;
+    };
+
+    struct PtrWrapper {
+      A*& getPtrRef() const;
+      void reset(A*);
+    };
+
+    void target(PtrWrapper p) {
+      if (p.getPtrRef()->x) {
+        *p.getPtrRef()->x;  // [[unsafe]]
+        p.reset(nullptr);
+        *p.getPtrRef()->x;  // [[unsafe]]
+      }
+    }
+  )cc",
+                       /*IgnoreSmartPointerDereference=*/false);
+}
+
 // FIXME: Add support for:
 // - constructors (copy, move)
 // - assignment operators (default, copy, move)
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index ae2eaf70de1c2..6cd424c20720d 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -3501,46 +3501,47 @@ TEST_F(FormatTest, UnderstandsAccessSpecifiers) {
                "label:\n"
                "  signals.baz();\n"
                "}");
-  verifyFormat("private[1];");
+
+  const auto Style = getLLVMStyle(FormatStyle::LK_C);
+  verifyFormat("private[1];", Style);
   verifyFormat("testArray[public] = 1;");
-  verifyFormat("public();");
+  verifyFormat("public();", Style);
   verifyFormat("myFunc(public);");
   verifyFormat("std::vector<int> testVec = {private};");
-  verifyFormat("private.p = 1;");
+  verifyFormat("private.p = 1;", Style);
   verifyFormat("void function(private...) {};");
   verifyFormat("if (private && public)");
-  verifyFormat("private &= true;");
+  verifyFormat("private &= true;", Style);
   verifyFormat("int x = private * public;");
-  verifyFormat("public *= private;");
+  verifyFormat("public *= private;", Style);
   verifyFormat("int x = public + private;");
-  verifyFormat("private++;");
+  verifyFormat("private++;", Style);
   verifyFormat("++private;");
-  verifyFormat("public += private;");
-  verifyFormat("public = public - private;");
-  verifyFormat("public->foo();");
-  verifyFormat("private--;");
+  verifyFormat("public += private;", Style);
+  verifyFormat("public = public - private;", Style);
+  verifyFormat("public->foo();", Style);
+  verifyFormat("private--;", Style);
   verifyFormat("--private;");
-  verifyFormat("public -= 1;");
+  verifyFormat("public -= 1;", Style);
   verifyFormat("if (!private && !public)");
-  verifyFormat("public != private;");
+  verifyFormat("public != private;", Style);
   verifyFormat("int x = public / private;");
-  verifyFormat("public /= 2;");
-  verifyFormat("public = public % 2;");
-  verifyFormat("public %= 2;");
+  verifyFormat("public /= 2;", Style);
+  verifyFormat("public = public % 2;", Style);
+  verifyFormat("public %= 2;", Style);
   verifyFormat("if (public < private)");
-  verifyFormat("public << private;");
-  verifyFormat("public <<= private;");
+  verifyFormat("public << private;", Style);
+  verifyFormat("public <<= private;", Style);
   verifyFormat("if (public > private)");
-  verifyFormat("public >> private;");
-  verifyFormat("public >>= private;");
-  verifyFormat("public ^ private;");
-  verifyFormat("public ^= private;");
-  verifyFormat("public | private;");
-  verifyFormat("public |= private;");
+  verifyFormat("public >> private;", Style);
+  verifyFormat("public >>= private;", Style);
+  verifyFormat("public ^ private;", Style);
+  verifyFormat("public ^= private;", Style);
+  verifyFormat("public | private;", Style);
+  verifyFormat("public |= private;", Style);
   verifyFormat("auto x = private ? 1 : 2;");
   verifyFormat("if (public == private)");
   verifyFormat("void foo(public, private)");
-  verifyFormat("public::foo();");
 
   verifyFormat("class A {\n"
                "public:\n"
@@ -28186,6 +28187,10 @@ TEST_F(FormatTest, RemoveParentheses) {
   verifyFormat("foo((a, b));", "foo(((a), b));", Style);
   verifyFormat("foo((a, b));", "foo((a, (b)));", Style);
   verifyFormat("foo((a, b, c));", "foo((a, ((b)), c));", Style);
+  verifyFormat("(..., (hash_a = hash_combine(hash_a, hash_b)));",
+               "(..., ((hash_a = hash_combine(hash_a, hash_b))));", Style);
+  verifyFormat("((hash_a = hash_combine(hash_a, hash_b)), ...);",
+               "(((hash_a = hash_combine(hash_a, hash_b))), ...);", Style);
   verifyFormat("return (0);", "return (((0)));", Style);
   verifyFormat("return (({ 0; }));", "return ((({ 0; })));", Style);
   verifyFormat("return ((... && std::is_convertible_v<TArgsLocal, TArgs>));",
diff --git a/clang/utils/TableGen/ClangAttrEmitter.cpp b/clang/utils/TableGen/ClangAttrEmitter.cpp
index af7478b7986f9..3d90bfbc75b32 100644
--- a/clang/utils/TableGen/ClangAttrEmitter.cpp
+++ b/clang/utils/TableGen/ClangAttrEmitter.cpp
@@ -4932,6 +4932,9 @@ void EmitClangAttrParsedAttrKinds(const RecordKeeper &Records,
           Matches = &Pragma;
         } else if (Variety == "HLSLAnnotation") {
           Matches = &HLSLAnnotation;
+          if (RawSpelling.compare(RawSpelling.lower()) != 0)
+            PrintError(S.getSpellingRecord().getLoc(),
+                       "HLSLAnnotation Attribute must be lower case.");
         }
 
         assert(Matches && "Unsupported spelling variety found");
diff --git a/clang/www/c_status.html b/clang/www/c_status.html
index 4e912987b69c6..4432e62b47520 100644
--- a/clang/www/c_status.html
+++ b/clang/www/c_status.html
@@ -265,51 +265,45 @@ <h2 id="c2y">C2y implementation status</h2>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3363.pdf">N3363</a></td>
       <td class="unknown" align="center">Unknown</td>
 	</tr>
-    <tr>
-      <td>Integer and arithmetic constant expressions, v. 2</td>
-      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3459.pdf">N3459</a></td>
-      <!-- first and second change only -->
-      <td class="unknown" align="center">Unknown</td>
-	</tr>
     <tr>
       <td>Preprocessor integer expressions, v. 2</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3505.pdf">N3505</a></td>
-      <td class="unknown" align="center">Unknown</td>
+      <td class="full" align="center">Yes</td>
 	</tr>
     <tr>
       <td>Initialization of anonymous structures and unions</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3451.pdf">N3451</a></td>
-      <td class="unknown" align="center">Unknown</td>
+      <td class="full" align="center">Yes</td>
 	</tr>
     <tr>
       <td>Slay Some Earthly Demons X</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3409.pdf">N3409</a></td>
-      <td class="unknown" align="center">Unknown</td>
+      <td class="unreleased" align="center">Clang 21</td>
 	</tr>
     <tr>
       <td>Slay Some Earthly Demons XI</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3410.pdf">N3410</a></td>
-      <td class="unknown" align="center">Unknown</td>
+      <td class="none" align="center">No</td>
 	</tr>
     <tr>
       <td>Slay Some Earthly Demons XII</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3411.pdf">N3411</a></td>
-      <td class="unknown" align="center">Unknown</td>
+      <td class="unreleased" align="center">Clang 21</td>
 	</tr>
     <tr>
       <td>Slay Some Earthly Demons XIII</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3478.pdf">N3478</a></td>
-      <td class="unknown" align="center">Unknown</td>
+      <td class="full" align="center">Yes</td>
 	</tr>
     <tr>
       <td>Slay Some Earthly Demons XIV</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3418.pdf">N3418</a></td>
-      <td class="unknown" align="center">Unknown</td>
+      <td class="none" align="center">No</td>
 	</tr>
     <tr>
       <td>Slay Some Earthly Demons XVI</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3481.pdf">N3481</a></td>
-      <td class="unknown" align="center">Unknown</td>
+      <td class="full" align="center">Yes</td>
 	</tr>
     <tr>
       <td>Slay Some Earthly Demons XVII</td>
@@ -319,7 +313,7 @@ <h2 id="c2y">C2y implementation status</h2>
     <tr>
       <td>Clarify the specification of the width macros</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3496.htm">N3496</a></td>
-      <td class="unknown" align="center">Unknown</td>
+      <td class="full" align="center">Clang 20</td>
 	</tr>
     <tr>
       <td>Complex operators</td>
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 919996bd700b4..b7888d2365acc 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -13537,7 +13537,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2285.html">2285</a></td>
     <td>CD5</td>
     <td>Issues with structured bindings</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 4</td>
   </tr>
   <tr id="2286">
     <td><a href="https://cplusplus.github.io/CWG/issues/2286.html">2286</a></td>
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index 3797654952319..64b731c19c517 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -223,7 +223,7 @@ <h2 id="cxx26">C++2c implementation status</h2>
  <tr>
   <td>Structured binding declaration as a condition</td>
   <td><a href="https://wg21.link/P0963R3">P0963R3</a></td>
-  <td class="none" align="center">No</td>
+  <td class="unreleased" align="center">Clang 21</td>
  </tr>
  <!--Poland, Fall 2024-->
  <tr>
diff --git a/clang/www/get_involved.html b/clang/www/get_involved.html
index 99fa03abb6234..1939289463f3d 100755
--- a/clang/www/get_involved.html
+++ b/clang/www/get_involved.html
@@ -83,7 +83,7 @@ <h2 id="criteria">Contributing Extensions to Clang</h2>
 for Clang. The benefits of the extension need to be evaluated against
 these costs. The Clang project uses the following criteria for this
 evaluation:</p>
-
+<p>
 <ol>
   <li>Evidence of a significant user community: This is based on a number of
   factors, including an existing user community, the perceived likelihood that
@@ -133,7 +133,36 @@ <h2 id="criteria">Contributing Extensions to Clang</h2>
   support the extension and what level of support is expected. The impacted
   project communities need to agree with that plan.</li>
 </ol>
-
+</p>
+<p>
+The Clang community uses an RFC process to evaluate potential extensions for
+inclusion in the tool. First, write a post in the <code>Clang Frontend</code>
+category of <a href="https://discourse.llvm.org/c/clang/6">Discourse</a>. The
+title should include <code>[RFC]</code> so it is clear that it is a proposed
+change. The post should have detailed information about the change itself, the
+motivation for needing the change, how it addresses the criteria listed above,
+and any other relevant details the community should be aware of.
+</p>
+<p>
+The community will discuss the proposal in Discourse, asking questions about
+the proposal to improve their understanding and giving support or dissent for
+the idea. Eventually, consensus will be determined as to whether the proposal
+should proceed or be rejected. If a proposal receives little or no feedback,
+that typically means that the proposal is rejected due to lack of interest.
+</p>
+<p>
+Sometimes, a consensus position is unclear and the proposal author will need
+additional guidance on what next steps to take. In such a case, the Clang Area
+Team may get involved. The Clang Area Team secretary will proactively look for
+RFCs that appear to not have a clear path forward to add them to the team's
+agenda. If anyone would like to put an RFC onto the team's agenda explicitly,
+they can tag any one of the <a href="https://github.com/llvm/llvm-project/blob/main/clang/AreaTeamMembers.txt">
+Clang Area Team</a> members in a comment on the RFC to get their attention. The
+area team will host a meeting to discuss the RFC and determine next steps for
+the proposal. In the event no clear community consensus position seems likely
+to ever form, the Clang Area Team will make a final judgement call on whether
+the RFC will proceed or not. <!-- TODO: link to area team processes -->
+</p>
 </div>
 </body>
 </html>
diff --git a/clang/www/get_started.html b/clang/www/get_started.html
index 8e4d36640be73..0087aba168839 100755
--- a/clang/www/get_started.html
+++ b/clang/www/get_started.html
@@ -128,7 +128,7 @@ <h3 id="buildWindows">Using Visual Studio</h3>
         project files.  Get it from:
         <a href="https://cmake.org/download/">
         https://cmake.org/download/</a></li>
-    <li><b>Visual Studio 2019 16.7 or later</b></li>
+    <li><b>Visual Studio 2019 16.7 or later</b>. This tutorial assumes Visual Studio 2022.</li>
     <li><b>Python</b>.  It is used to run the clang test suite. Get it from:
         <a href="https://www.python.org/download/">
         https://www.python.org/download/</a></li>
@@ -159,8 +159,8 @@ <h3 id="buildWindows">Using Visual Studio</h3>
     <li><tt>mkdir build</tt> (for building without polluting the source dir)</li>
     <li><tt>cd build</tt></li>
     <li>
-      If you are using Visual Studio 2019:
-      <tt>cmake -DLLVM_ENABLE_PROJECTS=clang -G "Visual Studio 16 2019" -A x64 -Thost=x64 ..\llvm</tt><br/>
+      If you are using Visual Studio 2022:
+      <tt>cmake -DLLVM_ENABLE_PROJECTS=clang -G "Visual Studio 17 2022" -A x64 -Thost=x64 ..\llvm</tt><br/>
       <tt>-Thost=x64</tt> is required, since the 32-bit linker will run out of memory.
     </li>
     <li>To generate x86 binaries instead of x64, pass <tt>-A Win32</tt>.</li>
@@ -198,12 +198,12 @@ <h3 id="buildWindowsNinja">Using Ninja alongside Visual Studio</h3>
       <li>If you open the start menu and search for "Command Prompt", you should
         see shortcuts created by Visual Studio to do this. To use native x64
         tools, choose the one titled "x64 Native Tools Command Prompt for VS
-        2017".</li>
+        2022".</li>
       <li> Alternatively, launch a regular <tt>cmd</tt> prompt and run the
-        appropriate vcvarsall.bat incantation. To get the 2017 x64 tools, this
+        appropriate vcvarsall.bat incantation. To get the 2022 x64 tools, this
         would be:<br/>
-        <tt>"C:\Program Files (x86)\Microsoft Visual
-          Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" x64</tt>
+        <tt>"C:\Program Files\Microsoft Visual
+          Studio\2022\Community\VC\Auxiliary\Build\vcvarsall.bat" x64</tt>
       </li>
     </ul>
   </li>
diff --git a/compiler-rt/include/orc_rt/c_api.h b/compiler-rt/include/orc_rt/c_api.h
index f428e94b1a7e8..6964ec0f5a361 100644
--- a/compiler-rt/include/orc_rt/c_api.h
+++ b/compiler-rt/include/orc_rt/c_api.h
@@ -48,46 +48,46 @@ ORC_RT_C_EXTERN_C_BEGIN
 typedef union {
   char *ValuePtr;
   char Value[sizeof(char *)];
-} orc_rt_CWrapperFunctionResultDataUnion;
+} orc_rt_WrapperFunctionResultDataUnion;
 
 /**
- * orc_rt_CWrapperFunctionResult is a kind of C-SmallVector with an
+ * orc_rt_WrapperFunctionResult is a kind of C-SmallVector with an
  * out-of-band error state.
  *
  * If Size == 0 and Data.ValuePtr is non-zero then the value is in the
  * 'out-of-band error' state, and Data.ValuePtr points at a malloc-allocated,
  * null-terminated string error message.
  *
- * If Size <= sizeof(orc_rt_CWrapperFunctionResultData) then the value is in
+ * If Size <= sizeof(orc_rt_WrapperFunctionResultData) then the value is in
  * the 'small' state and the content is held in the first Size bytes of
  * Data.Value.
  *
- * If Size > sizeof(orc_rt_CWrapperFunctionResultData) then the value is in the
+ * If Size > sizeof(orc_rt_WrapperFunctionResultData) then the value is in the
  * 'large' state and the content is held in the first Size bytes of the
  * memory pointed to by Data.ValuePtr. This memory must have been allocated by
  * malloc, and will be freed with free when this value is destroyed.
  */
 typedef struct {
-  orc_rt_CWrapperFunctionResultDataUnion Data;
+  orc_rt_WrapperFunctionResultDataUnion Data;
   size_t Size;
-} orc_rt_CWrapperFunctionResult;
+} orc_rt_WrapperFunctionResult;
 
 /**
- * Zero-initialize an orc_rt_CWrapperFunctionResult.
+ * Zero-initialize an orc_rt_WrapperFunctionResult.
  */
 static inline void
-orc_rt_CWrapperFunctionResultInit(orc_rt_CWrapperFunctionResult *R) {
+orc_rt_WrapperFunctionResultInit(orc_rt_WrapperFunctionResult *R) {
   R->Size = 0;
   R->Data.ValuePtr = 0;
 }
 
 /**
- * Create an orc_rt_CWrapperFunctionResult with an uninitialized buffer of
+ * Create an orc_rt_WrapperFunctionResult with an uninitialized buffer of
  * size Size. The buffer is returned via the DataPtr argument.
  */
-static inline orc_rt_CWrapperFunctionResult
-orc_rt_CWrapperFunctionResultAllocate(size_t Size) {
-  orc_rt_CWrapperFunctionResult R;
+static inline orc_rt_WrapperFunctionResult
+orc_rt_WrapperFunctionResultAllocate(size_t Size) {
+  orc_rt_WrapperFunctionResult R;
   R.Size = Size;
   // If Size is 0 ValuePtr must be 0 or it is considered an out-of-band error.
   R.Data.ValuePtr = 0;
@@ -99,9 +99,9 @@ orc_rt_CWrapperFunctionResultAllocate(size_t Size) {
 /**
  * Create an orc_rt_WrapperFunctionResult from the given data range.
  */
-static inline orc_rt_CWrapperFunctionResult
-orc_rt_CreateCWrapperFunctionResultFromRange(const char *Data, size_t Size) {
-  orc_rt_CWrapperFunctionResult R;
+static inline orc_rt_WrapperFunctionResult
+orc_rt_CreateWrapperFunctionResultFromRange(const char *Data, size_t Size) {
+  orc_rt_WrapperFunctionResult R;
   R.Size = Size;
   if (R.Size > sizeof(R.Data.Value)) {
     char *Tmp = (char *)malloc(Size);
@@ -113,28 +113,28 @@ orc_rt_CreateCWrapperFunctionResultFromRange(const char *Data, size_t Size) {
 }
 
 /**
- * Create an orc_rt_CWrapperFunctionResult by copying the given string,
+ * Create an orc_rt_WrapperFunctionResult by copying the given string,
  * including the null-terminator.
  *
  * This function copies the input string. The client is responsible for freeing
  * the ErrMsg arg.
  */
-static inline orc_rt_CWrapperFunctionResult
-orc_rt_CreateCWrapperFunctionResultFromString(const char *Source) {
-  return orc_rt_CreateCWrapperFunctionResultFromRange(Source,
-                                                      strlen(Source) + 1);
+static inline orc_rt_WrapperFunctionResult
+orc_rt_CreateWrapperFunctionResultFromString(const char *Source) {
+  return orc_rt_CreateWrapperFunctionResultFromRange(Source,
+                                                     strlen(Source) + 1);
 }
 
 /**
- * Create an orc_rt_CWrapperFunctionResult representing an out-of-band
+ * Create an orc_rt_WrapperFunctionResult representing an out-of-band
  * error.
  *
  * This function copies the input string. The client is responsible for freeing
  * the ErrMsg arg.
  */
-static inline orc_rt_CWrapperFunctionResult
-orc_rt_CreateCWrapperFunctionResultFromOutOfBandError(const char *ErrMsg) {
-  orc_rt_CWrapperFunctionResult R;
+static inline orc_rt_WrapperFunctionResult
+orc_rt_CreateWrapperFunctionResultFromOutOfBandError(const char *ErrMsg) {
+  orc_rt_WrapperFunctionResult R;
   R.Size = 0;
   char *Tmp = (char *)malloc(strlen(ErrMsg) + 1);
   strcpy(Tmp, ErrMsg);
@@ -143,11 +143,11 @@ orc_rt_CreateCWrapperFunctionResultFromOutOfBandError(const char *ErrMsg) {
 }
 
 /**
- * This should be called to destroy orc_rt_CWrapperFunctionResult values
+ * This should be called to destroy orc_rt_WrapperFunctionResult values
  * regardless of their state.
  */
 static inline void
-orc_rt_DisposeCWrapperFunctionResult(orc_rt_CWrapperFunctionResult *R) {
+orc_rt_DisposeWrapperFunctionResult(orc_rt_WrapperFunctionResult *R) {
   if (R->Size > sizeof(R->Data.Value) ||
       (R->Size == 0 && R->Data.ValuePtr))
     free(R->Data.ValuePtr);
@@ -155,22 +155,22 @@ orc_rt_DisposeCWrapperFunctionResult(orc_rt_CWrapperFunctionResult *R) {
 
 /**
  * Get a pointer to the data contained in the given
- * orc_rt_CWrapperFunctionResult.
+ * orc_rt_WrapperFunctionResult.
  */
 static inline char *
-orc_rt_CWrapperFunctionResultData(orc_rt_CWrapperFunctionResult *R) {
+orc_rt_WrapperFunctionResultData(orc_rt_WrapperFunctionResult *R) {
   assert((R->Size != 0 || R->Data.ValuePtr == NULL) &&
          "Cannot get data for out-of-band error value");
   return R->Size > sizeof(R->Data.Value) ? R->Data.ValuePtr : R->Data.Value;
 }
 
 /**
- * Safely get the size of the given orc_rt_CWrapperFunctionResult.
+ * Safely get the size of the given orc_rt_WrapperFunctionResult.
  *
  * Asserts that we're not trying to access the size of an error value.
  */
 static inline size_t
-orc_rt_CWrapperFunctionResultSize(const orc_rt_CWrapperFunctionResult *R) {
+orc_rt_WrapperFunctionResultSize(const orc_rt_WrapperFunctionResult *R) {
   assert((R->Size != 0 || R->Data.ValuePtr == NULL) &&
          "Cannot get size for out-of-band error value");
   return R->Size;
@@ -178,22 +178,22 @@ orc_rt_CWrapperFunctionResultSize(const orc_rt_CWrapperFunctionResult *R) {
 
 /**
  * Returns 1 if this value is equivalent to a value just initialized by
- * orc_rt_CWrapperFunctionResultInit, 0 otherwise.
+ * orc_rt_WrapperFunctionResultInit, 0 otherwise.
  */
 static inline size_t
-orc_rt_CWrapperFunctionResultEmpty(const orc_rt_CWrapperFunctionResult *R) {
+orc_rt_WrapperFunctionResultEmpty(const orc_rt_WrapperFunctionResult *R) {
   return R->Size == 0 && R->Data.ValuePtr == 0;
 }
 
 /**
  * Returns a pointer to the out-of-band error string for this
- * orc_rt_CWrapperFunctionResult, or null if there is no error.
+ * orc_rt_WrapperFunctionResult, or null if there is no error.
  *
- * The orc_rt_CWrapperFunctionResult retains ownership of the error
+ * The orc_rt_WrapperFunctionResult retains ownership of the error
  * string, so it should be copied if the caller wishes to preserve it.
  */
-static inline const char *orc_rt_CWrapperFunctionResultGetOutOfBandError(
-    const orc_rt_CWrapperFunctionResult *R) {
+static inline const char *orc_rt_WrapperFunctionResultGetOutOfBandError(
+    const orc_rt_WrapperFunctionResult *R) {
   return R->Size == 0 ? R->Data.ValuePtr : 0;
 }
 
diff --git a/compiler-rt/lib/ctx_profile/CtxInstrContextNode.h b/compiler-rt/lib/ctx_profile/CtxInstrContextNode.h
index fe8ddcdf79129..0fc4883305145 100644
--- a/compiler-rt/lib/ctx_profile/CtxInstrContextNode.h
+++ b/compiler-rt/lib/ctx_profile/CtxInstrContextNode.h
@@ -122,6 +122,12 @@ class ProfileWriter {
   virtual void startContextSection() = 0;
   virtual void writeContextual(const ctx_profile::ContextNode &RootNode) = 0;
   virtual void endContextSection() = 0;
+
+  virtual void startFlatSection() = 0;
+  virtual void writeFlat(ctx_profile::GUID Guid, const uint64_t *Buffer,
+                         size_t BufferSize) = 0;
+  virtual void endFlatSection() = 0;
+
   virtual ~ProfileWriter() = default;
 };
 } // namespace ctx_profile
diff --git a/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp b/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp
index 992aa94a6631d..d7ec8fde4ec7d 100644
--- a/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp
+++ b/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp
@@ -8,6 +8,8 @@
 
 #include "CtxInstrProfiling.h"
 #include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_atomic_clang.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_dense_map.h"
 #include "sanitizer_common/sanitizer_libc.h"
@@ -27,6 +29,20 @@ __sanitizer::SpinMutex AllContextsMutex;
 SANITIZER_GUARDED_BY(AllContextsMutex)
 __sanitizer::Vector<ContextRoot *> AllContextRoots;
 
+__sanitizer::atomic_uintptr_t AllFunctionsData = {};
+
+// Keep all the functions for which we collect a flat profile in a linked list.
+__sanitizer::SpinMutex FlatCtxArenaMutex;
+SANITIZER_GUARDED_BY(FlatCtxArenaMutex)
+Arena *FlatCtxArenaHead = nullptr;
+SANITIZER_GUARDED_BY(FlatCtxArenaMutex)
+Arena *FlatCtxArena = nullptr;
+
+// Set to true when we enter a root, and false when we exit - regardless if this
+// thread collects a contextual profile for that root.
+__thread bool IsUnderContext = false;
+__sanitizer::atomic_uint8_t ProfilingStarted = {};
+
 // utility to taint a pointer by setting the LSB. There is an assumption
 // throughout that the addresses of contexts are even (really, they should be
 // align(8), but "even"-ness is the minimum assumption)
@@ -109,7 +125,10 @@ void resetContextNode(ContextNode &Node) {
       resetContextNode(*Next);
 }
 
-void onContextEnter(ContextNode &Node) { ++Node.counters()[0]; }
+ContextNode *onContextEnter(ContextNode &Node) {
+  ++Node.counters()[0];
+  return &Node;
+}
 
 } // namespace
 
@@ -182,12 +201,75 @@ ContextNode *getCallsiteSlow(GUID Guid, ContextNode **InsertionPoint,
   return Ret;
 }
 
-ContextNode *__llvm_ctx_profile_get_context(void *Callee, GUID Guid,
-                                            uint32_t NumCounters,
+ContextNode *getFlatProfile(FunctionData &Data, GUID Guid,
+                            uint32_t NumCounters) {
+  if (ContextNode *Existing = Data.FlatCtx)
+    return Existing;
+  {
+    // We could instead try to take the lock and, if that fails, return
+    // TheScratchContext. But that could leave message pump loops more sparsely
+    // profiled than everything else. Maybe that doesn't matter, and we can
+    // optimize this later.
+    __sanitizer::GenericScopedLock<__sanitizer::StaticSpinMutex> L(&Data.Mutex);
+    if (ContextNode *Existing = Data.FlatCtx)
+      return Existing;
+
+    auto NeededSize = ContextNode::getAllocSize(NumCounters, 0);
+    char *AllocBuff = nullptr;
+    {
+      __sanitizer::GenericScopedLock<__sanitizer::SpinMutex> FL(
+          &FlatCtxArenaMutex);
+      if (FlatCtxArena)
+        AllocBuff = FlatCtxArena->tryBumpAllocate(NeededSize);
+      if (!AllocBuff) {
+        FlatCtxArena = Arena::allocateNewArena(getArenaAllocSize(NeededSize),
+                                               FlatCtxArena);
+        AllocBuff = FlatCtxArena->tryBumpAllocate(NeededSize);
+      }
+      if (!FlatCtxArenaHead)
+        FlatCtxArenaHead = FlatCtxArena;
+    }
+    auto *Ret = allocContextNode(AllocBuff, Guid, NumCounters, 0);
+    Data.FlatCtx = Ret;
+
+    Data.Next = reinterpret_cast<FunctionData *>(
+        __sanitizer::atomic_load_relaxed(&AllFunctionsData));
+    while (!__sanitizer::atomic_compare_exchange_strong(
+        &AllFunctionsData, reinterpret_cast<uintptr_t *>(&Data.Next),
+        reinterpret_cast<uintptr_t>(&Data),
+        __sanitizer::memory_order_release)) {
+    }
+  }
+
+  return Data.FlatCtx;
+}
+
+ContextNode *getUnhandledContext(FunctionData &Data, GUID Guid,
+                                 uint32_t NumCounters) {
+  // 1) if we are under a root (regardless if this thread is collecting or not a
+  // contextual profile for that root), do not collect a flat profile. We want
+  // to keep flat profiles only for activations that can't happen under a root,
+  // to avoid confusing profiles. We can, for example, combine flattened and
+  // flat profiles meaningfully, as we wouldn't double-count anything.
+  //
+  // 2) to avoid lengthy startup, don't bother with flat profiles until the
+  // profiling started. We would reset them anyway when profiling starts.
+  // HOWEVER. This does lose profiling for message pumps: those functions are
+  // entered once and never exit. They should be assumed to be entered before
+  // profiling starts - because profiling should start after the server is up
+  // and running (which is equivalent to "message pumps are set up").
+  if (IsUnderContext || !__sanitizer::atomic_load_relaxed(&ProfilingStarted))
+    return TheScratchContext;
+  return markAsScratch(
+      onContextEnter(*getFlatProfile(Data, Guid, NumCounters)));
+}
+
+ContextNode *__llvm_ctx_profile_get_context(FunctionData *Data, void *Callee,
+                                            GUID Guid, uint32_t NumCounters,
                                             uint32_t NumCallsites) {
   // fast "out" if we're not even doing contextual collection.
   if (!__llvm_ctx_profile_current_context_root)
-    return TheScratchContext;
+    return getUnhandledContext(*Data, Guid, NumCounters);
 
   // also fast "out" if the caller is scratch. We can see if it's scratch by
   // looking at the interior pointer into the subcontexts vector that the caller
@@ -196,7 +278,7 @@ ContextNode *__llvm_ctx_profile_get_context(void *Callee, GUID Guid,
   // precisely, aligned - 8 values)
   auto **CallsiteContext = consume(__llvm_ctx_profile_callsite[0]);
   if (!CallsiteContext || isScratch(CallsiteContext))
-    return TheScratchContext;
+    return getUnhandledContext(*Data, Guid, NumCounters);
 
   // if the callee isn't the expected one, return scratch.
   // Signal handler(s) could have been invoked at any point in the execution.
@@ -214,7 +296,7 @@ ContextNode *__llvm_ctx_profile_get_context(void *Callee, GUID Guid,
   // for that case.
   auto *ExpectedCallee = consume(__llvm_ctx_profile_expected_callee[0]);
   if (ExpectedCallee != Callee)
-    return TheScratchContext;
+    return getUnhandledContext(*Data, Guid, NumCounters);
 
   auto *Callsite = *CallsiteContext;
   // in the case of indirect calls, we will have all seen targets forming a
@@ -257,6 +339,7 @@ void setupContext(ContextRoot *Root, GUID Guid, uint32_t NumCounters,
 ContextNode *__llvm_ctx_profile_start_context(
     ContextRoot *Root, GUID Guid, uint32_t Counters,
     uint32_t Callsites) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
+  IsUnderContext = true;
   if (!Root->FirstMemBlock) {
     setupContext(Root, Guid, Counters, Callsites);
   }
@@ -272,6 +355,7 @@ ContextNode *__llvm_ctx_profile_start_context(
 
 void __llvm_ctx_profile_release_context(ContextRoot *Root)
     SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
+  IsUnderContext = false;
   if (__llvm_ctx_profile_current_context_root) {
     __llvm_ctx_profile_current_context_root = nullptr;
     Root->Taken.Unlock();
@@ -291,10 +375,12 @@ void __llvm_ctx_profile_start_collection() {
 
     resetContextNode(*Root->FirstNode);
   }
+  __sanitizer::atomic_store_relaxed(&ProfilingStarted, true);
   __sanitizer::Printf("[ctxprof] Initial NumMemUnits: %zu \n", NumMemUnits);
 }
 
 bool __llvm_ctx_profile_fetch(ProfileWriter &Writer) {
+  __sanitizer::atomic_store_relaxed(&ProfilingStarted, false);
   __sanitizer::GenericScopedLock<__sanitizer::SpinMutex> Lock(
       &AllContextsMutex);
 
@@ -310,17 +396,43 @@ bool __llvm_ctx_profile_fetch(ProfileWriter &Writer) {
     Writer.writeContextual(*Root->FirstNode);
   }
   Writer.endContextSection();
+  Writer.startFlatSection();
+  // The list progresses behind the head, so taking this snapshot allows the
+  // list to grow concurrently without causing a race condition with our
+  // traversing it.
+  const auto *Pos = reinterpret_cast<const FunctionData *>(
+      __sanitizer::atomic_load_relaxed(&AllFunctionsData));
+  for (; Pos; Pos = Pos->Next)
+    Writer.writeFlat(Pos->FlatCtx->guid(), Pos->FlatCtx->counters(),
+                     Pos->FlatCtx->counters_size());
+  Writer.endFlatSection();
   return true;
 }
 
 void __llvm_ctx_profile_free() {
-  __sanitizer::GenericScopedLock<__sanitizer::SpinMutex> Lock(
-      &AllContextsMutex);
-  for (int I = 0, E = AllContextRoots.Size(); I < E; ++I)
-    for (auto *A = AllContextRoots[I]->FirstMemBlock; A;) {
+  __sanitizer::atomic_store_relaxed(&ProfilingStarted, false);
+  {
+    __sanitizer::GenericScopedLock<__sanitizer::SpinMutex> Lock(
+        &AllContextsMutex);
+    for (int I = 0, E = AllContextRoots.Size(); I < E; ++I)
+      for (auto *A = AllContextRoots[I]->FirstMemBlock; A;) {
+        auto *C = A;
+        A = A->next();
+        __sanitizer::InternalFree(C);
+      }
+    AllContextRoots.Reset();
+  }
+  __sanitizer::atomic_store_relaxed(&AllFunctionsData, 0U);
+  {
+    __sanitizer::GenericScopedLock<__sanitizer::SpinMutex> Lock(
+        &FlatCtxArenaMutex);
+    FlatCtxArena = nullptr;
+    for (auto *A = FlatCtxArenaHead; A;) {
       auto *C = A;
-      A = A->next();
+      A = C->next();
       __sanitizer::InternalFree(C);
     }
-  AllContextRoots.Reset();
+
+    FlatCtxArenaHead = nullptr;
+  }
 }
diff --git a/compiler-rt/lib/ctx_profile/CtxInstrProfiling.h b/compiler-rt/lib/ctx_profile/CtxInstrProfiling.h
index 8a6949d4ec288..c41a77457178c 100644
--- a/compiler-rt/lib/ctx_profile/CtxInstrProfiling.h
+++ b/compiler-rt/lib/ctx_profile/CtxInstrProfiling.h
@@ -113,6 +113,28 @@ struct ContextRoot {
   static_assert(sizeof(Taken) == 1);
 };
 
+// This is allocated and zero-initialized by the compiler, the in-place
+// initialization serves mostly as self-documentation and for testing.
+// The design is influenced by the observation that typically (at least for
+// datacenter binaries, which is the motivating target of this profiler) less
+// than 10% of functions in a binary even appear in a profile (of any kind).
+//
+// 1) We could pre-allocate the flat profile storage in the compiler, just like
+// the flat instrumented profiling does. But that penalizes the static size of
+// the binary for little reason
+//
+// 2) We could do the above but zero-initialize the buffers (which should place
+// them in .bss), and dynamically populate them. This, though, would page-in
+// more memory upfront for the binary's runtime
+//
+// The current design trades off a bit of overhead at the first time a function
+// is encountered *for flat profiling* for avoiding size penalties.
+struct FunctionData {
+  FunctionData *Next = nullptr;
+  ContextNode *volatile FlatCtx = nullptr;
+  ::__sanitizer::StaticSpinMutex Mutex;
+};
+
 /// This API is exposed for testing. See the APIs below about the contract with
 /// LLVM.
 inline bool isScratch(const void *Ctx) {
@@ -152,7 +174,8 @@ void __llvm_ctx_profile_release_context(__ctx_profile::ContextRoot *Root);
 
 /// called for any other function than entry points, in the entry BB of such
 /// function. Same consideration about LSB of returned value as .._start_context
-ContextNode *__llvm_ctx_profile_get_context(void *Callee, GUID Guid,
+ContextNode *__llvm_ctx_profile_get_context(__ctx_profile::FunctionData *Data,
+                                            void *Callee, GUID Guid,
                                             uint32_t NumCounters,
                                             uint32_t NumCallsites);
 
diff --git a/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp b/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp
index 97292f9f1abff..01a8274774ecb 100644
--- a/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp
+++ b/compiler-rt/lib/ctx_profile/tests/CtxInstrProfilingTest.cpp
@@ -40,6 +40,7 @@ TEST(ArenaTest, Basic) {
 }
 
 TEST_F(ContextTest, Basic) {
+  __llvm_ctx_profile_start_collection();
   auto *Ctx = __llvm_ctx_profile_start_context(&Root, 1, 10, 4);
   ASSERT_NE(Ctx, nullptr);
   EXPECT_NE(Root.CurrentMem, nullptr);
@@ -58,6 +59,7 @@ TEST_F(ContextTest, Basic) {
 }
 
 TEST_F(ContextTest, Callsite) {
+  __llvm_ctx_profile_start_collection();
   auto *Ctx = __llvm_ctx_profile_start_context(&Root, 1, 10, 4);
   int FakeCalleeAddress = 0;
   const bool IsScratch = isScratch(Ctx);
@@ -67,7 +69,11 @@ TEST_F(ContextTest, Callsite) {
   __llvm_ctx_profile_expected_callee[0] = &FakeCalleeAddress;
   __llvm_ctx_profile_callsite[0] = &Ctx->subContexts()[2];
   // This is what the callee does
-  auto *Subctx = __llvm_ctx_profile_get_context(&FakeCalleeAddress, 2, 3, 1);
+  FunctionData FData = {0};
+  auto *Subctx =
+      __llvm_ctx_profile_get_context(&FData, &FakeCalleeAddress, 2, 3, 1);
+  // This should not have required creating a flat context.
+  EXPECT_EQ(FData.FlatCtx, nullptr);
   // We expect the subcontext to be appropriately placed and dimensioned
   EXPECT_EQ(Ctx->subContexts()[2], Subctx);
   EXPECT_EQ(Subctx->counters_size(), 3U);
@@ -81,29 +87,59 @@ TEST_F(ContextTest, Callsite) {
   __llvm_ctx_profile_release_context(&Root);
 }
 
-TEST_F(ContextTest, ScratchNoCollection) {
+TEST_F(ContextTest, ScratchNoCollectionProfilingNotStarted) {
+  // This test intentionally does not call __llvm_ctx_profile_start_collection.
   EXPECT_EQ(__llvm_ctx_profile_current_context_root, nullptr);
   int FakeCalleeAddress = 0;
   // this would be the very first function executing this. the TLS is empty,
   // too.
-  auto *Ctx = __llvm_ctx_profile_get_context(&FakeCalleeAddress, 2, 3, 1);
+  FunctionData FData = {0};
+  auto *Ctx =
+      __llvm_ctx_profile_get_context(&FData, &FakeCalleeAddress, 2, 3, 1);
+  // We never entered a context (_start_context was never called) - so the
+  // returned context must be a tagged pointer.
+  EXPECT_TRUE(isScratch(Ctx));
+  // Because we didn't start collection, no flat profile should have been
+  // allocated.
+  EXPECT_EQ(FData.FlatCtx, nullptr);
+}
+
+TEST_F(ContextTest, ScratchNoCollectionProfilingStarted) {
+  ASSERT_EQ(__llvm_ctx_profile_current_context_root, nullptr);
+  int FakeCalleeAddress = 0;
+  // Start collection, so the function gets a flat profile instead of scratch.
+  __llvm_ctx_profile_start_collection();
+  // this would be the very first function executing this. the TLS is empty,
+  // too.
+  FunctionData FData = {0};
+  auto *Ctx =
+      __llvm_ctx_profile_get_context(&FData, &FakeCalleeAddress, 2, 3, 1);
   // We never entered a context (_start_context was never called) - so the
-  // returned context must be scratch.
+  // returned context must be a tagged pointer.
   EXPECT_TRUE(isScratch(Ctx));
+  // Because we never entered a context, we should have allocated a flat context
+  EXPECT_NE(FData.FlatCtx, nullptr);
+  EXPECT_EQ(reinterpret_cast<uintptr_t>(FData.FlatCtx) + 1,
+            reinterpret_cast<uintptr_t>(Ctx));
 }
 
 TEST_F(ContextTest, ScratchDuringCollection) {
+  __llvm_ctx_profile_start_collection();
   auto *Ctx = __llvm_ctx_profile_start_context(&Root, 1, 10, 4);
   int FakeCalleeAddress = 0;
   int OtherFakeCalleeAddress = 0;
   __llvm_ctx_profile_expected_callee[0] = &FakeCalleeAddress;
   __llvm_ctx_profile_callsite[0] = &Ctx->subContexts()[2];
-  auto *Subctx =
-      __llvm_ctx_profile_get_context(&OtherFakeCalleeAddress, 2, 3, 1);
+  FunctionData FData[3] = {0};
+  auto *Subctx = __llvm_ctx_profile_get_context(
+      &FData[0], &OtherFakeCalleeAddress, 2, 3, 1);
   // We expected a different callee - so return scratch. It mimics what happens
   // in the case of a signal handler - in this case, OtherFakeCalleeAddress is
   // the signal handler.
   EXPECT_TRUE(isScratch(Subctx));
+  // We shouldn't have tried to return a flat context because we're under a
+  // root.
+  EXPECT_EQ(FData[0].FlatCtx, nullptr);
   EXPECT_EQ(__llvm_ctx_profile_expected_callee[0], nullptr);
   EXPECT_EQ(__llvm_ctx_profile_callsite[0], nullptr);
 
@@ -111,24 +147,27 @@ TEST_F(ContextTest, ScratchDuringCollection) {
   __llvm_ctx_profile_expected_callee[1] = &ThirdFakeCalleeAddress;
   __llvm_ctx_profile_callsite[1] = &Subctx->subContexts()[0];
 
-  auto *Subctx2 =
-      __llvm_ctx_profile_get_context(&ThirdFakeCalleeAddress, 3, 0, 0);
+  auto *Subctx2 = __llvm_ctx_profile_get_context(
+      &FData[1], &ThirdFakeCalleeAddress, 3, 0, 0);
   // We again expect scratch because the '0' position is where the runtime
   // looks, so it doesn't matter the '1' position is populated correctly.
   EXPECT_TRUE(isScratch(Subctx2));
+  EXPECT_EQ(FData[1].FlatCtx, nullptr);
 
   __llvm_ctx_profile_expected_callee[0] = &ThirdFakeCalleeAddress;
   __llvm_ctx_profile_callsite[0] = &Subctx->subContexts()[0];
-  auto *Subctx3 =
-      __llvm_ctx_profile_get_context(&ThirdFakeCalleeAddress, 3, 0, 0);
+  auto *Subctx3 = __llvm_ctx_profile_get_context(
+      &FData[2], &ThirdFakeCalleeAddress, 3, 0, 0);
   // We expect scratch here, too, because the value placed in
   // __llvm_ctx_profile_callsite is scratch
   EXPECT_TRUE(isScratch(Subctx3));
+  EXPECT_EQ(FData[2].FlatCtx, nullptr);
 
   __llvm_ctx_profile_release_context(&Root);
 }
 
 TEST_F(ContextTest, NeedMoreMemory) {
+  __llvm_ctx_profile_start_collection();
   auto *Ctx = __llvm_ctx_profile_start_context(&Root, 1, 10, 4);
   int FakeCalleeAddress = 0;
   const bool IsScratch = isScratch(Ctx);
@@ -136,9 +175,11 @@ TEST_F(ContextTest, NeedMoreMemory) {
   const auto *CurrentMem = Root.CurrentMem;
   __llvm_ctx_profile_expected_callee[0] = &FakeCalleeAddress;
   __llvm_ctx_profile_callsite[0] = &Ctx->subContexts()[2];
+  FunctionData FData = {0};
   // Allocate a massive subcontext to force new arena allocation
   auto *Subctx =
-      __llvm_ctx_profile_get_context(&FakeCalleeAddress, 3, 1 << 20, 1);
+      __llvm_ctx_profile_get_context(&FData, &FakeCalleeAddress, 3, 1 << 20, 1);
+  EXPECT_EQ(FData.FlatCtx, nullptr);
   EXPECT_EQ(Ctx->subContexts()[2], Subctx);
   EXPECT_NE(CurrentMem, Root.CurrentMem);
   EXPECT_NE(Root.CurrentMem, nullptr);
@@ -175,7 +216,9 @@ TEST_F(ContextTest, Dump) {
   int FakeCalleeAddress = 0;
   __llvm_ctx_profile_expected_callee[0] = &FakeCalleeAddress;
   __llvm_ctx_profile_callsite[0] = &Ctx->subContexts()[2];
-  auto *Subctx = __llvm_ctx_profile_get_context(&FakeCalleeAddress, 2, 3, 1);
+  FunctionData FData = {0};
+  auto *Subctx =
+      __llvm_ctx_profile_get_context(&FData, &FakeCalleeAddress, 2, 3, 1);
   (void)Subctx;
   __llvm_ctx_profile_release_context(&Root);
 
@@ -186,6 +229,9 @@ TEST_F(ContextTest, Dump) {
 
     int EnteredSectionCount = 0;
     int ExitedSectionCount = 0;
+    int EnteredFlatCount = 0;
+    int ExitedFlatCount = 0;
+    int FlatsWritten = 0;
 
     bool State = false;
 
@@ -217,6 +263,16 @@ TEST_F(ContextTest, Dump) {
       EXPECT_EQ(EnteredSectionCount, 1);
       ++ExitedSectionCount;
     }
+    void startFlatSection() override { ++EnteredFlatCount; }
+    void writeFlat(GUID Guid, const uint64_t *Buffer,
+                   size_t BufferSize) override {
+      ++FlatsWritten;
+      EXPECT_EQ(BufferSize, 3);
+      EXPECT_EQ(Buffer[0], 15U);
+      EXPECT_EQ(Buffer[1], 0U);
+      EXPECT_EQ(Buffer[2], 0U);
+    }
+    void endFlatSection() override { ++ExitedFlatCount; }
   };
 
   TestProfileWriter W(&Root, 1);
@@ -226,10 +282,17 @@ TEST_F(ContextTest, Dump) {
 
   // this resets all counters but not the internal structure.
   __llvm_ctx_profile_start_collection();
+  auto *Flat =
+      __llvm_ctx_profile_get_context(&FData, &FakeCalleeAddress, 2, 3, 1);
+  EXPECT_NE(FData.FlatCtx, nullptr);
+  FData.FlatCtx->counters()[0] = 15U;
   TestProfileWriter W2(&Root, 0);
   EXPECT_FALSE(W2.State);
   __llvm_ctx_profile_fetch(W2);
   EXPECT_TRUE(W2.State);
   EXPECT_EQ(W2.EnteredSectionCount, 1);
   EXPECT_EQ(W2.ExitedSectionCount, 1);
+  EXPECT_EQ(W2.EnteredFlatCount, 1);
+  EXPECT_EQ(W2.FlatsWritten, 1);
+  EXPECT_EQ(W2.ExitedFlatCount, 1);
 }
diff --git a/compiler-rt/lib/orc/coff_platform.cpp b/compiler-rt/lib/orc/coff_platform.cpp
index 49b805a0ec7d3..78144ec114a2f 100644
--- a/compiler-rt/lib/orc/coff_platform.cpp
+++ b/compiler-rt/lib/orc/coff_platform.cpp
@@ -594,19 +594,19 @@ void *COFFPlatformRuntimeState::findJITDylibBaseByPC(uint64_t PC) {
   return Range.Header;
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_coff_platform_bootstrap(char *ArgData, size_t ArgSize) {
   COFFPlatformRuntimeState::initialize();
   return WrapperFunctionResult().release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_coff_platform_shutdown(char *ArgData, size_t ArgSize) {
   COFFPlatformRuntimeState::destroy();
   return WrapperFunctionResult().release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_coff_register_jitdylib(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(
              ArgData, ArgSize,
@@ -617,7 +617,7 @@ __orc_rt_coff_register_jitdylib(char *ArgData, size_t ArgSize) {
       .release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_coff_deregister_jitdylib(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
              ArgData, ArgSize,
@@ -628,7 +628,7 @@ __orc_rt_coff_deregister_jitdylib(char *ArgData, size_t ArgSize) {
       .release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_coff_register_object_sections(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError(SPSExecutorAddr, SPSCOFFObjectSectionsMap,
                                   bool)>::
@@ -643,7 +643,7 @@ __orc_rt_coff_register_object_sections(char *ArgData, size_t ArgSize) {
           .release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_coff_deregister_object_sections(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError(SPSExecutorAddr, SPSCOFFObjectSectionsMap)>::
       handle(ArgData, ArgSize,
diff --git a/compiler-rt/lib/orc/common.h b/compiler-rt/lib/orc/common.h
index f34229a615341..70d9a5905b120 100644
--- a/compiler-rt/lib/orc/common.h
+++ b/compiler-rt/lib/orc/common.h
@@ -41,7 +41,7 @@ ORC_RT_IMPORT __orc_rt_Opaque __orc_rt_jit_dispatch_ctx ORC_RT_WEAK_IMPORT;
 /// This is declared for use by the runtime, but should be implemented in the
 /// executor or provided by a definition added to the JIT before the runtime
 /// is loaded.
-ORC_RT_IMPORT orc_rt_CWrapperFunctionResult
+ORC_RT_IMPORT orc_rt_WrapperFunctionResult
 __orc_rt_jit_dispatch(__orc_rt_Opaque *DispatchCtx, const void *FnTag,
                       const char *Data, size_t Size) ORC_RT_WEAK_IMPORT;
 
diff --git a/compiler-rt/lib/orc/dlfcn_wrapper.cpp b/compiler-rt/lib/orc/dlfcn_wrapper.cpp
index 3d7da18e3045b..02e37968cd9fd 100644
--- a/compiler-rt/lib/orc/dlfcn_wrapper.cpp
+++ b/compiler-rt/lib/orc/dlfcn_wrapper.cpp
@@ -23,7 +23,7 @@ extern "C" void *__orc_rt_jit_dlopen(const char *path, int mode);
 extern "C" int __orc_rt_jit_dlupdate(void *dso_handle);
 extern "C" int __orc_rt_jit_dlclose(void *dso_handle);
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_jit_dlerror_wrapper(const char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSString()>::handle(
              ArgData, ArgSize,
@@ -31,7 +31,7 @@ __orc_rt_jit_dlerror_wrapper(const char *ArgData, size_t ArgSize) {
       .release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_jit_dlopen_wrapper(const char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSExecutorAddr(SPSString, int32_t)>::handle(
              ArgData, ArgSize,
@@ -43,7 +43,7 @@ __orc_rt_jit_dlopen_wrapper(const char *ArgData, size_t ArgSize) {
 }
 
 #ifndef _WIN32
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_jit_dlupdate_wrapper(const char *ArgData, size_t ArgSize) {
   return WrapperFunction<int32_t(SPSExecutorAddr)>::handle(
              ArgData, ArgSize,
@@ -54,7 +54,7 @@ __orc_rt_jit_dlupdate_wrapper(const char *ArgData, size_t ArgSize) {
 }
 #endif
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_jit_dlclose_wrapper(const char *ArgData, size_t ArgSize) {
   return WrapperFunction<int32_t(SPSExecutorAddr)>::handle(
              ArgData, ArgSize,
diff --git a/compiler-rt/lib/orc/elfnix_platform.cpp b/compiler-rt/lib/orc/elfnix_platform.cpp
index 6473139e931d8..33b09b948b878 100644
--- a/compiler-rt/lib/orc/elfnix_platform.cpp
+++ b/compiler-rt/lib/orc/elfnix_platform.cpp
@@ -656,7 +656,7 @@ void destroyELFNixTLVMgr(void *ELFNixTLVMgr) {
 //                             JIT entry points
 //------------------------------------------------------------------------------
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
              ArgData, ArgSize,
@@ -668,7 +668,7 @@ __orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) {
       .release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError()>::handle(
              ArgData, ArgSize,
@@ -679,7 +679,7 @@ __orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) {
       .release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_elfnix_register_jitdylib(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(
              ArgData, ArgSize,
@@ -690,7 +690,7 @@ __orc_rt_elfnix_register_jitdylib(char *ArgData, size_t ArgSize) {
       .release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_elfnix_deregister_jitdylib(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
              ArgData, ArgSize,
@@ -701,7 +701,7 @@ __orc_rt_elfnix_deregister_jitdylib(char *ArgData, size_t ArgSize) {
       .release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_elfnix_register_init_sections(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError(SPSExecutorAddr,
                                   SPSSequence<SPSExecutorAddrRange>)>::
@@ -714,7 +714,7 @@ __orc_rt_elfnix_register_init_sections(char *ArgData, size_t ArgSize) {
           .release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_elfnix_deregister_init_sections(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError(SPSExecutorAddr,
                                   SPSSequence<SPSExecutorAddrRange>)>::
@@ -728,7 +728,7 @@ __orc_rt_elfnix_deregister_init_sections(char *ArgData, size_t ArgSize) {
 }
 
 /// Wrapper function for registering metadata on a per-object basis.
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
       handle(ArgData, ArgSize,
@@ -740,7 +740,7 @@ __orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) {
 }
 
 /// Wrapper for releasing per-object metadat.
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
       handle(ArgData, ArgSize,
@@ -776,7 +776,7 @@ ORC_RT_INTERFACE ptrdiff_t ___orc_rt_elfnix_tlsdesc_resolver_impl(
   return TLVPtr - ThreadPointer;
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
              ArgData, ArgSize,
diff --git a/compiler-rt/lib/orc/macho_platform.cpp b/compiler-rt/lib/orc/macho_platform.cpp
index 4b603fd95e316..1443c6c078fde 100644
--- a/compiler-rt/lib/orc/macho_platform.cpp
+++ b/compiler-rt/lib/orc/macho_platform.cpp
@@ -1349,7 +1349,7 @@ Error runWrapperFunctionCalls(std::vector<WrapperFunctionCall> WFCs) {
 //                             JIT entry points
 //------------------------------------------------------------------------------
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError()>::handle(
              ArgData, ArgSize,
@@ -1357,7 +1357,7 @@ __orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) {
       .release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError()>::handle(
              ArgData, ArgSize,
@@ -1365,7 +1365,7 @@ __orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) {
       .release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_macho_register_jitdylib(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(
              ArgData, ArgSize,
@@ -1376,7 +1376,7 @@ __orc_rt_macho_register_jitdylib(char *ArgData, size_t ArgSize) {
       .release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_macho_deregister_jitdylib(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
              ArgData, ArgSize,
@@ -1387,7 +1387,7 @@ __orc_rt_macho_deregister_jitdylib(char *ArgData, size_t ArgSize) {
       .release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_macho_register_object_platform_sections(char *ArgData,
                                                  size_t ArgSize) {
   return WrapperFunction<SPSError(SPSExecutorAddr,
@@ -1404,7 +1404,7 @@ __orc_rt_macho_register_object_platform_sections(char *ArgData,
           .release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_macho_register_object_symbol_table(char *ArgData, size_t ArgSize) {
   using SymtabContainer = std::vector<
       std::tuple<ExecutorAddr, ExecutorAddr,
@@ -1420,7 +1420,7 @@ __orc_rt_macho_register_object_symbol_table(char *ArgData, size_t ArgSize) {
           .release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_macho_deregister_object_symbol_table(char *ArgData, size_t ArgSize) {
   using SymtabContainer = std::vector<
       std::tuple<ExecutorAddr, ExecutorAddr,
@@ -1436,7 +1436,7 @@ __orc_rt_macho_deregister_object_symbol_table(char *ArgData, size_t ArgSize) {
           .release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_macho_deregister_object_platform_sections(char *ArgData,
                                                    size_t ArgSize) {
   return WrapperFunction<SPSError(SPSExecutorAddr,
@@ -1453,7 +1453,7 @@ __orc_rt_macho_deregister_object_platform_sections(char *ArgData,
           .release();
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_macho_run_wrapper_function_calls(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError(SPSSequence<SPSWrapperFunctionCall>)>::handle(
              ArgData, ArgSize, runWrapperFunctionCalls)
@@ -1479,7 +1479,7 @@ ORC_RT_INTERFACE void *__orc_rt_macho_tlv_get_addr_impl(TLVDescriptor *D) {
       reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
 }
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_macho_create_pthread_key(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
              ArgData, ArgSize,
diff --git a/compiler-rt/lib/orc/run_program_wrapper.cpp b/compiler-rt/lib/orc/run_program_wrapper.cpp
index c9565e742908f..970e25ece37b9 100644
--- a/compiler-rt/lib/orc/run_program_wrapper.cpp
+++ b/compiler-rt/lib/orc/run_program_wrapper.cpp
@@ -22,7 +22,7 @@ extern "C" int64_t __orc_rt_run_program(const char *JITDylibName,
                                         const char *EntrySymbolName, int argc,
                                         char *argv[]);
 
-ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
+ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
 __orc_rt_run_program_wrapper(const char *ArgData, size_t ArgSize) {
   return WrapperFunction<int64_t(SPSString, SPSString,
                                  SPSSequence<SPSString>)>::
diff --git a/compiler-rt/lib/orc/tests/unit/c_api_test.cpp b/compiler-rt/lib/orc/tests/unit/c_api_test.cpp
index 497cb937e2afc..b9bd2d557821d 100644
--- a/compiler-rt/lib/orc/tests/unit/c_api_test.cpp
+++ b/compiler-rt/lib/orc/tests/unit/c_api_test.cpp
@@ -13,25 +13,25 @@
 #include "orc_rt/c_api.h"
 #include "gtest/gtest.h"
 
-TEST(CAPITest, CWrapperFunctionResultInit) {
-  orc_rt_CWrapperFunctionResult R;
-  orc_rt_CWrapperFunctionResultInit(&R);
+TEST(CAPITest, WrapperFunctionResultInit) {
+  orc_rt_WrapperFunctionResult R;
+  orc_rt_WrapperFunctionResultInit(&R);
 
   EXPECT_EQ(R.Size, 0U);
   EXPECT_EQ(R.Data.ValuePtr, nullptr);
 
   // Check that this value isn't treated as an out-of-band error.
-  EXPECT_EQ(orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr);
+  EXPECT_EQ(orc_rt_WrapperFunctionResultGetOutOfBandError(&R), nullptr);
 
   // Check that we can dispose of the value.
-  orc_rt_DisposeCWrapperFunctionResult(&R);
+  orc_rt_DisposeWrapperFunctionResult(&R);
 }
 
-TEST(CAPITest, CWrapperFunctionResultAllocSmall) {
+TEST(CAPITest, WrapperFunctionResultAllocSmall) {
   constexpr size_t SmallAllocSize = sizeof(const char *);
 
-  auto R = orc_rt_CWrapperFunctionResultAllocate(SmallAllocSize);
-  char *DataPtr = orc_rt_CWrapperFunctionResultData(&R);
+  auto R = orc_rt_WrapperFunctionResultAllocate(SmallAllocSize);
+  char *DataPtr = orc_rt_WrapperFunctionResultData(&R);
 
   for (size_t I = 0; I != SmallAllocSize; ++I)
     DataPtr[I] = 0x55 + I;
@@ -44,24 +44,24 @@ TEST(CAPITest, CWrapperFunctionResultAllocSmall) {
         << "Unexpected value at index " << I;
 
   // Check that this value isn't treated as an out-of-band error.
-  EXPECT_EQ(orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr);
+  EXPECT_EQ(orc_rt_WrapperFunctionResultGetOutOfBandError(&R), nullptr);
 
-  // Check that orc_rt_CWrapperFunctionResult(Data|Result|Size) and
-  // orc_rt_CWrapperFunctionResultGetOutOfBandError behave as expected.
-  EXPECT_EQ(orc_rt_CWrapperFunctionResultData(&R), R.Data.Value);
-  EXPECT_EQ(orc_rt_CWrapperFunctionResultSize(&R), SmallAllocSize);
-  EXPECT_FALSE(orc_rt_CWrapperFunctionResultEmpty(&R));
-  EXPECT_EQ(orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr);
+  // Check that orc_rt_WrapperFunctionResult(Data|Result|Size) and
+  // orc_rt_WrapperFunctionResultGetOutOfBandError behave as expected.
+  EXPECT_EQ(orc_rt_WrapperFunctionResultData(&R), R.Data.Value);
+  EXPECT_EQ(orc_rt_WrapperFunctionResultSize(&R), SmallAllocSize);
+  EXPECT_FALSE(orc_rt_WrapperFunctionResultEmpty(&R));
+  EXPECT_EQ(orc_rt_WrapperFunctionResultGetOutOfBandError(&R), nullptr);
 
   // Check that we can dispose of the value.
-  orc_rt_DisposeCWrapperFunctionResult(&R);
+  orc_rt_DisposeWrapperFunctionResult(&R);
 }
 
-TEST(CAPITest, CWrapperFunctionResultAllocLarge) {
+TEST(CAPITest, WrapperFunctionResultAllocLarge) {
   constexpr size_t LargeAllocSize = sizeof(const char *) + 1;
 
-  auto R = orc_rt_CWrapperFunctionResultAllocate(LargeAllocSize);
-  char *DataPtr = orc_rt_CWrapperFunctionResultData(&R);
+  auto R = orc_rt_WrapperFunctionResultAllocate(LargeAllocSize);
+  char *DataPtr = orc_rt_WrapperFunctionResultData(&R);
 
   for (size_t I = 0; I != LargeAllocSize; ++I)
     DataPtr[I] = 0x55 + I;
@@ -75,28 +75,28 @@ TEST(CAPITest, CWrapperFunctionResultAllocLarge) {
         << "Unexpected value at index " << I;
 
   // Check that this value isn't treated as an out-of-band error.
-  EXPECT_EQ(orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr);
+  EXPECT_EQ(orc_rt_WrapperFunctionResultGetOutOfBandError(&R), nullptr);
 
-  // Check that orc_rt_CWrapperFunctionResult(Data|Result|Size) and
-  // orc_rt_CWrapperFunctionResultGetOutOfBandError behave as expected.
-  EXPECT_EQ(orc_rt_CWrapperFunctionResultData(&R), R.Data.ValuePtr);
-  EXPECT_EQ(orc_rt_CWrapperFunctionResultSize(&R), LargeAllocSize);
-  EXPECT_FALSE(orc_rt_CWrapperFunctionResultEmpty(&R));
-  EXPECT_EQ(orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr);
+  // Check that orc_rt_WrapperFunctionResult(Data|Result|Size) and
+  // orc_rt_WrapperFunctionResultGetOutOfBandError behave as expected.
+  EXPECT_EQ(orc_rt_WrapperFunctionResultData(&R), R.Data.ValuePtr);
+  EXPECT_EQ(orc_rt_WrapperFunctionResultSize(&R), LargeAllocSize);
+  EXPECT_FALSE(orc_rt_WrapperFunctionResultEmpty(&R));
+  EXPECT_EQ(orc_rt_WrapperFunctionResultGetOutOfBandError(&R), nullptr);
 
   // Check that we can dispose of the value.
-  orc_rt_DisposeCWrapperFunctionResult(&R);
+  orc_rt_DisposeWrapperFunctionResult(&R);
 }
 
-TEST(CAPITest, CWrapperFunctionResultFromRangeSmall) {
+TEST(CAPITest, WrapperFunctionResultFromRangeSmall) {
   constexpr size_t SmallAllocSize = sizeof(const char *);
 
   char Source[SmallAllocSize];
   for (size_t I = 0; I != SmallAllocSize; ++I)
     Source[I] = 0x55 + I;
 
-  orc_rt_CWrapperFunctionResult R =
-      orc_rt_CreateCWrapperFunctionResultFromRange(Source, SmallAllocSize);
+  orc_rt_WrapperFunctionResult R =
+      orc_rt_CreateWrapperFunctionResultFromRange(Source, SmallAllocSize);
 
   // Check that the inline storage in R.Data.Value contains the expected
   // sequence.
@@ -106,18 +106,18 @@ TEST(CAPITest, CWrapperFunctionResultFromRangeSmall) {
         << "Unexpected value at index " << I;
 
   // Check that we can dispose of the value.
-  orc_rt_DisposeCWrapperFunctionResult(&R);
+  orc_rt_DisposeWrapperFunctionResult(&R);
 }
 
-TEST(CAPITest, CWrapperFunctionResultFromRangeLarge) {
+TEST(CAPITest, WrapperFunctionResultFromRangeLarge) {
   constexpr size_t LargeAllocSize = sizeof(const char *) + 1;
 
   char Source[LargeAllocSize];
   for (size_t I = 0; I != LargeAllocSize; ++I)
     Source[I] = 0x55 + I;
 
-  orc_rt_CWrapperFunctionResult R =
-      orc_rt_CreateCWrapperFunctionResultFromRange(Source, LargeAllocSize);
+  orc_rt_WrapperFunctionResult R =
+      orc_rt_CreateWrapperFunctionResultFromRange(Source, LargeAllocSize);
 
   // Check that the inline storage in R.Data.Value contains the expected
   // sequence.
@@ -127,10 +127,10 @@ TEST(CAPITest, CWrapperFunctionResultFromRangeLarge) {
         << "Unexpected value at index " << I;
 
   // Check that we can dispose of the value.
-  orc_rt_DisposeCWrapperFunctionResult(&R);
+  orc_rt_DisposeWrapperFunctionResult(&R);
 }
 
-TEST(CAPITest, CWrapperFunctionResultFromStringSmall) {
+TEST(CAPITest, WrapperFunctionResultFromStringSmall) {
   constexpr size_t SmallAllocSize = sizeof(const char *);
 
   char Source[SmallAllocSize];
@@ -138,8 +138,8 @@ TEST(CAPITest, CWrapperFunctionResultFromStringSmall) {
     Source[I] = 'a' + I;
   Source[SmallAllocSize - 1] = '\0';
 
-  orc_rt_CWrapperFunctionResult R =
-      orc_rt_CreateCWrapperFunctionResultFromString(Source);
+  orc_rt_WrapperFunctionResult R =
+      orc_rt_CreateWrapperFunctionResultFromString(Source);
 
   // Check that the inline storage in R.Data.Value contains the expected
   // sequence.
@@ -151,10 +151,10 @@ TEST(CAPITest, CWrapperFunctionResultFromStringSmall) {
       << "Unexpected value at index " << (SmallAllocSize - 1);
 
   // Check that we can dispose of the value.
-  orc_rt_DisposeCWrapperFunctionResult(&R);
+  orc_rt_DisposeWrapperFunctionResult(&R);
 }
 
-TEST(CAPITest, CWrapperFunctionResultFromStringLarge) {
+TEST(CAPITest, WrapperFunctionResultFromStringLarge) {
   constexpr size_t LargeAllocSize = sizeof(const char *) + 1;
 
   char Source[LargeAllocSize];
@@ -162,8 +162,8 @@ TEST(CAPITest, CWrapperFunctionResultFromStringLarge) {
     Source[I] = 'a' + I;
   Source[LargeAllocSize - 1] = '\0';
 
-  orc_rt_CWrapperFunctionResult R =
-      orc_rt_CreateCWrapperFunctionResultFromString(Source);
+  orc_rt_WrapperFunctionResult R =
+      orc_rt_CreateWrapperFunctionResultFromString(Source);
 
   // Check that the inline storage in R.Data.Value contains the expected
   // sequence.
@@ -175,26 +175,28 @@ TEST(CAPITest, CWrapperFunctionResultFromStringLarge) {
       << "Unexpected value at index " << (LargeAllocSize - 1);
 
   // Check that we can dispose of the value.
-  orc_rt_DisposeCWrapperFunctionResult(&R);
+  orc_rt_DisposeWrapperFunctionResult(&R);
 }
 
-TEST(CAPITest, CWrapperFunctionResultFromOutOfBandError) {
+TEST(CAPITest, WrapperFunctionResultFromOutOfBandError) {
   constexpr const char *ErrMsg = "test error message";
-  orc_rt_CWrapperFunctionResult R =
-      orc_rt_CreateCWrapperFunctionResultFromOutOfBandError(ErrMsg);
+  orc_rt_WrapperFunctionResult R =
+      orc_rt_CreateWrapperFunctionResultFromOutOfBandError(ErrMsg);
 
 #ifndef NDEBUG
-  EXPECT_DEATH({ orc_rt_CWrapperFunctionResultData(&R); },
-               "Cannot get data for out-of-band error value");
-  EXPECT_DEATH({ orc_rt_CWrapperFunctionResultSize(&R); },
-               "Cannot get size for out-of-band error value");
+  EXPECT_DEATH(
+      { orc_rt_WrapperFunctionResultData(&R); },
+      "Cannot get data for out-of-band error value");
+  EXPECT_DEATH(
+      { orc_rt_WrapperFunctionResultSize(&R); },
+      "Cannot get size for out-of-band error value");
 #endif
 
-  EXPECT_FALSE(orc_rt_CWrapperFunctionResultEmpty(&R));
-  const char *OOBErrMsg = orc_rt_CWrapperFunctionResultGetOutOfBandError(&R);
+  EXPECT_FALSE(orc_rt_WrapperFunctionResultEmpty(&R));
+  const char *OOBErrMsg = orc_rt_WrapperFunctionResultGetOutOfBandError(&R);
   EXPECT_NE(OOBErrMsg, nullptr);
   EXPECT_NE(OOBErrMsg, ErrMsg);
   EXPECT_TRUE(strcmp(OOBErrMsg, ErrMsg) == 0);
 
-  orc_rt_DisposeCWrapperFunctionResult(&R);
+  orc_rt_DisposeWrapperFunctionResult(&R);
 }
diff --git a/compiler-rt/lib/orc/tests/unit/wrapper_function_utils_test.cpp b/compiler-rt/lib/orc/tests/unit/wrapper_function_utils_test.cpp
index 91be84f15188f..54ad5a9824cc5 100644
--- a/compiler-rt/lib/orc/tests/unit/wrapper_function_utils_test.cpp
+++ b/compiler-rt/lib/orc/tests/unit/wrapper_function_utils_test.cpp
@@ -29,8 +29,8 @@ TEST(WrapperFunctionUtilsTest, DefaultWrapperFunctionResult) {
 }
 
 TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromCStruct) {
-  orc_rt_CWrapperFunctionResult CR =
-      orc_rt_CreateCWrapperFunctionResultFromString(TestString);
+  orc_rt_WrapperFunctionResult CR =
+      orc_rt_CreateWrapperFunctionResultFromString(TestString);
   WrapperFunctionResult R(CR);
   EXPECT_EQ(R.size(), strlen(TestString) + 1);
   EXPECT_TRUE(strcmp(R.data(), TestString) == 0);
@@ -74,13 +74,13 @@ TEST(WrapperFunctionUtilsTest, WrapperFunctionCCallCreateEmpty) {
 
 static void voidNoop() {}
 
-static orc_rt_CWrapperFunctionResult voidNoopWrapper(const char *ArgData,
-                                                     size_t ArgSize) {
+static orc_rt_WrapperFunctionResult voidNoopWrapper(const char *ArgData,
+                                                    size_t ArgSize) {
   return WrapperFunction<void()>::handle(ArgData, ArgSize, voidNoop).release();
 }
 
-static orc_rt_CWrapperFunctionResult addWrapper(const char *ArgData,
-                                                size_t ArgSize) {
+static orc_rt_WrapperFunctionResult addWrapper(const char *ArgData,
+                                               size_t ArgSize) {
   return WrapperFunction<int32_t(int32_t, int32_t)>::handle(
              ArgData, ArgSize,
              [](int32_t X, int32_t Y) -> int32_t { return X + Y; })
@@ -89,11 +89,11 @@ static orc_rt_CWrapperFunctionResult addWrapper(const char *ArgData,
 
 extern "C" __orc_rt_Opaque __orc_rt_jit_dispatch_ctx{};
 
-extern "C" orc_rt_CWrapperFunctionResult
+extern "C" orc_rt_WrapperFunctionResult
 __orc_rt_jit_dispatch(__orc_rt_Opaque *Ctx, const void *FnTag,
                       const char *ArgData, size_t ArgSize) {
   using WrapperFunctionType =
-      orc_rt_CWrapperFunctionResult (*)(const char *, size_t);
+      orc_rt_WrapperFunctionResult (*)(const char *, size_t);
 
   return reinterpret_cast<WrapperFunctionType>(const_cast<void *>(FnTag))(
       ArgData, ArgSize);
@@ -120,8 +120,8 @@ class AddClass {
   int32_t X;
 };
 
-static orc_rt_CWrapperFunctionResult addMethodWrapper(const char *ArgData,
-                                                      size_t ArgSize) {
+static orc_rt_WrapperFunctionResult addMethodWrapper(const char *ArgData,
+                                                     size_t ArgSize) {
   return WrapperFunction<int32_t(SPSExecutorAddr, int32_t)>::handle(
              ArgData, ArgSize, makeMethodWrapperHandler(&AddClass::addMethod))
       .release();
@@ -136,8 +136,8 @@ TEST(WrapperFunctionUtilsTest, WrapperFunctionMethodCallAndHandleRet) {
   EXPECT_EQ(Result, (int32_t)3);
 }
 
-static orc_rt_CWrapperFunctionResult sumArrayWrapper(const char *ArgData,
-                                                     size_t ArgSize) {
+static orc_rt_WrapperFunctionResult sumArrayWrapper(const char *ArgData,
+                                                    size_t ArgSize) {
   return WrapperFunction<int8_t(SPSExecutorAddrRange)>::handle(
              ArgData, ArgSize,
              [](ExecutorAddrRange R) {
diff --git a/compiler-rt/lib/orc/wrapper_function_utils.h b/compiler-rt/lib/orc/wrapper_function_utils.h
index 3e372fe96d2d6..d6605f02f75e9 100644
--- a/compiler-rt/lib/orc/wrapper_function_utils.h
+++ b/compiler-rt/lib/orc/wrapper_function_utils.h
@@ -21,71 +21,71 @@
 
 namespace orc_rt {
 
-/// C++ wrapper function result: Same as CWrapperFunctionResult but
+/// C++ wrapper function result: Same as orc_rt_WrapperFunctionResult but
 /// auto-releases memory.
 class WrapperFunctionResult {
 public:
   /// Create a default WrapperFunctionResult.
-  WrapperFunctionResult() { orc_rt_CWrapperFunctionResultInit(&R); }
+  WrapperFunctionResult() { orc_rt_WrapperFunctionResultInit(&R); }
 
-  /// Create a WrapperFunctionResult from a CWrapperFunctionResult. This
+  /// Create a WrapperFunctionResult from a WrapperFunctionResult. This
   /// instance takes ownership of the result object and will automatically
   /// call dispose on the result upon destruction.
-  WrapperFunctionResult(orc_rt_CWrapperFunctionResult R) : R(R) {}
+  WrapperFunctionResult(orc_rt_WrapperFunctionResult R) : R(R) {}
 
   WrapperFunctionResult(const WrapperFunctionResult &) = delete;
   WrapperFunctionResult &operator=(const WrapperFunctionResult &) = delete;
 
   WrapperFunctionResult(WrapperFunctionResult &&Other) {
-    orc_rt_CWrapperFunctionResultInit(&R);
+    orc_rt_WrapperFunctionResultInit(&R);
     std::swap(R, Other.R);
   }
 
   WrapperFunctionResult &operator=(WrapperFunctionResult &&Other) {
-    orc_rt_CWrapperFunctionResult Tmp;
-    orc_rt_CWrapperFunctionResultInit(&Tmp);
+    orc_rt_WrapperFunctionResult Tmp;
+    orc_rt_WrapperFunctionResultInit(&Tmp);
     std::swap(Tmp, Other.R);
     std::swap(R, Tmp);
     return *this;
   }
 
-  ~WrapperFunctionResult() { orc_rt_DisposeCWrapperFunctionResult(&R); }
+  ~WrapperFunctionResult() { orc_rt_DisposeWrapperFunctionResult(&R); }
 
   /// Relinquish ownership of and return the
-  /// orc_rt_CWrapperFunctionResult.
-  orc_rt_CWrapperFunctionResult release() {
-    orc_rt_CWrapperFunctionResult Tmp;
-    orc_rt_CWrapperFunctionResultInit(&Tmp);
+  /// orc_rt_WrapperFunctionResult.
+  orc_rt_WrapperFunctionResult release() {
+    orc_rt_WrapperFunctionResult Tmp;
+    orc_rt_WrapperFunctionResultInit(&Tmp);
     std::swap(R, Tmp);
     return Tmp;
   }
 
   /// Get a pointer to the data contained in this instance.
-  char *data() { return orc_rt_CWrapperFunctionResultData(&R); }
+  char *data() { return orc_rt_WrapperFunctionResultData(&R); }
 
   /// Returns the size of the data contained in this instance.
-  size_t size() const { return orc_rt_CWrapperFunctionResultSize(&R); }
+  size_t size() const { return orc_rt_WrapperFunctionResultSize(&R); }
 
   /// Returns true if this value is equivalent to a default-constructed
   /// WrapperFunctionResult.
-  bool empty() const { return orc_rt_CWrapperFunctionResultEmpty(&R); }
+  bool empty() const { return orc_rt_WrapperFunctionResultEmpty(&R); }
 
   /// Create a WrapperFunctionResult with the given size and return a pointer
   /// to the underlying memory.
   static WrapperFunctionResult allocate(size_t Size) {
     WrapperFunctionResult R;
-    R.R = orc_rt_CWrapperFunctionResultAllocate(Size);
+    R.R = orc_rt_WrapperFunctionResultAllocate(Size);
     return R;
   }
 
   /// Copy from the given char range.
   static WrapperFunctionResult copyFrom(const char *Source, size_t Size) {
-    return orc_rt_CreateCWrapperFunctionResultFromRange(Source, Size);
+    return orc_rt_CreateWrapperFunctionResultFromRange(Source, Size);
   }
 
   /// Copy from the given null-terminated string (includes the null-terminator).
   static WrapperFunctionResult copyFrom(const char *Source) {
-    return orc_rt_CreateCWrapperFunctionResultFromString(Source);
+    return orc_rt_CreateWrapperFunctionResultFromString(Source);
   }
 
   /// Copy from the given std::string (includes the null terminator).
@@ -95,7 +95,7 @@ class WrapperFunctionResult {
 
   /// Create an out-of-band error by copying the given string.
   static WrapperFunctionResult createOutOfBandError(const char *Msg) {
-    return orc_rt_CreateCWrapperFunctionResultFromOutOfBandError(Msg);
+    return orc_rt_CreateWrapperFunctionResultFromOutOfBandError(Msg);
   }
 
   /// Create an out-of-band error by copying the given string.
@@ -116,11 +116,11 @@ class WrapperFunctionResult {
   /// If this value is an out-of-band error then this returns the error message,
   /// otherwise returns nullptr.
   const char *getOutOfBandError() const {
-    return orc_rt_CWrapperFunctionResultGetOutOfBandError(&R);
+    return orc_rt_WrapperFunctionResultGetOutOfBandError(&R);
   }
 
 private:
-  orc_rt_CWrapperFunctionResult R;
+  orc_rt_WrapperFunctionResult R;
 };
 
 namespace detail {
@@ -430,7 +430,7 @@ class WrapperFunctionCall {
   /// Run call returning raw WrapperFunctionResult.
   WrapperFunctionResult run() const {
     using FnTy =
-        orc_rt_CWrapperFunctionResult(const char *ArgData, size_t ArgSize);
+        orc_rt_WrapperFunctionResult(const char *ArgData, size_t ArgSize);
     return WrapperFunctionResult(
         FnAddr.toPtr<FnTy *>()(ArgData.data(), ArgData.size()));
   }
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp
index 5ff8d1832556f..a9533d6fc04ca 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp
@@ -334,9 +334,22 @@ static const load_command *NextCommand(const load_command *lc) {
   return (const load_command *)((const char *)lc + lc->cmdsize);
 }
 
-static void FindUUID(const load_command *first_lc, u8 *uuid_output) {
-  for (const load_command *lc = first_lc; lc->cmd != 0; lc = NextCommand(lc)) {
-    if (lc->cmd != LC_UUID) continue;
+#  ifdef MH_MAGIC_64
+static constexpr size_t header_size = sizeof(mach_header_64);
+#  else
+static constexpr size_t header_size = sizeof(mach_header);
+#  endif
+
+static void FindUUID(const load_command *first_lc, const mach_header *hdr,
+                     u8 *uuid_output) {
+  uint32_t curcmd = 0;
+  for (const load_command *lc = first_lc; curcmd < hdr->ncmds;
+       curcmd++, lc = NextCommand(lc)) {
+    CHECK_LT((const char *)lc,
+             (const char *)hdr + header_size + hdr->sizeofcmds);
+
+    if (lc->cmd != LC_UUID)
+      continue;
 
     const uuid_command *uuid_lc = (const uuid_command *)lc;
     const uint8_t *uuid = &uuid_lc->uuid[0];
@@ -345,9 +358,16 @@ static void FindUUID(const load_command *first_lc, u8 *uuid_output) {
   }
 }
 
-static bool IsModuleInstrumented(const load_command *first_lc) {
-  for (const load_command *lc = first_lc; lc->cmd != 0; lc = NextCommand(lc)) {
-    if (lc->cmd != LC_LOAD_DYLIB) continue;
+static bool IsModuleInstrumented(const load_command *first_lc,
+                                 const mach_header *hdr) {
+  uint32_t curcmd = 0;
+  for (const load_command *lc = first_lc; curcmd < hdr->ncmds;
+       curcmd++, lc = NextCommand(lc)) {
+    CHECK_LT((const char *)lc,
+             (const char *)hdr + header_size + hdr->sizeofcmds);
+
+    if (lc->cmd != LC_LOAD_DYLIB)
+      continue;
 
     const dylib_command *dylib_lc = (const dylib_command *)lc;
     uint32_t dylib_name_offset = dylib_lc->dylib.name.offset;
@@ -393,10 +413,10 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
           continue;
         }
       }
-      FindUUID((const load_command *)data_.current_load_cmd_addr,
+      FindUUID((const load_command *)data_.current_load_cmd_addr, hdr,
                data_.current_uuid);
       data_.current_instrumented = IsModuleInstrumented(
-          (const load_command *)data_.current_load_cmd_addr);
+          (const load_command *)data_.current_load_cmd_addr, hdr);
     }
 
     while (data_.current_load_cmd_count > 0) {
diff --git a/compiler-rt/lib/sanitizer_common/tests/sanitizer_procmaps_mac_test.cpp b/compiler-rt/lib/sanitizer_common/tests/sanitizer_procmaps_mac_test.cpp
index f622bba246309..7547528e2adb8 100644
--- a/compiler-rt/lib/sanitizer_common/tests/sanitizer_procmaps_mac_test.cpp
+++ b/compiler-rt/lib/sanitizer_common/tests/sanitizer_procmaps_mac_test.cpp
@@ -37,16 +37,15 @@ class MemoryMappingLayoutMock final : public MemoryMappingLayout {
     .uuid = {}
   };
 
-  static constexpr char dylib_name[] = "libclang_rt.\0\0\0"; // 8 bytes aligned, padded with zeros per loader.h
+  static constexpr char libclang_rt_dylib_name[] =
+      "libclang_rt.\0\0\0";  // 8 bytes aligned, padded with zeros per loader.h
+  static constexpr char uninstrumented_dylib_name[] =
+      "uninst___rt.\0\0\0";  // 8 bytes aligned, padded with zeros per loader.h
+
   static constexpr dylib_command mock_dylib_command = {
-    .cmd = LC_LOAD_DYLIB,
-    .cmdsize = sizeof(dylib_command) + sizeof(dylib_name),
-    .dylib = {
-      .name = {
-        .offset = sizeof(dylib_command)
-      }
-    }
-  };
+      .cmd = LC_LOAD_DYLIB,
+      .cmdsize = sizeof(dylib_command) + sizeof(libclang_rt_dylib_name),
+      .dylib = {.name = {.offset = sizeof(dylib_command)}}};
 
   static constexpr uuid_command mock_trap_command = {
     .cmd = LC_UUID,
@@ -59,51 +58,57 @@ class MemoryMappingLayoutMock final : public MemoryMappingLayout {
   std::vector<unsigned char> mock_header;
 
 public:
-  MemoryMappingLayoutMock(): MemoryMappingLayout(false) {
-    EXPECT_EQ(mock_uuid_command.cmdsize % 8, 0u);
-    EXPECT_EQ(mock_dylib_command.cmdsize % 8, 0u);
-
-    Reset();
-
-#ifdef MH_MAGIC_64
-    const struct mach_header_64 *header = (mach_header_64 *)_dyld_get_image_header(0); // Any header will do
-    const size_t header_size = sizeof(mach_header_64);
-#else
-    const struct mach_header *header = _dyld_get_image_header(0);
-    const size_t header_size = sizeof(mach_header);
-#endif
-    const size_t mock_header_size_with_extras = header_size + header->sizeofcmds +
-      mock_uuid_command.cmdsize + mock_dylib_command.cmdsize + sizeof(uuid_command);
-
-    mock_header.reserve(mock_header_size_with_extras);
-    // Copy the original header
-    copy((unsigned char *)header,
-      (unsigned char *)header + header_size + header->sizeofcmds,
-      back_inserter(mock_header));
-    // The following commands are not supposed to be processed
-    // by the (correct) ::Next method at all, since they're not
-    // accounted for in header->ncmds .
-    copy((unsigned char *)&mock_uuid_command,
-      ((unsigned char *)&mock_uuid_command) + mock_uuid_command.cmdsize,
-      back_inserter(mock_header));
-    copy((unsigned char *)&mock_dylib_command,
-      ((unsigned char *)&mock_dylib_command) + sizeof(dylib_command), // as mock_dylib_command.cmdsize contains the following string
-      back_inserter(mock_header));
-    copy((unsigned char *)dylib_name,
-      ((unsigned char *)dylib_name) + sizeof(dylib_name),
-      back_inserter(mock_header));
-
-    // Append a command w. huge size to have the test detect the read overrun
-    copy((unsigned char *)&mock_trap_command,
-      ((unsigned char *)&mock_trap_command) + sizeof(uuid_command),
-      back_inserter(mock_header));
-
-    start_load_cmd_addr = (const char *)(mock_header.data() + header_size);
-    sizeofcmds = header->sizeofcmds;
-
-    const char *last_byte_load_cmd_addr = (start_load_cmd_addr+sizeofcmds-1);
-    data_.current_image = -1; // So the loop in ::Next runs just once
-  }
+ MemoryMappingLayoutMock(bool instrumented) : MemoryMappingLayout(false) {
+   EXPECT_EQ(mock_uuid_command.cmdsize % 8, 0u);
+   EXPECT_EQ(mock_dylib_command.cmdsize % 8, 0u);
+
+   Reset();
+
+#  ifdef MH_MAGIC_64
+   const struct mach_header_64 *header =
+       (mach_header_64 *)_dyld_get_image_header(0);  // Any header will do
+   const size_t header_size = sizeof(mach_header_64);
+#  else
+   const struct mach_header *header = _dyld_get_image_header(0);
+   const size_t header_size = sizeof(mach_header);
+#  endif
+   const size_t mock_header_size_with_extras =
+       header_size + header->sizeofcmds + mock_uuid_command.cmdsize +
+       mock_dylib_command.cmdsize + sizeof(uuid_command);
+
+   mock_header.reserve(mock_header_size_with_extras);
+   // Copy the original header
+   copy((unsigned char *)header,
+        (unsigned char *)header + header_size + header->sizeofcmds,
+        back_inserter(mock_header));
+   // The following commands are not supposed to be processed
+   // by the (correct) ::Next method at all, since they're not
+   // accounted for in header->ncmds .
+   copy((unsigned char *)&mock_uuid_command,
+        ((unsigned char *)&mock_uuid_command) + mock_uuid_command.cmdsize,
+        back_inserter(mock_header));
+   copy((unsigned char *)&mock_dylib_command,
+        ((unsigned char *)&mock_dylib_command) +
+            sizeof(dylib_command),  // as mock_dylib_command.cmdsize contains
+                                    // the following string
+        back_inserter(mock_header));
+   const char(&dylib_name)[16] =
+       instrumented ? libclang_rt_dylib_name : uninstrumented_dylib_name;
+   copy((unsigned char *)dylib_name,
+        ((unsigned char *)dylib_name) + sizeof(dylib_name),
+        back_inserter(mock_header));
+
+   // Append a command w. huge size to have the test detect the read overrun
+   copy((unsigned char *)&mock_trap_command,
+        ((unsigned char *)&mock_trap_command) + sizeof(uuid_command),
+        back_inserter(mock_header));
+
+   start_load_cmd_addr = (const char *)(mock_header.data() + header_size);
+   sizeofcmds = header->sizeofcmds;
+
+   const char *last_byte_load_cmd_addr = (start_load_cmd_addr + sizeofcmds - 1);
+   data_.current_image = -1;  // So the loop in ::Next runs just once
+ }
 
   size_t SizeOfLoadCommands() {
     return sizeofcmds;
@@ -120,8 +125,20 @@ class MemoryMappingLayoutMock final : public MemoryMappingLayout {
   }
 };
 
-TEST(MemoryMappingLayout, Next) {
-  __sanitizer::MemoryMappingLayoutMock memory_mapping;
+TEST(MemoryMappingLayout, NextInstrumented) {
+  __sanitizer::MemoryMappingLayoutMock memory_mapping(true);
+  __sanitizer::MemoryMappedSegment segment;
+  size_t size = memory_mapping.SizeOfLoadCommands();
+  while (memory_mapping.Next(&segment)) {
+    size_t offset = memory_mapping.CurrentLoadCommandOffset();
+    EXPECT_LE(offset, size);
+  }
+  size_t final_offset = memory_mapping.CurrentLoadCommandOffset();
+  EXPECT_EQ(final_offset, size);  // All commands processed, no more, no less
+}
+
+TEST(MemoryMappingLayout, NextUnInstrumented) {
+  __sanitizer::MemoryMappingLayoutMock memory_mapping(false);
   __sanitizer::MemoryMappedSegment segment;
   size_t size = memory_mapping.SizeOfLoadCommands();
   while (memory_mapping.Next(&segment)) {
diff --git a/compiler-rt/test/ctx_profile/TestCases/generate-context.cpp b/compiler-rt/test/ctx_profile/TestCases/generate-context.cpp
index cdf819cbefc3b..bf33b4423fd1f 100644
--- a/compiler-rt/test/ctx_profile/TestCases/generate-context.cpp
+++ b/compiler-rt/test/ctx_profile/TestCases/generate-context.cpp
@@ -15,6 +15,7 @@
 #include <iostream>
 
 using namespace llvm::ctx_profile;
+extern "C" void __llvm_ctx_profile_start_collection();
 extern "C" bool __llvm_ctx_profile_fetch(ProfileWriter &);
 
 // avoid name mangling
@@ -36,6 +37,15 @@ __attribute__((noinline)) void theRoot() {
     someFunction(I);
   }
 }
+
+__attribute__((noinline)) void flatFct() {
+  printf("flat check 1\n");
+  someFunction(1);
+#pragma nounroll
+  for (auto I = 0; I < 2; ++I) {
+    someFunction(I);
+  }
+}
 }
 
 // Make sure the program actually ran correctly.
@@ -43,6 +53,10 @@ __attribute__((noinline)) void theRoot() {
 // CHECK-NEXT: check odd
 // CHECK-NEXT: check even
 // CHECK-NEXT: check odd
+// CHECK-NEXT: flat check 1
+// CHECK-NEXT: check odd
+// CHECK-NEXT: check even
+// CHECK-NEXT: check odd
 
 class TestProfileWriter : public ProfileWriter {
   void printProfile(const ContextNode &Node, const std::string &Indent,
@@ -73,6 +87,22 @@ class TestProfileWriter : public ProfileWriter {
   void writeContextual(const ContextNode &RootNode) override {
     printProfile(RootNode, "", "");
   }
+
+  void startFlatSection() override {
+    std::cout << "Entered Flat Section" << std::endl;
+  }
+
+  void writeFlat(GUID Guid, const uint64_t *Buffer,
+                 size_t BufferSize) override {
+    std::cout << "Flat: " << Guid << " " << Buffer[0];
+    for (size_t I = 1U; I < BufferSize; ++I)
+      std::cout << "," << Buffer[I];
+    std::cout << std::endl;
+  };
+
+  void endFlatSection() override {
+    std::cout << "Exited Flat Section" << std::endl;
+  }
 };
 
 // 8657661246551306189 is theRoot. We expect 2 callsites and 2 counters - one
@@ -100,6 +130,11 @@ class TestProfileWriter : public ProfileWriter {
 // CHECK-NEXT:   2 counters and 2 callsites
 // CHECK-NEXT:   Counter values: 2 1
 // CHECK-NEXT: Exited Context Section
+// CHECK-NEXT: Entered Flat Section
+// CHECK-NEXT: Flat: 6759619411192316602 3,1
+// This is flatFct (guid: 14569438697463215220)
+// CHECK-NEXT: Flat: 14569438697463215220 1,2
+// CHECK-NEXT: Exited Flat Section
 
 bool profileWriter() {
   TestProfileWriter W;
@@ -107,7 +142,9 @@ bool profileWriter() {
 }
 
 int main(int argc, char **argv) {
+  __llvm_ctx_profile_start_collection();
   theRoot();
+  flatFct();
   // This would be implemented in a specific RPC handler, but here we just call
   // it directly.
   return !profileWriter();
diff --git a/compiler-rt/test/tysan/lit.cfg.py b/compiler-rt/test/tysan/lit.cfg.py
index 817a3f0921e8d..f38e0211639da 100644
--- a/compiler-rt/test/tysan/lit.cfg.py
+++ b/compiler-rt/test/tysan/lit.cfg.py
@@ -144,4 +144,4 @@ def build_invocation(compile_flags):
     "Linux",
     "Darwin",
 ]:
-    config.unsupported = Tr
+    config.unsupported = True
diff --git a/flang-rt/CMakeLists.txt b/flang-rt/CMakeLists.txt
index 9f890a015621a..70b114041b7fd 100644
--- a/flang-rt/CMakeLists.txt
+++ b/flang-rt/CMakeLists.txt
@@ -223,7 +223,6 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
-
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
   find_compiler_rt_library(builtins FLANG_RT_BUILTINS_LIBRARY)
@@ -245,6 +244,10 @@ check_cxx_compiler_flag("-UTESTFLAG" FLANG_RT_SUPPORTS_UNDEFINE_FLAG)
 # Check whether -fno-lto is supported.
 check_cxx_compiler_flag(-fno-lto FLANG_RT_HAS_FNO_LTO_FLAG)
 
+# Different platform may have different name for the POSIX thread library.
+# For example, libpthread.a on AIX. Search for it as it is needed when
+# building the shared flang_rt.runtime.so.
+find_package(Threads)
 
 # function checks
 find_package(Backtrace)
@@ -252,6 +255,55 @@ set(HAVE_BACKTRACE ${Backtrace_FOUND})
 set(BACKTRACE_HEADER ${Backtrace_HEADER})
 
 
+if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  if (NOT DEFINED FLANG_RT_GCC_RESOURCE_DIR)
+    set(FLANG_RT_GCC_RESOURCE_DIR "FLANG_RT_GCC_RESOURCE_DIR-NOTFOUND")
+    execute_process(
+      COMMAND "${CMAKE_CXX_COMPILER}" -v -c "${FLANG_RT_SOURCE_DIR}/cmake/clang_gcc_root.cpp" ${CMAKE_CXX_FLAGS} -###
+      ERROR_FILE "${CMAKE_CURRENT_BINARY_DIR}/clang_gcc_root_result"
+    )
+    file(STRINGS "${CMAKE_CURRENT_BINARY_DIR}/clang_gcc_root_result" _errorresult)
+    foreach (_line IN LISTS _errorresult)
+      string(REGEX MATCH
+        "^Selected GCC installation: (.+)$"
+        _match
+        "${_line}")
+      if (CMAKE_MATCH_1)
+        set(FLANG_RT_GCC_RESOURCE_DIR "${CMAKE_MATCH_1}")
+        message(STATUS "Found GCC installation selected by Clang: ${FLANG_RT_GCC_RESOURCE_DIR}")
+        break()
+      endif ()
+    endforeach ()
+    set(FLANG_RT_GCC_RESOURCE_DIR "${FLANG_RT_GCC_RESOURCE_DIR}" CACHE INTERNAL "Path to GCC's resource dir selected by Clang" FORCE)
+  endif ()
+endif ()
+
+check_include_file("quadmath.h" FOUND_QUADMATH_H)
+if (FOUND_QUADMATH_H)
+  message(STATUS "quadmath.h found without additional include paths")
+  set(FLANG_RT_INCLUDE_QUADMATH_H "<quadmath.h>")
+elseif (FLANG_RT_GCC_RESOURCE_DIR)
+  cmake_push_check_state()
+    list(APPEND CMAKE_REQUIRED_INCLUDES "${FLANG_RT_GCC_RESOURCE_DIR}/include")
+    check_include_file("quadmath.h" FOUND_GCC_QUADMATH_H)
+  cmake_pop_check_state()
+  if (FOUND_GCC_QUADMATH_H)
+    message(STATUS "quadmath.h found in Clang's selected GCC installation")
+    set(FLANG_RT_INCLUDE_QUADMATH_H "\"${FLANG_RT_GCC_RESOURCE_DIR}/include/quadmath.h\"")
+  endif ()
+endif ()
+
+# Check if 128-bit float computations can be done via long double.
+check_cxx_source_compiles(
+  "#include <cfloat>
+   #if LDBL_MANT_DIG != 113
+   #error LDBL_MANT_DIG != 113
+   #endif
+   int main() { return 0; }
+  "
+  HAVE_LDBL_MANT_DIG_113)
+
+
 #####################
 # Build Preparation #
 #####################
@@ -273,7 +325,9 @@ set(CMAKE_CXX_STANDARD_REQUIRED YES)
 
 
 configure_file(cmake/config.h.cmake.in config.h)
-
+if (FLANG_RT_INCLUDE_QUADMATH_H)
+  configure_file("cmake/quadmath_wrapper.h.in" "${FLANG_RT_BINARY_DIR}/quadmath_wrapper.h")
+endif ()
 
 # The bootstrap build will create a phony target with the same as the top-level
 # directory ("flang-rt") and delegate it to the runtimes build dir.
diff --git a/flang-rt/cmake/clang_gcc_root.cpp b/flang-rt/cmake/clang_gcc_root.cpp
new file mode 100644
index 0000000000000..c3aeba4bb65d9
--- /dev/null
+++ b/flang-rt/cmake/clang_gcc_root.cpp
@@ -0,0 +1 @@
+int main() { return 0; }
\ No newline at end of file
diff --git a/flang-rt/cmake/modules/AddFlangRT.cmake b/flang-rt/cmake/modules/AddFlangRT.cmake
index a43f1c332187a..2292732daba6e 100644
--- a/flang-rt/cmake/modules/AddFlangRT.cmake
+++ b/flang-rt/cmake/modules/AddFlangRT.cmake
@@ -142,6 +142,9 @@ function (add_flangrt_library name)
   endif ()
   if (build_shared)
     add_library("${name_shared}" SHARED ${extra_args} ${ARG_ADDITIONAL_HEADERS} ${ARG_UNPARSED_ARGUMENTS})
+    if (Threads_FOUND) 
+      target_link_libraries(${name_shared} PUBLIC Threads::Threads)
+    endif ()
   endif ()
 
   if (libtargets)
@@ -195,7 +198,7 @@ function (add_flangrt_library name)
     # Use compiler-specific options to disable exceptions and RTTI.
     if (LLVM_COMPILER_IS_GCC_COMPATIBLE)
       target_compile_options(${tgtname} PRIVATE
-          $<$<COMPILE_LANGUAGE:CXX>:-fno-exceptions -fno-rtti -fno-unwind-tables -fno-asynchronous-unwind-tables>
+          $<$<COMPILE_LANGUAGE:CXX>:-fno-exceptions -fno-rtti -funwind-tables -fno-asynchronous-unwind-tables>
         )
     elseif (MSVC)
       target_compile_options(${tgtname} PRIVATE
diff --git a/flang-rt/cmake/quadmath_wrapper.h.in b/flang-rt/cmake/quadmath_wrapper.h.in
new file mode 100644
index 0000000000000..c29358efcdb0d
--- /dev/null
+++ b/flang-rt/cmake/quadmath_wrapper.h.in
@@ -0,0 +1,9 @@
+/*===-- cmake/quadmath_wrapper.h.in ---------------------=-----------*- C -*-===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===----------------------------------------------------------------------===*/
+
+#include ${FLANG_RT_INCLUDE_QUADMATH_H}
diff --git a/flang-rt/lib/quadmath/CMakeLists.txt b/flang-rt/lib/quadmath/CMakeLists.txt
index 4f113216b42c8..eaf8202d7bce9 100644
--- a/flang-rt/lib/quadmath/CMakeLists.txt
+++ b/flang-rt/lib/quadmath/CMakeLists.txt
@@ -78,8 +78,7 @@ target_include_directories(FortranFloat128MathILib INTERFACE
 
 if (FLANG_RUNTIME_F128_MATH_LIB)
   if (${FLANG_RUNTIME_F128_MATH_LIB} STREQUAL "libquadmath")
-    check_include_file(quadmath.h FOUND_QUADMATH_HEADER)
-    if(FOUND_QUADMATH_HEADER)
+    if(FLANG_RT_INCLUDE_QUADMATH_H)
       add_compile_definitions(HAS_QUADMATHLIB)
     else()
       message(FATAL_ERROR
diff --git a/flang-rt/lib/quadmath/complex-math.h b/flang-rt/lib/quadmath/complex-math.h
index 424ed84da4e01..b76bf03732938 100644
--- a/flang-rt/lib/quadmath/complex-math.h
+++ b/flang-rt/lib/quadmath/complex-math.h
@@ -13,7 +13,7 @@
 #include "flang/Runtime/entry-names.h"
 
 #if HAS_QUADMATHLIB
-#include "quadmath.h"
+#include "quadmath_wrapper.h"
 #define CAbs(x) cabsq(x)
 #define CAcos(x) cacosq(x)
 #define CAcosh(x) cacoshq(x)
diff --git a/flang-rt/lib/quadmath/math-entries.h b/flang-rt/lib/quadmath/math-entries.h
index 6e47f32cc8a43..91ad80c8197f4 100644
--- a/flang-rt/lib/quadmath/math-entries.h
+++ b/flang-rt/lib/quadmath/math-entries.h
@@ -112,7 +112,7 @@ DEFINE_FALLBACK_F128(Yn)
 
 #if HAS_QUADMATHLIB
 // Define wrapper callers for libquadmath.
-#include "quadmath.h"
+#include "quadmath_wrapper.h"
 DEFINE_SIMPLE_ALIAS(Abs, fabsq)
 DEFINE_SIMPLE_ALIAS(Acos, acosq)
 DEFINE_SIMPLE_ALIAS(Acosh, acoshq)
diff --git a/flang-rt/lib/runtime/command.cpp b/flang-rt/lib/runtime/command.cpp
index 8a5a61ac1ad44..9ada5bd59c0b7 100644
--- a/flang-rt/lib/runtime/command.cpp
+++ b/flang-rt/lib/runtime/command.cpp
@@ -21,9 +21,11 @@
 #define getcwd _getcwd
 #define PATH_MAX MAX_PATH
 
+#ifdef _MSC_VER
 // On Windows GetCurrentProcessId returns a DWORD aka uint32_t
 #include <processthreadsapi.h>
 inline pid_t getpid() { return GetCurrentProcessId(); }
+#endif
 #else
 #include <unistd.h> //getpid()
 
diff --git a/flang/docs/ArrayRepacking.md b/flang/docs/ArrayRepacking.md
new file mode 100755
index 0000000000000..544904fd968cc
--- /dev/null
+++ b/flang/docs/ArrayRepacking.md
@@ -0,0 +1,514 @@
+<!--===- docs/ArrayRepacking.md
+
+   Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+   See https://llvm.org/LICENSE.txt for license information.
+   SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+-->
+
+# Assumed-shape arrays repacking
+Fortran 90 introduced dummy arguments to be declared as assumed-shape arrays, which allowed to pass non-contiguous arrays to subprograms. In some cases, accessing non-contiguous arrays may result in poor program performance, and paying an overhead of copying a non-contiguous array into a contiguous memory (packing) before processing it may result in better performance. This document describes Flang compiler and runtime support for packing/unpacking of non-contiguous arrays.
+
+## A problem case
+
+[Example #1](#example-1) provides a way to compare performance of a repetitive access of a large array when the array is contiguous and non-contiguous. The `test` function remains the same in both cases to make sure that any difference in the code generation does not affect performance, and only the array layout in memory matters.
+
+The example might be compiled using any Fortran 90 compiler, e.g. `gfortran -cpp example1.f90 -O2 <additional-options>`. The table below contains performance information for different compilations and targets:
+
+| additional-options | AMD EPYC 9684X, GNU Fortran 13.2.0                           | Arm Neoverse V2, GNU Fortran 11.4.0                          |
+| ------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ |
+| `none`             | 20495466758      L1-dcache-loads<br/>            31403868      L1-dcache-prefetches<br/>     10116173649      L1-dcache-load-misses<br/>119,167,236,596      cycles | 20030549910      L1-dcache-loads<br/>   10233598442      L1-dcache-load-misses<br/>     1098681496      LLC-load-misses<br/>43,426,056,799      cycles |
+| `-DREPACKING`      | 20245847735      L1-dcache-loads<br/>         583614040      L1-dcache-prefetches<br/>         644552282      L1-dcache-load-misses<br/>  10,837,843,298      cycles | 20023110457      L1-dcache-loads<br/>              294393      L1-dcache-load-misses<br/>              321878      LLC-load-misses<br/>10,065,421,618      cycles |
+| `-frepack-arrays`  | 20248624699      L1-dcache-loads<br/>         584325700      L1-dcache-prefetches<br/>         644353154      L1-dcache-load-misses<br/>  10,850,830,504      cycles | 20023117997      L1-dcache-loads<br/>              275169      L1-dcache-load-misses<br/>              323902      LLC-load-misses<br/>10,066,689,166      cycles |
+
+The default version is much slower than the version with manual array repacking and [-frepack-arrays](https://gcc.gnu.org/onlinedocs/gfortran/Code-Gen-Options.html#index-frepack-arrays) due to the L1 data cache misses even considering the extra overhead required to pack/unpack the non-contiguous array.
+
+This artificial example was inspired by the innermost hot loop from `fourir` subroutine of [Polyhedron/capacita](https://fortran.uk/fortran-compiler-comparisons/the-polyhedron-solutions-benchmark-suite/) benchmark, which speeds up about 1.8x with GNU Fortran compiler's `-frepack-arrays` option on AMD EPYC 9684X and 1.3x - on Arm Neoverse V2.
+
+Having these results it seems reasonable to provide support for arrays repacking in Flang compiler, which may reduce the amount of effort to rewrite existing Fortran programs for better data cache utilization.
+
+## Implementations in other compilers
+
+### GNU Fortran compiler
+
+`-frepack-arrays` option of GNU Fortran compiler let's the compiler generate special subprogram prologue/epilogue code that performs automatic packing/unpacking of the assumed-shape dummy arrays. With some implementation limitations, the following happens for any such dummy array:
+
+**In subprogram prologue:** iff the array is not contiguous in any dimension, it is copied into a newly allocated contiguous chunk of memory, and the following subprogram code operates on the temporary. This is a `pack` action consisting of the allocation and copy-in.
+
+**In subprogram epilogue:** iff the array is not contiguous in any dimension, values from the temporary array are copied over to the original array and the temporary array is deallocated. This is `unpack` action consisting of the copy-out and deallocation. It makes sure any updates of the array done by the subprogram are propagated to the caller side.
+
+#### Facts and guesses about the implementation
+
+The dynamic checks for continuity and the array copy code is located completely in the [runtime](https://github.com/gcc-mirror/gcc/blob/3e08a4ecea27c54fda90e8f58641b1986ad957e1/libgfortran/generated/in_pack_r8.c#L35), so the compiler inserts unconditional calls in the subprogram prologue/epilogue.
+
+It looks like `gfortran` ignores `intent(out)/intent(in)` which could have helped to avoid some of the `pack/unpack` overhead.
+
+It looks like the `pack`/`unpack` actions are inserted early in the compilation pipeline, and these extra calls affect behavior of the later optimization passes. For example, `Polyhedron/fatigue2` slows down by about 2x with `-frepack-arrays`: this slowdown is not caused by the `pack`/`unpack` overhead, but is a consequence of worse function inlining decisions made after the calls insertion. The benchmarks becomes even faster than the original version with `-frepack-arrays` and proper `-finline-limit=` settings, but it does not look like the benchmark contains code that would benefit from the array repacking.
+
+It does not look like `gfortran` is able to eliminate the `pack`/`unpack` code after the function inlining, if the actual argument is statically known to be contiguous. So the overhead from the dynamic continuity checks is inevitable when `-frepack-arrays` is specified.
+
+It does not look like `gfortran` tries to optimize the insertion of `pack`/`unpack` code. For example, if a dummy array is only used under a condition within the subprogram, the repacking code might be inserted under the same condition to minimize the overhead on the unconditional path through the subprogram.
+
+### NVIDIA HPC Fortran compiler
+
+`nvfortran` compiler performs array repacking by default, and has few option to control this behavior (only [-M[no]target_temps](https://docs.nvidia.com/hpc-sdk/compilers/hpc-compilers-ref-guide/index.html#command-line-options-reference)). The compiler inserts `pack`/`unpack` code around the calls of subprograms that have assumed-shape dummy array arguments (a procedure having an assumed-shape dummy argument must have an explicit interface due to F2018 15.4.2.2, 1, (3), (b)):
+
+**Before the call:** iff the array is not contiguous in the innermost dimension, it is copied into a newly allocated contiguous chunk of memory, and the temporary array is passed to the callee.
+
+**After the call:** iff the array is not contiguous in the innermost dimension, values from the temporary array are copied to the original array and the temporary array is deallocated.
+
+#### Facts and guesses about the implementation
+
+The `pack` code is only generated if the actual argument may be non-contiguous in the innermost dimension, as determined statically, i.e. the compiler does not generate any dynamic continuity checks. For example:
+
+```Fortran
+interface
+  subroutine test1(x)
+    real :: x(:)
+  end subroutine test1
+  subroutine test2(x)
+    real :: x(:,:)
+  end subroutine test2
+end interface
+real :: x(m1,m2), y(1,m2)
+call test1(x(1,:)) ! case 1
+call test1(y(1,:)) ! case 2
+call test2(x(1:n,:)) ! case 3
+call test2(x(1:1,:)) ! case 4
+```
+
+In case 1, the `pack`/`unpack` code is generated without dynamically checking if `m1 == 1` (in which case the actual argument is actually contiguous).
+
+In case 2, the `pack`/`unpack` code is also generated, which is a room for improvement.
+
+In case 3 and 4, the `pack`/`unpack` code is not generated, because the actual argument is contiguous in the innermost dimension. There seems to be room for improvement in case 4, where it might be beneficial to repack the array in case `m1` is big enough to prevent the data cache utilization (depending on the actual processing of the array in `test2`, of course).
+
+`nvfortran` does optimize out the `unpack` copy-out code in case the dummy argument is declared `intent(in)`, but it does not optimize the `pack` copy-in in case it is declared `intent(out)`.
+
+It looks like `nvfortran` is not able to optimize the `pack`/`unpack` code after the function inlining (`-Minline=reshape`), even if the inline code makes it obvious that only a single element of the array is being accessed and there is no reason to copy-in/out the whole array.
+
+`nvfortran`'s implementation guarantees that an assumed-shape dummy array is contiguous in the innermost dimension, so when such a dummy is passed to a callee as an actual argument associated with the callee's assumed-shape dummy array, there is no need to `pack`/`unpack` it again around the callee's call site.
+
+## Known limitations of the array repacking
+
+`gfortran` documentation, expectedly, warns that the array repacking `can introduce significant overhead to the function call, especially when the passed data is noncontiguous`. A compiler has to try to minimize the overhead of the copy-in/out actions whenever possible, but it may not be always possible to guess correctly when the repacking is profitable. So the `gfortran`'s approach of giving the users control over the repacking seems reasonable. A compiler may decide to enable array repacking by default or under some optimization levels, but the correctness issues described below has to be taken into account as well as performance and usability (i.e. the need to specify a compiler option to enable/disable array repacking).
+
+**Difference between performance of nvfortran and gfortran **
+
+Array repacking creates a complete copy of an array section and let's the program code work on the temporary copy, then reflecting the updates back through another copy. If the original program intends to let different threads to work on different parts of the same array section, then the copy-in/out actions introduce a data race that has not existed in the original program. [Example #2](#example-2) produces inconsistent results when being compiled with either `nvfortran -mp` or `gfortran -fopenmp -frepack-arrays` and run with multiple threads. Note that the `repacking` subroutine and its call site might be written such that they are located in separate modules that do not have to be compiled with `-mp/-fopenmp`, so a compiler has no clue whether array repacking is safe. Even if explicitly instructed via `-frepack-arrays`, the compiler cannot avoid false-positive warnings about unsafety of array repacking, because it cannot know whether a function might be called in a multithreaded context (e.g. when `-mp/-fopenmp` is not specified).
+
+The array copies may also become a problem for OpenACC/OpenMP target data environment management. For example:
+
+```Fortran
+subroutine test(x)
+  real :: x(:)
+  !$acc serial present(x)
+  ...
+  !$acc end serial
+end subroutine test
+subroutine caller(n)
+  integer :: n
+  real :: x(n,n)
+  !$acc enter data create(x)
+  call test(x(1,:))
+end subroutine caller
+```
+
+The whole array `x` is expected to be present in the device data environment after the `enter data` construct, but the actual array being "seen" at the `serial` construct is a temporary copy of the array section, which has no corresponding memory on the device.
+
+A compiler could generate code that dynamically detect both of these situations, i.e. whether the point of repacking is happening in a multithreaded context or whether the array to be repacked has associated bookkeeping in the device data environment, and do not create copies. Such checks would introduce dependencies on the parallelization/offload runtime libraries, which are not linked unless compiler is instructed to do so via `-acc/-fopenacc/-mp/-fopenmp/etc.`
+
+So it does not seem practical/reasonable to enable the array repacking by default in a compiler that must produce correct code for all standard conformant programs. It is still beneficial to let users request array repacking, given that its behavior is properly documented and all the warning signs are in place.
+
+## Flang feature requirements
+
+### Correctness
+
+1. Support repacking of assumed-shape array dummy arguments or actual array arguments associated with such dummy arguments of any data types.
+2. When array repacking is enabled, Flang should strive to provide correct program behavior when OpenACC/OpenMP features are explicitly enabled during the compilation.
+   * It is unclear if the compiler/runtime can always prevent the array repacking to produce incorrect code. Thus, the implementation should do its best to prevent incorrect behavior or diagnose the incorrect behavior in runtime as soon as possible.
+   * To stress again, such implementation can only be enabled when users enable OpenACC/OpenMP explicitly during the compilation, so that the compiler can rely on the appropriate OpenACC/OpenMP runtime libraries to be linked into the resulting module.
+
+### Performance
+
+1. Minimize the overhead of array repacking, e.g. avoid copy-in/out whenever possible, execute copy-in/out only on the execution paths where the array is accessed.
+2. Provide different modes of repacking depending on the "continuity" meaning, i.e. one - array is contiguous in the innermost dimension, two - array is contiguous in all dimensions.
+3. Avoid generating repacking code, when the "continuity" can be statically proven (including after optimization passes like constant propagation, function inlining, etc.).
+4. Use a set of heuristics to avoid generating repacking code based on the array usage pattern, e.g. if an array is proven not to be used in an array expression or a loop, etc.
+5. Use a set of heuristics to avoid repacking actions dynamically, e.g. based on the array size, element size, byte stride(s) of the [innermost] dimension(s), etc.
+6. Minimize the impact of the IR changes, introduced by repacking, on the later optimization passes.
+
+### Usability
+
+1. Provide command line options to enable/disable array repacking, e.g. `-f[no-]repack-arrays` for `gfortran` cli compatibility.
+2. Provide command line options to instruct the compiler which performance heuristics to use with the default picked based on benchmarking.
+3. Provide consistent behavior of the temporary arrays with relation to `-fstack-arrays` (that forces all temporary arrays to be allocated on the stack).
+4. Produce correct debug information to substitute the original array with the copy array when accessing values in the debugger.
+5. Document potential correctness issues that array repacking may cause in multithreaded/offload execution.
+
+## Proposed design
+
+### Overview
+
+Controlled by cli options, Lowering will generate a `fir.pack_array` operation in a subprogram's prologue for each assumed-shape dummy array argument (including `OPTIONAL`). For each `fir.pack_array` it will also generate `fir.unpack_array` in the subprogram's epilogue. These new operations will represent the complete effects of `pack`/`unpack` actions, such as temp-allocation/copy-in/copy-out/temp-deallocation. While it is possible to represent the needed actions using existing FIR/HLFIR operations, it is worth keeping them more specific and compact for easier manipulation in the passes related to optimizing the `pack`/`unpack` actions.
+
+The new operations will hold all the information that customizes further handling of the `pack`/`unpack` actions, such as:
+
+* Optional array of attributes supporting an interface to generate a predicate that says if the repacking is safe in the current context.
+* The continuity mode: `innermost` vs `whole`.
+* Attributes selecting the heuristics (both compiler and runtime ones) that may be applied to avoid `pack`/`unpack` actions.
+* Other attributes, like `stack` vs `heap` to manage the temporary allocation according to `-fstack-arrays`, etc.
+
+Lowering will not try to optimize the insertion of new operations, except for obvious cases like `CONTIGUOUS` dummy arguments or arrays of elements bigger than the element size threshold. Further optimization passes will be responsible for optimizing the operations away or moving them around to satisfy the performance requirements.
+
+The following FIR passes should be implemented:
+
+* Deletion of `fir.pack_array`/`fir.unpack_array` that are statically proven to take a contiguous input array.
+* Deletion/merging of cascaded `fir.pack_array` operations.
+* Deletion of the new operations that are statically proven not to meet the array usage patterns that are considered to benefit from the array repacking.
+* Deletion of the new operations that are statically proven not to meet the dynamic conditions for repacking (such as the array size).
+* Repositioning of `fir.pack_array`/`fir.unpack_array` to execution paths where the array is actually accessed.
+* A pass converting the operations to the existing FIR operations and/or Fortran runtime calls.
+
+### New operations to represent pack/unpack
+
+#### fir.pack_array operation
+
+The operation has the following syntax:
+
+```
+%new_var = fir.pack_array %var
+    [stack ]
+    [innermost ]
+    [no_copy ]
+    [heuristics([none|loop-only]) ]
+    [constraints([max-size = <int>, ][max-element-size = <int>, ]
+    			 [min-stride = <int>]) ]
+    [typeparams %p1, ... ]
+    [<[acc.temp_copy_is_safe][omp.temp_copy_is_safe]>]
+    : !fir.box/class<!fir.array<...>>
+```
+
+The operation creates a new `!fir.box/class<!fir.array<>>` value to represent either the original `%var` or a newly allocated temporary array, maybe identical to `%var` by value.
+
+Arguments:
+
+* `stack` - indicates if `-fstack-arrays` is in effect for compiling this function.
+* `innermost` - tells that the repacking has to be done iff the array is not contiguous in the innermost dimension. This also describes what type of continuity can be expected from `%new_var`, i.e. `innermost` means that the resulting array is definitely contiguous in the innermost dimension, but may be non-contiguous in other dimensions (unless additional analysis proves otherwise). For 1-D arrays, `innermost` attribute is not valid.
+* `no_copy` - indicates that, in case a temporary array is created, `%var` to `%new_var` copy is not required (`intent(out)` dummy argument case).
+* `heuristics`
+  * `loop-only` - `fir.pack_array` can be optimized away, if the array is not used in a loop.
+  * `none` - `fir.pack_array` cannot be optimized based on the array usage pattern.
+* `constraints`
+  * `max-size` - constant integer attribute specifying the maximum byte size of an array that is eligible for repacking.
+  * `max-element-size` - constant integer attribute specifying the maximum byte element-size of an array that is eligible for repacking.
+  * `min-stride` - constant integer attribute specifying the minimum byte stride of the innermost dimension of an array that is eligible for repacking.
+* `typeparams` - type parameters of the element type.
+* `*.temp_copy_is_safe`: a list of attributes implementing `TempCopyIsSafe` attribute interface for generating a boolean value indicating whether using a temporary copy instead of the original array is safe in the current context.
+
+Memory effects are conservative, assuming that an allocation and copy may happen:
+
+* `MemAlloc` effect on either `AutomaticAllocationScopeResource` or `DefaultResource` depending on `stack` attribute.
+  * The memory allocation effect is especially important for operations with `stack` attribute so that `fir.pack_array` operations are not reordered inconsistently with their corresponding `fir.unpack_array` operations. This may cause issues with later lowering of this operations into `stacksave/stackrestore` pairs.
+* If there is no `no_copy`:
+  * `MemRead` effect on unknown value to indicate potential read from the original array. This effect prevents hoisting a `fir.pack_array` above any write to the original array.
+    * [TBD] we can relax that by having an additional argument taking `fir.box_addr %var` value, though, this adds some redundancy to the argument list.
+
+Alias analysis:
+
+* For the purpose of alias analysis `fir.pack_array` should be considered a pass-through operation, meaning that when FIR alias analysis is looking for the source of a pointer, emboxed in the result box of `fir.pack_array`, the search is continued through the operation's argument box def-use.
+
+#### fir.unpack_array operation
+
+The operation has the following syntax:
+
+```
+fir.unpack_array %new_var to %var [stack ] [no_copy ] : !fir.box/class<!fir.array<...>>
+```
+
+The operation is either a no-op or deallocates the temporary array, and maybe copies the temporary array into the original array.
+
+Arguments:
+
+* `%new_var` is a value produced by `fir.pack_array`. In general, `%new_var` may become a block argument, and getting the other properties from the producer may not be possible. So some arguments are duplicated on `fir.unpack_array`.
+* `stack` - indicates if `-fstack-arrays` is in effect for compiling this function.
+* `no_copy` - indicates that, in case a temporary array is created, `%new_var` to `%var` copy is not required (`intent(in)` dummy argument case).
+
+Memory effects are conservative, assuming that a copy and deallocation may happen:
+
+* `MemFree` effect on either `AutomaticAllocationScopeResource` or `DefaultResource` depending on `stack` attribute.
+* If there is no `no_copy`:
+  * `MemWrite` effect on unknown value to indicate potential write into the original array. This effect should prevent hoisting any reads/writes of the original array above a `fir.unpack_array`, since those hoisted reads/writes may address the original array memory that has not been updated from the temporary copy yet.
+
+### New attribute interface
+
+The `TempCopyIsSafe` attribute interface provides means to generate programming model specific predicates saying whether repacking is safe or not at the point where it needs to be done. For example the OpenMP MLIR dialect may provide an attribute implementing this interface to generate a runtime check at the point of packing array `x` inside subroutine `repacking`. A conservative implementation might look like this:
+
+```C
+  repacking_is_safe = omp_get_num_team() == 1 && omp_get_num_threads() == 1;
+```
+
+The attribute interface will be used during lowering of `fir.pack_array` to generate the predicate and make the packing conditional and safe. This will allow applying repacking correctly in programs compiled with `-fopenmp`, and get the benefits of repacking in the serial parts of those programs.
+
+Similarly, the OpenACC MLIR dialect may provide such an attribute to check if a device copy has been created for an array that is about to be packed, and prevent the repacking for cases like this:
+
+```Fortran
+program main
+  real :: x(2,10)
+  x = 0.0
+  !$acc enter data copyin(x)
+  do i=1,2
+     call repacking(x(i,:))
+  end do
+  !$acc exit data copyout(x)
+  print *, x
+contains
+  subroutine repacking(x)
+    real :: x(:)
+    !$acc parallel loop present(x)
+    do j=1,10
+       x(j) = x(j) + 1.0
+    end do
+  end subroutine repacking
+end program main
+```
+
+Note that it may not be possible to detect all the cases where repacking breaks the original OpenACC/OpenMP program behavior. In the following case, it is unclear how to synchronize the repacking in both routines (especially, when the OpenACC directives are not lexically visible at the point of repacking):
+
+```Fortran
+program main
+  real :: x(2,10)
+  x = 0.0
+  call repacking_in(x(1,:))
+  call repacking_out(x(1,:))
+  print *, x
+contains
+  subroutine repacking_in(x)
+    real :: x(:)
+    !$acc enter data copyin(x)
+  end subroutine repacking_in
+  subroutine repacking_out(x)
+    real :: x(:)
+    !$acc exit data copyout(x)
+  end subroutine repacking_out
+end program main
+```
+
+So the most conservative implementation of the predicate generator may be to always produce `false` value. Flang lowering may attach any number of such attributes to `fir.pack_array` depending on the compilation context and options.
+
+[TBD] define `TempCopyIsSafe` attribute interface so that OpenACC/OpenMP dialects can provide their specific attributes, which can be used to generate static/runtime checks for safety of the temporary copy in particular context.
+
+#### Alternatives/additions to the attribute interface
+
+The following ideas were expressed during the review, and they are worth considering.
+
+| |
+| - |
+| For OpenACC/OpenMP runtime to be able to detect/handle the presence of the original array and the temporary copy in the device data environment, descriptor flags/properties might be used to mark the copy's descriptor as such and provide a link to the original array (or its descriptor). It may be problematic to maintain such flags/properties, in general, because of the repacking that may happen, especially, in C code, where the flags/properties might be dropped. Moreover, a copy array might be repacked into another copy array multiple times, so a descriptor might need to keep a chain of associated arrays and it will have to be maintained as well.<br>An alternative to tracking the original-copy "association" might be compiler generated code notifying the OpenACC/OpenMP offload runtime about the copy being created/deleted for the original array, so that the offload runtime can disallow repacking or report an error when the repacking is definitely causing the program to behave incorrectly. The compiler may report the "association" to the runtime through callbacks provided by `TempCopyIsSafe` attribute interface, and this will require proper bookkeeping in the runtime specific to the array repacking. |
+| There may be some uses for an API allowing to statically determine whether a given descriptor (SSA value) represent the repacked copy of the original array. For example, it may be in the form of an API in the OpenACC `MappableType` interface. This can be done with some limitations for the values produced by `fir.pack_array` that are dynamic (i.e. the copy is created conditionally based on the runtime checks). |
+| The compiler can also try to statically determine the conditions where the array repacking might be unsafe, e.g. a presence of memory barriers or operations carrying implicit memory barriers, presence of atomic operations between the `pack/unpack` operations may indicate non-trivial handling of the array memory. Such checks may result in the removal of `pack/unpack` operations, and they can probably be done in a mandatory pass (not an optimization pass). At the same time, the result of the checks may depend on other optimization passes (e.g. inlining), so the behavior may be inconsistent between different optimization levels. |
+
+### Lowering Fortran to FIR
+
+Fortran code:
+
+```Fortran
+subroutine test(x,n)
+  character(n) :: x(2:)
+end subroutine test
+```
+
+HLFIR/FIR produced by Lowering:
+
+```
+  func.func @_QPtest(%arg0: !fir.box<!fir.array<?x!fir.char<1,?>>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i32> {fir.bindc_name = "n"}) {
+    %0 = fir.dummy_scope : !fir.dscope
+    %1:2 = hlfir.declare %arg1 dummy_scope %0 {uniq_name = "_QFtestEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+    %2 = fir.load %1#0 : !fir.ref<i32>
+    %c0_i32 = arith.constant 0 : i32
+    %3 = arith.cmpi sgt, %2, %c0_i32 : i32
+    %4 = arith.select %3, %2, %c0_i32 : i32
+    %c2_i64 = arith.constant 2 : i64
+    %5 = fir.convert %c2_i64 : (i64) -> index
+    %6 = fir.shift %5 : (index) -> !fir.shift<1>
+    %repacked = fir.pack_array %arg0 typeparams %4 : !fir.box<!fir.array<?x!fir.char<1,?>>>
+    %7:2 = hlfir.declare %repacked(%6) typeparams %4 dummy_scope %0 {uniq_name = "_QFtestEx"} : (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.shift<1>, i32, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.box<!fir.array<?x!fir.char<1,?>>>)
+    fir.unpack_array %repacked to %arg0 : !fir.box<!fir.array<?x!fir.char<1,?>>>
+    return
+  }
+```
+
+#### Restrictions:
+
+It is unsafe to create temporaries of assumed-shape dummy arrays that have `TARGET` attribute, because they can be accessed during the invocation of the subprograms not using direct reference of the dummy argument. Lowering must never produce the new operations for such dummy arguments. [TBD] a user option might be provided to override the default safe behavior.
+
+The copy creation is also restricted for `ASYNCHRONOUS` and `VOLATILE` arguments. Such dummy arguments might be changed during the execution of their subprogram in an unpredictable manner, so creating a copy for them might be incorrect (Fortran 2023, Note 5 of section 15.5.2.5).
+
+It does not make sense to generate the new operations for `CONTIGUOUS` arguments and for arguments with statically known element size that exceeds the `max-element-size` threshold.
+
+#### Optional behavior
+
+In case of the `whole` continuity mode or with 1-D array, Flang can propagate this information to `hlfir.declare` - this may improve optimizations down the road. This can be done iff the repacking has no dynamic constraints and/or heuristics. For example:
+
+```
+    %c0 = arith.constant 0 : index
+    %6:3 = fir.box_dims %arg0, %c0 : (!fir.box<!fir.array<?x!fir.char<1,?>>>, index) -> (index, index, index)
+    %c2_i64 = arith.constant 2 : i64
+    %7 = fir.convert %c2_i64 : (i64) -> index
+    %8 = fir.shape_shift %7, %6#1 : (index, index) -> !fir.shapeshift<1>
+    %repacked = fir.pack_array %arg0 typeparams %5 : !fir.box<!fir.array<?x!fir.char<1,?>>>
+    %9 = fir.box_addr %repacked : (!fir.box<!fir.array<?x!fir.char<1,?>>>) -> !fir.ref<!fir.array<?x!fir.char<1,?>>>
+    %10:2 = hlfir.declare %9(%8) typeparams %5 dummy_scope %0 {fortran_attrs = #fir.var_attrs<contiguous>, uniq_name = "_QFtestEx"} : (!fir.ref<!fir.array<?x!fir.char<1,?>>>, !fir.shapeshift<1>, i32, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.ref<!fir.array<?x!fir.char<1,?>>>)
+```
+
+This may complicate future `fir.pack_array` optimization passes, but it is worth considering.
+
+### Lowering new operations
+
+Lowering of the new operations (after all the optimizations) might be done in a FIR-to-FIR conversion pass.
+
+`fir.pack_array` lowering might be done in the following steps:
+
+* If there are dynamic constraints, generate a boolean `p1` that is set to true if repacking has to be done (depending on the constraints values and the original array descriptor). The IR might be cleaner if we generate a Fortran runtime call here.
+* If there are attributes implementing `TempCopyIsSafe` attribute interface, then use the interface method to generate boolean predicates for each such attribute: `p2`, ..., `pn`.
+  * [TBD] it seems that the runtime checks for the target offload programs will require a pure `PointerLike` value for the array start and the total byte size of the array (e.g. as `index` value).
+* Compute `p = p1 && p2 && ... && pn`.
+* Compute the total size of the temporary `required_size` (in elements).
+* Compute the total size of the allocation `allocation_size = p ? required_size : 0`.
+* If `stack` is present, then allocate memory on the stack `%space = fir.alloca <element-type>, %allocation_size`, otherwise, `%space = fir.allocmem !fir.array<?x<element-type>>, %allocation_size`.
+* Embox `%space` into a temporary descriptor `%new_box`.
+* `no_copy`:
+  * If present, then `%new_box` is a new result of `fir.pack_array`.
+  * If not present, then call Fortran runtime to do the copy into a pre-allocated temporary. Some part of the existing `PACK` runtime may be reused, but the intention is to have a shallow copy vs the deep copy implemented by `PACK`. The runtime call modifies `%new_box` and it becomes a new result of `fir.pack_array`.
+  * For `OPTIONAL` absent argument, the result of the operation is the original absent box.
+
+`fir.unpack_array` lowering:
+
+* If `no_copy` is not present, then call Fortran runtime to do the the copy into the original array. The copy must be made iff `fir.box_addr %var != fir.box_addr %new_var`.
+
+* If `stack` is not present, generate:
+
+  ```
+  %new_addr = fir.box_addr %new_var
+  %orig_addr = fir.box_addr %orig_addr
+  %cmp = arith.cmp neq, %new_addr, %orig_addr
+  fir.if %cmp {
+    fir.freemem %new_addr
+  }
+  ```
+  Alternatively, we can let the Fortran runtime do the heap deallocation.
+
+### Runtime
+
+[TBD] define the runtime APIs.
+
+### Optimization passes
+
+[TBD] describe in more details optimization passes listed in [Overview](#overview). Define their place in the pipeline (e.g. with relation to FIR inlining).
+
+#### Loop versioning
+
+There is an existing optimization pass (controlled via `-f[no-]version-loops-for-stride`) that creates specialized versions of the loop nests which process assumed-shape dummy arrays. The pass adds a dynamic check for the array(s) contiguity (in the innermost dimension) and exposes the contiguity by rewriting the array(s) accesses via raw pointer references. This transformation may enable more unit-stride vectorization. The pass is beneficial only if the actual array argument is contiguous in the innermost dimension.
+
+The array repacking is targeting better data cache utilization, and is not intended to enable more unit-strided vectorization for the assumed-shape arrays. At the same time, combining array repacking with the loop versioning may provide better performance for programs where the actual array arguments are non-contiguous, but then their repacked copies can be accessed using unit strides.
+
+In cases where `fir.pack_array` is statically known to produce a copy that is contiguous in the innermost dimension, the loop versioning pass can skip the generation of the dynamic checks.
+
+### Driver: user options
+
+The following user options are proposed:
+
+* `-frepack-arrays` - the option forces Flang to repack a non-contiguous assumed-shape dummy array into a temporary contiguous memory, which may result in faster accesses of the array. The compiler will insert special code in subprogram prologue to allocate a temporary array and copy the original array into the temporary; in subprogram epilogue, it will insert a copy from the temporary array into the original array and deallocate the temporary. The overhead of the allocation/deallocation and the copies may be significant depending on the array size. The compiler will try to optimize the unnecessary/unprofitable repacking.
+* `-frepack-arrays-opts=[none|loop-only]` - the option enables optimizations that may eliminate the array repacking code depending on the array usage pattern:
+  * `none` - no optimizations.
+  * `loop-only` - the array repacking code will be removed in any subprogram where the array is not used inside a loop or an array expression.
+* `-frepack-arrays-continuity=[whole|innermost]`:
+  * `whole` - the option will repack arrays that are non-contiguous in any dimension (default).
+  * `innermost` - the option will repack arrays that are non-contiguous in the innermost dimension.
+* `-frepack-arrays-max-size=<int>` - arrays bigger than the specified size will not be repacked.
+* `-frepack-arrays-max-element-size=<int>` - arrays with elements bigger than the specified size will not be repacked.
+* `-frepack-arrays-min-stride=<int>` - arrays with the byte stride of the innermost dimension less than the specified value will not be repacked (e.g. it may not be profitable to repack arrays with the stride in the innermost dimension that is less than the data cache line size).
+* [TBD] defaults for `-frepack-arrays-max-size`, `-frepack-arrays-max-element-size` and `-frepack-arrays-min-stride` will be defined based on benchmark tuning and might be target specific.
+
+## Examples
+### Example #1
+
+```fortran
+subroutine test(x,n,repeat,m1,m2,m3)
+  real x(:)
+  integer n,repeat,repeat_max
+  integer i,j,idx1,idx2,idx3
+  do j=1,repeat
+     idx1=(j-1)/m1+0 ! always 0
+     idx2=(j-1)/m2+n/3 ! always n/3
+     idx3=(j-1)/m3+2*n/3 ! always 2*n/3
+     do i=1,n/3
+        x(idx1+i)=x(idx1+i)+1.0
+        x(idx2+i)=x(idx2+i)+1.0
+        x(idx3+i)=x(idx3+i)+1.0
+     end do
+  end do
+end subroutine test
+
+subroutine wrapper(x,n)
+  interface
+     subroutine test(x,n,repeat,m1,m2,m3)
+       real x(:)
+       integer n,repeat,m1,m2,m3
+     end subroutine test
+  end interface
+  integer, parameter :: repeat=1000000
+  integer n
+  real x(n,n)
+#if REPACKING
+  real,allocatable :: tmp(:)
+  allocate(tmp(SIZE(x,1)))
+  tmp(:)=x(1,:)
+  call test(tmp,n,repeat,repeat,repeat,repeat)
+#else
+  call test(x(1,:),n,repeat,repeat,repeat,repeat)
+#endif
+#if REPACKING
+  x(1,:)=tmp(:)
+  deallocate(tmp)
+#endif
+  print *,'X: ',x(1,::n/2)
+end subroutine wrapper
+
+program main
+  interface
+     subroutine wrapper(x,n)
+       integer n
+       real x(n,n)
+     end subroutine wrapper
+  end interface
+  integer, parameter :: n=10000
+  real x(n,n)
+  x=0.0
+  call wrapper(x,n)
+end program main
+```
+
+### Example #2
+
+```Fortran
+program main
+  use omp_lib
+  real :: x(2,10)
+  x = 0.0
+  print *, x(1,:)
+  !$omp parallel do num_threads(10) shared(x)
+  do i=1,10
+     call repacking(x(1,:), i)
+  end do
+  !$omp end parallel do
+  print *, x(1,:)
+contains
+  subroutine repacking(x, i)
+    real :: x(:)
+    integer :: i
+    print *, i !
+    x(i) = x(i) + 1.0
+  end subroutine repacking
+end program main
+```
diff --git a/flang/docs/index.md b/flang/docs/index.md
index c35f634746e68..1de0ee2e6f0d6 100644
--- a/flang/docs/index.md
+++ b/flang/docs/index.md
@@ -41,6 +41,7 @@ on how to get in touch with us and to learn more about the current status.
    Aliasing
    AliasingAnalysisFIR
    ArrayComposition
+   ArrayRepacking
    AssumedRank
    BijectiveInternalNameUniquing
    Calls
diff --git a/flang/examples/FeatureList/FeatureList.cpp b/flang/examples/FeatureList/FeatureList.cpp
index ddb9dee5b8a8e..a1a908820e39c 100644
--- a/flang/examples/FeatureList/FeatureList.cpp
+++ b/flang/examples/FeatureList/FeatureList.cpp
@@ -456,8 +456,6 @@ struct NodeVisitor {
   READ_FEATURE(OmpBeginLoopDirective)
   READ_FEATURE(OmpBeginSectionsDirective)
   READ_FEATURE(OmpBlockDirective)
-  READ_FEATURE(OmpCancelType)
-  READ_FEATURE(OmpCancelType::Type)
   READ_FEATURE(OmpClause)
   READ_FEATURE(OmpClauseList)
   READ_FEATURE(OmpCriticalDirective)
@@ -551,7 +549,6 @@ struct NodeVisitor {
   READ_FEATURE(OpenMPAtomicConstruct)
   READ_FEATURE(OpenMPBlockConstruct)
   READ_FEATURE(OpenMPCancelConstruct)
-  READ_FEATURE(OpenMPCancelConstruct::If)
   READ_FEATURE(OpenMPCancellationPointConstruct)
   READ_FEATURE(OpenMPConstruct)
   READ_FEATURE(OpenMPCriticalConstruct)
diff --git a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
index 5f351fb97e5aa..589b3ecf8779d 100644
--- a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
+++ b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
@@ -265,9 +265,6 @@ void OpenMPCounterVisitor::Post(const OmpDirectiveNameModifier &c) {
   clauseDetails +=
       "name_modifier=" + llvm::omp::getOpenMPDirectiveName(c.v).str() + ";";
 }
-void OpenMPCounterVisitor::Post(const OmpCancelType::Type &c) {
-  clauseDetails += "type=" + std::string{OmpCancelType::EnumToString(c)} + ";";
-}
 void OpenMPCounterVisitor::Post(const OmpClause &c) {
   PostClauseCommon(normalize_clause_name(c.source.ToString()));
   clauseDetails.clear();
diff --git a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.h b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.h
index 59bdac8594cb7..4257e71963d7d 100644
--- a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.h
+++ b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.h
@@ -78,7 +78,6 @@ struct OpenMPCounterVisitor {
   void Post(const OmpMapType::Value &c);
   void Post(const OmpScheduleClause::Kind &c);
   void Post(const OmpDirectiveNameModifier &c);
-  void Post(const OmpCancelType::Type &c);
   void Post(const OmpClause &c);
   void PostClauseCommon(const ClauseInfo &ci);
 
diff --git a/flang/include/flang/Evaluate/tools.h b/flang/include/flang/Evaluate/tools.h
index 050990d1cd7ed..1414eaf14f7d6 100644
--- a/flang/include/flang/Evaluate/tools.h
+++ b/flang/include/flang/Evaluate/tools.h
@@ -464,9 +464,13 @@ struct ExtractCoindexedObjectHelper {
   }
 };
 
+static inline std::optional<CoarrayRef> ExtractCoarrayRef(const DataRef &x) {
+  return ExtractCoindexedObjectHelper{}(x);
+}
+
 template <typename A> std::optional<CoarrayRef> ExtractCoarrayRef(const A &x) {
   if (auto dataRef{ExtractDataRef(x, true)}) {
-    return ExtractCoindexedObjectHelper{}(*dataRef);
+    return ExtractCoarrayRef(*dataRef);
   } else {
     return ExtractCoindexedObjectHelper{}(x);
   }
diff --git a/flang/include/flang/Frontend/CompilerInstance.h b/flang/include/flang/Frontend/CompilerInstance.h
index 509c9f4b9e91a..e37ef5e236871 100644
--- a/flang/include/flang/Frontend/CompilerInstance.h
+++ b/flang/include/flang/Frontend/CompilerInstance.h
@@ -15,9 +15,8 @@
 
 #include "flang/Frontend/CompilerInvocation.h"
 #include "flang/Frontend/FrontendAction.h"
+#include "flang/Frontend/ParserActions.h"
 #include "flang/Frontend/PreprocessorOptions.h"
-#include "flang/Parser/parsing.h"
-#include "flang/Parser/provenance.h"
 #include "flang/Semantics/runtime-type-info.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Support/StringOstream.h"
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index 9e6724be33033..d6ee1511cdb4b 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -18,7 +18,7 @@
 #include "flang/Frontend/PreprocessorOptions.h"
 #include "flang/Frontend/TargetOptions.h"
 #include "flang/Lower/LoweringOptions.h"
-#include "flang/Parser/parsing.h"
+#include "flang/Parser/options.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Support/LangOptions.h"
 #include "mlir/Support/Timing.h"
@@ -29,7 +29,7 @@
 
 namespace llvm {
 class TargetMachine;
-}
+} // namespace llvm
 
 namespace Fortran::frontend {
 
diff --git a/flang/include/flang/Frontend/FrontendActions.h b/flang/include/flang/Frontend/FrontendActions.h
index 4e3d3cb2657db..f9a45bd6c0a56 100644
--- a/flang/include/flang/Frontend/FrontendActions.h
+++ b/flang/include/flang/Frontend/FrontendActions.h
@@ -13,10 +13,8 @@
 #ifndef FORTRAN_FRONTEND_FRONTENDACTIONS_H
 #define FORTRAN_FRONTEND_FRONTENDACTIONS_H
 
-#include "flang/Frontend/CodeGenOptions.h"
 #include "flang/Frontend/FrontendAction.h"
-#include "flang/Parser/parsing.h"
-#include "flang/Semantics/semantics.h"
+#include "flang/Frontend/ParserActions.h"
 
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/OwningOpRef.h"
@@ -26,21 +24,6 @@
 
 namespace Fortran::frontend {
 
-// TODO: This is a copy from f18.cpp. It doesn't really belong here and should
-// be moved to a more suitable place in future.
-struct MeasurementVisitor {
-  template <typename A>
-  bool Pre(const A &) {
-    return true;
-  }
-  template <typename A>
-  void Post(const A &) {
-    ++objects;
-    bytes += sizeof(A);
-  }
-  size_t objects{0}, bytes{0};
-};
-
 //===----------------------------------------------------------------------===//
 // Custom Consumer Actions
 //===----------------------------------------------------------------------===//
diff --git a/flang/include/flang/Frontend/ParserActions.h b/flang/include/flang/Frontend/ParserActions.h
new file mode 100644
index 0000000000000..8e7d1f71ec105
--- /dev/null
+++ b/flang/include/flang/Frontend/ParserActions.h
@@ -0,0 +1,67 @@
+//===- ParserActions.h -------------------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_PARSER_ACTIONS_H_
+#define FORTRAN_PARSER_ACTIONS_H_
+
+#include <string>
+
+namespace llvm {
+class raw_string_ostream;
+class raw_ostream;
+class StringRef;
+} // namespace llvm
+
+namespace Fortran::lower {
+class LoweringBridge;
+} // namespace Fortran::lower
+
+namespace Fortran::parser {
+class Parsing;
+class AllCookedSources;
+} // namespace Fortran::parser
+
+namespace lower::pft {
+class Program;
+} // namespace lower::pft
+
+//=== Frontend Parser helpers ===
+
+namespace Fortran::frontend {
+class CompilerInstance;
+
+parser::AllCookedSources &getAllCooked(CompilerInstance &ci);
+
+void parseAndLowerTree(CompilerInstance &ci, lower::LoweringBridge &lb);
+
+void dumpTree(CompilerInstance &ci);
+
+void dumpProvenance(CompilerInstance &ci);
+
+void dumpPreFIRTree(CompilerInstance &ci);
+
+void formatOrDumpPrescanner(std::string &buf,
+                            llvm::raw_string_ostream &outForPP,
+                            CompilerInstance &ci);
+
+void debugMeasureParseTree(CompilerInstance &ci, llvm::StringRef filename);
+
+void debugUnparseNoSema(CompilerInstance &ci, llvm::raw_ostream &out);
+
+void debugUnparseWithSymbols(CompilerInstance &ci);
+
+void debugUnparseWithModules(CompilerInstance &ci);
+
+void debugDumpParsingLog(CompilerInstance &ci);
+} // namespace Fortran::frontend
+
+#endif // FORTRAN_PARSER_ACTIONS_H_
diff --git a/flang/include/flang/Lower/Cuda.h b/flang/include/flang/Lower/Cuda.h
index d97045383d195..b6f849e3d63f0 100644
--- a/flang/include/flang/Lower/Cuda.h
+++ b/flang/include/flang/Lower/Cuda.h
@@ -20,27 +20,6 @@
 #include "mlir/Dialect/OpenACC/OpenACC.h"
 
 namespace Fortran::lower {
-// Check if the insertion point is currently in a device context. HostDevice
-// subprogram are not considered fully device context so it will return false
-// for it.
-// If the insertion point is inside an OpenACC region op, it is considered
-// device context.
-static bool inline isCudaDeviceContext(fir::FirOpBuilder &builder) {
-  if (builder.getRegion().getParentOfType<cuf::KernelOp>())
-    return true;
-  if (builder.getRegion()
-          .getParentOfType<mlir::acc::ComputeRegionOpInterface>())
-    return true;
-  if (auto funcOp = builder.getRegion().getParentOfType<mlir::func::FuncOp>()) {
-    if (auto cudaProcAttr =
-            funcOp.getOperation()->getAttrOfType<cuf::ProcAttributeAttr>(
-                cuf::getProcAttrName())) {
-      return cudaProcAttr.getValue() != cuf::ProcAttribute::Host &&
-             cudaProcAttr.getValue() != cuf::ProcAttribute::HostDevice;
-    }
-  }
-  return false;
-}
 
 static inline unsigned getAllocatorIdx(const Fortran::semantics::Symbol &sym) {
   std::optional<Fortran::common::CUDADataAttr> cudaAttr =
diff --git a/flang/include/flang/Optimizer/Builder/CUFCommon.h b/flang/include/flang/Optimizer/Builder/CUFCommon.h
index b99e330429622..e3c7b5098b83f 100644
--- a/flang/include/flang/Optimizer/Builder/CUFCommon.h
+++ b/flang/include/flang/Optimizer/Builder/CUFCommon.h
@@ -25,8 +25,10 @@ namespace cuf {
 mlir::gpu::GPUModuleOp getOrCreateGPUModule(mlir::ModuleOp mod,
                                             mlir::SymbolTable &symTab);
 
-bool isInCUDADeviceContext(mlir::Operation *op);
+bool isCUDADeviceContext(mlir::Operation *op);
+bool isCUDADeviceContext(mlir::Region &);
 bool isRegisteredDeviceGlobal(fir::GlobalOp op);
+bool isRegisteredDeviceAttr(std::optional<cuf::DataAttribute> attr);
 
 void genPointerSync(const mlir::Value box, fir::FirOpBuilder &builder);
 
diff --git a/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index 19fc2c22f0d49..ac80873dc374f 100644
--- a/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -60,7 +60,7 @@ class Entity : public mlir::Value {
   bool isVariable() const { return !isValue(); }
   bool isMutableBox() const { return hlfir::isBoxAddressType(getType()); }
   bool isProcedurePointer() const {
-    return fir::isBoxProcAddressType(getType());
+    return hlfir::isFortranProcedurePointerType(getType());
   }
   bool isBoxAddressOrValue() const {
     return hlfir::isBoxAddressOrValueType(getType());
diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index c82e5265970c5..589a936f8b8c7 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -187,15 +187,18 @@ struct IntrinsicLibrary {
   mlir::Value genAtanpi(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genAtomicAdd(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genAtomicAnd(mlir::Type, llvm::ArrayRef<mlir::Value>);
-  mlir::Value genAtomicCas(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  fir::ExtendedValue genAtomicCas(mlir::Type,
+                                  llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genAtomicDec(mlir::Type, llvm::ArrayRef<mlir::Value>);
-  mlir::Value genAtomicExch(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  fir::ExtendedValue genAtomicExch(mlir::Type,
+                                   llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genAtomicInc(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genAtomicMax(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genAtomicMin(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genAtomicOr(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genAtomicSub(mlir::Type, llvm::ArrayRef<mlir::Value>);
-  mlir::Value genAtomicXor(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  fir::ExtendedValue genAtomicXor(mlir::Type,
+                                  llvm::ArrayRef<fir::ExtendedValue>);
   fir::ExtendedValue
       genCommandArgumentCount(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genAsind(mlir::Type, llvm::ArrayRef<mlir::Value>);
@@ -223,6 +226,9 @@ struct IntrinsicLibrary {
   fir::ExtendedValue genCount(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
   void genCpuTime(llvm::ArrayRef<fir::ExtendedValue>);
   fir::ExtendedValue genCshift(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
+  template <const char *fctName, int extent>
+  fir::ExtendedValue genCUDALDXXFunc(mlir::Type,
+                                     llvm::ArrayRef<fir::ExtendedValue>);
   fir::ExtendedValue genCAssociatedCFunPtr(mlir::Type,
                                            llvm::ArrayRef<fir::ExtendedValue>);
   fir::ExtendedValue genCAssociatedCPtr(mlir::Type,
diff --git a/flang/include/flang/Optimizer/Builder/Runtime/CUDA/Descriptor.h b/flang/include/flang/Optimizer/Builder/Runtime/CUDA/Descriptor.h
new file mode 100644
index 0000000000000..14d262bf22a70
--- /dev/null
+++ b/flang/include/flang/Optimizer/Builder/Runtime/CUDA/Descriptor.h
@@ -0,0 +1,31 @@
+//===-- Descriptor.h - CUDA descritpor runtime API calls --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_OPTIMIZER_BUILDER_RUNTIME_CUDA_DESCRIPTOR_H_
+#define FORTRAN_OPTIMIZER_BUILDER_RUNTIME_CUDA_DESCRIPTOR_H_
+
+#include "mlir/IR/Value.h"
+
+namespace mlir {
+class Location;
+} // namespace mlir
+
+namespace fir {
+class FirOpBuilder;
+}
+
+namespace fir::runtime::cuda {
+
+/// Generate runtime call to sync the doublce descriptor referenced by
+/// \p hostPtr.
+void genSyncGlobalDescriptor(fir::FirOpBuilder &builder, mlir::Location loc,
+                             mlir::Value hostPtr);
+
+} // namespace fir::runtime::cuda
+
+#endif // FORTRAN_OPTIMIZER_BUILDER_RUNTIME_CUDA_DESCRIPTOR_H_
diff --git a/flang/include/flang/Optimizer/Builder/Runtime/RTBuilder.h b/flang/include/flang/Optimizer/Builder/Runtime/RTBuilder.h
index 5158abaa31ed1..722e9191be728 100644
--- a/flang/include/flang/Optimizer/Builder/Runtime/RTBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/Runtime/RTBuilder.h
@@ -30,7 +30,7 @@
 #include <cstdint>
 #include <functional>
 
-#ifdef _WIN32
+#ifdef _MSC_VER
 // On Windows* OS GetCurrentProcessId returns DWORD aka uint32_t
 typedef std::uint32_t pid_t;
 #endif
diff --git a/flang/include/flang/Optimizer/Builder/TemporaryStorage.h b/flang/include/flang/Optimizer/Builder/TemporaryStorage.h
index b17a75354e7d1..cdb23a64c5c8a 100644
--- a/flang/include/flang/Optimizer/Builder/TemporaryStorage.h
+++ b/flang/include/flang/Optimizer/Builder/TemporaryStorage.h
@@ -180,7 +180,7 @@ class AnyValueStack {
 /// dynamic type, bounds, and type parameters as the Nth variable that was
 /// pushed. It is implemented using runtime.
 /// Note that this is not meant to save POINTER or ALLOCATABLE descriptor
-/// addresses, use AnyDescriptorAddressStack instead.
+/// addresses, use AnyAddressStack instead.
 class AnyVariableStack {
 public:
   AnyVariableStack(mlir::Location loc, fir::FirOpBuilder &builder,
@@ -205,19 +205,21 @@ class AnyVariableStack {
   mlir::Value retValueBox;
 };
 
-/// Data structure to stack descriptor addresses. It stores the descriptor
-/// addresses as int_ptr values under the hood.
-class AnyDescriptorAddressStack : public AnyValueStack {
+/// Data structure to stack simple addresses (C pointers). It can be used to
+/// store data base addresses, descriptor addresses, procedure addresses, and
+/// pointer procedure address. It stores the addresses as int_ptr values under
+/// the hood.
+class AnyAddressStack : public AnyValueStack {
 public:
-  AnyDescriptorAddressStack(mlir::Location loc, fir::FirOpBuilder &builder,
-                            mlir::Type descriptorAddressType);
+  AnyAddressStack(mlir::Location loc, fir::FirOpBuilder &builder,
+                  mlir::Type addressType);
 
   void pushValue(mlir::Location loc, fir::FirOpBuilder &builder,
                  mlir::Value value);
   mlir::Value fetch(mlir::Location loc, fir::FirOpBuilder &builder);
 
 private:
-  mlir::Type descriptorAddressType;
+  mlir::Type addressType;
 };
 
 class TemporaryStorage;
@@ -281,8 +283,7 @@ class TemporaryStorage {
 
 private:
   std::variant<HomogeneousScalarStack, SimpleCopy, SSARegister, AnyValueStack,
-               AnyVariableStack, AnyVectorSubscriptStack,
-               AnyDescriptorAddressStack>
+               AnyVariableStack, AnyVectorSubscriptStack, AnyAddressStack>
       impl;
 };
 } // namespace fir::factory
diff --git a/flang/include/flang/Optimizer/Dialect/FIRAttr.td b/flang/include/flang/Optimizer/Dialect/FIRAttr.td
index e3474da6685af..8e86d82f38df4 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRAttr.td
+++ b/flang/include/flang/Optimizer/Dialect/FIRAttr.td
@@ -156,4 +156,26 @@ def fir_LocationKindAttr : EnumAttr<FIROpsDialect, fir_LocationKind, "loc_kind">
 def LocationKindArrayAttr : ArrayOfAttr<FIROpsDialect, "LocationKindArray",
     "loc_kind_array", "LocationKindAttr">;
 
+/// Optimization heuristics for fir.pack_array operation.
+def fir_PackArrayHeuristics
+    : I32BitEnumAttr<"PackArrayHeuristics", "",
+                     [
+                         /// fir.pack_array cannot be optimized based on the
+                         /// array usage pattern.
+                         I32BitEnumAttrCaseNone<"None", "none">,
+                         /// fir.pack_array can be optimized away, if the array
+                         /// is not used in a loop.
+                         I32BitEnumAttrCaseBit<"LoopOnly", 0, "loop_only">,
+]> {
+  let separator = ", ";
+  let cppNamespace = "::fir";
+  let genSpecializedAttr = 0;
+}
+
+def fir_PackArrayHeuristicsAttr
+    : EnumAttr<FIROpsDialect, fir_PackArrayHeuristics,
+               "pack_array_heuristics"> {
+  let assemblyFormat = "`<` $value `>`";
+}
+
 #endif // FIR_DIALECT_FIR_ATTRS
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index c83c57186b46d..7147a2401baa7 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -68,6 +68,12 @@ def IsBoxAddressOrValueTypePred
 def fir_BoxAddressOrValueType : Type<IsBoxAddressOrValueTypePred,
     "fir.box or fir.class type or reference">;
 
+def RefOfConstantSizeAggregateTypePred
+        : CPred<"::fir::isRefOfConstantSizeAggregateType($_self)">;
+def AnyRefOfConstantSizeAggregateType : TypeConstraint<
+      RefOfConstantSizeAggregateTypePred,
+      "a reference type to a constant size fir.array, fir.char, or fir.type">;
+
 //===----------------------------------------------------------------------===//
 // Memory SSA operations
 //===----------------------------------------------------------------------===//
@@ -342,6 +348,44 @@ def fir_StoreOp : fir_Op<"store", [FirAliasTagOpInterface]> {
   }];
 }
 
+def fir_CopyOp : fir_Op<"copy", []> {
+  let summary = "copy constant size memory";
+
+  let description = [{
+    Copy the memory from a source with compile time constant size to
+    a destination of the same type.
+
+    This is meant to be used for aggregate types where load and store
+    are not appropriate to make a copy because LLVM is not meant to
+    handle load and store of "big" aggregates.
+
+    Its "no_overlap" attribute allows indicating that the source and destination
+    are known to not overlap at compile time.
+
+    ```
+      !t =!fir.type<t{x:!fir.array<1000xi32>}>
+      fir.copy %x to %y : !fir.ref<!t>, !fir.ref<!t>
+    ```
+    TODO: add FirAliasTagOpInterface to carry TBAA.
+  }];
+
+  let arguments = (ins Arg<AnyRefOfConstantSizeAggregateType, "", [MemRead]>:$source,
+                       Arg<AnyRefOfConstantSizeAggregateType, "", [MemWrite]>:$destination,
+                       OptionalAttr<UnitAttr>:$no_overlap);
+
+  let builders = [OpBuilder<(ins "mlir::Value":$source,
+                                 "mlir::Value":$destination,
+                                  CArg<"bool", "false">:$no_overlap)>];
+
+  let assemblyFormat = [{
+    $source `to` $destination (`no_overlap` $no_overlap^)?
+    attr-dict `:` type(operands)
+  }];
+
+  let hasVerifier = 1;
+}
+
+
 def fir_SaveResultOp : fir_Op<"save_result", [AttrSizedOperandSegments]> {
   let summary = [{
     save an array, box, or record function result SSA-value to a memory location
@@ -3276,4 +3320,100 @@ def fir_DummyScopeOp : fir_Op<"dummy_scope",
   let assemblyFormat = "attr-dict `:` type(results)";
 }
 
+def fir_PackArrayOp
+    : fir_Op<"pack_array", [DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
+                            AllTypesMatch<["array", "result"]>]> {
+  let summary = "Pack non-contiguous array into a temporary";
+
+  let description = [{
+    The operation creates a new !fir.box/class<!fir.array<>> value
+    to represent either the original array or a newly allocated
+    temporary array, maybe identical to the original array by value.
+
+    Arguments:
+      - array is the original array.
+        It must have !fir.box/class<!fir.array<>> type.
+      - stack/heap attribute indicates where the temporary array
+        needs to be allocated.
+      - innermost/whole attribute identifies the contiguity mode.
+        innermost means that the repacking has to be done iff the original
+        array is not contiguous in the leading dimension.
+        whole means that the repacking has to be done iff the original
+        array is not contiguous in any dimension.
+        innermost is disallowed for 1D arrays in favor of whole.
+      - no_copy attribute indicates that the original array
+        is not copied into the temporary.
+      - typeparams specify the length parameters of the original array.
+        Even though the array is fully represented with a box, the explicit
+        length parameters might be specified to simplify computing
+        the size of the array's element in compilation time (e.g. constant
+        length parameters might be propagated after MLIR inlining).
+      - optional constraints attributes:
+        * max_size is an unsigned integer attribute specifying the maximum
+          byte size of an array that is eligible for repacking.
+        * max_element_size is an unsigned integer attribute specifying
+          the maximum byte element-size of an array that is eligible
+          for repacking.
+        * min_stride is an unsigned integer attribute specifying
+          the minimum byte stride of the innermost dimension of an array
+          that is eligible for repacking.
+      - heuristics attribute specifies conditions when the array repacking
+        may be optimized.
+  }];
+
+  let arguments = (ins AnyBoxedArray:$array, UnitAttr:$stack,
+      UnitAttr:$innermost, UnitAttr:$no_copy, OptionalAttr<UI64Attr>:$max_size,
+      OptionalAttr<UI64Attr>:$max_element_size,
+      OptionalAttr<UI64Attr>:$min_stride,
+      DefaultValuedAttr<fir_PackArrayHeuristicsAttr,
+                        "::fir::PackArrayHeuristics::None">:$heuristics,
+      Variadic<AnyIntegerType>:$typeparams);
+
+  let results = (outs AnyBoxedArray:$result);
+  let assemblyFormat = [{
+    $array (`stack` $stack^):(`heap`)?
+    (`innermost` $innermost^):(`whole`)?
+    (`no_copy` $no_copy^)?
+    (`constraints` custom<PackArrayConstraints>($max_size, $max_element_size, $min_stride)^)?
+    (`heuristics` $heuristics^)?
+    (`typeparams` $typeparams^)?
+    attr-dict `:` functional-type(operands, results)
+  }];
+
+  let hasVerifier = 1;
+}
+
+def fir_UnpackArrayOp
+    : fir_Op<"unpack_array", [SameTypeOperands,
+                              DeclareOpInterfaceMethods<
+                                  MemoryEffectsOpInterface>]> {
+  let summary = "Unpack values from temporary array into original array";
+
+  let description = [{
+    The operation is either a no-op or deallocates the temporary array,
+    and maybe copies the temporary array into the original array.
+
+    Arguments:
+      - temp is a fir.box/fir.class value produced by fir.pack_array.
+        It describes either the original array or the temporary array.
+      - original is the original array descriptor.
+      - stack/heap attribute indicates where the temporary array
+        was allocated.
+      - no_copy attribute indicates that the temporary array
+        is not copied into the original temporary array.
+  }];
+
+  let arguments = (ins AnyBoxedArray:$temp, AnyBoxedArray:$original,
+      UnitAttr:$stack, UnitAttr:$no_copy);
+
+  let assemblyFormat = [{
+    $temp `to` $original
+    (`stack` $stack^):(`heap`)?
+    (`no_copy` $no_copy^)?
+    attr-dict `:` type($original)
+  }];
+
+  let hasVerifier = 1;
+}
+
 #endif
diff --git a/flang/include/flang/Optimizer/Dialect/FIRType.h b/flang/include/flang/Optimizer/Dialect/FIRType.h
index 3d30f4e673682..76e0aa352bcd9 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRType.h
+++ b/flang/include/flang/Optimizer/Dialect/FIRType.h
@@ -498,6 +498,13 @@ inline bool isBoxProcAddressType(mlir::Type t) {
   return t && mlir::isa<fir::BoxProcType>(t);
 }
 
+inline bool isRefOfConstantSizeAggregateType(mlir::Type t) {
+  t = fir::dyn_cast_ptrEleTy(t);
+  return t &&
+         mlir::isa<fir::CharacterType, fir::RecordType, fir::SequenceType>(t) &&
+         !hasDynamicSize(t);
+}
+
 /// Return a string representation of `ty`.
 ///
 /// fir.array<10x10xf32> -> prefix_10x10xf32
diff --git a/flang/include/flang/Optimizer/Dialect/FIRTypes.td b/flang/include/flang/Optimizer/Dialect/FIRTypes.td
index 41e765c1cb7b9..fd5bbbe44751f 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRTypes.td
+++ b/flang/include/flang/Optimizer/Dialect/FIRTypes.td
@@ -658,5 +658,12 @@ def ArrayOrBoxOrRecord : TypeConstraint<Or<[fir_SequenceType.predicate,
     IsBaseBoxTypePred, fir_RecordType.predicate]>,
     "fir.box, fir.array or fir.type">;
 
+// Returns true iff the type is an array box or a reference to such type.
+def IsArrayBoxPred : CPred<"::fir::getBoxRank($_self) != 0">;
+
+// Any boxed array type (not a reference to a boxed array type).
+def AnyBoxedArray
+    : TypeConstraint<And<[BoxOrClassType.predicate, IsArrayBoxPred]>,
+                     "any boxed array">;
 
 #endif // FIR_DIALECT_FIR_TYPES
diff --git a/flang/include/flang/Optimizer/HLFIR/HLFIRDialect.h b/flang/include/flang/Optimizer/HLFIR/HLFIRDialect.h
index 15296aa7e8c75..5152dee14ad65 100644
--- a/flang/include/flang/Optimizer/HLFIR/HLFIRDialect.h
+++ b/flang/include/flang/Optimizer/HLFIR/HLFIRDialect.h
@@ -82,6 +82,17 @@ inline bool isPolymorphicType(mlir::Type type) {
   return fir::isPolymorphicType(type);
 }
 
+/// Is this the FIR type of a Fortran procedure pointer?
+inline bool isFortranProcedurePointerType(mlir::Type type) {
+  return fir::isBoxProcAddressType(type);
+}
+
+inline bool isFortranPointerObjectType(mlir::Type type) {
+  auto boxTy =
+      llvm::dyn_cast_or_null<fir::BaseBoxType>(fir::dyn_cast_ptrEleTy(type));
+  return boxTy && boxTy.isPointer();
+}
+
 /// Is this an SSA value type for the value of a Fortran procedure
 /// designator ?
 inline bool isFortranProcedureValue(mlir::Type type) {
diff --git a/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td b/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td
index 1b1ac61d4550f..ee0b5aa9760b1 100644
--- a/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td
+++ b/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td
@@ -91,10 +91,9 @@ def IsFortranVariablePred
 def AnyFortranVariable : Type<IsFortranVariablePred, "any HLFIR variable type">;
 
 
-def AnyFortranValue : TypeConstraint<Or<[AnyLogicalLike.predicate,
-    AnyIntegerLike.predicate, AnyRealLike.predicate,
-    AnyFirComplexLike.predicate,
-    hlfir_ExprType.predicate]>, "any Fortran value type">;
+def IsFortranValuePred : CPred<"::hlfir::isFortranValueType($_self)">;
+def AnyFortranValue
+        : TypeConstraint<IsFortranValuePred, "any Fortran value type">;
 
 
 def AnyFortranEntity : TypeConstraint<Or<[AnyFortranVariable.predicate,
diff --git a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
index c12066b1346f6..f69930d5b53b3 100644
--- a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
+++ b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
@@ -1378,6 +1378,8 @@ def hlfir_RegionAssignOp : hlfir_Op<"region_assign", [hlfir_OrderedAssignmentTre
     }
     mlir::Region* getSubTreeRegion() { return nullptr; }
     bool isPointerAssignment();
+    bool isPointerObjectAssignment();
+    bool isProcedurePointerAssignment();
   }];
 
   let hasCustomAssemblyFormat = 1;
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 3b3c6bdc448d7..ed3f5cdcc1ace 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -489,7 +489,9 @@ class ParseTreeDumper {
   NODE(parser, OmpOtherwiseClause)
   NODE(parser, OmpWhenClause)
   NODE(OmpWhenClause, Modifier)
+  NODE(parser, OmpDirectiveName)
   NODE(parser, OmpDirectiveSpecification)
+  NODE_ENUM(OmpDirectiveSpecification, Flags)
   NODE(parser, OmpTraitPropertyName)
   NODE(parser, OmpTraitScore)
   NODE(parser, OmpTraitPropertyExtension)
@@ -538,12 +540,11 @@ class ParseTreeDumper {
         "llvm::omp::Directive = ", llvm::omp::getOpenMPDirectiveName(x))
         .str();
   }
-  NODE(parser, OmpCancelType)
-  NODE_ENUM(OmpCancelType, Type)
   NODE(parser, OmpClause)
 #define GEN_FLANG_DUMP_PARSE_TREE_CLAUSES
 #include "llvm/Frontend/OpenMP/OMP.inc"
   NODE(parser, OmpClauseList)
+  NODE(parser, OmpCancellationConstructTypeClause)
   NODE(parser, OmpContainsClause)
   NODE(parser, OmpCriticalDirective)
   NODE(parser, OmpErrorDirective)
@@ -586,7 +587,6 @@ class ParseTreeDumper {
   NODE(OmpFromClause, Modifier)
   NODE(parser, OmpExpectation)
   NODE_ENUM(OmpExpectation, Value)
-  NODE(parser, OmpDirectiveNameModifier)
   NODE(parser, OmpHoldsClause)
   NODE(parser, OmpIfClause)
   NODE(OmpIfClause, Modifier)
@@ -687,7 +687,6 @@ class ParseTreeDumper {
   NODE(parser, OpenMPAtomicConstruct)
   NODE(parser, OpenMPBlockConstruct)
   NODE(parser, OpenMPCancelConstruct)
-  NODE(OpenMPCancelConstruct, If)
   NODE(parser, OpenMPCancellationPointConstruct)
   NODE(parser, OpenMPConstruct)
   NODE(parser, OpenMPCriticalConstruct)
diff --git a/flang/include/flang/Parser/options.h b/flang/include/flang/Parser/options.h
new file mode 100644
index 0000000000000..dc363acb7c28a
--- /dev/null
+++ b/flang/include/flang/Parser/options.h
@@ -0,0 +1,43 @@
+//===-- include/flang/Parser/options.h --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_PARSER_OPTIONS_H_
+#define FORTRAN_PARSER_OPTIONS_H_
+
+#include "characters.h"
+#include "flang/Support/Fortran-features.h"
+
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace Fortran::parser {
+
+struct Options {
+  Options() {}
+
+  using Predefinition = std::pair<std::string, std::optional<std::string>>;
+
+  bool isFixedForm{false};
+  int fixedFormColumns{72};
+  common::LanguageFeatureControl features;
+  std::vector<std::string> searchDirectories;
+  std::vector<std::string> intrinsicModuleDirectories;
+  std::vector<Predefinition> predefinitions;
+  bool instrumentedParse{false};
+  bool isModuleFile{false};
+  bool needProvenanceRangeToCharBlockMappings{false};
+  Fortran::parser::Encoding encoding{Fortran::parser::Encoding::UTF_8};
+  bool prescanAndReformat{false}; // -E
+  bool expandIncludeLinesInPreprocessedOutput{true};
+  bool showColors{false};
+};
+
+} // namespace Fortran::parser
+
+#endif // FORTRAN_PARSER_OPTIONS_H_
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index f11859bb09ddb..2b62866cc7712 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3464,6 +3464,19 @@ WRAPPER_CLASS(PauseStmt, std::optional<StopCode>);
 struct OmpClause;
 struct OmpDirectiveSpecification;
 
+struct OmpDirectiveName {
+  // No boilerplates: this class should be copyable, movable, etc.
+  constexpr OmpDirectiveName() = default;
+  constexpr OmpDirectiveName(const OmpDirectiveName &) = default;
+  // Construct from an already parsed text. Use Verbatim for this because
+  // Verbatim's source corresponds to an actual source location.
+  // This allows "construct<OmpDirectiveName>(Verbatim("<name>"))".
+  OmpDirectiveName(const Verbatim &name);
+  using WrapperTrait = std::true_type;
+  CharBlock source;
+  llvm::omp::Directive v{llvm::omp::Directive::OMPD_unknown};
+};
+
 // 2.1 Directives or clauses may accept a list or extended-list.
 //     A list item is a variable, array section or common block name (enclosed
 //     in slashes). An extended list item is a list item or a procedure Name.
@@ -3794,9 +3807,7 @@ struct OmpDeviceModifier {
 // [*] The IF clause is allowed on CANCEL in OpenMP 4.5, but only without
 // the directive-name-modifier. For the sake of uniformity CANCEL can be
 // considered a valid value in 4.5 as well.
-struct OmpDirectiveNameModifier {
-  WRAPPER_CLASS_BOILERPLATE(OmpDirectiveNameModifier, llvm::omp::Directive);
-};
+using OmpDirectiveNameModifier = OmpDirectiveName;
 
 // Ref: [5.1:205-209], [5.2:166-168]
 //
@@ -4036,6 +4047,12 @@ struct OmpBindClause {
   WRAPPER_CLASS_BOILERPLATE(OmpBindClause, Binding);
 };
 
+// Artificial clause to represent a cancellable construct.
+struct OmpCancellationConstructTypeClause {
+  TUPLE_CLASS_BOILERPLATE(OmpCancellationConstructTypeClause);
+  std::tuple<OmpDirectiveName, std::optional<ScalarLogicalExpr>> t;
+};
+
 // Ref: [5.2:214]
 //
 // contains-clause ->
@@ -4491,10 +4508,16 @@ struct OmpClauseList {
 // --- Directives and constructs
 
 struct OmpDirectiveSpecification {
-  CharBlock source;
+  ENUM_CLASS(Flags, None, DeprecatedSyntax);
   TUPLE_CLASS_BOILERPLATE(OmpDirectiveSpecification);
-  std::tuple<llvm::omp::Directive, std::optional<std::list<OmpArgument>>,
-      std::optional<OmpClauseList>>
+  llvm::omp::Directive DirId() const { //
+    return std::get<OmpDirectiveName>(t).v;
+  }
+  const OmpClauseList &Clauses() const;
+
+  CharBlock source;
+  std::tuple<OmpDirectiveName, std::optional<std::list<OmpArgument>>,
+      std::optional<OmpClauseList>, Flags>
       t;
 };
 
@@ -4842,26 +4865,18 @@ struct OmpLoopDirective {
   CharBlock source;
 };
 
-// 2.14.1 construct-type-clause -> PARALLEL | SECTIONS | DO | TASKGROUP
-struct OmpCancelType {
-  ENUM_CLASS(Type, Parallel, Sections, Do, Taskgroup)
-  WRAPPER_CLASS_BOILERPLATE(OmpCancelType, Type);
-  CharBlock source;
-};
-
 // 2.14.2 cancellation-point -> CANCELLATION POINT construct-type-clause
 struct OpenMPCancellationPointConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPCancellationPointConstruct);
   CharBlock source;
-  std::tuple<Verbatim, OmpCancelType> t;
+  std::tuple<Verbatim, OmpClauseList> t;
 };
 
 // 2.14.1 cancel -> CANCEL construct-type-clause [ [,] if-clause]
 struct OpenMPCancelConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPCancelConstruct);
-  WRAPPER_CLASS(If, ScalarLogicalExpr);
   CharBlock source;
-  std::tuple<Verbatim, OmpCancelType, std::optional<If>> t;
+  std::tuple<Verbatim, OmpClauseList> t;
 };
 
 // Ref: [5.0:254-255], [5.1:287-288], [5.2:322-323]
diff --git a/flang/include/flang/Parser/parsing.h b/flang/include/flang/Parser/parsing.h
index 116b6bd6f191f..3365628dc4e0c 100644
--- a/flang/include/flang/Parser/parsing.h
+++ b/flang/include/flang/Parser/parsing.h
@@ -9,41 +9,18 @@
 #ifndef FORTRAN_PARSER_PARSING_H_
 #define FORTRAN_PARSER_PARSING_H_
 
-#include "characters.h"
 #include "instrumented-parser.h"
 #include "message.h"
+#include "options.h"
 #include "parse-tree.h"
 #include "provenance.h"
 #include "flang/Parser/preprocessor.h"
-#include "flang/Support/Fortran-features.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
 #include <string>
-#include <utility>
-#include <vector>
 
 namespace Fortran::parser {
 
-struct Options {
-  Options() {}
-
-  using Predefinition = std::pair<std::string, std::optional<std::string>>;
-
-  bool isFixedForm{false};
-  int fixedFormColumns{72};
-  common::LanguageFeatureControl features;
-  std::vector<std::string> searchDirectories;
-  std::vector<std::string> intrinsicModuleDirectories;
-  std::vector<Predefinition> predefinitions;
-  bool instrumentedParse{false};
-  bool isModuleFile{false};
-  bool needProvenanceRangeToCharBlockMappings{false};
-  Fortran::parser::Encoding encoding{Fortran::parser::Encoding::UTF_8};
-  bool prescanAndReformat{false}; // -E
-  bool expandIncludeLinesInPreprocessedOutput{true};
-  bool showColors{false};
-};
-
 class Parsing {
 public:
   explicit Parsing(AllCookedSources &);
diff --git a/flang/include/flang/Parser/preprocessor.h b/flang/include/flang/Parser/preprocessor.h
index f8d3346065981..86528a7e68def 100644
--- a/flang/include/flang/Parser/preprocessor.h
+++ b/flang/include/flang/Parser/preprocessor.h
@@ -81,6 +81,7 @@ class Preprocessor {
   void Define(const std::string &macro, const std::string &value);
   void Undefine(std::string macro);
   bool IsNameDefined(const CharBlock &);
+  bool IsNameDefinedEmpty(const CharBlock &);
   bool IsFunctionLikeDefinition(const CharBlock &);
   bool AnyDefinitions() const { return !definitions_.empty(); }
   bool InConditional() const { return !ifStack_.empty(); }
diff --git a/flang/include/flang/Runtime/command.h b/flang/include/flang/Runtime/command.h
index 7ab3f6442dcf9..3add66dd50d4a 100644
--- a/flang/include/flang/Runtime/command.h
+++ b/flang/include/flang/Runtime/command.h
@@ -12,7 +12,7 @@
 #include "flang/Runtime/entry-names.h"
 #include <cstdint>
 
-#ifdef _WIN32
+#ifdef _MSC_VER
 // On Windows* OS GetCurrentProcessId returns DWORD aka uint32_t
 typedef std::uint32_t pid_t;
 #else
diff --git a/flang/include/flang/Semantics/expression.h b/flang/include/flang/Semantics/expression.h
index ce253e6f52e8f..b051c486641fe 100644
--- a/flang/include/flang/Semantics/expression.h
+++ b/flang/include/flang/Semantics/expression.h
@@ -342,7 +342,6 @@ class ExpressionAnalyzer {
       const semantics::Scope &, bool C919bAlreadyEnforced = false);
   MaybeExpr CompleteSubscripts(ArrayRef &&);
   MaybeExpr ApplySubscripts(DataRef &&, std::vector<Subscript> &&);
-  void CheckSubscripts(ArrayRef &);
   bool CheckRanks(const DataRef &); // Return false if error exists.
   bool CheckPolymorphic(const DataRef &); // ditto
   bool CheckDataRef(const DataRef &); // ditto
diff --git a/flang/include/flang/Support/OpenMP-utils.h b/flang/include/flang/Support/OpenMP-utils.h
index d8f82e1cf94f1..6d9db2b682c50 100644
--- a/flang/include/flang/Support/OpenMP-utils.h
+++ b/flang/include/flang/Support/OpenMP-utils.h
@@ -34,6 +34,7 @@ struct EntryBlockArgsEntry {
 /// Structure holding the information needed to create and bind entry block
 /// arguments associated to all clauses that can define them.
 struct EntryBlockArgs {
+  EntryBlockArgsEntry hasDeviceAddr;
   llvm::ArrayRef<mlir::Value> hostEvalVars;
   EntryBlockArgsEntry inReduction;
   EntryBlockArgsEntry map;
@@ -44,21 +45,21 @@ struct EntryBlockArgs {
   EntryBlockArgsEntry useDevicePtr;
 
   bool isValid() const {
-    return inReduction.isValid() && map.isValid() && priv.isValid() &&
-        reduction.isValid() && taskReduction.isValid() &&
+    return hasDeviceAddr.isValid() && inReduction.isValid() && map.isValid() &&
+        priv.isValid() && reduction.isValid() && taskReduction.isValid() &&
         useDeviceAddr.isValid() && useDevicePtr.isValid();
   }
 
   auto getSyms() const {
-    return llvm::concat<const semantics::Symbol *const>(inReduction.syms,
-        map.syms, priv.syms, reduction.syms, taskReduction.syms,
-        useDeviceAddr.syms, useDevicePtr.syms);
+    return llvm::concat<const semantics::Symbol *const>(hasDeviceAddr.syms,
+        inReduction.syms, map.syms, priv.syms, reduction.syms,
+        taskReduction.syms, useDeviceAddr.syms, useDevicePtr.syms);
   }
 
   auto getVars() const {
-    return llvm::concat<const mlir::Value>(hostEvalVars, inReduction.vars,
-        map.vars, priv.vars, reduction.vars, taskReduction.vars,
-        useDeviceAddr.vars, useDevicePtr.vars);
+    return llvm::concat<const mlir::Value>(hasDeviceAddr.vars, hostEvalVars,
+        inReduction.vars, map.vars, priv.vars, reduction.vars,
+        taskReduction.vars, useDeviceAddr.vars, useDevicePtr.vars);
   }
 };
 
diff --git a/flang/lib/Evaluate/check-expression.cpp b/flang/lib/Evaluate/check-expression.cpp
index 823f8fd49baeb..3d338b04e64bb 100644
--- a/flang/lib/Evaluate/check-expression.cpp
+++ b/flang/lib/Evaluate/check-expression.cpp
@@ -69,13 +69,14 @@ class IsConstantExprHelper
   bool operator()(const Component &component) const {
     return (*this)(component.base());
   }
-  // Forbid integer division by zero in constants.
+  // Prevent integer division by known zeroes in constant expressions.
   template <int KIND>
   bool operator()(
       const Divide<Type<TypeCategory::Integer, KIND>> &division) const {
     using T = Type<TypeCategory::Integer, KIND>;
-    if (const auto divisor{GetScalarConstantValue<T>(division.right())}) {
-      return !divisor->IsZero() && (*this)(division.left());
+    if ((*this)(division.left()) && (*this)(division.right())) {
+      const auto divisor{GetScalarConstantValue<T>(division.right())};
+      return !divisor || !divisor->IsZero();
     } else {
       return false;
     }
diff --git a/flang/lib/Evaluate/variable.cpp b/flang/lib/Evaluate/variable.cpp
index 841d0f71ed0e2..849194b492053 100644
--- a/flang/lib/Evaluate/variable.cpp
+++ b/flang/lib/Evaluate/variable.cpp
@@ -477,8 +477,11 @@ int BaseObject::Corank() const {
 int Component::Corank() const {
   if (int corank{symbol_->Corank()}; corank > 0) {
     return corank;
+  } else if (semantics::IsAllocatableOrObjectPointer(&*symbol_)) {
+    return 0; // coarray subobjects ca%a or ca%p are not coarrays
+  } else {
+    return base().Corank();
   }
-  return base().Corank();
 }
 
 int NamedEntity::Corank() const {
@@ -489,7 +492,14 @@ int NamedEntity::Corank() const {
       u_);
 }
 
-int ArrayRef::Corank() const { return base().Corank(); }
+int ArrayRef::Corank() const {
+  for (const Subscript &subs : subscript_) {
+    if (!std::holds_alternative<Triplet>(subs.u) && subs.Rank() > 0) {
+      return 0; // vector-valued subscript - subobject is not a coarray
+    }
+  }
+  return base().Corank();
+}
 
 int DataRef::Corank() const {
   return common::visit(common::visitors{
diff --git a/flang/lib/Frontend/CMakeLists.txt b/flang/lib/Frontend/CMakeLists.txt
index 80d63fca6fb76..c80373799b015 100644
--- a/flang/lib/Frontend/CMakeLists.txt
+++ b/flang/lib/Frontend/CMakeLists.txt
@@ -5,6 +5,7 @@ add_flang_library(flangFrontend
   CompilerInstance.cpp
   CompilerInvocation.cpp
   CodeGenOptions.cpp
+  ParserActions.cpp
   FrontendAction.cpp
   FrontendActions.cpp
   FrontendOptions.cpp
diff --git a/flang/lib/Frontend/FrontendAction.cpp b/flang/lib/Frontend/FrontendAction.cpp
index 9a555bc7cd23b..ab77d143fa4b6 100644
--- a/flang/lib/Frontend/FrontendAction.cpp
+++ b/flang/lib/Frontend/FrontendAction.cpp
@@ -15,6 +15,7 @@
 #include "flang/Frontend/FrontendActions.h"
 #include "flang/Frontend/FrontendOptions.h"
 #include "flang/Frontend/FrontendPluginRegistry.h"
+#include "flang/Parser/parsing.h"
 #include "clang/Basic/DiagnosticFrontend.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/VirtualFileSystem.h"
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index 72a656b3627f5..94de376aaf7d6 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -14,9 +14,8 @@
 #include "flang/Frontend/CompilerInstance.h"
 #include "flang/Frontend/CompilerInvocation.h"
 #include "flang/Frontend/FrontendOptions.h"
-#include "flang/Frontend/PreprocessorOptions.h"
+#include "flang/Frontend/ParserActions.h"
 #include "flang/Lower/Bridge.h"
-#include "flang/Lower/PFTBuilder.h"
 #include "flang/Lower/Support/Verifier.h"
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
 #include "flang/Optimizer/Dialect/Support/KindMapping.h"
@@ -25,13 +24,7 @@
 #include "flang/Optimizer/Support/InitFIR.h"
 #include "flang/Optimizer/Support/Utils.h"
 #include "flang/Optimizer/Transforms/Passes.h"
-#include "flang/Parser/dump-parse-tree.h"
-#include "flang/Parser/parsing.h"
-#include "flang/Parser/provenance.h"
-#include "flang/Parser/source.h"
-#include "flang/Parser/unparse.h"
 #include "flang/Semantics/runtime-type-info.h"
-#include "flang/Semantics/semantics.h"
 #include "flang/Semantics/unparse-with-symbols.h"
 #include "flang/Support/default-kinds.h"
 #include "flang/Tools/CrossToolHelpers.h"
@@ -317,9 +310,9 @@ bool CodeGenAction::beginSourceFileAction() {
   lower::LoweringBridge lb = Fortran::lower::LoweringBridge::create(
       *mlirCtx, ci.getSemanticsContext(), defKinds,
       ci.getSemanticsContext().intrinsics(),
-      ci.getSemanticsContext().targetCharacteristics(),
-      ci.getParsing().allCooked(), ci.getInvocation().getTargetOpts().triple,
-      kindMap, ci.getInvocation().getLoweringOpts(),
+      ci.getSemanticsContext().targetCharacteristics(), getAllCooked(ci),
+      ci.getInvocation().getTargetOpts().triple, kindMap,
+      ci.getInvocation().getLoweringOpts(),
       ci.getInvocation().getFrontendOpts().envDefaults,
       ci.getInvocation().getFrontendOpts().features, targetMachine,
       ci.getInvocation().getTargetOpts(), ci.getInvocation().getCodeGenOpts());
@@ -333,8 +326,7 @@ bool CodeGenAction::beginSourceFileAction() {
   }
 
   // Create a parse tree and lower it to FIR
-  Fortran::parser::Program &parseTree{*ci.getParsing().parseTree()};
-  lb.lower(parseTree, ci.getSemanticsContext());
+  parseAndLowerTree(ci, lb);
 
   // Fetch module from lb, so we can set
   mlirModule = lb.getModuleAndRelease();
@@ -431,19 +423,8 @@ void PrintPreprocessedAction::executeAction() {
   std::string buf;
   llvm::raw_string_ostream outForPP{buf};
 
-  // Format or dump the prescanner's output
   CompilerInstance &ci = this->getInstance();
-  if (ci.getInvocation().getPreprocessorOpts().showMacros) {
-    ci.getParsing().EmitPreprocessorMacros(outForPP);
-  } else if (ci.getInvocation().getPreprocessorOpts().noReformat) {
-    ci.getParsing().DumpCookedChars(outForPP);
-  } else {
-    ci.getParsing().EmitPreprocessedSource(
-        outForPP, !ci.getInvocation().getPreprocessorOpts().noLineDirectives);
-  }
-
-  // Print getDiagnostics from the prescanner
-  ci.getParsing().messages().Emit(llvm::errs(), ci.getAllCookedSources());
+  formatOrDumpPrescanner(buf, outForPP, ci);
 
   // If a pre-defined output stream exists, dump the preprocessed content there
   if (!ci.isOutputStreamNull()) {
@@ -463,60 +444,31 @@ void PrintPreprocessedAction::executeAction() {
 }
 
 void DebugDumpProvenanceAction::executeAction() {
-  this->getInstance().getParsing().DumpProvenance(llvm::outs());
+  dumpProvenance(this->getInstance());
 }
 
 void ParseSyntaxOnlyAction::executeAction() {}
 
 void DebugUnparseNoSemaAction::executeAction() {
-  auto &invoc = this->getInstance().getInvocation();
-  auto &parseTree{getInstance().getParsing().parseTree()};
-
-  // TODO: Options should come from CompilerInvocation
-  Unparse(llvm::outs(), *parseTree,
-          /*encoding=*/Fortran::parser::Encoding::UTF_8,
-          /*capitalizeKeywords=*/true, /*backslashEscapes=*/false,
-          /*preStatement=*/nullptr,
-          invoc.getUseAnalyzedObjectsForUnparse() ? &invoc.getAsFortran()
-                                                  : nullptr);
+  debugUnparseNoSema(this->getInstance(), llvm::outs());
 }
 
 void DebugUnparseAction::executeAction() {
-  auto &invoc = this->getInstance().getInvocation();
-  auto &parseTree{getInstance().getParsing().parseTree()};
-
   CompilerInstance &ci = this->getInstance();
   auto os{ci.createDefaultOutputFile(
       /*Binary=*/false, /*InFile=*/getCurrentFileOrBufferName())};
 
-  // TODO: Options should come from CompilerInvocation
-  Unparse(*os, *parseTree,
-          /*encoding=*/Fortran::parser::Encoding::UTF_8,
-          /*capitalizeKeywords=*/true, /*backslashEscapes=*/false,
-          /*preStatement=*/nullptr,
-          invoc.getUseAnalyzedObjectsForUnparse() ? &invoc.getAsFortran()
-                                                  : nullptr);
-
-  // Report fatal semantic errors
+  debugUnparseNoSema(ci, *os);
   reportFatalSemanticErrors();
 }
 
 void DebugUnparseWithSymbolsAction::executeAction() {
-  auto &parseTree{*getInstance().getParsing().parseTree()};
-
-  Fortran::semantics::UnparseWithSymbols(
-      llvm::outs(), parseTree, /*encoding=*/Fortran::parser::Encoding::UTF_8);
-
-  // Report fatal semantic errors
+  debugUnparseWithSymbols(this->getInstance());
   reportFatalSemanticErrors();
 }
 
 void DebugUnparseWithModulesAction::executeAction() {
-  auto &parseTree{*getInstance().getParsing().parseTree()};
-  CompilerInstance &ci{getInstance()};
-  Fortran::semantics::UnparseWithModules(
-      llvm::outs(), ci.getSemantics().context(), parseTree,
-      /*encoding=*/Fortran::parser::Encoding::UTF_8);
+  debugUnparseWithModules(this->getInstance());
   reportFatalSemanticErrors();
 }
 
@@ -540,12 +492,7 @@ void DebugDumpAllAction::executeAction() {
   CompilerInstance &ci = this->getInstance();
 
   // Dump parse tree
-  auto &parseTree{getInstance().getParsing().parseTree()};
-  llvm::outs() << "========================";
-  llvm::outs() << " Flang: parse tree dump ";
-  llvm::outs() << "========================\n";
-  Fortran::parser::DumpTree(llvm::outs(), parseTree,
-                            &ci.getInvocation().getAsFortran());
+  dumpTree(ci);
 
   if (!ci.getRtTyTables().schemata) {
     unsigned diagID = ci.getDiagnostics().getCustomDiagID(
@@ -564,21 +511,11 @@ void DebugDumpAllAction::executeAction() {
 }
 
 void DebugDumpParseTreeNoSemaAction::executeAction() {
-  auto &parseTree{getInstance().getParsing().parseTree()};
-
-  // Dump parse tree
-  Fortran::parser::DumpTree(
-      llvm::outs(), parseTree,
-      &this->getInstance().getInvocation().getAsFortran());
+  dumpTree(this->getInstance());
 }
 
 void DebugDumpParseTreeAction::executeAction() {
-  auto &parseTree{getInstance().getParsing().parseTree()};
-
-  // Dump parse tree
-  Fortran::parser::DumpTree(
-      llvm::outs(), parseTree,
-      &this->getInstance().getInvocation().getAsFortran());
+  dumpTree(this->getInstance());
 
   // Report fatal semantic errors
   reportFatalSemanticErrors();
@@ -586,62 +523,20 @@ void DebugDumpParseTreeAction::executeAction() {
 
 void DebugMeasureParseTreeAction::executeAction() {
   CompilerInstance &ci = this->getInstance();
-
-  // Parse. In case of failure, report and return.
-  ci.getParsing().Parse(llvm::outs());
-
-  if ((ci.getParsing().parseTree().has_value() &&
-       !ci.getParsing().consumedWholeFile()) ||
-      (!ci.getParsing().messages().empty() &&
-       (ci.getInvocation().getWarnAsErr() ||
-        ci.getParsing().messages().AnyFatalError()))) {
-    unsigned diagID = ci.getDiagnostics().getCustomDiagID(
-        clang::DiagnosticsEngine::Error, "Could not parse %0");
-    ci.getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
-
-    ci.getParsing().messages().Emit(llvm::errs(),
-                                    this->getInstance().getAllCookedSources());
-    return;
-  }
-
-  // Report the getDiagnostics from parsing
-  ci.getParsing().messages().Emit(llvm::errs(), ci.getAllCookedSources());
-
-  auto &parseTree{*ci.getParsing().parseTree()};
-
-  // Measure the parse tree
-  MeasurementVisitor visitor;
-  Fortran::parser::Walk(parseTree, visitor);
-  llvm::outs() << "Parse tree comprises " << visitor.objects
-               << " objects and occupies " << visitor.bytes
-               << " total bytes.\n";
+  debugMeasureParseTree(ci, getCurrentFileOrBufferName());
 }
 
 void DebugPreFIRTreeAction::executeAction() {
-  CompilerInstance &ci = this->getInstance();
   // Report and exit if fatal semantic errors are present
   if (reportFatalSemanticErrors()) {
     return;
   }
 
-  auto &parseTree{*ci.getParsing().parseTree()};
-
-  // Dump pre-FIR tree
-  if (auto ast{
-          Fortran::lower::createPFT(parseTree, ci.getSemanticsContext())}) {
-    Fortran::lower::dumpPFT(llvm::outs(), *ast);
-  } else {
-    unsigned diagID = ci.getDiagnostics().getCustomDiagID(
-        clang::DiagnosticsEngine::Error, "Pre FIR Tree is NULL.");
-    ci.getDiagnostics().Report(diagID);
-  }
+  dumpPreFIRTree(this->getInstance());
 }
 
 void DebugDumpParsingLogAction::executeAction() {
-  CompilerInstance &ci = this->getInstance();
-
-  ci.getParsing().Parse(llvm::errs());
-  ci.getParsing().DumpParsingLog(llvm::outs());
+  debugDumpParsingLog(this->getInstance());
 }
 
 void GetDefinitionAction::executeAction() {
@@ -1473,17 +1368,7 @@ void InitOnlyAction::executeAction() {
 void PluginParseTreeAction::executeAction() {}
 
 void DebugDumpPFTAction::executeAction() {
-  CompilerInstance &ci = this->getInstance();
-
-  if (auto ast = Fortran::lower::createPFT(*ci.getParsing().parseTree(),
-                                           ci.getSemantics().context())) {
-    Fortran::lower::dumpPFT(llvm::outs(), *ast);
-    return;
-  }
-
-  unsigned diagID = ci.getDiagnostics().getCustomDiagID(
-      clang::DiagnosticsEngine::Error, "Pre FIR Tree is NULL.");
-  ci.getDiagnostics().Report(diagID);
+  dumpPreFIRTree(this->getInstance());
 }
 
 Fortran::parser::Parsing &PluginParseTreeAction::getParsing() {
diff --git a/flang/lib/Frontend/ParserActions.cpp b/flang/lib/Frontend/ParserActions.cpp
new file mode 100644
index 0000000000000..cc7e72f696f96
--- /dev/null
+++ b/flang/lib/Frontend/ParserActions.cpp
@@ -0,0 +1,148 @@
+//===--- ParserActions.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Frontend/ParserActions.h"
+#include "flang/Frontend/CompilerInstance.h"
+#include "flang/Lower/Bridge.h"
+#include "flang/Lower/PFTBuilder.h"
+#include "flang/Parser/dump-parse-tree.h"
+#include "flang/Parser/parsing.h"
+#include "flang/Parser/provenance.h"
+#include "flang/Parser/source.h"
+#include "flang/Parser/unparse.h"
+#include "flang/Semantics/unparse-with-symbols.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace Fortran::frontend {
+
+parser::AllCookedSources &getAllCooked(CompilerInstance &ci) {
+  return ci.getParsing().allCooked();
+}
+
+void parseAndLowerTree(CompilerInstance &ci, lower::LoweringBridge &lb) {
+  parser::Program &parseTree{*ci.getParsing().parseTree()};
+  lb.lower(parseTree, ci.getSemanticsContext());
+}
+
+void dumpTree(CompilerInstance &ci) {
+  auto &parseTree{ci.getParsing().parseTree()};
+  llvm::outs() << "========================";
+  llvm::outs() << " Flang: parse tree dump ";
+  llvm::outs() << "========================\n";
+  parser::DumpTree(llvm::outs(), parseTree, &ci.getInvocation().getAsFortran());
+}
+
+void dumpProvenance(CompilerInstance &ci) {
+  ci.getParsing().DumpProvenance(llvm::outs());
+}
+
+void dumpPreFIRTree(CompilerInstance &ci) {
+  auto &parseTree{*ci.getParsing().parseTree()};
+
+  if (auto ast{lower::createPFT(parseTree, ci.getSemanticsContext())}) {
+    lower::dumpPFT(llvm::outs(), *ast);
+  } else {
+    unsigned diagID = ci.getDiagnostics().getCustomDiagID(
+        clang::DiagnosticsEngine::Error, "Pre FIR Tree is NULL.");
+    ci.getDiagnostics().Report(diagID);
+  }
+}
+
+void formatOrDumpPrescanner(std::string &buf,
+                            llvm::raw_string_ostream &outForPP,
+                            CompilerInstance &ci) {
+  if (ci.getInvocation().getPreprocessorOpts().showMacros) {
+    ci.getParsing().EmitPreprocessorMacros(outForPP);
+  } else if (ci.getInvocation().getPreprocessorOpts().noReformat) {
+    ci.getParsing().DumpCookedChars(outForPP);
+  } else {
+    ci.getParsing().EmitPreprocessedSource(
+        outForPP, !ci.getInvocation().getPreprocessorOpts().noLineDirectives);
+  }
+
+  // Print getDiagnostics from the prescanner
+  ci.getParsing().messages().Emit(llvm::errs(), ci.getAllCookedSources());
+}
+
+struct MeasurementVisitor {
+  template <typename A>
+  bool Pre(const A &) {
+    return true;
+  }
+  template <typename A>
+  void Post(const A &) {
+    ++objects;
+    bytes += sizeof(A);
+  }
+  size_t objects{0}, bytes{0};
+};
+
+void debugMeasureParseTree(CompilerInstance &ci, llvm::StringRef filename) {
+  // Parse. In case of failure, report and return.
+  ci.getParsing().Parse(llvm::outs());
+
+  if ((ci.getParsing().parseTree().has_value() &&
+       !ci.getParsing().consumedWholeFile()) ||
+      (!ci.getParsing().messages().empty() &&
+       (ci.getInvocation().getWarnAsErr() ||
+        ci.getParsing().messages().AnyFatalError()))) {
+    unsigned diagID = ci.getDiagnostics().getCustomDiagID(
+        clang::DiagnosticsEngine::Error, "Could not parse %0");
+    ci.getDiagnostics().Report(diagID) << filename;
+
+    ci.getParsing().messages().Emit(llvm::errs(), ci.getAllCookedSources());
+    return;
+  }
+
+  // Report the getDiagnostics from parsing
+  ci.getParsing().messages().Emit(llvm::errs(), ci.getAllCookedSources());
+
+  auto &parseTree{ci.getParsing().parseTree()};
+  MeasurementVisitor visitor;
+  parser::Walk(parseTree, visitor);
+  llvm::outs() << "Parse tree comprises " << visitor.objects
+               << " objects and occupies " << visitor.bytes
+               << " total bytes.\n";
+}
+
+void debugUnparseNoSema(CompilerInstance &ci, llvm::raw_ostream &out) {
+  auto &invoc = ci.getInvocation();
+  auto &parseTree{ci.getParsing().parseTree()};
+
+  // TODO: Options should come from CompilerInvocation
+  Unparse(out, *parseTree,
+          /*encoding=*/parser::Encoding::UTF_8,
+          /*capitalizeKeywords=*/true, /*backslashEscapes=*/false,
+          /*preStatement=*/nullptr,
+          invoc.getUseAnalyzedObjectsForUnparse() ? &invoc.getAsFortran()
+                                                  : nullptr);
+}
+
+void debugUnparseWithSymbols(CompilerInstance &ci) {
+  auto &parseTree{*ci.getParsing().parseTree()};
+
+  semantics::UnparseWithSymbols(llvm::outs(), parseTree,
+                                /*encoding=*/parser::Encoding::UTF_8);
+}
+
+void debugUnparseWithModules(CompilerInstance &ci) {
+  auto &parseTree{*ci.getParsing().parseTree()};
+  semantics::UnparseWithModules(llvm::outs(), ci.getSemantics().context(),
+                                parseTree,
+                                /*encoding=*/parser::Encoding::UTF_8);
+}
+
+void debugDumpParsingLog(CompilerInstance &ci) {
+  ci.getParsing().Parse(llvm::errs());
+  ci.getParsing().DumpParsingLog(llvm::outs());
+}
+} // namespace Fortran::frontend
diff --git a/flang/lib/Lower/Allocatable.cpp b/flang/lib/Lower/Allocatable.cpp
index 3d21e7a3fa8d5..9938bd573d1fa 100644
--- a/flang/lib/Lower/Allocatable.cpp
+++ b/flang/lib/Lower/Allocatable.cpp
@@ -470,7 +470,7 @@ class AllocateStmtHelper {
   void genSimpleAllocation(const Allocation &alloc,
                            const fir::MutableBoxValue &box) {
     bool isCudaSymbol = Fortran::semantics::HasCUDAAttr(alloc.getSymbol());
-    bool isCudaDeviceContext = Fortran::lower::isCudaDeviceContext(builder);
+    bool isCudaDeviceContext = cuf::isCUDADeviceContext(builder.getRegion());
     bool inlineAllocation = !box.isDerived() && !errorManager.hasStatSpec() &&
                             !alloc.type.IsPolymorphic() &&
                             !alloc.hasCoarraySpec() && !useAllocateRuntime &&
@@ -862,7 +862,7 @@ genDeallocate(fir::FirOpBuilder &builder,
               mlir::Value declaredTypeDesc = {},
               const Fortran::semantics::Symbol *symbol = nullptr) {
   bool isCudaSymbol = symbol && Fortran::semantics::HasCUDAAttr(*symbol);
-  bool isCudaDeviceContext = Fortran::lower::isCudaDeviceContext(builder);
+  bool isCudaDeviceContext = cuf::isCUDADeviceContext(builder.getRegion());
   bool inlineDeallocation =
       !box.isDerived() && !box.isPolymorphic() && !box.hasAssumedRank() &&
       !box.isUnlimitedPolymorphic() && !errorManager.hasStatSpec() &&
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 95f431983d442..93f54d88a029d 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -1274,6 +1274,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
         [](const fir::FortranVariableOpInterface &box) {
           return fir::FortranVariableOpInterface(box).isPointer();
         },
+        [](const fir::AbstractBox &box) {
+          return fir::isBoxProcAddressType(box.getAddr().getType());
+        },
         [](const auto &box) { return false; });
 
     copyVarHLFIR(loc, dst, src, isBoxAllocatable, isBoxPointer, flags);
@@ -4353,8 +4356,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
   void genForallPointerAssignment(
       mlir::Location loc, const Fortran::evaluate::Assignment &assign,
       const Fortran::evaluate::Assignment::BoundsSpec &lbExprs) {
-    if (Fortran::evaluate::IsProcedureDesignator(assign.rhs))
-      TODO(loc, "procedure pointer assignment inside FORALL");
     std::optional<Fortran::evaluate::DynamicType> lhsType =
         assign.lhs.GetType();
     // Polymorphic pointer assignment is delegated to the runtime, and
@@ -4383,7 +4384,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     Fortran::lower::StatementContext lhsContext;
     hlfir::Entity lhs = Fortran::lower::convertExprToHLFIR(
         loc, *this, assign.lhs, localSymbols, lhsContext);
-
     auto lhsYieldOp = builder->create<hlfir::YieldOp>(loc, lhs);
     Fortran::lower::genCleanUpInRegionIfAny(
         loc, *builder, lhsYieldOp.getCleanup(), lhsContext);
@@ -4391,6 +4391,23 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     // Lower RHS in its own region.
     builder->createBlock(&regionAssignOp.getRhsRegion());
     Fortran::lower::StatementContext rhsContext;
+    mlir::Value rhs =
+        genForallPointerAssignmentRhs(loc, lhs, assign, rhsContext);
+    auto rhsYieldOp = builder->create<hlfir::YieldOp>(loc, rhs);
+    Fortran::lower::genCleanUpInRegionIfAny(
+        loc, *builder, rhsYieldOp.getCleanup(), rhsContext);
+
+    builder->setInsertionPointAfter(regionAssignOp);
+  }
+
+  mlir::Value
+  genForallPointerAssignmentRhs(mlir::Location loc, mlir::Value lhs,
+                                const Fortran::evaluate::Assignment &assign,
+                                Fortran::lower::StatementContext &rhsContext) {
+    if (Fortran::evaluate::IsProcedureDesignator(assign.rhs))
+      return fir::getBase(Fortran::lower::convertExprToAddress(
+          loc, *this, assign.rhs, localSymbols, rhsContext));
+    // Data target.
     hlfir::Entity rhs = Fortran::lower::convertExprToHLFIR(
         loc, *this, assign.rhs, localSymbols, rhsContext);
     // Create pointer descriptor value from the RHS.
@@ -4398,12 +4415,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       rhs = hlfir::Entity{builder->create<fir::LoadOp>(loc, rhs)};
     auto lhsBoxType =
         llvm::cast<fir::BaseBoxType>(fir::unwrapRefType(lhs.getType()));
-    mlir::Value newBox = hlfir::genVariableBox(loc, *builder, rhs, lhsBoxType);
-    auto rhsYieldOp = builder->create<hlfir::YieldOp>(loc, newBox);
-    Fortran::lower::genCleanUpInRegionIfAny(
-        loc, *builder, rhsYieldOp.getCleanup(), rhsContext);
-
-    builder->setInsertionPointAfter(regionAssignOp);
+    return hlfir::genVariableBox(loc, *builder, rhs, lhsBoxType);
   }
 
   // Create the 2 x newRank array with the bounds to be passed to the runtime as
@@ -4689,7 +4701,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     mlir::Location loc = getCurrentLocation();
     fir::FirOpBuilder &builder = getFirOpBuilder();
 
-    bool isInDeviceContext = Fortran::lower::isCudaDeviceContext(builder);
+    bool isInDeviceContext = cuf::isCUDADeviceContext(builder.getRegion());
 
     bool isCUDATransfer =
         IsCUDADataTransfer(assign.lhs, assign.rhs) && !isInDeviceContext;
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 8a7be19f8040b..e87e78bcd3bf9 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -852,9 +852,8 @@ void Fortran::lower::defaultInitializeAtRuntime(
       }
       auto addrOf = builder.create<fir::AddrOfOp>(loc, global.resultType(),
                                                   global.getSymbol());
-      fir::LoadOp load = builder.create<fir::LoadOp>(loc, addrOf.getResult());
-      // FIXME: Use memcpy instead of store.
-      builder.create<fir::StoreOp>(loc, load, fir::getBase(exv));
+      builder.create<fir::CopyOp>(loc, addrOf, fir::getBase(exv),
+                                  /*noOverlap=*/true);
     } else {
       mlir::Value box = builder.createBox(loc, exv);
       fir::runtime::genDerivedTypeInitialize(builder, loc, box);
@@ -1902,7 +1901,7 @@ void Fortran::lower::genDeclareSymbol(
     bool force) {
   if (converter.getLoweringOptions().getLowerToHighLevelFIR() &&
       (!Fortran::semantics::IsProcedure(sym) ||
-       Fortran::semantics::IsPointer(sym)) &&
+       Fortran::semantics::IsPointer(sym.GetUltimate())) &&
       !sym.detailsIf<Fortran::semantics::CommonBlockDetails>()) {
     fir::FirOpBuilder &builder = converter.getFirOpBuilder();
     const mlir::Location loc = genLocation(converter, sym);
diff --git a/flang/lib/Lower/OpenMP/ClauseFinder.h b/flang/lib/Lower/OpenMP/ClauseFinder.h
new file mode 100644
index 0000000000000..3b77f2ca1d4cb
--- /dev/null
+++ b/flang/lib/Lower/OpenMP/ClauseFinder.h
@@ -0,0 +1,76 @@
+//===-- Lower/OpenMP/ClauseFinder.h --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+#ifndef FORTRAN_LOWER_CLAUSEFINDER_H
+#define FORTRAN_LOWER_CLAUSEFINDER_H
+
+#include "Clauses.h"
+
+namespace Fortran {
+namespace lower {
+namespace omp {
+
+class ClauseFinder {
+  using ClauseIterator = List<Clause>::const_iterator;
+
+public:
+  /// Utility to find a clause within a range in the clause list.
+  template <typename T>
+  static ClauseIterator findClause(ClauseIterator begin, ClauseIterator end) {
+    for (ClauseIterator it = begin; it != end; ++it) {
+      if (std::get_if<T>(&it->u))
+        return it;
+    }
+
+    return end;
+  }
+
+  /// Return the first instance of the given clause found in the clause list or
+  /// `nullptr` if not present. If more than one instance is expected, use
+  /// `findRepeatableClause` instead.
+  template <typename T>
+  static const T *findUniqueClause(const List<Clause> &clauses,
+                                   const parser::CharBlock **source = nullptr) {
+    ClauseIterator it = findClause<T>(clauses.begin(), clauses.end());
+    if (it != clauses.end()) {
+      if (source)
+        *source = &it->source;
+      return &std::get<T>(it->u);
+    }
+    return nullptr;
+  }
+
+  /// Call `callbackFn` for each occurrence of the given clause. Return `true`
+  /// if at least one instance was found.
+  template <typename T>
+  static bool findRepeatableClause(
+      const List<Clause> &clauses,
+      std::function<void(const T &, const parser::CharBlock &source)>
+          callbackFn) {
+    bool found = false;
+    ClauseIterator nextIt, endIt = clauses.end();
+    for (ClauseIterator it = clauses.begin(); it != endIt; it = nextIt) {
+      nextIt = findClause<T>(it, endIt);
+
+      if (nextIt != endIt) {
+        callbackFn(std::get<T>(nextIt->u), nextIt->source);
+        found = true;
+        ++nextIt;
+      }
+    }
+    return found;
+  }
+};
+} // namespace omp
+} // namespace lower
+} // namespace Fortran
+
+#endif // FORTRAN_LOWER_CLAUSEFINDER_H
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index e21d299570b86..dda2ac76df3b2 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -12,6 +12,7 @@
 
 #include "ClauseProcessor.h"
 #include "Clauses.h"
+#include "Utils.h"
 
 #include "flang/Lower/PFTBuilder.h"
 #include "flang/Parser/tools.h"
@@ -201,24 +202,6 @@ static void addUseDeviceClause(
     useDeviceSyms.push_back(object.sym());
 }
 
-static void convertLoopBounds(lower::AbstractConverter &converter,
-                              mlir::Location loc,
-                              mlir::omp::LoopRelatedClauseOps &result,
-                              std::size_t loopVarTypeSize) {
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  // The types of lower bound, upper bound, and step are converted into the
-  // type of the loop variable if necessary.
-  mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize);
-  for (unsigned it = 0; it < (unsigned)result.loopLowerBounds.size(); it++) {
-    result.loopLowerBounds[it] = firOpBuilder.createConvert(
-        loc, loopVarType, result.loopLowerBounds[it]);
-    result.loopUpperBounds[it] = firOpBuilder.createConvert(
-        loc, loopVarType, result.loopUpperBounds[it]);
-    result.loopSteps[it] =
-        firOpBuilder.createConvert(loc, loopVarType, result.loopSteps[it]);
-  }
-}
-
 //===----------------------------------------------------------------------===//
 // ClauseProcessor unique clauses
 //===----------------------------------------------------------------------===//
@@ -240,55 +223,8 @@ bool ClauseProcessor::processCollapse(
     mlir::Location currentLocation, lower::pft::Evaluation &eval,
     mlir::omp::LoopRelatedClauseOps &result,
     llvm::SmallVectorImpl<const semantics::Symbol *> &iv) const {
-  bool found = false;
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-
-  // Collect the loops to collapse.
-  lower::pft::Evaluation *doConstructEval = &eval.getFirstNestedEvaluation();
-  if (doConstructEval->getIf<parser::DoConstruct>()->IsDoConcurrent()) {
-    TODO(currentLocation, "Do Concurrent in Worksharing loop construct");
-  }
-
-  std::int64_t collapseValue = 1l;
-  if (auto *clause = findUniqueClause<omp::clause::Collapse>()) {
-    collapseValue = evaluate::ToInt64(clause->v).value();
-    found = true;
-  }
-
-  std::size_t loopVarTypeSize = 0;
-  do {
-    lower::pft::Evaluation *doLoop =
-        &doConstructEval->getFirstNestedEvaluation();
-    auto *doStmt = doLoop->getIf<parser::NonLabelDoStmt>();
-    assert(doStmt && "Expected do loop to be in the nested evaluation");
-    const auto &loopControl =
-        std::get<std::optional<parser::LoopControl>>(doStmt->t);
-    const parser::LoopControl::Bounds *bounds =
-        std::get_if<parser::LoopControl::Bounds>(&loopControl->u);
-    assert(bounds && "Expected bounds for worksharing do loop");
-    lower::StatementContext stmtCtx;
-    result.loopLowerBounds.push_back(fir::getBase(
-        converter.genExprValue(*semantics::GetExpr(bounds->lower), stmtCtx)));
-    result.loopUpperBounds.push_back(fir::getBase(
-        converter.genExprValue(*semantics::GetExpr(bounds->upper), stmtCtx)));
-    if (bounds->step) {
-      result.loopSteps.push_back(fir::getBase(
-          converter.genExprValue(*semantics::GetExpr(bounds->step), stmtCtx)));
-    } else { // If `step` is not present, assume it as `1`.
-      result.loopSteps.push_back(firOpBuilder.createIntegerConstant(
-          currentLocation, firOpBuilder.getIntegerType(32), 1));
-    }
-    iv.push_back(bounds->name.thing.symbol);
-    loopVarTypeSize = std::max(loopVarTypeSize,
-                               bounds->name.thing.symbol->GetUltimate().size());
-    collapseValue--;
-    doConstructEval =
-        &*std::next(doConstructEval->getNestedEvaluations().begin());
-  } while (collapseValue > 0);
-
-  convertLoopBounds(converter, currentLocation, result, loopVarTypeSize);
-
-  return found;
+  return collectLoopRelatedInfo(converter, currentLocation, eval, clauses,
+                                result, iv);
 }
 
 bool ClauseProcessor::processDevice(lower::StatementContext &stmtCtx,
@@ -913,14 +849,34 @@ bool ClauseProcessor::processDepend(mlir::omp::DependClauseOps &result) const {
 }
 
 bool ClauseProcessor::processHasDeviceAddr(
-    mlir::omp::HasDeviceAddrClauseOps &result,
-    llvm::SmallVectorImpl<const semantics::Symbol *> &isDeviceSyms) const {
-  return findRepeatableClause<omp::clause::HasDeviceAddr>(
-      [&](const omp::clause::HasDeviceAddr &devAddrClause,
-          const parser::CharBlock &) {
-        addUseDeviceClause(converter, devAddrClause.v, result.hasDeviceAddrVars,
-                           isDeviceSyms);
+    lower::StatementContext &stmtCtx, mlir::omp::HasDeviceAddrClauseOps &result,
+    llvm::SmallVectorImpl<const semantics::Symbol *> &hasDeviceSyms) const {
+  // For HAS_DEVICE_ADDR objects, implicitly map the top-level entities.
+  // Their address (or the whole descriptor, if the entity had one) will be
+  // passed to the target region.
+  std::map<Object, OmpMapParentAndMemberData> parentMemberIndices;
+  bool clauseFound = findRepeatableClause<omp::clause::HasDeviceAddr>(
+      [&](const omp::clause::HasDeviceAddr &clause,
+          const parser::CharBlock &source) {
+        mlir::Location location = converter.genLocation(source);
+        llvm::omp::OpenMPOffloadMappingFlags mapTypeBits =
+            llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
+            llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
+        omp::ObjectList baseObjects;
+        llvm::transform(clause.v, std::back_inserter(baseObjects),
+                        [&](const omp::Object &object) {
+                          if (auto maybeBase = getBaseObject(object, semaCtx))
+                            return *maybeBase;
+                          return object;
+                        });
+        processMapObjects(stmtCtx, location, baseObjects, mapTypeBits,
+                          parentMemberIndices, result.hasDeviceAddrVars,
+                          hasDeviceSyms);
       });
+
+  insertChildMapInfoIntoParent(converter, semaCtx, stmtCtx, parentMemberIndices,
+                               result.hasDeviceAddrVars, hasDeviceSyms);
+  return clauseFound;
 }
 
 bool ClauseProcessor::processIf(
@@ -972,8 +928,24 @@ void ClauseProcessor::processMapObjects(
     llvm::SmallVectorImpl<const semantics::Symbol *> &mapSyms,
     llvm::StringRef mapperIdNameRef) const {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+
+  // Create the mapper symbol from its name, if specified.
   mlir::FlatSymbolRefAttr mapperId;
-  std::string mapperIdName = mapperIdNameRef.str();
+  if (!mapperIdNameRef.empty() && !objects.empty()) {
+    std::string mapperIdName = mapperIdNameRef.str();
+    if (mapperIdName == "default") {
+      const omp::Object &object = objects.front();
+      auto &typeSpec = object.sym()->owner().IsDerivedType()
+                           ? *object.sym()->owner().derivedTypeSpec()
+                           : object.sym()->GetType()->derivedTypeSpec();
+      mapperIdName = typeSpec.name().ToString() + ".default";
+      mapperIdName = converter.mangleName(mapperIdName, *typeSpec.GetScope());
+    }
+    assert(converter.getModuleOp().lookupSymbol(mapperIdName) &&
+           "mapper not found");
+    mapperId =
+        mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(), mapperIdName);
+  }
 
   for (const omp::Object &object : objects) {
     llvm::SmallVector<mlir::Value> bounds;
@@ -1006,20 +978,6 @@ void ClauseProcessor::processMapObjects(
       }
     }
 
-    if (!mapperIdName.empty()) {
-      if (mapperIdName == "default") {
-        auto &typeSpec = object.sym()->owner().IsDerivedType()
-                             ? *object.sym()->owner().derivedTypeSpec()
-                             : object.sym()->GetType()->derivedTypeSpec();
-        mapperIdName = typeSpec.name().ToString() + ".default";
-        mapperIdName = converter.mangleName(mapperIdName, *typeSpec.GetScope());
-      }
-      assert(converter.getModuleOp().lookupSymbol(mapperIdName) &&
-             "mapper not found");
-      mapperId = mlir::FlatSymbolRefAttr::get(&converter.getMLIRContext(),
-                                              mapperIdName);
-      mapperIdName.clear();
-    }
     // Explicit map captures are captured ByRef by default,
     // optimisation passes may alter this to ByCopy or other capture
     // types to optimise
@@ -1093,15 +1051,15 @@ bool ClauseProcessor::processMap(
     }
 
     if (typeMods) {
+      // TODO: Still requires "self" modifier, an OpenMP 6.0+ feature
       if (llvm::is_contained(*typeMods, Map::MapTypeModifier::Always))
         mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
-      // Diagnose unimplemented map-type-modifiers.
-      if (llvm::any_of(*typeMods, [](Map::MapTypeModifier m) {
-            return m != Map::MapTypeModifier::Always;
-          })) {
-        TODO(currentLocation, "Map type modifiers (other than 'ALWAYS')"
-                              " are not implemented yet");
-      }
+      if (llvm::is_contained(*typeMods, Map::MapTypeModifier::Present))
+        mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
+      if (llvm::is_contained(*typeMods, Map::MapTypeModifier::Close))
+        mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
+      if (llvm::is_contained(*typeMods, Map::MapTypeModifier::OmpxHold))
+        mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
     }
 
     if (iterator) {
@@ -1137,19 +1095,20 @@ bool ClauseProcessor::processMotionClauses(lower::StatementContext &stmtCtx,
   auto callbackFn = [&](const auto &clause, const parser::CharBlock &source) {
     mlir::Location clauseLocation = converter.genLocation(source);
     const auto &[expectation, mapper, iterator, objects] = clause.t;
-    // TODO Support motion modifiers: present, mapper, iterator.
-    if (expectation) {
-      TODO(clauseLocation, "PRESENT modifier is not supported yet");
-    } else if (mapper) {
+
+    // TODO Support motion modifiers: mapper, iterator.
+    if (mapper) {
       TODO(clauseLocation, "Mapper modifier is not supported yet");
     } else if (iterator) {
       TODO(clauseLocation, "Iterator modifier is not supported yet");
     }
-    constexpr llvm::omp::OpenMPOffloadMappingFlags mapTypeBits =
+
+    llvm::omp::OpenMPOffloadMappingFlags mapTypeBits =
         std::is_same_v<llvm::remove_cvref_t<decltype(clause)>, omp::clause::To>
             ? llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO
             : llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM;
-
+    if (expectation && *expectation == omp::clause::To::Expectation::Present)
+      mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
     processMapObjects(stmtCtx, clauseLocation, objects, mapTypeBits,
                       parentMemberIndices, result.mapVars, mapSymbols);
   };
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index 889a09a8f0cd8..aa203689ab560 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -12,6 +12,7 @@
 #ifndef FORTRAN_LOWER_CLAUSEPROCESSOR_H
 #define FORTRAN_LOWER_CLAUSEPROCESSOR_H
 
+#include "ClauseFinder.h"
 #include "Clauses.h"
 #include "ReductionProcessor.h"
 #include "Utils.h"
@@ -71,8 +72,9 @@ class ClauseProcessor {
   bool processFinal(lower::StatementContext &stmtCtx,
                     mlir::omp::FinalClauseOps &result) const;
   bool processHasDeviceAddr(
+      lower::StatementContext &stmtCtx,
       mlir::omp::HasDeviceAddrClauseOps &result,
-      llvm::SmallVectorImpl<const semantics::Symbol *> &isDeviceSyms) const;
+      llvm::SmallVectorImpl<const semantics::Symbol *> &hasDeviceSyms) const;
   bool processHint(mlir::omp::HintClauseOps &result) const;
   bool processInclusive(mlir::Location currentLocation,
                         mlir::omp::InclusiveClauseOps &result) const;
@@ -148,10 +150,6 @@ class ClauseProcessor {
 private:
   using ClauseIterator = List<Clause>::const_iterator;
 
-  /// Utility to find a clause within a range in the clause list.
-  template <typename T>
-  static ClauseIterator findClause(ClauseIterator begin, ClauseIterator end);
-
   /// Return the first instance of the given clause found in the clause list or
   /// `nullptr` if not present. If more than one instance is expected, use
   /// `findRepeatableClause` instead.
@@ -199,45 +197,17 @@ void ClauseProcessor::processTODO(mlir::Location currentLocation,
     (checkUnhandledClause(it->id, std::get_if<Ts>(&it->u)), ...);
 }
 
-template <typename T>
-ClauseProcessor::ClauseIterator
-ClauseProcessor::findClause(ClauseIterator begin, ClauseIterator end) {
-  for (ClauseIterator it = begin; it != end; ++it) {
-    if (std::get_if<T>(&it->u))
-      return it;
-  }
-
-  return end;
-}
-
 template <typename T>
 const T *
 ClauseProcessor::findUniqueClause(const parser::CharBlock **source) const {
-  ClauseIterator it = findClause<T>(clauses.begin(), clauses.end());
-  if (it != clauses.end()) {
-    if (source)
-      *source = &it->source;
-    return &std::get<T>(it->u);
-  }
-  return nullptr;
+  return ClauseFinder::findUniqueClause<T>(clauses, source);
 }
 
 template <typename T>
 bool ClauseProcessor::findRepeatableClause(
     std::function<void(const T &, const parser::CharBlock &source)> callbackFn)
     const {
-  bool found = false;
-  ClauseIterator nextIt, endIt = clauses.end();
-  for (ClauseIterator it = clauses.begin(); it != endIt; it = nextIt) {
-    nextIt = findClause<T>(it, endIt);
-
-    if (nextIt != endIt) {
-      callbackFn(std::get<T>(nextIt->u), nextIt->source);
-      found = true;
-      ++nextIt;
-    }
-  }
-  return found;
+  return ClauseFinder::findRepeatableClause<T>(clauses, callbackFn);
 }
 
 template <typename T>
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index 84092b75e0b3c..9fa9abd9e8ceb 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -159,7 +159,12 @@ std::optional<Object> getBaseObject(const Object &object,
       return Object{SymbolAndDesignatorExtractor::symbol_addr(comp->symbol()),
                     ea.Designate(evaluate::DataRef{
                         SymbolAndDesignatorExtractor::AsRvalueRef(*comp)})};
-    } else if (base.UnwrapSymbolRef()) {
+    } else if (auto *symRef = base.UnwrapSymbolRef()) {
+      // This is the base symbol of the array reference, which is the same
+      // as the symbol in the input object,
+      // e.g. A(i) is represented as {Symbol(A), Designator(ArrayRef(A, i))}.
+      // Here we have the Symbol(A), which is what we started with.
+      assert(&**symRef == object.sym());
       return std::nullopt;
     }
   } else {
@@ -217,6 +222,7 @@ MAKE_EMPTY_CLASS(Simd, Simd);
 MAKE_EMPTY_CLASS(Threads, Threads);
 MAKE_EMPTY_CLASS(UnifiedAddress, UnifiedAddress);
 MAKE_EMPTY_CLASS(UnifiedSharedMemory, UnifiedSharedMemory);
+MAKE_EMPTY_CLASS(SelfMaps, SelfMaps);
 MAKE_EMPTY_CLASS(Unknown, Unknown);
 MAKE_EMPTY_CLASS(Untied, Untied);
 MAKE_EMPTY_CLASS(Weak, Weak);
diff --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/lib/Lower/OpenMP/Clauses.h
index 23c984412df3e..fe453eb05cde2 100644
--- a/flang/lib/Lower/OpenMP/Clauses.h
+++ b/flang/lib/Lower/OpenMP/Clauses.h
@@ -286,6 +286,7 @@ using To = tomp::clause::ToT<TypeTy, IdTy, ExprTy>;
 using UnifiedAddress = tomp::clause::UnifiedAddressT<TypeTy, IdTy, ExprTy>;
 using UnifiedSharedMemory =
     tomp::clause::UnifiedSharedMemoryT<TypeTy, IdTy, ExprTy>;
+using SelfMaps = tomp::clause::SelfMapsT<TypeTy, IdTy, ExprTy>;
 using Uniform = tomp::clause::UniformT<TypeTy, IdTy, ExprTy>;
 using Unknown = tomp::clause::UnknownT<TypeTy, IdTy, ExprTy>;
 using Untied = tomp::clause::UntiedT<TypeTy, IdTy, ExprTy>;
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 781b0dfceff9e..b88454c45da85 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -257,6 +257,11 @@ void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) {
     return;
 
   if (mlir::isa<mlir::omp::WsloopOp>(op) || mlir::isa<mlir::omp::SimdOp>(op)) {
+    mlir::omp::LoopRelatedClauseOps result;
+    llvm::SmallVector<const semantics::Symbol *> iv;
+    collectLoopRelatedInfo(converter, converter.getCurrentLocation(), eval,
+                           clauses, result, iv);
+
     // Update the original variable just before exiting the worksharing
     // loop. Conversion as follows:
     //
@@ -280,9 +285,8 @@ void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) {
     mlir::Value cmpOp;
     llvm::SmallVector<mlir::Value> vs;
     vs.reserve(loopOp.getIVs().size());
-    for (auto [iv, ub, step] :
-         llvm::zip_equal(loopOp.getIVs(), loopOp.getLoopUpperBounds(),
-                         loopOp.getLoopSteps())) {
+    for (auto [iv, ub, step] : llvm::zip_equal(
+             loopOp.getIVs(), result.loopUpperBounds, result.loopSteps)) {
       // v = iv + step
       // cmp = step < 0 ? v < ub : v > ub
       mlir::Value v = firOpBuilder.create<mlir::arith::AddIOp>(loc, iv, step);
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index b1568cc12a05a..77786742b3e13 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -321,6 +321,7 @@ static void bindEntryBlockArgs(lower::AbstractConverter &converter,
   // Process in clause name alphabetical order to match block arguments order.
   // Do not bind host_eval variables because they cannot be used inside of the
   // corresponding region, except for very specific cases handled separately.
+  bindMapLike(args.hasDeviceAddr.syms, op.getHasDeviceAddrBlockArgs());
   bindPrivateLike(args.inReduction.syms, args.inReduction.vars,
                   op.getInReductionBlockArgs());
   bindMapLike(args.map.syms, op.getMapBlockArgs());
@@ -1208,27 +1209,27 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info,
     if (privatize) {
       // DataSharingProcessor::processStep2() may create operations before/after
       // the one passed as argument. We need to treat loop wrappers and their
-      // nested loop as a unit, so we need to pass the top level wrapper (if
+      // nested loop as a unit, so we need to pass the bottom level wrapper (if
       // present). Otherwise, these operations will be inserted within a
       // wrapper region.
-      mlir::Operation *privatizationTopLevelOp = &op;
+      mlir::Operation *privatizationBottomLevelOp = &op;
       if (auto loopNest = llvm::dyn_cast<mlir::omp::LoopNestOp>(op)) {
         llvm::SmallVector<mlir::omp::LoopWrapperInterface> wrappers;
         loopNest.gatherWrappers(wrappers);
         if (!wrappers.empty())
-          privatizationTopLevelOp = &*wrappers.back();
+          privatizationBottomLevelOp = &*wrappers.front();
       }
 
       if (!info.dsp) {
         assert(tempDsp.has_value());
-        tempDsp->processStep2(privatizationTopLevelOp, isLoop);
+        tempDsp->processStep2(privatizationBottomLevelOp, isLoop);
       } else {
         if (isLoop && regionArgs.size() > 0) {
           for (const auto &regionArg : regionArgs) {
             info.dsp->pushLoopIV(info.converter.getSymbolAddress(*regionArg));
           }
         }
-        info.dsp->processStep2(privatizationTopLevelOp, isLoop);
+        info.dsp->processStep2(privatizationBottomLevelOp, isLoop);
       }
     }
   }
@@ -1654,7 +1655,7 @@ static void genTargetClauses(
   cp.processBare(clauseOps);
   cp.processDepend(clauseOps);
   cp.processDevice(stmtCtx, clauseOps);
-  cp.processHasDeviceAddr(clauseOps, hasDeviceAddrSyms);
+  cp.processHasDeviceAddr(stmtCtx, clauseOps, hasDeviceAddrSyms);
   if (!hostEvalInfo.empty()) {
     // Only process host_eval if compiling for the host device.
     processHostEvalClauses(converter, semaCtx, stmtCtx, eval, loc);
@@ -2200,6 +2201,10 @@ genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
     if (dsp.getAllSymbolsToPrivatize().contains(&sym))
       return;
 
+    // These symbols are mapped individually in processHasDeviceAddr.
+    if (llvm::is_contained(hasDeviceAddrSyms, &sym))
+      return;
+
     // Structure component symbols don't have bindings, and can only be
     // explicitly mapped individually. If a member is captured implicitly
     // we map the entirety of the derived type when we find its symbol.
@@ -2290,10 +2295,13 @@ genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
 
   auto targetOp = firOpBuilder.create<mlir::omp::TargetOp>(loc, clauseOps);
 
-  llvm::SmallVector<mlir::Value> mapBaseValues;
+  llvm::SmallVector<mlir::Value> hasDeviceAddrBaseValues, mapBaseValues;
+  extractMappedBaseValues(clauseOps.hasDeviceAddrVars, hasDeviceAddrBaseValues);
   extractMappedBaseValues(clauseOps.mapVars, mapBaseValues);
 
   EntryBlockArgs args;
+  args.hasDeviceAddr.syms = hasDeviceAddrSyms;
+  args.hasDeviceAddr.vars = hasDeviceAddrBaseValues;
   args.hostEvalVars = clauseOps.hostEvalVars;
   // TODO: Add in_reduction syms and vars.
   args.map.syms = mapSyms;
@@ -2741,18 +2749,20 @@ static void genCompositeDistributeParallelDoSimd(
   genParallelClauses(converter, semaCtx, stmtCtx, parallelItem->clauses, loc,
                      parallelClauseOps, parallelReductionSyms);
 
-  DataSharingProcessor dsp(converter, semaCtx, simdItem->clauses, eval,
-                           /*shouldCollectPreDeterminedSymbols=*/true,
-                           /*useDelayedPrivatization=*/true, symTable);
-  dsp.processStep1(&parallelClauseOps);
+  DataSharingProcessor parallelItemDSP(
+      converter, semaCtx, parallelItem->clauses, eval,
+      /*shouldCollectPreDeterminedSymbols=*/false,
+      /*useDelayedPrivatization=*/true, symTable);
+  parallelItemDSP.processStep1(&parallelClauseOps);
 
   EntryBlockArgs parallelArgs;
-  parallelArgs.priv.syms = dsp.getDelayedPrivSymbols();
+  parallelArgs.priv.syms = parallelItemDSP.getDelayedPrivSymbols();
   parallelArgs.priv.vars = parallelClauseOps.privateVars;
   parallelArgs.reduction.syms = parallelReductionSyms;
   parallelArgs.reduction.vars = parallelClauseOps.reductionVars;
   genParallelOp(converter, symTable, semaCtx, eval, loc, queue, parallelItem,
-                parallelClauseOps, parallelArgs, &dsp, /*isComposite=*/true);
+                parallelClauseOps, parallelArgs, &parallelItemDSP,
+                /*isComposite=*/true);
 
   // Clause processing.
   mlir::omp::DistributeOperands distributeClauseOps;
@@ -2769,6 +2779,11 @@ static void genCompositeDistributeParallelDoSimd(
   genSimdClauses(converter, semaCtx, simdItem->clauses, loc, simdClauseOps,
                  simdReductionSyms);
 
+  DataSharingProcessor simdItemDSP(converter, semaCtx, simdItem->clauses, eval,
+                                   /*shouldCollectPreDeterminedSymbols=*/true,
+                                   /*useDelayedPrivatization=*/true, symTable);
+  simdItemDSP.processStep1(&simdClauseOps);
+
   mlir::omp::LoopNestOperands loopNestClauseOps;
   llvm::SmallVector<const semantics::Symbol *> iv;
   genLoopNestClauses(converter, semaCtx, eval, simdItem->clauses, loc,
@@ -2790,7 +2805,8 @@ static void genCompositeDistributeParallelDoSimd(
   wsloopOp.setComposite(/*val=*/true);
 
   EntryBlockArgs simdArgs;
-  // TODO: Add private syms and vars.
+  simdArgs.priv.syms = simdItemDSP.getDelayedPrivSymbols();
+  simdArgs.priv.vars = simdClauseOps.privateVars;
   simdArgs.reduction.syms = simdReductionSyms;
   simdArgs.reduction.vars = simdClauseOps.reductionVars;
   auto simdOp =
@@ -2802,7 +2818,8 @@ static void genCompositeDistributeParallelDoSimd(
                 {{distributeOp, distributeArgs},
                  {wsloopOp, wsloopArgs},
                  {simdOp, simdArgs}},
-                llvm::omp::Directive::OMPD_distribute_parallel_do_simd, dsp);
+                llvm::omp::Directive::OMPD_distribute_parallel_do_simd,
+                simdItemDSP);
 }
 
 static void genCompositeDistributeSimd(lower::AbstractConverter &converter,
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
index f83079eb68688..aa1187669ca8b 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
@@ -617,10 +617,13 @@ void ReductionProcessor::processReductionArguments(
     // this isn't the same as the by-val and by-ref passing later in the
     // pipeline. Both styles assume that the variable is a reference at
     // this point
-    assert(mlir::isa<fir::ReferenceType>(symVal.getType()) &&
-           "reduction input var is a reference");
+    assert(fir::isa_ref_type(symVal.getType()) &&
+           "reduction input var is passed by reference");
+    mlir::Type elementType = fir::dyn_cast_ptrEleTy(symVal.getType());
+    mlir::Type refTy = fir::ReferenceType::get(elementType);
 
-    reductionVars.push_back(symVal);
+    reductionVars.push_back(
+        builder.createConvert(currentLocation, refTy, symVal));
     reduceVarByRef.push_back(doReductionByRef(symVal));
   }
 
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index 48bcf492fd368..f5bec97433a4a 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -14,6 +14,7 @@
 
 #include "Clauses.h"
 
+#include "ClauseFinder.h"
 #include <flang/Lower/AbstractConverter.h>
 #include <flang/Lower/ConvertType.h>
 #include <flang/Lower/DirectivesCommon.h>
@@ -302,7 +303,7 @@ mlir::Value createParentSymAndGenIntermediateMaps(
   /// Checks if an omp::Object is an array expression with a subscript, e.g.
   /// array(1,2).
   auto isArrayExprWithSubscript = [](omp::Object obj) {
-    if (auto maybeRef = evaluate::ExtractDataRef(*obj.ref())) {
+    if (auto maybeRef = evaluate::ExtractDataRef(obj.ref())) {
       evaluate::DataRef ref = *maybeRef;
       if (auto *arr = std::get_if<evaluate::ArrayRef>(&ref.u))
         return !arr->subscript().empty();
@@ -453,7 +454,7 @@ getComponentObject(std::optional<Object> object,
   if (!object)
     return std::nullopt;
 
-  auto ref = evaluate::ExtractDataRef(*object.value().ref());
+  auto ref = evaluate::ExtractDataRef(object.value().ref());
   if (!ref)
     return std::nullopt;
 
@@ -595,6 +596,80 @@ void lastprivateModifierNotSupported(const omp::clause::Lastprivate &lastp,
   }
 }
 
+static void convertLoopBounds(lower::AbstractConverter &converter,
+                              mlir::Location loc,
+                              mlir::omp::LoopRelatedClauseOps &result,
+                              std::size_t loopVarTypeSize) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+  // The types of lower bound, upper bound, and step are converted into the
+  // type of the loop variable if necessary.
+  mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize);
+  for (unsigned it = 0; it < (unsigned)result.loopLowerBounds.size(); it++) {
+    result.loopLowerBounds[it] = firOpBuilder.createConvert(
+        loc, loopVarType, result.loopLowerBounds[it]);
+    result.loopUpperBounds[it] = firOpBuilder.createConvert(
+        loc, loopVarType, result.loopUpperBounds[it]);
+    result.loopSteps[it] =
+        firOpBuilder.createConvert(loc, loopVarType, result.loopSteps[it]);
+  }
+}
+
+bool collectLoopRelatedInfo(
+    lower::AbstractConverter &converter, mlir::Location currentLocation,
+    lower::pft::Evaluation &eval, const omp::List<omp::Clause> &clauses,
+    mlir::omp::LoopRelatedClauseOps &result,
+    llvm::SmallVectorImpl<const semantics::Symbol *> &iv) {
+  bool found = false;
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+
+  // Collect the loops to collapse.
+  lower::pft::Evaluation *doConstructEval = &eval.getFirstNestedEvaluation();
+  if (doConstructEval->getIf<parser::DoConstruct>()->IsDoConcurrent()) {
+    TODO(currentLocation, "Do Concurrent in Worksharing loop construct");
+  }
+
+  std::int64_t collapseValue = 1l;
+  if (auto *clause =
+          ClauseFinder::findUniqueClause<omp::clause::Collapse>(clauses)) {
+    collapseValue = evaluate::ToInt64(clause->v).value();
+    found = true;
+  }
+
+  std::size_t loopVarTypeSize = 0;
+  do {
+    lower::pft::Evaluation *doLoop =
+        &doConstructEval->getFirstNestedEvaluation();
+    auto *doStmt = doLoop->getIf<parser::NonLabelDoStmt>();
+    assert(doStmt && "Expected do loop to be in the nested evaluation");
+    const auto &loopControl =
+        std::get<std::optional<parser::LoopControl>>(doStmt->t);
+    const parser::LoopControl::Bounds *bounds =
+        std::get_if<parser::LoopControl::Bounds>(&loopControl->u);
+    assert(bounds && "Expected bounds for worksharing do loop");
+    lower::StatementContext stmtCtx;
+    result.loopLowerBounds.push_back(fir::getBase(
+        converter.genExprValue(*semantics::GetExpr(bounds->lower), stmtCtx)));
+    result.loopUpperBounds.push_back(fir::getBase(
+        converter.genExprValue(*semantics::GetExpr(bounds->upper), stmtCtx)));
+    if (bounds->step) {
+      result.loopSteps.push_back(fir::getBase(
+          converter.genExprValue(*semantics::GetExpr(bounds->step), stmtCtx)));
+    } else { // If `step` is not present, assume it as `1`.
+      result.loopSteps.push_back(firOpBuilder.createIntegerConstant(
+          currentLocation, firOpBuilder.getIntegerType(32), 1));
+    }
+    iv.push_back(bounds->name.thing.symbol);
+    loopVarTypeSize = std::max(loopVarTypeSize,
+                               bounds->name.thing.symbol->GetUltimate().size());
+    collapseValue--;
+    doConstructEval =
+        &*std::next(doConstructEval->getNestedEvaluations().begin());
+  } while (collapseValue > 0);
+
+  convertLoopBounds(converter, currentLocation, result, loopVarTypeSize);
+
+  return found;
+}
 } // namespace omp
 } // namespace lower
 } // namespace Fortran
diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index 3943eb633b04e..30b4613837b9a 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -163,6 +163,11 @@ void genObjectList(const ObjectList &objects,
 void lastprivateModifierNotSupported(const omp::clause::Lastprivate &lastp,
                                      mlir::Location loc);
 
+bool collectLoopRelatedInfo(
+    lower::AbstractConverter &converter, mlir::Location currentLocation,
+    lower::pft::Evaluation &eval, const omp::List<omp::Clause> &clauses,
+    mlir::omp::LoopRelatedClauseOps &result,
+    llvm::SmallVectorImpl<const semantics::Symbol *> &iv);
 } // namespace omp
 } // namespace lower
 } // namespace Fortran
diff --git a/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp b/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
index 2bc79100b03b6..2ff8ec05782c1 100644
--- a/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
+++ b/flang/lib/Optimizer/Analysis/AliasAnalysis.cpp
@@ -51,40 +51,6 @@ static bool hasGlobalOpTargetAttr(mlir::Value v, fir::AddrOfOp op) {
       v, fir::GlobalOp::getTargetAttrName(globalOpName));
 }
 
-static mlir::Value
-getOriginalDef(mlir::Value v,
-               fir::AliasAnalysis::Source::Attributes &attributes,
-               bool &isCapturedInInternalProcedure, bool &approximateSource) {
-  mlir::Operation *defOp;
-  bool breakFromLoop = false;
-  while (!breakFromLoop && (defOp = v.getDefiningOp())) {
-    mlir::Type ty = defOp->getResultTypes()[0];
-    llvm::TypeSwitch<Operation *>(defOp)
-        .Case<fir::ConvertOp>([&](fir::ConvertOp op) { v = op.getValue(); })
-        .Case<fir::DeclareOp, hlfir::DeclareOp>([&](auto op) {
-          v = op.getMemref();
-          auto varIf = llvm::cast<fir::FortranVariableOpInterface>(defOp);
-          attributes |= getAttrsFromVariable(varIf);
-          isCapturedInInternalProcedure |=
-              varIf.isCapturedInInternalProcedure();
-        })
-        .Case<fir::CoordinateOp>([&](auto op) {
-          if (fir::AliasAnalysis::isPointerReference(ty))
-            attributes.set(fir::AliasAnalysis::Attribute::Pointer);
-          v = op->getOperand(0);
-          approximateSource = true;
-        })
-        .Case<hlfir::DesignateOp>([&](hlfir::DesignateOp op) {
-          auto varIf = llvm::cast<fir::FortranVariableOpInterface>(defOp);
-          attributes |= getAttrsFromVariable(varIf);
-          v = op.getMemref();
-          approximateSource = true;
-        })
-        .Default([&](auto op) { breakFromLoop = true; });
-  }
-  return v;
-}
-
 static bool isEvaluateInMemoryBlockArg(mlir::Value v) {
   if (auto evalInMem = llvm::dyn_cast_or_null<hlfir::EvaluateInMemoryOp>(
           v.getParentRegion()->getParentOp()))
@@ -621,38 +587,34 @@ AliasAnalysis::Source AliasAnalysis::getSource(mlir::Value v,
             if (mlir::isa<fir::PointerType>(boxTy.getEleTy()))
               attributes.set(Attribute::Pointer);
 
-            auto def = getOriginalDef(op.getMemref(), attributes,
-                                      isCapturedInInternalProcedure,
-                                      approximateSource);
-            if (auto addrOfOp = def.template getDefiningOp<fir::AddrOfOp>()) {
-              global = addrOfOp.getSymbol();
-
-              if (hasGlobalOpTargetAttr(def, addrOfOp))
-                attributes.set(Attribute::Target);
+            auto boxSrc = getSource(op.getMemref());
+            attributes |= boxSrc.attributes;
+            approximateSource |= boxSrc.approximateSource;
+            isCapturedInInternalProcedure |=
+                boxSrc.isCapturedInInternalProcedure;
 
+            global = llvm::dyn_cast<mlir::SymbolRefAttr>(boxSrc.origin.u);
+            if (global) {
               type = SourceKind::Global;
-            }
-            // TODO: Add support to fir.allocmem
-            else if (auto allocOp =
-                         def.template getDefiningOp<fir::AllocaOp>()) {
-              v = def;
-              defOp = v.getDefiningOp();
-              type = SourceKind::Allocate;
-            } else if (isDummyArgument(def)) {
-              defOp = nullptr;
-              v = def;
             } else {
-              type = SourceKind::Indirect;
+              auto def = llvm::cast<mlir::Value>(boxSrc.origin.u);
+              // TODO: Add support to fir.allocmem
+              if (auto allocOp = def.template getDefiningOp<fir::AllocaOp>()) {
+                v = def;
+                defOp = v.getDefiningOp();
+                type = SourceKind::Allocate;
+              } else if (isDummyArgument(def)) {
+                defOp = nullptr;
+                v = def;
+              } else {
+                type = SourceKind::Indirect;
+              }
             }
-            // TODO: This assignment is redundant but somehow works around an
-            // apparent MSVC bug reporting "undeclared identifier" at the next
-            // "breakFromLoop = true;".  See
-            // <https://github.com/llvm/llvm-project/pull/127845#issuecomment-2669829610>.
             breakFromLoop = true;
-          } else {
-            // No further tracking for addresses loaded from memory for now.
-            type = SourceKind::Indirect;
+            return;
           }
+          // No further tracking for addresses loaded from memory for now.
+          type = SourceKind::Indirect;
           breakFromLoop = true;
         })
         .Case<fir::AddrOfOp>([&](auto op) {
diff --git a/flang/lib/Optimizer/Builder/CMakeLists.txt b/flang/lib/Optimizer/Builder/CMakeLists.txt
index f0563d092e3dc..31ae395805faf 100644
--- a/flang/lib/Optimizer/Builder/CMakeLists.txt
+++ b/flang/lib/Optimizer/Builder/CMakeLists.txt
@@ -18,6 +18,7 @@ add_flang_library(FIRBuilder
   Runtime/Assign.cpp
   Runtime/Character.cpp
   Runtime/Command.cpp
+  Runtime/CUDA/Descriptor.cpp
   Runtime/Derived.cpp
   Runtime/EnvironmentDefaults.cpp
   Runtime/Exceptions.cpp
diff --git a/flang/lib/Optimizer/Builder/CUFCommon.cpp b/flang/lib/Optimizer/Builder/CUFCommon.cpp
index 39848205f47af..5f286c04a7ca0 100644
--- a/flang/lib/Optimizer/Builder/CUFCommon.cpp
+++ b/flang/lib/Optimizer/Builder/CUFCommon.cpp
@@ -12,6 +12,7 @@
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/NVVMDialect.h"
+#include "mlir/Dialect/OpenACC/OpenACC.h"
 
 /// Retrieve or create the CUDA Fortran GPU module in the give in \p mod.
 mlir::gpu::GPUModuleOp cuf::getOrCreateGPUModule(mlir::ModuleOp mod,
@@ -31,25 +32,34 @@ mlir::gpu::GPUModuleOp cuf::getOrCreateGPUModule(mlir::ModuleOp mod,
   return gpuMod;
 }
 
-bool cuf::isInCUDADeviceContext(mlir::Operation *op) {
-  if (!op)
+bool cuf::isCUDADeviceContext(mlir::Operation *op) {
+  if (!op || !op->getParentRegion())
     return false;
-  if (op->getParentOfType<cuf::KernelOp>() ||
-      op->getParentOfType<mlir::gpu::GPUFuncOp>())
+  return isCUDADeviceContext(*op->getParentRegion());
+}
+
+// Check if the insertion point is currently in a device context. HostDevice
+// subprogram are not considered fully device context so it will return false
+// for it.
+// If the insertion point is inside an OpenACC region op, it is considered
+// device context.
+bool cuf::isCUDADeviceContext(mlir::Region &region) {
+  if (region.getParentOfType<cuf::KernelOp>())
+    return true;
+  if (region.getParentOfType<mlir::acc::ComputeRegionOpInterface>())
     return true;
-  if (auto funcOp = op->getParentOfType<mlir::func::FuncOp>()) {
-    if (auto cudaProcAttr = funcOp->getAttrOfType<cuf::ProcAttributeAttr>(
-            cuf::getProcAttrName())) {
-      return cudaProcAttr.getValue() != cuf::ProcAttribute::Host;
+  if (auto funcOp = region.getParentOfType<mlir::func::FuncOp>()) {
+    if (auto cudaProcAttr =
+            funcOp.getOperation()->getAttrOfType<cuf::ProcAttributeAttr>(
+                cuf::getProcAttrName())) {
+      return cudaProcAttr.getValue() != cuf::ProcAttribute::Host &&
+             cudaProcAttr.getValue() != cuf::ProcAttribute::HostDevice;
     }
   }
   return false;
 }
 
-bool cuf::isRegisteredDeviceGlobal(fir::GlobalOp op) {
-  if (op.getConstant())
-    return false;
-  auto attr = op.getDataAttr();
+bool cuf::isRegisteredDeviceAttr(std::optional<cuf::DataAttribute> attr) {
   if (attr && (*attr == cuf::DataAttribute::Device ||
                *attr == cuf::DataAttribute::Managed ||
                *attr == cuf::DataAttribute::Constant))
@@ -57,6 +67,12 @@ bool cuf::isRegisteredDeviceGlobal(fir::GlobalOp op) {
   return false;
 }
 
+bool cuf::isRegisteredDeviceGlobal(fir::GlobalOp op) {
+  if (op.getConstant())
+    return false;
+  return isRegisteredDeviceAttr(op.getDataAttr());
+}
+
 void cuf::genPointerSync(const mlir::Value box, fir::FirOpBuilder &builder) {
   if (auto declareOp = box.getDefiningOp<hlfir::DeclareOp>()) {
     if (auto addrOfOp = declareOp.getMemref().getDefiningOp<fir::AddrOfOp>()) {
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index c52b0cbaf2019..f57ed41fd785d 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -16,12 +16,14 @@
 #include "flang/Optimizer/Builder/IntrinsicCall.h"
 #include "flang/Common/static-multimap-view.h"
 #include "flang/Optimizer/Builder/BoxValue.h"
+#include "flang/Optimizer/Builder/CUFCommon.h"
 #include "flang/Optimizer/Builder/Character.h"
 #include "flang/Optimizer/Builder/Complex.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/MutableBox.h"
 #include "flang/Optimizer/Builder/PPCIntrinsicCall.h"
 #include "flang/Optimizer/Builder/Runtime/Allocatable.h"
+#include "flang/Optimizer/Builder/Runtime/CUDA/Descriptor.h"
 #include "flang/Optimizer/Builder/Runtime/Character.h"
 #include "flang/Optimizer/Builder/Runtime/Command.h"
 #include "flang/Optimizer/Builder/Runtime/Derived.h"
@@ -38,6 +40,7 @@
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIROpsSupport.h"
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
+#include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "flang/Optimizer/Support/FatalError.h"
 #include "flang/Optimizer/Support/Utils.h"
 #include "flang/Runtime/entry-names.h"
@@ -103,6 +106,34 @@ using I = IntrinsicLibrary;
 /// argument is an optional variable in the current scope).
 static constexpr bool handleDynamicOptional = true;
 
+/// TODO: Move all CUDA Fortran intrinsic handlers into its own file similar to
+/// PPC.
+static const char __ldca_i4x4[] = "__ldca_i4x4_";
+static const char __ldca_i8x2[] = "__ldca_i8x2_";
+static const char __ldca_r2x2[] = "__ldca_r2x2_";
+static const char __ldca_r4x4[] = "__ldca_r4x4_";
+static const char __ldca_r8x2[] = "__ldca_r8x2_";
+static const char __ldcg_i4x4[] = "__ldcg_i4x4_";
+static const char __ldcg_i8x2[] = "__ldcg_i8x2_";
+static const char __ldcg_r2x2[] = "__ldcg_r2x2_";
+static const char __ldcg_r4x4[] = "__ldcg_r4x4_";
+static const char __ldcg_r8x2[] = "__ldcg_r8x2_";
+static const char __ldcs_i4x4[] = "__ldcs_i4x4_";
+static const char __ldcs_i8x2[] = "__ldcs_i8x2_";
+static const char __ldcs_r2x2[] = "__ldcs_r2x2_";
+static const char __ldcs_r4x4[] = "__ldcs_r4x4_";
+static const char __ldcs_r8x2[] = "__ldcs_r8x2_";
+static const char __ldcv_i4x4[] = "__ldcv_i4x4_";
+static const char __ldcv_i8x2[] = "__ldcv_i8x2_";
+static const char __ldcv_r2x2[] = "__ldcv_r2x2_";
+static const char __ldcv_r4x4[] = "__ldcv_r4x4_";
+static const char __ldcv_r8x2[] = "__ldcv_r8x2_";
+static const char __ldlu_i4x4[] = "__ldlu_i4x4_";
+static const char __ldlu_i8x2[] = "__ldlu_i8x2_";
+static const char __ldlu_r2x2[] = "__ldlu_r2x2_";
+static const char __ldlu_r4x4[] = "__ldlu_r4x4_";
+static const char __ldlu_r8x2[] = "__ldlu_r8x2_";
+
 /// Table that drives the fir generation depending on the intrinsic or intrinsic
 /// module procedure one to one mapping with Fortran arguments. If no mapping is
 /// defined here for a generic intrinsic, genRuntimeCall will be called
@@ -111,6 +142,106 @@ static constexpr bool handleDynamicOptional = true;
 /// argument must not be lowered by value. In which case, the lowering rules
 /// should be provided for all the intrinsic arguments for completeness.
 static constexpr IntrinsicHandler handlers[]{
+    {"__ldca_i4x4",
+     &I::genCUDALDXXFunc<__ldca_i4x4, 4>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldca_i8x2",
+     &I::genCUDALDXXFunc<__ldca_i8x2, 2>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldca_r2x2",
+     &I::genCUDALDXXFunc<__ldca_r2x2, 2>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldca_r4x4",
+     &I::genCUDALDXXFunc<__ldca_r4x4, 4>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldca_r8x2",
+     &I::genCUDALDXXFunc<__ldca_r8x2, 2>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldcg_i4x4",
+     &I::genCUDALDXXFunc<__ldcg_i4x4, 4>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldcg_i8x2",
+     &I::genCUDALDXXFunc<__ldcg_i8x2, 2>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldcg_r2x2",
+     &I::genCUDALDXXFunc<__ldcg_r2x2, 2>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldcg_r4x4",
+     &I::genCUDALDXXFunc<__ldcg_r4x4, 4>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldcg_r8x2",
+     &I::genCUDALDXXFunc<__ldcg_r8x2, 2>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldcs_i4x4",
+     &I::genCUDALDXXFunc<__ldcs_i4x4, 4>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldcs_i8x2",
+     &I::genCUDALDXXFunc<__ldcs_i8x2, 2>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldcs_r2x2",
+     &I::genCUDALDXXFunc<__ldcs_r2x2, 2>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldcs_r4x4",
+     &I::genCUDALDXXFunc<__ldcs_r4x4, 4>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldcs_r8x2",
+     &I::genCUDALDXXFunc<__ldcs_r8x2, 2>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldcv_i4x4",
+     &I::genCUDALDXXFunc<__ldcv_i4x4, 4>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldcv_i8x2",
+     &I::genCUDALDXXFunc<__ldcv_i8x2, 2>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldcv_r2x2",
+     &I::genCUDALDXXFunc<__ldcv_r2x2, 2>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldcv_r4x4",
+     &I::genCUDALDXXFunc<__ldcv_r4x4, 4>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldcv_r8x2",
+     &I::genCUDALDXXFunc<__ldcv_r8x2, 2>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldlu_i4x4",
+     &I::genCUDALDXXFunc<__ldlu_i4x4, 4>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldlu_i8x2",
+     &I::genCUDALDXXFunc<__ldlu_i8x2, 2>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldlu_r2x2",
+     &I::genCUDALDXXFunc<__ldlu_r2x2, 2>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldlu_r4x4",
+     &I::genCUDALDXXFunc<__ldlu_r4x4, 4>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
+    {"__ldlu_r8x2",
+     &I::genCUDALDXXFunc<__ldlu_r8x2, 2>,
+     {{{"a", asAddr}}},
+     /*isElemental=*/false},
     {"abort", &I::genAbort},
     {"abs", &I::genAbs},
     {"achar", &I::genChar},
@@ -2737,15 +2868,17 @@ mlir::Value IntrinsicLibrary::genAtomicOr(mlir::Type resultType,
 }
 
 // ATOMICCAS
-mlir::Value IntrinsicLibrary::genAtomicCas(mlir::Type resultType,
-                                           llvm::ArrayRef<mlir::Value> args) {
+fir::ExtendedValue
+IntrinsicLibrary::genAtomicCas(mlir::Type resultType,
+                               llvm::ArrayRef<fir::ExtendedValue> args) {
   assert(args.size() == 3);
   auto successOrdering = mlir::LLVM::AtomicOrdering::acq_rel;
   auto failureOrdering = mlir::LLVM::AtomicOrdering::monotonic;
   auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(resultType.getContext());
 
-  mlir::Value arg1 = args[1];
-  mlir::Value arg2 = args[2];
+  mlir::Value arg0 = fir::getBase(args[0]);
+  mlir::Value arg1 = fir::getBase(args[1]);
+  mlir::Value arg2 = fir::getBase(args[2]);
 
   auto bitCastFloat = [&](mlir::Value arg) -> mlir::Value {
     if (mlir::isa<mlir::Float32Type>(arg.getType()))
@@ -2766,7 +2899,7 @@ mlir::Value IntrinsicLibrary::genAtomicCas(mlir::Type resultType,
   }
 
   auto address =
-      builder.create<mlir::UnrealizedConversionCastOp>(loc, llvmPtrTy, args[0])
+      builder.create<mlir::UnrealizedConversionCastOp>(loc, llvmPtrTy, arg0)
           .getResult(0);
   auto cmpxchg = builder.create<mlir::LLVM::AtomicCmpXchgOp>(
       loc, address, arg1, arg2, successOrdering, failureOrdering);
@@ -2783,13 +2916,16 @@ mlir::Value IntrinsicLibrary::genAtomicDec(mlir::Type resultType,
 }
 
 // ATOMICEXCH
-mlir::Value IntrinsicLibrary::genAtomicExch(mlir::Type resultType,
-                                            llvm::ArrayRef<mlir::Value> args) {
+fir::ExtendedValue
+IntrinsicLibrary::genAtomicExch(mlir::Type resultType,
+                                llvm::ArrayRef<fir::ExtendedValue> args) {
   assert(args.size() == 2);
-  assert(args[1].getType().isIntOrFloat());
+  mlir::Value arg0 = fir::getBase(args[0]);
+  mlir::Value arg1 = fir::getBase(args[1]);
+  assert(arg1.getType().isIntOrFloat());
 
   mlir::LLVM::AtomicBinOp binOp = mlir::LLVM::AtomicBinOp::xchg;
-  return genAtomBinOp(builder, loc, binOp, args[0], args[1]);
+  return genAtomBinOp(builder, loc, binOp, arg0, arg1);
 }
 
 mlir::Value IntrinsicLibrary::genAtomicInc(mlir::Type resultType,
@@ -2824,13 +2960,13 @@ mlir::Value IntrinsicLibrary::genAtomicMin(mlir::Type resultType,
 }
 
 // ATOMICXOR
-mlir::Value IntrinsicLibrary::genAtomicXor(mlir::Type resultType,
-                                           llvm::ArrayRef<mlir::Value> args) {
+fir::ExtendedValue
+IntrinsicLibrary::genAtomicXor(mlir::Type resultType,
+                               llvm::ArrayRef<fir::ExtendedValue> args) {
   assert(args.size() == 2);
-  assert(mlir::isa<mlir::IntegerType>(args[1].getType()));
-
-  mlir::LLVM::AtomicBinOp binOp = mlir::LLVM::AtomicBinOp::_xor;
-  return genAtomBinOp(builder, loc, binOp, args[0], args[1]);
+  mlir::Value arg0 = fir::getBase(args[0]);
+  mlir::Value arg1 = fir::getBase(args[1]);
+  return genAtomBinOp(builder, loc, mlir::LLVM::AtomicBinOp::_xor, arg0, arg1);
 }
 
 // ASSOCIATED
@@ -3254,6 +3390,17 @@ void IntrinsicLibrary::genCFPointer(llvm::ArrayRef<fir::ExtendedValue> args) {
 
   fir::factory::associateMutableBox(builder, loc, *fPtr, getCPtrExtVal(*fPtr),
                                     /*lbounds=*/mlir::ValueRange{});
+
+  // If the pointer is a registered CUDA fortran variable, the descriptor needs
+  // to be synced.
+  if (auto declare = mlir::dyn_cast_or_null<hlfir::DeclareOp>(
+          fPtr->getAddr().getDefiningOp()))
+    if (declare.getMemref().getDefiningOp() &&
+        mlir::isa<fir::AddrOfOp>(declare.getMemref().getDefiningOp()))
+      if (cuf::isRegisteredDeviceAttr(declare.getDataAttr()) &&
+          !cuf::isCUDADeviceContext(builder.getRegion()))
+        fir::runtime::cuda::genSyncGlobalDescriptor(builder, loc,
+                                                    declare.getMemref());
 }
 
 // C_F_PROCPOINTER
@@ -3530,6 +3677,30 @@ IntrinsicLibrary::genCshift(mlir::Type resultType,
   return readAndAddCleanUp(resultMutableBox, resultType, "CSHIFT");
 }
 
+// __LDCA, __LDCS, __LDLU, __LDCV
+template <const char *fctName, int extent>
+fir::ExtendedValue
+IntrinsicLibrary::genCUDALDXXFunc(mlir::Type resultType,
+                                  llvm::ArrayRef<fir::ExtendedValue> args) {
+  assert(args.size() == 1);
+  mlir::Type resTy = fir::SequenceType::get(extent, resultType);
+  mlir::Value arg = fir::getBase(args[0]);
+  mlir::Value res = builder.create<fir::AllocaOp>(loc, resTy);
+  if (mlir::isa<fir::BaseBoxType>(arg.getType()))
+    arg = builder.create<fir::BoxAddrOp>(loc, arg);
+  mlir::Type refResTy = fir::ReferenceType::get(resTy);
+  mlir::FunctionType ftype =
+      mlir::FunctionType::get(arg.getContext(), {refResTy, refResTy}, {});
+  auto funcOp = builder.createFunction(loc, fctName, ftype);
+  llvm::SmallVector<mlir::Value> funcArgs;
+  funcArgs.push_back(res);
+  funcArgs.push_back(arg);
+  builder.create<fir::CallOp>(loc, funcOp, funcArgs);
+  mlir::Value ext =
+      builder.createIntegerConstant(loc, builder.getIndexType(), extent);
+  return fir::ArrayBoxValue(res, {ext});
+}
+
 // DATE_AND_TIME
 void IntrinsicLibrary::genDateAndTime(llvm::ArrayRef<fir::ExtendedValue> args) {
   assert(args.size() == 4 && "date_and_time has 4 args");
diff --git a/flang/lib/Optimizer/Builder/Runtime/CUDA/Descriptor.cpp b/flang/lib/Optimizer/Builder/Runtime/CUDA/Descriptor.cpp
new file mode 100644
index 0000000000000..90662c094c65e
--- /dev/null
+++ b/flang/lib/Optimizer/Builder/Runtime/CUDA/Descriptor.cpp
@@ -0,0 +1,34 @@
+
+//===-- Allocatable.cpp -- Allocatable statements lowering ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Builder/Runtime/CUDA/Descriptor.h"
+#include "flang/Optimizer/Builder/FIRBuilder.h"
+#include "flang/Optimizer/Builder/Runtime/RTBuilder.h"
+#include "flang/Runtime/CUDA/descriptor.h"
+
+using namespace Fortran::runtime::cuda;
+
+void fir::runtime::cuda::genSyncGlobalDescriptor(fir::FirOpBuilder &builder,
+                                                 mlir::Location loc,
+                                                 mlir::Value hostPtr) {
+  mlir::func::FuncOp callee =
+      fir::runtime::getRuntimeFunc<mkRTKey(CUFSyncGlobalDescriptor)>(loc,
+                                                                     builder);
+  auto fTy = callee.getFunctionType();
+  mlir::Value sourceFile = fir::factory::locationToFilename(builder, loc);
+  mlir::Value sourceLine =
+      fir::factory::locationToLineNo(builder, loc, fTy.getInput(2));
+  llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
+      builder, loc, fTy, hostPtr, sourceFile, sourceLine)};
+  builder.create<fir::CallOp>(loc, callee, args);
+}
diff --git a/flang/lib/Optimizer/Builder/TemporaryStorage.cpp b/flang/lib/Optimizer/Builder/TemporaryStorage.cpp
index 48c2cb2181a0b..9d2e9837a3df8 100644
--- a/flang/lib/Optimizer/Builder/TemporaryStorage.cpp
+++ b/flang/lib/Optimizer/Builder/TemporaryStorage.cpp
@@ -357,25 +357,33 @@ void fir::factory::AnyVectorSubscriptStack::destroy(
 }
 
 //===----------------------------------------------------------------------===//
-// fir::factory::AnyDescriptorAddressStack implementation.
+// fir::factory::AnyAddressStack implementation.
 //===----------------------------------------------------------------------===//
 
-fir::factory::AnyDescriptorAddressStack::AnyDescriptorAddressStack(
-    mlir::Location loc, fir::FirOpBuilder &builder,
-    mlir::Type descriptorAddressType)
+fir::factory::AnyAddressStack::AnyAddressStack(mlir::Location loc,
+                                               fir::FirOpBuilder &builder,
+                                               mlir::Type addressType)
     : AnyValueStack(loc, builder, builder.getIntPtrType()),
-      descriptorAddressType{descriptorAddressType} {}
-
-void fir::factory::AnyDescriptorAddressStack::pushValue(
-    mlir::Location loc, fir::FirOpBuilder &builder, mlir::Value variable) {
-  mlir::Value cast =
-      builder.createConvert(loc, builder.getIntPtrType(), variable);
+      addressType{addressType} {}
+
+void fir::factory::AnyAddressStack::pushValue(mlir::Location loc,
+                                              fir::FirOpBuilder &builder,
+                                              mlir::Value variable) {
+  mlir::Value cast = variable;
+  if (auto boxProcType = llvm::dyn_cast<fir::BoxProcType>(variable.getType())) {
+    cast =
+        builder.create<fir::BoxAddrOp>(loc, boxProcType.getEleTy(), variable);
+  }
+  cast = builder.createConvert(loc, builder.getIntPtrType(), cast);
   static_cast<AnyValueStack *>(this)->pushValue(loc, builder, cast);
 }
 
-mlir::Value
-fir::factory::AnyDescriptorAddressStack::fetch(mlir::Location loc,
-                                               fir::FirOpBuilder &builder) {
+mlir::Value fir::factory::AnyAddressStack::fetch(mlir::Location loc,
+                                                 fir::FirOpBuilder &builder) {
   mlir::Value addr = static_cast<AnyValueStack *>(this)->fetch(loc, builder);
-  return builder.createConvert(loc, descriptorAddressType, addr);
+  if (auto boxProcType = llvm::dyn_cast<fir::BoxProcType>(addressType)) {
+    mlir::Value cast = builder.createConvert(loc, boxProcType.getEleTy(), addr);
+    return builder.create<fir::EmboxProcOp>(loc, boxProcType, cast);
+  }
+  return builder.createConvert(loc, addressType, addr);
 }
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index a2743edd7844a..151836cb6b172 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -688,6 +688,28 @@ struct CmpcOpConversion : public fir::FIROpConversion<fir::CmpcOp> {
   }
 };
 
+static mlir::LogicalResult
+convertFpToInteger(mlir::ConversionPatternRewriter &rewriter,
+                   fir::ConvertOp opToReplace, mlir::Value op0,
+                   mlir::Type fromTy, mlir::Type toTy, bool isSigned) {
+  auto fromBits = mlir::LLVM::getPrimitiveTypeSizeInBits(fromTy);
+  auto toBits = mlir::LLVM::getPrimitiveTypeSizeInBits(toTy);
+  const bool useSaturatedIntrinsic = fromBits > toBits;
+  if (useSaturatedIntrinsic) {
+    const char* intrinsicStr = isSigned ? "llvm.fptosi.sat" : "llvm.fptoui.sat";
+    auto intrinsicName = mlir::StringAttr::get(opToReplace.getContext(), intrinsicStr);
+    rewriter.replaceOpWithNewOp<mlir::LLVM::CallIntrinsicOp>(
+        opToReplace, toTy, intrinsicName, op0);
+  } else {
+    if (isSigned) {
+      rewriter.replaceOpWithNewOp<mlir::LLVM::FPToSIOp>(opToReplace, toTy, op0);
+    } else {
+      rewriter.replaceOpWithNewOp<mlir::LLVM::FPToUIOp>(opToReplace, toTy, op0);
+    }
+  }
+  return mlir::success();
+}
+
 /// convert value of from-type to value of to-type
 struct ConvertOpConversion : public fir::FIROpConversion<fir::ConvertOp> {
   using FIROpConversion::FIROpConversion;
@@ -835,11 +857,11 @@ struct ConvertOpConversion : public fir::FIROpConversion<fir::ConvertOp> {
         return mlir::success();
       }
       if (mlir::isa<mlir::IntegerType>(toTy)) {
-        if (toTy.isUnsignedInteger())
-          rewriter.replaceOpWithNewOp<mlir::LLVM::FPToUIOp>(convert, toTy, op0);
-        else
-          rewriter.replaceOpWithNewOp<mlir::LLVM::FPToSIOp>(convert, toTy, op0);
-        return mlir::success();
+        // NOTE: We are checking the fir type here because toTy is an LLVM type
+        // which is signless, and we need to use the intrinsic that matches the
+        // sign of the output in fir.
+        const bool isSigned = !toFirTy.isUnsignedInteger();
+        return convertFpToInteger(rewriter, convert, op0, fromTy, toTy, isSigned);
       }
     } else if (mlir::isa<mlir::IntegerType>(fromTy)) {
       // Integer to integer conversion.
@@ -982,7 +1004,8 @@ struct EmboxCharOpConversion : public fir::FIROpConversion<fir::EmboxCharOp> {
 template <typename ModuleOp>
 static mlir::SymbolRefAttr
 getMallocInModule(ModuleOp mod, fir::AllocMemOp op,
-                  mlir::ConversionPatternRewriter &rewriter) {
+                  mlir::ConversionPatternRewriter &rewriter,
+                  mlir::Type indexType) {
   static constexpr char mallocName[] = "malloc";
   if (auto mallocFunc =
           mod.template lookupSymbol<mlir::LLVM::LLVMFuncOp>(mallocName))
@@ -992,7 +1015,6 @@ getMallocInModule(ModuleOp mod, fir::AllocMemOp op,
     return mlir::SymbolRefAttr::get(userMalloc);
 
   mlir::OpBuilder moduleBuilder(mod.getBodyRegion());
-  auto indexType = mlir::IntegerType::get(op.getContext(), 64);
   auto mallocDecl = moduleBuilder.create<mlir::LLVM::LLVMFuncOp>(
       op.getLoc(), mallocName,
       mlir::LLVM::LLVMFunctionType::get(getLlvmPtrType(op.getContext()),
@@ -1002,12 +1024,13 @@ getMallocInModule(ModuleOp mod, fir::AllocMemOp op,
 }
 
 /// Return the LLVMFuncOp corresponding to the standard malloc call.
-static mlir::SymbolRefAttr
-getMalloc(fir::AllocMemOp op, mlir::ConversionPatternRewriter &rewriter) {
+static mlir::SymbolRefAttr getMalloc(fir::AllocMemOp op,
+                                     mlir::ConversionPatternRewriter &rewriter,
+                                     mlir::Type indexType) {
   if (auto mod = op->getParentOfType<mlir::gpu::GPUModuleOp>())
-    return getMallocInModule(mod, op, rewriter);
+    return getMallocInModule(mod, op, rewriter, indexType);
   auto mod = op->getParentOfType<mlir::ModuleOp>();
-  return getMallocInModule(mod, op, rewriter);
+  return getMallocInModule(mod, op, rewriter, indexType);
 }
 
 /// Helper function for generating the LLVM IR that computes the distance
@@ -1067,7 +1090,12 @@ struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
     for (mlir::Value opnd : adaptor.getOperands())
       size = rewriter.create<mlir::LLVM::MulOp>(
           loc, ity, size, integerCast(loc, rewriter, ity, opnd));
-    heap->setAttr("callee", getMalloc(heap, rewriter));
+    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
+    auto mallocTy =
+        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
+    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
+      size = integerCast(loc, rewriter, mallocTy, size);
+    heap->setAttr("callee", getMalloc(heap, rewriter, mallocTy));
     rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
         heap, ::getLlvmPtrType(heap.getContext()), size,
         addLLVMOpBundleAttrs(rewriter, heap->getAttrs(), 1));
@@ -2116,7 +2144,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
       unsigned dim = iter.index();
       mlir::Value lb = one;
       if (!lbounds.empty()) {
-        lb = lbounds[dim];
+        lb = integerCast(loc, rewriter, lowerTy().indexType(), lbounds[dim]);
         auto extentIsEmpty = rewriter.create<mlir::LLVM::ICmpOp>(
             loc, mlir::LLVM::ICmpPredicate::eq, extent, zero);
         lb = rewriter.create<mlir::LLVM::SelectOp>(loc, extentIsEmpty, one, lb);
@@ -3539,6 +3567,36 @@ struct StoreOpConversion : public fir::FIROpConversion<fir::StoreOp> {
   }
 };
 
+/// `fir.copy` --> `llvm.memcpy` or `llvm.memmove`
+struct CopyOpConversion : public fir::FIROpConversion<fir::CopyOp> {
+  using FIROpConversion::FIROpConversion;
+
+  llvm::LogicalResult
+  matchAndRewrite(fir::CopyOp copy, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::Location loc = copy.getLoc();
+    mlir::Value llvmSource = adaptor.getSource();
+    mlir::Value llvmDestination = adaptor.getDestination();
+    mlir::Type i64Ty = mlir::IntegerType::get(rewriter.getContext(), 64);
+    mlir::Type copyTy = fir::unwrapRefType(copy.getSource().getType());
+    mlir::Value copySize =
+        genTypeStrideInBytes(loc, i64Ty, rewriter, convertType(copyTy));
+
+    mlir::LLVM::AliasAnalysisOpInterface newOp;
+    if (copy.getNoOverlap())
+      newOp = rewriter.create<mlir::LLVM::MemcpyOp>(
+          loc, llvmDestination, llvmSource, copySize, /*isVolatile=*/false);
+    else
+      newOp = rewriter.create<mlir::LLVM::MemmoveOp>(
+          loc, llvmDestination, llvmSource, copySize, /*isVolatile=*/false);
+
+    // TODO: propagate TBAA once FirAliasTagOpInterface added to CopyOp.
+    attachTBAATag(newOp, copyTy, copyTy, nullptr);
+    rewriter.eraseOp(copy);
+    return mlir::success();
+  }
+};
+
 namespace {
 
 /// Convert `fir.unboxchar` into two `llvm.extractvalue` instructions. One for
@@ -4142,11 +4200,11 @@ void fir::populateFIRToLLVMConversionPatterns(
       BoxOffsetOpConversion, BoxProcHostOpConversion, BoxRankOpConversion,
       BoxTypeCodeOpConversion, BoxTypeDescOpConversion, CallOpConversion,
       CmpcOpConversion, ConvertOpConversion, CoordinateOpConversion,
-      DTEntryOpConversion, DeclareOpConversion, DivcOpConversion,
-      EmboxOpConversion, EmboxCharOpConversion, EmboxProcOpConversion,
-      ExtractValueOpConversion, FieldIndexOpConversion, FirEndOpConversion,
-      FreeMemOpConversion, GlobalLenOpConversion, GlobalOpConversion,
-      InsertOnRangeOpConversion, IsPresentOpConversion,
+      CopyOpConversion, DTEntryOpConversion, DeclareOpConversion,
+      DivcOpConversion, EmboxOpConversion, EmboxCharOpConversion,
+      EmboxProcOpConversion, ExtractValueOpConversion, FieldIndexOpConversion,
+      FirEndOpConversion, FreeMemOpConversion, GlobalLenOpConversion,
+      GlobalOpConversion, InsertOnRangeOpConversion, IsPresentOpConversion,
       LenParamIndexOpConversion, LoadOpConversion, MulcOpConversion,
       NegcOpConversion, NoReassocOpConversion, SelectCaseOpConversion,
       SelectOpConversion, SelectRankOpConversion, SelectTypeOpConversion,
diff --git a/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp b/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
index 5c9da0321bcc4..43ed60fc0c292 100644
--- a/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
+++ b/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
@@ -541,37 +541,37 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
             callOp.getContext(),
             mlir::TypeRange{newInTypes}.drop_front(dropFront), newResTys));
         newCall = rewriter->create<fir::CallOp>(loc, newResTys, newOpers);
-        // Set ABI argument attributes on call operation since they are not
-        // accessible via a FuncOp in indirect calls.
-        if (hasByValOrSRetArgs(newInTyAndAttrs)) {
-          llvm::SmallVector<mlir::Attribute> argAttrsArray;
-          for (const auto &arg :
-               llvm::ArrayRef<fir::CodeGenSpecifics::TypeAndAttr>(
-                   newInTyAndAttrs)
-                   .drop_front(dropFront)) {
-            mlir::NamedAttrList argAttrs;
-            const auto &attr = std::get<fir::CodeGenSpecifics::Attributes>(arg);
-            if (attr.isByVal()) {
-              mlir::Type elemType =
-                  fir::dyn_cast_ptrOrBoxEleTy(std::get<mlir::Type>(arg));
-              argAttrs.set(mlir::LLVM::LLVMDialect::getByValAttrName(),
-                           mlir::TypeAttr::get(elemType));
-            } else if (attr.isSRet()) {
-              mlir::Type elemType =
-                  fir::dyn_cast_ptrOrBoxEleTy(std::get<mlir::Type>(arg));
-              argAttrs.set(mlir::LLVM::LLVMDialect::getStructRetAttrName(),
-                           mlir::TypeAttr::get(elemType));
-              if (auto align = attr.getAlignment()) {
-                argAttrs.set(mlir::LLVM::LLVMDialect::getAlignAttrName(),
-                             rewriter->getIntegerAttr(
-                                 rewriter->getIntegerType(32), align));
-              }
-            }
-            argAttrsArray.emplace_back(
-                argAttrs.getDictionary(rewriter->getContext()));
+      }
+      // Always set ABI argument attributes on call operations, even when
+      // direct, as required by
+      // https://llvm.org/docs/LangRef.html#parameter-attributes.
+      if (hasByValOrSRetArgs(newInTyAndAttrs)) {
+        llvm::SmallVector<mlir::Attribute> argAttrsArray;
+        for (const auto &arg :
+             llvm::ArrayRef<fir::CodeGenSpecifics::TypeAndAttr>(newInTyAndAttrs)
+                 .drop_front(dropFront)) {
+          mlir::NamedAttrList argAttrs;
+          const auto &attr = std::get<fir::CodeGenSpecifics::Attributes>(arg);
+          if (attr.isByVal()) {
+            mlir::Type elemType =
+                fir::dyn_cast_ptrOrBoxEleTy(std::get<mlir::Type>(arg));
+            argAttrs.set(mlir::LLVM::LLVMDialect::getByValAttrName(),
+                         mlir::TypeAttr::get(elemType));
+          } else if (attr.isSRet()) {
+            mlir::Type elemType =
+                fir::dyn_cast_ptrOrBoxEleTy(std::get<mlir::Type>(arg));
+            argAttrs.set(mlir::LLVM::LLVMDialect::getStructRetAttrName(),
+                         mlir::TypeAttr::get(elemType));
+          }
+          if (auto align = attr.getAlignment()) {
+            argAttrs.set(
+                mlir::LLVM::LLVMDialect::getAlignAttrName(),
+                rewriter->getIntegerAttr(rewriter->getIntegerType(32), align));
           }
-          newCall.setArgAttrsAttr(rewriter->getArrayAttr(argAttrsArray));
+          argAttrsArray.emplace_back(
+              argAttrs.getDictionary(rewriter->getContext()));
         }
+        newCall.setArgAttrsAttr(rewriter->getArrayAttr(argAttrsArray));
       }
       LLVM_DEBUG(llvm::dbgs() << "replacing call with " << newCall << '\n');
       if (wrap)
diff --git a/flang/lib/Optimizer/CodeGen/TypeConverter.cpp b/flang/lib/Optimizer/CodeGen/TypeConverter.cpp
index 89f498433806e..1a1d3a8cfb870 100644
--- a/flang/lib/Optimizer/CodeGen/TypeConverter.cpp
+++ b/flang/lib/Optimizer/CodeGen/TypeConverter.cpp
@@ -28,10 +28,27 @@
 
 namespace fir {
 
+static mlir::LowerToLLVMOptions MakeLowerOptions(mlir::ModuleOp module) {
+  llvm::StringRef dataLayoutString;
+  auto dataLayoutAttr = module->template getAttrOfType<mlir::StringAttr>(
+      mlir::LLVM::LLVMDialect::getDataLayoutAttrName());
+  if (dataLayoutAttr)
+    dataLayoutString = dataLayoutAttr.getValue();
+
+  auto options = mlir::LowerToLLVMOptions(module.getContext());
+  auto llvmDL = llvm::DataLayout(dataLayoutString);
+  if (llvmDL.getPointerSizeInBits(0) == 32) {
+    // FIXME: Should translateDataLayout in the MLIR layer be doing this?
+    options.overrideIndexBitwidth(32);
+  }
+  options.dataLayout = llvmDL;
+  return options;
+}
+
 LLVMTypeConverter::LLVMTypeConverter(mlir::ModuleOp module, bool applyTBAA,
                                      bool forceUnifiedTBAATree,
                                      const mlir::DataLayout &dl)
-    : mlir::LLVMTypeConverter(module.getContext()),
+    : mlir::LLVMTypeConverter(module.getContext(), MakeLowerOptions(module)),
       kindMapping(getKindMapping(module)),
       specifics(CodeGenSpecifics::get(
           module.getContext(), getTargetTriple(module), getKindMapping(module),
diff --git a/flang/lib/Optimizer/Dialect/FIRAttr.cpp b/flang/lib/Optimizer/Dialect/FIRAttr.cpp
index 4c78e223b4178..d190b307dc7f2 100644
--- a/flang/lib/Optimizer/Dialect/FIRAttr.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRAttr.cpp
@@ -300,5 +300,5 @@ void FIROpsDialect::registerAttributes() {
                 FortranProcedureFlagsEnumAttr, FortranVariableFlagsAttr,
                 LowerBoundAttr, PointIntervalAttr, RealAttr, ReduceAttr,
                 SubclassAttr, UpperBoundAttr, LocationKindAttr,
-                LocationKindArrayAttr>();
+                LocationKindArrayAttr, PackArrayHeuristicsAttr>();
 }
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index 7efb733eb565c..033d6453a619a 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -380,11 +380,16 @@ llvm::LogicalResult fir::AllocMemOp::verify() {
 
 // CHARACTERs and derived types with LEN PARAMETERs are dependent types that
 // require runtime values to fully define the type of an object.
-static bool validTypeParams(mlir::Type dynTy, mlir::ValueRange typeParams) {
+static bool validTypeParams(mlir::Type dynTy, mlir::ValueRange typeParams,
+                            bool allowParamsForBox = false) {
   dynTy = fir::unwrapAllRefAndSeqType(dynTy);
-  // A box value will contain type parameter values itself.
-  if (mlir::isa<fir::BoxType>(dynTy))
-    return typeParams.size() == 0;
+  if (mlir::isa<fir::BaseBoxType>(dynTy)) {
+    // A box value will contain type parameter values itself.
+    if (!allowParamsForBox)
+      return typeParams.size() == 0;
+
+    dynTy = fir::getFortranElementType(dynTy);
+  }
   // Derived type must have all type parameters satisfied.
   if (auto recTy = mlir::dyn_cast<fir::RecordType>(dynTy))
     return typeParams.size() == recTy.getNumLenParams();
@@ -3940,6 +3945,26 @@ void fir::StoreOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
   build(builder, result, value, memref, {});
 }
 
+//===----------------------------------------------------------------------===//
+// CopyOp
+//===----------------------------------------------------------------------===//
+
+void fir::CopyOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
+                        mlir::Value source, mlir::Value destination,
+                        bool noOverlap) {
+  mlir::UnitAttr noOverlapAttr =
+      noOverlap ? builder.getUnitAttr() : mlir::UnitAttr{};
+  build(builder, result, source, destination, noOverlapAttr);
+}
+
+llvm::LogicalResult fir::CopyOp::verify() {
+  mlir::Type sourceType = fir::unwrapRefType(getSource().getType());
+  mlir::Type destinationType = fir::unwrapRefType(getDestination().getType());
+  if (sourceType != destinationType)
+    return emitOpError("source and destination must have the same value type");
+  return mlir::success();
+}
+
 //===----------------------------------------------------------------------===//
 // StringLitOp
 //===----------------------------------------------------------------------===//
@@ -4541,6 +4566,111 @@ llvm::LogicalResult fir::DeclareOp::verify() {
   return fortranVar.verifyDeclareLikeOpImpl(getMemref());
 }
 
+//===----------------------------------------------------------------------===//
+// PackArrayOp
+//===----------------------------------------------------------------------===//
+
+llvm::LogicalResult fir::PackArrayOp::verify() {
+  mlir::Type arrayType = getArray().getType();
+  if (!validTypeParams(arrayType, getTypeparams(), /*allowParamsForBox=*/true))
+    return emitOpError("invalid type parameters");
+
+  if (getInnermost() && fir::getBoxRank(arrayType) == 1)
+    return emitOpError(
+        "'innermost' is invalid for 1D arrays, use 'whole' instead");
+  return mlir::success();
+}
+
+void fir::PackArrayOp::getEffects(
+    llvm::SmallVectorImpl<
+        mlir::SideEffects::EffectInstance<mlir::MemoryEffects::Effect>>
+        &effects) {
+  if (getStack())
+    effects.emplace_back(
+        mlir::MemoryEffects::Allocate::get(),
+        mlir::SideEffects::AutomaticAllocationScopeResource::get());
+  else
+    effects.emplace_back(mlir::MemoryEffects::Allocate::get(),
+                         mlir::SideEffects::DefaultResource::get());
+
+  if (!getNoCopy())
+    effects.emplace_back(mlir::MemoryEffects::Read::get(),
+                         mlir::SideEffects::DefaultResource::get());
+}
+
+static mlir::ParseResult
+parsePackArrayConstraints(mlir::OpAsmParser &parser, mlir::IntegerAttr &maxSize,
+                          mlir::IntegerAttr &maxElementSize,
+                          mlir::IntegerAttr &minStride) {
+  mlir::OperationName opName = mlir::OperationName(
+      fir::PackArrayOp::getOperationName(), parser.getContext());
+  struct {
+    llvm::StringRef name;
+    mlir::IntegerAttr &ref;
+  } attributes[] = {
+      {fir::PackArrayOp::getMaxSizeAttrName(opName), maxSize},
+      {fir::PackArrayOp::getMaxElementSizeAttrName(opName), maxElementSize},
+      {fir::PackArrayOp::getMinStrideAttrName(opName), minStride}};
+
+  mlir::NamedAttrList parsedAttrs;
+  if (succeeded(parser.parseOptionalAttrDict(parsedAttrs))) {
+    for (auto parsedAttr : parsedAttrs) {
+      for (auto opAttr : attributes) {
+        if (parsedAttr.getName() == opAttr.name)
+          opAttr.ref = mlir::cast<mlir::IntegerAttr>(parsedAttr.getValue());
+      }
+    }
+    return mlir::success();
+  }
+  return mlir::failure();
+}
+
+static void printPackArrayConstraints(mlir::OpAsmPrinter &p,
+                                      fir::PackArrayOp &op,
+                                      const mlir::IntegerAttr &maxSize,
+                                      const mlir::IntegerAttr &maxElementSize,
+                                      const mlir::IntegerAttr &minStride) {
+  llvm::SmallVector<mlir::NamedAttribute> attributes;
+  if (maxSize)
+    attributes.emplace_back(op.getMaxSizeAttrName(), maxSize);
+  if (maxElementSize)
+    attributes.emplace_back(op.getMaxElementSizeAttrName(), maxElementSize);
+  if (minStride)
+    attributes.emplace_back(op.getMinStrideAttrName(), minStride);
+
+  p.printOptionalAttrDict(attributes);
+}
+
+//===----------------------------------------------------------------------===//
+// UnpackArrayOp
+//===----------------------------------------------------------------------===//
+
+llvm::LogicalResult fir::UnpackArrayOp::verify() {
+  if (auto packOp = getTemp().getDefiningOp<fir::PackArrayOp>())
+    if (getStack() != packOp.getStack())
+      return emitOpError() << "the pack operation uses different memory for "
+                              "the temporary (stack vs heap): "
+                           << *packOp.getOperation() << "\n";
+  return mlir::success();
+}
+
+void fir::UnpackArrayOp::getEffects(
+    llvm::SmallVectorImpl<
+        mlir::SideEffects::EffectInstance<mlir::MemoryEffects::Effect>>
+        &effects) {
+  if (getStack())
+    effects.emplace_back(
+        mlir::MemoryEffects::Free::get(),
+        mlir::SideEffects::AutomaticAllocationScopeResource::get());
+  else
+    effects.emplace_back(mlir::MemoryEffects::Free::get(),
+                         mlir::SideEffects::DefaultResource::get());
+
+  if (!getNoCopy())
+    effects.emplace_back(mlir::MemoryEffects::Write::get(),
+                         mlir::SideEffects::DefaultResource::get());
+}
+
 //===----------------------------------------------------------------------===//
 // FIROpsDialect
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/Dialect/FIRType.cpp b/flang/lib/Optimizer/Dialect/FIRType.cpp
index d499df0b08139..dc0bee9b060c9 100644
--- a/flang/lib/Optimizer/Dialect/FIRType.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRType.cpp
@@ -554,70 +554,92 @@ std::string getTypeAsString(mlir::Type ty, const fir::KindMapping &kindMap,
   llvm::raw_string_ostream name{buf};
   if (!prefix.empty())
     name << "_";
-  while (ty) {
-    if (fir::isa_trivial(ty)) {
-      if (mlir::isa<mlir::IndexType>(ty)) {
-        name << "idx";
-      } else if (ty.isIntOrIndex()) {
-        name << 'i' << ty.getIntOrFloatBitWidth();
-      } else if (mlir::isa<mlir::FloatType>(ty)) {
-        name << 'f' << ty.getIntOrFloatBitWidth();
-      } else if (auto cplxTy = mlir::dyn_cast_or_null<mlir::ComplexType>(ty)) {
-        name << 'z';
-        auto floatTy = mlir::cast<mlir::FloatType>(cplxTy.getElementType());
-        name << floatTy.getWidth();
-      } else if (auto logTy = mlir::dyn_cast_or_null<fir::LogicalType>(ty)) {
-        name << 'l' << kindMap.getLogicalBitsize(logTy.getFKind());
+
+  std::function<void(mlir::Type)> appendTypeName = [&](mlir::Type ty) {
+    while (ty) {
+      if (fir::isa_trivial(ty)) {
+        if (mlir::isa<mlir::IndexType>(ty)) {
+          name << "idx";
+        } else if (ty.isIntOrIndex()) {
+          name << 'i' << ty.getIntOrFloatBitWidth();
+        } else if (mlir::isa<mlir::FloatType>(ty)) {
+          name << 'f' << ty.getIntOrFloatBitWidth();
+        } else if (auto cplxTy =
+                       mlir::dyn_cast_or_null<mlir::ComplexType>(ty)) {
+          name << 'z';
+          auto floatTy = mlir::cast<mlir::FloatType>(cplxTy.getElementType());
+          name << floatTy.getWidth();
+        } else if (auto logTy = mlir::dyn_cast_or_null<fir::LogicalType>(ty)) {
+          name << 'l' << kindMap.getLogicalBitsize(logTy.getFKind());
+        } else {
+          llvm::report_fatal_error("unsupported type");
+        }
+        break;
+      } else if (mlir::isa<mlir::NoneType>(ty)) {
+        name << "none";
+        break;
+      } else if (auto charTy = mlir::dyn_cast_or_null<fir::CharacterType>(ty)) {
+        name << 'c' << kindMap.getCharacterBitsize(charTy.getFKind());
+        if (charTy.getLen() == fir::CharacterType::unknownLen())
+          name << "xU";
+        else if (charTy.getLen() != fir::CharacterType::singleton())
+          name << "x" << charTy.getLen();
+        break;
+      } else if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(ty)) {
+        for (auto extent : seqTy.getShape()) {
+          if (extent == fir::SequenceType::getUnknownExtent())
+            name << "Ux";
+          else
+            name << extent << 'x';
+        }
+        ty = seqTy.getEleTy();
+      } else if (auto refTy = mlir::dyn_cast_or_null<fir::ReferenceType>(ty)) {
+        name << "ref_";
+        ty = refTy.getEleTy();
+      } else if (auto ptrTy = mlir::dyn_cast_or_null<fir::PointerType>(ty)) {
+        name << "ptr_";
+        ty = ptrTy.getEleTy();
+      } else if (auto ptrTy =
+                     mlir::dyn_cast_or_null<fir::LLVMPointerType>(ty)) {
+        name << "llvmptr_";
+        ty = ptrTy.getEleTy();
+      } else if (auto heapTy = mlir::dyn_cast_or_null<fir::HeapType>(ty)) {
+        name << "heap_";
+        ty = heapTy.getEleTy();
+      } else if (auto classTy = mlir::dyn_cast_or_null<fir::ClassType>(ty)) {
+        name << "class_";
+        ty = classTy.getEleTy();
+      } else if (auto boxTy = mlir::dyn_cast_or_null<fir::BoxType>(ty)) {
+        name << "box_";
+        ty = boxTy.getEleTy();
+      } else if (auto boxcharTy =
+                     mlir::dyn_cast_or_null<fir::BoxCharType>(ty)) {
+        name << "boxchar_";
+        ty = boxcharTy.getEleTy();
+      } else if (auto boxprocTy =
+                     mlir::dyn_cast_or_null<fir::BoxProcType>(ty)) {
+        name << "boxproc_";
+        auto procTy = mlir::dyn_cast<mlir::FunctionType>(boxprocTy.getEleTy());
+        assert(procTy.getNumResults() <= 1 &&
+               "function type with more than one result");
+        for (const auto &result : procTy.getResults())
+          appendTypeName(result);
+        name << "_args";
+        for (const auto &arg : procTy.getInputs()) {
+          name << '_';
+          appendTypeName(arg);
+        }
+        break;
+      } else if (auto recTy = mlir::dyn_cast_or_null<fir::RecordType>(ty)) {
+        name << "rec_" << recTy.getName();
+        break;
       } else {
         llvm::report_fatal_error("unsupported type");
       }
-      break;
-    } else if (mlir::isa<mlir::NoneType>(ty)) {
-      name << "none";
-      break;
-    } else if (auto charTy = mlir::dyn_cast_or_null<fir::CharacterType>(ty)) {
-      name << 'c' << kindMap.getCharacterBitsize(charTy.getFKind());
-      if (charTy.getLen() == fir::CharacterType::unknownLen())
-        name << "xU";
-      else if (charTy.getLen() != fir::CharacterType::singleton())
-        name << "x" << charTy.getLen();
-      break;
-    } else if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(ty)) {
-      for (auto extent : seqTy.getShape()) {
-        if (extent == fir::SequenceType::getUnknownExtent())
-          name << "Ux";
-        else
-          name << extent << 'x';
-      }
-      ty = seqTy.getEleTy();
-    } else if (auto refTy = mlir::dyn_cast_or_null<fir::ReferenceType>(ty)) {
-      name << "ref_";
-      ty = refTy.getEleTy();
-    } else if (auto ptrTy = mlir::dyn_cast_or_null<fir::PointerType>(ty)) {
-      name << "ptr_";
-      ty = ptrTy.getEleTy();
-    } else if (auto ptrTy = mlir::dyn_cast_or_null<fir::LLVMPointerType>(ty)) {
-      name << "llvmptr_";
-      ty = ptrTy.getEleTy();
-    } else if (auto heapTy = mlir::dyn_cast_or_null<fir::HeapType>(ty)) {
-      name << "heap_";
-      ty = heapTy.getEleTy();
-    } else if (auto classTy = mlir::dyn_cast_or_null<fir::ClassType>(ty)) {
-      name << "class_";
-      ty = classTy.getEleTy();
-    } else if (auto boxTy = mlir::dyn_cast_or_null<fir::BoxType>(ty)) {
-      name << "box_";
-      ty = boxTy.getEleTy();
-    } else if (auto boxcharTy = mlir::dyn_cast_or_null<fir::BoxCharType>(ty)) {
-      name << "boxchar_";
-      ty = boxcharTy.getEleTy();
-    } else if (auto recTy = mlir::dyn_cast_or_null<fir::RecordType>(ty)) {
-      name << "rec_" << recTy.getName();
-      break;
-    } else {
-      llvm::report_fatal_error("unsupported type");
     }
-  }
+  };
+
+  appendTypeName(ty);
   return buf;
 }
 
diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
index 383e6a2630537..8851a3a7187b9 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
@@ -1891,18 +1891,33 @@ llvm::LogicalResult hlfir::RegionAssignOp::verify() {
   return mlir::success();
 }
 
-bool hlfir::RegionAssignOp::isPointerAssignment() {
+static mlir::Type
+getNonVectorSubscriptedLhsType(hlfir::RegionAssignOp regionAssign) {
+  hlfir::YieldOp yieldOp = mlir::dyn_cast_or_null<hlfir::YieldOp>(
+      getTerminator(regionAssign.getLhsRegion()));
+  return yieldOp ? yieldOp.getEntity().getType() : mlir::Type{};
+}
+
+bool hlfir::RegionAssignOp::isPointerObjectAssignment() {
   if (!getUserDefinedAssignment().empty())
     return false;
-  hlfir::YieldOp yieldOp =
-      mlir::dyn_cast_or_null<hlfir::YieldOp>(getTerminator(getLhsRegion()));
-  if (!yieldOp)
+  mlir::Type lhsType = getNonVectorSubscriptedLhsType(*this);
+  return lhsType && hlfir::isFortranPointerObjectType(lhsType);
+}
+
+bool hlfir::RegionAssignOp::isProcedurePointerAssignment() {
+  if (!getUserDefinedAssignment().empty())
     return false;
-  mlir::Type lhsType = yieldOp.getEntity().getType();
-  if (!hlfir::isBoxAddressType(lhsType))
+  mlir::Type lhsType = getNonVectorSubscriptedLhsType(*this);
+  return lhsType && hlfir::isFortranProcedurePointerType(lhsType);
+}
+
+bool hlfir::RegionAssignOp::isPointerAssignment() {
+  if (!getUserDefinedAssignment().empty())
     return false;
-  auto baseBoxType = llvm::cast<fir::BaseBoxType>(fir::unwrapRefType(lhsType));
-  return baseBoxType.isPointer();
+  mlir::Type lhsType = getNonVectorSubscriptedLhsType(*this);
+  return lhsType && (hlfir::isFortranPointerObjectType(lhsType) ||
+                     hlfir::isFortranProcedurePointerType(lhsType));
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
index 7561daefa3b83..5cae7cf443c86 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
@@ -1277,11 +1277,13 @@ void OrderedAssignmentRewriter::saveNonVectorSubscriptedAddress(
         [&] { temp = insertSavedEntity(region, fir::factory::SSARegister{}); });
   else
     doBeforeLoopNest([&] {
-      if (var.isMutableBox())
-        temp =
-            insertSavedEntity(region, fir::factory::AnyDescriptorAddressStack{
-                                          loc, builder, var.getType()});
+      if (var.isMutableBox() || var.isProcedure() || var.isProcedurePointer())
+        // Store single C pointer to entity.
+        temp = insertSavedEntity(
+            region, fir::factory::AnyAddressStack{loc, builder, var.getType()});
       else
+        // Store the base address and dynamic shape/length/type information
+        // as descriptor.
         temp = insertSavedEntity(region, fir::factory::AnyVariableStack{
                                              loc, builder, var.getType()});
     });
diff --git a/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp b/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp
index ab4dc582d5804..2f9c01a129210 100644
--- a/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp
+++ b/flang/lib/Optimizer/OpenMP/MapInfoFinalization.cpp
@@ -243,20 +243,29 @@ class MapInfoFinalizationPass
   /// where necessary, although it does not seem strictly required.
   unsigned long getDescriptorMapType(unsigned long mapTypeFlag,
                                      mlir::Operation *target) {
+    using mapFlags = llvm::omp::OpenMPOffloadMappingFlags;
     if (llvm::isa_and_nonnull<mlir::omp::TargetExitDataOp,
                               mlir::omp::TargetUpdateOp>(target))
       return mapTypeFlag;
 
-    bool hasImplicitMap =
-        (llvm::omp::OpenMPOffloadMappingFlags(mapTypeFlag) &
-         llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT) ==
-        llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
+    mapFlags flags = mapFlags::OMP_MAP_TO |
+                     (mapFlags(mapTypeFlag) &
+                      (mapFlags::OMP_MAP_IMPLICIT | mapFlags::OMP_MAP_CLOSE |
+                       mapFlags::OMP_MAP_ALWAYS));
+    return llvm::to_underlying(flags);
+  }
 
-    return llvm::to_underlying(
-        hasImplicitMap
-            ? llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
-                  llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT
-            : llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO);
+  /// Check if the mapOp is present in the HasDeviceAddr clause on
+  /// the userOp. Only applies to TargetOp.
+  bool isHasDeviceAddr(mlir::omp::MapInfoOp mapOp, mlir::Operation *userOp) {
+    assert(userOp && "Expecting non-null argument");
+    if (auto targetOp = llvm::dyn_cast<mlir::omp::TargetOp>(userOp)) {
+      for (mlir::Value hda : targetOp.getHasDeviceAddrVars()) {
+        if (hda.getDefiningOp() == mapOp)
+          return true;
+      }
+    }
+    return false;
   }
 
   mlir::omp::MapInfoOp genDescriptorMemberMaps(mlir::omp::MapInfoOp op,
@@ -268,11 +277,11 @@ class MapInfoFinalizationPass
     // TODO: map the addendum segment of the descriptor, similarly to the
     // base address/data pointer member.
     mlir::Value descriptor = getDescriptorFromBoxMap(op, builder);
-    auto baseAddr = genBaseAddrMap(descriptor, op.getBounds(),
-                                   op.getMapType().value_or(0), builder);
+
     mlir::ArrayAttr newMembersAttr;
     mlir::SmallVector<mlir::Value> newMembers;
     llvm::SmallVector<llvm::SmallVector<int64_t>> memberIndices;
+    bool IsHasDeviceAddr = isHasDeviceAddr(op, target);
 
     if (!mapMemberUsers.empty() || !op.getMembers().empty())
       getMemberIndicesAsVectors(
@@ -286,6 +295,12 @@ class MapInfoFinalizationPass
     // member information to now have one new member for the base address, or
     // we are expanding a parent that is a descriptor and we have to adjust
     // all of its members to reflect the insertion of the base address.
+    //
+    // If we're expanding a top-level descriptor for a map operation that
+    // resulted from "has_device_addr" clause, then we want the base pointer
+    // from the descriptor to be used verbatim, i.e. without additional
+    // remapping. To avoid this remapping, simply don't generate any map
+    // information for the descriptor members.
     if (!mapMemberUsers.empty()) {
       // Currently, there should only be one user per map when this pass
       // is executed. Either a parent map, holding the current map in its
@@ -296,6 +311,8 @@ class MapInfoFinalizationPass
       assert(mapMemberUsers.size() == 1 &&
              "OMPMapInfoFinalization currently only supports single users of a "
              "MapInfoOp");
+      auto baseAddr = genBaseAddrMap(descriptor, op.getBounds(),
+                                     op.getMapType().value_or(0), builder);
       ParentAndPlacement mapUser = mapMemberUsers[0];
       adjustMemberIndices(memberIndices, mapUser.index);
       llvm::SmallVector<mlir::Value> newMemberOps;
@@ -307,7 +324,9 @@ class MapInfoFinalizationPass
       mapUser.parent.getMembersMutable().assign(newMemberOps);
       mapUser.parent.setMembersIndexAttr(
           builder.create2DI64ArrayAttr(memberIndices));
-    } else {
+    } else if (!IsHasDeviceAddr) {
+      auto baseAddr = genBaseAddrMap(descriptor, op.getBounds(),
+                                     op.getMapType().value_or(0), builder);
       newMembers.push_back(baseAddr);
       if (!op.getMembers().empty()) {
         for (auto &indices : memberIndices)
@@ -321,15 +340,26 @@ class MapInfoFinalizationPass
       }
     }
 
+    // Descriptors for objects listed on the `has_device_addr` will always
+    // be copied. This is because the descriptor can be rematerialized by the
+    // compiler, and so the address of the descriptor for a given object at
+    // one place in the code may differ from that address in another place.
+    // The contents of the descriptor (the base address in particular) will
+    // remain unchanged though.
+    uint64_t MapType = op.getMapType().value_or(0);
+    if (IsHasDeviceAddr) {
+      MapType |= llvm::to_underlying(
+          llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS);
+    }
+
     mlir::omp::MapInfoOp newDescParentMapOp =
         builder.create<mlir::omp::MapInfoOp>(
             op->getLoc(), op.getResult().getType(), descriptor,
             mlir::TypeAttr::get(fir::unwrapRefType(descriptor.getType())),
             /*varPtrPtr=*/mlir::Value{}, newMembers, newMembersAttr,
             /*bounds=*/mlir::SmallVector<mlir::Value>{},
-            builder.getIntegerAttr(
-                builder.getIntegerType(64, false),
-                getDescriptorMapType(op.getMapType().value_or(0), target)),
+            builder.getIntegerAttr(builder.getIntegerType(64, false),
+                                   getDescriptorMapType(MapType, target)),
             /*mapperId*/ mlir::FlatSymbolRefAttr(), op.getMapCaptureTypeAttr(),
             op.getNameAttr(),
             /*partial_map=*/builder.getBoolAttr(false));
@@ -448,6 +478,12 @@ class MapInfoFinalizationPass
       addOperands(useDevPtrMutableOpRange, target,
                   argIface.getUseDevicePtrBlockArgsStart() +
                       argIface.numUseDevicePtrBlockArgs());
+    } else if (auto targetOp = llvm::dyn_cast<mlir::omp::TargetOp>(target)) {
+      mlir::MutableOperandRange hasDevAddrMutableOpRange =
+          targetOp.getHasDeviceAddrVarsMutable();
+      addOperands(hasDevAddrMutableOpRange, target,
+                  argIface.getHasDeviceAddrBlockArgsStart() +
+                      argIface.numHasDeviceAddrBlockArgs());
     }
   }
 
diff --git a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
index 2ab2d84f1643d..0fbec8a204b8d 100644
--- a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
@@ -8,6 +8,7 @@
 
 #include "flang/Optimizer/Transforms/CUFOpConversion.h"
 #include "flang/Optimizer/Builder/CUFCommon.h"
+#include "flang/Optimizer/Builder/Runtime/CUDA/Descriptor.h"
 #include "flang/Optimizer/Builder/Runtime/RTBuilder.h"
 #include "flang/Optimizer/CodeGen/TypeConverter.h"
 #include "flang/Optimizer/Dialect/CUF/CUFOps.h"
@@ -904,16 +905,7 @@ struct CUFSyncDescriptorOpConversion
 
     auto hostAddr = builder.create<fir::AddrOfOp>(
         loc, fir::ReferenceType::get(globalOp.getType()), op.getGlobalName());
-    mlir::func::FuncOp callee =
-        fir::runtime::getRuntimeFunc<mkRTKey(CUFSyncGlobalDescriptor)>(loc,
-                                                                       builder);
-    auto fTy = callee.getFunctionType();
-    mlir::Value sourceFile = fir::factory::locationToFilename(builder, loc);
-    mlir::Value sourceLine =
-        fir::factory::locationToLineNo(builder, loc, fTy.getInput(2));
-    llvm::SmallVector<mlir::Value> args{fir::runtime::createArguments(
-        builder, loc, fTy, hostAddr, sourceFile, sourceLine)};
-    builder.create<fir::CallOp>(loc, callee, args);
+    fir::runtime::cuda::genSyncGlobalDescriptor(builder, loc, hostAddr);
     op.erase();
     return mlir::success();
   }
diff --git a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp
index 3baac62dd69f2..dd4d1783dac37 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp
@@ -1279,7 +1279,7 @@ void SimplifyIntrinsicsPass::runOnOperation() {
   fir::KindMapping kindMap = fir::getKindMapping(module);
   module.walk([&](mlir::Operation *op) {
     if (auto call = mlir::dyn_cast<fir::CallOp>(op)) {
-      if (cuf::isInCUDADeviceContext(op))
+      if (cuf::isCUDADeviceContext(op))
         return;
       if (mlir::SymbolRefAttr callee = call.getCalleeAttr()) {
         mlir::StringRef funcName = callee.getLeafReference().getValue();
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 43cd2ea1eb0e6..a9cdf8e7d2efa 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -27,15 +27,54 @@ namespace Fortran::parser {
 constexpr auto startOmpLine = skipStuffBeforeStatement >> "!$OMP "_sptok;
 constexpr auto endOmpLine = space >> endOfLine;
 
+// Given a parser P for a wrapper class, invoke P, and if it succeeds return
+// the wrapped object.
+template <typename Parser> struct UnwrapParser {
+  static_assert(
+      Parser::resultType::WrapperTrait::value && "Wrapper class required");
+  using resultType = decltype(Parser::resultType::v);
+  constexpr UnwrapParser(Parser p) : parser_(p) {}
+
+  std::optional<resultType> Parse(ParseState &state) const {
+    if (auto result{parser_.Parse(state)}) {
+      return result->v;
+    }
+    return std::nullopt;
+  }
+
+private:
+  const Parser parser_;
+};
+
+template <typename Parser> constexpr auto unwrap(const Parser &p) {
+  return UnwrapParser<Parser>(p);
+}
+
+// Check (without advancing the parsing location) if the next thing in the
+// input would be accepted by the "checked" parser, and if so, run the "parser"
+// parser.
+// The intended use is with the "checker" parser being some token, followed
+// by a more complex parser that consumes the token plus more things, e.g.
+// "PARALLEL"_id >= Parser<OmpDirectiveSpecification>{}.
+//
+// The >= has a higher precedence than ||, so it can be used just like >>
+// in an alternatives parser without parentheses.
+template <typename PA, typename PB>
+constexpr auto operator>=(PA checker, PB parser) {
+  return lookAhead(checker) >> parser;
+}
+
 /// Parse OpenMP directive name (this includes compound directives).
 struct OmpDirectiveNameParser {
-  using resultType = llvm::omp::Directive;
+  using resultType = OmpDirectiveName;
   using Token = TokenStringMatch<false, false>;
 
   std::optional<resultType> Parse(ParseState &state) const {
     for (const NameWithId &nid : directives()) {
       if (attempt(Token(nid.first.data())).Parse(state)) {
-        return nid.second;
+        OmpDirectiveName n;
+        n.v = nid.second;
+        return n;
       }
     }
     return std::nullopt;
@@ -218,7 +257,7 @@ TYPE_PARSER(construct<OmpTraitSelectorName::Value>(
 TYPE_PARSER(sourced(construct<OmpTraitSelectorName>(
     // Parse predefined names first (because of SIMD).
     construct<OmpTraitSelectorName>(Parser<OmpTraitSelectorName::Value>{}) ||
-    construct<OmpTraitSelectorName>(OmpDirectiveNameParser{}) ||
+    construct<OmpTraitSelectorName>(unwrap(OmpDirectiveNameParser{})) ||
     // identifier-or-string for extensions
     construct<OmpTraitSelectorName>(
         applyFunction(nameToString, Parser<Name>{})) ||
@@ -550,6 +589,9 @@ TYPE_PARSER(construct<OmpAffinityClause>(
     maybe(nonemptyList(Parser<OmpAffinityClause::Modifier>{}) / ":"),
     Parser<OmpObjectList>{}))
 
+TYPE_PARSER(construct<OmpCancellationConstructTypeClause>(
+    OmpDirectiveNameParser{}, maybe(parenthesized(scalarLogicalExpr))))
+
 // 2.15.3.1 DEFAULT (PRIVATE | FIRSTPRIVATE | SHARED | NONE)
 TYPE_PARSER(construct<OmpDefaultClause::DataSharingAttribute>(
     "PRIVATE" >> pure(OmpDefaultClause::DataSharingAttribute::Private) ||
@@ -775,12 +817,13 @@ TYPE_PARSER(construct<OmpMessageClause>(expr))
 
 TYPE_PARSER(construct<OmpHoldsClause>(indirect(expr)))
 TYPE_PARSER(construct<OmpAbsentClause>(many(maybe(","_tok) >>
-    construct<llvm::omp::Directive>(OmpDirectiveNameParser{}))))
+    construct<llvm::omp::Directive>(unwrap(OmpDirectiveNameParser{})))))
 TYPE_PARSER(construct<OmpContainsClause>(many(maybe(","_tok) >>
-    construct<llvm::omp::Directive>(OmpDirectiveNameParser{}))))
+    construct<llvm::omp::Directive>(unwrap(OmpDirectiveNameParser{})))))
 
-TYPE_PARSER("ABSENT" >> construct<OmpClause>(construct<OmpClause::Absent>(
-                            parenthesized(Parser<OmpAbsentClause>{}))) ||
+TYPE_PARSER( //
+    "ABSENT" >> construct<OmpClause>(construct<OmpClause::Absent>(
+                    parenthesized(Parser<OmpAbsentClause>{}))) ||
     "ACQUIRE" >> construct<OmpClause>(construct<OmpClause::Acquire>()) ||
     "ACQ_REL" >> construct<OmpClause>(construct<OmpClause::AcqRel>()) ||
     "AFFINITY" >> construct<OmpClause>(construct<OmpClause::Affinity>(
@@ -957,7 +1000,20 @@ TYPE_PARSER("ABSENT" >> construct<OmpClause>(construct<OmpClause::Absent>(
     "UPDATE" >> construct<OmpClause>(construct<OmpClause::Update>(
                     parenthesized(Parser<OmpUpdateClause>{}))) ||
     "WHEN" >> construct<OmpClause>(construct<OmpClause::When>(
-                  parenthesized(Parser<OmpWhenClause>{}))))
+                  parenthesized(Parser<OmpWhenClause>{}))) ||
+    // Cancellable constructs
+    "DO"_id >=
+        construct<OmpClause>(construct<OmpClause::CancellationConstructType>(
+            Parser<OmpCancellationConstructTypeClause>{})) ||
+    "PARALLEL"_id >=
+        construct<OmpClause>(construct<OmpClause::CancellationConstructType>(
+            Parser<OmpCancellationConstructTypeClause>{})) ||
+    "SECTIONS"_id >=
+        construct<OmpClause>(construct<OmpClause::CancellationConstructType>(
+            Parser<OmpCancellationConstructTypeClause>{})) ||
+    "TASKGROUP"_id >=
+        construct<OmpClause>(construct<OmpClause::CancellationConstructType>(
+            Parser<OmpCancellationConstructTypeClause>{})))
 
 // [Clause, [Clause], ...]
 TYPE_PARSER(sourced(construct<OmpClauseList>(
@@ -971,10 +1027,33 @@ TYPE_PARSER(sourced(construct<OmpErrorDirective>(
 
 // --- Parsers for directives and constructs --------------------------
 
-TYPE_PARSER(sourced(construct<OmpDirectiveSpecification>( //
-    OmpDirectiveNameParser{},
-    maybe(parenthesized(nonemptyList(Parser<OmpArgument>{}))),
-    maybe(Parser<OmpClauseList>{}))))
+OmpDirectiveSpecification static makeFlushFromOldSyntax1(Verbatim &&text,
+    std::optional<OmpClauseList> &&clauses,
+    std::optional<std::list<OmpArgument>> &&args,
+    OmpDirectiveSpecification::Flags &&flags) {
+  return OmpDirectiveSpecification{OmpDirectiveName(text), std::move(args),
+      std::move(clauses), std::move(flags)};
+}
+
+TYPE_PARSER(sourced(
+    // Parse the old syntax: FLUSH [clauses] [(objects)]
+    construct<OmpDirectiveSpecification>(
+        // Force this old-syntax parser to fail for FLUSH followed by '('.
+        // Otherwise it could succeed on the new syntax but have one of
+        // lists absent in the parsed result.
+        // E.g. for FLUSH(x) SEQ_CST it would find no clauses following
+        // the directive name, parse the argument list "(x)" and stop.
+        applyFunction<OmpDirectiveSpecification>(makeFlushFromOldSyntax1,
+            verbatim("FLUSH"_tok) / !lookAhead("("_tok),
+            maybe(Parser<OmpClauseList>{}),
+            maybe(parenthesized(nonemptyList(Parser<OmpArgument>{}))),
+            pure(OmpDirectiveSpecification::Flags::DeprecatedSyntax))) ||
+    // Parse the standard syntax: directive [(arguments)] [clauses]
+    construct<OmpDirectiveSpecification>( //
+        sourced(OmpDirectiveNameParser{}),
+        maybe(parenthesized(nonemptyList(Parser<OmpArgument>{}))),
+        maybe(Parser<OmpClauseList>{}),
+        pure(OmpDirectiveSpecification::Flags::None))))
 
 TYPE_PARSER(sourced(construct<OmpNothingDirective>("NOTHING" >> ok)))
 
@@ -1048,20 +1127,13 @@ TYPE_PARSER(sourced(construct<OmpLoopDirective>(first(
 TYPE_PARSER(sourced(construct<OmpBeginLoopDirective>(
     sourced(Parser<OmpLoopDirective>{}), Parser<OmpClauseList>{})))
 
-// 2.14.1 construct-type-clause -> PARALLEL | SECTIONS | DO | TASKGROUP
-TYPE_PARSER(sourced(construct<OmpCancelType>(
-    first("PARALLEL" >> pure(OmpCancelType::Type::Parallel),
-        "SECTIONS" >> pure(OmpCancelType::Type::Sections),
-        "DO" >> pure(OmpCancelType::Type::Do),
-        "TASKGROUP" >> pure(OmpCancelType::Type::Taskgroup)))))
-
 // 2.14.2 Cancellation Point construct
 TYPE_PARSER(sourced(construct<OpenMPCancellationPointConstruct>(
-    verbatim("CANCELLATION POINT"_tok), Parser<OmpCancelType>{})))
+    verbatim("CANCELLATION POINT"_tok), Parser<OmpClauseList>{})))
 
 // 2.14.1 Cancel construct
-TYPE_PARSER(sourced(construct<OpenMPCancelConstruct>(verbatim("CANCEL"_tok),
-    Parser<OmpCancelType>{}, maybe("IF" >> parenthesized(scalarLogicalExpr)))))
+TYPE_PARSER(sourced(construct<OpenMPCancelConstruct>(
+    verbatim("CANCEL"_tok), Parser<OmpClauseList>{})))
 
 TYPE_PARSER(sourced(construct<OmpFailClause>(
     parenthesized(indirect(Parser<OmpMemoryOrderClause>{})))))
@@ -1145,9 +1217,10 @@ TYPE_PARSER(
     sourced(construct<OpenMPStandaloneConstruct>(
                 Parser<OpenMPSimpleStandaloneConstruct>{}) ||
         construct<OpenMPStandaloneConstruct>(Parser<OpenMPFlushConstruct>{}) ||
-        construct<OpenMPStandaloneConstruct>(Parser<OpenMPCancelConstruct>{}) ||
+        // Try CANCELLATION POINT before CANCEL.
         construct<OpenMPStandaloneConstruct>(
             Parser<OpenMPCancellationPointConstruct>{}) ||
+        construct<OpenMPStandaloneConstruct>(Parser<OpenMPCancelConstruct>{}) ||
         construct<OpenMPStandaloneConstruct>(
             Parser<OmpMetadirectiveDirective>{}) ||
         construct<OpenMPStandaloneConstruct>(Parser<OpenMPDepobjConstruct>{})) /
diff --git a/flang/lib/Parser/parse-tree.cpp b/flang/lib/Parser/parse-tree.cpp
index 251b6919cf52f..5917f30b3c589 100644
--- a/flang/lib/Parser/parse-tree.cpp
+++ b/flang/lib/Parser/parse-tree.cpp
@@ -253,6 +253,23 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const Name &x) {
   return os << x.ToString();
 }
 
+OmpDirectiveName::OmpDirectiveName(const Verbatim &name) {
+  std::string_view nameView{name.source.begin(), name.source.size()};
+  std::string nameLower{ToLowerCaseLetters(nameView)};
+  // The function getOpenMPDirectiveKind will return OMPD_unknown in two cases:
+  // (1) if the given string doesn't match any actual directive, or
+  // (2) if the given string was "unknown".
+  // The Verbatim(<token>) parser will succeed as long as the given token
+  // matches the source.
+  // Since using "construct<OmpDirectiveName>(verbatim(...))" will succeed
+  // if the verbatim parser succeeds, in order to get OMPD_unknown the
+  // token given to Verbatim must be invalid. Because it's an internal issue
+  // asserting is ok.
+  v = llvm::omp::getOpenMPDirectiveKind(nameLower);
+  assert(v != llvm::omp::Directive::OMPD_unknown && "Invalid directive name");
+  source = name.source;
+}
+
 OmpDependenceType::Value OmpDoacross::GetDepType() const {
   return common::visit( //
       common::visitors{
@@ -319,4 +336,12 @@ namespace Fortran::parser {
 llvm::omp::Clause OmpClause::Id() const {
   return std::visit([](auto &&s) { return getClauseIdForClass(s); }, u);
 }
+
+const OmpClauseList &OmpDirectiveSpecification::Clauses() const {
+  static OmpClauseList empty{decltype(OmpClauseList::v){}};
+  if (auto &clauses = std::get<std::optional<OmpClauseList>>(t)) {
+    return *clauses;
+  }
+  return empty;
+}
 } // namespace Fortran::parser
diff --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
index 6c257f57c2d57..7e6a1c2ca6977 100644
--- a/flang/lib/Parser/preprocessor.cpp
+++ b/flang/lib/Parser/preprocessor.cpp
@@ -842,6 +842,15 @@ bool Preprocessor::IsNameDefined(const CharBlock &token) {
   return definitions_.find(token) != definitions_.end();
 }
 
+bool Preprocessor::IsNameDefinedEmpty(const CharBlock &token) {
+  if (auto it{definitions_.find(token)}; it != definitions_.end()) {
+    const Definition &def{it->second};
+    return !def.isFunctionLike() && def.replacement().SizeInChars() == 0;
+  } else {
+    return false;
+  }
+}
+
 bool Preprocessor::IsFunctionLikeDefinition(const CharBlock &token) {
   auto it{definitions_.find(token)};
   return it != definitions_.end() && it->second.isFunctionLike();
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index f479cc51871ef..b4ac82839b73f 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -145,9 +145,8 @@ void Prescanner::Statement() {
     if (inFixedForm_) {
       CHECK(IsFixedFormCommentChar(*at_));
     } else {
-      while (int n{IsSpaceOrTab(at_)}) {
-        at_ += n, ++column_;
-      }
+      at_ += line.payloadOffset;
+      column_ += line.payloadOffset;
       CHECK(*at_ == '!');
     }
     std::optional<int> condOffset;
@@ -597,6 +596,30 @@ const char *Prescanner::SkipWhiteSpace(const char *p) {
   return p;
 }
 
+const char *Prescanner::SkipWhiteSpaceIncludingEmptyMacros(
+    const char *p) const {
+  while (true) {
+    if (int n{IsSpaceOrTab(p)}) {
+      p += n;
+    } else if (preprocessor_.AnyDefinitions() && IsLegalIdentifierStart(*p)) {
+      // Skip keyword macros with empty definitions
+      const char *q{p + 1};
+      while (IsLegalInIdentifier(*q)) {
+        ++q;
+      }
+      if (preprocessor_.IsNameDefinedEmpty(
+              CharBlock{p, static_cast<std::size_t>(q - p)})) {
+        p = q;
+      } else {
+        break;
+      }
+    } else {
+      break;
+    }
+  }
+  return p;
+}
+
 const char *Prescanner::SkipWhiteSpaceAndCComments(const char *p) const {
   while (true) {
     if (int n{IsSpaceOrTab(p)}) {
@@ -1463,18 +1486,18 @@ Prescanner::IsFixedFormCompilerDirectiveLine(const char *start) const {
   *sp = '\0';
   if (const char *ss{IsCompilerDirectiveSentinel(
           sentinel, static_cast<std::size_t>(sp - sentinel))}) {
-    std::size_t payloadOffset = p - start;
-    return {LineClassification{
-        LineClassification::Kind::CompilerDirective, payloadOffset, ss}};
+    return {
+        LineClassification{LineClassification::Kind::CompilerDirective, 0, ss}};
   }
   return std::nullopt;
 }
 
 std::optional<Prescanner::LineClassification>
 Prescanner::IsFreeFormCompilerDirectiveLine(const char *start) const {
-  if (const char *p{SkipWhiteSpace(start)}; p && *p++ == '!') {
+  if (const char *p{SkipWhiteSpaceIncludingEmptyMacros(start)};
+      p && *p++ == '!') {
     if (auto maybePair{IsCompilerDirectiveSentinel(p)}) {
-      auto offset{static_cast<std::size_t>(maybePair->second - start)};
+      auto offset{static_cast<std::size_t>(p - start - 1)};
       return {LineClassification{LineClassification::Kind::CompilerDirective,
           offset, maybePair->first}};
     }
@@ -1529,7 +1552,7 @@ Prescanner::IsCompilerDirectiveSentinel(const char *p) const {
     if (int n{*p == '&' ? 1 : IsSpaceOrTab(p)}) {
       if (j > 0) {
         sentinel[j] = '\0';
-        p = SkipWhiteSpace(p + n);
+        p = SkipWhiteSpaceIncludingEmptyMacros(p + n);
         if (*p != '!') {
           if (const char *sp{IsCompilerDirectiveSentinel(sentinel, j)}) {
             return std::make_pair(sp, p);
diff --git a/flang/lib/Parser/prescan.h b/flang/lib/Parser/prescan.h
index e2440ad6fbc42..9cf1b389f5b19 100644
--- a/flang/lib/Parser/prescan.h
+++ b/flang/lib/Parser/prescan.h
@@ -182,6 +182,7 @@ class Prescanner {
   void SkipCComments();
   void SkipSpaces();
   static const char *SkipWhiteSpace(const char *);
+  const char *SkipWhiteSpaceIncludingEmptyMacros(const char *) const;
   const char *SkipWhiteSpaceAndCComments(const char *) const;
   const char *SkipCComment(const char *) const;
   bool NextToken(TokenSequence &);
diff --git a/flang/lib/Parser/program-parsers.cpp b/flang/lib/Parser/program-parsers.cpp
index e365cd24a6aed..d6e7f11297187 100644
--- a/flang/lib/Parser/program-parsers.cpp
+++ b/flang/lib/Parser/program-parsers.cpp
@@ -64,7 +64,7 @@ static constexpr auto programUnit{
     construct<ProgramUnit>(indirect(subroutineSubprogram)) ||
     construct<ProgramUnit>(indirect(Parser<Submodule>{})) ||
     construct<ProgramUnit>(indirect(Parser<BlockData>{})) ||
-    lookAhead(validFunctionStmt) >>
+    lookAhead(maybe(label) >> validFunctionStmt) >>
         construct<ProgramUnit>(indirect(functionSubprogram)) ||
     construct<ProgramUnit>(indirect(Parser<MainProgram>{}))};
 static constexpr auto normalProgramUnit{StartNewSubprogram{} >> programUnit /
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index 1df17b6d7382b..f9d5c141b9795 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2094,14 +2094,30 @@ class UnparseVisitor {
     Word(llvm::omp::getOpenMPDirectiveName(x).str());
   }
   void Unparse(const OmpDirectiveSpecification &x) {
-    using ArgList = std::list<parser::OmpArgument>;
-    Walk(std::get<llvm::omp::Directive>(x.t));
-    if (auto &args{std::get<std::optional<ArgList>>(x.t)}) {
-      Put("(");
-      Walk(*args);
-      Put(")");
+    auto unparseArgs{[&]() {
+      using ArgList = std::list<parser::OmpArgument>;
+      if (auto &args{std::get<std::optional<ArgList>>(x.t)}) {
+        Put("(");
+        Walk(*args);
+        Put(")");
+      }
+    }};
+    auto unparseClauses{[&]() { //
+      Walk(std::get<std::optional<OmpClauseList>>(x.t));
+    }};
+
+    Walk(std::get<OmpDirectiveName>(x.t));
+    auto flags{std::get<OmpDirectiveSpecification::Flags>(x.t)};
+    if (flags == OmpDirectiveSpecification::Flags::DeprecatedSyntax) {
+      if (x.DirId() == llvm::omp::Directive::OMPD_flush) {
+        // FLUSH clause arglist
+        unparseClauses();
+        unparseArgs();
+      }
+    } else {
+      unparseArgs();
+      unparseClauses();
     }
-    Walk(std::get<std::optional<OmpClauseList>>(x.t));
   }
   void Unparse(const OmpTraitScore &x) {
     Word("SCORE(");
@@ -2835,15 +2851,14 @@ class UnparseVisitor {
   void Unparse(const OpenMPCancellationPointConstruct &x) {
     BeginOpenMP();
     Word("!$OMP CANCELLATION POINT ");
-    Walk(std::get<OmpCancelType>(x.t));
+    Walk(std::get<OmpClauseList>(x.t));
     Put("\n");
     EndOpenMP();
   }
   void Unparse(const OpenMPCancelConstruct &x) {
     BeginOpenMP();
     Word("!$OMP CANCEL ");
-    Walk(std::get<OmpCancelType>(x.t));
-    Walk(std::get<std::optional<OpenMPCancelConstruct::If>>(x.t));
+    Walk(std::get<OmpClauseList>(x.t));
     Put("\n");
     EndOpenMP();
   }
@@ -3018,7 +3033,6 @@ class UnparseVisitor {
       OmpDeviceTypeClause, DeviceTypeDescription) // OMP device_type
   WALK_NESTED_ENUM(OmpReductionModifier, Value) // OMP reduction-modifier
   WALK_NESTED_ENUM(OmpExpectation, Value) // OMP motion-expectation
-  WALK_NESTED_ENUM(OmpCancelType, Type) // OMP cancel-type
   WALK_NESTED_ENUM(OmpOrderClause, Ordering) // OMP ordering
   WALK_NESTED_ENUM(OmpOrderModifier, Value) // OMP order-modifier
   WALK_NESTED_ENUM(OmpPrescriptiveness, Value) // OMP prescriptiveness
diff --git a/flang/lib/Semantics/assignment.cpp b/flang/lib/Semantics/assignment.cpp
index 8de20d3126a6c..935f5a03bdb6a 100644
--- a/flang/lib/Semantics/assignment.cpp
+++ b/flang/lib/Semantics/assignment.cpp
@@ -134,10 +134,16 @@ static std::optional<std::string> GetPointerComponentDesignatorName(
 // Checks C1594(5,6); false if check fails
 bool CheckCopyabilityInPureScope(parser::ContextualMessages &messages,
     const SomeExpr &expr, const Scope &scope) {
-  if (const Symbol * base{GetFirstSymbol(expr)}) {
-    if (const char *why{
-            WhyBaseObjectIsSuspicious(base->GetUltimate(), scope)}) {
-      if (auto pointer{GetPointerComponentDesignatorName(expr)}) {
+  if (auto pointer{GetPointerComponentDesignatorName(expr)}) {
+    if (const Symbol * base{GetFirstSymbol(expr)}) {
+      const char *why{WhyBaseObjectIsSuspicious(base->GetUltimate(), scope)};
+      if (!why) {
+        if (auto coarray{evaluate::ExtractCoarrayRef(expr)}) {
+          base = &coarray->GetLastSymbol();
+          why = "coindexed";
+        }
+      }
+      if (why) {
         evaluate::SayWithDeclaration(messages, *base,
             "A pure subprogram may not copy the value of '%s' because it is %s"
             " and has the POINTER potential subobject component '%s'"_err_en_US,
diff --git a/flang/lib/Semantics/check-allocate.cpp b/flang/lib/Semantics/check-allocate.cpp
index 223bee6eb6f11..b426dd81334bb 100644
--- a/flang/lib/Semantics/check-allocate.cpp
+++ b/flang/lib/Semantics/check-allocate.cpp
@@ -39,11 +39,10 @@ class AllocationCheckerHelper {
 public:
   AllocationCheckerHelper(
       const parser::Allocation &alloc, AllocateCheckerInfo &info)
-      : allocateInfo_{info}, allocateObject_{std::get<parser::AllocateObject>(
-                                 alloc.t)},
-        allocateShapeSpecRank_{ShapeSpecRank(alloc)}, allocateCoarraySpecRank_{
-                                                          CoarraySpecRank(
-                                                              alloc)} {}
+      : allocateInfo_{info}, allocation_{alloc},
+        allocateObject_{std::get<parser::AllocateObject>(alloc.t)},
+        allocateShapeSpecRank_{ShapeSpecRank(alloc)},
+        allocateCoarraySpecRank_{CoarraySpecRank(alloc)} {}
 
   bool RunChecks(SemanticsContext &context);
 
@@ -84,6 +83,7 @@ class AllocationCheckerHelper {
   }
 
   AllocateCheckerInfo &allocateInfo_;
+  const parser::Allocation &allocation_;
   const parser::AllocateObject &allocateObject_;
   const int allocateShapeSpecRank_{0};
   const int allocateCoarraySpecRank_{0};
@@ -693,6 +693,31 @@ bool AllocationCheckerHelper::RunCoarrayRelatedChecks(
                 corank_);
         return false;
       }
+      if (const auto &coarraySpec{
+              std::get<std::optional<parser::AllocateCoarraySpec>>(
+                  allocation_.t)}) {
+        int dim{0};
+        for (const auto &spec :
+            std::get<std::list<parser::AllocateCoshapeSpec>>(coarraySpec->t)) {
+          if (auto ubv{evaluate::ToInt64(
+                  GetExpr(context, std::get<parser::BoundExpr>(spec.t)))}) {
+            if (auto *lbx{GetExpr(context,
+                    std::get<std::optional<parser::BoundExpr>>(spec.t))}) {
+              auto lbv{evaluate::ToInt64(*lbx)};
+              if (lbv && *ubv < *lbv) {
+                context.Say(name_.source,
+                    "Upper cobound %jd is less than lower cobound %jd of codimension %d"_err_en_US,
+                    std::intmax_t{*ubv}, std::intmax_t{*lbv}, dim + 1);
+              }
+            } else if (*ubv < 1) {
+              context.Say(name_.source,
+                  "Upper cobound %jd of codimension %d is less than 1"_err_en_US,
+                  std::intmax_t{*ubv}, dim + 1);
+            }
+          }
+          ++dim;
+        }
+      }
     }
   } else { // Not a coarray
     if (hasAllocateCoarraySpec()) {
diff --git a/flang/lib/Semantics/check-call.cpp b/flang/lib/Semantics/check-call.cpp
index ecc1e3d27e3bf..205b4a780258c 100644
--- a/flang/lib/Semantics/check-call.cpp
+++ b/flang/lib/Semantics/check-call.cpp
@@ -790,6 +790,12 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
             "ALLOCATABLE %s must have INTENT(IN) to be associated with a coindexed actual argument"_err_en_US,
             dummyName);
       }
+      if (!actualIsCoindexed && actualLastSymbol && dummy.type.corank() == 0 &&
+          actualLastSymbol->Corank() > 0) {
+        messages.Say(
+            "ALLOCATABLE %s is not a coarray but actual argument has corank %d"_err_en_US,
+            dummyName, actualLastSymbol->Corank());
+      }
     } else if (evaluate::IsBareNullPointer(&actual)) {
       if (dummyIsOptional) {
       } else if (dummy.intent == common::Intent::Default &&
@@ -822,12 +828,6 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
           "ALLOCATABLE %s must be associated with an ALLOCATABLE actual argument"_err_en_US,
           dummyName);
     }
-    if (!actualIsCoindexed && actualLastSymbol &&
-        actualLastSymbol->Corank() != dummy.type.corank()) {
-      messages.Say(
-          "ALLOCATABLE %s has corank %d but actual argument has corank %d"_err_en_US,
-          dummyName, dummy.type.corank(), actualLastSymbol->Corank());
-    }
   }
 
   // 15.5.2.7 -- dummy is POINTER
@@ -926,6 +926,11 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
       messages.Say(
           "Actual argument associated with coarray %s must be a coarray"_err_en_US,
           dummyName);
+    } else if (actualType.corank() != dummy.type.corank() &&
+        dummyIsAllocatableOrPointer) {
+      messages.Say(
+          "ALLOCATABLE or POINTER %s has corank %d but actual argument has corank %d"_err_en_US,
+          dummyName, dummy.type.corank(), actualType.corank());
     }
     if (dummyIsVolatile) {
       if (!actualIsVolatile) {
@@ -1840,6 +1845,13 @@ static void CheckMove_Alloc(evaluate::ActualArguments &arguments,
   if (arguments.size() >= 2) {
     evaluate::CheckForCoindexedObject(
         messages, arguments[1], "move_alloc", "to");
+    int fromCR{GetCorank(arguments[0])};
+    int toCR{GetCorank(arguments[1])};
+    if (fromCR != toCR) {
+      messages.Say(*arguments[0]->sourceLocation(),
+          "FROM= argument to MOVE_ALLOC has corank %d, but TO= argument has corank %d"_err_en_US,
+          fromCR, toCR);
+    }
   }
   if (arguments.size() >= 3) {
     evaluate::CheckForCoindexedObject(
diff --git a/flang/lib/Semantics/check-coarray.cpp b/flang/lib/Semantics/check-coarray.cpp
index 6bed525d7f687..833a08899308a 100644
--- a/flang/lib/Semantics/check-coarray.cpp
+++ b/flang/lib/Semantics/check-coarray.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "check-coarray.h"
+#include "definable.h"
 #include "flang/Common/indirection.h"
 #include "flang/Evaluate/expression.h"
 #include "flang/Parser/message.h"
@@ -96,34 +97,37 @@ static void CheckCoindexedStatOrErrmsg(SemanticsContext &context,
   Fortran::common::visit(CoindexedCheck, statOrErrmsg.u);
 }
 
+static void CheckSyncStat(SemanticsContext &context,
+    const parser::StatOrErrmsg &statOrErrmsg, bool &gotStat, bool &gotMsg) {
+  common::visit(
+      common::visitors{
+          [&](const parser::StatVariable &stat) {
+            if (gotStat) {
+              context.Say( // C1172
+                  "The stat-variable in a sync-stat-list may not be repeated"_err_en_US);
+            }
+            gotStat = true;
+          },
+          [&](const parser::MsgVariable &var) {
+            WarnOnDeferredLengthCharacterScalar(context, GetExpr(context, var),
+                var.v.thing.thing.GetSource(), "ERRMSG=");
+            if (gotMsg) {
+              context.Say( // C1172
+                  "The errmsg-variable in a sync-stat-list may not be repeated"_err_en_US);
+            }
+            gotMsg = true;
+          },
+      },
+      statOrErrmsg.u);
+
+  CheckCoindexedStatOrErrmsg(context, statOrErrmsg, "sync-stat-list");
+}
+
 static void CheckSyncStatList(
     SemanticsContext &context, const std::list<parser::StatOrErrmsg> &list) {
   bool gotStat{false}, gotMsg{false};
-
   for (const parser::StatOrErrmsg &statOrErrmsg : list) {
-    common::visit(
-        common::visitors{
-            [&](const parser::StatVariable &stat) {
-              if (gotStat) {
-                context.Say( // C1172
-                    "The stat-variable in a sync-stat-list may not be repeated"_err_en_US);
-              }
-              gotStat = true;
-            },
-            [&](const parser::MsgVariable &var) {
-              WarnOnDeferredLengthCharacterScalar(context,
-                  GetExpr(context, var), var.v.thing.thing.GetSource(),
-                  "ERRMSG=");
-              if (gotMsg) {
-                context.Say( // C1172
-                    "The errmsg-variable in a sync-stat-list may not be repeated"_err_en_US);
-              }
-              gotMsg = true;
-            },
-        },
-        statOrErrmsg.u);
-
-    CheckCoindexedStatOrErrmsg(context, statOrErrmsg, "sync-stat-list");
+    CheckSyncStat(context, statOrErrmsg, gotStat, gotMsg);
   }
 }
 
@@ -260,7 +264,51 @@ void CoarrayChecker::Leave(const parser::EventWaitStmt &x) {
       context_, std::get<std::list<parser::EventWaitSpec>>(x.t));
 }
 
+static void CheckLockVariable(
+    SemanticsContext &context, const parser::LockVariable &lockVar) {
+  if (const SomeExpr * expr{GetExpr(lockVar)}) {
+    if (auto dyType{expr->GetType()}) {
+      auto at{parser::FindSourceLocation(lockVar)};
+      if (dyType->category() != TypeCategory::Derived ||
+          dyType->IsUnlimitedPolymorphic() ||
+          !IsLockType(&dyType->GetDerivedTypeSpec())) {
+        context.Say(at,
+            "Lock variable must have type LOCK_TYPE from ISO_FORTRAN_ENV"_err_en_US);
+      } else if (auto whyNot{WhyNotDefinable(at, context.FindScope(at),
+                     {DefinabilityFlag::DoNotNoteDefinition,
+                         DefinabilityFlag::AllowEventLockOrNotifyType},
+                     *expr)}) {
+        whyNot->set_severity(parser::Severity::Because);
+        context.Say(at, "Lock variable is not definable"_err_en_US)
+            .Attach(std::move(*whyNot));
+      }
+    }
+  }
+}
+
+void CoarrayChecker::Leave(const parser::LockStmt &x) {
+  CheckLockVariable(context_, std::get<parser::LockVariable>(x.t));
+  bool gotAcquired{false}, gotStat{false}, gotMsg{false};
+  for (const parser::LockStmt::LockStat &lockStat :
+      std::get<std::list<parser::LockStmt::LockStat>>(x.t)) {
+    if (const auto *statOrErrmsg{
+            std::get_if<parser::StatOrErrmsg>(&lockStat.u)}) {
+      CheckSyncStat(context_, *statOrErrmsg, gotStat, gotMsg);
+    } else {
+      CHECK(std::holds_alternative<
+          parser::Scalar<parser::Logical<parser::Variable>>>(lockStat.u));
+      if (gotAcquired) {
+        context_.Say(parser::FindSourceLocation(lockStat),
+            "Multiple ACQUIRED_LOCK specifiers"_err_en_US);
+      } else {
+        gotAcquired = true;
+      }
+    }
+  }
+}
+
 void CoarrayChecker::Leave(const parser::UnlockStmt &x) {
+  CheckLockVariable(context_, std::get<parser::LockVariable>(x.t));
   CheckSyncStatList(context_, std::get<std::list<parser::StatOrErrmsg>>(x.t));
 }
 
diff --git a/flang/lib/Semantics/check-coarray.h b/flang/lib/Semantics/check-coarray.h
index 0af9a880fd31a..a968585b48be7 100644
--- a/flang/lib/Semantics/check-coarray.h
+++ b/flang/lib/Semantics/check-coarray.h
@@ -28,6 +28,7 @@ struct SyncAllStmt;
 struct SyncImagesStmt;
 struct SyncMemoryStmt;
 struct SyncTeamStmt;
+struct LockStmt;
 struct UnlockStmt;
 } // namespace Fortran::parser
 
@@ -45,6 +46,7 @@ class CoarrayChecker : public virtual BaseChecker {
   void Leave(const parser::NotifyWaitStmt &);
   void Leave(const parser::EventPostStmt &);
   void Leave(const parser::EventWaitStmt &);
+  void Leave(const parser::LockStmt &);
   void Leave(const parser::UnlockStmt &);
   void Leave(const parser::CriticalStmt &);
   void Leave(const parser::ImageSelector &);
diff --git a/flang/lib/Semantics/check-deallocate.cpp b/flang/lib/Semantics/check-deallocate.cpp
index 7cac1c413b643..3bcd4d87b0906 100644
--- a/flang/lib/Semantics/check-deallocate.cpp
+++ b/flang/lib/Semantics/check-deallocate.cpp
@@ -89,6 +89,9 @@ void DeallocateChecker::Leave(const parser::DeallocateStmt &deallocateStmt) {
                           "Object in DEALLOCATE statement is not deallocatable"_err_en_US)
                       .Attach(std::move(
                           whyNot->set_severity(parser::Severity::Because)));
+                } else if (evaluate::ExtractCoarrayRef(*expr)) { // F'2023 C955
+                  context_.Say(source,
+                      "Component in DEALLOCATE statement may not be coindexed"_err_en_US);
                 }
               }
             },
diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index 25e9262e46a75..e5a01657e4a15 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -3486,7 +3486,7 @@ void CheckHelper::CheckDioDummyIsDefaultInteger(
 }
 
 void CheckHelper::CheckDioDummyIsScalar(const Symbol &subp, const Symbol &arg) {
-  if (arg.Rank() > 0 || arg.Corank() > 0) {
+  if (arg.Rank() > 0) {
     messages_.Say(arg.name(),
         "Dummy argument '%s' of a defined input/output procedure must be a scalar"_err_en_US,
         arg.name());
@@ -3643,6 +3643,13 @@ void CheckHelper::CheckDefinedIoProc(const Symbol &symbol,
       CheckDioArgCount(*specificSubp, ioKind, dummyArgs.size());
       int argCount{0};
       for (auto *arg : dummyArgs) {
+        if (arg && arg->Corank() > 0) {
+          evaluate::AttachDeclaration(
+              messages_.Say(arg->name(),
+                  "Dummy argument '%s' of defined input/output procedure '%s' may not be a coarray"_err_en_US,
+                  arg->name(), ultimate.name()),
+              *arg);
+        }
         switch (argCount++) {
         case 0:
           // dtv-type-spec, INTENT(INOUT) :: dtv
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index c6ed211549401..5fcebdca0bc5f 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -625,18 +625,28 @@ void OmpStructureChecker::CheckHintClause(
 }
 
 void OmpStructureChecker::Enter(const parser::OmpDirectiveSpecification &x) {
-  PushContextAndClauseSets(x.source, std::get<llvm::omp::Directive>(x.t));
+  // OmpDirectiveSpecification exists on its own only in METADIRECTIVE.
+  // In other cases it's a part of other constructs that handle directive
+  // context stack by themselves.
+  if (GetDirectiveNest(MetadirectiveNest)) {
+    PushContextAndClauseSets(
+        std::get<parser::OmpDirectiveName>(x.t).source, x.DirId());
+  }
 }
 
 void OmpStructureChecker::Leave(const parser::OmpDirectiveSpecification &) {
-  dirContext_.pop_back();
+  if (GetDirectiveNest(MetadirectiveNest)) {
+    dirContext_.pop_back();
+  }
 }
 
 void OmpStructureChecker::Enter(const parser::OmpMetadirectiveDirective &x) {
+  EnterDirectiveNest(MetadirectiveNest);
   PushContextAndClauseSets(x.source, llvm::omp::Directive::OMPD_metadirective);
 }
 
 void OmpStructureChecker::Leave(const parser::OmpMetadirectiveDirective &) {
+  ExitDirectiveNest(MetadirectiveNest);
   dirContext_.pop_back();
 }
 
@@ -1203,6 +1213,64 @@ void OmpStructureChecker::Enter(const parser::OpenMPBlockConstruct &x) {
     deviceConstructFound_ = true;
   }
 
+  if (GetContext().directive == llvm::omp::Directive::OMPD_single) {
+    std::set<Symbol *> singleCopyprivateSyms;
+    std::set<Symbol *> endSingleCopyprivateSyms;
+    bool foundNowait{false};
+    parser::CharBlock NowaitSource;
+
+    auto catchCopyPrivateNowaitClauses = [&](const auto &dir, bool endDir) {
+      for (auto &clause : std::get<parser::OmpClauseList>(dir.t).v) {
+        if (clause.Id() == llvm::omp::Clause::OMPC_copyprivate) {
+          for (const auto &ompObject : GetOmpObjectList(clause)->v) {
+            const auto *name{parser::Unwrap<parser::Name>(ompObject)};
+            if (Symbol * symbol{name->symbol}) {
+              if (singleCopyprivateSyms.count(symbol)) {
+                if (endDir) {
+                  context_.Warn(common::UsageWarning::OpenMPUsage, name->source,
+                      "The COPYPRIVATE clause with '%s' is already used on the SINGLE directive"_warn_en_US,
+                      name->ToString());
+                } else {
+                  context_.Say(name->source,
+                      "'%s' appears in more than one COPYPRIVATE clause on the SINGLE directive"_err_en_US,
+                      name->ToString());
+                }
+              } else if (endSingleCopyprivateSyms.count(symbol)) {
+                context_.Say(name->source,
+                    "'%s' appears in more than one COPYPRIVATE clause on the END SINGLE directive"_err_en_US,
+                    name->ToString());
+              } else {
+                if (endDir) {
+                  endSingleCopyprivateSyms.insert(symbol);
+                } else {
+                  singleCopyprivateSyms.insert(symbol);
+                }
+              }
+            }
+          }
+        } else if (clause.Id() == llvm::omp::Clause::OMPC_nowait) {
+          if (foundNowait) {
+            context_.Say(clause.source,
+                "At most one NOWAIT clause can appear on the SINGLE directive"_err_en_US);
+          } else {
+            foundNowait = !endDir;
+          }
+          if (!NowaitSource.ToString().size()) {
+            NowaitSource = clause.source;
+          }
+        }
+      }
+    };
+    catchCopyPrivateNowaitClauses(beginBlockDir, false);
+    catchCopyPrivateNowaitClauses(endBlockDir, true);
+    unsigned version{context_.langOptions().OpenMPVersion};
+    if (version <= 52 && NowaitSource.ToString().size() &&
+        (singleCopyprivateSyms.size() || endSingleCopyprivateSyms.size())) {
+      context_.Say(NowaitSource,
+          "NOWAIT clause must not be used with COPYPRIVATE clause on the SINGLE directive"_err_en_US);
+    }
+  }
+
   switch (beginDir.v) {
   case llvm::omp::Directive::OMPD_target:
     if (CheckTargetBlockOnlyTeams(block)) {
@@ -2176,9 +2244,13 @@ void OmpStructureChecker::Leave(const parser::OpenMPFlushConstruct &x) {
 
 void OmpStructureChecker::Enter(const parser::OpenMPCancelConstruct &x) {
   const auto &dir{std::get<parser::Verbatim>(x.t)};
-  const auto &type{std::get<parser::OmpCancelType>(x.t)};
+  const auto &clauses{std::get<parser::OmpClauseList>(x.t)};
   PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_cancel);
-  CheckCancellationNest(dir.source, type.v);
+
+  if (auto maybeConstruct{GetCancelType(
+          llvm::omp::Directive::OMPD_cancel, x.source, clauses)}) {
+    CheckCancellationNest(dir.source, *maybeConstruct);
+  }
 }
 
 void OmpStructureChecker::Leave(const parser::OpenMPCancelConstruct &) {
@@ -2228,13 +2300,37 @@ void OmpStructureChecker::Leave(const parser::OpenMPCriticalConstruct &) {
   dirContext_.pop_back();
 }
 
+void OmpStructureChecker::Enter(
+    const parser::OmpClause::CancellationConstructType &x) {
+  // Do not call CheckAllowed/CheckAllowedClause, because in case of an error
+  // it will print "CANCELLATION_CONSTRUCT_TYPE" as the clause name instead of
+  // the contained construct name.
+  auto &dirName{std::get<parser::OmpDirectiveName>(x.v.t)};
+  switch (dirName.v) {
+  case llvm::omp::Directive::OMPD_do:
+  case llvm::omp::Directive::OMPD_parallel:
+  case llvm::omp::Directive::OMPD_sections:
+  case llvm::omp::Directive::OMPD_taskgroup:
+    break;
+  default:
+    context_.Say(dirName.source, "%s is not a cancellable construct"_err_en_US,
+        parser::ToUpperCaseLetters(
+            llvm::omp::getOpenMPDirectiveName(dirName.v).str()));
+    break;
+  }
+}
+
 void OmpStructureChecker::Enter(
     const parser::OpenMPCancellationPointConstruct &x) {
   const auto &dir{std::get<parser::Verbatim>(x.t)};
-  const auto &type{std::get<parser::OmpCancelType>(x.t)};
+  const auto &clauses{std::get<parser::OmpClauseList>(x.t)};
   PushContextAndClauseSets(
       dir.source, llvm::omp::Directive::OMPD_cancellation_point);
-  CheckCancellationNest(dir.source, type.v);
+
+  if (auto maybeConstruct{GetCancelType(
+          llvm::omp::Directive::OMPD_cancellation_point, x.source, clauses)}) {
+    CheckCancellationNest(dir.source, *maybeConstruct);
+  }
 }
 
 void OmpStructureChecker::Leave(
@@ -2242,8 +2338,42 @@ void OmpStructureChecker::Leave(
   dirContext_.pop_back();
 }
 
+std::optional<llvm::omp::Directive> OmpStructureChecker::GetCancelType(
+    llvm::omp::Directive cancelDir, const parser::CharBlock &cancelSource,
+    const parser::OmpClauseList &clauses) {
+  // Given clauses from CANCEL or CANCELLATION_POINT, identify the construct
+  // to which the cancellation applies.
+  std::optional<llvm::omp::Directive> cancelee;
+  llvm::StringRef cancelName{llvm::omp::getOpenMPDirectiveName(cancelDir)};
+
+  for (const parser::OmpClause &clause : clauses.v) {
+    using CancellationConstructType =
+        parser::OmpClause::CancellationConstructType;
+    if (auto *cctype{std::get_if<CancellationConstructType>(&clause.u)}) {
+      if (cancelee) {
+        context_.Say(cancelSource,
+            "Multiple cancel-directive-name clauses are not allowed on the %s construct"_err_en_US,
+            parser::ToUpperCaseLetters(cancelName.str()));
+        return std::nullopt;
+      }
+      cancelee = std::get<parser::OmpDirectiveName>(cctype->v.t).v;
+    }
+  }
+
+  if (!cancelee) {
+    context_.Say(cancelSource,
+        "Missing cancel-directive-name clause on the %s construct"_err_en_US,
+        parser::ToUpperCaseLetters(cancelName.str()));
+    return std::nullopt;
+  }
+
+  return cancelee;
+}
+
 void OmpStructureChecker::CheckCancellationNest(
-    const parser::CharBlock &source, const parser::OmpCancelType::Type &type) {
+    const parser::CharBlock &source, llvm::omp::Directive type) {
+  llvm::StringRef typeName{llvm::omp::getOpenMPDirectiveName(type)};
+
   if (CurrentDirectiveIsNested()) {
     // If construct-type-clause is taskgroup, the cancellation construct must be
     // closely nested inside a task or a taskloop construct and the cancellation
@@ -2254,8 +2384,9 @@ void OmpStructureChecker::CheckCancellationNest(
     // that matches the type specified in construct-type-clause of the
     // cancellation construct.
     bool eligibleCancellation{false};
+
     switch (type) {
-    case parser::OmpCancelType::Type::Taskgroup:
+    case llvm::omp::Directive::OMPD_taskgroup:
       if (llvm::omp::nestedCancelTaskgroupAllowedSet.test(
               GetContextParent().directive)) {
         eligibleCancellation = true;
@@ -2281,38 +2412,37 @@ void OmpStructureChecker::CheckCancellationNest(
       }
       if (!eligibleCancellation) {
         context_.Say(source,
-            "With %s clause, %s construct must be closely nested inside TASK "
-            "or TASKLOOP construct and %s region must be closely nested inside "
-            "TASKGROUP region"_err_en_US,
-            parser::ToUpperCaseLetters(
-                parser::OmpCancelType::EnumToString(type)),
+            "With %s clause, %s construct must be closely nested inside TASK or TASKLOOP construct and %s region must be closely nested inside TASKGROUP region"_err_en_US,
+            parser::ToUpperCaseLetters(typeName.str()),
             ContextDirectiveAsFortran(), ContextDirectiveAsFortran());
       }
       return;
-    case parser::OmpCancelType::Type::Sections:
+    case llvm::omp::Directive::OMPD_sections:
       if (llvm::omp::nestedCancelSectionsAllowedSet.test(
               GetContextParent().directive)) {
         eligibleCancellation = true;
       }
       break;
-    case parser::OmpCancelType::Type::Do:
+    case llvm::omp::Directive::OMPD_do:
       if (llvm::omp::nestedCancelDoAllowedSet.test(
               GetContextParent().directive)) {
         eligibleCancellation = true;
       }
       break;
-    case parser::OmpCancelType::Type::Parallel:
+    case llvm::omp::Directive::OMPD_parallel:
       if (llvm::omp::nestedCancelParallelAllowedSet.test(
               GetContextParent().directive)) {
         eligibleCancellation = true;
       }
       break;
+    default:
+      // This should have been diagnosed by this point.
+      llvm_unreachable("Unexpected directive");
     }
     if (!eligibleCancellation) {
       context_.Say(source,
-          "With %s clause, %s construct cannot be closely nested inside %s "
-          "construct"_err_en_US,
-          parser::ToUpperCaseLetters(parser::OmpCancelType::EnumToString(type)),
+          "With %s clause, %s construct cannot be closely nested inside %s construct"_err_en_US,
+          parser::ToUpperCaseLetters(typeName.str()),
           ContextDirectiveAsFortran(),
           parser::ToUpperCaseLetters(
               getDirectiveName(GetContextParent().directive).str()));
@@ -2320,38 +2450,33 @@ void OmpStructureChecker::CheckCancellationNest(
   } else {
     // The cancellation directive cannot be orphaned.
     switch (type) {
-    case parser::OmpCancelType::Type::Taskgroup:
+    case llvm::omp::Directive::OMPD_taskgroup:
       context_.Say(source,
-          "%s %s directive is not closely nested inside "
-          "TASK or TASKLOOP"_err_en_US,
+          "%s %s directive is not closely nested inside TASK or TASKLOOP"_err_en_US,
           ContextDirectiveAsFortran(),
-          parser::ToUpperCaseLetters(
-              parser::OmpCancelType::EnumToString(type)));
+          parser::ToUpperCaseLetters(typeName.str()));
       break;
-    case parser::OmpCancelType::Type::Sections:
+    case llvm::omp::Directive::OMPD_sections:
       context_.Say(source,
-          "%s %s directive is not closely nested inside "
-          "SECTION or SECTIONS"_err_en_US,
+          "%s %s directive is not closely nested inside SECTION or SECTIONS"_err_en_US,
           ContextDirectiveAsFortran(),
-          parser::ToUpperCaseLetters(
-              parser::OmpCancelType::EnumToString(type)));
+          parser::ToUpperCaseLetters(typeName.str()));
       break;
-    case parser::OmpCancelType::Type::Do:
+    case llvm::omp::Directive::OMPD_do:
       context_.Say(source,
-          "%s %s directive is not closely nested inside "
-          "the construct that matches the DO clause type"_err_en_US,
+          "%s %s directive is not closely nested inside the construct that matches the DO clause type"_err_en_US,
           ContextDirectiveAsFortran(),
-          parser::ToUpperCaseLetters(
-              parser::OmpCancelType::EnumToString(type)));
+          parser::ToUpperCaseLetters(typeName.str()));
       break;
-    case parser::OmpCancelType::Type::Parallel:
+    case llvm::omp::Directive::OMPD_parallel:
       context_.Say(source,
-          "%s %s directive is not closely nested inside "
-          "the construct that matches the PARALLEL clause type"_err_en_US,
+          "%s %s directive is not closely nested inside the construct that matches the PARALLEL clause type"_err_en_US,
           ContextDirectiveAsFortran(),
-          parser::ToUpperCaseLetters(
-              parser::OmpCancelType::EnumToString(type)));
+          parser::ToUpperCaseLetters(typeName.str()));
       break;
+    default:
+      // This should have been diagnosed by this point.
+      llvm_unreachable("Unexpected directive");
     }
   }
 }
@@ -2903,12 +3028,6 @@ void OmpStructureChecker::Leave(const parser::OmpClauseList &) {
   // clause
   CheckMultListItems();
 
-  // 2.7.3 Single Construct Restriction
-  if (GetContext().directive == llvm::omp::Directive::OMPD_end_single) {
-    CheckNotAllowedIfClause(
-        llvm::omp::Clause::OMPC_copyprivate, {llvm::omp::Clause::OMPC_nowait});
-  }
-
   auto testThreadprivateVarErr = [&](Symbol sym, parser::Name name,
                                      llvmOmpClause clauseTy) {
     if (sym.test(Symbol::Flag::OmpThreadprivate))
@@ -3044,7 +3163,6 @@ CHECK_SIMPLE_CLAUSE(MemoryOrder, OMPC_memory_order)
 CHECK_SIMPLE_CLAUSE(Bind, OMPC_bind)
 CHECK_SIMPLE_CLAUSE(Align, OMPC_align)
 CHECK_SIMPLE_CLAUSE(Compare, OMPC_compare)
-CHECK_SIMPLE_CLAUSE(CancellationConstructType, OMPC_cancellation_construct_type)
 CHECK_SIMPLE_CLAUSE(OmpxAttribute, OMPC_ompx_attribute)
 CHECK_SIMPLE_CLAUSE(Weak, OMPC_weak)
 
@@ -3549,15 +3667,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Private &x) {
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Nowait &x) {
   CheckAllowedClause(llvm::omp::Clause::OMPC_nowait);
-  if (llvm::omp::noWaitClauseNotAllowedSet.test(GetContext().directive)) {
-    context_.Say(GetContext().clauseSource,
-        "%s clause is not allowed on the OMP %s directive,"
-        " use it on OMP END %s directive "_err_en_US,
-        parser::ToUpperCaseLetters(
-            getClauseName(llvm::omp::Clause::OMPC_nowait).str()),
-        parser::ToUpperCaseLetters(GetContext().directiveSource.ToString()),
-        parser::ToUpperCaseLetters(GetContext().directiveSource.ToString()));
-  }
 }
 
 bool OmpStructureChecker::IsDataRefTypeParamInquiry(
@@ -4288,15 +4397,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Copyprivate &x) {
   CheckIntentInPointer(symbols, llvm::omp::Clause::OMPC_copyprivate);
   CheckCopyingPolymorphicAllocatable(
       symbols, llvm::omp::Clause::OMPC_copyprivate);
-  if (GetContext().directive == llvm::omp::Directive::OMPD_single) {
-    context_.Say(GetContext().clauseSource,
-        "%s clause is not allowed on the OMP %s directive,"
-        " use it on OMP END %s directive "_err_en_US,
-        parser::ToUpperCaseLetters(
-            getClauseName(llvm::omp::Clause::OMPC_copyprivate).str()),
-        parser::ToUpperCaseLetters(GetContext().directiveSource.ToString()),
-        parser::ToUpperCaseLetters(GetContext().directiveSource.ToString()));
-  }
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Lastprivate &x) {
@@ -5518,6 +5618,10 @@ void OmpStructureChecker::Enter(
   CheckAllowedRequiresClause(llvm::omp::Clause::OMPC_unified_shared_memory);
 }
 
+void OmpStructureChecker::Enter(const parser::OmpClause::SelfMaps &x) {
+  CheckAllowedRequiresClause(llvm::omp::Clause::OMPC_self_maps);
+}
+
 void OmpStructureChecker::Enter(const parser::DoConstruct &x) {
   Base::Enter(x);
   loopStack_.push_back(&x);
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index b70ea58cf5578..4c4e65cd7a2db 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -275,8 +275,11 @@ class OmpStructureChecker
   void CheckTargetUpdate();
   void CheckDependenceType(const parser::OmpDependenceType::Value &x);
   void CheckTaskDependenceType(const parser::OmpTaskDependenceType::Value &x);
+  std::optional<llvm::omp::Directive> GetCancelType(
+      llvm::omp::Directive cancelDir, const parser::CharBlock &cancelSource,
+      const parser::OmpClauseList &clauses);
   void CheckCancellationNest(
-      const parser::CharBlock &source, const parser::OmpCancelType::Type &type);
+      const parser::CharBlock &source, llvm::omp::Directive type);
   std::int64_t GetOrdCollapseLevel(const parser::OpenMPLoopConstruct &x);
   void CheckReductionObjects(
       const parser::OmpObjectList &objects, llvm::omp::Clause clauseId);
@@ -326,7 +329,8 @@ class OmpStructureChecker
     TargetNest,
     DeclarativeNest,
     ContextSelectorNest,
-    LastType = ContextSelectorNest,
+    MetadirectiveNest,
+    LastType = MetadirectiveNest,
   };
   int directiveNest_[LastType + 1] = {0};
 
diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index 827defd605f7f..39a58a4e23363 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -265,93 +265,32 @@ MaybeExpr ExpressionAnalyzer::Designate(DataRef &&ref) {
   }
 }
 
-// Some subscript semantic checks must be deferred until all of the
-// subscripts are in hand.
-MaybeExpr ExpressionAnalyzer::CompleteSubscripts(ArrayRef &&ref) {
-  const Symbol &symbol{ref.GetLastSymbol().GetUltimate()};
-  int symbolRank{symbol.Rank()};
-  int subscripts{static_cast<int>(ref.size())};
-  if (subscripts == 0) {
-    return std::nullopt; // error recovery
-  } else if (subscripts != symbolRank) {
-    if (symbolRank != 0) {
-      Say("Reference to rank-%d object '%s' has %d subscripts"_err_en_US,
-          symbolRank, symbol.name(), subscripts);
-    }
-    return std::nullopt;
-  } else if (symbol.has<semantics::ObjectEntityDetails>() ||
-      symbol.has<semantics::AssocEntityDetails>()) {
-    // C928 & C1002
-    if (Triplet *last{std::get_if<Triplet>(&ref.subscript().back().u)}) {
-      if (!last->upper() && IsAssumedSizeArray(symbol)) {
-        Say("Assumed-size array '%s' must have explicit final "
-            "subscript upper bound value"_err_en_US,
-            symbol.name());
-        return std::nullopt;
-      }
-    }
-  } else {
-    // Shouldn't get here from Analyze(ArrayElement) without a valid base,
-    // which, if not an object, must be a construct entity from
-    // SELECT TYPE/RANK or ASSOCIATE.
-    CHECK(symbol.has<semantics::AssocEntityDetails>());
-  }
-  if (!semantics::IsNamedConstant(symbol) && !inDataStmtObject_) {
-    // Subscripts of named constants are checked in folding.
-    // Subscripts of DATA statement objects are checked in data statement
-    // conversion to initializers.
-    CheckSubscripts(ref);
-  }
-  return Designate(DataRef{std::move(ref)});
-}
-
-// Applies subscripts to a data reference.
-MaybeExpr ExpressionAnalyzer::ApplySubscripts(
-    DataRef &&dataRef, std::vector<Subscript> &&subscripts) {
-  if (subscripts.empty()) {
-    return std::nullopt; // error recovery
-  }
-  return common::visit(
-      common::visitors{
-          [&](SymbolRef &&symbol) {
-            return CompleteSubscripts(ArrayRef{symbol, std::move(subscripts)});
-          },
-          [&](Component &&c) {
-            return CompleteSubscripts(
-                ArrayRef{std::move(c), std::move(subscripts)});
-          },
-          [&](auto &&) -> MaybeExpr {
-            DIE("bad base for ArrayRef");
-            return std::nullopt;
-          },
-      },
-      std::move(dataRef.u));
-}
-
-void ExpressionAnalyzer::CheckSubscripts(ArrayRef &ref) {
-  // Fold subscript expressions and check for an empty triplet.
-  const Symbol &arraySymbol{ref.base().GetLastSymbol()};
-  Shape lb{GetLBOUNDs(foldingContext_, NamedEntity{arraySymbol})};
-  CHECK(lb.size() >= ref.subscript().size());
-  Shape ub{GetUBOUNDs(foldingContext_, NamedEntity{arraySymbol})};
-  CHECK(ub.size() >= ref.subscript().size());
+// Returns false if any dimension could be empty (e.g. A(1:0)) or has an error
+static bool FoldSubscripts(semantics::SemanticsContext &context,
+    const Symbol &arraySymbol, std::vector<Subscript> &subscripts, Shape &lb,
+    Shape &ub) {
+  FoldingContext &foldingContext{context.foldingContext()};
+  lb = GetLBOUNDs(foldingContext, NamedEntity{arraySymbol});
+  CHECK(lb.size() >= subscripts.size());
+  ub = GetUBOUNDs(foldingContext, NamedEntity{arraySymbol});
+  CHECK(ub.size() >= subscripts.size());
   bool anyPossiblyEmptyDim{false};
   int dim{0};
-  for (Subscript &ss : ref.subscript()) {
+  for (Subscript &ss : subscripts) {
     if (Triplet * triplet{std::get_if<Triplet>(&ss.u)}) {
-      auto expr{Fold(triplet->stride())};
+      auto expr{Fold(foldingContext, triplet->stride())};
       auto stride{ToInt64(expr)};
       triplet->set_stride(std::move(expr));
       std::optional<ConstantSubscript> lower, upper;
       if (auto expr{triplet->lower()}) {
-        *expr = Fold(std::move(*expr));
+        *expr = Fold(foldingContext, std::move(*expr));
         lower = ToInt64(*expr);
         triplet->set_lower(std::move(*expr));
       } else {
         lower = ToInt64(lb[dim]);
       }
       if (auto expr{triplet->upper()}) {
-        *expr = Fold(std::move(*expr));
+        *expr = Fold(foldingContext, std::move(*expr));
         upper = ToInt64(*expr);
         triplet->set_upper(std::move(*expr));
       } else {
@@ -359,8 +298,9 @@ void ExpressionAnalyzer::CheckSubscripts(ArrayRef &ref) {
       }
       if (stride) {
         if (*stride == 0) {
-          Say("Stride of triplet must not be zero"_err_en_US);
-          return;
+          foldingContext.messages().Say(
+              "Stride of triplet must not be zero"_err_en_US);
+          return false; // error
         }
         if (lower && upper) {
           if (*stride > 0) {
@@ -380,21 +320,53 @@ void ExpressionAnalyzer::CheckSubscripts(ArrayRef &ref) {
       }
     } else { // not triplet
       auto &expr{std::get<IndirectSubscriptIntegerExpr>(ss.u).value()};
-      expr = Fold(std::move(expr));
+      expr = Fold(foldingContext, std::move(expr));
       anyPossiblyEmptyDim |= expr.Rank() > 0; // vector subscript
     }
     ++dim;
   }
-  if (anyPossiblyEmptyDim) {
-    return;
+  return !anyPossiblyEmptyDim;
+}
+
+static void ValidateSubscriptValue(parser::ContextualMessages &messages,
+    const Symbol &symbol, ConstantSubscript val,
+    std::optional<ConstantSubscript> lb, std::optional<ConstantSubscript> ub,
+    int dim, const char *co = "") {
+  std::optional<parser::MessageFixedText> msg;
+  std::optional<ConstantSubscript> bound;
+  if (lb && val < *lb) {
+    msg =
+        "%ssubscript %jd is less than lower %sbound %jd for %sdimension %d of array"_err_en_US;
+    bound = *lb;
+  } else if (ub && val > *ub) {
+    msg =
+        "%ssubscript %jd is greater than upper %sbound %jd for %sdimension %d of array"_err_en_US;
+    bound = *ub;
+    if (dim + 1 == symbol.Rank() && IsDummy(symbol) && *bound == 1) {
+      // Old-school overindexing of a dummy array isn't fatal when
+      // it's on the last dimension and the extent is 1.
+      msg->set_severity(parser::Severity::Warning);
+    }
+  }
+  if (msg) {
+    AttachDeclaration(
+        messages.Say(std::move(*msg), co, static_cast<std::intmax_t>(val), co,
+            static_cast<std::intmax_t>(bound.value()), co, dim + 1),
+        symbol);
   }
-  dim = 0;
-  for (Subscript &ss : ref.subscript()) {
+}
+
+static void ValidateSubscripts(semantics::SemanticsContext &context,
+    const Symbol &arraySymbol, const std::vector<Subscript> &subscripts,
+    const Shape &lb, const Shape &ub) {
+  int dim{0};
+  for (const Subscript &ss : subscripts) {
     auto dimLB{ToInt64(lb[dim])};
     auto dimUB{ToInt64(ub[dim])};
     if (dimUB && dimLB && *dimUB < *dimLB) {
       AttachDeclaration(
-          Warn(common::UsageWarning::SubscriptedEmptyArray,
+          context.Warn(common::UsageWarning::SubscriptedEmptyArray,
+              context.foldingContext().messages().at(),
               "Empty array dimension %d should not be subscripted as an element or non-empty array section"_err_en_US,
               dim + 1),
           arraySymbol);
@@ -429,35 +401,105 @@ void ExpressionAnalyzer::CheckSubscripts(ArrayRef &ref) {
     }
     for (int j{0}; j < vals; ++j) {
       if (val[j]) {
-        std::optional<parser::MessageFixedText> msg;
-        std::optional<ConstantSubscript> bound;
-        if (dimLB && *val[j] < *dimLB) {
-          msg =
-              "Subscript %jd is less than lower bound %jd for dimension %d of array"_err_en_US;
-          bound = *dimLB;
-        } else if (dimUB && *val[j] > *dimUB) {
-          msg =
-              "Subscript %jd is greater than upper bound %jd for dimension %d of array"_err_en_US;
-          bound = *dimUB;
-          if (dim + 1 == arraySymbol.Rank() && IsDummy(arraySymbol) &&
-              *bound == 1) {
-            // Old-school overindexing of a dummy array isn't fatal when
-            // it's on the last dimension and the extent is 1.
-            msg->set_severity(parser::Severity::Warning);
-          }
-        }
-        if (msg) {
-          AttachDeclaration(
-              Say(std::move(*msg), static_cast<std::intmax_t>(*val[j]),
-                  static_cast<std::intmax_t>(bound.value()), dim + 1),
-              arraySymbol);
-        }
+        ValidateSubscriptValue(context.foldingContext().messages(), arraySymbol,
+            *val[j], dimLB, dimUB, dim);
       }
     }
     ++dim;
   }
 }
 
+static void CheckSubscripts(
+    semantics::SemanticsContext &context, ArrayRef &ref) {
+  const Symbol &arraySymbol{ref.base().GetLastSymbol()};
+  Shape lb, ub;
+  if (FoldSubscripts(context, arraySymbol, ref.subscript(), lb, ub)) {
+    ValidateSubscripts(context, arraySymbol, ref.subscript(), lb, ub);
+  }
+}
+
+static void CheckSubscripts(
+    semantics::SemanticsContext &context, CoarrayRef &ref) {
+  const Symbol &coarraySymbol{ref.GetBase().GetLastSymbol()};
+  Shape lb, ub;
+  if (FoldSubscripts(context, coarraySymbol, ref.subscript(), lb, ub)) {
+    ValidateSubscripts(context, coarraySymbol, ref.subscript(), lb, ub);
+  }
+  FoldingContext &foldingContext{context.foldingContext()};
+  int dim{0};
+  for (auto &expr : ref.cosubscript()) {
+    expr = Fold(foldingContext, std::move(expr));
+    if (auto val{ToInt64(expr)}) {
+      ValidateSubscriptValue(foldingContext.messages(), coarraySymbol, *val,
+          ToInt64(GetLCOBOUND(coarraySymbol, dim)),
+          ToInt64(GetUCOBOUND(coarraySymbol, dim)), dim, "co");
+    }
+    ++dim;
+  }
+}
+
+// Some subscript semantic checks must be deferred until all of the
+// subscripts are in hand.
+MaybeExpr ExpressionAnalyzer::CompleteSubscripts(ArrayRef &&ref) {
+  const Symbol &symbol{ref.GetLastSymbol().GetUltimate()};
+  int symbolRank{symbol.Rank()};
+  int subscripts{static_cast<int>(ref.size())};
+  if (subscripts == 0) {
+    return std::nullopt; // error recovery
+  } else if (subscripts != symbolRank) {
+    if (symbolRank != 0) {
+      Say("Reference to rank-%d object '%s' has %d subscripts"_err_en_US,
+          symbolRank, symbol.name(), subscripts);
+    }
+    return std::nullopt;
+  } else if (symbol.has<semantics::ObjectEntityDetails>() ||
+      symbol.has<semantics::AssocEntityDetails>()) {
+    // C928 & C1002
+    if (Triplet * last{std::get_if<Triplet>(&ref.subscript().back().u)}) {
+      if (!last->upper() && IsAssumedSizeArray(symbol)) {
+        Say("Assumed-size array '%s' must have explicit final subscript upper bound value"_err_en_US,
+            symbol.name());
+        return std::nullopt;
+      }
+    }
+  } else {
+    // Shouldn't get here from Analyze(ArrayElement) without a valid base,
+    // which, if not an object, must be a construct entity from
+    // SELECT TYPE/RANK or ASSOCIATE.
+    CHECK(symbol.has<semantics::AssocEntityDetails>());
+  }
+  if (!semantics::IsNamedConstant(symbol) && !inDataStmtObject_) {
+    // Subscripts of named constants are checked in folding.
+    // Subscripts of DATA statement objects are checked in data statement
+    // conversion to initializers.
+    CheckSubscripts(context_, ref);
+  }
+  return Designate(DataRef{std::move(ref)});
+}
+
+// Applies subscripts to a data reference.
+MaybeExpr ExpressionAnalyzer::ApplySubscripts(
+    DataRef &&dataRef, std::vector<Subscript> &&subscripts) {
+  if (subscripts.empty()) {
+    return std::nullopt; // error recovery
+  }
+  return common::visit(common::visitors{
+                           [&](SymbolRef &&symbol) {
+                             return CompleteSubscripts(
+                                 ArrayRef{symbol, std::move(subscripts)});
+                           },
+                           [&](Component &&c) {
+                             return CompleteSubscripts(
+                                 ArrayRef{std::move(c), std::move(subscripts)});
+                           },
+                           [&](auto &&) -> MaybeExpr {
+                             DIE("bad base for ArrayRef");
+                             return std::nullopt;
+                           },
+                       },
+      std::move(dataRef.u));
+}
+
 // C919a - only one part-ref of a data-ref may have rank > 0
 bool ExpressionAnalyzer::CheckRanks(const DataRef &dataRef) {
   return common::visit(
@@ -608,8 +650,25 @@ MaybeExpr ExpressionAnalyzer::Analyze(const parser::Designator &d) {
       dataRef = ExtractDataRef(std::move(result),
           /*intoSubstring=*/false, /*intoComplexPart=*/true);
     }
-    if (dataRef && !CheckDataRef(*dataRef)) {
-      result.reset();
+    if (dataRef) {
+      if (!CheckDataRef(*dataRef)) {
+        result.reset();
+      } else if (ExtractCoarrayRef(*dataRef).has_value()) {
+        if (auto dyType{result->GetType()};
+            dyType && dyType->category() == TypeCategory::Derived) {
+          if (!std::holds_alternative<CoarrayRef>(dataRef->u) &&
+              dyType->IsPolymorphic()) { // F'2023 C918
+            Say("The base of a polymorphic object may not be coindexed"_err_en_US);
+          }
+          if (const auto *derived{GetDerivedTypeSpec(*dyType)}) {
+            if (auto bad{FindPolymorphicAllocatablePotentialComponent(
+                    *derived)}) { // F'2023 C917
+              Say("A coindexed designator may not have a type with the polymorphic potential subobject component '%s'"_err_en_US,
+                  bad.BuildResultDesignatorName());
+            }
+          }
+        }
+      }
     }
   }
   return result;
@@ -1524,9 +1583,10 @@ MaybeExpr ExpressionAnalyzer::Analyze(const parser::CoindexedNamedObject &x) {
     // Reverse the chain of symbols so that the base is first and coarray
     // ultimate component is last.
     if (cosubsOk) {
-      return Designate(
-          DataRef{CoarrayRef{SymbolVector{reversed.crbegin(), reversed.crend()},
-              std::move(subscripts), std::move(cosubscripts)}});
+      CoarrayRef coarrayRef{SymbolVector{reversed.crbegin(), reversed.crend()},
+          std::move(subscripts), std::move(cosubscripts)};
+      CheckSubscripts(context_, coarrayRef);
+      return Designate(DataRef{std::move(coarrayRef)});
     }
   }
   return std::nullopt;
@@ -2487,6 +2547,15 @@ auto ExpressionAnalyzer::AnalyzeProcedureComponentRef(
           return CalleeAndArguments{
               ProcedureDesignator{*resolution}, std::move(arguments)};
         } else if (dataRef.has_value()) {
+          if (ExtractCoarrayRef(*dataRef)) {
+            if (IsProcedurePointer(*sym)) {
+              Say(sc.component.source,
+                  "Base of procedure component reference may not be coindexed"_err_en_US);
+            } else {
+              Say(sc.component.source,
+                  "A procedure binding may not be coindexed unless it can be resolved at compilation time"_err_en_US);
+            }
+          }
           if (sym->attrs().test(semantics::Attr::NOPASS)) {
             const auto *dtSpec{GetDerivedTypeSpec(dtExpr->GetType())};
             if (dtSpec && dtSpec->scope()) {
@@ -3293,15 +3362,24 @@ const Assignment *ExpressionAnalyzer::Analyze(const parser::AssignmentStmt &x) {
             "in a non-pointer intrinsic assignment statement");
         analyzer.CheckForAssumedRank("in an assignment statement");
         const Expr<SomeType> &lhs{analyzer.GetExpr(0)};
-        if (auto dyType{lhs.GetType()};
-            dyType && dyType->IsPolymorphic()) { // 10.2.1.2p1(1)
-          const Symbol *lastWhole0{UnwrapWholeSymbolOrComponentDataRef(lhs)};
-          const Symbol *lastWhole{
-              lastWhole0 ? &ResolveAssociations(*lastWhole0) : nullptr};
-          if (!lastWhole || !IsAllocatable(*lastWhole)) {
-            Say("Left-hand side of assignment may not be polymorphic unless assignment is to an entire allocatable"_err_en_US);
-          } else if (evaluate::IsCoarray(*lastWhole)) {
-            Say("Left-hand side of assignment may not be polymorphic if it is a coarray"_err_en_US);
+        if (auto dyType{lhs.GetType()}) {
+          if (dyType->IsPolymorphic()) { // 10.2.1.2p1(1)
+            const Symbol *lastWhole0{UnwrapWholeSymbolOrComponentDataRef(lhs)};
+            const Symbol *lastWhole{
+                lastWhole0 ? &ResolveAssociations(*lastWhole0) : nullptr};
+            if (!lastWhole || !IsAllocatable(*lastWhole)) {
+              Say("Left-hand side of assignment may not be polymorphic unless assignment is to an entire allocatable"_err_en_US);
+            } else if (evaluate::IsCoarray(*lastWhole)) {
+              Say("Left-hand side of assignment may not be polymorphic if it is a coarray"_err_en_US);
+            }
+          }
+          if (auto *derived{GetDerivedTypeSpec(*dyType)}) {
+            if (auto iter{FindAllocatableUltimateComponent(*derived)}) {
+              if (ExtractCoarrayRef(lhs)) {
+                Say("Left-hand side of assignment must not be coindexed due to allocatable ultimate component '%s'"_err_en_US,
+                    iter.BuildResultDesignatorName());
+              }
+            }
           }
         }
       }
diff --git a/flang/lib/Semantics/mod-file.cpp b/flang/lib/Semantics/mod-file.cpp
index ac176978a45a5..c3b46261228df 100644
--- a/flang/lib/Semantics/mod-file.cpp
+++ b/flang/lib/Semantics/mod-file.cpp
@@ -978,11 +978,9 @@ void ModFileWriter::PutObjectEntity(
         << ") " << symbol.name() << '\n';
   }
   if (symbol.test(Fortran::semantics::Symbol::Flag::CrayPointer)) {
-    if (!symbol.owner().crayPointers().empty()) {
-      for (const auto &[pointee, pointer] : symbol.owner().crayPointers()) {
-        if (pointer == symbol) {
-          os << "pointer(" << symbol.name() << "," << pointee << ")\n";
-        }
+    for (const auto &[pointee, pointer] : symbol.owner().crayPointers()) {
+      if (pointer == symbol) {
+        os << "pointer(" << symbol.name() << "," << pointee << ")\n";
       }
     }
   }
@@ -1725,6 +1723,17 @@ void SubprogramSymbolCollector::DoSymbol(
   if (!scope.IsDerivedType()) {
     need_.push_back(symbol);
   }
+  if (symbol.test(Fortran::semantics::Symbol::Flag::CrayPointer)) {
+    for (const auto &[pointee, pointer] : symbol.owner().crayPointers()) {
+      if (&*pointer == &symbol) {
+        auto iter{symbol.owner().find(pointee)};
+        CHECK(iter != symbol.owner().end());
+        DoSymbol(*iter->second);
+      }
+    }
+  } else if (symbol.test(Fortran::semantics::Symbol::Flag::CrayPointee)) {
+    DoSymbol(GetCrayPointer(symbol));
+  }
 }
 
 void SubprogramSymbolCollector::DoType(const DeclTypeSpec *type) {
diff --git a/flang/lib/Semantics/pointer-assignment.cpp b/flang/lib/Semantics/pointer-assignment.cpp
index 654c01856cfd4..23a64d05338be 100644
--- a/flang/lib/Semantics/pointer-assignment.cpp
+++ b/flang/lib/Semantics/pointer-assignment.cpp
@@ -154,6 +154,9 @@ bool PointerAssignmentChecker::CheckLeftHandSide(const SomeExpr &lhs) {
   } else if (evaluate::IsAssumedRank(lhs)) {
     Say("The left-hand side of a pointer assignment must not be an assumed-rank dummy argument"_err_en_US);
     return false;
+  } else if (evaluate::ExtractCoarrayRef(lhs)) { // F'2023 C1027
+    Say("The left-hand side of a pointer assignment must not be coindexed"_err_en_US);
+    return false;
   } else {
     return true;
   }
@@ -177,7 +180,7 @@ bool PointerAssignmentChecker::Check(const SomeExpr &rhs) {
     Say("An array section with a vector subscript may not be a pointer target"_err_en_US);
     return false;
   }
-  if (ExtractCoarrayRef(rhs)) { // C1026
+  if (ExtractCoarrayRef(rhs)) { // F'2023 C1029
     Say("A coindexed object may not be a pointer target"_err_en_US);
     return false;
   }
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 38888a4dc1461..ce96eb9b7782e 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -352,7 +352,7 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
   }
 
   bool Pre(const parser::OmpDirectiveSpecification &x) {
-    PushContext(x.source, std::get<llvm::omp::Directive>(x.t));
+    PushContext(x.source, x.DirId());
     return true;
   }
   void Post(const parser::OmpDirectiveSpecification &) { PopContext(); }
@@ -2404,6 +2404,24 @@ void OmpAttributeVisitor::ResolveOmpObjectList(
   }
 }
 
+/// True if either symbol is in a namelist or some other symbol in the same
+/// equivalence set as symbol is in a namelist.
+static bool SymbolOrEquivalentIsInNamelist(const Symbol &symbol) {
+  auto isInNamelist{[](const Symbol &sym) {
+    const Symbol &ultimate{sym.GetUltimate()};
+    return ultimate.test(Symbol::Flag::InNamelist);
+  }};
+
+  const EquivalenceSet *eqv{FindEquivalenceSet(symbol)};
+  if (!eqv) {
+    return isInNamelist(symbol);
+  }
+
+  return llvm::any_of(*eqv, [isInNamelist](const EquivalenceObject &obj) {
+    return isInNamelist(obj.symbol);
+  });
+}
+
 void OmpAttributeVisitor::ResolveOmpObject(
     const parser::OmpObject &ompObject, Symbol::Flag ompFlag) {
   common::visit(
@@ -2468,7 +2486,6 @@ void OmpAttributeVisitor::ResolveOmpObject(
                               .str()));
                 }
                 if (ompFlag == Symbol::Flag::OmpReduction) {
-                  const Symbol &ultimateSymbol{symbol->GetUltimate()};
                   // Using variables inside of a namelist in OpenMP reductions
                   // is allowed by the standard, but is not allowed for
                   // privatisation. This looks like an oversight. If the
@@ -2476,7 +2493,7 @@ void OmpAttributeVisitor::ResolveOmpObject(
                   // mapping for the reduction variable: resulting in incorrect
                   // results. Disabling this hoisting could make some real
                   // production code go slower. See discussion in #109303
-                  if (ultimateSymbol.test(Symbol::Flag::InNamelist)) {
+                  if (SymbolOrEquivalentIsInNamelist(*symbol)) {
                     context_.Say(name->source,
                         "Variable '%s' in NAMELIST cannot be in a REDUCTION clause"_err_en_US,
                         name->ToString());
@@ -2838,7 +2855,7 @@ void OmpAttributeVisitor::CheckObjectIsPrivatizable(
     clauseName = "LASTPRIVATE";
   }
 
-  if (ultimateSymbol.test(Symbol::Flag::InNamelist)) {
+  if (SymbolOrEquivalentIsInNamelist(symbol)) {
     context_.Say(name.source,
         "Variable '%s' in NAMELIST cannot be in a %s clause"_err_en_US,
         name.ToString(), clauseName.str());
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 4f80cdca0f4bb..e57abf8ac0912 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -1775,11 +1775,10 @@ bool OmpVisitor::Pre(const parser::OmpDirectiveSpecification &x) {
   // Disable the semantic analysis for it for now to allow the compiler to
   // parse METADIRECTIVE without flagging errors.
   AddOmpSourceRange(x.source);
-  auto dirId{std::get<llvm::omp::Directive>(x.t)};
   auto &maybeArgs{std::get<std::optional<std::list<parser::OmpArgument>>>(x.t)};
   auto &maybeClauses{std::get<std::optional<parser::OmpClauseList>>(x.t)};
 
-  switch (dirId) {
+  switch (x.DirId()) {
   case llvm::omp::Directive::OMPD_declare_mapper:
     if (maybeArgs && maybeClauses) {
       const parser::OmpArgument &first{maybeArgs->front()};
diff --git a/flang/lib/Semantics/type.cpp b/flang/lib/Semantics/type.cpp
index e867d7ad6e253..c5a75c4d619c5 100644
--- a/flang/lib/Semantics/type.cpp
+++ b/flang/lib/Semantics/type.cpp
@@ -567,11 +567,18 @@ const DeclTypeSpec &InstantiateHelper::InstantiateIntrinsicType(
       kind = *value;
     } else {
       foldingContext().messages().Say(symbolName,
-          "KIND parameter value (%jd) of intrinsic type %s "
-          "did not resolve to a supported value"_err_en_US,
+          "KIND parameter value (%jd) of intrinsic type %s did not resolve to a supported value"_err_en_US,
           *value,
           parser::ToUpperCaseLetters(EnumToString(intrinsic.category())));
     }
+  } else {
+    std::string exprString;
+    llvm::raw_string_ostream sstream(exprString);
+    copy.AsFortran(sstream);
+    foldingContext().messages().Say(symbolName,
+        "KIND parameter expression (%s) of intrinsic type %s did not resolve to a constant value"_err_en_US,
+        exprString,
+        parser::ToUpperCaseLetters(EnumToString(intrinsic.category())));
   }
   switch (spec.category()) {
   case DeclTypeSpec::Numeric:
diff --git a/flang/lib/Support/OpenMP-utils.cpp b/flang/lib/Support/OpenMP-utils.cpp
index 178a6e38dc0f2..97e7723f0be8c 100644
--- a/flang/lib/Support/OpenMP-utils.cpp
+++ b/flang/lib/Support/OpenMP-utils.cpp
@@ -18,10 +18,11 @@ mlir::Block *genEntryBlock(mlir::OpBuilder &builder, const EntryBlockArgs &args,
 
   llvm::SmallVector<mlir::Type> types;
   llvm::SmallVector<mlir::Location> locs;
-  unsigned numVars = args.hostEvalVars.size() + args.inReduction.vars.size() +
-      args.map.vars.size() + args.priv.vars.size() +
-      args.reduction.vars.size() + args.taskReduction.vars.size() +
-      args.useDeviceAddr.vars.size() + args.useDevicePtr.vars.size();
+  unsigned numVars = args.hasDeviceAddr.vars.size() + args.hostEvalVars.size() +
+      args.inReduction.vars.size() + args.map.vars.size() +
+      args.priv.vars.size() + args.reduction.vars.size() +
+      args.taskReduction.vars.size() + args.useDeviceAddr.vars.size() +
+      args.useDevicePtr.vars.size();
   types.reserve(numVars);
   locs.reserve(numVars);
 
@@ -34,6 +35,7 @@ mlir::Block *genEntryBlock(mlir::OpBuilder &builder, const EntryBlockArgs &args,
 
   // Populate block arguments in clause name alphabetical order to match
   // expected order by the BlockArgOpenMPOpInterface.
+  extractTypeLoc(args.hasDeviceAddr.vars);
   extractTypeLoc(args.hostEvalVars);
   extractTypeLoc(args.inReduction.vars);
   extractTypeLoc(args.map.vars);
diff --git a/flang/module/cudadevice.f90 b/flang/module/cudadevice.f90
index 4491e9f653270..baaa112f5d8c2 100644
--- a/flang/module/cudadevice.f90
+++ b/flang/module/cudadevice.f90
@@ -1118,6 +1118,31 @@ attributes(device) integer function match_any_syncjd(mask, val)
       !dir$ ignore_tkr (d) x
       complex(8), intent(in) :: x
     end function
+    attributes(device) pure function __ldca_i4x4(x) result(y)
+      !dir$ ignore_tkr (d) x
+      integer(4), dimension(4), intent(in) :: x
+      integer(4), dimension(4) :: y
+    end function
+    attributes(device) pure function __ldca_i8x2(x) result(y)
+      !dir$ ignore_tkr (d) x
+      integer(8), dimension(2), intent(in) :: x
+      integer(8), dimension(2) :: y
+    end function
+    attributes(device) pure function __ldca_r2x2(x) result(y)
+      !dir$ ignore_tkr (d) x
+      real(2), dimension(2), intent(in) :: x
+      real(2), dimension(2) :: y
+    end function
+    attributes(device) pure function __ldca_r4x4(x) result(y)
+      !dir$ ignore_tkr (d) x
+      real(4), dimension(4), intent(in) :: x
+      real(4), dimension(4) :: y
+    end function
+    attributes(device) pure function __ldca_r8x2(x) result(y)
+      !dir$ ignore_tkr (d) x
+      real(8), dimension(2), intent(in) :: x
+      real(8), dimension(2) :: y
+    end function
   end interface
 
   ! LDCS
@@ -1158,6 +1183,31 @@ attributes(device) integer function match_any_syncjd(mask, val)
       !dir$ ignore_tkr (d) x
       complex(8), intent(in) :: x
     end function
+    attributes(device) pure function __ldcs_i4x4(x) result(y)
+      !dir$ ignore_tkr (d) x
+      integer(4), dimension(4), intent(in) :: x
+      integer(4), dimension(4) :: y
+    end function
+    attributes(device) pure function __ldcs_i8x2(x) result(y)
+      !dir$ ignore_tkr (d) x
+      integer(8), dimension(2), intent(in) :: x
+      integer(8), dimension(2) :: y
+    end function
+    attributes(device) pure function __ldcs_r2x2(x) result(y)
+      !dir$ ignore_tkr (d) x
+      real(2), dimension(2), intent(in) :: x
+      real(2), dimension(2) :: y
+    end function
+    attributes(device) pure function __ldcs_r4x4(x) result(y)
+      !dir$ ignore_tkr (d) x
+      real(4), dimension(4), intent(in) :: x
+      real(4), dimension(4) :: y
+    end function
+    attributes(device) pure function __ldcs_r8x2(x) result(y)
+      !dir$ ignore_tkr (d) x
+      real(8), dimension(2), intent(in) :: x
+      real(8), dimension(2) :: y
+    end function
   end interface
 
   ! LDLU
@@ -1198,6 +1248,31 @@ attributes(device) integer function match_any_syncjd(mask, val)
       !dir$ ignore_tkr (d) x
       complex(8), intent(in) :: x
     end function
+    attributes(device) pure function __ldlu_i4x4(x) result(y)
+      !dir$ ignore_tkr (d) x
+      integer(4), dimension(4), intent(in) :: x
+      integer(4), dimension(4) :: y
+    end function
+    attributes(device) pure function __ldlu_i8x2(x) result(y)
+      !dir$ ignore_tkr (d) x
+      integer(8), dimension(2), intent(in) :: x
+      integer(8), dimension(2) :: y
+    end function
+    attributes(device) pure function __ldlu_r2x2(x) result(y)
+      !dir$ ignore_tkr (d) x
+      real(2), dimension(2), intent(in) :: x
+      real(2), dimension(2) :: y
+    end function
+    attributes(device) pure function __ldlu_r4x4(x) result(y)
+      !dir$ ignore_tkr (d) x
+      real(4), dimension(4), intent(in) :: x
+      real(4), dimension(4) :: y
+    end function
+    attributes(device) pure function __ldlu_r8x2(x) result(y)
+      !dir$ ignore_tkr (d) x
+      real(8), dimension(2), intent(in) :: x
+      real(8), dimension(2) :: y
+    end function
   end interface
 
   ! LDCV
@@ -1238,6 +1313,31 @@ attributes(device) integer function match_any_syncjd(mask, val)
       !dir$ ignore_tkr (d) x
       complex(8), intent(in) :: x
     end function
+    attributes(device) pure function __ldcv_i4x4(x) result(y)
+      !dir$ ignore_tkr (d) x
+      integer(4), dimension(4), intent(in) :: x
+      integer(4), dimension(4) :: y
+      end function
+    attributes(device) pure function __ldcv_i8x2(x) result(y)
+      !dir$ ignore_tkr (d) x
+      integer(8), dimension(2), intent(in) :: x
+      integer(8), dimension(2) :: y
+    end function
+    attributes(device) pure function __ldcv_r2x2(x) result(y)
+      !dir$ ignore_tkr (d) x
+      real(2), dimension(2), intent(in) :: x
+      real(2), dimension(2) :: y
+    end function
+    attributes(device) pure function __ldcv_r4x4(x) result(y)
+      !dir$ ignore_tkr (d) x
+      real(4), dimension(4), intent(in) :: x
+      real(4), dimension(4) :: y
+    end function
+    attributes(device) pure function __ldcv_r8x2(x) result(y)
+      !dir$ ignore_tkr (d) x
+      real(8), dimension(2), intent(in) :: x
+      real(8), dimension(2) :: y
+    end function
   end interface
 
   ! STWB
diff --git a/flang/runtime/CMakeLists.txt b/flang/runtime/CMakeLists.txt
index fbd1e1e2cc56e..f9378d4ed4312 100644
--- a/flang/runtime/CMakeLists.txt
+++ b/flang/runtime/CMakeLists.txt
@@ -132,6 +132,12 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "XL")
   set(NO_RTTI_FLAGS "-qnoeh -qnortti")
 endif ()
 
+# When compiling LLVM_ENABLE_RUNTIMES=flang-rt, the build system looks for the
+# full path of quadmath.h. In this non-runtimes build, preserve the old behavior
+# of just including <quadmath.h>.
+set(FLANG_RT_INCLUDE_QUADMATH_H "<quadmath.h>")
+configure_file("${FLANG_RT_SOURCE_DIR}/cmake/quadmath_wrapper.h.in" "quadmath_wrapper.h")
+
 configure_file("${FLANG_RT_SOURCE_DIR}/cmake/config.h.cmake.in" config.h)
 # include_directories is used here instead of target_include_directories
 # because add_flang_library creates multiple objects (STATIC/SHARED, OBJECT)
diff --git a/flang/test/Analysis/AliasAnalysis/source-kind.fir b/flang/test/Analysis/AliasAnalysis/source-kind.fir
new file mode 100644
index 0000000000000..b23f5bc464c8e
--- /dev/null
+++ b/flang/test/Analysis/AliasAnalysis/source-kind.fir
@@ -0,0 +1,26 @@
+// REQUIRES: asserts
+// RUN: fir-opt %s -pass-pipeline='builtin.module(func.func(test-fir-alias-analysis))' -debug-only=fir-alias-analysis 2>&1 | FileCheck %s
+
+// CHECK-LABEL: Testing : "_QFPtest"
+
+// Checking that the source kind of a load of a load is SourceKind::Indirect
+// CHECK: {test.ptr = "load_load"}
+// CHECK-NEXT: SourceKind: Indirect
+
+// Checking that the source kind of a load of an arg is SourceKind::Argument
+// CHECK: {test.ptr = "load_arg"}
+// CHECK-NEXT: SourceKind: Argument
+
+func.func @_QFPtest(%arg0: !fir.ref<!fir.box<!fir.ptr<f32>>> ) attributes {test.ptr = "func"} {
+  
+  %0 = fir.alloca !fir.llvm_ptr<!fir.box<!fir.ptr<f32>>>
+  %1 = fir.convert %arg0 : (!fir.ref<!fir.box<!fir.ptr<f32>>>) -> !fir.llvm_ptr<!fir.box<!fir.ptr<f32>>>
+  fir.store %1 to %0 : !fir.ref<!fir.llvm_ptr<!fir.box<!fir.ptr<f32>>>>
+  %2 = fir.load %0 : !fir.ref<!fir.llvm_ptr<!fir.box<!fir.ptr<f32>>>>
+  %3 = fir.convert %2 : (!fir.llvm_ptr<!fir.box<!fir.ptr<f32>>>) -> !fir.ref<!fir.box<!fir.ptr<f32>>>
+  %15 = fir.load %3 : !fir.ref<!fir.box<!fir.ptr<f32>>>
+  %16 = fir.box_addr %15 {test.ptr = "load_load"} : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
+  %17 = fir.load %arg0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
+  %18 = fir.box_addr %17 {test.ptr = "load_arg"} : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
+  return
+}
\ No newline at end of file
diff --git a/flang/test/Fir/CUDA/cuda-target-rewrite.mlir b/flang/test/Fir/CUDA/cuda-target-rewrite.mlir
index 0e7534e06c89c..831b5a0048008 100644
--- a/flang/test/Fir/CUDA/cuda-target-rewrite.mlir
+++ b/flang/test/Fir/CUDA/cuda-target-rewrite.mlir
@@ -1,6 +1,7 @@
-// REQUIRES: x86-registered-target
 // RUN: fir-opt --split-input-file --target-rewrite="target=x86_64-unknown-linux-gnu" %s | FileCheck %s
 
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
+
 gpu.module @testmod {
   gpu.func @_QPvcpowdk(%arg0: !fir.ref<complex<f64>> {cuf.data_attr = #cuf.cuda<device>, fir.bindc_name = "a"}) attributes {cuf.proc_attr = #cuf.cuda_proc<global>} {
     %0 = fir.alloca i64
@@ -15,22 +16,25 @@ gpu.module @testmod {
 // CHECK-LABEL: gpu.func @_QPvcpowdk
 // CHECK: %{{.*}} = fir.call @_FortranAzpowk(%{{.*}}, %{{.*}}, %{{.*}}) : (f64, f64, i64) -> tuple<f64, f64>
 // CHECK: func.func private @_FortranAzpowk(f64, f64, i64) -> tuple<f64, f64> attributes {fir.bindc_name = "_FortranAzpowk", fir.runtime}
+}
 
 // -----
 
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
+
 gpu.module @testmod {
   gpu.func @_QPtest(%arg0: complex<f64>) -> (complex<f64>) {
     gpu.return %arg0 : complex<f64>
   }
 }
+}
 
 // CHECK-LABEL: gpu.func @_QPtest
 // CHECK-SAME: (%arg0: f64, %arg1: f64) -> tuple<f64, f64> {
 // CHECK: gpu.return %{{.*}} : tuple<f64, f64>
 
-
 // -----
-module attributes {gpu.container_module} {
+module attributes {gpu.container_module, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
 
 gpu.module @testmod {
   gpu.func @_QPtest(%arg0: complex<f64>) -> () kernel {
diff --git a/flang/test/Fir/alloc-32.fir b/flang/test/Fir/alloc-32.fir
new file mode 100644
index 0000000000000..3eefc3225fac7
--- /dev/null
+++ b/flang/test/Fir/alloc-32.fir
@@ -0,0 +1,30 @@
+// RUN: %flang_fc1 -triple i686 -emit-llvm  %s -o - | FileCheck %s
+// REQUIRES: x86-registered-target
+
+// This is a check for calling malloc using i32 when on a 32-bit target (only).
+// It doesn't contain the comprehensive tests that alloc.fir has, and
+// that file should be used to exercise most code paths.
+
+module attributes {
+    fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.target_triple = "i686"
+} {
+
+// CHECK-LABEL: define ptr @allocmem_scalar_nonchar(
+// CHECK: call ptr @malloc(i32 4)
+func.func @allocmem_scalar_nonchar() -> !fir.heap<i32> {
+  %1 = fir.allocmem i32
+  return %1 : !fir.heap<i32>
+}
+
+// CHECK-LABEL: define ptr @allocmem_scalar_dynchar(
+// CHECK-SAME: i32 %[[len:.*]])
+// CHECK: %[[mul1:.*]] = sext i32 %[[len]] to i64
+// CHECK: %[[mul2:.*]] = mul i64 1, %[[mul1]]
+// CHECK: %[[trunc:.*]] = trunc i64 %[[mul2]] to i32
+// CHECK: call ptr @malloc(i32 %[[trunc]])
+func.func @allocmem_scalar_dynchar(%l : i32) -> !fir.heap<!fir.char<1,?>> {
+  %1 = fir.allocmem !fir.char<1,?>(%l : i32)
+  return %1 : !fir.heap<!fir.char<1,?>>
+}
+
+}
diff --git a/flang/test/Fir/alloc.fir b/flang/test/Fir/alloc.fir
index ba9b08dad7764..5b4930bb9cb34 100644
--- a/flang/test/Fir/alloc.fir
+++ b/flang/test/Fir/alloc.fir
@@ -2,6 +2,10 @@
 // RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
 
 // UNSUPPORTED: system-windows
+// Disabled on 32-bit targets due to the additional `trunc` opcodes required
+// UNSUPPORTED: target-x86
+// UNSUPPORTED: target=sparc-{{.*}}
+// UNSUPPORTED: target=sparcel-{{.*}}
 
 // CHECK-LABEL: define ptr @alloca_scalar_nonchar()
 // CHECK: alloca i32, i64 1
diff --git a/flang/test/Fir/convert-to-llvm.fir b/flang/test/Fir/convert-to-llvm.fir
index c7037019ee701..08ef4d8ed34f2 100644
--- a/flang/test/Fir/convert-to-llvm.fir
+++ b/flang/test/Fir/convert-to-llvm.fir
@@ -690,32 +690,134 @@ func.func @compare_complex_other(%a : complex<f64>, %b : complex<f64>) -> i1 {
 
 // Test `fir.convert` operation conversion from Float type.
 
-func.func @convert_from_float(%arg0 : f32) {
+func.func @convert_from_float_to_float(%arg0 : f32) {
   %0 = fir.convert %arg0 : (f32) -> f16
   %1 = fir.convert %arg0 : (f32) -> f32
   %2 = fir.convert %arg0 : (f32) -> f64
   %3 = fir.convert %arg0 : (f32) -> f80
   %4 = fir.convert %arg0 : (f32) -> f128
-  %5 = fir.convert %arg0 : (f32) -> i1
-  %6 = fir.convert %arg0 : (f32) -> i8
-  %7 = fir.convert %arg0 : (f32) -> i16
-  %8 = fir.convert %arg0 : (f32) -> i32
-  %9 = fir.convert %arg0 : (f32) -> i64
   return
 }
 
-// CHECK-LABEL: convert_from_float(
+// CHECK-LABEL: convert_from_float_to_float(
 // CHECK-SAME:                     %[[ARG0:.*]]: f32
 // CHECK:         %{{.*}} = llvm.fptrunc %[[ARG0]] : f32 to f16
 // CHECK-NOT:     f32 to f32
 // CHECK:         %{{.*}} = llvm.fpext %[[ARG0]] : f32 to f64
 // CHECK:         %{{.*}} = llvm.fpext %[[ARG0]] : f32 to f80
 // CHECK:         %{{.*}} = llvm.fpext %[[ARG0]] : f32 to f128
-// CHECK:         %{{.*}} = llvm.fptosi %[[ARG0]] : f32 to i1
-// CHECK:         %{{.*}} = llvm.fptosi %[[ARG0]] : f32 to i8
-// CHECK:         %{{.*}} = llvm.fptosi %[[ARG0]] : f32 to i16
-// CHECK:         %{{.*}} = llvm.fptosi %[[ARG0]] : f32 to i32
-// CHECK:         %{{.*}} = llvm.fptosi %[[ARG0]] : f32 to i64
+
+func.func @convert_from_f16_to_integer(%arg0 : f16) {
+  %1 = fir.convert %arg0 : (f16) -> i1
+  %2 = fir.convert %arg0 : (f16) -> i8
+  %3 = fir.convert %arg0 : (f16) -> i16
+  %4 = fir.convert %arg0 : (f16) -> i32
+  %5 = fir.convert %arg0 : (f16) -> i64
+  %6 = fir.convert %arg0 : (f16) -> i128
+  %7 = fir.convert %arg0 : (f16) -> ui8
+  %8 = fir.convert %arg0 : (f16) -> ui16
+  %9 = fir.convert %arg0 : (f16) -> ui32
+  %10 = fir.convert %arg0 : (f16) -> ui64
+  %11 = fir.convert %arg0 : (f16) -> ui128
+  return
+}
+
+// CHECK-LABEL: @convert_from_f16_to_integer(%arg0:
+// CHECK: llvm.call_intrinsic "llvm.fptosi.sat"(%{{.+}}) : (f16) -> i1
+// CHECK: llvm.call_intrinsic "llvm.fptosi.sat"(%{{.+}}) : (f16) -> i8
+// CHECK: llvm.fptosi %arg0 : f16 to i16
+// CHECK: llvm.fptosi %arg0 : f16 to i32
+// CHECK: llvm.fptosi %arg0 : f16 to i64
+// CHECK: llvm.fptosi %arg0 : f16 to i128
+// CHECK: llvm.call_intrinsic "llvm.fptoui.sat"(%{{.+}}) : (f16) -> i8
+// CHECK: llvm.fptoui %arg0 : f16 to i16
+// CHECK: llvm.fptoui %arg0 : f16 to i32
+// CHECK: llvm.fptoui %arg0 : f16 to i64
+// CHECK: llvm.fptoui %arg0 : f16 to i128
+
+func.func @convert_from_f32_to_integer(%arg0 : f32) {
+  %1 = fir.convert %arg0 : (f32) -> i1
+  %2 = fir.convert %arg0 : (f32) -> i8
+  %3 = fir.convert %arg0 : (f32) -> i16
+  %4 = fir.convert %arg0 : (f32) -> i32
+  %5 = fir.convert %arg0 : (f32) -> i64
+  %6 = fir.convert %arg0 : (f32) -> i128
+  %7 = fir.convert %arg0 : (f32) -> ui8
+  %8 = fir.convert %arg0 : (f32) -> ui16
+  %9 = fir.convert %arg0 : (f32) -> ui32
+  %10 = fir.convert %arg0 : (f32) -> ui64
+  %11 = fir.convert %arg0 : (f32) -> ui128
+  return
+}
+
+// CHECK-LABEL: @convert_from_f32_to_integer(%arg0:
+// CHECK: llvm.call_intrinsic "llvm.fptosi.sat"(%{{.+}}) : (f32) -> i1
+// CHECK: llvm.call_intrinsic "llvm.fptosi.sat"(%{{.+}}) : (f32) -> i8
+// CHECK: llvm.call_intrinsic "llvm.fptosi.sat"(%{{.+}}) : (f32) -> i16
+// CHECK: llvm.fptosi %arg0 : f32 to i32
+// CHECK: llvm.fptosi %arg0 : f32 to i64
+// CHECK: llvm.fptosi %arg0 : f32 to i128
+// CHECK: llvm.call_intrinsic "llvm.fptoui.sat"(%{{.+}}) : (f32) -> i8
+// CHECK: llvm.call_intrinsic "llvm.fptoui.sat"(%{{.+}}) : (f32) -> i16
+// CHECK: llvm.fptoui %arg0 : f32 to i32
+// CHECK: llvm.fptoui %arg0 : f32 to i64
+// CHECK: llvm.fptoui %arg0 : f32 to i128
+
+func.func @convert_from_f64_to_integer(%arg0 : f64) {
+  %1 = fir.convert %arg0 : (f64) -> i1
+  %2 = fir.convert %arg0 : (f64) -> i8
+  %3 = fir.convert %arg0 : (f64) -> i16
+  %4 = fir.convert %arg0 : (f64) -> i32
+  %5 = fir.convert %arg0 : (f64) -> i64
+  %6 = fir.convert %arg0 : (f64) -> i128
+  %7 = fir.convert %arg0 : (f64) -> ui8
+  %8 = fir.convert %arg0 : (f64) -> ui16
+  %9 = fir.convert %arg0 : (f64) -> ui32
+  %10 = fir.convert %arg0 : (f64) -> ui64
+  %11 = fir.convert %arg0 : (f64) -> ui128
+  return
+}
+
+// CHECK-LABEL: @convert_from_f64_to_integer(%arg0:
+// CHECK: llvm.call_intrinsic "llvm.fptosi.sat"(%{{.+}}) : (f64) -> i1
+// CHECK: llvm.call_intrinsic "llvm.fptosi.sat"(%{{.+}}) : (f64) -> i8
+// CHECK: llvm.call_intrinsic "llvm.fptosi.sat"(%{{.+}}) : (f64) -> i16
+// CHECK: llvm.call_intrinsic "llvm.fptosi.sat"(%{{.+}}) : (f64) -> i32
+// CHECK: llvm.fptosi %arg0 : f64 to i64
+// CHECK: llvm.fptosi %arg0 : f64 to i128
+// CHECK: llvm.call_intrinsic "llvm.fptoui.sat"(%{{.+}}) : (f64) -> i8
+// CHECK: llvm.call_intrinsic "llvm.fptoui.sat"(%{{.+}}) : (f64) -> i16
+// CHECK: llvm.call_intrinsic "llvm.fptoui.sat"(%{{.+}}) : (f64) -> i32
+// CHECK: llvm.fptoui %arg0 : f64 to i64
+// CHECK: llvm.fptoui %arg0 : f64 to i128
+
+func.func @convert_from_f128_to_integer(%arg0 : f128) {
+  %1 = fir.convert %arg0 : (f128) -> i1
+  %2 = fir.convert %arg0 : (f128) -> i8
+  %3 = fir.convert %arg0 : (f128) -> i16
+  %4 = fir.convert %arg0 : (f128) -> i32
+  %5 = fir.convert %arg0 : (f128) -> i64
+  %6 = fir.convert %arg0 : (f128) -> i128
+  %7 = fir.convert %arg0 : (f128) -> ui8
+  %8 = fir.convert %arg0 : (f128) -> ui16
+  %9 = fir.convert %arg0 : (f128) -> ui32
+  %10 = fir.convert %arg0 : (f128) -> ui64
+  %11 = fir.convert %arg0 : (f128) -> ui128
+  return
+}
+
+// CHECK-LABEL: @convert_from_f128_to_integer(%arg0:
+// CHECK: llvm.call_intrinsic "llvm.fptosi.sat"(%{{.+}}) : (f128) -> i1
+// CHECK: llvm.call_intrinsic "llvm.fptosi.sat"(%{{.+}}) : (f128) -> i8
+// CHECK: llvm.call_intrinsic "llvm.fptosi.sat"(%{{.+}}) : (f128) -> i16
+// CHECK: llvm.call_intrinsic "llvm.fptosi.sat"(%{{.+}}) : (f128) -> i32
+// CHECK: llvm.call_intrinsic "llvm.fptosi.sat"(%{{.+}}) : (f128) -> i64
+// CHECK: llvm.fptosi %arg0 : f128 to i128
+// CHECK: llvm.call_intrinsic "llvm.fptoui.sat"(%{{.+}}) : (f128) -> i8
+// CHECK: llvm.call_intrinsic "llvm.fptoui.sat"(%{{.+}}) : (f128) -> i16
+// CHECK: llvm.call_intrinsic "llvm.fptoui.sat"(%{{.+}}) : (f128) -> i32
+// CHECK: llvm.call_intrinsic "llvm.fptoui.sat"(%{{.+}}) : (f128) -> i64
+// CHECK: llvm.fptoui %arg0 : f128 to i128
 
 // -----
 
diff --git a/flang/test/Fir/copy-codegen.fir b/flang/test/Fir/copy-codegen.fir
new file mode 100644
index 0000000000000..eef1885c6a49c
--- /dev/null
+++ b/flang/test/Fir/copy-codegen.fir
@@ -0,0 +1,35 @@
+// Test fir.copy codegen.
+// RUN: fir-opt --fir-to-llvm-ir %s -o - | FileCheck %s
+
+!t=!fir.type<sometype{i:!fir.array<9xi32>}>
+
+module attributes {llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"} {
+
+func.func @test_copy_1(%arg0: !fir.ref<!t>, %arg1: !fir.ref<!t>) {
+  fir.copy %arg0 to %arg1 no_overlap : !fir.ref<!t>, !fir.ref<!t>
+  return
+}
+// CHECK-LABEL:   llvm.func @test_copy_1(
+// CHECK-SAME:                           %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !llvm.ptr,
+// CHECK-SAME:                           %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !llvm.ptr) {
+// CHECK:           %[[VAL_2:.*]] = llvm.mlir.zero : !llvm.ptr
+// CHECK:           %[[VAL_3:.*]] = llvm.getelementptr %[[VAL_2]][1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"sometype", (array<9 x i32>)>
+// CHECK:           %[[VAL_4:.*]] = llvm.ptrtoint %[[VAL_3]] : !llvm.ptr to i64
+// CHECK:           "llvm.intr.memcpy"(%[[VAL_1]], %[[VAL_0]], %[[VAL_4]]) <{isVolatile = false}> : (!llvm.ptr, !llvm.ptr, i64) -> ()
+// CHECK:           llvm.return
+// CHECK:         }
+
+func.func @test_copy_2(%arg0: !fir.ref<!t>, %arg1: !fir.ref<!t>) {
+  fir.copy %arg0 to %arg1 : !fir.ref<!t>, !fir.ref<!t>
+  return
+}
+// CHECK-LABEL:   llvm.func @test_copy_2(
+// CHECK-SAME:                           %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !llvm.ptr,
+// CHECK-SAME:                           %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !llvm.ptr) {
+// CHECK:           %[[VAL_2:.*]] = llvm.mlir.zero : !llvm.ptr
+// CHECK:           %[[VAL_3:.*]] = llvm.getelementptr %[[VAL_2]][1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"sometype", (array<9 x i32>)>
+// CHECK:           %[[VAL_4:.*]] = llvm.ptrtoint %[[VAL_3]] : !llvm.ptr to i64
+// CHECK:           "llvm.intr.memmove"(%[[VAL_1]], %[[VAL_0]], %[[VAL_4]]) <{isVolatile = false}> : (!llvm.ptr, !llvm.ptr, i64) -> ()
+// CHECK:           llvm.return
+// CHECK:         }
+}
diff --git a/flang/test/Fir/fir-ops.fir b/flang/test/Fir/fir-ops.fir
index 1bfcb3a9f3dc8..c5bcba9103db1 100644
--- a/flang/test/Fir/fir-ops.fir
+++ b/flang/test/Fir/fir-ops.fir
@@ -933,3 +933,37 @@ func.func @test_call_arg_attrs_indirect(%arg0: i16, %arg1: (i16)-> i16) -> i16 {
   %0 = fir.call %arg1(%arg0) : (i16 {llvm.noundef, llvm.signext}) -> (i16 {llvm.signext})
   return %0 : i16
 }
+
+// CHECK-LABEL: @test_copy(
+// CHECK-SAME:    %[[VAL_0:.*]]: !fir.ref<!fir.type<sometype{i:i32}>>,
+// CHECK-SAME:    %[[VAL_1:.*]]: !fir.ptr<!fir.type<sometype{i:i32}>>
+func.func @test_copy(%arg0: !fir.ref<!fir.type<sometype{i:i32}>>, %arg1: !fir.ptr<!fir.type<sometype{i:i32}>>) {
+  // CHECK: fir.copy %[[VAL_0]] to %[[VAL_1]] no_overlap : !fir.ref<!fir.type<sometype{i:i32}>>, !fir.ptr<!fir.type<sometype{i:i32}>>
+  fir.copy %arg0 to %arg1 no_overlap : !fir.ref<!fir.type<sometype{i:i32}>>, !fir.ptr<!fir.type<sometype{i:i32}>>
+  return
+}
+
+// CHECK-LABEL:   func.func @test_pack_unpack_array(
+// CHECK-SAME:                                      %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.ref<!fir.box<none>>,
+// CHECK-SAME:                                      %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !fir.box<!fir.array<?xi32>>) {
+func.func @test_pack_unpack_array(%arg0: !fir.ref<!fir.box<none>>, %arg1: !fir.box<!fir.array<?xi32>>) {
+// CHECK:           %[[VAL_2:.*]] = fir.pack_array %[[VAL_1]] heap whole : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+  %0 = fir.pack_array %arg1 heap whole : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+  %1 = fir.convert %0 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+  fir.store %1 to %arg0 : !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_4:.*]] = fir.pack_array %[[VAL_1]] stack whole : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+  %2 = fir.pack_array %arg1 stack whole : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+  %3 = fir.convert %2 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+  fir.store %3 to %arg0 : !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_6:.*]] = fir.pack_array %[[VAL_1]] heap whole no_copy : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+  %4 = fir.pack_array %arg1 heap whole no_copy constraints {} heuristics <none> : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+  %5 = fir.convert %4 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+  fir.store %5 to %arg0 : !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_8:.*]] = fir.pack_array %[[VAL_1]] stack whole constraints  {max_size = 100 : ui64, max_element_size = 1 : ui64, min_stride = 10 : ui64} heuristics <loop_only> : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+  %6 = fir.pack_array %arg1 stack whole constraints {max_size = 100 : ui64, max_element_size = 1 : ui64, min_stride = 10 : ui64} heuristics <loop_only> : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+  %7 = fir.convert %6 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+  fir.store %7 to %arg0 : !fir.ref<!fir.box<none>>
+// CHECK:           fir.unpack_array %[[VAL_8]] to %[[VAL_1]] stack no_copy : !fir.box<!fir.array<?xi32>>
+  fir.unpack_array %6 to %arg1 stack no_copy : !fir.box<!fir.array<?xi32>>
+  return
+}
diff --git a/flang/test/Fir/invalid.fir b/flang/test/Fir/invalid.fir
index 7e3f9d6498412..1876f8c7d965e 100644
--- a/flang/test/Fir/invalid.fir
+++ b/flang/test/Fir/invalid.fir
@@ -1018,3 +1018,131 @@ func.func @bad_is_assumed_size(%arg0: !fir.ref<!fir.array<*:none>>) {
   %1 = fir.is_assumed_size %arg0 : (!fir.ref<!fir.array<*:none>>) -> i1
   return
 }
+
+// -----
+
+!t=!fir.type<sometype{i:i32}>
+!t2=!fir.type<sometype2{j:i32}>
+func.func @bad_copy_1(%arg0: !fir.ref<!t>, %arg1: !fir.ref<!t2>) {
+  // expected-error@+1{{'fir.copy' op source and destination must have the same value type}}
+  fir.copy %arg0 to %arg1 no_overlap : !fir.ref<!t>, !fir.ref<!t2>
+  return
+}
+
+// -----
+
+!t=!fir.type<sometype{i:i32}>
+func.func @bad_copy_2(%arg0: !fir.ref<!t>, %arg1: !t) {
+  // expected-error@+1{{'fir.copy' op operand #0 must be a reference type to a constant size fir.array, fir.char, or fir.type, but got '!fir.type<sometype{i:i32}>'}}
+  fir.copy %arg1 to %arg0 no_overlap : !t, !fir.ref<!t>
+  return
+}
+
+// -----
+
+!t=!fir.array<?xi32>
+func.func @bad_copy_3(%arg0: !fir.ref<!t>, %arg1: !fir.ref<!t>) {
+  // expected-error@+1{{'fir.copy' op operand #0 must be a reference type to a constant size fir.array, fir.char, or fir.type, but got '!fir.ref<!fir.array<?xi32>>'}}
+  fir.copy %arg0 to %arg1 no_overlap : !fir.ref<!t>, !fir.ref<!t>
+  return
+}
+
+// -----
+
+!t=f32
+func.func @bad_copy_4(%arg0: !fir.ref<!t>, %arg1: !fir.ref<!t>) {
+  // expected-error@+1{{'fir.copy' op operand #0 must be a reference type to a constant size fir.array, fir.char, or fir.type, but got '!fir.ref<f32>'}}
+  fir.copy %arg0 to %arg1 no_overlap : !fir.ref<!t>, !fir.ref<!t>
+  return
+}
+
+// -----
+
+func.func @bad_pack_array1(%arg0: !fir.ref<!fir.box<!fir.array<?xi32>>>) {
+  // expected-error@+1{{op operand #0 must be any boxed array, but got '!fir.ref<!fir.box<!fir.array<?xi32>>>'}}
+  %0 = fir.pack_array %arg0 stack whole : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> !fir.ref<!fir.box<!fir.array<?xi32>>>
+  return
+}
+
+// -----
+
+func.func @bad_pack_array2(%arg0: !fir.box<!fir.array<?xi32>>) {
+  // expected-error@+1{{op failed to verify that all of {array, result} have same type}}
+  %0 = fir.pack_array %arg0 stack whole : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi64>>
+  return
+}
+
+// -----
+
+func.func @bad_pack_array3(%arg0: !fir.box<!fir.array<?x!fir.char<1,?>>>, %arg1: i32) {
+  // expected-error@+1{{op invalid type parameters}}
+  %0 = fir.pack_array %arg0 stack whole typeparams %arg1, %arg1 : (!fir.box<!fir.array<?x!fir.char<1,?>>>, i32, i32) -> !fir.box<!fir.array<?x!fir.char<1,?>>>
+  return
+}
+
+// -----
+
+func.func @bad_pack_array4(%arg0: !fir.box<!fir.array<?xf32>>) {
+  // expected-error@+1{{op attribute 'max_size' failed to satisfy constraint: 64-bit unsigned integer attribute}}
+  %0 = fir.pack_array %arg0 stack whole constraints {max_size = -1 : i64} : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// -----
+
+func.func @bad_pack_array5(%arg0: !fir.box<!fir.array<?xf32>>) {
+  // expected-error@+1{{op attribute 'max_element_size' failed to satisfy constraint: 64-bit unsigned integer attribute}}
+  %0 = fir.pack_array %arg0 stack whole constraints {max_element_size = -1 : i64} : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// -----
+
+func.func @bad_pack_array6(%arg0: !fir.box<!fir.array<?xf32>>) {
+  // expected-error@+1{{op attribute 'min_stride' failed to satisfy constraint: 64-bit unsigned integer attribute}}
+  %0 = fir.pack_array %arg0 stack whole constraints {min_stride = -1 : i64} : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// -----
+
+func.func @bad_pack_array7(%arg0: !fir.box<!fir.array<?xf32>>) {
+  // expected-error@+1{{op 'innermost' is invalid for 1D arrays, use 'whole' instead}}
+  %0 = fir.pack_array %arg0 stack innermost : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// -----
+
+func.func @bad_unpack_array1(%arg0: !fir.ref<!fir.box<!fir.array<?xi32>>>, %arg1: !fir.ref<!fir.box<!fir.array<?xi32>>>) {
+  // expected-error@+1{{op operand #0 must be any boxed array, but got '!fir.ref<!fir.box<!fir.array<?xi32>>>'}}
+  fir.unpack_array %arg0 to %arg1 stack : !fir.ref<!fir.box<!fir.array<?xi32>>>
+  return
+}
+
+// -----
+
+func.func @bad_unpack_array2(%arg0: !fir.box<!fir.array<?xf32>>) {
+  %0 = fir.pack_array %arg0 stack whole : (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  // expected-error@+1{{op the pack operation uses different memory for the temporary (stack vs heap)}}
+  fir.unpack_array %0 to %arg0 heap no_copy : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// -----
+
+func.func @bad_unpack_array3(%arg0: !fir.box<!fir.array<?xf32>>) {
+  %0 = fir.pack_array %arg0 heap whole no_copy: (!fir.box<!fir.array<?xf32>>) -> !fir.box<!fir.array<?xf32>>
+  // expected-error@+1{{op the pack operation uses different memory for the temporary (stack vs heap)}}
+  fir.unpack_array %0 to %arg0 stack : !fir.box<!fir.array<?xf32>>
+  return
+}
+
+// -----
+
+func.func @bad_unpack_array4(%arg0: !fir.box<!fir.array<?xf32>>, %arg1: !fir.box<!fir.array<?xi32>>) {
+  // expected-note@-1 {{prior use here}}
+  // expected-error@+1{{use of value '%arg0' expects different type than prior uses: '!fir.box<!fir.array<?xi32>>' vs '!fir.box<!fir.array<?xf32>>'}}
+  fir.unpack_array %arg0 to %arg1 stack : !fir.box<!fir.array<?xi32>>
+  return
+}
diff --git a/flang/test/Fir/struct-passing-return-loongarch64-bystack.fir b/flang/test/Fir/struct-passing-return-loongarch64-bystack.fir
index 5041a39e69798..91ac345d2d7af 100644
--- a/flang/test/Fir/struct-passing-return-loongarch64-bystack.fir
+++ b/flang/test/Fir/struct-passing-return-loongarch64-bystack.fir
@@ -1,7 +1,6 @@
 /// Test LoongArch64 ABI rewrite of struct passed and returned by value (BIND(C), VALUE derived types).
 /// This test test cases where the struct must be passed or returned on the stack.
 
-// REQUIRES: loongarch-registered-target
 // RUN: tco --target=loongarch64-unknown-linux-gnu %s | FileCheck %s
 
 !ty_int_toobig = !fir.type<int_toobig{i:!fir.array<5xi32>}>
diff --git a/flang/test/Fir/struct-passing-x86-64-byval.fir b/flang/test/Fir/struct-passing-x86-64-byval.fir
index 13cf52de581f7..8451c26095226 100644
--- a/flang/test/Fir/struct-passing-x86-64-byval.fir
+++ b/flang/test/Fir/struct-passing-x86-64-byval.fir
@@ -1,7 +1,6 @@
 // Test X86-64 ABI rewrite of struct passed by value (BIND(C), VALUE derived types).
 // This test test cases where the struct must be passed on the stack according
 // to the System V ABI.
-// REQUIRES: x86-registered-target
 // RUN: tco --target=x86_64-unknown-linux-gnu %s | FileCheck %s
 
 module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
diff --git a/flang/test/Fir/struct-passing-x86-64-one-field-inreg.fir b/flang/test/Fir/struct-passing-x86-64-one-field-inreg.fir
index e37e8dd4481d0..7feb698e0d5c5 100644
--- a/flang/test/Fir/struct-passing-x86-64-one-field-inreg.fir
+++ b/flang/test/Fir/struct-passing-x86-64-one-field-inreg.fir
@@ -1,6 +1,5 @@
 // Test X86-64 passing ABI of struct in registers for the simple case
 // where the struct has a single intrinsic component that is not a complex.
-// REQUIRES: x86-registered-target
 // RUN: fir-opt -target-rewrite="target=x86_64-unknown-linux-gnu" %s -o - | FileCheck %s
 
 module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
diff --git a/flang/test/Fir/struct-passing-x86-64-several-fields-inreg.fir b/flang/test/Fir/struct-passing-x86-64-several-fields-inreg.fir
index 9a0a41e1da542..afc8c2de85019 100644
--- a/flang/test/Fir/struct-passing-x86-64-several-fields-inreg.fir
+++ b/flang/test/Fir/struct-passing-x86-64-several-fields-inreg.fir
@@ -1,6 +1,5 @@
 // Test X86-64 passing ABI of struct in registers for the cases where the
 // struct has more than one field.
-// REQUIRES: x86-registered-target
 // RUN: fir-opt -target-rewrite="target=x86_64-unknown-linux-gnu" %s -o - | FileCheck %s
 
 
diff --git a/flang/test/Fir/struct-return-aarch64.fir b/flang/test/Fir/struct-return-aarch64.fir
index 8b75c2cac7b6b..57be5a9828284 100644
--- a/flang/test/Fir/struct-return-aarch64.fir
+++ b/flang/test/Fir/struct-return-aarch64.fir
@@ -125,7 +125,7 @@ func.func private @test_too_big() -> !too_big
 func.func @test_call_too_big(%arg0 : !fir.ref<!too_big>) {
   // CHECK: %[[STACK:.*]] = llvm.intr.stacksave : !llvm.ptr
   // CHECK: %[[ARG:.*]] = fir.alloca !fir.type<t7{x:i64,y:i64,z:i64}>
-  // CHECK: fir.call @test_too_big(%[[ARG]]) : (!fir.ref<!fir.type<t7{x:i64,y:i64,z:i64}>>) -> ()
+  // CHECK: fir.call @test_too_big(%[[ARG]]) : (!fir.ref<!fir.type<t7{x:i64,y:i64,z:i64}>> {llvm.align = 8 : i32, llvm.sret = !fir.type<t7{x:i64,y:i64,z:i64}>}) -> ()
   // CHECK: %[[CVT:.*]] = fir.convert %[[ARG]] : (!fir.ref<!fir.type<t7{x:i64,y:i64,z:i64}>>) -> !fir.ref<!fir.type<t7{x:i64,y:i64,z:i64}>>
   // CHECK: %[[LD:.*]] = fir.load %[[CVT]] : !fir.ref<!fir.type<t7{x:i64,y:i64,z:i64}>>
   // CHECK: llvm.intr.stackrestore %[[STACK]] : !llvm.ptr
@@ -145,7 +145,7 @@ func.func private @test_too_big_hfa() -> !too_big_hfa
 func.func @test_call_too_big_hfa(%arg0 : !fir.ref<!too_big_hfa>) {
   // CHECK: %[[STACK:.*]] = llvm.intr.stacksave : !llvm.ptr
   // CHECK: %[[ARG:.*]] = fir.alloca !fir.type<t8{i:!fir.array<5xf32>}>
-  // CHECK: fir.call @test_too_big_hfa(%[[ARG]]) : (!fir.ref<!fir.type<t8{i:!fir.array<5xf32>}>>) -> ()
+  // CHECK: fir.call @test_too_big_hfa(%[[ARG]]) : (!fir.ref<!fir.type<t8{i:!fir.array<5xf32>}>> {llvm.align = 8 : i32, llvm.sret = !fir.type<t8{i:!fir.array<5xf32>}>}) -> ()
   // CHECK: %[[CVT:.*]] = fir.convert %[[ARG]] : (!fir.ref<!fir.type<t8{i:!fir.array<5xf32>}>>) -> !fir.ref<!fir.type<t8{i:!fir.array<5xf32>}>>
   // CHECK: %[[LD:.*]] = fir.load %[[CVT]] : !fir.ref<!fir.type<t8{i:!fir.array<5xf32>}>>
   // CHECK: llvm.intr.stackrestore %[[STACK]] : !llvm.ptr
diff --git a/flang/test/Fir/struct-return-loongarch64-byreg.fir b/flang/test/Fir/struct-return-loongarch64-byreg.fir
index eb3d4d50d8866..baf9c74d6231a 100644
--- a/flang/test/Fir/struct-return-loongarch64-byreg.fir
+++ b/flang/test/Fir/struct-return-loongarch64-byreg.fir
@@ -7,7 +7,6 @@
 /// only the first example in each category checks the entire invocation process,
 /// while the other examples only check the signatures.
 
-// REQUIRES: loongarch-registered-target
 // RUN: fir-opt --split-input-file --target-rewrite="target=loongarch64-unknown-linux-gnu" %s | FileCheck %s
 
 
diff --git a/flang/test/Fir/struct-return-powerpc64-aix.fir b/flang/test/Fir/struct-return-powerpc64-aix.fir
index 3058ef6b7e965..5037ffca0c79d 100644
--- a/flang/test/Fir/struct-return-powerpc64-aix.fir
+++ b/flang/test/Fir/struct-return-powerpc64-aix.fir
@@ -15,7 +15,7 @@ func.func @test_call_t1(%arg0 : !fir.ref<!fir.type<t1<{c:!fir.char<1>}>>>) {
   return
   //CHECK: %[[STCK:.*]] = llvm.intr.stacksave : !llvm.ptr
   //CHECK: %[[ARG:.*]] = fir.alloca !fir.type<t1<{c:!fir.char<1>}>>
-  //CHECK: fir.call @test_t1(%[[ARG]]) : (!fir.ref<!fir.type<t1<{c:!fir.char<1>}>>>) -> ()
+  //CHECK: fir.call @test_t1(%[[ARG]]) : (!fir.ref<!fir.type<t1<{c:!fir.char<1>}>>> {llvm.align = 8 : i32, llvm.sret = !fir.type<t1<{c:!fir.char<1>}>>}) -> ()
   //CHECK: %[[CVT:.*]] = fir.convert %[[ARG]] : (!fir.ref<!fir.type<t1<{c:!fir.char<1>}>>>) -> !fir.ref<!fir.type<t1<{c:!fir.char<1>}>>>
   //CHECK: %[[LD:.*]] = fir.load %[[CVT]] : !fir.ref<!fir.type<t1<{c:!fir.char<1>}>>>
   //CHECK: llvm.intr.stackrestore %[[STCK]] : !llvm.ptr
@@ -34,7 +34,7 @@ func.func @test_call_t2(%arg0 : !fir.ref<!fir.type<t2<{i:i32}>>>) {
   return
   //CHECK: %[[STCK:.*]] = llvm.intr.stacksave : !llvm.ptr
   //CHECK: %[[ARG:.*]] = fir.alloca !fir.type<t2<{i:i32}>>
-  //CHECK: fir.call @test_t2(%[[ARG]]) : (!fir.ref<!fir.type<t2<{i:i32}>>>) -> ()
+  //CHECK: fir.call @test_t2(%[[ARG]]) : (!fir.ref<!fir.type<t2<{i:i32}>>> {llvm.align = 8 : i32, llvm.sret = !fir.type<t2<{i:i32}>>}) -> ()
   //CHECK: %[[CVT:.*]] = fir.convert %[[ARG]] : (!fir.ref<!fir.type<t2<{i:i32}>>>) -> !fir.ref<!fir.type<t2<{i:i32}>>>
   //CHECK: %[[LD:.*]] = fir.load %[[CVT]] : !fir.ref<!fir.type<t2<{i:i32}>>>
   //CHECK: llvm.intr.stackrestore %[[STCK]] : !llvm.ptr
@@ -53,7 +53,7 @@ func.func @test_call_t3(%arg0 : !fir.ref<!fir.type<t3<{r1:f32,r2:f32,r3:f32}>>>)
   return
   //CHECK: %[[STCK:.*]] = llvm.intr.stacksave : !llvm.ptr
   //CHECK: %[[ARG:.*]] = fir.alloca !fir.type<t3<{r1:f32,r2:f32,r3:f32}>>
-  //CHECK: fir.call @test_t3(%[[ARG]]) : (!fir.ref<!fir.type<t3<{r1:f32,r2:f32,r3:f32}>>>) -> ()
+  //CHECK: fir.call @test_t3(%[[ARG]]) : (!fir.ref<!fir.type<t3<{r1:f32,r2:f32,r3:f32}>>> {llvm.align = 8 : i32, llvm.sret = !fir.type<t3<{r1:f32,r2:f32,r3:f32}>>}) -> ()
   //CHECK: %[[CVT:.*]] = fir.convert %[[ARG]] : (!fir.ref<!fir.type<t3<{r1:f32,r2:f32,r3:f32}>>>) -> !fir.ref<!fir.type<t3<{r1:f32,r2:f32,r3:f32}>>>
   //CHECK: %[[LD:.*]] = fir.load %[[CVT]] : !fir.ref<!fir.type<t3<{r1:f32,r2:f32,r3:f32}>>>
   //CHECK: llvm.intr.stackrestore %[[STCK]] : !llvm.ptr
@@ -72,7 +72,7 @@ func.func @test_call_t4(%arg0 : !fir.ref<!fir.type<t4<{r:!fir.array<8xf32>}>>>)
   return
   //CHECK: %[[STCK:.*]] = llvm.intr.stacksave : !llvm.ptr
   //CHECK: %[[ARG:.*]] = fir.alloca !fir.type<t4<{r:!fir.array<8xf32>}>>
-  //CHECK: fir.call @test_t4(%[[ARG]]) : (!fir.ref<!fir.type<t4<{r:!fir.array<8xf32>}>>>) -> ()
+  //CHECK: fir.call @test_t4(%[[ARG]]) : (!fir.ref<!fir.type<t4<{r:!fir.array<8xf32>}>>> {llvm.align = 8 : i32, llvm.sret = !fir.type<t4<{r:!fir.array<8xf32>}>>}) -> ()
   //CHECK: %[[CVT:.*]] = fir.convert %[[ARG]] : (!fir.ref<!fir.type<t4<{r:!fir.array<8xf32>}>>>) -> !fir.ref<!fir.type<t4<{r:!fir.array<8xf32>}>>>
   //CHECK: %[[LD:.*]] = fir.load %[[CVT]] : !fir.ref<!fir.type<t4<{r:!fir.array<8xf32>}>>>
   //CHECK: llvm.intr.stackrestore %[[STCK]] : !llvm.ptr
@@ -91,7 +91,7 @@ func.func @test_call_t5(%arg0 : !fir.ref<!fir.type<t5<{c:!fir.char<1>,r:f32,i:i6
   return
   //CHECK: %[[STCK:.*]] = llvm.intr.stacksave : !llvm.ptr
   //CHECK: %[[ARG:.*]] = fir.alloca !fir.type<t5<{c:!fir.char<1>,r:f32,i:i64}>>
-  //CHECK: fir.call @test_t5(%[[ARG]]) : (!fir.ref<!fir.type<t5<{c:!fir.char<1>,r:f32,i:i64}>>>) -> ()
+  //CHECK: fir.call @test_t5(%[[ARG]]) : (!fir.ref<!fir.type<t5<{c:!fir.char<1>,r:f32,i:i64}>>> {llvm.align = 8 : i32, llvm.sret = !fir.type<t5<{c:!fir.char<1>,r:f32,i:i64}>>}) -> ()
   //CHECK: %[[CVT:.*]] = fir.convert %[[ARG]] : (!fir.ref<!fir.type<t5<{c:!fir.char<1>,r:f32,i:i64}>>>) -> !fir.ref<!fir.type<t5<{c:!fir.char<1>,r:f32,i:i64}>>>
   //CHECK: %[[LD:.*]] = fir.load %[[CVT]] : !fir.ref<!fir.type<t5<{c:!fir.char<1>,r:f32,i:i64}>>>
   //CHECK: llvm.intr.stackrestore %[[STCK]] : !llvm.ptr
diff --git a/flang/test/Fir/struct-return-ppc64le.fir b/flang/test/Fir/struct-return-ppc64le.fir
index a5906280a0ef8..a19f8885ee0c5 100644
--- a/flang/test/Fir/struct-return-ppc64le.fir
+++ b/flang/test/Fir/struct-return-ppc64le.fir
@@ -15,7 +15,7 @@ func.func @test_call_t1(%arg0 : !fir.ref<!fir.type<t1{c:!fir.char<1>}>>) {
   return
   //CHECK: %[[STCK:.*]] = llvm.intr.stacksave : !llvm.ptr
   //CHECK: %[[ARG:.*]] = fir.alloca !fir.type<t1{c:!fir.char<1>}>
-  //CHECK: fir.call @test_t1(%[[ARG]]) : (!fir.ref<!fir.type<t1{c:!fir.char<1>}>>) -> ()
+  //CHECK: fir.call @test_t1(%[[ARG]]) : (!fir.ref<!fir.type<t1{c:!fir.char<1>}>> {llvm.align = 8 : i32, llvm.sret = !fir.type<t1{c:!fir.char<1>}>}) -> ()
   //CHECK: %[[CVT:.*]] = fir.convert %[[ARG]] : (!fir.ref<!fir.type<t1{c:!fir.char<1>}>>) -> !fir.ref<!fir.type<t1{c:!fir.char<1>}>>
   //CHECK: %[[LD:.*]] = fir.load %[[CVT]] : !fir.ref<!fir.type<t1{c:!fir.char<1>}>>
   //CHECK: llvm.intr.stackrestore %[[STCK]] : !llvm.ptr
@@ -34,7 +34,7 @@ func.func @test_call_t2(%arg0 : !fir.ref<!fir.type<t2{i:i32}>>) {
   return
   //CHECK: %[[STCK:.*]] = llvm.intr.stacksave : !llvm.ptr
   //CHECK: %[[ARG:.*]] = fir.alloca !fir.type<t2{i:i32}>
-  //CHECK: fir.call @test_t2(%[[ARG]]) : (!fir.ref<!fir.type<t2{i:i32}>>) -> ()
+  //CHECK: fir.call @test_t2(%[[ARG]]) : (!fir.ref<!fir.type<t2{i:i32}>> {llvm.align = 8 : i32, llvm.sret = !fir.type<t2{i:i32}>}) -> ()
   //CHECK: %[[CVT:.*]] = fir.convert %[[ARG]] : (!fir.ref<!fir.type<t2{i:i32}>>) -> !fir.ref<!fir.type<t2{i:i32}>>
   //CHECK: %[[LD:.*]] = fir.load %[[CVT]] : !fir.ref<!fir.type<t2{i:i32}>>
   //CHECK: llvm.intr.stackrestore %[[STCK]] : !llvm.ptr
@@ -73,7 +73,7 @@ func.func @test_call_t4(%arg0 : !fir.ref<!fir.type<t4{r:!fir.array<9xf32>}>>) {
   return
   //CHECK: %[[STCK:.*]] = llvm.intr.stacksave : !llvm.ptr
   //CHECK: %[[ARG:.*]] = fir.alloca !fir.type<t4{r:!fir.array<9xf32>}>
-  //CHECK: fir.call @test_t4(%[[ARG]]) : (!fir.ref<!fir.type<t4{r:!fir.array<9xf32>}>>) -> ()
+  //CHECK: fir.call @test_t4(%[[ARG]]) : (!fir.ref<!fir.type<t4{r:!fir.array<9xf32>}>> {llvm.align = 8 : i32, llvm.sret = !fir.type<t4{r:!fir.array<9xf32>}>}) -> ()
   //CHECK: %[[CVT:.*]] = fir.convert %[[ARG]] : (!fir.ref<!fir.type<t4{r:!fir.array<9xf32>}>>) -> !fir.ref<!fir.type<t4{r:!fir.array<9xf32>}>>
   //CHECK: %[[LD:.*]] = fir.load %[[CVT]] : !fir.ref<!fir.type<t4{r:!fir.array<9xf32>}>>
   //CHECK: llvm.intr.stackrestore %[[STCK]] : !llvm.ptr
@@ -92,7 +92,7 @@ func.func @test_call_t5(%arg0 : !fir.ref<!fir.type<t5{c:!fir.char<1>,r:f32,i:i64
   return
   //CHECK: %[[STCK:.*]] = llvm.intr.stacksave : !llvm.ptr
   //CHECK: %[[ARG:.*]] = fir.alloca !fir.type<t5{c:!fir.char<1>,r:f32,i:i64}>
-  //CHECK: fir.call @test_t5(%[[ARG]]) : (!fir.ref<!fir.type<t5{c:!fir.char<1>,r:f32,i:i64}>>) -> ()
+  //CHECK: fir.call @test_t5(%[[ARG]]) : (!fir.ref<!fir.type<t5{c:!fir.char<1>,r:f32,i:i64}>> {llvm.align = 8 : i32, llvm.sret = !fir.type<t5{c:!fir.char<1>,r:f32,i:i64}>}) -> ()
   //CHECK: %[[CVT:.*]] = fir.convert %[[ARG]] : (!fir.ref<!fir.type<t5{c:!fir.char<1>,r:f32,i:i64}>>) -> !fir.ref<!fir.type<t5{c:!fir.char<1>,r:f32,i:i64}>>
   //CHECK: %[[LD:.*]] = fir.load %[[CVT]] : !fir.ref<!fir.type<t5{c:!fir.char<1>,r:f32,i:i64}>>
   //CHECK: llvm.intr.stackrestore %[[STCK]] : !llvm.ptr
diff --git a/flang/test/Fir/struct-return-x86-64.fir b/flang/test/Fir/struct-return-x86-64.fir
index f4c2add69ff7e..5d1e6129d8f69 100644
--- a/flang/test/Fir/struct-return-x86-64.fir
+++ b/flang/test/Fir/struct-return-x86-64.fir
@@ -1,5 +1,4 @@
 // Test X86-64 ABI rewrite of struct returned by value (BIND(C), VALUE derived types).
-// REQUIRES: x86-registered-target
 // RUN: fir-opt --target-rewrite %s | FileCheck %s
 
 !fits_in_reg = !fir.type<t1{i:f32,j:i32,k:f32}>
@@ -86,7 +85,7 @@ module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data
 // CHECK-SAME:                              %[[VAL_0:.*]]: !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>) {
 // CHECK:           %[[VAL_1:.*]] = llvm.intr.stacksave : !llvm.ptr
 // CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.type<t2{i:!fir.array<5xf32>}>
-// CHECK:           fir.call @test_sret(%[[VAL_2]]) : (!fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>) -> ()
+// CHECK:           fir.call @test_sret(%[[VAL_2]]) : (!fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>> {llvm.align = 8 : i32, llvm.sret = !fir.type<t2{i:!fir.array<5xf32>}>}) -> ()
 // CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>) -> !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>
 // CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]] : !fir.ref<!fir.type<t2{i:!fir.array<5xf32>}>>
 // CHECK:           llvm.intr.stackrestore %[[VAL_1]] : !llvm.ptr
diff --git a/flang/test/Fir/target-rewrite-complex-10-x86.fir b/flang/test/Fir/target-rewrite-complex-10-x86.fir
index a6dd178cfe70a..6404b4f766d39 100644
--- a/flang/test/Fir/target-rewrite-complex-10-x86.fir
+++ b/flang/test/Fir/target-rewrite-complex-10-x86.fir
@@ -1,8 +1,9 @@
 // Test COMPLEX(10) passing and returning on X86
-// REQUIRES: x86-registered-target
 // RUN: fir-opt --target-rewrite="target=x86_64-unknown-linux-gnu" %s | FileCheck %s --check-prefix=AMD64
 // RUN: tco -target="x86_64-unknown-linux-gnu" %s | FileCheck %s --check-prefix=AMD64_LLVM
 
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
+
 func.func @returncomplex10() -> complex<f80> {
   %1 = fir.zero_bits complex<f80>
   return %1 : complex<f80>
@@ -30,3 +31,4 @@ func.func @takecomplex10(%z: complex<f80>) {
 // AMD64:    fir.store %[[VAL_2]] to %[[VAL_3]] : !fir.ref<complex<f80>>
 
 // AMD64_LLVM: define void @takecomplex10(ptr byval({ x86_fp80, x86_fp80 }) align 16 captures(none) %0)
+}
diff --git a/flang/test/Fir/target-rewrite-complex.fir b/flang/test/Fir/target-rewrite-complex.fir
index e697abb6e3ab3..405b14031f0c5 100644
--- a/flang/test/Fir/target-rewrite-complex.fir
+++ b/flang/test/Fir/target-rewrite-complex.fir
@@ -235,7 +235,7 @@ func.func @callcomplex4(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i3
   // I32: [[ADDRC2:%[0-9A-Za-z]+]] = fir.alloca complex<f32>
   // I32: fir.store [[C]] to [[ADDRC2]] : !fir.ref<complex<f32>>
   // I32: [[T:%[0-9A-Za-z]+]] = fir.convert [[ADDRC2]] : (!fir.ref<complex<f32>>) -> !fir.ref<tuple<f32, f32>>
-  // I32: fir.call @paramcomplex4([[T]]) : (!fir.ref<tuple<f32, f32>>) -> ()
+  // I32: fir.call @paramcomplex4([[T]]) : (!fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}) -> ()
 
   // I32_MINGW: [[ADDRI64:%[0-9A-Za-z]+]] = fir.alloca i64
   // I32_MINGW: fir.store [[RES]] to [[ADDRI64]] : !fir.ref<i64>
@@ -244,7 +244,7 @@ func.func @callcomplex4(%arg0 : !fir.class<!fir.type<_QMpolymorphic_testTp1{a:i3
   // I32_MINGW: [[ADDRC2:%[0-9A-Za-z]+]] = fir.alloca complex<f32>
   // I32_MINGW: fir.store [[C]] to [[ADDRC2]] : !fir.ref<complex<f32>>
   // I32_MINGW: [[T:%[0-9A-Za-z]+]] = fir.convert [[ADDRC2]] : (!fir.ref<complex<f32>>) -> !fir.ref<tuple<f32, f32>>
-  // I32_MINGW: fir.call @paramcomplex4([[T]]) : (!fir.ref<tuple<f32, f32>>) -> ()
+  // I32_MINGW: fir.call @paramcomplex4([[T]]) : (!fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}) -> ()
 
   // X64: [[ADDRV:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:f32>
   // X64: fir.store [[RES]] to [[ADDRV]] : !fir.ref<!fir.vector<2:f32>>
@@ -442,12 +442,12 @@ func.func private @paramcomplex8(complex<f64>) -> ()
 // LOONGARCH64-LABEL: func @callcomplex8()
 func.func @callcomplex8() {
   // I32: [[RES:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
-  // I32: fir.call @returncomplex8([[RES]]) : (!fir.ref<tuple<f64, f64>>) -> ()
+  // I32: fir.call @returncomplex8([[RES]]) : (!fir.ref<tuple<f64, f64>> {llvm.align = 4 : i32, llvm.sret = tuple<f64, f64>}) -> ()
   // I32_MINGW: [[RES:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
-  // I32_MINGW: fir.call @returncomplex8([[RES]]) : (!fir.ref<tuple<f64, f64>>) -> ()
+  // I32_MINGW: fir.call @returncomplex8([[RES]]) : (!fir.ref<tuple<f64, f64>> {llvm.align = 8 : i32, llvm.sret = tuple<f64, f64>}) -> ()
   // X64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<f64, f64>
   // X64_MINGW: [[RES:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
-  // X64_MINGW: fir.call @returncomplex8([[RES]]) : (!fir.ref<tuple<f64, f64>>) -> ()
+  // X64_MINGW: fir.call @returncomplex8([[RES]]) : (!fir.ref<tuple<f64, f64>> {llvm.align = 8 : i32, llvm.sret = tuple<f64, f64>}) -> ()
   // AARCH64: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<f64, f64>
   // PPC64le: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<f64, f64>
   // SPARCV9: [[RES:%[0-9A-Za-z]+]] = fir.call @returncomplex8() : () -> tuple<f64, f64>
@@ -461,14 +461,14 @@ func.func @callcomplex8() {
   // I32: [[ADDRC:%[0-9A-Za-z]+]] = fir.alloca complex<f64>
   // I32: fir.store [[V]] to [[ADDRC]] : !fir.ref<complex<f64>>
   // I32: [[ADDRT:%[0-9A-Za-z]+]] = fir.convert [[ADDRC]] : (!fir.ref<complex<f64>>) -> !fir.ref<tuple<f64, f64>>
-  // I32: fir.call @paramcomplex8([[ADDRT]]) : (!fir.ref<tuple<f64, f64>>) -> ()
+  // I32: fir.call @paramcomplex8([[ADDRT]]) : (!fir.ref<tuple<f64, f64>> {llvm.align = 4 : i32, llvm.byval = tuple<f64, f64>}) -> ()
 
   // I32_MINGW: [[RESC:%[0-9A-Za-z]+]] = fir.convert [[RES]] : (!fir.ref<tuple<f64, f64>>) -> !fir.ref<complex<f64>>
   // I32_MINGW: [[V:%[0-9A-Za-z]+]]  = fir.load [[RESC]] : !fir.ref<complex<f64>>
   // I32_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.alloca complex<f64>
   // I32_MINGW: fir.store [[V]] to [[ADDRC]] : !fir.ref<complex<f64>>
   // I32_MINGW: [[ADDRT:%[0-9A-Za-z]+]] = fir.convert [[ADDRC]] : (!fir.ref<complex<f64>>) -> !fir.ref<tuple<f64, f64>>
-  // I32_MINGW: fir.call @paramcomplex8([[ADDRT]]) : (!fir.ref<tuple<f64, f64>>) -> ()
+  // I32_MINGW: fir.call @paramcomplex8([[ADDRT]]) : (!fir.ref<tuple<f64, f64>> {llvm.align = 4 : i32, llvm.byval = tuple<f64, f64>}) -> ()
 
   // X64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
   // X64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f64, f64>>
@@ -483,7 +483,7 @@ func.func @callcomplex8() {
   // X64_MINGW: [[ADDRC:%[0-9A-Za-z]+]] = fir.alloca complex<f64>
   // X64_MINGW: fir.store [[V]] to [[ADDRC]] : !fir.ref<complex<f64>>
   // X64_MINGW: [[ADDRT:%[0-9A-Za-z]+]] = fir.convert [[ADDRC]] : (!fir.ref<complex<f64>>) -> !fir.ref<tuple<f64, f64>>
-  // X64_MINGW: fir.call @paramcomplex8([[ADDRT]]) : (!fir.ref<tuple<f64, f64>>) -> ()
+  // X64_MINGW: fir.call @paramcomplex8([[ADDRT]]) : (!fir.ref<tuple<f64, f64>> {llvm.align = 8 : i32, llvm.byval = tuple<f64, f64>}) -> ()
 
   // AARCH64: [[ADDRT:%[0-9A-Za-z]+]] = fir.alloca tuple<f64, f64>
   // AARCH64: fir.store [[RES]] to [[ADDRT]] : !fir.ref<tuple<f64, f64>>
@@ -590,7 +590,7 @@ func.func @multipleparamscomplex4(%z1 : complex<f32>, %z2 : complex<f32>, %z3 :
   // I32-DAG: fir.store [[Z3_VAL]] to [[Z3_ADDRC]] : !fir.ref<complex<f32>>
   // I32-DAG: [[Z3_ADDRT:%[0-9A-Za-z]+]] = fir.convert [[Z3_ADDRC]] : (!fir.ref<complex<f32>>) -> !fir.ref<tuple<f32, f32>>
 
-  // I32: fir.call @calleemultipleparamscomplex4([[Z1_ADDRT]], [[Z2_ADDRT]], [[Z3_ADDRT]]) : (!fir.ref<tuple<f32, f32>>, !fir.ref<tuple<f32, f32>>, !fir.ref<tuple<f32, f32>>) -> ()
+  // I32: fir.call @calleemultipleparamscomplex4([[Z1_ADDRT]], [[Z2_ADDRT]], [[Z3_ADDRT]]) : (!fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}, !fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}, !fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}) -> ()
 
   // I32_MINGW-DAG: [[Z1_ADDR:%[0-9A-Za-z]+]] = fir.convert [[Z1]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
   // I32_MINGW-DAG: [[Z1_VAL:%[0-9A-Za-z]+]] = fir.load [[Z1_ADDR]] : !fir.ref<complex<f32>>
@@ -609,7 +609,7 @@ func.func @multipleparamscomplex4(%z1 : complex<f32>, %z2 : complex<f32>, %z3 :
   // I32_MINGW-DAG: fir.store [[Z3_VAL]] to [[Z3_ADDRC]] : !fir.ref<complex<f32>>
   // I32_MINGW-DAG: [[Z3_ADDRT:%[0-9A-Za-z]+]] = fir.convert [[Z3_ADDRC]] : (!fir.ref<complex<f32>>) -> !fir.ref<tuple<f32, f32>>
 
-  // I32_MINGW: fir.call @calleemultipleparamscomplex4([[Z1_ADDRT]], [[Z2_ADDRT]], [[Z3_ADDRT]]) : (!fir.ref<tuple<f32, f32>>, !fir.ref<tuple<f32, f32>>, !fir.ref<tuple<f32, f32>>) -> ()
+  // I32_MINGW: fir.call @calleemultipleparamscomplex4([[Z1_ADDRT]], [[Z2_ADDRT]], [[Z3_ADDRT]]) : (!fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}, !fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}, !fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}) -> ()
 
   // X64-DAG: [[Z3_ADDR:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:f32>
   // X64-DAG: fir.store [[Z3]] to [[Z3_ADDR]] : !fir.ref<!fir.vector<2:f32>>
@@ -838,7 +838,7 @@ func.func private @mlircomplexf32(%z1: complex<f32>, %z2: complex<f32>) -> compl
   // I32-DAG: fir.store [[Z2_VAL]] to [[Z2_ADDRC]] : !fir.ref<complex<f32>>
   // I32-DAG: [[Z2_ADDRT:%[0-9A-Za-z]+]] = fir.convert [[Z2_ADDRC]] : (!fir.ref<complex<f32>>) -> !fir.ref<tuple<f32, f32>>
 
-  // I32: [[VAL:%[0-9A-Za-z]+]] = fir.call @mlircomplexf32([[Z1_ADDRT]], [[Z2_ADDRT]]) : (!fir.ref<tuple<f32, f32>>, !fir.ref<tuple<f32, f32>>) -> i64
+  // I32: [[VAL:%[0-9A-Za-z]+]] = fir.call @mlircomplexf32([[Z1_ADDRT]], [[Z2_ADDRT]]) : (!fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}, !fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}) -> i64
 
   // I32_MINGW-DAG: [[Z1_ADDR:%[0-9A-Za-z]+]] = fir.convert [[Z1]] : (!fir.ref<tuple<f32, f32>>) -> !fir.ref<complex<f32>>
   // I32_MINGW-DAG: [[Z1_VAL:%[0-9A-Za-z]+]] = fir.load [[Z1_ADDR]] : !fir.ref<complex<f32>>
@@ -852,7 +852,7 @@ func.func private @mlircomplexf32(%z1: complex<f32>, %z2: complex<f32>) -> compl
   // I32_MINGW-DAG: fir.store [[Z2_VAL]] to [[Z2_ADDRC]] : !fir.ref<complex<f32>>
   // I32_MINGW-DAG: [[Z2_ADDRT:%[0-9A-Za-z]+]] = fir.convert [[Z2_ADDRC]] : (!fir.ref<complex<f32>>) -> !fir.ref<tuple<f32, f32>>
 
-  // I32_MINGW: [[VAL:%[0-9A-Za-z]+]] = fir.call @mlircomplexf32([[Z1_ADDRT]], [[Z2_ADDRT]]) : (!fir.ref<tuple<f32, f32>>, !fir.ref<tuple<f32, f32>>) -> i64
+  // I32_MINGW: [[VAL:%[0-9A-Za-z]+]] = fir.call @mlircomplexf32([[Z1_ADDRT]], [[Z2_ADDRT]]) : (!fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}, !fir.ref<tuple<f32, f32>> {llvm.align = 4 : i32, llvm.byval = tuple<f32, f32>}) -> i64
 
   // X64-DAG: [[Z2_ADDR:%[0-9A-Za-z]+]] = fir.alloca !fir.vector<2:f32>
   // X64-DAG: fir.store [[Z2]] to [[Z2_ADDR]] : !fir.ref<!fir.vector<2:f32>>
diff --git a/flang/test/Fir/target-rewrite-complex16.fir b/flang/test/Fir/target-rewrite-complex16.fir
index 86a5b0051ab2e..199aa73b879c5 100644
--- a/flang/test/Fir/target-rewrite-complex16.fir
+++ b/flang/test/Fir/target-rewrite-complex16.fir
@@ -66,7 +66,7 @@ func.func @addrof() {
 // CHECK-LABEL:   func.func @callcomplex16() {
 // CHECK:           %[[VAL_0:.*]] = llvm.intr.stacksave : !llvm.ptr
 // CHECK:           %[[VAL_1:.*]] = fir.alloca tuple<f128, f128>
-// CHECK:           fir.call @returncomplex16(%[[VAL_1]]) : (!fir.ref<tuple<f128, f128>>) -> ()
+// CHECK:           fir.call @returncomplex16(%[[VAL_1]]) : (!fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.sret = tuple<f128, f128>}) -> ()
 // CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<tuple<f128, f128>>) -> !fir.ref<complex<f128>>
 // CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_2]] : !fir.ref<complex<f128>>
 // CHECK:           llvm.intr.stackrestore %[[VAL_0]] : !llvm.ptr
@@ -74,7 +74,7 @@ func.func @addrof() {
 // CHECK:           %[[VAL_5:.*]] = fir.alloca complex<f128>
 // CHECK:           fir.store %[[VAL_3]] to %[[VAL_5]] : !fir.ref<complex<f128>>
 // CHECK:           %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<complex<f128>>) -> !fir.ref<tuple<f128, f128>>
-// CHECK:           fir.call @paramcomplex16(%[[VAL_6]]) : (!fir.ref<tuple<f128, f128>>) -> ()
+// CHECK:           fir.call @paramcomplex16(%[[VAL_6]]) : (!fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.byval = tuple<f128, f128>}) -> ()
 // CHECK:           llvm.intr.stackrestore %[[VAL_4]] : !llvm.ptr
 // CHECK:           return
 // CHECK:         }
@@ -98,7 +98,7 @@ func.func @addrof() {
 // CHECK:           %[[VAL_14:.*]] = fir.alloca complex<f128>
 // CHECK:           fir.store %[[VAL_4]] to %[[VAL_14]] : !fir.ref<complex<f128>>
 // CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.ref<complex<f128>>) -> !fir.ref<tuple<f128, f128>>
-// CHECK:           fir.call @calleemultipleparamscomplex16(%[[VAL_11]], %[[VAL_13]], %[[VAL_15]]) : (!fir.ref<tuple<f128, f128>>, !fir.ref<tuple<f128, f128>>, !fir.ref<tuple<f128, f128>>) -> ()
+// CHECK:           fir.call @calleemultipleparamscomplex16(%[[VAL_11]], %[[VAL_13]], %[[VAL_15]]) : (!fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.byval = tuple<f128, f128>}, !fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.byval = tuple<f128, f128>}, !fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.byval = tuple<f128, f128>}) -> ()
 // CHECK:           llvm.intr.stackrestore %[[VAL_9]] : !llvm.ptr
 // CHECK:           return
 // CHECK:         }
@@ -117,7 +117,7 @@ func.func @addrof() {
 // CHECK:           %[[VAL_11:.*]] = fir.alloca complex<f128>
 // CHECK:           fir.store %[[VAL_4]] to %[[VAL_11]] : !fir.ref<complex<f128>>
 // CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (!fir.ref<complex<f128>>) -> !fir.ref<tuple<f128, f128>>
-// CHECK:           fir.call @mlircomplexf128(%[[VAL_8]], %[[VAL_10]], %[[VAL_12]]) : (!fir.ref<tuple<f128, f128>>, !fir.ref<tuple<f128, f128>>, !fir.ref<tuple<f128, f128>>) -> ()
+// CHECK:           fir.call @mlircomplexf128(%[[VAL_8]], %[[VAL_10]], %[[VAL_12]]) : (!fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.sret = tuple<f128, f128>}, !fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.byval = tuple<f128, f128>}, !fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.byval = tuple<f128, f128>}) -> ()
 // CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_8]] : (!fir.ref<tuple<f128, f128>>) -> !fir.ref<complex<f128>>
 // CHECK:           %[[VAL_14:.*]] = fir.load %[[VAL_13]] : !fir.ref<complex<f128>>
 // CHECK:           llvm.intr.stackrestore %[[VAL_7]] : !llvm.ptr
diff --git a/flang/test/Fir/target-rewrite-indirect-calls.fir b/flang/test/Fir/target-rewrite-indirect-calls.fir
index dbb3d0823520c..58a9a260b9a98 100644
--- a/flang/test/Fir/target-rewrite-indirect-calls.fir
+++ b/flang/test/Fir/target-rewrite-indirect-calls.fir
@@ -16,7 +16,7 @@ func.func @test(%arg0: () -> (), %arg1: !fir.ref<!fir.type<t{a:!fir.array<5xf64>
 // CHECK:           %[[VAL_5:.*]] = llvm.intr.stacksave : !llvm.ptr
 // CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.type<t{a:!fir.array<5xf64>}>
 // CHECK:           fir.store %[[VAL_3]] to %[[VAL_6]] : !fir.ref<!fir.type<t{a:!fir.array<5xf64>}>>
-// CHECK:           fir.call %[[VAL_4]](%[[VAL_6]], %[[VAL_2]]) : (!fir.ref<!fir.type<t{a:!fir.array<5xf64>}>> {llvm.byval = !fir.type<t{a:!fir.array<5xf64>}>}, f64) -> ()
+// CHECK:           fir.call %[[VAL_4]](%[[VAL_6]], %[[VAL_2]]) : (!fir.ref<!fir.type<t{a:!fir.array<5xf64>}>> {llvm.align = 8 : i32, llvm.byval = !fir.type<t{a:!fir.array<5xf64>}>}, f64) -> ()
 // CHECK:           llvm.intr.stackrestore %[[VAL_5]] : !llvm.ptr
 // CHECK:           return
 // CHECK:         }
diff --git a/flang/test/Fir/target-rewrite-integer-loongarch64.fir b/flang/test/Fir/target-rewrite-integer-loongarch64.fir
index 8421cbbb41a9d..96d341caffb53 100644
--- a/flang/test/Fir/target-rewrite-integer-loongarch64.fir
+++ b/flang/test/Fir/target-rewrite-integer-loongarch64.fir
@@ -1,10 +1,11 @@
 /// Test i32 passing and returning on LoongArch64
 /// LoongArch64 LP64D ABI requires unsigned 32 bit integers to be sign extended.
 
-// REQUIRES: loongarch-registered-target
 // RUN: fir-opt --target-rewrite="target=loongarch64-unknown-linux-gnu" %s | FileCheck %s --check-prefix=LOONGARCH64
 // RUN: tco -target="loongarch64-unknown-linux-gnu" %s | FileCheck %s --check-prefix=LOONGARCH64_LLVM
 
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128", llvm.target_triple = "loongarch64-unknown-linux-gnu"} {
+
 // LOONGARCH64: func.func private @cfunc32(i32 {llvm.signext}) -> (i32 {llvm.signext}) attributes {fir.bindc_name = "cfunc32"}
 
 // LOONGARCH64_LLVM: declare signext i32 @cfunc32(i32 signext)
@@ -25,3 +26,4 @@ func.func @foo(%0: i32) -> i32 attributes {fir.bindc_name = "foo"} {
   %1 = fir.call @cfunc32(%0) fastmath<contract> : (i32) -> i32
   return %1 : i32
 }
+}
diff --git a/flang/test/Fir/target.fir b/flang/test/Fir/target.fir
index 62ce488a3ca2f..781d153f525ff 100644
--- a/flang/test/Fir/target.fir
+++ b/flang/test/Fir/target.fir
@@ -68,7 +68,7 @@ func.func @call4() {
   // AARCH64: = call { float, float } @gen4()
   // PPC: = call { float, float } @gen4()
   %1 = fir.call @gen4() : () -> complex<f32>
-  // I32: call void @sink4(ptr %
+  // I32: call void @sink4(ptr byval({ float, float }) align 4 %
   // X64: call void @sink4(<2 x float> %
   // AARCH64: call void @sink4([2 x float] %
   // PPC: call void @sink4(float %{{.*}}, float %{{.*}})
@@ -80,12 +80,12 @@ func.func @call4() {
 // X64-LABEL: define void @call8()
 // AARCH64-LABEL: define void @call8()
 func.func @call8() {
-  // I32: call void @gen8(ptr %
+  // I32: call void @gen8(ptr sret({ double, double }) align 4 %
   // X64: = call { double, double } @gen8()
   // AARCH64: = call { double, double } @gen8()
   // PPC: = call { double, double } @gen8()
   %1 = fir.call @gen8() : () -> complex<f64>
-  // I32: call void @sink8(ptr %
+  // I32: call void @sink8(ptr byval({ double, double }) align 4 %
   // X64: call void @sink8(double %{{[0-9]*}}, double %{{[0-9]*}})
   // AARCH64: call void @sink8([2 x double] %
   // PPC: call void @sink8(double %{{.*}}, double %{{.*}})
diff --git a/flang/test/HLFIR/order_assignments/forall-proc-pointer-assignment-codegen.fir b/flang/test/HLFIR/order_assignments/forall-proc-pointer-assignment-codegen.fir
new file mode 100644
index 0000000000000..c5fcc4d943927
--- /dev/null
+++ b/flang/test/HLFIR/order_assignments/forall-proc-pointer-assignment-codegen.fir
@@ -0,0 +1,222 @@
+// Test code generation of hlfir.region_assign representing procedure pointer
+// assignments inside FORALL.
+
+// RUN: fir-opt %s --lower-hlfir-ordered-assignments | FileCheck %s
+
+!t=!fir.type<t{p:!fir.boxproc<() -> i32>}>
+func.func @test_no_conflict(%arg0: !fir.ref<!fir.array<10x!t>> {fir.bindc_name = "x"}) {
+  %c10_i64 = arith.constant 10 : i64
+  %c1_i64 = arith.constant 1 : i64
+  %c10 = arith.constant 10 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c10 : (index) -> !fir.shape<1>
+  %2:2 = hlfir.declare %arg0(%1) dummy_scope %0 {uniq_name = "x"} : (!fir.ref<!fir.array<10x!t>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10x!t>>, !fir.ref<!fir.array<10x!t>>)
+  hlfir.forall lb {
+    hlfir.yield %c1_i64 : i64
+  } ub {
+    hlfir.yield %c10_i64 : i64
+  }  (%arg1: i64) {
+    hlfir.region_assign {
+      %3 = fir.address_of(@f1) : () -> i32
+      %4 = fir.emboxproc %3 : (() -> i32) -> !fir.boxproc<() -> ()>
+      hlfir.yield %4 : !fir.boxproc<() -> ()>
+    } to {
+      %3 = hlfir.designate %2#0 (%arg1)  : (!fir.ref<!fir.array<10x!t>>, i64) -> !fir.ref<!t>
+      %4 = hlfir.designate %3{"p"}   {fortran_attrs = #fir.var_attrs<pointer>} : (!fir.ref<!t>) -> !fir.ref<!fir.boxproc<() -> i32>>
+      hlfir.yield %4 : !fir.ref<!fir.boxproc<() -> i32>>
+    }
+  }
+  return
+}
+// CHECK-LABEL:   func.func @test_no_conflict(
+// CHECK:           %[[VAL_1:.*]] = arith.constant 10 : i64
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_3:.*]] = arith.constant 10 : index
+// CHECK:           %[[VAL_4:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare{{.*}}"x"
+// CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_2]] : (i64) -> index
+// CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_1]] : (i64) -> index
+// CHECK:           %[[VAL_9:.*]] = arith.constant 1 : index
+// CHECK:           fir.do_loop %[[VAL_10:.*]] = %[[VAL_7]] to %[[VAL_8]] step %[[VAL_9]] {
+// CHECK:             %[[VAL_11:.*]] = fir.convert %[[VAL_10]] : (index) -> i64
+// CHECK:             %[[VAL_12:.*]] = hlfir.designate %[[VAL_6]]#0 (%[[VAL_11]])  : (!fir.ref<!fir.array<10x!fir.type<t{p:!fir.boxproc<() -> i32>}>>>, i64) -> !fir.ref<!fir.type<t{p:!fir.boxproc<() -> i32>}>>
+// CHECK:             %[[VAL_13:.*]] = hlfir.designate %[[VAL_12]]{"p"}   {fortran_attrs = #fir.var_attrs<pointer>} : (!fir.ref<!fir.type<t{p:!fir.boxproc<() -> i32>}>>) -> !fir.ref<!fir.boxproc<() -> i32>>
+// CHECK:             %[[VAL_14:.*]] = fir.address_of(@f1) : () -> i32
+// CHECK:             %[[VAL_15:.*]] = fir.emboxproc %[[VAL_14]] : (() -> i32) -> !fir.boxproc<() -> ()>
+// CHECK:             %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (!fir.boxproc<() -> ()>) -> !fir.boxproc<() -> i32>
+// CHECK:             fir.store %[[VAL_16]] to %[[VAL_13]] : !fir.ref<!fir.boxproc<() -> i32>>
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+func.func @test_need_to_save_rhs(%arg0: !fir.ref<!fir.array<10x!t>> {fir.bindc_name = "x"}) {
+  %c10_i64 = arith.constant 10 : i64
+  %c1_i64 = arith.constant 1 : i64
+  %c10 = arith.constant 10 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c10 : (index) -> !fir.shape<1>
+  %2:2 = hlfir.declare %arg0(%1) dummy_scope %0 {uniq_name = "x"} : (!fir.ref<!fir.array<10x!t>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10x!t>>, !fir.ref<!fir.array<10x!t>>)
+  hlfir.forall lb {
+    hlfir.yield %c1_i64 : i64
+  } ub {
+    hlfir.yield %c10_i64 : i64
+  }  (%arg1: i64) {
+    hlfir.region_assign {
+      %3 = hlfir.designate %2#0 (%c10)  : (!fir.ref<!fir.array<10x!t>>, index) -> !fir.ref<!t>
+      %4 = hlfir.designate %3{"p"}   {fortran_attrs = #fir.var_attrs<pointer>} : (!fir.ref<!t>) -> !fir.ref<!fir.boxproc<() -> i32>>
+      %5 = fir.load %4 : !fir.ref<!fir.boxproc<() -> i32>>
+      hlfir.yield %5 : !fir.boxproc<() -> i32>
+    } to {
+      %3 = hlfir.designate %2#0 (%arg1)  : (!fir.ref<!fir.array<10x!t>>, i64) -> !fir.ref<!t>
+      %4 = hlfir.designate %3{"p"}   {fortran_attrs = #fir.var_attrs<pointer>} : (!fir.ref<!t>) -> !fir.ref<!fir.boxproc<() -> i32>>
+      hlfir.yield %4 : !fir.ref<!fir.boxproc<() -> i32>>
+    }
+  }
+  return
+}
+// CHECK-LABEL:   func.func @test_need_to_save_rhs(
+// CHECK:           %[[VAL_1:.*]] = fir.alloca i64
+// CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.box<!fir.heap<i64>>
+// CHECK:           %[[VAL_3:.*]] = fir.alloca i64
+// CHECK:           %[[VAL_4:.*]] = arith.constant 10 : i64
+// CHECK:           %[[VAL_5:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_6:.*]] = arith.constant 10 : index
+// CHECK:           %[[VAL_7:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare{{.*}}x
+// CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_5]] : (i64) -> index
+// CHECK:           %[[VAL_11:.*]] = fir.convert %[[VAL_4]] : (i64) -> index
+// CHECK:           %[[VAL_12:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_13:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_14:.*]] = arith.constant 1 : i64
+// CHECK:           fir.store %[[VAL_13]] to %[[VAL_3]] : !fir.ref<i64>
+// CHECK:           %[[VAL_19:.*]] = fir.call @_FortranACreateValueStack(
+// CHECK:           fir.do_loop %[[VAL_20:.*]] = %[[VAL_10]] to %[[VAL_11]] step %[[VAL_12]] {
+// CHECK:             %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (index) -> i64
+// CHECK:             %[[VAL_22:.*]] = hlfir.designate %[[VAL_9]]#0 (%[[VAL_6]])  : (!fir.ref<!fir.array<10x!fir.type<t{p:!fir.boxproc<() -> i32>}>>>, index) -> !fir.ref<!fir.type<t{p:!fir.boxproc<() -> i32>}>>
+// CHECK:             %[[VAL_23:.*]] = hlfir.designate %[[VAL_22]]{"p"}   {fortran_attrs = #fir.var_attrs<pointer>} : (!fir.ref<!fir.type<t{p:!fir.boxproc<() -> i32>}>>) -> !fir.ref<!fir.boxproc<() -> i32>>
+// CHECK:             %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<!fir.boxproc<() -> i32>>
+// CHECK:             %[[VAL_25:.*]] = fir.box_addr %[[VAL_24]] : (!fir.boxproc<() -> i32>) -> (() -> i32)
+// CHECK:             %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (() -> i32) -> i64
+// CHECK:             fir.store %[[VAL_26]] to %[[VAL_1]] : !fir.ref<i64>
+// CHECK:             %[[VAL_27:.*]] = fir.embox %[[VAL_1]] : (!fir.ref<i64>) -> !fir.box<i64>
+// CHECK:             %[[VAL_28:.*]] = fir.convert %[[VAL_27]] : (!fir.box<i64>) -> !fir.box<none>
+// CHECK:             fir.call @_FortranAPushValue(%[[VAL_19]], %[[VAL_28]]) : (!fir.llvm_ptr<i8>, !fir.box<none>) -> ()
+// CHECK:           }
+// CHECK:           %[[VAL_29:.*]] = fir.convert %[[VAL_5]] : (i64) -> index
+// CHECK:           %[[VAL_30:.*]] = fir.convert %[[VAL_4]] : (i64) -> index
+// CHECK:           %[[VAL_31:.*]] = arith.constant 1 : index
+// CHECK:           fir.store %[[VAL_13]] to %[[VAL_3]] : !fir.ref<i64>
+// CHECK:           fir.do_loop %[[VAL_32:.*]] = %[[VAL_29]] to %[[VAL_30]] step %[[VAL_31]] {
+// CHECK:             %[[VAL_33:.*]] = fir.convert %[[VAL_32]] : (index) -> i64
+// CHECK:             %[[VAL_34:.*]] = hlfir.designate %[[VAL_9]]#0 (%[[VAL_33]])  : (!fir.ref<!fir.array<10x!fir.type<t{p:!fir.boxproc<() -> i32>}>>>, i64) -> !fir.ref<!fir.type<t{p:!fir.boxproc<() -> i32>}>>
+// CHECK:             %[[VAL_35:.*]] = hlfir.designate %[[VAL_34]]{"p"}   {fortran_attrs = #fir.var_attrs<pointer>} : (!fir.ref<!fir.type<t{p:!fir.boxproc<() -> i32>}>>) -> !fir.ref<!fir.boxproc<() -> i32>>
+// CHECK:             %[[VAL_36:.*]] = fir.load %[[VAL_3]] : !fir.ref<i64>
+// CHECK:             %[[VAL_37:.*]] = arith.addi %[[VAL_36]], %[[VAL_14]] : i64
+// CHECK:             fir.store %[[VAL_37]] to %[[VAL_3]] : !fir.ref<i64>
+// CHECK:             %[[VAL_38:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.box<!fir.heap<i64>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:             fir.call @_FortranAValueAt(%[[VAL_19]], %[[VAL_36]], %[[VAL_38]]) : (!fir.llvm_ptr<i8>, i64, !fir.ref<!fir.box<none>>) -> ()
+// CHECK:             %[[VAL_39:.*]] = fir.load %[[VAL_2]] : !fir.ref<!fir.box<!fir.heap<i64>>>
+// CHECK:             %[[VAL_40:.*]] = fir.box_addr %[[VAL_39]] : (!fir.box<!fir.heap<i64>>) -> !fir.heap<i64>
+// CHECK:             %[[VAL_41:.*]] = fir.load %[[VAL_40]] : !fir.heap<i64>
+// CHECK:             %[[VAL_42:.*]] = fir.convert %[[VAL_41]] : (i64) -> (() -> i32)
+// CHECK:             %[[VAL_43:.*]] = fir.emboxproc %[[VAL_42]] : (() -> i32) -> !fir.boxproc<() -> i32>
+// CHECK:             fir.store %[[VAL_43]] to %[[VAL_35]] : !fir.ref<!fir.boxproc<() -> i32>>
+// CHECK:           }
+// CHECK:           fir.call @_FortranADestroyValueStack(%[[VAL_19]]) : (!fir.llvm_ptr<i8>) -> ()
+// CHECK:           return
+// CHECK:         }
+
+func.func @test_need_to_save_lhs(%arg0: !fir.ref<!fir.array<10x!t>>) {
+  %c11_i64 = arith.constant 11 : i64
+  %c10_i64 = arith.constant 10 : i64
+  %c1_i64 = arith.constant 1 : i64
+  %c10 = arith.constant 10 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1 = fir.shape %c10 : (index) -> !fir.shape<1>
+  %2:2 = hlfir.declare %arg0(%1) dummy_scope %0 {uniq_name = "x"} : (!fir.ref<!fir.array<10x!t>>, !fir.shape<1>, !fir.dscope) -> (!fir.ref<!fir.array<10x!t>>, !fir.ref<!fir.array<10x!t>>)
+  hlfir.forall lb {
+    hlfir.yield %c1_i64 : i64
+  } ub {
+    hlfir.yield %c10_i64 : i64
+  }  (%arg1: i64) {
+    hlfir.region_assign {
+      %3 = fir.address_of(@f1) : () -> i32
+      %4 = fir.emboxproc %3 : (() -> i32) -> !fir.boxproc<() -> ()>
+      hlfir.yield %4 : !fir.boxproc<() -> ()>
+    } to {
+      %3 = arith.subi %c11_i64, %arg1 : i64
+      %4 = hlfir.designate %2#0 (%3)  : (!fir.ref<!fir.array<10x!t>>, i64) -> !fir.ref<!t>
+      %5 = hlfir.designate %4{"p"}   {fortran_attrs = #fir.var_attrs<pointer>} : (!fir.ref<!t>) -> !fir.ref<!fir.boxproc<() -> i32>>
+      %6 = fir.load %5 : !fir.ref<!fir.boxproc<() -> i32>>
+      %7 = fir.box_addr %6 : (!fir.boxproc<() -> i32>) -> (() -> i32)
+      %8 = fir.call %7() proc_attrs<pure> : () -> i32
+      %9 = fir.convert %8 : (i32) -> i64
+      %10 = hlfir.designate %2#0 (%9)  : (!fir.ref<!fir.array<10x!t>>, i64) -> !fir.ref<!t>
+      %11 = hlfir.designate %10{"p"}   {fortran_attrs = #fir.var_attrs<pointer>} : (!fir.ref<!t>) -> !fir.ref<!fir.boxproc<() -> i32>>
+      hlfir.yield %11 : !fir.ref<!fir.boxproc<() -> i32>>
+    }
+  }
+  return
+}
+// CHECK-LABEL:   func.func @test_need_to_save_lhs(
+// CHECK:           %[[VAL_1:.*]] = fir.alloca i64
+// CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.box<!fir.heap<i64>>
+// CHECK:           %[[VAL_3:.*]] = fir.alloca i64
+// CHECK:           %[[VAL_4:.*]] = arith.constant 11 : i64
+// CHECK:           %[[VAL_5:.*]] = arith.constant 10 : i64
+// CHECK:           %[[VAL_6:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_7:.*]] = arith.constant 10 : index
+// CHECK:           %[[VAL_8:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_7]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare{{.*}}"x"
+// CHECK:           %[[VAL_11:.*]] = fir.convert %[[VAL_6]] : (i64) -> index
+// CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_5]] : (i64) -> index
+// CHECK:           %[[VAL_13:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_14:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_15:.*]] = arith.constant 1 : i64
+// CHECK:           fir.store %[[VAL_14]] to %[[VAL_3]] : !fir.ref<i64>
+// CHECK:           %[[VAL_20:.*]] = fir.call @_FortranACreateValueStack(
+// CHECK:           fir.do_loop %[[VAL_21:.*]] = %[[VAL_11]] to %[[VAL_12]] step %[[VAL_13]] {
+// CHECK:             %[[VAL_22:.*]] = fir.convert %[[VAL_21]] : (index) -> i64
+// CHECK:             %[[VAL_23:.*]] = arith.subi %[[VAL_4]], %[[VAL_22]] : i64
+// CHECK:             %[[VAL_24:.*]] = hlfir.designate %[[VAL_10]]#0 (%[[VAL_23]])  : (!fir.ref<!fir.array<10x!fir.type<t{p:!fir.boxproc<() -> i32>}>>>, i64) -> !fir.ref<!fir.type<t{p:!fir.boxproc<() -> i32>}>>
+// CHECK:             %[[VAL_25:.*]] = hlfir.designate %[[VAL_24]]{"p"}   {fortran_attrs = #fir.var_attrs<pointer>} : (!fir.ref<!fir.type<t{p:!fir.boxproc<() -> i32>}>>) -> !fir.ref<!fir.boxproc<() -> i32>>
+// CHECK:             %[[VAL_26:.*]] = fir.load %[[VAL_25]] : !fir.ref<!fir.boxproc<() -> i32>>
+// CHECK:             %[[VAL_27:.*]] = fir.box_addr %[[VAL_26]] : (!fir.boxproc<() -> i32>) -> (() -> i32)
+// CHECK:             %[[VAL_28:.*]] = fir.call %[[VAL_27]]() proc_attrs<pure> : () -> i32
+// CHECK:             %[[VAL_29:.*]] = fir.convert %[[VAL_28]] : (i32) -> i64
+// CHECK:             %[[VAL_30:.*]] = hlfir.designate %[[VAL_10]]#0 (%[[VAL_29]])  : (!fir.ref<!fir.array<10x!fir.type<t{p:!fir.boxproc<() -> i32>}>>>, i64) -> !fir.ref<!fir.type<t{p:!fir.boxproc<() -> i32>}>>
+// CHECK:             %[[VAL_31:.*]] = hlfir.designate %[[VAL_30]]{"p"}   {fortran_attrs = #fir.var_attrs<pointer>} : (!fir.ref<!fir.type<t{p:!fir.boxproc<() -> i32>}>>) -> !fir.ref<!fir.boxproc<() -> i32>>
+// CHECK:             %[[VAL_32:.*]] = fir.convert %[[VAL_31]] : (!fir.ref<!fir.boxproc<() -> i32>>) -> i64
+// CHECK:             fir.store %[[VAL_32]] to %[[VAL_1]] : !fir.ref<i64>
+// CHECK:             %[[VAL_33:.*]] = fir.embox %[[VAL_1]] : (!fir.ref<i64>) -> !fir.box<i64>
+// CHECK:             %[[VAL_34:.*]] = fir.convert %[[VAL_33]] : (!fir.box<i64>) -> !fir.box<none>
+// CHECK:             fir.call @_FortranAPushValue(%[[VAL_20]], %[[VAL_34]]) : (!fir.llvm_ptr<i8>, !fir.box<none>) -> ()
+// CHECK:           }
+// CHECK:           %[[VAL_35:.*]] = fir.convert %[[VAL_6]] : (i64) -> index
+// CHECK:           %[[VAL_36:.*]] = fir.convert %[[VAL_5]] : (i64) -> index
+// CHECK:           %[[VAL_37:.*]] = arith.constant 1 : index
+// CHECK:           fir.store %[[VAL_14]] to %[[VAL_3]] : !fir.ref<i64>
+// CHECK:           fir.do_loop %[[VAL_38:.*]] = %[[VAL_35]] to %[[VAL_36]] step %[[VAL_37]] {
+// CHECK:             %[[VAL_39:.*]] = fir.convert %[[VAL_38]] : (index) -> i64
+// CHECK:             %[[VAL_40:.*]] = fir.load %[[VAL_3]] : !fir.ref<i64>
+// CHECK:             %[[VAL_41:.*]] = arith.addi %[[VAL_40]], %[[VAL_15]] : i64
+// CHECK:             fir.store %[[VAL_41]] to %[[VAL_3]] : !fir.ref<i64>
+// CHECK:             %[[VAL_42:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.box<!fir.heap<i64>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:             fir.call @_FortranAValueAt(%[[VAL_20]], %[[VAL_40]], %[[VAL_42]]) : (!fir.llvm_ptr<i8>, i64, !fir.ref<!fir.box<none>>) -> ()
+// CHECK:             %[[VAL_43:.*]] = fir.load %[[VAL_2]] : !fir.ref<!fir.box<!fir.heap<i64>>>
+// CHECK:             %[[VAL_44:.*]] = fir.box_addr %[[VAL_43]] : (!fir.box<!fir.heap<i64>>) -> !fir.heap<i64>
+// CHECK:             %[[VAL_45:.*]] = fir.load %[[VAL_44]] : !fir.heap<i64>
+// CHECK:             %[[VAL_46:.*]] = fir.convert %[[VAL_45]] : (i64) -> !fir.ref<!fir.boxproc<() -> i32>>
+// CHECK:             %[[VAL_47:.*]] = fir.address_of(@f1) : () -> i32
+// CHECK:             %[[VAL_48:.*]] = fir.emboxproc %[[VAL_47]] : (() -> i32) -> !fir.boxproc<() -> ()>
+// CHECK:             %[[VAL_49:.*]] = fir.convert %[[VAL_48]] : (!fir.boxproc<() -> ()>) -> !fir.boxproc<() -> i32>
+// CHECK:             fir.store %[[VAL_49]] to %[[VAL_46]] : !fir.ref<!fir.boxproc<() -> i32>>
+// CHECK:           }
+// CHECK:           fir.call @_FortranADestroyValueStack(%[[VAL_20]]) : (!fir.llvm_ptr<i8>) -> ()
+// CHECK:           return
+// CHECK:         }
+
+func.func private @f1() -> i32 attributes {fir.proc_attrs = #fir.proc_attrs<pure>}
diff --git a/flang/test/HLFIR/order_assignments/forall-proc-pointer-assignment-scheduling-character.f90 b/flang/test/HLFIR/order_assignments/forall-proc-pointer-assignment-scheduling-character.f90
new file mode 100644
index 0000000000000..d2d1939890882
--- /dev/null
+++ b/flang/test/HLFIR/order_assignments/forall-proc-pointer-assignment-scheduling-character.f90
@@ -0,0 +1,126 @@
+! Test analysis of character procedure pointer assignments inside FORALL.
+! Character procedure gets their own tests because they are tracked differently
+! in FIR because of the length of the function result.
+
+! RUN: bbc -hlfir -o /dev/null -pass-pipeline="builtin.module(lower-hlfir-ordered-assignments)" \
+! RUN: --debug-only=flang-ordered-assignment -flang-dbg-order-assignment-schedule-only -I nw %s 2>&1 | FileCheck %s
+! REQUIRES: asserts
+
+module char_proc_ptr_forall
+  type :: t
+    procedure(f1), nopass, pointer :: p
+  end type
+contains
+  pure character(2) function f1()
+    f1 = "01"
+  end function
+  pure character(2) function f2()
+    f2 = "02"
+  end function
+  pure character(2) function f3()
+    f3 = "03"
+  end function
+  pure character(2) function f4()
+    f4 = "04"
+  end function
+  pure character(2) function f5()
+    f5 = "05"
+  end function
+  pure character(2) function f6()
+    f6 = "06"
+  end function
+  pure character(2) function f7()
+    f7 = "07"
+  end function
+  pure character(2) function f8()
+    f8 = "08"
+  end function
+  pure character(2) function f9()
+    f9 = "09"
+  end function
+  pure character(2) function f10()
+    f10 = "10"
+  end function
+
+  integer pure function decode(c)
+    character(2), intent(in) :: c
+    decode = modulo(iachar(c(2:2))-49,10)+1 
+  end function
+
+  subroutine test_no_conflict(x)
+    type(t) :: x(10)
+    forall(i=1:10) x(i)%p => f1
+  end subroutine
+! CHECK: ------------ scheduling forall in _QMchar_proc_ptr_forallPtest_no_conflict ------------
+! CHECK-NEXT: run 1 evaluate: forall/region_assign1
+
+  subroutine test_need_to_save_rhs(x)
+    type(t) :: x(10)
+    forall(i=1:10) x(i)%p => x(11-i)%p
+  end subroutine
+! CHECK: ------------ scheduling forall in _QMchar_proc_ptr_forallPtest_need_to_save_rhs ------------
+! CHECK-NEXT: conflict: R/W
+! CHECK-NEXT: run 1 save    : forall/region_assign1/rhs
+! CHECK-NEXT: run 2 evaluate: forall/region_assign1
+
+  subroutine test_need_to_save_lhs(x)
+    type(t) :: x(10)
+    forall(i=1:10) x(decode(x(11-i)%p()))%p => f1
+  end subroutine
+! CHECK: ------------ scheduling forall in _QMchar_proc_ptr_forallPtest_need_to_save_lhs ------------
+! CHECK: conflict: R/W
+! CHECK-NEXT: run 1 save    : forall/region_assign1/lhs
+! CHECK-NEXT: run 2 evaluate: forall/region_assign1
+
+  subroutine test_need_to_save_lhs_and_rhs(x)
+    type(t) :: x(10)
+    forall(i=1:10) x(decode(x(11-i)%p()))%p => x(modulo(-2*i, 11))%p
+  end subroutine
+! CHECK: ------------ scheduling forall in _QMchar_proc_ptr_forallPtest_need_to_save_lhs_and_rhs ------------
+! CHECK: conflict: R/W
+! CHECK-NEXT: run 1 save    : forall/region_assign1/rhs
+! CHECK: conflict: R/W
+! CHECK-NEXT: run 1 save    : forall/region_assign1/lhs
+! CHECK-NEXT: run 2 evaluate: forall/region_assign1
+
+
+! End-to-end test utilities for debugging purposes.
+
+  subroutine reset(a)
+    type(t) :: a(:)
+    a = [t(f10), t(f9), t(f8), t(f7), t(f6), t(f5), t(f4), t(f3), t(f2), t(f1)]
+  end subroutine
+
+  subroutine print(a)
+    type(t) :: a(:)
+    print *, [(decode(a(i)%p()), i=1,10)]
+  end subroutine
+
+  logical function check_equal(a, expected)
+    type(t) :: a(:)
+    integer :: expected(:)
+    check_equal = all([(decode(a(i)%p()), i=1,10)].eq.expected)
+    if (.not.check_equal) then
+      print *, "expected:", expected
+      print *, "got:", [(decode(a(i)%p()), i=1,10)]
+    end if
+  end function
+end module
+
+! End-to-end test for debugging purposes (not verified by lit).
+  use char_proc_ptr_forall
+  type(t) :: a(10)
+
+  call reset(a)
+  call test_need_to_save_rhs(a)
+  if (.not.check_equal(a, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])) stop 1
+
+  call reset(a)
+  call test_need_to_save_lhs(a)
+  if (.not.check_equal(a, [1, 1, 1, 1, 1, 1, 1, 1, 1, 1])) stop 2
+
+  call reset(a)
+  call test_need_to_save_lhs_and_rhs(a)
+  if (.not.check_equal(a, [2, 4, 6, 8, 10, 1, 3, 5, 7, 9])) stop 3
+  print *, "PASS"
+end
diff --git a/flang/test/HLFIR/order_assignments/forall-proc-pointer-assignment-scheduling.f90 b/flang/test/HLFIR/order_assignments/forall-proc-pointer-assignment-scheduling.f90
new file mode 100644
index 0000000000000..ba9c203453d95
--- /dev/null
+++ b/flang/test/HLFIR/order_assignments/forall-proc-pointer-assignment-scheduling.f90
@@ -0,0 +1,123 @@
+! Test analysis of procedure pointer assignments inside FORALL.
+
+! RUN: bbc -hlfir -o /dev/null -pass-pipeline="builtin.module(lower-hlfir-ordered-assignments)" \
+! RUN: --debug-only=flang-ordered-assignment -flang-dbg-order-assignment-schedule-only -I nw %s 2>&1 | FileCheck %s
+! REQUIRES: asserts
+
+module proc_ptr_forall
+  type :: t
+    procedure(f1), nopass, pointer :: p
+  end type
+contains
+  pure integer function f1()
+    f1 = 1
+  end function
+  pure integer function f2()
+    f2 = 2
+  end function
+  pure integer function f3()
+    f3 = 3
+  end function
+  pure integer function f4()
+    f4 = 4
+  end function
+  pure integer function f5()
+    f5 = 5
+  end function
+  pure integer function f6()
+    f6 = 6
+  end function
+  pure integer function f7()
+    f7 = 7
+  end function
+  pure integer function f8()
+    f8 = 8
+  end function
+  pure integer function f9()
+    f9 = 9
+  end function
+  pure integer function f10()
+    f10 = 10
+  end function
+
+  subroutine test_no_conflict(x)
+    type(t) :: x(10)
+    forall(i=1:10) x(i)%p => f1
+  end subroutine
+! CHECK: ------------ scheduling forall in _QMproc_ptr_forallPtest_no_conflict ------------
+! CHECK-NEXT: run 1 evaluate: forall/region_assign1
+
+  subroutine test_need_to_save_rhs(x)
+    type(t) :: x(10)
+    forall(i=1:10) x(i)%p => x(11-i)%p
+  end subroutine
+! CHECK: ------------ scheduling forall in _QMproc_ptr_forallPtest_need_to_save_rhs ------------
+! CHECK-NEXT: conflict: R/W
+! CHECK-NEXT: run 1 save    : forall/region_assign1/rhs
+! CHECK-NEXT: run 2 evaluate: forall/region_assign1
+
+  subroutine test_need_to_save_lhs(x)
+    type(t) :: x(10)
+    forall(i=1:10) x(x(11-i)%p())%p => f1
+  end subroutine
+! CHECK: ------------ scheduling forall in _QMproc_ptr_forallPtest_need_to_save_lhs ------------
+! CHECK-NEXT: unknown effect: %{{.*}} = fir.call
+! CHECK-NEXT: unknown effect: %{{.*}} = fir.call
+! CHECK-NEXT: conflict: R/W
+! CHECK-NEXT: run 1 save    : forall/region_assign1/lhs
+! CHECK-NEXT: run 2 evaluate: forall/region_assign1
+
+  subroutine test_need_to_save_lhs_and_rhs(x)
+    type(t) :: x(10)
+    forall(i=1:10) x(x(11-i)%p())%p => x(modulo(-2*i, 11))%p
+  end subroutine
+! CHECK: ------------ scheduling forall in _QMproc_ptr_forallPtest_need_to_save_lhs_and_rhs ------------
+! CHECK-NEXT: unknown effect: %{{.*}} = fir.call
+! CHECK-NEXT: conflict: R/W
+! CHECK-NEXT: run 1 save    : forall/region_assign1/rhs
+! CHECK-NEXT: unknown effect: %{{.*}} = fir.call
+! CHECK-NEXT: conflict: R/W
+! CHECK-NEXT: run 1 save    : forall/region_assign1/lhs
+! CHECK-NEXT: run 2 evaluate: forall/region_assign1
+
+
+! End-to-end test utilities for debugging purposes.
+
+  subroutine reset(a)
+    type(t) :: a(:)
+    a = [t(f10), t(f9), t(f8), t(f7), t(f6), t(f5), t(f4), t(f3), t(f2), t(f1)]
+  end subroutine
+
+  subroutine print(a)
+    type(t) :: a(:)
+    print *, [(a(i)%p(), i=1,10)]
+  end subroutine
+
+  logical function check_equal(a, expected)
+    type(t) :: a(:)
+    integer :: expected(:)
+    check_equal = all([(a(i)%p(), i=1,10)].eq.expected)
+    if (.not.check_equal) then
+      print *, "expected:", expected
+      print *, "got:", [(a(i)%p(), i=1,10)]
+    end if
+  end function
+end module
+
+! End-to-end test for debugging purposes (not verified by lit).
+  use proc_ptr_forall
+  type(t) :: a(10)
+
+  call reset(a)
+  call test_need_to_save_rhs(a)
+  if (.not.check_equal(a, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])) stop 1
+
+  call reset(a)
+  call test_need_to_save_lhs(a)
+  if (.not.check_equal(a, [1, 1, 1, 1, 1, 1, 1, 1, 1, 1])) stop 2
+
+  call reset(a)
+  call test_need_to_save_lhs_and_rhs(a)
+  if (.not.check_equal(a, [2, 4, 6, 8, 10, 1, 3, 5, 7, 9])) stop 3
+  print *, "PASS"
+end
diff --git a/flang/test/Integration/OpenMP/map-types-and-sizes.f90 b/flang/test/Integration/OpenMP/map-types-and-sizes.f90
index 70ae353ced214..35eec823e30de 100644
--- a/flang/test/Integration/OpenMP/map-types-and-sizes.f90
+++ b/flang/test/Integration/OpenMP/map-types-and-sizes.f90
@@ -6,7 +6,7 @@
 ! added to this directory and sub-directories.
 !===----------------------------------------------------------------------===!
 
-!RUN: %flang_fc1 -emit-llvm -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa -flang-deprecated-no-hlfir %s -o - | FileCheck %s
+!RUN: %flang_fc1 -emit-llvm -fopenmp -fopenmp-version=51 -fopenmp-targets=amdgcn-amd-amdhsa -flang-deprecated-no-hlfir %s -o - | FileCheck %s
 
 !===============================================================================
 ! Check MapTypes for target implicit captures
@@ -39,6 +39,37 @@ subroutine mapType_ptr
   !$omp end target
 end subroutine mapType_ptr
 
+!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [1 x i64] [i64 4]
+!CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [1 x i64] [i64 4097]
+subroutine map_present_target_data
+  integer :: x
+!$omp target data map(present, to: x)
+!$omp end target data
+end subroutine
+
+!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [1 x i64] [i64 4]
+!CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [1 x i64] [i64 4097]
+subroutine map_present_update
+  integer :: x
+!$omp target update to(present: x)
+end subroutine
+
+!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [1 x i64] [i64 4]
+!CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [1 x i64] [i64 1027]
+subroutine map_close
+  integer :: x
+!$omp target data map(close, tofrom: x)
+!$omp end target data
+end subroutine
+
+!CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [1 x i64] [i64 4]
+!CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [1 x i64] [i64 8195]
+subroutine map_ompx_hold
+  integer :: x
+!$omp target data map(ompx_hold, tofrom: x)
+!$omp end target data
+end subroutine
+
 !CHECK: @.offload_sizes{{.*}} = private unnamed_addr constant [4 x i64] [i64 0, i64 24, i64 8, i64 0]
 !CHECK: @.offload_maptypes{{.*}} = private unnamed_addr constant [4 x i64] [i64 32, i64 281474976711169, i64 281474976711171, i64 281474976711187]
 subroutine mapType_allocatable
diff --git a/flang/test/Integration/OpenMP/parallel-private-reduction-worstcase.f90 b/flang/test/Integration/OpenMP/parallel-private-reduction-worstcase.f90
index 7e735b6499504..cf77c46346b7f 100644
--- a/flang/test/Integration/OpenMP/parallel-private-reduction-worstcase.f90
+++ b/flang/test/Integration/OpenMP/parallel-private-reduction-worstcase.f90
@@ -219,5 +219,5 @@ subroutine worst_case(a, b, c, d)
 !                [var extent was non-zero: malloc a private array]
 ! CHECK:         br label %omp.private.init5
 
-! CHECK:       omp.par.outlined.exit.exitStub:                   ; preds = %omp.region.cont52
+! CHECK:       omp.par.exit.exitStub:                           ; preds = %omp.region.cont52
 ! CHECK-NEXT:    ret void
diff --git a/flang/test/Integration/OpenMP/private-global.f90 b/flang/test/Integration/OpenMP/private-global.f90
index 1aacfb4c87198..8f8de8cdedd3b 100644
--- a/flang/test/Integration/OpenMP/private-global.f90
+++ b/flang/test/Integration/OpenMP/private-global.f90
@@ -31,14 +31,14 @@ program bug
 ! CHECK:         %[[FIFTY:.*]] = alloca i32, i64 1, align 4
 ! CHECK:         %[[INTERMEDIATE:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, align 8
 ! CHECK:         %[[TABLE_BOX_ADDR2:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, i64 1, align 8
-! CHECK:         call void @llvm.memcpy.p0.p0.i32(ptr %[[INTERMEDIATE]], ptr %[[PRIV_BOX_ALLOC]], i32 48, i1 false)
+! CHECK:         call void @llvm.memcpy.p0.p0.i32(ptr %[[INTERMEDIATE]], ptr %[[PRIV_BOX_ALLOC]], i32 {{4[48]}}, i1 false)
 ! CHECK:         store i32 50, ptr %[[FIFTY]], align 4
 ! CHECK:         %[[FIFTY_BOX_VAL:.*]] = insertvalue { ptr, i64, i32, i8, i8, i8, i8 } { ptr undef, i64 4, i32 20240719, i8 0, i8 9, i8 0, i8 0 }, ptr %[[FIFTY]], 0
-! CHECK:         store { ptr, i64, i32, i8, i8, i8, i8 } %[[FIFTY_BOX_VAL]], ptr %[[BOXED_FIFTY]], align 8
-! CHECK:         call void @llvm.memcpy.p0.p0.i32(ptr %[[TABLE_BOX_ADDR2]], ptr %[[INTERMEDIATE]], i32 48, i1 false)
+! CHECK:         store { ptr, i64, i32, i8, i8, i8, i8 } %[[FIFTY_BOX_VAL]], ptr %[[BOXED_FIFTY]], align {{[48]}}
+! CHECK:         call void @llvm.memcpy.p0.p0.i32(ptr %[[TABLE_BOX_ADDR2]], ptr %[[INTERMEDIATE]], i32 {{4[48]}}, i1 false)
 ! CHECK:         call void @_FortranAAssign(ptr %[[TABLE_BOX_ADDR2]], ptr %[[BOXED_FIFTY]], ptr @{{.*}}, i32 9)
-! CHECK:         call void @llvm.memcpy.p0.p0.i32(ptr %[[TABLE_BOX_ADDR]], ptr %[[PRIV_BOX_ALLOC]], i32 48, i1 false)
-! CHECK:         %[[PRIV_TABLE:.*]] = call ptr @malloc(i64 40)
+! CHECK:         call void @llvm.memcpy.p0.p0.i32(ptr %[[TABLE_BOX_ADDR]], ptr %[[PRIV_BOX_ALLOC]], i32 {{4[48]}}, i1 false)
+! CHECK:         %[[PRIV_TABLE:.*]] = call ptr @malloc(i{{(32)|(64)}} 40)
 ! ...
 ! check that we use the private copy of table for table/=50
 ! CHECK:       omp.par.region3:
diff --git a/flang/test/Lower/CUDA/cuda-device-proc.cuf b/flang/test/Lower/CUDA/cuda-device-proc.cuf
index c651d34c55093..554e6fcfa5327 100644
--- a/flang/test/Lower/CUDA/cuda-device-proc.cuf
+++ b/flang/test/Lower/CUDA/cuda-device-proc.cuf
@@ -150,21 +150,33 @@ end subroutine
 ! CHECK: fir.convert %{{.*}} : (f64) -> i64
 ! CHECK: fir.call @llvm.nvvm.match.any.sync.i64p
 
-attributes(device) subroutine testAtomic()
-  integer :: a, istat, j
+attributes(device) subroutine testAtomic(aa, n)
+  integer :: aa(*)
+  integer, intent(in) :: n
+  integer :: a, istat, j, i
   real :: r
   istat = atomicexch(a,0)
   istat = atomicexch(r, 0.0)
   istat = atomicxor(a, j)
   istat = atomiccas(a, i, 14)
+  do i = 1, n
+    istat = atomicxor(aa, i)
+    istat = atomiccas(aa, i, 14)
+    istat = atomicexch(aa, 0)
+  end do
 end subroutine
 
-! CHECK-LABEL: func.func @_QPtestatomic()
+! CHECK-LABEL: func.func @_QPtestatomic
 ! CHECK: llvm.atomicrmw xchg %{{.*}}, %c0{{.*}} seq_cst : !llvm.ptr, i32
 ! CHECK: llvm.atomicrmw xchg %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, f32
 ! CHECK: llvm.atomicrmw _xor %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i32
 ! CHECK: %[[ADDR:.*]] = builtin.unrealized_conversion_cast %{{.*}}#1 : !fir.ref<i32> to !llvm.ptr
 ! CHECK: llvm.cmpxchg %[[ADDR]], %{{.*}}, %c14{{.*}} acq_rel monotonic : !llvm.ptr, i32
+! CHECK: fir.do_loop
+! CHECK: llvm.atomicrmw _xor %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i32
+! CHECK: %[[ADDR:.*]] = builtin.unrealized_conversion_cast %{{.*}}#1 : !fir.ref<!fir.array<?xi32>> to !llvm.ptr 
+! CHECK: llvm.cmpxchg %[[ADDR]], %{{.*}}, %c14{{.*}} acq_rel monotonic : !llvm.ptr, i32
+! CHECK: llvm.atomicrmw xchg %{{.*}}, %c0{{.*}} seq_cst : !llvm.ptr, i32
 
 attributes(device) subroutine testAtomic2()
   integer(8) :: a, i, istat
@@ -198,3 +210,115 @@ end subroutine
 ! CHECK: %[[CAST:.*]] = builtin.unrealized_conversion_cast %{{.*}}#1 : !fir.ref<f64> to !llvm.ptr
 ! CHECK: %[[ATOMIC:.*]] = llvm.cmpxchg %[[CAST]], %[[BCAST1]], %[[BCAST2]] acq_rel monotonic : !llvm.ptr, i64
 ! CHECK: %[[RES:.*]] = llvm.extractvalue %[[ATOMIC]][1] : !llvm.struct<(i64, i1)> 
+
+attributes(global) subroutine __ldXXi4(b)
+  integer, device :: b(*)
+  integer, device :: x(4)
+  x(1:4) = __ldca(b(i:j))
+  x = __ldcg(b(i:j))
+  x = __ldcs(b(i:j))
+  x(1:4) = __ldlu(b(i:j))
+  x(1:4) = __ldcv(b(i:j))
+end
+
+! CHECK-LABEL: func.func @_QP__ldxxi4
+! CHECK: fir.call @__ldca_i4x4_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<4xi32>>, !fir.ref<!fir.array<?xi32>>) -> ()
+! CHECK: fir.call @__ldcg_i4x4_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<4xi32>>, !fir.ref<!fir.array<?xi32>>) -> ()
+! CHECK: fir.call @__ldcs_i4x4_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<4xi32>>, !fir.ref<!fir.array<?xi32>>) -> ()
+! CHECK: fir.call @__ldlu_i4x4_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<4xi32>>, !fir.ref<!fir.array<?xi32>>) -> ()
+! CHECK: fir.call @__ldcv_i4x4_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<4xi32>>, !fir.ref<!fir.array<?xi32>>) -> ()
+
+attributes(global) subroutine __ldXXi8(b)
+  integer(8), device :: b(*)
+  integer(8), device :: x(2)
+  x(1:2) = __ldca(b(i:j))
+  x = __ldcg(b(i:j))
+  x = __ldcs(b(i:j))
+  x(1:2) = __ldlu(b(i:j))
+  x(1:2) = __ldcv(b(i:j))
+end
+
+! CHECK-LABEL: func.func @_QP__ldxxi8
+! CHECK: fir.call @__ldca_i8x2_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<2xi64>>, !fir.ref<!fir.array<?xi64>>) -> ()
+! CHECK: fir.call @__ldcg_i8x2_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<2xi64>>, !fir.ref<!fir.array<?xi64>>) -> ()
+! CHECK: fir.call @__ldcs_i8x2_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<2xi64>>, !fir.ref<!fir.array<?xi64>>) -> ()
+! CHECK: fir.call @__ldlu_i8x2_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<2xi64>>, !fir.ref<!fir.array<?xi64>>) -> ()
+! CHECK: fir.call @__ldcv_i8x2_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<2xi64>>, !fir.ref<!fir.array<?xi64>>) -> ()
+
+attributes(global) subroutine __ldXXr4(b)
+  real, device :: b(*)
+  real, device :: x(4)
+  x(1:4) = __ldca(b(i:j))
+  x = __ldcg(b(i:j))
+  x = __ldcs(b(i:j))
+  x(1:4) = __ldlu(b(i:j))
+  x(1:4) = __ldcv(b(i:j))
+end
+
+! CHECK-LABEL: func.func @_QP__ldxxr4
+! CHECK: fir.call @__ldca_r4x4_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<4xf32>>, !fir.ref<!fir.array<?xf32>>) -> ()
+! CHECK: fir.call @__ldcg_r4x4_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<4xf32>>, !fir.ref<!fir.array<?xf32>>) -> ()
+! CHECK: fir.call @__ldcs_r4x4_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<4xf32>>, !fir.ref<!fir.array<?xf32>>) -> ()
+! CHECK: fir.call @__ldlu_r4x4_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<4xf32>>, !fir.ref<!fir.array<?xf32>>) -> ()
+! CHECK: fir.call @__ldcv_r4x4_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<4xf32>>, !fir.ref<!fir.array<?xf32>>) -> ()
+
+attributes(global) subroutine __ldXXr2(b)
+  real(2), device :: b(*)
+  real(2), device :: x(2)
+  x(1:2) = __ldca(b(i:j))
+  x = __ldcg(b(i:j))
+  x = __ldcs(b(i:j))
+  x(1:2) = __ldlu(b(i:j))
+  x(1:2) = __ldcv(b(i:j))
+end
+
+! CHECK-LABEL: func.func @_QP__ldxxr2
+! CHECK: fir.call @__ldca_r2x2_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<2xf16>>, !fir.ref<!fir.array<?xf16>>) -> ()
+! CHECK: fir.call @__ldcg_r2x2_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<2xf16>>, !fir.ref<!fir.array<?xf16>>) -> ()
+! CHECK: fir.call @__ldcs_r2x2_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<2xf16>>, !fir.ref<!fir.array<?xf16>>) -> ()
+! CHECK: fir.call @__ldlu_r2x2_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<2xf16>>, !fir.ref<!fir.array<?xf16>>) -> ()
+! CHECK: fir.call @__ldcv_r2x2_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<2xf16>>, !fir.ref<!fir.array<?xf16>>) -> ()
+
+attributes(global) subroutine __ldXXr8(b)
+  real(8), device :: b(*)
+  real(8), device :: x(2)
+  x(1:2) = __ldca(b(i:j))
+  x = __ldcg(b(i:j))
+  x = __ldcs(b(i:j))
+  x(1:2) = __ldlu(b(i:j))
+  x(1:2) = __ldcv(b(i:j))
+end
+
+! CHECK-LABEL: func.func @_QP__ldxxr8
+! CHECK: fir.call @__ldca_r8x2_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<2xf64>>, !fir.ref<!fir.array<?xf64>>) -> ()
+! CHECK: fir.call @__ldcg_r8x2_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<2xf64>>, !fir.ref<!fir.array<?xf64>>) -> ()
+! CHECK: fir.call @__ldcs_r8x2_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<2xf64>>, !fir.ref<!fir.array<?xf64>>) -> ()
+! CHECK: fir.call @__ldlu_r8x2_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<2xf64>>, !fir.ref<!fir.array<?xf64>>) -> ()
+! CHECK: fir.call @__ldcv_r8x2_(%{{.*}}, %{{.*}}) fastmath<contract> : (!fir.ref<!fir.array<2xf64>>, !fir.ref<!fir.array<?xf64>>) -> ()
+
+
+! CHECK-DAG: func.func private @__ldca_i4x4_(!fir.ref<!fir.array<4xi32>>, !fir.ref<!fir.array<4xi32>>)
+! CHECK-DAG: func.func private @__ldcg_i4x4_(!fir.ref<!fir.array<4xi32>>, !fir.ref<!fir.array<4xi32>>)
+! CHECK-DAG: func.func private @__ldcs_i4x4_(!fir.ref<!fir.array<4xi32>>, !fir.ref<!fir.array<4xi32>>)
+! CHECK-DAG: func.func private @__ldlu_i4x4_(!fir.ref<!fir.array<4xi32>>, !fir.ref<!fir.array<4xi32>>)
+! CHECK-DAG: func.func private @__ldcv_i4x4_(!fir.ref<!fir.array<4xi32>>, !fir.ref<!fir.array<4xi32>>)
+! CHECK-DAG: func.func private @__ldca_i8x2_(!fir.ref<!fir.array<2xi64>>, !fir.ref<!fir.array<2xi64>>)
+! CHECK-DAG: func.func private @__ldcg_i8x2_(!fir.ref<!fir.array<2xi64>>, !fir.ref<!fir.array<2xi64>>)
+! CHECK-DAG: func.func private @__ldcs_i8x2_(!fir.ref<!fir.array<2xi64>>, !fir.ref<!fir.array<2xi64>>)
+! CHECK-DAG: func.func private @__ldlu_i8x2_(!fir.ref<!fir.array<2xi64>>, !fir.ref<!fir.array<2xi64>>)
+! CHECK-DAG: func.func private @__ldcv_i8x2_(!fir.ref<!fir.array<2xi64>>, !fir.ref<!fir.array<2xi64>>)
+! CHECK-DAG: func.func private @__ldca_r4x4_(!fir.ref<!fir.array<4xf32>>, !fir.ref<!fir.array<4xf32>>)
+! CHECK-DAG: func.func private @__ldcg_r4x4_(!fir.ref<!fir.array<4xf32>>, !fir.ref<!fir.array<4xf32>>)
+! CHECK-DAG: func.func private @__ldcs_r4x4_(!fir.ref<!fir.array<4xf32>>, !fir.ref<!fir.array<4xf32>>)
+! CHECK-DAG: func.func private @__ldlu_r4x4_(!fir.ref<!fir.array<4xf32>>, !fir.ref<!fir.array<4xf32>>)
+! CHECK-DAG: func.func private @__ldcv_r4x4_(!fir.ref<!fir.array<4xf32>>, !fir.ref<!fir.array<4xf32>>)
+! CHECK-DAG: func.func private @__ldca_r2x2_(!fir.ref<!fir.array<2xf16>>, !fir.ref<!fir.array<2xf16>>)
+! CHECK-DAG: func.func private @__ldcg_r2x2_(!fir.ref<!fir.array<2xf16>>, !fir.ref<!fir.array<2xf16>>)
+! CHECK-DAG: func.func private @__ldcs_r2x2_(!fir.ref<!fir.array<2xf16>>, !fir.ref<!fir.array<2xf16>>)
+! CHECK-DAG: func.func private @__ldlu_r2x2_(!fir.ref<!fir.array<2xf16>>, !fir.ref<!fir.array<2xf16>>)
+! CHECK-DAG: func.func private @__ldcv_r2x2_(!fir.ref<!fir.array<2xf16>>, !fir.ref<!fir.array<2xf16>>)
+! CHECK-DAG: func.func private @__ldca_r8x2_(!fir.ref<!fir.array<2xf64>>, !fir.ref<!fir.array<2xf64>>)
+! CHECK-DAG: func.func private @__ldcg_r8x2_(!fir.ref<!fir.array<2xf64>>, !fir.ref<!fir.array<2xf64>>)
+! CHECK-DAG: func.func private @__ldcs_r8x2_(!fir.ref<!fir.array<2xf64>>, !fir.ref<!fir.array<2xf64>>)
+! CHECK-DAG: func.func private @__ldlu_r8x2_(!fir.ref<!fir.array<2xf64>>, !fir.ref<!fir.array<2xf64>>)
+! CHECK-DAG: func.func private @__ldcv_r8x2_(!fir.ref<!fir.array<2xf64>>, !fir.ref<!fir.array<2xf64>>)
diff --git a/flang/test/Lower/CUDA/cuda-pointer.cuf b/flang/test/Lower/CUDA/cuda-pointer.cuf
index 2a9dbe54c2922..e9614751673e0 100644
--- a/flang/test/Lower/CUDA/cuda-pointer.cuf
+++ b/flang/test/Lower/CUDA/cuda-pointer.cuf
@@ -2,10 +2,31 @@
 
 ! Test lowering of CUDA pointers.
 
+module mod1
+
+integer, device, pointer :: x(:)
+
+contains
+
 subroutine allocate_pointer
   real, device, pointer :: pr(:)
   allocate(pr(10))
 end 
 
-! CHECK-LABEL: func.func @_QPallocate_pointer()
+! CHECK-LABEL: func.func @_QMmod1Pallocate_pointer()
 ! CHECK-COUNT-2: fir.embox %{{.*}} {allocator_idx = 2 : i32} : (!fir.ptr<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.ptr<!fir.array<?xf32>>>
+
+subroutine c_f_pointer_sync
+  use iso_c_binding
+  use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
+  type(c_devptr) :: cd1
+  integer, parameter :: N = 2000
+  call c_f_pointer(cd1, x, (/ 2000 /))
+end 
+
+! CHECK-LABEL: func.func @_QMmod1Pc_f_pointer_sync()
+! CHECK: %[[ADDR_X:.*]] = fir.address_of(@_QMmod1Ex) : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+! CHECK: %[[CONV:.*]] = fir.convert %[[ADDR_X]] : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> !fir.llvm_ptr<i8>
+! CHECK: fir.call @_FortranACUFSyncGlobalDescriptor(%[[CONV]], %{{.*}}, %{{.*}}) fastmath<contract> : (!fir.llvm_ptr<i8>, !fir.ref<i8>, i32) -> ()
+
+end module
diff --git a/flang/test/Lower/HLFIR/structure-constructor.f90 b/flang/test/Lower/HLFIR/structure-constructor.f90
index d881508e7d3ae..628d3ee0dd771 100644
--- a/flang/test/Lower/HLFIR/structure-constructor.f90
+++ b/flang/test/Lower/HLFIR/structure-constructor.f90
@@ -99,8 +99,7 @@ end subroutine test3
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}> {bindc_name = "res", uniq_name = "_QFtest3Eres"}
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFtest3Eres"} : (!fir.ref<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>, !fir.ref<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>)
 ! CHECK:           %[[ADDR:.*]] = fir.address_of(@_QQ_QMtypesTt3.DerivedInit) : !fir.ref<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>
-! CHECK:           %[[LOAD:.*]] = fir.load %[[ADDR]] : !fir.ref<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>
-! CHECK:           fir.store %[[LOAD]] to %[[VAL_3]]#1 : !fir.ref<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>
+! CHECK:           fir.copy %[[ADDR]] to %[[VAL_3]]#1 no_overlap : !fir.ref<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>, !fir.ref<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>
 ! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest3Ex"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>)
 ! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>, !fir.ref<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>)
 ! CHECK:           %[[VAL_12:.*]] = fir.embox %[[VAL_11]]#0 : (!fir.ref<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>) -> !fir.box<!fir.type<_QMtypesTt3{r:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>
@@ -132,8 +131,7 @@ end subroutine test4
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}> {bindc_name = "res", uniq_name = "_QFtest4Eres"}
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFtest4Eres"} : (!fir.ref<!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>, !fir.ref<!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>)
 ! CHECK:           %[[ADDR:.*]] = fir.address_of(@_QQ_QMtypesTt4.DerivedInit) : !fir.ref<!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>
-! CHECK:           %[[LOAD:.*]] = fir.load %[[ADDR]] : !fir.ref<!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>
-! CHECK:           fir.store %[[LOAD]] to %[[VAL_3]]#1 : !fir.ref<!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>
+! CHECK:           fir.copy %[[ADDR]] to %[[VAL_3]]#1 no_overlap : !fir.ref<!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>, !fir.ref<!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 2 : index
 ! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_0]] typeparams %[[VAL_10]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest4Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>>, index, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>>)
 ! CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>, !fir.ref<!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>)
@@ -173,8 +171,7 @@ end subroutine test5
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}> {bindc_name = "res", uniq_name = "_QFtest5Eres"}
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFtest5Eres"} : (!fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>, !fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>)
 ! CHECK:           %[[ADDR:.*]] = fir.address_of(@_QQ_QMtypesTt5.DerivedInit) : !fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>
-! CHECK:           %[[LOAD:.*]] = fir.load %[[ADDR]] : !fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>
-! CHECK:           fir.store %[[LOAD]] to %[[VAL_3]]#1 : !fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>
+! CHECK:           fir.copy %[[ADDR]] to %[[VAL_3]]#1 no_overlap : !fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>, !fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>
 ! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest5Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>)
 ! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>, !fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>)
 ! CHECK:           %[[VAL_12:.*]] = fir.embox %[[VAL_11]]#0 : (!fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>) -> !fir.box<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>
@@ -221,8 +218,7 @@ end subroutine test6
 ! CHECK:           %[[VAL_11:.*]] = fir.alloca !fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}> {bindc_name = "res", uniq_name = "_QFtest6Eres"}
 ! CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFtest6Eres"} : (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>, !fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>)
 ! CHECK:           %[[ADDR:.*]] = fir.address_of(@_QQ_QMtypesTt6.DerivedInit) : !fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>
-! CHECK:           %[[LOAD:.*]] = fir.load %[[ADDR]] : !fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>
-! CHECK:           fir.store %[[LOAD]] to %[[VAL_12]]#1 : !fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>
+! CHECK:           fir.copy %[[ADDR]] to %[[VAL_12]]#1 no_overlap : !fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>, !fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>
 ! CHECK:           %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest6Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>, !fir.dscope) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>)
 ! CHECK:           %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>, !fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>)
 ! CHECK:           %[[VAL_21:.*]] = fir.embox %[[VAL_20]]#0 : (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> !fir.box<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>
@@ -307,8 +303,7 @@ end subroutine test7
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}> {bindc_name = "x", uniq_name = "_QFtest7Ex"}
 ! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFtest7Ex"} : (!fir.ref<!fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>, !fir.ref<!fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>)
 ! CHECK:           %[[ADDR:.*]] = fir.address_of(@_QQ_QMtypesTt7.DerivedInit) : !fir.ref<!fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>
-! CHECK:           %[[LOAD:.*]] = fir.load %[[ADDR]] : !fir.ref<!fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>
-! CHECK:           fir.store %[[LOAD]] to %[[VAL_4]]#1 : !fir.ref<!fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>
+! CHECK:           fir.copy %[[ADDR]] to %[[VAL_4]]#1 no_overlap : !fir.ref<!fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>, !fir.ref<!fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>
 ! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>, !fir.ref<!fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>)
 ! CHECK:           %[[VAL_12:.*]] = fir.embox %[[VAL_11]]#0 : (!fir.ref<!fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>) -> !fir.box<!fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>
 ! CHECK:           %[[VAL_13:.*]] = fir.address_of(@_QQclX{{.*}}) : !fir.ref<!fir.char<1,{{[0-9]*}}>>
@@ -336,8 +331,7 @@ end subroutine test8
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.type<_QMtypesTt8{c:!fir.box<!fir.heap<!fir.char<1,11>>>}> {bindc_name = "res", uniq_name = "_QFtest8Eres"}
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest8Eres"} : (!fir.ref<!fir.type<_QMtypesTt8{c:!fir.box<!fir.heap<!fir.char<1,11>>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt8{c:!fir.box<!fir.heap<!fir.char<1,11>>>}>>, !fir.ref<!fir.type<_QMtypesTt8{c:!fir.box<!fir.heap<!fir.char<1,11>>>}>>)
 ! CHECK:           %[[ADDR:.*]] = fir.address_of(@_QQ_QMtypesTt8.DerivedInit) : !fir.ref<!fir.type<_QMtypesTt8{c:!fir.box<!fir.heap<!fir.char<1,11>>>}>>
-! CHECK:           %[[LOAD:.*]] = fir.load %[[ADDR]] : !fir.ref<!fir.type<_QMtypesTt8{c:!fir.box<!fir.heap<!fir.char<1,11>>>}>>
-! CHECK:           fir.store %[[LOAD]] to %[[VAL_2]]#1 : !fir.ref<!fir.type<_QMtypesTt8{c:!fir.box<!fir.heap<!fir.char<1,11>>>}>>
+! CHECK:           fir.copy %[[ADDR]] to %[[VAL_2]]#1 no_overlap : !fir.ref<!fir.type<_QMtypesTt8{c:!fir.box<!fir.heap<!fir.char<1,11>>>}>>, !fir.ref<!fir.type<_QMtypesTt8{c:!fir.box<!fir.heap<!fir.char<1,11>>>}>>
 ! CHECK:           %[[VAL_9:.*]] = fir.alloca !fir.box<!fir.heap<!fir.char<1,12>>> {bindc_name = "x", uniq_name = "_QFtest8Ex"}
 ! CHECK:           %[[VAL_10:.*]] = arith.constant 12 : index
 ! CHECK:           %[[VAL_11:.*]] = fir.zero_bits !fir.heap<!fir.char<1,12>>
@@ -382,8 +376,7 @@ end subroutine test9
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.type<_QMtypesTt8{c:!fir.box<!fir.heap<!fir.char<1,11>>>}> {bindc_name = "res", uniq_name = "_QFtest9Eres"}
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest9Eres"} : (!fir.ref<!fir.type<_QMtypesTt8{c:!fir.box<!fir.heap<!fir.char<1,11>>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt8{c:!fir.box<!fir.heap<!fir.char<1,11>>>}>>, !fir.ref<!fir.type<_QMtypesTt8{c:!fir.box<!fir.heap<!fir.char<1,11>>>}>>)
 ! CHECK:           %[[ADDR:.*]] = fir.address_of(@_QQ_QMtypesTt8.DerivedInit) : !fir.ref<!fir.type<_QMtypesTt8{c:!fir.box<!fir.heap<!fir.char<1,11>>>}>>
-! CHECK:           %[[LOAD:.*]] = fir.load %[[ADDR]] : !fir.ref<!fir.type<_QMtypesTt8{c:!fir.box<!fir.heap<!fir.char<1,11>>>}>>
-! CHECK:           fir.store %[[LOAD]] to %[[VAL_2]]#1 : !fir.ref<!fir.type<_QMtypesTt8{c:!fir.box<!fir.heap<!fir.char<1,11>>>}>>
+! CHECK:           fir.copy %[[ADDR]] to %[[VAL_2]]#1 no_overlap : !fir.ref<!fir.type<_QMtypesTt8{c:!fir.box<!fir.heap<!fir.char<1,11>>>}>>, !fir.ref<!fir.type<_QMtypesTt8{c:!fir.box<!fir.heap<!fir.char<1,11>>>}>>
 ! CHECK:           %[[VAL_9:.*]] = arith.constant 12 : index
 ! CHECK:           %[[VAL_10:.*]] = fir.alloca !fir.char<1,12> {bindc_name = "x", uniq_name = "_QFtest9Ex"}
 ! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] typeparams %[[VAL_9]] {uniq_name = "_QFtest9Ex"} : (!fir.ref<!fir.char<1,12>>, index) -> (!fir.ref<!fir.char<1,12>>, !fir.ref<!fir.char<1,12>>)
diff --git a/flang/test/Lower/OpenMP/Todo/from-expectation-modifier.f90 b/flang/test/Lower/OpenMP/Todo/from-expectation-modifier.f90
deleted file mode 100644
index 7be70ba9102e1..0000000000000
--- a/flang/test/Lower/OpenMP/Todo/from-expectation-modifier.f90
+++ /dev/null
@@ -1,8 +0,0 @@
-!RUN: %not_todo_cmd bbc -emit-hlfir -fopenmp -fopenmp-version=52 -o - %s 2>&1 | FileCheck %s
-!RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 -o - %s 2>&1 | FileCheck %s
-
-!CHECK: not yet implemented: PRESENT modifier is not supported yet
-subroutine f00(x)
-  integer :: x
-  !$omp target update from(present: x)
-end
diff --git a/flang/test/Lower/OpenMP/Todo/map-modifiers-close.f90 b/flang/test/Lower/OpenMP/Todo/map-modifiers-close.f90
deleted file mode 100644
index c30f45d83864b..0000000000000
--- a/flang/test/Lower/OpenMP/Todo/map-modifiers-close.f90
+++ /dev/null
@@ -1,10 +0,0 @@
-! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-
-!CHECK: Map type modifiers (other than 'ALWAYS') are not implemented yet
-subroutine f00()
-  integer :: x
-  !$omp target map(close: x)
-  x = x + 1
-  !$omp end target
-end
diff --git a/flang/test/Lower/OpenMP/Todo/map-modifiers-ompxhold.f90 b/flang/test/Lower/OpenMP/Todo/map-modifiers-ompxhold.f90
deleted file mode 100644
index 0b5f2f5ca5e24..0000000000000
--- a/flang/test/Lower/OpenMP/Todo/map-modifiers-ompxhold.f90
+++ /dev/null
@@ -1,11 +0,0 @@
-! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-
-!CHECK: Map type modifiers (other than 'ALWAYS') are not implemented yet
-subroutine f01()
-  integer :: x
-  !$omp target map(ompx_hold: x)
-  x = x + 1
-  !$omp end target
-end
-
diff --git a/flang/test/Lower/OpenMP/Todo/map-modifiers-present.f90 b/flang/test/Lower/OpenMP/Todo/map-modifiers-present.f90
deleted file mode 100644
index 836b3a0a84bec..0000000000000
--- a/flang/test/Lower/OpenMP/Todo/map-modifiers-present.f90
+++ /dev/null
@@ -1,11 +0,0 @@
-! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-
-!CHECK: Map type modifiers (other than 'ALWAYS') are not implemented yet
-subroutine f02()
-  integer :: x
-  !$omp target map(present: x)
-  x = x + 1
-  !$omp end target
-end
-
diff --git a/flang/test/Lower/OpenMP/Todo/taskloop-cancel.f90 b/flang/test/Lower/OpenMP/Todo/taskloop-cancel.f90
new file mode 100644
index 0000000000000..5045c621e4d77
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/taskloop-cancel.f90
@@ -0,0 +1,14 @@
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s -fopenmp-version=50 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s -fopenmp-version=50 2>&1 | FileCheck %s
+
+! CHECK: not yet implemented: Taskloop construct
+subroutine omp_taskloop
+integer :: i
+!$omp parallel
+    !$omp taskloop
+      do i = 1, 10
+      !$omp cancel taskgroup
+      end do
+    !$omp end taskloop
+!$omp end parallel
+end subroutine omp_taskloop
diff --git a/flang/test/Lower/OpenMP/Todo/to-expectation-modifier.f90 b/flang/test/Lower/OpenMP/Todo/to-expectation-modifier.f90
deleted file mode 100644
index f29fb8fffb1eb..0000000000000
--- a/flang/test/Lower/OpenMP/Todo/to-expectation-modifier.f90
+++ /dev/null
@@ -1,8 +0,0 @@
-!RUN: %not_todo_cmd bbc -emit-hlfir -fopenmp -fopenmp-version=52 -o - %s 2>&1 | FileCheck %s
-!RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 -o - %s 2>&1 | FileCheck %s
-
-!CHECK: not yet implemented: PRESENT modifier is not supported yet
-subroutine f00(x)
-  integer :: x
-  !$omp target update to(present: x)
-end
diff --git a/flang/test/Lower/OpenMP/distribute-parallel-do-simd.f90 b/flang/test/Lower/OpenMP/distribute-parallel-do-simd.f90
index bea7f037cecf3..142bc02ae8c1d 100644
--- a/flang/test/Lower/OpenMP/distribute-parallel-do-simd.f90
+++ b/flang/test/Lower/OpenMP/distribute-parallel-do-simd.f90
@@ -8,10 +8,10 @@
 subroutine distribute_parallel_do_simd_num_threads()
   !$omp teams
 
-  ! CHECK:      omp.parallel num_threads({{.*}}) private({{.*}}) {
+  ! CHECK:      omp.parallel num_threads({{.*}}) {
   ! CHECK:      omp.distribute {
   ! CHECK-NEXT: omp.wsloop {
-  ! CHECK-NEXT: omp.simd {
+  ! CHECK-NEXT: omp.simd private({{.*}}) {
   ! CHECK-NEXT: omp.loop_nest
   !$omp distribute parallel do simd num_threads(10)
   do index_ = 1, 10
@@ -25,10 +25,10 @@ end subroutine distribute_parallel_do_simd_num_threads
 subroutine distribute_parallel_do_simd_dist_schedule()
   !$omp teams
 
-  ! CHECK:      omp.parallel private({{.*}}) {
+  ! CHECK:      omp.parallel  {
   ! CHECK:      omp.distribute dist_schedule_static dist_schedule_chunk_size({{.*}}) {
   ! CHECK-NEXT: omp.wsloop {
-  ! CHECK-NEXT: omp.simd {
+  ! CHECK-NEXT: omp.simd private({{.*}}) {
   ! CHECK-NEXT: omp.loop_nest
   !$omp distribute parallel do simd dist_schedule(static, 4)
   do index_ = 1, 10
@@ -42,10 +42,10 @@ end subroutine distribute_parallel_do_simd_dist_schedule
 subroutine distribute_parallel_do_simd_schedule()
   !$omp teams
 
-  ! CHECK:      omp.parallel private({{.*}}) {
+  ! CHECK:      omp.parallel {
   ! CHECK:      omp.distribute {
   ! CHECK-NEXT: omp.wsloop schedule(static = {{.*}}) {
-  ! CHECK-NEXT: omp.simd {
+  ! CHECK-NEXT: omp.simd private({{.*}}) {
   ! CHECK-NEXT: omp.loop_nest
   !$omp distribute parallel do simd schedule(static, 4)
   do index_ = 1, 10
@@ -59,10 +59,10 @@ end subroutine distribute_parallel_do_simd_schedule
 subroutine distribute_parallel_do_simd_simdlen()
   !$omp teams
 
-  ! CHECK:      omp.parallel private({{.*}}) {
+  ! CHECK:      omp.parallel {
   ! CHECK:      omp.distribute {
   ! CHECK-NEXT: omp.wsloop {
-  ! CHECK-NEXT: omp.simd simdlen(4) {
+  ! CHECK-NEXT: omp.simd simdlen(4) private({{.*}}) {
   ! CHECK-NEXT: omp.loop_nest
   !$omp distribute parallel do simd simdlen(4)
   do index_ = 1, 10
@@ -83,14 +83,14 @@ subroutine distribute_parallel_do_simd_private()
   ! CHECK: omp.teams {
   !$omp teams
 
-  ! CHECK:      omp.parallel private(@{{.*}} %[[X]]#0 -> %[[X_ARG:[^,]+]],
-  ! CHECK-SAME:                      @{{.*}} %[[INDEX]]#0 -> %[[INDEX_ARG:.*]] : !fir.ref<i64>, !fir.ref<i32>) {
-  ! CHECK:      %[[X_PRIV:.*]]:2 = hlfir.declare %[[X_ARG]]
-  ! CHECK:      %[[INDEX_PRIV:.*]]:2 = hlfir.declare %[[INDEX_ARG]]
+  ! CHECK:      omp.parallel {
   ! CHECK:      omp.distribute {
   ! CHECK-NEXT: omp.wsloop {
-  ! CHECK-NEXT: omp.simd {
+  ! CHECK-NEXT: omp.simd private(@{{.*}} %[[X]]#0 -> %[[X_ARG:[^,]+]],
+  ! CHECK-SAME:                  @{{.*}} %[[INDEX]]#0 -> %[[INDEX_ARG:.*]] : !fir.ref<i64>, !fir.ref<i32>) {
   ! CHECK-NEXT: omp.loop_nest
+  ! CHECK:      %[[X_PRIV:.*]]:2 = hlfir.declare %[[X_ARG]]
+  ! CHECK:      %[[INDEX_PRIV:.*]]:2 = hlfir.declare %[[INDEX_ARG]]
   !$omp distribute parallel do simd private(x)
   do index_ = 1, 10
   end do
@@ -98,3 +98,48 @@ subroutine distribute_parallel_do_simd_private()
 
   !$omp end teams
 end subroutine distribute_parallel_do_simd_private
+
+! CHECK-LABEL:   func.func @_QPlastprivate_cond_in_composite_construct
+subroutine lastprivate_cond_in_composite_construct(x_min, x_max, y_min, y_max)
+implicit none
+integer :: x_min,x_max,y_min,y_max
+integer :: i,j
+
+! CHECK:           omp.target {{.*}} {
+! CHECK:             %[[X_MAX_MAPPED:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "{{.*}}x_max"}
+! CHECK:             omp.teams {
+! CHECK:               omp.parallel {
+! CHECK:                 omp.distribute {
+! CHECK:                   omp.wsloop {
+! CHECK:                     omp.simd private({{.*}}) {
+! CHECK:                       omp.loop_nest (%[[I_IV:.*]], %[[J_IV:.*]]) : i32 = ({{.*}}) to ({{.*}}) inclusive step ({{.*}}) {
+! CHECK:                         %[[Y_MAX_PRIV:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "{{.*}}y_max"}
+
+! CHECK:                         %[[I_UB:.*]] = fir.load %[[X_MAX_MAPPED]]#0 : !fir.ref<i32>
+! CHECK:                         %[[I_STEP:.*]] = arith.constant 1 : i32
+! CHECK:                         %[[J_UB:.*]] = fir.load %[[Y_MAX_PRIV]]#0 : !fir.ref<i32>
+! CHECK:                         %[[J_STEP:.*]] = arith.constant 1 : i32
+
+! CHECK:                         %[[VAL_55:.*]] = arith.addi %[[I_IV]], %[[I_STEP]] : i32
+! CHECK:                         %[[VAL_56:.*]] = arith.constant 0 : i32
+! CHECK:                         %[[VAL_57:.*]] = arith.cmpi slt, %[[I_STEP]], %[[VAL_56]] : i32
+! CHECK:                         %[[VAL_58:.*]] = arith.cmpi slt, %[[VAL_55]], %[[I_UB]] : i32
+! CHECK:                         %[[VAL_59:.*]] = arith.cmpi sgt, %[[VAL_55]], %[[I_UB]] : i32
+! CHECK:                         %[[VAL_60:.*]] = arith.select %[[VAL_57]], %[[VAL_58]], %[[VAL_59]] : i1
+
+! CHECK:                         %[[VAL_61:.*]] = arith.addi %[[J_IV]], %[[J_STEP]] : i32
+! CHECK:                         %[[VAL_62:.*]] = arith.constant 0 : i32
+! CHECK:                         %[[VAL_63:.*]] = arith.cmpi slt, %[[J_STEP]], %[[VAL_62]] : i32
+! CHECK:                         %[[VAL_64:.*]] = arith.cmpi slt, %[[VAL_61]], %[[J_UB]] : i32
+! CHECK:                         %[[VAL_65:.*]] = arith.cmpi sgt, %[[VAL_61]], %[[J_UB]] : i32
+! CHECK:                         %[[VAL_66:.*]] = arith.select %[[VAL_63]], %[[VAL_64]], %[[VAL_65]] : i1
+
+! CHECK:                         %[[LASTPRIV_CMP:.*]] = arith.andi %[[VAL_60]], %[[VAL_66]] : i1
+! CHECK:                         fir.if %[[LASTPRIV_CMP]] {
+
+!$omp target teams distribute parallel do simd collapse(2) private(y_max)
+  do i=x_min,x_max
+    do j=y_min,y_max
+    enddo
+  enddo
+end subroutine
diff --git a/flang/test/Lower/OpenMP/has_device_addr-mapinfo.f90 b/flang/test/Lower/OpenMP/has_device_addr-mapinfo.f90
new file mode 100644
index 0000000000000..8d8c0433758ee
--- /dev/null
+++ b/flang/test/Lower/OpenMP/has_device_addr-mapinfo.f90
@@ -0,0 +1,23 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=51 %s -mmlir -mlir-print-op-generic -o - | FileCheck %s
+!RUN: bbc -emit-hlfir -fopenmp -fopenmp-version=51 %s -mlir-print-op-generic -o - | FileCheck %s
+
+! Check that we don't generate member information for the descriptor of `a`
+! on entry to the target region.
+
+integer function s(a)
+  integer :: a(:)
+  integer :: t
+  !$omp target data map(to:a) use_device_addr(a)
+  !$omp target map(from:t) has_device_addr(a)
+  t = size(a, 1)
+  !$omp end target
+  !$omp end target data
+  s = t
+end
+
+! Check that the map.info for `a` only takes a single parameter.
+
+!CHECK-DAG: %[[MAP_A:[0-9]+]] = "omp.map.info"(%[[STORAGE_A:[0-9#]+]]) <{map_capture_type = #omp<variable_capture_kind(ByRef)>, map_type = 517 : ui64, name = "a", operandSegmentSizes = array<i32: 1, 0, 0, 0>, partial_map = false, var_type = !fir.box<!fir.array<?xi32>>}> : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> !fir.ref<!fir.array<?xi32>>
+!CHECK-DAG: %[[MAP_T:[0-9]+]] = "omp.map.info"(%[[STORAGE_T:[0-9#]+]]) <{map_capture_type = #omp<variable_capture_kind(ByRef)>, map_type = 2 : ui64, name = "t", operandSegmentSizes = array<i32: 1, 0, 0, 0>, partial_map = false, var_type = i32}> : (!fir.ref<i32>) -> !fir.ref<i32>
+
+!CHECK: "omp.target"(%[[MAP_A]], %[[MAP_T]])
diff --git a/flang/test/Lower/OpenMP/lastprivate-iv.f90 b/flang/test/Lower/OpenMP/lastprivate-iv.f90
index 7918b47a400b9..e90c7e253ac06 100644
--- a/flang/test/Lower/OpenMP/lastprivate-iv.f90
+++ b/flang/test/Lower/OpenMP/lastprivate-iv.f90
@@ -14,11 +14,13 @@
 !CHECK-NEXT:   omp.loop_nest (%[[IV:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) {
 !CHECK:          %[[I:.*]]:2 = hlfir.declare %[[I_MEM]] {uniq_name = "_QFlastprivate_iv_incEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:          hlfir.assign %[[IV]] to %[[I]]#1 : i32, !fir.ref<i32>
-!CHECK:          %[[V:.*]] = arith.addi %[[IV]], %[[STEP]] : i32
+!CHECK:          %[[UB_2:.*]] = arith.constant 10 : i32
+!CHECK:          %[[STEP_2:.*]]  = arith.constant 3 : i32
+!CHECK:          %[[V:.*]] = arith.addi %[[IV]], %[[STEP_2]] : i32
 !CHECK:          %[[C0:.*]] = arith.constant 0 : i32
-!CHECK:          %[[STEP_NEG:.*]] = arith.cmpi slt, %[[STEP]], %[[C0]] : i32
-!CHECK:          %[[V_LT:.*]] = arith.cmpi slt, %[[V]], %[[UB]] : i32
-!CHECK:          %[[V_GT:.*]] = arith.cmpi sgt, %[[V]], %[[UB]] : i32
+!CHECK:          %[[STEP_NEG:.*]] = arith.cmpi slt, %[[STEP_2]], %[[C0]] : i32
+!CHECK:          %[[V_LT:.*]] = arith.cmpi slt, %[[V]], %[[UB_2]] : i32
+!CHECK:          %[[V_GT:.*]] = arith.cmpi sgt, %[[V]], %[[UB_2]] : i32
 !CHECK:          %[[CMP:.*]] = arith.select %[[STEP_NEG]], %[[V_LT]], %[[V_GT]] : i1
 !CHECK:          fir.if %[[CMP]] {
 !CHECK:            hlfir.assign %[[V]] to %[[I]]#1 : i32, !fir.ref<i32>
@@ -48,11 +50,13 @@ subroutine lastprivate_iv_inc()
 !CHECK-NEXT:   omp.loop_nest (%[[IV:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]]) {
 !CHECK:          %[[I:.*]]:2 = hlfir.declare %[[I_MEM]] {uniq_name = "_QFlastprivate_iv_decEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:          hlfir.assign %[[IV]] to %[[I]]#1 : i32, !fir.ref<i32>
-!CHECK:          %[[V:.*]] = arith.addi %[[IV]], %[[STEP]] : i32
+!CHECK:          %[[UB_2:.*]] = arith.constant 1 : i32
+!CHECK:          %[[STEP_2:.*]]  = arith.constant -3 : i32
+!CHECK:          %[[V:.*]] = arith.addi %[[IV]], %[[STEP_2]] : i32
 !CHECK:          %[[C0:.*]] = arith.constant 0 : i32
-!CHECK:          %[[STEP_NEG:.*]] = arith.cmpi slt, %[[STEP]], %[[C0]] : i32
-!CHECK:          %[[V_LT:.*]] = arith.cmpi slt, %[[V]], %[[UB]] : i32
-!CHECK:          %[[V_GT:.*]] = arith.cmpi sgt, %[[V]], %[[UB]] : i32
+!CHECK:          %[[STEP_NEG:.*]] = arith.cmpi slt, %[[STEP_2]], %[[C0]] : i32
+!CHECK:          %[[V_LT:.*]] = arith.cmpi slt, %[[V]], %[[UB_2]] : i32
+!CHECK:          %[[V_GT:.*]] = arith.cmpi sgt, %[[V]], %[[UB_2]] : i32
 !CHECK:          %[[CMP:.*]] = arith.select %[[STEP_NEG]], %[[V_LT]], %[[V_GT]] : i1
 !CHECK:          fir.if %[[CMP]] {
 !CHECK:            hlfir.assign %[[V]] to %[[I]]#1 : i32, !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/lastprivate-simd.f90 b/flang/test/Lower/OpenMP/lastprivate-simd.f90
index df42b35637de7..c542f1f99dc25 100644
--- a/flang/test/Lower/OpenMP/lastprivate-simd.f90
+++ b/flang/test/Lower/OpenMP/lastprivate-simd.f90
@@ -38,7 +38,7 @@ subroutine simd_ivs
 ! CHECK:         %[[IDO2_PRIV_DECL:.*]]:2 = hlfir.declare %[[IDO2_PRIV_ARG]] {uniq_name = "{{.*}}Eido2"}
 ! CHECK:         %[[IDO3_PRIV_DECL:.*]]:2 = hlfir.declare %[[IDO3_PRIV_ARG]] {uniq_name = "{{.*}}Eido3"}
 
-! CHECK:         fir.if %33 {
+! CHECK:         fir.if %{{.*}} {
 ! CHECK:           hlfir.assign %{{.*}} to %[[IDO1_PRIV_DECL]]#1
 ! CHECK:           hlfir.assign %{{.*}} to %[[IDO2_PRIV_DECL]]#1
 ! CHECK:           hlfir.assign %{{.*}} to %[[IDO3_PRIV_DECL]]#1
diff --git a/flang/test/Lower/OpenMP/map-modifiers.f90 b/flang/test/Lower/OpenMP/map-modifiers.f90
new file mode 100644
index 0000000000000..64d7869cbb836
--- /dev/null
+++ b/flang/test/Lower/OpenMP/map-modifiers.f90
@@ -0,0 +1,32 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=51 %s -o - | FileCheck %s
+
+subroutine map_present_target_data
+    integer :: x
+!CHECK: %[[MAP:.*]] = omp.map.info {{.*}} map_clauses(present, to) {{.*}} {name = "x"}
+!CHECK: omp.target_data map_entries(%[[MAP]] : {{.*}}) {
+!$omp target data map(present, to: x)
+!$omp end target data
+end subroutine
+
+subroutine map_present_update
+    integer :: x
+!CHECK: %[[MAP:.*]] = omp.map.info {{.*}} map_clauses(present, to) {{.*}} {name = "x"}
+!CHECK: omp.target_update map_entries(%[[MAP]] : {{.*}})
+!$omp target update to(present: x)
+end subroutine
+
+subroutine map_close
+    integer :: x
+!CHECK: %[[MAP:.*]] = omp.map.info {{.*}} map_clauses(close, tofrom) {{.*}} {name = "x"}
+!CHECK: omp.target_data map_entries(%[[MAP]] : {{.*}}) {
+!$omp target data map(close, tofrom: x)
+!$omp end target data
+end subroutine
+
+subroutine map_ompx_hold
+    integer :: x
+!CHECK: %[[MAP:.*]] = omp.map.info {{.*}} map_clauses(ompx_hold, tofrom) {{.*}} {name = "x"}
+!CHECK: omp.target_data map_entries(%[[MAP]] : {{.*}}) {
+!$omp target data map(ompx_hold, tofrom: x)
+!$omp end target data
+end subroutine
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-mixed.f90 b/flang/test/Lower/OpenMP/parallel-reduction-mixed.f90
index be25169e7d83e..f769fd3a278ba 100644
--- a/flang/test/Lower/OpenMP/parallel-reduction-mixed.f90
+++ b/flang/test/Lower/OpenMP/parallel-reduction-mixed.f90
@@ -36,7 +36,7 @@ end subroutine proc
 
 !CHECK: [[MALLOC_BB]]:
 !CHECK-NOT: omp.par.{{.*}}:
-!CHECK: call ptr @malloc(i64 80)
+!CHECK: call ptr @malloc(i{{(32)|(64)}} 80)
 
 !CHECK: %[[RED_ARR_0:.*]] = getelementptr inbounds [2 x ptr], ptr %red.array, i64 0, i64 0
 !CHECK: store ptr %[[F_priv]], ptr %[[RED_ARR_0:.*]]
diff --git a/flang/test/Lower/OpenMP/parallel-wsloop-lastpriv.f90 b/flang/test/Lower/OpenMP/parallel-wsloop-lastpriv.f90
index 72482fc184861..14b83ce60a674 100644
--- a/flang/test/Lower/OpenMP/parallel-wsloop-lastpriv.f90
+++ b/flang/test/Lower/OpenMP/parallel-wsloop-lastpriv.f90
@@ -23,11 +23,13 @@ subroutine omp_do_lastprivate(a)
 
   ! CHECK-NEXT: hlfir.assign %[[ARG1]] to %[[I_PVT_DECL]]#1 : i32, !fir.ref<i32>
   ! CHECK-NEXT: fir.call @_QPfoo(%[[I_PVT_DECL]]#1, %[[A_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
-  ! CHECK:      %[[NEXT_ARG1:.*]] = arith.addi %[[ARG1]], %[[STEP]] : i32
+  ! CHECK:      %[[UB_2:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK:      %[[STEP_2:.*]] = arith.constant 1 : i32
+  ! CHECK:      %[[NEXT_ARG1:.*]] = arith.addi %[[ARG1]], %[[STEP_2]] : i32
   ! CHECK:      %[[ZERO:.*]] = arith.constant 0 : i32
-  ! CHECK:      %[[STEP_DIR:.*]] = arith.cmpi slt, %[[STEP]], %[[ZERO]] : i32
-  ! CHECK:      %[[LT_UB:.*]] = arith.cmpi slt, %[[NEXT_ARG1]], %[[UB]] : i32
-  ! CHECK:      %[[GT_UB:.*]] = arith.cmpi sgt, %[[NEXT_ARG1]], %[[UB]] : i32
+  ! CHECK:      %[[STEP_DIR:.*]] = arith.cmpi slt, %[[STEP_2]], %[[ZERO]] : i32
+  ! CHECK:      %[[LT_UB:.*]] = arith.cmpi slt, %[[NEXT_ARG1]], %[[UB_2]] : i32
+  ! CHECK:      %[[GT_UB:.*]] = arith.cmpi sgt, %[[NEXT_ARG1]], %[[UB_2]] : i32
   ! CHECK:      %[[SEL:.*]] = arith.select %[[STEP_DIR]], %[[LT_UB]], %[[GT_UB]] : i1
   ! CHECK:      fir.if %[[SEL]] {
   ! CHECK:        hlfir.assign %[[NEXT_ARG1]] to %[[I_PVT_DECL]]#1 : i32, !fir.ref<i32>
@@ -67,11 +69,13 @@ subroutine omp_do_lastprivate2(a, n)
 
   ! CHECK: hlfir.assign %[[ARG2]] to %[[I_PVT_DECL]]#1 : i32, !fir.ref<i32>
   ! CHECK: fir.call @_QPfoo(%[[I_PVT_DECL]]#1, %[[A_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
-  ! CHECK: %[[NEXT_ARG2:.*]] = arith.addi %[[ARG2]], %[[STEP]] : i32
+  ! CHECK: %[[UB_2:.*]] = fir.load %[[N_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK: %[[STEP_2:.*]] = arith.constant 1 : i32
+  ! CHECK: %[[NEXT_ARG2:.*]] = arith.addi %[[ARG2]], %[[STEP_2]] : i32
   ! CHECK: %[[ZERO:.*]] = arith.constant 0 : i32
-  ! CHECK: %[[STEP_DIR:.*]] = arith.cmpi slt, %[[STEP]], %[[ZERO]] : i32
-  ! CHECK: %[[LT_UB:.*]] = arith.cmpi slt, %[[NEXT_ARG2]], %[[UB]] : i32
-  ! CHECK: %[[GT_UB:.*]] = arith.cmpi sgt, %[[NEXT_ARG2]], %[[UB]] : i32
+  ! CHECK: %[[STEP_DIR:.*]] = arith.cmpi slt, %[[STEP_2]], %[[ZERO]] : i32
+  ! CHECK: %[[LT_UB:.*]] = arith.cmpi slt, %[[NEXT_ARG2]], %[[UB_2]] : i32
+  ! CHECK: %[[GT_UB:.*]] = arith.cmpi sgt, %[[NEXT_ARG2]], %[[UB_2]] : i32
   ! CHECK: %[[SEL:.*]] = arith.select %[[STEP_DIR]], %[[LT_UB]], %[[GT_UB]] : i1
   ! CHECK: fir.if %[[SEL]] {
   ! CHECK:   hlfir.assign %[[NEXT_ARG2]] to %[[I_PVT_DECL]]#1 : i32, !fir.ref<i32>
@@ -112,17 +116,24 @@ subroutine omp_do_lastprivate_collapse2(a)
   ! CHECK-NEXT: hlfir.assign %[[ARG1]] to %[[I_PVT_DECL]]#1 : i32, !fir.ref<i32>
   ! CHECK-NEXT: hlfir.assign %[[ARG2]] to %[[J_PVT_DECL]]#1 : i32, !fir.ref<i32>
   ! CHECK-NEXT: fir.call @_QPfoo(%[[I_PVT_DECL]]#1, %[[A_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
-  ! CHECK:      %[[NEXT_ARG1:.*]] = arith.addi %[[ARG1]], %[[STEP1]] : i32
+
+  ! CHECK:      %[[UB1_2:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK:      %[[STEP1_2:.*]] = arith.constant 1 : i32
+  ! CHECK:      %[[UB2_2:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK:      %[[STEP2_2:.*]] = arith.constant 1 : i32
+
+  ! CHECK:      %[[NEXT_ARG1:.*]] = arith.addi %[[ARG1]], %[[STEP1_2]] : i32
   ! CHECK:      %[[ZERO1:.*]] = arith.constant 0 : i32
-  ! CHECK:      %[[STEP1_END:.*]] = arith.cmpi slt, %[[STEP1]], %[[ZERO1]] : i32
-  ! CHECK:      %[[LT_UB1:.*]] = arith.cmpi slt, %[[NEXT_ARG1]], %[[UB1]] : i32
-  ! CHECK:      %[[GT_UB1:.*]] = arith.cmpi sgt, %[[NEXT_ARG1]], %[[UB1]] : i32
+  ! CHECK:      %[[STEP1_END:.*]] = arith.cmpi slt, %[[STEP1_2]], %[[ZERO1]] : i32
+  ! CHECK:      %[[LT_UB1:.*]] = arith.cmpi slt, %[[NEXT_ARG1]], %[[UB1_2]] : i32
+  ! CHECK:      %[[GT_UB1:.*]] = arith.cmpi sgt, %[[NEXT_ARG1]], %[[UB1_2]] : i32
   ! CHECK:      %[[SEL1:.*]] = arith.select %[[STEP1_END]], %[[LT_UB1]], %[[GT_UB1]] : i1
-  ! CHECK:      %[[NEXT_ARG2:.*]] = arith.addi %[[ARG2]], %[[STEP2]] : i32
+
+  ! CHECK:      %[[NEXT_ARG2:.*]] = arith.addi %[[ARG2]], %[[STEP2_2]] : i32
   ! CHECK:      %[[ZERO2:.*]] = arith.constant 0 : i32
-  ! CHECK:      %[[STEP2_END:.*]] = arith.cmpi slt, %[[STEP2]], %[[ZERO2]] : i32
-  ! CHECK:      %[[LT_UB2:.*]] = arith.cmpi slt, %[[NEXT_ARG2]], %[[UB2]] : i32
-  ! CHECK:      %[[GT_UB2:.*]] = arith.cmpi sgt, %[[NEXT_ARG2]], %[[UB2]] : i32
+  ! CHECK:      %[[STEP2_END:.*]] = arith.cmpi slt, %[[STEP2_2]], %[[ZERO2]] : i32
+  ! CHECK:      %[[LT_UB2:.*]] = arith.cmpi slt, %[[NEXT_ARG2]], %[[UB2_2]] : i32
+  ! CHECK:      %[[GT_UB2:.*]] = arith.cmpi sgt, %[[NEXT_ARG2]], %[[UB2_2]] : i32
   ! CHECK:      %[[SEL2:.*]] = arith.select %[[STEP2_END]], %[[LT_UB2]], %[[GT_UB2]] : i1
   ! CHECK:      %[[AND:.*]] = arith.andi %[[SEL1]], %[[SEL2]] : i1
   ! CHECK:      fir.if %[[AND]] {
@@ -173,24 +184,32 @@ subroutine omp_do_lastprivate_collapse3(a)
   ! CHECK-NEXT: hlfir.assign %[[ARG2]] to %[[J_PVT_DECL]]#1 : i32, !fir.ref<i32>
   ! CHECK-NEXT: hlfir.assign %[[ARG3]] to %[[K_PVT_DECL]]#1 : i32, !fir.ref<i32>
   ! CHECK-NEXT: fir.call @_QPfoo(%[[I_PVT_DECL]]#1, %[[A_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
-  ! CHECK:      %[[NEXT_ARG1:.*]] = arith.addi %[[ARG1]], %[[STEP1]] : i32
+
+  ! CHECK:      %[[UB1_2:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK:      %[[STEP1_2:.*]] = arith.constant 1 : i32
+  ! CHECK:      %[[UB2_2:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK:      %[[STEP2_2:.*]] = arith.constant 1 : i32
+  ! CHECK:      %[[UB3_2:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK:      %[[STEP3_2:.*]] = arith.constant 1 : i32
+
+  ! CHECK:      %[[NEXT_ARG1:.*]] = arith.addi %[[ARG1]], %[[STEP1_2]] : i32
   ! CHECK:      %[[ZERO1:.*]] = arith.constant 0 : i32
-  ! CHECK:      %[[STEP1_END:.*]] = arith.cmpi slt, %[[STEP1]], %[[ZERO1]] : i32
-  ! CHECK:      %[[LT_UB1:.*]] = arith.cmpi slt, %[[NEXT_ARG1]], %[[UB1]] : i32
-  ! CHECK:      %[[GT_UB1:.*]] = arith.cmpi sgt, %[[NEXT_ARG1]], %[[UB1]] : i32
+  ! CHECK:      %[[STEP1_END:.*]] = arith.cmpi slt, %[[STEP1_2]], %[[ZERO1]] : i32
+  ! CHECK:      %[[LT_UB1:.*]] = arith.cmpi slt, %[[NEXT_ARG1]], %[[UB1_2]] : i32
+  ! CHECK:      %[[GT_UB1:.*]] = arith.cmpi sgt, %[[NEXT_ARG1]], %[[UB1_2]] : i32
   ! CHECK:      %[[SEL1:.*]] = arith.select %[[STEP1_END]], %[[LT_UB1]], %[[GT_UB1]] : i1
-  ! CHECK:      %[[NEXT_ARG2:.*]] = arith.addi %[[ARG2]], %[[STEP2]] : i32
+  ! CHECK:      %[[NEXT_ARG2:.*]] = arith.addi %[[ARG2]], %[[STEP2_2]] : i32
   ! CHECK:      %[[ZERO2:.*]] = arith.constant 0 : i32
-  ! CHECK:      %[[STEP2_END:.*]] = arith.cmpi slt, %[[STEP2]], %[[ZERO2]] : i32
-  ! CHECK:      %[[LT_UB2:.*]] = arith.cmpi slt, %[[NEXT_ARG2]], %[[UB2]] : i32
-  ! CHECK:      %[[GT_UB2:.*]] = arith.cmpi sgt, %[[NEXT_ARG2]], %[[UB2]] : i32
+  ! CHECK:      %[[STEP2_END:.*]] = arith.cmpi slt, %[[STEP2_2]], %[[ZERO2]] : i32
+  ! CHECK:      %[[LT_UB2:.*]] = arith.cmpi slt, %[[NEXT_ARG2]], %[[UB2_2]] : i32
+  ! CHECK:      %[[GT_UB2:.*]] = arith.cmpi sgt, %[[NEXT_ARG2]], %[[UB2_2]] : i32
   ! CHECK:      %[[SEL2:.*]] = arith.select %[[STEP2_END]], %[[LT_UB2]], %[[GT_UB2]] : i1
   ! CHECK:      %[[AND1:.*]] = arith.andi %[[SEL1]], %[[SEL2]] : i1
-  ! CHECK:      %[[NEXT_ARG3:.*]] = arith.addi %[[ARG3]], %[[STEP3]] : i32
+  ! CHECK:      %[[NEXT_ARG3:.*]] = arith.addi %[[ARG3]], %[[STEP3_2]] : i32
   ! CHECK:      %[[ZERO3:.*]] = arith.constant 0 : i32
-  ! CHECK:      %[[STEP3_END:.*]] = arith.cmpi slt, %[[STEP3]], %[[ZERO3]] : i32
-  ! CHECK:      %[[LT_UB3:.*]] = arith.cmpi slt, %[[NEXT_ARG3]], %[[UB3]] : i32
-  ! CHECK:      %[[GT_UB3:.*]] = arith.cmpi sgt, %[[NEXT_ARG3]], %[[UB3]] : i32
+  ! CHECK:      %[[STEP3_END:.*]] = arith.cmpi slt, %[[STEP3_2]], %[[ZERO3]] : i32
+  ! CHECK:      %[[LT_UB3:.*]] = arith.cmpi slt, %[[NEXT_ARG3]], %[[UB3_2]] : i32
+  ! CHECK:      %[[GT_UB3:.*]] = arith.cmpi sgt, %[[NEXT_ARG3]], %[[UB3_2]] : i32
   ! CHECK:      %[[SEL3:.*]] = arith.select %[[STEP3_END]], %[[LT_UB3]], %[[GT_UB3]] : i1
   ! CHECK:      %[[AND2:.*]] = arith.andi %[[AND1]], %[[SEL3]] : i1
   ! CHECK:      fir.if %[[AND2]] {
diff --git a/flang/test/Lower/OpenMP/private-derived-type.f90 b/flang/test/Lower/OpenMP/private-derived-type.f90
index e7e1e707a52b6..e188672d60bd6 100644
--- a/flang/test/Lower/OpenMP/private-derived-type.f90
+++ b/flang/test/Lower/OpenMP/private-derived-type.f90
@@ -30,8 +30,7 @@ end subroutine s4
 ! CHECK:           %[[VAL_DERIVED_ALLOCA:.*]] = fir.alloca !fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}> {bindc_name = "v", uniq_name = "_QFs4Ev"}
 ! CHECK:           %[[VAL_DERIVED_DECLARE:.*]]:2 = hlfir.declare %[[VAL_DERIVED_ALLOCA]] {{.*}}
 ! CHECK:           %[[ADDR:.*]] = fir.address_of(@_QQ_QFs4Ty3.DerivedInit) : !fir.ref<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>
-! CHECK:           %[[LOAD:.*]] = fir.load %[[ADDR]] {{.*}}
-! CHECK:           fir.store %[[LOAD]] to %[[VAL_DERIVED_DECLARE]]#1 : !fir.ref<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>
+! CHECK:           fir.copy %[[ADDR]] to %[[VAL_DERIVED_DECLARE]]#1 no_overlap : {{.*}}
 ! CHECK:           omp.parallel {
 ! CHECK:             omp.wsloop private(@[[DERIVED_PRIV]] %{{.*}}#0 -> %{{.*}}, @{{.*}} %{{.*}}#0 -> %{{.*}} : !fir.ref<!fir.type<_QFs4Ty3{x:!fir.box<!fir.heap<i32>>}>>, !fir.ref<i32>) {
 ! CHECK:           }
diff --git a/flang/test/Lower/OpenMP/privatization-proc-ptr.f90 b/flang/test/Lower/OpenMP/privatization-proc-ptr.f90
new file mode 100644
index 0000000000000..168580edac878
--- /dev/null
+++ b/flang/test/Lower/OpenMP/privatization-proc-ptr.f90
@@ -0,0 +1,165 @@
+! Test privatization of procedure pointers.
+
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+!RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+program proc_ptr_test
+  implicit none
+
+contains
+
+!CHECK: omp.private {type = private} @_QFFtest_namesEpf2_private_boxproc_z32_args_ref_3x4xf32_ref_z32 : !fir.boxproc<(!fir.ref<!fir.array<3x4xf32>>, !fir.ref<complex<f32>>) -> complex<f32>>
+!CHECK: omp.private {type = private} @_QFFtest_namesEpf1_private_boxproc_f32_args_ref_f32 : !fir.boxproc<(!fir.ref<f32>) -> f32>
+!CHECK: omp.private {type = private} @_QFFtest_namesEpf0_private_boxproc_i32_args : !fir.boxproc<() -> i32>
+!CHECK: omp.private {type = private} @_QFFtest_namesEps2_private_boxproc__args_ref_i32_boxchar_c8xU : !fir.boxproc<(!fir.ref<i32>, !fir.boxchar<1>) -> ()>
+!CHECK: omp.private {type = private} @_QFFtest_namesEps1_private_boxproc__args_ref_i32 : !fir.boxproc<(!fir.ref<i32>) -> ()>
+!CHECK: omp.private {type = private} @_QFFtest_namesEps0_private_boxproc__args : !fir.boxproc<() -> ()>
+
+!CHECK: omp.private {type = private} @_QFFtest_lastprivateEps_private_boxproc__args_ref_i32 : !fir.boxproc<(!fir.ref<i32>) -> ()>
+!CHECK: omp.private {type = private} @_QFFtest_lastprivateEpf_private_boxproc_i32_args_ref_i32 : !fir.boxproc<(!fir.ref<i32>) -> i32>
+
+!CHECK: omp.private {type = firstprivate} @_QFFtest_firstprivateEps_firstprivate_boxproc__args_ref_i32 : !fir.boxproc<(!fir.ref<i32>) -> ()> copy {
+!CHECK:  ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.boxproc<(!fir.ref<i32>) -> ()>>, %[[ARG1:.*]]: !fir.ref<!fir.boxproc<(!fir.ref<i32>) -> ()>>):
+!CHECK:    %[[TEMP:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.boxproc<(!fir.ref<i32>) -> ()>>
+!CHECK:    fir.store %[[TEMP]] to %[[ARG1]] : !fir.ref<!fir.boxproc<(!fir.ref<i32>) -> ()>>
+!CHECK:    omp.yield(%[[ARG1]] : !fir.ref<!fir.boxproc<(!fir.ref<i32>) -> ()>>)
+!CHECK: }
+
+!CHECK: omp.private {type = firstprivate} @_QFFtest_firstprivateEpf_firstprivate_boxproc_i32_args_ref_i32 : !fir.boxproc<(!fir.ref<i32>) -> i32>
+!CHECK: omp.private {type = private} @_QFFtest_privateEps_private_boxproc__args_ref_i32 : !fir.boxproc<(!fir.ref<i32>) -> ()>
+!CHECK: omp.private {type = private} @_QFFtest_privateEpf_private_boxproc_i32_args_ref_i32 : !fir.boxproc<(!fir.ref<i32>) -> i32>
+
+!CHECK-LABEL: func private @_QFPtest_private
+!CHECK:       %[[PF:.*]]:2 = hlfir.declare %{{.*}} {{{.*}}uniq_name = "_QFFtest_privateEpf"}
+!CHECK:       %[[PS:.*]]:2 = hlfir.declare %{{.*}} {{{.*}}uniq_name = "_QFFtest_privateEps"}
+!CHECK:       omp.parallel
+!CHECK:         %[[PRIV_PF:.*]]:2 = hlfir.declare %{{.*}} {{{.*}}uniq_name = "_QFFtest_privateEpf"}
+!CHECK:         %[[PRIV_PS:.*]]:2 = hlfir.declare %{{.*}} {{{.*}}uniq_name = "_QFFtest_privateEps"}
+!CHECK:         %[[PF_VAL:.*]] = fir.load %[[PRIV_PF]]#0
+!CHECK:         %[[PF_BOX:.*]] = fir.box_addr %[[PF_VAL]]
+!CHECK:         fir.call %[[PF_BOX]]({{.*}})
+!CHECK:         %[[PS_VAL:.*]] = fir.load %[[PRIV_PS]]#0
+!CHECK:         %[[PS_BOX:.*]] = fir.box_addr %[[PS_VAL]]
+!CHECK:         fir.call %[[PS_BOX]]({{.*}})
+subroutine test_private
+  procedure(f), pointer :: pf
+  procedure(sub), pointer :: ps
+  integer :: res
+
+  !$omp parallel private(pf, ps)
+    pf => f
+    ps => sub
+    res = pf(123)
+    call ps(456)
+  !$omp end parallel
+end subroutine
+
+!CHECK-LABEL: func private @_QFPtest_firstprivate
+!CHECK:       %[[PF:.*]]:2 = hlfir.declare %{{.*}} {{{.*}}uniq_name = "_QFFtest_firstprivateEpf"}
+!CHECK:       %[[PS:.*]]:2 = hlfir.declare %{{.*}} {{{.*}}uniq_name = "_QFFtest_firstprivateEps"}
+!CHECK:       omp.parallel
+!CHECK:         %[[PRIV_PF:.*]]:2 = hlfir.declare %{{.*}} {{{.*}}uniq_name = "_QFFtest_firstprivateEpf"}
+!CHECK:         %[[PRIV_PS:.*]]:2 = hlfir.declare %{{.*}} {{{.*}}uniq_name = "_QFFtest_firstprivateEps"}
+subroutine test_firstprivate
+  procedure(f), pointer :: pf
+  procedure(sub), pointer :: ps
+
+  !$omp parallel firstprivate(pf, ps)
+  !$omp end parallel
+end subroutine
+
+!CHECK-LABEL: func private @_QFPtest_lastprivate
+!CHECK:       %[[PF:.*]]:2 = hlfir.declare %{{.*}} {{{.*}}uniq_name = "_QFFtest_lastprivateEpf"}
+!CHECK:       %[[PS:.*]]:2 = hlfir.declare %{{.*}} {{{.*}}uniq_name = "_QFFtest_lastprivateEps"}
+!CHECK:       omp.parallel
+!CHECK:         %[[PRIV_PF:.*]]:2 = hlfir.declare %{{.*}} {{{.*}}uniq_name = "_QFFtest_lastprivateEpf"}
+!CHECK:         %[[PRIV_PS:.*]]:2 = hlfir.declare %{{.*}} {{{.*}}uniq_name = "_QFFtest_lastprivateEps"}
+!CHECK:         %[[PF_VAL:.*]] = fir.load %[[PRIV_PF]]#0
+!CHECK:         fir.store %[[PF_VAL]] to %[[PF]]#0
+!CHECK:         %[[PS_VAL:.*]] = fir.load %[[PRIV_PS]]#0
+!CHECK:         fir.store %[[PS_VAL]] to %[[PS]]#0
+subroutine test_lastprivate
+  procedure(f), pointer :: pf
+  procedure(sub), pointer :: ps
+  integer :: i
+
+  !$omp parallel do lastprivate(pf, ps)
+  do i = 1, 5
+  end do
+  !$omp end parallel do
+end subroutine
+
+!CHECK-LABEL: func private @_QFPtest_sections
+!CHECK:       %[[PF:.*]]:2 = hlfir.declare %{{.*}} {{{.*}}uniq_name = "_QFFtest_sectionsEpf"}
+!CHECK:       %[[PS:.*]]:2 = hlfir.declare %{{.*}} {{{.*}}uniq_name = "_QFFtest_sectionsEps"}
+!CHECK:       %[[PRIV_PF:.*]]:2 = hlfir.declare %{{.*}} {{{.*}}uniq_name = "_QFFtest_sectionsEpf"}
+!CHECK:       %[[PF_VAL:.*]] = fir.load %[[PF]]#0
+!CHECK:       fir.store %[[PF_VAL]] to %[[PRIV_PF]]#0
+!CHECK:       %[[PRIV_PS:.*]]:2 = hlfir.declare %{{.*}} {{{.*}}uniq_name = "_QFFtest_sectionsEps"}
+!CHECK:       %[[PS_VAL:.*]] = fir.load %[[PS]]#0
+!CHECK:       fir.store %[[PS_VAL]] to %[[PRIV_PS]]#0
+!CHECK:       omp.sections
+!CHECK:         %[[PF_VAL:.*]] = fir.load %[[PRIV_PF]]#0
+!CHECK:         fir.store %[[PF_VAL]] to %[[PF]]#0
+!CHECK:         %[[PS_VAL:.*]] = fir.load %[[PRIV_PS]]#0
+!CHECK:         fir.store %[[PS_VAL]] to %[[PS]]#0
+subroutine test_sections
+  procedure(f), pointer :: pf
+  procedure(sub), pointer :: ps
+
+  !$omp sections firstprivate(pf, ps) lastprivate(pf, ps)
+  !$omp end sections
+end subroutine
+
+integer function f(arg)
+  integer :: arg
+  f = arg
+end function
+
+subroutine sub(arg)
+  integer :: arg
+end subroutine
+
+subroutine test_names
+  procedure(s0), pointer :: ps0
+  procedure(s1), pointer :: ps1
+  procedure(s2), pointer :: ps2
+
+  procedure(f0), pointer :: pf0
+  procedure(f1), pointer :: pf1
+  procedure(f2), pointer :: pf2
+
+  !$omp parallel private(ps0, ps1, ps2, pf0, pf1, pf2)
+  !$omp end parallel
+end subroutine
+
+subroutine s0
+end subroutine
+
+subroutine s1(i)
+  integer :: i
+end subroutine
+
+subroutine s2(i, j)
+  integer :: i
+  character(*) :: j
+end subroutine
+
+integer function f0
+  f0 = 0
+end function
+
+real function f1(r)
+  real :: r
+
+  f1 = 0.0
+end function
+
+function f2(a, c)
+  real :: a(3, 4)
+  complex :: f2, c
+
+  f2 = (0.0, 0.0)
+end function
+
+end program
diff --git a/flang/test/Lower/OpenMP/reduction-equivalence.f90 b/flang/test/Lower/OpenMP/reduction-equivalence.f90
new file mode 100644
index 0000000000000..f46e148ac61cd
--- /dev/null
+++ b/flang/test/Lower/OpenMP/reduction-equivalence.f90
@@ -0,0 +1,48 @@
+! RUN: bbc -emit-hlfir -fopenmp -o - %s | FileCheck %s
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s | FileCheck %s
+
+! Test that we can reduce variables used in an equivalence statement.
+! Unlike POINTER variables these are lowered to an unboxed !fir.ptr<>.
+
+subroutine reduction_equivalence()
+  integer::va
+  equivalence (va,vva)
+  va=1
+
+!$omp parallel reduction(+:va)
+  va=1
+!$omp end parallel
+end subroutine reduction_equivalence
+
+
+! CHECK-LABEL:   omp.declare_reduction @add_reduction_i32 : i32 init {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: i32):
+! CHECK:           %[[VAL_1:.*]] = arith.constant 0 : i32
+! CHECK:           omp.yield(%[[VAL_1]] : i32)
+! CHECK-LABEL:   } combiner {
+! CHECK:         ^bb0(%[[VAL_0:.*]]: i32, %[[VAL_1:.*]]: i32):
+! CHECK:           %[[VAL_2:.*]] = arith.addi %[[VAL_0]], %[[VAL_1]] : i32
+! CHECK:           omp.yield(%[[VAL_2]] : i32)
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPreduction_equivalence() {
+! CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.array<4xi8> {uniq_name = "_QFreduction_equivalenceEva"}
+! CHECK:           %[[VAL_1:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_2:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_1]] : (!fir.ref<!fir.array<4xi8>>, index) -> !fir.ref<i8>
+! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<i8>) -> !fir.ptr<i32>
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFreduction_equivalenceEva"} : (!fir.ptr<i32>) -> (!fir.ptr<i32>, !fir.ptr<i32>)
+! CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_6:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_5]] : (!fir.ref<!fir.array<4xi8>>, index) -> !fir.ref<i8>
+! CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<i8>) -> !fir.ptr<f32>
+! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_equivalenceEvva"} : (!fir.ptr<f32>) -> (!fir.ptr<f32>, !fir.ptr<f32>)
+! CHECK:           %[[VAL_9:.*]] = arith.constant 1 : i32
+! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : i32, !fir.ptr<i32>
+! CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_4]]#0 : (!fir.ptr<i32>) -> !fir.ref<i32>
+! CHECK:           omp.parallel reduction(@add_reduction_i32 %[[VAL_10]] -> %[[VAL_11:.*]] : !fir.ref<i32>) {
+! CHECK:             %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFreduction_equivalenceEva"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:             %[[VAL_13:.*]] = arith.constant 1 : i32
+! CHECK:             hlfir.assign %[[VAL_13]] to %[[VAL_12]]#0 : i32, !fir.ref<i32>
+! CHECK:             omp.terminator
+! CHECK:           }
+! CHECK:           return
+! CHECK:         }
diff --git a/flang/test/Lower/default-initialization.f90 b/flang/test/Lower/default-initialization.f90
index d3664c1cc04f2..7b93db43613f1 100644
--- a/flang/test/Lower/default-initialization.f90
+++ b/flang/test/Lower/default-initialization.f90
@@ -23,8 +23,7 @@ module test_dinit
   subroutine local
     !CHECK: %[[x:.*]]  = fir.alloca !fir.type<_QMtest_dinitTt{i:i32}> {bindc_name = "x", uniq_name = "_QMtest_dinitFlocalEx"}
     !CHECK: %[[ADDR:.*]] = fir.address_of(@_QQ_QMtest_dinitTt.DerivedInit) : !fir.ref<!fir.type<_QMtest_dinitTt{i:i32}>>
-    !CHECK: %[[LOAD:.*]] = fir.load %[[ADDR]] : !fir.ref<!fir.type<_QMtest_dinitTt{i:i32}>>
-    !CHECK: fir.store %[[LOAD]] to %[[x]] : !fir.ref<!fir.type<_QMtest_dinitTt{i:i32}>>
+    !CHECK: fir.copy %[[ADDR]] to %[[x]] no_overlap : !fir.ref<!fir.type<_QMtest_dinitTt{i:i32}>>, !fir.ref<!fir.type<_QMtest_dinitTt{i:i32}>>
     type(t) :: x
     print *, x%i
   end subroutine 
@@ -47,8 +46,7 @@ subroutine local_array()
   subroutine local_alloc_comp
     !CHECK: %[[x:.*]] = fir.alloca !fir.type<_QMtest_dinitTt_alloc_comp{i:!fir.box<!fir.heap<!fir.array<?xf32>>>}> {bindc_name = "x", uniq_name = "_QMtest_dinitFlocal_alloc_compEx"}
     !CHECK: %[[ADDR:.*]] = fir.address_of(@_QQ_QMtest_dinitTt_alloc_comp.DerivedInit) : !fir.ref<!fir.type<_QMtest_dinitTt_alloc_comp{i:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>
-    !CHECK: %[[LOAD:.*]] = fir.load %[[ADDR]] : !fir.ref<!fir.type<_QMtest_dinitTt_alloc_comp{i:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>
-    !CHECK: fir.store %[[LOAD]] to %[[x]] : !fir.ref<!fir.type<_QMtest_dinitTt_alloc_comp{i:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>
+    !CHECK: fir.copy %[[ADDR]] to %[[x]] no_overlap : !fir.ref<!fir.type<_QMtest_dinitTt_alloc_comp{i:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>, !fir.ref<!fir.type<_QMtest_dinitTt_alloc_comp{i:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>
     type(t_alloc_comp) :: x
   end subroutine 
 
@@ -57,8 +55,7 @@ subroutine local_alloc_comp
   function result()
     !CHECK: %[[x:.*]] = fir.alloca !fir.type<_QMtest_dinitTt{i:i32}> {bindc_name = "result", uniq_name = "_QMtest_dinitFresultEresult"}
     !CHECK: %[[ADDR:.*]] = fir.address_of(@_QQ_QMtest_dinitTt.DerivedInit) : !fir.ref<!fir.type<_QMtest_dinitTt{i:i32}>>
-    !CHECK: %[[LOAD:.*]] = fir.load %[[ADDR]] : !fir.ref<!fir.type<_QMtest_dinitTt{i:i32}>>
-    !CHECK: fir.store %[[LOAD]] to %[[x]] : !fir.ref<!fir.type<_QMtest_dinitTt{i:i32}>>
+    !CHECK: fir.copy %[[ADDR]] to %[[x]] no_overlap : !fir.ref<!fir.type<_QMtest_dinitTt{i:i32}>>, !fir.ref<!fir.type<_QMtest_dinitTt{i:i32}>>
     type(t) :: result
   end function
 
@@ -67,8 +64,7 @@ function result()
   ! CHECK-SAME: %[[x:.*]]: !fir.ref<!fir.type<_QMtest_dinitTt{i:i32}>>
   subroutine intent_out(x)
     !CHECK: %[[ADDR:.*]] = fir.address_of(@_QQ_QMtest_dinitTt.DerivedInit) : !fir.ref<!fir.type<_QMtest_dinitTt{i:i32}>>
-    !CHECK: %[[LOAD:.*]] = fir.load %[[ADDR]] : !fir.ref<!fir.type<_QMtest_dinitTt{i:i32}>>
-    !CHECK: fir.store %[[LOAD]] to %[[x]] : !fir.ref<!fir.type<_QMtest_dinitTt{i:i32}>>
+    !CHECK: fir.copy %[[ADDR]] to %[[x]] no_overlap : !fir.ref<!fir.type<_QMtest_dinitTt{i:i32}>>, !fir.ref<!fir.type<_QMtest_dinitTt{i:i32}>>
     type(t), intent(out) :: x
   end subroutine
 
@@ -95,8 +91,7 @@ subroutine local_eq()
     ! CHECK: %[[xcoor:.*]] = fir.coordinate_of %[[equiv]], %c0{{.*}} : (!fir.ref<!fir.array<4xi8>>, index) -> !fir.ref<i8>
     ! CHECK: %[[x:.*]] = fir.convert %[[xcoor]] : (!fir.ref<i8>) -> !fir.ptr<!fir.type<_QMtest_dinitTtseq{i:i32}>>
     ! CHECK: %[[ADDR:.*]] = fir.address_of(@_QQ_QMtest_dinitTtseq.DerivedInit) : !fir.ref<!fir.type<_QMtest_dinitTtseq{i:i32}>>
-    ! CHECK: %[[LOAD:.*]] = fir.load %[[ADDR]] : !fir.ref<!fir.type<_QMtest_dinitTtseq{i:i32}>>
-    ! CHECK: fir.store %[[LOAD]] to %[[x]] : !fir.ptr<!fir.type<_QMtest_dinitTtseq{i:i32}>>
+    ! CHECK: fir.copy %[[ADDR]] to %[[x]] no_overlap : !fir.ref<!fir.type<_QMtest_dinitTtseq{i:i32}>>, !fir.ptr<!fir.type<_QMtest_dinitTtseq{i:i32}>>
     equivalence (x, zi)
     print *, i
   end subroutine
@@ -113,15 +108,13 @@ subroutine local_eq2()
     ! CHECK: %[[xcoor:.*]] = fir.coordinate_of %[[equiv]], %c0{{.*}} : (!fir.ref<!fir.array<4xi8>>, index) -> !fir.ref<i8>
     ! CHECK: %[[x:.*]] = fir.convert %[[xcoor]] : (!fir.ref<i8>) -> !fir.ptr<!fir.type<_QMtest_dinitTtseq{i:i32}>>
     ! CHECK: %[[ADDR:.*]] = fir.address_of(@_QQ_QMtest_dinitTtseq.DerivedInit) : !fir.ref<!fir.type<_QMtest_dinitTtseq{i:i32}>>
-    ! CHECK: %[[LOAD:.*]] = fir.load %[[ADDR]] : !fir.ref<!fir.type<_QMtest_dinitTtseq{i:i32}>>
-    ! CHECK: fir.store %[[LOAD]] to %[[x]] : !fir.ptr<!fir.type<_QMtest_dinitTtseq{i:i32}>>
+    ! CHECK: fir.copy %[[ADDR]] to %[[x]] no_overlap : !fir.ref<!fir.type<_QMtest_dinitTtseq{i:i32}>>, !fir.ptr<!fir.type<_QMtest_dinitTtseq{i:i32}>>
 
   
     ! CHECK: %[[ycoor:.*]] = fir.coordinate_of %[[equiv]], %c0{{.*}} : (!fir.ref<!fir.array<4xi8>>, index) -> !fir.ref<i8>
     ! CHECK: %[[y:.*]] = fir.convert %[[ycoor]] : (!fir.ref<i8>) -> !fir.ptr<!fir.type<_QMtest_dinitTtseq{i:i32}>>
     ! CHECK: %[[ADDR:.*]] = fir.address_of(@_QQ_QMtest_dinitTtseq.DerivedInit) : !fir.ref<!fir.type<_QMtest_dinitTtseq{i:i32}>>
-    ! CHECK: %[[LOAD:.*]] = fir.load %[[ADDR]] : !fir.ref<!fir.type<_QMtest_dinitTtseq{i:i32}>>
-    ! CHECK: fir.store %[[LOAD]] to %[[y]] : !fir.ptr<!fir.type<_QMtest_dinitTtseq{i:i32}>>
+    ! CHECK: fir.copy %[[ADDR]] to %[[y]] no_overlap : !fir.ref<!fir.type<_QMtest_dinitTtseq{i:i32}>>, !fir.ptr<!fir.type<_QMtest_dinitTtseq{i:i32}>>
     equivalence (x, y)
     print *, y%i
   end subroutine
diff --git a/flang/test/Lower/derived-type-finalization.f90 b/flang/test/Lower/derived-type-finalization.f90
index 0be9b91cdc0eb..3ea58cd719f4a 100644
--- a/flang/test/Lower/derived-type-finalization.f90
+++ b/flang/test/Lower/derived-type-finalization.f90
@@ -203,8 +203,7 @@ subroutine test_avoid_double_finalization(a)
 ! CHECK-LABEL: func.func @_QMderived_type_finalizationPtest_avoid_double_finalization(
 ! CHECK: %[[b:.*]] = fir.alloca !fir.type<_QMderived_type_finalizationTt3{t:!fir.type<_QMderived_type_finalizationTt2{a:!fir.box<!fir.heap<!fir.array<?xi32>>>}>}> {bindc_name = "b", uniq_name = "_QMderived_type_finalizationFtest_avoid_double_finalizationEb"}
 ! CHECK: %[[ADDR:.*]] = fir.address_of(@_QQ_QMderived_type_finalizationTt3.DerivedInit) : !fir.ref<!fir.type<_QMderived_type_finalizationTt3{t:!fir.type<_QMderived_type_finalizationTt2{a:!fir.box<!fir.heap<!fir.array<?xi32>>>}>}>>
-! CHECK: %[[LOAD:.*]] = fir.load %[[ADDR]] : !fir.ref<!fir.type<_QMderived_type_finalizationTt3{t:!fir.type<_QMderived_type_finalizationTt2{a:!fir.box<!fir.heap<!fir.array<?xi32>>>}>}>>
-! CHECK: fir.store %[[LOAD]] to %[[b]] : !fir.ref<!fir.type<_QMderived_type_finalizationTt3{t:!fir.type<_QMderived_type_finalizationTt2{a:!fir.box<!fir.heap<!fir.array<?xi32>>>}>}>>
+! CHECK: fir.copy %[[ADDR]] to %[[b]] no_overlap : !fir.ref<!fir.type<_QMderived_type_finalizationTt3{t:!fir.type<_QMderived_type_finalizationTt2{a:!fir.box<!fir.heap<!fir.array<?xi32>>>}>}>>
 ! CHECK-NOT: fir.call @_FortranADestroy
 ! CHECK: fir.call @_FortranAAssign(
 ! CHECK: fir.call @_FortranADestroy(
diff --git a/flang/test/Lower/derived-type-temp.f90 b/flang/test/Lower/derived-type-temp.f90
index d7b8ce3973295..f4e55fe095c6d 100644
--- a/flang/test/Lower/derived-type-temp.f90
+++ b/flang/test/Lower/derived-type-temp.f90
@@ -1,5 +1,5 @@
 ! Test lowering of derived type temporary creation and init
-! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s
 
 program derived_temp_init
   type t1
@@ -13,7 +13,7 @@ program derived_temp_init
   y = t2(x)
 end
 
-! CHECK: %[[x:.*]] =  fir.alloca !fir.type<_QFTt1{i:!fir.box<!fir.heap<i32>>}> {bindc_name = "x", uniq_name = "_QFEx"}
+! CHECK: %[[ALLOC:.*]] =  fir.alloca !fir.type<_QFTt1{i:!fir.box<!fir.heap<i32>>}> {bindc_name = "x", uniq_name = "_QFEx"}
+! CHECK: %[[x:.*]]:2 = hlfir.declare %[[ALLOC]] {{.*}}
 ! CHECK: %[[ADDR:.*]] = fir.address_of(@_QQ_QFTt1.DerivedInit) : !fir.ref<!fir.type<_QFTt1{i:!fir.box<!fir.heap<i32>>}>>
-! CHECK: %[[LOAD:.*]] = fir.load %[[ADDR]] : !fir.ref<!fir.type<_QFTt1{i:!fir.box<!fir.heap<i32>>}>>
-! CHECK: fir.store %[[LOAD]] to %[[x]] : !fir.ref<!fir.type<_QFTt1{i:!fir.box<!fir.heap<i32>>}>>
+! CHECK: fir.copy %[[ADDR]] to %[[x]]#1 no_overlap : !fir.ref<!fir.type<_QFTt1{i:!fir.box<!fir.heap<i32>>}>>, !fir.ref<!fir.type<_QFTt1{i:!fir.box<!fir.heap<i32>>}>>
diff --git a/flang/test/Lower/forall/character-1.f90 b/flang/test/Lower/forall/character-1.f90
index d5f968ba93450..69064ddfcf0be 100644
--- a/flang/test/Lower/forall/character-1.f90
+++ b/flang/test/Lower/forall/character-1.f90
@@ -2,6 +2,10 @@
 ! RUN: %flang -emit-llvm -flang-deprecated-no-hlfir -S -mmlir -disable-external-name-interop %s -o - | FileCheck %s
 ! Test from Fortran source through to LLVM IR.
 ! UNSUPPORTED: system-windows
+! Disabled on 32-bit targets due to the additional `trunc` opcodes required
+! UNSUPPORTED: target-x86
+! UNSUPPORTED: target=sparc-{{.*}}
+! UNSUPPORTED: target=sparcel-{{.*}}
 
 ! Assumed size array of assumed length character.
 program test
diff --git a/flang/test/Lower/forall/forall-allocatable-2.f90 b/flang/test/Lower/forall/forall-allocatable-2.f90
index fb212c085dcff..16bfc286d9482 100644
--- a/flang/test/Lower/forall/forall-allocatable-2.f90
+++ b/flang/test/Lower/forall/forall-allocatable-2.f90
@@ -19,8 +19,7 @@ end subroutine forall_with_allocatable2
 ! CHECK:         %[[VAL_1:.*]] = fir.alloca i32 {adapt.valuebyref, bindc_name = "i"}
 ! CHECK:         %[[VAL_2:.*]] = fir.alloca !fir.type<_QFforall_with_allocatable2Tt{i:i32,arr:!fir.box<!fir.heap<!fir.array<?xf32>>>}> {bindc_name = "thing", uniq_name = "_QFforall_with_allocatable2Ething"}
 ! CHECK:         %[[ADDR:.*]] =  fir.address_of(@_QQ_QFforall_with_allocatable2Tt.DerivedInit) : !fir.ref<!fir.type<_QFforall_with_allocatable2Tt{i:i32,arr:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>
-! CHECK:         %[[LOAD:.*]] = fir.load %[[ADDR]] : !fir.ref<!fir.type<_QFforall_with_allocatable2Tt{i:i32,arr:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>
-! CHECK:         fir.store %[[LOAD]] to %[[VAL_2]] : !fir.ref<!fir.type<_QFforall_with_allocatable2Tt{i:i32,arr:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>
+! CHECK:         fir.copy %[[ADDR]] to %[[VAL_2]] no_overlap : !fir.ref<!fir.type<_QFforall_with_allocatable2Tt{i:i32,arr:!fir.box<!fir.heap<!fir.array<?xf32>>>}>>
 ! CHECK:         %[[VAL_9:.*]] = arith.constant 5 : i32
 ! CHECK:         %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i32) -> index
 ! CHECK:         %[[VAL_11:.*]] = arith.constant 15 : i32
diff --git a/flang/test/Lower/pointer-default-init.f90 b/flang/test/Lower/pointer-default-init.f90
index 661db22b2eaa2..dd9c7539b4f05 100644
--- a/flang/test/Lower/pointer-default-init.f90
+++ b/flang/test/Lower/pointer-default-init.f90
@@ -40,8 +40,7 @@ subroutine test_local()
 ! CHECK-LABEL:   func.func @_QPtest_local() {
 ! CHECK: %[[x:.*]] = fir.alloca !fir.type<_QMtestTt{i:i32,x:!fir.box<!fir.ptr<!fir.array<?xf32>>>}> {bindc_name = "x", uniq_name = "_QFtest_localEx"}
 ! CHECK: %[[ADDR:.*]] = fir.address_of(@_QQ_QMtestTt.DerivedInit) : !fir.ref<!fir.type<_QMtestTt{i:i32,x:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>
-! CHECK: %[[LOAD:.*]] = fir.load %[[ADDR]] : !fir.ref<!fir.type<_QMtestTt{i:i32,x:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>
-! CHECK: fir.store %[[LOAD]] to %[[x]] : !fir.ref<!fir.type<_QMtestTt{i:i32,x:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>
+! CHECK: fir.copy %[[ADDR]] to %[[x]] no_overlap : !fir.ref<!fir.type<_QMtestTt{i:i32,x:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>, !fir.ref<!fir.type<_QMtestTt{i:i32,x:!fir.box<!fir.ptr<!fir.array<?xf32>>>}>>
 
 subroutine test_saved()
   use test, only : t
diff --git a/flang/test/Parser/OpenMP/if-clause.f90 b/flang/test/Parser/OpenMP/if-clause.f90
index b3e3913f8bd1c..d7ab86ca6d2cf 100644
--- a/flang/test/Parser/OpenMP/if-clause.f90
+++ b/flang/test/Parser/OpenMP/if-clause.f90
@@ -11,34 +11,34 @@ program openmp_parse_if
 
   ! CHECK: OmpSimpleStandaloneDirective -> llvm::omp::Directive = target update
   ! CHECK-NEXT: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: OmpDirectiveNameModifier -> llvm::omp::Directive = target update
+  ! CHECK-NEXT: OmpDirectiveName -> llvm::omp::Directive = target update
   !$omp target update if(target update: cond) to(i)
 
   ! CHECK: OmpSimpleStandaloneDirective -> llvm::omp::Directive = target enter data
   ! CHECK: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: OmpDirectiveNameModifier -> llvm::omp::Directive = target enter data
+  ! CHECK-NEXT: OmpDirectiveName -> llvm::omp::Directive = target enter data
   !$omp target enter data map(to: i) if(target enter data: cond)
 
   ! CHECK: OmpSimpleStandaloneDirective -> llvm::omp::Directive = target exit data
   ! CHECK: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: OmpDirectiveNameModifier -> llvm::omp::Directive = target exit data
+  ! CHECK-NEXT: OmpDirectiveName -> llvm::omp::Directive = target exit data
   !$omp target exit data map(from: i) if(target exit data: cond)
 
   ! CHECK: OmpBlockDirective -> llvm::omp::Directive = target data
   ! CHECK: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: OmpDirectiveNameModifier -> llvm::omp::Directive = target data
+  ! CHECK-NEXT: OmpDirectiveName -> llvm::omp::Directive = target data
   !$omp target data map(tofrom: i) if(target data: cond)
   !$omp end target data
 
   ! CHECK: OmpLoopDirective -> llvm::omp::Directive = target teams distribute parallel do simd
   ! CHECK: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: OmpDirectiveNameModifier -> llvm::omp::Directive = target
+  ! CHECK-NEXT: OmpDirectiveName -> llvm::omp::Directive = target
   ! CHECK: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: OmpDirectiveNameModifier -> llvm::omp::Directive = teams
+  ! CHECK-NEXT: OmpDirectiveName -> llvm::omp::Directive = teams
   ! CHECK: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: OmpDirectiveNameModifier -> llvm::omp::Directive = parallel
+  ! CHECK-NEXT: OmpDirectiveName -> llvm::omp::Directive = parallel
   ! CHECK: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: OmpDirectiveNameModifier -> llvm::omp::Directive = simd
+  ! CHECK-NEXT: OmpDirectiveName -> llvm::omp::Directive = simd
   !$omp target teams distribute parallel do simd if(target: cond) &
   !$omp&    if(teams: cond) if(parallel: cond) if(simd: cond)
   do i = 1, 10
@@ -47,13 +47,13 @@ program openmp_parse_if
 
   ! CHECK: OmpBlockDirective -> llvm::omp::Directive = task
   ! CHECK-NEXT: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: OmpDirectiveNameModifier -> llvm::omp::Directive = task
+  ! CHECK-NEXT: OmpDirectiveName -> llvm::omp::Directive = task
   !$omp task if(task: cond)
   !$omp end task
 
   ! CHECK: OmpLoopDirective -> llvm::omp::Directive = taskloop
   ! CHECK-NEXT: OmpClause -> If -> OmpIfClause
-  ! CHECK-NEXT: DirectiveNameModifier -> llvm::omp::Directive = taskloop
+  ! CHECK-NEXT: OmpDirectiveName -> llvm::omp::Directive = taskloop
   !$omp taskloop if(taskloop: cond)
   do i = 1, 10
   end do
diff --git a/flang/test/Parser/OpenMP/metadirective-dirspec.f90 b/flang/test/Parser/OpenMP/metadirective-dirspec.f90
index 73520c41fe77d..bf749d1f48c10 100644
--- a/flang/test/Parser/OpenMP/metadirective-dirspec.f90
+++ b/flang/test/Parser/OpenMP/metadirective-dirspec.f90
@@ -25,7 +25,7 @@ subroutine f00(x)
 !PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
 !PARSE-TREE: | | | | | | | bool = 'true'
 !PARSE-TREE: | | OmpDirectiveSpecification
-!PARSE-TREE: | | | llvm::omp::Directive = allocate
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = allocate
 !PARSE-TREE: | | | OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 'x'
 !PARSE-TREE: | | | OmpClauseList ->
 
@@ -51,7 +51,7 @@ subroutine f01(x)
 !PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
 !PARSE-TREE: | | | | | | | bool = 'true'
 !PARSE-TREE: | | OmpDirectiveSpecification
-!PARSE-TREE: | | | llvm::omp::Directive = critical
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = critical
 !PARSE-TREE: | | | OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 'x'
 !PARSE-TREE: | | | OmpClauseList ->
 
@@ -76,7 +76,7 @@ subroutine f02
 !PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
 !PARSE-TREE: | | | | | | | bool = 'true'
 !PARSE-TREE: | | OmpDirectiveSpecification
-!PARSE-TREE: | | | llvm::omp::Directive = declare mapper
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = declare mapper
 !PARSE-TREE: | | | OmpArgument -> OmpMapperSpecifier
 !PARSE-TREE: | | | | Name = 'mymapper'
 !PARSE-TREE: | | | | TypeSpec -> IntrinsicTypeSpec -> IntegerTypeSpec ->
@@ -120,7 +120,7 @@ subroutine f03
 !PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
 !PARSE-TREE: | | | | | | | bool = 'true'
 !PARSE-TREE: | | OmpDirectiveSpecification
-!PARSE-TREE: | | | llvm::omp::Directive = declare reduction
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = declare reduction
 !PARSE-TREE: | | | OmpArgument -> OmpReductionSpecifier
 !PARSE-TREE: | | | | OmpReductionIdentifier -> DefinedOperator -> IntrinsicOperator = Add
 !PARSE-TREE: | | | | OmpTypeNameList -> OmpTypeSpecifier -> TypeSpec -> DerivedTypeSpec
@@ -158,7 +158,7 @@ subroutine f04
 !PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
 !PARSE-TREE: | | | | | | | bool = 'true'
 !PARSE-TREE: | | OmpDirectiveSpecification
-!PARSE-TREE: | | | llvm::omp::Directive = declare simd
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = declare simd
 !PARSE-TREE: | | | OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 'f04'
 !PARSE-TREE: | | | OmpClauseList ->
 !PARSE-TREE: ImplicitPart ->
@@ -183,7 +183,7 @@ subroutine f05
 !PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
 !PARSE-TREE: | | | | | | | bool = 'true'
 !PARSE-TREE: | | OmpDirectiveSpecification
-!PARSE-TREE: | | | llvm::omp::Directive = declare target
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = declare target
 !PARSE-TREE: | | | OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 'f05'
 !PARSE-TREE: | | | OmpClauseList ->
 !PARSE-TREE: ImplicitPart ->
@@ -210,7 +210,7 @@ subroutine f06(x, y)
 !PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
 !PARSE-TREE: | | | | | | | bool = 'true'
 !PARSE-TREE: | | OmpDirectiveSpecification
-!PARSE-TREE: | | | llvm::omp::Directive = flush
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = flush
 !PARSE-TREE: | | | OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 'x'
 !PARSE-TREE: | | | OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 'y'
 !PARSE-TREE: | | | OmpClauseList ->
@@ -237,6 +237,6 @@ subroutine f07
 !PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
 !PARSE-TREE: | | | | | | | bool = 'true'
 !PARSE-TREE: | | OmpDirectiveSpecification
-!PARSE-TREE: | | | llvm::omp::Directive = threadprivate
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = threadprivate
 !PARSE-TREE: | | | OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 't'
 !PARSE-TREE: | | | OmpClauseList ->
diff --git a/flang/test/Parser/OpenMP/metadirective-flush.f90 b/flang/test/Parser/OpenMP/metadirective-flush.f90
new file mode 100644
index 0000000000000..8403663200f93
--- /dev/null
+++ b/flang/test/Parser/OpenMP/metadirective-flush.f90
@@ -0,0 +1,54 @@
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=52 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=52 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine f00()
+  integer :: x
+  !$omp metadirective when(user={condition(.true.)}: flush seq_cst (x))
+end
+
+!UNPARSE: SUBROUTINE f00
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(USER={CONDITION(.true._4)}: FLUSH SEQ_CST(x))
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = User
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Condition
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> Scalar -> Expr = '.true._4'
+!PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | | | | bool = 'true'
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = flush
+!PARSE-TREE: | | | OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | | OmpClauseList -> OmpClause -> SeqCst
+!PARSE-TREE: | | | Flags = DeprecatedSyntax
+
+subroutine f01()
+  integer :: x
+  !$omp metadirective when(user={condition(.true.)}: flush(x) seq_cst)
+end
+
+!UNPARSE: SUBROUTINE f01
+!UNPARSE:  INTEGER x
+!UNPARSE: !$OMP METADIRECTIVE  WHEN(USER={CONDITION(.true._4)}: FLUSH(x) SEQ_CST)
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpMetadirectiveDirective
+!PARSE-TREE: | OmpClauseList -> OmpClause -> When -> OmpWhenClause
+!PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE: | | | OmpTraitSetSelectorName -> Value = User
+!PARSE-TREE: | | | OmpTraitSelector
+!PARSE-TREE: | | | | OmpTraitSelectorName -> Value = Condition
+!PARSE-TREE: | | | | Properties
+!PARSE-TREE: | | | | | OmpTraitProperty -> Scalar -> Expr = '.true._4'
+!PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | | | | bool = 'true'
+!PARSE-TREE: | | OmpDirectiveSpecification
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = flush
+!PARSE-TREE: | | | OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | | | OmpClauseList -> OmpClause -> SeqCst
+!PARSE-TREE: | | | Flags = None
diff --git a/flang/test/Parser/OpenMP/metadirective-v50.f90 b/flang/test/Parser/OpenMP/metadirective-v50.f90
index d7c3121b8f1b8..6fef3376470a6 100644
--- a/flang/test/Parser/OpenMP/metadirective-v50.f90
+++ b/flang/test/Parser/OpenMP/metadirective-v50.f90
@@ -24,8 +24,8 @@ subroutine f01
 !PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
 !PARSE-TREE: | | | | | | | bool = 'true'
 !PARSE-TREE: | | OmpDirectiveSpecification
-!PARSE-TREE: | | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = nothing
 !PARSE-TREE: | | | OmpClauseList ->
 !PARSE-TREE: | OmpClause -> Default -> OmpDefaultClause -> OmpDirectiveSpecification
-!PARSE-TREE: | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = nothing
 !PARSE-TREE: | | OmpClauseList ->
diff --git a/flang/test/Parser/OpenMP/metadirective.f90 b/flang/test/Parser/OpenMP/metadirective.f90
index dce31c2e7db26..1185ac897ecf6 100644
--- a/flang/test/Parser/OpenMP/metadirective.f90
+++ b/flang/test/Parser/OpenMP/metadirective.f90
@@ -20,7 +20,7 @@ subroutine f00
 !PARSE-TREE: | | | OmpTraitSelector
 !PARSE-TREE: | | | | OmpTraitSelectorName -> llvm::omp::Directive = parallel
 !PARSE-TREE: | | OmpDirectiveSpecification
-!PARSE-TREE: | | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = nothing
 !PARSE-TREE: | | | OmpClauseList ->
 
 subroutine f01
@@ -47,7 +47,7 @@ subroutine f01
 !PARSE-TREE: | | | | | OmpTraitProperty -> Scalar -> Expr = '1_4'
 !PARSE-TREE: | | | | | | LiteralConstant -> IntLiteralConstant = '1'
 !PARSE-TREE: | | OmpDirectiveSpecification
-!PARSE-TREE: | | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = nothing
 !PARSE-TREE: | | | OmpClauseList ->
 
 subroutine f02
@@ -74,7 +74,7 @@ subroutine f02
 !PARSE-TREE: | | | | | OmpTraitProperty -> Scalar -> Expr = '7_4'
 !PARSE-TREE: | | | | | | LiteralConstant -> IntLiteralConstant = '7'
 !PARSE-TREE: | | OmpDirectiveSpecification
-!PARSE-TREE: | | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = nothing
 !PARSE-TREE: | | | OmpClauseList ->
 
 subroutine f03
@@ -98,7 +98,7 @@ subroutine f03
 !PARSE-TREE: | | | | Properties
 !PARSE-TREE: | | | | | OmpTraitProperty -> OmpClause -> AcqRel
 !PARSE-TREE: | | OmpDirectiveSpecification
-!PARSE-TREE: | | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = nothing
 !PARSE-TREE: | | | OmpClauseList ->
 
 subroutine f04
@@ -132,7 +132,7 @@ subroutine f04
 !PARSE-TREE: | | | | | | | OmpTraitPropertyExtension -> Scalar -> Expr = '1_4'
 !PARSE-TREE: | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
 !PARSE-TREE: | | OmpDirectiveSpecification
-!PARSE-TREE: | | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = nothing
 !PARSE-TREE: | | | OmpClauseList ->
 
 subroutine f05(x)
@@ -168,12 +168,12 @@ subroutine f05(x)
 !PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
 !PARSE-TREE: | | | | | | | bool = 'true'
 !PARSE-TREE: | | OmpDirectiveSpecification
-!PARSE-TREE: | | | llvm::omp::Directive = parallel do
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = parallel do
 !PARSE-TREE: | | | OmpClauseList -> OmpClause -> Reduction -> OmpReductionClause
 !PARSE-TREE: | | | | Modifier -> OmpReductionIdentifier -> DefinedOperator -> IntrinsicOperator = Add
 !PARSE-TREE: | | | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'x'
 !PARSE-TREE: | OmpClause -> Otherwise -> OmpOtherwiseClause -> OmpDirectiveSpecification
-!PARSE-TREE: | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = nothing
 !PARSE-TREE: | | OmpClauseList ->
 
 subroutine f06
@@ -207,7 +207,7 @@ subroutine f06
 !PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
 !PARSE-TREE: | | | | | | | bool = 'true'
 !PARSE-TREE: | | OmpDirectiveSpecification
-!PARSE-TREE: | | | llvm::omp::Directive = nothing
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = nothing
 !PARSE-TREE: | | | OmpClauseList ->
 
 subroutine f07
@@ -232,7 +232,7 @@ subroutine f07
 !PARSE-TREE: | | | | Properties
 !PARSE-TREE: | | | | | OmpTraitProperty -> OmpTraitPropertyName -> string = 'amd'
 !PARSE-TREE: | | OmpDirectiveSpecification
-!PARSE-TREE: | | | llvm::omp::Directive = declare simd
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = declare simd
 !PARSE-TREE: | | | OmpClauseList ->
 !PARSE-TREE: | OmpClause -> When -> OmpWhenClause
 !PARSE-TREE: | | Modifier -> OmpContextSelectorSpecification -> OmpTraitSetSelector
@@ -244,8 +244,8 @@ subroutine f07
 !PARSE-TREE: | | | | | | LiteralConstant -> LogicalLiteralConstant
 !PARSE-TREE: | | | | | | | bool = 'true'
 !PARSE-TREE: | | OmpDirectiveSpecification
-!PARSE-TREE: | | | llvm::omp::Directive = declare target
+!PARSE-TREE: | | | OmpDirectiveName -> llvm::omp::Directive = declare target
 !PARSE-TREE: | | | OmpClauseList ->
 !PARSE-TREE: | OmpClause -> Otherwise -> OmpOtherwiseClause -> OmpDirectiveSpecification
-!PARSE-TREE: | | llvm::omp::Directive = nothing
-!PARSE-TREE: | | OmpClauseList ->
\ No newline at end of file
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = nothing
+!PARSE-TREE: | | OmpClauseList ->
diff --git a/flang/test/Parser/func-label.f b/flang/test/Parser/func-label.f
new file mode 100644
index 0000000000000..d78545177f64e
--- /dev/null
+++ b/flang/test/Parser/func-label.f
@@ -0,0 +1,7 @@
+! RUN: %flang_fc1 -fsyntax-only -pedantic %s 2>&1 | FileCheck --allow-empty %s
+!CHECK-NOT: error:
+ 1    function fun()
+ 2    end function
+ 3    write(6,*) "pass"
+ 4    end
+
diff --git a/flang/test/Preprocessing/bug126459.F90 b/flang/test/Preprocessing/bug126459.F90
new file mode 100644
index 0000000000000..fae8a07659f72
--- /dev/null
+++ b/flang/test/Preprocessing/bug126459.F90
@@ -0,0 +1,5 @@
+! RUN: %flang -E -fopenmp %s 2>&1 | FileCheck %s
+!CHECK: NDIR=0
+#define BLANKMACRO
+BLANKMACRO !$ NDIR=0
+end
diff --git a/flang/test/Semantics/OpenMP/cancel.f90 b/flang/test/Semantics/OpenMP/cancel.f90
new file mode 100644
index 0000000000000..581c4bdb97646
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/cancel.f90
@@ -0,0 +1,29 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp
+
+subroutine f00
+!$omp parallel
+!ERROR: Missing cancel-directive-name clause on the CANCEL construct
+!$omp cancel
+!$omp end parallel
+end
+
+subroutine f01
+!$omp parallel
+!ERROR: Multiple cancel-directive-name clauses are not allowed on the CANCEL construct
+!$omp cancel parallel parallel
+!$omp end parallel
+end
+
+subroutine f02
+!$omp parallel
+!ERROR: Missing cancel-directive-name clause on the CANCELLATION POINT construct
+!$omp cancellation point
+!$omp end parallel
+end
+
+subroutine f03
+!$omp parallel
+!ERROR: Multiple cancel-directive-name clauses are not allowed on the CANCELLATION POINT construct
+!$omp cancellation point parallel parallel
+!$omp end parallel
+end
diff --git a/flang/test/Semantics/OpenMP/clause-validity01.f90 b/flang/test/Semantics/OpenMP/clause-validity01.f90
index e8114154a809b..5e0d91914c441 100644
--- a/flang/test/Semantics/OpenMP/clause-validity01.f90
+++ b/flang/test/Semantics/OpenMP/clause-validity01.f90
@@ -330,11 +330,12 @@
   !$omp parallel
   b = 1
   !ERROR: LASTPRIVATE clause is not allowed on the SINGLE directive
-  !ERROR: NOWAIT clause is not allowed on the OMP SINGLE directive, use it on OMP END SINGLE directive 
+  !ERROR: NOWAIT clause must not be used with COPYPRIVATE clause on the SINGLE directive
   !$omp single private(a) lastprivate(c) nowait
   a = 3.14
-  !ERROR: Clause NOWAIT is not allowed if clause COPYPRIVATE appears on the END SINGLE directive
   !ERROR: COPYPRIVATE variable 'a' may not appear on a PRIVATE or FIRSTPRIVATE clause on a SINGLE construct
+  !ERROR: At most one NOWAIT clause can appear on the SINGLE directive
+  !ERROR: At most one NOWAIT clause can appear on the SINGLE directive
   !ERROR: At most one NOWAIT clause can appear on the END SINGLE directive
   !$omp end single copyprivate(a) nowait nowait
   c = 2
@@ -351,7 +352,6 @@
   a = 1.0
   !ERROR: COPYPRIVATE clause is not allowed on the END WORKSHARE directive
   !$omp end workshare nowait copyprivate(a)
-  !ERROR: NOWAIT clause is not allowed on the OMP WORKSHARE directive, use it on OMP END WORKSHARE directive 
   !$omp workshare nowait
   !$omp end workshare
   !$omp end parallel
@@ -420,7 +420,6 @@
   !$omp parallel
   !ERROR: No ORDERED clause with a parameter can be specified on the DO SIMD directive
   !ERROR: NOGROUP clause is not allowed on the DO SIMD directive
-  !ERROR: NOWAIT clause is not allowed on the OMP DO SIMD directive, use it on OMP END DO SIMD directive 
   !$omp do simd ordered(2) NOGROUP nowait
   do i = 1, N
      do j = 1, N
diff --git a/flang/test/Semantics/OpenMP/equivalence-namelist.f90 b/flang/test/Semantics/OpenMP/equivalence-namelist.f90
new file mode 100644
index 0000000000000..dc0a1712ed8f5
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/equivalence-namelist.f90
@@ -0,0 +1,32 @@
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+
+! The openmp standard only dissallows namelist for privatization, but flang
+! also does not allow it for reduction as this would be difficult to support.
+!
+! Variables in equivalence with variables in the namelist pose the same
+! implementation problems.
+
+subroutine test01()
+  integer::va
+  equivalence (va,vva)
+  namelist /na1/vva
+  va=1
+
+!ERROR: Variable 'va' in NAMELIST cannot be in a REDUCTION clause
+!$omp parallel reduction(+:va)
+  write(*,na1)
+!$omp end parallel
+end subroutine test01
+
+
+subroutine test02()
+  integer::va
+  equivalence (va,vva)
+  namelist /na1/vva
+  va=1
+
+!ERROR: Variable 'va' in NAMELIST cannot be in a PRIVATE clause
+!$omp parallel private(va)
+  write(*,na1)
+!$omp end parallel
+end subroutine test02
diff --git a/flang/test/Semantics/OpenMP/single03.f90 b/flang/test/Semantics/OpenMP/single03.f90
new file mode 100644
index 0000000000000..dc2c2fd27eb04
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/single03.f90
@@ -0,0 +1,54 @@
+! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp -fopenmp-version=52
+!
+! OpenMP Version 5.2
+!
+! 2.10.2 single Construct
+! Copyprivate and Nowait clauses are allowed in both clause and end clause
+
+subroutine omp_single
+    integer, save :: i
+    integer       :: j
+    i = 10; j = 11
+
+    !ERROR: COPYPRIVATE variable 'i' is not PRIVATE or THREADPRIVATE in outer context
+    !ERROR: NOWAIT clause must not be used with COPYPRIVATE clause on the SINGLE directive
+    !$omp single copyprivate(i) nowait
+        print *, "omp single", i
+    !$omp end single
+
+    !$omp parallel private(i)
+        !$omp single copyprivate(i)
+            print *, "omp single", i
+        !$omp end single
+    !$omp end parallel
+
+    !$omp parallel
+        !ERROR: NOWAIT clause must not be used with COPYPRIVATE clause on the SINGLE directive
+        !$omp single nowait
+            print *, "omp single", i
+        !ERROR: COPYPRIVATE variable 'i' is not PRIVATE or THREADPRIVATE in outer context
+        !$omp end single copyprivate(i)
+
+        !ERROR: COPYPRIVATE variable 'i' is not PRIVATE or THREADPRIVATE in outer context
+        !$omp single copyprivate(i)
+            print *, "omp single", i
+        !ERROR: NOWAIT clause must not be used with COPYPRIVATE clause on the SINGLE directive
+        !$omp end single nowait
+
+        !ERROR: COPYPRIVATE variable 'j' may not appear on a PRIVATE or FIRSTPRIVATE clause on a SINGLE construct
+        !$omp single private(j) copyprivate(j)
+            print *, "omp single", j
+        !ERROR: COPYPRIVATE variable 'j' may not appear on a PRIVATE or FIRSTPRIVATE clause on a SINGLE construct
+        !WARNING: The COPYPRIVATE clause with 'j' is already used on the SINGLE directive
+        !$omp end single copyprivate(j)
+
+        !$omp single nowait
+            print *, "omp single", j
+        !ERROR: At most one NOWAIT clause can appear on the SINGLE directive
+        !$omp end single nowait
+    !$omp end parallel
+
+    !$omp single nowait
+        print *, "omp single", i
+    !$omp end single
+end subroutine omp_single
diff --git a/flang/test/Semantics/OpenMP/single04.f90 b/flang/test/Semantics/OpenMP/single04.f90
new file mode 100644
index 0000000000000..9505745c600e9
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/single04.f90
@@ -0,0 +1,81 @@
+! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp -fopenmp-version=52
+!
+! OpenMP Version 5.2
+!
+! 2.10.2 single Construct
+! Valid and invalid testcases for copyprivate and nowait clause on the single directive
+
+program single
+    ! Valid testcases
+    !$omp single
+        print *, x
+    !$omp end single
+
+    !$omp single nowait
+        print *, x
+    !$omp end single
+
+    !$omp single copyprivate(x, y, z)
+        print *, x
+    !$omp end single
+
+    !$omp single
+        print *, x
+    !$omp end single copyprivate(x)
+
+    ! Invalid testcases
+    !$omp single
+        print *, x
+    !ERROR: NOWAIT clause must not be used with COPYPRIVATE clause on the SINGLE directive
+    !$omp end single copyprivate(x) nowait
+
+    !ERROR: 'x' appears in more than one COPYPRIVATE clause on the SINGLE directive
+    !$omp single copyprivate(x) copyprivate(x)
+        print *, x
+    !$omp end single
+
+    !$omp single
+        print *, x
+    !ERROR: 'x' appears in more than one COPYPRIVATE clause on the END SINGLE directive
+    !$omp end single copyprivate(x) copyprivate(x)
+
+    !ERROR: At most one NOWAIT clause can appear on the SINGLE directive
+    !$omp single nowait nowait
+        print *, x
+    !$omp end single
+
+    !$omp single
+        print *, x
+    !ERROR: At most one NOWAIT clause can appear on the END SINGLE directive
+    !$omp end single nowait nowait
+
+    !ERROR: NOWAIT clause must not be used with COPYPRIVATE clause on the SINGLE directive
+    !$omp single copyprivate(x) nowait
+        print *, x
+    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive
+    !ERROR: At most one NOWAIT clause can appear on the SINGLE directive
+    !$omp end single copyprivate(x) nowait
+
+    !$omp single copyprivate(x)
+        print *, x
+    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive
+    !ERROR: NOWAIT clause must not be used with COPYPRIVATE clause on the SINGLE directive
+    !$omp end single copyprivate(x) nowait
+
+    !ERROR: NOWAIT clause must not be used with COPYPRIVATE clause on the SINGLE directive
+    !$omp single copyprivate(x, y) nowait
+        print *, x
+    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive
+    !ERROR: 'z' appears in more than one COPYPRIVATE clause on the END SINGLE directive
+    !ERROR: At most one NOWAIT clause can appear on the SINGLE directive
+    !$omp end single copyprivate(x, z) copyprivate(z) nowait
+
+    !ERROR: NOWAIT clause must not be used with COPYPRIVATE clause on the SINGLE directive
+    !$omp single copyprivate(x) nowait copyprivate(y) copyprivate(z)
+        print *, x
+    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive
+    !WARNING: The COPYPRIVATE clause with 'y' is already used on the SINGLE directive
+    !WARNING: The COPYPRIVATE clause with 'z' is already used on the SINGLE directive
+    !ERROR: At most one NOWAIT clause can appear on the SINGLE directive
+    !$omp end single copyprivate(x, y, z) nowait
+end program
diff --git a/flang/test/Semantics/OpenMP/threadprivate04.f90 b/flang/test/Semantics/OpenMP/threadprivate04.f90
index 3d8c7fb8de8fa..d261f33b4dbd7 100644
--- a/flang/test/Semantics/OpenMP/threadprivate04.f90
+++ b/flang/test/Semantics/OpenMP/threadprivate04.f90
@@ -14,7 +14,6 @@ program main
   !$omp parallel num_threads(x1)
   !$omp end parallel
 
-  !ERROR: COPYPRIVATE clause is not allowed on the OMP SINGLE directive, use it on OMP END SINGLE directive 
   !$omp single copyprivate(x2, /blk1/)
   !$omp end single
 
diff --git a/flang/test/Semantics/allocate12.f90 b/flang/test/Semantics/allocate12.f90
index ecc28d8dbe27d..c5fc7e2b84e4b 100644
--- a/flang/test/Semantics/allocate12.f90
+++ b/flang/test/Semantics/allocate12.f90
@@ -112,6 +112,7 @@ subroutine C941_C942b_C950(xsrc, x1, a2, b2, cx1, ca2, cb1, cb2, c1, c2)
   ! Valid construct
   allocate(c1%ct2(2,5)%t1(2)%t0%array(10))
 
+  !ERROR: cosubscript 2 is less than lower cobound 5 for codimension 1 of array
   !ERROR: Allocatable object must not be coindexed in ALLOCATE
   allocate(b1%x, b2(1)%x, cb1[2]%x, SOURCE=xsrc)
   !ERROR: Allocatable object must not be coindexed in ALLOCATE
diff --git a/flang/test/Semantics/assign02.f90 b/flang/test/Semantics/assign02.f90
index a40d204982b2f..6775506c21a3b 100644
--- a/flang/test/Semantics/assign02.f90
+++ b/flang/test/Semantics/assign02.f90
@@ -146,7 +146,7 @@ function f2()
     end
   end
 
-  ! C1026 (R1037) A data-target shall not be a coindexed object.
+  ! F'2023 C1029 A data-target shall not be a coindexed object.
   subroutine s10
     real, target, save :: a[*]
     real, pointer :: b
@@ -154,6 +154,17 @@ subroutine s10
     b => a[1]
   end
 
+  ! F'2023 C1027 the LHS may not be coindexed
+  subroutine s11
+    type t
+      real, pointer :: p
+    end type
+    type(t), save :: ca[*]
+    real, target :: x
+    !ERROR: The left-hand side of a pointer assignment must not be coindexed
+    ca[1]%p => x
+  end
+
 end
 
 module m2
diff --git a/flang/test/Semantics/assign11.f90 b/flang/test/Semantics/assign11.f90
index eaa9533409502..37216526b5f33 100644
--- a/flang/test/Semantics/assign11.f90
+++ b/flang/test/Semantics/assign11.f90
@@ -4,9 +4,15 @@ program test
   class(*), allocatable :: pa
   class(*), pointer :: pp
   class(*), allocatable :: pac[:]
+  type t
+    real, allocatable :: a
+  end type
+  type(t) auc[*]
   pa = 1 ! ok
   !ERROR: Left-hand side of assignment may not be polymorphic unless assignment is to an entire allocatable
   pp = 1
   !ERROR: Left-hand side of assignment may not be polymorphic if it is a coarray
   pac = 1
+  !ERROR: Left-hand side of assignment must not be coindexed due to allocatable ultimate component '%a'
+  auc[1] = t()
 end
diff --git a/flang/test/Semantics/bindings01.f90 b/flang/test/Semantics/bindings01.f90
index 7c2dc6448bb3f..dc44db09c4a6f 100644
--- a/flang/test/Semantics/bindings01.f90
+++ b/flang/test/Semantics/bindings01.f90
@@ -293,6 +293,48 @@ subroutine t2p
   end
 end
 
+module m12
+  type t
+    procedure(sub), pointer, nopass :: pp
+   contains
+    procedure, non_overridable, nopass :: tbp1 => sub
+    procedure, nopass :: tbp2 => sub
+    generic :: gen1 => tbp1
+    generic :: gen2 => tbp2
+  end type
+ contains
+  subroutine sub
+  end
+  subroutine test(x, y)
+    class(t) :: x[*]
+    type(t) :: y[*]
+    call x%pp ! ok
+    call y%pp ! ok
+    !ERROR: Base of procedure component reference may not be coindexed
+    call x[1]%pp
+    !ERROR: Base of procedure component reference may not be coindexed
+    call y[1]%pp
+    call x%tbp1 ! ok
+    call y%tbp1 ! ok
+    call x[1]%tbp1 ! ok
+    call y[1]%tbp1 ! ok
+    call x%tbp2 ! ok
+    call y%tbp2 ! ok
+    !ERROR: A procedure binding may not be coindexed unless it can be resolved at compilation time
+    call x[1]%tbp2
+    call y[1]%tbp2 ! ok
+    call x%gen1 ! ok
+    call y%gen1 ! ok
+    call x[1]%gen1 ! ok
+    call y[1]%gen1 ! ok
+    call x%gen2 ! ok
+    call y%gen2 ! ok
+    !ERROR: A procedure binding may not be coindexed unless it can be resolved at compilation time
+    call x[1]%gen2
+    call y[1]%gen2 ! ok
+  end
+end
+
 program test
   use m1
   type,extends(t) :: t2
diff --git a/flang/test/Semantics/c_f_pointer.f90 b/flang/test/Semantics/c_f_pointer.f90
index 6921438ccded0..de5673f8dc1d9 100644
--- a/flang/test/Semantics/c_f_pointer.f90
+++ b/flang/test/Semantics/c_f_pointer.f90
@@ -36,6 +36,7 @@ program test
   call c_f_pointer(scalarC, scalarIntF, [1_8])
   !ERROR: FPTR= argument to C_F_POINTER() may not have a deferred type parameter
   call c_f_pointer(scalarC, charDeferredF)
+  !ERROR: cosubscript 0 is less than lower cobound 1 for codimension 1 of array
   !ERROR: FPTR= argument to C_F_POINTER() may not be a coindexed object
   !ERROR: A coindexed object may not be a pointer target
   call c_f_pointer(scalarC, coindexed[0]%p)
diff --git a/flang/test/Semantics/call04.f90 b/flang/test/Semantics/call04.f90
index 3b079aa4fb2b1..8a99ceee6b637 100644
--- a/flang/test/Semantics/call04.f90
+++ b/flang/test/Semantics/call04.f90
@@ -26,7 +26,7 @@ subroutine s01c(x)
   end subroutine
   subroutine s01b ! C846 - can only be caught at a call via explicit interface
     !ERROR: ALLOCATABLE coarray 'coarray' may not be associated with INTENT(OUT) dummy argument 'x='
-    !ERROR: ALLOCATABLE dummy argument 'x=' has corank 0 but actual argument has corank 1
+    !ERROR: ALLOCATABLE dummy argument 'x=' is not a coarray but actual argument has corank 1
     call s01a(coarray)
     call s01c(coarray) ! ok, dummy is not allocatable
   end subroutine
diff --git a/flang/test/Semantics/call06.f90 b/flang/test/Semantics/call06.f90
index 3e3c5aa61b570..2a4dfc45ea927 100644
--- a/flang/test/Semantics/call06.f90
+++ b/flang/test/Semantics/call06.f90
@@ -41,9 +41,9 @@ subroutine test(x)
     call s01(allofunc()) ! subtle: ALLOCATABLE function result isn't
     call s02(cov) ! ok
     call s03(com) ! ok
-    !ERROR: ALLOCATABLE dummy argument 'x=' has corank 1 but actual argument has corank 2
+    !ERROR: ALLOCATABLE or POINTER dummy argument 'x=' has corank 1 but actual argument has corank 2
     call s02(com)
-    !ERROR: ALLOCATABLE dummy argument 'x=' has corank 2 but actual argument has corank 1
+    !ERROR: ALLOCATABLE or POINTER dummy argument 'x=' has corank 2 but actual argument has corank 1
     call s03(cov)
     call s04(cov[1]) ! ok
     !ERROR: ALLOCATABLE dummy argument 'x=' must have INTENT(IN) to be associated with a coindexed actual argument
diff --git a/flang/test/Semantics/call12.f90 b/flang/test/Semantics/call12.f90
index cd4006a53b3e7..e7c0fd8b9b8cb 100644
--- a/flang/test/Semantics/call12.f90
+++ b/flang/test/Semantics/call12.f90
@@ -104,4 +104,11 @@ pure subroutine internal
       localhp = hasPtr(z%a)
     end subroutine
   end function
+  pure subroutine test2(hpd, hhpd)
+    use used
+    type(hasHiddenPtr), intent(in out) :: hpd, hhpd[*]
+    hpd = hhpd ! ok
+    !ERROR: A pure subprogram may not copy the value of 'hhpd' because it is coindexed and has the POINTER potential subobject component '%a%p'
+    hpd = hhpd[1]
+  end subroutine
 end module
diff --git a/flang/test/Semantics/coarrays02.f90 b/flang/test/Semantics/coarrays02.f90
index a9f4958204936..dc907161250ab 100644
--- a/flang/test/Semantics/coarrays02.f90
+++ b/flang/test/Semantics/coarrays02.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1
 ! More coarray error tests.
-module m
+module m1
   integer :: local[*] ! ok in module
 end
 program main
@@ -48,3 +48,51 @@ function func2()
   !ERROR: Local variable 'local' without the SAVE or ALLOCATABLE attribute may not have a coarray potential subobject component '%comp'
   type(t) :: local
 end
+
+module m2
+  type t0
+    integer n
+  end type
+  type t1
+    class(t0), allocatable :: a
+  end type
+  type t2
+    type(t1) c
+  end type
+ contains
+  subroutine test(x)
+    type(t2), intent(in) :: x[*]
+    !ERROR: The base of a polymorphic object may not be coindexed
+    call sub1(x[1]%c%a)
+    !ERROR: A coindexed designator may not have a type with the polymorphic potential subobject component '%a'
+    call sub2(x[1]%c)
+  end
+  subroutine sub1(x)
+    type(t0), intent(in) :: x
+  end
+  subroutine sub2(x)
+    type(t1), intent(in) :: x
+  end
+end
+
+module m3
+  type t
+    real, allocatable :: a(:)
+    real, pointer :: p(:)
+    real arr(2)
+  end type
+ contains
+  subroutine sub(ca)
+    real, intent(in) :: ca(:)[*]
+  end
+  subroutine test(cat)
+    type(t), intent(in) :: cat[*]
+    call sub(cat%arr(1:2)) ! ok
+    !ERROR: Actual argument associated with coarray dummy argument 'ca=' must be a coarray
+    call sub(cat%arr([1]))
+    !ERROR: Actual argument associated with coarray dummy argument 'ca=' must be a coarray
+    call sub(cat%a)
+    !ERROR: Actual argument associated with coarray dummy argument 'ca=' must be a coarray
+    call sub(cat%p)
+  end
+end
diff --git a/flang/test/Semantics/deallocate05.f90 b/flang/test/Semantics/deallocate05.f90
index bdc6998094064..382957f1005cc 100644
--- a/flang/test/Semantics/deallocate05.f90
+++ b/flang/test/Semantics/deallocate05.f90
@@ -27,6 +27,11 @@ Program deallocatetest
 Character(256) :: ee
 Procedure(Real) :: prp
 
+type at
+  real, allocatable :: a
+end type
+type(at) :: c[*]
+
 Allocate(rp)
 Deallocate(rp)
 
@@ -67,4 +72,7 @@ Program deallocatetest
 !ERROR: ERRMSG may not be duplicated in a DEALLOCATE statement
 Deallocate(x, stat=s, errmsg=ee, errmsg=ee)
 
+!ERROR: Component in DEALLOCATE statement may not be coindexed
+deallocate(c[1]%a)
+
 End Program deallocatetest
diff --git a/flang/test/Semantics/expr-errors06.f90 b/flang/test/Semantics/expr-errors06.f90
index 07cda25e5cb0c..7d4283d35cdcb 100644
--- a/flang/test/Semantics/expr-errors06.f90
+++ b/flang/test/Semantics/expr-errors06.f90
@@ -7,37 +7,37 @@ subroutine subr(da)
   !ERROR: DATA statement designator 'a(0_8)' is out of range
   !ERROR: DATA statement designator 'a(11_8)' is out of range
   data a(0)/0./, a(10+1)/0./
-  !ERROR: Subscript 0 is less than lower bound 1 for dimension 1 of array
+  !ERROR: subscript 0 is less than lower bound 1 for dimension 1 of array
   print *, a(0)
-  !ERROR: Subscript 0 is less than lower bound 1 for dimension 1 of array
+  !ERROR: subscript 0 is less than lower bound 1 for dimension 1 of array
   print *, a(1-1)
-  !ERROR: Subscript 11 is greater than upper bound 10 for dimension 1 of array
+  !ERROR: subscript 11 is greater than upper bound 10 for dimension 1 of array
   print *, a(11)
-  !ERROR: Subscript 11 is greater than upper bound 10 for dimension 1 of array
+  !ERROR: subscript 11 is greater than upper bound 10 for dimension 1 of array
   print *, a(10+1)
   !ERROR: Subscript value (0) is out of range on dimension 1 in reference to a constant array value
   print *, n(0)
   !ERROR: Subscript value (3) is out of range on dimension 1 in reference to a constant array value
   print *, n(4-1)
   print *, a(1:12:3) ! ok
-  !ERROR: Subscript 13 is greater than upper bound 10 for dimension 1 of array
+  !ERROR: subscript 13 is greater than upper bound 10 for dimension 1 of array
   print *, a(1:13:3)
   print *, a(10:-1:-3) ! ok
-  !ERROR: Subscript -2 is less than lower bound 1 for dimension 1 of array
+  !ERROR: subscript -2 is less than lower bound 1 for dimension 1 of array
   print *, a(10:-2:-3)
   print *, a(-1:-2) ! empty section is ok
   print *, a(0:11:-1) ! empty section is ok
-  !ERROR: Subscript 0 is less than lower bound 1 for dimension 1 of array
+  !ERROR: subscript 0 is less than lower bound 1 for dimension 1 of array
   print *, a(0:0:unknown) ! lower==upper, can ignore stride
-  !ERROR: Subscript 11 is greater than upper bound 10 for dimension 1 of array
+  !ERROR: subscript 11 is greater than upper bound 10 for dimension 1 of array
   print *, a(11:11:unknown) ! lower==upper, can ignore stride
-  !ERROR: Subscript 0 is less than lower bound 1 for dimension 1 of array
+  !ERROR: subscript 0 is less than lower bound 1 for dimension 1 of array
   print *, da(0,1)
-  !ERROR: Subscript 3 is greater than upper bound 2 for dimension 1 of array
+  !ERROR: subscript 3 is greater than upper bound 2 for dimension 1 of array
   print *, da(3,1)
-  !ERROR: Subscript 0 is less than lower bound 1 for dimension 2 of array
+  !ERROR: subscript 0 is less than lower bound 1 for dimension 2 of array
   print *, da(1,0)
-  !WARNING: Subscript 2 is greater than upper bound 1 for dimension 2 of array
+  !WARNING: subscript 2 is greater than upper bound 1 for dimension 2 of array
   print *, da(1,2)
   print *, empty([(j,j=1,0)],1) ! ok
   print *, empty(1:0,1) ! ok
diff --git a/flang/test/Semantics/io11.f90 b/flang/test/Semantics/io11.f90
index 7565d35aeb407..3529929003b01 100644
--- a/flang/test/Semantics/io11.f90
+++ b/flang/test/Semantics/io11.f90
@@ -57,7 +57,7 @@ module m3
   private
 contains
   ! Error bad # of args
-  subroutine unformattedReadProc(dtv, unit, iostat, iomsg, iotype) 
+  subroutine unformattedReadProc(dtv, unit, iostat, iomsg, iotype)
     class(t), intent(inout) :: dtv
     integer, intent(in) :: unit
     integer, intent(out) :: iostat
@@ -119,7 +119,7 @@ subroutine formattedReadProc(dtv, unit, iotype, vlist, iostat, iomsg)
 end module m5
 
 module m6
-  interface read(formatted) 
+  interface read(formatted)
     procedure :: formattedReadProc
   end interface
 
@@ -169,7 +169,7 @@ module m8
 contains
   subroutine formattedWriteProc(dtv, unit, iotype, vlist, iostat, iomsg)
     !ERROR: Dummy argument 'dtv' of a defined input/output procedure must have intent 'INTENT(IN)'
-    class(t), intent(inout) :: dtv ! Error, must be intent(inout)
+    class(t), intent(inout) :: dtv ! Error, must be intent(in)
     integer, intent(in) :: unit
     character(len=*), intent(in) :: iotype
     integer, intent(in) :: vlist(:)
@@ -195,7 +195,7 @@ subroutine formattedReadProc(dtv, unit, iotype, vlist, iostat, iomsg)
     !ERROR: Dummy argument 'unit' of a defined input/output procedure may not have any attributes
     integer,  pointer, intent(in) :: unit
     character(len=*), intent(in) :: iotype
-    integer, intent(in) :: vlist(:) 
+    integer, intent(in) :: vlist(:)
     integer, intent(out) :: iostat
     character(len=*), intent(inout) :: iomsg
 
@@ -416,7 +416,7 @@ subroutine formattedReadProc(dtv,unit,iotype,v_list,iostat,iomsg)
 end module
 
 module m19
-  ! Test two different defined input/output procedures specified as a 
+  ! Test two different defined input/output procedures specified as a
   ! type-bound procedure and as a generic for the same derived type
   type t
     integer c
@@ -446,7 +446,7 @@ subroutine unformattedReadProc(dtv,unit,iostat,iomsg)
 end module
 
 module m20
-  ! Test read and write defined input/output procedures specified as a 
+  ! Test read and write defined input/output procedures specified as a
   ! type-bound procedure and as a generic for the same derived type
   type t
     integer c
@@ -744,3 +744,25 @@ subroutine absWrite(dtv, unit, iotype, v_list, iostat, iomsg)
     procedure write2
   end interface
 end
+
+module m29
+  type t
+  end type
+  interface write(formatted)
+    subroutine wf(dtv, unit, iotype, v_list, iostat, iomsg)
+    import t
+    !ERROR: Dummy argument 'dtv' of defined input/output procedure 'wf' may not be a coarray
+    class(t), intent(in) :: dtv[*]
+    !ERROR: Dummy argument 'unit' of defined input/output procedure 'wf' may not be a coarray
+    integer, intent(in) :: unit[*]
+    !ERROR: Dummy argument 'iotype' of defined input/output procedure 'wf' may not be a coarray
+    character(len=*), intent(in) :: iotype[*]
+    !ERROR: Dummy argument 'v_list' of defined input/output procedure 'wf' may not be a coarray
+    integer, intent(in) :: v_list(:)[*]
+    !ERROR: Dummy argument 'iostat' of defined input/output procedure 'wf' may not be a coarray
+    integer, intent(out) :: iostat[*]
+    !ERROR: Dummy argument 'iomsg' of defined input/output procedure 'wf' may not be a coarray
+    character(len=*), intent(inout) :: iomsg[*]
+    end
+  end interface
+end
diff --git a/flang/test/Semantics/lockstmt03.f90 b/flang/test/Semantics/lockstmt03.f90
index 8079bc5c7c85c..8ebe3de62d7ee 100644
--- a/flang/test/Semantics/lockstmt03.f90
+++ b/flang/test/Semantics/lockstmt03.f90
@@ -1,5 +1,4 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1
-! XFAIL: *
 ! This test checks for semantic errors in lock statements based on the
 ! statement specification in section 11.6.10 of the Fortran 2018 standard.
 
@@ -10,14 +9,16 @@ program test_lock_stmt
   character(len=128) error_message, msg_array(10), coindexed_msg[*], repeated_msg
   integer status, stat_array(10), coindexed_int[*], non_bool, repeated_stat
   logical non_integer, bool, bool_array(10), non_char, coindexed_logical[*], repeated_bool
-  type(lock_type) :: lock_var[*], lock_array(10)[*], non_coarray_lock
+  type(lock_type) :: lock_var[*], lock_array(10)[*]
+  !ERROR: Variable 'non_coarray_lock' with EVENT_TYPE or LOCK_TYPE must be a coarray
+  type(lock_type) :: non_coarray_lock
   type(event_type) :: not_lock_var[*]
 
   !___ non-standard-conforming statements ___
 
 ! type mismatches
 
-  !ERROR: to be determined
+  !ERROR: Lock variable must have type LOCK_TYPE from ISO_FORTRAN_ENV
   lock(not_lock_var)
 
   !ERROR: Must have LOGICAL type, but is INTEGER(4)
@@ -45,50 +46,65 @@ program test_lock_stmt
 
 ! corank mismatch
 
-  !ERROR: to be determined
-  lock(non_coarray_lock)
+  lock(non_coarray_lock) ! caught above
 
 ! C1173 - stat-variable and errmsg-variable shall not be a coindexed object
 
-  !ERROR: to be determined
+  !ERROR: The stat-variable or errmsg-variable in a sync-stat-list may not be a coindexed object
   lock(lock_var, stat=coindexed_int[1])
 
-  !ERROR: to be determined
+  !ERROR: The stat-variable or errmsg-variable in a sync-stat-list may not be a coindexed object
   lock(lock_var, errmsg=coindexed_msg[1])
 
-  !ERROR: to be determined
+  !ERROR: The stat-variable or errmsg-variable in a sync-stat-list may not be a coindexed object
+  !ERROR: The stat-variable or errmsg-variable in a sync-stat-list may not be a coindexed object
   lock(lock_var, acquired_lock=coindexed_logical[1], stat=coindexed_int[1], errmsg=coindexed_msg[1])
 
 ! C1181 - No specifier shall appear more than once in a given lock-stat-list
 
-  !ERROR: to be determined
+  !ERROR: Multiple ACQUIRED_LOCK specifiers
   lock(lock_var, acquired_lock=bool, acquired_lock=repeated_bool)
 
-  !ERROR: to be determined
+  !ERROR: The stat-variable in a sync-stat-list may not be repeated
   lock(lock_var, stat=status, stat=repeated_stat)
 
-  !ERROR: to be determined
+  !ERROR: The errmsg-variable in a sync-stat-list may not be repeated
   lock(lock_var, errmsg=error_message, errmsg=repeated_msg)
 
-  !ERROR: to be determined
+  !ERROR: Multiple ACQUIRED_LOCK specifiers
   lock(lock_var, acquired_lock=bool, stat=status, errmsg=error_message, acquired_lock=repeated_bool)
 
-  !ERROR: to be determined
+  !ERROR: The stat-variable in a sync-stat-list may not be repeated
   lock(lock_var, acquired_lock=bool, stat=status, errmsg=error_message, stat=repeated_stat)
 
-  !ERROR: to be determined
+  !ERROR: The errmsg-variable in a sync-stat-list may not be repeated
   lock(lock_var, acquired_lock=bool, stat=status, errmsg=error_message, errmsg=repeated_msg)
 
-  !ERROR: to be determined
+  !ERROR: The stat-variable in a sync-stat-list may not be repeated
+  !ERROR: Multiple ACQUIRED_LOCK specifiers
   lock(lock_var, acquired_lock=bool, stat=status, errmsg=error_message, acquired_lock=repeated_bool, stat=repeated_stat)
 
-  !ERROR: to be determined
+  !ERROR: The errmsg-variable in a sync-stat-list may not be repeated
+  !ERROR: Multiple ACQUIRED_LOCK specifiers
   lock(lock_var, acquired_lock=bool, stat=status, errmsg=error_message, acquired_lock=repeated_bool, errmsg=repeated_msg)
 
-  !ERROR: to be determined
+  !ERROR: The stat-variable in a sync-stat-list may not be repeated
+  !ERROR: The errmsg-variable in a sync-stat-list may not be repeated
   lock(lock_var, acquired_lock=bool, stat=status, errmsg=error_message, stat=repeated_stat, errmsg=repeated_msg)
 
-  !ERROR: to be determined
+  !ERROR: The stat-variable in a sync-stat-list may not be repeated
+  !ERROR: The errmsg-variable in a sync-stat-list may not be repeated
+  !ERROR: Multiple ACQUIRED_LOCK specifiers
   lock(lock_var, acquired_lock=bool, stat=status, errmsg=error_message, acquired_lock=repeated_bool, stat=repeated_stat, errmsg=repeated_msg)
 
+ contains
+  subroutine lockit(x)
+    type(lock_type), intent(in) :: x[*]
+    !ERROR: Lock variable is not definable
+    !BECAUSE: 'x' is an INTENT(IN) dummy argument
+    lock(x)
+    !ERROR: Lock variable is not definable
+    !BECAUSE: 'x' is an INTENT(IN) dummy argument
+    unlock(x)
+  end
 end program test_lock_stmt
diff --git a/flang/test/Semantics/modfile74.f90 b/flang/test/Semantics/modfile74.f90
new file mode 100644
index 0000000000000..f24ce8024c100
--- /dev/null
+++ b/flang/test/Semantics/modfile74.f90
@@ -0,0 +1,19 @@
+! RUN: %python %S/test_modfile.py %s %flang_fc1
+module m
+ contains
+  function f() result(ptr)
+    character :: str
+    pointer(ptr, str)
+    ptr = 0
+  end
+end
+
+!Expect: m.mod
+!module m
+!contains
+!function f() result(ptr)
+!integer(8)::ptr
+!pointer(ptr,str)
+!character(1_8,1)::str
+!end
+!end
diff --git a/flang/test/Semantics/move_alloc.f90 b/flang/test/Semantics/move_alloc.f90
index 3303a002039bb..73db594129eab 100644
--- a/flang/test/Semantics/move_alloc.f90
+++ b/flang/test/Semantics/move_alloc.f90
@@ -1,7 +1,7 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1
 ! Check for semantic errors in move_alloc() subroutine calls
 program main
-  integer, allocatable :: a(:)[:], b(:)[:], f(:), g(:)
+  integer, allocatable :: a(:)[:], b(:)[:], f(:), g(:), h(:)[:,:]
   type alloc_component
     integer, allocatable :: a(:)
   end type
@@ -73,4 +73,7 @@ program main
   !ERROR: Argument #2 to MOVE_ALLOC must be allocatable
   call move_alloc(f, g(::2))
 
+  !ERROR: FROM= argument to MOVE_ALLOC has corank 1, but TO= argument has corank 2
+  call move_alloc(a, h)
+
 end program main
diff --git a/flang/test/Semantics/pdt04.f90 b/flang/test/Semantics/pdt04.f90
new file mode 100644
index 0000000000000..7578a763cc947
--- /dev/null
+++ b/flang/test/Semantics/pdt04.f90
@@ -0,0 +1,11 @@
+!RUN: not %flang_fc1 %s 2>&1 | FileCheck %s
+!CHECK: error: KIND parameter expression (int(1_4/0_4,kind=8)) of intrinsic type CHARACTER did not resolve to a constant value
+!CHECK: in the context: instantiation of parameterized derived type 'ty(j=1_4,k=0_4)'
+!CHECK: warning: INTEGER(4) division by zero
+program main
+  type ty(j,k)
+    integer, kind :: j, k
+    character(kind=j/k) a
+  end type
+  type(ty(1,0)) x
+end
diff --git a/flang/test/Semantics/resolve105.f90 b/flang/test/Semantics/resolve105.f90
index 13623207c0bf7..5c2c7e40b0bde 100644
--- a/flang/test/Semantics/resolve105.f90
+++ b/flang/test/Semantics/resolve105.f90
@@ -1,9 +1,9 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1
 ! Test instantiation of components that are procedure pointers.
-! 
 program test
   type dtype(kindParam)
     integer, kind :: kindParam = 4
+    !ERROR: KIND parameter expression (kindparam) of intrinsic type REAL did not resolve to a constant value
     !ERROR: KIND parameter value (66) of intrinsic type REAL did not resolve to a supported value
     !ERROR: KIND parameter value (55) of intrinsic type REAL did not resolve to a supported value
     procedure (real(kindParam)), pointer, nopass :: field => null()
diff --git a/libc/CMakeLists.txt b/libc/CMakeLists.txt
index daec56ffd8ccb..ad39ff6fbcb1e 100644
--- a/libc/CMakeLists.txt
+++ b/libc/CMakeLists.txt
@@ -34,6 +34,7 @@ if (CMAKE_BUILD_TYPE STREQUAL "Debug")
   add_definitions("-D_DEBUG")
 endif()
 
+
 # Default to C++17
 set(CMAKE_CXX_STANDARD 17)
 
diff --git a/libc/cmake/modules/LLVMLibCTestRules.cmake b/libc/cmake/modules/LLVMLibCTestRules.cmake
index 03b4b251649e7..45a36abd8ce1b 100644
--- a/libc/cmake/modules/LLVMLibCTestRules.cmake
+++ b/libc/cmake/modules/LLVMLibCTestRules.cmake
@@ -36,8 +36,9 @@ function(_get_common_test_compile_options output_var c_test flags)
     if(NOT LIBC_WNO_ERROR)
       # list(APPEND compile_options "-Werror")
     endif()
-    # list(APPEND compile_options "-Wconversion")
-    # list(APPEND compile_options "-Wno-sign-conversion")
+    list(APPEND compile_options "-Wconversion")
+    # FIXME: convert to -Wsign-conversion
+    list(APPEND compile_options "-Wno-sign-conversion")
     list(APPEND compile_options "-Wimplicit-fallthrough")
     list(APPEND compile_options "-Wwrite-strings")
     # Silence this warning because _Complex is a part of C99.
diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 6c2be4d3b0b99..ab1917259519b 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -93,6 +93,9 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.strings.bcmp
     libc.src.strings.bcopy
     libc.src.strings.bzero
+    libc.src.strings.ffs
+    libc.src.strings.ffsl
+    libc.src.strings.ffsll
     libc.src.strings.index
     libc.src.strings.rindex
     libc.src.strings.strcasecmp
@@ -908,6 +911,10 @@ if(LLVM_LIBC_FULL_BUILD)
     # sched.h entrypoints
     libc.src.sched.__sched_getcpucount
 
+    # strings.h entrypoints
+    libc.src.strings.strcasecmp_l
+    libc.src.strings.strncasecmp_l
+
     # setjmp.h entrypoints
     libc.src.setjmp.longjmp
     libc.src.setjmp.setjmp
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 779654422e35b..a29478898fe70 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -93,6 +93,9 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.strings.bcmp
     libc.src.strings.bcopy
     libc.src.strings.bzero
+    libc.src.strings.ffs
+    libc.src.strings.ffsl
+    libc.src.strings.ffsll
     libc.src.strings.index
     libc.src.strings.rindex
     libc.src.strings.strcasecmp
@@ -935,6 +938,10 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.string.strcoll_l
     libc.src.string.strxfrm_l
 
+    # strings.h entrypoints
+    libc.src.strings.strcasecmp_l
+    libc.src.strings.strncasecmp_l
+
     # assert.h entrypoints
     libc.src.assert.__assert_fail
 
diff --git a/libc/docs/headers/stdfix.rst b/libc/docs/headers/stdfix.rst
index 5189439c63306..20f68b4a1ed35 100644
--- a/libc/docs/headers/stdfix.rst
+++ b/libc/docs/headers/stdfix.rst
@@ -1,6 +1,6 @@
-================
-StdFix Functions
-================
+========
+stdfix.h
+========
 
 .. include:: ../check.rst
 
@@ -89,9 +89,9 @@ The following functions are included in the ISO/IEC TR 18037:2008 standard.
 ================== =========
 Type Generic Macro Available
 ================== =========
-absfx
-countlsfx
-roundfx
+absfx              |check|
+countlsfx          |check|
+roundfx            |check|
 ================== =========
 
 
diff --git a/libc/hdr/CMakeLists.txt b/libc/hdr/CMakeLists.txt
index 7f523c50e8694..b337a8c9fc2a6 100644
--- a/libc/hdr/CMakeLists.txt
+++ b/libc/hdr/CMakeLists.txt
@@ -183,6 +183,15 @@ add_proxy_header_library(
     libc.include.link
 )
 
+add_proxy_header_library(
+  locale_macros
+  HDRS
+    locale_macros.h
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-macros.locale_macros
+    libc.include.locale
+)
+
 add_proxy_header_library(
   sys_auxv_macros
   HDRS
diff --git a/libc/hdr/locale_macros.h b/libc/hdr/locale_macros.h
new file mode 100644
index 0000000000000..7d94f6ddabe41
--- /dev/null
+++ b/libc/hdr/locale_macros.h
@@ -0,0 +1,22 @@
+//===-- Definition of macros from locale.h --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_LOCALE_MACROS_H
+#define LLVM_LIBC_HDR_LOCALE_MACROS_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-macros/locale-macros.h"
+
+#else // Overlay mode
+
+#error "macros not available in overlay mode"
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_LOCALE_MACROS_H
diff --git a/libc/include/llvm-libc-macros/stdfix-macros.h b/libc/include/llvm-libc-macros/stdfix-macros.h
index 554ebe544a42e..04097e14e9747 100644
--- a/libc/include/llvm-libc-macros/stdfix-macros.h
+++ b/libc/include/llvm-libc-macros/stdfix-macros.h
@@ -323,6 +323,45 @@
 #define ULACCUM_EPSILON 0x1.0p-32ULK
 #endif // ULACCUM_EPSILON
 
+#define absfx(x)                                                               \
+  _Generic((x),                                                                \
+      fract: absr,                                                             \
+      short fract: abshr,                                                      \
+      long fract: abslr,                                                       \
+      accum: absk,                                                             \
+      short accum: abshk,                                                      \
+      long accum: abslk)(x)
+
+#define countlsfx(x)                                                           \
+  _Generic((x),                                                                \
+      fract: countlsr,                                                         \
+      short fract: countlshr,                                                  \
+      long fract: countlslr,                                                   \
+      accum: countlsk,                                                         \
+      short accum: countlshk,                                                  \
+      long accum: countlslk,                                                   \
+      unsigned fract: countlsur,                                               \
+      unsigned short fract: countlsuhr,                                        \
+      unsigned long fract: countlsulr,                                         \
+      unsigned accum: countlsuk,                                               \
+      unsigned short accum: countlsuhk,                                        \
+      unsigned long accum: countlsulk)(x)
+
+#define roundfx(x, y)                                                          \
+  _Generic((x),                                                                \
+      fract: roundr,                                                           \
+      short fract: roundhr,                                                    \
+      long fract: roundlr,                                                     \
+      accum: roundk,                                                           \
+      short accum: roundhk,                                                    \
+      long accum: roundlk,                                                     \
+      unsigned fract: roundur,                                                 \
+      unsigned short fract: rounduhr,                                          \
+      unsigned long fract: roundulr,                                           \
+      unsigned accum: rounduk,                                                 \
+      unsigned short accum: rounduhk,                                          \
+      unsigned long accum: roundulk)(x, y)
+
 #endif // LIBC_COMPILER_HAS_FIXED_POINT
 
 #endif // LLVM_LIBC_MACROS_STDFIX_MACROS_H
diff --git a/libc/include/stdfix.yaml b/libc/include/stdfix.yaml
index 707e5a2b09819..29c4d8b5c8960 100644
--- a/libc/include/stdfix.yaml
+++ b/libc/include/stdfix.yaml
@@ -1,6 +1,12 @@
 header: stdfix.h
 header_template: stdfix.h.def
-macros: []
+macros:
+  - macro_name: absfx
+    macro_header: stdfix-macros.h
+  - macro_name: countlsfx
+    macro_header: stdfix-macros.h
+  - macro_name: roundfx
+    macro_header: stdfix-macros.h
 types:
   - type_name: stdfix-types
 enums: []
diff --git a/libc/include/strings.yaml b/libc/include/strings.yaml
index b6aa8f6d60b27..855800d9dbc3d 100644
--- a/libc/include/strings.yaml
+++ b/libc/include/strings.yaml
@@ -3,6 +3,7 @@ header_template: strings.h.def
 macros: []
 types:
   - type_name: size_t
+  - type_name: locale_t
 enums: []
 objects: []
 functions:
@@ -29,6 +30,24 @@ functions:
     arguments:
       - type: void *
       - type: size_t
+  - name: ffs
+    standards:
+      - POSIX
+    return_type: int
+    arguments:
+      - type: int
+  - name: ffsl
+    standards:
+      - POSIX
+    return_type: int
+    arguments:
+      - type: long
+  - name: ffsll
+    standards:
+      - POSIX
+    return_type: int
+    arguments:
+      - type: long long
   - name: index
     standards:
       - BSDExtensions
@@ -50,6 +69,14 @@ functions:
     arguments:
       - type: const char *
       - type: const char *
+  - name: strcasecmp_l
+    standards:
+      - BSDExtensions
+    return_type: int
+    arguments:
+      - type: const char *
+      - type: const char *
+      - type: locale_t
   - name: strncasecmp
     standards:
       - BSDExtensions
@@ -58,3 +85,12 @@ functions:
       - type: const char *
       - type: const char *
       - type: size_t
+  - name: strncasecmp_l
+    standards:
+      - BSDExtensions
+    return_type: int
+    arguments:
+      - type: const char *
+      - type: const char *
+      - type: size_t
+      - type: locale_t
diff --git a/libc/src/__support/CPP/bit.h b/libc/src/__support/CPP/bit.h
index 82b9eb5128262..e491f3e032669 100644
--- a/libc/src/__support/CPP/bit.h
+++ b/libc/src/__support/CPP/bit.h
@@ -101,7 +101,7 @@ countr_zero(T value) {
     shift >>= 1;
     mask >>= shift;
   }
-  return zero_bits;
+  return static_cast<int>(zero_bits);
 }
 #if __has_builtin(__builtin_ctzs)
 ADD_SPECIALIZATION(countr_zero, unsigned short, __builtin_ctzs)
@@ -140,7 +140,7 @@ countl_zero(T value) {
     else
       zero_bits |= shift;
   }
-  return zero_bits;
+  return static_cast<int>(zero_bits);
 }
 #if __has_builtin(__builtin_clzs)
 ADD_SPECIALIZATION(countl_zero, unsigned short, __builtin_clzs)
@@ -162,7 +162,7 @@ ADD_SPECIALIZATION(countl_zero, unsigned long long, __builtin_clzll)
 template <typename T>
 [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t<cpp::is_unsigned_v<T>, int>
 countl_one(T value) {
-  return cpp::countl_zero<T>(~value);
+  return cpp::countl_zero<T>(static_cast<T>(~value));
 }
 
 /// Count the number of ones from the least significant bit to the first
@@ -175,7 +175,7 @@ countl_one(T value) {
 template <typename T>
 [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t<cpp::is_unsigned_v<T>, int>
 countr_one(T value) {
-  return cpp::countr_zero<T>(~value);
+  return cpp::countr_zero<T>(static_cast<T>(~value));
 }
 
 /// Returns the number of bits needed to represent value if value is nonzero.
@@ -226,7 +226,7 @@ rotr(T value, int rotate);
 template <typename T>
 [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t<cpp::is_unsigned_v<T>, T>
 rotl(T value, int rotate) {
-  constexpr unsigned N = cpp::numeric_limits<T>::digits;
+  constexpr int N = cpp::numeric_limits<T>::digits;
   rotate = rotate % N;
   if (!rotate)
     return value;
@@ -238,7 +238,7 @@ rotl(T value, int rotate) {
 template <typename T>
 [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t<cpp::is_unsigned_v<T>, T>
 rotr(T value, int rotate) {
-  constexpr unsigned N = cpp::numeric_limits<T>::digits;
+  constexpr int N = cpp::numeric_limits<T>::digits;
   rotate = rotate % N;
   if (!rotate)
     return value;
diff --git a/libc/src/__support/CPP/span.h b/libc/src/__support/CPP/span.h
index a41c9b744e370..9234a26d201cd 100644
--- a/libc/src/__support/CPP/span.h
+++ b/libc/src/__support/CPP/span.h
@@ -11,6 +11,7 @@
 #include <stddef.h> // For size_t
 
 #include "array.h" // For array
+#include "limits.h"
 #include "src/__support/macros/config.h"
 #include "type_traits.h" // For remove_cv_t, enable_if_t, is_same_v, is_const_v
 
@@ -48,7 +49,8 @@ template <typename T> class span {
   using const_reference = const T &;
   using iterator = T *;
 
-  LIBC_INLINE_VAR static constexpr size_type dynamic_extent = -1;
+  LIBC_INLINE_VAR static constexpr size_type dynamic_extent =
+      cpp::numeric_limits<size_type>::max();
 
   LIBC_INLINE constexpr span() : span_data(nullptr), span_size(0) {}
 
@@ -58,7 +60,7 @@ template <typename T> class span {
       : span_data(first), span_size(count) {}
 
   LIBC_INLINE constexpr span(pointer first, pointer end)
-      : span_data(first), span_size(end - first) {}
+      : span_data(first), span_size(static_cast<size_t>(end - first)) {}
 
   template <typename U, size_t N,
             cpp::enable_if_t<is_compatible_v<U>, bool> = true>
diff --git a/libc/src/__support/CPP/string.h b/libc/src/__support/CPP/string.h
index dbc0ae04e5e6f..1ac04c7f1f9dc 100644
--- a/libc/src/__support/CPP/string.h
+++ b/libc/src/__support/CPP/string.h
@@ -67,7 +67,8 @@ class string {
       : string(cstr, ::LIBC_NAMESPACE::internal::string_length(cstr)) {}
   LIBC_INLINE string(size_t size_, char value) {
     resize(size_);
-    inline_memset((void *)buffer_, value, size_);
+    static_assert(sizeof(char) == sizeof(uint8_t));
+    inline_memset((void *)buffer_, static_cast<uint8_t>(value), size_);
   }
 
   LIBC_INLINE string &operator=(const string &other) {
diff --git a/libc/src/__support/CPP/string_view.h b/libc/src/__support/CPP/string_view.h
index 745c62c35f0a0..aa15814b2e149 100644
--- a/libc/src/__support/CPP/string_view.h
+++ b/libc/src/__support/CPP/string_view.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIBC_SRC___SUPPORT_CPP_STRING_VIEW_H
 #define LLVM_LIBC_SRC___SUPPORT_CPP_STRING_VIEW_H
 
+#include "limits.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
 
@@ -40,7 +41,7 @@ class string_view {
   LIBC_INLINE static constexpr size_t length(const char *Str) {
     for (const char *End = Str;; ++End)
       if (*End == '\0')
-        return End - Str;
+        return static_cast<size_t>(End - Str);
   }
 
   LIBC_INLINE bool equals(string_view Other) const {
@@ -61,7 +62,8 @@ class string_view {
 
   // special value equal to the maximum value representable by the type
   // size_type.
-  LIBC_INLINE_VAR static constexpr size_t npos = -1;
+  LIBC_INLINE_VAR static constexpr size_t npos =
+      cpp::numeric_limits<size_t>::max();
 
   LIBC_INLINE constexpr string_view() : Data(nullptr), Len(0) {}
 
diff --git a/libc/src/__support/FPUtil/FMA.h b/libc/src/__support/FPUtil/FMA.h
index 1e40d06dc1462..2cafb4c0974e3 100644
--- a/libc/src/__support/FPUtil/FMA.h
+++ b/libc/src/__support/FPUtil/FMA.h
@@ -24,6 +24,8 @@ LIBC_INLINE OutType fma(InType x, InType y, InType z) {
 }
 
 #ifdef LIBC_TARGET_CPU_HAS_FMA
+
+#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT
 template <> LIBC_INLINE float fma(float x, float y, float z) {
 #if __has_builtin(__builtin_elementwise_fma)
   return __builtin_elementwise_fma(x, y, z);
@@ -31,7 +33,9 @@ template <> LIBC_INLINE float fma(float x, float y, float z) {
   return __builtin_fmaf(x, y, z);
 #endif
 }
+#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
 
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 template <> LIBC_INLINE double fma(double x, double y, double z) {
 #if __has_builtin(__builtin_elementwise_fma)
   return __builtin_elementwise_fma(x, y, z);
@@ -39,6 +43,7 @@ template <> LIBC_INLINE double fma(double x, double y, double z) {
   return __builtin_fma(x, y, z);
 #endif
 }
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 #endif // LIBC_TARGET_CPU_HAS_FMA
 
 } // namespace fputil
diff --git a/libc/src/__support/FPUtil/FPBits.h b/libc/src/__support/FPUtil/FPBits.h
index 90b6e406e0f31..bee8d0a8dc47d 100644
--- a/libc/src/__support/FPUtil/FPBits.h
+++ b/libc/src/__support/FPUtil/FPBits.h
@@ -247,11 +247,11 @@ template <FPType fp_type> struct FPStorage : public FPLayout<fp_type> {
     using UP::UP;
 
     LIBC_INLINE constexpr BiasedExponent(Exponent exp)
-        : UP(static_cast<int32_t>(exp) + EXP_BIAS) {}
+        : UP(static_cast<uint32_t>(static_cast<int32_t>(exp) + EXP_BIAS)) {}
 
     // Cast operator to get convert from BiasedExponent to Exponent.
     LIBC_INLINE constexpr operator Exponent() const {
-      return Exponent(UP::value - EXP_BIAS);
+      return Exponent(static_cast<int32_t>(UP::value - EXP_BIAS));
     }
 
     LIBC_INLINE constexpr BiasedExponent &operator++() {
@@ -686,7 +686,7 @@ struct FPRepImpl : public FPRepSem<fp_type, RetT> {
   }
 
   LIBC_INLINE constexpr void set_biased_exponent(StorageType biased) {
-    UP::set_biased_exponent(BiasedExponent((int32_t)biased));
+    UP::set_biased_exponent(BiasedExponent(static_cast<uint32_t>(biased)));
   }
 
   LIBC_INLINE constexpr int get_exponent() const {
diff --git a/libc/src/__support/FPUtil/NormalFloat.h b/libc/src/__support/FPUtil/NormalFloat.h
index b4cbb5042a68b..a2f285fc6fb95 100644
--- a/libc/src/__support/FPUtil/NormalFloat.h
+++ b/libc/src/__support/FPUtil/NormalFloat.h
@@ -105,7 +105,7 @@ template <typename T> struct NormalFloat {
 
     constexpr int SUBNORMAL_EXPONENT = -FPBits<T>::EXP_BIAS + 1;
     if (exponent < SUBNORMAL_EXPONENT) {
-      unsigned shift = SUBNORMAL_EXPONENT - exponent;
+      unsigned shift = static_cast<unsigned>(SUBNORMAL_EXPONENT - exponent);
       // Since exponent > subnormalExponent, shift is strictly greater than
       // zero.
       if (shift <= FPBits<T>::FRACTION_LEN + 1) {
@@ -160,7 +160,7 @@ template <typename T> struct NormalFloat {
     if (bits.is_subnormal()) {
       unsigned shift = evaluate_normalization_shift(bits.get_mantissa());
       mantissa = static_cast<StorageType>(bits.get_mantissa() << shift);
-      exponent = 1 - FPBits<T>::EXP_BIAS - shift;
+      exponent = 1 - FPBits<T>::EXP_BIAS - static_cast<int32_t>(shift);
     } else {
       exponent = bits.get_biased_exponent() - FPBits<T>::EXP_BIAS;
       mantissa = ONE | bits.get_mantissa();
diff --git a/libc/src/__support/FPUtil/aarch64/FEnvImpl.h b/libc/src/__support/FPUtil/aarch64/FEnvImpl.h
index 18b0631324f8f..914155a01631d 100644
--- a/libc/src/__support/FPUtil/aarch64/FEnvImpl.h
+++ b/libc/src/__support/FPUtil/aarch64/FEnvImpl.h
@@ -110,7 +110,7 @@ LIBC_INLINE int enable_except(int excepts) {
       (controlWord >> FEnv::ExceptionControlFlagsBitPosition) & 0x1F;
   controlWord |= (newExcepts << FEnv::ExceptionControlFlagsBitPosition);
   FEnv::writeControlWord(controlWord);
-  return FEnv::exceptionStatusToMacro(oldExcepts);
+  return FEnv::exceptionStatusToMacro(static_cast<uint32_t>(oldExcepts));
 }
 
 LIBC_INLINE int disable_except(int excepts) {
@@ -120,12 +120,12 @@ LIBC_INLINE int disable_except(int excepts) {
       (controlWord >> FEnv::ExceptionControlFlagsBitPosition) & 0x1F;
   controlWord &= ~(disabledExcepts << FEnv::ExceptionControlFlagsBitPosition);
   FEnv::writeControlWord(controlWord);
-  return FEnv::exceptionStatusToMacro(oldExcepts);
+  return FEnv::exceptionStatusToMacro(static_cast<uint32_t>(oldExcepts));
 }
 
 LIBC_INLINE int get_except() {
   uint32_t controlWord = FEnv::getControlWord();
-  int enabledExcepts =
+  uint32_t enabledExcepts =
       (controlWord >> FEnv::ExceptionControlFlagsBitPosition) & 0x1F;
   return FEnv::exceptionStatusToMacro(enabledExcepts);
 }
@@ -250,8 +250,10 @@ LIBC_INLINE int set_round(int mode) {
   }
 
   uint32_t controlWord = FEnv::getControlWord();
-  controlWord &= ~(0x3 << FEnv::RoundingControlBitPosition);
-  controlWord |= (bitValue << FEnv::RoundingControlBitPosition);
+  controlWord &=
+      static_cast<uint32_t>(~(0x3 << FEnv::RoundingControlBitPosition));
+  controlWord |=
+      static_cast<uint32_t>(bitValue << FEnv::RoundingControlBitPosition);
   FEnv::writeControlWord(controlWord);
 
   return 0;
diff --git a/libc/src/__support/FPUtil/aarch64/fenv_darwin_impl.h b/libc/src/__support/FPUtil/aarch64/fenv_darwin_impl.h
index 969e70796d1f1..dcce76b6116be 100644
--- a/libc/src/__support/FPUtil/aarch64/fenv_darwin_impl.h
+++ b/libc/src/__support/FPUtil/aarch64/fenv_darwin_impl.h
@@ -63,7 +63,7 @@ struct FEnv {
   // __fpcr_flush_to_zero bit in the FPCR register.  This control bit is
   // located in a different place from FE_FLUSHTOZERO status bit relative to
   // the other exceptions.
-  LIBC_INLINE static uint32_t exception_value_from_status(int status) {
+  LIBC_INLINE static uint32_t exception_value_from_status(uint32_t status) {
     return ((status & FE_INVALID) ? EX_INVALID : 0) |
            ((status & FE_DIVBYZERO) ? EX_DIVBYZERO : 0) |
            ((status & FE_OVERFLOW) ? EX_OVERFLOW : 0) |
@@ -72,7 +72,7 @@ struct FEnv {
            ((status & FE_FLUSHTOZERO) ? EX_FLUSHTOZERO : 0);
   }
 
-  LIBC_INLINE static uint32_t exception_value_from_control(int control) {
+  LIBC_INLINE static uint32_t exception_value_from_control(uint32_t control) {
     return ((control & __fpcr_trap_invalid) ? EX_INVALID : 0) |
            ((control & __fpcr_trap_divbyzero) ? EX_DIVBYZERO : 0) |
            ((control & __fpcr_trap_overflow) ? EX_OVERFLOW : 0) |
@@ -81,7 +81,7 @@ struct FEnv {
            ((control & __fpcr_flush_to_zero) ? EX_FLUSHTOZERO : 0);
   }
 
-  LIBC_INLINE static int exception_value_to_status(uint32_t excepts) {
+  LIBC_INLINE static uint32_t exception_value_to_status(uint32_t excepts) {
     return ((excepts & EX_INVALID) ? FE_INVALID : 0) |
            ((excepts & EX_DIVBYZERO) ? FE_DIVBYZERO : 0) |
            ((excepts & EX_OVERFLOW) ? FE_OVERFLOW : 0) |
@@ -90,7 +90,7 @@ struct FEnv {
            ((excepts & EX_FLUSHTOZERO) ? FE_FLUSHTOZERO : 0);
   }
 
-  LIBC_INLINE static int exception_value_to_control(uint32_t excepts) {
+  LIBC_INLINE static uint32_t exception_value_to_control(uint32_t excepts) {
     return ((excepts & EX_INVALID) ? __fpcr_trap_invalid : 0) |
            ((excepts & EX_DIVBYZERO) ? __fpcr_trap_divbyzero : 0) |
            ((excepts & EX_OVERFLOW) ? __fpcr_trap_overflow : 0) |
@@ -113,34 +113,37 @@ struct FEnv {
 };
 
 LIBC_INLINE int enable_except(int excepts) {
-  uint32_t new_excepts = FEnv::exception_value_from_status(excepts);
+  uint32_t new_excepts =
+      FEnv::exception_value_from_status(static_cast<uint32_t>(excepts));
   uint32_t control_word = FEnv::get_control_word();
   uint32_t old_excepts = FEnv::exception_value_from_control(control_word);
   if (new_excepts != old_excepts) {
     control_word |= FEnv::exception_value_to_control(new_excepts);
     FEnv::set_control_word(control_word);
   }
-  return FEnv::exception_value_to_status(old_excepts);
+  return static_cast<int>(FEnv::exception_value_to_status(old_excepts));
 }
 
 LIBC_INLINE int disable_except(int excepts) {
-  uint32_t disabled_excepts = FEnv::exception_value_from_status(excepts);
+  uint32_t disabled_excepts =
+      FEnv::exception_value_from_status(static_cast<uint32_t>(excepts));
   uint32_t control_word = FEnv::get_control_word();
   uint32_t old_excepts = FEnv::exception_value_from_control(control_word);
   control_word &= ~FEnv::exception_value_to_control(disabled_excepts);
   FEnv::set_control_word(control_word);
-  return FEnv::exception_value_to_status(old_excepts);
+  return static_cast<int>(FEnv::exception_value_to_status(old_excepts));
 }
 
 LIBC_INLINE int get_except() {
   uint32_t control_word = FEnv::get_control_word();
   uint32_t enabled_excepts = FEnv::exception_value_from_control(control_word);
-  return FEnv::exception_value_to_status(enabled_excepts);
+  return static_cast<int>(FEnv::exception_value_to_status(enabled_excepts));
 }
 
 LIBC_INLINE int clear_except(int excepts) {
   uint32_t status_word = FEnv::get_status_word();
-  uint32_t except_value = FEnv::exception_value_from_status(excepts);
+  uint32_t except_value =
+      FEnv::exception_value_from_status(static_cast<uint32_t>(excepts));
   status_word &= ~FEnv::exception_value_to_status(except_value);
   FEnv::set_status_word(status_word);
   return 0;
@@ -148,13 +151,16 @@ LIBC_INLINE int clear_except(int excepts) {
 
 LIBC_INLINE int test_except(int excepts) {
   uint32_t statusWord = FEnv::get_status_word();
-  uint32_t ex_value = FEnv::exception_value_from_status(excepts);
-  return statusWord & FEnv::exception_value_to_status(ex_value);
+  uint32_t ex_value =
+      FEnv::exception_value_from_status(static_cast<uint32_t>(excepts));
+  return static_cast<int>(statusWord &
+                          FEnv::exception_value_to_status(ex_value));
 }
 
 LIBC_INLINE int set_except(int excepts) {
   uint32_t status_word = FEnv::get_status_word();
-  uint32_t new_exceptions = FEnv::exception_value_from_status(excepts);
+  uint32_t new_exceptions =
+      FEnv::exception_value_from_status(static_cast<uint32_t>(excepts));
   status_word |= FEnv::exception_value_to_status(new_exceptions);
   FEnv::set_status_word(status_word);
   return 0;
@@ -174,7 +180,8 @@ LIBC_INLINE int raise_except(int excepts) {
                          : "s0", "s1" /* s0 and s1 are clobbered */);
   };
 
-  uint32_t to_raise = FEnv::exception_value_from_status(excepts);
+  uint32_t to_raise =
+      FEnv::exception_value_from_status(static_cast<uint32_t>(excepts));
   int result = 0;
 
   if (to_raise & FEnv::EX_INVALID) {
@@ -237,7 +244,7 @@ LIBC_INLINE int get_round() {
 }
 
 LIBC_INLINE int set_round(int mode) {
-  uint16_t bit_value;
+  uint32_t bit_value;
   switch (mode) {
   case FE_TONEAREST:
     bit_value = FEnv::TONEAREST;
@@ -256,7 +263,7 @@ LIBC_INLINE int set_round(int mode) {
   }
 
   uint32_t control_word = FEnv::get_control_word();
-  control_word &= ~(0x3 << FEnv::ROUNDING_CONTROL_BIT_POSITION);
+  control_word &= ~(0x3u << FEnv::ROUNDING_CONTROL_BIT_POSITION);
   control_word |= (bit_value << FEnv::ROUNDING_CONTROL_BIT_POSITION);
   FEnv::set_control_word(control_word);
 
diff --git a/libc/src/__support/FPUtil/double_double.h b/libc/src/__support/FPUtil/double_double.h
index b24ffd4aa456f..c27885aadc028 100644
--- a/libc/src/__support/FPUtil/double_double.h
+++ b/libc/src/__support/FPUtil/double_double.h
@@ -100,6 +100,26 @@ LIBC_INLINE NumberPair<T> exact_mult(const NumberPair<T> &as, T a, T b) {
   return r;
 }
 
+// The templated exact multiplication needs template version of
+// LIBC_TARGET_CPU_HAS_FMA_* macro to correctly select the implementation.
+// These can be moved to "src/__support/macros/properties/cpu_features.h" if
+// other part of libc needed.
+template <typename T> struct TargetHasFmaInstruction {
+  static constexpr bool VALUE = false;
+};
+
+#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT
+template <> struct TargetHasFmaInstruction<float> {
+  static constexpr bool VALUE = true;
+};
+#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
+
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
+template <> struct TargetHasFmaInstruction<double> {
+  static constexpr bool VALUE = true;
+};
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
+
 // Note: When FMA instruction is not available, the `exact_mult` function is
 // only correct for round-to-nearest mode.  See:
 //   Zimmermann, P., "Note on the Veltkamp/Dekker Algorithms with Directed
@@ -111,15 +131,15 @@ template <typename T = double, size_t SPLIT_B = DefaultSplit<T>::VALUE>
 LIBC_INLINE NumberPair<T> exact_mult(T a, T b) {
   NumberPair<T> r{0.0, 0.0};
 
-#ifdef LIBC_TARGET_CPU_HAS_FMA
-  r.hi = a * b;
-  r.lo = fputil::multiply_add(a, b, -r.hi);
-#else
-  // Dekker's Product.
-  NumberPair<T> as = split(a);
+  if constexpr (TargetHasFmaInstruction<T>::VALUE) {
+    r.hi = a * b;
+    r.lo = fputil::multiply_add(a, b, -r.hi);
+  } else {
+    // Dekker's Product.
+    NumberPair<T> as = split(a);
 
-  r = exact_mult<T, SPLIT_B>(as, a, b);
-#endif // LIBC_TARGET_CPU_HAS_FMA
+    r = exact_mult<T, SPLIT_B>(as, a, b);
+  }
 
   return r;
 }
diff --git a/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h b/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h
index 9492d52da0455..0ba836d17a085 100644
--- a/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h
+++ b/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h
@@ -24,7 +24,7 @@ namespace x86 {
 LIBC_INLINE void normalize(int &exponent,
                            FPBits<long double>::StorageType &mantissa) {
   const unsigned int shift = static_cast<unsigned int>(
-      cpp::countl_zero(static_cast<uint64_t>(mantissa)) -
+      static_cast<size_t>(cpp::countl_zero(static_cast<uint64_t>(mantissa))) -
       (8 * sizeof(uint64_t) - 1 - FPBits<long double>::FRACTION_LEN));
   exponent -= shift;
   mantissa <<= shift;
diff --git a/libc/src/__support/FPUtil/multiply_add.h b/libc/src/__support/FPUtil/multiply_add.h
index ae00e08673d08..8260702e2c9f4 100644
--- a/libc/src/__support/FPUtil/multiply_add.h
+++ b/libc/src/__support/FPUtil/multiply_add.h
@@ -46,6 +46,7 @@ multiply_add(T x, T y, T z) {
 namespace LIBC_NAMESPACE_DECL {
 namespace fputil {
 
+#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT
 LIBC_INLINE float multiply_add(float x, float y, float z) {
 #if __has_builtin(__builtin_elementwise_fma)
   return __builtin_elementwise_fma(x, y, z);
@@ -53,7 +54,9 @@ LIBC_INLINE float multiply_add(float x, float y, float z) {
   return __builtin_fmaf(x, y, z);
 #endif
 }
+#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
 
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 LIBC_INLINE double multiply_add(double x, double y, double z) {
 #if __has_builtin(__builtin_elementwise_fma)
   return __builtin_elementwise_fma(x, y, z);
@@ -61,6 +64,7 @@ LIBC_INLINE double multiply_add(double x, double y, double z) {
   return __builtin_fma(x, y, z);
 #endif
 }
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
 } // namespace fputil
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/OSUtil/darwin/io.h b/libc/src/__support/OSUtil/darwin/io.h
index a5f7ecbd70362..69df99da522fb 100644
--- a/libc/src/__support/OSUtil/darwin/io.h
+++ b/libc/src/__support/OSUtil/darwin/io.h
@@ -17,7 +17,8 @@ namespace LIBC_NAMESPACE_DECL {
 
 LIBC_INLINE void write_to_stderr(cpp::string_view msg) {
   LIBC_NAMESPACE::syscall_impl(4 /*SYS_write*/, 2 /* stderr */,
-                               reinterpret_cast<long>(msg.data()), msg.size());
+                               reinterpret_cast<long>(msg.data()),
+                               static_cast<long>(msg.size()));
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/big_int.h b/libc/src/__support/big_int.h
index e726a094b5dac..fa06bb485de2c 100644
--- a/libc/src/__support/big_int.h
+++ b/libc/src/__support/big_int.h
@@ -284,8 +284,8 @@ LIBC_INLINE constexpr cpp::array<word, N> shift(cpp::array<word, N> array,
     if (i < 0)
       return 0;
     if (i >= int(N))
-      return is_neg ? -1 : 0;
-    return array[i];
+      return is_neg ? cpp::numeric_limits<word>::max() : 0;
+    return array[static_cast<unsigned>(i)];
   };
   const size_t index_offset = offset / WORD_BITS;
   const size_t bit_offset = offset % WORD_BITS;
@@ -296,7 +296,7 @@ LIBC_INLINE constexpr cpp::array<word, N> shift(cpp::array<word, N> array,
   for (size_t index = 0; index < N; ++index) {
     const word part1 = safe_get_at(index + index_offset);
     const word part2 = safe_get_at(index + index_offset + 1);
-    word &dst = out[at(index)];
+    word &dst = out[static_cast<unsigned>(at(index))];
     if (bit_offset == 0)
       dst = part1; // no crosstalk between parts.
     else if constexpr (direction == LEFT)
@@ -696,7 +696,8 @@ struct BigInt {
     }
     BigInt quotient;
     WordType x_word = static_cast<WordType>(x);
-    constexpr size_t LOG2_WORD_SIZE = cpp::bit_width(WORD_SIZE) - 1;
+    constexpr size_t LOG2_WORD_SIZE =
+        static_cast<size_t>(cpp::bit_width(WORD_SIZE) - 1);
     constexpr size_t HALF_WORD_SIZE = WORD_SIZE >> 1;
     constexpr WordType HALF_MASK = ((WordType(1) << HALF_WORD_SIZE) - 1);
     // lower = smallest multiple of WORD_SIZE that is >= e.
@@ -1008,12 +1009,12 @@ struct BigInt {
       BigInt subtractor = divider;
       int cur_bit = multiword::countl_zero(subtractor.val) -
                     multiword::countl_zero(remainder.val);
-      subtractor <<= cur_bit;
+      subtractor <<= static_cast<size_t>(cur_bit);
       for (; cur_bit >= 0 && remainder > 0; --cur_bit, subtractor >>= 1) {
         if (remainder < subtractor)
           continue;
         remainder -= subtractor;
-        quotient.set_bit(cur_bit);
+        quotient.set_bit(static_cast<size_t>(cur_bit));
       }
     }
     return Division{quotient, remainder};
@@ -1275,26 +1276,28 @@ rotr(T value, int rotate);
 template <typename T>
 [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t<is_big_int_v<T>, T>
 rotl(T value, int rotate) {
-  constexpr unsigned N = cpp::numeric_limits<T>::digits;
+  constexpr int N = cpp::numeric_limits<T>::digits;
   rotate = rotate % N;
   if (!rotate)
     return value;
   if (rotate < 0)
     return cpp::rotr<T>(value, -rotate);
-  return (value << rotate) | (value >> (N - rotate));
+  return (value << static_cast<size_t>(rotate)) |
+         (value >> (N - static_cast<size_t>(rotate)));
 }
 
 // Specialization of cpp::rotr ('bit.h') for BigInt.
 template <typename T>
 [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t<is_big_int_v<T>, T>
 rotr(T value, int rotate) {
-  constexpr unsigned N = cpp::numeric_limits<T>::digits;
+  constexpr int N = cpp::numeric_limits<T>::digits;
   rotate = rotate % N;
   if (!rotate)
     return value;
   if (rotate < 0)
     return cpp::rotl<T>(value, -rotate);
-  return (value >> rotate) | (value << (N - rotate));
+  return (value >> static_cast<size_t>(rotate)) |
+         (value << (N - static_cast<size_t>(rotate)));
 }
 
 } // namespace cpp
@@ -1311,7 +1314,7 @@ mask_trailing_ones() {
   T out; // zero initialized
   for (size_t i = 0; i <= QUOTIENT; ++i)
     out[i] = i < QUOTIENT
-                 ? -1
+                 ? cpp::numeric_limits<typename T::word_type>::max()
                  : mask_trailing_ones<typename T::word_type, REMAINDER>();
   return out;
 }
@@ -1327,7 +1330,7 @@ LIBC_INLINE constexpr cpp::enable_if_t<is_big_int_v<T>, T> mask_leading_ones() {
   T out; // zero initialized
   for (size_t i = QUOTIENT; i < T::WORD_COUNT; ++i)
     out[i] = i > QUOTIENT
-                 ? -1
+                 ? cpp::numeric_limits<typename T::word_type>::max()
                  : mask_leading_ones<typename T::word_type, REMAINDER>();
   return out;
 }
@@ -1380,8 +1383,7 @@ first_trailing_zero(T value) {
 template <typename T>
 [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t<is_big_int_v<T>, int>
 first_trailing_one(T value) {
-  return value == cpp::numeric_limits<T>::max() ? 0
-                                                : cpp::countr_zero(value) + 1;
+  return value == 0 ? 0 : cpp::countr_zero(value) + 1;
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/float_to_string.h b/libc/src/__support/float_to_string.h
index d5de6f38cb655..4b03eaf4d02f2 100644
--- a/libc/src/__support/float_to_string.h
+++ b/libc/src/__support/float_to_string.h
@@ -287,7 +287,7 @@ LIBC_INLINE UInt<MID_INT_SIZE> get_table_negative(int exponent, size_t i) {
   size_t ten_blocks = i;
   size_t five_blocks = 0;
   if (shift_amount < 0) {
-    int block_shifts = (-shift_amount) / BLOCK_SIZE;
+    int block_shifts = (-shift_amount) / static_cast<int>(BLOCK_SIZE);
     if (block_shifts < static_cast<int>(ten_blocks)) {
       ten_blocks = ten_blocks - block_shifts;
       five_blocks = block_shifts;
diff --git a/libc/src/__support/high_precision_decimal.h b/libc/src/__support/high_precision_decimal.h
index 922dce484aa6b..cb4b50c315447 100644
--- a/libc/src/__support/high_precision_decimal.h
+++ b/libc/src/__support/high_precision_decimal.h
@@ -264,7 +264,7 @@ class HighPrecisionDecimal {
   LIBC_INLINE void left_shift(uint32_t shift_amount) {
     uint32_t new_digits = this->get_num_new_digits(shift_amount);
 
-    int32_t read_index = this->num_digits - 1;
+    int32_t read_index = static_cast<int32_t>(this->num_digits - 1);
     uint32_t write_index = this->num_digits + new_digits;
 
     uint64_t accumulator = 0;
@@ -329,7 +329,7 @@ class HighPrecisionDecimal {
         if (saw_dot) {
           break;
         }
-        this->decimal_point = total_digits;
+        this->decimal_point = static_cast<int32_t>(total_digits);
         saw_dot = true;
       } else {
         if (num_string[num_cur] == '0' && this->num_digits == 0) {
@@ -350,7 +350,7 @@ class HighPrecisionDecimal {
     }
 
     if (!saw_dot)
-      this->decimal_point = total_digits;
+      this->decimal_point = static_cast<int32_t>(total_digits);
 
     if (num_cur < num_len &&
         (num_string[num_cur] == 'e' || num_string[num_cur] == 'E')) {
@@ -393,7 +393,7 @@ class HighPrecisionDecimal {
         this->left_shift(MAX_SHIFT_AMOUNT);
         shift_amount -= MAX_SHIFT_AMOUNT;
       }
-      this->left_shift(shift_amount);
+      this->left_shift(static_cast<uint32_t>(shift_amount));
     }
     // Right
     else {
@@ -401,7 +401,7 @@ class HighPrecisionDecimal {
         this->right_shift(MAX_SHIFT_AMOUNT);
         shift_amount += MAX_SHIFT_AMOUNT;
       }
-      this->right_shift(-shift_amount);
+      this->right_shift(static_cast<uint32_t>(-shift_amount));
     }
   }
 
@@ -424,8 +424,8 @@ class HighPrecisionDecimal {
       result *= 10;
       ++cur_digit;
     }
-    return result + static_cast<unsigned int>(
-                        this->should_round_up(this->decimal_point, round));
+    return result +
+           static_cast<T>(this->should_round_up(this->decimal_point, round));
   }
 
   // Extra functions for testing.
diff --git a/libc/src/__support/integer_literals.h b/libc/src/__support/integer_literals.h
index 0298ec7d088d6..f68b7ef12c879 100644
--- a/libc/src/__support/integer_literals.h
+++ b/libc/src/__support/integer_literals.h
@@ -47,7 +47,7 @@ LIBC_INLINE constexpr T accumulate(int base, const uint8_t *digits,
                                    size_t size) {
   T value{};
   for (; size; ++digits, --size) {
-    value *= base;
+    value *= static_cast<unsigned int>(base);
     value += *digits;
   }
   return value;
diff --git a/libc/src/__support/integer_to_string.h b/libc/src/__support/integer_to_string.h
index 13dd83e9ed2b1..65bdcf16b3867 100644
--- a/libc/src/__support/integer_to_string.h
+++ b/libc/src/__support/integer_to_string.h
@@ -405,7 +405,7 @@ template <typename T, typename Fmt = radix::Dec> class IntegerToString {
     // Returns the absolute value of 'value' as 'UNSIGNED_T'.
     LIBC_INLINE static UNSIGNED_T abs(T value) {
       if (cpp::is_unsigned_v<T> || value >= 0)
-        return value; // already of the right sign.
+        return static_cast<UNSIGNED_T>(value); // already of the right sign.
 
       // Signed integers are asymmetric (e.g., int8_t ∈ [-128, 127]).
       // Thus negating the type's minimum value would overflow.
@@ -422,7 +422,8 @@ template <typename T, typename Fmt = radix::Dec> class IntegerToString {
       if (value == cpp::numeric_limits<T>::min()) {
         return cpp::bit_cast<UNSIGNED_T>(value);
       } else {
-        return -value; // legal and representable both as T and UNSIGNED_T.`
+        return static_cast<UNSIGNED_T>(
+            -value); // legal and representable both as T and UNSIGNED_T.`
       }
     }
 
diff --git a/libc/src/__support/macros/properties/cpu_features.h b/libc/src/__support/macros/properties/cpu_features.h
index d2cea367516db..1714775ca334d 100644
--- a/libc/src/__support/macros/properties/cpu_features.h
+++ b/libc/src/__support/macros/properties/cpu_features.h
@@ -45,6 +45,21 @@
 #if defined(__ARM_FEATURE_FMA) || (defined(__AVX2__) && defined(__FMA__)) ||   \
     defined(__NVPTX__) || defined(__AMDGPU__) || defined(__LIBC_RISCV_USE_FMA)
 #define LIBC_TARGET_CPU_HAS_FMA
+// Provide a more fine-grained control of FMA instruction for ARM targets.
+#if defined(__ARM_FP)
+#if (__ARM_FP & 0x2)
+#define LIBC_TARGET_CPU_HAS_FMA_HALF
+#endif // LIBC_TARGET_CPU_HAS_FMA_HALF
+#if (__ARM_FP & 0x4)
+#define LIBC_TARGET_CPU_HAS_FMA_FLOAT
+#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
+#if (__ARM_FP & 0x8)
+#define LIBC_TARGET_CPU_HAS_FMA_DOUBLE
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
+#else
+#define LIBC_TARGET_CPU_HAS_FMA_FLOAT
+#define LIBC_TARGET_CPU_HAS_FMA_DOUBLE
+#endif
 #endif
 
 #if defined(LIBC_TARGET_ARCH_IS_AARCH64) ||                                    \
diff --git a/libc/src/__support/math_extras.h b/libc/src/__support/math_extras.h
index 6f4a006aad270..47df2a43250c8 100644
--- a/libc/src/__support/math_extras.h
+++ b/libc/src/__support/math_extras.h
@@ -146,8 +146,7 @@ first_trailing_zero(T value) {
 template <typename T>
 [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t<cpp::is_unsigned_v<T>, int>
 first_trailing_one(T value) {
-  return value == cpp::numeric_limits<T>::max() ? 0
-                                                : cpp::countr_zero(value) + 1;
+  return value == 0 ? 0 : cpp::countr_zero(value) + 1;
 }
 
 template <typename T>
diff --git a/libc/src/__support/memory_size.h b/libc/src/__support/memory_size.h
index cdd6a10222de1..3d40b113bcb68 100644
--- a/libc/src/__support/memory_size.h
+++ b/libc/src/__support/memory_size.h
@@ -77,7 +77,8 @@ class SafeMemSize {
     if (!cpp::has_single_bit(alignment) || alignment > MAX_MEM_SIZE || !valid())
       return SafeMemSize{type{-1}};
 
-    type offset = offset_to(value, alignment);
+    type offset =
+        static_cast<type>(offset_to(static_cast<size_t>(value), alignment));
 
     if (LIBC_UNLIKELY(offset > static_cast<type>(MAX_MEM_SIZE) - value))
       return SafeMemSize{type{-1}};
diff --git a/libc/src/__support/str_to_float.h b/libc/src/__support/str_to_float.h
index b4d5646822df3..48c88309c58e9 100644
--- a/libc/src/__support/str_to_float.h
+++ b/libc/src/__support/str_to_float.h
@@ -108,11 +108,11 @@ eisel_lemire(ExpandedFloat<T> init_num,
   }
 
   // Normalization
-  uint32_t clz = cpp::countl_zero<StorageType>(mantissa);
+  uint32_t clz = static_cast<uint32_t>(cpp::countl_zero<StorageType>(mantissa));
   mantissa <<= clz;
 
-  int32_t exp2 =
-      exp10_to_exp2(exp10) + FPBits::STORAGE_LEN + FPBits::EXP_BIAS - clz;
+  int32_t exp2 = exp10_to_exp2(exp10) + FPBits::STORAGE_LEN + FPBits::EXP_BIAS -
+                 static_cast<int32_t>(clz);
 
   // Multiplication
   const uint64_t *power_of_ten =
@@ -225,8 +225,8 @@ eisel_lemire<long double>(ExpandedFloat<long double> init_num,
   }
 
   // Normalization
-  uint32_t clz = cpp::countl_zero(mantissa) -
-                 ((sizeof(UInt128) - sizeof(StorageType)) * CHAR_BIT);
+  int32_t clz = static_cast<int32_t>(cpp::countl_zero(mantissa)) -
+                ((sizeof(UInt128) - sizeof(StorageType)) * CHAR_BIT);
   mantissa <<= clz;
 
   int32_t exp2 =
@@ -802,7 +802,7 @@ LIBC_INLINE FloatConvertReturn<T> binary_exp_to_float(ExpandedFloat<T> init_num,
 
   // Handle subnormals.
   if (biased_exponent <= 0) {
-    amount_to_shift_right += 1 - biased_exponent;
+    amount_to_shift_right += static_cast<uint32_t>(1 - biased_exponent);
     biased_exponent = 0;
 
     if (amount_to_shift_right > FPBits::STORAGE_LEN) {
@@ -909,7 +909,7 @@ decimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT,
       cpp::numeric_limits<StorageType>::max() / BASE;
   while (true) {
     if (isdigit(src[index])) {
-      uint32_t digit = b36_char_to_int(src[index]);
+      uint32_t digit = static_cast<uint32_t>(b36_char_to_int(src[index]));
       seen_digit = true;
 
       if (mantissa < bitstype_max_div_by_base) {
@@ -956,7 +956,7 @@ decimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT,
       if (result.has_error())
         output.error = result.error;
       int32_t add_to_exponent = result.value;
-      index += result.parsed_len;
+      index += static_cast<size_t>(result.parsed_len);
 
       // Here we do this operation as int64 to avoid overflow.
       int64_t temp_exponent = static_cast<int64_t>(exponent) +
@@ -1020,7 +1020,7 @@ hexadecimal_string_to_float(const char *__restrict src,
       cpp::numeric_limits<StorageType>::max() / BASE;
   while (true) {
     if (isalnum(src[index])) {
-      uint32_t digit = b36_char_to_int(src[index]);
+      uint32_t digit = static_cast<uint32_t>(b36_char_to_int(src[index]));
       if (digit < BASE)
         seen_digit = true;
       else
@@ -1070,7 +1070,7 @@ hexadecimal_string_to_float(const char *__restrict src,
         output.error = result.error;
 
       int32_t add_to_exponent = result.value;
-      index += result.parsed_len;
+      index += static_cast<size_t>(result.parsed_len);
 
       // Here we do this operation as int64 to avoid overflow.
       int64_t temp_exponent = static_cast<int64_t>(exponent) +
@@ -1135,7 +1135,7 @@ LIBC_INLINE StrToNumResult<T> strtofloatingpoint(const char *__restrict src) {
 
   int error = 0;
 
-  ptrdiff_t index = first_non_whitespace(src) - src;
+  size_t index = static_cast<size_t>(first_non_whitespace(src) - src);
 
   if (src[index] == '+' || src[index] == '-') {
     sign = src[index];
@@ -1245,7 +1245,7 @@ LIBC_INLINE StrToNumResult<T> strtofloatingpoint(const char *__restrict src) {
   // special 80 bit long doubles. Otherwise it should be inlined out.
   set_implicit_bit<T>(result);
 
-  return {result.get_val(), index, error};
+  return {result.get_val(), static_cast<ptrdiff_t>(index), error};
 }
 
 template <class T> LIBC_INLINE StrToNumResult<T> strtonan(const char *arg) {
diff --git a/libc/src/__support/str_to_integer.h b/libc/src/__support/str_to_integer.h
index 8e569e8a7feb0..9212ad25d0820 100644
--- a/libc/src/__support/str_to_integer.h
+++ b/libc/src/__support/str_to_integer.h
@@ -96,7 +96,7 @@ strtointeger(const char *__restrict src, int base,
   if (base < 0 || base == 1 || base > 36)
     return {0, 0, EINVAL};
 
-  src_cur = first_non_whitespace(src, src_len) - src;
+  src_cur = static_cast<size_t>(first_non_whitespace(src, src_len) - src);
 
   char result_sign = '+';
   if (src[src_cur] == '+' || src[src_cur] == '-') {
@@ -119,7 +119,7 @@ strtointeger(const char *__restrict src, int base,
   ResultType const abs_max =
       (is_positive ? cpp::numeric_limits<T>::max() : NEGATIVE_MAX);
   ResultType const abs_max_div_by_base =
-      static_cast<ResultType>(abs_max / base);
+      abs_max / static_cast<ResultType>(base);
 
   while (src_cur < src_len && isalnum(src[src_cur])) {
     int cur_digit = b36_char_to_int(src[src_cur]);
@@ -141,17 +141,17 @@ strtointeger(const char *__restrict src, int base,
       result = abs_max;
       error_val = ERANGE;
     } else {
-      result = static_cast<ResultType>(result * base);
+      result = result * static_cast<ResultType>(base);
     }
-    if (result > abs_max - cur_digit) {
+    if (result > abs_max - static_cast<ResultType>(cur_digit)) {
       result = abs_max;
       error_val = ERANGE;
     } else {
-      result = static_cast<ResultType>(result + cur_digit);
+      result = result + static_cast<ResultType>(cur_digit);
     }
   }
 
-  ptrdiff_t str_len = is_number ? (src_cur) : 0;
+  ptrdiff_t str_len = is_number ? static_cast<ptrdiff_t>(src_cur) : 0;
 
   if (error_val == ERANGE) {
     if (is_positive || IS_UNSIGNED)
diff --git a/libc/src/math/generic/acosf.cpp b/libc/src/math/generic/acosf.cpp
index 3c097a7871380..509a5ebc4973e 100644
--- a/libc/src/math/generic/acosf.cpp
+++ b/libc/src/math/generic/acosf.cpp
@@ -20,6 +20,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 static constexpr size_t N_EXCEPTS = 4;
 
 // Exceptional values when |x| <= 0.5
@@ -34,6 +35,7 @@ static constexpr fputil::ExceptValues<float, N_EXCEPTS> ACOSF_EXCEPTS = {{
     // x = -0x1.04c444p-12, acosf(x) = 0x1.923p0 (RZ)
     {0xb9826222, 0x3fc91800, 1, 0, 1},
 }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float, acosf, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
@@ -51,9 +53,11 @@ LLVM_LIBC_FUNCTION(float, acosf, (float x)) {
       //   acos(x) = pi/2 - asin(x)
       //           ~ pi/2 - x - x^3 / 6
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
       // Check for exceptional values
       if (auto r = ACOSF_EXCEPTS.lookup(x_uint); LIBC_UNLIKELY(r.has_value()))
         return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
       double xd = static_cast<double>(x);
       return static_cast<float>(fputil::multiply_add(
diff --git a/libc/src/math/generic/acosf16.cpp b/libc/src/math/generic/acosf16.cpp
index 858da3bfaa918..202a950fbb5dd 100644
--- a/libc/src/math/generic/acosf16.cpp
+++ b/libc/src/math/generic/acosf16.cpp
@@ -27,6 +27,7 @@ namespace LIBC_NAMESPACE_DECL {
 static constexpr float PI_OVER_2 = 0x1.921fb6p0f;
 static constexpr float PI = 0x1.921fb6p1f;
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 static constexpr size_t N_EXCEPTS = 2;
 
 static constexpr fputil::ExceptValues<float16, N_EXCEPTS> ACOSF16_EXCEPTS{{
@@ -34,6 +35,7 @@ static constexpr fputil::ExceptValues<float16, N_EXCEPTS> ACOSF16_EXCEPTS{{
     {0xacaf, 0x3e93, 1, 0, 0},
     {0xb874, 0x4052, 1, 0, 1},
 }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, acosf16, (float16 x)) {
   using FPBits = fputil::FPBits<float16>;
@@ -64,9 +66,11 @@ LLVM_LIBC_FUNCTION(float16, acosf16, (float16 x)) {
 
   float xf = x;
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   // Handle exceptional values
   if (auto r = ACOSF16_EXCEPTS.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
     return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // |x| == 0x1p0, x is 1 or -1
   // if x is (-)1, return pi, else
diff --git a/libc/src/math/generic/acoshf.cpp b/libc/src/math/generic/acoshf.cpp
index 6158063d30521..c4927fa27a84b 100644
--- a/libc/src/math/generic/acoshf.cpp
+++ b/libc/src/math/generic/acoshf.cpp
@@ -22,7 +22,6 @@ namespace LIBC_NAMESPACE_DECL {
 LLVM_LIBC_FUNCTION(float, acoshf, (float x)) {
   using FPBits_t = typename fputil::FPBits<float>;
   FPBits_t xbits(x);
-  uint32_t x_u = xbits.uintval();
 
   if (LIBC_UNLIKELY(x <= 1.0f)) {
     if (x == 1.0f)
@@ -33,6 +32,8 @@ LLVM_LIBC_FUNCTION(float, acoshf, (float x)) {
     return FPBits_t::quiet_nan().get_val();
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+  uint32_t x_u = xbits.uintval();
   if (LIBC_UNLIKELY(x_u >= 0x4f8ffb03)) {
     if (LIBC_UNLIKELY(xbits.is_inf_or_nan()))
       return x;
@@ -64,6 +65,10 @@ LLVM_LIBC_FUNCTION(float, acoshf, (float x)) {
       return round_result_slightly_up(0x1.451436p6f);
     }
   }
+#else
+  if (LIBC_UNLIKELY(xbits.is_inf_or_nan()))
+    return x;
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   double x_d = static_cast<double>(x);
   // acosh(x) = log(x + sqrt(x^2 - 1))
diff --git a/libc/src/math/generic/asinf.cpp b/libc/src/math/generic/asinf.cpp
index 3a89def8f6e0c..da854417e85fe 100644
--- a/libc/src/math/generic/asinf.cpp
+++ b/libc/src/math/generic/asinf.cpp
@@ -21,6 +21,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 static constexpr size_t N_EXCEPTS = 2;
 
 // Exceptional values when |x| <= 0.5
@@ -40,6 +41,7 @@ static constexpr fputil::ExceptValues<float, N_EXCEPTS> ASINF_EXCEPTS_HI = {{
     // x = 0x1.ee836cp-1, asinf(x) = 0x1.4f0654p0 (RZ)
     {0x3f7741b6, 0x3fa7832a, 1, 0, 0},
 }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float, asinf, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
@@ -74,18 +76,20 @@ LLVM_LIBC_FUNCTION(float, asinf, (float x)) {
       // |x| < 2^-125. For targets without FMA instructions, we simply use
       // double for intermediate results as it is more efficient than using an
       // emulated version of FMA.
-#if defined(LIBC_TARGET_CPU_HAS_FMA)
+#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT)
       return fputil::multiply_add(x, 0x1.0p-25f, x);
 #else
       double xd = static_cast<double>(x);
       return static_cast<float>(fputil::multiply_add(xd, 0x1.0p-25, xd));
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
     }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
     // Check for exceptional values
     if (auto r = ASINF_EXCEPTS_LO.lookup_odd(x_abs, x_sign);
         LIBC_UNLIKELY(r.has_value()))
       return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
     // For |x| <= 0.5, we approximate asinf(x) by:
     //   asin(x) = x * P(x^2)
@@ -111,10 +115,12 @@ LLVM_LIBC_FUNCTION(float, asinf, (float x)) {
     return FPBits::quiet_nan().get_val();
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   // Check for exceptional values
   if (auto r = ASINF_EXCEPTS_HI.lookup_odd(x_abs, x_sign);
       LIBC_UNLIKELY(r.has_value()))
     return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // When |x| > 0.5, we perform range reduction as follow:
   //
diff --git a/libc/src/math/generic/asinhf.cpp b/libc/src/math/generic/asinhf.cpp
index 1d68ac9ea13bc..9cdb7b8394e9d 100644
--- a/libc/src/math/generic/asinhf.cpp
+++ b/libc/src/math/generic/asinhf.cpp
@@ -49,6 +49,7 @@ LLVM_LIBC_FUNCTION(float, asinhf, (float x)) {
   double x_sign = SIGN[x_u >> 31];
   double x_d = x;
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   // Helper functions to set results for exceptional cases.
   auto round_result_slightly_down = [x_sign](float r) -> float {
     return fputil::multiply_add(static_cast<float>(x_sign), r,
@@ -95,6 +96,10 @@ LLVM_LIBC_FUNCTION(float, asinhf, (float x)) {
       return round_result_slightly_down(0x1.e1b92p3f);
     }
   }
+#else
+  if (LIBC_UNLIKELY(xbits.is_inf_or_nan()))
+    return x;
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // asinh(x) = log(x + sqrt(x^2 + 1))
   return static_cast<float>(
diff --git a/libc/src/math/generic/atan2f.cpp b/libc/src/math/generic/atan2f.cpp
index 5ac2b29438ea9..726cae9c8462b 100644
--- a/libc/src/math/generic/atan2f.cpp
+++ b/libc/src/math/generic/atan2f.cpp
@@ -131,7 +131,7 @@ float atan2f_double_double(double num_d, double den_d, double q_d, int idx,
     num_r = num_d;
     den_r = den_d;
   }
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
   q.lo = fputil::multiply_add(q.hi, -den_r, num_r) / den_r;
 #else
   // Compute `(num_r - q.hi * den_r) / den_r` accurately without FMA
@@ -140,7 +140,7 @@ float atan2f_double_double(double num_d, double den_d, double q_d, int idx,
   double t1 = fputil::multiply_add(q_hi_dd.hi, -den_r, num_r); // Exact
   double t2 = fputil::multiply_add(q_hi_dd.lo, -den_r, t1);
   q.lo = t2 / den_r;
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
   // Taylor polynomial, evaluating using Horner's scheme:
   //   P = x - x^3/3 + x^5/5 -x^7/7 + x^9/9 - x^11/11 + x^13/13 - x^15/15
diff --git a/libc/src/math/generic/atanf.cpp b/libc/src/math/generic/atanf.cpp
index 5e0788efbeb88..46196dbe4162c 100644
--- a/libc/src/math/generic/atanf.cpp
+++ b/libc/src/math/generic/atanf.cpp
@@ -52,12 +52,12 @@ LLVM_LIBC_FUNCTION(float, atanf, (float x)) {
       return x;
     // x <= 2^-12;
     if (LIBC_UNLIKELY(x_abs < 0x3980'0000)) {
-#if defined(LIBC_TARGET_CPU_HAS_FMA)
+#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT)
       return fputil::multiply_add(x, -0x1.0p-25f, x);
 #else
       double x_d = static_cast<double>(x);
       return static_cast<float>(fputil::multiply_add(x_d, -0x1.0p-25, x_d));
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
     }
     // Use Taylor polynomial:
     //   atan(x) ~ x * (1 - x^2 / 3 + x^4 / 5 - x^6 / 7 + x^8 / 9 - x^10 / 11).
diff --git a/libc/src/math/generic/cbrt.cpp b/libc/src/math/generic/cbrt.cpp
index ee7d69b2c211f..ce227e6650c84 100644
--- a/libc/src/math/generic/cbrt.cpp
+++ b/libc/src/math/generic/cbrt.cpp
@@ -58,7 +58,7 @@ double intial_approximation(double x) {
 
 // Get the error term for Newton iteration:
 //   h(x) = x^3 * a^2 - 1,
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 double get_error(const DoubleDouble &x_3, const DoubleDouble &a_sq) {
   return fputil::multiply_add(x_3.hi, a_sq.hi, -1.0) +
          fputil::multiply_add(x_3.lo, a_sq.hi, x_3.hi * a_sq.lo);
diff --git a/libc/src/math/generic/cos.cpp b/libc/src/math/generic/cos.cpp
index 568b1254c6f02..b60082bf9c308 100644
--- a/libc/src/math/generic/cos.cpp
+++ b/libc/src/math/generic/cos.cpp
@@ -20,11 +20,11 @@
 #include "src/math/generic/range_reduction_double_common.h"
 #include "src/math/generic/sincos_eval.h"
 
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 #include "range_reduction_double_fma.h"
 #else
 #include "range_reduction_double_nofma.h"
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/math/generic/cosf.cpp b/libc/src/math/generic/cosf.cpp
index 972ffa923aedf..6ea24f9ccd3fa 100644
--- a/libc/src/math/generic/cosf.cpp
+++ b/libc/src/math/generic/cosf.cpp
@@ -20,6 +20,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 // Exceptional cases for cosf.
 static constexpr size_t N_EXCEPTS = 6;
 
@@ -38,6 +39,7 @@ static constexpr fputil::ExceptValues<float, N_EXCEPTS> COSF_EXCEPTS{{
     // x = 0x1.ddebdep120, cos(x) = 0x1.114438p-1 (RZ)
     {0x7beef5ef, 0x3f08a21c, 1, 0, 0},
 }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float, cosf, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
@@ -101,15 +103,17 @@ LLVM_LIBC_FUNCTION(float, cosf, (float x)) {
     // |x| < 2^-125. For targets without FMA instructions, we simply use
     // double for intermediate results as it is more efficient than using an
     // emulated version of FMA.
-#if defined(LIBC_TARGET_CPU_HAS_FMA)
+#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT)
     return fputil::multiply_add(xbits.get_val(), -0x1.0p-25f, 1.0f);
 #else
     return static_cast<float>(fputil::multiply_add(xd, -0x1.0p-25, 1.0));
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   if (auto r = COSF_EXCEPTS.lookup(x_abs); LIBC_UNLIKELY(r.has_value()))
     return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // x is inf or nan.
   if (LIBC_UNLIKELY(x_abs >= 0x7f80'0000U)) {
diff --git a/libc/src/math/generic/cosf16.cpp b/libc/src/math/generic/cosf16.cpp
index 534e07854474e..4d42db981ce71 100644
--- a/libc/src/math/generic/cosf16.cpp
+++ b/libc/src/math/generic/cosf16.cpp
@@ -19,6 +19,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 constexpr size_t N_EXCEPTS = 4;
 
 constexpr fputil::ExceptValues<float16, N_EXCEPTS> COSF16_EXCEPTS{{
@@ -28,6 +29,7 @@ constexpr fputil::ExceptValues<float16, N_EXCEPTS> COSF16_EXCEPTS{{
     {0x5c49, 0xb8c6, 0, 1, 0},
     {0x7acc, 0xa474, 0, 1, 0},
 }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, cosf16, (float16 x)) {
   using FPBits = fputil::FPBits<float16>;
@@ -53,9 +55,11 @@ LLVM_LIBC_FUNCTION(float16, cosf16, (float16 x)) {
   //          = cos(k * pi/32) * cos(y * pi/32) -
   //            sin(k * pi/32) * sin(y * pi/32)
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   // Handle exceptional values
   if (auto r = COSF16_EXCEPTS.lookup(x_abs); LIBC_UNLIKELY(r.has_value()))
     return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // cos(+/-0) = 1
   if (LIBC_UNLIKELY(x_abs == 0U))
diff --git a/libc/src/math/generic/coshf16.cpp b/libc/src/math/generic/coshf16.cpp
index cca7581c70e0e..6668e77000f0c 100644
--- a/libc/src/math/generic/coshf16.cpp
+++ b/libc/src/math/generic/coshf16.cpp
@@ -20,6 +20,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 static constexpr fputil::ExceptValues<float16, 9> COSHF16_EXCEPTS_POS = {{
     // x = 0x1.6ap-5, coshf16(x) = 0x1p+0 (RZ)
     {0x29a8U, 0x3c00U, 1U, 0U, 1U},
@@ -51,6 +52,7 @@ static constexpr fputil::ExceptValues<float16, 4> COSHF16_EXCEPTS_NEG = {{
     // x = -0x1.5fp+3, coshf16(x) = 0x1.c54p+14 (RZ)
     {0xc97cU, 0x7715U, 1U, 0U, 1U},
 }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, coshf16, (float16 x)) {
   using FPBits = fputil::FPBits<float16>;
@@ -89,6 +91,7 @@ LLVM_LIBC_FUNCTION(float16, coshf16, (float16 x)) {
     }
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   if (x_bits.is_pos()) {
     if (auto r = COSHF16_EXCEPTS_POS.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
       return r.value();
@@ -96,6 +99,7 @@ LLVM_LIBC_FUNCTION(float16, coshf16, (float16 x)) {
     if (auto r = COSHF16_EXCEPTS_NEG.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
       return r.value();
   }
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   return eval_sinh_or_cosh</*IsSinh=*/false>(x);
 }
diff --git a/libc/src/math/generic/cospif.cpp b/libc/src/math/generic/cospif.cpp
index 4ef1539539921..29566f4fceacf 100644
--- a/libc/src/math/generic/cospif.cpp
+++ b/libc/src/math/generic/cospif.cpp
@@ -50,11 +50,11 @@ LLVM_LIBC_FUNCTION(float, cospif, (float x)) {
   // The exhautive test passes for smaller values
   if (LIBC_UNLIKELY(x_abs < 0x38A2'F984U)) {
 
-#if defined(LIBC_TARGET_CPU_HAS_FMA)
+#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT)
     return fputil::multiply_add(xbits.get_val(), -0x1.0p-25f, 1.0f);
 #else
     return static_cast<float>(fputil::multiply_add(xd, -0x1.0p-25, 1.0));
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
   }
 
   // Numbers greater or equal to 2^23 are always integers or NaN
diff --git a/libc/src/math/generic/erff.cpp b/libc/src/math/generic/erff.cpp
index 15357452759ea..016afe4a68140 100644
--- a/libc/src/math/generic/erff.cpp
+++ b/libc/src/math/generic/erff.cpp
@@ -141,6 +141,7 @@ LLVM_LIBC_FUNCTION(float, erff, (float x)) {
     return ONE[sign] + SMALL[sign];
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   // Exceptional mask = common 0 bits of 2 exceptional values.
   constexpr uint32_t EXCEPT_MASK = 0x809a'6184U;
 
@@ -155,6 +156,7 @@ LLVM_LIBC_FUNCTION(float, erff, (float x)) {
     if (x_abs == 0U)
       return x;
   }
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // Polynomial approximation:
   //   erf(x) ~ x * (c0 + c1 * x^2 + c2 * x^4 + ... + c7 * x^14)
diff --git a/libc/src/math/generic/exp10f16.cpp b/libc/src/math/generic/exp10f16.cpp
index 006dd5c554428..31abf3b4f89b2 100644
--- a/libc/src/math/generic/exp10f16.cpp
+++ b/libc/src/math/generic/exp10f16.cpp
@@ -26,7 +26,8 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT
 static constexpr size_t N_EXP10F16_EXCEPTS = 5;
 #else
 static constexpr size_t N_EXP10F16_EXCEPTS = 8;
@@ -44,7 +45,7 @@ static constexpr fputil::ExceptValues<float16, N_EXP10F16_EXCEPTS>
         {0xbf0aU, 0x2473U, 1U, 0U, 0U},
         // x = -0x1.e1cp+1, exp10f16(x) = 0x1.694p-13 (RZ)
         {0xc387U, 0x09a5U, 1U, 0U, 0U},
-#ifndef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_TARGET_CPU_HAS_FMA_FLOAT
         // x = 0x1.0cp+1, exp10f16(x) = 0x1.f04p+6 (RZ)
         {0x4030U, 0x57c1U, 1U, 0U, 1U},
         // x = 0x1.1b8p+1, exp10f16(x) = 0x1.47cp+7 (RZ)
@@ -53,6 +54,7 @@ static constexpr fputil::ExceptValues<float16, N_EXP10F16_EXCEPTS>
         {0x446eU, 0x7690U, 1U, 0U, 1U},
 #endif
     }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, exp10f16, (float16 x)) {
   using FPBits = fputil::FPBits<float16>;
@@ -119,8 +121,10 @@ LLVM_LIBC_FUNCTION(float16, exp10f16, (float16 x)) {
     }
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   if (auto r = EXP10F16_EXCEPTS.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
     return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // 10^x = 2^((hi + mid) * log2(10)) * 10^lo
   auto [exp2_hi_mid, exp10_lo] = exp10_range_reduction(x);
diff --git a/libc/src/math/generic/exp10m1f.cpp b/libc/src/math/generic/exp10m1f.cpp
index c0e302eea7b08..e973b2921c2e4 100644
--- a/libc/src/math/generic/exp10m1f.cpp
+++ b/libc/src/math/generic/exp10m1f.cpp
@@ -22,6 +22,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 static constexpr size_t N_EXCEPTS_LO = 11;
 
 static constexpr fputil::ExceptValues<float, N_EXCEPTS_LO> EXP10M1F_EXCEPTS_LO =
@@ -94,6 +95,7 @@ static constexpr fputil::ExceptValues<float, N_EXCEPTS_HI> EXP10M1F_EXCEPTS_HI =
         // x = -0x1.ca4322p-5, exp10m1f(x) = -0x1.ef073p-4 (RZ)
         {0xbd65'2191U, 0xbdf7'8398U, 0U, 1U, 1U},
     }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float, exp10m1f, (float x)) {
   using FPBits = fputil::FPBits<float>;
@@ -119,8 +121,10 @@ LLVM_LIBC_FUNCTION(float, exp10m1f, (float x)) {
 
   // When |x| <= log10(2) * 2^(-6)
   if (LIBC_UNLIKELY(x_abs <= 0x3b9a'209bU)) {
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
     if (auto r = EXP10M1F_EXCEPTS_LO.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
       return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
     double dx = x;
     double dx_sq = dx * dx;
@@ -192,8 +196,10 @@ LLVM_LIBC_FUNCTION(float, exp10m1f, (float x)) {
     }
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   if (auto r = EXP10M1F_EXCEPTS_HI.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
     return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // Range reduction: 10^x = 2^(mid + hi) * 10^lo
   //   rr = (2^(mid + hi), lo)
diff --git a/libc/src/math/generic/exp10m1f16.cpp b/libc/src/math/generic/exp10m1f16.cpp
index 449aedf254ca5..545c479694811 100644
--- a/libc/src/math/generic/exp10m1f16.cpp
+++ b/libc/src/math/generic/exp10m1f16.cpp
@@ -24,6 +24,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 static constexpr fputil::ExceptValues<float16, 3> EXP10M1F16_EXCEPTS_LO = {{
     // (input, RZ output, RU offset, RD offset, RN offset)
     // x = 0x1.5c4p-4, exp10m1f16(x) = 0x1.bacp-3 (RZ)
@@ -34,7 +35,7 @@ static constexpr fputil::ExceptValues<float16, 3> EXP10M1F16_EXCEPTS_LO = {{
     {0x9788U, 0x9c53U, 0U, 1U, 0U},
 }};
 
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT
 static constexpr size_t N_EXP10M1F16_EXCEPTS_HI = 3;
 #else
 static constexpr size_t N_EXP10M1F16_EXCEPTS_HI = 6;
@@ -49,7 +50,7 @@ static constexpr fputil::ExceptValues<float16, N_EXP10M1F16_EXCEPTS_HI>
         {0x3657U, 0x3df6U, 1U, 0U, 0U},
         // x = 0x1.d04p-2, exp10m1f16(x) = 0x1.d7p+0 (RZ)
         {0x3741U, 0x3f5cU, 1U, 0U, 1U},
-#ifndef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_TARGET_CPU_HAS_FMA_FLOAT
         // x = 0x1.0cp+1, exp10m1f16(x) = 0x1.ec4p+6 (RZ)
         {0x4030U, 0x57b1U, 1U, 0U, 1U},
         // x = 0x1.1b8p+1, exp10m1f16(x) = 0x1.45cp+7 (RZ)
@@ -58,6 +59,7 @@ static constexpr fputil::ExceptValues<float16, N_EXP10M1F16_EXCEPTS_HI>
         {0x44bdU, 0x7aaeU, 1U, 0U, 1U},
 #endif
     }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, exp10m1f16, (float16 x)) {
   using FPBits = fputil::FPBits<float16>;
@@ -122,9 +124,11 @@ LLVM_LIBC_FUNCTION(float16, exp10m1f16, (float16 x)) {
       if (LIBC_UNLIKELY(x_abs == 0))
         return x;
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
       if (auto r = EXP10M1F16_EXCEPTS_LO.lookup(x_u);
           LIBC_UNLIKELY(r.has_value()))
         return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
       float xf = x;
       // Degree-5 minimax polynomial generated by Sollya with the following
@@ -153,8 +157,10 @@ LLVM_LIBC_FUNCTION(float16, exp10m1f16, (float16 x)) {
     }
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   if (auto r = EXP10M1F16_EXCEPTS_HI.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
     return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // exp10(x) = exp2((hi + mid) * log2(10)) * exp10(lo)
   auto [exp2_hi_mid, exp10_lo] = exp10_range_reduction(x);
diff --git a/libc/src/math/generic/exp2.cpp b/libc/src/math/generic/exp2.cpp
index 2c612777c9cb5..726f88b6457fc 100644
--- a/libc/src/math/generic/exp2.cpp
+++ b/libc/src/math/generic/exp2.cpp
@@ -35,11 +35,11 @@ using LIBC_NAMESPACE::operator""_u128;
 
 // Error bounds:
 // Errors when using double precision.
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 constexpr double ERR_D = 0x1.0p-63;
 #else
 constexpr double ERR_D = 0x1.8p-63;
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
 #ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 // Errors when using double-double precision.
diff --git a/libc/src/math/generic/exp2f16.cpp b/libc/src/math/generic/exp2f16.cpp
index 858053fe2bccb..5c039c59df1af 100644
--- a/libc/src/math/generic/exp2f16.cpp
+++ b/libc/src/math/generic/exp2f16.cpp
@@ -21,6 +21,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 static constexpr fputil::ExceptValues<float16, 3> EXP2F16_EXCEPTS = {{
     // (input, RZ output, RU offset, RD offset, RN offset)
     // x = 0x1.714p-11, exp2f16(x) = 0x1p+0 (RZ)
@@ -30,6 +31,7 @@ static constexpr fputil::ExceptValues<float16, 3> EXP2F16_EXCEPTS = {{
     // x = -0x1.d5cp-4, exp2f16(x) = 0x1.d8cp-1 (RZ)
     {0xaf57U, 0x3b63U, 1U, 0U, 0U},
 }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, exp2f16, (float16 x)) {
   using FPBits = fputil::FPBits<float16>;
@@ -82,8 +84,10 @@ LLVM_LIBC_FUNCTION(float16, exp2f16, (float16 x)) {
     }
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   if (auto r = EXP2F16_EXCEPTS.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
     return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // exp2(x) = exp2(hi + mid) * exp2(lo)
   auto [exp2_hi_mid, exp2_lo] = exp2_range_reduction(x);
diff --git a/libc/src/math/generic/exp2f_impl.h b/libc/src/math/generic/exp2f_impl.h
index ae2d0628c1205..5c6c2bd415188 100644
--- a/libc/src/math/generic/exp2f_impl.h
+++ b/libc/src/math/generic/exp2f_impl.h
@@ -27,10 +27,6 @@ namespace LIBC_NAMESPACE_DECL {
 namespace generic {
 
 LIBC_INLINE float exp2f(float x) {
-  constexpr uint32_t EXVAL1 = 0x3b42'9d37U;
-  constexpr uint32_t EXVAL2 = 0xbcf3'a937U;
-  constexpr uint32_t EXVAL_MASK = EXVAL1 & EXVAL2;
-
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
 
@@ -46,6 +42,11 @@ LIBC_INLINE float exp2f(float x) {
         return 1.0f + x;
       }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+      constexpr uint32_t EXVAL1 = 0x3b42'9d37U;
+      constexpr uint32_t EXVAL2 = 0xbcf3'a937U;
+      constexpr uint32_t EXVAL_MASK = EXVAL1 & EXVAL2;
+
       // Check exceptional values.
       if (LIBC_UNLIKELY((x_u & EXVAL_MASK) == EXVAL_MASK)) {
         if (LIBC_UNLIKELY(x_u == EXVAL1)) { // x = 0x1.853a6ep-9f
@@ -54,6 +55,7 @@ LIBC_INLINE float exp2f(float x) {
           return fputil::round_result_slightly_down(0x1.f58d62p-1f);
         }
       }
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
       // Minimax polynomial generated by Sollya with:
       // > P = fpminimax((2^x - 1)/x, 5, [|D...|], [-2^-5, 2^-5]);
diff --git a/libc/src/math/generic/exp2m1f.cpp b/libc/src/math/generic/exp2m1f.cpp
index 2060dc34cc9bf..4913a5e4277e4 100644
--- a/libc/src/math/generic/exp2m1f.cpp
+++ b/libc/src/math/generic/exp2m1f.cpp
@@ -23,6 +23,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 static constexpr size_t N_EXCEPTS_LO = 8;
 
 static constexpr fputil::ExceptValues<float, N_EXCEPTS_LO> EXP2M1F_EXCEPTS_LO =
@@ -58,6 +59,7 @@ static constexpr fputil::ExceptValues<float, N_EXCEPTS_HI> EXP2M1F_EXCEPTS_HI =
         // x = -0x1.de7b9cp-5, exp2m1f(x) = -0x1.4508f4p-5 (RZ)
         {0xbd6f'3dceU, 0xbd22'847aU, 0U, 1U, 1U},
     }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float, exp2m1f, (float x)) {
   using FPBits = fputil::FPBits<float>;
@@ -70,8 +72,10 @@ LLVM_LIBC_FUNCTION(float, exp2m1f, (float x)) {
   if (LIBC_UNLIKELY(x_abs >= 0x4300'0000U || x_abs <= 0x3d00'0000U)) {
     // |x| <= 2^-5
     if (x_abs <= 0x3d00'0000U) {
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
       if (auto r = EXP2M1F_EXCEPTS_LO.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
         return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
       // Minimax polynomial generated by Sollya with:
       // > display = hexadecimal;
@@ -121,8 +125,10 @@ LLVM_LIBC_FUNCTION(float, exp2m1f, (float x)) {
     return -1.0f;
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   if (auto r = EXP2M1F_EXCEPTS_HI.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
     return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // For -25 < x < 128, to compute 2^x, we perform the following range
   // reduction: find hi, mid, lo such that:
diff --git a/libc/src/math/generic/exp2m1f16.cpp b/libc/src/math/generic/exp2m1f16.cpp
index 6a1cd2328a050..61633cd2cfcfb 100644
--- a/libc/src/math/generic/exp2m1f16.cpp
+++ b/libc/src/math/generic/exp2m1f16.cpp
@@ -24,6 +24,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 static constexpr fputil::ExceptValues<float16, 6> EXP2M1F16_EXCEPTS_LO = {{
     // (input, RZ output, RU offset, RD offset, RN offset)
     // x = 0x1.cf4p-13, exp2m1f16(x) = 0x1.41p-13 (RZ)
@@ -40,7 +41,7 @@ static constexpr fputil::ExceptValues<float16, 6> EXP2M1F16_EXCEPTS_LO = {{
     {0x973fU, 0x9505U, 0U, 1U, 0U},
 }};
 
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT
 static constexpr size_t N_EXP2M1F16_EXCEPTS_HI = 6;
 #else
 static constexpr size_t N_EXP2M1F16_EXCEPTS_HI = 7;
@@ -51,13 +52,13 @@ static constexpr fputil::ExceptValues<float16, N_EXP2M1F16_EXCEPTS_HI>
         // (input, RZ output, RU offset, RD offset, RN offset)
         // x = 0x1.e58p-3, exp2m1f16(x) = 0x1.6dcp-3 (RZ)
         {0x3396U, 0x31b7U, 1U, 0U, 0U},
-#ifndef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_TARGET_CPU_HAS_FMA_FLOAT
         // x = 0x1.2e8p-2, exp2m1f16(x) = 0x1.d14p-3 (RZ)
         {0x34baU, 0x3345U, 1U, 0U, 0U},
 #endif
         // x = 0x1.ad8p-2, exp2m1f16(x) = 0x1.598p-2 (RZ)
         {0x36b6U, 0x3566U, 1U, 0U, 0U},
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT
         // x = 0x1.edcp-2, exp2m1f16(x) = 0x1.964p-2 (RZ)
         {0x37b7U, 0x3659U, 1U, 0U, 1U},
 #endif
@@ -67,11 +68,12 @@ static constexpr fputil::ExceptValues<float16, N_EXP2M1F16_EXCEPTS_HI>
         {0xb3ccU, 0xb0f9U, 0U, 1U, 0U},
         // x = -0x1.294p-1, exp2m1f16(x) = -0x1.53p-2 (RZ)
         {0xb8a5U, 0xb54cU, 0U, 1U, 1U},
-#ifndef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_TARGET_CPU_HAS_FMA_FLOAT
         // x = -0x1.a34p-1, exp2m1f16(x) = -0x1.bb4p-2 (RZ)
         {0xba8dU, 0xb6edU, 0U, 1U, 1U},
 #endif
     }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, exp2m1f16, (float16 x)) {
   using FPBits = fputil::FPBits<float16>;
@@ -132,9 +134,11 @@ LLVM_LIBC_FUNCTION(float16, exp2m1f16, (float16 x)) {
 
     // When |x| <= 2^(-3).
     if (x_abs <= 0x3000U) {
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
       if (auto r = EXP2M1F16_EXCEPTS_LO.lookup(x_u);
           LIBC_UNLIKELY(r.has_value()))
         return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
       float xf = x;
       // Degree-5 minimax polynomial generated by Sollya with the following
@@ -149,8 +153,10 @@ LLVM_LIBC_FUNCTION(float16, exp2m1f16, (float16 x)) {
     }
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   if (auto r = EXP2M1F16_EXCEPTS_HI.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
     return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // exp2(x) = exp2(hi + mid) * exp2(lo)
   auto [exp2_hi_mid, exp2_lo] = exp2_range_reduction(x);
diff --git a/libc/src/math/generic/expf.cpp b/libc/src/math/generic/expf.cpp
index ee5c2a32b6c6b..fa507d4d9322c 100644
--- a/libc/src/math/generic/expf.cpp
+++ b/libc/src/math/generic/expf.cpp
@@ -28,10 +28,12 @@ LLVM_LIBC_FUNCTION(float, expf, (float x)) {
   uint32_t x_u = xbits.uintval();
   uint32_t x_abs = x_u & 0x7fff'ffffU;
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   // Exceptional values
   if (LIBC_UNLIKELY(x_u == 0xc236'bd8cU)) { // x = -0x1.6d7b18p+5f
     return 0x1.108a58p-66f - x * 0x1.0p-95f;
   }
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // When |x| >= 89, |x| < 2^-25, or x is nan
   if (LIBC_UNLIKELY(x_abs >= 0x42b2'0000U || x_abs <= 0x3280'0000U)) {
diff --git a/libc/src/math/generic/expf16.cpp b/libc/src/math/generic/expf16.cpp
index 0548ef3932ae9..1af9b3ec9ad6e 100644
--- a/libc/src/math/generic/expf16.cpp
+++ b/libc/src/math/generic/expf16.cpp
@@ -22,6 +22,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 static constexpr fputil::ExceptValues<float16, 2> EXPF16_EXCEPTS_LO = {{
     // (input, RZ output, RU offset, RD offset, RN offset)
     // x = 0x1.de4p-8, expf16(x) = 0x1.01cp+0 (RZ)
@@ -39,6 +40,7 @@ static constexpr fputil::ExceptValues<float16, 3> EXPF16_EXCEPTS_HI = {{
     // x = -0x1.55p-5, expf16(x) = 0x1.ebp-1 (RZ)
     {0xa954U, 0x3bacU, 1U, 0U, 0U},
 }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, expf16, (float16 x)) {
   using FPBits = fputil::FPBits<float16>;
@@ -95,8 +97,10 @@ LLVM_LIBC_FUNCTION(float16, expf16, (float16 x)) {
 
     // When 0 < |x| <= 2^(-5).
     if (x_abs <= 0x2800U && !x_bits.is_zero()) {
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
       if (auto r = EXPF16_EXCEPTS_LO.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
         return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
       float xf = x;
       // Degree-3 minimax polynomial generated by Sollya with the following
@@ -109,8 +113,10 @@ LLVM_LIBC_FUNCTION(float16, expf16, (float16 x)) {
     }
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   if (auto r = EXPF16_EXCEPTS_HI.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
     return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // exp(x) = exp(hi + mid) * exp(lo)
   auto [exp_hi_mid, exp_lo] = exp_range_reduction(x);
diff --git a/libc/src/math/generic/expm1f.cpp b/libc/src/math/generic/expm1f.cpp
index d5e9e85ed4bd3..1e44e943d9258 100644
--- a/libc/src/math/generic/expm1f.cpp
+++ b/libc/src/math/generic/expm1f.cpp
@@ -38,14 +38,14 @@ LLVM_LIBC_FUNCTION(float, expm1f, (float x)) {
     return 0x1.8dbe62p-3f;
   }
 
-#if !defined(LIBC_TARGET_CPU_HAS_FMA)
+#if !defined(LIBC_TARGET_CPU_HAS_FMA_DOUBLE)
   if (LIBC_UNLIKELY(x_u == 0xbdc1'c6cbU)) { // x = -0x1.838d96p-4f
     int round_mode = fputil::quick_get_round();
     if (round_mode == FE_TONEAREST || round_mode == FE_DOWNWARD)
       return -0x1.71c884p-4f;
     return -0x1.71c882p-4f;
   }
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
   // When |x| > 25*log(2), or nan
   if (LIBC_UNLIKELY(x_abs >= 0x418a'a123U)) {
@@ -102,12 +102,12 @@ LLVM_LIBC_FUNCTION(float, expm1f, (float x)) {
         // 2^-76. For targets without FMA instructions, we simply use double for
         // intermediate results as it is more efficient than using an emulated
         // version of FMA.
-#if defined(LIBC_TARGET_CPU_HAS_FMA)
-      return fputil::fma<float>(x, x, x);
+#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT)
+      return fputil::multiply_add(x, x, x);
 #else
       double xd = x;
       return static_cast<float>(fputil::multiply_add(xd, xd, xd));
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
     }
 
     constexpr double COEFFS[] = {0x1p-1,
diff --git a/libc/src/math/generic/expm1f16.cpp b/libc/src/math/generic/expm1f16.cpp
index 4ce0efd1f461b..2188dfbda8ef9 100644
--- a/libc/src/math/generic/expm1f16.cpp
+++ b/libc/src/math/generic/expm1f16.cpp
@@ -23,13 +23,14 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 static constexpr fputil::ExceptValues<float16, 1> EXPM1F16_EXCEPTS_LO = {{
     // (input, RZ output, RU offset, RD offset, RN offset)
     // x = 0x1.564p-5, expm1f16(x) = 0x1.5d4p-5 (RZ)
     {0x2959U, 0x2975U, 1U, 0U, 1U},
 }};
 
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT
 static constexpr size_t N_EXPM1F16_EXCEPTS_HI = 2;
 #else
 static constexpr size_t N_EXPM1F16_EXCEPTS_HI = 3;
@@ -42,11 +43,12 @@ static constexpr fputil::ExceptValues<float16, N_EXPM1F16_EXCEPTS_HI>
         {0x3f0dU, 0x44d3U, 1U, 0U, 1U},
         // x = -0x1.e28p-3, expm1f16(x) = -0x1.adcp-3 (RZ)
         {0xb38aU, 0xb2b7U, 0U, 1U, 1U},
-#ifndef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_TARGET_CPU_HAS_FMA_FLOAT
         // x = 0x1.a08p-3, exp10m1f(x) = 0x1.cdcp-3 (RZ)
         {0x3282U, 0x3337U, 1U, 0U, 0U},
 #endif
     }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, expm1f16, (float16 x)) {
   using FPBits = fputil::FPBits<float16>;
@@ -105,9 +107,12 @@ LLVM_LIBC_FUNCTION(float16, expm1f16, (float16 x)) {
 
     // When 0 < |x| <= 2^(-3).
     if (x_abs <= 0x3000U && !x_bits.is_zero()) {
+
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
       if (auto r = EXPM1F16_EXCEPTS_LO.lookup(x_u);
           LIBC_UNLIKELY(r.has_value()))
         return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
       float xf = x;
       // Degree-5 minimax polynomial generated by Sollya with the following
@@ -121,8 +126,10 @@ LLVM_LIBC_FUNCTION(float16, expm1f16, (float16 x)) {
     }
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   if (auto r = EXPM1F16_EXCEPTS_HI.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
     return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // exp(x) = exp(hi + mid) * exp(lo)
   auto [exp_hi_mid, exp_lo] = exp_range_reduction(x);
diff --git a/libc/src/math/generic/fmul.cpp b/libc/src/math/generic/fmul.cpp
index e759e48cd6989..daad64873f27a 100644
--- a/libc/src/math/generic/fmul.cpp
+++ b/libc/src/math/generic/fmul.cpp
@@ -21,7 +21,7 @@ LLVM_LIBC_FUNCTION(float, fmul, (double x, double y)) {
   // correctly rounded for all rounding modes, so we fall
   // back to the generic `fmul` implementation
 
-#ifndef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
   return fputil::generic::mul<float>(x, y);
 #else
   fputil::DoubleDouble prod = fputil::exact_mult(x, y);
diff --git a/libc/src/math/generic/hypotf.cpp b/libc/src/math/generic/hypotf.cpp
index 959c0420ae149..ec48f62163a48 100644
--- a/libc/src/math/generic/hypotf.cpp
+++ b/libc/src/math/generic/hypotf.cpp
@@ -55,7 +55,7 @@ LLVM_LIBC_FUNCTION(float, hypotf, (float x, float y)) {
 
   // These squares are exact.
   double a_sq = ad * ad;
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
   double sum_sq = fputil::multiply_add(bd, bd, a_sq);
 #else
   double b_sq = bd * bd;
@@ -72,7 +72,7 @@ LLVM_LIBC_FUNCTION(float, hypotf, (float x, float y)) {
     double r_d = result.get_val();
 
     // Perform rounding correction.
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
     double sum_sq_lo = fputil::multiply_add(bd, bd, a_sq - sum_sq);
     double err = sum_sq_lo - fputil::multiply_add(r_d, r_d, -sum_sq);
 #else
diff --git a/libc/src/math/generic/log.cpp b/libc/src/math/generic/log.cpp
index 04eebab975cd5..0cd4424ee0baf 100644
--- a/libc/src/math/generic/log.cpp
+++ b/libc/src/math/generic/log.cpp
@@ -800,13 +800,13 @@ LLVM_LIBC_FUNCTION(double, log, (double x)) {
   fputil::DoubleDouble r1;
 
   // Perform exact range reduction
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
   u = fputil::multiply_add(r, m, -1.0); // exact
 #else
   uint64_t c_m = x_m & 0x3FFF'E000'0000'0000ULL;
   double c = FPBits_t(c_m).get_val();
   u = fputil::multiply_add(r, m - c, CD[index]); // exact
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
   // Exact sum:
   //   r1.hi + r1.lo = e_x * log(2)_hi - log(r)_hi + u
diff --git a/libc/src/math/generic/log10.cpp b/libc/src/math/generic/log10.cpp
index fd8d5a8aae938..1c4e559ba083c 100644
--- a/libc/src/math/generic/log10.cpp
+++ b/libc/src/math/generic/log10.cpp
@@ -802,13 +802,13 @@ LLVM_LIBC_FUNCTION(double, log10, (double x)) {
   fputil::DoubleDouble r1;
 
   // Perform exact range reduction
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
   u = fputil::multiply_add(r, m, -1.0); // exact
 #else
   uint64_t c_m = x_m & 0x3FFF'E000'0000'0000ULL;
   double c = FPBits_t(c_m).get_val();
   u = fputil::multiply_add(r, m - c, CD[index]); // exact
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
   // Error of u_sq = ulp(u^2);
   u_sq = u * u;
diff --git a/libc/src/math/generic/log10f.cpp b/libc/src/math/generic/log10f.cpp
index c635fa4ef9b63..81e7cdbcfe5a1 100644
--- a/libc/src/math/generic/log10f.cpp
+++ b/libc/src/math/generic/log10f.cpp
@@ -139,13 +139,14 @@ LLVM_LIBC_FUNCTION(float, log10f, (float x)) {
       return 9.0f;
     case 0x5015'02f9U: // x = 10,000,000,000
       return 10.0f;
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
     case 0x0efe'ee7aU: // x = 0x1.fddcf4p-98f
       return fputil::round_result_slightly_up(-0x1.d33a46p+4f);
     case 0x3f5f'de1bU: // x = 0x1.bfbc36p-1f
       return fputil::round_result_slightly_up(-0x1.dd2c6ep-5f);
     case 0x3f80'70d8U: // x = 0x1.00e1bp0f
       return fputil::round_result_slightly_up(0x1.8762c4p-10f);
-#ifndef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
     case 0x08ae'a356U: // x = 0x1.5d46acp-110f
       return fputil::round_result_slightly_up(-0x1.07d3b4p+5f);
     case 0x120b'93dcU: // x = 0x1.1727b8p-91f
@@ -156,7 +157,8 @@ LLVM_LIBC_FUNCTION(float, log10f, (float x)) {
       return fputil::round_result_slightly_down(0x1.2c9314p+3f);
     case 0x7956'ba5eU: // x = 69683218960000541503257137270226944.0
       return fputil::round_result_slightly_up(0x1.16bebap+5f);
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
     }
   }
 
@@ -194,12 +196,12 @@ LLVM_LIBC_FUNCTION(float, log10f, (float x)) {
 
   float u = xbits.get_val();
   double v;
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT
   v = static_cast<double>(fputil::multiply_add(u, R[index], -1.0f)); // Exact.
 #else
   v = fputil::multiply_add(static_cast<double>(u),
                            static_cast<double>(R[index]), -1.0); // Exact
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
 
   // Degree-5 polynomial approximation of log10 generated by:
   // > P = fpminimax(log10(1 + x)/x, 4, [|D...|], [-2^-8, 2^-7]);
diff --git a/libc/src/math/generic/log10f16.cpp b/libc/src/math/generic/log10f16.cpp
index 990bcabaf6871..2626af4ddda15 100644
--- a/libc/src/math/generic/log10f16.cpp
+++ b/libc/src/math/generic/log10f16.cpp
@@ -23,7 +23,8 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT
 static constexpr size_t N_LOG10F16_EXCEPTS = 11;
 #else
 static constexpr size_t N_LOG10F16_EXCEPTS = 17;
@@ -36,7 +37,7 @@ static constexpr fputil::ExceptValues<float16, N_LOG10F16_EXCEPTS>
         {0x338fU, 0xb903U, 0U, 1U, 0U},
         // x = 0x1.fep-3, log10f16(x) = -0x1.35p-1 (RZ)
         {0x33f8U, 0xb8d4U, 0U, 1U, 1U},
-#ifndef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_TARGET_CPU_HAS_FMA_FLOAT
         // x = 0x1.394p-1, log10f16(x) = -0x1.b4cp-3 (RZ)
         {0x38e5U, 0xb2d3U, 0U, 1U, 1U},
 #endif
@@ -47,7 +48,7 @@ static constexpr fputil::ExceptValues<float16, N_LOG10F16_EXCEPTS>
         // x = 0x1.f3p-1, log10f16(x) = -0x1.6dcp-7 (RZ)
         {0x3bccU, 0xa1b7U, 0U, 1U, 1U},
 // x = 0x1.f38p-1, log10f16(x) = -0x1.5f8p-7 (RZ)
-#ifndef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_TARGET_CPU_HAS_FMA_FLOAT
         {0x3bceU, 0xa17eU, 0U, 1U, 1U},
         // x = 0x1.fd8p-1, log10f16(x) = -0x1.168p-9 (RZ)
         {0x3bf6U, 0x985aU, 0U, 1U, 1U},
@@ -71,6 +72,7 @@ static constexpr fputil::ExceptValues<float16, N_LOG10F16_EXCEPTS>
         // x = 0x1.674p+13, log10f16(x) = 0x1.03cp+2 (RZ)
         {0x719dU, 0x440fU, 1U, 0U, 0U},
     }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, log10f16, (float16 x)) {
   using FPBits = fputil::FPBits<float16>;
@@ -110,8 +112,10 @@ LLVM_LIBC_FUNCTION(float16, log10f16, (float16 x)) {
     return FPBits::inf().get_val();
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   if (auto r = LOG10F16_EXCEPTS.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
     return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // To compute log10(x), we perform the following range reduction:
   //   x = 2^m * 1.mant,
diff --git a/libc/src/math/generic/log1p.cpp b/libc/src/math/generic/log1p.cpp
index b1f02164b6a28..058409fed081d 100644
--- a/libc/src/math/generic/log1p.cpp
+++ b/libc/src/math/generic/log1p.cpp
@@ -1009,7 +1009,7 @@ LLVM_LIBC_FUNCTION(double, log1p, (double x)) {
   fputil::DoubleDouble v_lo = fputil::exact_mult(m_dd.lo, r);
 
   // Perform exact range reduction
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
   v_hi = fputil::multiply_add(r, m_dd.hi, -1.0); // Exact.
 #else
   // c = 1 + idx * 2^-7.
@@ -1017,7 +1017,7 @@ LLVM_LIBC_FUNCTION(double, log1p, (double x)) {
                       uint64_t(0x3FF0'0000'0000'0000ULL))
                  .get_val();
   v_hi = fputil::multiply_add(r, m_dd.hi - c, RCM1[idx]); // Exact
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
   // Range reduction output:
   //   -0x1.69000000000edp-8 < v_hi + v_lo < 0x1.7f00000000081p-8
diff --git a/libc/src/math/generic/log1pf.cpp b/libc/src/math/generic/log1pf.cpp
index 869cb077cc434..7f614293029de 100644
--- a/libc/src/math/generic/log1pf.cpp
+++ b/libc/src/math/generic/log1pf.cpp
@@ -90,6 +90,7 @@ LLVM_LIBC_FUNCTION(float, log1pf, (float x)) {
 
   // Use log1p(x) = log(1 + x) for |x| > 2^-6;
   if (x_a > 0x3c80'0000U) {
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
     // Hard-to-round cases.
     switch (x_u) {
     case 0x41078febU: // x = 0x1.0f1fd6p3
@@ -108,7 +109,7 @@ LLVM_LIBC_FUNCTION(float, log1pf, (float x)) {
       fputil::set_errno_if_required(ERANGE);
       fputil::raise_except_if_required(FE_DIVBYZERO);
       return FPBits::inf(Sign::NEG).get_val();
-#ifndef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
     case 0x4cc1c80bU: // x = 0x1.839016p+26f
       return fputil::round_result_slightly_down(0x1.26fc04p+4f);
     case 0x5ee8984eU: // x = 0x1.d1309cp+62f
@@ -117,13 +118,21 @@ LLVM_LIBC_FUNCTION(float, log1pf, (float x)) {
       return fputil::round_result_slightly_up(0x1.af66cp+5f);
     case 0x79e7ec37U: // x = 0x1.cfd86ep+116f
       return fputil::round_result_slightly_up(0x1.43ff6ep+6f);
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
     }
+#else
+    if (x == -1.0f) {
+      fputil::set_errno_if_required(ERANGE);
+      fputil::raise_except_if_required(FE_DIVBYZERO);
+      return FPBits::inf(Sign::NEG).get_val();
+    }
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
     return internal::log(xd + 1.0);
   }
 
   // |x| <= 2^-6.
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   // Hard-to round cases.
   switch (x_u) {
   case 0x35400003U: // x = 0x1.800006p-21f
@@ -137,6 +146,7 @@ LLVM_LIBC_FUNCTION(float, log1pf, (float x)) {
   case 0xbb0ec8c4U: // x = -0x1.1d9188p-9
     return fputil::round_result_slightly_up(-0x1.1de14ap-9f);
   }
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // Polymial generated by Sollya with:
   // > fpminimax(log(1 + x)/x, 7, [|D...|], [-2^-6; 2^-6]);
diff --git a/libc/src/math/generic/log2.cpp b/libc/src/math/generic/log2.cpp
index f46ff724a4f37..27ca2fc350f17 100644
--- a/libc/src/math/generic/log2.cpp
+++ b/libc/src/math/generic/log2.cpp
@@ -915,13 +915,13 @@ LLVM_LIBC_FUNCTION(double, log2, (double x)) {
   fputil::DoubleDouble r1;
 
   // Perform exact range reduction
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
   u = fputil::multiply_add(r, m, -1.0); // exact
 #else
   uint64_t c_m = x_m & 0x3FFF'E000'0000'0000ULL;
   double c = FPBits_t(c_m).get_val();
   u = fputil::multiply_add(r, m - c, CD[index]); // exact
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
   // Exact sum:
   //   r1.hi + r1.lo = e_x * log(2)_hi - log(r)_hi + u
diff --git a/libc/src/math/generic/log2f.cpp b/libc/src/math/generic/log2f.cpp
index 111f3f130bcab..b25ec41f277b6 100644
--- a/libc/src/math/generic/log2f.cpp
+++ b/libc/src/math/generic/log2f.cpp
@@ -97,11 +97,11 @@ LLVM_LIBC_FUNCTION(float, log2f, (float x)) {
 
   float u = xbits.get_val();
   double v;
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT
   v = static_cast<double>(fputil::multiply_add(u, R[index], -1.0f)); // Exact.
 #else
   v = fputil::multiply_add(static_cast<double>(u), RD[index], -1.0); // Exact
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
 
   double extra_factor = static_cast<double>(m) + LOG2_R[index];
 
diff --git a/libc/src/math/generic/log2f16.cpp b/libc/src/math/generic/log2f16.cpp
index ff4e0268b53d0..34be780388f2c 100644
--- a/libc/src/math/generic/log2f16.cpp
+++ b/libc/src/math/generic/log2f16.cpp
@@ -23,7 +23,8 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT
 static constexpr size_t N_LOG2F16_EXCEPTS = 2;
 #else
 static constexpr size_t N_LOG2F16_EXCEPTS = 9;
@@ -32,7 +33,7 @@ static constexpr size_t N_LOG2F16_EXCEPTS = 9;
 static constexpr fputil::ExceptValues<float16, N_LOG2F16_EXCEPTS>
     LOG2F16_EXCEPTS = {{
 // (input, RZ output, RU offset, RD offset, RN offset)
-#ifndef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_TARGET_CPU_HAS_FMA_FLOAT
         // x = 0x1.224p-1, log2f16(x) = -0x1.a34p-1 (RZ)
         {0x3889U, 0xba8dU, 0U, 1U, 0U},
         // x = 0x1.e34p-1, log2f16(x) = -0x1.558p-4 (RZ)
@@ -40,7 +41,7 @@ static constexpr fputil::ExceptValues<float16, N_LOG2F16_EXCEPTS>
 #endif
         // x = 0x1.e8cp-1, log2f16(x) = -0x1.128p-4 (RZ)
         {0x3ba3U, 0xac4aU, 0U, 1U, 0U},
-#ifndef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_TARGET_CPU_HAS_FMA_FLOAT
         // x = 0x1.f98p-1, log2f16(x) = -0x1.2ep-6 (RZ)
         {0x3be6U, 0xa4b8U, 0U, 1U, 0U},
         // x = 0x1.facp-1, log2f16(x) = -0x1.e7p-7 (RZ)
@@ -48,7 +49,7 @@ static constexpr fputil::ExceptValues<float16, N_LOG2F16_EXCEPTS>
 #endif
         // x = 0x1.fb4p-1, log2f16(x) = -0x1.b88p-7 (RZ)
         {0x3bedU, 0xa2e2U, 0U, 1U, 1U},
-#ifndef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_TARGET_CPU_HAS_FMA_FLOAT
         // x = 0x1.fecp-1, log2f16(x) = -0x1.cep-9 (RZ)
         {0x3bfbU, 0x9b38U, 0U, 1U, 1U},
         // x = 0x1.ffcp-1, log2f16(x) = -0x1.714p-11 (RZ)
@@ -57,6 +58,7 @@ static constexpr fputil::ExceptValues<float16, N_LOG2F16_EXCEPTS>
         {0x3c89U, 0x31cbU, 1U, 0U, 1U},
 #endif
     }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, log2f16, (float16 x)) {
   using FPBits = fputil::FPBits<float16>;
@@ -96,8 +98,10 @@ LLVM_LIBC_FUNCTION(float16, log2f16, (float16 x)) {
     return FPBits::inf().get_val();
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   if (auto r = LOG2F16_EXCEPTS.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
     return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // To compute log2(x), we perform the following range reduction:
   //   x = 2^m * 1.mant,
diff --git a/libc/src/math/generic/logf.cpp b/libc/src/math/generic/logf.cpp
index 30c00edafe21d..032d658a941be 100644
--- a/libc/src/math/generic/logf.cpp
+++ b/libc/src/math/generic/logf.cpp
@@ -66,6 +66,7 @@ LLVM_LIBC_FUNCTION(float, logf, (float x)) {
 
   // Small inputs
   if (x_u < 0x4c5d65a5U) {
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
     // Hard-to-round cases.
     switch (x_u) {
     case 0x3f7f4d6fU: // x = 0x1.fe9adep-1f
@@ -80,6 +81,7 @@ LLVM_LIBC_FUNCTION(float, logf, (float x)) {
       return round_result_slightly_up(-0x1.6d7b18p+5f);
 #endif // LIBC_TARGET_CPU_HAS_FMA
     }
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
     // Subnormal inputs.
     if (LIBC_UNLIKELY(x_u < FPBits::min_normal().uintval())) {
       if (x == 0.0f) {
@@ -94,6 +96,7 @@ LLVM_LIBC_FUNCTION(float, logf, (float x)) {
       x_u = xbits.uintval();
     }
   } else {
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
     // Hard-to-round cases.
     switch (x_u) {
     case 0x4c5d65a5U: // x = 0x1.bacb4ap+25f
@@ -104,15 +107,16 @@ LLVM_LIBC_FUNCTION(float, logf, (float x)) {
       return round_result_slightly_down(0x1.08b512p+6f);
     case 0x7a17f30aU: // x = 0x1.2fe614p+117f
       return round_result_slightly_up(0x1.451436p+6f);
-#ifndef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
     case 0x500ffb03U: // x = 0x1.1ff606p+33f
       return round_result_slightly_up(0x1.6fdd34p+4f);
     case 0x5cd69e88U: // x = 0x1.ad3d1p+58f
       return round_result_slightly_up(0x1.45c146p+5f);
     case 0x5ee8984eU: // x = 0x1.d1309cp+62f;
       return round_result_slightly_up(0x1.5c9442p+5f);
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
     }
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
     // Exceptional inputs.
     if (LIBC_UNLIKELY(x_u > FPBits::max_normal().uintval())) {
       if (x_u == 0x8000'0000U) {
@@ -152,11 +156,11 @@ LLVM_LIBC_FUNCTION(float, logf, (float x)) {
 
   float u = xbits.get_val();
   double v;
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT
   v = static_cast<double>(fputil::multiply_add(u, R[index], -1.0f)); // Exact.
 #else
   v = fputil::multiply_add(static_cast<double>(u), RD[index], -1.0); // Exact
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
 
   // Degree-5 polynomial approximation of log generated by Sollya with:
   // > P = fpminimax(log(1 + x)/x, 4, [|1, D...|], [-2^-8, 2^-7]);
diff --git a/libc/src/math/generic/logf16.cpp b/libc/src/math/generic/logf16.cpp
index 802225a810550..8e0d7d8f226cb 100644
--- a/libc/src/math/generic/logf16.cpp
+++ b/libc/src/math/generic/logf16.cpp
@@ -23,7 +23,8 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT
 static constexpr size_t N_LOGF16_EXCEPTS = 5;
 #else
 static constexpr size_t N_LOGF16_EXCEPTS = 11;
@@ -32,7 +33,7 @@ static constexpr size_t N_LOGF16_EXCEPTS = 11;
 static constexpr fputil::ExceptValues<float16, N_LOGF16_EXCEPTS>
     LOGF16_EXCEPTS = {{
 // (input, RZ output, RU offset, RD offset, RN offset)
-#ifndef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_TARGET_CPU_HAS_FMA_FLOAT
         // x = 0x1.61cp-13, logf16(x) = -0x1.16p+3 (RZ)
         {0x0987U, 0xc858U, 0U, 1U, 0U},
         // x = 0x1.f2p-12, logf16(x) = -0x1.e98p+2 (RZ)
@@ -42,7 +43,7 @@ static constexpr fputil::ExceptValues<float16, N_LOGF16_EXCEPTS>
         {0x1935U, 0xc5f9U, 0U, 1U, 0U},
         // x = 0x1.5ep-8, logf16(x) = -0x1.4ecp+2 (RZ)
         {0x1d78U, 0xc53bU, 0U, 1U, 0U},
-#ifndef LIBC_TARGET_CPU_HAS_FMA
+#ifndef LIBC_TARGET_CPU_HAS_FMA_FLOAT
         // x = 0x1.fdp-1, logf16(x) = -0x1.81p-8 (RZ)
         {0x3bf4U, 0x9e04U, 0U, 1U, 1U},
         // x = 0x1.fep-1, logf16(x) = -0x1.008p-8 (RZ)
@@ -52,7 +53,7 @@ static constexpr fputil::ExceptValues<float16, N_LOGF16_EXCEPTS>
         {0x3bfcU, 0x9801U, 0U, 1U, 0U},
         // x = 0x1.ff8p-1, logf16(x) = -0x1p-10 (RZ)
         {0x3bfeU, 0x9400U, 0U, 1U, 1U},
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT
         // x = 0x1.4c4p+1, logf16(x) = 0x1.e84p-1 (RZ)
         {0x4131U, 0x3ba1U, 1U, 0U, 1U},
 #else
@@ -64,6 +65,7 @@ static constexpr fputil::ExceptValues<float16, N_LOGF16_EXCEPTS>
         {0x6354U, 0x46d7U, 1U, 0U, 1U},
 #endif
     }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, logf16, (float16 x)) {
   using FPBits = fputil::FPBits<float16>;
@@ -103,8 +105,10 @@ LLVM_LIBC_FUNCTION(float16, logf16, (float16 x)) {
     return FPBits::inf().get_val();
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   if (auto r = LOGF16_EXCEPTS.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
     return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // To compute log(x), we perform the following range reduction:
   //   x = 2^m * 1.mant,
diff --git a/libc/src/math/generic/pow.cpp b/libc/src/math/generic/pow.cpp
index a2a0bb698f81a..8a12934f6c4ba 100644
--- a/libc/src/math/generic/pow.cpp
+++ b/libc/src/math/generic/pow.cpp
@@ -394,14 +394,14 @@ LLVM_LIBC_FUNCTION(double, pow, (double x, double y)) {
   DoubleDouble dx_c0;
 
   // Perform exact range reduction and exact product dx * c0.
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
   dx = fputil::multiply_add(RD[idx_x], m_x.get_val(), -1.0); // Exact
   dx_c0 = fputil::exact_mult(COEFFS[0], dx);
 #else
   double c = FPBits(m_x.uintval() & 0x3fff'e000'0000'0000).get_val();
   dx = fputil::multiply_add(RD[idx_x], m_x.get_val() - c, CD[idx_x]); // Exact
   dx_c0 = fputil::exact_mult<double, 28>(dx, COEFFS[0]);              // Exact
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
   double dx2 = dx * dx;
   double c0 = fputil::multiply_add(dx, COEFFS[2], COEFFS[1]);
diff --git a/libc/src/math/generic/powf.cpp b/libc/src/math/generic/powf.cpp
index 7f4417d275702..2d7deca3c77bb 100644
--- a/libc/src/math/generic/powf.cpp
+++ b/libc/src/math/generic/powf.cpp
@@ -165,11 +165,11 @@ alignas(16) constexpr DoubleDouble LOG2_R_DD[128] = {
 };
 #else
 
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 constexpr uint64_t ERR = 64;
 #else
 constexpr uint64_t ERR = 128;
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
 // We choose the precision of the high part to be 53 - 24 - 8, so that when
 //   y * (e_x + LOG2_R_DD[i].hi) is exact.
@@ -851,11 +851,11 @@ LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) {
   //   log2(m_x) = log2( (1 + dx) / r )
   //             = log2(1 + dx) - log2(r).
   double dx;
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT
   dx = static_cast<double>(fputil::multiply_add(m_x, R[idx_x], -1.0f)); // Exact
 #else
   dx = fputil::multiply_add(static_cast<double>(m_x), RD[idx_x], -1.0); // Exact
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
 
   // Degree-5 polynomial approximation:
   //   dx * P(dx) ~ log2(1 + dx)
diff --git a/libc/src/math/generic/range_reduction_double_common.h b/libc/src/math/generic/range_reduction_double_common.h
index 711a12219c847..f3dcdb937333c 100644
--- a/libc/src/math/generic/range_reduction_double_common.h
+++ b/libc/src/math/generic/range_reduction_double_common.h
@@ -20,14 +20,14 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 static constexpr unsigned SPLIT = fputil::DefaultSplit<double>::VALUE;
 #else
 // When there is no-FMA instructions, in order to have exact product of 2 double
 // precision with directional roundings, we need to lower the precision of the
 // constants by at least 1 bit, and use a different splitting constant.
 static constexpr unsigned SPLIT = 28;
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
 using LIBC_NAMESPACE::fputil::DoubleDouble;
 using Float128 = LIBC_NAMESPACE::fputil::DyadicFloat<128>;
diff --git a/libc/src/math/generic/sin.cpp b/libc/src/math/generic/sin.cpp
index b32486dff487c..4a58dcf4b173f 100644
--- a/libc/src/math/generic/sin.cpp
+++ b/libc/src/math/generic/sin.cpp
@@ -21,11 +21,11 @@
 #include "src/math/generic/range_reduction_double_common.h"
 #include "src/math/generic/sincos_eval.h"
 
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 #include "range_reduction_double_fma.h"
 #else
 #include "range_reduction_double_nofma.h"
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -52,7 +52,7 @@ LLVM_LIBC_FUNCTION(double, sin, (double x)) {
         if (LIBC_UNLIKELY(x == 0.0))
           return x + x; // Make sure it works with FTZ/DAZ.
 
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
         return fputil::multiply_add(x, -0x1.0p-54, x);
 #else
         if (LIBC_UNLIKELY(x_e < 4)) {
@@ -63,7 +63,7 @@ LLVM_LIBC_FUNCTION(double, sin, (double x)) {
             return FPBits(xbits.uintval() - 1).get_val();
         }
         return fputil::multiply_add(x, -0x1.0p-54, x);
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
       }
       // No range reduction needed.
       k = 0;
diff --git a/libc/src/math/generic/sincos.cpp b/libc/src/math/generic/sincos.cpp
index 166ce46603140..0ac2f7f997527 100644
--- a/libc/src/math/generic/sincos.cpp
+++ b/libc/src/math/generic/sincos.cpp
@@ -22,11 +22,11 @@
 #include "src/math/generic/range_reduction_double_common.h"
 #include "src/math/generic/sincos_eval.h"
 
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 #include "range_reduction_double_fma.h"
 #else
 #include "range_reduction_double_nofma.h"
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -57,7 +57,7 @@ LLVM_LIBC_FUNCTION(void, sincos, (double x, double *sin_x, double *cos_x)) {
         }
 
         // For |x| < 2^-27, max(|sin(x) - x|, |cos(x) - 1|) < ulp(x)/2.
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
         *sin_x = fputil::multiply_add(x, -0x1.0p-54, x);
         *cos_x = fputil::multiply_add(x, -x, 1.0);
 #else
@@ -71,7 +71,7 @@ LLVM_LIBC_FUNCTION(void, sincos, (double x, double *sin_x, double *cos_x)) {
             *sin_x = FPBits(xbits.uintval() - 1).get_val();
         }
         *sin_x = fputil::multiply_add(x, -0x1.0p-54, x);
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
         return;
       }
       // No range reduction needed.
diff --git a/libc/src/math/generic/sincos_eval.h b/libc/src/math/generic/sincos_eval.h
index 6cd1da4721bf5..41a4c75849ff4 100644
--- a/libc/src/math/generic/sincos_eval.h
+++ b/libc/src/math/generic/sincos_eval.h
@@ -65,7 +65,7 @@ LIBC_INLINE double sincos_eval(const DoubleDouble &u, DoubleDouble &sin_u,
   double u_hi_neg_half = (-0.5) * u.hi;
   DoubleDouble v;
 
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
   v.hi = fputil::multiply_add(u.hi, u_hi_neg_half, 1.0);
   v.lo = 1.0 - v.hi; // Exact
   v.lo = fputil::multiply_add(u.hi, u_hi_neg_half, v.lo);
@@ -73,7 +73,7 @@ LIBC_INLINE double sincos_eval(const DoubleDouble &u, DoubleDouble &sin_u,
   DoubleDouble u_hi_sq_neg_half = fputil::exact_mult(u.hi, u_hi_neg_half);
   v = fputil::exact_add(1.0, u_hi_sq_neg_half.hi);
   v.lo += u_hi_sq_neg_half.lo;
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
   // r1 ~ -1/720 + u_hi^2 / 40320
   double r1 = fputil::multiply_add(u_hi_sq, 0x1.a01a01a01a01ap-16,
diff --git a/libc/src/math/generic/sincosf.cpp b/libc/src/math/generic/sincosf.cpp
index ccaa29c10c4c6..623ef636afb1e 100644
--- a/libc/src/math/generic/sincosf.cpp
+++ b/libc/src/math/generic/sincosf.cpp
@@ -19,6 +19,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 // Exceptional values
 static constexpr int N_EXCEPTS = 6;
 
@@ -48,6 +49,7 @@ static constexpr uint32_t EXCEPT_OUTPUTS_COS[N_EXCEPTS][4] = {
     {0x3f78142e, 1, 0, 1}, // x = 0x1.2b9622p67, cos(x) = 0x1.f0285cp-1 (RZ)
     {0x3f08a21c, 1, 0, 0}, // x = 0x1.ddebdep120, cos(x) = 0x1.114438p-1 (RZ)
 };
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(void, sincosf, (float x, float *sinp, float *cosp)) {
   using FPBits = typename fputil::FPBits<float>;
@@ -130,14 +132,14 @@ LLVM_LIBC_FUNCTION(void, sincosf, (float x, float *sinp, float *cosp)) {
     // |x| < 2^-125. For targets without FMA instructions, we simply use
     // double for intermediate results as it is more efficient than using an
     // emulated version of FMA.
-#if defined(LIBC_TARGET_CPU_HAS_FMA)
+#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT)
     *sinp = fputil::multiply_add(x, -0x1.0p-25f, x);
     *cosp = fputil::multiply_add(FPBits(x_abs).get_val(), -0x1.0p-25f, 1.0f);
 #else
     *sinp = static_cast<float>(fputil::multiply_add(xd, -0x1.0p-25, xd));
     *cosp = static_cast<float>(fputil::multiply_add(
         static_cast<double>(FPBits(x_abs).get_val()), -0x1.0p-25, 1.0));
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
     return;
   }
 
@@ -152,6 +154,7 @@ LLVM_LIBC_FUNCTION(void, sincosf, (float x, float *sinp, float *cosp)) {
     return;
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   // Check exceptional values.
   for (int i = 0; i < N_EXCEPTS; ++i) {
     if (LIBC_UNLIKELY(x_abs == EXCEPT_INPUTS[i])) {
@@ -178,6 +181,7 @@ LLVM_LIBC_FUNCTION(void, sincosf, (float x, float *sinp, float *cosp)) {
       return;
     }
   }
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // Combine the results with the sine and cosine of sum formulas:
   //   sin(x) = sin((k + y)*pi/32)
diff --git a/libc/src/math/generic/sincosf16_utils.h b/libc/src/math/generic/sincosf16_utils.h
index 133896b5de7a3..05cab09d2089b 100644
--- a/libc/src/math/generic/sincosf16_utils.h
+++ b/libc/src/math/generic/sincosf16_utils.h
@@ -40,7 +40,7 @@ constexpr float SIN_K_PI_OVER_32[64] = {
 
 LIBC_INLINE int32_t range_reduction_sincospif16(float x, float &y) {
   float kf = fputil::nearest_integer(x * 32);
-  y = fputil::multiply_add<float>(x, 32.0, -kf);
+  y = fputil::multiply_add(x, 32.0f, -kf);
 
   return static_cast<int32_t>(kf);
 }
diff --git a/libc/src/math/generic/sincosf_utils.h b/libc/src/math/generic/sincosf_utils.h
index 726a5ab9b64be..6eaf820e5c1b0 100644
--- a/libc/src/math/generic/sincosf_utils.h
+++ b/libc/src/math/generic/sincosf_utils.h
@@ -14,7 +14,7 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
 
-#if defined(LIBC_TARGET_CPU_HAS_FMA)
+#if defined(LIBC_TARGET_CPU_HAS_FMA_DOUBLE)
 #include "range_reduction_fma.h"
 // using namespace LIBC_NAMESPACE::fma;
 using LIBC_NAMESPACE::fma::FAST_PASS_BOUND;
@@ -27,7 +27,7 @@ using LIBC_NAMESPACE::fma::small_range_reduction;
 using LIBC_NAMESPACE::generic::FAST_PASS_BOUND;
 using LIBC_NAMESPACE::generic::large_range_reduction;
 using LIBC_NAMESPACE::generic::small_range_reduction;
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -108,7 +108,7 @@ LIBC_INLINE void sincosf_eval(double xd, uint32_t x_abs, double &sin_k,
 //   => pi * x = (k + y) * pi / 32
 static LIBC_INLINE int64_t range_reduction_sincospi(double x, double &y) {
   double kd = fputil::nearest_integer(x * 32);
-  y = fputil::multiply_add<double>(x, 32.0, -kd);
+  y = fputil::multiply_add(x, 32.0, -kd);
 
   return static_cast<int64_t>(kd);
 }
diff --git a/libc/src/math/generic/sinf.cpp b/libc/src/math/generic/sinf.cpp
index cea267d4c683e..d27ce843a2c92 100644
--- a/libc/src/math/generic/sinf.cpp
+++ b/libc/src/math/generic/sinf.cpp
@@ -19,7 +19,7 @@
 #include "src/__support/macros/optimization.h"            // LIBC_UNLIKELY
 #include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
 
-#if defined(LIBC_TARGET_CPU_HAS_FMA)
+#if defined(LIBC_TARGET_CPU_HAS_FMA_DOUBLE)
 #include "range_reduction_fma.h"
 #else
 #include "range_reduction.h"
@@ -101,11 +101,11 @@ LLVM_LIBC_FUNCTION(float, sinf, (float x)) {
       // |x| < 2^-125. For targets without FMA instructions, we simply use
       // double for intermediate results as it is more efficient than using an
       // emulated version of FMA.
-#if defined(LIBC_TARGET_CPU_HAS_FMA)
+#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT)
       return fputil::multiply_add(x, -0x1.0p-25f, x);
 #else
       return static_cast<float>(fputil::multiply_add(xd, -0x1.0p-25, xd));
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
     }
 
     // |x| < pi/16.
@@ -124,6 +124,7 @@ LLVM_LIBC_FUNCTION(float, sinf, (float x)) {
     return static_cast<float>(xd * result);
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   if (LIBC_UNLIKELY(x_abs == 0x4619'9998U)) { // x = 0x1.33333p13
     float r = -0x1.63f4bap-2f;
     int rounding = fputil::quick_get_round();
@@ -132,6 +133,7 @@ LLVM_LIBC_FUNCTION(float, sinf, (float x)) {
       r = -0x1.63f4bcp-2f;
     return xbits.is_neg() ? -r : r;
   }
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   if (LIBC_UNLIKELY(x_abs >= 0x7f80'0000U)) {
     if (x_abs == 0x7f80'0000U) {
diff --git a/libc/src/math/generic/sinf16.cpp b/libc/src/math/generic/sinf16.cpp
index 7c15951cccecf..85e55a614588a 100644
--- a/libc/src/math/generic/sinf16.cpp
+++ b/libc/src/math/generic/sinf16.cpp
@@ -19,6 +19,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 constexpr size_t N_EXCEPTS = 4;
 
 constexpr fputil::ExceptValues<float16, N_EXCEPTS> SINF16_EXCEPTS{{
@@ -28,6 +29,7 @@ constexpr fputil::ExceptValues<float16, N_EXCEPTS> SINF16_EXCEPTS{{
     {0x5cb0, 0xbbff, 0, 1, 0},
     {0x51f5, 0xb80f, 0, 1, 0},
 }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, sinf16, (float16 x)) {
   using FPBits = fputil::FPBits<float16>;
@@ -53,11 +55,14 @@ LLVM_LIBC_FUNCTION(float16, sinf16, (float16 x)) {
   //   	      = sin(k * pi/32) * cos(y * pi/32) +
   //   	        sin(y * pi/32) * cos(k * pi/32)
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   // Handle exceptional values
   bool x_sign = x_u >> 15;
+
   if (auto r = SINF16_EXCEPTS.lookup_odd(x_abs, x_sign);
       LIBC_UNLIKELY(r.has_value()))
     return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   int rounding = fputil::quick_get_round();
 
diff --git a/libc/src/math/generic/sinhf.cpp b/libc/src/math/generic/sinhf.cpp
index 371dd6e67e66e..d6158fd302536 100644
--- a/libc/src/math/generic/sinhf.cpp
+++ b/libc/src/math/generic/sinhf.cpp
@@ -24,11 +24,13 @@ LLVM_LIBC_FUNCTION(float, sinhf, (float x)) {
   if (LIBC_UNLIKELY(x_abs >= 0x42b4'0000U || x_abs <= 0x3da0'0000U)) {
     // |x| <= 0.078125
     if (x_abs <= 0x3da0'0000U) {
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
       // |x| = 0.0005589424981735646724700927734375
       if (LIBC_UNLIKELY(x_abs == 0x3a12'85ffU)) {
         if (fputil::fenv_is_round_to_nearest())
           return x;
       }
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
       // |x| <= 2^-26
       if (LIBC_UNLIKELY(x_abs <= 0x3280'0000U)) {
diff --git a/libc/src/math/generic/sinhf16.cpp b/libc/src/math/generic/sinhf16.cpp
index e2dd009dc72c6..680e1cc49a1d8 100644
--- a/libc/src/math/generic/sinhf16.cpp
+++ b/libc/src/math/generic/sinhf16.cpp
@@ -20,6 +20,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 static constexpr fputil::ExceptValues<float16, 16> SINHF16_EXCEPTS_POS = {{
     // x = 0x1.714p-5, sinhf16(x) = 0x1.714p-5 (RZ)
     {0x29c5U, 0x29c5U, 1U, 0U, 1U},
@@ -81,6 +82,7 @@ static constexpr fputil::ExceptValues<float16, 12> SINHF16_EXCEPTS_NEG = {{
     // x = -0x1.5fp+3, sinhf16(x) = -0x1.c54p+14 (RZ)
     {0xc97cU, 0xf715U, 0U, 1U, 1U},
 }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, sinhf16, (float16 x)) {
   using FPBits = fputil::FPBits<float16>;
@@ -130,6 +132,7 @@ LLVM_LIBC_FUNCTION(float16, sinhf16, (float16 x)) {
     return FPBits(static_cast<uint16_t>(x_u)).get_val();
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   if (x_bits.is_pos()) {
     if (auto r = SINHF16_EXCEPTS_POS.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
       return r.value();
@@ -137,6 +140,7 @@ LLVM_LIBC_FUNCTION(float16, sinhf16, (float16 x)) {
     if (auto r = SINHF16_EXCEPTS_NEG.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
       return r.value();
   }
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   return eval_sinh_or_cosh</*IsSinh=*/true>(x);
 }
diff --git a/libc/src/math/generic/tan.cpp b/libc/src/math/generic/tan.cpp
index 19d31a8441efb..a899a2128d384 100644
--- a/libc/src/math/generic/tan.cpp
+++ b/libc/src/math/generic/tan.cpp
@@ -22,11 +22,11 @@
 #include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
 #include "src/math/generic/range_reduction_double_common.h"
 
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 #include "range_reduction_double_fma.h"
 #else
 #include "range_reduction_double_nofma.h"
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -140,7 +140,7 @@ LLVM_LIBC_FUNCTION(double, tan, (double x)) {
         if (LIBC_UNLIKELY(x == 0.0))
           return x + x; // Make sure it works with FTZ/DAZ.
 
-#ifdef LIBC_TARGET_CPU_HAS_FMA
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
         return fputil::multiply_add(x, 0x1.0p-54, x);
 #else
         if (LIBC_UNLIKELY(x_e < 4)) {
@@ -150,7 +150,7 @@ LLVM_LIBC_FUNCTION(double, tan, (double x)) {
             return FPBits(xbits.uintval() + 1).get_val();
         }
         return fputil::multiply_add(x, 0x1.0p-54, x);
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
       }
       // No range reduction needed.
       k = 0;
diff --git a/libc/src/math/generic/tanf.cpp b/libc/src/math/generic/tanf.cpp
index 6fd5f9a103676..a15aa9796cbd8 100644
--- a/libc/src/math/generic/tanf.cpp
+++ b/libc/src/math/generic/tanf.cpp
@@ -21,6 +21,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 // Exceptional cases for tanf.
 constexpr size_t N_EXCEPTS = 6;
 
@@ -39,11 +40,11 @@ constexpr fputil::ExceptValues<float, N_EXCEPTS> TANF_EXCEPTS{{
     // x = 0x1.a6ce12p86, tan(x) = -0x1.c5612ep-1 (RZ)
     {0x6ad36709, 0xbf62b097, 0, 1, 0},
 }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float, tanf, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
-  bool x_sign = xbits.uintval() >> 31;
   uint32_t x_abs = xbits.uintval() & 0x7fff'ffffU;
 
   // |x| < pi/32
@@ -74,11 +75,11 @@ LLVM_LIBC_FUNCTION(float, tanf, (float x)) {
       // |x| < 2^-125. For targets without FMA instructions, we simply use
       // double for intermediate results as it is more efficient than using an
       // emulated version of FMA.
-#if defined(LIBC_TARGET_CPU_HAS_FMA)
+#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT)
       return fputil::multiply_add(x, 0x1.0p-25f, x);
 #else
       return static_cast<float>(fputil::multiply_add(xd, 0x1.0p-25, xd));
-#endif // LIBC_TARGET_CPU_HAS_FMA
+#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
     }
 
     // |x| < pi/32
@@ -92,6 +93,8 @@ LLVM_LIBC_FUNCTION(float, tanf, (float x)) {
     return static_cast<float>(xd * result);
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+  bool x_sign = xbits.uintval() >> 31;
   // Check for exceptional values
   if (LIBC_UNLIKELY(x_abs == 0x3f8a1f62U)) {
     // |x| = 0x1.143ec4p0
@@ -104,6 +107,7 @@ LLVM_LIBC_FUNCTION(float, tanf, (float x)) {
 
     return tmp;
   }
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // |x| > 0x1.ada6a8p+27f
   if (LIBC_UNLIKELY(x_abs > 0x4d56'd354U)) {
@@ -115,10 +119,12 @@ LLVM_LIBC_FUNCTION(float, tanf, (float x)) {
       }
       return x + FPBits::quiet_nan().get_val();
     }
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
     // Other large exceptional values
     if (auto r = TANF_EXCEPTS.lookup_odd(x_abs, x_sign);
         LIBC_UNLIKELY(r.has_value()))
       return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   }
 
   // For |x| >= pi/32, we use the definition of tan(x) function:
diff --git a/libc/src/math/generic/tanf16.cpp b/libc/src/math/generic/tanf16.cpp
index 48aa51e456a8a..97d201b65bbe6 100644
--- a/libc/src/math/generic/tanf16.cpp
+++ b/libc/src/math/generic/tanf16.cpp
@@ -19,6 +19,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 constexpr size_t N_EXCEPTS = 9;
 
 constexpr fputil::ExceptValues<float16, N_EXCEPTS> TANF16_EXCEPTS{{
@@ -33,6 +34,7 @@ constexpr fputil::ExceptValues<float16, N_EXCEPTS> TANF16_EXCEPTS{{
     {0x6f4d, 0xbe19, 0, 1, 1},
     {0x7330, 0xcb62, 0, 1, 0},
 }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, tanf16, (float16 x)) {
   using FPBits = fputil::FPBits<float16>;
@@ -40,13 +42,15 @@ LLVM_LIBC_FUNCTION(float16, tanf16, (float16 x)) {
 
   uint16_t x_u = xbits.uintval();
   uint16_t x_abs = x_u & 0x7fff;
-  bool x_sign = x_u >> 15;
   float xf = x;
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+  bool x_sign = x_u >> 15;
   // Handle exceptional values
   if (auto r = TANF16_EXCEPTS.lookup_odd(x_abs, x_sign);
       LIBC_UNLIKELY(r.has_value()))
     return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // |x| <= 0x1.d1p-5
   if (LIBC_UNLIKELY(x_abs <= 0x2b44)) {
diff --git a/libc/src/math/generic/tanhf16.cpp b/libc/src/math/generic/tanhf16.cpp
index 0266b5cfc2df1..4c43cfd29153f 100644
--- a/libc/src/math/generic/tanhf16.cpp
+++ b/libc/src/math/generic/tanhf16.cpp
@@ -24,12 +24,14 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 static constexpr fputil::ExceptValues<float16, 2> TANHF16_EXCEPTS = {{
     // x = 0x1.f54p+0, tanhf16(x) = 0x1.ecp-1 (RZ)
     {0x3fd5U, 0x3bb0U, 1U, 0U, 0U},
     // x = -0x1.f54p+0, tanhf16(x) = -0x1.ecp-1 (RZ)
     {0xbfd5U, 0xbbb0U, 0U, 1U, 0U},
 }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, tanhf16, (float16 x)) {
   using FPBits = fputil::FPBits<float16>;
@@ -98,8 +100,10 @@ LLVM_LIBC_FUNCTION(float16, tanhf16, (float16 x)) {
     return fputil::cast<float16>(-0x1.ffcp-1);
   }
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   if (auto r = TANHF16_EXCEPTS.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
     return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
   // For atanh(-1 + 2^(-11)) < x < atanh(1 - 2^(-11)), to compute tanh(x), we
   // perform the following range reduction: find hi, mid, lo, such that:
diff --git a/libc/src/math/generic/tanpif16.cpp b/libc/src/math/generic/tanpif16.cpp
index cf4f9917d4537..71cf25c9741a1 100644
--- a/libc/src/math/generic/tanpif16.cpp
+++ b/libc/src/math/generic/tanpif16.cpp
@@ -19,6 +19,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 constexpr size_t N_EXCEPTS = 21;
 
 constexpr fputil::ExceptValues<float16, N_EXCEPTS> TANPIF16_EXCEPTS{{
@@ -35,6 +36,7 @@ constexpr fputil::ExceptValues<float16, N_EXCEPTS> TANPIF16_EXCEPTS{{
     {0x4135, 0xc1ee, 0, 1, 0}, {0x42cb, 0x41ee, 1, 0, 0},
     {0x4335, 0xc1ee, 0, 1, 0},
 }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, tanpif16, (float16 x)) {
   using FPBits = typename fputil::FPBits<float16>;
@@ -48,10 +50,13 @@ LLVM_LIBC_FUNCTION(float16, tanpif16, (float16 x)) {
     if (LIBC_UNLIKELY(x_abs == 0U))
       return x;
 
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
     bool x_sign = x_u >> 15;
+
     if (auto r = TANPIF16_EXCEPTS.lookup_odd(x_abs, x_sign);
         LIBC_UNLIKELY(r.has_value()))
       return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
   }
 
   // Numbers greater or equal to 2^10 are integers, or infinity, or NaN
diff --git a/libc/src/stdio/printf_core/parser.h b/libc/src/stdio/printf_core/parser.h
index acbbaa25b1c9d..89556f1a9e5f2 100644
--- a/libc/src/stdio/printf_core/parser.h
+++ b/libc/src/stdio/printf_core/parser.h
@@ -57,7 +57,8 @@ template <typename T> using int_type_of_v = typename int_type_of<T>::type;
     if (!temp.has_value()) {                                                   \
       section.has_conv = false;                                                \
     } else {                                                                   \
-      dst = cpp::bit_cast<int_type_of_v<arg_type>>(temp.value());              \
+      dst = static_cast<decltype(dst)>(                                        \
+          cpp::bit_cast<int_type_of_v<arg_type>>(temp.value()));               \
     }                                                                          \
   }
 #else
@@ -130,7 +131,7 @@ template <typename ArgProvider> class Parser {
       } else if (internal::isdigit(str[cur_pos])) {
         auto result = internal::strtointeger<int>(str + cur_pos, 10);
         section.min_width = result.value;
-        cur_pos = cur_pos + result.parsed_len;
+        cur_pos = cur_pos + static_cast<size_t>(result.parsed_len);
       }
       if (section.min_width < 0) {
         section.min_width =
@@ -153,7 +154,7 @@ template <typename ArgProvider> class Parser {
         } else if (internal::isdigit(str[cur_pos])) {
           auto result = internal::strtointeger<int>(str + cur_pos, 10);
           section.precision = result.value;
-          cur_pos = cur_pos + result.parsed_len;
+          cur_pos = cur_pos + static_cast<size_t>(result.parsed_len);
         }
       }
 
@@ -265,7 +266,8 @@ template <typename ArgProvider> class Parser {
       case ('m'):
         // %m is an odd conversion in that it doesn't consume an argument, it
         // just takes the current value of errno as its argument.
-        section.conv_val_raw = static_cast<int>(libc_errno);
+        section.conv_val_raw =
+            static_cast<fputil::FPBits<double>::StorageType>(libc_errno);
         break;
 #endif // LIBC_COPT_PRINTF_DISABLE_STRERROR
 #ifndef LIBC_COPT_PRINTF_DISABLE_WRITE_INT
@@ -356,7 +358,7 @@ template <typename ArgProvider> class Parser {
       }
       if (internal::isdigit(str[*local_pos])) {
         const auto result = internal::strtointeger<int>(str + *local_pos, 10);
-        *local_pos += result.parsed_len;
+        *local_pos += static_cast<size_t>(result.parsed_len);
         return {lm, static_cast<size_t>(cpp::max(0, result.value))};
       }
       return {lm, 0};
@@ -405,10 +407,10 @@ template <typename ArgProvider> class Parser {
   LIBC_INLINE size_t parse_index(size_t *local_pos) {
     if (internal::isdigit(str[*local_pos])) {
       auto result = internal::strtointeger<int>(str + *local_pos, 10);
-      size_t index = result.value;
-      if (str[*local_pos + result.parsed_len] != '$')
+      size_t index = static_cast<size_t>(result.value);
+      if (str[*local_pos + static_cast<size_t>(result.parsed_len)] != '$')
         return 0;
-      *local_pos = 1 + result.parsed_len + *local_pos;
+      *local_pos = static_cast<size_t>(1 + result.parsed_len) + *local_pos;
       return index;
     }
     return 0;
diff --git a/libc/src/stdio/printf_core/writer.h b/libc/src/stdio/printf_core/writer.h
index 5526a478ea620..28c9a28270875 100644
--- a/libc/src/stdio/printf_core/writer.h
+++ b/libc/src/stdio/printf_core/writer.h
@@ -135,7 +135,8 @@ class Writer final {
     chars_written += static_cast<int>(length);
 
     if (LIBC_LIKELY(wb->buff_cur + length <= wb->buff_len)) {
-      inline_memset(wb->buff + wb->buff_cur, new_char, length);
+      inline_memset(wb->buff + wb->buff_cur,
+                    static_cast<unsigned char>(new_char), length);
       wb->buff_cur += length;
       return WRITE_OK;
     }
diff --git a/libc/src/stdio/scanf_core/parser.h b/libc/src/stdio/scanf_core/parser.h
index 6cc5b30eb0db2..1e2f26e0d3fdd 100644
--- a/libc/src/stdio/scanf_core/parser.h
+++ b/libc/src/stdio/scanf_core/parser.h
@@ -78,7 +78,7 @@ template <typename ArgProvider> class Parser {
       if (internal::isdigit(str[cur_pos])) {
         auto result = internal::strtointeger<int>(str + cur_pos, 10);
         section.max_width = result.value;
-        cur_pos = cur_pos + result.parsed_len;
+        cur_pos = cur_pos + static_cast<size_t>(result.parsed_len);
       }
 
       // TODO(michaelrj): add posix allocate flag support.
@@ -150,10 +150,11 @@ template <typename ArgProvider> class Parser {
             char b = str[cur_pos + 1];
             char start = (a < b ? a : b);
             char end = (a < b ? b : a);
-            scan_set.set_range(start, end);
+            scan_set.set_range(static_cast<size_t>(start),
+                               static_cast<size_t>(end));
             cur_pos += 2;
           } else {
-            scan_set.set(str[cur_pos]);
+            scan_set.set(static_cast<size_t>(str[cur_pos]));
             ++cur_pos;
           }
         }
@@ -237,10 +238,10 @@ template <typename ArgProvider> class Parser {
   LIBC_INLINE size_t parse_index(size_t *local_pos) {
     if (internal::isdigit(str[*local_pos])) {
       auto result = internal::strtointeger<int>(str + *local_pos, 10);
-      size_t index = result.value;
-      if (str[*local_pos + result.parsed_len] != '$')
+      size_t index = static_cast<size_t>(result.value);
+      if (str[*local_pos + static_cast<size_t>(result.parsed_len)] != '$')
         return 0;
-      *local_pos = 1 + result.parsed_len + *local_pos;
+      *local_pos = static_cast<size_t>(1 + result.parsed_len) + *local_pos;
       return index;
     }
     return 0;
diff --git a/libc/src/stdlib/quick_sort.h b/libc/src/stdlib/quick_sort.h
index 9ab2830250018..8ba0098854d19 100644
--- a/libc/src/stdlib/quick_sort.h
+++ b/libc/src/stdlib/quick_sort.h
@@ -167,7 +167,9 @@ LIBC_INLINE void quick_sort_impl(A &array, const void *ancestor_pivot,
   }
 }
 
-constexpr size_t ilog2(size_t n) { return cpp::bit_width(n) - 1; }
+constexpr size_t ilog2(size_t n) {
+  return static_cast<size_t>(cpp::bit_width(n)) - 1;
+}
 
 template <typename A, typename F>
 LIBC_INLINE void quick_sort(A &array, const F &is_less) {
diff --git a/libc/src/string/memory_utils/utils.h b/libc/src/string/memory_utils/utils.h
index 5c9bc72208f85..bdf0b8652188b 100644
--- a/libc/src/string/memory_utils/utils.h
+++ b/libc/src/string/memory_utils/utils.h
@@ -263,7 +263,7 @@ LIBC_INLINE void store_aligned(ValueType value, Ptr dst) {
   static_assert(sizeof(ValueType) >= (sizeof(T) + ... + sizeof(TS)));
   constexpr size_t SHIFT = sizeof(T) * 8;
   if constexpr (Endian::IS_LITTLE) {
-    store<T>(assume_aligned<sizeof(T)>(dst), value & ~T(0));
+    store<T>(assume_aligned<sizeof(T)>(dst), T(value & T(~0)));
     if constexpr (sizeof...(TS) > 0)
       store_aligned<ValueType, TS...>(value >> SHIFT, dst + sizeof(T));
   } else if constexpr (Endian::IS_BIG) {
@@ -297,7 +297,7 @@ LIBC_INLINE void adjust(ptrdiff_t offset, T1 *__restrict &p1,
                         T2 *__restrict &p2, size_t &count) {
   p1 += offset;
   p2 += offset;
-  count -= offset;
+  count -= static_cast<size_t>(offset);
 }
 
 // Advances p1 and p2 so p1 gets aligned to the next SIZE bytes boundary
@@ -306,7 +306,8 @@ LIBC_INLINE void adjust(ptrdiff_t offset, T1 *__restrict &p1,
 template <size_t SIZE, typename T1, typename T2>
 void align_p1_to_next_boundary(T1 *__restrict &p1, T2 *__restrict &p2,
                                size_t &count) {
-  adjust(distance_to_next_aligned<SIZE>(p1), p1, p2, count);
+  adjust(static_cast<ptrdiff_t>(distance_to_next_aligned<SIZE>(p1)), p1, p2,
+         count);
   p1 = assume_aligned<SIZE>(p1);
 }
 
diff --git a/libc/src/string/string_utils.h b/libc/src/string/string_utils.h
index 583d35014d398..e4659f65c93e2 100644
--- a/libc/src/string/string_utils.h
+++ b/libc/src/string/string_utils.h
@@ -66,7 +66,7 @@ LIBC_INLINE size_t string_length_wide_read(const char *src) {
   for (; reinterpret_cast<uintptr_t>(char_ptr) % sizeof(Word) != 0;
        ++char_ptr) {
     if (*char_ptr == '\0')
-      return char_ptr - src;
+      return static_cast<size_t>(char_ptr - src);
   }
   // Step 2: read blocks
   for (const Word *block_ptr = reinterpret_cast<const Word *>(char_ptr);
@@ -77,7 +77,7 @@ LIBC_INLINE size_t string_length_wide_read(const char *src) {
   for (; *char_ptr != '\0'; ++char_ptr) {
     ;
   }
-  return char_ptr - src;
+  return static_cast<size_t>(char_ptr - src);
 }
 
 // Returns the length of a string, denoted by the first occurrence
@@ -169,7 +169,7 @@ LIBC_INLINE size_t complementary_span(const char *src, const char *segment) {
   for (; *src && !bitset.test(*reinterpret_cast<const unsigned char *>(src));
        ++src)
     ;
-  return src - initial;
+  return static_cast<size_t>(src - initial);
 }
 
 // Given the similarities between strtok and strtok_r, we can implement both
@@ -189,12 +189,14 @@ LIBC_INLINE char *string_token(char *__restrict src,
   if (LIBC_UNLIKELY(src == nullptr && ((src = *saveptr) == nullptr)))
     return nullptr;
 
+  static_assert(sizeof(char) == sizeof(cpp::byte),
+                "bitset of 256 assumes char is 8 bits");
   cpp::bitset<256> delimiter_set;
   for (; *delimiter_string != '\0'; ++delimiter_string)
-    delimiter_set.set(*delimiter_string);
+    delimiter_set.set(static_cast<size_t>(*delimiter_string));
 
   if constexpr (SkipDelim)
-    for (; *src != '\0' && delimiter_set.test(*src); ++src)
+    for (; *src != '\0' && delimiter_set.test(static_cast<size_t>(*src)); ++src)
       ;
   if (*src == '\0') {
     *saveptr = src;
@@ -202,7 +204,7 @@ LIBC_INLINE char *string_token(char *__restrict src,
   }
   char *token = src;
   for (; *src != '\0'; ++src) {
-    if (delimiter_set.test(*src)) {
+    if (delimiter_set.test(static_cast<size_t>(*src))) {
       *src = '\0';
       ++src;
       break;
diff --git a/libc/src/strings/CMakeLists.txt b/libc/src/strings/CMakeLists.txt
index 5e84c7be1f7d6..d312495ae8d91 100644
--- a/libc/src/strings/CMakeLists.txt
+++ b/libc/src/strings/CMakeLists.txt
@@ -54,6 +54,36 @@ add_entrypoint_object(
     bcopy.h
 )
 
+add_entrypoint_object(
+  ffs
+  SRCS
+    ffs.cpp
+  HDRS
+    ffs.h
+  DEPENDS
+    libc.src.__support.math_extras
+)
+
+add_entrypoint_object(
+  ffsl
+  SRCS
+    ffsl.cpp
+  HDRS
+    ffsl.h
+  DEPENDS
+    libc.src.__support.math_extras
+)
+
+add_entrypoint_object(
+  ffsll
+  SRCS
+    ffsll.cpp
+  HDRS
+    ffsll.h
+  DEPENDS
+    libc.src.__support.math_extras
+)
+
 add_entrypoint_object(
   index
   SRCS
@@ -85,6 +115,18 @@ add_entrypoint_object(
     libc.src.string.memory_utils.inline_strcmp
 )
 
+add_entrypoint_object(
+  strcasecmp_l
+  SRCS
+    strcasecmp_l.cpp
+  HDRS
+    strcasecmp_l.h
+  DEPENDS
+    libc.hdr.types.locale_t
+    libc.src.__support.ctype_utils
+    libc.src.string.memory_utils.inline_strcmp
+)
+
 add_entrypoint_object(
   strncasecmp
   SRCS
@@ -95,3 +137,15 @@ add_entrypoint_object(
     libc.src.__support.ctype_utils
     libc.src.string.memory_utils.inline_strcmp
 )
+
+add_entrypoint_object(
+  strncasecmp_l
+  SRCS
+    strncasecmp_l.cpp
+  HDRS
+    strncasecmp_l.h
+  DEPENDS
+    libc.hdr.types.locale_t
+    libc.src.__support.ctype_utils
+    libc.src.string.memory_utils.inline_strcmp
+)
diff --git a/libc/src/strings/ffs.cpp b/libc/src/strings/ffs.cpp
new file mode 100644
index 0000000000000..5e1efafeab56a
--- /dev/null
+++ b/libc/src/strings/ffs.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of ffs ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/strings/ffs.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/math_extras.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, ffs, (int i)) {
+  return first_trailing_one(static_cast<unsigned>(i));
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/strings/ffs.h b/libc/src/strings/ffs.h
new file mode 100644
index 0000000000000..bf43c43caedb2
--- /dev/null
+++ b/libc/src/strings/ffs.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for ffs ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRINGS_FFS_H
+#define LLVM_LIBC_SRC_STRINGS_FFS_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int ffs(int i);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_STRINGS_FFS_H
diff --git a/libc/src/strings/ffsl.cpp b/libc/src/strings/ffsl.cpp
new file mode 100644
index 0000000000000..b1b25598e405b
--- /dev/null
+++ b/libc/src/strings/ffsl.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of ffsl --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/strings/ffsl.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/math_extras.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, ffsl, (long i)) {
+  return first_trailing_one(static_cast<unsigned long>(i));
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/strings/ffsl.h b/libc/src/strings/ffsl.h
new file mode 100644
index 0000000000000..1feca010b2ebb
--- /dev/null
+++ b/libc/src/strings/ffsl.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for ffsl --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRINGS_FFSL_H
+#define LLVM_LIBC_SRC_STRINGS_FFSL_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int ffsl(long i);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_STRINGS_FFSL_H
diff --git a/libc/src/strings/ffsll.cpp b/libc/src/strings/ffsll.cpp
new file mode 100644
index 0000000000000..be16e81a54711
--- /dev/null
+++ b/libc/src/strings/ffsll.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of ffsll -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/strings/ffsll.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/math_extras.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, ffsll, (long long i)) {
+  return first_trailing_one(static_cast<unsigned long long>(i));
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/strings/ffsll.h b/libc/src/strings/ffsll.h
new file mode 100644
index 0000000000000..f059b8ab89b4e
--- /dev/null
+++ b/libc/src/strings/ffsll.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for ffsll -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRINGS_FFSLL_H
+#define LLVM_LIBC_SRC_STRINGS_FFSLL_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int ffsll(long long i);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_STRINGS_FFSLL_H
diff --git a/libc/src/strings/strcasecmp_l.cpp b/libc/src/strings/strcasecmp_l.cpp
new file mode 100644
index 0000000000000..95117cb27a564
--- /dev/null
+++ b/libc/src/strings/strcasecmp_l.cpp
@@ -0,0 +1,27 @@
+//===-- Implementation of strcasecmp_l ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/strings/strcasecmp_l.h"
+
+#include "src/__support/common.h"
+#include "src/__support/ctype_utils.h"
+#include "src/__support/macros/config.h"
+#include "src/string/memory_utils/inline_strcmp.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, strcasecmp_l,
+                   (const char *left, const char *right, locale_t)) {
+  auto case_cmp = [](char a, char b) {
+    return LIBC_NAMESPACE::internal::tolower(a) -
+           LIBC_NAMESPACE::internal::tolower(b);
+  };
+  return inline_strcmp(left, right, case_cmp);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/strings/strcasecmp_l.h b/libc/src/strings/strcasecmp_l.h
new file mode 100644
index 0000000000000..48074460205ce
--- /dev/null
+++ b/libc/src/strings/strcasecmp_l.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for strcasecmp_l ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRINGS_STRCASECMP_L_H
+#define LLVM_LIBC_SRC_STRINGS_STRCASECMP_L_H
+
+#include "hdr/types/locale_t.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int strcasecmp_l(const char *left, const char *right, locale_t locale);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_STRINGS_STRCASECMP_L_H
diff --git a/libc/src/strings/strncasecmp_l.cpp b/libc/src/strings/strncasecmp_l.cpp
new file mode 100644
index 0000000000000..91ac7e5e89107
--- /dev/null
+++ b/libc/src/strings/strncasecmp_l.cpp
@@ -0,0 +1,27 @@
+//===-- Implementation of strncasecmp_l -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/strings/strncasecmp_l.h"
+
+#include "src/__support/common.h"
+#include "src/__support/ctype_utils.h"
+#include "src/__support/macros/config.h"
+#include "src/string/memory_utils/inline_strcmp.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, strncasecmp_l,
+                   (const char *left, const char *right, size_t n, locale_t)) {
+  auto case_cmp = [](char a, char b) {
+    return LIBC_NAMESPACE::internal::tolower(a) -
+           LIBC_NAMESPACE::internal::tolower(b);
+  };
+  return inline_strncmp(left, right, n, case_cmp);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/strings/strncasecmp_l.h b/libc/src/strings/strncasecmp_l.h
new file mode 100644
index 0000000000000..6c6c7f17b4438
--- /dev/null
+++ b/libc/src/strings/strncasecmp_l.h
@@ -0,0 +1,23 @@
+//===-- Implementation header for strncasecmp_l -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRINGS_STRNCASECMP_L_H
+#define LLVM_LIBC_SRC_STRINGS_STRNCASECMP_L_H
+
+#include "hdr/types/locale_t.h"
+#include "src/__support/macros/config.h"
+#include <stddef.h>
+
+namespace LIBC_NAMESPACE_DECL {
+
+int strncasecmp_l(const char *left, const char *right, size_t n,
+                  locale_t locale);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_STRINGS_STRNCASECMP_L_H
diff --git a/libc/test/UnitTest/ExecuteFunction.h b/libc/test/UnitTest/ExecuteFunction.h
index 511249be0f3c9..93ab6e9733804 100644
--- a/libc/test/UnitTest/ExecuteFunction.h
+++ b/libc/test/UnitTest/ExecuteFunction.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIBC_TEST_UNITTEST_EXECUTEFUNCTION_H
 #define LLVM_LIBC_TEST_UNITTEST_EXECUTEFUNCTION_H
 
+#include "src/__support/CPP/limits.h"
 #include "src/__support/macros/config.h"
 #include <stdint.h>
 
@@ -25,7 +26,7 @@ struct ProcessStatus {
   int platform_defined;
   const char *failure = nullptr;
 
-  static constexpr unsigned TIMEOUT = ~0U;
+  static constexpr int TIMEOUT = cpp::numeric_limits<int>::max();
 
   static ProcessStatus error(const char *error) { return {0, error}; }
   static ProcessStatus timed_out_ps() {
@@ -41,9 +42,8 @@ struct ProcessStatus {
   int get_fatal_signal();
 };
 
-ProcessStatus
-invoke_in_subprocess(FunctionCaller *func,
-                     unsigned timeout_ms = ProcessStatus::TIMEOUT);
+ProcessStatus invoke_in_subprocess(FunctionCaller *func,
+                                   int timeout_ms = ProcessStatus::TIMEOUT);
 
 const char *signal_as_string(int signum);
 
diff --git a/libc/test/UnitTest/ExecuteFunctionUnix.cpp b/libc/test/UnitTest/ExecuteFunctionUnix.cpp
index c90d7e0cb9ff1..c0e85c2144005 100644
--- a/libc/test/UnitTest/ExecuteFunctionUnix.cpp
+++ b/libc/test/UnitTest/ExecuteFunctionUnix.cpp
@@ -34,7 +34,7 @@ int ProcessStatus::get_fatal_signal() {
   return WTERMSIG(platform_defined);
 }
 
-ProcessStatus invoke_in_subprocess(FunctionCaller *func, unsigned timeout_ms) {
+ProcessStatus invoke_in_subprocess(FunctionCaller *func, int timeout_ms) {
   int pipe_fds[2];
   if (::pipe(pipe_fds) == -1) {
     delete func;
diff --git a/libc/test/UnitTest/LibcTest.cpp b/libc/test/UnitTest/LibcTest.cpp
index 4395c7cf6e201..fec45982f3e63 100644
--- a/libc/test/UnitTest/LibcTest.cpp
+++ b/libc/test/UnitTest/LibcTest.cpp
@@ -158,13 +158,13 @@ int Test::runTests(const TestOptions &Options) {
     }
 
     tlog << green << "[ RUN      ] " << reset << TestName << '\n';
-    [[maybe_unused]] const uint64_t start_time = clock();
+    [[maybe_unused]] const uint64_t start_time = static_cast<uint64_t>(clock());
     RunContext Ctx;
     T->SetUp();
     T->setContext(&Ctx);
     T->Run();
     T->TearDown();
-    [[maybe_unused]] const uint64_t end_time = clock();
+    [[maybe_unused]] const uint64_t end_time = static_cast<uint64_t>(clock());
     switch (Ctx.status()) {
     case RunContext::RunResult::Fail:
       tlog << red << "[  FAILED  ] " << reset << TestName << '\n';
diff --git a/libc/test/UnitTest/MemoryMatcher.h b/libc/test/UnitTest/MemoryMatcher.h
index 617231520bf20..cbbf3286c02ac 100644
--- a/libc/test/UnitTest/MemoryMatcher.h
+++ b/libc/test/UnitTest/MemoryMatcher.h
@@ -49,7 +49,7 @@ class MemoryMatcher : public Matcher<MemoryView> {
   MemoryView expected;
   MemoryView actual;
   bool mismatch_size = false;
-  size_t mismatch_index = -1;
+  size_t mismatch_index = cpp::numeric_limits<size_t>::max();
 
 public:
   MemoryMatcher(MemoryView expectedValue) : expected(expectedValue) {}
diff --git a/libc/test/src/__support/CPP/bit_test.cpp b/libc/test/src/__support/CPP/bit_test.cpp
index 1f2315281bc1d..89e2a75e0680f 100644
--- a/libc/test/src/__support/CPP/bit_test.cpp
+++ b/libc/test/src/__support/CPP/bit_test.cpp
@@ -24,6 +24,7 @@ using UnsignedTypes = testing::TypeList<
     unsigned char, unsigned short, unsigned int, unsigned long,
     unsigned long long, UInt<128>>;
 
+#ifdef FAKE_MACRO_DISABLE
 TYPED_TEST(LlvmLibcBitTest, HasSingleBit, UnsignedTypes) {
   constexpr auto ZERO = T(0);
   constexpr auto ALL_ONES = T(~ZERO);
@@ -46,32 +47,33 @@ TYPED_TEST(LlvmLibcBitTest, HasSingleBit, UnsignedTypes) {
     EXPECT_FALSE(has_single_bit<T>(two_bits_value));
   }
 }
+#endif
 
 TYPED_TEST(LlvmLibcBitTest, CountLZero, UnsignedTypes) {
   EXPECT_EQ(countl_zero<T>(T(0)), cpp::numeric_limits<T>::digits);
   int expected = 0;
-  for (T value = ~T(0); value; value >>= 1, ++expected)
+  for (T value = T(~0); value; value >>= 1, ++expected)
     EXPECT_EQ(countl_zero<T>(value), expected);
 }
 
 TYPED_TEST(LlvmLibcBitTest, CountRZero, UnsignedTypes) {
   EXPECT_EQ(countr_zero<T>(T(0)), cpp::numeric_limits<T>::digits);
   int expected = 0;
-  for (T value = ~T(0); value; value <<= 1, ++expected)
+  for (T value = T(~0); value; value <<= 1, ++expected)
     EXPECT_EQ(countr_zero<T>(value), expected);
 }
 
 TYPED_TEST(LlvmLibcBitTest, CountLOne, UnsignedTypes) {
   EXPECT_EQ(countl_one<T>(T(0)), 0);
   int expected = cpp::numeric_limits<T>::digits;
-  for (T value = ~T(0); value; value <<= 1, --expected)
+  for (T value = T(~0); value; value <<= 1, --expected)
     EXPECT_EQ(countl_one<T>(value), expected);
 }
 
 TYPED_TEST(LlvmLibcBitTest, CountROne, UnsignedTypes) {
   EXPECT_EQ(countr_one<T>(T(0)), 0);
   int expected = cpp::numeric_limits<T>::digits;
-  for (T value = ~T(0); value; value >>= 1, --expected)
+  for (T value = T(~0); value; value >>= 1, --expected)
     EXPECT_EQ(countr_one<T>(value), expected);
 }
 
@@ -163,7 +165,7 @@ TEST(LlvmLibcBitTest, BitFloor) {
 
 TYPED_TEST(LlvmLibcBitTest, RotateIsInvariantForZeroAndOne, UnsignedTypes) {
   constexpr T all_zeros = T(0);
-  constexpr T all_ones = ~T(0);
+  constexpr T all_ones = T(~0);
   for (int i = 0; i < cpp::numeric_limits<T>::digits; ++i) {
     EXPECT_EQ(rotl<T>(all_zeros, i), all_zeros);
     EXPECT_EQ(rotl<T>(all_ones, i), all_ones);
@@ -227,8 +229,9 @@ TEST(LlvmLibcBitTest, Rotr) {
 TYPED_TEST(LlvmLibcBitTest, CountOnes, UnsignedTypes) {
   EXPECT_EQ(popcount(T(0)), 0);
   for (int i = 0; i != cpp::numeric_limits<T>::digits; ++i)
-    EXPECT_EQ(popcount<T>(cpp::numeric_limits<T>::max() >> i),
-              cpp::numeric_limits<T>::digits - i);
+    EXPECT_EQ(
+        popcount<T>(cpp::numeric_limits<T>::max() >> static_cast<size_t>(i)),
+        cpp::numeric_limits<T>::digits - i);
 }
 
 } // namespace cpp
diff --git a/libc/test/src/__support/arg_list_test.cpp b/libc/test/src/__support/arg_list_test.cpp
index 79a715e910687..70db0aafa631a 100644
--- a/libc/test/src/__support/arg_list_test.cpp
+++ b/libc/test/src/__support/arg_list_test.cpp
@@ -72,9 +72,9 @@ long int check_primitives(int first, ...) {
   count += args.next_var<unsigned long>();
   count += args.next_var<long long>();
   count += args.next_var<unsigned long long>();
-  count += args.next_var<double>();
-  count += args.next_var<double>();
-  count += args.next_var<long double>();
+  count += static_cast<long int>(args.next_var<double>());
+  count += static_cast<long int>(args.next_var<double>());
+  count += static_cast<long int>(args.next_var<long double>());
   count += *args.next_var<int *>();
   return count;
 }
@@ -112,7 +112,8 @@ long int check_struct_type(int first, ...) {
 
   S s = args.next_var<S>();
   int last = args.next_var<int>();
-  return s.c + s.s + s.i + s.l + s.f + s.d + last;
+  return static_cast<long int>(s.c + s.s + s.i + static_cast<long>(s.l) + s.f +
+                               s.d + last);
 }
 
 TEST(LlvmLibcArgListTest, TestStructTypes) {
diff --git a/libc/test/src/__support/big_int_test.cpp b/libc/test/src/__support/big_int_test.cpp
index 2666ed978dad7..93a66d59650b4 100644
--- a/libc/test/src/__support/big_int_test.cpp
+++ b/libc/test/src/__support/big_int_test.cpp
@@ -197,8 +197,8 @@ TYPED_TEST(LlvmLibcUIntClassTest, CountBits, Types) {
     for (size_t i = 0; i < T::BITS; ++i) {
       const auto l_one = T::all_ones() << i; // 0b111...000
       const auto r_one = T::all_ones() >> i; // 0b000...111
-      const int zeros = i;
-      const int ones = T::BITS - zeros;
+      const int zeros = static_cast<int>(i);
+      const int ones = static_cast<int>(T::BITS - static_cast<size_t>(zeros));
       ASSERT_EQ(cpp::countr_one(r_one), ones);
       ASSERT_EQ(cpp::countl_one(l_one), ones);
       ASSERT_EQ(cpp::countr_zero(l_one), zeros);
@@ -871,13 +871,13 @@ TEST(LlvmLibcUIntClassTest, ConstructorFromUInt128Tests) {
   ASSERT_EQ(static_cast<int>(c >> 64), 123);
   ASSERT_EQ(static_cast<uint64_t>(d), static_cast<uint64_t>(b));
   ASSERT_EQ(static_cast<uint64_t>(d >> 64), static_cast<uint64_t>(b >> 64));
-  ASSERT_EQ(c + d, LL_Int128(a + b));
+  ASSERT_EQ(c + d, LL_Int128(a + static_cast<__uint128_t>(b)));
 
   ASSERT_EQ(static_cast<int>(e), 1);
   ASSERT_EQ(static_cast<int>(e >> 64), 123);
   ASSERT_EQ(static_cast<uint64_t>(f), static_cast<uint64_t>(b));
   ASSERT_EQ(static_cast<uint64_t>(f >> 64), static_cast<uint64_t>(b >> 64));
-  ASSERT_EQ(LL_UInt192(e + f), LL_UInt192(a + b));
+  ASSERT_EQ(LL_UInt192(e + f), LL_UInt192(a + static_cast<__uint128_t>(b)));
 }
 
 TEST(LlvmLibcUIntClassTest, WordTypeUInt128Tests) {
diff --git a/libc/test/src/__support/blockstore_test.cpp b/libc/test/src/__support/blockstore_test.cpp
index de7bd72b96c80..364155cd92cfa 100644
--- a/libc/test/src/__support/blockstore_test.cpp
+++ b/libc/test/src/__support/blockstore_test.cpp
@@ -27,7 +27,7 @@ class LlvmLibcBlockStoreTest : public LIBC_NAMESPACE::testing::Test {
     for (auto iter = block_store.begin(); iter != end; ++iter, ++i) {
       Element &e = *iter;
       if (REVERSE) {
-        int j = ELEMENT_COUNT - 1 - i;
+        int j = static_cast<int>(ELEMENT_COUNT - 1) - i;
         ASSERT_EQ(e.a, j);
         ASSERT_EQ(e.b, long(j * 2));
         ASSERT_EQ(e.c, unsigned(j * 3));
diff --git a/libc/test/src/__support/fixedvector_test.cpp b/libc/test/src/__support/fixedvector_test.cpp
index b73df04d03746..8be18a52ec015 100644
--- a/libc/test/src/__support/fixedvector_test.cpp
+++ b/libc/test/src/__support/fixedvector_test.cpp
@@ -92,7 +92,7 @@ TEST(LlvmLibcFixedVectorTest, ForwardIteration) {
   LIBC_NAMESPACE::FixedVector<int, 5> vec(arr.begin(), arr.end());
   ASSERT_EQ(vec.size(), arr.size());
   for (auto it = vec.begin(); it != vec.end(); ++it) {
-    auto idx = it - vec.begin();
+    auto idx = static_cast<size_t>(it - vec.begin());
     ASSERT_EQ(*it, arr[idx]);
   }
 }
@@ -102,7 +102,7 @@ TEST(LlvmLibcFixedVectorTest, ConstForwardIteration) {
   const LIBC_NAMESPACE::FixedVector<int, 5> vec(arr.begin(), arr.end());
   ASSERT_EQ(vec.size(), arr.size());
   for (auto it = vec.begin(); it != vec.end(); ++it) {
-    auto idx = it - vec.begin();
+    auto idx = static_cast<size_t>(it - vec.begin());
     ASSERT_EQ(*it, arr[idx]);
   }
 }
diff --git a/libc/test/src/__support/hash_test.cpp b/libc/test/src/__support/hash_test.cpp
index f23a43a3bc5e4..94c884cc7fb17 100644
--- a/libc/test/src/__support/hash_test.cpp
+++ b/libc/test/src/__support/hash_test.cpp
@@ -78,7 +78,7 @@ TEST(LlvmLibcHashTest, Avalanche) {
       }
       for (size_t i = 0; i < sz; ++i) {
         for (size_t j = 0; j < 8; ++j) {
-          uint8_t mask = 1 << j;
+          uint8_t mask = static_cast<uint8_t>(1 << j);
           mem.data[i] ^= mask;
           {
             LIBC_NAMESPACE::internal::HashState state{0xabcdef1234567890};
diff --git a/libc/test/src/__support/integer_to_string_test.cpp b/libc/test/src/__support/integer_to_string_test.cpp
index 6ec2dc6eba521..7d4006512093e 100644
--- a/libc/test/src/__support/integer_to_string_test.cpp
+++ b/libc/test/src/__support/integer_to_string_test.cpp
@@ -41,7 +41,7 @@ TEST(LlvmLibcIntegerToStringTest, UINT8) {
   EXPECT(type, 12, "12");
   EXPECT(type, 123, "123");
   EXPECT(type, UINT8_MAX, "255");
-  EXPECT(type, -1, "255");
+  EXPECT(type, static_cast<uint8_t>(-1), "255");
 }
 
 TEST(LlvmLibcIntegerToStringTest, INT8) {
@@ -65,7 +65,7 @@ TEST(LlvmLibcIntegerToStringTest, UINT16) {
   EXPECT(type, 1234, "1234");
   EXPECT(type, 12345, "12345");
   EXPECT(type, UINT16_MAX, "65535");
-  EXPECT(type, -1, "65535");
+  EXPECT(type, static_cast<uint16_t>(-1), "65535");
 }
 
 TEST(LlvmLibcIntegerToStringTest, INT16) {
@@ -99,7 +99,7 @@ TEST(LlvmLibcIntegerToStringTest, UINT32) {
   EXPECT(type, 123456789, "123456789");
   EXPECT(type, 1234567890, "1234567890");
   EXPECT(type, UINT32_MAX, "4294967295");
-  EXPECT(type, -1, "4294967295");
+  EXPECT(type, static_cast<uint32_t>(-1), "4294967295");
 }
 
 TEST(LlvmLibcIntegerToStringTest, INT32) {
@@ -144,7 +144,7 @@ TEST(LlvmLibcIntegerToStringTest, UINT64) {
   EXPECT(type, 1234567890, "1234567890");
   EXPECT(type, 1234567890123456789, "1234567890123456789");
   EXPECT(type, UINT64_MAX, "18446744073709551615");
-  EXPECT(type, -1, "18446744073709551615");
+  EXPECT(type, static_cast<uint64_t>(-1), "18446744073709551615");
 }
 
 TEST(LlvmLibcIntegerToStringTest, INT64) {
@@ -181,7 +181,8 @@ TEST(LlvmLibcIntegerToStringTest, UINT64_Base_8) {
   EXPECT(type, 0, "0");
   EXPECT(type, 012345, "12345");
   EXPECT(type, 0123456701234567012345, "123456701234567012345");
-  EXPECT(type, 01777777777777777777777, "1777777777777777777777");
+  EXPECT(type, static_cast<int64_t>(01777777777777777777777),
+         "1777777777777777777777");
 }
 
 TEST(LlvmLibcIntegerToStringTest, UINT64_Base_16) {
diff --git a/libc/test/src/__support/math_extras_test.cpp b/libc/test/src/__support/math_extras_test.cpp
index 08c090017c1a1..f5d4dae371827 100644
--- a/libc/test/src/__support/math_extras_test.cpp
+++ b/libc/test/src/__support/math_extras_test.cpp
@@ -72,34 +72,41 @@ TEST(LlvmLibcBlockMathExtrasTest, mask_trailing_ones) {
 
 TYPED_TEST(LlvmLibcBitTest, FirstLeadingZero, UnsignedTypesNoBigInt) {
   EXPECT_EQ(first_leading_zero<T>(cpp::numeric_limits<T>::max()), 0);
-  for (int i = 0U; i != cpp::numeric_limits<T>::digits; ++i)
-    EXPECT_EQ(first_leading_zero<T>(~(T(1) << i)),
-              cpp::numeric_limits<T>::digits - i);
+  for (int i = 0U; i != cpp::numeric_limits<T>::digits; ++i) {
+    auto lhs = T(~(T(1) << size_t(i)));
+    EXPECT_EQ(first_leading_zero<T>(lhs), cpp::numeric_limits<T>::digits - i);
+  }
 }
 
 TYPED_TEST(LlvmLibcBitTest, FirstLeadingOne, UnsignedTypesNoBigInt) {
   EXPECT_EQ(first_leading_one<T>(static_cast<T>(0)), 0);
-  for (int i = 0U; i != cpp::numeric_limits<T>::digits; ++i)
-    EXPECT_EQ(first_leading_one<T>(T(1) << i),
-              cpp::numeric_limits<T>::digits - i);
+  for (int i = 0U; i != cpp::numeric_limits<T>::digits; ++i) {
+    auto lhs = T(T(1) << size_t(i));
+    EXPECT_EQ(first_leading_one<T>(lhs), cpp::numeric_limits<T>::digits - i);
+  }
 }
 
 TYPED_TEST(LlvmLibcBitTest, FirstTrailingZero, UnsignedTypesNoBigInt) {
   EXPECT_EQ(first_trailing_zero<T>(cpp::numeric_limits<T>::max()), 0);
-  for (int i = 0U; i != cpp::numeric_limits<T>::digits; ++i)
-    EXPECT_EQ(first_trailing_zero<T>(~(T(1) << i)), i + 1);
+  for (int i = 0U; i != cpp::numeric_limits<T>::digits; ++i) {
+    auto lhs = T(~(T(1) << size_t(i)));
+    EXPECT_EQ(first_trailing_zero<T>(lhs), i + 1);
+  }
 }
 
 TYPED_TEST(LlvmLibcBitTest, FirstTrailingOne, UnsignedTypesNoBigInt) {
-  EXPECT_EQ(first_trailing_one<T>(cpp::numeric_limits<T>::max()), 0);
-  for (int i = 0U; i != cpp::numeric_limits<T>::digits; ++i)
-    EXPECT_EQ(first_trailing_one<T>(T(1) << i), i + 1);
+  EXPECT_EQ(first_trailing_one<T>(static_cast<T>(0)), 0);
+  EXPECT_EQ(first_trailing_one<T>(cpp::numeric_limits<T>::max()), 1);
+  for (int i = 0U; i != cpp::numeric_limits<T>::digits; ++i) {
+    auto lhs = T(T(1) << size_t(i));
+    EXPECT_EQ(first_trailing_one<T>(lhs), i + 1);
+  }
 }
 
 TYPED_TEST(LlvmLibcBitTest, CountZeros, UnsignedTypesNoBigInt) {
   EXPECT_EQ(count_zeros(T(0)), cpp::numeric_limits<T>::digits);
   for (int i = 0; i != cpp::numeric_limits<T>::digits; ++i)
-    EXPECT_EQ(count_zeros<T>(cpp::numeric_limits<T>::max() >> i), i);
+    EXPECT_EQ(count_zeros<T>(cpp::numeric_limits<T>::max() >> size_t(i)), i);
 }
 
 using UnsignedTypes = testing::TypeList<
diff --git a/libc/test/src/__support/str_to_double_test.cpp b/libc/test/src/__support/str_to_double_test.cpp
index 03d21852dac2b..ccfa44f12d8ef 100644
--- a/libc/test/src/__support/str_to_double_test.cpp
+++ b/libc/test/src/__support/str_to_double_test.cpp
@@ -104,7 +104,7 @@ TEST(LlvmLibcStrToDblTest, SimpleDecimalConversionExtraTypes) {
       internal::simple_decimal_conversion<double>("123456789012345678900");
 
   double_output_mantissa = double_result.num.mantissa;
-  output_exp2 = double_result.num.exponent;
+  output_exp2 = static_cast<uint32_t>(double_result.num.exponent);
 
   EXPECT_EQ(double_output_mantissa, uint64_t(0x1AC53A7E04BCDA));
   EXPECT_EQ(output_exp2, uint32_t(1089));
diff --git a/libc/test/src/__support/str_to_float_test.cpp b/libc/test/src/__support/str_to_float_test.cpp
index fa4c642ec9cdb..66f7db742eb45 100644
--- a/libc/test/src/__support/str_to_float_test.cpp
+++ b/libc/test/src/__support/str_to_float_test.cpp
@@ -59,7 +59,7 @@ TEST(LlvmLibcStrToFltTest, SimpleDecimalConversionExtraTypes) {
   auto float_result =
       internal::simple_decimal_conversion<float>("123456789012345678900");
   float_output_mantissa = float_result.num.mantissa;
-  output_exp2 = float_result.num.exponent;
+  output_exp2 = static_cast<uint32_t>(float_result.num.exponent);
   EXPECT_EQ(float_output_mantissa, uint32_t(0xd629d4));
   EXPECT_EQ(output_exp2, uint32_t(193));
   EXPECT_EQ(float_result.error, 0);
diff --git a/libc/test/src/__support/str_to_fp_test.h b/libc/test/src/__support/str_to_fp_test.h
index db4e62a5cd0d1..c7bc57b845fe0 100644
--- a/libc/test/src/__support/str_to_fp_test.h
+++ b/libc/test/src/__support/str_to_fp_test.h
@@ -31,7 +31,7 @@ template <typename T> struct LlvmLibcStrToFloatTest : public testing::Test {
     ASSERT_TRUE(result.has_value());
 
     actual_output_mantissa = result->mantissa;
-    actual_output_exp2 = result->exponent;
+    actual_output_exp2 = static_cast<uint32_t>(result->exponent);
 
     EXPECT_EQ(actual_output_mantissa, expectedOutputMantissa);
     EXPECT_EQ(actual_output_exp2, expectedOutputExp2);
@@ -55,7 +55,7 @@ template <typename T> struct LlvmLibcStrToFloatTest : public testing::Test {
     ASSERT_TRUE(result.has_value());
 
     actual_output_mantissa = result->mantissa;
-    actual_output_exp2 = result->exponent;
+    actual_output_exp2 = static_cast<uint32_t>(result->exponent);
 
     EXPECT_EQ(actual_output_mantissa, expectedOutputMantissa);
     EXPECT_EQ(actual_output_exp2, expectedOutputExp2);
@@ -72,7 +72,7 @@ template <typename T> struct LlvmLibcStrToFloatTest : public testing::Test {
     auto result = internal::simple_decimal_conversion<T>(numStart);
 
     actual_output_mantissa = result.num.mantissa;
-    actual_output_exp2 = result.num.exponent;
+    actual_output_exp2 = static_cast<uint32_t>(result.num.exponent);
 
     EXPECT_EQ(actual_output_mantissa, expectedOutputMantissa);
     EXPECT_EQ(actual_output_exp2, expectedOutputExp2);
diff --git a/libc/test/src/locale/CMakeLists.txt b/libc/test/src/locale/CMakeLists.txt
index 3192004db26dd..5032ce9c324e2 100644
--- a/libc/test/src/locale/CMakeLists.txt
+++ b/libc/test/src/locale/CMakeLists.txt
@@ -7,7 +7,7 @@ add_libc_test(
   SRCS
     locale_test.cpp
   DEPENDS
-    libc.include.locale
+    libc.hdr.locale_macros
     libc.src.locale.newlocale
     libc.src.locale.uselocale
     libc.src.locale.freelocale
@@ -20,6 +20,6 @@ add_libc_test(
   SRCS
     localeconv_test.cpp
   DEPENDS
-    libc.include.locale
+    libc.hdr.locale_macros
     libc.src.locale.localeconv
 )
diff --git a/libc/test/src/locale/locale_test.cpp b/libc/test/src/locale/locale_test.cpp
index bc48bb851f4e4..d800e8fa5fcf9 100644
--- a/libc/test/src/locale/locale_test.cpp
+++ b/libc/test/src/locale/locale_test.cpp
@@ -6,14 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/locale_macros.h"
 #include "src/locale/freelocale.h"
 #include "src/locale/newlocale.h"
 #include "src/locale/uselocale.h"
-
 #include "test/UnitTest/Test.h"
 
-#include "include/llvm-libc-macros/locale-macros.h"
-
 TEST(LlvmLibcLocale, DefaultLocale) {
   locale_t new_locale = LIBC_NAMESPACE::newlocale(LC_ALL, "C", nullptr);
   EXPECT_NE(new_locale, static_cast<locale_t>(nullptr));
diff --git a/libc/test/src/locale/localeconv_test.cpp b/libc/test/src/locale/localeconv_test.cpp
index 79264276dec35..4a7888176d539 100644
--- a/libc/test/src/locale/localeconv_test.cpp
+++ b/libc/test/src/locale/localeconv_test.cpp
@@ -6,9 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "include/llvm-libc-macros/locale-macros.h"
+#include "hdr/locale_macros.h"
 #include "src/locale/localeconv.h"
-
 #include "test/UnitTest/Test.h"
 
 TEST(LlvmLibcLocale, DefaultLocale) {
diff --git a/libc/test/src/math/FModTest.h b/libc/test/src/math/FModTest.h
index 32c009ab88286..f761dba79d529 100644
--- a/libc/test/src/math/FModTest.h
+++ b/libc/test/src/math/FModTest.h
@@ -54,7 +54,7 @@ class FmodTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 
     // fmod (+inf, y) == aNaN plus invalid exception.
     TEST_SPECIAL(inf, 3.0, aNaN, true, FE_INVALID);
-    TEST_SPECIAL(inf, -1.1L, aNaN, true, FE_INVALID);
+    TEST_SPECIAL(inf, static_cast<float>(-1.1L), aNaN, true, FE_INVALID);
     TEST_SPECIAL(inf, 0.0, aNaN, true, FE_INVALID);
     TEST_SPECIAL(inf, neg_zero, aNaN, true, FE_INVALID);
     TEST_SPECIAL(inf, min_denormal, aNaN, true, FE_INVALID);
@@ -65,7 +65,7 @@ class FmodTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 
     // fmod (-inf, y) == aNaN plus invalid exception.
     TEST_SPECIAL(neg_inf, 3.0, aNaN, true, FE_INVALID);
-    TEST_SPECIAL(neg_inf, -1.1L, aNaN, true, FE_INVALID);
+    TEST_SPECIAL(neg_inf, static_cast<float>(-1.1L), aNaN, true, FE_INVALID);
     TEST_SPECIAL(neg_inf, 0.0, aNaN, true, FE_INVALID);
     TEST_SPECIAL(neg_inf, neg_zero, aNaN, true, FE_INVALID);
     TEST_SPECIAL(neg_inf, min_denormal, aNaN, true, FE_INVALID);
@@ -76,7 +76,7 @@ class FmodTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 
     // fmod (x, +0) == aNaN plus invalid exception.
     TEST_SPECIAL(3.0, 0.0, aNaN, true, FE_INVALID);
-    TEST_SPECIAL(-1.1L, 0.0, aNaN, true, FE_INVALID);
+    TEST_SPECIAL(static_cast<float>(-1.1L), 0.0, aNaN, true, FE_INVALID);
     TEST_SPECIAL(0.0, 0.0, aNaN, true, FE_INVALID);
     TEST_SPECIAL(neg_zero, 0.0, aNaN, true, FE_INVALID);
     TEST_SPECIAL(min_denormal, 0.0, aNaN, true, FE_INVALID);
@@ -85,7 +85,7 @@ class FmodTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 
     // fmod (x, -0) == aNaN plus invalid exception.
     TEST_SPECIAL(3.0, neg_zero, aNaN, true, FE_INVALID);
-    TEST_SPECIAL(-1.1L, neg_zero, aNaN, true, FE_INVALID);
+    TEST_SPECIAL(static_cast<float>(-1.1L), neg_zero, aNaN, true, FE_INVALID);
     TEST_SPECIAL(0.0, neg_zero, aNaN, true, FE_INVALID);
     TEST_SPECIAL(neg_zero, neg_zero, aNaN, true, FE_INVALID);
     TEST_SPECIAL(min_denormal, neg_zero, aNaN, true, FE_INVALID);
diff --git a/libc/test/src/stdbit/stdc_first_trailing_one_uc_test.cpp b/libc/test/src/stdbit/stdc_first_trailing_one_uc_test.cpp
index 5ca4cfca1b479..f507e9ac12c2a 100644
--- a/libc/test/src/stdbit/stdc_first_trailing_one_uc_test.cpp
+++ b/libc/test/src/stdbit/stdc_first_trailing_one_uc_test.cpp
@@ -11,7 +11,7 @@
 #include "test/UnitTest/Test.h"
 
 TEST(LlvmLibcStdcFirstTrailingOneUcTest, ALL) {
-  EXPECT_EQ(LIBC_NAMESPACE::stdc_first_trailing_one_uc(UCHAR_MAX), 0U);
+  EXPECT_EQ(LIBC_NAMESPACE::stdc_first_trailing_one_uc(UCHAR_MAX), 1U);
 }
 
 TEST(LlvmLibcStdcFirstTrailingOneUcTest, OneHot) {
diff --git a/libc/test/src/stdbit/stdc_first_trailing_one_ui_test.cpp b/libc/test/src/stdbit/stdc_first_trailing_one_ui_test.cpp
index 137c8a42e407d..3109d6888d534 100644
--- a/libc/test/src/stdbit/stdc_first_trailing_one_ui_test.cpp
+++ b/libc/test/src/stdbit/stdc_first_trailing_one_ui_test.cpp
@@ -11,7 +11,7 @@
 #include "test/UnitTest/Test.h"
 
 TEST(LlvmLibcStdcFirstTrailingOneUiTest, ALL) {
-  EXPECT_EQ(LIBC_NAMESPACE::stdc_first_trailing_one_ui(UINT_MAX), 0U);
+  EXPECT_EQ(LIBC_NAMESPACE::stdc_first_trailing_one_ui(UINT_MAX), 1U);
 }
 
 TEST(LlvmLibcStdcFirstTrailingOneUiTest, OneHot) {
diff --git a/libc/test/src/stdbit/stdc_first_trailing_one_ul_test.cpp b/libc/test/src/stdbit/stdc_first_trailing_one_ul_test.cpp
index 3fc1f3f16c60d..8d1f39acff8d3 100644
--- a/libc/test/src/stdbit/stdc_first_trailing_one_ul_test.cpp
+++ b/libc/test/src/stdbit/stdc_first_trailing_one_ul_test.cpp
@@ -11,7 +11,7 @@
 #include "test/UnitTest/Test.h"
 
 TEST(LlvmLibcStdcFirstTrailingOneUlTest, ALL) {
-  EXPECT_EQ(LIBC_NAMESPACE::stdc_first_trailing_one_ul(ULONG_MAX), 0U);
+  EXPECT_EQ(LIBC_NAMESPACE::stdc_first_trailing_one_ul(ULONG_MAX), 1U);
 }
 
 TEST(LlvmLibcStdcFirstTrailingOneUlTest, OneHot) {
diff --git a/libc/test/src/stdbit/stdc_first_trailing_one_ull_test.cpp b/libc/test/src/stdbit/stdc_first_trailing_one_ull_test.cpp
index 5719e09a5120a..12ef0537e3281 100644
--- a/libc/test/src/stdbit/stdc_first_trailing_one_ull_test.cpp
+++ b/libc/test/src/stdbit/stdc_first_trailing_one_ull_test.cpp
@@ -11,7 +11,7 @@
 #include "test/UnitTest/Test.h"
 
 TEST(LlvmLibcStdcFirstTrailingOneUllTest, ALL) {
-  EXPECT_EQ(LIBC_NAMESPACE::stdc_first_trailing_one_ull(ULLONG_MAX), 0U);
+  EXPECT_EQ(LIBC_NAMESPACE::stdc_first_trailing_one_ull(ULLONG_MAX), 1U);
 }
 
 TEST(LlvmLibcStdcFirstTrailingOneUllTest, OneHot) {
diff --git a/libc/test/src/stdbit/stdc_first_trailing_one_us_test.cpp b/libc/test/src/stdbit/stdc_first_trailing_one_us_test.cpp
index 46c69acfb7f07..6a506e6cc9632 100644
--- a/libc/test/src/stdbit/stdc_first_trailing_one_us_test.cpp
+++ b/libc/test/src/stdbit/stdc_first_trailing_one_us_test.cpp
@@ -11,7 +11,7 @@
 #include "test/UnitTest/Test.h"
 
 TEST(LlvmLibcStdcFirstTrailingOneUsTest, ALL) {
-  EXPECT_EQ(LIBC_NAMESPACE::stdc_first_trailing_one_us(USHRT_MAX), 0U);
+  EXPECT_EQ(LIBC_NAMESPACE::stdc_first_trailing_one_us(USHRT_MAX), 1U);
 }
 
 TEST(LlvmLibcStdcFirstTrailingOneUsTest, OneHot) {
diff --git a/libc/test/src/stdfix/CMakeLists.txt b/libc/test/src/stdfix/CMakeLists.txt
index 8dc0eb854e65c..a3dc25762f549 100644
--- a/libc/test/src/stdfix/CMakeLists.txt
+++ b/libc/test/src/stdfix/CMakeLists.txt
@@ -4,6 +4,7 @@ endif()
 
 add_custom_target(libc-stdfix-tests)
 
+
 foreach(suffix IN ITEMS hr r lr hk k lk)
   add_libc_test(
     abs${suffix}_test
@@ -176,3 +177,25 @@ add_libc_test(
     libc.src.__support.fixed_point.fx_rep
     libc.src.__support.FPUtil.basic_operations
 )
+
+set(macros_depends)
+
+foreach(s IN ITEMS r hr lr k hk lk)
+  list(APPEND macros_depends
+    libc.src.stdfix.abs${s}
+    libc.src.stdfix.countls${s}
+    libc.src.stdfix.countlsu${s}
+    libc.src.stdfix.round${s}
+    libc.src.stdfix.roundu${s}
+  )
+endforeach()
+
+add_libc_test(
+  macros_test
+  SUITE libc-stdfix-tests
+  SRCS macros_test.cpp
+  DEPENDS
+    libc.include.stdfix
+    ${macros_depends}
+)
+
diff --git a/libc/test/src/stdfix/macros_test.cpp b/libc/test/src/stdfix/macros_test.cpp
new file mode 100644
index 0000000000000..b52abf3f07999
--- /dev/null
+++ b/libc/test/src/stdfix/macros_test.cpp
@@ -0,0 +1,113 @@
+//===-- Unittests for absfx -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AbsTest.h"
+#include "src/stdfix/abshk.h"
+#include "src/stdfix/abshr.h"
+#include "src/stdfix/absk.h"
+#include "src/stdfix/abslk.h"
+#include "src/stdfix/abslr.h"
+#include "src/stdfix/absr.h"
+
+#include "src/stdfix/countlshk.h"
+#include "src/stdfix/countlshr.h"
+#include "src/stdfix/countlsk.h"
+#include "src/stdfix/countlslk.h"
+#include "src/stdfix/countlslr.h"
+#include "src/stdfix/countlsr.h"
+#include "src/stdfix/countlsuhk.h"
+#include "src/stdfix/countlsuhr.h"
+#include "src/stdfix/countlsuk.h"
+#include "src/stdfix/countlsulk.h"
+#include "src/stdfix/countlsulr.h"
+#include "src/stdfix/countlsur.h"
+
+#include "src/stdfix/roundhk.h"
+#include "src/stdfix/roundhr.h"
+#include "src/stdfix/roundk.h"
+#include "src/stdfix/roundlk.h"
+#include "src/stdfix/roundlr.h"
+#include "src/stdfix/roundr.h"
+#include "src/stdfix/rounduhk.h"
+#include "src/stdfix/rounduhr.h"
+#include "src/stdfix/rounduk.h"
+#include "src/stdfix/roundulk.h"
+#include "src/stdfix/roundulr.h"
+#include "src/stdfix/roundur.h"
+
+using LIBC_NAMESPACE::abshk;
+using LIBC_NAMESPACE::abshr;
+using LIBC_NAMESPACE::absk;
+using LIBC_NAMESPACE::abslk;
+using LIBC_NAMESPACE::abslr;
+using LIBC_NAMESPACE::absr;
+using LIBC_NAMESPACE::countlshk;
+using LIBC_NAMESPACE::countlshr;
+using LIBC_NAMESPACE::countlsk;
+using LIBC_NAMESPACE::countlslk;
+using LIBC_NAMESPACE::countlslr;
+using LIBC_NAMESPACE::countlsr;
+using LIBC_NAMESPACE::countlsuhk;
+using LIBC_NAMESPACE::countlsuhr;
+using LIBC_NAMESPACE::countlsuk;
+using LIBC_NAMESPACE::countlsulk;
+using LIBC_NAMESPACE::countlsulr;
+using LIBC_NAMESPACE::countlsur;
+using LIBC_NAMESPACE::roundhk;
+using LIBC_NAMESPACE::roundhr;
+using LIBC_NAMESPACE::roundk;
+using LIBC_NAMESPACE::roundlk;
+using LIBC_NAMESPACE::roundlr;
+using LIBC_NAMESPACE::roundr;
+using LIBC_NAMESPACE::rounduhk;
+using LIBC_NAMESPACE::rounduhr;
+using LIBC_NAMESPACE::rounduk;
+using LIBC_NAMESPACE::roundulk;
+using LIBC_NAMESPACE::roundulr;
+using LIBC_NAMESPACE::roundur;
+
+TEST(LlvmLibcAbsfxTest, Basic) {
+  ASSERT_EQ(absfx(-0.5r), absr(-0.5r));
+  ASSERT_EQ(absfx(-0.5hr), abshr(-0.5hr));
+  ASSERT_EQ(absfx(-0.5lr), abslr(-0.5lr));
+  ASSERT_EQ(absfx(-0.5k), absk(-0.5k));
+  ASSERT_EQ(absfx(-0.5hk), abshk(-0.5hk));
+  ASSERT_EQ(absfx(-0.5lk), abslk(-0.5lk));
+}
+
+TEST(LlvmLibcRoundfxTest, Basic) {
+  ASSERT_EQ(roundfx(0.75r, 0), roundr(0.75r, 0));
+  ASSERT_EQ(roundfx(0.75hr, 0), roundhr(0.75hr, 0));
+  ASSERT_EQ(roundfx(0.75lr, 0), roundlr(0.75lr, 0));
+  ASSERT_EQ(roundfx(0.75k, 0), roundk(0.75k, 0));
+  ASSERT_EQ(roundfx(0.75hk, 0), roundhk(0.75hk, 0));
+  ASSERT_EQ(roundfx(0.75lk, 0), roundlk(0.75lk, 0));
+
+  ASSERT_EQ(roundfx(0.75ur, 0), roundur(0.75ur, 0));
+  ASSERT_EQ(roundfx(0.75uhr, 0), rounduhr(0.75uhr, 0));
+  ASSERT_EQ(roundfx(0.75ulr, 0), roundulr(0.75ulr, 0));
+  ASSERT_EQ(roundfx(0.75uk, 0), rounduk(0.75uk, 0));
+  ASSERT_EQ(roundfx(0.75uhk, 0), rounduhk(0.75uhk, 0));
+  ASSERT_EQ(roundfx(0.75ulk, 0), roundulk(0.75ulk, 0));
+}
+
+TEST(LlvmLibcCountlsfxTest, Basic) {
+  ASSERT_EQ(countlsfx(0.5r), countlsr(0.5r));
+  ASSERT_EQ(countlsfx(0.5hr), countlshr(0.5hr));
+  ASSERT_EQ(countlsfx(0.5lr), countlslr(0.5lr));
+  ASSERT_EQ(countlsfx(0.5k), countlsk(0.5k));
+  ASSERT_EQ(countlsfx(0.5hk), countlshk(0.5hk));
+  ASSERT_EQ(countlsfx(0.5lk), countlslk(0.5lk));
+
+  ASSERT_EQ(countlsfx(0.5ur), countlsur(0.5ur));
+  ASSERT_EQ(countlsfx(0.5uhr), countlsuhr(0.5uhr));
+  ASSERT_EQ(countlsfx(0.5ulr), countlsulr(0.5ulr));
+  ASSERT_EQ(countlsfx(0.5uk), countlsuk(0.5uk));
+  ASSERT_EQ(countlsfx(0.5uhk), countlsuhk(0.5uhk));
+  ASSERT_EQ(countlsfx(0.5ulk), countlsulk(0.5ulk));
+}
diff --git a/libc/test/src/stdio/printf_core/parser_test.cpp b/libc/test/src/stdio/printf_core/parser_test.cpp
index c277b304faea9..9d192828860f7 100644
--- a/libc/test/src/stdio/printf_core/parser_test.cpp
+++ b/libc/test/src/stdio/printf_core/parser_test.cpp
@@ -97,7 +97,8 @@ TEST(LlvmLibcPrintfParserTest, EvalOneArg) {
   expected.has_conv = true;
 
   expected.raw_string = {str, 2};
-  expected.conv_val_raw = arg1;
+  expected.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg1);
   expected.conv_name = 'd';
 
   ASSERT_PFORMAT_EQ(expected, format_arr[0]);
@@ -132,7 +133,8 @@ TEST(LlvmLibcPrintfParserTest, EvalOneArgWithFlags) {
       LIBC_NAMESPACE::printf_core::FormatFlags::LEADING_ZEROES |
       LIBC_NAMESPACE::printf_core::FormatFlags::SPACE_PREFIX |
       LIBC_NAMESPACE::printf_core::FormatFlags::ALTERNATE_FORM);
-  expected.conv_val_raw = arg1;
+  expected.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg1);
   expected.conv_name = 'd';
 
   ASSERT_PFORMAT_EQ(expected, format_arr[0]);
@@ -149,7 +151,8 @@ TEST(LlvmLibcPrintfParserTest, EvalOneArgWithWidth) {
 
   expected.raw_string = {str, 4};
   expected.min_width = 12;
-  expected.conv_val_raw = arg1;
+  expected.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg1);
   expected.conv_name = 'd';
 
   ASSERT_PFORMAT_EQ(expected, format_arr[0]);
@@ -166,7 +169,8 @@ TEST(LlvmLibcPrintfParserTest, EvalOneArgWithPrecision) {
 
   expected.raw_string = {str, 5};
   expected.precision = 34;
-  expected.conv_val_raw = arg1;
+  expected.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg1);
   expected.conv_name = 'd';
 
   ASSERT_PFORMAT_EQ(expected, format_arr[0]);
@@ -183,7 +187,8 @@ TEST(LlvmLibcPrintfParserTest, EvalOneArgWithTrivialPrecision) {
 
   expected.raw_string = {str, 3};
   expected.precision = 0;
-  expected.conv_val_raw = arg1;
+  expected.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg1);
   expected.conv_name = 'd';
 
   ASSERT_PFORMAT_EQ(expected, format_arr[0]);
@@ -200,7 +205,8 @@ TEST(LlvmLibcPrintfParserTest, EvalOneArgWithShortLengthModifier) {
 
   expected.raw_string = {str, 3};
   expected.length_modifier = LIBC_NAMESPACE::printf_core::LengthModifier::h;
-  expected.conv_val_raw = arg1;
+  expected.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg1);
   expected.conv_name = 'd';
 
   ASSERT_PFORMAT_EQ(expected, format_arr[0]);
@@ -217,7 +223,8 @@ TEST(LlvmLibcPrintfParserTest, EvalOneArgWithLongLengthModifier) {
 
   expected.raw_string = {str, 4};
   expected.length_modifier = LIBC_NAMESPACE::printf_core::LengthModifier::ll;
-  expected.conv_val_raw = arg1;
+  expected.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg1);
   expected.conv_name = 'd';
 
   ASSERT_PFORMAT_EQ(expected, format_arr[0]);
@@ -235,7 +242,8 @@ TEST(LlvmLibcPrintfParserTest, EvalOneArgWithBitWidthLengthModifier) {
   expected.raw_string = {str, 5};
   expected.length_modifier = LIBC_NAMESPACE::printf_core::LengthModifier::w;
   expected.bit_width = 32;
-  expected.conv_val_raw = arg1;
+  expected.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg1);
   expected.conv_name = 'd';
 
   ASSERT_PFORMAT_EQ(expected, format_arr[0]);
@@ -253,7 +261,8 @@ TEST(LlvmLibcPrintfParserTest, EvalOneArgWithFastBitWidthLengthModifier) {
   expected.raw_string = {str, 6};
   expected.length_modifier = LIBC_NAMESPACE::printf_core::LengthModifier::wf;
   expected.bit_width = 32;
-  expected.conv_val_raw = arg1;
+  expected.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg1);
   expected.conv_name = 'd';
 
   ASSERT_PFORMAT_EQ(expected, format_arr[0]);
@@ -276,7 +285,8 @@ TEST(LlvmLibcPrintfParserTest, EvalOneArgWithAllOptions) {
   expected.min_width = 56;
   expected.precision = 78;
   expected.length_modifier = LIBC_NAMESPACE::printf_core::LengthModifier::j;
-  expected.conv_val_raw = arg1;
+  expected.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg1);
   expected.conv_name = 'd';
 
   ASSERT_PFORMAT_EQ(expected, format_arr[0]);
@@ -294,7 +304,8 @@ TEST(LlvmLibcPrintfParserTest, EvalThreeArgs) {
   expected0.has_conv = true;
 
   expected0.raw_string = {str, 2};
-  expected0.conv_val_raw = arg1;
+  expected0.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg1);
   expected0.conv_name = 'd';
 
   ASSERT_PFORMAT_EQ(expected0, format_arr[0]);
@@ -329,7 +340,8 @@ TEST(LlvmLibcPrintfParserTest, EvalOneArgWithOverflowingWidthAndPrecision) {
   expected.flags = LIBC_NAMESPACE::printf_core::FormatFlags::LEFT_JUSTIFIED;
   expected.min_width = INT_MAX;
   expected.precision = INT_MAX;
-  expected.conv_val_raw = arg1;
+  expected.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg1);
   expected.conv_name = 'd';
 
   ASSERT_PFORMAT_EQ(expected, format_arr[0]);
@@ -351,7 +363,8 @@ TEST(LlvmLibcPrintfParserTest,
   expected.flags = LIBC_NAMESPACE::printf_core::FormatFlags::LEFT_JUSTIFIED;
   expected.min_width = INT_MAX;
   expected.precision = arg2;
-  expected.conv_val_raw = arg3;
+  expected.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg3);
   expected.conv_name = 'd';
 
   ASSERT_PFORMAT_EQ(expected, format_arr[0]);
@@ -369,7 +382,8 @@ TEST(LlvmLibcPrintfParserTest, IndexModeOneArg) {
   expected.has_conv = true;
 
   expected.raw_string = {str, 4};
-  expected.conv_val_raw = arg1;
+  expected.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg1);
   expected.conv_name = 'd';
 
   ASSERT_PFORMAT_EQ(expected, format_arr[0]);
@@ -387,7 +401,8 @@ TEST(LlvmLibcPrintfParserTest, IndexModeThreeArgsSequential) {
   expected0.has_conv = true;
 
   expected0.raw_string = {str, 4};
-  expected0.conv_val_raw = arg1;
+  expected0.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg1);
   expected0.conv_name = 'd';
 
   ASSERT_PFORMAT_EQ(expected0, format_arr[0]);
@@ -421,7 +436,8 @@ TEST(LlvmLibcPrintfParserTest, IndexModeThreeArgsReverse) {
   expected0.has_conv = true;
 
   expected0.raw_string = {str, 4};
-  expected0.conv_val_raw = arg1;
+  expected0.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg1);
   expected0.conv_name = 'd';
 
   ASSERT_PFORMAT_EQ(expected0, format_arr[0]);
@@ -485,7 +501,8 @@ TEST(LlvmLibcPrintfParserTest, IndexModeComplexParsing) {
 
   expected1.raw_string = {str + 12, 6};
   expected1.length_modifier = LIBC_NAMESPACE::printf_core::LengthModifier::ll;
-  expected1.conv_val_raw = arg3;
+  expected1.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg3);
   expected1.conv_name = 'u';
 
   EXPECT_PFORMAT_EQ(expected1, format_arr[1]);
@@ -546,7 +563,8 @@ TEST(LlvmLibcPrintfParserTest, IndexModeComplexParsing) {
   expected9.raw_string = {str + 41, 7};
   expected9.min_width = 1;
   expected9.precision = 1;
-  expected9.conv_val_raw = arg1;
+  expected9.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg1);
   expected9.conv_name = 'c';
 
   EXPECT_PFORMAT_EQ(expected9, format_arr[9]);
@@ -566,14 +584,16 @@ TEST(LlvmLibcPrintfParserTest, IndexModeGapCheck) {
 
   expected0.has_conv = true;
   expected0.raw_string = {str, 4};
-  expected0.conv_val_raw = arg1;
+  expected0.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg1);
   expected0.conv_name = 'd';
 
   EXPECT_PFORMAT_EQ(expected0, format_arr[0]);
 
   expected1.has_conv = true;
   expected1.raw_string = {str + 4, 4};
-  expected1.conv_val_raw = arg2;
+  expected1.conv_val_raw =
+      static_cast<LIBC_NAMESPACE::fputil::FPBits<double>::StorageType>(arg2);
   expected1.conv_name = 'd';
 
   EXPECT_PFORMAT_EQ(expected1, format_arr[1]);
diff --git a/libc/test/src/stdlib/SortingTest.h b/libc/test/src/stdlib/SortingTest.h
index 681a420ea7274..d845280aaa27e 100644
--- a/libc/test/src/stdlib/SortingTest.h
+++ b/libc/test/src/stdlib/SortingTest.h
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/macros/config.h"
-#include "src/stdlib/qsort.h"
 #include "test/UnitTest/Test.h"
 
 class SortingTest : public LIBC_NAMESPACE::testing::Test {
diff --git a/libc/test/src/string/memmove_test.cpp b/libc/test/src/string/memmove_test.cpp
index e280b5d25f6f4..1e225e58ca1eb 100644
--- a/libc/test/src/string/memmove_test.cpp
+++ b/libc/test/src/string/memmove_test.cpp
@@ -93,7 +93,8 @@ TEST(LlvmLibcMemmoveTest, SizeSweep) {
   Randomize(Buffer);
   for (int Size = 0; Size < kMaxSize; ++Size)
     for (int Overlap = -1; Overlap < Size;) {
-      ASSERT_TRUE(CheckMemmove<Adaptor>(Buffer, Size, Overlap));
+      ASSERT_TRUE(
+          CheckMemmove<Adaptor>(Buffer, static_cast<size_t>(Size), Overlap));
       // Prevent quadratic behavior by skipping offset above kDenseOverlap.
       if (Overlap > kDenseOverlap)
         Overlap *= 2;
diff --git a/libc/test/src/string/memory_utils/memory_check_utils.h b/libc/test/src/string/memory_utils/memory_check_utils.h
index 3ddd15b181c8e..db9cfda672b21 100644
--- a/libc/test/src/string/memory_utils/memory_check_utils.h
+++ b/libc/test/src/string/memory_utils/memory_check_utils.h
@@ -157,7 +157,7 @@ inline uint16_t Checksum(cpp::span<char> dst) {
   uint16_t sum1 = 0;
   uint16_t sum2 = 0;
   for (char c : dst) {
-    sum1 = (sum1 + c) % 255U;
+    sum1 = (sum1 + static_cast<uint16_t>(c)) % 255U;
     sum2 = (sum2 + sum1) % 255U;
   }
   return static_cast<uint16_t>((sum2 << 8) | sum1);
@@ -185,8 +185,10 @@ template <auto FnImpl>
 inline bool CheckMemmove(cpp::span<char> buffer, size_t size, int overlap) {
   LIBC_ASSERT(buffer.size() > (2 * size + 1));
   const size_t half_size = buffer.size() / 2;
-  LIBC_ASSERT((size_t)(overlap >= 0 ? overlap : -overlap) < half_size);
-  cpp::span<char> head = buffer.first(half_size + overlap).last(size);
+  LIBC_ASSERT(static_cast<size_t>(overlap >= 0 ? overlap : -overlap) <
+              half_size);
+  cpp::span<char> head =
+      buffer.first(half_size + static_cast<size_t>(overlap)).last(size);
   cpp::span<char> tail = buffer.last(half_size).first(size);
   LIBC_ASSERT(head.size() == size);
   LIBC_ASSERT(tail.size() == size);
diff --git a/libc/test/src/string/memory_utils/op_tests.cpp b/libc/test/src/string/memory_utils/op_tests.cpp
index c6197d1afa266..2057ab3eaaded 100644
--- a/libc/test/src/string/memory_utils/op_tests.cpp
+++ b/libc/test/src/string/memory_utils/op_tests.cpp
@@ -174,7 +174,7 @@ TYPED_TEST(LlvmLibcOpTest, Memset, MemsetImplementations) {
     static constexpr auto HeadTailImpl = SetAdaptor<Impl::head_tail>;
     Buffer DstBuffer(2 * kSize);
     for (size_t size = kSize; size < 2 * kSize; ++size) {
-      const char value = size % 10;
+      const uint8_t value = size % 10;
       auto dst = DstBuffer.span().subspan(0, size);
       ASSERT_TRUE(CheckMemset<HeadTailImpl>(dst, value, size));
     }
@@ -185,7 +185,7 @@ TYPED_TEST(LlvmLibcOpTest, Memset, MemsetImplementations) {
       static constexpr auto LoopImpl = SetAdaptor<Impl::loop_and_tail>;
       Buffer DstBuffer(3 * kSize);
       for (size_t size = kSize; size < 3 * kSize; ++size) {
-        const char value = size % 10;
+        const uint8_t value = size % 10;
         auto dst = DstBuffer.span().subspan(0, size);
         ASSERT_TRUE((CheckMemset<LoopImpl>(dst, value, size)));
       }
diff --git a/libc/test/src/string/memory_utils/utils_test.cpp b/libc/test/src/string/memory_utils/utils_test.cpp
index 4c1accd551685..4dff0684b9111 100644
--- a/libc/test/src/string/memory_utils/utils_test.cpp
+++ b/libc/test/src/string/memory_utils/utils_test.cpp
@@ -47,11 +47,11 @@ TEST(LlvmLibcUtilsTest, DistanceToAlignDown) {
 TEST(LlvmLibcUtilsTest, Adjust2) {
   char a, b;
   const size_t base_size = 10;
-  for (ptrdiff_t I = -2; I < 2; ++I) {
+  for (uintptr_t I = 0; I < 4; ++I) {
     auto *p1 = &a;
     auto *p2 = &b;
     size_t size = base_size;
-    adjust(I, p1, p2, size);
+    adjust(static_cast<ptrdiff_t>(I), p1, p2, size);
     EXPECT_EQ(intptr_t(p1), intptr_t(&a + I));
     EXPECT_EQ(intptr_t(p2), intptr_t(&b + I));
     EXPECT_EQ(size, base_size - I);
diff --git a/libc/test/src/string/memset_test.cpp b/libc/test/src/string/memset_test.cpp
index 774a321ddfce6..46d6ce7478772 100644
--- a/libc/test/src/string/memset_test.cpp
+++ b/libc/test/src/string/memset_test.cpp
@@ -27,7 +27,7 @@ TEST(LlvmLibcMemsetTest, SizeSweep) {
   static constexpr size_t kMaxSize = 400;
   Buffer DstBuffer(kMaxSize);
   for (size_t size = 0; size < kMaxSize; ++size) {
-    const char value = size % 10;
+    const uint8_t value = size % 10;
     auto dst = DstBuffer.span().subspan(0, size);
     ASSERT_TRUE((CheckMemset<Adaptor>(dst, value, size)));
   }
diff --git a/libc/test/src/strings/CMakeLists.txt b/libc/test/src/strings/CMakeLists.txt
index 10f96b8531f68..5f70dc024f6ce 100644
--- a/libc/test/src/strings/CMakeLists.txt
+++ b/libc/test/src/strings/CMakeLists.txt
@@ -12,6 +12,36 @@ add_libc_test(
     LibcMemoryHelpers
 )
 
+add_libc_test(
+  ffs_test
+  SUITE
+    libc-strings-tests
+  SRCS
+    ffs_test.cpp
+  DEPENDS
+    libc.src.strings.ffs
+)
+
+add_libc_test(
+  ffsl_test
+  SUITE
+    libc-strings-tests
+  SRCS
+    ffsl_test.cpp
+  DEPENDS
+    libc.src.strings.ffsl
+)
+
+add_libc_test(
+  ffsll_test
+  SUITE
+    libc-strings-tests
+  SRCS
+    ffsll_test.cpp
+  DEPENDS
+    libc.src.strings.ffsll
+)
+
 add_libc_test(
   index_test
   SUITE
@@ -44,6 +74,19 @@ add_libc_test(
     libc.src.strings.strcasecmp
 )
 
+add_libc_test(
+  strcasecmp_l_test
+  SUITE
+    libc-strings-tests
+  SRCS
+    strcasecmp_l_test.cpp
+  DEPENDS
+    libc.hdr.locale_macros
+    libc.src.locale.freelocale
+    libc.src.locale.newlocale
+    libc.src.strings.strcasecmp_l
+)
+
 add_libc_test(
   strncasecmp_test
   SUITE
@@ -54,5 +97,18 @@ add_libc_test(
     libc.src.strings.strncasecmp
 )
 
+add_libc_test(
+  strncasecmp_l_test
+  SUITE
+    libc-strings-tests
+  SRCS
+    strncasecmp_l_test.cpp
+  DEPENDS
+    libc.hdr.locale_macros
+    libc.src.locale.freelocale
+    libc.src.locale.newlocale
+    libc.src.strings.strncasecmp_l
+)
+
 add_libc_multi_impl_test(bcmp libc-strings-tests SRCS bcmp_test.cpp)
 add_libc_multi_impl_test(bzero libc-strings-tests SRCS bzero_test.cpp)
diff --git a/libc/test/src/strings/bcopy_test.cpp b/libc/test/src/strings/bcopy_test.cpp
index f6bb859b4ae9f..6daa634cab888 100644
--- a/libc/test/src/strings/bcopy_test.cpp
+++ b/libc/test/src/strings/bcopy_test.cpp
@@ -87,7 +87,8 @@ TEST(LlvmLibcBcopyTest, SizeSweep) {
   Randomize(Buffer);
   for (int Size = 0; Size < kMaxSize; ++Size)
     for (int Overlap = -1; Overlap < Size;) {
-      ASSERT_TRUE(CheckMemmove<Adaptor>(Buffer, Size, Overlap));
+      ASSERT_TRUE(
+          CheckMemmove<Adaptor>(Buffer, static_cast<size_t>(Size), Overlap));
       // Prevent quadratic behavior by skipping offset above kDenseOverlap.
       if (Overlap > kDenseOverlap)
         Overlap *= 2;
diff --git a/libc/test/src/strings/ffs_test.cpp b/libc/test/src/strings/ffs_test.cpp
new file mode 100644
index 0000000000000..17cd412b2df05
--- /dev/null
+++ b/libc/test/src/strings/ffs_test.cpp
@@ -0,0 +1,32 @@
+//===-- Unittests for ffs -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/strings/ffs.h"
+
+#include "src/__support/macros/config.h"
+#include "test/UnitTest/Test.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+TEST(LlvmLibcFfsTest, SimpleFfs) {
+  ASSERT_EQ(ffs(0x00000000), 0);
+  ASSERT_EQ(ffs(0x00000001), 1);
+  ASSERT_EQ(ffs(0x00000020), 6);
+  ASSERT_EQ(ffs(0x00000400), 11);
+  ASSERT_EQ(ffs(0x00008000), 16);
+  ASSERT_EQ(ffs(0x00010000), 17);
+  ASSERT_EQ(ffs(0x00200000), 22);
+  ASSERT_EQ(ffs(0x04000000), 27);
+  ASSERT_EQ(ffs(0x80000000), 32);
+  ASSERT_EQ(ffs(0xfbe71), 1);
+  ASSERT_EQ(ffs(0xfbe70), 5);
+  ASSERT_EQ(ffs(0x10), 5);
+  ASSERT_EQ(ffs(0x100), 9);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/strings/ffsl_test.cpp b/libc/test/src/strings/ffsl_test.cpp
new file mode 100644
index 0000000000000..cf5c32f2fec6b
--- /dev/null
+++ b/libc/test/src/strings/ffsl_test.cpp
@@ -0,0 +1,42 @@
+//===-- Unittests for ffsl ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/strings/ffsl.h"
+
+#include "src/__support/macros/config.h"
+#include "test/UnitTest/Test.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+TEST(LlvmLibcFfslTest, SimpleFfsl) {
+  ASSERT_EQ(ffsl(0x00000000L), 0);
+  ASSERT_EQ(ffsl(0x00000001L), 1);
+  ASSERT_EQ(ffsl(0x00000020L), 6);
+  ASSERT_EQ(ffsl(0x00000400L), 11);
+  ASSERT_EQ(ffsl(0x00008000L), 16);
+  ASSERT_EQ(ffsl(0x00010000L), 17);
+  ASSERT_EQ(ffsl(0x00200000L), 22);
+  ASSERT_EQ(ffsl(0x04000000L), 27);
+  ASSERT_EQ(ffsl(0x80000000L), 32);
+#ifdef __LP64__
+  ASSERT_EQ(ffsl(0x0000000100000000L), 33);
+  ASSERT_EQ(ffsl(0x0000002000000000L), 38);
+  ASSERT_EQ(ffsl(0x0000040000000000L), 43);
+  ASSERT_EQ(ffsl(0x0000800000000000L), 48);
+  ASSERT_EQ(ffsl(0x0001000000000000L), 49);
+  ASSERT_EQ(ffsl(0x0020000000000000L), 54);
+  ASSERT_EQ(ffsl(0x0400000000000000L), 59);
+  ASSERT_EQ(ffsl(0x8000000000000000L), 64);
+#endif
+  ASSERT_EQ(ffsl(0xfbe71L), 1);
+  ASSERT_EQ(ffsl(0xfbe70L), 5);
+  ASSERT_EQ(ffsl(0x10L), 5);
+  ASSERT_EQ(ffsl(0x100L), 9);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/strings/ffsll_test.cpp b/libc/test/src/strings/ffsll_test.cpp
new file mode 100644
index 0000000000000..bdd7a2ed30f3c
--- /dev/null
+++ b/libc/test/src/strings/ffsll_test.cpp
@@ -0,0 +1,40 @@
+//===-- Unittests for ffsll -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/strings/ffsll.h"
+
+#include "src/__support/macros/config.h"
+#include "test/UnitTest/Test.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+TEST(LlvmLibcFfsllTest, SimpleFfsll) {
+  ASSERT_EQ(ffsll(0x0000000000000000LL), 0);
+  ASSERT_EQ(ffsll(0x0000000000000001LL), 1);
+  ASSERT_EQ(ffsll(0x0000000000000020LL), 6);
+  ASSERT_EQ(ffsll(0x0000000000000400LL), 11);
+  ASSERT_EQ(ffsll(0x0000000000008000LL), 16);
+  ASSERT_EQ(ffsll(0x0000000000010000LL), 17);
+  ASSERT_EQ(ffsll(0x0000000000200000LL), 22);
+  ASSERT_EQ(ffsll(0x0000000004000000LL), 27);
+  ASSERT_EQ(ffsll(0x0000000080000000LL), 32);
+  ASSERT_EQ(ffsll(0x0000000100000000LL), 33);
+  ASSERT_EQ(ffsll(0x0000002000000000LL), 38);
+  ASSERT_EQ(ffsll(0x0000040000000000LL), 43);
+  ASSERT_EQ(ffsll(0x0000800000000000LL), 48);
+  ASSERT_EQ(ffsll(0x0001000000000000LL), 49);
+  ASSERT_EQ(ffsll(0x0020000000000000LL), 54);
+  ASSERT_EQ(ffsll(0x0400000000000000LL), 59);
+  ASSERT_EQ(ffsll(0x8000000000000000LL), 64);
+  ASSERT_EQ(ffsll(0xfbe71LL), 1);
+  ASSERT_EQ(ffsll(0xfbe70LL), 5);
+  ASSERT_EQ(ffsll(0x10LL), 5);
+  ASSERT_EQ(ffsll(0x100LL), 9);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/strings/strcasecmp_l_test.cpp b/libc/test/src/strings/strcasecmp_l_test.cpp
new file mode 100644
index 0000000000000..33a47b6547d68
--- /dev/null
+++ b/libc/test/src/strings/strcasecmp_l_test.cpp
@@ -0,0 +1,21 @@
+//===-- Unittests for strcasecmp_l ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/locale_macros.h"
+#include "src/locale/freelocale.h"
+#include "src/locale/newlocale.h"
+#include "src/strings/strcasecmp_l.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcStrCaseCmpLTest, Case) {
+  locale_t locale = LIBC_NAMESPACE::newlocale(LC_ALL, "C", nullptr);
+  ASSERT_EQ(LIBC_NAMESPACE::strcasecmp_l("hello", "HELLO", locale), 0);
+  ASSERT_LT(LIBC_NAMESPACE::strcasecmp_l("hello1", "hello2", locale), 0);
+  ASSERT_GT(LIBC_NAMESPACE::strcasecmp_l("hello2", "hello1", locale), 0);
+  LIBC_NAMESPACE::freelocale(locale);
+}
diff --git a/libc/test/src/strings/strncasecmp_l_test.cpp b/libc/test/src/strings/strncasecmp_l_test.cpp
new file mode 100644
index 0000000000000..271f31808c7c3
--- /dev/null
+++ b/libc/test/src/strings/strncasecmp_l_test.cpp
@@ -0,0 +1,22 @@
+//===-- Unittests for strncasecmp_l ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/locale_macros.h"
+#include "src/locale/freelocale.h"
+#include "src/locale/newlocale.h"
+#include "src/strings/strncasecmp_l.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcStrNCaseCmpLTest, Case) {
+  locale_t locale = LIBC_NAMESPACE::newlocale(LC_ALL, "C", nullptr);
+  ASSERT_EQ(LIBC_NAMESPACE::strncasecmp_l("hello", "HELLO", 3, locale), 0);
+  ASSERT_EQ(LIBC_NAMESPACE::strncasecmp_l("abcXX", "ABCYY", 3, locale), 0);
+  ASSERT_LT(LIBC_NAMESPACE::strncasecmp_l("hello1", "hello2", 6, locale), 0);
+  ASSERT_GT(LIBC_NAMESPACE::strncasecmp_l("hello2", "hello1", 6, locale), 0);
+  LIBC_NAMESPACE::freelocale(locale);
+}
diff --git a/libcxx/docs/ReleaseNotes/21.rst b/libcxx/docs/ReleaseNotes/21.rst
index f45a448e52cc0..633d114693c82 100644
--- a/libcxx/docs/ReleaseNotes/21.rst
+++ b/libcxx/docs/ReleaseNotes/21.rst
@@ -39,7 +39,9 @@ Implemented Papers
 ------------------
 
 - N4258: Cleaning-up noexcept in the Library (`Github <https://github.com/llvm/llvm-project/issues/99937>`__)
+- P0767R1: Deprecate POD (`Github <https://github.com/llvm/llvm-project/issues/104013>`__)
 - P1361R2: Integration of chrono with text formatting (`Github <https://github.com/llvm/llvm-project/issues/100014>`__)
+- P2255R2: A type trait to detect reference binding to temporary (implemented the type traits only) (`Github <https://github.com/llvm/llvm-project/issues/105180>`__)
 
 Improvements and New Features
 -----------------------------
@@ -58,6 +60,8 @@ Improvements and New Features
 Deprecations and Removals
 -------------------------
 
+- ``std::is_pod`` and ``std::is_pod_v`` are deprecated in C++20 and later.
+
 Upcoming Deprecations and Removals
 ----------------------------------
 
diff --git a/libcxx/docs/Status/Cxx20Papers.csv b/libcxx/docs/Status/Cxx20Papers.csv
index cb25006797055..4430b785334c1 100644
--- a/libcxx/docs/Status/Cxx20Papers.csv
+++ b/libcxx/docs/Status/Cxx20Papers.csv
@@ -13,7 +13,7 @@
 "`P0616R0 <https://wg21.link/P0616R0>`__","de-pessimize legacy <numeric> algorithms with std::move","2017-11 (Albuquerque)","|Complete|","12",""
 "`P0653R2 <https://wg21.link/P0653R2>`__","Utility to convert a pointer to a raw pointer","2017-11 (Albuquerque)","|Complete|","6",""
 "`P0718R2 <https://wg21.link/P0718R2>`__","Atomic shared_ptr","2017-11 (Albuquerque)","","",""
-"`P0767R1 <https://wg21.link/P0767R1>`__","Deprecate POD","2017-11 (Albuquerque)","|Complete|","7",""
+"`P0767R1 <https://wg21.link/P0767R1>`__","Deprecate POD","2017-11 (Albuquerque)","|Complete|","21","It was previously erroneously marked as complete in LLVM 7."
 "`P0768R1 <https://wg21.link/P0768R1>`__","Library Support for the Spaceship (Comparison) Operator","2017-11 (Albuquerque)","|Complete|","",""
 "`P0777R1 <https://wg21.link/P0777R1>`__","Treating Unnecessary ``decay``\ ","2017-11 (Albuquerque)","|Complete|","7",""
 "","","","","",""
diff --git a/libcxx/docs/Status/Cxx23Papers.csv b/libcxx/docs/Status/Cxx23Papers.csv
index bfaa63a7c224e..2589954729a30 100644
--- a/libcxx/docs/Status/Cxx23Papers.csv
+++ b/libcxx/docs/Status/Cxx23Papers.csv
@@ -43,7 +43,7 @@
 "`P0627R6 <https://wg21.link/P0627R6>`__","Function to mark unreachable code","2022-02 (Virtual)","|Complete|","15",""
 "`P1206R7 <https://wg21.link/P1206R7>`__","``ranges::to``: A function to convert any range to a container","2022-02 (Virtual)","|Complete|","17",""
 "`P1413R3 <https://wg21.link/P1413R3>`__","Deprecate ``std::aligned_storage`` and ``std::aligned_union``","2022-02 (Virtual)","|Complete|","","``std::aligned_storage_t`` and ``std::aligned_union_t`` are marked deprecated, but clang doesn't issue a diagnostic for deprecated using template declarations."
-"`P2255R2 <https://wg21.link/P2255R2>`__","A type trait to detect reference binding to temporary","2022-02 (Virtual)","","",""
+"`P2255R2 <https://wg21.link/P2255R2>`__","A type trait to detect reference binding to temporary","2022-02 (Virtual)","|Partial|","","Implemented the type traits only."
 "`P2273R3 <https://wg21.link/P2273R3>`__","Making ``std::unique_ptr`` constexpr","2022-02 (Virtual)","|Complete|","16",""
 "`P2387R3 <https://wg21.link/P2387R3>`__","Pipe support for user-defined range adaptors","2022-02 (Virtual)","|Complete|","19",""
 "`P2440R1 <https://wg21.link/P2440R1>`__","``ranges::iota``, ``ranges::shift_left`` and ``ranges::shift_right``","2022-02 (Virtual)","","",""
@@ -82,7 +82,6 @@
 "`P2499R0 <https://wg21.link/P2499R0>`__","``string_view`` range constructor should be ``explicit``","2022-07 (Virtual)","|Complete|","16",""
 "`P2502R2 <https://wg21.link/P2502R2>`__","``std::generator``: Synchronous Coroutine Generator for Ranges","2022-07 (Virtual)","","",""
 "`P2508R1 <https://wg21.link/P2508R1>`__","Exposing ``std::basic-format-string``","2022-07 (Virtual)","|Complete|","15",""
-"`P2513R4 <https://wg21.link/P2513R4>`__","``char8_t`` Compatibility and Portability Fixes","2022-07 (Virtual)","","",""
 "`P2517R1 <https://wg21.link/P2517R1>`__","Add a conditional ``noexcept`` specification to ``std::apply``","2022-07 (Virtual)","|Complete|","3.9",""
 "`P2520R0 <https://wg21.link/P2520R0>`__","``move_iterator`` should be a random access iterator","2022-07 (Virtual)","|Complete|","17","Implemented as a DR in C++20"
 "`P2540R1 <https://wg21.link/P2540R1>`__","Empty Product for certain Views","2022-07 (Virtual)","","",""
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index ee1d7b872b364..d7c36d6b438fb 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -867,6 +867,8 @@ set(files
   __type_traits/negation.h
   __type_traits/promote.h
   __type_traits/rank.h
+  __type_traits/reference_constructs_from_temporary.h
+  __type_traits/reference_converts_from_temporary.h
   __type_traits/remove_all_extents.h
   __type_traits/remove_const.h
   __type_traits/remove_const_ref.h
diff --git a/libcxx/include/__memory/allocator_traits.h b/libcxx/include/__memory/allocator_traits.h
index 2d9ab847e9f25..57b591e5b4b31 100644
--- a/libcxx/include/__memory/allocator_traits.h
+++ b/libcxx/include/__memory/allocator_traits.h
@@ -138,6 +138,7 @@ template <class _Alloc>
 using __propagate_on_container_swap _LIBCPP_NODEBUG =
     __detected_or_t<false_type, __propagate_on_container_swap_member, _Alloc>;
 
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
 // __is_always_equal
 template <class _Tp>
 using __is_always_equal_member _LIBCPP_NODEBUG = typename _Tp::is_always_equal;
@@ -147,7 +148,6 @@ using __is_always_equal _LIBCPP_NODEBUG =
     __detected_or_t<typename is_empty<_Alloc>::type, __is_always_equal_member, _Alloc>;
 
 // __allocator_traits_rebind
-_LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <class _Tp, class _Up, class = void>
 struct __has_rebind_other : false_type {};
 template <class _Tp, class _Up>
diff --git a/libcxx/include/__type_traits/is_pod.h b/libcxx/include/__type_traits/is_pod.h
index a57662400394a..b6aacf3b2b202 100644
--- a/libcxx/include/__type_traits/is_pod.h
+++ b/libcxx/include/__type_traits/is_pod.h
@@ -19,11 +19,12 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
-struct _LIBCPP_TEMPLATE_VIS _LIBCPP_NO_SPECIALIZATIONS is_pod : public integral_constant<bool, __is_pod(_Tp)> {};
+struct _LIBCPP_TEMPLATE_VIS
+_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_NO_SPECIALIZATIONS is_pod : public integral_constant<bool, __is_pod(_Tp)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
-_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_pod_v = __is_pod(_Tp);
+_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_pod_v = __is_pod(_Tp);
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/reference_constructs_from_temporary.h b/libcxx/include/__type_traits/reference_constructs_from_temporary.h
new file mode 100644
index 0000000000000..1666032b89bed
--- /dev/null
+++ b/libcxx/include/__type_traits/reference_constructs_from_temporary.h
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_REFERENCE_CONSTRUCTS_FROM_TEMPORARY_H
+#define _LIBCPP___TYPE_TRAITS_REFERENCE_CONSTRUCTS_FROM_TEMPORARY_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23 && __has_builtin(__reference_constructs_from_temporary)
+
+template <class _Tp, class _Up>
+struct _LIBCPP_TEMPLATE_VIS _LIBCPP_NO_SPECIALIZATIONS reference_constructs_from_temporary
+    : public bool_constant<__reference_constructs_from_temporary(_Tp, _Up)> {};
+
+template <class _Tp, class _Up>
+_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool reference_constructs_from_temporary_v =
+    __reference_constructs_from_temporary(_Tp, _Up);
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_REFERENCE_CONSTRUCTS_FROM_TEMPORARY_H
diff --git a/libcxx/include/__type_traits/reference_converts_from_temporary.h b/libcxx/include/__type_traits/reference_converts_from_temporary.h
new file mode 100644
index 0000000000000..043bdbf38dada
--- /dev/null
+++ b/libcxx/include/__type_traits/reference_converts_from_temporary.h
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_REFERENCE_CONVERTS_FROM_TEMPORARY_H
+#define _LIBCPP___TYPE_TRAITS_REFERENCE_CONVERTS_FROM_TEMPORARY_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23 && __has_builtin(__reference_converts_from_temporary)
+
+template <class _Tp, class _Up>
+struct _LIBCPP_TEMPLATE_VIS _LIBCPP_NO_SPECIALIZATIONS reference_converts_from_temporary
+    : public bool_constant<__reference_converts_from_temporary(_Tp, _Up)> {};
+
+template <class _Tp, class _Up>
+_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool reference_converts_from_temporary_v =
+    __reference_converts_from_temporary(_Tp, _Up);
+
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___TYPE_TRAITS_REFERENCE_CONVERTS_FROM_TEMPORARY_H
diff --git a/libcxx/include/future b/libcxx/include/future
index 22b387d3ba094..a08687485bd99 100644
--- a/libcxx/include/future
+++ b/libcxx/include/future
@@ -865,8 +865,7 @@ void __async_assoc_state<_Rp, _Fp>::__execute() {
 
 template <class _Rp, class _Fp>
 void __async_assoc_state<_Rp, _Fp>::__on_zero_shared() _NOEXCEPT {
-  if (base::__state_ & base::__constructed)
-    this->wait();
+  this->wait();
   base::__on_zero_shared();
 }
 
@@ -903,8 +902,7 @@ void __async_assoc_state<void, _Fp>::__execute() {
 
 template <class _Fp>
 void __async_assoc_state<void, _Fp>::__on_zero_shared() _NOEXCEPT {
-  if (base::__state_ & base::__constructed)
-    this->wait();
+  this->wait();
   base::__on_zero_shared();
 }
 
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index 81c7378d9e3b8..b9964dac84acd 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -369,6 +369,14 @@ module std_core [system] {
     module negation                                   { header "__type_traits/negation.h" }
     module promote                                    { header "__type_traits/promote.h" }
     module rank                                       { header "__type_traits/rank.h" }
+    module reference_constructs_from_temporary {
+      header "__type_traits/reference_constructs_from_temporary.h"
+      export std_core.type_traits.integral_constant
+    }
+    module reference_converts_from_temporary {
+      header "__type_traits/reference_converts_from_temporary.h"
+      export std_core.type_traits.integral_constant
+    }
     module remove_all_extents                         { header "__type_traits/remove_all_extents.h" }
     module remove_const_ref                           { header "__type_traits/remove_const_ref.h" }
     module remove_const                               { header "__type_traits/remove_const.h" }
diff --git a/libcxx/include/regex b/libcxx/include/regex
index eebd68af54297..96229c6a6ee42 100644
--- a/libcxx/include/regex
+++ b/libcxx/include/regex
@@ -3955,7 +3955,7 @@ _ForwardIterator basic_regex<_CharT, _Traits>::__parse_character_escape(
       ++__first;
       break;
     default:
-      if (*__first != '_' && !__traits_.isctype(*__first, ctype_base::alnum)) {
+      if (!__traits_.isctype(*__first, ctype_base::alnum)) {
         if (__str)
           *__str = *__first;
         else
diff --git a/libcxx/include/type_traits b/libcxx/include/type_traits
index ffcddb0176615..a03a60917cd54 100644
--- a/libcxx/include/type_traits
+++ b/libcxx/include/type_traits
@@ -18,13 +18,11 @@ namespace std
 
     // helper class:
     template <class T, T v> struct integral_constant;
-    typedef integral_constant<bool, true>  true_type;   // C++11
-    typedef integral_constant<bool, false> false_type;  // C++11
+    typedef integral_constant<bool, true>  true_type;           // since C++11
+    typedef integral_constant<bool, false> false_type;          // since C++11
 
-    template <bool B>                                   // C++14
-    using bool_constant = integral_constant<bool, B>;   // C++14
-    typedef bool_constant<true> true_type;              // C++14
-    typedef bool_constant<false> false_type;            // C++14
+    template <bool B>
+    using bool_constant = integral_constant<bool, B>;           // since C++17
 
     // helper traits
     template <bool, class T = void> struct enable_if;
@@ -32,7 +30,7 @@ namespace std
 
     // Primary classification traits:
     template <class T> struct is_void;
-    template <class T> struct is_null_pointer;  // C++14
+    template <class T> struct is_null_pointer;                  // since C++14
     template <class T> struct is_integral;
     template <class T> struct is_floating_point;
     template <class T> struct is_array;
@@ -51,7 +49,7 @@ namespace std
     template <class T> struct is_arithmetic;
     template <class T> struct is_fundamental;
     template <class T> struct is_member_pointer;
-    template <class T> struct is_scoped_enum; // C++23
+    template <class T> struct is_scoped_enum;                   // since C++23
     template <class T> struct is_scalar;
     template <class T> struct is_object;
     template <class T> struct is_compound;
@@ -75,9 +73,9 @@ namespace std
     template <class T> struct remove_pointer;
     template <class T> struct add_pointer;
 
-    template<class T> struct type_identity;                     // C++20
+    template<class T> struct type_identity;                     // since C++20
     template<class T>
-      using type_identity_t = typename type_identity<T>::type;  // C++20
+      using type_identity_t = typename type_identity<T>::type;  // since C++20
 
     // Integral properties:
     template <class T> struct is_signed;
@@ -91,20 +89,20 @@ namespace std
     template <class T> struct remove_extent;
     template <class T> struct remove_all_extents;
 
-    template <class T> struct is_bounded_array;                 // C++20
-    template <class T> struct is_unbounded_array;               // C++20
+    template <class T> struct is_bounded_array;                 // since C++20
+    template <class T> struct is_unbounded_array;               // since C++20
 
     // Member introspection:
-    template <class T> struct is_pod;
     template <class T> struct is_trivial;
+    template <class T> struct is_pod;                           // deprecated in C++20
     template <class T> struct is_trivially_copyable;
     template <class T> struct is_standard_layout;
-    template <class T> struct is_literal_type; // Deprecated in C++17; removed in C++20
+    template <class T> struct is_literal_type;                  // deprecated in C++17; removed in C++20
     template <class T> struct is_empty;
     template <class T> struct is_polymorphic;
     template <class T> struct is_abstract;
-    template <class T> struct is_final; // C++14
-    template <class T> struct is_aggregate; // C++17
+    template <class T> struct is_final;                         // since C++14
+    template <class T> struct is_aggregate;                     // since C++17
 
     template <class T, class... Args> struct is_constructible;
     template <class T>                struct is_default_constructible;
@@ -113,8 +111,8 @@ namespace std
     template <class T, class U>       struct is_assignable;
     template <class T>                struct is_copy_assignable;
     template <class T>                struct is_move_assignable;
-    template <class T, class U>       struct is_swappable_with;       // C++17
-    template <class T>                struct is_swappable;            // C++17
+    template <class T, class U>       struct is_swappable_with; // since C++17
+    template <class T>                struct is_swappable;      // since C++17
     template <class T>                struct is_destructible;
 
     template <class T, class... Args> struct is_trivially_constructible;
@@ -133,292 +131,328 @@ namespace std
     template <class T, class U>       struct is_nothrow_assignable;
     template <class T>                struct is_nothrow_copy_assignable;
     template <class T>                struct is_nothrow_move_assignable;
-    template <class T, class U>       struct is_nothrow_swappable_with; // C++17
-    template <class T>                struct is_nothrow_swappable;      // C++17
+    template <class T, class U>
+      struct is_nothrow_swappable_with;                         // since C++17
+    template <class T>
+      struct is_nothrow_swappable;                              // since C++17
     template <class T>                struct is_nothrow_destructible;
 
-    template<class T> struct is_implicit_lifetime;                     // Since C++23
+    template <class T> struct is_implicit_lifetime;             // since C++23
 
     template <class T> struct has_virtual_destructor;
 
-    template<class T> struct has_unique_object_representations;         // C++17
+    template <class T>
+      struct has_unique_object_representations;                 // since C++17
+
+    template<class T, class U>
+      struct reference_constructs_from_temporary;               // since C++23
+    template<class T, class U>
+      struct reference_converts_from_temporary;                 // since C++23
 
     // Relationships between types:
     template <class T, class U> struct is_same;
     template <class Base, class Derived> struct is_base_of;
-    template <class Base, class Derived> struct is_virtual_base_of;     // C++26
+    template <class Base, class Derived>
+      struct is_virtual_base_of;                                // since C++26
 
     template <class From, class To> struct is_convertible;
-    template <typename From, typename To> struct is_nothrow_convertible;                  // C++20
-    template <typename From, typename To> inline constexpr bool is_nothrow_convertible_v; // C++20
+    template <class From, class To>
+      struct is_nothrow_convertible;                            // since C++20
 
-    template <class Fn, class... ArgTypes> struct is_invocable;
-    template <class R, class Fn, class... ArgTypes> struct is_invocable_r;
+    template <class Fn, class... ArgTypes> struct is_invocable; // since C++17
+    template <class R, class Fn, class... ArgTypes>
+      struct is_invocable_r;                                    // since C++17
 
-    template <class Fn, class... ArgTypes> struct is_nothrow_invocable;
-    template <class R, class Fn, class... ArgTypes> struct is_nothrow_invocable_r;
+    template <class Fn, class... ArgTypes>
+      struct is_nothrow_invocable;                              // since C++17
+    template <class R, class Fn, class... ArgTypes>
+      struct is_nothrow_invocable_r;                            // since C++17
 
     // Alignment properties and transformations:
     template <class T> struct alignment_of;
     template <size_t Len, size_t Align = most_stringent_alignment_requirement>
-        struct aligned_storage;                                 // deprecated in C++23
+      struct aligned_storage;                                   // deprecated in C++23
     template <size_t Len, class... Types> struct aligned_union; // deprecated in C++23
-    template <class T> struct remove_cvref; // C++20
+    template <class T> struct remove_cvref;                     // since C++20
 
     template <class T> struct decay;
     template <class... T> struct common_type;
     template <class T> struct underlying_type;
-    template <class> class result_of; // undefined; deprecated in C++17; removed in C++20
-    template <class Fn, class... ArgTypes> class result_of<Fn(ArgTypes...)>; // deprecated in C++17; removed in C++20
-    template <class Fn, class... ArgTypes> struct invoke_result;  // C++17
+    template <class> struct result_of;                          // undefined; deprecated in C++17; removed in C++20
+    template <class Fn, class... ArgTypes>
+      struct result_of<Fn(ArgTypes...)>;                        // deprecated in C++17; removed in C++20
+    template <class Fn, class... ArgTypes>
+      struct invoke_result;                                     // since C++17
 
     // const-volatile modifications:
     template <class T>
-      using remove_const_t    = typename remove_const<T>::type;  // C++14
+      using remove_const_t = typename remove_const<T>::type;    // since C++14
     template <class T>
-      using remove_volatile_t = typename remove_volatile<T>::type;  // C++14
+      using remove_volatile_t
+        = typename remove_volatile<T>::type;                    // since C++14
     template <class T>
-      using remove_cv_t       = typename remove_cv<T>::type;  // C++14
+      using remove_cv_t    = typename remove_cv<T>::type;       // since C++14
     template <class T>
-      using add_const_t       = typename add_const<T>::type;  // C++14
+      using add_const_t    = typename add_const<T>::type;       // since C++14
     template <class T>
-      using add_volatile_t    = typename add_volatile<T>::type;  // C++14
+      using add_volatile_t = typename add_volatile<T>::type;    // since C++14
     template <class T>
-      using add_cv_t          = typename add_cv<T>::type;  // C++14
+      using add_cv_t       = typename add_cv<T>::type;          // since C++14
 
     // reference modifications:
     template <class T>
-      using remove_reference_t     = typename remove_reference<T>::type;  // C++14
+      using remove_reference_t
+        = typename remove_reference<T>::type;                   // since C++14
     template <class T>
-      using add_lvalue_reference_t = typename add_lvalue_reference<T>::type;  // C++14
+      using add_lvalue_reference_t
+        = typename add_lvalue_reference<T>::type;               // since C++14
     template <class T>
-      using add_rvalue_reference_t = typename add_rvalue_reference<T>::type;  // C++14
+      using add_rvalue_reference_t
+        = typename add_rvalue_reference<T>::type;               // since C++14
 
     // sign modifications:
     template <class T>
-      using make_signed_t   = typename make_signed<T>::type;  // C++14
+      using make_signed_t   = typename make_signed<T>::type;    // since C++14
     template <class T>
-      using make_unsigned_t = typename make_unsigned<T>::type;  // C++14
+      using make_unsigned_t = typename make_unsigned<T>::type;  // since C++14
 
     // array modifications:
     template <class T>
-      using remove_extent_t      = typename remove_extent<T>::type;  // C++14
+      using remove_extent_t
+        = typename remove_extent<T>::type;                      // since C++14
     template <class T>
-      using remove_all_extents_t = typename remove_all_extents<T>::type;  // C++14
+      using remove_all_extents_t
+        = typename remove_all_extents<T>::type;                 // since C++14
 
     template <class T>
       inline constexpr bool is_bounded_array_v
-        = is_bounded_array<T>::value;                                     // C++20
+        = is_bounded_array<T>::value;                           // since C++20
       inline constexpr bool is_unbounded_array_v
-        = is_unbounded_array<T>::value;                                   // C++20
+        = is_unbounded_array<T>::value;                         // since C++20
 
     // pointer modifications:
     template <class T>
-      using remove_pointer_t = typename remove_pointer<T>::type;  // C++14
+      using remove_pointer_t
+        = typename remove_pointer<T>::type;                     // since C++14
     template <class T>
-      using add_pointer_t    = typename add_pointer<T>::type;  // C++14
+      using add_pointer_t = typename add_pointer<T>::type;      // since C++14
 
     // other transformations:
     template <size_t Len, size_t Align=default-alignment>
-      using aligned_storage_t = typename aligned_storage<Len,Align>::type;  // C++14
+      using aligned_storage_t
+        = typename aligned_storage<Len,Align>::type;            // since C++14
     template <size_t Len, class... Types>
-      using aligned_union_t   = typename aligned_union<Len,Types...>::type;  // C++14
+      using aligned_union_t
+        = typename aligned_union<Len,Types...>::type;           // since C++14
     template <class T>
-      using remove_cvref_t    = typename remove_cvref<T>::type;  // C++20
+      using remove_cvref_t
+        = typename remove_cvref<T>::type;                       // since C++20
     template <class T>
-      using decay_t           = typename decay<T>::type;  // C++14
+      using decay_t     = typename decay<T>::type;              // since C++14
     template <bool b, class T=void>
-      using enable_if_t       = typename enable_if<b,T>::type;  // C++14
+      using enable_if_t = typename enable_if<b,T>::type;        // since C++14
     template <bool b, class T, class F>
-      using conditional_t     = typename conditional<b,T,F>::type;  // C++14
+      using conditional_t
+        = typename conditional<b,T,F>::type;                    // since C++14
     template <class... T>
-      using common_type_t     = typename common_type<T...>::type;  // C++14
+      using common_type_t
+        = typename common_type<T...>::type;                     // since C++14
     template <class T>
-      using underlying_type_t = typename underlying_type<T>::type;  // C++14
+      using underlying_type_t
+        = typename underlying_type<T>::type;                    // since C++14
     template <class T>
-      using result_of_t       = typename result_of<T>::type;  // C++14; deprecated in C++17; removed in C++20
+      using result_of_t = typename result_of<T>::type;          // since C++14; deprecated in C++17; removed in C++20
     template <class Fn, class... ArgTypes>
-      using invoke_result_t   = typename invoke_result<Fn, ArgTypes...>::type;  // C++17
+      using invoke_result_t
+       = typename invoke_result<Fn, ArgTypes...>::type;         // since C++17
 
     template <class...>
-      using void_t = void;   // C++17
+      using void_t = void;                                      // since C++17
 
       // See C++14 20.10.4.1, primary type categories
       template <class T> inline constexpr bool is_void_v
-        = is_void<T>::value;                                             // C++17
+        = is_void<T>::value;                                    // since C++17
       template <class T> inline constexpr bool is_null_pointer_v
-        = is_null_pointer<T>::value;                                     // C++17
+        = is_null_pointer<T>::value;                            // since C++17
       template <class T> inline constexpr bool is_integral_v
-        = is_integral<T>::value;                                         // C++17
+        = is_integral<T>::value;                                // since C++17
       template <class T> inline constexpr bool is_floating_point_v
-        = is_floating_point<T>::value;                                   // C++17
+        = is_floating_point<T>::value;                          // since C++17
       template <class T> inline constexpr bool is_array_v
-        = is_array<T>::value;                                            // C++17
+        = is_array<T>::value;                                   // since C++17
       template <class T> inline constexpr bool is_pointer_v
-        = is_pointer<T>::value;                                          // C++17
+        = is_pointer<T>::value;                                 // since C++17
       template <class T> inline constexpr bool is_lvalue_reference_v
-        = is_lvalue_reference<T>::value;                                 // C++17
+        = is_lvalue_reference<T>::value;                        // since C++17
       template <class T> inline constexpr bool is_rvalue_reference_v
-        = is_rvalue_reference<T>::value;                                 // C++17
+        = is_rvalue_reference<T>::value;                        // since C++17
       template <class T> inline constexpr bool is_member_object_pointer_v
-        = is_member_object_pointer<T>::value;                            // C++17
+        = is_member_object_pointer<T>::value;                   // since C++17
       template <class T> inline constexpr bool is_member_function_pointer_v
-        = is_member_function_pointer<T>::value;                          // C++17
+        = is_member_function_pointer<T>::value;                 // since C++17
       template <class T> inline constexpr bool is_enum_v
-        = is_enum<T>::value;                                             // C++17
+        = is_enum<T>::value;                                    // since C++17
       template <class T> inline constexpr bool is_union_v
-        = is_union<T>::value;                                            // C++17
+        = is_union<T>::value;                                   // since C++17
       template <class T> inline constexpr bool is_class_v
-        = is_class<T>::value;                                            // C++17
+        = is_class<T>::value;                                   // since C++17
       template <class T> inline constexpr bool is_function_v
-        = is_function<T>::value;                                         // C++17
+        = is_function<T>::value;                                // since C++17
 
       // See C++14 20.10.4.2, composite type categories
       template <class T> inline constexpr bool is_reference_v
-        = is_reference<T>::value;                                        // C++17
+        = is_reference<T>::value;                               // since C++17
       template <class T> inline constexpr bool is_arithmetic_v
-        = is_arithmetic<T>::value;                                       // C++17
+        = is_arithmetic<T>::value;                              // since C++17
       template <class T> inline constexpr bool is_fundamental_v
-        = is_fundamental<T>::value;                                      // C++17
+        = is_fundamental<T>::value;                             // since C++17
       template <class T> inline constexpr bool is_object_v
-        = is_object<T>::value;                                           // C++17
+        = is_object<T>::value;                                  // since C++17
       template <class T> inline constexpr bool is_scalar_v
-        = is_scalar<T>::value;                                           // C++17
+        = is_scalar<T>::value;                                  // since C++17
       template <class T> inline constexpr bool is_compound_v
-        = is_compound<T>::value;                                         // C++17
+        = is_compound<T>::value;                                // since C++17
       template <class T> inline constexpr bool is_member_pointer_v
-        = is_member_pointer<T>::value;                                   // C++17
+        = is_member_pointer<T>::value;                          // since C++17
       template <class T> inline constexpr bool is_scoped_enum_v
-        = is_scoped_enum<T>::value;                                      // C++23
+        = is_scoped_enum<T>::value;                             // since C++23
 
       // See C++14 20.10.4.3, type properties
       template <class T> inline constexpr bool is_const_v
-        = is_const<T>::value;                                            // C++17
+        = is_const<T>::value;                                   // since C++17
       template <class T> inline constexpr bool is_volatile_v
-        = is_volatile<T>::value;                                         // C++17
+        = is_volatile<T>::value;                                // since C++17
       template <class T> inline constexpr bool is_trivial_v
-        = is_trivial<T>::value;                                          // C++17
+        = is_trivial<T>::value;                                 // since C++17
       template <class T> inline constexpr bool is_trivially_copyable_v
-        = is_trivially_copyable<T>::value;                               // C++17
+        = is_trivially_copyable<T>::value;                      // since C++17
       template <class T> inline constexpr bool is_standard_layout_v
-        = is_standard_layout<T>::value;                                  // C++17
+        = is_standard_layout<T>::value;                         // since C++17
       template <class T> inline constexpr bool is_pod_v
-        = is_pod<T>::value;                                              // C++17
+        = is_pod<T>::value;                                     // since C++17; deprecated in C++20
       template <class T> inline constexpr bool is_literal_type_v
-        = is_literal_type<T>::value;                                     // C++17; deprecated in C++17; removed in C++20
+        = is_literal_type<T>::value;                            // since C++17; deprecated in C++17; removed in C++20
       template <class T> inline constexpr bool is_empty_v
-        = is_empty<T>::value;                                            // C++17
+        = is_empty<T>::value;                                   // since C++17
       template <class T> inline constexpr bool is_polymorphic_v
-        = is_polymorphic<T>::value;                                      // C++17
+        = is_polymorphic<T>::value;                             // since C++17
       template <class T> inline constexpr bool is_abstract_v
-        = is_abstract<T>::value;                                         // C++17
+        = is_abstract<T>::value;                                // since C++17
       template <class T> inline constexpr bool is_final_v
-        = is_final<T>::value;                                            // C++17
+        = is_final<T>::value;                                   // since C++17
       template <class T> inline constexpr bool is_aggregate_v
-        = is_aggregate<T>::value;                                        // C++17
+        = is_aggregate<T>::value;                               // since C++17
       template <class T> inline constexpr bool is_signed_v
-        = is_signed<T>::value;                                           // C++17
+        = is_signed<T>::value;                                  // since C++17
       template <class T> inline constexpr bool is_unsigned_v
-        = is_unsigned<T>::value;                                         // C++17
+        = is_unsigned<T>::value;                                // since C++17
       template <class T, class... Args> inline constexpr bool is_constructible_v
-        = is_constructible<T, Args...>::value;                           // C++17
+        = is_constructible<T, Args...>::value;                  // since C++17
       template <class T> inline constexpr bool is_default_constructible_v
-        = is_default_constructible<T>::value;                            // C++17
+        = is_default_constructible<T>::value;                   // since C++17
       template <class T> inline constexpr bool is_copy_constructible_v
-        = is_copy_constructible<T>::value;                               // C++17
+        = is_copy_constructible<T>::value;                      // since C++17
       template <class T> inline constexpr bool is_move_constructible_v
-        = is_move_constructible<T>::value;                               // C++17
+        = is_move_constructible<T>::value;                      // since C++17
       template <class T, class U> inline constexpr bool is_assignable_v
-        = is_assignable<T, U>::value;                                    // C++17
+        = is_assignable<T, U>::value;                           // since C++17
       template <class T> inline constexpr bool is_copy_assignable_v
-        = is_copy_assignable<T>::value;                                  // C++17
+        = is_copy_assignable<T>::value;                         // since C++17
       template <class T> inline constexpr bool is_move_assignable_v
-        = is_move_assignable<T>::value;                                  // C++17
+        = is_move_assignable<T>::value;                         // since C++17
       template <class T, class U> inline constexpr bool is_swappable_with_v
-        = is_swappable_with<T, U>::value;                                // C++17
+        = is_swappable_with<T, U>::value;                       // since C++17
       template <class T> inline constexpr bool is_swappable_v
-        = is_swappable<T>::value;                                        // C++17
+        = is_swappable<T>::value;                               // since C++17
       template <class T> inline constexpr bool is_destructible_v
-        = is_destructible<T>::value;                                     // C++17
+        = is_destructible<T>::value;                            // since C++17
       template <class T, class... Args> inline constexpr bool is_trivially_constructible_v
-        = is_trivially_constructible<T, Args...>::value;                 // C++17
+        = is_trivially_constructible<T, Args...>::value;        // since C++17
       template <class T> inline constexpr bool is_trivially_default_constructible_v
-        = is_trivially_default_constructible<T>::value;                  // C++17
+        = is_trivially_default_constructible<T>::value;         // since C++17
       template <class T> inline constexpr bool is_trivially_copy_constructible_v
-        = is_trivially_copy_constructible<T>::value;                     // C++17
+        = is_trivially_copy_constructible<T>::value;            // since C++17
       template <class T> inline constexpr bool is_trivially_move_constructible_v
-        = is_trivially_move_constructible<T>::value;                     // C++17
+        = is_trivially_move_constructible<T>::value;            // since C++17
       template <class T, class U> inline constexpr bool is_trivially_assignable_v
-        = is_trivially_assignable<T, U>::value;                          // C++17
+        = is_trivially_assignable<T, U>::value;                 // since C++17
       template <class T> inline constexpr bool is_trivially_copy_assignable_v
-        = is_trivially_copy_assignable<T>::value;                        // C++17
+        = is_trivially_copy_assignable<T>::value;               // since C++17
       template <class T> inline constexpr bool is_trivially_move_assignable_v
-        = is_trivially_move_assignable<T>::value;                        // C++17
+        = is_trivially_move_assignable<T>::value;               // since C++17
       template <class T> inline constexpr bool is_trivially_destructible_v
-        = is_trivially_destructible<T>::value;                           // C++17
+        = is_trivially_destructible<T>::value;                  // since C++17
       template <class T, class... Args> inline constexpr bool is_nothrow_constructible_v
-        = is_nothrow_constructible<T, Args...>::value;                   // C++17
+        = is_nothrow_constructible<T, Args...>::value;          // since C++17
       template <class T> inline constexpr bool is_nothrow_default_constructible_v
-        = is_nothrow_default_constructible<T>::value;                    // C++17
+        = is_nothrow_default_constructible<T>::value;           // since C++17
       template <class T> inline constexpr bool is_nothrow_copy_constructible_v
-        = is_nothrow_copy_constructible<T>::value;                       // C++17
+        = is_nothrow_copy_constructible<T>::value;              // since C++17
       template <class T> inline constexpr bool is_nothrow_move_constructible_v
-        = is_nothrow_move_constructible<T>::value;                       // C++17
+        = is_nothrow_move_constructible<T>::value;              // since C++17
       template <class T, class U> inline constexpr bool is_nothrow_assignable_v
-        = is_nothrow_assignable<T, U>::value;                            // C++17
+        = is_nothrow_assignable<T, U>::value;                   // since C++17
       template <class T> inline constexpr bool is_nothrow_copy_assignable_v
-        = is_nothrow_copy_assignable<T>::value;                          // C++17
+        = is_nothrow_copy_assignable<T>::value;                 // since C++17
       template <class T> inline constexpr bool is_nothrow_move_assignable_v
-        = is_nothrow_move_assignable<T>::value;                          // C++17
+        = is_nothrow_move_assignable<T>::value;                 // since C++17
       template <class T, class U> inline constexpr bool is_nothrow_swappable_with_v
-        = is_nothrow_swappable_with<T, U>::value;                       // C++17
+        = is_nothrow_swappable_with<T, U>::value;               // since C++17
       template <class T> inline constexpr bool is_nothrow_swappable_v
-        = is_nothrow_swappable<T>::value;                               // C++17
+        = is_nothrow_swappable<T>::value;                       // since C++17
       template <class T> inline constexpr bool is_nothrow_destructible_v
-        = is_nothrow_destructible<T>::value;                             // C++17
-      template<class T>
-        constexpr bool is_implicit_lifetime_v = is_implicit_lifetime<T>::value;   // Since C++23
+        = is_nothrow_destructible<T>::value;                    // since C++17
+      template <class T> inline constexpr bool is_implicit_lifetime_v
+        = is_implicit_lifetime<T>::value;                       // since C++23
       template <class T> inline constexpr bool has_virtual_destructor_v
-        = has_virtual_destructor<T>::value;                              // C++17
-      template<class T> inline constexpr bool has_unique_object_representations_v // C++17
-        = has_unique_object_representations<T>::value;
+        = has_virtual_destructor<T>::value;                     // since C++17
+      template<class T> inline constexpr bool has_unique_object_representations_v
+        = has_unique_object_representations<T>::value;          // since C++17
+      template<class T, class U>
+        constexpr bool reference_constructs_from_temporary_v
+          = reference_constructs_from_temporary<T, U>::value;   // since C++23
+      template<class T, class U>
+        constexpr bool reference_converts_from_temporary_v
+          = reference_converts_from_temporary<T, U>::value;     // since C++23
 
       // See C++14 20.10.5, type property queries
       template <class T> inline constexpr size_t alignment_of_v
-        = alignment_of<T>::value;                                        // C++17
+        = alignment_of<T>::value;                               // since C++17
       template <class T> inline constexpr size_t rank_v
-        = rank<T>::value;                                                // C++17
+        = rank<T>::value;                                       // since C++17
       template <class T, unsigned I = 0> inline constexpr size_t extent_v
-        = extent<T, I>::value;                                           // C++17
+        = extent<T, I>::value;                                  // since C++17
 
       // See C++14 20.10.6, type relations
       template <class T, class U> inline constexpr bool is_same_v
-        = is_same<T, U>::value;                                          // C++17
+        = is_same<T, U>::value;                                 // since C++17
       template <class Base, class Derived> inline constexpr bool is_base_of_v
-        = is_base_of<Base, Derived>::value;                              // C++17
+        = is_base_of<Base, Derived>::value;                     // since C++17
       template <class Base, class Derived> inline constexpr bool is_virtual_base_of_v
-        = is_virtual_base_of<Base, Derived>::value;                      // C++26
+        = is_virtual_base_of<Base, Derived>::value;             // since C++26
       template <class From, class To> inline constexpr bool is_convertible_v
-        = is_convertible<From, To>::value;                               // C++17
+        = is_convertible<From, To>::value;                      // since C++17
+      template <class From, class To> inline constexpr bool is_nothrow_convertible_v
+        = is_nothrow_convertible<From, To>::value;              // since C++20
       template <class Fn, class... ArgTypes> inline constexpr bool is_invocable_v
-        = is_invocable<Fn, ArgTypes...>::value;                          // C++17
+        = is_invocable<Fn, ArgTypes...>::value;                 // since C++17
       template <class R, class Fn, class... ArgTypes> inline constexpr bool is_invocable_r_v
-        = is_invocable_r<R, Fn, ArgTypes...>::value;                     // C++17
+        = is_invocable_r<R, Fn, ArgTypes...>::value;            // since C++17
       template <class Fn, class... ArgTypes> inline constexpr bool is_nothrow_invocable_v
-        = is_nothrow_invocable<Fn, ArgTypes...>::value;                  // C++17
+        = is_nothrow_invocable<Fn, ArgTypes...>::value;         // since C++17
       template <class R, class Fn, class... ArgTypes> inline constexpr bool is_nothrow_invocable_r_v
-        = is_nothrow_invocable_r<R, Fn, ArgTypes...>::value;             // C++17
+        = is_nothrow_invocable_r<R, Fn, ArgTypes...>::value;    // since C++17
 
       // [meta.logical], logical operator traits:
-      template<class... B> struct conjunction;                           // C++17
-      template<class... B>
-        inline constexpr bool conjunction_v = conjunction<B...>::value;  // C++17
-      template<class... B> struct disjunction;                           // C++17
-      template<class... B>
-        inline constexpr bool disjunction_v = disjunction<B...>::value;  // C++17
-      template<class B> struct negation;                                 // C++17
-      template<class B>
-        inline constexpr bool negation_v = negation<B>::value;           // C++17
+      template<class... B> struct conjunction;                  // since C++17
+      template<class... B> inline constexpr bool conjunction_v
+        = conjunction<B...>::value;                             // since C++17
+      template<class... B> struct disjunction;                  // since C++17
+      template<class... B> inline constexpr bool disjunction_v
+        = disjunction<B...>::value;                             // since C++17
+      template<class B> struct negation;                        // since C++17
+      template<class B> inline constexpr bool negation_v
+        = negation<B>::value;                                   // since C++17
 
 }
 
@@ -523,6 +557,8 @@ namespace std
 
 #  if _LIBCPP_STD_VER >= 23
 #    include <__type_traits/is_implicit_lifetime.h>
+#    include <__type_traits/reference_constructs_from_temporary.h>
+#    include <__type_traits/reference_converts_from_temporary.h>
 #  endif
 
 #  include <version>
diff --git a/libcxx/modules/std/type_traits.inc b/libcxx/modules/std/type_traits.inc
index f544f95c7aaaa..6823c86ed153b 100644
--- a/libcxx/modules/std/type_traits.inc
+++ b/libcxx/modules/std/type_traits.inc
@@ -106,8 +106,14 @@ export namespace std {
 
   using std::has_unique_object_representations;
 
-  // using std::reference_constructs_from_temporary;
-  // using std::reference_converts_from_temporary;
+#if _LIBCPP_STD_VER >= 23
+#  if __has_builtin(__reference_constructs_from_temporary)
+  using std::reference_constructs_from_temporary;
+#  endif
+#  if __has_builtin(__reference_converts_from_temporary)
+  using std::reference_converts_from_temporary;
+#  endif
+#endif
 
   // [meta.unary.prop.query], type property queries
   using std::alignment_of;
@@ -284,8 +290,14 @@ export namespace std {
   using std::is_unbounded_array_v;
   using std::is_unsigned_v;
   using std::is_volatile_v;
-  // using std::reference_constructs_from_temporary_v;
-  // using std::reference_converts_from_temporary_v;
+#if _LIBCPP_STD_VER >= 23
+#  if __has_builtin(__reference_constructs_from_temporary)
+  using std::reference_constructs_from_temporary_v;
+#  endif
+#  if __has_builtin(__reference_converts_from_temporary)
+  using std::reference_converts_from_temporary_v;
+#  endif
+#endif
 
   // [meta.unary.prop.query], type property queries
   using std::alignment_of_v;
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
index 066ed1191dd05..92f6a622c329c 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_strong.pass.cpp
@@ -1,3 +1,4 @@
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
index e83a143df3f04..3bee003b2143f 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.compare_exchange_weak.pass.cpp
@@ -1,3 +1,4 @@
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp
index ef3705d1db275..3d4700406984c 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.ctor.pass.cpp
@@ -1,3 +1,4 @@
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
index bc92b3dc36225..504d135c4f3d7 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.load.pass.cpp
@@ -1,3 +1,4 @@
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
index ab0d4a220c94f..1afa42528e14f 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.store.pass.cpp
@@ -1,3 +1,4 @@
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp b/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
index dcec2fb628545..39178d2393be2 100644
--- a/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.ref/assert.wait.pass.cpp
@@ -1,3 +1,4 @@
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/compare_exchange_strong.verify.cpp b/libcxx/test/libcxx/atomics/atomics.ref/compare_exchange_strong.verify.cpp
new file mode 100644
index 0000000000000..427c82dbd0af2
--- /dev/null
+++ b/libcxx/test/libcxx/atomics/atomics.ref/compare_exchange_strong.verify.cpp
@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: diagnose-if-support
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <atomic>
+
+// bool compare_exchange_strong(T& expected, T desired, memory_order success,
+// memory_order failure) const noexcept;
+//
+// Preconditions: failure is memory_order::relaxed, memory_order::consume,
+// memory_order::acquire, or memory_order::seq_cst.
+
+#include <atomic>
+
+void test() {
+  using T = int;
+
+  T x(T(1));
+  std::atomic_ref const a(x);
+
+  T expected(T(2));
+  T const desired(T(3));
+  std::memory_order const success = std::memory_order_relaxed;
+  // clang-format off
+  a.compare_exchange_strong(expected, desired, success, std::memory_order_relaxed);
+  a.compare_exchange_strong(expected, desired, success, std::memory_order_consume);
+  a.compare_exchange_strong(expected, desired, success, std::memory_order_acquire);
+  a.compare_exchange_strong(expected, desired, success, std::memory_order_seq_cst);
+  a.compare_exchange_strong(expected, desired, success, std::memory_order_release); // expected-warning {{memory order argument to atomic operation is invalid}}
+  a.compare_exchange_strong(expected, desired, success, std::memory_order_acq_rel); // expected-warning {{memory order argument to atomic operation is invalid}}
+  // clang-format on
+}
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/compare_exchange_weak.verify.cpp b/libcxx/test/libcxx/atomics/atomics.ref/compare_exchange_weak.verify.cpp
new file mode 100644
index 0000000000000..6a1046074dd26
--- /dev/null
+++ b/libcxx/test/libcxx/atomics/atomics.ref/compare_exchange_weak.verify.cpp
@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: diagnose-if-support
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <atomic>
+
+// bool compare_exchange_weak(T& expected, T desired, memory_order success,
+// memory_order failure) const noexcept;
+//
+// Preconditions: failure is memory_order::relaxed, memory_order::consume,
+// memory_order::acquire, or memory_order::seq_cst.
+
+#include <atomic>
+
+void test() {
+  using T = int;
+
+  T x(T(42));
+  std::atomic_ref const a(x);
+
+  T expected(T(2));
+  T const desired(T(3));
+  std::memory_order const success = std::memory_order_relaxed;
+  // clang-format off
+  a.compare_exchange_weak(expected, desired, success, std::memory_order_relaxed);
+  a.compare_exchange_weak(expected, desired, success, std::memory_order_consume);
+  a.compare_exchange_weak(expected, desired, success, std::memory_order_acquire);
+  a.compare_exchange_weak(expected, desired, success, std::memory_order_seq_cst);
+  a.compare_exchange_weak(expected, desired, success, std::memory_order_release); // expected-warning {{memory order argument to atomic operation is invalid}}
+  a.compare_exchange_weak(expected, desired, success, std::memory_order_acq_rel); // expected-warning {{memory order argument to atomic operation is invalid}}
+  // clang-format on
+}
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/load.verify.cpp b/libcxx/test/libcxx/atomics/atomics.ref/load.verify.cpp
new file mode 100644
index 0000000000000..9fdecb16d9d9b
--- /dev/null
+++ b/libcxx/test/libcxx/atomics/atomics.ref/load.verify.cpp
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: diagnose-if-support
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <atomic>
+
+// T load(memory_order order = memory_order::seq_cst) const noexcept;
+//
+// Preconditions: order is memory_order::relaxed, memory_order::consume, memory_order::acquire, or memory_order::seq_cst.
+
+#include <atomic>
+
+void test() {
+  using T = int;
+
+  T x(T(1));
+  std::atomic_ref const a(x);
+
+  // clang-format off
+  (void)a.load(std::memory_order_relaxed);
+  (void)a.load(std::memory_order_consume);
+  (void)a.load(std::memory_order_acquire);
+  (void)a.load(std::memory_order_seq_cst);
+  (void)a.load(std::memory_order_release); // expected-warning {{memory order argument to atomic operation is invalid}}
+  (void)a.load(std::memory_order_acq_rel); // expected-warning {{memory order argument to atomic operation is invalid}}
+  // clang-format on
+}
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/store.verify.cpp b/libcxx/test/libcxx/atomics/atomics.ref/store.verify.cpp
new file mode 100644
index 0000000000000..0be4e5ebfa2cc
--- /dev/null
+++ b/libcxx/test/libcxx/atomics/atomics.ref/store.verify.cpp
@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: diagnose-if-support
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <atomic>
+
+// void store(T desired, memory_order order = memory_order::seq_cst) const noexcept;
+//
+// Preconditions: order is memory_order::relaxed, memory_order::release, or memory_order::seq_cst.
+
+#include <atomic>
+
+void test() {
+  using T = int;
+
+  T x(T(1));
+  std::atomic_ref const a(x);
+
+  T const desired(T(2));
+
+  // clang-format off
+  a.store(desired, std::memory_order_relaxed);
+  a.store(desired, std::memory_order_release);
+  a.store(desired, std::memory_order_seq_cst);
+  a.store(desired, std::memory_order_consume); // expected-warning {{memory order argument to atomic operation is invalid}}
+  a.store(desired, std::memory_order_acquire); // expected-warning {{memory order argument to atomic operation is invalid}}
+  a.store(desired, std::memory_order_acq_rel); // expected-warning {{memory order argument to atomic operation is invalid}}
+  // clang-format on
+}
diff --git a/libcxx/test/libcxx/atomics/atomics.ref/wait.verify.cpp b/libcxx/test/libcxx/atomics/atomics.ref/wait.verify.cpp
new file mode 100644
index 0000000000000..718e716ebdb32
--- /dev/null
+++ b/libcxx/test/libcxx/atomics/atomics.ref/wait.verify.cpp
@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: diagnose-if-support
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <atomic>
+
+// void wait(T old, memory_order order = memory_order::seq_cst) const noexcept;
+//
+// Preconditions: order is memory_order::relaxed, memory_order::consume, memory_order::acquire, or memory_order::seq_cst.
+
+#include <atomic>
+
+void test() {
+  using T = int;
+
+  T x(T(1));
+  std::atomic_ref const a(x);
+
+  T const old(T(2));
+
+  // clang-format off
+  a.wait(old, std::memory_order_relaxed);
+  a.wait(old, std::memory_order_consume);
+  a.wait(old, std::memory_order_acquire);
+  a.wait(old, std::memory_order_seq_cst);
+  a.wait(old, std::memory_order_release); // expected-warning {{memory order argument to atomic operation is invalid}}
+  a.wait(old, std::memory_order_acq_rel); // expected-warning {{memory order argument to atomic operation is invalid}}
+  // clang-format on
+}
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.search/search.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.search/search.pass.cpp
index b4b841f273c46..3bf2fd9d6bb04 100644
--- a/libcxx/test/std/algorithms/alg.nonmodifying/alg.search/search.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.search/search.pass.cpp
@@ -24,126 +24,113 @@
 #include "test_iterators.h"
 
 struct MySearcherC {
-    template <typename Iterator>
-    std::pair<Iterator, Iterator>
-    TEST_CONSTEXPR operator() (Iterator b, Iterator e) const
-    {
-        return std::make_pair(b, e);
-    }
+  template <typename Iterator>
+  std::pair<Iterator, Iterator> TEST_CONSTEXPR operator()(Iterator b, Iterator e) const {
+    return std::make_pair(b, e);
+  }
 };
 
 #if TEST_STD_VER > 17
 TEST_CONSTEXPR bool test_constexpr() {
-    int ia[] = {0, 1, 2, 3};
-    int ib[] = {0, 1, 5, 3};
-    int ic[] = {0, 1, 2, 0, 1, 2, 3, 0, 1, 2, 3, 4};
-    return    (std::search(std::begin(ic), std::end(ic), std::begin(ia), std::end(ia)) == ic+3)
-           && (std::search(std::begin(ic), std::end(ic), std::begin(ib), std::end(ib)) == std::end(ic))
-           && (std::search(std::begin(ic), std::end(ic), MySearcherC()) == std::begin(ic))
-           ;
-    }
+  int ia[] = {0, 1, 2, 3};
+  int ib[] = {0, 1, 5, 3};
+  int ic[] = {0, 1, 2, 0, 1, 2, 3, 0, 1, 2, 3, 4};
+  return (std::search(std::begin(ic), std::end(ic), std::begin(ia), std::end(ia)) == ic + 3) &&
+         (std::search(std::begin(ic), std::end(ic), std::begin(ib), std::end(ib)) == std::end(ic)) &&
+         (std::search(std::begin(ic), std::end(ic), MySearcherC()) == std::begin(ic));
+}
 #endif
 
 int searcher_called = 0;
 
 struct MySearcher {
-    template <typename Iterator>
-    std::pair<Iterator, Iterator>
-    operator() (Iterator b, Iterator e) const
-    {
-        ++searcher_called;
-        return std::make_pair(b, e);
-    }
+  template <typename Iterator>
+  std::pair<Iterator, Iterator> operator()(Iterator b, Iterator e) const {
+    ++searcher_called;
+    return std::make_pair(b, e);
+  }
 };
 
 namespace User {
 struct S {
-    S(int x) : x_(x) {}
-    int x_;
+  S(int x) : x_(x) {}
+  int x_;
 };
-bool operator==(S lhs, S rhs)
-{
-    return lhs.x_ == rhs.x_;
-}
+bool operator==(S lhs, S rhs) { return lhs.x_ == rhs.x_; }
 template <class T, class U>
 void make_pair(T&&, U&&) = delete;
 } // namespace User
 
 template <class Iter1, class Iter2>
-void
-test()
-{
-    int ia[] = {0, 1, 2, 3, 4, 5};
-    const unsigned sa = sizeof(ia)/sizeof(ia[0]);
-    assert(std::search(Iter1(ia), Iter1(ia+sa), Iter2(ia), Iter2(ia)) == Iter1(ia));
-    assert(std::search(Iter1(ia), Iter1(ia+sa), Iter2(ia), Iter2(ia+1)) == Iter1(ia));
-    assert(std::search(Iter1(ia), Iter1(ia+sa), Iter2(ia+1), Iter2(ia+2)) == Iter1(ia+1));
-    assert(std::search(Iter1(ia), Iter1(ia+sa), Iter2(ia+2), Iter2(ia+2)) == Iter1(ia));
-    assert(std::search(Iter1(ia), Iter1(ia+sa), Iter2(ia+2), Iter2(ia+3)) == Iter1(ia+2));
-    assert(std::search(Iter1(ia), Iter1(ia+sa), Iter2(ia+2), Iter2(ia+3)) == Iter1(ia+2));
-    assert(std::search(Iter1(ia), Iter1(ia), Iter2(ia+2), Iter2(ia+3)) == Iter1(ia));
-    assert(std::search(Iter1(ia), Iter1(ia+sa), Iter2(ia+sa-1), Iter2(ia+sa)) == Iter1(ia+sa-1));
-    assert(std::search(Iter1(ia), Iter1(ia+sa), Iter2(ia+sa-3), Iter2(ia+sa)) == Iter1(ia+sa-3));
-    assert(std::search(Iter1(ia), Iter1(ia+sa), Iter2(ia), Iter2(ia+sa)) == Iter1(ia));
-    assert(std::search(Iter1(ia), Iter1(ia+sa-1), Iter2(ia), Iter2(ia+sa)) == Iter1(ia+sa-1));
-    assert(std::search(Iter1(ia), Iter1(ia+1), Iter2(ia), Iter2(ia+sa)) == Iter1(ia+1));
-    int ib[] = {0, 1, 2, 0, 1, 2, 3, 0, 1, 2, 3, 4};
-    const unsigned sb = sizeof(ib)/sizeof(ib[0]);
-    int ic[] = {1};
-    assert(std::search(Iter1(ib), Iter1(ib+sb), Iter2(ic), Iter2(ic+1)) == Iter1(ib+1));
-    int id[] = {1, 2};
-    assert(std::search(Iter1(ib), Iter1(ib+sb), Iter2(id), Iter2(id+2)) == Iter1(ib+1));
-    int ie[] = {1, 2, 3};
-    assert(std::search(Iter1(ib), Iter1(ib+sb), Iter2(ie), Iter2(ie+3)) == Iter1(ib+4));
-    int ig[] = {1, 2, 3, 4};
-    assert(std::search(Iter1(ib), Iter1(ib+sb), Iter2(ig), Iter2(ig+4)) == Iter1(ib+8));
-    int ih[] = {0, 1, 1, 1, 1, 2, 3, 0, 1, 2, 3, 4};
-    const unsigned sh = sizeof(ih)/sizeof(ih[0]);
-    int ii[] = {1, 1, 2};
-    assert(std::search(Iter1(ih), Iter1(ih+sh), Iter2(ii), Iter2(ii+3)) == Iter1(ih+3));
-    int ij[] = {0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0};
-    const unsigned sj = sizeof(ij)/sizeof(ij[0]);
-    int ik[] = {0, 0, 0, 0, 1, 1, 1, 1, 0, 0};
-    const unsigned sk = sizeof(ik)/sizeof(ik[0]);
-    assert(std::search(Iter1(ij), Iter1(ij+sj), Iter2(ik), Iter2(ik+sk)) == Iter1(ij+6));
+void test() {
+  int ia[]          = {0, 1, 2, 3, 4, 5};
+  const unsigned sa = sizeof(ia) / sizeof(ia[0]);
+  assert(std::search(Iter1(ia), Iter1(ia + sa), Iter2(ia), Iter2(ia)) == Iter1(ia));
+  assert(std::search(Iter1(ia), Iter1(ia + sa), Iter2(ia), Iter2(ia + 1)) == Iter1(ia));
+  assert(std::search(Iter1(ia), Iter1(ia + sa), Iter2(ia + 1), Iter2(ia + 2)) == Iter1(ia + 1));
+  assert(std::search(Iter1(ia), Iter1(ia + sa), Iter2(ia + 2), Iter2(ia + 2)) == Iter1(ia));
+  assert(std::search(Iter1(ia), Iter1(ia + sa), Iter2(ia + 2), Iter2(ia + 3)) == Iter1(ia + 2));
+  assert(std::search(Iter1(ia), Iter1(ia + sa), Iter2(ia + 2), Iter2(ia + 3)) == Iter1(ia + 2));
+  assert(std::search(Iter1(ia), Iter1(ia), Iter2(ia + 2), Iter2(ia + 3)) == Iter1(ia));
+  assert(std::search(Iter1(ia), Iter1(ia + sa), Iter2(ia + sa - 1), Iter2(ia + sa)) == Iter1(ia + sa - 1));
+  assert(std::search(Iter1(ia), Iter1(ia + sa), Iter2(ia + sa - 3), Iter2(ia + sa)) == Iter1(ia + sa - 3));
+  assert(std::search(Iter1(ia), Iter1(ia + sa), Iter2(ia), Iter2(ia + sa)) == Iter1(ia));
+  assert(std::search(Iter1(ia), Iter1(ia + sa - 1), Iter2(ia), Iter2(ia + sa)) == Iter1(ia + sa - 1));
+  assert(std::search(Iter1(ia), Iter1(ia + 1), Iter2(ia), Iter2(ia + sa)) == Iter1(ia + 1));
+  int ib[]          = {0, 1, 2, 0, 1, 2, 3, 0, 1, 2, 3, 4};
+  const unsigned sb = sizeof(ib) / sizeof(ib[0]);
+  int ic[]          = {1};
+  assert(std::search(Iter1(ib), Iter1(ib + sb), Iter2(ic), Iter2(ic + 1)) == Iter1(ib + 1));
+  int id[] = {1, 2};
+  assert(std::search(Iter1(ib), Iter1(ib + sb), Iter2(id), Iter2(id + 2)) == Iter1(ib + 1));
+  int ie[] = {1, 2, 3};
+  assert(std::search(Iter1(ib), Iter1(ib + sb), Iter2(ie), Iter2(ie + 3)) == Iter1(ib + 4));
+  int ig[] = {1, 2, 3, 4};
+  assert(std::search(Iter1(ib), Iter1(ib + sb), Iter2(ig), Iter2(ig + 4)) == Iter1(ib + 8));
+  int ih[]          = {0, 1, 1, 1, 1, 2, 3, 0, 1, 2, 3, 4};
+  const unsigned sh = sizeof(ih) / sizeof(ih[0]);
+  int ii[]          = {1, 1, 2};
+  assert(std::search(Iter1(ih), Iter1(ih + sh), Iter2(ii), Iter2(ii + 3)) == Iter1(ih + 3));
+  int ij[]          = {0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0};
+  const unsigned sj = sizeof(ij) / sizeof(ij[0]);
+  int ik[]          = {0, 0, 0, 0, 1, 1, 1, 1, 0, 0};
+  const unsigned sk = sizeof(ik) / sizeof(ik[0]);
+  assert(std::search(Iter1(ij), Iter1(ij + sj), Iter2(ik), Iter2(ik + sk)) == Iter1(ij + 6));
 }
 
 template <class Iter>
-void
-adl_test()
-{
-    User::S ua[] = {1};
-    assert(std::search(Iter(ua), Iter(ua), Iter(ua), Iter(ua)) == Iter(ua));
+void adl_test() {
+  User::S ua[] = {1};
+  assert(std::search(Iter(ua), Iter(ua), Iter(ua), Iter(ua)) == Iter(ua));
 }
 
-int main(int, char**)
-{
-    test<forward_iterator<const int*>, forward_iterator<const int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*> >();
-
-    adl_test<forward_iterator<User::S*> >();
-    adl_test<random_access_iterator<User::S*> >();
+int main(int, char**) {
+  test<forward_iterator<const int*>, forward_iterator<const int*> >();
+  test<forward_iterator<const int*>, bidirectional_iterator<const int*> >();
+  test<forward_iterator<const int*>, random_access_iterator<const int*> >();
+  test<bidirectional_iterator<const int*>, forward_iterator<const int*> >();
+  test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*> >();
+  test<bidirectional_iterator<const int*>, random_access_iterator<const int*> >();
+  test<random_access_iterator<const int*>, forward_iterator<const int*> >();
+  test<random_access_iterator<const int*>, bidirectional_iterator<const int*> >();
+  test<random_access_iterator<const int*>, random_access_iterator<const int*> >();
+
+  adl_test<forward_iterator<User::S*> >();
+  adl_test<random_access_iterator<User::S*> >();
 
 #if TEST_STD_VER > 14
-{
-    typedef int * RI;
-    static_assert((std::is_same<RI, decltype(std::search(RI(), RI(), MySearcher()))>::value), "" );
+  {
+    typedef int* RI;
+    static_assert((std::is_same<RI, decltype(std::search(RI(), RI(), MySearcher()))>::value), "");
 
     RI it(nullptr);
     assert(it == std::search(it, it, MySearcher()));
     assert(searcher_called == 1);
-}
+  }
 #endif
 
 #if TEST_STD_VER > 17
-    static_assert(test_constexpr());
+  static_assert(test_constexpr());
 #endif
 
   return 0;
diff --git a/libcxx/test/std/re/re.alg/re.alg.match/ecma.pass.cpp b/libcxx/test/std/re/re.alg/re.alg.match/ecma.pass.cpp
index 4face8ba02bbe..a0699634c2bc9 100644
--- a/libcxx/test/std/re/re.alg/re.alg.match/ecma.pass.cpp
+++ b/libcxx/test/std/re/re.alg/re.alg.match/ecma.pass.cpp
@@ -7,6 +7,8 @@
 //===----------------------------------------------------------------------===//
 //
 
+// XFAIL: FROZEN-CXX03-HEADERS-FIXME
+
 // <regex>
 
 // template <class BidirectionalIterator, class Allocator, class charT, class traits>
@@ -669,6 +671,21 @@ int main(int, char**)
         assert(m.position(0) == 0);
         assert(m.str(0) == s);
     }
+    {
+      std::cmatch m;
+      const char s[] = "$_se";
+      assert(std::regex_match(s, m, std::regex("\\$\\_se")));
+      assert(m.size() == 1);
+      assert(!m.prefix().matched);
+      assert(m.prefix().first == s);
+      assert(m.prefix().second == m[0].first);
+      assert(!m.suffix().matched);
+      assert(m.suffix().first == m[0].second);
+      assert(m.suffix().second == s + std::char_traits<char>::length(s));
+      assert(m.length(0) >= 0 && static_cast<size_t>(m.length(0)) == std::char_traits<char>::length(s));
+      assert(m.position(0) == 0);
+      assert(m.str(0) == s);
+    }
 
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
     {
@@ -1305,6 +1322,21 @@ int main(int, char**)
         assert(m.position(0) == 0);
         assert(m.str(0) == s);
     }
+    {
+      std::wcmatch m;
+      const wchar_t s[] = L"$_se";
+      assert(std::regex_match(s, m, std::wregex(L"\\$\\_se")));
+      assert(m.size() == 1);
+      assert(!m.prefix().matched);
+      assert(m.prefix().first == s);
+      assert(m.prefix().second == m[0].first);
+      assert(!m.suffix().matched);
+      assert(m.suffix().first == m[0].second);
+      assert(m.suffix().second == s + std::char_traits<wchar_t>::length(s));
+      assert(m.length(0) >= 0 && static_cast<std::size_t>(m.length(0)) == std::char_traits<wchar_t>::length(s));
+      assert(m.position(0) == 0);
+      assert(m.str(0) == s);
+    }
 #endif // TEST_HAS_NO_WIDE_CHARACTERS
 
   return 0;
diff --git a/libcxx/test/std/thread/futures/futures.async/thread_create_failure.pass.cpp b/libcxx/test/std/thread/futures/futures.async/thread_create_failure.pass.cpp
deleted file mode 100644
index d4f7cb771781c..0000000000000
--- a/libcxx/test/std/thread/futures/futures.async/thread_create_failure.pass.cpp
+++ /dev/null
@@ -1,62 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// UNSUPPORTED: no-threads, no-exceptions
-
-// ASan seems to try to create threadsm which obviouly doesn't work in this test.
-// UNSUPPORTED: asan
-// UNSUPPORTED: hwasan
-
-// UNSUPPORTED: c++03
-
-// There is no way to limit the number of threads on windows
-// UNSUPPORTED: windows
-
-// AIX, FreeBSD, and macOS seem to limit the number of processes, not threads via RLIMIT_NPROC
-// XFAIL: target={{.+}}-aix{{.*}}
-// XFAIL: target={{.+}}-apple-{{.*}}
-// XFAIL: freebsd
-
-// This test makes sure that we fail gracefully in care the thread creation fails. This is only reliably possible on
-// systems that allow limiting the number of threads that can be created. See https://llvm.org/PR125428 for more details
-
-#include <cassert>
-#include <future>
-#include <system_error>
-
-#if __has_include(<sys/resource.h>)
-#  include <sys/resource.h>
-#  ifdef RLIMIT_NPROC
-void force_thread_creation_failure() {
-  rlimit lim = {1, 1};
-  assert(setrlimit(RLIMIT_NPROC, &lim) == 0);
-}
-#  else
-#    error "No known way to force only one thread being available"
-#  endif
-#else
-#  error "No known way to force only one thread being available"
-#endif
-
-int main(int, char**) {
-  force_thread_creation_failure();
-
-  try {
-    std::future<int> fut = std::async(std::launch::async, [] { return 1; });
-    assert(false);
-  } catch (const std::system_error&) {
-  }
-
-  try {
-    std::future<void> fut = std::async(std::launch::async, [] { return; });
-    assert(false);
-  } catch (const std::system_error&) {
-  }
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/common.h b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/common.h
index be9149de89a6c..b51444915a5c5 100644
--- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/common.h
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/common.h
@@ -39,9 +39,52 @@ struct A {
   A& operator=(const A&);
 };
 
-class Abstract
-{
-    virtual ~Abstract() = 0;
+class Abstract {
+  virtual ~Abstract() = 0;
 };
 
+// Types for reference_{constructs/converts}_from_temporary
+
+#if TEST_STD_VER >= 23
+
+class NonPODClass {
+public:
+  NonPODClass(int);
+};
+enum Enum { EV };
+struct Base {
+  Enum e;
+  int i;
+  float f;
+  NonPODClass* p;
+};
+// Not PODs
+struct Derived : Base {};
+
+template <class T, class RefType = T&>
+class ConvertsToRef {
+public:
+  operator RefType() const { return static_cast<RefType>(obj); }
+  mutable T obj = 42;
+};
+template <class T, class RefType = T&>
+class ConvertsToRefPrivate {
+  operator RefType() const { return static_cast<RefType>(obj); }
+  mutable T obj = 42;
+};
+
+class ExplicitConversionRvalueRef {
+public:
+  operator int();
+  explicit operator int&&();
+};
+
+class ExplicitConversionRef {
+public:
+  operator int();
+  explicit operator int&();
+};
+
+#endif
+
 #endif // TEST_META_UNARY_COMP_COMMON_H
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.pass.cpp
index 681ad13a07dfd..24adec37431e7 100644
--- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
 // These compilers don't support __builtin_is_implicit_lifetime yet.
-// UNSUPPORTED: clang-18, clang-19, gcc-14, apple-clang-15, apple-clang-16, apple-clang-17
+// UNSUPPORTED: clang-18, clang-19, gcc-14, apple-clang-15, apple-clang-16
 
 // <type_traits>
 
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.verify.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.verify.cpp
index 34462f9bf0ec6..4bcb10d0b7579 100644
--- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.verify.cpp
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.verify.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
 // These compilers don't support __builtin_is_implicit_lifetime yet.
-// UNSUPPORTED: clang-18, clang-19, gcc-14, apple-clang-15, apple-clang-16, apple-clang-17
+// UNSUPPORTED: clang-18, clang-19, gcc-14, apple-clang-15, apple-clang-16
 
 // <type_traits>
 
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_pod.deprecated.verify.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_pod.deprecated.verify.cpp
new file mode 100644
index 0000000000000..0f9b1856591d8
--- /dev/null
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_pod.deprecated.verify.cpp
@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+
+// <type_traits>
+
+// is_pod and is_pod_v are deprecated in C++20 by P0767R1
+
+#include <type_traits>
+
+static_assert(std::is_pod<int>::value); // expected-warning {{'is_pod<int>' is deprecated}}
+static_assert(std::is_pod_v<int>);      // expected-warning {{'is_pod_v<int>' is deprecated}}
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_pod.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_pod.pass.cpp
index 87fe6ebbee883..887033fc72d86 100644
--- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_pod.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_pod.pass.cpp
@@ -10,6 +10,8 @@
 
 // is_pod
 
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <type_traits>
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/reference_constructs_from_temporary.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/reference_constructs_from_temporary.pass.cpp
new file mode 100644
index 0000000000000..5b3753c67381f
--- /dev/null
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/reference_constructs_from_temporary.pass.cpp
@@ -0,0 +1,90 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++23
+
+// These compilers don't support std::reference_converts_from_temporary yet.
+// UNSUPPORTED: android, apple-clang-15, apple-clang-16, clang-19.1
+
+// <type_traits>
+
+// template<class T, class U> struct reference_constructs_from_temporary;
+
+// template<class T, class U>
+// constexpr bool reference_constructs_from_temporary_v
+//   = reference_constructs_from_temporary<T, U>::value;
+
+#include <cassert>
+#include <type_traits>
+
+#include "common.h"
+#include "test_macros.h"
+
+template <typename T, typename U, bool Expected>
+constexpr void test_reference_constructs_from_temporary() {
+  assert((std::reference_constructs_from_temporary<T, U>::value == Expected));
+  assert((std::reference_constructs_from_temporary_v<T, U> == Expected));
+}
+
+constexpr bool test() {
+  test_reference_constructs_from_temporary<int&, int&, false>();
+  test_reference_constructs_from_temporary<int&, int&, false>();
+  test_reference_constructs_from_temporary<int&, int&&, false>();
+
+  test_reference_constructs_from_temporary<const int&, int&, false>();
+  test_reference_constructs_from_temporary<const int&, const int&, false>();
+  test_reference_constructs_from_temporary<const int&, int&&, false>();
+
+  test_reference_constructs_from_temporary<int&, long&, false>(); // doesn't construct
+
+  test_reference_constructs_from_temporary<const int&, long&, true>();
+  test_reference_constructs_from_temporary<const int&, long&&, true>();
+  test_reference_constructs_from_temporary<int&&, long&, true>();
+
+  assert((std::is_constructible_v<int&, ConvertsToRef<int, int&>>));
+  test_reference_constructs_from_temporary<int&, ConvertsToRef<int, int&>, false>();
+
+  assert((std::is_constructible_v<int&&, ConvertsToRef<int, int&&>>));
+  test_reference_constructs_from_temporary<int&&, ConvertsToRef<int, int&&>, false>();
+
+  assert((std::is_constructible_v<const int&, ConvertsToRef<int, const int&>>));
+  test_reference_constructs_from_temporary<int&&, ConvertsToRef<int, const int&>, false>();
+
+  assert((std::is_constructible_v<const int&, ConvertsToRef<long, long&>>));
+  test_reference_constructs_from_temporary<const int&, ConvertsToRef<long, long&>, true>();
+#ifndef TEST_COMPILER_GCC
+  test_reference_constructs_from_temporary<const int&, ConvertsToRefPrivate<long, long&>, false>();
+#endif
+
+  // Test that it doesn't accept non-reference types as input.
+  test_reference_constructs_from_temporary<int, long, false>();
+
+  test_reference_constructs_from_temporary<const int&, long, true>();
+
+  // Additional checks
+  test_reference_constructs_from_temporary<const Base&, Derived, true>();
+  test_reference_constructs_from_temporary<int&&, int, true>();
+  test_reference_constructs_from_temporary<const int&, int, true>();
+  test_reference_constructs_from_temporary<int&&, int&&, false>();
+  test_reference_constructs_from_temporary<const int&, int&&, false>();
+  test_reference_constructs_from_temporary<int&&, long&&, true>();
+  test_reference_constructs_from_temporary<int&&, long, true>();
+
+  test_reference_constructs_from_temporary<int&, ExplicitConversionRef, false>();
+  test_reference_constructs_from_temporary<const int&, ExplicitConversionRef, false>();
+  test_reference_constructs_from_temporary<int&&, ExplicitConversionRvalueRef, false>();
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/reference_converts_from_temporary.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/reference_converts_from_temporary.pass.cpp
new file mode 100644
index 0000000000000..849e286c8cdab
--- /dev/null
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/reference_converts_from_temporary.pass.cpp
@@ -0,0 +1,90 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++23
+
+// These compilers don't support std::reference_converts_from_temporary yet.
+// UNSUPPORTED: android, apple-clang-15, apple-clang-16, clang-18, clang-19.1
+
+// <type_traits>
+
+// template<class T, class U> struct reference_converts_from_temporary;
+
+// template<class T, class U>
+// constexpr bool reference_converts_from_temporary_v
+//   = reference_converts_from_temporary<T, U>::value;
+
+#include <cassert>
+#include <type_traits>
+
+#include "common.h"
+#include "test_macros.h"
+
+template <typename T, typename U, bool Expected>
+constexpr void test_reference_converts_from_temporary() {
+  assert((std::reference_converts_from_temporary<T, U>::value == Expected));
+  assert((std::reference_converts_from_temporary_v<T, U> == Expected));
+}
+
+constexpr bool test() {
+  test_reference_converts_from_temporary<int&, int&, false>();
+  test_reference_converts_from_temporary<int&, int&, false>();
+  test_reference_converts_from_temporary<int&, int&&, false>();
+
+  test_reference_converts_from_temporary<const int&, int&, false>();
+  test_reference_converts_from_temporary<const int&, const int&, false>();
+  test_reference_converts_from_temporary<const int&, int&&, false>();
+
+  test_reference_converts_from_temporary<int&, long&, false>(); // doesn't construct
+
+  test_reference_converts_from_temporary<const int&, long&, true>();
+  test_reference_converts_from_temporary<const int&, long&&, true>();
+  test_reference_converts_from_temporary<int&&, long&, true>();
+
+  assert((std::is_constructible_v<int&, ConvertsToRef<int, int&>>));
+  test_reference_converts_from_temporary<int&, ConvertsToRef<int, int&>, false>();
+
+  assert((std::is_constructible_v<int&&, ConvertsToRef<int, int&&>>));
+  test_reference_converts_from_temporary<int&&, ConvertsToRef<int, int&&>, false>();
+
+  assert((std::is_constructible_v<const int&, ConvertsToRef<int, const int&>>));
+  test_reference_converts_from_temporary<int&&, ConvertsToRef<int, const int&>, false>();
+
+  assert((std::is_constructible_v<const int&, ConvertsToRef<long, long&>>));
+  test_reference_converts_from_temporary<const int&, ConvertsToRef<long, long&>, true>();
+#ifndef TEST_COMPILER_GCC
+  test_reference_converts_from_temporary<const int&, ConvertsToRefPrivate<long, long&>, false>();
+#endif
+
+  // Test that it doesn't accept non-reference types as input.
+  test_reference_converts_from_temporary<int, long, false>();
+
+  test_reference_converts_from_temporary<const int&, long, true>();
+
+  // Additional checks
+  test_reference_converts_from_temporary<const Base&, Derived, true>();
+  test_reference_converts_from_temporary<int&&, int, true>();
+  test_reference_converts_from_temporary<const int&, int, true>();
+  test_reference_converts_from_temporary<int&&, int&&, false>();
+  test_reference_converts_from_temporary<const int&, int&&, false>();
+  test_reference_converts_from_temporary<int&&, long&&, true>();
+  test_reference_converts_from_temporary<int&&, long, true>();
+
+  test_reference_converts_from_temporary<int&, ExplicitConversionRef, false>();
+  test_reference_converts_from_temporary<const int&, ExplicitConversionRef, true>();
+  test_reference_converts_from_temporary<int&&, ExplicitConversionRvalueRef, true>();
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}
diff --git a/libcxx/utils/ci/docker-compose.yml b/libcxx/utils/ci/docker-compose.yml
index 16db1b0e3acb3..68e89db5cf452 100644
--- a/libcxx/utils/ci/docker-compose.yml
+++ b/libcxx/utils/ci/docker-compose.yml
@@ -10,7 +10,7 @@ services:
       dockerfile: Dockerfile
       target: actions-builder
       args:
-        BASE_IMAGE: ghcr.io/actions/actions-runner:2.319.1
+        BASE_IMAGE: ghcr.io/actions/actions-runner:2.322.0
         <<: *compiler_versions
 
   android-buildkite-builder:
diff --git a/libunwind/src/Registers.hpp b/libunwind/src/Registers.hpp
index 452f46a0d56ea..2c3bfb7e8428a 100644
--- a/libunwind/src/Registers.hpp
+++ b/libunwind/src/Registers.hpp
@@ -4126,7 +4126,7 @@ inline reg_t Registers_riscv::getRegister(int regNum) const {
     return _registers[regNum];
   if (regNum == UNW_RISCV_VLENB) {
     reg_t vlenb;
-    __asm__("csrr %0, 0xC22" : "=r"(vlenb));
+    __asm__ volatile("csrr %0, 0xC22" : "=r"(vlenb));
     return vlenb;
   }
   _LIBUNWIND_ABORT("unsupported riscv register");
diff --git a/lld/COFF/Chunks.h b/lld/COFF/Chunks.h
index 06e9aae0e6f6e..9f68d5a325cc5 100644
--- a/lld/COFF/Chunks.h
+++ b/lld/COFF/Chunks.h
@@ -59,7 +59,8 @@ class Chunk {
     SectionKind,
     SectionECKind,
     OtherKind,
-    ImportThunkKind
+    ImportThunkKind,
+    ECExportThunkKind
   };
   Kind kind() const { return chunkKind; }
 
@@ -827,7 +828,10 @@ static const uint8_t ECExportThunkCode[] = {
 
 class ECExportThunkChunk : public NonSectionCodeChunk {
 public:
-  explicit ECExportThunkChunk(Defined *targetSym) : target(targetSym) {}
+  explicit ECExportThunkChunk(Defined *targetSym)
+      : NonSectionCodeChunk(ECExportThunkKind), target(targetSym) {}
+  static bool classof(const Chunk *c) { return c->kind() == ECExportThunkKind; }
+
   size_t getSize() const override { return sizeof(ECExportThunkCode); };
   void writeTo(uint8_t *buf) const override;
   MachineTypes getMachine() const override { return AMD64; }
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index c20c049017018..bb64a5c0b9e96 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -786,6 +786,13 @@ void LinkerDriver::addWinSysRootLibSearchPaths() {
                                       path))
       searchPaths.push_back(saver().save(path));
   }
+
+  // Libraries specified by `/nodefaultlib:` may not be found in incomplete
+  // search paths before lld infers a machine type from input files.
+  std::set<std::string> noDefaultLibs;
+  for (const std::string &path : ctx.config.noDefaultLibs)
+    noDefaultLibs.insert(findLib(path).lower());
+  ctx.config.noDefaultLibs = noDefaultLibs;
 }
 
 // Parses LIB environment which contains a list of search paths.
diff --git a/lld/COFF/Options.td b/lld/COFF/Options.td
index b6fd3d0daaef9..4e401a5fd1d6d 100644
--- a/lld/COFF/Options.td
+++ b/lld/COFF/Options.td
@@ -342,6 +342,7 @@ def fastfail : F<"fastfail">;
 def kernel : F<"kernel">;
 def pdbcompress : F<"pdbcompress">;
 def emitpogophaseinfo : F<"emitpogophaseinfo">;
+def noexp : F<"noexp">;
 
 def delay : P_priv<"delay">;
 def errorreport : P_priv<"errorreport">;
diff --git a/lld/COFF/Writer.cpp b/lld/COFF/Writer.cpp
index 8474c9826a31d..a8148446a2657 100644
--- a/lld/COFF/Writer.cpp
+++ b/lld/COFF/Writer.cpp
@@ -1986,10 +1986,17 @@ static void maybeAddAddressTakenFunction(SymbolRVASet &addressTakenSyms,
     // Common is always data, so it is ignored.
     break;
   case Symbol::DefinedAbsoluteKind:
-  case Symbol::DefinedSyntheticKind:
     // Absolute is never code, synthetic generally isn't and usually isn't
     // determinable.
     break;
+  case Symbol::DefinedSyntheticKind:
+    // For EC export thunks, mark both the thunk itself and its target.
+    if (auto expChunk = dyn_cast_or_null<ECExportThunkChunk>(
+            cast<Defined>(s)->getChunk())) {
+      addSymbolToRVASet(addressTakenSyms, cast<Defined>(s));
+      addSymbolToRVASet(addressTakenSyms, expChunk->target);
+    }
+    break;
   case Symbol::LazyArchiveKind:
   case Symbol::LazyObjectKind:
   case Symbol::LazyDLLSymbolKind:
@@ -2054,9 +2061,11 @@ void Writer::createGuardCFTables() {
     // with /guard:cf.
     for (ObjFile *file : ctx.objFileInstances) {
       if (file->hasGuardCF()) {
-        Symbol *flagSym = ctx.symtab.findUnderscore("__guard_flags");
-        cast<DefinedAbsolute>(flagSym)->setVA(
-            uint32_t(GuardFlags::CF_INSTRUMENTED));
+        ctx.forEachSymtab([&](SymbolTable &symtab) {
+          Symbol *flagSym = symtab.findUnderscore("__guard_flags");
+          cast<DefinedAbsolute>(flagSym)->setVA(
+              uint32_t(GuardFlags::CF_INSTRUMENTED));
+        });
         break;
       }
     }
@@ -2138,8 +2147,10 @@ void Writer::createGuardCFTables() {
     guardFlags |= uint32_t(GuardFlags::CF_LONGJUMP_TABLE_PRESENT);
   if (config->guardCF & GuardCFLevel::EHCont)
     guardFlags |= uint32_t(GuardFlags::EH_CONTINUATION_TABLE_PRESENT);
-  Symbol *flagSym = ctx.symtab.findUnderscore("__guard_flags");
-  cast<DefinedAbsolute>(flagSym)->setVA(guardFlags);
+  ctx.forEachSymtab([guardFlags](SymbolTable &symtab) {
+    Symbol *flagSym = symtab.findUnderscore("__guard_flags");
+    cast<DefinedAbsolute>(flagSym)->setVA(guardFlags);
+  });
 }
 
 // Take a list of input sections containing symbol table indices and add those
@@ -2210,10 +2221,12 @@ void Writer::maybeAddRVATable(SymbolRVASet tableSymbols, StringRef tableSym,
     tableChunk = make<RVATableChunk>(std::move(tableSymbols));
   rdataSec->addChunk(tableChunk);
 
-  Symbol *t = ctx.symtab.findUnderscore(tableSym);
-  Symbol *c = ctx.symtab.findUnderscore(countSym);
-  replaceSymbol<DefinedSynthetic>(t, t->getName(), tableChunk);
-  cast<DefinedAbsolute>(c)->setVA(tableChunk->getSize() / (hasFlag ? 5 : 4));
+  ctx.forEachSymtab([&](SymbolTable &symtab) {
+    Symbol *t = symtab.findUnderscore(tableSym);
+    Symbol *c = symtab.findUnderscore(countSym);
+    replaceSymbol<DefinedSynthetic>(t, t->getName(), tableChunk);
+    cast<DefinedAbsolute>(c)->setVA(tableChunk->getSize() / (hasFlag ? 5 : 4));
+  });
 }
 
 // Create CHPE metadata chunks.
diff --git a/lld/ELF/Arch/LoongArch.cpp b/lld/ELF/Arch/LoongArch.cpp
index dbf024eadf100..709b31ed4e01a 100644
--- a/lld/ELF/Arch/LoongArch.cpp
+++ b/lld/ELF/Arch/LoongArch.cpp
@@ -58,6 +58,8 @@ enum Op {
   LD_W = 0x28800000,
   LD_D = 0x28c00000,
   JIRL = 0x4c000000,
+  B = 0x50000000,
+  BL = 0x54000000,
 };
 
 enum Reg {
@@ -152,6 +154,10 @@ static uint32_t setJ20(uint32_t insn, uint32_t imm) {
   return (insn & 0xfe00001f) | (extractBits(imm, 19, 0) << 5);
 }
 
+static uint32_t setJ5(uint32_t insn, uint32_t imm) {
+  return (insn & 0xfffffc1f) | (extractBits(imm, 4, 0) << 5);
+}
+
 static uint32_t setK12(uint32_t insn, uint32_t imm) {
   return (insn & 0xffc003ff) | (extractBits(imm, 11, 0) << 10);
 }
@@ -759,10 +765,10 @@ static bool isPairRelaxable(ArrayRef<Relocation> relocs, size_t i) {
 
 // Relax code sequence.
 // From:
-//   pcalau12i $a0, %pc_hi20(sym)
-//   addi.w/d $a0, $a0, %pc_lo12(sym)
+//   pcalau12i     $a0, %pc_hi20(sym) | %ld_pc_hi20(sym)  | %gd_pc_hi20(sym)
+//   addi.w/d $a0, $a0, %pc_lo12(sym) | %got_pc_lo12(sym) | %got_pc_lo12(sym)
 // To:
-//   pcaddi $a0, %pc_lo12(sym)
+//   pcaddi $a0, %pc_lo12(sym) | %got_pc_lo12(sym) | %got_pc_lo12(sym)
 //
 // From:
 //   pcalau12i $a0, %got_pc_hi20(sym_got)
@@ -776,6 +782,10 @@ static void relaxPCHi20Lo12(Ctx &ctx, const InputSection &sec, size_t i,
   if (!((rHi20.type == R_LARCH_PCALA_HI20 &&
          rLo12.type == R_LARCH_PCALA_LO12) ||
         (rHi20.type == R_LARCH_GOT_PC_HI20 &&
+         rLo12.type == R_LARCH_GOT_PC_LO12) ||
+        (rHi20.type == R_LARCH_TLS_GD_PC_HI20 &&
+         rLo12.type == R_LARCH_GOT_PC_LO12) ||
+        (rHi20.type == R_LARCH_TLS_LD_PC_HI20 &&
          rLo12.type == R_LARCH_GOT_PC_LO12)))
     return;
 
@@ -796,6 +806,8 @@ static void relaxPCHi20Lo12(Ctx &ctx, const InputSection &sec, size_t i,
   else if (rHi20.expr == RE_LOONGARCH_PAGE_PC ||
            rHi20.expr == RE_LOONGARCH_GOT_PAGE_PC)
     dest = rHi20.sym->getVA(ctx);
+  else if (rHi20.expr == RE_LOONGARCH_TLSGD_PAGE_PC)
+    dest = ctx.in.got->getGlobalDynAddr(*rHi20.sym);
   else {
     Err(ctx) << getErrorLoc(ctx, (const uint8_t *)loc) << "unknown expr ("
              << rHi20.expr << ") against symbol " << rHi20.sym
@@ -825,11 +837,74 @@ static void relaxPCHi20Lo12(Ctx &ctx, const InputSection &sec, size_t i,
     return;
 
   sec.relaxAux->relocTypes[i] = R_LARCH_RELAX;
-  sec.relaxAux->relocTypes[i + 2] = R_LARCH_PCREL20_S2;
+  if (rHi20.type == R_LARCH_TLS_GD_PC_HI20)
+    sec.relaxAux->relocTypes[i + 2] = R_LARCH_TLS_GD_PCREL20_S2;
+  else if (rHi20.type == R_LARCH_TLS_LD_PC_HI20)
+    sec.relaxAux->relocTypes[i + 2] = R_LARCH_TLS_LD_PCREL20_S2;
+  else
+    sec.relaxAux->relocTypes[i + 2] = R_LARCH_PCREL20_S2;
   sec.relaxAux->writes.push_back(insn(PCADDI, getD5(nextInsn), 0, 0));
   remove = 4;
 }
 
+// Relax code sequence.
+// From:
+//   pcaddu18i $ra, %call36(foo)
+//   jirl $ra, $ra, 0
+// To:
+//   b/bl foo
+static void relaxCall36(Ctx &ctx, const InputSection &sec, size_t i,
+                        uint64_t loc, Relocation &r, uint32_t &remove) {
+  const uint64_t dest =
+      (r.expr == R_PLT_PC ? r.sym->getPltVA(ctx) : r.sym->getVA(ctx)) +
+      r.addend;
+
+  const int64_t displace = dest - loc;
+  // Check if the displace aligns 4 bytes or exceeds the range of b[l].
+  if ((displace & 0x3) != 0 || !isInt<28>(displace))
+    return;
+
+  const uint32_t nextInsn = read32le(sec.content().data() + r.offset + 4);
+  if (getD5(nextInsn) == R_RA) {
+    // convert jirl to bl
+    sec.relaxAux->relocTypes[i] = R_LARCH_B26;
+    sec.relaxAux->writes.push_back(insn(BL, 0, 0, 0));
+    remove = 4;
+  } else if (getD5(nextInsn) == R_ZERO) {
+    // convert jirl to b
+    sec.relaxAux->relocTypes[i] = R_LARCH_B26;
+    sec.relaxAux->writes.push_back(insn(B, 0, 0, 0));
+    remove = 4;
+  }
+}
+
+// Relax code sequence.
+// From:
+//   lu12i.w $rd, %le_hi20_r(sym)
+//   add.w/d $rd, $rd, $tp, %le_add_r(sym)
+//   addi/ld/st.w/d $rd, $rd, %le_lo12_r(sym)
+// To:
+//   addi/ld/st.w/d $rd, $tp, %le_lo12_r(sym)
+static void relaxTlsLe(Ctx &ctx, const InputSection &sec, size_t i,
+                       uint64_t loc, Relocation &r, uint32_t &remove) {
+  uint64_t val = r.sym->getVA(ctx, r.addend);
+  // Check if the val exceeds the range of addi/ld/st.
+  if (!isInt<12>(val))
+    return;
+  uint32_t currInsn = read32le(sec.content().data() + r.offset);
+  switch (r.type) {
+  case R_LARCH_TLS_LE_HI20_R:
+  case R_LARCH_TLS_LE_ADD_R:
+    sec.relaxAux->relocTypes[i] = R_LARCH_RELAX;
+    remove = 4;
+    break;
+  case R_LARCH_TLS_LE_LO12_R:
+    sec.relaxAux->writes.push_back(setJ5(currInsn, R_TP));
+    sec.relaxAux->relocTypes[i] = R_LARCH_TLS_LE_LO12_R;
+    break;
+  }
+}
+
 static bool relax(Ctx &ctx, InputSection &sec) {
   const uint64_t secAddr = sec.getVA();
   const MutableArrayRef<Relocation> relocs = sec.relocs();
@@ -870,10 +945,22 @@ static bool relax(Ctx &ctx, InputSection &sec) {
     }
     case R_LARCH_PCALA_HI20:
     case R_LARCH_GOT_PC_HI20:
+    case R_LARCH_TLS_GD_PC_HI20:
+    case R_LARCH_TLS_LD_PC_HI20:
       // The overflow check for i+2 will be carried out in isPairRelaxable.
       if (isPairRelaxable(relocs, i))
         relaxPCHi20Lo12(ctx, sec, i, loc, r, relocs[i + 2], remove);
       break;
+    case R_LARCH_CALL36:
+      if (relaxable(relocs, i))
+        relaxCall36(ctx, sec, i, loc, r, remove);
+      break;
+    case R_LARCH_TLS_LE_HI20_R:
+    case R_LARCH_TLS_LE_ADD_R:
+    case R_LARCH_TLS_LE_LO12_R:
+      if (relaxable(relocs, i))
+        relaxTlsLe(ctx, sec, i, loc, r, remove);
+      break;
     }
 
     // For all anchors whose offsets are <= r.offset, they are preceded by
@@ -977,6 +1064,23 @@ void LoongArch::finalizeRelax(int passes) const {
             // RelExpr is needed for relocating.
             r.expr = r.sym->hasFlag(NEEDS_PLT) ? R_PLT_PC : R_PC;
             break;
+          case R_LARCH_B26:
+          case R_LARCH_TLS_LE_LO12_R:
+            skip = 4;
+            write32le(p, aux.writes[writesIdx++]);
+            break;
+          case R_LARCH_TLS_GD_PCREL20_S2:
+            // Note: R_LARCH_TLS_LD_PCREL20_S2 must also use R_TLSGD_PC instead
+            // of R_TLSLD_PC due to historical reasons. In fact, right now TLSLD
+            // behaves exactly like TLSGD on LoongArch.
+            //
+            // This reason has also been mentioned in mold commit:
+            // https://github.com/rui314/mold/commit/5dfa1cf07c03bd57cb3d493b652ef22441bcd71c
+          case R_LARCH_TLS_LD_PCREL20_S2:
+            skip = 4;
+            write32le(p, aux.writes[writesIdx++]);
+            r.expr = R_TLSGD_PC;
+            break;
           default:
             llvm_unreachable("unsupported type");
           }
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index b5872b85efd3a..d14faca31d58f 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -136,6 +136,9 @@ enum LtoKind : uint8_t {UnifiedThin, UnifiedRegular, Default};
 // For -z gcs=
 enum class GcsPolicy { Implicit, Never, Always };
 
+// For some options that resemble -z bti-report={none,warning,error}
+enum class ReportPolicy { None, Warning, Error };
+
 struct SymbolVersion {
   llvm::StringRef name;
   bool isExternCpp;
@@ -225,11 +228,11 @@ struct Config {
   llvm::StringRef whyExtract;
   llvm::StringRef cmseInputLib;
   llvm::StringRef cmseOutputLib;
-  StringRef zBtiReport = "none";
-  StringRef zCetReport = "none";
-  StringRef zPauthReport = "none";
-  StringRef zGcsReport = "none";
-  StringRef zExecuteOnlyReport = "none";
+  ReportPolicy zBtiReport = ReportPolicy::None;
+  ReportPolicy zCetReport = ReportPolicy::None;
+  ReportPolicy zPauthReport = ReportPolicy::None;
+  ReportPolicy zGcsReport = ReportPolicy::None;
+  ReportPolicy zExecuteOnlyReport = ReportPolicy::None;
   bool ltoBBAddrMap;
   llvm::StringRef ltoBasicBlockSections;
   std::pair<llvm::StringRef, llvm::StringRef> thinLTOObjectSuffixReplace;
@@ -748,6 +751,14 @@ ELFSyncStream InternalErr(Ctx &ctx, const uint8_t *buf);
 
 #define CHECK2(E, S) lld::check2((E), [&] { return toStr(ctx, S); })
 
+inline DiagLevel toDiagLevel(ReportPolicy policy) {
+  if (policy == ReportPolicy::Error)
+    return DiagLevel::Err;
+  else if (policy == ReportPolicy::Warning)
+    return DiagLevel::Warn;
+  return DiagLevel::None;
+}
+
 } // namespace lld::elf
 
 #endif
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index c5de522daa177..18f9fed0d08e2 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -396,18 +396,18 @@ static void checkOptions(Ctx &ctx) {
       ErrAlways(ctx) << "-z pac-plt only supported on AArch64";
     if (ctx.arg.zForceBti)
       ErrAlways(ctx) << "-z force-bti only supported on AArch64";
-    if (ctx.arg.zBtiReport != "none")
+    if (ctx.arg.zBtiReport != ReportPolicy::None)
       ErrAlways(ctx) << "-z bti-report only supported on AArch64";
-    if (ctx.arg.zPauthReport != "none")
+    if (ctx.arg.zPauthReport != ReportPolicy::None)
       ErrAlways(ctx) << "-z pauth-report only supported on AArch64";
-    if (ctx.arg.zGcsReport != "none")
+    if (ctx.arg.zGcsReport != ReportPolicy::None)
       ErrAlways(ctx) << "-z gcs-report only supported on AArch64";
     if (ctx.arg.zGcs != GcsPolicy::Implicit)
       ErrAlways(ctx) << "-z gcs only supported on AArch64";
   }
 
   if (ctx.arg.emachine != EM_AARCH64 && ctx.arg.emachine != EM_ARM &&
-      ctx.arg.zExecuteOnlyReport != "none")
+      ctx.arg.zExecuteOnlyReport != ReportPolicy::None)
     ErrAlways(ctx)
         << "-z execute-only-report only supported on AArch64 and ARM";
 
@@ -423,7 +423,7 @@ static void checkOptions(Ctx &ctx) {
     ErrAlways(ctx) << "--relax-gp is only supported on RISC-V targets";
 
   if (ctx.arg.emachine != EM_386 && ctx.arg.emachine != EM_X86_64 &&
-      ctx.arg.zCetReport != "none")
+      ctx.arg.zCetReport != ReportPolicy::None)
     ErrAlways(ctx) << "-z cet-report only supported on X86 and X86_64";
 
   if (ctx.arg.pie && ctx.arg.shared)
@@ -1272,11 +1272,6 @@ static void parseClangOption(Ctx &ctx, StringRef opt, const Twine &msg) {
   ErrAlways(ctx) << msg << ": " << StringRef(err).trim();
 }
 
-// Checks the parameter of the bti-report and cet-report options.
-static bool isValidReportString(StringRef arg) {
-  return arg == "none" || arg == "warning" || arg == "error";
-}
-
 // Process a remap pattern 'from-glob=to-file'.
 static bool remapInputs(Ctx &ctx, StringRef line, const Twine &location) {
   SmallVector<StringRef, 0> fields;
@@ -1638,12 +1633,17 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
       if (option.first != reportArg.first)
         continue;
       arg->claim();
-      if (!isValidReportString(option.second)) {
+      if (option.second == "none")
+        *reportArg.second = ReportPolicy::None;
+      else if (option.second == "warning")
+        *reportArg.second = ReportPolicy::Warning;
+      else if (option.second == "error")
+        *reportArg.second = ReportPolicy::Error;
+      else {
         ErrAlways(ctx) << "unknown -z " << reportArg.first
                        << "= value: " << option.second;
         continue;
       }
-      *reportArg.second = option.second;
     }
   }
 
@@ -2820,17 +2820,13 @@ static void readSecurityNotes(Ctx &ctx) {
   bool hasValidPauthAbiCoreInfo = llvm::any_of(
       ctx.aarch64PauthAbiCoreInfo, [](uint8_t c) { return c != 0; });
 
-  auto report = [&](StringRef config) -> ELFSyncStream {
-    if (config == "error")
-      return {ctx, DiagLevel::Err};
-    else if (config == "warning")
-      return {ctx, DiagLevel::Warn};
-    return {ctx, DiagLevel::None};
+  auto report = [&](ReportPolicy policy) -> ELFSyncStream {
+    return {ctx, toDiagLevel(policy)};
   };
-  auto reportUnless = [&](StringRef config, bool cond) -> ELFSyncStream {
+  auto reportUnless = [&](ReportPolicy policy, bool cond) -> ELFSyncStream {
     if (cond)
       return {ctx, DiagLevel::None};
-    return report(config);
+    return {ctx, toDiagLevel(policy)};
   };
   for (ELFFileBase *f : ctx.objectFiles) {
     uint32_t features = f->andFeatures;
@@ -2860,13 +2856,13 @@ static void readSecurityNotes(Ctx &ctx) {
 
     if (ctx.arg.zForceBti && !(features & GNU_PROPERTY_AARCH64_FEATURE_1_BTI)) {
       features |= GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
-      if (ctx.arg.zBtiReport == "none")
+      if (ctx.arg.zBtiReport == ReportPolicy::None)
         Warn(ctx) << f
                   << ": -z force-bti: file does not have "
                      "GNU_PROPERTY_AARCH64_FEATURE_1_BTI property";
     } else if (ctx.arg.zForceIbt &&
                !(features & GNU_PROPERTY_X86_FEATURE_1_IBT)) {
-      if (ctx.arg.zCetReport == "none")
+      if (ctx.arg.zCetReport == ReportPolicy::None)
         Warn(ctx) << f
                   << ": -z force-ibt: file does not have "
                      "GNU_PROPERTY_X86_FEATURE_1_IBT property";
diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp
index 120f5271cf229..0d5b383f4b596 100644
--- a/lld/ELF/LinkerScript.cpp
+++ b/lld/ELF/LinkerScript.cpp
@@ -563,7 +563,7 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd,
           continue;
 
         if (!cmd->matchesFile(*sec->file) || pat.excludesFile(*sec->file) ||
-            sec->parent == &outCmd || !flagsMatch(sec))
+            !flagsMatch(sec))
           continue;
 
         if (sec->parent) {
@@ -626,7 +626,7 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd,
 
     for (InputSectionDescription *isd : scd->sc.commands) {
       for (InputSectionBase *sec : isd->sectionBases) {
-        if (sec->parent == &outCmd || !flagsMatch(sec))
+        if (!flagsMatch(sec))
           continue;
         bool isSpill = sec->parent && isa<OutputSection>(sec->parent);
         if (!sec->parent || (isSpill && outCmd.name == "/DISCARD/")) {
@@ -1559,6 +1559,8 @@ bool LinkerScript::spillSections() {
   if (potentialSpillLists.empty())
     return false;
 
+  DenseSet<PotentialSpillSection *> skippedSpills;
+
   bool spilled = false;
   for (SectionCommand *cmd : reverse(sectionCommands)) {
     auto *osd = dyn_cast<OutputDesc>(cmd);
@@ -1585,16 +1587,34 @@ bool LinkerScript::spillSections() {
         if (isa<PotentialSpillSection>(isec))
           continue;
 
-        // Find the next potential spill location and remove it from the list.
         auto it = potentialSpillLists.find(isec);
         if (it == potentialSpillLists.end())
-          continue;
+          break;
+
+        // Consume spills until finding one that might help, then consume it.
+        auto canSpillHelp = [&](PotentialSpillSection *spill) {
+          // Spills to the same region that overflowed cannot help.
+          if (hasRegionOverflowed(osec->memRegion) &&
+              spill->getParent()->memRegion == osec->memRegion)
+            return false;
+          if (hasRegionOverflowed(osec->lmaRegion) &&
+              spill->getParent()->lmaRegion == osec->lmaRegion)
+            return false;
+          return true;
+        };
         PotentialSpillList &list = it->second;
-        PotentialSpillSection *spill = list.head;
-        if (spill->next)
-          list.head = spill->next;
-        else
-          potentialSpillLists.erase(isec);
+        PotentialSpillSection *spill;
+        for (spill = list.head; spill; spill = spill->next) {
+          if (list.head->next)
+            list.head = spill->next;
+          else
+            potentialSpillLists.erase(isec);
+          if (canSpillHelp(spill))
+            break;
+          skippedSpills.insert(spill);
+        }
+        if (!spill)
+          continue;
 
         // Replace the next spill location with the spilled section and adjust
         // its properties to match the new location. Note that the alignment of
@@ -1629,6 +1649,15 @@ bool LinkerScript::spillSections() {
     }
   }
 
+  // Clean up any skipped spills.
+  DenseSet<InputSectionDescription *> isds;
+  for (PotentialSpillSection *s : skippedSpills)
+    isds.insert(s->isd);
+  for (InputSectionDescription *isd : isds)
+    llvm::erase_if(isd->sections, [&](InputSection *s) {
+      return skippedSpills.contains(dyn_cast<PotentialSpillSection>(s));
+    });
+
   return spilled;
 }
 
diff --git a/lld/ELF/ScriptParser.cpp b/lld/ELF/ScriptParser.cpp
index 671a5ec4deccb..4c52bfda7a70e 100644
--- a/lld/ELF/ScriptParser.cpp
+++ b/lld/ELF/ScriptParser.cpp
@@ -988,20 +988,8 @@ OutputDesc *ScriptParser::readOverlaySectionDescription() {
       ctx.script->createOutputSection(readName(), getCurrentLocation());
   osd->osec.inOverlay = true;
   expect("{");
-  while (auto tok = till("}")) {
-    uint64_t withFlags = 0;
-    uint64_t withoutFlags = 0;
-    if (tok == "INPUT_SECTION_FLAGS") {
-      std::tie(withFlags, withoutFlags) = readInputSectionFlags();
-      tok = till("");
-    }
-    if (tok == "CLASS")
-      osd->osec.commands.push_back(make<InputSectionDescription>(
-          StringRef{}, withFlags, withoutFlags, readSectionClassName()));
-    else
-      osd->osec.commands.push_back(
-          readInputSectionRules(tok, withFlags, withoutFlags));
-  }
+  while (auto tok = till("}"))
+    osd->osec.commands.push_back(readInputSectionDescription(tok));
   osd->osec.phdrs = readOutputSectionPhdrs();
   return osd;
 }
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index f249bb198e98d..2cea6a44b391a 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -2181,17 +2181,13 @@ template <class ELFT> void Writer<ELFT>::checkExecuteOnly() {
 // Check which input sections of RX output sections don't have the
 // SHF_AARCH64_PURECODE or SHF_ARM_PURECODE flag set.
 template <class ELFT> void Writer<ELFT>::checkExecuteOnlyReport() {
-  if (ctx.arg.zExecuteOnlyReport == "none")
+  if (ctx.arg.zExecuteOnlyReport == ReportPolicy::None)
     return;
 
   auto reportUnless = [&](bool cond) -> ELFSyncStream {
     if (cond)
       return {ctx, DiagLevel::None};
-    if (ctx.arg.zExecuteOnlyReport == "error")
-      return {ctx, DiagLevel::Err};
-    if (ctx.arg.zExecuteOnlyReport == "warning")
-      return {ctx, DiagLevel::Warn};
-    return {ctx, DiagLevel::None};
+    return {ctx, toDiagLevel(ctx.arg.zExecuteOnlyReport)};
   };
 
   uint64_t purecodeFlag =
diff --git a/lld/test/COFF/Inputs/defaultlib.yaml b/lld/test/COFF/Inputs/defaultlib.yaml
new file mode 100644
index 0000000000000..5b61095d65822
--- /dev/null
+++ b/lld/test/COFF/Inputs/defaultlib.yaml
@@ -0,0 +1,24 @@
+--- !COFF
+header:
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: [  ]
+sections:
+  - Name:            .drectve
+    Characteristics: [ IMAGE_SCN_LNK_INFO, IMAGE_SCN_LNK_REMOVE ]
+    Alignment:       1
+    SectionData:     202F44454641554C544C49423A64656661756C742E6C6962
+    SizeOfRawData:   24
+symbols:
+  - Name:            .drectve
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition:
+      Length:          24
+      NumberOfRelocations: 0
+      NumberOfLinenumbers: 0
+      CheckSum:        2526009418
+      Number:          1
+...
diff --git a/lld/test/COFF/arm64x-guardcf.s b/lld/test/COFF/arm64x-guardcf.s
new file mode 100644
index 0000000000000..750bf0b3862c5
--- /dev/null
+++ b/lld/test/COFF/arm64x-guardcf.s
@@ -0,0 +1,122 @@
+// REQUIRES: aarch64, x86
+// RUN: split-file %s %t.dir && cd %t.dir
+
+// RUN: llvm-mc -filetype=obj -triple=aarch64-windows func-gfids.s -o func-gfids-arm64.obj
+// RUN: llvm-mc -filetype=obj -triple=arm64ec-windows func-gfids.s -o func-gfids-arm64ec.obj
+// RUN: llvm-mc -filetype=obj -triple=aarch64-windows func-exp.s -o func-exp-arm64.obj
+// RUN: llvm-mc -filetype=obj -triple=arm64ec-windows func-exp.s -o func-exp-arm64ec.obj
+// RUN: llvm-mc -filetype=obj -triple=aarch64-windows dllmain.s -o dllmain-arm64.obj
+// RUN: llvm-mc -filetype=obj -triple=arm64ec-windows dllmain.s -o dllmain-arm64ec.obj
+// RUN: llvm-mc -filetype=obj -triple=x86_64-windows func-amd64.s -o func-amd64.obj
+// RUN: llvm-mc -filetype=obj -triple=arm64ec-windows %S/Inputs/loadconfig-arm64ec.s -o loadconfig-arm64ec.obj
+// RUN: llvm-mc -filetype=obj -triple=aarch64-windows %S/Inputs/loadconfig-arm64.s -o loadconfig-arm64.obj
+
+
+// Check that CF guard tables contain both native and EC symbols and are referenced from both load configs.
+
+// RUN: lld-link -dll -noentry -machine:arm64x func-gfids-arm64.obj func-gfids-arm64ec.obj func-amd64.obj -guard:cf -out:out.dll \
+// RUN:          loadconfig-arm64ec.obj loadconfig-arm64.obj
+// RUN: llvm-readobj --coff-load-config out.dll | FileCheck --check-prefix=LOADCFG %s
+
+// LOADCFG:      LoadConfig [
+// LOADCFG:        GuardCFFunctionCount: 3
+// LOADCFG-NEXT:   GuardFlags [ (0x10500)
+// LOADCFG-NEXT:     CF_FUNCTION_TABLE_PRESENT (0x400)
+// LOADCFG-NEXT:     CF_INSTRUMENTED (0x100)
+// LOADCFG-NEXT:     CF_LONGJUMP_TABLE_PRESENT (0x10000)
+// LOADCFG-NEXT:   ]
+// LOADCFG:      ]
+// LOADCFG:      GuardFidTable [
+// LOADCFG-NEXT:   0x180001000
+// LOADCFG-NEXT:   0x180002000
+// LOADCFG-NEXT:   0x180003000
+// LOADCFG-NEXT: ]
+// LOADCFG:      HybridObject {
+// LOADCFG:        LoadConfig [
+// LOADCFG:          GuardCFFunctionCount: 3
+// LOADCFG-NEXT:     GuardFlags [ (0x10500)
+// LOADCFG-NEXT:       CF_FUNCTION_TABLE_PRESENT (0x400)
+// LOADCFG-NEXT:       CF_INSTRUMENTED (0x100)
+// LOADCFG-NEXT:       CF_LONGJUMP_TABLE_PRESENT (0x10000)
+// LOADCFG-NEXT:     ]
+// LOADCFG:        ]
+// LOADCFG:        GuardFidTable [
+// LOADCFG-NEXT:     0x180001000
+// LOADCFG-NEXT:     0x180002000
+// LOADCFG-NEXT:     0x180003000
+// LOADCFG-NEXT:   ]
+// LOADCFG:      ]
+
+
+// Check that exports from both views are present in CF guard tables.
+
+// RUN: lld-link -dll -noentry -machine:arm64x func-exp-arm64.obj func-exp-arm64ec.obj -guard:cf -out:out-exp.dll \
+// RUN:          loadconfig-arm64ec.obj loadconfig-arm64.obj
+// RUN: llvm-readobj --coff-load-config out-exp.dll | FileCheck --check-prefix=LOADCFG %s
+
+
+// Check that entry points from both views are present in CF guard tables.
+
+// RUN: lld-link -dll -machine:arm64x dllmain-arm64.obj dllmain-arm64ec.obj -guard:cf -out:out-entry.dll \
+// RUN:          loadconfig-arm64ec.obj loadconfig-arm64.obj
+// RUN: llvm-readobj --coff-load-config out-entry.dll | FileCheck --check-prefix=LOADCFG %s
+
+
+// Check that both load configs are marked as instrumented if any input object was built with /guard:cf.
+
+// RUN: lld-link -dll -noentry -machine:arm64x func-gfids-arm64ec.obj -out:out-nocfg.dll \
+// RUN:          loadconfig-arm64ec.obj loadconfig-arm64.obj
+
+// RUN: llvm-readobj --coff-load-config out-nocfg.dll | FileCheck --check-prefix=LOADCFG-INST %s
+
+// LOADCFG-INST:      LoadConfig [
+// LOADCFG-INST:        GuardFlags [ (0x100)
+// LOADCFG-INST-NEXT:     CF_INSTRUMENTED (0x100)
+// LOADCFG-INST-NEXT:   ]
+// LOADCFG-INST:      ]
+// LOADCFG-INST:      HybridObject {
+// LOADCFG-INST:        LoadConfig [
+// LOADCFG-INST:          GuardFlags [ (0x100)
+// LOADCFG-INST-NEXT:       CF_INSTRUMENTED (0x100)
+// LOADCFG-INST-NEXT:     ]
+// LOADCFG-INST:        ]
+// LOADCFG-INST:      ]
+
+#--- func-gfids.s
+        .def @feat.00; .scl 3; .type 0; .endef
+        .globl @feat.00
+@feat.00 = 0x800
+
+        .globl func
+func:
+        ret
+
+        .section .gfids$y,"dr"
+        .symidx func
+
+#--- func-amd64.s
+        .def @feat.00; .scl 3; .type 0; .endef
+        .globl @feat.00
+@feat.00 = 0x800
+
+        .globl func_amd64
+func_amd64:
+        ret
+
+        .section .gfids$y,"dr"
+        .symidx func_amd64
+
+#--- func-exp.s
+        .def func; .scl 2; .type 32; .endef
+        .globl func
+func:
+        ret
+
+        .section .drectve
+        .ascii "-export:func"
+
+#--- dllmain.s
+        .def _DllMainCRTStartup; .scl 2; .type 32; .endef
+        .globl _DllMainCRTStartup
+_DllMainCRTStartup:
+        ret
diff --git a/lld/test/COFF/build-id-sym.s b/lld/test/COFF/build-id-sym.s
index faed7bc40fd06..004a012cba0be 100644
--- a/lld/test/COFF/build-id-sym.s
+++ b/lld/test/COFF/build-id-sym.s
@@ -1,6 +1,6 @@
 # REQUIRES: x86
 # RUN: llvm-mc -triple=x86_64-windows-msvc -filetype=obj -o %t.obj %s
-# RUN: lld-link -debug:symtab -entry:main %t.obj -build-id -Brepro -out:%t.exe
+# RUN: lld-link -debug:symtab -entry:main %t.obj -build-id -Brepro -out:%t.exe -guard:cf
 # RUN: llvm-objdump -s -t %t.exe | FileCheck %s
 
 # Check __buildid points to 0x14000203c which is after the signature RSDS.
@@ -21,3 +21,6 @@ main:
 .section .bss,"bw",discard,__buildid
 .global __buildid
 __buildid:
+
+.data
+.quad __buildid
diff --git a/lld/test/COFF/nodefaultlib.test b/lld/test/COFF/nodefaultlib.test
index ca8f09bdfc76c..ceeb1f393b141 100644
--- a/lld/test/COFF/nodefaultlib.test
+++ b/lld/test/COFF/nodefaultlib.test
@@ -36,3 +36,12 @@ MSVC stamps uppercase references in OBJ directives, thus ensure that passing low
 
 UPPERCASE-NOT: OLDNAMES
 UPPERCASE-NOT: LIBCMT
+
+# RUN: yaml2obj -o %T/defaultlib.obj %p/Inputs/defaultlib.yaml
+# RUN: mkdir -p %t.dir/sysroot/VC/Tools/MSVC/1.1.1.1/lib/x64
+# RUN: cp %p/Inputs/ret42.lib %t.dir/sysroot/VC/Tools/MSVC/1.1.1.1/lib/x64/default.lib
+
+# RUN: lld-link /winsysroot:%t.dir/sysroot /out:%t.exe /entry:main /subsystem:console %T/defaultlib.obj
+# RUN: not lld-link /winsysroot:%t.dir/sysroot /out:%t.exe /entry:main /subsystem:console /nodefaultlib:default.lib %T/defaultlib.obj 2>&1 | FileCheck -check-prefix=CHECK4 %s
+
+CHECK4: error: <root>: undefined symbol: main
diff --git a/lld/test/ELF/linkerscript/overlay-keep.test b/lld/test/ELF/linkerscript/overlay-keep.test
new file mode 100644
index 0000000000000..1b163523eead5
--- /dev/null
+++ b/lld/test/ELF/linkerscript/overlay-keep.test
@@ -0,0 +1,51 @@
+# REQUIRES: x86
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 a.s -o a.o
+# RUN: ld.lld a.o --gc-sections --script nokeep.t -o a
+# RUN: llvm-objdump --section-headers a | FileCheck --check-prefix=NOKEEP %s
+# RUN: ld.lld a.o --gc-sections --script keep.t -o a
+# RUN: llvm-objdump --section-headers a | FileCheck --check-prefix=KEEP %s
+
+# NOKEEP:      Sections:
+# NOKEEP-NEXT: Idx Name          Size
+# NOKEEP-NEXT:   0               00000000
+# NOKEEP-NEXT:   1 .text         00000004
+# NOKEEP-NEXT:   2 .keep1        00000000
+
+# KEEP:      Sections:
+# KEEP-NEXT: Idx Name          Size
+# KEEP-NEXT:   0               00000000
+# KEEP-NEXT:   1 .text         00000004
+# KEEP-NEXT:   2 .keep1        00000004
+# KEEP-NEXT:   3 .keep2        00000004
+
+#--- a.s
+.global _start
+_start:
+ .long 1
+
+.section .keep1, "a"
+keep1:
+ .long 2
+
+.section .keep2, "a"
+keep2:
+ .long 3
+
+#--- nokeep.t
+SECTIONS {
+  .text : { *(.text) }
+  OVERLAY 0x1000 : AT ( 0x4000 ) {
+    .keep1 { *(.keep1) }
+    .keep2 { *(.keep2) }
+  }
+}
+
+#--- keep.t
+SECTIONS {
+  .text : { *(.text) }
+  OVERLAY 0x1000 : AT ( 0x4000 ) {
+    .keep1 { KEEP(*(.keep1)) }
+    .keep2 { KEEP(*(.keep2)) }
+  }
+}
diff --git a/lld/test/ELF/linkerscript/section-class.test b/lld/test/ELF/linkerscript/section-class.test
index 5c30b6343653e..27273f3855c32 100644
--- a/lld/test/ELF/linkerscript/section-class.test
+++ b/lld/test/ELF/linkerscript/section-class.test
@@ -450,3 +450,103 @@ SECTIONS {
 
 # TO-DISCARD: error: section '.two_byte_section' cannot spill from/to /DISCARD/
 # TO-DISCARD-WARN: warning: section '.two_byte_section' cannot spill from/to /DISCARD/
+
+#--- same-mem-region.lds
+## Spills to the same memory region that overflowed do not consume address assignment passes.
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 0
+  b : ORIGIN = 0, LENGTH = 3
+  c : ORIGIN = 3, LENGTH = 3
+  d : ORIGIN = 6, LENGTH = 3
+}
+SECTIONS {
+  CLASS(class) { *(.one_byte_section .two_byte_section) }
+  .a00 : { CLASS(class) } >a AT>c
+  .a01 : { CLASS(class) } >a AT>d
+  .a02 : { CLASS(class) } >a AT>d
+  .a03 : { CLASS(class) } >a AT>d
+  .a04 : { CLASS(class) } >a AT>d
+  .a05 : { CLASS(class) } >a AT>d
+  .a06 : { CLASS(class) } >a AT>d
+  .a07 : { CLASS(class) } >a AT>d
+  .a08 : { CLASS(class) } >a AT>d
+  .a09 : { CLASS(class) } >a AT>d
+  .a10 : { CLASS(class) } >a AT>d
+  .a11 : { CLASS(class) } >a AT>d
+  .a12 : { CLASS(class) } >a AT>d
+  .a13 : { CLASS(class) } >a AT>d
+  .a14 : { CLASS(class) } >a AT>d
+  .a15 : { CLASS(class) } >a AT>d
+  .a16 : { CLASS(class) } >a AT>d
+  .a17 : { CLASS(class) } >a AT>d
+  .a18 : { CLASS(class) } >a AT>d
+  .a19 : { CLASS(class) } >a AT>d
+  .a20 : { CLASS(class) } >a AT>d
+  .a21 : { CLASS(class) } >a AT>d
+  .a22 : { CLASS(class) } >a AT>d
+  .a23 : { CLASS(class) } >a AT>d
+  .a24 : { CLASS(class) } >a AT>d
+  .a25 : { CLASS(class) } >a AT>d
+  .a26 : { CLASS(class) } >a AT>d
+  .a27 : { CLASS(class) } >a AT>d
+  .a28 : { CLASS(class) } >a AT>d
+  .a29 : { CLASS(class) } >a AT>d
+  .a30 : { CLASS(class) } >a AT>d
+  .b : { CLASS(class) } >b AT>d
+}
+
+# RUN: ld.lld -T same-mem-region.lds -o same-mem-region spill.o
+# RUN: llvm-readelf -S same-mem-region | FileCheck %s --check-prefix=SAME-MEM-REGION
+
+# SAME-MEM-REGION:      Name          Type     Address          Off    Size
+# SAME-MEM-REGION:      .b PROGBITS 0000000000000000 001000 000003
+
+#--- same-lma-region.lds
+## Spills to the same load region that overflowed do not consume address assignment passes.
+MEMORY {
+  a : ORIGIN = 0, LENGTH = 0
+  b : ORIGIN = 0, LENGTH = 3
+  c : ORIGIN = 3, LENGTH = 3
+  d : ORIGIN = 6, LENGTH = 3
+}
+SECTIONS {
+  CLASS(class) { *(.one_byte_section .two_byte_section) }
+  .a00 : { CLASS(class) } >c AT>a
+  .a01 : { CLASS(class) } >d AT>a
+  .a02 : { CLASS(class) } >d AT>a
+  .a03 : { CLASS(class) } >d AT>a
+  .a04 : { CLASS(class) } >d AT>a
+  .a05 : { CLASS(class) } >d AT>a
+  .a06 : { CLASS(class) } >d AT>a
+  .a07 : { CLASS(class) } >d AT>a
+  .a08 : { CLASS(class) } >d AT>a
+  .a09 : { CLASS(class) } >d AT>a
+  .a10 : { CLASS(class) } >d AT>a
+  .a11 : { CLASS(class) } >d AT>a
+  .a12 : { CLASS(class) } >d AT>a
+  .a13 : { CLASS(class) } >d AT>a
+  .a14 : { CLASS(class) } >d AT>a
+  .a15 : { CLASS(class) } >d AT>a
+  .a16 : { CLASS(class) } >d AT>a
+  .a17 : { CLASS(class) } >d AT>a
+  .a18 : { CLASS(class) } >d AT>a
+  .a19 : { CLASS(class) } >d AT>a
+  .a20 : { CLASS(class) } >d AT>a
+  .a21 : { CLASS(class) } >d AT>a
+  .a22 : { CLASS(class) } >d AT>a
+  .a23 : { CLASS(class) } >d AT>a
+  .a24 : { CLASS(class) } >d AT>a
+  .a25 : { CLASS(class) } >d AT>a
+  .a26 : { CLASS(class) } >d AT>a
+  .a27 : { CLASS(class) } >d AT>a
+  .a28 : { CLASS(class) } >d AT>a
+  .a29 : { CLASS(class) } >d AT>a
+  .a30 : { CLASS(class) } >d AT>a
+  .b : { CLASS(class) } >d AT>b
+}
+
+# RUN: ld.lld -T same-lma-region.lds -o same-lma-region spill.o
+# RUN: llvm-readelf -S same-lma-region | FileCheck %s --check-prefix=SAME-LMA-REGION
+
+# SAME-LMA-REGION:      Name          Type     Address          Off    Size
+# SAME-LMA-REGION:      .b PROGBITS 0000000000000006 001006 000003
diff --git a/lld/test/ELF/loongarch-relax-call36-2.s b/lld/test/ELF/loongarch-relax-call36-2.s
new file mode 100644
index 0000000000000..383913dbe6277
--- /dev/null
+++ b/lld/test/ELF/loongarch-relax-call36-2.s
@@ -0,0 +1,65 @@
+# REQUIRES: loongarch
+## Relax R_LARCH_CALL36. This test tests boundary cases and some special symbols.
+
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=loongarch64 -mattr=+relax a.s -o a.o
+
+# RUN: ld.lld -T lds a.o -o a
+# RUN: llvm-objdump -d --no-show-raw-insn a | FileCheck %s --check-prefixes=RELAX,RELAX-MID
+
+## Unsure whether this needs a diagnostic. GNU ld allows this.
+# RUN: ld.lld -T lds -pie a.o -o a.pie
+# RUN: llvm-objdump -d --no-show-raw-insn a.pie | FileCheck %s --check-prefixes=RELAX,RELAX-MID
+
+# RUN: ld.lld -T lds -pie -z notext -z ifunc-noplt a.o -o a.ifunc-noplt
+# RUN: llvm-objdump -d --no-show-raw-insn a.ifunc-noplt | FileCheck %s --check-prefixes=RELAX,NORELAX-MID
+
+# RELAX-LABEL:  <_start>:
+## offset = 0x10000000 - 0x8000000 = 0x8000000(134217728), hi=512, lo18=0
+# RELAX-NEXT:    8000000:  pcaddu18i $ra, 512
+# RELAX-NEXT:              jirl   $ra, $ra, 0
+# RELAX-NEXT:              bl     134217720
+# RELAX-NEXT:              bl     -134217728
+## offset = 12 - 0x8000010 = -0x8000004(-134217732), hi=512, lo18=-4
+# RELAX-NEXT:    8000010:  pcaddu18i $ra, -512
+# RELAX-NEXT:              jirl   $ra, $ra, -4
+# RELAX-EMPTY:
+
+# RELAX-MID-LABEL:  <.mid>:
+## offset = 0x8010000 - 0x8008000 = 32768
+# RELAX-MID-NEXT:    8008000:  bl     32768
+# RELAX-MID-NEXT:              b      32764
+# RELAX-MID-EMPTY:
+
+# NORELAX-MID-LABEL: <.mid>:
+# NORELAX-MID-NEXT:  8008000:  pcaddu18i $ra, 0
+# NORELAX-MID-NEXT:            jirl   $ra, $ra, 0
+# NORELAX-MID-NEXT:            pcaddu18i $t0, 0
+# NORELAX-MID-NEXT:            jr     $t0
+# NORELAX-MID-EMPTY:
+
+#--- a.s
+.global _start, ifunc
+_start:
+  call36 pos       # exceed positive range (.text+0x7fffffc), not relaxed
+  call36 pos       # relaxed
+  call36 neg       # relaxed
+  call36 neg       # exceed negative range (.text+16-0x8000000), not relaxed
+
+.section .mid,"ax",@progbits
+.balign 16
+  call36 ifunc      # enable ifunc, not relaxed
+  tail36 $t0, ifunc # enable ifunc, not relaxed
+
+.type ifunc, @gnu_indirect_function
+ifunc:
+  ret
+
+#--- lds
+SECTIONS {
+  .text 0x8000000 : { *(.text) }
+  .mid  0x8008000 : { *(.mid) }
+  .iplt 0x8010000 : { *(.iplt) }
+}
+neg = 12;
+pos = 0x10000000;
diff --git a/lld/test/ELF/loongarch-relax-call36.s b/lld/test/ELF/loongarch-relax-call36.s
new file mode 100644
index 0000000000000..fa6e79dfa5803
--- /dev/null
+++ b/lld/test/ELF/loongarch-relax-call36.s
@@ -0,0 +1,136 @@
+# REQUIRES: loongarch
+## Relax R_LARCH_CALL36, which involves the macro instructions call36/tail36.
+
+# RUN: rm -rf %t && split-file %s %t && cd %t
+
+# RUN: llvm-mc -filetype=obj -triple=loongarch64 -mattr=+relax a.s -o a.64.o
+# RUN: llvm-mc -filetype=obj -triple=loongarch64 -mattr=+relax b.s -o b.64.o
+# RUN: ld.lld -shared -soname=b.so b.64.o -o b.64.so
+# RUN: ld.lld -T lds a.64.o b.64.so -o 64
+# RUN: llvm-objdump -td --no-show-raw-insn 64 | FileCheck %s --check-prefix=RELAX
+
+## --no-relax disables relaxation.
+# RUN: ld.lld -T lds a.64.o b.64.so --no-relax -o 64.norelax
+# RUN: llvm-objdump -td --no-show-raw-insn 64.norelax | FileCheck %s --check-prefix=NORELAX
+
+# RELAX:       {{0*}}00010000 g       .text  {{0*}}0000001c _start
+# RELAX:       {{0*}}0001001c g       .text  {{0*}}00000000 _start_end
+# RELAX:       {{0*}}00010808 g       .mid   {{0*}}00000000 mid_end
+# RELAX:       {{0*}}10010010 g       .high  {{0*}}00000000 high_end
+
+# RELAX-LABEL: <_start>:
+## offset = 0x10018 - 0x10000 = 24
+# RELAX-NEXT:      10000:  bl     24 <a>
+# RELAX-NEXT:              b      20 <a>
+# RELAX-NEXT:              nop
+# RELAX-NEXT:              nop
+## offset = .plt(0x10400)+32 - 0x10010 = 1040
+# RELAX-NEXT:      10010:  bl     1040 <bar+0x10420>
+# RELAX-NEXT:              b      1036 <bar+0x10420>
+# RELAX-EMPTY:
+# RELAX-NEXT: <a>:
+# RELAX-NEXT:      10018:  ret
+# RELAX-EMPTY:
+
+# RELAX-LABEL: <.mid>:
+## offset = 0x10000 - 0x10800 = -2048
+# RELAX-NEXT:      10800:  bl     -2048 <_start>
+# RELAX-NEXT:              b      -2052 <_start>
+# RELAX-EMPTY:
+
+# RELAX-LABEL: <.mid2>:
+## offset = 0x10000 - 0x1010000 = -16777216
+# RELAX-NEXT:    1010000:  bl     -16777216 <_start>
+# RELAX-NEXT:              b      -16777220 <_start>
+# RELAX-EMPTY:
+
+# RELAX-LABEL: <.high>:
+## offset = 0x10000 - 0x10010000 = -0x10000000, hi=-1024, lo18=0
+# RELAX-NEXT:   10010000:  pcaddu18i $ra, -1024
+# RELAX-NEXT:              jirl   $ra, $ra, 0
+# RELAX-NEXT:              pcaddu18i $t0, -1024
+# RELAX-NEXT:              jirl   $zero, $t0, -8
+# RELAX-EMPTY:
+
+
+# NORELAX-LABEL: <_start>:
+## offset = 0x10020 - 0x10000 = 0x20, hi=0, lo18=32
+# NORELAX-NEXT:    10000:  pcaddu18i $ra, 0
+# NORELAX-NEXT:            jirl   $ra, $ra, 32
+## offset = 0x10020 - 0x10008 = 0x18, hi=0, lo18=24
+# NORELAX-NEXT:    10008:  pcaddu18i $t0, 0
+# NORELAX-NEXT:            jirl   $zero, $t0, 24
+## offset = .plt(0x10400)+32 - 0x10010 = 0x410, hi=0, lo18=1040
+# NORELAX-NEXT:    10010:  pcaddu18i $ra, 0
+# NORELAX-NEXT:            jirl   $ra, $ra, 1040
+## offset = .plt(0x10400)+32 - 0x10018 = 0x408, hi=0, lo18=1032
+# NORELAX-NEXT:    10018:  pcaddu18i $t0, 0
+# NORELAX-NEXT:            jirl   $zero, $t0, 1032
+# NORELAX-EMPTY:
+# NORELAX-NEXT: <a>:
+# NORELAX-NEXT:      10020:  ret
+# NORELAX-EMPTY:
+
+# NORELAX-LABEL: <.mid>:
+## offset = 0x10000 - 0x10800 = -0x800, hi=0, lo18=-2048
+# NORELAX-NEXT:    10800:  pcaddu18i $ra, 0
+# NORELAX-NEXT:            jirl   $ra, $ra, -2048
+# NORELAX-NEXT:            pcaddu18i $t0, 0
+# NORELAX-NEXT:            jirl   $zero, $t0, -2056
+# NORELAX-EMPTY:
+
+# NORELAX-LABEL: <.mid2>:
+## offset = 0x10000 - 0x1010000 = -0x1000000, hi=-64, lo18=0
+# NORELAX-NEXT:  1010000:  pcaddu18i $ra, -64
+# NORELAX-NEXT:            jirl   $ra, $ra, 0
+# NORELAX-NEXT:            pcaddu18i $t0, -64
+# NORELAX-NEXT:            jirl   $zero, $t0, -8
+# NORELAX-EMPTY:
+
+# NORELAX-LABEL: <.high>:
+## offset = 0x10000 - 0x10010000 = -0x10000000, hi=-1024, lo18=0
+# NORELAX-NEXT: 10010000:  pcaddu18i $ra, -1024
+# NORELAX-NEXT:            jirl   $ra, $ra, 0
+# NORELAX-NEXT:            pcaddu18i $t0, -1024
+# NORELAX-NEXT:            jirl   $zero, $t0, -8
+# NORELAX-EMPTY:
+
+#--- a.s
+.global _start, _start_end
+_start:
+  call36 a          # relaxed. la64: bl
+  tail36 $t0, a     # relaxed. la64: b
+.balign 16
+  call36 bar        # PLT call36 can be relaxed. la64: bl
+  tail36 $t0, bar   # PLT tail36 can be relaxed. la64: bl
+
+a:
+  ret
+.size _start, . - _start
+_start_end:
+
+.section .mid,"ax",@progbits
+  call36 _start         # relaxed. la64: bl
+  tail36 $t0, _start    # relaxed. la64: b
+
+.section .mid2,"ax",@progbits
+  call36 _start         # relaxed. la64: bl
+  tail36 $t0, _start    # relaxed. la64: b
+
+.section .high,"ax",@progbits
+  call36 _start         # exceed range, not relaxed
+  tail36 $t0, _start    # exceed range, not relaxed
+
+#--- b.s
+.globl bar
+bar:
+  ret
+
+#--- lds
+SECTIONS {
+  .text 0x10000 : { *(.text) }
+  .plt 0x10400 : { *(.plt) }
+  .mid 0x10800 : { *(.mid); mid_end = .; }
+  .mid2 0x1010000 : { *(.mid2) }
+  .high 0x10010000 : { *(.high); high_end = .; }
+}
diff --git a/lld/test/ELF/loongarch-relax-emit-relocs.s b/lld/test/ELF/loongarch-relax-emit-relocs.s
index 7031ffd9f6686..909b65075a695 100644
--- a/lld/test/ELF/loongarch-relax-emit-relocs.s
+++ b/lld/test/ELF/loongarch-relax-emit-relocs.s
@@ -2,11 +2,11 @@
 ## Test that we can handle --emit-relocs while relaxing.
 
 # RUN: llvm-mc --filetype=obj --triple=loongarch32 --mattr=+relax %s -o %t.32.o
-# RUN: llvm-mc --filetype=obj --triple=loongarch64 --mattr=+relax %s -o %t.64.o
+# RUN: llvm-mc --filetype=obj --triple=loongarch64 --mattr=+relax --defsym ELF64=1 %s -o %t.64.o
 # RUN: ld.lld -Ttext=0x10000 -section-start=.got=0x20000 --emit-relocs %t.32.o -o %t.32
 # RUN: ld.lld -Ttext=0x10000 -section-start=.got=0x20000 --emit-relocs %t.64.o -o %t.64
-# RUN: llvm-objdump -dr %t.32 | FileCheck %s --check-prefix=RELAX
-# RUN: llvm-objdump -dr %t.64 | FileCheck %s --check-prefix=RELAX
+# RUN: llvm-objdump -dr %t.32 | FileCheck %s --check-prefixes=RELAX,RELAX32
+# RUN: llvm-objdump -dr %t.64 | FileCheck %s --check-prefixes=RELAX,RELAX64
 
 ## -r should keep original relocations.
 # RUN: ld.lld -r %t.64.o -o %t.64.r
@@ -27,10 +27,45 @@
 # RELAX-NEXT:     R_LARCH_RELAX *ABS*
 # RELAX-NEXT:     R_LARCH_PCREL20_S2 _start
 # RELAX-NEXT:     R_LARCH_RELAX *ABS*
-# RELAX-NEXT:   nop
-# RELAX-NEXT:     R_LARCH_ALIGN *ABS*+0xc
-# RELAX-NEXT:   nop
-# RELAX-NEXT:   ret
+
+# RELAX64-NEXT:  bl  -8
+# RELAX64-NEXT:    R_LARCH_B26 _start
+# RELAX64-NEXT:    R_LARCH_RELAX *ABS*
+# RELAX64-NEXT:  b   -12
+# RELAX64-NEXT:    R_LARCH_B26 _start
+# RELAX64-NEXT:    R_LARCH_RELAX *ABS*
+
+# RELAX-NEXT:   lu12i.w   $a0, 0
+# RELAX-NEXT:     R_LARCH_TLS_LE_HI20 a
+# RELAX-NEXT:   ori       $a0, $a0, 0
+# RELAX-NEXT:     R_LARCH_TLS_LE_LO12 a
+# RELAX-NEXT:   pcaddi    $a0, [[#]]
+# RELAX-NEXT:     R_LARCH_RELAX a
+# RELAX-NEXT:     R_LARCH_RELAX *ABS*
+# RELAX-NEXT:     R_LARCH_TLS_GD_PCREL20_S2 a
+# RELAX-NEXT:     R_LARCH_RELAX *ABS*
+# RELAX-NEXT:   pcaddi    $a0, [[#]]
+# RELAX-NEXT:     R_LARCH_RELAX a
+# RELAX-NEXT:     R_LARCH_RELAX *ABS*
+# RELAX-NEXT:     R_LARCH_TLS_LD_PCREL20_S2 a
+# RELAX-NEXT:     R_LARCH_RELAX *ABS*
+# RELAX-NEXT:   addi.{{[dw]}} $a0, $tp, 0
+# RELAX-NEXT:     R_LARCH_RELAX a
+# RELAX-NEXT:     R_LARCH_RELAX *ABS*
+# RELAX-NEXT:     R_LARCH_RELAX a
+# RELAX-NEXT:     R_LARCH_RELAX *ABS*
+# RELAX-NEXT:     R_LARCH_TLS_LE_LO12_R a
+# RELAX-NEXT:     R_LARCH_RELAX *ABS*
+
+# RELAX32-NEXT:  nop
+# RELAX32-NEXT:    R_LARCH_ALIGN *ABS*+0xc
+# RELAX32-NEXT:  ret
+
+# RELAX64-NEXT:  nop
+# RELAX64-NEXT:    R_LARCH_ALIGN *ABS*+0xc
+# RELAX64-NEXT:  nop
+# RELAX64-NEXT:  nop
+# RELAX64-NEXT:  ret
 
 # NORELAX:      <_start>:
 # NORELAX-NEXT:   pcalau12i $a0, 0
@@ -45,8 +80,44 @@
 # NORELAX-NEXT:   ld.d      $a0, $a0, 0
 # NORELAX-NEXT:     R_LARCH_GOT_PC_LO12 _start
 # NORELAX-NEXT:     R_LARCH_RELAX *ABS*
-# NORELAX-NEXT:   ret
+# NORELAX-NEXT:   pcaddu18i $ra, 0
+# NORELAX-NEXT:     R_LARCH_CALL36 _start
+# NORELAX-NEXT:     R_LARCH_RELAX *ABS*
+# NORELAX-NEXT:   jirl   $ra, $ra, -16
+# NORELAX-NEXT:   pcaddu18i $a0, 0
+# NORELAX-NEXT:     R_LARCH_CALL36 _start
+# NORELAX-NEXT:     R_LARCH_RELAX *ABS*
+# NORELAX-NEXT:   jirl $zero, $a0, -24
+# NORELAX-NEXT:   lu12i.w   $a0, 0
+# NORELAX-NEXT:     R_LARCH_TLS_LE_HI20 a
+# NORELAX-NEXT:   ori       $a0, $a0, 0
+# NORELAX-NEXT:     R_LARCH_TLS_LE_LO12 a
+# NORELAX-NEXT:   pcalau12i $a0, 16
+# NORELAX-NEXT:     R_LARCH_TLS_GD_PC_HI20 a
+# NORELAX-NEXT:     R_LARCH_RELAX *ABS*
+# NORELAX-NEXT:   addi.d    $a0, $a0, 8
+# NORELAX-NEXT:     R_LARCH_GOT_PC_LO12 a
+# NORELAX-NEXT:     R_LARCH_RELAX *ABS*
+# NORELAX-NEXT:   pcalau12i $a0, 16
+# NORELAX-NEXT:     R_LARCH_TLS_LD_PC_HI20 a
+# NORELAX-NEXT:     R_LARCH_RELAX *ABS*
+# NORELAX-NEXT:   addi.d    $a0, $a0, 8
+# NORELAX-NEXT:     R_LARCH_GOT_PC_LO12 a
+# NORELAX-NEXT:     R_LARCH_RELAX *ABS*
+# NORELAX-NEXT:   lu12i.w   $a0, 0
+# NORELAX-NEXT:     R_LARCH_TLS_LE_HI20_R a
+# NORELAX-NEXT:     R_LARCH_RELAX *ABS*
+# NORELAX-NEXT:   add.d     $a0, $a0, $tp
+# NORELAX-NEXT:     R_LARCH_TLS_LE_ADD_R a
+# NORELAX-NEXT:     R_LARCH_RELAX *ABS*
+# NORELAX-NEXT:   addi.d    $a0, $a0, 0
+# NORELAX-NEXT:     R_LARCH_TLS_LE_LO12_R a
+# NORELAX-NEXT:     R_LARCH_RELAX *ABS*
+# NORELAX-NEXT:   nop
 # NORELAX-NEXT:     R_LARCH_ALIGN *ABS*+0xc
+# NORELAX-NEXT:   nop
+# NORELAX-NEXT:   nop
+# NORELAX-NEXT:   ret
 
 # CHECKR:      <_start>:
 # CHECKR-NEXT:   pcalau12i $a0, 0
@@ -61,15 +132,83 @@
 # CHECKR-NEXT:   ld.d      $a0, $a0, 0
 # CHECKR-NEXT:     R_LARCH_GOT_PC_LO12 _start
 # CHECKR-NEXT:     R_LARCH_RELAX *ABS*
+# CHECKR-NEXT:   pcaddu18i $ra, 0
+# CHECKR-NEXT:     R_LARCH_CALL36 _start
+# CHECKR-NEXT:     R_LARCH_RELAX *ABS*
+# CHECKR-NEXT:   jirl   $ra, $ra, 0
+# CHECKR-NEXT:   pcaddu18i $a0, 0
+# CHECKR-NEXT:     R_LARCH_CALL36 _start
+# CHECKR-NEXT:     R_LARCH_RELAX *ABS*
+# CHECKR-NEXT:   jr     $a0
+# CHECKR-NEXT:   lu12i.w   $a0, 0
+# CHECKR-NEXT:     R_LARCH_TLS_LE_HI20 a
+# CHECKR-NEXT:   ori       $a0, $a0, 0
+# CHECKR-NEXT:     R_LARCH_TLS_LE_LO12 a
+# CHECKR-NEXT:   pcalau12i $a0, 0
+# CHECKR-NEXT:     R_LARCH_TLS_GD_PC_HI20 a
+# CHECKR-NEXT:     R_LARCH_RELAX *ABS*
+# CHECKR-NEXT:   addi.d    $a0, $a0, 0
+# CHECKR-NEXT:     R_LARCH_GOT_PC_LO12 a
+# CHECKR-NEXT:     R_LARCH_RELAX *ABS*
+# CHECKR-NEXT:   pcalau12i $a0, 0
+# CHECKR-NEXT:     R_LARCH_TLS_LD_PC_HI20 a
+# CHECKR-NEXT:     R_LARCH_RELAX *ABS*
+# CHECKR-NEXT:   addi.d    $a0, $a0, 0
+# CHECKR-NEXT:     R_LARCH_GOT_PC_LO12 a
+# CHECKR-NEXT:     R_LARCH_RELAX *ABS*
+# CHECKR-NEXT:   lu12i.w   $a0, 0
+# CHECKR-NEXT:     R_LARCH_TLS_LE_HI20_R a
+# CHECKR-NEXT:     R_LARCH_RELAX *ABS*
+# CHECKR-NEXT:   add.d     $a0, $a0, $tp
+# CHECKR-NEXT:     R_LARCH_TLS_LE_ADD_R a
+# CHECKR-NEXT:     R_LARCH_RELAX *ABS*
+# CHECKR-NEXT:   addi.d    $a0, $a0, 0
+# CHECKR-NEXT:     R_LARCH_TLS_LE_LO12_R a
+# CHECKR-NEXT:     R_LARCH_RELAX *ABS*
 # CHECKR-NEXT:   nop
 # CHECKR-NEXT:     R_LARCH_ALIGN *ABS*+0xc
 # CHECKR-NEXT:   nop
 # CHECKR-NEXT:   nop
 # CHECKR-NEXT:   ret
 
+.macro add dst, src1, src2, src3
+.ifdef ELF64
+  add.d \dst, \src1, \src2, \src3
+.else
+  add.w \dst, \src1, \src2, \src3
+.endif
+.endm
+
+.macro addi dst, src1, src2
+.ifdef ELF64
+  addi.d \dst, \src1, \src2
+.else
+  addi.w \dst, \src1, \src2
+.endif
+.endm
+
 .global _start
 _start:
   la.pcrel $a0, _start
   la.got   $a0, _start
+
+.ifdef ELF64
+  call36 _start
+  tail36 $a0, _start
+.endif
+
+  la.tls.le $a0, a  # without R_LARCH_RELAX reloaction
+  la.tls.gd $a0, a
+  la.tls.ld $a0, a
+
+  lu12i.w $a0, %le_hi20_r(a)
+  add $a0, $a0, $tp, %le_add_r(a)
+  addi $a0, $a0, %le_lo12_r(a)
+
   .p2align 4
   ret
+
+.section .tbss,"awT",@nobits
+.globl a
+a:
+.zero 4
diff --git a/lld/test/ELF/loongarch-relax-tls-le.s b/lld/test/ELF/loongarch-relax-tls-le.s
new file mode 100644
index 0000000000000..a91374b6e81a3
--- /dev/null
+++ b/lld/test/ELF/loongarch-relax-tls-le.s
@@ -0,0 +1,115 @@
+# REQUIRES: loongarch
+
+# RUN: llvm-mc --filetype=obj --triple=loongarch32 -mattr=+relax --defsym ELF32=1 %s -o %t.32.o
+# RUN: llvm-mc --filetype=obj --triple=loongarch64 -mattr=+relax %s -o %t.64.o
+
+# RUN: ld.lld %t.32.o -o %t.32
+# RUN: llvm-objdump -d --no-show-raw-insn %t.32 | FileCheck --check-prefixes=RELAX32 %s
+
+# RUN: ld.lld %t.64.o -o %t.64
+# RUN: llvm-objdump -d --no-show-raw-insn %t.64 | FileCheck --check-prefixes=RELAX64 %s
+
+# RELAX32-LABEL: <_start>:
+## .LANCHOR0@tprel = 8
+# RELAX32-NEXT:    addi.w  $a0, $tp, 8 
+# RELAX32-NEXT:    ld.w    $a1, $a0, 0
+# RELAX32-NEXT:    ld.w    $a2, $tp, 8
+## .a@tprel - 4 = 0x7fc
+# RELAX32-NEXT:    addi.w  $a1, $zero, 1
+# RELAX32-NEXT:    addi.w $a1, $a1, 2
+# RELAX32-NEXT:    st.w   $a1, $tp, 2044
+## .a@tprel = 0x800
+# RELAX32-NEXT:    lu12i.w $a0, 1
+# RELAX32-NEXT:    add.w   $a0, $a0, $tp
+# RELAX32-NEXT:    addi.w  $a0, $a0, -2048
+
+# RELAX64-LABEL: <_start>:
+## .LANCHOR0@tprel = 8
+# RELAX64-NEXT:    addi.d  $a0, $tp, 8 
+# RELAX64-NEXT:    ld.d    $a1, $a0, 0
+# RELAX64-NEXT:    ld.d    $a2, $tp, 8
+## .a@tprel - 4 = 0x7fc
+# RELAX64-NEXT:    addi.d  $a1, $zero, 1
+# RELAX64-NEXT:    addi.d $a1, $a1, 2
+# RELAX64-NEXT:    st.d   $a1, $tp, 2044
+## .a@tprel = 0x800
+# RELAX64-NEXT:    lu12i.w $a0, 1
+# RELAX64-NEXT:    add.d   $a0, $a0, $tp
+# RELAX64-NEXT:    addi.d  $a0, $a0, -2048
+
+.macro add dst, src1, src2, src3
+.ifdef ELF32
+  add.w \dst, \src1, \src2, \src3
+.else
+  add.d \dst, \src1, \src2, \src3
+.endif
+.endm
+.macro inst op dst, src1, src2
+.ifdef ELF32
+  .ifc      \op, addi
+    addi.w  \dst, \src1, \src2
+  .else;    .ifc   \op, ld
+    ld.w    \dst, \src1, \src2
+  .else;    .ifc   \op, st
+    st.w    \dst, \src1, \src2
+  .else;    .ifc   \op, ldptr
+    ldptr.w \dst, \src1, \src2
+  .else
+    .error "Unknown op in ELF32 mode"
+  .endif; .endif; .endif; .endif
+.else
+  .ifc      \op, addi
+    addi.d  \dst, \src1, \src2
+  .else;    .ifc   \op, ld
+    ld.d    \dst, \src1, \src2
+  .else;    .ifc   \op, st
+    st.d    \dst, \src1, \src2
+  .else;    .ifc   \op, ldptr
+    ldptr.d \dst, \src1, \src2
+  .else
+    .error "Unknown op in ELF64 mode"
+  .endif; .endif; .endif; .endif
+.endif
+.endm
+
+.macro addi dst, src1, src2
+  inst addi \dst, \src1, \src2
+.endm
+.macro ld dst, src1, src2
+  inst ld \dst, \src1, \src2
+.endm
+.macro st dst, src1, src2
+  inst st \dst, \src1, \src2
+.endm
+.macro ldptr dst, src1, src2
+  inst ldptr \dst, \src1, \src2
+.endm
+
+_start:
+  ## Test instructions not in pairs.
+  lu12i.w $a0, %le_hi20_r(.LANCHOR0)
+  add $a0, $a0, $tp, %le_add_r(.LANCHOR0)
+  addi $a0, $a0, %le_lo12_r(.LANCHOR0)
+  ld $a1, $a0, 0
+  ld $a2, $a0, %le_lo12_r(.LANCHOR0)
+
+  ## hi20(a-4) = hi20(0x7fc) = 0. relaxable
+  ## Test non-adjacent instructions.
+  lu12i.w $a0, %le_hi20_r(a-4)
+  addi $a1, $zero, 0x1
+  add $a0, $a0, $tp, %le_add_r(a-4)
+  addi $a1, $a1, 0x2
+  st $a1, $a0, %le_lo12_r(a-4)
+
+  ## hi20(a) = hi20(0x800) = 1. not relaxable
+  lu12i.w $a0, %le_hi20_r(a)
+  add $a0, $a0, $tp, %le_add_r(a)
+  addi $a0, $a0, %le_lo12_r(a)
+
+.section .tbss,"awT",@nobits
+.space 8
+.LANCHOR0:
+.space 0x800-8
+.globl a
+a:
+.zero 4
diff --git a/lld/test/ELF/loongarch-tls-gd.s b/lld/test/ELF/loongarch-tls-gd.s
index 2aecb44c17a34..140aa4cea6bad 100644
--- a/lld/test/ELF/loongarch-tls-gd.s
+++ b/lld/test/ELF/loongarch-tls-gd.s
@@ -1,14 +1,17 @@
 # REQUIRES: loongarch
 # RUN: rm -rf %t && split-file %s %t
 
-## LoongArch psABI doesn't specify TLS relaxation. Though the code sequences are not
-## relaxed, dynamic relocations can be omitted for GD->LE relaxation.
+## LoongArch psABI doesn't specify TLS relaxation. It can be handled the same way as gcc:
+## (a) code sequence can be converted from `pcalau12i+addi.[wd]` to `pcaddi`.
+## (b) dynamic relocations can be omitted for GD->LE relaxation.
 
 # RUN: llvm-mc --filetype=obj --triple=loongarch32 %t/a.s -o %t/a.32.o
+# RUN: llvm-mc --filetype=obj --triple=loongarch32 -mattr=+relax %t/a.s -o %t/a.32.relax.o
 # RUN: llvm-mc --filetype=obj --triple=loongarch32 %t/bc.s -o %t/bc.32.o
 # RUN: ld.lld -shared -soname=bc.so %t/bc.32.o -o %t/bc.32.so
 # RUN: llvm-mc --filetype=obj --triple=loongarch32 %t/tga.s -o %t/tga.32.o
 # RUN: llvm-mc --filetype=obj --triple=loongarch64 %t/a.s -o %t/a.64.o
+# RUN: llvm-mc --filetype=obj --triple=loongarch64 %t/a.s -mattr=+relax -o %t/a.64.relax.o
 # RUN: llvm-mc --filetype=obj --triple=loongarch64 %t/bc.s -o %t/bc.64.o
 # RUN: ld.lld -shared -soname=bc.so %t/bc.64.o -o %t/bc.64.so
 # RUN: llvm-mc --filetype=obj --triple=loongarch64 %t/tga.s -o %t/tga.64.o
@@ -17,6 +20,9 @@
 # RUN: ld.lld -shared %t/a.32.o %t/bc.32.o -o %t/gd.32.so
 # RUN: llvm-readobj -r %t/gd.32.so | FileCheck --check-prefix=GD32-REL %s
 # RUN: llvm-objdump -d --no-show-raw-insn %t/gd.32.so | FileCheck --check-prefix=GD32 %s
+# RUN: ld.lld -shared %t/a.32.relax.o %t/bc.32.o -o %t/gd.32.relax.so
+# RUN: llvm-readobj -r %t/gd.32.relax.so | FileCheck --check-prefix=GD32-REL-RELAX %s
+# RUN: llvm-objdump -d --no-show-raw-insn %t/gd.32.relax.so | FileCheck --check-prefix=GD32-RELAX %s
 
 ## LA32 GD -> LE
 # RUN: ld.lld %t/a.32.o %t/bc.32.o %t/tga.32.o -o %t/le.32
@@ -35,6 +41,9 @@
 # RUN: ld.lld -shared %t/a.64.o %t/bc.64.o -o %t/gd.64.so
 # RUN: llvm-readobj -r %t/gd.64.so | FileCheck --check-prefix=GD64-REL %s
 # RUN: llvm-objdump -d --no-show-raw-insn %t/gd.64.so | FileCheck --check-prefix=GD64 %s
+# RUN: ld.lld -shared %t/a.64.relax.o %t/bc.64.o -o %t/gd.64.relax.so
+# RUN: llvm-readobj -r %t/gd.64.relax.so | FileCheck --check-prefix=GD64-REL-RELAX %s
+# RUN: llvm-objdump -d --no-show-raw-insn %t/gd.64.relax.so | FileCheck --check-prefix=GD64-RELAX %s
 
 ## LA64 GD -> LE
 # RUN: ld.lld %t/a.64.o %t/bc.64.o %t/tga.64.o -o %t/le.64
@@ -66,6 +75,21 @@
 # GD32-NEXT:        addi.w $a0, $a0, 792
 # GD32-NEXT:        bl 44
 
+# GD32-REL-RELAX:      .rela.dyn {
+# GD32-REL-RELAX-NEXT:   0x20300 R_LARCH_TLS_DTPMOD32 a 0x0
+# GD32-REL-RELAX-NEXT:   0x20304 R_LARCH_TLS_DTPREL32 a 0x0
+# GD32-REL-RELAX-NEXT:   0x20308 R_LARCH_TLS_DTPMOD32 b 0x0
+# GD32-REL-RELAX-NEXT:   0x2030C R_LARCH_TLS_DTPREL32 b 0x0
+# GD32-REL-RELAX-NEXT: }
+
+## &DTPMOD(a) - . = 0x20300 - 0x10250 = 16428<<2
+# GD32-RELAX:      10250: pcaddi $a0, 16428
+# GD32-RELAX-NEXT:        bl 44
+
+## &DTPMOD(b) - . = 0x20308 - 0x10258 = 16428<<2
+# GD32-RELAX:      10258: pcaddi $a0, 16428
+# GD32-RELAX-NEXT:        bl 36
+
 # GD64-REL:      .rela.dyn {
 # GD64-REL-NEXT:   0x204C0 R_LARCH_TLS_DTPMOD64 a 0x0
 # GD64-REL-NEXT:   0x204C8 R_LARCH_TLS_DTPREL64 a 0x0
@@ -83,6 +107,21 @@
 # GD64-NEXT:        addi.d $a0, $a0, 1232
 # GD64-NEXT:        bl 36
 
+# GD64-REL-RELAX:      .rela.dyn {
+# GD64-REL-RELAX-NEXT:   0x204C0 R_LARCH_TLS_DTPMOD64 a 0x0
+# GD64-REL-RELAX-NEXT:   0x204C8 R_LARCH_TLS_DTPREL64 a 0x0
+# GD64-REL-RELAX-NEXT:   0x204D0 R_LARCH_TLS_DTPMOD64 b 0x0
+# GD64-REL-RELAX-NEXT:   0x204D8 R_LARCH_TLS_DTPREL64 b 0x0
+# GD64-REL-RELAX-NEXT: }
+
+## &DTPMOD(a) - . = 0x204c0 - 0x10398 = 16458<<2
+# GD64-RELAX:      10398: pcaddi $a0, 16458
+# GD64-RELAX-NEXT:        bl 52
+
+## &DTPMOD(b) - . = 0x204d0 - 0x103a0 = 16460<<2
+# GD64-RELAX:      103a0: pcaddi $a0, 16460
+# GD64-RELAX-NEXT:        bl 44
+
 # NOREL: no relocations
 
 ## .got contains pre-populated values: [a@dtpmod, a@dtprel, b@dtpmod, b@dtprel]
diff --git a/lld/test/ELF/loongarch-tls-ld.s b/lld/test/ELF/loongarch-tls-ld.s
index a5be3ad905b76..27adb1e17702a 100644
--- a/lld/test/ELF/loongarch-tls-ld.s
+++ b/lld/test/ELF/loongarch-tls-ld.s
@@ -1,12 +1,15 @@
 # REQUIRES: loongarch
 # RUN: rm -rf %t && split-file %s %t
 
-## LoongArch psABI doesn't specify TLS relaxation. Though the code sequences are not
-## relaxed, dynamic relocations can be omitted for LD->LE relaxation.
+## LoongArch psABI doesn't specify TLS relaxation. It can be handled the same way as gcc:
+## (a) code sequence can be converted from `pcalau12i+addi.[wd]` to `pcaddi`.
+## (b) dynamic relocations can be omitted for LD->LE relaxation.
 
 # RUN: llvm-mc --filetype=obj --triple=loongarch32 --position-independent %t/a.s -o %t/a.32.o
+# RUN: llvm-mc --filetype=obj --triple=loongarch32 --position-independent -mattr=+relax %t/a.s -o %t/a.32.relax.o
 # RUN: llvm-mc --filetype=obj --triple=loongarch32 %t/tga.s -o %t/tga.32.o
 # RUN: llvm-mc --filetype=obj --triple=loongarch64 --position-independent %t/a.s -o %t/a.64.o
+# RUN: llvm-mc --filetype=obj --triple=loongarch64 --position-independent -mattr=+relax %t/a.s -o %t/a.64.relax.o
 # RUN: llvm-mc --filetype=obj --triple=loongarch64 %t/tga.s -o %t/tga.64.o
 
 ## LA32 LD
@@ -14,24 +17,34 @@
 # RUN: llvm-readobj -r %t/ld.32.so | FileCheck --check-prefix=LD32-REL %s
 # RUN: llvm-readelf -x .got %t/ld.32.so | FileCheck --check-prefix=LD32-GOT %s
 # RUN: llvm-objdump -d --no-show-raw-insn %t/ld.32.so | FileCheck --check-prefixes=LD32 %s
+# RUN: ld.lld -shared %t/a.32.relax.o -o %t/ld.32.relax.so
+# RUN: llvm-objdump -d --no-show-raw-insn %t/ld.32.relax.so | FileCheck --check-prefixes=LD32-RELAX %s
 
 ## LA32 LD -> LE
 # RUN: ld.lld %t/a.32.o %t/tga.32.o -o %t/le.32
 # RUN: llvm-readelf -r %t/le.32 | FileCheck --check-prefix=NOREL %s
 # RUN: llvm-readelf -x .got %t/le.32 | FileCheck --check-prefix=LE32-GOT %s
 # RUN: llvm-objdump -d --no-show-raw-insn %t/le.32 | FileCheck --check-prefixes=LE32 %s
+# RUN: ld.lld %t/a.32.relax.o %t/tga.32.o -o %t/le.32.relax
+# RUN: llvm-readelf -x .got %t/le.32.relax | FileCheck --check-prefix=LE32-GOT-RELAX %s
+# RUN: llvm-objdump -d --no-show-raw-insn %t/le.32.relax | FileCheck --check-prefixes=LE32-RELAX %s
 
 ## LA64 LD
 # RUN: ld.lld -shared %t/a.64.o -o %t/ld.64.so
 # RUN: llvm-readobj -r %t/ld.64.so | FileCheck --check-prefix=LD64-REL %s
 # RUN: llvm-readelf -x .got %t/ld.64.so | FileCheck --check-prefix=LD64-GOT %s
 # RUN: llvm-objdump -d --no-show-raw-insn %t/ld.64.so | FileCheck --check-prefixes=LD64 %s
+# RUN: ld.lld -shared %t/a.64.relax.o -o %t/ld.64.relax.so
+# RUN: llvm-objdump -d --no-show-raw-insn %t/ld.64.relax.so | FileCheck --check-prefixes=LD64-RELAX %s
 
 ## LA64 LD -> LE
 # RUN: ld.lld %t/a.64.o %t/tga.64.o -o %t/le.64
 # RUN: llvm-readelf -r %t/le.64 | FileCheck --check-prefix=NOREL %s
 # RUN: llvm-readelf -x .got %t/le.64 | FileCheck --check-prefix=LE64-GOT %s
 # RUN: llvm-objdump -d --no-show-raw-insn %t/le.64 | FileCheck --check-prefixes=LE64 %s
+# RUN: ld.lld %t/a.64.relax.o %t/tga.64.o -o %t/le.64.relax
+# RUN: llvm-readelf -x .got %t/le.64.relax | FileCheck --check-prefix=LE64-GOT-RELAX %s
+# RUN: llvm-objdump -d --no-show-raw-insn %t/le.64.relax | FileCheck --check-prefixes=LE64-RELAX %s
 
 ## a@dtprel = st_value(a) = 0 is a link-time constant.
 # LD32-REL:      .rela.dyn {
@@ -56,6 +69,14 @@
 # LD64-NEXT:        addi.d $a0, $a0, 1024
 # LD64-NEXT:        bl 40
 
+## LA32: &DTPMOD(a) - . = 0x20280 - 0x101cc = 16429<<2
+# LD32-RELAX:      101cc: pcaddi  $a0, 16429
+# LD32-RELAX-NEXT:        bl 48
+
+## LA64: &DTPMOD(a) - . = 0x20400 - 0x102e0 = 16456<<2
+# LD64-RELAX:      102e0: pcaddi  $a0, 16456
+# LD64-RELAX-NEXT:        bl 44
+
 # NOREL: no relocations
 
 ## a is local - its DTPMOD/DTPREL slots are link-time constants.
@@ -66,6 +87,12 @@
 # LE64-GOT: section '.got':
 # LE64-GOT-NEXT: 0x000301d8 01000000 00000000 00000000 00000000
 
+# LE32-GOT-RELAX: section '.got':
+# LE32-GOT-RELAX-NEXT: 0x0003011c 01000000 00000000
+
+# LE64-GOT-RELAX: section '.got':
+# LE64-GOT-RELAX-NEXT: 0x000301d0 01000000 00000000 00000000 00000000
+
 ## LA32: DTPMOD(.LANCHOR0) - . = 0x30120 - 0x20114: 0x10 pages, page offset 0x120
 # LE32:      20114: pcalau12i $a0, 16
 # LE32-NEXT:        addi.w $a0, $a0, 288
@@ -76,6 +103,15 @@
 # LE64-NEXT:        addi.d $a0, $a0, 472
 # LE64-NEXT:        bl 4
 
+## LA32: DTPMOD(.LANCHOR0) - . = 0x3011c - 0x20114 = 16386<<2
+# LE32-RELAX:      20114: pcaddi $a0, 16386
+# LE32-RELAX-NEXT:        bl 4
+
+## LA64: DTPMOD(.LANCHOR0) - . = 0x301d0 - 0x201c8 = 16386<<2
+# LE64-RELAX:      201c8: pcaddi $a0, 16386
+# LE64-RELAX-NEXT:        bl 4
+
+
 #--- a.s
 la.tls.ld $a0, .LANCHOR0
 bl %plt(__tls_get_addr)
diff --git a/lld/test/ELF/loongarch-tls-le.s b/lld/test/ELF/loongarch-tls-le.s
index 394c60f67bce8..981baa6fd726f 100644
--- a/lld/test/ELF/loongarch-tls-le.s
+++ b/lld/test/ELF/loongarch-tls-le.s
@@ -1,14 +1,20 @@
 # REQUIRES: loongarch
 
 # RUN: llvm-mc --filetype=obj --triple=loongarch32 --defsym ELF32=1 %s -o %t.32.o
+# RUN: llvm-mc --filetype=obj --triple=loongarch32 --defsym ELF32=1 -mattr=+relax %s -o %t.32.relax.o
 # RUN: llvm-mc --filetype=obj --triple=loongarch64 %s -o %t.64.o
+# RUN: llvm-mc --filetype=obj --triple=loongarch64 -mattr=+relax %s -o %t.64.relax.o
 
 # RUN: ld.lld %t.32.o -o %t.32
 # RUN: llvm-nm -p %t.32 | FileCheck --check-prefixes=NM %s
 # RUN: llvm-objdump -d --no-show-raw-insn %t.32 | FileCheck --check-prefixes=LE,LE32 %s
+# RUN: ld.lld %t.32.relax.o -o %t.32.relax
+# RUN: llvm-objdump -d --no-show-raw-insn %t.32.relax | FileCheck --check-prefixes=LE,LE32-RELAX %s
 
 # RUN: ld.lld %t.64.o -o %t.64
 # RUN: llvm-objdump -d --no-show-raw-insn %t.64 | FileCheck --check-prefixes=LE,LE64 %s
+# RUN: ld.lld %t.64.relax.o -o %t.64.relax
+# RUN: llvm-objdump -d --no-show-raw-insn %t.64.relax | FileCheck --check-prefixes=LE,LE64-RELAX %s
 
 # RUN: not ld.lld -shared %t.32.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR --implicit-check-not=error:
 
@@ -37,12 +43,22 @@
 # LE32-NEXT: add.w   $a0, $a0, $tp
 # LE32-NEXT: addi.w  $a0, $a0, -2048
 
+# LE32-RELAX:      addi.w  $a0, $tp, 8
+# LE32-RELAX-NEXT: lu12i.w $a0, 1
+# LE32-RELAX-NEXT: add.w   $a0, $a0, $tp
+# LE32-RELAX-NEXT: addi.w  $a0, $a0, -2048
+
 # LE64:      add.d   $a0, $a0, $tp
 # LE64-NEXT: addi.d  $a0, $a0, 8
 # LE64-NEXT: lu12i.w $a0, 1
 # LE64-NEXT: add.d   $a0, $a0, $tp
 # LE64-NEXT: addi.d  $a0, $a0, -2048
 
+# LE64-RELAX:      addi.d  $a0, $tp, 8
+# LE64-RELAX-NEXT: lu12i.w $a0, 1
+# LE64-RELAX-NEXT: add.d   $a0, $a0, $tp
+# LE64-RELAX-NEXT: addi.d  $a0, $a0, -2048
+
 # LE-EMPTY:
 
 .macro add dst, src1, src2, src3
diff --git a/lldb/bindings/interface/SBProcessDocstrings.i b/lldb/bindings/interface/SBProcessDocstrings.i
index b5b22052a6094..2438a6bfc124c 100644
--- a/lldb/bindings/interface/SBProcessDocstrings.i
+++ b/lldb/bindings/interface/SBProcessDocstrings.i
@@ -260,12 +260,12 @@ SBProcess supports thread iteration. For example (from test/lldbutil.py), ::
 ) lldb::SBProcess::DeallocateMemory;
 
 %feature("docstring", "
-    Get a list of all the memory regions associated with this process.
-    ```
+    Get a list of all the memory regions associated with this process. ::
+    
         readable_regions = []
         for region in process.GetMemoryRegions():
             if region.IsReadable():
                 readable_regions.append(region)
-    ```
+    
 "
 ) lldb::SBProcess::GetMemoryRegions;
diff --git a/lldb/bindings/interface/SBProgressDocstrings.i b/lldb/bindings/interface/SBProgressDocstrings.i
index 7b7e1dc79187c..8d252ef1f370c 100644
--- a/lldb/bindings/interface/SBProgressDocstrings.i
+++ b/lldb/bindings/interface/SBProgressDocstrings.i
@@ -23,7 +23,7 @@ specify a new detail. Some implementations differ on throttling updates and this
 if the progress is deterministic or non-deterministic. For DAP, non-deterministic update messages have a higher
 throttling rate than deterministic ones.
 
-Below are examples in Python for deterministic and non-deterministic progresses.
+Below are examples in Python for deterministic and non-deterministic progresses. ::
 
     deterministic_progress1 = lldb.SBProgress('Deterministic Progress', 'Detail', 3, lldb.SBDebugger) 
     for i in range(3): 
@@ -44,7 +44,7 @@ If you don't call Finalize() when the progress is not done, the progress object
 garbage collected by the Python runtime, the end event will eventually get sent, but it is best not to 
 rely on the garbage collection when using lldb.SBProgress.
 
-Non-deterministic progresses behave the same, but omit the total in the constructor.
+Non-deterministic progresses behave the same, but omit the total in the constructor. ::
 
     non_deterministic_progress = lldb.SBProgress('Non deterministic progress, 'Detail', lldb.SBDebugger)
     for i in range(10):
diff --git a/lldb/bindings/interface/SBSaveCoreOptionsDocstrings.i b/lldb/bindings/interface/SBSaveCoreOptionsDocstrings.i
index 08bbdf89d68de..6efbe45d2d3ab 100644
--- a/lldb/bindings/interface/SBSaveCoreOptionsDocstrings.i
+++ b/lldb/bindings/interface/SBSaveCoreOptionsDocstrings.i
@@ -4,16 +4,13 @@
 SBSaveCoreOptions includes API's to specify the memory regions and threads to include
 when generating a core file. It extends the existing SaveCoreStyle option.
 
-* eSaveCoreFull will save off all thread and memory regions, ignoring the memory regions and threads in
-the options object.
+* eSaveCoreFull will save off all thread and memory regions, ignoring the memory regions and threads in the options object.
 
-* eSaveCoreDirtyOnly pages will capture all threads and all rw- memory regions, in addition to the regions specified
-in the options object if they are not already captured.
+* eSaveCoreDirtyOnly pages will capture all threads and all rw- memory regions, in addition to the regions specified in the options object if they are not already captured.
 
 * eSaveCoreStackOnly will capture all threads, but no memory regions unless specified.
 
-* eSaveCoreCustomOnly Custom defers entirely to the SBSaveCoreOptions object and will only save what is specified. 
-  Picking custom and specifying nothing will result in an error being returned.
+* eSaveCoreCustomOnly Custom defers entirely to the SBSaveCoreOptions object and will only save what is specified. Picking custom and specifying nothing will result in an error being returned.
 
 Note that currently ELF Core files are not supported."
 ) lldb::SBSaveCoreOptions;
diff --git a/lldb/docs/resources/build.rst b/lldb/docs/resources/build.rst
index 214c5f63f2c73..9ba2405fc9628 100644
--- a/lldb/docs/resources/build.rst
+++ b/lldb/docs/resources/build.rst
@@ -340,7 +340,7 @@ On macOS the LLDB test suite requires libc++. Either add
   code-signed with identity ``lldb_codesign`` (see :ref:`CodeSigning`).
 * ``LLDB_USE_SYSTEM_DEBUGSERVER:BOOL``: Use the system's debugserver, so lldb is
   functional without setting up code-signing.
-
+* ``LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS:BOOL``: Detect missing packages or modules at configuration time.
 
 .. _CMakeCaches:
 
diff --git a/lldb/docs/use/symbolfilejson.rst b/lldb/docs/use/symbolfilejson.rst
index 9d15d704f3574..2ad28a14e9320 100644
--- a/lldb/docs/use/symbolfilejson.rst
+++ b/lldb/docs/use/symbolfilejson.rst
@@ -74,14 +74,61 @@ sections
 * ``type``: a string representing the section type (see below).
 * ``address``: a number representing the section file address.
 * ``size``: a number representing the section size in bytes.
+* ``read``: a boolean value indicating if the section has read permissions.
+* ``write``: a boolean value indicating if the section has write permissions.
+* ``execute``: a boolean value indicating if the section has execute permissions.
+* ``subsections``: An array of child sections in the same format.
+* ``user_id``: a number representing the section identifier.
+* ``file_offset``: a number that represents the offset in the file for this section's contents.
+* ``file_size``: a number that represents the size in bytes in the file for this section's contents.
+* ``alignment``: a boolean value indicating if the section has execute permissions.
+* ``fake``: a boolean value indicating if the section is fake.
+* ``encrypted``: a boolean value indicating if the section is encrypted.
+* ``thread_specific``: a boolean value indicating if the section is thread specific.
 
 .. code-block:: JSON
 
   {
-      "name": "__TEXT",
-      "type": "code",
-      "address": 0,
-      "size": 546,
+    "user_id": 256,
+    "name": "__TEXT",
+    "type": "code",
+    "address": 0,
+    "size": 546,
+    "file_offset": 0,
+    "file_size": 4096,
+    "read": true,
+    "write": false,
+    "executable": true,
+    "subsections": [
+      {
+        "name": "__text",
+        "type": "code",
+        "address": 0,
+        "size": 200,
+        "alignment": 2,
+        "read": true,
+        "write": false,
+        "execute": true
+      },
+      {
+          "name": "__fake",
+          "address": 200,
+          "size": 10,
+          "fake": true
+      },
+      {
+          "name": "__encrypted",
+          "address": 210,
+          "size": 20,
+          "encrypted": true
+      },
+      {
+          "name": "__tls",
+          "address": 230,
+          "size": 30,
+          "thread_specific": true
+      }
+    ]
   }
 
 The ``type`` field accepts the following values: ``code``, ``container``,
diff --git a/lldb/include/lldb/Core/Mangled.h b/lldb/include/lldb/Core/Mangled.h
index 165e67838e84b..9ca28917ccffa 100644
--- a/lldb/include/lldb/Core/Mangled.h
+++ b/lldb/include/lldb/Core/Mangled.h
@@ -276,10 +276,12 @@ class Mangled {
   void Encode(DataEncoder &encoder, ConstStringTable &strtab) const;
 
 private:
-  /// Mangled member variables.
-  ConstString m_mangled;           ///< The mangled version of the name
-  mutable ConstString m_demangled; ///< Mutable so we can get it on demand with
-                                   ///a const version of this object
+  ///< The mangled version of the name.
+  ConstString m_mangled;
+
+  ///< Mutable so we can get it on demand with
+  ///< a const version of this object.
+  mutable ConstString m_demangled;
 };
 
 Stream &operator<<(Stream &s, const Mangled &obj);
diff --git a/lldb/include/lldb/Core/Module.h b/lldb/include/lldb/Core/Module.h
index 90e0f4b6a3aac..9aa05ed3eb074 100644
--- a/lldb/include/lldb/Core/Module.h
+++ b/lldb/include/lldb/Core/Module.h
@@ -608,7 +608,11 @@ class Module : public std::enable_shared_from_this<Module>,
   virtual SymbolFile *GetSymbolFile(bool can_create = true,
                                     Stream *feedback_strm = nullptr);
 
-  Symtab *GetSymtab();
+  /// Get the module's symbol table
+  ///
+  /// If the symbol table has already been loaded, this function returns it.
+  /// Otherwise, it will only be loaded when can_create is true.
+  Symtab *GetSymtab(bool can_create = true);
 
   /// Get a reference to the UUID value contained in this object.
   ///
diff --git a/lldb/include/lldb/Core/Progress.h b/lldb/include/lldb/Core/Progress.h
index 5876eae717e96..3003568e8946b 100644
--- a/lldb/include/lldb/Core/Progress.h
+++ b/lldb/include/lldb/Core/Progress.h
@@ -9,7 +9,6 @@
 #ifndef LLDB_CORE_PROGRESS_H
 #define LLDB_CORE_PROGRESS_H
 
-#include "lldb/Host/Alarm.h"
 #include "lldb/Utility/Timeout.h"
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-types.h"
@@ -18,6 +17,7 @@
 #include <cstdint>
 #include <mutex>
 #include <optional>
+#include <string>
 
 namespace lldb_private {
 
@@ -115,21 +115,6 @@ class Progress {
   /// Used to indicate a non-deterministic progress report
   static constexpr uint64_t kNonDeterministicTotal = UINT64_MAX;
 
-  /// Data belonging to this Progress event that is used for bookkeeping by
-  /// ProgressManager.
-  struct ProgressData {
-    /// The title of the progress activity, also used as a category.
-    std::string title;
-    /// A unique integer identifier for progress reporting.
-    uint64_t progress_id;
-    /// The optional debugger ID to report progress to. If this has no value
-    /// then all debuggers will receive this event.
-    std::optional<lldb::user_id_t> debugger_id;
-
-    /// The origin of the progress event, wheter it is internal or external.
-    Origin origin;
-  };
-
 private:
   void ReportProgress();
   static std::atomic<uint64_t> g_id;
@@ -141,8 +126,18 @@ class Progress {
   // Minimum amount of time between two progress reports.
   const Timeout<std::nano> m_minimum_report_time;
 
-  /// Data needed by the debugger to broadcast a progress event.
-  const ProgressData m_progress_data;
+  /// The title of the progress activity, also used as a category.
+  const std::string m_title;
+
+  /// A unique integer identifier for progress reporting.
+  const uint64_t m_progress_id;
+
+  /// The optional debugger ID to report progress to. If this has no value
+  /// then all debuggers will receive this event.
+  const std::optional<lldb::user_id_t> m_debugger_id;
+
+  /// The origin of the progress event, whether it is internal or external.
+  const Origin m_origin;
 
   /// How much work ([0...m_total]) that has been completed.
   std::atomic<uint64_t> m_completed = 0;
@@ -161,60 +156,6 @@ class Progress {
   std::optional<uint64_t> m_prev_completed;
 };
 
-/// A class used to group progress reports by category. This is done by using a
-/// map that maintains a refcount of each category of progress reports that have
-/// come in. Keeping track of progress reports this way will be done if a
-/// debugger is listening to the eBroadcastBitProgressByCategory broadcast bit.
-class ProgressManager {
-public:
-  ProgressManager();
-  ~ProgressManager();
-
-  /// Control the refcount of the progress report category as needed.
-  void Increment(const Progress::ProgressData &);
-  void Decrement(const Progress::ProgressData &);
-
-  static void Initialize();
-  static void Terminate();
-  static bool Enabled();
-  static ProgressManager &Instance();
-
-protected:
-  enum class EventType {
-    Begin,
-    End,
-  };
-  static void ReportProgress(const Progress::ProgressData &progress_data,
-                             EventType type);
-
-  static std::optional<ProgressManager> &InstanceImpl();
-
-  /// Helper function for reporting progress when the alarm in the corresponding
-  /// entry in the map expires.
-  void Expire(llvm::StringRef key);
-
-  /// Entry used for bookkeeping.
-  struct Entry {
-    /// Reference count used for overlapping events.
-    uint64_t refcount = 0;
-
-    /// Data used to emit progress events.
-    Progress::ProgressData data;
-
-    /// Alarm handle used when the refcount reaches zero.
-    Alarm::Handle handle = Alarm::INVALID_HANDLE;
-  };
-
-  /// Map used for bookkeeping.
-  llvm::StringMap<Entry> m_entries;
-
-  /// Mutex to provide the map.
-  std::mutex m_entries_mutex;
-
-  /// Alarm instance to coalesce progress events.
-  Alarm m_alarm;
-};
-
 } // namespace lldb_private
 
 #endif // LLDB_CORE_PROGRESS_H
diff --git a/lldb/include/lldb/Core/Section.h b/lldb/include/lldb/Core/Section.h
index 17b3cb454949f..f0f5a0b3499c0 100644
--- a/lldb/include/lldb/Core/Section.h
+++ b/lldb/include/lldb/Core/Section.h
@@ -101,14 +101,25 @@ class SectionList {
 };
 
 struct JSONSection {
+  std::optional<lldb::user_id_t> user_id;
   std::string name;
   std::optional<lldb::SectionType> type;
   std::optional<uint64_t> address;
   std::optional<uint64_t> size;
+  std::optional<uint64_t> file_offset;
+  std::optional<uint64_t> file_size;
+  std::optional<uint64_t> log2align;
+  std::optional<uint64_t> flags;
+
   // Section permissions;
   std::optional<bool> read;
   std::optional<bool> write;
   std::optional<bool> execute;
+
+  std::optional<bool> fake;
+  std::optional<bool> encrypted;
+  std::optional<bool> thread_specific;
+
   std::vector<JSONSection> subsections;
 };
 
diff --git a/lldb/include/lldb/Core/Telemetry.h b/lldb/include/lldb/Core/Telemetry.h
index 7d8716f1659b5..56259e4a673bf 100644
--- a/lldb/include/lldb/Core/Telemetry.h
+++ b/lldb/include/lldb/Core/Telemetry.h
@@ -13,12 +13,14 @@
 #include "lldb/Interpreter/CommandReturnObject.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/StructuredData.h"
+#include "lldb/Utility/UUID.h"
 #include "lldb/lldb-forward.h"
 #include "llvm/ADT/FunctionExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/JSON.h"
 #include "llvm/Telemetry/Telemetry.h"
+#include <atomic>
 #include <chrono>
 #include <ctime>
 #include <memory>
@@ -28,6 +30,17 @@
 namespace lldb_private {
 namespace telemetry {
 
+struct LLDBConfig : public ::llvm::telemetry::Config {
+  // If true, we will collect full details about a debug command (eg., args and
+  // original command). Note: This may contain PII, hence can only be enabled by
+  // the vendor while creating the Manager.
+  const bool detailed_command_telemetry;
+
+  explicit LLDBConfig(bool enable_telemetry, bool detailed_command_telemetry)
+      : ::llvm::telemetry::Config(enable_telemetry),
+        detailed_command_telemetry(detailed_command_telemetry) {}
+};
+
 // We expect each (direct) subclass of LLDBTelemetryInfo to
 // have an LLDBEntryKind in the form 0b11xxxxxxxx
 // Specifically:
@@ -37,6 +50,7 @@ namespace telemetry {
 // must have their LLDBEntryKind in the similar form (ie., share common prefix)
 struct LLDBEntryKind : public ::llvm::telemetry::EntryKind {
   static const llvm::telemetry::KindType BaseInfo = 0b11000000;
+  static const llvm::telemetry::KindType CommandInfo = 0b11010000;
   static const llvm::telemetry::KindType DebuggerInfo = 0b11000100;
 };
 
@@ -66,6 +80,52 @@ struct LLDBBaseTelemetryInfo : public llvm::telemetry::TelemetryInfo {
   void serialize(llvm::telemetry::Serializer &serializer) const override;
 };
 
+struct CommandInfo : public LLDBBaseTelemetryInfo {
+  /// If the command is/can be associated with a target entry this field
+  /// contains that target's UUID. <EMPTY> otherwise.
+  UUID target_uuid;
+  /// A unique ID for a command so the manager can match the start entry with
+  /// its end entry. These values only need to be unique within the same
+  /// session. Necessary because we'd send off an entry right before a command's
+  /// execution and another right after. This is to avoid losing telemetry if
+  /// the command does not execute successfully.
+  uint64_t command_id;
+  /// The command name(eg., "breakpoint set")
+  std::string command_name;
+  /// These two fields are not collected by default due to PII risks.
+  /// Vendor may allow them by setting the
+  /// LLDBConfig::detailed_command_telemetry.
+  /// @{
+  std::optional<std::string> original_command;
+  std::optional<std::string> args;
+  /// @}
+  /// Return status of a command and any error description in case of error.
+  std::optional<lldb::ReturnStatus> ret_status;
+  std::optional<std::string> error_data;
+
+  CommandInfo() = default;
+
+  llvm::telemetry::KindType getKind() const override {
+    return LLDBEntryKind::CommandInfo;
+  }
+
+  static bool classof(const llvm::telemetry::TelemetryInfo *T) {
+    return (T->getKind() & LLDBEntryKind::CommandInfo) ==
+           LLDBEntryKind::CommandInfo;
+  }
+
+  void serialize(llvm::telemetry::Serializer &serializer) const override;
+
+  static uint64_t GetNextId();
+
+private:
+  // We assign each command (in the same session) a unique id so that their
+  // "start" and "end" entries can be matched up.
+  // These values don't need to be unique across runs (because they are
+  // secondary-key), hence a simple counter is sufficent.
+  static std::atomic<uint64_t> g_command_id_seed;
+};
+
 struct DebuggerInfo : public LLDBBaseTelemetryInfo {
   std::string lldb_version;
 
@@ -93,24 +153,21 @@ class TelemetryManager : public llvm::telemetry::Manager {
 public:
   llvm::Error preDispatch(llvm::telemetry::TelemetryInfo *entry) override;
 
-  const llvm::telemetry::Config *GetConfig();
+  const LLDBConfig *GetConfig() { return m_config.get(); }
 
   virtual llvm::StringRef GetInstanceName() const = 0;
 
   static TelemetryManager *GetInstance();
 
-  static TelemetryManager *GetInstanceIfEnabled();
-
 protected:
-  TelemetryManager(std::unique_ptr<llvm::telemetry::Config> config);
+  TelemetryManager(std::unique_ptr<LLDBConfig> config);
 
   static void SetInstance(std::unique_ptr<TelemetryManager> manger);
 
 private:
-  std::unique_ptr<llvm::telemetry::Config> m_config;
+  std::unique_ptr<LLDBConfig> m_config;
   // Each instance of a TelemetryManager is assigned a unique ID.
   const std::string m_id;
-
   static std::unique_ptr<TelemetryManager> g_instance;
 };
 
@@ -118,39 +175,54 @@ class TelemetryManager : public llvm::telemetry::Manager {
 template <typename Info> struct ScopedDispatcher {
   // The debugger pointer is optional because we may not have a debugger yet.
   // In that case, caller must set the debugger later.
-  ScopedDispatcher(llvm::unique_function<void(Info *info)> callback,
+  ScopedDispatcher(Debugger *debugger = nullptr) {
+    // Start the timer.
+    m_start_time = std::chrono::steady_clock::now();
+    this->debugger = debugger;
+  }
+  ScopedDispatcher(llvm::unique_function<void(Info *info)> final_callback,
                    Debugger *debugger = nullptr)
-      : m_callback(std::move(callback)) {
+      : m_final_callback(std::move(final_callback)) {
     // Start the timer.
     m_start_time = std::chrono::steady_clock::now();
-    m_info.debugger = debugger;
+    this->debugger = debugger;
   }
 
-  void SetDebugger(Debugger *debugger) { m_info.debugger = debugger; }
+  void SetDebugger(Debugger *debugger) { this->debugger = debugger; }
 
-  ~ScopedDispatcher() {
-    // If Telemetry is disabled (either at buildtime or runtime),
-    // then don't do anything.
-    TelemetryManager *manager = TelemetryManager::GetInstanceIfEnabled();
-    if (!manager)
-      return;
+  void DispatchOnExit(llvm::unique_function<void(Info *info)> final_callback) {
+    // We probably should not be overriding previously set cb.
+    assert(!m_final_callback);
+    m_final_callback = std::move(final_callback);
+  }
 
-    m_info.start_time = m_start_time;
-    // Populate common fields that we can only set now.
-    m_info.end_time = std::chrono::steady_clock::now();
-    // The callback will set the remaining fields.
-    m_callback(&m_info);
+  void DispatchNow(llvm::unique_function<void(Info *info)> populate_fields_cb) {
+    TelemetryManager *manager = TelemetryManager::GetInstance();
+    if (!manager->GetConfig()->EnableTelemetry)
+      return;
+    Info info;
+    // Populate the common fields we know about.
+    info.start_time = m_start_time;
+    info.end_time = std::chrono::steady_clock::now();
+    info.debugger = debugger;
+    // The callback will set the rest.
+    populate_fields_cb(&info);
     // And then we dispatch.
-    if (llvm::Error er = manager->dispatch(&m_info)) {
+    if (llvm::Error er = manager->dispatch(&info)) {
       LLDB_LOG_ERROR(GetLog(LLDBLog::Object), std::move(er),
-                     "Failed to dispatch entry of type: {0}", m_info.getKind());
+                     "Failed to dispatch entry of type: {0}", info.getKind());
     }
   }
 
+  ~ScopedDispatcher() {
+    if (m_final_callback)
+      DispatchNow(std::move(m_final_callback));
+  }
+
 private:
   SteadyTimePoint m_start_time;
-  llvm::unique_function<void(Info *info)> m_callback;
-  Info m_info;
+  llvm::unique_function<void(Info *info)> m_final_callback;
+  Debugger *debugger;
 };
 
 } // namespace telemetry
diff --git a/lldb/include/lldb/Host/Alarm.h b/lldb/include/lldb/Host/Alarm.h
deleted file mode 100644
index 23b1ff1af5689..0000000000000
--- a/lldb/include/lldb/Host/Alarm.h
+++ /dev/null
@@ -1,115 +0,0 @@
-//===-- Alarm.h -------------------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLDB_HOST_ALARM_H
-#define LLDB_HOST_ALARM_H
-
-#include "lldb/Host/HostThread.h"
-#include "lldb/lldb-types.h"
-#include "llvm/Support/Chrono.h"
-
-#include <condition_variable>
-#include <mutex>
-
-namespace lldb_private {
-
-/// \class Alarm abstraction that enables scheduling a callback function after a
-/// specified timeout. Creating an alarm for a callback returns a Handle that
-/// can be used to restart or cancel the alarm.
-class Alarm {
-public:
-  using Handle = uint64_t;
-  using Callback = std::function<void()>;
-  using TimePoint = llvm::sys::TimePoint<>;
-  using Duration = std::chrono::milliseconds;
-
-  Alarm(Duration timeout, bool run_callback_on_exit = false);
-  ~Alarm();
-
-  /// Create an alarm for the given callback. The alarm will expire and the
-  /// callback will be called after the timeout.
-  ///
-  /// \returns
-  ///   Handle which can be used to restart or cancel the alarm.
-  Handle Create(Callback callback);
-
-  /// Restart the alarm for the given Handle. The alarm will expire and the
-  /// callback will be called after the timeout.
-  ///
-  /// \returns
-  ///   True if the alarm was successfully restarted. False if there is no alarm
-  ///   for the given Handle or the alarm already expired.
-  bool Restart(Handle handle);
-
-  /// Cancel the alarm for the given Handle. The alarm and its handle will be
-  /// removed.
-  ///
-  /// \returns
-  ///   True if the alarm was successfully canceled and the Handle removed.
-  ///   False if there is no alarm for the given Handle or the alarm already
-  ///   expired.
-  bool Cancel(Handle handle);
-
-  static constexpr Handle INVALID_HANDLE = 0;
-
-private:
-  /// Helper functions to start, stop and check the status of the alarm thread.
-  /// @{
-  void StartAlarmThread();
-  void StopAlarmThread();
-  bool AlarmThreadRunning();
-  /// @}
-
-  /// Return an unique, monotonically increasing handle.
-  static Handle GetNextUniqueHandle();
-
-  /// Helper to compute the next time the alarm thread needs to wake up.
-  TimePoint GetNextExpiration() const;
-
-  /// Alarm entry.
-  struct Entry {
-    Handle handle;
-    Callback callback;
-    TimePoint expiration;
-
-    Entry(Callback callback, TimePoint expiration);
-    bool operator==(const Entry &rhs) { return handle == rhs.handle; }
-  };
-
-  /// List of alarm entries.
-  std::vector<Entry> m_entries;
-
-  /// Timeout between when an alarm is created and when it fires.
-  Duration m_timeout;
-
-  /// The alarm thread.
-  /// @{
-  HostThread m_alarm_thread;
-  lldb::thread_result_t AlarmThread();
-  /// @}
-
-  /// Synchronize access between the alarm thread and the main thread.
-  std::mutex m_alarm_mutex;
-
-  /// Condition variable used to wake up the alarm thread.
-  std::condition_variable m_alarm_cv;
-
-  /// Flag to signal the alarm thread that something changed and we need to
-  /// recompute the next alarm.
-  bool m_recompute_next_alarm = false;
-
-  /// Flag to signal the alarm thread to exit.
-  bool m_exit = false;
-
-  /// Flag to signal we should run all callbacks on exit.
-  bool m_run_callbacks_on_exit = false;
-};
-
-} // namespace lldb_private
-
-#endif // LLDB_HOST_ALARM_H
diff --git a/lldb/include/lldb/Target/Language.h b/lldb/include/lldb/Target/Language.h
index 38ca458159edc..b699a90aff8e4 100644
--- a/lldb/include/lldb/Target/Language.h
+++ b/lldb/include/lldb/Target/Language.h
@@ -354,14 +354,13 @@ class Language : public PluginInterface {
 
   virtual llvm::StringRef GetInstanceVariableName() { return {}; }
 
-  /// Returns true if this SymbolContext should be ignored when setting
-  /// breakpoints by line (number or regex). Helpful for languages that create
-  /// artificial functions without meaningful user code associated with them
-  /// (e.g. code that gets expanded in late compilation stages, like by
-  /// CoroSplitter).
-  virtual bool IgnoreForLineBreakpoints(const SymbolContext &) const {
-    return false;
-  }
+  /// Given a symbol context list of matches which supposedly represent the
+  /// same file and line number in a CU, erases those that should be ignored
+  /// when setting breakpoints by line (number or regex). Helpful for languages
+  /// that create split a single source-line into many functions (e.g. call
+  /// sites transformed by CoroSplitter).
+  virtual void
+  FilterForLineBreakpoints(llvm::SmallVectorImpl<SymbolContext> &) const {}
 
   /// Returns a boolean indicating whether two symbol contexts are equal for the
   /// purposes of frame comparison. If the plugin has no opinion, it should
diff --git a/lldb/include/lldb/Utility/XcodeSDK.h b/lldb/include/lldb/Utility/XcodeSDK.h
index 3a6cbb9c98f6b..ceb8abb8c502d 100644
--- a/lldb/include/lldb/Utility/XcodeSDK.h
+++ b/lldb/include/lldb/Utility/XcodeSDK.h
@@ -65,7 +65,9 @@ class XcodeSDK {
   /// parameter. For example, "MacOSX.10.14.sdk".
   XcodeSDK(std::string &&name) : m_name(std::move(name)) {}
   XcodeSDK(std::string name, FileSpec sysroot)
-      : m_name(std::move(name)), m_sysroot(std::move(sysroot)) {}
+      : m_name(std::move(name)), m_sysroot(std::move(sysroot)) {
+    assert(!m_sysroot || m_name == m_sysroot.GetFilename().GetStringRef());
+  }
   static XcodeSDK GetAnyMacOS() { return XcodeSDK("MacOSX.sdk"); }
 
   /// The merge function follows a strict order to maintain monotonicity:
diff --git a/lldb/include/lldb/lldb-enumerations.h b/lldb/include/lldb/lldb-enumerations.h
index fecf9cbb765f7..e30ce5be57b69 100644
--- a/lldb/include/lldb/lldb-enumerations.h
+++ b/lldb/include/lldb/lldb-enumerations.h
@@ -1356,9 +1356,9 @@ enum DebuggerBroadcastBit {
   eBroadcastBitWarning = (1 << 1),
   eBroadcastBitError = (1 << 2),
   eBroadcastSymbolChange = (1 << 3),
-  eBroadcastBitProgressCategory = (1 << 4),
+  eBroadcastBitProgressCategory = (1 << 4), ///< Deprecated
   eBroadcastBitExternalProgress = (1 << 5),
-  eBroadcastBitExternalProgressCategory = (1 << 6),
+  eBroadcastBitExternalProgressCategory = (1 << 6), ///< Deprecated
 };
 
 /// Used for expressing severity in logs and diagnostics.
diff --git a/lldb/packages/Python/lldbsuite/test/lldbplatformutil.py b/lldb/packages/Python/lldbsuite/test/lldbplatformutil.py
index 3d8c713562e9b..a1ab06076dcb2 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbplatformutil.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbplatformutil.py
@@ -35,7 +35,7 @@ def check_first_register_readable(test_case):
         test_case.expect("register read r0", substrs=["r0 = 0x"])
     elif arch in ["powerpc64le"]:
         test_case.expect("register read r0", substrs=["r0 = 0x"])
-    elif re.match("^rv(32|64)", arch):
+    elif arch in ["riscv64", "riscv32"]:
         test_case.expect("register read zero", substrs=["zero = 0x"])
     else:
         # TODO: Add check for other architectures
@@ -240,6 +240,10 @@ def getArchitecture():
         arch = "x86_64"
     if arch in ["armv7l", "armv8l"]:
         arch = "arm"
+    if re.match("rv64*", arch):
+        arch = "riscv64"
+    if re.match("rv32*", arch):
+        arch = "riscv32"
     return arch
 
 
diff --git a/lldb/packages/Python/lldbsuite/test/lldbtest.py b/lldb/packages/Python/lldbsuite/test/lldbtest.py
index 81b286340560d..590024ef77119 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbtest.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbtest.py
@@ -1344,6 +1344,13 @@ def isAArch64(self):
         arch = self.getArchitecture().lower()
         return arch in ["aarch64", "arm64", "arm64e"]
 
+    def isARM(self):
+        """Returns true if the architecture is ARM, meaning 32-bit ARM. Which could
+        be M profile, A profile Armv7-a, or the AArch32 mode of Armv8-a."""
+        return not self.isAArch64() and (
+            self.getArchitecture().lower().startswith("arm")
+        )
+
     def isAArch64SVE(self):
         return self.isAArch64() and "sve" in self.getCPUInfo()
 
@@ -1393,6 +1400,10 @@ def isLoongArchLSX(self):
     def isLoongArchLASX(self):
         return self.isLoongArch() and "lasx" in self.getCPUInfo()
 
+    def isRISCV(self):
+        """Returns true if the architecture is RISCV64 or RISCV32."""
+        return self.getArchitecture() in ["riscv64", "riscv32"]
+
     def getArchitecture(self):
         """Returns the architecture in effect the test suite is running with."""
         return lldbplatformutil.getArchitecture()
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py
index fb96b3d4de560..3d3ecb9aa8f95 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py
@@ -682,7 +682,7 @@ def assert_valid_reg_info(self, reg_info):
         self.assertTrue("name" in reg_info)
         self.assertTrue("bitsize" in reg_info)
 
-        if not self.getArchitecture() == "aarch64":
+        if not (self.getArchitecture() == "aarch64" or self.isRISCV()):
             self.assertTrue("offset" in reg_info)
 
         self.assertTrue("encoding" in reg_info)
diff --git a/lldb/source/API/SBLineEntry.cpp b/lldb/source/API/SBLineEntry.cpp
index 216ea6d18eab8..0f4936f32a074 100644
--- a/lldb/source/API/SBLineEntry.cpp
+++ b/lldb/source/API/SBLineEntry.cpp
@@ -137,7 +137,7 @@ void SBLineEntry::SetLine(uint32_t line) {
 void SBLineEntry::SetColumn(uint32_t column) {
   LLDB_INSTRUMENT_VA(this, column);
 
-  ref().line = column;
+  ref().column = column;
 }
 
 bool SBLineEntry::operator==(const SBLineEntry &rhs) const {
diff --git a/lldb/source/API/SystemInitializerFull.cpp b/lldb/source/API/SystemInitializerFull.cpp
index 31f3a9f30b81f..9cc3779d1895f 100644
--- a/lldb/source/API/SystemInitializerFull.cpp
+++ b/lldb/source/API/SystemInitializerFull.cpp
@@ -68,9 +68,6 @@ llvm::Error SystemInitializerFull::Initialize() {
   const char *arg0 = "lldb";
   llvm::cl::ParseCommandLineOptions(1, &arg0);
 
-  // Initialize the progress manager.
-  ProgressManager::Initialize();
-
 #define LLDB_PLUGIN(p) LLDB_PLUGIN_INITIALIZE(p);
 #include "Plugins/Plugins.def"
 
@@ -104,9 +101,6 @@ void SystemInitializerFull::Terminate() {
 #define LLDB_PLUGIN(p) LLDB_PLUGIN_TERMINATE(p);
 #include "Plugins/Plugins.def"
 
-  // Terminate the progress manager.
-  ProgressManager::Terminate();
-
   // Now shutdown the common parts, in reverse order.
   SystemInitializerCommon::Terminate();
 }
diff --git a/lldb/source/Breakpoint/BreakpointResolver.cpp b/lldb/source/Breakpoint/BreakpointResolver.cpp
index 5fe544908c39e..91fdff4a455da 100644
--- a/lldb/source/Breakpoint/BreakpointResolver.cpp
+++ b/lldb/source/Breakpoint/BreakpointResolver.cpp
@@ -207,16 +207,15 @@ bool operator<(const SourceLoc lhs, const SourceLoc rhs) {
 void BreakpointResolver::SetSCMatchesByLine(
     SearchFilter &filter, SymbolContextList &sc_list, bool skip_prologue,
     llvm::StringRef log_ident, uint32_t line, std::optional<uint16_t> column) {
-  llvm::SmallVector<SymbolContext, 16> all_scs;
-
-  for (const auto &sc : sc_list) {
-    if (Language::GetGlobalLanguageProperties()
-            .GetEnableFilterForLineBreakpoints())
-      if (Language *lang = Language::FindPlugin(sc.GetLanguage());
-          lang && lang->IgnoreForLineBreakpoints(sc))
-        continue;
-    all_scs.push_back(sc);
-  }
+  llvm::SmallVector<SymbolContext, 16> all_scs(sc_list.begin(), sc_list.end());
+
+  // Let the language plugin filter `sc_list`. Because all symbol contexts in
+  // sc_list are assumed to belong to the same File, Line and CU, the code below
+  // assumes they have the same language.
+  if (!sc_list.IsEmpty() && Language::GetGlobalLanguageProperties()
+                                .GetEnableFilterForLineBreakpoints())
+    if (Language *lang = Language::FindPlugin(sc_list[0].GetLanguage()))
+      lang->FilterForLineBreakpoints(all_scs);
 
   while (all_scs.size()) {
     uint32_t closest_line = UINT32_MAX;
diff --git a/lldb/source/Commands/CommandObjectDWIMPrint.cpp b/lldb/source/Commands/CommandObjectDWIMPrint.cpp
index 04142427717bd..17c60297a521e 100644
--- a/lldb/source/Commands/CommandObjectDWIMPrint.cpp
+++ b/lldb/source/Commands/CommandObjectDWIMPrint.cpp
@@ -101,6 +101,10 @@ void CommandObjectDWIMPrint::DoExecute(StringRef command,
   // Add a hint if object description was requested, but no description
   // function was implemented.
   auto maybe_add_hint = [&](llvm::StringRef output) {
+    static bool note_shown = false;
+    if (note_shown)
+      return;
+
     // Identify the default output of object description for Swift and
     // Objective-C
     // "<Name: 0x...>. The regex is:
@@ -110,17 +114,14 @@ void CommandObjectDWIMPrint::DoExecute(StringRef command,
     // - Followed by 5 or more hex digits.
     // - Followed by ">".
     // - End with zero or more whitespace characters.
-    const std::regex swift_class_regex("^<\\S+: 0x[[:xdigit:]]{5,}>\\s*$");
+    static const std::regex swift_class_regex(
+        "^<\\S+: 0x[[:xdigit:]]{5,}>\\s*$");
 
     if (GetDebugger().GetShowDontUsePoHint() && target_ptr &&
         (language == lldb::eLanguageTypeSwift ||
          language == lldb::eLanguageTypeObjC) &&
         std::regex_match(output.data(), swift_class_regex)) {
 
-      static bool note_shown = false;
-      if (note_shown)
-        return;
-
       result.AppendNote(
           "object description requested, but type doesn't implement "
           "a custom object description. Consider using \"p\" instead of "
diff --git a/lldb/source/Core/Debugger.cpp b/lldb/source/Core/Debugger.cpp
index dbe3d72e5efa4..8c705f889983a 100644
--- a/lldb/source/Core/Debugger.cpp
+++ b/lldb/source/Core/Debugger.cpp
@@ -1548,7 +1548,7 @@ void Debugger::ReportProgress(uint64_t progress_id, std::string title,
                               std::string details, uint64_t completed,
                               uint64_t total,
                               std::optional<lldb::user_id_t> debugger_id,
-                              uint32_t progress_category_bit) {
+                              uint32_t progress_broadcast_bit) {
   // Check if this progress is for a specific debugger.
   if (debugger_id) {
     // It is debugger specific, grab it and deliver the event if the debugger
@@ -1558,7 +1558,7 @@ void Debugger::ReportProgress(uint64_t progress_id, std::string title,
       PrivateReportProgress(*debugger_sp, progress_id, std::move(title),
                             std::move(details), completed, total,
                             /*is_debugger_specific*/ true,
-                            progress_category_bit);
+                            progress_broadcast_bit);
     return;
   }
   // The progress event is not debugger specific, iterate over all debuggers
@@ -1569,7 +1569,7 @@ void Debugger::ReportProgress(uint64_t progress_id, std::string title,
     for (pos = g_debugger_list_ptr->begin(); pos != end; ++pos)
       PrivateReportProgress(*(*pos), progress_id, title, details, completed,
                             total, /*is_debugger_specific*/ false,
-                            progress_category_bit);
+                            progress_broadcast_bit);
   }
 }
 
diff --git a/lldb/source/Core/Mangled.cpp b/lldb/source/Core/Mangled.cpp
index 51c22495a16d7..ddaaedea04183 100644
--- a/lldb/source/Core/Mangled.cpp
+++ b/lldb/source/Core/Mangled.cpp
@@ -270,50 +270,51 @@ bool Mangled::GetRichManglingInfo(RichManglingContext &context,
 // name. The result is cached and will be kept until a new string value is
 // supplied to this object, or until the end of the object's lifetime.
 ConstString Mangled::GetDemangledName() const {
-  // Check to make sure we have a valid mangled name and that we haven't
-  // already decoded our mangled name.
-  if (m_mangled && m_demangled.IsNull()) {
-    // Don't bother running anything that isn't mangled
-    const char *mangled_name = m_mangled.GetCString();
-    ManglingScheme mangling_scheme =
-        GetManglingScheme(m_mangled.GetStringRef());
-    if (mangling_scheme != eManglingSchemeNone &&
-        !m_mangled.GetMangledCounterpart(m_demangled)) {
-      // We didn't already mangle this name, demangle it and if all goes well
-      // add it to our map.
-      char *demangled_name = nullptr;
-      switch (mangling_scheme) {
-      case eManglingSchemeMSVC:
-        demangled_name = GetMSVCDemangledStr(mangled_name);
-        break;
-      case eManglingSchemeItanium: {
-        demangled_name = GetItaniumDemangledStr(mangled_name);
-        break;
-      }
-      case eManglingSchemeRustV0:
-        demangled_name = GetRustV0DemangledStr(m_mangled);
-        break;
-      case eManglingSchemeD:
-        demangled_name = GetDLangDemangledStr(m_mangled);
-        break;
-      case eManglingSchemeSwift:
-        // Demangling a swift name requires the swift compiler. This is
-        // explicitly unsupported on llvm.org.
-        break;
-      case eManglingSchemeNone:
-        llvm_unreachable("eManglingSchemeNone was handled already");
-      }
-      if (demangled_name) {
-        m_demangled.SetStringWithMangledCounterpart(
-            llvm::StringRef(demangled_name), m_mangled);
-        free(demangled_name);
-      }
-    }
-    if (m_demangled.IsNull()) {
-      // Set the demangled string to the empty string to indicate we tried to
-      // parse it once and failed.
-      m_demangled.SetCString("");
-    }
+  if (!m_mangled)
+    return m_demangled;
+
+  // Re-use previously demangled names.
+  if (!m_demangled.IsNull())
+    return m_demangled;
+
+  if (m_mangled.GetMangledCounterpart(m_demangled) && !m_demangled.IsNull())
+    return m_demangled;
+
+  // We didn't already mangle this name, demangle it and if all goes well
+  // add it to our map.
+  char *demangled_name = nullptr;
+  switch (GetManglingScheme(m_mangled.GetStringRef())) {
+  case eManglingSchemeMSVC:
+    demangled_name = GetMSVCDemangledStr(m_mangled);
+    break;
+  case eManglingSchemeItanium: {
+    demangled_name = GetItaniumDemangledStr(m_mangled.GetCString());
+    break;
+  }
+  case eManglingSchemeRustV0:
+    demangled_name = GetRustV0DemangledStr(m_mangled);
+    break;
+  case eManglingSchemeD:
+    demangled_name = GetDLangDemangledStr(m_mangled);
+    break;
+  case eManglingSchemeSwift:
+    // Demangling a swift name requires the swift compiler. This is
+    // explicitly unsupported on llvm.org.
+    break;
+  case eManglingSchemeNone:
+    // Don't bother demangling anything that isn't mangled.
+    break;
+  }
+
+  if (demangled_name) {
+    m_demangled.SetStringWithMangledCounterpart(demangled_name, m_mangled);
+    free(demangled_name);
+  }
+
+  if (m_demangled.IsNull()) {
+    // Set the demangled string to the empty string to indicate we tried to
+    // parse it once and failed.
+    m_demangled.SetCString("");
   }
 
   return m_demangled;
diff --git a/lldb/source/Core/Module.cpp b/lldb/source/Core/Module.cpp
index 33668c5d20dde..d70f292abaea4 100644
--- a/lldb/source/Core/Module.cpp
+++ b/lldb/source/Core/Module.cpp
@@ -1022,8 +1022,8 @@ SymbolFile *Module::GetSymbolFile(bool can_create, Stream *feedback_strm) {
   return m_symfile_up ? m_symfile_up->GetSymbolFile() : nullptr;
 }
 
-Symtab *Module::GetSymtab() {
-  if (SymbolFile *symbols = GetSymbolFile())
+Symtab *Module::GetSymtab(bool can_create) {
+  if (SymbolFile *symbols = GetSymbolFile(can_create))
     return symbols->GetSymtab();
   return nullptr;
 }
diff --git a/lldb/source/Core/Progress.cpp b/lldb/source/Core/Progress.cpp
index 63f9804320809..d29dce0a688c0 100644
--- a/lldb/source/Core/Progress.cpp
+++ b/lldb/source/Core/Progress.cpp
@@ -31,11 +31,11 @@ Progress::Progress(std::string title, std::string details,
                    Timeout<std::nano> minimum_report_time,
                    Progress::Origin origin)
     : m_total(total.value_or(Progress::kNonDeterministicTotal)),
-      m_minimum_report_time(minimum_report_time),
-      m_progress_data{title, ++g_id,
-                      debugger ? std::optional<user_id_t>(debugger->GetID())
-                               : std::nullopt,
-                      origin},
+      m_minimum_report_time(minimum_report_time), m_title(title),
+      m_progress_id(++g_id),
+      m_debugger_id(debugger ? std::optional<user_id_t>(debugger->GetID())
+                             : std::nullopt),
+      m_origin(origin),
       m_last_report_time_ns(
           std::chrono::nanoseconds(
               std::chrono::steady_clock::now().time_since_epoch())
@@ -44,27 +44,19 @@ Progress::Progress(std::string title, std::string details,
   std::lock_guard<std::mutex> guard(m_mutex);
   ReportProgress();
 
-  // Report to the ProgressManager if that subsystem is enabled.
-  if (ProgressManager::Enabled())
-    ProgressManager::Instance().Increment(m_progress_data);
-
   // Start signpost interval right before the meaningful work starts.
-  g_progress_signposts->startInterval(this, m_progress_data.title);
+  g_progress_signposts->startInterval(this, m_title);
 }
 
 Progress::~Progress() {
   // End signpost interval as soon as possible.
-  g_progress_signposts->endInterval(this, m_progress_data.title);
+  g_progress_signposts->endInterval(this, m_title);
 
   // Make sure to always report progress completed when this object is
   // destructed so it indicates the progress dialog/activity should go away.
   std::lock_guard<std::mutex> guard(m_mutex);
   m_completed = m_total;
   ReportProgress();
-
-  // Report to the ProgressManager if that subsystem is enabled.
-  if (ProgressManager::Enabled())
-    ProgressManager::Instance().Decrement(m_progress_data);
 }
 
 void Progress::Increment(uint64_t amount,
@@ -109,128 +101,11 @@ void Progress::ReportProgress() {
     return; // An overflow in the m_completed counter. Just ignore these events.
 
   // Change the category bit if we're an internal or external progress.
-  uint32_t progress_category_bit =
-      m_progress_data.origin == Progress::Origin::eExternal
-          ? lldb::eBroadcastBitExternalProgress
-          : lldb::eBroadcastBitProgress;
-
-  Debugger::ReportProgress(m_progress_data.progress_id, m_progress_data.title,
-                           m_details, completed, m_total,
-                           m_progress_data.debugger_id, progress_category_bit);
-  m_prev_completed = completed;
-}
-
-ProgressManager::ProgressManager()
-    : m_entries(), m_alarm(std::chrono::milliseconds(100)) {}
-
-ProgressManager::~ProgressManager() {}
-
-void ProgressManager::Initialize() {
-  assert(!InstanceImpl() && "Already initialized.");
-  InstanceImpl().emplace();
-}
-
-void ProgressManager::Terminate() {
-  assert(InstanceImpl() && "Already terminated.");
-  InstanceImpl().reset();
-}
-
-bool ProgressManager::Enabled() { return InstanceImpl().operator bool(); }
-
-ProgressManager &ProgressManager::Instance() {
-  assert(InstanceImpl() && "ProgressManager must be initialized");
-  return *InstanceImpl();
-}
-
-std::optional<ProgressManager> &ProgressManager::InstanceImpl() {
-  static std::optional<ProgressManager> g_progress_manager;
-  return g_progress_manager;
-}
-
-void ProgressManager::Increment(const Progress::ProgressData &progress_data) {
-  std::lock_guard<std::mutex> lock(m_entries_mutex);
-
-  llvm::StringRef key = progress_data.title;
-  bool new_entry = !m_entries.contains(key);
-  Entry &entry = m_entries[progress_data.title];
-
-  if (new_entry) {
-    // This is a new progress event. Report progress and store the progress
-    // data.
-    ReportProgress(progress_data, EventType::Begin);
-    entry.data = progress_data;
-  } else if (entry.refcount == 0) {
-    // This is an existing entry that was scheduled to be deleted but a new one
-    // came in before the timer expired.
-    assert(entry.handle != Alarm::INVALID_HANDLE);
-
-    if (!m_alarm.Cancel(entry.handle)) {
-      // The timer expired before we had a chance to cancel it. We have to treat
-      // this as an entirely new progress event.
-      ReportProgress(progress_data, EventType::Begin);
-    }
-    // Clear the alarm handle.
-    entry.handle = Alarm::INVALID_HANDLE;
-  }
-
-  // Regardless of how we got here, we need to bump the reference count.
-  entry.refcount++;
-}
-
-void ProgressManager::Decrement(const Progress::ProgressData &progress_data) {
-  std::lock_guard<std::mutex> lock(m_entries_mutex);
-  llvm::StringRef key = progress_data.title;
-
-  auto it = m_entries.find(key);
-  if (it == m_entries.end())
-    return;
-
-  Entry &entry = it->second;
-  entry.refcount--;
-
-  if (entry.refcount == 0) {
-    assert(entry.handle == Alarm::INVALID_HANDLE);
-
-    // Copy the key to a std::string so we can pass it by value to the lambda.
-    // The underlying StringRef will not exist by the time the callback is
-    // called.
-    std::string key_str = std::string(key);
-
-    // Start a timer. If it expires before we see another progress event, it
-    // will be reported.
-    entry.handle = m_alarm.Create([=]() { Expire(key_str); });
-  }
-}
-
-void ProgressManager::ReportProgress(
-    const Progress::ProgressData &progress_data, EventType type) {
-  // The category bit only keeps track of when progress report categories have
-  // started and ended, so clear the details and reset other fields when
-  // broadcasting to it since that bit doesn't need that information.
-  const uint64_t completed =
-      (type == EventType::Begin) ? 0 : Progress::kNonDeterministicTotal;
-  const uint32_t progress_category_bit =
-      progress_data.origin == Progress::Origin::eExternal
-          ? lldb::eBroadcastBitExternalProgressCategory
-          : lldb::eBroadcastBitProgressCategory;
-  Debugger::ReportProgress(progress_data.progress_id, progress_data.title, "",
-                           completed, Progress::kNonDeterministicTotal,
-                           progress_data.debugger_id, progress_category_bit);
-}
+  uint32_t progress_category_bit = m_origin == Progress::Origin::eExternal
+                                       ? lldb::eBroadcastBitExternalProgress
+                                       : lldb::eBroadcastBitProgress;
 
-void ProgressManager::Expire(llvm::StringRef key) {
-  std::lock_guard<std::mutex> lock(m_entries_mutex);
-
-  // This shouldn't happen but be resilient anyway.
-  if (!m_entries.contains(key))
-    return;
-
-  // A new event came in and the alarm fired before we had a chance to restart
-  // it.
-  if (m_entries[key].refcount != 0)
-    return;
-
-  // We're done with this entry.
-  ReportProgress(m_entries[key].data, EventType::End);
-  m_entries.erase(key);
+  Debugger::ReportProgress(m_progress_id, m_title, m_details, completed,
+                           m_total, m_debugger_id, progress_category_bit);
+  m_prev_completed = completed;
 }
diff --git a/lldb/source/Core/Section.cpp b/lldb/source/Core/Section.cpp
index 608e2a5fc3093..27dcf987b0278 100644
--- a/lldb/source/Core/Section.cpp
+++ b/lldb/source/Core/Section.cpp
@@ -160,10 +160,9 @@ const char *Section::GetTypeAsCString() const {
 }
 
 Section::Section(const ModuleSP &module_sp, ObjectFile *obj_file,
-                 user_id_t sect_id, ConstString name,
-                 SectionType sect_type, addr_t file_addr, addr_t byte_size,
-                 lldb::offset_t file_offset, lldb::offset_t file_size,
-                 uint32_t log2align, uint32_t flags,
+                 user_id_t sect_id, ConstString name, SectionType sect_type,
+                 addr_t file_addr, addr_t byte_size, lldb::offset_t file_offset,
+                 lldb::offset_t file_size, uint32_t log2align, uint32_t flags,
                  uint32_t target_byte_size /*=1*/)
     : ModuleChild(module_sp), UserID(sect_id), Flags(flags),
       m_obj_file(obj_file), m_type(sect_type), m_parent_wp(), m_name(name),
@@ -171,15 +170,14 @@ Section::Section(const ModuleSP &module_sp, ObjectFile *obj_file,
       m_file_offset(file_offset), m_file_size(file_size),
       m_log2align(log2align), m_children(), m_fake(false), m_encrypted(false),
       m_thread_specific(false), m_readable(false), m_writable(false),
-      m_executable(false), m_relocated(false), m_target_byte_size(target_byte_size) {
-}
+      m_executable(false), m_relocated(false),
+      m_target_byte_size(target_byte_size) {}
 
 Section::Section(const lldb::SectionSP &parent_section_sp,
                  const ModuleSP &module_sp, ObjectFile *obj_file,
-                 user_id_t sect_id, ConstString name,
-                 SectionType sect_type, addr_t file_addr, addr_t byte_size,
-                 lldb::offset_t file_offset, lldb::offset_t file_size,
-                 uint32_t log2align, uint32_t flags,
+                 user_id_t sect_id, ConstString name, SectionType sect_type,
+                 addr_t file_addr, addr_t byte_size, lldb::offset_t file_offset,
+                 lldb::offset_t file_size, uint32_t log2align, uint32_t flags,
                  uint32_t target_byte_size /*=1*/)
     : ModuleChild(module_sp), UserID(sect_id), Flags(flags),
       m_obj_file(obj_file), m_type(sect_type), m_parent_wp(), m_name(name),
@@ -187,7 +185,8 @@ Section::Section(const lldb::SectionSP &parent_section_sp,
       m_file_offset(file_offset), m_file_size(file_size),
       m_log2align(log2align), m_children(), m_fake(false), m_encrypted(false),
       m_thread_specific(false), m_readable(false), m_writable(false),
-      m_executable(false), m_relocated(false), m_target_byte_size(target_byte_size) {
+      m_executable(false), m_relocated(false),
+      m_target_byte_size(target_byte_size) {
   if (parent_section_sp)
     m_parent_wp = parent_section_sp;
 }
@@ -469,7 +468,6 @@ bool Section::ContainsOnlyDebugInfo() const {
   return false;
 }
 
-
 #pragma mark SectionList
 
 SectionList &SectionList::operator=(const SectionList &rhs) {
@@ -554,8 +552,7 @@ SectionSP SectionList::GetSectionAtIndex(size_t idx) const {
   return sect_sp;
 }
 
-SectionSP
-SectionList::FindSectionByName(ConstString section_dstr) const {
+SectionSP SectionList::FindSectionByName(ConstString section_dstr) const {
   SectionSP sect_sp;
   // Check if we have a valid section string
   if (section_dstr && !m_sections.empty()) {
@@ -693,7 +690,14 @@ bool fromJSON(const llvm::json::Value &value,
          o.map("address", section.address) && o.map("size", section.size) &&
          o.map("read", section.read) && o.map("write", section.write) &&
          o.map("execute", section.execute) &&
-         o.mapOptional("subsections", section.subsections);
+         o.mapOptional("subsections", section.subsections) &&
+         o.map("user_id", section.user_id) &&
+         o.map("file_offset", section.file_offset) &&
+         o.map("file_size", section.file_size) &&
+         o.map("alignment", section.log2align) &&
+         o.map("flags", section.flags) && o.map("fake", section.fake) &&
+         o.map("encrypted", section.encrypted) &&
+         o.map("thread_specific", section.thread_specific);
 }
 
 bool fromJSON(const llvm::json::Value &value, lldb::SectionType &type,
diff --git a/lldb/source/Core/Telemetry.cpp b/lldb/source/Core/Telemetry.cpp
index 367e94d6566de..be929a644706e 100644
--- a/lldb/source/Core/Telemetry.cpp
+++ b/lldb/source/Core/Telemetry.cpp
@@ -60,6 +60,22 @@ void LLDBBaseTelemetryInfo::serialize(Serializer &serializer) const {
     serializer.write("end_time", ToNanosec(end_time.value()));
 }
 
+void CommandInfo::serialize(Serializer &serializer) const {
+  LLDBBaseTelemetryInfo::serialize(serializer);
+
+  serializer.write("target_uuid", target_uuid.GetAsString());
+  serializer.write("command_id", command_id);
+  serializer.write("command_name", command_name);
+  if (original_command.has_value())
+    serializer.write("original_command", original_command.value());
+  if (args.has_value())
+    serializer.write("args", args.value());
+  if (ret_status.has_value())
+    serializer.write("ret_status", ret_status.value());
+  if (error_data.has_value())
+    serializer.write("error_data", error_data.value());
+}
+
 void DebuggerInfo::serialize(Serializer &serializer) const {
   LLDBBaseTelemetryInfo::serialize(serializer);
 
@@ -67,7 +83,10 @@ void DebuggerInfo::serialize(Serializer &serializer) const {
   serializer.write("is_exit_entry", is_exit_entry);
 }
 
-TelemetryManager::TelemetryManager(std::unique_ptr<Config> config)
+std::atomic<uint64_t> CommandInfo::g_command_id_seed = 0;
+uint64_t CommandInfo::GetNextId() { return g_command_id_seed.fetch_add(1); }
+
+TelemetryManager::TelemetryManager(std::unique_ptr<LLDBConfig> config)
     : m_config(std::move(config)), m_id(MakeUUID()) {}
 
 llvm::Error TelemetryManager::preDispatch(TelemetryInfo *entry) {
@@ -79,23 +98,41 @@ llvm::Error TelemetryManager::preDispatch(TelemetryInfo *entry) {
   return llvm::Error::success();
 }
 
-const Config *TelemetryManager::GetConfig() { return m_config.get(); }
+class NoOpTelemetryManager : public TelemetryManager {
+public:
+  llvm::Error preDispatch(llvm::telemetry::TelemetryInfo *entry) override {
+    // Does nothing.
+    return llvm::Error::success();
+  }
 
-std::unique_ptr<TelemetryManager> TelemetryManager::g_instance = nullptr;
-TelemetryManager *TelemetryManager::GetInstance() {
-  if (!Config::BuildTimeEnableTelemetry)
-    return nullptr;
-  return g_instance.get();
-}
+  explicit NoOpTelemetryManager()
+      : TelemetryManager(std::make_unique<LLDBConfig>(
+            /*EnableTelemetry*/ false, /*DetailedCommand*/ false)) {}
+
+  virtual llvm::StringRef GetInstanceName() const override {
+    return "NoOpTelemetryManager";
+  }
+
+  llvm::Error dispatch(llvm::telemetry::TelemetryInfo *entry) override {
+    // Does nothing.
+    return llvm::Error::success();
+  }
 
-TelemetryManager *TelemetryManager::GetInstanceIfEnabled() {
-  // Telemetry may be enabled at build time but disabled at runtime.
-  if (TelemetryManager *ins = TelemetryManager::GetInstance()) {
-    if (ins->GetConfig()->EnableTelemetry)
-      return ins;
+  static NoOpTelemetryManager *GetInstance() {
+    static std::unique_ptr<NoOpTelemetryManager> g_ins =
+        std::make_unique<NoOpTelemetryManager>();
+    return g_ins.get();
   }
+};
 
-  return nullptr;
+std::unique_ptr<TelemetryManager> TelemetryManager::g_instance = nullptr;
+TelemetryManager *TelemetryManager::GetInstance() {
+  // If Telemetry is disabled or if there is no default instance, then use the
+  // NoOp manager. We use a dummy instance to avoid having to do nullchecks in
+  // various places.
+  if (!Config::BuildTimeEnableTelemetry || !g_instance)
+    return NoOpTelemetryManager::GetInstance();
+  return g_instance.get();
 }
 
 void TelemetryManager::SetInstance(std::unique_ptr<TelemetryManager> manager) {
diff --git a/lldb/source/Host/CMakeLists.txt b/lldb/source/Host/CMakeLists.txt
index f4be151756b3b..52ef67feeb6ab 100644
--- a/lldb/source/Host/CMakeLists.txt
+++ b/lldb/source/Host/CMakeLists.txt
@@ -13,7 +13,6 @@ macro(add_host_subdirectory group)
 endmacro()
 
 add_host_subdirectory(common
-  common/Alarm.cpp
   common/FileAction.cpp
   common/FileCache.cpp
   common/File.cpp
@@ -136,6 +135,7 @@ else()
 
   elseif (CMAKE_SYSTEM_NAME MATCHES "AIX")
     add_host_subdirectory(aix
+      aix/Host.cpp
       aix/HostInfoAIX.cpp
       )
   endif()
diff --git a/lldb/source/Host/aix/Host.cpp b/lldb/source/Host/aix/Host.cpp
new file mode 100644
index 0000000000000..751c4fbcc9368
--- /dev/null
+++ b/lldb/source/Host/aix/Host.cpp
@@ -0,0 +1,25 @@
+//===-- source/Host/aix/Host.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Host/Host.h"
+#include "lldb/Utility/Status.h"
+
+using namespace lldb_private;
+
+uint32_t Host::FindProcessesImpl(const ProcessInstanceInfoMatch &match_info,
+                                 ProcessInstanceInfoList &process_infos) {
+  return 0;
+}
+
+bool Host::GetProcessInfo(lldb::pid_t pid, ProcessInstanceInfo &process_info) {
+  return false;
+}
+
+Status Host::ShellExpandArguments(ProcessLaunchInfo &launch_info) {
+  return Status("unimplemented");
+}
diff --git a/lldb/source/Host/common/Alarm.cpp b/lldb/source/Host/common/Alarm.cpp
deleted file mode 100644
index afc770d20d7b1..0000000000000
--- a/lldb/source/Host/common/Alarm.cpp
+++ /dev/null
@@ -1,222 +0,0 @@
-//===-- Alarm.cpp ---------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lldb/Host/Alarm.h"
-#include "lldb/Host/ThreadLauncher.h"
-#include "lldb/Utility/LLDBLog.h"
-#include "lldb/Utility/Log.h"
-
-using namespace lldb;
-using namespace lldb_private;
-
-Alarm::Alarm(Duration timeout, bool run_callback_on_exit)
-    : m_timeout(timeout), m_run_callbacks_on_exit(run_callback_on_exit) {
-  StartAlarmThread();
-}
-
-Alarm::~Alarm() { StopAlarmThread(); }
-
-Alarm::Handle Alarm::Create(std::function<void()> callback) {
-  // Gracefully deal with the unlikely event that the alarm thread failed to
-  // launch.
-  if (!AlarmThreadRunning())
-    return INVALID_HANDLE;
-
-  // Compute the next expiration before we take the lock. This ensures that
-  // waiting on the lock doesn't eat into the timeout.
-  const TimePoint expiration = GetNextExpiration();
-
-  Handle handle = INVALID_HANDLE;
-
-  {
-    std::lock_guard<std::mutex> alarm_guard(m_alarm_mutex);
-
-    // Create a new unique entry and remember its handle.
-    m_entries.emplace_back(callback, expiration);
-    handle = m_entries.back().handle;
-
-    // Tell the alarm thread we need to recompute the next alarm.
-    m_recompute_next_alarm = true;
-  }
-
-  m_alarm_cv.notify_one();
-  return handle;
-}
-
-bool Alarm::Restart(Handle handle) {
-  // Gracefully deal with the unlikely event that the alarm thread failed to
-  // launch.
-  if (!AlarmThreadRunning())
-    return false;
-
-  // Compute the next expiration before we take the lock. This ensures that
-  // waiting on the lock doesn't eat into the timeout.
-  const TimePoint expiration = GetNextExpiration();
-
-  {
-    std::lock_guard<std::mutex> alarm_guard(m_alarm_mutex);
-
-    // Find the entry corresponding to the given handle.
-    const auto it =
-        std::find_if(m_entries.begin(), m_entries.end(),
-                     [handle](Entry &entry) { return entry.handle == handle; });
-    if (it == m_entries.end())
-      return false;
-
-    // Update the expiration.
-    it->expiration = expiration;
-
-    // Tell the alarm thread we need to recompute the next alarm.
-    m_recompute_next_alarm = true;
-  }
-
-  m_alarm_cv.notify_one();
-  return true;
-}
-
-bool Alarm::Cancel(Handle handle) {
-  // Gracefully deal with the unlikely event that the alarm thread failed to
-  // launch.
-  if (!AlarmThreadRunning())
-    return false;
-
-  {
-    std::lock_guard<std::mutex> alarm_guard(m_alarm_mutex);
-
-    const auto it =
-        std::find_if(m_entries.begin(), m_entries.end(),
-                     [handle](Entry &entry) { return entry.handle == handle; });
-
-    if (it == m_entries.end())
-      return false;
-
-    m_entries.erase(it);
-  }
-
-  // No need to notify the alarm thread. This only affects the alarm thread if
-  // we removed the entry that corresponds to the next alarm. If that's the
-  // case, the thread will wake up as scheduled, find no expired events, and
-  // recompute the next alarm time.
-  return true;
-}
-
-Alarm::Entry::Entry(Alarm::Callback callback, Alarm::TimePoint expiration)
-    : handle(Alarm::GetNextUniqueHandle()), callback(std::move(callback)),
-      expiration(std::move(expiration)) {}
-
-void Alarm::StartAlarmThread() {
-  if (!m_alarm_thread.IsJoinable()) {
-    llvm::Expected<HostThread> alarm_thread = ThreadLauncher::LaunchThread(
-        "lldb.debugger.alarm-thread", [this] { return AlarmThread(); },
-        8 * 1024 * 1024); // Use larger 8MB stack for this thread
-    if (alarm_thread) {
-      m_alarm_thread = *alarm_thread;
-    } else {
-      LLDB_LOG_ERROR(GetLog(LLDBLog::Host), alarm_thread.takeError(),
-                     "failed to launch host thread: {0}");
-    }
-  }
-}
-
-void Alarm::StopAlarmThread() {
-  if (m_alarm_thread.IsJoinable()) {
-    {
-      std::lock_guard<std::mutex> alarm_guard(m_alarm_mutex);
-      m_exit = true;
-    }
-    m_alarm_cv.notify_one();
-    m_alarm_thread.Join(nullptr);
-  }
-}
-
-bool Alarm::AlarmThreadRunning() { return m_alarm_thread.IsJoinable(); }
-
-lldb::thread_result_t Alarm::AlarmThread() {
-  bool exit = false;
-  std::optional<TimePoint> next_alarm;
-
-  const auto predicate = [this] { return m_exit || m_recompute_next_alarm; };
-
-  while (!exit) {
-    // Synchronization between the main thread and the alarm thread using a
-    // mutex and condition variable. There are 2 reasons the thread can wake up:
-    //
-    // 1. The timeout for the next alarm expired.
-    //
-    // 2. The condition variable is notified that one of our shared variables
-    //    (see predicate) was modified. Either the thread is asked to shut down
-    //    or a new alarm came in and we need to recompute the next timeout.
-    //
-    // Below we only deal with the timeout expiring and fall through for dealing
-    // with the rest.
-    llvm::SmallVector<Callback, 1> callbacks;
-    {
-      std::unique_lock<std::mutex> alarm_lock(m_alarm_mutex);
-      if (next_alarm) {
-        if (!m_alarm_cv.wait_until(alarm_lock, *next_alarm, predicate)) {
-          // The timeout for the next alarm expired.
-
-          // Clear the next timeout to signal that we need to recompute the next
-          // timeout.
-          next_alarm.reset();
-
-          // Iterate over all the callbacks. Call the ones that have expired
-          // and remove them from the list.
-          const TimePoint now = std::chrono::system_clock::now();
-          auto it = m_entries.begin();
-          while (it != m_entries.end()) {
-            if (it->expiration <= now) {
-              callbacks.emplace_back(std::move(it->callback));
-              it = m_entries.erase(it);
-            } else {
-              it++;
-            }
-          }
-        }
-      } else {
-        m_alarm_cv.wait(alarm_lock, predicate);
-      }
-
-      // Fall through after waiting on the condition variable. At this point
-      // either the predicate is true or we woke up because an alarm expired.
-
-      // The alarm thread is shutting down.
-      if (m_exit) {
-        exit = true;
-        if (m_run_callbacks_on_exit) {
-          for (Entry &entry : m_entries)
-            callbacks.emplace_back(std::move(entry.callback));
-        }
-      }
-
-      // A new alarm was added or an alarm expired. Either way we need to
-      // recompute when this thread should wake up for the next alarm.
-      if (m_recompute_next_alarm || !next_alarm) {
-        for (Entry &entry : m_entries) {
-          if (!next_alarm || entry.expiration < *next_alarm)
-            next_alarm = entry.expiration;
-        }
-        m_recompute_next_alarm = false;
-      }
-    }
-
-    // Outside the lock, call the callbacks.
-    for (Callback &callback : callbacks)
-      callback();
-  }
-  return {};
-}
-
-Alarm::TimePoint Alarm::GetNextExpiration() const {
-  return std::chrono::system_clock::now() + m_timeout;
-}
-
-Alarm::Handle Alarm::GetNextUniqueHandle() {
-  static std::atomic<Handle> g_next_handle = 1;
-  return g_next_handle++;
-}
diff --git a/lldb/source/Host/macosx/objcxx/MemoryMonitorMacOSX.mm b/lldb/source/Host/macosx/objcxx/MemoryMonitorMacOSX.mm
index cb6c2457df1e3..c3a430bda7fa5 100644
--- a/lldb/source/Host/macosx/objcxx/MemoryMonitorMacOSX.mm
+++ b/lldb/source/Host/macosx/objcxx/MemoryMonitorMacOSX.mm
@@ -32,7 +32,6 @@ void Start() override {
       }
     });
     dispatch_activate(m_memory_pressure_source);
-    printf("Started\n");
   }
 
   void Stop() override {
diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp
index c363f20081f9e..24cf755d1f740 100644
--- a/lldb/source/Interpreter/CommandInterpreter.cpp
+++ b/lldb/source/Interpreter/CommandInterpreter.cpp
@@ -46,7 +46,9 @@
 #include "Commands/CommandObjectWatchpoint.h"
 
 #include "lldb/Core/Debugger.h"
+#include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
+#include "lldb/Core/Telemetry.h"
 #include "lldb/Host/StreamFile.h"
 #include "lldb/Utility/ErrorMessages.h"
 #include "lldb/Utility/LLDBLog.h"
@@ -88,6 +90,7 @@
 #include "llvm/Support/Path.h"
 #include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/ScopedPrinter.h"
+#include "llvm/Telemetry/Telemetry.h"
 
 #if defined(__APPLE__)
 #include <TargetConditionals.h>
@@ -1883,9 +1886,48 @@ bool CommandInterpreter::HandleCommand(const char *command_line,
                                        LazyBool lazy_add_to_history,
                                        CommandReturnObject &result,
                                        bool force_repeat_command) {
+  telemetry::ScopedDispatcher<telemetry::CommandInfo> helper(&m_debugger);
+  const bool detailed_command_telemetry =
+      telemetry::TelemetryManager::GetInstance()
+          ->GetConfig()
+          ->detailed_command_telemetry;
+  const int command_id = telemetry::CommandInfo::GetNextId();
+
   std::string command_string(command_line);
   std::string original_command_string(command_string);
   std::string real_original_command_string(command_string);
+  std::string parsed_command_args;
+  CommandObject *cmd_obj = nullptr;
+
+  helper.DispatchNow([&](lldb_private::telemetry::CommandInfo *info) {
+    info->command_id = command_id;
+    if (Target *target = GetExecutionContext().GetTargetPtr()) {
+      // If we have a target attached to this command, then get the UUID.
+      info->target_uuid = target->GetExecutableModule() != nullptr
+                              ? target->GetExecutableModule()->GetUUID()
+                              : UUID();
+    }
+    if (detailed_command_telemetry)
+      info->original_command = original_command_string;
+    // The rest (eg., command_name, args, etc) hasn't been parsed yet;
+    // Those will be collected by the on-exit-callback.
+  });
+
+  helper.DispatchOnExit([&](lldb_private::telemetry::CommandInfo *info) {
+    // TODO: this is logging the time the command-handler finishes.
+    // But we may want a finer-grain durations too?
+    // (ie., the execute_time recorded below?)
+    info->command_id = command_id;
+    llvm::StringRef command_name =
+        cmd_obj ? cmd_obj->GetCommandName() : "<not found>";
+    info->command_name = command_name.str();
+    info->ret_status = result.GetStatus();
+    if (std::string error_str = result.GetErrorString(); !error_str.empty())
+      info->error_data = std::move(error_str);
+
+    if (detailed_command_telemetry)
+      info->args = parsed_command_args;
+  });
 
   Log *log = GetLog(LLDBLog::Commands);
   LLDB_LOGF(log, "Processing command: %s", command_line);
@@ -1991,7 +2033,7 @@ bool CommandInterpreter::HandleCommand(const char *command_line,
   // From 1 above, we can determine whether the Execute function wants raw
   // input or not.
 
-  CommandObject *cmd_obj = ResolveCommandImpl(command_string, result);
+  cmd_obj = ResolveCommandImpl(command_string, result);
 
   // We have to preprocess the whole command string for Raw commands, since we
   // don't know the structure of the command.  For parsed commands, we only
@@ -2053,37 +2095,36 @@ bool CommandInterpreter::HandleCommand(const char *command_line,
     if (add_to_history)
       m_command_history.AppendString(original_command_string);
 
-    std::string remainder;
     const std::size_t actual_cmd_name_len = cmd_obj->GetCommandName().size();
     if (actual_cmd_name_len < command_string.length())
-      remainder = command_string.substr(actual_cmd_name_len);
+      parsed_command_args = command_string.substr(actual_cmd_name_len);
 
     // Remove any initial spaces
-    size_t pos = remainder.find_first_not_of(k_white_space);
+    size_t pos = parsed_command_args.find_first_not_of(k_white_space);
     if (pos != 0 && pos != std::string::npos)
-      remainder.erase(0, pos);
+      parsed_command_args.erase(0, pos);
 
     LLDB_LOGF(
         log, "HandleCommand, command line after removing command name(s): '%s'",
-        remainder.c_str());
+        parsed_command_args.c_str());
 
     // To test whether or not transcript should be saved, `transcript_item` is
     // used instead of `GetSaveTranscript()`. This is because the latter will
     // fail when the command is "settings set interpreter.save-transcript true".
     if (transcript_item) {
       transcript_item->AddStringItem("commandName", cmd_obj->GetCommandName());
-      transcript_item->AddStringItem("commandArguments", remainder);
+      transcript_item->AddStringItem("commandArguments", parsed_command_args);
     }
 
     ElapsedTime elapsed(execute_time);
     cmd_obj->SetOriginalCommandString(real_original_command_string);
     // Set the indent to the position of the command in the command line.
-    pos = real_original_command_string.rfind(remainder);
+    pos = real_original_command_string.rfind(parsed_command_args);
     std::optional<uint16_t> indent;
     if (pos != std::string::npos)
       indent = pos;
     result.SetDiagnosticIndent(indent);
-    cmd_obj->Execute(remainder.c_str(), result);
+    cmd_obj->Execute(parsed_command_args.c_str(), result);
   }
 
   LLDB_LOGF(log, "HandleCommand, command %s",
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
index ff17028e6662a..c91a29ace1f68 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
@@ -458,7 +458,14 @@ __lldb_apple_objc_v2_get_shared_cache_class_info (void *objc_opt_ro_ptr,
 
         if (objc_opt->version == 16)
         {
-            const objc_clsopt_v16_t* clsopt = (const objc_clsopt_v16_t*)((uint8_t *)objc_opt + objc_opt_v16->largeSharedCachesClassOffset);
+            int32_t large_offset = objc_opt_v16->largeSharedCachesClassOffset;
+            const objc_clsopt_v16_t* clsopt = (const objc_clsopt_v16_t*)((uint8_t *)objc_opt + large_offset);
+            // Work around a bug in some version shared cache builder where the offset overflows 2GiB (rdar://146432183).
+            uint32_t unsigned_offset = (uint32_t)large_offset;
+            if (unsigned_offset > 0x7fffffff && unsigned_offset < 0x82000000) {
+               clsopt = (const objc_clsopt_v16_t*)((uint8_t *)objc_opt + unsigned_offset);
+               DEBUG_PRINTF("warning: applying largeSharedCachesClassOffset overflow workaround!\n");
+            }
             const size_t max_class_infos = class_infos_byte_size/sizeof(ClassInfo);
 
             DEBUG_PRINTF("max_class_infos = %llu\n", (uint64_t)max_class_infos);
diff --git a/lldb/source/Plugins/ObjectFile/JSON/ObjectFileJSON.cpp b/lldb/source/Plugins/ObjectFile/JSON/ObjectFileJSON.cpp
index 0f9676b836b50..cb8ba05d461d4 100644
--- a/lldb/source/Plugins/ObjectFile/JSON/ObjectFileJSON.cpp
+++ b/lldb/source/Plugins/ObjectFile/JSON/ObjectFileJSON.cpp
@@ -179,39 +179,35 @@ void ObjectFileJSON::CreateSections(SectionList &unified_section_list) {
     return;
   m_sections_up = std::make_unique<SectionList>();
 
-  lldb::user_id_t id = 1;
+  lldb::user_id_t id = 0;
   for (const auto &json_section : m_sections) {
     auto make_section = [this, &id](const JSONSection &section,
                                     SectionSP parent_section_sp =
                                         nullptr) -> SectionSP {
       SectionSP section_sp;
+      auto sect_id = section.user_id.value_or(id + 1);
+      if (!section.user_id.has_value())
+        ++id;
+      const auto name = ConstString(section.name);
+      const auto sect_type = section.type.value_or(eSectionTypeCode);
+      const auto vm_addr = section.address.value_or(0);
+      const auto vm_size = section.size.value_or(0);
+      const auto file_offset = section.file_offset.value_or(0);
+      const auto file_size = section.file_size.value_or(0);
+      const auto log2align = section.log2align.value_or(0);
+      const auto flags = section.flags.value_or(0);
       if (parent_section_sp) {
         section_sp = std::make_shared<Section>(
-            parent_section_sp, GetModule(), this,
-            /*sect_id=*/id++,
-            /*name=*/ConstString(section.name),
-            /*sect_type=*/section.type.value_or(eSectionTypeCode),
-            /*file_vm_addr=*/section.address.value_or(0) -
-                parent_section_sp->GetFileAddress(),
-            /*vm_size=*/section.size.value_or(0),
-            /*file_offset=*/0,
-            /*file_size=*/0,
-            /*log2align=*/0,
-            /*flags=*/0);
+            parent_section_sp, GetModule(), this, sect_id, name, sect_type,
+            vm_addr - parent_section_sp->GetFileAddress(), vm_size, file_offset,
+            file_size, log2align, flags);
 
       } else {
         section_sp = std::make_shared<Section>(
-            GetModule(), this,
-            /*sect_id=*/id++,
-            /*name=*/ConstString(section.name),
-            /*sect_type=*/section.type.value_or(eSectionTypeCode),
-            /*file_vm_addr=*/section.address.value_or(0),
-            /*vm_size=*/section.size.value_or(0),
-            /*file_offset=*/0,
-            /*file_size=*/0,
-            /*log2align=*/0,
-            /*flags=*/0);
+            GetModule(), this, sect_id, name, sect_type, vm_addr, vm_size,
+            file_offset, file_size, log2align, flags);
       }
+      // Set permissions
       uint32_t permissions = 0;
       if (section.read.value_or(0))
         permissions |= lldb::ePermissionsReadable;
@@ -221,6 +217,9 @@ void ObjectFileJSON::CreateSections(SectionList &unified_section_list) {
         permissions |= lldb::ePermissionsExecutable;
       if (permissions)
         section_sp->SetPermissions(permissions);
+      section_sp->SetIsFake(section.fake.value_or(false));
+      section_sp->SetIsEncrypted(section.encrypted.value_or(false));
+      section_sp->SetIsThreadSpecific(section.thread_specific.value_or(false));
       return section_sp;
     };
     auto section_sp = make_section(json_section);
diff --git a/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.cpp b/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.cpp
index fe1c7256e22bc..c54eb659e1744 100644
--- a/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.cpp
+++ b/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.cpp
@@ -86,10 +86,12 @@ bool UnwindAssemblyInstEmulation::GetNonCallSiteUnwindPlanFromAssembly(
   const bool show_address = true;
   const bool show_bytes = true;
   const bool show_control_flow_kind = false;
-  m_cfa_reg_info = *m_inst_emulator_up->GetRegisterInfo(
+
+  m_state.cfa_reg_info = *m_inst_emulator_up->GetRegisterInfo(
       unwind_plan.GetRegisterKind(), unwind_plan.GetInitialCFARegister());
-  m_fp_is_cfa = false;
-  m_register_values.clear();
+  m_state.fp_is_cfa = false;
+  m_state.register_values.clear();
+
   m_pushed_regs.clear();
 
   // Initialize the CFA with a known value. In the 32 bit case it will be
@@ -97,8 +99,8 @@ bool UnwindAssemblyInstEmulation::GetNonCallSiteUnwindPlanFromAssembly(
   // byte size to be safe for any future address sizes
   m_initial_sp = (1ull << ((addr_byte_size * 8) - 1));
   RegisterValue cfa_reg_value;
-  cfa_reg_value.SetUInt(m_initial_sp, m_cfa_reg_info.byte_size);
-  SetRegisterValue(m_cfa_reg_info, cfa_reg_value);
+  cfa_reg_value.SetUInt(m_initial_sp, m_state.cfa_reg_info.byte_size);
+  SetRegisterValue(m_state.cfa_reg_info, cfa_reg_value);
 
   const InstructionList &inst_list = disasm_sp->GetInstructionList();
   const size_t num_instructions = inst_list.GetSize();
@@ -107,33 +109,25 @@ bool UnwindAssemblyInstEmulation::GetNonCallSiteUnwindPlanFromAssembly(
     Instruction *inst = inst_list.GetInstructionAtIndex(0).get();
     const lldb::addr_t base_addr = inst->GetAddress().GetFileAddress();
 
-    // Map for storing the unwind plan row and the value of the registers at a
-    // given offset. When we see a forward branch we add a new entry to this map
-    // with the actual unwind plan row and register context for the target
-    // address of the branch as the current data have to be valid for the target
-    // address of the branch too if we are in the same function.
-    std::map<lldb::addr_t, std::pair<UnwindPlan::RowSP, RegisterValueMap>>
-        saved_unwind_states;
+    // Map for storing the unwind state at a given offset. When we see a forward
+    // branch we add a new entry to this map with the actual unwind plan row and
+    // register context for the target address of the branch as the current data
+    // have to be valid for the target address of the branch too if we are in
+    // the same function.
+    std::map<lldb::addr_t, UnwindState> saved_unwind_states;
 
-    // Make a copy of the current instruction Row and save it in m_curr_row so
+    // Make a copy of the current instruction Row and save it in m_state so
     // we can add updates as we process the instructions.
-    UnwindPlan::RowSP last_row =
-        std::make_shared<UnwindPlan::Row>(*unwind_plan.GetLastRow());
-    m_curr_row = std::make_shared<UnwindPlan::Row>(*last_row);
+    m_state.row = *unwind_plan.GetLastRow();
 
     // Add the initial state to the save list with offset 0.
-    saved_unwind_states.insert({0, {last_row, m_register_values}});
-
-    // cache the stack pointer register number (in whatever register numbering
-    // this UnwindPlan uses) for quick reference during instruction parsing.
-    RegisterInfo sp_reg_info = *m_inst_emulator_up->GetRegisterInfo(
-        eRegisterKindGeneric, LLDB_REGNUM_GENERIC_SP);
+    auto condition_block_start_state =
+        saved_unwind_states.emplace(0, m_state).first;
 
     // The architecture dependent condition code of the last processed
     // instruction.
     EmulateInstruction::InstructionCondition last_condition =
         EmulateInstruction::UnconditionalCondition;
-    lldb::addr_t condition_block_start_offset = 0;
 
     for (size_t idx = 0; idx < num_instructions; ++idx) {
       m_curr_row_modified = false;
@@ -151,78 +145,28 @@ bool UnwindAssemblyInstEmulation::GetNonCallSiteUnwindPlanFromAssembly(
       --it; // Move it to the row corresponding to the current offset
 
       // If the offset of m_curr_row don't match with the offset we see in
-      // saved_unwind_states then we have to update m_curr_row and
-      // m_register_values based on the saved values. It is happening after we
-      // processed an epilogue and a return to caller instruction.
-      if (it->second.first->GetOffset() != m_curr_row->GetOffset()) {
-        UnwindPlan::Row *newrow = new UnwindPlan::Row;
-        *newrow = *it->second.first;
-        m_curr_row.reset(newrow);
-        m_register_values = it->second.second;
-        // re-set the CFA register ivars to match the new m_curr_row.
-        if (sp_reg_info.name &&
-            m_curr_row->GetCFAValue().IsRegisterPlusOffset()) {
-          uint32_t row_cfa_regnum =
-              m_curr_row->GetCFAValue().GetRegisterNumber();
-          lldb::RegisterKind row_kind = m_unwind_plan_ptr->GetRegisterKind();
-          // set m_cfa_reg_info to the row's CFA reg.
-          m_cfa_reg_info =
-              *m_inst_emulator_up->GetRegisterInfo(row_kind, row_cfa_regnum);
-          // set m_fp_is_cfa.
-          if (sp_reg_info.kinds[row_kind] == row_cfa_regnum)
-            m_fp_is_cfa = false;
-          else
-            m_fp_is_cfa = true;
-        }
-      }
+      // saved_unwind_states then we have to update current unwind state to
+      // the saved values. It is happening after we processed an epilogue and a
+      // return to caller instruction.
+      if (it->second.row.GetOffset() != m_state.row.GetOffset())
+        m_state = it->second;
 
       m_inst_emulator_up->SetInstruction(inst->GetOpcode(), inst->GetAddress(),
                                          nullptr);
 
       if (last_condition != m_inst_emulator_up->GetInstructionCondition()) {
-        if (m_inst_emulator_up->GetInstructionCondition() !=
-                EmulateInstruction::UnconditionalCondition &&
-            saved_unwind_states.count(current_offset) == 0) {
-          // If we don't have a saved row for the current offset then save our
-          // current state because we will have to restore it after the
-          // conditional block.
-          auto new_row = std::make_shared<UnwindPlan::Row>(*m_curr_row.get());
-          saved_unwind_states.insert(
-              {current_offset, {new_row, m_register_values}});
-        }
-
         // If the last instruction was conditional with a different condition
-        // then the then current condition then restore the condition.
+        // than the current condition then restore the state.
         if (last_condition != EmulateInstruction::UnconditionalCondition) {
-          const auto &saved_state =
-              saved_unwind_states.at(condition_block_start_offset);
-          m_curr_row = std::make_shared<UnwindPlan::Row>(*saved_state.first);
-          m_curr_row->SetOffset(current_offset);
-          m_register_values = saved_state.second;
-          // re-set the CFA register ivars to match the new m_curr_row.
-          if (sp_reg_info.name &&
-              m_curr_row->GetCFAValue().IsRegisterPlusOffset()) {
-            uint32_t row_cfa_regnum =
-                m_curr_row->GetCFAValue().GetRegisterNumber();
-            lldb::RegisterKind row_kind = m_unwind_plan_ptr->GetRegisterKind();
-            // set m_cfa_reg_info to the row's CFA reg.
-            m_cfa_reg_info =
-                *m_inst_emulator_up->GetRegisterInfo(row_kind, row_cfa_regnum);
-            // set m_fp_is_cfa.
-            if (sp_reg_info.kinds[row_kind] == row_cfa_regnum)
-              m_fp_is_cfa = false;
-            else
-              m_fp_is_cfa = true;
-          }
+          m_state = condition_block_start_state->second;
+          m_state.row.SetOffset(current_offset);
           // The last instruction might already created a row for this offset
           // and we want to overwrite it.
-          bool replace_existing = true;
-          unwind_plan.InsertRow(std::make_shared<UnwindPlan::Row>(*m_curr_row),
-                                replace_existing);
+          saved_unwind_states.insert_or_assign(current_offset, m_state);
         }
 
         // We are starting a new conditional block at the actual offset
-        condition_block_start_offset = current_offset;
+        condition_block_start_state = it;
       }
 
       if (log && log->GetVerbose()) {
@@ -245,11 +189,10 @@ bool UnwindAssemblyInstEmulation::GetNonCallSiteUnwindPlanFromAssembly(
       if (m_forward_branch_offset != 0 &&
           range.ContainsFileAddress(inst->GetAddress().GetFileAddress() +
                                     m_forward_branch_offset)) {
-        auto newrow = std::make_shared<UnwindPlan::Row>(*m_curr_row.get());
-        newrow->SetOffset(current_offset + m_forward_branch_offset);
-        saved_unwind_states.insert({current_offset + m_forward_branch_offset,
-                                    {newrow, m_register_values}});
-        unwind_plan.InsertRow(newrow);
+        if (auto [it, inserted] = saved_unwind_states.emplace(
+                current_offset + m_forward_branch_offset, m_state);
+            inserted)
+          it->second.row.SetOffset(current_offset + m_forward_branch_offset);
       }
 
       // Were there any changes to the CFI while evaluating this instruction?
@@ -258,21 +201,18 @@ bool UnwindAssemblyInstEmulation::GetNonCallSiteUnwindPlanFromAssembly(
         // current address
         if (saved_unwind_states.count(current_offset +
                                       inst->GetOpcode().GetByteSize()) == 0) {
-          m_curr_row->SetOffset(current_offset +
+          m_state.row.SetOffset(current_offset +
                                 inst->GetOpcode().GetByteSize());
-          unwind_plan.InsertRow(m_curr_row);
-          saved_unwind_states.insert(
-              {current_offset + inst->GetOpcode().GetByteSize(),
-               {m_curr_row, m_register_values}});
-
-          // Allocate a new Row for m_curr_row, copy the current state
-          // into it
-          UnwindPlan::Row *newrow = new UnwindPlan::Row;
-          *newrow = *m_curr_row.get();
-          m_curr_row.reset(newrow);
+          saved_unwind_states.emplace(
+              current_offset + inst->GetOpcode().GetByteSize(), m_state);
         }
       }
     }
+    for (auto &[_, state] : saved_unwind_states) {
+      unwind_plan.InsertRow(
+          std::make_shared<UnwindPlan::Row>(std::move(state.row)),
+          /*replace_existing=*/true);
+    }
   }
 
   if (log && log->GetVerbose()) {
@@ -339,14 +279,14 @@ uint64_t UnwindAssemblyInstEmulation::MakeRegisterKindValuePair(
 
 void UnwindAssemblyInstEmulation::SetRegisterValue(
     const RegisterInfo &reg_info, const RegisterValue &reg_value) {
-  m_register_values[MakeRegisterKindValuePair(reg_info)] = reg_value;
+  m_state.register_values[MakeRegisterKindValuePair(reg_info)] = reg_value;
 }
 
 bool UnwindAssemblyInstEmulation::GetRegisterValue(const RegisterInfo &reg_info,
                                                    RegisterValue &reg_value) {
   const uint64_t reg_id = MakeRegisterKindValuePair(reg_info);
-  RegisterValueMap::const_iterator pos = m_register_values.find(reg_id);
-  if (pos != m_register_values.end()) {
+  RegisterValueMap::const_iterator pos = m_state.register_values.find(reg_id);
+  if (pos != m_state.register_values.end()) {
     reg_value = pos->second;
     return true; // We had a real value that comes from an opcode that wrote
                  // to it...
@@ -444,7 +384,7 @@ size_t UnwindAssemblyInstEmulation::WriteMemory(
         generic_regnum != LLDB_REGNUM_GENERIC_SP) {
       if (m_pushed_regs.try_emplace(reg_num, addr).second) {
         const int32_t offset = addr - m_initial_sp;
-        m_curr_row->SetRegisterLocationToAtCFAPlusOffset(reg_num, offset,
+        m_state.row.SetRegisterLocationToAtCFAPlusOffset(reg_num, offset,
                                                          /*can_replace=*/true);
         m_curr_row_modified = true;
       }
@@ -548,13 +488,14 @@ bool UnwindAssemblyInstEmulation::WriteRegister(
     // CFA offset
     // with the same amount.
     lldb::RegisterKind kind = m_unwind_plan_ptr->GetRegisterKind();
-    if (m_fp_is_cfa && reg_info->kinds[kind] == m_cfa_reg_info.kinds[kind] &&
+    if (m_state.fp_is_cfa &&
+        reg_info->kinds[kind] == m_state.cfa_reg_info.kinds[kind] &&
         context.GetInfoType() ==
             EmulateInstruction::eInfoTypeRegisterPlusOffset &&
         context.info.RegisterPlusOffset.reg.kinds[kind] ==
-            m_cfa_reg_info.kinds[kind]) {
+            m_state.cfa_reg_info.kinds[kind]) {
       const int64_t offset = context.info.RegisterPlusOffset.signed_offset;
-      m_curr_row->GetCFAValue().IncOffset(-1 * offset);
+      m_state.row.GetCFAValue().IncOffset(-1 * offset);
       m_curr_row_modified = true;
     }
   } break;
@@ -590,25 +531,25 @@ bool UnwindAssemblyInstEmulation::WriteRegister(
       case EmulateInstruction::eInfoTypeAddress:
         if (auto it = m_pushed_regs.find(reg_num);
             it != m_pushed_regs.end() && context.info.address == it->second) {
-          m_curr_row->SetRegisterLocationToSame(reg_num,
+          m_state.row.SetRegisterLocationToSame(reg_num,
                                                 false /*must_replace*/);
           m_curr_row_modified = true;
 
           // FP has been restored to its original value, we are back
           // to using SP to calculate the CFA.
-          if (m_fp_is_cfa) {
-            m_fp_is_cfa = false;
+          if (m_state.fp_is_cfa) {
+            m_state.fp_is_cfa = false;
             lldb::RegisterKind sp_reg_kind = eRegisterKindGeneric;
             uint32_t sp_reg_num = LLDB_REGNUM_GENERIC_SP;
             RegisterInfo sp_reg_info =
                 *m_inst_emulator_up->GetRegisterInfo(sp_reg_kind, sp_reg_num);
             RegisterValue sp_reg_val;
             if (GetRegisterValue(sp_reg_info, sp_reg_val)) {
-              m_cfa_reg_info = sp_reg_info;
+              m_state.cfa_reg_info = sp_reg_info;
               const uint32_t cfa_reg_num =
                   sp_reg_info.kinds[m_unwind_plan_ptr->GetRegisterKind()];
               assert(cfa_reg_num != LLDB_INVALID_REGNUM);
-              m_curr_row->GetCFAValue().SetIsRegisterPlusOffset(
+              m_state.row.GetCFAValue().SetIsRegisterPlusOffset(
                   cfa_reg_num, m_initial_sp - sp_reg_val.GetAsUInt64());
             }
           }
@@ -620,7 +561,7 @@ bool UnwindAssemblyInstEmulation::WriteRegister(
              generic_regnum == LLDB_REGNUM_GENERIC_FLAGS) &&
             "eInfoTypeISA used for popping a register other the PC/FLAGS");
         if (generic_regnum != LLDB_REGNUM_GENERIC_FLAGS) {
-          m_curr_row->SetRegisterLocationToSame(reg_num,
+          m_state.row.SetRegisterLocationToSame(reg_num,
                                                 false /*must_replace*/);
           m_curr_row_modified = true;
         }
@@ -633,26 +574,26 @@ bool UnwindAssemblyInstEmulation::WriteRegister(
   } break;
 
   case EmulateInstruction::eContextSetFramePointer:
-    if (!m_fp_is_cfa) {
-      m_fp_is_cfa = true;
-      m_cfa_reg_info = *reg_info;
+    if (!m_state.fp_is_cfa) {
+      m_state.fp_is_cfa = true;
+      m_state.cfa_reg_info = *reg_info;
       const uint32_t cfa_reg_num =
           reg_info->kinds[m_unwind_plan_ptr->GetRegisterKind()];
       assert(cfa_reg_num != LLDB_INVALID_REGNUM);
-      m_curr_row->GetCFAValue().SetIsRegisterPlusOffset(
+      m_state.row.GetCFAValue().SetIsRegisterPlusOffset(
           cfa_reg_num, m_initial_sp - reg_value.GetAsUInt64());
       m_curr_row_modified = true;
     }
     break;
 
   case EmulateInstruction::eContextRestoreStackPointer:
-    if (m_fp_is_cfa) {
-      m_fp_is_cfa = false;
-      m_cfa_reg_info = *reg_info;
+    if (m_state.fp_is_cfa) {
+      m_state.fp_is_cfa = false;
+      m_state.cfa_reg_info = *reg_info;
       const uint32_t cfa_reg_num =
           reg_info->kinds[m_unwind_plan_ptr->GetRegisterKind()];
       assert(cfa_reg_num != LLDB_INVALID_REGNUM);
-      m_curr_row->GetCFAValue().SetIsRegisterPlusOffset(
+      m_state.row.GetCFAValue().SetIsRegisterPlusOffset(
           cfa_reg_num, m_initial_sp - reg_value.GetAsUInt64());
       m_curr_row_modified = true;
     }
@@ -661,9 +602,9 @@ bool UnwindAssemblyInstEmulation::WriteRegister(
   case EmulateInstruction::eContextAdjustStackPointer:
     // If we have created a frame using the frame pointer, don't follow
     // subsequent adjustments to the stack pointer.
-    if (!m_fp_is_cfa) {
-      m_curr_row->GetCFAValue().SetIsRegisterPlusOffset(
-          m_curr_row->GetCFAValue().GetRegisterNumber(),
+    if (!m_state.fp_is_cfa) {
+      m_state.row.GetCFAValue().SetIsRegisterPlusOffset(
+          m_state.row.GetCFAValue().GetRegisterNumber(),
           m_initial_sp - reg_value.GetAsUInt64());
       m_curr_row_modified = true;
     }
diff --git a/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.h b/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.h
index d03a8ce4e9241..96a0881eaee9d 100644
--- a/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.h
+++ b/lldb/source/Plugins/UnwindAssembly/InstEmulation/UnwindAssemblyInstEmulation.h
@@ -63,10 +63,8 @@ class UnwindAssemblyInstEmulation : public lldb_private::UnwindAssembly {
   UnwindAssemblyInstEmulation(const lldb_private::ArchSpec &arch,
                               lldb_private::EmulateInstruction *inst_emulator)
       : UnwindAssembly(arch), m_inst_emulator_up(inst_emulator),
-        m_range_ptr(nullptr), m_unwind_plan_ptr(nullptr), m_curr_row(),
-        m_initial_sp(0), m_cfa_reg_info(), m_fp_is_cfa(false),
-        m_register_values(), m_pushed_regs(), m_curr_row_modified(false),
-        m_forward_branch_offset(0) {
+        m_range_ptr(nullptr), m_unwind_plan_ptr(nullptr), m_initial_sp(0),
+        m_curr_row_modified(false), m_forward_branch_offset(0) {
     if (m_inst_emulator_up) {
       m_inst_emulator_up->SetBaton(this);
       m_inst_emulator_up->SetCallbacks(ReadMemory, WriteMemory, ReadRegister,
@@ -124,16 +122,20 @@ class UnwindAssemblyInstEmulation : public lldb_private::UnwindAssembly {
   bool GetRegisterValue(const lldb_private::RegisterInfo &reg_info,
                         lldb_private::RegisterValue &reg_value);
 
+  typedef std::map<uint64_t, lldb_private::RegisterValue> RegisterValueMap;
+  struct UnwindState {
+    lldb_private::UnwindPlan::Row row = {};
+    lldb_private::RegisterInfo cfa_reg_info = {};
+    bool fp_is_cfa = false;
+    RegisterValueMap register_values = {};
+  };
+
   std::unique_ptr<lldb_private::EmulateInstruction> m_inst_emulator_up;
   lldb_private::AddressRange *m_range_ptr;
   lldb_private::UnwindPlan *m_unwind_plan_ptr;
-  lldb_private::UnwindPlan::RowSP m_curr_row;
+  UnwindState m_state;
   typedef std::map<uint64_t, uint64_t> PushedRegisterToAddrMap;
   uint64_t m_initial_sp;
-  lldb_private::RegisterInfo m_cfa_reg_info;
-  bool m_fp_is_cfa;
-  typedef std::map<uint64_t, lldb_private::RegisterValue> RegisterValueMap;
-  RegisterValueMap m_register_values;
   PushedRegisterToAddrMap m_pushed_regs;
 
   // While processing the instruction stream, we need to communicate some state
diff --git a/lldb/source/Target/Statistics.cpp b/lldb/source/Target/Statistics.cpp
index 8173d20457e21..b5d2e7bda1edf 100644
--- a/lldb/source/Target/Statistics.cpp
+++ b/lldb/source/Target/Statistics.cpp
@@ -316,7 +316,7 @@ llvm::json::Value DebuggerStats::ReportStatistics(
     ModuleStats module_stat;
     module_stat.symtab_parse_time = module->GetSymtabParseTime().get().count();
     module_stat.symtab_index_time = module->GetSymtabIndexTime().get().count();
-    Symtab *symtab = module->GetSymtab();
+    Symtab *symtab = module->GetSymtab(/*can_create=*/false);
     if (symtab) {
       module_stat.symtab_loaded_from_cache = symtab->GetWasLoadedFromCache();
       if (module_stat.symtab_loaded_from_cache)
@@ -325,7 +325,7 @@ llvm::json::Value DebuggerStats::ReportStatistics(
       if (module_stat.symtab_saved_to_cache)
         ++symtabs_saved;
     }
-    SymbolFile *sym_file = module->GetSymbolFile();
+    SymbolFile *sym_file = module->GetSymbolFile(/*can_create=*/false);
     if (sym_file) {
       if (!summary_only) {
         if (sym_file->GetObjectFile() != module->GetObjectFile())
diff --git a/lldb/source/Utility/XcodeSDK.cpp b/lldb/source/Utility/XcodeSDK.cpp
index 02cf7866e22fb..004b4717e315b 100644
--- a/lldb/source/Utility/XcodeSDK.cpp
+++ b/lldb/source/Utility/XcodeSDK.cpp
@@ -155,10 +155,6 @@ bool XcodeSDK::Info::operator==(const Info &other) const {
 }
 
 void XcodeSDK::Merge(const XcodeSDK &other) {
-  auto add_internal_sdk_suffix = [](llvm::StringRef sdk) {
-    return (sdk.substr(0, sdk.size() - 3) + "Internal.sdk").str();
-  };
-
   // The "bigger" SDK always wins.
   auto l = Parse();
   auto r = other.Parse();
@@ -168,12 +164,14 @@ void XcodeSDK::Merge(const XcodeSDK &other) {
     // The Internal flag always wins.
     if (!l.internal && r.internal) {
       if (llvm::StringRef(m_name).ends_with(".sdk"))
-        m_name = add_internal_sdk_suffix(m_name);
-
-      if (m_sysroot.GetFileNameExtension() == ".sdk")
-        m_sysroot.SetFilename(add_internal_sdk_suffix(m_sysroot.GetFilename()));
+        m_name =
+            m_name.substr(0, m_name.size() - 3) + std::string("Internal.sdk");
     }
   }
+
+  // We changed the SDK name. Adjust the sysroot accordingly.
+  if (m_sysroot && m_sysroot.GetFilename().GetStringRef() != m_name)
+    m_sysroot.SetFilename(m_name);
 }
 
 std::string XcodeSDK::GetCanonicalName(XcodeSDK::Info info) {
diff --git a/lldb/source/ValueObject/ValueObjectChild.cpp b/lldb/source/ValueObject/ValueObjectChild.cpp
index ea211530591cf..d7f1ad08415e3 100644
--- a/lldb/source/ValueObject/ValueObjectChild.cpp
+++ b/lldb/source/ValueObject/ValueObjectChild.cpp
@@ -164,8 +164,9 @@ bool ValueObjectChild::UpdateValue() {
             const bool thread_and_frame_only_if_stopped = true;
             ExecutionContext exe_ctx(GetExecutionContextRef().Lock(
                 thread_and_frame_only_if_stopped));
-            if (auto type_bit_size = GetCompilerType().GetBitSize(
-                    exe_ctx.GetBestExecutionContextScope())) {
+            if (auto type_bit_size =
+                    llvm::expectedToOptional(GetCompilerType().GetBitSize(
+                        exe_ctx.GetBestExecutionContextScope()))) {
               uint64_t bitfield_end =
                   m_bitfield_bit_size + m_bitfield_bit_offset;
               if (bitfield_end > *type_bit_size) {
diff --git a/lldb/test/API/functionalities/asan/TestMemoryHistory.py b/lldb/test/API/functionalities/asan/TestMemoryHistory.py
index 41ab25823f5cc..b04182a543719 100644
--- a/lldb/test/API/functionalities/asan/TestMemoryHistory.py
+++ b/lldb/test/API/functionalities/asan/TestMemoryHistory.py
@@ -19,6 +19,7 @@ def test(self):
         self.asan_tests()
 
     @skipIf(oslist=no_match(["macosx"]))
+    @skipIf(bugnumber="rdar://144997976")
     def test_libsanitizers_asan(self):
         try:
             self.build(make_targets=["libsanitizers"])
diff --git a/lldb/test/API/functionalities/asan/TestReportData.py b/lldb/test/API/functionalities/asan/TestReportData.py
index 5e4c179e2a481..fabc985d0ed44 100644
--- a/lldb/test/API/functionalities/asan/TestReportData.py
+++ b/lldb/test/API/functionalities/asan/TestReportData.py
@@ -20,6 +20,7 @@ def test(self):
         self.asan_tests()
 
     @skipIf(oslist=no_match(["macosx"]))
+    @skipIf(bugnumber="rdar://144997976")
     def test_libsanitizers_asan(self):
         try:
             self.build(make_targets=["libsanitizers"])
diff --git a/lldb/test/API/functionalities/json/object-file/TestObjectFileJSON.py b/lldb/test/API/functionalities/json/object-file/TestObjectFileJSON.py
index 510788b43d0db..5145a6c0cd53a 100644
--- a/lldb/test/API/functionalities/json/object-file/TestObjectFileJSON.py
+++ b/lldb/test/API/functionalities/json/object-file/TestObjectFileJSON.py
@@ -64,6 +64,7 @@ def test_module(self):
         foo_file_addr = TEXT_file_addr + 0x100
         bar_file_addr = DATA_file_addr + 0x10
         TEXT_size = 0x222
+        text_size = 0x20
         DATA_size = 0x333
         foo_size = 0x11
         bar_size = 0x22
@@ -74,10 +75,22 @@ def test_module(self):
             "type": "sharedlibrary",
             "sections": [
                 {
+                    "user_id": 0x100,
+                    "name": "__PAGEZERO",
+                    "type": "container",
+                    "address": 0,
+                    "size": 0x100000000,
+                    "flags": 0x101
+              },
+              {
+                    "user_id": 0x200,
                     "name": "__TEXT",
                     "type": "container",
                     "address": TEXT_file_addr,
                     "size": TEXT_size,
+                    "flags": 0x202,
+                    "file_offset": 0,
+                    "file_size": TEXT_size,
                     "read": True,
                     "write": False,
                     "execute": True,
@@ -86,10 +99,29 @@ def test_module(self):
                             "name": "__text",
                             "type": "code",
                             "address": TEXT_file_addr,
-                            "size": TEXT_size,
+                            "size": text_size,
+                            "alignment": 2,
                             "read": True,
                             "write": False,
                             "execute": True,
+                        },
+                        {
+                            "name": "__fake",
+                            "address": TEXT_file_addr + 1 * text_size,
+                            "size": text_size,
+                            "fake": True
+                        },
+                        {
+                            "name": "__encrypted",
+                            "address": TEXT_file_addr + 2 * text_size,
+                            "size": text_size,
+                            "encrypted": True
+                        },
+                        {
+                            "name": "__tls",
+                            "address": TEXT_file_addr + 2 * text_size,
+                            "size": text_size,
+                            "thread_specific": True
                         }
                     ],
                 },
@@ -101,6 +133,9 @@ def test_module(self):
                     "read": True,
                     "write": True,
                     "execute": False,
+                    "flags": 0x303,
+                    "file_offset": DATA_file_addr - TEXT_file_addr,
+                    "file_size": DATA_size,
                 },
             ],
             "symbols": [
@@ -127,21 +162,34 @@ def test_module(self):
 
         TEXT_section = module.GetSectionAtIndex(0)
         self.assertTrue(TEXT_section.IsValid())
+        self.assertEqual(TEXT_section.GetName(), "__PAGEZERO")
+        self.assertEqual(TEXT_section.file_addr, 0)
+        self.assertEqual(TEXT_section.size, 0x100000000)
+        self.assertEqual(TEXT_section.GetSectionType(), lldb.eSectionTypeContainer)
+        self.assertEqual(TEXT_section.GetNumSubSections(), 0)
+        text_permissions = TEXT_section.GetPermissions()
+        self.assertFalse((text_permissions & lldb.ePermissionsReadable) != 0)
+        self.assertFalse((text_permissions & lldb.ePermissionsWritable) != 0)
+        self.assertFalse((text_permissions & lldb.ePermissionsExecutable) != 0)
+
+        TEXT_section = module.GetSectionAtIndex(1)
+        self.assertTrue(TEXT_section.IsValid())
         self.assertEqual(TEXT_section.GetName(), "__TEXT")
         self.assertEqual(TEXT_section.file_addr, TEXT_file_addr)
         self.assertEqual(TEXT_section.size, TEXT_size)
+        self.assertEqual(TEXT_section.file_offset, 0)
+        self.assertEqual(TEXT_section.file_size, TEXT_size)
         self.assertEqual(TEXT_section.GetSectionType(), lldb.eSectionTypeContainer)
-        self.assertEqual(TEXT_section.GetNumSubSections(), 1)
+        self.assertEqual(TEXT_section.GetNumSubSections(), 4)
         text_permissions = TEXT_section.GetPermissions()
         self.assertTrue((text_permissions & lldb.ePermissionsReadable) != 0)
         self.assertFalse((text_permissions & lldb.ePermissionsWritable) != 0)
         self.assertTrue((text_permissions & lldb.ePermissionsExecutable) != 0)
 
         text_section = TEXT_section.GetSubSectionAtIndex(0)
-        self.assertTrue(text_section.IsValid())
         self.assertEqual(text_section.GetName(), "__text")
-        self.assertEqual(text_section.file_addr, TEXT_file_addr)
-        self.assertEqual(text_section.size, TEXT_size)
+        self.assertEqual(text_section.size, text_size)
+        self.assertEqual(text_section.GetAlignment(), 4)
         self.assertEqual(text_section.GetSectionType(), lldb.eSectionTypeCode)
         self.assertEqual(text_section.GetNumSubSections(), 0)
         text_permissions = text_section.GetPermissions()
@@ -149,11 +197,13 @@ def test_module(self):
         self.assertFalse((text_permissions & lldb.ePermissionsWritable) != 0)
         self.assertTrue((text_permissions & lldb.ePermissionsExecutable) != 0)
 
-        DATA_section = module.GetSectionAtIndex(1)
+        DATA_section = module.GetSectionAtIndex(2)
         self.assertTrue(DATA_section.IsValid())
         self.assertEqual(DATA_section.GetName(), "__DATA")
         self.assertEqual(DATA_section.file_addr, DATA_file_addr)
         self.assertEqual(DATA_section.size, DATA_size)
+        self.assertEqual(DATA_section.file_offset, DATA_file_addr - TEXT_file_addr)
+        self.assertEqual(DATA_section.file_size, DATA_size)
         self.assertEqual(DATA_section.GetSectionType(), lldb.eSectionTypeData)
         data_permissions = DATA_section.GetPermissions()
         self.assertTrue((data_permissions & lldb.ePermissionsReadable) != 0)
@@ -170,6 +220,19 @@ def test_module(self):
         self.assertEqual(bar_symbol.addr.GetFileAddress(), bar_file_addr)
         self.assertEqual(bar_symbol.GetSize(), bar_size)
 
+        # Verify the user_ids and flags are set correctly since there is no API
+        # for this on lldb.SBSection
+        self.expect("target modules dump sections c.json",
+            substrs = [
+                "0x0000000000000100 container              [0x0000000000000000-0x0000000100000000)  ---  0x00000000 0x00000000 0x00000101 c.json.__PAGEZERO",
+                "0x0000000000000200 container              [0x0000000100000000-0x0000000100000222)  r-x  0x00000000 0x00000222 0x00000202 c.json.__TEXT",
+                "0x0000000000000001 code                   [0x0000000100000000-0x0000000100000020)  r-x  0x00000000 0x00000000 0x00000000 c.json.__TEXT.__text",
+                "0x0000000000000002 code                   [0x0000000100000020-0x0000000100000040)  ---  0x00000000 0x00000000 0x00000000 c.json.__TEXT.__fake",
+                "0x0000000000000003 code                   [0x0000000100000040-0x0000000100000060)  ---  0x00000000 0x00000000 0x00000000 c.json.__TEXT.__encrypted",
+                "0x0000000000000004 code                   [0x0000000100000040-0x0000000100000060)  ---  0x00000000 0x00000000 0x00000000 c.json.__TEXT.__tls",
+                "0x0000000000000005 data                   [0x0000000100001000-0x0000000100001333)  rw-  0x00001000 0x00000333 0x00000303 c.json.__DATA"
+            ])
+
         error = target.SetSectionLoadAddress(TEXT_section, TEXT_file_addr + slide)
         self.assertSuccess(error)
         error = target.SetSectionLoadAddress(DATA_section, DATA_file_addr + slide)
diff --git a/lldb/test/API/macosx/no-nlist-memory-module/Makefile b/lldb/test/API/macosx/no-nlist-memory-module/Makefile
index 4858a7bbaf924..456e57ac8e65c 100644
--- a/lldb/test/API/macosx/no-nlist-memory-module/Makefile
+++ b/lldb/test/API/macosx/no-nlist-memory-module/Makefile
@@ -1,19 +1,13 @@
 C_SOURCES := main.c
 LD_EXTRAS = -Wl,-rpath "-Wl,$(shell pwd)" -L. -lno-nlists -lhas-nlists
 
-.PHONY: build-libno-nlists build-libhas-nlists
-all: build-libno-nlists build-libhas-nlists a.out
-
 include Makefile.rules
 
-build-libno-nlists: no-nlists.c no-nlist-sect.s
-	$(CC) $(CFLAGS) -c -o no-nlists.o $(<D)/no-nlists.c
-	$(CC) $(CFLAGS) -c -o no-nlist-sect.o $(<D)/no-nlist-sect.s
-	$(LD) $(LDFLAGS) -dynamiclib -o libno-nlists.dylib no-nlists.o no-nlist-sect.o -install_name "@executable_path/libno-nlists.dylib"
+a.out: dylib_HasNlists dylib_NoNlists
 
-build-libhas-nlists: has-nlists.c
-	$(CC) $(CFLAGS) -c -o has-nlists.o $(<D)/has-nlists.c
-	$(LD) $(LDFLAGS) -dynamiclib -o libhas-nlists.dylib has-nlists.o  -install_name "@executable_path/libhas-nlists.dylib"
+dylib_HasNlists:
+	"$(MAKE)" -f $(MAKEFILE_RULES) \
+	DYLIB_ONLY=YES DYLIB_NAME=has-nlists DYLIB_C_SOURCES=has-nlists.c
 
-clean::
-	rm -rf *.o *.dylib a.out *.dSYM
+dylib_NoNlists:
+	"$(MAKE)" VPATH=$(SRCDIR) -I $(SRCDIR) -f $(SRCDIR)/NoNlists.mk
diff --git a/lldb/test/API/macosx/no-nlist-memory-module/NoNlists.mk b/lldb/test/API/macosx/no-nlist-memory-module/NoNlists.mk
new file mode 100644
index 0000000000000..046390c028186
--- /dev/null
+++ b/lldb/test/API/macosx/no-nlist-memory-module/NoNlists.mk
@@ -0,0 +1,12 @@
+DYLIB_ONLY := YES
+DYLIB_NAME := no-nlists
+DYLIB_C_SOURCES := no-nlists.c
+DYLIB_OBJECTS += no-nlist-sect.o
+
+no-nlist-sect.o:
+	$(CC) $(CFLAGS) -c -o no-nlist-sect.o $(SRCDIR)/no-nlist-sect.s
+
+include Makefile.rules
+
+clean::
+	rm -rf *.o *.dylib a.out *.dSYM
diff --git a/lldb/test/API/python_api/sbprogress/TestSBProgress.py b/lldb/test/API/python_api/sbprogress/TestSBProgress.py
index 2a0689a52a185..b64c4b9c7206e 100644
--- a/lldb/test/API/python_api/sbprogress/TestSBProgress.py
+++ b/lldb/test/API/python_api/sbprogress/TestSBProgress.py
@@ -43,7 +43,7 @@ def test_progress_finalize_non_deterministic_progress(self):
         progress = lldb.SBProgress("Test SBProgress", "Test finalize", self.dbg)
         listener = lldb.SBListener("Test listener")
         broadcaster = self.dbg.GetBroadcaster()
-        broadcaster.AddListener(listener, lldb.eBroadcastBitExternalProgressCategory)
+        broadcaster.AddListener(listener, lldb.eBroadcastBitExternalProgress)
         event = lldb.SBEvent()
         progress.Finalize()
         self.assertTrue(listener.WaitForEvent(5, event))
@@ -57,7 +57,7 @@ def test_progress_finalize_deterministic_progress(self):
         progress = lldb.SBProgress("Test SBProgress", "Test finalize", 13, self.dbg)
         listener = lldb.SBListener("Test listener")
         broadcaster = self.dbg.GetBroadcaster()
-        broadcaster.AddListener(listener, lldb.eBroadcastBitExternalProgressCategory)
+        broadcaster.AddListener(listener, lldb.eBroadcastBitExternalProgress)
         event = lldb.SBEvent()
         progress.Finalize()
         self.assertTrue(listener.WaitForEvent(5, event))
diff --git a/lldb/test/API/tools/lldb-dap/terminated-event/TestDAP_terminatedEvent.py b/lldb/test/API/tools/lldb-dap/terminated-event/TestDAP_terminatedEvent.py
index 6d1c25e8e4534..b0abe2a38dac4 100644
--- a/lldb/test/API/tools/lldb-dap/terminated-event/TestDAP_terminatedEvent.py
+++ b/lldb/test/API/tools/lldb-dap/terminated-event/TestDAP_terminatedEvent.py
@@ -43,7 +43,7 @@ def test_terminated_event(self):
         self.continue_to_breakpoints(breakpoint_ids)
         self.continue_to_exit()
 
-        statistics = self.dap_server.wait_for_terminated()["statistics"]
+        statistics = self.dap_server.wait_for_terminated()["body"]["$__lldb_statistics"]
         self.assertGreater(statistics["totalDebugInfoByteSize"], 0)
         self.assertGreater(statistics["totalDebugInfoEnabled"], 0)
         self.assertGreater(statistics["totalModuleCountHasDebugInfo"], 0)
diff --git a/lldb/test/API/tools/lldb-server/TestLldbGdbServer.py b/lldb/test/API/tools/lldb-server/TestLldbGdbServer.py
index 592037db502aa..12c97bc62dcef 100644
--- a/lldb/test/API/tools/lldb-server/TestLldbGdbServer.py
+++ b/lldb/test/API/tools/lldb-server/TestLldbGdbServer.py
@@ -195,8 +195,17 @@ def test_qRegisterInfo_contains_required_generics_debugserver(self):
         # Ensure we have a stack pointer register.
         self.assertIn("sp", generic_regs)
 
-        # Ensure we have a flags register.
-        self.assertIn("flags", generic_regs)
+        # Ensure we have a flags register. RISC-V doesn't have a flags register
+        if not self.isRISCV():
+            self.assertIn("flags", generic_regs)
+
+        if self.isRISCV() or self.isAArch64() or self.isARM():
+            # Specific register for a return address
+            self.assertIn("ra", generic_regs)
+
+            # Function arguments registers
+            for i in range(1, 5 if self.isARM() else 9):
+                self.assertIn(f"arg{i}", generic_regs)
 
     def test_qRegisterInfo_contains_at_least_one_register_set(self):
         self.build()
diff --git a/lldb/test/API/tools/lldb-server/main.cpp b/lldb/test/API/tools/lldb-server/main.cpp
index 484da8aa4bef9..c661f5b4e82c4 100644
--- a/lldb/test/API/tools/lldb-server/main.cpp
+++ b/lldb/test/API/tools/lldb-server/main.cpp
@@ -128,6 +128,15 @@ static void swap_chars() {
                :
                : "r"('0'), "r"('1'), "r"(&g_c1), "r"(&g_c2)
                : "memory");
+#elif defined(__riscv)
+  asm volatile("sb %1, (%2)\n\t"
+               "sb %0, (%3)\n\t"
+               "sb %0, (%2)\n\t"
+               "sb %1, (%3)\n\t"
+               :
+               : "r"('0'), "r"('1'), "r"(&g_c1), "r"(&g_c2)
+               : "memory");
+
 #else
 #warning This may generate unpredictible assembly and cause the single-stepping test to fail.
 #warning Please add appropriate assembly for your target.
diff --git a/lldb/test/API/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py b/lldb/test/API/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py
index c1a92eeb5d5d1..603aa0033f086 100644
--- a/lldb/test/API/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py
+++ b/lldb/test/API/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py
@@ -63,7 +63,7 @@ def test_g_target_xml_returns_correct_data(self):
             self.assertEqual(q_info_reg["format"], xml_info_reg.get("format"))
             self.assertEqual(q_info_reg["bitsize"], xml_info_reg.get("bitsize"))
 
-            if not self.isAArch64():
+            if not (self.isAArch64() or self.isRISCV()):
                 self.assertEqual(q_info_reg["offset"], xml_info_reg.get("offset"))
 
             self.assertEqual(q_info_reg["encoding"], xml_info_reg.get("encoding"))
diff --git a/lldb/test/Shell/Commands/command-statistics-dump.test b/lldb/test/Shell/Commands/command-statistics-dump.test
new file mode 100644
index 0000000000000..bad7de0ecf61f
--- /dev/null
+++ b/lldb/test/Shell/Commands/command-statistics-dump.test
@@ -0,0 +1,40 @@
+# This test validates that statistics generation does not force loading
+# symbol tables. In order to avoid other sources of symbol loading we
+# create the target without loading dependents and do not actually
+# run it. Running the target is a problem because there are various
+# instrumentation plugins (e.g. ASAN) that are always enabled and force
+# symbol loading. If you see this test start to fail we may have added
+# a new source of symbol loading unexpectedly.
+
+# Build a simple test executable.
+# RUN: %clang_host -g %S/Inputs/main.c -o %t-main.exe
+
+# When we enable symbol preload and dump stats there should be a non-zero
+# time for parsing symbol tables for the main module.
+# RUN: %lldb -O "settings set plugin.jit-loader.gdb.enable off" \
+# RUN:       -O "settings set target.preload-symbols true" \
+# RUN:       -o 'target create --no-dependents "%t-main.exe"' \
+# RUN:       -o "statistics dump" \
+# RUN:       -o "q" \
+# RUN:       | FileCheck %s -check-prefixes=CHECK,PRELOAD_TRUE
+
+# Find the module stats for the main executable and make sure
+# we are looking at the symbol parse time for that module.
+# CHECK: "modules": [
+# CHECK: {
+# CHECK: "path": {{.*}}-main.exe
+# CHECK-NOT: }
+
+# PRELOAD_TRUE: "symbolTableParseTime":
+# PRELOAD_TRUE-SAME: {{[1-9]+}}
+
+# When we disable symbol preload and dump stats the symbol table
+# for main should not be parsed and have a time of 0.
+# RUN: %lldb -O "settings set plugin.jit-loader.gdb.enable off" \
+# RUN:       -O "settings set target.preload-symbols false" \
+# RUN:       -o 'target create --no-dependents "%t-main.exe"' \
+# RUN:       -o "statistics dump" \
+# RUN:       -o "q" \
+# RUN:       | FileCheck %s -check-prefixes=CHECK,PRELOAD_FALSE
+
+# PRELOAD_FALSE: "symbolTableParseTime": 0,
diff --git a/lldb/tools/lldb-dap/DAP.cpp b/lldb/tools/lldb-dap/DAP.cpp
index 1f7b25e7c5bcc..edd3b31be8ff7 100644
--- a/lldb/tools/lldb-dap/DAP.cpp
+++ b/lldb/tools/lldb-dap/DAP.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "DAP.h"
+#include "DAPLog.h"
 #include "Handler/ResponseHandler.h"
 #include "JSONUtils.h"
 #include "LLDBUtils.h"
@@ -25,6 +26,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -61,10 +63,10 @@ const char DEV_NULL[] = "/dev/null";
 
 namespace lldb_dap {
 
-DAP::DAP(std::string name, llvm::StringRef path, std::ofstream *log,
+DAP::DAP(llvm::StringRef client_name, llvm::StringRef path, std::ofstream *log,
          lldb::IOObjectSP input, lldb::IOObjectSP output, ReplMode repl_mode,
          std::vector<std::string> pre_init_commands)
-    : name(std::move(name)), debug_adapter_path(path), log(log),
+    : client_name(client_name), debug_adapter_path(path), log(log),
       input(std::move(input)), output(std::move(output)),
       broadcaster("lldb-dap"), exception_breakpoints(),
       pre_init_commands(std::move(pre_init_commands)),
@@ -239,14 +241,7 @@ void DAP::SendJSON(const llvm::json::Value &json) {
   std::lock_guard<std::mutex> locker(mutex);
   SendJSON(json_str);
 
-  if (log) {
-    auto now = std::chrono::duration<double>(
-        std::chrono::system_clock::now().time_since_epoch());
-    *log << llvm::formatv("{0:f9} {1} <-- ", now.count(), name).str()
-         << std::endl
-         << "Content-Length: " << json_str.size() << "\r\n\r\n"
-         << llvm::formatv("{0:2}", json).str() << std::endl;
-  }
+  DAP_LOG(log, "({0}) <-- {1}", client_name, json_str);
 }
 
 // Read a JSON packet from the "in" stream.
@@ -270,13 +265,7 @@ std::string DAP::ReadJSON() {
   if (!input.read_full(log, length, json_str))
     return json_str;
 
-  if (log) {
-    auto now = std::chrono::duration<double>(
-        std::chrono::system_clock::now().time_since_epoch());
-    *log << llvm::formatv("{0:f9} {1} --> ", now.count(), name).str()
-         << std::endl
-         << "Content-Length: " << length << "\r\n\r\n";
-  }
+  DAP_LOG(log, "({0}) --> {1}", client_name, json_str);
   return json_str;
 }
 
@@ -711,22 +700,15 @@ PacketStatus DAP::GetNextObject(llvm::json::Object &object) {
 
   llvm::StringRef json_sref(json);
   llvm::Expected<llvm::json::Value> json_value = llvm::json::parse(json_sref);
-  if (auto error = json_value.takeError()) {
-    std::string error_str = llvm::toString(std::move(error));
-    if (log)
-      *log << "error: failed to parse JSON: " << error_str << std::endl
-           << json << std::endl;
+  if (!json_value) {
+    DAP_LOG_ERROR(log, json_value.takeError(),
+                  "({1}) failed to parse JSON: {0}", client_name);
     return PacketStatus::JSONMalformed;
   }
 
-  if (log) {
-    *log << llvm::formatv("{0:2}", *json_value).str() << std::endl;
-  }
-
   llvm::json::Object *object_ptr = json_value->getAsObject();
   if (!object_ptr) {
-    if (log)
-      *log << "error: json packet isn't a object" << std::endl;
+    DAP_LOG(log, "({0}) error: json packet isn't a object", client_name);
     return PacketStatus::JSONNotObject;
   }
   object = *object_ptr;
@@ -744,9 +726,7 @@ bool DAP::HandleObject(const llvm::json::Object &object) {
       return true; // Success
     }
 
-    if (log)
-      *log << "error: unhandled command \"" << command.data() << "\""
-           << std::endl;
+    DAP_LOG(log, "({0}) error: unhandled command '{1}'", client_name, command);
     return false; // Fail
   }
 
diff --git a/lldb/tools/lldb-dap/DAP.h b/lldb/tools/lldb-dap/DAP.h
index 8b2e498a28c95..3ff1992b61f5b 100644
--- a/lldb/tools/lldb-dap/DAP.h
+++ b/lldb/tools/lldb-dap/DAP.h
@@ -145,7 +145,7 @@ struct SendEventRequestHandler : public lldb::SBCommandPluginInterface {
 };
 
 struct DAP {
-  std::string name;
+  llvm::StringRef client_name;
   llvm::StringRef debug_adapter_path;
   std::ofstream *log;
   InputStream input;
@@ -210,7 +210,7 @@ struct DAP {
   // will contain that expression.
   std::string last_nonempty_var_expression;
 
-  DAP(std::string name, llvm::StringRef path, std::ofstream *log,
+  DAP(llvm::StringRef client_name, llvm::StringRef path, std::ofstream *log,
       lldb::IOObjectSP input, lldb::IOObjectSP output, ReplMode repl_mode,
       std::vector<std::string> pre_init_commands);
   ~DAP();
diff --git a/lldb/tools/lldb-dap/DAPLog.h b/lldb/tools/lldb-dap/DAPLog.h
new file mode 100644
index 0000000000000..75a0a7d63a4f1
--- /dev/null
+++ b/lldb/tools/lldb-dap/DAPLog.h
@@ -0,0 +1,58 @@
+//===-- DAPLog.h ----------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_TOOLS_LLDB_DAP_DAPLOG_H
+#define LLDB_TOOLS_LLDB_DAP_DAPLOG_H
+
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FormatAdapters.h"
+#include "llvm/Support/FormatVariadic.h"
+#include <chrono>
+#include <fstream>
+#include <string>
+
+// Write a message to log, if logging is enabled.
+#define DAP_LOG(log, ...)                                                      \
+  do {                                                                         \
+    ::std::ofstream *log_private = (log);                                      \
+    if (log_private) {                                                         \
+      ::std::chrono::duration<double> now{                                     \
+          ::std::chrono::system_clock::now().time_since_epoch()};              \
+      *log_private << ::llvm::formatv("{0:f9} ", now.count()).str()            \
+                   << ::llvm::formatv(__VA_ARGS__).str() << std::endl;         \
+    }                                                                          \
+  } while (0)
+
+// Write message to log, if error is set. In the log message refer to the error
+// with {0}. Error is cleared regardless of whether logging is enabled.
+#define DAP_LOG_ERROR(log, error, ...)                                         \
+  do {                                                                         \
+    ::std::ofstream *log_private = (log);                                      \
+    ::llvm::Error error_private = (error);                                     \
+    if (log_private && error_private) {                                        \
+      ::std::chrono::duration<double> now{                                     \
+          std::chrono::system_clock::now().time_since_epoch()};                \
+      *log_private << ::llvm::formatv("{0:f9} ", now.count()).str()            \
+                   << ::lldb_dap::FormatError(::std::move(error_private),      \
+                                              __VA_ARGS__)                     \
+                   << std::endl;                                               \
+    } else                                                                     \
+      ::llvm::consumeError(::std::move(error_private));                        \
+  } while (0)
+
+namespace lldb_dap {
+
+template <typename... Args>
+inline auto FormatError(llvm::Error error, const char *format, Args &&...args) {
+  return llvm::formatv(format, llvm::toString(std::move(error)),
+                       std::forward<Args>(args)...)
+      .str();
+}
+} // namespace lldb_dap
+
+#endif
diff --git a/lldb/tools/lldb-dap/Handler/InitializeRequestHandler.cpp b/lldb/tools/lldb-dap/Handler/InitializeRequestHandler.cpp
index 5bb73a7ec0d85..7b7d8d5cedaa6 100644
--- a/lldb/tools/lldb-dap/Handler/InitializeRequestHandler.cpp
+++ b/lldb/tools/lldb-dap/Handler/InitializeRequestHandler.cpp
@@ -111,7 +111,7 @@ void ProgressEventThreadFunction(DAP &dap) {
 // them prevent multiple threads from writing simultaneously so no locking
 // is required.
 static void EventThreadFunction(DAP &dap) {
-  llvm::set_thread_name(dap.name + ".event_handler");
+  llvm::set_thread_name(dap.client_name + ".event_handler");
   lldb::SBEvent event;
   lldb::SBListener listener = dap.debugger.GetListener();
   dap.broadcaster.AddListener(listener, eBroadcastBitStopEventThread);
diff --git a/lldb/tools/lldb-dap/JSONUtils.cpp b/lldb/tools/lldb-dap/JSONUtils.cpp
index 7094bf60bfbc2..932145b1799bd 100644
--- a/lldb/tools/lldb-dap/JSONUtils.cpp
+++ b/lldb/tools/lldb-dap/JSONUtils.cpp
@@ -1526,7 +1526,8 @@ static void addStatistic(lldb::SBTarget &target, llvm::json::Object &event) {
     const char *key = keys.GetStringAtIndex(i);
     FilterAndGetValueForKey(statistics, key, stats_body);
   }
-  event.try_emplace("statistics", std::move(stats_body));
+  llvm::json::Object body{{"$__lldb_statistics", std::move(stats_body)}};
+  event.try_emplace("body", std::move(body));
 }
 
 llvm::json::Object CreateTerminatedEventObject(lldb::SBTarget &target) {
diff --git a/lldb/tools/lldb-dap/OutputRedirector.cpp b/lldb/tools/lldb-dap/OutputRedirector.cpp
index 9ff25ad4149dd..fe278faca87bf 100644
--- a/lldb/tools/lldb-dap/OutputRedirector.cpp
+++ b/lldb/tools/lldb-dap/OutputRedirector.cpp
@@ -38,7 +38,7 @@ Expected<int> OutputRedirector::GetWriteFileDescriptor() {
   return m_fd;
 }
 
-Error OutputRedirector::RedirectTo(std::FILE *fileOverride,
+Error OutputRedirector::RedirectTo(std::FILE *file_override,
                                    std::function<void(StringRef)> callback) {
   assert(m_fd == kInvalidDescriptor && "Output readirector already started.");
   int new_fd[2];
@@ -56,8 +56,8 @@ Error OutputRedirector::RedirectTo(std::FILE *fileOverride,
   int read_fd = new_fd[0];
   m_fd = new_fd[1];
 
-  if (fileOverride) {
-    int override_fd = fileno(fileOverride);
+  if (file_override) {
+    int override_fd = fileno(file_override);
 
     // Backup the FD to restore once redirection is complete.
     m_original_fd = override_fd;
diff --git a/lldb/tools/lldb-dap/OutputRedirector.h b/lldb/tools/lldb-dap/OutputRedirector.h
index 45571c0d5f344..77b1b76ec4d89 100644
--- a/lldb/tools/lldb-dap/OutputRedirector.h
+++ b/lldb/tools/lldb-dap/OutputRedirector.h
@@ -24,10 +24,16 @@ class OutputRedirector {
   /// Creates writable file descriptor that will invoke the given callback on
   /// each write in a background thread.
   ///
+  /// \param[in] file_override
+  ///     Updates the file descriptor to the redirection pipe, if not null.
+  ///
+  /// \param[in] callback
+  ///     A callback invoked when any data is written to the file handle.
+  ///
   /// \return
   ///     \a Error::success if the redirection was set up correctly, or an error
   ///     otherwise.
-  llvm::Error RedirectTo(std::FILE *overrideFile,
+  llvm::Error RedirectTo(std::FILE *file_override,
                          std::function<void(llvm::StringRef)> callback);
 
   llvm::Expected<int> GetWriteFileDescriptor();
diff --git a/lldb/tools/lldb-dap/lldb-dap.cpp b/lldb/tools/lldb-dap/lldb-dap.cpp
index a5d9978e30248..ab40b75e5425d 100644
--- a/lldb/tools/lldb-dap/lldb-dap.cpp
+++ b/lldb/tools/lldb-dap/lldb-dap.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "DAP.h"
+#include "DAPLog.h"
 #include "EventHelper.h"
 #include "Handler/RequestHandler.h"
 #include "RunInTerminal.h"
@@ -294,8 +295,7 @@ serveConnection(const Socket::SocketProtocol &protocol, const std::string &name,
   }
 
   std::string address = llvm::join(listener->GetListeningConnectionURI(), ", ");
-  if (log)
-    *log << "started with connection listeners " << address << "\n";
+  DAP_LOG(log, "started with connection listeners {0}", address);
 
   llvm::outs() << "Listening for: " << address << "\n";
   // Ensure listening address are flushed for calles to retrieve the resolve
@@ -315,13 +315,8 @@ serveConnection(const Socket::SocketProtocol &protocol, const std::string &name,
                                           &dap_sessions_mutex, &dap_sessions,
                                           &clientCount](
                                              std::unique_ptr<Socket> sock) {
-    std::string name = llvm::formatv("client_{0}", clientCount++).str();
-    if (log) {
-      auto now = std::chrono::duration<double>(
-          std::chrono::system_clock::now().time_since_epoch());
-      *log << llvm::formatv("{0:f9}", now.count()).str()
-           << " client connected: " << name << "\n";
-    }
+    std::string client_name = llvm::formatv("client_{0}", clientCount++).str();
+    DAP_LOG(log, "({0}) client connected", client_name);
 
     lldb::IOObjectSP io(std::move(sock));
 
@@ -329,8 +324,8 @@ serveConnection(const Socket::SocketProtocol &protocol, const std::string &name,
     // client.
     std::thread client([=, &dap_sessions_condition, &dap_sessions_mutex,
                         &dap_sessions]() {
-      llvm::set_thread_name(name + ".runloop");
-      DAP dap = DAP(name, program_path, log, io, io, default_repl_mode,
+      llvm::set_thread_name(client_name + ".runloop");
+      DAP dap = DAP(client_name, program_path, log, io, io, default_repl_mode,
                     pre_init_commands);
 
       if (auto Err = dap.ConfigureIO()) {
@@ -351,13 +346,7 @@ serveConnection(const Socket::SocketProtocol &protocol, const std::string &name,
                                     "DAP session error: ");
       }
 
-      if (log) {
-        auto now = std::chrono::duration<double>(
-            std::chrono::system_clock::now().time_since_epoch());
-        *log << llvm::formatv("{0:f9}", now.count()).str()
-             << " client closed: " << name << "\n";
-      }
-
+      DAP_LOG(log, "({0}) client disconnected", client_name);
       std::unique_lock<std::mutex> lock(dap_sessions_mutex);
       dap_sessions.erase(io.get());
       std::notify_all_at_thread_exit(dap_sessions_condition, std::move(lock));
@@ -374,9 +363,9 @@ serveConnection(const Socket::SocketProtocol &protocol, const std::string &name,
     return status.takeError();
   }
 
-  if (log)
-    *log << "lldb-dap server shutdown requested, disconnecting remaining "
-            "clients...\n";
+  DAP_LOG(
+      log,
+      "lldb-dap server shutdown requested, disconnecting remaining clients...");
 
   bool client_failed = false;
   {
@@ -385,7 +374,7 @@ serveConnection(const Socket::SocketProtocol &protocol, const std::string &name,
       auto error = dap->Disconnect();
       if (error.Fail()) {
         client_failed = true;
-        llvm::errs() << "DAP client " << dap->name
+        llvm::errs() << "DAP client " << dap->client_name
                      << " disconnected failed: " << error.GetCString() << "\n";
       }
       // Close the socket to ensure the DAP::Loop read finishes.
diff --git a/lldb/unittests/API/CMakeLists.txt b/lldb/unittests/API/CMakeLists.txt
index 52e9a5e991515..fe2ff684a5d92 100644
--- a/lldb/unittests/API/CMakeLists.txt
+++ b/lldb/unittests/API/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_lldb_unittest(APITests
   SBCommandInterpreterTest.cpp
+  SBLineEntryTest.cpp
 
   LINK_LIBS
     liblldb
diff --git a/lldb/unittests/API/SBLineEntryTest.cpp b/lldb/unittests/API/SBLineEntryTest.cpp
new file mode 100644
index 0000000000000..518893b554bd1
--- /dev/null
+++ b/lldb/unittests/API/SBLineEntryTest.cpp
@@ -0,0 +1,26 @@
+//===-- SBLineEntryTest.cpp -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===/
+
+#include "gtest/gtest.h"
+
+#include "lldb/API/LLDB.h"
+
+TEST(SBLineEntryTest, SetLineAndColumn) {
+  constexpr uint32_t expected_line_no = 40;
+  constexpr uint32_t expected_column_no = 20;
+
+  lldb::SBLineEntry line_entry{};
+  line_entry.SetLine(expected_line_no);
+  line_entry.SetColumn(expected_column_no);
+
+  const uint32_t line_no = line_entry.GetLine();
+  const uint32_t column_no = line_entry.GetColumn();
+
+  EXPECT_EQ(line_no, line_no);
+  EXPECT_EQ(column_no, expected_column_no);
+}
diff --git a/lldb/unittests/Callback/TestBreakpointSetCallback.cpp b/lldb/unittests/Callback/TestBreakpointSetCallback.cpp
index 3dba4a9eb719e..c29cac5d2e5e7 100644
--- a/lldb/unittests/Callback/TestBreakpointSetCallback.cpp
+++ b/lldb/unittests/Callback/TestBreakpointSetCallback.cpp
@@ -52,8 +52,7 @@ class BreakpointSetCallbackTest : public ::testing::Test {
 
   DebuggerSP m_debugger_sp;
   PlatformSP m_platform_sp;
-  SubsystemRAII<FileSystem, HostInfo, PlatformMacOSX, ProgressManager>
-      subsystems;
+  SubsystemRAII<FileSystem, HostInfo, PlatformMacOSX> subsystems;
 };
 
 TEST_F(BreakpointSetCallbackTest, TestBreakpointSetCallback) {
diff --git a/lldb/unittests/Core/ProgressReportTest.cpp b/lldb/unittests/Core/ProgressReportTest.cpp
index 0943d7b990809..592e13a79e8d7 100644
--- a/lldb/unittests/Core/ProgressReportTest.cpp
+++ b/lldb/unittests/Core/ProgressReportTest.cpp
@@ -57,8 +57,7 @@ class ProgressReportTest : public ::testing::Test {
 
   DebuggerSP m_debugger_sp;
   ListenerSP m_listener_sp;
-  SubsystemRAII<FileSystem, HostInfo, PlatformMacOSX, ProgressManager>
-      subsystems;
+  SubsystemRAII<FileSystem, HostInfo, PlatformMacOSX> subsystems;
 };
 
 TEST_F(ProgressReportTest, TestReportCreation) {
@@ -313,119 +312,6 @@ TEST_F(ProgressReportTest, TestMinimumReportTime) {
   ASSERT_FALSE(listener_sp->GetEvent(event_sp, TIMEOUT));
 }
 
-TEST_F(ProgressReportTest, TestProgressManager) {
-  ListenerSP listener_sp =
-      CreateListenerFor(lldb::eBroadcastBitProgressCategory);
-  EventSP event_sp;
-  const ProgressEventData *data;
-
-  // Create three progress events with the same category then try to pop 2
-  // events from the queue in a row before the progress reports are destroyed.
-  // Since only 1 event should've been broadcast for this category, the second
-  // GetEvent() call should return false.
-  {
-    Progress progress1("Progress report 1", "Starting report 1");
-    Progress progress2("Progress report 1", "Starting report 2");
-    Progress progress3("Progress report 1", "Starting report 3");
-    ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT));
-    ASSERT_FALSE(listener_sp->GetEvent(event_sp, TIMEOUT));
-  }
-
-  data = ProgressEventData::GetEventDataFromEvent(event_sp.get());
-
-  EXPECT_EQ(data->GetDetails(), "");
-  EXPECT_FALSE(data->IsFinite());
-  EXPECT_FALSE(data->GetCompleted());
-  EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal);
-  EXPECT_EQ(data->GetMessage(), "Progress report 1");
-
-  // Pop another event from the queue, this should be the event for the final
-  // report for this category.
-  ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT));
-  data = ProgressEventData::GetEventDataFromEvent(event_sp.get());
-
-  EXPECT_EQ(data->GetDetails(), "");
-  EXPECT_FALSE(data->IsFinite());
-  EXPECT_TRUE(data->GetCompleted());
-  EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal);
-  EXPECT_EQ(data->GetMessage(), "Progress report 1");
-}
-
-TEST_F(ProgressReportTest, TestOverlappingEvents) {
-  ListenerSP listener_sp =
-      CreateListenerFor(lldb::eBroadcastBitProgressCategory);
-  EventSP event_sp;
-  const ProgressEventData *data;
-
-  // Create two progress reports of the same category that overlap with each
-  // other. Here we want to ensure that the ID broadcasted for the initial and
-  // final reports for this category are the same.
-  std::unique_ptr<Progress> overlap_progress1 =
-      std::make_unique<Progress>("Overlapping report 1", "Starting report 1");
-  std::unique_ptr<Progress> overlap_progress2 =
-      std::make_unique<Progress>("Overlapping report 1", "Starting report 2");
-  overlap_progress1.reset();
-
-  ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT));
-  data = ProgressEventData::GetEventDataFromEvent(event_sp.get());
-  // Get the ID used in the first report for this category.
-  uint64_t expected_progress_id = data->GetID();
-
-  EXPECT_EQ(data->GetDetails(), "");
-  EXPECT_FALSE(data->IsFinite());
-  EXPECT_FALSE(data->GetCompleted());
-  EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal);
-  EXPECT_EQ(data->GetMessage(), "Overlapping report 1");
-
-  overlap_progress2.reset();
-
-  ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT));
-  data = ProgressEventData::GetEventDataFromEvent(event_sp.get());
-
-  EXPECT_EQ(data->GetDetails(), "");
-  EXPECT_FALSE(data->IsFinite());
-  EXPECT_TRUE(data->GetCompleted());
-  EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal);
-  EXPECT_EQ(data->GetMessage(), "Overlapping report 1");
-  // The progress ID for the final report should be the same as that for the
-  // initial report.
-  EXPECT_EQ(data->GetID(), expected_progress_id);
-}
-
-TEST_F(ProgressReportTest, TestProgressManagerDisjointReports) {
-  ListenerSP listener_sp =
-      CreateListenerFor(lldb::eBroadcastBitProgressCategory);
-  EventSP event_sp;
-  const ProgressEventData *data;
-  uint64_t expected_progress_id;
-
-  { Progress progress("Coalesced report 1", "Starting report 1"); }
-  { Progress progress("Coalesced report 1", "Starting report 2"); }
-  { Progress progress("Coalesced report 1", "Starting report 3"); }
-
-  ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT));
-  data = ProgressEventData::GetEventDataFromEvent(event_sp.get());
-  expected_progress_id = data->GetID();
-
-  EXPECT_EQ(data->GetDetails(), "");
-  EXPECT_FALSE(data->IsFinite());
-  EXPECT_FALSE(data->GetCompleted());
-  EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal);
-  EXPECT_EQ(data->GetMessage(), "Coalesced report 1");
-
-  ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT));
-  data = ProgressEventData::GetEventDataFromEvent(event_sp.get());
-
-  EXPECT_EQ(data->GetID(), expected_progress_id);
-  EXPECT_EQ(data->GetDetails(), "");
-  EXPECT_FALSE(data->IsFinite());
-  EXPECT_TRUE(data->GetCompleted());
-  EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal);
-  EXPECT_EQ(data->GetMessage(), "Coalesced report 1");
-
-  ASSERT_FALSE(listener_sp->GetEvent(event_sp, TIMEOUT));
-}
-
 TEST_F(ProgressReportTest, TestExternalReportCreation) {
   ListenerSP listener_sp =
       CreateListenerFor(lldb::eBroadcastBitExternalProgress);
diff --git a/lldb/unittests/Core/TelemetryTest.cpp b/lldb/unittests/Core/TelemetryTest.cpp
index 7fc2b3d0264e3..72db8c17f09ea 100644
--- a/lldb/unittests/Core/TelemetryTest.cpp
+++ b/lldb/unittests/Core/TelemetryTest.cpp
@@ -52,8 +52,8 @@ class TestDestination : public llvm::telemetry::Destination {
 class FakePlugin : public telemetry::TelemetryManager {
 public:
   FakePlugin()
-      : telemetry::TelemetryManager(
-            std::make_unique<llvm::telemetry::Config>(true)) {}
+      : telemetry::TelemetryManager(std::make_unique<telemetry::LLDBConfig>(
+            /*enable_telemetry=*/true, /*detailed_command_telemetry=*/true)) {}
 
   // TelemetryManager interface
   llvm::Error preDispatch(llvm::telemetry::TelemetryInfo *entry) override {
diff --git a/lldb/unittests/Host/AlarmTest.cpp b/lldb/unittests/Host/AlarmTest.cpp
deleted file mode 100644
index 94e72af3ffe8d..0000000000000
--- a/lldb/unittests/Host/AlarmTest.cpp
+++ /dev/null
@@ -1,172 +0,0 @@
-//===-- AlarmTest.cpp -----------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lldb/Host/Alarm.h"
-#include "gtest/gtest.h"
-
-#include <chrono>
-#include <thread>
-
-using namespace lldb_private;
-using namespace std::chrono_literals;
-
-// Increase the timeout tenfold when running under ASan as it can have about the
-// same performance overhead.
-#if __has_feature(address_sanitizer)
-static constexpr auto TEST_TIMEOUT = 10000ms;
-#else
-static constexpr auto TEST_TIMEOUT = 1000ms;
-#endif
-
-// The time between scheduling a callback and it getting executed. This should
-// NOT be increased under ASan.
-static constexpr auto ALARM_TIMEOUT = 500ms;
-
-// If there are any pending callbacks, make sure they run before the Alarm
-// object is destroyed.
-static constexpr bool RUN_CALLBACKS_ON_EXIT = true;
-
-TEST(AlarmTest, Create) {
-  std::mutex m;
-
-  std::vector<Alarm::TimePoint> callbacks_actual;
-  std::vector<Alarm::TimePoint> callbacks_expected;
-
-  Alarm alarm(ALARM_TIMEOUT, RUN_CALLBACKS_ON_EXIT);
-
-  // Create 5 alarms some time apart.
-  for (size_t i = 0; i < 5; ++i) {
-    callbacks_actual.emplace_back();
-    callbacks_expected.emplace_back(std::chrono::system_clock::now() +
-                                    ALARM_TIMEOUT);
-
-    alarm.Create([&callbacks_actual, &m, i]() {
-      std::lock_guard<std::mutex> guard(m);
-      callbacks_actual[i] = std::chrono::system_clock::now();
-    });
-
-    std::this_thread::sleep_for(ALARM_TIMEOUT / 5);
-  }
-
-  // Leave plenty of time for all the alarms to fire.
-  std::this_thread::sleep_for(TEST_TIMEOUT);
-
-  // Acquire the lock to check the callbacks.
-  std::lock_guard<std::mutex> guard(m);
-
-  // Make sure all the alarms fired around the expected time.
-  for (size_t i = 0; i < 5; ++i)
-    EXPECT_GE(callbacks_actual[i], callbacks_expected[i]);
-}
-
-TEST(AlarmTest, Exit) {
-  std::mutex m;
-
-  std::vector<Alarm::Handle> handles;
-  std::vector<bool> callbacks;
-
-  {
-    Alarm alarm(ALARM_TIMEOUT, RUN_CALLBACKS_ON_EXIT);
-
-    // Create 5 alarms.
-    for (size_t i = 0; i < 5; ++i) {
-      callbacks.emplace_back(false);
-
-      handles.push_back(alarm.Create([&callbacks, &m, i]() {
-        std::lock_guard<std::mutex> guard(m);
-        callbacks[i] = true;
-      }));
-    }
-
-    // Let the alarm go out of scope before any alarm had a chance to fire.
-  }
-
-  // Acquire the lock to check the callbacks.
-  std::lock_guard<std::mutex> guard(m);
-
-  // Make sure none of the alarms fired.
-  for (bool callback : callbacks)
-    EXPECT_TRUE(callback);
-}
-
-TEST(AlarmTest, Cancel) {
-  std::mutex m;
-
-  std::vector<Alarm::Handle> handles;
-  std::vector<bool> callbacks;
-
-  Alarm alarm(ALARM_TIMEOUT, RUN_CALLBACKS_ON_EXIT);
-
-  // Create 5 alarms.
-  for (size_t i = 0; i < 5; ++i) {
-    callbacks.emplace_back(false);
-
-    handles.push_back(alarm.Create([&callbacks, &m, i]() {
-      std::lock_guard<std::mutex> guard(m);
-      callbacks[i] = true;
-    }));
-  }
-
-  // Make sure we can cancel the first 4 alarms.
-  for (size_t i = 0; i < 4; ++i)
-    EXPECT_TRUE(alarm.Cancel(handles[i]));
-
-  // Leave plenty of time for all the alarms to fire.
-  std::this_thread::sleep_for(TEST_TIMEOUT);
-
-  // Acquire the lock to check the callbacks.
-  std::lock_guard<std::mutex> guard(m);
-
-  // Make sure none of the first 4 alarms fired.
-  for (size_t i = 0; i < 4; ++i)
-    EXPECT_FALSE(callbacks[i]);
-
-  // Make sure the fifth alarm still fired.
-  EXPECT_TRUE(callbacks[4]);
-}
-
-TEST(AlarmTest, Restart) {
-  std::mutex m;
-
-  std::vector<Alarm::Handle> handles;
-  std::vector<Alarm::TimePoint> callbacks_actual;
-  std::vector<Alarm::TimePoint> callbacks_expected;
-
-  Alarm alarm(ALARM_TIMEOUT, RUN_CALLBACKS_ON_EXIT);
-
-  // Create 5 alarms some time apart.
-  for (size_t i = 0; i < 5; ++i) {
-    callbacks_actual.emplace_back();
-    callbacks_expected.emplace_back(std::chrono::system_clock::now() +
-                                    ALARM_TIMEOUT);
-
-    handles.push_back(alarm.Create([&callbacks_actual, &m, i]() {
-      std::lock_guard<std::mutex> guard(m);
-      callbacks_actual[i] = std::chrono::system_clock::now();
-    }));
-
-    std::this_thread::sleep_for(ALARM_TIMEOUT / 5);
-  }
-
-  // Update the last 2 alarms.
-  for (size_t i = 3; i < 5; ++i) {
-    std::lock_guard<std::mutex> guard(m);
-    callbacks_expected[i] = std::chrono::system_clock::now() + ALARM_TIMEOUT;
-    EXPECT_TRUE(alarm.Restart(handles[i]));
-  }
-
-  // Leave plenty of time for all the alarms to fire.
-  std::this_thread::sleep_for(TEST_TIMEOUT);
-
-  // Acquire the lock to check the callbacks.
-  std::lock_guard<std::mutex> guard(m);
-
-  // Make sure all the alarms around the expected time.
-  for (size_t i = 0; i < 5; ++i)
-    EXPECT_GE(callbacks_actual[i], callbacks_expected[i]);
-}
diff --git a/lldb/unittests/Host/CMakeLists.txt b/lldb/unittests/Host/CMakeLists.txt
index 7c09932b39c2a..c959478970d18 100644
--- a/lldb/unittests/Host/CMakeLists.txt
+++ b/lldb/unittests/Host/CMakeLists.txt
@@ -1,5 +1,4 @@
 set (FILES
-  AlarmTest.cpp
   ConnectionFileDescriptorTest.cpp
   FileActionTest.cpp
   FileSystemTest.cpp
diff --git a/lldb/unittests/Symbol/SymtabTest.cpp b/lldb/unittests/Symbol/SymtabTest.cpp
index 7b8892e5b5c0f..e431c31e1eac1 100644
--- a/lldb/unittests/Symbol/SymtabTest.cpp
+++ b/lldb/unittests/Symbol/SymtabTest.cpp
@@ -6,8 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "Plugins/ObjectFile/ELF/ObjectFileELF.h"
 #include "Plugins/ObjectFile/Mach-O/ObjectFileMachO.h"
 #include "Plugins/SymbolFile/DWARF/SymbolFileDWARF.h"
+#include "Plugins/SymbolFile/Symtab/SymbolFileSymtab.h"
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 #include "TestingSupport/SubsystemRAII.h"
 #include "TestingSupport/TestUtilities.h"
@@ -31,7 +33,7 @@ using namespace lldb_private::plugin::dwarf;
 
 class SymtabTest : public testing::Test {
   SubsystemRAII<FileSystem, HostInfo, ObjectFileMachO, SymbolFileDWARF,
-                TypeSystemClang>
+                ObjectFileELF, SymbolFileSymtab, TypeSystemClang>
       subsystem;
 };
 
@@ -718,3 +720,44 @@ TEST_F(SymtabTest, TestDecodeCStringMaps) {
                                                  Symtab::eVisibilityAny);
   ASSERT_NE(symbol, nullptr);
 }
+
+TEST_F(SymtabTest, TestSymtabCreatedOnDemand) {
+  auto ExpectedFile = TestFile::fromYaml(R"(
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_EXEC
+  Machine:         EM_X86_64
+  Entry:           0x0000000000400180
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    Address:         0x0000000000400180
+    AddressAlign:    0x0000000000000010
+    Content:         554889E58B042500106000890425041060005DC3
+Symbols:
+  - Name:            _start
+    Type:            STT_FUNC
+    Section:         .text
+    Value:           0x0000000000400180
+    Size:            0x0000000000000014
+    Binding:         STB_GLOBAL
+...
+)");
+  ASSERT_THAT_EXPECTED(ExpectedFile, llvm::Succeeded());
+  auto module_sp = std::make_shared<Module>(ExpectedFile->moduleSpec());
+
+  // The symbol table should not be loaded by default.
+  Symtab *module_symtab = module_sp->GetSymtab(/*can_create=*/false);
+  ASSERT_EQ(module_symtab, nullptr);
+
+  // But it should be created on demand.
+  module_symtab = module_sp->GetSymtab(/*can_create=*/true);
+  ASSERT_NE(module_symtab, nullptr);
+
+  // And we should be able to get it again once it has been created.
+  Symtab *cached_module_symtab = module_sp->GetSymtab(/*can_create=*/false);
+  ASSERT_EQ(module_symtab, cached_module_symtab);
+}
diff --git a/lldb/unittests/Utility/XcodeSDKTest.cpp b/lldb/unittests/Utility/XcodeSDKTest.cpp
index bca7f1291c425..4db6a50fcf860 100644
--- a/lldb/unittests/Utility/XcodeSDKTest.cpp
+++ b/lldb/unittests/Utility/XcodeSDKTest.cpp
@@ -66,13 +66,14 @@ TEST(XcodeSDKTest, MergeTest) {
   empty.Merge(XcodeSDK("MacOSX10.14.Internal.sdk"));
   EXPECT_EQ(empty.GetString(), llvm::StringRef("MacOSX10.14.Internal.sdk"));
   EXPECT_FALSE(empty.GetSysroot());
-  empty.Merge(XcodeSDK("MacOSX9.5.Internal.sdk", FileSpec{"/Path/To/9.5.sdk"}));
+  empty.Merge(XcodeSDK("MacOSX9.5.Internal.sdk",
+                       FileSpec{"/Path/To/MacOSX9.5.Internal.sdk"}));
   EXPECT_FALSE(empty.GetSysroot());
-  empty.Merge(XcodeSDK("MacOSX12.5.sdk", FileSpec{"/Path/To/12.5.sdk"}));
-  EXPECT_EQ(empty.GetSysroot(), FileSpec{"/Path/To/12.5.sdk"});
+  empty.Merge(XcodeSDK("MacOSX12.5.sdk", FileSpec{"/Path/To/MacOSX12.5.sdk"}));
+  EXPECT_EQ(empty.GetSysroot(), FileSpec{"/Path/To/MacOSX12.5.sdk"});
   empty.Merge(XcodeSDK("MacOSX11.5.Internal.sdk",
-                       FileSpec{"/Path/To/12.5.Internal.sdk"}));
-  EXPECT_EQ(empty.GetSysroot(), FileSpec{"/Path/To/12.5.Internal.sdk"});
+                       FileSpec{"/Path/To/MacOSX11.5.Internal.sdk"}));
+  EXPECT_EQ(empty.GetSysroot(), FileSpec{"/Path/To/MacOSX12.5.Internal.sdk"});
 }
 
 #ifndef _WIN32
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index 9a5cd1d985886..c5d3e23a47f0e 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -1189,6 +1189,7 @@ endif()
 if (UNIX AND ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
           add_compile_definitions(_XOPEN_SOURCE=700)
           add_compile_definitions(_LARGE_FILE_API)
+          add_compile_options(-pthread)
 
   # Modules should be built with -shared -Wl,-G, so we can use runtime linking
   # with plugins.
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index def6addd595e8..81319cb62d436 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1733,6 +1733,10 @@ The AMDGPU backend supports the following LLVM IR attributes.
      "amdgpu-sgpr-hazard-mem-wait-cull-threshold"     Sets the number of active SGPR hazards that must be present before
                                                       inserting a cull sequence at a memory wait.
 
+     "amdgpu-promote-alloca-to-vector-max-regs"       Maximum vector size (in 32b registers) to create when promoting alloca.
+
+     "amdgpu-promote-alloca-to-vector-vgpr-ratio"     Ratio of VGPRs to budget for promoting alloca to vectors.
+
      ================================================ ==========================================================
 
 Calling Conventions
@@ -2320,6 +2324,9 @@ if needed.
   as position independent code. See :ref:`amdgpu-code-conventions` for
   information on conventions used in the isa generation.
 
+``.amdgpu.kernel.runtime.handle``
+  Symbols used for device enqueue.
+
 .. _amdgpu-note-records:
 
 Note Records
diff --git a/llvm/docs/CodeReview.rst b/llvm/docs/CodeReview.rst
index 4cfd91435247c..89b7adf13eaec 100644
--- a/llvm/docs/CodeReview.rst
+++ b/llvm/docs/CodeReview.rst
@@ -272,3 +272,21 @@ Developers should participate in code reviews as both reviewers and
 authors. If someone is kind enough to review your code, you should return the
 favor for someone else. Note that anyone is welcome to review and give feedback
 on a patch, but approval of patches should be consistent with the policy above.
+
+Upstreaming Changes to LLVM
+===========================
+
+When upstreaming your own changes from a downstream project to LLVM, simply
+follow the process outlined above.
+
+When upstreaming changes originally written by someone else:  
+
+* Ensure that there are no obstacles to upstreaming the code. In some cases,
+  this simply means checking with the original author(s) to ensure they are
+  aware of and approve the upstreaming. In other cases, licensing
+  considerations may be more complex.
+* Properly attribute the original changes, e.g., by creating a commit with
+  multiple authors (`GitHub guide <https://docs.github.com/en/pull-requests/committing-changes-to-your-project/creating-and-editing-commits/creating-a-commit-with-multiple-authors>`_).
+* Invite the original author(s) to review the changes, and also include
+  additional reviewers. Specifically, an LGTM from a (co-)author should not be
+  taken as approval to land a change.
diff --git a/llvm/docs/CommandGuide/llvm-objcopy.rst b/llvm/docs/CommandGuide/llvm-objcopy.rst
index 8dc1357635e1b..96f3247780b3b 100644
--- a/llvm/docs/CommandGuide/llvm-objcopy.rst
+++ b/llvm/docs/CommandGuide/llvm-objcopy.rst
@@ -57,9 +57,10 @@ multiple file formats.
 
 .. option:: --discard-all, -x
 
- Remove most local symbols from the output. Different file formats may limit
- this to a subset of the local symbols. For example, file and section symbols in
- ELF objects will not be discarded. Additionally, remove all debug sections.
+ Remove most local symbols not referenced by relocations from the output.
+ Different file formats may limit this to a subset of the local symbols. For
+ example, file and section symbols in ELF objects will not be discarded.
+ Additionally, remove all debug sections.
 
 .. option:: --dump-section <section>=<file>
 
@@ -386,7 +387,7 @@ them.
 
 .. option:: --discard-locals, -X
 
- Remove local symbols starting with ".L" from the output.
+ Remove local symbols starting with ".L" not referenced by relocations from the output.
 
 .. option:: --extract-dwo
 
diff --git a/llvm/docs/HowToCrossCompileLLVM.rst b/llvm/docs/HowToCrossCompileLLVM.rst
index de38852c5a786..d69e4dbc766c5 100644
--- a/llvm/docs/HowToCrossCompileLLVM.rst
+++ b/llvm/docs/HowToCrossCompileLLVM.rst
@@ -1,215 +1,238 @@
 ===================================================================
-How To Cross-Compile Clang/LLVM using Clang/LLVM
+How to cross-compile Clang/LLVM using Clang/LLVM
 ===================================================================
 
 Introduction
-============
+------------
 
 This document contains information about building LLVM and
-Clang on host machine, targeting another platform.
+Clang on a host machine, targeting another platform.
 
 For more information on how to use Clang as a cross-compiler,
 please check https://clang.llvm.org/docs/CrossCompilation.html.
 
-TODO: Add MIPS and other platforms to this document.
+This document describes cross-building a compiler in a single stage, using an
+existing ``clang`` install as the host compiler.
 
-Cross-Compiling from x86_64 to ARM
-==================================
+.. note::
+  These instructions have been tested for targeting 32-bit ARM, AArch64, or
+  64-bit RISC-V from an x86_64 Linux host. But should be equally applicable to
+  any other target.
 
-In this use case, we'll be using CMake and Ninja, on a Debian-based Linux
-system, cross-compiling from an x86_64 host (most Intel and AMD chips
-nowadays) to a hard-float ARM target (most ARM targets nowadays).
-
-The packages you'll need are:
-
- * ``cmake``
- * ``ninja-build`` (from backports in Ubuntu)
- * ``gcc-4.7-arm-linux-gnueabihf``
- * ``gcc-4.7-multilib-arm-linux-gnueabihf``
- * ``binutils-arm-linux-gnueabihf``
- * ``libgcc1-armhf-cross``
- * ``libsfgcc1-armhf-cross``
- * ``libstdc++6-armhf-cross``
- * ``libstdc++6-4.7-dev-armhf-cross``
-
-Configuring CMake
------------------
-
-For more information on how to configure CMake for LLVM/Clang,
-see :doc:`CMake`.
-
-The CMake options you need to add are:
-
- * ``-DCMAKE_SYSTEM_NAME=<target-system>``
- * ``-DCMAKE_INSTALL_PREFIX=<install-dir>``
- * ``-DLLVM_HOST_TRIPLE=arm-linux-gnueabihf``
- * ``-DLLVM_TARGETS_TO_BUILD=ARM``
-
-Note: ``CMAKE_CROSSCOMPILING`` is always set automatically when ``CMAKE_SYSTEM_NAME`` is set. Don't put ``-DCMAKE_CROSSCOMPILING=TRUE`` in your options.
-
-Also note that ``LLVM_HOST_TRIPLE`` specifies the triple of the system
-that the cross built LLVM is going to run on - the flag is named based
-on the autoconf build/host/target nomenclature. (This flag implicitly sets
-other defaults, such as ``LLVM_DEFAULT_TARGET_TRIPLE``.)
+Setting up a sysroot
+--------------------
 
-If you're compiling with GCC, you can use architecture options for your target,
-and the compiler driver will detect everything that it needs:
+You will need a sysroot that contains essential build dependencies compiled
+for the target architecture. In this case, we will be using CMake and Ninja on
+a Linux host and compiling against a Debian sysroot. Detailed instructions on
+producing sysroots are outside of the scope of this documentation, but the
+following instructions should work on any Linux distribution with these
+pre-requisites:
 
- * ``-DCMAKE_CXX_FLAGS='-march=armv7-a -mcpu=cortex-a9 -mfloat-abi=hard'``
+ * ``binfmt_misc`` configured to execute ``qemu-user`` for binaries of the
+   target architecture. This is done by installing the ``qemu-user-static``
+   and ``binfmt-support`` packages on Debian-derived distributions.
+ * Root access (setups involving ``proot`` or other tools to avoid this
+   requirement may be possible, but aren't described here).
+ * The ``debootstrap`` tool. This is available in most distributions.
 
-However, if you're using Clang, the driver might not be up-to-date with your
-specific Linux distribution, version or GCC layout, so you'll need to fudge.
+The following snippet will initialise sysroots for 32-bit Arm, AArch64, and
+64-bit RISC-V (just pick the target(s) you are interested in):
 
-In addition to the ones above, you'll also need:
+   .. code-block:: bash
 
- * ``--target=arm-linux-gnueabihf`` or whatever is the triple of your cross GCC.
- * ``'--sysroot=/usr/arm-linux-gnueabihf'``, ``'--sysroot=/opt/gcc/arm-linux-gnueabihf'``
-   or whatever is the location of your GCC's sysroot (where /lib, /bin etc are).
- * Appropriate use of ``-I`` and ``-L``, depending on how the cross GCC is installed,
-   and where are the libraries and headers.
+    sudo debootstrap --arch=armhf --variant=minbase --include=build-essential,symlinks stable sysroot-deb-armhf-stable
+    sudo debootstrap --arch=arm64 --variant=minbase --include=build-essential,symlinks stable sysroot-deb-arm64-stable
+    sudo debootstrap --arch=riscv64 --variant=minbase --include=build-essential,symlinks unstable sysroot-deb-riscv64-unstable
 
-You may also want to set the ``LLVM_NATIVE_TOOL_DIR`` option - pointing
-at a directory with prebuilt LLVM tools (``llvm-tblgen``, ``clang-tblgen``
-etc) for the build host, allowing you to them reuse them if available.
-E.g. ``-DLLVM_NATIVE_TOOL_DIR=<path-to-native-llvm-build>/bin``.
-If the option isn't set (or the directory doesn't contain all needed tools),
-the LLVM cross build will automatically launch a nested build to build the
-tools that are required.
+The created sysroot may contain absolute symlinks, which will resolve to a
+location within the host when accessed during compilation, so we must convert
+any absolute symlinks to relative ones:
 
-The CXX flags define the target, cpu (which in this case
-defaults to ``fpu=VFP3`` with NEON), and forcing the hard-float ABI. If you're
-using Clang as a cross-compiler, you will *also* have to set ``--sysroot``
-to make sure it picks the correct linker.
+   .. code-block:: bash
 
-When using Clang, it's important that you choose the triple to be *identical*
-to the GCC triple and the sysroot. This will make it easier for Clang to
-find the correct tools and include headers. But that won't mean all headers and
-libraries will be found. You'll still need to use ``-I`` and ``-L`` to locate
-those extra ones, depending on your distribution.
+    sudo chroot sysroot-of-your-choice symlinks -cr .
 
-Most of the time, what you want is to have a native compiler to the
-platform itself, but not others. So there's rarely a point in compiling
-all back-ends. For that reason, you should also set the
-``TARGETS_TO_BUILD`` to only build the back-end you're targeting to.
 
-You must set the ``CMAKE_INSTALL_PREFIX``, otherwise a ``ninja install``
-will copy ARM binaries to your root filesystem, which is not what you
-want.
+Configuring CMake and building
+------------------------------
 
-Hacks
------
+For more information on how to configure CMake for LLVM/Clang,
+see :doc:`CMake`. Following CMake's recommended practice, we will create a
+`toolchain file
+<https://cmake.org/cmake/help/book/mastering-cmake/chapter/Cross%20Compiling%20With%20CMake.html#toolchain-files>`_. 
 
-There are some bugs in current LLVM, which require some fiddling before
-running CMake:
+The following assumes you have a system install of ``clang`` and ``lld`` that
+will be used for cross compiling and that the listed commands are executed
+from within the root of a checkout of the ``llvm-project`` git repository.
 
-#. If you're using Clang as the cross-compiler, there is a problem in
-   the LLVM ARM back-end that is producing absolute relocations on
-   position-independent code (``R_ARM_THM_MOVW_ABS_NC``), so for now, you
-   should disable PIC:
+First, set variables in your shell session that will be used throughout the
+build instructions:
 
    .. code-block:: bash
 
-      -DLLVM_ENABLE_PIC=False
+    SYSROOT=$HOME/sysroot-deb-arm64-stable
+    TARGET=aarch64-linux-gnu
+    CFLAGS=""
 
-   This is not a problem, since Clang/LLVM libraries are statically
-   linked anyway, it shouldn't affect much.
+To customise details of the compilation target or choose a different
+architecture altogether, change the ``SYSROOT``,
+``TARGET``, and ``CFLAGS`` variables to something matching your target. For
+example, for 64-bit RISC-V you might set
+``SYSROOT=$HOME/sysroot-deb-riscv64-unstable``, ``TARGET=riscv64-linux-gnu``
+and ``CFLAGS="-march=rva20u64"``. Refer to documentation such as your target's
+compiler documentation or processor manual for guidance on which ``CFLAGS``
+settings may be appropriate. The specified ``TARGET`` should match the triple
+used within the sysroot (i.e. ``$SYSROOT/usr/lib/$TARGET`` should exist).
 
-#. The ARM libraries won't be installed in your system.
-   But the CMake prepare step, which checks for
-   dependencies, will check the *host* libraries, not the *target*
-   ones. Below there's a list of some dependencies, but your project could
-   have more, or this document could be outdated. You'll see the errors
-   while linking as an indication of that.
+Then execute the following snippet to create a toolchain file:
 
-   Debian based distros have a way to add ``multiarch``, which adds
-   a new architecture and allows you to install packages for those
-   systems. See https://wiki.debian.org/Multiarch/HOWTO for more info.
+   .. code-block:: bash
 
-   But not all distros will have that, and possibly not an easy way to
-   install them in any anyway, so you'll have to build/download
-   them separately.
+    cat - <<EOF > $TARGET-clang.cmake
+    set(CMAKE_SYSTEM_NAME Linux)
+    set(CMAKE_SYSROOT "$SYSROOT")
+    set(CMAKE_C_COMPILER_TARGET $TARGET)
+    set(CMAKE_CXX_COMPILER_TARGET $TARGET)
+    set(CMAKE_C_FLAGS_INIT "$CFLAGS")
+    set(CMAKE_CXX_FLAGS_INIT "$CFLAGS")
+    set(CMAKE_LINKER_TYPE LLD)
+    set(CMAKE_C_COMPILER clang)
+    set(CMAKE_CXX_COMPILER clang++)
+    set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+    set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
+    set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
+    set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
+    EOF
+
+
+Then configure and build by invoking ``cmake``:
 
-   A quick way of getting the libraries is to download them from
-   a distribution repository, like Debian (http://packages.debian.org/jessie/),
-   and download the missing libraries. Note that the ``libXXX``
-   will have the shared objects (``.so``) and the ``libXXX-dev`` will
-   give you the headers and the static (``.a``) library. Just in
-   case, download both.
+   .. code-block:: bash
 
-   The ones you need for ARM are: ``libtinfo``, ``zlib1g``,
-   ``libxml2`` and ``liblzma``. In the Debian repository you'll
-   find downloads for all architectures.
+    cmake -G Ninja \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DLLVM_ENABLE_PROJECTS="lld;clang" \
+      -DCMAKE_TOOLCHAIN_FILE=$(pwd)/$TARGET-clang.cmake \
+      -DLLVM_HOST_TRIPLE=$TARGET \
+      -DCMAKE_INSTALL_PREFIX=$HOME/clang-$TARGET \
+      -S llvm \
+      -B build/$TARGET
+    cmake --build build/$TARGET
+
+These options from the toolchain file and ``cmake`` invocation above are
+important:
+
+ * ``CMAKE_SYSTEM_NAME``: Perhaps surprisingly, explicitly setting this
+   variable `causes CMake to set
+   CMAKE_CROSSCOMPIILING <https://cmake.org/cmake/help/latest/variable/CMAKE_CROSSCOMPILING.html#variable:CMAKE_CROSSCOMPILING>`_.
+ * ``CMAKE_{C,CXX}_COMPILER_TARGET``: This will be used to set the
+   ``--target`` argument to ``clang``. The triple should match the triple used
+   within the sysroot (i.e. ``$SYSROOT/usr/lib/$TARGET`` should exist).
+ * ``CMAKE_FIND_ROOT_PATH_MODE_*``: These `control the search behaviour for
+   finding libraries, includes or binaries
+   <https://cmake.org/cmake/help/book/mastering-cmake/chapter/Cross%20Compiling%20With%20CMake.html#finding-external-libraries-programs-and-other-files>`_.
+   Setting these prevents files for the host being used in the build.
+ * ``LLVM_HOST_TRIPLE``: Specifies the target triple of the system the built
+   LLVM will run on, which also implicitly sets other defaults such as
+   ``LLVM_DEFAULT_TARGET_TRIPLE``. For example, if you are using an x86_64
+   host to compile for RISC-V, this will be a RISC-V triple.
+ * ``CMAKE_SYSROOT``: The path to the sysroot containing libraries and headers
+   for the target.
+ * ``CMAKE_INSTALL_PREFIX``: Setting this avoids installing binaries compiled
+   for the target system into system directories for the host system. It is
+   not required unless you are going to use the ``install`` target.
+
+See `LLVM's build documentation
+<https://llvm.org/docs/CMake.html#frequently-used-cmake-variables>`_ for more
+guidance on CMake variables (e.g. ``LLVM_TARGETS_TO_BUILD`` may be useful if
+your cross-compiled binaries only need to support compiling for one target).
+
+Working around a ninja dependency issue
+---------------------------------------
+
+If you followed the instructions above to create a sysroot, you may run into a
+`longstanding problem related to path canonicalization in ninja
+<https://github.com/ninja-build/ninja/issues/1330>`_. GCC canonicalizes system
+headers in dependency files, so when ninja reads them it does not need to do
+so. Clang does not do this, and unfortunately ninja does not implement the
+canonicalization logic at all, meaning for some system headers with symlinks
+in the paths, it can incorrectly compute a non-existing path and consider it
+as always modified.
+
+If you are suffering from this issue, you will find any attempt at an
+incremental build (including the suggested command to build the ``install``
+target in the next section) results in recompiling everything.  ``ninja -C
+build/$TARGET -t deps`` shows files in ``$SYSROOT/include/*`` that
+do not exist (as the ``$SYSROOT/include`` folder does not exist) and you can
+further confirm these files are causing ``ninja`` to determine a rebuild is
+necessary with ``ninja -C build/$TARGET -d deps``.
+
+A workaround is to create a symlink so that the incorrect
+``$SYSROOT/include/*`` dependencies resolve to files within
+``$SYSROOT/usr/include/*``. This works in practice for the simple
+cross-compilation use case described here, but is not a general solution.
 
-   After you download and unpack all ``.deb`` packages, copy all
-   ``.so`` and ``.a`` to a directory, make the appropriate
-   symbolic links (if necessary), and add the relevant ``-L``
-   and ``-I`` paths to ``-DCMAKE_CXX_FLAGS`` above.
+   .. code-block:: bash
 
+    sudo ln -s usr/include $SYSROOT/include
 
-Running CMake and Building
---------------------------
+Testing the just-built compiler
+-------------------------------
 
-Finally, if you're using your platform compiler, run:
+Confirm the ``clang`` binary was built for the expected target architecture:
 
    .. code-block:: bash
 
-     $ cmake -G Ninja <source-dir> -DCMAKE_BUILD_TYPE=<type> <options above>
+    $ file -L ./build/aarch64-linux-gnu/bin/clang
+    ./build/aarch64-linux-gnu/bin/clang: ELF 64-bit LSB pie executable, ARM aarch64, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-aarch64.so.1, for GNU/Linux 3.7.0, BuildID[sha1]=516b8b366a790fcd3563bee4aec0cdfcb90bb1c7, not stripped
 
-If you're using Clang as the cross-compiler, run:
+If you have ``qemu-user`` installed you can test the produced target binary
+either by invoking ``qemu-{target}-static`` directly:
 
    .. code-block:: bash
 
-     $ CC='clang' CXX='clang++' cmake -G Ninja <source-dir> -DCMAKE_BUILD_TYPE=<type> <options above>
-
-If you have ``clang``/``clang++`` on the path, it should just work, and special
-Ninja files will be created in the build directory. I strongly suggest
-you to run ``cmake`` on a separate build directory, *not* inside the
-source tree.
+    $ qemu-aarch64-static -L $SYSROOT ./build/aarch64-linux-gnu/bin/clang --version
+    clang version 21.0.0git (https://github.com/llvm/llvm-project cedfdc6e889c5c614a953ed1f44bcb45a405f8da)
+    Target: aarch64-unknown-linux-gnu
+    Thread model: posix
+    InstalledDir: /home/asb/llvm-project/build/aarch64-linux-gnu/bin
 
-To build, simply type:
+Or, if binfmt_misc is configured (as was necessary for debootstrap):
 
    .. code-block:: bash
 
-     $ ninja
+    $ export QEMU_LD_PREFIX=$SYSROOT; ./build/aarch64-linux-gnu/bin/clang --version
+    clang version 21.0.0git (https://github.com/llvm/llvm-project cedfdc6e889c5c614a953ed1f44bcb45a405f8da)
+    Target: aarch64-unknown-linux-gnu
+    Thread model: posix
+    InstalledDir: /home/asb/llvm-project/build/aarch64-linux-gnu/bin
 
-It should automatically find out how many cores you have, what are
-the rules that needs building and will build the whole thing.
-
-You can't run ``ninja check-all`` on this tree because the created
-binaries are targeted to ARM, not x86_64.
-
-Installing and Using
+Installing and using
 --------------------
 
-After the LLVM/Clang has built successfully, you should install it
-via:
-
-   .. code-block:: bash
-
-     $ ninja install
+.. note::
+  Use of the ``install`` target requires that you have set
+  ``CMAKE_INSTALL_PREFIX`` otherwise it will attempt to install in
+  directories under `/` on your host.
 
-which will create a sysroot on the install-dir. You can then tar
-that directory into a binary with the full triple name (for easy
-identification), like:
+If you want to transfer a copy of the built compiler to another machine, you
+can first install it to a location on the host via:
 
    .. code-block:: bash
 
-     $ ln -sf <install-dir> arm-linux-gnueabihf-clang
-     $ tar zchf arm-linux-gnueabihf-clang.tar.gz arm-linux-gnueabihf-clang
+    cmake --build build/$TARGET --target=install
 
-If you copy that tarball to your target board, you'll be able to use
-it for running the test-suite, for example. Follow the guidelines at
-https://llvm.org/docs/lnt/quickstart.html, unpack the tarball in the
-test directory, and use options:
+This will install the LLVM/Clang headers, binaries, libraries, and other files
+to paths within ``CMAKE_INSTALL_PREFIX``. Then tar that directory for transfer
+to a device that runs the target architecture natively:
 
    .. code-block:: bash
 
-     $ ./sandbox/bin/python sandbox/bin/lnt runtest nt \
-         --sandbox sandbox \
-         --test-suite `pwd`/test-suite \
-         --cc `pwd`/arm-linux-gnueabihf-clang/bin/clang \
-         --cxx `pwd`/arm-linux-gnueabihf-clang/bin/clang++
+    tar -czvf clang-$TARGET.tar.gz -C $HOME clang-$TARGET
 
-Remember to add the ``-jN`` options to ``lnt`` to the number of CPUs
-on your board. Also, the path to your clang has to be absolute, so
-you'll need the `pwd` trick above.
+The generated toolchain is portable, but requires compatible versions of any
+shared libraries it links against. This means using a sysroot that is as
+similar to your target operating system as possible is desirable. Other `CMake
+variables <https://llvm.org/docs/CMake.html#frequently-used-cmake-variables>`_
+may be helpful, for instance ``LLVM_STATIC_LINK_CXX_STDLIB``.
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 8f80344b854ee..b0de05da2fd1f 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -6319,21 +6319,24 @@ The following ``tag:`` values are valid:
   DW_TAG_union_type       = 23
 
 For ``DW_TAG_array_type``, the ``elements:`` should be :ref:`subrange
-descriptors <DISubrange>`, each representing the range of subscripts at that
-level of indexing. The ``DIFlagVector`` flag to ``flags:`` indicates that an
-array type is a native packed vector. The optional ``dataLocation`` is a
-DIExpression that describes how to get from an object's address to the actual
-raw data, if they aren't equivalent. This is only supported for array types,
-particularly to describe Fortran arrays, which have an array descriptor in
-addition to the array data. Alternatively it can also be DIVariable which
-has the address of the actual raw data. The Fortran language supports pointer
-arrays which can be attached to actual arrays, this attachment between pointer
-and pointee is called association.  The optional ``associated`` is a
-DIExpression that describes whether the pointer array is currently associated.
-The optional ``allocated`` is a DIExpression that describes whether the
-allocatable array is currently allocated.  The optional ``rank`` is a
-DIExpression that describes the rank (number of dimensions) of fortran assumed
-rank array (rank is known at runtime).
+descriptors <DISubrange>` or :ref:`subrange descriptors
+<DISubrangeType>`, each representing the range of subscripts at that
+level of indexing. The ``DIFlagVector`` flag to ``flags:`` indicates
+that an array type is a native packed vector. The optional
+``dataLocation`` is a DIExpression that describes how to get from an
+object's address to the actual raw data, if they aren't
+equivalent. This is only supported for array types, particularly to
+describe Fortran arrays, which have an array descriptor in addition to
+the array data. Alternatively it can also be DIVariable which has the
+address of the actual raw data. The Fortran language supports pointer
+arrays which can be attached to actual arrays, this attachment between
+pointer and pointee is called association.  The optional
+``associated`` is a DIExpression that describes whether the pointer
+array is currently associated.  The optional ``allocated`` is a
+DIExpression that describes whether the allocatable array is currently
+allocated.  The optional ``rank`` is a DIExpression that describes the
+rank (number of dimensions) of fortran assumed rank array (rank is
+known at runtime).
 
 For ``DW_TAG_enumeration_type``, the ``elements:`` should be :ref:`enumerator
 descriptors <DIEnumerator>`, each representing the definition of an enumeration
@@ -6378,6 +6381,50 @@ DISubrange
     !12 = !DIGlobalVariable(name: "count", scope: !8, file: !6, line: 22, type: !9)
     !13 = !DISubrange(count: !12, lowerBound: 0)
 
+.. _DISubrangeType:
+
+DISubrangeType
+""""""""""""""
+
+``DISubrangeType`` is similar to ``DISubrange``, but it is also a
+``DIType``.  It may be used as the type of an object, but could also
+be used as an array index.
+
+Like ``DISubrange``, it can hold a lower bound and count, or a lower
+bound and upper bound.  A ``DISubrangeType`` refers to the underlying
+type of which it is a subrange; this type can be an integer type or an
+enumeration type.
+
+A ``DISubrangeType`` may also have a stride -- unlike ``DISubrange``,
+this stride is a bit stride.  The stride is only useful when a
+``DISubrangeType`` is used as an array index type.
+
+Finally, ``DISubrangeType`` may have a bias.  In Ada, a program can
+request that a subrange value be stored in the minimum number of bits
+required.  In this situation, the stored value is biased by the lower
+bound -- e.g., a range ``-7 .. 0`` may take 3 bits in memory, and the
+value -5 would be stored as 2 (a bias of -7).
+
+.. code-block:: text
+
+    ; Scopes used in rest of example
+    !0 = !DIFile(filename: "vla.c", directory: "/path/to/file")
+    !1 = distinct !DICompileUnit(language: DW_LANG_C99, file: !0)
+    !2 = distinct !DISubprogram(name: "foo", scope: !1, file: !0, line: 5)
+
+    ; Base type used in example.
+    !3 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+
+    ; A simple subrange with a name.
+    !4 = !DISubrange(name: "subrange", file: !0, line: 17, size: 32,
+                     align: 32, baseType: !3, lowerBound: 18, count: 12)
+    ; A subrange with a bias.
+    !5 = !DISubrange(name: "biased", lowerBound: -7, upperBound: 0,
+                     bias: -7, size: 3)
+    ; A subrange with a bit stride.
+    !6 = !DISubrange(name: "biased", lowerBound: 0, upperBound: 7,
+                     stride: 3)
+
 .. _DIEnumerator:
 
 DIEnumerator
diff --git a/llvm/docs/NVPTXUsage.rst b/llvm/docs/NVPTXUsage.rst
index bf5bcb6111829..621879fc5648b 100644
--- a/llvm/docs/NVPTXUsage.rst
+++ b/llvm/docs/NVPTXUsage.rst
@@ -1446,6 +1446,38 @@ The last argument `i1 %unpack` is a compile-time constant which when set, indica
 For more information, refer to the
 `PTX ISA <https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-st>`__.
 
+Store Intrinsics
+----------------
+
+'``llvm.nvvm.st.bulk.*``'
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+.. code-block:: llvm
+
+  declare void @llvm.nvvm.st.bulk(ptr addrspace(1) %dst, i64 %size, i64 immarg %initval)
+  declare void @llvm.nvvm.st.bulk.shared.cta(ptr addrspace(3) %dst, i64 %size, i64 immarg %initval)
+
+Overview:
+"""""""""
+
+The '``@llvm.nvvm.st.bulk.*``' intrinsics initialize a region of shared memory 
+starting from the location specified by the destination address operand `%dst`.
+
+The integer operand `%size` specifies the amount of memory to be initialized in 
+terms of number of bytes and must be a multiple of 8. Otherwise, the behavior 
+is undefined.
+
+The integer immediate operand `%initval` specifies the initialization value for 
+the memory locations. The only numeric value allowed is 0.
+
+The ``@llvm.nvvm.st.bulk.shared.cta`` and ``@llvm.nvvm.st.bulk`` intrinsics are 
+similar but the latter uses generic addressing (see `Generic Addressing <https://docs.nvidia.com/cuda/parallel-thread-execution/#generic-addressing>`__).
+
+For more information, refer `PTX ISA <https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-st-bulk>`__.
+
 Other Intrinsics
 ----------------
 
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index 62c6a4fd80fd4..831e8fd5d186b 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -454,7 +454,7 @@ The current vendor extensions supported are:
   LLVM implements `version 0.2 of the Qualcomm uC CSR extension specification <https://github.com/quic/riscv-unified-db/releases/latest>`__ by Qualcomm.  All instructions are prefixed with `qc.` as described in the specification. These instructions are only available for riscv32.
 
 ``experimental-Xqciint``
-  LLVM implements `version 0.2 of the Qualcomm uC Interrupts extension specification <https://github.com/quic/riscv-unified-db/releases/latest>`__ by Qualcomm.  All instructions are prefixed with `qc.` as described in the specification. These instructions are only available for riscv32.
+  LLVM implements `version 0.4 of the Qualcomm uC Interrupts extension specification <https://github.com/quic/riscv-unified-db/releases/latest>`__ by Qualcomm.  All instructions are prefixed with `qc.` as described in the specification. These instructions are only available for riscv32.
 
 ``experimental-Xqcilia``
   LLVM implements `version 0.2 of the Qualcomm uC Large Immediate Arithmetic extension specification <https://github.com/quic/riscv-unified-db/releases/latest>`__ by Qualcomm.  All instructions are prefixed with `qc.` as described in the specification. These instructions are only available for riscv32.
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index f2a477d9bdcc9..3d608c636a8f6 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -159,6 +159,8 @@ Changes to the LLVM tools
 ---------------------------------
 
 * llvm-objcopy now supports the `--update-section` flag for intermediate Mach-O object files.
+* llvm-strip now supports continuing to process files on encountering an error.
+* In llvm-objcopy's ELF port, `--discard-locals` and `--discard-all` now allow and preserve symbols referenced by relocations.
 
 Changes to LLDB
 ---------------------------------
diff --git a/llvm/include/llvm/ADT/APFloat.h b/llvm/include/llvm/ADT/APFloat.h
index 00d92740259eb..25fc8a00f7ddd 100644
--- a/llvm/include/llvm/ADT/APFloat.h
+++ b/llvm/include/llvm/ADT/APFloat.h
@@ -791,6 +791,8 @@ class IEEEFloat final {
 
   /// Sign bit of the number.
   unsigned int sign : 1;
+
+  friend class IEEEFloatUnitTestHelper;
 };
 
 hash_code hash_value(const IEEEFloat &Arg);
diff --git a/llvm/include/llvm/ADT/EquivalenceClasses.h b/llvm/include/llvm/ADT/EquivalenceClasses.h
index 4f98b84cf97d2..c375d6e77b12a 100644
--- a/llvm/include/llvm/ADT/EquivalenceClasses.h
+++ b/llvm/include/llvm/ADT/EquivalenceClasses.h
@@ -144,7 +144,7 @@ class EquivalenceClasses {
     operator=(RHS);
   }
 
-  const EquivalenceClasses &operator=(const EquivalenceClasses &RHS) {
+  EquivalenceClasses &operator=(const EquivalenceClasses &RHS) {
     TheMapping.clear();
     for (iterator I = RHS.begin(), E = RHS.end(); I != E; ++I)
       if (I->isLeader()) {
diff --git a/llvm/include/llvm/ADT/GenericUniformityImpl.h b/llvm/include/llvm/ADT/GenericUniformityImpl.h
index bd09f4fe43e08..d10355fff1bea 100644
--- a/llvm/include/llvm/ADT/GenericUniformityImpl.h
+++ b/llvm/include/llvm/ADT/GenericUniformityImpl.h
@@ -57,6 +57,9 @@
 
 namespace llvm {
 
+// Forward decl from llvm/CodeGen/MachineInstr.h
+class MachineInstr;
+
 /// Construct a specially modified post-order traversal of cycles.
 ///
 /// The ModifiedPO is contructed using a virtually modified CFG as follows:
@@ -342,6 +345,9 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
       typename SyncDependenceAnalysisT::DivergenceDescriptor;
   using BlockLabelMapT = typename SyncDependenceAnalysisT::BlockLabelMap;
 
+  using TemporalDivergenceTuple =
+      std::tuple<ConstValueRefT, InstructionT *, const CycleT *>;
+
   GenericUniformityAnalysisImpl(const DominatorTreeT &DT, const CycleInfoT &CI,
                                 const TargetTransformInfo *TTI)
       : Context(CI.getSSAContext()), F(*Context.getFunction()), CI(CI),
@@ -396,6 +402,11 @@ template <typename ContextT> class GenericUniformityAnalysisImpl {
 
   void print(raw_ostream &out) const;
 
+  SmallVector<TemporalDivergenceTuple, 8> TemporalDivergenceList;
+
+  void recordTemporalDivergence(ConstValueRefT, const InstructionT *,
+                                const CycleT *);
+
 protected:
   /// \brief Value/block pair representing a single phi input.
   struct PhiInput {
@@ -1129,6 +1140,13 @@ void GenericUniformityAnalysisImpl<ContextT>::compute() {
   }
 }
 
+template <typename ContextT>
+void GenericUniformityAnalysisImpl<ContextT>::recordTemporalDivergence(
+    ConstValueRefT Val, const InstructionT *User, const CycleT *Cycle) {
+  TemporalDivergenceList.emplace_back(Val, const_cast<InstructionT *>(User),
+                                      Cycle);
+}
+
 template <typename ContextT>
 bool GenericUniformityAnalysisImpl<ContextT>::isAlwaysUniform(
     const InstructionT &Instr) const {
@@ -1146,6 +1164,12 @@ template <typename ContextT>
 void GenericUniformityAnalysisImpl<ContextT>::print(raw_ostream &OS) const {
   bool haveDivergentArgs = false;
 
+  // When we print Value, LLVM IR instruction, we want to print extra new line.
+  // In LLVM IR print function for Value does not print new line at the end.
+  // In MIR print for MachineInstr prints new line at the end.
+  constexpr bool IsMIR = std::is_same<InstructionT, MachineInstr>::value;
+  std::string NewLine = IsMIR ? "" : "\n";
+
   // Control flow instructions may be divergent even if their inputs are
   // uniform. Thus, although exceedingly rare, it is possible to have a program
   // with no divergent values but with divergent control structures.
@@ -1180,6 +1204,16 @@ void GenericUniformityAnalysisImpl<ContextT>::print(raw_ostream &OS) const {
     }
   }
 
+  if (!TemporalDivergenceList.empty()) {
+    OS << "\nTEMPORAL DIVERGENCE LIST:\n";
+
+    for (auto [Val, UseInst, Cycle] : TemporalDivergenceList) {
+      OS << "Value         :" << Context.print(Val) << NewLine
+         << "Used by       :" << Context.print(UseInst) << NewLine
+         << "Outside cycle :" << Cycle->print(Context) << "\n\n";
+    }
+  }
+
   for (auto &block : F) {
     OS << "\nBLOCK " << Context.print(&block) << '\n';
 
@@ -1191,7 +1225,7 @@ void GenericUniformityAnalysisImpl<ContextT>::print(raw_ostream &OS) const {
         OS << "  DIVERGENT: ";
       else
         OS << "             ";
-      OS << Context.print(value) << '\n';
+      OS << Context.print(value) << NewLine;
     }
 
     OS << "TERMINATORS\n";
@@ -1203,13 +1237,21 @@ void GenericUniformityAnalysisImpl<ContextT>::print(raw_ostream &OS) const {
         OS << "  DIVERGENT: ";
       else
         OS << "             ";
-      OS << Context.print(T) << '\n';
+      OS << Context.print(T) << NewLine;
     }
 
     OS << "END BLOCK\n";
   }
 }
 
+template <typename ContextT>
+iterator_range<
+    typename GenericUniformityInfo<ContextT>::TemporalDivergenceTuple *>
+GenericUniformityInfo<ContextT>::getTemporalDivergenceList() const {
+  return make_range(DA->TemporalDivergenceList.begin(),
+                    DA->TemporalDivergenceList.end());
+}
+
 template <typename ContextT>
 bool GenericUniformityInfo<ContextT>::hasDivergence() const {
   return DA->hasDivergence();
diff --git a/llvm/include/llvm/ADT/GenericUniformityInfo.h b/llvm/include/llvm/ADT/GenericUniformityInfo.h
index e53afccc020b4..9376fa6ee0bae 100644
--- a/llvm/include/llvm/ADT/GenericUniformityInfo.h
+++ b/llvm/include/llvm/ADT/GenericUniformityInfo.h
@@ -40,6 +40,9 @@ template <typename ContextT> class GenericUniformityInfo {
   using CycleInfoT = GenericCycleInfo<ContextT>;
   using CycleT = typename CycleInfoT::CycleT;
 
+  using TemporalDivergenceTuple =
+      std::tuple<ConstValueRefT, InstructionT *, const CycleT *>;
+
   GenericUniformityInfo(const DominatorTreeT &DT, const CycleInfoT &CI,
                         const TargetTransformInfo *TTI = nullptr);
   GenericUniformityInfo() = default;
@@ -78,6 +81,8 @@ template <typename ContextT> class GenericUniformityInfo {
 
   void print(raw_ostream &Out) const;
 
+  iterator_range<TemporalDivergenceTuple *> getTemporalDivergenceList() const;
+
 private:
   using ImplT = GenericUniformityAnalysisImpl<ContextT>;
 
diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h
index ace5f60b572d7..78b7e94c2b3a1 100644
--- a/llvm/include/llvm/ADT/STLExtras.h
+++ b/llvm/include/llvm/ADT/STLExtras.h
@@ -320,7 +320,8 @@ template <typename T> class Callable<T, true> {
 
 /// Returns true if the given container only contains a single element.
 template <typename ContainerTy> bool hasSingleElement(ContainerTy &&C) {
-  auto B = std::begin(C), E = std::end(C);
+  auto B = adl_begin(C);
+  auto E = adl_end(C);
   return B != E && std::next(B) == E;
 }
 
@@ -375,8 +376,7 @@ inline mapped_iterator<ItTy, FuncTy> map_iterator(ItTy I, FuncTy F) {
 
 template <class ContainerTy, class FuncTy>
 auto map_range(ContainerTy &&C, FuncTy F) {
-  return make_range(map_iterator(std::begin(C), F),
-                    map_iterator(std::end(C), F));
+  return make_range(map_iterator(adl_begin(C), F), map_iterator(adl_end(C), F));
 }
 
 /// A base type of mapped iterator, that is useful for building derived
@@ -573,11 +573,9 @@ iterator_range<filter_iterator<detail::IterOfRange<RangeT>, PredicateT>>
 make_filter_range(RangeT &&Range, PredicateT Pred) {
   using FilterIteratorT =
       filter_iterator<detail::IterOfRange<RangeT>, PredicateT>;
-  return make_range(
-      FilterIteratorT(std::begin(std::forward<RangeT>(Range)),
-                      std::end(std::forward<RangeT>(Range)), Pred),
-      FilterIteratorT(std::end(std::forward<RangeT>(Range)),
-                      std::end(std::forward<RangeT>(Range)), Pred));
+  auto B = adl_begin(Range);
+  auto E = adl_end(Range);
+  return make_range(FilterIteratorT(B, E, Pred), FilterIteratorT(E, E, Pred));
 }
 
 /// A pseudo-iterator adaptor that is designed to implement "early increment"
@@ -657,8 +655,8 @@ iterator_range<early_inc_iterator_impl<detail::IterOfRange<RangeT>>>
 make_early_inc_range(RangeT &&Range) {
   using EarlyIncIteratorT =
       early_inc_iterator_impl<detail::IterOfRange<RangeT>>;
-  return make_range(EarlyIncIteratorT(std::begin(std::forward<RangeT>(Range))),
-                    EarlyIncIteratorT(std::end(std::forward<RangeT>(Range))));
+  return make_range(EarlyIncIteratorT(adl_begin(Range)),
+                    EarlyIncIteratorT(adl_end(Range)));
 }
 
 // Forward declarations required by zip_shortest/zip_equal/zip_first/zip_longest
@@ -1111,8 +1109,8 @@ class concat_iterator
   /// We need the full range to know how to switch between each of the
   /// iterators.
   template <typename... RangeTs>
-  explicit concat_iterator(RangeTs &&... Ranges)
-      : Begins(std::begin(Ranges)...), Ends(std::end(Ranges)...) {}
+  explicit concat_iterator(RangeTs &&...Ranges)
+      : Begins(adl_begin(Ranges)...), Ends(adl_end(Ranges)...) {}
 
   using BaseT::operator++;
 
@@ -1141,13 +1139,12 @@ template <typename ValueT, typename... RangeTs> class concat_range {
 public:
   using iterator =
       concat_iterator<ValueT,
-                      decltype(std::begin(std::declval<RangeTs &>()))...>;
+                      decltype(adl_begin(std::declval<RangeTs &>()))...>;
 
 private:
   std::tuple<RangeTs...> Ranges;
 
-  template <size_t... Ns>
-  iterator begin_impl(std::index_sequence<Ns...>) {
+  template <size_t... Ns> iterator begin_impl(std::index_sequence<Ns...>) {
     return iterator(std::get<Ns>(Ranges)...);
   }
   template <size_t... Ns>
@@ -1155,12 +1152,12 @@ template <typename ValueT, typename... RangeTs> class concat_range {
     return iterator(std::get<Ns>(Ranges)...);
   }
   template <size_t... Ns> iterator end_impl(std::index_sequence<Ns...>) {
-    return iterator(make_range(std::end(std::get<Ns>(Ranges)),
-                               std::end(std::get<Ns>(Ranges)))...);
+    return iterator(make_range(adl_end(std::get<Ns>(Ranges)),
+                               adl_end(std::get<Ns>(Ranges)))...);
   }
   template <size_t... Ns> iterator end_impl(std::index_sequence<Ns...>) const {
-    return iterator(make_range(std::end(std::get<Ns>(Ranges)),
-                               std::end(std::get<Ns>(Ranges)))...);
+    return iterator(make_range(adl_end(std::get<Ns>(Ranges)),
+                               adl_end(std::get<Ns>(Ranges)))...);
   }
 
 public:
@@ -1437,7 +1434,7 @@ template <typename EltTy, typename FirstTy> class first_or_second_type {
 
 /// Given a container of pairs, return a range over the first elements.
 template <typename ContainerTy> auto make_first_range(ContainerTy &&c) {
-  using EltTy = decltype((*std::begin(c)));
+  using EltTy = decltype(*adl_begin(c));
   return llvm::map_range(std::forward<ContainerTy>(c),
                          [](EltTy elt) -> typename detail::first_or_second_type<
                                            EltTy, decltype((elt.first))>::type {
@@ -1447,7 +1444,7 @@ template <typename ContainerTy> auto make_first_range(ContainerTy &&c) {
 
 /// Given a container of pairs, return a range over the second elements.
 template <typename ContainerTy> auto make_second_range(ContainerTy &&c) {
-  using EltTy = decltype((*std::begin(c)));
+  using EltTy = decltype(*adl_begin(c));
   return llvm::map_range(
       std::forward<ContainerTy>(c),
       [](EltTy elt) ->
@@ -2126,7 +2123,7 @@ void append_values(Container &C, Args &&...Values) {
 
 /// Given a sequence container Cont, replace the range [ContIt, ContEnd) with
 /// the range [ValIt, ValEnd) (which is not from the same container).
-template<typename Container, typename RandomAccessIterator>
+template <typename Container, typename RandomAccessIterator>
 void replace(Container &Cont, typename Container::iterator ContIt,
              typename Container::iterator ContEnd, RandomAccessIterator ValIt,
              RandomAccessIterator ValEnd) {
@@ -2134,21 +2131,24 @@ void replace(Container &Cont, typename Container::iterator ContIt,
     if (ValIt == ValEnd) {
       Cont.erase(ContIt, ContEnd);
       return;
-    } else if (ContIt == ContEnd) {
+    }
+    if (ContIt == ContEnd) {
       Cont.insert(ContIt, ValIt, ValEnd);
       return;
     }
-    *ContIt++ = *ValIt++;
+    *ContIt = *ValIt;
+    ++ContIt;
+    ++ValIt;
   }
 }
 
 /// Given a sequence container Cont, replace the range [ContIt, ContEnd) with
 /// the range R.
-template<typename Container, typename Range = std::initializer_list<
-                                 typename Container::value_type>>
+template <typename Container, typename Range = std::initializer_list<
+                                  typename Container::value_type>>
 void replace(Container &Cont, typename Container::iterator ContIt,
-             typename Container::iterator ContEnd, Range R) {
-  replace(Cont, ContIt, ContEnd, R.begin(), R.end());
+             typename Container::iterator ContEnd, Range &&R) {
+  replace(Cont, ContIt, ContEnd, adl_begin(R), adl_end(R));
 }
 
 /// An STL-style algorithm similar to std::for_each that applies a second
@@ -2561,19 +2561,19 @@ bool hasNItemsOrLess(
 
 /// Returns true if the given container has exactly N items
 template <typename ContainerTy> bool hasNItems(ContainerTy &&C, unsigned N) {
-  return hasNItems(std::begin(C), std::end(C), N);
+  return hasNItems(adl_begin(C), adl_end(C), N);
 }
 
 /// Returns true if the given container has N or more items
 template <typename ContainerTy>
 bool hasNItemsOrMore(ContainerTy &&C, unsigned N) {
-  return hasNItemsOrMore(std::begin(C), std::end(C), N);
+  return hasNItemsOrMore(adl_begin(C), adl_end(C), N);
 }
 
 /// Returns true if the given container has N or less items
 template <typename ContainerTy>
 bool hasNItemsOrLess(ContainerTy &&C, unsigned N) {
-  return hasNItemsOrLess(std::begin(C), std::end(C), N);
+  return hasNItemsOrLess(adl_begin(C), adl_end(C), N);
 }
 
 /// Returns a raw pointer that represents the same address as the argument.
diff --git a/llvm/include/llvm/ADT/bit.h b/llvm/include/llvm/ADT/bit.h
index c42b5e686bdc9..915a7de50aaff 100644
--- a/llvm/include/llvm/ADT/bit.h
+++ b/llvm/include/llvm/ADT/bit.h
@@ -15,6 +15,7 @@
 #define LLVM_ADT_BIT_H
 
 #include "llvm/Support/Compiler.h"
+#include <cstddef> // for std::size_t
 #include <cstdint>
 #include <limits>
 #include <type_traits>
diff --git a/llvm/include/llvm/ADT/ilist_node.h b/llvm/include/llvm/ADT/ilist_node.h
index bfd50f8b3fb71..67384546a9275 100644
--- a/llvm/include/llvm/ADT/ilist_node.h
+++ b/llvm/include/llvm/ADT/ilist_node.h
@@ -18,6 +18,8 @@
 #include "llvm/ADT/ilist_node_base.h"
 #include "llvm/ADT/ilist_node_options.h"
 
+#include <type_traits>
+
 namespace llvm {
 
 namespace ilist_detail {
@@ -147,9 +149,13 @@ class ilist_node_impl
   ///
   /// This requires sentinel tracking to be explicitly enabled.  Use the
   /// ilist_sentinel_tracking<true> option to get this API.
-  bool isSentinel() const {
-    static_assert(OptionsT::is_sentinel_tracking_explicit,
-                  "Use ilist_sentinel_tracking<true> to enable isSentinel()");
+  ///
+  /// Rather than using static_assert to enforce the API is not called when
+  /// configured with is_sentinel_tracking_explicit=false, the method is
+  /// conditionally provided using std::enable_if.  This way, clients of
+  /// ilist_node_impl can be fully instantiated for DLLExport on Windows.
+  template <typename T = OptionsT>
+  std::enable_if_t<T::is_sentinel_tracking_explicit, bool> isSentinel() const {
     return node_base_type::isSentinel();
   }
 };
diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
index 9509e5234840f..66ea5ecbc4bfb 100644
--- a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
+++ b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
@@ -1571,7 +1571,8 @@ void BlockFrequencyInfoImpl<BT>::initTransitionProbabilities(
     SmallPtrSet<const BlockT *, 2> UniqueSuccs;
     for (const auto SI : children<const BlockT *>(BB)) {
       // Ignore cold blocks
-      if (!BlockIndex.contains(SI))
+      auto BlockIndexIt = BlockIndex.find(SI);
+      if (BlockIndexIt == BlockIndex.end())
         continue;
       // Ignore parallel edges between BB and SI blocks
       if (!UniqueSuccs.insert(SI).second)
@@ -1583,7 +1584,7 @@ void BlockFrequencyInfoImpl<BT>::initTransitionProbabilities(
 
       auto EdgeProb =
           Scaled64::getFraction(EP.getNumerator(), EP.getDenominator());
-      size_t Dst = BlockIndex.find(SI)->second;
+      size_t Dst = BlockIndexIt->second;
       Succs[Src].push_back(std::make_pair(Dst, EdgeProb));
       SumProb[Src] += EdgeProb;
     }
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 3f03c3acddfd1..d46859bcb0517 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1986,27 +1986,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       }
       return Cost;
     }
-    case Intrinsic::get_active_lane_mask: {
-      EVT ResVT = getTLI()->getValueType(DL, RetTy, true);
-      EVT ArgType = getTLI()->getValueType(DL, ICA.getArgTypes()[0], true);
-
-      // If we're not expanding the intrinsic then we assume this is cheap
-      // to implement.
-      if (!getTLI()->shouldExpandGetActiveLaneMask(ResVT, ArgType)) {
-        return getTypeLegalizationCost(RetTy).first;
-      }
-
-      // Create the expanded types that will be used to calculate the uadd_sat
-      // operation.
-      Type *ExpRetTy = VectorType::get(
-          ICA.getArgTypes()[0], cast<VectorType>(RetTy)->getElementCount());
-      IntrinsicCostAttributes Attrs(Intrinsic::uadd_sat, ExpRetTy, {}, FMF);
-      InstructionCost Cost =
-          thisT()->getTypeBasedIntrinsicInstrCost(Attrs, CostKind);
-      Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, ExpRetTy, RetTy,
-                                          CmpInst::ICMP_ULT, CostKind);
-      return Cost;
-    }
     case Intrinsic::experimental_cttz_elts: {
       EVT ArgType = getTLI()->getValueType(DL, ICA.getArgTypes()[0], true);
 
@@ -2054,6 +2033,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
       return Cost;
     }
+    case Intrinsic::get_active_lane_mask:
     case Intrinsic::experimental_vector_match:
       return thisT()->getTypeBasedIntrinsicInstrCost(ICA, CostKind);
     case Intrinsic::modf:
@@ -2394,6 +2374,27 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
           thisT()->getArithmeticInstrCost(BinaryOperator::And, RetTy, CostKind);
       return Cost;
     }
+    case Intrinsic::get_active_lane_mask: {
+      Type *ArgTy = ICA.getArgTypes()[0];
+      EVT ResVT = getTLI()->getValueType(DL, RetTy, true);
+      EVT ArgVT = getTLI()->getValueType(DL, ArgTy, true);
+
+      // If we're not expanding the intrinsic then we assume this is cheap
+      // to implement.
+      if (!getTLI()->shouldExpandGetActiveLaneMask(ResVT, ArgVT))
+        return getTypeLegalizationCost(RetTy).first;
+
+      // Create the expanded types that will be used to calculate the uadd_sat
+      // operation.
+      Type *ExpRetTy =
+          VectorType::get(ArgTy, cast<VectorType>(RetTy)->getElementCount());
+      IntrinsicCostAttributes Attrs(Intrinsic::uadd_sat, ExpRetTy, {}, FMF);
+      InstructionCost Cost =
+          thisT()->getTypeBasedIntrinsicInstrCost(Attrs, CostKind);
+      Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, ExpRetTy, RetTy,
+                                          CmpInst::ICMP_ULT, CostKind);
+      return Cost;
+    }
     case Intrinsic::abs:
       ISD = ISD::ABS;
       break;
diff --git a/llvm/include/llvm/CodeGen/DetectDeadLanes.h b/llvm/include/llvm/CodeGen/DetectDeadLanes.h
index 349bf794627b0..5efe65962cf07 100644
--- a/llvm/include/llvm/CodeGen/DetectDeadLanes.h
+++ b/llvm/include/llvm/CodeGen/DetectDeadLanes.h
@@ -29,6 +29,7 @@
 #define LLVM_CODEGEN_DETECTDEADLANES_H
 
 #include "llvm/ADT/BitVector.h"
+#include "llvm/CodeGen/MachinePassManager.h"
 #include "llvm/MC/LaneBitmask.h"
 #include <deque>
 
@@ -115,6 +116,13 @@ class DeadLaneDetector {
   BitVector DefinedByCopy;
 };
 
+class DetectDeadLanesPass : public PassInfoMixin<DetectDeadLanesPass> {
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+  static bool isRequired() { return true; }
+};
+
 } // end namespace llvm
 
 #endif // LLVM_CODEGEN_DETECTDEADLANES_H
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h b/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
index 78a92c86b91e4..72483fbea5805 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
@@ -33,6 +33,12 @@ template <typename Pattern>
   return P.match(MRI, &MI);
 }
 
+template <typename Pattern>
+[[nodiscard]] bool mi_match(const MachineInstr &MI,
+                            const MachineRegisterInfo &MRI, Pattern &&P) {
+  return P.match(MRI, &MI);
+}
+
 // TODO: Extend for N use.
 template <typename SubPatternT> struct OneUse_match {
   SubPatternT SubPat;
@@ -337,6 +343,19 @@ template <> struct bind_helper<MachineInstr *> {
   }
 };
 
+template <> struct bind_helper<const MachineInstr *> {
+  static bool bind(const MachineRegisterInfo &MRI, const MachineInstr *&MI,
+                   Register Reg) {
+    MI = MRI.getVRegDef(Reg);
+    return MI;
+  }
+  static bool bind(const MachineRegisterInfo &MRI, const MachineInstr *&MI,
+                   const MachineInstr *Inst) {
+    MI = Inst;
+    return MI;
+  }
+};
+
 template <> struct bind_helper<LLT> {
   static bool bind(const MachineRegisterInfo &MRI, LLT &Ty, Register Reg) {
     Ty = MRI.getType(Reg);
@@ -368,6 +387,9 @@ template <typename Class> struct bind_ty {
 
 inline bind_ty<Register> m_Reg(Register &R) { return R; }
 inline bind_ty<MachineInstr *> m_MInstr(MachineInstr *&MI) { return MI; }
+inline bind_ty<const MachineInstr *> m_MInstr(const MachineInstr *&MI) {
+  return MI;
+}
 inline bind_ty<LLT> m_Type(LLT &Ty) { return Ty; }
 inline bind_ty<CmpInst::Predicate> m_Pred(CmpInst::Predicate &P) { return P; }
 inline operand_type_match m_Pred() { return operand_type_match(); }
@@ -418,7 +440,7 @@ inline bind_ty<const ConstantFP *> m_GFCst(const ConstantFP *&C) { return C; }
 
 // General helper for all the binary generic MI such as G_ADD/G_SUB etc
 template <typename LHS_P, typename RHS_P, unsigned Opcode,
-          bool Commutable = false>
+          bool Commutable = false, unsigned Flags = MachineInstr::NoFlags>
 struct BinaryOp_match {
   LHS_P L;
   RHS_P R;
@@ -426,18 +448,20 @@ struct BinaryOp_match {
   BinaryOp_match(const LHS_P &LHS, const RHS_P &RHS) : L(LHS), R(RHS) {}
   template <typename OpTy>
   bool match(const MachineRegisterInfo &MRI, OpTy &&Op) {
-    MachineInstr *TmpMI;
+    const MachineInstr *TmpMI;
     if (mi_match(Op, MRI, m_MInstr(TmpMI))) {
       if (TmpMI->getOpcode() == Opcode && TmpMI->getNumOperands() == 3) {
-        return (L.match(MRI, TmpMI->getOperand(1).getReg()) &&
-                R.match(MRI, TmpMI->getOperand(2).getReg())) ||
-               // NOTE: When trying the alternative operand ordering
-               // with a commutative operation, it is imperative to always run
-               // the LHS sub-pattern  (i.e. `L`) before the RHS sub-pattern
-               // (i.e. `R`). Otherwsie, m_DeferredReg/Type will not work as
-               // expected.
-               (Commutable && (L.match(MRI, TmpMI->getOperand(2).getReg()) &&
-                               R.match(MRI, TmpMI->getOperand(1).getReg())));
+        if ((!L.match(MRI, TmpMI->getOperand(1).getReg()) ||
+             !R.match(MRI, TmpMI->getOperand(2).getReg())) &&
+            // NOTE: When trying the alternative operand ordering
+            // with a commutative operation, it is imperative to always run
+            // the LHS sub-pattern  (i.e. `L`) before the RHS sub-pattern
+            // (i.e. `R`). Otherwise, m_DeferredReg/Type will not work as
+            // expected.
+            (!Commutable || !L.match(MRI, TmpMI->getOperand(2).getReg()) ||
+             !R.match(MRI, TmpMI->getOperand(1).getReg())))
+          return false;
+        return (TmpMI->getFlags() & Flags) == Flags;
       }
     }
     return false;
@@ -464,7 +488,7 @@ struct BinaryOpc_match {
                // NOTE: When trying the alternative operand ordering
                // with a commutative operation, it is imperative to always run
                // the LHS sub-pattern  (i.e. `L`) before the RHS sub-pattern
-               // (i.e. `R`). Otherwsie, m_DeferredReg/Type will not work as
+               // (i.e. `R`). Otherwise, m_DeferredReg/Type will not work as
                // expected.
                (Commutable && (L.match(MRI, TmpMI->getOperand(2).getReg()) &&
                                R.match(MRI, TmpMI->getOperand(1).getReg())));
@@ -559,6 +583,19 @@ inline BinaryOp_match<LHS, RHS, TargetOpcode::G_OR, true> m_GOr(const LHS &L,
   return BinaryOp_match<LHS, RHS, TargetOpcode::G_OR, true>(L, R);
 }
 
+template <typename LHS, typename RHS>
+inline BinaryOp_match<LHS, RHS, TargetOpcode::G_OR, true,
+                      MachineInstr::Disjoint>
+m_GDisjointOr(const LHS &L, const RHS &R) {
+  return BinaryOp_match<LHS, RHS, TargetOpcode::G_OR, true,
+                        MachineInstr::Disjoint>(L, R);
+}
+
+template <typename LHS, typename RHS>
+inline auto m_GAddLike(const LHS &L, const RHS &R) {
+  return m_any_of(m_GAdd(L, R), m_GDisjointOr(L, R));
+}
+
 template <typename LHS, typename RHS>
 inline BinaryOp_match<LHS, RHS, TargetOpcode::G_SHL, false>
 m_GShl(const LHS &L, const RHS &R) {
@@ -717,7 +754,7 @@ struct CompareOp_match {
     // NOTE: When trying the alternative operand ordering
     // with a commutative operation, it is imperative to always run
     // the LHS sub-pattern  (i.e. `L`) before the RHS sub-pattern
-    // (i.e. `R`). Otherwsie, m_DeferredReg/Type will not work as expected.
+    // (i.e. `R`). Otherwise, m_DeferredReg/Type will not work as expected.
     if (Commutable && L.match(MRI, RHS) && R.match(MRI, LHS) &&
         P.match(MRI, CmpInst::getSwappedPredicate(TmpPred)))
       return true;
diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index 102b1eb07358e..b26cabe801ee8 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -994,8 +994,17 @@ class MachineInstr
 
   /// Return true if this is an indirect branch, such as a
   /// branch through a register.
-  bool isIndirectBranch(QueryType Type = AnyInBundle) const {
-    return hasProperty(MCID::IndirectBranch, Type);
+  bool isIndirectBranch(QueryType Type = AnyInBundle,
+                        bool IncludeJumpTable = true) const {
+    return hasProperty(MCID::IndirectBranch, Type) &&
+           (IncludeJumpTable || !llvm::any_of(operands(), [](const auto &Op) {
+              return Op.isJTI();
+            }));
+  }
+
+  bool isComputedGoto(QueryType Type = AnyInBundle) const {
+    // Jump tables are not considered computed gotos.
+    return isIndirectBranch(Type, /*IncludeJumpTable=*/false);
   }
 
   /// Return true if this is a branch which may fall
diff --git a/llvm/include/llvm/CodeGen/MachinePipeliner.h b/llvm/include/llvm/CodeGen/MachinePipeliner.h
index 0362b501ed347..120b559c20305 100644
--- a/llvm/include/llvm/CodeGen/MachinePipeliner.h
+++ b/llvm/include/llvm/CodeGen/MachinePipeliner.h
@@ -269,7 +269,7 @@ class SwingSchedulerDAG : public ScheduleDAGInstrs {
   using InstrMapTy = DenseMap<MachineInstr *, MachineInstr *>;
 
   /// Instructions to change when emitting the final schedule.
-  DenseMap<SUnit *, std::pair<unsigned, int64_t>> InstrChanges;
+  DenseMap<SUnit *, std::pair<Register, int64_t>> InstrChanges;
 
   /// We may create a new instruction, so remember it because it
   /// must be deleted when the pass is finished.
@@ -374,12 +374,12 @@ class SwingSchedulerDAG : public ScheduleDAGInstrs {
 
   /// Return the new base register that was stored away for the changed
   /// instruction.
-  unsigned getInstrBaseReg(SUnit *SU) const {
-    DenseMap<SUnit *, std::pair<unsigned, int64_t>>::const_iterator It =
+  Register getInstrBaseReg(SUnit *SU) const {
+    DenseMap<SUnit *, std::pair<Register, int64_t>>::const_iterator It =
         InstrChanges.find(SU);
     if (It != InstrChanges.end())
       return It->second.first;
-    return 0;
+    return Register();
   }
 
   void addMutation(std::unique_ptr<ScheduleDAGMutation> Mutation) {
@@ -415,7 +415,7 @@ class SwingSchedulerDAG : public ScheduleDAGInstrs {
   bool computeDelta(const MachineInstr &MI, int &Delta) const;
   MachineInstr *findDefInLoop(Register Reg);
   bool canUseLastOffsetValue(MachineInstr *MI, unsigned &BasePos,
-                             unsigned &OffsetPos, unsigned &NewBase,
+                             unsigned &OffsetPos, Register &NewBase,
                              int64_t &NewOffset);
   void postProcessDAG();
   /// Set the Minimum Initiation Interval for this schedule attempt.
diff --git a/llvm/include/llvm/CodeGen/ModuloSchedule.h b/llvm/include/llvm/CodeGen/ModuloSchedule.h
index 64598ce449a44..49dc746d3ee35 100644
--- a/llvm/include/llvm/CodeGen/ModuloSchedule.h
+++ b/llvm/include/llvm/CodeGen/ModuloSchedule.h
@@ -160,10 +160,10 @@ class ModuloSchedule {
 /// rewriting the old loop and inserting prologs and epilogs as required.
 class ModuloScheduleExpander {
 public:
-  using InstrChangesTy = DenseMap<MachineInstr *, std::pair<unsigned, int64_t>>;
+  using InstrChangesTy = DenseMap<MachineInstr *, std::pair<Register, int64_t>>;
 
 private:
-  using ValueMapTy = DenseMap<unsigned, unsigned>;
+  using ValueMapTy = DenseMap<Register, Register>;
   using MBBVectorTy = SmallVectorImpl<MachineBasicBlock *>;
   using InstrMapTy = DenseMap<MachineInstr *, MachineInstr *>;
 
@@ -183,7 +183,7 @@ class ModuloScheduleExpander {
   /// The first element in the pair is the max difference in stages. The
   /// second is true if the register defines a Phi value and loop value is
   /// scheduled before the Phi.
-  std::map<unsigned, std::pair<unsigned, bool>> RegToStageDiff;
+  std::map<Register, std::pair<unsigned, bool>> RegToStageDiff;
 
   /// Instructions to change when emitting the final schedule.
   InstrChangesTy InstrChanges;
@@ -221,21 +221,21 @@ class ModuloScheduleExpander {
   void updateInstruction(MachineInstr *NewMI, bool LastDef,
                          unsigned CurStageNum, unsigned InstrStageNum,
                          ValueMapTy *VRMap);
-  MachineInstr *findDefInLoop(unsigned Reg);
-  unsigned getPrevMapVal(unsigned StageNum, unsigned PhiStage, unsigned LoopVal,
+  MachineInstr *findDefInLoop(Register Reg);
+  Register getPrevMapVal(unsigned StageNum, unsigned PhiStage, Register LoopVal,
                          unsigned LoopStage, ValueMapTy *VRMap,
                          MachineBasicBlock *BB);
   void rewritePhiValues(MachineBasicBlock *NewBB, unsigned StageNum,
                         ValueMapTy *VRMap, InstrMapTy &InstrMap);
   void rewriteScheduledInstr(MachineBasicBlock *BB, InstrMapTy &InstrMap,
                              unsigned CurStageNum, unsigned PhiNum,
-                             MachineInstr *Phi, unsigned OldReg,
-                             unsigned NewReg, unsigned PrevReg = 0);
+                             MachineInstr *Phi, Register OldReg,
+                             Register NewReg, Register PrevReg = Register());
   bool isLoopCarried(MachineInstr &Phi);
 
   /// Return the max. number of stages/iterations that can occur between a
   /// register definition and its uses.
-  unsigned getStagesForReg(int Reg, unsigned CurStage) {
+  unsigned getStagesForReg(Register Reg, unsigned CurStage) {
     std::pair<unsigned, bool> Stages = RegToStageDiff[Reg];
     if ((int)CurStage > Schedule.getNumStages() - 1 && Stages.first == 0 &&
         Stages.second)
@@ -249,7 +249,7 @@ class ModuloScheduleExpander {
   /// This is not the case if the loop value is scheduled prior to the
   /// Phi in the same stage.  This function returns the number of stages
   /// or iterations needed between the Phi definition and any uses.
-  unsigned getStagesForPhi(int Reg) {
+  unsigned getStagesForPhi(Register Reg) {
     std::pair<unsigned, bool> Stages = RegToStageDiff[Reg];
     if (Stages.second)
       return Stages.first;
@@ -374,7 +374,7 @@ class PeelingModuloScheduleExpander {
 /// It unrolls the kernel enough to avoid overlap of register lifetime.
 class ModuloScheduleExpanderMVE {
 private:
-  using ValueMapTy = DenseMap<unsigned, unsigned>;
+  using ValueMapTy = DenseMap<Register, Register>;
   using MBBVectorTy = SmallVectorImpl<MachineBasicBlock *>;
   using InstrMapTy = DenseMap<MachineInstr *, MachineInstr *>;
 
diff --git a/llvm/include/llvm/CodeGen/PatchableFunction.h b/llvm/include/llvm/CodeGen/PatchableFunction.h
new file mode 100644
index 0000000000000..d81a92f9eef26
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/PatchableFunction.h
@@ -0,0 +1,30 @@
+//===- llvm/CodeGen/PatchableFunction.h -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_PATCHABLEFUNCTION_H
+#define LLVM_CODEGEN_PATCHABLEFUNCTION_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+
+class PatchableFunctionPass : public PassInfoMixin<PatchableFunctionPass> {
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+
+  MachineFunctionProperties getRequiredProperties() const {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::NoVRegs);
+  }
+  static bool isRequired() { return true; }
+};
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_PATCHABLEFUNCTION_H
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index 1c1c94e7193ae..2283f99202e2f 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -421,6 +421,8 @@ struct SDNodeFlags {
 
     PoisonGeneratingFlags = NoUnsignedWrap | NoSignedWrap | Exact | Disjoint |
                             NonNeg | NoNaNs | NoInfs | SameSign,
+    FastMathFlags = NoNaNs | NoInfs | NoSignedZeros | AllowReciprocal |
+                    AllowContract | ApproximateFuncs | AllowReassociation,
   };
 
   /// Default constructor turns off all optimization flags.
diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index 3206cc4518821..502c0f6489773 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -593,7 +593,7 @@ class TargetRegisterInfo : public MCRegisterInfo {
 
   /// Returns true if PhysReg cannot be written to in inline asm statements.
   virtual bool isInlineAsmReadOnlyReg(const MachineFunction &MF,
-                                      unsigned PhysReg) const {
+                                      MCRegister PhysReg) const {
     return false;
   }
 
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h b/llvm/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h
index 7fb61b6a021af..6bfa5506017df 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h
@@ -83,35 +83,9 @@ class EHFrameCFIBlockInspector {
   };
 };
 
-/// Supports registration/deregistration of EH-frames in a target process.
-class EHFrameRegistrar {
-public:
-  virtual ~EHFrameRegistrar();
-  virtual Error registerEHFrames(orc::ExecutorAddrRange EHFrameSection) = 0;
-  virtual Error deregisterEHFrames(orc::ExecutorAddrRange EHFrameSection) = 0;
-};
-
-/// Registers / Deregisters EH-frames in the current process.
-class InProcessEHFrameRegistrar final : public EHFrameRegistrar {
-public:
-  Error registerEHFrames(orc::ExecutorAddrRange EHFrameSection) override;
-
-  Error deregisterEHFrames(orc::ExecutorAddrRange EHFrameSection) override;
-};
-
-using StoreFrameRangeFunction = std::function<void(
-    orc::ExecutorAddr EHFrameSectionAddr, size_t EHFrameSectionSize)>;
-
-/// Creates a pass that records the address and size of the EH frame section.
-/// If no eh-frame section is found then the address and size will both be given
-/// as zero.
-///
-/// Authors of JITLinkContexts can use this function to register a post-fixup
-/// pass that records the range of the eh-frame section. This range can
-/// be used after finalization to register and deregister the frame.
-LinkGraphPassFunction
-createEHFrameRecorderPass(const Triple &TT,
-                          StoreFrameRangeFunction StoreFrameRange);
+/// Returns a pointer to the DWARF eh-frame section if the graph contains a
+/// non-empty one, otherwise returns null.
+Section *getEHFrameSection(LinkGraph &G);
 
 } // end namespace jitlink
 } // end namespace llvm
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
index fbbd0d3d74b15..b333ef8d9ecd5 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
@@ -1653,11 +1653,11 @@ class ExecutionSession {
   /// (using registerJITDispatchHandler) and called from the executor.
   template <typename SPSSignature, typename HandlerT>
   static JITDispatchHandlerFunction wrapAsyncWithSPS(HandlerT &&H) {
-    return [H = std::forward<HandlerT>(H)](
-               SendResultFunction SendResult,
-               const char *ArgData, size_t ArgSize) mutable {
-      shared::WrapperFunction<SPSSignature>::handleAsync(ArgData, ArgSize, H,
-                                                         std::move(SendResult));
+    return [H = std::forward<HandlerT>(H)](SendResultFunction SendResult,
+                                           const char *ArgData,
+                                           size_t ArgSize) mutable {
+      shared::WrapperFunction<SPSSignature>::handleAsync(
+          ArgData, ArgSize, std::move(SendResult), H);
     };
   }
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EHFrameRegistrationPlugin.h b/llvm/include/llvm/ExecutionEngine/Orc/EHFrameRegistrationPlugin.h
index 6cbbc0c94a37f..6f25b998fb005 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/EHFrameRegistrationPlugin.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/EHFrameRegistrationPlugin.h
@@ -19,37 +19,37 @@
 #include <mutex>
 #include <vector>
 
-namespace llvm {
-
-namespace jitlink {
-class EHFrameRegistrar;
-} // namespace jitlink
-
-namespace orc {
+namespace llvm::orc {
 
+/// Adds AllocationActions to register and deregister eh-frame sections in the
+/// absence of native Platform support.
 class EHFrameRegistrationPlugin : public LinkGraphLinkingLayer::Plugin {
 public:
-  EHFrameRegistrationPlugin(
-      ExecutionSession &ES,
-      std::unique_ptr<jitlink::EHFrameRegistrar> Registrar);
+  static Expected<std::unique_ptr<EHFrameRegistrationPlugin>>
+  Create(ExecutionSession &ES);
+
+  EHFrameRegistrationPlugin(ExecutorAddr RegisterEHFrame,
+                            ExecutorAddr DeregisterEHFrame)
+      : RegisterEHFrame(RegisterEHFrame), DeregisterEHFrame(DeregisterEHFrame) {
+  }
+
   void modifyPassConfig(MaterializationResponsibility &MR,
                         jitlink::LinkGraph &G,
                         jitlink::PassConfiguration &PassConfig) override;
-  Error notifyEmitted(MaterializationResponsibility &MR) override;
-  Error notifyFailed(MaterializationResponsibility &MR) override;
-  Error notifyRemovingResources(JITDylib &JD, ResourceKey K) override;
+  Error notifyFailed(MaterializationResponsibility &MR) override {
+    return Error::success();
+  }
+  Error notifyRemovingResources(JITDylib &JD, ResourceKey K) override {
+    return Error::success();
+  }
   void notifyTransferringResources(JITDylib &JD, ResourceKey DstKey,
-                                   ResourceKey SrcKey) override;
+                                   ResourceKey SrcKey) override {}
 
 private:
-  std::mutex EHFramePluginMutex;
-  ExecutionSession &ES;
-  std::unique_ptr<jitlink::EHFrameRegistrar> Registrar;
-  DenseMap<MaterializationResponsibility *, ExecutorAddrRange> InProcessLinks;
-  DenseMap<ResourceKey, std::vector<ExecutorAddrRange>> EHFrameRanges;
+  ExecutorAddr RegisterEHFrame;
+  ExecutorAddr DeregisterEHFrame;
 };
 
-} // end namespace orc
-} // end namespace llvm
+} // namespace llvm::orc
 
 #endif // LLVM_EXECUTIONENGINE_ORC_EHFRAMEREGISTRATIONPLUGIN_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h
deleted file mode 100644
index 182e9ed1041a6..0000000000000
--- a/llvm/include/llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h
+++ /dev/null
@@ -1,58 +0,0 @@
-//===-- EPCEHFrameRegistrar.h - EPC based eh-frame registration -*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// ExecutorProcessControl based eh-frame registration.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_EXECUTIONENGINE_ORC_EPCEHFRAMEREGISTRAR_H
-#define LLVM_EXECUTIONENGINE_ORC_EPCEHFRAMEREGISTRAR_H
-
-#include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
-#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
-
-namespace llvm {
-namespace orc {
-
-class ExecutionSession;
-
-/// Register/Deregisters EH frames in a remote process via a
-/// ExecutorProcessControl instance.
-class EPCEHFrameRegistrar : public jitlink::EHFrameRegistrar {
-public:
-  /// Create from a ExecutorProcessControl instance alone. This will use
-  /// the EPC's lookupSymbols method to find the registration/deregistration
-  /// function addresses by name.
-  ///
-  /// If RegistrationFunctionsDylib is non-None then it will be searched to
-  /// find the registration functions. If it is None then the process dylib
-  /// will be loaded to find the registration functions.
-  static Expected<std::unique_ptr<EPCEHFrameRegistrar>>
-  Create(ExecutionSession &ES);
-
-  /// Create a EPCEHFrameRegistrar with the given ExecutorProcessControl
-  /// object and registration/deregistration function addresses.
-  EPCEHFrameRegistrar(ExecutionSession &ES,
-                      ExecutorAddr RegisterEHFrameSectionWrapper,
-                      ExecutorAddr DeregisterEHFRameSectionWrapper)
-      : ES(ES), RegisterEHFrameSectionWrapper(RegisterEHFrameSectionWrapper),
-        DeregisterEHFrameSectionWrapper(DeregisterEHFRameSectionWrapper) {}
-
-  Error registerEHFrames(ExecutorAddrRange EHFrameSection) override;
-  Error deregisterEHFrames(ExecutorAddrRange EHFrameSection) override;
-
-private:
-  ExecutionSession &ES;
-  ExecutorAddr RegisterEHFrameSectionWrapper;
-  ExecutorAddr DeregisterEHFrameSectionWrapper;
-};
-
-} // end namespace orc
-} // end namespace llvm
-
-#endif // LLVM_EXECUTIONENGINE_ORC_EPCEHFRAMEREGISTRAR_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h b/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h
index 9f9ff06b2c2f6..8815777f35e07 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/MemoryMapper.h
@@ -157,6 +157,7 @@ class SharedMemoryMapper final : public MemoryMapper {
   struct Reservation {
     void *LocalAddr;
     size_t Size;
+    int SharedMemoryId;
   };
 
   ExecutorProcessControl &EPC;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
index 0f59edd429332..927939e63bbfd 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
@@ -44,8 +44,8 @@ extern const char *MemoryWriteUInt64sWrapperName;
 extern const char *MemoryWriteBuffersWrapperName;
 extern const char *MemoryWritePointersWrapperName;
 
-extern const char *RegisterEHFrameSectionWrapperName;
-extern const char *DeregisterEHFrameSectionWrapperName;
+extern const char *RegisterEHFrameSectionAllocActionName;
+extern const char *DeregisterEHFrameSectionAllocActionName;
 
 extern const char *RunAsMainWrapperName;
 extern const char *RunAsVoidFunctionWrapperName;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h
index 2521bda0b42cf..7ac8bd3489683 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h
@@ -527,7 +527,7 @@ class WrapperFunction<SPSRetTagT(SPSTagTs...)> {
   /// Handle a call to an async wrapper function.
   template <typename HandlerT, typename SendResultT>
   static void handleAsync(const char *ArgData, size_t ArgSize,
-                          HandlerT &&Handler, SendResultT &&SendResult) {
+                          SendResultT &&SendResult, HandlerT &&Handler) {
     using WFAHH = detail::WrapperFunctionAsyncHandlerHelper<
         std::remove_reference_t<HandlerT>, ResultSerializer, SPSTagTs...>;
     WFAHH::applyAsync(std::forward<HandlerT>(Handler),
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h
index d375f6034616a..0c56fa885cf4d 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h
@@ -35,9 +35,9 @@ Error deregisterEHFrameSection(const void *EHFrameSectionAddr,
 } // end namespace llvm
 
 extern "C" LLVM_ABI llvm::orc::shared::CWrapperFunctionResult
-llvm_orc_registerEHFrameSectionWrapper(const char *Data, uint64_t Size);
+llvm_orc_registerEHFrameSectionAllocAction(const char *Data, uint64_t Size);
 
 extern "C" LLVM_ABI llvm::orc::shared::CWrapperFunctionResult
-llvm_orc_deregisterEHFrameSectionWrapper(const char *Data, uint64_t Size);
+llvm_orc_deregisterEHFrameSectionAllocAction(const char *Data, uint64_t Size);
 
 #endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_REGISTEREHFRAMES_H
diff --git a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
index 5dc1c4cb54c7c..e0714e812e5cd 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
@@ -1142,6 +1142,11 @@ struct UnifiedSharedMemoryT {
   using EmptyTrait = std::true_type;
 };
 
+template <typename T, typename I, typename E> //
+struct SelfMapsT {
+  using EmptyTrait = std::true_type;
+};
+
 // V5.2: [5.10] `uniform` clause
 template <typename T, typename I, typename E> //
 struct UniformT {
@@ -1245,7 +1250,7 @@ using EmptyClausesT = std::variant<
     ReverseOffloadT<T, I, E>, SeqCstT<T, I, E>, SimdT<T, I, E>,
     ThreadsT<T, I, E>, UnifiedAddressT<T, I, E>, UnifiedSharedMemoryT<T, I, E>,
     UnknownT<T, I, E>, UntiedT<T, I, E>, UseT<T, I, E>, WeakT<T, I, E>,
-    WriteT<T, I, E>, NoOpenmpConstructsT<T, I, E>>;
+    WriteT<T, I, E>, NoOpenmpConstructsT<T, I, E>, SelfMapsT<T, I, E>>;
 
 template <typename T, typename I, typename E>
 using IncompleteClausesT =
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 8a2f30a7995dc..75347061d9a92 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -105,6 +105,7 @@ def OMPC_CancellationConstructType : Clause<"cancellation_construct_type"> {
     OMP_CANCELLATION_CONSTRUCT_Taskgroup,
     OMP_CANCELLATION_CONSTRUCT_None
   ];
+  let flangClass = "OmpCancellationConstructTypeClause";
 }
 def OMPC_Contains : Clause<"contains"> {
   let clangClass = "OMPContainsClause";
@@ -505,6 +506,9 @@ def OMPC_UnifiedAddress : Clause<"unified_address"> {
 def OMPC_UnifiedSharedMemory : Clause<"unified_shared_memory"> {
   let clangClass = "OMPUnifiedSharedMemoryClause";
 }
+def OMPC_SelfMaps : Clause<"self_maps"> {
+  let clangClass = "OMPSelfMapsClause";
+}
 def OMPC_Uniform : Clause<"uniform"> {
   let flangClass = "Name";
   let isValueList = true;
@@ -647,12 +651,16 @@ def OMP_BeginDeclareVariant : Directive<"begin declare variant"> {
 }
 def OMP_Cancel : Directive<"cancel"> {
   let allowedOnceClauses = [
+    VersionedClause<OMPC_CancellationConstructType>,
     VersionedClause<OMPC_If>,
   ];
   let association = AS_None;
   let category = CA_Executable;
 }
 def OMP_CancellationPoint : Directive<"cancellation point"> {
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_CancellationConstructType>,
+  ];
   let association = AS_None;
   let category = CA_Executable;
 }
@@ -935,7 +943,7 @@ def OMP_Requires : Directive<"requires"> {
     // TODO: Correct this supprted version number whenever complete
     // implementation of reverse_offload is available.
     VersionedClause<OMPC_AtomicDefaultMemOrder>,
-    VersionedClause<OMPC_DynamicAllocators>,
+    VersionedClause<OMPC_DynamicAllocators>, VersionedClause<OMPC_SelfMaps>,
     VersionedClause<OMPC_ReverseOffload, 99>,
   ];
   let association = AS_None;
@@ -1023,9 +1031,11 @@ def OMP_Single : Directive<"single"> {
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_CopyPrivate>,
     VersionedClause<OMPC_FirstPrivate>,
-    VersionedClause<OMPC_NoWait>,
     VersionedClause<OMPC_Private>,
   ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_NoWait>,
+  ];
   let association = AS_Block;
   let category = CA_Executable;
 }
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index b624f9005bd72..ea58985cbebda 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -5269,4 +5269,18 @@ foreach shape = ["16x64b", "16x128b", "16x256b", "32x32b", "16x32bx2"] in {
   }
 }
 
+//
+// Bulk store intrinsics
+//
+
+def int_nvvm_st_bulk: DefaultAttrsIntrinsic<[],
+  [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],
+  [IntrArgMemOnly, IntrWriteMem,
+    WriteOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>, ImmArg<ArgIndex<2>>]>;
+
+def int_nvvm_st_bulk_shared_cta : DefaultAttrsIntrinsic<[],
+  [llvm_shared_ptr_ty, llvm_i64_ty, llvm_i64_ty],
+  [IntrArgMemOnly, IntrWriteMem,
+    WriteOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>, ImmArg<ArgIndex<2>>]>;
+
 } // let TargetPrefix = "nvvm"
diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h
index 2fd2d6887022c..91ccd76c41e07 100644
--- a/llvm/include/llvm/IR/Module.h
+++ b/llvm/include/llvm/IR/Module.h
@@ -1067,6 +1067,9 @@ class LLVM_ABI Module {
 
   /// Set the target variant version build SDK version metadata.
   void setDarwinTargetVariantSDKVersion(VersionTuple Version);
+
+  /// Returns target-abi from MDString, null if target-abi is absent.
+  StringRef getTargetABIFromMD();
 };
 
 /// Given "llvm.used" or "llvm.compiler.used" as a global name, collect the
diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h
index 3c586a1dd21d8..2650456c49841 100644
--- a/llvm/include/llvm/IR/ModuleSummaryIndex.h
+++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h
@@ -23,6 +23,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/iterator_range.h"
 #include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/Module.h"
@@ -1289,6 +1290,72 @@ struct TypeIdSummary {
   std::map<uint64_t, WholeProgramDevirtResolution> WPDRes;
 };
 
+class CfiFunctionIndex {
+  DenseMap<GlobalValue::GUID, std::set<std::string, std::less<>>> Index;
+  using IndexIterator =
+      DenseMap<GlobalValue::GUID,
+               std::set<std::string, std::less<>>>::const_iterator;
+  using NestedIterator = std::set<std::string, std::less<>>::const_iterator;
+
+public:
+  // Iterates keys of the DenseMap.
+  class GUIDIterator : public iterator_adaptor_base<GUIDIterator, IndexIterator,
+                                                    std::forward_iterator_tag,
+                                                    GlobalValue::GUID> {
+    using base = GUIDIterator::iterator_adaptor_base;
+
+  public:
+    GUIDIterator() = default;
+    explicit GUIDIterator(IndexIterator I) : base(I) {}
+
+    GlobalValue::GUID operator*() const { return this->wrapped()->first; }
+  };
+
+  CfiFunctionIndex() = default;
+  template <typename It> CfiFunctionIndex(It B, It E) {
+    for (; B != E; ++B)
+      emplace(*B);
+  }
+
+  std::vector<StringRef> symbols() const {
+    std::vector<StringRef> Symbols;
+    for (auto &[GUID, Syms] : Index)
+      Symbols.insert(Symbols.end(), Syms.begin(), Syms.end());
+    return Symbols;
+  }
+
+  GUIDIterator guid_begin() const { return GUIDIterator(Index.begin()); }
+  GUIDIterator guid_end() const { return GUIDIterator(Index.end()); }
+  iterator_range<GUIDIterator> guids() const {
+    return make_range(guid_begin(), guid_end());
+  }
+
+  iterator_range<NestedIterator> forGuid(GlobalValue::GUID GUID) const {
+    auto I = Index.find(GUID);
+    if (I == Index.end())
+      return make_range(NestedIterator{}, NestedIterator{});
+    return make_range(I->second.begin(), I->second.end());
+  }
+
+  template <typename... Args> void emplace(Args &&...A) {
+    StringRef S(std::forward<Args>(A)...);
+    GlobalValue::GUID GUID =
+        GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(S));
+    Index[GUID].emplace(S);
+  }
+
+  size_t count(StringRef S) const {
+    GlobalValue::GUID GUID =
+        GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(S));
+    auto I = Index.find(GUID);
+    if (I == Index.end())
+      return 0;
+    return I->second.count(S);
+  }
+
+  bool empty() const { return Index.empty(); }
+};
+
 /// 160 bits SHA1
 using ModuleHash = std::array<uint32_t, 5>;
 
@@ -1418,8 +1485,8 @@ class ModuleSummaryIndex {
   /// True if some of the FunctionSummary contains a ParamAccess.
   bool HasParamAccess = false;
 
-  std::set<std::string, std::less<>> CfiFunctionDefs;
-  std::set<std::string, std::less<>> CfiFunctionDecls;
+  CfiFunctionIndex CfiFunctionDefs;
+  CfiFunctionIndex CfiFunctionDecls;
 
   // Used in cases where we want to record the name of a global, but
   // don't have the string owned elsewhere (e.g. the Strtab on a module).
@@ -1667,19 +1734,11 @@ class ModuleSummaryIndex {
     return I == OidGuidMap.end() ? 0 : I->second;
   }
 
-  std::set<std::string, std::less<>> &cfiFunctionDefs() {
-    return CfiFunctionDefs;
-  }
-  const std::set<std::string, std::less<>> &cfiFunctionDefs() const {
-    return CfiFunctionDefs;
-  }
+  CfiFunctionIndex &cfiFunctionDefs() { return CfiFunctionDefs; }
+  const CfiFunctionIndex &cfiFunctionDefs() const { return CfiFunctionDefs; }
 
-  std::set<std::string, std::less<>> &cfiFunctionDecls() {
-    return CfiFunctionDecls;
-  }
-  const std::set<std::string, std::less<>> &cfiFunctionDecls() const {
-    return CfiFunctionDecls;
-  }
+  CfiFunctionIndex &cfiFunctionDecls() { return CfiFunctionDecls; }
+  const CfiFunctionIndex &cfiFunctionDecls() const { return CfiFunctionDecls; }
 
   /// Add a global value summary for a value.
   void addGlobalValueSummary(const GlobalValue &GV,
diff --git a/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h b/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h
index 14b8ce5df8536..d5a91763a981c 100644
--- a/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h
+++ b/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h
@@ -9,8 +9,10 @@
 #ifndef LLVM_IR_MODULESUMMARYINDEXYAML_H
 #define LLVM_IR_MODULESUMMARYINDEXYAML_H
 
+#include "llvm/ADT/StringRef.h"
 #include "llvm/IR/ModuleSummaryIndex.h"
 #include "llvm/Support/YAMLTraits.h"
+#include <algorithm>
 
 namespace llvm {
 namespace yaml {
@@ -345,11 +347,11 @@ template <> struct MappingTraits<ModuleSummaryIndex> {
                    index.WithGlobalValueDeadStripping);
 
     if (io.outputting()) {
-      std::vector<std::string> CfiFunctionDefs(index.CfiFunctionDefs.begin(),
-                                               index.CfiFunctionDefs.end());
+      auto CfiFunctionDefs = index.CfiFunctionDefs.symbols();
+      llvm::sort(CfiFunctionDefs);
       io.mapOptional("CfiFunctionDefs", CfiFunctionDefs);
-      std::vector<std::string> CfiFunctionDecls(index.CfiFunctionDecls.begin(),
-                                                index.CfiFunctionDecls.end());
+      auto CfiFunctionDecls(index.CfiFunctionDecls.symbols());
+      llvm::sort(CfiFunctionDecls);
       io.mapOptional("CfiFunctionDecls", CfiFunctionDecls);
     } else {
       std::vector<std::string> CfiFunctionDefs;
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.def b/llvm/include/llvm/IR/RuntimeLibcalls.def
index c6ac341d71a20..2545aebc73391 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.def
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.def
@@ -444,6 +444,7 @@ HANDLE_LIBCALL(SINTTOFP_I32_F64, "__floatsidf")
 HANDLE_LIBCALL(SINTTOFP_I32_F80, "__floatsixf")
 HANDLE_LIBCALL(SINTTOFP_I32_F128, "__floatsitf")
 HANDLE_LIBCALL(SINTTOFP_I32_PPCF128, "__gcc_itoq")
+HANDLE_LIBCALL(SINTTOFP_I64_BF16, "__floatdibf")
 HANDLE_LIBCALL(SINTTOFP_I64_F16, "__floatdihf")
 HANDLE_LIBCALL(SINTTOFP_I64_F32, "__floatdisf")
 HANDLE_LIBCALL(SINTTOFP_I64_F64, "__floatdidf")
@@ -462,6 +463,7 @@ HANDLE_LIBCALL(UINTTOFP_I32_F64, "__floatunsidf")
 HANDLE_LIBCALL(UINTTOFP_I32_F80, "__floatunsixf")
 HANDLE_LIBCALL(UINTTOFP_I32_F128, "__floatunsitf")
 HANDLE_LIBCALL(UINTTOFP_I32_PPCF128, "__gcc_utoq")
+HANDLE_LIBCALL(UINTTOFP_I64_BF16, "__floatundibf")
 HANDLE_LIBCALL(UINTTOFP_I64_F16, "__floatundihf")
 HANDLE_LIBCALL(UINTTOFP_I64_F32, "__floatundisf")
 HANDLE_LIBCALL(UINTTOFP_I64_F64, "__floatundidf")
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index a05e876806ab5..36be3d552f556 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -89,7 +89,7 @@ void initializeDXILResourceTypeWrapperPassPass(PassRegistry &);
 void initializeDeadMachineInstructionElimPass(PassRegistry &);
 void initializeDebugifyMachineModulePass(PassRegistry &);
 void initializeDependenceAnalysisWrapperPassPass(PassRegistry &);
-void initializeDetectDeadLanesPass(PassRegistry &);
+void initializeDetectDeadLanesLegacyPass(PassRegistry &);
 void initializeDomOnlyPrinterWrapperPassPass(PassRegistry &);
 void initializeDomOnlyViewerWrapperPassPass(PassRegistry &);
 void initializeDomPrinterWrapperPassPass(PassRegistry &);
@@ -225,7 +225,7 @@ void initializeOptimizePHIsLegacyPass(PassRegistry &);
 void initializePEIPass(PassRegistry &);
 void initializePHIEliminationPass(PassRegistry &);
 void initializePartiallyInlineLibCallsLegacyPassPass(PassRegistry &);
-void initializePatchableFunctionPass(PassRegistry &);
+void initializePatchableFunctionLegacyPass(PassRegistry &);
 void initializePeepholeOptimizerLegacyPass(PassRegistry &);
 void initializePhiValuesWrapperPassPass(PassRegistry &);
 void initializePhysicalRegisterUsageInfoWrapperLegacyPass(PassRegistry &);
diff --git a/llvm/include/llvm/MC/MCAsmInfo.h b/llvm/include/llvm/MC/MCAsmInfo.h
index aba8eaed82d1a..7f03097eba060 100644
--- a/llvm/include/llvm/MC/MCAsmInfo.h
+++ b/llvm/include/llvm/MC/MCAsmInfo.h
@@ -709,7 +709,7 @@ class MCAsmInfo {
   bool shouldUseMotorolaIntegers() const { return UseMotorolaIntegers; }
 
   StringRef getVariantKindName(uint32_t Kind) const;
-  uint32_t getVariantKindForName(StringRef Name) const;
+  std::optional<uint32_t> getVariantKindForName(StringRef Name) const;
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCExpr.h b/llvm/include/llvm/MC/MCExpr.h
index ab96b39d9f296..a302a81ac468a 100644
--- a/llvm/include/llvm/MC/MCExpr.h
+++ b/llvm/include/llvm/MC/MCExpr.h
@@ -198,7 +198,6 @@ class MCSymbolRefExpr : public MCExpr {
   // cleaner approach.
   enum VariantKind : uint16_t {
     VK_None,
-    VK_Invalid,
 
     VK_GOT,
     VK_GOTENT,
diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h
index d6a957322ea11..ee747253fd869 100644
--- a/llvm/include/llvm/MC/MCObjectStreamer.h
+++ b/llvm/include/llvm/MC/MCObjectStreamer.h
@@ -141,8 +141,8 @@ class MCObjectStreamer : public MCStreamer {
                          SMLoc Loc) override;
   void emitDwarfLocDirective(unsigned FileNo, unsigned Line, unsigned Column,
                              unsigned Flags, unsigned Isa,
-                             unsigned Discriminator,
-                             StringRef FileName) override;
+                             unsigned Discriminator, StringRef FileName,
+                             StringRef Comment = {}) override;
   void emitDwarfAdvanceLineAddr(int64_t LineDelta, const MCSymbol *LastLabel,
                                 const MCSymbol *Label,
                                 unsigned PointerSize) override;
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index 1dc6ddfd7566b..9d63c1e66bdae 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -887,7 +887,8 @@ class MCStreamer {
   virtual void emitDwarfLocDirective(unsigned FileNo, unsigned Line,
                                      unsigned Column, unsigned Flags,
                                      unsigned Isa, unsigned Discriminator,
-                                     StringRef FileName);
+                                     StringRef FileName,
+                                     StringRef Comment = {});
 
   /// This implements the '.loc_label Name' directive.
   virtual void emitDwarfLocLabelDirective(SMLoc Loc, StringRef Name);
diff --git a/llvm/include/llvm/Object/ELFObjectFile.h b/llvm/include/llvm/Object/ELFObjectFile.h
index 60c72062a3f6a..bafc92cafe539 100644
--- a/llvm/include/llvm/Object/ELFObjectFile.h
+++ b/llvm/include/llvm/Object/ELFObjectFile.h
@@ -410,6 +410,9 @@ template <class ELFT> class ELFObjectFile : public ELFObjectFileBase {
     case ELF::EM_ARM:
       Type = ELF::SHT_ARM_ATTRIBUTES;
       break;
+    case ELF::EM_AARCH64:
+      Type = ELF::SHT_AARCH64_ATTRIBUTES;
+      break;
     case ELF::EM_RISCV:
       Type = ELF::SHT_RISCV_ATTRIBUTES;
       break;
diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
index 25899d04dc664..90a0cdf803560 100644
--- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
@@ -26,6 +26,7 @@
 #include "llvm/CodeGen/CallBrPrepare.h"
 #include "llvm/CodeGen/CodeGenPrepare.h"
 #include "llvm/CodeGen/DeadMachineInstructionElim.h"
+#include "llvm/CodeGen/DetectDeadLanes.h"
 #include "llvm/CodeGen/DwarfEHPrepare.h"
 #include "llvm/CodeGen/EarlyIfConversion.h"
 #include "llvm/CodeGen/ExpandLargeDivRem.h"
@@ -58,6 +59,7 @@
 #include "llvm/CodeGen/MachineVerifier.h"
 #include "llvm/CodeGen/OptimizePHIs.h"
 #include "llvm/CodeGen/PHIElimination.h"
+#include "llvm/CodeGen/PatchableFunction.h"
 #include "llvm/CodeGen/PeepholeOptimizer.h"
 #include "llvm/CodeGen/PostRASchedulerList.h"
 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index f99a5f2c74bf3..ebfdaf82169d8 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -138,6 +138,7 @@ MACHINE_FUNCTION_ANALYSIS("virtregmap", VirtRegMapAnalysis())
 #define MACHINE_FUNCTION_PASS(NAME, CREATE_PASS)
 #endif
 MACHINE_FUNCTION_PASS("dead-mi-elimination", DeadMachineInstructionElimPass())
+MACHINE_FUNCTION_PASS("detect-dead-lanes", DetectDeadLanesPass())
 MACHINE_FUNCTION_PASS("early-ifcvt", EarlyIfConverterPass())
 MACHINE_FUNCTION_PASS("early-machinelicm", EarlyMachineLICMPass())
 MACHINE_FUNCTION_PASS("early-tailduplication", EarlyTailDuplicatePass())
@@ -151,6 +152,7 @@ MACHINE_FUNCTION_PASS("machine-scheduler", MachineSchedulerPass(TM))
 MACHINE_FUNCTION_PASS("machinelicm", MachineLICMPass())
 MACHINE_FUNCTION_PASS("no-op-machine-function", NoOpMachineFunctionPass())
 MACHINE_FUNCTION_PASS("opt-phis", OptimizePHIsPass())
+MACHINE_FUNCTION_PASS("patchable-function", PatchableFunctionPass())
 MACHINE_FUNCTION_PASS("peephole-opt", PeepholeOptimizerPass())
 MACHINE_FUNCTION_PASS("phi-node-elimination", PHIEliminationPass())
 MACHINE_FUNCTION_PASS("post-RA-sched", PostRASchedulerPass(TM))
@@ -249,7 +251,6 @@ DUMMY_MACHINE_FUNCTION_PASS("break-false-deps", BreakFalseDepsPass)
 DUMMY_MACHINE_FUNCTION_PASS("cfguard-longjmp", CFGuardLongjmpPass)
 DUMMY_MACHINE_FUNCTION_PASS("cfi-fixup", CFIFixupPass)
 DUMMY_MACHINE_FUNCTION_PASS("cfi-instr-inserter", CFIInstrInserterPass)
-DUMMY_MACHINE_FUNCTION_PASS("detect-dead-lanes", DetectDeadLanesPass)
 DUMMY_MACHINE_FUNCTION_PASS("dot-machine-cfg", MachineCFGPrinter)
 DUMMY_MACHINE_FUNCTION_PASS("fentry-insert", FEntryInserterPass)
 DUMMY_MACHINE_FUNCTION_PASS("fs-profile-loader", MIRProfileLoaderNewPass)
@@ -270,7 +271,6 @@ DUMMY_MACHINE_FUNCTION_PASS("machine-sanmd", MachineSanitizerBinaryMetadata)
 DUMMY_MACHINE_FUNCTION_PASS("machine-uniformity", MachineUniformityInfoWrapperPass)
 DUMMY_MACHINE_FUNCTION_PASS("machineinstr-printer", MachineFunctionPrinterPass)
 DUMMY_MACHINE_FUNCTION_PASS("mirfs-discriminators", MIRAddFSDiscriminatorsPass)
-DUMMY_MACHINE_FUNCTION_PASS("patchable-function", PatchableFunctionPass)
 DUMMY_MACHINE_FUNCTION_PASS("postra-machine-sink", PostRAMachineSinkingPass)
 DUMMY_MACHINE_FUNCTION_PASS("print-machine-uniformity", MachineUniformityInfoPrinterPass)
 DUMMY_MACHINE_FUNCTION_PASS("processimpdefs", ProcessImplicitDefsPass)
diff --git a/llvm/include/llvm/ProfileData/CtxInstrContextNode.h b/llvm/include/llvm/ProfileData/CtxInstrContextNode.h
index fe8ddcdf79129..0fc4883305145 100644
--- a/llvm/include/llvm/ProfileData/CtxInstrContextNode.h
+++ b/llvm/include/llvm/ProfileData/CtxInstrContextNode.h
@@ -122,6 +122,12 @@ class ProfileWriter {
   virtual void startContextSection() = 0;
   virtual void writeContextual(const ctx_profile::ContextNode &RootNode) = 0;
   virtual void endContextSection() = 0;
+
+  virtual void startFlatSection() = 0;
+  virtual void writeFlat(ctx_profile::GUID Guid, const uint64_t *Buffer,
+                         size_t BufferSize) = 0;
+  virtual void endFlatSection() = 0;
+
   virtual ~ProfileWriter() = default;
 };
 } // namespace ctx_profile
diff --git a/llvm/include/llvm/ProfileData/PGOCtxProfReader.h b/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
index 33f03120a835a..4b0c944a5258c 100644
--- a/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
+++ b/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
@@ -174,6 +174,7 @@ using CtxProfContextualProfiles =
     std::map<GlobalValue::GUID, PGOCtxProfContext>;
 struct PGOCtxProfile {
   CtxProfContextualProfiles Contexts;
+  CtxProfFlatProfile FlatProfiles;
 
   PGOCtxProfile() = default;
   PGOCtxProfile(const PGOCtxProfile &) = delete;
@@ -192,10 +193,12 @@ class PGOCtxProfileReader final {
   Expected<std::pair<std::optional<uint32_t>, PGOCtxProfContext>>
   readProfile(PGOCtxProfileBlockIDs Kind);
 
+  bool tryGetNextKnownBlockID(PGOCtxProfileBlockIDs &ID);
   bool canEnterBlockWithID(PGOCtxProfileBlockIDs ID);
   Error enterBlockWithID(PGOCtxProfileBlockIDs ID);
 
   Error loadContexts(CtxProfContextualProfiles &P);
+  Error loadFlatProfiles(CtxProfFlatProfile &P);
 
 public:
   PGOCtxProfileReader(StringRef Buffer)
diff --git a/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h b/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
index 0ff11c998f02a..c5a724d9a2142 100644
--- a/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
+++ b/llvm/include/llvm/ProfileData/PGOCtxProfWriter.h
@@ -22,10 +22,14 @@ namespace llvm {
 enum PGOCtxProfileRecords { Invalid = 0, Version, Guid, CalleeIndex, Counters };
 
 enum PGOCtxProfileBlockIDs {
-  ProfileMetadataBlockID = bitc::FIRST_APPLICATION_BLOCKID,
+  FIRST_VALID = bitc::FIRST_APPLICATION_BLOCKID,
+  ProfileMetadataBlockID = FIRST_VALID,
   ContextsSectionBlockID = ProfileMetadataBlockID + 1,
   ContextRootBlockID = ContextsSectionBlockID + 1,
   ContextNodeBlockID = ContextRootBlockID + 1,
+  FlatProfilesSectionBlockID = ContextNodeBlockID + 1,
+  FlatProfileBlockID = FlatProfilesSectionBlockID + 1,
+  LAST_VALID = FlatProfileBlockID
 };
 
 /// Write one or more ContextNodes to the provided raw_fd_stream.
@@ -83,6 +87,11 @@ class PGOCtxProfileWriter final : public ctx_profile::ProfileWriter {
   void writeContextual(const ctx_profile::ContextNode &RootNode) override;
   void endContextSection() override;
 
+  void startFlatSection() override;
+  void writeFlat(ctx_profile::GUID Guid, const uint64_t *Buffer,
+                 size_t BufferSize) override;
+  void endFlatSection() override;
+
   // constants used in writing which a reader may find useful.
   static constexpr unsigned CodeLen = 2;
   static constexpr uint32_t CurrentVersion = 2;
diff --git a/llvm/include/llvm/SandboxIR/Region.h b/llvm/include/llvm/SandboxIR/Region.h
index 1a051941e5598..14f35c9c4d8a9 100644
--- a/llvm/include/llvm/SandboxIR/Region.h
+++ b/llvm/include/llvm/SandboxIR/Region.h
@@ -120,6 +120,7 @@ class Region {
   void remove(Instruction *I);
   friend class Context; // The callbacks need to call add() and remove().
   friend class RegionInternalsAttorney; // For unit tests.
+  friend class RegionsFromBBs;          // For add().
 
   /// Set \p I as the \p Idx'th element in the auxiliary vector.
   /// NOTE: This is for internal use, it does not set the metadata.
diff --git a/llvm/include/llvm/Support/AArch64AttributeParser.h b/llvm/include/llvm/Support/AArch64AttributeParser.h
new file mode 100644
index 0000000000000..823ae180a5c5a
--- /dev/null
+++ b/llvm/include/llvm/Support/AArch64AttributeParser.h
@@ -0,0 +1,29 @@
+//=== - AArch64AttributeParser.h-AArch64 Attribute Information Printer - ===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_AARCH64ATTRIBUTEPARSER_H
+#define LLVM_SUPPORT_AARCH64ATTRIBUTEPARSER_H
+
+#include "llvm/Support/ELFAttrParserExtended.h"
+#include "llvm/Support/ELFAttributes.h"
+
+namespace llvm {
+
+class AArch64AttributeParser : public ELFExtendedAttrParser {
+  static std::vector<SubsectionAndTagToTagName> &returnTagsNamesMap();
+
+public:
+  AArch64AttributeParser(ScopedPrinter *Sw)
+      : ELFExtendedAttrParser(Sw, returnTagsNamesMap()) {}
+  AArch64AttributeParser()
+      : ELFExtendedAttrParser(nullptr, returnTagsNamesMap()) {}
+};
+
+} // namespace llvm
+
+#endif // LLVM_SUPPORT_AARCH64ATTRIBUTEPARSER_H
diff --git a/llvm/include/llvm/Support/AArch64BuildAttributes.h b/llvm/include/llvm/Support/AArch64BuildAttributes.h
index 2479992cf8e79..477c7f4f7efee 100644
--- a/llvm/include/llvm/Support/AArch64BuildAttributes.h
+++ b/llvm/include/llvm/Support/AArch64BuildAttributes.h
@@ -22,7 +22,7 @@
 
 namespace llvm {
 
-namespace AArch64BuildAttrs {
+namespace AArch64BuildAttributes {
 
 /// AArch64 build attributes vendors IDs (a.k.a subsection name)
 enum VendorID : unsigned {
@@ -69,7 +69,7 @@ enum FeatureAndBitsFlag : unsigned {
   Feature_PAC_Flag = 1 << 1,
   Feature_GCS_Flag = 1 << 2
 };
-} // namespace AArch64BuildAttrs
+} // namespace AArch64BuildAttributes
 } // namespace llvm
 
-#endif // LLVM_SUPPORT_AARCH64BUILDATTRIBUTES_H
\ No newline at end of file
+#endif // LLVM_SUPPORT_AARCH64BUILDATTRIBUTES_H
diff --git a/llvm/include/llvm/Support/ARMAttributeParser.h b/llvm/include/llvm/Support/ARMAttributeParser.h
index d1d953120ae7e..749f9cd2213d7 100644
--- a/llvm/include/llvm/Support/ARMAttributeParser.h
+++ b/llvm/include/llvm/Support/ARMAttributeParser.h
@@ -10,15 +10,15 @@
 #define LLVM_SUPPORT_ARMATTRIBUTEPARSER_H
 
 #include "ARMBuildAttributes.h"
-#include "ELFAttributeParser.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/ELFAttrParserCompact.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
 
 class ScopedPrinter;
 
-class ARMAttributeParser : public ELFAttributeParser {
+class ARMAttributeParser : public ELFCompactAttrParser {
   struct DisplayHandler {
     ARMBuildAttrs::AttrType attribute;
     Error (ARMAttributeParser::*routine)(ARMBuildAttrs::AttrType);
@@ -74,9 +74,10 @@ class ARMAttributeParser : public ELFAttributeParser {
 
 public:
   ARMAttributeParser(ScopedPrinter *sw)
-      : ELFAttributeParser(sw, ARMBuildAttrs::getARMAttributeTags(), "aeabi") {}
+      : ELFCompactAttrParser(sw, ARMBuildAttrs::getARMAttributeTags(),
+                             "aeabi") {}
   ARMAttributeParser()
-      : ELFAttributeParser(ARMBuildAttrs::getARMAttributeTags(), "aeabi") {}
+      : ELFCompactAttrParser(ARMBuildAttrs::getARMAttributeTags(), "aeabi") {}
 };
 }
 
diff --git a/llvm/include/llvm/Support/CSKYAttributeParser.h b/llvm/include/llvm/Support/CSKYAttributeParser.h
index e926ebe5e306e..08257a744a959 100644
--- a/llvm/include/llvm/Support/CSKYAttributeParser.h
+++ b/llvm/include/llvm/Support/CSKYAttributeParser.h
@@ -10,10 +10,10 @@
 #define LLVM_SUPPORT_CSKYATTRIBUTEPARSER_H
 
 #include "llvm/Support/CSKYAttributes.h"
-#include "llvm/Support/ELFAttributeParser.h"
+#include "llvm/Support/ELFAttrParserCompact.h"
 
 namespace llvm {
-class CSKYAttributeParser : public ELFAttributeParser {
+class CSKYAttributeParser : public ELFCompactAttrParser {
   struct DisplayHandler {
     CSKYAttrs::AttrType attribute;
     Error (CSKYAttributeParser::*routine)(unsigned);
@@ -33,9 +33,9 @@ class CSKYAttributeParser : public ELFAttributeParser {
 
 public:
   CSKYAttributeParser(ScopedPrinter *sw)
-      : ELFAttributeParser(sw, CSKYAttrs::getCSKYAttributeTags(), "csky") {}
+      : ELFCompactAttrParser(sw, CSKYAttrs::getCSKYAttributeTags(), "csky") {}
   CSKYAttributeParser()
-      : ELFAttributeParser(CSKYAttrs::getCSKYAttributeTags(), "csky") {}
+      : ELFCompactAttrParser(CSKYAttrs::getCSKYAttributeTags(), "csky") {}
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Support/ConvertUTF.h b/llvm/include/llvm/Support/ConvertUTF.h
index c892bb3c03cb5..25d46178457d6 100644
--- a/llvm/include/llvm/Support/ConvertUTF.h
+++ b/llvm/include/llvm/Support/ConvertUTF.h
@@ -47,7 +47,7 @@
 
     Conversions between UTF32, UTF-16, and UTF-8.  Header file.
 
-    Several funtions are included here, forming a complete set of
+    Several functions are included here, forming a complete set of
     conversions between the three formats.  UTF-7 is not included
     here, but is handled in a separate source file.
 
diff --git a/llvm/include/llvm/Support/ELFAttrParserCompact.h b/llvm/include/llvm/Support/ELFAttrParserCompact.h
new file mode 100644
index 0000000000000..34f740bc9f0a2
--- /dev/null
+++ b/llvm/include/llvm/Support/ELFAttrParserCompact.h
@@ -0,0 +1,92 @@
+//===- ELF AttributeParser.h - ELF Attribute Parser -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_ELFCOMPACTATTRPARSER_H
+#define LLVM_SUPPORT_ELFCOMPACTATTRPARSER_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/DataExtractor.h"
+#include "llvm/Support/ELFAttributeParser.h"
+#include "llvm/Support/ELFAttributes.h"
+#include "llvm/Support/Error.h"
+
+#include <optional>
+#include <unordered_map>
+
+namespace llvm {
+class StringRef;
+class ScopedPrinter;
+
+class ELFCompactAttrParser : public ELFAttributeParser {
+  StringRef vendor;
+  std::unordered_map<unsigned, unsigned> attributes;
+  std::unordered_map<unsigned, StringRef> attributesStr;
+
+  virtual Error handler(uint64_t tag, bool &handled) = 0;
+
+protected:
+  ScopedPrinter *sw;
+  TagNameMap tagToStringMap;
+  DataExtractor de{ArrayRef<uint8_t>{}, true, 0};
+  DataExtractor::Cursor cursor{0};
+
+  void printAttribute(unsigned tag, unsigned value, StringRef valueDesc);
+
+  Error parseStringAttribute(const char *name, unsigned tag,
+                             ArrayRef<const char *> strings);
+  Error parseAttributeList(uint32_t length);
+  void parseIndexList(SmallVectorImpl<uint8_t> &indexList);
+  Error parseSubsection(uint32_t length);
+
+  void setAttributeString(unsigned tag, StringRef value) {
+    attributesStr.emplace(tag, value);
+  }
+
+public:
+  virtual ~ELFCompactAttrParser() { static_cast<void>(!cursor.takeError()); }
+  Error integerAttribute(unsigned tag);
+  Error stringAttribute(unsigned tag);
+
+  ELFCompactAttrParser(ScopedPrinter *sw, TagNameMap tagNameMap,
+                       StringRef vendor)
+      : vendor(vendor), sw(sw), tagToStringMap(tagNameMap) {}
+  ELFCompactAttrParser(TagNameMap tagNameMap, StringRef vendor)
+      : vendor(vendor), sw(nullptr), tagToStringMap(tagNameMap) {}
+
+  Error parse(ArrayRef<uint8_t> section, llvm::endianness endian) override;
+
+  std::optional<unsigned> getAttributeValue(unsigned tag) const override {
+    auto I = attributes.find(tag);
+    if (I == attributes.end())
+      return std::nullopt;
+    return I->second;
+  }
+  std::optional<unsigned>
+  getAttributeValue(StringRef buildAttributeSubsectionName,
+                    unsigned tag) const override {
+    assert("" == buildAttributeSubsectionName &&
+           "buildAttributeSubsectionName must be an empty string");
+    return getAttributeValue(tag);
+  }
+  std::optional<StringRef> getAttributeString(unsigned tag) const override {
+    auto I = attributesStr.find(tag);
+    if (I == attributesStr.end())
+      return std::nullopt;
+    return I->second;
+  }
+  std::optional<StringRef>
+  getAttributeString(StringRef buildAttributeSubsectionName,
+                     unsigned tag) const override {
+    assert("" == buildAttributeSubsectionName &&
+           "buildAttributeSubsectionName must be an empty string");
+    return getAttributeString(tag);
+  }
+};
+
+} // namespace llvm
+#endif // LLVM_SUPPORT_ELFCOMPACTATTRPARSER_H
diff --git a/llvm/include/llvm/Support/ELFAttrParserExtended.h b/llvm/include/llvm/Support/ELFAttrParserExtended.h
new file mode 100644
index 0000000000000..63712c38d7312
--- /dev/null
+++ b/llvm/include/llvm/Support/ELFAttrParserExtended.h
@@ -0,0 +1,56 @@
+//===- ELF AttributeParser.h - ELF Attribute Parser -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_ELFEXTENDEDATTRPARSER_H
+#define LLVM_SUPPORT_ELFEXTENDEDATTRPARSER_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/DataExtractor.h"
+#include "llvm/Support/ELFAttributeParser.h"
+#include "llvm/Support/ELFAttributes.h"
+#include "llvm/Support/Error.h"
+#include <optional>
+#include <vector>
+
+namespace llvm {
+class StringRef;
+class ScopedPrinter;
+
+class ELFExtendedAttrParser : public ELFAttributeParser {
+protected:
+  ScopedPrinter *Sw;
+  DataExtractor De{ArrayRef<uint8_t>{}, true, 0};
+  DataExtractor::Cursor Cursor{0};
+
+  // Data structure for holding Extended ELF Build Attribute subsection
+  SmallVector<BuildAttributeSubSection, 8> SubSectionVec;
+  // Maps SubsectionName + Tag to tags names. Required for printing comments.
+  const std::vector<SubsectionAndTagToTagName> TagsNamesMap;
+  StringRef getTagName(const StringRef &BuildAttrSubsectionName,
+                       const unsigned Tag);
+
+public:
+  Error parse(ArrayRef<uint8_t> Section, llvm::endianness Endian) override;
+
+  std::optional<unsigned> getAttributeValue(unsigned Tag) const override;
+  std::optional<unsigned> getAttributeValue(StringRef BuildAttrSubsectionName,
+                                            unsigned Tag) const override;
+  std::optional<StringRef> getAttributeString(unsigned Tag) const override;
+  std::optional<StringRef> getAttributeString(StringRef BuildAttrSubsectionName,
+                                              unsigned Tag) const override;
+
+  ELFExtendedAttrParser(
+      ScopedPrinter *Sw,
+      const std::vector<SubsectionAndTagToTagName> TagsNamesMap)
+      : Sw(Sw), TagsNamesMap(TagsNamesMap) {}
+  ELFExtendedAttrParser(
+      const std::vector<SubsectionAndTagToTagName> TagsNamesMap)
+      : Sw(nullptr), TagsNamesMap(TagsNamesMap) {}
+};
+} // namespace llvm
+#endif // LLVM_SUPPORT_ELFEXTENDEDATTRPARSER_H
diff --git a/llvm/include/llvm/Support/ELFAttributeParser.h b/llvm/include/llvm/Support/ELFAttributeParser.h
index ffb92468fb37e..97350edb793c9 100644
--- a/llvm/include/llvm/Support/ELFAttributeParser.h
+++ b/llvm/include/llvm/Support/ELFAttributeParser.h
@@ -9,69 +9,34 @@
 #ifndef LLVM_SUPPORT_ELFATTRIBUTEPARSER_H
 #define LLVM_SUPPORT_ELFATTRIBUTEPARSER_H
 
-#include "ELFAttributes.h"
 #include "llvm/ADT/ArrayRef.h"
-#include "llvm/Support/DataExtractor.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
 
-#include <optional>
-#include <unordered_map>
-
 namespace llvm {
-class StringRef;
-class ScopedPrinter;
 
 class ELFAttributeParser {
-  StringRef vendor;
-  std::unordered_map<unsigned, unsigned> attributes;
-  std::unordered_map<unsigned, StringRef> attributesStr;
-
-  virtual Error handler(uint64_t tag, bool &handled) = 0;
-
-protected:
-  ScopedPrinter *sw;
-  TagNameMap tagToStringMap;
-  DataExtractor de{ArrayRef<uint8_t>{}, true, 0};
-  DataExtractor::Cursor cursor{0};
-
-  void printAttribute(unsigned tag, unsigned value, StringRef valueDesc);
-
-  Error parseStringAttribute(const char *name, unsigned tag,
-                             ArrayRef<const char *> strings);
-  Error parseAttributeList(uint32_t length);
-  void parseIndexList(SmallVectorImpl<uint8_t> &indexList);
-  Error parseSubsection(uint32_t length);
-
-  void setAttributeString(unsigned tag, StringRef value) {
-    attributesStr.emplace(tag, value);
-  }
-
 public:
-  virtual ~ELFAttributeParser() { static_cast<void>(!cursor.takeError()); }
-  Error integerAttribute(unsigned tag);
-  Error stringAttribute(unsigned tag);
-
-  ELFAttributeParser(ScopedPrinter *sw, TagNameMap tagNameMap, StringRef vendor)
-      : vendor(vendor), sw(sw), tagToStringMap(tagNameMap) {}
-
-  ELFAttributeParser(TagNameMap tagNameMap, StringRef vendor)
-      : vendor(vendor), sw(nullptr), tagToStringMap(tagNameMap) {}
-
-  Error parse(ArrayRef<uint8_t> section, llvm::endianness endian);
+  virtual ~ELFAttributeParser() {}
 
-  std::optional<unsigned> getAttributeValue(unsigned tag) const {
-    auto I = attributes.find(tag);
-    if (I == attributes.end())
-      return std::nullopt;
-    return I->second;
+  virtual Error parse(ArrayRef<uint8_t> Section, llvm::endianness Endian) {
+    return llvm::Error::success();
+  }
+  virtual std::optional<unsigned>
+  getAttributeValue(StringRef BuildAttrSubsectionName, unsigned Tag) const {
+    return std::nullopt;
+  }
+  virtual std::optional<unsigned> getAttributeValue(unsigned Tag) const {
+    return std::nullopt;
+  }
+  virtual std::optional<StringRef>
+  getAttributeString(StringRef BuildAttrSubsectionName, unsigned Tag) const {
+    return std::nullopt;
   }
-  std::optional<StringRef> getAttributeString(unsigned tag) const {
-    auto I = attributesStr.find(tag);
-    if (I == attributesStr.end())
-      return std::nullopt;
-    return I->second;
+  virtual std::optional<StringRef> getAttributeString(unsigned Tag) const {
+    return std::nullopt;
   }
 };
 
 } // namespace llvm
-#endif
+#endif // LLVM_SUPPORT_ELFATTRIBUTEPARSER_H
diff --git a/llvm/include/llvm/Support/ELFAttributes.h b/llvm/include/llvm/Support/ELFAttributes.h
index 295d0f4669812..6782aec6050ad 100644
--- a/llvm/include/llvm/Support/ELFAttributes.h
+++ b/llvm/include/llvm/Support/ELFAttributes.h
@@ -15,6 +15,7 @@
 
 namespace llvm {
 
+// Tag to string: ELF compact build attribute section
 struct TagNameItem {
   unsigned attr;
   StringRef tagName;
@@ -22,6 +23,34 @@ struct TagNameItem {
 
 using TagNameMap = ArrayRef<TagNameItem>;
 
+// Build Attribute storage for ELF extended attribute section
+struct BuildAttributeItem {
+  enum Types : uint8_t {
+    NumericAttribute = 0,
+    TextAttribute,
+  } Type;
+  unsigned Tag;
+  unsigned IntValue;
+  std::string StringValue;
+  BuildAttributeItem(Types Ty, unsigned Tg, unsigned IV, std::string SV)
+      : Type(Ty), Tag(Tg), IntValue(IV), StringValue(std::move(SV)) {}
+};
+struct BuildAttributeSubSection {
+  StringRef Name;
+  unsigned IsOptional;
+  unsigned ParameterType;
+  SmallVector<BuildAttributeItem, 64> Content;
+};
+
+// Tag to string: ELF extended build attribute section
+struct SubsectionAndTagToTagName {
+  StringRef SubsectionName;
+  unsigned Tag;
+  StringRef TagName;
+  SubsectionAndTagToTagName(StringRef SN, unsigned Tg, StringRef TN)
+      : SubsectionName(SN), Tag(Tg), TagName(TN) {}
+};
+
 namespace ELFAttrs {
 
 enum AttrType : unsigned { File = 1, Section = 2, Symbol = 3 };
diff --git a/llvm/include/llvm/Support/ErrorOr.h b/llvm/include/llvm/Support/ErrorOr.h
index d195cbb04ec7d..1daac9a436c91 100644
--- a/llvm/include/llvm/Support/ErrorOr.h
+++ b/llvm/include/llvm/Support/ErrorOr.h
@@ -16,6 +16,7 @@
 #define LLVM_SUPPORT_ERROROR_H
 
 #include <cassert>
+#include <functional> // for std::reference_wrapper
 #include <system_error>
 #include <type_traits>
 #include <utility>
diff --git a/llvm/include/llvm/Support/HexagonAttributeParser.h b/llvm/include/llvm/Support/HexagonAttributeParser.h
index 1116dd42b1ad0..462bfc4e2df1e 100644
--- a/llvm/include/llvm/Support/HexagonAttributeParser.h
+++ b/llvm/include/llvm/Support/HexagonAttributeParser.h
@@ -9,11 +9,11 @@
 #ifndef LLVM_SUPPORT_HEXAGONATTRIBUTEPARSER_H
 #define LLVM_SUPPORT_HEXAGONATTRIBUTEPARSER_H
 
-#include "llvm/Support/ELFAttributeParser.h"
+#include "llvm/Support/ELFAttrParserCompact.h"
 #include "llvm/Support/HexagonAttributes.h"
 
 namespace llvm {
-class HexagonAttributeParser : public ELFAttributeParser {
+class HexagonAttributeParser : public ELFCompactAttrParser {
   struct DisplayHandler {
     HexagonAttrs::AttrType Attribute;
     Error (HexagonAttributeParser::*Routine)(unsigned);
@@ -25,11 +25,11 @@ class HexagonAttributeParser : public ELFAttributeParser {
 
 public:
   HexagonAttributeParser(ScopedPrinter *SP)
-      : ELFAttributeParser(SP, HexagonAttrs::getHexagonAttributeTags(),
-                           "hexagon") {}
+      : ELFCompactAttrParser(SP, HexagonAttrs::getHexagonAttributeTags(),
+                             "hexagon") {}
   HexagonAttributeParser()
-      : ELFAttributeParser(HexagonAttrs::getHexagonAttributeTags(), "hexagon") {
-  }
+      : ELFCompactAttrParser(HexagonAttrs::getHexagonAttributeTags(),
+                             "hexagon") {}
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Support/JSON.h b/llvm/include/llvm/Support/JSON.h
index 14a5c7142ed8c..3e3f783ffb857 100644
--- a/llvm/include/llvm/Support/JSON.h
+++ b/llvm/include/llvm/Support/JSON.h
@@ -472,6 +472,14 @@ class Value {
     return LLVM_LIKELY(Type == T_Array) ? &as<json::Array>() : nullptr;
   }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  void print(llvm::raw_ostream &OS) const;
+  LLVM_DUMP_METHOD void dump() const {
+    print(llvm::dbgs());
+    llvm::dbgs() << '\n';
+  }
+#endif // !NDEBUG || LLVM_ENABLE_DUMP
+
 private:
   void destroy();
   void copyFrom(const Value &M);
diff --git a/llvm/include/llvm/Support/LockFileManager.h b/llvm/include/llvm/Support/LockFileManager.h
index 92c7ceed6a929..cf02b41a6f729 100644
--- a/llvm/include/llvm/Support/LockFileManager.h
+++ b/llvm/include/llvm/Support/LockFileManager.h
@@ -9,36 +9,24 @@
 #define LLVM_SUPPORT_LOCKFILEMANAGER_H
 
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Support/Error.h"
 #include <optional>
+#include <string>
 #include <system_error>
-#include <utility> // for std::pair
+#include <variant>
 
 namespace llvm {
 class StringRef;
 
-/// Class that manages the creation of a lock file to aid
-/// implicit coordination between different processes.
+/// Class that manages the creation of a lock file to aid implicit coordination
+/// between different processes.
 ///
-/// The implicit coordination works by creating a ".lock" file alongside
-/// the file that we're coordinating for, using the atomicity of the file
-/// system to ensure that only a single process can create that ".lock" file.
-/// When the lock file is removed, the owning process has finished the
-/// operation.
+/// The implicit coordination works by creating a ".lock" file, using the
+/// atomicity of the file system to ensure that only a single process can create
+/// that ".lock" file. When the lock file is removed, the owning process has
+/// finished the operation.
 class LockFileManager {
 public:
-  /// Describes the state of a lock file.
-  enum LockFileState {
-    /// The lock file has been created and is owned by this instance
-    /// of the object.
-    LFS_Owned,
-    /// The lock file already exists and is owned by some other
-    /// instance.
-    LFS_Shared,
-    /// An error occurred while trying to create or find the lock
-    /// file.
-    LFS_Error
-  };
-
   /// Describes the result of waiting for the owner to release the lock.
   enum WaitForUnlockResult {
     /// The lock was released successfully.
@@ -54,27 +42,32 @@ class LockFileManager {
   SmallString<128> LockFileName;
   SmallString<128> UniqueLockFileName;
 
-  std::optional<std::pair<std::string, int>> Owner;
-  std::error_code ErrorCode;
-  std::string ErrorDiagMsg;
+  struct OwnerUnknown {};
+  struct OwnedByUs {};
+  struct OwnedByAnother {
+    std::string OwnerHostName;
+    int OwnerPID;
+  };
+  std::variant<OwnerUnknown, OwnedByUs, OwnedByAnother> Owner;
 
   LockFileManager(const LockFileManager &) = delete;
   LockFileManager &operator=(const LockFileManager &) = delete;
 
-  static std::optional<std::pair<std::string, int>>
-  readLockFile(StringRef LockFileName);
+  static std::optional<OwnedByAnother> readLockFile(StringRef LockFileName);
 
   static bool processStillExecuting(StringRef Hostname, int PID);
 
 public:
-
+  /// Does not try to acquire the lock.
   LockFileManager(StringRef FileName);
-  ~LockFileManager();
 
-  /// Determine the state of the lock file.
-  LockFileState getState() const;
+  /// Unlocks the lock if previously acquired by \c tryLock().
+  ~LockFileManager();
 
-  operator LockFileState() const { return getState(); }
+  /// Tries to acquire the lock without blocking.
+  /// \returns true if the lock was successfully acquired, false if the lock is
+  /// already held by someone else, or \c Error in case of unexpected failure.
+  Expected<bool> tryLock();
 
   /// For a shared lock, wait until the owner releases the lock.
   /// Total timeout for the file to appear is ~1.5 minutes.
@@ -84,15 +77,6 @@ class LockFileManager {
   /// Remove the lock file.  This may delete a different lock file than
   /// the one previously read if there is a race.
   std::error_code unsafeRemoveLockFile();
-
-  /// Get error message, or "" if there is no error.
-  std::string getErrorMessage() const;
-
-  /// Set error and error message
-  void setError(const std::error_code &EC, StringRef ErrorMsg = "") {
-    ErrorCode = EC;
-    ErrorDiagMsg = ErrorMsg.str();
-  }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/Support/MSP430AttributeParser.h b/llvm/include/llvm/Support/MSP430AttributeParser.h
index bc9b214944708..3a4f1b43d9d18 100644
--- a/llvm/include/llvm/Support/MSP430AttributeParser.h
+++ b/llvm/include/llvm/Support/MSP430AttributeParser.h
@@ -14,11 +14,11 @@
 #ifndef LLVM_SUPPORT_MSP430ATTRIBUTEPARSER_H
 #define LLVM_SUPPORT_MSP430ATTRIBUTEPARSER_H
 
-#include "llvm/Support/ELFAttributeParser.h"
+#include "llvm/Support/ELFAttrParserCompact.h"
 #include "llvm/Support/MSP430Attributes.h"
 
 namespace llvm {
-class MSP430AttributeParser : public ELFAttributeParser {
+class MSP430AttributeParser : public ELFCompactAttrParser {
   struct DisplayHandler {
     MSP430Attrs::AttrType Attribute;
     Error (MSP430AttributeParser::*Routine)(MSP430Attrs::AttrType);
@@ -34,10 +34,10 @@ class MSP430AttributeParser : public ELFAttributeParser {
 
 public:
   MSP430AttributeParser(ScopedPrinter *SW)
-      : ELFAttributeParser(SW, MSP430Attrs::getMSP430AttributeTags(),
-                           "mspabi") {}
+      : ELFCompactAttrParser(SW, MSP430Attrs::getMSP430AttributeTags(),
+                             "mspabi") {}
   MSP430AttributeParser()
-      : ELFAttributeParser(MSP430Attrs::getMSP430AttributeTags(), "mspabi") {}
+      : ELFCompactAttrParser(MSP430Attrs::getMSP430AttributeTags(), "mspabi") {}
 };
 } // namespace llvm
 
diff --git a/llvm/include/llvm/Support/MathExtras.h b/llvm/include/llvm/Support/MathExtras.h
index 5a6f51adc07f3..07f7bab0f0a3a 100644
--- a/llvm/include/llvm/Support/MathExtras.h
+++ b/llvm/include/llvm/Support/MathExtras.h
@@ -723,7 +723,7 @@ std::enable_if_t<std::is_signed_v<T>, T> AddOverflow(T X, T Y, T &Result) {
 }
 
 /// Subtract two signed integers, computing the two's complement truncated
-/// result, returning true if an overflow ocurred.
+/// result, returning true if an overflow occurred.
 template <typename T>
 std::enable_if_t<std::is_signed_v<T>, T> SubOverflow(T X, T Y, T &Result) {
 #if __has_builtin(__builtin_sub_overflow)
@@ -749,7 +749,7 @@ std::enable_if_t<std::is_signed_v<T>, T> SubOverflow(T X, T Y, T &Result) {
 }
 
 /// Multiply two signed integers, computing the two's complement truncated
-/// result, returning true if an overflow ocurred.
+/// result, returning true if an overflow occurred.
 template <typename T>
 std::enable_if_t<std::is_signed_v<T>, T> MulOverflow(T X, T Y, T &Result) {
 #if __has_builtin(__builtin_mul_overflow)
diff --git a/llvm/include/llvm/Support/Mustache.h b/llvm/include/llvm/Support/Mustache.h
new file mode 100644
index 0000000000000..41173b96d1a9a
--- /dev/null
+++ b/llvm/include/llvm/Support/Mustache.h
@@ -0,0 +1,127 @@
+//===--- Mustache.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of the Mustache templating language supports version 1.4.2
+// currently relies on llvm::json::Value for data input.
+// See the Mustache spec for more information
+// (https://mustache.github.io/mustache.5.html).
+//
+// Current Features Supported:
+// - Variables
+// - Sections
+// - Inverted Sections
+// - Partials
+// - Comments
+// - Lambdas
+// - Unescaped Variables
+//
+// Features Not Supported:
+// - Set Delimiter
+// - Blocks
+// - Parents
+// - Dynamic Names
+//
+// The Template class is a container class that outputs the Mustache template
+// string and is the main class for users. It stores all the lambdas and the
+// ASTNode Tree. When the Template is instantiated it tokenizes the Template
+// String and creates a vector of Tokens. Then it calls a basic recursive
+// descent parser to construct the ASTNode Tree. The ASTNodes are all stored
+// in an arena allocator which is freed once the template class goes out of
+// scope.
+//
+// Usage:
+// \code
+//   // Creating a simple template and rendering it
+//   auto Template = Template("Hello, {{name}}!");
+//   Value Data = {{"name", "World"}};
+//   std::string Out;
+//   raw_string_ostream OS(Out);
+//   T.render(Data, OS);
+//   // Out == "Hello, World!"
+//
+//   // Creating a template with a partial and rendering it
+//   auto Template = Template("{{>partial}}");
+//   Template.registerPartial("partial", "Hello, {{name}}!");
+//   Value Data = {{"name", "World"}};
+//   std::string Out;
+//   raw_string_ostream OS(Out);
+//   T.render(Data, OS);
+//   // Out == "Hello, World!"
+//
+//   // Creating a template with a lambda and rendering it
+//   Value D = Object{};
+//   auto T = Template("Hello, {{lambda}}!");
+//   Lambda L = []() -> llvm::json::Value { return "World"; };
+//   T.registerLambda("lambda", L);
+//   std::string Out;
+//   raw_string_ostream OS(Out);
+//   T.render(D, OS);
+//   // Out == "Hello, World!"
+// \endcode
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_MUSTACHE
+#define LLVM_SUPPORT_MUSTACHE
+
+#include "Error.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/StringSaver.h"
+#include <functional>
+#include <vector>
+
+namespace llvm::mustache {
+
+using Lambda = std::function<llvm::json::Value()>;
+using SectionLambda = std::function<llvm::json::Value(std::string)>;
+
+class ASTNode;
+
+// A Template represents the container for the AST and the partials
+// and Lambdas that are registered with it.
+class Template {
+public:
+  Template(StringRef TemplateStr);
+
+  Template(const Template &) = delete;
+
+  Template &operator=(const Template &) = delete;
+
+  Template(Template &&Other) noexcept;
+
+  Template &operator=(Template &&Other) noexcept;
+
+  void render(const llvm::json::Value &Data, llvm::raw_ostream &OS);
+
+  void registerPartial(std::string Name, std::string Partial);
+
+  void registerLambda(std::string Name, Lambda Lambda);
+
+  void registerLambda(std::string Name, SectionLambda Lambda);
+
+  // By default the Mustache Spec Specifies that HTML special characters
+  // should be escaped. This function allows the user to specify which
+  // characters should be escaped.
+  void overrideEscapeCharacters(DenseMap<char, std::string> Escapes);
+
+private:
+  StringMap<ASTNode *> Partials;
+  StringMap<Lambda> Lambdas;
+  StringMap<SectionLambda> SectionLambdas;
+  DenseMap<char, std::string> Escapes;
+  // The allocator for the ASTNode Tree
+  llvm::BumpPtrAllocator AstAllocator;
+  // Allocator for each render call resets after each render
+  llvm::BumpPtrAllocator RenderAllocator;
+  ASTNode *Tree;
+};
+} // namespace llvm::mustache
+
+#endif // LLVM_SUPPORT_MUSTACHE
diff --git a/llvm/include/llvm/Support/RISCVAttributeParser.h b/llvm/include/llvm/Support/RISCVAttributeParser.h
index 9f295504de959..4a74ed321d7a2 100644
--- a/llvm/include/llvm/Support/RISCVAttributeParser.h
+++ b/llvm/include/llvm/Support/RISCVAttributeParser.h
@@ -9,11 +9,11 @@
 #ifndef LLVM_SUPPORT_RISCVATTRIBUTEPARSER_H
 #define LLVM_SUPPORT_RISCVATTRIBUTEPARSER_H
 
-#include "llvm/Support/ELFAttributeParser.h"
+#include "llvm/Support/ELFAttrParserCompact.h"
 #include "llvm/Support/RISCVAttributes.h"
 
 namespace llvm {
-class RISCVAttributeParser : public ELFAttributeParser {
+class RISCVAttributeParser : public ELFCompactAttrParser {
   struct DisplayHandler {
     RISCVAttrs::AttrType attribute;
     Error (RISCVAttributeParser::*routine)(unsigned);
@@ -28,9 +28,10 @@ class RISCVAttributeParser : public ELFAttributeParser {
 
 public:
   RISCVAttributeParser(ScopedPrinter *sw)
-      : ELFAttributeParser(sw, RISCVAttrs::getRISCVAttributeTags(), "riscv") {}
+      : ELFCompactAttrParser(sw, RISCVAttrs::getRISCVAttributeTags(), "riscv") {
+  }
   RISCVAttributeParser()
-      : ELFAttributeParser(RISCVAttrs::getRISCVAttributeTags(), "riscv") {}
+      : ELFCompactAttrParser(RISCVAttrs::getRISCVAttributeTags(), "riscv") {}
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromBBs.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromBBs.h
new file mode 100644
index 0000000000000..66c9cd42fb878
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromBBs.h
@@ -0,0 +1,38 @@
+//===- RegionsFromBBs.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A SandboxIR function pass that builds one region per BB and then runs a
+// pipeline of region passes on them. This is useful to test region passes in
+// isolation without relying on the output of other vectorizer components.
+//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_REGIONSFROMBBS_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_REGIONSFROMBBS_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/SandboxIR/Pass.h"
+#include "llvm/SandboxIR/PassManager.h"
+
+namespace llvm::sandboxir {
+
+class RegionsFromBBs final : public FunctionPass {
+  // The PM containing the pipeline of region passes.
+  RegionPassManager RPM;
+
+public:
+  RegionsFromBBs(StringRef Pipeline);
+  bool runOnFunction(Function &F, const Analyses &A) final;
+  void printPipeline(raw_ostream &OS) const final {
+    OS << getName() << "\n";
+    RPM.printPipeline(OS);
+  }
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_REGIONSFROMBBS_H
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index d645bf8f7b621..b0ba25c3c16ac 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -943,18 +943,21 @@ Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP,
 
   // If the base value for this address is a literal integer value, fold the
   // getelementptr to the resulting integer value casted to the pointer type.
-  APInt BasePtr(BitWidth, 0);
+  APInt BasePtr(DL.getPointerTypeSizeInBits(Ptr->getType()), 0);
   if (auto *CE = dyn_cast<ConstantExpr>(Ptr)) {
     if (CE->getOpcode() == Instruction::IntToPtr) {
       if (auto *Base = dyn_cast<ConstantInt>(CE->getOperand(0)))
-        BasePtr = Base->getValue().zextOrTrunc(BitWidth);
+        BasePtr = Base->getValue().zextOrTrunc(BasePtr.getBitWidth());
     }
   }
 
   auto *PTy = cast<PointerType>(Ptr->getType());
   if ((Ptr->isNullValue() || BasePtr != 0) &&
       !DL.isNonIntegralPointerType(PTy)) {
-    Constant *C = ConstantInt::get(Ptr->getContext(), Offset + BasePtr);
+    // If the index size is smaller than the pointer size, add to the low
+    // bits only.
+    BasePtr.insertBits(BasePtr.trunc(BitWidth) + Offset, 0);
+    Constant *C = ConstantInt::get(Ptr->getContext(), BasePtr);
     return ConstantExpr::getIntToPtr(C, ResTy);
   }
 
diff --git a/llvm/lib/Analysis/CostModel.cpp b/llvm/lib/Analysis/CostModel.cpp
index 68cb536bf7891..cec0fb6b98dea 100644
--- a/llvm/lib/Analysis/CostModel.cpp
+++ b/llvm/lib/Analysis/CostModel.cpp
@@ -28,30 +28,81 @@
 
 using namespace llvm;
 
-static cl::opt<TargetTransformInfo::TargetCostKind> CostKind(
+enum class OutputCostKind {
+  RecipThroughput,
+  Latency,
+  CodeSize,
+  SizeAndLatency,
+  All,
+};
+
+static cl::opt<OutputCostKind> CostKind(
     "cost-kind", cl::desc("Target cost kind"),
-    cl::init(TargetTransformInfo::TCK_RecipThroughput),
-    cl::values(clEnumValN(TargetTransformInfo::TCK_RecipThroughput,
-                          "throughput", "Reciprocal throughput"),
-               clEnumValN(TargetTransformInfo::TCK_Latency,
-                          "latency", "Instruction latency"),
-               clEnumValN(TargetTransformInfo::TCK_CodeSize,
-                          "code-size", "Code size"),
-               clEnumValN(TargetTransformInfo::TCK_SizeAndLatency,
-                          "size-latency", "Code size and latency")));
+    cl::init(OutputCostKind::RecipThroughput),
+    cl::values(clEnumValN(OutputCostKind::RecipThroughput, "throughput",
+                          "Reciprocal throughput"),
+               clEnumValN(OutputCostKind::Latency, "latency",
+                          "Instruction latency"),
+               clEnumValN(OutputCostKind::CodeSize, "code-size", "Code size"),
+               clEnumValN(OutputCostKind::SizeAndLatency, "size-latency",
+                          "Code size and latency"),
+               clEnumValN(OutputCostKind::All, "all", "Print all cost kinds")));
 
-static cl::opt<bool> TypeBasedIntrinsicCost("type-based-intrinsic-cost",
-    cl::desc("Calculate intrinsics cost based only on argument types"),
-    cl::init(false));
+enum class IntrinsicCostStrategy {
+  InstructionCost,
+  IntrinsicCost,
+  TypeBasedIntrinsicCost,
+};
 
-static cl::opt<bool> PreferIntrinsicCost(
-    "prefer-intrinsic-cost",
-    cl::desc("Prefer using getIntrinsicInstrCost over getInstructionCost"),
-    cl::init(false));
+static cl::opt<IntrinsicCostStrategy> IntrinsicCost(
+    "intrinsic-cost-strategy",
+    cl::desc("Costing strategy for intrinsic instructions"),
+    cl::init(IntrinsicCostStrategy::InstructionCost),
+    cl::values(
+        clEnumValN(IntrinsicCostStrategy::InstructionCost, "instruction-cost",
+                   "Use TargetTransformInfo::getInstructionCost"),
+        clEnumValN(IntrinsicCostStrategy::IntrinsicCost, "intrinsic-cost",
+                   "Use TargetTransformInfo::getIntrinsicInstrCost"),
+        clEnumValN(
+            IntrinsicCostStrategy::TypeBasedIntrinsicCost,
+            "type-based-intrinsic-cost",
+            "Calculate the intrinsic cost based only on argument types")));
 
 #define CM_NAME "cost-model"
 #define DEBUG_TYPE CM_NAME
 
+static InstructionCost getCost(Instruction &Inst, TTI::TargetCostKind CostKind,
+                               TargetTransformInfo &TTI,
+                               TargetLibraryInfo &TLI) {
+  auto *II = dyn_cast<IntrinsicInst>(&Inst);
+  if (II && IntrinsicCost != IntrinsicCostStrategy::InstructionCost) {
+    IntrinsicCostAttributes ICA(
+        II->getIntrinsicID(), *II, InstructionCost::getInvalid(),
+        /*TypeBasedOnly=*/IntrinsicCost ==
+            IntrinsicCostStrategy::TypeBasedIntrinsicCost,
+        &TLI);
+    return TTI.getIntrinsicInstrCost(ICA, CostKind);
+  }
+
+  return TTI.getInstructionCost(&Inst, CostKind);
+}
+
+static TTI::TargetCostKind
+OutputCostKindToTargetCostKind(OutputCostKind CostKind) {
+  switch (CostKind) {
+  case OutputCostKind::RecipThroughput:
+    return TTI::TCK_RecipThroughput;
+  case OutputCostKind::Latency:
+    return TTI::TCK_Latency;
+  case OutputCostKind::CodeSize:
+    return TTI::TCK_CodeSize;
+  case OutputCostKind::SizeAndLatency:
+    return TTI::TCK_SizeAndLatency;
+  default:
+    llvm_unreachable("Unexpected OutputCostKind!");
+  };
+}
+
 PreservedAnalyses CostModelPrinterPass::run(Function &F,
                                             FunctionAnalysisManager &AM) {
   auto &TTI = AM.getResult<TargetIRAnalysis>(F);
@@ -59,25 +110,30 @@ PreservedAnalyses CostModelPrinterPass::run(Function &F,
   OS << "Printing analysis 'Cost Model Analysis' for function '" << F.getName() << "':\n";
   for (BasicBlock &B : F) {
     for (Instruction &Inst : B) {
-      // TODO: Use a pass parameter instead of cl::opt CostKind to determine
-      // which cost kind to print.
-      InstructionCost Cost;
-      auto *II = dyn_cast<IntrinsicInst>(&Inst);
-      if (II && (PreferIntrinsicCost || TypeBasedIntrinsicCost)) {
-        IntrinsicCostAttributes ICA(
-            II->getIntrinsicID(), *II, InstructionCost::getInvalid(),
-            /*TypeBasedOnly=*/TypeBasedIntrinsicCost, &TLI);
-        Cost = TTI.getIntrinsicInstrCost(ICA, CostKind);
+      OS << "Cost Model: ";
+      if (CostKind == OutputCostKind::All) {
+        OS << "Found costs of ";
+        InstructionCost RThru =
+            getCost(Inst, TTI::TCK_RecipThroughput, TTI, TLI);
+        InstructionCost CodeSize = getCost(Inst, TTI::TCK_CodeSize, TTI, TLI);
+        InstructionCost Lat = getCost(Inst, TTI::TCK_Latency, TTI, TLI);
+        InstructionCost SizeLat =
+            getCost(Inst, TTI::TCK_SizeAndLatency, TTI, TLI);
+        if (RThru == CodeSize && RThru == Lat && RThru == SizeLat)
+          OS << RThru;
+        else
+          OS << "RThru:" << RThru << " CodeSize:" << CodeSize << " Lat:" << Lat
+             << " SizeLat:" << SizeLat;
+        OS << " for: " << Inst << "\n";
       } else {
-        Cost = TTI.getInstructionCost(&Inst, CostKind);
+        InstructionCost Cost =
+            getCost(Inst, OutputCostKindToTargetCostKind(CostKind), TTI, TLI);
+        if (auto CostVal = Cost.getValue())
+          OS << "Found an estimated cost of " << *CostVal;
+        else
+          OS << "Invalid cost";
+        OS << " for instruction: " << Inst << "\n";
       }
-
-      if (auto CostVal = Cost.getValue())
-        OS << "Cost Model: Found an estimated cost of " << *CostVal;
-      else
-        OS << "Cost Model: Invalid cost";
-
-      OS << " for instruction: " << Inst << "\n";
     }
   }
   return PreservedAnalyses::all();
diff --git a/llvm/lib/Analysis/UniformityAnalysis.cpp b/llvm/lib/Analysis/UniformityAnalysis.cpp
index 592de1067e191..71e72ef678d35 100644
--- a/llvm/lib/Analysis/UniformityAnalysis.cpp
+++ b/llvm/lib/Analysis/UniformityAnalysis.cpp
@@ -79,13 +79,12 @@ template <>
 void llvm::GenericUniformityAnalysisImpl<
     SSAContext>::propagateTemporalDivergence(const Instruction &I,
                                              const Cycle &DefCycle) {
-  if (isDivergent(I))
-    return;
   for (auto *User : I.users()) {
     auto *UserInstr = cast<Instruction>(User);
     if (DefCycle.contains(UserInstr->getParent()))
       continue;
     markDivergent(*UserInstr);
+    recordTemporalDivergence(&I, UserInstr, &DefCycle);
   }
 }
 
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 08dc34f89130a..2920738445d5b 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -6208,13 +6208,14 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     else if (Bits.isNegative())
       Known.signBitMustBeOne();
 
-    if (Ty->isIEEE()) {
+    if (Ty->isIEEELikeFPTy()) {
       // IEEE floats are NaN when all bits of the exponent plus at least one of
       // the fraction bits are 1. This means:
       //   - If we assume unknown bits are 0 and the value is NaN, it will
       //     always be NaN
       //   - If we assume unknown bits are 1 and the value is not NaN, it can
       //     never be NaN
+      // Note: They do not hold for x86_fp80 format.
       if (APFloat(Ty->getFltSemantics(), Bits.One).isNaN())
         Known.KnownFPClasses = fcNan;
       else if (!APFloat(Ty->getFltSemantics(), ~Bits.Zero).isNaN())
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index e4dc7d6ed86b0..40e755902b724 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -8075,20 +8075,18 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
       break;
 
     case bitc::FS_CFI_FUNCTION_DEFS: {
-      std::set<std::string, std::less<>> &CfiFunctionDefs =
-          TheIndex.cfiFunctionDefs();
+      auto &CfiFunctionDefs = TheIndex.cfiFunctionDefs();
       for (unsigned I = 0; I != Record.size(); I += 2)
-        CfiFunctionDefs.insert(
-            {Strtab.data() + Record[I], static_cast<size_t>(Record[I + 1])});
+        CfiFunctionDefs.emplace(Strtab.data() + Record[I],
+                                static_cast<size_t>(Record[I + 1]));
       break;
     }
 
     case bitc::FS_CFI_FUNCTION_DECLS: {
-      std::set<std::string, std::less<>> &CfiFunctionDecls =
-          TheIndex.cfiFunctionDecls();
+      auto &CfiFunctionDecls = TheIndex.cfiFunctionDecls();
       for (unsigned I = 0; I != Record.size(); I += 2)
-        CfiFunctionDecls.insert(
-            {Strtab.data() + Record[I], static_cast<size_t>(Record[I + 1])});
+        CfiFunctionDecls.emplace(Strtab.data() + Record[I],
+                                 static_cast<size_t>(Record[I + 1]));
       break;
     }
 
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index a3556af9a683c..f6510a7a1549d 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -5064,33 +5064,29 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
       getReferencedTypeIds(FS, ReferencedTypeIds);
   }
 
-  if (!Index.cfiFunctionDefs().empty()) {
-    for (auto &S : Index.cfiFunctionDefs()) {
-      if (DefOrUseGUIDs.contains(
-              GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(S)))) {
-        NameVals.push_back(StrtabBuilder.add(S));
-        NameVals.push_back(S.size());
-      }
+  SmallVector<StringRef, 4> Functions;
+  auto EmitCfiFunctions = [&](const CfiFunctionIndex &CfiIndex,
+                              bitc::GlobalValueSummarySymtabCodes Code) {
+    if (CfiIndex.empty())
+      return;
+    for (GlobalValue::GUID GUID : DefOrUseGUIDs) {
+      auto Defs = CfiIndex.forGuid(GUID);
+      Functions.insert(Functions.end(), Defs.begin(), Defs.end());
     }
-    if (!NameVals.empty()) {
-      Stream.EmitRecord(bitc::FS_CFI_FUNCTION_DEFS, NameVals);
-      NameVals.clear();
+    if (Functions.empty())
+      return;
+    llvm::sort(Functions);
+    for (const auto &S : Functions) {
+      NameVals.push_back(StrtabBuilder.add(S));
+      NameVals.push_back(S.size());
     }
-  }
+    Stream.EmitRecord(Code, NameVals);
+    NameVals.clear();
+    Functions.clear();
+  };
 
-  if (!Index.cfiFunctionDecls().empty()) {
-    for (auto &S : Index.cfiFunctionDecls()) {
-      if (DefOrUseGUIDs.contains(
-              GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(S)))) {
-        NameVals.push_back(StrtabBuilder.add(S));
-        NameVals.push_back(S.size());
-      }
-    }
-    if (!NameVals.empty()) {
-      Stream.EmitRecord(bitc::FS_CFI_FUNCTION_DECLS, NameVals);
-      NameVals.clear();
-    }
-  }
+  EmitCfiFunctions(Index.cfiFunctionDefs(), bitc::FS_CFI_FUNCTION_DEFS);
+  EmitCfiFunctions(Index.cfiFunctionDecls(), bitc::FS_CFI_FUNCTION_DECLS);
 
   // Walk the GUIDs that were referenced, and write the
   // corresponding type id records.
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 60d911d0383ed..b7b083028ef65 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -2057,6 +2057,16 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) {
     }
   }
 
+  auto RecordSourceLine = [this](auto &DL, auto Flags) {
+    SmallString<128> LocationString;
+    if (Asm->OutStreamer->isVerboseAsm()) {
+      raw_svector_ostream OS(LocationString);
+      DL.print(OS);
+    }
+    recordSourceLine(DL.getLine(), DL.getCol(), DL.getScope(), Flags,
+                     LocationString);
+  };
+
   // When we emit a line-0 record, we don't update PrevInstLoc; so look at
   // the last line number actually emitted, to see if it was line 0.
   unsigned LastAsmLine =
@@ -2084,8 +2094,7 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) {
     // But we might be coming back to it after a line 0 record.
     if ((LastAsmLine == 0 && DL.getLine() != 0) || Flags) {
       // Reinstate the source location but not marked as a statement.
-      const MDNode *Scope = DL.getScope();
-      recordSourceLine(DL.getLine(), DL.getCol(), Scope, Flags);
+      RecordSourceLine(DL, Flags);
     }
     return;
   }
@@ -2136,8 +2145,7 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) {
   if (DL.getLine() && (DL.getLine() != OldLine || ForceIsStmt))
     Flags |= DWARF2_FLAG_IS_STMT;
 
-  const MDNode *Scope = DL.getScope();
-  recordSourceLine(DL.getLine(), DL.getCol(), Scope, Flags);
+  RecordSourceLine(DL, Flags);
 
   // If we're not at line 0, remember this location.
   if (DL.getLine())
@@ -2272,7 +2280,8 @@ findPrologueEndLoc(const MachineFunction *MF) {
 static void recordSourceLine(AsmPrinter &Asm, unsigned Line, unsigned Col,
                              const MDNode *S, unsigned Flags, unsigned CUID,
                              uint16_t DwarfVersion,
-                             ArrayRef<std::unique_ptr<DwarfCompileUnit>> DCUs) {
+                             ArrayRef<std::unique_ptr<DwarfCompileUnit>> DCUs,
+                             StringRef Comment = {}) {
   StringRef Fn;
   unsigned FileNo = 1;
   unsigned Discriminator = 0;
@@ -2286,7 +2295,7 @@ static void recordSourceLine(AsmPrinter &Asm, unsigned Line, unsigned Col,
                  .getOrCreateSourceID(Scope->getFile());
   }
   Asm.OutStreamer->emitDwarfLocDirective(FileNo, Line, Col, Flags, 0,
-                                         Discriminator, Fn);
+                                         Discriminator, Fn, Comment);
 }
 
 const MachineInstr *
@@ -2617,10 +2626,10 @@ void DwarfDebug::endFunctionImpl(const MachineFunction *MF) {
 // Register a source line with debug info. Returns the  unique label that was
 // emitted and which provides correspondence to the source line list.
 void DwarfDebug::recordSourceLine(unsigned Line, unsigned Col, const MDNode *S,
-                                  unsigned Flags) {
+                                  unsigned Flags, StringRef Location) {
   ::recordSourceLine(*Asm, Line, Col, S, Flags,
                      Asm->OutStreamer->getContext().getDwarfCompileUnitID(),
-                     getDwarfVersion(), getUnits());
+                     getDwarfVersion(), getUnits(), Location);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
index 9662c617d730e..58e6d39f76ae0 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
@@ -679,7 +679,7 @@ class DwarfDebug : public DebugHandlerBase {
   /// label that was emitted and which provides correspondence to the
   /// source line list.
   void recordSourceLine(unsigned Line, unsigned Col, const MDNode *Scope,
-                        unsigned Flags);
+                        unsigned Flags, StringRef Location = {});
 
   /// Populate LexicalScope entries with variables' info.
   void collectEntityInfo(DwarfCompileUnit &TheCU, const DISubprogram *SP,
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
index b77ecf1372405..383fbfb3fbd2b 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
@@ -1478,7 +1478,7 @@ void DwarfUnit::constructSubrangeDIE(DIE &DW_Subrange, const DISubrangeType *SR,
 
   AddBoundTypeEntry(dwarf::DW_AT_upper_bound, SR->getUpperBound());
 
-  AddBoundTypeEntry(dwarf::DW_AT_byte_stride, SR->getStride());
+  AddBoundTypeEntry(dwarf::DW_AT_bit_stride, SR->getStride());
 
   AddBoundTypeEntry(dwarf::DW_AT_GNU_bias, SR->getBias());
 }
@@ -1670,7 +1670,7 @@ void DwarfUnit::constructArrayTypeDIE(DIE &Buffer, const DICompositeType *CTy) {
   DINodeArray Elements = CTy->getElements();
   for (DINode *E : Elements) {
     if (auto *Element = dyn_cast_or_null<DISubrangeType>(E)) {
-      DIE &TyDIE = createAndAddDIE(CTy->getTag(), Buffer, CTy);
+      DIE &TyDIE = createAndAddDIE(Element->getTag(), Buffer, CTy);
       constructSubrangeDIE(TyDIE, Element, true);
     } else if (auto *Element = dyn_cast_or_null<DISubrange>(E))
       constructSubrangeDIE(Buffer, Element);
diff --git a/llvm/lib/CodeGen/AsmPrinter/WinException.cpp b/llvm/lib/CodeGen/AsmPrinter/WinException.cpp
index d51ac2ac89f35..062283975851a 100644
--- a/llvm/lib/CodeGen/AsmPrinter/WinException.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/WinException.cpp
@@ -290,6 +290,12 @@ void WinException::endFuncletImpl() {
       // functions that need it in the end anyway.
     }
 
+    if (!MF->getEHContTargets().empty()) {
+      // Copy the function's EH Continuation targets to a module-level list.
+      EHContTargets.insert(EHContTargets.end(), MF->getEHContTargets().begin(),
+                           MF->getEHContTargets().end());
+    }
+
     // Switch back to the funclet start .text section now that we are done
     // writing to .xdata, and emit an .seh_endproc directive to mark the end of
     // the function.
diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp
index beb7fb284a376..b36d2f743d512 100644
--- a/llvm/lib/CodeGen/CodeGen.cpp
+++ b/llvm/lib/CodeGen/CodeGen.cpp
@@ -33,7 +33,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeCodeGenPrepareLegacyPassPass(Registry);
   initializeDeadMachineInstructionElimPass(Registry);
   initializeDebugifyMachineModulePass(Registry);
-  initializeDetectDeadLanesPass(Registry);
+  initializeDetectDeadLanesLegacyPass(Registry);
   initializeDwarfEHPrepareLegacyPassPass(Registry);
   initializeEarlyIfConverterLegacyPass(Registry);
   initializeEarlyIfPredicatorPass(Registry);
@@ -103,7 +103,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeOptimizePHIsLegacyPass(Registry);
   initializePEIPass(Registry);
   initializePHIEliminationPass(Registry);
-  initializePatchableFunctionPass(Registry);
+  initializePatchableFunctionLegacyPass(Registry);
   initializePeepholeOptimizerLegacyPass(Registry);
   initializePostMachineSchedulerLegacyPass(Registry);
   initializePostRAHazardRecognizerPass(Registry);
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 088062afab177..d5fbd4c380746 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -4672,8 +4672,8 @@ class TypePromotionHelper {
   static void addPromotedInst(InstrToOrigTy &PromotedInsts,
                               Instruction *ExtOpnd, bool IsSExt) {
     ExtType ExtTy = IsSExt ? SignExtension : ZeroExtension;
-    InstrToOrigTy::iterator It = PromotedInsts.find(ExtOpnd);
-    if (It != PromotedInsts.end()) {
+    auto [It, Inserted] = PromotedInsts.try_emplace(ExtOpnd);
+    if (!Inserted) {
       // If the new extension is same as original, the information in
       // PromotedInsts[ExtOpnd] is still correct.
       if (It->second.getInt() == ExtTy)
@@ -4684,7 +4684,7 @@ class TypePromotionHelper {
       // BothExtension.
       ExtTy = BothExtension;
     }
-    PromotedInsts[ExtOpnd] = TypeIsSExt(ExtOpnd->getType(), ExtTy);
+    It->second = TypeIsSExt(ExtOpnd->getType(), ExtTy);
   }
 
   /// Utility function to query the original type of instruction \p Opnd
diff --git a/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp b/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp
index ae893176d0784..e8581f632f8ee 100644
--- a/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp
+++ b/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp
@@ -67,9 +67,9 @@ void CriticalAntiDepBreaker::StartBlock(MachineBasicBlock *BB) {
   for (const MachineBasicBlock *Succ : BB->successors())
     for (const auto &LI : Succ->liveins()) {
       for (MCRegAliasIterator AI(LI.PhysReg, TRI, true); AI.isValid(); ++AI) {
-        unsigned Reg = (*AI).id();
-        Classes[Reg] = reinterpret_cast<TargetRegisterClass *>(-1);
-        KillIndices[Reg] = BBSize;
+        MCRegister Reg = *AI;
+        Classes[Reg.id()] = reinterpret_cast<TargetRegisterClass *>(-1);
+        KillIndices[Reg.id()] = BBSize;
         DefIndices[Reg] = ~0u;
       }
     }
@@ -85,10 +85,10 @@ void CriticalAntiDepBreaker::StartBlock(MachineBasicBlock *BB) {
     if (!IsReturnBlock && !Pristine.test(Reg))
       continue;
     for (MCRegAliasIterator AI(*I, TRI, true); AI.isValid(); ++AI) {
-      unsigned Reg = (*AI).id();
-      Classes[Reg] = reinterpret_cast<TargetRegisterClass *>(-1);
-      KillIndices[Reg] = BBSize;
-      DefIndices[Reg] = ~0u;
+      MCRegister Reg = *AI;
+      Classes[Reg.id()] = reinterpret_cast<TargetRegisterClass *>(-1);
+      KillIndices[Reg.id()] = BBSize;
+      DefIndices[Reg.id()] = ~0u;
     }
   }
 }
@@ -182,7 +182,8 @@ void CriticalAntiDepBreaker::PrescanInstruction(MachineInstr &MI) {
     MachineOperand &MO = MI.getOperand(i);
     if (!MO.isReg()) continue;
     Register Reg = MO.getReg();
-    if (Reg == 0) continue;
+    if (!Reg)
+      continue;
     const TargetRegisterClass *NewRC = nullptr;
 
     if (i < MI.getDesc().getNumOperands())
@@ -209,7 +210,7 @@ void CriticalAntiDepBreaker::PrescanInstruction(MachineInstr &MI) {
 
     // If we're still willing to consider this register, note the reference.
     if (Classes[Reg.id()] != reinterpret_cast<TargetRegisterClass *>(-1))
-      RegRefs.insert(std::make_pair(Reg, &MO));
+      RegRefs.emplace(Reg, &MO);
 
     if (MO.isUse() && Special) {
       if (!KeepRegs.test(Reg.id())) {
@@ -278,7 +279,8 @@ void CriticalAntiDepBreaker::ScanInstruction(MachineInstr &MI, unsigned Count) {
 
       if (!MO.isReg()) continue;
       Register Reg = MO.getReg();
-      if (Reg == 0) continue;
+      if (!Reg)
+        continue;
       if (!MO.isDef()) continue;
 
       // Ignore two-addr defs.
@@ -308,7 +310,8 @@ void CriticalAntiDepBreaker::ScanInstruction(MachineInstr &MI, unsigned Count) {
     MachineOperand &MO = MI.getOperand(i);
     if (!MO.isReg()) continue;
     Register Reg = MO.getReg();
-    if (Reg == 0) continue;
+    if (!Reg)
+      continue;
     if (!MO.isUse()) continue;
 
     const TargetRegisterClass *NewRC = nullptr;
@@ -322,15 +325,15 @@ void CriticalAntiDepBreaker::ScanInstruction(MachineInstr &MI, unsigned Count) {
     else if (!NewRC || Classes[Reg.id()] != NewRC)
       Classes[Reg.id()] = reinterpret_cast<TargetRegisterClass *>(-1);
 
-    RegRefs.insert(std::make_pair(Reg, &MO));
+    RegRefs.emplace(Reg, &MO);
 
     // It wasn't previously live but now it is, this is a kill.
     // Repeat for all aliases.
     for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) {
-      unsigned AliasReg = (*AI).id();
-      if (KillIndices[AliasReg] == ~0u) {
-        KillIndices[AliasReg] = Count;
-        DefIndices[AliasReg] = ~0u;
+      MCRegister AliasReg = *AI;
+      if (KillIndices[AliasReg.id()] == ~0u) {
+        KillIndices[AliasReg.id()] = Count;
+        DefIndices[AliasReg.id()] = ~0u;
       }
     }
   }
@@ -347,10 +350,9 @@ void CriticalAntiDepBreaker::ScanInstruction(MachineInstr &MI, unsigned Count) {
 // RegRefs because the def is inserted by PrescanInstruction and not erased
 // during ScanInstruction. So checking for an instruction with definitions of
 // both NewReg and AntiDepReg covers it.
-bool
-CriticalAntiDepBreaker::isNewRegClobberedByRefs(RegRefIter RegRefBegin,
-                                                RegRefIter RegRefEnd,
-                                                unsigned NewReg) {
+bool CriticalAntiDepBreaker::isNewRegClobberedByRefs(RegRefIter RegRefBegin,
+                                                     RegRefIter RegRefEnd,
+                                                     MCRegister NewReg) {
   for (RegRefIter I = RegRefBegin; I != RegRefEnd; ++I ) {
     MachineOperand *RefOper = I->second;
 
@@ -389,15 +391,12 @@ CriticalAntiDepBreaker::isNewRegClobberedByRefs(RegRefIter RegRefBegin,
   return false;
 }
 
-unsigned CriticalAntiDepBreaker::
-findSuitableFreeRegister(RegRefIter RegRefBegin,
-                         RegRefIter RegRefEnd,
-                         unsigned AntiDepReg,
-                         unsigned LastNewReg,
-                         const TargetRegisterClass *RC,
-                         SmallVectorImpl<unsigned> &Forbid) {
+MCRegister CriticalAntiDepBreaker::findSuitableFreeRegister(
+    RegRefIter RegRefBegin, RegRefIter RegRefEnd, MCRegister AntiDepReg,
+    MCRegister LastNewReg, const TargetRegisterClass *RC,
+    const SmallVectorImpl<Register> &Forbid) {
   ArrayRef<MCPhysReg> Order = RegClassInfo.getOrder(RC);
-  for (unsigned NewReg : Order) {
+  for (MCRegister NewReg : Order) {
     // Don't replace a register with itself.
     if (NewReg == AntiDepReg) continue;
     // Don't replace a register with one that was recently used to repair
@@ -410,17 +409,18 @@ findSuitableFreeRegister(RegRefIter RegRefBegin,
     if (isNewRegClobberedByRefs(RegRefBegin, RegRefEnd, NewReg)) continue;
     // If NewReg is dead and NewReg's most recent def is not before
     // AntiDepReg's kill, it's safe to replace AntiDepReg with NewReg.
-    assert(((KillIndices[AntiDepReg] == ~0u) != (DefIndices[AntiDepReg] == ~0u))
-           && "Kill and Def maps aren't consistent for AntiDepReg!");
+    assert(((KillIndices[AntiDepReg.id()] == ~0u) !=
+            (DefIndices[AntiDepReg.id()] == ~0u)) &&
+           "Kill and Def maps aren't consistent for AntiDepReg!");
     assert(((KillIndices[NewReg] == ~0u) != (DefIndices[NewReg] == ~0u))
            && "Kill and Def maps aren't consistent for NewReg!");
     if (KillIndices[NewReg] != ~0u ||
         Classes[NewReg] == reinterpret_cast<TargetRegisterClass *>(-1) ||
-        KillIndices[AntiDepReg] > DefIndices[NewReg])
+        KillIndices[AntiDepReg.id()] > DefIndices[NewReg])
       continue;
     // If NewReg overlaps any of the forbidden registers, we can't use it.
     bool Forbidden = false;
-    for (unsigned R : Forbid)
+    for (Register R : Forbid)
       if (TRI->regsOverlap(NewReg, R)) {
         Forbidden = true;
         break;
@@ -430,7 +430,7 @@ findSuitableFreeRegister(RegRefIter RegRefBegin,
   }
 
   // No registers are free and available!
-  return 0;
+  return MCRegister();
 }
 
 unsigned CriticalAntiDepBreaker::
@@ -517,7 +517,7 @@ BreakAntiDependencies(const std::vector<SUnit> &SUnits,
   // fix that remaining critical edge too. This is a little more involved,
   // because unlike the most recent register, less recent registers should
   // still be considered, though only if no other registers are available.
-  std::vector<unsigned> LastNewReg(TRI->getNumRegs(), 0);
+  std::vector<MCRegister> LastNewReg(TRI->getNumRegs(), MCRegister());
 
   // Attempt to break anti-dependence edges on the critical path. Walk the
   // instructions from the bottom up, tracking information about liveness
@@ -549,22 +549,22 @@ BreakAntiDependencies(const std::vector<SUnit> &SUnits,
     // anti-dependencies. The current code here only knows how to break one
     // edge per instruction. Note that we'd have to be able to break all of
     // the anti-dependencies in an instruction in order to be effective.
-    unsigned AntiDepReg = 0;
+    MCRegister AntiDepReg;
     if (&MI == CriticalPathMI) {
       if (const SDep *Edge = CriticalPathStep(CriticalPathSU)) {
         const SUnit *NextSU = Edge->getSUnit();
 
         // Only consider anti-dependence edges.
         if (Edge->getKind() == SDep::Anti) {
-          AntiDepReg = Edge->getReg();
-          assert(AntiDepReg != 0 && "Anti-dependence on reg0?");
+          AntiDepReg = Edge->getReg().asMCReg();
+          assert(AntiDepReg && "Anti-dependence on reg0?");
           if (!MRI.isAllocatable(AntiDepReg))
             // Don't break anti-dependencies on non-allocatable registers.
-            AntiDepReg = 0;
-          else if (KeepRegs.test(AntiDepReg))
+            AntiDepReg = MCRegister();
+          else if (KeepRegs.test(AntiDepReg.id()))
             // Don't break anti-dependencies if a use down below requires
             // this exact register.
-            AntiDepReg = 0;
+            AntiDepReg = MCRegister();
           else {
             // If the SUnit has other dependencies on the SUnit that it
             // anti-depends on, don't bother breaking the anti-dependency
@@ -579,7 +579,7 @@ BreakAntiDependencies(const std::vector<SUnit> &SUnits,
                       ? (P.getKind() != SDep::Anti || P.getReg() != AntiDepReg)
                       : (P.getKind() == SDep::Data &&
                          P.getReg() == AntiDepReg)) {
-                AntiDepReg = 0;
+                AntiDepReg = MCRegister();
                 break;
               }
           }
@@ -595,7 +595,7 @@ BreakAntiDependencies(const std::vector<SUnit> &SUnits,
 
     PrescanInstruction(MI);
 
-    SmallVector<unsigned, 2> ForbidRegs;
+    SmallVector<Register, 2> ForbidRegs;
 
     // If MI's defs have a special allocation requirement, don't allow
     // any def registers to be changed. Also assume all registers
@@ -603,7 +603,7 @@ BreakAntiDependencies(const std::vector<SUnit> &SUnits,
     if (MI.isCall() || MI.hasExtraDefRegAllocReq() || TII->isPredicated(MI))
       // If this instruction's defs have special allocation requirement, don't
       // break this anti-dependency.
-      AntiDepReg = 0;
+      AntiDepReg = MCRegister();
     else if (AntiDepReg) {
       // If this instruction has a use of AntiDepReg, breaking it
       // is invalid.  If the instruction defines other registers,
@@ -612,9 +612,10 @@ BreakAntiDependencies(const std::vector<SUnit> &SUnits,
       for (const MachineOperand &MO : MI.operands()) {
         if (!MO.isReg()) continue;
         Register Reg = MO.getReg();
-        if (Reg == 0) continue;
+        if (!Reg)
+          continue;
         if (MO.isUse() && TRI->regsOverlap(AntiDepReg, Reg)) {
-          AntiDepReg = 0;
+          AntiDepReg = MCRegister();
           break;
         }
         if (MO.isDef() && Reg != AntiDepReg)
@@ -624,25 +625,24 @@ BreakAntiDependencies(const std::vector<SUnit> &SUnits,
 
     // Determine AntiDepReg's register class, if it is live and is
     // consistently used within a single class.
-    const TargetRegisterClass *RC = AntiDepReg != 0 ? Classes[AntiDepReg]
-                                                    : nullptr;
-    assert((AntiDepReg == 0 || RC != nullptr) &&
+    const TargetRegisterClass *RC =
+        AntiDepReg ? Classes[AntiDepReg.id()] : nullptr;
+    assert((!AntiDepReg || RC != nullptr) &&
            "Register should be live if it's causing an anti-dependence!");
     if (RC == reinterpret_cast<TargetRegisterClass *>(-1))
-      AntiDepReg = 0;
+      AntiDepReg = MCRegister();
 
     // Look for a suitable register to use to break the anti-dependence.
     //
     // TODO: Instead of picking the first free register, consider which might
     // be the best.
-    if (AntiDepReg != 0) {
-      std::pair<std::multimap<unsigned, MachineOperand *>::iterator,
-                std::multimap<unsigned, MachineOperand *>::iterator>
-        Range = RegRefs.equal_range(AntiDepReg);
-      if (unsigned NewReg = findSuitableFreeRegister(Range.first, Range.second,
-                                                     AntiDepReg,
-                                                     LastNewReg[AntiDepReg],
-                                                     RC, ForbidRegs)) {
+    if (AntiDepReg) {
+      std::pair<std::multimap<MCRegister, MachineOperand *>::iterator,
+                std::multimap<MCRegister, MachineOperand *>::iterator>
+          Range = RegRefs.equal_range(AntiDepReg);
+      if (MCRegister NewReg = findSuitableFreeRegister(
+              Range.first, Range.second, AntiDepReg, LastNewReg[AntiDepReg], RC,
+              ForbidRegs)) {
         LLVM_DEBUG(dbgs() << "Breaking anti-dependence edge on "
                           << printReg(AntiDepReg, TRI) << " with "
                           << RegRefs.count(AntiDepReg) << " references"
@@ -650,8 +650,10 @@ BreakAntiDependencies(const std::vector<SUnit> &SUnits,
 
         // Update the references to the old register to refer to the new
         // register.
-        for (std::multimap<unsigned, MachineOperand *>::iterator
-             Q = Range.first, QE = Range.second; Q != QE; ++Q) {
+        for (std::multimap<MCRegister, MachineOperand *>::iterator
+                 Q = Range.first,
+                 QE = Range.second;
+             Q != QE; ++Q) {
           Q->second->setReg(NewReg);
           // If the SU for the instruction being updated has debug information
           // related to the anti-dependency register, make sure to update that
@@ -665,22 +667,22 @@ BreakAntiDependencies(const std::vector<SUnit> &SUnits,
         // We just went back in time and modified history; the
         // liveness information for the anti-dependence reg is now
         // inconsistent. Set the state as if it were dead.
-        Classes[NewReg] = Classes[AntiDepReg];
-        DefIndices[NewReg] = DefIndices[AntiDepReg];
-        KillIndices[NewReg] = KillIndices[AntiDepReg];
-        assert(((KillIndices[NewReg] == ~0u) !=
-                (DefIndices[NewReg] == ~0u)) &&
-             "Kill and Def maps aren't consistent for NewReg!");
-
-        Classes[AntiDepReg] = nullptr;
-        DefIndices[AntiDepReg] = KillIndices[AntiDepReg];
-        KillIndices[AntiDepReg] = ~0u;
-        assert(((KillIndices[AntiDepReg] == ~0u) !=
-                (DefIndices[AntiDepReg] == ~0u)) &&
-             "Kill and Def maps aren't consistent for AntiDepReg!");
+        Classes[NewReg.id()] = Classes[AntiDepReg.id()];
+        DefIndices[NewReg.id()] = DefIndices[AntiDepReg.id()];
+        KillIndices[NewReg.id()] = KillIndices[AntiDepReg.id()];
+        assert(((KillIndices[NewReg.id()] == ~0u) !=
+                (DefIndices[NewReg.id()] == ~0u)) &&
+               "Kill and Def maps aren't consistent for NewReg!");
+
+        Classes[AntiDepReg.id()] = nullptr;
+        DefIndices[AntiDepReg.id()] = KillIndices[AntiDepReg.id()];
+        KillIndices[AntiDepReg.id()] = ~0u;
+        assert(((KillIndices[AntiDepReg.id()] == ~0u) !=
+                (DefIndices[AntiDepReg.id()] == ~0u)) &&
+               "Kill and Def maps aren't consistent for AntiDepReg!");
 
         RegRefs.erase(AntiDepReg);
-        LastNewReg[AntiDepReg] = NewReg;
+        LastNewReg[AntiDepReg.id()] = NewReg;
         ++Broken;
       }
     }
diff --git a/llvm/lib/CodeGen/CriticalAntiDepBreaker.h b/llvm/lib/CodeGen/CriticalAntiDepBreaker.h
index 640506b6e9ed4..a96e2ed8cf1a3 100644
--- a/llvm/lib/CodeGen/CriticalAntiDepBreaker.h
+++ b/llvm/lib/CodeGen/CriticalAntiDepBreaker.h
@@ -53,10 +53,10 @@ class LLVM_LIBRARY_VISIBILITY CriticalAntiDepBreaker : public AntiDepBreaker {
     std::vector<const TargetRegisterClass *> Classes;
 
     /// Map registers to all their references within a live range.
-    std::multimap<unsigned, MachineOperand *> RegRefs;
+    std::multimap<MCRegister, MachineOperand *> RegRefs;
 
     using RegRefIter =
-        std::multimap<unsigned, MachineOperand *>::const_iterator;
+        std::multimap<MCRegister, MachineOperand *>::const_iterator;
 
     /// The index of the most recent kill (proceeding bottom-up),
     /// or ~0u if the register is not live.
@@ -96,15 +96,13 @@ class LLVM_LIBRARY_VISIBILITY CriticalAntiDepBreaker : public AntiDepBreaker {
   private:
     void PrescanInstruction(MachineInstr &MI);
     void ScanInstruction(MachineInstr &MI, unsigned Count);
-    bool isNewRegClobberedByRefs(RegRefIter RegRefBegin,
-                                 RegRefIter RegRefEnd,
-                                 unsigned NewReg);
-    unsigned findSuitableFreeRegister(RegRefIter RegRefBegin,
-                                      RegRefIter RegRefEnd,
-                                      unsigned AntiDepReg,
-                                      unsigned LastNewReg,
-                                      const TargetRegisterClass *RC,
-                                      SmallVectorImpl<unsigned> &Forbid);
+    bool isNewRegClobberedByRefs(RegRefIter RegRefBegin, RegRefIter RegRefEnd,
+                                 MCRegister NewReg);
+    MCRegister
+    findSuitableFreeRegister(RegRefIter RegRefBegin, RegRefIter RegRefEnd,
+                             MCRegister AntiDepReg, MCRegister LastNewReg,
+                             const TargetRegisterClass *RC,
+                             const SmallVectorImpl<Register> &Forbid);
   };
 
 } // end namespace llvm
diff --git a/llvm/lib/CodeGen/DetectDeadLanes.cpp b/llvm/lib/CodeGen/DetectDeadLanes.cpp
index 01da473268706..d2522fd5eb0e3 100644
--- a/llvm/lib/CodeGen/DetectDeadLanes.cpp
+++ b/llvm/lib/CodeGen/DetectDeadLanes.cpp
@@ -373,19 +373,9 @@ LaneBitmask DeadLaneDetector::determineInitialUsedLanes(Register Reg) {
 
 namespace {
 
-class DetectDeadLanes : public MachineFunctionPass {
+class DetectDeadLanes {
 public:
-  bool runOnMachineFunction(MachineFunction &MF) override;
-
-  static char ID;
-  DetectDeadLanes() : MachineFunctionPass(ID) {}
-
-  StringRef getPassName() const override { return "Detect Dead Lanes"; }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesCFG();
-    MachineFunctionPass::getAnalysisUsage(AU);
-  }
+  bool run(MachineFunction &MF);
 
 private:
   /// update the operand status.
@@ -407,12 +397,29 @@ class DetectDeadLanes : public MachineFunctionPass {
   const TargetRegisterInfo *TRI = nullptr;
 };
 
+struct DetectDeadLanesLegacy : public MachineFunctionPass {
+  static char ID;
+  DetectDeadLanesLegacy() : MachineFunctionPass(ID) {}
+
+  StringRef getPassName() const override { return "Detect Dead Lanes"; }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override {
+    return DetectDeadLanes().run(MF);
+  }
+};
+
 } // end anonymous namespace
 
-char DetectDeadLanes::ID = 0;
-char &llvm::DetectDeadLanesID = DetectDeadLanes::ID;
+char DetectDeadLanesLegacy::ID = 0;
+char &llvm::DetectDeadLanesID = DetectDeadLanesLegacy::ID;
 
-INITIALIZE_PASS(DetectDeadLanes, DEBUG_TYPE, "Detect Dead Lanes", false, false)
+INITIALIZE_PASS(DetectDeadLanesLegacy, DEBUG_TYPE, "Detect Dead Lanes", false,
+                false)
 
 bool DetectDeadLanes::isUndefRegAtInput(
     const MachineOperand &MO, const DeadLaneDetector::VRegInfo &RegInfo) const {
@@ -537,7 +544,17 @@ DetectDeadLanes::modifySubRegisterOperandStatus(const DeadLaneDetector &DLD,
   return std::make_pair(Changed, Again);
 }
 
-bool DetectDeadLanes::runOnMachineFunction(MachineFunction &MF) {
+PreservedAnalyses
+DetectDeadLanesPass::run(MachineFunction &MF,
+                         MachineFunctionAnalysisManager &MFAM) {
+  if (!DetectDeadLanes().run(MF))
+    return PreservedAnalyses::all();
+  auto PA = getMachineFunctionPassPreservedAnalyses();
+  PA.preserveSet<CFGAnalyses>();
+  return PA;
+}
+
+bool DetectDeadLanes::run(MachineFunction &MF) {
   // Don't bother if we won't track subregister liveness later.  This pass is
   // required for correctness if subregister liveness is enabled because the
   // register coalescer cannot deal with hidden dead defs. However without
diff --git a/llvm/lib/CodeGen/ImplicitNullChecks.cpp b/llvm/lib/CodeGen/ImplicitNullChecks.cpp
index c586e07c7d042..0e0ffa51d647b 100644
--- a/llvm/lib/CodeGen/ImplicitNullChecks.cpp
+++ b/llvm/lib/CodeGen/ImplicitNullChecks.cpp
@@ -195,7 +195,7 @@ class ImplicitNullChecks : public MachineFunctionPass {
   /// to be used. \p PrevInsts is the set of instruction seen since
   /// the explicit null check on \p PointerReg.
   SuitabilityResult isSuitableMemoryOp(const MachineInstr &MI,
-                                       unsigned PointerReg,
+                                       Register PointerReg,
                                        ArrayRef<MachineInstr *> PrevInsts);
 
   /// Returns true if \p DependenceMI can clobber the liveIns in NullSucc block
@@ -317,7 +317,7 @@ bool ImplicitNullChecks::runOnMachineFunction(MachineFunction &MF) {
 
 // Return true if any register aliasing \p Reg is live-in into \p MBB.
 static bool AnyAliasLiveIn(const TargetRegisterInfo *TRI,
-                           MachineBasicBlock *MBB, unsigned Reg) {
+                           MachineBasicBlock *MBB, Register Reg) {
   for (MCRegAliasIterator AR(Reg, TRI, /*IncludeSelf*/ true); AR.isValid();
        ++AR)
     if (MBB->isLiveIn(*AR))
@@ -362,7 +362,7 @@ ImplicitNullChecks::areMemoryOpsAliased(const MachineInstr &MI,
 
 ImplicitNullChecks::SuitabilityResult
 ImplicitNullChecks::isSuitableMemoryOp(const MachineInstr &MI,
-                                       unsigned PointerReg,
+                                       Register PointerReg,
                                        ArrayRef<MachineInstr *> PrevInsts) {
   // Implementation restriction for faulting_op insertion
   // TODO: This could be relaxed if we find a test case which warrants it.
@@ -706,14 +706,11 @@ bool ImplicitNullChecks::analyzeBlockForNullChecks(
 /// faults.  The FAULTING instruction is inserted at the end of MBB.
 MachineInstr *ImplicitNullChecks::insertFaultingInstr(
     MachineInstr *MI, MachineBasicBlock *MBB, MachineBasicBlock *HandlerMBB) {
-  const unsigned NoRegister = 0; // Guaranteed to be the NoRegister value for
-                                 // all targets.
-
   DebugLoc DL;
   unsigned NumDefs = MI->getDesc().getNumDefs();
   assert(NumDefs <= 1 && "other cases unhandled!");
 
-  unsigned DefReg = NoRegister;
+  Register DefReg;
   if (NumDefs != 0) {
     DefReg = MI->getOperand(0).getReg();
     assert(NumDefs == 1 && "expected exactly one def!");
diff --git a/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp b/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
index 49c8a0e466337..b888b81e92b92 100644
--- a/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
+++ b/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
@@ -20,7 +20,7 @@ static cl::opt<bool>
                        cl::Hidden,
                        cl::desc("Use Stable Hashing for MIR VReg Renaming"));
 
-using VRegRenameMap = std::map<unsigned, unsigned>;
+using VRegRenameMap = std::map<Register, Register>;
 
 bool VRegRenamer::doVRegRenaming(const VRegRenameMap &VRM) {
   bool Changed = false;
@@ -45,7 +45,7 @@ VRegRenamer::getVRegRenameMap(const std::vector<NamedVReg> &VRegs) {
 
   VRegRenameMap VRM;
   for (const auto &VReg : VRegs) {
-    const unsigned Reg = VReg.getReg();
+    const Register Reg = VReg.getReg();
     VRM[Reg] = createVirtualRegisterWithLowerName(Reg, GetUniqueVRegName(VReg));
   }
   return VRM;
@@ -77,7 +77,7 @@ std::string VRegRenamer::getInstructionOpcodeHash(MachineInstr &MI) {
     case MachineOperand::MO_Register:
       if (MO.getReg().isVirtual())
         return MRI.getVRegDef(MO.getReg())->getOpcode();
-      return MO.getReg();
+      return MO.getReg().id();
     case MachineOperand::MO_Immediate:
       return MO.getImm();
     case MachineOperand::MO_TargetIndex:
@@ -136,8 +136,8 @@ std::string VRegRenamer::getInstructionOpcodeHash(MachineInstr &MI) {
   return OS.str();
 }
 
-unsigned VRegRenamer::createVirtualRegister(unsigned VReg) {
-  assert(Register::isVirtualRegister(VReg) && "Expected Virtual Registers");
+Register VRegRenamer::createVirtualRegister(Register VReg) {
+  assert(VReg.isVirtual() && "Expected Virtual Registers");
   std::string Name = getInstructionOpcodeHash(*MRI.getVRegDef(VReg));
   return createVirtualRegisterWithLowerName(VReg, Name);
 }
@@ -160,10 +160,10 @@ bool VRegRenamer::renameInstsInMBB(MachineBasicBlock *MBB) {
         NamedVReg(MO.getReg(), Prefix + getInstructionOpcodeHash(Candidate)));
   }
 
-  return VRegs.size() ? doVRegRenaming(getVRegRenameMap(VRegs)) : false;
+  return !VRegs.empty() ? doVRegRenaming(getVRegRenameMap(VRegs)) : false;
 }
 
-unsigned VRegRenamer::createVirtualRegisterWithLowerName(unsigned VReg,
+Register VRegRenamer::createVirtualRegisterWithLowerName(Register VReg,
                                                          StringRef Name) {
   std::string LowerName = Name.lower();
   const TargetRegisterClass *RC = MRI.getRegClassOrNull(VReg);
diff --git a/llvm/lib/CodeGen/MIRVRegNamerUtils.h b/llvm/lib/CodeGen/MIRVRegNamerUtils.h
index a059bc5333c65..055bb9576c048 100644
--- a/llvm/lib/CodeGen/MIRVRegNamerUtils.h
+++ b/llvm/lib/CodeGen/MIRVRegNamerUtils.h
@@ -61,19 +61,19 @@ class VRegRenamer {
 
   /// For all the VRegs that are candidates for renaming,
   /// return a mapping from old vregs to new vregs with names.
-  std::map<unsigned, unsigned>
+  std::map<Register, Register>
   getVRegRenameMap(const std::vector<NamedVReg> &VRegs);
 
   /// Perform replacing of registers based on the <old,new> vreg map.
-  bool doVRegRenaming(const std::map<unsigned, unsigned> &VRegRenameMap);
+  bool doVRegRenaming(const std::map<Register, Register> &VRegRenameMap);
 
   /// createVirtualRegister - Given an existing vreg, create a named vreg to
   /// take its place. The name is determined by calling
   /// getInstructionOpcodeHash.
-  unsigned createVirtualRegister(unsigned VReg);
+  Register createVirtualRegister(Register VReg);
 
   /// Create a vreg with name and return it.
-  unsigned createVirtualRegisterWithLowerName(unsigned VReg, StringRef Name);
+  Register createVirtualRegisterWithLowerName(Register VReg, StringRef Name);
 
   /// Linearly traverse the MachineBasicBlock and rename each instruction's
   /// vreg definition based on the semantics of the instruction.
diff --git a/llvm/lib/CodeGen/MachineCopyPropagation.cpp b/llvm/lib/CodeGen/MachineCopyPropagation.cpp
index 1105b8c15515f..5102639a3a30a 100644
--- a/llvm/lib/CodeGen/MachineCopyPropagation.cpp
+++ b/llvm/lib/CodeGen/MachineCopyPropagation.cpp
@@ -971,6 +971,19 @@ void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) {
 
     forwardUses(MI);
 
+    // It's possible that the previous transformation has resulted in a no-op
+    // register move (i.e. one where source and destination registers are the
+    // same and are not referring to a reserved register). If so, delete it.
+    CopyOperands = isCopyInstr(MI, *TII, UseCopyInstr);
+    if (CopyOperands &&
+        CopyOperands->Source->getReg() == CopyOperands->Destination->getReg() &&
+        !MRI->isReserved(CopyOperands->Source->getReg())) {
+      MI.eraseFromParent();
+      NumDeletes++;
+      Changed = true;
+      continue;
+    }
+
     // Not a copy.
     SmallVector<Register, 4> Defs;
     const MachineOperand *RegMask = nullptr;
diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp
index a34b204fed6dc..19e8b44b69b01 100644
--- a/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -710,7 +710,7 @@ void SwingSchedulerDAG::schedule() {
       Stages[SU->getInstr()] = Schedule.stageScheduled(SU);
     }
   }
-  DenseMap<MachineInstr *, std::pair<unsigned, int64_t>> NewInstrChanges;
+  DenseMap<MachineInstr *, std::pair<Register, int64_t>> NewInstrChanges;
   for (auto &KV : NewMIs) {
     Cycles[KV.first] = Cycles[KV.second];
     Stages[KV.first] = Stages[KV.second];
@@ -756,27 +756,27 @@ void SwingSchedulerDAG::finishBlock() {
 /// Return the register values for  the operands of a Phi instruction.
 /// This function assume the instruction is a Phi.
 static void getPhiRegs(MachineInstr &Phi, MachineBasicBlock *Loop,
-                       unsigned &InitVal, unsigned &LoopVal) {
+                       Register &InitVal, Register &LoopVal) {
   assert(Phi.isPHI() && "Expecting a Phi.");
 
-  InitVal = 0;
-  LoopVal = 0;
+  InitVal = Register();
+  LoopVal = Register();
   for (unsigned i = 1, e = Phi.getNumOperands(); i != e; i += 2)
     if (Phi.getOperand(i + 1).getMBB() != Loop)
       InitVal = Phi.getOperand(i).getReg();
     else
       LoopVal = Phi.getOperand(i).getReg();
 
-  assert(InitVal != 0 && LoopVal != 0 && "Unexpected Phi structure.");
+  assert(InitVal && LoopVal && "Unexpected Phi structure.");
 }
 
 /// Return the Phi register value that comes the loop block.
-static unsigned getLoopPhiReg(const MachineInstr &Phi,
+static Register getLoopPhiReg(const MachineInstr &Phi,
                               const MachineBasicBlock *LoopBB) {
   for (unsigned i = 1, e = Phi.getNumOperands(); i != e; i += 2)
     if (Phi.getOperand(i + 1).getMBB() == LoopBB)
       return Phi.getOperand(i).getReg();
-  return 0;
+  return Register();
 }
 
 /// Return true if SUb can be reached from SUa following the chain edges.
@@ -937,8 +937,8 @@ void SwingSchedulerDAG::updatePhiDependences() {
   for (SUnit &I : SUnits) {
     RemoveDeps.clear();
     // Set to true if the instruction has an operand defined by a Phi.
-    unsigned HasPhiUse = 0;
-    unsigned HasPhiDef = 0;
+    Register HasPhiUse;
+    Register HasPhiDef;
     MachineInstr *MI = I.getInstr();
     // Iterate over each operand, and we process the definitions.
     for (const MachineOperand &MO : MI->operands()) {
@@ -1017,7 +1017,8 @@ void SwingSchedulerDAG::changeDependences() {
   // If so, we update the base and offset of the instruction and change
   // the dependences.
   for (SUnit &I : SUnits) {
-    unsigned BasePos = 0, OffsetPos = 0, NewBase = 0;
+    unsigned BasePos = 0, OffsetPos = 0;
+    Register NewBase;
     int64_t NewOffset = 0;
     if (!canUseLastOffsetValue(I.getInstr(), BasePos, OffsetPos, NewBase,
                                NewOffset))
@@ -1982,7 +1983,7 @@ static void computeLiveOuts(MachineFunction &MF, RegPressureTracker &RPTracker,
   const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
   MachineRegisterInfo &MRI = MF.getRegInfo();
   SmallVector<VRegMaskOrUnit, 8> LiveOutRegs;
-  SmallSet<unsigned, 4> Uses;
+  SmallSet<Register, 4> Uses;
   for (SUnit *SU : NS) {
     const MachineInstr *MI = SU->getInstr();
     if (MI->isPHI())
@@ -2646,7 +2647,7 @@ bool SwingSchedulerDAG::computeDelta(const MachineInstr &MI, int &Delta) const {
 bool SwingSchedulerDAG::canUseLastOffsetValue(MachineInstr *MI,
                                               unsigned &BasePos,
                                               unsigned &OffsetPos,
-                                              unsigned &NewBase,
+                                              Register &NewBase,
                                               int64_t &Offset) {
   // Get the load instruction.
   if (TII->isPostIncrement(*MI))
@@ -2662,7 +2663,7 @@ bool SwingSchedulerDAG::canUseLastOffsetValue(MachineInstr *MI,
   if (!Phi || !Phi->isPHI())
     return false;
   // Get the register defined in the loop block.
-  unsigned PrevReg = getLoopPhiReg(*Phi, MI->getParent());
+  Register PrevReg = getLoopPhiReg(*Phi, MI->getParent());
   if (!PrevReg)
     return false;
 
@@ -2702,10 +2703,10 @@ bool SwingSchedulerDAG::canUseLastOffsetValue(MachineInstr *MI,
 void SwingSchedulerDAG::applyInstrChange(MachineInstr *MI,
                                          SMSchedule &Schedule) {
   SUnit *SU = getSUnit(MI);
-  DenseMap<SUnit *, std::pair<unsigned, int64_t>>::iterator It =
+  DenseMap<SUnit *, std::pair<Register, int64_t>>::iterator It =
       InstrChanges.find(SU);
   if (It != InstrChanges.end()) {
-    std::pair<unsigned, int64_t> RegAndOffset = It->second;
+    std::pair<Register, int64_t> RegAndOffset = It->second;
     unsigned BasePos, OffsetPos;
     if (!TII->getBaseAndOffsetPosition(*MI, BasePos, OffsetPos))
       return;
@@ -2789,10 +2790,10 @@ bool SwingSchedulerDAG::mayOverlapInLaterIter(
     if (!DefB || !DefO || !DefB->isPHI() || !DefO->isPHI())
       return true;
 
-    unsigned InitValB = 0;
-    unsigned LoopValB = 0;
-    unsigned InitValO = 0;
-    unsigned LoopValO = 0;
+    Register InitValB;
+    Register LoopValB;
+    Register InitValO;
+    Register LoopValO;
     getPhiRegs(*DefB, BB, InitValB, LoopValB);
     getPhiRegs(*DefO, BB, InitValO, LoopValO);
     MachineInstr *InitDefB = MRI.getVRegDef(InitValB);
@@ -3062,7 +3063,7 @@ void SMSchedule::orderDependence(const SwingSchedulerDAG *SSD, SUnit *SU,
       unsigned BasePos, OffsetPos;
       if (ST.getInstrInfo()->getBaseAndOffsetPosition(*MI, BasePos, OffsetPos))
         if (MI->getOperand(BasePos).getReg() == Reg)
-          if (unsigned NewReg = SSD->getInstrBaseReg(SU))
+          if (Register NewReg = SSD->getInstrBaseReg(SU))
             Reg = NewReg;
       bool Reads, Writes;
       std::tie(Reads, Writes) =
@@ -3180,8 +3181,8 @@ bool SMSchedule::isLoopCarried(const SwingSchedulerDAG *SSD,
   unsigned DefCycle = cycleScheduled(DefSU);
   int DefStage = stageScheduled(DefSU);
 
-  unsigned InitVal = 0;
-  unsigned LoopVal = 0;
+  Register InitVal;
+  Register LoopVal;
   getPhiRegs(Phi, Phi.getParent(), InitVal, LoopVal);
   SUnit *UseSU = SSD->getSUnit(MRI.getVRegDef(LoopVal));
   if (!UseSU)
@@ -3212,7 +3213,7 @@ bool SMSchedule::isLoopCarriedDefOfUse(const SwingSchedulerDAG *SSD,
     return false;
   if (!isLoopCarried(SSD, *Phi))
     return false;
-  unsigned LoopReg = getLoopPhiReg(*Phi, Phi->getParent());
+  Register LoopReg = getLoopPhiReg(*Phi, Phi->getParent());
   for (MachineOperand &DMO : Def->all_defs()) {
     if (DMO.getReg() == LoopReg)
       return true;
@@ -3434,8 +3435,8 @@ void SwingSchedulerDAG::checkValidNodeOrder(const NodeSetType &Circuits) const {
 /// In this case p and p' overlap, which means that two registers are needed.
 /// Instead, this function changes the load to use p' and updates the offset.
 void SwingSchedulerDAG::fixupRegisterOverlaps(std::deque<SUnit *> &Instrs) {
-  unsigned OverlapReg = 0;
-  unsigned NewBaseReg = 0;
+  Register OverlapReg;
+  Register NewBaseReg;
   for (SUnit *SU : Instrs) {
     MachineInstr *MI = SU->getInstr();
     for (unsigned i = 0, e = MI->getNumOperands(); i < e; ++i) {
@@ -3445,8 +3446,8 @@ void SwingSchedulerDAG::fixupRegisterOverlaps(std::deque<SUnit *> &Instrs) {
       if (MO.isReg() && MO.isUse() && MO.getReg() == OverlapReg) {
         // Check that the instruction appears in the InstrChanges structure,
         // which contains instructions that can have the offset updated.
-        DenseMap<SUnit *, std::pair<unsigned, int64_t>>::iterator It =
-          InstrChanges.find(SU);
+        DenseMap<SUnit *, std::pair<Register, int64_t>>::iterator It =
+            InstrChanges.find(SU);
         if (It != InstrChanges.end()) {
           unsigned BasePos, OffsetPos;
           // Update the base register and adjust the offset.
@@ -3461,8 +3462,8 @@ void SwingSchedulerDAG::fixupRegisterOverlaps(std::deque<SUnit *> &Instrs) {
             NewMIs[MI] = NewMI;
           }
         }
-        OverlapReg = 0;
-        NewBaseReg = 0;
+        OverlapReg = Register();
+        NewBaseReg = Register();
         break;
       }
       // Look for an instruction of the form p' = op(p), which uses and defines
diff --git a/llvm/lib/CodeGen/MachineTraceMetrics.cpp b/llvm/lib/CodeGen/MachineTraceMetrics.cpp
index ffe52d6d60b7c..c2d4aa087d32d 100644
--- a/llvm/lib/CodeGen/MachineTraceMetrics.cpp
+++ b/llvm/lib/CodeGen/MachineTraceMetrics.cpp
@@ -1071,7 +1071,7 @@ computeInstrHeights(const MachineBasicBlock *MBB) {
       } else {
         // For register units, the def latency is not included because we don't
         // know the def yet.
-        RegUnits[LI.Reg].Cycle = LI.Height;
+        RegUnits[LI.Reg.id()].Cycle = LI.Height;
       }
     }
   }
diff --git a/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp b/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
index b5dc487e7cb0e..8c95dc71d4e21 100644
--- a/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
+++ b/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp
@@ -117,12 +117,12 @@ void llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::
     if (!Op.getReg().isVirtual())
       continue;
     auto Reg = Op.getReg();
-    if (isDivergent(Reg))
-      continue;
     for (MachineInstr &UserInstr : RegInfo.use_instructions(Reg)) {
       if (DefCycle.contains(UserInstr.getParent()))
         continue;
       markDivergent(UserInstr);
+
+      recordTemporalDivergence(Reg, &UserInstr, &DefCycle);
     }
   }
 }
@@ -193,7 +193,7 @@ INITIALIZE_PASS_END(MachineUniformityAnalysisPass, "machine-uniformity",
 
 void MachineUniformityAnalysisPass::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesAll();
-  AU.addRequired<MachineCycleInfoWrapperPass>();
+  AU.addRequiredTransitive<MachineCycleInfoWrapperPass>();
   AU.addRequired<MachineDominatorTreeWrapperPass>();
   MachineFunctionPass::getAnalysisUsage(AU);
 }
diff --git a/llvm/lib/CodeGen/ModuloSchedule.cpp b/llvm/lib/CodeGen/ModuloSchedule.cpp
index f9fe812f7e65c..bff0b7af989bc 100644
--- a/llvm/lib/CodeGen/ModuloSchedule.cpp
+++ b/llvm/lib/CodeGen/ModuloSchedule.cpp
@@ -38,34 +38,34 @@ void ModuloSchedule::print(raw_ostream &OS) {
 /// Return the register values for  the operands of a Phi instruction.
 /// This function assume the instruction is a Phi.
 static void getPhiRegs(MachineInstr &Phi, MachineBasicBlock *Loop,
-                       unsigned &InitVal, unsigned &LoopVal) {
+                       Register &InitVal, Register &LoopVal) {
   assert(Phi.isPHI() && "Expecting a Phi.");
 
-  InitVal = 0;
-  LoopVal = 0;
+  InitVal = Register();
+  LoopVal = Register();
   for (unsigned i = 1, e = Phi.getNumOperands(); i != e; i += 2)
     if (Phi.getOperand(i + 1).getMBB() != Loop)
       InitVal = Phi.getOperand(i).getReg();
     else
       LoopVal = Phi.getOperand(i).getReg();
 
-  assert(InitVal != 0 && LoopVal != 0 && "Unexpected Phi structure.");
+  assert(InitVal && LoopVal && "Unexpected Phi structure.");
 }
 
 /// Return the Phi register value that comes from the incoming block.
-static unsigned getInitPhiReg(MachineInstr &Phi, MachineBasicBlock *LoopBB) {
+static Register getInitPhiReg(MachineInstr &Phi, MachineBasicBlock *LoopBB) {
   for (unsigned i = 1, e = Phi.getNumOperands(); i != e; i += 2)
     if (Phi.getOperand(i + 1).getMBB() != LoopBB)
       return Phi.getOperand(i).getReg();
-  return 0;
+  return Register();
 }
 
 /// Return the Phi register value that comes the loop block.
-static unsigned getLoopPhiReg(MachineInstr &Phi, MachineBasicBlock *LoopBB) {
+static Register getLoopPhiReg(MachineInstr &Phi, MachineBasicBlock *LoopBB) {
   for (unsigned i = 1, e = Phi.getNumOperands(); i != e; i += 2)
     if (Phi.getOperand(i + 1).getMBB() == LoopBB)
       return Phi.getOperand(i).getReg();
-  return 0;
+  return Register();
 }
 
 void ModuloScheduleExpander::expand() {
@@ -341,7 +341,7 @@ void ModuloScheduleExpander::generateEpilog(
 
 /// Replace all uses of FromReg that appear outside the specified
 /// basic block with ToReg.
-static void replaceRegUsesAfterLoop(unsigned FromReg, unsigned ToReg,
+static void replaceRegUsesAfterLoop(Register FromReg, Register ToReg,
                                     MachineBasicBlock *MBB,
                                     MachineRegisterInfo &MRI,
                                     LiveIntervals &LIS) {
@@ -355,7 +355,7 @@ static void replaceRegUsesAfterLoop(unsigned FromReg, unsigned ToReg,
 
 /// Return true if the register has a use that occurs outside the
 /// specified loop.
-static bool hasUseAfterLoop(unsigned Reg, MachineBasicBlock *BB,
+static bool hasUseAfterLoop(Register Reg, MachineBasicBlock *BB,
                             MachineRegisterInfo &MRI) {
   for (const MachineOperand &MO : MRI.use_operands(Reg))
     if (MO.getParent()->getParent() != BB)
@@ -389,14 +389,14 @@ void ModuloScheduleExpander::generateExistingPhis(
        BBI != BBE; ++BBI) {
     Register Def = BBI->getOperand(0).getReg();
 
-    unsigned InitVal = 0;
-    unsigned LoopVal = 0;
+    Register InitVal;
+    Register LoopVal;
     getPhiRegs(*BBI, BB, InitVal, LoopVal);
 
-    unsigned PhiOp1 = 0;
+    Register PhiOp1;
     // The Phi value from the loop body typically is defined in the loop, but
     // not always. So, we need to check if the value is defined in the loop.
-    unsigned PhiOp2 = LoopVal;
+    Register PhiOp2 = LoopVal;
     if (auto It = VRMap[LastStageNum].find(LoopVal);
         It != VRMap[LastStageNum].end())
       PhiOp2 = It->second;
@@ -407,11 +407,14 @@ void ModuloScheduleExpander::generateExistingPhis(
     if (NumStages == 0) {
       // We don't need to generate a Phi anymore, but we need to rename any uses
       // of the Phi value.
-      unsigned NewReg = VRMap[PrevStage][LoopVal];
+      Register NewReg = VRMap[PrevStage][LoopVal];
       rewriteScheduledInstr(NewBB, InstrMap, CurStageNum, 0, &*BBI, Def,
                             InitVal, NewReg);
-      if (VRMap[CurStageNum].count(LoopVal))
-        VRMap[CurStageNum][Def] = VRMap[CurStageNum][LoopVal];
+      auto It = VRMap[CurStageNum].find(LoopVal);
+      if (It != VRMap[CurStageNum].end()) {
+        Register Reg = It->second;
+        VRMap[CurStageNum][Def] = Reg;
+      }
     }
     // Adjust the number of Phis needed depending on the number of prologs left,
     // and the distance from where the Phi is first scheduled. The number of
@@ -422,7 +425,7 @@ void ModuloScheduleExpander::generateExistingPhis(
       MaxPhis = std::max((int)MaxPhis - (int)LoopValStage, 1);
     unsigned NumPhis = std::min(NumStages, MaxPhis);
 
-    unsigned NewReg = 0;
+    Register NewReg;
     unsigned AccessStage = (LoopValStage != -1) ? LoopValStage : StageScheduled;
     // In the epilog, we may need to look back one stage to get the correct
     // Phi name, because the epilog and prolog blocks execute the same stage.
@@ -464,10 +467,12 @@ void ModuloScheduleExpander::generateExistingPhis(
           InstOp1 = MRI.getVRegDef(PhiOp1);
           int PhiOpStage = Schedule.getStage(InstOp1);
           int StageAdj = (PhiOpStage != -1 ? PhiStage - PhiOpStage : 0);
-          if (PhiOpStage != -1 && PrologStage - StageAdj >= Indirects + np &&
-              VRMap[PrologStage - StageAdj - Indirects - np].count(PhiOp1)) {
-            PhiOp1 = VRMap[PrologStage - StageAdj - Indirects - np][PhiOp1];
-            break;
+          if (PhiOpStage != -1 && PrologStage - StageAdj >= Indirects + np) {
+            auto &M = VRMap[PrologStage - StageAdj - Indirects - np];
+            if (auto It = M.find(PhiOp1); It != M.end()) {
+              PhiOp1 = It->second;
+              break;
+            }
           }
           ++Indirects;
         }
@@ -564,14 +569,14 @@ void ModuloScheduleExpander::generateExistingPhis(
       // We define the Phis after creating the new pipelined code, so
       // we need to rename the Phi values in scheduled instructions.
 
-      unsigned PrevReg = 0;
+      Register PrevReg;
       if (InKernel && VRMap[PrevStage - np].count(LoopVal))
         PrevReg = VRMap[PrevStage - np][LoopVal];
       rewriteScheduledInstr(NewBB, InstrMap, CurStageNum, np, &*BBI, Def,
                             NewReg, PrevReg);
       // If the Phi has been scheduled, use the new name for rewriting.
       if (VRMap[CurStageNum - np].count(Def)) {
-        unsigned R = VRMap[CurStageNum - np][Def];
+        Register R = VRMap[CurStageNum - np][Def];
         rewriteScheduledInstr(NewBB, InstrMap, CurStageNum, np, &*BBI, R,
                               NewReg);
       }
@@ -597,8 +602,11 @@ void ModuloScheduleExpander::generateExistingPhis(
 
     // Check if we need to rename a Phi that has been eliminated due to
     // scheduling.
-    if (NumStages == 0 && IsLast && VRMap[CurStageNum].count(LoopVal))
-      replaceRegUsesAfterLoop(Def, VRMap[CurStageNum][LoopVal], BB, MRI, LIS);
+    if (NumStages == 0 && IsLast) {
+      auto It = VRMap[CurStageNum].find(LoopVal);
+      if (It != VRMap[CurStageNum].end())
+        replaceRegUsesAfterLoop(Def, It->second, BB, MRI, LIS);
+    }
   }
 }
 
@@ -645,7 +653,7 @@ void ModuloScheduleExpander::generatePhis(
       if (!InKernel && (unsigned)StageScheduled > PrologStage)
         continue;
 
-      unsigned PhiOp2;
+      Register PhiOp2;
       if (InKernel) {
         PhiOp2 = VRMap[PrevStage][Def];
         if (MachineInstr *InstOp2 = MRI.getVRegDef(PhiOp2))
@@ -679,7 +687,7 @@ void ModuloScheduleExpander::generatePhis(
         // VRMapPhi (after Kernel) = {0: %Phi1, 1: %Phi0}
         // VRMapPhi (after Epilog0) = {0: %Phi3, 1: %Phi2}
 
-        unsigned PhiOp1 = VRMap[PrologStage][Def];
+        Register PhiOp1 = VRMap[PrologStage][Def];
         if (np <= PrologStage)
           PhiOp1 = VRMap[PrologStage - np][Def];
         if (!InKernel) {
@@ -811,7 +819,7 @@ void ModuloScheduleExpander::splitLifetimes(MachineBasicBlock *KernelBB,
          I != E; ++I) {
       if (I->isPHI() && I->getParent() == KernelBB) {
         // Get the loop carried definition.
-        unsigned LCDef = getLoopPhiReg(PHI, KernelBB);
+        Register LCDef = getLoopPhiReg(PHI, KernelBB);
         if (!LCDef)
           continue;
         MachineInstr *MI = MRI.getVRegDef(LCDef);
@@ -819,12 +827,12 @@ void ModuloScheduleExpander::splitLifetimes(MachineBasicBlock *KernelBB,
           continue;
         // Search through the rest of the block looking for uses of the Phi
         // definition. If one occurs, then split the lifetime.
-        unsigned SplitReg = 0;
+        Register SplitReg;
         for (auto &BBJ : make_range(MachineBasicBlock::instr_iterator(MI),
                                     KernelBB->instr_end()))
           if (BBJ.readsRegister(Def, /*TRI=*/nullptr)) {
             // We split the lifetime when we find the first use.
-            if (SplitReg == 0) {
+            if (!SplitReg) {
               SplitReg = MRI.createVirtualRegister(MRI.getRegClass(Def));
               BuildMI(*KernelBB, MI, MI->getDebugLoc(),
                       TII->get(TargetOpcode::COPY), SplitReg)
@@ -1012,7 +1020,7 @@ MachineInstr *ModuloScheduleExpander::cloneAndChangeInstr(
   MachineInstr *NewMI = MF.CloneMachineInstr(OldMI);
   auto It = InstrChanges.find(OldMI);
   if (It != InstrChanges.end()) {
-    std::pair<unsigned, int64_t> RegAndOffset = It->second;
+    std::pair<Register, int64_t> RegAndOffset = It->second;
     unsigned BasePos, OffsetPos;
     if (!TII->getBaseAndOffsetPosition(*OldMI, BasePos, OffsetPos))
       return nullptr;
@@ -1065,7 +1073,7 @@ void ModuloScheduleExpander::updateInstruction(MachineInstr *NewMI,
 /// Return the instruction in the loop that defines the register.
 /// If the definition is a Phi, then follow the Phi operand to
 /// the instruction in the loop.
-MachineInstr *ModuloScheduleExpander::findDefInLoop(unsigned Reg) {
+MachineInstr *ModuloScheduleExpander::findDefInLoop(Register Reg) {
   SmallPtrSet<MachineInstr *, 8> Visited;
   MachineInstr *Def = MRI.getVRegDef(Reg);
   while (Def->isPHI()) {
@@ -1081,10 +1089,10 @@ MachineInstr *ModuloScheduleExpander::findDefInLoop(unsigned Reg) {
 }
 
 /// Return the new name for the value from the previous stage.
-unsigned ModuloScheduleExpander::getPrevMapVal(
-    unsigned StageNum, unsigned PhiStage, unsigned LoopVal, unsigned LoopStage,
+Register ModuloScheduleExpander::getPrevMapVal(
+    unsigned StageNum, unsigned PhiStage, Register LoopVal, unsigned LoopStage,
     ValueMapTy *VRMap, MachineBasicBlock *BB) {
-  unsigned PrevVal = 0;
+  Register PrevVal;
   if (StageNum > PhiStage) {
     MachineInstr *LoopInst = MRI.getVRegDef(LoopVal);
     if (PhiStage == LoopStage && VRMap[StageNum - 1].count(LoopVal))
@@ -1118,8 +1126,8 @@ void ModuloScheduleExpander::rewritePhiValues(MachineBasicBlock *NewBB,
                                               ValueMapTy *VRMap,
                                               InstrMapTy &InstrMap) {
   for (auto &PHI : BB->phis()) {
-    unsigned InitVal = 0;
-    unsigned LoopVal = 0;
+    Register InitVal;
+    Register LoopVal;
     getPhiRegs(PHI, BB, InitVal, LoopVal);
     Register PhiDef = PHI.getOperand(0).getReg();
 
@@ -1129,7 +1137,7 @@ void ModuloScheduleExpander::rewritePhiValues(MachineBasicBlock *NewBB,
     if (NumPhis > StageNum)
       NumPhis = StageNum;
     for (unsigned np = 0; np <= NumPhis; ++np) {
-      unsigned NewVal =
+      Register NewVal =
           getPrevMapVal(StageNum - np, PhiStage, LoopVal, LoopStage, VRMap, BB);
       if (!NewVal)
         NewVal = InitVal;
@@ -1144,8 +1152,8 @@ void ModuloScheduleExpander::rewritePhiValues(MachineBasicBlock *NewBB,
 /// basic block, and we don't change the uses in the new instruction.
 void ModuloScheduleExpander::rewriteScheduledInstr(
     MachineBasicBlock *BB, InstrMapTy &InstrMap, unsigned CurStageNum,
-    unsigned PhiNum, MachineInstr *Phi, unsigned OldReg, unsigned NewReg,
-    unsigned PrevReg) {
+    unsigned PhiNum, MachineInstr *Phi, Register OldReg, Register NewReg,
+    Register PrevReg) {
   bool InProlog = (CurStageNum < (unsigned)Schedule.getNumStages() - 1);
   int StagePhi = Schedule.getStage(Phi) + PhiNum;
   // Rewrite uses that have been scheduled already to use the new
@@ -1166,7 +1174,7 @@ void ModuloScheduleExpander::rewriteScheduledInstr(
     MachineInstr *OrigMI = OrigInstr->second;
     int StageSched = Schedule.getStage(OrigMI);
     int CycleSched = Schedule.getCycle(OrigMI);
-    unsigned ReplaceReg = 0;
+    Register ReplaceReg;
     // This is the stage for the scheduled instruction.
     if (StagePhi == StageSched && Phi->isPHI()) {
       int CyclePhi = Schedule.getCycle(Phi);
@@ -1208,8 +1216,8 @@ bool ModuloScheduleExpander::isLoopCarried(MachineInstr &Phi) {
   int DefCycle = Schedule.getCycle(&Phi);
   int DefStage = Schedule.getStage(&Phi);
 
-  unsigned InitVal = 0;
-  unsigned LoopVal = 0;
+  Register InitVal;
+  Register LoopVal;
   getPhiRegs(Phi, Phi.getParent(), InitVal, LoopVal);
   MachineInstr *Use = MRI.getVRegDef(LoopVal);
   if (!Use || Use->isPHI())
@@ -1273,7 +1281,7 @@ class KernelRewriter {
   DenseMap<const TargetRegisterClass *, Register> Undefs;
   // Map from <LoopReg, InitReg> to phi register for all created phis. Note that
   // this map is only used when InitReg is non-undef.
-  DenseMap<std::pair<unsigned, unsigned>, Register> Phis;
+  DenseMap<std::pair<Register, Register>, Register> Phis;
   // Map from LoopReg to phi register where the InitReg is undef.
   DenseMap<Register, Register> UndefPhis;
 
@@ -2319,11 +2327,11 @@ void ModuloScheduleExpanderMVE::updateInstrUse(
     MachineInstr *DefInst = MRI.getVRegDef(OrigReg);
     if (!DefInst || DefInst->getParent() != OrigKernel)
       continue;
-    unsigned InitReg = 0;
-    unsigned DefReg = OrigReg;
+    Register InitReg;
+    Register DefReg = OrigReg;
     if (DefInst->isPHI()) {
       ++DiffStage;
-      unsigned LoopReg;
+      Register LoopReg;
       getPhiRegs(*DefInst, OrigKernel, InitReg, LoopReg);
       // LoopReg is guaranteed to be defined within the loop by canApply()
       DefReg = LoopReg;
@@ -2364,7 +2372,7 @@ void ModuloScheduleExpanderMVE::updateInstrUse(
 /// canApply() guarantees that at most only one such phi exists.
 static MachineInstr *getLoopPhiUser(Register Reg, MachineBasicBlock *Loop) {
   for (MachineInstr &Phi : Loop->phis()) {
-    unsigned InitVal, LoopVal;
+    Register InitVal, LoopVal;
     getPhiRegs(Phi, Loop, InitVal, LoopVal);
     if (LoopVal == Reg)
       return &Phi;
@@ -2499,7 +2507,7 @@ void ModuloScheduleExpanderMVE::mergeRegUsesAfterPipeline(Register OrigReg,
   // original loop
   if (!LoopPhis.empty()) {
     for (MachineInstr *Phi : LoopPhis) {
-      unsigned InitReg, LoopReg;
+      Register InitReg, LoopReg;
       getPhiRegs(*Phi, OrigKernel, InitReg, LoopReg);
       Register NewInit = MRI.createVirtualRegister(MRI.getRegClass(InitReg));
       BuildMI(*NewPreheader, NewPreheader->getFirstNonPHI(), Phi->getDebugLoc(),
@@ -2703,7 +2711,7 @@ bool ModuloScheduleExpanderMVE::canApply(MachineLoop &L) {
 
   // Put some constraints on the operands of the phis to simplify the
   // transformation
-  DenseSet<unsigned> UsedByPhi;
+  DenseSet<Register> UsedByPhi;
   for (MachineInstr &MI : BB->phis()) {
     // Registers defined by phis must be used only inside the loop and be never
     // used by phis.
@@ -2719,7 +2727,7 @@ bool ModuloScheduleExpanderMVE::canApply(MachineLoop &L) {
     // A source register from the loop block must be defined inside the loop.
     // A register defined inside the loop must be referenced by only one phi at
     // most.
-    unsigned InitVal, LoopVal;
+    Register InitVal, LoopVal;
     getPhiRegs(MI, MI.getParent(), InitVal, LoopVal);
     if (!Register(LoopVal).isVirtual() ||
         MRI.getVRegDef(LoopVal)->getParent() != BB) {
diff --git a/llvm/lib/CodeGen/PatchableFunction.cpp b/llvm/lib/CodeGen/PatchableFunction.cpp
index 75c2dfeca58d5..07e6c1d90e786 100644
--- a/llvm/lib/CodeGen/PatchableFunction.cpp
+++ b/llvm/lib/CodeGen/PatchableFunction.cpp
@@ -11,6 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/CodeGen/PatchableFunction.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -23,21 +24,37 @@
 using namespace llvm;
 
 namespace {
-struct PatchableFunction : public MachineFunctionPass {
-  static char ID; // Pass identification, replacement for typeid
-  PatchableFunction() : MachineFunctionPass(ID) {
-    initializePatchableFunctionPass(*PassRegistry::getPassRegistry());
+struct PatchableFunction {
+  bool run(MachineFunction &F);
+};
+
+struct PatchableFunctionLegacy : public MachineFunctionPass {
+  static char ID;
+  PatchableFunctionLegacy() : MachineFunctionPass(ID) {
+    initializePatchableFunctionLegacyPass(*PassRegistry::getPassRegistry());
+  }
+  bool runOnMachineFunction(MachineFunction &F) override {
+    return PatchableFunction().run(F);
   }
 
-  bool runOnMachineFunction(MachineFunction &F) override;
-   MachineFunctionProperties getRequiredProperties() const override {
+  MachineFunctionProperties getRequiredProperties() const override {
     return MachineFunctionProperties().set(
         MachineFunctionProperties::Property::NoVRegs);
   }
 };
+
+} // namespace
+
+PreservedAnalyses
+PatchableFunctionPass::run(MachineFunction &MF,
+                           MachineFunctionAnalysisManager &MFAM) {
+  MFPropsModifier _(*this, MF);
+  if (!PatchableFunction().run(MF))
+    return PreservedAnalyses::all();
+  return getMachineFunctionPassPreservedAnalyses();
 }
 
-bool PatchableFunction::runOnMachineFunction(MachineFunction &MF) {
+bool PatchableFunction::run(MachineFunction &MF) {
   MachineBasicBlock &FirstMBB = *MF.begin();
 
   if (MF.getFunction().hasFnAttribute("patchable-function-entry")) {
@@ -62,7 +79,7 @@ bool PatchableFunction::runOnMachineFunction(MachineFunction &MF) {
   return false;
 }
 
-char PatchableFunction::ID = 0;
-char &llvm::PatchableFunctionID = PatchableFunction::ID;
-INITIALIZE_PASS(PatchableFunction, "patchable-function",
+char PatchableFunctionLegacy::ID = 0;
+char &llvm::PatchableFunctionID = PatchableFunctionLegacy::ID;
+INITIALIZE_PASS(PatchableFunctionLegacy, "patchable-function",
                 "Implement the 'patchable-function' attribute", false, false)
diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
index 27fa0b43d74f6..6601e70e495e6 100644
--- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
+++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
@@ -415,8 +415,7 @@ bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const {
       GV->setAlignment(Align(16));
       Value *PatternPtr = GV;
       Value *NumBytes = Builder.CreateMul(
-          Builder.getInt64(DL.getTypeSizeInBits(Memset->getValue()->getType()) /
-                           8),
+          Builder.getInt64(DL.getTypeAllocSize(Memset->getValue()->getType())),
           Memset->getLength());
       CallInst *MemsetPattern16Call =
           Builder.CreateCall(MSP, {Memset->getRawDest(), PatternPtr, NumBytes});
diff --git a/llvm/lib/CodeGen/RegisterBankInfo.cpp b/llvm/lib/CodeGen/RegisterBankInfo.cpp
index a0061ece8add6..eaef13969a7a1 100644
--- a/llvm/lib/CodeGen/RegisterBankInfo.cpp
+++ b/llvm/lib/CodeGen/RegisterBankInfo.cpp
@@ -282,13 +282,13 @@ RegisterBankInfo::getPartialMapping(unsigned StartIdx, unsigned Length,
   ++NumPartialMappingsAccessed;
 
   hash_code Hash = hashPartialMapping(StartIdx, Length, &RegBank);
-  const auto &It = MapOfPartialMappings.find(Hash);
-  if (It != MapOfPartialMappings.end())
+  auto [It, Inserted] = MapOfPartialMappings.try_emplace(Hash);
+  if (!Inserted)
     return *It->second;
 
   ++NumPartialMappingsCreated;
 
-  auto &PartMapping = MapOfPartialMappings[Hash];
+  auto &PartMapping = It->second;
   PartMapping = std::make_unique<PartialMapping>(StartIdx, Length, RegBank);
   return *PartMapping;
 }
@@ -316,13 +316,13 @@ RegisterBankInfo::getValueMapping(const PartialMapping *BreakDown,
   ++NumValueMappingsAccessed;
 
   hash_code Hash = hashValueMapping(BreakDown, NumBreakDowns);
-  const auto &It = MapOfValueMappings.find(Hash);
-  if (It != MapOfValueMappings.end())
+  auto [It, Inserted] = MapOfValueMappings.try_emplace(Hash);
+  if (!Inserted)
     return *It->second;
 
   ++NumValueMappingsCreated;
 
-  auto &ValMapping = MapOfValueMappings[Hash];
+  auto &ValMapping = It->second;
   ValMapping = std::make_unique<ValueMapping>(BreakDown, NumBreakDowns);
   return *ValMapping;
 }
@@ -390,13 +390,13 @@ RegisterBankInfo::getInstructionMappingImpl(
 
   hash_code Hash =
       hashInstructionMapping(ID, Cost, OperandsMapping, NumOperands);
-  const auto &It = MapOfInstructionMappings.find(Hash);
-  if (It != MapOfInstructionMappings.end())
+  auto [It, Inserted] = MapOfInstructionMappings.try_emplace(Hash);
+  if (!Inserted)
     return *It->second;
 
   ++NumInstructionMappingsCreated;
 
-  auto &InstrMapping = MapOfInstructionMappings[Hash];
+  auto &InstrMapping = It->second;
   InstrMapping = std::make_unique<InstructionMapping>(
       ID, Cost, OperandsMapping, NumOperands);
   return *InstrMapping;
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index ef5f2210573e0..bceaf25ee14e4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -5390,6 +5390,12 @@ SDValue DAGCombiner::visitAVG(SDNode *N) {
     }
   }
 
+  // Fold avgfloors(x,y) -> avgflooru(x,y) if both x and y are non-negative
+  if (Opcode == ISD::AVGFLOORS && hasOperation(ISD::AVGFLOORU, VT)) {
+    if (DAG.SignBitIsZero(N0) && DAG.SignBitIsZero(N1))
+      return DAG.getNode(ISD::AVGFLOORU, DL, VT, N0, N1);
+  }
+
   return SDValue();
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 2b8818482a333..9e8227de261f1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -5075,6 +5075,9 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
   if (Node->getOpcode() == ISD::BR_CC ||
       Node->getOpcode() == ISD::SELECT_CC)
     OVT = Node->getOperand(2).getSimpleValueType();
+  // Preserve fast math flags
+  SDNodeFlags FastMathFlags = Node->getFlags() & SDNodeFlags::FastMathFlags;
+  SelectionDAG::FlagInserter FlagsInserter(DAG, FastMathFlags);
   MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
   SDLoc dl(Node);
   SDValue Tmp1, Tmp2, Tmp3, Tmp4;
@@ -5280,7 +5283,6 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
     Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
     // Perform the larger operation, then round down.
     Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
-    Tmp1->setFlags(Node->getFlags());
     if (TruncOp != ISD::FP_ROUND)
       Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
     else
@@ -5409,8 +5411,7 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
   case ISD::FATAN2:
     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
     Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
-    Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
-                       Node->getFlags());
+    Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
     Results.push_back(
         DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3,
                     DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
@@ -5521,8 +5522,7 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
   case ISD::FSINCOS:
   case ISD::FSINCOSPI: {
     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
-    Tmp2 = DAG.getNode(Node->getOpcode(), dl, DAG.getVTList(NVT, NVT), Tmp1,
-                       Node->getFlags());
+    Tmp2 = DAG.getNode(Node->getOpcode(), dl, DAG.getVTList(NVT, NVT), Tmp1);
     Tmp3 = DAG.getIntPtrConstant(0, dl, /*isTarget=*/true);
     for (unsigned ResNum = 0; ResNum < Node->getNumValues(); ResNum++)
       Results.push_back(
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
index b6abad830c371..5210372dd935f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
@@ -233,6 +233,10 @@ bool DAGTypeLegalizer::run() {
     assert(N->getNodeId() == ReadyToProcess &&
            "Node should be ready if on worklist!");
 
+    // Preserve fast math flags
+    SDNodeFlags FastMathFlags = N->getFlags() & SDNodeFlags::FastMathFlags;
+    SelectionDAG::FlagInserter FlagsInserter(DAG, FastMathFlags);
+
     LLVM_DEBUG(dbgs() << "\nLegalizing node: "; N->dump(&DAG));
     if (IgnoreNodeResults(N)) {
       LLVM_DEBUG(dbgs() << "Ignoring node results\n");
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index d2df323fce638..14bb1d943d2d6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -1970,7 +1970,11 @@ void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
   bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
   bool IsSEH = isAsynchronousEHPersonality(Pers);
   MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
-  if (!IsSEH)
+  if (IsSEH) {
+    // For SEH, EHCont Guard needs to know that this catchpad is a target.
+    CatchPadMBB->setIsEHContTarget(true);
+    DAG.getMachineFunction().setHasEHContTarget(true);
+  } else
     CatchPadMBB->setIsEHScopeEntry();
   // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
   if (IsMSVCCXX || IsCoreCLR)
@@ -1981,8 +1985,6 @@ void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
   // Update machine-CFG edge.
   MachineBasicBlock *TargetMBB = FuncInfo.getMBB(I.getSuccessor());
   FuncInfo.MBB->addSuccessor(TargetMBB);
-  TargetMBB->setIsEHContTarget(true);
-  DAG.getMachineFunction().setHasEHContTarget(true);
 
   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
   bool IsSEH = isAsynchronousEHPersonality(Pers);
@@ -1996,6 +1998,10 @@ void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
     return;
   }
 
+  // For non-SEH, EHCont Guard needs to know that this catchret is a target.
+  TargetMBB->setIsEHContTarget(true);
+  DAG.getMachineFunction().setHasEHContTarget(true);
+
   // Figure out the funclet membership for the catchret's successor.
   // This will be used by the FuncletLayout pass to determine how to order the
   // BB's.
@@ -2028,58 +2034,6 @@ void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
   }
 }
 
-// In wasm EH, even though a catchpad may not catch an exception if a tag does
-// not match, it is OK to add only the first unwind destination catchpad to the
-// successors, because there will be at least one invoke instruction within the
-// catch scope that points to the next unwind destination, if one exists, so
-// CFGSort cannot mess up with BB sorting order.
-// (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic
-// call within them, and catchpads only consisting of 'catch (...)' have a
-// '__cxa_end_catch' call within them, both of which generate invokes in case
-// the next unwind destination exists, i.e., the next unwind destination is not
-// the caller.)
-//
-// Having at most one EH pad successor is also simpler and helps later
-// transformations.
-//
-// For example,
-// current:
-//   invoke void @foo to ... unwind label %catch.dispatch
-// catch.dispatch:
-//   %0 = catchswitch within ... [label %catch.start] unwind label %next
-// catch.start:
-//   ...
-//   ... in this BB or some other child BB dominated by this BB there will be an
-//   invoke that points to 'next' BB as an unwind destination
-//
-// next: ; We don't need to add this to 'current' BB's successor
-//   ...
-static void findWasmUnwindDestinations(
-    FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
-    BranchProbability Prob,
-    SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
-        &UnwindDests) {
-  while (EHPadBB) {
-    BasicBlock::const_iterator Pad = EHPadBB->getFirstNonPHIIt();
-    if (isa<CleanupPadInst>(Pad)) {
-      // Stop on cleanup pads.
-      UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
-      UnwindDests.back().first->setIsEHScopeEntry();
-      break;
-    } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
-      // Add the catchpad handlers to the possible destinations. We don't
-      // continue to the unwind destination of the catchswitch for wasm.
-      for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
-        UnwindDests.emplace_back(FuncInfo.getMBB(CatchPadBB), Prob);
-        UnwindDests.back().first->setIsEHScopeEntry();
-      }
-      break;
-    } else {
-      continue;
-    }
-  }
-}
-
 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
 /// many places it could ultimately go. In the IR, we have a single unwind
 /// destination, but in the machine CFG, we enumerate all the possible blocks.
@@ -2100,13 +2054,6 @@ static void findUnwindDestinations(
   bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
   bool IsSEH = isAsynchronousEHPersonality(Personality);
 
-  if (IsWasmCXX) {
-    findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
-    assert(UnwindDests.size() <= 1 &&
-           "There should be at most one unwind destination for wasm");
-    return;
-  }
-
   while (EHPadBB) {
     BasicBlock::const_iterator Pad = EHPadBB->getFirstNonPHIIt();
     BasicBlock *NewEHPadBB = nullptr;
@@ -2116,10 +2063,13 @@ static void findUnwindDestinations(
       break;
     } else if (isa<CleanupPadInst>(Pad)) {
       // Stop on cleanup pads. Cleanups are always funclet entries for all known
-      // personalities.
+      // personalities except Wasm. And in Wasm this becomes a catch_all(_ref),
+      // which always catches an exception.
       UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
       UnwindDests.back().first->setIsEHScopeEntry();
-      UnwindDests.back().first->setIsEHFuncletEntry();
+      // In Wasm, EH scopes are not funclets
+      if (!IsWasmCXX)
+        UnwindDests.back().first->setIsEHFuncletEntry();
       break;
     } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
       // Add the catchpad handlers to the possible destinations.
@@ -12039,7 +11989,7 @@ SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
           assert(isa<AllocaInst>(PHIOp) &&
                  FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
                  "Didn't codegen value into a register!??");
-          Reg = FuncInfo.CreateRegs(PHIOp);
+          Reg = FuncInfo.CreateRegs(&PN);
           CopyValueToVirtualRegister(PHIOp, Reg);
         }
       }
diff --git a/llvm/lib/CodeGen/SlotIndexes.cpp b/llvm/lib/CodeGen/SlotIndexes.cpp
index 1b92a5aa59d18..eecc379776ea0 100644
--- a/llvm/lib/CodeGen/SlotIndexes.cpp
+++ b/llvm/lib/CodeGen/SlotIndexes.cpp
@@ -212,6 +212,7 @@ void SlotIndexes::repairIndexesInRange(MachineBasicBlock *MBB,
   IndexList::iterator ListI = endIdx.listEntry()->getIterator();
   MachineBasicBlock::iterator MBBI = End;
   bool pastStart = false;
+  bool OldIndexesRemoved = false;
   while (ListI != ListB || MBBI != Begin || (includeStart && !pastStart)) {
     assert(ListI->getIndex() >= startIdx.getIndex() &&
            (includeStart || !pastStart) &&
@@ -220,6 +221,8 @@ void SlotIndexes::repairIndexesInRange(MachineBasicBlock *MBB,
     MachineInstr *SlotMI = ListI->getInstr();
     MachineInstr *MI = (MBBI != MBB->end() && !pastStart) ? &*MBBI : nullptr;
     bool MBBIAtBegin = MBBI == Begin && (!includeStart || pastStart);
+    bool MIIndexNotFound = MI && !mi2iMap.contains(MI);
+    bool SlotMIRemoved = false;
 
     if (SlotMI == MI && !MBBIAtBegin) {
       --ListI;
@@ -227,25 +230,31 @@ void SlotIndexes::repairIndexesInRange(MachineBasicBlock *MBB,
         --MBBI;
       else
         pastStart = true;
-    } else if (MI && !mi2iMap.contains(MI)) {
+    } else if (MIIndexNotFound || OldIndexesRemoved) {
       if (MBBI != Begin)
         --MBBI;
       else
         pastStart = true;
     } else {
-      --ListI;
-      if (SlotMI)
+      // We ran through all the indexes on the interval
+      //   -> The only thing left is to go through all the
+      //   remaining MBB instructions and update their indexes
+      if (ListI == ListB)
+        OldIndexesRemoved = true;
+      else
+        --ListI;
+      if (SlotMI) {
         removeMachineInstrFromMaps(*SlotMI);
+        SlotMIRemoved = true;
+      }
     }
-  }
 
-  // In theory this could be combined with the previous loop, but it is tricky
-  // to update the IndexList while we are iterating it.
-  for (MachineBasicBlock::iterator I = End; I != Begin;) {
-    --I;
-    MachineInstr &MI = *I;
-    if (!MI.isDebugOrPseudoInstr() && !mi2iMap.contains(&MI))
-      insertMachineInstrInMaps(MI);
+    MachineInstr *InstrToInsert = SlotMIRemoved ? SlotMI : MI;
+
+    // Insert instruction back into the maps after passing it/removing the index
+    if ((MIIndexNotFound || SlotMIRemoved) && InstrToInsert->getParent() &&
+        !InstrToInsert->isDebugOrPseudoInstr())
+      insertMachineInstrInMaps(*InstrToInsert);
   }
 }
 
diff --git a/llvm/lib/CodeGen/TailDuplicator.cpp b/llvm/lib/CodeGen/TailDuplicator.cpp
index b796c9bb776c8..e08c8b79b3556 100644
--- a/llvm/lib/CodeGen/TailDuplicator.cpp
+++ b/llvm/lib/CodeGen/TailDuplicator.cpp
@@ -601,8 +601,11 @@ bool TailDuplicator::shouldTailDuplicate(bool IsSimple,
   // that rearrange the predecessors of the indirect branch.
 
   bool HasIndirectbr = false;
-  if (!TailBB.empty())
+  bool HasComputedGoto = false;
+  if (!TailBB.empty()) {
     HasIndirectbr = TailBB.back().isIndirectBranch();
+    HasComputedGoto = TailBB.back().isComputedGoto();
+  }
 
   if (HasIndirectbr && PreRegAlloc)
     MaxDuplicateCount = TailDupIndirectBranchSize;
@@ -660,7 +663,12 @@ bool TailDuplicator::shouldTailDuplicate(bool IsSimple,
   // Duplicating a BB which has both multiple predecessors and successors will
   // may cause huge amount of PHI nodes. If we want to remove this limitation,
   // we have to address https://github.com/llvm/llvm-project/issues/78578.
-  if (TailBB.pred_size() > TailDupPredSize &&
+  // NB. This basically unfactors computed gotos that were factored early on in
+  // the compilation process to speed up edge based data flow. If we do not
+  // unfactor them again, it can seriously pessimize code with many computed
+  // jumps in the source code, such as interpreters. Therefore we do not
+  // restrict the computed gotos.
+  if (!HasComputedGoto && TailBB.pred_size() > TailDupPredSize &&
       TailBB.succ_size() > TailDupSuccSize) {
     // If TailBB or any of its successors contains a phi, we may have to add a
     // large number of additional phis with additional incoming values.
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index f5ea3c0b47d6a..3da66a4113334 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -312,6 +312,8 @@ RTLIB::Libcall RTLIB::getSINTTOFP(EVT OpVT, EVT RetVT) {
     if (RetVT == MVT::ppcf128)
       return SINTTOFP_I32_PPCF128;
   } else if (OpVT == MVT::i64) {
+    if (RetVT == MVT::bf16)
+      return SINTTOFP_I64_BF16;
     if (RetVT == MVT::f16)
       return SINTTOFP_I64_F16;
     if (RetVT == MVT::f32)
@@ -358,6 +360,8 @@ RTLIB::Libcall RTLIB::getUINTTOFP(EVT OpVT, EVT RetVT) {
     if (RetVT == MVT::ppcf128)
       return UINTTOFP_I32_PPCF128;
   } else if (OpVT == MVT::i64) {
+    if (RetVT == MVT::bf16)
+      return UINTTOFP_I64_BF16;
     if (RetVT == MVT::f16)
       return UINTTOFP_I64_F16;
     if (RetVT == MVT::f32)
diff --git a/llvm/lib/CodeGen/WinEHPrepare.cpp b/llvm/lib/CodeGen/WinEHPrepare.cpp
index 253a3887009a2..37f6726bafca9 100644
--- a/llvm/lib/CodeGen/WinEHPrepare.cpp
+++ b/llvm/lib/CodeGen/WinEHPrepare.cpp
@@ -866,9 +866,6 @@ void WinEHPrepareImpl::demotePHIsOnFunclets(Function &F,
   for (BasicBlock &BB : make_early_inc_range(F)) {
     if (!BB.isEHPad())
       continue;
-    if (DemoteCatchSwitchPHIOnly &&
-        !isa<CatchSwitchInst>(BB.getFirstNonPHIIt()))
-      continue;
 
     for (Instruction &I : make_early_inc_range(BB)) {
       auto *PN = dyn_cast<PHINode>(&I);
@@ -876,6 +873,23 @@ void WinEHPrepareImpl::demotePHIsOnFunclets(Function &F,
       if (!PN)
         break;
 
+      // If DemoteCatchSwitchPHIOnly is true, we only demote a PHI when
+      // 1. The PHI is within a catchswitch BB
+      // 2. The PHI has a catchswitch BB has one of its incoming blocks
+      if (DemoteCatchSwitchPHIOnly) {
+        bool IsCatchSwitchBB = isa<CatchSwitchInst>(BB.getFirstNonPHIIt());
+        bool HasIncomingCatchSwitchBB = false;
+        for (unsigned I = 0, E = PN->getNumIncomingValues(); I < E; ++I) {
+          if (isa<CatchSwitchInst>(
+                  PN->getIncomingBlock(I)->getFirstNonPHIIt())) {
+            HasIncomingCatchSwitchBB = true;
+            break;
+          }
+        }
+        if (!IsCatchSwitchBB && !HasIncomingCatchSwitchBB)
+          break;
+      }
+
       AllocaInst *SpillSlot = insertPHILoads(PN, F);
       if (SpillSlot)
         insertPHIStores(PN, SpillSlot);
diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
index 540dfdad5831b..72e5f701f89a7 100644
--- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
@@ -639,20 +639,6 @@ Error EHFrameNullTerminator::operator()(LinkGraph &G) {
   return Error::success();
 }
 
-EHFrameRegistrar::~EHFrameRegistrar() = default;
-
-Error InProcessEHFrameRegistrar::registerEHFrames(
-    orc::ExecutorAddrRange EHFrameSection) {
-  return orc::registerEHFrameSection(EHFrameSection.Start.toPtr<void *>(),
-                                     EHFrameSection.size());
-}
-
-Error InProcessEHFrameRegistrar::deregisterEHFrames(
-    orc::ExecutorAddrRange EHFrameSection) {
-  return orc::deregisterEHFrameSection(EHFrameSection.Start.toPtr<void *>(),
-                                       EHFrameSection.size());
-}
-
 EHFrameCFIBlockInspector EHFrameCFIBlockInspector::FromEdgeScan(Block &B) {
   if (B.edges_empty())
     return EHFrameCFIBlockInspector(nullptr);
@@ -678,36 +664,24 @@ EHFrameCFIBlockInspector::EHFrameCFIBlockInspector(Edge &CIEEdge,
                                                    Edge *LSDAEdge)
     : CIEEdge(&CIEEdge), PCBeginEdge(&PCBeginEdge), LSDAEdge(LSDAEdge) {}
 
-LinkGraphPassFunction
-createEHFrameRecorderPass(const Triple &TT,
-                          StoreFrameRangeFunction StoreRangeAddress) {
+Section *getEHFrameSection(LinkGraph &G) {
   const char *EHFrameSectionName = nullptr;
-  if (TT.getObjectFormat() == Triple::MachO)
+  switch (G.getTargetTriple().getObjectFormat()) {
+  case Triple::MachO:
     EHFrameSectionName = "__TEXT,__eh_frame";
-  else
+    break;
+  case Triple::ELF:
     EHFrameSectionName = ".eh_frame";
+    break;
+  default:
+    return nullptr;
+  }
 
-  auto RecordEHFrame =
-      [EHFrameSectionName,
-       StoreFrameRange = std::move(StoreRangeAddress)](LinkGraph &G) -> Error {
-    // Search for a non-empty eh-frame and record the address of the first
-    // symbol in it.
-    orc::ExecutorAddr Addr;
-    size_t Size = 0;
-    if (auto *S = G.findSectionByName(EHFrameSectionName)) {
-      auto R = SectionRange(*S);
-      Addr = R.getStart();
-      Size = R.getSize();
-    }
-    if (!Addr && Size != 0)
-      return make_error<JITLinkError>(
-          StringRef(EHFrameSectionName) +
-          " section can not have zero address with non-zero size");
-    StoreFrameRange(Addr, Size);
-    return Error::success();
-  };
+  if (auto *S = G.findSectionByName(EHFrameSectionName))
+    if (!S->empty())
+      return S;
 
-  return RecordEHFrame;
+  return nullptr;
 }
 
 } // end namespace jitlink
diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
index c701a05ee301f..431f9590a874e 100644
--- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
@@ -18,7 +18,6 @@ add_llvm_component_library(LLVMOrcJIT
   EHFrameRegistrationPlugin.cpp
   EPCDynamicLibrarySearchGenerator.cpp
   EPCDebugObjectRegistrar.cpp
-  EPCEHFrameRegistrar.cpp
   EPCGenericDylibManager.cpp
   EPCGenericJITLinkMemoryManager.cpp
   EPCGenericRTDyldMemoryManager.cpp
diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/DebugInfoSupport.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/DebugInfoSupport.cpp
index 5a058bd712a3e..9b842180fa7a7 100644
--- a/llvm/lib/ExecutionEngine/Orc/Debugging/DebugInfoSupport.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Debugging/DebugInfoSupport.cpp
@@ -33,10 +33,9 @@ static DenseSet<StringRef> DWARFSectionNames = {
 static void preserveDWARFSection(LinkGraph &G, Section &Sec) {
   DenseMap<Block *, Symbol *> Preserved;
   for (auto Sym : Sec.symbols()) {
-    if (Sym->isLive())
-      Preserved[&Sym->getBlock()] = Sym;
-    else if (!Preserved.count(&Sym->getBlock()))
-      Preserved[&Sym->getBlock()] = Sym;
+    auto [It, Inserted] = Preserved.try_emplace(&Sym->getBlock());
+    if (Inserted || Sym->isLive())
+      It->second = Sym;
   }
   for (auto Block : Sec.blocks()) {
     auto &PSym = Preserved[Block];
diff --git a/llvm/lib/ExecutionEngine/Orc/EHFrameRegistrationPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/EHFrameRegistrationPlugin.cpp
index 161bd68ff0854..b34c3434b256c 100644
--- a/llvm/lib/ExecutionEngine/Orc/EHFrameRegistrationPlugin.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/EHFrameRegistrationPlugin.cpp
@@ -10,6 +10,7 @@
 
 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
 #include "llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h"
+#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
 
 #define DEBUG_TYPE "orc"
 
@@ -17,14 +18,26 @@ using namespace llvm::jitlink;
 
 namespace llvm::orc {
 
-EHFrameRegistrationPlugin::EHFrameRegistrationPlugin(
-    ExecutionSession &ES, std::unique_ptr<EHFrameRegistrar> Registrar)
-    : ES(ES), Registrar(std::move(Registrar)) {}
+Expected<std::unique_ptr<EHFrameRegistrationPlugin>>
+EHFrameRegistrationPlugin::Create(ExecutionSession &ES) {
+  // Lookup addresseses of the registration/deregistration functions in the
+  // bootstrap map.
+  ExecutorAddr RegisterEHFrameSectionAllocAction;
+  ExecutorAddr DeregisterEHFrameSectionAllocAction;
+  if (auto Err = ES.getExecutorProcessControl().getBootstrapSymbols(
+          {{RegisterEHFrameSectionAllocAction,
+            rt::RegisterEHFrameSectionAllocActionName},
+           {DeregisterEHFrameSectionAllocAction,
+            rt::DeregisterEHFrameSectionAllocActionName}}))
+    return std::move(Err);
+
+  return std::make_unique<EHFrameRegistrationPlugin>(
+      RegisterEHFrameSectionAllocAction, DeregisterEHFrameSectionAllocAction);
+}
 
 void EHFrameRegistrationPlugin::modifyPassConfig(
     MaterializationResponsibility &MR, LinkGraph &LG,
     PassConfiguration &PassConfig) {
-
   if (LG.getTargetTriple().isOSBinFormatMachO())
     PassConfig.PrePrunePasses.insert(
         PassConfig.PrePrunePasses.begin(), [](LinkGraph &G) {
@@ -33,92 +46,20 @@ void EHFrameRegistrationPlugin::modifyPassConfig(
           return Error::success();
         });
 
-  PassConfig.PostFixupPasses.push_back(createEHFrameRecorderPass(
-      LG.getTargetTriple(), [this, &MR](ExecutorAddr Addr, size_t Size) {
-        if (Addr) {
-          std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
-          assert(!InProcessLinks.count(&MR) &&
-                 "Link for MR already being tracked?");
-          InProcessLinks[&MR] = {Addr, Size};
-        }
-      }));
-}
-
-Error EHFrameRegistrationPlugin::notifyEmitted(
-    MaterializationResponsibility &MR) {
-
-  ExecutorAddrRange EmittedRange;
-  {
-    std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
-
-    auto EHFrameRangeItr = InProcessLinks.find(&MR);
-    if (EHFrameRangeItr == InProcessLinks.end())
-      return Error::success();
-
-    EmittedRange = EHFrameRangeItr->second;
-    assert(EmittedRange.Start && "eh-frame addr to register can not be null");
-    InProcessLinks.erase(EHFrameRangeItr);
-  }
-
-  if (auto Err = MR.withResourceKeyDo(
-          [&](ResourceKey K) { EHFrameRanges[K].push_back(EmittedRange); }))
-    return Err;
-
-  return Registrar->registerEHFrames(EmittedRange);
-}
-
-Error EHFrameRegistrationPlugin::notifyFailed(
-    MaterializationResponsibility &MR) {
-  std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
-  InProcessLinks.erase(&MR);
-  return Error::success();
-}
-
-Error EHFrameRegistrationPlugin::notifyRemovingResources(JITDylib &JD,
-                                                         ResourceKey K) {
-  std::vector<ExecutorAddrRange> RangesToRemove;
-
-  ES.runSessionLocked([&] {
-    auto I = EHFrameRanges.find(K);
-    if (I != EHFrameRanges.end()) {
-      RangesToRemove = std::move(I->second);
-      EHFrameRanges.erase(I);
+  PassConfig.PostFixupPasses.push_back([this](LinkGraph &G) -> Error {
+    if (auto *EHFrame = getEHFrameSection(G)) {
+      using namespace shared;
+      auto R = SectionRange(*EHFrame).getRange();
+      G.allocActions().push_back(
+          {cantFail(
+               WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
+                   RegisterEHFrame, R)),
+           cantFail(
+               WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>(
+                   DeregisterEHFrame, R))});
     }
+    return Error::success();
   });
-
-  Error Err = Error::success();
-  while (!RangesToRemove.empty()) {
-    auto RangeToRemove = RangesToRemove.back();
-    RangesToRemove.pop_back();
-    assert(RangeToRemove.Start && "Untracked eh-frame range must not be null");
-    Err = joinErrors(std::move(Err),
-                     Registrar->deregisterEHFrames(RangeToRemove));
-  }
-
-  return Err;
-}
-
-void EHFrameRegistrationPlugin::notifyTransferringResources(
-    JITDylib &JD, ResourceKey DstKey, ResourceKey SrcKey) {
-  auto SI = EHFrameRanges.find(SrcKey);
-  if (SI == EHFrameRanges.end())
-    return;
-
-  auto DI = EHFrameRanges.find(DstKey);
-  if (DI != EHFrameRanges.end()) {
-    auto &SrcRanges = SI->second;
-    auto &DstRanges = DI->second;
-    DstRanges.reserve(DstRanges.size() + SrcRanges.size());
-    for (auto &SrcRange : SrcRanges)
-      DstRanges.push_back(std::move(SrcRange));
-    EHFrameRanges.erase(SI);
-  } else {
-    // We need to move SrcKey's ranges over without invalidating the SI
-    // iterator.
-    auto Tmp = std::move(SI->second);
-    EHFrameRanges.erase(SI);
-    EHFrameRanges[DstKey] = std::move(Tmp);
-  }
 }
 
 } // namespace llvm::orc
diff --git a/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp b/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp
deleted file mode 100644
index f15315260ab01..0000000000000
--- a/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp
+++ /dev/null
@@ -1,49 +0,0 @@
-//===------ EPCEHFrameRegistrar.cpp - EPC-based eh-frame registration -----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h"
-
-#include "llvm/ExecutionEngine/Orc/Core.h"
-#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
-
-using namespace llvm::orc::shared;
-
-namespace llvm {
-namespace orc {
-
-Expected<std::unique_ptr<EPCEHFrameRegistrar>>
-EPCEHFrameRegistrar::Create(ExecutionSession &ES) {
-
-  // Lookup addresseses of the registration/deregistration functions in the
-  // bootstrap map.
-  ExecutorAddr RegisterEHFrameSectionWrapper;
-  ExecutorAddr DeregisterEHFrameSectionWrapper;
-  if (auto Err = ES.getExecutorProcessControl().getBootstrapSymbols(
-          {{RegisterEHFrameSectionWrapper,
-            rt::RegisterEHFrameSectionWrapperName},
-           {DeregisterEHFrameSectionWrapper,
-            rt::DeregisterEHFrameSectionWrapperName}}))
-    return std::move(Err);
-
-  return std::make_unique<EPCEHFrameRegistrar>(
-      ES, RegisterEHFrameSectionWrapper, DeregisterEHFrameSectionWrapper);
-}
-
-Error EPCEHFrameRegistrar::registerEHFrames(ExecutorAddrRange EHFrameSection) {
-  return ES.callSPSWrapper<void(SPSExecutorAddrRange)>(
-      RegisterEHFrameSectionWrapper, EHFrameSection);
-}
-
-Error EPCEHFrameRegistrar::deregisterEHFrames(
-    ExecutorAddrRange EHFrameSection) {
-  return ES.callSPSWrapper<void(SPSExecutorAddrRange)>(
-      DeregisterEHFrameSectionWrapper, EHFrameSection);
-}
-
-} // end namespace orc
-} // end namespace llvm
diff --git a/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp b/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp
index 060f17c957ef9..fec7062ff79a6 100644
--- a/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp
@@ -28,8 +28,9 @@ EPCGenericRTDyldMemoryManager::CreateWithDefaultBootstrapSymbols(
            {SAs.Finalize, rt::SimpleExecutorMemoryManagerFinalizeWrapperName},
            {SAs.Deallocate,
             rt::SimpleExecutorMemoryManagerDeallocateWrapperName},
-           {SAs.RegisterEHFrame, rt::RegisterEHFrameSectionWrapperName},
-           {SAs.DeregisterEHFrame, rt::DeregisterEHFrameSectionWrapperName}}))
+           {SAs.RegisterEHFrame, rt::RegisterEHFrameSectionAllocActionName},
+           {SAs.DeregisterEHFrame,
+            rt::DeregisterEHFrameSectionAllocActionName}}))
     return std::move(Err);
   return std::make_unique<EPCGenericRTDyldMemoryManager>(EPC, std::move(SAs));
 }
diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
index 1b649227b38a0..3e359f82eeb56 100644
--- a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
@@ -506,10 +506,9 @@ Error DLLImportDefinitionGenerator::tryToGenerate(
     if (Deinterned.starts_with(getImpPrefix()))
       Deinterned = Deinterned.drop_front(StringRef(getImpPrefix()).size());
     // Don't degrade the required state
-    if (ToLookUpSymbols.count(Deinterned) &&
-        ToLookUpSymbols[Deinterned] == SymbolLookupFlags::RequiredSymbol)
-      continue;
-    ToLookUpSymbols[Deinterned] = KV.second;
+    auto [It, Inserted] = ToLookUpSymbols.try_emplace(Deinterned);
+    if (Inserted || It->second != SymbolLookupFlags::RequiredSymbol)
+      It->second = KV.second;
   }
 
   for (auto &KV : ToLookUpSymbols)
diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
index a48b2624eccda..7d385f4cf2fbb 100644
--- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
@@ -14,7 +14,6 @@
 #include "llvm/ExecutionEngine/Orc/EHFrameRegistrationPlugin.h"
 #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
 #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
-#include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h"
 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
@@ -1252,12 +1251,11 @@ Expected<JITDylibSP> setUpGenericLLVMIRPlatform(LLJIT &J) {
     // Otherwise fall back to standard unwind registration.
     if (UseEHFrames) {
       auto &ES = J.getExecutionSession();
-      if (auto EHFrameRegistrar = EPCEHFrameRegistrar::Create(ES)) {
-        OLL->addPlugin(std::make_unique<EHFrameRegistrationPlugin>(
-            ES, std::move(*EHFrameRegistrar)));
+      if (auto EHFP = EHFrameRegistrationPlugin::Create(ES)) {
+        OLL->addPlugin(std::move(*EHFP));
         LLVM_DEBUG(dbgs() << "Enabled eh-frame support.\n");
       } else
-        return EHFrameRegistrar.takeError();
+        return EHFP.takeError();
     }
   }
 
@@ -1342,8 +1340,8 @@ LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) {
 // In-process LLJIT uses eh-frame section wrappers via EPC, so we need to force
 // them to be linked in.
 LLVM_ATTRIBUTE_USED void linkComponents() {
-  errs() << (void *)&llvm_orc_registerEHFrameSectionWrapper
-         << (void *)&llvm_orc_deregisterEHFrameSectionWrapper;
+  errs() << (void *)&llvm_orc_registerEHFrameSectionAllocAction
+         << (void *)&llvm_orc_deregisterEHFrameSectionAllocAction;
 }
 
 } // End namespace orc.
diff --git a/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp b/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp
index 1989d8ca101e1..0b84541258ac1 100644
--- a/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp
@@ -220,10 +220,11 @@ void SharedMemoryMapper::reserve(size_t NumBytes,
                                  OnReservedFunction OnReserved) {
 #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
 
+  int SharedMemoryId = -1;
   EPC.callSPSWrapperAsync<
       rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>(
       SAs.Reserve,
-      [this, NumBytes, OnReserved = std::move(OnReserved)](
+      [this, NumBytes, OnReserved = std::move(OnReserved), SharedMemoryId](
           Error SerializationErr,
           Expected<std::pair<ExecutorAddr, std::string>> Result) mutable {
         if (SerializationErr) {
@@ -248,7 +249,7 @@ void SharedMemoryMapper::reserve(size_t NumBytes,
             SharedMemoryName.size());
         auto HashedName = BLAKE3::hash<sizeof(key_t)>(Data);
         key_t Key = *reinterpret_cast<key_t *>(HashedName.data());
-        int SharedMemoryId =
+        SharedMemoryId =
             shmget(Key, NumBytes, IPC_CREAT | __IPC_SHAREAS | 0700);
         if (SharedMemoryId < 0) {
           return OnReserved(errorCodeToError(
@@ -298,7 +299,8 @@ void SharedMemoryMapper::reserve(size_t NumBytes,
 #endif
         {
           std::lock_guard<std::mutex> Lock(Mutex);
-          Reservations.insert({RemoteAddr, {LocalAddr, NumBytes}});
+          Reservations.insert(
+              {RemoteAddr, {LocalAddr, NumBytes, SharedMemoryId}});
         }
 
         OnReserved(ExecutorAddrRange(RemoteAddr, NumBytes));
@@ -396,7 +398,8 @@ void SharedMemoryMapper::release(ArrayRef<ExecutorAddr> Bases,
 #if defined(LLVM_ON_UNIX)
 
 #if defined(__MVS__)
-      if (shmdt(Reservations[Base].LocalAddr) < 0)
+      if (shmdt(Reservations[Base].LocalAddr) < 0 ||
+          shmctl(Reservations[Base].SharedMemoryId, IPC_RMID, NULL) < 0)
         Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
 #else
       if (munmap(Reservations[Base].LocalAddr, Reservations[Base].Size) != 0)
diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp
index d3b3f121cfcd9..4e5b800100b9c 100644
--- a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp
@@ -52,10 +52,10 @@ const char *MemoryWriteBuffersWrapperName =
 const char *MemoryWritePointersWrapperName =
     "__llvm_orc_bootstrap_mem_write_pointers_wrapper";
 
-const char *RegisterEHFrameSectionWrapperName =
-    "llvm_orc_registerEHFrameSectionWrapper";
-const char *DeregisterEHFrameSectionWrapperName =
-    "llvm_orc_deregisterEHFrameSectionWrapper";
+const char *RegisterEHFrameSectionAllocActionName =
+    "llvm_orc_registerEHFrameAllocAction";
+const char *DeregisterEHFrameSectionAllocActionName =
+    "llvm_orc_deregisterEHFrameAllocAction";
 
 const char *RunAsMainWrapperName = "__llvm_orc_bootstrap_run_as_main_wrapper";
 const char *RunAsVoidFunctionWrapperName =
diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.cpp
index c95b7ac5159fe..a30e87243ada8 100644
--- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.cpp
@@ -22,10 +22,10 @@ void addDefaultBootstrapValuesForHostProcess(
     StringMap<ExecutorAddr> &BootstrapSymbols) {
 
   // FIXME: We probably shouldn't set these on Windows?
-  BootstrapSymbols[rt::RegisterEHFrameSectionWrapperName] =
-      ExecutorAddr::fromPtr(&llvm_orc_registerEHFrameSectionWrapper);
-  BootstrapSymbols[rt::DeregisterEHFrameSectionWrapperName] =
-      ExecutorAddr::fromPtr(&llvm_orc_deregisterEHFrameSectionWrapper);
+  BootstrapSymbols[rt::RegisterEHFrameSectionAllocActionName] =
+      ExecutorAddr::fromPtr(&llvm_orc_registerEHFrameSectionAllocAction);
+  BootstrapSymbols[rt::DeregisterEHFrameSectionAllocActionName] =
+      ExecutorAddr::fromPtr(&llvm_orc_deregisterEHFrameSectionAllocAction);
 
 #ifdef __APPLE__
   if (!dlsym(RTLD_DEFAULT, "__unw_add_find_dynamic_unwind_sections"))
diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp
index f11a73bb5c7ac..fbaf2bae25f4b 100644
--- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp
@@ -164,14 +164,14 @@ static Error deregisterEHFrameWrapper(ExecutorAddrRange EHFrame) {
 }
 
 extern "C" orc::shared::CWrapperFunctionResult
-llvm_orc_registerEHFrameSectionWrapper(const char *Data, uint64_t Size) {
+llvm_orc_registerEHFrameSectionAllocAction(const char *Data, uint64_t Size) {
   return WrapperFunction<SPSError(SPSExecutorAddrRange)>::handle(
              Data, Size, registerEHFrameWrapper)
       .release();
 }
 
 extern "C" orc::shared::CWrapperFunctionResult
-llvm_orc_deregisterEHFrameSectionWrapper(const char *Data, uint64_t Size) {
+llvm_orc_deregisterEHFrameSectionAllocAction(const char *Data, uint64_t Size) {
   return WrapperFunction<SPSError(SPSExecutorAddrRange)>::handle(
              Data, Size, deregisterEHFrameWrapper)
       .release();
diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp
index 4862b2d3e7f79..0ac68122317a9 100644
--- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp
@@ -207,10 +207,10 @@ Error SimpleRemoteEPCServer::sendSetupMessage(
          "Dispatch function name should not be set");
   EI.BootstrapSymbols[ExecutorSessionObjectName] = ExecutorAddr::fromPtr(this);
   EI.BootstrapSymbols[DispatchFnName] = ExecutorAddr::fromPtr(jitDispatchEntry);
-  EI.BootstrapSymbols[rt::RegisterEHFrameSectionWrapperName] =
-      ExecutorAddr::fromPtr(&llvm_orc_registerEHFrameSectionWrapper);
-  EI.BootstrapSymbols[rt::DeregisterEHFrameSectionWrapperName] =
-      ExecutorAddr::fromPtr(&llvm_orc_deregisterEHFrameSectionWrapper);
+  EI.BootstrapSymbols[rt::RegisterEHFrameSectionAllocActionName] =
+      ExecutorAddr::fromPtr(&llvm_orc_registerEHFrameSectionAllocAction);
+  EI.BootstrapSymbols[rt::DeregisterEHFrameSectionAllocActionName] =
+      ExecutorAddr::fromPtr(&llvm_orc_deregisterEHFrameSectionAllocAction);
 
   using SPSSerialize =
       shared::SPSArgList<shared::SPSSimpleRemoteEPCExecutorInfo>;
diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
index 06fc2eb32f3bd..def117448ab6a 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
@@ -1680,16 +1680,16 @@ RuntimeDyldELF::processRelocationRef(
       SectionEntry &Section = Sections[SectionID];
 
       // Look for an existing stub.
-      StubMap::const_iterator i = Stubs.find(Value);
-      if (i != Stubs.end()) {
+      auto [It, Inserted] = Stubs.try_emplace(Value);
+      if (!Inserted) {
         resolveRelocation(Section, Offset,
-                          Section.getLoadAddressWithOffset(i->second), RelType,
+                          Section.getLoadAddressWithOffset(It->second), RelType,
                           0);
         LLVM_DEBUG(dbgs() << " Stub function found\n");
       } else {
         // Create a new stub function.
         LLVM_DEBUG(dbgs() << " Create a new stub function\n");
-        Stubs[Value] = Section.getStubOffset();
+        It->second = Section.getStubOffset();
         uint8_t *StubTargetAddr = createStubFunction(
             Section.getAddressWithOffset(Section.getStubOffset()));
         RelocationEntry RE(SectionID, StubTargetAddr - Section.getAddress(),
@@ -1745,15 +1745,15 @@ RuntimeDyldELF::processRelocationRef(
       Value.Addend += Addend;
 
       //  Look up for existing stub.
-      StubMap::const_iterator i = Stubs.find(Value);
-      if (i != Stubs.end()) {
-        RelocationEntry RE(SectionID, Offset, RelType, i->second);
+      auto [It, Inserted] = Stubs.try_emplace(Value);
+      if (!Inserted) {
+        RelocationEntry RE(SectionID, Offset, RelType, It->second);
         addRelocationForSection(RE, SectionID);
         LLVM_DEBUG(dbgs() << " Stub function found\n");
       } else {
         // Create a new stub function.
         LLVM_DEBUG(dbgs() << " Create a new stub function\n");
-        Stubs[Value] = Section.getStubOffset();
+        It->second = Section.getStubOffset();
 
         unsigned AbiVariant = Obj.getPlatformFlags();
 
@@ -1945,17 +1945,17 @@ RuntimeDyldELF::processRelocationRef(
           RangeOverflow) {
         // It is an external symbol (either Value.SymbolName is set, or
         // SymType is SymbolRef::ST_Unknown) or out of range.
-        StubMap::const_iterator i = Stubs.find(Value);
-        if (i != Stubs.end()) {
+        auto [It, Inserted] = Stubs.try_emplace(Value);
+        if (!Inserted) {
           // Symbol function stub already created, just relocate to it
           resolveRelocation(Section, Offset,
-                            Section.getLoadAddressWithOffset(i->second),
+                            Section.getLoadAddressWithOffset(It->second),
                             RelType, 0);
           LLVM_DEBUG(dbgs() << " Stub function found\n");
         } else {
           // Create a new stub function.
           LLVM_DEBUG(dbgs() << " Create a new stub function\n");
-          Stubs[Value] = Section.getStubOffset();
+          It->second = Section.getStubOffset();
           uint8_t *StubTargetAddr = createStubFunction(
               Section.getAddressWithOffset(Section.getStubOffset()),
               AbiVariant);
@@ -2127,10 +2127,10 @@ RuntimeDyldELF::processRelocationRef(
         // This is a call to an external function.
         // Look for an existing stub.
         SectionEntry *Section = &Sections[SectionID];
-        StubMap::const_iterator i = Stubs.find(Value);
+        auto [It, Inserted] = Stubs.try_emplace(Value);
         uintptr_t StubAddress;
-        if (i != Stubs.end()) {
-          StubAddress = uintptr_t(Section->getAddress()) + i->second;
+        if (!Inserted) {
+          StubAddress = uintptr_t(Section->getAddress()) + It->second;
           LLVM_DEBUG(dbgs() << " Stub function found\n");
         } else {
           // Create a new stub function (equivalent to a PLT entry).
@@ -2140,7 +2140,7 @@ RuntimeDyldELF::processRelocationRef(
           StubAddress = alignTo(BaseAddress + Section->getStubOffset(),
                                 getStubAlignment());
           unsigned StubOffset = StubAddress - BaseAddress;
-          Stubs[Value] = StubOffset;
+          It->second = StubOffset;
           createStubFunction((uint8_t *)StubAddress);
 
           // Bump our stub offset counter
diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h b/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h
index f54ed0eae0076..3ca6309874de5 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h
@@ -121,13 +121,13 @@ class RuntimeDyldCOFFAArch64 : public RuntimeDyldCOFF {
     OriginalRelValueRef.Addend = Addend;
     OriginalRelValueRef.SymbolName = TargetName.data();
 
-    auto Stub = Stubs.find(OriginalRelValueRef);
-    if (Stub == Stubs.end()) {
+    auto [Stub, Inserted] = Stubs.try_emplace(OriginalRelValueRef);
+    if (Inserted) {
       LLVM_DEBUG(dbgs() << " Create a new stub function for "
                         << TargetName.data() << "\n");
 
       StubOffset = Section.getStubOffset();
-      Stubs[OriginalRelValueRef] = StubOffset;
+      Stub->second = StubOffset;
       createStubFunction(Section.getAddressWithOffset(StubOffset));
       Section.advanceStubOffset(getMaxStubSize());
     } else {
diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h b/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h
index 984a8d765c845..2a45535f68fa0 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h
@@ -160,13 +160,13 @@ class RuntimeDyldCOFFX86_64 : public RuntimeDyldCOFF {
     OriginalRelValueRef.Addend = Addend;
     OriginalRelValueRef.SymbolName = TargetName.data();
 
-    auto Stub = Stubs.find(OriginalRelValueRef);
-    if (Stub == Stubs.end()) {
+    auto [Stub, Inserted] = Stubs.try_emplace(OriginalRelValueRef);
+    if (Inserted) {
       LLVM_DEBUG(dbgs() << " Create a new stub function for "
                         << TargetName.data() << "\n");
 
       StubOffset = Section.getStubOffset();
-      Stubs[OriginalRelValueRef] = StubOffset;
+      Stub->second = StubOffset;
       createStubFunction(Section.getAddressWithOffset(StubOffset));
       Section.advanceStubOffset(getMaxStubSize());
     } else {
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 7ce097269ba8b..7718d8a98172f 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -1602,14 +1602,6 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createParallel(
   SmallVector<BasicBlock *, 32> Blocks;
   OI.collectBlocks(ParallelRegionBlockSet, Blocks);
 
-  // Ensure a single exit node for the outlined region by creating one.
-  // We might have multiple incoming edges to the exit now due to finalizations,
-  // e.g., cancel calls that cause the control flow to leave the region.
-  BasicBlock *PRegOutlinedExitBB = PRegExitBB;
-  PRegExitBB = SplitBlock(PRegExitBB, &*PRegExitBB->getFirstInsertionPt());
-  PRegOutlinedExitBB->setName("omp.par.outlined.exit");
-  Blocks.push_back(PRegOutlinedExitBB);
-
   CodeExtractorAnalysisCache CEAC(*OuterFn);
   CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
                           /* AggregateArgs */ false,
@@ -3779,6 +3771,8 @@ OpenMPIRBuilder::createReductions(const LocationDescription &Loc,
 
   // Emit a call to the runtime function that orchestrates the reduction.
   // Declare the reduction function in the process.
+  Type *IndexTy = Builder.getIndexTy(
+      M.getDataLayout(), M.getDataLayout().getDefaultGlobalsAddressSpace());
   Function *Func = Builder.GetInsertBlock()->getParent();
   Module *Module = Func->getParent();
   uint32_t SrcLocStrSize;
@@ -3794,7 +3788,7 @@ OpenMPIRBuilder::createReductions(const LocationDescription &Loc,
   Constant *NumVariables = Builder.getInt32(NumReductions);
   const DataLayout &DL = Module->getDataLayout();
   unsigned RedArrayByteSize = DL.getTypeStoreSize(RedArrayTy);
-  Constant *RedArraySize = Builder.getInt64(RedArrayByteSize);
+  Constant *RedArraySize = ConstantInt::get(IndexTy, RedArrayByteSize);
   Function *ReductionFunc = getFreshReductionFunc(*Module);
   Value *Lock = getOMPCriticalRegionLock(".reduction");
   Function *ReduceFunc = getOrCreateRuntimeFunctionPtr(
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 88bb2359cd055..ae68da0182dc4 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -96,6 +96,10 @@ static cl::opt<bool> PrintInstDebugLocs(
     "print-inst-debug-locs", cl::Hidden,
     cl::desc("Pretty print debug locations of instructions when dumping"));
 
+static cl::opt<bool> PrintProfData(
+    "print-prof-data", cl::Hidden,
+    cl::desc("Pretty print perf data (branch weights, etc) when dumping"));
+
 // Make virtual table appear in this compilation unit.
 AssemblyAnnotationWriter::~AssemblyAnnotationWriter() = default;
 
@@ -4161,6 +4165,13 @@ void AssemblyWriter::printFunction(const Function *F) {
     writeOperand(F->getPersonalityFn(), /*PrintType=*/true);
   }
 
+  if (PrintProfData) {
+    if (auto *MDProf = F->getMetadata(LLVMContext::MD_prof)) {
+      Out << " ";
+      MDProf->print(Out, TheModule, /*IsForDebug=*/true);
+    }
+  }
+
   if (F->isDeclaration()) {
     Out << '\n';
   } else {
@@ -4287,6 +4298,14 @@ void AssemblyWriter::printInfoComment(const Value &V) {
       }
     }
   }
+  if (PrintProfData) {
+    if (auto *I = dyn_cast<Instruction>(&V)) {
+      if (auto *MD = I->getMetadata(LLVMContext::MD_prof)) {
+        Out << " ; ";
+        MD->print(Out, TheModule, /*IsForDebug=*/true);
+      }
+    }
+  }
 
   if (PrintInstAddrs)
     Out << " ; " << &V;
diff --git a/llvm/lib/IR/Module.cpp b/llvm/lib/IR/Module.cpp
index c7b9f8744d8d3..c7daaafe13e3f 100644
--- a/llvm/lib/IR/Module.cpp
+++ b/llvm/lib/IR/Module.cpp
@@ -915,3 +915,11 @@ VersionTuple Module::getDarwinTargetVariantSDKVersion() const {
 void Module::setDarwinTargetVariantSDKVersion(VersionTuple Version) {
   addSDKVersionMD(Version, *this, "darwin.target_variant.SDK Version");
 }
+
+StringRef Module::getTargetABIFromMD() {
+  StringRef TargetABI;
+  if (auto *TargetABIMD =
+          dyn_cast_or_null<MDString>(getModuleFlag("target-abi")))
+    TargetABI = TargetABIMD->getString();
+  return TargetABI;
+}
diff --git a/llvm/lib/IR/User.cpp b/llvm/lib/IR/User.cpp
index b0aa785deb9af..ab44cb4b8a3f7 100644
--- a/llvm/lib/IR/User.cpp
+++ b/llvm/lib/IR/User.cpp
@@ -146,6 +146,9 @@ void *User::allocateFixedOperandUser(size_t Size, unsigned Us,
   Use *Start = reinterpret_cast<Use *>(Storage + DescBytesToAllocate);
   Use *End = Start + Us;
   User *Obj = reinterpret_cast<User *>(End);
+  Obj->NumUserOperands = Us;
+  Obj->HasHungOffUses = false;
+  Obj->HasDescriptor = DescBytes != 0;
   for (; Start != End; Start++)
     new (Start) Use(Obj);
 
@@ -172,6 +175,9 @@ void *User::operator new(size_t Size, HungOffOperandsAllocMarker) {
   void *Storage = ::operator new(Size + sizeof(Use *));
   Use **HungOffOperandList = static_cast<Use **>(Storage);
   User *Obj = reinterpret_cast<User *>(HungOffOperandList + 1);
+  Obj->NumUserOperands = 0;
+  Obj->HasHungOffUses = true;
+  Obj->HasDescriptor = false;
   *HungOffOperandList = nullptr;
   return Obj;
 }
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index dfd4b5188907d..e895a46b8cd77 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -1437,12 +1437,10 @@ class InProcessThinBackend : public ThinBackendProc {
                         OnWrite, ShouldEmitImportsFiles, ThinLTOParallelism),
         AddStream(std::move(AddStream)), Cache(std::move(Cache)),
         ShouldEmitIndexFiles(ShouldEmitIndexFiles) {
-    for (auto &Name : CombinedIndex.cfiFunctionDefs())
-      CfiFunctionDefs.insert(
-          GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
-    for (auto &Name : CombinedIndex.cfiFunctionDecls())
-      CfiFunctionDecls.insert(
-          GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
+    auto &Defs = CombinedIndex.cfiFunctionDefs();
+    CfiFunctionDefs.insert(Defs.guid_begin(), Defs.guid_end());
+    auto &Decls = CombinedIndex.cfiFunctionDecls();
+    CfiFunctionDecls.insert(Decls.guid_begin(), Decls.guid_end());
   }
 
   virtual Error runThinLTOBackendThread(
@@ -1965,12 +1963,10 @@ Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache,
 
   // Any functions referenced by the jump table in the regular LTO object must
   // be exported.
-  for (auto &Def : ThinLTO.CombinedIndex.cfiFunctionDefs())
-    ExportedGUIDs.insert(
-        GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Def)));
-  for (auto &Decl : ThinLTO.CombinedIndex.cfiFunctionDecls())
-    ExportedGUIDs.insert(
-        GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Decl)));
+  auto &Defs = ThinLTO.CombinedIndex.cfiFunctionDefs();
+  ExportedGUIDs.insert(Defs.guid_begin(), Defs.guid_end());
+  auto &Decls = ThinLTO.CombinedIndex.cfiFunctionDecls();
+  ExportedGUIDs.insert(Decls.guid_begin(), Decls.guid_end());
 
   auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
     const auto &ExportList = ExportLists.find(ModuleIdentifier);
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index b38252a3272e8..139c39abf8e6b 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -221,8 +221,13 @@ createTargetMachine(const Config &Conf, const Target *TheTarget, Module &M) {
   else
     CodeModel = M.getCodeModel();
 
+  TargetOptions TargetOpts = Conf.Options;
+  if (TargetOpts.MCOptions.ABIName.empty()) {
+    TargetOpts.MCOptions.ABIName = M.getTargetABIFromMD();
+  }
+
   std::unique_ptr<TargetMachine> TM(TheTarget->createTargetMachine(
-      TheTriple.str(), Conf.CPU, Features.getString(), Conf.Options, RelocModel,
+      TheTriple.str(), Conf.CPU, Features.getString(), TargetOpts, RelocModel,
       CodeModel, Conf.CGOptLevel));
 
   assert(TM && "Failed to create target machine");
diff --git a/llvm/lib/MC/MCAsmInfo.cpp b/llvm/lib/MC/MCAsmInfo.cpp
index 9821bb298d7d4..69374f69bc8e5 100644
--- a/llvm/lib/MC/MCAsmInfo.cpp
+++ b/llvm/lib/MC/MCAsmInfo.cpp
@@ -145,9 +145,9 @@ StringRef MCAsmInfo::getVariantKindName(uint32_t Kind) const {
   return It->second;
 }
 
-uint32_t MCAsmInfo::getVariantKindForName(StringRef Name) const {
+std::optional<uint32_t> MCAsmInfo::getVariantKindForName(StringRef Name) const {
   auto It = NameToVariantKind.find(Name.lower());
   if (It != NameToVariantKind.end())
     return It->second;
-  return MCSymbolRefExpr::VK_Invalid;
+  return {};
 }
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index f1b34ca65abd9..fe6bb8c965147 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -294,8 +294,8 @@ class MCAsmStreamer final : public MCStreamer {
                                unsigned CUID = 0) override;
   void emitDwarfLocDirective(unsigned FileNo, unsigned Line, unsigned Column,
                              unsigned Flags, unsigned Isa,
-                             unsigned Discriminator,
-                             StringRef FileName) override;
+                             unsigned Discriminator, StringRef FileName,
+                             StringRef Location = {}) override;
   virtual void emitDwarfLocLabelDirective(SMLoc Loc, StringRef Name) override;
 
   MCSymbol *getDwarfLineTableSymbol(unsigned CUID) override;
@@ -1688,7 +1688,8 @@ void MCAsmStreamer::emitDwarfFile0Directive(
 void MCAsmStreamer::emitDwarfLocDirective(unsigned FileNo, unsigned Line,
                                           unsigned Column, unsigned Flags,
                                           unsigned Isa, unsigned Discriminator,
-                                          StringRef FileName) {
+                                          StringRef FileName,
+                                          StringRef Comment) {
   // If target doesn't support .loc/.file directive, we need to record the lines
   // same way like we do in object mode.
   if (MAI->isAIX()) {
@@ -1696,7 +1697,7 @@ void MCAsmStreamer::emitDwarfLocDirective(unsigned FileNo, unsigned Line,
     // first one gets a line entry.
     MCDwarfLineEntry::make(this, getCurrentSectionOnly());
     this->MCStreamer::emitDwarfLocDirective(FileNo, Line, Column, Flags, Isa,
-                                            Discriminator, FileName);
+                                            Discriminator, FileName, Comment);
     return;
   }
 
@@ -1727,12 +1728,15 @@ void MCAsmStreamer::emitDwarfLocDirective(unsigned FileNo, unsigned Line,
 
   if (IsVerboseAsm) {
     OS.PadToColumn(MAI->getCommentColumn());
-    OS << MAI->getCommentString() << ' ' << FileName << ':'
-       << Line << ':' << Column;
+    OS << MAI->getCommentString() << ' ';
+    if (Comment.empty())
+      OS << FileName << ':' << Line << ':' << Column;
+    else
+      OS << Comment;
   }
   EmitEOL();
   this->MCStreamer::emitDwarfLocDirective(FileNo, Line, Column, Flags, Isa,
-                                          Discriminator, FileName);
+                                          Discriminator, FileName, Comment);
 }
 
 void MCAsmStreamer::emitDwarfLocLabelDirective(SMLoc Loc, StringRef Name) {
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index 9aee1abcd0d67..41c08809301ee 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -418,13 +418,14 @@ void MCObjectStreamer::emitDwarfLocDirective(unsigned FileNo, unsigned Line,
                                              unsigned Column, unsigned Flags,
                                              unsigned Isa,
                                              unsigned Discriminator,
-                                             StringRef FileName) {
+                                             StringRef FileName,
+                                             StringRef Comment) {
   // In case we see two .loc directives in a row, make sure the
   // first one gets a line entry.
   MCDwarfLineEntry::make(this, getCurrentSectionOnly());
 
   this->MCStreamer::emitDwarfLocDirective(FileNo, Line, Column, Flags, Isa,
-                                          Discriminator, FileName);
+                                          Discriminator, FileName, Comment);
 }
 
 static const MCExpr *buildSymbolDiff(MCObjectStreamer &OS, const MCSymbol *A,
diff --git a/llvm/lib/MC/MCParser/AsmLexer.cpp b/llvm/lib/MC/MCParser/AsmLexer.cpp
index 41ff94d125a16..23836438027c0 100644
--- a/llvm/lib/MC/MCParser/AsmLexer.cpp
+++ b/llvm/lib/MC/MCParser/AsmLexer.cpp
@@ -698,8 +698,10 @@ size_t AsmLexer::peekTokens(MutableArrayRef<AsmToken> Buf,
 
     Buf[ReadCount] = Token;
 
-    if (Token.is(AsmToken::Eof))
+    if (Token.is(AsmToken::Eof)) {
+      ReadCount++;
       break;
+    }
   }
 
   SetError(SavedErrLoc, SavedErr);
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 1e1b63717ded5..0cd691e9298ed 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -1228,10 +1228,10 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc,
 
     // Lookup the symbol variant if used.
     if (!Split.second.empty()) {
-      Variant =
-          MCSymbolRefExpr::VariantKind(MAI.getVariantKindForName(Split.second));
-      if (Variant != MCSymbolRefExpr::VK_Invalid) {
+      auto MaybeVariant = MAI.getVariantKindForName(Split.second);
+      if (MaybeVariant) {
         SymbolName = Split.first;
+        Variant = MCSymbolRefExpr::VariantKind(*MaybeVariant);
       } else if (MAI.doesAllowAtInName() && !MAI.useParensForSymbolVariant()) {
         Variant = MCSymbolRefExpr::VK_None;
       } else {
@@ -1279,11 +1279,11 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc,
       std::pair<StringRef, StringRef> Split = IDVal.split('@');
       MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
       if (Split.first.size() != IDVal.size()) {
-        Variant = MCSymbolRefExpr::VariantKind(
-            MAI.getVariantKindForName(Split.second));
-        if (Variant == MCSymbolRefExpr::VK_Invalid)
+        auto MaybeVariant = MAI.getVariantKindForName(Split.second);
+        if (!MaybeVariant)
           return TokError("invalid variant '" + Split.second + "'");
         IDVal = Split.first;
+        Variant = MCSymbolRefExpr::VariantKind(*MaybeVariant);
       }
       if (IDVal == "f" || IDVal == "b") {
         MCSymbol *Sym =
@@ -1470,12 +1470,12 @@ bool AsmParser::parseExpression(const MCExpr *&Res, SMLoc &EndLoc) {
     if (Lexer.isNot(AsmToken::Identifier))
       return TokError("unexpected symbol modifier following '@'");
 
-    auto Variant = MCSymbolRefExpr::VariantKind(
-        MAI.getVariantKindForName(getTok().getIdentifier()));
-    if (Variant == MCSymbolRefExpr::VK_Invalid)
+    auto Variant = MAI.getVariantKindForName(getTok().getIdentifier());
+    if (!Variant)
       return TokError("invalid variant '" + getTok().getIdentifier() + "'");
 
-    const MCExpr *ModifiedRes = applyModifierToExpr(Res, Variant);
+    const MCExpr *ModifiedRes =
+        applyModifierToExpr(Res, MCSymbolRefExpr::VariantKind(*Variant));
     if (!ModifiedRes) {
       return TokError("invalid modifier '" + getTok().getIdentifier() +
                       "' (no symbols present)");
diff --git a/llvm/lib/MC/MCParser/MasmParser.cpp b/llvm/lib/MC/MCParser/MasmParser.cpp
index b7adce918b467..c1a451a7119d5 100644
--- a/llvm/lib/MC/MCParser/MasmParser.cpp
+++ b/llvm/lib/MC/MCParser/MasmParser.cpp
@@ -1985,7 +1985,10 @@ bool MasmParser::parseStatement(ParseStatementInfo &Info,
 
   // If macros are enabled, check to see if this is a macro instantiation.
   if (const MCAsmMacro *M = getContext().lookupMacro(IDVal.lower())) {
-    return handleMacroEntry(M, IDLoc);
+    AsmToken::TokenKind ArgumentEndTok = parseOptionalToken(AsmToken::LParen)
+                                             ? AsmToken::RParen
+                                             : AsmToken::EndOfStatement;
+    return handleMacroEntry(M, IDLoc, ArgumentEndTok);
   }
 
   // Otherwise, we have a normal instruction or directive.
@@ -2829,7 +2832,7 @@ bool MasmParser::handleMacroEntry(const MCAsmMacro *M, SMLoc NameLoc,
   }
 
   MCAsmMacroArguments A;
-  if (parseMacroArguments(M, A, ArgumentEndTok))
+  if (parseMacroArguments(M, A, ArgumentEndTok) || parseToken(ArgumentEndTok))
     return true;
 
   // Macro instantiation is lexical, unfortunately. We construct a new buffer
@@ -2913,10 +2916,6 @@ bool MasmParser::handleMacroInvocation(const MCAsmMacro *M, SMLoc NameLoc) {
       eatToEndOfStatement();
   }
 
-  // Consume the right-parenthesis on the other side of the arguments.
-  if (parseRParen())
-    return true;
-
   // Exit values may require lexing, unfortunately. We construct a new buffer to
   // hold the exit value.
   std::unique_ptr<MemoryBuffer> MacroValue =
diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp
index 99b7651a9ab31..f040954efb6b5 100644
--- a/llvm/lib/MC/MCStreamer.cpp
+++ b/llvm/lib/MC/MCStreamer.cpp
@@ -238,7 +238,7 @@ void MCStreamer::emitCFIMTETaggedFrame() {
 void MCStreamer::emitDwarfLocDirective(unsigned FileNo, unsigned Line,
                                        unsigned Column, unsigned Flags,
                                        unsigned Isa, unsigned Discriminator,
-                                       StringRef FileName) {
+                                       StringRef FileName, StringRef Comment) {
   getContext().setCurrentDwarfLoc(FileNo, Line, Column, Flags, Isa,
                                   Discriminator);
 }
diff --git a/llvm/lib/MC/MCSubtargetInfo.cpp b/llvm/lib/MC/MCSubtargetInfo.cpp
index f59ec2f7c2602..d86eaad48420d 100644
--- a/llvm/lib/MC/MCSubtargetInfo.cpp
+++ b/llvm/lib/MC/MCSubtargetInfo.cpp
@@ -317,14 +317,18 @@ FeatureBitset MCSubtargetInfo::ApplyFeatureFlag(StringRef FS) {
 
 bool MCSubtargetInfo::checkFeatures(StringRef FS) const {
   SubtargetFeatures T(FS);
-  FeatureBitset Set, All;
-  for (std::string F : T.getFeatures()) {
-    ::ApplyFeatureFlag(Set, F, ProcFeatures);
-    if (F[0] == '-')
-      F[0] = '+';
-    ::ApplyFeatureFlag(All, F, ProcFeatures);
-  }
-  return (FeatureBits & All) == Set;
+  return all_of(T.getFeatures(), [this](const std::string &F) {
+    assert(SubtargetFeatures::hasFlag(F) &&
+           "Feature flags should start with '+' or '-'");
+    const SubtargetFeatureKV *FeatureEntry =
+        Find(SubtargetFeatures::StripFlag(F), ProcFeatures);
+    if (!FeatureEntry)
+      report_fatal_error(Twine("'") + F +
+                         "' is not a recognized feature for this target");
+
+    return FeatureBits.test(FeatureEntry->Value) ==
+           SubtargetFeatures::isEnabled(F);
+  });
 }
 
 const MCSchedModel &MCSubtargetInfo::getSchedModelForCPU(StringRef CPU) const {
diff --git a/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp b/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
index 1a31bfa2e0db3..5a785d7afd1e4 100644
--- a/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
+++ b/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
@@ -364,7 +364,7 @@ static Error updateAndRemoveSymbols(const CommonConfig &Config,
   // (like GroupSection or RelocationSection). This way, we know which
   // symbols are still 'needed' and which are not.
   if (Config.StripUnneeded || !Config.UnneededSymbolsToRemove.empty() ||
-      !Config.OnlySection.empty()) {
+      !Config.OnlySection.empty() || Config.DiscardMode != DiscardType::None) {
     for (SectionBase &Sec : Obj.sections())
       Sec.markSymbols();
   }
@@ -386,22 +386,23 @@ static Error updateAndRemoveSymbols(const CommonConfig &Config,
     if (Config.StripDebug && Sym.Type == STT_FILE)
       return true;
 
-    if ((Config.DiscardMode == DiscardType::All ||
-         (Config.DiscardMode == DiscardType::Locals &&
-          StringRef(Sym.Name).starts_with(".L"))) &&
-        Sym.Binding == STB_LOCAL && Sym.getShndx() != SHN_UNDEF &&
-        Sym.Type != STT_FILE && Sym.Type != STT_SECTION)
-      return true;
-
     if ((Config.StripUnneeded ||
          Config.UnneededSymbolsToRemove.matches(Sym.Name)) &&
         (!Obj.isRelocatable() || isUnneededSymbol(Sym)))
       return true;
 
-    // We want to remove undefined symbols if all references have been stripped.
-    if (!Config.OnlySection.empty() && !Sym.Referenced &&
-        Sym.getShndx() == SHN_UNDEF)
-      return true;
+    if (!Sym.Referenced) {
+      if ((Config.DiscardMode == DiscardType::All ||
+           (Config.DiscardMode == DiscardType::Locals &&
+            StringRef(Sym.Name).starts_with(".L"))) &&
+          Sym.Binding == STB_LOCAL && Sym.getShndx() != SHN_UNDEF &&
+          Sym.Type != STT_FILE && Sym.Type != STT_SECTION)
+        return true;
+      // We want to remove undefined symbols if all references have been
+      // stripped.
+      if (!Config.OnlySection.empty() && Sym.getShndx() == SHN_UNDEF)
+        return true;
+    }
 
     return false;
   };
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 8080059f0bb03..2e62d6e6f4cc6 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -84,6 +84,7 @@
 #include "llvm/CodeGen/CodeGenPrepare.h"
 #include "llvm/CodeGen/ComplexDeinterleavingPass.h"
 #include "llvm/CodeGen/DeadMachineInstructionElim.h"
+#include "llvm/CodeGen/DetectDeadLanes.h"
 #include "llvm/CodeGen/DwarfEHPrepare.h"
 #include "llvm/CodeGen/EarlyIfConversion.h"
 #include "llvm/CodeGen/EdgeBundles.h"
@@ -129,6 +130,7 @@
 #include "llvm/CodeGen/MachineVerifier.h"
 #include "llvm/CodeGen/OptimizePHIs.h"
 #include "llvm/CodeGen/PHIElimination.h"
+#include "llvm/CodeGen/PatchableFunction.h"
 #include "llvm/CodeGen/PeepholeOptimizer.h"
 #include "llvm/CodeGen/PostRASchedulerList.h"
 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
diff --git a/llvm/lib/ProfileData/PGOCtxProfReader.cpp b/llvm/lib/ProfileData/PGOCtxProfReader.cpp
index 63a6b013baba6..5cc4c94c74b76 100644
--- a/llvm/lib/ProfileData/PGOCtxProfReader.cpp
+++ b/llvm/lib/ProfileData/PGOCtxProfReader.cpp
@@ -57,13 +57,24 @@ Error PGOCtxProfileReader::unsupported(const Twine &Msg) {
   return make_error<InstrProfError>(instrprof_error::unsupported_version, Msg);
 }
 
-bool PGOCtxProfileReader::canEnterBlockWithID(PGOCtxProfileBlockIDs ID) {
+bool PGOCtxProfileReader::tryGetNextKnownBlockID(PGOCtxProfileBlockIDs &ID) {
   auto Blk = advance();
   if (!Blk) {
     consumeError(Blk.takeError());
     return false;
   }
-  return Blk->Kind == BitstreamEntry::SubBlock && Blk->ID == ID;
+  if (Blk->Kind != BitstreamEntry::SubBlock)
+    return false;
+  if (PGOCtxProfileBlockIDs::FIRST_VALID > Blk->ID ||
+      PGOCtxProfileBlockIDs::LAST_VALID < Blk->ID)
+    return false;
+  ID = static_cast<PGOCtxProfileBlockIDs>(Blk->ID);
+  return true;
+}
+
+bool PGOCtxProfileReader::canEnterBlockWithID(PGOCtxProfileBlockIDs ID) {
+  PGOCtxProfileBlockIDs Test = {};
+  return tryGetNextKnownBlockID(Test) && Test == ID;
 }
 
 Error PGOCtxProfileReader::enterBlockWithID(PGOCtxProfileBlockIDs ID) {
@@ -71,10 +82,14 @@ Error PGOCtxProfileReader::enterBlockWithID(PGOCtxProfileBlockIDs ID) {
   return Error::success();
 }
 
+// Note: we use PGOCtxProfContext for flat profiles also, as the latter are
+// structurally similar. Alternative modeling here seems a bit overkill at the
+// moment.
 Expected<std::pair<std::optional<uint32_t>, PGOCtxProfContext>>
 PGOCtxProfileReader::readProfile(PGOCtxProfileBlockIDs Kind) {
   assert((Kind == PGOCtxProfileBlockIDs::ContextRootBlockID ||
-          Kind == PGOCtxProfileBlockIDs::ContextNodeBlockID) &&
+          Kind == PGOCtxProfileBlockIDs::ContextNodeBlockID ||
+          Kind == PGOCtxProfileBlockIDs::FlatProfileBlockID) &&
          "Unexpected profile kind");
   RET_ON_ERR(enterBlockWithID(Kind));
 
@@ -176,14 +191,24 @@ Error PGOCtxProfileReader::readMetadata() {
 }
 
 Error PGOCtxProfileReader::loadContexts(CtxProfContextualProfiles &P) {
-  if (canEnterBlockWithID(PGOCtxProfileBlockIDs::ContextsSectionBlockID)) {
-    RET_ON_ERR(enterBlockWithID(PGOCtxProfileBlockIDs::ContextsSectionBlockID));
-    while (canEnterBlockWithID(PGOCtxProfileBlockIDs::ContextRootBlockID)) {
-      EXPECT_OR_RET(E, readProfile(PGOCtxProfileBlockIDs::ContextRootBlockID));
-      auto Key = E->second.guid();
-      if (!P.insert({Key, std::move(E->second)}).second)
-        return wrongValue("Duplicate roots");
-    }
+  RET_ON_ERR(enterBlockWithID(PGOCtxProfileBlockIDs::ContextsSectionBlockID));
+  while (canEnterBlockWithID(PGOCtxProfileBlockIDs::ContextRootBlockID)) {
+    EXPECT_OR_RET(E, readProfile(PGOCtxProfileBlockIDs::ContextRootBlockID));
+    auto Key = E->second.guid();
+    if (!P.insert({Key, std::move(E->second)}).second)
+      return wrongValue("Duplicate roots");
+  }
+  return Error::success();
+}
+
+Error PGOCtxProfileReader::loadFlatProfiles(CtxProfFlatProfile &P) {
+  RET_ON_ERR(
+      enterBlockWithID(PGOCtxProfileBlockIDs::FlatProfilesSectionBlockID));
+  while (canEnterBlockWithID(PGOCtxProfileBlockIDs::FlatProfileBlockID)) {
+    EXPECT_OR_RET(E, readProfile(PGOCtxProfileBlockIDs::FlatProfileBlockID));
+    auto Guid = E->second.guid();
+    if (!P.insert({Guid, std::move(E->second.counters())}).second)
+      return wrongValue("Duplicate flat profile entries");
   }
   return Error::success();
 }
@@ -191,7 +216,19 @@ Error PGOCtxProfileReader::loadContexts(CtxProfContextualProfiles &P) {
 Expected<PGOCtxProfile> PGOCtxProfileReader::loadProfiles() {
   RET_ON_ERR(readMetadata());
   PGOCtxProfile Ret;
-  RET_ON_ERR(loadContexts(Ret.Contexts));
+  PGOCtxProfileBlockIDs Test = {};
+  for (auto I = 0; I < 2; ++I) {
+    if (!tryGetNextKnownBlockID(Test))
+      break;
+    if (Test == PGOCtxProfileBlockIDs::ContextsSectionBlockID) {
+      RET_ON_ERR(loadContexts(Ret.Contexts));
+    } else if (Test == PGOCtxProfileBlockIDs::FlatProfilesSectionBlockID) {
+      RET_ON_ERR(loadFlatProfiles(Ret.FlatProfiles));
+    } else {
+      return wrongValue("Unexpected section");
+    }
+  }
+
   return std::move(Ret);
 }
 
@@ -287,5 +324,17 @@ void llvm::convertCtxProfToYaml(raw_ostream &OS, const PGOCtxProfile &Profile) {
     toYaml(Out, Profile.Contexts);
     Out.postflightKey(nullptr);
   }
+  if (!Profile.FlatProfiles.empty()) {
+    Out.preflightKey("FlatProfiles", false, false, UseDefault, SaveInfo);
+    Out.beginSequence();
+    size_t ElemID = 0;
+    for (const auto &[Guid, Counters] : Profile.FlatProfiles) {
+      Out.preflightElement(ElemID++, SaveInfo);
+      toYaml(Out, Guid, Counters, {});
+      Out.postflightElement(nullptr);
+    }
+    Out.endSequence();
+    Out.postflightKey(nullptr);
+  }
   Out.endMapping();
 }
diff --git a/llvm/lib/ProfileData/PGOCtxProfWriter.cpp b/llvm/lib/ProfileData/PGOCtxProfWriter.cpp
index d4184da1c2509..e906836b16b2b 100644
--- a/llvm/lib/ProfileData/PGOCtxProfWriter.cpp
+++ b/llvm/lib/ProfileData/PGOCtxProfWriter.cpp
@@ -13,6 +13,7 @@
 #include "llvm/ProfileData/PGOCtxProfWriter.h"
 #include "llvm/Bitstream/BitCodeEnums.h"
 #include "llvm/ProfileData/CtxInstrContextNode.h"
+#include "llvm/ProfileData/PGOCtxProfReader.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/YAMLTraits.h"
@@ -59,6 +60,11 @@ PGOCtxProfileWriter::PGOCtxProfileWriter(
     DescribeRecord(PGOCtxProfileRecords::Guid, "GUID");
     DescribeRecord(PGOCtxProfileRecords::CalleeIndex, "CalleeIndex");
     DescribeRecord(PGOCtxProfileRecords::Counters, "Counters");
+    DescribeBlock(PGOCtxProfileBlockIDs::FlatProfilesSectionBlockID,
+                  "FlatProfiles");
+    DescribeBlock(PGOCtxProfileBlockIDs::FlatProfileBlockID, "Flat");
+    DescribeRecord(PGOCtxProfileRecords::Guid, "GUID");
+    DescribeRecord(PGOCtxProfileRecords::Counters, "Counters");
   }
   Writer.ExitBlock();
   Writer.EnterSubblock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID, CodeLen);
@@ -108,12 +114,26 @@ void PGOCtxProfileWriter::startContextSection() {
   Writer.EnterSubblock(PGOCtxProfileBlockIDs::ContextsSectionBlockID, CodeLen);
 }
 
+void PGOCtxProfileWriter::startFlatSection() {
+  Writer.EnterSubblock(PGOCtxProfileBlockIDs::FlatProfilesSectionBlockID,
+                       CodeLen);
+}
+
 void PGOCtxProfileWriter::endContextSection() { Writer.ExitBlock(); }
+void PGOCtxProfileWriter::endFlatSection() { Writer.ExitBlock(); }
 
 void PGOCtxProfileWriter::writeContextual(const ContextNode &RootNode) {
   writeImpl(std::nullopt, RootNode);
 }
 
+void PGOCtxProfileWriter::writeFlat(ctx_profile::GUID Guid,
+                                    const uint64_t *Buffer, size_t Size) {
+  Writer.EnterSubblock(PGOCtxProfileBlockIDs::FlatProfileBlockID, CodeLen);
+  writeGuid(Guid);
+  writeCounters({Buffer, Size});
+  Writer.ExitBlock();
+}
+
 namespace {
 
 /// Representation of the context node suitable for yaml serialization /
@@ -123,8 +143,13 @@ struct SerializableCtxRepresentation {
   std::vector<uint64_t> Counters;
   std::vector<std::vector<SerializableCtxRepresentation>> Callsites;
 };
+
+using SerializableFlatProfileRepresentation =
+    std::pair<ctx_profile::GUID, std::vector<uint64_t>>;
+
 struct SerializableProfileRepresentation {
   std::vector<SerializableCtxRepresentation> Contexts;
+  std::vector<SerializableFlatProfileRepresentation> FlatProfiles;
 };
 
 ctx_profile::ContextNode *
@@ -164,6 +189,7 @@ createNode(std::vector<std::unique_ptr<char[]>> &Nodes,
 
 LLVM_YAML_IS_SEQUENCE_VECTOR(SerializableCtxRepresentation)
 LLVM_YAML_IS_SEQUENCE_VECTOR(std::vector<SerializableCtxRepresentation>)
+LLVM_YAML_IS_SEQUENCE_VECTOR(SerializableFlatProfileRepresentation)
 template <> struct yaml::MappingTraits<SerializableCtxRepresentation> {
   static void mapping(yaml::IO &IO, SerializableCtxRepresentation &SCR) {
     IO.mapRequired("Guid", SCR.Guid);
@@ -175,6 +201,15 @@ template <> struct yaml::MappingTraits<SerializableCtxRepresentation> {
 template <> struct yaml::MappingTraits<SerializableProfileRepresentation> {
   static void mapping(yaml::IO &IO, SerializableProfileRepresentation &SPR) {
     IO.mapOptional("Contexts", SPR.Contexts);
+    IO.mapOptional("FlatProfiles", SPR.FlatProfiles);
+  }
+};
+
+template <> struct yaml::MappingTraits<SerializableFlatProfileRepresentation> {
+  static void mapping(yaml::IO &IO,
+                      SerializableFlatProfileRepresentation &SFPR) {
+    IO.mapRequired("Guid", SFPR.first);
+    IO.mapRequired("Counters", SFPR.second);
   }
 };
 
@@ -201,6 +236,12 @@ Error llvm::createCtxProfFromYAML(StringRef Profile, raw_ostream &Out) {
     }
     Writer.endContextSection();
   }
+  if (!SPR.FlatProfiles.empty()) {
+    Writer.startFlatSection();
+    for (const auto &[Guid, Counters] : SPR.FlatProfiles)
+      Writer.writeFlat(Guid, Counters.data(), Counters.size());
+    Writer.endFlatSection();
+  }
   if (EC)
     return createStringError(EC, "failed to write output");
   return Error::success();
diff --git a/llvm/lib/Support/AArch64AttributeParser.cpp b/llvm/lib/Support/AArch64AttributeParser.cpp
new file mode 100644
index 0000000000000..c675ef2a3b3df
--- /dev/null
+++ b/llvm/lib/Support/AArch64AttributeParser.cpp
@@ -0,0 +1,21 @@
+//===-- AArch64AttributeParser.cpp - AArch64 Build Attributes PArser------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with
+// LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "llvm/Support/AArch64AttributeParser.h"
+
+std::vector<llvm::SubsectionAndTagToTagName> &
+llvm::AArch64AttributeParser::returnTagsNamesMap() {
+  static std::vector<SubsectionAndTagToTagName> TagsNamesMap = {
+      {"aeabi_pauthabi", 1, "Tag_PAuth_Platform"},
+      {"aeabi_pauthabi", 2, "Tag_PAuth_Schema"},
+      {"aeabi_feature_and_bits", 0, "Tag_Feature_BTI"},
+      {"aeabi_feature_and_bits", 1, "Tag_Feature_PAC"},
+      {"aeabi_feature_and_bits", 2, "Tag_Feature_GCS"}};
+  return TagsNamesMap;
+}
diff --git a/llvm/lib/Support/AArch64BuildAttributes.cpp b/llvm/lib/Support/AArch64BuildAttributes.cpp
index e36667ca711e0..4a6b2fd538803 100644
--- a/llvm/lib/Support/AArch64BuildAttributes.cpp
+++ b/llvm/lib/Support/AArch64BuildAttributes.cpp
@@ -10,9 +10,9 @@
 #include "llvm/ADT/StringSwitch.h"
 
 using namespace llvm;
-using namespace llvm::AArch64BuildAttrs;
+using namespace llvm::AArch64BuildAttributes;
 
-StringRef AArch64BuildAttrs::getVendorName(unsigned Vendor) {
+StringRef AArch64BuildAttributes::getVendorName(unsigned Vendor) {
   switch (Vendor) {
   case AEABI_FEATURE_AND_BITS:
     return "aeabi_feature_and_bits";
@@ -25,14 +25,14 @@ StringRef AArch64BuildAttrs::getVendorName(unsigned Vendor) {
     return "";
   }
 }
-VendorID AArch64BuildAttrs::getVendorID(StringRef Vendor) {
+VendorID AArch64BuildAttributes::getVendorID(StringRef Vendor) {
   return StringSwitch<VendorID>(Vendor)
       .Case("aeabi_feature_and_bits", AEABI_FEATURE_AND_BITS)
       .Case("aeabi_pauthabi", AEABI_PAUTHABI)
       .Default(VENDOR_UNKNOWN);
 }
 
-StringRef AArch64BuildAttrs::getOptionalStr(unsigned Optional) {
+StringRef AArch64BuildAttributes::getOptionalStr(unsigned Optional) {
   switch (Optional) {
   case REQUIRED:
     return "required";
@@ -43,18 +43,18 @@ StringRef AArch64BuildAttrs::getOptionalStr(unsigned Optional) {
     return "";
   }
 }
-SubsectionOptional AArch64BuildAttrs::getOptionalID(StringRef Optional) {
+SubsectionOptional AArch64BuildAttributes::getOptionalID(StringRef Optional) {
   return StringSwitch<SubsectionOptional>(Optional)
       .Case("required", REQUIRED)
       .Case("optional", OPTIONAL)
       .Default(OPTIONAL_NOT_FOUND);
 }
-StringRef AArch64BuildAttrs::getSubsectionOptionalUnknownError() {
+StringRef AArch64BuildAttributes::getSubsectionOptionalUnknownError() {
   return "unknown AArch64 build attributes optionality, expected "
          "required|optional";
 }
 
-StringRef AArch64BuildAttrs::getTypeStr(unsigned Type) {
+StringRef AArch64BuildAttributes::getTypeStr(unsigned Type) {
   switch (Type) {
   case ULEB128:
     return "uleb128";
@@ -65,17 +65,17 @@ StringRef AArch64BuildAttrs::getTypeStr(unsigned Type) {
     return "";
   }
 }
-SubsectionType AArch64BuildAttrs::getTypeID(StringRef Type) {
+SubsectionType AArch64BuildAttributes::getTypeID(StringRef Type) {
   return StringSwitch<SubsectionType>(Type)
       .Cases("uleb128", "ULEB128", ULEB128)
       .Cases("ntbs", "NTBS", NTBS)
       .Default(TYPE_NOT_FOUND);
 }
-StringRef AArch64BuildAttrs::getSubsectionTypeUnknownError() {
+StringRef AArch64BuildAttributes::getSubsectionTypeUnknownError() {
   return "unknown AArch64 build attributes type, expected uleb128|ntbs";
 }
 
-StringRef AArch64BuildAttrs::getPauthABITagsStr(unsigned PauthABITag) {
+StringRef AArch64BuildAttributes::getPauthABITagsStr(unsigned PauthABITag) {
   switch (PauthABITag) {
   case TAG_PAUTH_PLATFORM:
     return "Tag_PAuth_Platform";
@@ -87,7 +87,7 @@ StringRef AArch64BuildAttrs::getPauthABITagsStr(unsigned PauthABITag) {
   }
 }
 
-PauthABITags AArch64BuildAttrs::getPauthABITagsID(StringRef PauthABITag) {
+PauthABITags AArch64BuildAttributes::getPauthABITagsID(StringRef PauthABITag) {
   return StringSwitch<PauthABITags>(PauthABITag)
       .Case("Tag_PAuth_Platform", TAG_PAUTH_PLATFORM)
       .Case("Tag_PAuth_Schema", TAG_PAUTH_SCHEMA)
@@ -95,7 +95,7 @@ PauthABITags AArch64BuildAttrs::getPauthABITagsID(StringRef PauthABITag) {
 }
 
 StringRef
-AArch64BuildAttrs::getFeatureAndBitsTagsStr(unsigned FeatureAndBitsTag) {
+AArch64BuildAttributes::getFeatureAndBitsTagsStr(unsigned FeatureAndBitsTag) {
   switch (FeatureAndBitsTag) {
   case TAG_FEATURE_BTI:
     return "Tag_Feature_BTI";
@@ -110,7 +110,7 @@ AArch64BuildAttrs::getFeatureAndBitsTagsStr(unsigned FeatureAndBitsTag) {
 }
 
 FeatureAndBitsTags
-AArch64BuildAttrs::getFeatureAndBitsTagsID(StringRef FeatureAndBitsTag) {
+AArch64BuildAttributes::getFeatureAndBitsTagsID(StringRef FeatureAndBitsTag) {
   return StringSwitch<FeatureAndBitsTags>(FeatureAndBitsTag)
       .Case("Tag_Feature_BTI", TAG_FEATURE_BTI)
       .Case("Tag_Feature_PAC", TAG_FEATURE_PAC)
diff --git a/llvm/lib/Support/APFloat.cpp b/llvm/lib/Support/APFloat.cpp
index cfc3c3b4974c5..e058010f9d267 100644
--- a/llvm/lib/Support/APFloat.cpp
+++ b/llvm/lib/Support/APFloat.cpp
@@ -1657,7 +1657,8 @@ APFloat::opStatus IEEEFloat::normalize(roundingMode rounding_mode,
   /* Before rounding normalize the exponent of fcNormal numbers.  */
   omsb = significandMSB() + 1;
 
-  if (omsb) {
+  // Only skip this `if` if the value is exactly zero.
+  if (omsb || lost_fraction != lfExactlyZero) {
     /* OMSB is numbered from 1.  We want to place it in the integer
        bit numbered PRECISION if possible, with a compensating change in
        the exponent.  */
@@ -1839,7 +1840,7 @@ APFloat::opStatus IEEEFloat::addOrSubtractSpecials(const IEEEFloat &rhs,
 /* Add or subtract two normal numbers.  */
 lostFraction IEEEFloat::addOrSubtractSignificand(const IEEEFloat &rhs,
                                                  bool subtract) {
-  integerPart carry;
+  integerPart carry = 0;
   lostFraction lost_fraction;
   int bits;
 
@@ -1857,11 +1858,13 @@ lostFraction IEEEFloat::addOrSubtractSignificand(const IEEEFloat &rhs,
           "This floating point format does not support signed values");
 
     IEEEFloat temp_rhs(rhs);
+    bool lost_fraction_is_from_rhs = false;
 
     if (bits == 0)
       lost_fraction = lfExactlyZero;
     else if (bits > 0) {
       lost_fraction = temp_rhs.shiftSignificandRight(bits - 1);
+      lost_fraction_is_from_rhs = true;
       shiftSignificandLeft(1);
     } else {
       lost_fraction = shiftSignificandRight(-bits - 1);
@@ -1869,23 +1872,40 @@ lostFraction IEEEFloat::addOrSubtractSignificand(const IEEEFloat &rhs,
     }
 
     // Should we reverse the subtraction.
-    if (compareAbsoluteValue(temp_rhs) == cmpLessThan) {
-      carry = temp_rhs.subtractSignificand
-        (*this, lost_fraction != lfExactlyZero);
+    cmpResult cmp_result = compareAbsoluteValue(temp_rhs);
+    if (cmp_result == cmpLessThan) {
+      bool borrow =
+          lost_fraction != lfExactlyZero && !lost_fraction_is_from_rhs;
+      if (borrow) {
+        // The lost fraction is being subtracted, borrow from the significand
+        // and invert `lost_fraction`.
+        if (lost_fraction == lfLessThanHalf)
+          lost_fraction = lfMoreThanHalf;
+        else if (lost_fraction == lfMoreThanHalf)
+          lost_fraction = lfLessThanHalf;
+      }
+      carry = temp_rhs.subtractSignificand(*this, borrow);
       copySignificand(temp_rhs);
       sign = !sign;
-    } else {
-      carry = subtractSignificand
-        (temp_rhs, lost_fraction != lfExactlyZero);
+    } else if (cmp_result == cmpGreaterThan) {
+      bool borrow = lost_fraction != lfExactlyZero && lost_fraction_is_from_rhs;
+      if (borrow) {
+        // The lost fraction is being subtracted, borrow from the significand
+        // and invert `lost_fraction`.
+        if (lost_fraction == lfLessThanHalf)
+          lost_fraction = lfMoreThanHalf;
+        else if (lost_fraction == lfMoreThanHalf)
+          lost_fraction = lfLessThanHalf;
+      }
+      carry = subtractSignificand(temp_rhs, borrow);
+    } else { // cmpEqual
+      zeroSignificand();
+      if (lost_fraction != lfExactlyZero && lost_fraction_is_from_rhs) {
+        // rhs is slightly larger due to the lost fraction, flip the sign.
+        sign = !sign;
+      }
     }
 
-    /* Invert the lost fraction - it was on the RHS and
-       subtracted.  */
-    if (lost_fraction == lfLessThanHalf)
-      lost_fraction = lfMoreThanHalf;
-    else if (lost_fraction == lfMoreThanHalf)
-      lost_fraction = lfLessThanHalf;
-
     /* The code above is intended to ensure that no borrow is
        necessary.  */
     assert(!carry);
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index a6d8a25818866..2754c97fce6c1 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -144,6 +144,7 @@ add_llvm_component_library(LLVMSupport
   APInt.cpp
   APSInt.cpp
   ARMBuildAttributes.cpp
+  AArch64AttributeParser.cpp
   AArch64BuildAttributes.cpp
   ARMAttributeParser.cpp
   ARMWinEH.cpp
@@ -182,8 +183,9 @@ add_llvm_component_library(LLVMSupport
   DAGDeltaAlgorithm.cpp
   DJB.cpp
   DynamicAPInt.cpp
-  ELFAttributeParser.cpp
   ELFAttributes.cpp
+  ELFAttrParserCompact.cpp
+  ELFAttrParserExtended.cpp
   Error.cpp
   ErrorHandling.cpp
   ExponentialBackoff.cpp
@@ -218,6 +220,7 @@ add_llvm_component_library(LLVMSupport
   MD5.cpp
   MSP430Attributes.cpp
   MSP430AttributeParser.cpp
+  Mustache.cpp      
   NativeFormatting.cpp
   OptimizedStructLayout.cpp
   Optional.cpp
diff --git a/llvm/lib/Support/CSKYAttributeParser.cpp b/llvm/lib/Support/CSKYAttributeParser.cpp
index 40ee617b981b2..1e46bf8127df6 100644
--- a/llvm/lib/Support/CSKYAttributeParser.cpp
+++ b/llvm/lib/Support/CSKYAttributeParser.cpp
@@ -16,19 +16,19 @@ const CSKYAttributeParser::DisplayHandler
     CSKYAttributeParser::displayRoutines[] = {
         {
             CSKYAttrs::CSKY_ARCH_NAME,
-            &ELFAttributeParser::stringAttribute,
+            &ELFCompactAttrParser::stringAttribute,
         },
         {
             CSKYAttrs::CSKY_CPU_NAME,
-            &ELFAttributeParser::stringAttribute,
+            &ELFCompactAttrParser::stringAttribute,
         },
         {
             CSKYAttrs::CSKY_ISA_FLAGS,
-            &ELFAttributeParser::integerAttribute,
+            &ELFCompactAttrParser::integerAttribute,
         },
         {
             CSKYAttrs::CSKY_ISA_EXT_FLAGS,
-            &ELFAttributeParser::integerAttribute,
+            &ELFCompactAttrParser::integerAttribute,
         },
         {
             CSKYAttrs::CSKY_DSP_VERSION,
@@ -60,7 +60,7 @@ const CSKYAttributeParser::DisplayHandler
         },
         {
             CSKYAttrs::CSKY_FPU_NUMBER_MODULE,
-            &ELFAttributeParser::stringAttribute,
+            &ELFCompactAttrParser::stringAttribute,
         },
         {
             CSKYAttrs::CSKY_FPU_HARDFP,
diff --git a/llvm/lib/Support/ELFAttributeParser.cpp b/llvm/lib/Support/ELFAttrParserCompact.cpp
similarity index 87%
rename from llvm/lib/Support/ELFAttributeParser.cpp
rename to llvm/lib/Support/ELFAttrParserCompact.cpp
index 26c3d54e17ade..14c5533678a75 100644
--- a/llvm/lib/Support/ELFAttributeParser.cpp
+++ b/llvm/lib/Support/ELFAttrParserCompact.cpp
@@ -1,12 +1,15 @@
-//===--- ELFAttributeParser.cpp - ELF Attribute Parser --------------------===//
+//===--- ELFCompactAttrParser.cpp - ELF Compact Attribute Parser ------  ===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-//===----------------------------------------------------------------------===//
+// ELF Compact Attribute Parser parse ELF build atrributes that are held
+// in a single Build Attributes Subsection.
+//
+//===--------------------------------------------------------------------===//
 
-#include "llvm/Support/ELFAttributeParser.h"
+#include "llvm/Support/ELFAttrParserCompact.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/ScopedPrinter.h"
@@ -20,8 +23,8 @@ static constexpr EnumEntry<unsigned> tagNames[] = {
     {"Tag_Symbol", ELFAttrs::Symbol},
 };
 
-Error ELFAttributeParser::parseStringAttribute(const char *name, unsigned tag,
-                                               ArrayRef<const char *> strings) {
+Error ELFCompactAttrParser::parseStringAttribute(
+    const char *name, unsigned tag, ArrayRef<const char *> strings) {
   uint64_t value = de.getULEB128(cursor);
   if (value >= strings.size()) {
     printAttribute(tag, value, "");
@@ -33,7 +36,7 @@ Error ELFAttributeParser::parseStringAttribute(const char *name, unsigned tag,
   return Error::success();
 }
 
-Error ELFAttributeParser::integerAttribute(unsigned tag) {
+Error ELFCompactAttrParser::integerAttribute(unsigned tag) {
   StringRef tagName =
       ELFAttrs::attrTypeAsString(tag, tagToStringMap, /*hasTagPrefix=*/false);
   uint64_t value = de.getULEB128(cursor);
@@ -49,7 +52,7 @@ Error ELFAttributeParser::integerAttribute(unsigned tag) {
   return Error::success();
 }
 
-Error ELFAttributeParser::stringAttribute(unsigned tag) {
+Error ELFCompactAttrParser::stringAttribute(unsigned tag) {
   StringRef tagName =
       ELFAttrs::attrTypeAsString(tag, tagToStringMap, /*hasTagPrefix=*/false);
   StringRef desc = de.getCStrRef(cursor);
@@ -65,8 +68,8 @@ Error ELFAttributeParser::stringAttribute(unsigned tag) {
   return Error::success();
 }
 
-void ELFAttributeParser::printAttribute(unsigned tag, unsigned value,
-                                        StringRef valueDesc) {
+void ELFCompactAttrParser::printAttribute(unsigned tag, unsigned value,
+                                          StringRef valueDesc) {
   attributes.insert(std::make_pair(tag, value));
 
   if (sw) {
@@ -82,7 +85,7 @@ void ELFAttributeParser::printAttribute(unsigned tag, unsigned value,
   }
 }
 
-void ELFAttributeParser::parseIndexList(SmallVectorImpl<uint8_t> &indexList) {
+void ELFCompactAttrParser::parseIndexList(SmallVectorImpl<uint8_t> &indexList) {
   for (;;) {
     uint64_t value = de.getULEB128(cursor);
     if (!cursor || !value)
@@ -91,7 +94,7 @@ void ELFAttributeParser::parseIndexList(SmallVectorImpl<uint8_t> &indexList) {
   }
 }
 
-Error ELFAttributeParser::parseAttributeList(uint32_t length) {
+Error ELFCompactAttrParser::parseAttributeList(uint32_t length) {
   uint64_t pos;
   uint64_t end = cursor.tell() + length;
   while ((pos = cursor.tell()) < end) {
@@ -119,7 +122,7 @@ Error ELFAttributeParser::parseAttributeList(uint32_t length) {
   return Error::success();
 }
 
-Error ELFAttributeParser::parseSubsection(uint32_t length) {
+Error ELFCompactAttrParser::parseSubsection(uint32_t length) {
   uint64_t end = cursor.tell() - sizeof(length) + length;
   StringRef vendorName = de.getCStrRef(cursor);
   if (sw) {
@@ -188,8 +191,8 @@ Error ELFAttributeParser::parseSubsection(uint32_t length) {
   return Error::success();
 }
 
-Error ELFAttributeParser::parse(ArrayRef<uint8_t> section,
-                                llvm::endianness endian) {
+Error ELFCompactAttrParser::parse(ArrayRef<uint8_t> section,
+                                  llvm::endianness endian) {
   unsigned sectionNumber = 0;
   de = DataExtractor(section, endian == llvm::endianness::little, 0);
 
diff --git a/llvm/lib/Support/ELFAttrParserExtended.cpp b/llvm/lib/Support/ELFAttrParserExtended.cpp
new file mode 100644
index 0000000000000..c08ed5e6fe86e
--- /dev/null
+++ b/llvm/lib/Support/ELFAttrParserExtended.cpp
@@ -0,0 +1,187 @@
+//===-ELFAttrParserExtended.cpp-ELF Extended Attribute Information Printer-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM
+// Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===------------------------------------------------------------------===//
+
+#include "llvm/Support/ELFAttrParserExtended.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/AArch64BuildAttributes.h"
+#include "llvm/Support/ELFAttributes.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ScopedPrinter.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cstdint>
+
+using namespace llvm;
+using namespace ELFAttrs;
+
+std::optional<unsigned>
+ELFExtendedAttrParser::getAttributeValue(unsigned Tag) const {
+  assert(
+      0 &&
+      "use getAttributeValue overloaded version accepting Stringref, unsigned");
+  return std::nullopt;
+}
+
+std::optional<unsigned>
+ELFExtendedAttrParser::getAttributeValue(StringRef BuildAttrSubsectionName,
+                                         unsigned Tag) const {
+  for (const auto &SubSection : SubSectionVec) {
+    if (BuildAttrSubsectionName == SubSection.Name)
+      for (const auto &BAItem : SubSection.Content) {
+        if (Tag == BAItem.Tag)
+          return std::optional<unsigned>(BAItem.IntValue);
+      }
+  }
+  return std::nullopt;
+}
+
+std::optional<StringRef>
+ELFExtendedAttrParser::getAttributeString(unsigned Tag) const {
+  assert(
+      0 &&
+      "use getAttributeValue overloaded version accepting Stringref, unsigned");
+  return std::nullopt;
+}
+
+std::optional<StringRef>
+ELFExtendedAttrParser::getAttributeString(StringRef BuildAttrSubsectionName,
+                                          unsigned Tag) const {
+  for (const auto &SubSection : SubSectionVec) {
+    if (BuildAttrSubsectionName == SubSection.Name)
+      for (const auto &BAItem : SubSection.Content) {
+        if (Tag == BAItem.Tag)
+          return std::optional<StringRef>(BAItem.StringValue);
+      }
+  }
+  return std::nullopt;
+}
+
+StringRef
+ELFExtendedAttrParser::getTagName(const StringRef &BuildAttrSubsectionName,
+                                  const unsigned Tag) {
+  for (const auto &Entry : TagsNamesMap) {
+    if (BuildAttrSubsectionName == Entry.SubsectionName)
+      if (Tag == Entry.Tag)
+        return Entry.TagName;
+  }
+  return "";
+}
+
+Error ELFExtendedAttrParser::parse(ArrayRef<uint8_t> Section,
+                                   llvm::endianness Endian) {
+
+  unsigned SectionNumber = 0;
+  De = DataExtractor(Section, Endian == llvm::endianness::little, 0);
+
+  // Early returns have specific errors. Consume the Error in Cursor.
+  struct ClearCursorError {
+    DataExtractor::Cursor &Cursor;
+    ~ClearCursorError() { consumeError(Cursor.takeError()); }
+  } Clear{Cursor};
+
+  /*
+      ELF Extended Build Attributes Layout:
+      <format-version: ‘A’> --> Currently, there is only one version: 'A' (0x41)
+      [ <uint32: subsection-length> <NTBS: vendor-name> <bytes: vendor-data> ]
+        --> subsection-length: Offset from the start of this subsection to the
+     start of the next one.
+        --> vendor-name: Null-terminated byte string.
+        --> vendor-data expands to:
+          [ <uint8: optional> <uint8: parameter type> <attribute>* ]
+            --> optional: 0 = required, 1 = optional.
+            --> parameter type: 0 = ULEB128, 1 = NTBS.
+            --> attribute: <tag, value>* pair. Tag is ULEB128, value is of
+     <parameter type>.
+  */
+
+  // Get format-version
+  uint8_t FormatVersion = De.getU8(Cursor);
+  if (ELFAttrs::Format_Version != FormatVersion)
+    return createStringError(errc::invalid_argument,
+                             "unrecognized format-version: 0x" +
+                                 utohexstr(FormatVersion));
+
+  while (!De.eof(Cursor)) {
+    uint32_t ExtBASubsectionLength = De.getU32(Cursor);
+    // Minimal valid Extended Build Attributes subsection header size is at
+    // least 8: length(4) name(at least a single char + null) optionality(1) and
+    // type(1)
+    if (ExtBASubsectionLength < 8)
+      return createStringError(
+          errc::invalid_argument,
+          "invalid Extended Build Attributes subsection size at offset: " +
+              utohexstr(Cursor.tell() - 4));
+
+    StringRef VendorName = De.getCStrRef(Cursor);
+    uint8_t IsOptional = De.getU8(Cursor);
+    StringRef IsOptionalStr = IsOptional ? "optional" : "required";
+    uint8_t Type = De.getU8(Cursor);
+    StringRef TypeStr = Type ? "ntbs" : "uleb128";
+
+    BuildAttributeSubSection BASubSection;
+    BASubSection.Name = VendorName;
+    BASubSection.IsOptional = IsOptional;
+    BASubSection.ParameterType = Type;
+
+    if (Sw) {
+      Sw->startLine() << "Section " << ++SectionNumber << " {\n";
+      Sw->indent();
+      Sw->printNumber("SectionLength", ExtBASubsectionLength);
+      Sw->startLine() << "VendorName" << ": " << VendorName
+                      << " Optionality: " << IsOptionalStr
+                      << " Type: " << TypeStr << "\n";
+      Sw->startLine() << "Attributes {\n";
+      Sw->indent();
+    }
+
+    // Offset in Section
+    uint64_t OffsetInSection = Cursor.tell();
+    // Size: 4 bytes, Vendor Name: VendorName.size() + 1 (null termination),
+    // optionality: 1, size: 1
+    uint32_t BytesAllButAttributes = 4 + (VendorName.size() + 1) + 1 + 1;
+    while (Cursor.tell() <
+           (OffsetInSection + ExtBASubsectionLength - BytesAllButAttributes)) {
+
+      uint64_t Tag = De.getULEB128(Cursor);
+
+      StringRef TagName = getTagName(VendorName, Tag);
+
+      uint64_t ValueInt = 0;
+      std::string ValueStr = "";
+      if (Type) { // type==1 --> ntbs
+        ValueStr = De.getCStrRef(Cursor);
+        if (Sw)
+          Sw->printString("" != TagName ? TagName : utostr(Tag), ValueStr);
+      } else { // type==0 --> uleb128
+        ValueInt = De.getULEB128(Cursor);
+        if (Sw)
+          Sw->printNumber("" != TagName ? TagName : utostr(Tag), ValueInt);
+      }
+
+      // populate data structure
+      BuildAttributeItem BAItem(static_cast<BuildAttributeItem::Types>(Type),
+                                Tag, ValueInt, ValueStr);
+      BASubSection.Content.push_back(BAItem);
+    }
+    if (Sw) {
+      // Close 'Attributes'
+      Sw->unindent();
+      Sw->startLine() << "}\n";
+      // Close 'Section'
+      Sw->unindent();
+      Sw->startLine() << "}\n";
+    }
+
+    // populate data structure
+    SubSectionVec.push_back(BASubSection);
+  }
+
+  return Cursor.takeError();
+}
diff --git a/llvm/lib/Support/HexagonAttributeParser.cpp b/llvm/lib/Support/HexagonAttributeParser.cpp
index 2143162d11c79..3baa976574930 100644
--- a/llvm/lib/Support/HexagonAttributeParser.cpp
+++ b/llvm/lib/Support/HexagonAttributeParser.cpp
@@ -14,31 +14,31 @@ const HexagonAttributeParser::DisplayHandler
     HexagonAttributeParser::DisplayRoutines[] = {
         {
             HexagonAttrs::ARCH,
-            &ELFAttributeParser::integerAttribute,
+            &ELFCompactAttrParser::integerAttribute,
         },
         {
             HexagonAttrs::HVXARCH,
-            &ELFAttributeParser::integerAttribute,
+            &ELFCompactAttrParser::integerAttribute,
         },
         {
             HexagonAttrs::HVXIEEEFP,
-            &ELFAttributeParser::integerAttribute,
+            &ELFCompactAttrParser::integerAttribute,
         },
         {
             HexagonAttrs::HVXQFLOAT,
-            &ELFAttributeParser::integerAttribute,
+            &ELFCompactAttrParser::integerAttribute,
         },
         {
             HexagonAttrs::ZREG,
-            &ELFAttributeParser::integerAttribute,
+            &ELFCompactAttrParser::integerAttribute,
         },
         {
             HexagonAttrs::AUDIO,
-            &ELFAttributeParser::integerAttribute,
+            &ELFCompactAttrParser::integerAttribute,
         },
         {
             HexagonAttrs::CABAC,
-            &ELFAttributeParser::integerAttribute,
+            &ELFCompactAttrParser::integerAttribute,
         }};
 
 Error HexagonAttributeParser::handler(uint64_t Tag, bool &Handled) {
diff --git a/llvm/lib/Support/JSON.cpp b/llvm/lib/Support/JSON.cpp
index a5c617bb4a076..98fef5dcf68a3 100644
--- a/llvm/lib/Support/JSON.cpp
+++ b/llvm/lib/Support/JSON.cpp
@@ -182,6 +182,10 @@ void Value::destroy() {
   }
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void Value::print(llvm::raw_ostream &OS) const { OS << *this; }
+#endif // !NDEBUG || LLVM_ENABLE_DUMP
+
 bool operator==(const Value &L, const Value &R) {
   if (L.kind() != R.kind())
     return false;
diff --git a/llvm/lib/Support/LockFileManager.cpp b/llvm/lib/Support/LockFileManager.cpp
index 9a45a9966458e..7cf9db379974f 100644
--- a/llvm/lib/Support/LockFileManager.cpp
+++ b/llvm/lib/Support/LockFileManager.cpp
@@ -51,7 +51,7 @@ using namespace llvm;
 /// \param LockFileName The name of the lock file to read.
 ///
 /// \returns The process ID of the process that owns this lock file
-std::optional<std::pair<std::string, int>>
+std::optional<LockFileManager::OwnedByAnother>
 LockFileManager::readLockFile(StringRef LockFileName) {
   // Read the owning host and PID out of the lock file. If it appears that the
   // owning process is dead, the lock file is invalid.
@@ -69,8 +69,10 @@ LockFileManager::readLockFile(StringRef LockFileName) {
   PIDStr = PIDStr.substr(PIDStr.find_first_not_of(' '));
   int PID;
   if (!PIDStr.getAsInteger(10, PID)) {
-    auto Owner = std::make_pair(std::string(Hostname), PID);
-    if (processStillExecuting(Owner.first, Owner.second))
+    OwnedByAnother Owner;
+    Owner.OwnerHostName = Hostname;
+    Owner.OwnerPID = PID;
+    if (processStillExecuting(Owner.OwnerHostName, Owner.OwnerPID))
       return Owner;
   }
 
@@ -158,41 +160,40 @@ class RemoveUniqueLockFileOnSignal {
 } // end anonymous namespace
 
 LockFileManager::LockFileManager(StringRef FileName)
-{
-  this->FileName = FileName;
-  if (std::error_code EC = sys::fs::make_absolute(this->FileName)) {
-    std::string S("failed to obtain absolute path for ");
-    S.append(std::string(this->FileName));
-    setError(EC, S);
-    return;
-  }
-  LockFileName = this->FileName;
+    : FileName(FileName), Owner(OwnerUnknown{}) {}
+
+Expected<bool> LockFileManager::tryLock() {
+  assert(std::holds_alternative<OwnerUnknown>(Owner) &&
+         "lock has already been attempted");
+
+  SmallString<128> AbsoluteFileName(FileName);
+  if (std::error_code EC = sys::fs::make_absolute(AbsoluteFileName))
+    return createStringError(EC, "failed to obtain absolute path for " +
+                                     AbsoluteFileName);
+  LockFileName = AbsoluteFileName;
   LockFileName += ".lock";
 
   // If the lock file already exists, don't bother to try to create our own
   // lock file; it won't work anyway. Just figure out who owns this lock file.
-  if ((Owner = readLockFile(LockFileName)))
-    return;
+  if (auto LockFileOwner = readLockFile(LockFileName)) {
+    Owner = std::move(*LockFileOwner);
+    return false;
+  }
 
   // Create a lock file that is unique to this instance.
   UniqueLockFileName = LockFileName;
   UniqueLockFileName += "-%%%%%%%%";
   int UniqueLockFileID;
   if (std::error_code EC = sys::fs::createUniqueFile(
-          UniqueLockFileName, UniqueLockFileID, UniqueLockFileName)) {
-    std::string S("failed to create unique file ");
-    S.append(std::string(UniqueLockFileName));
-    setError(EC, S);
-    return;
-  }
+          UniqueLockFileName, UniqueLockFileID, UniqueLockFileName))
+    return createStringError(EC, "failed to create unique file " +
+                                     UniqueLockFileName);
 
   // Write our process ID to our unique lock file.
   {
     SmallString<256> HostID;
-    if (auto EC = getHostID(HostID)) {
-      setError(EC, "failed to get host id");
-      return;
-    }
+    if (auto EC = getHostID(HostID))
+      return createStringError(EC, "failed to get host id");
 
     raw_fd_ostream Out(UniqueLockFileID, /*shouldClose=*/true);
     Out << HostID << ' ' << sys::Process::getProcessId();
@@ -201,13 +202,12 @@ LockFileManager::LockFileManager(StringRef FileName)
     if (Out.has_error()) {
       // We failed to write out PID, so report the error, remove the
       // unique lock file, and fail.
-      std::string S("failed to write to ");
-      S.append(std::string(UniqueLockFileName));
-      setError(Out.error(), S);
+      Error Err = createStringError(Out.error(),
+                                    "failed to write to " + UniqueLockFileName);
       sys::fs::remove(UniqueLockFileName);
       // Don't call report_fatal_error.
       Out.clear_error();
-      return;
+      return std::move(Err);
     }
   }
 
@@ -221,23 +221,21 @@ LockFileManager::LockFileManager(StringRef FileName)
         sys::fs::create_link(UniqueLockFileName, LockFileName);
     if (!EC) {
       RemoveUniqueFile.lockAcquired();
-      return;
+      Owner = OwnedByUs{};
+      return true;
     }
 
-    if (EC != errc::file_exists) {
-      std::string S("failed to create link ");
-      raw_string_ostream OSS(S);
-      OSS << LockFileName.str() << " to " << UniqueLockFileName.str();
-      setError(EC, S);
-      return;
-    }
+    if (EC != errc::file_exists)
+      return createStringError(EC, "failed to create link " + LockFileName +
+                                       " to " + UniqueLockFileName);
 
     // Someone else managed to create the lock file first. Read the process ID
     // from the lock file.
-    if ((Owner = readLockFile(LockFileName))) {
+    if (auto LockFileOwner = readLockFile(LockFileName)) {
       // Wipe out our unique lock file (it's useless now)
       sys::fs::remove(UniqueLockFileName);
-      return;
+      Owner = std::move(*LockFileOwner);
+      return false;
     }
 
     if (!sys::fs::exists(LockFileName)) {
@@ -248,39 +246,14 @@ LockFileManager::LockFileManager(StringRef FileName)
 
     // There is a lock file that nobody owns; try to clean it up and get
     // ownership.
-    if ((EC = sys::fs::remove(LockFileName))) {
-      std::string S("failed to remove lockfile ");
-      S.append(std::string(UniqueLockFileName));
-      setError(EC, S);
-      return;
-    }
+    if ((EC = sys::fs::remove(LockFileName)))
+      return createStringError(EC, "failed to remove lockfile " +
+                                       UniqueLockFileName);
   }
 }
 
-LockFileManager::LockFileState LockFileManager::getState() const {
-  if (Owner)
-    return LFS_Shared;
-
-  if (ErrorCode)
-    return LFS_Error;
-
-  return LFS_Owned;
-}
-
-std::string LockFileManager::getErrorMessage() const {
-  if (ErrorCode) {
-    std::string Str(ErrorDiagMsg);
-    std::string ErrCodeMsg = ErrorCode.message();
-    raw_string_ostream OSS(Str);
-    if (!ErrCodeMsg.empty())
-      OSS << ": " << ErrCodeMsg;
-    return Str;
-  }
-  return "";
-}
-
 LockFileManager::~LockFileManager() {
-  if (getState() != LFS_Owned)
+  if (!std::holds_alternative<OwnedByUs>(Owner))
     return;
 
   // Since we own the lock, remove the lock file and our own unique lock file.
@@ -293,8 +266,9 @@ LockFileManager::~LockFileManager() {
 
 LockFileManager::WaitForUnlockResult
 LockFileManager::waitForUnlock(const unsigned MaxSeconds) {
-  if (getState() != LFS_Shared)
-    return Res_Success;
+  auto *LockFileOwner = std::get_if<OwnedByAnother>(&Owner);
+  assert(LockFileOwner &&
+         "waiting for lock to be unlocked without knowing the owner");
 
   // Since we don't yet have an event-based method to wait for the lock file,
   // use randomized exponential backoff, similar to Ethernet collision
@@ -307,15 +281,12 @@ LockFileManager::waitForUnlock(const unsigned MaxSeconds) {
   while (Backoff.waitForNextAttempt()) {
     // FIXME: implement event-based waiting
     if (sys::fs::access(LockFileName.c_str(), sys::fs::AccessMode::Exist) ==
-        errc::no_such_file_or_directory) {
-      // If the original file wasn't created, somone thought the lock was dead.
-      if (!sys::fs::exists(FileName))
-        return Res_OwnerDied;
+        errc::no_such_file_or_directory)
       return Res_Success;
-    }
 
     // If the process owning the lock died without cleaning up, just bail out.
-    if (!processStillExecuting((*Owner).first, (*Owner).second))
+    if (!processStillExecuting(LockFileOwner->OwnerHostName,
+                               LockFileOwner->OwnerPID))
       return Res_OwnerDied;
   }
 
diff --git a/llvm/lib/Support/Mustache.cpp b/llvm/lib/Support/Mustache.cpp
new file mode 100644
index 0000000000000..2735bf2ca3b9b
--- /dev/null
+++ b/llvm/lib/Support/Mustache.cpp
@@ -0,0 +1,793 @@
+//===-- Mustache.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "llvm/Support/Mustache.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/raw_ostream.h"
+#include <sstream>
+
+using namespace llvm;
+using namespace llvm::mustache;
+
+namespace {
+
+using Accessor = SmallVector<std::string>;
+
+static bool isFalsey(const json::Value &V) {
+  return V.getAsNull() || (V.getAsBoolean() && !V.getAsBoolean().value()) ||
+         (V.getAsArray() && V.getAsArray()->empty());
+}
+
+static Accessor splitMustacheString(StringRef Str) {
+  // We split the mustache string into an accessor.
+  // For example:
+  //    "a.b.c" would be split into {"a", "b", "c"}
+  // We make an exception for a single dot which
+  // refers to the current context.
+  Accessor Tokens;
+  if (Str == ".") {
+    Tokens.emplace_back(Str);
+    return Tokens;
+  }
+  while (!Str.empty()) {
+    StringRef Part;
+    std::tie(Part, Str) = Str.split(".");
+    Tokens.emplace_back(Part.trim());
+  }
+  return Tokens;
+}
+} // namespace
+
+namespace llvm::mustache {
+
+class Token {
+public:
+  enum class Type {
+    Text,
+    Variable,
+    Partial,
+    SectionOpen,
+    SectionClose,
+    InvertSectionOpen,
+    UnescapeVariable,
+    Comment,
+  };
+
+  Token(std::string Str)
+      : TokenType(Type::Text), RawBody(std::move(Str)), TokenBody(RawBody),
+        AccessorValue({}), Indentation(0) {};
+
+  Token(std::string RawBody, std::string TokenBody, char Identifier)
+      : RawBody(std::move(RawBody)), TokenBody(std::move(TokenBody)),
+        Indentation(0) {
+    TokenType = getTokenType(Identifier);
+    if (TokenType == Type::Comment)
+      return;
+    StringRef AccessorStr(this->TokenBody);
+    if (TokenType != Type::Variable)
+      AccessorStr = AccessorStr.substr(1);
+    AccessorValue = splitMustacheString(StringRef(AccessorStr).trim());
+  }
+
+  Accessor getAccessor() const { return AccessorValue; }
+
+  Type getType() const { return TokenType; }
+
+  void setIndentation(size_t NewIndentation) { Indentation = NewIndentation; }
+
+  size_t getIndentation() const { return Indentation; }
+
+  static Type getTokenType(char Identifier) {
+    switch (Identifier) {
+    case '#':
+      return Type::SectionOpen;
+    case '/':
+      return Type::SectionClose;
+    case '^':
+      return Type::InvertSectionOpen;
+    case '!':
+      return Type::Comment;
+    case '>':
+      return Type::Partial;
+    case '&':
+      return Type::UnescapeVariable;
+    default:
+      return Type::Variable;
+    }
+  }
+
+  Type TokenType;
+  // RawBody is the original string that was tokenized.
+  std::string RawBody;
+  // TokenBody is the original string with the identifier removed.
+  std::string TokenBody;
+  Accessor AccessorValue;
+  size_t Indentation;
+};
+
+class ASTNode {
+public:
+  enum Type {
+    Root,
+    Text,
+    Partial,
+    Variable,
+    UnescapeVariable,
+    Section,
+    InvertSection,
+  };
+
+  ASTNode(llvm::BumpPtrAllocator &Alloc, llvm::StringMap<ASTNode *> &Partials,
+          llvm::StringMap<Lambda> &Lambdas,
+          llvm::StringMap<SectionLambda> &SectionLambdas,
+          llvm::DenseMap<char, std::string> &Escapes)
+      : Allocator(Alloc), Partials(Partials), Lambdas(Lambdas),
+        SectionLambdas(SectionLambdas), Escapes(Escapes), Ty(Type::Root),
+        Parent(nullptr), ParentContext(nullptr) {}
+
+  ASTNode(std::string Body, ASTNode *Parent, llvm::BumpPtrAllocator &Alloc,
+          llvm::StringMap<ASTNode *> &Partials,
+          llvm::StringMap<Lambda> &Lambdas,
+          llvm::StringMap<SectionLambda> &SectionLambdas,
+          llvm::DenseMap<char, std::string> &Escapes)
+      : Allocator(Alloc), Partials(Partials), Lambdas(Lambdas),
+        SectionLambdas(SectionLambdas), Escapes(Escapes), Ty(Type::Text),
+        Body(std::move(Body)), Parent(Parent), ParentContext(nullptr) {}
+
+  // Constructor for Section/InvertSection/Variable/UnescapeVariable Nodes
+  ASTNode(Type Ty, Accessor Accessor, ASTNode *Parent,
+          llvm::BumpPtrAllocator &Alloc, llvm::StringMap<ASTNode *> &Partials,
+          llvm::StringMap<Lambda> &Lambdas,
+          llvm::StringMap<SectionLambda> &SectionLambdas,
+          llvm::DenseMap<char, std::string> &Escapes)
+      : Allocator(Alloc), Partials(Partials), Lambdas(Lambdas),
+        SectionLambdas(SectionLambdas), Escapes(Escapes), Ty(Ty),
+        Parent(Parent), AccessorValue(std::move(Accessor)),
+        ParentContext(nullptr) {}
+
+  void addChild(ASTNode *Child) { Children.emplace_back(Child); };
+
+  void setRawBody(std::string NewBody) { RawBody = std::move(NewBody); };
+
+  void setIndentation(size_t NewIndentation) { Indentation = NewIndentation; };
+
+  void render(const llvm::json::Value &Data, llvm::raw_ostream &OS);
+
+private:
+  void renderLambdas(const llvm::json::Value &Contexts, llvm::raw_ostream &OS,
+                     Lambda &L);
+
+  void renderSectionLambdas(const llvm::json::Value &Contexts,
+                            llvm::raw_ostream &OS, SectionLambda &L);
+
+  void renderPartial(const llvm::json::Value &Contexts, llvm::raw_ostream &OS,
+                     ASTNode *Partial);
+
+  void renderChild(const llvm::json::Value &Context, llvm::raw_ostream &OS);
+
+  const llvm::json::Value *findContext();
+
+  llvm::BumpPtrAllocator &Allocator;
+  StringMap<ASTNode *> &Partials;
+  StringMap<Lambda> &Lambdas;
+  StringMap<SectionLambda> &SectionLambdas;
+  DenseMap<char, std::string> &Escapes;
+  Type Ty;
+  size_t Indentation = 0;
+  std::string RawBody;
+  std::string Body;
+  ASTNode *Parent;
+  // TODO: switch implementation to SmallVector<T>
+  std::vector<ASTNode *> Children;
+  const Accessor AccessorValue;
+  const llvm::json::Value *ParentContext;
+};
+
+// A wrapper for arena allocator for ASTNodes
+ASTNode *createRootNode(void *Node, llvm::BumpPtrAllocator &Alloc,
+                        llvm::StringMap<ASTNode *> &Partials,
+                        llvm::StringMap<Lambda> &Lambdas,
+                        llvm::StringMap<SectionLambda> &SectionLambdas,
+                        llvm::DenseMap<char, std::string> &Escapes) {
+  return new (Node) ASTNode(Alloc, Partials, Lambdas, SectionLambdas, Escapes);
+}
+
+ASTNode *createNode(void *Node, ASTNode::Type T, Accessor A, ASTNode *Parent,
+                    llvm::BumpPtrAllocator &Alloc,
+                    llvm::StringMap<ASTNode *> &Partials,
+                    llvm::StringMap<Lambda> &Lambdas,
+                    llvm::StringMap<SectionLambda> &SectionLambdas,
+                    llvm::DenseMap<char, std::string> &Escapes) {
+  return new (Node) ASTNode(T, std::move(A), Parent, Alloc, Partials, Lambdas,
+                            SectionLambdas, Escapes);
+}
+
+ASTNode *createTextNode(void *Node, std::string Body, ASTNode *Parent,
+                        llvm::BumpPtrAllocator &Alloc,
+                        llvm::StringMap<ASTNode *> &Partials,
+                        llvm::StringMap<Lambda> &Lambdas,
+                        llvm::StringMap<SectionLambda> &SectionLambdas,
+                        llvm::DenseMap<char, std::string> &Escapes) {
+  return new (Node) ASTNode(std::move(Body), Parent, Alloc, Partials, Lambdas,
+                            SectionLambdas, Escapes);
+}
+
+// Function to check if there is meaningful text behind.
+// We determine if a token has meaningful text behind
+// if the right of previous token contains anything that is
+// not a newline.
+// For example:
+//  "Stuff {{#Section}}" (returns true)
+//   vs
+//  "{{#Section}} \n" (returns false)
+// We make an exception for when previous token is empty
+// and the current token is the second token.
+// For example:
+//  "{{#Section}}"
+bool hasTextBehind(size_t Idx, const ArrayRef<Token> &Tokens) {
+  if (Idx == 0)
+    return true;
+
+  size_t PrevIdx = Idx - 1;
+  if (Tokens[PrevIdx].getType() != Token::Type::Text)
+    return true;
+
+  const Token &PrevToken = Tokens[PrevIdx];
+  StringRef TokenBody = StringRef(PrevToken.RawBody).rtrim(" \r\t\v");
+  return !TokenBody.ends_with("\n") && !(TokenBody.empty() && Idx == 1);
+}
+
+// Function to check if there's no meaningful text ahead.
+// We determine if a token has text ahead if the left of previous
+// token does not start with a newline.
+bool hasTextAhead(size_t Idx, const ArrayRef<Token> &Tokens) {
+  if (Idx >= Tokens.size() - 1)
+    return true;
+
+  size_t NextIdx = Idx + 1;
+  if (Tokens[NextIdx].getType() != Token::Type::Text)
+    return true;
+
+  const Token &NextToken = Tokens[NextIdx];
+  StringRef TokenBody = StringRef(NextToken.RawBody).ltrim(" ");
+  return !TokenBody.starts_with("\r\n") && !TokenBody.starts_with("\n");
+}
+
+bool requiresCleanUp(Token::Type T) {
+  // We must clean up all the tokens that could contain child nodes.
+  return T == Token::Type::SectionOpen || T == Token::Type::InvertSectionOpen ||
+         T == Token::Type::SectionClose || T == Token::Type::Comment ||
+         T == Token::Type::Partial;
+}
+
+// Adjust next token body if there is no text ahead.
+// For example:
+// The template string
+//  "{{! Comment }} \nLine 2"
+// would be considered as no text ahead and should be rendered as
+//  " Line 2"
+void stripTokenAhead(SmallVectorImpl<Token> &Tokens, size_t Idx) {
+  Token &NextToken = Tokens[Idx + 1];
+  StringRef NextTokenBody = NextToken.TokenBody;
+  // Cut off the leading newline which could be \n or \r\n.
+  if (NextTokenBody.starts_with("\r\n"))
+    NextToken.TokenBody = NextTokenBody.substr(2).str();
+  else if (NextTokenBody.starts_with("\n"))
+    NextToken.TokenBody = NextTokenBody.substr(1).str();
+}
+
+// Adjust previous token body if there no text behind.
+// For example:
+//  The template string
+//  " \t{{#section}}A{{/section}}"
+// would be considered as having no text ahead and would be render as
+//  "A"
+// The exception for this is partial tag which requires us to
+// keep track of the indentation once it's rendered.
+void stripTokenBefore(SmallVectorImpl<Token> &Tokens, size_t Idx,
+                      Token &CurrentToken, Token::Type CurrentType) {
+  Token &PrevToken = Tokens[Idx - 1];
+  StringRef PrevTokenBody = PrevToken.TokenBody;
+  StringRef Unindented = PrevTokenBody.rtrim(" \r\t\v");
+  size_t Indentation = PrevTokenBody.size() - Unindented.size();
+  if (CurrentType != Token::Type::Partial)
+    PrevToken.TokenBody = Unindented.str();
+  CurrentToken.setIndentation(Indentation);
+}
+
+// Simple tokenizer that splits the template into tokens.
+// The mustache spec allows {{{ }}} to unescape variables,
+// but we don't support that here. An unescape variable
+// is represented only by {{& variable}}.
+SmallVector<Token> tokenize(StringRef Template) {
+  SmallVector<Token> Tokens;
+  StringLiteral Open("{{");
+  StringLiteral Close("}}");
+  size_t Start = 0;
+  size_t DelimiterStart = Template.find(Open);
+  if (DelimiterStart == StringRef::npos) {
+    Tokens.emplace_back(Template.str());
+    return Tokens;
+  }
+  while (DelimiterStart != StringRef::npos) {
+    if (DelimiterStart != Start)
+      Tokens.emplace_back(Template.substr(Start, DelimiterStart - Start).str());
+    size_t DelimiterEnd = Template.find(Close, DelimiterStart);
+    if (DelimiterEnd == StringRef::npos)
+      break;
+
+    // Extract the Interpolated variable without delimiters.
+    size_t InterpolatedStart = DelimiterStart + Open.size();
+    size_t InterpolatedEnd = DelimiterEnd - DelimiterStart - Close.size();
+    std::string Interpolated =
+        Template.substr(InterpolatedStart, InterpolatedEnd).str();
+    std::string RawBody = Open.str() + Interpolated + Close.str();
+    Tokens.emplace_back(RawBody, Interpolated, Interpolated[0]);
+    Start = DelimiterEnd + Close.size();
+    DelimiterStart = Template.find(Open, Start);
+  }
+
+  if (Start < Template.size())
+    Tokens.emplace_back(Template.substr(Start).str());
+
+  // Fix up white spaces for:
+  //   - open sections
+  //   - inverted sections
+  //   - close sections
+  //   - comments
+  //
+  // This loop attempts to find standalone tokens and tries to trim out
+  // the surrounding whitespace.
+  // For example:
+  // if you have the template string
+  //  {{#section}} \n Example \n{{/section}}
+  // The output should would be
+  // For example:
+  //  \n Example \n
+  size_t LastIdx = Tokens.size() - 1;
+  for (size_t Idx = 0, End = Tokens.size(); Idx < End; ++Idx) {
+    Token &CurrentToken = Tokens[Idx];
+    Token::Type CurrentType = CurrentToken.getType();
+    // Check if token type requires cleanup.
+    bool RequiresCleanUp = requiresCleanUp(CurrentType);
+
+    if (!RequiresCleanUp)
+      continue;
+
+    // We adjust the token body if there's no text behind or ahead.
+    // A token is considered to have no text ahead if the right of the previous
+    // token is a newline followed by spaces.
+    // A token is considered to have no text behind if the left of the next
+    // token is spaces followed by a newline.
+    // eg.
+    //  "Line 1\n {{#section}} \n Line 2 \n {{/section}} \n Line 3"
+    bool HasTextBehind = hasTextBehind(Idx, Tokens);
+    bool HasTextAhead = hasTextAhead(Idx, Tokens);
+
+    if ((!HasTextAhead && !HasTextBehind) || (!HasTextAhead && Idx == 0))
+      stripTokenAhead(Tokens, Idx);
+
+    if ((!HasTextBehind && !HasTextAhead) || (!HasTextBehind && Idx == LastIdx))
+      stripTokenBefore(Tokens, Idx, CurrentToken, CurrentType);
+  }
+  return Tokens;
+}
+
+// Custom stream to escape strings.
+class EscapeStringStream : public raw_ostream {
+public:
+  explicit EscapeStringStream(llvm::raw_ostream &WrappedStream,
+                              DenseMap<char, std::string> &Escape)
+      : Escape(Escape), WrappedStream(WrappedStream) {
+    SetUnbuffered();
+  }
+
+protected:
+  void write_impl(const char *Ptr, size_t Size) override {
+    llvm::StringRef Data(Ptr, Size);
+    for (char C : Data) {
+      auto It = Escape.find(C);
+      if (It != Escape.end())
+        WrappedStream << It->getSecond();
+      else
+        WrappedStream << C;
+    }
+  }
+
+  uint64_t current_pos() const override { return WrappedStream.tell(); }
+
+private:
+  DenseMap<char, std::string> &Escape;
+  llvm::raw_ostream &WrappedStream;
+};
+
+// Custom stream to add indentation used to for rendering partials.
+class AddIndentationStringStream : public raw_ostream {
+public:
+  explicit AddIndentationStringStream(llvm::raw_ostream &WrappedStream,
+                                      size_t Indentation)
+      : Indentation(Indentation), WrappedStream(WrappedStream) {
+    SetUnbuffered();
+  }
+
+protected:
+  void write_impl(const char *Ptr, size_t Size) override {
+    llvm::StringRef Data(Ptr, Size);
+    SmallString<0> Indent;
+    Indent.resize(Indentation, ' ');
+    for (char C : Data) {
+      WrappedStream << C;
+      if (C == '\n')
+        WrappedStream << Indent;
+    }
+  }
+
+  uint64_t current_pos() const override { return WrappedStream.tell(); }
+
+private:
+  size_t Indentation;
+  llvm::raw_ostream &WrappedStream;
+};
+
+class Parser {
+public:
+  Parser(StringRef TemplateStr, BumpPtrAllocator &Allocator)
+      : ASTAllocator(Allocator), TemplateStr(TemplateStr) {}
+
+  ASTNode *parse(llvm::BumpPtrAllocator &RenderAlloc,
+                 llvm::StringMap<ASTNode *> &Partials,
+                 llvm::StringMap<Lambda> &Lambdas,
+                 llvm::StringMap<SectionLambda> &SectionLambdas,
+                 llvm::DenseMap<char, std::string> &Escapes);
+
+private:
+  void parseMustache(ASTNode *Parent, llvm::BumpPtrAllocator &Alloc,
+                     llvm::StringMap<ASTNode *> &Partials,
+                     llvm::StringMap<Lambda> &Lambdas,
+                     llvm::StringMap<SectionLambda> &SectionLambdas,
+                     llvm::DenseMap<char, std::string> &Escapes);
+
+  BumpPtrAllocator &ASTAllocator;
+  SmallVector<Token> Tokens;
+  size_t CurrentPtr;
+  StringRef TemplateStr;
+};
+
+ASTNode *Parser::parse(llvm::BumpPtrAllocator &RenderAlloc,
+                       llvm::StringMap<ASTNode *> &Partials,
+                       llvm::StringMap<Lambda> &Lambdas,
+                       llvm::StringMap<SectionLambda> &SectionLambdas,
+                       llvm::DenseMap<char, std::string> &Escapes) {
+  Tokens = tokenize(TemplateStr);
+  CurrentPtr = 0;
+  void *Root = ASTAllocator.Allocate(sizeof(ASTNode), alignof(ASTNode));
+  ASTNode *RootNode = createRootNode(Root, RenderAlloc, Partials, Lambdas,
+                                     SectionLambdas, Escapes);
+  parseMustache(RootNode, RenderAlloc, Partials, Lambdas, SectionLambdas,
+                Escapes);
+  return RootNode;
+}
+
+void Parser::parseMustache(ASTNode *Parent, llvm::BumpPtrAllocator &Alloc,
+                           llvm::StringMap<ASTNode *> &Partials,
+                           llvm::StringMap<Lambda> &Lambdas,
+                           llvm::StringMap<SectionLambda> &SectionLambdas,
+                           llvm::DenseMap<char, std::string> &Escapes) {
+
+  while (CurrentPtr < Tokens.size()) {
+    Token CurrentToken = Tokens[CurrentPtr];
+    CurrentPtr++;
+    Accessor A = CurrentToken.getAccessor();
+    ASTNode *CurrentNode;
+    void *Node = ASTAllocator.Allocate(sizeof(ASTNode), alignof(ASTNode));
+
+    switch (CurrentToken.getType()) {
+    case Token::Type::Text: {
+      CurrentNode =
+          createTextNode(Node, std::move(CurrentToken.TokenBody), Parent, Alloc,
+                         Partials, Lambdas, SectionLambdas, Escapes);
+      Parent->addChild(CurrentNode);
+      break;
+    }
+    case Token::Type::Variable: {
+      CurrentNode =
+          createNode(Node, ASTNode::Variable, std::move(A), Parent, Alloc,
+                     Partials, Lambdas, SectionLambdas, Escapes);
+      Parent->addChild(CurrentNode);
+      break;
+    }
+    case Token::Type::UnescapeVariable: {
+      CurrentNode =
+          createNode(Node, ASTNode::UnescapeVariable, std::move(A), Parent,
+                     Alloc, Partials, Lambdas, SectionLambdas, Escapes);
+      Parent->addChild(CurrentNode);
+      break;
+    }
+    case Token::Type::Partial: {
+      CurrentNode =
+          createNode(Node, ASTNode::Partial, std::move(A), Parent, Alloc,
+                     Partials, Lambdas, SectionLambdas, Escapes);
+      CurrentNode->setIndentation(CurrentToken.getIndentation());
+      Parent->addChild(CurrentNode);
+      break;
+    }
+    case Token::Type::SectionOpen: {
+      CurrentNode = createNode(Node, ASTNode::Section, A, Parent, Alloc,
+                               Partials, Lambdas, SectionLambdas, Escapes);
+      size_t Start = CurrentPtr;
+      parseMustache(CurrentNode, Alloc, Partials, Lambdas, SectionLambdas,
+                    Escapes);
+      const size_t End = CurrentPtr - 1;
+      std::string RawBody;
+      for (std::size_t I = Start; I < End; I++)
+        RawBody += Tokens[I].RawBody;
+      CurrentNode->setRawBody(std::move(RawBody));
+      Parent->addChild(CurrentNode);
+      break;
+    }
+    case Token::Type::InvertSectionOpen: {
+      CurrentNode = createNode(Node, ASTNode::InvertSection, A, Parent, Alloc,
+                               Partials, Lambdas, SectionLambdas, Escapes);
+      size_t Start = CurrentPtr;
+      parseMustache(CurrentNode, Alloc, Partials, Lambdas, SectionLambdas,
+                    Escapes);
+      const size_t End = CurrentPtr - 1;
+      std::string RawBody;
+      for (size_t Idx = Start; Idx < End; Idx++)
+        RawBody += Tokens[Idx].RawBody;
+      CurrentNode->setRawBody(std::move(RawBody));
+      Parent->addChild(CurrentNode);
+      break;
+    }
+    case Token::Type::Comment:
+      break;
+    case Token::Type::SectionClose:
+      return;
+    }
+  }
+}
+void toMustacheString(const json::Value &Data, raw_ostream &OS) {
+  switch (Data.kind()) {
+  case json::Value::Null:
+    return;
+  case json::Value::Number: {
+    auto Num = *Data.getAsNumber();
+    std::ostringstream SS;
+    SS << Num;
+    OS << SS.str();
+    return;
+  }
+  case json::Value::String: {
+    auto Str = *Data.getAsString();
+    OS << Str.str();
+    return;
+  }
+
+  case json::Value::Array: {
+    auto Arr = *Data.getAsArray();
+    if (Arr.empty())
+      return;
+    [[fallthrough]];
+  }
+  case json::Value::Object:
+  case json::Value::Boolean: {
+    llvm::json::OStream JOS(OS, 2);
+    JOS.value(Data);
+    break;
+  }
+  }
+}
+
+void ASTNode::render(const json::Value &Data, raw_ostream &OS) {
+  ParentContext = &Data;
+  const json::Value *ContextPtr = Ty == Root ? ParentContext : findContext();
+  const json::Value &Context = ContextPtr ? *ContextPtr : nullptr;
+
+  switch (Ty) {
+  case Root:
+    renderChild(Data, OS);
+    return;
+  case Text:
+    OS << Body;
+    return;
+  case Partial: {
+    auto Partial = Partials.find(AccessorValue[0]);
+    if (Partial != Partials.end())
+      renderPartial(Data, OS, Partial->getValue());
+    return;
+  }
+  case Variable: {
+    auto Lambda = Lambdas.find(AccessorValue[0]);
+    if (Lambda != Lambdas.end())
+      renderLambdas(Data, OS, Lambda->getValue());
+    else {
+      EscapeStringStream ES(OS, Escapes);
+      toMustacheString(Context, ES);
+    }
+    return;
+  }
+  case UnescapeVariable: {
+    auto Lambda = Lambdas.find(AccessorValue[0]);
+    if (Lambda != Lambdas.end())
+      renderLambdas(Data, OS, Lambda->getValue());
+    else
+      toMustacheString(Context, OS);
+    return;
+  }
+  case Section: {
+    // Sections are not rendered if the context is falsey.
+    auto SectionLambda = SectionLambdas.find(AccessorValue[0]);
+    bool IsLambda = SectionLambda != SectionLambdas.end();
+    if (isFalsey(Context) && !IsLambda)
+      return;
+
+    if (IsLambda) {
+      renderSectionLambdas(Data, OS, SectionLambda->getValue());
+      return;
+    }
+
+    if (Context.getAsArray()) {
+      const json::Array *Arr = Context.getAsArray();
+      for (const json::Value &V : *Arr)
+        renderChild(V, OS);
+      return;
+    }
+    renderChild(Context, OS);
+    return;
+  }
+  case InvertSection: {
+    bool IsLambda =
+        SectionLambdas.find(AccessorValue[0]) != SectionLambdas.end();
+    if (!isFalsey(Context) || IsLambda)
+      return;
+    renderChild(Context, OS);
+    return;
+  }
+  }
+  llvm_unreachable("Invalid ASTNode type");
+}
+
+const json::Value *ASTNode::findContext() {
+  // The mustache spec allows for dot notation to access nested values
+  // a single dot refers to the current context.
+  // We attempt to find the JSON context in the current node, if it is not
+  // found, then we traverse the parent nodes to find the context until we
+  // reach the root node or the context is found.
+  if (AccessorValue.empty())
+    return nullptr;
+  if (AccessorValue[0] == ".")
+    return ParentContext;
+
+  const json::Object *CurrentContext = ParentContext->getAsObject();
+  StringRef CurrentAccessor = AccessorValue[0];
+  ASTNode *CurrentParent = Parent;
+
+  while (!CurrentContext || !CurrentContext->get(CurrentAccessor)) {
+    if (CurrentParent->Ty != Root) {
+      CurrentContext = CurrentParent->ParentContext->getAsObject();
+      CurrentParent = CurrentParent->Parent;
+      continue;
+    }
+    return nullptr;
+  }
+  const json::Value *Context = nullptr;
+  for (auto [Idx, Acc] : enumerate(AccessorValue)) {
+    const json::Value *CurrentValue = CurrentContext->get(Acc);
+    if (!CurrentValue)
+      return nullptr;
+    if (Idx < AccessorValue.size() - 1) {
+      CurrentContext = CurrentValue->getAsObject();
+      if (!CurrentContext)
+        return nullptr;
+    } else
+      Context = CurrentValue;
+  }
+  return Context;
+}
+
+void ASTNode::renderChild(const json::Value &Contexts, llvm::raw_ostream &OS) {
+  for (ASTNode *Child : Children)
+    Child->render(Contexts, OS);
+}
+
+void ASTNode::renderPartial(const json::Value &Contexts, llvm::raw_ostream &OS,
+                            ASTNode *Partial) {
+  AddIndentationStringStream IS(OS, Indentation);
+  Partial->render(Contexts, IS);
+}
+
+void ASTNode::renderLambdas(const json::Value &Contexts, llvm::raw_ostream &OS,
+                            Lambda &L) {
+  json::Value LambdaResult = L();
+  std::string LambdaStr;
+  raw_string_ostream Output(LambdaStr);
+  toMustacheString(LambdaResult, Output);
+  Parser P = Parser(LambdaStr, Allocator);
+  ASTNode *LambdaNode =
+      P.parse(Allocator, Partials, Lambdas, SectionLambdas, Escapes);
+
+  EscapeStringStream ES(OS, Escapes);
+  if (Ty == Variable) {
+    LambdaNode->render(Contexts, ES);
+    return;
+  }
+  LambdaNode->render(Contexts, OS);
+}
+
+void ASTNode::renderSectionLambdas(const json::Value &Contexts,
+                                   llvm::raw_ostream &OS, SectionLambda &L) {
+  json::Value Return = L(RawBody);
+  if (isFalsey(Return))
+    return;
+  std::string LambdaStr;
+  raw_string_ostream Output(LambdaStr);
+  toMustacheString(Return, Output);
+  Parser P = Parser(LambdaStr, Allocator);
+  ASTNode *LambdaNode =
+      P.parse(Allocator, Partials, Lambdas, SectionLambdas, Escapes);
+  LambdaNode->render(Contexts, OS);
+  return;
+}
+
+void Template::render(const json::Value &Data, llvm::raw_ostream &OS) {
+  Tree->render(Data, OS);
+  RenderAllocator.Reset();
+}
+
+void Template::registerPartial(std::string Name, std::string Partial) {
+  Parser P = Parser(Partial, AstAllocator);
+  ASTNode *PartialTree =
+      P.parse(RenderAllocator, Partials, Lambdas, SectionLambdas, Escapes);
+  Partials.insert(std::make_pair(Name, PartialTree));
+}
+
+void Template::registerLambda(std::string Name, Lambda L) { Lambdas[Name] = L; }
+
+void Template::registerLambda(std::string Name, SectionLambda L) {
+  SectionLambdas[Name] = L;
+}
+
+void Template::overrideEscapeCharacters(DenseMap<char, std::string> E) {
+  Escapes = std::move(E);
+}
+
+Template::Template(StringRef TemplateStr) {
+  Parser P = Parser(TemplateStr, AstAllocator);
+  Tree = P.parse(RenderAllocator, Partials, Lambdas, SectionLambdas, Escapes);
+  // The default behavior is to escape html entities.
+  DenseMap<char, std::string> HtmlEntities = {{'&', "&amp;"},
+                                              {'<', "&lt;"},
+                                              {'>', "&gt;"},
+                                              {'"', "&quot;"},
+                                              {'\'', "&#39;"}};
+  overrideEscapeCharacters(HtmlEntities);
+}
+
+Template::Template(Template &&Other) noexcept
+    : Partials(std::move(Other.Partials)), Lambdas(std::move(Other.Lambdas)),
+      SectionLambdas(std::move(Other.SectionLambdas)),
+      Escapes(std::move(Other.Escapes)),
+      AstAllocator(std::move(Other.AstAllocator)),
+      RenderAllocator(std::move(Other.RenderAllocator)), Tree(Other.Tree) {
+  Other.Tree = nullptr;
+}
+
+Template &Template::operator=(Template &&Other) noexcept {
+  if (this != &Other) {
+    Partials = std::move(Other.Partials);
+    Lambdas = std::move(Other.Lambdas);
+    SectionLambdas = std::move(Other.SectionLambdas);
+    Escapes = std::move(Other.Escapes);
+    AstAllocator = std::move(Other.AstAllocator);
+    RenderAllocator = std::move(Other.RenderAllocator);
+    Tree = Other.Tree;
+    Other.Tree = nullptr;
+  }
+  return *this;
+}
+} // namespace llvm::mustache
diff --git a/llvm/lib/Support/RISCVAttributeParser.cpp b/llvm/lib/Support/RISCVAttributeParser.cpp
index 20392e4bae19f..41c48859566e4 100644
--- a/llvm/lib/Support/RISCVAttributeParser.cpp
+++ b/llvm/lib/Support/RISCVAttributeParser.cpp
@@ -15,19 +15,19 @@ const RISCVAttributeParser::DisplayHandler
     RISCVAttributeParser::displayRoutines[] = {
         {
             RISCVAttrs::ARCH,
-            &ELFAttributeParser::stringAttribute,
+            &ELFCompactAttrParser::stringAttribute,
         },
         {
             RISCVAttrs::PRIV_SPEC,
-            &ELFAttributeParser::integerAttribute,
+            &ELFCompactAttrParser::integerAttribute,
         },
         {
             RISCVAttrs::PRIV_SPEC_MINOR,
-            &ELFAttributeParser::integerAttribute,
+            &ELFCompactAttrParser::integerAttribute,
         },
         {
             RISCVAttrs::PRIV_SPEC_REVISION,
-            &ELFAttributeParser::integerAttribute,
+            &ELFCompactAttrParser::integerAttribute,
         },
         {
             RISCVAttrs::STACK_ALIGN,
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 49d48357b7fc5..47c0a1fab08f7 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -361,7 +361,7 @@ void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
   if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
           M.getModuleFlag("branch-target-enforcement"))) {
     if (!BTE->isZero()) {
-      BAFlags |= AArch64BuildAttrs::FeatureAndBitsFlag::Feature_BTI_Flag;
+      BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_BTI_Flag;
       GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
     }
   }
@@ -369,7 +369,7 @@ void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
   if (const auto *GCS = mdconst::extract_or_null<ConstantInt>(
           M.getModuleFlag("guarded-control-stack"))) {
     if (!GCS->isZero()) {
-      BAFlags |= AArch64BuildAttrs::FeatureAndBitsFlag::Feature_GCS_Flag;
+      BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_GCS_Flag;
       GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_GCS;
     }
   }
@@ -377,7 +377,7 @@ void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
   if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
           M.getModuleFlag("sign-return-address"))) {
     if (!Sign->isZero()) {
-      BAFlags |= AArch64BuildAttrs::FeatureAndBitsFlag::Feature_PAC_Flag;
+      BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_PAC_Flag;
       GNUFlags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
     }
   }
@@ -480,35 +480,42 @@ void AArch64AsmPrinter::emitAttributes(unsigned Flags,
 
   if (PAuthABIPlatform || PAuthABIVersion) {
     TS->emitAtributesSubsection(
-        AArch64BuildAttrs::getVendorName(AArch64BuildAttrs::AEABI_PAUTHABI),
-        AArch64BuildAttrs::SubsectionOptional::REQUIRED,
-        AArch64BuildAttrs::SubsectionType::ULEB128);
-    TS->emitAttribute(
-        AArch64BuildAttrs::getVendorName(AArch64BuildAttrs::AEABI_PAUTHABI),
-        AArch64BuildAttrs::TAG_PAUTH_PLATFORM, PAuthABIPlatform, "");
-    TS->emitAttribute(
-        AArch64BuildAttrs::getVendorName(AArch64BuildAttrs::AEABI_PAUTHABI),
-        AArch64BuildAttrs::TAG_PAUTH_SCHEMA, PAuthABIVersion, "");
-  }
-
-  unsigned BTIValue = (Flags & AArch64BuildAttrs::Feature_BTI_Flag) ? 1 : 0;
-  unsigned PACValue = (Flags & AArch64BuildAttrs::Feature_PAC_Flag) ? 1 : 0;
-  unsigned GCSValue = (Flags & AArch64BuildAttrs::Feature_GCS_Flag) ? 1 : 0;
+        AArch64BuildAttributes::getVendorName(
+            AArch64BuildAttributes::AEABI_PAUTHABI),
+        AArch64BuildAttributes::SubsectionOptional::REQUIRED,
+        AArch64BuildAttributes::SubsectionType::ULEB128);
+    TS->emitAttribute(AArch64BuildAttributes::getVendorName(
+                          AArch64BuildAttributes::AEABI_PAUTHABI),
+                      AArch64BuildAttributes::TAG_PAUTH_PLATFORM,
+                      PAuthABIPlatform, "");
+    TS->emitAttribute(AArch64BuildAttributes::getVendorName(
+                          AArch64BuildAttributes::AEABI_PAUTHABI),
+                      AArch64BuildAttributes::TAG_PAUTH_SCHEMA, PAuthABIVersion,
+                      "");
+  }
+
+  unsigned BTIValue =
+      (Flags & AArch64BuildAttributes::Feature_BTI_Flag) ? 1 : 0;
+  unsigned PACValue =
+      (Flags & AArch64BuildAttributes::Feature_PAC_Flag) ? 1 : 0;
+  unsigned GCSValue =
+      (Flags & AArch64BuildAttributes::Feature_GCS_Flag) ? 1 : 0;
 
   if (BTIValue || PACValue || GCSValue) {
-    TS->emitAtributesSubsection(AArch64BuildAttrs::getVendorName(
-                                    AArch64BuildAttrs::AEABI_FEATURE_AND_BITS),
-                                AArch64BuildAttrs::SubsectionOptional::OPTIONAL,
-                                AArch64BuildAttrs::SubsectionType::ULEB128);
-    TS->emitAttribute(AArch64BuildAttrs::getVendorName(
-                          AArch64BuildAttrs::AEABI_FEATURE_AND_BITS),
-                      AArch64BuildAttrs::TAG_FEATURE_BTI, BTIValue, "");
-    TS->emitAttribute(AArch64BuildAttrs::getVendorName(
-                          AArch64BuildAttrs::AEABI_FEATURE_AND_BITS),
-                      AArch64BuildAttrs::TAG_FEATURE_PAC, PACValue, "");
-    TS->emitAttribute(AArch64BuildAttrs::getVendorName(
-                          AArch64BuildAttrs::AEABI_FEATURE_AND_BITS),
-                      AArch64BuildAttrs::TAG_FEATURE_GCS, GCSValue, "");
+    TS->emitAtributesSubsection(
+        AArch64BuildAttributes::getVendorName(
+            AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
+        AArch64BuildAttributes::SubsectionOptional::OPTIONAL,
+        AArch64BuildAttributes::SubsectionType::ULEB128);
+    TS->emitAttribute(AArch64BuildAttributes::getVendorName(
+                          AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
+                      AArch64BuildAttributes::TAG_FEATURE_BTI, BTIValue, "");
+    TS->emitAttribute(AArch64BuildAttributes::getVendorName(
+                          AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
+                      AArch64BuildAttributes::TAG_FEATURE_PAC, PACValue, "");
+    TS->emitAttribute(AArch64BuildAttributes::getVendorName(
+                          AArch64BuildAttributes::AEABI_FEATURE_AND_BITS),
+                      AArch64BuildAttributes::TAG_FEATURE_GCS, GCSValue, "");
   }
 }
 
diff --git a/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
index 3ddc694cc8384..5ef439f8224c1 100644
--- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
@@ -93,7 +93,7 @@ class AArch64FastISel final : public FastISel {
       unsigned Reg;
       int FI;
     } Base;
-    unsigned OffsetReg = 0;
+    Register OffsetReg;
     unsigned Shift = 0;
     int64_t Offset = 0;
     const GlobalValue *GV = nullptr;
@@ -108,23 +108,19 @@ class AArch64FastISel final : public FastISel {
     bool isRegBase() const { return Kind == RegBase; }
     bool isFIBase() const { return Kind == FrameIndexBase; }
 
-    void setReg(unsigned Reg) {
+    void setReg(Register Reg) {
       assert(isRegBase() && "Invalid base register access!");
-      Base.Reg = Reg;
+      Base.Reg = Reg.id();
     }
 
-    unsigned getReg() const {
+    Register getReg() const {
       assert(isRegBase() && "Invalid base register access!");
       return Base.Reg;
     }
 
-    void setOffsetReg(unsigned Reg) {
-      OffsetReg = Reg;
-    }
+    void setOffsetReg(Register Reg) { OffsetReg = Reg; }
 
-    unsigned getOffsetReg() const {
-      return OffsetReg;
-    }
+    Register getOffsetReg() const { return OffsetReg; }
 
     void setFI(unsigned FI) {
       assert(isFIBase() && "Invalid base frame index  access!");
@@ -197,24 +193,23 @@ class AArch64FastISel final : public FastISel {
                          const Value *Cond);
   bool optimizeIntExtLoad(const Instruction *I, MVT RetVT, MVT SrcVT);
   bool optimizeSelect(const SelectInst *SI);
-  unsigned getRegForGEPIndex(const Value *Idx);
+  Register getRegForGEPIndex(const Value *Idx);
 
   // Emit helper routines.
-  unsigned emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,
+  Register emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,
                       const Value *RHS, bool SetFlags = false,
-                      bool WantResult = true,  bool IsZExt = false);
-  unsigned emitAddSub_rr(bool UseAdd, MVT RetVT, unsigned LHSReg,
-                         unsigned RHSReg, bool SetFlags = false,
+                      bool WantResult = true, bool IsZExt = false);
+  Register emitAddSub_rr(bool UseAdd, MVT RetVT, Register LHSReg,
+                         Register RHSReg, bool SetFlags = false,
                          bool WantResult = true);
-  unsigned emitAddSub_ri(bool UseAdd, MVT RetVT, unsigned LHSReg,
-                         uint64_t Imm, bool SetFlags = false,
-                         bool WantResult = true);
-  unsigned emitAddSub_rs(bool UseAdd, MVT RetVT, unsigned LHSReg,
-                         unsigned RHSReg, AArch64_AM::ShiftExtendType ShiftType,
+  Register emitAddSub_ri(bool UseAdd, MVT RetVT, Register LHSReg, uint64_t Imm,
+                         bool SetFlags = false, bool WantResult = true);
+  Register emitAddSub_rs(bool UseAdd, MVT RetVT, Register LHSReg,
+                         Register RHSReg, AArch64_AM::ShiftExtendType ShiftType,
                          uint64_t ShiftImm, bool SetFlags = false,
                          bool WantResult = true);
-  unsigned emitAddSub_rx(bool UseAdd, MVT RetVT, unsigned LHSReg,
-                         unsigned RHSReg, AArch64_AM::ShiftExtendType ExtType,
+  Register emitAddSub_rx(bool UseAdd, MVT RetVT, Register LHSReg,
+                         Register RHSReg, AArch64_AM::ShiftExtendType ExtType,
                          uint64_t ShiftImm, bool SetFlags = false,
                          bool WantResult = true);
 
@@ -222,51 +217,51 @@ class AArch64FastISel final : public FastISel {
   bool emitCompareAndBranch(const BranchInst *BI);
   bool emitCmp(const Value *LHS, const Value *RHS, bool IsZExt);
   bool emitICmp(MVT RetVT, const Value *LHS, const Value *RHS, bool IsZExt);
-  bool emitICmp_ri(MVT RetVT, unsigned LHSReg, uint64_t Imm);
+  bool emitICmp_ri(MVT RetVT, Register LHSReg, uint64_t Imm);
   bool emitFCmp(MVT RetVT, const Value *LHS, const Value *RHS);
-  unsigned emitLoad(MVT VT, MVT ResultVT, Address Addr, bool WantZExt = true,
+  Register emitLoad(MVT VT, MVT ResultVT, Address Addr, bool WantZExt = true,
                     MachineMemOperand *MMO = nullptr);
-  bool emitStore(MVT VT, unsigned SrcReg, Address Addr,
+  bool emitStore(MVT VT, Register SrcReg, Address Addr,
                  MachineMemOperand *MMO = nullptr);
-  bool emitStoreRelease(MVT VT, unsigned SrcReg, unsigned AddrReg,
+  bool emitStoreRelease(MVT VT, Register SrcReg, Register AddrReg,
                         MachineMemOperand *MMO = nullptr);
-  unsigned emitIntExt(MVT SrcVT, unsigned SrcReg, MVT DestVT, bool isZExt);
-  unsigned emiti1Ext(unsigned SrcReg, MVT DestVT, bool isZExt);
-  unsigned emitAdd(MVT RetVT, const Value *LHS, const Value *RHS,
+  Register emitIntExt(MVT SrcVT, Register SrcReg, MVT DestVT, bool isZExt);
+  Register emiti1Ext(Register SrcReg, MVT DestVT, bool isZExt);
+  Register emitAdd(MVT RetVT, const Value *LHS, const Value *RHS,
                    bool SetFlags = false, bool WantResult = true,
                    bool IsZExt = false);
-  unsigned emitAdd_ri_(MVT VT, unsigned Op0, int64_t Imm);
-  unsigned emitSub(MVT RetVT, const Value *LHS, const Value *RHS,
+  Register emitAdd_ri_(MVT VT, Register Op0, int64_t Imm);
+  Register emitSub(MVT RetVT, const Value *LHS, const Value *RHS,
                    bool SetFlags = false, bool WantResult = true,
                    bool IsZExt = false);
-  unsigned emitSubs_rr(MVT RetVT, unsigned LHSReg, unsigned RHSReg,
+  Register emitSubs_rr(MVT RetVT, Register LHSReg, Register RHSReg,
                        bool WantResult = true);
-  unsigned emitSubs_rs(MVT RetVT, unsigned LHSReg, unsigned RHSReg,
+  Register emitSubs_rs(MVT RetVT, Register LHSReg, Register RHSReg,
                        AArch64_AM::ShiftExtendType ShiftType, uint64_t ShiftImm,
                        bool WantResult = true);
-  unsigned emitLogicalOp(unsigned ISDOpc, MVT RetVT, const Value *LHS,
+  Register emitLogicalOp(unsigned ISDOpc, MVT RetVT, const Value *LHS,
                          const Value *RHS);
-  unsigned emitLogicalOp_ri(unsigned ISDOpc, MVT RetVT, unsigned LHSReg,
+  Register emitLogicalOp_ri(unsigned ISDOpc, MVT RetVT, Register LHSReg,
                             uint64_t Imm);
-  unsigned emitLogicalOp_rs(unsigned ISDOpc, MVT RetVT, unsigned LHSReg,
-                            unsigned RHSReg, uint64_t ShiftImm);
-  unsigned emitAnd_ri(MVT RetVT, unsigned LHSReg, uint64_t Imm);
-  unsigned emitMul_rr(MVT RetVT, unsigned Op0, unsigned Op1);
-  unsigned emitSMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1);
-  unsigned emitUMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1);
-  unsigned emitLSL_rr(MVT RetVT, unsigned Op0Reg, unsigned Op1Reg);
-  unsigned emitLSL_ri(MVT RetVT, MVT SrcVT, unsigned Op0Reg, uint64_t Imm,
+  Register emitLogicalOp_rs(unsigned ISDOpc, MVT RetVT, Register LHSReg,
+                            Register RHSReg, uint64_t ShiftImm);
+  Register emitAnd_ri(MVT RetVT, Register LHSReg, uint64_t Imm);
+  Register emitMul_rr(MVT RetVT, Register Op0, Register Op1);
+  Register emitSMULL_rr(MVT RetVT, Register Op0, Register Op1);
+  Register emitUMULL_rr(MVT RetVT, Register Op0, Register Op1);
+  Register emitLSL_rr(MVT RetVT, Register Op0Reg, Register Op1Reg);
+  Register emitLSL_ri(MVT RetVT, MVT SrcVT, Register Op0Reg, uint64_t Imm,
                       bool IsZExt = true);
-  unsigned emitLSR_rr(MVT RetVT, unsigned Op0Reg, unsigned Op1Reg);
-  unsigned emitLSR_ri(MVT RetVT, MVT SrcVT, unsigned Op0Reg, uint64_t Imm,
+  Register emitLSR_rr(MVT RetVT, Register Op0Reg, Register Op1Reg);
+  Register emitLSR_ri(MVT RetVT, MVT SrcVT, Register Op0Reg, uint64_t Imm,
                       bool IsZExt = true);
-  unsigned emitASR_rr(MVT RetVT, unsigned Op0Reg, unsigned Op1Reg);
-  unsigned emitASR_ri(MVT RetVT, MVT SrcVT, unsigned Op0Reg, uint64_t Imm,
+  Register emitASR_rr(MVT RetVT, Register Op0Reg, Register Op1Reg);
+  Register emitASR_ri(MVT RetVT, MVT SrcVT, Register Op0Reg, uint64_t Imm,
                       bool IsZExt = false);
 
-  unsigned materializeInt(const ConstantInt *CI, MVT VT);
-  unsigned materializeFP(const ConstantFP *CFP, MVT VT);
-  unsigned materializeGV(const GlobalValue *GV);
+  Register materializeInt(const ConstantInt *CI, MVT VT);
+  Register materializeFP(const ConstantFP *CFP, MVT VT);
+  Register materializeGV(const GlobalValue *GV);
 
   // Call handling routines.
 private:
@@ -351,12 +346,10 @@ Register AArch64FastISel::fastMaterializeAlloca(const AllocaInst *AI) {
          "Alloca should always return a pointer.");
 
   // Don't handle dynamic allocas.
-  if (!FuncInfo.StaticAllocaMap.count(AI))
+  auto SI = FuncInfo.StaticAllocaMap.find(AI);
+  if (SI == FuncInfo.StaticAllocaMap.end())
     return Register();
 
-  DenseMap<const AllocaInst *, int>::iterator SI =
-      FuncInfo.StaticAllocaMap.find(AI);
-
   if (SI != FuncInfo.StaticAllocaMap.end()) {
     Register ResultReg = createResultReg(&AArch64::GPR64spRegClass);
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(AArch64::ADDXri),
@@ -370,9 +363,9 @@ Register AArch64FastISel::fastMaterializeAlloca(const AllocaInst *AI) {
   return Register();
 }
 
-unsigned AArch64FastISel::materializeInt(const ConstantInt *CI, MVT VT) {
+Register AArch64FastISel::materializeInt(const ConstantInt *CI, MVT VT) {
   if (VT > MVT::i64)
-    return 0;
+    return Register();
 
   if (!CI->isZero())
     return fastEmit_i(VT, VT, ISD::Constant, CI->getZExtValue());
@@ -387,14 +380,14 @@ unsigned AArch64FastISel::materializeInt(const ConstantInt *CI, MVT VT) {
   return ResultReg;
 }
 
-unsigned AArch64FastISel::materializeFP(const ConstantFP *CFP, MVT VT) {
+Register AArch64FastISel::materializeFP(const ConstantFP *CFP, MVT VT) {
   // Positive zero (+0.0) has to be materialized with a fmov from the zero
   // register, because the immediate version of fmov cannot encode zero.
   if (CFP->isNullValue())
     return fastMaterializeFloatZero(CFP);
 
   if (VT != MVT::f32 && VT != MVT::f64)
-    return 0;
+    return Register();
 
   const APFloat Val = CFP->getValueAPF();
   bool Is64Bit = (VT == MVT::f64);
@@ -442,27 +435,27 @@ unsigned AArch64FastISel::materializeFP(const ConstantFP *CFP, MVT VT) {
   return ResultReg;
 }
 
-unsigned AArch64FastISel::materializeGV(const GlobalValue *GV) {
+Register AArch64FastISel::materializeGV(const GlobalValue *GV) {
   // We can't handle thread-local variables quickly yet.
   if (GV->isThreadLocal())
-    return 0;
+    return Register();
 
   // MachO still uses GOT for large code-model accesses, but ELF requires
   // movz/movk sequences, which FastISel doesn't handle yet.
   if (!Subtarget->useSmallAddressing() && !Subtarget->isTargetMachO())
-    return 0;
+    return Register();
 
   if (FuncInfo.MF->getInfo<AArch64FunctionInfo>()->hasELFSignedGOT())
-    return 0;
+    return Register();
 
   unsigned OpFlags = Subtarget->ClassifyGlobalReference(GV, TM);
 
   EVT DestEVT = TLI.getValueType(DL, GV->getType(), true);
   if (!DestEVT.isSimple())
-    return 0;
+    return Register();
 
   Register ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
-  unsigned ResultReg;
+  Register ResultReg;
 
   if (OpFlags & AArch64II::MO_GOT) {
     // ADRP + LDRX
@@ -516,7 +509,7 @@ unsigned AArch64FastISel::materializeGV(const GlobalValue *GV) {
       // are not exactly 1:1 with FastISel so we cannot easily abstract this
       // out. At some point, it would be nice to find a way to not have this
       // duplciate code.
-      unsigned DstReg = createResultReg(&AArch64::GPR64commonRegClass);
+      Register DstReg = createResultReg(&AArch64::GPR64commonRegClass);
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(AArch64::MOVKXi),
               DstReg)
           .addReg(ADRPReg)
@@ -979,7 +972,7 @@ bool AArch64FastISel::computeCallAddress(const Value *V, Address &Addr) {
   // If all else fails, try to materialize the value in a register.
   if (!Addr.getGlobalValue()) {
     Addr.setReg(getRegForValue(V));
-    return Addr.getReg() != 0;
+    return Addr.getReg().isValid();
   }
 
   return false;
@@ -1075,7 +1068,7 @@ bool AArch64FastISel::simplifyAddress(Address &Addr, MVT VT) {
   }
 
   if (RegisterOffsetNeedsLowering) {
-    unsigned ResultReg = 0;
+    Register ResultReg;
     if (Addr.getReg()) {
       if (Addr.getExtendType() == AArch64_AM::SXTW ||
           Addr.getExtendType() == AArch64_AM::UXTW   )
@@ -1109,7 +1102,7 @@ bool AArch64FastISel::simplifyAddress(Address &Addr, MVT VT) {
   // Since the offset is too large for the load/store instruction get the
   // reg+offset into a register.
   if (ImmediateOffsetNeedsLowering) {
-    unsigned ResultReg;
+    Register ResultReg;
     if (Addr.getReg())
       // Try to fold the immediate into the add instruction.
       ResultReg = emitAdd_ri_(MVT::i64, Addr.getReg(), Offset);
@@ -1164,14 +1157,14 @@ void AArch64FastISel::addLoadStoreOperands(Address &Addr,
     MIB.addMemOperand(MMO);
 }
 
-unsigned AArch64FastISel::emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,
+Register AArch64FastISel::emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,
                                      const Value *RHS, bool SetFlags,
-                                     bool WantResult,  bool IsZExt) {
+                                     bool WantResult, bool IsZExt) {
   AArch64_AM::ShiftExtendType ExtendType = AArch64_AM::InvalidShiftExtend;
   bool NeedExtend = false;
   switch (RetVT.SimpleTy) {
   default:
-    return 0;
+    return Register();
   case MVT::i1:
     NeedExtend = true;
     break;
@@ -1210,12 +1203,12 @@ unsigned AArch64FastISel::emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,
 
   Register LHSReg = getRegForValue(LHS);
   if (!LHSReg)
-    return 0;
+    return Register();
 
   if (NeedExtend)
     LHSReg = emitIntExt(SrcVT, LHSReg, RetVT, IsZExt);
 
-  unsigned ResultReg = 0;
+  Register ResultReg;
   if (const auto *C = dyn_cast<ConstantInt>(RHS)) {
     uint64_t Imm = IsZExt ? C->getZExtValue() : C->getSExtValue();
     if (C->isNegative())
@@ -1236,7 +1229,7 @@ unsigned AArch64FastISel::emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,
       isValueAvailable(RHS)) {
     Register RHSReg = getRegForValue(RHS);
     if (!RHSReg)
-      return 0;
+      return Register();
     return emitAddSub_rx(UseAdd, RetVT, LHSReg, RHSReg, ExtendType, 0,
                          SetFlags, WantResult);
   }
@@ -1255,7 +1248,7 @@ unsigned AArch64FastISel::emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,
       uint64_t ShiftVal = cast<ConstantInt>(MulRHS)->getValue().logBase2();
       Register RHSReg = getRegForValue(MulLHS);
       if (!RHSReg)
-        return 0;
+        return Register();
       ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, RHSReg, AArch64_AM::LSL,
                                 ShiftVal, SetFlags, WantResult);
       if (ResultReg)
@@ -1278,7 +1271,7 @@ unsigned AArch64FastISel::emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,
         if (ShiftType != AArch64_AM::InvalidShiftExtend) {
           Register RHSReg = getRegForValue(SI->getOperand(0));
           if (!RHSReg)
-            return 0;
+            return Register();
           ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, RHSReg, ShiftType,
                                     ShiftVal, SetFlags, WantResult);
           if (ResultReg)
@@ -1290,7 +1283,7 @@ unsigned AArch64FastISel::emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,
 
   Register RHSReg = getRegForValue(RHS);
   if (!RHSReg)
-    return 0;
+    return Register();
 
   if (NeedExtend)
     RHSReg = emitIntExt(SrcVT, RHSReg, RetVT, IsZExt);
@@ -1298,17 +1291,17 @@ unsigned AArch64FastISel::emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS,
   return emitAddSub_rr(UseAdd, RetVT, LHSReg, RHSReg, SetFlags, WantResult);
 }
 
-unsigned AArch64FastISel::emitAddSub_rr(bool UseAdd, MVT RetVT, unsigned LHSReg,
-                                        unsigned RHSReg, bool SetFlags,
+Register AArch64FastISel::emitAddSub_rr(bool UseAdd, MVT RetVT, Register LHSReg,
+                                        Register RHSReg, bool SetFlags,
                                         bool WantResult) {
   assert(LHSReg && RHSReg && "Invalid register number.");
 
   if (LHSReg == AArch64::SP || LHSReg == AArch64::WSP ||
       RHSReg == AArch64::SP || RHSReg == AArch64::WSP)
-    return 0;
+    return Register();
 
   if (RetVT != MVT::i32 && RetVT != MVT::i64)
-    return 0;
+    return Register();
 
   static const unsigned OpcTable[2][2][2] = {
     { { AArch64::SUBWrr,  AArch64::SUBXrr  },
@@ -1320,7 +1313,7 @@ unsigned AArch64FastISel::emitAddSub_rr(bool UseAdd, MVT RetVT, unsigned LHSReg,
   unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
   const TargetRegisterClass *RC =
       Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
-  unsigned ResultReg;
+  Register ResultReg;
   if (WantResult)
     ResultReg = createResultReg(RC);
   else
@@ -1335,13 +1328,13 @@ unsigned AArch64FastISel::emitAddSub_rr(bool UseAdd, MVT RetVT, unsigned LHSReg,
   return ResultReg;
 }
 
-unsigned AArch64FastISel::emitAddSub_ri(bool UseAdd, MVT RetVT, unsigned LHSReg,
+Register AArch64FastISel::emitAddSub_ri(bool UseAdd, MVT RetVT, Register LHSReg,
                                         uint64_t Imm, bool SetFlags,
                                         bool WantResult) {
   assert(LHSReg && "Invalid register number.");
 
   if (RetVT != MVT::i32 && RetVT != MVT::i64)
-    return 0;
+    return Register();
 
   unsigned ShiftImm;
   if (isUInt<12>(Imm))
@@ -1350,7 +1343,7 @@ unsigned AArch64FastISel::emitAddSub_ri(bool UseAdd, MVT RetVT, unsigned LHSReg,
     ShiftImm = 12;
     Imm >>= 12;
   } else
-    return 0;
+    return Register();
 
   static const unsigned OpcTable[2][2][2] = {
     { { AArch64::SUBWri,  AArch64::SUBXri  },
@@ -1365,7 +1358,7 @@ unsigned AArch64FastISel::emitAddSub_ri(bool UseAdd, MVT RetVT, unsigned LHSReg,
     RC = Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
   else
     RC = Is64Bit ? &AArch64::GPR64spRegClass : &AArch64::GPR32spRegClass;
-  unsigned ResultReg;
+  Register ResultReg;
   if (WantResult)
     ResultReg = createResultReg(RC);
   else
@@ -1380,8 +1373,8 @@ unsigned AArch64FastISel::emitAddSub_ri(bool UseAdd, MVT RetVT, unsigned LHSReg,
   return ResultReg;
 }
 
-unsigned AArch64FastISel::emitAddSub_rs(bool UseAdd, MVT RetVT, unsigned LHSReg,
-                                        unsigned RHSReg,
+Register AArch64FastISel::emitAddSub_rs(bool UseAdd, MVT RetVT, Register LHSReg,
+                                        Register RHSReg,
                                         AArch64_AM::ShiftExtendType ShiftType,
                                         uint64_t ShiftImm, bool SetFlags,
                                         bool WantResult) {
@@ -1390,11 +1383,11 @@ unsigned AArch64FastISel::emitAddSub_rs(bool UseAdd, MVT RetVT, unsigned LHSReg,
          RHSReg != AArch64::SP && RHSReg != AArch64::WSP);
 
   if (RetVT != MVT::i32 && RetVT != MVT::i64)
-    return 0;
+    return Register();
 
   // Don't deal with undefined shifts.
   if (ShiftImm >= RetVT.getSizeInBits())
-    return 0;
+    return Register();
 
   static const unsigned OpcTable[2][2][2] = {
     { { AArch64::SUBWrs,  AArch64::SUBXrs  },
@@ -1406,7 +1399,7 @@ unsigned AArch64FastISel::emitAddSub_rs(bool UseAdd, MVT RetVT, unsigned LHSReg,
   unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
   const TargetRegisterClass *RC =
       Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
-  unsigned ResultReg;
+  Register ResultReg;
   if (WantResult)
     ResultReg = createResultReg(RC);
   else
@@ -1422,8 +1415,8 @@ unsigned AArch64FastISel::emitAddSub_rs(bool UseAdd, MVT RetVT, unsigned LHSReg,
   return ResultReg;
 }
 
-unsigned AArch64FastISel::emitAddSub_rx(bool UseAdd, MVT RetVT, unsigned LHSReg,
-                                        unsigned RHSReg,
+Register AArch64FastISel::emitAddSub_rx(bool UseAdd, MVT RetVT, Register LHSReg,
+                                        Register RHSReg,
                                         AArch64_AM::ShiftExtendType ExtType,
                                         uint64_t ShiftImm, bool SetFlags,
                                         bool WantResult) {
@@ -1432,10 +1425,10 @@ unsigned AArch64FastISel::emitAddSub_rx(bool UseAdd, MVT RetVT, unsigned LHSReg,
          RHSReg != AArch64::XZR && RHSReg != AArch64::WZR);
 
   if (RetVT != MVT::i32 && RetVT != MVT::i64)
-    return 0;
+    return Register();
 
   if (ShiftImm >= 4)
-    return 0;
+    return Register();
 
   static const unsigned OpcTable[2][2][2] = {
     { { AArch64::SUBWrx,  AArch64::SUBXrx  },
@@ -1450,7 +1443,7 @@ unsigned AArch64FastISel::emitAddSub_rx(bool UseAdd, MVT RetVT, unsigned LHSReg,
     RC = Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
   else
     RC = Is64Bit ? &AArch64::GPR64spRegClass : &AArch64::GPR32spRegClass;
-  unsigned ResultReg;
+  Register ResultReg;
   if (WantResult)
     ResultReg = createResultReg(RC);
   else
@@ -1491,12 +1484,14 @@ bool AArch64FastISel::emitCmp(const Value *LHS, const Value *RHS, bool IsZExt) {
 bool AArch64FastISel::emitICmp(MVT RetVT, const Value *LHS, const Value *RHS,
                                bool IsZExt) {
   return emitSub(RetVT, LHS, RHS, /*SetFlags=*/true, /*WantResult=*/false,
-                 IsZExt) != 0;
+                 IsZExt)
+      .isValid();
 }
 
-bool AArch64FastISel::emitICmp_ri(MVT RetVT, unsigned LHSReg, uint64_t Imm) {
+bool AArch64FastISel::emitICmp_ri(MVT RetVT, Register LHSReg, uint64_t Imm) {
   return emitAddSub_ri(/*UseAdd=*/false, RetVT, LHSReg, Imm,
-                       /*SetFlags=*/true, /*WantResult=*/false) != 0;
+                       /*SetFlags=*/true, /*WantResult=*/false)
+      .isValid();
 }
 
 bool AArch64FastISel::emitFCmp(MVT RetVT, const Value *LHS, const Value *RHS) {
@@ -1532,7 +1527,7 @@ bool AArch64FastISel::emitFCmp(MVT RetVT, const Value *LHS, const Value *RHS) {
   return true;
 }
 
-unsigned AArch64FastISel::emitAdd(MVT RetVT, const Value *LHS, const Value *RHS,
+Register AArch64FastISel::emitAdd(MVT RetVT, const Value *LHS, const Value *RHS,
                                   bool SetFlags, bool WantResult, bool IsZExt) {
   return emitAddSub(/*UseAdd=*/true, RetVT, LHS, RHS, SetFlags, WantResult,
                     IsZExt);
@@ -1543,8 +1538,8 @@ unsigned AArch64FastISel::emitAdd(MVT RetVT, const Value *LHS, const Value *RHS,
 /// First try to emit an add with an immediate operand using emitAddSub_ri. If
 /// that fails, then try to materialize the immediate into a register and use
 /// emitAddSub_rr instead.
-unsigned AArch64FastISel::emitAdd_ri_(MVT VT, unsigned Op0, int64_t Imm) {
-  unsigned ResultReg;
+Register AArch64FastISel::emitAdd_ri_(MVT VT, Register Op0, int64_t Imm) {
+  Register ResultReg;
   if (Imm < 0)
     ResultReg = emitAddSub_ri(false, VT, Op0, -Imm);
   else
@@ -1553,35 +1548,35 @@ unsigned AArch64FastISel::emitAdd_ri_(MVT VT, unsigned Op0, int64_t Imm) {
   if (ResultReg)
     return ResultReg;
 
-  unsigned CReg = fastEmit_i(VT, VT, ISD::Constant, Imm);
+  Register CReg = fastEmit_i(VT, VT, ISD::Constant, Imm);
   if (!CReg)
-    return 0;
+    return Register();
 
   ResultReg = emitAddSub_rr(true, VT, Op0, CReg);
   return ResultReg;
 }
 
-unsigned AArch64FastISel::emitSub(MVT RetVT, const Value *LHS, const Value *RHS,
+Register AArch64FastISel::emitSub(MVT RetVT, const Value *LHS, const Value *RHS,
                                   bool SetFlags, bool WantResult, bool IsZExt) {
   return emitAddSub(/*UseAdd=*/false, RetVT, LHS, RHS, SetFlags, WantResult,
                     IsZExt);
 }
 
-unsigned AArch64FastISel::emitSubs_rr(MVT RetVT, unsigned LHSReg,
-                                      unsigned RHSReg, bool WantResult) {
+Register AArch64FastISel::emitSubs_rr(MVT RetVT, Register LHSReg,
+                                      Register RHSReg, bool WantResult) {
   return emitAddSub_rr(/*UseAdd=*/false, RetVT, LHSReg, RHSReg,
                        /*SetFlags=*/true, WantResult);
 }
 
-unsigned AArch64FastISel::emitSubs_rs(MVT RetVT, unsigned LHSReg,
-                                      unsigned RHSReg,
+Register AArch64FastISel::emitSubs_rs(MVT RetVT, Register LHSReg,
+                                      Register RHSReg,
                                       AArch64_AM::ShiftExtendType ShiftType,
                                       uint64_t ShiftImm, bool WantResult) {
   return emitAddSub_rs(/*UseAdd=*/false, RetVT, LHSReg, RHSReg, ShiftType,
                        ShiftImm, /*SetFlags=*/true, WantResult);
 }
 
-unsigned AArch64FastISel::emitLogicalOp(unsigned ISDOpc, MVT RetVT,
+Register AArch64FastISel::emitLogicalOp(unsigned ISDOpc, MVT RetVT,
                                         const Value *LHS, const Value *RHS) {
   // Canonicalize immediates to the RHS first.
   if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS))
@@ -1600,9 +1595,9 @@ unsigned AArch64FastISel::emitLogicalOp(unsigned ISDOpc, MVT RetVT,
 
   Register LHSReg = getRegForValue(LHS);
   if (!LHSReg)
-    return 0;
+    return Register();
 
-  unsigned ResultReg = 0;
+  Register ResultReg;
   if (const auto *C = dyn_cast<ConstantInt>(RHS)) {
     uint64_t Imm = C->getZExtValue();
     ResultReg = emitLogicalOp_ri(ISDOpc, RetVT, LHSReg, Imm);
@@ -1625,7 +1620,7 @@ unsigned AArch64FastISel::emitLogicalOp(unsigned ISDOpc, MVT RetVT,
 
       Register RHSReg = getRegForValue(MulLHS);
       if (!RHSReg)
-        return 0;
+        return Register();
       ResultReg = emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, RHSReg, ShiftVal);
       if (ResultReg)
         return ResultReg;
@@ -1639,7 +1634,7 @@ unsigned AArch64FastISel::emitLogicalOp(unsigned ISDOpc, MVT RetVT,
         uint64_t ShiftVal = C->getZExtValue();
         Register RHSReg = getRegForValue(SI->getOperand(0));
         if (!RHSReg)
-          return 0;
+          return Register();
         ResultReg = emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, RHSReg, ShiftVal);
         if (ResultReg)
           return ResultReg;
@@ -1648,7 +1643,7 @@ unsigned AArch64FastISel::emitLogicalOp(unsigned ISDOpc, MVT RetVT,
 
   Register RHSReg = getRegForValue(RHS);
   if (!RHSReg)
-    return 0;
+    return Register();
 
   MVT VT = std::max(MVT::i32, RetVT.SimpleTy);
   ResultReg = fastEmit_rr(VT, VT, ISDOpc, LHSReg, RHSReg);
@@ -1659,8 +1654,8 @@ unsigned AArch64FastISel::emitLogicalOp(unsigned ISDOpc, MVT RetVT,
   return ResultReg;
 }
 
-unsigned AArch64FastISel::emitLogicalOp_ri(unsigned ISDOpc, MVT RetVT,
-                                           unsigned LHSReg, uint64_t Imm) {
+Register AArch64FastISel::emitLogicalOp_ri(unsigned ISDOpc, MVT RetVT,
+                                           Register LHSReg, uint64_t Imm) {
   static_assert((ISD::AND + 1 == ISD::OR) && (ISD::AND + 2 == ISD::XOR),
                 "ISD nodes are not consecutive!");
   static const unsigned OpcTable[3][2] = {
@@ -1673,7 +1668,7 @@ unsigned AArch64FastISel::emitLogicalOp_ri(unsigned ISDOpc, MVT RetVT,
   unsigned RegSize;
   switch (RetVT.SimpleTy) {
   default:
-    return 0;
+    return Register();
   case MVT::i1:
   case MVT::i8:
   case MVT::i16:
@@ -1692,7 +1687,7 @@ unsigned AArch64FastISel::emitLogicalOp_ri(unsigned ISDOpc, MVT RetVT,
   }
 
   if (!AArch64_AM::isLogicalImmediate(Imm, RegSize))
-    return 0;
+    return Register();
 
   Register ResultReg =
       fastEmitInst_ri(Opc, RC, LHSReg,
@@ -1704,8 +1699,8 @@ unsigned AArch64FastISel::emitLogicalOp_ri(unsigned ISDOpc, MVT RetVT,
   return ResultReg;
 }
 
-unsigned AArch64FastISel::emitLogicalOp_rs(unsigned ISDOpc, MVT RetVT,
-                                           unsigned LHSReg, unsigned RHSReg,
+Register AArch64FastISel::emitLogicalOp_rs(unsigned ISDOpc, MVT RetVT,
+                                           Register LHSReg, Register RHSReg,
                                            uint64_t ShiftImm) {
   static_assert((ISD::AND + 1 == ISD::OR) && (ISD::AND + 2 == ISD::XOR),
                 "ISD nodes are not consecutive!");
@@ -1717,13 +1712,13 @@ unsigned AArch64FastISel::emitLogicalOp_rs(unsigned ISDOpc, MVT RetVT,
 
   // Don't deal with undefined shifts.
   if (ShiftImm >= RetVT.getSizeInBits())
-    return 0;
+    return Register();
 
   const TargetRegisterClass *RC;
   unsigned Opc;
   switch (RetVT.SimpleTy) {
   default:
-    return 0;
+    return Register();
   case MVT::i1:
   case MVT::i8:
   case MVT::i16:
@@ -1746,19 +1741,18 @@ unsigned AArch64FastISel::emitLogicalOp_rs(unsigned ISDOpc, MVT RetVT,
   return ResultReg;
 }
 
-unsigned AArch64FastISel::emitAnd_ri(MVT RetVT, unsigned LHSReg,
-                                     uint64_t Imm) {
+Register AArch64FastISel::emitAnd_ri(MVT RetVT, Register LHSReg, uint64_t Imm) {
   return emitLogicalOp_ri(ISD::AND, RetVT, LHSReg, Imm);
 }
 
-unsigned AArch64FastISel::emitLoad(MVT VT, MVT RetVT, Address Addr,
+Register AArch64FastISel::emitLoad(MVT VT, MVT RetVT, Address Addr,
                                    bool WantZExt, MachineMemOperand *MMO) {
   if (!TLI.allowsMisalignedMemoryAccesses(VT))
-    return 0;
+    return Register();
 
   // Simplify this down to something we can handle.
   if (!simplifyAddress(Addr, VT))
-    return 0;
+    return Register();
 
   unsigned ScaleFactor = getImplicitScaleFactor(VT);
   if (!ScaleFactor)
@@ -1869,7 +1863,7 @@ unsigned AArch64FastISel::emitLoad(MVT VT, MVT RetVT, Address Addr,
 
   // Loading an i1 requires special handling.
   if (VT == MVT::i1) {
-    unsigned ANDReg = emitAnd_ri(MVT::i32, ResultReg, 1);
+    Register ANDReg = emitAnd_ri(MVT::i32, ResultReg, 1);
     assert(ANDReg && "Unexpected AND instruction emission failure.");
     ResultReg = ANDReg;
   }
@@ -1896,7 +1890,7 @@ bool AArch64FastISel::selectAddSub(const Instruction *I) {
   if (VT.isVector())
     return selectOperator(I, I->getOpcode());
 
-  unsigned ResultReg;
+  Register ResultReg;
   switch (I->getOpcode()) {
   default:
     llvm_unreachable("Unexpected instruction.");
@@ -1922,7 +1916,7 @@ bool AArch64FastISel::selectLogicalOp(const Instruction *I) {
   if (VT.isVector())
     return selectOperator(I, I->getOpcode());
 
-  unsigned ResultReg;
+  Register ResultReg;
   switch (I->getOpcode()) {
   default:
     llvm_unreachable("Unexpected instruction.");
@@ -1991,7 +1985,7 @@ bool AArch64FastISel::selectLoad(const Instruction *I) {
     }
   }
 
-  unsigned ResultReg =
+  Register ResultReg =
       emitLoad(VT, RetVT, Addr, WantZExt, createMachineMemOperandFor(I));
   if (!ResultReg)
     return false;
@@ -2054,8 +2048,8 @@ bool AArch64FastISel::selectLoad(const Instruction *I) {
   return true;
 }
 
-bool AArch64FastISel::emitStoreRelease(MVT VT, unsigned SrcReg,
-                                       unsigned AddrReg,
+bool AArch64FastISel::emitStoreRelease(MVT VT, Register SrcReg,
+                                       Register AddrReg,
                                        MachineMemOperand *MMO) {
   unsigned Opc;
   switch (VT.SimpleTy) {
@@ -2076,7 +2070,7 @@ bool AArch64FastISel::emitStoreRelease(MVT VT, unsigned SrcReg,
   return true;
 }
 
-bool AArch64FastISel::emitStore(MVT VT, unsigned SrcReg, Address Addr,
+bool AArch64FastISel::emitStore(MVT VT, Register SrcReg, Address Addr,
                                 MachineMemOperand *MMO) {
   if (!TLI.allowsMisalignedMemoryAccesses(VT))
     return false;
@@ -2130,7 +2124,7 @@ bool AArch64FastISel::emitStore(MVT VT, unsigned SrcReg, Address Addr,
 
   // Storing an i1 requires special handling.
   if (VTIsi1 && SrcReg != AArch64::WZR) {
-    unsigned ANDReg = emitAnd_ri(MVT::i32, SrcReg, 1);
+    Register ANDReg = emitAnd_ri(MVT::i32, SrcReg, 1);
     assert(ANDReg && "Unexpected AND instruction emission failure.");
     SrcReg = ANDReg;
   }
@@ -2170,7 +2164,7 @@ bool AArch64FastISel::selectStore(const Instruction *I) {
 
   // Get the value to be stored into a register. Use the zero register directly
   // when possible to avoid an unnecessary copy and a wasted register.
-  unsigned SrcReg = 0;
+  Register SrcReg;
   if (const auto *CI = dyn_cast<ConstantInt>(Op0)) {
     if (CI->isZero())
       SrcReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR;
@@ -2490,7 +2484,7 @@ bool AArch64FastISel::selectBranch(const Instruction *I) {
   }
 
   Register CondReg = getRegForValue(BI->getCondition());
-  if (CondReg == 0)
+  if (!CondReg)
     return false;
 
   // i1 conditions come as i32 values, test the lowest bit with tb(n)z.
@@ -2515,7 +2509,7 @@ bool AArch64FastISel::selectBranch(const Instruction *I) {
 bool AArch64FastISel::selectIndirectBr(const Instruction *I) {
   const IndirectBrInst *BI = cast<IndirectBrInst>(I);
   Register AddrReg = getRegForValue(BI->getOperand(0));
-  if (AddrReg == 0)
+  if (!AddrReg)
     return false;
 
   // Authenticated indirectbr is not implemented yet.
@@ -2543,7 +2537,7 @@ bool AArch64FastISel::selectCmp(const Instruction *I) {
 
   // Try to optimize or fold the cmp.
   CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
-  unsigned ResultReg = 0;
+  Register ResultReg;
   switch (Predicate) {
   default:
     break;
@@ -2831,7 +2825,7 @@ bool AArch64FastISel::selectFPToInt(const Instruction *I, bool Signed) {
     return false;
 
   Register SrcReg = getRegForValue(I->getOperand(0));
-  if (SrcReg == 0)
+  if (!SrcReg)
     return false;
 
   EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType(), true);
@@ -3120,7 +3114,7 @@ bool AArch64FastISel::finishCall(CallLoweringInfo &CLI, unsigned NumBytes) {
   for (unsigned i = 0; i != RVLocs.size(); ++i) {
     CCValAssign &VA = RVLocs[i];
     MVT CopyVT = VA.getValVT();
-    unsigned CopyReg = ResultReg + i;
+    Register CopyReg = ResultReg + i;
 
     // TODO: Handle big-endian results
     if (CopyVT.isVector() && !Subtarget->isLittleEndian())
@@ -3249,7 +3243,7 @@ bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
     } else
       return false;
   } else {
-    unsigned CallReg = 0;
+    Register CallReg;
     if (Symbol) {
       Register ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(AArch64::ADRP),
@@ -3330,7 +3324,7 @@ bool AArch64FastISel::tryEmitSmallMemCpy(Address Dest, Address Src,
       }
     }
 
-    unsigned ResultReg = emitLoad(VT, VT, Src);
+    Register ResultReg = emitLoad(VT, VT, Src);
     if (!ResultReg)
       return false;
 
@@ -3457,7 +3451,7 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
     // ldr x0, [x0]
     // ldr x0, [x0]
     // ...
-    unsigned DestReg;
+    Register DestReg;
     unsigned Depth = cast<ConstantInt>(II->getOperand(0))->getZExtValue();
     while (Depth--) {
       DestReg = fastEmitInst_ri(AArch64::LDRXui, &AArch64::GPR64RegClass,
@@ -3639,7 +3633,7 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
     if (!Op0Reg)
       return false;
 
-    unsigned ResultReg = fastEmit_r(VT, VT, ISD::FSQRT, Op0Reg);
+    Register ResultReg = fastEmit_r(VT, VT, ISD::FSQRT, Op0Reg);
     if (!ResultReg)
       return false;
 
@@ -3691,7 +3685,7 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
       break;
     }
 
-    unsigned ResultReg1 = 0, ResultReg2 = 0, MulReg = 0;
+    Register ResultReg1, ResultReg2, MulReg;
     AArch64CC::CondCode CC = AArch64CC::Invalid;
     switch (IID) {
     default: llvm_unreachable("Unexpected intrinsic!");
@@ -3735,7 +3729,7 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
         // LHSReg and RHSReg cannot be killed by this Mul, since they are
         // reused in the next instruction.
         MulReg = emitMul_rr(VT, LHSReg, RHSReg);
-        unsigned SMULHReg = fastEmit_rr(VT, VT, ISD::MULHS, LHSReg, RHSReg);
+        Register SMULHReg = fastEmit_rr(VT, VT, ISD::MULHS, LHSReg, RHSReg);
         emitSubs_rs(VT, SMULHReg, MulReg, AArch64_AM::ASR, 63,
                     /*WantResult=*/false);
       }
@@ -3764,7 +3758,7 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
         // LHSReg and RHSReg cannot be killed by this Mul, since they are
         // reused in the next instruction.
         MulReg = emitMul_rr(VT, LHSReg, RHSReg);
-        unsigned UMULHReg = fastEmit_rr(VT, VT, ISD::MULHU, LHSReg, RHSReg);
+        Register UMULHReg = fastEmit_rr(VT, VT, ISD::MULHU, LHSReg, RHSReg);
         emitSubs_rr(VT, AArch64::XZR, UMULHReg, /*WantResult=*/false);
       }
       break;
@@ -3862,7 +3856,7 @@ bool AArch64FastISel::selectRet(const Instruction *I) {
     return false;
 
   // Build a list of return value registers.
-  SmallVector<unsigned, 4> RetRegs;
+  SmallVector<Register, 4> RetRegs;
 
   if (Ret->getNumOperands() > 0) {
     CallingConv::ID CC = F.getCallingConv();
@@ -3891,10 +3885,10 @@ bool AArch64FastISel::selectRet(const Instruction *I) {
       return false;
 
     Register Reg = getRegForValue(RV);
-    if (Reg == 0)
+    if (!Reg)
       return false;
 
-    unsigned SrcReg = Reg + VA.getValNo();
+    Register SrcReg = Reg + VA.getValNo();
     Register DestReg = VA.getLocReg();
     // Avoid a cross-class copy. This is very unlikely.
     if (!MRI.getRegClass(SrcReg)->contains(DestReg))
@@ -3924,7 +3918,7 @@ bool AArch64FastISel::selectRet(const Instruction *I) {
 
       bool IsZExt = Outs[0].Flags.isZExt();
       SrcReg = emitIntExt(RVVT, SrcReg, DestVT, IsZExt);
-      if (SrcReg == 0)
+      if (!SrcReg)
         return false;
     }
 
@@ -3943,7 +3937,7 @@ bool AArch64FastISel::selectRet(const Instruction *I) {
 
   MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
                                     TII.get(AArch64::RET_ReallyLR));
-  for (unsigned RetReg : RetRegs)
+  for (Register RetReg : RetRegs)
     MIB.addReg(RetReg, RegState::Implicit);
   return true;
 }
@@ -3979,7 +3973,7 @@ bool AArch64FastISel::selectTrunc(const Instruction *I) {
   // generate a COPY. We cannot mark the source register also as result
   // register, because this can incorrectly transfer the kill flag onto the
   // source register.
-  unsigned ResultReg;
+  Register ResultReg;
   if (SrcVT == MVT::i64) {
     uint64_t Mask = 0;
     switch (DestVT.SimpleTy) {
@@ -4013,7 +4007,7 @@ bool AArch64FastISel::selectTrunc(const Instruction *I) {
   return true;
 }
 
-unsigned AArch64FastISel::emiti1Ext(unsigned SrcReg, MVT DestVT, bool IsZExt) {
+Register AArch64FastISel::emiti1Ext(Register SrcReg, MVT DestVT, bool IsZExt) {
   assert((DestVT == MVT::i8 || DestVT == MVT::i16 || DestVT == MVT::i32 ||
           DestVT == MVT::i64) &&
          "Unexpected value type.");
@@ -4022,7 +4016,7 @@ unsigned AArch64FastISel::emiti1Ext(unsigned SrcReg, MVT DestVT, bool IsZExt) {
     DestVT = MVT::i32;
 
   if (IsZExt) {
-    unsigned ResultReg = emitAnd_ri(MVT::i32, SrcReg, 1);
+    Register ResultReg = emitAnd_ri(MVT::i32, SrcReg, 1);
     assert(ResultReg && "Unexpected AND instruction emission failure.");
     if (DestVT == MVT::i64) {
       // We're ZExt i1 to i64.  The ANDWri Wd, Ws, #1 implicitly clears the
@@ -4039,17 +4033,19 @@ unsigned AArch64FastISel::emiti1Ext(unsigned SrcReg, MVT DestVT, bool IsZExt) {
   } else {
     if (DestVT == MVT::i64) {
       // FIXME: We're SExt i1 to i64.
-      return 0;
+      return Register();
     }
     return fastEmitInst_rii(AArch64::SBFMWri, &AArch64::GPR32RegClass, SrcReg,
                             0, 0);
   }
 }
 
-unsigned AArch64FastISel::emitMul_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
-  unsigned Opc, ZReg;
+Register AArch64FastISel::emitMul_rr(MVT RetVT, Register Op0, Register Op1) {
+  unsigned Opc;
+  Register ZReg;
   switch (RetVT.SimpleTy) {
-  default: return 0;
+  default:
+    return Register();
   case MVT::i8:
   case MVT::i16:
   case MVT::i32:
@@ -4064,29 +4060,30 @@ unsigned AArch64FastISel::emitMul_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
   return fastEmitInst_rrr(Opc, RC, Op0, Op1, ZReg);
 }
 
-unsigned AArch64FastISel::emitSMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
+Register AArch64FastISel::emitSMULL_rr(MVT RetVT, Register Op0, Register Op1) {
   if (RetVT != MVT::i64)
-    return 0;
+    return Register();
 
   return fastEmitInst_rrr(AArch64::SMADDLrrr, &AArch64::GPR64RegClass,
                           Op0, Op1, AArch64::XZR);
 }
 
-unsigned AArch64FastISel::emitUMULL_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
+Register AArch64FastISel::emitUMULL_rr(MVT RetVT, Register Op0, Register Op1) {
   if (RetVT != MVT::i64)
-    return 0;
+    return Register();
 
   return fastEmitInst_rrr(AArch64::UMADDLrrr, &AArch64::GPR64RegClass,
                           Op0, Op1, AArch64::XZR);
 }
 
-unsigned AArch64FastISel::emitLSL_rr(MVT RetVT, unsigned Op0Reg,
-                                     unsigned Op1Reg) {
+Register AArch64FastISel::emitLSL_rr(MVT RetVT, Register Op0Reg,
+                                     Register Op1Reg) {
   unsigned Opc = 0;
   bool NeedTrunc = false;
   uint64_t Mask = 0;
   switch (RetVT.SimpleTy) {
-  default: return 0;
+  default:
+    return Register();
   case MVT::i8:  Opc = AArch64::LSLVWr; NeedTrunc = true; Mask = 0xff;   break;
   case MVT::i16: Opc = AArch64::LSLVWr; NeedTrunc = true; Mask = 0xffff; break;
   case MVT::i32: Opc = AArch64::LSLVWr;                                  break;
@@ -4104,7 +4101,7 @@ unsigned AArch64FastISel::emitLSL_rr(MVT RetVT, unsigned Op0Reg,
   return ResultReg;
 }
 
-unsigned AArch64FastISel::emitLSL_ri(MVT RetVT, MVT SrcVT, unsigned Op0,
+Register AArch64FastISel::emitLSL_ri(MVT RetVT, MVT SrcVT, Register Op0,
                                      uint64_t Shift, bool IsZExt) {
   assert(RetVT.SimpleTy >= SrcVT.SimpleTy &&
          "Unexpected source/return type pair.");
@@ -4135,7 +4132,7 @@ unsigned AArch64FastISel::emitLSL_ri(MVT RetVT, MVT SrcVT, unsigned Op0,
 
   // Don't deal with undefined shifts.
   if (Shift >= DstBits)
-    return 0;
+    return Register();
 
   // For immediate shifts we can fold the zero-/sign-extension into the shift.
   // {S|U}BFM Wd, Wn, #r, #s
@@ -4182,13 +4179,14 @@ unsigned AArch64FastISel::emitLSL_ri(MVT RetVT, MVT SrcVT, unsigned Op0,
   return fastEmitInst_rii(Opc, RC, Op0, ImmR, ImmS);
 }
 
-unsigned AArch64FastISel::emitLSR_rr(MVT RetVT, unsigned Op0Reg,
-                                     unsigned Op1Reg) {
+Register AArch64FastISel::emitLSR_rr(MVT RetVT, Register Op0Reg,
+                                     Register Op1Reg) {
   unsigned Opc = 0;
   bool NeedTrunc = false;
   uint64_t Mask = 0;
   switch (RetVT.SimpleTy) {
-  default: return 0;
+  default:
+    return Register();
   case MVT::i8:  Opc = AArch64::LSRVWr; NeedTrunc = true; Mask = 0xff;   break;
   case MVT::i16: Opc = AArch64::LSRVWr; NeedTrunc = true; Mask = 0xffff; break;
   case MVT::i32: Opc = AArch64::LSRVWr; break;
@@ -4207,7 +4205,7 @@ unsigned AArch64FastISel::emitLSR_rr(MVT RetVT, unsigned Op0Reg,
   return ResultReg;
 }
 
-unsigned AArch64FastISel::emitLSR_ri(MVT RetVT, MVT SrcVT, unsigned Op0,
+Register AArch64FastISel::emitLSR_ri(MVT RetVT, MVT SrcVT, Register Op0,
                                      uint64_t Shift, bool IsZExt) {
   assert(RetVT.SimpleTy >= SrcVT.SimpleTy &&
          "Unexpected source/return type pair.");
@@ -4238,7 +4236,7 @@ unsigned AArch64FastISel::emitLSR_ri(MVT RetVT, MVT SrcVT, unsigned Op0,
 
   // Don't deal with undefined shifts.
   if (Shift >= DstBits)
-    return 0;
+    return Register();
 
   // For immediate shifts we can fold the zero-/sign-extension into the shift.
   // {S|U}BFM Wd, Wn, #r, #s
@@ -4273,7 +4271,7 @@ unsigned AArch64FastISel::emitLSR_ri(MVT RetVT, MVT SrcVT, unsigned Op0,
   if (!IsZExt) {
     Op0 = emitIntExt(SrcVT, Op0, RetVT, IsZExt);
     if (!Op0)
-      return 0;
+      return Register();
     SrcVT = RetVT;
     SrcBits = SrcVT.getSizeInBits();
     IsZExt = true;
@@ -4298,13 +4296,14 @@ unsigned AArch64FastISel::emitLSR_ri(MVT RetVT, MVT SrcVT, unsigned Op0,
   return fastEmitInst_rii(Opc, RC, Op0, ImmR, ImmS);
 }
 
-unsigned AArch64FastISel::emitASR_rr(MVT RetVT, unsigned Op0Reg,
-                                     unsigned Op1Reg) {
+Register AArch64FastISel::emitASR_rr(MVT RetVT, Register Op0Reg,
+                                     Register Op1Reg) {
   unsigned Opc = 0;
   bool NeedTrunc = false;
   uint64_t Mask = 0;
   switch (RetVT.SimpleTy) {
-  default: return 0;
+  default:
+    return Register();
   case MVT::i8:  Opc = AArch64::ASRVWr; NeedTrunc = true; Mask = 0xff;   break;
   case MVT::i16: Opc = AArch64::ASRVWr; NeedTrunc = true; Mask = 0xffff; break;
   case MVT::i32: Opc = AArch64::ASRVWr;                                  break;
@@ -4323,7 +4322,7 @@ unsigned AArch64FastISel::emitASR_rr(MVT RetVT, unsigned Op0Reg,
   return ResultReg;
 }
 
-unsigned AArch64FastISel::emitASR_ri(MVT RetVT, MVT SrcVT, unsigned Op0,
+Register AArch64FastISel::emitASR_ri(MVT RetVT, MVT SrcVT, Register Op0,
                                      uint64_t Shift, bool IsZExt) {
   assert(RetVT.SimpleTy >= SrcVT.SimpleTy &&
          "Unexpected source/return type pair.");
@@ -4354,7 +4353,7 @@ unsigned AArch64FastISel::emitASR_ri(MVT RetVT, MVT SrcVT, unsigned Op0,
 
   // Don't deal with undefined shifts.
   if (Shift >= DstBits)
-    return 0;
+    return Register();
 
   // For immediate shifts we can fold the zero-/sign-extension into the shift.
   // {S|U}BFM Wd, Wn, #r, #s
@@ -4403,7 +4402,7 @@ unsigned AArch64FastISel::emitASR_ri(MVT RetVT, MVT SrcVT, unsigned Op0,
   return fastEmitInst_rii(Opc, RC, Op0, ImmR, ImmS);
 }
 
-unsigned AArch64FastISel::emitIntExt(MVT SrcVT, unsigned SrcReg, MVT DestVT,
+Register AArch64FastISel::emitIntExt(MVT SrcVT, Register SrcReg, MVT DestVT,
                                      bool IsZExt) {
   assert(DestVT != MVT::i1 && "ZeroExt/SignExt an i1?");
 
@@ -4415,14 +4414,14 @@ unsigned AArch64FastISel::emitIntExt(MVT SrcVT, unsigned SrcReg, MVT DestVT,
        (DestVT != MVT::i32) && (DestVT != MVT::i64)) ||
       ((SrcVT !=  MVT::i1) && (SrcVT !=  MVT::i8) &&
        (SrcVT !=  MVT::i16) && (SrcVT !=  MVT::i32)))
-    return 0;
+    return Register();
 
   unsigned Opc;
   unsigned Imm = 0;
 
   switch (SrcVT.SimpleTy) {
   default:
-    return 0;
+    return Register();
   case MVT::i1:
     return emiti1Ext(SrcReg, DestVT, IsZExt);
   case MVT::i8:
@@ -4604,7 +4603,7 @@ bool AArch64FastISel::selectIntExt(const Instruction *I) {
     }
   }
 
-  unsigned ResultReg = emitIntExt(SrcVT, SrcReg, RetVT, IsZExt);
+  Register ResultReg = emitIntExt(SrcVT, SrcReg, RetVT, IsZExt);
   if (!ResultReg)
     return false;
 
@@ -4697,8 +4696,7 @@ bool AArch64FastISel::selectMul(const Instruction *I) {
       if (!Src0Reg)
         return false;
 
-      unsigned ResultReg =
-          emitLSL_ri(VT, SrcVT, Src0Reg, ShiftVal, IsZExt);
+      Register ResultReg = emitLSL_ri(VT, SrcVT, Src0Reg, ShiftVal, IsZExt);
 
       if (ResultReg) {
         updateValueMap(I, ResultReg);
@@ -4714,7 +4712,7 @@ bool AArch64FastISel::selectMul(const Instruction *I) {
   if (!Src1Reg)
     return false;
 
-  unsigned ResultReg = emitMul_rr(VT, Src0Reg, Src1Reg);
+  Register ResultReg = emitMul_rr(VT, Src0Reg, Src1Reg);
 
   if (!ResultReg)
     return false;
@@ -4732,7 +4730,7 @@ bool AArch64FastISel::selectShift(const Instruction *I) {
     return selectOperator(I, I->getOpcode());
 
   if (const auto *C = dyn_cast<ConstantInt>(I->getOperand(1))) {
-    unsigned ResultReg = 0;
+    Register ResultReg;
     uint64_t ShiftVal = C->getZExtValue();
     MVT SrcVT = RetVT;
     bool IsZExt = I->getOpcode() != Instruction::AShr;
@@ -4788,7 +4786,7 @@ bool AArch64FastISel::selectShift(const Instruction *I) {
   if (!Op1Reg)
     return false;
 
-  unsigned ResultReg = 0;
+  Register ResultReg;
   switch (I->getOpcode()) {
   default: llvm_unreachable("Unexpected instruction.");
   case Instruction::Shl:
@@ -4906,7 +4904,7 @@ bool AArch64FastISel::selectSDiv(const Instruction *I) {
     return false;
 
   if (cast<BinaryOperator>(I)->isExact()) {
-    unsigned ResultReg = emitASR_ri(VT, VT, Src0Reg, Lg2);
+    Register ResultReg = emitASR_ri(VT, VT, Src0Reg, Lg2);
     if (!ResultReg)
       return false;
     updateValueMap(I, ResultReg);
@@ -4914,7 +4912,7 @@ bool AArch64FastISel::selectSDiv(const Instruction *I) {
   }
 
   int64_t Pow2MinusOne = (1ULL << Lg2) - 1;
-  unsigned AddReg = emitAdd_ri_(VT, Src0Reg, Pow2MinusOne);
+  Register AddReg = emitAdd_ri_(VT, Src0Reg, Pow2MinusOne);
   if (!AddReg)
     return false;
 
@@ -4938,8 +4936,8 @@ bool AArch64FastISel::selectSDiv(const Instruction *I) {
 
   // Divide by Pow2 --> ashr. If we're dividing by a negative value we must also
   // negate the result.
-  unsigned ZeroReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR;
-  unsigned ResultReg;
+  Register ZeroReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR;
+  Register ResultReg;
   if (C.isNegative())
     ResultReg = emitAddSub_rs(/*UseAdd=*/false, VT, ZeroReg, SelectReg,
                               AArch64_AM::ASR, Lg2);
@@ -4956,11 +4954,11 @@ bool AArch64FastISel::selectSDiv(const Instruction *I) {
 /// This is mostly a copy of the existing FastISel getRegForGEPIndex code. We
 /// have to duplicate it for AArch64, because otherwise we would fail during the
 /// sign-extend emission.
-unsigned AArch64FastISel::getRegForGEPIndex(const Value *Idx) {
+Register AArch64FastISel::getRegForGEPIndex(const Value *Idx) {
   Register IdxN = getRegForValue(Idx);
-  if (IdxN == 0)
+  if (!IdxN)
     // Unhandled operand. Halt "fast" selection and bail.
-    return 0;
+    return Register();
 
   // If the index is smaller or larger than intptr_t, truncate or extend it.
   MVT PtrVT = TLI.getPointerTy(DL);
@@ -5015,12 +5013,12 @@ bool AArch64FastISel::selectGetElementPtr(const Instruction *I) {
 
       // N = N + Idx * ElementSize;
       uint64_t ElementSize = GTI.getSequentialElementStride(DL);
-      unsigned IdxN = getRegForGEPIndex(Idx);
+      Register IdxN = getRegForGEPIndex(Idx);
       if (!IdxN)
         return false;
 
       if (ElementSize != 1) {
-        unsigned C = fastEmit_i(VT, VT, ISD::Constant, ElementSize);
+        Register C = fastEmit_i(VT, VT, ISD::Constant, ElementSize);
         if (!C)
           return false;
         IdxN = emitMul_rr(VT, IdxN, C);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 07bcd802962fa..3ca9107cb2ce5 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -7380,23 +7380,12 @@ bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(SDNode *Root, SDValue N,
     return false;
 
   SDValue VScale = N.getOperand(1);
-  int64_t MulImm = std::numeric_limits<int64_t>::max();
-  if (VScale.getOpcode() == ISD::VSCALE) {
-    MulImm = cast<ConstantSDNode>(VScale.getOperand(0))->getSExtValue();
-  } else if (auto C = dyn_cast<ConstantSDNode>(VScale)) {
-    int64_t ByteOffset = C->getSExtValue();
-    const auto KnownVScale =
-        Subtarget->getSVEVectorSizeInBits() / AArch64::SVEBitsPerBlock;
-
-    if (!KnownVScale || ByteOffset % KnownVScale != 0)
-      return false;
-
-    MulImm = ByteOffset / KnownVScale;
-  } else
+  if (VScale.getOpcode() != ISD::VSCALE)
     return false;
 
   TypeSize TS = MemVT.getSizeInBits();
   int64_t MemWidthBytes = static_cast<int64_t>(TS.getKnownMinValue()) / 8;
+  int64_t MulImm = cast<ConstantSDNode>(VScale.getOperand(0))->getSExtValue();
 
   if ((MulImm % MemWidthBytes) != 0)
     return false;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index e7d141d22c7c7..07efa24711fe0 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -984,6 +984,23 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
 #undef LCALLNAME5
   }
 
+  if (Subtarget->outlineAtomics() && !Subtarget->hasLSFE()) {
+    setOperationAction(ISD::ATOMIC_LOAD_FADD, MVT::f16, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FADD, MVT::f32, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FADD, MVT::f64, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FADD, MVT::bf16, LibCall);
+
+    setOperationAction(ISD::ATOMIC_LOAD_FMAX, MVT::f16, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMAX, MVT::f32, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMAX, MVT::f64, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMAX, MVT::bf16, LibCall);
+
+    setOperationAction(ISD::ATOMIC_LOAD_FMIN, MVT::f16, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMIN, MVT::f32, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMIN, MVT::f64, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMIN, MVT::bf16, LibCall);
+  }
+
   if (Subtarget->hasLSE128()) {
     // Custom lowering because i128 is not legal. Must be replaced by 2x64
     // values. ATOMIC_LOAD_AND also needs op legalisation to emit LDCLRP.
@@ -1700,6 +1717,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::BITCAST, VT, Custom);
       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
       setOperationAction(ISD::FABS, VT, Legal);
+      setOperationAction(ISD::FCOPYSIGN, VT, Custom);
       setOperationAction(ISD::FNEG, VT, Legal);
       setOperationAction(ISD::FP_EXTEND, VT, Custom);
       setOperationAction(ISD::FP_ROUND, VT, Custom);
@@ -7906,7 +7924,6 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
 
   // Assign locations to all of the incoming arguments.
   SmallVector<CCValAssign, 16> ArgLocs;
-  DenseMap<unsigned, SDValue> CopiedRegs;
   CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
 
   // At this point, Ins[].VT may already be promoted to i32. To correctly
@@ -10690,7 +10707,7 @@ SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
   // a SVE FCOPYSIGN.
   if (!VT.isVector() && !Subtarget->isNeonAvailable() &&
       Subtarget->isSVEorStreamingSVEAvailable()) {
-    if (VT != MVT::f16 && VT != MVT::f32 && VT != MVT::f64)
+    if (VT != MVT::f16 && VT != MVT::f32 && VT != MVT::f64 && VT != MVT::bf16)
       return SDValue();
     EVT SVT = getPackedSVEVectorVT(VT);
 
@@ -17488,7 +17505,7 @@ bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI,
 
       if (UseScalable)
         Shuffle = Builder.CreateInsertVector(
-            STVTy, UndefValue::get(STVTy), Shuffle,
+            STVTy, PoisonValue::get(STVTy), Shuffle,
             ConstantInt::get(Type::getInt64Ty(STVTy->getContext()), 0));
 
       Ops.push_back(Shuffle);
@@ -27907,6 +27924,12 @@ AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
   if (CanUseLSE128)
     return AtomicExpansionKind::None;
 
+  // If LSFE available, use atomic FP instructions in preference to expansion
+  if (Subtarget->hasLSFE() && (AI->getOperation() == AtomicRMWInst::FAdd ||
+                               AI->getOperation() == AtomicRMWInst::FMax ||
+                               AI->getOperation() == AtomicRMWInst::FMin))
+    return AtomicExpansionKind::None;
+
   // Nand is not supported in LSE.
   // Leave 128 bits to LLSC or CmpXChg.
   if (AI->getOperation() != AtomicRMWInst::Nand && Size < 128 &&
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 5e6db9d007a55..2d7a9d6f00bd0 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -543,6 +543,20 @@ let Predicates = [HasLSE] in {
   defm : LDOPregister_patterns_mod<"LDCLR", "atomic_load_and", "ORN">;
 }
 
+defm atomic_load_fadd  : binary_atomic_op_fp<atomic_load_fadd>;
+defm atomic_load_fmin  : binary_atomic_op_fp<atomic_load_fmin>;
+defm atomic_load_fmax  : binary_atomic_op_fp<atomic_load_fmax>;
+
+let Predicates = [HasLSFE] in {
+  defm : LDFPOPregister_patterns<"LDFADD",   "atomic_load_fadd">;
+  defm : LDFPOPregister_patterns<"LDFMAXNM", "atomic_load_fmax">;
+  defm : LDFPOPregister_patterns<"LDFMINNM", "atomic_load_fmin">;
+
+  defm : LDBFPOPregister_patterns<"LDBFADD",   "atomic_load_fadd">;
+  defm : LDBFPOPregister_patterns<"LDBFMAXNM", "atomic_load_fmax">;
+  defm : LDBFPOPregister_patterns<"LDBFMINNM", "atomic_load_fmin">;
+}
+
 // v8.9a/v9.4a FEAT_LRCPC patterns
 let Predicates = [HasRCPC3, HasNEON] in {
   // LDAP1 loads
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 7a8b743faf4a3..255cd0ec5840c 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -12493,6 +12493,36 @@ multiclass LDOPregister_patterns_mod<string inst, string op, string mod> {
                         (i32 (!cast<Instruction>(mod#Wrr) WZR, GPR32:$Rm))>;
 }
 
+let Predicates = [HasLSFE] in
+multiclass LDFPOPregister_patterns_ord_dag<string inst, string suffix, string op,
+                                         ValueType vt, dag data> {
+  def : Pat<(!cast<PatFrag>(op#"_"#vt#"_monotonic") FPR64:$Rn, data),
+            (!cast<Instruction>(inst # suffix) data, FPR64:$Rn)>;
+  def : Pat<(!cast<PatFrag>(op#"_"#vt#"_acquire") FPR64:$Rn, data),
+            (!cast<Instruction>(inst # "A" # suffix) data, FPR64:$Rn)>;
+  def : Pat<(!cast<PatFrag>(op#"_"#vt#"_release") FPR64:$Rn, data),
+            (!cast<Instruction>(inst # "L" # suffix) data, FPR64:$Rn)>;
+  def : Pat<(!cast<PatFrag>(op#"_"#vt#"_acq_rel") FPR64:$Rn, data),
+            (!cast<Instruction>(inst # "AL" # suffix) data, FPR64:$Rn)>;
+  def : Pat<(!cast<PatFrag>(op#"_"#vt#"_seq_cst") FPR64:$Rn, data),
+            (!cast<Instruction>(inst # "AL" # suffix) data, FPR64:$Rn)>;
+}
+
+multiclass LDFPOPregister_patterns_ord<string inst, string suffix, string op,
+                                     ValueType vt, dag RHS> {
+  defm : LDFPOPregister_patterns_ord_dag<inst, suffix, op, vt, RHS>;
+}
+
+multiclass LDFPOPregister_patterns<string inst, string op> {
+  defm : LDFPOPregister_patterns_ord<inst, "H", op, f16, (f16 FPR16:$Rm)>;
+  defm : LDFPOPregister_patterns_ord<inst, "S", op, f32, (f32 FPR32:$Rm)>;
+  defm : LDFPOPregister_patterns_ord<inst, "D", op, f64, (f64 FPR64:$Rm)>;
+}
+
+multiclass LDBFPOPregister_patterns<string inst, string op> {
+  defm : LDFPOPregister_patterns_ord<inst, "", op, bf16, (bf16 FPR16:$Rm)>;
+}
+
 let Predicates = [HasLSE] in
 multiclass CASregister_patterns_ord_dag<string inst, string suffix, string op,
                                         ValueType vt, dag OLD, dag NEW> {
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index c836f3138a45f..6c61e3a613f6f 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -1030,9 +1030,7 @@ def add_and_or_is_add : PatFrags<(ops node:$lhs, node:$rhs),
    return CurDAG->isADDLike(SDValue(N,0));
 }]> {
   let GISelPredicateCode = [{
-     return MI.getOpcode() == TargetOpcode::G_ADD ||
-            (MI.getOpcode() == TargetOpcode::G_OR &&
-             MI.getFlag(MachineInstr::MIFlag::Disjoint));
+     return mi_match(MI, MRI, m_GAddLike(m_Reg(), m_Reg()));
   }];
 }
 
@@ -10564,7 +10562,7 @@ let Predicates = [HasLSFE] in {
   defm LDFMAXNML  : AtomicFPLoad<0b01, 0b110, "ldfmaxnml">;
   defm LDFMINNMA  : AtomicFPLoad<0b10, 0b111, "ldfminnma">;
   defm LDFMINNMAL : AtomicFPLoad<0b11, 0b111, "ldfminnmal">;
-  defm LDFMINMN   : AtomicFPLoad<0b00, 0b111, "ldfminnm">;
+  defm LDFMINNM   : AtomicFPLoad<0b00, 0b111, "ldfminnm">;
   defm LDFMINNML  : AtomicFPLoad<0b01, 0b111, "ldfminnml">;
   // BFloat16
   def LDBFADDA    : BaseAtomicFPLoad<FPR16, 0b00, 0b10, 0b000, "ldbfadda">;
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index f5ffc72cae537..c6eb77e3bc3ba 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -391,7 +391,7 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
   void mirFileLoaded(MachineFunction &MF) const override;
 
   // Return the known range for the bit length of SVE data registers. A value
-  // of 0 means nothing is known about that particular limit beyond what's
+  // of 0 means nothing is known about that particular limit beyong what's
   // implied by the architecture.
   unsigned getMaxSVEVectorSizeInBits() const {
     assert(isSVEorStreamingSVEAvailable() &&
@@ -405,16 +405,6 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
     return MinSVEVectorSizeInBits;
   }
 
-  // Return the known bit length of SVE data registers. A value of 0 means the
-  // length is unkown beyond what's implied by the architecture.
-  unsigned getSVEVectorSizeInBits() const {
-    assert(isSVEorStreamingSVEAvailable() &&
-           "Tried to get SVE vector length without SVE support!");
-    if (MinSVEVectorSizeInBits == MaxSVEVectorSizeInBits)
-      return MaxSVEVectorSizeInBits;
-    return 0;
-  }
-
   bool useSVEForFixedLengthVectors() const {
     if (!isSVEorStreamingSVEAvailable())
       return false;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index ba019e1a4ecd5..7cec8a17dfaaa 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -18,6 +18,7 @@
 #include "llvm/CodeGen/BasicTTIImpl.h"
 #include "llvm/CodeGen/CostTable.h"
 #include "llvm/CodeGen/TargetLowering.h"
+#include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IntrinsicsAArch64.h"
@@ -884,12 +885,11 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
 
     const auto LegalisationCost = getTypeLegalizationCost(RetTy);
     if (OpInfoZ.isUniform()) {
-      // FIXME: The costs could be lower if the codegen is better.
       static const CostTblEntry FshlTbl[] = {
-          {Intrinsic::fshl, MVT::v4i32, 3}, // ushr + shl + orr
-          {Intrinsic::fshl, MVT::v2i64, 3}, {Intrinsic::fshl, MVT::v16i8, 4},
-          {Intrinsic::fshl, MVT::v8i16, 4}, {Intrinsic::fshl, MVT::v2i32, 3},
-          {Intrinsic::fshl, MVT::v8i8, 4},  {Intrinsic::fshl, MVT::v4i16, 4}};
+          {Intrinsic::fshl, MVT::v4i32, 2}, // shl + usra
+          {Intrinsic::fshl, MVT::v2i64, 2}, {Intrinsic::fshl, MVT::v16i8, 2},
+          {Intrinsic::fshl, MVT::v8i16, 2}, {Intrinsic::fshl, MVT::v2i32, 2},
+          {Intrinsic::fshl, MVT::v8i8, 2},  {Intrinsic::fshl, MVT::v4i16, 2}};
       // Costs for both fshl & fshr are the same, so just pass Intrinsic::fshl
       // to avoid having to duplicate the costs.
       const auto *Entry =
@@ -3532,23 +3532,111 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
   default:
     return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
                                          Op2Info);
+  case ISD::SREM:
   case ISD::SDIV:
-    if (Op2Info.isConstant() && Op2Info.isUniform() && Op2Info.isPowerOf2()) {
-      // On AArch64, scalar signed division by constants power-of-two are
-      // normally expanded to the sequence ADD + CMP + SELECT + SRA.
-      // The OperandValue properties many not be same as that of previous
-      // operation; conservatively assume OP_None.
-      InstructionCost Cost = getArithmeticInstrCost(
-          Instruction::Add, Ty, CostKind,
-          Op1Info.getNoProps(), Op2Info.getNoProps());
-      Cost += getArithmeticInstrCost(Instruction::Sub, Ty, CostKind,
-                                     Op1Info.getNoProps(), Op2Info.getNoProps());
-      Cost += getArithmeticInstrCost(
-          Instruction::Select, Ty, CostKind,
-          Op1Info.getNoProps(), Op2Info.getNoProps());
-      Cost += getArithmeticInstrCost(Instruction::AShr, Ty, CostKind,
-                                     Op1Info.getNoProps(), Op2Info.getNoProps());
-      return Cost;
+    /*
+    Notes for sdiv/srem specific costs:
+    1. This only considers the cases where the divisor is constant, uniform and
+    (pow-of-2/non-pow-of-2). Other cases are not important since they either
+    result in some form of (ldr + adrp), corresponding to constant vectors, or
+    scalarization of the division operation.
+    2. Constant divisors, either negative in whole or partially, don't result in
+    significantly different codegen as compared to positive constant divisors.
+    So, we don't consider negative divisors seperately.
+    3. If the codegen is significantly different with SVE, it has been indicated
+    using comments at appropriate places.
+
+    sdiv specific cases:
+    -----------------------------------------------------------------------
+    codegen                       | pow-of-2               | Type
+    -----------------------------------------------------------------------
+    add + cmp + csel + asr        | Y                      | i64
+    add + cmp + csel + asr        | Y                      | i32
+    -----------------------------------------------------------------------
+
+    srem specific cases:
+    -----------------------------------------------------------------------
+    codegen                       | pow-of-2               | Type
+    -----------------------------------------------------------------------
+    negs + and + and + csneg      | Y                      | i64
+    negs + and + and + csneg      | Y                      | i32
+    -----------------------------------------------------------------------
+
+    other sdiv/srem cases:
+    -------------------------------------------------------------------------
+    commom codegen            | + srem     | + sdiv     | pow-of-2  | Type
+    -------------------------------------------------------------------------
+    smulh + asr + add + add   | -          | -          | N         | i64
+    smull + lsr + add + add   | -          | -          | N         | i32
+    usra                      | and + sub  | sshr       | Y         | <2 x i64>
+    2 * (scalar code)         | -          | -          | N         | <2 x i64>
+    usra                      | bic + sub  | sshr + neg | Y         | <4 x i32>
+    smull2 + smull + uzp2     | mls        | -          | N         | <4 x i32>
+           + sshr  + usra     |            |            |           |
+    -------------------------------------------------------------------------
+    */
+    if (Op2Info.isConstant() && Op2Info.isUniform()) {
+      InstructionCost AddCost =
+          getArithmeticInstrCost(Instruction::Add, Ty, CostKind,
+                                 Op1Info.getNoProps(), Op2Info.getNoProps());
+      InstructionCost AsrCost =
+          getArithmeticInstrCost(Instruction::AShr, Ty, CostKind,
+                                 Op1Info.getNoProps(), Op2Info.getNoProps());
+      InstructionCost MulCost =
+          getArithmeticInstrCost(Instruction::Mul, Ty, CostKind,
+                                 Op1Info.getNoProps(), Op2Info.getNoProps());
+      // add/cmp/csel/csneg should have similar cost while asr/negs/and should
+      // have similar cost.
+      auto VT = TLI->getValueType(DL, Ty);
+      if (LT.second.isScalarInteger() && VT.getSizeInBits() <= 64) {
+        if (Op2Info.isPowerOf2()) {
+          return ISD == ISD::SDIV ? (3 * AddCost + AsrCost)
+                                  : (3 * AsrCost + AddCost);
+        } else {
+          return MulCost + AsrCost + 2 * AddCost;
+        }
+      } else if (VT.isVector()) {
+        InstructionCost UsraCost = 2 * AsrCost;
+        if (Op2Info.isPowerOf2()) {
+          // Division with scalable types corresponds to native 'asrd'
+          // instruction when SVE is available.
+          // e.g. %1 = sdiv <vscale x 4 x i32> %a, splat (i32 8)
+          if (Ty->isScalableTy() && ST->hasSVE())
+            return 2 * AsrCost;
+          return UsraCost +
+                 (ISD == ISD::SDIV
+                      ? (LT.second.getScalarType() == MVT::i64 ? 1 : 2) *
+                            AsrCost
+                      : 2 * AddCost);
+        } else if (LT.second == MVT::v2i64) {
+          return VT.getVectorNumElements() *
+                 getArithmeticInstrCost(Opcode, Ty->getScalarType(), CostKind,
+                                        Op1Info.getNoProps(),
+                                        Op2Info.getNoProps());
+        } else {
+          // When SVE is available, we get:
+          // smulh + lsr + add/sub + asr + add/sub.
+          if (Ty->isScalableTy() && ST->hasSVE())
+            return MulCost /*smulh cost*/ + 2 * AddCost + 2 * AsrCost;
+          return 2 * MulCost + AddCost /*uzp2 cost*/ + AsrCost + UsraCost;
+        }
+      }
+    }
+    if (Op2Info.isConstant() && !Op2Info.isUniform() &&
+        LT.second.isFixedLengthVector()) {
+      // FIXME: When the constant vector is non-uniform, this may result in
+      // loading the vector from constant pool or in some cases, may also result
+      // in scalarization. For now, we are approximating this with the
+      // scalarization cost.
+      auto ExtractCost = 2 * getVectorInstrCost(Instruction::ExtractElement, Ty,
+                                                CostKind, -1, nullptr, nullptr);
+      auto InsertCost = getVectorInstrCost(Instruction::InsertElement, Ty,
+                                           CostKind, -1, nullptr, nullptr);
+      unsigned NElts = cast<FixedVectorType>(Ty)->getNumElements();
+      return ExtractCost + InsertCost +
+             NElts * getArithmeticInstrCost(Opcode, Ty->getScalarType(),
+                                            CostKind, Op1Info.getNoProps(),
+                                            Op2Info.getNoProps());
     }
     [[fallthrough]];
   case ISD::UDIV:
@@ -3588,23 +3676,6 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
                                   AddCost * 2 + ShrCost;
         return DivCost + (ISD == ISD::UREM ? MulCost + AddCost : 0);
       }
-
-      // TODO: Fix SDIV and SREM costs, similar to the above.
-      if (TLI->isOperationLegalOrCustom(ISD::MULHU, VT) &&
-          Op2Info.isUniform() && !VT.isScalableVector()) {
-        // Vector signed division by constant are expanded to the
-        // sequence MULHS + ADD/SUB + SRA + SRL + ADD.
-        InstructionCost MulCost =
-            getArithmeticInstrCost(Instruction::Mul, Ty, CostKind,
-                                   Op1Info.getNoProps(), Op2Info.getNoProps());
-        InstructionCost AddCost =
-            getArithmeticInstrCost(Instruction::Add, Ty, CostKind,
-                                   Op1Info.getNoProps(), Op2Info.getNoProps());
-        InstructionCost ShrCost =
-            getArithmeticInstrCost(Instruction::AShr, Ty, CostKind,
-                                   Op1Info.getNoProps(), Op2Info.getNoProps());
-        return MulCost * 2 + AddCost * 2 + ShrCost * 2 + 1;
-      }
     }
 
     // div i128's are lowered as libcalls.  Pass nullptr as (u)divti3 calls are
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index 6a973b0160e23..1e7296fa23486 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -7847,10 +7847,10 @@ bool AArch64AsmParser::parseDirectiveAeabiSubSectionHeader(SMLoc L) {
 
   // Consume the name (subsection name)
   StringRef SubsectionName;
-  AArch64BuildAttrs::VendorID SubsectionNameID;
+  AArch64BuildAttributes::VendorID SubsectionNameID;
   if (Parser.getTok().is(AsmToken::Identifier)) {
     SubsectionName = Parser.getTok().getIdentifier();
-    SubsectionNameID = AArch64BuildAttrs::getVendorID(SubsectionName);
+    SubsectionNameID = AArch64BuildAttributes::getVendorID(SubsectionName);
   } else {
     Error(Parser.getTok().getLoc(), "subsection name not found");
     return true;
@@ -7867,14 +7867,14 @@ bool AArch64AsmParser::parseDirectiveAeabiSubSectionHeader(SMLoc L) {
       getTargetStreamer().getAtributesSubsectionByName(SubsectionName);
 
   // Consume the first parameter (optionality parameter)
-  AArch64BuildAttrs::SubsectionOptional IsOptional;
+  AArch64BuildAttributes::SubsectionOptional IsOptional;
   // options: optional/required
   if (Parser.getTok().is(AsmToken::Identifier)) {
     StringRef Optionality = Parser.getTok().getIdentifier();
-    IsOptional = AArch64BuildAttrs::getOptionalID(Optionality);
-    if (AArch64BuildAttrs::OPTIONAL_NOT_FOUND == IsOptional) {
+    IsOptional = AArch64BuildAttributes::getOptionalID(Optionality);
+    if (AArch64BuildAttributes::OPTIONAL_NOT_FOUND == IsOptional) {
       Error(Parser.getTok().getLoc(),
-            AArch64BuildAttrs::getSubsectionOptionalUnknownError());
+            AArch64BuildAttributes::getSubsectionOptionalUnknownError());
       return true;
     }
     if (SubsectionExists) {
@@ -7882,10 +7882,10 @@ bool AArch64AsmParser::parseDirectiveAeabiSubSectionHeader(SMLoc L) {
         Error(Parser.getTok().getLoc(),
               "optionality mismatch! subsection '" + SubsectionName +
                   "' already exists with optionality defined as '" +
-                  AArch64BuildAttrs::getOptionalStr(
+                  AArch64BuildAttributes::getOptionalStr(
                       SubsectionExists->IsOptional) +
                   "' and not '" +
-                  AArch64BuildAttrs::getOptionalStr(IsOptional) + "'");
+                  AArch64BuildAttributes::getOptionalStr(IsOptional) + "'");
         return true;
       }
     }
@@ -7895,15 +7895,15 @@ bool AArch64AsmParser::parseDirectiveAeabiSubSectionHeader(SMLoc L) {
     return true;
   }
   // Check for possible IsOptional unaccepted values for known subsections
-  if (AArch64BuildAttrs::AEABI_FEATURE_AND_BITS == SubsectionNameID) {
-    if (AArch64BuildAttrs::REQUIRED == IsOptional) {
+  if (AArch64BuildAttributes::AEABI_FEATURE_AND_BITS == SubsectionNameID) {
+    if (AArch64BuildAttributes::REQUIRED == IsOptional) {
       Error(Parser.getTok().getLoc(),
             "aeabi_feature_and_bits must be marked as optional");
       return true;
     }
   }
-  if (AArch64BuildAttrs::AEABI_PAUTHABI == SubsectionNameID) {
-    if (AArch64BuildAttrs::OPTIONAL == IsOptional) {
+  if (AArch64BuildAttributes::AEABI_PAUTHABI == SubsectionNameID) {
+    if (AArch64BuildAttributes::OPTIONAL == IsOptional) {
       Error(Parser.getTok().getLoc(),
             "aeabi_pauthabi must be marked as required");
       return true;
@@ -7916,23 +7916,24 @@ bool AArch64AsmParser::parseDirectiveAeabiSubSectionHeader(SMLoc L) {
   }
 
   // Consume the second parameter (type parameter)
-  AArch64BuildAttrs::SubsectionType Type;
+  AArch64BuildAttributes::SubsectionType Type;
   if (Parser.getTok().is(AsmToken::Identifier)) {
     StringRef Name = Parser.getTok().getIdentifier();
-    Type = AArch64BuildAttrs::getTypeID(Name);
-    if (AArch64BuildAttrs::TYPE_NOT_FOUND == Type) {
+    Type = AArch64BuildAttributes::getTypeID(Name);
+    if (AArch64BuildAttributes::TYPE_NOT_FOUND == Type) {
       Error(Parser.getTok().getLoc(),
-            AArch64BuildAttrs::getSubsectionTypeUnknownError());
+            AArch64BuildAttributes::getSubsectionTypeUnknownError());
       return true;
     }
     if (SubsectionExists) {
       if (Type != SubsectionExists->ParameterType) {
-        Error(
-            Parser.getTok().getLoc(),
-            "type mismatch! subsection '" + SubsectionName +
-                "' already exists with type defined as '" +
-                AArch64BuildAttrs::getTypeStr(SubsectionExists->ParameterType) +
-                "' and not '" + AArch64BuildAttrs::getTypeStr(Type) + "'");
+        Error(Parser.getTok().getLoc(),
+              "type mismatch! subsection '" + SubsectionName +
+                  "' already exists with type defined as '" +
+                  AArch64BuildAttributes::getTypeStr(
+                      SubsectionExists->ParameterType) +
+                  "' and not '" + AArch64BuildAttributes::getTypeStr(Type) +
+                  "'");
         return true;
       }
     }
@@ -7942,9 +7943,9 @@ bool AArch64AsmParser::parseDirectiveAeabiSubSectionHeader(SMLoc L) {
     return true;
   }
   // Check for possible unaccepted 'type' values for known subsections
-  if (AArch64BuildAttrs::AEABI_FEATURE_AND_BITS == SubsectionNameID ||
-      AArch64BuildAttrs::AEABI_PAUTHABI == SubsectionNameID) {
-    if (AArch64BuildAttrs::NTBS == Type) {
+  if (AArch64BuildAttributes::AEABI_FEATURE_AND_BITS == SubsectionNameID ||
+      AArch64BuildAttributes::AEABI_PAUTHABI == SubsectionNameID) {
+    if (AArch64BuildAttributes::NTBS == Type) {
       Error(Parser.getTok().getLoc(),
             SubsectionName + " must be marked as ULEB128");
       return true;
@@ -7980,13 +7981,14 @@ bool AArch64AsmParser::parseDirectiveAeabiAArch64Attr(SMLoc L) {
   StringRef ActiveSubsectionName = ActiveSubsection->VendorName;
   unsigned ActiveSubsectionType = ActiveSubsection->ParameterType;
 
-  unsigned ActiveSubsectionID = AArch64BuildAttrs::VENDOR_UNKNOWN;
-  if (AArch64BuildAttrs::getVendorName(AArch64BuildAttrs::AEABI_PAUTHABI) ==
+  unsigned ActiveSubsectionID = AArch64BuildAttributes::VENDOR_UNKNOWN;
+  if (AArch64BuildAttributes::getVendorName(
+          AArch64BuildAttributes::AEABI_PAUTHABI) == ActiveSubsectionName)
+    ActiveSubsectionID = AArch64BuildAttributes::AEABI_PAUTHABI;
+  if (AArch64BuildAttributes::getVendorName(
+          AArch64BuildAttributes::AEABI_FEATURE_AND_BITS) ==
       ActiveSubsectionName)
-    ActiveSubsectionID = AArch64BuildAttrs::AEABI_PAUTHABI;
-  if (AArch64BuildAttrs::getVendorName(
-          AArch64BuildAttrs::AEABI_FEATURE_AND_BITS) == ActiveSubsectionName)
-    ActiveSubsectionID = AArch64BuildAttrs::AEABI_FEATURE_AND_BITS;
+    ActiveSubsectionID = AArch64BuildAttributes::AEABI_FEATURE_AND_BITS;
 
   StringRef TagStr = "";
   unsigned Tag;
@@ -7995,7 +7997,7 @@ bool AArch64AsmParser::parseDirectiveAeabiAArch64Attr(SMLoc L) {
   } else if (Parser.getTok().is(AsmToken::Identifier)) {
     TagStr = Parser.getTok().getIdentifier();
     switch (ActiveSubsectionID) {
-    case AArch64BuildAttrs::VENDOR_UNKNOWN:
+    case AArch64BuildAttributes::VENDOR_UNKNOWN:
       // Tag was provided as an unrecognized string instead of an unsigned
       // integer
       Error(Parser.getTok().getLoc(), "unrecognized Tag: '" + TagStr +
@@ -8003,18 +8005,18 @@ bool AArch64AsmParser::parseDirectiveAeabiAArch64Attr(SMLoc L) {
                                           "tags have to be an unsigned int.");
       return true;
       break;
-    case AArch64BuildAttrs::AEABI_PAUTHABI:
-      Tag = AArch64BuildAttrs::getPauthABITagsID(TagStr);
-      if (AArch64BuildAttrs::PAUTHABI_TAG_NOT_FOUND == Tag) {
+    case AArch64BuildAttributes::AEABI_PAUTHABI:
+      Tag = AArch64BuildAttributes::getPauthABITagsID(TagStr);
+      if (AArch64BuildAttributes::PAUTHABI_TAG_NOT_FOUND == Tag) {
         Error(Parser.getTok().getLoc(), "unknown AArch64 build attribute '" +
                                             TagStr + "' for subsection '" +
                                             ActiveSubsectionName + "'");
         return true;
       }
       break;
-    case AArch64BuildAttrs::AEABI_FEATURE_AND_BITS:
-      Tag = AArch64BuildAttrs::getFeatureAndBitsTagsID(TagStr);
-      if (AArch64BuildAttrs::FEATURE_AND_BITS_TAG_NOT_FOUND == Tag) {
+    case AArch64BuildAttributes::AEABI_FEATURE_AND_BITS:
+      Tag = AArch64BuildAttributes::getFeatureAndBitsTagsID(TagStr);
+      if (AArch64BuildAttributes::FEATURE_AND_BITS_TAG_NOT_FOUND == Tag) {
         Error(Parser.getTok().getLoc(), "unknown AArch64 build attribute '" +
                                             TagStr + "' for subsection '" +
                                             ActiveSubsectionName + "'");
@@ -8038,7 +8040,7 @@ bool AArch64AsmParser::parseDirectiveAeabiAArch64Attr(SMLoc L) {
   unsigned ValueInt = unsigned(-1);
   std::string ValueStr = "";
   if (Parser.getTok().is(AsmToken::Integer)) {
-    if (AArch64BuildAttrs::NTBS == ActiveSubsectionType) {
+    if (AArch64BuildAttributes::NTBS == ActiveSubsectionType) {
       Error(
           Parser.getTok().getLoc(),
           "active subsection type is NTBS (string), found ULEB128 (unsigned)");
@@ -8046,7 +8048,7 @@ bool AArch64AsmParser::parseDirectiveAeabiAArch64Attr(SMLoc L) {
     }
     ValueInt = getTok().getIntVal();
   } else if (Parser.getTok().is(AsmToken::Identifier)) {
-    if (AArch64BuildAttrs::ULEB128 == ActiveSubsectionType) {
+    if (AArch64BuildAttributes::ULEB128 == ActiveSubsectionType) {
       Error(
           Parser.getTok().getLoc(),
           "active subsection type is ULEB128 (unsigned), found NTBS (string)");
@@ -8054,7 +8056,7 @@ bool AArch64AsmParser::parseDirectiveAeabiAArch64Attr(SMLoc L) {
     }
     ValueStr = Parser.getTok().getIdentifier();
   } else if (Parser.getTok().is(AsmToken::String)) {
-    if (AArch64BuildAttrs::ULEB128 == ActiveSubsectionType) {
+    if (AArch64BuildAttributes::ULEB128 == ActiveSubsectionType) {
       Error(
           Parser.getTok().getLoc(),
           "active subsection type is ULEB128 (unsigned), found NTBS (string)");
@@ -8067,7 +8069,7 @@ bool AArch64AsmParser::parseDirectiveAeabiAArch64Attr(SMLoc L) {
   }
   // Check for possible unaccepted values for known tags
   // (AEABI_FEATURE_AND_BITS)
-  if (ActiveSubsectionID == AArch64BuildAttrs::AEABI_FEATURE_AND_BITS) {
+  if (ActiveSubsectionID == AArch64BuildAttributes::AEABI_FEATURE_AND_BITS) {
     if (0 != ValueInt && 1 != ValueInt) {
       Error(Parser.getTok().getLoc(),
             "unknown AArch64 build attributes Value for Tag '" + TagStr +
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
index a0953e13aa373..98bd102d8f4c1 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
@@ -161,10 +161,10 @@ class AArch64TargetAsmStreamer : public AArch64TargetStreamer {
       return;
     }
 
-    unsigned VendorID = AArch64BuildAttrs::getVendorID(VendorName);
+    unsigned VendorID = AArch64BuildAttributes::getVendorID(VendorName);
 
     switch (VendorID) {
-    case AArch64BuildAttrs::VENDOR_UNKNOWN:
+    case AArch64BuildAttributes::VENDOR_UNKNOWN:
       if (unsigned(-1) != Value) {
         OS << "\t.aeabi_attribute" << "\t" << Tag << ", " << Value;
         AArch64TargetStreamer::emitAttribute(VendorName, Tag, Value, "");
@@ -176,7 +176,7 @@ class AArch64TargetAsmStreamer : public AArch64TargetStreamer {
       }
       break;
     // Note: AEABI_FEATURE_AND_BITS takes only unsigned values
-    case AArch64BuildAttrs::AEABI_FEATURE_AND_BITS:
+    case AArch64BuildAttributes::AEABI_FEATURE_AND_BITS:
       switch (Tag) {
       default: // allow emitting any attribute by number
         OS << "\t.aeabi_attribute" << "\t" << Tag << ", " << Value;
@@ -184,17 +184,17 @@ class AArch64TargetAsmStreamer : public AArch64TargetStreamer {
         // (important for llvm-mc asm parsing)
         AArch64TargetStreamer::emitAttribute(VendorName, Tag, Value, "");
         break;
-      case AArch64BuildAttrs::TAG_FEATURE_BTI:
-      case AArch64BuildAttrs::TAG_FEATURE_GCS:
-      case AArch64BuildAttrs::TAG_FEATURE_PAC:
+      case AArch64BuildAttributes::TAG_FEATURE_BTI:
+      case AArch64BuildAttributes::TAG_FEATURE_GCS:
+      case AArch64BuildAttributes::TAG_FEATURE_PAC:
         OS << "\t.aeabi_attribute" << "\t" << Tag << ", " << Value << "\t// "
-           << AArch64BuildAttrs::getFeatureAndBitsTagsStr(Tag);
+           << AArch64BuildAttributes::getFeatureAndBitsTagsStr(Tag);
         AArch64TargetStreamer::emitAttribute(VendorName, Tag, Value, "");
         break;
       }
       break;
     // Note: AEABI_PAUTHABI takes only unsigned values
-    case AArch64BuildAttrs::AEABI_PAUTHABI:
+    case AArch64BuildAttributes::AEABI_PAUTHABI:
       switch (Tag) {
       default: // allow emitting any attribute by number
         OS << "\t.aeabi_attribute" << "\t" << Tag << ", " << Value;
@@ -202,10 +202,10 @@ class AArch64TargetAsmStreamer : public AArch64TargetStreamer {
         // (important for llvm-mc asm parsing)
         AArch64TargetStreamer::emitAttribute(VendorName, Tag, Value, "");
         break;
-      case AArch64BuildAttrs::TAG_PAUTH_PLATFORM:
-      case AArch64BuildAttrs::TAG_PAUTH_SCHEMA:
+      case AArch64BuildAttributes::TAG_PAUTH_PLATFORM:
+      case AArch64BuildAttributes::TAG_PAUTH_SCHEMA:
         OS << "\t.aeabi_attribute" << "\t" << Tag << ", " << Value << "\t// "
-           << AArch64BuildAttrs::getPauthABITagsStr(Tag);
+           << AArch64BuildAttributes::getPauthABITagsStr(Tag);
         AArch64TargetStreamer::emitAttribute(VendorName, Tag, Value, "");
         break;
       }
@@ -215,42 +215,43 @@ class AArch64TargetAsmStreamer : public AArch64TargetStreamer {
   }
 
   void emitAtributesSubsection(
-      StringRef SubsectionName, AArch64BuildAttrs::SubsectionOptional Optional,
-      AArch64BuildAttrs::SubsectionType ParameterType) override {
+      StringRef SubsectionName,
+      AArch64BuildAttributes::SubsectionOptional Optional,
+      AArch64BuildAttributes::SubsectionType ParameterType) override {
     // The AArch64 build attributes assembly subsection header format:
     // ".aeabi_subsection name, optional, parameter type"
     // optional: required (0) optional (1)
     // parameter type: uleb128 or ULEB128 (0) ntbs or NTBS (1)
-    unsigned SubsectionID = AArch64BuildAttrs::getVendorID(SubsectionName);
+    unsigned SubsectionID = AArch64BuildAttributes::getVendorID(SubsectionName);
 
     assert((0 == Optional || 1 == Optional) &&
-           AArch64BuildAttrs::getSubsectionOptionalUnknownError().data());
+           AArch64BuildAttributes::getSubsectionOptionalUnknownError().data());
     assert((0 == ParameterType || 1 == ParameterType) &&
-           AArch64BuildAttrs::getSubsectionTypeUnknownError().data());
+           AArch64BuildAttributes::getSubsectionTypeUnknownError().data());
 
     std::string SubsectionTag = ".aeabi_subsection";
     StringRef OptionalStr = getOptionalStr(Optional);
     StringRef ParameterStr = getTypeStr(ParameterType);
 
     switch (SubsectionID) {
-    case AArch64BuildAttrs::VENDOR_UNKNOWN: {
+    case AArch64BuildAttributes::VENDOR_UNKNOWN: {
       // Private subsection
       break;
     }
-    case AArch64BuildAttrs::AEABI_PAUTHABI: {
-      assert(AArch64BuildAttrs::REQUIRED == Optional &&
+    case AArch64BuildAttributes::AEABI_PAUTHABI: {
+      assert(AArch64BuildAttributes::REQUIRED == Optional &&
              "subsection .aeabi-pauthabi should be marked as "
              "required and not as optional");
-      assert(AArch64BuildAttrs::ULEB128 == ParameterType &&
+      assert(AArch64BuildAttributes::ULEB128 == ParameterType &&
              "subsection .aeabi-pauthabi should be "
              "marked as uleb128 and not as ntbs");
       break;
     }
-    case AArch64BuildAttrs::AEABI_FEATURE_AND_BITS: {
-      assert(AArch64BuildAttrs::OPTIONAL == Optional &&
+    case AArch64BuildAttributes::AEABI_FEATURE_AND_BITS: {
+      assert(AArch64BuildAttributes::OPTIONAL == Optional &&
              "subsection .aeabi_feature_and_bits should be "
              "marked as optional and not as required");
-      assert(AArch64BuildAttrs::ULEB128 == ParameterType &&
+      assert(AArch64BuildAttributes::ULEB128 == ParameterType &&
              "subsection .aeabi_feature_and_bits should "
              "be marked as uleb128 and not as ntbs");
       break;
@@ -416,8 +417,8 @@ AArch64ELFStreamer &AArch64TargetELFStreamer::getStreamer() {
 }
 
 void AArch64TargetELFStreamer::emitAtributesSubsection(
-    StringRef VendorName, AArch64BuildAttrs::SubsectionOptional IsOptional,
-    AArch64BuildAttrs::SubsectionType ParameterType) {
+    StringRef VendorName, AArch64BuildAttributes::SubsectionOptional IsOptional,
+    AArch64BuildAttributes::SubsectionType ParameterType) {
   AArch64TargetStreamer::emitAtributesSubsection(VendorName, IsOptional,
                                                  ParameterType);
 }
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp
index 028d9196613cb..04defbc977462 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp
@@ -154,8 +154,8 @@ MCTargetStreamer *llvm::createAArch64NullTargetStreamer(MCStreamer &S) {
 }
 
 void AArch64TargetStreamer::emitAtributesSubsection(
-    StringRef VendorName, AArch64BuildAttrs::SubsectionOptional IsOptional,
-    AArch64BuildAttrs::SubsectionType ParameterType) {
+    StringRef VendorName, AArch64BuildAttributes::SubsectionOptional IsOptional,
+    AArch64BuildAttributes::SubsectionType ParameterType) {
 
   // If exists, return.
   for (MCELFStreamer::AttributeSubSection &SubSection : AttributeSubSections) {
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h
index 9cbb104c0eb9e..43e099f919999 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h
@@ -97,8 +97,8 @@ class AArch64TargetStreamer : public MCTargetStreamer {
   /// Build attributes implementation
   virtual void
   emitAtributesSubsection(StringRef VendorName,
-                          AArch64BuildAttrs::SubsectionOptional IsOptional,
-                          AArch64BuildAttrs::SubsectionType ParameterType);
+                          AArch64BuildAttributes::SubsectionOptional IsOptional,
+                          AArch64BuildAttributes::SubsectionType ParameterType);
   virtual void emitAttribute(StringRef VendorName, unsigned Tag, unsigned Value,
                              std::string String);
   void activateAtributesSubsection(StringRef VendorName);
@@ -124,8 +124,9 @@ class AArch64TargetELFStreamer : public AArch64TargetStreamer {
 
   /// Build attributes implementation
   void emitAtributesSubsection(
-      StringRef VendorName, AArch64BuildAttrs::SubsectionOptional IsOptional,
-      AArch64BuildAttrs::SubsectionType ParameterType) override;
+      StringRef VendorName,
+      AArch64BuildAttributes::SubsectionOptional IsOptional,
+      AArch64BuildAttributes::SubsectionType ParameterType) override;
   void emitAttribute(StringRef VendorName, unsigned Tag, unsigned Value,
                      std::string String) override;
   void emitInst(uint32_t Inst) override;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 57297288eecb4..90ba5cfb2f65a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -156,8 +156,8 @@ struct AMDGPULowerBufferFatPointersPass
   const TargetMachine &TM;
 };
 
-void initializeAMDGPUReserveWWMRegsPass(PassRegistry &);
-extern char &AMDGPUReserveWWMRegsID;
+void initializeAMDGPUReserveWWMRegsLegacyPass(PassRegistry &);
+extern char &AMDGPUReserveWWMRegsLegacyID;
 
 void initializeAMDGPURewriteOutArgumentsPass(PassRegistry &);
 extern char &AMDGPURewriteOutArgumentsID;
@@ -358,6 +358,13 @@ class SIModeRegisterPass : public PassInfoMixin<SIModeRegisterPass> {
   PreservedAnalyses run(MachineFunction &F, MachineFunctionAnalysisManager &AM);
 };
 
+class SIMemoryLegalizerPass : public PassInfoMixin<SIMemoryLegalizerPass> {
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+  static bool isRequired() { return true; }
+};
+
 FunctionPass *createAMDGPUAnnotateUniformValuesLegacy();
 
 ModulePass *createAMDGPUPrintfRuntimeBinding();
@@ -422,7 +429,7 @@ class SIAnnotateControlFlowPass
 void initializeSIAnnotateControlFlowLegacyPass(PassRegistry &);
 extern char &SIAnnotateControlFlowLegacyPassID;
 
-void initializeSIMemoryLegalizerPass(PassRegistry&);
+void initializeSIMemoryLegalizerLegacyPass(PassRegistry &);
 extern char &SIMemoryLegalizerID;
 
 void initializeSIModeRegisterLegacyPass(PassRegistry &);
@@ -456,9 +463,9 @@ void initializeAMDGPUExternalAAWrapperPass(PassRegistry&);
 
 void initializeAMDGPUArgumentUsageInfoPass(PassRegistry &);
 
-ModulePass *createAMDGPUOpenCLEnqueuedBlockLoweringLegacyPass();
-void initializeAMDGPUOpenCLEnqueuedBlockLoweringLegacyPass(PassRegistry &);
-extern char &AMDGPUOpenCLEnqueuedBlockLoweringLegacyID;
+ModulePass *createAMDGPUExportKernelRuntimeHandlesLegacyPass();
+void initializeAMDGPUExportKernelRuntimeHandlesLegacyPass(PassRegistry &);
+extern char &AMDGPUExportKernelRuntimeHandlesLegacyID;
 
 void initializeGCNNSAReassignLegacyPass(PassRegistry &);
 extern char &GCNNSAReassignID;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index effc8d2ed6b49..1c8dc09d3060b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -119,6 +119,12 @@ def FeatureUnalignedDSAccess : SubtargetFeature<"unaligned-ds-access",
   "Hardware supports unaligned local and region loads and stores"
 >;
 
+def FeatureRelaxedBufferOOBMode : SubtargetFeature<"relaxed-buffer-oob-mode",
+  "RelaxedBufferOOBMode",
+  "true",
+  "Disable strict out-of-bounds buffer guarantees. An OOB access may potentially cause an adjacent access to be treated as if it were also OOB"
+>;
+
 def FeatureApertureRegs : SubtargetFeature<"aperture-regs",
   "HasApertureRegs",
   "true",
@@ -238,6 +244,12 @@ def FeatureInstFwdPrefetchBug : SubtargetFeature<"inst-fwd-prefetch-bug",
   "S_INST_PREFETCH instruction causes shader to hang"
 >;
 
+def FeatureSafeSmemPrefetch : SubtargetFeature<"safe-smem-prefetch",
+  "HasSafeSmemPrefetch",
+  "true",
+  "SMEM prefetches do not fail on illegal address"
+>;
+
 def FeatureVcmpxExecWARHazard : SubtargetFeature<"vcmpx-exec-war-hazard",
   "HasVcmpxExecWARHazard",
   "true",
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td b/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td
index 80969fce3d77f..e891fdba4e03e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td
@@ -91,7 +91,15 @@ def CSR_AMDGPU_AGPRs : CalleeSavedRegs<
 >;
 
 def CSR_AMDGPU_SGPRs : CalleeSavedRegs<
-  (sequence "SGPR%u", 30, 105)
+  // Ensure that s30-s31 (return address), s32 (stack pointer), s33 (frame pointer),
+  // and s34 (base pointer) are callee-saved. The striped layout starts from s40,
+  // with a stripe width of 8. The last stripe is 10 wide instead of 8, to avoid
+  // ending with a 2-wide stripe.
+  (add (sequence "SGPR%u", 30, 39),
+       (sequence "SGPR%u", 48, 55),
+       (sequence "SGPR%u", 64, 71),
+       (sequence "SGPR%u", 80, 87),
+       (sequence "SGPR%u", 96, 105))
 >;
 
 def CSR_AMDGPU_SI_Gfx_SGPRs : CalleeSavedRegs<
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUExportKernelRuntimeHandles.cpp b/llvm/lib/Target/AMDGPU/AMDGPUExportKernelRuntimeHandles.cpp
new file mode 100644
index 0000000000000..a42d94d56b802
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUExportKernelRuntimeHandles.cpp
@@ -0,0 +1,110 @@
+//===- AMDGPUExportKernelRuntimeHandles.cpp - Lower enqueued block --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// \file
+//
+// Give any globals used for OpenCL block enqueue runtime handles external
+// linkage so the runtime may access them. These should behave like internal
+// functions for purposes of linking, but need to have an external symbol in the
+// final object for the runtime to access them.
+//
+// TODO: This could be replaced with a new linkage type or global object
+// metadata that produces an external symbol in the final object, but allows
+// rename on IR linking. Alternatively if we can rely on
+// GlobalValue::getGlobalIdentifier we can just make these external symbols to
+// begin with.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUExportKernelRuntimeHandles.h"
+#include "AMDGPU.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Pass.h"
+
+#define DEBUG_TYPE "amdgpu-export-kernel-runtime-handles"
+
+using namespace llvm;
+
+namespace {
+
+/// Lower enqueued blocks.
+class AMDGPUExportKernelRuntimeHandlesLegacy : public ModulePass {
+public:
+  static char ID;
+
+  explicit AMDGPUExportKernelRuntimeHandlesLegacy() : ModulePass(ID) {}
+
+private:
+  bool runOnModule(Module &M) override;
+};
+
+} // end anonymous namespace
+
+char AMDGPUExportKernelRuntimeHandlesLegacy::ID = 0;
+
+char &llvm::AMDGPUExportKernelRuntimeHandlesLegacyID =
+    AMDGPUExportKernelRuntimeHandlesLegacy::ID;
+
+INITIALIZE_PASS(AMDGPUExportKernelRuntimeHandlesLegacy, DEBUG_TYPE,
+                "Externalize enqueued block runtime handles", false, false)
+
+ModulePass *llvm::createAMDGPUExportKernelRuntimeHandlesLegacyPass() {
+  return new AMDGPUExportKernelRuntimeHandlesLegacy();
+}
+
+static bool exportKernelRuntimeHandles(Module &M) {
+  bool Changed = false;
+
+  const StringLiteral HandleSectionName(".amdgpu.kernel.runtime.handle");
+
+  for (GlobalVariable &GV : M.globals()) {
+    if (GV.getSection() == HandleSectionName) {
+      GV.setLinkage(GlobalValue::ExternalLinkage);
+      GV.setDSOLocal(false);
+      Changed = true;
+    }
+  }
+
+  if (!Changed)
+    return false;
+
+  // FIXME: We shouldn't really need to export the kernel address. We can
+  // initialize the runtime handle with the kernel descriptor.
+  for (Function &F : M) {
+    if (F.getCallingConv() != CallingConv::AMDGPU_KERNEL)
+      continue;
+
+    const MDNode *Associated = F.getMetadata(LLVMContext::MD_associated);
+    if (!Associated)
+      continue;
+
+    auto *VM = cast<ValueAsMetadata>(Associated->getOperand(0));
+    auto *Handle = dyn_cast<GlobalObject>(VM->getValue());
+    if (Handle && Handle->getSection() == HandleSectionName) {
+      F.setLinkage(GlobalValue::ExternalLinkage);
+      F.setVisibility(GlobalValue::ProtectedVisibility);
+    }
+  }
+
+  return Changed;
+}
+
+bool AMDGPUExportKernelRuntimeHandlesLegacy::runOnModule(Module &M) {
+  return exportKernelRuntimeHandles(M);
+}
+
+PreservedAnalyses
+AMDGPUExportKernelRuntimeHandlesPass::run(Module &M,
+                                          ModuleAnalysisManager &MAM) {
+  if (!exportKernelRuntimeHandles(M))
+    return PreservedAnalyses::all();
+
+  PreservedAnalyses PA;
+  PA.preserveSet<AllAnalysesOn<Function>>();
+  return PA;
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUOpenCLEnqueuedBlockLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUExportKernelRuntimeHandles.h
similarity index 50%
rename from llvm/lib/Target/AMDGPU/AMDGPUOpenCLEnqueuedBlockLowering.h
rename to llvm/lib/Target/AMDGPU/AMDGPUExportKernelRuntimeHandles.h
index 16ed7c18d8523..6e7ebc977668b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUOpenCLEnqueuedBlockLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUExportKernelRuntimeHandles.h
@@ -1,4 +1,4 @@
-//===- AMDGPUOpenCLEnqueuedBlockLowering.h -----------------------*- C++-*-===//
+//===- AMDGPUExportKernelRuntimeHandles.h -----------------------*- C++-*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,18 +6,18 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIB_TARGET_AMDGPU_OPENCLENQUEUEDBLOCKLOWERING_H
-#define LLVM_LIB_TARGET_AMDGPU_OPENCLENQUEUEDBLOCKLOWERING_H
+#ifndef LLVM_LIB_TARGET_AMDGPU_EXPORTKERNELRUNTIMEHANDLES_H
+#define LLVM_LIB_TARGET_AMDGPU_EXPORTKERNELRUNTIMEHANDLES_H
 
 #include "llvm/IR/PassManager.h"
 
 namespace llvm {
-class AMDGPUOpenCLEnqueuedBlockLoweringPass
-    : public PassInfoMixin<AMDGPUOpenCLEnqueuedBlockLoweringPass> {
+class AMDGPUExportKernelRuntimeHandlesPass
+    : public PassInfoMixin<AMDGPUExportKernelRuntimeHandlesPass> {
 public:
-  AMDGPUOpenCLEnqueuedBlockLoweringPass() = default;
+  AMDGPUExportKernelRuntimeHandlesPass() = default;
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
 };
 } // namespace llvm
 
-#endif // LLVM_LIB_TARGET_AMDGPU_OPENCLENQUEUEDBLOCKLOWERING_H
+#endif // LLVM_LIB_TARGET_AMDGPU_EXPORTKERNELRUNTIMEHANDLES_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
index fb258547e8fb9..a5660432603fd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
@@ -16,6 +16,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPU.h"
+#include "AMDGPUGlobalISelUtils.h"
 #include "SILowerI1Copies.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -78,6 +79,9 @@ class DivergenceLoweringHelper : public PhiLoweringHelper {
                            Register DstReg, Register PrevReg,
                            Register CurReg) override;
   void constrainAsLaneMask(Incoming &In) override;
+
+  bool lowerTemporalDivergence();
+  bool lowerTemporalDivergenceI1();
 };
 
 DivergenceLoweringHelper::DivergenceLoweringHelper(
@@ -188,6 +192,100 @@ void DivergenceLoweringHelper::constrainAsLaneMask(Incoming &In) {
   In.Reg = Copy.getReg(0);
 }
 
+void replaceUsesOfRegInInstWith(Register Reg, MachineInstr *Inst,
+                                Register NewReg) {
+  for (MachineOperand &Op : Inst->operands()) {
+    if (Op.isReg() && Op.getReg() == Reg)
+      Op.setReg(NewReg);
+  }
+}
+
+bool DivergenceLoweringHelper::lowerTemporalDivergence() {
+  AMDGPU::IntrinsicLaneMaskAnalyzer ILMA(*MF);
+  DenseMap<Register, Register> TDCache;
+
+  for (auto [Reg, UseInst, _] : MUI->getTemporalDivergenceList()) {
+    if (MRI->getType(Reg) == LLT::scalar(1) || MUI->isDivergent(Reg) ||
+        ILMA.isS32S64LaneMask(Reg))
+      continue;
+
+    Register CachedTDCopy = TDCache.lookup(Reg);
+    if (CachedTDCopy) {
+      replaceUsesOfRegInInstWith(Reg, UseInst, CachedTDCopy);
+      continue;
+    }
+
+    MachineInstr *Inst = MRI->getVRegDef(Reg);
+    MachineBasicBlock *MBB = Inst->getParent();
+    B.setInsertPt(*MBB, MBB->SkipPHIsAndLabels(std::next(Inst->getIterator())));
+
+    Register VgprReg = MRI->createGenericVirtualRegister(MRI->getType(Reg));
+    B.buildInstr(AMDGPU::COPY, {VgprReg}, {Reg})
+        .addUse(ExecReg, RegState::Implicit);
+
+    replaceUsesOfRegInInstWith(Reg, UseInst, VgprReg);
+    TDCache[Reg] = VgprReg;
+  }
+  return false;
+}
+
+bool DivergenceLoweringHelper::lowerTemporalDivergenceI1() {
+  MachineRegisterInfo::VRegAttrs BoolS1 = {ST->getBoolRC(), LLT::scalar(1)};
+  initializeLaneMaskRegisterAttributes(BoolS1);
+  MachineSSAUpdater SSAUpdater(*MF);
+
+  // In case of use outside muliple nested cycles or muliple uses we only need
+  // to merge lane mask across largest relevant cycle.
+  SmallDenseMap<Register, std::pair<const MachineCycle *, Register>> LRCCache;
+  for (auto [Reg, UseInst, LRC] : MUI->getTemporalDivergenceList()) {
+    if (MRI->getType(Reg) != LLT::scalar(1))
+      continue;
+
+    auto [LRCCacheIter, RegNotCached] = LRCCache.try_emplace(Reg);
+    auto &CycleMergedMask = LRCCacheIter->getSecond();
+    const MachineCycle *&CachedLRC = CycleMergedMask.first;
+    if (RegNotCached || LRC->contains(CachedLRC)) {
+      CachedLRC = LRC;
+    }
+  }
+
+  for (auto &LRCCacheEntry : LRCCache) {
+    Register Reg = LRCCacheEntry.first;
+    auto &CycleMergedMask = LRCCacheEntry.getSecond();
+    const MachineCycle *Cycle = CycleMergedMask.first;
+
+    Register MergedMask = MRI->createVirtualRegister(BoolS1);
+    SSAUpdater.Initialize(MergedMask);
+
+    MachineBasicBlock *MBB = MRI->getVRegDef(Reg)->getParent();
+    SSAUpdater.AddAvailableValue(MBB, MergedMask);
+
+    for (auto Entry : Cycle->getEntries()) {
+      for (MachineBasicBlock *Pred : Entry->predecessors()) {
+        if (!Cycle->contains(Pred)) {
+          B.setInsertPt(*Pred, Pred->getFirstTerminator());
+          auto ImplDef = B.buildInstr(AMDGPU::IMPLICIT_DEF, {BoolS1}, {});
+          SSAUpdater.AddAvailableValue(Pred, ImplDef.getReg(0));
+        }
+      }
+    }
+
+    buildMergeLaneMasks(*MBB, MBB->getFirstTerminator(), {}, MergedMask,
+                        SSAUpdater.GetValueInMiddleOfBlock(MBB), Reg);
+
+    CycleMergedMask.second = MergedMask;
+  }
+
+  for (auto [Reg, UseInst, Cycle] : MUI->getTemporalDivergenceList()) {
+    if (MRI->getType(Reg) != LLT::scalar(1))
+      continue;
+
+    replaceUsesOfRegInInstWith(Reg, UseInst, LRCCache.lookup(Reg).second);
+  }
+
+  return false;
+}
+
 } // End anonymous namespace.
 
 INITIALIZE_PASS_BEGIN(AMDGPUGlobalISelDivergenceLowering, DEBUG_TYPE,
@@ -218,5 +316,21 @@ bool AMDGPUGlobalISelDivergenceLowering::runOnMachineFunction(
 
   DivergenceLoweringHelper Helper(&MF, &DT, &PDT, &MUI);
 
-  return Helper.lowerPhis();
+  bool Changed = false;
+  // Temporal divergence lowering needs to inspect list of instructions used
+  // outside cycle with divergent exit provided by uniformity analysis. Uniform
+  // instructions from the list require lowering, no instruction is deleted.
+  // Thus it needs to be run before lowerPhis that deletes phis that require
+  // lowering and replaces them with new instructions.
+
+  // Non-i1 temporal divergence lowering.
+  Changed |= Helper.lowerTemporalDivergence();
+  // This covers both uniform and divergent i1s. Lane masks are in sgpr and need
+  // to be updated in each iteration.
+  Changed |= Helper.lowerTemporalDivergenceI1();
+  // Temporal divergence lowering of divergent i1 phi used outside of the cycle
+  // could also be handled by lowerPhis but we do it in lowerTempDivergenceI1
+  // since in some case lowerPhis does unnecessary lane mask merging.
+  Changed |= Helper.lowerPhis();
+  return Changed;
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
index 0b18c6b0e923a..47e32c4864809 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
@@ -91,25 +91,17 @@ void IntrinsicLaneMaskAnalyzer::initLaneMaskIntrinsics(MachineFunction &MF) {
       GIntrinsic *GI = dyn_cast<GIntrinsic>(&MI);
       if (GI && GI->is(Intrinsic::amdgcn_if_break)) {
         S32S64LaneMask.insert(MI.getOperand(3).getReg());
-        findLCSSAPhi(MI.getOperand(0).getReg());
+        S32S64LaneMask.insert(MI.getOperand(0).getReg());
       }
 
       if (MI.getOpcode() == AMDGPU::SI_IF ||
           MI.getOpcode() == AMDGPU::SI_ELSE) {
-        findLCSSAPhi(MI.getOperand(0).getReg());
+        S32S64LaneMask.insert(MI.getOperand(0).getReg());
       }
     }
   }
 }
 
-void IntrinsicLaneMaskAnalyzer::findLCSSAPhi(Register Reg) {
-  S32S64LaneMask.insert(Reg);
-  for (const MachineInstr &LCSSAPhi : MRI.use_instructions(Reg)) {
-    if (LCSSAPhi.isPHI())
-      S32S64LaneMask.insert(LCSSAPhi.getOperand(0).getReg());
-  }
-}
-
 static LLT getReadAnyLaneSplitTy(LLT Ty) {
   if (Ty.isVector()) {
     LLT ElTy = Ty.getElementType();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h
index 27f8fed86d647..70cfdacec700c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h
@@ -47,8 +47,6 @@ class IntrinsicLaneMaskAnalyzer {
 
 private:
   void initLaneMaskIntrinsics(MachineFunction &MF);
-  // This will not be needed when we turn off LCSSA for global-isel.
-  void findLCSSAPhi(Register Reg);
 };
 
 void buildReadAnyLane(MachineIRBuilder &B, Register SgprDst, Register VgprSrc,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
index ee8a700f988dc..2991778a1bbc7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
@@ -21,6 +21,8 @@
 #include "llvm/IR/Module.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/Target/TargetLoweringObjectFile.h"
+
 using namespace llvm;
 
 static std::pair<Type *, Align> getArgumentTypeAlign(const Argument &Arg,
@@ -38,6 +40,27 @@ static std::pair<Type *, Align> getArgumentTypeAlign(const Argument &Arg,
   return std::pair(Ty, *ArgAlign);
 }
 
+/// Find the mangled symbol name for the runtime handle for \p EnqueuedBlock
+static std::string getEnqueuedBlockSymbolName(const AMDGPUTargetMachine &TM,
+                                              const Function &EnqueuedBlock) {
+  const MDNode *Associated =
+      EnqueuedBlock.getMetadata(LLVMContext::MD_associated);
+  if (!Associated)
+    return "";
+
+  auto *VM = cast<ValueAsMetadata>(Associated->getOperand(0));
+  auto *RuntimeHandle =
+      dyn_cast<GlobalVariable>(VM->getValue()->stripPointerCasts());
+  if (!RuntimeHandle ||
+      RuntimeHandle->getSection() != ".amdgpu.kernel.runtime.handle")
+    return "";
+
+  SmallString<128> Name;
+  TM.getNameWithPrefix(Name, RuntimeHandle,
+                       TM.getObjFileLowering()->getMangler());
+  return Name.str().str();
+}
+
 namespace llvm {
 
 static cl::opt<bool> DumpHSAMetadata(
@@ -230,7 +253,8 @@ void MetadataStreamerMsgPackV4::emitKernelLanguage(const Function &Func,
   Kern[".language_version"] = LanguageVersion;
 }
 
-void MetadataStreamerMsgPackV4::emitKernelAttrs(const Function &Func,
+void MetadataStreamerMsgPackV4::emitKernelAttrs(const AMDGPUTargetMachine &TM,
+                                                const Function &Func,
                                                 msgpack::MapDocNode Kern) {
 
   if (auto *Node = Func.getMetadata("reqd_work_group_size"))
@@ -244,11 +268,13 @@ void MetadataStreamerMsgPackV4::emitKernelAttrs(const Function &Func,
             mdconst::extract<ConstantInt>(Node->getOperand(1))->getZExtValue()),
         /*Copy=*/true);
   }
-  if (Func.hasFnAttribute("runtime-handle")) {
-    Kern[".device_enqueue_symbol"] = Kern.getDocument()->getNode(
-        Func.getFnAttribute("runtime-handle").getValueAsString().str(),
-        /*Copy=*/true);
+
+  std::string HandleName = getEnqueuedBlockSymbolName(TM, Func);
+  if (!HandleName.empty()) {
+    Kern[".device_enqueue_symbol"] =
+        Kern.getDocument()->getNode(std::move(HandleName), /*Copy=*/true);
   }
+
   if (Func.hasFnAttribute("device-init"))
     Kern[".kind"] = Kern.getDocument()->getNode("init");
   else if (Func.hasFnAttribute("device-fini"))
@@ -567,12 +593,13 @@ void MetadataStreamerMsgPackV4::emitKernel(const MachineFunction &MF,
   auto Kernels =
       getRootMetadata("amdhsa.kernels").getArray(/*Convert=*/true);
 
+  auto &TM = static_cast<const AMDGPUTargetMachine &>(MF.getTarget());
   {
     Kern[".name"] = Kern.getDocument()->getNode(Func.getName());
     Kern[".symbol"] = Kern.getDocument()->getNode(
         (Twine(Func.getName()) + Twine(".kd")).str(), /*Copy=*/true);
     emitKernelLanguage(Func, Kern);
-    emitKernelAttrs(Func, Kern);
+    emitKernelAttrs(TM, Func, Kern);
     emitKernelArgs(MF, Kern);
   }
 
@@ -698,9 +725,10 @@ void MetadataStreamerMsgPackV5::emitHiddenKernelArgs(
     emitKernelArg(DL, Int8PtrTy, Align(8), "hidden_queue_ptr", Offset, Args);
 }
 
-void MetadataStreamerMsgPackV5::emitKernelAttrs(const Function &Func,
+void MetadataStreamerMsgPackV5::emitKernelAttrs(const AMDGPUTargetMachine &TM,
+                                                const Function &Func,
                                                 msgpack::MapDocNode Kern) {
-  MetadataStreamerMsgPackV4::emitKernelAttrs(Func, Kern);
+  MetadataStreamerMsgPackV4::emitKernelAttrs(TM, Func, Kern);
 
   if (Func.getFnAttribute("uniform-work-group-size").getValueAsBool())
     Kern[".uniform_work_group_size"] = Kern.getDocument()->getNode(1);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
index fd76666dc360b..22dfcb4a4ec1d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
@@ -23,6 +23,7 @@
 
 namespace llvm {
 
+class AMDGPUTargetMachine;
 class AMDGPUTargetStreamer;
 class Argument;
 class DataLayout;
@@ -59,7 +60,8 @@ class MetadataStreamer {
   virtual void emitVersion() = 0;
   virtual void emitHiddenKernelArgs(const MachineFunction &MF, unsigned &Offset,
                                     msgpack::ArrayDocNode Args) = 0;
-  virtual void emitKernelAttrs(const Function &Func,
+  virtual void emitKernelAttrs(const AMDGPUTargetMachine &TM,
+                               const Function &Func,
                                msgpack::MapDocNode Kern) = 0;
 };
 
@@ -100,7 +102,8 @@ class LLVM_EXTERNAL_VISIBILITY MetadataStreamerMsgPackV4
 
   void emitKernelLanguage(const Function &Func, msgpack::MapDocNode Kern);
 
-  void emitKernelAttrs(const Function &Func, msgpack::MapDocNode Kern) override;
+  void emitKernelAttrs(const AMDGPUTargetMachine &TM, const Function &Func,
+                       msgpack::MapDocNode Kern) override;
 
   void emitKernelArgs(const MachineFunction &MF, msgpack::MapDocNode Kern);
 
@@ -146,7 +149,8 @@ class MetadataStreamerMsgPackV5 : public MetadataStreamerMsgPackV4 {
   void emitVersion() override;
   void emitHiddenKernelArgs(const MachineFunction &MF, unsigned &Offset,
                             msgpack::ArrayDocNode Args) override;
-  void emitKernelAttrs(const Function &Func, msgpack::MapDocNode Kern) override;
+  void emitKernelAttrs(const AMDGPUTargetMachine &TM, const Function &Func,
+                       msgpack::MapDocNode Kern) override;
 
 public:
   MetadataStreamerMsgPackV5() = default;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp
index 179d8aa46f802..639089c75a33e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp
@@ -256,7 +256,7 @@ bool optimizeSection(ArrayRef<SmallVector<IntrinsicInst *, 4>> MergeableInsts) {
         VecOp = B.CreateExtractElement(NewCalls[0], Idx->getValue().urem(4));
         LLVM_DEBUG(dbgs() << "Add: " << *VecOp << "\n");
       } else {
-        VecOp = UndefValue::get(II->getType());
+        VecOp = PoisonValue::get(II->getType());
         for (unsigned I = 0; I < NumElts; ++I) {
           VecOp = B.CreateInsertElement(
               VecOp,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
index b3e371cdff8fd..b25619b4c5422 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
@@ -400,8 +400,8 @@ class AMDGPUInsertDelayAlu {
     if (Emit) {
       assert(State == BlockState[&MBB] &&
              "Basic block state should not have changed on final pass!");
-    } else if (State != BlockState[&MBB]) {
-      BlockState[&MBB] = std::move(State);
+    } else if (DelayState &BS = BlockState[&MBB]; State != BS) {
+      BS = std::move(State);
       Changed = true;
     }
     return Changed;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index ebe740f884ea6..ef076814ffdab 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -1574,35 +1574,59 @@ Value *GCNTTIImpl::simplifyAMDGCNLaneIntrinsicDemanded(
   const unsigned LastElt = DemandedElts.getActiveBits() - 1;
   const unsigned MaskLen = LastElt - FirstElt + 1;
 
-  // TODO: Handle general subvector extract.
-  if (MaskLen != 1)
+  unsigned OldNumElts = VT->getNumElements();
+  if (MaskLen == OldNumElts && MaskLen != 1)
     return nullptr;
 
   Type *EltTy = VT->getElementType();
-  if (!isTypeLegal(EltTy))
+  Type *NewVT = MaskLen == 1 ? EltTy : FixedVectorType::get(EltTy, MaskLen);
+
+  // Theoretically we should support these intrinsics for any legal type. Avoid
+  // introducing cases that aren't direct register types like v3i16.
+  if (!isTypeLegal(NewVT))
     return nullptr;
 
   Value *Src = II.getArgOperand(0);
 
-  assert(FirstElt == LastElt);
-  Value *Extract = IC.Builder.CreateExtractElement(Src, FirstElt);
-
   // Make sure convergence tokens are preserved.
   // TODO: CreateIntrinsic should allow directly copying bundles
   SmallVector<OperandBundleDef, 2> OpBundles;
   II.getOperandBundlesAsDefs(OpBundles);
 
   Module *M = IC.Builder.GetInsertBlock()->getModule();
-  Function *Remangled = Intrinsic::getOrInsertDeclaration(
-      M, II.getIntrinsicID(), {Extract->getType()});
+  Function *Remangled =
+      Intrinsic::getOrInsertDeclaration(M, II.getIntrinsicID(), {NewVT});
+
+  if (MaskLen == 1) {
+    Value *Extract = IC.Builder.CreateExtractElement(Src, FirstElt);
+
+    // TODO: Preserve callsite attributes?
+    CallInst *NewCall = IC.Builder.CreateCall(Remangled, {Extract}, OpBundles);
+
+    return IC.Builder.CreateInsertElement(PoisonValue::get(II.getType()),
+                                          NewCall, FirstElt);
+  }
+
+  SmallVector<int> ExtractMask(MaskLen, -1);
+  for (unsigned I = 0; I != MaskLen; ++I) {
+    if (DemandedElts[FirstElt + I])
+      ExtractMask[I] = FirstElt + I;
+  }
+
+  Value *Extract = IC.Builder.CreateShuffleVector(Src, ExtractMask);
 
   // TODO: Preserve callsite attributes?
   CallInst *NewCall = IC.Builder.CreateCall(Remangled, {Extract}, OpBundles);
 
+  SmallVector<int> InsertMask(OldNumElts, -1);
+  for (unsigned I = 0; I != MaskLen; ++I) {
+    if (DemandedElts[FirstElt + I])
+      InsertMask[FirstElt + I] = I;
+  }
+
   // FIXME: If the call has a convergence bundle, we end up leaving the dead
   // call behind.
-  return IC.Builder.CreateInsertElement(PoisonValue::get(II.getType()), NewCall,
-                                        FirstElt);
+  return IC.Builder.CreateShuffleVector(NewCall, InsertMask);
 }
 
 std::optional<Value *> GCNTTIImpl::simplifyDemandedVectorEltsIntrinsic(
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index 608b43b59eed3..5dd1fe14e5626 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -1620,7 +1620,7 @@ void SplitPtrStructs::killAndReplaceSplitInstructions(
         Dbg->setExpression(*RsrcExpr);
         Dbg->replaceVariableLocationOp(I, Rsrc);
       } else {
-        Dbg->replaceVariableLocationOp(I, UndefValue::get(I->getType()));
+        Dbg->replaceVariableLocationOp(I, PoisonValue::get(I->getType()));
       }
     }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUOpenCLEnqueuedBlockLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUOpenCLEnqueuedBlockLowering.cpp
deleted file mode 100644
index fbd15ad176e3b..0000000000000
--- a/llvm/lib/Target/AMDGPU/AMDGPUOpenCLEnqueuedBlockLowering.cpp
+++ /dev/null
@@ -1,136 +0,0 @@
-//===- AMDGPUOpenCLEnqueuedBlockLowering.cpp - Lower enqueued block -------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// \file
-// This post-linking pass replaces the function pointer of enqueued
-// block kernel with a global variable (runtime handle) and adds
-// "runtime-handle" attribute to the enqueued block kernel.
-//
-// In LLVM CodeGen the runtime-handle metadata will be translated to
-// RuntimeHandle metadata in code object. Runtime allocates a global buffer
-// for each kernel with RuntimeHandle metadata and saves the kernel address
-// required for the AQL packet into the buffer. __enqueue_kernel function
-// in device library knows that the invoke function pointer in the block
-// literal is actually runtime handle and loads the kernel address from it
-// and put it into AQL packet for dispatching.
-//
-// This cannot be done in FE since FE cannot create a unique global variable
-// with external linkage across LLVM modules. The global variable with internal
-// linkage does not work since optimization passes will try to replace loads
-// of the global variable with its initialization value.
-//
-// It also identifies the kernels directly or indirectly enqueues kernels
-// and adds "calls-enqueue-kernel" function attribute to them, which will
-// be used to determine whether to emit runtime metadata for the kernel
-// enqueue related hidden kernel arguments.
-//
-//===----------------------------------------------------------------------===//
-
-#include "AMDGPUOpenCLEnqueuedBlockLowering.h"
-#include "AMDGPU.h"
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/Mangler.h"
-#include "llvm/IR/Module.h"
-#include "llvm/Pass.h"
-#include "llvm/Support/Debug.h"
-
-#define DEBUG_TYPE "amdgpu-lower-enqueued-block"
-
-using namespace llvm;
-
-namespace {
-
-/// Lower enqueued blocks.
-class AMDGPUOpenCLEnqueuedBlockLowering {
-public:
-  bool run(Module &M);
-};
-
-class AMDGPUOpenCLEnqueuedBlockLoweringLegacy : public ModulePass {
-public:
-  static char ID;
-
-  explicit AMDGPUOpenCLEnqueuedBlockLoweringLegacy() : ModulePass(ID) {}
-
-private:
-  bool runOnModule(Module &M) override;
-};
-
-} // end anonymous namespace
-
-char AMDGPUOpenCLEnqueuedBlockLoweringLegacy::ID = 0;
-
-char &llvm::AMDGPUOpenCLEnqueuedBlockLoweringLegacyID =
-    AMDGPUOpenCLEnqueuedBlockLoweringLegacy::ID;
-
-INITIALIZE_PASS(AMDGPUOpenCLEnqueuedBlockLoweringLegacy, DEBUG_TYPE,
-                "Lower OpenCL enqueued blocks", false, false)
-
-ModulePass *llvm::createAMDGPUOpenCLEnqueuedBlockLoweringLegacyPass() {
-  return new AMDGPUOpenCLEnqueuedBlockLoweringLegacy();
-}
-
-bool AMDGPUOpenCLEnqueuedBlockLoweringLegacy::runOnModule(Module &M) {
-  AMDGPUOpenCLEnqueuedBlockLowering Impl;
-  return Impl.run(M);
-}
-
-PreservedAnalyses
-AMDGPUOpenCLEnqueuedBlockLoweringPass::run(Module &M, ModuleAnalysisManager &) {
-  AMDGPUOpenCLEnqueuedBlockLowering Impl;
-  if (Impl.run(M))
-    return PreservedAnalyses::none();
-  return PreservedAnalyses::all();
-}
-
-bool AMDGPUOpenCLEnqueuedBlockLowering::run(Module &M) {
-  DenseSet<Function *> Callers;
-  auto &C = M.getContext();
-  bool Changed = false;
-
-  // ptr kernel_object, i32 private_segment_size, i32 group_segment_size
-  StructType *HandleTy = nullptr;
-
-  for (auto &F : M.functions()) {
-    if (F.hasFnAttribute("enqueued-block")) {
-      if (!F.hasName()) {
-        SmallString<64> Name;
-        Mangler::getNameWithPrefix(Name, "__amdgpu_enqueued_kernel",
-                                   M.getDataLayout());
-        F.setName(Name);
-      }
-      LLVM_DEBUG(dbgs() << "found enqueued kernel: " << F.getName() << '\n');
-      auto RuntimeHandle = (F.getName() + ".runtime_handle").str();
-      if (!HandleTy) {
-        Type *Int32 = Type::getInt32Ty(C);
-        HandleTy =
-            StructType::create(C, {PointerType::getUnqual(C), Int32, Int32},
-                               "block.runtime.handle.t");
-      }
-
-      auto *GV = new GlobalVariable(
-          M, HandleTy,
-          /*isConstant=*/true, GlobalValue::ExternalLinkage,
-          /*Initializer=*/Constant::getNullValue(HandleTy), RuntimeHandle,
-          /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
-          AMDGPUAS::GLOBAL_ADDRESS,
-          /*isExternallyInitialized=*/true);
-      LLVM_DEBUG(dbgs() << "runtime handle created: " << *GV << '\n');
-
-      F.replaceAllUsesWith(ConstantExpr::getAddrSpaceCast(GV, F.getType()));
-      F.addFnAttr("runtime-handle", RuntimeHandle);
-      F.setLinkage(GlobalValue::ExternalLinkage);
-      Changed = true;
-    }
-  }
-
-  return Changed;
-}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 1050855176c04..c5d3207acf0f5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -17,10 +17,10 @@
 #define MODULE_PASS(NAME, CREATE_PASS)
 #endif
 MODULE_PASS("amdgpu-always-inline", AMDGPUAlwaysInlinePass())
+MODULE_PASS("amdgpu-export-kernel-runtime-handles", AMDGPUExportKernelRuntimeHandlesPass())
 MODULE_PASS("amdgpu-lower-buffer-fat-pointers",
             AMDGPULowerBufferFatPointersPass(*this))
 MODULE_PASS("amdgpu-lower-ctor-dtor", AMDGPUCtorDtorLoweringPass())
-MODULE_PASS("amdgpu-lower-enqueued-block", AMDGPUOpenCLEnqueuedBlockLoweringPass())
 MODULE_PASS("amdgpu-lower-module-lds", AMDGPULowerModuleLDSPass(*this))
 MODULE_PASS("amdgpu-perf-hint",
             AMDGPUPerfHintAnalysisPass(
@@ -99,6 +99,7 @@ FUNCTION_PASS_WITH_PARAMS(
 MACHINE_FUNCTION_PASS("amdgpu-insert-delay-alu", AMDGPUInsertDelayAluPass())
 MACHINE_FUNCTION_PASS("amdgpu-isel", AMDGPUISelDAGToDAGPass(*this))
 MACHINE_FUNCTION_PASS("amdgpu-pre-ra-long-branch-reg", GCNPreRALongBranchRegPass())
+MACHINE_FUNCTION_PASS("amdgpu-reserve-wwm-regs", AMDGPUReserveWWMRegsPass())
 MACHINE_FUNCTION_PASS("amdgpu-rewrite-partial-reg-uses", GCNRewritePartialRegUsesPass())
 MACHINE_FUNCTION_PASS("amdgpu-pre-ra-optimizations", GCNPreRAOptimizationsPass())
 MACHINE_FUNCTION_PASS("amdgpu-nsa-reassign", GCNNSAReassignPass())
@@ -112,6 +113,7 @@ MACHINE_FUNCTION_PASS("si-load-store-opt", SILoadStoreOptimizerPass())
 MACHINE_FUNCTION_PASS("si-lower-control-flow", SILowerControlFlowPass())
 MACHINE_FUNCTION_PASS("si-lower-sgpr-spills", SILowerSGPRSpillsPass())
 MACHINE_FUNCTION_PASS("si-lower-wwm-copies", SILowerWWMCopiesPass())
+MACHINE_FUNCTION_PASS("si-memory-legalizer", SIMemoryLegalizerPass())
 MACHINE_FUNCTION_PASS("si-mode-register", SIModeRegisterPass())
 MACHINE_FUNCTION_PASS("si-opt-vgpr-liverange", SIOptimizeVGPRLiveRangePass())
 MACHINE_FUNCTION_PASS("si-optimize-exec-masking", SIOptimizeExecMaskingPass())
@@ -131,7 +133,6 @@ DUMMY_MACHINE_FUNCTION_PASS("amdgpu-set-wave-priority", AMDGPUSetWavePriorityPas
 DUMMY_MACHINE_FUNCTION_PASS("si-insert-hard-clauses", SIInsertHardClausesPass())
 DUMMY_MACHINE_FUNCTION_PASS("si-insert-waitcnts", SIInsertWaitcntsPass())
 DUMMY_MACHINE_FUNCTION_PASS("si-late-branch-lowering", SILateBranchLoweringPass())
-DUMMY_MACHINE_FUNCTION_PASS("si-memory-legalizer", SIMemoryLegalizerPass())
 DUMMY_MACHINE_FUNCTION_PASS("si-pre-emit-peephole", SIPreEmitPeepholePass())
 // TODO: Move amdgpu-preload-kern-arg-prolog to MACHINE_FUNCTION_PASS since it
 // already exists.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 226044c892b9e..5c0ce1113b6c9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -66,6 +66,19 @@ static cl::opt<unsigned> PromoteAllocaToVectorLimit(
     cl::desc("Maximum byte size to consider promote alloca to vector"),
     cl::init(0));
 
+static cl::opt<unsigned> PromoteAllocaToVectorMaxRegs(
+    "amdgpu-promote-alloca-to-vector-max-regs",
+    cl::desc(
+        "Maximum vector size (in 32b registers) to use when promoting alloca"),
+    cl::init(16));
+
+// Use up to 1/4 of available register budget for vectorization.
+// FIXME: Increase the limit for whole function budgets? Perhaps x2?
+static cl::opt<unsigned> PromoteAllocaToVectorVGPRRatio(
+    "amdgpu-promote-alloca-to-vector-vgpr-ratio",
+    cl::desc("Ratio of VGPRs to budget for promoting alloca to vectors"),
+    cl::init(4));
+
 static cl::opt<unsigned>
     LoopUserWeight("promote-alloca-vector-loop-user-weight",
                    cl::desc("The bonus weight of users of allocas within loop "
@@ -84,6 +97,8 @@ class AMDGPUPromoteAllocaImpl {
   uint32_t LocalMemLimit = 0;
   uint32_t CurrentLocalMemUsage = 0;
   unsigned MaxVGPRs;
+  unsigned VGPRBudgetRatio;
+  unsigned MaxVectorRegs;
 
   bool IsAMDGCN = false;
   bool IsAMDHSA = false;
@@ -112,6 +127,8 @@ class AMDGPUPromoteAllocaImpl {
 
   void sortAllocasToPromote(SmallVectorImpl<AllocaInst *> &Allocas);
 
+  void setFunctionLimits(const Function &F);
+
 public:
   AMDGPUPromoteAllocaImpl(TargetMachine &TM, LoopInfo &LI) : TM(TM), LI(LI) {
 
@@ -298,6 +315,19 @@ void AMDGPUPromoteAllocaImpl::sortAllocasToPromote(
   // clang-format on
 }
 
+void AMDGPUPromoteAllocaImpl::setFunctionLimits(const Function &F) {
+  // Load per function limits, overriding with global options where appropriate.
+  MaxVectorRegs = F.getFnAttributeAsParsedInteger(
+      "amdgpu-promote-alloca-to-vector-max-regs", PromoteAllocaToVectorMaxRegs);
+  if (PromoteAllocaToVectorMaxRegs.getNumOccurrences())
+    MaxVectorRegs = PromoteAllocaToVectorMaxRegs;
+  VGPRBudgetRatio = F.getFnAttributeAsParsedInteger(
+      "amdgpu-promote-alloca-to-vector-vgpr-ratio",
+      PromoteAllocaToVectorVGPRRatio);
+  if (PromoteAllocaToVectorVGPRRatio.getNumOccurrences())
+    VGPRBudgetRatio = PromoteAllocaToVectorVGPRRatio;
+}
+
 bool AMDGPUPromoteAllocaImpl::run(Function &F, bool PromoteToLDS) {
   Mod = F.getParent();
   DL = &Mod->getDataLayout();
@@ -307,15 +337,14 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool PromoteToLDS) {
     return false;
 
   MaxVGPRs = getMaxVGPRs(TM, F);
+  setFunctionLimits(F);
 
   bool SufficientLDS = PromoteToLDS && hasSufficientLocalMem(F);
 
-  // Use up to 1/4 of available register budget for vectorization.
-  // FIXME: Increase the limit for whole function budgets? Perhaps x2?
   unsigned VectorizationBudget =
       (PromoteAllocaToVectorLimit ? PromoteAllocaToVectorLimit * 8
                                   : (MaxVGPRs * 32)) /
-      4;
+      VGPRBudgetRatio;
 
   SmallVector<AllocaInst *, 16> Allocas;
   for (Instruction &I : F.getEntryBlock()) {
@@ -400,7 +429,8 @@ static Value *calculateVectorIndex(
 }
 
 static Value *GEPToVectorIndex(GetElementPtrInst *GEP, AllocaInst *Alloca,
-                               Type *VecElemTy, const DataLayout &DL) {
+                               Type *VecElemTy, const DataLayout &DL,
+                               SmallVector<Instruction *> &NewInsts) {
   // TODO: Extracting a "multiple of X" from a GEP might be a useful generic
   // helper.
   unsigned BW = DL.getIndexTypeSizeInBits(GEP->getType());
@@ -414,22 +444,40 @@ static Value *GEPToVectorIndex(GetElementPtrInst *GEP, AllocaInst *Alloca,
   if (VarOffsets.size() > 1)
     return nullptr;
 
-  if (VarOffsets.size() == 1) {
-    // Only handle cases where we don't need to insert extra arithmetic
-    // instructions.
-    const auto &VarOffset = VarOffsets.front();
-    if (!ConstOffset.isZero() || VarOffset.second != VecElemSize)
-      return nullptr;
-    return VarOffset.first;
-  }
-
   APInt Quot;
   uint64_t Rem;
   APInt::udivrem(ConstOffset, VecElemSize, Quot, Rem);
   if (Rem != 0)
     return nullptr;
 
-  return ConstantInt::get(GEP->getContext(), Quot);
+  ConstantInt *ConstIndex = ConstantInt::get(GEP->getContext(), Quot);
+  if (VarOffsets.size() == 0)
+    return ConstIndex;
+
+  IRBuilder<> Builder(GEP);
+
+  const auto &VarOffset = VarOffsets.front();
+  APInt::udivrem(VarOffset.second, VecElemSize, Quot, Rem);
+  if (Rem != 0 || Quot.isZero())
+    return nullptr;
+
+  Value *Offset = VarOffset.first;
+  if (!Quot.isOne()) {
+    auto *OffsetType = dyn_cast<IntegerType>(Offset->getType());
+    if (!OffsetType)
+      return nullptr;
+    ConstantInt *ConstMul = ConstantInt::get(OffsetType, Quot.getZExtValue());
+    Offset = Builder.CreateMul(Offset, ConstMul);
+    if (Instruction *NewInst = dyn_cast<Instruction>(Offset))
+      NewInsts.push_back(NewInst);
+  }
+  if (ConstOffset.isZero())
+    return Offset;
+
+  Value *IndexAdd = Builder.CreateAdd(ConstIndex, Offset);
+  if (Instruction *NewInst = dyn_cast<Instruction>(IndexAdd))
+    NewInsts.push_back(NewInst);
+  return IndexAdd;
 }
 
 /// Promotes a single user of the alloca to a vector form.
@@ -737,23 +785,44 @@ bool AMDGPUPromoteAllocaImpl::tryPromoteAllocaToVector(AllocaInst &Alloca) {
   Type *AllocaTy = Alloca.getAllocatedType();
   auto *VectorTy = dyn_cast<FixedVectorType>(AllocaTy);
   if (auto *ArrayTy = dyn_cast<ArrayType>(AllocaTy)) {
-    if (VectorType::isValidElementType(ArrayTy->getElementType()) &&
-        ArrayTy->getNumElements() > 0)
-      VectorTy = FixedVectorType::get(ArrayTy->getElementType(),
-                                      ArrayTy->getNumElements());
+    uint64_t NumElems = 1;
+    Type *ElemTy;
+    do {
+      NumElems *= ArrayTy->getNumElements();
+      ElemTy = ArrayTy->getElementType();
+    } while ((ArrayTy = dyn_cast<ArrayType>(ElemTy)));
+
+    // Check for array of vectors
+    auto *InnerVectorTy = dyn_cast<FixedVectorType>(ElemTy);
+    if (InnerVectorTy) {
+      NumElems *= InnerVectorTy->getNumElements();
+      ElemTy = InnerVectorTy->getElementType();
+    }
+
+    if (VectorType::isValidElementType(ElemTy) && NumElems > 0) {
+      unsigned ElementSize = DL->getTypeSizeInBits(ElemTy) / 8;
+      unsigned AllocaSize = DL->getTypeStoreSize(AllocaTy);
+      // Expand vector if required to match padding of inner type,
+      // i.e. odd size subvectors.
+      // Storage size of new vector must match that of alloca for correct
+      // behaviour of byte offsets and GEP computation.
+      if (NumElems * ElementSize != AllocaSize)
+        NumElems = AllocaSize / ElementSize;
+      if (NumElems > 0 && (AllocaSize % ElementSize) == 0)
+        VectorTy = FixedVectorType::get(ElemTy, NumElems);
+    }
   }
 
-  // FIXME: There is no reason why we can't support larger arrays, we
-  // are just being conservative for now.
-  // FIXME: We also reject alloca's of the form [ 2 x [ 2 x i32 ]] or
-  // equivalent. Potentially these could also be promoted but we don't currently
-  // handle this case
   if (!VectorTy) {
     LLVM_DEBUG(dbgs() << "  Cannot convert type to vector\n");
     return false;
   }
 
-  if (VectorTy->getNumElements() > 16 || VectorTy->getNumElements() < 2) {
+  const unsigned MaxElements =
+      (MaxVectorRegs * 32) / DL->getTypeSizeInBits(VectorTy->getElementType());
+
+  if (VectorTy->getNumElements() > MaxElements ||
+      VectorTy->getNumElements() < 2) {
     LLVM_DEBUG(dbgs() << "  " << *VectorTy
                       << " has an unsupported number of elements\n");
     return false;
@@ -763,11 +832,14 @@ bool AMDGPUPromoteAllocaImpl::tryPromoteAllocaToVector(AllocaInst &Alloca) {
   SmallVector<Instruction *> WorkList;
   SmallVector<Instruction *> UsersToRemove;
   SmallVector<Instruction *> DeferredInsts;
+  SmallVector<Instruction *> NewGEPInsts;
   DenseMap<MemTransferInst *, MemTransferInfo> TransferInfo;
 
   const auto RejectUser = [&](Instruction *Inst, Twine Msg) {
     LLVM_DEBUG(dbgs() << "  Cannot promote alloca to vector: " << Msg << "\n"
                       << "    " << *Inst << "\n");
+    for (auto *Inst : reverse(NewGEPInsts))
+      Inst->eraseFromParent();
     return false;
   };
 
@@ -817,7 +889,7 @@ bool AMDGPUPromoteAllocaImpl::tryPromoteAllocaToVector(AllocaInst &Alloca) {
     if (auto *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
       // If we can't compute a vector index from this GEP, then we can't
       // promote this alloca to vector.
-      Value *Index = GEPToVectorIndex(GEP, &Alloca, VecEltTy, *DL);
+      Value *Index = GEPToVectorIndex(GEP, &Alloca, VecEltTy, *DL, NewGEPInsts);
       if (!Index)
         return RejectUser(Inst, "cannot compute vector index for GEP");
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalize.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalize.cpp
index 8d3e7829e10e1..d5a83903e2b13 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalize.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalize.cpp
@@ -89,13 +89,12 @@ const RegBankLegalizeRules &getRules(const GCNSubtarget &ST,
   static SmallDenseMap<unsigned, std::unique_ptr<RegBankLegalizeRules>>
       CacheForRuleSet;
   std::lock_guard<std::mutex> Lock(GlobalMutex);
-  if (!CacheForRuleSet.contains(ST.getGeneration())) {
-    auto Rules = std::make_unique<RegBankLegalizeRules>(ST, MRI);
-    CacheForRuleSet[ST.getGeneration()] = std::move(Rules);
-  } else {
-    CacheForRuleSet[ST.getGeneration()]->refreshRefs(ST, MRI);
-  }
-  return *CacheForRuleSet[ST.getGeneration()];
+  auto [It, Inserted] = CacheForRuleSet.try_emplace(ST.getGeneration());
+  if (Inserted)
+    It->second = std::make_unique<RegBankLegalizeRules>(ST, MRI);
+  else
+    It->second->refreshRefs(ST, MRI);
+  return *It->second;
 }
 
 class AMDGPURegBankLegalizeCombiner {
@@ -312,6 +311,12 @@ bool AMDGPURegBankLegalize::runOnMachineFunction(MachineFunction &MF) {
     }
 
     // Opcodes that also support S1.
+    if (Opc == G_FREEZE &&
+        MRI.getType(MI->getOperand(0).getReg()) != LLT::scalar(1)) {
+      RBLHelper.applyMappingTrivial(*MI);
+      continue;
+    }
+
     if ((Opc == AMDGPU::G_CONSTANT || Opc == AMDGPU::G_FCONSTANT ||
          Opc == AMDGPU::G_IMPLICIT_DEF)) {
       Register Dst = MI->getOperand(0).getReg();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
index 3c007987b8494..3383175fc1bdb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
@@ -134,6 +134,26 @@ void RegBankLegalizeHelper::lower(MachineInstr &MI,
   switch (Mapping.LoweringMethod) {
   case DoNotLower:
     return;
+  case VccExtToSel: {
+    LLT Ty = MRI.getType(MI.getOperand(0).getReg());
+    Register Src = MI.getOperand(1).getReg();
+    unsigned Opc = MI.getOpcode();
+    if (Ty == S32 || Ty == S16) {
+      auto True = B.buildConstant({VgprRB, Ty}, Opc == G_SEXT ? -1 : 1);
+      auto False = B.buildConstant({VgprRB, Ty}, 0);
+      B.buildSelect(MI.getOperand(0).getReg(), Src, True, False);
+    }
+    if (Ty == S64) {
+      auto True = B.buildConstant({VgprRB, S32}, Opc == G_SEXT ? -1 : 1);
+      auto False = B.buildConstant({VgprRB, S32}, 0);
+      auto Sel = B.buildSelect({VgprRB, S32}, Src, True, False);
+      B.buildMergeValues(
+          MI.getOperand(0).getReg(),
+          {Sel.getReg(0), Opc == G_SEXT ? Sel.getReg(0) : False.getReg(0)});
+    }
+    MI.eraseFromParent();
+    return;
+  }
   case UniExtToSel: {
     LLT Ty = MRI.getType(MI.getOperand(0).getReg());
     auto True = B.buildConstant({SgprRB, Ty},
@@ -276,6 +296,8 @@ LLT RegBankLegalizeHelper::getTyFromID(RegBankLLTMappingApplyID ID) {
   case Sgpr64:
   case Vgpr64:
     return LLT::scalar(64);
+  case VgprP0:
+    return LLT::pointer(0, 64);
   case SgprP1:
   case VgprP1:
     return LLT::pointer(1, 64);
@@ -383,6 +405,7 @@ RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) {
     return SgprRB;
   case Vgpr32:
   case Vgpr64:
+  case VgprP0:
   case VgprP1:
   case VgprP3:
   case VgprP4:
@@ -425,6 +448,7 @@ void RegBankLegalizeHelper::applyMappingDst(
     case SgprV4S32:
     case Vgpr32:
     case Vgpr64:
+    case VgprP0:
     case VgprP1:
     case VgprP3:
     case VgprP4:
@@ -555,6 +579,7 @@ void RegBankLegalizeHelper::applyMappingSrc(
     // vgpr scalars, pointers and vectors
     case Vgpr32:
     case Vgpr64:
+    case VgprP0:
     case VgprP1:
     case VgprP3:
     case VgprP4:
@@ -653,7 +678,8 @@ void RegBankLegalizeHelper::applyMappingPHI(MachineInstr &MI) {
   // We accept all types that can fit in some register class.
   // Uniform G_PHIs have all sgpr registers.
   // Divergent G_PHIs have vgpr dst but inputs can be sgpr or vgpr.
-  if (Ty == LLT::scalar(32) || Ty == LLT::pointer(4, 64)) {
+  if (Ty == LLT::scalar(32) || Ty == LLT::pointer(1, 64) ||
+      Ty == LLT::pointer(4, 64)) {
     return;
   }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 33018ae9677a3..6ee15709d2fa6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -50,6 +50,8 @@ bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
     return MRI.getType(Reg) == LLT::scalar(32);
   case S64:
     return MRI.getType(Reg) == LLT::scalar(64);
+  case P0:
+    return MRI.getType(Reg) == LLT::pointer(0, 64);
   case P1:
     return MRI.getType(Reg) == LLT::pointer(1, 64);
   case P3:
@@ -58,6 +60,8 @@ bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
     return MRI.getType(Reg) == LLT::pointer(4, 64);
   case P5:
     return MRI.getType(Reg) == LLT::pointer(5, 32);
+  case V4S32:
+    return MRI.getType(Reg) == LLT::fixed_vector(4, 32);
   case B32:
     return MRI.getType(Reg).getSizeInBits() == 32;
   case B64:
@@ -78,6 +82,8 @@ bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
     return MRI.getType(Reg) == LLT::scalar(32) && MUI.isUniform(Reg);
   case UniS64:
     return MRI.getType(Reg) == LLT::scalar(64) && MUI.isUniform(Reg);
+  case UniP0:
+    return MRI.getType(Reg) == LLT::pointer(0, 64) && MUI.isUniform(Reg);
   case UniP1:
     return MRI.getType(Reg) == LLT::pointer(1, 64) && MUI.isUniform(Reg);
   case UniP3:
@@ -104,6 +110,8 @@ bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
     return MRI.getType(Reg) == LLT::scalar(32) && MUI.isDivergent(Reg);
   case DivS64:
     return MRI.getType(Reg) == LLT::scalar(64) && MUI.isDivergent(Reg);
+  case DivP0:
+    return MRI.getType(Reg) == LLT::pointer(0, 64) && MUI.isDivergent(Reg);
   case DivP1:
     return MRI.getType(Reg) == LLT::pointer(1, 64) && MUI.isDivergent(Reg);
   case DivP3:
@@ -433,16 +441,21 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
   addRulesForGOpcs({G_XOR, G_OR, G_AND}, StandardB)
       .Any({{UniS1}, {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32AExt}}})
       .Any({{DivS1}, {{Vcc}, {Vcc, Vcc}}})
+      .Div(B32, {{VgprB32}, {VgprB32, VgprB32}})
+      .Uni(B64, {{SgprB64}, {SgprB64, SgprB64}})
       .Div(B64, {{VgprB64}, {VgprB64, VgprB64}, SplitTo32});
 
   addRulesForGOpcs({G_SHL}, Standard)
+      .Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})
       .Uni(S64, {{Sgpr64}, {Sgpr64, Sgpr32}})
       .Div(S64, {{Vgpr64}, {Vgpr64, Vgpr32}});
 
   // Note: we only write S1 rules for G_IMPLICIT_DEF, G_CONSTANT, G_FCONSTANT
   // and G_FREEZE here, rest is trivially regbankselected earlier
+  addRulesForGOpcs({G_IMPLICIT_DEF}).Any({{UniS1}, {{Sgpr32Trunc}, {}}});
   addRulesForGOpcs({G_CONSTANT})
       .Any({{UniS1, _}, {{Sgpr32Trunc}, {None}, UniCstExt}});
+  addRulesForGOpcs({G_FREEZE}).Any({{DivS1}, {{Vcc}, {Vcc}}});
 
   addRulesForGOpcs({G_ICMP})
       .Any({{UniS1, _, S32}, {{Sgpr32Trunc}, {None, Sgpr32, Sgpr32}}})
@@ -473,6 +486,7 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
 
   addRulesForGOpcs({G_ZEXT, G_SEXT})
       .Any({{UniS32, S1}, {{Sgpr32}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
+      .Any({{DivS32, S1}, {{Vgpr32}, {Vcc}, VccExtToSel}})
       .Any({{UniS64, S32}, {{Sgpr64}, {Sgpr32}, Ext32To64}})
       .Any({{DivS64, S32}, {{Vgpr64}, {Vgpr32}, Ext32To64}});
 
@@ -527,9 +541,12 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
 
   // clang-format off
   addRulesForGOpcs({G_LOAD})
+      .Any({{DivB32, DivP0}, {{VgprB32}, {VgprP0}}})
+
       .Any({{DivB32, DivP1}, {{VgprB32}, {VgprP1}}})
       .Any({{{UniB256, UniP1}, isAlign4 && isUL}, {{SgprB256}, {SgprP1}}})
       .Any({{{UniB512, UniP1}, isAlign4 && isUL}, {{SgprB512}, {SgprP1}}})
+      .Any({{{UniB32, UniP1}, !isAlign4 || !isUL}, {{UniInVgprB32}, {SgprP1}}})
       .Any({{{UniB256, UniP1}, !isAlign4 || !isUL}, {{UniInVgprB256}, {VgprP1}, SplitLoad}})
       .Any({{{UniB512, UniP1}, !isAlign4 || !isUL}, {{UniInVgprB512}, {VgprP1}, SplitLoad}})
 
@@ -558,15 +575,26 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
   // clang-format on
 
   addRulesForGOpcs({G_AMDGPU_BUFFER_LOAD}, Vector)
+      .Div(S32, {{Vgpr32}, {SgprV4S32, Vgpr32, Vgpr32, Sgpr32}})
+      .Uni(S32, {{UniInVgprS32}, {SgprV4S32, Vgpr32, Vgpr32, Sgpr32}})
       .Div(V4S32, {{VgprV4S32}, {SgprV4S32, Vgpr32, Vgpr32, Sgpr32}})
       .Uni(V4S32, {{UniInVgprV4S32}, {SgprV4S32, Vgpr32, Vgpr32, Sgpr32}});
 
   addRulesForGOpcs({G_STORE})
+      .Any({{S32, P0}, {{}, {Vgpr32, VgprP0}}})
       .Any({{S32, P1}, {{}, {Vgpr32, VgprP1}}})
       .Any({{S64, P1}, {{}, {Vgpr64, VgprP1}}})
       .Any({{V4S32, P1}, {{}, {VgprV4S32, VgprP1}}});
 
-  addRulesForGOpcs({G_PTR_ADD}).Any({{DivP1}, {{VgprP1}, {VgprP1, Vgpr64}}});
+  addRulesForGOpcs({G_AMDGPU_BUFFER_STORE})
+      .Any({{S32}, {{}, {Vgpr32, SgprV4S32, Vgpr32, Vgpr32, Sgpr32}}});
+
+  addRulesForGOpcs({G_PTR_ADD})
+      .Any({{UniP1}, {{SgprP1}, {SgprP1, Sgpr64}}})
+      .Any({{DivP1}, {{VgprP1}, {VgprP1, Vgpr64}}})
+      .Any({{DivP0}, {{VgprP0}, {VgprP0, Vgpr64}}});
+
+  addRulesForGOpcs({G_INTTOPTR}).Any({{UniP4}, {{SgprP4}, {Sgpr64}}});
 
   addRulesForGOpcs({G_ABS}, Standard).Uni(S16, {{Sgpr32Trunc}, {Sgpr32SExt}});
 
@@ -582,15 +610,24 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
       .Any({{UniS32, S32}, {{UniInVgprS32}, {Vgpr32}}}, !hasSALUFloat);
 
   addRulesForGOpcs({G_UITOFP})
+      .Any({{DivS32, S32}, {{Vgpr32}, {Vgpr32}}})
       .Any({{UniS32, S32}, {{Sgpr32}, {Sgpr32}}}, hasSALUFloat)
       .Any({{UniS32, S32}, {{UniInVgprS32}, {Vgpr32}}}, !hasSALUFloat);
 
   using namespace Intrinsic;
 
+  addRulesForIOpcs({amdgcn_s_getpc}).Any({{UniS64, _}, {{Sgpr64}, {None}}});
+
   // This is "intrinsic lane mask" it was set to i32/i64 in llvm-ir.
   addRulesForIOpcs({amdgcn_end_cf}).Any({{_, S32}, {{}, {None, Sgpr32}}});
 
   addRulesForIOpcs({amdgcn_if_break}, Standard)
       .Uni(S32, {{Sgpr32}, {IntrId, Vcc, Sgpr32}});
 
+  addRulesForIOpcs({amdgcn_mbcnt_lo, amdgcn_mbcnt_hi}, Standard)
+      .Div(S32, {{}, {Vgpr32, None, Vgpr32, Vgpr32}});
+
+  addRulesForIOpcs({amdgcn_readfirstlane})
+      .Any({{UniS32, _, DivS32}, {{}, {Sgpr32, None, Vgpr32}}});
+
 } // end initialize rules
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
index 8280751e1dbdd..6bde7f2cd676d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
@@ -50,16 +50,19 @@ enum UniformityLLTOpPredicateID {
   DivS64,
 
   // pointers
+  P0,
   P1,
   P3,
   P4,
   P5,
 
+  UniP0,
   UniP1,
   UniP3,
   UniP4,
   UniP5,
 
+  DivP0,
   DivP1,
   DivP3,
   DivP4,
@@ -124,6 +127,7 @@ enum RegBankLLTMappingApplyID {
   // vgpr scalars, pointers, vectors and B-types
   Vgpr32,
   Vgpr64,
+  VgprP0,
   VgprP1,
   VgprP3,
   VgprP4,
@@ -162,6 +166,7 @@ enum RegBankLLTMappingApplyID {
 // vgpr. Lower it to two S32 vgpr ANDs.
 enum LoweringMethodID {
   DoNotLower,
+  VccExtToSel,
   UniExtToSel,
   VgprToVccCopy,
   SplitTo32,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.cpp
index abd7dcecc93ad..8a0c9faa34631 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankSelect.cpp
@@ -83,6 +83,7 @@ class RegBankSelectHelper {
   MachineRegisterInfo &MRI;
   AMDGPU::IntrinsicLaneMaskAnalyzer &ILMA;
   const MachineUniformityInfo &MUI;
+  const SIRegisterInfo &TRI;
   const RegisterBank *SgprRB;
   const RegisterBank *VgprRB;
   const RegisterBank *VccRB;
@@ -91,14 +92,29 @@ class RegBankSelectHelper {
   RegBankSelectHelper(MachineIRBuilder &B,
                       AMDGPU::IntrinsicLaneMaskAnalyzer &ILMA,
                       const MachineUniformityInfo &MUI,
-                      const RegisterBankInfo &RBI)
-      : B(B), MRI(*B.getMRI()), ILMA(ILMA), MUI(MUI),
+                      const SIRegisterInfo &TRI, const RegisterBankInfo &RBI)
+      : B(B), MRI(*B.getMRI()), ILMA(ILMA), MUI(MUI), TRI(TRI),
         SgprRB(&RBI.getRegBank(AMDGPU::SGPRRegBankID)),
         VgprRB(&RBI.getRegBank(AMDGPU::VGPRRegBankID)),
         VccRB(&RBI.getRegBank(AMDGPU::VCCRegBankID)) {}
 
+  // Temporal divergence copy: COPY to vgpr with implicit use of $exec inside of
+  // the cycle
+  // Note: uniformity analysis does not consider that registers with vgpr def
+  // are divergent (you can have uniform value in vgpr).
+  // - TODO: implicit use of $exec could be implemented as indicator that
+  //   instruction is divergent
+  bool isTemporalDivergenceCopy(Register Reg) {
+    MachineInstr *MI = MRI.getVRegDef(Reg);
+    if (!MI->isCopy() || MI->getNumImplicitOperands() != 1)
+      return false;
+
+    return MI->implicit_operands().begin()->getReg() == TRI.getExec();
+  }
+
   const RegisterBank *getRegBankToAssign(Register Reg) {
-    if (MUI.isUniform(Reg) || ILMA.isS32S64LaneMask(Reg))
+    if (!isTemporalDivergenceCopy(Reg) &&
+        (MUI.isUniform(Reg) || ILMA.isS32S64LaneMask(Reg)))
       return SgprRB;
     if (MRI.getType(Reg) == LLT::scalar(1))
       return VccRB;
@@ -209,7 +225,8 @@ bool AMDGPURegBankSelect::runOnMachineFunction(MachineFunction &MF) {
       getAnalysis<MachineUniformityAnalysisPass>().getUniformityInfo();
   MachineRegisterInfo &MRI = *B.getMRI();
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
-  RegBankSelectHelper RBSHelper(B, ILMA, MUI, *ST.getRegBankInfo());
+  RegBankSelectHelper RBSHelper(B, ILMA, MUI, *ST.getRegisterInfo(),
+                                *ST.getRegBankInfo());
   // Virtual registers at this point don't have register banks.
   // Virtual registers in def and use operands of already inst-selected
   // instruction have register class.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index efe92e0fecc12..c19ee14ab1574 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3460,7 +3460,7 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
     applyMappingMAD_64_32(B, OpdMapper);
     return;
   case AMDGPU::G_PREFETCH: {
-    if (!Subtarget.hasPrefetch()) {
+    if (!Subtarget.hasPrefetch() || !Subtarget.hasSafeSmemPrefetch()) {
       MI.eraseFromParent();
       return;
     }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUReserveWWMRegs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUReserveWWMRegs.cpp
index e0348e192977b..f255bfc128d6b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUReserveWWMRegs.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUReserveWWMRegs.cpp
@@ -14,6 +14,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "AMDGPUReserveWWMRegs.h"
 #include "AMDGPU.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 #include "SIMachineFunctionInfo.h"
@@ -27,12 +28,12 @@ using namespace llvm;
 
 namespace {
 
-class AMDGPUReserveWWMRegs : public MachineFunctionPass {
+class AMDGPUReserveWWMRegsLegacy : public MachineFunctionPass {
 public:
   static char ID;
 
-  AMDGPUReserveWWMRegs() : MachineFunctionPass(ID) {
-    initializeAMDGPUReserveWWMRegsPass(*PassRegistry::getPassRegistry());
+  AMDGPUReserveWWMRegsLegacy() : MachineFunctionPass(ID) {
+    initializeAMDGPUReserveWWMRegsLegacyPass(*PassRegistry::getPassRegistry());
   }
 
   bool runOnMachineFunction(MachineFunction &MF) override;
@@ -47,16 +48,34 @@ class AMDGPUReserveWWMRegs : public MachineFunctionPass {
   }
 };
 
+class AMDGPUReserveWWMRegs {
+public:
+  bool run(MachineFunction &MF);
+};
+
 } // End anonymous namespace.
 
-INITIALIZE_PASS(AMDGPUReserveWWMRegs, DEBUG_TYPE,
+INITIALIZE_PASS(AMDGPUReserveWWMRegsLegacy, DEBUG_TYPE,
                 "AMDGPU Reserve WWM Registers", false, false)
 
-char AMDGPUReserveWWMRegs::ID = 0;
+char AMDGPUReserveWWMRegsLegacy::ID = 0;
+
+char &llvm::AMDGPUReserveWWMRegsLegacyID = AMDGPUReserveWWMRegsLegacy::ID;
 
-char &llvm::AMDGPUReserveWWMRegsID = AMDGPUReserveWWMRegs::ID;
+bool AMDGPUReserveWWMRegsLegacy::runOnMachineFunction(MachineFunction &MF) {
+  return AMDGPUReserveWWMRegs().run(MF);
+}
+
+PreservedAnalyses
+AMDGPUReserveWWMRegsPass::run(MachineFunction &MF,
+                              MachineFunctionAnalysisManager &) {
+  AMDGPUReserveWWMRegs().run(MF);
+  // TODO: This should abandon RegisterClassInfo once it is turned into an
+  // analysis.
+  return PreservedAnalyses::all();
+}
 
-bool AMDGPUReserveWWMRegs::runOnMachineFunction(MachineFunction &MF) {
+bool AMDGPUReserveWWMRegs::run(MachineFunction &MF) {
   SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
 
   bool Changed = false;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUReserveWWMRegs.h b/llvm/lib/Target/AMDGPU/AMDGPUReserveWWMRegs.h
new file mode 100644
index 0000000000000..7c58590347f3e
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUReserveWWMRegs.h
@@ -0,0 +1,24 @@
+//===- AMDGPUReserveWWMRegs.h -----------------------------------*- C++- *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPURESERVEWWMREGS_H
+#define LLVM_LIB_TARGET_AMDGPU_AMDGPURESERVEWWMREGS_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+class AMDGPUReserveWWMRegsPass
+    : public PassInfoMixin<AMDGPUReserveWWMRegsPass> {
+public:
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+  static bool isRequired() { return true; }
+};
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPURESERVEWWMREGS_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp
index 65aee2f70104b..65cbb13628301 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp
@@ -1545,18 +1545,16 @@ PreservedAnalyses AMDGPUSplitModulePass::run(Module &M,
                       << "'\n");
 
     while (true) {
-      llvm::LockFileManager Locked(LockFilePath.str());
-      switch (Locked) {
-      case LockFileManager::LFS_Error:
+      llvm::LockFileManager Lock(LockFilePath.str());
+      bool Owned;
+      if (Error Err = Lock.tryLock().moveInto(Owned)) {
+        consumeError(std::move(Err));
         LLVM_DEBUG(
             dbgs() << "[amdgpu-split-module] unable to acquire lockfile, debug "
                       "output may be mangled by other processes\n");
-        Locked.unsafeRemoveLockFile();
-        break;
-      case LockFileManager::LFS_Owned:
-        break;
-      case LockFileManager::LFS_Shared: {
-        switch (Locked.waitForUnlock()) {
+        Lock.unsafeRemoveLockFile();
+      } else if (!Owned) {
+        switch (Lock.waitForUnlock()) {
         case LockFileManager::Res_Success:
           break;
         case LockFileManager::Res_OwnerDied:
@@ -1566,11 +1564,9 @@ PreservedAnalyses AMDGPUSplitModulePass::run(Module &M,
               dbgs()
               << "[amdgpu-split-module] unable to acquire lockfile, debug "
                  "output may be mangled by other processes\n");
-          Locked.unsafeRemoveLockFile();
+          Lock.unsafeRemoveLockFile();
           break; // give up
         }
-        break;
-      }
       }
 
       splitAMDGPUModule(TTIGetter, M, N, ModuleCallback);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index ce3dcd920bce3..d5d375e473d04 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -19,12 +19,13 @@
 #include "AMDGPUAliasAnalysis.h"
 #include "AMDGPUCtorDtorLowering.h"
 #include "AMDGPUExportClustering.h"
+#include "AMDGPUExportKernelRuntimeHandles.h"
 #include "AMDGPUIGroupLP.h"
 #include "AMDGPUISelDAGToDAG.h"
 #include "AMDGPUMacroFusion.h"
-#include "AMDGPUOpenCLEnqueuedBlockLowering.h"
 #include "AMDGPUPerfHintAnalysis.h"
 #include "AMDGPURemoveIncompatibleFunctions.h"
+#include "AMDGPUReserveWWMRegs.h"
 #include "AMDGPUSplitModule.h"
 #include "AMDGPUTargetObjectFile.h"
 #include "AMDGPUTargetTransformInfo.h"
@@ -517,7 +518,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeAMDGPULowerKernelArgumentsPass(*PR);
   initializeAMDGPUPromoteKernelArgumentsPass(*PR);
   initializeAMDGPULowerKernelAttributesPass(*PR);
-  initializeAMDGPUOpenCLEnqueuedBlockLoweringLegacyPass(*PR);
+  initializeAMDGPUExportKernelRuntimeHandlesLegacyPass(*PR);
   initializeAMDGPUPostLegalizerCombinerPass(*PR);
   initializeAMDGPUPreLegalizerCombinerPass(*PR);
   initializeAMDGPURegBankCombinerPass(*PR);
@@ -528,7 +529,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeAMDGPURemoveIncompatibleFunctionsLegacyPass(*PR);
   initializeAMDGPULowerModuleLDSLegacyPass(*PR);
   initializeAMDGPULowerBufferFatPointersPass(*PR);
-  initializeAMDGPUReserveWWMRegsPass(*PR);
+  initializeAMDGPUReserveWWMRegsLegacyPass(*PR);
   initializeAMDGPURewriteOutArgumentsPass(*PR);
   initializeAMDGPURewriteUndefForPHILegacyPass(*PR);
   initializeAMDGPUUnifyMetadataPass(*PR);
@@ -541,7 +542,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeSILowerControlFlowLegacyPass(*PR);
   initializeSIPreEmitPeepholePass(*PR);
   initializeSILateBranchLoweringPass(*PR);
-  initializeSIMemoryLegalizerPass(*PR);
+  initializeSIMemoryLegalizerLegacyPass(*PR);
   initializeSIOptimizeExecMaskingLegacyPass(*PR);
   initializeSIPreAllocateWWMRegsLegacyPass(*PR);
   initializeSIFormMemoryClausesLegacyPass(*PR);
@@ -1225,8 +1226,8 @@ void AMDGPUPassConfig::addIRPasses() {
   if (Arch == Triple::r600)
     addPass(createR600OpenCLImageTypeLoweringPass());
 
-  // Replace OpenCL enqueued block function pointers with global variables.
-  addPass(createAMDGPUOpenCLEnqueuedBlockLoweringLegacyPass());
+  // Make enqueued block runtime handles externally visible.
+  addPass(createAMDGPUExportKernelRuntimeHandlesLegacyPass());
 
   // Lower LDS accesses to global memory pass if address sanitizer is enabled.
   if (EnableSwLowerLDS)
@@ -1382,7 +1383,11 @@ bool GCNPassConfig::addPreISel() {
   // control flow modifications.
   addPass(createAMDGPURewriteUndefForPHILegacyPass());
 
-  addPass(createLCSSAPass());
+  // SDAG requires LCSSA, GlobalISel does not. Disable LCSSA for -global-isel
+  // with -new-reg-bank-select and without any of the fallback options.
+  if (!getCGPassBuilderOption().EnableGlobalISelOption ||
+      !isGlobalISelAbortEnabled() || !NewRegBankSelect)
+    addPass(createLCSSAPass());
 
   if (TM->getOptLevel() > CodeGenOptLevel::Less)
     addPass(&AMDGPUPerfHintAnalysisLegacyID);
@@ -1599,7 +1604,7 @@ bool GCNPassConfig::addRegAssignAndRewriteFast() {
   addPass(createWWMRegAllocPass(false));
 
   addPass(&SILowerWWMCopiesLegacyID);
-  addPass(&AMDGPUReserveWWMRegsID);
+  addPass(&AMDGPUReserveWWMRegsLegacyID);
 
   // For allocating per-thread VGPRs.
   addPass(createVGPRAllocPass(false));
@@ -1636,7 +1641,7 @@ bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
   addPass(createWWMRegAllocPass(true));
   addPass(&SILowerWWMCopiesLegacyID);
   addPass(createVirtRegRewriter(false));
-  addPass(&AMDGPUReserveWWMRegsID);
+  addPass(&AMDGPUReserveWWMRegsLegacyID);
 
   // For allocating per-thread VGPRs.
   addPass(createVGPRAllocPass(true));
@@ -1972,7 +1977,7 @@ void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {
   addPass(AMDGPUAlwaysInlinePass());
   addPass(AlwaysInlinerPass());
 
-  addPass(AMDGPUOpenCLEnqueuedBlockLoweringPass());
+  addPass(AMDGPUExportKernelRuntimeHandlesPass());
 
   if (EnableSwLowerLDS)
     addPass(AMDGPUSwLowerLDSPass(TM));
@@ -2086,7 +2091,9 @@ void AMDGPUCodeGenPassBuilder::addPreISel(AddIRPass &addPass) const {
   // control flow modifications.
   addPass(AMDGPURewriteUndefForPHIPass());
 
-  addPass(LCSSAPass());
+  if (!getCGPassBuilderOption().EnableGlobalISelOption ||
+      !isGlobalISelAbortEnabled() || !NewRegBankSelect)
+    addPass(LCSSAPass());
 
   if (TM.getOptLevel() > CodeGenOptLevel::Less)
     addPass(AMDGPUPerfHintAnalysisPass(TM));
@@ -2151,7 +2158,8 @@ void AMDGPUCodeGenPassBuilder::addPreEmitPass(AddMachinePass &addPass) const {
   if (isPassEnabled(EnableVOPD, CodeGenOptLevel::Less)) {
     // TODO: addPass(GCNCreateVOPDPass());
   }
-  // TODO: addPass(SIMemoryLegalizerPass());
+
+  addPass(SIMemoryLegalizerPass());
   // TODO: addPass(SIInsertWaitcntsPass());
 
   // TODO: addPass(SIModeRegisterPass());
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
index 4df55eac5d76b..bfdd8cf1bc2b1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp
@@ -164,6 +164,46 @@ class AMDGPUWaitSGPRHazards {
       BuildMI(MBB, MI, MI->getDebugLoc(), TII->get(AMDGPU::DS_NOP));
   }
 
+  unsigned mergeMasks(unsigned Mask1, unsigned Mask2) {
+    unsigned Mask = 0xffff;
+    Mask = AMDGPU::DepCtr::encodeFieldSaSdst(
+        Mask, std::min(AMDGPU::DepCtr::decodeFieldSaSdst(Mask1),
+                       AMDGPU::DepCtr::decodeFieldSaSdst(Mask2)));
+    Mask = AMDGPU::DepCtr::encodeFieldVaVcc(
+        Mask, std::min(AMDGPU::DepCtr::decodeFieldVaVcc(Mask1),
+                       AMDGPU::DepCtr::decodeFieldVaVcc(Mask2)));
+    Mask = AMDGPU::DepCtr::encodeFieldVmVsrc(
+        Mask, std::min(AMDGPU::DepCtr::decodeFieldVmVsrc(Mask1),
+                       AMDGPU::DepCtr::decodeFieldVmVsrc(Mask2)));
+    Mask = AMDGPU::DepCtr::encodeFieldVaSdst(
+        Mask, std::min(AMDGPU::DepCtr::decodeFieldVaSdst(Mask1),
+                       AMDGPU::DepCtr::decodeFieldVaSdst(Mask2)));
+    Mask = AMDGPU::DepCtr::encodeFieldVaVdst(
+        Mask, std::min(AMDGPU::DepCtr::decodeFieldVaVdst(Mask1),
+                       AMDGPU::DepCtr::decodeFieldVaVdst(Mask2)));
+    Mask = AMDGPU::DepCtr::encodeFieldHoldCnt(
+        Mask, std::min(AMDGPU::DepCtr::decodeFieldHoldCnt(Mask1),
+                       AMDGPU::DepCtr::decodeFieldHoldCnt(Mask2)));
+    Mask = AMDGPU::DepCtr::encodeFieldVaSsrc(
+        Mask, std::min(AMDGPU::DepCtr::decodeFieldVaSsrc(Mask1),
+                       AMDGPU::DepCtr::decodeFieldVaSsrc(Mask2)));
+    return Mask;
+  }
+
+  bool mergeConsecutiveWaitAlus(MachineBasicBlock::instr_iterator &MI,
+                                unsigned Mask) {
+    auto MBB = MI->getParent();
+    if (MI == MBB->instr_begin())
+      return false;
+
+    auto It = prev_nodbg(MI, MBB->instr_begin());
+    if (It->getOpcode() != AMDGPU::S_WAITCNT_DEPCTR)
+      return false;
+
+    It->getOperand(0).setImm(mergeMasks(Mask, It->getOperand(0).getImm()));
+    return true;
+  }
+
   bool runOnMachineBasicBlock(MachineBasicBlock &MBB, bool Emit) {
     enum { WA_VALU = 0x1, WA_SALU = 0x2, WA_VCC = 0x4 };
 
@@ -362,10 +402,12 @@ class AMDGPUWaitSGPRHazards {
           Mask = AMDGPU::DepCtr::encodeFieldVaSdst(Mask, 0);
         }
         if (Emit) {
-          auto NewMI = BuildMI(MBB, MI, MI->getDebugLoc(),
-                               TII->get(AMDGPU::S_WAITCNT_DEPCTR))
-                           .addImm(Mask);
-          updateGetPCBundle(NewMI);
+          if (!mergeConsecutiveWaitAlus(MI, Mask)) {
+            auto NewMI = BuildMI(MBB, MI, MI->getDebugLoc(),
+                                 TII->get(AMDGPU::S_WAITCNT_DEPCTR))
+                             .addImm(Mask);
+            updateGetPCBundle(NewMI);
+          }
           Emitted = true;
         }
       }
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 408da0536237e..09a3096602fc3 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -55,6 +55,7 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPUCombinerHelper.cpp
   AMDGPUCtorDtorLowering.cpp
   AMDGPUExportClustering.cpp
+  AMDGPUExportKernelRuntimeHandles.cpp
   AMDGPUFrameLowering.cpp
   AMDGPUGlobalISelDivergenceLowering.cpp
   AMDGPUGlobalISelUtils.cpp
@@ -84,7 +85,6 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPUMCResourceInfo.cpp
   AMDGPUMarkLastScratchLoad.cpp
   AMDGPUMIRFormatter.cpp
-  AMDGPUOpenCLEnqueuedBlockLowering.cpp
   AMDGPUPerfHintAnalysis.cpp
   AMDGPUPostLegalizerCombiner.cpp
   AMDGPUPreLegalizerCombiner.cpp
diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
index afed8b999d8eb..728ce125eba2d 100644
--- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
+++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
@@ -176,9 +176,12 @@ static DecodeStatus decodeSrcOp(MCInst &Inst, unsigned EncSize,
 
 // Decoder for registers. Imm(7-bit) is number of register, uses decodeSrcOp to
 // get register class. Used by SGPR only operands.
-#define DECODE_OPERAND_REG_7(RegClass, OpWidth)                                \
+#define DECODE_OPERAND_SREG_7(RegClass, OpWidth)                               \
   DECODE_SrcOp(Decode##RegClass##RegisterClass, 7, OpWidth, Imm, false, 0)
 
+#define DECODE_OPERAND_SREG_8(RegClass, OpWidth)                               \
+  DECODE_SrcOp(Decode##RegClass##RegisterClass, 8, OpWidth, Imm, false, 0)
+
 // Decoder for registers. Imm(10-bit): Imm{7-0} is number of register,
 // Imm{9} is acc(agpr or vgpr) Imm{8} should be 0 (see VOP3Pe_SMFMAC).
 // Set Imm{8} to 1 (IS_VGPR) to decode using 'enum10' from decodeSrcOp.
@@ -270,20 +273,21 @@ DECODE_OPERAND_REG_8(VReg_384)
 DECODE_OPERAND_REG_8(VReg_512)
 DECODE_OPERAND_REG_8(VReg_1024)
 
-DECODE_OPERAND_REG_7(SReg_32, OPW32)
-DECODE_OPERAND_REG_7(SReg_32_XM0, OPW32)
-DECODE_OPERAND_REG_7(SReg_32_XEXEC, OPW32)
-DECODE_OPERAND_REG_7(SReg_32_XM0_XEXEC, OPW32)
-DECODE_OPERAND_REG_7(SReg_32_XEXEC_HI, OPW32)
-DECODE_OPERAND_REG_7(SReg_64, OPW64)
-DECODE_OPERAND_REG_7(SReg_64_XEXEC, OPW64)
-DECODE_OPERAND_REG_7(SReg_64_XEXEC_XNULL, OPW64)
-DECODE_OPERAND_REG_7(SReg_96, OPW96)
-DECODE_OPERAND_REG_7(SReg_128, OPW128)
-DECODE_OPERAND_REG_7(SReg_128_XNULL, OPW128)
-DECODE_OPERAND_REG_7(SReg_256, OPW256)
-DECODE_OPERAND_REG_7(SReg_256_XNULL, OPW256)
-DECODE_OPERAND_REG_7(SReg_512, OPW512)
+DECODE_OPERAND_SREG_7(SReg_32, OPW32)
+DECODE_OPERAND_SREG_7(SReg_32_XM0, OPW32)
+DECODE_OPERAND_SREG_7(SReg_32_XEXEC, OPW32)
+DECODE_OPERAND_SREG_7(SReg_32_XM0_XEXEC, OPW32)
+DECODE_OPERAND_SREG_7(SReg_32_XEXEC_HI, OPW32)
+DECODE_OPERAND_SREG_7(SReg_64_XEXEC, OPW64)
+DECODE_OPERAND_SREG_7(SReg_64_XEXEC_XNULL, OPW64)
+DECODE_OPERAND_SREG_7(SReg_96, OPW96)
+DECODE_OPERAND_SREG_7(SReg_128, OPW128)
+DECODE_OPERAND_SREG_7(SReg_128_XNULL, OPW128)
+DECODE_OPERAND_SREG_7(SReg_256, OPW256)
+DECODE_OPERAND_SREG_7(SReg_256_XNULL, OPW256)
+DECODE_OPERAND_SREG_7(SReg_512, OPW512)
+
+DECODE_OPERAND_SREG_8(SReg_64, OPW64)
 
 DECODE_OPERAND_REG_8(AGPR_32)
 DECODE_OPERAND_REG_8(AReg_64)
diff --git a/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp b/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp
index d40a1a2a10d9b..798279b279da3 100644
--- a/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp
@@ -38,15 +38,15 @@ namespace {
 
 class GCNCreateVOPD : public MachineFunctionPass {
 private:
-    class VOPDCombineInfo {
-    public:
-      VOPDCombineInfo() = default;
-      VOPDCombineInfo(MachineInstr *First, MachineInstr *Second)
-          : FirstMI(First), SecondMI(Second) {}
-
-      MachineInstr *FirstMI;
-      MachineInstr *SecondMI;
-    };
+  class VOPDCombineInfo {
+  public:
+    VOPDCombineInfo() = default;
+    VOPDCombineInfo(MachineInstr *First, MachineInstr *Second)
+        : FirstMI(First), SecondMI(Second) {}
+
+    MachineInstr *FirstMI;
+    MachineInstr *SecondMI;
+  };
 
 public:
   static char ID;
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 6664a70572ded..f7f03fe5911bd 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -78,6 +78,7 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   bool BackOffBarrier = false;
   bool UnalignedScratchAccess = false;
   bool UnalignedAccessMode = false;
+  bool RelaxedBufferOOBMode = false;
   bool HasApertureRegs = false;
   bool SupportsXNACK = false;
   bool KernargPreload = false;
@@ -232,6 +233,7 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   bool HasVMEMtoScalarWriteHazard = false;
   bool HasSMEMtoVectorWriteHazard = false;
   bool HasInstFwdPrefetchBug = false;
+  bool HasSafeSmemPrefetch = false;
   bool HasVcmpxExecWARHazard = false;
   bool HasLdsBranchVmemWARHazard = false;
   bool HasNSAtoVMEMBug = false;
@@ -607,6 +609,8 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
     return UnalignedAccessMode;
   }
 
+  bool hasRelaxedBufferOOBMode() const { return RelaxedBufferOOBMode; }
+
   bool hasApertureRegs() const {
     return HasApertureRegs;
   }
@@ -960,6 +964,8 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
 
   bool hasPrefetch() const { return GFX12Insts; }
 
+  bool hasSafeSmemPrefetch() const { return HasSafeSmemPrefetch; }
+
   // Has s_cmpk_* instructions.
   bool hasSCmpK() const { return getGeneration() < GFX12; }
 
diff --git a/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp b/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
index df0c2080e0795..3be7e5a16c825 100644
--- a/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
@@ -48,7 +48,7 @@ class SIAnnotateControlFlow {
 
   ConstantInt *BoolTrue;
   ConstantInt *BoolFalse;
-  UndefValue *BoolUndef;
+  PoisonValue *BoolPoison;
   Constant *IntMaskZero;
 
   Function *If = nullptr;
@@ -120,7 +120,7 @@ void SIAnnotateControlFlow::initialize(const GCNSubtarget &ST) {
 
   BoolTrue = ConstantInt::getTrue(Context);
   BoolFalse = ConstantInt::getFalse(Context);
-  BoolUndef = PoisonValue::get(Boolean);
+  BoolPoison = PoisonValue::get(Boolean);
   IntMaskZero = ConstantInt::get(IntMask, 0);
 }
 
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index fe095414e5172..9743320601ed4 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -582,7 +582,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
 
     setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, MVT::i16, Custom);
     setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i16, Custom);
-    setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i16, Custom);
+    setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i1, Custom);
 
     setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i32, Custom);
 
@@ -855,7 +855,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
   if (Subtarget->hasMad64_32())
     setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, MVT::i32, Custom);
 
-  if (Subtarget->hasPrefetch())
+  if (Subtarget->hasPrefetch() && Subtarget->hasSafeSmemPrefetch())
     setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
 
   if (Subtarget->hasIEEEMinMax()) {
@@ -1883,6 +1883,20 @@ bool SITargetLowering::allowsMisalignedMemoryAccessesImpl(
            Subtarget->hasUnalignedBufferAccessEnabled();
   }
 
+  // Ensure robust out-of-bounds guarantees for buffer accesses are met if
+  // RelaxedBufferOOBMode is disabled. Normally hardware will ensure proper
+  // out-of-bounds behavior, but in the edge case where an access starts
+  // out-of-bounds and then enter in-bounds, the entire access would be treated
+  // as out-of-bounds. Prevent misaligned memory accesses by requiring the
+  // natural alignment of buffer accesses.
+  if (AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER ||
+      AddrSpace == AMDGPUAS::BUFFER_RESOURCE ||
+      AddrSpace == AMDGPUAS::BUFFER_STRIDED_POINTER) {
+    if (!Subtarget->hasRelaxedBufferOOBMode() &&
+        Alignment < Align(PowerOf2Ceil(divideCeil(Size, 8))))
+      return false;
+  }
+
   // Smaller than dword value must be aligned.
   if (Size < 32)
     return false;
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 13e01b53639d2..239f2664f59f3 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -130,10 +130,10 @@ enum WaitEventType {
 // We reserve a fixed number of VGPR slots in the scoring tables for
 // special tokens like SCMEM_LDS (needed for buffer load to LDS).
 enum RegisterMapping {
-  SQ_MAX_PGM_VGPRS = 512, // Maximum programmable VGPRs across all targets.
-  AGPR_OFFSET = 256,      // Maximum programmable ArchVGPRs across all targets.
-  SQ_MAX_PGM_SGPRS = 128, // Maximum programmable SGPRs across all targets.
-  NUM_EXTRA_VGPRS = 9,    // Reserved slots for DS.
+  SQ_MAX_PGM_VGPRS = 1024, // Maximum programmable VGPRs across all targets.
+  AGPR_OFFSET = 512,       // Maximum programmable ArchVGPRs across all targets.
+  SQ_MAX_PGM_SGPRS = 128,  // Maximum programmable SGPRs across all targets.
+  NUM_EXTRA_VGPRS = 9,     // Reserved slots for DS.
   // Artificial register slots to track LDS writes into specific LDS locations
   // if a location is known. When slots are exhausted or location is
   // unknown use the first slot. The first slot is also always updated in
@@ -748,27 +748,32 @@ RegInterval WaitcntBrackets::getRegInterval(const MachineInstr *MI,
 
   RegInterval Result;
 
-  unsigned Reg = TRI->getEncodingValue(AMDGPU::getMCReg(Op.getReg(), *ST)) &
-                 AMDGPU::HWEncoding::REG_IDX_MASK;
+  MCRegister MCReg = AMDGPU::getMCReg(Op.getReg(), *ST);
+  unsigned RegIdx = TRI->getHWRegIndex(MCReg);
+  assert(isUInt<8>(RegIdx));
 
+  const TargetRegisterClass *RC = TRI->getPhysRegBaseClass(Op.getReg());
+  unsigned Size = TRI->getRegSizeInBits(*RC);
+
+  // AGPRs/VGPRs are tracked every 16 bits, SGPRs by 32 bits
   if (TRI->isVectorRegister(*MRI, Op.getReg())) {
-    assert(Reg <= SQ_MAX_PGM_VGPRS);
+    unsigned Reg = RegIdx << 1 | (AMDGPU::isHi16Reg(MCReg, *TRI) ? 1 : 0);
+    assert(Reg < AGPR_OFFSET);
     Result.first = Reg;
     if (TRI->isAGPR(*MRI, Op.getReg()))
       Result.first += AGPR_OFFSET;
     assert(Result.first >= 0 && Result.first < SQ_MAX_PGM_VGPRS);
-  } else if (TRI->isSGPRReg(*MRI, Op.getReg()) && Reg < SQ_MAX_PGM_SGPRS) {
+    assert(Size % 16 == 0);
+    Result.second = Result.first + (Size / 16);
+  } else if (TRI->isSGPRReg(*MRI, Op.getReg()) && RegIdx < SQ_MAX_PGM_SGPRS) {
     // SGPRs including VCC, TTMPs and EXEC but excluding read-only scalar
     // sources like SRC_PRIVATE_BASE.
-    Result.first = Reg + NUM_ALL_VGPRS;
+    Result.first = RegIdx + NUM_ALL_VGPRS;
+    Result.second = Result.first + divideCeil(Size, 32);
   } else {
     return {-1, -1};
   }
 
-  const TargetRegisterClass *RC = TRI->getPhysRegBaseClass(Op.getReg());
-  unsigned Size = TRI->getRegSizeInBits(*RC);
-  Result.second = Result.first + ((Size + 16) / 32);
-
   return Result;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index e1fb4c9db0a84..1e025f481ffa9 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -1337,6 +1337,7 @@ bool SIInstrInfo::getConstValDefinedInReg(const MachineInstr &MI,
   case AMDGPU::S_MOV_B64:
   case AMDGPU::V_MOV_B64_e32:
   case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
+  case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
   case AMDGPU::S_MOV_B64_IMM_PSEUDO:
   case AMDGPU::V_MOV_B64_PSEUDO: {
     const MachineOperand &Src0 = MI.getOperand(1);
@@ -2186,7 +2187,13 @@ bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
     MI.getMF()->getRegInfo().constrainRegClass(MI.getOperand(0).getReg(),
                                                &AMDGPU::SReg_32_XM0RegClass);
     break;
-
+  case AMDGPU::AV_MOV_B32_IMM_PSEUDO: {
+    Register Dst = MI.getOperand(0).getReg();
+    bool IsAGPR = SIRegisterInfo::isAGPRClass(RI.getPhysRegBaseClass(Dst));
+    MI.setDesc(
+        get(IsAGPR ? AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::V_MOV_B32_e32));
+    break;
+  }
   case AMDGPU::V_MOV_B64_PSEUDO: {
     Register Dst = MI.getOperand(0).getReg();
     Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
@@ -3423,6 +3430,7 @@ bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) {
   case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
   case AMDGPU::V_ACCVGPR_READ_B32_e64:
   case AMDGPU::V_ACCVGPR_MOV_B32:
+  case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
     return true;
   default:
     return false;
@@ -6059,7 +6067,7 @@ bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
       if (i == OpIdx)
         continue;
       const MachineOperand &Op = MI.getOperand(i);
-      if (!Op.isReg() && !Op.isFI() &&
+      if (!Op.isReg() && !Op.isFI() && !Op.isRegMask() &&
           !isInlineConstant(Op, InstDesc.operands()[i]) &&
           !Op.isIdenticalTo(*MO))
         return false;
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index cd5d3e3d31ab4..985337da1909d 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -140,6 +140,25 @@ def V_MOV_B64_PSEUDO : VPseudoInstSI <(outs VReg_64:$vdst),
   let UseNamedOperandTable = 1;
 }
 
+// 32-bit materialize immediate which supports AGPR or VGPR. Typically
+// this should just expand to V_MOV_B32, unless $vdst happens to be
+// allocated to an AGPR in which case it will lower to
+// V_ACCVGPR_WRITE_B32. This should always use an inline immediate
+// operand, as v_accvgpr_write_b32 does not support literal constants.
+def AV_MOV_B32_IMM_PSEUDO
+    : VPseudoInstSI<(outs AV_32:$vdst), (ins VCSrc_b32:$src0)> {
+  let isReMaterializable = 1;
+  let isAsCheapAsAMove = 1;
+
+  // Imprecise, technically if AGPR it's VOP3 and VOP1 for AGPR. But
+  // this tricks the rematerialize logic into working for it.
+  let VOP3 = 1;
+  let isMoveImm = 1;
+  let SchedRW = [Write32Bit];
+  let Size = 4;
+  let UseNamedOperandTable = 1;
+}
+
 // 64-bit vector move with dpp. Expanded post-RA.
 def V_MOV_B64_DPP_PSEUDO : VOP_DPP_Pseudo <"v_mov_b64_dpp", VOP_I64_I64> {
   let Size = 16; // Requires two 8-byte v_mov_b32_dpp to complete.
diff --git a/llvm/lib/Target/AMDGPU/SILowerI1Copies.h b/llvm/lib/Target/AMDGPU/SILowerI1Copies.h
index a407e3e014edd..fd90328c2b926 100644
--- a/llvm/lib/Target/AMDGPU/SILowerI1Copies.h
+++ b/llvm/lib/Target/AMDGPU/SILowerI1Copies.h
@@ -15,6 +15,7 @@
 #include "GCNSubtarget.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachinePostDominators.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/MachineSSAUpdater.h"
 
 namespace llvm {
@@ -72,6 +73,11 @@ class PhiLoweringHelper {
     LaneMaskRegAttrs = MRI->getVRegAttrs(LaneMask);
   }
 
+  void
+  initializeLaneMaskRegisterAttributes(MachineRegisterInfo::VRegAttrs Attrs) {
+    LaneMaskRegAttrs = Attrs;
+  }
+
   bool isLaneMaskReg(Register Reg) const {
     return TII->getRegisterInfo().isSGPRReg(*MRI, Reg) &&
            TII->getRegisterInfo().getRegSizeInBits(Reg, *MRI) ==
diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index 34953f9c08db7..56fec409d11ae 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -21,8 +21,10 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachinePassManager.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/MemoryModelRelaxationAnnotations.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/Support/AtomicOrdering.h"
 #include "llvm/TargetParser/TargetParser.h"
 
@@ -625,9 +627,9 @@ class SIGfx12CacheControl : public SIGfx11CacheControl {
   }
 };
 
-class SIMemoryLegalizer final : public MachineFunctionPass {
+class SIMemoryLegalizer final {
 private:
-
+  const MachineModuleInfo &MMI;
   /// Cache Control.
   std::unique_ptr<SICacheControl> CC = nullptr;
 
@@ -661,10 +663,16 @@ class SIMemoryLegalizer final : public MachineFunctionPass {
   bool expandAtomicCmpxchgOrRmw(const SIMemOpInfo &MOI,
                                 MachineBasicBlock::iterator &MI);
 
+public:
+  SIMemoryLegalizer(const MachineModuleInfo &MMI) : MMI(MMI) {};
+  bool run(MachineFunction &MF);
+};
+
+class SIMemoryLegalizerLegacy final : public MachineFunctionPass {
 public:
   static char ID;
 
-  SIMemoryLegalizer() : MachineFunctionPass(ID) {}
+  SIMemoryLegalizerLegacy() : MachineFunctionPass(ID) {}
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.setPreservesCFG();
@@ -2767,11 +2775,26 @@ bool SIMemoryLegalizer::expandAtomicCmpxchgOrRmw(const SIMemOpInfo &MOI,
   return Changed;
 }
 
-bool SIMemoryLegalizer::runOnMachineFunction(MachineFunction &MF) {
-  bool Changed = false;
-
+bool SIMemoryLegalizerLegacy::runOnMachineFunction(MachineFunction &MF) {
   const MachineModuleInfo &MMI =
       getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
+  return SIMemoryLegalizer(MMI).run(MF);
+}
+
+PreservedAnalyses
+SIMemoryLegalizerPass::run(MachineFunction &MF,
+                           MachineFunctionAnalysisManager &MFAM) {
+  auto *MMI = MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(MF)
+                  .getCachedResult<MachineModuleAnalysis>(
+                      *MF.getFunction().getParent());
+  assert(MMI && "MachineModuleAnalysis must be available");
+  if (!SIMemoryLegalizer(MMI->getMMI()).run(MF))
+    return PreservedAnalyses::all();
+  return getMachineFunctionPassPreservedAnalyses().preserveSet<CFGAnalyses>();
+}
+
+bool SIMemoryLegalizer::run(MachineFunction &MF) {
+  bool Changed = false;
 
   SIMemOpAccess MOA(MMI.getObjFileInfo<AMDGPUMachineModuleInfo>());
   CC = SICacheControl::create(MF.getSubtarget<GCNSubtarget>());
@@ -2812,11 +2835,11 @@ bool SIMemoryLegalizer::runOnMachineFunction(MachineFunction &MF) {
   return Changed;
 }
 
-INITIALIZE_PASS(SIMemoryLegalizer, DEBUG_TYPE, PASS_NAME, false, false)
+INITIALIZE_PASS(SIMemoryLegalizerLegacy, DEBUG_TYPE, PASS_NAME, false, false)
 
-char SIMemoryLegalizer::ID = 0;
-char &llvm::SIMemoryLegalizerID = SIMemoryLegalizer::ID;
+char SIMemoryLegalizerLegacy::ID = 0;
+char &llvm::SIMemoryLegalizerID = SIMemoryLegalizerLegacy::ID;
 
 FunctionPass *llvm::createSIMemoryLegalizerPass() {
-  return new SIMemoryLegalizer();
+  return new SIMemoryLegalizerLegacy();
 }
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index 5e62ceac281b8..eeac9c1ad1084 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -423,6 +423,12 @@ let SubtargetPredicate = isGFX11Plus in {
   }
 } // End SubtargetPredicate = isGFX11Plus
 
+let SubtargetPredicate = isGFX12Plus in {
+  let hasSideEffects = 1, Defs = [SCC] in {
+    def S_ALLOC_VGPR : SOP1_0_32 <"s_alloc_vgpr">;
+  }
+} // End SubtargetPredicate = isGFX12Plus
+
 class SOP1_F32_Inst<string opName, SDPatternOperator Op, ValueType vt0=f32,
                     ValueType vt1=vt0> :
   SOP1_32<opName, [(set vt0:$sdst, (UniformUnaryFrag<Op> vt1:$src0))]>;
@@ -2048,6 +2054,7 @@ defm S_GET_BARRIER_STATE_M0       : SOP1_M0_Real_gfx12<0x050>;
 defm S_BARRIER_SIGNAL_IMM         : SOP1_IMM_Real_gfx12<0x04e>;
 defm S_BARRIER_SIGNAL_ISFIRST_IMM : SOP1_IMM_Real_gfx12<0x04f>;
 defm S_GET_BARRIER_STATE_IMM      : SOP1_IMM_Real_gfx12<0x050>;
+defm S_ALLOC_VGPR                 : SOP1_Real_gfx12<0x053>;
 defm S_SLEEP_VAR                  : SOP1_IMM_Real_gfx12<0x058>;
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index b51cf536467b9..ac6b07bad3e35 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -164,6 +164,18 @@ inline unsigned getSaSdstBitWidth() { return 1; }
 /// \returns SaSdst bit shift
 inline unsigned getSaSdstBitShift() { return 0; }
 
+/// \returns VaSsrc width
+inline unsigned getVaSsrcBitWidth() { return 1; }
+
+/// \returns VaSsrc bit shift
+inline unsigned getVaSsrcBitShift() { return 8; }
+
+/// \returns HoldCnt bit shift
+inline unsigned getHoldCntWidth() { return 1; }
+
+/// \returns HoldCnt bit shift
+inline unsigned getHoldCntBitShift() { return 7; }
+
 } // end anonymous namespace
 
 namespace llvm {
@@ -1740,6 +1752,14 @@ unsigned decodeFieldVaVcc(unsigned Encoded) {
   return unpackBits(Encoded, getVaVccBitShift(), getVaVccBitWidth());
 }
 
+unsigned decodeFieldVaSsrc(unsigned Encoded) {
+  return unpackBits(Encoded, getVaSsrcBitShift(), getVaSsrcBitWidth());
+}
+
+unsigned decodeFieldHoldCnt(unsigned Encoded) {
+  return unpackBits(Encoded, getHoldCntBitShift(), getHoldCntWidth());
+}
+
 unsigned encodeFieldVmVsrc(unsigned Encoded, unsigned VmVsrc) {
   return packBits(VmVsrc, Encoded, getVmVsrcBitShift(), getVmVsrcBitWidth());
 }
@@ -1780,6 +1800,22 @@ unsigned encodeFieldVaVcc(unsigned VaVcc) {
   return encodeFieldVaVcc(0xffff, VaVcc);
 }
 
+unsigned encodeFieldVaSsrc(unsigned Encoded, unsigned VaSsrc) {
+  return packBits(VaSsrc, Encoded, getVaSsrcBitShift(), getVaSsrcBitWidth());
+}
+
+unsigned encodeFieldVaSsrc(unsigned VaSsrc) {
+  return encodeFieldVaSsrc(0xffff, VaSsrc);
+}
+
+unsigned encodeFieldHoldCnt(unsigned Encoded, unsigned HoldCnt) {
+  return packBits(HoldCnt, Encoded, getHoldCntBitShift(), getHoldCntWidth());
+}
+
+unsigned encodeFieldHoldCnt(unsigned HoldCnt) {
+  return encodeFieldHoldCnt(0xffff, HoldCnt);
+}
+
 } // namespace DepCtr
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index f54d5a273ca37..184f40bccfff8 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -1180,6 +1180,12 @@ unsigned decodeFieldVaSdst(unsigned Encoded);
 /// \returns Decoded VaVcc from given immediate \p Encoded.
 unsigned decodeFieldVaVcc(unsigned Encoded);
 
+/// \returns Decoded SaSrc from given immediate \p Encoded.
+unsigned decodeFieldVaSsrc(unsigned Encoded);
+
+/// \returns Decoded HoldCnt from given immediate \p Encoded.
+unsigned decodeFieldHoldCnt(unsigned Encoded);
+
 /// \returns \p VmVsrc as an encoded Depctr immediate.
 unsigned encodeFieldVmVsrc(unsigned VmVsrc);
 
@@ -1210,6 +1216,18 @@ unsigned encodeFieldVaVcc(unsigned VaVcc);
 /// \returns \p Encoded combined with encoded \p VaVcc.
 unsigned encodeFieldVaVcc(unsigned Encoded, unsigned VaVcc);
 
+/// \returns \p HoldCnt as an encoded Depctr immediate.
+unsigned encodeFieldHoldCnt(unsigned HoldCnt);
+
+/// \returns \p Encoded combined with encoded \p HoldCnt.
+unsigned encodeFieldHoldCnt(unsigned HoldCnt, unsigned Encoded);
+
+/// \returns \p VaSsrc as an encoded Depctr immediate.
+unsigned encodeFieldVaSsrc(unsigned VaSsrc);
+
+/// \returns \p Encoded combined with encoded \p VaSsrc.
+unsigned encodeFieldVaSsrc(unsigned Encoded, unsigned VaSsrc);
+
 } // namespace DepCtr
 
 namespace Exp {
diff --git a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp
index 22ebe175ff62f..9ae0054521b05 100644
--- a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp
@@ -245,7 +245,7 @@ isAsmClobberable(const MachineFunction &MF, MCRegister PhysReg) const {
 }
 
 bool ARMBaseRegisterInfo::isInlineAsmReadOnlyReg(const MachineFunction &MF,
-                                                 unsigned PhysReg) const {
+                                                 MCRegister PhysReg) const {
   const ARMSubtarget &STI = MF.getSubtarget<ARMSubtarget>();
   const ARMFrameLowering *TFI = getFrameLowering(MF);
 
@@ -256,7 +256,7 @@ bool ARMBaseRegisterInfo::isInlineAsmReadOnlyReg(const MachineFunction &MF,
   if (hasBasePointer(MF))
     markSuperRegs(Reserved, BasePtr);
   assert(checkAllSuperRegsMarked(Reserved));
-  return Reserved.test(PhysReg);
+  return Reserved.test(PhysReg.id());
 }
 
 const TargetRegisterClass *
diff --git a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.h b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.h
index 68a28043fd32e..69e10ac2a54d2 100644
--- a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.h
+++ b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.h
@@ -88,7 +88,7 @@ class ARMBaseRegisterInfo : public ARMGenRegisterInfo {
   bool isAsmClobberable(const MachineFunction &MF,
                        MCRegister PhysReg) const override;
   bool isInlineAsmReadOnlyReg(const MachineFunction &MF,
-                              unsigned PhysReg) const override;
+                              MCRegister PhysReg) const override;
 
   const TargetRegisterClass *
   getPointerRegClass(const MachineFunction &MF,
diff --git a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
index 9e1f3fcbdc43f..59b4923c0c5b8 100644
--- a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
@@ -2144,7 +2144,7 @@ static void CMSEPushCalleeSaves(const TargetInstrInfo &TII,
 
 static void CMSEPopCalleeSaves(const TargetInstrInfo &TII,
                                MachineBasicBlock &MBB,
-                               MachineBasicBlock::iterator MBBI, int JumpReg,
+                               MachineBasicBlock::iterator MBBI,
                                bool Thumb1Only) {
   const DebugLoc &DL = MBBI->getDebugLoc();
   if (Thumb1Only) {
@@ -2418,7 +2418,7 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
 
       CMSERestoreFPRegs(MBB, MBBI, DL, OriginalClearRegs); // restore FP registers
 
-      CMSEPopCalleeSaves(*TII, MBB, MBBI, JumpReg, AFI->isThumb1OnlyFunction());
+      CMSEPopCalleeSaves(*TII, MBB, MBBI, AFI->isThumb1OnlyFunction());
 
       MI.eraseFromParent();
       return true;
diff --git a/llvm/lib/Target/ARM/ARMFastISel.cpp b/llvm/lib/Target/ARM/ARMFastISel.cpp
index eb7f196be4f1f..765c65c5fcb24 100644
--- a/llvm/lib/Target/ARM/ARMFastISel.cpp
+++ b/llvm/lib/Target/ARM/ARMFastISel.cpp
@@ -83,24 +83,51 @@ using namespace llvm;
 namespace {
 
   // All possible address modes, plus some.
-  struct Address {
-    enum {
-      RegBase,
-      FrameIndexBase
-    } BaseType = RegBase;
-
-    union {
-      unsigned Reg;
-      int FI;
-    } Base;
-
-    int Offset = 0;
-
-    // Innocuous defaults for our address.
-    Address() {
-      Base.Reg = 0;
-    }
-  };
+class Address {
+public:
+  using BaseKind = enum { RegBase, FrameIndexBase };
+
+private:
+  BaseKind Kind = RegBase;
+  union {
+    unsigned Reg;
+    int FI;
+  } Base;
+
+  int Offset = 0;
+
+public:
+  // Innocuous defaults for our address.
+  Address() { Base.Reg = 0; }
+
+  void setKind(BaseKind K) { Kind = K; }
+  BaseKind getKind() const { return Kind; }
+  bool isRegBase() const { return Kind == RegBase; }
+  bool isFIBase() const { return Kind == FrameIndexBase; }
+
+  void setReg(Register Reg) {
+    assert(isRegBase() && "Invalid base register access!");
+    Base.Reg = Reg.id();
+  }
+
+  Register getReg() const {
+    assert(isRegBase() && "Invalid base register access!");
+    return Base.Reg;
+  }
+
+  void setFI(int FI) {
+    assert(isFIBase() && "Invalid base frame index  access!");
+    Base.FI = FI;
+  }
+
+  int getFI() const {
+    assert(isFIBase() && "Invalid base frame index access!");
+    return Base.FI;
+  }
+
+  void setOffset(int O) { Offset = O; }
+  int getOffset() { return Offset; }
+};
 
 class ARMFastISel final : public FastISel {
   /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
@@ -132,17 +159,16 @@ class ARMFastISel final : public FastISel {
   private:
     // Code from FastISel.cpp.
 
-    unsigned fastEmitInst_r(unsigned MachineInstOpcode,
-                            const TargetRegisterClass *RC, unsigned Op0);
-    unsigned fastEmitInst_rr(unsigned MachineInstOpcode,
-                             const TargetRegisterClass *RC,
-                             unsigned Op0, unsigned Op1);
-    unsigned fastEmitInst_ri(unsigned MachineInstOpcode,
-                             const TargetRegisterClass *RC,
-                             unsigned Op0, uint64_t Imm);
-    unsigned fastEmitInst_i(unsigned MachineInstOpcode,
-                            const TargetRegisterClass *RC,
-                            uint64_t Imm);
+    Register fastEmitInst_r(unsigned MachineInstOpcode,
+                            const TargetRegisterClass *RC, Register Op0);
+    Register fastEmitInst_rr(unsigned MachineInstOpcode,
+                             const TargetRegisterClass *RC, Register Op0,
+                             Register Op1);
+    Register fastEmitInst_ri(unsigned MachineInstOpcode,
+                             const TargetRegisterClass *RC, Register Op0,
+                             uint64_t Imm);
+    Register fastEmitInst_i(unsigned MachineInstOpcode,
+                            const TargetRegisterClass *RC, uint64_t Imm);
 
     // Backend specific FastISel code.
 
@@ -188,21 +214,21 @@ class ARMFastISel final : public FastISel {
     bool ARMEmitLoad(MVT VT, Register &ResultReg, Address &Addr,
                      MaybeAlign Alignment = std::nullopt, bool isZExt = true,
                      bool allocReg = true);
-    bool ARMEmitStore(MVT VT, unsigned SrcReg, Address &Addr,
+    bool ARMEmitStore(MVT VT, Register SrcReg, Address &Addr,
                       MaybeAlign Alignment = std::nullopt);
     bool ARMComputeAddress(const Value *Obj, Address &Addr);
     void ARMSimplifyAddress(Address &Addr, MVT VT, bool useAM3);
     bool ARMIsMemCpySmall(uint64_t Len);
     bool ARMTryEmitSmallMemCpy(Address Dest, Address Src, uint64_t Len,
                                MaybeAlign Alignment);
-    unsigned ARMEmitIntExt(MVT SrcVT, unsigned SrcReg, MVT DestVT, bool isZExt);
-    unsigned ARMMaterializeFP(const ConstantFP *CFP, MVT VT);
-    unsigned ARMMaterializeInt(const Constant *C, MVT VT);
-    unsigned ARMMaterializeGV(const GlobalValue *GV, MVT VT);
-    unsigned ARMMoveToFPReg(MVT VT, unsigned SrcReg);
-    unsigned ARMMoveToIntReg(MVT VT, unsigned SrcReg);
+    Register ARMEmitIntExt(MVT SrcVT, Register SrcReg, MVT DestVT, bool isZExt);
+    Register ARMMaterializeFP(const ConstantFP *CFP, MVT VT);
+    Register ARMMaterializeInt(const Constant *C, MVT VT);
+    Register ARMMaterializeGV(const GlobalValue *GV, MVT VT);
+    Register ARMMoveToFPReg(MVT VT, Register SrcReg);
+    Register ARMMoveToIntReg(MVT VT, Register SrcReg);
     unsigned ARMSelectCallOp(bool UseReg);
-    unsigned ARMLowerPICELF(const GlobalValue *GV, MVT VT);
+    Register ARMLowerPICELF(const GlobalValue *GV, MVT VT);
 
     const TargetLowering *getTargetLowering() { return &TLI; }
 
@@ -219,7 +245,7 @@ class ARMFastISel final : public FastISel {
                          CallingConv::ID CC,
                          unsigned &NumBytes,
                          bool isVarArg);
-    unsigned getLibcallReg(const Twine &Name);
+    Register getLibcallReg(const Twine &Name);
     bool FinishCall(MVT RetVT, SmallVectorImpl<Register> &UsedRegs,
                     const Instruction *I, CallingConv::ID CC,
                     unsigned &NumBytes, bool isVarArg);
@@ -291,9 +317,9 @@ ARMFastISel::AddOptionalDefs(const MachineInstrBuilder &MIB) {
   return MIB;
 }
 
-unsigned ARMFastISel::fastEmitInst_r(unsigned MachineInstOpcode,
+Register ARMFastISel::fastEmitInst_r(unsigned MachineInstOpcode,
                                      const TargetRegisterClass *RC,
-                                     unsigned Op0) {
+                                     Register Op0) {
   Register ResultReg = createResultReg(RC);
   const MCInstrDesc &II = TII.get(MachineInstOpcode);
 
@@ -313,9 +339,9 @@ unsigned ARMFastISel::fastEmitInst_r(unsigned MachineInstOpcode,
   return ResultReg;
 }
 
-unsigned ARMFastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
+Register ARMFastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
                                       const TargetRegisterClass *RC,
-                                      unsigned Op0, unsigned Op1) {
+                                      Register Op0, Register Op1) {
   Register ResultReg = createResultReg(RC);
   const MCInstrDesc &II = TII.get(MachineInstOpcode);
 
@@ -340,9 +366,9 @@ unsigned ARMFastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
   return ResultReg;
 }
 
-unsigned ARMFastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
+Register ARMFastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
                                       const TargetRegisterClass *RC,
-                                      unsigned Op0, uint64_t Imm) {
+                                      Register Op0, uint64_t Imm) {
   Register ResultReg = createResultReg(RC);
   const MCInstrDesc &II = TII.get(MachineInstOpcode);
 
@@ -365,7 +391,7 @@ unsigned ARMFastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
   return ResultReg;
 }
 
-unsigned ARMFastISel::fastEmitInst_i(unsigned MachineInstOpcode,
+Register ARMFastISel::fastEmitInst_i(unsigned MachineInstOpcode,
                                      const TargetRegisterClass *RC,
                                      uint64_t Imm) {
   Register ResultReg = createResultReg(RC);
@@ -386,8 +412,9 @@ unsigned ARMFastISel::fastEmitInst_i(unsigned MachineInstOpcode,
 
 // TODO: Don't worry about 64-bit now, but when this is fixed remove the
 // checks from the various callers.
-unsigned ARMFastISel::ARMMoveToFPReg(MVT VT, unsigned SrcReg) {
-  if (VT == MVT::f64) return 0;
+Register ARMFastISel::ARMMoveToFPReg(MVT VT, Register SrcReg) {
+  if (VT == MVT::f64)
+    return Register();
 
   Register MoveReg = createResultReg(TLI.getRegClassFor(VT));
   AddOptionalDefs(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
@@ -396,8 +423,9 @@ unsigned ARMFastISel::ARMMoveToFPReg(MVT VT, unsigned SrcReg) {
   return MoveReg;
 }
 
-unsigned ARMFastISel::ARMMoveToIntReg(MVT VT, unsigned SrcReg) {
-  if (VT == MVT::i64) return 0;
+Register ARMFastISel::ARMMoveToIntReg(MVT VT, Register SrcReg) {
+  if (VT == MVT::i64)
+    return Register();
 
   Register MoveReg = createResultReg(TLI.getRegClassFor(VT));
   AddOptionalDefs(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
@@ -409,7 +437,7 @@ unsigned ARMFastISel::ARMMoveToIntReg(MVT VT, unsigned SrcReg) {
 // For double width floating point we need to materialize two constants
 // (the high and the low) into integer registers then use a move to get
 // the combined constant into an FP reg.
-unsigned ARMFastISel::ARMMaterializeFP(const ConstantFP *CFP, MVT VT) {
+Register ARMFastISel::ARMMaterializeFP(const ConstantFP *CFP, MVT VT) {
   const APFloat Val = CFP->getValueAPF();
   bool is64bit = VT == MVT::f64;
 
@@ -448,9 +476,9 @@ unsigned ARMFastISel::ARMMaterializeFP(const ConstantFP *CFP, MVT VT) {
   return DestReg;
 }
 
-unsigned ARMFastISel::ARMMaterializeInt(const Constant *C, MVT VT) {
+Register ARMFastISel::ARMMaterializeInt(const Constant *C, MVT VT) {
   if (VT != MVT::i32 && VT != MVT::i16 && VT != MVT::i8 && VT != MVT::i1)
-    return 0;
+    return Register();
 
   // If we can do this in a single instruction without a constant pool entry
   // do so now.
@@ -483,7 +511,7 @@ unsigned ARMFastISel::ARMMaterializeInt(const Constant *C, MVT VT) {
     }
   }
 
-  unsigned ResultReg = 0;
+  Register ResultReg;
   if (Subtarget->useMovt())
     ResultReg = fastEmit_i(VT, VT, ISD::Constant, CI->getZExtValue());
 
@@ -492,7 +520,7 @@ unsigned ARMFastISel::ARMMaterializeInt(const Constant *C, MVT VT) {
 
   // Load from constant pool.  For now 32-bit only.
   if (VT != MVT::i32)
-    return 0;
+    return Register();
 
   // MachineConstantPool wants an explicit alignment.
   Align Alignment = DL.getPrefTypeAlign(C->getType());
@@ -517,13 +545,14 @@ bool ARMFastISel::isPositionIndependent() const {
   return TLI.isPositionIndependent();
 }
 
-unsigned ARMFastISel::ARMMaterializeGV(const GlobalValue *GV, MVT VT) {
+Register ARMFastISel::ARMMaterializeGV(const GlobalValue *GV, MVT VT) {
   // For now 32-bit only.
-  if (VT != MVT::i32 || GV->isThreadLocal()) return 0;
+  if (VT != MVT::i32 || GV->isThreadLocal())
+    return Register();
 
   // ROPI/RWPI not currently supported.
   if (Subtarget->isROPI() || Subtarget->isRWPI())
-    return 0;
+    return Register();
 
   bool IsIndirect = Subtarget->isGVIndirectSymbol(GV);
   const TargetRegisterClass *RC = isThumb2 ? &ARM::rGPRRegClass
@@ -533,7 +562,8 @@ unsigned ARMFastISel::ARMMaterializeGV(const GlobalValue *GV, MVT VT) {
   // FastISel TLS support on non-MachO is broken, punt to SelectionDAG.
   const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV);
   bool IsThreadLocal = GVar && GVar->isThreadLocal();
-  if (!Subtarget->isTargetMachO() && IsThreadLocal) return 0;
+  if (!Subtarget->isTargetMachO() && IsThreadLocal)
+    return Register();
 
   bool IsPositionIndependent = isPositionIndependent();
   // Use movw+movt when possible, it avoids constant pool entries.
@@ -637,7 +667,7 @@ Register ARMFastISel::fastMaterializeConstant(const Constant *C) {
   return Register();
 }
 
-// TODO: unsigned ARMFastISel::TargetMaterializeFloatZero(const ConstantFP *CF);
+// TODO: Register ARMFastISel::TargetMaterializeFloatZero(const ConstantFP *CF);
 
 Register ARMFastISel::fastMaterializeAlloca(const AllocaInst *AI) {
   // Don't handle dynamic allocas.
@@ -735,7 +765,7 @@ bool ARMFastISel::ARMComputeAddress(const Value *Obj, Address &Addr) {
       break;
     case Instruction::GetElementPtr: {
       Address SavedAddr = Addr;
-      int TmpOffset = Addr.Offset;
+      int TmpOffset = Addr.getOffset();
 
       // Iterate through the GEP folding the constants into offsets where
       // we can.
@@ -771,7 +801,7 @@ bool ARMFastISel::ARMComputeAddress(const Value *Obj, Address &Addr) {
       }
 
       // Try to grab the base operand now.
-      Addr.Offset = TmpOffset;
+      Addr.setOffset(TmpOffset);
       if (ARMComputeAddress(U->getOperand(0), Addr)) return true;
 
       // We failed, restore everything and try the other options.
@@ -785,8 +815,8 @@ bool ARMFastISel::ARMComputeAddress(const Value *Obj, Address &Addr) {
       DenseMap<const AllocaInst*, int>::iterator SI =
         FuncInfo.StaticAllocaMap.find(AI);
       if (SI != FuncInfo.StaticAllocaMap.end()) {
-        Addr.BaseType = Address::FrameIndexBase;
-        Addr.Base.FI = SI->second;
+        Addr.setKind(Address::FrameIndexBase);
+        Addr.setFI(SI->second);
         return true;
       }
       break;
@@ -794,8 +824,9 @@ bool ARMFastISel::ARMComputeAddress(const Value *Obj, Address &Addr) {
   }
 
   // Try to get this in a register if nothing else has worked.
-  if (Addr.Base.Reg == 0) Addr.Base.Reg = getRegForValue(Obj);
-  return Addr.Base.Reg != 0;
+  if (!Addr.getReg())
+    Addr.setReg(getRegForValue(Obj));
+  return Addr.getReg();
 }
 
 void ARMFastISel::ARMSimplifyAddress(Address &Addr, MVT VT, bool useAM3) {
@@ -808,45 +839,45 @@ void ARMFastISel::ARMSimplifyAddress(Address &Addr, MVT VT, bool useAM3) {
     case MVT::i32:
       if (!useAM3) {
         // Integer loads/stores handle 12-bit offsets.
-        needsLowering = ((Addr.Offset & 0xfff) != Addr.Offset);
+        needsLowering = ((Addr.getOffset() & 0xfff) != Addr.getOffset());
         // Handle negative offsets.
         if (needsLowering && isThumb2)
-          needsLowering = !(Subtarget->hasV6T2Ops() && Addr.Offset < 0 &&
-                            Addr.Offset > -256);
+          needsLowering = !(Subtarget->hasV6T2Ops() && Addr.getOffset() < 0 &&
+                            Addr.getOffset() > -256);
       } else {
         // ARM halfword load/stores and signed byte loads use +/-imm8 offsets.
-        needsLowering = (Addr.Offset > 255 || Addr.Offset < -255);
+        needsLowering = (Addr.getOffset() > 255 || Addr.getOffset() < -255);
       }
       break;
     case MVT::f32:
     case MVT::f64:
       // Floating point operands handle 8-bit offsets.
-      needsLowering = ((Addr.Offset & 0xff) != Addr.Offset);
+      needsLowering = ((Addr.getOffset() & 0xff) != Addr.getOffset());
       break;
   }
 
   // If this is a stack pointer and the offset needs to be simplified then
   // put the alloca address into a register, set the base type back to
   // register and continue. This should almost never happen.
-  if (needsLowering && Addr.BaseType == Address::FrameIndexBase) {
+  if (needsLowering && Addr.isFIBase()) {
     const TargetRegisterClass *RC = isThumb2 ? &ARM::tGPRRegClass
                                              : &ARM::GPRRegClass;
     Register ResultReg = createResultReg(RC);
     unsigned Opc = isThumb2 ? ARM::t2ADDri : ARM::ADDri;
-    AddOptionalDefs(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
-                            TII.get(Opc), ResultReg)
-                            .addFrameIndex(Addr.Base.FI)
-                            .addImm(0));
-    Addr.Base.Reg = ResultReg;
-    Addr.BaseType = Address::RegBase;
+    AddOptionalDefs(
+        BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc), ResultReg)
+            .addFrameIndex(Addr.getFI())
+            .addImm(0));
+    Addr.setKind(Address::RegBase);
+    Addr.setReg(ResultReg);
   }
 
   // Since the offset is too large for the load/store instruction
   // get the reg+offset into a register.
   if (needsLowering) {
-    Addr.Base.Reg = fastEmit_ri_(MVT::i32, ISD::ADD, Addr.Base.Reg,
-                                 Addr.Offset, MVT::i32);
-    Addr.Offset = 0;
+    Addr.setReg(fastEmit_ri_(MVT::i32, ISD::ADD, Addr.getReg(),
+                             Addr.getOffset(), MVT::i32));
+    Addr.setOffset(0);
   }
 }
 
@@ -857,12 +888,12 @@ void ARMFastISel::AddLoadStoreOperands(MVT VT, Address &Addr,
   // addrmode5 output depends on the selection dag addressing dividing the
   // offset by 4 that it then later multiplies. Do this here as well.
   if (VT.SimpleTy == MVT::f32 || VT.SimpleTy == MVT::f64)
-    Addr.Offset /= 4;
+    Addr.setOffset(Addr.getOffset() / 4);
 
   // Frame base works a bit differently. Handle it separately.
-  if (Addr.BaseType == Address::FrameIndexBase) {
-    int FI = Addr.Base.FI;
-    int Offset = Addr.Offset;
+  if (Addr.isFIBase()) {
+    int FI = Addr.getFI();
+    int Offset = Addr.getOffset();
     MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
         MachinePointerInfo::getFixedStack(*FuncInfo.MF, FI, Offset), Flags,
         MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
@@ -872,25 +903,27 @@ void ARMFastISel::AddLoadStoreOperands(MVT VT, Address &Addr,
     // ARM halfword load/stores and signed byte loads need an additional
     // operand.
     if (useAM3) {
-      int Imm = (Addr.Offset < 0) ? (0x100 | -Addr.Offset) : Addr.Offset;
+      int Imm = (Addr.getOffset() < 0) ? (0x100 | -Addr.getOffset())
+                                       : Addr.getOffset();
       MIB.addReg(0);
       MIB.addImm(Imm);
     } else {
-      MIB.addImm(Addr.Offset);
+      MIB.addImm(Addr.getOffset());
     }
     MIB.addMemOperand(MMO);
   } else {
     // Now add the rest of the operands.
-    MIB.addReg(Addr.Base.Reg);
+    MIB.addReg(Addr.getReg());
 
     // ARM halfword load/stores and signed byte loads need an additional
     // operand.
     if (useAM3) {
-      int Imm = (Addr.Offset < 0) ? (0x100 | -Addr.Offset) : Addr.Offset;
+      int Imm = (Addr.getOffset() < 0) ? (0x100 | -Addr.getOffset())
+                                       : Addr.getOffset();
       MIB.addReg(0);
       MIB.addImm(Imm);
     } else {
-      MIB.addImm(Addr.Offset);
+      MIB.addImm(Addr.getOffset());
     }
   }
   AddOptionalDefs(MIB);
@@ -909,7 +942,8 @@ bool ARMFastISel::ARMEmitLoad(MVT VT, Register &ResultReg, Address &Addr,
     case MVT::i1:
     case MVT::i8:
       if (isThumb2) {
-        if (Addr.Offset < 0 && Addr.Offset > -256 && Subtarget->hasV6T2Ops())
+        if (Addr.getOffset() < 0 && Addr.getOffset() > -256 &&
+            Subtarget->hasV6T2Ops())
           Opc = isZExt ? ARM::t2LDRBi8 : ARM::t2LDRSBi8;
         else
           Opc = isZExt ? ARM::t2LDRBi12 : ARM::t2LDRSBi12;
@@ -929,7 +963,8 @@ bool ARMFastISel::ARMEmitLoad(MVT VT, Register &ResultReg, Address &Addr,
         return false;
 
       if (isThumb2) {
-        if (Addr.Offset < 0 && Addr.Offset > -256 && Subtarget->hasV6T2Ops())
+        if (Addr.getOffset() < 0 && Addr.getOffset() > -256 &&
+            Subtarget->hasV6T2Ops())
           Opc = isZExt ? ARM::t2LDRHi8 : ARM::t2LDRSHi8;
         else
           Opc = isZExt ? ARM::t2LDRHi12 : ARM::t2LDRSHi12;
@@ -945,7 +980,8 @@ bool ARMFastISel::ARMEmitLoad(MVT VT, Register &ResultReg, Address &Addr,
         return false;
 
       if (isThumb2) {
-        if (Addr.Offset < 0 && Addr.Offset > -256 && Subtarget->hasV6T2Ops())
+        if (Addr.getOffset() < 0 && Addr.getOffset() > -256 &&
+            Subtarget->hasV6T2Ops())
           Opc = ARM::t2LDRi8;
         else
           Opc = ARM::t2LDRi12;
@@ -985,7 +1021,7 @@ bool ARMFastISel::ARMEmitLoad(MVT VT, Register &ResultReg, Address &Addr,
   // Create the base instruction, then add the operands.
   if (allocReg)
     ResultReg = createResultReg(RC);
-  assert(ResultReg > 255 && "Expected an allocated virtual register.");
+  assert(ResultReg.isVirtual() && "Expected an allocated virtual register.");
   MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
                                     TII.get(Opc), ResultReg);
   AddLoadStoreOperands(VT, Addr, MIB, MachineMemOperand::MOLoad, useAM3);
@@ -1038,7 +1074,7 @@ bool ARMFastISel::SelectLoad(const Instruction *I) {
   return true;
 }
 
-bool ARMFastISel::ARMEmitStore(MVT VT, unsigned SrcReg, Address &Addr,
+bool ARMFastISel::ARMEmitStore(MVT VT, Register SrcReg, Address &Addr,
                                MaybeAlign Alignment) {
   unsigned StrOpc;
   bool useAM3 = false;
@@ -1058,7 +1094,8 @@ bool ARMFastISel::ARMEmitStore(MVT VT, unsigned SrcReg, Address &Addr,
     }
     case MVT::i8:
       if (isThumb2) {
-        if (Addr.Offset < 0 && Addr.Offset > -256 && Subtarget->hasV6T2Ops())
+        if (Addr.getOffset() < 0 && Addr.getOffset() > -256 &&
+            Subtarget->hasV6T2Ops())
           StrOpc = ARM::t2STRBi8;
         else
           StrOpc = ARM::t2STRBi12;
@@ -1072,7 +1109,8 @@ bool ARMFastISel::ARMEmitStore(MVT VT, unsigned SrcReg, Address &Addr,
         return false;
 
       if (isThumb2) {
-        if (Addr.Offset < 0 && Addr.Offset > -256 && Subtarget->hasV6T2Ops())
+        if (Addr.getOffset() < 0 && Addr.getOffset() > -256 &&
+            Subtarget->hasV6T2Ops())
           StrOpc = ARM::t2STRHi8;
         else
           StrOpc = ARM::t2STRHi12;
@@ -1087,7 +1125,8 @@ bool ARMFastISel::ARMEmitStore(MVT VT, unsigned SrcReg, Address &Addr,
         return false;
 
       if (isThumb2) {
-        if (Addr.Offset < 0 && Addr.Offset > -256 && Subtarget->hasV6T2Ops())
+        if (Addr.getOffset() < 0 && Addr.getOffset() > -256 &&
+            Subtarget->hasV6T2Ops())
           StrOpc = ARM::t2STRi8;
         else
           StrOpc = ARM::t2STRi12;
@@ -1135,7 +1174,7 @@ bool ARMFastISel::ARMEmitStore(MVT VT, unsigned SrcReg, Address &Addr,
 
 bool ARMFastISel::SelectStore(const Instruction *I) {
   Value *Op0 = I->getOperand(0);
-  unsigned SrcReg = 0;
+  Register SrcReg;
 
   // Atomic stores need special handling.
   if (cast<StoreInst>(I)->isAtomic())
@@ -1163,7 +1202,8 @@ bool ARMFastISel::SelectStore(const Instruction *I) {
 
   // Get the value to be stored into a register.
   SrcReg = getRegForValue(Op0);
-  if (SrcReg == 0) return false;
+  if (!SrcReg)
+    return false;
 
   // See if we can handle this address.
   Address Addr;
@@ -1289,7 +1329,8 @@ bool ARMFastISel::SelectBranch(const Instruction *I) {
   }
 
   Register CmpReg = getRegForValue(BI->getCondition());
-  if (CmpReg == 0) return false;
+  if (!CmpReg)
+    return false;
 
   // We've been divorced from our compare!  Our block was split, and
   // now our compare lives in a predecessor block.  We musn't
@@ -1320,7 +1361,8 @@ bool ARMFastISel::SelectBranch(const Instruction *I) {
 
 bool ARMFastISel::SelectIndirectBr(const Instruction *I) {
   Register AddrReg = getRegForValue(I->getOperand(0));
-  if (AddrReg == 0) return false;
+  if (!AddrReg)
+    return false;
 
   unsigned Opc = isThumb2 ? ARM::tBRIND : ARM::BX;
   assert(isThumb2 || Subtarget->hasV4TOps());
@@ -1411,21 +1453,25 @@ bool ARMFastISel::ARMEmitCmp(const Value *Src1Value, const Value *Src2Value,
   }
 
   Register SrcReg1 = getRegForValue(Src1Value);
-  if (SrcReg1 == 0) return false;
+  if (!SrcReg1)
+    return false;
 
-  unsigned SrcReg2 = 0;
+  Register SrcReg2;
   if (!UseImm) {
     SrcReg2 = getRegForValue(Src2Value);
-    if (SrcReg2 == 0) return false;
+    if (!SrcReg2)
+      return false;
   }
 
   // We have i1, i8, or i16, we need to either zero extend or sign extend.
   if (needsExt) {
     SrcReg1 = ARMEmitIntExt(SrcVT, SrcReg1, MVT::i32, isZExt);
-    if (SrcReg1 == 0) return false;
+    if (!SrcReg1)
+      return false;
     if (!UseImm) {
       SrcReg2 = ARMEmitIntExt(SrcVT, SrcReg2, MVT::i32, isZExt);
-      if (SrcReg2 == 0) return false;
+      if (!SrcReg2)
+        return false;
     }
   }
 
@@ -1474,7 +1520,7 @@ bool ARMFastISel::SelectCmp(const Instruction *I) {
                                            : &ARM::GPRRegClass;
   Register DestReg = createResultReg(RC);
   Constant *Zero = ConstantInt::get(Type::getInt32Ty(*Context), 0);
-  unsigned ZeroReg = fastMaterializeConstant(Zero);
+  Register ZeroReg = fastMaterializeConstant(Zero);
   // ARMEmitCmp emits a FMSTAT when necessary, so it's always safe to use CPSR.
   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(MovCCOpc), DestReg)
           .addReg(ZeroReg).addImm(1)
@@ -1493,7 +1539,8 @@ bool ARMFastISel::SelectFPExt(const Instruction *I) {
       !V->getType()->isFloatTy()) return false;
 
   Register Op = getRegForValue(V);
-  if (Op == 0) return false;
+  if (!Op)
+    return false;
 
   Register Result = createResultReg(&ARM::DPRRegClass);
   AddOptionalDefs(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
@@ -1512,7 +1559,8 @@ bool ARMFastISel::SelectFPTrunc(const Instruction *I) {
         V->getType()->isDoubleTy())) return false;
 
   Register Op = getRegForValue(V);
-  if (Op == 0) return false;
+  if (!Op)
+    return false;
 
   Register Result = createResultReg(&ARM::SPRRegClass);
   AddOptionalDefs(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
@@ -1540,19 +1588,22 @@ bool ARMFastISel::SelectIToFP(const Instruction *I, bool isSigned) {
     return false;
 
   Register SrcReg = getRegForValue(Src);
-  if (SrcReg == 0) return false;
+  if (!SrcReg)
+    return false;
 
   // Handle sign-extension.
   if (SrcVT == MVT::i16 || SrcVT == MVT::i8) {
     SrcReg = ARMEmitIntExt(SrcVT, SrcReg, MVT::i32,
                                        /*isZExt*/!isSigned);
-    if (SrcReg == 0) return false;
+    if (!SrcReg)
+      return false;
   }
 
   // The conversion routine works on fp-reg to fp-reg and the operand above
   // was an integer, move it to the fp registers if possible.
-  unsigned FP = ARMMoveToFPReg(MVT::f32, SrcReg);
-  if (FP == 0) return false;
+  Register FP = ARMMoveToFPReg(MVT::f32, SrcReg);
+  if (!FP)
+    return false;
 
   unsigned Opc;
   if (Ty->isFloatTy()) Opc = isSigned ? ARM::VSITOS : ARM::VUITOS;
@@ -1577,7 +1628,8 @@ bool ARMFastISel::SelectFPToI(const Instruction *I, bool isSigned) {
     return false;
 
   Register Op = getRegForValue(I->getOperand(0));
-  if (Op == 0) return false;
+  if (!Op)
+    return false;
 
   unsigned Opc;
   Type *OpTy = I->getOperand(0)->getType();
@@ -1593,8 +1645,9 @@ bool ARMFastISel::SelectFPToI(const Instruction *I, bool isSigned) {
 
   // This result needs to be in an integer register, but the conversion only
   // takes place in fp-regs.
-  unsigned IntReg = ARMMoveToIntReg(DstVT, ResultReg);
-  if (IntReg == 0) return false;
+  Register IntReg = ARMMoveToIntReg(DstVT, ResultReg);
+  if (!IntReg)
+    return false;
 
   updateValueMap(I, IntReg);
   return true;
@@ -1609,9 +1662,11 @@ bool ARMFastISel::SelectSelect(const Instruction *I) {
   if (VT != MVT::i32) return false;
 
   Register CondReg = getRegForValue(I->getOperand(0));
-  if (CondReg == 0) return false;
+  if (!CondReg)
+    return false;
   Register Op1Reg = getRegForValue(I->getOperand(1));
-  if (Op1Reg == 0) return false;
+  if (!Op1Reg)
+    return false;
 
   // Check to see if we can use an immediate in the conditional move.
   int Imm = 0;
@@ -1628,10 +1683,11 @@ bool ARMFastISel::SelectSelect(const Instruction *I) {
       (ARM_AM::getSOImmVal(Imm) != -1);
   }
 
-  unsigned Op2Reg = 0;
+  Register Op2Reg;
   if (!UseImm) {
     Op2Reg = getRegForValue(I->getOperand(2));
-    if (Op2Reg == 0) return false;
+    if (!Op2Reg)
+      return false;
   }
 
   unsigned TstOpc = isThumb2 ? ARM::t2TSTri : ARM::TSTri;
@@ -1757,12 +1813,14 @@ bool ARMFastISel::SelectBinaryIntOp(const Instruction *I, unsigned ISDOpcode) {
   }
 
   Register SrcReg1 = getRegForValue(I->getOperand(0));
-  if (SrcReg1 == 0) return false;
+  if (!SrcReg1)
+    return false;
 
   // TODO: Often the 2nd operand is an immediate, which can be encoded directly
   // in the instruction, rather then materializing the value in a register.
   Register SrcReg2 = getRegForValue(I->getOperand(1));
-  if (SrcReg2 == 0) return false;
+  if (!SrcReg2)
+    return false;
 
   Register ResultReg = createResultReg(&ARM::GPRnopcRegClass);
   SrcReg1 = constrainOperandRegClass(TII.get(Opc), SrcReg1, 1);
@@ -1808,10 +1866,12 @@ bool ARMFastISel::SelectBinaryFPOp(const Instruction *I, unsigned ISDOpcode) {
       break;
   }
   Register Op1 = getRegForValue(I->getOperand(0));
-  if (Op1 == 0) return false;
+  if (!Op1)
+    return false;
 
   Register Op2 = getRegForValue(I->getOperand(1));
-  if (Op2 == 0) return false;
+  if (!Op2)
+    return false;
 
   Register ResultReg = createResultReg(TLI.getRegClassFor(VT.SimpleTy));
   AddOptionalDefs(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
@@ -1953,7 +2013,7 @@ bool ARMFastISel::ProcessCallArgs(SmallVectorImpl<Value*> &Args,
       case CCValAssign::SExt: {
         MVT DestVT = VA.getLocVT();
         Arg = ARMEmitIntExt(ArgVT, Arg, DestVT, /*isZExt*/false);
-        assert(Arg != 0 && "Failed to emit a sext");
+        assert(Arg && "Failed to emit a sext");
         ArgVT = DestVT;
         break;
       }
@@ -1962,13 +2022,13 @@ bool ARMFastISel::ProcessCallArgs(SmallVectorImpl<Value*> &Args,
       case CCValAssign::ZExt: {
         MVT DestVT = VA.getLocVT();
         Arg = ARMEmitIntExt(ArgVT, Arg, DestVT, /*isZExt*/true);
-        assert(Arg != 0 && "Failed to emit a zext");
+        assert(Arg && "Failed to emit a zext");
         ArgVT = DestVT;
         break;
       }
       case CCValAssign::BCvt: {
-        unsigned BC = fastEmit_r(ArgVT, VA.getLocVT(), ISD::BITCAST, Arg);
-        assert(BC != 0 && "Failed to emit a bitcast!");
+        Register BC = fastEmit_r(ArgVT, VA.getLocVT(), ISD::BITCAST, Arg);
+        assert(BC && "Failed to emit a bitcast!");
         Arg = BC;
         ArgVT = VA.getLocVT();
         break;
@@ -2007,9 +2067,9 @@ bool ARMFastISel::ProcessCallArgs(SmallVectorImpl<Value*> &Args,
         continue;
 
       Address Addr;
-      Addr.BaseType = Address::RegBase;
-      Addr.Base.Reg = ARM::SP;
-      Addr.Offset = VA.getLocMemOffset();
+      Addr.setKind(Address::RegBase);
+      Addr.setReg(ARM::SP);
+      Addr.setOffset(VA.getLocMemOffset());
 
       bool EmitRet = ARMEmitStore(ArgVT, Arg, Addr); (void)EmitRet;
       assert(EmitRet && "Could not emit a store for argument!");
@@ -2091,7 +2151,7 @@ bool ARMFastISel::SelectRet(const Instruction *I) {
     return false;
 
   // Build a list of return value registers.
-  SmallVector<unsigned, 4> RetRegs;
+  SmallVector<Register, 4> RetRegs;
 
   CallingConv::ID CC = F.getCallingConv();
   if (Ret->getNumOperands() > 0) {
@@ -2106,7 +2166,7 @@ bool ARMFastISel::SelectRet(const Instruction *I) {
 
     const Value *RV = Ret->getOperand(0);
     Register Reg = getRegForValue(RV);
-    if (Reg == 0)
+    if (!Reg)
       return false;
 
     // Only handle a single return value for now.
@@ -2122,7 +2182,7 @@ bool ARMFastISel::SelectRet(const Instruction *I) {
     if (!VA.isRegLoc())
       return false;
 
-    unsigned SrcReg = Reg + VA.getValNo();
+    Register SrcReg = Reg + VA.getValNo();
     EVT RVEVT = TLI.getValueType(DL, RV->getType());
     if (!RVEVT.isSimple()) return false;
     MVT RVVT = RVEVT.getSimpleVT();
@@ -2138,7 +2198,8 @@ bool ARMFastISel::SelectRet(const Instruction *I) {
       // nothing.
       if (Outs[0].Flags.isZExt() || Outs[0].Flags.isSExt()) {
         SrcReg = ARMEmitIntExt(RVVT, SrcReg, DestVT, Outs[0].Flags.isZExt());
-        if (SrcReg == 0) return false;
+        if (!SrcReg)
+          return false;
       }
     }
 
@@ -2167,7 +2228,7 @@ bool ARMFastISel::SelectRet(const Instruction *I) {
   MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
                                     TII.get(RetOpc));
   AddOptionalDefs(MIB);
-  for (unsigned R : RetRegs)
+  for (Register R : RetRegs)
     MIB.addReg(R, RegState::Implicit);
   return true;
 }
@@ -2179,11 +2240,12 @@ unsigned ARMFastISel::ARMSelectCallOp(bool UseReg) {
     return isThumb2 ? ARM::tBL : ARM::BL;
 }
 
-unsigned ARMFastISel::getLibcallReg(const Twine &Name) {
+Register ARMFastISel::getLibcallReg(const Twine &Name) {
   // Manually compute the global's type to avoid building it when unnecessary.
   Type *GVTy = PointerType::get(*Context, /*AS=*/0);
   EVT LCREVT = TLI.getValueType(DL, GVTy);
-  if (!LCREVT.isSimple()) return 0;
+  if (!LCREVT.isSimple())
+    return Register();
 
   GlobalValue *GV = M.getNamedGlobal(Name.str());
   if (!GV)
@@ -2231,7 +2293,8 @@ bool ARMFastISel::ARMEmitLibcall(const Instruction *I, RTLIB::Libcall Call) {
   ArgFlags.reserve(I->getNumOperands());
   for (Value *Op :  I->operands()) {
     Register Arg = getRegForValue(Op);
-    if (Arg == 0) return false;
+    if (!Arg)
+      return false;
 
     Type *ArgTy = Op->getType();
     MVT ArgVT;
@@ -2256,7 +2319,8 @@ bool ARMFastISel::ARMEmitLibcall(const Instruction *I, RTLIB::Libcall Call) {
   Register CalleeReg;
   if (Subtarget->genLongCalls()) {
     CalleeReg = getLibcallReg(TLI.getLibcallName(Call));
-    if (CalleeReg == 0) return false;
+    if (!CalleeReg)
+      return false;
   }
 
   // Issue the call.
@@ -2397,7 +2461,8 @@ bool ARMFastISel::SelectCall(const Instruction *I,
     else
       CalleeReg = getRegForValue(Callee);
 
-    if (CalleeReg == 0) return false;
+    if (!CalleeReg)
+      return false;
   }
 
   // Issue the call.
@@ -2477,8 +2542,8 @@ bool ARMFastISel::ARMTryEmitSmallMemCpy(Address Dest, Address Src, uint64_t Len,
 
     unsigned Size = VT.getSizeInBits()/8;
     Len -= Size;
-    Dest.Offset += Size;
-    Src.Offset += Size;
+    Dest.setOffset(Dest.getOffset() + Size);
+    Src.setOffset(Src.getOffset() + Size);
   }
 
   return true;
@@ -2499,14 +2564,14 @@ bool ARMFastISel::SelectIntrinsicCall(const IntrinsicInst &I) {
     const ARMBaseRegisterInfo *RegInfo =
         static_cast<const ARMBaseRegisterInfo *>(Subtarget->getRegisterInfo());
     Register FramePtr = RegInfo->getFrameRegister(*(FuncInfo.MF));
-    unsigned SrcReg = FramePtr;
+    Register SrcReg = FramePtr;
 
     // Recursively load frame address
     // ldr r0 [fp]
     // ldr r0 [r0]
     // ldr r0 [r0]
     // ...
-    unsigned DestReg;
+    Register DestReg;
     unsigned Depth = cast<ConstantInt>(I.getOperand(0))->getZExtValue();
     while (Depth--) {
       DestReg = createResultReg(RC);
@@ -2604,12 +2669,12 @@ bool ARMFastISel::SelectTrunc(const Instruction *I) {
   return true;
 }
 
-unsigned ARMFastISel::ARMEmitIntExt(MVT SrcVT, unsigned SrcReg, MVT DestVT,
+Register ARMFastISel::ARMEmitIntExt(MVT SrcVT, Register SrcReg, MVT DestVT,
                                     bool isZExt) {
   if (DestVT != MVT::i32 && DestVT != MVT::i16 && DestVT != MVT::i8)
-    return 0;
+    return Register();
   if (SrcVT != MVT::i16 && SrcVT != MVT::i8 && SrcVT != MVT::i1)
-    return 0;
+    return Register();
 
   // Table of which combinations can be emitted as a single instruction,
   // and which will require two.
@@ -2704,7 +2769,7 @@ unsigned ARMFastISel::ARMEmitIntExt(MVT SrcVT, unsigned SrcReg, MVT DestVT,
   // 16-bit Thumb instructions always set CPSR (unless they're in an IT block).
   bool setsCPSR = &ARM::tGPRRegClass == RC;
   unsigned LSLOpc = isThumb2 ? ARM::tLSLri : ARM::MOVsi;
-  unsigned ResultReg;
+  Register ResultReg;
   // MOVsi encodes shift and immediate in shift operand addressing mode.
   // The following condition has the same value when emitting two
   // instruction sequences: both are shifts.
@@ -2762,8 +2827,9 @@ bool ARMFastISel::SelectIntExt(const Instruction *I) {
 
   MVT SrcVT = SrcEVT.getSimpleVT();
   MVT DestVT = DestEVT.getSimpleVT();
-  unsigned ResultReg = ARMEmitIntExt(SrcVT, SrcReg, DestVT, isZExt);
-  if (ResultReg == 0) return false;
+  Register ResultReg = ARMEmitIntExt(SrcVT, SrcReg, DestVT, isZExt);
+  if (!ResultReg)
+    return false;
   updateValueMap(I, ResultReg);
   return true;
 }
@@ -2796,16 +2862,19 @@ bool ARMFastISel::SelectShift(const Instruction *I,
 
   Value *Src1Value = I->getOperand(0);
   Register Reg1 = getRegForValue(Src1Value);
-  if (Reg1 == 0) return false;
+  if (!Reg1)
+    return false;
 
-  unsigned Reg2 = 0;
+  Register Reg2;
   if (Opc == ARM::MOVsr) {
     Reg2 = getRegForValue(Src2Value);
-    if (Reg2 == 0) return false;
+    if (!Reg2)
+      return false;
   }
 
   Register ResultReg = createResultReg(&ARM::GPRnopcRegClass);
-  if(ResultReg == 0) return false;
+  if (!ResultReg)
+    return false;
 
   MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
                                     TII.get(Opc), ResultReg)
@@ -2953,7 +3022,7 @@ bool ARMFastISel::tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,
   return true;
 }
 
-unsigned ARMFastISel::ARMLowerPICELF(const GlobalValue *GV, MVT VT) {
+Register ARMFastISel::ARMLowerPICELF(const GlobalValue *GV, MVT VT) {
   bool UseGOT_PREL = !GV->isDSOLocal();
   LLVMContext *Context = &MF->getFunction().getContext();
   unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
@@ -3062,7 +3131,7 @@ bool ARMFastISel::fastLowerArguments() {
   const TargetRegisterClass *RC = &ARM::rGPRRegClass;
   for (const Argument &Arg : F->args()) {
     unsigned ArgNo = Arg.getArgNo();
-    unsigned SrcReg = GPRArgRegs[ArgNo];
+    MCRegister SrcReg = GPRArgRegs[ArgNo];
     Register DstReg = FuncInfo.MF->addLiveIn(SrcReg, RC);
     // FIXME: Unfortunately it's necessary to emit a copy from the livein copy.
     // Without this, EmitLiveInCopies may eliminate the livein if its only
diff --git a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
index 646d57770164a..77ed246edbadf 100644
--- a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
+++ b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
@@ -1113,8 +1113,9 @@ bool BPFAbstractMemberAccess::transformGEPChain(CallInst *Call,
                               Call->getIterator());
 
   // Generate a BitCast
-  auto *BCInst =
-      new BitCastInst(Base, PointerType::getUnqual(BB->getContext()));
+  auto *BCInst = new BitCastInst(
+      Base, PointerType::get(BB->getContext(),
+                             Base->getType()->getPointerAddressSpace()));
   BCInst->insertBefore(Call->getIterator());
 
   // Generate a GetElementPtr
diff --git a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
index c0baf301e0624..4bb1b1eb22eba 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
@@ -2022,8 +2022,9 @@ static bool isTargetConstant(const SDValue &V) {
 }
 
 unsigned HexagonDAGToDAGISel::getUsesInFunction(const Value *V) {
-  if (GAUsesInFunction.count(V))
-    return GAUsesInFunction[V];
+  auto [It, Inserted] = GAUsesInFunction.try_emplace(V);
+  if (!Inserted)
+    return It->second;
 
   unsigned Result = 0;
   const Function &CurF = CurDAG->getMachineFunction().getFunction();
@@ -2033,7 +2034,7 @@ unsigned HexagonDAGToDAGISel::getUsesInFunction(const Value *V) {
       ++Result;
   }
 
-  GAUsesInFunction[V] = Result;
+  It->second = Result;
 
   return Result;
 }
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
index 98b1dde8fa3fc..725067e0c9bdd 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
@@ -728,6 +728,24 @@ class HexagonAsmBackend : public MCAsmBackend {
               MCContext &Context = Asm.getContext();
               auto &RF = cast<MCRelaxableFragment>(*Frags[K]);
               auto &Inst = const_cast<MCInst &>(RF.getInst());
+
+              const bool WouldTraverseLabel = llvm::any_of(
+                  Asm.symbols(), [&Asm, &RF, &Inst](MCSymbol const &sym) {
+                    uint64_t Offset = 0;
+                    const bool HasOffset = Asm.getSymbolOffset(sym, Offset);
+                    const unsigned PacketSizeBytes =
+                        HexagonMCInstrInfo::bundleSize(Inst) *
+                        HEXAGON_INSTR_SIZE;
+                    const bool OffsetPastSym =
+                        Offset <= (Asm.getFragmentOffset(RF) + PacketSizeBytes);
+                    return !sym.isVariable() && Offset != 0 && HasOffset &&
+                           OffsetPastSym;
+                  });
+              if (WouldTraverseLabel) {
+                Size = 0;
+                break;
+              }
+
               while (Size > 0 &&
                      HexagonMCInstrInfo::bundleSize(Inst) < MaxPacketSize) {
                 MCInst *Nop = Context.createMCInst();
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 3e755c25fd91a..b62c15ddb97d3 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -73,8 +73,6 @@
 
 using namespace llvm;
 
-static std::atomic<unsigned> GlobalUniqueCallSite;
-
 static cl::opt<bool> sched4reg(
     "nvptx-sched4reg",
     cl::desc("NVPTX Specific: schedule for register pressue"), cl::init(false));
@@ -500,7 +498,7 @@ static SDValue MaybeBitcast(SelectionDAG &DAG, SDLoc DL, EVT VT,
 // NVPTXTargetLowering Constructor.
 NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
                                          const NVPTXSubtarget &STI)
-    : TargetLowering(TM), nvTM(&TM), STI(STI) {
+    : TargetLowering(TM), nvTM(&TM), STI(STI), GlobalUniqueCallSite(0) {
   // always lower memset, memcpy, and memmove intrinsics to load/store
   // instructions, rather
   // then generating calls to memset, mempcy or memmove.
@@ -1474,7 +1472,7 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   unsigned FirstVAArg = CLI.NumFixedArgs; // position of the first variadic
   unsigned VAOffset = 0;                  // current offset in the param array
 
-  unsigned UniqueCallSite = GlobalUniqueCallSite.fetch_add(1);
+  const unsigned UniqueCallSite = GlobalUniqueCallSite++;
   SDValue TempChain = Chain;
   Chain = DAG.getCALLSEQ_START(Chain, UniqueCallSite, 0, dl);
   SDValue InGlue = Chain.getValue(1);
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index f41c569a65544..ff0241886223b 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -273,6 +273,8 @@ class NVPTXTargetLowering : public TargetLowering {
 
 private:
   const NVPTXSubtarget &STI; // cache the subtarget here
+  mutable unsigned GlobalUniqueCallSite;
+
   SDValue getParamSymbol(SelectionDAG &DAG, int idx, EVT) const;
 
   SDValue LowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index c6be31e20f643..f6150ee9db26e 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -1982,93 +1982,72 @@ class ATOMIC_SHARED_CHK <dag ops, dag frag>
 class ATOMIC_GENERIC_CHK <dag ops, dag frag>
  : PatFrag<ops, frag, AS_match.generic>;
 
-multiclass F_ATOMIC_2_imp<ValueType ptrT, NVPTXRegClass ptrclass,
+multiclass F_ATOMIC_2<
   ValueType regT, NVPTXRegClass regclass,
   string SpaceStr, string TypeStr, string OpcStr, PatFrag IntOp,
-  Operand IMMType, SDNode IMM, list<Predicate> Pred> {
+  Operand IMMType, SDNode IMM, list<Predicate> Pred = []> {
   let mayLoad = 1, mayStore = 1, hasSideEffects = 1 in {
-    def reg : NVPTXInst<(outs regclass:$dst), (ins ptrclass:$addr, regclass:$b),
-      !strconcat("atom", SpaceStr, OpcStr, TypeStr, " \t$dst, [$addr], $b;"),
-      [(set (regT regclass:$dst), (IntOp (ptrT ptrclass:$addr), (regT regclass:$b)))]>,
+    def r : NVPTXInst<(outs regclass:$dst), (ins ADDR:$addr, regclass:$b),
+      "atom" # SpaceStr # OpcStr # TypeStr # " \t$dst, [$addr], $b;",
+      [(set (regT regclass:$dst), (IntOp addr:$addr, (regT regclass:$b)))]>,
     Requires<Pred>;
-    def imm : NVPTXInst<(outs regclass:$dst), (ins ptrclass:$addr, IMMType:$b),
-      !strconcat("atom", SpaceStr, OpcStr, TypeStr, " \t$dst, [$addr], $b;", ""),
-      [(set (regT regclass:$dst), (IntOp (ptrT ptrclass:$addr), IMM:$b))]>,
-    Requires<!if(!or(!eq(TypeStr, ".f16"), !eq(TypeStr, ".bf16")), [Predicate<"false">], Pred)>;
+    if !not(!or(!eq(TypeStr, ".f16"), !eq(TypeStr, ".bf16"))) then
+      def i : NVPTXInst<(outs regclass:$dst), (ins ADDR:$addr, IMMType:$b),
+        "atom" # SpaceStr # OpcStr # TypeStr # " \t$dst, [$addr], $b;",
+        [(set (regT regclass:$dst), (IntOp addr:$addr, IMM:$b))]>,
+      Requires<Pred>;
   }
 }
-multiclass F_ATOMIC_2<ValueType regT, NVPTXRegClass regclass, string SpaceStr, string TypeStr,
-  string OpcStr, PatFrag IntOp, Operand IMMType, SDNode IMM,
-  list<Predicate> Pred = []> {
-  defm p32 : F_ATOMIC_2_imp<i32, Int32Regs, regT, regclass, SpaceStr, TypeStr, OpcStr,
-    IntOp, IMMType, IMM, Pred>;
-  defm p64 : F_ATOMIC_2_imp<i64, Int64Regs, regT, regclass, SpaceStr, TypeStr, OpcStr,
-    IntOp, IMMType, IMM, Pred>;
-}
 
 // has 2 operands, neg the second one
-multiclass F_ATOMIC_2_NEG_imp<ValueType ptrT, NVPTXRegClass ptrclass,
+multiclass F_ATOMIC_2_NEG<
   ValueType regT, NVPTXRegClass regclass,
   string SpaceStr, string TypeStr, string OpcStr, PatFrag IntOp,
-  list<Predicate> Pred> {
+  list<Predicate> Pred = []> {
   let mayLoad = 1, mayStore = 1, hasSideEffects = 1 in {
-    def reg : NVPTXInst<(outs regclass:$dst), (ins ptrclass:$addr, regclass:$b),
+    def reg : NVPTXInst<(outs regclass:$dst), (ins ADDR:$addr, regclass:$b),
       !strconcat(
         "{{ \n\t",
         ".reg \t.s", TypeStr, " temp; \n\t",
         "neg.s", TypeStr, " \ttemp, $b; \n\t",
         "atom", SpaceStr, OpcStr, ".u", TypeStr, " \t$dst, [$addr], temp; \n\t",
         "}}"),
-      [(set (regT regclass:$dst), (IntOp (ptrT ptrclass:$addr), (regT regclass:$b)))]>,
+      [(set (regT regclass:$dst), (IntOp addr:$addr, (regT regclass:$b)))]>,
     Requires<Pred>;
   }
 }
-multiclass F_ATOMIC_2_NEG<ValueType regT, NVPTXRegClass regclass, string SpaceStr,
-  string TypeStr, string OpcStr, PatFrag IntOp, list<Predicate> Pred = []> {
- defm p32: F_ATOMIC_2_NEG_imp<i32, Int32Regs, regT, regclass, SpaceStr, TypeStr, OpcStr,
-   IntOp, Pred> ;
- defm p64: F_ATOMIC_2_NEG_imp<i64, Int64Regs, regT, regclass, SpaceStr, TypeStr, OpcStr,
-   IntOp, Pred> ;
-}
 
 // has 3 operands
-multiclass F_ATOMIC_3_imp<ValueType ptrT, NVPTXRegClass ptrclass,
+multiclass F_ATOMIC_3<
   ValueType regT, NVPTXRegClass regclass, string SemStr,
   string SpaceStr, string TypeStr, string OpcStr, PatFrag IntOp,
-  Operand IMMType, list<Predicate> Pred> {
+  Operand IMMType, list<Predicate> Pred = []> {
   let mayLoad = 1, mayStore = 1, hasSideEffects = 1 in {
-    def reg : NVPTXInst<(outs regclass:$dst),
-      (ins ptrclass:$addr, regclass:$b, regclass:$c),
-      !strconcat("atom", SemStr, SpaceStr, OpcStr, TypeStr, " \t$dst, [$addr], $b, $c;"),
-      [(set (regT regclass:$dst), (IntOp (ptrT ptrclass:$addr), (regT regclass:$b), (regT regclass:$c)))]>,
+    def rr : NVPTXInst<(outs regclass:$dst),
+      (ins ADDR:$addr, regclass:$b, regclass:$c),
+      "atom" # SemStr # SpaceStr # OpcStr # TypeStr # " \t$dst, [$addr], $b, $c;",
+      [(set (regT regclass:$dst), (IntOp addr:$addr, regT:$b, regT:$c))]>,
     Requires<Pred>;
 
-    def imm1 : NVPTXInst<(outs regclass:$dst),
-      (ins ptrclass:$addr, IMMType:$b, regclass:$c),
-      !strconcat("atom", SemStr, SpaceStr, OpcStr, TypeStr, " \t$dst, [$addr], $b, $c;"),
-      [(set (regT regclass:$dst), (IntOp (ptrT ptrclass:$addr), imm:$b, (regT regclass:$c)))]>,
+    def ir : NVPTXInst<(outs regclass:$dst),
+      (ins ADDR:$addr, IMMType:$b, regclass:$c),
+      "atom" # SemStr # SpaceStr # OpcStr # TypeStr # " \t$dst, [$addr], $b, $c;",
+      [(set (regT regclass:$dst), (IntOp addr:$addr, imm:$b, regT:$c))]>,
     Requires<Pred>;
 
-    def imm2 : NVPTXInst<(outs regclass:$dst),
-      (ins ptrclass:$addr, regclass:$b, IMMType:$c),
-      !strconcat("atom", SemStr, SpaceStr, OpcStr, TypeStr, " \t$dst, [$addr], $b, $c;", ""),
-      [(set (regT regclass:$dst), (IntOp (ptrT ptrclass:$addr), (regT regclass:$b), imm:$c))]>,
+    def ri : NVPTXInst<(outs regclass:$dst),
+      (ins ADDR:$addr, regclass:$b, IMMType:$c),
+      "atom" # SemStr # SpaceStr # OpcStr # TypeStr # " \t$dst, [$addr], $b, $c;",
+      [(set (regT regclass:$dst), (IntOp addr:$addr, regT:$b, imm:$c))]>,
     Requires<Pred>;
 
-    def imm3 : NVPTXInst<(outs regclass:$dst),
-      (ins ptrclass:$addr, IMMType:$b, IMMType:$c),
-      !strconcat("atom", SemStr, SpaceStr, OpcStr, TypeStr, " \t$dst, [$addr], $b, $c;"),
-      [(set (regT regclass:$dst), (IntOp (ptrT ptrclass:$addr), imm:$b, imm:$c))]>,
+    def ii : NVPTXInst<(outs regclass:$dst),
+      (ins ADDR:$addr, IMMType:$b, IMMType:$c),
+      "atom" # SemStr # SpaceStr # OpcStr # TypeStr # " \t$dst, [$addr], $b, $c;",
+      [(set (regT regclass:$dst), (IntOp addr:$addr, imm:$b, imm:$c))]>,
     Requires<Pred>;
   }
 }
-multiclass F_ATOMIC_3<ValueType regT, NVPTXRegClass regclass, string SemStr, string SpaceStr,
-  string TypeStr, string OpcStr, PatFrag IntOp, Operand IMMType, list<Predicate> Pred = []> {
-  defm p32 : F_ATOMIC_3_imp<i32, Int32Regs, regT, regclass, SemStr, SpaceStr, TypeStr,
-    OpcStr, IntOp, IMMType, Pred>;
-  defm p64 : F_ATOMIC_3_imp<i64, Int64Regs, regT, regclass, SemStr, SpaceStr, TypeStr,
-    OpcStr, IntOp, IMMType, Pred>;
-}
 
 // atom_add
 
@@ -2540,27 +2519,15 @@ multiclass ATOM2P_impl<string AsmStr,  Intrinsic Intr,
                        list<Predicate> Preds> {
   let AddedComplexity = 1 in {
     def : ATOM23_impl<AsmStr, regT, regclass, Preds,
-                      (ins Int16Regs:$src, regclass:$b),
-                      (Intr i16:$src, regT:$b)>;
-    def : ATOM23_impl<AsmStr, regT, regclass, Preds,
-                      (ins Int32Regs:$src, regclass:$b),
-                      (Intr i32:$src, regT:$b)>;
-    def : ATOM23_impl<AsmStr, regT, regclass, Preds,
-                      (ins Int64Regs:$src, regclass:$b),
-                      (Intr i64:$src, regT:$b)>;
+                      (ins ADDR:$src, regclass:$b),
+                      (Intr addr:$src, regT:$b)>;
   }
   // tablegen can't infer argument types from Intrinsic (though it can
   // from Instruction) so we have to enforce specific type on
   // immediates via explicit cast to ImmTy.
   def : ATOM23_impl<AsmStr, regT, regclass, Preds,
-                    (ins Int16Regs:$src, ImmType:$b),
-                    (Intr i16:$src, (ImmTy Imm:$b))>;
-  def : ATOM23_impl<AsmStr, regT, regclass, Preds,
-                    (ins Int32Regs:$src, ImmType:$b),
-                    (Intr i32:$src, (ImmTy Imm:$b))>;
-  def : ATOM23_impl<AsmStr, regT, regclass, Preds,
-                    (ins Int64Regs:$src, ImmType:$b),
-                    (Intr i64:$src, (ImmTy Imm:$b))>;
+                    (ins ADDR:$src, ImmType:$b),
+                    (Intr addr:$src, (ImmTy Imm:$b))>;
 }
 
 multiclass ATOM3P_impl<string AsmStr,  Intrinsic Intr,
@@ -2570,32 +2537,20 @@ multiclass ATOM3P_impl<string AsmStr,  Intrinsic Intr,
   // Variants for register/immediate permutations of $b and $c
   let AddedComplexity = 2 in {
     def : ATOM23_impl<AsmStr, regT, regclass, Preds,
-                      (ins Int32Regs:$src, regclass:$b, regclass:$c),
-                      (Intr i32:$src, regT:$b, regT:$c)>;
-    def : ATOM23_impl<AsmStr, regT, regclass, Preds,
-                      (ins Int64Regs:$src, regclass:$b, regclass:$c),
-                      (Intr i64:$src, regT:$b, regT:$c)>;
+                      (ins ADDR:$src, regclass:$b, regclass:$c),
+                      (Intr addr:$src, regT:$b, regT:$c)>;
   }
   let AddedComplexity = 1 in {
     def : ATOM23_impl<AsmStr, regT, regclass, Preds,
-                      (ins Int32Regs:$src, ImmType:$b, regclass:$c),
-                      (Intr i32:$src, (ImmTy Imm:$b), regT:$c)>;
-    def : ATOM23_impl<AsmStr, regT, regclass, Preds,
-                      (ins Int64Regs:$src, ImmType:$b, regclass:$c),
-                      (Intr i64:$src, (ImmTy Imm:$b), regT:$c)>;
+                      (ins ADDR:$src, ImmType:$b, regclass:$c),
+                      (Intr addr:$src, (ImmTy Imm:$b), regT:$c)>;
     def : ATOM23_impl<AsmStr, regT, regclass, Preds,
-                      (ins Int32Regs:$src, regclass:$b, ImmType:$c),
-                      (Intr i32:$src, regT:$b, (ImmTy Imm:$c))>;
-    def : ATOM23_impl<AsmStr, regT, regclass, Preds,
-                      (ins Int64Regs:$src, regclass:$b, ImmType:$c),
-                      (Intr i64:$src, regT:$b, (ImmTy Imm:$c))>;
+                      (ins ADDR:$src, regclass:$b, ImmType:$c),
+                      (Intr addr:$src, regT:$b, (ImmTy Imm:$c))>;
   }
   def : ATOM23_impl<AsmStr, regT, regclass, Preds,
-                    (ins Int32Regs:$src, ImmType:$b, ImmType:$c),
-                    (Intr i32:$src, (ImmTy Imm:$b), (ImmTy Imm:$c))>;
-  def : ATOM23_impl<AsmStr, regT, regclass, Preds,
-                    (ins Int64Regs:$src, ImmType:$b, ImmType:$c),
-                    (Intr i64:$src, (ImmTy Imm:$b), (ImmTy Imm:$c))>;
+                    (ins ADDR:$src, ImmType:$b, ImmType:$c),
+                    (Intr addr:$src, (ImmTy Imm:$b), (ImmTy Imm:$c))>;
 }
 
 // Constructs intrinsic name and instruction asm strings.
@@ -7777,3 +7732,17 @@ foreach shape = ["16x64b", "16x128b", "16x256b", "32x32b", "16x32bx2"] in {
 }
 
 } // isConvergent
+
+// Bulk store instructions
+                            
+def INT_NVVM_ST_BULK_GENERIC :
+  NVPTXInst<(outs), (ins ADDR:$dest_addr, Int64Regs:$size),
+            "st.bulk [$dest_addr], $size, 0;",
+            [(int_nvvm_st_bulk addr:$dest_addr, i64:$size, (i64 0))]>,
+            Requires<[hasSM<100>, hasPTX<86>]>;
+
+def INT_NVVM_ST_BULK_SHARED_CTA:
+  NVPTXInst<(outs), (ins ADDR:$dest_addr, Int64Regs:$size),
+            "st.bulk.shared::cta [$dest_addr], $size, 0;",
+            [(int_nvvm_st_bulk_shared_cta addr:$dest_addr, i64:$size, (i64 0))]>,
+            Requires<[hasSM<100>, hasPTX<86>]>;
diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index cd0704cad2462..be0df763029a5 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -109,10 +109,10 @@ struct DenseMapInfo<std::pair<const MCSymbol *, MCSymbolRefExpr::VariantKind>> {
   using TOCKey = std::pair<const MCSymbol *, MCSymbolRefExpr::VariantKind>;
 
   static inline TOCKey getEmptyKey() {
-    return {nullptr, MCSymbolRefExpr::VariantKind::VK_None};
+    return {nullptr, MCSymbolRefExpr::VK_None};
   }
   static inline TOCKey getTombstoneKey() {
-    return {nullptr, MCSymbolRefExpr::VariantKind::VK_Invalid};
+    return {(const MCSymbol *)1, MCSymbolRefExpr::VK_None};
   }
   static unsigned getHashValue(const TOCKey &PairVal) {
     return detail::combineHashValue(
diff --git a/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp b/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp
index 0930206c0ae9b..a86da814f10e9 100644
--- a/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp
+++ b/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp
@@ -241,9 +241,11 @@ class PPCBoolRetToInt : public FunctionPass {
       ++NumBoolCallPromotion;
     ++NumBoolToIntPromotion;
 
-    for (Value *V : Defs)
-      if (!BoolToIntMap.count(V))
-        BoolToIntMap[V] = translate(V);
+    for (Value *V : Defs) {
+      auto [It, Inserted] = BoolToIntMap.try_emplace(V);
+      if (Inserted)
+        It->second = translate(V);
+    }
 
     // Replace the operands of the translated instructions. They were set to
     // zero in the translate function.
diff --git a/llvm/lib/Target/RISCV/CMakeLists.txt b/llvm/lib/Target/RISCV/CMakeLists.txt
index 5d1ea50eba494..e8d00f4df7c86 100644
--- a/llvm/lib/Target/RISCV/CMakeLists.txt
+++ b/llvm/lib/Target/RISCV/CMakeLists.txt
@@ -48,6 +48,7 @@ add_llvm_target(RISCVCodeGen
   RISCVISelDAGToDAG.cpp
   RISCVISelLowering.cpp
   RISCVLandingPadSetup.cpp
+  RISCVLoadStoreOptimizer.cpp
   RISCVMachineFunctionInfo.cpp
   RISCVMergeBaseOffset.cpp
   RISCVOptWInstrs.cpp
diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
index 6dfebc1989e92..de742ac428de2 100644
--- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
+++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
@@ -644,6 +644,12 @@ void RISCVDisassembler::addSPOperands(MCInst &MI) const {
 #define TRY_TO_DECODE_FEATURE_ANY(FEATURES, DECODER_TABLE, DESC)               \
   TRY_TO_DECODE((STI.getFeatureBits() & (FEATURES)).any(), DECODER_TABLE, DESC)
 
+static constexpr FeatureBitset XCVFeatureGroup = {
+    RISCV::FeatureVendorXCVbitmanip, RISCV::FeatureVendorXCVelw,
+    RISCV::FeatureVendorXCVmac,      RISCV::FeatureVendorXCVmem,
+    RISCV::FeatureVendorXCValu,      RISCV::FeatureVendorXCVsimd,
+    RISCV::FeatureVendorXCVbi};
+
 static constexpr FeatureBitset XRivosFeatureGroup = {
     RISCV::FeatureVendorXRivosVisni,
     RISCV::FeatureVendorXRivosVizip,
@@ -667,6 +673,14 @@ static constexpr FeatureBitset XSfSystemGroup = {
     RISCV::FeatureVendorXSiFivecflushdlone,
 };
 
+static constexpr FeatureBitset XTHeadGroup = {
+    RISCV::FeatureVendorXTHeadBa,      RISCV::FeatureVendorXTHeadBb,
+    RISCV::FeatureVendorXTHeadBs,      RISCV::FeatureVendorXTHeadCondMov,
+    RISCV::FeatureVendorXTHeadCmo,     RISCV::FeatureVendorXTHeadFMemIdx,
+    RISCV::FeatureVendorXTHeadMac,     RISCV::FeatureVendorXTHeadMemIdx,
+    RISCV::FeatureVendorXTHeadMemPair, RISCV::FeatureVendorXTHeadSync,
+    RISCV::FeatureVendorXTHeadVdot};
+
 DecodeStatus RISCVDisassembler::getInstruction32(MCInst &MI, uint64_t &Size,
                                                  ArrayRef<uint8_t> Bytes,
                                                  uint64_t Address,
@@ -681,28 +695,8 @@ DecodeStatus RISCVDisassembler::getInstruction32(MCInst &MI, uint64_t &Size,
 
   TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXVentanaCondOps,
                         DecoderTableXVentana32, "XVentanaCondOps");
-  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXTHeadBa, DecoderTableXTHeadBa32,
-                        "XTHeadBa");
-  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXTHeadBb, DecoderTableXTHeadBb32,
-                        "XTHeadBb");
-  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXTHeadBs, DecoderTableXTHeadBs32,
-                        "XTHeadBs");
-  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXTHeadCondMov,
-                        DecoderTableXTHeadCondMov32, "XTHeadCondMov");
-  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXTHeadCmo, DecoderTableXTHeadCmo32,
-                        "XTHeadCmo");
-  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXTHeadFMemIdx,
-                        DecoderTableXTHeadFMemIdx32, "XTHeadFMemIdx");
-  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXTHeadMac, DecoderTableXTHeadMac32,
-                        "XTHeadMac");
-  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXTHeadMemIdx,
-                        DecoderTableXTHeadMemIdx32, "XTHeadMemIdx");
-  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXTHeadMemPair,
-                        DecoderTableXTHeadMemPair32, "XTHeadMemPair");
-  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXTHeadSync,
-                        DecoderTableXTHeadSync32, "XTHeadSync");
-  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXTHeadVdot,
-                        DecoderTableXTHeadVdot32, "XTHeadVdot");
+  TRY_TO_DECODE_FEATURE_ANY(XTHeadGroup, DecoderTableXTHead32,
+                            "T-Head extensions");
   TRY_TO_DECODE_FEATURE_ANY(XSfVectorGroup, DecoderTableXSfvector32,
                             "SiFive vector extensions");
   TRY_TO_DECODE_FEATURE_ANY(XSfSystemGroup, DecoderTableXSfsystem32,
@@ -713,20 +707,8 @@ DecodeStatus RISCVDisassembler::getInstruction32(MCInst &MI, uint64_t &Size,
                         "MIPS mips.lsp");
   TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXMIPSCMove,
                         DecoderTableXmipscmove32, "MIPS mips.ccmov");
-  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXCVbitmanip,
-                        DecoderTableXCVbitmanip32, "CORE-V Bit Manipulation");
-  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXCVelw, DecoderTableXCVelw32,
-                        "CORE-V Event load");
-  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXCVmac, DecoderTableXCVmac32,
-                        "CORE-V MAC");
-  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXCVmem, DecoderTableXCVmem32,
-                        "CORE-V MEM");
-  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXCValu, DecoderTableXCValu32,
-                        "CORE-V ALU");
-  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXCVsimd, DecoderTableXCVsimd32,
-                        "CORE-V SIMD extensions");
-  TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXCVbi, DecoderTableXCVbi32,
-                        "CORE-V Immediate Branching");
+  TRY_TO_DECODE_FEATURE_ANY(XCVFeatureGroup, DecoderTableXCV32,
+                            "CORE-V extensions");
   TRY_TO_DECODE_FEATURE_ANY(XqciFeatureGroup, DecoderTableXqci32,
                             "Qualcomm uC Extensions");
 
diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index 35db027509d94..21119f624339c 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -1281,7 +1281,8 @@ def HasVendorXMIPSLSP
 
 def FeatureVendorXwchc
     : RISCVExtension<2, 2,
-                     "WCH/QingKe additional compressed opcodes">;
+                     "WCH/QingKe additional compressed opcodes",
+                     [FeatureStdExtZca]>;
 def HasVendorXwchc
     : Predicate<"Subtarget->hasVendorXwchc()">,
       AssemblerPredicate<(all_of FeatureVendorXwchc),
@@ -1351,7 +1352,7 @@ def HasVendorXqcicm
                          "'Xqcicm' (Qualcomm uC Conditional Move Extension)">;
 
 def FeatureVendorXqciint
-    : RISCVExperimentalExtension<0, 2, "Qualcomm uC Interrupts Extension",
+    : RISCVExperimentalExtension<0, 4, "Qualcomm uC Interrupts Extension",
                                  [FeatureStdExtZca]>;
 def HasVendorXqciint
     : Predicate<"Subtarget->hasVendorXqciint()">,
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index 6b61a236cf570..e1314d4fee8a0 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -1008,10 +1008,9 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
     // The frame pointer does need to be reserved from register allocation.
     assert(MF.getRegInfo().isReserved(FPReg) && "FP not reserved");
 
-    // Xqccmp with hasFP will update FP using `qc.cm.pushfp`, so we don't need
-    // to update it again, but we do need to emit the `.cfi_def_cfa` below.
-    if (RVFI->getPushPopKind(MF) !=
-        RISCVMachineFunctionInfo::PushPopKind::VendorXqccmp) {
+    // Some stack management variants automatically keep FP updated, so we don't
+    // need an instruction to do so.
+    if (!RVFI->hasImplicitFPUpdates(MF)) {
       RI->adjustReg(
           MBB, MBBI, DL, FPReg, SPReg,
           StackOffset::getFixed(RealStackSize - RVFI->getVarArgsSaveSize()),
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index f3cce950ed7b5..8aa684c56bde0 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -3573,6 +3573,13 @@ bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal) {
       /*Decrement=*/true);
 }
 
+bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NoDec(SDValue N, SDValue &SplatVal) {
+  return selectVSplatImmHelper(
+      N, SplatVal, *CurDAG, *Subtarget,
+      [](int64_t Imm) { return (isInt<5>(Imm) && Imm != -16) || Imm == 16; },
+      /*Decrement=*/false);
+}
+
 bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NonZero(SDValue N,
                                                       SDValue &SplatVal) {
   return selectVSplatImmHelper(
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
index 5048a80fdd18f..db09ad146b655 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
@@ -137,6 +137,7 @@ class RISCVDAGToDAGISel : public SelectionDAGISel {
     return selectVSplatUimm(N, Bits, Val);
   }
   bool selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal);
+  bool selectVSplatSimm5Plus1NoDec(SDValue N, SDValue &SplatVal);
   bool selectVSplatSimm5Plus1NonZero(SDValue N, SDValue &SplatVal);
   // Matches the splat of a value which can be extended or truncated, such that
   // only the bottom 8 bits are preserved.
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 36d113a63cd1d..27a4bbce1f5fc 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -4361,27 +4361,19 @@ static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
     if ((LoC >> 31) == HiC)
       return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
 
-    // If vl is equal to VLMAX or fits in 4 bits and Hi constant is equal to Lo,
-    // we could use vmv.v.x whose EEW = 32 to lower it. This allows us to use
-    // vlmax vsetvli or vsetivli to change the VL.
-    // FIXME: Support larger constants?
-    // FIXME: Support non-constant VLs by saturating?
+    // Use vmv.v.x with EEW=32.  Use either a vsetivli or vsetvli to change
+    // VL.  This can temporarily increase VL if VL less than VLMAX.
     if (LoC == HiC) {
       SDValue NewVL;
-      if (isAllOnesConstant(VL) ||
-          (isa<RegisterSDNode>(VL) &&
-           cast<RegisterSDNode>(VL)->getReg() == RISCV::X0))
-        NewVL = DAG.getRegister(RISCV::X0, MVT::i32);
-      else if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->getAsZExtVal()))
+      if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->getAsZExtVal()))
         NewVL = DAG.getNode(ISD::ADD, DL, VL.getValueType(), VL, VL);
-
-      if (NewVL) {
-        MVT InterVT =
-            MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
-        auto InterVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterVT,
-                                    DAG.getUNDEF(InterVT), Lo, NewVL);
-        return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
-      }
+      else
+        NewVL = DAG.getRegister(RISCV::X0, MVT::i32);
+      MVT InterVT =
+          MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
+      auto InterVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterVT,
+                                  DAG.getUNDEF(InterVT), Lo, NewVL);
+      return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
     }
   }
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index f767223f96cd2..2fdf6bd36e88f 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -2767,6 +2767,42 @@ MachineInstr *RISCVInstrInfo::emitLdStWithAddr(MachineInstr &MemI,
       .setMIFlags(MemI.getFlags());
 }
 
+// TODO: At the moment, MIPS introduced paring of instructions operating with
+// word or double word. This should be extended with more instructions when more
+// vendors support load/store pairing.
+bool RISCVInstrInfo::isPairableLdStInstOpc(unsigned Opc) {
+  switch (Opc) {
+  default:
+    return false;
+  case RISCV::SW:
+  case RISCV::SD:
+  case RISCV::LD:
+  case RISCV::LW:
+    return true;
+  }
+}
+
+bool RISCVInstrInfo::isLdStSafeToPair(const MachineInstr &LdSt,
+                                      const TargetRegisterInfo *TRI) {
+  // If this is a volatile load/store, don't mess with it.
+  if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)
+    return false;
+
+  if (LdSt.getOperand(1).isFI())
+    return true;
+
+  assert(LdSt.getOperand(1).isReg() && "Expected a reg operand.");
+  // Can't cluster if the instruction modifies the base register
+  // or it is update form. e.g. ld x5,8(x5)
+  if (LdSt.modifiesRegister(LdSt.getOperand(1).getReg(), TRI))
+    return false;
+
+  if (!LdSt.getOperand(2).isImm())
+    return false;
+
+  return true;
+}
+
 bool RISCVInstrInfo::getMemOperandsWithOffsetWidth(
     const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
     int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index 329ed22c6efe5..656cb38e11297 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -301,6 +301,12 @@ class RISCVInstrInfo : public RISCVGenInstrInfo {
 
   bool isHighLatencyDef(int Opc) const override;
 
+  /// Return true if pairing the given load or store may be paired with another.
+  static bool isPairableLdStInstOpc(unsigned Opc);
+
+  static bool isLdStSafeToPair(const MachineInstr &LdSt,
+                               const TargetRegisterInfo *TRI);
+
 protected:
   const RISCVSubtarget &STI;
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index 5ce826b136832..aab368680e814 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -523,7 +523,6 @@ include "RISCVInstrFormats.td"
 // Instruction Class Templates
 //===----------------------------------------------------------------------===//
 
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 class BranchCC_rri<bits<3> funct3, string opcodestr>
     : RVInstB<funct3, OPC_BRANCH, (outs),
               (ins GPR:$rs1, GPR:$rs2, simm13_lsb0:$imm12),
@@ -531,6 +530,9 @@ class BranchCC_rri<bits<3> funct3, string opcodestr>
       Sched<[WriteJmp, ReadJmp, ReadJmp]> {
   let isBranch = 1;
   let isTerminator = 1;
+  let hasSideEffects = 0;
+  let mayLoad = 0;
+  let mayStore = 0;
 }
 
 let hasSideEffects = 0, mayLoad = 1, mayStore = 0 in {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index 2bd61883760e5..b2c5261ae6c2d 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -877,9 +877,9 @@ foreach mti = AllMasks in
 // 11.1. Vector Single-Width Integer Add and Subtract
 defm : VPatBinarySDNode_VV_VX_VI<add, "PseudoVADD">;
 defm : VPatBinarySDNode_VV_VX<sub, "PseudoVSUB">;
-// Handle VRSUB specially since it's the only integer binary op with reversed
-// pattern operands
 foreach vti = AllIntegerVectors in {
+  // Handle VRSUB specially since it's the only integer binary op with reversed
+  // pattern operands
   // FIXME: The AddedComplexity here is covering up a missing matcher for
   // widening vwsub.vx which can recognize a extended folded into the
   // scalar of the splat.
@@ -896,6 +896,15 @@ foreach vti = AllIntegerVectors in {
                    (vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs1,
                    simm5:$rs2, vti.AVL, vti.Log2SEW, TA_MA)>;
   }
+
+  let Predicates = GetVTypePredicates<vti>.Predicates in {
+    // Match VSUB with a small immediate to vadd.vi by negating the immediate.
+    def : Pat<(sub (vti.Vector vti.RegClass:$rs1),
+                   (vti.Vector (SplatPat_simm5_plus1_nodec simm5_plus1:$rs2))),
+              (!cast<Instruction>("PseudoVADD_VI_"#vti.LMul.MX)
+                    (vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs1,
+                    (NegImm simm5_plus1:$rs2), vti.AVL, vti.Log2SEW, TA_MA)>;
+  }
 }
 
 // 11.2. Vector Widening Integer Add and Subtract
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index 43cfc9d1e77ca..5d98ffedcbb9a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -598,6 +598,8 @@ def SplatPat_uimm5 : ComplexPattern<vAny, 1, "selectVSplatUimmBits<5>", [], [],
 def SplatPat_uimm6 : ComplexPattern<vAny, 1, "selectVSplatUimmBits<6>", [], [], 3>;
 def SplatPat_simm5_plus1
     : ComplexPattern<vAny, 1, "selectVSplatSimm5Plus1", [], [], 3>;
+def SplatPat_simm5_plus1_nodec
+    : ComplexPattern<vAny, 1, "selectVSplatSimm5Plus1NoDec", [], [], 3>;
 def SplatPat_simm5_plus1_nonzero
     : ComplexPattern<vAny, 1, "selectVSplatSimm5Plus1NonZero", [], [], 3>;
 
@@ -1992,10 +1994,10 @@ multiclass VPatAVGADDVL_VV_VX_RM<SDNode vop, int vxrm, string suffix = ""> {
 // 11.1. Vector Single-Width Integer Add and Subtract
 defm : VPatBinaryVL_VV_VX_VI<riscv_add_vl, "PseudoVADD">;
 defm : VPatBinaryVL_VV_VX<riscv_sub_vl, "PseudoVSUB">;
-// Handle VRSUB specially since it's the only integer binary op with reversed
-// pattern operands
 foreach vti = AllIntegerVectors in {
   let Predicates = GetVTypePredicates<vti>.Predicates in {
+    // Handle VRSUB specially since it's the only integer binary op with
+    // reversed pattern operands
     def : Pat<(riscv_sub_vl (vti.Vector (SplatPat (XLenVT GPR:$rs2))),
                             (vti.Vector vti.RegClass:$rs1),
                             vti.RegClass:$passthru, (vti.Mask VMV0:$vm), VLOpFrag),
@@ -2008,6 +2010,15 @@ foreach vti = AllIntegerVectors in {
               (!cast<Instruction>("PseudoVRSUB_VI_"# vti.LMul.MX#"_MASK")
                    vti.RegClass:$passthru, vti.RegClass:$rs1, simm5:$rs2,
                    (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
+
+    // Match VSUB with a small immediate to vadd.vi by negating the immediate.
+    def : Pat<(riscv_sub_vl (vti.Vector vti.RegClass:$rs1),
+                            (vti.Vector (SplatPat_simm5_plus1_nodec simm5_plus1:$rs2)),
+                            vti.RegClass:$passthru, (vti.Mask VMV0:$vm), VLOpFrag),
+              (!cast<Instruction>("PseudoVADD_VI_"#vti.LMul.MX#"_MASK")
+                     vti.RegClass:$passthru, vti.RegClass:$rs1,
+                     (NegImm simm5_plus1:$rs2), (vti.Mask VMV0:$vm),
+                     GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>;
   }
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
index bade4863ad348..344b8e13ba2ae 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-let DecoderNamespace = "XCVbitmanip" in {
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
   class CVInstBitManipRII<bits<2> funct2, bits<3> funct3, dag outs, dag ins,
                       string opcodestr, string argstr>
       : RVInstIBase<funct3, OPC_CUSTOM_2, outs, ins, opcodestr, argstr> {
@@ -36,10 +36,9 @@ let DecoderNamespace = "XCVbitmanip" in {
                 (ins GPR:$rs1), opcodestr, "$rd, $rs1"> {
     let rs2 = 0b00000;
   }
-}
+} // hasSideEffects = 0, mayLoad = 0, mayStore = 0
 
-let Predicates = [HasVendorXCVbitmanip, IsRV32],
-    hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
+let Predicates = [HasVendorXCVbitmanip, IsRV32], DecoderNamespace = "XCV" in {
   def CV_EXTRACT : CVBitManipRII<0b00, 0b000, "cv.extract">;
   def CV_EXTRACTU : CVBitManipRII<0b01, 0b000, "cv.extractu">;
 
@@ -54,6 +53,7 @@ let Predicates = [HasVendorXCVbitmanip, IsRV32],
     def CV_INSERT : CVInstBitManipRII<0b10, 0b000, (outs GPR:$rd_wb),
                              (ins GPR:$rd, GPR:$rs1, uimm5:$is3, uimm5:$is2),
                              "cv.insert", "$rd, $rs1, $is3, $is2">;
+    let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
     def CV_INSERTR : RVInstR<0b0011010, 0b011, OPC_CUSTOM_1, (outs GPR:$rd_wb),
                              (ins GPR:$rd, GPR:$rs1, GPR:$rs2),
                              "cv.insertr", "$rd, $rs1, $rs2">;
@@ -73,7 +73,10 @@ class CVInstMac<bits<7> funct7, bits<3> funct3, string opcodestr>
     : RVInstR<funct7, funct3, OPC_CUSTOM_1,
               (outs GPR:$rd_wb), (ins GPR:$rd, GPR:$rs1, GPR:$rs2),
               opcodestr, "$rd, $rs1, $rs2"> {
-  let DecoderNamespace = "XCVmac";
+  let Constraints = "$rd = $rd_wb";
+  let hasSideEffects = 0;
+  let mayLoad = 0;
+  let mayStore = 0;
 }
 
 class CVInstMacMulN<bits<2> funct2, bits<3> funct3, dag outs, dag ins,
@@ -84,19 +87,23 @@ class CVInstMacMulN<bits<2> funct2, bits<3> funct3, dag outs, dag ins,
 
   let Inst{31-30} = funct2;
   let Inst{29-25} = imm5;
-  let DecoderNamespace = "XCVmac";
+
+  let hasSideEffects = 0;
+  let mayLoad = 0;
+  let mayStore = 0;
 }
 
 class CVInstMacN<bits<2> funct2, bits<3> funct3, string opcodestr>
     : CVInstMacMulN<funct2, funct3, (outs GPR:$rd_wb),
-                    (ins GPR:$rd, GPR:$rs1, GPR:$rs2, uimm5:$imm5), opcodestr>;
+                    (ins GPR:$rd, GPR:$rs1, GPR:$rs2, uimm5:$imm5), opcodestr> {
+  let Constraints = "$rd = $rd_wb";
+}
 
 class CVInstMulN<bits<2> funct2, bits<3> funct3, string opcodestr>
     : CVInstMacMulN<funct2, funct3, (outs GPR:$rd),
                     (ins GPR:$rs1, GPR:$rs2, uimm5:$imm5), opcodestr>;
 
-let Predicates = [HasVendorXCVmac, IsRV32], hasSideEffects = 0, mayLoad = 0,
-    mayStore = 0, Constraints = "$rd = $rd_wb" in {
+let Predicates = [HasVendorXCVmac, IsRV32], DecoderNamespace = "XCV" in {
   // 32x32 bit macs
   def CV_MAC      : CVInstMac<0b1001000, 0b011, "cv.mac">,
                     Sched<[]>;
@@ -122,9 +129,7 @@ let Predicates = [HasVendorXCVmac, IsRV32], hasSideEffects = 0, mayLoad = 0,
                     Sched<[]>;
   def CV_MACHHURN : CVInstMacN<0b11, 0b111, "cv.machhurn">,
                     Sched<[]>;
-} // Predicates = [HasVendorXCVmac, IsRV32], hasSideEffects = 0, mayLoad = 0...
 
-let Predicates = [HasVendorXCVmac, IsRV32], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
   // Signed 16x16 bit muls with imm
   def CV_MULSN    : CVInstMulN<0b00, 0b100, "cv.mulsn">,
                     Sched<[]>;
@@ -144,7 +149,7 @@ let Predicates = [HasVendorXCVmac, IsRV32], hasSideEffects = 0, mayLoad = 0, may
                     Sched<[]>;
   def CV_MULHHURN : CVInstMulN<0b11, 0b101, "cv.mulhhurn">,
                     Sched<[]>;
-} // Predicates = [HasVendorXCVmac, IsRV32], hasSideEffects = 0, mayLoad = 0...
+} // Predicates = [HasVendorXCVmac, IsRV32], DecoderNamespace = "XCV"
 
 let Predicates = [HasVendorXCVmac, IsRV32] in {
   // Xcvmac Pseudo Instructions
@@ -161,7 +166,7 @@ let Predicates = [HasVendorXCVmac, IsRV32] in {
                   (CV_MULHHUN GPR:$rd1, GPR:$rs1, GPR:$rs2, 0)>;
 } // Predicates = [HasVendorXCVmac, IsRV32]
 
-let DecoderNamespace = "XCValu" in {
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
   class CVInstAluRRI<bits<2> funct2, bits<3> funct3, string opcodestr>
       : RVInstRBase<funct3, OPC_CUSTOM_2, (outs GPR:$rd),
                     (ins GPR:$rs1, GPR:$rs2, uimm5:$imm5), opcodestr,
@@ -178,7 +183,9 @@ let DecoderNamespace = "XCValu" in {
 
   class CVInstAluRRNR<bits<7> funct7, bits<3> funct3, string opcodestr>
     : RVInstR<funct7, funct3, OPC_CUSTOM_1, (outs GPR:$rd_wb),
-              (ins GPR:$rd, GPR:$rs1, GPR:$rs2), opcodestr, "$rd, $rs1, $rs2">;
+              (ins GPR:$rd, GPR:$rs1, GPR:$rs2), opcodestr, "$rd, $rs1, $rs2"> {
+    let Constraints = "$rd = $rd_wb";
+  }
 
   class CVInstAluRI<bits<7> funct7, bits<3> funct3, string opcodestr>
       : RVInstIBase<funct3, OPC_CUSTOM_1, (outs GPR:$rd),
@@ -195,11 +202,9 @@ let DecoderNamespace = "XCValu" in {
               opcodestr, "$rd, $rs1"> {
      let rs2 = 0b00000;
   }
+} // hasSideEffects = 0, mayLoad = 0, mayStore = 0
 
-} // DecoderNamespace = "XCValu"
-
-let Predicates = [HasVendorXCValu, IsRV32],
-  hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
+let Predicates = [HasVendorXCValu, IsRV32], DecoderNamespace = "XCV" in {
   // General ALU Operations
   def CV_ABS    : CVInstAluR<0b0101000, 0b011, "cv.abs">,
                   Sched<[]>;
@@ -249,12 +254,7 @@ let Predicates = [HasVendorXCValu, IsRV32],
                   Sched<[]>;
   def CV_SUBURN : CVInstAluRRI<0b11, 0b011, "cv.suburn">,
                   Sched<[]>;
-} // Predicates = [HasVendorXCValu, IsRV32],
-  //   hasSideEffects = 0, mayLoad = 0, mayStore = 0
 
-let Predicates = [HasVendorXCValu, IsRV32],
-  hasSideEffects = 0, mayLoad = 0, mayStore = 0,
-  Constraints = "$rd = $rd_wb" in {
   def CV_ADDNR   : CVInstAluRRNR<0b1000000, 0b011, "cv.addnr">,
                    Sched<[]>;
   def CV_ADDUNR  : CVInstAluRRNR<0b1000001, 0b011, "cv.addunr">,
@@ -272,9 +272,7 @@ let Predicates = [HasVendorXCValu, IsRV32],
   def CV_SUBURNR : CVInstAluRRNR<0b1000111, 0b011, "cv.suburnr">,
                    Sched<[]>;
 
-} // Predicates = [HasVendorXCValu, IsRV32],
-  //   hasSideEffects = 0, mayLoad = 0, mayStore = 0,
-  //   Constraints = "$rd = $rd_wb"
+} // Predicates = [HasVendorXCValu, IsRV32]
 
 let Predicates = [HasVendorXCValu, IsRV32] in {
   def : MnemonicAlias<"cv.slet", "cv.sle">;
@@ -288,7 +286,6 @@ class CVInstSIMDRR<bits<5> funct5, bit F, bit funct1, bits<3> funct3,
   let Inst{31-27} = funct5;
   let Inst{26} = F;
   let Inst{25} = funct1;
-  let DecoderNamespace = "XCVsimd";
 }
 
 class CVInstSIMDRI<bits<5> funct5, bit F, bits<3> funct3, RISCVOpcode opcode,
@@ -300,9 +297,9 @@ class CVInstSIMDRI<bits<5> funct5, bit F, bits<3> funct3, RISCVOpcode opcode,
   let Inst{26} = F;
   let Inst{25} = imm6{0}; // funct1 unused
   let Inst{24-20} = imm6{5-1};
-  let DecoderNamespace = "XCVsimd";
 }
 
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
 class CVSIMDRR<bits<5> funct5, bit F, bit funct1, bits<3> funct3,
                string opcodestr>
     : CVInstSIMDRR<funct5, F, funct1, funct3, OPC_CUSTOM_3, (outs GPR:$rd),
@@ -315,30 +312,24 @@ class CVSIMDRRWb<bits<5> funct5, bit F, bit funct1, bits<3> funct3,
   let Constraints = "$rd = $rd_wb";
 }
 
-class CVSIMDRI<bits<5> funct5, bit F, bits<3> funct3, string opcodestr>
+class CVSIMDRI<bits<5> funct5, bit F, bits<3> funct3, string opcodestr,
+               Operand immtype = simm6>
     : CVInstSIMDRI<funct5, F, funct3, OPC_CUSTOM_3, (outs GPR:$rd),
-              (ins GPR:$rs1, simm6:$imm6), opcodestr, "$rd, $rs1, $imm6">;
+                   (ins GPR:$rs1, immtype:$imm6), opcodestr, "$rd, $rs1, $imm6">;
 
-class CVSIMDRIWb<bits<5> funct5, bit F, bits<3> funct3, string opcodestr>
+class CVSIMDRIWb<bits<5> funct5, bit F, bits<3> funct3, string opcodestr,
+                 Operand immtype = simm6>
     : CVInstSIMDRI<funct5, F, funct3, OPC_CUSTOM_3,
-                   (outs GPR:$rd_wb), (ins GPR:$rd, GPR:$rs1, simm6:$imm6),
+                   (outs GPR:$rd_wb), (ins GPR:$rd, GPR:$rs1, immtype:$imm6),
                    opcodestr, "$rd, $rs1, $imm6"> {
   let Constraints = "$rd = $rd_wb";
 }
 
-class CVSIMDRU<bits<5> funct5, bit F, bits<3> funct3, string opcodestr,
-               Operand immtype = uimm6>
-    : CVInstSIMDRI<funct5, F, funct3, OPC_CUSTOM_3,
-                   (outs GPR:$rd), (ins GPR:$rs1, immtype:$imm6),
-                   opcodestr, "$rd, $rs1, $imm6">;
+class CVSIMDRU<bits<5> funct5, bit F, bits<3> funct3, string opcodestr>
+    : CVSIMDRI<funct5, F, funct3, opcodestr, uimm6>;
 
 class CVSIMDRUWb<bits<5> funct5, bit F, bits<3> funct3, string opcodestr>
-    : CVInstSIMDRI<funct5, F, funct3, OPC_CUSTOM_3,
-                   (outs GPR:$rd_wb),
-                   (ins GPR:$rd, GPR:$rs1, uimm6:$imm6),
-                   opcodestr, "$rd, $rs1, $imm6"> {
-  let Constraints = "$rd = $rd_wb";
-}
+    : CVSIMDRIWb<funct5, F, funct3, opcodestr, uimm6>;
 
 class CVSIMDR<bits<5> funct5, bit F, bit funct1, bits<3> funct3,
               string opcodestr>
@@ -346,6 +337,7 @@ class CVSIMDR<bits<5> funct5, bit F, bit funct1, bits<3> funct3,
                   (ins GPR:$rs1), opcodestr, "$rd, $rs1"> {
   let rs2 = 0b00000;
 }
+} // hasSideEffects = 0, mayLoad = 0, mayStore = 0
 
 multiclass CVSIMDBinarySigned<bits<5> funct5, bit F, bit funct1, string mnemonic> {
   def CV_ # NAME # _H : CVSIMDRR<funct5, F, funct1, 0b000, "cv." # mnemonic # ".h">;
@@ -370,8 +362,8 @@ multiclass CVSIMDShift<bits<5> funct5, bit F, bit funct1, string mnemonic> {
   def CV_ # NAME # _B : CVSIMDRR<funct5, F, funct1, 0b001, "cv." # mnemonic # ".b">;
   def CV_ # NAME # _SC_H : CVSIMDRR<funct5, F, funct1, 0b100, "cv." # mnemonic # ".sc.h">;
   def CV_ # NAME # _SC_B : CVSIMDRR<funct5, F, funct1, 0b101, "cv." # mnemonic # ".sc.b">;
-  def CV_ # NAME # _SCI_H : CVSIMDRU<funct5, F, 0b110, "cv." # mnemonic # ".sci.h", uimm4>;
-  def CV_ # NAME # _SCI_B : CVSIMDRU<funct5, F, 0b111, "cv." # mnemonic # ".sci.b", uimm3>;
+  def CV_ # NAME # _SCI_H : CVSIMDRI<funct5, F, 0b110, "cv." # mnemonic # ".sci.h", uimm4>;
+  def CV_ # NAME # _SCI_B : CVSIMDRI<funct5, F, 0b111, "cv." # mnemonic # ".sci.b", uimm3>;
 }
 
 multiclass CVSIMDBinarySignedWb<bits<5> funct5, bit F, bit funct1, string mnemonic> {
@@ -393,8 +385,7 @@ multiclass CVSIMDBinaryUnsignedWb<bits<5> funct5, bit F, bit funct1, string mnem
 }
 
 
-let Predicates = [HasVendorXCVsimd, IsRV32],
-    hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
+let Predicates = [HasVendorXCVsimd, IsRV32], DecoderNamespace = "XCV" in {
   defm ADD :    CVSIMDBinarySigned<0b00000, 0, 0, "add">;
   defm SUB :    CVSIMDBinarySigned<0b00001, 0, 0, "sub">;
   defm AVG :    CVSIMDBinarySigned<0b00010, 0, 0, "avg">;
@@ -496,11 +487,14 @@ class CVInstImmBranch<bits<3> funct3, dag outs, dag ins,
     : RVInstB<funct3, OPC_CUSTOM_0, outs, ins, opcodestr, argstr> {
   bits<5> imm5;
   let rs2 = imm5;
-  let DecoderNamespace = "XCVbi";
+  let isBranch = 1;
+  let isTerminator = 1;
+  let hasSideEffects = 0;
+  let mayLoad = 0;
+  let mayStore = 0;
 }
 
-let Predicates = [HasVendorXCVbi, IsRV32], hasSideEffects = 0, mayLoad = 0,
-    mayStore = 0, isBranch = 1, isTerminator = 1 in {
+let Predicates = [HasVendorXCVbi, IsRV32], DecoderNamespace = "XCV" in {
   // Immediate branching operations
   def CV_BEQIMM : CVInstImmBranch<0b110, (outs),
         (ins GPR:$rs1, simm5:$imm5, simm13_lsb0:$imm12),
@@ -526,18 +520,19 @@ def CVrr : Operand<i32>,
    let MIOperandInfo = (ops GPR:$base, GPR:$offset);
 }
 
+let hasSideEffects = 0, mayLoad = 1, mayStore = 0 in {
 class CVLoad_ri_inc<bits<3> funct3, string opcodestr>
-    : RVInstI<funct3, OPC_CUSTOM_0, (outs GPR:$rd, GPR:$rs1_wb), (ins GPRMem:$rs1, simm12:$imm12),
+    : RVInstI<funct3, OPC_CUSTOM_0, (outs GPR:$rd, GPR:$rs1_wb),
+              (ins GPRMem:$rs1, simm12:$imm12),
               opcodestr, "$rd, (${rs1}), ${imm12}"> {
   let Constraints = "$rs1_wb = $rs1";
-  let DecoderNamespace = "XCVmem";
 }
 
 class CVLoad_rr_inc<bits<7> funct7, bits<3> funct3, string opcodestr>
-    : RVInstR<funct7, funct3, OPC_CUSTOM_1, (outs GPR:$rd, GPR:$rs1_wb), (ins GPRMem:$rs1, GPR:$rs2),
+    : RVInstR<funct7, funct3, OPC_CUSTOM_1, (outs GPR:$rd, GPR:$rs1_wb),
+              (ins GPRMem:$rs1, GPR:$rs2),
               opcodestr, "$rd, (${rs1}), ${rs2}"> {
   let Constraints = "$rs1_wb = $rs1";
-  let DecoderNamespace = "XCVmem";
 }
 
 class CVLoad_rr<bits<7> funct7, bits<3> funct3, string opcodestr>
@@ -551,11 +546,10 @@ class CVLoad_rr<bits<7> funct7, bits<3> funct3, string opcodestr>
   let Inst{19-15} = cvrr{9-5};
   let Inst{14-12} = funct3;
   let Inst{11-7} = rd;
-  let DecoderNamespace = "XCVmem";
 }
+} // hasSideEffects = 0, mayLoad = 1, mayStore = 0
 
-let Predicates = [HasVendorXCVmem, IsRV32], hasSideEffects = 0,
-                 mayLoad = 1, mayStore = 0, Constraints = "$rs1_wb = $rs1" in {
+let Predicates = [HasVendorXCVmem, IsRV32], DecoderNamespace = "XCV" in {
   // Register-Immediate load with post-increment
   def CV_LB_ri_inc  : CVLoad_ri_inc<0b000, "cv.lb">;
   def CV_LBU_ri_inc : CVLoad_ri_inc<0b100, "cv.lbu">;
@@ -569,10 +563,7 @@ let Predicates = [HasVendorXCVmem, IsRV32], hasSideEffects = 0,
   def CV_LH_rr_inc  : CVLoad_rr_inc<0b0000001, 0b011, "cv.lh">;
   def CV_LHU_rr_inc : CVLoad_rr_inc<0b0001001, 0b011, "cv.lhu">;
   def CV_LW_rr_inc  : CVLoad_rr_inc<0b0000010, 0b011, "cv.lw">;
-}
 
-let Predicates = [HasVendorXCVmem, IsRV32], hasSideEffects = 0, 
-                 mayLoad = 1, mayStore = 0 in {
   // Register-Register load
   def CV_LB_rr  : CVLoad_rr<0b0000100, 0b011, "cv.lb">;
   def CV_LBU_rr : CVLoad_rr<0b0001100, 0b011, "cv.lbu">;
@@ -581,17 +572,17 @@ let Predicates = [HasVendorXCVmem, IsRV32], hasSideEffects = 0,
   def CV_LW_rr  : CVLoad_rr<0b0000110, 0b011, "cv.lw">;
 }
 
+let hasSideEffects = 0, mayLoad = 0, mayStore = 1 in {
 class CVStore_ri_inc<bits<3> funct3, string opcodestr>
-    : RVInstS<funct3, OPC_CUSTOM_1, (outs GPR:$rs1_wb), 
+    : RVInstS<funct3, OPC_CUSTOM_1, (outs GPR:$rs1_wb),
               (ins GPR:$rs2, GPR:$rs1, simm12:$imm12),
               opcodestr, "$rs2, (${rs1}), ${imm12}"> {
   let Constraints = "$rs1_wb = $rs1";
-  let DecoderNamespace = "XCVmem";
 }
 
-class CVStore_rr_inc<bits<3> funct3, bits<7> funct7, dag outs, dag ins,
-                     string opcodestr, string argstr>
-    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatOther> {
+class CVStore_rr_inc<bits<3> funct3, bits<7> funct7, string opcodestr>
+    : RVInst<(outs GPR:$rs1_wb), (ins GPR:$rs2, GPR:$rs1, GPR:$rs3), opcodestr,
+             "$rs2, (${rs1}), ${rs3}", [], InstFormatOther> {
   bits<5> rs3;
   bits<5> rs2;
   bits<5> rs1;
@@ -602,13 +593,13 @@ class CVStore_rr_inc<bits<3> funct3, bits<7> funct7, dag outs, dag ins,
   let Inst{14-12} = funct3;
   let Inst{11-7} = rs3;
   let Inst{6-0} = OPC_CUSTOM_1.Value;
-  let DecoderNamespace = "XCVmem";
+  let Constraints = "$rs1_wb = $rs1";
 }
 
 
-class CVStore_rr<bits<3> funct3, bits<7> funct7, dag outs, dag ins,
-                     string opcodestr, string argstr>
-    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatOther> {
+class CVStore_rr<bits<3> funct3, bits<7> funct7, string opcodestr>
+    : RVInst<(outs), (ins GPR:$rs2, CVrr:$cvrr), opcodestr, "$rs2, $cvrr", [],
+             InstFormatOther> {
   bits<5> rs2;
   bits<10> cvrr;
 
@@ -618,44 +609,26 @@ class CVStore_rr<bits<3> funct3, bits<7> funct7, dag outs, dag ins,
   let Inst{14-12} = funct3;
   let Inst{11-7} = cvrr{4-0};
   let Inst{6-0} = OPC_CUSTOM_1.Value;
-  let DecoderNamespace = "XCVmem";
 }
+} // hasSideEffects = 0, mayLoad = 0, mayStore = 1
 
-let Predicates = [HasVendorXCVmem, IsRV32], hasSideEffects = 0, 
-                 mayLoad = 0, mayStore = 1, Constraints = "$rs1_wb = $rs1" in {
+let Predicates = [HasVendorXCVmem, IsRV32], DecoderNamespace = "XCV" in {
   // Register-Immediate store with post-increment
   def CV_SB_ri_inc : CVStore_ri_inc<0b000, "cv.sb">;
   def CV_SH_ri_inc : CVStore_ri_inc<0b001, "cv.sh">;
   def CV_SW_ri_inc : CVStore_ri_inc<0b010, "cv.sw">;
 
   // Register-Register store with post-increment
-  def CV_SB_rr_inc : CVStore_rr_inc<0b011, 0b0010000, 
-                     (outs GPR:$rs1_wb), (ins GPR:$rs2, GPR:$rs1, GPR:$rs3),
-                      "cv.sb", "$rs2, (${rs1}), ${rs3}">;
-  def CV_SH_rr_inc : CVStore_rr_inc<0b011, 0b0010001, 
-                     (outs GPR:$rs1_wb), (ins GPR:$rs2, GPR:$rs1, GPR:$rs3),
-                      "cv.sh", "$rs2, (${rs1}), ${rs3}">;
-  def CV_SW_rr_inc : CVStore_rr_inc<0b011, 0b0010010, 
-                     (outs GPR:$rs1_wb), (ins GPR:$rs2, GPR:$rs1, GPR:$rs3),
-                      "cv.sw", "$rs2, (${rs1}), ${rs3}">;
-}
-
+  def CV_SB_rr_inc : CVStore_rr_inc<0b011, 0b0010000, "cv.sb">;
+  def CV_SH_rr_inc : CVStore_rr_inc<0b011, 0b0010001, "cv.sh">;
+  def CV_SW_rr_inc : CVStore_rr_inc<0b011, 0b0010010, "cv.sw">;
 
-let Predicates = [HasVendorXCVmem, IsRV32], hasSideEffects = 0, 
-                  mayLoad = 0, mayStore = 1 in {
   // Register-Register store
-  def CV_SB_rr : CVStore_rr<0b011, 0b0010100,
-                 (outs), (ins GPR:$rs2, CVrr:$cvrr),
-                 "cv.sb", "$rs2, $cvrr">;
-  def CV_SH_rr : CVStore_rr<0b011, 0b0010101,
-                 (outs), (ins GPR:$rs2, CVrr:$cvrr),
-                 "cv.sh", "$rs2, $cvrr">;
-  def CV_SW_rr : CVStore_rr<0b011, 0b0010110,
-                 (outs), (ins GPR:$rs2, CVrr:$cvrr),
-                 "cv.sw", "$rs2, $cvrr">;
-} 
-
-let DecoderNamespace = "XCVelw" in
+  def CV_SB_rr : CVStore_rr<0b011, 0b0010100, "cv.sb">;
+  def CV_SH_rr : CVStore_rr<0b011, 0b0010101, "cv.sh">;
+  def CV_SW_rr : CVStore_rr<0b011, 0b0010110, "cv.sw">;
+}
+
 class CVLoad_ri<bits<3> funct3, string opcodestr>
     : RVInstI<funct3, OPC_CUSTOM_0, (outs GPR:$rd), 
       (ins GPRMem:$rs1, simm12:$imm12), opcodestr, "$rd, ${imm12}(${rs1})">;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
index 942ced8c64815..f69b24ae7d9c5 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
@@ -43,7 +43,7 @@ class THInstVdotVV<bits<6> funct6, RISCVVFormat opv, dag outs, dag ins,
     : RVInstVV<funct6, opv, outs, ins, opcodestr, argstr> {
   let Inst{26} = 0;
   let Inst{6-0} = OPC_CUSTOM_0.Value;
-  let DecoderNamespace = "XTHeadVdot";
+  let DecoderNamespace = "XTHead";
 }
 
 class THInstVdotVX<bits<6> funct6, RISCVVFormat opv, dag outs, dag ins,
@@ -51,7 +51,7 @@ class THInstVdotVX<bits<6> funct6, RISCVVFormat opv, dag outs, dag ins,
     : RVInstVX<funct6, opv, outs, ins, opcodestr, argstr> {
   let Inst{26} = 1;
   let Inst{6-0} = OPC_CUSTOM_0.Value;
-  let DecoderNamespace = "XTHeadVdot";
+  let DecoderNamespace = "XTHead";
 }
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
@@ -76,8 +76,7 @@ class THVdotALUrVX<bits<6> funct6, RISCVVFormat opv, string opcodestr,
 }
 } // hasSideEffects = 0, mayLoad = 0, mayStore = 0
 
-let Predicates = [HasVendorXTHeadBa], DecoderNamespace = "XTHeadBa",
-    hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 class THShiftALU_rri<bits<3> funct3, string opcodestr>
     : RVInstRBase<funct3, OPC_CUSTOM_0, (outs GPR:$rd),
                   (ins GPR:$rs1, GPR:$rs2, uimm2:$uimm2),
@@ -87,13 +86,13 @@ class THShiftALU_rri<bits<3> funct3, string opcodestr>
   let Inst{26-25} = uimm2;
 }
 
-let Predicates = [HasVendorXTHeadBb], DecoderNamespace = "XTHeadBb",
-    hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 class THShift_ri<bits<5> funct5, bits<3> funct3, string opcodestr>
     : RVInstIShift<funct5, funct3, OPC_CUSTOM_0, (outs GPR:$rd),
                    (ins GPR:$rs1, uimmlog2xlen:$shamt),
                    opcodestr, "$rd, $rs1, $shamt">;
 
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 class THBitfieldExtract_rii<bits<3> funct3, string opcodestr>
     : RVInstIBase<funct3, OPC_CUSTOM_0, (outs GPR:$rd),
                   (ins GPR:$rs1, uimmlog2xlen:$msb, uimmlog2xlen:$lsb),
@@ -104,20 +103,18 @@ class THBitfieldExtract_rii<bits<3> funct3, string opcodestr>
   let Inst{25-20} = lsb;
 }
 
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 class THRev_r<bits<5> funct5, bits<2> funct2, string opcodestr>
     : RVInstIUnary<{funct5, funct2, 0b00000}, 0b001, OPC_CUSTOM_0,
                    (outs GPR:$rd), (ins GPR:$rs1), opcodestr, "$rd, $rs1">;
-}
 
-let Predicates = [HasVendorXTHeadBb, IsRV64], DecoderNamespace = "XTHeadBb",
-    hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 class THShiftW_ri<bits<7> funct7, bits<3> funct3, string opcodestr>
     : RVInstIShiftW<funct7, funct3, OPC_CUSTOM_0, (outs GPR:$rd),
                     (ins GPR:$rs1, uimm5:$shamt),
                     opcodestr, "$rd, $rs1, $shamt">;
 
-let Predicates = [HasVendorXTHeadCondMov], DecoderNamespace = "XTHeadCondMov",
-    hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCommutable = 1 in
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCommutable = 1 in
 class THCondMov_rr<bits<7> funct7, string opcodestr>
     : RVInstR<funct7, 0b001, OPC_CUSTOM_0, (outs GPR:$rd_wb),
               (ins GPR:$rd, GPR:$rs1, GPR:$rs2),
@@ -125,8 +122,7 @@ class THCondMov_rr<bits<7> funct7, string opcodestr>
   let Constraints = "$rd_wb = $rd";
 }
 
-let Predicates = [HasVendorXTHeadMac], DecoderNamespace = "XTHeadMac",
-    hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCommutable = 1 in
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCommutable = 1 in
 class THMulAccumulate_rr<bits<7> funct7, string opcodestr>
     : RVInstR<funct7, 0b001, OPC_CUSTOM_0, (outs GPR:$rd_wb),
               (ins GPR:$rd, GPR:$rs1, GPR:$rs2),
@@ -134,8 +130,7 @@ class THMulAccumulate_rr<bits<7> funct7, string opcodestr>
   let Constraints = "$rd_wb = $rd";
 }
 
-let Predicates = [HasVendorXTHeadMemPair], DecoderNamespace = "XTHeadMemPair",
-    hasSideEffects = 0, mayLoad = 1, mayStore = 0 in
+let hasSideEffects = 0, mayLoad = 1, mayStore = 0 in
 class THLoadPair<bits<5> funct5, string opcodestr>
   : RVInstRBase<0b100, OPC_CUSTOM_0,
                 (outs GPR:$rd, GPR:$rs2),
@@ -148,8 +143,7 @@ class THLoadPair<bits<5> funct5, string opcodestr>
   let Constraints = "@earlyclobber $rd,@earlyclobber $rs2";
 }
 
-let Predicates = [HasVendorXTHeadMemPair], DecoderNamespace = "XTHeadMemPair",
-    hasSideEffects = 0, mayLoad = 0, mayStore = 1 in
+let hasSideEffects = 0, mayLoad = 0, mayStore = 1 in
 class THStorePair<bits<5> funct5, string opcodestr>
   : RVInstRBase<0b101, OPC_CUSTOM_0, (outs),
               (ins GPR:$rd, GPR:$rs2, GPR:$rs1, uimm2:$uimm2, uimm7:$const3or4),
@@ -249,6 +243,7 @@ multiclass THVdotVMAQA<string opcodestr, bits<6> funct6>
 // Instructions
 //===----------------------------------------------------------------------===//
 
+let DecoderNamespace = "XTHead" in {
 let Predicates = [HasVendorXTHeadBa] in
 def TH_ADDSL : THShiftALU_rri<0b001, "th.addsl">,
                Sched<[WriteSHXADD, ReadSHXADD, ReadSHXADD]>;
@@ -263,12 +258,13 @@ def TH_REV : THRev_r<0b10000, 0b01, "th.rev">;
 def TH_TSTNBZ : THRev_r<0b10000, 0b00, "th.tstnbz">;
 } // Predicates = [HasVendorXTHeadBb]
 
-let Predicates = [HasVendorXTHeadBb, IsRV64], IsSignExtendingOpW = 1 in {
+let Predicates = [HasVendorXTHeadBb, IsRV64],
+    IsSignExtendingOpW = 1 in {
 def TH_SRRIW : THShiftW_ri<0b0001010, 0b001, "th.srriw">;
 def TH_REVW : THRev_r<0b10010, 0b00, "th.revw">;
 } // Predicates = [HasVendorXTHeadBb, IsRV64]
 
-let Predicates = [HasVendorXTHeadBs], DecoderNamespace = "XTHeadBs",
+let Predicates = [HasVendorXTHeadBs],
     IsSignExtendingOpW = 1 in
 def TH_TST : THShift_ri<0b10001, 0b001, "th.tst">,
              Sched<[WriteSingleBitImm, ReadSingleBitImm]>;
@@ -310,7 +306,7 @@ def TH_SDD : THStorePair<0b11111, "th.sdd">,
              Sched<[WriteSTD, WriteSTD, ReadStoreData, ReadMemBase]>;
 }
 
-let Predicates = [HasVendorXTHeadMemIdx], DecoderNamespace = "XTHeadMemIdx" in {
+let Predicates = [HasVendorXTHeadMemIdx] in {
 // T-Head Load/Store + Update instructions.
 def TH_LBIA : THLoadUpdate<0b00011, "th.lbia">,
               Sched<[WriteLDB, ReadMemBase]>;
@@ -390,7 +386,7 @@ def TH_SURW : THStoreIndexed<GPR, 0b01010, "th.surw">,
               Sched<[WriteLDB, ReadMemBase]>;
 }
 
-let Predicates = [HasVendorXTHeadMemIdx, IsRV64], DecoderNamespace = "XTHeadMemIdx" in {
+let Predicates = [HasVendorXTHeadMemIdx, IsRV64] in {
 // T-Head Load/Store + Update instructions.
 def TH_LWUIA : THLoadUpdate<0b11011, "th.lwuia">,
                Sched<[WriteLDH, ReadMemBase]>;
@@ -426,37 +422,34 @@ def TH_SURD : THStoreIndexed<GPR, 0b01110, "th.surd">,
 
 // T-Head Load/Store Indexed instructions for floating point registers.
 
-let Predicates = [HasVendorXTHeadFMemIdx, HasStdExtF],
-    DecoderNamespace = "XTHeadFMemIdx" in {
+let Predicates = [HasVendorXTHeadFMemIdx, HasStdExtF] in {
 def TH_FLRW : THLoadIndexed<FPR32, 0b01000, "th.flrw">,
               Sched<[WriteFLD32, ReadFMemBase]>;
 def TH_FSRW : THStoreIndexed<FPR32, 0b01000, "th.fsrw">,
               Sched<[WriteFST32, ReadFStoreData, ReadFMemBase]>;
 }
 
-let Predicates = [HasVendorXTHeadFMemIdx, HasStdExtD],
-    DecoderNamespace = "XTHeadFMemIdx" in {
+let Predicates = [HasVendorXTHeadFMemIdx, HasStdExtD] in {
 def TH_FLRD : THLoadIndexed<FPR64, 0b01100, "th.flrd">,
               Sched<[WriteFLD64, ReadFMemBase]>;
 def TH_FSRD : THStoreIndexed<FPR64, 0b01100, "th.fsrd">,
               Sched<[WriteFST64, ReadFStoreData, ReadFMemBase]>;
 }
 
-let Predicates = [HasVendorXTHeadFMemIdx, HasStdExtF, IsRV64],
-    DecoderNamespace = "XTHeadFMemIdx" in {
+let Predicates = [HasVendorXTHeadFMemIdx, HasStdExtF, IsRV64] in {
 def TH_FLURW : THLoadIndexed<FPR32, 0b01010, "th.flurw">,
                Sched<[WriteFLD32, ReadFMemBase]>;
 def TH_FSURW : THStoreIndexed<FPR32, 0b01010, "th.fsurw">,
                Sched<[WriteFST32, ReadFStoreData, ReadFMemBase]>;
 }
 
-let Predicates = [HasVendorXTHeadFMemIdx, HasStdExtD, IsRV64],
-    DecoderNamespace = "XTHeadFMemIdx" in {
+let Predicates = [HasVendorXTHeadFMemIdx, HasStdExtD, IsRV64] in {
 def TH_FLURD : THLoadIndexed<FPR64, 0b01110, "th.flurd">,
                Sched<[WriteFLD64, ReadFMemBase]>;
 def TH_FSURD : THStoreIndexed<FPR64, 0b01110, "th.fsurd">,
                Sched<[WriteFST64, ReadFStoreData, ReadFMemBase]>;
 }
+} // DecoderNamespace = "XTHead"
 
 let Predicates = [HasVendorXTHeadVdot] in {
 defm THVdotVMAQA      : THVdotVMAQA<"th.vmaqa",     0b100000>;
@@ -692,7 +685,7 @@ let Predicates = [HasVendorXTHeadMemPair] in {
             (TH_SWD GPR:$rd1, GPR:$rd2, GPR:$rs1, uimm2_3:$uimm2_3, 3)>;
 }
 
-let Predicates = [HasVendorXTHeadCmo], DecoderNamespace = "XTHeadCmo" in {
+let Predicates = [HasVendorXTHeadCmo], DecoderNamespace = "XTHead" in {
 def TH_DCACHE_CSW    : THCacheInst_r<0b00001, "th.dcache.csw">;
 def TH_DCACHE_ISW    : THCacheInst_r<0b00010, "th.dcache.isw">;
 def TH_DCACHE_CISW   : THCacheInst_r<0b00011, "th.dcache.cisw">;
@@ -717,7 +710,7 @@ def TH_L2CACHE_IALL  : THCacheInst_void<0b10110, "th.l2cache.iall">;
 def TH_L2CACHE_CIALL : THCacheInst_void<0b10111, "th.l2cache.ciall">;
 }
 
-let Predicates = [HasVendorXTHeadSync], DecoderNamespace = "XTHeadSync" in {
+let Predicates = [HasVendorXTHeadSync], DecoderNamespace = "XTHead" in {
 def TH_SFENCE_VMAS : THCacheInst_rr<0b0000010, "th.sfence.vmas">;
 def TH_SYNC        : THCacheInst_void<0b11000, "th.sync">;
 def TH_SYNC_S      : THCacheInst_void<0b11001, "th.sync.s">;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
index 8e84a01821f56..32cfade8697b9 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
@@ -568,7 +568,7 @@ let Predicates = [HasVendorXqciint, IsRV32], hasSideEffects = 1 in {
 
   let mayLoad = 0, mayStore = 1,
     Uses = [X1, X2, X5, X6, X7, X8, X10, X11, X12, X13, X14, X15, X16, X17, X28, X29, X30, X31],
-    Defs = [X2] in {
+    Defs = [X2, X8] in {
   def QC_C_MIENTER      : QCIRVInst16CI_NONE<0b10000, "qc.c.mienter">;
   def QC_C_MIENTER_NEST : QCIRVInst16CI_NONE<0b10001, "qc.c.mienter.nest">;
   } // mayLoad = 1, mayStore = 1, Uses = [...], Defs = [...]
diff --git a/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
new file mode 100644
index 0000000000000..46fd3158e07ca
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVLoadStoreOptimizer.cpp
@@ -0,0 +1,403 @@
+//===----- RISCVLoadStoreOptimizer.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Load/Store Pairing: It identifies pairs of load or store instructions
+// operating on consecutive memory locations and merges them into a single
+// paired instruction, leveraging hardware support for paired memory accesses.
+// Much of the pairing logic is adapted from the AArch64LoadStoreOpt pass.
+//
+// NOTE: The AArch64LoadStoreOpt pass performs additional optimizations such as
+// merging zero store instructions, promoting loads that read directly from a
+// preceding store, and merging base register updates with load/store
+// instructions (via pre-/post-indexed addressing). These advanced
+// transformations are not yet implemented in the RISC-V pass but represent
+// potential future enhancements for further optimizing RISC-V memory
+// operations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "RISCV.h"
+#include "RISCVTargetMachine.h"
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Target/TargetOptions.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "riscv-load-store-opt"
+#define RISCV_LOAD_STORE_OPT_NAME "RISC-V Load / Store Optimizer"
+
+// The LdStLimit limits number of instructions how far we search for load/store
+// pairs.
+static cl::opt<unsigned> LdStLimit("riscv-load-store-scan-limit", cl::init(128),
+                                   cl::Hidden);
+
+namespace {
+
+struct RISCVLoadStoreOpt : public MachineFunctionPass {
+  static char ID;
+  bool runOnMachineFunction(MachineFunction &Fn) override;
+
+  RISCVLoadStoreOpt() : MachineFunctionPass(ID) {}
+
+  MachineFunctionProperties getRequiredProperties() const override {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::NoVRegs);
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addRequired<AAResultsWrapperPass>();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  StringRef getPassName() const override { return RISCV_LOAD_STORE_OPT_NAME; }
+
+  // Find and pair load/store instructions.
+  bool tryToPairLdStInst(MachineBasicBlock::iterator &MBBI);
+
+  // Convert load/store pairs to single instructions.
+  bool tryConvertToLdStPair(MachineBasicBlock::iterator First,
+                            MachineBasicBlock::iterator Second);
+
+  // Scan the instructions looking for a load/store that can be combined
+  // with the current instruction into a load/store pair.
+  // Return the matching instruction if one is found, else MBB->end().
+  MachineBasicBlock::iterator findMatchingInsn(MachineBasicBlock::iterator I,
+                                               bool &MergeForward);
+
+  MachineBasicBlock::iterator
+  mergePairedInsns(MachineBasicBlock::iterator I,
+                   MachineBasicBlock::iterator Paired, bool MergeForward);
+
+private:
+  AliasAnalysis *AA;
+  MachineRegisterInfo *MRI;
+  const RISCVInstrInfo *TII;
+  const RISCVRegisterInfo *TRI;
+  LiveRegUnits ModifiedRegUnits, UsedRegUnits;
+};
+} // end anonymous namespace
+
+char RISCVLoadStoreOpt::ID = 0;
+INITIALIZE_PASS(RISCVLoadStoreOpt, DEBUG_TYPE, RISCV_LOAD_STORE_OPT_NAME, false,
+                false)
+
+bool RISCVLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
+  if (skipFunction(Fn.getFunction()))
+    return false;
+  const RISCVSubtarget &Subtarget = Fn.getSubtarget<RISCVSubtarget>();
+  if (!Subtarget.useLoadStorePairs())
+    return false;
+
+  bool MadeChange = false;
+  TII = Subtarget.getInstrInfo();
+  TRI = Subtarget.getRegisterInfo();
+  MRI = &Fn.getRegInfo();
+  AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
+  ModifiedRegUnits.init(*TRI);
+  UsedRegUnits.init(*TRI);
+
+  for (MachineBasicBlock &MBB : Fn) {
+    LLVM_DEBUG(dbgs() << "MBB: " << MBB.getName() << "\n");
+
+    for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
+         MBBI != E;) {
+      if (TII->isPairableLdStInstOpc(MBBI->getOpcode()) &&
+          tryToPairLdStInst(MBBI))
+        MadeChange = true;
+      else
+        ++MBBI;
+    }
+  }
+  return MadeChange;
+}
+
+// Find loads and stores that can be merged into a single load or store pair
+// instruction.
+bool RISCVLoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) {
+  MachineInstr &MI = *MBBI;
+
+  // If this is volatile, it is not a candidate.
+  if (MI.hasOrderedMemoryRef())
+    return false;
+
+  if (!TII->isLdStSafeToPair(MI, TRI))
+    return false;
+
+  // Look ahead for a pairable instruction.
+  MachineBasicBlock::iterator E = MI.getParent()->end();
+  bool MergeForward;
+  MachineBasicBlock::iterator Paired = findMatchingInsn(MBBI, MergeForward);
+  if (Paired != E) {
+    MBBI = mergePairedInsns(MBBI, Paired, MergeForward);
+    return true;
+  }
+  return false;
+}
+
+// Merge two adjacent load/store instructions into a paired instruction
+// (LDP/SDP/SWP/LWP) if the effective address is 8-byte aligned in case of
+// SWP/LWP 16-byte aligned in case of LDP/SDP. This function selects the
+// appropriate paired opcode, verifies that the memory operand is properly
+// aligned, and checks that the offset is valid. If all conditions are met, it
+// builds and inserts the paired instruction.
+bool RISCVLoadStoreOpt::tryConvertToLdStPair(
+    MachineBasicBlock::iterator First, MachineBasicBlock::iterator Second) {
+  unsigned PairOpc;
+  Align RequiredAlignment;
+  switch (First->getOpcode()) {
+  default:
+    llvm_unreachable("Unsupported load/store instruction for pairing");
+  case RISCV::SW:
+    PairOpc = RISCV::MIPS_SWP;
+    RequiredAlignment = Align(8);
+    break;
+  case RISCV::LW:
+    PairOpc = RISCV::MIPS_LWP;
+    RequiredAlignment = Align(8);
+    break;
+  case RISCV::SD:
+    PairOpc = RISCV::MIPS_SDP;
+    RequiredAlignment = Align(16);
+    break;
+  case RISCV::LD:
+    PairOpc = RISCV::MIPS_LDP;
+    RequiredAlignment = Align(16);
+    break;
+  }
+
+  MachineFunction *MF = First->getMF();
+  const MachineMemOperand *MMO = *First->memoperands_begin();
+  Align MMOAlign = MMO->getAlign();
+
+  if (MMOAlign < RequiredAlignment)
+    return false;
+
+  int64_t Offset = First->getOperand(2).getImm();
+  if (!isUInt<7>(Offset))
+    return false;
+
+  MachineInstrBuilder MIB = BuildMI(
+      *MF,
+      First->getDebugLoc().get() ? First->getDebugLoc() : Second->getDebugLoc(),
+      TII->get(PairOpc));
+  MIB.add(First->getOperand(0))
+      .add(Second->getOperand(0))
+      .add(First->getOperand(1))
+      .add(First->getOperand(2))
+      .cloneMergedMemRefs({&*First, &*Second});
+
+  First->getParent()->insert(First, MIB);
+
+  First->removeFromParent();
+  Second->removeFromParent();
+
+  return true;
+}
+
+static bool mayAlias(MachineInstr &MIa,
+                     SmallVectorImpl<MachineInstr *> &MemInsns,
+                     AliasAnalysis *AA) {
+  for (MachineInstr *MIb : MemInsns)
+    if (MIa.mayAlias(AA, *MIb, /*UseTBAA*/ false))
+      return true;
+
+  return false;
+}
+
+// Scan the instructions looking for a load/store that can be combined with the
+// current instruction into a wider equivalent or a load/store pair.
+// TODO: Extend pairing logic to consider reordering both instructions
+// to a safe "middle" position rather than only merging forward/backward.
+// This requires more sophisticated checks for aliasing, register
+// liveness, and potential scheduling hazards.
+MachineBasicBlock::iterator
+RISCVLoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
+                                    bool &MergeForward) {
+  MachineBasicBlock::iterator E = I->getParent()->end();
+  MachineBasicBlock::iterator MBBI = I;
+  MachineInstr &FirstMI = *I;
+  MBBI = next_nodbg(MBBI, E);
+
+  bool MayLoad = FirstMI.mayLoad();
+  Register Reg = FirstMI.getOperand(0).getReg();
+  Register BaseReg = FirstMI.getOperand(1).getReg();
+  int64_t Offset = FirstMI.getOperand(2).getImm();
+  int64_t OffsetStride = (*FirstMI.memoperands_begin())->getSize().getValue();
+
+  MergeForward = false;
+
+  // Track which register units have been modified and used between the first
+  // insn (inclusive) and the second insn.
+  ModifiedRegUnits.clear();
+  UsedRegUnits.clear();
+
+  // Remember any instructions that read/write memory between FirstMI and MI.
+  SmallVector<MachineInstr *, 4> MemInsns;
+
+  for (unsigned Count = 0; MBBI != E && Count < LdStLimit;
+       MBBI = next_nodbg(MBBI, E)) {
+    MachineInstr &MI = *MBBI;
+
+    // Don't count transient instructions towards the search limit since there
+    // may be different numbers of them if e.g. debug information is present.
+    if (!MI.isTransient())
+      ++Count;
+
+    if (MI.getOpcode() == FirstMI.getOpcode() &&
+        TII->isLdStSafeToPair(MI, TRI)) {
+      Register MIBaseReg = MI.getOperand(1).getReg();
+      int64_t MIOffset = MI.getOperand(2).getImm();
+
+      if (BaseReg == MIBaseReg) {
+        if ((Offset != MIOffset + OffsetStride) &&
+            (Offset + OffsetStride != MIOffset)) {
+          LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits,
+                                            TRI);
+          MemInsns.push_back(&MI);
+          continue;
+        }
+
+        // If the destination register of one load is the same register or a
+        // sub/super register of the other load, bail and keep looking.
+        if (MayLoad &&
+            TRI->isSuperOrSubRegisterEq(Reg, MI.getOperand(0).getReg())) {
+          LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits,
+                                            TRI);
+          MemInsns.push_back(&MI);
+          continue;
+        }
+
+        // If the BaseReg has been modified, then we cannot do the optimization.
+        if (!ModifiedRegUnits.available(BaseReg))
+          return E;
+
+        // If the Rt of the second instruction was not modified or used between
+        // the two instructions and none of the instructions between the second
+        // and first alias with the second, we can combine the second into the
+        // first.
+        if (ModifiedRegUnits.available(MI.getOperand(0).getReg()) &&
+            !(MI.mayLoad() &&
+              !UsedRegUnits.available(MI.getOperand(0).getReg())) &&
+            !mayAlias(MI, MemInsns, AA)) {
+
+          MergeForward = false;
+          return MBBI;
+        }
+
+        // Likewise, if the Rt of the first instruction is not modified or used
+        // between the two instructions and none of the instructions between the
+        // first and the second alias with the first, we can combine the first
+        // into the second.
+        if (!(MayLoad &&
+              !UsedRegUnits.available(FirstMI.getOperand(0).getReg())) &&
+            !mayAlias(FirstMI, MemInsns, AA)) {
+
+          if (ModifiedRegUnits.available(FirstMI.getOperand(0).getReg())) {
+            MergeForward = true;
+            return MBBI;
+          }
+        }
+        // Unable to combine these instructions due to interference in between.
+        // Keep looking.
+      }
+    }
+
+    // If the instruction wasn't a matching load or store.  Stop searching if we
+    // encounter a call instruction that might modify memory.
+    if (MI.isCall())
+      return E;
+
+    // Update modified / uses register units.
+    LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
+
+    // Otherwise, if the base register is modified, we have no match, so
+    // return early.
+    if (!ModifiedRegUnits.available(BaseReg))
+      return E;
+
+    // Update list of instructions that read/write memory.
+    if (MI.mayLoadOrStore())
+      MemInsns.push_back(&MI);
+  }
+  return E;
+}
+
+MachineBasicBlock::iterator
+RISCVLoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
+                                    MachineBasicBlock::iterator Paired,
+                                    bool MergeForward) {
+  MachineBasicBlock::iterator E = I->getParent()->end();
+  MachineBasicBlock::iterator NextI = next_nodbg(I, E);
+  // If NextI is the second of the two instructions to be merged, we need
+  // to skip one further. Either way we merge will invalidate the iterator,
+  // and we don't need to scan the new instruction, as it's a pairwise
+  // instruction, which we're not considering for further action anyway.
+  if (NextI == Paired)
+    NextI = next_nodbg(NextI, E);
+
+  // Insert our new paired instruction after whichever of the paired
+  // instructions MergeForward indicates.
+  MachineBasicBlock::iterator InsertionPoint = MergeForward ? Paired : I;
+  MachineBasicBlock::iterator DeletionPoint = MergeForward ? I : Paired;
+  int Offset = I->getOperand(2).getImm();
+  int PairedOffset = Paired->getOperand(2).getImm();
+  bool InsertAfter = (Offset < PairedOffset) ^ MergeForward;
+
+  if (!MergeForward)
+    Paired->getOperand(1).setIsKill(false);
+
+  // Kill flags may become invalid when moving stores for pairing.
+  if (I->getOperand(0).isUse()) {
+    if (!MergeForward) {
+      // Check if the Paired store's source register has a kill flag and clear
+      // it only if there are intermediate uses between I and Paired.
+      MachineOperand &PairedRegOp = Paired->getOperand(0);
+      if (PairedRegOp.isKill()) {
+        for (auto It = std::next(I); It != Paired; ++It) {
+          if (It->readsRegister(PairedRegOp.getReg(), TRI)) {
+            PairedRegOp.setIsKill(false);
+            break;
+          }
+        }
+      }
+    } else {
+      // Clear kill flags of the first store's register in the forward
+      // direction.
+      Register Reg = I->getOperand(0).getReg();
+      for (MachineInstr &MI : make_range(std::next(I), std::next(Paired)))
+        MI.clearRegisterKills(Reg, TRI);
+    }
+  }
+
+  MachineInstr *ToInsert = DeletionPoint->removeFromParent();
+  MachineBasicBlock &MBB = *InsertionPoint->getParent();
+  MachineBasicBlock::iterator First, Second;
+
+  if (!InsertAfter) {
+    First = MBB.insert(InsertionPoint, ToInsert);
+    Second = InsertionPoint;
+  } else {
+    Second = MBB.insertAfter(InsertionPoint, ToInsert);
+    First = InsertionPoint;
+  }
+
+  if (tryConvertToLdStPair(First, Second)) {
+    LLVM_DEBUG(dbgs() << "Pairing load/store:\n    ");
+    LLVM_DEBUG(prev_nodbg(NextI, MBB.begin())->print(dbgs()));
+  }
+
+  return NextI;
+}
+
+// Returns an instance of the Load / Store Optimization pass.
+FunctionPass *llvm::createRISCVLoadStoreOptPass() {
+  return new RISCVLoadStoreOpt();
+}
diff --git a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.cpp b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.cpp
index a2fdbcc7f4d28..1c8eda10f1958 100644
--- a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.cpp
@@ -100,6 +100,29 @@ RISCVMachineFunctionInfo::getPushPopKind(const MachineFunction &MF) const {
   return PushPopKind::None;
 }
 
+bool RISCVMachineFunctionInfo::hasImplicitFPUpdates(
+    const MachineFunction &MF) const {
+  switch (getInterruptStackKind(MF)) {
+  case InterruptStackKind::QCINest:
+  case InterruptStackKind::QCINoNest:
+    // QC.C.MIENTER and QC.C.MIENTER.NEST both update FP on function entry.
+    return true;
+  default:
+    break;
+  }
+
+  switch (getPushPopKind(MF)) {
+  case PushPopKind::VendorXqccmp:
+    // When using Xqccmp, we will use `QC.CM.PUSHFP` when Frame Pointers are
+    // enabled, which will update FP.
+    return true;
+  default:
+    break;
+  }
+
+  return false;
+}
+
 void RISCVMachineFunctionInfo::initializeBaseYamlFields(
     const yaml::RISCVMachineFunctionInfo &YamlMFI) {
   VarArgsFrameIndex = YamlMFI.VarArgsFrameIndex;
diff --git a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
index f24355650ab22..4d06dea7414f1 100644
--- a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
@@ -166,6 +166,12 @@ class RISCVMachineFunctionInfo : public MachineFunctionInfo {
   unsigned getQCIInterruptStackSize() const { return QCIInterruptStackSize; }
   void setQCIInterruptStackSize(unsigned Size) { QCIInterruptStackSize = Size; }
 
+  // Some Stack Management Variants automatically update FP in a frame-pointer
+  // convention compatible way - which means we don't need to manually update
+  // the FP, but we still need to emit the correct CFI information for
+  // calculating the CFA based on FP.
+  bool hasImplicitFPUpdates(const MachineFunction &MF) const;
+
   void initializeBaseYamlFields(const yaml::RISCVMachineFunctionInfo &YamlMFI);
 
   void addSExt32Register(Register Reg);
diff --git a/llvm/lib/Target/RISCV/RISCVPushPopOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVPushPopOptimizer.cpp
index 0ead9a4009fab..eae7e8697f0ad 100644
--- a/llvm/lib/Target/RISCV/RISCVPushPopOptimizer.cpp
+++ b/llvm/lib/Target/RISCV/RISCVPushPopOptimizer.cpp
@@ -69,16 +69,6 @@ static unsigned getPopRetOpcode(unsigned PopOpcode, bool IsReturnZero) {
   }
 }
 
-// Check if POP instruction was inserted into the MBB and return iterator to it.
-static MachineBasicBlock::iterator containsPop(MachineBasicBlock &MBB) {
-  for (MachineBasicBlock::iterator MBBI = MBB.begin(); MBBI != MBB.end();
-       MBBI = next_nodbg(MBBI, MBB.end()))
-    if (MBBI->getFlag(MachineInstr::FrameDestroy) && isPop(MBBI->getOpcode()))
-      return MBBI;
-
-  return MBB.end();
-}
-
 bool RISCVPushPopOpt::usePopRet(MachineBasicBlock::iterator &MBBI,
                                 MachineBasicBlock::iterator &NextI,
                                 bool IsReturnZero) {
@@ -150,19 +140,28 @@ bool RISCVPushPopOpt::runOnMachineFunction(MachineFunction &Fn) {
 
   TII = Subtarget->getInstrInfo();
   TRI = Subtarget->getRegisterInfo();
+
   // Resize the modified and used register unit trackers.  We do this once
   // per function and then clear the register units each time we determine
   // correct return value for the POP.
   ModifiedRegUnits.init(*TRI);
   UsedRegUnits.init(*TRI);
+
   bool Modified = false;
   for (auto &MBB : Fn) {
-    MachineBasicBlock::iterator MBBI = containsPop(MBB);
-    MachineBasicBlock::iterator NextI = next_nodbg(MBBI, MBB.end());
-    if (MBBI != MBB.end() && NextI != MBB.end() &&
-        NextI->getOpcode() == RISCV::PseudoRET)
-      Modified |= usePopRet(MBBI, NextI, adjustRetVal(MBBI));
+    // RET should be the only terminator.
+    auto RetMBBI = MBB.getFirstTerminator();
+    if (RetMBBI == MBB.end() || RetMBBI->getOpcode() != RISCV::PseudoRET ||
+        RetMBBI == MBB.begin())
+      continue;
+
+    // The previous instruction should be a POP.
+    auto PopMBBI = prev_nodbg(RetMBBI, MBB.begin());
+    if (isPop(PopMBBI->getOpcode()) &&
+        PopMBBI->getFlag(MachineInstr::FrameDestroy))
+      Modified |= usePopRet(PopMBBI, RetMBBI, adjustRetVal(PopMBBI));
   }
+
   return Modified;
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.cpp b/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
index 1b54c278820fc..3c996c82fcec4 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
@@ -62,14 +62,14 @@ static cl::opt<unsigned> RISCVMinimumJumpTableEntries(
     "riscv-min-jump-table-entries", cl::Hidden,
     cl::desc("Set minimum number of entries to use a jump table on RISCV"));
 
-static cl::opt<bool>
-    UseMIPSLoadStorePairsOpt("mips-riscv-load-store-pairs",
-                             cl::desc("RISCV: Optimize for load-store bonding"),
-                             cl::init(false), cl::Hidden);
+static cl::opt<bool> UseMIPSLoadStorePairsOpt(
+    "use-riscv-mips-load-store-pairs",
+    cl::desc("Enable the load/store pair optimization pass"), cl::init(false),
+    cl::Hidden);
 
-static cl::opt<bool>
-    UseCCMovInsn("riscv-ccmov", cl::desc("RISCV: Use 'mips.ccmov' instruction"),
-                 cl::init(true), cl::Hidden);
+static cl::opt<bool> UseCCMovInsn("use-riscv-ccmov",
+                                  cl::desc("Use 'mips.ccmov' instruction"),
+                                  cl::init(true), cl::Hidden);
 
 void RISCVSubtarget::anchor() {}
 
@@ -248,6 +248,10 @@ void RISCVSubtarget::overridePostRASchedPolicy(MachineSchedPolicy &Policy,
   }
 }
 
+bool RISCVSubtarget::useLoadStorePairs() const {
+  return UseMIPSLoadStorePairsOpt && HasVendorXMIPSLSP;
+}
+
 bool RISCVSubtarget::useCCMovInsn() const {
   return UseCCMovInsn && HasVendorXMIPSCMove;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h
index cc9aef2d52556..035ce4745cfd9 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.h
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h
@@ -354,6 +354,7 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
   unsigned getMaxPrefetchIterationsAhead() const override {
     return TuneInfo->MaxPrefetchIterationsAhead;
   };
+  bool enableWritePrefetching() const override { return true; }
 
   unsigned getMinimumJumpTableEntries() const;
 
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index 52bb10f9ba19b..f78e5f8147d98 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -145,6 +145,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
   initializeRISCVDAGToDAGISelLegacyPass(*PR);
   initializeRISCVMoveMergePass(*PR);
   initializeRISCVPushPopOptPass(*PR);
+  initializeRISCVLoadStoreOptPass(*PR);
 }
 
 static StringRef computeDataLayout(const Triple &TT,
@@ -551,6 +552,8 @@ void RISCVPassConfig::addPreSched2() {
 
   // Emit KCFI checks for indirect calls.
   addPass(createKCFIPass());
+  if (TM->getOptLevel() != CodeGenOptLevel::None)
+    addPass(createRISCVLoadStoreOptPass());
 }
 
 void RISCVPassConfig::addPreEmitPass() {
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index 24f9912e40be5..cbec1c95eadc3 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -1008,8 +1008,8 @@ SPIRVType *SPIRVGlobalRegistry::findSPIRVType(
   Register Reg = DT.find(Ty, &MIRBuilder.getMF());
   if (Reg.isValid())
     return getSPIRVTypeForVReg(Reg);
-  if (ForwardPointerTypes.contains(Ty))
-    return ForwardPointerTypes[Ty];
+  if (auto It = ForwardPointerTypes.find(Ty); It != ForwardPointerTypes.end())
+    return It->second;
   return restOfCreateSPIRVType(Ty, MIRBuilder, AccQual, EmitIR);
 }
 
@@ -1103,14 +1103,15 @@ SPIRVType *SPIRVGlobalRegistry::createSPIRVType(
   // Null pointer means we have a loop in type definitions, make and
   // return corresponding OpTypeForwardPointer.
   if (SpvElementType == nullptr) {
-    if (!ForwardPointerTypes.contains(Ty))
-      ForwardPointerTypes[Ty] = getOpTypeForwardPointer(SC, MIRBuilder);
-    return ForwardPointerTypes[Ty];
+    auto [It, Inserted] = ForwardPointerTypes.try_emplace(Ty);
+    if (Inserted)
+      It->second = getOpTypeForwardPointer(SC, MIRBuilder);
+    return It->second;
   }
   // If we have forward pointer associated with this type, use its register
   // operand to create OpTypePointer.
-  if (ForwardPointerTypes.contains(Ty)) {
-    Register Reg = getSPIRVTypeID(ForwardPointerTypes[Ty]);
+  if (auto It = ForwardPointerTypes.find(Ty); It != ForwardPointerTypes.end()) {
+    Register Reg = getSPIRVTypeID(It->second);
     return getOpTypePointer(SC, SpvElementType, MIRBuilder, Reg);
   }
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index baacd58b28028..3779a4b6ccd34 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -120,8 +120,9 @@ addConstantsToTrack(MachineFunction &MF, SPIRVGlobalRegistry *GR,
   }
   for (MachineInstr *MI : ToErase) {
     Register Reg = MI->getOperand(2).getReg();
-    if (RegsAlreadyAddedToDT.contains(MI))
-      Reg = RegsAlreadyAddedToDT[MI];
+    auto It = RegsAlreadyAddedToDT.find(MI);
+    if (It != RegsAlreadyAddedToDT.end())
+      Reg = It->second;
     auto *RC = MRI.getRegClassOrNull(MI->getOperand(0).getReg());
     if (!MRI.getRegClassOrNull(Reg) && RC)
       MRI.setRegClass(Reg, RC);
@@ -652,7 +653,7 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
   }
   for (MachineInstr *MI : ToErase) {
     auto It = RegsAlreadyAddedToDT.find(MI);
-    if (RegsAlreadyAddedToDT.contains(MI))
+    if (It != RegsAlreadyAddedToDT.end())
       MRI.replaceRegWith(MI->getOperand(0).getReg(), It->second);
     MI->eraseFromParent();
   }
diff --git a/llvm/lib/Target/SystemZ/SystemZMachineScheduler.cpp b/llvm/lib/Target/SystemZ/SystemZMachineScheduler.cpp
index 4bc979de795dc..5e2365f1dc513 100644
--- a/llvm/lib/Target/SystemZ/SystemZMachineScheduler.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZMachineScheduler.cpp
@@ -94,14 +94,16 @@ void SystemZPostRASchedStrategy::enterMBB(MachineBasicBlock *NextMBB) {
   // scheduled. If this is not possible, we are done.
   MachineBasicBlock *SinglePredMBB =
     getSingleSchedPred(MBB, MLI->getLoopFor(MBB));
-  if (SinglePredMBB == nullptr ||
-      SchedStates.find(SinglePredMBB) == SchedStates.end())
+  if (SinglePredMBB == nullptr)
+    return;
+  auto It = SchedStates.find(SinglePredMBB);
+  if (It == SchedStates.end())
     return;
 
   LLVM_DEBUG(dbgs() << "** Continued scheduling from "
                     << printMBBReference(*SinglePredMBB) << "\n";);
 
-  HazardRec->copyState(SchedStates[SinglePredMBB]);
+  HazardRec->copyState(It->second);
   LLVM_DEBUG(HazardRec->dumpState(););
 
   // Emit incoming terminator(s). Be optimistic and assume that branch
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
index 277d353d1db10..640be5fe8e8c9 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
@@ -1850,13 +1850,12 @@ bool WebAssemblyCFGStackify::fixCallUnwindMismatches(MachineFunction &MF) {
 
       // If the EH pad on the stack top is where this instruction should unwind
       // next, we're good.
-      MachineBasicBlock *UnwindDest = getFakeCallerBlock(MF);
+      MachineBasicBlock *UnwindDest = nullptr;
       for (auto *Succ : MBB.successors()) {
         // Even though semantically a BB can have multiple successors in case an
-        // exception is not caught by a catchpad, in our backend implementation
-        // it is guaranteed that a BB can have at most one EH pad successor. For
-        // details, refer to comments in findWasmUnwindDestinations function in
-        // SelectionDAGBuilder.cpp.
+        // exception is not caught by a catchpad, the first unwind destination
+        // should appear first in the successor list, based on the calculation
+        // in findUnwindDestinations() in SelectionDAGBuilder.cpp.
         if (Succ->isEHPad()) {
           UnwindDest = Succ;
           break;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
index 550d8b64dca35..2d11019291a7a 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
@@ -659,9 +659,6 @@ static bool canLongjmp(const Value *Callee) {
   // Every catchpad generated by Wasm C++ contains __cxa_end_catch, so we
   // intentionally treat it as longjmpable to work around this problem. This is
   // a hacky fix but an easy one.
-  //
-  // The comment block in findWasmUnwindDestinations() in
-  // SelectionDAGBuilder.cpp is addressing a similar problem.
   if (CalleeName == "__cxa_end_catch")
     return WebAssembly::WasmEnableSjLj;
   if (CalleeName == "__cxa_begin_catch" ||
diff --git a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
index 684580d6fca06..24bf0dd378641 100644
--- a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
+++ b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
@@ -101,7 +101,8 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
 
   getActionDefinitionsBuilder({G_LROUND, G_LLROUND, G_FCOS, G_FCOSH, G_FACOS,
                                G_FSIN, G_FSINH, G_FASIN, G_FTAN, G_FTANH,
-                               G_FATAN, G_FATAN2})
+                               G_FATAN, G_FATAN2, G_FPOW, G_FEXP, G_FEXP2,
+                               G_FEXP10, G_FLOG, G_FLOG2, G_FLOG10})
       .libcall();
 
   // merge/unmerge
diff --git a/llvm/lib/Target/X86/X86FastISel.cpp b/llvm/lib/Target/X86/X86FastISel.cpp
index a0fe5960bc99a..9f3d9dca402cb 100644
--- a/llvm/lib/Target/X86/X86FastISel.cpp
+++ b/llvm/lib/Target/X86/X86FastISel.cpp
@@ -75,15 +75,15 @@ class X86FastISel final : public FastISel {
                           const DebugLoc &DL);
 
   bool X86FastEmitLoad(MVT VT, X86AddressMode &AM, MachineMemOperand *MMO,
-                       unsigned &ResultReg, unsigned Alignment = 1);
+                       Register &ResultReg, unsigned Alignment = 1);
 
   bool X86FastEmitStore(EVT VT, const Value *Val, X86AddressMode &AM,
                         MachineMemOperand *MMO = nullptr, bool Aligned = false);
-  bool X86FastEmitStore(EVT VT, unsigned ValReg, X86AddressMode &AM,
+  bool X86FastEmitStore(EVT VT, Register ValReg, X86AddressMode &AM,
                         MachineMemOperand *MMO = nullptr, bool Aligned = false);
 
-  bool X86FastEmitExtend(ISD::NodeType Opc, EVT DstVT, unsigned Src, EVT SrcVT,
-                         unsigned &ResultReg);
+  bool X86FastEmitExtend(ISD::NodeType Opc, EVT DstVT, Register Src, EVT SrcVT,
+                         Register &ResultReg);
 
   bool X86SelectAddress(const Value *V, X86AddressMode &AM);
   bool X86SelectCallAddress(const Value *V, X86AddressMode &AM);
@@ -134,9 +134,9 @@ class X86FastISel final : public FastISel {
 
   bool handleConstantAddresses(const Value *V, X86AddressMode &AM);
 
-  unsigned X86MaterializeInt(const ConstantInt *CI, MVT VT);
-  unsigned X86MaterializeFP(const ConstantFP *CFP, MVT VT);
-  unsigned X86MaterializeGV(const GlobalValue *GV, MVT VT);
+  Register X86MaterializeInt(const ConstantInt *CI, MVT VT);
+  Register X86MaterializeFP(const ConstantFP *CFP, MVT VT);
+  Register X86MaterializeGV(const GlobalValue *GV, MVT VT);
   Register fastMaterializeConstant(const Constant *C) override;
 
   Register fastMaterializeAlloca(const AllocaInst *C) override;
@@ -163,9 +163,9 @@ class X86FastISel final : public FastISel {
   const MachineInstrBuilder &addFullAddress(const MachineInstrBuilder &MIB,
                                             X86AddressMode &AM);
 
-  unsigned fastEmitInst_rrrr(unsigned MachineInstOpcode,
-                             const TargetRegisterClass *RC, unsigned Op0,
-                             unsigned Op1, unsigned Op2, unsigned Op3);
+  Register fastEmitInst_rrrr(unsigned MachineInstOpcode,
+                             const TargetRegisterClass *RC, Register Op0,
+                             Register Op1, Register Op2, Register Op3);
 };
 
 } // end anonymous namespace.
@@ -313,7 +313,7 @@ bool X86FastISel::isTypeLegal(Type *Ty, MVT &VT, bool AllowI1) {
 /// The address is either pre-computed, i.e. Ptr, or a GlobalAddress, i.e. GV.
 /// Return true and the result register by reference if it is possible.
 bool X86FastISel::X86FastEmitLoad(MVT VT, X86AddressMode &AM,
-                                  MachineMemOperand *MMO, unsigned &ResultReg,
+                                  MachineMemOperand *MMO, Register &ResultReg,
                                   unsigned Alignment) {
   bool HasSSE1 = Subtarget->hasSSE1();
   bool HasSSE2 = Subtarget->hasSSE2();
@@ -475,7 +475,7 @@ bool X86FastISel::X86FastEmitLoad(MVT VT, X86AddressMode &AM,
 /// type VT. The address is either pre-computed, consisted of a base ptr, Ptr
 /// and a displacement offset, or a GlobalAddress,
 /// i.e. V. Return true if it is possible.
-bool X86FastISel::X86FastEmitStore(EVT VT, unsigned ValReg, X86AddressMode &AM,
+bool X86FastISel::X86FastEmitStore(EVT VT, Register ValReg, X86AddressMode &AM,
                                    MachineMemOperand *MMO, bool Aligned) {
   bool HasSSE1 = Subtarget->hasSSE1();
   bool HasSSE2 = Subtarget->hasSSE2();
@@ -686,7 +686,7 @@ bool X86FastISel::X86FastEmitStore(EVT VT, const Value *Val,
   }
 
   Register ValReg = getRegForValue(Val);
-  if (ValReg == 0)
+  if (!ValReg)
     return false;
 
   return X86FastEmitStore(VT, ValReg, AM, MMO, Aligned);
@@ -695,11 +695,10 @@ bool X86FastISel::X86FastEmitStore(EVT VT, const Value *Val,
 /// X86FastEmitExtend - Emit a machine instruction to extend a value Src of
 /// type SrcVT to type DstVT using the specified extension opcode Opc (e.g.
 /// ISD::SIGN_EXTEND).
-bool X86FastISel::X86FastEmitExtend(ISD::NodeType Opc, EVT DstVT,
-                                    unsigned Src, EVT SrcVT,
-                                    unsigned &ResultReg) {
-  unsigned RR = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(), Opc, Src);
-  if (RR == 0)
+bool X86FastISel::X86FastEmitExtend(ISD::NodeType Opc, EVT DstVT, Register Src,
+                                    EVT SrcVT, Register &ResultReg) {
+  Register RR = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(), Opc, Src);
+  if (!RR)
     return false;
 
   ResultReg = RR;
@@ -899,7 +898,7 @@ bool X86FastISel::X86SelectAddress(const Value *V, X86AddressMode &AM) {
 
     // Pattern-match simple GEPs.
     uint64_t Disp = (int32_t)AM.Disp;
-    unsigned IndexReg = AM.IndexReg;
+    Register IndexReg = AM.IndexReg;
     unsigned Scale = AM.Scale;
     MVT PtrVT = TLI.getValueType(DL, U->getType()).getSimpleVT();
 
@@ -933,13 +932,12 @@ bool X86FastISel::X86SelectAddress(const Value *V, X86AddressMode &AM) {
           Op = cast<AddOperator>(Op)->getOperand(0);
           continue;
         }
-        if (IndexReg == 0 &&
-            (!AM.GV || !Subtarget->isPICStyleRIPRel()) &&
+        if (!IndexReg && (!AM.GV || !Subtarget->isPICStyleRIPRel()) &&
             (S == 1 || S == 2 || S == 4 || S == 8)) {
           // Scaled-index addressing.
           Scale = S;
           IndexReg = getRegForGEPIndex(PtrVT, Op);
-          if (IndexReg == 0)
+          if (!IndexReg)
             return false;
           break;
         }
@@ -1206,7 +1204,7 @@ bool X86FastISel::X86SelectRet(const Instruction *I) {
     return false;
 
   // Build a list of return value registers.
-  SmallVector<unsigned, 4> RetRegs;
+  SmallVector<Register, 4> RetRegs;
 
   if (Ret->getNumOperands() > 0) {
     SmallVector<ISD::OutputArg, 4> Outs;
@@ -1219,7 +1217,7 @@ bool X86FastISel::X86SelectRet(const Instruction *I) {
 
     const Value *RV = Ret->getOperand(0);
     Register Reg = getRegForValue(RV);
-    if (Reg == 0)
+    if (!Reg)
       return false;
 
     // Only handle a single return value for now.
@@ -1240,7 +1238,7 @@ bool X86FastISel::X86SelectRet(const Instruction *I) {
     if (VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1)
       return false;
 
-    unsigned SrcReg = Reg + VA.getValNo();
+    Register SrcReg = Reg + VA.getValNo();
     EVT SrcVT = TLI.getValueType(DL, RV->getType());
     EVT DstVT = VA.getValVT();
     // Special handling for extended integers.
@@ -1290,7 +1288,7 @@ bool X86FastISel::X86SelectRet(const Instruction *I) {
     Register Reg = X86MFInfo->getSRetReturnReg();
     assert(Reg &&
            "SRetReturnReg should have been set in LowerFormalArguments()!");
-    unsigned RetReg = Subtarget->isTarget64BitLP64() ? X86::RAX : X86::EAX;
+    Register RetReg = Subtarget->isTarget64BitLP64() ? X86::RAX : X86::EAX;
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
             TII.get(TargetOpcode::COPY), RetReg).addReg(Reg);
     RetRegs.push_back(RetReg);
@@ -1306,7 +1304,7 @@ bool X86FastISel::X86SelectRet(const Instruction *I) {
     MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
                   TII.get(Subtarget->is64Bit() ? X86::RET64 : X86::RET32));
   }
-  for (unsigned Reg : RetRegs)
+  for (Register Reg : RetRegs)
     MIB.addReg(Reg, RegState::Implicit);
   return true;
 }
@@ -1345,7 +1343,7 @@ bool X86FastISel::X86SelectLoad(const Instruction *I) {
   if (!X86SelectAddress(Ptr, AM))
     return false;
 
-  unsigned ResultReg = 0;
+  Register ResultReg;
   if (!X86FastEmitLoad(VT, AM, createMachineMemOperandFor(LI), ResultReg,
                        LI->getAlign().value()))
     return false;
@@ -1402,7 +1400,8 @@ static unsigned X86ChooseCmpImmediateOpcode(EVT VT, const ConstantInt *RHSC) {
 bool X86FastISel::X86FastEmitCompare(const Value *Op0, const Value *Op1, EVT VT,
                                      const DebugLoc &CurMIMD) {
   Register Op0Reg = getRegForValue(Op0);
-  if (Op0Reg == 0) return false;
+  if (!Op0Reg)
+    return false;
 
   // Handle 'null' like i32/i64 0.
   if (isa<ConstantPointerNull>(Op1))
@@ -1424,7 +1423,8 @@ bool X86FastISel::X86FastEmitCompare(const Value *Op0, const Value *Op1, EVT VT,
   if (CompareOpc == 0) return false;
 
   Register Op1Reg = getRegForValue(Op1);
-  if (Op1Reg == 0) return false;
+  if (!Op1Reg)
+    return false;
   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, CurMIMD, TII.get(CompareOpc))
     .addReg(Op0Reg)
     .addReg(Op1Reg);
@@ -1445,7 +1445,7 @@ bool X86FastISel::X86SelectCmp(const Instruction *I) {
 
   // Try to optimize or fold the cmp.
   CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
-  unsigned ResultReg = 0;
+  Register ResultReg;
   switch (Predicate) {
   default: break;
   case CmpInst::FCMP_FALSE: {
@@ -1535,7 +1535,7 @@ bool X86FastISel::X86SelectZExt(const Instruction *I) {
     return false;
 
   Register ResultReg = getRegForValue(I->getOperand(0));
-  if (ResultReg == 0)
+  if (!ResultReg)
     return false;
 
   // Handle zero-extension from i1 to i8, which is common.
@@ -1545,7 +1545,7 @@ bool X86FastISel::X86SelectZExt(const Instruction *I) {
     ResultReg = fastEmitZExtFromI1(MVT::i8, ResultReg);
     SrcVT = MVT::i8;
 
-    if (ResultReg == 0)
+    if (!ResultReg)
       return false;
   }
 
@@ -1579,7 +1579,7 @@ bool X86FastISel::X86SelectZExt(const Instruction *I) {
   } else if (DstVT != MVT::i8) {
     ResultReg = fastEmit_r(MVT::i8, DstVT.getSimpleVT(), ISD::ZERO_EXTEND,
                            ResultReg);
-    if (ResultReg == 0)
+    if (!ResultReg)
       return false;
   }
 
@@ -1593,7 +1593,7 @@ bool X86FastISel::X86SelectSExt(const Instruction *I) {
     return false;
 
   Register ResultReg = getRegForValue(I->getOperand(0));
-  if (ResultReg == 0)
+  if (!ResultReg)
     return false;
 
   // Handle sign-extension from i1 to i8.
@@ -1601,7 +1601,7 @@ bool X86FastISel::X86SelectSExt(const Instruction *I) {
   if (SrcVT == MVT::i1) {
     // Set the high bits to zero.
     Register ZExtReg = fastEmitZExtFromI1(MVT::i8, ResultReg);
-    if (ZExtReg == 0)
+    if (!ZExtReg)
       return false;
 
     // Negate the result to make an 8-bit sign extended value.
@@ -1623,7 +1623,7 @@ bool X86FastISel::X86SelectSExt(const Instruction *I) {
   } else if (DstVT != MVT::i8) {
     ResultReg = fastEmit_r(MVT::i8, DstVT.getSimpleVT(), ISD::SIGN_EXTEND,
                            ResultReg);
-    if (ResultReg == 0)
+    if (!ResultReg)
       return false;
   }
 
@@ -1730,7 +1730,8 @@ bool X86FastISel::X86SelectBranch(const Instruction *I) {
       }
       if (TestOpc) {
         Register OpReg = getRegForValue(TI->getOperand(0));
-        if (OpReg == 0) return false;
+        if (!OpReg)
+          return false;
 
         BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TestOpc))
           .addReg(OpReg).addImm(1);
@@ -1752,7 +1753,7 @@ bool X86FastISel::X86SelectBranch(const Instruction *I) {
     // Fake request the condition, otherwise the intrinsic might be completely
     // optimized away.
     Register TmpReg = getRegForValue(BI->getCondition());
-    if (TmpReg == 0)
+    if (!TmpReg)
       return false;
 
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::JCC_1))
@@ -1765,11 +1766,12 @@ bool X86FastISel::X86SelectBranch(const Instruction *I) {
   // Note that i1 essentially gets ANY_EXTEND'ed to i8 where it isn't used
   // in an explicit cast, so make sure to handle that correctly.
   Register OpReg = getRegForValue(BI->getCondition());
-  if (OpReg == 0) return false;
+  if (!OpReg)
+    return false;
 
   // In case OpReg is a K register, COPY to a GPR
   if (MRI.getRegClass(OpReg) == &X86::VK1RegClass) {
-    unsigned KOpReg = OpReg;
+    Register KOpReg = OpReg;
     OpReg = createResultReg(&X86::GR32RegClass);
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
             TII.get(TargetOpcode::COPY), OpReg)
@@ -1786,7 +1788,8 @@ bool X86FastISel::X86SelectBranch(const Instruction *I) {
 }
 
 bool X86FastISel::X86SelectShift(const Instruction *I) {
-  unsigned CReg = 0, OpReg = 0;
+  Register CReg;
+  unsigned OpReg;
   const TargetRegisterClass *RC = nullptr;
   if (I->getType()->isIntegerTy(8)) {
     CReg = X86::CL;
@@ -1833,10 +1836,12 @@ bool X86FastISel::X86SelectShift(const Instruction *I) {
     return false;
 
   Register Op0Reg = getRegForValue(I->getOperand(0));
-  if (Op0Reg == 0) return false;
+  if (!Op0Reg)
+    return false;
 
   Register Op1Reg = getRegForValue(I->getOperand(1));
-  if (Op1Reg == 0) return false;
+  if (!Op1Reg)
+    return false;
   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
           CReg).addReg(Op1Reg);
 
@@ -1942,10 +1947,10 @@ bool X86FastISel::X86SelectDivRem(const Instruction *I) {
   const DivRemEntry &TypeEntry = OpTable[TypeIndex];
   const DivRemEntry::DivRemResult &OpEntry = TypeEntry.ResultTable[OpIndex];
   Register Op0Reg = getRegForValue(I->getOperand(0));
-  if (Op0Reg == 0)
+  if (!Op0Reg)
     return false;
   Register Op1Reg = getRegForValue(I->getOperand(1));
-  if (Op1Reg == 0)
+  if (!Op1Reg)
     return false;
 
   // Move op0 into low-order input register.
@@ -1990,7 +1995,7 @@ bool X86FastISel::X86SelectDivRem(const Instruction *I) {
   // won't generate explicit references to the GR8_NOREX registers. If
   // the allocator and/or the backend get enhanced to be more robust in
   // that regard, this can be, and should be, removed.
-  unsigned ResultReg = 0;
+  Register ResultReg;
   if ((I->getOpcode() == Instruction::SRem ||
        I->getOpcode() == Instruction::URem) &&
       OpEntry.DivRemResultReg == X86::AH && Subtarget->is64Bit()) {
@@ -2095,7 +2100,7 @@ bool X86FastISel::X86FastEmitCMoveSelect(MVT RetVT, const Instruction *I) {
     // Fake request the condition, otherwise the intrinsic might be completely
     // optimized away.
     Register TmpReg = getRegForValue(Cond);
-    if (TmpReg == 0)
+    if (!TmpReg)
       return false;
 
     NeedTest = false;
@@ -2108,12 +2113,12 @@ bool X86FastISel::X86FastEmitCMoveSelect(MVT RetVT, const Instruction *I) {
     // whereas lsb is zero. Therefore, we have to truncate Op0Reg to i1 for
     // the select. This is achieved by performing TEST against 1.
     Register CondReg = getRegForValue(Cond);
-    if (CondReg == 0)
+    if (!CondReg)
       return false;
 
     // In case OpReg is a K register, COPY to a GPR
     if (MRI.getRegClass(CondReg) == &X86::VK1RegClass) {
-      unsigned KCondReg = CondReg;
+      Register KCondReg = CondReg;
       CondReg = createResultReg(&X86::GR32RegClass);
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
               TII.get(TargetOpcode::COPY), CondReg)
@@ -2192,7 +2197,7 @@ bool X86FastISel::X86FastEmitSSESelect(MVT RetVT, const Instruction *I) {
     return false;
 
   const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);
-  unsigned ResultReg;
+  Register ResultReg;
 
   if (Subtarget->hasAVX512()) {
     // If we have AVX512 we can use a mask compare and masked movss/sd.
@@ -2214,7 +2219,7 @@ bool X86FastISel::X86FastEmitSSESelect(MVT RetVT, const Instruction *I) {
     // LHS in the input. The mask input comes from the compare.
     unsigned MovOpcode =
       (RetVT == MVT::f32) ? X86::VMOVSSZrrk : X86::VMOVSDZrrk;
-    unsigned MovReg = fastEmitInst_rrrr(MovOpcode, VR128X, RHSReg, CmpReg,
+    Register MovReg = fastEmitInst_rrrr(MovOpcode, VR128X, RHSReg, CmpReg,
                                         ImplicitDefReg, LHSReg);
 
     ResultReg = createResultReg(RC);
@@ -2309,12 +2314,12 @@ bool X86FastISel::X86FastEmitPseudoSelect(MVT RetVT, const Instruction *I) {
       return false;
   } else {
     Register CondReg = getRegForValue(Cond);
-    if (CondReg == 0)
+    if (!CondReg)
       return false;
 
     // In case OpReg is a K register, COPY to a GPR
     if (MRI.getRegClass(CondReg) == &X86::VK1RegClass) {
-      unsigned KCondReg = CondReg;
+      Register KCondReg = CondReg;
       CondReg = createResultReg(&X86::GR32RegClass);
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
               TII.get(TargetOpcode::COPY), CondReg)
@@ -2359,7 +2364,7 @@ bool X86FastISel::X86SelectSelect(const Instruction *I) {
     // No need for a select anymore - this is an unconditional move.
     if (Opnd) {
       Register OpReg = getRegForValue(Opnd);
-      if (OpReg == 0)
+      if (!OpReg)
         return false;
       const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);
       Register ResultReg = createResultReg(RC);
@@ -2404,7 +2409,7 @@ bool X86FastISel::X86SelectIntToFP(const Instruction *I, bool IsSigned) {
 
   // Select integer to float/double conversion.
   Register OpReg = getRegForValue(I->getOperand(0));
-  if (OpReg == 0)
+  if (!OpReg)
     return false;
 
   unsigned Opcode;
@@ -2458,10 +2463,10 @@ bool X86FastISel::X86SelectFPExtOrFPTrunc(const Instruction *I,
   bool HasAVX = Subtarget->hasAVX();
 
   Register OpReg = getRegForValue(I->getOperand(0));
-  if (OpReg == 0)
+  if (!OpReg)
     return false;
 
-  unsigned ImplicitDefReg;
+  Register ImplicitDefReg;
   if (HasAVX) {
     ImplicitDefReg = createResultReg(RC);
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
@@ -2566,7 +2571,7 @@ bool X86FastISel::TryEmitSmallMemcpy(X86AddressMode DestAM,
     else
       VT = MVT::i8;
 
-    unsigned Reg;
+    Register Reg;
     bool RV = X86FastEmitLoad(VT, SrcAM, nullptr, Reg);
     RV &= X86FastEmitStore(VT, Reg, DestAM);
     assert(RV && "Failed to emit load or store??");
@@ -2592,7 +2597,7 @@ bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
 
     const Value *Op = II->getArgOperand(0);
     Register InputReg = getRegForValue(Op);
-    if (InputReg == 0)
+    if (!InputReg)
       return false;
 
     // F16C only allows converting from float to half and from half to float.
@@ -2605,7 +2610,7 @@ bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
         return false;
     }
 
-    unsigned ResultReg = 0;
+    Register ResultReg;
     const TargetRegisterClass *RC = TLI.getRegClassFor(MVT::v8i16);
     if (IsFloatToHalf) {
       // 'InputReg' is implicitly promoted from register class FR32 to
@@ -2816,11 +2821,11 @@ bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
     const Value *SrcVal = II->getArgOperand(0);
     Register SrcReg = getRegForValue(SrcVal);
 
-    if (SrcReg == 0)
+    if (!SrcReg)
       return false;
 
     const TargetRegisterClass *RC = TLI.getRegClassFor(VT);
-    unsigned ImplicitDefReg = 0;
+    Register ImplicitDefReg;
     if (AVXLevel > 0) {
       ImplicitDefReg = createResultReg(RC);
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
@@ -2887,10 +2892,10 @@ bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
     }
 
     Register LHSReg = getRegForValue(LHS);
-    if (LHSReg == 0)
+    if (!LHSReg)
       return false;
 
-    unsigned ResultReg = 0;
+    Register ResultReg;
     // Check if we have an immediate version.
     if (const auto *CI = dyn_cast<ConstantInt>(RHS)) {
       static const uint16_t Opc[2][4] = {
@@ -2910,10 +2915,10 @@ bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
         ResultReg = fastEmit_ri(VT, VT, BaseOpc, LHSReg, CI->getZExtValue());
     }
 
-    unsigned RHSReg;
+    Register RHSReg;
     if (!ResultReg) {
       RHSReg = getRegForValue(RHS);
-      if (RHSReg == 0)
+      if (!RHSReg)
         return false;
       ResultReg = fastEmit_rr(VT, VT, BaseOpc, LHSReg, RHSReg);
     }
@@ -3010,7 +3015,7 @@ bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
         break;
       unsigned Idx = cast<ConstantInt>(Index)->getZExtValue();
 
-      if (Idx == 0) {
+      if (!Idx) {
         Op = IE->getOperand(1);
         break;
       }
@@ -3018,7 +3023,7 @@ bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
     }
 
     Register Reg = getRegForValue(Op);
-    if (Reg == 0)
+    if (!Reg)
       return false;
 
     Register ResultReg = createResultReg(TLI.getRegClassFor(VT));
@@ -3162,7 +3167,7 @@ bool X86FastISel::fastLowerArguments() {
   for (auto const &Arg : F->args()) {
     MVT VT = TLI.getSimpleValueType(DL, Arg.getType());
     const TargetRegisterClass *RC = TLI.getRegClassFor(VT);
-    unsigned SrcReg;
+    MCRegister SrcReg;
     switch (VT.SimpleTy) {
     default: llvm_unreachable("Unexpected value type.");
     case MVT::i32: SrcReg = GPR32ArgRegs[GPRIdx++]; break;
@@ -3281,7 +3286,7 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
       return false;
 
   SmallVector<MVT, 16> OutVTs;
-  SmallVector<unsigned, 16> ArgRegs;
+  SmallVector<Register, 16> ArgRegs;
 
   // If this is a constant i1/i8/i16 argument, promote to i32 to avoid an extra
   // instruction. This is safe because it is common to all FastISel supported
@@ -3302,7 +3307,7 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
     // Codegen this as an argument + "and 1".
     MVT VT;
     auto *TI = dyn_cast<TruncInst>(Val);
-    unsigned ResultReg;
+    Register ResultReg;
     if (TI && TI->getType()->isIntegerTy(1) && CLI.CB &&
         (TI->getParent() == CLI.CB->getParent()) && TI->hasOneUse()) {
       Value *PrevVal = TI->getOperand(0);
@@ -3356,7 +3361,7 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
     if (ArgVT == MVT::x86mmx)
       return false;
 
-    unsigned ArgReg = ArgRegs[VA.getValNo()];
+    Register ArgReg = ArgRegs[VA.getValNo()];
 
     // Promote the value if needed.
     switch (VA.getLocInfo()) {
@@ -3384,7 +3389,7 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
         ArgReg = fastEmitZExtFromI1(MVT::i8, ArgReg);
         ArgVT = MVT::i8;
 
-        if (ArgReg == 0)
+        if (!ArgReg)
           return false;
       }
 
@@ -3588,7 +3593,7 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
   for (unsigned i = 0; i != RVLocs.size(); ++i) {
     CCValAssign &VA = RVLocs[i];
     EVT CopyVT = VA.getValVT();
-    unsigned CopyReg = ResultReg + i;
+    Register CopyReg = ResultReg + i;
     Register SrcReg = VA.getLocReg();
 
     // If this is x86-64, and we disabled SSE, we can't return FP values
@@ -3683,7 +3688,8 @@ X86FastISel::fastSelectInstruction(const Instruction *I)  {
     if (DstVT.bitsLT(SrcVT))
       return X86SelectTrunc(I);
     Register Reg = getRegForValue(I->getOperand(0));
-    if (Reg == 0) return false;
+    if (!Reg)
+      return false;
     updateValueMap(I, Reg);
     return true;
   }
@@ -3722,9 +3728,9 @@ X86FastISel::fastSelectInstruction(const Instruction *I)  {
   return false;
 }
 
-unsigned X86FastISel::X86MaterializeInt(const ConstantInt *CI, MVT VT) {
+Register X86FastISel::X86MaterializeInt(const ConstantInt *CI, MVT VT) {
   if (VT > MVT::i64)
-    return 0;
+    return Register();
 
   uint64_t Imm = CI->getZExtValue();
   if (Imm == 0) {
@@ -3770,7 +3776,7 @@ unsigned X86FastISel::X86MaterializeInt(const ConstantInt *CI, MVT VT) {
   return fastEmitInst_i(Opc, TLI.getRegClassFor(VT), Imm);
 }
 
-unsigned X86FastISel::X86MaterializeFP(const ConstantFP *CFP, MVT VT) {
+Register X86FastISel::X86MaterializeFP(const ConstantFP *CFP, MVT VT) {
   if (CFP->isNullValue())
     return fastMaterializeFloatZero(CFP);
 
@@ -3846,13 +3852,13 @@ unsigned X86FastISel::X86MaterializeFP(const ConstantFP *CFP, MVT VT) {
   return ResultReg;
 }
 
-unsigned X86FastISel::X86MaterializeGV(const GlobalValue *GV, MVT VT) {
+Register X86FastISel::X86MaterializeGV(const GlobalValue *GV, MVT VT) {
   // Can't handle large GlobalValues yet.
   if (TM.getCodeModel() != CodeModel::Small &&
       TM.getCodeModel() != CodeModel::Medium)
-    return 0;
+    return Register();
   if (TM.isLargeGlobalValue(GV))
-    return 0;
+    return Register();
 
   // Materialize addresses with LEA/MOV instructions.
   X86AddressMode AM;
@@ -3881,7 +3887,7 @@ unsigned X86FastISel::X86MaterializeGV(const GlobalValue *GV, MVT VT) {
     }
     return ResultReg;
   }
-  return 0;
+  return Register();
 }
 
 Register X86FastISel::fastMaterializeConstant(const Constant *C) {
@@ -4034,10 +4040,10 @@ bool X86FastISel::tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,
   return true;
 }
 
-unsigned X86FastISel::fastEmitInst_rrrr(unsigned MachineInstOpcode,
+Register X86FastISel::fastEmitInst_rrrr(unsigned MachineInstOpcode,
                                         const TargetRegisterClass *RC,
-                                        unsigned Op0, unsigned Op1,
-                                        unsigned Op2, unsigned Op3) {
+                                        Register Op0, Register Op1,
+                                        Register Op2, Register Op3) {
   const MCInstrDesc &II = TII.get(MachineInstOpcode);
 
   Register ResultReg = createResultReg(RC);
@@ -4065,7 +4071,6 @@ unsigned X86FastISel::fastEmitInst_rrrr(unsigned MachineInstOpcode,
   return ResultReg;
 }
 
-
 namespace llvm {
   FastISel *X86::createFastISel(FunctionLoweringInfo &funcInfo,
                                 const TargetLibraryInfo *libInfo) {
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index deab638b7e546..6aefb08aa7d1f 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2634,7 +2634,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
           ISD::FSIN,   ISD::STRICT_FSIN,
           ISD::FSINH,  ISD::STRICT_FSINH,
           ISD::FTAN,   ISD::STRICT_FTAN,
-          ISD::FTANH,  ISD::STRICT_FTANH})
+          ISD::FTANH,  ISD::STRICT_FTANH,
+          // TODO: Add ISD:::STRICT_FMODF too once implemented.
+          ISD::FMODF})
       if (isOperationExpand(Op, MVT::f32))
         setOperationAction(Op, MVT::f32, Promote);
   // clang-format on
@@ -31883,7 +31885,6 @@ X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
 
   IRBuilder<> Builder(AI);
   Builder.CollectMetadataToCopy(AI, {LLVMContext::MD_pcsections});
-  Module *M = Builder.GetInsertBlock()->getParent()->getParent();
   auto SSID = AI->getSyncScopeID();
   // We must restrict the ordering to avoid generating loads with Release or
   // ReleaseAcquire orderings.
@@ -31905,21 +31906,10 @@ X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
   // otherwise, we might be able to be more aggressive on relaxed idempotent
   // rmw. In practice, they do not look useful, so we don't try to be
   // especially clever.
-  if (SSID == SyncScope::SingleThread)
-    // FIXME: we could just insert an ISD::MEMBARRIER here, except we are at
-    // the IR level, so we must wrap it in an intrinsic.
-    return nullptr;
-
-  if (!Subtarget.hasMFence())
-    // FIXME: it might make sense to use a locked operation here but on a
-    // different cache-line to prevent cache-line bouncing. In practice it
-    // is probably a small win, and x86 processors without mfence are rare
-    // enough that we do not bother.
-    return nullptr;
 
-  Function *MFence =
-      llvm::Intrinsic::getOrInsertDeclaration(M, Intrinsic::x86_sse2_mfence);
-  Builder.CreateCall(MFence, {});
+  // Use `fence seq_cst` over `llvm.x64.sse2.mfence` here to get the correct
+  // lowering for SSID == SyncScope::SingleThread and avoidMFence || !hasMFence
+  Builder.CreateFence(AtomicOrdering::SequentiallyConsistent, SSID);
 
   // Finally we can emit the atomic load.
   LoadInst *Loaded = Builder.CreateAlignedLoad(
@@ -32007,7 +31997,7 @@ static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget &Subtarget,
   // cross-thread fence.
   if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
       FenceSSID == SyncScope::System) {
-    if (Subtarget.hasMFence())
+    if (!Subtarget.avoidMFence() && Subtarget.hasMFence())
       return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
 
     SDValue Chain = Op.getOperand(0);
@@ -39652,19 +39642,15 @@ static SDValue combineX86ShuffleChainWithExtract(
 /// into either a single instruction if there is a special purpose instruction
 /// for this operation, or into a PSHUFB instruction which is a fully general
 /// instruction but should only be used to replace chains over a certain depth.
-static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
-                                      ArrayRef<int> BaseMask, int Depth,
-                                      ArrayRef<const SDNode *> SrcNodes,
-                                      bool AllowVariableCrossLaneMask,
-                                      bool AllowVariablePerLaneMask,
-                                      bool IsMaskedShuffle,
-                                      SelectionDAG &DAG,
-                                      const X86Subtarget &Subtarget) {
+static SDValue combineX86ShuffleChain(
+    ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
+    ArrayRef<const SDNode *> SrcNodes, bool AllowVariableCrossLaneMask,
+    bool AllowVariablePerLaneMask, bool IsMaskedShuffle, SelectionDAG &DAG,
+    const SDLoc &DL, const X86Subtarget &Subtarget) {
   assert(!BaseMask.empty() && "Cannot combine an empty shuffle mask!");
   assert((Inputs.size() == 1 || Inputs.size() == 2) &&
          "Unexpected number of shuffle inputs!");
 
-  SDLoc DL(Root);
   unsigned RootOpc = Root.getOpcode();
   MVT RootVT = Root.getSimpleValueType();
   unsigned RootSizeInBits = RootVT.getSizeInBits();
@@ -40535,7 +40521,7 @@ static SDValue combineX86ShuffleChainWithExtract(
   if (SDValue WideShuffle = combineX86ShuffleChain(
           WideInputs, WideRoot, WideMask, Depth, SrcNodes,
           AllowVariableCrossLaneMask, AllowVariablePerLaneMask, IsMaskedShuffle,
-          DAG, Subtarget)) {
+          DAG, SDLoc(WideRoot), Subtarget)) {
     WideShuffle =
         extractSubVector(WideShuffle, 0, DAG, SDLoc(Root), RootSizeInBits);
     return DAG.getBitcast(RootVT, WideShuffle);
@@ -40884,7 +40870,7 @@ static SDValue combineX86ShufflesRecursively(
     ArrayRef<SDValue> SrcOps, int SrcOpIndex, SDValue Root,
     ArrayRef<int> RootMask, ArrayRef<const SDNode *> SrcNodes, unsigned Depth,
     unsigned MaxDepth, bool AllowVariableCrossLaneMask,
-    bool AllowVariablePerLaneMask, SelectionDAG &DAG,
+    bool AllowVariablePerLaneMask, SelectionDAG &DAG, const SDLoc &DL,
     const X86Subtarget &Subtarget) {
   assert(!RootMask.empty() &&
          (RootMask.size() > 1 || (RootMask[0] == 0 && SrcOpIndex == 0)) &&
@@ -40892,7 +40878,6 @@ static SDValue combineX86ShufflesRecursively(
   MVT RootVT = Root.getSimpleValueType();
   assert(RootVT.isVector() && "Shuffles operate on vector types!");
   unsigned RootSizeInBits = RootVT.getSizeInBits();
-  SDLoc DL(Root);
 
   // Bound the depth of our recursive combine because this is ultimately
   // quadratic in nature.
@@ -41186,7 +41171,7 @@ static SDValue combineX86ShufflesRecursively(
       }
       if (SDValue Res = combineX86ShufflesRecursively(
               Ops, i, Root, ResolvedMask, CombinedNodes, Depth + 1, MaxDepth,
-              AllowCrossLaneVar, AllowPerLaneVar, DAG, Subtarget))
+              AllowCrossLaneVar, AllowPerLaneVar, DAG, DL, Subtarget))
         return Res;
     }
   }
@@ -41306,7 +41291,7 @@ static SDValue combineX86ShufflesRecursively(
     // Try to combine into a single shuffle instruction.
     if (SDValue Shuffle = combineX86ShuffleChain(
             Ops, Root, Mask, Depth, CombinedNodes, AllowVariableCrossLaneMask,
-            AllowVariablePerLaneMask, IsMaskedShuffle, DAG, Subtarget))
+            AllowVariablePerLaneMask, IsMaskedShuffle, DAG, DL, Subtarget))
       return Shuffle;
 
     // If all the operands come from the same larger vector, fallthrough and try
@@ -41334,7 +41319,7 @@ static SDValue combineX86ShufflesRecursively(SDValue Op, SelectionDAG &DAG,
   return combineX86ShufflesRecursively(
       {Op}, 0, Op, {0}, {}, /*Depth*/ 0, X86::MaxShuffleCombineDepth,
       /*AllowCrossLaneVarMask*/ true, /*AllowPerLaneVarMask*/ true, DAG,
-      Subtarget);
+      SDLoc(Op), Subtarget);
 }
 
 /// Get the PSHUF-style mask from PSHUF node.
@@ -41378,6 +41363,22 @@ static SmallVector<int, 4> getPSHUFShuffleMask(SDValue N) {
   }
 }
 
+/// Get the expanded blend mask from a BLENDI node.
+/// For v16i16 nodes, this will splat the repeated i8 mask.
+static APInt getBLENDIBlendMask(SDValue V) {
+  assert(V.getOpcode() == X86ISD::BLENDI && "Unknown blend shuffle");
+  unsigned NumElts = V.getSimpleValueType().getVectorNumElements();
+  APInt Mask = V.getConstantOperandAPInt(2);
+  if (Mask.getBitWidth() > NumElts)
+    Mask = Mask.trunc(NumElts);
+  if (NumElts == 16) {
+    assert(Mask.getBitWidth() == 8 && "Unexpected v16i16 blend mask width");
+    Mask = APInt::getSplat(16, Mask);
+  }
+  assert(Mask.getBitWidth() == NumElts && "Unexpected blend mask width");
+  return Mask;
+}
+
 /// Search for a combinable shuffle across a chain ending in pshufd.
 ///
 /// We walk up the chain and look for a combinable shuffle, skipping over
@@ -41914,8 +41915,8 @@ static SDValue canonicalizeLaneShuffleWithRepeatedOps(SDValue V,
 
 static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
                                       ArrayRef<SDValue> Ops, SelectionDAG &DAG,
-                                      TargetLowering::DAGCombinerInfo &DCI,
-                                      const X86Subtarget &Subtarget);
+                                      const X86Subtarget &Subtarget,
+                                      unsigned Depth = 0);
 
 /// Try to combine x86 target specific shuffles.
 static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
@@ -41971,7 +41972,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
               {BC}, 0, BC, DemandedMask, {}, /*Depth*/ 0,
               X86::MaxShuffleCombineDepth,
               /*AllowCrossLaneVarMask*/ true,
-              /*AllowPerLaneVarMask*/ true, DAG, Subtarget))
+              /*AllowPerLaneVarMask*/ true, DAG, DL, Subtarget))
         return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
                            DAG.getBitcast(SrcVT, Res));
     }
@@ -42275,7 +42276,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
         unsigned SrcBits = SrcVT.getScalarSizeInBits();
         if ((EltBits % SrcBits) == 0 && SrcBits >= 32) {
           unsigned NewSize = SrcVT.getVectorNumElements();
-          APInt BlendMask = N.getConstantOperandAPInt(2).zextOrTrunc(NumElts);
+          APInt BlendMask = getBLENDIBlendMask(N);
           APInt NewBlendMask = APIntOps::ScaleBitMask(BlendMask, NewSize);
           return DAG.getBitcast(
               VT, DAG.getNode(X86ISD::BLENDI, DL, SrcVT, N0.getOperand(0),
@@ -42662,7 +42663,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
     if (CanConcat) {
       SDValue Ops[] = {N.getOperand(0), N.getOperand(2)};
       if (SDValue ConcatSrc =
-              combineConcatVectorOps(DL, WideVT, Ops, DAG, DCI, Subtarget)) {
+              combineConcatVectorOps(DL, WideVT, Ops, DAG, Subtarget)) {
         SDValue Mask = widenSubVector(N.getOperand(1), false, Subtarget, DAG,
                                       DL, WideVT.getSizeInBits());
         SDValue Perm = DAG.getNode(X86ISD::VPERMV, DL, WideVT, Mask, ConcatSrc);
@@ -42677,7 +42678,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
       // See if we can concatenate the commuted operands.
       if (CanConcat) {
         if (SDValue ConcatSrc = combineConcatVectorOps(
-                DL, WideVT, {N.getOperand(2), N.getOperand(0)}, DAG, DCI,
+                DL, WideVT, {N.getOperand(2), N.getOperand(0)}, DAG,
                 Subtarget)) {
           ShuffleVectorSDNode::commuteMask(Mask);
           Mask.append(NumElts, SM_SentinelUndef);
@@ -42707,7 +42708,7 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
                    m_InsertSubvector(m_Undef(), m_Value(Ops[1]), m_Zero())) &&
           Ops[0].getValueType() == HalfVT && Ops[1].getValueType() == HalfVT) {
         if (SDValue ConcatSrc =
-                combineConcatVectorOps(DL, VT, Ops, DAG, DCI, Subtarget)) {
+                combineConcatVectorOps(DL, VT, Ops, DAG, Subtarget)) {
           for (int &M : Mask)
             M = (M < (int)NumElts ? M : (M - (NumElts / 2)));
           return lowerShuffleWithPERMV(DL, VT, Mask, ConcatSrc,
@@ -43974,7 +43975,7 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
     SDValue NewShuffle = combineX86ShufflesRecursively(
         {Op}, 0, Op, DemandedMask, {}, 0, X86::MaxShuffleCombineDepth - Depth,
         /*AllowCrossLaneVarMask*/ true, /*AllowPerLaneVarMask*/ true, TLO.DAG,
-        Subtarget);
+        SDLoc(Op), Subtarget);
     if (NewShuffle)
       return TLO.CombineTo(Op, NewShuffle);
   }
@@ -44746,7 +44747,10 @@ bool X86TargetLowering::isSplatValueForTargetNode(SDValue Op,
 // Helper to peek through bitops/trunc/setcc to determine size of source vector.
 // Allows combineBitcastvxi1 to determine what size vector generated a <X x i1>.
 static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size,
-                                      bool AllowTruncate) {
+                                      bool AllowTruncate, unsigned Depth) {
+  // Limit recursion.
+  if (Depth >= SelectionDAG::MaxRecursionDepth)
+    return false;
   switch (Src.getOpcode()) {
   case ISD::TRUNCATE:
     if (!AllowTruncate)
@@ -44755,17 +44759,22 @@ static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size,
   case ISD::SETCC:
     return Src.getOperand(0).getValueSizeInBits() == Size;
   case ISD::FREEZE:
-    return checkBitcastSrcVectorSize(Src.getOperand(0), Size, AllowTruncate);
+    return checkBitcastSrcVectorSize(Src.getOperand(0), Size, AllowTruncate,
+                                     Depth + 1);
   case ISD::AND:
   case ISD::XOR:
   case ISD::OR:
-    return checkBitcastSrcVectorSize(Src.getOperand(0), Size, AllowTruncate) &&
-           checkBitcastSrcVectorSize(Src.getOperand(1), Size, AllowTruncate);
+    return checkBitcastSrcVectorSize(Src.getOperand(0), Size, AllowTruncate,
+                                     Depth + 1) &&
+           checkBitcastSrcVectorSize(Src.getOperand(1), Size, AllowTruncate,
+                                     Depth + 1);
   case ISD::SELECT:
   case ISD::VSELECT:
     return Src.getOperand(0).getScalarValueSizeInBits() == 1 &&
-           checkBitcastSrcVectorSize(Src.getOperand(1), Size, AllowTruncate) &&
-           checkBitcastSrcVectorSize(Src.getOperand(2), Size, AllowTruncate);
+           checkBitcastSrcVectorSize(Src.getOperand(1), Size, AllowTruncate,
+                                     Depth + 1) &&
+           checkBitcastSrcVectorSize(Src.getOperand(2), Size, AllowTruncate,
+                                     Depth + 1);
   case ISD::BUILD_VECTOR:
     return ISD::isBuildVectorAllZeros(Src.getNode()) ||
            ISD::isBuildVectorAllOnes(Src.getNode());
@@ -44936,7 +44945,7 @@ static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src,
     // For cases such as (i4 bitcast (v4i1 setcc v4i64 v1, v2))
     // sign-extend to a 256-bit operation to avoid truncation.
     if (Subtarget.hasAVX() &&
-        checkBitcastSrcVectorSize(Src, 256, Subtarget.hasAVX2())) {
+        checkBitcastSrcVectorSize(Src, 256, Subtarget.hasAVX2(), 0)) {
       SExtVT = MVT::v4i64;
       PropagateSExt = true;
     }
@@ -44948,8 +44957,8 @@ static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src,
     // If the setcc operand is 128-bit, prefer sign-extending to 128-bit over
     // 256-bit because the shuffle is cheaper than sign extending the result of
     // the compare.
-    if (Subtarget.hasAVX() && (checkBitcastSrcVectorSize(Src, 256, true) ||
-                               checkBitcastSrcVectorSize(Src, 512, true))) {
+    if (Subtarget.hasAVX() && (checkBitcastSrcVectorSize(Src, 256, true, 0) ||
+                               checkBitcastSrcVectorSize(Src, 512, true, 0))) {
       SExtVT = MVT::v8i32;
       PropagateSExt = true;
     }
@@ -44974,7 +44983,7 @@ static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src,
       break;
     }
     // Split if this is a <64 x i8> comparison result.
-    if (checkBitcastSrcVectorSize(Src, 512, false)) {
+    if (checkBitcastSrcVectorSize(Src, 512, false, 0)) {
       SExtVT = MVT::v64i8;
       break;
     }
@@ -45497,6 +45506,11 @@ static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
   if (SDValue V = combineCastedMaskArithmetic(N, DAG, DCI, Subtarget))
     return V;
 
+  // bitcast(v1Ty insert_vector_elt(X, Y, 0)) --> Y
+  if (N0.getOpcode() == ISD::INSERT_VECTOR_ELT && SrcVT.getScalarType() == VT &&
+      SrcVT.getVectorNumElements() == 1)
+    return N0.getOperand(1);
+
   // Convert a bitcasted integer logic operation that has one bitcasted
   // floating-point operand into a floating-point logic operation. This may
   // create a load of a constant, but that is cheaper than materializing the
@@ -51561,7 +51575,7 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
               {SrcVec}, 0, SrcVec, ShuffleMask, {}, /*Depth*/ 1,
               X86::MaxShuffleCombineDepth,
               /*AllowVarCrossLaneMask*/ true,
-              /*AllowVarPerLaneMask*/ true, DAG, Subtarget))
+              /*AllowVarPerLaneMask*/ true, DAG, SDLoc(SrcVec), Subtarget))
         return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Shuffle,
                            N0.getOperand(1));
     }
@@ -57802,8 +57816,8 @@ CastIntSETCCtoFP(MVT VT, ISD::CondCode CC, unsigned NumSignificantBitsLHS,
 /// ISD::INSERT_SUBVECTOR). The ops are assumed to be of the same type.
 static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
                                       ArrayRef<SDValue> Ops, SelectionDAG &DAG,
-                                      TargetLowering::DAGCombinerInfo &DCI,
-                                      const X86Subtarget &Subtarget) {
+                                      const X86Subtarget &Subtarget,
+                                      unsigned Depth) {
   assert(Subtarget.hasAVX() && "AVX assumed for concat_vectors");
   unsigned EltSizeInBits = VT.getScalarSizeInBits();
 
@@ -57815,6 +57829,9 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
       }))
     return getZeroVector(VT, Subtarget, DAG, DL);
 
+  if (Depth >= SelectionDAG::MaxRecursionDepth)
+    return SDValue(); // Limit search depth.
+
   SDValue Op0 = Ops[0];
   bool IsSplat = llvm::all_equal(Ops);
   unsigned NumOps = Ops.size();
@@ -57945,25 +57962,47 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
       }
       return AllConstants || AllSubs;
     };
+    auto CombineSubOperand = [&](MVT VT, ArrayRef<SDValue> SubOps, unsigned I) {
+      bool AllConstants = true;
+      SmallVector<SDValue> Subs;
+      for (SDValue SubOp : SubOps) {
+        SDValue BC = peekThroughBitcasts(SubOp.getOperand(I));
+        AllConstants &= ISD::isBuildVectorOfConstantSDNodes(BC.getNode()) ||
+                        ISD::isBuildVectorOfConstantFPSDNodes(BC.getNode());
+        Subs.push_back(SubOp.getOperand(I));
+      }
+      if (AllConstants)
+        return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs);
+      return combineConcatVectorOps(DL, VT, Subs, DAG, Subtarget, Depth + 1);
+    };
 
     switch (Op0.getOpcode()) {
     case ISD::VECTOR_SHUFFLE: {
-      if (NumOps == 2 && VT.is256BitVector() &&
-          (EltSizeInBits >= 32 || Subtarget.hasInt256()) &&
-          (IsConcatFree(VT, Ops, 0) || IsConcatFree(VT, Ops, 1))) {
-        int NumSubElts = Op0.getValueType().getVectorNumElements();
-        SmallVector<int> NewMask;
-        for (int M : cast<ShuffleVectorSDNode>(Ops[0])->getMask()) {
-          M = M >= NumSubElts ? M + NumSubElts : M;
-          NewMask.push_back(M);
-        }
-        for (int M : cast<ShuffleVectorSDNode>(Ops[1])->getMask()) {
-          if (0 <= M)
-            M = (M >= NumSubElts ? M + NumSubElts : M) + NumSubElts;
-          NewMask.push_back(M);
+      // TODO: Relax VBMI requirement for repeated shuffle ops - currently
+      // limited to targets that should always have good cross lane shuffles.
+      if (!IsSplat && NumOps == 2 && VT.is256BitVector() &&
+          (EltSizeInBits >= 32 || Subtarget.hasInt256())) {
+        SDValue Concat0 = CombineSubOperand(VT, Ops, 0);
+        SDValue Concat1 = CombineSubOperand(VT, Ops, 1);
+        if (Concat0 || Concat1 ||
+            (Ops[0].getOperand(0) == Ops[1].getOperand(0) &&
+             Ops[0].getOperand(1) == Ops[1].getOperand(1) &&
+             Subtarget.hasVBMI())) {
+          int NumSubElts = Op0.getValueType().getVectorNumElements();
+          SmallVector<int> NewMask;
+          for (int M : cast<ShuffleVectorSDNode>(Ops[0])->getMask()) {
+            M = M >= NumSubElts ? M + NumSubElts : M;
+            NewMask.push_back(M);
+          }
+          for (int M : cast<ShuffleVectorSDNode>(Ops[1])->getMask()) {
+            if (0 <= M)
+              M = (M >= NumSubElts ? M + NumSubElts : M) + NumSubElts;
+            NewMask.push_back(M);
+          }
+          Concat0 = Concat0 ? Concat0 : ConcatSubOperand(VT, Ops, 0);
+          Concat1 = Concat1 ? Concat1 : ConcatSubOperand(VT, Ops, 1);
+          return DAG.getVectorShuffle(VT, DL, Concat0, Concat1, NewMask);
         }
-        return DAG.getVectorShuffle(VT, DL, ConcatSubOperand(VT, Ops, 0),
-                                    ConcatSubOperand(VT, Ops, 1), NewMask);
       }
       break;
     }
@@ -57987,20 +58026,27 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
     case X86ISD::MOVDDUP:
     case X86ISD::MOVSHDUP:
     case X86ISD::MOVSLDUP: {
-      if (!IsSplat)
+      if (!IsSplat && (VT.is256BitVector() ||
+                       (VT.is512BitVector() && Subtarget.useAVX512Regs())))
         return DAG.getNode(Op0.getOpcode(), DL, VT,
                            ConcatSubOperand(VT, Ops, 0));
       break;
     }
     case X86ISD::SHUFP: {
       // Add SHUFPD support if/when necessary.
-      if (!IsSplat && VT.getScalarType() == MVT::f32 &&
+      if (!IsSplat &&
+          (VT == MVT::v8f32 ||
+           (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) &&
           llvm::all_of(Ops, [Op0](SDValue Op) {
             return Op.getOperand(2) == Op0.getOperand(2);
           })) {
-        return DAG.getNode(Op0.getOpcode(), DL, VT,
-                           ConcatSubOperand(VT, Ops, 0),
-                           ConcatSubOperand(VT, Ops, 1), Op0.getOperand(2));
+        SDValue Concat0 = CombineSubOperand(VT, Ops, 0);
+        SDValue Concat1 = CombineSubOperand(VT, Ops, 1);
+        if (Concat0 || Concat1)
+          return DAG.getNode(Op0.getOpcode(), DL, VT,
+                             Concat0 ? Concat0 : ConcatSubOperand(VT, Ops, 0),
+                             Concat1 ? Concat1 : ConcatSubOperand(VT, Ops, 1),
+                             Op0.getOperand(2));
       }
       break;
     }
@@ -58026,7 +58072,9 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
     case X86ISD::PSHUFLW:
     case X86ISD::PSHUFD:
       if (!IsSplat && NumOps == 2 && VT.is256BitVector() &&
-          Subtarget.hasInt256() && Op0.getOperand(1) == Ops[1].getOperand(1)) {
+          Subtarget.hasInt256() && llvm::all_of(Ops, [Op0](SDValue Op) {
+            return Op.getOperand(1) == Op0.getOperand(1);
+          })) {
         return DAG.getNode(Op0.getOpcode(), DL, VT,
                            ConcatSubOperand(VT, Ops, 0), Op0.getOperand(1));
       }
@@ -58062,12 +58110,17 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
         MVT SrcVT = Op0.getOperand(0).getSimpleValueType();
         SrcVT = MVT::getVectorVT(SrcVT.getScalarType(),
                                  NumOps * SrcVT.getVectorNumElements());
-        return DAG.getNode(Op0.getOpcode(), DL, VT,
-                           ConcatSubOperand(SrcVT, Ops, 0),
-                           ConcatSubOperand(SrcVT, Ops, 1));
+        SDValue Concat0 = CombineSubOperand(SrcVT, Ops, 0);
+        SDValue Concat1 = CombineSubOperand(SrcVT, Ops, 1);
+        if (Concat0 || Concat1)
+          return DAG.getNode(
+              Op0.getOpcode(), DL, VT,
+              Concat0 ? Concat0 : ConcatSubOperand(SrcVT, Ops, 0),
+              Concat1 ? Concat1 : ConcatSubOperand(SrcVT, Ops, 1));
       }
       break;
     case X86ISD::VPERMV:
+      // TODO: Handle 256-bit and NumOps == 4 cases.
       if (!IsSplat && NumOps == 2 &&
           (VT.is512BitVector() && Subtarget.useAVX512Regs())) {
         MVT OpVT = Op0.getSimpleValueType();
@@ -58093,7 +58146,9 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
       }
       break;
     case X86ISD::VPERMV3:
-      if (!IsSplat && NumOps == 2 && VT.is512BitVector()) {
+      // TODO: Handle 256-bit and NumOps == 4 cases.
+      if (!IsSplat && NumOps == 2 &&
+          (VT.is512BitVector() && Subtarget.useAVX512Regs())) {
         MVT OpVT = Op0.getSimpleValueType();
         int NumSrcElts = OpVT.getVectorNumElements();
         SmallVector<int, 64> ConcatMask;
@@ -58115,10 +58170,14 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
           }
         }
         if (ConcatMask.size() == (NumOps * NumSrcElts)) {
-          SDValue Src0 = ConcatSubOperand(VT, Ops, 0);
-          SDValue Src1 = ConcatSubOperand(VT, Ops, 2);
-          return lowerShuffleWithPERMV(DL, VT, ConcatMask, Src0, Src1,
-                                       Subtarget, DAG);
+          SDValue Concat0 = CombineSubOperand(VT, Ops, 0);
+          SDValue Concat1 = CombineSubOperand(VT, Ops, 2);
+          if (Concat0 || Concat1)
+            return lowerShuffleWithPERMV(
+                DL, VT, ConcatMask,
+                Concat0 ? Concat0 : ConcatSubOperand(VT, Ops, 0),
+                Concat1 ? Concat1 : ConcatSubOperand(VT, Ops, 2), Subtarget,
+                DAG);
         }
       }
       break;
@@ -58266,23 +58325,36 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
     case ISD::OR:
     case ISD::XOR:
     case X86ISD::ANDNP:
-      if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) ||
+      // TODO: AVX2+ targets should only use CombineSubOperand like AVX1.
+      if (!IsSplat && (VT.is256BitVector() ||
                        (VT.is512BitVector() && Subtarget.useAVX512Regs()))) {
-        return DAG.getNode(Op0.getOpcode(), DL, VT,
-                           ConcatSubOperand(VT, Ops, 0),
-                           ConcatSubOperand(VT, Ops, 1));
+        // Don't concatenate root AVX1 NOT patterns.
+        if (Op0.getOpcode() == ISD::XOR && Depth == 0 &&
+            !Subtarget.hasInt256() && llvm::all_of(Ops, [](SDValue X) {
+              return ISD::isBuildVectorAllOnes(X.getOperand(1).getNode());
+            }))
+          break;
+        SDValue Concat0 = CombineSubOperand(VT, Ops, 0);
+        SDValue Concat1 = CombineSubOperand(VT, Ops, 1);
+        if (Concat0 || Concat1 || Subtarget.hasInt256())
+          return DAG.getNode(Op0.getOpcode(), DL, VT,
+                             Concat0 ? Concat0 : ConcatSubOperand(VT, Ops, 0),
+                             Concat1 ? Concat1 : ConcatSubOperand(VT, Ops, 1));
       }
       break;
     case X86ISD::PCMPEQ:
     case X86ISD::PCMPGT:
-      if (!IsSplat && VT.is256BitVector() &&
-          (Subtarget.hasInt256() || VT == MVT::v8i32) &&
-          (IsConcatFree(VT, Ops, 0) || IsConcatFree(VT, Ops, 1))) {
-        if (Subtarget.hasInt256())
+      if (!IsSplat && VT.is256BitVector() && Subtarget.hasInt256()) {
+        SDValue Concat0 = CombineSubOperand(VT, Ops, 0);
+        SDValue Concat1 = CombineSubOperand(VT, Ops, 1);
+        if (Concat0 || Concat1)
           return DAG.getNode(Op0.getOpcode(), DL, VT,
-                             ConcatSubOperand(VT, Ops, 0),
-                             ConcatSubOperand(VT, Ops, 1));
+                             Concat0 ? Concat0 : ConcatSubOperand(VT, Ops, 0),
+                             Concat1 ? Concat1 : ConcatSubOperand(VT, Ops, 1));
+        break;
+      }
 
+      if (!IsSplat && VT == MVT::v8i32) {
         // Without AVX2, see if we can cast the values to v8f32 and use fcmp.
         // TODO: Handle v4f64 as well?
         unsigned MaxSigBitsLHS = 0, MaxSigBitsRHS = 0;
@@ -58307,8 +58379,10 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
 
         if (std::optional<unsigned> CastOpc =
                 CastIntSETCCtoFP(FpVT, ICC, MaxSigBitsLHS, MaxSigBitsRHS)) {
-          SDValue LHS = ConcatSubOperand(VT, Ops, 0);
-          SDValue RHS = ConcatSubOperand(VT, Ops, 1);
+          SDValue LHS = CombineSubOperand(VT, Ops, 0);
+          SDValue RHS = CombineSubOperand(VT, Ops, 1);
+          LHS = LHS ? LHS : ConcatSubOperand(VT, Ops, 0);
+          RHS = RHS ? RHS : ConcatSubOperand(VT, Ops, 1);
           LHS = DAG.getNode(*CastOpc, DL, FpVT, LHS);
           RHS = DAG.getNode(*CastOpc, DL, FpVT, RHS);
 
@@ -58350,9 +58424,12 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
       if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) ||
                        (VT.is512BitVector() && Subtarget.useAVX512Regs() &&
                         (EltSizeInBits >= 32 || Subtarget.useBWIRegs())))) {
-        return DAG.getNode(Op0.getOpcode(), DL, VT,
-                           ConcatSubOperand(VT, Ops, 0),
-                           ConcatSubOperand(VT, Ops, 1));
+        SDValue Concat0 = CombineSubOperand(VT, Ops, 0);
+        SDValue Concat1 = CombineSubOperand(VT, Ops, 1);
+        if (Concat0 || Concat1)
+          return DAG.getNode(Op0.getOpcode(), DL, VT,
+                             Concat0 ? Concat0 : ConcatSubOperand(VT, Ops, 0),
+                             Concat1 ? Concat1 : ConcatSubOperand(VT, Ops, 1));
       }
       break;
     // Due to VADD, VSUB, VMUL can executed on more ports than VINSERT and
@@ -58361,14 +58438,17 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
     case ISD::FADD:
     case ISD::FSUB:
     case ISD::FMUL:
-      if (!IsSplat && (IsConcatFree(VT, Ops, 0) || IsConcatFree(VT, Ops, 1)) &&
-          (VT.is256BitVector() ||
-           (VT.is512BitVector() && Subtarget.useAVX512Regs()))) {
-        return DAG.getNode(Op0.getOpcode(), DL, VT,
-                           ConcatSubOperand(VT, Ops, 0),
-                           ConcatSubOperand(VT, Ops, 1));
+      if (!IsSplat && (VT.is256BitVector() ||
+                       (VT.is512BitVector() && Subtarget.useAVX512Regs()))) {
+        SDValue Concat0 = CombineSubOperand(VT, Ops, 0);
+        SDValue Concat1 = CombineSubOperand(VT, Ops, 1);
+        if (Concat0 || Concat1)
+          return DAG.getNode(Op0.getOpcode(), DL, VT,
+                             Concat0 ? Concat0 : ConcatSubOperand(VT, Ops, 0),
+                             Concat1 ? Concat1 : ConcatSubOperand(VT, Ops, 1));
       }
       break;
+    // Always prefer to concatenate high latency FDIV instructions.
     case ISD::FDIV:
       if (!IsSplat && (VT.is256BitVector() ||
                        (VT.is512BitVector() && Subtarget.useAVX512Regs()))) {
@@ -58383,9 +58463,12 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
     case X86ISD::FHSUB:
       if (!IsSplat && VT.is256BitVector() &&
           (VT.isFloatingPoint() || Subtarget.hasInt256())) {
-        return DAG.getNode(Op0.getOpcode(), DL, VT,
-                           ConcatSubOperand(VT, Ops, 0),
-                           ConcatSubOperand(VT, Ops, 1));
+        SDValue Concat0 = CombineSubOperand(VT, Ops, 0);
+        SDValue Concat1 = CombineSubOperand(VT, Ops, 1);
+        if (Concat0 || Concat1)
+          return DAG.getNode(Op0.getOpcode(), DL, VT,
+                             Concat0 ? Concat0 : ConcatSubOperand(VT, Ops, 0),
+                             Concat1 ? Concat1 : ConcatSubOperand(VT, Ops, 1));
       }
       break;
     case X86ISD::PACKSS:
@@ -58395,9 +58478,13 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
         MVT SrcVT = Op0.getOperand(0).getSimpleValueType();
         SrcVT = MVT::getVectorVT(SrcVT.getScalarType(),
                                  NumOps * SrcVT.getVectorNumElements());
-        return DAG.getNode(Op0.getOpcode(), DL, VT,
-                           ConcatSubOperand(SrcVT, Ops, 0),
-                           ConcatSubOperand(SrcVT, Ops, 1));
+        SDValue Concat0 = CombineSubOperand(SrcVT, Ops, 0);
+        SDValue Concat1 = CombineSubOperand(SrcVT, Ops, 1);
+        if (Concat0 || Concat1)
+          return DAG.getNode(
+              Op0.getOpcode(), DL, VT,
+              Concat0 ? Concat0 : ConcatSubOperand(SrcVT, Ops, 0),
+              Concat1 ? Concat1 : ConcatSubOperand(SrcVT, Ops, 1));
       }
       break;
     case X86ISD::PALIGNR:
@@ -58407,23 +58494,22 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
           llvm::all_of(Ops, [Op0](SDValue Op) {
             return Op0.getOperand(2) == Op.getOperand(2);
           })) {
-        return DAG.getNode(Op0.getOpcode(), DL, VT,
-                           ConcatSubOperand(VT, Ops, 0),
-                           ConcatSubOperand(VT, Ops, 1), Op0.getOperand(2));
+        SDValue Concat0 = CombineSubOperand(VT, Ops, 0);
+        SDValue Concat1 = CombineSubOperand(VT, Ops, 1);
+        if (Concat0 || Concat1)
+          return DAG.getNode(Op0.getOpcode(), DL, VT,
+                             Concat0 ? Concat0 : ConcatSubOperand(VT, Ops, 0),
+                             Concat1 ? Concat1 : ConcatSubOperand(VT, Ops, 1),
+                             Op0.getOperand(2));
       }
       break;
     case X86ISD::BLENDI:
       if (NumOps == 2 && VT.is512BitVector() && Subtarget.useBWIRegs()) {
-        uint64_t Mask0 = Ops[0].getConstantOperandVal(2);
-        uint64_t Mask1 = Ops[1].getConstantOperandVal(2);
-        // MVT::v16i16 has repeated blend mask.
-        if (Op0.getSimpleValueType() == MVT::v16i16) {
-          Mask0 = (Mask0 << 8) | Mask0;
-          Mask1 = (Mask1 << 8) | Mask1;
-        }
-        uint64_t Mask = (Mask1 << (VT.getVectorNumElements() / 2)) | Mask0;
-        MVT MaskSVT = MVT::getIntegerVT(VT.getVectorNumElements());
-        MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());
+        unsigned NumElts = VT.getVectorNumElements();
+        APInt Mask = getBLENDIBlendMask(Ops[0]).zext(NumElts);
+        Mask.insertBits(getBLENDIBlendMask(Ops[1]), NumElts / 2);
+        MVT MaskSVT = MVT::getIntegerVT(NumElts);
+        MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
         SDValue Sel =
             DAG.getBitcast(MaskVT, DAG.getConstant(Mask, DL, MaskSVT));
         return DAG.getSelect(DL, VT, Sel, ConcatSubOperand(VT, Ops, 1),
@@ -58569,7 +58655,7 @@ static SDValue combineCONCAT_VECTORS(SDNode *N, SelectionDAG &DAG,
 
   if (Subtarget.hasAVX() && TLI.isTypeLegal(VT) && TLI.isTypeLegal(SrcVT)) {
     if (SDValue R = combineConcatVectorOps(SDLoc(N), VT.getSimpleVT(), Ops, DAG,
-                                           DCI, Subtarget))
+                                           Subtarget))
       return R;
   }
 
@@ -58671,7 +58757,7 @@ static SDValue combineINSERT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
   SmallVector<SDValue, 2> SubVectorOps;
   if (collectConcatOps(N, SubVectorOps, DAG)) {
     if (SDValue Fold =
-            combineConcatVectorOps(dl, OpVT, SubVectorOps, DAG, DCI, Subtarget))
+            combineConcatVectorOps(dl, OpVT, SubVectorOps, DAG, Subtarget))
       return Fold;
 
     // If we're inserting all zeros into the upper half, change this to
@@ -58743,6 +58829,14 @@ static SDValue combineINSERT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
     }
   }
 
+  // Attempt to recursively combine to a shuffle.
+  if (isTargetShuffle(peekThroughBitcasts(Vec).getOpcode()) &&
+      isTargetShuffle(peekThroughBitcasts(SubVec).getOpcode())) {
+    SDValue Op(N, 0);
+    if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
+      return Res;
+  }
+
   return SDValue();
 }
 
diff --git a/llvm/lib/Target/X86/X86InstrBuilder.h b/llvm/lib/Target/X86/X86InstrBuilder.h
index 45c5f8aa82e97..98f1018d3938a 100644
--- a/llvm/lib/Target/X86/X86InstrBuilder.h
+++ b/llvm/lib/Target/X86/X86InstrBuilder.h
@@ -40,27 +40,20 @@ namespace llvm {
 /// with BP or SP and Disp being offsetted accordingly.  The displacement may
 /// also include the offset of a global value.
 struct X86AddressMode {
-  enum {
-    RegBase,
-    FrameIndexBase
-  } BaseType;
+  enum { RegBase, FrameIndexBase } BaseType = RegBase;
 
-  union {
-    unsigned Reg;
+  union BaseUnion {
+    Register Reg;
     int FrameIndex;
-  } Base;
 
-  unsigned Scale;
-  unsigned IndexReg;
-  int Disp;
-  const GlobalValue *GV;
-  unsigned GVOpFlags;
+    BaseUnion() : Reg() {}
+  } Base;
 
-  X86AddressMode()
-    : BaseType(RegBase), Scale(1), IndexReg(0), Disp(0), GV(nullptr),
-      GVOpFlags(0) {
-    Base.Reg = 0;
-  }
+  unsigned Scale = 1;
+  Register IndexReg;
+  int Disp = 0;
+  const GlobalValue *GV = nullptr;
+  unsigned GVOpFlags = 0;
 
   void getFullAddress(SmallVectorImpl<MachineOperand> &MO) {
     assert(Scale == 1 || Scale == 2 || Scale == 4 || Scale == 8);
@@ -121,7 +114,7 @@ static inline X86AddressMode getAddressFromInstr(const MachineInstr *MI,
 /// with no scale, index or displacement. An example is: DWORD PTR [EAX].
 ///
 static inline const MachineInstrBuilder &
-addDirectMem(const MachineInstrBuilder &MIB, unsigned Reg) {
+addDirectMem(const MachineInstrBuilder &MIB, Register Reg) {
   // Because memory references are always represented with five
   // values, this adds: Reg, 1, NoReg, 0, NoReg to the instruction.
   return MIB.addReg(Reg).addImm(1).addReg(0).addImm(0).addReg(0);
@@ -130,7 +123,7 @@ addDirectMem(const MachineInstrBuilder &MIB, unsigned Reg) {
 /// Replace the address used in the instruction with the direct memory
 /// reference.
 static inline void setDirectAddressInInstr(MachineInstr *MI, unsigned Operand,
-                                           unsigned Reg) {
+                                           Register Reg) {
   // Direct memory address is in a form of: Reg/FI, 1 (Scale), NoReg, 0, NoReg.
   MI->getOperand(Operand).ChangeToRegister(Reg, /*isDef=*/false);
   MI->getOperand(Operand + 1).setImm(1);
@@ -154,16 +147,16 @@ addOffset(const MachineInstrBuilder &MIB, const MachineOperand& Offset) {
 /// displacement. An example is: DWORD PTR [EAX + 4].
 ///
 static inline const MachineInstrBuilder &
-addRegOffset(const MachineInstrBuilder &MIB,
-             unsigned Reg, bool isKill, int Offset) {
+addRegOffset(const MachineInstrBuilder &MIB, Register Reg, bool isKill,
+             int Offset) {
   return addOffset(MIB.addReg(Reg, getKillRegState(isKill)), Offset);
 }
 
 /// addRegReg - This function is used to add a memory reference of the form:
 /// [Reg + Reg].
 static inline const MachineInstrBuilder &
-addRegReg(const MachineInstrBuilder &MIB, unsigned Reg1, bool isKill1,
-          unsigned SubReg1, unsigned Reg2, bool isKill2, unsigned SubReg2) {
+addRegReg(const MachineInstrBuilder &MIB, Register Reg1, bool isKill1,
+          unsigned SubReg1, Register Reg2, bool isKill2, unsigned SubReg2) {
   return MIB.addReg(Reg1, getKillRegState(isKill1), SubReg1)
       .addImm(1)
       .addReg(Reg2, getKillRegState(isKill2), SubReg2)
@@ -224,7 +217,7 @@ addFrameReference(const MachineInstrBuilder &MIB, int FI, int Offset = 0) {
 ///
 static inline const MachineInstrBuilder &
 addConstantPoolReference(const MachineInstrBuilder &MIB, unsigned CPI,
-                         unsigned GlobalBaseReg, unsigned char OpFlags) {
+                         Register GlobalBaseReg, unsigned char OpFlags) {
   //FIXME: factor this
   return MIB.addReg(GlobalBaseReg).addImm(1).addReg(0)
     .addConstantPoolIndex(CPI, 0, OpFlags).addReg(0);
diff --git a/llvm/lib/Target/X86/X86InstrConditionalCompare.td b/llvm/lib/Target/X86/X86InstrConditionalCompare.td
index 35af8405f1abe..ba8cf6cc3bc67 100644
--- a/llvm/lib/Target/X86/X86InstrConditionalCompare.td
+++ b/llvm/lib/Target/X86/X86InstrConditionalCompare.td
@@ -36,7 +36,7 @@ class Ctest<bits<8> o, Format f, X86TypeInfo t, DAGOperand op1, DAGOperand op2>:
 //===----------------------------------------------------------------------===//
 // CCMP Instructions
 //
-let SchedRW = [WriteALU] in {
+let SchedRW = [WriteALU], isCompare = 1 in {
   def CCMP8rr : Ccmp<0x38, MRMDestReg, Xi8,  GR8,  GR8>;
   def CCMP16rr: Ccmp<0x39, MRMDestReg, Xi16, GR16, GR16>, PD;
   def CCMP32rr: Ccmp<0x39, MRMDestReg, Xi32, GR32, GR32>;
@@ -55,7 +55,7 @@ let SchedRW = [WriteALU] in {
   def CCMP64ri32: Ccmp<0x81, MRM7r, Xi64, GR64, i64i32imm>;
 }
 
-let mayLoad = 1 in {
+let mayLoad = 1, isCompare = 1 in {
   let SchedRW = [WriteALU.Folded] in {
     def CCMP16mi8: Ccmp<0x83, MRM7m, Xi16, i16mem, i16i8imm>, PD;
     def CCMP32mi8: Ccmp<0x83, MRM7m, Xi32, i32mem, i32i8imm>;
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 5fe7203c052d8..24002cbbcab57 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -3304,10 +3304,28 @@ static unsigned getNewOpcFromTable(ArrayRef<X86TableEntry> Table,
   return (I == Table.end() || I->OldOpc != Opc) ? 0U : I->NewOpc;
 }
 unsigned X86::getNFVariant(unsigned Opc) {
+#if defined(EXPENSIVE_CHECKS) && !defined(NDEBUG)
+  // Make sure the tables are sorted.
+  static std::atomic<bool> NFTableChecked(false);
+  if (!NFTableChecked.load(std::memory_order_relaxed)) {
+    assert(llvm::is_sorted(X86NFTransformTable) &&
+           "X86NFTransformTable is not sorted!");
+    NFTableChecked.store(true, std::memory_order_relaxed);
+  }
+#endif
   return getNewOpcFromTable(X86NFTransformTable, Opc);
 }
 
 unsigned X86::getNonNDVariant(unsigned Opc) {
+#if defined(EXPENSIVE_CHECKS) && !defined(NDEBUG)
+  // Make sure the tables are sorted.
+  static std::atomic<bool> NDTableChecked(false);
+  if (!NDTableChecked.load(std::memory_order_relaxed)) {
+    assert(llvm::is_sorted(X86ND2NonNDTable) &&
+           "X86ND2NonNDTableis not sorted!");
+    NDTableChecked.store(true, std::memory_order_relaxed);
+  }
+#endif
   return getNewOpcFromTable(X86ND2NonNDTable, Opc);
 }
 
@@ -4854,6 +4872,10 @@ bool X86InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
   case X86::CMP32ri:
   case X86::CMP16ri:
   case X86::CMP8ri:
+  case X86::CCMP64ri32:
+  case X86::CCMP32ri:
+  case X86::CCMP16ri:
+  case X86::CCMP8ri:
     SrcReg = MI.getOperand(0).getReg();
     SrcReg2 = 0;
     if (MI.getOperand(1).isImm()) {
@@ -4951,6 +4973,18 @@ bool X86InstrInfo::isRedundantFlagInstr(const MachineInstr &FlagI,
     }
     return false;
   }
+  case X86::CCMP64ri32:
+  case X86::CCMP32ri:
+  case X86::CCMP16ri:
+  case X86::CCMP8ri: {
+    // The CCMP instruction should not be optimized if the scc/dfv in it is not
+    // same as the one in previous CCMP instruction.
+    if ((FlagI.getOpcode() != OI.getOpcode()) ||
+        (OI.getOperand(2).getImm() != FlagI.getOperand(2).getImm()) ||
+        (OI.getOperand(3).getImm() != FlagI.getOperand(3).getImm()))
+      return false;
+    [[fallthrough]];
+  }
   case X86::CMP64ri32:
   case X86::CMP32ri:
   case X86::CMP16ri:
@@ -5352,10 +5386,12 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
   MachineInstr *MI = nullptr;
   MachineInstr *Sub = nullptr;
   MachineInstr *Movr0Inst = nullptr;
+  SmallVector<std::pair<MachineInstr *, unsigned>, 4> InstsToUpdate;
   bool NoSignFlag = false;
   bool ClearsOverflowFlag = false;
   bool ShouldUpdateCC = false;
   bool IsSwapped = false;
+  bool HasNF = Subtarget.hasNF();
   unsigned OpNo = 0;
   X86::CondCode NewCC = X86::COND_INVALID;
   int64_t ImmDelta = 0;
@@ -5441,6 +5477,16 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
           continue;
         }
 
+        // Try to replace non-NF with NF instructions.
+        if (HasNF && Inst.registerDefIsDead(X86::EFLAGS, TRI)) {
+          unsigned NewOp = X86::getNFVariant(Inst.getOpcode());
+          if (!NewOp)
+            return false;
+
+          InstsToUpdate.push_back(std::make_pair(&Inst, NewOp));
+          continue;
+        }
+
         // Cannot do anything for any other EFLAG changes.
         return false;
       }
@@ -5637,6 +5683,12 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
       return false;
   }
 
+  // Replace non-NF with NF instructions.
+  for (auto &Inst : InstsToUpdate) {
+    Inst.first->setDesc(get(Inst.second));
+    Inst.first->removeOperand(Inst.first->getNumOperands() - 1);
+  }
+
   // Make sure Sub instruction defines EFLAGS and mark the def live.
   MachineOperand *FlagDef =
       Sub->findRegisterDefOperand(X86::EFLAGS, /*TRI=*/nullptr);
diff --git a/llvm/lib/Target/X86/X86InstrSSE.td b/llvm/lib/Target/X86/X86InstrSSE.td
index 6aadb788c851e..2a7ab1e310618 100644
--- a/llvm/lib/Target/X86/X86InstrSSE.td
+++ b/llvm/lib/Target/X86/X86InstrSSE.td
@@ -6121,8 +6121,9 @@ let Predicates = [HasAVX, NoAVX10_2] in {
                                     v8i16, VR128, load, i128mem, 0,
                                     SchedWriteMPSAD.XMM>, VEX, VVVV, WIG;
   }
+}
 
-let Uses = [MXCSR], mayRaiseFPException = 1 in {
+let Predicates = [HasAVX], Uses = [MXCSR], mayRaiseFPException = 1 in {
   let ExeDomain = SSEPackedSingle in
   defm VDPPS : SS41I_binop_rmi_int<0x40, "vdpps", int_x86_sse41_dpps,
                                    VR128, load, f128mem, 0,
@@ -6136,7 +6137,6 @@ let Uses = [MXCSR], mayRaiseFPException = 1 in {
                                     VR256, load, i256mem, 0,
                                     SchedWriteDPPS.YMM>, VEX, VVVV, VEX_L, WIG;
 }
-}
 
 let Predicates = [HasAVX2, NoAVX10_2] in {
   let isCommutable = 0 in {
diff --git a/llvm/lib/Target/X86/X86PreTileConfig.cpp b/llvm/lib/Target/X86/X86PreTileConfig.cpp
index e8d90da1fb1e0..d176778afef92 100644
--- a/llvm/lib/Target/X86/X86PreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86PreTileConfig.cpp
@@ -293,29 +293,30 @@ bool X86PreTileConfig::runOnMachineFunction(MachineFunction &MF) {
   SmallVector<MachineBasicBlock *, 8> CfgLiveInBBs;
   for (auto &MBB : MF) {
     size_t Pos = 0;
+    auto &Info = BBVisitedInfo[&MBB];
     for (auto &MI : MBB) {
       ++Pos;
       if (isAMXInstruction(MI)) {
         // If there's call before the AMX, we need to reload tile config.
-        if (BBVisitedInfo[&MBB].LastCall)
-          CfgNeedInsert.insert(BBVisitedInfo[&MBB].LastCall);
+        if (Info.LastCall)
+          CfgNeedInsert.insert(Info.LastCall);
         else // Otherwise, we need tile config to live in this BB.
-          BBVisitedInfo[&MBB].NeedTileCfgLiveIn = true;
+          Info.NeedTileCfgLiveIn = true;
         // Always record the first AMX in case there's shape def after it.
-        if (!BBVisitedInfo[&MBB].FirstAMX)
-          BBVisitedInfo[&MBB].FirstAMX = MIRef(&MI, &MBB, Pos);
+        if (!Info.FirstAMX)
+          Info.FirstAMX = MIRef(&MI, &MBB, Pos);
       } else if (MI.isCall() && isDestructiveCall(MI, AMXRegs)) {
         // Record the call only if the callee clobbers all AMX registers.
-        BBVisitedInfo[&MBB].LastCall = MIRef(&MI, &MBB, Pos);
+        Info.LastCall = MIRef(&MI, &MBB, Pos);
       }
     }
-    if (BBVisitedInfo[&MBB].NeedTileCfgLiveIn) {
+    if (Info.NeedTileCfgLiveIn) {
       if (&MBB == &MF.front())
         CfgNeedInsert.insert(MIRef(&MBB));
       else
         CfgLiveInBBs.push_back(&MBB);
     }
-    if (BBVisitedInfo[&MBB].FirstAMX || BBVisitedInfo[&MBB].HasAMXRegLiveIn)
+    if (Info.FirstAMX || Info.HasAMXRegLiveIn)
       for (auto *Succ : MBB.successors())
         if (!isLoopBackEdge(Succ, &MBB))
           BBVisitedInfo[Succ].HasAMXRegLiveIn = true;
@@ -325,10 +326,11 @@ bool X86PreTileConfig::runOnMachineFunction(MachineFunction &MF) {
   while (!CfgLiveInBBs.empty()) {
     MachineBasicBlock *MBB = CfgLiveInBBs.pop_back_val();
     for (auto *Pred : MBB->predecessors()) {
+      auto &Info = BBVisitedInfo[Pred];
       if (BBVisitedInfo[Pred].LastCall) {
-        CfgNeedInsert.insert(BBVisitedInfo[Pred].LastCall);
-      } else if (!BBVisitedInfo[Pred].NeedTileCfgLiveIn) {
-        BBVisitedInfo[Pred].NeedTileCfgLiveIn = true;
+        CfgNeedInsert.insert(Info.LastCall);
+      } else if (!Info.NeedTileCfgLiveIn) {
+        Info.NeedTileCfgLiveIn = true;
         if (Pred == &MF.front())
           CfgNeedInsert.insert(MIRef(Pred));
         else
@@ -344,16 +346,16 @@ bool X86PreTileConfig::runOnMachineFunction(MachineFunction &MF) {
   // Avoid to insert ldtilecfg before any shape defs.
   SmallVector<MachineBasicBlock *, 8> WorkList;
   for (auto &I : ShapeBBs) {
+    auto &Info = BBVisitedInfo[I.first];
     // TODO: We can hoist shapes across BBs here.
-    if (BBVisitedInfo[I.first].HasAMXRegLiveIn) {
+    if (Info.HasAMXRegLiveIn) {
       // We are not able to config tile registers since the shape to config
       // is not defined yet. Emit error message and continue. The function
       // would not config tile registers.
       emitErrorMsg(MF);
       return false;
     }
-    if (BBVisitedInfo[I.first].FirstAMX &&
-        BBVisitedInfo[I.first].FirstAMX < I.second.back() &&
+    if (Info.FirstAMX && Info.FirstAMX < I.second.back() &&
         !hoistShapesInBB(I.first, I.second)) {
       emitErrorMsg(MF);
       return false;
@@ -363,8 +365,9 @@ bool X86PreTileConfig::runOnMachineFunction(MachineFunction &MF) {
   while (!WorkList.empty()) {
     MachineBasicBlock *MBB = WorkList.pop_back_val();
     for (auto *Pred : MBB->predecessors()) {
-      if (!BBVisitedInfo[Pred].TileCfgForbidden && !isLoopBackEdge(MBB, Pred)) {
-        BBVisitedInfo[Pred].TileCfgForbidden = true;
+      auto &Info = BBVisitedInfo[Pred];
+      if (!Info.TileCfgForbidden && !isLoopBackEdge(MBB, Pred)) {
+        Info.TileCfgForbidden = true;
         WorkList.push_back(Pred);
       }
     }
@@ -400,8 +403,9 @@ bool X86PreTileConfig::runOnMachineFunction(MachineFunction &MF) {
     // A given point might be forked due to shape conditions are not met.
     for (MIRef I : InsertPoints) {
       // Make sure we insert ldtilecfg after the last shape def in MBB.
-      if (ShapeBBs.count(I.MBB) && I < ShapeBBs[I.MBB].back())
-        I = ShapeBBs[I.MBB].back();
+      auto It = ShapeBBs.find(I.MBB);
+      if (It != ShapeBBs.end() && I < It->second.back())
+        I = It->second.back();
       // There're chances the MBB is sunk more than once. Record it to avoid
       // multi insert.
       if (VisitedOrInserted.insert(I).second) {
diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h
index 722076ca88c9c..8f2d326a69398 100644
--- a/llvm/lib/Target/X86/X86Subtarget.h
+++ b/llvm/lib/Target/X86/X86Subtarget.h
@@ -280,6 +280,9 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   /// supports it.
   bool hasMFence() const { return hasSSE2() || is64Bit(); }
 
+  /// Avoid use of `mfence` for`fence seq_cst`, and instead use `lock or`.
+  bool avoidMFence() const { return is64Bit(); }
+
   const Triple &getTargetTriple() const { return TargetTriple; }
 
   bool isTargetDarwin() const { return TargetTriple.isOSDarwin(); }
diff --git a/llvm/lib/Target/Xtensa/Disassembler/XtensaDisassembler.cpp b/llvm/lib/Target/Xtensa/Disassembler/XtensaDisassembler.cpp
index b1027488904df..b949b893953fe 100644
--- a/llvm/lib/Target/Xtensa/Disassembler/XtensaDisassembler.cpp
+++ b/llvm/lib/Target/Xtensa/Disassembler/XtensaDisassembler.cpp
@@ -73,8 +73,50 @@ static DecodeStatus DecodeARRegisterClass(MCInst &Inst, uint64_t RegNo,
   return MCDisassembler::Success;
 }
 
+static const MCPhysReg MRDecoderTable[] = {Xtensa::M0, Xtensa::M1, Xtensa::M2,
+                                           Xtensa::M3};
+
+static DecodeStatus DecodeMRRegisterClass(MCInst &Inst, uint64_t RegNo,
+                                          uint64_t Address,
+                                          const void *Decoder) {
+  if (RegNo >= std::size(MRDecoderTable))
+    return MCDisassembler::Fail;
+
+  MCPhysReg Reg = MRDecoderTable[RegNo];
+  Inst.addOperand(MCOperand::createReg(Reg));
+  return MCDisassembler::Success;
+}
+
+static const MCPhysReg MR01DecoderTable[] = {Xtensa::M0, Xtensa::M1};
+
+static DecodeStatus DecodeMR01RegisterClass(MCInst &Inst, uint64_t RegNo,
+                                            uint64_t Address,
+                                            const void *Decoder) {
+  if (RegNo > 2)
+    return MCDisassembler::Fail;
+
+  MCPhysReg Reg = MR01DecoderTable[RegNo];
+  Inst.addOperand(MCOperand::createReg(Reg));
+  return MCDisassembler::Success;
+}
+
+static const MCPhysReg MR23DecoderTable[] = {Xtensa::M2, Xtensa::M3};
+
+static DecodeStatus DecodeMR23RegisterClass(MCInst &Inst, uint64_t RegNo,
+                                            uint64_t Address,
+                                            const void *Decoder) {
+  if (RegNo != 2 && RegNo != 3)
+    return MCDisassembler::Fail;
+
+  MCPhysReg Reg = MR23DecoderTable[RegNo - 2];
+  Inst.addOperand(MCOperand::createReg(Reg));
+  return MCDisassembler::Success;
+}
+
 const MCPhysReg SRDecoderTable[] = {
-    Xtensa::SAR, 3, Xtensa::WINDOWBASE, 72, Xtensa::WINDOWSTART, 73};
+    Xtensa::SAR, 3,  Xtensa::ACCLO,      16, Xtensa::ACCHI,       17,
+    Xtensa::M0,  32, Xtensa::M1,         33, Xtensa::M2,          34,
+    Xtensa::M3,  35, Xtensa::WINDOWBASE, 72, Xtensa::WINDOWSTART, 73};
 
 static DecodeStatus DecodeSRRegisterClass(MCInst &Inst, uint64_t RegNo,
                                           uint64_t Address,
diff --git a/llvm/lib/Target/Xtensa/XtensaDSPInstrInfo.td b/llvm/lib/Target/Xtensa/XtensaDSPInstrInfo.td
new file mode 100644
index 0000000000000..d80df46320643
--- /dev/null
+++ b/llvm/lib/Target/Xtensa/XtensaDSPInstrInfo.td
@@ -0,0 +1,353 @@
+//===- XtensaDSPInstrInfo.td - Xtensa Target Description ---*- tablegen -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the Xtensa DSP instructions in TableGen format.
+//
+//===----------------------------------------------------------------------===//
+
+// Multiply
+class UMUL_AA<bits<4> oper1, string instrAsm>
+  : RRR_Inst<0x04, oper1, 0x07, (outs), (ins AR:$s, AR:$t),
+             instrAsm#"\t$s, $t", []>, Requires<[HasMAC16]> {
+  let r = 0;
+  let Defs = [M1, M2, ACCLO, ACCHI];
+}
+
+def UMUL_AA_LL : UMUL_AA<0x00, "umul.aa.ll">;
+def UMUL_AA_HL : UMUL_AA<0x01, "umul.aa.hl">;
+def UMUL_AA_LH : UMUL_AA<0x02, "umul.aa.lh">;
+def UMUL_AA_HH : UMUL_AA<0x03, "umul.aa.hh">;
+
+class MUL_AA<bits<4> oper1, string instrAsm>
+  : RRR_Inst<0x04, oper1, 0x07, (outs), (ins AR:$s, AR:$t),
+             instrAsm#"\t$s, $t", []>, Requires<[HasMAC16]> {
+  let r = 0;
+  let Defs = [M1, M2, ACCLO, ACCHI];
+}
+
+def MUL_AA_LL : MUL_AA<0x04, "mul.aa.ll">;
+def MUL_AA_HL : MUL_AA<0x05, "mul.aa.hl">;
+def MUL_AA_LH : MUL_AA<0x06, "mul.aa.lh">;
+def MUL_AA_HH : MUL_AA<0x07, "mul.aa.hh">;
+
+class MUL_AD<bits<4> oper1, string instrAsm>
+  : RRR_Inst<0x04, oper1, 0x03, (outs), (ins AR:$s, MR23:$y),
+             instrAsm#"\t$s, $y", []>, Requires<[HasMAC16]> {
+  bits<2> y;
+
+  let r = 0;
+  let t{3} = 0;
+  let t{2} = y{0};
+  let t{1-0} = 0;
+  let Defs = [M1, M2, ACCLO, ACCHI];
+}
+
+def MUL_AD_LL : MUL_AD<0x04, "mul.ad.ll">;
+def MUL_AD_HL : MUL_AD<0x05, "mul.ad.hl">;
+def MUL_AD_LH : MUL_AD<0x06, "mul.ad.lh">;
+def MUL_AD_HH : MUL_AD<0x07, "mul.ad.hh">;
+
+class MUL_DA<bits<4> oper1, string instrAsm>
+  : RRR_Inst<0x04, oper1, 0x06, (outs), (ins MR01:$x, AR:$t),
+             instrAsm#"\t$x, $t", []>, Requires<[HasMAC16]> {
+  bits<2> x;
+
+  let r{3} = 0;
+  let r{2} = x{0};
+  let r{1-0} = 0;
+  let s = 0;
+  let Defs = [M1, M2, ACCLO, ACCHI];
+}
+
+def MUL_DA_LL : MUL_DA<0x04, "mul.da.ll">;
+def MUL_DA_HL : MUL_DA<0x05, "mul.da.hl">;
+def MUL_DA_LH : MUL_DA<0x06, "mul.da.lh">;
+def MUL_DA_HH : MUL_DA<0x07, "mul.da.hh">;
+
+class MUL_DD<bits<4> oper1, string instrAsm>
+  : RRR_Inst<0x04, oper1, 0x02, (outs), (ins MR01:$x, MR23:$y),
+             instrAsm#"\t$x, $y", []>, Requires<[HasMAC16]> {
+  bits<2> x;
+  bits<2> y;
+
+  let r{3} = 0;
+  let r{2} = x{0};
+  let r{1-0} = 0;
+  let s = 0;
+  let t{3} = 0;
+  let t{2} = y{0};
+  let t{1-0} = 0;
+  let Defs = [M1, M2, ACCLO, ACCHI];
+}
+
+def MUL_DD_LL : MUL_DD<0x04, "mul.dd.ll">;
+def MUL_DD_HL : MUL_DD<0x05, "mul.dd.hl">;
+def MUL_DD_LH : MUL_DD<0x06, "mul.dd.lh">;
+def MUL_DD_HH : MUL_DD<0x07, "mul.dd.hh">;
+
+class MULA_AA<bits<4> oper1, string instrAsm>
+  : RRR_Inst<0x04, oper1, 0x07, (outs), (ins AR:$s, AR:$t),
+             instrAsm#"\t$s, $t", []>, Requires<[HasMAC16]> {
+  let r = 0;
+  let Defs = [M1, M2, ACCLO, ACCHI];
+}
+
+def MULA_AA_LL : MULA_AA<0x08, "mula.aa.ll">;
+def MULA_AA_HL : MULA_AA<0x09, "mula.aa.hl">;
+def MULA_AA_LH : MULA_AA<0x0A, "mula.aa.lh">;
+def MULA_AA_HH : MULA_AA<0x0B, "mula.aa.hh">;
+
+class MULA_AD<bits<4> oper1, string instrAsm>
+  : RRR_Inst<0x04, oper1, 0x03, (outs), (ins AR:$s, MR23:$y),
+             instrAsm#"\t$s, $y", []>, Requires<[HasMAC16]> {
+  bits<2> y;
+
+  let r = 0;
+  let t{3} = 0;
+  let t{2} = y{0};
+  let t{1-0} = 0;
+
+  let Uses = [ACCLO, ACCHI];
+  let Defs = [M1, M2, ACCLO, ACCHI];
+}
+
+def MULA_AD_LL : MULA_AD<0x08, "mula.ad.ll">;
+def MULA_AD_HL : MULA_AD<0x09, "mula.ad.hl">;
+def MULA_AD_LH : MULA_AD<0x0A, "mula.ad.lh">;
+def MULA_AD_HH : MULA_AD<0x0B, "mula.ad.hh">;
+
+class MULA_DA<bits<4> oper1, string instrAsm>
+  : RRR_Inst<0x04, oper1, 0x06, (outs), (ins MR01:$x, AR:$t),
+             instrAsm#"\t$x, $t", []>, Requires<[HasMAC16]> {
+  bits<2> x;
+
+  let r{3} = 0;
+  let r{2} = x{0};
+  let r{1-0} = 0;
+  let s = 0;
+
+  let Uses = [ACCLO, ACCHI];
+  let Defs = [M1, M2, ACCLO, ACCHI];
+}
+
+def MULA_DA_LL : MULA_DA<0x08, "mula.da.ll">;
+def MULA_DA_HL : MULA_DA<0x09, "mula.da.hl">;
+def MULA_DA_LH : MULA_DA<0x0A, "mula.da.lh">;
+def MULA_DA_HH : MULA_DA<0x0B, "mula.da.hh">;
+
+class MULA_DD<bits<4> oper1, string instrAsm>
+  : RRR_Inst<0x04, oper1, 0x02, (outs), (ins MR01:$x, MR23:$y),
+             instrAsm#"\t$x, $y", []>, Requires<[HasMAC16]> {
+  bits<2> x;
+  bits<2> y;
+
+  let r{3} = 0;
+  let r{2} = x{0};
+  let r{1-0} = 0;
+  let s = 0;
+  let t{3} = 0;
+  let t{2} = y{0};
+  let t{1-0} = 0;
+
+  let Uses = [ACCLO, ACCHI];
+  let Defs = [M1, M2, ACCLO, ACCHI];
+}
+
+def MULA_DD_LL : MULA_DD<0x08, "mula.dd.ll">;
+def MULA_DD_HL : MULA_DD<0x09, "mula.dd.hl">;
+def MULA_DD_LH : MULA_DD<0x0A, "mula.dd.lh">;
+def MULA_DD_HH : MULA_DD<0x0B, "mula.dd.hh">;
+
+class MULS_AA<bits<4> oper1, string instrAsm>
+  : RRR_Inst<0x04, oper1, 0x07, (outs), (ins AR:$s, AR:$t),
+             instrAsm#"\t$s, $t", []>, Requires<[HasMAC16]> {
+  let r = 0;
+  let Uses = [ACCLO, ACCHI];
+  let Defs = [M1, M2, ACCLO, ACCHI];
+}
+
+def MULS_AA_LL : MULS_AA<0x0C, "muls.aa.ll">;
+def MULS_AA_HL : MULS_AA<0x0D, "muls.aa.hl">;
+def MULS_AA_LH : MULS_AA<0x0E, "muls.aa.lh">;
+def MULS_AA_HH : MULS_AA<0x0F, "muls.aa.hh">;
+
+class MULS_AD<bits<4> oper1, string instrAsm>
+  : RRR_Inst<0x04, oper1, 0x03, (outs), (ins AR:$s, MR23:$y),
+             instrAsm#"\t$s, $y", []>, Requires<[HasMAC16]> {
+  bits<2> y;
+
+  let r = 0;
+  let t{3} = 0;
+  let t{2} = y{0};
+  let t{1-0} = 0;
+
+  let Uses = [ACCLO, ACCHI];
+  let Defs = [M1, M2, ACCLO, ACCHI];
+}
+
+def MULS_AD_LL : MULS_AD<0x0C, "muls.ad.ll">;
+def MULS_AD_HL : MULS_AD<0x0D, "muls.ad.hl">;
+def MULS_AD_LH : MULS_AD<0x0E, "muls.ad.lh">;
+def MULS_AD_HH : MULS_AD<0x0F, "muls.ad.hh">;
+
+class MULS_DA<bits<4> oper1, string instrAsm>
+  : RRR_Inst<0x04, oper1, 0x06, (outs), (ins MR01:$x, AR:$t),
+             instrAsm#"\t$x, $t", []>, Requires<[HasMAC16]> {
+  bits<2> x;
+
+  let r{3} = 0;
+  let r{2} = x{0};
+  let r{1-0} = 0;
+  let s = 0;
+
+  let Uses = [ACCLO, ACCHI];
+  let Defs = [M1, M2, ACCLO, ACCHI];
+}
+
+def MULS_DA_LL : MULS_DA<0x0C, "muls.da.ll">;
+def MULS_DA_HL : MULS_DA<0x0D, "muls.da.hl">;
+def MULS_DA_LH : MULS_DA<0x0E, "muls.da.lh">;
+def MULS_DA_HH : MULS_DA<0x0F, "muls.da.hh">;
+
+class MULS_DD<bits<4> oper1, string instrAsm>
+  : RRR_Inst<0x04, oper1, 0x02, (outs), (ins MR01:$x, MR23:$y),
+             instrAsm#"\t$x, $y", []>, Requires<[HasMAC16]> {
+  bits<2> x;
+  bits<2> y;
+
+  let r{3} = 0;
+  let r{2} = x{0};
+  let r{1-0} = 0;
+  let s = 0;
+  let t{3} = 0;
+  let t{2} = y{0};
+  let t{1-0} = 0;
+
+  let Uses = [ACCLO, ACCHI];
+  let Defs = [M1, M2, ACCLO, ACCHI];
+}
+
+def MULS_DD_LL : MULS_DD<0x0C, "muls.dd.ll">;
+def MULS_DD_HL : MULS_DD<0x0D, "muls.dd.hl">;
+def MULS_DD_LH : MULS_DD<0x0E, "muls.dd.lh">;
+def MULS_DD_HH : MULS_DD<0x0F, "muls.dd.hh">;
+
+//===----------------------------------------------------------------------===//
+// Multiply-accumulate with load
+
+class MULA_DA_LDDEC<bits<4> oper1, string instrAsm>
+  : RRR_Inst<0x04, oper1, 0x05, (outs MR:$w, AR:$d), (ins AR:$s, MR01:$x, AR:$t),
+             instrAsm#"\t $w, $s, $x, $t", []>, Requires<[HasMAC16]> {
+  bits<2> x;
+  bits<2> w;
+
+  let Constraints = "$s = $d";
+  let mayLoad = 1;
+  let r{3} = 0;
+  let r{2} = x{0};
+  let r{1-0} = w{1-0};
+  let Uses = [ACCLO, ACCHI];
+  let Defs = [M1, M2, ACCLO, ACCHI];
+}
+
+def MULA_DA_LL_LDDEC : MULA_DA_LDDEC<0x08, "mula.da.ll.lddec">;
+def MULA_DA_HL_LDDEC : MULA_DA_LDDEC<0x09, "mula.da.hl.lddec">;
+def MULA_DA_LH_LDDEC : MULA_DA_LDDEC<0x0A, "mula.da.lh.lddec">;
+def MULA_DA_HH_LDDEC : MULA_DA_LDDEC<0x0B, "mula.da.hh.lddec">;
+
+class MULA_DA_LDINC<bits<4> oper1, string instrAsm>
+  : RRR_Inst<0x04, oper1, 0x04, (outs MR:$w, AR:$d), (ins AR:$s, MR:$x, AR:$t),
+             instrAsm#"\t $w, $s, $x, $t", []>, Requires<[HasMAC16]> {
+  bits<1> x;
+  bits<2> w;
+
+  let Constraints = "$s = $d";
+  let mayLoad = 1;
+  let r{3} = 0;
+  let r{2} = x{0};
+  let r{1-0} = w{1-0};
+  let Uses = [ACCLO, ACCHI];
+  let Defs = [M1, M2, ACCLO, ACCHI];
+}
+
+def MULA_DA_LL_LDINC: MULA_DA_LDINC<0x08, "mula.da.ll.ldinc">;
+def MULA_DA_HL_LDINC: MULA_DA_LDINC<0x09, "mula.da.hl.ldinc">;
+def MULA_DA_LH_LDINC: MULA_DA_LDINC<0x0A, "mula.da.lh.ldinc">;
+def MULA_DA_HH_LDINC: MULA_DA_LDINC<0x0B, "mula.da.hh.ldinc">;
+
+class MULA_DD_LDDEC<bits<4> oper1, string instrAsm>
+  : RRR_Inst<0x04, oper1, 0x01, (outs MR:$w, AR:$d), (ins AR:$s, MR01:$x, MR23:$y),
+             instrAsm#"\t $w, $s, $x, $y", []>, Requires<[HasMAC16]> {
+  bits<2> x;
+  bits<2> y;
+  bits<2> w;
+
+  let Constraints = "$s = $d";
+  let mayLoad = 1;
+  let r{3} = 0;
+  let r{2} = x{0};
+  let r{1-0} = w{1-0};
+  let t{3} = 0;
+  let t{2} = y{0};
+  let t{1-0} = 0;
+  let Uses = [ACCLO, ACCHI];
+  let Defs = [M1, M2, ACCLO, ACCHI];
+}
+
+def MULA_DD_LL_LDDEC : MULA_DD_LDDEC<0x08, "mula.dd.ll.lddec">;
+def MULA_DD_HL_LDDEC : MULA_DD_LDDEC<0x09, "mula.dd.hl.lddec">;
+def MULA_DD_LH_LDDEC : MULA_DD_LDDEC<0x0A, "mula.dd.lh.lddec">;
+def MULA_DD_HH_LDDEC : MULA_DD_LDDEC<0x0B, "mula.dd.hh.lddec">;
+
+class MULA_DD_LDINC<bits<4> oper1, string instrAsm>
+  : RRR_Inst<0x04, oper1, 0x00, (outs MR:$w, AR:$d), (ins AR:$s, MR01:$x, MR23:$y),
+             instrAsm#"\t $w, $s, $x, $y", []>, Requires<[HasMAC16]> {
+  bits<2> x;
+  bits<2> y;
+  bits<2> w;
+
+  let Constraints = "$s = $d";
+  let mayLoad = 1;
+  let r{3} = 0;
+  let r{2} = x{0};
+  let r{1-0} = w{1-0};
+  let t{3} = 0;
+  let t{2} = y{0};
+  let t{1-0} = 0;
+  let Uses = [ACCLO, ACCHI];
+  let Defs = [M1, M2, ACCLO, ACCHI];
+}
+
+def MULA_DD_LL_LDINC : MULA_DD_LDINC<0x08, "mula.dd.ll.ldinc">;
+def MULA_DD_HL_LDINC : MULA_DD_LDINC<0x09, "mula.dd.hl.ldinc">;
+def MULA_DD_LH_LDINC : MULA_DD_LDINC<0x0A, "mula.dd.lh.ldinc">;
+def MULA_DD_HH_LDINC : MULA_DD_LDINC<0x0B, "mula.dd.hh.ldinc">;
+
+def LDDEC : RRR_Inst<0x04, 0x00, 0x09, (outs MR:$w, AR:$d), (ins AR:$s),
+                    "lddec\t $w, $s", []>, Requires<[HasMAC16]> {
+  bits<2> w;
+
+  let Constraints = "$s = $d";
+  let mayLoad = 1;
+  let r{3-2} = 0;
+  let r{1-0} = w{1-0};
+  let t = 0x00;
+}
+
+def LDINC : RRR_Inst<0x04, 0x00, 0x08, (outs MR:$w, AR:$d), (ins AR:$s),
+                    "ldinc\t $w, $s", []>, Requires<[HasMAC16]> {
+  bits<2> w;
+
+  let Constraints = "$s = $d";
+  let mayLoad = 1;
+  let r{3-2} = 0;
+  let r{1-0} = w{1-0};
+  let t = 0;
+}
diff --git a/llvm/lib/Target/Xtensa/XtensaFeatures.td b/llvm/lib/Target/Xtensa/XtensaFeatures.td
index b0cbeb9456796..184828cd253f3 100644
--- a/llvm/lib/Target/Xtensa/XtensaFeatures.td
+++ b/llvm/lib/Target/Xtensa/XtensaFeatures.td
@@ -13,6 +13,11 @@ def FeatureWindowed : SubtargetFeature<"windowed", "HasWindowed", "true",
 def HasWindowed : Predicate<"Subtarget->hasWindowed()">,
                   AssemblerPredicate<(all_of FeatureWindowed)>;
 
+def FeatureMAC16 : SubtargetFeature<"mac16", "HasMAC16", "true",
+                                    "Enable Xtensa MAC16 instructions">;
+def HasMAC16 : Predicate<"Subtarget->hasMAC16()">,
+               AssemblerPredicate<(all_of FeatureMAC16)>;
+
 def FeatureBoolean          : SubtargetFeature<"bool", "HasBoolean", "true",
                                                "Enable Xtensa Boolean extension">;
 def HasBoolean              : Predicate<"Subtarget->hasBoolean()">,
diff --git a/llvm/lib/Target/Xtensa/XtensaInstrInfo.td b/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
index e52dcbf1377c5..1f397e3ecac35 100644
--- a/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
+++ b/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
@@ -849,3 +849,8 @@ let Constraints = "$dr = $r,@earlyclobber $dr" in {
   def MOVT : RRR_Inst<0x00, 0x03, 0x0D, (outs AR:$dr), (ins AR:$r, AR:$s, BR:$t),
                    "movt\t$r, $s, $t", []>, Requires<[HasBoolean]>;
 }
+
+//===----------------------------------------------------------------------===//
+// DSP Instructions
+//===----------------------------------------------------------------------===//
+include "XtensaDSPInstrInfo.td"
diff --git a/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td b/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td
index 920354ff41ba5..2934018318406 100644
--- a/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td
+++ b/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td
@@ -83,7 +83,19 @@ def BREG : SRReg<4, "br", ["BR","4"]>;
 def WINDOWBASE : SRReg<72, "windowbase", ["WINDOWBASE", "72"]>;
 def WINDOWSTART : SRReg<73, "windowstart", ["WINDOWSTART", "73"]>;
 
-def SR :  RegisterClass<"Xtensa", [i32], 32, (add SAR, BREG,
+// MAC16 Option registers
+def ACCLO : SRReg<16, "acclo", ["ACCLO", "16"]>;
+def ACCHI : SRReg<17, "acchi", ["ACCHI", "17"]>;
+def M0    : SRReg<32, "m0", ["M0", "32"]>;
+def M1    : SRReg<33, "m1", ["M1", "33"]>;
+def M2    : SRReg<34, "m2", ["M2", "34"]>;
+def M3    : SRReg<35, "m3", ["M3", "35"]>;
+
+def MR01 :  RegisterClass<"Xtensa", [i32], 32, (add M0, M1)>;
+def MR23 :  RegisterClass<"Xtensa", [i32], 32, (add M2, M3)>;
+def MR   :  RegisterClass<"Xtensa", [i32], 32, (add MR01, MR23)>;
+
+def SR :  RegisterClass<"Xtensa", [i32], 32, (add SAR, BREG, MR,
   WINDOWBASE, WINDOWSTART)>;
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/Xtensa/XtensaSubtarget.h b/llvm/lib/Target/Xtensa/XtensaSubtarget.h
index 962bed2c2e36f..770f73905b337 100644
--- a/llvm/lib/Target/Xtensa/XtensaSubtarget.h
+++ b/llvm/lib/Target/Xtensa/XtensaSubtarget.h
@@ -66,6 +66,8 @@ class XtensaSubtarget : public XtensaGenSubtargetInfo {
 
   bool hasDensity() const { return HasDensity; }
 
+  bool hasMAC16() const { return HasMAC16; }
+
   bool hasWindowed() const { return HasWindowed; }
 
   bool hasBoolean() const { return HasBoolean; }
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 62fa3af502e29..2f5f1089067bf 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -785,7 +785,8 @@ static void buildFrameDebugInfo(Function &F, coro::Shape &Shape,
   // fields confilicts with each other.
   unsigned UnknownTypeNum = 0;
   for (unsigned Index = 0; Index < FrameTy->getNumElements(); Index++) {
-    if (!OffsetCache.contains(Index))
+    auto OCIt = OffsetCache.find(Index);
+    if (OCIt == OffsetCache.end())
       continue;
 
     std::string Name;
@@ -797,8 +798,8 @@ static void buildFrameDebugInfo(Function &F, coro::Shape &Shape,
     Type *Ty = FrameTy->getElementType(Index);
     assert(Ty->isSized() && "We can't handle type which is not sized.\n");
     SizeInBits = Layout.getTypeSizeInBits(Ty).getFixedValue();
-    AlignInBits = OffsetCache[Index].first * 8;
-    OffsetInBits = OffsetCache[Index].second * 8;
+    AlignInBits = OCIt->second.first * 8;
+    OffsetInBits = OCIt->second.second * 8;
 
     if (auto It = NameCache.find(Index); It != NameCache.end()) {
       Name = It->second.str();
diff --git a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
index 6fe81f7b063ec..1fca1b54065d4 100644
--- a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
@@ -406,15 +406,16 @@ static FunctionSummary *calculatePrevailingSummary(
     }
   }
 
+  auto &CPS = CachedPrevailingSummary[VI];
   if (Local) {
     assert(!Prevailing);
-    CachedPrevailingSummary[VI] = Local;
+    CPS = Local;
   } else if (Prevailing) {
     assert(!Local);
-    CachedPrevailingSummary[VI] = Prevailing;
+    CPS = Prevailing;
   }
 
-  return CachedPrevailingSummary[VI];
+  return CPS;
 }
 
 bool llvm::thinLTOPropagateFunctionAttrs(
diff --git a/llvm/lib/Transforms/IPO/GlobalDCE.cpp b/llvm/lib/Transforms/IPO/GlobalDCE.cpp
index eca36fb31cea0..250c0443cb4af 100644
--- a/llvm/lib/Transforms/IPO/GlobalDCE.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalDCE.cpp
@@ -67,16 +67,13 @@ void GlobalDCEPass::ComputeDependencies(Value *V,
     Deps.insert(GV);
   } else if (auto *CE = dyn_cast<Constant>(V)) {
     // Avoid walking the whole tree of a big ConstantExprs multiple times.
-    auto Where = ConstantDependenciesCache.find(CE);
-    if (Where != ConstantDependenciesCache.end()) {
-      auto const &K = Where->second;
-      Deps.insert(K.begin(), K.end());
-    } else {
-      SmallPtrSetImpl<GlobalValue *> &LocalDeps = ConstantDependenciesCache[CE];
+    auto [Where, Inserted] = ConstantDependenciesCache.try_emplace(CE);
+    SmallPtrSetImpl<GlobalValue *> &LocalDeps = Where->second;
+    if (Inserted) {
       for (User *CEUser : CE->users())
         ComputeDependencies(CEUser, LocalDeps);
-      Deps.insert(LocalDeps.begin(), LocalDeps.end());
     }
+    Deps.insert(LocalDeps.begin(), LocalDeps.end());
   }
 }
 
diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
index 3a3c8ef654dcc..596a3c1f01f64 100644
--- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -1699,9 +1699,9 @@ void LowerTypeTestsModule::buildBitSetsFromFunctionsNative(
 
       if (IsExported) {
         if (IsJumpTableCanonical)
-          ExportSummary->cfiFunctionDefs().insert(std::string(F->getName()));
+          ExportSummary->cfiFunctionDefs().emplace(F->getName());
         else
-          ExportSummary->cfiFunctionDecls().insert(std::string(F->getName()));
+          ExportSummary->cfiFunctionDecls().emplace(F->getName());
       }
 
       if (!IsJumpTableCanonical) {
@@ -2207,9 +2207,9 @@ bool LowerTypeTestsModule::lower() {
     bool IsExported = false;
     if (Function *F = dyn_cast<Function>(&GO)) {
       IsJumpTableCanonical = isJumpTableCanonical(F);
-      if (ExportedFunctions.count(F->getName())) {
-        IsJumpTableCanonical |=
-            ExportedFunctions[F->getName()].Linkage == CFL_Definition;
+      if (auto It = ExportedFunctions.find(F->getName());
+          It != ExportedFunctions.end()) {
+        IsJumpTableCanonical |= It->second.Linkage == CFL_Definition;
         IsExported = true;
       // TODO: The logic here checks only that the function is address taken,
       // not that the address takers are live. This can be updated to check
@@ -2407,9 +2407,9 @@ bool LowerTypeTestsModule::lower() {
             cast<MDString>(AliasMD->getOperand(0))->getString();
         StringRef Aliasee = cast<MDString>(AliasMD->getOperand(1))->getString();
 
-        if (!ExportedFunctions.count(Aliasee) ||
-            ExportedFunctions[Aliasee].Linkage != CFL_Definition ||
-            !M.getNamedAlias(Aliasee))
+        if (auto It = ExportedFunctions.find(Aliasee);
+            It == ExportedFunctions.end() ||
+            It->second.Linkage != CFL_Definition || !M.getNamedAlias(Aliasee))
           continue;
 
         GlobalValue::VisibilityTypes Visibility =
diff --git a/llvm/lib/Transforms/IPO/MergeFunctions.cpp b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
index d2443a83535e4..c463cda68d4b5 100644
--- a/llvm/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
@@ -923,6 +923,8 @@ void MergeFunctions::mergeTwoFunctions(Function *F, Function *G) {
                                       F->getAddressSpace(), "", F->getParent());
     NewF->copyAttributesFrom(F);
     NewF->takeName(F);
+    NewF->setComdat(F->getComdat());
+    F->setComdat(nullptr);
     NewF->IsNewDbgInfoFormat = F->IsNewDbgInfoFormat;
     // Ensure CFI type metadata is propagated to the new function.
     copyMetadataIfPresent(F, NewF, "type");
diff --git a/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp
index ec2594b0d9dc1..a614e2d169534 100644
--- a/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp
@@ -148,8 +148,9 @@ void PseudoProbeVerifier::verifyProbeFactors(
   auto &PrevProbeFactors = FunctionProbeFactors[F->getName()];
   for (const auto &I : ProbeFactors) {
     float CurProbeFactor = I.second;
-    if (PrevProbeFactors.count(I.first)) {
-      float PrevProbeFactor = PrevProbeFactors[I.first];
+    auto [It, Inserted] = PrevProbeFactors.try_emplace(I.first);
+    if (!Inserted) {
+      float PrevProbeFactor = It->second;
       if (std::abs(CurProbeFactor - PrevProbeFactor) >
           DistributionFactorVariance) {
         if (!BannerPrinted) {
@@ -163,7 +164,7 @@ void PseudoProbeVerifier::verifyProbeFactors(
     }
 
     // Update
-    PrevProbeFactors[I.first] = I.second;
+    It->second = I.second;
   }
 }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index 330358ee749e5..6cc241781d112 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -3120,15 +3120,16 @@ static Value *matchOrConcat(Instruction &Or, InstCombiner::BuilderTy &Builder) {
     return ConcatIntrinsicCalls(Intrinsic::bitreverse, UpperBRev, LowerBRev);
 
   // iX ext split: extending or(zext(x),shl(zext(y),bw/2) pattern
-  // to consume sext/ashr: or(zext(sext(x)),shl(zext(sext(ashr(x))),bw/2)
+  // to consume sext/ashr:
+  // or(zext(sext(x)),shl(zext(sext(ashr(x,xbw-1))),bw/2)
+  // or(zext(x),shl(zext(ashr(x,xbw-1)),bw/2)
   Value *X;
-  if (match(LowerSrc, m_SExt(m_Value(X))) &&
+  if (match(LowerSrc, m_SExtOrSelf(m_Value(X))) &&
       match(UpperSrc,
-            m_SExt(m_AShr(
+            m_SExtOrSelf(m_AShr(
                 m_Specific(X),
-                m_SpecificInt(X->getType()->getScalarSizeInBits() - 1))))) {
+                m_SpecificInt(X->getType()->getScalarSizeInBits() - 1)))))
     return Builder.CreateSExt(X, Ty);
-  }
 
   return nullptr;
 }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index c1f5e286ab3ed..c29cba6f675c5 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -360,7 +360,10 @@ void PointerReplacer::replace(Instruction *I) {
 
     IC.InsertNewInstWith(NewI, LT->getIterator());
     IC.replaceInstUsesWith(*LT, NewI);
-    WorkMap[LT] = NewI;
+    // LT has actually been replaced by NewI. It is useless to insert LT into
+    // the map. Instead, we insert NewI into the map to indicate this is the
+    // replacement (new value).
+    WorkMap[NewI] = NewI;
   } else if (auto *PHI = dyn_cast<PHINode>(I)) {
     Type *NewTy = getReplacement(PHI->getIncomingValue(0))->getType();
     auto *NewPHI = PHINode::Create(NewTy, PHI->getNumIncomingValues(),
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index ba925275faba6..a077c85ffc410 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -3129,6 +3129,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   /// fine).
   ///
   /// Caller guarantees that this intrinsic does not access memory.
+  ///
+  /// TODO: "horizontal"/"pairwise" intrinsics are often incorrectly matched by
+  ///       by this handler.
   [[maybe_unused]] bool
   maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I,
                                   unsigned int trailingFlags) {
@@ -3653,14 +3656,18 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   // Instrument generic vector reduction intrinsics
   // by ORing together all their fields.
   //
-  // The return type does not need to be the same type as the fields
+  // If AllowShadowCast is true, the return type does not need to be the same
+  // type as the fields
   // e.g., declare i32 @llvm.aarch64.neon.uaddv.i32.v16i8(<16 x i8>)
-  void handleVectorReduceIntrinsic(IntrinsicInst &I) {
+  void handleVectorReduceIntrinsic(IntrinsicInst &I, bool AllowShadowCast) {
     assert(I.arg_size() == 1);
 
     IRBuilder<> IRB(&I);
     Value *S = IRB.CreateOrReduce(getShadow(&I, 0));
-    S = CreateShadowCast(IRB, S, getShadowTy(&I));
+    if (AllowShadowCast)
+      S = CreateShadowCast(IRB, S, getShadowTy(&I));
+    else
+      assert(S->getType() == getShadowTy(&I));
     setShadow(&I, S);
     setOriginForNaryOp(I);
   }
@@ -3669,13 +3676,18 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   // e.g., call float @llvm.vector.reduce.fadd.f32.v2f32(float %a0, <2 x float>
   // %a1)
   //       shadow = shadow[a0] | shadow[a1.0] | shadow[a1.1]
+  //
+  // The type of the return value, initial starting value, and elements of the
+  // vector must be identical.
   void handleVectorReduceWithStarterIntrinsic(IntrinsicInst &I) {
     assert(I.arg_size() == 2);
 
     IRBuilder<> IRB(&I);
     Value *Shadow0 = getShadow(&I, 0);
     Value *Shadow1 = IRB.CreateOrReduce(getShadow(&I, 1));
+    assert(Shadow0->getType() == Shadow1->getType());
     Value *S = IRB.CreateOr(Shadow0, Shadow1);
+    assert(S->getType() == getShadowTy(&I));
     setShadow(&I, S);
     setOriginForNaryOp(I);
   }
@@ -4458,21 +4470,17 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     case Intrinsic::vector_reduce_add:
     case Intrinsic::vector_reduce_xor:
     case Intrinsic::vector_reduce_mul:
-    // Add reduction to scalar
-    case Intrinsic::aarch64_neon_faddv:
-    case Intrinsic::aarch64_neon_saddv:
-    case Intrinsic::aarch64_neon_uaddv:
-    // Floating-point min/max (vector)
-    // The f{min,max}"nm"v variants handle NaN differently than f{min,max}v,
-    // but our shadow propagation is the same.
-    case Intrinsic::aarch64_neon_fmaxv:
-    case Intrinsic::aarch64_neon_fminv:
-    case Intrinsic::aarch64_neon_fmaxnmv:
-    case Intrinsic::aarch64_neon_fminnmv:
-    // Sum long across vector
-    case Intrinsic::aarch64_neon_saddlv:
-    case Intrinsic::aarch64_neon_uaddlv:
-      handleVectorReduceIntrinsic(I);
+    // Signed/Unsigned Min/Max
+    // TODO: handling similarly to AND/OR may be more precise.
+    case Intrinsic::vector_reduce_smax:
+    case Intrinsic::vector_reduce_smin:
+    case Intrinsic::vector_reduce_umax:
+    case Intrinsic::vector_reduce_umin:
+    // TODO: this has no false positives, but arguably we should check that all
+    // the bits are initialized.
+    case Intrinsic::vector_reduce_fmax:
+    case Intrinsic::vector_reduce_fmin:
+      handleVectorReduceIntrinsic(I, /*AllowShadowCast=*/false);
       break;
 
     case Intrinsic::vector_reduce_fadd:
@@ -4871,6 +4879,18 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       setOrigin(&I, getCleanOrigin());
       break;
 
+    // TODO: handling max/min similarly to AND/OR may be more precise
+    // Floating-Point Maximum/Minimum Pairwise
+    case Intrinsic::aarch64_neon_fmaxp:
+    case Intrinsic::aarch64_neon_fminp:
+    // Floating-Point Maximum/Minimum Number Pairwise
+    case Intrinsic::aarch64_neon_fmaxnmp:
+    case Intrinsic::aarch64_neon_fminnmp:
+    // Signed/Unsigned Maximum/Minimum Pairwise
+    case Intrinsic::aarch64_neon_smaxp:
+    case Intrinsic::aarch64_neon_sminp:
+    case Intrinsic::aarch64_neon_umaxp:
+    case Intrinsic::aarch64_neon_uminp:
     // Add Pairwise
     case Intrinsic::aarch64_neon_addp:
     // Floating-point Add Pairwise
@@ -4903,6 +4923,29 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       break;
     }
 
+    // Add reduction to scalar
+    case Intrinsic::aarch64_neon_faddv:
+    case Intrinsic::aarch64_neon_saddv:
+    case Intrinsic::aarch64_neon_uaddv:
+    // Signed/Unsigned min/max (Vector)
+    // TODO: handling similarly to AND/OR may be more precise.
+    case Intrinsic::aarch64_neon_smaxv:
+    case Intrinsic::aarch64_neon_sminv:
+    case Intrinsic::aarch64_neon_umaxv:
+    case Intrinsic::aarch64_neon_uminv:
+    // Floating-point min/max (vector)
+    // The f{min,max}"nm"v variants handle NaN differently than f{min,max}v,
+    // but our shadow propagation is the same.
+    case Intrinsic::aarch64_neon_fmaxv:
+    case Intrinsic::aarch64_neon_fminv:
+    case Intrinsic::aarch64_neon_fmaxnmv:
+    case Intrinsic::aarch64_neon_fminnmv:
+    // Sum long across vector
+    case Intrinsic::aarch64_neon_saddlv:
+    case Intrinsic::aarch64_neon_uaddlv:
+      handleVectorReduceIntrinsic(I, /*AllowShadowCast=*/true);
+      break;
+
     // Saturating extract narrow
     case Intrinsic::aarch64_neon_sqxtn:
     case Intrinsic::aarch64_neon_sqxtun:
diff --git a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp
index aa6bee23ad5ff..ffc2aec77ff91 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp
@@ -12,6 +12,7 @@
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/IR/Analysis.h"
 #include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
@@ -53,6 +54,7 @@ class CtxInstrumentationLowerer final {
   ModuleAnalysisManager &MAM;
   Type *ContextNodeTy = nullptr;
   Type *ContextRootTy = nullptr;
+  Type *FunctionDataTy = nullptr;
 
   DenseMap<const Function *, Constant *> ContextRootMap;
   Function *StartCtx = nullptr;
@@ -120,6 +122,12 @@ CtxInstrumentationLowerer::CtxInstrumentationLowerer(Module &M,
                                           PointerTy,          /*CurrentMem*/
                                           SanitizerMutexType, /*Taken*/
                                       });
+  FunctionDataTy =
+      StructType::get(M.getContext(), {
+                                          PointerTy,          /*FlatCtx*/
+                                          SanitizerMutexType, /*Mutex*/
+                                      });
+
   // The Context header.
   ContextNodeTy = StructType::get(M.getContext(), {
                                                       I64Ty,     /*Guid*/
@@ -163,7 +171,8 @@ CtxInstrumentationLowerer::CtxInstrumentationLowerer(Module &M,
   GetCtx = cast<Function>(
       M.getOrInsertFunction(CompilerRtAPINames::GetCtx,
                             FunctionType::get(PointerTy,
-                                              {PointerTy, /*Callee*/
+                                              {PointerTy, /*FunctionData*/
+                                               PointerTy, /*Callee*/
                                                I64Ty,     /*Guid*/
                                                I32Ty,     /*NumCounters*/
                                                I32Ty},    /*NumCallsites*/
@@ -224,7 +233,6 @@ bool CtxInstrumentationLowerer::lowerFunction(Function &F) {
       assert(Mark->getIndex()->isZero());
 
       IRBuilder<> Builder(Mark);
-
       Guid = Builder.getInt64(
           AssignGUIDPass::getGUID(cast<Function>(*Mark->getNameValue())));
       // The type of the context of this function is now knowable since we have
@@ -248,9 +256,14 @@ bool CtxInstrumentationLowerer::lowerFunction(Function &F) {
         ORE.emit(
             [&] { return OptimizationRemark(DEBUG_TYPE, "Entrypoint", &F); });
       } else {
-        Context =
-            Builder.CreateCall(GetCtx, {&F, Guid, Builder.getInt32(NumCounters),
-                                        Builder.getInt32(NumCallsites)});
+        // Make up a compact name, these names end up taking up a lot of space
+        // in the binary.
+        auto *FData = new GlobalVariable(
+            M, FunctionDataTy, false, GlobalVariable::InternalLinkage,
+            Constant::getNullValue(FunctionDataTy));
+        Context = Builder.CreateCall(GetCtx, {FData, &F, Guid,
+                                              Builder.getInt32(NumCounters),
+                                              Builder.getInt32(NumCallsites)});
         ORE.emit([&] {
           return OptimizationRemark(DEBUG_TYPE, "RegularFunction", &F);
         });
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 5ad07e83d1273..a8055979acaa2 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1517,9 +1517,10 @@ void PGOUseFunc::populateCoverage(IndexedInstrProfReader *PGOReader) {
     CoveredBlocksToProcess.pop();
     for (auto *BB : InverseDependencies[CoveredBlock]) {
       // If CoveredBlock is covered then BB is covered.
-      if (Coverage[BB])
+      bool &Cov = Coverage[BB];
+      if (Cov)
         continue;
-      Coverage[BB] = true;
+      Cov = true;
       CoveredBlocksToProcess.push(BB);
     }
   }
@@ -1563,14 +1564,15 @@ void PGOUseFunc::populateCoverage(IndexedInstrProfReader *PGOReader) {
     // successors, e.g., when a block calls a function that may call exit(). In
     // those cases, BFI could find its successor to be covered while BCI could
     // find its successor to be dead.
-    if (Coverage[&BB] == IsBlockDead(BB).value_or(false)) {
+    const bool &Cov = Coverage[&BB];
+    if (Cov == IsBlockDead(BB).value_or(false)) {
       LLVM_DEBUG(
           dbgs() << "Found inconsistent block covearge for " << BB.getName()
-                 << ": BCI=" << (Coverage[&BB] ? "Covered" : "Dead") << " BFI="
+                 << ": BCI=" << (Cov ? "Covered" : "Dead") << " BFI="
                  << (IsBlockDead(BB).value() ? "Dead" : "Covered") << "\n");
       ++NumCorruptCoverage;
     }
-    if (Coverage[&BB])
+    if (Cov)
       ++NumCoveredBlocks;
   }
   if (PGOVerifyBFI && NumCorruptCoverage) {
diff --git a/llvm/lib/Transforms/Scalar/CallSiteSplitting.cpp b/llvm/lib/Transforms/Scalar/CallSiteSplitting.cpp
index e644636d42f63..2e955e061111f 100644
--- a/llvm/lib/Transforms/Scalar/CallSiteSplitting.cpp
+++ b/llvm/lib/Transforms/Scalar/CallSiteSplitting.cpp
@@ -397,9 +397,10 @@ static void splitCallSite(CallBase &CB,
         continue;
       PHINode *NewPN = PHINode::Create(CurrentI->getType(), Preds.size());
       NewPN->setDebugLoc(CurrentI->getDebugLoc());
-      for (auto &Mapping : ValueToValueMaps)
-        NewPN->addIncoming(Mapping[CurrentI],
-                           cast<Instruction>(Mapping[CurrentI])->getParent());
+      for (auto &Mapping : ValueToValueMaps) {
+        Value *V = Mapping[CurrentI];
+        NewPN->addIncoming(V, cast<Instruction>(V)->getParent());
+      }
       NewPN->insertBefore(*TailBB, TailBB->begin());
       CurrentI->replaceAllUsesWith(NewPN);
     }
diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
index 8e74b8645fad9..314a5d15f0f88 100644
--- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
+++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
@@ -1212,6 +1212,34 @@ static bool processAnd(BinaryOperator *BinOp, LazyValueInfo *LVI) {
   return true;
 }
 
+static bool processTrunc(TruncInst *TI, LazyValueInfo *LVI) {
+  if (TI->hasNoSignedWrap() && TI->hasNoUnsignedWrap())
+    return false;
+
+  ConstantRange Range =
+      LVI->getConstantRangeAtUse(TI->getOperandUse(0), /*UndefAllowed=*/false);
+  uint64_t DestWidth = TI->getDestTy()->getScalarSizeInBits();
+  bool Changed = false;
+
+  if (!TI->hasNoUnsignedWrap()) {
+    if (Range.getActiveBits() <= DestWidth) {
+      TI->setHasNoUnsignedWrap(true);
+      ++NumNUW;
+      Changed = true;
+    }
+  }
+
+  if (!TI->hasNoSignedWrap()) {
+    if (Range.getMinSignedBits() <= DestWidth) {
+      TI->setHasNoSignedWrap(true);
+      ++NumNSW;
+      Changed = true;
+    }
+  }
+
+  return Changed;
+}
+
 static bool runImpl(Function &F, LazyValueInfo *LVI, DominatorTree *DT,
                     const SimplifyQuery &SQ) {
   bool FnChanged = false;
@@ -1275,6 +1303,9 @@ static bool runImpl(Function &F, LazyValueInfo *LVI, DominatorTree *DT,
       case Instruction::And:
         BBChanged |= processAnd(cast<BinaryOperator>(&II), LVI);
         break;
+      case Instruction::Trunc:
+        BBChanged |= processTrunc(cast<TruncInst>(&II), LVI);
+        break;
       }
     }
 
diff --git a/llvm/lib/Transforms/Scalar/GuardWidening.cpp b/llvm/lib/Transforms/Scalar/GuardWidening.cpp
index 2978b7990a6eb..3ba5b79293bcd 100644
--- a/llvm/lib/Transforms/Scalar/GuardWidening.cpp
+++ b/llvm/lib/Transforms/Scalar/GuardWidening.cpp
@@ -665,8 +665,8 @@ Value *GuardWideningImpl::freezeAndPush(Value *Orig,
       CacheOfFreezes[Def] = FI;
     }
 
-    if (CacheOfFreezes.count(Def))
-      U.set(CacheOfFreezes[Def]);
+    if (auto It = CacheOfFreezes.find(Def); It != CacheOfFreezes.end())
+      U.set(It->second);
     return true;
   };
 
diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index ff3c93661c9e6..ce13f87d601a7 100644
--- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -638,17 +638,19 @@ bool MemCpyOptPass::processStoreOfLoad(StoreInst *SI, LoadInst *LI,
       (EnableMemCpyOptWithoutLibcalls ||
        (TLI->has(LibFunc_memcpy) && TLI->has(LibFunc_memmove)))) {
     MemoryLocation LoadLoc = MemoryLocation::get(LI);
-    MemoryUseOrDef *LoadAccess = MSSA->getMemoryAccess(LI),
-                   *StoreAccess = MSSA->getMemoryAccess(SI);
-
-    // We use MSSA to check if an instruction may store to the memory we load
-    // from in between the load and the store. If such an instruction is found,
-    // we try to promote there instead of at the store position.
-    auto *Clobber = MSSA->getWalker()->getClobberingMemoryAccess(
-        StoreAccess->getDefiningAccess(), LoadLoc, BAA);
-    Instruction *P = MSSA->dominates(LoadAccess, Clobber)
-                         ? cast<MemoryUseOrDef>(Clobber)->getMemoryInst()
-                         : SI;
+
+    // We use alias analysis to check if an instruction may store to
+    // the memory we load from in between the load and the store. If
+    // such an instruction is found, we try to promote there instead
+    // of at the store position.
+    // TODO: Can use MSSA for this.
+    Instruction *P = SI;
+    for (auto &I : make_range(++LI->getIterator(), SI->getIterator())) {
+      if (isModSet(BAA.getModRefInfo(&I, LoadLoc))) {
+        P = &I;
+        break;
+      }
+    }
 
     // If we found an instruction that may write to the loaded memory,
     // we can try to promote at this position instead of the store
diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
index b1f742b838f2a..d1054b9b045ca 100644
--- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -361,6 +361,8 @@ class StructurizeCFG {
 
   void rebuildSSA();
 
+  void setDebugLoc(BranchInst *Br, BasicBlock *BB);
+
 public:
   void init(Region *R);
   bool run(Region *R, DominatorTree *DT);
@@ -595,14 +597,6 @@ void StructurizeCFG::collectInfos() {
     // Find the last back edges
     analyzeLoops(RN);
   }
-
-  // Reset the collected term debug locations
-  TermDL.clear();
-
-  for (BasicBlock &BB : *Func) {
-    if (const DebugLoc &DL = BB.getTerminator()->getDebugLoc())
-      TermDL[&BB] = DL;
-  }
 }
 
 /// Insert the missing branch conditions
@@ -852,16 +846,17 @@ void StructurizeCFG::setPhiValues() {
     BasicBlock *To = AddedPhi.first;
     const BBVector &From = AddedPhi.second;
 
-    if (!DeletedPhis.count(To))
+    auto It = DeletedPhis.find(To);
+    if (It == DeletedPhis.end())
       continue;
 
-    PhiMap &Map = DeletedPhis[To];
+    PhiMap &Map = It->second;
     SmallVector<BasicBlock *> &UndefBlks = UndefBlksMap[To];
     for (const auto &[Phi, Incoming] : Map) {
-      Value *Undef = UndefValue::get(Phi->getType());
+      Value *Poison = PoisonValue::get(Phi->getType());
       Updater.Initialize(Phi->getType(), "");
-      Updater.AddAvailableValue(&Func->getEntryBlock(), Undef);
-      Updater.AddAvailableValue(To, Undef);
+      Updater.AddAvailableValue(&Func->getEntryBlock(), Poison);
+      Updater.AddAvailableValue(To, Poison);
 
       // Use leader phi's incoming if there is.
       auto LeaderIt = PhiClasses.findLeader(Phi);
@@ -890,7 +885,7 @@ void StructurizeCFG::setPhiValues() {
         if (Updater.HasValueForBlock(UB))
           continue;
 
-        Updater.AddAvailableValue(UB, Undef);
+        Updater.AddAvailableValue(UB, Poison);
       }
 
       for (BasicBlock *FI : From)
@@ -923,12 +918,24 @@ void StructurizeCFG::simplifyAffectedPhis() {
   } while (Changed);
 }
 
+void StructurizeCFG::setDebugLoc(BranchInst *Br, BasicBlock *BB) {
+  auto I = TermDL.find(BB);
+  if (I == TermDL.end())
+    return;
+
+  Br->setDebugLoc(I->second);
+  TermDL.erase(I);
+}
+
 /// Remove phi values from all successors and then remove the terminator.
 void StructurizeCFG::killTerminator(BasicBlock *BB) {
   Instruction *Term = BB->getTerminator();
   if (!Term)
     return;
 
+  if (const DebugLoc &DL = Term->getDebugLoc())
+    TermDL[BB] = DL;
+
   for (BasicBlock *Succ : successors(BB))
     delPhiValues(BB, Succ);
 
@@ -973,7 +980,7 @@ void StructurizeCFG::changeExit(RegionNode *Node, BasicBlock *NewExit,
     BasicBlock *BB = Node->getNodeAs<BasicBlock>();
     killTerminator(BB);
     BranchInst *Br = BranchInst::Create(NewExit, BB);
-    Br->setDebugLoc(TermDL[BB]);
+    setDebugLoc(Br, BB);
     addPhiValues(BB, NewExit);
     if (IncludeDominator)
       DT->changeImmediateDominator(NewExit, BB);
@@ -989,10 +996,14 @@ BasicBlock *StructurizeCFG::getNextFlow(BasicBlock *Dominator) {
                                         Func, Insert);
   FlowSet.insert(Flow);
 
-  // use a temporary variable to avoid a use-after-free if the map's storage is
-  // reallocated
-  DebugLoc DL = TermDL[Dominator];
-  TermDL[Flow] = std::move(DL);
+  if (auto *Term = Dominator->getTerminator()) {
+    if (const DebugLoc &DL = Term->getDebugLoc())
+      TermDL[Flow] = DL;
+  } else if (DebugLoc DLTemp = TermDL.lookup(Dominator)) {
+    // Use a temporary copy to avoid a use-after-free if the map's storage
+    // is reallocated.
+    TermDL[Flow] = DLTemp;
+  }
 
   DT->addNewBlock(Flow, Dominator);
   ParentRegion->getRegionInfo()->setRegionFor(Flow, ParentRegion);
@@ -1087,7 +1098,7 @@ void StructurizeCFG::wireFlow(bool ExitUseAllowed,
 
     // let it point to entry and next block
     BranchInst *Br = BranchInst::Create(Entry, Next, BoolPoison, Flow);
-    Br->setDebugLoc(TermDL[Flow]);
+    setDebugLoc(Br, Flow);
     Conditions.push_back(Br);
     addPhiValues(Flow, Entry);
     DT->changeImmediateDominator(Entry, Flow);
@@ -1128,7 +1139,7 @@ void StructurizeCFG::handleLoops(bool ExitUseAllowed,
   LoopEnd = needPrefix(false);
   BasicBlock *Next = needPostfix(LoopEnd, ExitUseAllowed);
   BranchInst *Br = BranchInst::Create(Next, LoopStart, BoolPoison, LoopEnd);
-  Br->setDebugLoc(TermDL[LoopEnd]);
+  setDebugLoc(Br, LoopEnd);
   LoopConds.push_back(Br);
   addPhiValues(LoopEnd, LoopStart);
   setPrevNode(Next);
@@ -1181,9 +1192,9 @@ void StructurizeCFG::rebuildSSA() {
           continue;
 
         if (!Initialized) {
-          Value *Undef = UndefValue::get(I.getType());
+          Value *Poison = PoisonValue::get(I.getType());
           Updater.Initialize(I.getType(), "");
-          Updater.AddAvailableValue(&Func->getEntryBlock(), Undef);
+          Updater.AddAvailableValue(&Func->getEntryBlock(), Poison);
           Updater.AddAvailableValue(BB, &I);
           Initialized = true;
         }
diff --git a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
index 7a5326c255831..594dd067006e2 100644
--- a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
@@ -76,42 +76,46 @@ static bool setNoReturn(Function &F) {
   return true;
 }
 
+static bool setMemoryEffects(Function &F, MemoryEffects ME) {
+  MemoryEffects OrigME = F.getMemoryEffects();
+  MemoryEffects NewME = OrigME & ME;
+  if (OrigME == NewME)
+    return false;
+  F.setMemoryEffects(NewME);
+  return true;
+}
+
 static bool setOnlyAccessesInaccessibleMemory(Function &F) {
-  if (F.onlyAccessesInaccessibleMemory())
+  if (!setMemoryEffects(F, MemoryEffects::inaccessibleMemOnly()))
     return false;
-  F.setOnlyAccessesInaccessibleMemory();
   ++NumInaccessibleMemOnly;
   return true;
 }
 
 static bool setOnlyReadsMemory(Function &F) {
-  if (F.onlyReadsMemory())
+  if (!setMemoryEffects(F, MemoryEffects::readOnly()))
     return false;
-  F.setOnlyReadsMemory();
   ++NumReadOnly;
   return true;
 }
 
 static bool setOnlyWritesMemory(Function &F) {
-  if (F.onlyWritesMemory()) // writeonly or readnone
+  if (!setMemoryEffects(F, MemoryEffects::writeOnly()))
     return false;
   ++NumWriteOnly;
-  F.setOnlyWritesMemory();
   return true;
 }
 
 static bool setOnlyAccessesArgMemory(Function &F) {
-  if (F.onlyAccessesArgMemory())
+  if (!setMemoryEffects(F, MemoryEffects::argMemOnly()))
     return false;
-  F.setOnlyAccessesArgMemory();
   ++NumArgMemOnly;
   return true;
 }
 
 static bool setOnlyAccessesInaccessibleMemOrArgMem(Function &F) {
-  if (F.onlyAccessesInaccessibleMemOrArgMem())
+  if (!setMemoryEffects(F, MemoryEffects::inaccessibleOrArgMemOnly()))
     return false;
-  F.setOnlyAccessesInaccessibleMemOrArgMem();
   ++NumInaccessibleMemOrArgMemOnly;
   return true;
 }
diff --git a/llvm/lib/Transforms/Utils/CloneFunction.cpp b/llvm/lib/Transforms/Utils/CloneFunction.cpp
index 58d400ac396be..dd1b4fe718053 100644
--- a/llvm/lib/Transforms/Utils/CloneFunction.cpp
+++ b/llvm/lib/Transforms/Utils/CloneFunction.cpp
@@ -40,6 +40,18 @@ using namespace llvm;
 
 #define DEBUG_TYPE "clone-function"
 
+namespace {
+void collectDebugInfoFromInstructions(const Function &F,
+                                      DebugInfoFinder &DIFinder) {
+  const Module *M = F.getParent();
+  if (M) {
+    // Inspect instructions to process e.g. DILexicalBlocks of inlined functions
+    for (const auto &I : instructions(F))
+      DIFinder.processInstruction(*M, I);
+  }
+}
+} // namespace
+
 /// See comments in Cloning.h.
 BasicBlock *llvm::CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap,
                                   const Twine &NameSuffix, Function *F,
@@ -135,6 +147,12 @@ void llvm::CloneFunctionAttributesInto(Function *NewFunc,
 DISubprogram *llvm::CollectDebugInfoForCloning(const Function &F,
                                                CloneFunctionChangeType Changes,
                                                DebugInfoFinder &DIFinder) {
+  // CloneModule takes care of cloning debug info for ClonedModule. Cloning into
+  // DifferentModule is taken care of separately in ClonedFunctionInto as part
+  // of llvm.dbg.cu update.
+  if (Changes >= CloneFunctionChangeType::DifferentModule)
+    return nullptr;
+
   DISubprogram *SPClonedWithinModule = nullptr;
   if (Changes < CloneFunctionChangeType::DifferentModule) {
     SPClonedWithinModule = F.getSubprogram();
@@ -142,12 +160,7 @@ DISubprogram *llvm::CollectDebugInfoForCloning(const Function &F,
   if (SPClonedWithinModule)
     DIFinder.processSubprogram(SPClonedWithinModule);
 
-  const Module *M = F.getParent();
-  if (Changes != CloneFunctionChangeType::ClonedModule && M) {
-    // Inspect instructions to process e.g. DILexicalBlocks of inlined functions
-    for (const auto &I : instructions(F))
-      DIFinder.processInstruction(*M, I);
-  }
+  collectDebugInfoFromInstructions(F, DIFinder);
 
   return SPClonedWithinModule;
 }
@@ -156,32 +169,32 @@ MetadataSetTy
 llvm::FindDebugInfoToIdentityMap(CloneFunctionChangeType Changes,
                                  DebugInfoFinder &DIFinder,
                                  DISubprogram *SPClonedWithinModule) {
+  if (Changes >= CloneFunctionChangeType::DifferentModule)
+    return {};
+
+  if (DIFinder.subprogram_count() == 0)
+    assert(!SPClonedWithinModule &&
+           "Subprogram should be in DIFinder->subprogram_count()...");
+
   MetadataSetTy MD;
 
-  if (Changes < CloneFunctionChangeType::DifferentModule &&
-      DIFinder.subprogram_count() > 0) {
-    // Avoid cloning types, compile units, and (other) subprograms.
-    for (DISubprogram *ISP : DIFinder.subprograms()) {
-      if (ISP != SPClonedWithinModule)
-        MD.insert(ISP);
-    }
+  // Avoid cloning types, compile units, and (other) subprograms.
+  for (DISubprogram *ISP : DIFinder.subprograms())
+    if (ISP != SPClonedWithinModule)
+      MD.insert(ISP);
 
-    // If a subprogram isn't going to be cloned skip its lexical blocks as well.
-    for (DIScope *S : DIFinder.scopes()) {
-      auto *LScope = dyn_cast<DILocalScope>(S);
-      if (LScope && LScope->getSubprogram() != SPClonedWithinModule)
-        MD.insert(S);
-    }
+  // If a subprogram isn't going to be cloned skip its lexical blocks as well.
+  for (DIScope *S : DIFinder.scopes()) {
+    auto *LScope = dyn_cast<DILocalScope>(S);
+    if (LScope && LScope->getSubprogram() != SPClonedWithinModule)
+      MD.insert(S);
+  }
 
     for (DICompileUnit *CU : DIFinder.compile_units())
       MD.insert(CU);
 
     for (DIType *Type : DIFinder.types())
       MD.insert(Type);
-  } else {
-    assert(!SPClonedWithinModule &&
-           "Subprogram should be in DIFinder->subprogram_count()...");
-  }
 
   return MD;
 }
@@ -351,6 +364,10 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
   SmallPtrSet<const void *, 8> Visited;
   for (auto *Operand : NMD->operands())
     Visited.insert(Operand);
+
+  // Collect and clone all the compile units referenced from the instructions in
+  // the function (e.g. as a scope).
+  collectDebugInfoFromInstructions(*OldFunc, DIFinder);
   for (auto *Unit : DIFinder.compile_units()) {
     MDNode *MappedUnit =
         MapMetadata(Unit, VMap, RF_None, TypeMapper, Materializer);
diff --git a/llvm/lib/Transforms/Utils/LCSSA.cpp b/llvm/lib/Transforms/Utils/LCSSA.cpp
index d76a0c2e4ce75..c3c3cdf50a985 100644
--- a/llvm/lib/Transforms/Utils/LCSSA.cpp
+++ b/llvm/lib/Transforms/Utils/LCSSA.cpp
@@ -99,10 +99,10 @@ formLCSSAForInstructionsImpl(SmallVectorImpl<Instruction *> &Worklist,
     BasicBlock *InstBB = I->getParent();
     Loop *L = LI.getLoopFor(InstBB);
     assert(L && "Instruction belongs to a BB that's not part of a loop");
-    if (!LoopExitBlocks.count(L))
-      L->getExitBlocks(LoopExitBlocks[L]);
-    assert(LoopExitBlocks.count(L));
-    const SmallVectorImpl<BasicBlock *> &ExitBlocks = LoopExitBlocks[L];
+    auto [It, Inserted] = LoopExitBlocks.try_emplace(L);
+    if (Inserted)
+      L->getExitBlocks(It->second);
+    const SmallVectorImpl<BasicBlock *> &ExitBlocks = It->second;
 
     if (ExitBlocks.empty())
       continue;
@@ -389,9 +389,10 @@ static bool formLCSSAImpl(Loop &L, const DominatorTree &DT, const LoopInfo *LI,
   }
 #endif
 
-  if (!LoopExitBlocks.count(&L))
-    L.getExitBlocks(LoopExitBlocks[&L]);
-  const SmallVectorImpl<BasicBlock *> &ExitBlocks = LoopExitBlocks[&L];
+  auto [It, Inserted] = LoopExitBlocks.try_emplace(&L);
+  if (Inserted)
+    L.getExitBlocks(It->second);
+  const SmallVectorImpl<BasicBlock *> &ExitBlocks = It->second;
   if (ExitBlocks.empty())
     return false;
 
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index eb03689f31949..833dc7331aecd 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -6883,9 +6883,10 @@ static bool switchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
     for (const auto &I : Results) {
       PHINode *PHI = I.first;
       Constant *Value = I.second;
-      if (!ResultLists.count(PHI))
+      auto [It, Inserted] = ResultLists.try_emplace(PHI);
+      if (Inserted)
         PHIs.push_back(PHI);
-      ResultLists[PHI].push_back(std::make_pair(CaseVal, Value));
+      It->second.push_back(std::make_pair(CaseVal, Value));
     }
   }
 
diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index 38670ba304e53..7dac6d0059b26 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -8,6 +8,7 @@ add_llvm_component_library(LLVMVectorize
   SandboxVectorizer/Interval.cpp
   SandboxVectorizer/Legality.cpp
   SandboxVectorizer/Passes/BottomUpVec.cpp
+  SandboxVectorizer/Passes/RegionsFromBBs.cpp
   SandboxVectorizer/Passes/RegionsFromMetadata.cpp
   SandboxVectorizer/Passes/SeedCollection.cpp
   SandboxVectorizer/Passes/TransactionAcceptOrRevert.cpp
@@ -22,6 +23,7 @@ add_llvm_component_library(LLVMVectorize
   VectorCombine.cpp
   VPlan.cpp
   VPlanAnalysis.cpp
+  VPlanConstruction.cpp
   VPlanHCFGBuilder.cpp
   VPlanRecipes.cpp
   VPlanSLP.cpp
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 74ddf906ff9fd..bab2c6efd4035 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -494,12 +494,7 @@ class InnerLoopVectorizer {
         MinProfitableTripCount(MinProfitableTripCount), UF(UnrollFactor),
         Builder(PSE.getSE()->getContext()), Legal(LVL), Cost(CM), BFI(BFI),
         PSI(PSI), RTChecks(RTChecks), Plan(Plan),
-        VectorPHVPB(Plan.getEntry()->getSingleSuccessor()) {
-    // Query this against the original loop and save it here because the profile
-    // of the original loop header may change as the transformation happens.
-    OptForSizeBasedOnProfile = llvm::shouldOptimizeForSize(
-        OrigLoop->getHeader(), PSI, BFI, PGSOQueryType::IRPass);
-  }
+        VectorPHVPB(Plan.getEntry()->getSingleSuccessor()) {}
 
   virtual ~InnerLoopVectorizer() = default;
 
@@ -672,10 +667,6 @@ class InnerLoopVectorizer {
   BlockFrequencyInfo *BFI;
   ProfileSummaryInfo *PSI;
 
-  // Whether this loop should be optimized for size based on profile guided size
-  // optimizatios.
-  bool OptForSizeBasedOnProfile;
-
   /// Structure to hold information about generated runtime checks, responsible
   /// for cleaning the checks, if vectorization turns out unprofitable.
   GeneratedRTChecks &RTChecks;
@@ -968,8 +959,6 @@ enum ScalarEpilogueLowering {
   CM_ScalarEpilogueNotAllowedUsePredicate
 };
 
-using InstructionVFPair = std::pair<Instruction *, ElementCount>;
-
 /// LoopVectorizationCostModel - estimates the expected speedups due to
 /// vectorization.
 /// In many cases vectorization is not profitable. This can happen because of
@@ -989,13 +978,18 @@ class LoopVectorizationCostModel {
                              AssumptionCache *AC,
                              OptimizationRemarkEmitter *ORE, const Function *F,
                              const LoopVectorizeHints *Hints,
-                             InterleavedAccessInfo &IAI)
+                             InterleavedAccessInfo &IAI,
+                             ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI)
       : ScalarEpilogueStatus(SEL), TheLoop(L), PSE(PSE), LI(LI), Legal(Legal),
         TTI(TTI), TLI(TLI), DB(DB), AC(AC), ORE(ORE), TheFunction(F),
         Hints(Hints), InterleaveInfo(IAI) {
     if (TTI.supportsScalableVectors() || ForceTargetSupportsScalableVectors)
       initializeVScaleForTuning();
     CostKind = F->hasMinSize() ? TTI::TCK_CodeSize : TTI::TCK_RecipThroughput;
+    // Query this against the original loop and save it here because the profile
+    // of the original loop header may change as the transformation happens.
+    OptForSize = llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI,
+                                             PGSOQueryType::IRPass);
   }
 
   /// \return An upper bound for the vectorization factors (both fixed and
@@ -1832,6 +1826,10 @@ class LoopVectorizationCostModel {
 
   /// The kind of cost that we are calculating
   TTI::TargetCostKind CostKind;
+
+  /// Whether this loop should be optimized for size based on function attribute
+  /// or profile information.
+  bool OptForSize;
 };
 } // end namespace llvm
 
@@ -2325,8 +2323,8 @@ emitTransformedIndex(IRBuilderBase &B, Value *Index, Value *StartValue,
   llvm_unreachable("invalid enum");
 }
 
-std::optional<unsigned> getMaxVScale(const Function &F,
-                                     const TargetTransformInfo &TTI) {
+static std::optional<unsigned> getMaxVScale(const Function &F,
+                                            const TargetTransformInfo &TTI) {
   if (std::optional<unsigned> MaxVScale = TTI.getMaxVScale())
     return MaxVScale;
 
@@ -2604,9 +2602,8 @@ BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) {
   if (!SCEVCheckBlock)
     return nullptr;
 
-  assert(!(SCEVCheckBlock->getParent()->hasOptSize() ||
-           (OptForSizeBasedOnProfile &&
-            Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) &&
+  assert((!Cost->OptForSize ||
+          Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled) &&
          "Cannot SCEV check stride or overflow when optimizing for size");
   assert(!LoopBypassBlocks.empty() &&
          "Should already be a bypass block due to iteration count check");
@@ -2618,10 +2615,6 @@ BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) {
 }
 
 BasicBlock *InnerLoopVectorizer::emitMemRuntimeChecks(BasicBlock *Bypass) {
-  // VPlan-native path does not do any analysis for runtime checks currently.
-  if (EnableVPlanNativePath)
-    return nullptr;
-
   BasicBlock *const MemCheckBlock =
       RTChecks.emitMemRuntimeChecks(Bypass, LoopVectorPreHeader);
 
@@ -2631,7 +2624,11 @@ BasicBlock *InnerLoopVectorizer::emitMemRuntimeChecks(BasicBlock *Bypass) {
   if (!MemCheckBlock)
     return nullptr;
 
-  if (MemCheckBlock->getParent()->hasOptSize() || OptForSizeBasedOnProfile) {
+  // VPlan-native path does not do any analysis for runtime checks currently.
+  assert((!EnableVPlanNativePath || OrigLoop->begin() == OrigLoop->end()) &&
+         "Runtime checks are not supported for outer loops yet");
+
+  if (Cost->OptForSize) {
     assert(Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled &&
            "Cannot emit memory checks when optimizing for size, unless forced "
            "to vectorize.");
@@ -4396,10 +4393,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
   sort(InvalidCosts, [&Numbering](RecipeVFPair &A, RecipeVFPair &B) {
     if (Numbering[A.first] != Numbering[B.first])
       return Numbering[A.first] < Numbering[B.first];
-    const auto &LHS = A.second;
-    const auto &RHS = B.second;
-    return std::make_tuple(LHS.isScalable(), LHS.getKnownMinValue()) <
-           std::make_tuple(RHS.isScalable(), RHS.getKnownMinValue());
+    return ElementCount::isKnownLT(A.second, B.second);
   });
 
   // For a list of ordered recipe-VF pairs:
@@ -9312,14 +9306,17 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
             return !CM.requiresScalarEpilogue(VF.isVector());
           },
           Range);
-  VPlanPtr Plan = VPlan::createInitialVPlan(Legal->getWidestInductionType(),
-                                            PSE, RequiresScalarEpilogueCheck,
-                                            CM.foldTailByMasking(), OrigLoop);
-
+  auto Plan = std::make_unique<VPlan>(OrigLoop);
   // Build hierarchical CFG.
+  // Convert to VPlan-transform and consoliate all transforms for VPlan
+  // creation.
   VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan);
   HCFGBuilder.buildHierarchicalCFG();
 
+  VPlanTransforms::introduceTopLevelVectorLoopRegion(
+      *Plan, Legal->getWidestInductionType(), PSE, RequiresScalarEpilogueCheck,
+      CM.foldTailByMasking(), OrigLoop);
+
   // Don't use getDecisionAndClampRange here, because we don't know the UF
   // so this function is better to be conservative, rather than to split
   // it up into different VPlans.
@@ -9615,13 +9612,14 @@ VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) {
   assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
 
   // Create new empty VPlan
-  auto Plan = VPlan::createInitialVPlan(Legal->getWidestInductionType(), PSE,
-                                        true, false, OrigLoop);
-
+  auto Plan = std::make_unique<VPlan>(OrigLoop);
   // Build hierarchical CFG
   VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan);
   HCFGBuilder.buildHierarchicalCFG();
 
+  VPlanTransforms::introduceTopLevelVectorLoopRegion(
+      *Plan, Legal->getWidestInductionType(), PSE, true, false, OrigLoop);
+
   for (ElementCount VF : Range)
     Plan->addVF(VF);
 
@@ -10081,7 +10079,7 @@ static bool processLoopInVPlanNativePath(
       getScalarEpilogueLowering(F, L, Hints, PSI, BFI, TTI, TLI, *LVL, &IAI);
 
   LoopVectorizationCostModel CM(SEL, L, PSE, LI, LVL, *TTI, TLI, DB, AC, ORE, F,
-                                &Hints, IAI);
+                                &Hints, IAI, PSI, BFI);
   // Use the planner for outer loop vectorization.
   // TODO: CM is not used at this point inside the planner. Turn CM into an
   // optional argument if we don't need it in the future.
@@ -10169,19 +10167,56 @@ static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE) {
   }
 }
 
-static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
-                                       VectorizationFactor &VF, Loop *L,
-                                       PredicatedScalarEvolution &PSE,
-                                       ScalarEpilogueLowering SEL,
-                                       std::optional<unsigned> VScale) {
-  InstructionCost CheckCost = Checks.getCost();
-  if (!CheckCost.isValid())
+/// For loops with uncountable early exits, find the cost of doing work when
+/// exiting the loop early, such as calculating the final exit values of
+/// variables used outside the loop.
+/// TODO: This is currently overly pessimistic because the loop may not take
+/// the early exit, but better to keep this conservative for now. In future,
+/// it might be possible to relax this by using branch probabilities.
+static InstructionCost calculateEarlyExitCost(VPCostContext &CostCtx,
+                                              VPlan &Plan, ElementCount VF) {
+  InstructionCost Cost = 0;
+  for (auto *ExitVPBB : Plan.getExitBlocks()) {
+    for (auto *PredVPBB : ExitVPBB->getPredecessors()) {
+      // If the predecessor is not the middle.block, then it must be the
+      // vector.early.exit block, which may contain work to calculate the exit
+      // values of variables used outside the loop.
+      if (PredVPBB != Plan.getMiddleBlock()) {
+        LLVM_DEBUG(dbgs() << "Calculating cost of work in exit block "
+                          << PredVPBB->getName() << ":\n");
+        Cost += PredVPBB->cost(VF, CostCtx);
+      }
+    }
+  }
+  return Cost;
+}
+
+/// This function determines whether or not it's still profitable to vectorize
+/// the loop given the extra work we have to do outside of the loop:
+///  1. Perform the runtime checks before entering the loop to ensure it's safe
+///     to vectorize.
+///  2. In the case of loops with uncountable early exits, we may have to do
+///     extra work when exiting the loop early, such as calculating the final
+///     exit values of variables used outside the loop.
+static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
+                                        VectorizationFactor &VF, Loop *L,
+                                        PredicatedScalarEvolution &PSE,
+                                        VPCostContext &CostCtx, VPlan &Plan,
+                                        ScalarEpilogueLowering SEL,
+                                        std::optional<unsigned> VScale) {
+  InstructionCost TotalCost = Checks.getCost();
+  if (!TotalCost.isValid())
     return false;
 
+  // Add on the cost of any work required in the vector early exit block, if
+  // one exists.
+  TotalCost += calculateEarlyExitCost(CostCtx, Plan, VF.Width);
+
   // When interleaving only scalar and vector cost will be equal, which in turn
   // would lead to a divide by 0. Fall back to hard threshold.
   if (VF.Width.isScalar()) {
-    if (CheckCost > VectorizeMemoryCheckThreshold) {
+    // TODO: Should we rename VectorizeMemoryCheckThreshold?
+    if (TotalCost > VectorizeMemoryCheckThreshold) {
       LLVM_DEBUG(
           dbgs()
           << "LV: Interleaving only is not profitable due to runtime checks\n");
@@ -10207,7 +10242,9 @@ static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
   //  The total cost of the vector loop is
   //    RtC + VecC * (TC / VF) + EpiC
   //  where
-  //  * RtC is the cost of the generated runtime checks
+  //  * RtC is the cost of the generated runtime checks plus the cost of
+  //    performing any additional work in the vector.early.exit block for loops
+  //    with uncountable early exits.
   //  * VecC is the cost of a single vector iteration.
   //  * TC is the actual trip count of the loop
   //  * VF is the vectorization factor
@@ -10225,7 +10262,7 @@ static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
   // the computations are performed on doubles, not integers and the result
   // is rounded up, hence we get an upper estimate of the TC.
   unsigned IntVF = getEstimatedRuntimeVF(VF.Width, VScale);
-  uint64_t RtC = *CheckCost.getValue();
+  uint64_t RtC = *TotalCost.getValue();
   uint64_t Div = ScalarC * IntVF - *VF.Cost.getValue();
   uint64_t MinTC1 = Div == 0 ? 0 : divideCeil(RtC * IntVF, Div);
 
@@ -10553,8 +10590,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
         // iteration count is low. However, setting the epilogue policy to
         // `CM_ScalarEpilogueNotAllowedLowTripLoop` prevents vectorizing loops
         // with runtime checks. It's more effective to let
-        // `areRuntimeChecksProfitable` determine if vectorization is beneficial
-        // for the loop.
+        // `isOutsideLoopWorkProfitable` determine if vectorization is
+        // beneficial for the loop.
         if (SEL != CM_ScalarEpilogueNotNeededUsePredicate)
           SEL = CM_ScalarEpilogueNotAllowedLowTripLoop;
       } else {
@@ -10618,7 +10655,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
 
   // Use the cost model.
   LoopVectorizationCostModel CM(SEL, L, PSE, LI, &LVL, *TTI, TLI, DB, AC, ORE,
-                                F, &Hints, IAI);
+                                F, &Hints, IAI, PSI, BFI);
   // Use the planner for vectorization.
   LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, IAI, PSE, Hints,
                                ORE);
@@ -10652,9 +10689,12 @@ bool LoopVectorizePass::processLoop(Loop *L) {
     // Check if it is profitable to vectorize with runtime checks.
     bool ForceVectorization =
         Hints.getForce() == LoopVectorizeHints::FK_Enabled;
+    VPCostContext CostCtx(CM.TTI, *CM.TLI, CM.Legal->getWidestInductionType(),
+                          CM, CM.CostKind);
     if (!ForceVectorization &&
-        !areRuntimeChecksProfitable(Checks, VF, L, PSE, SEL,
-                                    CM.getVScaleForTuning())) {
+        !isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx,
+                                     LVP.getPlanFor(VF.Width), SEL,
+                                     CM.getVScaleForTuning())) {
       ORE->emit([&]() {
         return OptimizationRemarkAnalysisAliasing(
                    DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
index f745073a1eba6..c525608804955 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/PassRegistry.def
@@ -31,6 +31,7 @@ REGION_PASS("bottom-up-vec", ::llvm::sandboxir::BottomUpVec)
 #endif
 
 FUNCTION_PASS_WITH_PARAMS("seed-collection", ::llvm::sandboxir::SeedCollection)
+FUNCTION_PASS_WITH_PARAMS("regions-from-bbs", ::llvm::sandboxir::RegionsFromBBs)
 FUNCTION_PASS_WITH_PARAMS("regions-from-metadata", ::llvm::sandboxir::RegionsFromMetadata)
 
 #undef FUNCTION_PASS_WITH_PARAMS
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromBBs.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromBBs.cpp
new file mode 100644
index 0000000000000..6801524732ce6
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromBBs.cpp
@@ -0,0 +1,35 @@
+//===- RegionsFromBBs.cpp - A helper to test RegionPasses -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromBBs.h"
+#include "llvm/SandboxIR/Function.h"
+#include "llvm/SandboxIR/Region.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.h"
+
+namespace llvm::sandboxir {
+
+RegionsFromBBs::RegionsFromBBs(StringRef Pipeline)
+    : FunctionPass("regions-from-bbs"),
+      RPM("rpm", Pipeline, SandboxVectorizerPassBuilder::createRegionPass) {}
+
+bool RegionsFromBBs::runOnFunction(Function &F, const Analyses &A) {
+  SmallVector<std::unique_ptr<Region>, 16> Regions;
+  // Create a region for each BB.
+  for (BasicBlock &BB : F) {
+    Regions.push_back(std::make_unique<Region>(F.getContext(), A.getTTI()));
+    auto &RgnPtr = Regions.back();
+    for (Instruction &I : BB)
+      RgnPtr->add(&I);
+  }
+  // For each region run the region pass pipeline.
+  for (auto &RgnPtr : Regions)
+    RPM.runOnRegion(*RgnPtr, A);
+  return false;
+}
+
+} // namespace llvm::sandboxir
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
index 389f9cc4cae7c..013ccf6e3d945 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizerPassBuilder.cpp
@@ -3,6 +3,7 @@
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/NullPass.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/PrintInstructionCount.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromBBs.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/RegionsFromMetadata.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/TransactionAcceptOrRevert.h"
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 563784e4af924..e595347d62bf5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -19,6 +19,7 @@
 #include "VPlan.h"
 #include "LoopVectorizationPlanner.h"
 #include "VPlanCFG.h"
+#include "VPlanDominatorTree.h"
 #include "VPlanHelpers.h"
 #include "VPlanPatternMatch.h"
 #include "VPlanTransforms.h"
@@ -221,7 +222,7 @@ VPTransformState::VPTransformState(const TargetTransformInfo *TTI,
                                    Loop *CurrentParentLoop, Type *CanonicalIVTy)
     : TTI(TTI), VF(VF), CFG(DT), LI(LI), Builder(Builder), ILV(ILV), Plan(Plan),
       CurrentParentLoop(CurrentParentLoop), LVer(nullptr),
-      TypeAnalysis(CanonicalIVTy) {}
+      TypeAnalysis(CanonicalIVTy), VPDT(*Plan) {}
 
 Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
   if (Def->isLiveIn())
@@ -264,7 +265,11 @@ Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) {
     return Data.VPV2Vector[Def];
 
   auto GetBroadcastInstrs = [this, Def](Value *V) {
-    bool SafeToHoist = Def->isDefinedOutsideLoopRegions();
+    bool SafeToHoist =
+        !Def->hasDefiningRecipe() ||
+        VPDT.properlyDominates(Def->getDefiningRecipe()->getParent(),
+                               Plan->getVectorPreheader());
+
     if (VF.isScalar())
       return V;
     // Place the code for broadcasting invariant variables in the new preheader.
@@ -880,85 +885,6 @@ VPlan::~VPlan() {
     delete BackedgeTakenCount;
 }
 
-VPlanPtr VPlan::createInitialVPlan(Type *InductionTy,
-                                   PredicatedScalarEvolution &PSE,
-                                   bool RequiresScalarEpilogueCheck,
-                                   bool TailFolded, Loop *TheLoop) {
-  auto Plan = std::make_unique<VPlan>(TheLoop);
-  VPBlockBase *ScalarHeader = Plan->getScalarHeader();
-
-  // Connect entry only to vector preheader initially. Entry will also be
-  // connected to the scalar preheader later, during skeleton creation when
-  // runtime guards are added as needed. Note that when executing the VPlan for
-  // an epilogue vector loop, the original entry block here will be replaced by
-  // a new VPIRBasicBlock wrapping the entry to the epilogue vector loop after
-  // generating code for the main vector loop.
-  VPBasicBlock *VecPreheader = Plan->createVPBasicBlock("vector.ph");
-  VPBlockUtils::connectBlocks(Plan->getEntry(), VecPreheader);
-
-  // Create SCEV and VPValue for the trip count.
-  // We use the symbolic max backedge-taken-count, which works also when
-  // vectorizing loops with uncountable early exits.
-  const SCEV *BackedgeTakenCountSCEV = PSE.getSymbolicMaxBackedgeTakenCount();
-  assert(!isa<SCEVCouldNotCompute>(BackedgeTakenCountSCEV) &&
-         "Invalid loop count");
-  ScalarEvolution &SE = *PSE.getSE();
-  const SCEV *TripCount = SE.getTripCountFromExitCount(BackedgeTakenCountSCEV,
-                                                       InductionTy, TheLoop);
-  Plan->TripCount =
-      vputils::getOrCreateVPValueForSCEVExpr(*Plan, TripCount, SE);
-
-  // Create VPRegionBlock, with empty header and latch blocks, to be filled
-  // during processing later.
-  VPBasicBlock *HeaderVPBB = Plan->createVPBasicBlock("vector.body");
-  VPBasicBlock *LatchVPBB = Plan->createVPBasicBlock("vector.latch");
-  VPBlockUtils::insertBlockAfter(LatchVPBB, HeaderVPBB);
-  auto *TopRegion = Plan->createVPRegionBlock(
-      HeaderVPBB, LatchVPBB, "vector loop", false /*isReplicator*/);
-
-  VPBlockUtils::insertBlockAfter(TopRegion, VecPreheader);
-  VPBasicBlock *MiddleVPBB = Plan->createVPBasicBlock("middle.block");
-  VPBlockUtils::insertBlockAfter(MiddleVPBB, TopRegion);
-
-  VPBasicBlock *ScalarPH = Plan->createVPBasicBlock("scalar.ph");
-  VPBlockUtils::connectBlocks(ScalarPH, ScalarHeader);
-  if (!RequiresScalarEpilogueCheck) {
-    VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
-    return Plan;
-  }
-
-  // If needed, add a check in the middle block to see if we have completed
-  // all of the iterations in the first vector loop.  Three cases:
-  // 1) If (N - N%VF) == N, then we *don't* need to run the remainder.
-  //    Thus if tail is to be folded, we know we don't need to run the
-  //    remainder and we can set the condition to true.
-  // 2) If we require a scalar epilogue, there is no conditional branch as
-  //    we unconditionally branch to the scalar preheader.  Do nothing.
-  // 3) Otherwise, construct a runtime check.
-  BasicBlock *IRExitBlock = TheLoop->getUniqueLatchExitBlock();
-  VPIRBasicBlock *VPExitBlock = Plan->getExitBlock(IRExitBlock);
-  // The connection order corresponds to the operands of the conditional branch.
-  VPBlockUtils::insertBlockAfter(VPExitBlock, MiddleVPBB);
-  VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
-
-  auto *ScalarLatchTerm = TheLoop->getLoopLatch()->getTerminator();
-  // Here we use the same DebugLoc as the scalar loop latch terminator instead
-  // of the corresponding compare because they may have ended up with
-  // different line numbers and we want to avoid awkward line stepping while
-  // debugging. Eg. if the compare has got a line number inside the loop.
-  VPBuilder Builder(MiddleVPBB);
-  VPValue *Cmp =
-      TailFolded
-          ? Plan->getOrAddLiveIn(ConstantInt::getTrue(
-                IntegerType::getInt1Ty(TripCount->getType()->getContext())))
-          : Builder.createICmp(CmpInst::ICMP_EQ, Plan->getTripCount(),
-                               &Plan->getVectorTripCount(),
-                               ScalarLatchTerm->getDebugLoc(), "cmp.n");
-  Builder.createNaryOp(VPInstruction::BranchOnCond, {Cmp},
-                       ScalarLatchTerm->getDebugLoc());
-  return Plan;
-}
-
 void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
                              VPTransformState &State) {
   Type *TCTy = TripCountV->getType();
@@ -1008,6 +934,10 @@ void VPlan::execute(VPTransformState *State) {
   State->CFG.PrevVPBB = nullptr;
   State->CFG.ExitBB = State->CFG.PrevBB->getSingleSuccessor();
 
+  // Update VPDominatorTree since VPBasicBlock may be removed after State was
+  // constructed.
+  State->VPDT.recalculate(*this);
+
   // Disconnect VectorPreHeader from ExitBB in both the CFG and DT.
   BasicBlock *VectorPreHeader = State->CFG.PrevBB;
   cast<BranchInst>(VectorPreHeader->getTerminator())->setSuccessor(0, nullptr);
@@ -1135,11 +1065,13 @@ void VPlan::printLiveIns(raw_ostream &O) const {
   }
 
   O << "\n";
-  if (TripCount->isLiveIn())
-    O << "Live-in ";
-  TripCount->printAsOperand(O, SlotTracker);
-  O << " = original trip-count";
-  O << "\n";
+  if (TripCount) {
+    if (TripCount->isLiveIn())
+      O << "Live-in ";
+    TripCount->printAsOperand(O, SlotTracker);
+    O << " = original trip-count";
+    O << "\n";
+  }
 }
 
 LLVM_DUMP_METHOD
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b277ed4816b8e..43dc30c40bb53 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1641,10 +1641,8 @@ class VPVectorPointerRecipe : public VPRecipeWithIRFlags,
 class VPHeaderPHIRecipe : public VPSingleDefRecipe {
 protected:
   VPHeaderPHIRecipe(unsigned char VPDefID, Instruction *UnderlyingInstr,
-                    VPValue *Start = nullptr, DebugLoc DL = {})
-      : VPSingleDefRecipe(VPDefID, ArrayRef<VPValue *>(), UnderlyingInstr, DL) {
-    if (Start)
-      addOperand(Start);
+                    VPValue *Start, DebugLoc DL = {})
+      : VPSingleDefRecipe(VPDefID, ArrayRef<VPValue *>({Start}), UnderlyingInstr, DL) {
   }
 
 public:
@@ -3503,21 +3501,6 @@ class VPlan {
     VPBB->setPlan(this);
   }
 
-  /// Create initial VPlan, having an "entry" VPBasicBlock (wrapping
-  /// original scalar pre-header) which contains SCEV expansions that need
-  /// to happen before the CFG is modified (when executing a VPlan for the
-  /// epilogue vector loop, the original entry needs to be replaced by a new
-  /// one); a VPBasicBlock for the vector pre-header, followed by a region for
-  /// the vector loop, followed by the middle VPBasicBlock. If a check is needed
-  /// to guard executing the scalar epilogue loop, it will be added to the
-  /// middle block, together with VPBasicBlocks for the scalar preheader and
-  /// exit blocks. \p InductionTy is the type of the canonical induction and
-  /// used for related values, like the trip count expression.
-  static VPlanPtr createInitialVPlan(Type *InductionTy,
-                                     PredicatedScalarEvolution &PSE,
-                                     bool RequiresScalarEpilogueCheck,
-                                     bool TailFolded, Loop *TheLoop);
-
   /// Prepare the plan for execution, setting up the required live-in values.
   void prepareToExecute(Value *TripCount, Value *VectorTripCount,
                         VPTransformState &State);
@@ -3579,11 +3562,18 @@ class VPlan {
     return TripCount;
   }
 
+  /// Set the trip count assuming it is currently null; if it is not - use
+  /// resetTripCount().
+  void setTripCount(VPValue *NewTripCount) {
+    assert(!TripCount && NewTripCount && "TripCount should not be set yet.");
+    TripCount = NewTripCount;
+  }
+
   /// Resets the trip count for the VPlan. The caller must make sure all uses of
   /// the original trip count have been replaced.
   void resetTripCount(VPValue *NewTripCount) {
     assert(TripCount && NewTripCount && TripCount->getNumUsers() == 0 &&
-           "TripCount always must be set");
+           "TripCount must be set when resetting");
     TripCount = NewTripCount;
   }
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
new file mode 100644
index 0000000000000..f58f0290b5fa9
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -0,0 +1,101 @@
+//===-- VPlanConstruction.cpp - Transforms for initial VPlan construction -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements transforms for initial VPlan construction.
+///
+//===----------------------------------------------------------------------===//
+
+#include "LoopVectorizationPlanner.h"
+#include "VPlan.h"
+#include "VPlanCFG.h"
+#include "VPlanTransforms.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+
+using namespace llvm;
+
+void VPlanTransforms::introduceTopLevelVectorLoopRegion(
+    VPlan &Plan, Type *InductionTy, PredicatedScalarEvolution &PSE,
+    bool RequiresScalarEpilogueCheck, bool TailFolded, Loop *TheLoop) {
+  // TODO: Generalize to introduce all loop regions.
+  auto *HeaderVPBB = cast<VPBasicBlock>(Plan.getEntry()->getSingleSuccessor());
+  VPBlockUtils::disconnectBlocks(Plan.getEntry(), HeaderVPBB);
+
+  VPBasicBlock *OriginalLatch =
+      cast<VPBasicBlock>(HeaderVPBB->getSinglePredecessor());
+  VPBlockUtils::disconnectBlocks(OriginalLatch, HeaderVPBB);
+  VPBasicBlock *VecPreheader = Plan.createVPBasicBlock("vector.ph");
+  VPBlockUtils::connectBlocks(Plan.getEntry(), VecPreheader);
+  assert(OriginalLatch->getNumSuccessors() == 0 &&
+         "Plan should end at top level latch");
+
+  // Create SCEV and VPValue for the trip count.
+  // We use the symbolic max backedge-taken-count, which works also when
+  // vectorizing loops with uncountable early exits.
+  const SCEV *BackedgeTakenCountSCEV = PSE.getSymbolicMaxBackedgeTakenCount();
+  assert(!isa<SCEVCouldNotCompute>(BackedgeTakenCountSCEV) &&
+         "Invalid loop count");
+  ScalarEvolution &SE = *PSE.getSE();
+  const SCEV *TripCount = SE.getTripCountFromExitCount(BackedgeTakenCountSCEV,
+                                                       InductionTy, TheLoop);
+  Plan.setTripCount(
+      vputils::getOrCreateVPValueForSCEVExpr(Plan, TripCount, SE));
+
+  // Create VPRegionBlock, with existing header and new empty latch block, to be
+  // filled.
+  VPBasicBlock *LatchVPBB = Plan.createVPBasicBlock("vector.latch");
+  VPBlockUtils::insertBlockAfter(LatchVPBB, OriginalLatch);
+  auto *TopRegion = Plan.createVPRegionBlock(
+      HeaderVPBB, LatchVPBB, "vector loop", false /*isReplicator*/);
+  // All VPBB's reachable shallowly from HeaderVPBB belong to top level loop,
+  // because VPlan is expected to end at top level latch.
+  for (VPBlockBase *VPBB : vp_depth_first_shallow(HeaderVPBB))
+    VPBB->setParent(TopRegion);
+
+  VPBlockUtils::insertBlockAfter(TopRegion, VecPreheader);
+  VPBasicBlock *MiddleVPBB = Plan.createVPBasicBlock("middle.block");
+  VPBlockUtils::insertBlockAfter(MiddleVPBB, TopRegion);
+
+  VPBasicBlock *ScalarPH = Plan.createVPBasicBlock("scalar.ph");
+  VPBlockUtils::connectBlocks(ScalarPH, Plan.getScalarHeader());
+  if (!RequiresScalarEpilogueCheck) {
+    VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
+    return;
+  }
+
+  // If needed, add a check in the middle block to see if we have completed
+  // all of the iterations in the first vector loop.  Three cases:
+  // 1) If (N - N%VF) == N, then we *don't* need to run the remainder.
+  //    Thus if tail is to be folded, we know we don't need to run the
+  //    remainder and we can set the condition to true.
+  // 2) If we require a scalar epilogue, there is no conditional branch as
+  //    we unconditionally branch to the scalar preheader.  Do nothing.
+  // 3) Otherwise, construct a runtime check.
+  BasicBlock *IRExitBlock = TheLoop->getUniqueLatchExitBlock();
+  auto *VPExitBlock = Plan.getExitBlock(IRExitBlock);
+  // The connection order corresponds to the operands of the conditional branch.
+  VPBlockUtils::insertBlockAfter(VPExitBlock, MiddleVPBB);
+  VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
+
+  auto *ScalarLatchTerm = TheLoop->getLoopLatch()->getTerminator();
+  // Here we use the same DebugLoc as the scalar loop latch terminator instead
+  // of the corresponding compare because they may have ended up with
+  // different line numbers and we want to avoid awkward line stepping while
+  // debugging. Eg. if the compare has got a line number inside the loop.
+  VPBuilder Builder(MiddleVPBB);
+  VPValue *Cmp =
+      TailFolded
+          ? Plan.getOrAddLiveIn(ConstantInt::getTrue(
+                IntegerType::getInt1Ty(TripCount->getType()->getContext())))
+          : Builder.createICmp(CmpInst::ICMP_EQ, Plan.getTripCount(),
+                               &Plan.getVectorTripCount(),
+                               ScalarLatchTerm->getDebugLoc(), "cmp.n");
+  Builder.createNaryOp(VPInstruction::BranchOnCond, {Cmp},
+                       ScalarLatchTerm->getDebugLoc());
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h b/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h
index 787cd940c4e26..995c6b8b2c2fb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanDominatorTree.h
@@ -39,8 +39,10 @@ class VPDominatorTree : public DominatorTreeBase<VPBlockBase, false> {
 
 public:
   VPDominatorTree() = default;
+  explicit VPDominatorTree(VPlan &Plan) { recalculate(Plan); }
 
   /// Returns true if \p A properly dominates \p B.
+  using Base::properlyDominates;
   bool properlyDominates(const VPRecipeBase *A, const VPRecipeBase *B);
 };
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
index a5e8e852bace8..4b8a2420b3037 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
@@ -23,6 +23,7 @@
 
 #include "VPlanHCFGBuilder.h"
 #include "LoopVectorizationPlanner.h"
+#include "VPlanCFG.h"
 #include "llvm/Analysis/LoopIterator.h"
 
 #define DEBUG_TYPE "loop-vectorize"
@@ -178,9 +179,10 @@ VPBasicBlock *PlainCFGBuilder::getOrCreateVPBB(BasicBlock *BB) {
   VPBasicBlock *VPBB = Plan.createVPBasicBlock(Name);
   BB2VPBB[BB] = VPBB;
 
-  // Get or create a region for the loop containing BB.
+  // Get or create a region for the loop containing BB, except for the top
+  // region of TheLoop which is created later.
   Loop *LoopOfBB = LI->getLoopFor(BB);
-  if (!LoopOfBB || !doesContainLoop(LoopOfBB, TheLoop))
+  if (!LoopOfBB || LoopOfBB == TheLoop || !doesContainLoop(LoopOfBB, TheLoop))
     return VPBB;
 
   auto *RegionOfVPBB = Loop2Region.lookup(LoopOfBB);
@@ -194,12 +196,8 @@ VPBasicBlock *PlainCFGBuilder::getOrCreateVPBB(BasicBlock *BB) {
   assert(!RegionOfVPBB &&
          "First visit of a header basic block expects to register its region.");
   // Handle a header - take care of its Region.
-  if (LoopOfBB == TheLoop) {
-    RegionOfVPBB = Plan.getVectorLoopRegion();
-  } else {
-    RegionOfVPBB = Plan.createVPRegionBlock(Name.str(), false /*isReplicator*/);
-    RegionOfVPBB->setParent(Loop2Region[LoopOfBB->getParentLoop()]);
-  }
+  RegionOfVPBB = Plan.createVPRegionBlock(Name.str(), false /*isReplicator*/);
+  RegionOfVPBB->setParent(Loop2Region[LoopOfBB->getParentLoop()]);
   RegionOfVPBB->setEntry(VPBB);
   Loop2Region[LoopOfBB] = RegionOfVPBB;
   return VPBB;
@@ -353,29 +351,6 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
 // Main interface to build the plain CFG.
 void PlainCFGBuilder::buildPlainCFG(
     DenseMap<VPBlockBase *, BasicBlock *> &VPB2IRBB) {
-  // 0. Reuse the top-level region, vector-preheader and exit VPBBs from the
-  // skeleton. These were created directly rather than via getOrCreateVPBB(),
-  // revisit them now to update BB2VPBB. Note that header/entry and
-  // latch/exiting VPBB's of top-level region have yet to be created.
-  VPRegionBlock *TheRegion = Plan.getVectorLoopRegion();
-  BasicBlock *ThePreheaderBB = TheLoop->getLoopPreheader();
-  assert((ThePreheaderBB->getTerminator()->getNumSuccessors() == 1) &&
-         "Unexpected loop preheader");
-  auto *VectorPreheaderVPBB =
-      cast<VPBasicBlock>(TheRegion->getSinglePredecessor());
-  // ThePreheaderBB conceptually corresponds to both Plan.getPreheader() (which
-  // wraps the original preheader BB) and Plan.getEntry() (which represents the
-  // new vector preheader); here we're interested in setting BB2VPBB to the
-  // latter.
-  BB2VPBB[ThePreheaderBB] = VectorPreheaderVPBB;
-  Loop2Region[LI->getLoopFor(TheLoop->getHeader())] = TheRegion;
-
-  // The existing vector region's entry and exiting VPBBs correspond to the loop
-  // header and latch.
-  VPBasicBlock *VectorHeaderVPBB = TheRegion->getEntryBasicBlock();
-  VPBasicBlock *VectorLatchVPBB = TheRegion->getExitingBasicBlock();
-  BB2VPBB[TheLoop->getHeader()] = VectorHeaderVPBB;
-  VectorHeaderVPBB->clearSuccessors();
 
   // 1. Scan the body of the loop in a topological order to visit each basic
   // block after having visited its predecessor basic blocks. Create a VPBB for
@@ -386,6 +361,9 @@ void PlainCFGBuilder::buildPlainCFG(
 
   // Loop PH needs to be explicitly visited since it's not taken into account by
   // LoopBlocksDFS.
+  BasicBlock *ThePreheaderBB = TheLoop->getLoopPreheader();
+  assert((ThePreheaderBB->getTerminator()->getNumSuccessors() == 1) &&
+         "Unexpected loop preheader");
   for (auto &I : *ThePreheaderBB) {
     if (I.getType()->isVoidTy())
       continue;
@@ -403,21 +381,18 @@ void PlainCFGBuilder::buildPlainCFG(
     // Set VPBB predecessors in the same order as they are in the incoming BB.
     if (!isHeaderBB(BB, LoopForBB)) {
       setVPBBPredsFromBB(VPBB, BB);
-    } else {
-      // BB is a loop header, set the predecessor for the region, except for the
-      // top region, whose predecessor was set when creating VPlan's skeleton.
-      assert(isHeaderVPBB(VPBB) && "isHeaderBB and isHeaderVPBB disagree");
-      if (TheRegion != Region)
-        setRegionPredsFromBB(Region, BB);
+    } else if (Region) {
+      // BB is a loop header and there's a corresponding region, set the
+      // predecessor for it.
+      setRegionPredsFromBB(Region, BB);
     }
 
     // Create VPInstructions for BB.
     createVPInstructionsForVPBB(VPBB, BB);
 
-    if (TheLoop->getLoopLatch() == BB) {
-      VPBB->setOneSuccessor(VectorLatchVPBB);
-      VectorLatchVPBB->clearPredecessors();
-      VectorLatchVPBB->setPredecessors({VPBB});
+    if (BB == TheLoop->getLoopLatch()) {
+      VPBasicBlock *HeaderVPBB = getOrCreateVPBB(LoopForBB->getHeader());
+      VPBlockUtils::connectBlocks(VPBB, HeaderVPBB);
       continue;
     }
 
@@ -451,9 +426,8 @@ void PlainCFGBuilder::buildPlainCFG(
     if (BB == LoopForBB->getLoopLatch()) {
       // For a latch we need to set the successor of the region rather than that
       // of VPBB and it should be set to the exit, i.e., non-header successor,
-      // except for the top region, whose successor was set when creating
-      // VPlan's skeleton.
-      assert(TheRegion != Region &&
+      // except for the top region, which is handled elsewhere.
+      assert(LoopForBB != TheLoop &&
              "Latch of the top region should have been handled earlier");
       Region->setOneSuccessor(isHeaderVPBB(Successor0) ? Successor1
                                                        : Successor0);
@@ -482,6 +456,9 @@ void PlainCFGBuilder::buildPlainCFG(
   // corresponding VPlan operands.
   fixHeaderPhis();
 
+  VPBlockUtils::connectBlocks(Plan.getEntry(),
+                              getOrCreateVPBB(TheLoop->getHeader()));
+
   for (const auto &[IRBB, VPB] : BB2VPBB)
     VPB2IRBB[VPB] = IRBB;
 }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index 8bdbf556efbb3..d53b72bb2258a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -16,6 +16,7 @@
 #define LLVM_TRANSFORMS_VECTORIZE_VPLANHELPERS_H
 
 #include "VPlanAnalysis.h"
+#include "VPlanDominatorTree.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
@@ -360,6 +361,9 @@ struct VPTransformState {
 
   /// VPlan-based type analysis.
   VPTypeAnalysis TypeAnalysis;
+
+  /// VPlan-based dominator tree.
+  VPDominatorTree VPDT;
 };
 
 /// Struct to hold various analysis needed for cost computations.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 5010bf029d140..0d3c9aeec6be4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -27,7 +27,6 @@
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Intrinsics.h"
-#include "llvm/IR/PatternMatch.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
 #include "llvm/IR/VectorBuilder.h"
@@ -743,6 +742,18 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
     return Ctx.TTI.getArithmeticReductionCost(
         Instruction::Or, cast<VectorType>(VecTy), std::nullopt, Ctx.CostKind);
   }
+  case VPInstruction::ExtractFirstActive: {
+    // Calculate the cost of determining the lane index.
+    auto *PredTy = toVectorTy(Ctx.Types.inferScalarType(getOperand(1)), VF);
+    IntrinsicCostAttributes Attrs(Intrinsic::experimental_cttz_elts,
+                                  Type::getInt64Ty(Ctx.LLVMCtx),
+                                  {PredTy, Type::getInt1Ty(Ctx.LLVMCtx)});
+    InstructionCost Cost = Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
+    // Add on the cost of extracting the element.
+    auto *VecTy = toVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF);
+    return Cost + Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy,
+                                             Ctx.CostKind);
+  }
   default:
     // TODO: Compute cost other VPInstructions once the legacy cost model has
     // been retired.
@@ -3615,16 +3626,6 @@ void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent,
                              VPSlotTracker &SlotTracker) const {
   O << Indent << "WIDEN-PHI ";
 
-  auto *OriginalPhi = cast<PHINode>(getUnderlyingValue());
-  // Unless all incoming values are modeled in VPlan  print the original PHI
-  // directly.
-  // TODO: Remove once all VPWidenPHIRecipe instances keep all relevant incoming
-  // values as VPValues.
-  if (getNumOperands() != OriginalPhi->getNumOperands()) {
-    O << VPlanIngredient(OriginalPhi);
-    return;
-  }
-
   printAsOperand(O, SlotTracker);
   O << " = phi ";
   printOperands(O, SlotTracker);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 7646350ca0ed2..ecf52673480e2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -737,67 +737,76 @@ static VPWidenInductionRecipe *getOptimizableIVOf(VPValue *VPV) {
   return IsWideIVInc() ? WideIV : nullptr;
 }
 
-void VPlanTransforms::optimizeInductionExitUsers(
-    VPlan &Plan, DenseMap<VPValue *, VPValue *> &EndValues) {
+/// Attempts to optimize the induction variable exit values for users in the
+/// exit block coming from the latch in the original scalar loop.
+static VPValue *
+optimizeLatchExitInductionUser(VPlan &Plan, VPTypeAnalysis &TypeInfo,
+                               VPBlockBase *PredVPBB, VPValue *Op,
+                               DenseMap<VPValue *, VPValue *> &EndValues) {
   using namespace VPlanPatternMatch;
-  SmallVector<VPIRBasicBlock *> ExitVPBBs(Plan.getExitBlocks());
-  if (ExitVPBBs.size() != 1)
-    return;
-
-  VPIRBasicBlock *ExitVPBB = ExitVPBBs[0];
-  VPBlockBase *PredVPBB = ExitVPBB->getSinglePredecessor();
-  if (!PredVPBB)
-    return;
-  assert(PredVPBB == Plan.getMiddleBlock() &&
-         "predecessor must be the middle block");
 
-  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
-  VPBuilder B(Plan.getMiddleBlock()->getTerminator());
-  for (VPRecipeBase &R : *ExitVPBB) {
-    auto *ExitIRI = cast<VPIRInstruction>(&R);
-    if (!isa<PHINode>(ExitIRI->getInstruction()))
-      break;
+  VPValue *Incoming;
+  if (!match(Op, m_VPInstruction<VPInstruction::ExtractFromEnd>(
+                     m_VPValue(Incoming), m_SpecificInt(1))))
+    return nullptr;
 
-    VPValue *Incoming;
-    if (!match(ExitIRI->getOperand(0),
-               m_VPInstruction<VPInstruction::ExtractFromEnd>(
-                   m_VPValue(Incoming), m_SpecificInt(1))))
-      continue;
+  auto *WideIV = getOptimizableIVOf(Incoming);
+  if (!WideIV)
+    return nullptr;
 
-    auto *WideIV = getOptimizableIVOf(Incoming);
-    if (!WideIV)
-      continue;
-    VPValue *EndValue = EndValues.lookup(WideIV);
-    assert(EndValue && "end value must have been pre-computed");
+  VPValue *EndValue = EndValues.lookup(WideIV);
+  assert(EndValue && "end value must have been pre-computed");
+
+  // `getOptimizableIVOf()` always returns the pre-incremented IV, so if it
+  // changed it means the exit is using the incremented value, so we don't
+  // need to subtract the step.
+  if (Incoming != WideIV)
+    return EndValue;
+
+  // Otherwise, subtract the step from the EndValue.
+  VPBuilder B(cast<VPBasicBlock>(PredVPBB)->getTerminator());
+  VPValue *Step = WideIV->getStepValue();
+  Type *ScalarTy = TypeInfo.inferScalarType(WideIV);
+  if (ScalarTy->isIntegerTy())
+    return B.createNaryOp(Instruction::Sub, {EndValue, Step}, {}, "ind.escape");
+  if (ScalarTy->isPointerTy()) {
+    auto *Zero = Plan.getOrAddLiveIn(
+        ConstantInt::get(Step->getLiveInIRValue()->getType(), 0));
+    return B.createPtrAdd(EndValue,
+                          B.createNaryOp(Instruction::Sub, {Zero, Step}), {},
+                          "ind.escape");
+  }
+  if (ScalarTy->isFloatingPointTy()) {
+    const auto &ID = WideIV->getInductionDescriptor();
+    return B.createNaryOp(
+        ID.getInductionBinOp()->getOpcode() == Instruction::FAdd
+            ? Instruction::FSub
+            : Instruction::FAdd,
+        {EndValue, Step}, {ID.getInductionBinOp()->getFastMathFlags()});
+  }
+  llvm_unreachable("all possible induction types must be handled");
+  return nullptr;
+}
 
-    if (Incoming != WideIV) {
-      ExitIRI->setOperand(0, EndValue);
-      continue;
-    }
+void VPlanTransforms::optimizeInductionExitUsers(
+    VPlan &Plan, DenseMap<VPValue *, VPValue *> &EndValues) {
+  VPBlockBase *MiddleVPBB = Plan.getMiddleBlock();
+  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
+  for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
+    for (VPRecipeBase &R : *ExitVPBB) {
+      auto *ExitIRI = cast<VPIRInstruction>(&R);
+      if (!isa<PHINode>(ExitIRI->getInstruction()))
+        break;
 
-    VPValue *Escape = nullptr;
-    VPValue *Step = WideIV->getStepValue();
-    Type *ScalarTy = TypeInfo.inferScalarType(WideIV);
-    if (ScalarTy->isIntegerTy()) {
-      Escape =
-          B.createNaryOp(Instruction::Sub, {EndValue, Step}, {}, "ind.escape");
-    } else if (ScalarTy->isPointerTy()) {
-      auto *Zero = Plan.getOrAddLiveIn(
-          ConstantInt::get(Step->getLiveInIRValue()->getType(), 0));
-      Escape = B.createPtrAdd(EndValue,
-                              B.createNaryOp(Instruction::Sub, {Zero, Step}),
-                              {}, "ind.escape");
-    } else if (ScalarTy->isFloatingPointTy()) {
-      const auto &ID = WideIV->getInductionDescriptor();
-      Escape = B.createNaryOp(
-          ID.getInductionBinOp()->getOpcode() == Instruction::FAdd
-              ? Instruction::FSub
-              : Instruction::FAdd,
-          {EndValue, Step}, {ID.getInductionBinOp()->getFastMathFlags()});
-    } else {
-      llvm_unreachable("all possible induction types must be handled");
+      for (auto [Idx, PredVPBB] : enumerate(ExitVPBB->getPredecessors())) {
+        if (PredVPBB == MiddleVPBB)
+          if (VPValue *Escape = optimizeLatchExitInductionUser(
+                  Plan, TypeInfo, PredVPBB, ExitIRI->getOperand(Idx),
+                  EndValues))
+            ExitIRI->setOperand(Idx, Escape);
+        // TODO: Optimize early exit induction users in follow-on patch.
+      }
     }
-    ExitIRI->setOperand(0, Escape);
   }
 }
 
@@ -2175,32 +2184,36 @@ void VPlanTransforms::materializeLiveInBroadcasts(VPlan &Plan) {
   if (Plan.hasScalarVFOnly())
     return;
 
+#ifndef NDEBUG
   VPDominatorTree VPDT;
   VPDT.recalculate(Plan);
+#endif
+
   auto *VectorPreheader = Plan.getVectorPreheader();
-  VPBuilder Builder(VectorPreheader);
   for (VPValue *LiveIn : Plan.getLiveIns()) {
     if (all_of(LiveIn->users(),
-               [LiveIn](VPUser *U) {
-                 return cast<VPRecipeBase>(U)->usesScalars(LiveIn);
-               }) ||
+               [LiveIn](VPUser *U) { return U->usesScalars(LiveIn); }) ||
         !LiveIn->getLiveInIRValue() ||
         isa<Constant>(LiveIn->getLiveInIRValue()))
       continue;
 
-    // Add explicit broadcast if the vector preheader dominates all users.
-    // TODO: Find valid insert point for all users.
-    if (all_of(LiveIn->users(), [&VPDT, VectorPreheader](VPUser *U) {
-          return VectorPreheader != cast<VPRecipeBase>(U)->getParent() &&
-                 VPDT.dominates(VectorPreheader,
-                                cast<VPRecipeBase>(U)->getParent());
-        })) {
-      auto *Broadcast =
-          Builder.createNaryOp(VPInstruction::Broadcast, {LiveIn});
-      LiveIn->replaceUsesWithIf(Broadcast, [LiveIn, Broadcast](VPUser &U,
-                                                               unsigned Idx) {
-        return Broadcast != &U && !cast<VPRecipeBase>(&U)->usesScalars(LiveIn);
-      });
+    // Add explicit broadcast at the insert point that dominates all users.
+    VPBasicBlock *HoistBlock = VectorPreheader;
+    VPBasicBlock::iterator HoistPoint = VectorPreheader->end();
+    for (VPUser *User : LiveIn->users()) {
+      if (cast<VPRecipeBase>(User)->getParent() == VectorPreheader)
+        HoistPoint = HoistBlock->begin();
+      else
+        assert(VPDT.dominates(VectorPreheader,
+                              cast<VPRecipeBase>(User)->getParent()) &&
+               "All users must be in the vector preheader or dominated by it");
     }
+
+    VPBuilder Builder(cast<VPBasicBlock>(HoistBlock), HoistPoint);
+    auto *Broadcast = Builder.createNaryOp(VPInstruction::Broadcast, {LiveIn});
+    LiveIn->replaceUsesWithIf(
+        Broadcast, [LiveIn, Broadcast](VPUser &U, unsigned Idx) {
+          return Broadcast != &U && !U.usesScalars(LiveIn);
+        });
   }
 }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index a48bf31cd0af3..8a9a81e00fe23 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -52,6 +52,21 @@ struct VPlanTransforms {
       verifyVPlanIsValid(Plan);
   }
 
+  /// Introduce the top-level VPRegionBlock for the main loop in \p Plan. Coming
+  /// into this function, \p Plan's top-level loop is modeled using a plain CFG.
+  /// This transform wraps the plain CFG of the top-level loop within a
+  /// VPRegionBlock and creates a VPValue expression for the original trip
+  /// count. It will also introduce a dedicated VPBasicBlock for the vector
+  /// pre-header as well a VPBasicBlock as exit block of the region
+  /// (middle.block). If a check is needed to guard executing the scalar
+  /// epilogue loop, it will be added to the middle block, together with
+  /// VPBasicBlocks for the scalar preheader and exit blocks. \p InductionTy is
+  /// the type of the canonical induction and used for related values, like the
+  /// trip count expression.
+  static void introduceTopLevelVectorLoopRegion(
+      VPlan &Plan, Type *InductionTy, PredicatedScalarEvolution &PSE,
+      bool RequiresScalarEpilogueCheck, bool TailFolded, Loop *TheLoop);
+
   /// Replaces the VPInstructions in \p Plan with corresponding
   /// widen recipes.
   static void
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 4d4a1a6e04d32..019d79567b4ae 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -2037,7 +2037,6 @@ bool VectorCombine::foldShuffleOfSelects(Instruction &I) {
                      m_Mask(Mask))))
     return false;
 
-  auto *DstVecTy = dyn_cast<FixedVectorType>(I.getType());
   auto *C1VecTy = dyn_cast<FixedVectorType>(C1->getType());
   auto *C2VecTy = dyn_cast<FixedVectorType>(C2->getType());
   if (!C1VecTy || !C2VecTy || C1VecTy != C2VecTy)
@@ -2051,24 +2050,26 @@ bool VectorCombine::foldShuffleOfSelects(Instruction &I) {
        (SI0FOp->getFastMathFlags() != SI1FOp->getFastMathFlags())))
     return false;
 
+  auto *SrcVecTy = dyn_cast<FixedVectorType>(T1->getType());
+  auto *DstVecTy = dyn_cast<FixedVectorType>(I.getType());
   auto SK = TargetTransformInfo::SK_PermuteTwoSrc;
   auto SelOp = Instruction::Select;
   InstructionCost OldCost = TTI.getCmpSelInstrCost(
-      SelOp, T1->getType(), C1VecTy, CmpInst::BAD_ICMP_PREDICATE, CostKind);
-  OldCost += TTI.getCmpSelInstrCost(SelOp, T2->getType(), C2VecTy,
+      SelOp, SrcVecTy, C1VecTy, CmpInst::BAD_ICMP_PREDICATE, CostKind);
+  OldCost += TTI.getCmpSelInstrCost(SelOp, SrcVecTy, C2VecTy,
                                     CmpInst::BAD_ICMP_PREDICATE, CostKind);
-  OldCost += TTI.getShuffleCost(SK, DstVecTy, Mask, CostKind, 0, nullptr,
+  OldCost += TTI.getShuffleCost(SK, SrcVecTy, Mask, CostKind, 0, nullptr,
                                 {I.getOperand(0), I.getOperand(1)}, &I);
 
-  auto *C1C2VecTy = cast<FixedVectorType>(
-      toVectorTy(Type::getInt1Ty(I.getContext()), DstVecTy->getNumElements()));
   InstructionCost NewCost =
-      TTI.getShuffleCost(SK, C1C2VecTy, Mask, CostKind, 0, nullptr, {C1, C2});
+      TTI.getShuffleCost(SK, C1VecTy, Mask, CostKind, 0, nullptr, {C1, C2});
   NewCost +=
-      TTI.getShuffleCost(SK, DstVecTy, Mask, CostKind, 0, nullptr, {T1, T2});
+      TTI.getShuffleCost(SK, SrcVecTy, Mask, CostKind, 0, nullptr, {T1, T2});
   NewCost +=
-      TTI.getShuffleCost(SK, DstVecTy, Mask, CostKind, 0, nullptr, {F1, F2});
-  NewCost += TTI.getCmpSelInstrCost(SelOp, DstVecTy, DstVecTy,
+      TTI.getShuffleCost(SK, SrcVecTy, Mask, CostKind, 0, nullptr, {F1, F2});
+  auto *C1C2ShuffledVecTy = cast<FixedVectorType>(
+      toVectorTy(Type::getInt1Ty(I.getContext()), DstVecTy->getNumElements()));
+  NewCost += TTI.getCmpSelInstrCost(SelOp, DstVecTy, C1C2ShuffledVecTy,
                                     CmpInst::BAD_ICMP_PREDICATE, CostKind);
 
   LLVM_DEBUG(dbgs() << "Found a shuffle feeding two selects: " << I
diff --git a/llvm/lib/WindowsDriver/MSVCPaths.cpp b/llvm/lib/WindowsDriver/MSVCPaths.cpp
index a7bffbb20eba1..1fc89749955df 100644
--- a/llvm/lib/WindowsDriver/MSVCPaths.cpp
+++ b/llvm/lib/WindowsDriver/MSVCPaths.cpp
@@ -444,6 +444,13 @@ bool getWindowsSDKDir(vfs::FileSystem &VFS, std::optional<StringRef> WinSdkDir,
     return !WindowsSDKLibVersion.empty();
   }
   if (Major == 10) {
+    if (WinSdkVersion) {
+      // Use the user-provided version as-is.
+      WindowsSDKIncludeVersion = WinSdkVersion->str();
+      WindowsSDKLibVersion = WindowsSDKIncludeVersion;
+      return true;
+    }
+
     if (!getWindows10SDKVersionFromPath(VFS, Path, WindowsSDKIncludeVersion))
       return false;
     WindowsSDKLibVersion = WindowsSDKIncludeVersion;
@@ -475,6 +482,12 @@ bool getUniversalCRTSdkDir(vfs::FileSystem &VFS,
           Path, nullptr))
     return false;
 
+  if (WinSdkVersion) {
+    // Use the user-provided version as-is.
+    UCRTVersion = WinSdkVersion->str();
+    return true;
+  }
+
   return getWindows10SDKVersionFromPath(VFS, Path, UCRTVersion);
 }
 
diff --git a/llvm/test/Analysis/CostModel/AArch64/abs.ll b/llvm/test/Analysis/CostModel/AArch64/abs.ll
index 6406cf1124d48..1d57cbb346bf2 100644
--- a/llvm/test/Analysis/CostModel/AArch64/abs.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/abs.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64 < %s | FileCheck %s --check-prefix=RECIP
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=aarch64 < %s | FileCheck %s --check-prefix=SIZE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
@@ -27,43 +26,24 @@ declare <32 x i8>  @llvm.abs.v32i8(<32 x i8>, i1)
 declare <64 x i8>  @llvm.abs.v64i8(<64 x i8>, i1)
 
 define i32 @abs(i32 %arg) {
-; RECIP-LABEL: 'abs'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> undef, i1 false)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> undef, i1 false)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> undef, i1 false)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <2 x i32> @llvm.abs.v2i32(<2 x i32> undef, i1 false)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> undef, i1 false)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> undef, i1 false)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> undef, i1 false)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = call <2 x i16> @llvm.abs.v2i16(<2 x i16> undef, i1 false)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = call <4 x i16> @llvm.abs.v4i16(<4 x i16> undef, i1 false)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> undef, i1 false)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> undef, i1 false)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> undef, i1 false)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I8 = call <8 x i8> @llvm.abs.v8i8(<8 x i8> undef, i1 false)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> undef, i1 false)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> undef, i1 false)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> undef, i1 false)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'abs'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> undef, i1 false)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> undef, i1 false)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> undef, i1 false)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <2 x i32> @llvm.abs.v2i32(<2 x i32> undef, i1 false)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> undef, i1 false)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> undef, i1 false)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> undef, i1 false)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I16 = call <2 x i16> @llvm.abs.v2i16(<2 x i16> undef, i1 false)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = call <4 x i16> @llvm.abs.v4i16(<4 x i16> undef, i1 false)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> undef, i1 false)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> undef, i1 false)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> undef, i1 false)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I8 = call <8 x i8> @llvm.abs.v8i8(<8 x i8> undef, i1 false)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> undef, i1 false)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> undef, i1 false)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> undef, i1 false)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-LABEL: 'abs'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.abs.v2i32(<2 x i32> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.abs.v4i32(<4 x i32> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.abs.v16i32(<16 x i32> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I16 = call <2 x i16> @llvm.abs.v2i16(<2 x i16> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.abs.v4i16(<4 x i16> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16I16 = call <16 x i16> @llvm.abs.v16i16(<16 x i16> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V32I16 = call <32 x i16> @llvm.abs.v32i16(<32 x i16> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.abs.v8i8(<8 x i8> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V32I8 = call <32 x i8> @llvm.abs.v32i8(<32 x i8> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V64I8 = call <64 x i8> @llvm.abs.v64i8(<64 x i8> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
 ;
   %V2I64 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> undef, i1 false)
   %V4I64 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> undef, i1 false)
diff --git a/llvm/test/Analysis/CostModel/AArch64/arith-fp.ll b/llvm/test/Analysis/CostModel/AArch64/arith-fp.ll
index 3e9d2259cf5f4..1efd41426c4ef 100644
--- a/llvm/test/Analysis/CostModel/AArch64/arith-fp.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/arith-fp.ll
@@ -1,43 +1,22 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -enable-no-nans-fp-math  -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output -mtriple=aarch64 -mattr=+fullfp16 | FileCheck %s --check-prefix=RECIP
-; RUN: opt < %s -enable-no-nans-fp-math  -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output -mtriple=aarch64 -mattr=+fullfp16 | FileCheck %s --check-prefix=SIZE
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-BASE
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=aarch64 -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-FP16
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
-define i32 @fadd(i32 %arg) {
-; RECIP-LABEL: 'fadd'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = fadd half undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fadd <4 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fadd <8 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = fadd <16 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fadd float undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fadd <2 x float> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fadd <4 x float> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fadd <8 x float> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fadd double undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fadd <2 x double> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fadd <4 x double> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'fadd'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = fadd half undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fadd <4 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fadd <8 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F16 = fadd <16 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fadd float undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fadd <2 x float> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fadd <4 x float> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fadd <8 x float> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fadd double undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fadd <2 x double> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fadd <4 x double> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+define void @fadd() {
+; CHECK-LABEL: 'fadd'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %F32 = fadd float undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F32 = fadd <2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V4F32 = fadd <4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F32 = fadd <8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %F64 = fadd double undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F64 = fadd <2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F64 = fadd <4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %F128 = fadd fp128 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:3 SizeLat:1 for: %V2F129 = fadd <2 x fp128> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
-  %F16 = fadd half undef, undef
-  %V4F16 = fadd <4 x half> undef, undef
-  %V8F16 = fadd <8 x half> undef, undef
-  %V16F16 = fadd <16 x half> undef, undef
-
   %F32 = fadd float undef, undef
   %V2F32 = fadd <2 x float> undef, undef
   %V4F32 = fadd <4 x float> undef, undef
@@ -47,43 +26,47 @@ define i32 @fadd(i32 %arg) {
   %V2F64 = fadd <2 x double> undef, undef
   %V4F64 = fadd <4 x double> undef, undef
 
-  ret i32 undef
+  %F128 = fadd fp128 undef, undef
+  %V2F129 = fadd <2 x fp128> undef, undef
+
+  ret void
 }
 
-define i32 @fsub(i32 %arg) {
-; RECIP-LABEL: 'fsub'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = fsub half undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fsub <4 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fsub <8 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = fsub <16 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fsub float undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fsub <2 x float> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fsub <4 x float> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fsub <8 x float> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fsub double undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fsub <2 x double> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fsub <4 x double> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'fsub'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = fsub half undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fsub <4 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fsub <8 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F16 = fsub <16 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fsub float undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fsub <2 x float> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fsub <4 x float> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fsub <8 x float> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fsub double undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fsub <2 x double> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fsub <4 x double> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+define void @fadd_fp16() {
+; CHECK-BASE-LABEL: 'fadd_fp16'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fadd half undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fadd <4 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fadd <8 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fadd <16 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
-  %F16 = fsub half undef, undef
-  %V4F16 = fsub <4 x half> undef, undef
-  %V8F16 = fsub <8 x half> undef, undef
-  %V16F16 = fsub <16 x half> undef, undef
+; CHECK-FP16-LABEL: 'fadd_fp16'
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fadd half undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fadd <4 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fadd <8 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fadd <16 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  %F16 = fadd half undef, undef
+  %V4F16 = fadd <4 x half> undef, undef
+  %V8F16 = fadd <8 x half> undef, undef
+  %V16F16 = fadd <16 x half> undef, undef
+  ret void
+}
 
+define void @fsub() {
+; CHECK-LABEL: 'fsub'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %F32 = fsub float undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F32 = fsub <2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V4F32 = fsub <4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F32 = fsub <8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %F64 = fsub double undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F64 = fsub <2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F64 = fsub <4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %F128 = fsub fp128 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:3 SizeLat:1 for: %V2F129 = fsub <2 x fp128> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
   %F32 = fsub float undef, undef
   %V2F32 = fsub <2 x float> undef, undef
   %V4F32 = fsub <4 x float> undef, undef
@@ -93,40 +76,47 @@ define i32 @fsub(i32 %arg) {
   %V2F64 = fsub <2 x double> undef, undef
   %V4F64 = fsub <4 x double> undef, undef
 
-  ret i32 undef
+  %F128 = fsub fp128 undef, undef
+  %V2F129 = fsub <2 x fp128> undef, undef
+
+  ret void
 }
 
-define i32 @fneg_idiom(i32 %arg) {
-; RECIP-LABEL: 'fneg_idiom'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = fsub half 0xH8000, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fsub <4 x half> splat (half 0xH8000), undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fsub <8 x half> splat (half 0xH8000), undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fsub float -0.000000e+00, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fsub <2 x float> splat (float -0.000000e+00), undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fsub <4 x float> splat (float -0.000000e+00), undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fsub <8 x float> splat (float -0.000000e+00), undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fsub double -0.000000e+00, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fsub <2 x double> splat (double -0.000000e+00), undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fsub <4 x double> splat (double -0.000000e+00), undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'fneg_idiom'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = fsub half 0xH8000, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fsub <4 x half> splat (half 0xH8000), undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fsub <8 x half> splat (half 0xH8000), undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fsub float -0.000000e+00, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fsub <2 x float> splat (float -0.000000e+00), undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fsub <4 x float> splat (float -0.000000e+00), undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fsub <8 x float> splat (float -0.000000e+00), undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fsub double -0.000000e+00, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fsub <2 x double> splat (double -0.000000e+00), undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fsub <4 x double> splat (double -0.000000e+00), undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+define void @fsub_fp16() {
+; CHECK-BASE-LABEL: 'fsub_fp16'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fsub half undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fsub <4 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fsub <8 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fsub <16 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
-  %F16 = fsub half -0.0, undef
-  %V4F16 = fsub <4 x half> <half -0.0, half -0.0, half -0.0, half -0.0>, undef
-  %V8F16 = fsub <8 x half> <half -0.0, half -0.0, half -0.0, half -0.0, half -0.0, half -0.0, half -0.0, half -0.0>, undef
+; CHECK-FP16-LABEL: 'fsub_fp16'
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fsub half undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fsub <4 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fsub <8 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fsub <16 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  %F16 = fsub half undef, undef
+  %V4F16 = fsub <4 x half> undef, undef
+  %V8F16 = fsub <8 x half> undef, undef
+  %V16F16 = fsub <16 x half> undef, undef
+  ret void
+}
 
+define void @fneg_idiom() {
+; CHECK-LABEL: 'fneg_idiom'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %F32 = fsub float -0.000000e+00, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F32 = fsub <2 x float> splat (float -0.000000e+00), undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V4F32 = fsub <4 x float> splat (float -0.000000e+00), undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F32 = fsub <8 x float> splat (float -0.000000e+00), undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %F64 = fsub double -0.000000e+00, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F64 = fsub <2 x double> splat (double -0.000000e+00), undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F64 = fsub <4 x double> splat (double -0.000000e+00), undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %F128 = fsub fp128 0xL00000000000000008000000000000000, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:3 SizeLat:1 for: %V2F129 = fsub <2 x fp128> splat (fp128 0xL00000000000000008000000000000000), undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
   %F32 = fsub float -0.0, undef
   %V2F32 = fsub <2 x float> <float -0.0, float -0.0>, undef
   %V4F32 = fsub <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, undef
@@ -136,46 +126,44 @@ define i32 @fneg_idiom(i32 %arg) {
   %V2F64 = fsub <2 x double> <double -0.0, double -0.0>, undef
   %V4F64 = fsub <4 x double> <double -0.0, double -0.0, double -0.0, double -0.0>, undef
 
-  ret i32 undef
+  %F128 = fsub fp128 0xL00000000000000008000000000000000, undef
+  %V2F129 = fsub <2 x fp128> <fp128 0xL00000000000000008000000000000000, fp128 0xL00000000000000008000000000000000>, undef
+
+  ret void
 }
 
-define i32 @fneg(i32 %arg) {
-; RECIP-LABEL: 'fneg'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = fneg half undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = fneg <2 x half> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fneg <4 x half> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fneg <8 x half> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = fneg <16 x half> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fneg float undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fneg <2 x float> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fneg <4 x float> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fneg <8 x float> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fneg double undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fneg <2 x double> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fneg <4 x double> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'fneg'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = fneg half undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = fneg <2 x half> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fneg <4 x half> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fneg <8 x half> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F16 = fneg <16 x half> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fneg float undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fneg <2 x float> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fneg <4 x float> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fneg <8 x float> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fneg double undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fneg <2 x double> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fneg <4 x double> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+define void @fneg_idiom_fp16() {
+; CHECK-BASE-LABEL: 'fneg_idiom_fp16'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fsub half 0xH8000, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fsub <4 x half> splat (half 0xH8000), undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fsub <8 x half> splat (half 0xH8000), undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
-  %F16 = fneg half undef
-  %V2F16 = fneg <2 x half> undef
-  %V4F16 = fneg <4 x half> undef
-  %V8F16 = fneg <8 x half> undef
-  %V16F16 = fneg <16 x half> undef
+; CHECK-FP16-LABEL: 'fneg_idiom_fp16'
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fsub half 0xH8000, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fsub <4 x half> splat (half 0xH8000), undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fsub <8 x half> splat (half 0xH8000), undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  %F16 = fsub half -0.0, undef
+  %V4F16 = fsub <4 x half> <half -0.0, half -0.0, half -0.0, half -0.0>, undef
+  %V8F16 = fsub <8 x half> <half -0.0, half -0.0, half -0.0, half -0.0, half -0.0, half -0.0, half -0.0, half -0.0>, undef
+  ret void
+}
 
+define void @fneg() {
+; CHECK-LABEL: 'fneg'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %F32 = fneg float undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F32 = fneg <2 x float> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V4F32 = fneg <4 x float> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F32 = fneg <8 x float> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %F64 = fneg double undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F64 = fneg <2 x double> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F64 = fneg <4 x double> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F128 = fneg fp128 undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V2F129 = fneg <2 x fp128> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
   %F32 = fneg float undef
   %V2F32 = fneg <2 x float> undef
   %V4F32 = fneg <4 x float> undef
@@ -185,75 +173,59 @@ define i32 @fneg(i32 %arg) {
   %V2F64 = fneg <2 x double> undef
   %V4F64 = fneg <4 x double> undef
 
-  ret i32 undef
+  %F128 = fneg fp128 undef
+  %V2F129 = fneg <2 x fp128> undef
+
+  ret void
 }
 
-define i32 @fmulfneg(i32 %arg) {
-; RECIP-LABEL: 'fmulfneg'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %F16 = fneg half undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16M = fmul half %F16, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = fneg <2 x half> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F16M = fmul <2 x half> %V2F16, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fneg <4 x half> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F16M = fmul <4 x half> %V4F16, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fneg <8 x half> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F16M = fmul <8 x half> %V8F16, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = fneg <16 x half> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16M = fmul <16 x half> %V16F16, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %F32 = fneg float undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32M = fmul float %F32, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fneg <2 x float> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32M = fmul <2 x float> %V2F32, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fneg <4 x float> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32M = fmul <4 x float> %V4F32, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fneg <8 x float> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32M = fmul <8 x float> %V8F32, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %F64 = fneg double undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64M = fmul double %F64, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fneg <2 x double> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64M = fmul <2 x double> %V2F64, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fneg <4 x double> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64M = fmul <4 x double> %V4F64, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'fmulfneg'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = fneg half undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16M = fmul half %F16, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = fneg <2 x half> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16M = fmul <2 x half> %V2F16, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fneg <4 x half> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16M = fmul <4 x half> %V4F16, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fneg <8 x half> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16M = fmul <8 x half> %V8F16, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F16 = fneg <16 x half> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F16M = fmul <16 x half> %V16F16, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fneg float undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32M = fmul float %F32, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fneg <2 x float> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32M = fmul <2 x float> %V2F32, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fneg <4 x float> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32M = fmul <4 x float> %V4F32, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fneg <8 x float> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32M = fmul <8 x float> %V8F32, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fneg double undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64M = fmul double %F64, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fneg <2 x double> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64M = fmul <2 x double> %V2F64, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fneg <4 x double> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64M = fmul <4 x double> %V4F64, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+define void @fneg_fp16() {
+; CHECK-BASE-LABEL: 'fneg_fp16'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fneg half undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16 = fneg <2 x half> undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fneg <4 x half> undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fneg <8 x half> undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fneg <16 x half> undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-FP16-LABEL: 'fneg_fp16'
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fneg half undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16 = fneg <2 x half> undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fneg <4 x half> undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fneg <8 x half> undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fneg <16 x half> undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %F16 = fneg half undef
-  %F16M = fmul half %F16, undef
   %V2F16 = fneg <2 x half> undef
-  %V2F16M = fmul <2 x half> %V2F16, undef
   %V4F16 = fneg <4 x half> undef
-  %V4F16M = fmul <4 x half> %V4F16, undef
   %V8F16 = fneg <8 x half> undef
-  %V8F16M = fmul <8 x half> %V8F16, undef
   %V16F16 = fneg <16 x half> undef
-  %V16F16M = fmul <16 x half> %V16F16, undef
+  ret void
+}
 
+define void @fmulfneg() {
+; CHECK-LABEL: 'fmulfneg'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:3 SizeLat:1 for: %F32 = fneg float undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F32M = fmul float %F32, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F32 = fneg <2 x float> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F32M = fmul <2 x float> %V2F32, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V4F32 = fneg <4 x float> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F32M = fmul <4 x float> %V4F32, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F32 = fneg <8 x float> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V8F32M = fmul <8 x float> %V8F32, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:3 SizeLat:1 for: %F64 = fneg double undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F64M = fmul double %F64, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F64 = fneg <2 x double> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F64M = fmul <2 x double> %V2F64, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F64 = fneg <4 x double> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V4F64M = fmul <4 x double> %V4F64, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F128 = fneg fp128 undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %F128M = fmul fp128 %F128, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V2F128 = fneg <2 x fp128> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:3 SizeLat:1 for: %V2F128M = fmul <2 x fp128> %V2F128, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
   %F32 = fneg float undef
   %F32M = fmul float %F32, undef
   %V2F32 = fneg <2 x float> undef
@@ -270,75 +242,76 @@ define i32 @fmulfneg(i32 %arg) {
   %V4F64 = fneg <4 x double> undef
   %V4F64M = fmul <4 x double> %V4F64, undef
 
-  ret i32 undef
+  %F128 = fneg fp128 undef
+  %F128M = fmul fp128 %F128, undef
+  %V2F128 = fneg <2 x fp128> undef
+  %V2F128M = fmul <2 x fp128> %V2F128, undef
+
+  ret void
 }
 
-define i32 @fnegfmul(i32 %arg) {
-; RECIP-LABEL: 'fnegfmul'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16M = fmul half undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %F16 = fneg half %F16M
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F16M = fmul <2 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = fneg <2 x half> %V2F16M
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F16M = fmul <4 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fneg <4 x half> %V4F16M
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F16M = fmul <8 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fneg <8 x half> %V8F16M
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16M = fmul <16 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = fneg <16 x half> %V16F16M
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32M = fmul float undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %F32 = fneg float %F32M
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32M = fmul <2 x float> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fneg <2 x float> %V2F32M
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32M = fmul <4 x float> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fneg <4 x float> %V4F32M
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32M = fmul <8 x float> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fneg <8 x float> %V8F32M
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64M = fmul double undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %F64 = fneg double %F64M
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64M = fmul <2 x double> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fneg <2 x double> %V2F64M
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64M = fmul <4 x double> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fneg <4 x double> %V4F64M
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'fnegfmul'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16M = fmul half undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = fneg half %F16M
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16M = fmul <2 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = fneg <2 x half> %V2F16M
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16M = fmul <4 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fneg <4 x half> %V4F16M
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16M = fmul <8 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fneg <8 x half> %V8F16M
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F16M = fmul <16 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F16 = fneg <16 x half> %V16F16M
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32M = fmul float undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fneg float %F32M
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32M = fmul <2 x float> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fneg <2 x float> %V2F32M
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32M = fmul <4 x float> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fneg <4 x float> %V4F32M
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32M = fmul <8 x float> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fneg <8 x float> %V8F32M
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64M = fmul double undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fneg double %F64M
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64M = fmul <2 x double> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fneg <2 x double> %V2F64M
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64M = fmul <4 x double> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fneg <4 x double> %V4F64M
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+define void @fmulneg_fp16() {
+; CHECK-BASE-LABEL: 'fmulneg_fp16'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fneg half undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16M = fmul half %F16, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16 = fneg <2 x half> undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16M = fmul <2 x half> %V2F16, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fneg <4 x half> undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16M = fmul <4 x half> %V4F16, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fneg <8 x half> undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16M = fmul <8 x half> %V8F16, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fneg <16 x half> undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16M = fmul <16 x half> %V16F16, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
-  %F16M = fmul half undef, undef
-  %F16 = fneg half %F16M
-  %V2F16M = fmul <2 x half> undef, undef
-  %V2F16 = fneg <2 x half> %V2F16M
-  %V4F16M = fmul <4 x half> undef, undef
-  %V4F16 = fneg <4 x half> %V4F16M
-  %V8F16M = fmul <8 x half> undef, undef
-  %V8F16 = fneg <8 x half> %V8F16M
-  %V16F16M = fmul <16 x half> undef, undef
-  %V16F16 = fneg <16 x half> %V16F16M
+; CHECK-FP16-LABEL: 'fmulneg_fp16'
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fneg half undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16M = fmul half %F16, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16 = fneg <2 x half> undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16M = fmul <2 x half> %V2F16, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fneg <4 x half> undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16M = fmul <4 x half> %V4F16, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fneg <8 x half> undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16M = fmul <8 x half> %V8F16, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fneg <16 x half> undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16M = fmul <16 x half> %V16F16, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  %F16 = fneg half undef
+  %F16M = fmul half %F16, undef
+  %V2F16 = fneg <2 x half> undef
+  %V2F16M = fmul <2 x half> %V2F16, undef
+  %V4F16 = fneg <4 x half> undef
+  %V4F16M = fmul <4 x half> %V4F16, undef
+  %V8F16 = fneg <8 x half> undef
+  %V8F16M = fmul <8 x half> %V8F16, undef
+  %V16F16 = fneg <16 x half> undef
+  %V16F16M = fmul <16 x half> %V16F16, undef
+  ret void
+}
 
+define void @fnegfmul() {
+; CHECK-LABEL: 'fnegfmul'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F32M = fmul float undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:3 SizeLat:1 for: %F32 = fneg float %F32M
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F32M = fmul <2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F32 = fneg <2 x float> %V2F32M
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F32M = fmul <4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V4F32 = fneg <4 x float> %V4F32M
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V8F32M = fmul <8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F32 = fneg <8 x float> %V8F32M
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F64M = fmul double undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:3 SizeLat:1 for: %F64 = fneg double %F64M
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F64M = fmul <2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F64 = fneg <2 x double> %V2F64M
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V4F64M = fmul <4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F64 = fneg <4 x double> %V4F64M
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %F128M = fmul fp128 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F128 = fneg fp128 %F128M
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:3 SizeLat:1 for: %V2F128M = fmul <2 x fp128> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V2F128 = fneg <2 x fp128> %V2F128M
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
   %F32M = fmul float undef, undef
   %F32 = fneg float %F32M
   %V2F32M = fmul <2 x float> undef, undef
@@ -355,43 +328,67 @@ define i32 @fnegfmul(i32 %arg) {
   %V4F64M = fmul <4 x double> undef, undef
   %V4F64 = fneg <4 x double> %V4F64M
 
-  ret i32 undef
+  %F128M = fmul fp128 undef, undef
+  %F128 = fneg fp128 %F128M
+  %V2F128M = fmul <2 x fp128> undef, undef
+  %V2F128 = fneg <2 x fp128> %V2F128M
+
+  ret void
 }
 
-define i32 @fmul(i32 %arg) {
-; RECIP-LABEL: 'fmul'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16 = fmul half undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F16 = fmul <4 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F16 = fmul <8 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16 = fmul <16 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = fmul float undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fmul <2 x float> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fmul <4 x float> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fmul <8 x float> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = fmul double undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fmul <2 x double> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fmul <4 x double> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'fmul'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = fmul half undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fmul <4 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fmul <8 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F16 = fmul <16 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fmul float undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fmul <2 x float> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fmul <4 x float> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fmul <8 x float> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fmul double undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fmul <2 x double> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fmul <4 x double> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+define void @fnegfmul_fp16() {
+; CHECK-BASE-LABEL: 'fnegfmul_fp16'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16M = fmul half undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fneg half %F16M
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16M = fmul <2 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16 = fneg <2 x half> %V2F16M
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16M = fmul <4 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fneg <4 x half> %V4F16M
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16M = fmul <8 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fneg <8 x half> %V8F16M
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16M = fmul <16 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fneg <16 x half> %V16F16M
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
-  %F16 = fmul half undef, undef
-  %V4F16 = fmul <4 x half> undef, undef
-  %V8F16 = fmul <8 x half> undef, undef
-  %V16F16 = fmul <16 x half> undef, undef
+; CHECK-FP16-LABEL: 'fnegfmul_fp16'
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16M = fmul half undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fneg half %F16M
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16M = fmul <2 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16 = fneg <2 x half> %V2F16M
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16M = fmul <4 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fneg <4 x half> %V4F16M
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16M = fmul <8 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fneg <8 x half> %V8F16M
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16M = fmul <16 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fneg <16 x half> %V16F16M
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  %F16M = fmul half undef, undef
+  %F16 = fneg half %F16M
+  %V2F16M = fmul <2 x half> undef, undef
+  %V2F16 = fneg <2 x half> %V2F16M
+  %V4F16M = fmul <4 x half> undef, undef
+  %V4F16 = fneg <4 x half> %V4F16M
+  %V8F16M = fmul <8 x half> undef, undef
+  %V8F16 = fneg <8 x half> %V8F16M
+  %V16F16M = fmul <16 x half> undef, undef
+  %V16F16 = fneg <16 x half> %V16F16M
+  ret void
+}
 
+define void @fmul() {
+; CHECK-LABEL: 'fmul'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F32 = fmul float undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F32 = fmul <2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F32 = fmul <4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V8F32 = fmul <8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F64 = fmul double undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F64 = fmul <2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V4F64 = fmul <4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %F128 = fmul fp128 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:3 SizeLat:1 for: %V2F128 = fmul <2 x fp128> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
   %F32 = fmul float undef, undef
   %V2F32 = fmul <2 x float> undef, undef
   %V4F32 = fmul <4 x float> undef, undef
@@ -401,43 +398,40 @@ define i32 @fmul(i32 %arg) {
   %V2F64 = fmul <2 x double> undef, undef
   %V4F64 = fmul <4 x double> undef, undef
 
-  ret i32 undef
+  %F128 = fmul fp128 undef, undef
+  %V2F128 = fmul <2 x fp128> undef, undef
+
+  ret void
 }
 
-define i32 @fdiv(i32 %arg) {
-; RECIP-LABEL: 'fdiv'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16 = fdiv half undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F16 = fdiv <4 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F16 = fdiv <8 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16 = fdiv <16 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = fdiv float undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fdiv <2 x float> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fdiv <4 x float> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fdiv <8 x float> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = fdiv double undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fdiv <2 x double> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fdiv <4 x double> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'fdiv'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %F16 = fdiv half undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F16 = fdiv <4 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F16 = fdiv <8 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16 = fdiv <16 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %F32 = fdiv float undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = fdiv <2 x float> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F32 = fdiv <4 x float> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fdiv <8 x float> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %F64 = fdiv double undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = fdiv <2 x double> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fdiv <4 x double> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+define void @fmul_fp16() {
+; CHECK-LABEL: 'fmul_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fmul half undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fmul <4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fmul <8 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fmul <16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
-  %F16 = fdiv half undef, undef
-  %V4F16 = fdiv <4 x half> undef, undef
-  %V8F16 = fdiv <8 x half> undef, undef
-  %V16F16 = fdiv <16 x half> undef, undef
+  %F16 = fmul half undef, undef
+  %V4F16 = fmul <4 x half> undef, undef
+  %V8F16 = fmul <8 x half> undef, undef
+  %V16F16 = fmul <16 x half> undef, undef
+  ret void
+}
 
+define void @fdiv() {
+; CHECK-LABEL: 'fdiv'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %F32 = fdiv float undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %V2F32 = fdiv <2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %V4F32 = fdiv <4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8F32 = fdiv <8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %F64 = fdiv double undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %V2F64 = fdiv <2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4F64 = fdiv <4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %F128 = fdiv fp128 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V2F128 = fdiv <2 x fp128> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
   %F32 = fdiv float undef, undef
   %V2F32 = fdiv <2 x float> undef, undef
   %V4F32 = fdiv <4 x float> undef, undef
@@ -447,43 +441,40 @@ define i32 @fdiv(i32 %arg) {
   %V2F64 = fdiv <2 x double> undef, undef
   %V4F64 = fdiv <4 x double> undef, undef
 
-  ret i32 undef
+  %F128 = fdiv fp128 undef, undef
+  %V2F128 = fdiv <2 x fp128> undef, undef
+
+  ret void
 }
 
-define i32 @frem(i32 %arg) {
-; RECIP-LABEL: 'frem'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F16 = frem half undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V4F16 = frem <4 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V8F16 = frem <8 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 216 for instruction: %V16F16 = frem <16 x half> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F32 = frem float undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2F32 = frem <2 x float> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V4F32 = frem <4 x float> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V8F32 = frem <8 x float> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F64 = frem double undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V2F64 = frem <2 x double> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V4F64 = frem <4 x double> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'frem'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %F16 = frem half undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F16 = frem <4 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F16 = frem <8 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16 = frem <16 x half> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %F32 = frem float undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F32 = frem <2 x float> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F32 = frem <4 x float> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = frem <8 x float> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %F64 = frem double undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = frem <2 x double> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = frem <4 x double> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+define void @fdiv_fp16() {
+; CHECK-LABEL: 'fdiv_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %F16 = fdiv half undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %V4F16 = fdiv <4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %V8F16 = fdiv <8 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V16F16 = fdiv <16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
-  %F16 = frem half undef, undef
-  %V4F16 = frem <4 x half> undef, undef
-  %V8F16 = frem <8 x half> undef, undef
-  %V16F16 = frem <16 x half> undef, undef
+  %F16 = fdiv half undef, undef
+  %V4F16 = fdiv <4 x half> undef, undef
+  %V8F16 = fdiv <8 x half> undef, undef
+  %V16F16 = fdiv <16 x half> undef, undef
+  ret void
+}
 
+define void @frem() {
+; CHECK-LABEL: 'frem'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %F32 = frem float undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %V2F32 = frem <2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:52 CodeSize:4 Lat:4 SizeLat:4 for: %V4F32 = frem <4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:104 CodeSize:4 Lat:4 SizeLat:4 for: %V8F32 = frem <8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %F64 = frem double undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %V2F64 = frem <2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:4 Lat:4 SizeLat:4 for: %V4F64 = frem <4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %F128 = frem fp128 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V2F128 = frem <2 x fp128> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
   %F32 = frem float undef, undef
   %V2F32 = frem <2 x float> undef, undef
   %V4F32 = frem <4 x float> undef, undef
@@ -493,43 +484,40 @@ define i32 @frem(i32 %arg) {
   %V2F64 = frem <2 x double> undef, undef
   %V4F64 = frem <4 x double> undef, undef
 
-  ret i32 undef
+  %F128 = frem fp128 undef, undef
+  %V2F128 = frem <2 x fp128> undef, undef
+
+  ret void
 }
 
-define i32 @fsqrt(i32 %arg) {
-; RECIP-LABEL: 'fsqrt'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = call half @llvm.sqrt.f16(half undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = call <4 x half> @llvm.sqrt.v4f16(<4 x half> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = call <8 x half> @llvm.sqrt.v8f16(<8 x half> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16 = call <16 x half> @llvm.sqrt.v16f16(<16 x half> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = call float @llvm.sqrt.f32(float undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = call <2 x float> @llvm.sqrt.v2f32(<2 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call <4 x float> @llvm.sqrt.v4f32(<4 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = call <8 x float> @llvm.sqrt.v8f32(<8 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = call double @llvm.sqrt.f64(double undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = call <2 x double> @llvm.sqrt.v2f64(<2 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = call <4 x double> @llvm.sqrt.v4f64(<4 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'fsqrt'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = call half @llvm.sqrt.f16(half undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = call <4 x half> @llvm.sqrt.v4f16(<4 x half> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = call <8 x half> @llvm.sqrt.v8f16(<8 x half> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16 = call <16 x half> @llvm.sqrt.v16f16(<16 x half> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = call float @llvm.sqrt.f32(float undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = call <2 x float> @llvm.sqrt.v2f32(<2 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call <4 x float> @llvm.sqrt.v4f32(<4 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = call <8 x float> @llvm.sqrt.v8f32(<8 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = call double @llvm.sqrt.f64(double undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = call <2 x double> @llvm.sqrt.v2f64(<2 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = call <4 x double> @llvm.sqrt.v4f64(<4 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+define void @frem_fp16() {
+; CHECK-LABEL: 'frem_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %F16 = frem half undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:52 CodeSize:4 Lat:4 SizeLat:4 for: %V4F16 = frem <4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:108 CodeSize:4 Lat:4 SizeLat:4 for: %V8F16 = frem <8 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:216 CodeSize:4 Lat:4 SizeLat:4 for: %V16F16 = frem <16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
-  %F16 = call half @llvm.sqrt.f16(half undef)
-  %V4F16 = call <4 x half> @llvm.sqrt.v4f16(<4 x half> undef)
-  %V8F16 = call <8 x half> @llvm.sqrt.v8f16(<8 x half> undef)
-  %V16F16 = call <16 x half> @llvm.sqrt.v16f16(<16 x half> undef)
+  %F16 = frem half undef, undef
+  %V4F16 = frem <4 x half> undef, undef
+  %V8F16 = frem <8 x half> undef, undef
+  %V16F16 = frem <16 x half> undef, undef
+  ret void
+}
 
+define void @fsqrt() {
+; CHECK-LABEL: 'fsqrt'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %F32 = call float @llvm.sqrt.f32(float undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2F32 = call <2 x float> @llvm.sqrt.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4F32 = call <4 x float> @llvm.sqrt.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8F32 = call <8 x float> @llvm.sqrt.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %F64 = call double @llvm.sqrt.f64(double undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2F64 = call <2 x double> @llvm.sqrt.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4F64 = call <4 x double> @llvm.sqrt.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:10 SizeLat:10 for: %F128 = call fp128 @llvm.sqrt.f128(fp128 undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:2 Lat:20 SizeLat:20 for: %V2F128 = call <2 x fp128> @llvm.sqrt.v2f128(<2 x fp128> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
   %F32 = call float @llvm.sqrt.f32(float undef)
   %V2F32 = call <2 x float> @llvm.sqrt.v2f32(<2 x float> undef)
   %V4F32 = call <4 x float> @llvm.sqrt.v4f32(<4 x float> undef)
@@ -539,43 +527,47 @@ define i32 @fsqrt(i32 %arg) {
   %V2F64 = call <2 x double> @llvm.sqrt.v2f64(<2 x double> undef)
   %V4F64 = call <4 x double> @llvm.sqrt.v4f64(<4 x double> undef)
 
-  ret i32 undef
+  %F128 = call fp128 @llvm.sqrt.fp128(fp128 undef)
+  %V2F128 = call <2 x fp128> @llvm.sqrt.v2fp128(<2 x fp128> undef)
+
+  ret void
 }
 
-define i32 @fabs(i32 %arg) {
-; RECIP-LABEL: 'fabs'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = call half @llvm.fabs.f16(half undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = call <4 x half> @llvm.fabs.v4f16(<4 x half> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = call <8 x half> @llvm.fabs.v8f16(<8 x half> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F16 = call <16 x half> @llvm.fabs.v16f16(<16 x half> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = call float @llvm.fabs.f32(float undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = call <2 x float> @llvm.fabs.v2f32(<2 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call <4 x float> @llvm.fabs.v4f32(<4 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = call <8 x float> @llvm.fabs.v8f32(<8 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = call double @llvm.fabs.f64(double undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = call <2 x double> @llvm.fabs.v2f64(<2 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = call <4 x double> @llvm.fabs.v4f64(<4 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'fabs'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = call half @llvm.fabs.f16(half undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = call <4 x half> @llvm.fabs.v4f16(<4 x half> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = call <8 x half> @llvm.fabs.v8f16(<8 x half> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F16 = call <16 x half> @llvm.fabs.v16f16(<16 x half> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = call float @llvm.fabs.f32(float undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = call <2 x float> @llvm.fabs.v2f32(<2 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call <4 x float> @llvm.fabs.v4f32(<4 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = call <8 x float> @llvm.fabs.v8f32(<8 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = call double @llvm.fabs.f64(double undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = call <2 x double> @llvm.fabs.v2f64(<2 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = call <4 x double> @llvm.fabs.v4f64(<4 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+define void @fsqrt_fp16() {
+; CHECK-BASE-LABEL: 'fsqrt_fp16'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 1 for: %F16 = call half @llvm.sqrt.f16(half undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 10 for: %V4F16 = call <4 x half> @llvm.sqrt.v4f16(<4 x half> undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 22 for: %V8F16 = call <8 x half> @llvm.sqrt.v8f16(<8 x half> undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 44 for: %V16F16 = call <16 x half> @llvm.sqrt.v16f16(<16 x half> undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
-  %F16 = call half @llvm.fabs.f16(half undef)
-  %V4F16 = call <4 x half> @llvm.fabs.v4f16(<4 x half> undef)
-  %V8F16 = call <8 x half> @llvm.fabs.v8f16(<8 x half> undef)
-  %V2F16 = call <16 x half> @llvm.fabs.v16f16(<16 x half> undef)
+; CHECK-FP16-LABEL: 'fsqrt_fp16'
+; CHECK-FP16-NEXT:  Cost Model: Found costs of 1 for: %F16 = call half @llvm.sqrt.f16(half undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of 1 for: %V4F16 = call <4 x half> @llvm.sqrt.v4f16(<4 x half> undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of 1 for: %V8F16 = call <8 x half> @llvm.sqrt.v8f16(<8 x half> undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of 4 for: %V16F16 = call <16 x half> @llvm.sqrt.v16f16(<16 x half> undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  %F16 = call half @llvm.sqrt.f16(half undef)
+  %V4F16 = call <4 x half> @llvm.sqrt.v4f16(<4 x half> undef)
+  %V8F16 = call <8 x half> @llvm.sqrt.v8f16(<8 x half> undef)
+  %V16F16 = call <16 x half> @llvm.sqrt.v16f16(<16 x half> undef)
+  ret void
+}
 
+define void @fabs() {
+; CHECK-LABEL: 'fabs'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %F32 = call float @llvm.fabs.f32(float undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2F32 = call <2 x float> @llvm.fabs.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4F32 = call <4 x float> @llvm.fabs.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8F32 = call <8 x float> @llvm.fabs.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %F64 = call double @llvm.fabs.f64(double undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2F64 = call <2 x double> @llvm.fabs.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4F64 = call <4 x double> @llvm.fabs.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:10 SizeLat:10 for: %F128 = call fp128 @llvm.fabs.f128(fp128 undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:2 Lat:20 SizeLat:20 for: %V2F128 = call <2 x fp128> @llvm.fabs.v2f128(<2 x fp128> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
   %F32 = call float @llvm.fabs.f32(float undef)
   %V2F32 = call <2 x float> @llvm.fabs.v2f32(<2 x float> undef)
   %V4F32 = call <4 x float> @llvm.fabs.v4f32(<4 x float> undef)
@@ -585,43 +577,40 @@ define i32 @fabs(i32 %arg) {
   %V2F64 = call <2 x double> @llvm.fabs.v2f64(<2 x double> undef)
   %V4F64 = call <4 x double> @llvm.fabs.v4f64(<4 x double> undef)
 
-  ret i32 undef
+  %F128 = call fp128 @llvm.fabs.fp128(fp128 undef)
+  %V2F128 = call <2 x fp128> @llvm.fabs.v2fp128(<2 x fp128> undef)
+
+  ret void
 }
 
-define i32 @fcopysign(i32 %arg) {
-; RECIP-LABEL: 'fcopysign'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16 = call half @llvm.copysign.f16(half undef, half undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F16 = call <4 x half> @llvm.copysign.v4f16(<4 x half> undef, <4 x half> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F16 = call <8 x half> @llvm.copysign.v8f16(<8 x half> undef, <8 x half> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16 = call <16 x half> @llvm.copysign.v16f16(<16 x half> undef, <16 x half> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = call float @llvm.copysign.f32(float undef, float undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = call <2 x float> @llvm.copysign.v2f32(<2 x float> undef, <2 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = call <4 x float> @llvm.copysign.v4f32(<4 x float> undef, <4 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = call <8 x float> @llvm.copysign.v8f32(<8 x float> undef, <8 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = call double @llvm.copysign.f64(double undef, double undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = call <2 x double> @llvm.copysign.v2f64(<2 x double> undef, <2 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = call <4 x double> @llvm.copysign.v4f64(<4 x double> undef, <4 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'fcopysign'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16 = call half @llvm.copysign.f16(half undef, half undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F16 = call <4 x half> @llvm.copysign.v4f16(<4 x half> undef, <4 x half> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F16 = call <8 x half> @llvm.copysign.v8f16(<8 x half> undef, <8 x half> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16 = call <16 x half> @llvm.copysign.v16f16(<16 x half> undef, <16 x half> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = call float @llvm.copysign.f32(float undef, float undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = call <2 x float> @llvm.copysign.v2f32(<2 x float> undef, <2 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = call <4 x float> @llvm.copysign.v4f32(<4 x float> undef, <4 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = call <8 x float> @llvm.copysign.v8f32(<8 x float> undef, <8 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = call double @llvm.copysign.f64(double undef, double undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = call <2 x double> @llvm.copysign.v2f64(<2 x double> undef, <2 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = call <4 x double> @llvm.copysign.v4f64(<4 x double> undef, <4 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+define void @fabs_fp16() {
+; CHECK-LABEL: 'fabs_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %F16 = call half @llvm.fabs.f16(half undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4F16 = call <4 x half> @llvm.fabs.v4f16(<4 x half> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8F16 = call <8 x half> @llvm.fabs.v8f16(<8 x half> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2F16 = call <16 x half> @llvm.fabs.v16f16(<16 x half> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
-  %F16 = call half @llvm.copysign.f16(half undef, half undef)
-  %V4F16 = call <4 x half> @llvm.copysign.v4f16(<4 x half> undef, <4 x half> undef)
-  %V8F16 = call <8 x half> @llvm.copysign.v8f16(<8 x half> undef, <8 x half> undef)
-  %V16F16 = call <16 x half> @llvm.copysign.v16f16(<16 x half> undef, <16 x half> undef)
+  %F16 = call half @llvm.fabs.f16(half undef)
+  %V4F16 = call <4 x half> @llvm.fabs.v4f16(<4 x half> undef)
+  %V8F16 = call <8 x half> @llvm.fabs.v8f16(<8 x half> undef)
+  %V2F16 = call <16 x half> @llvm.fabs.v16f16(<16 x half> undef)
+  ret void
+}
 
+define void @fcopysign() {
+; CHECK-LABEL: 'fcopysign'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %F32 = call float @llvm.copysign.f32(float undef, float undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V2F32 = call <2 x float> @llvm.copysign.v2f32(<2 x float> undef, <2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V4F32 = call <4 x float> @llvm.copysign.v4f32(<4 x float> undef, <4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8F32 = call <8 x float> @llvm.copysign.v8f32(<8 x float> undef, <8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %F64 = call double @llvm.copysign.f64(double undef, double undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V2F64 = call <2 x double> @llvm.copysign.v2f64(<2 x double> undef, <2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4F64 = call <4 x double> @llvm.copysign.v4f64(<4 x double> undef, <4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:10 SizeLat:10 for: %F128 = call fp128 @llvm.copysign.f128(fp128 undef, fp128 undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:2 Lat:20 SizeLat:20 for: %V2F128 = call <2 x fp128> @llvm.copysign.v2f128(<2 x fp128> undef, <2 x fp128> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
   %F32 = call float @llvm.copysign.f32(float undef, float undef)
   %V2F32 = call <2 x float> @llvm.copysign.v2f32(<2 x float> undef, <2 x float> undef)
   %V4F32 = call <4 x float> @llvm.copysign.v4f32(<4 x float> undef, <4 x float> undef)
@@ -631,43 +620,47 @@ define i32 @fcopysign(i32 %arg) {
   %V2F64 = call <2 x double> @llvm.copysign.v2f64(<2 x double> undef, <2 x double> undef)
   %V4F64 = call <4 x double> @llvm.copysign.v4f64(<4 x double> undef, <4 x double> undef)
 
-  ret i32 undef
+  %F128 = call fp128 @llvm.copysign.fp128(fp128 undef, fp128 undef)
+  %V2F128 = call <2 x fp128> @llvm.copysign.v2fp128(<2 x fp128> undef, <2 x fp128> undef)
+
+  ret void
 }
 
-define i32 @fma(i32 %arg) {
-; RECIP-LABEL: 'fma'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = call float @llvm.fma.f32(float undef, float undef, float undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = call <4 x half> @llvm.fma.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = call <8 x half> @llvm.fma.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16 = call <16 x half> @llvm.fma.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = call float @llvm.fma.f32(float undef, float undef, float undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = call <2 x float> @llvm.fma.v2f32(<2 x float> undef, <2 x float> undef, <2 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call <4 x float> @llvm.fma.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = call <8 x float> @llvm.fma.v8f32(<8 x float> undef, <8 x float> undef, <8 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = call double @llvm.fma.f64(double undef, double undef, double undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = call <2 x double> @llvm.fma.v2f64(<2 x double> undef, <2 x double> undef, <2 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = call <4 x double> @llvm.fma.v4f64(<4 x double> undef, <4 x double> undef, <4 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'fma'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = call float @llvm.fma.f32(float undef, float undef, float undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = call <4 x half> @llvm.fma.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = call <8 x half> @llvm.fma.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16 = call <16 x half> @llvm.fma.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = call float @llvm.fma.f32(float undef, float undef, float undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = call <2 x float> @llvm.fma.v2f32(<2 x float> undef, <2 x float> undef, <2 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call <4 x float> @llvm.fma.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = call <8 x float> @llvm.fma.v8f32(<8 x float> undef, <8 x float> undef, <8 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = call double @llvm.fma.f64(double undef, double undef, double undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = call <2 x double> @llvm.fma.v2f64(<2 x double> undef, <2 x double> undef, <2 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = call <4 x double> @llvm.fma.v4f64(<4 x double> undef, <4 x double> undef, <4 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
-;
-  %F16 = call float @llvm.fma.f32(float undef, float undef, float undef)
-  %V4F16 = call <4 x half> @llvm.fma.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
-  %V8F16 = call <8 x half> @llvm.fma.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
-  %V16F16 = call <16 x half> @llvm.fma.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
+define void @fcopysign_fp16() {
+; CHECK-BASE-LABEL: 'fcopysign_fp16'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 1 for: %F16 = call half @llvm.copysign.f16(half undef, half undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 2 for: %V4F16 = call <4 x half> @llvm.copysign.v4f16(<4 x half> undef, <4 x half> undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 2 for: %V8F16 = call <8 x half> @llvm.copysign.v8f16(<8 x half> undef, <8 x half> undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 4 for: %V16F16 = call <16 x half> @llvm.copysign.v16f16(<16 x half> undef, <16 x half> undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-FP16-LABEL: 'fcopysign_fp16'
+; CHECK-FP16-NEXT:  Cost Model: Found costs of 2 for: %F16 = call half @llvm.copysign.f16(half undef, half undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of 2 for: %V4F16 = call <4 x half> @llvm.copysign.v4f16(<4 x half> undef, <4 x half> undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of 2 for: %V8F16 = call <8 x half> @llvm.copysign.v8f16(<8 x half> undef, <8 x half> undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of 4 for: %V16F16 = call <16 x half> @llvm.copysign.v16f16(<16 x half> undef, <16 x half> undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  %F16 = call half @llvm.copysign.f16(half undef, half undef)
+  %V4F16 = call <4 x half> @llvm.copysign.v4f16(<4 x half> undef, <4 x half> undef)
+  %V8F16 = call <8 x half> @llvm.copysign.v8f16(<8 x half> undef, <8 x half> undef)
+  %V16F16 = call <16 x half> @llvm.copysign.v16f16(<16 x half> undef, <16 x half> undef)
+  ret void
+}
 
+define void @fma() {
+; CHECK-LABEL: 'fma'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %F32 = call float @llvm.fma.f32(float undef, float undef, float undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2F32 = call <2 x float> @llvm.fma.v2f32(<2 x float> undef, <2 x float> undef, <2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4F32 = call <4 x float> @llvm.fma.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8F32 = call <8 x float> @llvm.fma.v8f32(<8 x float> undef, <8 x float> undef, <8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %F64 = call double @llvm.fma.f64(double undef, double undef, double undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2F64 = call <2 x double> @llvm.fma.v2f64(<2 x double> undef, <2 x double> undef, <2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4F64 = call <4 x double> @llvm.fma.v4f64(<4 x double> undef, <4 x double> undef, <4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:10 SizeLat:10 for: %F128 = call fp128 @llvm.fma.f128(fp128 undef, fp128 undef, fp128 undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:2 Lat:20 SizeLat:20 for: %V2F128 = call <2 x fp128> @llvm.fma.v2f128(<2 x fp128> undef, <2 x fp128> undef, <2 x fp128> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
   %F32 = call float @llvm.fma.f32(float undef, float undef, float undef)
   %V2F32 = call <2 x float> @llvm.fma.v2f32(<2 x float> undef, <2 x float> undef, <2 x float> undef)
   %V4F32 = call <4 x float> @llvm.fma.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> undef)
@@ -677,61 +670,80 @@ define i32 @fma(i32 %arg) {
   %V2F64 = call <2 x double> @llvm.fma.v2f64(<2 x double> undef, <2 x double> undef, <2 x double> undef)
   %V4F64 = call <4 x double> @llvm.fma.v4f64(<4 x double> undef, <4 x double> undef, <4 x double> undef)
 
-  ret i32 undef
+  %F128 = call fp128 @llvm.fma.fp128(fp128 undef, fp128 undef, fp128 undef)
+  %V2F128 = call <2 x fp128> @llvm.fma.v2fp128(<2 x fp128> undef, <2 x fp128> undef, <2 x fp128> undef)
+
+  ret void
+}
+
+define void @fma_fp16() {
+; CHECK-BASE-LABEL: 'fma_fp16'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 1 for: %F16 = call half @llvm.fma.f16(half undef, half undef, half undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 10 for: %V4F16 = call <4 x half> @llvm.fma.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 22 for: %V8F16 = call <8 x half> @llvm.fma.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 44 for: %V16F16 = call <16 x half> @llvm.fma.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-FP16-LABEL: 'fma_fp16'
+; CHECK-FP16-NEXT:  Cost Model: Found costs of 1 for: %F16 = call half @llvm.fma.f16(half undef, half undef, half undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of 1 for: %V4F16 = call <4 x half> @llvm.fma.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of 1 for: %V8F16 = call <8 x half> @llvm.fma.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of 4 for: %V16F16 = call <16 x half> @llvm.fma.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  %F16 = call half @llvm.fma.f16(half undef, half undef, half undef)
+  %V4F16 = call <4 x half> @llvm.fma.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
+  %V8F16 = call <8 x half> @llvm.fma.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
+  %V16F16 = call <16 x half> @llvm.fma.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
+  ret void
+}
+
+define void @fmuladd() {
+; CHECK-LABEL: 'fmuladd'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %F32 = call float @llvm.fmuladd.f32(float undef, float undef, float undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2F32 = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> undef, <2 x float> undef, <2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4F32 = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8F32 = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> undef, <8 x float> undef, <8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %F64 = call double @llvm.fmuladd.f64(double undef, double undef, double undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2F64 = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> undef, <2 x double> undef, <2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4F64 = call <4 x double> @llvm.fmuladd.v4f64(<4 x double> undef, <4 x double> undef, <4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:2 Lat:6 SizeLat:2 for: %F128 = call fp128 @llvm.fmuladd.f128(fp128 undef, fp128 undef, fp128 undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:2 Lat:6 SizeLat:2 for: %V2F128 = call <2 x fp128> @llvm.fmuladd.v2f128(<2 x fp128> undef, <2 x fp128> undef, <2 x fp128> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  %F32 = call float @llvm.fmuladd.f32(float undef, float undef, float undef)
+  %V2F32 = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> undef, <2 x float> undef, <2 x float> undef)
+  %V4F32 = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> undef)
+  %V8F32 = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> undef, <8 x float> undef, <8 x float> undef)
+
+  %F64 = call double @llvm.fmuladd.f64(double undef, double undef, double undef)
+  %V2F64 = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> undef, <2 x double> undef, <2 x double> undef)
+  %V4F64 = call <4 x double> @llvm.fmuladd.v4f64(<4 x double> undef, <4 x double> undef, <4 x double> undef)
+
+  %F128 = call fp128 @llvm.fmuladd.f128(fp128 undef, fp128 undef, fp128 undef)
+  %V2F128 = call <2 x fp128> @llvm.fmuladd.v2f128(<2 x fp128> undef, <2 x fp128> undef, <2 x fp128> undef)
+
+  ret void
 }
 
-declare half @llvm.sqrt.f16(half)
-declare <4 x half> @llvm.sqrt.v4f16(<4 x half>)
-declare <8 x half> @llvm.sqrt.v8f16(<8 x half>)
-declare <16 x half> @llvm.sqrt.v16f16(<16 x half>)
-
-declare float @llvm.sqrt.f32(float)
-declare <2 x float> @llvm.sqrt.v2f32(<2 x float>)
-declare <4 x float> @llvm.sqrt.v4f32(<4 x float>)
-declare <8 x float> @llvm.sqrt.v8f32(<8 x float>)
-
-declare double @llvm.sqrt.f64(double)
-declare <2 x double> @llvm.sqrt.v2f64(<2 x double>)
-declare <4 x double> @llvm.sqrt.v4f64(<4 x double>)
-
-declare half @llvm.fabs.f16(half)
-declare <4 x half> @llvm.fabs.v4f16(<4 x half>)
-declare <8 x half> @llvm.fabs.v8f16(<8 x half>)
-declare <16 x half> @llvm.fabs.v16f16(<16 x half>)
-
-declare float @llvm.fabs.f32(float)
-declare <2 x float> @llvm.fabs.v2f32(<2 x float>)
-declare <4 x float> @llvm.fabs.v4f32(<4 x float>)
-declare <8 x float> @llvm.fabs.v8f32(<8 x float>)
-
-declare double @llvm.fabs.f64(double)
-declare <2 x double> @llvm.fabs.v2f64(<2 x double>)
-declare <4 x double> @llvm.fabs.v4f64(<4 x double>)
-
-declare half @llvm.copysign.f16(half, half)
-declare <4 x half> @llvm.copysign.v4f16(<4 x half>, <4 x half>)
-declare <8 x half> @llvm.copysign.v8f16(<8 x half>, <8 x half>)
-declare <16 x half> @llvm.copysign.v16f16(<16 x half>, <16 x half>)
-
-declare float @llvm.copysign.f32(float, float)
-declare <2 x float> @llvm.copysign.v2f32(<2 x float>, <2 x float>)
-declare <4 x float> @llvm.copysign.v4f32(<4 x float>, <4 x float>)
-declare <8 x float> @llvm.copysign.v8f32(<8 x float>, <8 x float>)
-
-declare double @llvm.copysign.f64(double, double)
-declare <2 x double> @llvm.copysign.v2f64(<2 x double>, <2 x double>)
-declare <4 x double> @llvm.copysign.v4f64(<4 x double>, <4 x double>)
-
-declare half @llvm.fma.f16(half, half, half)
-declare <4 x half> @llvm.fma.v4f16(<4 x half>, <4 x half>, <4 x half>)
-declare <8 x half> @llvm.fma.v8f16(<8 x half>, <8 x half>, <8 x half>)
-declare <16 x half> @llvm.fma.v16f16(<16 x half>, <16 x half>, <16 x half>)
-
-declare float @llvm.fma.f32(float, float, float)
-declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>)
-declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>)
-declare <8 x float> @llvm.fma.v8f32(<8 x float>, <8 x float>, <8 x float>)
-
-declare double @llvm.fma.f64(double, double, double)
-declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>)
-declare <4 x double> @llvm.fma.v4f64(<4 x double>, <4 x double>, <4 x double>)
+define void @fmuladd_fp16() {
+; CHECK-BASE-LABEL: 'fmuladd_fp16'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 1 for: %F16 = call half @llvm.fmuladd.f16(half undef, half undef, half undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:2 Lat:6 SizeLat:2 for: %V4F16 = call <4 x half> @llvm.fmuladd.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:2 Lat:6 SizeLat:2 for: %V8F16 = call <8 x half> @llvm.fmuladd.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:2 Lat:6 SizeLat:2 for: %V16F16 = call <16 x half> @llvm.fmuladd.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-FP16-LABEL: 'fmuladd_fp16'
+; CHECK-FP16-NEXT:  Cost Model: Found costs of 1 for: %F16 = call half @llvm.fmuladd.f16(half undef, half undef, half undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of 1 for: %V4F16 = call <4 x half> @llvm.fmuladd.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of 1 for: %V8F16 = call <8 x half> @llvm.fmuladd.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of 4 for: %V16F16 = call <16 x half> @llvm.fmuladd.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  %F16 = call half @llvm.fmuladd.f32(half undef, half undef, half undef)
+  %V4F16 = call <4 x half> @llvm.fmuladd.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
+  %V8F16 = call <8 x half> @llvm.fmuladd.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
+  %V16F16 = call <16 x half> @llvm.fmuladd.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/AArch64/arith-overflow.ll b/llvm/test/Analysis/CostModel/AArch64/arith-overflow.ll
index 91379b1f9f874..8b52b123d6d25 100644
--- a/llvm/test/Analysis/CostModel/AArch64/arith-overflow.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/arith-overflow.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64 < %s | FileCheck %s --check-prefix=RECIP
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=aarch64 < %s | FileCheck %s --check-prefix=SIZE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
@@ -25,43 +24,24 @@ declare {<32 x i8>, <32 x i1>}  @llvm.sadd.with.overflow.v32i8(<32 x i8>, <32 x
 declare {<64 x i8>, <64 x i1>}  @llvm.sadd.with.overflow.v64i8(<64 x i8>, <64 x i8>)
 
 define i32 @sadd(i32 %arg) {
-; RECIP-LABEL: 'sadd'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.sadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 undef, i16 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.sadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.sadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.sadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 undef, i8 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.sadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.sadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.sadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'sadd'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.sadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 undef, i16 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.sadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.sadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.sadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 undef, i8 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.sadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.sadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.sadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-LABEL: 'sadd'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:13 CodeSize:4 Lat:4 SizeLat:4 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:13 CodeSize:4 Lat:4 SizeLat:4 for: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.sadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %I16 = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 undef, i16 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.sadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.sadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.sadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %I8 = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 undef, i8 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.sadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.sadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.sadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
 ;
   %I64 = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
   %V2I64 = call {<2 x i64>, <2 x i1>} @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
@@ -107,43 +87,24 @@ declare {<32 x i8>, <32 x i1>}  @llvm.uadd.with.overflow.v32i8(<32 x i8>, <32 x
 declare {<64 x i8>, <64 x i1>}  @llvm.uadd.with.overflow.v64i8(<64 x i8>, <64 x i8>)
 
 define i32 @uadd(i32 %arg) {
-; RECIP-LABEL: 'uadd'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.uadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 undef, i16 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.uadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.uadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.uadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 undef, i8 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.uadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.uadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.uadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'uadd'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.uadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 undef, i16 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.uadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.uadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.uadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 undef, i8 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.uadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.uadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.uadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-LABEL: 'uadd'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:2 Lat:2 SizeLat:2 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:2 Lat:2 SizeLat:2 for: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.uadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %I16 = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 undef, i16 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.uadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.uadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:2 Lat:2 SizeLat:2 for: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.uadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %I8 = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 undef, i8 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.uadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.uadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:2 Lat:2 SizeLat:2 for: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.uadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
 ;
   %I64 = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
   %V2I64 = call {<2 x i64>, <2 x i1>} @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
@@ -189,43 +150,24 @@ declare {<32 x i8>, <32 x i1>}  @llvm.ssub.with.overflow.v32i8(<32 x i8>, <32 x
 declare {<64 x i8>, <64 x i1>}  @llvm.ssub.with.overflow.v64i8(<64 x i8>, <64 x i8>)
 
 define i32 @ssub(i32 %arg) {
-; RECIP-LABEL: 'ssub'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.ssub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 undef, i16 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.ssub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.ssub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.ssub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 undef, i8 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.ssub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.ssub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.ssub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'ssub'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.ssub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 undef, i16 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.ssub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.ssub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.ssub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 undef, i8 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.ssub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.ssub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.ssub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-LABEL: 'ssub'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:13 CodeSize:4 Lat:4 SizeLat:4 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:13 CodeSize:4 Lat:4 SizeLat:4 for: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.ssub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %I16 = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 undef, i16 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.ssub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.ssub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.ssub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %I8 = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 undef, i8 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.ssub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.ssub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.ssub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
 ;
   %I64 = call {i64, i1} @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
   %V2I64 = call {<2 x i64>, <2 x i1>} @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
@@ -271,43 +213,24 @@ declare {<32 x i8>, <32 x i1>}  @llvm.usub.with.overflow.v32i8(<32 x i8>, <32 x
 declare {<64 x i8>, <64 x i1>}  @llvm.usub.with.overflow.v64i8(<64 x i8>, <64 x i8>)
 
 define i32 @usub(i32 %arg) {
-; RECIP-LABEL: 'usub'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.usub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 undef, i16 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.usub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.usub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.usub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 undef, i8 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.usub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.usub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.usub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'usub'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.usub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 undef, i16 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.usub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.usub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.usub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 undef, i8 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.usub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.usub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.usub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-LABEL: 'usub'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:2 Lat:2 SizeLat:2 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:2 Lat:2 SizeLat:2 for: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.usub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %I16 = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 undef, i16 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.usub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.usub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:2 Lat:2 SizeLat:2 for: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.usub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %I8 = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 undef, i8 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.usub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:2 Lat:2 SizeLat:2 for: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.usub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:2 Lat:2 SizeLat:2 for: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.usub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
 ;
   %I64 = call {i64, i1} @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
   %V2I64 = call {<2 x i64>, <2 x i1>} @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
@@ -353,43 +276,24 @@ declare {<32 x i8>, <32 x i1>}  @llvm.smul.with.overflow.v32i8(<32 x i8>, <32 x
 declare {<64 x i8>, <64 x i1>}  @llvm.smul.with.overflow.v64i8(<64 x i8>, <64 x i8>)
 
 define i32 @smul(i32 %arg) {
-; RECIP-LABEL: 'smul'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 undef, i64 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 undef, i32 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 152 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I16 = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 undef, i16 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.smul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.smul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.smul.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I8 = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 undef, i8 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.smul.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.smul.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.smul.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'smul'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 undef, i64 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 undef, i32 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I16 = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 undef, i16 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.smul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.smul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.smul.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I8 = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 undef, i8 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.smul.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.smul.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.smul.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-LABEL: 'smul'
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %I64 = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 undef, i64 undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:34 CodeSize:8 Lat:8 SizeLat:8 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:68 CodeSize:8 Lat:8 SizeLat:8 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:136 CodeSize:8 Lat:8 SizeLat:8 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %I32 = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:38 CodeSize:8 Lat:8 SizeLat:8 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:76 CodeSize:8 Lat:8 SizeLat:8 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:152 CodeSize:8 Lat:8 SizeLat:8 for: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 5 for: %I16 = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 undef, i16 undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:8 SizeLat:8 for: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.smul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:8 Lat:8 SizeLat:8 for: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.smul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:8 Lat:8 SizeLat:8 for: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.smul.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 5 for: %I8 = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 undef, i8 undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:8 SizeLat:8 for: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.smul.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:8 Lat:8 SizeLat:8 for: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.smul.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:8 Lat:8 SizeLat:8 for: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.smul.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
 ;
   %I64 = call {i64, i1} @llvm.smul.with.overflow.i64(i64 undef, i64 undef)
   %V2I64 = call {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
@@ -435,43 +339,24 @@ declare {<32 x i8>, <32 x i1>}  @llvm.umul.with.overflow.v32i8(<32 x i8>, <32 x
 declare {<64 x i8>, <64 x i1>}  @llvm.umul.with.overflow.v64i8(<64 x i8>, <64 x i8>)
 
 define i32 @umul(i32 %arg) {
-; RECIP-LABEL: 'umul'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 undef, i64 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 undef, i32 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 148 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 undef, i16 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.umul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.umul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.umul.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 undef, i8 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.umul.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.umul.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.umul.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'umul'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 undef, i64 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 undef, i32 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 undef, i16 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.umul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.umul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.umul.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 undef, i8 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.umul.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.umul.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.umul.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-LABEL: 'umul'
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %I64 = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 undef, i64 undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:33 CodeSize:7 Lat:7 SizeLat:7 for: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:66 CodeSize:7 Lat:7 SizeLat:7 for: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:132 CodeSize:7 Lat:7 SizeLat:7 for: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %I32 = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:37 CodeSize:7 Lat:7 SizeLat:7 for: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:74 CodeSize:7 Lat:7 SizeLat:7 for: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:148 CodeSize:7 Lat:7 SizeLat:7 for: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I16 = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 undef, i16 undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:11 CodeSize:7 Lat:7 SizeLat:7 for: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.umul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:7 Lat:7 SizeLat:7 for: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.umul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:44 CodeSize:7 Lat:7 SizeLat:7 for: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.umul.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I8 = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 undef, i8 undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:11 CodeSize:7 Lat:7 SizeLat:7 for: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.umul.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:7 Lat:7 SizeLat:7 for: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.umul.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:44 CodeSize:7 Lat:7 SizeLat:7 for: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.umul.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
 ;
   %I64 = call {i64, i1} @llvm.umul.with.overflow.i64(i64 undef, i64 undef)
   %V2I64 = call {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
diff --git a/llvm/test/Analysis/CostModel/AArch64/arith-ssat.ll b/llvm/test/Analysis/CostModel/AArch64/arith-ssat.ll
index 2267d9b88c970..254715ac909e1 100644
--- a/llvm/test/Analysis/CostModel/AArch64/arith-ssat.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/arith-ssat.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64 < %s | FileCheck %s --check-prefix=RECIP
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=aarch64 < %s | FileCheck %s --check-prefix=SIZE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
@@ -32,57 +31,31 @@ declare <32 x i8>  @llvm.sadd.sat.v32i8(<32 x i8>, <32 x i8>)
 declare <64 x i8>  @llvm.sadd.sat.v64i8(<64 x i8>, <64 x i8>)
 
 define i32 @add(i32 %arg) {
-; RECIP-LABEL: 'add'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V3I32 = call <3 x i32> @llvm.sadd.sat.v3i32(<3 x i32> undef, <3 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I8 = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I8 = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'add'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V3I32 = call <3 x i32> @llvm.sadd.sat.v3i32(<3 x i32> undef, <3 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I8 = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I8 = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-LABEL: 'add'
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V3I32 = call <3 x i32> @llvm.sadd.sat.v3i32(<3 x i32> undef, <3 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 6 for: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 6 for: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
 ;
   %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
   %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
@@ -141,55 +114,30 @@ declare <32 x i8>  @llvm.ssub.sat.v32i8(<32 x i8>, <32 x i8>)
 declare <64 x i8>  @llvm.ssub.sat.v64i8(<64 x i8>, <64 x i8>)
 
 define i32 @sub(i32 %arg) {
-; RECIP-LABEL: 'sub'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I8 = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I8 = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'sub'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I8 = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I8 = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-LABEL: 'sub'
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 6 for: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 6 for: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
 ;
   %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
   %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
diff --git a/llvm/test/Analysis/CostModel/AArch64/arith-usat.ll b/llvm/test/Analysis/CostModel/AArch64/arith-usat.ll
index 5a131f23847b1..dba42b1e6ebb0 100644
--- a/llvm/test/Analysis/CostModel/AArch64/arith-usat.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/arith-usat.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64 < %s | FileCheck %s --check-prefix=RECIP
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=aarch64 < %s | FileCheck %s --check-prefix=SIZE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
@@ -31,55 +30,30 @@ declare <32 x i8>  @llvm.uadd.sat.v32i8(<32 x i8>, <32 x i8>)
 declare <64 x i8>  @llvm.uadd.sat.v64i8(<64 x i8>, <64 x i8>)
 
 define i32 @add(i32 %arg) {
-; RECIP-LABEL: 'add'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I16 = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I8 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I8 = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I8 = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'add'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I16 = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I8 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I8 = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I8 = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-LABEL: 'add'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
 ;
   %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
   %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
@@ -137,55 +111,30 @@ declare <32 x i8>  @llvm.usub.sat.v32i8(<32 x i8>, <32 x i8>)
 declare <64 x i8>  @llvm.usub.sat.v64i8(<64 x i8>, <64 x i8>)
 
 define i32 @sub(i32 %arg) {
-; RECIP-LABEL: 'sub'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I16 = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I8 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I8 = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I8 = call <8 x i8> @llvm.usub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SIZE-LABEL: 'sub'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I16 = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I16 = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I8 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I8 = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I8 = call <8 x i8> @llvm.usub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-LABEL: 'sub'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.usub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
 ;
   %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
   %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
diff --git a/llvm/test/Analysis/CostModel/AArch64/arith-widening.ll b/llvm/test/Analysis/CostModel/AArch64/arith-widening.ll
index afb75f288d0c1..7e1588f427be4 100644
--- a/llvm/test/Analysis/CostModel/AArch64/arith-widening.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/arith-widening.ll
@@ -1,135 +1,71 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s --check-prefix=RECIP
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output | FileCheck %s --check-prefix=SIZE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define void @extaddv1(i8 %i8, i16 %i16, i32 %i32, i64 %i64)  {
-; RECIP-LABEL: 'extaddv1'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext i8 %i8 to i16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = add i16 %i16, %sw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext i8 %i8 to i16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_16 = sext i8 %i8 to i16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = add i16 %sl1_8_16, %sl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext i8 %i8 to i16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = add i16 %i16, %zw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_16 = zext i8 %i8 to i16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_16 = zext i8 %i8 to i16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = add i16 %zl1_8_16, %zl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext i8 %i8 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = add i32 %i32, %sw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext i8 %i8 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext i8 %i8 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = add i32 %sl1_8_32, %sl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext i8 %i8 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = add i32 %i32, %zw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext i8 %i8 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext i8 %i8 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = add i32 %zl1_8_32, %zl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext i8 %i8 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = add i64 %i64, %sw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext i8 %i8 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext i8 %i8 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = add i64 %sl1_8_64, %sl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_8_64 = zext i8 %i8 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = add i64 %i64, %zw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_64 = zext i8 %i8 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_8_64 = zext i8 %i8 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = add i64 %zl1_8_64, %zl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_32 = sext i16 %i16 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = add i32 %i32, %sw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_32 = sext i16 %i16 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_32 = sext i16 %i16 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = add i32 %sl1_16_32, %sl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_32 = zext i16 %i16 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = add i32 %i32, %zw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_32 = zext i16 %i16 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_32 = zext i16 %i16 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = add i32 %zl1_16_32, %zl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext i16 %i16 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = add i64 %i64, %sw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext i16 %i16 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext i16 %i16 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = add i64 %sl1_16_64, %sl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_16_64 = zext i16 %i16 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = add i64 %i64, %zw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_64 = zext i16 %i16 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_64 = zext i16 %i16 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = add i64 %zl1_16_64, %zl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_32_64 = sext i32 %i32 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = add i64 %i64, %sw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_32_64 = sext i32 %i32 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_32_64 = sext i32 %i32 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = add i64 %sl1_32_64, %sl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext i32 %i32 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = add i64 %i64, %zw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext i32 %i32 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext i32 %i32 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = add i64 %zl1_32_64, %zl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'extaddv1'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext i8 %i8 to i16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = add i16 %i16, %sw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext i8 %i8 to i16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_16 = sext i8 %i8 to i16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = add i16 %sl1_8_16, %sl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext i8 %i8 to i16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = add i16 %i16, %zw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_16 = zext i8 %i8 to i16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_16 = zext i8 %i8 to i16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = add i16 %zl1_8_16, %zl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext i8 %i8 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = add i32 %i32, %sw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext i8 %i8 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext i8 %i8 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = add i32 %sl1_8_32, %sl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext i8 %i8 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = add i32 %i32, %zw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext i8 %i8 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext i8 %i8 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = add i32 %zl1_8_32, %zl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext i8 %i8 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = add i64 %i64, %sw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext i8 %i8 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext i8 %i8 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = add i64 %sl1_8_64, %sl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_8_64 = zext i8 %i8 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = add i64 %i64, %zw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_64 = zext i8 %i8 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_8_64 = zext i8 %i8 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = add i64 %zl1_8_64, %zl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_32 = sext i16 %i16 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = add i32 %i32, %sw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_32 = sext i16 %i16 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_32 = sext i16 %i16 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = add i32 %sl1_16_32, %sl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_32 = zext i16 %i16 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = add i32 %i32, %zw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_32 = zext i16 %i16 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_32 = zext i16 %i16 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = add i32 %zl1_16_32, %zl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext i16 %i16 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = add i64 %i64, %sw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext i16 %i16 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext i16 %i16 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = add i64 %sl1_16_64, %sl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_16_64 = zext i16 %i16 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = add i64 %i64, %zw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_64 = zext i16 %i16 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_64 = zext i16 %i16 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = add i64 %zl1_16_64, %zl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_32_64 = sext i32 %i32 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = add i64 %i64, %sw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_32_64 = sext i32 %i32 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_32_64 = sext i32 %i32 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = add i64 %sl1_32_64, %sl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext i32 %i32 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = add i64 %i64, %zw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext i32 %i32 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext i32 %i32 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = add i64 %zl1_32_64, %zl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'extaddv1'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_16 = sext i8 %i8 to i16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_16 = add i16 %i16, %sw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_16 = sext i8 %i8 to i16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_16 = sext i8 %i8 to i16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_16 = add i16 %sl1_8_16, %sl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_16 = zext i8 %i8 to i16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_16 = add i16 %i16, %zw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_16 = zext i8 %i8 to i16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_16 = zext i8 %i8 to i16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_16 = add i16 %zl1_8_16, %zl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_32 = sext i8 %i8 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_32 = add i32 %i32, %sw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_32 = sext i8 %i8 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_32 = sext i8 %i8 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_32 = add i32 %sl1_8_32, %sl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_32 = zext i8 %i8 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_32 = add i32 %i32, %zw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_32 = zext i8 %i8 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_32 = zext i8 %i8 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_32 = add i32 %zl1_8_32, %zl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_64 = sext i8 %i8 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_64 = add i64 %i64, %sw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_64 = sext i8 %i8 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_64 = sext i8 %i8 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_64 = add i64 %sl1_8_64, %sl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_8_64 = zext i8 %i8 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_64 = add i64 %i64, %zw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_8_64 = zext i8 %i8 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_8_64 = zext i8 %i8 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_64 = add i64 %zl1_8_64, %zl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_16_32 = sext i16 %i16 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_16_32 = add i32 %i32, %sw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_16_32 = sext i16 %i16 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_16_32 = sext i16 %i16 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_16_32 = add i32 %sl1_16_32, %sl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_16_32 = zext i16 %i16 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_16_32 = add i32 %i32, %zw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_16_32 = zext i16 %i16 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_16_32 = zext i16 %i16 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_16_32 = add i32 %zl1_16_32, %zl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_16_64 = sext i16 %i16 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_16_64 = add i64 %i64, %sw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_16_64 = sext i16 %i16 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_16_64 = sext i16 %i16 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_16_64 = add i64 %sl1_16_64, %sl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_16_64 = zext i16 %i16 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_16_64 = add i64 %i64, %zw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_16_64 = zext i16 %i16 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_16_64 = zext i16 %i16 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_16_64 = add i64 %zl1_16_64, %zl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_32_64 = sext i32 %i32 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_32_64 = add i64 %i64, %sw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_32_64 = sext i32 %i32 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_32_64 = sext i32 %i32 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_32_64 = add i64 %sl1_32_64, %sl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_32_64 = zext i32 %i32 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_32_64 = add i64 %i64, %zw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_32_64 = zext i32 %i32 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_32_64 = zext i32 %i32 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_32_64 = add i64 %zl1_32_64, %zl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %sw_8_16 = sext i8 %i8 to i16
   %asw_8_16 = add i16 %i16, %sw_8_16
@@ -207,131 +143,68 @@ define void @extaddv1(i8 %i8, i16 %i16, i32 %i32, i64 %i64)  {
 }
 
 define void @extaddv2(<2 x i8> %i8, <2 x i16> %i16, <2 x i32> %i32, <2 x i64> %i64)  {
-; RECIP-LABEL: 'extaddv2'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext <2 x i8> %i8 to <2 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = add <2 x i16> %i16, %sw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext <2 x i8> %i8 to <2 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_16 = sext <2 x i8> %i8 to <2 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = add <2 x i16> %sl1_8_16, %sl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext <2 x i8> %i8 to <2 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = add <2 x i16> %i16, %zw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_16 = zext <2 x i8> %i8 to <2 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_16 = zext <2 x i8> %i8 to <2 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = add <2 x i16> %zl1_8_16, %zl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext <2 x i8> %i8 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = add <2 x i32> %i32, %sw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext <2 x i8> %i8 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext <2 x i8> %i8 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = add <2 x i32> %sl1_8_32, %sl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext <2 x i8> %i8 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = add <2 x i32> %i32, %zw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext <2 x i8> %i8 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext <2 x i8> %i8 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = add <2 x i32> %zl1_8_32, %zl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext <2 x i8> %i8 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = add <2 x i64> %i64, %sw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext <2 x i8> %i8 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext <2 x i8> %i8 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = add <2 x i64> %sl1_8_64, %sl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_64 = zext <2 x i8> %i8 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = add <2 x i64> %i64, %zw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_64 = zext <2 x i8> %i8 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_64 = zext <2 x i8> %i8 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = add <2 x i64> %zl1_8_64, %zl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_32 = sext <2 x i16> %i16 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = add <2 x i32> %i32, %sw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_32 = sext <2 x i16> %i16 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_32 = sext <2 x i16> %i16 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = add <2 x i32> %sl1_16_32, %sl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_32 = zext <2 x i16> %i16 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = add <2 x i32> %i32, %zw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_32 = zext <2 x i16> %i16 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_32 = zext <2 x i16> %i16 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = add <2 x i32> %zl1_16_32, %zl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext <2 x i16> %i16 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = add <2 x i64> %i64, %sw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext <2 x i16> %i16 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext <2 x i16> %i16 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = add <2 x i64> %sl1_16_64, %sl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_64 = zext <2 x i16> %i16 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = add <2 x i64> %i64, %zw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_64 = zext <2 x i16> %i16 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_64 = zext <2 x i16> %i16 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = add <2 x i64> %zl1_16_64, %zl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_32_64 = sext <2 x i32> %i32 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = add <2 x i64> %i64, %sw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <2 x i32> %i32 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <2 x i32> %i32 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = add <2 x i64> %sl1_32_64, %sl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext <2 x i32> %i32 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = add <2 x i64> %i64, %zw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <2 x i32> %i32 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <2 x i32> %i32 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = add <2 x i64> %zl1_32_64, %zl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'extaddv2'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext <2 x i8> %i8 to <2 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = add <2 x i16> %i16, %sw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext <2 x i8> %i8 to <2 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_16 = sext <2 x i8> %i8 to <2 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = add <2 x i16> %sl1_8_16, %sl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext <2 x i8> %i8 to <2 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = add <2 x i16> %i16, %zw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_16 = zext <2 x i8> %i8 to <2 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_16 = zext <2 x i8> %i8 to <2 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = add <2 x i16> %zl1_8_16, %zl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext <2 x i8> %i8 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = add <2 x i32> %i32, %sw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext <2 x i8> %i8 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext <2 x i8> %i8 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = add <2 x i32> %sl1_8_32, %sl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext <2 x i8> %i8 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = add <2 x i32> %i32, %zw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext <2 x i8> %i8 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext <2 x i8> %i8 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = add <2 x i32> %zl1_8_32, %zl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext <2 x i8> %i8 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = add <2 x i64> %i64, %sw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext <2 x i8> %i8 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext <2 x i8> %i8 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = add <2 x i64> %sl1_8_64, %sl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_64 = zext <2 x i8> %i8 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = add <2 x i64> %i64, %zw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_64 = zext <2 x i8> %i8 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_64 = zext <2 x i8> %i8 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = add <2 x i64> %zl1_8_64, %zl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_32 = sext <2 x i16> %i16 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = add <2 x i32> %i32, %sw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_32 = sext <2 x i16> %i16 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_32 = sext <2 x i16> %i16 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = add <2 x i32> %sl1_16_32, %sl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_32 = zext <2 x i16> %i16 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = add <2 x i32> %i32, %zw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_32 = zext <2 x i16> %i16 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_32 = zext <2 x i16> %i16 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = add <2 x i32> %zl1_16_32, %zl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext <2 x i16> %i16 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = add <2 x i64> %i64, %sw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext <2 x i16> %i16 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext <2 x i16> %i16 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = add <2 x i64> %sl1_16_64, %sl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_64 = zext <2 x i16> %i16 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = add <2 x i64> %i64, %zw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_64 = zext <2 x i16> %i16 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_64 = zext <2 x i16> %i16 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = add <2 x i64> %zl1_16_64, %zl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_32_64 = sext <2 x i32> %i32 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = add <2 x i64> %i64, %sw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <2 x i32> %i32 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <2 x i32> %i32 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = add <2 x i64> %sl1_32_64, %sl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext <2 x i32> %i32 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = add <2 x i64> %i64, %zw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <2 x i32> %i32 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <2 x i32> %i32 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = add <2 x i64> %zl1_32_64, %zl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'extaddv2'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_16 = sext <2 x i8> %i8 to <2 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_16 = add <2 x i16> %i16, %sw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_16 = sext <2 x i8> %i8 to <2 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_16 = sext <2 x i8> %i8 to <2 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_16 = add <2 x i16> %sl1_8_16, %sl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_16 = zext <2 x i8> %i8 to <2 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_16 = add <2 x i16> %i16, %zw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_16 = zext <2 x i8> %i8 to <2 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_16 = zext <2 x i8> %i8 to <2 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_16 = add <2 x i16> %zl1_8_16, %zl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_32 = sext <2 x i8> %i8 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_32 = add <2 x i32> %i32, %sw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_32 = sext <2 x i8> %i8 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_32 = sext <2 x i8> %i8 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_32 = add <2 x i32> %sl1_8_32, %sl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_32 = zext <2 x i8> %i8 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_32 = add <2 x i32> %i32, %zw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_32 = zext <2 x i8> %i8 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_32 = zext <2 x i8> %i8 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_32 = add <2 x i32> %zl1_8_32, %zl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_64 = sext <2 x i8> %i8 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_64 = add <2 x i64> %i64, %sw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_64 = sext <2 x i8> %i8 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_64 = sext <2 x i8> %i8 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_64 = add <2 x i64> %sl1_8_64, %sl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_64 = zext <2 x i8> %i8 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_64 = add <2 x i64> %i64, %zw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_64 = zext <2 x i8> %i8 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_64 = zext <2 x i8> %i8 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_64 = add <2 x i64> %zl1_8_64, %zl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_16_32 = sext <2 x i16> %i16 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_16_32 = add <2 x i32> %i32, %sw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_16_32 = sext <2 x i16> %i16 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_16_32 = sext <2 x i16> %i16 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_16_32 = add <2 x i32> %sl1_16_32, %sl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_16_32 = zext <2 x i16> %i16 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_16_32 = add <2 x i32> %i32, %zw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_16_32 = zext <2 x i16> %i16 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_16_32 = zext <2 x i16> %i16 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_16_32 = add <2 x i32> %zl1_16_32, %zl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_16_64 = sext <2 x i16> %i16 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_16_64 = add <2 x i64> %i64, %sw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_16_64 = sext <2 x i16> %i16 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_16_64 = sext <2 x i16> %i16 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_16_64 = add <2 x i64> %sl1_16_64, %sl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_16_64 = zext <2 x i16> %i16 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_16_64 = add <2 x i64> %i64, %zw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_16_64 = zext <2 x i16> %i16 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_16_64 = zext <2 x i16> %i16 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_16_64 = add <2 x i64> %zl1_16_64, %zl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sw_32_64 = sext <2 x i32> %i32 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_32_64 = add <2 x i64> %i64, %sw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_32_64 = sext <2 x i32> %i32 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_32_64 = sext <2 x i32> %i32 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_32_64 = add <2 x i64> %sl1_32_64, %sl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_32_64 = zext <2 x i32> %i32 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_32_64 = add <2 x i64> %i64, %zw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_32_64 = zext <2 x i32> %i32 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_32_64 = zext <2 x i32> %i32 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_32_64 = add <2 x i64> %zl1_32_64, %zl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %sw_8_16 = sext <2 x i8> %i8 to <2 x i16>
   %asw_8_16 = add <2 x i16> %i16, %sw_8_16
@@ -409,131 +282,68 @@ define void @extaddv2(<2 x i8> %i8, <2 x i16> %i16, <2 x i32> %i32, <2 x i64> %i
 }
 
 define void @extaddv4(<4 x i8> %i8, <4 x i16> %i16, <4 x i32> %i32, <4 x i64> %i64)  {
-; RECIP-LABEL: 'extaddv4'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext <4 x i8> %i8 to <4 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = add <4 x i16> %i16, %sw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext <4 x i8> %i8 to <4 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_16 = sext <4 x i8> %i8 to <4 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = add <4 x i16> %sl1_8_16, %sl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext <4 x i8> %i8 to <4 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = add <4 x i16> %i16, %zw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_16 = zext <4 x i8> %i8 to <4 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_16 = zext <4 x i8> %i8 to <4 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = add <4 x i16> %zl1_8_16, %zl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext <4 x i8> %i8 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = add <4 x i32> %i32, %sw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext <4 x i8> %i8 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext <4 x i8> %i8 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = add <4 x i32> %sl1_8_32, %sl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext <4 x i8> %i8 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = add <4 x i32> %i32, %zw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext <4 x i8> %i8 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext <4 x i8> %i8 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = add <4 x i32> %zl1_8_32, %zl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sw_8_64 = sext <4 x i8> %i8 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asw_8_64 = add <4 x i64> %i64, %sw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sl1_8_64 = sext <4 x i8> %i8 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sl2_8_64 = sext <4 x i8> %i8 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asl_8_64 = add <4 x i64> %sl1_8_64, %sl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zw_8_64 = zext <4 x i8> %i8 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azw_8_64 = add <4 x i64> %i64, %zw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zl1_8_64 = zext <4 x i8> %i8 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zl2_8_64 = zext <4 x i8> %i8 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azl_8_64 = add <4 x i64> %zl1_8_64, %zl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_16_32 = sext <4 x i16> %i16 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = add <4 x i32> %i32, %sw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_16_32 = sext <4 x i16> %i16 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_16_32 = sext <4 x i16> %i16 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = add <4 x i32> %sl1_16_32, %sl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_16_32 = zext <4 x i16> %i16 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = add <4 x i32> %i32, %zw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_32 = zext <4 x i16> %i16 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_32 = zext <4 x i16> %i16 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = add <4 x i32> %zl1_16_32, %zl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sw_16_64 = sext <4 x i16> %i16 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asw_16_64 = add <4 x i64> %i64, %sw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sl1_16_64 = sext <4 x i16> %i16 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sl2_16_64 = sext <4 x i16> %i16 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asl_16_64 = add <4 x i64> %sl1_16_64, %sl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zw_16_64 = zext <4 x i16> %i16 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azw_16_64 = add <4 x i64> %i64, %zw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zl1_16_64 = zext <4 x i16> %i16 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zl2_16_64 = zext <4 x i16> %i16 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azl_16_64 = add <4 x i64> %zl1_16_64, %zl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_32_64 = sext <4 x i32> %i32 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asw_32_64 = add <4 x i64> %i64, %sw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <4 x i32> %i32 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <4 x i32> %i32 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asl_32_64 = add <4 x i64> %sl1_32_64, %sl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext <4 x i32> %i32 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azw_32_64 = add <4 x i64> %i64, %zw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <4 x i32> %i32 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <4 x i32> %i32 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azl_32_64 = add <4 x i64> %zl1_32_64, %zl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'extaddv4'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext <4 x i8> %i8 to <4 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = add <4 x i16> %i16, %sw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext <4 x i8> %i8 to <4 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_16 = sext <4 x i8> %i8 to <4 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = add <4 x i16> %sl1_8_16, %sl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext <4 x i8> %i8 to <4 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = add <4 x i16> %i16, %zw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_16 = zext <4 x i8> %i8 to <4 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_16 = zext <4 x i8> %i8 to <4 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = add <4 x i16> %zl1_8_16, %zl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext <4 x i8> %i8 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = add <4 x i32> %i32, %sw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext <4 x i8> %i8 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext <4 x i8> %i8 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = add <4 x i32> %sl1_8_32, %sl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext <4 x i8> %i8 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = add <4 x i32> %i32, %zw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext <4 x i8> %i8 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext <4 x i8> %i8 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = add <4 x i32> %zl1_8_32, %zl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext <4 x i8> %i8 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = add <4 x i64> %i64, %sw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext <4 x i8> %i8 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext <4 x i8> %i8 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = add <4 x i64> %sl1_8_64, %sl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_64 = zext <4 x i8> %i8 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = add <4 x i64> %i64, %zw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_64 = zext <4 x i8> %i8 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_64 = zext <4 x i8> %i8 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = add <4 x i64> %zl1_8_64, %zl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_16_32 = sext <4 x i16> %i16 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = add <4 x i32> %i32, %sw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_16_32 = sext <4 x i16> %i16 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_16_32 = sext <4 x i16> %i16 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = add <4 x i32> %sl1_16_32, %sl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_16_32 = zext <4 x i16> %i16 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = add <4 x i32> %i32, %zw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_32 = zext <4 x i16> %i16 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_32 = zext <4 x i16> %i16 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = add <4 x i32> %zl1_16_32, %zl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext <4 x i16> %i16 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = add <4 x i64> %i64, %sw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext <4 x i16> %i16 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext <4 x i16> %i16 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = add <4 x i64> %sl1_16_64, %sl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_64 = zext <4 x i16> %i16 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = add <4 x i64> %i64, %zw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_64 = zext <4 x i16> %i16 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_64 = zext <4 x i16> %i16 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = add <4 x i64> %zl1_16_64, %zl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_32_64 = sext <4 x i32> %i32 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = add <4 x i64> %i64, %sw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <4 x i32> %i32 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <4 x i32> %i32 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = add <4 x i64> %sl1_32_64, %sl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext <4 x i32> %i32 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = add <4 x i64> %i64, %zw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <4 x i32> %i32 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <4 x i32> %i32 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = add <4 x i64> %zl1_32_64, %zl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'extaddv4'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_16 = sext <4 x i8> %i8 to <4 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_16 = add <4 x i16> %i16, %sw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_16 = sext <4 x i8> %i8 to <4 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_16 = sext <4 x i8> %i8 to <4 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_16 = add <4 x i16> %sl1_8_16, %sl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_16 = zext <4 x i8> %i8 to <4 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_16 = add <4 x i16> %i16, %zw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_16 = zext <4 x i8> %i8 to <4 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_16 = zext <4 x i8> %i8 to <4 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_16 = add <4 x i16> %zl1_8_16, %zl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_32 = sext <4 x i8> %i8 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_32 = add <4 x i32> %i32, %sw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_32 = sext <4 x i8> %i8 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_32 = sext <4 x i8> %i8 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_32 = add <4 x i32> %sl1_8_32, %sl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_32 = zext <4 x i8> %i8 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_32 = add <4 x i32> %i32, %zw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_32 = zext <4 x i8> %i8 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_32 = zext <4 x i8> %i8 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_32 = add <4 x i32> %zl1_8_32, %zl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sw_8_64 = sext <4 x i8> %i8 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_64 = add <4 x i64> %i64, %sw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_8_64 = sext <4 x i8> %i8 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_8_64 = sext <4 x i8> %i8 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_64 = add <4 x i64> %sl1_8_64, %sl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zw_8_64 = zext <4 x i8> %i8 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_64 = add <4 x i64> %i64, %zw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_8_64 = zext <4 x i8> %i8 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_8_64 = zext <4 x i8> %i8 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_64 = add <4 x i64> %zl1_8_64, %zl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sw_16_32 = sext <4 x i16> %i16 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_16_32 = add <4 x i32> %i32, %sw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_16_32 = sext <4 x i16> %i16 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_16_32 = sext <4 x i16> %i16 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_16_32 = add <4 x i32> %sl1_16_32, %sl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_16_32 = zext <4 x i16> %i16 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_16_32 = add <4 x i32> %i32, %zw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_16_32 = zext <4 x i16> %i16 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_16_32 = zext <4 x i16> %i16 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_16_32 = add <4 x i32> %zl1_16_32, %zl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sw_16_64 = sext <4 x i16> %i16 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asw_16_64 = add <4 x i64> %i64, %sw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_16_64 = sext <4 x i16> %i16 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_16_64 = sext <4 x i16> %i16 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asl_16_64 = add <4 x i64> %sl1_16_64, %sl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zw_16_64 = zext <4 x i16> %i16 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azw_16_64 = add <4 x i64> %i64, %zw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_16_64 = zext <4 x i16> %i16 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_16_64 = zext <4 x i16> %i16 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azl_16_64 = add <4 x i64> %zl1_16_64, %zl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sw_32_64 = sext <4 x i32> %i32 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asw_32_64 = add <4 x i64> %i64, %sw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_32_64 = sext <4 x i32> %i32 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_32_64 = sext <4 x i32> %i32 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asl_32_64 = add <4 x i64> %sl1_32_64, %sl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_32_64 = zext <4 x i32> %i32 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azw_32_64 = add <4 x i64> %i64, %zw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_32_64 = zext <4 x i32> %i32 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_32_64 = zext <4 x i32> %i32 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azl_32_64 = add <4 x i64> %zl1_32_64, %zl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %sw_8_16 = sext <4 x i8> %i8 to <4 x i16>
   %asw_8_16 = add <4 x i16> %i16, %sw_8_16
@@ -611,131 +421,68 @@ define void @extaddv4(<4 x i8> %i8, <4 x i16> %i16, <4 x i32> %i32, <4 x i64> %i
 }
 
 define void @extaddv8(<8 x i8> %i8, <8 x i16> %i16, <8 x i32> %i32, <8 x i64> %i64)  {
-; RECIP-LABEL: 'extaddv8'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = add <8 x i16> %i16, %sw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = add <8 x i16> %sl1_8_16, %sl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = add <8 x i16> %i16, %zw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = add <8 x i16> %zl1_8_16, %zl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sw_8_32 = sext <8 x i8> %i8 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asw_8_32 = add <8 x i32> %i32, %sw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sl1_8_32 = sext <8 x i8> %i8 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sl2_8_32 = sext <8 x i8> %i8 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asl_8_32 = add <8 x i32> %sl1_8_32, %sl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zw_8_32 = zext <8 x i8> %i8 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azw_8_32 = add <8 x i32> %i32, %zw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zl1_8_32 = zext <8 x i8> %i8 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zl2_8_32 = zext <8 x i8> %i8 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azl_8_32 = add <8 x i32> %zl1_8_32, %zl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %sw_8_64 = sext <8 x i8> %i8 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asw_8_64 = add <8 x i64> %i64, %sw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %sl1_8_64 = sext <8 x i8> %i8 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %sl2_8_64 = sext <8 x i8> %i8 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asl_8_64 = add <8 x i64> %sl1_8_64, %sl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %zw_8_64 = zext <8 x i8> %i8 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azw_8_64 = add <8 x i64> %i64, %zw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %zl1_8_64 = zext <8 x i8> %i8 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %zl2_8_64 = zext <8 x i8> %i8 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azl_8_64 = add <8 x i64> %zl1_8_64, %zl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_16_32 = sext <8 x i16> %i16 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asw_16_32 = add <8 x i32> %i32, %sw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_16_32 = sext <8 x i16> %i16 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_16_32 = sext <8 x i16> %i16 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asl_16_32 = add <8 x i32> %sl1_16_32, %sl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_16_32 = zext <8 x i16> %i16 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azw_16_32 = add <8 x i32> %i32, %zw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_32 = zext <8 x i16> %i16 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_32 = zext <8 x i16> %i16 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azl_16_32 = add <8 x i32> %zl1_16_32, %zl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sw_16_64 = sext <8 x i16> %i16 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asw_16_64 = add <8 x i64> %i64, %sw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sl1_16_64 = sext <8 x i16> %i16 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sl2_16_64 = sext <8 x i16> %i16 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asl_16_64 = add <8 x i64> %sl1_16_64, %sl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %zw_16_64 = zext <8 x i16> %i16 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azw_16_64 = add <8 x i64> %i64, %zw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %zl1_16_64 = zext <8 x i16> %i16 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %zl2_16_64 = zext <8 x i16> %i16 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azl_16_64 = add <8 x i64> %zl1_16_64, %zl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_32_64 = sext <8 x i32> %i32 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asw_32_64 = add <8 x i64> %i64, %sw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <8 x i32> %i32 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <8 x i32> %i32 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asl_32_64 = add <8 x i64> %sl1_32_64, %sl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext <8 x i32> %i32 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azw_32_64 = add <8 x i64> %i64, %zw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <8 x i32> %i32 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <8 x i32> %i32 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azl_32_64 = add <8 x i64> %zl1_32_64, %zl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'extaddv8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = add <8 x i16> %i16, %sw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = add <8 x i16> %sl1_8_16, %sl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = add <8 x i16> %i16, %zw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = add <8 x i16> %zl1_8_16, %zl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext <8 x i8> %i8 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = add <8 x i32> %i32, %sw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext <8 x i8> %i8 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext <8 x i8> %i8 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = add <8 x i32> %sl1_8_32, %sl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext <8 x i8> %i8 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = add <8 x i32> %i32, %zw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext <8 x i8> %i8 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext <8 x i8> %i8 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = add <8 x i32> %zl1_8_32, %zl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext <8 x i8> %i8 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = add <8 x i64> %i64, %sw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext <8 x i8> %i8 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext <8 x i8> %i8 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = add <8 x i64> %sl1_8_64, %sl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_64 = zext <8 x i8> %i8 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = add <8 x i64> %i64, %zw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_64 = zext <8 x i8> %i8 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_64 = zext <8 x i8> %i8 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = add <8 x i64> %zl1_8_64, %zl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_16_32 = sext <8 x i16> %i16 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = add <8 x i32> %i32, %sw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_16_32 = sext <8 x i16> %i16 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_16_32 = sext <8 x i16> %i16 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = add <8 x i32> %sl1_16_32, %sl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_16_32 = zext <8 x i16> %i16 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = add <8 x i32> %i32, %zw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_32 = zext <8 x i16> %i16 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_32 = zext <8 x i16> %i16 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = add <8 x i32> %zl1_16_32, %zl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext <8 x i16> %i16 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = add <8 x i64> %i64, %sw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext <8 x i16> %i16 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext <8 x i16> %i16 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = add <8 x i64> %sl1_16_64, %sl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_64 = zext <8 x i16> %i16 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = add <8 x i64> %i64, %zw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_64 = zext <8 x i16> %i16 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_64 = zext <8 x i16> %i16 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = add <8 x i64> %zl1_16_64, %zl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_32_64 = sext <8 x i32> %i32 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = add <8 x i64> %i64, %sw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <8 x i32> %i32 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <8 x i32> %i32 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = add <8 x i64> %sl1_32_64, %sl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext <8 x i32> %i32 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = add <8 x i64> %i64, %zw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <8 x i32> %i32 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <8 x i32> %i32 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = add <8 x i64> %zl1_32_64, %zl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'extaddv8'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sw_8_16 = sext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_16 = add <8 x i16> %i16, %sw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_8_16 = sext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_8_16 = sext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_16 = add <8 x i16> %sl1_8_16, %sl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_8_16 = zext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_16 = add <8 x i16> %i16, %zw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_8_16 = zext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_8_16 = zext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_16 = add <8 x i16> %zl1_8_16, %zl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sw_8_32 = sext <8 x i8> %i8 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_32 = add <8 x i32> %i32, %sw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_8_32 = sext <8 x i8> %i8 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_8_32 = sext <8 x i8> %i8 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_32 = add <8 x i32> %sl1_8_32, %sl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zw_8_32 = zext <8 x i8> %i8 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_32 = add <8 x i32> %i32, %zw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_8_32 = zext <8 x i8> %i8 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_8_32 = zext <8 x i8> %i8 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_32 = add <8 x i32> %zl1_8_32, %zl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %sw_8_64 = sext <8 x i8> %i8 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_64 = add <8 x i64> %i64, %sw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_8_64 = sext <8 x i8> %i8 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_8_64 = sext <8 x i8> %i8 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_64 = add <8 x i64> %sl1_8_64, %sl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %zw_8_64 = zext <8 x i8> %i8 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_64 = add <8 x i64> %i64, %zw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_8_64 = zext <8 x i8> %i8 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_8_64 = zext <8 x i8> %i8 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_64 = add <8 x i64> %zl1_8_64, %zl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sw_16_32 = sext <8 x i16> %i16 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asw_16_32 = add <8 x i32> %i32, %sw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_16_32 = sext <8 x i16> %i16 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_16_32 = sext <8 x i16> %i16 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asl_16_32 = add <8 x i32> %sl1_16_32, %sl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_16_32 = zext <8 x i16> %i16 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azw_16_32 = add <8 x i32> %i32, %zw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_16_32 = zext <8 x i16> %i16 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_16_32 = zext <8 x i16> %i16 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azl_16_32 = add <8 x i32> %zl1_16_32, %zl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %sw_16_64 = sext <8 x i16> %i16 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asw_16_64 = add <8 x i64> %i64, %sw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_16_64 = sext <8 x i16> %i16 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_16_64 = sext <8 x i16> %i16 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asl_16_64 = add <8 x i64> %sl1_16_64, %sl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %zw_16_64 = zext <8 x i16> %i16 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azw_16_64 = add <8 x i64> %i64, %zw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_16_64 = zext <8 x i16> %i16 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_16_64 = zext <8 x i16> %i16 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azl_16_64 = add <8 x i64> %zl1_16_64, %zl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sw_32_64 = sext <8 x i32> %i32 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asw_32_64 = add <8 x i64> %i64, %sw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_32_64 = sext <8 x i32> %i32 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_32_64 = sext <8 x i32> %i32 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asl_32_64 = add <8 x i64> %sl1_32_64, %sl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_32_64 = zext <8 x i32> %i32 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azw_32_64 = add <8 x i64> %i64, %zw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_32_64 = zext <8 x i32> %i32 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_32_64 = zext <8 x i32> %i32 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azl_32_64 = add <8 x i64> %zl1_32_64, %zl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %sw_8_16 = sext <8 x i8> %i8 to <8 x i16>
   %asw_8_16 = add <8 x i16> %i16, %sw_8_16
@@ -813,131 +560,68 @@ define void @extaddv8(<8 x i8> %i8, <8 x i16> %i16, <8 x i32> %i32, <8 x i64> %i
 }
 
 define void @extaddv16(<16 x i8> %i8, <16 x i16> %i16, <16 x i32> %i32, <16 x i64> %i64)  {
-; RECIP-LABEL: 'extaddv16'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_8_16 = sext <16 x i8> %i8 to <16 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asw_8_16 = add <16 x i16> %i16, %sw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_8_16 = sext <16 x i8> %i8 to <16 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_8_16 = sext <16 x i8> %i8 to <16 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asl_8_16 = add <16 x i16> %sl1_8_16, %sl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_8_16 = zext <16 x i8> %i8 to <16 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azw_8_16 = add <16 x i16> %i16, %zw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_16 = zext <16 x i8> %i8 to <16 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_8_16 = zext <16 x i8> %i8 to <16 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azl_8_16 = add <16 x i16> %zl1_8_16, %zl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sw_8_32 = sext <16 x i8> %i8 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asw_8_32 = add <16 x i32> %i32, %sw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sl1_8_32 = sext <16 x i8> %i8 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sl2_8_32 = sext <16 x i8> %i8 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asl_8_32 = add <16 x i32> %sl1_8_32, %sl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %zw_8_32 = zext <16 x i8> %i8 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azw_8_32 = add <16 x i32> %i32, %zw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %zl1_8_32 = zext <16 x i8> %i8 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %zl2_8_32 = zext <16 x i8> %i8 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azl_8_32 = add <16 x i32> %zl1_8_32, %zl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %sw_8_64 = sext <16 x i8> %i8 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %asw_8_64 = add <16 x i64> %i64, %sw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %sl1_8_64 = sext <16 x i8> %i8 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %sl2_8_64 = sext <16 x i8> %i8 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %asl_8_64 = add <16 x i64> %sl1_8_64, %sl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %zw_8_64 = zext <16 x i8> %i8 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %azw_8_64 = add <16 x i64> %i64, %zw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %zl1_8_64 = zext <16 x i8> %i8 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %zl2_8_64 = zext <16 x i8> %i8 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %azl_8_64 = add <16 x i64> %zl1_8_64, %zl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_16_32 = sext <16 x i16> %i16 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asw_16_32 = add <16 x i32> %i32, %sw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_16_32 = sext <16 x i16> %i16 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_16_32 = sext <16 x i16> %i16 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asl_16_32 = add <16 x i32> %sl1_16_32, %sl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_16_32 = zext <16 x i16> %i16 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azw_16_32 = add <16 x i32> %i32, %zw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_32 = zext <16 x i16> %i16 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_32 = zext <16 x i16> %i16 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azl_16_32 = add <16 x i32> %zl1_16_32, %zl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %sw_16_64 = sext <16 x i16> %i16 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %asw_16_64 = add <16 x i64> %i64, %sw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %sl1_16_64 = sext <16 x i16> %i16 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %sl2_16_64 = sext <16 x i16> %i16 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %asl_16_64 = add <16 x i64> %sl1_16_64, %sl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %zw_16_64 = zext <16 x i16> %i16 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %azw_16_64 = add <16 x i64> %i64, %zw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %zl1_16_64 = zext <16 x i16> %i16 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %zl2_16_64 = zext <16 x i16> %i16 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %azl_16_64 = add <16 x i64> %zl1_16_64, %zl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_32_64 = sext <16 x i32> %i32 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %asw_32_64 = add <16 x i64> %i64, %sw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <16 x i32> %i32 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <16 x i32> %i32 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %asl_32_64 = add <16 x i64> %sl1_32_64, %sl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext <16 x i32> %i32 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %azw_32_64 = add <16 x i64> %i64, %zw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <16 x i32> %i32 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <16 x i32> %i32 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %azl_32_64 = add <16 x i64> %zl1_32_64, %zl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'extaddv16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_8_16 = sext <16 x i8> %i8 to <16 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = add <16 x i16> %i16, %sw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_8_16 = sext <16 x i8> %i8 to <16 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_8_16 = sext <16 x i8> %i8 to <16 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = add <16 x i16> %sl1_8_16, %sl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_8_16 = zext <16 x i8> %i8 to <16 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = add <16 x i16> %i16, %zw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_16 = zext <16 x i8> %i8 to <16 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_8_16 = zext <16 x i8> %i8 to <16 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = add <16 x i16> %zl1_8_16, %zl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext <16 x i8> %i8 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = add <16 x i32> %i32, %sw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext <16 x i8> %i8 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext <16 x i8> %i8 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = add <16 x i32> %sl1_8_32, %sl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext <16 x i8> %i8 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = add <16 x i32> %i32, %zw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext <16 x i8> %i8 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext <16 x i8> %i8 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = add <16 x i32> %zl1_8_32, %zl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext <16 x i8> %i8 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = add <16 x i64> %i64, %sw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext <16 x i8> %i8 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext <16 x i8> %i8 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = add <16 x i64> %sl1_8_64, %sl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_64 = zext <16 x i8> %i8 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = add <16 x i64> %i64, %zw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_64 = zext <16 x i8> %i8 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_64 = zext <16 x i8> %i8 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = add <16 x i64> %zl1_8_64, %zl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_16_32 = sext <16 x i16> %i16 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = add <16 x i32> %i32, %sw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_16_32 = sext <16 x i16> %i16 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_16_32 = sext <16 x i16> %i16 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = add <16 x i32> %sl1_16_32, %sl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_16_32 = zext <16 x i16> %i16 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = add <16 x i32> %i32, %zw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_32 = zext <16 x i16> %i16 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_32 = zext <16 x i16> %i16 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = add <16 x i32> %zl1_16_32, %zl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext <16 x i16> %i16 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = add <16 x i64> %i64, %sw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext <16 x i16> %i16 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext <16 x i16> %i16 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = add <16 x i64> %sl1_16_64, %sl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_64 = zext <16 x i16> %i16 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = add <16 x i64> %i64, %zw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_64 = zext <16 x i16> %i16 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_64 = zext <16 x i16> %i16 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = add <16 x i64> %zl1_16_64, %zl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_32_64 = sext <16 x i32> %i32 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = add <16 x i64> %i64, %sw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <16 x i32> %i32 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <16 x i32> %i32 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = add <16 x i64> %sl1_32_64, %sl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext <16 x i32> %i32 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = add <16 x i64> %i64, %zw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <16 x i32> %i32 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <16 x i32> %i32 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = add <16 x i64> %zl1_32_64, %zl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'extaddv16'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sw_8_16 = sext <16 x i8> %i8 to <16 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_16 = add <16 x i16> %i16, %sw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_8_16 = sext <16 x i8> %i8 to <16 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_8_16 = sext <16 x i8> %i8 to <16 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_16 = add <16 x i16> %sl1_8_16, %sl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_8_16 = zext <16 x i8> %i8 to <16 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_16 = add <16 x i16> %i16, %zw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_8_16 = zext <16 x i8> %i8 to <16 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_8_16 = zext <16 x i8> %i8 to <16 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_16 = add <16 x i16> %zl1_8_16, %zl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %sw_8_32 = sext <16 x i8> %i8 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_32 = add <16 x i32> %i32, %sw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_8_32 = sext <16 x i8> %i8 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_8_32 = sext <16 x i8> %i8 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_32 = add <16 x i32> %sl1_8_32, %sl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %zw_8_32 = zext <16 x i8> %i8 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_32 = add <16 x i32> %i32, %zw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_8_32 = zext <16 x i8> %i8 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_8_32 = zext <16 x i8> %i8 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_32 = add <16 x i32> %zl1_8_32, %zl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:15 CodeSize:1 Lat:1 SizeLat:1 for: %sw_8_64 = sext <16 x i8> %i8 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_64 = add <16 x i64> %i64, %sw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:15 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_8_64 = sext <16 x i8> %i8 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:15 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_8_64 = sext <16 x i8> %i8 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_64 = add <16 x i64> %sl1_8_64, %sl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:15 CodeSize:1 Lat:1 SizeLat:1 for: %zw_8_64 = zext <16 x i8> %i8 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_64 = add <16 x i64> %i64, %zw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:15 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_8_64 = zext <16 x i8> %i8 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:15 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_8_64 = zext <16 x i8> %i8 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_64 = add <16 x i64> %zl1_8_64, %zl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sw_16_32 = sext <16 x i16> %i16 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asw_16_32 = add <16 x i32> %i32, %sw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_16_32 = sext <16 x i16> %i16 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_16_32 = sext <16 x i16> %i16 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asl_16_32 = add <16 x i32> %sl1_16_32, %sl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_16_32 = zext <16 x i16> %i16 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azw_16_32 = add <16 x i32> %i32, %zw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_16_32 = zext <16 x i16> %i16 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_16_32 = zext <16 x i16> %i16 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azl_16_32 = add <16 x i32> %zl1_16_32, %zl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: %sw_16_64 = sext <16 x i16> %i16 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %asw_16_64 = add <16 x i64> %i64, %sw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_16_64 = sext <16 x i16> %i16 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_16_64 = sext <16 x i16> %i16 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %asl_16_64 = add <16 x i64> %sl1_16_64, %sl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: %zw_16_64 = zext <16 x i16> %i16 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %azw_16_64 = add <16 x i64> %i64, %zw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_16_64 = zext <16 x i16> %i16 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_16_64 = zext <16 x i16> %i16 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %azl_16_64 = add <16 x i64> %zl1_16_64, %zl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sw_32_64 = sext <16 x i32> %i32 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %asw_32_64 = add <16 x i64> %i64, %sw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_32_64 = sext <16 x i32> %i32 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_32_64 = sext <16 x i32> %i32 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %asl_32_64 = add <16 x i64> %sl1_32_64, %sl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_32_64 = zext <16 x i32> %i32 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %azw_32_64 = add <16 x i64> %i64, %zw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_32_64 = zext <16 x i32> %i32 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_32_64 = zext <16 x i32> %i32 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %azl_32_64 = add <16 x i64> %zl1_32_64, %zl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %sw_8_16 = sext <16 x i8> %i8 to <16 x i16>
   %asw_8_16 = add <16 x i16> %i16, %sw_8_16
@@ -1015,131 +699,68 @@ define void @extaddv16(<16 x i8> %i8, <16 x i16> %i16, <16 x i32> %i32, <16 x i6
 }
 
 define void @extsubv1(i8 %i8, i16 %i16, i32 %i32, i64 %i64)  {
-; RECIP-LABEL: 'extsubv1'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext i8 %i8 to i16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = sub i16 %i16, %sw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext i8 %i8 to i16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_16 = sext i8 %i8 to i16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = sub i16 %sl1_8_16, %sl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext i8 %i8 to i16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = sub i16 %i16, %zw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_16 = zext i8 %i8 to i16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_16 = zext i8 %i8 to i16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = sub i16 %zl1_8_16, %zl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext i8 %i8 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = sub i32 %i32, %sw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext i8 %i8 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext i8 %i8 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = sub i32 %sl1_8_32, %sl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext i8 %i8 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = sub i32 %i32, %zw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext i8 %i8 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext i8 %i8 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = sub i32 %zl1_8_32, %zl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext i8 %i8 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = sub i64 %i64, %sw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext i8 %i8 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext i8 %i8 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = sub i64 %sl1_8_64, %sl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_8_64 = zext i8 %i8 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = sub i64 %i64, %zw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_64 = zext i8 %i8 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_8_64 = zext i8 %i8 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = sub i64 %zl1_8_64, %zl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_32 = sext i16 %i16 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = sub i32 %i32, %sw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_32 = sext i16 %i16 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_32 = sext i16 %i16 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = sub i32 %sl1_16_32, %sl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_32 = zext i16 %i16 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = sub i32 %i32, %zw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_32 = zext i16 %i16 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_32 = zext i16 %i16 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = sub i32 %zl1_16_32, %zl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext i16 %i16 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = sub i64 %i64, %sw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext i16 %i16 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext i16 %i16 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = sub i64 %sl1_16_64, %sl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_16_64 = zext i16 %i16 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = sub i64 %i64, %zw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_64 = zext i16 %i16 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_64 = zext i16 %i16 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = sub i64 %zl1_16_64, %zl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_32_64 = sext i32 %i32 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = sub i64 %i64, %sw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_32_64 = sext i32 %i32 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_32_64 = sext i32 %i32 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = sub i64 %sl1_32_64, %sl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext i32 %i32 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = sub i64 %i64, %zw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext i32 %i32 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext i32 %i32 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = sub i64 %zl1_32_64, %zl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'extsubv1'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext i8 %i8 to i16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = sub i16 %i16, %sw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext i8 %i8 to i16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_16 = sext i8 %i8 to i16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = sub i16 %sl1_8_16, %sl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext i8 %i8 to i16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = sub i16 %i16, %zw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_16 = zext i8 %i8 to i16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_16 = zext i8 %i8 to i16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = sub i16 %zl1_8_16, %zl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext i8 %i8 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = sub i32 %i32, %sw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext i8 %i8 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext i8 %i8 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = sub i32 %sl1_8_32, %sl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext i8 %i8 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = sub i32 %i32, %zw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext i8 %i8 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext i8 %i8 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = sub i32 %zl1_8_32, %zl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext i8 %i8 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = sub i64 %i64, %sw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext i8 %i8 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext i8 %i8 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = sub i64 %sl1_8_64, %sl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_8_64 = zext i8 %i8 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = sub i64 %i64, %zw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_64 = zext i8 %i8 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_8_64 = zext i8 %i8 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = sub i64 %zl1_8_64, %zl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_32 = sext i16 %i16 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = sub i32 %i32, %sw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_32 = sext i16 %i16 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_32 = sext i16 %i16 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = sub i32 %sl1_16_32, %sl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_32 = zext i16 %i16 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = sub i32 %i32, %zw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_32 = zext i16 %i16 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_32 = zext i16 %i16 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = sub i32 %zl1_16_32, %zl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext i16 %i16 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = sub i64 %i64, %sw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext i16 %i16 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext i16 %i16 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = sub i64 %sl1_16_64, %sl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_16_64 = zext i16 %i16 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = sub i64 %i64, %zw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_64 = zext i16 %i16 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_64 = zext i16 %i16 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = sub i64 %zl1_16_64, %zl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_32_64 = sext i32 %i32 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = sub i64 %i64, %sw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_32_64 = sext i32 %i32 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_32_64 = sext i32 %i32 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = sub i64 %sl1_32_64, %sl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext i32 %i32 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = sub i64 %i64, %zw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext i32 %i32 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext i32 %i32 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = sub i64 %zl1_32_64, %zl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'extsubv1'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_16 = sext i8 %i8 to i16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_16 = sub i16 %i16, %sw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_16 = sext i8 %i8 to i16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_16 = sext i8 %i8 to i16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_16 = sub i16 %sl1_8_16, %sl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_16 = zext i8 %i8 to i16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_16 = sub i16 %i16, %zw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_16 = zext i8 %i8 to i16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_16 = zext i8 %i8 to i16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_16 = sub i16 %zl1_8_16, %zl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_32 = sext i8 %i8 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_32 = sub i32 %i32, %sw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_32 = sext i8 %i8 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_32 = sext i8 %i8 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_32 = sub i32 %sl1_8_32, %sl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_32 = zext i8 %i8 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_32 = sub i32 %i32, %zw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_32 = zext i8 %i8 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_32 = zext i8 %i8 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_32 = sub i32 %zl1_8_32, %zl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_64 = sext i8 %i8 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_64 = sub i64 %i64, %sw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_64 = sext i8 %i8 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_64 = sext i8 %i8 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_64 = sub i64 %sl1_8_64, %sl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_8_64 = zext i8 %i8 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_64 = sub i64 %i64, %zw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_8_64 = zext i8 %i8 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_8_64 = zext i8 %i8 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_64 = sub i64 %zl1_8_64, %zl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_16_32 = sext i16 %i16 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_16_32 = sub i32 %i32, %sw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_16_32 = sext i16 %i16 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_16_32 = sext i16 %i16 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_16_32 = sub i32 %sl1_16_32, %sl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_16_32 = zext i16 %i16 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_16_32 = sub i32 %i32, %zw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_16_32 = zext i16 %i16 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_16_32 = zext i16 %i16 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_16_32 = sub i32 %zl1_16_32, %zl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_16_64 = sext i16 %i16 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_16_64 = sub i64 %i64, %sw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_16_64 = sext i16 %i16 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_16_64 = sext i16 %i16 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_16_64 = sub i64 %sl1_16_64, %sl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_16_64 = zext i16 %i16 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_16_64 = sub i64 %i64, %zw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_16_64 = zext i16 %i16 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_16_64 = zext i16 %i16 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_16_64 = sub i64 %zl1_16_64, %zl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_32_64 = sext i32 %i32 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_32_64 = sub i64 %i64, %sw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_32_64 = sext i32 %i32 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_32_64 = sext i32 %i32 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_32_64 = sub i64 %sl1_32_64, %sl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_32_64 = zext i32 %i32 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_32_64 = sub i64 %i64, %zw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_32_64 = zext i32 %i32 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_32_64 = zext i32 %i32 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_32_64 = sub i64 %zl1_32_64, %zl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %sw_8_16 = sext i8 %i8 to i16
   %asw_8_16 = sub i16 %i16, %sw_8_16
@@ -1217,131 +838,68 @@ define void @extsubv1(i8 %i8, i16 %i16, i32 %i32, i64 %i64)  {
 }
 
 define void @extsubv2(<2 x i8> %i8, <2 x i16> %i16, <2 x i32> %i32, <2 x i64> %i64)  {
-; RECIP-LABEL: 'extsubv2'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext <2 x i8> %i8 to <2 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = sub <2 x i16> %i16, %sw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext <2 x i8> %i8 to <2 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_16 = sext <2 x i8> %i8 to <2 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = sub <2 x i16> %sl1_8_16, %sl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext <2 x i8> %i8 to <2 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = sub <2 x i16> %i16, %zw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_16 = zext <2 x i8> %i8 to <2 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_16 = zext <2 x i8> %i8 to <2 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = sub <2 x i16> %zl1_8_16, %zl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext <2 x i8> %i8 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = sub <2 x i32> %i32, %sw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext <2 x i8> %i8 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext <2 x i8> %i8 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = sub <2 x i32> %sl1_8_32, %sl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext <2 x i8> %i8 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = sub <2 x i32> %i32, %zw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext <2 x i8> %i8 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext <2 x i8> %i8 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = sub <2 x i32> %zl1_8_32, %zl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext <2 x i8> %i8 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = sub <2 x i64> %i64, %sw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext <2 x i8> %i8 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext <2 x i8> %i8 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = sub <2 x i64> %sl1_8_64, %sl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_64 = zext <2 x i8> %i8 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = sub <2 x i64> %i64, %zw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_64 = zext <2 x i8> %i8 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_64 = zext <2 x i8> %i8 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = sub <2 x i64> %zl1_8_64, %zl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_32 = sext <2 x i16> %i16 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = sub <2 x i32> %i32, %sw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_32 = sext <2 x i16> %i16 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_32 = sext <2 x i16> %i16 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = sub <2 x i32> %sl1_16_32, %sl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_32 = zext <2 x i16> %i16 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = sub <2 x i32> %i32, %zw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_32 = zext <2 x i16> %i16 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_32 = zext <2 x i16> %i16 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = sub <2 x i32> %zl1_16_32, %zl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext <2 x i16> %i16 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = sub <2 x i64> %i64, %sw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext <2 x i16> %i16 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext <2 x i16> %i16 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = sub <2 x i64> %sl1_16_64, %sl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_64 = zext <2 x i16> %i16 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = sub <2 x i64> %i64, %zw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_64 = zext <2 x i16> %i16 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_64 = zext <2 x i16> %i16 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = sub <2 x i64> %zl1_16_64, %zl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_32_64 = sext <2 x i32> %i32 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = sub <2 x i64> %i64, %sw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <2 x i32> %i32 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <2 x i32> %i32 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = sub <2 x i64> %sl1_32_64, %sl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext <2 x i32> %i32 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = sub <2 x i64> %i64, %zw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <2 x i32> %i32 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <2 x i32> %i32 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = sub <2 x i64> %zl1_32_64, %zl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'extsubv2'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext <2 x i8> %i8 to <2 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = sub <2 x i16> %i16, %sw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext <2 x i8> %i8 to <2 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_16 = sext <2 x i8> %i8 to <2 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = sub <2 x i16> %sl1_8_16, %sl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext <2 x i8> %i8 to <2 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = sub <2 x i16> %i16, %zw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_16 = zext <2 x i8> %i8 to <2 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_16 = zext <2 x i8> %i8 to <2 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = sub <2 x i16> %zl1_8_16, %zl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext <2 x i8> %i8 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = sub <2 x i32> %i32, %sw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext <2 x i8> %i8 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext <2 x i8> %i8 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = sub <2 x i32> %sl1_8_32, %sl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext <2 x i8> %i8 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = sub <2 x i32> %i32, %zw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext <2 x i8> %i8 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext <2 x i8> %i8 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = sub <2 x i32> %zl1_8_32, %zl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext <2 x i8> %i8 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = sub <2 x i64> %i64, %sw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext <2 x i8> %i8 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext <2 x i8> %i8 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = sub <2 x i64> %sl1_8_64, %sl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_64 = zext <2 x i8> %i8 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = sub <2 x i64> %i64, %zw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_64 = zext <2 x i8> %i8 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_64 = zext <2 x i8> %i8 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = sub <2 x i64> %zl1_8_64, %zl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_32 = sext <2 x i16> %i16 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = sub <2 x i32> %i32, %sw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_32 = sext <2 x i16> %i16 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_32 = sext <2 x i16> %i16 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = sub <2 x i32> %sl1_16_32, %sl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_32 = zext <2 x i16> %i16 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = sub <2 x i32> %i32, %zw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_32 = zext <2 x i16> %i16 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_32 = zext <2 x i16> %i16 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = sub <2 x i32> %zl1_16_32, %zl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext <2 x i16> %i16 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = sub <2 x i64> %i64, %sw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext <2 x i16> %i16 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext <2 x i16> %i16 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = sub <2 x i64> %sl1_16_64, %sl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_64 = zext <2 x i16> %i16 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = sub <2 x i64> %i64, %zw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_64 = zext <2 x i16> %i16 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_64 = zext <2 x i16> %i16 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = sub <2 x i64> %zl1_16_64, %zl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_32_64 = sext <2 x i32> %i32 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = sub <2 x i64> %i64, %sw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <2 x i32> %i32 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <2 x i32> %i32 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = sub <2 x i64> %sl1_32_64, %sl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext <2 x i32> %i32 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = sub <2 x i64> %i64, %zw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <2 x i32> %i32 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <2 x i32> %i32 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = sub <2 x i64> %zl1_32_64, %zl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'extsubv2'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_16 = sext <2 x i8> %i8 to <2 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_16 = sub <2 x i16> %i16, %sw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_16 = sext <2 x i8> %i8 to <2 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_16 = sext <2 x i8> %i8 to <2 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_16 = sub <2 x i16> %sl1_8_16, %sl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_16 = zext <2 x i8> %i8 to <2 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_16 = sub <2 x i16> %i16, %zw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_16 = zext <2 x i8> %i8 to <2 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_16 = zext <2 x i8> %i8 to <2 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_16 = sub <2 x i16> %zl1_8_16, %zl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_32 = sext <2 x i8> %i8 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_32 = sub <2 x i32> %i32, %sw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_32 = sext <2 x i8> %i8 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_32 = sext <2 x i8> %i8 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_32 = sub <2 x i32> %sl1_8_32, %sl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_32 = zext <2 x i8> %i8 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_32 = sub <2 x i32> %i32, %zw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_32 = zext <2 x i8> %i8 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_32 = zext <2 x i8> %i8 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_32 = sub <2 x i32> %zl1_8_32, %zl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_64 = sext <2 x i8> %i8 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_64 = sub <2 x i64> %i64, %sw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_64 = sext <2 x i8> %i8 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_64 = sext <2 x i8> %i8 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_64 = sub <2 x i64> %sl1_8_64, %sl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_64 = zext <2 x i8> %i8 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_64 = sub <2 x i64> %i64, %zw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_64 = zext <2 x i8> %i8 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_64 = zext <2 x i8> %i8 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_64 = sub <2 x i64> %zl1_8_64, %zl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_16_32 = sext <2 x i16> %i16 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_16_32 = sub <2 x i32> %i32, %sw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_16_32 = sext <2 x i16> %i16 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_16_32 = sext <2 x i16> %i16 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_16_32 = sub <2 x i32> %sl1_16_32, %sl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_16_32 = zext <2 x i16> %i16 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_16_32 = sub <2 x i32> %i32, %zw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_16_32 = zext <2 x i16> %i16 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_16_32 = zext <2 x i16> %i16 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_16_32 = sub <2 x i32> %zl1_16_32, %zl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_16_64 = sext <2 x i16> %i16 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_16_64 = sub <2 x i64> %i64, %sw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_16_64 = sext <2 x i16> %i16 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_16_64 = sext <2 x i16> %i16 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_16_64 = sub <2 x i64> %sl1_16_64, %sl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_16_64 = zext <2 x i16> %i16 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_16_64 = sub <2 x i64> %i64, %zw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_16_64 = zext <2 x i16> %i16 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_16_64 = zext <2 x i16> %i16 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_16_64 = sub <2 x i64> %zl1_16_64, %zl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sw_32_64 = sext <2 x i32> %i32 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_32_64 = sub <2 x i64> %i64, %sw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_32_64 = sext <2 x i32> %i32 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_32_64 = sext <2 x i32> %i32 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_32_64 = sub <2 x i64> %sl1_32_64, %sl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_32_64 = zext <2 x i32> %i32 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_32_64 = sub <2 x i64> %i64, %zw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_32_64 = zext <2 x i32> %i32 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_32_64 = zext <2 x i32> %i32 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_32_64 = sub <2 x i64> %zl1_32_64, %zl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %sw_8_16 = sext <2 x i8> %i8 to <2 x i16>
   %asw_8_16 = sub <2 x i16> %i16, %sw_8_16
@@ -1419,131 +977,68 @@ define void @extsubv2(<2 x i8> %i8, <2 x i16> %i16, <2 x i32> %i32, <2 x i64> %i
 }
 
 define void @extsubv4(<4 x i8> %i8, <4 x i16> %i16, <4 x i32> %i32, <4 x i64> %i64)  {
-; RECIP-LABEL: 'extsubv4'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext <4 x i8> %i8 to <4 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = sub <4 x i16> %i16, %sw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext <4 x i8> %i8 to <4 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_16 = sext <4 x i8> %i8 to <4 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = sub <4 x i16> %sl1_8_16, %sl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext <4 x i8> %i8 to <4 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = sub <4 x i16> %i16, %zw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_16 = zext <4 x i8> %i8 to <4 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_16 = zext <4 x i8> %i8 to <4 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = sub <4 x i16> %zl1_8_16, %zl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext <4 x i8> %i8 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = sub <4 x i32> %i32, %sw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext <4 x i8> %i8 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext <4 x i8> %i8 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = sub <4 x i32> %sl1_8_32, %sl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext <4 x i8> %i8 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = sub <4 x i32> %i32, %zw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext <4 x i8> %i8 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext <4 x i8> %i8 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = sub <4 x i32> %zl1_8_32, %zl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sw_8_64 = sext <4 x i8> %i8 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asw_8_64 = sub <4 x i64> %i64, %sw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sl1_8_64 = sext <4 x i8> %i8 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sl2_8_64 = sext <4 x i8> %i8 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asl_8_64 = sub <4 x i64> %sl1_8_64, %sl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zw_8_64 = zext <4 x i8> %i8 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azw_8_64 = sub <4 x i64> %i64, %zw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zl1_8_64 = zext <4 x i8> %i8 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zl2_8_64 = zext <4 x i8> %i8 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azl_8_64 = sub <4 x i64> %zl1_8_64, %zl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_16_32 = sext <4 x i16> %i16 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = sub <4 x i32> %i32, %sw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_16_32 = sext <4 x i16> %i16 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_16_32 = sext <4 x i16> %i16 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = sub <4 x i32> %sl1_16_32, %sl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_16_32 = zext <4 x i16> %i16 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = sub <4 x i32> %i32, %zw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_32 = zext <4 x i16> %i16 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_32 = zext <4 x i16> %i16 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = sub <4 x i32> %zl1_16_32, %zl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sw_16_64 = sext <4 x i16> %i16 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asw_16_64 = sub <4 x i64> %i64, %sw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sl1_16_64 = sext <4 x i16> %i16 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sl2_16_64 = sext <4 x i16> %i16 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asl_16_64 = sub <4 x i64> %sl1_16_64, %sl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zw_16_64 = zext <4 x i16> %i16 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azw_16_64 = sub <4 x i64> %i64, %zw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zl1_16_64 = zext <4 x i16> %i16 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zl2_16_64 = zext <4 x i16> %i16 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azl_16_64 = sub <4 x i64> %zl1_16_64, %zl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_32_64 = sext <4 x i32> %i32 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asw_32_64 = sub <4 x i64> %i64, %sw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <4 x i32> %i32 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <4 x i32> %i32 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asl_32_64 = sub <4 x i64> %sl1_32_64, %sl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext <4 x i32> %i32 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azw_32_64 = sub <4 x i64> %i64, %zw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <4 x i32> %i32 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <4 x i32> %i32 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azl_32_64 = sub <4 x i64> %zl1_32_64, %zl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'extsubv4'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext <4 x i8> %i8 to <4 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = sub <4 x i16> %i16, %sw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext <4 x i8> %i8 to <4 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_16 = sext <4 x i8> %i8 to <4 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = sub <4 x i16> %sl1_8_16, %sl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext <4 x i8> %i8 to <4 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = sub <4 x i16> %i16, %zw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_16 = zext <4 x i8> %i8 to <4 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_16 = zext <4 x i8> %i8 to <4 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = sub <4 x i16> %zl1_8_16, %zl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext <4 x i8> %i8 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = sub <4 x i32> %i32, %sw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext <4 x i8> %i8 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext <4 x i8> %i8 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = sub <4 x i32> %sl1_8_32, %sl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext <4 x i8> %i8 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = sub <4 x i32> %i32, %zw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext <4 x i8> %i8 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext <4 x i8> %i8 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = sub <4 x i32> %zl1_8_32, %zl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext <4 x i8> %i8 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = sub <4 x i64> %i64, %sw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext <4 x i8> %i8 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext <4 x i8> %i8 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = sub <4 x i64> %sl1_8_64, %sl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_64 = zext <4 x i8> %i8 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = sub <4 x i64> %i64, %zw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_64 = zext <4 x i8> %i8 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_64 = zext <4 x i8> %i8 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = sub <4 x i64> %zl1_8_64, %zl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_16_32 = sext <4 x i16> %i16 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = sub <4 x i32> %i32, %sw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_16_32 = sext <4 x i16> %i16 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_16_32 = sext <4 x i16> %i16 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = sub <4 x i32> %sl1_16_32, %sl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_16_32 = zext <4 x i16> %i16 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = sub <4 x i32> %i32, %zw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_32 = zext <4 x i16> %i16 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_32 = zext <4 x i16> %i16 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = sub <4 x i32> %zl1_16_32, %zl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext <4 x i16> %i16 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = sub <4 x i64> %i64, %sw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext <4 x i16> %i16 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext <4 x i16> %i16 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = sub <4 x i64> %sl1_16_64, %sl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_64 = zext <4 x i16> %i16 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = sub <4 x i64> %i64, %zw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_64 = zext <4 x i16> %i16 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_64 = zext <4 x i16> %i16 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = sub <4 x i64> %zl1_16_64, %zl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_32_64 = sext <4 x i32> %i32 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = sub <4 x i64> %i64, %sw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <4 x i32> %i32 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <4 x i32> %i32 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = sub <4 x i64> %sl1_32_64, %sl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext <4 x i32> %i32 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = sub <4 x i64> %i64, %zw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <4 x i32> %i32 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <4 x i32> %i32 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = sub <4 x i64> %zl1_32_64, %zl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'extsubv4'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_16 = sext <4 x i8> %i8 to <4 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_16 = sub <4 x i16> %i16, %sw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_16 = sext <4 x i8> %i8 to <4 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_16 = sext <4 x i8> %i8 to <4 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_16 = sub <4 x i16> %sl1_8_16, %sl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_16 = zext <4 x i8> %i8 to <4 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_16 = sub <4 x i16> %i16, %zw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_16 = zext <4 x i8> %i8 to <4 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_16 = zext <4 x i8> %i8 to <4 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_16 = sub <4 x i16> %zl1_8_16, %zl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_32 = sext <4 x i8> %i8 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_32 = sub <4 x i32> %i32, %sw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_32 = sext <4 x i8> %i8 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_32 = sext <4 x i8> %i8 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_32 = sub <4 x i32> %sl1_8_32, %sl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_32 = zext <4 x i8> %i8 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_32 = sub <4 x i32> %i32, %zw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_32 = zext <4 x i8> %i8 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_32 = zext <4 x i8> %i8 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_32 = sub <4 x i32> %zl1_8_32, %zl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sw_8_64 = sext <4 x i8> %i8 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_64 = sub <4 x i64> %i64, %sw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_8_64 = sext <4 x i8> %i8 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_8_64 = sext <4 x i8> %i8 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_64 = sub <4 x i64> %sl1_8_64, %sl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zw_8_64 = zext <4 x i8> %i8 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_64 = sub <4 x i64> %i64, %zw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_8_64 = zext <4 x i8> %i8 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_8_64 = zext <4 x i8> %i8 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_64 = sub <4 x i64> %zl1_8_64, %zl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sw_16_32 = sext <4 x i16> %i16 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_16_32 = sub <4 x i32> %i32, %sw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_16_32 = sext <4 x i16> %i16 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_16_32 = sext <4 x i16> %i16 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_16_32 = sub <4 x i32> %sl1_16_32, %sl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_16_32 = zext <4 x i16> %i16 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_16_32 = sub <4 x i32> %i32, %zw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_16_32 = zext <4 x i16> %i16 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_16_32 = zext <4 x i16> %i16 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_16_32 = sub <4 x i32> %zl1_16_32, %zl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sw_16_64 = sext <4 x i16> %i16 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asw_16_64 = sub <4 x i64> %i64, %sw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_16_64 = sext <4 x i16> %i16 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_16_64 = sext <4 x i16> %i16 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asl_16_64 = sub <4 x i64> %sl1_16_64, %sl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zw_16_64 = zext <4 x i16> %i16 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azw_16_64 = sub <4 x i64> %i64, %zw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_16_64 = zext <4 x i16> %i16 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_16_64 = zext <4 x i16> %i16 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azl_16_64 = sub <4 x i64> %zl1_16_64, %zl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sw_32_64 = sext <4 x i32> %i32 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asw_32_64 = sub <4 x i64> %i64, %sw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_32_64 = sext <4 x i32> %i32 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_32_64 = sext <4 x i32> %i32 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asl_32_64 = sub <4 x i64> %sl1_32_64, %sl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_32_64 = zext <4 x i32> %i32 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azw_32_64 = sub <4 x i64> %i64, %zw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_32_64 = zext <4 x i32> %i32 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_32_64 = zext <4 x i32> %i32 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azl_32_64 = sub <4 x i64> %zl1_32_64, %zl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %sw_8_16 = sext <4 x i8> %i8 to <4 x i16>
   %asw_8_16 = sub <4 x i16> %i16, %sw_8_16
@@ -1621,131 +1116,68 @@ define void @extsubv4(<4 x i8> %i8, <4 x i16> %i16, <4 x i32> %i32, <4 x i64> %i
 }
 
 define void @extsubv8(<8 x i8> %i8, <8 x i16> %i16, <8 x i32> %i32, <8 x i64> %i64)  {
-; RECIP-LABEL: 'extsubv8'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = sub <8 x i16> %i16, %sw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = sub <8 x i16> %sl1_8_16, %sl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = sub <8 x i16> %i16, %zw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = sub <8 x i16> %zl1_8_16, %zl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sw_8_32 = sext <8 x i8> %i8 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asw_8_32 = sub <8 x i32> %i32, %sw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sl1_8_32 = sext <8 x i8> %i8 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sl2_8_32 = sext <8 x i8> %i8 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asl_8_32 = sub <8 x i32> %sl1_8_32, %sl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zw_8_32 = zext <8 x i8> %i8 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azw_8_32 = sub <8 x i32> %i32, %zw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zl1_8_32 = zext <8 x i8> %i8 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zl2_8_32 = zext <8 x i8> %i8 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azl_8_32 = sub <8 x i32> %zl1_8_32, %zl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %sw_8_64 = sext <8 x i8> %i8 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asw_8_64 = sub <8 x i64> %i64, %sw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %sl1_8_64 = sext <8 x i8> %i8 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %sl2_8_64 = sext <8 x i8> %i8 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asl_8_64 = sub <8 x i64> %sl1_8_64, %sl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %zw_8_64 = zext <8 x i8> %i8 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azw_8_64 = sub <8 x i64> %i64, %zw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %zl1_8_64 = zext <8 x i8> %i8 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %zl2_8_64 = zext <8 x i8> %i8 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azl_8_64 = sub <8 x i64> %zl1_8_64, %zl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_16_32 = sext <8 x i16> %i16 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asw_16_32 = sub <8 x i32> %i32, %sw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_16_32 = sext <8 x i16> %i16 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_16_32 = sext <8 x i16> %i16 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asl_16_32 = sub <8 x i32> %sl1_16_32, %sl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_16_32 = zext <8 x i16> %i16 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azw_16_32 = sub <8 x i32> %i32, %zw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_32 = zext <8 x i16> %i16 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_32 = zext <8 x i16> %i16 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azl_16_32 = sub <8 x i32> %zl1_16_32, %zl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sw_16_64 = sext <8 x i16> %i16 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asw_16_64 = sub <8 x i64> %i64, %sw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sl1_16_64 = sext <8 x i16> %i16 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sl2_16_64 = sext <8 x i16> %i16 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asl_16_64 = sub <8 x i64> %sl1_16_64, %sl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %zw_16_64 = zext <8 x i16> %i16 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azw_16_64 = sub <8 x i64> %i64, %zw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %zl1_16_64 = zext <8 x i16> %i16 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %zl2_16_64 = zext <8 x i16> %i16 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azl_16_64 = sub <8 x i64> %zl1_16_64, %zl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_32_64 = sext <8 x i32> %i32 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asw_32_64 = sub <8 x i64> %i64, %sw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <8 x i32> %i32 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <8 x i32> %i32 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asl_32_64 = sub <8 x i64> %sl1_32_64, %sl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext <8 x i32> %i32 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azw_32_64 = sub <8 x i64> %i64, %zw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <8 x i32> %i32 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <8 x i32> %i32 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azl_32_64 = sub <8 x i64> %zl1_32_64, %zl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'extsubv8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = sub <8 x i16> %i16, %sw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = sub <8 x i16> %sl1_8_16, %sl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = sub <8 x i16> %i16, %zw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = sub <8 x i16> %zl1_8_16, %zl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext <8 x i8> %i8 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = sub <8 x i32> %i32, %sw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext <8 x i8> %i8 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext <8 x i8> %i8 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = sub <8 x i32> %sl1_8_32, %sl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext <8 x i8> %i8 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = sub <8 x i32> %i32, %zw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext <8 x i8> %i8 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext <8 x i8> %i8 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = sub <8 x i32> %zl1_8_32, %zl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext <8 x i8> %i8 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = sub <8 x i64> %i64, %sw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext <8 x i8> %i8 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext <8 x i8> %i8 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = sub <8 x i64> %sl1_8_64, %sl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_64 = zext <8 x i8> %i8 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = sub <8 x i64> %i64, %zw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_64 = zext <8 x i8> %i8 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_64 = zext <8 x i8> %i8 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = sub <8 x i64> %zl1_8_64, %zl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_16_32 = sext <8 x i16> %i16 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = sub <8 x i32> %i32, %sw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_16_32 = sext <8 x i16> %i16 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_16_32 = sext <8 x i16> %i16 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = sub <8 x i32> %sl1_16_32, %sl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_16_32 = zext <8 x i16> %i16 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = sub <8 x i32> %i32, %zw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_32 = zext <8 x i16> %i16 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_32 = zext <8 x i16> %i16 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = sub <8 x i32> %zl1_16_32, %zl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext <8 x i16> %i16 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = sub <8 x i64> %i64, %sw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext <8 x i16> %i16 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext <8 x i16> %i16 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = sub <8 x i64> %sl1_16_64, %sl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_64 = zext <8 x i16> %i16 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = sub <8 x i64> %i64, %zw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_64 = zext <8 x i16> %i16 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_64 = zext <8 x i16> %i16 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = sub <8 x i64> %zl1_16_64, %zl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_32_64 = sext <8 x i32> %i32 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = sub <8 x i64> %i64, %sw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <8 x i32> %i32 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <8 x i32> %i32 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = sub <8 x i64> %sl1_32_64, %sl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext <8 x i32> %i32 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = sub <8 x i64> %i64, %zw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <8 x i32> %i32 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <8 x i32> %i32 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = sub <8 x i64> %zl1_32_64, %zl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'extsubv8'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sw_8_16 = sext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_16 = sub <8 x i16> %i16, %sw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_8_16 = sext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_8_16 = sext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_16 = sub <8 x i16> %sl1_8_16, %sl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_8_16 = zext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_16 = sub <8 x i16> %i16, %zw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_8_16 = zext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_8_16 = zext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_16 = sub <8 x i16> %zl1_8_16, %zl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sw_8_32 = sext <8 x i8> %i8 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_32 = sub <8 x i32> %i32, %sw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_8_32 = sext <8 x i8> %i8 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_8_32 = sext <8 x i8> %i8 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_32 = sub <8 x i32> %sl1_8_32, %sl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zw_8_32 = zext <8 x i8> %i8 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_32 = sub <8 x i32> %i32, %zw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_8_32 = zext <8 x i8> %i8 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_8_32 = zext <8 x i8> %i8 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_32 = sub <8 x i32> %zl1_8_32, %zl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %sw_8_64 = sext <8 x i8> %i8 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_64 = sub <8 x i64> %i64, %sw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_8_64 = sext <8 x i8> %i8 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_8_64 = sext <8 x i8> %i8 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_64 = sub <8 x i64> %sl1_8_64, %sl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %zw_8_64 = zext <8 x i8> %i8 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_64 = sub <8 x i64> %i64, %zw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_8_64 = zext <8 x i8> %i8 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_8_64 = zext <8 x i8> %i8 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_64 = sub <8 x i64> %zl1_8_64, %zl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sw_16_32 = sext <8 x i16> %i16 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asw_16_32 = sub <8 x i32> %i32, %sw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_16_32 = sext <8 x i16> %i16 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_16_32 = sext <8 x i16> %i16 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asl_16_32 = sub <8 x i32> %sl1_16_32, %sl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_16_32 = zext <8 x i16> %i16 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azw_16_32 = sub <8 x i32> %i32, %zw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_16_32 = zext <8 x i16> %i16 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_16_32 = zext <8 x i16> %i16 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azl_16_32 = sub <8 x i32> %zl1_16_32, %zl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %sw_16_64 = sext <8 x i16> %i16 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asw_16_64 = sub <8 x i64> %i64, %sw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_16_64 = sext <8 x i16> %i16 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_16_64 = sext <8 x i16> %i16 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asl_16_64 = sub <8 x i64> %sl1_16_64, %sl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %zw_16_64 = zext <8 x i16> %i16 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azw_16_64 = sub <8 x i64> %i64, %zw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_16_64 = zext <8 x i16> %i16 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_16_64 = zext <8 x i16> %i16 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azl_16_64 = sub <8 x i64> %zl1_16_64, %zl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sw_32_64 = sext <8 x i32> %i32 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asw_32_64 = sub <8 x i64> %i64, %sw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_32_64 = sext <8 x i32> %i32 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_32_64 = sext <8 x i32> %i32 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asl_32_64 = sub <8 x i64> %sl1_32_64, %sl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_32_64 = zext <8 x i32> %i32 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azw_32_64 = sub <8 x i64> %i64, %zw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_32_64 = zext <8 x i32> %i32 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_32_64 = zext <8 x i32> %i32 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azl_32_64 = sub <8 x i64> %zl1_32_64, %zl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %sw_8_16 = sext <8 x i8> %i8 to <8 x i16>
   %asw_8_16 = sub <8 x i16> %i16, %sw_8_16
@@ -1823,131 +1255,68 @@ define void @extsubv8(<8 x i8> %i8, <8 x i16> %i16, <8 x i32> %i32, <8 x i64> %i
 }
 
 define void @extsubv16(<16 x i8> %i8, <16 x i16> %i16, <16 x i32> %i32, <16 x i64> %i64)  {
-; RECIP-LABEL: 'extsubv16'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_8_16 = sext <16 x i8> %i8 to <16 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asw_8_16 = sub <16 x i16> %i16, %sw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_8_16 = sext <16 x i8> %i8 to <16 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_8_16 = sext <16 x i8> %i8 to <16 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asl_8_16 = sub <16 x i16> %sl1_8_16, %sl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_8_16 = zext <16 x i8> %i8 to <16 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azw_8_16 = sub <16 x i16> %i16, %zw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_16 = zext <16 x i8> %i8 to <16 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_8_16 = zext <16 x i8> %i8 to <16 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azl_8_16 = sub <16 x i16> %zl1_8_16, %zl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sw_8_32 = sext <16 x i8> %i8 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asw_8_32 = sub <16 x i32> %i32, %sw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sl1_8_32 = sext <16 x i8> %i8 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sl2_8_32 = sext <16 x i8> %i8 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asl_8_32 = sub <16 x i32> %sl1_8_32, %sl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %zw_8_32 = zext <16 x i8> %i8 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azw_8_32 = sub <16 x i32> %i32, %zw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %zl1_8_32 = zext <16 x i8> %i8 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %zl2_8_32 = zext <16 x i8> %i8 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azl_8_32 = sub <16 x i32> %zl1_8_32, %zl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %sw_8_64 = sext <16 x i8> %i8 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %asw_8_64 = sub <16 x i64> %i64, %sw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %sl1_8_64 = sext <16 x i8> %i8 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %sl2_8_64 = sext <16 x i8> %i8 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %asl_8_64 = sub <16 x i64> %sl1_8_64, %sl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %zw_8_64 = zext <16 x i8> %i8 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %azw_8_64 = sub <16 x i64> %i64, %zw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %zl1_8_64 = zext <16 x i8> %i8 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %zl2_8_64 = zext <16 x i8> %i8 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %azl_8_64 = sub <16 x i64> %zl1_8_64, %zl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_16_32 = sext <16 x i16> %i16 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asw_16_32 = sub <16 x i32> %i32, %sw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_16_32 = sext <16 x i16> %i16 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_16_32 = sext <16 x i16> %i16 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asl_16_32 = sub <16 x i32> %sl1_16_32, %sl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_16_32 = zext <16 x i16> %i16 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azw_16_32 = sub <16 x i32> %i32, %zw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_32 = zext <16 x i16> %i16 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_32 = zext <16 x i16> %i16 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azl_16_32 = sub <16 x i32> %zl1_16_32, %zl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %sw_16_64 = sext <16 x i16> %i16 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %asw_16_64 = sub <16 x i64> %i64, %sw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %sl1_16_64 = sext <16 x i16> %i16 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %sl2_16_64 = sext <16 x i16> %i16 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %asl_16_64 = sub <16 x i64> %sl1_16_64, %sl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %zw_16_64 = zext <16 x i16> %i16 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %azw_16_64 = sub <16 x i64> %i64, %zw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %zl1_16_64 = zext <16 x i16> %i16 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %zl2_16_64 = zext <16 x i16> %i16 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %azl_16_64 = sub <16 x i64> %zl1_16_64, %zl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_32_64 = sext <16 x i32> %i32 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %asw_32_64 = sub <16 x i64> %i64, %sw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <16 x i32> %i32 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <16 x i32> %i32 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %asl_32_64 = sub <16 x i64> %sl1_32_64, %sl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext <16 x i32> %i32 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %azw_32_64 = sub <16 x i64> %i64, %zw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <16 x i32> %i32 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <16 x i32> %i32 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %azl_32_64 = sub <16 x i64> %zl1_32_64, %zl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'extsubv16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_8_16 = sext <16 x i8> %i8 to <16 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = sub <16 x i16> %i16, %sw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_8_16 = sext <16 x i8> %i8 to <16 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_8_16 = sext <16 x i8> %i8 to <16 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = sub <16 x i16> %sl1_8_16, %sl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_8_16 = zext <16 x i8> %i8 to <16 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = sub <16 x i16> %i16, %zw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_16 = zext <16 x i8> %i8 to <16 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_8_16 = zext <16 x i8> %i8 to <16 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = sub <16 x i16> %zl1_8_16, %zl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext <16 x i8> %i8 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = sub <16 x i32> %i32, %sw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext <16 x i8> %i8 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext <16 x i8> %i8 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = sub <16 x i32> %sl1_8_32, %sl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext <16 x i8> %i8 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = sub <16 x i32> %i32, %zw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext <16 x i8> %i8 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext <16 x i8> %i8 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = sub <16 x i32> %zl1_8_32, %zl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext <16 x i8> %i8 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = sub <16 x i64> %i64, %sw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext <16 x i8> %i8 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext <16 x i8> %i8 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = sub <16 x i64> %sl1_8_64, %sl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_64 = zext <16 x i8> %i8 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = sub <16 x i64> %i64, %zw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_64 = zext <16 x i8> %i8 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_64 = zext <16 x i8> %i8 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = sub <16 x i64> %zl1_8_64, %zl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_16_32 = sext <16 x i16> %i16 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = sub <16 x i32> %i32, %sw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_16_32 = sext <16 x i16> %i16 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_16_32 = sext <16 x i16> %i16 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = sub <16 x i32> %sl1_16_32, %sl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_16_32 = zext <16 x i16> %i16 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = sub <16 x i32> %i32, %zw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_32 = zext <16 x i16> %i16 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_32 = zext <16 x i16> %i16 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = sub <16 x i32> %zl1_16_32, %zl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext <16 x i16> %i16 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = sub <16 x i64> %i64, %sw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext <16 x i16> %i16 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext <16 x i16> %i16 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = sub <16 x i64> %sl1_16_64, %sl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_64 = zext <16 x i16> %i16 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = sub <16 x i64> %i64, %zw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_64 = zext <16 x i16> %i16 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_64 = zext <16 x i16> %i16 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = sub <16 x i64> %zl1_16_64, %zl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sw_32_64 = sext <16 x i32> %i32 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = sub <16 x i64> %i64, %sw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <16 x i32> %i32 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <16 x i32> %i32 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = sub <16 x i64> %sl1_32_64, %sl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext <16 x i32> %i32 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = sub <16 x i64> %i64, %zw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <16 x i32> %i32 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <16 x i32> %i32 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = sub <16 x i64> %zl1_32_64, %zl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'extsubv16'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sw_8_16 = sext <16 x i8> %i8 to <16 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_16 = sub <16 x i16> %i16, %sw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_8_16 = sext <16 x i8> %i8 to <16 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_8_16 = sext <16 x i8> %i8 to <16 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_16 = sub <16 x i16> %sl1_8_16, %sl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_8_16 = zext <16 x i8> %i8 to <16 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_16 = sub <16 x i16> %i16, %zw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_8_16 = zext <16 x i8> %i8 to <16 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_8_16 = zext <16 x i8> %i8 to <16 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_16 = sub <16 x i16> %zl1_8_16, %zl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %sw_8_32 = sext <16 x i8> %i8 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_32 = sub <16 x i32> %i32, %sw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_8_32 = sext <16 x i8> %i8 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_8_32 = sext <16 x i8> %i8 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_32 = sub <16 x i32> %sl1_8_32, %sl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %zw_8_32 = zext <16 x i8> %i8 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_32 = sub <16 x i32> %i32, %zw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_8_32 = zext <16 x i8> %i8 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_8_32 = zext <16 x i8> %i8 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_32 = sub <16 x i32> %zl1_8_32, %zl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:15 CodeSize:1 Lat:1 SizeLat:1 for: %sw_8_64 = sext <16 x i8> %i8 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_64 = sub <16 x i64> %i64, %sw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:15 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_8_64 = sext <16 x i8> %i8 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:15 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_8_64 = sext <16 x i8> %i8 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_64 = sub <16 x i64> %sl1_8_64, %sl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:15 CodeSize:1 Lat:1 SizeLat:1 for: %zw_8_64 = zext <16 x i8> %i8 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_64 = sub <16 x i64> %i64, %zw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:15 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_8_64 = zext <16 x i8> %i8 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:15 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_8_64 = zext <16 x i8> %i8 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_64 = sub <16 x i64> %zl1_8_64, %zl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sw_16_32 = sext <16 x i16> %i16 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asw_16_32 = sub <16 x i32> %i32, %sw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_16_32 = sext <16 x i16> %i16 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_16_32 = sext <16 x i16> %i16 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asl_16_32 = sub <16 x i32> %sl1_16_32, %sl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_16_32 = zext <16 x i16> %i16 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azw_16_32 = sub <16 x i32> %i32, %zw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_16_32 = zext <16 x i16> %i16 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_16_32 = zext <16 x i16> %i16 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azl_16_32 = sub <16 x i32> %zl1_16_32, %zl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: %sw_16_64 = sext <16 x i16> %i16 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %asw_16_64 = sub <16 x i64> %i64, %sw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_16_64 = sext <16 x i16> %i16 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_16_64 = sext <16 x i16> %i16 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %asl_16_64 = sub <16 x i64> %sl1_16_64, %sl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: %zw_16_64 = zext <16 x i16> %i16 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %azw_16_64 = sub <16 x i64> %i64, %zw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_16_64 = zext <16 x i16> %i16 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_16_64 = zext <16 x i16> %i16 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %azl_16_64 = sub <16 x i64> %zl1_16_64, %zl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sw_32_64 = sext <16 x i32> %i32 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %asw_32_64 = sub <16 x i64> %i64, %sw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_32_64 = sext <16 x i32> %i32 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_32_64 = sext <16 x i32> %i32 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %asl_32_64 = sub <16 x i64> %sl1_32_64, %sl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_32_64 = zext <16 x i32> %i32 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %azw_32_64 = sub <16 x i64> %i64, %zw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_32_64 = zext <16 x i32> %i32 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_32_64 = zext <16 x i32> %i32 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %azl_32_64 = sub <16 x i64> %zl1_32_64, %zl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %sw_8_16 = sext <16 x i8> %i8 to <16 x i16>
   %asw_8_16 = sub <16 x i16> %i16, %sw_8_16
@@ -2025,131 +1394,68 @@ define void @extsubv16(<16 x i8> %i8, <16 x i16> %i16, <16 x i32> %i32, <16 x i6
 }
 
 define void @extmulv1(i8 %i8, i16 %i16, i32 %i32, i64 %i64)  {
-; RECIP-LABEL: 'extmulv1'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext i8 %i8 to i16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = mul i16 %i16, %sw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext i8 %i8 to i16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_16 = sext i8 %i8 to i16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = mul i16 %sl1_8_16, %sl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext i8 %i8 to i16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = mul i16 %i16, %zw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_16 = zext i8 %i8 to i16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_16 = zext i8 %i8 to i16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = mul i16 %zl1_8_16, %zl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext i8 %i8 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = mul i32 %i32, %sw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext i8 %i8 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext i8 %i8 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = mul i32 %sl1_8_32, %sl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext i8 %i8 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = mul i32 %i32, %zw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext i8 %i8 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext i8 %i8 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = mul i32 %zl1_8_32, %zl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext i8 %i8 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = mul i64 %i64, %sw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext i8 %i8 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext i8 %i8 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = mul i64 %sl1_8_64, %sl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_8_64 = zext i8 %i8 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = mul i64 %i64, %zw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_64 = zext i8 %i8 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_8_64 = zext i8 %i8 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = mul i64 %zl1_8_64, %zl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_32 = sext i16 %i16 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = mul i32 %i32, %sw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_32 = sext i16 %i16 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_32 = sext i16 %i16 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = mul i32 %sl1_16_32, %sl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_32 = zext i16 %i16 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = mul i32 %i32, %zw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_32 = zext i16 %i16 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_32 = zext i16 %i16 to i32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = mul i32 %zl1_16_32, %zl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext i16 %i16 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = mul i64 %i64, %sw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext i16 %i16 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext i16 %i16 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = mul i64 %sl1_16_64, %sl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_16_64 = zext i16 %i16 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = mul i64 %i64, %zw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_64 = zext i16 %i16 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_64 = zext i16 %i16 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = mul i64 %zl1_16_64, %zl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_32_64 = sext i32 %i32 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = mul i64 %i64, %sw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_32_64 = sext i32 %i32 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_32_64 = sext i32 %i32 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = mul i64 %sl1_32_64, %sl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext i32 %i32 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = mul i64 %i64, %zw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext i32 %i32 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext i32 %i32 to i64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = mul i64 %zl1_32_64, %zl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'extmulv1'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext i8 %i8 to i16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = mul i16 %i16, %sw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext i8 %i8 to i16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_16 = sext i8 %i8 to i16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = mul i16 %sl1_8_16, %sl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext i8 %i8 to i16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = mul i16 %i16, %zw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_16 = zext i8 %i8 to i16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_16 = zext i8 %i8 to i16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = mul i16 %zl1_8_16, %zl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext i8 %i8 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = mul i32 %i32, %sw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext i8 %i8 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext i8 %i8 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = mul i32 %sl1_8_32, %sl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext i8 %i8 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = mul i32 %i32, %zw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext i8 %i8 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext i8 %i8 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = mul i32 %zl1_8_32, %zl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext i8 %i8 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = mul i64 %i64, %sw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext i8 %i8 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext i8 %i8 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = mul i64 %sl1_8_64, %sl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_8_64 = zext i8 %i8 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = mul i64 %i64, %zw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_64 = zext i8 %i8 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_8_64 = zext i8 %i8 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = mul i64 %zl1_8_64, %zl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_32 = sext i16 %i16 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = mul i32 %i32, %sw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_32 = sext i16 %i16 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_32 = sext i16 %i16 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = mul i32 %sl1_16_32, %sl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_32 = zext i16 %i16 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = mul i32 %i32, %zw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_32 = zext i16 %i16 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_32 = zext i16 %i16 to i32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = mul i32 %zl1_16_32, %zl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext i16 %i16 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = mul i64 %i64, %sw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext i16 %i16 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext i16 %i16 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = mul i64 %sl1_16_64, %sl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_16_64 = zext i16 %i16 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = mul i64 %i64, %zw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_64 = zext i16 %i16 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_64 = zext i16 %i16 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = mul i64 %zl1_16_64, %zl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_32_64 = sext i32 %i32 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = mul i64 %i64, %sw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_32_64 = sext i32 %i32 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_32_64 = sext i32 %i32 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = mul i64 %sl1_32_64, %sl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zw_32_64 = zext i32 %i32 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = mul i64 %i64, %zw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext i32 %i32 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext i32 %i32 to i64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = mul i64 %zl1_32_64, %zl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'extmulv1'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_16 = sext i8 %i8 to i16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_16 = mul i16 %i16, %sw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_16 = sext i8 %i8 to i16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_16 = sext i8 %i8 to i16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_16 = mul i16 %sl1_8_16, %sl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_16 = zext i8 %i8 to i16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_16 = mul i16 %i16, %zw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_16 = zext i8 %i8 to i16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_16 = zext i8 %i8 to i16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_16 = mul i16 %zl1_8_16, %zl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_32 = sext i8 %i8 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_32 = mul i32 %i32, %sw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_32 = sext i8 %i8 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_32 = sext i8 %i8 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_32 = mul i32 %sl1_8_32, %sl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_32 = zext i8 %i8 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_32 = mul i32 %i32, %zw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_32 = zext i8 %i8 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_32 = zext i8 %i8 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_32 = mul i32 %zl1_8_32, %zl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_64 = sext i8 %i8 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_64 = mul i64 %i64, %sw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_64 = sext i8 %i8 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_64 = sext i8 %i8 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_64 = mul i64 %sl1_8_64, %sl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_8_64 = zext i8 %i8 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_64 = mul i64 %i64, %zw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_8_64 = zext i8 %i8 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_8_64 = zext i8 %i8 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_64 = mul i64 %zl1_8_64, %zl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_16_32 = sext i16 %i16 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_16_32 = mul i32 %i32, %sw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_16_32 = sext i16 %i16 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_16_32 = sext i16 %i16 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_16_32 = mul i32 %sl1_16_32, %sl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_16_32 = zext i16 %i16 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_16_32 = mul i32 %i32, %zw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_16_32 = zext i16 %i16 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_16_32 = zext i16 %i16 to i32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_16_32 = mul i32 %zl1_16_32, %zl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_16_64 = sext i16 %i16 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_16_64 = mul i64 %i64, %sw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_16_64 = sext i16 %i16 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_16_64 = sext i16 %i16 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_16_64 = mul i64 %sl1_16_64, %sl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_16_64 = zext i16 %i16 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_16_64 = mul i64 %i64, %zw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_16_64 = zext i16 %i16 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_16_64 = zext i16 %i16 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_16_64 = mul i64 %zl1_16_64, %zl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_32_64 = sext i32 %i32 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_32_64 = mul i64 %i64, %sw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_32_64 = sext i32 %i32 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_32_64 = sext i32 %i32 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_32_64 = mul i64 %sl1_32_64, %sl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zw_32_64 = zext i32 %i32 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_32_64 = mul i64 %i64, %zw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_32_64 = zext i32 %i32 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_32_64 = zext i32 %i32 to i64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_32_64 = mul i64 %zl1_32_64, %zl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %sw_8_16 = sext i8 %i8 to i16
   %asw_8_16 = mul i16 %i16, %sw_8_16
@@ -2227,131 +1533,68 @@ define void @extmulv1(i8 %i8, i16 %i16, i32 %i32, i64 %i64)  {
 }
 
 define void @extmulv2(<2 x i8> %i8, <2 x i16> %i16, <2 x i32> %i32, <2 x i64> %i64)  {
-; RECIP-LABEL: 'extmulv2'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext <2 x i8> %i8 to <2 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = mul <2 x i16> %i16, %sw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext <2 x i8> %i8 to <2 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_16 = sext <2 x i8> %i8 to <2 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = mul <2 x i16> %sl1_8_16, %sl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext <2 x i8> %i8 to <2 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = mul <2 x i16> %i16, %zw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_16 = zext <2 x i8> %i8 to <2 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_16 = zext <2 x i8> %i8 to <2 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = mul <2 x i16> %zl1_8_16, %zl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext <2 x i8> %i8 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = mul <2 x i32> %i32, %sw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext <2 x i8> %i8 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext <2 x i8> %i8 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = mul <2 x i32> %sl1_8_32, %sl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext <2 x i8> %i8 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = mul <2 x i32> %i32, %zw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext <2 x i8> %i8 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext <2 x i8> %i8 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = mul <2 x i32> %zl1_8_32, %zl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext <2 x i8> %i8 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %asw_8_64 = mul <2 x i64> %i64, %sw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext <2 x i8> %i8 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext <2 x i8> %i8 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %asl_8_64 = mul <2 x i64> %sl1_8_64, %sl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_64 = zext <2 x i8> %i8 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %azw_8_64 = mul <2 x i64> %i64, %zw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_64 = zext <2 x i8> %i8 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_64 = zext <2 x i8> %i8 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %azl_8_64 = mul <2 x i64> %zl1_8_64, %zl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_32 = sext <2 x i16> %i16 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = mul <2 x i32> %i32, %sw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_32 = sext <2 x i16> %i16 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_32 = sext <2 x i16> %i16 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = mul <2 x i32> %sl1_16_32, %sl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_32 = zext <2 x i16> %i16 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = mul <2 x i32> %i32, %zw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_32 = zext <2 x i16> %i16 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_32 = zext <2 x i16> %i16 to <2 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = mul <2 x i32> %zl1_16_32, %zl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext <2 x i16> %i16 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %asw_16_64 = mul <2 x i64> %i64, %sw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext <2 x i16> %i16 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext <2 x i16> %i16 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %asl_16_64 = mul <2 x i64> %sl1_16_64, %sl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_64 = zext <2 x i16> %i16 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %azw_16_64 = mul <2 x i64> %i64, %zw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_64 = zext <2 x i16> %i16 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_64 = zext <2 x i16> %i16 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %azl_16_64 = mul <2 x i64> %zl1_16_64, %zl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_32_64 = sext <2 x i32> %i32 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %asw_32_64 = mul <2 x i64> %i64, %sw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <2 x i32> %i32 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <2 x i32> %i32 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = mul <2 x i64> %sl1_32_64, %sl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_32_64 = zext <2 x i32> %i32 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %azw_32_64 = mul <2 x i64> %i64, %zw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <2 x i32> %i32 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <2 x i32> %i32 to <2 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = mul <2 x i64> %zl1_32_64, %zl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'extmulv2'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext <2 x i8> %i8 to <2 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = mul <2 x i16> %i16, %sw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext <2 x i8> %i8 to <2 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_16 = sext <2 x i8> %i8 to <2 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = mul <2 x i16> %sl1_8_16, %sl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext <2 x i8> %i8 to <2 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = mul <2 x i16> %i16, %zw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_16 = zext <2 x i8> %i8 to <2 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_16 = zext <2 x i8> %i8 to <2 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = mul <2 x i16> %zl1_8_16, %zl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext <2 x i8> %i8 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = mul <2 x i32> %i32, %sw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext <2 x i8> %i8 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext <2 x i8> %i8 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = mul <2 x i32> %sl1_8_32, %sl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext <2 x i8> %i8 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = mul <2 x i32> %i32, %zw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext <2 x i8> %i8 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext <2 x i8> %i8 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = mul <2 x i32> %zl1_8_32, %zl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext <2 x i8> %i8 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = mul <2 x i64> %i64, %sw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext <2 x i8> %i8 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext <2 x i8> %i8 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = mul <2 x i64> %sl1_8_64, %sl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_64 = zext <2 x i8> %i8 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = mul <2 x i64> %i64, %zw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_64 = zext <2 x i8> %i8 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_64 = zext <2 x i8> %i8 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = mul <2 x i64> %zl1_8_64, %zl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_32 = sext <2 x i16> %i16 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = mul <2 x i32> %i32, %sw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_32 = sext <2 x i16> %i16 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_32 = sext <2 x i16> %i16 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = mul <2 x i32> %sl1_16_32, %sl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_32 = zext <2 x i16> %i16 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = mul <2 x i32> %i32, %zw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_32 = zext <2 x i16> %i16 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_32 = zext <2 x i16> %i16 to <2 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = mul <2 x i32> %zl1_16_32, %zl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext <2 x i16> %i16 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = mul <2 x i64> %i64, %sw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext <2 x i16> %i16 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext <2 x i16> %i16 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = mul <2 x i64> %sl1_16_64, %sl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_64 = zext <2 x i16> %i16 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = mul <2 x i64> %i64, %zw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_64 = zext <2 x i16> %i16 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_64 = zext <2 x i16> %i16 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = mul <2 x i64> %zl1_16_64, %zl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_32_64 = sext <2 x i32> %i32 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = mul <2 x i64> %i64, %sw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <2 x i32> %i32 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <2 x i32> %i32 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = mul <2 x i64> %sl1_32_64, %sl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_32_64 = zext <2 x i32> %i32 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = mul <2 x i64> %i64, %zw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <2 x i32> %i32 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <2 x i32> %i32 to <2 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = mul <2 x i64> %zl1_32_64, %zl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'extmulv2'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_16 = sext <2 x i8> %i8 to <2 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_16 = mul <2 x i16> %i16, %sw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_16 = sext <2 x i8> %i8 to <2 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_16 = sext <2 x i8> %i8 to <2 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_16 = mul <2 x i16> %sl1_8_16, %sl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_16 = zext <2 x i8> %i8 to <2 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_16 = mul <2 x i16> %i16, %zw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_16 = zext <2 x i8> %i8 to <2 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_16 = zext <2 x i8> %i8 to <2 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_16 = mul <2 x i16> %zl1_8_16, %zl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_32 = sext <2 x i8> %i8 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_32 = mul <2 x i32> %i32, %sw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_32 = sext <2 x i8> %i8 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_32 = sext <2 x i8> %i8 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_32 = mul <2 x i32> %sl1_8_32, %sl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_32 = zext <2 x i8> %i8 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_32 = mul <2 x i32> %i32, %zw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_32 = zext <2 x i8> %i8 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_32 = zext <2 x i8> %i8 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_32 = mul <2 x i32> %zl1_8_32, %zl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_64 = sext <2 x i8> %i8 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_64 = mul <2 x i64> %i64, %sw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_64 = sext <2 x i8> %i8 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_64 = sext <2 x i8> %i8 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_64 = mul <2 x i64> %sl1_8_64, %sl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_64 = zext <2 x i8> %i8 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_64 = mul <2 x i64> %i64, %zw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_64 = zext <2 x i8> %i8 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_64 = zext <2 x i8> %i8 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_64 = mul <2 x i64> %zl1_8_64, %zl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_16_32 = sext <2 x i16> %i16 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_16_32 = mul <2 x i32> %i32, %sw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_16_32 = sext <2 x i16> %i16 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_16_32 = sext <2 x i16> %i16 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_16_32 = mul <2 x i32> %sl1_16_32, %sl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_16_32 = zext <2 x i16> %i16 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_16_32 = mul <2 x i32> %i32, %zw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_16_32 = zext <2 x i16> %i16 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_16_32 = zext <2 x i16> %i16 to <2 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_16_32 = mul <2 x i32> %zl1_16_32, %zl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_16_64 = sext <2 x i16> %i16 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:1 Lat:1 SizeLat:1 for: %asw_16_64 = mul <2 x i64> %i64, %sw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_16_64 = sext <2 x i16> %i16 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_16_64 = sext <2 x i16> %i16 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:1 Lat:1 SizeLat:1 for: %asl_16_64 = mul <2 x i64> %sl1_16_64, %sl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_16_64 = zext <2 x i16> %i16 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:1 Lat:1 SizeLat:1 for: %azw_16_64 = mul <2 x i64> %i64, %zw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_16_64 = zext <2 x i16> %i16 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_16_64 = zext <2 x i16> %i16 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:1 Lat:1 SizeLat:1 for: %azl_16_64 = mul <2 x i64> %zl1_16_64, %zl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_32_64 = sext <2 x i32> %i32 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:1 Lat:1 SizeLat:1 for: %asw_32_64 = mul <2 x i64> %i64, %sw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_32_64 = sext <2 x i32> %i32 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_32_64 = sext <2 x i32> %i32 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_32_64 = mul <2 x i64> %sl1_32_64, %sl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_32_64 = zext <2 x i32> %i32 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:1 Lat:1 SizeLat:1 for: %azw_32_64 = mul <2 x i64> %i64, %zw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_32_64 = zext <2 x i32> %i32 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_32_64 = zext <2 x i32> %i32 to <2 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_32_64 = mul <2 x i64> %zl1_32_64, %zl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %sw_8_16 = sext <2 x i8> %i8 to <2 x i16>
   %asw_8_16 = mul <2 x i16> %i16, %sw_8_16
@@ -2429,131 +1672,68 @@ define void @extmulv2(<2 x i8> %i8, <2 x i16> %i16, <2 x i32> %i32, <2 x i64> %i
 }
 
 define void @extmulv4(<4 x i8> %i8, <4 x i16> %i16, <4 x i32> %i32, <4 x i64> %i64)  {
-; RECIP-LABEL: 'extmulv4'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext <4 x i8> %i8 to <4 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = mul <4 x i16> %i16, %sw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext <4 x i8> %i8 to <4 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_16 = sext <4 x i8> %i8 to <4 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = mul <4 x i16> %sl1_8_16, %sl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext <4 x i8> %i8 to <4 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = mul <4 x i16> %i16, %zw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_16 = zext <4 x i8> %i8 to <4 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_16 = zext <4 x i8> %i8 to <4 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = mul <4 x i16> %zl1_8_16, %zl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext <4 x i8> %i8 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = mul <4 x i32> %i32, %sw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext <4 x i8> %i8 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext <4 x i8> %i8 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = mul <4 x i32> %sl1_8_32, %sl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext <4 x i8> %i8 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = mul <4 x i32> %i32, %zw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext <4 x i8> %i8 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext <4 x i8> %i8 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = mul <4 x i32> %zl1_8_32, %zl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sw_8_64 = sext <4 x i8> %i8 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %asw_8_64 = mul <4 x i64> %i64, %sw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sl1_8_64 = sext <4 x i8> %i8 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sl2_8_64 = sext <4 x i8> %i8 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %asl_8_64 = mul <4 x i64> %sl1_8_64, %sl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zw_8_64 = zext <4 x i8> %i8 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %azw_8_64 = mul <4 x i64> %i64, %zw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zl1_8_64 = zext <4 x i8> %i8 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zl2_8_64 = zext <4 x i8> %i8 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %azl_8_64 = mul <4 x i64> %zl1_8_64, %zl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_32 = sext <4 x i16> %i16 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = mul <4 x i32> %i32, %sw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_16_32 = sext <4 x i16> %i16 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_16_32 = sext <4 x i16> %i16 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = mul <4 x i32> %sl1_16_32, %sl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_32 = zext <4 x i16> %i16 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = mul <4 x i32> %i32, %zw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_32 = zext <4 x i16> %i16 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_32 = zext <4 x i16> %i16 to <4 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = mul <4 x i32> %zl1_16_32, %zl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sw_16_64 = sext <4 x i16> %i16 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %asw_16_64 = mul <4 x i64> %i64, %sw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sl1_16_64 = sext <4 x i16> %i16 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sl2_16_64 = sext <4 x i16> %i16 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %asl_16_64 = mul <4 x i64> %sl1_16_64, %sl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zw_16_64 = zext <4 x i16> %i16 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %azw_16_64 = mul <4 x i64> %i64, %zw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zl1_16_64 = zext <4 x i16> %i16 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zl2_16_64 = zext <4 x i16> %i16 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %azl_16_64 = mul <4 x i64> %zl1_16_64, %zl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sw_32_64 = sext <4 x i32> %i32 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %asw_32_64 = mul <4 x i64> %i64, %sw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <4 x i32> %i32 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <4 x i32> %i32 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asl_32_64 = mul <4 x i64> %sl1_32_64, %sl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %zw_32_64 = zext <4 x i32> %i32 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %azw_32_64 = mul <4 x i64> %i64, %zw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <4 x i32> %i32 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <4 x i32> %i32 to <4 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azl_32_64 = mul <4 x i64> %zl1_32_64, %zl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'extmulv4'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext <4 x i8> %i8 to <4 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = mul <4 x i16> %i16, %sw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext <4 x i8> %i8 to <4 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_16 = sext <4 x i8> %i8 to <4 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = mul <4 x i16> %sl1_8_16, %sl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext <4 x i8> %i8 to <4 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = mul <4 x i16> %i16, %zw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_16 = zext <4 x i8> %i8 to <4 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_16 = zext <4 x i8> %i8 to <4 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = mul <4 x i16> %zl1_8_16, %zl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext <4 x i8> %i8 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = mul <4 x i32> %i32, %sw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext <4 x i8> %i8 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext <4 x i8> %i8 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = mul <4 x i32> %sl1_8_32, %sl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext <4 x i8> %i8 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = mul <4 x i32> %i32, %zw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext <4 x i8> %i8 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext <4 x i8> %i8 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = mul <4 x i32> %zl1_8_32, %zl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext <4 x i8> %i8 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = mul <4 x i64> %i64, %sw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext <4 x i8> %i8 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext <4 x i8> %i8 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = mul <4 x i64> %sl1_8_64, %sl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_64 = zext <4 x i8> %i8 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = mul <4 x i64> %i64, %zw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_64 = zext <4 x i8> %i8 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_64 = zext <4 x i8> %i8 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = mul <4 x i64> %zl1_8_64, %zl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_32 = sext <4 x i16> %i16 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = mul <4 x i32> %i32, %sw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_16_32 = sext <4 x i16> %i16 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_16_32 = sext <4 x i16> %i16 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = mul <4 x i32> %sl1_16_32, %sl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_32 = zext <4 x i16> %i16 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = mul <4 x i32> %i32, %zw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_32 = zext <4 x i16> %i16 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_32 = zext <4 x i16> %i16 to <4 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = mul <4 x i32> %zl1_16_32, %zl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext <4 x i16> %i16 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = mul <4 x i64> %i64, %sw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext <4 x i16> %i16 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext <4 x i16> %i16 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = mul <4 x i64> %sl1_16_64, %sl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_64 = zext <4 x i16> %i16 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = mul <4 x i64> %i64, %zw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_64 = zext <4 x i16> %i16 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_64 = zext <4 x i16> %i16 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = mul <4 x i64> %zl1_16_64, %zl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_32_64 = sext <4 x i32> %i32 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = mul <4 x i64> %i64, %sw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <4 x i32> %i32 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <4 x i32> %i32 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = mul <4 x i64> %sl1_32_64, %sl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_32_64 = zext <4 x i32> %i32 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = mul <4 x i64> %i64, %zw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <4 x i32> %i32 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <4 x i32> %i32 to <4 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = mul <4 x i64> %zl1_32_64, %zl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'extmulv4'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_16 = sext <4 x i8> %i8 to <4 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_16 = mul <4 x i16> %i16, %sw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_16 = sext <4 x i8> %i8 to <4 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_16 = sext <4 x i8> %i8 to <4 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_16 = mul <4 x i16> %sl1_8_16, %sl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_16 = zext <4 x i8> %i8 to <4 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_16 = mul <4 x i16> %i16, %zw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_16 = zext <4 x i8> %i8 to <4 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_16 = zext <4 x i8> %i8 to <4 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_16 = mul <4 x i16> %zl1_8_16, %zl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_32 = sext <4 x i8> %i8 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_32 = mul <4 x i32> %i32, %sw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_32 = sext <4 x i8> %i8 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl2_8_32 = sext <4 x i8> %i8 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_32 = mul <4 x i32> %sl1_8_32, %sl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_32 = zext <4 x i8> %i8 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_32 = mul <4 x i32> %i32, %zw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl1_8_32 = zext <4 x i8> %i8 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zl2_8_32 = zext <4 x i8> %i8 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_32 = mul <4 x i32> %zl1_8_32, %zl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sw_8_64 = sext <4 x i8> %i8 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_64 = mul <4 x i64> %i64, %sw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_8_64 = sext <4 x i8> %i8 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_8_64 = sext <4 x i8> %i8 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_64 = mul <4 x i64> %sl1_8_64, %sl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zw_8_64 = zext <4 x i8> %i8 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_64 = mul <4 x i64> %i64, %zw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_8_64 = zext <4 x i8> %i8 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_8_64 = zext <4 x i8> %i8 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_64 = mul <4 x i64> %zl1_8_64, %zl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_16_32 = sext <4 x i16> %i16 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_16_32 = mul <4 x i32> %i32, %sw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_16_32 = sext <4 x i16> %i16 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_16_32 = sext <4 x i16> %i16 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_16_32 = mul <4 x i32> %sl1_16_32, %sl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_16_32 = zext <4 x i16> %i16 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_16_32 = mul <4 x i32> %i32, %zw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_16_32 = zext <4 x i16> %i16 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_16_32 = zext <4 x i16> %i16 to <4 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_16_32 = mul <4 x i32> %zl1_16_32, %zl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sw_16_64 = sext <4 x i16> %i16 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:1 Lat:1 SizeLat:1 for: %asw_16_64 = mul <4 x i64> %i64, %sw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_16_64 = sext <4 x i16> %i16 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_16_64 = sext <4 x i16> %i16 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:1 Lat:1 SizeLat:1 for: %asl_16_64 = mul <4 x i64> %sl1_16_64, %sl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zw_16_64 = zext <4 x i16> %i16 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:1 Lat:1 SizeLat:1 for: %azw_16_64 = mul <4 x i64> %i64, %zw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_16_64 = zext <4 x i16> %i16 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_16_64 = zext <4 x i16> %i16 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:1 Lat:1 SizeLat:1 for: %azl_16_64 = mul <4 x i64> %zl1_16_64, %zl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %sw_32_64 = sext <4 x i32> %i32 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:1 Lat:1 SizeLat:1 for: %asw_32_64 = mul <4 x i64> %i64, %sw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_32_64 = sext <4 x i32> %i32 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_32_64 = sext <4 x i32> %i32 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asl_32_64 = mul <4 x i64> %sl1_32_64, %sl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %zw_32_64 = zext <4 x i32> %i32 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:1 Lat:1 SizeLat:1 for: %azw_32_64 = mul <4 x i64> %i64, %zw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_32_64 = zext <4 x i32> %i32 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_32_64 = zext <4 x i32> %i32 to <4 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azl_32_64 = mul <4 x i64> %zl1_32_64, %zl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %sw_8_16 = sext <4 x i8> %i8 to <4 x i16>
   %asw_8_16 = mul <4 x i16> %i16, %sw_8_16
@@ -2631,131 +1811,68 @@ define void @extmulv4(<4 x i8> %i8, <4 x i16> %i16, <4 x i32> %i32, <4 x i64> %i
 }
 
 define void @extmulv8(<8 x i8> %i8, <8 x i16> %i16, <8 x i32> %i32, <8 x i64> %i64)  {
-; RECIP-LABEL: 'extmulv8'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = mul <8 x i16> %i16, %sw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = mul <8 x i16> %sl1_8_16, %sl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = mul <8 x i16> %i16, %zw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = mul <8 x i16> %zl1_8_16, %zl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sw_8_32 = sext <8 x i8> %i8 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asw_8_32 = mul <8 x i32> %i32, %sw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sl1_8_32 = sext <8 x i8> %i8 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sl2_8_32 = sext <8 x i8> %i8 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asl_8_32 = mul <8 x i32> %sl1_8_32, %sl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zw_8_32 = zext <8 x i8> %i8 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azw_8_32 = mul <8 x i32> %i32, %zw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zl1_8_32 = zext <8 x i8> %i8 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %zl2_8_32 = zext <8 x i8> %i8 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azl_8_32 = mul <8 x i32> %zl1_8_32, %zl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %sw_8_64 = sext <8 x i8> %i8 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %asw_8_64 = mul <8 x i64> %i64, %sw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %sl1_8_64 = sext <8 x i8> %i8 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %sl2_8_64 = sext <8 x i8> %i8 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %asl_8_64 = mul <8 x i64> %sl1_8_64, %sl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %zw_8_64 = zext <8 x i8> %i8 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %azw_8_64 = mul <8 x i64> %i64, %zw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %zl1_8_64 = zext <8 x i8> %i8 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %zl2_8_64 = zext <8 x i8> %i8 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %azl_8_64 = mul <8 x i64> %zl1_8_64, %zl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sw_16_32 = sext <8 x i16> %i16 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asw_16_32 = mul <8 x i32> %i32, %sw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_16_32 = sext <8 x i16> %i16 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_16_32 = sext <8 x i16> %i16 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asl_16_32 = mul <8 x i32> %sl1_16_32, %sl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %zw_16_32 = zext <8 x i16> %i16 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azw_16_32 = mul <8 x i32> %i32, %zw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_32 = zext <8 x i16> %i16 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_32 = zext <8 x i16> %i16 to <8 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azl_16_32 = mul <8 x i32> %zl1_16_32, %zl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sw_16_64 = sext <8 x i16> %i16 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %asw_16_64 = mul <8 x i64> %i64, %sw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sl1_16_64 = sext <8 x i16> %i16 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sl2_16_64 = sext <8 x i16> %i16 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %asl_16_64 = mul <8 x i64> %sl1_16_64, %sl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %zw_16_64 = zext <8 x i16> %i16 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %azw_16_64 = mul <8 x i64> %i64, %zw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %zl1_16_64 = zext <8 x i16> %i16 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %zl2_16_64 = zext <8 x i16> %i16 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %azl_16_64 = mul <8 x i64> %zl1_16_64, %zl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %sw_32_64 = sext <8 x i32> %i32 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %asw_32_64 = mul <8 x i64> %i64, %sw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <8 x i32> %i32 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <8 x i32> %i32 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asl_32_64 = mul <8 x i64> %sl1_32_64, %sl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %zw_32_64 = zext <8 x i32> %i32 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %azw_32_64 = mul <8 x i64> %i64, %zw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <8 x i32> %i32 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <8 x i32> %i32 to <8 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azl_32_64 = mul <8 x i64> %zl1_32_64, %zl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'extmulv8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = mul <8 x i16> %i16, %sw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = mul <8 x i16> %sl1_8_16, %sl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = mul <8 x i16> %i16, %zw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = mul <8 x i16> %zl1_8_16, %zl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext <8 x i8> %i8 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = mul <8 x i32> %i32, %sw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext <8 x i8> %i8 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext <8 x i8> %i8 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = mul <8 x i32> %sl1_8_32, %sl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext <8 x i8> %i8 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = mul <8 x i32> %i32, %zw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext <8 x i8> %i8 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext <8 x i8> %i8 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = mul <8 x i32> %zl1_8_32, %zl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext <8 x i8> %i8 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = mul <8 x i64> %i64, %sw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext <8 x i8> %i8 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext <8 x i8> %i8 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = mul <8 x i64> %sl1_8_64, %sl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_64 = zext <8 x i8> %i8 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = mul <8 x i64> %i64, %zw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_64 = zext <8 x i8> %i8 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_64 = zext <8 x i8> %i8 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = mul <8 x i64> %zl1_8_64, %zl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_32 = sext <8 x i16> %i16 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = mul <8 x i32> %i32, %sw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_16_32 = sext <8 x i16> %i16 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_16_32 = sext <8 x i16> %i16 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = mul <8 x i32> %sl1_16_32, %sl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_32 = zext <8 x i16> %i16 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = mul <8 x i32> %i32, %zw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_32 = zext <8 x i16> %i16 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_32 = zext <8 x i16> %i16 to <8 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = mul <8 x i32> %zl1_16_32, %zl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext <8 x i16> %i16 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = mul <8 x i64> %i64, %sw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext <8 x i16> %i16 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext <8 x i16> %i16 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = mul <8 x i64> %sl1_16_64, %sl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_64 = zext <8 x i16> %i16 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = mul <8 x i64> %i64, %zw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_64 = zext <8 x i16> %i16 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_64 = zext <8 x i16> %i16 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = mul <8 x i64> %zl1_16_64, %zl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_32_64 = sext <8 x i32> %i32 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = mul <8 x i64> %i64, %sw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <8 x i32> %i32 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <8 x i32> %i32 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = mul <8 x i64> %sl1_32_64, %sl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_32_64 = zext <8 x i32> %i32 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = mul <8 x i64> %i64, %zw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <8 x i32> %i32 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <8 x i32> %i32 to <8 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = mul <8 x i64> %zl1_32_64, %zl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'extmulv8'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sw_8_16 = sext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asw_8_16 = mul <8 x i16> %i16, %sw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_8_16 = sext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_8_16 = sext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_16 = mul <8 x i16> %sl1_8_16, %sl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %zw_8_16 = zext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azw_8_16 = mul <8 x i16> %i16, %zw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_8_16 = zext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_8_16 = zext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_16 = mul <8 x i16> %zl1_8_16, %zl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sw_8_32 = sext <8 x i8> %i8 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_32 = mul <8 x i32> %i32, %sw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_8_32 = sext <8 x i8> %i8 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_8_32 = sext <8 x i8> %i8 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_32 = mul <8 x i32> %sl1_8_32, %sl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zw_8_32 = zext <8 x i8> %i8 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_32 = mul <8 x i32> %i32, %zw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_8_32 = zext <8 x i8> %i8 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_8_32 = zext <8 x i8> %i8 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_32 = mul <8 x i32> %zl1_8_32, %zl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %sw_8_64 = sext <8 x i8> %i8 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:56 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_64 = mul <8 x i64> %i64, %sw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_8_64 = sext <8 x i8> %i8 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_8_64 = sext <8 x i8> %i8 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:56 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_64 = mul <8 x i64> %sl1_8_64, %sl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %zw_8_64 = zext <8 x i8> %i8 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:56 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_64 = mul <8 x i64> %i64, %zw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_8_64 = zext <8 x i8> %i8 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_8_64 = zext <8 x i8> %i8 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:56 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_64 = mul <8 x i64> %zl1_8_64, %zl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %sw_16_32 = sext <8 x i16> %i16 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asw_16_32 = mul <8 x i32> %i32, %sw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_16_32 = sext <8 x i16> %i16 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_16_32 = sext <8 x i16> %i16 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asl_16_32 = mul <8 x i32> %sl1_16_32, %sl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %zw_16_32 = zext <8 x i16> %i16 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azw_16_32 = mul <8 x i32> %i32, %zw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_16_32 = zext <8 x i16> %i16 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_16_32 = zext <8 x i16> %i16 to <8 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azl_16_32 = mul <8 x i32> %zl1_16_32, %zl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %sw_16_64 = sext <8 x i16> %i16 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:56 CodeSize:1 Lat:1 SizeLat:1 for: %asw_16_64 = mul <8 x i64> %i64, %sw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_16_64 = sext <8 x i16> %i16 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_16_64 = sext <8 x i16> %i16 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:56 CodeSize:1 Lat:1 SizeLat:1 for: %asl_16_64 = mul <8 x i64> %sl1_16_64, %sl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %zw_16_64 = zext <8 x i16> %i16 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:56 CodeSize:1 Lat:1 SizeLat:1 for: %azw_16_64 = mul <8 x i64> %i64, %zw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_16_64 = zext <8 x i16> %i16 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_16_64 = zext <8 x i16> %i16 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:56 CodeSize:1 Lat:1 SizeLat:1 for: %azl_16_64 = mul <8 x i64> %zl1_16_64, %zl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %sw_32_64 = sext <8 x i32> %i32 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:56 CodeSize:1 Lat:1 SizeLat:1 for: %asw_32_64 = mul <8 x i64> %i64, %sw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_32_64 = sext <8 x i32> %i32 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_32_64 = sext <8 x i32> %i32 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asl_32_64 = mul <8 x i64> %sl1_32_64, %sl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %zw_32_64 = zext <8 x i32> %i32 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:56 CodeSize:1 Lat:1 SizeLat:1 for: %azw_32_64 = mul <8 x i64> %i64, %zw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_32_64 = zext <8 x i32> %i32 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_32_64 = zext <8 x i32> %i32 to <8 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azl_32_64 = mul <8 x i64> %zl1_32_64, %zl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %sw_8_16 = sext <8 x i8> %i8 to <8 x i16>
   %asw_8_16 = mul <8 x i16> %i16, %sw_8_16
@@ -2833,131 +1950,68 @@ define void @extmulv8(<8 x i8> %i8, <8 x i16> %i16, <8 x i32> %i32, <8 x i64> %i
 }
 
 define void @extmulv16(<16 x i8> %i8, <16 x i16> %i16, <16 x i32> %i32, <16 x i64> %i64)  {
-; RECIP-LABEL: 'extmulv16'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sw_8_16 = sext <16 x i8> %i8 to <16 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asw_8_16 = mul <16 x i16> %i16, %sw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_8_16 = sext <16 x i8> %i8 to <16 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_8_16 = sext <16 x i8> %i8 to <16 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %asl_8_16 = mul <16 x i16> %sl1_8_16, %sl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %zw_8_16 = zext <16 x i8> %i8 to <16 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azw_8_16 = mul <16 x i16> %i16, %zw_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_16 = zext <16 x i8> %i8 to <16 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_8_16 = zext <16 x i8> %i8 to <16 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %azl_8_16 = mul <16 x i16> %zl1_8_16, %zl2_8_16
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sw_8_32 = sext <16 x i8> %i8 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asw_8_32 = mul <16 x i32> %i32, %sw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sl1_8_32 = sext <16 x i8> %i8 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sl2_8_32 = sext <16 x i8> %i8 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asl_8_32 = mul <16 x i32> %sl1_8_32, %sl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %zw_8_32 = zext <16 x i8> %i8 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azw_8_32 = mul <16 x i32> %i32, %zw_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %zl1_8_32 = zext <16 x i8> %i8 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %zl2_8_32 = zext <16 x i8> %i8 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azl_8_32 = mul <16 x i32> %zl1_8_32, %zl2_8_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %sw_8_64 = sext <16 x i8> %i8 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %asw_8_64 = mul <16 x i64> %i64, %sw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %sl1_8_64 = sext <16 x i8> %i8 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %sl2_8_64 = sext <16 x i8> %i8 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %asl_8_64 = mul <16 x i64> %sl1_8_64, %sl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %zw_8_64 = zext <16 x i8> %i8 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %azw_8_64 = mul <16 x i64> %i64, %zw_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %zl1_8_64 = zext <16 x i8> %i8 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %zl2_8_64 = zext <16 x i8> %i8 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %azl_8_64 = mul <16 x i64> %zl1_8_64, %zl2_8_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %sw_16_32 = sext <16 x i16> %i16 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asw_16_32 = mul <16 x i32> %i32, %sw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_16_32 = sext <16 x i16> %i16 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_16_32 = sext <16 x i16> %i16 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %asl_16_32 = mul <16 x i32> %sl1_16_32, %sl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %zw_16_32 = zext <16 x i16> %i16 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azw_16_32 = mul <16 x i32> %i32, %zw_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_32 = zext <16 x i16> %i16 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_32 = zext <16 x i16> %i16 to <16 x i32>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %azl_16_32 = mul <16 x i32> %zl1_16_32, %zl2_16_32
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %sw_16_64 = sext <16 x i16> %i16 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %asw_16_64 = mul <16 x i64> %i64, %sw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %sl1_16_64 = sext <16 x i16> %i16 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %sl2_16_64 = sext <16 x i16> %i16 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %asl_16_64 = mul <16 x i64> %sl1_16_64, %sl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %zw_16_64 = zext <16 x i16> %i16 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %azw_16_64 = mul <16 x i64> %i64, %zw_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %zl1_16_64 = zext <16 x i16> %i16 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %zl2_16_64 = zext <16 x i16> %i16 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %azl_16_64 = mul <16 x i64> %zl1_16_64, %zl2_16_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %sw_32_64 = sext <16 x i32> %i32 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %asw_32_64 = mul <16 x i64> %i64, %sw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <16 x i32> %i32 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <16 x i32> %i32 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %asl_32_64 = mul <16 x i64> %sl1_32_64, %sl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %zw_32_64 = zext <16 x i32> %i32 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %azw_32_64 = mul <16 x i64> %i64, %zw_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <16 x i32> %i32 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <16 x i32> %i32 to <16 x i64>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %azl_32_64 = mul <16 x i64> %zl1_32_64, %zl2_32_64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'extmulv16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_16 = sext <16 x i8> %i8 to <16 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_16 = mul <16 x i16> %i16, %sw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_8_16 = sext <16 x i8> %i8 to <16 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_8_16 = sext <16 x i8> %i8 to <16 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = mul <16 x i16> %sl1_8_16, %sl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_16 = zext <16 x i8> %i8 to <16 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_16 = mul <16 x i16> %i16, %zw_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_16 = zext <16 x i8> %i8 to <16 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_8_16 = zext <16 x i8> %i8 to <16 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = mul <16 x i16> %zl1_8_16, %zl2_8_16
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_32 = sext <16 x i8> %i8 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_32 = mul <16 x i32> %i32, %sw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_32 = sext <16 x i8> %i8 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_32 = sext <16 x i8> %i8 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_32 = mul <16 x i32> %sl1_8_32, %sl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_32 = zext <16 x i8> %i8 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_32 = mul <16 x i32> %i32, %zw_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_32 = zext <16 x i8> %i8 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_32 = zext <16 x i8> %i8 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_32 = mul <16 x i32> %zl1_8_32, %zl2_8_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_8_64 = sext <16 x i8> %i8 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_8_64 = mul <16 x i64> %i64, %sw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_64 = sext <16 x i8> %i8 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_8_64 = sext <16 x i8> %i8 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_64 = mul <16 x i64> %sl1_8_64, %sl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_8_64 = zext <16 x i8> %i8 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_8_64 = mul <16 x i64> %i64, %zw_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_8_64 = zext <16 x i8> %i8 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_8_64 = zext <16 x i8> %i8 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_64 = mul <16 x i64> %zl1_8_64, %zl2_8_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_32 = sext <16 x i16> %i16 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_32 = mul <16 x i32> %i32, %sw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_16_32 = sext <16 x i16> %i16 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_16_32 = sext <16 x i16> %i16 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_32 = mul <16 x i32> %sl1_16_32, %sl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_32 = zext <16 x i16> %i16 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_32 = mul <16 x i32> %i32, %zw_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_16_32 = zext <16 x i16> %i16 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_16_32 = zext <16 x i16> %i16 to <16 x i32>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_32 = mul <16 x i32> %zl1_16_32, %zl2_16_32
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_16_64 = sext <16 x i16> %i16 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_16_64 = mul <16 x i64> %i64, %sw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_16_64 = sext <16 x i16> %i16 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl2_16_64 = sext <16 x i16> %i16 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_16_64 = mul <16 x i64> %sl1_16_64, %sl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_16_64 = zext <16 x i16> %i16 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_16_64 = mul <16 x i64> %i64, %zw_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl1_16_64 = zext <16 x i16> %i16 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zl2_16_64 = zext <16 x i16> %i16 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_16_64 = mul <16 x i64> %zl1_16_64, %zl2_16_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sw_32_64 = sext <16 x i32> %i32 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asw_32_64 = mul <16 x i64> %i64, %sw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl1_32_64 = sext <16 x i32> %i32 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sl2_32_64 = sext <16 x i32> %i32 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_32_64 = mul <16 x i64> %sl1_32_64, %sl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zw_32_64 = zext <16 x i32> %i32 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azw_32_64 = mul <16 x i64> %i64, %zw_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_32_64 = zext <16 x i32> %i32 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl2_32_64 = zext <16 x i32> %i32 to <16 x i64>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_32_64 = mul <16 x i64> %zl1_32_64, %zl2_32_64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'extmulv16'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %sw_8_16 = sext <16 x i8> %i8 to <16 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_16 = mul <16 x i16> %i16, %sw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_8_16 = sext <16 x i8> %i8 to <16 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_8_16 = sext <16 x i8> %i8 to <16 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_16 = mul <16 x i16> %sl1_8_16, %sl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %zw_8_16 = zext <16 x i8> %i8 to <16 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_16 = mul <16 x i16> %i16, %zw_8_16
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_8_16 = zext <16 x i8> %i8 to <16 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_8_16 = zext <16 x i8> %i8 to <16 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_16 = mul <16 x i16> %zl1_8_16, %zl2_8_16
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %sw_8_32 = sext <16 x i8> %i8 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_32 = mul <16 x i32> %i32, %sw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_8_32 = sext <16 x i8> %i8 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_8_32 = sext <16 x i8> %i8 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_32 = mul <16 x i32> %sl1_8_32, %sl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %zw_8_32 = zext <16 x i8> %i8 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_32 = mul <16 x i32> %i32, %zw_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_8_32 = zext <16 x i8> %i8 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_8_32 = zext <16 x i8> %i8 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_32 = mul <16 x i32> %zl1_8_32, %zl2_8_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:15 CodeSize:1 Lat:1 SizeLat:1 for: %sw_8_64 = sext <16 x i8> %i8 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:112 CodeSize:1 Lat:1 SizeLat:1 for: %asw_8_64 = mul <16 x i64> %i64, %sw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:15 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_8_64 = sext <16 x i8> %i8 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:15 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_8_64 = sext <16 x i8> %i8 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:112 CodeSize:1 Lat:1 SizeLat:1 for: %asl_8_64 = mul <16 x i64> %sl1_8_64, %sl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:15 CodeSize:1 Lat:1 SizeLat:1 for: %zw_8_64 = zext <16 x i8> %i8 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:112 CodeSize:1 Lat:1 SizeLat:1 for: %azw_8_64 = mul <16 x i64> %i64, %zw_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:15 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_8_64 = zext <16 x i8> %i8 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:15 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_8_64 = zext <16 x i8> %i8 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:112 CodeSize:1 Lat:1 SizeLat:1 for: %azl_8_64 = mul <16 x i64> %zl1_8_64, %zl2_8_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %sw_16_32 = sext <16 x i16> %i16 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asw_16_32 = mul <16 x i32> %i32, %sw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_16_32 = sext <16 x i16> %i16 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_16_32 = sext <16 x i16> %i16 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %asl_16_32 = mul <16 x i32> %sl1_16_32, %sl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %zw_16_32 = zext <16 x i16> %i16 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azw_16_32 = mul <16 x i32> %i32, %zw_16_32
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_16_32 = zext <16 x i16> %i16 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_16_32 = zext <16 x i16> %i16 to <16 x i32>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %azl_16_32 = mul <16 x i32> %zl1_16_32, %zl2_16_32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: %sw_16_64 = sext <16 x i16> %i16 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:112 CodeSize:1 Lat:1 SizeLat:1 for: %asw_16_64 = mul <16 x i64> %i64, %sw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: %sl1_16_64 = sext <16 x i16> %i16 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: %sl2_16_64 = sext <16 x i16> %i16 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:112 CodeSize:1 Lat:1 SizeLat:1 for: %asl_16_64 = mul <16 x i64> %sl1_16_64, %sl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: %zw_16_64 = zext <16 x i16> %i16 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:112 CodeSize:1 Lat:1 SizeLat:1 for: %azw_16_64 = mul <16 x i64> %i64, %zw_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: %zl1_16_64 = zext <16 x i16> %i16 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:1 Lat:1 SizeLat:1 for: %zl2_16_64 = zext <16 x i16> %i16 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:112 CodeSize:1 Lat:1 SizeLat:1 for: %azl_16_64 = mul <16 x i64> %zl1_16_64, %zl2_16_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %sw_32_64 = sext <16 x i32> %i32 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:112 CodeSize:1 Lat:1 SizeLat:1 for: %asw_32_64 = mul <16 x i64> %i64, %sw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl1_32_64 = sext <16 x i32> %i32 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %sl2_32_64 = sext <16 x i32> %i32 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %asl_32_64 = mul <16 x i64> %sl1_32_64, %sl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %zw_32_64 = zext <16 x i32> %i32 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:112 CodeSize:1 Lat:1 SizeLat:1 for: %azw_32_64 = mul <16 x i64> %i64, %zw_32_64
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_32_64 = zext <16 x i32> %i32 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl2_32_64 = zext <16 x i32> %i32 to <16 x i64>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %azl_32_64 = mul <16 x i64> %zl1_32_64, %zl2_32_64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %sw_8_16 = sext <16 x i8> %i8 to <16 x i16>
   %asw_8_16 = mul <16 x i16> %i16, %sw_8_16
@@ -3035,25 +2089,15 @@ define void @extmulv16(<16 x i8> %i8, <16 x i16> %i16, <16 x i32> %i32, <16 x i6
 }
 
 define void @extmul_const(<8 x i8> %i8, <8 x i16> %i16, <8 x i32> %i32, <8 x i64> %i64)  {
-; RECIP-LABEL: 'extmul_const'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = mul <8 x i16> %sl1_8_16, splat (i16 10)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = mul <8 x i16> %zl1_8_16, splat (i16 10)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_16b = zext <8 x i8> %i8 to <8 x i16>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %and = and <8 x i16> %sl1_8_16, splat (i16 255)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %aal_8_16 = mul <8 x i16> %zl1_8_16b, %and
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'extmul_const'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sl1_8_16 = sext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %asl_8_16 = mul <8 x i16> %sl1_8_16, splat (i16 10)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_16 = zext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %azl_8_16 = mul <8 x i16> %zl1_8_16, splat (i16 10)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %zl1_8_16b = zext <8 x i8> %i8 to <8 x i16>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %and = and <8 x i16> %sl1_8_16, splat (i16 255)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %aal_8_16 = mul <8 x i16> %zl1_8_16b, %and
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'extmul_const'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sl1_8_16 = sext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %asl_8_16 = mul <8 x i16> %sl1_8_16, splat (i16 10)
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_8_16 = zext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %azl_8_16 = mul <8 x i16> %zl1_8_16, splat (i16 10)
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %zl1_8_16b = zext <8 x i8> %i8 to <8 x i16>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %and = and <8 x i16> %sl1_8_16, splat (i16 255)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %aal_8_16 = mul <8 x i16> %zl1_8_16b, %and
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %sl1_8_16 = sext <8 x i8> %i8 to <8 x i16>
   %asl_8_16 = mul <8 x i16> %sl1_8_16, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>
diff --git a/llvm/test/Analysis/CostModel/AArch64/arith.ll b/llvm/test/Analysis/CostModel/AArch64/arith.ll
index 2355a612809c9..d3c1e07396921 100644
--- a/llvm/test/Analysis/CostModel/AArch64/arith.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/arith.ll
@@ -1,33 +1,20 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefix=RECIP
-; RUN: opt -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefix=SIZE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define void @i1() {
-; RECIP-LABEL: 'i1'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = add i1 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = sub i1 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = mul i1 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = ashr i1 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = lshr i1 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = shl i1 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i = and i1 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j = or i1 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k = xor i1 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'i1'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = add i1 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = sub i1 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = mul i1 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = ashr i1 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = lshr i1 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = shl i1 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i = and i1 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j = or i1 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k = xor i1 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'i1'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %c = add i1 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %d = sub i1 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %e = mul i1 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f = ashr i1 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %g = lshr i1 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %h = shl i1 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %i = and i1 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %j = or i1 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %k = xor i1 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %c = add i1 undef, undef
   %d = sub i1 undef, undef
@@ -42,29 +29,17 @@ define void @i1() {
 }
 
 define void @i8() {
-; RECIP-LABEL: 'i8'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = add i8 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = sub i8 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = mul i8 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = ashr i8 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = lshr i8 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = shl i8 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i = and i8 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j = or i8 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k = xor i8 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'i8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = add i8 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = sub i8 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = mul i8 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = ashr i8 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = lshr i8 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = shl i8 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i = and i8 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j = or i8 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k = xor i8 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'i8'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %c = add i8 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %d = sub i8 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %e = mul i8 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f = ashr i8 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %g = lshr i8 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %h = shl i8 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %i = and i8 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %j = or i8 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %k = xor i8 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %c = add i8 undef, undef
   %d = sub i8 undef, undef
@@ -79,29 +54,17 @@ define void @i8() {
 }
 
 define void @i16() {
-; RECIP-LABEL: 'i16'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = add i16 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = sub i16 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = mul i16 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = ashr i16 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = lshr i16 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = shl i16 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i = and i16 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j = or i16 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k = xor i16 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = add i16 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = sub i16 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = mul i16 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = ashr i16 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = lshr i16 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = shl i16 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i = and i16 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j = or i16 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k = xor i16 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'i16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %c = add i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %d = sub i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %e = mul i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f = ashr i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %g = lshr i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %h = shl i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %i = and i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %j = or i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %k = xor i16 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %c = add i16 undef, undef
   %d = sub i16 undef, undef
@@ -116,29 +79,17 @@ define void @i16() {
 }
 
 define void @i32() {
-; RECIP-LABEL: 'i32'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = add i32 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = sub i32 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = mul i32 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = ashr i32 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = lshr i32 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = shl i32 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i = and i32 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j = or i32 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k = xor i32 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'i32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = add i32 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = sub i32 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = mul i32 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = ashr i32 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = lshr i32 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = shl i32 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i = and i32 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j = or i32 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k = xor i32 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'i32'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %c = add i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %d = sub i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %e = mul i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f = ashr i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %g = lshr i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %h = shl i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %i = and i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %j = or i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %k = xor i32 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %c = add i32 undef, undef
   %d = sub i32 undef, undef
@@ -153,29 +104,17 @@ define void @i32() {
 }
 
 define void @i64() {
-; RECIP-LABEL: 'i64'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = add i64 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = sub i64 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = mul i64 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = ashr i64 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = lshr i64 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = shl i64 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i = and i64 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j = or i64 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k = xor i64 undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'i64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = add i64 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = sub i64 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = mul i64 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = ashr i64 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = lshr i64 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = shl i64 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i = and i64 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j = or i64 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k = xor i64 undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'i64'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %c = add i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %d = sub i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %e = mul i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f = ashr i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %g = lshr i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %h = shl i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %i = and i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %j = or i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %k = xor i64 undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %c = add i64 undef, undef
   %d = sub i64 undef, undef
@@ -191,83 +130,44 @@ define void @i64() {
 
 
 define void @vi8() {
-; RECIP-LABEL: 'vi8'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c2 = add <2 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d2 = sub <2 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e2 = mul <2 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f2 = ashr <2 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g2 = lshr <2 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h2 = shl <2 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c4 = add <4 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d4 = sub <4 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e4 = mul <4 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f4 = ashr <4 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g4 = lshr <4 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h4 = shl <4 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i4 = and <4 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j4 = or <4 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k4 = xor <4 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c8 = add <8 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d8 = sub <8 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e8 = mul <8 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f8 = ashr <8 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g8 = lshr <8 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h8 = shl <8 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i8 = and <8 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j8 = or <8 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k8 = xor <8 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c16 = add <16 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d16 = sub <16 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e16 = mul <16 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = ashr <16 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g16 = lshr <16 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h16 = shl <16 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i16 = and <16 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j16 = or <16 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k16 = xor <16 x i8> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'vi8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c2 = add <2 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d2 = sub <2 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e2 = mul <2 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f2 = ashr <2 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g2 = lshr <2 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h2 = shl <2 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c4 = add <4 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d4 = sub <4 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e4 = mul <4 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f4 = ashr <4 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g4 = lshr <4 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h4 = shl <4 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i4 = and <4 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j4 = or <4 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k4 = xor <4 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c8 = add <8 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d8 = sub <8 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e8 = mul <8 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f8 = ashr <8 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g8 = lshr <8 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h8 = shl <8 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i8 = and <8 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j8 = or <8 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k8 = xor <8 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c16 = add <16 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d16 = sub <16 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e16 = mul <16 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = ashr <16 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g16 = lshr <16 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h16 = shl <16 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i16 = and <16 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j16 = or <16 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k16 = xor <16 x i8> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'vi8'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %c2 = add <2 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %d2 = sub <2 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %e2 = mul <2 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f2 = ashr <2 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %g2 = lshr <2 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %h2 = shl <2 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %i2 = and <2 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %j2 = or <2 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %k2 = xor <2 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %c4 = add <4 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %d4 = sub <4 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %e4 = mul <4 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f4 = ashr <4 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %g4 = lshr <4 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %h4 = shl <4 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %i4 = and <4 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %j4 = or <4 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %k4 = xor <4 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %c8 = add <8 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %d8 = sub <8 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %e8 = mul <8 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f8 = ashr <8 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %g8 = lshr <8 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %h8 = shl <8 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %i8 = and <8 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %j8 = or <8 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %k8 = xor <8 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %c16 = add <16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %d16 = sub <16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %e16 = mul <16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f16 = ashr <16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %g16 = lshr <16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %h16 = shl <16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %i16 = and <16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %j16 = or <16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %k16 = xor <16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %c2 = add <2 x i8> undef, undef
   %d2 = sub <2 x i8> undef, undef
@@ -309,83 +209,44 @@ define void @vi8() {
 }
 
 define void @vi16() {
-; RECIP-LABEL: 'vi16'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c2 = add <2 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d2 = sub <2 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e2 = mul <2 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f2 = ashr <2 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g2 = lshr <2 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h2 = shl <2 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c4 = add <4 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d4 = sub <4 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e4 = mul <4 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f4 = ashr <4 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g4 = lshr <4 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h4 = shl <4 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i4 = and <4 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j4 = or <4 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k4 = xor <4 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c8 = add <8 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d8 = sub <8 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e8 = mul <8 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f8 = ashr <8 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g8 = lshr <8 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h8 = shl <8 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i8 = and <8 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j8 = or <8 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k8 = xor <8 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %c16 = add <16 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %d16 = sub <16 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e16 = mul <16 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f16 = ashr <16 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %g16 = lshr <16 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %h16 = shl <16 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i16 = and <16 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j16 = or <16 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k16 = xor <16 x i16> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'vi16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c2 = add <2 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d2 = sub <2 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e2 = mul <2 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f2 = ashr <2 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g2 = lshr <2 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h2 = shl <2 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c4 = add <4 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d4 = sub <4 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e4 = mul <4 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f4 = ashr <4 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g4 = lshr <4 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h4 = shl <4 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i4 = and <4 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j4 = or <4 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k4 = xor <4 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c8 = add <8 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d8 = sub <8 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e8 = mul <8 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f8 = ashr <8 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g8 = lshr <8 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h8 = shl <8 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i8 = and <8 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j8 = or <8 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k8 = xor <8 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c16 = add <16 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d16 = sub <16 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e16 = mul <16 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = ashr <16 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g16 = lshr <16 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h16 = shl <16 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i16 = and <16 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j16 = or <16 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k16 = xor <16 x i16> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'vi16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %c2 = add <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %d2 = sub <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %e2 = mul <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f2 = ashr <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %g2 = lshr <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %h2 = shl <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %i2 = and <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %j2 = or <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %k2 = xor <2 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %c4 = add <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %d4 = sub <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %e4 = mul <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f4 = ashr <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %g4 = lshr <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %h4 = shl <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %i4 = and <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %j4 = or <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %k4 = xor <4 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %c8 = add <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %d8 = sub <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %e8 = mul <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f8 = ashr <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %g8 = lshr <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %h8 = shl <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %i8 = and <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %j8 = or <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %k8 = xor <8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %c16 = add <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %d16 = sub <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %f16 = ashr <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %g16 = lshr <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %h16 = shl <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %i16 = and <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %j16 = or <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %k16 = xor <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %c2 = add <2 x i16> undef, undef
   %d2 = sub <2 x i16> undef, undef
@@ -427,83 +288,44 @@ define void @vi16() {
 }
 
 define void @vi32() {
-; RECIP-LABEL: 'vi32'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c2 = add <2 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d2 = sub <2 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e2 = mul <2 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f2 = ashr <2 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g2 = lshr <2 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h2 = shl <2 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c4 = add <4 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d4 = sub <4 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e4 = mul <4 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f4 = ashr <4 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g4 = lshr <4 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h4 = shl <4 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i4 = and <4 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j4 = or <4 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k4 = xor <4 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %c8 = add <8 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %d8 = sub <8 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e8 = mul <8 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f8 = ashr <8 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %g8 = lshr <8 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %h8 = shl <8 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i8 = and <8 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j8 = or <8 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k8 = xor <8 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c16 = add <16 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d16 = sub <16 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e16 = mul <16 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f16 = ashr <16 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %g16 = lshr <16 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %h16 = shl <16 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i16 = and <16 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j16 = or <16 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k16 = xor <16 x i32> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'vi32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c2 = add <2 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d2 = sub <2 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e2 = mul <2 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f2 = ashr <2 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g2 = lshr <2 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h2 = shl <2 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c4 = add <4 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d4 = sub <4 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e4 = mul <4 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f4 = ashr <4 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g4 = lshr <4 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h4 = shl <4 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i4 = and <4 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j4 = or <4 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k4 = xor <4 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c8 = add <8 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d8 = sub <8 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e8 = mul <8 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f8 = ashr <8 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g8 = lshr <8 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h8 = shl <8 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i8 = and <8 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j8 = or <8 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k8 = xor <8 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c16 = add <16 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d16 = sub <16 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e16 = mul <16 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = ashr <16 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g16 = lshr <16 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h16 = shl <16 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i16 = and <16 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j16 = or <16 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k16 = xor <16 x i32> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'vi32'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %c2 = add <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %d2 = sub <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %e2 = mul <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f2 = ashr <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %g2 = lshr <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %h2 = shl <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %i2 = and <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %j2 = or <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %k2 = xor <2 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %c4 = add <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %d4 = sub <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %e4 = mul <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f4 = ashr <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %g4 = lshr <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %h4 = shl <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %i4 = and <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %j4 = or <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %k4 = xor <4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %c8 = add <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %d8 = sub <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %e8 = mul <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %f8 = ashr <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %g8 = lshr <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %h8 = shl <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %i8 = and <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %j8 = or <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %k8 = xor <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %c16 = add <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %d16 = sub <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %f16 = ashr <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %g16 = lshr <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %h16 = shl <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %i16 = and <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %j16 = or <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %k16 = xor <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %c2 = add <2 x i32> undef, undef
   %d2 = sub <2 x i32> undef, undef
@@ -545,83 +367,44 @@ define void @vi32() {
 }
 
 define void @vi64() {
-; RECIP-LABEL: 'vi64'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c2 = add <2 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d2 = sub <2 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %e2 = mul <2 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f2 = ashr <2 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g2 = lshr <2 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h2 = shl <2 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %c4 = add <4 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %d4 = sub <4 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %e4 = mul <4 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f4 = ashr <4 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %g4 = lshr <4 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %h4 = shl <4 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i4 = and <4 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %j4 = or <4 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %k4 = xor <4 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %c8 = add <8 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %d8 = sub <8 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %e8 = mul <8 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f8 = ashr <8 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %g8 = lshr <8 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %h8 = shl <8 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %i8 = and <8 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %j8 = or <8 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %k8 = xor <8 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %c16 = add <16 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %d16 = sub <16 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %e16 = mul <16 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f16 = ashr <16 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %g16 = lshr <16 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %h16 = shl <16 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %i16 = and <16 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %j16 = or <16 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %k16 = xor <16 x i64> undef, undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'vi64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c2 = add <2 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d2 = sub <2 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e2 = mul <2 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f2 = ashr <2 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g2 = lshr <2 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h2 = shl <2 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i2 = and <2 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j2 = or <2 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k2 = xor <2 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c4 = add <4 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d4 = sub <4 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e4 = mul <4 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f4 = ashr <4 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g4 = lshr <4 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h4 = shl <4 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i4 = and <4 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j4 = or <4 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k4 = xor <4 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c8 = add <8 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d8 = sub <8 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e8 = mul <8 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f8 = ashr <8 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g8 = lshr <8 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h8 = shl <8 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i8 = and <8 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j8 = or <8 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k8 = xor <8 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c16 = add <16 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d16 = sub <16 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e16 = mul <16 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = ashr <16 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g16 = lshr <16 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h16 = shl <16 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %i16 = and <16 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %j16 = or <16 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %k16 = xor <16 x i64> undef, undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'vi64'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %c2 = add <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %d2 = sub <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:1 Lat:1 SizeLat:1 for: %e2 = mul <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f2 = ashr <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %g2 = lshr <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %h2 = shl <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %i2 = and <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %j2 = or <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %k2 = xor <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %c4 = add <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %d4 = sub <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:1 Lat:1 SizeLat:1 for: %e4 = mul <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %f4 = ashr <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %g4 = lshr <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %h4 = shl <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %i4 = and <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %j4 = or <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %k4 = xor <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %c8 = add <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %d8 = sub <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:56 CodeSize:1 Lat:1 SizeLat:1 for: %e8 = mul <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %f8 = ashr <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %g8 = lshr <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %h8 = shl <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %i8 = and <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %j8 = or <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:1 SizeLat:1 for: %k8 = xor <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %c16 = add <16 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %d16 = sub <16 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:112 CodeSize:1 Lat:1 SizeLat:1 for: %e16 = mul <16 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %f16 = ashr <16 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %g16 = lshr <16 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %h16 = shl <16 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %i16 = and <16 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %j16 = or <16 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %k16 = xor <16 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %c2 = add <2 x i64> undef, undef
   %d2 = sub <2 x i64> undef, undef
diff --git a/llvm/test/Analysis/CostModel/AArch64/div.ll b/llvm/test/Analysis/CostModel/AArch64/div.ll
index fba6d9ab31793..5f01710a1e60f 100644
--- a/llvm/test/Analysis/CostModel/AArch64/div.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/div.ll
@@ -288,28 +288,28 @@ define void @udiv_uniform() {
 define void @sdiv_const() {
 ; CHECK-LABEL: 'sdiv_const'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I128 = sdiv i128 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I64 = sdiv i64 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V2i64 = sdiv <2 x i64> undef, <i64 6, i64 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V4i64 = sdiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V8i64 = sdiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sdiv i32 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V2i32 = sdiv <2 x i32> undef, <i32 4, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V4i32 = sdiv <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V8i32 = sdiv <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V16i32 = sdiv <16 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sdiv i16 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V2i16 = sdiv <2 x i16> undef, <i16 4, i16 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V4i16 = sdiv <4 x i16> undef, <i16 4, i16 5, i16 6, i16 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8i16 = sdiv <8 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 176 for instruction: %V16i16 = sdiv <16 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 352 for instruction: %V32i16 = sdiv <32 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = sdiv i8 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V2i8 = sdiv <2 x i8> undef, <i8 4, i8 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V4i8 = sdiv <4 x i8> undef, <i8 4, i8 5, i8 6, i8 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8i8 = sdiv <8 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 168 for instruction: %V16i8 = sdiv <16 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 336 for instruction: %V32i8 = sdiv <32 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 672 for instruction: %V64i8 = sdiv <64 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = sdiv i64 undef, 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i64 = sdiv <2 x i64> undef, <i64 6, i64 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4i64 = sdiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8i64 = sdiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = sdiv i32 undef, 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i32 = sdiv <2 x i32> undef, <i32 4, i32 5>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4i32 = sdiv <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8i32 = sdiv <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V16i32 = sdiv <16 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = sdiv i16 undef, 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i16 = sdiv <2 x i16> undef, <i16 4, i16 5>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4i16 = sdiv <4 x i16> undef, <i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8i16 = sdiv <8 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V16i16 = sdiv <16 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V32i16 = sdiv <32 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = sdiv i8 undef, 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i8 = sdiv <2 x i8> undef, <i8 4, i8 5>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4i8 = sdiv <4 x i8> undef, <i8 4, i8 5, i8 6, i8 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8i8 = sdiv <8 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V16i8 = sdiv <16 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V32i8 = sdiv <32 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V64i8 = sdiv <64 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I128 = sdiv i128 undef, 7
@@ -405,28 +405,28 @@ define void @udiv_const() {
 define void @sdiv_uniformconst() {
 ; CHECK-LABEL: 'sdiv_uniformconst'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I128 = sdiv i128 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I64 = sdiv i64 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2i64 = sdiv <2 x i64> undef, splat (i64 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4i64 = sdiv <4 x i64> undef, splat (i64 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8i64 = sdiv <8 x i64> undef, splat (i64 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sdiv i32 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2i32 = sdiv <2 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4i32 = sdiv <4 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8i32 = sdiv <8 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V16i32 = sdiv <16 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sdiv i16 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2i16 = sdiv <2 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4i16 = sdiv <4 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i16 = sdiv <8 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V16i16 = sdiv <16 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %V32i16 = sdiv <32 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = sdiv i8 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2i8 = sdiv <2 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4i8 = sdiv <4 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8i8 = sdiv <8 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i8 = sdiv <16 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %V32i8 = sdiv <32 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %V64i8 = sdiv <64 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = sdiv i64 undef, 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i64 = sdiv <2 x i64> undef, splat (i64 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4i64 = sdiv <4 x i64> undef, splat (i64 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8i64 = sdiv <8 x i64> undef, splat (i64 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = sdiv i32 undef, 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i32 = sdiv <2 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i32 = sdiv <4 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8i32 = sdiv <8 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16i32 = sdiv <16 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = sdiv i16 undef, 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i16 = sdiv <2 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i16 = sdiv <4 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i16 = sdiv <8 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16i16 = sdiv <16 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32i16 = sdiv <32 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = sdiv i8 undef, 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i8 = sdiv <2 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i8 = sdiv <4 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i8 = sdiv <8 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i8 = sdiv <16 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32i8 = sdiv <32 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64i8 = sdiv <64 x i8> undef, splat (i8 7)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I128 = sdiv i128 undef, 7
@@ -521,28 +521,28 @@ define void @udiv_uniformconst() {
 define void @sdiv_constpow2() {
 ; CHECK-LABEL: 'sdiv_constpow2'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I128 = sdiv i128 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I64 = sdiv i64 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V2i64 = sdiv <2 x i64> undef, <i64 8, i64 16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V4i64 = sdiv <4 x i64> undef, <i64 2, i64 4, i64 8, i64 16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V8i64 = sdiv <8 x i64> undef, <i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I32 = sdiv i32 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V2i32 = sdiv <2 x i32> undef, <i32 2, i32 4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V4i32 = sdiv <4 x i32> undef, <i32 2, i32 4, i32 8, i32 16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V8i32 = sdiv <8 x i32> undef, <i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V16i32 = sdiv <16 x i32> undef, <i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256, i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I16 = sdiv i16 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V2i16 = sdiv <2 x i16> undef, <i16 2, i16 4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V4i16 = sdiv <4 x i16> undef, <i16 2, i16 4, i16 8, i16 16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8i16 = sdiv <8 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 176 for instruction: %V16i16 = sdiv <16 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 352 for instruction: %V32i16 = sdiv <32 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I8 = sdiv i8 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V2i8 = sdiv <2 x i8> undef, <i8 2, i8 4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V4i8 = sdiv <4 x i8> undef, <i8 2, i8 4, i8 8, i8 16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8i8 = sdiv <8 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 168 for instruction: %V16i8 = sdiv <16 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 336 for instruction: %V32i8 = sdiv <32 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 672 for instruction: %V64i8 = sdiv <64 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = sdiv i64 undef, 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i64 = sdiv <2 x i64> undef, <i64 8, i64 16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4i64 = sdiv <4 x i64> undef, <i64 2, i64 4, i64 8, i64 16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8i64 = sdiv <8 x i64> undef, <i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = sdiv i32 undef, 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i32 = sdiv <2 x i32> undef, <i32 2, i32 4>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4i32 = sdiv <4 x i32> undef, <i32 2, i32 4, i32 8, i32 16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8i32 = sdiv <8 x i32> undef, <i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V16i32 = sdiv <16 x i32> undef, <i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256, i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = sdiv i16 undef, 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i16 = sdiv <2 x i16> undef, <i16 2, i16 4>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4i16 = sdiv <4 x i16> undef, <i16 2, i16 4, i16 8, i16 16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8i16 = sdiv <8 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V16i16 = sdiv <16 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V32i16 = sdiv <32 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = sdiv i8 undef, 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i8 = sdiv <2 x i8> undef, <i8 2, i8 4>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4i8 = sdiv <4 x i8> undef, <i8 2, i8 4, i8 8, i8 16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8i8 = sdiv <8 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V16i8 = sdiv <16 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V32i8 = sdiv <32 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V64i8 = sdiv <64 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I128 = sdiv i128 undef, 16
@@ -637,28 +637,28 @@ define void @udiv_constpow2() {
 define void @sdiv_uniformconstpow2() {
 ; CHECK-LABEL: 'sdiv_uniformconstpow2'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I128 = sdiv i128 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I64 = sdiv i64 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2i64 = sdiv <2 x i64> undef, splat (i64 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V4i64 = sdiv <4 x i64> undef, splat (i64 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V8i64 = sdiv <8 x i64> undef, splat (i64 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I32 = sdiv i32 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2i32 = sdiv <2 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V4i32 = sdiv <4 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V8i32 = sdiv <8 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V16i32 = sdiv <16 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I16 = sdiv i16 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2i16 = sdiv <2 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V4i16 = sdiv <4 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V8i16 = sdiv <8 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V16i16 = sdiv <16 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V32i16 = sdiv <32 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I8 = sdiv i8 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2i8 = sdiv <2 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V4i8 = sdiv <4 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V8i8 = sdiv <8 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %V16i8 = sdiv <16 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V32i8 = sdiv <32 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V64i8 = sdiv <64 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = sdiv i64 undef, 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i64 = sdiv <2 x i64> undef, splat (i64 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i64 = sdiv <4 x i64> undef, splat (i64 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8i64 = sdiv <8 x i64> undef, splat (i64 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = sdiv i32 undef, 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i32 = sdiv <2 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i32 = sdiv <4 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8i32 = sdiv <8 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16i32 = sdiv <16 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = sdiv i16 undef, 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i16 = sdiv <2 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i16 = sdiv <4 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i16 = sdiv <8 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16i16 = sdiv <16 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32i16 = sdiv <32 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = sdiv i8 undef, 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i8 = sdiv <2 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i8 = sdiv <4 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i8 = sdiv <8 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i8 = sdiv <16 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32i8 = sdiv <32 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64i8 = sdiv <64 x i8> undef, splat (i8 16)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I128 = sdiv i128 undef, 16
@@ -753,28 +753,28 @@ define void @udiv_uniformconstpow2() {
 define void @sdiv_constnegpow2() {
 ; CHECK-LABEL: 'sdiv_constnegpow2'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I128 = sdiv i128 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I64 = sdiv i64 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V2i64 = sdiv <2 x i64> undef, <i64 -8, i64 -16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V4i64 = sdiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V8i64 = sdiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sdiv i32 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V2i32 = sdiv <2 x i32> undef, <i32 -2, i32 -4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V4i32 = sdiv <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V8i32 = sdiv <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V16i32 = sdiv <16 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256, i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sdiv i16 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V2i16 = sdiv <2 x i16> undef, <i16 -2, i16 -4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V4i16 = sdiv <4 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8i16 = sdiv <8 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 176 for instruction: %V16i16 = sdiv <16 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 352 for instruction: %V32i16 = sdiv <32 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = sdiv i8 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V2i8 = sdiv <2 x i8> undef, <i8 -2, i8 -4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V4i8 = sdiv <4 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8i8 = sdiv <8 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 168 for instruction: %V16i8 = sdiv <16 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 336 for instruction: %V32i8 = sdiv <32 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 672 for instruction: %V64i8 = sdiv <64 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = sdiv i64 undef, -16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i64 = sdiv <2 x i64> undef, <i64 -8, i64 -16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4i64 = sdiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8i64 = sdiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = sdiv i32 undef, -16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i32 = sdiv <2 x i32> undef, <i32 -2, i32 -4>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4i32 = sdiv <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8i32 = sdiv <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V16i32 = sdiv <16 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256, i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = sdiv i16 undef, -16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i16 = sdiv <2 x i16> undef, <i16 -2, i16 -4>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4i16 = sdiv <4 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8i16 = sdiv <8 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V16i16 = sdiv <16 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V32i16 = sdiv <32 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = sdiv i8 undef, -16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i8 = sdiv <2 x i8> undef, <i8 -2, i8 -4>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4i8 = sdiv <4 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8i8 = sdiv <8 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V16i8 = sdiv <16 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V32i8 = sdiv <32 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V64i8 = sdiv <64 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I128 = sdiv i128 undef, -16
@@ -869,28 +869,28 @@ define void @udiv_constnegpow2() {
 define void @sdiv_uniformconstnegpow2() {
 ; CHECK-LABEL: 'sdiv_uniformconstnegpow2'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I128 = sdiv i128 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I64 = sdiv i64 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2i64 = sdiv <2 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4i64 = sdiv <4 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8i64 = sdiv <8 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = sdiv i32 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2i32 = sdiv <2 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4i32 = sdiv <4 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8i32 = sdiv <8 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V16i32 = sdiv <16 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = sdiv i16 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2i16 = sdiv <2 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4i16 = sdiv <4 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i16 = sdiv <8 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V16i16 = sdiv <16 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %V32i16 = sdiv <32 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = sdiv i8 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2i8 = sdiv <2 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4i8 = sdiv <4 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8i8 = sdiv <8 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i8 = sdiv <16 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %V32i8 = sdiv <32 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %V64i8 = sdiv <64 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = sdiv i64 undef, -16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i64 = sdiv <2 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4i64 = sdiv <4 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8i64 = sdiv <8 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = sdiv i32 undef, -16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i32 = sdiv <2 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i32 = sdiv <4 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8i32 = sdiv <8 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16i32 = sdiv <16 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = sdiv i16 undef, -16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i16 = sdiv <2 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i16 = sdiv <4 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i16 = sdiv <8 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16i16 = sdiv <16 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32i16 = sdiv <32 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = sdiv i8 undef, -16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i8 = sdiv <2 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i8 = sdiv <4 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i8 = sdiv <8 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i8 = sdiv <16 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32i8 = sdiv <32 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64i8 = sdiv <64 x i8> undef, splat (i8 -16)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I128 = sdiv i128 undef, -16
diff --git a/llvm/test/Analysis/CostModel/AArch64/div_cte.ll b/llvm/test/Analysis/CostModel/AArch64/div_cte.ll
index 68e287477042d..e92d827db02e2 100644
--- a/llvm/test/Analysis/CostModel/AArch64/div_cte.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/div_cte.ll
@@ -7,7 +7,7 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define <16 x i8> @sdiv8xi16(<16 x i8> %x) {
 ; CHECK-LABEL: 'sdiv8xi16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %div = sdiv <16 x i8> %x, splat (i8 9)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %div = sdiv <16 x i8> %x, splat (i8 9)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %div
 ;
   %div = sdiv <16 x i8> %x, <i8 9, i8 9, i8 9, i8 9, i8 9, i8 9, i8 9, i8 9, i8 9, i8 9, i8 9, i8 9, i8 9, i8 9, i8 9, i8 9>
@@ -16,7 +16,7 @@ define <16 x i8> @sdiv8xi16(<16 x i8> %x) {
 
 define <8 x i16> @sdiv16xi8(<8 x i16> %x) {
 ; CHECK-LABEL: 'sdiv16xi8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %div = sdiv <8 x i16> %x, splat (i16 9)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %div = sdiv <8 x i16> %x, splat (i16 9)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %div
 ;
   %div = sdiv <8 x i16> %x, <i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9>
@@ -25,7 +25,7 @@ define <8 x i16> @sdiv16xi8(<8 x i16> %x) {
 
 define <4 x i32> @sdiv32xi4(<4 x i32> %x) {
 ; CHECK-LABEL: 'sdiv32xi4'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %div = sdiv <4 x i32> %x, splat (i32 9)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %div = sdiv <4 x i32> %x, splat (i32 9)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %div
 ;
   %div = sdiv <4 x i32> %x, <i32 9, i32 9, i32 9, i32 9>
diff --git a/llvm/test/Analysis/CostModel/AArch64/fshl.ll b/llvm/test/Analysis/CostModel/AArch64/fshl.ll
index 8c6466ab470b4..59e86aefeeb6c 100644
--- a/llvm/test/Analysis/CostModel/AArch64/fshl.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/fshl.ll
@@ -1,17 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 2
-; RUN: opt -passes="print<cost-model>" -cost-kind=throughput -disable-output -mtriple=arm64-apple-ios < %s 2>&1 | FileCheck %s --check-prefix=RECIP
-; RUN: opt -passes="print<cost-model>" -cost-kind=code-size -disable-output -mtriple=arm64-apple-ios < %s 2>&1 | FileCheck %s --check-prefix=SIZE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define i8 @fshl_i8_3rd_arg_const(i8 %a, i8 %b) {
-; RECIP-LABEL: 'fshl_i8_3rd_arg_const'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fshl = tail call i8 @llvm.fshl.i8(i8 %a, i8 %b, i8 9)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %fshl
-;
-; SIZE-LABEL: 'fshl_i8_3rd_arg_const'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fshl = tail call i8 @llvm.fshl.i8(i8 %a, i8 %b, i8 9)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %fshl
+; CHECK-LABEL: 'fshl_i8_3rd_arg_const'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %fshl = tail call i8 @llvm.fshl.i8(i8 %a, i8 %b, i8 9)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %fshl
 ;
 entry:
   %fshl = tail call i8 @llvm.fshl.i8(i8 %a, i8 %b, i8 9)
@@ -19,13 +14,9 @@ entry:
 }
 
 define i8 @fshl_i8_3rd_arg_var(i8 %a, i8 %b, i8 %c) {
-; RECIP-LABEL: 'fshl_i8_3rd_arg_var'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshl = tail call i8 @llvm.fshl.i8(i8 %a, i8 %b, i8 %c)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %fshl
-;
-; SIZE-LABEL: 'fshl_i8_3rd_arg_var'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshl = tail call i8 @llvm.fshl.i8(i8 %a, i8 %b, i8 %c)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %fshl
+; CHECK-LABEL: 'fshl_i8_3rd_arg_var'
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %fshl = tail call i8 @llvm.fshl.i8(i8 %a, i8 %b, i8 %c)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %fshl
 ;
 entry:
   %fshl = tail call i8 @llvm.fshl.i8(i8 %a, i8 %b, i8 %c)
@@ -35,13 +26,9 @@ entry:
 declare i8 @llvm.fshl.i8(i8, i8, i8)
 
 define i16 @fshl_i16(i16 %a, i16 %b) {
-; RECIP-LABEL: 'fshl_i16'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fshl = tail call i16 @llvm.fshl.i16(i16 %a, i16 %b, i16 9)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %fshl
-;
-; SIZE-LABEL: 'fshl_i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fshl = tail call i16 @llvm.fshl.i16(i16 %a, i16 %b, i16 9)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %fshl
+; CHECK-LABEL: 'fshl_i16'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %fshl = tail call i16 @llvm.fshl.i16(i16 %a, i16 %b, i16 9)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %fshl
 ;
 entry:
   %fshl = tail call i16 @llvm.fshl.i16(i16 %a, i16 %b, i16 9)
@@ -51,13 +38,9 @@ entry:
 declare i16 @llvm.fshl.i16(i16, i16, i16)
 
 define i32 @fshl_i32_3rd_arg_const(i32 %a, i32 %b) {
-; RECIP-LABEL: 'fshl_i32_3rd_arg_const'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %fshl = tail call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 9)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %fshl
-;
-; SIZE-LABEL: 'fshl_i32_3rd_arg_const'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %fshl = tail call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 9)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %fshl
+; CHECK-LABEL: 'fshl_i32_3rd_arg_const'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %fshl = tail call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 9)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %fshl
 ;
 entry:
   %fshl = tail call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 9)
@@ -65,13 +48,9 @@ entry:
 }
 
 define i32 @fshl_i32_3rd_arg_var(i32 %a, i32 %b, i32 %c) {
-; RECIP-LABEL: 'fshl_i32_3rd_arg_var'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshl = tail call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 %c)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %fshl
-;
-; SIZE-LABEL: 'fshl_i32_3rd_arg_var'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshl = tail call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 %c)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %fshl
+; CHECK-LABEL: 'fshl_i32_3rd_arg_var'
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %fshl = tail call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 %c)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %fshl
 ;
 entry:
   %fshl = tail call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 %c)
@@ -81,13 +60,9 @@ entry:
 declare i32 @llvm.fshl.i32(i32, i32, i32)
 
 define i64 @fshl_i64_3rd_arg_const(i64 %a, i64 %b) {
-; RECIP-LABEL: 'fshl_i64_3rd_arg_const'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %fshl = tail call i64 @llvm.fshl.i64(i64 %a, i64 %b, i64 9)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %fshl
-;
-; SIZE-LABEL: 'fshl_i64_3rd_arg_const'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %fshl = tail call i64 @llvm.fshl.i64(i64 %a, i64 %b, i64 9)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %fshl
+; CHECK-LABEL: 'fshl_i64_3rd_arg_const'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %fshl = tail call i64 @llvm.fshl.i64(i64 %a, i64 %b, i64 9)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %fshl
 ;
 entry:
   %fshl = tail call i64 @llvm.fshl.i64(i64 %a, i64 %b, i64 9)
@@ -95,13 +70,9 @@ entry:
 }
 
 define i64 @fshl_i64_3rd_arg_var(i64 %a, i64 %b, i64 %c) {
-; RECIP-LABEL: 'fshl_i64_3rd_arg_var'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshl = tail call i64 @llvm.fshl.i64(i64 %a, i64 %b, i64 %c)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %fshl
-;
-; SIZE-LABEL: 'fshl_i64_3rd_arg_var'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshl = tail call i64 @llvm.fshl.i64(i64 %a, i64 %b, i64 %c)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %fshl
+; CHECK-LABEL: 'fshl_i64_3rd_arg_var'
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %fshl = tail call i64 @llvm.fshl.i64(i64 %a, i64 %b, i64 %c)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %fshl
 ;
 entry:
   %fshl = tail call i64 @llvm.fshl.i64(i64 %a, i64 %b, i64 %c)
@@ -111,13 +82,9 @@ entry:
 declare i64 @llvm.fshl.i64(i64, i64, i64)
 
 define i19 @fshl_i19(i19 %a, i19 %b) {
-; RECIP-LABEL: 'fshl_i19'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fshl = tail call i19 @llvm.fshl.i19(i19 %a, i19 %b, i19 9)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i19 %fshl
-;
-; SIZE-LABEL: 'fshl_i19'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fshl = tail call i19 @llvm.fshl.i19(i19 %a, i19 %b, i19 9)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i19 %fshl
+; CHECK-LABEL: 'fshl_i19'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %fshl = tail call i19 @llvm.fshl.i19(i19 %a, i19 %b, i19 9)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i19 %fshl
 ;
 entry:
   %fshl = tail call i19 @llvm.fshl.i19(i19 %a, i19 %b, i19 9)
@@ -128,13 +95,9 @@ declare i19 @llvm.fshl.i19(i19, i19, i19)
 
 
 define <16 x i8> @fshl_v16i8_3rd_arg_vec_const_all_lanes_same(<16 x i8> %a, <16 x i8> %b) {
-; RECIP-LABEL: 'fshl_v16i8_3rd_arg_vec_const_all_lanes_same'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fshl = tail call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> splat (i8 3))
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %fshl
-;
-; SIZE-LABEL: 'fshl_v16i8_3rd_arg_vec_const_all_lanes_same'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fshl = tail call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> splat (i8 3))
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %fshl
+; CHECK-LABEL: 'fshl_v16i8_3rd_arg_vec_const_all_lanes_same'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %fshl = tail call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> splat (i8 3))
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %fshl
 ;
 entry:
   %fshl = tail call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
@@ -142,13 +105,9 @@ entry:
 }
 
 define <16 x i8> @fshl_v16i8_3rd_arg_vec_const_lanes_different(<16 x i8> %a, <16 x i8> %b) {
-; RECIP-LABEL: 'fshl_v16i8_3rd_arg_vec_const_lanes_different'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshl = tail call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 9, i8 1, i8 13, i8 7, i8 31, i8 23, i8 43, i8 51, i8 3, i8 3, i8 17, i8 3, i8 11, i8 15, i8 3, i8 3>)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %fshl
-;
-; SIZE-LABEL: 'fshl_v16i8_3rd_arg_vec_const_lanes_different'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshl = tail call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 9, i8 1, i8 13, i8 7, i8 31, i8 23, i8 43, i8 51, i8 3, i8 3, i8 17, i8 3, i8 11, i8 15, i8 3, i8 3>)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %fshl
+; CHECK-LABEL: 'fshl_v16i8_3rd_arg_vec_const_lanes_different'
+; CHECK-NEXT:  Cost Model: Found costs of 6 for: %fshl = tail call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 9, i8 1, i8 13, i8 7, i8 31, i8 23, i8 43, i8 51, i8 3, i8 3, i8 17, i8 3, i8 11, i8 15, i8 3, i8 3>)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %fshl
 ;
 entry:
   %fshl = tail call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 9, i8 1, i8 13, i8 7, i8 31, i8 23, i8 43, i8 51, i8 3, i8 3, i8 17, i8 3, i8 11, i8 15, i8 3, i8 3>)
@@ -156,13 +115,9 @@ entry:
 }
 
 define <16 x i8> @fshl_v16i8_3rd_arg_var(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
-; RECIP-LABEL: 'fshl_v16i8_3rd_arg_var'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshl = tail call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %fshl
-;
-; SIZE-LABEL: 'fshl_v16i8_3rd_arg_var'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshl = tail call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %fshl
+; CHECK-LABEL: 'fshl_v16i8_3rd_arg_var'
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %fshl = tail call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %fshl
 ;
 entry:
   %fshl = tail call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c)
@@ -172,13 +127,9 @@ entry:
 declare <16 x i8> @llvm.fshl.v16i8(<16 x i8>, <16 x i8>, <16 x i8>)
 
 define <8 x i16> @fshl_v8i16_3rd_arg_vec_const_all_lanes_same(<8 x i16> %a, <8 x i16> %b) {
-; RECIP-LABEL: 'fshl_v8i16_3rd_arg_vec_const_all_lanes_same'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fshl = tail call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> splat (i16 3))
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %fshl
-;
-; SIZE-LABEL: 'fshl_v8i16_3rd_arg_vec_const_all_lanes_same'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fshl = tail call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> splat (i16 3))
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %fshl
+; CHECK-LABEL: 'fshl_v8i16_3rd_arg_vec_const_all_lanes_same'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %fshl = tail call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> splat (i16 3))
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i16> %fshl
 ;
 entry:
   %fshl = tail call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
@@ -186,13 +137,9 @@ entry:
 }
 
 define <8 x i16> @fshl_v8i16_3rd_arg_vec_const_lanes_different(<8 x i16> %a, <8 x i16> %b) {
-; RECIP-LABEL: 'fshl_v8i16_3rd_arg_vec_const_lanes_different'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshl = tail call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> <i16 3, i16 1, i16 13, i16 8, i16 7, i16 31, i16 43, i16 51>)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %fshl
-;
-; SIZE-LABEL: 'fshl_v8i16_3rd_arg_vec_const_lanes_different'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshl = tail call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> <i16 3, i16 1, i16 13, i16 8, i16 7, i16 31, i16 43, i16 51>)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %fshl
+; CHECK-LABEL: 'fshl_v8i16_3rd_arg_vec_const_lanes_different'
+; CHECK-NEXT:  Cost Model: Found costs of 6 for: %fshl = tail call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> <i16 3, i16 1, i16 13, i16 8, i16 7, i16 31, i16 43, i16 51>)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i16> %fshl
 ;
 entry:
   %fshl = tail call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> <i16 3, i16 1, i16 13, i16 8, i16 7, i16 31, i16 43, i16 51>)
@@ -200,13 +147,9 @@ entry:
 }
 
 define <8 x i16> @fshl_v8i16_3rd_arg_var(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
-; RECIP-LABEL: 'fshl_v8i16_3rd_arg_var'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshl = tail call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %fshl
-;
-; SIZE-LABEL: 'fshl_v8i16_3rd_arg_var'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshl = tail call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %fshl
+; CHECK-LABEL: 'fshl_v8i16_3rd_arg_var'
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %fshl = tail call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i16> %fshl
 ;
 entry:
   %fshl = tail call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c)
@@ -216,13 +159,9 @@ entry:
 declare <8 x i16> @llvm.fshl.v8i16(<8 x i16>, <8 x i16>, <8 x i16>)
 
 define <4 x i32> @fshl_v4i32_3rd_arg_vec_const_all_lanes_same(<4 x i32> %a, <4 x i32> %b) {
-; RECIP-LABEL: 'fshl_v4i32_3rd_arg_vec_const_all_lanes_same'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fshl = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> splat (i32 3))
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %fshl
-;
-; SIZE-LABEL: 'fshl_v4i32_3rd_arg_vec_const_all_lanes_same'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fshl = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> splat (i32 3))
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %fshl
+; CHECK-LABEL: 'fshl_v4i32_3rd_arg_vec_const_all_lanes_same'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %fshl = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> splat (i32 3))
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %fshl
 ;
 entry:
   %fshl = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 3, i32 3, i32 3, i32 3>)
@@ -230,13 +169,9 @@ entry:
 }
 
 define <4 x i32> @fshl_v4i32_3rd_arg_vec_const_lanes_different(<4 x i32> %a, <4 x i32> %b) {
-; RECIP-LABEL: 'fshl_v4i32_3rd_arg_vec_const_lanes_different'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshl = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 1, i32 3, i32 11, i32 2>)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %fshl
-;
-; SIZE-LABEL: 'fshl_v4i32_3rd_arg_vec_const_lanes_different'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshl = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 1, i32 3, i32 11, i32 2>)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %fshl
+; CHECK-LABEL: 'fshl_v4i32_3rd_arg_vec_const_lanes_different'
+; CHECK-NEXT:  Cost Model: Found costs of 6 for: %fshl = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 1, i32 3, i32 11, i32 2>)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %fshl
 ;
 entry:
   %fshl = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 1, i32 3, i32 11, i32 2>)
@@ -244,13 +179,9 @@ entry:
 }
 
 define <4 x i32> @fshl_v4i32_3rd_arg_var(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
-; RECIP-LABEL: 'fshl_v4i32_3rd_arg_var'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshl = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %fshl
-;
-; SIZE-LABEL: 'fshl_v4i32_3rd_arg_var'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshl = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %fshl
+; CHECK-LABEL: 'fshl_v4i32_3rd_arg_var'
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %fshl = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %fshl
 ;
 entry:
   %fshl = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c)
@@ -260,13 +191,9 @@ entry:
 declare <4 x i32> @llvm.fshl.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
 
 define <2 x i64> @fshl_v2i64_3rd_arg_vec_const_all_lanes_same(<2 x i64> %a, <2 x i64> %b) {
-; RECIP-LABEL: 'fshl_v2i64_3rd_arg_vec_const_all_lanes_same'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fshl = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> splat (i64 1))
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %fshl
-;
-; SIZE-LABEL: 'fshl_v2i64_3rd_arg_vec_const_all_lanes_same'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fshl = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> splat (i64 1))
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %fshl
+; CHECK-LABEL: 'fshl_v2i64_3rd_arg_vec_const_all_lanes_same'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %fshl = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> splat (i64 1))
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i64> %fshl
 ;
 entry:
   %fshl = tail call <2 x i64> @llvm.fshl.v4i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> <i64 1, i64 1>)
@@ -274,13 +201,9 @@ entry:
 }
 
 define <2 x i64> @fshl_v2i64_3rd_arg_vec_const_lanes_different(<2 x i64> %a, <2 x i64> %b) {
-; RECIP-LABEL: 'fshl_v2i64_3rd_arg_vec_const_lanes_different'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshl = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> <i64 1, i64 2>)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %fshl
-;
-; SIZE-LABEL: 'fshl_v2i64_3rd_arg_vec_const_lanes_different'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshl = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> <i64 1, i64 2>)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %fshl
+; CHECK-LABEL: 'fshl_v2i64_3rd_arg_vec_const_lanes_different'
+; CHECK-NEXT:  Cost Model: Found costs of 6 for: %fshl = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> <i64 1, i64 2>)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i64> %fshl
 ;
 entry:
   %fshl = tail call <2 x i64> @llvm.fshl.v4i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> <i64 1, i64 2>)
@@ -288,13 +211,9 @@ entry:
 }
 
 define <2 x i64> @fshl_v2i64_3rd_arg_var(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
-; RECIP-LABEL: 'fshl_v2i64_3rd_arg_var'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshl = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %fshl
-;
-; SIZE-LABEL: 'fshl_v2i64_3rd_arg_var'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshl = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %fshl
+; CHECK-LABEL: 'fshl_v2i64_3rd_arg_var'
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %fshl = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i64> %fshl
 ;
 entry:
   %fshl = tail call <2 x i64> @llvm.fshl.v4i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c)
@@ -304,13 +223,9 @@ entry:
 declare <2 x i64> @llvm.fshl.v4i64(<2 x i64>, <2 x i64>, <2 x i64>)
 
 define <4 x i30> @fshl_v4i30_3rd_arg_var(<4 x i30> %a, <4 x i30> %b, <4 x i30> %c) {
-; RECIP-LABEL: 'fshl_v4i30_3rd_arg_var'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %fshl = tail call <4 x i30> @llvm.fshl.v4i30(<4 x i30> %a, <4 x i30> %b, <4 x i30> %c)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i30> %fshl
-;
-; SIZE-LABEL: 'fshl_v4i30_3rd_arg_var'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %fshl = tail call <4 x i30> @llvm.fshl.v4i30(<4 x i30> %a, <4 x i30> %b, <4 x i30> %c)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i30> %fshl
+; CHECK-LABEL: 'fshl_v4i30_3rd_arg_var'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:10 Lat:10 SizeLat:10 for: %fshl = tail call <4 x i30> @llvm.fshl.v4i30(<4 x i30> %a, <4 x i30> %b, <4 x i30> %c)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i30> %fshl
 ;
 entry:
   %fshl = tail call <4 x i30> @llvm.fshl.v4i30(<4 x i30> %a, <4 x i30> %b, <4 x i30> %c)
@@ -320,13 +235,9 @@ entry:
 declare <4 x i30> @llvm.fshl.v4i30(<4 x i30>, <4 x i30>, <4 x i30>)
 
 define <2 x i66> @fshl_v2i66_3rd_arg_vec_const_lanes_different(<2 x i66> %a, <2 x i66> %b) {
-; RECIP-LABEL: 'fshl_v2i66_3rd_arg_vec_const_lanes_different'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %fshl = tail call <2 x i66> @llvm.fshl.v2i66(<2 x i66> %a, <2 x i66> %b, <2 x i66> <i66 1, i66 2>)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i66> %fshl
-;
-; SIZE-LABEL: 'fshl_v2i66_3rd_arg_vec_const_lanes_different'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshl = tail call <2 x i66> @llvm.fshl.v2i66(<2 x i66> %a, <2 x i66> %b, <2 x i66> <i66 1, i66 2>)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i66> %fshl
+; CHECK-LABEL: 'fshl_v2i66_3rd_arg_vec_const_lanes_different'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:32 CodeSize:6 Lat:6 SizeLat:6 for: %fshl = tail call <2 x i66> @llvm.fshl.v2i66(<2 x i66> %a, <2 x i66> %b, <2 x i66> <i66 1, i66 2>)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i66> %fshl
 ;
 entry:
   %fshl = tail call <2 x i66> @llvm.fshl.v4i66(<2 x i66> %a, <2 x i66> %b, <2 x i66> <i66 1, i66 2>)
@@ -335,13 +246,9 @@ entry:
 declare <2 x i66> @llvm.fshl.v4i66(<2 x i66>, <2 x i66>, <2 x i66>)
 
 define i66 @fshl_i66(i66 %a, i66 %b) {
-; RECIP-LABEL: 'fshl_i66'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fshl = tail call i66 @llvm.fshl.i66(i66 %a, i66 %b, i66 9)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i66 %fshl
-;
-; SIZE-LABEL: 'fshl_i66'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fshl = tail call i66 @llvm.fshl.i66(i66 %a, i66 %b, i66 9)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i66 %fshl
+; CHECK-LABEL: 'fshl_i66'
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %fshl = tail call i66 @llvm.fshl.i66(i66 %a, i66 %b, i66 9)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i66 %fshl
 ;
 entry:
   %fshl = tail call i66 @llvm.fshl.i66(i66 %a, i66 %b, i66 9)
@@ -351,13 +258,9 @@ entry:
 declare i66 @llvm.fshl.i66(i66, i66, i66)
 
 define <2 x i128> @fshl_v2i128_3rd_arg_vec_const_lanes_different(<2 x i128> %a, <2 x i128> %b) {
-; RECIP-LABEL: 'fshl_v2i128_3rd_arg_vec_const_lanes_different'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %fshl = tail call <2 x i128> @llvm.fshl.v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> <i128 1, i128 2>)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i128> %fshl
-;
-; SIZE-LABEL: 'fshl_v2i128_3rd_arg_vec_const_lanes_different'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshl = tail call <2 x i128> @llvm.fshl.v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> <i128 1, i128 2>)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i128> %fshl
+; CHECK-LABEL: 'fshl_v2i128_3rd_arg_vec_const_lanes_different'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:32 CodeSize:6 Lat:6 SizeLat:6 for: %fshl = tail call <2 x i128> @llvm.fshl.v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> <i128 1, i128 2>)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i128> %fshl
 ;
 entry:
   %fshl = tail call <2 x i128> @llvm.fshl.v4i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> <i128 1, i128 2>)
@@ -366,13 +269,9 @@ entry:
 declare <2 x i128> @llvm.fshl.v4i128(<2 x i128>, <2 x i128>, <2 x i128>)
 
 define i128 @fshl_i128(i128 %a, i128 %b) {
-; RECIP-LABEL: 'fshl_i128'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %fshl = tail call i128 @llvm.fshl.i128(i128 %a, i128 %b, i128 9)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i128 %fshl
-;
-; SIZE-LABEL: 'fshl_i128'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshl = tail call i128 @llvm.fshl.i128(i128 %a, i128 %b, i128 9)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i128 %fshl
+; CHECK-LABEL: 'fshl_i128'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:6 Lat:6 SizeLat:6 for: %fshl = tail call i128 @llvm.fshl.i128(i128 %a, i128 %b, i128 9)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i128 %fshl
 ;
 entry:
   %fshl = tail call i128 @llvm.fshl.i128(i128 %a, i128 %b, i128 9)
diff --git a/llvm/test/Analysis/CostModel/AArch64/fshr.ll b/llvm/test/Analysis/CostModel/AArch64/fshr.ll
index 120b1c4c4c4ef..53cede900c398 100644
--- a/llvm/test/Analysis/CostModel/AArch64/fshr.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/fshr.ll
@@ -1,17 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 2
-; RUN: opt -passes="print<cost-model>" -cost-kind=throughput -disable-output -mtriple=arm64-apple-ios < %s 2>&1 | FileCheck %s --check-prefix=RECIP
-; RUN: opt -passes="print<cost-model>" -cost-kind=code-size -disable-output -mtriple=arm64-apple-ios < %s 2>&1 | FileCheck %s --check-prefix=SIZE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define i8 @fshr_i8_3rd_arg_const(i8 %a, i8 %b) {
-; RECIP-LABEL: 'fshr_i8_3rd_arg_const'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fshr = tail call i8 @llvm.fshr.i8(i8 %a, i8 %b, i8 9)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %fshr
-;
-; SIZE-LABEL: 'fshr_i8_3rd_arg_const'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fshr = tail call i8 @llvm.fshr.i8(i8 %a, i8 %b, i8 9)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %fshr
+; CHECK-LABEL: 'fshr_i8_3rd_arg_const'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %fshr = tail call i8 @llvm.fshr.i8(i8 %a, i8 %b, i8 9)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %fshr
 ;
 entry:
   %fshr = tail call i8 @llvm.fshr.i8(i8 %a, i8 %b, i8 9)
@@ -19,13 +14,9 @@ entry:
 }
 
 define i8 @fshr_i8_3rd_arg_var(i8 %a, i8 %b, i8 %c) {
-; RECIP-LABEL: 'fshr_i8_3rd_arg_var'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshr = tail call i8 @llvm.fshr.i8(i8 %a, i8 %b, i8 %c)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %fshr
-;
-; SIZE-LABEL: 'fshr_i8_3rd_arg_var'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshr = tail call i8 @llvm.fshr.i8(i8 %a, i8 %b, i8 %c)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %fshr
+; CHECK-LABEL: 'fshr_i8_3rd_arg_var'
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %fshr = tail call i8 @llvm.fshr.i8(i8 %a, i8 %b, i8 %c)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %fshr
 ;
 entry:
   %fshr = tail call i8 @llvm.fshr.i8(i8 %a, i8 %b, i8 %c)
@@ -35,13 +26,9 @@ entry:
 declare i8 @llvm.fshr.i8(i8, i8, i8)
 
 define i16 @fshr_i16(i16 %a, i16 %b) {
-; RECIP-LABEL: 'fshr_i16'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fshr = tail call i16 @llvm.fshr.i16(i16 %a, i16 %b, i16 9)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %fshr
-;
-; SIZE-LABEL: 'fshr_i16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fshr = tail call i16 @llvm.fshr.i16(i16 %a, i16 %b, i16 9)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %fshr
+; CHECK-LABEL: 'fshr_i16'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %fshr = tail call i16 @llvm.fshr.i16(i16 %a, i16 %b, i16 9)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %fshr
 ;
 entry:
   %fshr = tail call i16 @llvm.fshr.i16(i16 %a, i16 %b, i16 9)
@@ -51,13 +38,9 @@ entry:
 declare i16 @llvm.fshr.i16(i16, i16, i16)
 
 define i32 @fshr_i32_3rd_arg_const(i32 %a, i32 %b) {
-; RECIP-LABEL: 'fshr_i32_3rd_arg_const'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %fshr = tail call i32 @llvm.fshr.i32(i32 %a, i32 %b, i32 9)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %fshr
-;
-; SIZE-LABEL: 'fshr_i32_3rd_arg_const'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %fshr = tail call i32 @llvm.fshr.i32(i32 %a, i32 %b, i32 9)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %fshr
+; CHECK-LABEL: 'fshr_i32_3rd_arg_const'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %fshr = tail call i32 @llvm.fshr.i32(i32 %a, i32 %b, i32 9)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %fshr
 ;
 entry:
   %fshr = tail call i32 @llvm.fshr.i32(i32 %a, i32 %b, i32 9)
@@ -65,13 +48,9 @@ entry:
 }
 
 define i32 @fshr_i32_3rd_arg_var(i32 %a, i32 %b, i32 %c) {
-; RECIP-LABEL: 'fshr_i32_3rd_arg_var'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshr = tail call i32 @llvm.fshr.i32(i32 %a, i32 %b, i32 %c)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %fshr
-;
-; SIZE-LABEL: 'fshr_i32_3rd_arg_var'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshr = tail call i32 @llvm.fshr.i32(i32 %a, i32 %b, i32 %c)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %fshr
+; CHECK-LABEL: 'fshr_i32_3rd_arg_var'
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %fshr = tail call i32 @llvm.fshr.i32(i32 %a, i32 %b, i32 %c)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %fshr
 ;
 entry:
   %fshr = tail call i32 @llvm.fshr.i32(i32 %a, i32 %b, i32 %c)
@@ -81,13 +60,9 @@ entry:
 declare i32 @llvm.fshr.i32(i32, i32, i32)
 
 define i64 @fshr_i64_3rd_arg_const(i64 %a, i64 %b) {
-; RECIP-LABEL: 'fshr_i64_3rd_arg_const'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %fshr = tail call i64 @llvm.fshr.i64(i64 %a, i64 %b, i64 9)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %fshr
-;
-; SIZE-LABEL: 'fshr_i64_3rd_arg_const'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %fshr = tail call i64 @llvm.fshr.i64(i64 %a, i64 %b, i64 9)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %fshr
+; CHECK-LABEL: 'fshr_i64_3rd_arg_const'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %fshr = tail call i64 @llvm.fshr.i64(i64 %a, i64 %b, i64 9)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %fshr
 ;
 entry:
   %fshr = tail call i64 @llvm.fshr.i64(i64 %a, i64 %b, i64 9)
@@ -95,13 +70,9 @@ entry:
 }
 
 define i64 @fshr_i64_3rd_arg_var(i64 %a, i64 %b, i64 %c) {
-; RECIP-LABEL: 'fshr_i64_3rd_arg_var'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshr = tail call i64 @llvm.fshr.i64(i64 %a, i64 %b, i64 %c)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %fshr
-;
-; SIZE-LABEL: 'fshr_i64_3rd_arg_var'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshr = tail call i64 @llvm.fshr.i64(i64 %a, i64 %b, i64 %c)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %fshr
+; CHECK-LABEL: 'fshr_i64_3rd_arg_var'
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %fshr = tail call i64 @llvm.fshr.i64(i64 %a, i64 %b, i64 %c)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %fshr
 ;
 entry:
   %fshr = tail call i64 @llvm.fshr.i64(i64 %a, i64 %b, i64 %c)
@@ -111,13 +82,9 @@ entry:
 declare i64 @llvm.fshr.i64(i64, i64, i64)
 
 define i19 @fshr_i19(i19 %a, i19 %b) {
-; RECIP-LABEL: 'fshr_i19'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fshr = tail call i19 @llvm.fshr.i19(i19 %a, i19 %b, i19 9)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i19 %fshr
-;
-; SIZE-LABEL: 'fshr_i19'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fshr = tail call i19 @llvm.fshr.i19(i19 %a, i19 %b, i19 9)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i19 %fshr
+; CHECK-LABEL: 'fshr_i19'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %fshr = tail call i19 @llvm.fshr.i19(i19 %a, i19 %b, i19 9)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i19 %fshr
 ;
 entry:
   %fshr = tail call i19 @llvm.fshr.i19(i19 %a, i19 %b, i19 9)
@@ -128,13 +95,9 @@ declare i19 @llvm.fshr.i19(i19, i19, i19)
 
 
 define <16 x i8> @fshr_v16i8_3rd_arg_vec_const_all_lanes_same(<16 x i8> %a, <16 x i8> %b) {
-; RECIP-LABEL: 'fshr_v16i8_3rd_arg_vec_const_all_lanes_same'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fshr = tail call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> splat (i8 3))
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %fshr
-;
-; SIZE-LABEL: 'fshr_v16i8_3rd_arg_vec_const_all_lanes_same'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fshr = tail call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> splat (i8 3))
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %fshr
+; CHECK-LABEL: 'fshr_v16i8_3rd_arg_vec_const_all_lanes_same'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %fshr = tail call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> splat (i8 3))
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %fshr
 ;
 entry:
   %fshr = tail call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
@@ -142,13 +105,9 @@ entry:
 }
 
 define <16 x i8> @fshr_v16i8_3rd_arg_vec_const_lanes_different(<16 x i8> %a, <16 x i8> %b) {
-; RECIP-LABEL: 'fshr_v16i8_3rd_arg_vec_const_lanes_different'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshr = tail call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 9, i8 1, i8 13, i8 7, i8 31, i8 23, i8 43, i8 51, i8 3, i8 3, i8 17, i8 3, i8 11, i8 15, i8 3, i8 3>)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %fshr
-;
-; SIZE-LABEL: 'fshr_v16i8_3rd_arg_vec_const_lanes_different'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshr = tail call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 9, i8 1, i8 13, i8 7, i8 31, i8 23, i8 43, i8 51, i8 3, i8 3, i8 17, i8 3, i8 11, i8 15, i8 3, i8 3>)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %fshr
+; CHECK-LABEL: 'fshr_v16i8_3rd_arg_vec_const_lanes_different'
+; CHECK-NEXT:  Cost Model: Found costs of 6 for: %fshr = tail call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 9, i8 1, i8 13, i8 7, i8 31, i8 23, i8 43, i8 51, i8 3, i8 3, i8 17, i8 3, i8 11, i8 15, i8 3, i8 3>)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %fshr
 ;
 entry:
   %fshr = tail call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 9, i8 1, i8 13, i8 7, i8 31, i8 23, i8 43, i8 51, i8 3, i8 3, i8 17, i8 3, i8 11, i8 15, i8 3, i8 3>)
@@ -156,13 +115,9 @@ entry:
 }
 
 define <16 x i8> @fshr_v16i8_3rd_arg_var(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
-; RECIP-LABEL: 'fshr_v16i8_3rd_arg_var'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshr = tail call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %fshr
-;
-; SIZE-LABEL: 'fshr_v16i8_3rd_arg_var'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshr = tail call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %fshr
+; CHECK-LABEL: 'fshr_v16i8_3rd_arg_var'
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %fshr = tail call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %fshr
 ;
 entry:
   %fshr = tail call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c)
@@ -172,13 +127,9 @@ entry:
 declare <16 x i8> @llvm.fshr.v16i8(<16 x i8>, <16 x i8>, <16 x i8>)
 
 define <8 x i16> @fshr_v8i16_3rd_arg_vec_const_all_lanes_same(<8 x i16> %a, <8 x i16> %b) {
-; RECIP-LABEL: 'fshr_v8i16_3rd_arg_vec_const_all_lanes_same'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fshr = tail call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> splat (i16 3))
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %fshr
-;
-; SIZE-LABEL: 'fshr_v8i16_3rd_arg_vec_const_all_lanes_same'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fshr = tail call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> splat (i16 3))
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %fshr
+; CHECK-LABEL: 'fshr_v8i16_3rd_arg_vec_const_all_lanes_same'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %fshr = tail call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> splat (i16 3))
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i16> %fshr
 ;
 entry:
   %fshr = tail call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
@@ -186,13 +137,9 @@ entry:
 }
 
 define <8 x i16> @fshr_v8i16_3rd_arg_vec_const_lanes_different(<8 x i16> %a, <8 x i16> %b) {
-; RECIP-LABEL: 'fshr_v8i16_3rd_arg_vec_const_lanes_different'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshr = tail call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> <i16 3, i16 1, i16 13, i16 8, i16 7, i16 31, i16 43, i16 51>)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %fshr
-;
-; SIZE-LABEL: 'fshr_v8i16_3rd_arg_vec_const_lanes_different'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshr = tail call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> <i16 3, i16 1, i16 13, i16 8, i16 7, i16 31, i16 43, i16 51>)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %fshr
+; CHECK-LABEL: 'fshr_v8i16_3rd_arg_vec_const_lanes_different'
+; CHECK-NEXT:  Cost Model: Found costs of 6 for: %fshr = tail call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> <i16 3, i16 1, i16 13, i16 8, i16 7, i16 31, i16 43, i16 51>)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i16> %fshr
 ;
 entry:
   %fshr = tail call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> <i16 3, i16 1, i16 13, i16 8, i16 7, i16 31, i16 43, i16 51>)
@@ -200,13 +147,9 @@ entry:
 }
 
 define <8 x i16> @fshr_v8i16_3rd_arg_var(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
-; RECIP-LABEL: 'fshr_v8i16_3rd_arg_var'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshr = tail call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %fshr
-;
-; SIZE-LABEL: 'fshr_v8i16_3rd_arg_var'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshr = tail call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %fshr
+; CHECK-LABEL: 'fshr_v8i16_3rd_arg_var'
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %fshr = tail call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i16> %fshr
 ;
 entry:
   %fshr = tail call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c)
@@ -216,13 +159,9 @@ entry:
 declare <8 x i16> @llvm.fshr.v8i16(<8 x i16>, <8 x i16>, <8 x i16>)
 
 define <4 x i32> @fshr_v4i32_3rd_arg_vec_const_all_lanes_same(<4 x i32> %a, <4 x i32> %b) {
-; RECIP-LABEL: 'fshr_v4i32_3rd_arg_vec_const_all_lanes_same'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fshr = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> splat (i32 3))
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %fshr
-;
-; SIZE-LABEL: 'fshr_v4i32_3rd_arg_vec_const_all_lanes_same'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fshr = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> splat (i32 3))
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %fshr
+; CHECK-LABEL: 'fshr_v4i32_3rd_arg_vec_const_all_lanes_same'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %fshr = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> splat (i32 3))
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %fshr
 ;
 entry:
   %fshr = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 3, i32 3, i32 3, i32 3>)
@@ -230,13 +169,9 @@ entry:
 }
 
 define <4 x i32> @fshr_v4i32_3rd_arg_vec_const_lanes_different(<4 x i32> %a, <4 x i32> %b) {
-; RECIP-LABEL: 'fshr_v4i32_3rd_arg_vec_const_lanes_different'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshr = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 1, i32 3, i32 11, i32 2>)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %fshr
-;
-; SIZE-LABEL: 'fshr_v4i32_3rd_arg_vec_const_lanes_different'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshr = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 1, i32 3, i32 11, i32 2>)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %fshr
+; CHECK-LABEL: 'fshr_v4i32_3rd_arg_vec_const_lanes_different'
+; CHECK-NEXT:  Cost Model: Found costs of 6 for: %fshr = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 1, i32 3, i32 11, i32 2>)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %fshr
 ;
 entry:
   %fshr = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 1, i32 3, i32 11, i32 2>)
@@ -244,13 +179,9 @@ entry:
 }
 
 define <4 x i32> @fshr_v4i32_3rd_arg_var(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
-; RECIP-LABEL: 'fshr_v4i32_3rd_arg_var'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshr = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %fshr
-;
-; SIZE-LABEL: 'fshr_v4i32_3rd_arg_var'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshr = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %fshr
+; CHECK-LABEL: 'fshr_v4i32_3rd_arg_var'
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %fshr = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %fshr
 ;
 entry:
   %fshr = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c)
@@ -260,13 +191,9 @@ entry:
 declare <4 x i32> @llvm.fshr.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
 
 define <2 x i64> @fshr_v2i64_3rd_arg_vec_const_all_lanes_same(<2 x i64> %a, <2 x i64> %b) {
-; RECIP-LABEL: 'fshr_v2i64_3rd_arg_vec_const_all_lanes_same'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fshr = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> splat (i64 1))
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %fshr
-;
-; SIZE-LABEL: 'fshr_v2i64_3rd_arg_vec_const_all_lanes_same'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fshr = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> splat (i64 1))
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %fshr
+; CHECK-LABEL: 'fshr_v2i64_3rd_arg_vec_const_all_lanes_same'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %fshr = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> splat (i64 1))
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i64> %fshr
 ;
 entry:
   %fshr = tail call <2 x i64> @llvm.fshr.v4i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> <i64 1, i64 1>)
@@ -274,13 +201,9 @@ entry:
 }
 
 define <2 x i64> @fshr_v2i64_3rd_arg_vec_const_lanes_different(<2 x i64> %a, <2 x i64> %b) {
-; RECIP-LABEL: 'fshr_v2i64_3rd_arg_vec_const_lanes_different'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshr = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> <i64 1, i64 2>)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %fshr
-;
-; SIZE-LABEL: 'fshr_v2i64_3rd_arg_vec_const_lanes_different'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshr = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> <i64 1, i64 2>)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %fshr
+; CHECK-LABEL: 'fshr_v2i64_3rd_arg_vec_const_lanes_different'
+; CHECK-NEXT:  Cost Model: Found costs of 6 for: %fshr = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> <i64 1, i64 2>)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i64> %fshr
 ;
 entry:
   %fshr = tail call <2 x i64> @llvm.fshr.v4i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> <i64 1, i64 2>)
@@ -288,13 +211,9 @@ entry:
 }
 
 define <2 x i64> @fshr_v2i64_3rd_arg_var(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
-; RECIP-LABEL: 'fshr_v2i64_3rd_arg_var'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshr = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %fshr
-;
-; SIZE-LABEL: 'fshr_v2i64_3rd_arg_var'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %fshr = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %fshr
+; CHECK-LABEL: 'fshr_v2i64_3rd_arg_var'
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %fshr = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i64> %fshr
 ;
 entry:
   %fshr = tail call <2 x i64> @llvm.fshr.v4i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c)
@@ -304,13 +223,9 @@ entry:
 declare <2 x i64> @llvm.fshr.v4i64(<2 x i64>, <2 x i64>, <2 x i64>)
 
 define <4 x i30> @fshr_v4i30_3rd_arg_var(<4 x i30> %a, <4 x i30> %b, <4 x i30> %c) {
-; RECIP-LABEL: 'fshr_v4i30_3rd_arg_var'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %fshr = tail call <4 x i30> @llvm.fshr.v4i30(<4 x i30> %a, <4 x i30> %b, <4 x i30> %c)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i30> %fshr
-;
-; SIZE-LABEL: 'fshr_v4i30_3rd_arg_var'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %fshr = tail call <4 x i30> @llvm.fshr.v4i30(<4 x i30> %a, <4 x i30> %b, <4 x i30> %c)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i30> %fshr
+; CHECK-LABEL: 'fshr_v4i30_3rd_arg_var'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:10 Lat:10 SizeLat:10 for: %fshr = tail call <4 x i30> @llvm.fshr.v4i30(<4 x i30> %a, <4 x i30> %b, <4 x i30> %c)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i30> %fshr
 ;
 entry:
   %fshr = tail call <4 x i30> @llvm.fshr.v4i30(<4 x i30> %a, <4 x i30> %b, <4 x i30> %c)
@@ -320,13 +235,9 @@ entry:
 declare <4 x i30> @llvm.fshr.v4i30(<4 x i30>, <4 x i30>, <4 x i30>)
 
 define <2 x i66> @fshr_v2i66_3rd_arg_vec_const_lanes_different(<2 x i66> %a, <2 x i66> %b) {
-; RECIP-LABEL: 'fshr_v2i66_3rd_arg_vec_const_lanes_different'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %fshr = tail call <2 x i66> @llvm.fshr.v2i66(<2 x i66> %a, <2 x i66> %b, <2 x i66> <i66 1, i66 2>)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i66> %fshr
-;
-; SIZE-LABEL: 'fshr_v2i66_3rd_arg_vec_const_lanes_different'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshr = tail call <2 x i66> @llvm.fshr.v2i66(<2 x i66> %a, <2 x i66> %b, <2 x i66> <i66 1, i66 2>)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i66> %fshr
+; CHECK-LABEL: 'fshr_v2i66_3rd_arg_vec_const_lanes_different'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:32 CodeSize:6 Lat:6 SizeLat:6 for: %fshr = tail call <2 x i66> @llvm.fshr.v2i66(<2 x i66> %a, <2 x i66> %b, <2 x i66> <i66 1, i66 2>)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i66> %fshr
 ;
 entry:
   %fshr = tail call <2 x i66> @llvm.fshr.v4i66(<2 x i66> %a, <2 x i66> %b, <2 x i66> <i66 1, i66 2>)
@@ -335,13 +246,9 @@ entry:
 declare <2 x i66> @llvm.fshr.v4i66(<2 x i66>, <2 x i66>, <2 x i66>)
 
 define i66 @fshr_i66(i66 %a, i66 %b) {
-; RECIP-LABEL: 'fshr_i66'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fshr = tail call i66 @llvm.fshr.i66(i66 %a, i66 %b, i66 9)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i66 %fshr
-;
-; SIZE-LABEL: 'fshr_i66'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fshr = tail call i66 @llvm.fshr.i66(i66 %a, i66 %b, i66 9)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i66 %fshr
+; CHECK-LABEL: 'fshr_i66'
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %fshr = tail call i66 @llvm.fshr.i66(i66 %a, i66 %b, i66 9)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i66 %fshr
 ;
 entry:
   %fshr = tail call i66 @llvm.fshr.i66(i66 %a, i66 %b, i66 9)
@@ -351,13 +258,9 @@ entry:
 declare i66 @llvm.fshr.i66(i66, i66, i66)
 
 define <2 x i128> @fshr_v2i128_3rd_arg_vec_const_lanes_different(<2 x i128> %a, <2 x i128> %b) {
-; RECIP-LABEL: 'fshr_v2i128_3rd_arg_vec_const_lanes_different'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %fshr = tail call <2 x i128> @llvm.fshr.v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> <i128 1, i128 2>)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i128> %fshr
-;
-; SIZE-LABEL: 'fshr_v2i128_3rd_arg_vec_const_lanes_different'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshr = tail call <2 x i128> @llvm.fshr.v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> <i128 1, i128 2>)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i128> %fshr
+; CHECK-LABEL: 'fshr_v2i128_3rd_arg_vec_const_lanes_different'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:32 CodeSize:6 Lat:6 SizeLat:6 for: %fshr = tail call <2 x i128> @llvm.fshr.v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> <i128 1, i128 2>)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i128> %fshr
 ;
 entry:
   %fshr = tail call <2 x i128> @llvm.fshr.v4i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> <i128 1, i128 2>)
@@ -366,13 +269,9 @@ entry:
 declare <2 x i128> @llvm.fshr.v4i128(<2 x i128>, <2 x i128>, <2 x i128>)
 
 define i128 @fshr_i128(i128 %a, i128 %b) {
-; RECIP-LABEL: 'fshr_i128'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %fshr = tail call i128 @llvm.fshr.i128(i128 %a, i128 %b, i128 9)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i128 %fshr
-;
-; SIZE-LABEL: 'fshr_i128'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fshr = tail call i128 @llvm.fshr.i128(i128 %a, i128 %b, i128 9)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i128 %fshr
+; CHECK-LABEL: 'fshr_i128'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:6 Lat:6 SizeLat:6 for: %fshr = tail call i128 @llvm.fshr.i128(i128 %a, i128 %b, i128 9)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i128 %fshr
 ;
 entry:
   %fshr = tail call i128 @llvm.fshr.i128(i128 %a, i128 %b, i128 9)
diff --git a/llvm/test/Analysis/CostModel/AArch64/gep.ll b/llvm/test/Analysis/CostModel/AArch64/gep.ll
index 979d105670943..a1239b246f2fb 100644
--- a/llvm/test/Analysis/CostModel/AArch64/gep.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/gep.ll
@@ -1,20 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 2
-; RUN: opt -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefix=RECIP
-; RUN: opt -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefix=SIZE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-linux-gnu"
 
 define i8 @test1(ptr %p) {
-; RECIP-LABEL: 'test1'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i8, ptr %p, i32 1
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i8, ptr %a, align 1
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %v
-;
-; SIZE-LABEL: 'test1'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i8, ptr %p, i32 1
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i8, ptr %a, align 1
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %v
+; CHECK-LABEL: 'test1'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %a = getelementptr inbounds i8, ptr %p, i32 1
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %a, align 1
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
 ;
   %a = getelementptr inbounds i8, ptr %p, i32 1
   %v = load i8, ptr %a
@@ -22,15 +16,10 @@ define i8 @test1(ptr %p) {
 }
 
 define i16 @test2(ptr %p) {
-; RECIP-LABEL: 'test2'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i16, ptr %p, i32 1
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i16, ptr %a, align 2
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %v
-;
-; SIZE-LABEL: 'test2'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i16, ptr %p, i32 1
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i16, ptr %a, align 2
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %v
+; CHECK-LABEL: 'test2'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %a = getelementptr inbounds i16, ptr %p, i32 1
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i16, ptr %a, align 2
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %v
 ;
   %a = getelementptr inbounds i16, ptr %p, i32 1
   %v = load i16, ptr %a
@@ -38,15 +27,10 @@ define i16 @test2(ptr %p) {
 }
 
 define i32 @test3(ptr %p) {
-; RECIP-LABEL: 'test3'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i32, ptr %p, i32 1
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i32, ptr %a, align 4
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %v
-;
-; SIZE-LABEL: 'test3'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i32, ptr %p, i32 1
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i32, ptr %a, align 4
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %v
+; CHECK-LABEL: 'test3'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %a = getelementptr inbounds i32, ptr %p, i32 1
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i32, ptr %a, align 4
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %v
 ;
   %a = getelementptr inbounds i32, ptr %p, i32 1
   %v = load i32, ptr %a
@@ -54,15 +38,10 @@ define i32 @test3(ptr %p) {
 }
 
 define i64 @test4(ptr %p) {
-; RECIP-LABEL: 'test4'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i64, ptr %p, i32 1
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i64, ptr %a, align 8
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %v
-;
-; SIZE-LABEL: 'test4'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i64, ptr %p, i32 1
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i64, ptr %a, align 8
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %v
+; CHECK-LABEL: 'test4'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %a = getelementptr inbounds i64, ptr %p, i32 1
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i64, ptr %a, align 8
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %v
 ;
   %a = getelementptr inbounds i64, ptr %p, i32 1
   %v = load i64, ptr %a
@@ -70,15 +49,10 @@ define i64 @test4(ptr %p) {
 }
 
 define i8 @test5(ptr %p) {
-; RECIP-LABEL: 'test5'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i8, ptr %p, i32 1024
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i8, ptr %a, align 1
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %v
-;
-; SIZE-LABEL: 'test5'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i8, ptr %p, i32 1024
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i8, ptr %a, align 1
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %v
+; CHECK-LABEL: 'test5'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %a = getelementptr inbounds i8, ptr %p, i32 1024
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %a, align 1
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
 ;
   %a = getelementptr inbounds i8, ptr %p, i32 1024
   %v = load i8, ptr %a
@@ -86,15 +60,10 @@ define i8 @test5(ptr %p) {
 }
 
 define i16 @test6(ptr %p) {
-; RECIP-LABEL: 'test6'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i16, ptr %p, i32 1024
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i16, ptr %a, align 2
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %v
-;
-; SIZE-LABEL: 'test6'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i16, ptr %p, i32 1024
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i16, ptr %a, align 2
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %v
+; CHECK-LABEL: 'test6'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %a = getelementptr inbounds i16, ptr %p, i32 1024
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i16, ptr %a, align 2
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %v
 ;
   %a = getelementptr inbounds i16, ptr %p, i32 1024
   %v = load i16, ptr %a
@@ -102,15 +71,10 @@ define i16 @test6(ptr %p) {
 }
 
 define i32 @test7(ptr %p) {
-; RECIP-LABEL: 'test7'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i32, ptr %p, i32 1024
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i32, ptr %a, align 4
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %v
-;
-; SIZE-LABEL: 'test7'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i32, ptr %p, i32 1024
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i32, ptr %a, align 4
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %v
+; CHECK-LABEL: 'test7'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %a = getelementptr inbounds i32, ptr %p, i32 1024
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i32, ptr %a, align 4
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %v
 ;
   %a = getelementptr inbounds i32, ptr %p, i32 1024
   %v = load i32, ptr %a
@@ -118,15 +82,10 @@ define i32 @test7(ptr %p) {
 }
 
 define i64 @test8(ptr %p) {
-; RECIP-LABEL: 'test8'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i64, ptr %p, i32 1024
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i64, ptr %a, align 8
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %v
-;
-; SIZE-LABEL: 'test8'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i64, ptr %p, i32 1024
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i64, ptr %a, align 8
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %v
+; CHECK-LABEL: 'test8'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %a = getelementptr inbounds i64, ptr %p, i32 1024
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i64, ptr %a, align 8
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %v
 ;
   %a = getelementptr inbounds i64, ptr %p, i32 1024
   %v = load i64, ptr %a
@@ -134,15 +93,10 @@ define i64 @test8(ptr %p) {
 }
 
 define i8 @test9(ptr %p) {
-; RECIP-LABEL: 'test9'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i8, ptr %p, i32 4096
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i8, ptr %a, align 1
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %v
-;
-; SIZE-LABEL: 'test9'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i8, ptr %p, i32 4096
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i8, ptr %a, align 1
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %v
+; CHECK-LABEL: 'test9'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %a = getelementptr inbounds i8, ptr %p, i32 4096
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %a, align 1
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
 ;
   %a = getelementptr inbounds i8, ptr %p, i32 4096
   %v = load i8, ptr %a
@@ -150,15 +104,10 @@ define i8 @test9(ptr %p) {
 }
 
 define i16 @test10(ptr %p) {
-; RECIP-LABEL: 'test10'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i16, ptr %p, i32 4096
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i16, ptr %a, align 2
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %v
-;
-; SIZE-LABEL: 'test10'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i16, ptr %p, i32 4096
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i16, ptr %a, align 2
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %v
+; CHECK-LABEL: 'test10'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %a = getelementptr inbounds i16, ptr %p, i32 4096
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i16, ptr %a, align 2
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %v
 ;
   %a = getelementptr inbounds i16, ptr %p, i32 4096
   %v = load i16, ptr %a
@@ -166,15 +115,10 @@ define i16 @test10(ptr %p) {
 }
 
 define i32 @test11(ptr %p) {
-; RECIP-LABEL: 'test11'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i32, ptr %p, i32 4096
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i32, ptr %a, align 4
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %v
-;
-; SIZE-LABEL: 'test11'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i32, ptr %p, i32 4096
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i32, ptr %a, align 4
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %v
+; CHECK-LABEL: 'test11'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %a = getelementptr inbounds i32, ptr %p, i32 4096
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i32, ptr %a, align 4
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %v
 ;
   %a = getelementptr inbounds i32, ptr %p, i32 4096
   %v = load i32, ptr %a
@@ -182,15 +126,10 @@ define i32 @test11(ptr %p) {
 }
 
 define i64 @test12(ptr %p) {
-; RECIP-LABEL: 'test12'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i64, ptr %p, i32 4096
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i64, ptr %a, align 8
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %v
-;
-; SIZE-LABEL: 'test12'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i64, ptr %p, i32 4096
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i64, ptr %a, align 8
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %v
+; CHECK-LABEL: 'test12'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %a = getelementptr inbounds i64, ptr %p, i32 4096
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i64, ptr %a, align 8
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %v
 ;
   %a = getelementptr inbounds i64, ptr %p, i32 4096
   %v = load i64, ptr %a
@@ -198,15 +137,10 @@ define i64 @test12(ptr %p) {
 }
 
 define i8 @test13(ptr %p) {
-; RECIP-LABEL: 'test13'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i8, ptr %p, i32 -64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i8, ptr %a, align 1
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %v
-;
-; SIZE-LABEL: 'test13'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i8, ptr %p, i32 -64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i8, ptr %a, align 1
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %v
+; CHECK-LABEL: 'test13'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %a = getelementptr inbounds i8, ptr %p, i32 -64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %a, align 1
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
 ;
   %a = getelementptr inbounds i8, ptr %p, i32 -64
   %v = load i8, ptr %a
@@ -214,15 +148,10 @@ define i8 @test13(ptr %p) {
 }
 
 define i16 @test14(ptr %p) {
-; RECIP-LABEL: 'test14'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i16, ptr %p, i32 -64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i16, ptr %a, align 2
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %v
-;
-; SIZE-LABEL: 'test14'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i16, ptr %p, i32 -64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i16, ptr %a, align 2
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %v
+; CHECK-LABEL: 'test14'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %a = getelementptr inbounds i16, ptr %p, i32 -64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i16, ptr %a, align 2
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %v
 ;
   %a = getelementptr inbounds i16, ptr %p, i32 -64
   %v = load i16, ptr %a
@@ -230,15 +159,10 @@ define i16 @test14(ptr %p) {
 }
 
 define i32 @test15(ptr %p) {
-; RECIP-LABEL: 'test15'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i32, ptr %p, i32 -64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i32, ptr %a, align 4
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %v
-;
-; SIZE-LABEL: 'test15'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i32, ptr %p, i32 -64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i32, ptr %a, align 4
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %v
+; CHECK-LABEL: 'test15'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %a = getelementptr inbounds i32, ptr %p, i32 -64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i32, ptr %a, align 4
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %v
 ;
   %a = getelementptr inbounds i32, ptr %p, i32 -64
   %v = load i32, ptr %a
@@ -246,15 +170,10 @@ define i32 @test15(ptr %p) {
 }
 
 define i64 @test16(ptr %p) {
-; RECIP-LABEL: 'test16'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i64, ptr %p, i32 -64
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i64, ptr %a, align 8
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %v
-;
-; SIZE-LABEL: 'test16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i64, ptr %p, i32 -64
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i64, ptr %a, align 8
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %v
+; CHECK-LABEL: 'test16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %a = getelementptr inbounds i64, ptr %p, i32 -64
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i64, ptr %a, align 8
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %v
 ;
   %a = getelementptr inbounds i64, ptr %p, i32 -64
   %v = load i64, ptr %a
@@ -262,15 +181,10 @@ define i64 @test16(ptr %p) {
 }
 
 define i8 @test17(ptr %p) {
-; RECIP-LABEL: 'test17'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i8, ptr %p, i32 -1024
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i8, ptr %a, align 1
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %v
-;
-; SIZE-LABEL: 'test17'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i8, ptr %p, i32 -1024
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i8, ptr %a, align 1
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %v
+; CHECK-LABEL: 'test17'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %a = getelementptr inbounds i8, ptr %p, i32 -1024
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %a, align 1
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
 ;
   %a = getelementptr inbounds i8, ptr %p, i32 -1024
   %v = load i8, ptr %a
@@ -278,15 +192,10 @@ define i8 @test17(ptr %p) {
 }
 
 define i16 @test18(ptr %p) {
-; RECIP-LABEL: 'test18'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i16, ptr %p, i32 -1024
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i16, ptr %a, align 2
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %v
-;
-; SIZE-LABEL: 'test18'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i16, ptr %p, i32 -1024
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i16, ptr %a, align 2
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %v
+; CHECK-LABEL: 'test18'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %a = getelementptr inbounds i16, ptr %p, i32 -1024
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i16, ptr %a, align 2
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %v
 ;
   %a = getelementptr inbounds i16, ptr %p, i32 -1024
   %v = load i16, ptr %a
@@ -294,15 +203,10 @@ define i16 @test18(ptr %p) {
 }
 
 define i32 @test19(ptr %p) {
-; RECIP-LABEL: 'test19'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i32, ptr %p, i32 -1024
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i32, ptr %a, align 4
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %v
-;
-; SIZE-LABEL: 'test19'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i32, ptr %p, i32 -1024
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i32, ptr %a, align 4
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %v
+; CHECK-LABEL: 'test19'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %a = getelementptr inbounds i32, ptr %p, i32 -1024
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i32, ptr %a, align 4
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %v
 ;
   %a = getelementptr inbounds i32, ptr %p, i32 -1024
   %v = load i32, ptr %a
@@ -310,15 +214,10 @@ define i32 @test19(ptr %p) {
 }
 
 define i64 @test20(ptr %p) {
-; RECIP-LABEL: 'test20'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i64, ptr %p, i32 -1024
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i64, ptr %a, align 8
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %v
-;
-; SIZE-LABEL: 'test20'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i64, ptr %p, i32 -1024
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i64, ptr %a, align 8
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %v
+; CHECK-LABEL: 'test20'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %a = getelementptr inbounds i64, ptr %p, i32 -1024
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i64, ptr %a, align 8
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %v
 ;
   %a = getelementptr inbounds i64, ptr %p, i32 -1024
   %v = load i64, ptr %a
@@ -326,15 +225,10 @@ define i64 @test20(ptr %p) {
 }
 
 define i8 @test21(ptr %p, i32 %i) {
-; RECIP-LABEL: 'test21'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i8, ptr %p, i32 %i
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i8, ptr %a, align 1
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %v
-;
-; SIZE-LABEL: 'test21'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i8, ptr %p, i32 %i
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i8, ptr %a, align 1
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %v
+; CHECK-LABEL: 'test21'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %a = getelementptr inbounds i8, ptr %p, i32 %i
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %a, align 1
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
 ;
   %a = getelementptr inbounds i8, ptr %p, i32 %i
   %v = load i8, ptr %a
@@ -342,15 +236,10 @@ define i8 @test21(ptr %p, i32 %i) {
 }
 
 define i16 @test22(ptr %p, i32 %i) {
-; RECIP-LABEL: 'test22'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i16, ptr %p, i32 %i
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i16, ptr %a, align 2
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %v
-;
-; SIZE-LABEL: 'test22'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i16, ptr %p, i32 %i
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i16, ptr %a, align 2
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %v
+; CHECK-LABEL: 'test22'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %a = getelementptr inbounds i16, ptr %p, i32 %i
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i16, ptr %a, align 2
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %v
 ;
   %a = getelementptr inbounds i16, ptr %p, i32 %i
   %v = load i16, ptr %a
@@ -358,15 +247,10 @@ define i16 @test22(ptr %p, i32 %i) {
 }
 
 define i32 @test23(ptr %p, i32 %i) {
-; RECIP-LABEL: 'test23'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i32, ptr %p, i32 %i
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i32, ptr %a, align 4
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %v
-;
-; SIZE-LABEL: 'test23'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i32, ptr %p, i32 %i
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i32, ptr %a, align 4
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %v
+; CHECK-LABEL: 'test23'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %a = getelementptr inbounds i32, ptr %p, i32 %i
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i32, ptr %a, align 4
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %v
 ;
   %a = getelementptr inbounds i32, ptr %p, i32 %i
   %v = load i32, ptr %a
@@ -374,15 +258,10 @@ define i32 @test23(ptr %p, i32 %i) {
 }
 
 define i64 @test24(ptr %p, i32 %i) {
-; RECIP-LABEL: 'test24'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i64, ptr %p, i32 %i
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i64, ptr %a, align 8
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %v
-;
-; SIZE-LABEL: 'test24'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i64, ptr %p, i32 %i
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i64, ptr %a, align 8
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %v
+; CHECK-LABEL: 'test24'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %a = getelementptr inbounds i64, ptr %p, i32 %i
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i64, ptr %a, align 8
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %v
 ;
   %a = getelementptr inbounds i64, ptr %p, i32 %i
   %v = load i64, ptr %a
@@ -390,15 +269,10 @@ define i64 @test24(ptr %p, i32 %i) {
 }
 
 define i8 @test25(ptr %p) {
-; RECIP-LABEL: 'test25'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i8, ptr %p, i32 -128
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i8, ptr %a, align 1
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %v
-;
-; SIZE-LABEL: 'test25'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i8, ptr %p, i32 -128
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i8, ptr %a, align 1
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %v
+; CHECK-LABEL: 'test25'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %a = getelementptr inbounds i8, ptr %p, i32 -128
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %a, align 1
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
 ;
   %a = getelementptr inbounds i8, ptr %p, i32 -128
   %v = load i8, ptr %a
@@ -406,15 +280,10 @@ define i8 @test25(ptr %p) {
 }
 
 define i16 @test26(ptr %p) {
-; RECIP-LABEL: 'test26'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i16, ptr %p, i32 -128
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i16, ptr %a, align 2
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %v
-;
-; SIZE-LABEL: 'test26'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i16, ptr %p, i32 -128
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i16, ptr %a, align 2
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %v
+; CHECK-LABEL: 'test26'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %a = getelementptr inbounds i16, ptr %p, i32 -128
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i16, ptr %a, align 2
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %v
 ;
   %a = getelementptr inbounds i16, ptr %p, i32 -128
   %v = load i16, ptr %a
@@ -422,15 +291,10 @@ define i16 @test26(ptr %p) {
 }
 
 define i32 @test27(ptr %p) {
-; RECIP-LABEL: 'test27'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i32, ptr %p, i32 -128
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i32, ptr %a, align 4
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %v
-;
-; SIZE-LABEL: 'test27'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i32, ptr %p, i32 -128
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i32, ptr %a, align 4
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %v
+; CHECK-LABEL: 'test27'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %a = getelementptr inbounds i32, ptr %p, i32 -128
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i32, ptr %a, align 4
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %v
 ;
   %a = getelementptr inbounds i32, ptr %p, i32 -128
   %v = load i32, ptr %a
@@ -438,15 +302,10 @@ define i32 @test27(ptr %p) {
 }
 
 define i64 @test28(ptr %p) {
-; RECIP-LABEL: 'test28'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i64, ptr %p, i32 -128
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i64, ptr %a, align 8
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %v
-;
-; SIZE-LABEL: 'test28'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i64, ptr %p, i32 -128
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i64, ptr %a, align 8
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %v
+; CHECK-LABEL: 'test28'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %a = getelementptr inbounds i64, ptr %p, i32 -128
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i64, ptr %a, align 8
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %v
 ;
   %a = getelementptr inbounds i64, ptr %p, i32 -128
   %v = load i64, ptr %a
@@ -454,15 +313,10 @@ define i64 @test28(ptr %p) {
 }
 
 define i8 @test29(ptr %p) {
-; RECIP-LABEL: 'test29'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i8, ptr %p, i32 -256
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i8, ptr %a, align 1
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %v
-;
-; SIZE-LABEL: 'test29'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = getelementptr inbounds i8, ptr %p, i32 -256
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i8, ptr %a, align 1
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %v
+; CHECK-LABEL: 'test29'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %a = getelementptr inbounds i8, ptr %p, i32 -256
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %a, align 1
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
 ;
   %a = getelementptr inbounds i8, ptr %p, i32 -256
   %v = load i8, ptr %a
@@ -470,15 +324,10 @@ define i8 @test29(ptr %p) {
 }
 
 define i16 @test30(ptr %p) {
-; RECIP-LABEL: 'test30'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i16, ptr %p, i32 -256
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i16, ptr %a, align 2
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %v
-;
-; SIZE-LABEL: 'test30'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i16, ptr %p, i32 -256
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i16, ptr %a, align 2
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %v
+; CHECK-LABEL: 'test30'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %a = getelementptr inbounds i16, ptr %p, i32 -256
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i16, ptr %a, align 2
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %v
 ;
   %a = getelementptr inbounds i16, ptr %p, i32 -256
   %v = load i16, ptr %a
@@ -486,15 +335,10 @@ define i16 @test30(ptr %p) {
 }
 
 define i32 @test31(ptr %p) {
-; RECIP-LABEL: 'test31'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i32, ptr %p, i32 -256
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i32, ptr %a, align 4
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %v
-;
-; SIZE-LABEL: 'test31'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i32, ptr %p, i32 -256
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i32, ptr %a, align 4
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %v
+; CHECK-LABEL: 'test31'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %a = getelementptr inbounds i32, ptr %p, i32 -256
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i32, ptr %a, align 4
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %v
 ;
   %a = getelementptr inbounds i32, ptr %p, i32 -256
   %v = load i32, ptr %a
@@ -502,15 +346,10 @@ define i32 @test31(ptr %p) {
 }
 
 define i64 @test32(ptr %p) {
-; RECIP-LABEL: 'test32'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i64, ptr %p, i32 -256
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i64, ptr %a, align 8
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %v
-;
-; SIZE-LABEL: 'test32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i64, ptr %p, i32 -256
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i64, ptr %a, align 8
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %v
+; CHECK-LABEL: 'test32'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %a = getelementptr inbounds i64, ptr %p, i32 -256
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i64, ptr %a, align 8
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %v
 ;
   %a = getelementptr inbounds i64, ptr %p, i32 -256
   %v = load i64, ptr %a
@@ -518,15 +357,10 @@ define i64 @test32(ptr %p) {
 }
 
 define i8 @test33(ptr %p) {
-; RECIP-LABEL: 'test33'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i8, ptr %p, i32 -512
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i8, ptr %a, align 1
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i8 %v
-;
-; SIZE-LABEL: 'test33'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i8, ptr %p, i32 -512
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i8, ptr %a, align 1
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %v
+; CHECK-LABEL: 'test33'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %a = getelementptr inbounds i8, ptr %p, i32 -512
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i8, ptr %a, align 1
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i8 %v
 ;
   %a = getelementptr inbounds i8, ptr %p, i32 -512
   %v = load i8, ptr %a
@@ -534,15 +368,10 @@ define i8 @test33(ptr %p) {
 }
 
 define i16 @test34(ptr %p) {
-; RECIP-LABEL: 'test34'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i16, ptr %p, i32 -512
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i16, ptr %a, align 2
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %v
-;
-; SIZE-LABEL: 'test34'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i16, ptr %p, i32 -512
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i16, ptr %a, align 2
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %v
+; CHECK-LABEL: 'test34'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %a = getelementptr inbounds i16, ptr %p, i32 -512
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i16, ptr %a, align 2
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i16 %v
 ;
   %a = getelementptr inbounds i16, ptr %p, i32 -512
   %v = load i16, ptr %a
@@ -550,15 +379,10 @@ define i16 @test34(ptr %p) {
 }
 
 define i32 @test35(ptr %p) {
-; RECIP-LABEL: 'test35'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i32, ptr %p, i32 -512
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i32, ptr %a, align 4
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %v
-;
-; SIZE-LABEL: 'test35'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i32, ptr %p, i32 -512
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i32, ptr %a, align 4
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %v
+; CHECK-LABEL: 'test35'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %a = getelementptr inbounds i32, ptr %p, i32 -512
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i32, ptr %a, align 4
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %v
 ;
   %a = getelementptr inbounds i32, ptr %p, i32 -512
   %v = load i32, ptr %a
@@ -566,15 +390,10 @@ define i32 @test35(ptr %p) {
 }
 
 define i64 @test36(ptr %p) {
-; RECIP-LABEL: 'test36'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i64, ptr %p, i32 -512
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i64, ptr %a, align 8
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %v
-;
-; SIZE-LABEL: 'test36'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = getelementptr inbounds i64, ptr %p, i32 -512
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = load i64, ptr %a, align 8
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %v
+; CHECK-LABEL: 'test36'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %a = getelementptr inbounds i64, ptr %p, i32 -512
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v = load i64, ptr %a, align 8
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i64 %v
 ;
   %a = getelementptr inbounds i64, ptr %p, i32 -512
   %v = load i64, ptr %a
diff --git a/llvm/test/Analysis/CostModel/AArch64/insert-extract.ll b/llvm/test/Analysis/CostModel/AArch64/insert-extract.ll
index babc1c42c7438..3baeb075f1813 100644
--- a/llvm/test/Analysis/CostModel/AArch64/insert-extract.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/insert-extract.ll
@@ -1,86 +1,50 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output | FileCheck %s
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mcpu=neoverse-n1 | FileCheck %s
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mcpu=neoverse-n2 | FileCheck %s
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mcpu=neoverse-v1 | FileCheck %s
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mcpu=neoverse-v2 | FileCheck %s
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -mcpu=kryo | FileCheck %s --check-prefix=KRYO
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output | FileCheck %s
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=neoverse-n1 | FileCheck %s
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=neoverse-n2 | FileCheck %s
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=neoverse-v1 | FileCheck %s
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=neoverse-v2 | FileCheck %s
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mcpu=kryo | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
 
 define void @vectorInstrCost() {
 ; CHECK-LABEL: 'vectorInstrCost'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %ta0 = extractelement <8 x i1> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %ta1 = extractelement <8 x i1> undef, i32 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t1 = extractelement <8 x i8> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = extractelement <8 x i8> undef, i32 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t3 = extractelement <4 x i16> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t4 = extractelement <4 x i16> undef, i32 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t5 = extractelement <2 x i32> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t6 = extractelement <2 x i32> undef, i32 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t7 = extractelement <2 x i64> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t8 = extractelement <2 x i64> undef, i32 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %t9 = extractelement <4 x half> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t10 = extractelement <4 x half> undef, i32 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %t11 = extractelement <2 x float> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t12 = extractelement <2 x float> undef, i32 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %t13 = extractelement <2 x double> undef, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t14 = extractelement <2 x double> undef, i32 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %t31 = insertelement <8 x i1> undef, i1 false, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %t41 = insertelement <8 x i1> undef, i1 true, i32 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t30 = insertelement <8 x i8> undef, i8 0, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t40 = insertelement <8 x i8> undef, i8 1, i32 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t50 = insertelement <4 x i16> undef, i16 2, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t60 = insertelement <4 x i16> undef, i16 3, i32 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t70 = insertelement <2 x i32> undef, i32 4, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t80 = insertelement <2 x i32> undef, i32 5, i32 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t90 = insertelement <2 x i64> undef, i64 6, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t100 = insertelement <2 x i64> undef, i64 7, i32 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %t110 = insertelement <4 x half> zeroinitializer, half 0xH0000, i64 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t120 = insertelement <4 x half> zeroinitializer, half 0xH0000, i64 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %t130 = insertelement <2 x float> zeroinitializer, float 0.000000e+00, i64 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t140 = insertelement <2 x float> zeroinitializer, float 0.000000e+00, i64 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %t150 = insertelement <2 x double> zeroinitializer, double 0.000000e+00, i64 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t160 = insertelement <2 x double> zeroinitializer, double 0.000000e+00, i64 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %ta0 = extractelement <8 x i1> undef, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %ta1 = extractelement <8 x i1> undef, i32 1
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t1 = extractelement <8 x i8> undef, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t2 = extractelement <8 x i8> undef, i32 1
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t3 = extractelement <4 x i16> undef, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t4 = extractelement <4 x i16> undef, i32 1
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t5 = extractelement <2 x i32> undef, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t6 = extractelement <2 x i32> undef, i32 1
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t7 = extractelement <2 x i64> undef, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t8 = extractelement <2 x i64> undef, i32 1
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %t9 = extractelement <4 x half> undef, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t10 = extractelement <4 x half> undef, i32 1
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %t11 = extractelement <2 x float> undef, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t12 = extractelement <2 x float> undef, i32 1
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %t13 = extractelement <2 x double> undef, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t14 = extractelement <2 x double> undef, i32 1
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %t31 = insertelement <8 x i1> undef, i1 false, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %t41 = insertelement <8 x i1> undef, i1 true, i32 1
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t30 = insertelement <8 x i8> undef, i8 0, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t40 = insertelement <8 x i8> undef, i8 1, i32 1
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t50 = insertelement <4 x i16> undef, i16 2, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t60 = insertelement <4 x i16> undef, i16 3, i32 1
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t70 = insertelement <2 x i32> undef, i32 4, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t80 = insertelement <2 x i32> undef, i32 5, i32 1
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t90 = insertelement <2 x i64> undef, i64 6, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t100 = insertelement <2 x i64> undef, i64 7, i32 1
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %t110 = insertelement <4 x half> zeroinitializer, half 0xH0000, i64 0
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t120 = insertelement <4 x half> zeroinitializer, half 0xH0000, i64 1
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %t130 = insertelement <2 x float> zeroinitializer, float 0.000000e+00, i64 0
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t140 = insertelement <2 x float> zeroinitializer, float 0.000000e+00, i64 1
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %t150 = insertelement <2 x double> zeroinitializer, double 0.000000e+00, i64 0
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %t160 = insertelement <2 x double> zeroinitializer, double 0.000000e+00, i64 1
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
-; KRYO-LABEL: 'vectorInstrCost'
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %ta0 = extractelement <8 x i1> undef, i32 0
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %ta1 = extractelement <8 x i1> undef, i32 1
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t1 = extractelement <8 x i8> undef, i32 0
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = extractelement <8 x i8> undef, i32 1
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t3 = extractelement <4 x i16> undef, i32 0
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t4 = extractelement <4 x i16> undef, i32 1
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t5 = extractelement <2 x i32> undef, i32 0
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t6 = extractelement <2 x i32> undef, i32 1
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t7 = extractelement <2 x i64> undef, i32 0
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t8 = extractelement <2 x i64> undef, i32 1
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %t9 = extractelement <4 x half> undef, i32 0
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t10 = extractelement <4 x half> undef, i32 1
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %t11 = extractelement <2 x float> undef, i32 0
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t12 = extractelement <2 x float> undef, i32 1
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %t13 = extractelement <2 x double> undef, i32 0
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t14 = extractelement <2 x double> undef, i32 1
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %t31 = insertelement <8 x i1> undef, i1 false, i32 0
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %t41 = insertelement <8 x i1> undef, i1 true, i32 1
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t30 = insertelement <8 x i8> undef, i8 0, i32 0
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t40 = insertelement <8 x i8> undef, i8 1, i32 1
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t50 = insertelement <4 x i16> undef, i16 2, i32 0
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t60 = insertelement <4 x i16> undef, i16 3, i32 1
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t70 = insertelement <2 x i32> undef, i32 4, i32 0
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t80 = insertelement <2 x i32> undef, i32 5, i32 1
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t90 = insertelement <2 x i64> undef, i64 6, i32 0
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t100 = insertelement <2 x i64> undef, i64 7, i32 1
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %t110 = insertelement <4 x half> zeroinitializer, half 0xH0000, i64 0
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t120 = insertelement <4 x half> zeroinitializer, half 0xH0000, i64 1
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %t130 = insertelement <2 x float> zeroinitializer, float 0.000000e+00, i64 0
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t140 = insertelement <2 x float> zeroinitializer, float 0.000000e+00, i64 1
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %t150 = insertelement <2 x double> zeroinitializer, double 0.000000e+00, i64 0
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t160 = insertelement <2 x double> zeroinitializer, double 0.000000e+00, i64 1
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-
   %ta0 = extractelement <8 x i1> undef, i32 0
   %ta1 = extractelement <8 x i1> undef, i32 1
   %t1 = extractelement <8 x i8> undef, i32 0
@@ -121,14 +85,9 @@ define void @vectorInstrCost() {
 
 define <8 x i8> @LD1_B(<8 x i8> %vec, ptr noundef %i) {
 ; CHECK-LABEL: 'LD1_B'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = load i8, ptr %i, align 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2 = insertelement <8 x i8> %vec, i8 %v1, i32 1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i8> %v2
-;
-; KRYO-LABEL: 'LD1_B'
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = load i8, ptr %i, align 1
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2 = insertelement <8 x i8> %vec, i8 %v1, i32 1
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i8> %v2
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i8, ptr %i, align 1
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %v2 = insertelement <8 x i8> %vec, i8 %v1, i32 1
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %v2
 ;
 entry:
   %v1 = load i8, ptr %i, align 1
@@ -138,14 +97,9 @@ entry:
 
 define <4 x i16> @LD1_H(<4 x i16> %vec, ptr noundef %i) {
 ; CHECK-LABEL: 'LD1_H'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = load i16, ptr %i, align 2
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2 = insertelement <4 x i16> %vec, i16 %v1, i32 2
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i16> %v2
-;
-; KRYO-LABEL: 'LD1_H'
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = load i16, ptr %i, align 2
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2 = insertelement <4 x i16> %vec, i16 %v1, i32 2
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i16> %v2
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i16, ptr %i, align 2
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %v2 = insertelement <4 x i16> %vec, i16 %v1, i32 2
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i16> %v2
 ;
 entry:
   %v1 = load i16, ptr %i, align 2
@@ -155,14 +109,9 @@ entry:
 
 define <4 x i32> @LD1_W(<4 x i32> %vec, ptr noundef %i) {
 ; CHECK-LABEL: 'LD1_W'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = load i32, ptr %i, align 4
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2 = insertelement <4 x i32> %vec, i32 %v1, i32 3
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %v2
-;
-; KRYO-LABEL: 'LD1_W'
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = load i32, ptr %i, align 4
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2 = insertelement <4 x i32> %vec, i32 %v1, i32 3
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %v2
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i32, ptr %i, align 4
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %v2 = insertelement <4 x i32> %vec, i32 %v1, i32 3
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %v2
 ;
 entry:
   %v1 = load i32, ptr %i, align 4
@@ -172,14 +121,9 @@ entry:
 
 define <2 x i64> @LD1_X(<2 x i64> %vec, ptr noundef %i) {
 ; CHECK-LABEL: 'LD1_X'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = load i64, ptr %i, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2 = insertelement <2 x i64> %vec, i64 %v1, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %v2
-;
-; KRYO-LABEL: 'LD1_X'
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = load i64, ptr %i, align 8
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2 = insertelement <2 x i64> %vec, i64 %v1, i32 0
-; KRYO-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %v2
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v1 = load i64, ptr %i, align 8
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %v2 = insertelement <2 x i64> %vec, i64 %v1, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i64> %v2
 ;
 entry:
   %v1 = load i64, ptr %i, align 8
diff --git a/llvm/test/Analysis/CostModel/AArch64/min-max.ll b/llvm/test/Analysis/CostModel/AArch64/min-max.ll
index ab048a8021381..ed4b9dd2ba571 100644
--- a/llvm/test/Analysis/CostModel/AArch64/min-max.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/min-max.ll
@@ -1,57 +1,32 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output -cost-kind=throughput | FileCheck %s --check-prefixes=RECIP,RECIP-NOF16
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -mattr=+fullfp16 -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output -cost-kind=throughput | FileCheck %s --check-prefixes=RECIP,RECIP-F16
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output -cost-kind=throughput | FileCheck %s --check-prefixes=SIZE,SIZE-NOF16
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -mattr=+fullfp16 -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output -cost-kind=throughput | FileCheck %s --check-prefixes=SIZE,SIZE-F16
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-NOF16
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 -mattr=+fullfp16 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-F16
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define void @umin() {
-; RECIP-LABEL: 'umin'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1i8 = call <1 x i8> @llvm.umin.v1i8(<1 x i8> undef, <1 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V3i8 = call <3 x i8> @llvm.umin.v3i8(<3 x i8> undef, <3 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i8 = call <4 x i8> @llvm.umin.v4i8(<4 x i8> undef, <4 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i8 = call <8 x i8> @llvm.umin.v8i8(<8 x i8> undef, <8 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32i8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64i8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = call <2 x i16> @llvm.umin.v2i16(<2 x i16> undef, <2 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = call <4 x i16> @llvm.umin.v4i16(<4 x i16> undef, <4 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16i16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = call <2 x i32> @llvm.umin.v2i32(<2 x i32> undef, <2 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'umin'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1i8 = call <1 x i8> @llvm.umin.v1i8(<1 x i8> undef, <1 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V3i8 = call <3 x i8> @llvm.umin.v3i8(<3 x i8> undef, <3 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i8 = call <4 x i8> @llvm.umin.v4i8(<4 x i8> undef, <4 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i8 = call <8 x i8> @llvm.umin.v8i8(<8 x i8> undef, <8 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32i8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64i8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = call <2 x i16> @llvm.umin.v2i16(<2 x i16> undef, <2 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = call <4 x i16> @llvm.umin.v4i16(<4 x i16> undef, <4 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16i16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = call <2 x i32> @llvm.umin.v2i32(<2 x i32> undef, <2 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-LABEL: 'umin'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %i8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %i16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %i32 = call i32 @llvm.umin.i32(i32 undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %i64 = call i64 @llvm.umin.i64(i64 undef, i64 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V1i8 = call <1 x i8> @llvm.umin.v1i8(<1 x i8> undef, <1 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V3i8 = call <3 x i8> @llvm.umin.v3i8(<3 x i8> undef, <3 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4i8 = call <4 x i8> @llvm.umin.v4i8(<4 x i8> undef, <4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8i8 = call <8 x i8> @llvm.umin.v8i8(<8 x i8> undef, <8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V16i8 = call <16 x i8> @llvm.umin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V32i8 = call <32 x i8> @llvm.umin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V64i8 = call <64 x i8> @llvm.umin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2i16 = call <2 x i16> @llvm.umin.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4i16 = call <4 x i16> @llvm.umin.v4i16(<4 x i16> undef, <4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8i16 = call <8 x i16> @llvm.umin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16i16 = call <16 x i16> @llvm.umin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2i32 = call <2 x i32> @llvm.umin.v2i32(<2 x i32> undef, <2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4i32 = call <4 x i32> @llvm.umin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8i32 = call <8 x i32> @llvm.umin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V2i64 = call <2 x i64> @llvm.umin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4i64 = call <4 x i64> @llvm.umin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %i8 = call i8 @llvm.umin.i8(i8 undef, i8 undef)
   %i16 = call i16 @llvm.umin.i16(i16 undef, i16 undef)
@@ -77,51 +52,28 @@ define void @umin() {
 }
 
 define void @umax() {
-; RECIP-LABEL: 'umax'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1i8 = call <1 x i8> @llvm.umax.v1i8(<1 x i8> undef, <1 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V3i8 = call <3 x i8> @llvm.umax.v3i8(<3 x i8> undef, <3 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i8 = call <4 x i8> @llvm.umax.v4i8(<4 x i8> undef, <4 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i8 = call <8 x i8> @llvm.umax.v8i8(<8 x i8> undef, <8 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32i8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64i8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = call <2 x i16> @llvm.umax.v2i16(<2 x i16> undef, <2 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = call <4 x i16> @llvm.umax.v4i16(<4 x i16> undef, <4 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16i16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = call <2 x i32> @llvm.umax.v2i32(<2 x i32> undef, <2 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'umax'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1i8 = call <1 x i8> @llvm.umax.v1i8(<1 x i8> undef, <1 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V3i8 = call <3 x i8> @llvm.umax.v3i8(<3 x i8> undef, <3 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i8 = call <4 x i8> @llvm.umax.v4i8(<4 x i8> undef, <4 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i8 = call <8 x i8> @llvm.umax.v8i8(<8 x i8> undef, <8 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32i8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64i8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = call <2 x i16> @llvm.umax.v2i16(<2 x i16> undef, <2 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = call <4 x i16> @llvm.umax.v4i16(<4 x i16> undef, <4 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16i16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = call <2 x i32> @llvm.umax.v2i32(<2 x i32> undef, <2 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-LABEL: 'umax'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %i8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %i16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %i32 = call i32 @llvm.umax.i32(i32 undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %i64 = call i64 @llvm.umax.i64(i64 undef, i64 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V1i8 = call <1 x i8> @llvm.umax.v1i8(<1 x i8> undef, <1 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V3i8 = call <3 x i8> @llvm.umax.v3i8(<3 x i8> undef, <3 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4i8 = call <4 x i8> @llvm.umax.v4i8(<4 x i8> undef, <4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8i8 = call <8 x i8> @llvm.umax.v8i8(<8 x i8> undef, <8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V16i8 = call <16 x i8> @llvm.umax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V32i8 = call <32 x i8> @llvm.umax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V64i8 = call <64 x i8> @llvm.umax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2i16 = call <2 x i16> @llvm.umax.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4i16 = call <4 x i16> @llvm.umax.v4i16(<4 x i16> undef, <4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8i16 = call <8 x i16> @llvm.umax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16i16 = call <16 x i16> @llvm.umax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2i32 = call <2 x i32> @llvm.umax.v2i32(<2 x i32> undef, <2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4i32 = call <4 x i32> @llvm.umax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8i32 = call <8 x i32> @llvm.umax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V2i64 = call <2 x i64> @llvm.umax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4i64 = call <4 x i64> @llvm.umax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %i8 = call i8 @llvm.umax.i8(i8 undef, i8 undef)
   %i16 = call i16 @llvm.umax.i16(i16 undef, i16 undef)
@@ -147,51 +99,28 @@ define void @umax() {
 }
 
 define void @smin() {
-; RECIP-LABEL: 'smin'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1i8 = call <1 x i8> @llvm.smin.v1i8(<1 x i8> undef, <1 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V3i8 = call <3 x i8> @llvm.smin.v3i8(<3 x i8> undef, <3 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i8 = call <4 x i8> @llvm.smin.v4i8(<4 x i8> undef, <4 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i8 = call <8 x i8> @llvm.smin.v8i8(<8 x i8> undef, <8 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32i8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64i8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = call <2 x i16> @llvm.smin.v2i16(<2 x i16> undef, <2 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = call <4 x i16> @llvm.smin.v4i16(<4 x i16> undef, <4 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16i16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = call <2 x i32> @llvm.smin.v2i32(<2 x i32> undef, <2 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'smin'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1i8 = call <1 x i8> @llvm.smin.v1i8(<1 x i8> undef, <1 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V3i8 = call <3 x i8> @llvm.smin.v3i8(<3 x i8> undef, <3 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i8 = call <4 x i8> @llvm.smin.v4i8(<4 x i8> undef, <4 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i8 = call <8 x i8> @llvm.smin.v8i8(<8 x i8> undef, <8 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32i8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64i8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = call <2 x i16> @llvm.smin.v2i16(<2 x i16> undef, <2 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = call <4 x i16> @llvm.smin.v4i16(<4 x i16> undef, <4 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16i16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = call <2 x i32> @llvm.smin.v2i32(<2 x i32> undef, <2 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-LABEL: 'smin'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %i8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %i16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %i32 = call i32 @llvm.smin.i32(i32 undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %i64 = call i64 @llvm.smin.i64(i64 undef, i64 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V1i8 = call <1 x i8> @llvm.smin.v1i8(<1 x i8> undef, <1 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V3i8 = call <3 x i8> @llvm.smin.v3i8(<3 x i8> undef, <3 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4i8 = call <4 x i8> @llvm.smin.v4i8(<4 x i8> undef, <4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8i8 = call <8 x i8> @llvm.smin.v8i8(<8 x i8> undef, <8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V16i8 = call <16 x i8> @llvm.smin.v16i8(<16 x i8> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V32i8 = call <32 x i8> @llvm.smin.v32i8(<32 x i8> undef, <32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V64i8 = call <64 x i8> @llvm.smin.v64i8(<64 x i8> undef, <64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2i16 = call <2 x i16> @llvm.smin.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4i16 = call <4 x i16> @llvm.smin.v4i16(<4 x i16> undef, <4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8i16 = call <8 x i16> @llvm.smin.v8i16(<8 x i16> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16i16 = call <16 x i16> @llvm.smin.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2i32 = call <2 x i32> @llvm.smin.v2i32(<2 x i32> undef, <2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4i32 = call <4 x i32> @llvm.smin.v4i32(<4 x i32> undef, <4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8i32 = call <8 x i32> @llvm.smin.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V2i64 = call <2 x i64> @llvm.smin.v2i64(<2 x i64> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4i64 = call <4 x i64> @llvm.smin.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %i8 = call i8 @llvm.smin.i8(i8 undef, i8 undef)
   %i16 = call i16 @llvm.smin.i16(i16 undef, i16 undef)
@@ -217,51 +146,28 @@ define void @smin() {
 }
 
 define void @smax() {
-; RECIP-LABEL: 'smax'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1i8 = call <1 x i8> @llvm.smax.v1i8(<1 x i8> undef, <1 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V3i8 = call <3 x i8> @llvm.smax.v3i8(<3 x i8> undef, <3 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i8 = call <4 x i8> @llvm.smax.v4i8(<4 x i8> undef, <4 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i8 = call <8 x i8> @llvm.smax.v8i8(<8 x i8> undef, <8 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32i8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64i8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = call <2 x i16> @llvm.smax.v2i16(<2 x i16> undef, <2 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = call <4 x i16> @llvm.smax.v4i16(<4 x i16> undef, <4 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16i16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = call <2 x i32> @llvm.smax.v2i32(<2 x i32> undef, <2 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'smax'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %i64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V1i8 = call <1 x i8> @llvm.smax.v1i8(<1 x i8> undef, <1 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V3i8 = call <3 x i8> @llvm.smax.v3i8(<3 x i8> undef, <3 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i8 = call <4 x i8> @llvm.smax.v4i8(<4 x i8> undef, <4 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i8 = call <8 x i8> @llvm.smax.v8i8(<8 x i8> undef, <8 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32i8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64i8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i16 = call <2 x i16> @llvm.smax.v2i16(<2 x i16> undef, <2 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i16 = call <4 x i16> @llvm.smax.v4i16(<4 x i16> undef, <4 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16i16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = call <2 x i32> @llvm.smax.v2i32(<2 x i32> undef, <2 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-LABEL: 'smax'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %i8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %i16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %i32 = call i32 @llvm.smax.i32(i32 undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %i64 = call i64 @llvm.smax.i64(i64 undef, i64 undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V1i8 = call <1 x i8> @llvm.smax.v1i8(<1 x i8> undef, <1 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V3i8 = call <3 x i8> @llvm.smax.v3i8(<3 x i8> undef, <3 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4i8 = call <4 x i8> @llvm.smax.v4i8(<4 x i8> undef, <4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8i8 = call <8 x i8> @llvm.smax.v8i8(<8 x i8> undef, <8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V16i8 = call <16 x i8> @llvm.smax.v16i8(<16 x i8> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V32i8 = call <32 x i8> @llvm.smax.v32i8(<32 x i8> undef, <32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V64i8 = call <64 x i8> @llvm.smax.v64i8(<64 x i8> undef, <64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2i16 = call <2 x i16> @llvm.smax.v2i16(<2 x i16> undef, <2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4i16 = call <4 x i16> @llvm.smax.v4i16(<4 x i16> undef, <4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8i16 = call <8 x i16> @llvm.smax.v8i16(<8 x i16> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16i16 = call <16 x i16> @llvm.smax.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2i32 = call <2 x i32> @llvm.smax.v2i32(<2 x i32> undef, <2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4i32 = call <4 x i32> @llvm.smax.v4i32(<4 x i32> undef, <4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8i32 = call <8 x i32> @llvm.smax.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V2i64 = call <2 x i64> @llvm.smax.v2i64(<2 x i64> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4i64 = call <4 x i64> @llvm.smax.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %i8 = call i8 @llvm.smax.i8(i8 undef, i8 undef)
   %i16 = call i16 @llvm.smax.i16(i16 undef, i16 undef)
@@ -287,37 +193,21 @@ define void @smax() {
 }
 
 define void @minnum16() {
-; RECIP-NOF16-LABEL: 'minnum16'
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.minnum.f16(half undef, half undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2f16 = call <2 x half> @llvm.minnum.v2f16(<2 x half> undef, <2 x half> undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4f16 = call <4 x half> @llvm.minnum.v4f16(<4 x half> undef, <4 x half> undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8f16 = call <8 x half> @llvm.minnum.v8f16(<8 x half> undef, <8 x half> undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16f16 = call <16 x half> @llvm.minnum.v16f16(<16 x half> undef, <16 x half> undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; RECIP-F16-LABEL: 'minnum16'
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.minnum.f16(half undef, half undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f16 = call <2 x half> @llvm.minnum.v2f16(<2 x half> undef, <2 x half> undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f16 = call <4 x half> @llvm.minnum.v4f16(<4 x half> undef, <4 x half> undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8f16 = call <8 x half> @llvm.minnum.v8f16(<8 x half> undef, <8 x half> undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16f16 = call <16 x half> @llvm.minnum.v16f16(<16 x half> undef, <16 x half> undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-NOF16-LABEL: 'minnum16'
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.minnum.f16(half undef, half undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2f16 = call <2 x half> @llvm.minnum.v2f16(<2 x half> undef, <2 x half> undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4f16 = call <4 x half> @llvm.minnum.v4f16(<4 x half> undef, <4 x half> undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8f16 = call <8 x half> @llvm.minnum.v8f16(<8 x half> undef, <8 x half> undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16f16 = call <16 x half> @llvm.minnum.v16f16(<16 x half> undef, <16 x half> undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-F16-LABEL: 'minnum16'
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.minnum.f16(half undef, half undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f16 = call <2 x half> @llvm.minnum.v2f16(<2 x half> undef, <2 x half> undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f16 = call <4 x half> @llvm.minnum.v4f16(<4 x half> undef, <4 x half> undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8f16 = call <8 x half> @llvm.minnum.v8f16(<8 x half> undef, <8 x half> undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16f16 = call <16 x half> @llvm.minnum.v16f16(<16 x half> undef, <16 x half> undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NOF16-LABEL: 'minnum16'
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 1 for: %f16 = call half @llvm.minnum.f16(half undef, half undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 4 for: %V2f16 = call <2 x half> @llvm.minnum.v2f16(<2 x half> undef, <2 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 10 for: %V4f16 = call <4 x half> @llvm.minnum.v4f16(<4 x half> undef, <4 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 22 for: %V8f16 = call <8 x half> @llvm.minnum.v8f16(<8 x half> undef, <8 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 44 for: %V16f16 = call <16 x half> @llvm.minnum.v16f16(<16 x half> undef, <16 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-F16-LABEL: 'minnum16'
+; CHECK-F16-NEXT:  Cost Model: Found costs of 1 for: %f16 = call half @llvm.minnum.f16(half undef, half undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 1 for: %V2f16 = call <2 x half> @llvm.minnum.v2f16(<2 x half> undef, <2 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 1 for: %V4f16 = call <4 x half> @llvm.minnum.v4f16(<4 x half> undef, <4 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 1 for: %V8f16 = call <8 x half> @llvm.minnum.v8f16(<8 x half> undef, <8 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 4 for: %V16f16 = call <16 x half> @llvm.minnum.v16f16(<16 x half> undef, <16 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %f16 = call half @llvm.minnum.f16(half undef, half undef)
   %V2f16 = call <2 x half> @llvm.minnum.v2f16(<2 x half> undef, <2 x half> undef)
@@ -328,37 +218,21 @@ define void @minnum16() {
 }
 
 define void @maxnum16() {
-; RECIP-NOF16-LABEL: 'maxnum16'
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.maxnum.f16(half undef, half undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2f16 = call <2 x half> @llvm.maxnum.v2f16(<2 x half> undef, <2 x half> undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4f16 = call <4 x half> @llvm.maxnum.v4f16(<4 x half> undef, <4 x half> undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8f16 = call <8 x half> @llvm.maxnum.v8f16(<8 x half> undef, <8 x half> undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16f16 = call <16 x half> @llvm.maxnum.v16f16(<16 x half> undef, <16 x half> undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; RECIP-F16-LABEL: 'maxnum16'
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.maxnum.f16(half undef, half undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f16 = call <2 x half> @llvm.maxnum.v2f16(<2 x half> undef, <2 x half> undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f16 = call <4 x half> @llvm.maxnum.v4f16(<4 x half> undef, <4 x half> undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8f16 = call <8 x half> @llvm.maxnum.v8f16(<8 x half> undef, <8 x half> undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16f16 = call <16 x half> @llvm.maxnum.v16f16(<16 x half> undef, <16 x half> undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-NOF16-LABEL: 'maxnum16'
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.maxnum.f16(half undef, half undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2f16 = call <2 x half> @llvm.maxnum.v2f16(<2 x half> undef, <2 x half> undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4f16 = call <4 x half> @llvm.maxnum.v4f16(<4 x half> undef, <4 x half> undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8f16 = call <8 x half> @llvm.maxnum.v8f16(<8 x half> undef, <8 x half> undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16f16 = call <16 x half> @llvm.maxnum.v16f16(<16 x half> undef, <16 x half> undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-F16-LABEL: 'maxnum16'
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.maxnum.f16(half undef, half undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f16 = call <2 x half> @llvm.maxnum.v2f16(<2 x half> undef, <2 x half> undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f16 = call <4 x half> @llvm.maxnum.v4f16(<4 x half> undef, <4 x half> undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8f16 = call <8 x half> @llvm.maxnum.v8f16(<8 x half> undef, <8 x half> undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16f16 = call <16 x half> @llvm.maxnum.v16f16(<16 x half> undef, <16 x half> undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NOF16-LABEL: 'maxnum16'
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 1 for: %f16 = call half @llvm.maxnum.f16(half undef, half undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 4 for: %V2f16 = call <2 x half> @llvm.maxnum.v2f16(<2 x half> undef, <2 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 10 for: %V4f16 = call <4 x half> @llvm.maxnum.v4f16(<4 x half> undef, <4 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 22 for: %V8f16 = call <8 x half> @llvm.maxnum.v8f16(<8 x half> undef, <8 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 44 for: %V16f16 = call <16 x half> @llvm.maxnum.v16f16(<16 x half> undef, <16 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-F16-LABEL: 'maxnum16'
+; CHECK-F16-NEXT:  Cost Model: Found costs of 1 for: %f16 = call half @llvm.maxnum.f16(half undef, half undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 1 for: %V2f16 = call <2 x half> @llvm.maxnum.v2f16(<2 x half> undef, <2 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 1 for: %V4f16 = call <4 x half> @llvm.maxnum.v4f16(<4 x half> undef, <4 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 1 for: %V8f16 = call <8 x half> @llvm.maxnum.v8f16(<8 x half> undef, <8 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 4 for: %V16f16 = call <16 x half> @llvm.maxnum.v16f16(<16 x half> undef, <16 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %f16 = call half @llvm.maxnum.f16(half undef, half undef)
   %V2f16 = call <2 x half> @llvm.maxnum.v2f16(<2 x half> undef, <2 x half> undef)
@@ -369,25 +243,15 @@ define void @maxnum16() {
 }
 
 define void @minnum() {
-; RECIP-LABEL: 'minnum'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f32 = call float @llvm.minnum.f32(float undef, float undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64 = call double @llvm.minnum.f64(double undef, double undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f32 = call <2 x float> @llvm.minnum.v2f32(<2 x float> undef, <2 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f32 = call <4 x float> @llvm.minnum.v4f32(<4 x float> undef, <4 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8f32 = call <8 x float> @llvm.minnum.v8f32(<8 x float> undef, <8 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f64 = call <2 x double> @llvm.minnum.v2f64(<2 x double> undef, <2 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4f64 = call <4 x double> @llvm.minnum.v4f64(<4 x double> undef, <4 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'minnum'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f32 = call float @llvm.minnum.f32(float undef, float undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64 = call double @llvm.minnum.f64(double undef, double undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f32 = call <2 x float> @llvm.minnum.v2f32(<2 x float> undef, <2 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f32 = call <4 x float> @llvm.minnum.v4f32(<4 x float> undef, <4 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8f32 = call <8 x float> @llvm.minnum.v8f32(<8 x float> undef, <8 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f64 = call <2 x double> @llvm.minnum.v2f64(<2 x double> undef, <2 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4f64 = call <4 x double> @llvm.minnum.v4f64(<4 x double> undef, <4 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-LABEL: 'minnum'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f32 = call float @llvm.minnum.f32(float undef, float undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f64 = call double @llvm.minnum.f64(double undef, double undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2f32 = call <2 x float> @llvm.minnum.v2f32(<2 x float> undef, <2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4f32 = call <4 x float> @llvm.minnum.v4f32(<4 x float> undef, <4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8f32 = call <8 x float> @llvm.minnum.v8f32(<8 x float> undef, <8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2f64 = call <2 x double> @llvm.minnum.v2f64(<2 x double> undef, <2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4f64 = call <4 x double> @llvm.minnum.v4f64(<4 x double> undef, <4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %f32 = call float @llvm.minnum.f32(float undef, float undef)
   %f64 = call double @llvm.minnum.f64(double undef, double undef)
@@ -400,25 +264,15 @@ define void @minnum() {
 }
 
 define void @maxnum() {
-; RECIP-LABEL: 'maxnum'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f32 = call float @llvm.maxnum.f32(float undef, float undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64 = call double @llvm.maxnum.f64(double undef, double undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f32 = call <2 x float> @llvm.maxnum.v2f32(<2 x float> undef, <2 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f32 = call <4 x float> @llvm.maxnum.v4f32(<4 x float> undef, <4 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8f32 = call <8 x float> @llvm.maxnum.v8f32(<8 x float> undef, <8 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f64 = call <2 x double> @llvm.maxnum.v2f64(<2 x double> undef, <2 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4f64 = call <4 x double> @llvm.maxnum.v4f64(<4 x double> undef, <4 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'maxnum'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f32 = call float @llvm.maxnum.f32(float undef, float undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64 = call double @llvm.maxnum.f64(double undef, double undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f32 = call <2 x float> @llvm.maxnum.v2f32(<2 x float> undef, <2 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f32 = call <4 x float> @llvm.maxnum.v4f32(<4 x float> undef, <4 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8f32 = call <8 x float> @llvm.maxnum.v8f32(<8 x float> undef, <8 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f64 = call <2 x double> @llvm.maxnum.v2f64(<2 x double> undef, <2 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4f64 = call <4 x double> @llvm.maxnum.v4f64(<4 x double> undef, <4 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-LABEL: 'maxnum'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f32 = call float @llvm.maxnum.f32(float undef, float undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f64 = call double @llvm.maxnum.f64(double undef, double undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2f32 = call <2 x float> @llvm.maxnum.v2f32(<2 x float> undef, <2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4f32 = call <4 x float> @llvm.maxnum.v4f32(<4 x float> undef, <4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8f32 = call <8 x float> @llvm.maxnum.v8f32(<8 x float> undef, <8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2f64 = call <2 x double> @llvm.maxnum.v2f64(<2 x double> undef, <2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4f64 = call <4 x double> @llvm.maxnum.v4f64(<4 x double> undef, <4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %f32 = call float @llvm.maxnum.f32(float undef, float undef)
   %f64 = call double @llvm.maxnum.f64(double undef, double undef)
@@ -432,37 +286,21 @@ define void @maxnum() {
 
 
 define void @minimum16() {
-; RECIP-NOF16-LABEL: 'minimum16'
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.minimum.f16(half undef, half undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2f16 = call <2 x half> @llvm.minimum.v2f16(<2 x half> undef, <2 x half> undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4f16 = call <4 x half> @llvm.minimum.v4f16(<4 x half> undef, <4 x half> undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8f16 = call <8 x half> @llvm.minimum.v8f16(<8 x half> undef, <8 x half> undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16f16 = call <16 x half> @llvm.minimum.v16f16(<16 x half> undef, <16 x half> undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; RECIP-F16-LABEL: 'minimum16'
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.minimum.f16(half undef, half undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f16 = call <2 x half> @llvm.minimum.v2f16(<2 x half> undef, <2 x half> undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f16 = call <4 x half> @llvm.minimum.v4f16(<4 x half> undef, <4 x half> undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8f16 = call <8 x half> @llvm.minimum.v8f16(<8 x half> undef, <8 x half> undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16f16 = call <16 x half> @llvm.minimum.v16f16(<16 x half> undef, <16 x half> undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-NOF16-LABEL: 'minimum16'
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.minimum.f16(half undef, half undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2f16 = call <2 x half> @llvm.minimum.v2f16(<2 x half> undef, <2 x half> undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4f16 = call <4 x half> @llvm.minimum.v4f16(<4 x half> undef, <4 x half> undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8f16 = call <8 x half> @llvm.minimum.v8f16(<8 x half> undef, <8 x half> undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16f16 = call <16 x half> @llvm.minimum.v16f16(<16 x half> undef, <16 x half> undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-F16-LABEL: 'minimum16'
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.minimum.f16(half undef, half undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f16 = call <2 x half> @llvm.minimum.v2f16(<2 x half> undef, <2 x half> undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f16 = call <4 x half> @llvm.minimum.v4f16(<4 x half> undef, <4 x half> undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8f16 = call <8 x half> @llvm.minimum.v8f16(<8 x half> undef, <8 x half> undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16f16 = call <16 x half> @llvm.minimum.v16f16(<16 x half> undef, <16 x half> undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NOF16-LABEL: 'minimum16'
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 1 for: %f16 = call half @llvm.minimum.f16(half undef, half undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 4 for: %V2f16 = call <2 x half> @llvm.minimum.v2f16(<2 x half> undef, <2 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 10 for: %V4f16 = call <4 x half> @llvm.minimum.v4f16(<4 x half> undef, <4 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 22 for: %V8f16 = call <8 x half> @llvm.minimum.v8f16(<8 x half> undef, <8 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 44 for: %V16f16 = call <16 x half> @llvm.minimum.v16f16(<16 x half> undef, <16 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-F16-LABEL: 'minimum16'
+; CHECK-F16-NEXT:  Cost Model: Found costs of 1 for: %f16 = call half @llvm.minimum.f16(half undef, half undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 1 for: %V2f16 = call <2 x half> @llvm.minimum.v2f16(<2 x half> undef, <2 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 1 for: %V4f16 = call <4 x half> @llvm.minimum.v4f16(<4 x half> undef, <4 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 1 for: %V8f16 = call <8 x half> @llvm.minimum.v8f16(<8 x half> undef, <8 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 4 for: %V16f16 = call <16 x half> @llvm.minimum.v16f16(<16 x half> undef, <16 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %f16 = call half @llvm.minimum.f16(half undef, half undef)
   %V2f16 = call <2 x half> @llvm.minimum.v2f16(<2 x half> undef, <2 x half> undef)
@@ -473,37 +311,21 @@ define void @minimum16() {
 }
 
 define void @maximum16() {
-; RECIP-NOF16-LABEL: 'maximum16'
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.maximum.f16(half undef, half undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2f16 = call <2 x half> @llvm.maximum.v2f16(<2 x half> undef, <2 x half> undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4f16 = call <4 x half> @llvm.maximum.v4f16(<4 x half> undef, <4 x half> undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8f16 = call <8 x half> @llvm.maximum.v8f16(<8 x half> undef, <8 x half> undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16f16 = call <16 x half> @llvm.maximum.v16f16(<16 x half> undef, <16 x half> undef)
-; RECIP-NOF16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; RECIP-F16-LABEL: 'maximum16'
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.maximum.f16(half undef, half undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f16 = call <2 x half> @llvm.maximum.v2f16(<2 x half> undef, <2 x half> undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f16 = call <4 x half> @llvm.maximum.v4f16(<4 x half> undef, <4 x half> undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8f16 = call <8 x half> @llvm.maximum.v8f16(<8 x half> undef, <8 x half> undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16f16 = call <16 x half> @llvm.maximum.v16f16(<16 x half> undef, <16 x half> undef)
-; RECIP-F16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-NOF16-LABEL: 'maximum16'
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.maximum.f16(half undef, half undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2f16 = call <2 x half> @llvm.maximum.v2f16(<2 x half> undef, <2 x half> undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4f16 = call <4 x half> @llvm.maximum.v4f16(<4 x half> undef, <4 x half> undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8f16 = call <8 x half> @llvm.maximum.v8f16(<8 x half> undef, <8 x half> undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16f16 = call <16 x half> @llvm.maximum.v16f16(<16 x half> undef, <16 x half> undef)
-; SIZE-NOF16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-F16-LABEL: 'maximum16'
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.maximum.f16(half undef, half undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f16 = call <2 x half> @llvm.maximum.v2f16(<2 x half> undef, <2 x half> undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f16 = call <4 x half> @llvm.maximum.v4f16(<4 x half> undef, <4 x half> undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8f16 = call <8 x half> @llvm.maximum.v8f16(<8 x half> undef, <8 x half> undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16f16 = call <16 x half> @llvm.maximum.v16f16(<16 x half> undef, <16 x half> undef)
-; SIZE-F16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NOF16-LABEL: 'maximum16'
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 1 for: %f16 = call half @llvm.maximum.f16(half undef, half undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 4 for: %V2f16 = call <2 x half> @llvm.maximum.v2f16(<2 x half> undef, <2 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 10 for: %V4f16 = call <4 x half> @llvm.maximum.v4f16(<4 x half> undef, <4 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 22 for: %V8f16 = call <8 x half> @llvm.maximum.v8f16(<8 x half> undef, <8 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of 44 for: %V16f16 = call <16 x half> @llvm.maximum.v16f16(<16 x half> undef, <16 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-F16-LABEL: 'maximum16'
+; CHECK-F16-NEXT:  Cost Model: Found costs of 1 for: %f16 = call half @llvm.maximum.f16(half undef, half undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 1 for: %V2f16 = call <2 x half> @llvm.maximum.v2f16(<2 x half> undef, <2 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 1 for: %V4f16 = call <4 x half> @llvm.maximum.v4f16(<4 x half> undef, <4 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 1 for: %V8f16 = call <8 x half> @llvm.maximum.v8f16(<8 x half> undef, <8 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 4 for: %V16f16 = call <16 x half> @llvm.maximum.v16f16(<16 x half> undef, <16 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %f16 = call half @llvm.maximum.f16(half undef, half undef)
   %V2f16 = call <2 x half> @llvm.maximum.v2f16(<2 x half> undef, <2 x half> undef)
@@ -514,25 +336,15 @@ define void @maximum16() {
 }
 
 define void @minimum() {
-; RECIP-LABEL: 'minimum'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f32 = call float @llvm.minimum.f32(float undef, float undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64 = call double @llvm.minimum.f64(double undef, double undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f32 = call <2 x float> @llvm.minimum.v2f32(<2 x float> undef, <2 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f32 = call <4 x float> @llvm.minimum.v4f32(<4 x float> undef, <4 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8f32 = call <8 x float> @llvm.minimum.v8f32(<8 x float> undef, <8 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f64 = call <2 x double> @llvm.minimum.v2f64(<2 x double> undef, <2 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4f64 = call <4 x double> @llvm.minimum.v4f64(<4 x double> undef, <4 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'minimum'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f32 = call float @llvm.minimum.f32(float undef, float undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64 = call double @llvm.minimum.f64(double undef, double undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f32 = call <2 x float> @llvm.minimum.v2f32(<2 x float> undef, <2 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f32 = call <4 x float> @llvm.minimum.v4f32(<4 x float> undef, <4 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8f32 = call <8 x float> @llvm.minimum.v8f32(<8 x float> undef, <8 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f64 = call <2 x double> @llvm.minimum.v2f64(<2 x double> undef, <2 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4f64 = call <4 x double> @llvm.minimum.v4f64(<4 x double> undef, <4 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-LABEL: 'minimum'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f32 = call float @llvm.minimum.f32(float undef, float undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f64 = call double @llvm.minimum.f64(double undef, double undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2f32 = call <2 x float> @llvm.minimum.v2f32(<2 x float> undef, <2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4f32 = call <4 x float> @llvm.minimum.v4f32(<4 x float> undef, <4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8f32 = call <8 x float> @llvm.minimum.v8f32(<8 x float> undef, <8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2f64 = call <2 x double> @llvm.minimum.v2f64(<2 x double> undef, <2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4f64 = call <4 x double> @llvm.minimum.v4f64(<4 x double> undef, <4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %f32 = call float @llvm.minimum.f32(float undef, float undef)
   %f64 = call double @llvm.minimum.f64(double undef, double undef)
@@ -545,25 +357,15 @@ define void @minimum() {
 }
 
 define void @maximum() {
-; RECIP-LABEL: 'maximum'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f32 = call float @llvm.maximum.f32(float undef, float undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64 = call double @llvm.maximum.f64(double undef, double undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f32 = call <2 x float> @llvm.maximum.v2f32(<2 x float> undef, <2 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f32 = call <4 x float> @llvm.maximum.v4f32(<4 x float> undef, <4 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8f32 = call <8 x float> @llvm.maximum.v8f32(<8 x float> undef, <8 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f64 = call <2 x double> @llvm.maximum.v2f64(<2 x double> undef, <2 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4f64 = call <4 x double> @llvm.maximum.v4f64(<4 x double> undef, <4 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'maximum'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f32 = call float @llvm.maximum.f32(float undef, float undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64 = call double @llvm.maximum.f64(double undef, double undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f32 = call <2 x float> @llvm.maximum.v2f32(<2 x float> undef, <2 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f32 = call <4 x float> @llvm.maximum.v4f32(<4 x float> undef, <4 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8f32 = call <8 x float> @llvm.maximum.v8f32(<8 x float> undef, <8 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f64 = call <2 x double> @llvm.maximum.v2f64(<2 x double> undef, <2 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4f64 = call <4 x double> @llvm.maximum.v4f64(<4 x double> undef, <4 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-LABEL: 'maximum'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f32 = call float @llvm.maximum.f32(float undef, float undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %f64 = call double @llvm.maximum.f64(double undef, double undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2f32 = call <2 x float> @llvm.maximum.v2f32(<2 x float> undef, <2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4f32 = call <4 x float> @llvm.maximum.v4f32(<4 x float> undef, <4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8f32 = call <8 x float> @llvm.maximum.v8f32(<8 x float> undef, <8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2f64 = call <2 x double> @llvm.maximum.v2f64(<2 x double> undef, <2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4f64 = call <4 x double> @llvm.maximum.v4f64(<4 x double> undef, <4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %f32 = call float @llvm.maximum.f32(float undef, float undef)
   %f64 = call double @llvm.maximum.f64(double undef, double undef)
diff --git a/llvm/test/Analysis/CostModel/AArch64/mul.ll b/llvm/test/Analysis/CostModel/AArch64/mul.ll
index f31248d53f8ff..7f65ddea582eb 100644
--- a/llvm/test/Analysis/CostModel/AArch64/mul.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/mul.ll
@@ -1,188 +1,131 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s --check-prefix=RECIP
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output | FileCheck %s --check-prefix=COST
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
 
 ; Verify the cost of (vector) multiply instructions.
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define <2 x i8> @t1(<2 x i8> %a, <2 x i8> %b)  {
-; RECIP-LABEL: 't1'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <2 x i8> %a, %b
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i8> %1
-;
-; COST-LABEL: 't1'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <2 x i8> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i8> %1
+; CHECK-LABEL: 't1'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = mul <2 x i8> %a, %b
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i8> %1
 ;
   %1 = mul <2 x i8> %a, %b
   ret <2 x i8> %1
 }
 
 define <4 x i8> @t2(<4 x i8> %a, <4 x i8> %b)  {
-; RECIP-LABEL: 't2'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <4 x i8> %a, %b
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %1
-;
-; COST-LABEL: 't2'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <4 x i8> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i8> %1
+; CHECK-LABEL: 't2'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = mul <4 x i8> %a, %b
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %1
 ;
   %1 = mul <4 x i8> %a, %b
   ret <4 x i8> %1
 }
 
 define <8 x i8> @t3(<8 x i8> %a, <8 x i8> %b)  {
-; RECIP-LABEL: 't3'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <8 x i8> %a, %b
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i8> %1
-;
-; COST-LABEL: 't3'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <8 x i8> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i8> %1
+; CHECK-LABEL: 't3'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = mul <8 x i8> %a, %b
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %1
 ;
   %1 = mul <8 x i8> %a, %b
   ret <8 x i8> %1
 }
 
 define <16 x i8> @t4(<16 x i8> %a, <16 x i8> %b)  {
-; RECIP-LABEL: 't4'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <16 x i8> %a, %b
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %1
-;
-; COST-LABEL: 't4'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <16 x i8> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %1
+; CHECK-LABEL: 't4'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = mul <16 x i8> %a, %b
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %1
 ;
   %1 = mul <16 x i8> %a, %b
   ret <16 x i8> %1
 }
 
 define <32 x i8> @t5(<32 x i8> %a, <32 x i8> %b)  {
-; RECIP-LABEL: 't5'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = mul <32 x i8> %a, %b
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %1
-;
-; COST-LABEL: 't5'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <32 x i8> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %1
+; CHECK-LABEL: 't5'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %1 = mul <32 x i8> %a, %b
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <32 x i8> %1
 ;
   %1 = mul <32 x i8> %a, %b
   ret <32 x i8> %1
 }
 
 define <2 x i16> @t6(<2 x i16> %a, <2 x i16> %b)  {
-; RECIP-LABEL: 't6'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <2 x i16> %a, %b
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i16> %1
-;
-; COST-LABEL: 't6'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <2 x i16> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i16> %1
+; CHECK-LABEL: 't6'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = mul <2 x i16> %a, %b
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i16> %1
 ;
   %1 = mul <2 x i16> %a, %b
   ret <2 x i16> %1
 }
 
 define <4 x i16> @t7(<4 x i16> %a, <4 x i16> %b)  {
-; RECIP-LABEL: 't7'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <4 x i16> %a, %b
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i16> %1
-;
-; COST-LABEL: 't7'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <4 x i16> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i16> %1
+; CHECK-LABEL: 't7'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = mul <4 x i16> %a, %b
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i16> %1
 ;
   %1 = mul <4 x i16> %a, %b
   ret <4 x i16> %1
 }
 
 define <8 x i16> @t8(<8 x i16> %a, <8 x i16> %b)  {
-; RECIP-LABEL: 't8'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <8 x i16> %a, %b
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %1
-;
-; COST-LABEL: 't8'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <8 x i16> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %1
+; CHECK-LABEL: 't8'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = mul <8 x i16> %a, %b
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i16> %1
 ;
   %1 = mul <8 x i16> %a, %b
   ret <8 x i16> %1
 }
 
 define <16 x i16> @t9(<16 x i16> %a, <16 x i16> %b)  {
-; RECIP-LABEL: 't9'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = mul <16 x i16> %a, %b
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %1
-;
-; COST-LABEL: 't9'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <16 x i16> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %1
+; CHECK-LABEL: 't9'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %1 = mul <16 x i16> %a, %b
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i16> %1
 ;
   %1 = mul <16 x i16> %a, %b
   ret <16 x i16> %1
 }
 
 define <2 x i32> @t10(<2 x i32> %a, <2 x i32> %b)  {
-; RECIP-LABEL: 't10'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <2 x i32> %a, %b
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %1
-;
-; COST-LABEL: 't10'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <2 x i32> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i32> %1
+; CHECK-LABEL: 't10'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = mul <2 x i32> %a, %b
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i32> %1
 ;
   %1 = mul <2 x i32> %a, %b
   ret <2 x i32> %1
 }
 
 define <4 x i32> @t11(<4 x i32> %a, <4 x i32> %b)  {
-; RECIP-LABEL: 't11'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <4 x i32> %a, %b
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %1
-;
-; COST-LABEL: 't11'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <4 x i32> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %1
+; CHECK-LABEL: 't11'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = mul <4 x i32> %a, %b
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %1
 ;
   %1 = mul <4 x i32> %a, %b
   ret <4 x i32> %1
 }
 
 define <8 x i32> @t12(<8 x i32> %a, <8 x i32> %b)  {
-; RECIP-LABEL: 't12'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = mul <8 x i32> %a, %b
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %1
-;
-; COST-LABEL: 't12'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul <8 x i32> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %1
+; CHECK-LABEL: 't12'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %1 = mul <8 x i32> %a, %b
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i32> %1
 ;
   %1 = mul <8 x i32> %a, %b
   ret <8 x i32> %1
 }
 
 define <2 x i64> @t13(<2 x i64> %a, <2 x i64> %b)  {
-; RECIP-LABEL: 't13'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %1 = mul nsw <2 x i64> %a, %b
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %1
-;
-; COST-LABEL: 't13'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul nsw <2 x i64> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %1
+; CHECK-LABEL: 't13'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:1 Lat:1 SizeLat:1 for: %1 = mul nsw <2 x i64> %a, %b
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i64> %1
 ;
   %1 = mul nsw <2 x i64> %a, %b
   ret <2 x i64> %1
 }
 
 define <4 x i64> @t14(<4 x i64> %a, <4 x i64> %b)  {
-; RECIP-LABEL: 't14'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %1 = mul nsw <4 x i64> %a, %b
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %1
-;
-; COST-LABEL: 't14'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul nsw <4 x i64> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %1
+; CHECK-LABEL: 't14'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:1 Lat:1 SizeLat:1 for: %1 = mul nsw <4 x i64> %a, %b
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i64> %1
 ;
   %1 = mul nsw <4 x i64> %a, %b
   ret <4 x i64> %1
diff --git a/llvm/test/Analysis/CostModel/AArch64/reduce-add.ll b/llvm/test/Analysis/CostModel/AArch64/reduce-add.ll
index 7341363f36d18..521264be8b31c 100644
--- a/llvm/test/Analysis/CostModel/AArch64/reduce-add.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/reduce-add.ll
@@ -1,47 +1,27 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s --check-prefix=RECIP
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output | FileCheck %s --check-prefix=COST
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define void @reduce() {
-; RECIP-LABEL: 'reduce'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1i8 = call i8 @llvm.vector.reduce.add.v1i8(<1 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V3i8 = call i8 @llvm.vector.reduce.add.v3i8(<3 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i8 = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i8 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16i8 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32i8 = call i8 @llvm.vector.reduce.add.v32i8(<32 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64i8 = call i8 @llvm.vector.reduce.add.v64i8(<64 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i16 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16i16 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i32 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; COST-LABEL: 'reduce'
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V1i8 = call i8 @llvm.vector.reduce.add.v1i8(<1 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V3i8 = call i8 @llvm.vector.reduce.add.v3i8(<3 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i8 = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i8 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16i8 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32i8 = call i8 @llvm.vector.reduce.add.v32i8(<32 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64i8 = call i8 @llvm.vector.reduce.add.v64i8(<64 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i16 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16i16 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i32 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i32 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'reduce'
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V1i8 = call i8 @llvm.vector.reduce.add.v1i8(<1 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V3i8 = call i8 @llvm.vector.reduce.add.v3i8(<3 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V4i8 = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8i8 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16i8 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %V32i8 = call i8 @llvm.vector.reduce.add.v32i8(<32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 5 for: %V64i8 = call i8 @llvm.vector.reduce.add.v64i8(<64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V2i16 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V4i16 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8i16 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %V16i16 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V2i32 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V4i32 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %V8i32 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V2i64 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %V4i64 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %V1i8 = call i8 @llvm.vector.reduce.add.v1i8(<1 x i8> undef)
   %V3i8 = call i8 @llvm.vector.reduce.add.v3i8(<3 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/AArch64/reduce-and.ll b/llvm/test/Analysis/CostModel/AArch64/reduce-and.ll
index 632cc4dc3e7d8..b484f8f6c60ba 100644
--- a/llvm/test/Analysis/CostModel/AArch64/reduce-and.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/reduce-and.ll
@@ -1,61 +1,34 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s --check-prefix=RECIP
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output | FileCheck %s --check-prefix=COST
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define void @reduce() {
-; RECIP-LABEL: 'reduce'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1i8 = call i8 @llvm.vector.reduce.and.v1i8(<1 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V3i8 = call i8 @llvm.vector.reduce.and.v3i8(<3 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4i8 = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8i8 = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V16i8 = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V32i8 = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64i8 = call i8 @llvm.vector.reduce.and.v64i8(<64 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4i16 = call i16 @llvm.vector.reduce.and.v4i16(<4 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i16 = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16i16 = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4i32 = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i32 = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i64 = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; COST-LABEL: 'reduce'
-; COST-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1i8 = call i8 @llvm.vector.reduce.and.v1i8(<1 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V3i8 = call i8 @llvm.vector.reduce.and.v3i8(<3 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4i8 = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8i8 = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V16i8 = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V32i8 = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64i8 = call i8 @llvm.vector.reduce.and.v64i8(<64 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4i16 = call i16 @llvm.vector.reduce.and.v4i16(<4 x i16> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i16 = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16i16 = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4i32 = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i32 = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i64 = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'reduce'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %V1 = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 5 for: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 9 for: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %V1i8 = call i8 @llvm.vector.reduce.and.v1i8(<1 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V3i8 = call i8 @llvm.vector.reduce.and.v3i8(<3 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %V4i8 = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 15 for: %V8i8 = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 17 for: %V16i8 = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 18 for: %V32i8 = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 20 for: %V64i8 = call i8 @llvm.vector.reduce.and.v64i8(<64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %V4i16 = call i16 @llvm.vector.reduce.and.v4i16(<4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 9 for: %V8i16 = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 10 for: %V16i16 = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %V2i32 = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 5 for: %V4i32 = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 6 for: %V8i32 = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %V2i64 = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4i64 = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 
   %V1   = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/AArch64/reduce-fadd.ll b/llvm/test/Analysis/CostModel/AArch64/reduce-fadd.ll
index 79be838ed2f9e..32a12d83e5534 100644
--- a/llvm/test/Analysis/CostModel/AArch64/reduce-fadd.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/reduce-fadd.ll
@@ -1,31 +1,19 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt -passes='print<cost-model>' 2>&1 -cost-kind=throughput -disable-output -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=RECIP,RECIP-NOFP16
-; RUN: opt -passes='print<cost-model>' 2>&1 -cost-kind=throughput -disable-output -mtriple=aarch64-linux-gnu -mattr=+fullfp16 < %s | FileCheck %s --check-prefixes=RECIP,RECIP-FP16
-; RUN: opt -passes='print<cost-model>' 2>&1 -cost-kind=throughput -disable-output -mtriple=aarch64-linux-gnu -mattr=+bf16 < %s | FileCheck %s --check-prefixes=RECIP,RECIPBF16
-; RUN: opt -passes='print<cost-model>' 2>&1 -cost-kind=code-size -disable-output -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=SIZE,SIZE-NOFP16
-; RUN: opt -passes='print<cost-model>' 2>&1 -cost-kind=code-size -disable-output -mtriple=aarch64-linux-gnu -mattr=+fullfp16 < %s | FileCheck %s --check-prefixes=SIZE,SIZE-FP16
-; RUN: opt -passes='print<cost-model>' 2>&1 -cost-kind=code-size -disable-output -mtriple=aarch64-linux-gnu -mattr=+bf16 < %s | FileCheck %s --check-prefixes=SIZE,SIZE-BF16
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP16
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-F16
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s -mattr=+bf16 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define void @strict_fp_reductions() {
-; RECIP-LABEL: 'strict_fp_reductions'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fadd_v2f32 = call float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %fadd_v4f32 = call float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %fadd_v8f32 = call float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fadd_v2f64 = call double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %fadd_v4f64 = call double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %fadd_v4f128 = call fp128 @llvm.vector.reduce.fadd.v4f128(fp128 undef, <4 x fp128> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'strict_fp_reductions'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fadd_v2f32 = call float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %fadd_v4f32 = call float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %fadd_v8f32 = call float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fadd_v2f64 = call double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %fadd_v4f64 = call double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_v4f128 = call fp128 @llvm.vector.reduce.fadd.v4f128(fp128 undef, <4 x fp128> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'strict_fp_reductions'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:6 Lat:10 SizeLat:6 for: %fadd_v2f32 = call float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:14 Lat:22 SizeLat:14 for: %fadd_v4f32 = call float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:28 Lat:44 SizeLat:28 for: %fadd_v8f32 = call float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:6 Lat:10 SizeLat:6 for: %fadd_v2f64 = call double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:12 Lat:20 SizeLat:12 for: %fadd_v4f64 = call double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:8 Lat:16 SizeLat:8 for: %fadd_v4f128 = call fp128 @llvm.vector.reduce.fadd.v4f128(fp128 undef, <4 x fp128> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %fadd_v2f32 = call float @llvm.vector.reduce.fadd.v2f32(float 0.0, <2 x float> undef)
   %fadd_v4f32 = call float @llvm.vector.reduce.fadd.v4f32(float 0.0, <4 x float> undef)
@@ -37,33 +25,26 @@ define void @strict_fp_reductions() {
 }
 
 define void @strict_fp_reductions_fp16() {
-; RECIP-NOFP16-LABEL: 'strict_fp_reductions_fp16'
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_v2f16 = call half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %fadd_v4f16 = call half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %fadd_v8f16 = call half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %fadd_v16f16 = call half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; RECIP-FP16-LABEL: 'strict_fp_reductions_fp16'
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fadd_v2f16 = call half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %fadd_v4f16 = call half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %fadd_v8f16 = call half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %fadd_v16f16 = call half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; RECIPBF16-LABEL: 'strict_fp_reductions_fp16'
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_v2f16 = call half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %fadd_v4f16 = call half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %fadd_v8f16 = call half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %fadd_v16f16 = call half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'strict_fp_reductions_fp16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %fadd_v2f16 = call half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %fadd_v4f16 = call half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %fadd_v8f16 = call half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %fadd_v16f16 = call half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-NOFP16-LABEL: 'strict_fp_reductions_fp16'
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:6 Lat:10 SizeLat:6 for: %fadd_v2f16 = call half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:18 CodeSize:14 Lat:22 SizeLat:14 for: %fadd_v4f16 = call half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:38 CodeSize:30 Lat:46 SizeLat:30 for: %fadd_v8f16 = call half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:76 CodeSize:60 Lat:92 SizeLat:60 for: %fadd_v16f16 = call half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-F16-LABEL: 'strict_fp_reductions_fp16'
+; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:6 Lat:10 SizeLat:6 for: %fadd_v2f16 = call half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:14 Lat:22 SizeLat:14 for: %fadd_v4f16 = call half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:30 CodeSize:30 Lat:46 SizeLat:30 for: %fadd_v8f16 = call half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:60 CodeSize:60 Lat:92 SizeLat:60 for: %fadd_v16f16 = call half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-BF16-LABEL: 'strict_fp_reductions_fp16'
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:6 Lat:10 SizeLat:6 for: %fadd_v2f16 = call half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:18 CodeSize:14 Lat:22 SizeLat:14 for: %fadd_v4f16 = call half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:38 CodeSize:30 Lat:46 SizeLat:30 for: %fadd_v8f16 = call half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:76 CodeSize:60 Lat:92 SizeLat:60 for: %fadd_v16f16 = call half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %fadd_v2f16 = call half @llvm.vector.reduce.fadd.v2f16(half 0.0, <2 x half> undef)
   %fadd_v4f16 = call half @llvm.vector.reduce.fadd.v4f16(half 0.0, <4 x half> undef)
@@ -73,62 +54,40 @@ define void @strict_fp_reductions_fp16() {
 }
 
 define void @strict_fp_reductions_bf16() {
-; RECIP-NOFP16-LABEL: 'strict_fp_reductions_bf16'
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %fadd_v4f8 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; RECIP-FP16-LABEL: 'strict_fp_reductions_bf16'
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %fadd_v4f8 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NOFP16-LABEL: 'strict_fp_reductions_bf16'
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:18 CodeSize:14 Lat:22 SizeLat:14 for: %fadd_v4f8 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
-; RECIPBF16-LABEL: 'strict_fp_reductions_bf16'
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %fadd_v4f8 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-F16-LABEL: 'strict_fp_reductions_bf16'
+; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:18 CodeSize:14 Lat:22 SizeLat:14 for: %fadd_v4f8 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
-; SIZE-LABEL: 'strict_fp_reductions_bf16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %fadd_v4f8 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-BF16-LABEL: 'strict_fp_reductions_bf16'
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:14 Lat:22 SizeLat:14 for: %fadd_v4f8 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %fadd_v4f8 = call bfloat @llvm.vector.reduce.fadd.v4f8(bfloat 0.0, <4 x bfloat> undef)
   ret void
 }
 
 define void @fast_fp_reductions() {
-; RECIP-LABEL: 'fast_fp_reductions'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %fadd_v2f32 = call fast float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %fadd_v2f32_reassoc = call reassoc float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_v4f32 = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_v4f32_reassoc = call reassoc float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fadd_v8f32 = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fadd_v8f32_reassoc = call reassoc float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %fadd_v13f32 = call fast float @llvm.vector.reduce.fadd.v13f32(float 0.000000e+00, <13 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fadd_v5f32_reassoc = call reassoc float @llvm.vector.reduce.fadd.v5f32(float 0.000000e+00, <5 x float> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %fadd_v2f64 = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %fadd_v2f64_reassoc = call reassoc double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_v4f64 = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_v4f64_reassoc = call reassoc double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_v7f64 = call fast double @llvm.vector.reduce.fadd.v7f64(double 0.000000e+00, <7 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_v9f64_reassoc = call reassoc double @llvm.vector.reduce.fadd.v9f64(double 0.000000e+00, <9 x double> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %fadd_v4f128 = call reassoc fp128 @llvm.vector.reduce.fadd.v4f128(fp128 undef, <4 x fp128> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'fast_fp_reductions'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %fadd_v2f32 = call fast float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %fadd_v2f32_reassoc = call reassoc float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_v4f32 = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_v4f32_reassoc = call reassoc float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fadd_v8f32 = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fadd_v8f32_reassoc = call reassoc float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %fadd_v13f32 = call fast float @llvm.vector.reduce.fadd.v13f32(float 0.000000e+00, <13 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fadd_v5f32_reassoc = call reassoc float @llvm.vector.reduce.fadd.v5f32(float 0.000000e+00, <5 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %fadd_v2f64 = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %fadd_v2f64_reassoc = call reassoc double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_v4f64 = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_v4f64_reassoc = call reassoc double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_v7f64 = call fast double @llvm.vector.reduce.fadd.v7f64(double 0.000000e+00, <7 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_v9f64_reassoc = call reassoc double @llvm.vector.reduce.fadd.v9f64(double 0.000000e+00, <9 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_v4f128 = call reassoc fp128 @llvm.vector.reduce.fadd.v4f128(fp128 undef, <4 x fp128> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'fast_fp_reductions'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %fadd_v2f32 = call fast float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %fadd_v2f32_reassoc = call reassoc float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %fadd_v4f32 = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %fadd_v4f32_reassoc = call reassoc float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %fadd_v8f32 = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %fadd_v8f32_reassoc = call reassoc float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 5 for: %fadd_v13f32 = call fast float @llvm.vector.reduce.fadd.v13f32(float 0.000000e+00, <13 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %fadd_v5f32_reassoc = call reassoc float @llvm.vector.reduce.fadd.v5f32(float 0.000000e+00, <5 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %fadd_v2f64 = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %fadd_v2f64_reassoc = call reassoc double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %fadd_v4f64 = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %fadd_v4f64_reassoc = call reassoc double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %fadd_v7f64 = call fast double @llvm.vector.reduce.fadd.v7f64(double 0.000000e+00, <7 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %fadd_v9f64_reassoc = call reassoc double @llvm.vector.reduce.fadd.v9f64(double 0.000000e+00, <9 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:2 Lat:6 SizeLat:2 for: %fadd_v4f128 = call reassoc fp128 @llvm.vector.reduce.fadd.v4f128(fp128 undef, <4 x fp128> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %fadd_v2f32 = call fast float @llvm.vector.reduce.fadd.v2f32(float 0.0, <2 x float> undef)
   %fadd_v2f32_reassoc = call reassoc float @llvm.vector.reduce.fadd.v2f32(float 0.0, <2 x float> undef)
@@ -157,83 +116,44 @@ define void @fast_fp_reductions() {
 }
 
 define void @fast_fp_reductions_fp16() {
-; RECIP-NOFP16-LABEL: 'fast_fp_reductions_fp16'
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %fadd_v2f16_fast = call fast half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %fadd_v2f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %fadd_v4f16_fast = call fast half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %fadd_v4f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %fadd_v8f16 = call fast half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %fadd_v8f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %fadd_v16f16 = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %fadd_v16f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %fadd_v11f16 = call fast half @llvm.vector.reduce.fadd.v11f16(half 0xH0000, <11 x half> undef)
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %fadd_v13f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v13f16(half 0xH0000, <13 x half> undef)
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; RECIP-FP16-LABEL: 'fast_fp_reductions_fp16'
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_v2f16_fast = call fast half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_v2f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_v4f16_fast = call fast half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_v4f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fadd_v8f16 = call fast half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fadd_v8f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_v16f16 = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_v16f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_v11f16 = call fast half @llvm.vector.reduce.fadd.v11f16(half 0xH0000, <11 x half> undef)
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_v13f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v13f16(half 0xH0000, <13 x half> undef)
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; RECIPBF16-LABEL: 'fast_fp_reductions_fp16'
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %fadd_v2f16_fast = call fast half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %fadd_v2f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %fadd_v4f16_fast = call fast half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %fadd_v4f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %fadd_v8f16 = call fast half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %fadd_v8f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %fadd_v16f16 = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %fadd_v16f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %fadd_v11f16 = call fast half @llvm.vector.reduce.fadd.v11f16(half 0xH0000, <11 x half> undef)
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %fadd_v13f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v13f16(half 0xH0000, <13 x half> undef)
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-NOFP16-LABEL: 'fast_fp_reductions_fp16'
-; SIZE-NOFP16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_v2f16_fast = call fast half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; SIZE-NOFP16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_v2f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; SIZE-NOFP16-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_v4f16_fast = call fast half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; SIZE-NOFP16-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_v4f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; SIZE-NOFP16-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %fadd_v8f16 = call fast half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; SIZE-NOFP16-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %fadd_v8f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; SIZE-NOFP16-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %fadd_v16f16 = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; SIZE-NOFP16-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %fadd_v16f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; SIZE-NOFP16-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %fadd_v11f16 = call fast half @llvm.vector.reduce.fadd.v11f16(half 0xH0000, <11 x half> undef)
-; SIZE-NOFP16-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %fadd_v13f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v13f16(half 0xH0000, <13 x half> undef)
-; SIZE-NOFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
-;
-; SIZE-FP16-LABEL: 'fast_fp_reductions_fp16'
-; SIZE-FP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_v2f16_fast = call fast half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; SIZE-FP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_v2f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; SIZE-FP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_v4f16_fast = call fast half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; SIZE-FP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_v4f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; SIZE-FP16-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fadd_v8f16 = call fast half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; SIZE-FP16-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fadd_v8f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; SIZE-FP16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_v16f16 = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; SIZE-FP16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_v16f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; SIZE-FP16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_v11f16 = call fast half @llvm.vector.reduce.fadd.v11f16(half 0xH0000, <11 x half> undef)
-; SIZE-FP16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_v13f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v13f16(half 0xH0000, <13 x half> undef)
-; SIZE-FP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
-;
-; SIZE-BF16-LABEL: 'fast_fp_reductions_fp16'
-; SIZE-BF16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_v2f16_fast = call fast half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; SIZE-BF16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_v2f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; SIZE-BF16-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_v4f16_fast = call fast half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; SIZE-BF16-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_v4f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; SIZE-BF16-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %fadd_v8f16 = call fast half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; SIZE-BF16-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %fadd_v8f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; SIZE-BF16-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %fadd_v16f16 = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; SIZE-BF16-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %fadd_v16f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; SIZE-BF16-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %fadd_v11f16 = call fast half @llvm.vector.reduce.fadd.v11f16(half 0xH0000, <11 x half> undef)
-; SIZE-BF16-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %fadd_v13f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v13f16(half 0xH0000, <13 x half> undef)
-; SIZE-BF16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-NOFP16-LABEL: 'fast_fp_reductions_fp16'
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:6 SizeLat:4 for: %fadd_v2f16_fast = call fast half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:6 SizeLat:4 for: %fadd_v2f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f16_fast = call fast half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:30 CodeSize:27 Lat:33 SizeLat:27 for: %fadd_v8f16 = call fast half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:30 CodeSize:27 Lat:33 SizeLat:27 for: %fadd_v8f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:44 Lat:52 SizeLat:44 for: %fadd_v16f16 = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:44 Lat:52 SizeLat:44 for: %fadd_v16f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:38 CodeSize:35 Lat:41 SizeLat:35 for: %fadd_v11f16 = call fast half @llvm.vector.reduce.fadd.v11f16(half 0xH0000, <11 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:38 CodeSize:35 Lat:41 SizeLat:35 for: %fadd_v13f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v13f16(half 0xH0000, <13 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-F16-LABEL: 'fast_fp_reductions_fp16'
+; CHECK-F16-NEXT:  Cost Model: Found costs of 2 for: %fadd_v2f16_fast = call fast half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 2 for: %fadd_v2f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 2 for: %fadd_v4f16_fast = call fast half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 2 for: %fadd_v4f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 3 for: %fadd_v8f16 = call fast half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 3 for: %fadd_v8f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 4 for: %fadd_v16f16 = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 4 for: %fadd_v16f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 4 for: %fadd_v11f16 = call fast half @llvm.vector.reduce.fadd.v11f16(half 0xH0000, <11 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of 4 for: %fadd_v13f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v13f16(half 0xH0000, <13 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-BF16-LABEL: 'fast_fp_reductions_fp16'
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:6 SizeLat:4 for: %fadd_v2f16_fast = call fast half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:6 SizeLat:4 for: %fadd_v2f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f16_fast = call fast half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:30 CodeSize:27 Lat:33 SizeLat:27 for: %fadd_v8f16 = call fast half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:30 CodeSize:27 Lat:33 SizeLat:27 for: %fadd_v8f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:44 Lat:52 SizeLat:44 for: %fadd_v16f16 = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:44 Lat:52 SizeLat:44 for: %fadd_v16f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:38 CodeSize:35 Lat:41 SizeLat:35 for: %fadd_v11f16 = call fast half @llvm.vector.reduce.fadd.v11f16(half 0xH0000, <11 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:38 CodeSize:35 Lat:41 SizeLat:35 for: %fadd_v13f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v13f16(half 0xH0000, <13 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %fadd_v2f16_fast = call fast half @llvm.vector.reduce.fadd.v2f16(half 0.0, <2 x half> undef)
   %fadd_v2f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v2f16(half 0.0, <2 x half> undef)
@@ -254,21 +174,17 @@ define void @fast_fp_reductions_fp16() {
 }
 
 define void @fast_fp_reductions_bf16() {
-; RECIP-NOFP16-LABEL: 'fast_fp_reductions_bf16'
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %fadd_v4f8 = call reassoc bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR8000, <4 x bfloat> undef)
-; RECIP-NOFP16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; RECIP-FP16-LABEL: 'fast_fp_reductions_bf16'
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %fadd_v4f8 = call reassoc bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR8000, <4 x bfloat> undef)
-; RECIP-FP16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NOFP16-LABEL: 'fast_fp_reductions_bf16'
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f8 = call reassoc bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR8000, <4 x bfloat> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
-; RECIPBF16-LABEL: 'fast_fp_reductions_bf16'
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_v4f8 = call reassoc bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR8000, <4 x bfloat> undef)
-; RECIPBF16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-F16-LABEL: 'fast_fp_reductions_bf16'
+; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f8 = call reassoc bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR8000, <4 x bfloat> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
-; SIZE-LABEL: 'fast_fp_reductions_bf16'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_v4f8 = call reassoc bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR8000, <4 x bfloat> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-BF16-LABEL: 'fast_fp_reductions_bf16'
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f8 = call reassoc bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR8000, <4 x bfloat> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %fadd_v4f8 = call reassoc bfloat @llvm.vector.reduce.fadd.v4f8(bfloat -0.0, <4 x bfloat> undef)
   ret void
diff --git a/llvm/test/Analysis/CostModel/AArch64/reduce-or.ll b/llvm/test/Analysis/CostModel/AArch64/reduce-or.ll
index ba747b8321ee8..519b8ecf6dc76 100644
--- a/llvm/test/Analysis/CostModel/AArch64/reduce-or.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/reduce-or.ll
@@ -1,61 +1,34 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s --check-prefix=RECIP
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output | FileCheck %s --check-prefix=COST
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define void @reduce() {
-; RECIP-LABEL: 'reduce'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1i8 = call i8 @llvm.vector.reduce.or.v1i8(<1 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V3i8 = call i8 @llvm.vector.reduce.or.v3i8(<3 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4i8 = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8i8 = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V16i8 = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V32i8 = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64i8 = call i8 @llvm.vector.reduce.or.v64i8(<64 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4i16 = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i16 = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16i16 = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4i32 = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i32 = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i64 = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; COST-LABEL: 'reduce'
-; COST-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1i8 = call i8 @llvm.vector.reduce.or.v1i8(<1 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V3i8 = call i8 @llvm.vector.reduce.or.v3i8(<3 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4i8 = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8i8 = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V16i8 = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V32i8 = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64i8 = call i8 @llvm.vector.reduce.or.v64i8(<64 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4i16 = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i16 = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16i16 = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4i32 = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i32 = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i64 = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'reduce'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %V1 = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 5 for: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 9 for: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %V1i8 = call i8 @llvm.vector.reduce.or.v1i8(<1 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V3i8 = call i8 @llvm.vector.reduce.or.v3i8(<3 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %V4i8 = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 15 for: %V8i8 = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 17 for: %V16i8 = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 18 for: %V32i8 = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 20 for: %V64i8 = call i8 @llvm.vector.reduce.or.v64i8(<64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %V4i16 = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 9 for: %V8i16 = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 10 for: %V16i16 = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %V2i32 = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 5 for: %V4i32 = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 6 for: %V8i32 = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %V2i64 = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4i64 = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 
   %V1   = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/AArch64/reduce-xor.ll b/llvm/test/Analysis/CostModel/AArch64/reduce-xor.ll
index b916c38ad220f..2a8609d2f418b 100644
--- a/llvm/test/Analysis/CostModel/AArch64/reduce-xor.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/reduce-xor.ll
@@ -1,61 +1,34 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s --check-prefix=RECIP
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output | FileCheck %s --check-prefix=COST
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define void @reduce() {
-; RECIP-LABEL: 'reduce'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i1 @llvm.vector.reduce.xor.v2i1(<2 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i1 @llvm.vector.reduce.xor.v4i1(<4 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i1 @llvm.vector.reduce.xor.v8i1(<8 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i1 @llvm.vector.reduce.xor.v16i1(<16 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i1 @llvm.vector.reduce.xor.v32i1(<32 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64 = call i1 @llvm.vector.reduce.xor.v64i1(<64 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V128 = call i1 @llvm.vector.reduce.xor.v128i1(<128 x i1> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1i8 = call i8 @llvm.vector.reduce.xor.v1i8(<1 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V3i8 = call i8 @llvm.vector.reduce.xor.v3i8(<3 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4i8 = call i8 @llvm.vector.reduce.xor.v4i8(<4 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8i8 = call i8 @llvm.vector.reduce.xor.v8i8(<8 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V16i8 = call i8 @llvm.vector.reduce.xor.v16i8(<16 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V32i8 = call i8 @llvm.vector.reduce.xor.v32i8(<32 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64i8 = call i8 @llvm.vector.reduce.xor.v64i8(<64 x i8> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4i16 = call i16 @llvm.vector.reduce.xor.v4i16(<4 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i16 = call i16 @llvm.vector.reduce.xor.v8i16(<8 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16i16 = call i16 @llvm.vector.reduce.xor.v16i16(<16 x i16> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = call i32 @llvm.vector.reduce.xor.v2i32(<2 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4i32 = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i32 = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i64 = call i64 @llvm.vector.reduce.xor.v2i64(<2 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> undef)
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; COST-LABEL: 'reduce'
-; COST-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i1 @llvm.vector.reduce.xor.v2i1(<2 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i1 @llvm.vector.reduce.xor.v4i1(<4 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i1 @llvm.vector.reduce.xor.v8i1(<8 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i1 @llvm.vector.reduce.xor.v16i1(<16 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32 = call i1 @llvm.vector.reduce.xor.v32i1(<32 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V64 = call i1 @llvm.vector.reduce.xor.v64i1(<64 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V128 = call i1 @llvm.vector.reduce.xor.v128i1(<128 x i1> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1i8 = call i8 @llvm.vector.reduce.xor.v1i8(<1 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V3i8 = call i8 @llvm.vector.reduce.xor.v3i8(<3 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4i8 = call i8 @llvm.vector.reduce.xor.v4i8(<4 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8i8 = call i8 @llvm.vector.reduce.xor.v8i8(<8 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V16i8 = call i8 @llvm.vector.reduce.xor.v16i8(<16 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V32i8 = call i8 @llvm.vector.reduce.xor.v32i8(<32 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64i8 = call i8 @llvm.vector.reduce.xor.v64i8(<64 x i8> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4i16 = call i16 @llvm.vector.reduce.xor.v4i16(<4 x i16> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i16 = call i16 @llvm.vector.reduce.xor.v8i16(<8 x i16> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16i16 = call i16 @llvm.vector.reduce.xor.v16i16(<16 x i16> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = call i32 @llvm.vector.reduce.xor.v2i32(<2 x i32> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4i32 = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i32 = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i64 = call i64 @llvm.vector.reduce.xor.v2i64(<2 x i64> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> undef)
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'reduce'
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %V1 = call i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V2 = call i1 @llvm.vector.reduce.xor.v2i1(<2 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V4 = call i1 @llvm.vector.reduce.xor.v4i1(<4 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8 = call i1 @llvm.vector.reduce.xor.v8i1(<8 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16 = call i1 @llvm.vector.reduce.xor.v16i1(<16 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %V32 = call i1 @llvm.vector.reduce.xor.v32i1(<32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 5 for: %V64 = call i1 @llvm.vector.reduce.xor.v64i1(<64 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 9 for: %V128 = call i1 @llvm.vector.reduce.xor.v128i1(<128 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %V1i8 = call i8 @llvm.vector.reduce.xor.v1i8(<1 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V3i8 = call i8 @llvm.vector.reduce.xor.v3i8(<3 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %V4i8 = call i8 @llvm.vector.reduce.xor.v4i8(<4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 15 for: %V8i8 = call i8 @llvm.vector.reduce.xor.v8i8(<8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 17 for: %V16i8 = call i8 @llvm.vector.reduce.xor.v16i8(<16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 18 for: %V32i8 = call i8 @llvm.vector.reduce.xor.v32i8(<32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 20 for: %V64i8 = call i8 @llvm.vector.reduce.xor.v64i8(<64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 7 for: %V4i16 = call i16 @llvm.vector.reduce.xor.v4i16(<4 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 9 for: %V8i16 = call i16 @llvm.vector.reduce.xor.v8i16(<8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 10 for: %V16i16 = call i16 @llvm.vector.reduce.xor.v16i16(<16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %V2i32 = call i32 @llvm.vector.reduce.xor.v2i32(<2 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 5 for: %V4i32 = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 6 for: %V8i32 = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %V2i64 = call i64 @llvm.vector.reduce.xor.v2i64(<2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4i64 = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 
   %V1   = call i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/AArch64/rem.ll b/llvm/test/Analysis/CostModel/AArch64/rem.ll
index f17533b713611..de38ba0507f62 100644
--- a/llvm/test/Analysis/CostModel/AArch64/rem.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/rem.ll
@@ -5,7 +5,7 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define void @srem() {
 ; CHECK-LABEL: 'srem'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %I128 = srem i128 undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I128 = srem i128 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I64 = srem i64 undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i64 = srem <2 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i64 = srem <4 x i64> undef, undef
@@ -287,29 +287,29 @@ define void @urem_uniform() {
 
 define void @srem_const() {
 ; CHECK-LABEL: 'srem_const'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %I128 = srem i128 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %I64 = srem i64 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i64 = srem <2 x i64> undef, <i64 6, i64 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i64 = srem <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8i64 = srem <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = srem i32 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i32 = srem <2 x i32> undef, <i32 4, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i32 = srem <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8i32 = srem <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16i32 = srem <16 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = srem i16 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i16 = srem <2 x i16> undef, <i16 4, i16 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i16 = srem <4 x i16> undef, <i16 4, i16 5, i16 6, i16 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8i16 = srem <8 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16i16 = srem <16 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V32i16 = srem <32 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = srem i8 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i8 = srem <2 x i8> undef, <i8 4, i8 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i8 = srem <4 x i8> undef, <i8 4, i8 5, i8 6, i8 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8i8 = srem <8 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16i8 = srem <16 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V32i8 = srem <32 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 448 for instruction: %V64i8 = srem <64 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I128 = srem i128 undef, 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = srem i64 undef, 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2i64 = srem <2 x i64> undef, <i64 6, i64 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4i64 = srem <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8i64 = srem <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = srem i32 undef, 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2i32 = srem <2 x i32> undef, <i32 4, i32 5>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4i32 = srem <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8i32 = srem <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16i32 = srem <16 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = srem i16 undef, 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2i16 = srem <2 x i16> undef, <i16 4, i16 5>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4i16 = srem <4 x i16> undef, <i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8i16 = srem <8 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16i16 = srem <16 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32i16 = srem <32 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = srem i8 undef, 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2i8 = srem <2 x i8> undef, <i8 4, i8 5>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4i8 = srem <4 x i8> undef, <i8 4, i8 5, i8 6, i8 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8i8 = srem <8 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16i8 = srem <16 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32i8 = srem <32 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64i8 = srem <64 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I128 = srem i128 undef, 7
@@ -404,29 +404,29 @@ define void @urem_const() {
 
 define void @srem_uniformconst() {
 ; CHECK-LABEL: 'srem_uniformconst'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %I128 = srem i128 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %I64 = srem i64 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V2i64 = srem <2 x i64> undef, splat (i64 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V4i64 = srem <4 x i64> undef, splat (i64 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V8i64 = srem <8 x i64> undef, splat (i64 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = srem i32 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i32 = srem <2 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i32 = srem <4 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8i32 = srem <8 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16i32 = srem <16 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = srem i16 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i16 = srem <2 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i16 = srem <4 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8i16 = srem <8 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16i16 = srem <16 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V32i16 = srem <32 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = srem i8 undef, 7
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i8 = srem <2 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i8 = srem <4 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8i8 = srem <8 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16i8 = srem <16 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V32i8 = srem <32 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 448 for instruction: %V64i8 = srem <64 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I128 = srem i128 undef, 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = srem i64 undef, 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i64 = srem <2 x i64> undef, splat (i64 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4i64 = srem <4 x i64> undef, splat (i64 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8i64 = srem <8 x i64> undef, splat (i64 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = srem i32 undef, 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i32 = srem <2 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i32 = srem <4 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8i32 = srem <8 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16i32 = srem <16 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = srem i16 undef, 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i16 = srem <2 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i16 = srem <4 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i16 = srem <8 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16i16 = srem <16 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32i16 = srem <32 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = srem i8 undef, 7
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i8 = srem <2 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i8 = srem <4 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i8 = srem <8 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i8 = srem <16 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32i8 = srem <32 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64i8 = srem <64 x i8> undef, splat (i8 7)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I128 = srem i128 undef, 7
@@ -520,29 +520,29 @@ define void @urem_uniformconst() {
 
 define void @srem_constpow2() {
 ; CHECK-LABEL: 'srem_constpow2'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %I128 = srem i128 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I64 = srem i64 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i64 = srem <2 x i64> undef, <i64 8, i64 16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i64 = srem <4 x i64> undef, <i64 2, i64 4, i64 8, i64 16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8i64 = srem <8 x i64> undef, <i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I32 = srem i32 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i32 = srem <2 x i32> undef, <i32 2, i32 4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i32 = srem <4 x i32> undef, <i32 2, i32 4, i32 8, i32 16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8i32 = srem <8 x i32> undef, <i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16i32 = srem <16 x i32> undef, <i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256, i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I16 = srem i16 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i16 = srem <2 x i16> undef, <i16 2, i16 4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i16 = srem <4 x i16> undef, <i16 2, i16 4, i16 8, i16 16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8i16 = srem <8 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16i16 = srem <16 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V32i16 = srem <32 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I8 = srem i8 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i8 = srem <2 x i8> undef, <i8 2, i8 4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i8 = srem <4 x i8> undef, <i8 2, i8 4, i8 8, i8 16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8i8 = srem <8 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16i8 = srem <16 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V32i8 = srem <32 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 448 for instruction: %V64i8 = srem <64 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I128 = srem i128 undef, 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = srem i64 undef, 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2i64 = srem <2 x i64> undef, <i64 8, i64 16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4i64 = srem <4 x i64> undef, <i64 2, i64 4, i64 8, i64 16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8i64 = srem <8 x i64> undef, <i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = srem i32 undef, 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2i32 = srem <2 x i32> undef, <i32 2, i32 4>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4i32 = srem <4 x i32> undef, <i32 2, i32 4, i32 8, i32 16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8i32 = srem <8 x i32> undef, <i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16i32 = srem <16 x i32> undef, <i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256, i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = srem i16 undef, 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2i16 = srem <2 x i16> undef, <i16 2, i16 4>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4i16 = srem <4 x i16> undef, <i16 2, i16 4, i16 8, i16 16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8i16 = srem <8 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16i16 = srem <16 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32i16 = srem <32 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = srem i8 undef, 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2i8 = srem <2 x i8> undef, <i8 2, i8 4>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4i8 = srem <4 x i8> undef, <i8 2, i8 4, i8 8, i8 16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8i8 = srem <8 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16i8 = srem <16 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32i8 = srem <32 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64i8 = srem <64 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I128 = srem i128 undef, 16
@@ -636,29 +636,29 @@ define void @urem_constpow2() {
 
 define void @srem_uniformconstpow2() {
 ; CHECK-LABEL: 'srem_uniformconstpow2'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %I128 = srem i128 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I64 = srem i64 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V2i64 = srem <2 x i64> undef, splat (i64 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V4i64 = srem <4 x i64> undef, splat (i64 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8i64 = srem <8 x i64> undef, splat (i64 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I32 = srem i32 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V2i32 = srem <2 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V4i32 = srem <4 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8i32 = srem <8 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 176 for instruction: %V16i32 = srem <16 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I16 = srem i16 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V2i16 = srem <2 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V4i16 = srem <4 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8i16 = srem <8 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 176 for instruction: %V16i16 = srem <16 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 352 for instruction: %V32i16 = srem <32 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %I8 = srem i8 undef, 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V2i8 = srem <2 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V4i8 = srem <4 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8i8 = srem <8 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 176 for instruction: %V16i8 = srem <16 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 352 for instruction: %V32i8 = srem <32 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 704 for instruction: %V64i8 = srem <64 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I128 = srem i128 undef, 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = srem i64 undef, 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i64 = srem <2 x i64> undef, splat (i64 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4i64 = srem <4 x i64> undef, splat (i64 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8i64 = srem <8 x i64> undef, splat (i64 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = srem i32 undef, 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i32 = srem <2 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i32 = srem <4 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8i32 = srem <8 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16i32 = srem <16 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = srem i16 undef, 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i16 = srem <2 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i16 = srem <4 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i16 = srem <8 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16i16 = srem <16 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32i16 = srem <32 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = srem i8 undef, 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i8 = srem <2 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i8 = srem <4 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i8 = srem <8 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i8 = srem <16 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32i8 = srem <32 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64i8 = srem <64 x i8> undef, splat (i8 16)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I128 = srem i128 undef, 16
@@ -752,29 +752,29 @@ define void @urem_uniformconstpow2() {
 
 define void @srem_constnegpow2() {
 ; CHECK-LABEL: 'srem_constnegpow2'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %I128 = srem i128 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %I64 = srem i64 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i64 = srem <2 x i64> undef, <i64 -8, i64 -16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i64 = srem <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8i64 = srem <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = srem i32 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i32 = srem <2 x i32> undef, <i32 -2, i32 -4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i32 = srem <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8i32 = srem <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16i32 = srem <16 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256, i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = srem i16 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i16 = srem <2 x i16> undef, <i16 -2, i16 -4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i16 = srem <4 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8i16 = srem <8 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16i16 = srem <16 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V32i16 = srem <32 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = srem i8 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i8 = srem <2 x i8> undef, <i8 -2, i8 -4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i8 = srem <4 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8i8 = srem <8 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16i8 = srem <16 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V32i8 = srem <32 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 448 for instruction: %V64i8 = srem <64 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I128 = srem i128 undef, -16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = srem i64 undef, -16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2i64 = srem <2 x i64> undef, <i64 -8, i64 -16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4i64 = srem <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8i64 = srem <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = srem i32 undef, -16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2i32 = srem <2 x i32> undef, <i32 -2, i32 -4>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4i32 = srem <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8i32 = srem <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16i32 = srem <16 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256, i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = srem i16 undef, -16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2i16 = srem <2 x i16> undef, <i16 -2, i16 -4>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4i16 = srem <4 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8i16 = srem <8 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16i16 = srem <16 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32i16 = srem <32 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = srem i8 undef, -16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2i8 = srem <2 x i8> undef, <i8 -2, i8 -4>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4i8 = srem <4 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8i8 = srem <8 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16i8 = srem <16 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32i8 = srem <32 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64i8 = srem <64 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I128 = srem i128 undef, -16
@@ -868,29 +868,29 @@ define void @urem_constnegpow2() {
 
 define void @srem_uniformconstnegpow2() {
 ; CHECK-LABEL: 'srem_uniformconstnegpow2'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %I128 = srem i128 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %I64 = srem i64 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V2i64 = srem <2 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V4i64 = srem <4 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V8i64 = srem <8 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = srem i32 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i32 = srem <2 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i32 = srem <4 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8i32 = srem <8 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16i32 = srem <16 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I16 = srem i16 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i16 = srem <2 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i16 = srem <4 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8i16 = srem <8 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16i16 = srem <16 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V32i16 = srem <32 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I8 = srem i8 undef, -16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2i8 = srem <2 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4i8 = srem <4 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8i8 = srem <8 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16i8 = srem <16 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V32i8 = srem <32 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 448 for instruction: %V64i8 = srem <64 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %I128 = srem i128 undef, -16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = srem i64 undef, -16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i64 = srem <2 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4i64 = srem <4 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8i64 = srem <8 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = srem i32 undef, -16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i32 = srem <2 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i32 = srem <4 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8i32 = srem <8 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16i32 = srem <16 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = srem i16 undef, -16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i16 = srem <2 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i16 = srem <4 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i16 = srem <8 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16i16 = srem <16 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32i16 = srem <32 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = srem i8 undef, -16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i8 = srem <2 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i8 = srem <4 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i8 = srem <8 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i8 = srem <16 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32i8 = srem <32 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64i8 = srem <64 x i8> undef, splat (i8 -16)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I128 = srem i128 undef, -16
diff --git a/llvm/test/Analysis/CostModel/AArch64/select.ll b/llvm/test/Analysis/CostModel/AArch64/select.ll
index 07e4cd4c906d0..76bd5c9bb0af6 100644
--- a/llvm/test/Analysis/CostModel/AArch64/select.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/select.ll
@@ -1,49 +1,28 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s  -passes="print<cost-model>" 2>&1 -disable-output -mtriple=arm64-apple-ios -mcpu=cyclone | FileCheck %s --check-prefix=RECIP
-; RUN: opt < %s  -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=aarch64-- | FileCheck %s --check-prefix=SIZE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
+
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
 
 define void @select() {
-    ; Scalar values
-; RECIP-LABEL: 'select'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = select i1 undef, i8 undef, i8 undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = select i1 undef, i16 undef, i16 undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = select i1 undef, i32 undef, i32 undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4 = select i1 undef, i64 undef, i64 undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = select i1 undef, float undef, float undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = select i1 undef, double undef, double undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v13b = select <16 x i1> undef, <16 x i16> undef, <16 x i16> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v15b = select <8 x i1> undef, <8 x i32> undef, <8 x i32> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v15c = select <16 x i1> undef, <16 x i32> undef, <16 x i32> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16a = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v16b = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v16c = select <16 x i1> undef, <16 x i64> undef, <16 x i64> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2float = select <2 x i1> undef, <2 x float> undef, <2 x float> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4float = select <4 x i1> undef, <4 x float> undef, <4 x float> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2double = select <2 x i1> undef, <2 x double> undef, <2 x double> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4half = select <4 x i1> undef, <4 x half> undef, <4 x half> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8half = select <8 x i1> undef, <8 x half> undef, <8 x half> undef
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SIZE-LABEL: 'select'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = select i1 undef, i8 undef, i8 undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = select i1 undef, i16 undef, i16 undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = select i1 undef, i32 undef, i32 undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4 = select i1 undef, i64 undef, i64 undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = select i1 undef, float undef, float undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = select i1 undef, double undef, double undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v13b = select <16 x i1> undef, <16 x i16> undef, <16 x i16> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v15b = select <8 x i1> undef, <8 x i32> undef, <8 x i32> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v15c = select <16 x i1> undef, <16 x i32> undef, <16 x i32> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16a = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16b = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16c = select <16 x i1> undef, <16 x i64> undef, <16 x i64> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2float = select <2 x i1> undef, <2 x float> undef, <2 x float> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4float = select <4 x i1> undef, <4 x float> undef, <4 x float> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2double = select <2 x i1> undef, <2 x double> undef, <2 x double> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4half = select <4 x i1> undef, <4 x half> undef, <4 x half> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8half = select <8 x i1> undef, <8 x half> undef, <8 x half> undef
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'select'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v1 = select i1 undef, i8 undef, i8 undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2 = select i1 undef, i16 undef, i16 undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v3 = select i1 undef, i32 undef, i32 undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4 = select i1 undef, i64 undef, i64 undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v5 = select i1 undef, float undef, float undef
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v6 = select i1 undef, double undef, double undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:1 Lat:1 SizeLat:1 for: %v13b = select <16 x i1> undef, <16 x i16> undef, <16 x i16> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:1 SizeLat:1 for: %v15b = select <8 x i1> undef, <8 x i32> undef, <8 x i32> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:1 Lat:1 SizeLat:1 for: %v15c = select <16 x i1> undef, <16 x i32> undef, <16 x i32> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:80 CodeSize:1 Lat:1 SizeLat:1 for: %v16a = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:160 CodeSize:1 Lat:1 SizeLat:1 for: %v16b = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:320 CodeSize:1 Lat:1 SizeLat:1 for: %v16c = select <16 x i1> undef, <16 x i64> undef, <16 x i64> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %v2float = select <2 x i1> undef, <2 x float> undef, <2 x float> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %v4float = select <4 x i1> undef, <4 x float> undef, <4 x float> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %v2double = select <2 x i1> undef, <2 x double> undef, <2 x double> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %v4half = select <4 x i1> undef, <4 x half> undef, <4 x half> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: %v8half = select <8 x i1> undef, <8 x half> undef, <8 x half> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %v1 = select i1 undef, i8 undef, i8 undef
   %v2 = select i1 undef, i16 undef, i16 undef
diff --git a/llvm/test/Analysis/CostModel/AArch64/shuffle-broadcast.ll b/llvm/test/Analysis/CostModel/AArch64/shuffle-broadcast.ll
index 86d2cb0038f4a..f62758f56ca94 100644
--- a/llvm/test/Analysis/CostModel/AArch64/shuffle-broadcast.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/shuffle-broadcast.ll
@@ -1,37 +1,22 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s --check-prefix=RECIP
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output | FileCheck %s --check-prefix=COST
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define void @broadcast() {
-; RECIP-LABEL: 'broadcast'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> zeroinitializer
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> zeroinitializer
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> zeroinitializer
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> zeroinitializer
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> zeroinitializer
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v12 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> zeroinitializer
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v13 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> zeroinitializer
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v14 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> zeroinitializer
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v15 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> zeroinitializer
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> zeroinitializer
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v17 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> zeroinitializer
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; COST-LABEL: 'broadcast'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> zeroinitializer
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> zeroinitializer
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> zeroinitializer
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> zeroinitializer
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> zeroinitializer
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v12 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> zeroinitializer
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v13 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> zeroinitializer
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v14 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> zeroinitializer
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v15 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> zeroinitializer
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> zeroinitializer
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v17 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> zeroinitializer
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'broadcast'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v7 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v9 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v10 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v11 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v12 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v13 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v14 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v15 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v17 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %v7 = shufflevector <2 x i8> undef, <2 x i8>undef, <2 x i32> zeroinitializer
   %v8 = shufflevector <4 x i8> undef, <4 x i8>undef, <4 x i32> zeroinitializer
diff --git a/llvm/test/Analysis/CostModel/AArch64/shuffle-load.ll b/llvm/test/Analysis/CostModel/AArch64/shuffle-load.ll
index bf81d7aa5e689..156d85b5b100f 100644
--- a/llvm/test/Analysis/CostModel/AArch64/shuffle-load.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/shuffle-load.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -mtriple=aarch64--linux-gnu -passes="print<cost-model>" 2>&1 -disable-output | FileCheck %s
-; RUN: opt < %s -mtriple=aarch64--linux-gnu -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output | FileCheck %s --check-prefix=CODESIZE
+; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output | FileCheck %s
 
 ; These tests check the costs of ld1r instructions, through the
 ; isLegalBroadcastLoad method.
@@ -11,98 +10,51 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 ; and splats as that is how getShuffleCost currently recognizes them.
 define void @shuffle() {
 ; CHECK-LABEL: 'shuffle'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %lv2i8 = load <2 x i8>, ptr undef, align 2
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv2i8 = shufflevector <2 x i8> %lv2i8, <2 x i8> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %lv4i8 = load <4 x i8>, ptr undef, align 4
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv4i8 = shufflevector <4 x i8> %lv4i8, <4 x i8> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv8i8 = load <8 x i8>, ptr undef, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv8i8 = shufflevector <8 x i8> %lv8i8, <8 x i8> undef, <8 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv16i8 = load <16 x i8>, ptr undef, align 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv16i8 = shufflevector <16 x i8> %lv16i8, <16 x i8> undef, <16 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %lv2i16 = load <2 x i16>, ptr undef, align 4
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv2i16 = shufflevector <2 x i16> %lv2i16, <2 x i16> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv4i16 = load <4 x i16>, ptr undef, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv4i16 = shufflevector <4 x i16> %lv4i16, <4 x i16> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv8i16 = load <8 x i16>, ptr undef, align 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv8i16 = shufflevector <8 x i16> %lv8i16, <8 x i16> undef, <8 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %lv16i16 = load <16 x i16>, ptr undef, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sv16i16 = shufflevector <16 x i16> %lv16i16, <16 x i16> undef, <16 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv2i32 = load <2 x i32>, ptr undef, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv2i32 = shufflevector <2 x i32> %lv2i32, <2 x i32> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv4i32 = load <4 x i32>, ptr undef, align 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv4i32 = shufflevector <4 x i32> %lv4i32, <4 x i32> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %lv8i32 = load <8 x i32>, ptr undef, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sv8i32 = shufflevector <8 x i32> %lv8i32, <8 x i32> undef, <8 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv2i64 = load <2 x i64>, ptr undef, align 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv2i64 = shufflevector <2 x i64> %lv2i64, <2 x i64> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %lv4i64 = load <4 x i64>, ptr undef, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sv4i64 = shufflevector <4 x i64> %lv4i64, <4 x i64> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv2f16 = load <2 x half>, ptr undef, align 4
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv2f16 = shufflevector <2 x half> %lv2f16, <2 x half> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv4f16 = load <4 x half>, ptr undef, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv4f16 = shufflevector <4 x half> %lv4f16, <4 x half> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv8f16 = load <8 x half>, ptr undef, align 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv8f16 = shufflevector <8 x half> %lv8f16, <8 x half> undef, <8 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %lv16f16 = load <16 x half>, ptr undef, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sv16f16 = shufflevector <16 x half> %lv16f16, <16 x half> undef, <16 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv2f32 = load <2 x float>, ptr undef, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv2f32 = shufflevector <2 x float> %lv2f32, <2 x float> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv4f32 = load <4 x float>, ptr undef, align 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv4f32 = shufflevector <4 x float> %lv4f32, <4 x float> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %lv8f32 = load <8 x float>, ptr undef, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sv8f32 = shufflevector <8 x float> %lv8f32, <8 x float> undef, <8 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv2f64 = load <2 x double>, ptr undef, align 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv2f64 = shufflevector <2 x double> %lv2f64, <2 x double> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %lv4f64 = load <4 x double>, ptr undef, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sv4f64 = shufflevector <4 x double> %lv4f64, <4 x double> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; CODESIZE-LABEL: 'shuffle'
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv2i8 = load <2 x i8>, ptr undef, align 2
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv2i8 = shufflevector <2 x i8> %lv2i8, <2 x i8> undef, <2 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv4i8 = load <4 x i8>, ptr undef, align 4
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv4i8 = shufflevector <4 x i8> %lv4i8, <4 x i8> undef, <4 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv8i8 = load <8 x i8>, ptr undef, align 8
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv8i8 = shufflevector <8 x i8> %lv8i8, <8 x i8> undef, <8 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv16i8 = load <16 x i8>, ptr undef, align 16
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv16i8 = shufflevector <16 x i8> %lv16i8, <16 x i8> undef, <16 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv2i16 = load <2 x i16>, ptr undef, align 4
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sv2i16 = shufflevector <2 x i16> %lv2i16, <2 x i16> undef, <2 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv4i16 = load <4 x i16>, ptr undef, align 8
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv4i16 = shufflevector <4 x i16> %lv4i16, <4 x i16> undef, <4 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv8i16 = load <8 x i16>, ptr undef, align 16
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv8i16 = shufflevector <8 x i16> %lv8i16, <8 x i16> undef, <8 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %lv16i16 = load <16 x i16>, ptr undef, align 32
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv16i16 = shufflevector <16 x i16> %lv16i16, <16 x i16> undef, <16 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv2i32 = load <2 x i32>, ptr undef, align 8
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv2i32 = shufflevector <2 x i32> %lv2i32, <2 x i32> undef, <2 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv4i32 = load <4 x i32>, ptr undef, align 16
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv4i32 = shufflevector <4 x i32> %lv4i32, <4 x i32> undef, <4 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %lv8i32 = load <8 x i32>, ptr undef, align 32
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv8i32 = shufflevector <8 x i32> %lv8i32, <8 x i32> undef, <8 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv2i64 = load <2 x i64>, ptr undef, align 16
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv2i64 = shufflevector <2 x i64> %lv2i64, <2 x i64> undef, <2 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %lv4i64 = load <4 x i64>, ptr undef, align 32
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv4i64 = shufflevector <4 x i64> %lv4i64, <4 x i64> undef, <4 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv2f16 = load <2 x half>, ptr undef, align 4
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv2f16 = shufflevector <2 x half> %lv2f16, <2 x half> undef, <2 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv4f16 = load <4 x half>, ptr undef, align 8
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv4f16 = shufflevector <4 x half> %lv4f16, <4 x half> undef, <4 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv8f16 = load <8 x half>, ptr undef, align 16
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv8f16 = shufflevector <8 x half> %lv8f16, <8 x half> undef, <8 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %lv16f16 = load <16 x half>, ptr undef, align 32
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv16f16 = shufflevector <16 x half> %lv16f16, <16 x half> undef, <16 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv2f32 = load <2 x float>, ptr undef, align 8
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv2f32 = shufflevector <2 x float> %lv2f32, <2 x float> undef, <2 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv4f32 = load <4 x float>, ptr undef, align 16
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv4f32 = shufflevector <4 x float> %lv4f32, <4 x float> undef, <4 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %lv8f32 = load <8 x float>, ptr undef, align 32
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv8f32 = shufflevector <8 x float> %lv8f32, <8 x float> undef, <8 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lv2f64 = load <2 x double>, ptr undef, align 16
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv2f64 = shufflevector <2 x double> %lv2f64, <2 x double> undef, <2 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %lv4f64 = load <4 x double>, ptr undef, align 32
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sv4f64 = shufflevector <4 x double> %lv4f64, <4 x double> undef, <4 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:4 SizeLat:1 for: %lv2i8 = load <2 x i8>, ptr undef, align 2
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sv2i8 = shufflevector <2 x i8> %lv2i8, <2 x i8> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %lv4i8 = load <4 x i8>, ptr undef, align 4
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sv4i8 = shufflevector <4 x i8> %lv4i8, <4 x i8> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv8i8 = load <8 x i8>, ptr undef, align 8
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv8i8 = shufflevector <8 x i8> %lv8i8, <8 x i8> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv16i8 = load <16 x i8>, ptr undef, align 16
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv16i8 = shufflevector <16 x i8> %lv16i8, <16 x i8> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:4 SizeLat:1 for: %lv2i16 = load <2 x i16>, ptr undef, align 4
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %sv2i16 = shufflevector <2 x i16> %lv2i16, <2 x i16> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv4i16 = load <4 x i16>, ptr undef, align 8
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv4i16 = shufflevector <4 x i16> %lv4i16, <4 x i16> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv8i16 = load <8 x i16>, ptr undef, align 16
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv8i16 = shufflevector <8 x i16> %lv8i16, <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %lv16i16 = load <16 x i16>, ptr undef, align 32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:0 Lat:2 SizeLat:2 for: %sv16i16 = shufflevector <16 x i16> %lv16i16, <16 x i16> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv2i32 = load <2 x i32>, ptr undef, align 8
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv2i32 = shufflevector <2 x i32> %lv2i32, <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv4i32 = load <4 x i32>, ptr undef, align 16
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv4i32 = shufflevector <4 x i32> %lv4i32, <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %lv8i32 = load <8 x i32>, ptr undef, align 32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:0 Lat:2 SizeLat:2 for: %sv8i32 = shufflevector <8 x i32> %lv8i32, <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv2i64 = load <2 x i64>, ptr undef, align 16
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv2i64 = shufflevector <2 x i64> %lv2i64, <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %lv4i64 = load <4 x i64>, ptr undef, align 32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:0 Lat:2 SizeLat:2 for: %sv4i64 = shufflevector <4 x i64> %lv4i64, <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv2f16 = load <2 x half>, ptr undef, align 4
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv2f16 = shufflevector <2 x half> %lv2f16, <2 x half> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv4f16 = load <4 x half>, ptr undef, align 8
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv4f16 = shufflevector <4 x half> %lv4f16, <4 x half> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv8f16 = load <8 x half>, ptr undef, align 16
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv8f16 = shufflevector <8 x half> %lv8f16, <8 x half> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %lv16f16 = load <16 x half>, ptr undef, align 32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:0 Lat:2 SizeLat:2 for: %sv16f16 = shufflevector <16 x half> %lv16f16, <16 x half> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv2f32 = load <2 x float>, ptr undef, align 8
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv2f32 = shufflevector <2 x float> %lv2f32, <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv4f32 = load <4 x float>, ptr undef, align 16
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv4f32 = shufflevector <4 x float> %lv4f32, <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %lv8f32 = load <8 x float>, ptr undef, align 32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:0 Lat:2 SizeLat:2 for: %sv8f32 = shufflevector <8 x float> %lv8f32, <8 x float> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %lv2f64 = load <2 x double>, ptr undef, align 16
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:1 SizeLat:1 for: %sv2f64 = shufflevector <2 x double> %lv2f64, <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %lv4f64 = load <4 x double>, ptr undef, align 32
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:0 Lat:2 SizeLat:2 for: %sv4f64 = shufflevector <4 x double> %lv4f64, <4 x double> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %lv2i8 = load <2 x i8>, ptr undef
   %sv2i8 = shufflevector <2 x i8> %lv2i8, <2 x i8> undef, <2 x i32> zeroinitializer
@@ -162,16 +114,10 @@ define void @shuffle() {
 
 define <4 x half> @ld1r_4h_float_shuff(ptr nocapture %x) {
 ; CHECK-LABEL: 'ld1r_4h_float_shuff'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load half, ptr %x, align 2
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %tmp1 = insertelement <4 x half> undef, half %tmp, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <4 x half> %tmp1, <4 x half> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x half> %lane
-;
-; CODESIZE-LABEL: 'ld1r_4h_float_shuff'
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load half, ptr %x, align 2
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %tmp1 = insertelement <4 x half> undef, half %tmp, i32 0
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <4 x half> %tmp1, <4 x half> undef, <4 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x half> %lane
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load half, ptr %x, align 2
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %tmp1 = insertelement <4 x half> undef, half %tmp, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %lane = shufflevector <4 x half> %tmp1, <4 x half> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x half> %lane
 ;
 entry:
   %tmp = load half, ptr %x, align 2
@@ -182,16 +128,10 @@ entry:
 
 define <8 x half> @ld1r_8h_float_shuff(ptr nocapture %x) {
 ; CHECK-LABEL: 'ld1r_8h_float_shuff'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load half, ptr %x, align 2
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %tmp1 = insertelement <8 x half> undef, half %tmp, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <8 x half> %tmp1, <8 x half> undef, <8 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x half> %lane
-;
-; CODESIZE-LABEL: 'ld1r_8h_float_shuff'
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load half, ptr %x, align 2
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %tmp1 = insertelement <8 x half> undef, half %tmp, i32 0
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <8 x half> %tmp1, <8 x half> undef, <8 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x half> %lane
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load half, ptr %x, align 2
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %tmp1 = insertelement <8 x half> undef, half %tmp, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %lane = shufflevector <8 x half> %tmp1, <8 x half> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x half> %lane
 ;
 entry:
   %tmp = load half, ptr %x, align 2
@@ -202,16 +142,10 @@ entry:
 
 define <2 x float> @ld1r_2s_float_shuff(ptr nocapture %x) {
 ; CHECK-LABEL: 'ld1r_2s_float_shuff'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load float, ptr %x, align 4
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %tmp1 = insertelement <2 x float> undef, float %tmp, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <2 x float> %tmp1, <2 x float> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %lane
-;
-; CODESIZE-LABEL: 'ld1r_2s_float_shuff'
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load float, ptr %x, align 4
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %tmp1 = insertelement <2 x float> undef, float %tmp, i32 0
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <2 x float> %tmp1, <2 x float> undef, <2 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x float> %lane
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load float, ptr %x, align 4
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %tmp1 = insertelement <2 x float> undef, float %tmp, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %lane = shufflevector <2 x float> %tmp1, <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x float> %lane
 ;
 entry:
   %tmp = load float, ptr %x, align 4
@@ -222,16 +156,10 @@ entry:
 
 define <4 x float> @ld1r_4s_float_shuff(ptr nocapture %x) {
 ; CHECK-LABEL: 'ld1r_4s_float_shuff'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load float, ptr %x, align 4
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %tmp1 = insertelement <4 x float> undef, float %tmp, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <4 x float> %tmp1, <4 x float> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %lane
-;
-; CODESIZE-LABEL: 'ld1r_4s_float_shuff'
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load float, ptr %x, align 4
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %tmp1 = insertelement <4 x float> undef, float %tmp, i32 0
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <4 x float> %tmp1, <4 x float> undef, <4 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x float> %lane
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load float, ptr %x, align 4
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %tmp1 = insertelement <4 x float> undef, float %tmp, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %lane = shufflevector <4 x float> %tmp1, <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x float> %lane
 ;
 entry:
   %tmp = load float, ptr %x, align 4
@@ -242,16 +170,10 @@ entry:
 
 define <2 x double> @ld1r_2d_double_shuff(ptr nocapture %x) {
 ; CHECK-LABEL: 'ld1r_2d_double_shuff'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load double, ptr %x, align 4
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %tmp1 = insertelement <2 x double> undef, double %tmp, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <2 x double> %tmp1, <2 x double> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %lane
-;
-; CODESIZE-LABEL: 'ld1r_2d_double_shuff'
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load double, ptr %x, align 4
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %tmp1 = insertelement <2 x double> undef, double %tmp, i32 0
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <2 x double> %tmp1, <2 x double> undef, <2 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x double> %lane
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load double, ptr %x, align 4
+; CHECK-NEXT:  Cost Model: Found costs of 0 for: %tmp1 = insertelement <2 x double> undef, double %tmp, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %lane = shufflevector <2 x double> %tmp1, <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x double> %lane
 ;
 entry:
   %tmp = load double, ptr %x, align 4
@@ -264,16 +186,10 @@ entry:
 
 define <8 x i8> @ld1r_8b_int_shuff(ptr nocapture %x) {
 ; CHECK-LABEL: 'ld1r_8b_int_shuff'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load i8, ptr %x, align 2
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %tmp1 = insertelement <8 x i8> undef, i8 %tmp, i8 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i8> %lane
-;
-; CODESIZE-LABEL: 'ld1r_8b_int_shuff'
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load i8, ptr %x, align 2
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %tmp1 = insertelement <8 x i8> undef, i8 %tmp, i8 0
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i8> %lane
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load i8, ptr %x, align 2
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %tmp1 = insertelement <8 x i8> undef, i8 %tmp, i8 0
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %lane = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %lane
 ;
 entry:
   %tmp = load i8, ptr %x, align 2
@@ -284,16 +200,10 @@ entry:
 
 define <16 x i8> @ld1r_16b_int_shuff(ptr nocapture %x) {
 ; CHECK-LABEL: 'ld1r_16b_int_shuff'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load i8, ptr %x, align 2
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %tmp1 = insertelement <16 x i8> undef, i8 %tmp, i8 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <16 x i8> %tmp1, <16 x i8> undef, <16 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %lane
-;
-; CODESIZE-LABEL: 'ld1r_16b_int_shuff'
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load i8, ptr %x, align 2
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %tmp1 = insertelement <16 x i8> undef, i8 %tmp, i8 0
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <16 x i8> %tmp1, <16 x i8> undef, <16 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %lane
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load i8, ptr %x, align 2
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %tmp1 = insertelement <16 x i8> undef, i8 %tmp, i8 0
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %lane = shufflevector <16 x i8> %tmp1, <16 x i8> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %lane
 ;
 entry:
   %tmp = load i8, ptr %x, align 2
@@ -304,16 +214,10 @@ entry:
 
 define <4 x i16> @ld1r_4h_int_shuff(ptr nocapture %x) {
 ; CHECK-LABEL: 'ld1r_4h_int_shuff'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load i16, ptr %x, align 2
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %tmp1 = insertelement <4 x i16> undef, i16 %tmp, i16 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <4 x i16> %tmp1, <4 x i16> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i16> %lane
-;
-; CODESIZE-LABEL: 'ld1r_4h_int_shuff'
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load i16, ptr %x, align 2
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %tmp1 = insertelement <4 x i16> undef, i16 %tmp, i16 0
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <4 x i16> %tmp1, <4 x i16> undef, <4 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i16> %lane
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load i16, ptr %x, align 2
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %tmp1 = insertelement <4 x i16> undef, i16 %tmp, i16 0
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %lane = shufflevector <4 x i16> %tmp1, <4 x i16> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i16> %lane
 ;
 entry:
   %tmp = load i16, ptr %x, align 2
@@ -324,16 +228,10 @@ entry:
 
 define <8 x i16> @ld1r_8h_int_shuff(ptr nocapture %x) {
 ; CHECK-LABEL: 'ld1r_8h_int_shuff'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load i16, ptr %x, align 2
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %tmp1 = insertelement <8 x i16> undef, i16 %tmp, i16 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <8 x i16> %tmp1, <8 x i16> undef, <8 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %lane
-;
-; CODESIZE-LABEL: 'ld1r_8h_int_shuff'
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load i16, ptr %x, align 2
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %tmp1 = insertelement <8 x i16> undef, i16 %tmp, i16 0
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <8 x i16> %tmp1, <8 x i16> undef, <8 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %lane
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load i16, ptr %x, align 2
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %tmp1 = insertelement <8 x i16> undef, i16 %tmp, i16 0
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %lane = shufflevector <8 x i16> %tmp1, <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i16> %lane
 ;
 entry:
   %tmp = load i16, ptr %x, align 2
@@ -344,16 +242,10 @@ entry:
 
 define <2 x i32> @ld1r_2s_int_shuff(ptr nocapture %x) {
 ; CHECK-LABEL: 'ld1r_2s_int_shuff'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load i32, ptr %x, align 4
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %tmp1 = insertelement <2 x i32> undef, i32 %tmp, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %lane
-;
-; CODESIZE-LABEL: 'ld1r_2s_int_shuff'
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load i32, ptr %x, align 4
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %tmp1 = insertelement <2 x i32> undef, i32 %tmp, i32 0
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <2 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i32> %lane
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load i32, ptr %x, align 4
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %tmp1 = insertelement <2 x i32> undef, i32 %tmp, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %lane = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i32> %lane
 ;
 entry:
   %tmp = load i32, ptr %x, align 4
@@ -364,16 +256,10 @@ entry:
 
 define <4 x i32> @ld1r_4s_int_shuff(ptr nocapture %x) {
 ; CHECK-LABEL: 'ld1r_4s_int_shuff'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load i32, ptr %x, align 4
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %tmp1 = insertelement <4 x i32> undef, i32 %tmp, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <4 x i32> %tmp1, <4 x i32> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lane
-;
-; CODESIZE-LABEL: 'ld1r_4s_int_shuff'
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load i32, ptr %x, align 4
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %tmp1 = insertelement <4 x i32> undef, i32 %tmp, i32 0
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <4 x i32> %tmp1, <4 x i32> undef, <4 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %lane
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load i32, ptr %x, align 4
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %tmp1 = insertelement <4 x i32> undef, i32 %tmp, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %lane = shufflevector <4 x i32> %tmp1, <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %lane
 ;
 entry:
   %tmp = load i32, ptr %x, align 4
@@ -384,16 +270,10 @@ entry:
 
 define <2 x i64> @ld1r_2d_int_shuff(ptr nocapture %x) {
 ; CHECK-LABEL: 'ld1r_2d_int_shuff'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load i64, ptr %x, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %tmp1 = insertelement <2 x i64> undef, i64 %tmp, i32 0
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <2 x i64> %tmp1, <2 x i64> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %lane
-;
-; CODESIZE-LABEL: 'ld1r_2d_int_shuff'
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %tmp = load i64, ptr %x, align 8
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %tmp1 = insertelement <2 x i64> undef, i64 %tmp, i32 0
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %lane = shufflevector <2 x i64> %tmp1, <2 x i64> undef, <2 x i32> zeroinitializer
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %lane
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %tmp = load i64, ptr %x, align 8
+; CHECK-NEXT:  Cost Model: Found costs of 3 for: %tmp1 = insertelement <2 x i64> undef, i64 %tmp, i32 0
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %lane = shufflevector <2 x i64> %tmp1, <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <2 x i64> %lane
 ;
 entry:
   %tmp = load i64, ptr %x, align 8
@@ -404,106 +284,55 @@ entry:
 
 define void @vld2(ptr %p) {
 ; CHECK-LABEL: 'vld2'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8 = load <4 x i8>, ptr %p, align 4
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_1 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = load <8 x i8>, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_1 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = load <16 x i8>, ptr %p, align 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_1 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i8 = load <32 x i8>, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_0 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_1 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = load <4 x i16>, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_0 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_1 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = load <8 x i16>, ptr %p, align 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_1 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = load <16 x i16>, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_0 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_1 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i16 = load <32 x i16>, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i16_0 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i16_1 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = load <4 x i32>, ptr %p, align 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_1 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = load <8 x i32>, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_0 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_1 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32 = load <16 x i32>, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i32_0 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i32_1 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i32 = load <32 x i32>, ptr %p, align 128
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i32_0 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i32_1 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i64 = load <4 x i64>, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i64 = load <8 x i64>, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_0 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_1 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i64 = load <16 x i64>, ptr %p, align 128
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_0 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_1 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i64 = load <32 x i64>, ptr %p, align 256
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_0 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_1 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; CODESIZE-LABEL: 'vld2'
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = load <4 x i8>, ptr %p, align 4
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> <i32 0, i32 2>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_1 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> <i32 1, i32 3>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = load <8 x i8>, ptr %p, align 8
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_1 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = load <16 x i8>, ptr %p, align 16
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_1 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i8 = load <32 x i8>, ptr %p, align 32
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_0 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_1 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = load <4 x i16>, ptr %p, align 8
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_0 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> <i32 0, i32 2>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_1 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> <i32 1, i32 3>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = load <8 x i16>, ptr %p, align 16
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_1 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = load <16 x i16>, ptr %p, align 32
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_0 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_1 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i16 = load <32 x i16>, ptr %p, align 64
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i16_0 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i16_1 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = load <4 x i32>, ptr %p, align 16
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> <i32 0, i32 2>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_1 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> <i32 1, i32 3>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = load <8 x i32>, ptr %p, align 32
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_0 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_1 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32 = load <16 x i32>, ptr %p, align 64
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i32_0 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i32_1 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i32 = load <32 x i32>, ptr %p, align 128
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i32_0 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i32_1 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i64 = load <4 x i64>, ptr %p, align 32
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> <i32 0, i32 2>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> <i32 1, i32 3>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i64 = load <8 x i64>, ptr %p, align 64
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_0 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_1 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i64 = load <16 x i64>, ptr %p, align 128
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_0 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_1 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i64 = load <32 x i64>, ptr %p, align 256
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_0 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_1 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %v4i8 = load <4 x i8>, ptr %p, align 4
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i8_0 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i8_1 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v8i8 = load <8 x i8>, ptr %p, align 8
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i8_0 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i8_1 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v16i8 = load <16 x i8>, ptr %p, align 16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i8_0 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i8_1 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v32i8 = load <32 x i8>, ptr %p, align 32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v32i8_0 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v32i8_1 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v4i16 = load <4 x i16>, ptr %p, align 8
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i16_0 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i16_1 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v8i16 = load <8 x i16>, ptr %p, align 16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i16_0 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i16_1 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v16i16 = load <16 x i16>, ptr %p, align 32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i16_0 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i16_1 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v32i16 = load <32 x i16>, ptr %p, align 64
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v32i16_0 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v32i16_1 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v4i32 = load <4 x i32>, ptr %p, align 16
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i32_0 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i32_1 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v8i32 = load <8 x i32>, ptr %p, align 32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i32_0 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i32_1 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v16i32 = load <16 x i32>, ptr %p, align 64
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v16i32_0 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v16i32_1 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:8 Lat:4 SizeLat:8 for: %v32i32 = load <32 x i32>, ptr %p, align 128
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v32i32_0 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v32i32_1 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v2i64 = load <4 x i64>, ptr %p, align 32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i64_0 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i64_1 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v4i64 = load <8 x i64>, ptr %p, align 64
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v4i64_0 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v4i64_1 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:8 Lat:4 SizeLat:8 for: %v8i64 = load <16 x i64>, ptr %p, align 128
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8i64_0 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8i64_1 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:16 Lat:4 SizeLat:16 for: %v16i64 = load <32 x i64>, ptr %p, align 256
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16i64_0 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16i64_1 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %v4i8 = load <4 x i8>, ptr %p
   %v4i8_0 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> <i32 0, i32 2>
@@ -563,138 +392,71 @@ define void @vld2(ptr %p) {
 
 define void @vld3(ptr %p) {
 ; CHECK-LABEL: 'vld3'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8 = load <6 x i8>, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_0 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_1 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> <i32 1, i32 4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_2 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> <i32 2, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = load <12 x i8>, ptr %p, align 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_0 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_1 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_2 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = load <24 x i8>, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_1 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_2 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i8 = load <48 x i8>, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_1 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_2 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = load <6 x i16>, ptr %p, align 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i16_0 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i16_1 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> <i32 1, i32 4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i16_2 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> <i32 2, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = load <12 x i16>, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_0 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_1 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_2 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i16 = load <24 x i16>, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_1 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_2 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i16 = load <48 x i16>, ptr %p, align 128
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16_0 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16_1 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16_2 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i32 = load <6 x i32>, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> <i32 1, i32 4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_2 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> <i32 2, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i32 = load <12 x i32>, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_1 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_2 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i32 = load <24 x i32>, ptr %p, align 128
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_0 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_1 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_2 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i32 = load <48 x i32>, ptr %p, align 256
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_0 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_1 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_2 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i64 = load <6 x i64>, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> <i32 1, i32 4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_2 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> <i32 2, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i64 = load <12 x i64>, ptr %p, align 128
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_0 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_1 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_2 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8i64 = load <24 x i64>, ptr %p, align 256
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_0 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_1 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_2 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16i64 = load <48 x i64>, ptr %p, align 512
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_0 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_1 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_2 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; CODESIZE-LABEL: 'vld3'
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8 = load <6 x i8>, ptr %p, align 8
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_0 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> <i32 0, i32 3>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_1 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> <i32 1, i32 4>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_2 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> <i32 2, i32 5>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = load <12 x i8>, ptr %p, align 16
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_0 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_1 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_2 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = load <24 x i8>, ptr %p, align 32
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_1 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_2 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i8 = load <48 x i8>, ptr %p, align 64
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_1 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_2 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = load <6 x i16>, ptr %p, align 16
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i16_0 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> <i32 0, i32 3>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i16_1 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> <i32 1, i32 4>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i16_2 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> <i32 2, i32 5>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = load <12 x i16>, ptr %p, align 32
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_0 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_1 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_2 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i16 = load <24 x i16>, ptr %p, align 64
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_1 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_2 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i16 = load <48 x i16>, ptr %p, align 128
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16_0 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16_1 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16_2 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i32 = load <6 x i32>, ptr %p, align 32
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> <i32 0, i32 3>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> <i32 1, i32 4>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_2 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> <i32 2, i32 5>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i32 = load <12 x i32>, ptr %p, align 64
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_1 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_2 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i32 = load <24 x i32>, ptr %p, align 128
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_0 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_1 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_2 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i32 = load <48 x i32>, ptr %p, align 256
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_0 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_1 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_2 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i64 = load <6 x i64>, ptr %p, align 64
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> <i32 0, i32 3>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> <i32 1, i32 4>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_2 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> <i32 2, i32 5>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i64 = load <12 x i64>, ptr %p, align 128
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_0 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_1 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_2 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8i64 = load <24 x i64>, ptr %p, align 256
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_0 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_1 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_2 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16i64 = load <48 x i64>, ptr %p, align 512
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_0 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_1 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_2 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v2i8 = load <6 x i8>, ptr %p, align 8
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v2i8_0 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v2i8_1 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> <i32 1, i32 4>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v2i8_2 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> <i32 2, i32 5>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v4i8 = load <12 x i8>, ptr %p, align 16
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v4i8_0 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v4i8_1 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v4i8_2 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v8i8 = load <24 x i8>, ptr %p, align 32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i8_0 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i8_1 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i8_2 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v16i8 = load <48 x i8>, ptr %p, align 64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i8_0 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i8_1 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i8_2 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v2i16 = load <6 x i16>, ptr %p, align 16
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v2i16_0 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v2i16_1 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> <i32 1, i32 4>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v2i16_2 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> <i32 2, i32 5>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v4i16 = load <12 x i16>, ptr %p, align 32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i16_0 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i16_1 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i16_2 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8i16 = load <24 x i16>, ptr %p, align 64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i16_0 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i16_1 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i16_2 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:8 Lat:4 SizeLat:8 for: %v16i16 = load <48 x i16>, ptr %p, align 128
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v16i16_0 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v16i16_1 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v16i16_2 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v2i32 = load <6 x i32>, ptr %p, align 32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i32_0 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i32_1 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> <i32 1, i32 4>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i32_2 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> <i32 2, i32 5>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v4i32 = load <12 x i32>, ptr %p, align 64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i32_0 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i32_1 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i32_2 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:8 Lat:4 SizeLat:8 for: %v8i32 = load <24 x i32>, ptr %p, align 128
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v8i32_0 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v8i32_1 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v8i32_2 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:16 Lat:4 SizeLat:16 for: %v16i32 = load <48 x i32>, ptr %p, align 256
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v16i32_0 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v16i32_1 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v16i32_2 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v2i64 = load <6 x i64>, ptr %p, align 64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i64_0 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i64_1 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> <i32 1, i32 4>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i64_2 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> <i32 2, i32 5>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:8 Lat:4 SizeLat:8 for: %v4i64 = load <12 x i64>, ptr %p, align 128
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v4i64_0 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v4i64_1 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v4i64_2 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:16 Lat:4 SizeLat:16 for: %v8i64 = load <24 x i64>, ptr %p, align 256
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8i64_0 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8i64_1 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8i64_2 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:32 CodeSize:32 Lat:4 SizeLat:32 for: %v16i64 = load <48 x i64>, ptr %p, align 512
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16i64_0 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16i64_1 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16i64_2 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %v2i8 = load <6 x i8>, ptr %p
   %v2i8_0 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> <i32 0, i32 3>
@@ -769,170 +531,87 @@ define void @vld3(ptr %p) {
 
 define void @vld4(ptr %p) {
 ; CHECK-LABEL: 'vld4'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8 = load <8 x i8>, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_0 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 0, i32 4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_1 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 1, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_2 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 2, i32 6>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_3 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 3, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = load <16 x i8>, ptr %p, align 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_0 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_1 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_2 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_3 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = load <32 x i8>, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_1 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_2 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_3 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i8 = load <64 x i8>, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_1 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_2 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_3 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = load <8 x i16>, ptr %p, align 16
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i16_0 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 0, i32 4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i16_1 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 1, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i16_2 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 2, i32 6>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i16_3 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 3, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = load <16 x i16>, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_0 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_1 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_2 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_3 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i16 = load <32 x i16>, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_1 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_2 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_3 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i16 = load <64 x i16>, ptr %p, align 128
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16_0 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16_1 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16_2 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16_3 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i32 = load <8 x i32>, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 0, i32 4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 1, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_2 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 2, i32 6>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_3 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 3, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i32 = load <16 x i32>, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_1 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_2 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_3 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i32 = load <32 x i32>, ptr %p, align 128
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_0 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_1 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_2 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_3 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i32 = load <64 x i32>, ptr %p, align 256
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_0 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_1 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_2 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_3 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i64 = load <8 x i64>, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 0, i32 4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 1, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_2 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 2, i32 6>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_3 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 3, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i64 = load <16 x i64>, ptr %p, align 128
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_0 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_1 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_2 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_3 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8i64 = load <32 x i64>, ptr %p, align 256
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_0 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_1 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_2 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_3 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16i64 = load <64 x i64>, ptr %p, align 512
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_0 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_1 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_2 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_3 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; CODESIZE-LABEL: 'vld4'
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8 = load <8 x i8>, ptr %p, align 8
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_0 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 0, i32 4>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_1 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 1, i32 5>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_2 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 2, i32 6>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_3 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 3, i32 7>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = load <16 x i8>, ptr %p, align 16
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_0 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_1 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_2 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_3 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = load <32 x i8>, ptr %p, align 32
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_1 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_2 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_3 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i8 = load <64 x i8>, ptr %p, align 64
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_1 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_2 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_3 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = load <8 x i16>, ptr %p, align 16
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i16_0 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 0, i32 4>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i16_1 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 1, i32 5>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i16_2 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 2, i32 6>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i16_3 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 3, i32 7>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = load <16 x i16>, ptr %p, align 32
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_0 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_1 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_2 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_3 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i16 = load <32 x i16>, ptr %p, align 64
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_1 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_2 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_3 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i16 = load <64 x i16>, ptr %p, align 128
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16_0 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16_1 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16_2 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16_3 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i32 = load <8 x i32>, ptr %p, align 32
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 0, i32 4>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 1, i32 5>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_2 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 2, i32 6>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_3 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 3, i32 7>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i32 = load <16 x i32>, ptr %p, align 64
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_1 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_2 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_3 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i32 = load <32 x i32>, ptr %p, align 128
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_0 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_1 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_2 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_3 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i32 = load <64 x i32>, ptr %p, align 256
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_0 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_1 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_2 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_3 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i64 = load <8 x i64>, ptr %p, align 64
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 0, i32 4>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 1, i32 5>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_2 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 2, i32 6>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_3 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 3, i32 7>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i64 = load <16 x i64>, ptr %p, align 128
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_0 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_1 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_2 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_3 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8i64 = load <32 x i64>, ptr %p, align 256
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_0 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_1 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_2 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_3 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16i64 = load <64 x i64>, ptr %p, align 512
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_0 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_1 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_2 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_3 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
-; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v2i8 = load <8 x i8>, ptr %p, align 8
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v2i8_0 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 0, i32 4>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v2i8_1 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 1, i32 5>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v2i8_2 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 2, i32 6>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v2i8_3 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 3, i32 7>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v4i8 = load <16 x i8>, ptr %p, align 16
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v4i8_0 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v4i8_1 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v4i8_2 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v4i8_3 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v8i8 = load <32 x i8>, ptr %p, align 32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i8_0 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i8_1 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i8_2 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i8_3 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v16i8 = load <64 x i8>, ptr %p, align 64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i8_0 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i8_1 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i8_2 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i8_3 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %v2i16 = load <8 x i16>, ptr %p, align 16
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v2i16_0 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 0, i32 4>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v2i16_1 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 1, i32 5>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v2i16_2 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 2, i32 6>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v2i16_3 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 3, i32 7>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v4i16 = load <16 x i16>, ptr %p, align 32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i16_0 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i16_1 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i16_2 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i16_3 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8i16 = load <32 x i16>, ptr %p, align 64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i16_0 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i16_1 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i16_2 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i16_3 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:8 Lat:4 SizeLat:8 for: %v16i16 = load <64 x i16>, ptr %p, align 128
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v16i16_0 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v16i16_1 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v16i16_2 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v16i16_3 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %v2i32 = load <8 x i32>, ptr %p, align 32
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i32_0 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 0, i32 4>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i32_1 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 1, i32 5>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i32_2 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 2, i32 6>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i32_3 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 3, i32 7>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v4i32 = load <16 x i32>, ptr %p, align 64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i32_0 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i32_1 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i32_2 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i32_3 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:8 Lat:4 SizeLat:8 for: %v8i32 = load <32 x i32>, ptr %p, align 128
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v8i32_0 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v8i32_1 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v8i32_2 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v8i32_3 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:16 Lat:4 SizeLat:16 for: %v16i32 = load <64 x i32>, ptr %p, align 256
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v16i32_0 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v16i32_1 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v16i32_2 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v16i32_3 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v2i64 = load <8 x i64>, ptr %p, align 64
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i64_0 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 0, i32 4>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i64_1 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 1, i32 5>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i64_2 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 2, i32 6>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i64_3 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 3, i32 7>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:8 Lat:4 SizeLat:8 for: %v4i64 = load <16 x i64>, ptr %p, align 128
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v4i64_0 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v4i64_1 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v4i64_2 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v4i64_3 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:16 Lat:4 SizeLat:16 for: %v8i64 = load <32 x i64>, ptr %p, align 256
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8i64_0 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8i64_1 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8i64_2 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8i64_3 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:32 CodeSize:32 Lat:4 SizeLat:32 for: %v16i64 = load <64 x i64>, ptr %p, align 512
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16i64_0 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16i64_1 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16i64_2 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16i64_3 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %v2i8 = load <8 x i8>, ptr %p
   %v2i8_0 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 0, i32 4>
diff --git a/llvm/test/Analysis/CostModel/AArch64/shuffle-reverse.ll b/llvm/test/Analysis/CostModel/AArch64/shuffle-reverse.ll
index e1bf345a09da1..7dc4a2efbd660 100644
--- a/llvm/test/Analysis/CostModel/AArch64/shuffle-reverse.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/shuffle-reverse.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s --check-prefix=RECIP
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -passes="print<cost-model>" -cost-kind=code-size 2>&1 -disable-output | FileCheck %s --check-prefix=COST
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
@@ -10,63 +9,34 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 ;; Reverse shuffles should be lowered to vrev and possibly a vext (for quadwords, on neon)
 define void @reverse() {
-; RECIP-LABEL: 'reverse'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> <i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64 = shufflevector <2 x i64> undef, <2 x i64> undef, <2 x i32> <i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = shufflevector <2 x half> undef, <2 x half> undef, <2 x i32> <i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f16 = shufflevector <4 x half> undef, <4 x half> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2bf16 = shufflevector <2 x bfloat> undef, <2 x bfloat> undef, <2 x i32> <i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4bf16 = shufflevector <4 x bfloat> undef, <4 x bfloat> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8bf16 = shufflevector <8 x bfloat> undef, <8 x bfloat> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16bf16 = shufflevector <16 x bfloat> undef, <16 x bfloat> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> <i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f32 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = shufflevector <2 x double> undef, <2 x double> undef, <2 x i32> <i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f64 = shufflevector <4 x double> undef, <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; COST-LABEL: 'reverse'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> <i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64 = shufflevector <2 x i64> undef, <2 x i64> undef, <2 x i32> <i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = shufflevector <2 x half> undef, <2 x half> undef, <2 x i32> <i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f16 = shufflevector <4 x half> undef, <4 x half> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2bf16 = shufflevector <2 x bfloat> undef, <2 x bfloat> undef, <2 x i32> <i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4bf16 = shufflevector <4 x bfloat> undef, <4 x bfloat> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8bf16 = shufflevector <8 x bfloat> undef, <8 x bfloat> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16bf16 = shufflevector <16 x bfloat> undef, <16 x bfloat> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> <i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f32 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = shufflevector <2 x double> undef, <2 x double> undef, <2 x i32> <i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f64 = shufflevector <4 x double> undef, <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'reverse'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i8 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i16 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v4i16 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i32 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v4i32 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2i64 = shufflevector <2 x i64> undef, <2 x i64> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2f16 = shufflevector <2 x half> undef, <2 x half> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v4f16 = shufflevector <4 x half> undef, <4 x half> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2bf16 = shufflevector <2 x bfloat> undef, <2 x bfloat> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v4bf16 = shufflevector <4 x bfloat> undef, <4 x bfloat> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v8bf16 = shufflevector <8 x bfloat> undef, <8 x bfloat> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v16bf16 = shufflevector <16 x bfloat> undef, <16 x bfloat> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2f32 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v4f32 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v2f64 = shufflevector <2 x double> undef, <2 x double> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v4f64 = shufflevector <4 x double> undef, <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %v2i8 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> <i32 1, i32 0>
   %v4i8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
@@ -106,59 +76,32 @@ define void @reverse() {
 }
 
 define void @vrev64() {
-; RECIP-LABEL: 'vrev64'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64 = shufflevector <8 x i64> undef, <8 x i64> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64 = shufflevector <16 x i64> undef, <16 x i64> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = shufflevector <4 x half> undef, <4 x half> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4bf16 = shufflevector <4 x bfloat> undef, <4 x bfloat> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8bf16 = shufflevector <8 x bfloat> undef, <8 x bfloat> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16bf16 = shufflevector <16 x bfloat> undef, <16 x bfloat> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f32 = shufflevector <16 x float> undef, <16 x float> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f64 = shufflevector <4 x double> undef, <4 x double> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f64 = shufflevector <8 x double> undef, <8 x double> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f64 = shufflevector <16 x double> undef, <16 x double> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; COST-LABEL: 'vrev64'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; COST-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64 = shufflevector <8 x i64> undef, <8 x i64> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; COST-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64 = shufflevector <16 x i64> undef, <16 x i64> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = shufflevector <4 x half> undef, <4 x half> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4bf16 = shufflevector <4 x bfloat> undef, <4 x bfloat> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8bf16 = shufflevector <8 x bfloat> undef, <8 x bfloat> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16bf16 = shufflevector <16 x bfloat> undef, <16 x bfloat> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; COST-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f32 = shufflevector <16 x float> undef, <16 x float> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f64 = shufflevector <4 x double> undef, <4 x double> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f64 = shufflevector <8 x double> undef, <8 x double> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-; COST-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f64 = shufflevector <16 x double> undef, <16 x double> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'vrev64'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i16 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i32 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16i32 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v4i64 = shufflevector <4 x i64> undef, <4 x i64> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8i64 = shufflevector <8 x i64> undef, <8 x i64> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16i64 = shufflevector <16 x i64> undef, <16 x i64> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4f16 = shufflevector <4 x half> undef, <4 x half> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4bf16 = shufflevector <4 x bfloat> undef, <4 x bfloat> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8bf16 = shufflevector <8 x bfloat> undef, <8 x bfloat> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v16bf16 = shufflevector <16 x bfloat> undef, <16 x bfloat> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4f32 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16f32 = shufflevector <16 x float> undef, <16 x float> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v4f64 = shufflevector <4 x double> undef, <4 x double> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8f64 = shufflevector <8 x double> undef, <8 x double> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16f64 = shufflevector <16 x double> undef, <16 x double> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %v4i8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
   %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
@@ -196,43 +139,24 @@ define void @vrev64() {
 }
 
 define void @vrev32() {
-; RECIP-LABEL: 'vrev32'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64 = shufflevector <8 x i64> undef, <8 x i64> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64 = shufflevector <16 x i64> undef, <16 x i64> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8bf16 = shufflevector <8 x bfloat> undef, <8 x bfloat> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16bf16 = shufflevector <16 x bfloat> undef, <16 x bfloat> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f32 = shufflevector <16 x float> undef, <16 x float> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f64 = shufflevector <8 x double> undef, <8 x double> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f64 = shufflevector <16 x double> undef, <16 x double> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; COST-LABEL: 'vrev32'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; COST-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64 = shufflevector <8 x i64> undef, <8 x i64> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; COST-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64 = shufflevector <16 x i64> undef, <16 x i64> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8bf16 = shufflevector <8 x bfloat> undef, <8 x bfloat> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16bf16 = shufflevector <16 x bfloat> undef, <16 x bfloat> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; COST-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f32 = shufflevector <16 x float> undef, <16 x float> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f64 = shufflevector <8 x double> undef, <8 x double> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-; COST-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f64 = shufflevector <16 x double> undef, <16 x double> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'vrev32'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16i32 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8i64 = shufflevector <8 x i64> undef, <8 x i64> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16i64 = shufflevector <16 x i64> undef, <16 x i64> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8f16 = shufflevector <8 x half> undef, <8 x half> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8bf16 = shufflevector <8 x bfloat> undef, <8 x bfloat> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v16bf16 = shufflevector <16 x bfloat> undef, <16 x bfloat> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v8f32 = shufflevector <8 x float> undef, <8 x float> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16f32 = shufflevector <16 x float> undef, <16 x float> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v8f64 = shufflevector <8 x double> undef, <8 x double> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16f64 = shufflevector <16 x double> undef, <16 x double> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
   %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
@@ -262,27 +186,16 @@ define void @vrev32() {
 }
 
 define void @vrev16() {
-; RECIP-LABEL: 'vrev16'
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64 = shufflevector <16 x i64> undef, <16 x i64> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16bf16 = shufflevector <16 x bfloat> undef, <16 x bfloat> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16f32 = shufflevector <16 x float> undef, <16 x float> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f64 = shufflevector <16 x double> undef, <16 x double> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; COST-LABEL: 'vrev16'
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; COST-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64 = shufflevector <16 x i64> undef, <16 x i64> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16bf16 = shufflevector <16 x bfloat> undef, <16 x bfloat> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; COST-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16f32 = shufflevector <16 x float> undef, <16 x float> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; COST-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f64 = shufflevector <16 x double> undef, <16 x double> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
-; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-LABEL: 'vrev16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v16i32 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16i64 = shufflevector <16 x i64> undef, <16 x i64> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v16f16 = shufflevector <16 x half> undef, <16 x half> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+; CHECK-NEXT:  Cost Model: Found costs of 2 for: %v16bf16 = shufflevector <16 x bfloat> undef, <16 x bfloat> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+; CHECK-NEXT:  Cost Model: Found costs of 4 for: %v16f32 = shufflevector <16 x float> undef, <16 x float> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+; CHECK-NEXT:  Cost Model: Found costs of 8 for: %v16f64 = shufflevector <16 x double> undef, <16 x double> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
 
diff --git a/llvm/test/Analysis/CostModel/AArch64/sincos.ll b/llvm/test/Analysis/CostModel/AArch64/sincos.ll
index e32c51667e0ad..32408acb582d0 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sincos.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sincos.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "sincos"
 ; RUN: opt < %s -mtriple=aarch64-gnu-linux -mattr=+neon,+sve -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s
-; RUN: opt < %s -mtriple=aarch64-gnu-linux -mattr=+neon,+sve -vector-library=ArmPL -passes="print<cost-model>" -prefer-intrinsic-cost -cost-kind=throughput 2>&1 -disable-output | FileCheck %s -check-prefix=CHECK-VECLIB
+; RUN: opt < %s -mtriple=aarch64-gnu-linux -mattr=+neon,+sve -vector-library=ArmPL -passes="print<cost-model>" -intrinsic-cost-strategy=intrinsic-cost -cost-kind=throughput 2>&1 -disable-output | FileCheck %s -check-prefix=CHECK-VECLIB
 
 define void @sincos() {
 ; CHECK-LABEL: 'sincos'
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-div.ll b/llvm/test/Analysis/CostModel/AArch64/sve-div.ll
index d20ebbf0ea9c1..c38cdcd8b9a34 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-div.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-div.ll
@@ -163,42 +163,42 @@ define void @udiv() {
 
 define void @sdiv_uniformconst() {
 ; CHECK-LABEL: 'sdiv_uniformconst'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sdiv <2 x i64> undef, splat (i64 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = sdiv <4 x i64> undef, splat (i64 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8i64 = sdiv <8 x i64> undef, splat (i64 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = sdiv <2 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4i32 = sdiv <4 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = sdiv <8 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16i32 = sdiv <16 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2i16 = sdiv <2 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4i16 = sdiv <4 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i16 = sdiv <8 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16i16 = sdiv <16 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32i16 = sdiv <32 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2i8 = sdiv <2 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4i8 = sdiv <4 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8i8 = sdiv <8 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i8 = sdiv <16 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32i8 = sdiv <32 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V64i8 = sdiv <64 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV2i64 = sdiv <vscale x 2 x i64> undef, splat (i64 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV4i64 = sdiv <vscale x 4 x i64> undef, splat (i64 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV8i64 = sdiv <vscale x 8 x i64> undef, splat (i64 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV2i32 = sdiv <vscale x 2 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV4i32 = sdiv <vscale x 4 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV8i32 = sdiv <vscale x 8 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV16i32 = sdiv <vscale x 16 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV2i16 = sdiv <vscale x 2 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV4i16 = sdiv <vscale x 4 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV8i16 = sdiv <vscale x 8 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NV16i16 = sdiv <vscale x 16 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %NV32i16 = sdiv <vscale x 32 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV2i8 = sdiv <vscale x 2 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV4i8 = sdiv <vscale x 4 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV8i8 = sdiv <vscale x 8 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NV16i8 = sdiv <vscale x 16 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %NV32i8 = sdiv <vscale x 32 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %NV64i8 = sdiv <vscale x 64 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i64 = sdiv <2 x i64> undef, splat (i64 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4i64 = sdiv <4 x i64> undef, splat (i64 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8i64 = sdiv <8 x i64> undef, splat (i64 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i32 = sdiv <2 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i32 = sdiv <4 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8i32 = sdiv <8 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16i32 = sdiv <16 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i16 = sdiv <2 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i16 = sdiv <4 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i16 = sdiv <8 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16i16 = sdiv <16 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32i16 = sdiv <32 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i8 = sdiv <2 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i8 = sdiv <4 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i8 = sdiv <8 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i8 = sdiv <16 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32i8 = sdiv <32 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64i8 = sdiv <64 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i64 = sdiv <vscale x 2 x i64> undef, splat (i64 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV4i64 = sdiv <vscale x 4 x i64> undef, splat (i64 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV8i64 = sdiv <vscale x 8 x i64> undef, splat (i64 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i32 = sdiv <vscale x 2 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV4i32 = sdiv <vscale x 4 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV8i32 = sdiv <vscale x 8 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV16i32 = sdiv <vscale x 16 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i16 = sdiv <vscale x 2 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV4i16 = sdiv <vscale x 4 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV8i16 = sdiv <vscale x 8 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV16i16 = sdiv <vscale x 16 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV32i16 = sdiv <vscale x 32 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i8 = sdiv <vscale x 2 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV4i8 = sdiv <vscale x 4 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV8i8 = sdiv <vscale x 8 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV16i8 = sdiv <vscale x 16 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV32i8 = sdiv <vscale x 32 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV64i8 = sdiv <vscale x 64 x i8> undef, splat (i8 7)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V2i64 = sdiv <2 x i64> undef, splat (i64 7)
@@ -321,42 +321,42 @@ define void @udiv_uniformconst() {
 
 define void @sdiv_uniformconstpow2() {
 ; CHECK-LABEL: 'sdiv_uniformconstpow2'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2i64 = sdiv <2 x i64> undef, splat (i64 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V4i64 = sdiv <4 x i64> undef, splat (i64 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V8i64 = sdiv <8 x i64> undef, splat (i64 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2i32 = sdiv <2 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V4i32 = sdiv <4 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V8i32 = sdiv <8 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V16i32 = sdiv <16 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2i16 = sdiv <2 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V4i16 = sdiv <4 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V8i16 = sdiv <8 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V16i16 = sdiv <16 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V32i16 = sdiv <32 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2i8 = sdiv <2 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V4i8 = sdiv <4 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V8i8 = sdiv <8 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %V16i8 = sdiv <16 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V32i8 = sdiv <32 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V64i8 = sdiv <64 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i64 = sdiv <vscale x 2 x i64> undef, splat (i64 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV4i64 = sdiv <vscale x 4 x i64> undef, splat (i64 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV8i64 = sdiv <vscale x 8 x i64> undef, splat (i64 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i32 = sdiv <vscale x 2 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV4i32 = sdiv <vscale x 4 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV8i32 = sdiv <vscale x 8 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV16i32 = sdiv <vscale x 16 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i16 = sdiv <vscale x 2 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV4i16 = sdiv <vscale x 4 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV8i16 = sdiv <vscale x 8 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV16i16 = sdiv <vscale x 16 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV32i16 = sdiv <vscale x 32 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i8 = sdiv <vscale x 2 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV4i8 = sdiv <vscale x 4 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV8i8 = sdiv <vscale x 8 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV16i8 = sdiv <vscale x 16 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV32i8 = sdiv <vscale x 32 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV64i8 = sdiv <vscale x 64 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i64 = sdiv <2 x i64> undef, splat (i64 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i64 = sdiv <4 x i64> undef, splat (i64 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8i64 = sdiv <8 x i64> undef, splat (i64 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i32 = sdiv <2 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i32 = sdiv <4 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8i32 = sdiv <8 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16i32 = sdiv <16 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i16 = sdiv <2 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i16 = sdiv <4 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i16 = sdiv <8 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16i16 = sdiv <16 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32i16 = sdiv <32 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i8 = sdiv <2 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i8 = sdiv <4 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i8 = sdiv <8 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i8 = sdiv <16 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32i8 = sdiv <32 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64i8 = sdiv <64 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV2i64 = sdiv <vscale x 2 x i64> undef, splat (i64 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV4i64 = sdiv <vscale x 4 x i64> undef, splat (i64 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV8i64 = sdiv <vscale x 8 x i64> undef, splat (i64 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV2i32 = sdiv <vscale x 2 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV4i32 = sdiv <vscale x 4 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV8i32 = sdiv <vscale x 8 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV16i32 = sdiv <vscale x 16 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV2i16 = sdiv <vscale x 2 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV4i16 = sdiv <vscale x 4 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV8i16 = sdiv <vscale x 8 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV16i16 = sdiv <vscale x 16 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV32i16 = sdiv <vscale x 32 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV2i8 = sdiv <vscale x 2 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV4i8 = sdiv <vscale x 4 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV8i8 = sdiv <vscale x 8 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV16i8 = sdiv <vscale x 16 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV32i8 = sdiv <vscale x 32 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV64i8 = sdiv <vscale x 64 x i8> undef, splat (i8 16)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V2i64 = sdiv <2 x i64> undef, splat (i64 16)
@@ -479,42 +479,42 @@ define void @udiv_uniformconstpow2() {
 
 define void @sdiv_uniformconstnegpow2() {
 ; CHECK-LABEL: 'sdiv_uniformconstnegpow2'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = sdiv <2 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = sdiv <4 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8i64 = sdiv <8 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = sdiv <2 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4i32 = sdiv <4 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = sdiv <8 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16i32 = sdiv <16 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2i16 = sdiv <2 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4i16 = sdiv <4 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i16 = sdiv <8 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16i16 = sdiv <16 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32i16 = sdiv <32 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2i8 = sdiv <2 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4i8 = sdiv <4 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8i8 = sdiv <8 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i8 = sdiv <16 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32i8 = sdiv <32 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V64i8 = sdiv <64 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV2i64 = sdiv <vscale x 2 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV4i64 = sdiv <vscale x 4 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV8i64 = sdiv <vscale x 8 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV2i32 = sdiv <vscale x 2 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV4i32 = sdiv <vscale x 4 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV8i32 = sdiv <vscale x 8 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV16i32 = sdiv <vscale x 16 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV2i16 = sdiv <vscale x 2 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV4i16 = sdiv <vscale x 4 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV8i16 = sdiv <vscale x 8 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NV16i16 = sdiv <vscale x 16 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %NV32i16 = sdiv <vscale x 32 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV2i8 = sdiv <vscale x 2 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV4i8 = sdiv <vscale x 4 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV8i8 = sdiv <vscale x 8 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NV16i8 = sdiv <vscale x 16 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %NV32i8 = sdiv <vscale x 32 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %NV64i8 = sdiv <vscale x 64 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i64 = sdiv <2 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4i64 = sdiv <4 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8i64 = sdiv <8 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i32 = sdiv <2 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i32 = sdiv <4 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8i32 = sdiv <8 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16i32 = sdiv <16 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i16 = sdiv <2 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i16 = sdiv <4 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i16 = sdiv <8 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16i16 = sdiv <16 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32i16 = sdiv <32 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i8 = sdiv <2 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i8 = sdiv <4 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i8 = sdiv <8 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i8 = sdiv <16 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32i8 = sdiv <32 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64i8 = sdiv <64 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i64 = sdiv <vscale x 2 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV4i64 = sdiv <vscale x 4 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV8i64 = sdiv <vscale x 8 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i32 = sdiv <vscale x 2 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV4i32 = sdiv <vscale x 4 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV8i32 = sdiv <vscale x 8 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV16i32 = sdiv <vscale x 16 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i16 = sdiv <vscale x 2 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV4i16 = sdiv <vscale x 4 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV8i16 = sdiv <vscale x 8 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV16i16 = sdiv <vscale x 16 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV32i16 = sdiv <vscale x 32 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i8 = sdiv <vscale x 2 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV4i8 = sdiv <vscale x 4 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV8i8 = sdiv <vscale x 8 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV16i8 = sdiv <vscale x 16 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV32i8 = sdiv <vscale x 32 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV64i8 = sdiv <vscale x 64 x i8> undef, splat (i8 -16)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V2i64 = sdiv <2 x i64> undef, splat (i64 -16)
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
index 0bf776b5c97e3..09fd1de99d6c1 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
@@ -1,31 +1,31 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=aarch64--linux-gnu -mattr=+sve | FileCheck %s --check-prefix=CHECK-VSCALE-1
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -mcpu=neoverse-v1 -disable-output -S -mtriple=aarch64--linux-gnu -mattr=+sve | FileCheck %s --check-prefix=CHECK-VSCALE-2
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -type-based-intrinsic-cost -disable-output -S -mtriple=aarch64--linux-gnu -mattr=+sve | FileCheck %s --check-prefix=TYPE_BASED_ONLY
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -S -mtriple=aarch64--linux-gnu -mattr=+sve | FileCheck %s --check-prefix=CHECK-VSCALE-1
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -mcpu=neoverse-v1 -disable-output -S -mtriple=aarch64--linux-gnu -mattr=+sve | FileCheck %s --check-prefix=CHECK-VSCALE-2
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -intrinsic-cost-strategy=type-based-intrinsic-cost -disable-output -S -mtriple=aarch64--linux-gnu -mattr=+sve | FileCheck %s --check-prefix=TYPE_BASED_ONLY
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 define void @vector_insert_extract(<vscale x 4 x i32> %v0, <vscale x 16 x i32> %v1, <16 x i32> %v2) {
 ; CHECK-VSCALE-1-LABEL: 'vector_insert_extract'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %extract_fixed_from_scalable = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> %v0, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %insert_fixed_into_scalable = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> %v0, <16 x i32> %v2, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_scalable_from_scalable = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> %v1, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_scalable_into_scalable = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> %v1, <vscale x 4 x i32> %v0, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 54 for: %extract_fixed_from_scalable = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> %v0, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 54 for: %insert_fixed_into_scalable = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> %v0, <16 x i32> %v2, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %extract_scalable_from_scalable = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> %v1, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %insert_scalable_into_scalable = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> %v1, <vscale x 4 x i32> %v0, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'vector_insert_extract'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %extract_fixed_from_scalable = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> %v0, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %insert_fixed_into_scalable = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> %v0, <16 x i32> %v2, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_scalable_from_scalable = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> %v1, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_scalable_into_scalable = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> %v1, <vscale x 4 x i32> %v0, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 54 for: %extract_fixed_from_scalable = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> %v0, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 54 for: %insert_fixed_into_scalable = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> %v0, <16 x i32> %v2, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %extract_scalable_from_scalable = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> %v1, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %insert_scalable_into_scalable = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> %v1, <vscale x 4 x i32> %v0, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'vector_insert_extract'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %extract_fixed_from_scalable = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> %v0, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %insert_fixed_into_scalable = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> %v0, <16 x i32> %v2, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %extract_scalable_from_scalable = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> %v1, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %insert_scalable_into_scalable = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> %v1, <vscale x 4 x i32> %v0, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %extract_fixed_from_scalable = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> %v0, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %insert_fixed_into_scalable = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> %v0, <16 x i32> %v2, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %extract_scalable_from_scalable = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> %v1, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %insert_scalable_into_scalable = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> %v1, <vscale x 4 x i32> %v0, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %extract_fixed_from_scalable = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> %v0, i64 0)
   %insert_fixed_into_scalable = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> %v0, <16 x i32> %v2, i64 0)
@@ -40,40 +40,40 @@ declare <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x
 
 define void @vector_insert_extract_idxzero_128b() #1 {
 ; CHECK-VSCALE-1-LABEL: 'vector_insert_extract_idxzero_128b'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %insert_legal_fixed_into_scalable = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> undef, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %extract_legal_fixed_from_scalable = call <2 x double> @llvm.vector.extract.v2f64.nxv2f64(<vscale x 2 x double> undef, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> undef, <2 x float> undef, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> undef, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> undef, <vscale x 2 x float> undef, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4f32_nxv8f32 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> undef, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 0 for: %insert_legal_fixed_into_scalable = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 0 for: %extract_legal_fixed_from_scalable = call <2 x double> @llvm.vector.extract.v2f64.nxv2f64(<vscale x 2 x double> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 42 for: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 4 for: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> undef, <2 x float> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 12 for: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> undef, <vscale x 2 x float> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %extract_nxv4f32_nxv8f32 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'vector_insert_extract_idxzero_128b'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %insert_legal_fixed_into_scalable = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> undef, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %extract_legal_fixed_from_scalable = call <2 x double> @llvm.vector.extract.v2f64.nxv2f64(<vscale x 2 x double> undef, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> undef, <2 x float> undef, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> undef, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> undef, <vscale x 2 x float> undef, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4f32_nxv8f32 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> undef, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 0 for: %insert_legal_fixed_into_scalable = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 0 for: %extract_legal_fixed_from_scalable = call <2 x double> @llvm.vector.extract.v2f64.nxv2f64(<vscale x 2 x double> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 42 for: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 4 for: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> undef, <2 x float> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 12 for: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> undef, <vscale x 2 x float> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %extract_nxv4f32_nxv8f32 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'vector_insert_extract_idxzero_128b'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %insert_legal_fixed_into_scalable = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> undef, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %extract_legal_fixed_from_scalable = call <2 x double> @llvm.vector.extract.v2f64.nxv2f64(<vscale x 2 x double> undef, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> undef, <2 x float> undef, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> undef, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> undef, <vscale x 2 x float> undef, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %extract_nxv4f32_nxv8f32 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> undef, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %insert_legal_fixed_into_scalable = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> undef, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %extract_legal_fixed_from_scalable = call <2 x double> @llvm.vector.extract.v2f64.nxv2f64(<vscale x 2 x double> undef, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> undef, <2 x float> undef, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> undef, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> undef, <vscale x 2 x float> undef, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %extract_nxv4f32_nxv8f32 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> undef, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %insert_legal_fixed_into_scalable = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> undef, <4 x float> undef, i64 0)
   %extract_legal_fixed_from_scalable = call <2 x double> @llvm.vector.extract.v2f64.nxv2f64(<vscale x 2 x double> undef, i64 0)
@@ -97,40 +97,40 @@ declare <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x
 
 define void @vector_insert_extract_idxzero_256b() #2 {
 ; CHECK-VSCALE-1-LABEL: 'vector_insert_extract_idxzero_256b'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %insert_legal_fixed_into_scalable = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> undef, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %extract_legal_fixed_from_scalable = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> undef, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> undef, <2 x float> undef, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> undef, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> undef, <vscale x 2 x float> undef, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4f32_nxv8f32 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> undef, i64 0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 0 for: %insert_legal_fixed_into_scalable = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 0 for: %extract_legal_fixed_from_scalable = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 42 for: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 4 for: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> undef, <2 x float> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 12 for: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> undef, <vscale x 2 x float> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %extract_nxv4f32_nxv8f32 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> undef, i64 0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'vector_insert_extract_idxzero_256b'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %insert_legal_fixed_into_scalable = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> undef, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %extract_legal_fixed_from_scalable = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> undef, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> undef, <2 x float> undef, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> undef, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> undef, <vscale x 2 x float> undef, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %extract_nxv4f32_nxv8f32 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> undef, i64 0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 0 for: %insert_legal_fixed_into_scalable = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 0 for: %extract_legal_fixed_from_scalable = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 42 for: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 4 for: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> undef, <2 x float> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 12 for: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> undef, <vscale x 2 x float> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %extract_nxv4f32_nxv8f32 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> undef, i64 0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'vector_insert_extract_idxzero_256b'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %insert_legal_fixed_into_scalable = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> undef, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %extract_legal_fixed_from_scalable = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> undef, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> undef, <2 x float> undef, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> undef, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> undef, <vscale x 2 x float> undef, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %extract_nxv4f32_nxv8f32 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> undef, i64 0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %insert_legal_fixed_into_scalable = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> undef, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %extract_legal_fixed_from_scalable = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> undef, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> undef, <vscale x 2 x i1> undef, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> undef, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> undef, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> undef, <2 x float> undef, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> undef, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> undef, <vscale x 2 x float> undef, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %extract_nxv4f32_nxv8f32 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> undef, i64 0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %insert_legal_fixed_into_scalable = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> undef, i64 0)
   %extract_legal_fixed_from_scalable = call <8 x float> @llvm.vector.extract.v8f32.nx4f32(<vscale x 4 x float> undef, i64 0)
@@ -148,121 +148,121 @@ declare <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float>, i64
 
 define void @reductions(<vscale x 4 x i32> %v0, <vscale x 4 x i64> %v1, <vscale x 4 x float> %v2, <vscale x 4 x double> %v3) {
 ; CHECK-VSCALE-1-LABEL: 'reductions'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %add_nxv1i32 = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %add_nxv4i32 = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %add_nxv4i64 = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %mul_nxv1i32 = call i32 @llvm.vector.reduce.mul.nxv1i32(<vscale x 1 x i32> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %mul_nxv4i32 = call i32 @llvm.vector.reduce.mul.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %mul_nxv4i64 = call i64 @llvm.vector.reduce.mul.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %and_nxv1i32 = call i32 @llvm.vector.reduce.and.nxv1i32(<vscale x 1 x i32> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %and_nxv4i32 = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %and_nxv4i64 = call i64 @llvm.vector.reduce.and.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %or_nxv1i32 = call i32 @llvm.vector.reduce.or.nxv1i32(<vscale x 1 x i32> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %or_nxv4i32 = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %or_nxv4i64 = call i64 @llvm.vector.reduce.or.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %xor_nxv1i32 = call i32 @llvm.vector.reduce.xor.nxv1i32(<vscale x 1 x i32> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %xor_nxv4i32 = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %xor_nxv4i64 = call i64 @llvm.vector.reduce.xor.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %umin_nxv1i64 = call i64 @llvm.vector.reduce.umin.nxv1i64(<vscale x 1 x i64> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %umin_nxv4i32 = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %umin_nxv4i64 = call i64 @llvm.vector.reduce.umin.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %smin_nxv1i64 = call i64 @llvm.vector.reduce.smin.nxv1i64(<vscale x 1 x i64> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %smin_nxv4i32 = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %smin_nxv4i64 = call i64 @llvm.vector.reduce.smin.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %umax_nxv1i64 = call i64 @llvm.vector.reduce.umax.nxv1i64(<vscale x 1 x i64> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %umax_nxv4i32 = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %umax_nxv4i64 = call i64 @llvm.vector.reduce.umax.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %smax_nxv1i64 = call i64 @llvm.vector.reduce.smax.nxv1i64(<vscale x 1 x i64> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %smax_nxv4i32 = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %smax_nxv4i64 = call i64 @llvm.vector.reduce.smax.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %fadd_nxv1f32 = call fast float @llvm.vector.reduce.fadd.nxv1f32(float 0.000000e+00, <vscale x 1 x float> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_nxv4f32 = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v2)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fadd_nxv4f64 = call fast double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v3)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %fmin_nxv1f32 = call fast float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fmin_nxv4f32 = call fast float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> %v2)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fmin_nxv4f64 = call fast double @llvm.vector.reduce.fmin.nxv4f64(<vscale x 4 x double> %v3)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %fmax_nxv1f32 = call fast float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fmax_nxv4f32 = call fast float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> %v2)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fmax_nxv4f64 = call fast double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double> %v3)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %add_nxv1i32 = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %add_nxv4i32 = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %add_nxv4i64 = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %mul_nxv1i32 = call i32 @llvm.vector.reduce.mul.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %mul_nxv4i32 = call i32 @llvm.vector.reduce.mul.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %mul_nxv4i64 = call i64 @llvm.vector.reduce.mul.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %and_nxv1i32 = call i32 @llvm.vector.reduce.and.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %and_nxv4i32 = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %and_nxv4i64 = call i64 @llvm.vector.reduce.and.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %or_nxv1i32 = call i32 @llvm.vector.reduce.or.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %or_nxv4i32 = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %or_nxv4i64 = call i64 @llvm.vector.reduce.or.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %xor_nxv1i32 = call i32 @llvm.vector.reduce.xor.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %xor_nxv4i32 = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %xor_nxv4i64 = call i64 @llvm.vector.reduce.xor.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %umin_nxv1i64 = call i64 @llvm.vector.reduce.umin.nxv1i64(<vscale x 1 x i64> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %umin_nxv4i32 = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %umin_nxv4i64 = call i64 @llvm.vector.reduce.umin.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %smin_nxv1i64 = call i64 @llvm.vector.reduce.smin.nxv1i64(<vscale x 1 x i64> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %smin_nxv4i32 = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %smin_nxv4i64 = call i64 @llvm.vector.reduce.smin.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %umax_nxv1i64 = call i64 @llvm.vector.reduce.umax.nxv1i64(<vscale x 1 x i64> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %umax_nxv4i32 = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %umax_nxv4i64 = call i64 @llvm.vector.reduce.umax.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %smax_nxv1i64 = call i64 @llvm.vector.reduce.smax.nxv1i64(<vscale x 1 x i64> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %smax_nxv4i32 = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %smax_nxv4i64 = call i64 @llvm.vector.reduce.smax.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %fadd_nxv1f32 = call fast float @llvm.vector.reduce.fadd.nxv1f32(float 0.000000e+00, <vscale x 1 x float> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %fadd_nxv4f32 = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v2)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:3 Lat:5 SizeLat:3 for: %fadd_nxv4f64 = call fast double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v3)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %fmin_nxv1f32 = call fast float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %fmin_nxv4f32 = call fast float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> %v2)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 4 for: %fmin_nxv4f64 = call fast double @llvm.vector.reduce.fmin.nxv4f64(<vscale x 4 x double> %v3)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %fmax_nxv1f32 = call fast float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %fmax_nxv4f32 = call fast float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> %v2)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 4 for: %fmax_nxv4f64 = call fast double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double> %v3)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'reductions'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %add_nxv1i32 = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %add_nxv4i32 = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %add_nxv4i64 = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %mul_nxv1i32 = call i32 @llvm.vector.reduce.mul.nxv1i32(<vscale x 1 x i32> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %mul_nxv4i32 = call i32 @llvm.vector.reduce.mul.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %mul_nxv4i64 = call i64 @llvm.vector.reduce.mul.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %and_nxv1i32 = call i32 @llvm.vector.reduce.and.nxv1i32(<vscale x 1 x i32> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %and_nxv4i32 = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %and_nxv4i64 = call i64 @llvm.vector.reduce.and.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %or_nxv1i32 = call i32 @llvm.vector.reduce.or.nxv1i32(<vscale x 1 x i32> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %or_nxv4i32 = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %or_nxv4i64 = call i64 @llvm.vector.reduce.or.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %xor_nxv1i32 = call i32 @llvm.vector.reduce.xor.nxv1i32(<vscale x 1 x i32> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %xor_nxv4i32 = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %xor_nxv4i64 = call i64 @llvm.vector.reduce.xor.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %umin_nxv1i64 = call i64 @llvm.vector.reduce.umin.nxv1i64(<vscale x 1 x i64> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %umin_nxv4i32 = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %umin_nxv4i64 = call i64 @llvm.vector.reduce.umin.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %smin_nxv1i64 = call i64 @llvm.vector.reduce.smin.nxv1i64(<vscale x 1 x i64> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %smin_nxv4i32 = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %smin_nxv4i64 = call i64 @llvm.vector.reduce.smin.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %umax_nxv1i64 = call i64 @llvm.vector.reduce.umax.nxv1i64(<vscale x 1 x i64> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %umax_nxv4i32 = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %umax_nxv4i64 = call i64 @llvm.vector.reduce.umax.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %smax_nxv1i64 = call i64 @llvm.vector.reduce.smax.nxv1i64(<vscale x 1 x i64> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %smax_nxv4i32 = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> %v0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %smax_nxv4i64 = call i64 @llvm.vector.reduce.smax.nxv4i64(<vscale x 4 x i64> %v1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %fadd_nxv1f32 = call fast float @llvm.vector.reduce.fadd.nxv1f32(float 0.000000e+00, <vscale x 1 x float> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_nxv4f32 = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v2)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fadd_nxv4f64 = call fast double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v3)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %fmin_nxv1f32 = call fast float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fmin_nxv4f32 = call fast float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> %v2)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fmin_nxv4f64 = call fast double @llvm.vector.reduce.fmin.nxv4f64(<vscale x 4 x double> %v3)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %fmax_nxv1f32 = call fast float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fmax_nxv4f32 = call fast float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> %v2)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fmax_nxv4f64 = call fast double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double> %v3)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %add_nxv1i32 = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %add_nxv4i32 = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %add_nxv4i64 = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %mul_nxv1i32 = call i32 @llvm.vector.reduce.mul.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %mul_nxv4i32 = call i32 @llvm.vector.reduce.mul.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %mul_nxv4i64 = call i64 @llvm.vector.reduce.mul.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %and_nxv1i32 = call i32 @llvm.vector.reduce.and.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %and_nxv4i32 = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %and_nxv4i64 = call i64 @llvm.vector.reduce.and.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %or_nxv1i32 = call i32 @llvm.vector.reduce.or.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %or_nxv4i32 = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %or_nxv4i64 = call i64 @llvm.vector.reduce.or.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %xor_nxv1i32 = call i32 @llvm.vector.reduce.xor.nxv1i32(<vscale x 1 x i32> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %xor_nxv4i32 = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %xor_nxv4i64 = call i64 @llvm.vector.reduce.xor.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %umin_nxv1i64 = call i64 @llvm.vector.reduce.umin.nxv1i64(<vscale x 1 x i64> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %umin_nxv4i32 = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %umin_nxv4i64 = call i64 @llvm.vector.reduce.umin.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %smin_nxv1i64 = call i64 @llvm.vector.reduce.smin.nxv1i64(<vscale x 1 x i64> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %smin_nxv4i32 = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %smin_nxv4i64 = call i64 @llvm.vector.reduce.smin.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %umax_nxv1i64 = call i64 @llvm.vector.reduce.umax.nxv1i64(<vscale x 1 x i64> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %umax_nxv4i32 = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %umax_nxv4i64 = call i64 @llvm.vector.reduce.umax.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %smax_nxv1i64 = call i64 @llvm.vector.reduce.smax.nxv1i64(<vscale x 1 x i64> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %smax_nxv4i32 = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %smax_nxv4i64 = call i64 @llvm.vector.reduce.smax.nxv4i64(<vscale x 4 x i64> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %fadd_nxv1f32 = call fast float @llvm.vector.reduce.fadd.nxv1f32(float 0.000000e+00, <vscale x 1 x float> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %fadd_nxv4f32 = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v2)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:3 Lat:5 SizeLat:3 for: %fadd_nxv4f64 = call fast double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v3)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %fmin_nxv1f32 = call fast float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %fmin_nxv4f32 = call fast float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> %v2)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 4 for: %fmin_nxv4f64 = call fast double @llvm.vector.reduce.fmin.nxv4f64(<vscale x 4 x double> %v3)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %fmax_nxv1f32 = call fast float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %fmax_nxv4f32 = call fast float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> %v2)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 4 for: %fmax_nxv4f64 = call fast double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double> %v3)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'reductions'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %add_nxv1i32 = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %add_nxv4i32 = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %v0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %add_nxv4i64 = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> %v1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %mul_nxv1i32 = call i32 @llvm.vector.reduce.mul.nxv1i32(<vscale x 1 x i32> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %mul_nxv4i32 = call i32 @llvm.vector.reduce.mul.nxv4i32(<vscale x 4 x i32> %v0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %mul_nxv4i64 = call i64 @llvm.vector.reduce.mul.nxv4i64(<vscale x 4 x i64> %v1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %and_nxv1i32 = call i32 @llvm.vector.reduce.and.nxv1i32(<vscale x 1 x i32> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %and_nxv4i32 = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> %v0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %and_nxv4i64 = call i64 @llvm.vector.reduce.and.nxv4i64(<vscale x 4 x i64> %v1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %or_nxv1i32 = call i32 @llvm.vector.reduce.or.nxv1i32(<vscale x 1 x i32> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %or_nxv4i32 = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> %v0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %or_nxv4i64 = call i64 @llvm.vector.reduce.or.nxv4i64(<vscale x 4 x i64> %v1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %xor_nxv1i32 = call i32 @llvm.vector.reduce.xor.nxv1i32(<vscale x 1 x i32> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %xor_nxv4i32 = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> %v0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %xor_nxv4i64 = call i64 @llvm.vector.reduce.xor.nxv4i64(<vscale x 4 x i64> %v1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %umin_nxv1i64 = call i64 @llvm.vector.reduce.umin.nxv1i64(<vscale x 1 x i64> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %umin_nxv4i32 = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> %v0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %umin_nxv4i64 = call i64 @llvm.vector.reduce.umin.nxv4i64(<vscale x 4 x i64> %v1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %smin_nxv1i64 = call i64 @llvm.vector.reduce.smin.nxv1i64(<vscale x 1 x i64> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %smin_nxv4i32 = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> %v0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %smin_nxv4i64 = call i64 @llvm.vector.reduce.smin.nxv4i64(<vscale x 4 x i64> %v1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %umax_nxv1i64 = call i64 @llvm.vector.reduce.umax.nxv1i64(<vscale x 1 x i64> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %umax_nxv4i32 = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> %v0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %umax_nxv4i64 = call i64 @llvm.vector.reduce.umax.nxv4i64(<vscale x 4 x i64> %v1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %smax_nxv1i64 = call i64 @llvm.vector.reduce.smax.nxv1i64(<vscale x 1 x i64> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %smax_nxv4i32 = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> %v0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %smax_nxv4i64 = call i64 @llvm.vector.reduce.smax.nxv4i64(<vscale x 4 x i64> %v1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %fadd_nxv1f32 = call fast float @llvm.vector.reduce.fadd.nxv1f32(float 0.000000e+00, <vscale x 1 x float> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fadd_nxv4f32 = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v2)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %fadd_nxv4f64 = call fast double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v3)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %fmin_nxv1f32 = call fast float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fmin_nxv4f32 = call fast float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> %v2)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fmin_nxv4f64 = call fast double @llvm.vector.reduce.fmin.nxv4f64(<vscale x 4 x double> %v3)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %fmax_nxv1f32 = call fast float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %fmax_nxv4f32 = call fast float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> %v2)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fmax_nxv4f64 = call fast double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double> %v3)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %add_nxv1i32 = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 2 for: %add_nxv4i32 = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %v0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 3 for: %add_nxv4i64 = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> %v1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %mul_nxv1i32 = call i32 @llvm.vector.reduce.mul.nxv1i32(<vscale x 1 x i32> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %mul_nxv4i32 = call i32 @llvm.vector.reduce.mul.nxv4i32(<vscale x 4 x i32> %v0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %mul_nxv4i64 = call i64 @llvm.vector.reduce.mul.nxv4i64(<vscale x 4 x i64> %v1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %and_nxv1i32 = call i32 @llvm.vector.reduce.and.nxv1i32(<vscale x 1 x i32> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 2 for: %and_nxv4i32 = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> %v0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 3 for: %and_nxv4i64 = call i64 @llvm.vector.reduce.and.nxv4i64(<vscale x 4 x i64> %v1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %or_nxv1i32 = call i32 @llvm.vector.reduce.or.nxv1i32(<vscale x 1 x i32> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 2 for: %or_nxv4i32 = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> %v0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 3 for: %or_nxv4i64 = call i64 @llvm.vector.reduce.or.nxv4i64(<vscale x 4 x i64> %v1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %xor_nxv1i32 = call i32 @llvm.vector.reduce.xor.nxv1i32(<vscale x 1 x i32> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 2 for: %xor_nxv4i32 = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> %v0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 3 for: %xor_nxv4i64 = call i64 @llvm.vector.reduce.xor.nxv4i64(<vscale x 4 x i64> %v1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %umin_nxv1i64 = call i64 @llvm.vector.reduce.umin.nxv1i64(<vscale x 1 x i64> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 2 for: %umin_nxv4i32 = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> %v0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 3 for: %umin_nxv4i64 = call i64 @llvm.vector.reduce.umin.nxv4i64(<vscale x 4 x i64> %v1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %smin_nxv1i64 = call i64 @llvm.vector.reduce.smin.nxv1i64(<vscale x 1 x i64> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 2 for: %smin_nxv4i32 = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> %v0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 3 for: %smin_nxv4i64 = call i64 @llvm.vector.reduce.smin.nxv4i64(<vscale x 4 x i64> %v1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %umax_nxv1i64 = call i64 @llvm.vector.reduce.umax.nxv1i64(<vscale x 1 x i64> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 2 for: %umax_nxv4i32 = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> %v0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 3 for: %umax_nxv4i64 = call i64 @llvm.vector.reduce.umax.nxv4i64(<vscale x 4 x i64> %v1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %smax_nxv1i64 = call i64 @llvm.vector.reduce.smax.nxv1i64(<vscale x 1 x i64> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 2 for: %smax_nxv4i32 = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> %v0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 3 for: %smax_nxv4i64 = call i64 @llvm.vector.reduce.smax.nxv4i64(<vscale x 4 x i64> %v1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %fadd_nxv1f32 = call fast float @llvm.vector.reduce.fadd.nxv1f32(float 0.000000e+00, <vscale x 1 x float> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 2 for: %fadd_nxv4f32 = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v2)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:3 Lat:5 SizeLat:3 for: %fadd_nxv4f64 = call fast double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v3)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %fmin_nxv1f32 = call fast float @llvm.vector.reduce.fmin.nxv1f32(<vscale x 1 x float> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 2 for: %fmin_nxv4f32 = call fast float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> %v2)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 4 for: %fmin_nxv4f64 = call fast double @llvm.vector.reduce.fmin.nxv4f64(<vscale x 4 x double> %v3)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %fmax_nxv1f32 = call fast float @llvm.vector.reduce.fmax.nxv1f32(<vscale x 1 x float> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 2 for: %fmax_nxv4f32 = call fast float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> %v2)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 4 for: %fmax_nxv4f64 = call fast double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double> %v3)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %add_nxv1i32 = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> undef)
   %add_nxv4i32 = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %v0)
@@ -307,25 +307,25 @@ define void @reductions(<vscale x 4 x i32> %v0, <vscale x 4 x i64> %v1, <vscale
 
 define void @strict_fp_reductions(<vscale x 4 x float> %v0, <vscale x 4 x double> %v1) {
 ; CHECK-VSCALE-1-LABEL: 'strict_fp_reductions'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_nxv4f32 = call float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_nxv4f64 = call double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %fmul_nxv4f32 = call float @llvm.vector.reduce.fmul.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %fmul_nxv4f64 = call double @llvm.vector.reduce.fmul.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:4 Lat:12 SizeLat:4 for: %fadd_nxv4f32 = call float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:4 Lat:12 SizeLat:4 for: %fadd_nxv4f64 = call double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %fmul_nxv4f32 = call float @llvm.vector.reduce.fmul.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %fmul_nxv4f64 = call double @llvm.vector.reduce.fmul.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'strict_fp_reductions'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_nxv4f32 = call float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %fadd_nxv4f64 = call double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %fmul_nxv4f32 = call float @llvm.vector.reduce.fmul.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %fmul_nxv4f64 = call double @llvm.vector.reduce.fmul.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:8 Lat:24 SizeLat:8 for: %fadd_nxv4f32 = call float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:8 Lat:24 SizeLat:8 for: %fadd_nxv4f64 = call double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %fmul_nxv4f32 = call float @llvm.vector.reduce.fmul.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %fmul_nxv4f64 = call double @llvm.vector.reduce.fmul.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'strict_fp_reductions'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_nxv4f32 = call float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %fadd_nxv4f64 = call double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %fmul_nxv4f32 = call float @llvm.vector.reduce.fmul.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %fmul_nxv4f64 = call double @llvm.vector.reduce.fmul.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:4 Lat:12 SizeLat:4 for: %fadd_nxv4f32 = call float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:4 Lat:12 SizeLat:4 for: %fadd_nxv4f64 = call double @llvm.vector.reduce.fadd.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %fmul_nxv4f32 = call float @llvm.vector.reduce.fmul.nxv4f32(float 0.000000e+00, <vscale x 4 x float> %v0)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %fmul_nxv4f64 = call double @llvm.vector.reduce.fmul.nxv4f64(double 0.000000e+00, <vscale x 4 x double> %v1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %fadd_nxv4f32 = call float @llvm.vector.reduce.fadd.nxv4f32(float 0.0, <vscale x 4 x float> %v0)
   %fadd_nxv4f64 = call double @llvm.vector.reduce.fadd.nxv4f64(double 0.0, <vscale x 4 x double> %v1)
@@ -365,19 +365,19 @@ declare double @llvm.vector.reduce.fmax.nxv4f64(<vscale x 4 x double>)
 
 define void  @count_zeroes(<vscale x 4 x i32> %A) {
 ; CHECK-VSCALE-1-LABEL: 'count_zeroes'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %ctlz = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %ctlz = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %cttz = call <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'count_zeroes'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %ctlz = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %ctlz = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %cttz = call <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'count_zeroes'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %ctlz = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 2 for: %ctlz = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 2 for: %cttz = call <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %ctlz = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
   %cttz = call <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32> %A, i1 true)
@@ -389,94 +389,94 @@ declare <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32>, i1)
 
 define void @vector_reverse() #0 {
 ; CHECK-VSCALE-1-LABEL: 'vector_reverse'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv32i8 = call <vscale x 32 x i8> @llvm.vector.reverse.nxv32i8(<vscale x 32 x i8> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2i16 = call <vscale x 2 x i16> @llvm.vector.reverse.nxv2i16(<vscale x 2 x i16> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4i16 = call <vscale x 4 x i16> @llvm.vector.reverse.nxv4i16(<vscale x 4 x i16> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8i16 = call <vscale x 8 x i16> @llvm.vector.reverse.nxv8i16(<vscale x 8 x i16> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.reverse.nxv16i16(<vscale x 16 x i16> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.reverse.nxv8i32(<vscale x 8 x i32> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.reverse.nxv2i64(<vscale x 2 x i64> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.reverse.nxv4i64(<vscale x 4 x i64> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2f16 = call <vscale x 2 x half> @llvm.vector.reverse.nxv2f16(<vscale x 2 x half> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4f16 = call <vscale x 4 x half> @llvm.vector.reverse.nxv4f16(<vscale x 4 x half> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8f16 = call <vscale x 8 x half> @llvm.vector.reverse.nxv8f16(<vscale x 8 x half> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv16f16 = call <vscale x 16 x half> @llvm.vector.reverse.nxv16f16(<vscale x 16 x half> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2f32 = call <vscale x 2 x float> @llvm.vector.reverse.nxv2f32(<vscale x 2 x float> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4f32 = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv8f32 = call <vscale x 8 x float> @llvm.vector.reverse.nxv8f32(<vscale x 8 x float> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2f64 = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv4f64 = call <vscale x 4 x double> @llvm.vector.reverse.nxv4f64(<vscale x 4 x double> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2bf16 = call <vscale x 2 x bfloat> @llvm.vector.reverse.nxv2bf16(<vscale x 2 x bfloat> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.reverse.nxv4bf16(<vscale x 4 x bfloat> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.reverse.nxv8bf16(<vscale x 8 x bfloat> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.reverse.nxv16bf16(<vscale x 16 x bfloat> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.reverse.nxv16i1(<vscale x 16 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.reverse.nxv8i1(<vscale x 8 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.reverse.nxv4i1(<vscale x 4 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %reverse_nxv32i8 = call <vscale x 32 x i8> @llvm.vector.reverse.nxv32i8(<vscale x 32 x i8> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv2i16 = call <vscale x 2 x i16> @llvm.vector.reverse.nxv2i16(<vscale x 2 x i16> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv4i16 = call <vscale x 4 x i16> @llvm.vector.reverse.nxv4i16(<vscale x 4 x i16> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv8i16 = call <vscale x 8 x i16> @llvm.vector.reverse.nxv8i16(<vscale x 8 x i16> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %reverse_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.reverse.nxv16i16(<vscale x 16 x i16> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %reverse_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.reverse.nxv8i32(<vscale x 8 x i32> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.reverse.nxv2i64(<vscale x 2 x i64> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %reverse_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.reverse.nxv4i64(<vscale x 4 x i64> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv2f16 = call <vscale x 2 x half> @llvm.vector.reverse.nxv2f16(<vscale x 2 x half> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv4f16 = call <vscale x 4 x half> @llvm.vector.reverse.nxv4f16(<vscale x 4 x half> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv8f16 = call <vscale x 8 x half> @llvm.vector.reverse.nxv8f16(<vscale x 8 x half> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %reverse_nxv16f16 = call <vscale x 16 x half> @llvm.vector.reverse.nxv16f16(<vscale x 16 x half> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv2f32 = call <vscale x 2 x float> @llvm.vector.reverse.nxv2f32(<vscale x 2 x float> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv4f32 = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %reverse_nxv8f32 = call <vscale x 8 x float> @llvm.vector.reverse.nxv8f32(<vscale x 8 x float> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv2f64 = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %reverse_nxv4f64 = call <vscale x 4 x double> @llvm.vector.reverse.nxv4f64(<vscale x 4 x double> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv2bf16 = call <vscale x 2 x bfloat> @llvm.vector.reverse.nxv2bf16(<vscale x 2 x bfloat> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.reverse.nxv4bf16(<vscale x 4 x bfloat> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.reverse.nxv8bf16(<vscale x 8 x bfloat> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %reverse_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.reverse.nxv16bf16(<vscale x 16 x bfloat> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.reverse.nxv16i1(<vscale x 16 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.reverse.nxv8i1(<vscale x 8 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.reverse.nxv4i1(<vscale x 4 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'vector_reverse'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv32i8 = call <vscale x 32 x i8> @llvm.vector.reverse.nxv32i8(<vscale x 32 x i8> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2i16 = call <vscale x 2 x i16> @llvm.vector.reverse.nxv2i16(<vscale x 2 x i16> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4i16 = call <vscale x 4 x i16> @llvm.vector.reverse.nxv4i16(<vscale x 4 x i16> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8i16 = call <vscale x 8 x i16> @llvm.vector.reverse.nxv8i16(<vscale x 8 x i16> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.reverse.nxv16i16(<vscale x 16 x i16> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.reverse.nxv8i32(<vscale x 8 x i32> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.reverse.nxv2i64(<vscale x 2 x i64> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.reverse.nxv4i64(<vscale x 4 x i64> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2f16 = call <vscale x 2 x half> @llvm.vector.reverse.nxv2f16(<vscale x 2 x half> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4f16 = call <vscale x 4 x half> @llvm.vector.reverse.nxv4f16(<vscale x 4 x half> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8f16 = call <vscale x 8 x half> @llvm.vector.reverse.nxv8f16(<vscale x 8 x half> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv16f16 = call <vscale x 16 x half> @llvm.vector.reverse.nxv16f16(<vscale x 16 x half> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2f32 = call <vscale x 2 x float> @llvm.vector.reverse.nxv2f32(<vscale x 2 x float> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4f32 = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv8f32 = call <vscale x 8 x float> @llvm.vector.reverse.nxv8f32(<vscale x 8 x float> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2f64 = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv4f64 = call <vscale x 4 x double> @llvm.vector.reverse.nxv4f64(<vscale x 4 x double> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2bf16 = call <vscale x 2 x bfloat> @llvm.vector.reverse.nxv2bf16(<vscale x 2 x bfloat> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.reverse.nxv4bf16(<vscale x 4 x bfloat> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.reverse.nxv8bf16(<vscale x 8 x bfloat> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %reverse_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.reverse.nxv16bf16(<vscale x 16 x bfloat> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.reverse.nxv16i1(<vscale x 16 x i1> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.reverse.nxv8i1(<vscale x 8 x i1> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.reverse.nxv4i1(<vscale x 4 x i1> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %reverse_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %reverse_nxv32i8 = call <vscale x 32 x i8> @llvm.vector.reverse.nxv32i8(<vscale x 32 x i8> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv2i16 = call <vscale x 2 x i16> @llvm.vector.reverse.nxv2i16(<vscale x 2 x i16> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv4i16 = call <vscale x 4 x i16> @llvm.vector.reverse.nxv4i16(<vscale x 4 x i16> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv8i16 = call <vscale x 8 x i16> @llvm.vector.reverse.nxv8i16(<vscale x 8 x i16> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %reverse_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.reverse.nxv16i16(<vscale x 16 x i16> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %reverse_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.reverse.nxv8i32(<vscale x 8 x i32> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.reverse.nxv2i64(<vscale x 2 x i64> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %reverse_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.reverse.nxv4i64(<vscale x 4 x i64> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv2f16 = call <vscale x 2 x half> @llvm.vector.reverse.nxv2f16(<vscale x 2 x half> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv4f16 = call <vscale x 4 x half> @llvm.vector.reverse.nxv4f16(<vscale x 4 x half> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv8f16 = call <vscale x 8 x half> @llvm.vector.reverse.nxv8f16(<vscale x 8 x half> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %reverse_nxv16f16 = call <vscale x 16 x half> @llvm.vector.reverse.nxv16f16(<vscale x 16 x half> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv2f32 = call <vscale x 2 x float> @llvm.vector.reverse.nxv2f32(<vscale x 2 x float> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv4f32 = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %reverse_nxv8f32 = call <vscale x 8 x float> @llvm.vector.reverse.nxv8f32(<vscale x 8 x float> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv2f64 = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %reverse_nxv4f64 = call <vscale x 4 x double> @llvm.vector.reverse.nxv4f64(<vscale x 4 x double> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv2bf16 = call <vscale x 2 x bfloat> @llvm.vector.reverse.nxv2bf16(<vscale x 2 x bfloat> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.reverse.nxv4bf16(<vscale x 4 x bfloat> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.reverse.nxv8bf16(<vscale x 8 x bfloat> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %reverse_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.reverse.nxv16bf16(<vscale x 16 x bfloat> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.reverse.nxv16i1(<vscale x 16 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.reverse.nxv8i1(<vscale x 8 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.reverse.nxv4i1(<vscale x 4 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %reverse_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'vector_reverse'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv32i8 = call <vscale x 32 x i8> @llvm.vector.reverse.nxv32i8(<vscale x 32 x i8> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv2i16 = call <vscale x 2 x i16> @llvm.vector.reverse.nxv2i16(<vscale x 2 x i16> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv4i16 = call <vscale x 4 x i16> @llvm.vector.reverse.nxv4i16(<vscale x 4 x i16> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv8i16 = call <vscale x 8 x i16> @llvm.vector.reverse.nxv8i16(<vscale x 8 x i16> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.reverse.nxv16i16(<vscale x 16 x i16> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.reverse.nxv8i32(<vscale x 8 x i32> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.reverse.nxv2i64(<vscale x 2 x i64> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.reverse.nxv4i64(<vscale x 4 x i64> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv2f16 = call <vscale x 2 x half> @llvm.vector.reverse.nxv2f16(<vscale x 2 x half> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv4f16 = call <vscale x 4 x half> @llvm.vector.reverse.nxv4f16(<vscale x 4 x half> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv8f16 = call <vscale x 8 x half> @llvm.vector.reverse.nxv8f16(<vscale x 8 x half> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv16f16 = call <vscale x 16 x half> @llvm.vector.reverse.nxv16f16(<vscale x 16 x half> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv2f32 = call <vscale x 2 x float> @llvm.vector.reverse.nxv2f32(<vscale x 2 x float> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv4f32 = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv8f32 = call <vscale x 8 x float> @llvm.vector.reverse.nxv8f32(<vscale x 8 x float> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv2f64 = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv4f64 = call <vscale x 4 x double> @llvm.vector.reverse.nxv4f64(<vscale x 4 x double> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv2bf16 = call <vscale x 2 x bfloat> @llvm.vector.reverse.nxv2bf16(<vscale x 2 x bfloat> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.reverse.nxv4bf16(<vscale x 4 x bfloat> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.reverse.nxv8bf16(<vscale x 8 x bfloat> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.reverse.nxv16bf16(<vscale x 16 x bfloat> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.reverse.nxv16i1(<vscale x 16 x i1> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.reverse.nxv8i1(<vscale x 8 x i1> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.reverse.nxv4i1(<vscale x 4 x i1> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %reverse_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv32i8 = call <vscale x 32 x i8> @llvm.vector.reverse.nxv32i8(<vscale x 32 x i8> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv2i16 = call <vscale x 2 x i16> @llvm.vector.reverse.nxv2i16(<vscale x 2 x i16> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv4i16 = call <vscale x 4 x i16> @llvm.vector.reverse.nxv4i16(<vscale x 4 x i16> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv8i16 = call <vscale x 8 x i16> @llvm.vector.reverse.nxv8i16(<vscale x 8 x i16> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.reverse.nxv16i16(<vscale x 16 x i16> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.reverse.nxv8i32(<vscale x 8 x i32> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.reverse.nxv2i64(<vscale x 2 x i64> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.reverse.nxv4i64(<vscale x 4 x i64> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv2f16 = call <vscale x 2 x half> @llvm.vector.reverse.nxv2f16(<vscale x 2 x half> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv4f16 = call <vscale x 4 x half> @llvm.vector.reverse.nxv4f16(<vscale x 4 x half> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv8f16 = call <vscale x 8 x half> @llvm.vector.reverse.nxv8f16(<vscale x 8 x half> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv16f16 = call <vscale x 16 x half> @llvm.vector.reverse.nxv16f16(<vscale x 16 x half> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv2f32 = call <vscale x 2 x float> @llvm.vector.reverse.nxv2f32(<vscale x 2 x float> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv4f32 = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv8f32 = call <vscale x 8 x float> @llvm.vector.reverse.nxv8f32(<vscale x 8 x float> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv2f64 = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv4f64 = call <vscale x 4 x double> @llvm.vector.reverse.nxv4f64(<vscale x 4 x double> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv2bf16 = call <vscale x 2 x bfloat> @llvm.vector.reverse.nxv2bf16(<vscale x 2 x bfloat> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.reverse.nxv4bf16(<vscale x 4 x bfloat> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.reverse.nxv8bf16(<vscale x 8 x bfloat> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.reverse.nxv16bf16(<vscale x 16 x bfloat> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.reverse.nxv16i1(<vscale x 16 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.reverse.nxv8i1(<vscale x 8 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.reverse.nxv4i1(<vscale x 4 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %reverse_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 
   %reverse_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.reverse.nxv16i8(<vscale x 16 x i8> undef)
@@ -538,40 +538,40 @@ declare <vscale x 2 x i1> @llvm.vector.reverse.nxv2i1(<vscale x 2 x i1>)
 
 define void @unsupported_fp_ops(<vscale x 4 x float> %vec, i32 %extraarg) {
 ; CHECK-VSCALE-1-LABEL: 'unsupported_fp_ops'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %sin = call <vscale x 4 x float> @llvm.sin.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %cos = call <vscale x 4 x float> @llvm.cos.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %pow = call <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float> %vec, <vscale x 4 x float> %vec)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 %extraarg)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %exp = call <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %exp2 = call <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %log = call <vscale x 4 x float> @llvm.log.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %log2 = call <vscale x 4 x float> @llvm.log2.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %log10 = call <vscale x 4 x float> @llvm.log10.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %sin = call <vscale x 4 x float> @llvm.sin.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %cos = call <vscale x 4 x float> @llvm.cos.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %pow = call <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float> %vec, <vscale x 4 x float> %vec)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 %extraarg)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %exp = call <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %exp2 = call <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %log = call <vscale x 4 x float> @llvm.log.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %log2 = call <vscale x 4 x float> @llvm.log2.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %log10 = call <vscale x 4 x float> @llvm.log10.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'unsupported_fp_ops'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %sin = call <vscale x 4 x float> @llvm.sin.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %cos = call <vscale x 4 x float> @llvm.cos.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %pow = call <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float> %vec, <vscale x 4 x float> %vec)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 %extraarg)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %exp = call <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %exp2 = call <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %log = call <vscale x 4 x float> @llvm.log.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %log2 = call <vscale x 4 x float> @llvm.log2.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %log10 = call <vscale x 4 x float> @llvm.log10.nxv4f32(<vscale x 4 x float> %vec)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %sin = call <vscale x 4 x float> @llvm.sin.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %cos = call <vscale x 4 x float> @llvm.cos.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %pow = call <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float> %vec, <vscale x 4 x float> %vec)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 %extraarg)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %exp = call <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %exp2 = call <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %log = call <vscale x 4 x float> @llvm.log.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %log2 = call <vscale x 4 x float> @llvm.log2.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %log10 = call <vscale x 4 x float> @llvm.log10.nxv4f32(<vscale x 4 x float> %vec)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'unsupported_fp_ops'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %sin = call <vscale x 4 x float> @llvm.sin.nxv4f32(<vscale x 4 x float> %vec)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %cos = call <vscale x 4 x float> @llvm.cos.nxv4f32(<vscale x 4 x float> %vec)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %pow = call <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float> %vec, <vscale x 4 x float> %vec)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 %extraarg)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %exp = call <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> %vec)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %exp2 = call <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> %vec)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %log = call <vscale x 4 x float> @llvm.log.nxv4f32(<vscale x 4 x float> %vec)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %log2 = call <vscale x 4 x float> @llvm.log2.nxv4f32(<vscale x 4 x float> %vec)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %log10 = call <vscale x 4 x float> @llvm.log10.nxv4f32(<vscale x 4 x float> %vec)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %sin = call <vscale x 4 x float> @llvm.sin.nxv4f32(<vscale x 4 x float> %vec)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %cos = call <vscale x 4 x float> @llvm.cos.nxv4f32(<vscale x 4 x float> %vec)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %pow = call <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float> %vec, <vscale x 4 x float> %vec)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 %extraarg)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %exp = call <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> %vec)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %exp2 = call <vscale x 4 x float> @llvm.exp2.nxv4f32(<vscale x 4 x float> %vec)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %log = call <vscale x 4 x float> @llvm.log.nxv4f32(<vscale x 4 x float> %vec)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %log2 = call <vscale x 4 x float> @llvm.log2.nxv4f32(<vscale x 4 x float> %vec)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %log10 = call <vscale x 4 x float> @llvm.log10.nxv4f32(<vscale x 4 x float> %vec)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 
   %sin = call <vscale x 4 x float> @llvm.sin.nxv4f32(<vscale x 4 x float> %vec)
@@ -588,16 +588,16 @@ define void @unsupported_fp_ops(<vscale x 4 x float> %vec, i32 %extraarg) {
 
 define void @powi(<vscale x 4 x float> %vec) {
 ; CHECK-VSCALE-1-LABEL: 'powi'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 42)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:7 Lat:21 SizeLat:7 for: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 42)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'powi'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 42)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:7 Lat:21 SizeLat:7 for: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 42)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'powi'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 42)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 42)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> %vec, i32 42)
   ret void
@@ -615,196 +615,196 @@ declare <vscale x 4 x float> @llvm.log10.nxv4f32(<vscale x 4 x float>)
 
 define void @vector_splice() #0 {
 ; CHECK-VSCALE-1-LABEL: 'vector_splice'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv32i8 = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2i16 = call <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4i16 = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv8i16 = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2f16 = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x half> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4f16 = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x half> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv8f16 = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x half> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv16f16 = call <vscale x 16 x half> @llvm.vector.splice.nxv16f16(<vscale x 16 x half> zeroinitializer, <vscale x 16 x half> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2f32 = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x float> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4f32 = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv8f32 = call <vscale x 8 x float> @llvm.vector.splice.nxv8f32(<vscale x 8 x float> zeroinitializer, <vscale x 8 x float> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2f64 = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv4f64 = call <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double> zeroinitializer, <vscale x 4 x double> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2bf16 = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> zeroinitializer, <vscale x 2 x bfloat> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %splice_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %splice_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %splice_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %splice_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv16i8_neg = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv32i8_neg = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1i16_neg = call <vscale x 1 x i16> @llvm.vector.splice.nxv1i16(<vscale x 1 x i16> zeroinitializer, <vscale x 1 x i16> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2i16_neg = call <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4i16_neg = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv8i16_neg = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv16i16_neg = call <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4i32_neg = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv8i32_neg = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1i64_neg = call <vscale x 1 x i64> @llvm.vector.splice.nxv1i64(<vscale x 1 x i64> zeroinitializer, <vscale x 1 x i64> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2i64_neg = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv4i64_neg = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1f16_neg = call <vscale x 1 x half> @llvm.vector.splice.nxv1f16(<vscale x 1 x half> zeroinitializer, <vscale x 1 x half> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2f16_neg = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x half> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4f16_neg = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x half> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv8f16_neg = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x half> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv16f16_neg = call <vscale x 16 x half> @llvm.vector.splice.nxv16f16(<vscale x 16 x half> zeroinitializer, <vscale x 16 x half> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1f32_neg = call <vscale x 1 x float> @llvm.vector.splice.nxv1f32(<vscale x 1 x float> zeroinitializer, <vscale x 1 x float> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2f32_neg = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x float> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4f32_neg = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv8f32_neg = call <vscale x 8 x float> @llvm.vector.splice.nxv8f32(<vscale x 8 x float> zeroinitializer, <vscale x 8 x float> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1f64_neg = call <vscale x 1 x double> @llvm.vector.splice.nxv1f64(<vscale x 1 x double> zeroinitializer, <vscale x 1 x double> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2f64_neg = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv4f64_neg = call <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double> zeroinitializer, <vscale x 4 x double> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1bf16_neg = call <vscale x 1 x bfloat> @llvm.vector.splice.nxv1bf16(<vscale x 1 x bfloat> zeroinitializer, <vscale x 1 x bfloat> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2bf16_neg = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> zeroinitializer, <vscale x 2 x bfloat> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4bf16_neg = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv8bf16_neg = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv16bf16_neg = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv16i1_neg = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv8i1_neg = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv4i1_neg = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv2i1_neg = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1i1_neg = call <vscale x 1 x i1> @llvm.vector.splice.nxv1i1(<vscale x 1 x i1> zeroinitializer, <vscale x 1 x i1> zeroinitializer, i32 -1)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %splice_nxv32i8 = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv2i16 = call <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv4i16 = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv8i16 = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %splice_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %splice_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %splice_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv2f16 = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x half> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv4f16 = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x half> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv8f16 = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x half> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %splice_nxv16f16 = call <vscale x 16 x half> @llvm.vector.splice.nxv16f16(<vscale x 16 x half> zeroinitializer, <vscale x 16 x half> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv2f32 = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x float> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv4f32 = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %splice_nxv8f32 = call <vscale x 8 x float> @llvm.vector.splice.nxv8f32(<vscale x 8 x float> zeroinitializer, <vscale x 8 x float> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv2f64 = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %splice_nxv4f64 = call <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double> zeroinitializer, <vscale x 4 x double> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv2bf16 = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> zeroinitializer, <vscale x 2 x bfloat> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 2 for: %splice_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 4 for: %splice_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 4 for: %splice_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 4 for: %splice_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 4 for: %splice_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv16i8_neg = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv32i8_neg = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1i16_neg = call <vscale x 1 x i16> @llvm.vector.splice.nxv1i16(<vscale x 1 x i16> zeroinitializer, <vscale x 1 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv2i16_neg = call <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv4i16_neg = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv8i16_neg = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv16i16_neg = call <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv4i32_neg = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv8i32_neg = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1i64_neg = call <vscale x 1 x i64> @llvm.vector.splice.nxv1i64(<vscale x 1 x i64> zeroinitializer, <vscale x 1 x i64> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv2i64_neg = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv4i64_neg = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1f16_neg = call <vscale x 1 x half> @llvm.vector.splice.nxv1f16(<vscale x 1 x half> zeroinitializer, <vscale x 1 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv2f16_neg = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv4f16_neg = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv8f16_neg = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv16f16_neg = call <vscale x 16 x half> @llvm.vector.splice.nxv16f16(<vscale x 16 x half> zeroinitializer, <vscale x 16 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1f32_neg = call <vscale x 1 x float> @llvm.vector.splice.nxv1f32(<vscale x 1 x float> zeroinitializer, <vscale x 1 x float> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv2f32_neg = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x float> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv4f32_neg = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv8f32_neg = call <vscale x 8 x float> @llvm.vector.splice.nxv8f32(<vscale x 8 x float> zeroinitializer, <vscale x 8 x float> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1f64_neg = call <vscale x 1 x double> @llvm.vector.splice.nxv1f64(<vscale x 1 x double> zeroinitializer, <vscale x 1 x double> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv2f64_neg = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv4f64_neg = call <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double> zeroinitializer, <vscale x 4 x double> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1bf16_neg = call <vscale x 1 x bfloat> @llvm.vector.splice.nxv1bf16(<vscale x 1 x bfloat> zeroinitializer, <vscale x 1 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv2bf16_neg = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> zeroinitializer, <vscale x 2 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv4bf16_neg = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv8bf16_neg = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv16bf16_neg = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv16i1_neg = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv8i1_neg = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv4i1_neg = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv2i1_neg = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1i1_neg = call <vscale x 1 x i1> @llvm.vector.splice.nxv1i1(<vscale x 1 x i1> zeroinitializer, <vscale x 1 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'vector_splice'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv32i8 = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2i16 = call <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4i16 = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv8i16 = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2f16 = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x half> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4f16 = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x half> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv8f16 = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x half> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv16f16 = call <vscale x 16 x half> @llvm.vector.splice.nxv16f16(<vscale x 16 x half> zeroinitializer, <vscale x 16 x half> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2f32 = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x float> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4f32 = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv8f32 = call <vscale x 8 x float> @llvm.vector.splice.nxv8f32(<vscale x 8 x float> zeroinitializer, <vscale x 8 x float> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2f64 = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv4f64 = call <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double> zeroinitializer, <vscale x 4 x double> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv2bf16 = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> zeroinitializer, <vscale x 2 x bfloat> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %splice_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %splice_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %splice_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %splice_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %splice_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %splice_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv16i8_neg = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv32i8_neg = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1i16_neg = call <vscale x 1 x i16> @llvm.vector.splice.nxv1i16(<vscale x 1 x i16> zeroinitializer, <vscale x 1 x i16> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2i16_neg = call <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4i16_neg = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv8i16_neg = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv16i16_neg = call <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4i32_neg = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv8i32_neg = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1i64_neg = call <vscale x 1 x i64> @llvm.vector.splice.nxv1i64(<vscale x 1 x i64> zeroinitializer, <vscale x 1 x i64> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2i64_neg = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv4i64_neg = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1f16_neg = call <vscale x 1 x half> @llvm.vector.splice.nxv1f16(<vscale x 1 x half> zeroinitializer, <vscale x 1 x half> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2f16_neg = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x half> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4f16_neg = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x half> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv8f16_neg = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x half> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv16f16_neg = call <vscale x 16 x half> @llvm.vector.splice.nxv16f16(<vscale x 16 x half> zeroinitializer, <vscale x 16 x half> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1f32_neg = call <vscale x 1 x float> @llvm.vector.splice.nxv1f32(<vscale x 1 x float> zeroinitializer, <vscale x 1 x float> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2f32_neg = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x float> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4f32_neg = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv8f32_neg = call <vscale x 8 x float> @llvm.vector.splice.nxv8f32(<vscale x 8 x float> zeroinitializer, <vscale x 8 x float> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1f64_neg = call <vscale x 1 x double> @llvm.vector.splice.nxv1f64(<vscale x 1 x double> zeroinitializer, <vscale x 1 x double> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2f64_neg = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv4f64_neg = call <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double> zeroinitializer, <vscale x 4 x double> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1bf16_neg = call <vscale x 1 x bfloat> @llvm.vector.splice.nxv1bf16(<vscale x 1 x bfloat> zeroinitializer, <vscale x 1 x bfloat> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv2bf16_neg = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> zeroinitializer, <vscale x 2 x bfloat> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv4bf16_neg = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %splice_nxv8bf16_neg = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv16bf16_neg = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv16i1_neg = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv8i1_neg = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv4i1_neg = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %splice_nxv2i1_neg = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1i1_neg = call <vscale x 1 x i1> @llvm.vector.splice.nxv1i1(<vscale x 1 x i1> zeroinitializer, <vscale x 1 x i1> zeroinitializer, i32 -1)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %splice_nxv32i8 = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv2i16 = call <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv4i16 = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv8i16 = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %splice_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %splice_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %splice_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv2f16 = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x half> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv4f16 = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x half> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv8f16 = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x half> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %splice_nxv16f16 = call <vscale x 16 x half> @llvm.vector.splice.nxv16f16(<vscale x 16 x half> zeroinitializer, <vscale x 16 x half> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv2f32 = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x float> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv4f32 = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %splice_nxv8f32 = call <vscale x 8 x float> @llvm.vector.splice.nxv8f32(<vscale x 8 x float> zeroinitializer, <vscale x 8 x float> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv2f64 = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %splice_nxv4f64 = call <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double> zeroinitializer, <vscale x 4 x double> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv2bf16 = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> zeroinitializer, <vscale x 2 x bfloat> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %splice_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 2 for: %splice_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 4 for: %splice_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 4 for: %splice_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 4 for: %splice_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 4 for: %splice_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv16i8_neg = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv32i8_neg = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1i16_neg = call <vscale x 1 x i16> @llvm.vector.splice.nxv1i16(<vscale x 1 x i16> zeroinitializer, <vscale x 1 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv2i16_neg = call <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv4i16_neg = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv8i16_neg = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv16i16_neg = call <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv4i32_neg = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv8i32_neg = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1i64_neg = call <vscale x 1 x i64> @llvm.vector.splice.nxv1i64(<vscale x 1 x i64> zeroinitializer, <vscale x 1 x i64> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv2i64_neg = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv4i64_neg = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1f16_neg = call <vscale x 1 x half> @llvm.vector.splice.nxv1f16(<vscale x 1 x half> zeroinitializer, <vscale x 1 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv2f16_neg = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv4f16_neg = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv8f16_neg = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv16f16_neg = call <vscale x 16 x half> @llvm.vector.splice.nxv16f16(<vscale x 16 x half> zeroinitializer, <vscale x 16 x half> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1f32_neg = call <vscale x 1 x float> @llvm.vector.splice.nxv1f32(<vscale x 1 x float> zeroinitializer, <vscale x 1 x float> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv2f32_neg = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x float> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv4f32_neg = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv8f32_neg = call <vscale x 8 x float> @llvm.vector.splice.nxv8f32(<vscale x 8 x float> zeroinitializer, <vscale x 8 x float> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1f64_neg = call <vscale x 1 x double> @llvm.vector.splice.nxv1f64(<vscale x 1 x double> zeroinitializer, <vscale x 1 x double> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv2f64_neg = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv4f64_neg = call <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double> zeroinitializer, <vscale x 4 x double> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1bf16_neg = call <vscale x 1 x bfloat> @llvm.vector.splice.nxv1bf16(<vscale x 1 x bfloat> zeroinitializer, <vscale x 1 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv2bf16_neg = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> zeroinitializer, <vscale x 2 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv4bf16_neg = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 3 for: %splice_nxv8bf16_neg = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv16bf16_neg = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv16i1_neg = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv8i1_neg = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv4i1_neg = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 6 for: %splice_nxv2i1_neg = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1i1_neg = call <vscale x 1 x i1> @llvm.vector.splice.nxv1i1(<vscale x 1 x i1> zeroinitializer, <vscale x 1 x i1> zeroinitializer, i32 -1)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'vector_splice'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv32i8 = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv2i16 = call <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv4i16 = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv8i16 = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv2f16 = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x half> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv4f16 = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x half> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv8f16 = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x half> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv16f16 = call <vscale x 16 x half> @llvm.vector.splice.nxv16f16(<vscale x 16 x half> zeroinitializer, <vscale x 16 x half> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv2f32 = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x float> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv4f32 = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv8f32 = call <vscale x 8 x float> @llvm.vector.splice.nxv8f32(<vscale x 8 x float> zeroinitializer, <vscale x 8 x float> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv2f64 = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv4f64 = call <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double> zeroinitializer, <vscale x 4 x double> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv2bf16 = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> zeroinitializer, <vscale x 2 x bfloat> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv16i8_neg = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv32i8_neg = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1i16_neg = call <vscale x 1 x i16> @llvm.vector.splice.nxv1i16(<vscale x 1 x i16> zeroinitializer, <vscale x 1 x i16> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv2i16_neg = call <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv4i16_neg = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv8i16_neg = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv16i16_neg = call <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv4i32_neg = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv8i32_neg = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1i64_neg = call <vscale x 1 x i64> @llvm.vector.splice.nxv1i64(<vscale x 1 x i64> zeroinitializer, <vscale x 1 x i64> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv2i64_neg = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv4i64_neg = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1f16_neg = call <vscale x 1 x half> @llvm.vector.splice.nxv1f16(<vscale x 1 x half> zeroinitializer, <vscale x 1 x half> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv2f16_neg = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x half> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv4f16_neg = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x half> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv8f16_neg = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x half> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv16f16_neg = call <vscale x 16 x half> @llvm.vector.splice.nxv16f16(<vscale x 16 x half> zeroinitializer, <vscale x 16 x half> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1f32_neg = call <vscale x 1 x float> @llvm.vector.splice.nxv1f32(<vscale x 1 x float> zeroinitializer, <vscale x 1 x float> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv2f32_neg = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x float> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv4f32_neg = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv8f32_neg = call <vscale x 8 x float> @llvm.vector.splice.nxv8f32(<vscale x 8 x float> zeroinitializer, <vscale x 8 x float> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1f64_neg = call <vscale x 1 x double> @llvm.vector.splice.nxv1f64(<vscale x 1 x double> zeroinitializer, <vscale x 1 x double> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv2f64_neg = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv4f64_neg = call <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double> zeroinitializer, <vscale x 4 x double> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1bf16_neg = call <vscale x 1 x bfloat> @llvm.vector.splice.nxv1bf16(<vscale x 1 x bfloat> zeroinitializer, <vscale x 1 x bfloat> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv2bf16_neg = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> zeroinitializer, <vscale x 2 x bfloat> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv4bf16_neg = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv8bf16_neg = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv16bf16_neg = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv16i1_neg = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv8i1_neg = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv4i1_neg = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv2i1_neg = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %splice_nxv1i1_neg = call <vscale x 1 x i1> @llvm.vector.splice.nxv1i1(<vscale x 1 x i1> zeroinitializer, <vscale x 1 x i1> zeroinitializer, i32 -1)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv32i8 = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv2i16 = call <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv4i16 = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv8i16 = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv16i16 = call <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv4i32 = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv8i32 = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv2i64 = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv4i64 = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv2f16 = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x half> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv4f16 = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x half> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv8f16 = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x half> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv16f16 = call <vscale x 16 x half> @llvm.vector.splice.nxv16f16(<vscale x 16 x half> zeroinitializer, <vscale x 16 x half> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv2f32 = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x float> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv4f32 = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv8f32 = call <vscale x 8 x float> @llvm.vector.splice.nxv8f32(<vscale x 8 x float> zeroinitializer, <vscale x 8 x float> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv2f64 = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv4f64 = call <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double> zeroinitializer, <vscale x 4 x double> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv2bf16 = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> zeroinitializer, <vscale x 2 x bfloat> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv4bf16 = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv8bf16 = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv16bf16 = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv16i1 = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv8i1 = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv4i1 = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv2i1 = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv16i8_neg = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv32i8_neg = call <vscale x 32 x i8> @llvm.vector.splice.nxv32i8(<vscale x 32 x i8> zeroinitializer, <vscale x 32 x i8> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1i16_neg = call <vscale x 1 x i16> @llvm.vector.splice.nxv1i16(<vscale x 1 x i16> zeroinitializer, <vscale x 1 x i16> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv2i16_neg = call <vscale x 2 x i16> @llvm.vector.splice.nxv2i16(<vscale x 2 x i16> zeroinitializer, <vscale x 2 x i16> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv4i16_neg = call <vscale x 4 x i16> @llvm.vector.splice.nxv4i16(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv8i16_neg = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv16i16_neg = call <vscale x 16 x i16> @llvm.vector.splice.nxv16i16(<vscale x 16 x i16> zeroinitializer, <vscale x 16 x i16> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv4i32_neg = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv8i32_neg = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1i64_neg = call <vscale x 1 x i64> @llvm.vector.splice.nxv1i64(<vscale x 1 x i64> zeroinitializer, <vscale x 1 x i64> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv2i64_neg = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv4i64_neg = call <vscale x 4 x i64> @llvm.vector.splice.nxv4i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1f16_neg = call <vscale x 1 x half> @llvm.vector.splice.nxv1f16(<vscale x 1 x half> zeroinitializer, <vscale x 1 x half> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv2f16_neg = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x half> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv4f16_neg = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x half> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv8f16_neg = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x half> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv16f16_neg = call <vscale x 16 x half> @llvm.vector.splice.nxv16f16(<vscale x 16 x half> zeroinitializer, <vscale x 16 x half> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1f32_neg = call <vscale x 1 x float> @llvm.vector.splice.nxv1f32(<vscale x 1 x float> zeroinitializer, <vscale x 1 x float> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv2f32_neg = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x float> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv4f32_neg = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv8f32_neg = call <vscale x 8 x float> @llvm.vector.splice.nxv8f32(<vscale x 8 x float> zeroinitializer, <vscale x 8 x float> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1f64_neg = call <vscale x 1 x double> @llvm.vector.splice.nxv1f64(<vscale x 1 x double> zeroinitializer, <vscale x 1 x double> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv2f64_neg = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x double> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv4f64_neg = call <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double> zeroinitializer, <vscale x 4 x double> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1bf16_neg = call <vscale x 1 x bfloat> @llvm.vector.splice.nxv1bf16(<vscale x 1 x bfloat> zeroinitializer, <vscale x 1 x bfloat> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv2bf16_neg = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> zeroinitializer, <vscale x 2 x bfloat> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv4bf16_neg = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> zeroinitializer, <vscale x 4 x bfloat> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv8bf16_neg = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> zeroinitializer, <vscale x 8 x bfloat> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv16bf16_neg = call <vscale x 16 x bfloat> @llvm.vector.splice.nxv16bf16(<vscale x 16 x bfloat> zeroinitializer, <vscale x 16 x bfloat> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv16i1_neg = call <vscale x 16 x i1> @llvm.vector.splice.nxv16i1(<vscale x 16 x i1> zeroinitializer, <vscale x 16 x i1> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv8i1_neg = call <vscale x 8 x i1> @llvm.vector.splice.nxv8i1(<vscale x 8 x i1> zeroinitializer, <vscale x 8 x i1> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv4i1_neg = call <vscale x 4 x i1> @llvm.vector.splice.nxv4i1(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv2i1_neg = call <vscale x 2 x i1> @llvm.vector.splice.nxv2i1(<vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %splice_nxv1i1_neg = call <vscale x 1 x i1> @llvm.vector.splice.nxv1i1(<vscale x 1 x i1> zeroinitializer, <vscale x 1 x i1> zeroinitializer, i32 -1)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 
   %splice_nxv16i8 = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, i32 1)
@@ -912,73 +912,73 @@ declare <vscale x 4 x double> @llvm.vector.splice.nxv4f64(<vscale x 4 x double>,
 
 define void @get_lane_mask() #0 {
 ; CHECK-VSCALE-1-LABEL: 'get_lane_mask'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv16i1_i64 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv8i1_i64 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv4i1_i64 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv2i1_i64 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv16i1_i32 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 undef, i32 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv8i1_i32 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32 undef, i32 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv4i1_i32 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i32(i32 undef, i32 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv2i1_i32 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i32(i32 undef, i32 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %mask_nxv32i1_i64 = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %mask_nxv16i1_i16 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i16(i16 undef, i16 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %mask_v8i1_i64 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask_v4i1_i64 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %mask_v2i1_i64 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %mask_v16i1_i32 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 undef, i32 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv16i1_i64 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv8i1_i64 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv4i1_i64 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv2i1_i64 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv16i1_i32 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv8i1_i32 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv4i1_i32 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv2i1_i32 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:33 Lat:33 SizeLat:33 for: %mask_nxv32i1_i64 = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %mask_nxv16i1_i16 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i16(i16 undef, i16 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 32 for: %mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 16 for: %mask_v8i1_i64 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 8 for: %mask_v4i1_i64 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 4 for: %mask_v2i1_i64 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 32 for: %mask_v16i1_i32 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 16 for: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 8 for: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 4 for: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:33 Lat:33 SizeLat:33 for: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'get_lane_mask'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv16i1_i64 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv8i1_i64 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv4i1_i64 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv2i1_i64 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv16i1_i32 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 undef, i32 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv8i1_i32 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32 undef, i32 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv4i1_i32 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i32(i32 undef, i32 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask_nxv2i1_i32 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i32(i32 undef, i32 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %mask_nxv32i1_i64 = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %mask_nxv16i1_i16 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i16(i16 undef, i16 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %mask_v8i1_i64 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask_v4i1_i64 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %mask_v2i1_i64 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %mask_v16i1_i32 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 undef, i32 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv16i1_i64 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv8i1_i64 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv4i1_i64 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv2i1_i64 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv16i1_i32 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv8i1_i32 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv4i1_i32 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv2i1_i32 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:33 Lat:33 SizeLat:33 for: %mask_nxv32i1_i64 = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %mask_nxv16i1_i16 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i16(i16 undef, i16 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 32 for: %mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 16 for: %mask_v8i1_i64 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 8 for: %mask_v4i1_i64 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 4 for: %mask_v2i1_i64 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 32 for: %mask_v16i1_i32 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 16 for: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 8 for: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 4 for: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:33 Lat:33 SizeLat:33 for: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'get_lane_mask'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %mask_nxv16i1_i64 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 undef, i64 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %mask_nxv8i1_i64 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 undef, i64 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %mask_nxv4i1_i64 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 undef, i64 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %mask_nxv2i1_i64 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 undef, i64 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %mask_nxv16i1_i32 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 undef, i32 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %mask_nxv8i1_i32 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32 undef, i32 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %mask_nxv4i1_i32 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i32(i32 undef, i32 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %mask_nxv2i1_i32 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i32(i32 undef, i32 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %mask_nxv32i1_i64 = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 undef, i64 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %mask_nxv16i1_i16 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i16(i16 undef, i16 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %mask_v8i1_i64 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 undef, i64 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask_v4i1_i64 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 undef, i64 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %mask_v2i1_i64 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 undef, i64 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %mask_v16i1_i32 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 undef, i32 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv16i1_i64 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 undef, i64 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv8i1_i64 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 undef, i64 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv4i1_i64 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 undef, i64 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv2i1_i64 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 undef, i64 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv16i1_i32 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 undef, i32 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv8i1_i32 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i32(i32 undef, i32 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv4i1_i32 = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i32(i32 undef, i32 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 1 for: %mask_nxv2i1_i32 = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i32(i32 undef, i32 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:33 Lat:33 SizeLat:33 for: %mask_nxv32i1_i64 = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 undef, i64 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %mask_nxv16i1_i16 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i16(i16 undef, i16 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 32 for: %mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 16 for: %mask_v8i1_i64 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 undef, i64 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 8 for: %mask_v4i1_i64 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 undef, i64 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 4 for: %mask_v2i1_i64 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 undef, i64 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 32 for: %mask_v16i1_i32 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 undef, i32 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 16 for: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 8 for: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 4 for: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:33 Lat:33 SizeLat:33 for: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %mask_nxv16i1_i64 = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 undef, i64 undef)
   %mask_nxv8i1_i64 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 undef, i64 undef)
@@ -1011,25 +1011,25 @@ define void @get_lane_mask() #0 {
 
 define void @fshr() #0 {
 ; CHECK-VSCALE-1-LABEL: 'fshr'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %1 = call <vscale x 16 x i8> @llvm.fshr.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %2 = call <vscale x 8 x i16> @llvm.fshr.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %3 = call <vscale x 4 x i32> @llvm.fshr.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %4 = call <vscale x 2 x i64> @llvm.fshr.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 5 for: %1 = call <vscale x 16 x i8> @llvm.fshr.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 5 for: %2 = call <vscale x 8 x i16> @llvm.fshr.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 5 for: %3 = call <vscale x 4 x i32> @llvm.fshr.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 5 for: %4 = call <vscale x 2 x i64> @llvm.fshr.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'fshr'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %1 = call <vscale x 16 x i8> @llvm.fshr.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %2 = call <vscale x 8 x i16> @llvm.fshr.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %3 = call <vscale x 4 x i32> @llvm.fshr.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %4 = call <vscale x 2 x i64> @llvm.fshr.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 5 for: %1 = call <vscale x 16 x i8> @llvm.fshr.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 5 for: %2 = call <vscale x 8 x i16> @llvm.fshr.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 5 for: %3 = call <vscale x 4 x i32> @llvm.fshr.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 5 for: %4 = call <vscale x 2 x i64> @llvm.fshr.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'fshr'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %1 = call <vscale x 16 x i8> @llvm.fshr.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %2 = call <vscale x 8 x i16> @llvm.fshr.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %3 = call <vscale x 4 x i32> @llvm.fshr.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %4 = call <vscale x 2 x i64> @llvm.fshr.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 7 for: %1 = call <vscale x 16 x i8> @llvm.fshr.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 7 for: %2 = call <vscale x 8 x i16> @llvm.fshr.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 7 for: %3 = call <vscale x 4 x i32> @llvm.fshr.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 7 for: %4 = call <vscale x 2 x i64> @llvm.fshr.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   call <vscale x 16 x i8> @llvm.fshr.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
   call <vscale x 8 x i16> @llvm.fshr.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
@@ -1040,25 +1040,25 @@ define void @fshr() #0 {
 
 define void @fshl() #0 {
 ; CHECK-VSCALE-1-LABEL: 'fshl'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %1 = call <vscale x 16 x i8> @llvm.fshl.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %2 = call <vscale x 8 x i16> @llvm.fshl.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %3 = call <vscale x 4 x i32> @llvm.fshl.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %4 = call <vscale x 2 x i64> @llvm.fshl.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 5 for: %1 = call <vscale x 16 x i8> @llvm.fshl.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 5 for: %2 = call <vscale x 8 x i16> @llvm.fshl.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 5 for: %3 = call <vscale x 4 x i32> @llvm.fshl.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 5 for: %4 = call <vscale x 2 x i64> @llvm.fshl.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'fshl'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %1 = call <vscale x 16 x i8> @llvm.fshl.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %2 = call <vscale x 8 x i16> @llvm.fshl.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %3 = call <vscale x 4 x i32> @llvm.fshl.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %4 = call <vscale x 2 x i64> @llvm.fshl.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 5 for: %1 = call <vscale x 16 x i8> @llvm.fshl.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 5 for: %2 = call <vscale x 8 x i16> @llvm.fshl.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 5 for: %3 = call <vscale x 4 x i32> @llvm.fshl.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 5 for: %4 = call <vscale x 2 x i64> @llvm.fshl.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'fshl'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %1 = call <vscale x 16 x i8> @llvm.fshl.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %2 = call <vscale x 8 x i16> @llvm.fshl.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %3 = call <vscale x 4 x i32> @llvm.fshl.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %4 = call <vscale x 2 x i64> @llvm.fshl.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 7 for: %1 = call <vscale x 16 x i8> @llvm.fshl.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 7 for: %2 = call <vscale x 8 x i16> @llvm.fshl.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 7 for: %3 = call <vscale x 4 x i32> @llvm.fshl.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> undef, <vscale x 4 x i32> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 7 for: %4 = call <vscale x 2 x i64> @llvm.fshl.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i64> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   call <vscale x 16 x i8> @llvm.fshl.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
   call <vscale x 8 x i16> @llvm.fshl.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i16> undef, <vscale x 8 x i16> undef)
@@ -1069,16 +1069,16 @@ define void @fshl() #0 {
 
 define <vscale x 4 x i32> @masked_gather_nxv4i32(<vscale x 4 x ptr> %ld, <vscale x 4 x i1> %masks, <vscale x 4 x i32> %passthru) {
 ; CHECK-VSCALE-1-LABEL: 'masked_gather_nxv4i32'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %res = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> %ld, i32 0, <vscale x 4 x i1> %masks, <vscale x 4 x i32> %passthru)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <vscale x 4 x i32> %res
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 40 for: %res = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> %ld, i32 0, <vscale x 4 x i1> %masks, <vscale x 4 x i32> %passthru)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <vscale x 4 x i32> %res
 ;
 ; CHECK-VSCALE-2-LABEL: 'masked_gather_nxv4i32'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> %ld, i32 0, <vscale x 4 x i1> %masks, <vscale x 4 x i32> %passthru)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <vscale x 4 x i32> %res
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 80 for: %res = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> %ld, i32 0, <vscale x 4 x i1> %masks, <vscale x 4 x i32> %passthru)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <vscale x 4 x i32> %res
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_gather_nxv4i32'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %res = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> %ld, i32 0, <vscale x 4 x i1> %masks, <vscale x 4 x i32> %passthru)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <vscale x 4 x i32> %res
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %res = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> %ld, i32 0, <vscale x 4 x i1> %masks, <vscale x 4 x i32> %passthru)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <vscale x 4 x i32> %res
 ;
   %res = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x ptr> %ld, i32 0, <vscale x 4 x i1> %masks, <vscale x 4 x i32> %passthru)
   ret <vscale x 4 x i32> %res
@@ -1086,16 +1086,16 @@ define <vscale x 4 x i32> @masked_gather_nxv4i32(<vscale x 4 x ptr> %ld, <vscale
 
 define <vscale x 8 x i32> @masked_gather_nxv8i32(<vscale x 8 x ptr> %ld, <vscale x 8 x i1> %masks, <vscale x 8 x i32> %passthru) {
 ; CHECK-VSCALE-1-LABEL: 'masked_gather_nxv8i32'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %res = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> %ld, i32 0, <vscale x 8 x i1> %masks, <vscale x 8 x i32> %passthru)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <vscale x 8 x i32> %res
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 80 for: %res = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> %ld, i32 0, <vscale x 8 x i1> %masks, <vscale x 8 x i32> %passthru)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <vscale x 8 x i32> %res
 ;
 ; CHECK-VSCALE-2-LABEL: 'masked_gather_nxv8i32'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %res = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> %ld, i32 0, <vscale x 8 x i1> %masks, <vscale x 8 x i32> %passthru)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <vscale x 8 x i32> %res
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 160 for: %res = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> %ld, i32 0, <vscale x 8 x i1> %masks, <vscale x 8 x i32> %passthru)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <vscale x 8 x i32> %res
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_gather_nxv8i32'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: %res = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> %ld, i32 0, <vscale x 8 x i1> %masks, <vscale x 8 x i32> %passthru)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <vscale x 8 x i32> %res
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: %res = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32.nxv8p0(<vscale x 8 x ptr> %ld, i32 0, <vscale x 8 x i1> %masks, <vscale x 8 x i32> %passthru)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <vscale x 8 x i32> %res
 ;
   %res = call <vscale x 8 x i32> @llvm.masked.gather.nxv8i32(<vscale x 8 x ptr> %ld, i32 0, <vscale x 8 x i1> %masks, <vscale x 8 x i32> %passthru)
   ret <vscale x 8 x i32> %res
@@ -1103,16 +1103,16 @@ define <vscale x 8 x i32> @masked_gather_nxv8i32(<vscale x 8 x ptr> %ld, <vscale
 
 define <4 x i32> @masked_gather_v4i32(<4 x ptr> %ld, <4 x i1> %masks, <4 x i32> %passthru) {
 ; CHECK-VSCALE-1-LABEL: 'masked_gather_v4i32'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ld, i32 0, <4 x i1> %masks, <4 x i32> %passthru)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:32 Lat:32 SizeLat:32 for: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ld, i32 0, <4 x i1> %masks, <4 x i32> %passthru)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %res
 ;
 ; CHECK-VSCALE-2-LABEL: 'masked_gather_v4i32'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ld, i32 0, <4 x i1> %masks, <4 x i32> %passthru)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:32 Lat:32 SizeLat:32 for: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ld, i32 0, <4 x i1> %masks, <4 x i32> %passthru)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %res
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_gather_v4i32'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ld, i32 0, <4 x i1> %masks, <4 x i32> %passthru)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 36 for: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ld, i32 0, <4 x i1> %masks, <4 x i32> %passthru)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %res
 ;
   %res = call <4 x i32> @llvm.masked.gather.v4i32(<4 x ptr> %ld, i32 0, <4 x i1> %masks, <4 x i32> %passthru)
   ret <4 x i32> %res
@@ -1120,16 +1120,16 @@ define <4 x i32> @masked_gather_v4i32(<4 x ptr> %ld, <4 x i1> %masks, <4 x i32>
 
 define <1 x i128> @masked_gather_v1i128(<1 x ptr> %ld, <1 x i1> %masks, <1 x i128> %passthru) {
 ; CHECK-VSCALE-1-LABEL: 'masked_gather_v1i128'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0(<1 x ptr> %ld, i32 0, <1 x i1> %masks, <1 x i128> %passthru)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <1 x i128> %res
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:9 for: %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0(<1 x ptr> %ld, i32 0, <1 x i1> %masks, <1 x i128> %passthru)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <1 x i128> %res
 ;
 ; CHECK-VSCALE-2-LABEL: 'masked_gather_v1i128'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0(<1 x ptr> %ld, i32 0, <1 x i1> %masks, <1 x i128> %passthru)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <1 x i128> %res
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:9 for: %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0(<1 x ptr> %ld, i32 0, <1 x i1> %masks, <1 x i128> %passthru)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <1 x i128> %res
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_gather_v1i128'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0(<1 x ptr> %ld, i32 0, <1 x i1> %masks, <1 x i128> %passthru)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <1 x i128> %res
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 1 for: %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0(<1 x ptr> %ld, i32 0, <1 x i1> %masks, <1 x i128> %passthru)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <1 x i128> %res
 ;
   %res = call <1 x i128> @llvm.masked.gather.v1i128.v1p0(<1 x ptr> %ld, i32 0, <1 x i1> %masks, <1 x i128> %passthru)
   ret <1 x i128> %res
@@ -1137,16 +1137,16 @@ define <1 x i128> @masked_gather_v1i128(<1 x ptr> %ld, <1 x i1> %masks, <1 x i12
 
 define void @masked_scatter_nxv4i32(<vscale x 4 x i32> %data, <vscale x 4 x ptr> %ptrs, <vscale x 4 x i1> %masks) {
 ; CHECK-VSCALE-1-LABEL: 'masked_scatter_nxv4i32'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> %data, <vscale x 4 x ptr> %ptrs, i32 0, <vscale x 4 x i1> %masks)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 40 for: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> %data, <vscale x 4 x ptr> %ptrs, i32 0, <vscale x 4 x i1> %masks)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'masked_scatter_nxv4i32'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> %data, <vscale x 4 x ptr> %ptrs, i32 0, <vscale x 4 x i1> %masks)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 80 for: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> %data, <vscale x 4 x ptr> %ptrs, i32 0, <vscale x 4 x i1> %masks)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_scatter_nxv4i32'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> %data, <vscale x 4 x ptr> %ptrs, i32 0, <vscale x 4 x i1> %masks)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> %data, <vscale x 4 x ptr> %ptrs, i32 0, <vscale x 4 x i1> %masks)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 
   call void @llvm.masked.scatter.nxv4i32(<vscale x 4 x i32> %data, <vscale x 4 x ptr> %ptrs, i32 0, <vscale x 4 x i1> %masks)
@@ -1155,16 +1155,16 @@ define void @masked_scatter_nxv4i32(<vscale x 4 x i32> %data, <vscale x 4 x ptr>
 
 define void @masked_scatter_nxv8i32(<vscale x 8 x i32> %data, <vscale x 8 x ptr> %ptrs, <vscale x 8 x i1> %masks) {
 ; CHECK-VSCALE-1-LABEL: 'masked_scatter_nxv8i32'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> %data, <vscale x 8 x ptr> %ptrs, i32 0, <vscale x 8 x i1> %masks)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 80 for: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> %data, <vscale x 8 x ptr> %ptrs, i32 0, <vscale x 8 x i1> %masks)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'masked_scatter_nxv8i32'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> %data, <vscale x 8 x ptr> %ptrs, i32 0, <vscale x 8 x i1> %masks)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 160 for: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> %data, <vscale x 8 x ptr> %ptrs, i32 0, <vscale x 8 x i1> %masks)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_scatter_nxv8i32'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> %data, <vscale x 8 x ptr> %ptrs, i32 0, <vscale x 8 x i1> %masks)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.masked.scatter.nxv8i32.nxv8p0(<vscale x 8 x i32> %data, <vscale x 8 x ptr> %ptrs, i32 0, <vscale x 8 x i1> %masks)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 
   call void @llvm.masked.scatter.nxv8i32(<vscale x 8 x i32> %data, <vscale x 8 x ptr> %ptrs, i32 0, <vscale x 8 x i1> %masks)
@@ -1173,16 +1173,16 @@ define void @masked_scatter_nxv8i32(<vscale x 8 x i32> %data, <vscale x 8 x ptr>
 
 define void @masked_scatter_v4i32(<4 x i32> %data, <4 x ptr> %ptrs, <4 x i1> %masks) {
 ; CHECK-VSCALE-1-LABEL: 'masked_scatter_v4i32'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %data, <4 x ptr> %ptrs, i32 0, <4 x i1> %masks)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:32 Lat:32 SizeLat:32 for: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %data, <4 x ptr> %ptrs, i32 0, <4 x i1> %masks)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'masked_scatter_v4i32'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %data, <4 x ptr> %ptrs, i32 0, <4 x i1> %masks)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:32 Lat:32 SizeLat:32 for: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %data, <4 x ptr> %ptrs, i32 0, <4 x i1> %masks)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_scatter_v4i32'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %data, <4 x ptr> %ptrs, i32 0, <4 x i1> %masks)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 28 for: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %data, <4 x ptr> %ptrs, i32 0, <4 x i1> %masks)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 
   call void @llvm.masked.scatter.v4i32(<4 x i32> %data, <4 x ptr> %ptrs, i32 0, <4 x i1> %masks)
@@ -1191,16 +1191,16 @@ define void @masked_scatter_v4i32(<4 x i32> %data, <4 x ptr> %ptrs, <4 x i1> %ma
 
 define void @masked_scatter_v1i128(<1 x i128> %data, <1 x ptr> %ptrs, <1 x i1> %masks) {
 ; CHECK-VSCALE-1-LABEL: 'masked_scatter_v1i128'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v1i128.v1p0(<1 x i128> %data, <1 x ptr> %ptrs, i32 0, <1 x i1> %masks)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:9 for: call void @llvm.masked.scatter.v1i128.v1p0(<1 x i128> %data, <1 x ptr> %ptrs, i32 0, <1 x i1> %masks)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'masked_scatter_v1i128'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.scatter.v1i128.v1p0(<1 x i128> %data, <1 x ptr> %ptrs, i32 0, <1 x i1> %masks)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:9 Lat:8 SizeLat:9 for: call void @llvm.masked.scatter.v1i128.v1p0(<1 x i128> %data, <1 x ptr> %ptrs, i32 0, <1 x i1> %masks)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'masked_scatter_v1i128'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v1i128.v1p0(<1 x i128> %data, <1 x ptr> %ptrs, i32 0, <1 x i1> %masks)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 1 for: call void @llvm.masked.scatter.v1i128.v1p0(<1 x i128> %data, <1 x ptr> %ptrs, i32 0, <1 x i1> %masks)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 
   call void @llvm.masked.scatter.v1i128.v1p0(<1 x i128> %data, <1 x ptr> %ptrs, i32 0, <1 x i1> %masks)
@@ -1209,16 +1209,16 @@ define void @masked_scatter_v1i128(<1 x i128> %data, <1 x ptr> %ptrs, <1 x i1> %
 
 define void @histogram_nxv2i64(<vscale x 2 x ptr> %buckets, <vscale x 2 x i1> %mask) #3 {
 ; CHECK-VSCALE-1-LABEL: 'histogram_nxv2i64'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> %buckets, i64 1, <vscale x 2 x i1> %mask)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 8 for: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> %buckets, i64 1, <vscale x 2 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'histogram_nxv2i64'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> %buckets, i64 1, <vscale x 2 x i1> %mask)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 8 for: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> %buckets, i64 1, <vscale x 2 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_nxv2i64'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> %buckets, i64 1, <vscale x 2 x i1> %mask)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 8 for: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> %buckets, i64 1, <vscale x 2 x i1> %mask)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> %buckets, i64 1, <vscale x 2 x i1> %mask)
   ret void
@@ -1226,16 +1226,16 @@ define void @histogram_nxv2i64(<vscale x 2 x ptr> %buckets, <vscale x 2 x i1> %m
 
 define void @histogram_nxv4i32(<vscale x 4 x ptr> %buckets, <vscale x 4 x i1> %mask) #3 {
 ; CHECK-VSCALE-1-LABEL: 'histogram_nxv4i32'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 8 for: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'histogram_nxv4i32'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 8 for: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_nxv4i32'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 8 for: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> %buckets, i32 1, <vscale x 4 x i1> %mask)
   ret void
@@ -1243,16 +1243,16 @@ define void @histogram_nxv4i32(<vscale x 4 x ptr> %buckets, <vscale x 4 x i1> %m
 
 define void @histogram_nxv8i16(<vscale x 8 x ptr> %buckets, <vscale x 8 x i1> %mask) #3 {
 ; CHECK-VSCALE-1-LABEL: 'histogram_nxv8i16'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16(<vscale x 8 x ptr> %buckets, i16 1, <vscale x 8 x i1> %mask)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16(<vscale x 8 x ptr> %buckets, i16 1, <vscale x 8 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'histogram_nxv8i16'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16(<vscale x 8 x ptr> %buckets, i16 1, <vscale x 8 x i1> %mask)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16(<vscale x 8 x ptr> %buckets, i16 1, <vscale x 8 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_nxv8i16'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16(<vscale x 8 x ptr> %buckets, i16 1, <vscale x 8 x i1> %mask)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16(<vscale x 8 x ptr> %buckets, i16 1, <vscale x 8 x i1> %mask)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   call void @llvm.experimental.vector.histogram.add.nxv8p0.i16(<vscale x 8 x ptr> %buckets, i16 1, <vscale x 8 x i1> %mask)
   ret void
@@ -1260,16 +1260,16 @@ define void @histogram_nxv8i16(<vscale x 8 x ptr> %buckets, <vscale x 8 x i1> %m
 
 define void @histogram_nxv16i8(<vscale x 16 x ptr> %buckets, <vscale x 16 x i1> %mask) #3 {
 ; CHECK-VSCALE-1-LABEL: 'histogram_nxv16i8'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8(<vscale x 16 x ptr> %buckets, i8 1, <vscale x 16 x i1> %mask)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 32 for: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8(<vscale x 16 x ptr> %buckets, i8 1, <vscale x 16 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'histogram_nxv16i8'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8(<vscale x 16 x ptr> %buckets, i8 1, <vscale x 16 x i1> %mask)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 32 for: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8(<vscale x 16 x ptr> %buckets, i8 1, <vscale x 16 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_nxv16i8'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8(<vscale x 16 x ptr> %buckets, i8 1, <vscale x 16 x i1> %mask)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 32 for: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8(<vscale x 16 x ptr> %buckets, i8 1, <vscale x 16 x i1> %mask)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   call void @llvm.experimental.vector.histogram.add.nxv16p0.i64(<vscale x 16 x ptr> %buckets, i8 1, <vscale x 16 x i1> %mask)
   ret void
@@ -1277,16 +1277,16 @@ define void @histogram_nxv16i8(<vscale x 16 x ptr> %buckets, <vscale x 16 x i1>
 
 define void @histogram_v2i64(<2 x ptr> %buckets, <2 x i1> %mask) {
 ; CHECK-VSCALE-1-LABEL: 'histogram_v2i64'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'histogram_v2i64'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_v2i64'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
   ret void
@@ -1294,16 +1294,16 @@ define void @histogram_v2i64(<2 x ptr> %buckets, <2 x i1> %mask) {
 
 define void @histogram_v4i32(<4 x ptr> %buckets, <4 x i1> %mask) {
 ; CHECK-VSCALE-1-LABEL: 'histogram_v4i32'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'histogram_v4i32'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_v4i32'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
   ret void
@@ -1311,16 +1311,16 @@ define void @histogram_v4i32(<4 x ptr> %buckets, <4 x i1> %mask) {
 
 define void @histogram_v8i16(<8 x ptr> %buckets, <8 x i1> %mask) {
 ; CHECK-VSCALE-1-LABEL: 'histogram_v8i16'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'histogram_v8i16'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_v8i16'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
   ret void
@@ -1328,16 +1328,16 @@ define void @histogram_v8i16(<8 x ptr> %buckets, <8 x i1> %mask) {
 
 define void @histogram_v16i8(<16 x ptr> %buckets, <16 x i1> %mask) {
 ; CHECK-VSCALE-1-LABEL: 'histogram_v16i8'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'histogram_v16i8'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_v16i8'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   call void @llvm.experimental.vector.histogram.add.v16p0.i64(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
   ret void
@@ -1345,16 +1345,16 @@ define void @histogram_v16i8(<16 x ptr> %buckets, <16 x i1> %mask) {
 
 define void @histogram_nxv4i64(<vscale x 4 x ptr> %buckets, <vscale x 4 x i1> %mask) #3 {
 ; CHECK-VSCALE-1-LABEL: 'histogram_nxv4i64'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i64(<vscale x 4 x ptr> %buckets, i64 1, <vscale x 4 x i1> %mask)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.experimental.vector.histogram.add.nxv4p0.i64(<vscale x 4 x ptr> %buckets, i64 1, <vscale x 4 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'histogram_nxv4i64'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i64(<vscale x 4 x ptr> %buckets, i64 1, <vscale x 4 x i1> %mask)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.experimental.vector.histogram.add.nxv4p0.i64(<vscale x 4 x ptr> %buckets, i64 1, <vscale x 4 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_nxv4i64'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i64(<vscale x 4 x ptr> %buckets, i64 1, <vscale x 4 x i1> %mask)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 16 for: call void @llvm.experimental.vector.histogram.add.nxv4p0.i64(<vscale x 4 x ptr> %buckets, i64 1, <vscale x 4 x i1> %mask)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   call void @llvm.experimental.vector.histogram.add.nxv4p0.i64(<vscale x 4 x ptr> %buckets, i64 1, <vscale x 4 x i1> %mask)
   ret void
@@ -1362,37 +1362,37 @@ define void @histogram_nxv4i64(<vscale x 4 x ptr> %buckets, <vscale x 4 x i1> %m
 
 define void @match() #3 {
 ; CHECK-VSCALE-1-LABEL: 'match'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %match_nxv16i8_v16i8 = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> undef, <16 x i8> undef, <vscale x 16 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %match_nxv8i16_v8i16 = call <vscale x 8 x i1> @llvm.experimental.vector.match.nxv8i16.v8i16(<vscale x 8 x i16> undef, <8 x i16> undef, <vscale x 8 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %match_nxv4i32_v4i32 = call <vscale x 4 x i1> @llvm.experimental.vector.match.nxv4i32.v4i32(<vscale x 4 x i32> undef, <4 x i32> undef, <vscale x 4 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %match_nxv2i64_v2i64 = call <vscale x 2 x i1> @llvm.experimental.vector.match.nxv2i64.v2i64(<vscale x 2 x i64> undef, <2 x i64> undef, <vscale x 2 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %match_v16i8_v16i8 = call <16 x i1> @llvm.experimental.vector.match.v16i8.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %match_v8i16_v8i16 = call <8 x i1> @llvm.experimental.vector.match.v8i16.v8i16(<8 x i16> undef, <8 x i16> undef, <8 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %match_v4i32_v4i32 = call <4 x i1> @llvm.experimental.vector.match.v4i32.v4i32(<4 x i32> undef, <4 x i32> undef, <4 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %match_v2i64_v2i64 = call <2 x i1> @llvm.experimental.vector.match.v2i64.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i1> undef)
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 4 for: %match_nxv16i8_v16i8 = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> undef, <16 x i8> undef, <vscale x 16 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 4 for: %match_nxv8i16_v8i16 = call <vscale x 8 x i1> @llvm.experimental.vector.match.nxv8i16.v8i16(<vscale x 8 x i16> undef, <8 x i16> undef, <vscale x 8 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 21 for: %match_nxv4i32_v4i32 = call <vscale x 4 x i1> @llvm.experimental.vector.match.nxv4i32.v4i32(<vscale x 4 x i32> undef, <4 x i32> undef, <vscale x 4 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 11 for: %match_nxv2i64_v2i64 = call <vscale x 2 x i1> @llvm.experimental.vector.match.nxv2i64.v2i64(<vscale x 2 x i64> undef, <2 x i64> undef, <vscale x 2 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 14 for: %match_v16i8_v16i8 = call <16 x i1> @llvm.experimental.vector.match.v16i8.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 14 for: %match_v8i16_v8i16 = call <8 x i1> @llvm.experimental.vector.match.v8i16.v8i16(<8 x i16> undef, <8 x i16> undef, <8 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 21 for: %match_v4i32_v4i32 = call <4 x i1> @llvm.experimental.vector.match.v4i32.v4i32(<4 x i32> undef, <4 x i32> undef, <4 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of 11 for: %match_v2i64_v2i64 = call <2 x i1> @llvm.experimental.vector.match.v2i64.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i1> undef)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'match'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %match_nxv16i8_v16i8 = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> undef, <16 x i8> undef, <vscale x 16 x i1> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %match_nxv8i16_v8i16 = call <vscale x 8 x i1> @llvm.experimental.vector.match.nxv8i16.v8i16(<vscale x 8 x i16> undef, <8 x i16> undef, <vscale x 8 x i1> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %match_nxv4i32_v4i32 = call <vscale x 4 x i1> @llvm.experimental.vector.match.nxv4i32.v4i32(<vscale x 4 x i32> undef, <4 x i32> undef, <vscale x 4 x i1> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %match_nxv2i64_v2i64 = call <vscale x 2 x i1> @llvm.experimental.vector.match.nxv2i64.v2i64(<vscale x 2 x i64> undef, <2 x i64> undef, <vscale x 2 x i1> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %match_v16i8_v16i8 = call <16 x i1> @llvm.experimental.vector.match.v16i8.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i1> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %match_v8i16_v8i16 = call <8 x i1> @llvm.experimental.vector.match.v8i16.v8i16(<8 x i16> undef, <8 x i16> undef, <8 x i1> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %match_v4i32_v4i32 = call <4 x i1> @llvm.experimental.vector.match.v4i32.v4i32(<4 x i32> undef, <4 x i32> undef, <4 x i1> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %match_v2i64_v2i64 = call <2 x i1> @llvm.experimental.vector.match.v2i64.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i1> undef)
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 4 for: %match_nxv16i8_v16i8 = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> undef, <16 x i8> undef, <vscale x 16 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 4 for: %match_nxv8i16_v8i16 = call <vscale x 8 x i1> @llvm.experimental.vector.match.nxv8i16.v8i16(<vscale x 8 x i16> undef, <8 x i16> undef, <vscale x 8 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 21 for: %match_nxv4i32_v4i32 = call <vscale x 4 x i1> @llvm.experimental.vector.match.nxv4i32.v4i32(<vscale x 4 x i32> undef, <4 x i32> undef, <vscale x 4 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 11 for: %match_nxv2i64_v2i64 = call <vscale x 2 x i1> @llvm.experimental.vector.match.nxv2i64.v2i64(<vscale x 2 x i64> undef, <2 x i64> undef, <vscale x 2 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 14 for: %match_v16i8_v16i8 = call <16 x i1> @llvm.experimental.vector.match.v16i8.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 14 for: %match_v8i16_v8i16 = call <8 x i1> @llvm.experimental.vector.match.v8i16.v8i16(<8 x i16> undef, <8 x i16> undef, <8 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 21 for: %match_v4i32_v4i32 = call <4 x i1> @llvm.experimental.vector.match.v4i32.v4i32(<4 x i32> undef, <4 x i32> undef, <4 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of 11 for: %match_v2i64_v2i64 = call <2 x i1> @llvm.experimental.vector.match.v2i64.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i1> undef)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'match'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %match_nxv16i8_v16i8 = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> undef, <16 x i8> undef, <vscale x 16 x i1> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %match_nxv8i16_v8i16 = call <vscale x 8 x i1> @llvm.experimental.vector.match.nxv8i16.v8i16(<vscale x 8 x i16> undef, <8 x i16> undef, <vscale x 8 x i1> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %match_nxv4i32_v4i32 = call <vscale x 4 x i1> @llvm.experimental.vector.match.nxv4i32.v4i32(<vscale x 4 x i32> undef, <4 x i32> undef, <vscale x 4 x i1> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %match_nxv2i64_v2i64 = call <vscale x 2 x i1> @llvm.experimental.vector.match.nxv2i64.v2i64(<vscale x 2 x i64> undef, <2 x i64> undef, <vscale x 2 x i1> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %match_v16i8_v16i8 = call <16 x i1> @llvm.experimental.vector.match.v16i8.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i1> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %match_v8i16_v8i16 = call <8 x i1> @llvm.experimental.vector.match.v8i16.v8i16(<8 x i16> undef, <8 x i16> undef, <8 x i1> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %match_v4i32_v4i32 = call <4 x i1> @llvm.experimental.vector.match.v4i32.v4i32(<4 x i32> undef, <4 x i32> undef, <4 x i1> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %match_v2i64_v2i64 = call <2 x i1> @llvm.experimental.vector.match.v2i64.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i1> undef)
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 4 for: %match_nxv16i8_v16i8 = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> undef, <16 x i8> undef, <vscale x 16 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 4 for: %match_nxv8i16_v8i16 = call <vscale x 8 x i1> @llvm.experimental.vector.match.nxv8i16.v8i16(<vscale x 8 x i16> undef, <8 x i16> undef, <vscale x 8 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 21 for: %match_nxv4i32_v4i32 = call <vscale x 4 x i1> @llvm.experimental.vector.match.nxv4i32.v4i32(<vscale x 4 x i32> undef, <4 x i32> undef, <vscale x 4 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 11 for: %match_nxv2i64_v2i64 = call <vscale x 2 x i1> @llvm.experimental.vector.match.nxv2i64.v2i64(<vscale x 2 x i64> undef, <2 x i64> undef, <vscale x 2 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 14 for: %match_v16i8_v16i8 = call <16 x i1> @llvm.experimental.vector.match.v16i8.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 14 for: %match_v8i16_v8i16 = call <8 x i1> @llvm.experimental.vector.match.v8i16.v8i16(<8 x i16> undef, <8 x i16> undef, <8 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 21 for: %match_v4i32_v4i32 = call <4 x i1> @llvm.experimental.vector.match.v4i32.v4i32(<4 x i32> undef, <4 x i32> undef, <4 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of 11 for: %match_v2i64_v2i64 = call <2 x i1> @llvm.experimental.vector.match.v2i64.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i1> undef)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 
   %match_nxv16i8_v16i8 = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> undef, <16 x i8> undef, <vscale x 16 x i1> undef)
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-rem.ll b/llvm/test/Analysis/CostModel/AArch64/sve-rem.ll
index d18ed3296585b..ed88f1b390b68 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-rem.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-rem.ll
@@ -163,42 +163,42 @@ define void @urem() {
 
 define void @srem_uniformconst() {
 ; CHECK-LABEL: 'srem_uniformconst'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i64 = srem <2 x i64> undef, splat (i64 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4i64 = srem <4 x i64> undef, splat (i64 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8i64 = srem <8 x i64> undef, splat (i64 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = srem <2 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4i32 = srem <4 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8i32 = srem <8 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16i32 = srem <16 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2i16 = srem <2 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4i16 = srem <4 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i16 = srem <8 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16i16 = srem <16 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V32i16 = srem <32 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2i8 = srem <2 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4i8 = srem <4 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8i8 = srem <8 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16i8 = srem <16 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V32i8 = srem <32 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V64i8 = srem <64 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV2i64 = srem <vscale x 2 x i64> undef, splat (i64 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV4i64 = srem <vscale x 4 x i64> undef, splat (i64 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NV8i64 = srem <vscale x 8 x i64> undef, splat (i64 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV2i32 = srem <vscale x 2 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV4i32 = srem <vscale x 4 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV8i32 = srem <vscale x 8 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NV16i32 = srem <vscale x 16 x i32> undef, splat (i32 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV2i16 = srem <vscale x 2 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV4i16 = srem <vscale x 4 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV8i16 = srem <vscale x 8 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV16i16 = srem <vscale x 16 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %NV32i16 = srem <vscale x 32 x i16> undef, splat (i16 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV2i8 = srem <vscale x 2 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV4i8 = srem <vscale x 4 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV8i8 = srem <vscale x 8 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %NV16i8 = srem <vscale x 16 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %NV32i8 = srem <vscale x 32 x i8> undef, splat (i8 7)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %NV64i8 = srem <vscale x 64 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i64 = srem <2 x i64> undef, splat (i64 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4i64 = srem <4 x i64> undef, splat (i64 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8i64 = srem <8 x i64> undef, splat (i64 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i32 = srem <2 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i32 = srem <4 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8i32 = srem <8 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16i32 = srem <16 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i16 = srem <2 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i16 = srem <4 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i16 = srem <8 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16i16 = srem <16 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32i16 = srem <32 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i8 = srem <2 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i8 = srem <4 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i8 = srem <8 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i8 = srem <16 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32i8 = srem <32 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64i8 = srem <64 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i64 = srem <vscale x 2 x i64> undef, splat (i64 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV4i64 = srem <vscale x 4 x i64> undef, splat (i64 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV8i64 = srem <vscale x 8 x i64> undef, splat (i64 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i32 = srem <vscale x 2 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV4i32 = srem <vscale x 4 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV8i32 = srem <vscale x 8 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV16i32 = srem <vscale x 16 x i32> undef, splat (i32 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i16 = srem <vscale x 2 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV4i16 = srem <vscale x 4 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV8i16 = srem <vscale x 8 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV16i16 = srem <vscale x 16 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV32i16 = srem <vscale x 32 x i16> undef, splat (i16 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i8 = srem <vscale x 2 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV4i8 = srem <vscale x 4 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV8i8 = srem <vscale x 8 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV16i8 = srem <vscale x 16 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV32i8 = srem <vscale x 32 x i8> undef, splat (i8 7)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV64i8 = srem <vscale x 64 x i8> undef, splat (i8 7)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V2i64 = srem <2 x i64> undef, splat (i64 7)
@@ -321,42 +321,42 @@ define void @urem_uniformconst() {
 
 define void @srem_uniformconstpow2() {
 ; CHECK-LABEL: 'srem_uniformconstpow2'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V2i64 = srem <2 x i64> undef, splat (i64 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V4i64 = srem <4 x i64> undef, splat (i64 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V8i64 = srem <8 x i64> undef, splat (i64 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V2i32 = srem <2 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4i32 = srem <4 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V8i32 = srem <8 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 116 for instruction: %V16i32 = srem <16 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V2i16 = srem <2 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4i16 = srem <4 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V8i16 = srem <8 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 106 for instruction: %V16i16 = srem <16 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 212 for instruction: %V32i16 = srem <32 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V2i8 = srem <2 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4i8 = srem <4 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V8i8 = srem <8 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 101 for instruction: %V16i8 = srem <16 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 202 for instruction: %V32i8 = srem <32 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 404 for instruction: %V64i8 = srem <64 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %NV2i64 = srem <vscale x 2 x i64> undef, splat (i64 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %NV4i64 = srem <vscale x 4 x i64> undef, splat (i64 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %NV8i64 = srem <vscale x 8 x i64> undef, splat (i64 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %NV2i32 = srem <vscale x 2 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %NV4i32 = srem <vscale x 4 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %NV8i32 = srem <vscale x 8 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %NV16i32 = srem <vscale x 16 x i32> undef, splat (i32 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %NV2i16 = srem <vscale x 2 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %NV4i16 = srem <vscale x 4 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %NV8i16 = srem <vscale x 8 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %NV16i16 = srem <vscale x 16 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %NV32i16 = srem <vscale x 32 x i16> undef, splat (i16 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %NV2i8 = srem <vscale x 2 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %NV4i8 = srem <vscale x 4 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %NV8i8 = srem <vscale x 8 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %NV16i8 = srem <vscale x 16 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %NV32i8 = srem <vscale x 32 x i8> undef, splat (i8 16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %NV64i8 = srem <vscale x 64 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i64 = srem <2 x i64> undef, splat (i64 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4i64 = srem <4 x i64> undef, splat (i64 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8i64 = srem <8 x i64> undef, splat (i64 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i32 = srem <2 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i32 = srem <4 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8i32 = srem <8 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16i32 = srem <16 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i16 = srem <2 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i16 = srem <4 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i16 = srem <8 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16i16 = srem <16 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32i16 = srem <32 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i8 = srem <2 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i8 = srem <4 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i8 = srem <8 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i8 = srem <16 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32i8 = srem <32 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64i8 = srem <64 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV2i64 = srem <vscale x 2 x i64> undef, splat (i64 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV4i64 = srem <vscale x 4 x i64> undef, splat (i64 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV8i64 = srem <vscale x 8 x i64> undef, splat (i64 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV2i32 = srem <vscale x 2 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV4i32 = srem <vscale x 4 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV8i32 = srem <vscale x 8 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV16i32 = srem <vscale x 16 x i32> undef, splat (i32 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV2i16 = srem <vscale x 2 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV4i16 = srem <vscale x 4 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV8i16 = srem <vscale x 8 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV16i16 = srem <vscale x 16 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV32i16 = srem <vscale x 32 x i16> undef, splat (i16 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV2i8 = srem <vscale x 2 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV4i8 = srem <vscale x 4 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV8i8 = srem <vscale x 8 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %NV16i8 = srem <vscale x 16 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV32i8 = srem <vscale x 32 x i8> undef, splat (i8 16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV64i8 = srem <vscale x 64 x i8> undef, splat (i8 16)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V2i64 = srem <2 x i64> undef, splat (i64 16)
@@ -479,42 +479,42 @@ define void @urem_uniformconstpow2() {
 
 define void @srem_uniformconstnegpow2() {
 ; CHECK-LABEL: 'srem_uniformconstnegpow2'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i64 = srem <2 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4i64 = srem <4 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8i64 = srem <8 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = srem <2 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4i32 = srem <4 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8i32 = srem <8 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16i32 = srem <16 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2i16 = srem <2 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4i16 = srem <4 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i16 = srem <8 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16i16 = srem <16 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V32i16 = srem <32 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2i8 = srem <2 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4i8 = srem <4 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8i8 = srem <8 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16i8 = srem <16 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V32i8 = srem <32 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V64i8 = srem <64 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV2i64 = srem <vscale x 2 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV4i64 = srem <vscale x 4 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NV8i64 = srem <vscale x 8 x i64> undef, splat (i64 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV2i32 = srem <vscale x 2 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV4i32 = srem <vscale x 4 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %NV8i32 = srem <vscale x 8 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %NV16i32 = srem <vscale x 16 x i32> undef, splat (i32 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV2i16 = srem <vscale x 2 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV4i16 = srem <vscale x 4 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV8i16 = srem <vscale x 8 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV16i16 = srem <vscale x 16 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %NV32i16 = srem <vscale x 32 x i16> undef, splat (i16 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV2i8 = srem <vscale x 2 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %NV4i8 = srem <vscale x 4 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV8i8 = srem <vscale x 8 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %NV16i8 = srem <vscale x 16 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %NV32i8 = srem <vscale x 32 x i8> undef, splat (i8 -16)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %NV64i8 = srem <vscale x 64 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2i64 = srem <2 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4i64 = srem <4 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8i64 = srem <8 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i32 = srem <2 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i32 = srem <4 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8i32 = srem <8 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16i32 = srem <16 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i16 = srem <2 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i16 = srem <4 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i16 = srem <8 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16i16 = srem <16 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32i16 = srem <32 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2i8 = srem <2 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4i8 = srem <4 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8i8 = srem <8 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i8 = srem <16 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32i8 = srem <32 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64i8 = srem <64 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i64 = srem <vscale x 2 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV4i64 = srem <vscale x 4 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV8i64 = srem <vscale x 8 x i64> undef, splat (i64 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i32 = srem <vscale x 2 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV4i32 = srem <vscale x 4 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV8i32 = srem <vscale x 8 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV16i32 = srem <vscale x 16 x i32> undef, splat (i32 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i16 = srem <vscale x 2 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV4i16 = srem <vscale x 4 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV8i16 = srem <vscale x 8 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV16i16 = srem <vscale x 16 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV32i16 = srem <vscale x 32 x i16> undef, splat (i16 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV2i8 = srem <vscale x 2 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV4i8 = srem <vscale x 4 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV8i8 = srem <vscale x 8 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %NV16i8 = srem <vscale x 16 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %NV32i8 = srem <vscale x 32 x i8> undef, splat (i8 -16)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %NV64i8 = srem <vscale x 64 x i8> undef, splat (i8 -16)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V2i64 = srem <2 x i64> undef, splat (i64 -16)
diff --git a/llvm/test/Analysis/CostModel/RISCV/cast.ll b/llvm/test/Analysis/CostModel/RISCV/cast.ll
index 04048b8ba17f1..e498ccc733040 100644
--- a/llvm/test/Analysis/CostModel/RISCV/cast.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/cast.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -mtriple=riscv32 -mattr=+v,+zfh,+zvfh -passes="print<cost-model>" -cost-kind=throughput --type-based-intrinsic-cost=true 2>&1 -disable-output | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: opt < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh -passes="print<cost-model>" -cost-kind=throughput --type-based-intrinsic-cost=true 2>&1 -disable-output | FileCheck %s --check-prefixes=CHECK,RV64
+; RUN: opt < %s -mtriple=riscv32 -mattr=+v,+zfh,+zvfh -passes="print<cost-model>" -cost-kind=throughput -intrinsic-cost-strategy=type-based-intrinsic-cost 2>&1 -disable-output | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: opt < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh -passes="print<cost-model>" -cost-kind=throughput -intrinsic-cost-strategy=type-based-intrinsic-cost 2>&1 -disable-output | FileCheck %s --check-prefixes=CHECK,RV64
 
 define void @sext() {
 ; RV32-LABEL: 'sext'
diff --git a/llvm/test/Analysis/CostModel/RISCV/cmp.ll b/llvm/test/Analysis/CostModel/RISCV/cmp.ll
index b820baf5acf85..69d4f27ac41be 100644
--- a/llvm/test/Analysis/CostModel/RISCV/cmp.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/cmp.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=riscv32 -mattr=+v,+f -passes="print<cost-model>" -cost-kind=throughput --type-based-intrinsic-cost=true 2>&1 -disable-output | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: opt < %s -mtriple=riscv64 -mattr=+v,+f -passes="print<cost-model>" -cost-kind=throughput --type-based-intrinsic-cost=true 2>&1 -disable-output | FileCheck %s --check-prefixes=CHECK,RV64
+; RUN: opt < %s -mtriple=riscv32 -mattr=+v,+f -passes="print<cost-model>" -cost-kind=throughput -intrinsic-cost-strategy=type-based-intrinsic-cost 2>&1 -disable-output | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: opt < %s -mtriple=riscv64 -mattr=+v,+f -passes="print<cost-model>" -cost-kind=throughput -intrinsic-cost-strategy=type-based-intrinsic-cost 2>&1 -disable-output | FileCheck %s --check-prefixes=CHECK,RV64
 
 define void @icmp() {
 ; RV32-LABEL: 'icmp'
diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-expandload-compressstore.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-expandload-compressstore.ll
index ea5f3444fa248..e0f80f541bc19 100644
--- a/llvm/test/Analysis/CostModel/RISCV/rvv-expandload-compressstore.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/rvv-expandload-compressstore.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
 ; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+zvfhmin,+zvfbfmin | FileCheck %s
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+zvfhmin,+zvfbfmin --type-based-intrinsic-cost=true | FileCheck %s --check-prefixes=TYPEBASED
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+zvfhmin,+zvfbfmin -intrinsic-cost-strategy=type-based-intrinsic-cost | FileCheck %s --check-prefixes=TYPEBASED
 
 define void @expand_load() {
 ; CHECK-LABEL: 'expand_load'
diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-select.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-select.ll
index 6ab8ac64d64e4..748492c32630e 100644
--- a/llvm/test/Analysis/CostModel/RISCV/rvv-select.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/rvv-select.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=riscv64 -mattr=+v,+f,+d,+zfh,+zvfh -riscv-v-fixed-length-vector-lmul-max=1 < %s | FileCheck %s
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=riscv64 -mattr=+v,+f,+d,+zfh,+zvfh -riscv-v-fixed-length-vector-lmul-max=1 --type-based-intrinsic-cost=true < %s | FileCheck %s
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=riscv64 -mattr=+v,+f,+d,+zfh,+zvfh -riscv-v-fixed-length-vector-lmul-max=1 -intrinsic-cost-strategy=type-based-intrinsic-cost < %s | FileCheck %s
 ; Check that we don't crash querying costs when vectors are not enabled.
 ; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=riscv64
 
diff --git a/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll b/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll
index 7fc91e7049aa3..04b1d18fca8ee 100644
--- a/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
 ; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+zvfhmin,+zvfbfmin | FileCheck %s --check-prefixes=CHECK,ARGBASED
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+zvfhmin,+zvfbfmin --type-based-intrinsic-cost=true | FileCheck %s --check-prefixes=CHECK,TYPEBASED
+; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+zvfhmin,+zvfbfmin -intrinsic-cost-strategy=type-based-intrinsic-cost | FileCheck %s --check-prefixes=CHECK,TYPEBASED
 
 define void @unsupported_fp_ops(<vscale x 4 x float> %vec, i32 %extraarg) {
 ; CHECK-LABEL: 'unsupported_fp_ops'
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
index 992d050cf9ca0..fc9a126f79a83 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
@@ -4,306 +4,141 @@
 ; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+lsfe -O1 | FileCheck %s --check-prefixes=CHECK,-O1
 
 define dso_local half @atomicrmw_fadd_half_aligned_monotonic(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fadd_half_aligned_monotonic:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fadd_half_aligned_monotonic:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; CHECK:    ldfadd h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fadd_half_aligned_acquire(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fadd_half_aligned_acquire:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fadd_half_aligned_acquire:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; CHECK:    ldfadda h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, half %value acquire, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fadd_half_aligned_release(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fadd_half_aligned_release:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fadd_half_aligned_release:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_release:
+; CHECK:    ldfaddl h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, half %value release, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fadd_half_aligned_acq_rel(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; CHECK:    ldfaddal h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fadd_half_aligned_seq_cst(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; CHECK:    ldfaddal h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 2
     ret half %r
 }
 
 define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; CHECK:    ldbfadd h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; CHECK:    ldbfadda h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_release:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_release:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; CHECK:    ldbfaddl h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; CHECK:    ldbfaddal h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; CHECK:    ldbfaddal h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 2
     ret bfloat %r
 }
 
 define dso_local float @atomicrmw_fadd_float_aligned_monotonic(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fadd_float_aligned_monotonic:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fadd_float_aligned_monotonic:
-; -O1:    ldxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; CHECK:    ldfadd s0, s0, [x0]
     %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fadd_float_aligned_acquire(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fadd_float_aligned_acquire:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fadd_float_aligned_acquire:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; CHECK:    ldfadda s0, s0, [x0]
     %r = atomicrmw fadd ptr %ptr, float %value acquire, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fadd_float_aligned_release(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fadd_float_aligned_release:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fadd_float_aligned_release:
-; -O1:    ldxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_release:
+; CHECK:    ldfaddl s0, s0, [x0]
     %r = atomicrmw fadd ptr %ptr, float %value release, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fadd_float_aligned_acq_rel(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; CHECK:    ldfaddal s0, s0, [x0]
     %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fadd_float_aligned_seq_cst(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; CHECK:    ldfaddal s0, s0, [x0]
     %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 4
     ret float %r
 }
 
 define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fadd_double_aligned_monotonic:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fadd_double_aligned_monotonic:
-; -O1:    ldxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; CHECK:    ldfadd d0, d0, [x0]
     %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fadd_double_aligned_acquire(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fadd_double_aligned_acquire:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fadd_double_aligned_acquire:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; CHECK:    ldfadda d0, d0, [x0]
     %r = atomicrmw fadd ptr %ptr, double %value acquire, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fadd_double_aligned_release(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fadd_double_aligned_release:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fadd_double_aligned_release:
-; -O1:    ldxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_release:
+; CHECK:    ldfaddl d0, d0, [x0]
     %r = atomicrmw fadd ptr %ptr, double %value release, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fadd_double_aligned_acq_rel(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; CHECK:    ldfaddal d0, d0, [x0]
     %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fadd_double_aligned_seq_cst(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; CHECK:    ldfaddal d0, d0, [x0]
     %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 8
     ret double %r
 }
@@ -964,306 +799,141 @@ define dso_local double @atomicrmw_fsub_double_unaligned_seq_cst(ptr %ptr, doubl
 }
 
 define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmax_half_aligned_monotonic:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fmax_half_aligned_monotonic:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; CHECK:    ldfmaxnm h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmax_half_aligned_acquire(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmax_half_aligned_acquire:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fmax_half_aligned_acquire:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; CHECK:    ldfmaxnma h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value acquire, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmax_half_aligned_release:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fmax_half_aligned_release:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_release:
+; CHECK:    ldfmaxnml h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value release, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmax_half_aligned_acq_rel(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; CHECK:    ldfmaxnmal h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmax_half_aligned_seq_cst(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; CHECK:    ldfmaxnmal h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 2
     ret half %r
 }
 
 define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; CHECK:    ldbfmaxnm h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; CHECK:    ldbfmaxnma h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_release:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_release:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; CHECK:    ldbfmaxnml h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; CHECK:    ldbfmaxnmal h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; CHECK:    ldbfmaxnmal h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 2
     ret bfloat %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmax_float_aligned_monotonic:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fmax_float_aligned_monotonic:
-; -O1:    ldxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; CHECK:    ldfmaxnm s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_acquire(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmax_float_aligned_acquire:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fmax_float_aligned_acquire:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; CHECK:    ldfmaxnma s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value acquire, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmax_float_aligned_release:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fmax_float_aligned_release:
-; -O1:    ldxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_release:
+; CHECK:    ldfmaxnml s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value release, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_acq_rel(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; CHECK:    ldfmaxnmal s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_seq_cst(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; CHECK:    ldfmaxnmal s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 4
     ret float %r
 }
 
 define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmax_double_aligned_monotonic:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fmax_double_aligned_monotonic:
-; -O1:    ldxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; CHECK:    ldfmaxnm d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmax_double_aligned_acquire(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmax_double_aligned_acquire:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fmax_double_aligned_acquire:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; CHECK:    ldfmaxnma d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value acquire, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmax_double_aligned_release:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fmax_double_aligned_release:
-; -O1:    ldxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_release:
+; CHECK:    ldfmaxnml d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value release, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmax_double_aligned_acq_rel(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; CHECK:    ldfmaxnmal d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmax_double_aligned_seq_cst(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; CHECK:    ldfmaxnmal d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 8
     ret double %r
 }
@@ -1444,306 +1114,141 @@ define dso_local double @atomicrmw_fmax_double_unaligned_seq_cst(ptr %ptr, doubl
 }
 
 define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmin_half_aligned_monotonic:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fmin_half_aligned_monotonic:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; CHECK:    ldfminnm h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmin_half_aligned_acquire(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmin_half_aligned_acquire:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fmin_half_aligned_acquire:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; CHECK:    ldfminnma h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value acquire, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmin_half_aligned_release:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fmin_half_aligned_release:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_release:
+; CHECK:    ldfminnml h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value release, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmin_half_aligned_acq_rel(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; CHECK:    ldfminnmal h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmin_half_aligned_seq_cst(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
-; -O0:    ldaxrh w0, [x9]
-; -O0:    cmp w0, w10, uxth
-; -O0:    stlxrh w8, w11, [x9]
-; -O0:    subs w8, w8, w0, uxth
-;
-; -O1-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; CHECK:    ldfminnmal h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 2
     ret half %r
 }
 
 define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; CHECK:    ldbfminnm h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; CHECK:    ldbfminnma h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_release:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_release:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; CHECK:    ldbfminnml h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; CHECK:    ldbfminnmal h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; CHECK:    ldbfminnmal h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 2
     ret bfloat %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmin_float_aligned_monotonic:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fmin_float_aligned_monotonic:
-; -O1:    ldxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; CHECK:    ldfminnm s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_acquire(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmin_float_aligned_acquire:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fmin_float_aligned_acquire:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; CHECK:    ldfminnma s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value acquire, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmin_float_aligned_release:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fmin_float_aligned_release:
-; -O1:    ldxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_release:
+; CHECK:    ldfminnml s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value release, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_acq_rel(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; CHECK:    ldfminnmal s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_seq_cst(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
-; -O0:    ldaxr w0, [x9]
-; -O0:    cmp w0, w10
-; -O0:    stlxr w8, w11, [x9]
-; -O0:    subs w8, w0, w8
-;
-; -O1-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; CHECK:    ldfminnmal s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 4
     ret float %r
 }
 
 define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmin_double_aligned_monotonic:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fmin_double_aligned_monotonic:
-; -O1:    ldxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; CHECK:    ldfminnm d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmin_double_aligned_acquire(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmin_double_aligned_acquire:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fmin_double_aligned_acquire:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; CHECK:    ldfminnma d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value acquire, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmin_double_aligned_release:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fmin_double_aligned_release:
-; -O1:    ldxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_release:
+; CHECK:    ldfminnml d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value release, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmin_double_aligned_acq_rel(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; CHECK:    ldfminnmal d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmin_double_aligned_seq_cst(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
-; -O0:    ldaxr x0, [x9]
-; -O0:    cmp x0, x10
-; -O0:    stlxr w8, x11, [x9]
-; -O0:    subs x8, x0, x8
-;
-; -O1-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; CHECK:    ldfminnmal d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 8
     ret double %r
 }
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
index 6c46407177297..a22cc5806d86d 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
@@ -4,321 +4,141 @@
 ; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64_be -mattr=+lsfe -O1 | FileCheck %s --check-prefixes=CHECK,-O1
 
 define dso_local half @atomicrmw_fadd_half_aligned_monotonic(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fadd_half_aligned_monotonic:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_half_aligned_monotonic:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; CHECK:    ldfadd h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fadd_half_aligned_acquire(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fadd_half_aligned_acquire:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_half_aligned_acquire:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; CHECK:    ldfadda h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, half %value acquire, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fadd_half_aligned_release(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fadd_half_aligned_release:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_half_aligned_release:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_release:
+; CHECK:    ldfaddl h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, half %value release, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fadd_half_aligned_acq_rel(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; CHECK:    ldfaddal h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fadd_half_aligned_seq_cst(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; CHECK:    ldfaddal h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 2
     ret half %r
 }
 
 define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; CHECK:    ldbfadd h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; CHECK:    ldbfadda h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_release:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_release:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; CHECK:    ldbfaddl h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; CHECK:    ldbfaddal h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; CHECK:    ldbfaddal h0, h0, [x0]
     %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 2
     ret bfloat %r
 }
 
 define dso_local float @atomicrmw_fadd_float_aligned_monotonic(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fadd_float_aligned_monotonic:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_float_aligned_monotonic:
-; -O1:    ldxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; CHECK:    ldfadd s0, s0, [x0]
     %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fadd_float_aligned_acquire(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fadd_float_aligned_acquire:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_float_aligned_acquire:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; CHECK:    ldfadda s0, s0, [x0]
     %r = atomicrmw fadd ptr %ptr, float %value acquire, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fadd_float_aligned_release(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fadd_float_aligned_release:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_float_aligned_release:
-; -O1:    ldxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_release:
+; CHECK:    ldfaddl s0, s0, [x0]
     %r = atomicrmw fadd ptr %ptr, float %value release, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fadd_float_aligned_acq_rel(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; CHECK:    ldfaddal s0, s0, [x0]
     %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fadd_float_aligned_seq_cst(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; CHECK:    ldfaddal s0, s0, [x0]
     %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 4
     ret float %r
 }
 
 define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fadd_double_aligned_monotonic:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_double_aligned_monotonic:
-; -O1:    ldxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; CHECK:    ldfadd d0, d0, [x0]
     %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fadd_double_aligned_acquire(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fadd_double_aligned_acquire:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_double_aligned_acquire:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; CHECK:    ldfadda d0, d0, [x0]
     %r = atomicrmw fadd ptr %ptr, double %value acquire, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fadd_double_aligned_release(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fadd_double_aligned_release:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_double_aligned_release:
-; -O1:    ldxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_release:
+; CHECK:    ldfaddl d0, d0, [x0]
     %r = atomicrmw fadd ptr %ptr, double %value release, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fadd_double_aligned_acq_rel(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; CHECK:    ldfaddal d0, d0, [x0]
     %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fadd_double_aligned_seq_cst(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; CHECK:    ldfaddal d0, d0, [x0]
     %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 8
     ret double %r
 }
@@ -994,321 +814,141 @@ define dso_local double @atomicrmw_fsub_double_unaligned_seq_cst(ptr %ptr, doubl
 }
 
 define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmax_half_aligned_monotonic:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_half_aligned_monotonic:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; CHECK:    ldfmaxnm h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmax_half_aligned_acquire(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmax_half_aligned_acquire:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_half_aligned_acquire:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; CHECK:    ldfmaxnma h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value acquire, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmax_half_aligned_release:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_half_aligned_release:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_release:
+; CHECK:    ldfmaxnml h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value release, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmax_half_aligned_acq_rel(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; CHECK:    ldfmaxnmal h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmax_half_aligned_seq_cst(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; CHECK:    ldfmaxnmal h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 2
     ret half %r
 }
 
 define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; CHECK:    ldbfmaxnm h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; CHECK:    ldbfmaxnma h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_release:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_release:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; CHECK:    ldbfmaxnml h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; CHECK:    ldbfmaxnmal h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; CHECK:    ldbfmaxnmal h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 2
     ret bfloat %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmax_float_aligned_monotonic:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_float_aligned_monotonic:
-; -O1:    ldxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; CHECK:    ldfmaxnm s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_acquire(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmax_float_aligned_acquire:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_float_aligned_acquire:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; CHECK:    ldfmaxnma s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value acquire, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmax_float_aligned_release:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_float_aligned_release:
-; -O1:    ldxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_release:
+; CHECK:    ldfmaxnml s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value release, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_acq_rel(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; CHECK:    ldfmaxnmal s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_seq_cst(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; CHECK:    ldfmaxnmal s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 4
     ret float %r
 }
 
 define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmax_double_aligned_monotonic:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_double_aligned_monotonic:
-; -O1:    ldxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; CHECK:    ldfmaxnm d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmax_double_aligned_acquire(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmax_double_aligned_acquire:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_double_aligned_acquire:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; CHECK:    ldfmaxnma d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value acquire, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmax_double_aligned_release:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_double_aligned_release:
-; -O1:    ldxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_release:
+; CHECK:    ldfmaxnml d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value release, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmax_double_aligned_acq_rel(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; CHECK:    ldfmaxnmal d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmax_double_aligned_seq_cst(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; CHECK:    ldfmaxnmal d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 8
     ret double %r
 }
@@ -1489,321 +1129,141 @@ define dso_local double @atomicrmw_fmax_double_unaligned_seq_cst(ptr %ptr, doubl
 }
 
 define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmin_half_aligned_monotonic:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_half_aligned_monotonic:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; CHECK:    ldfminnm h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmin_half_aligned_acquire(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmin_half_aligned_acquire:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_half_aligned_acquire:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; CHECK:    ldfminnma h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value acquire, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmin_half_aligned_release:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_half_aligned_release:
-; -O1:    ldxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_release:
+; CHECK:    ldfminnml h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value release, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmin_half_aligned_acq_rel(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; CHECK:    ldfminnmal h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmin_half_aligned_seq_cst(ptr %ptr, half %value) {
-; -O0-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
-; -O1:    ldaxrh w8, [x0]
-; -O1:    stlxrh w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; CHECK:    ldfminnmal h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 2
     ret half %r
 }
 
 define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; CHECK:    ldbfminnm h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; CHECK:    ldbfminnma h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_release:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_release:
-; -O1:    ldxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; CHECK:    ldbfminnml h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; CHECK:    ldbfminnmal h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 2
     ret bfloat %r
 }
 
 define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
-; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
-; -O0:    add w8, w8, w9
-; -O0:    add w8, w8, w9
-; -O0:    ldaxrh w9, [x11]
-; -O0:    cmp w9, w8, uxth
-; -O0:    stlxrh w10, w12, [x11]
-; -O0:    subs w8, w9, w8, uxth
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
-; -O1:    ldaxrh w9, [x0]
-; -O1:    add w9, w9, w8
-; -O1:    add w9, w10, w9
-; -O1:    stlxrh w10, w9, [x0]
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; CHECK:    ldbfminnmal h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 2
     ret bfloat %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmin_float_aligned_monotonic:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_float_aligned_monotonic:
-; -O1:    ldxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; CHECK:    ldfminnm s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_acquire(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmin_float_aligned_acquire:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_float_aligned_acquire:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; CHECK:    ldfminnma s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value acquire, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmin_float_aligned_release:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_float_aligned_release:
-; -O1:    ldxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_release:
+; CHECK:    ldfminnml s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value release, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_acq_rel(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; CHECK:    ldfminnmal s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_seq_cst(ptr %ptr, float %value) {
-; -O0-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
-; -O0:    ldaxr w9, [x11]
-; -O0:    cmp w9, w8
-; -O0:    stlxr w10, w12, [x11]
-; -O0:    subs w8, w9, w8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
-; -O1:    ldaxr w8, [x0]
-; -O1:    stlxr w9, w8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; CHECK:    ldfminnmal s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 4
     ret float %r
 }
 
 define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmin_double_aligned_monotonic:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_double_aligned_monotonic:
-; -O1:    ldxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; CHECK:    ldfminnm d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmin_double_aligned_acquire(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmin_double_aligned_acquire:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_double_aligned_acquire:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; CHECK:    ldfminnma d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value acquire, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmin_double_aligned_release:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_double_aligned_release:
-; -O1:    ldxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_release:
+; CHECK:    ldfminnml d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value release, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmin_double_aligned_acq_rel(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; CHECK:    ldfminnmal d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmin_double_aligned_seq_cst(ptr %ptr, double %value) {
-; -O0-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
-; -O0:    ldaxr x9, [x11]
-; -O0:    cmp x9, x8
-; -O0:    stlxr w10, x12, [x11]
-; -O0:    subs x8, x9, x8
-; -O0:    subs w8, w8, #1
-;
-; -O1-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
-; -O1:    ldaxr x8, [x0]
-; -O1:    stlxr w9, x8, [x0]
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; CHECK:    ldfminnmal d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 8
     ret double %r
 }
diff --git a/llvm/test/CodeGen/AArch64/aarch64-known-bits-hadd.ll b/llvm/test/CodeGen/AArch64/aarch64-known-bits-hadd.ll
index 0506e1ed9710b..b07ba763d20ec 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-known-bits-hadd.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-known-bits-hadd.ll
@@ -35,9 +35,8 @@ define <8 x i16> @rhaddu_zext(<8 x i8> %a0, <8 x i8> %a1) {
 define <8 x i16> @hadds_zext(<8 x i8> %a0, <8 x i8> %a1) {
 ; CHECK-LABEL: hadds_zext:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    shadd v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    uhadd   v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ushll   v0.8h, v0.8b, #0
 ; CHECK-NEXT:    ret
   %x0 = zext <8 x i8> %a0 to <8 x i16>
   %x1 = zext <8 x i8> %a1 to <8 x i16>
diff --git a/llvm/test/CodeGen/AArch64/arm64ec-eh.ll b/llvm/test/CodeGen/AArch64/arm64ec-eh.ll
index 4b6bbb01c6c2d..64c4a871647f1 100644
--- a/llvm/test/CodeGen/AArch64/arm64ec-eh.ll
+++ b/llvm/test/CodeGen/AArch64/arm64ec-eh.ll
@@ -22,7 +22,6 @@ define dso_local i32 @test() #0 personality ptr @__C_specific_handler {
 ; CHECK-NEXT:    bl      "#ext"
 ; CHECK-NEXT:  .Ltmp1:
 ; CHECK-NEXT:  .LBB0_1:
-; CHECK-NEXT:  $ehgcr_0_1:
 ; CHECK-NEXT:    stur    w0, [x29, #-4]
 ; CHECK-NEXT:    .seh_startepilogue
 ; CHECK-NEXT:    ldp     x29, x30, [sp, #16]             // 16-byte Folded Reload
@@ -32,6 +31,7 @@ define dso_local i32 @test() #0 personality ptr @__C_specific_handler {
 ; CHECK-NEXT:    .seh_endepilogue
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:  $ehgcr_0_2:
 ; CHECK-NEXT:    mov     w0, wzr
 ; CHECK-NEXT:    b       .LBB0_1
   %1 = alloca i32, align 4
diff --git a/llvm/test/CodeGen/AArch64/fp16_fast_math.ll b/llvm/test/CodeGen/AArch64/fp16_fast_math.ll
new file mode 100644
index 0000000000000..b7d2de708a110
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fp16_fast_math.ll
@@ -0,0 +1,90 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=aarch64 -mattr=-fullfp16 -stop-after=finalize-isel | FileCheck %s --check-prefixes=CHECK-CVT
+; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 -stop-after=finalize-isel | FileCheck %s --check-prefixes=CHECK-FP16
+
+; Check that the output instructions have the same fast math flags as the input
+; fadd, even when fadd is promoted to float type.
+
+define half @normal_fadd(half %x, half %y) {
+  ; CHECK-CVT-LABEL: name: normal_fadd
+  ; CHECK-CVT: bb.0.entry:
+  ; CHECK-CVT-NEXT:   liveins: $h0, $h1
+  ; CHECK-CVT-NEXT: {{  $}}
+  ; CHECK-CVT-NEXT:   [[COPY:%[0-9]+]]:fpr16 = COPY $h1
+  ; CHECK-CVT-NEXT:   [[COPY1:%[0-9]+]]:fpr16 = COPY $h0
+  ; CHECK-CVT-NEXT:   [[FCVTSHr:%[0-9]+]]:fpr32 = nofpexcept FCVTSHr [[COPY]], implicit $fpcr
+  ; CHECK-CVT-NEXT:   [[FCVTSHr1:%[0-9]+]]:fpr32 = nofpexcept FCVTSHr [[COPY1]], implicit $fpcr
+  ; CHECK-CVT-NEXT:   [[FADDSrr:%[0-9]+]]:fpr32 = nofpexcept FADDSrr killed [[FCVTSHr1]], killed [[FCVTSHr]], implicit $fpcr
+  ; CHECK-CVT-NEXT:   [[FCVTHSr:%[0-9]+]]:fpr16 = nofpexcept FCVTHSr killed [[FADDSrr]], implicit $fpcr
+  ; CHECK-CVT-NEXT:   $h0 = COPY [[FCVTHSr]]
+  ; CHECK-CVT-NEXT:   RET_ReallyLR implicit $h0
+  ;
+  ; CHECK-FP16-LABEL: name: normal_fadd
+  ; CHECK-FP16: bb.0.entry:
+  ; CHECK-FP16-NEXT:   liveins: $h0, $h1
+  ; CHECK-FP16-NEXT: {{  $}}
+  ; CHECK-FP16-NEXT:   [[COPY:%[0-9]+]]:fpr16 = COPY $h1
+  ; CHECK-FP16-NEXT:   [[COPY1:%[0-9]+]]:fpr16 = COPY $h0
+  ; CHECK-FP16-NEXT:   [[FADDHrr:%[0-9]+]]:fpr16 = nofpexcept FADDHrr [[COPY1]], [[COPY]], implicit $fpcr
+  ; CHECK-FP16-NEXT:   $h0 = COPY [[FADDHrr]]
+  ; CHECK-FP16-NEXT:   RET_ReallyLR implicit $h0
+entry:
+  %add = fadd half %x, %y
+  ret half %add
+}
+
+define half @fast_fadd(half %x, half %y) {
+  ; CHECK-CVT-LABEL: name: fast_fadd
+  ; CHECK-CVT: bb.0.entry:
+  ; CHECK-CVT-NEXT:   liveins: $h0, $h1
+  ; CHECK-CVT-NEXT: {{  $}}
+  ; CHECK-CVT-NEXT:   [[COPY:%[0-9]+]]:fpr16 = COPY $h1
+  ; CHECK-CVT-NEXT:   [[COPY1:%[0-9]+]]:fpr16 = COPY $h0
+  ; CHECK-CVT-NEXT:   [[FCVTSHr:%[0-9]+]]:fpr32 = nnan ninf nsz arcp contract afn reassoc nofpexcept FCVTSHr [[COPY]], implicit $fpcr
+  ; CHECK-CVT-NEXT:   [[FCVTSHr1:%[0-9]+]]:fpr32 = nnan ninf nsz arcp contract afn reassoc nofpexcept FCVTSHr [[COPY1]], implicit $fpcr
+  ; CHECK-CVT-NEXT:   [[FADDSrr:%[0-9]+]]:fpr32 = nnan ninf nsz arcp contract afn reassoc nofpexcept FADDSrr killed [[FCVTSHr1]], killed [[FCVTSHr]], implicit $fpcr
+  ; CHECK-CVT-NEXT:   [[FCVTHSr:%[0-9]+]]:fpr16 = nnan ninf nsz arcp contract afn reassoc nofpexcept FCVTHSr killed [[FADDSrr]], implicit $fpcr
+  ; CHECK-CVT-NEXT:   $h0 = COPY [[FCVTHSr]]
+  ; CHECK-CVT-NEXT:   RET_ReallyLR implicit $h0
+  ;
+  ; CHECK-FP16-LABEL: name: fast_fadd
+  ; CHECK-FP16: bb.0.entry:
+  ; CHECK-FP16-NEXT:   liveins: $h0, $h1
+  ; CHECK-FP16-NEXT: {{  $}}
+  ; CHECK-FP16-NEXT:   [[COPY:%[0-9]+]]:fpr16 = COPY $h1
+  ; CHECK-FP16-NEXT:   [[COPY1:%[0-9]+]]:fpr16 = COPY $h0
+  ; CHECK-FP16-NEXT:   [[FADDHrr:%[0-9]+]]:fpr16 = nnan ninf nsz arcp contract afn reassoc nofpexcept FADDHrr [[COPY1]], [[COPY]], implicit $fpcr
+  ; CHECK-FP16-NEXT:   $h0 = COPY [[FADDHrr]]
+  ; CHECK-FP16-NEXT:   RET_ReallyLR implicit $h0
+entry:
+  %add = fadd fast half %x, %y
+  ret half %add
+}
+
+define half @ninf_fadd(half %x, half %y) {
+  ; CHECK-CVT-LABEL: name: ninf_fadd
+  ; CHECK-CVT: bb.0.entry:
+  ; CHECK-CVT-NEXT:   liveins: $h0, $h1
+  ; CHECK-CVT-NEXT: {{  $}}
+  ; CHECK-CVT-NEXT:   [[COPY:%[0-9]+]]:fpr16 = COPY $h1
+  ; CHECK-CVT-NEXT:   [[COPY1:%[0-9]+]]:fpr16 = COPY $h0
+  ; CHECK-CVT-NEXT:   [[FCVTSHr:%[0-9]+]]:fpr32 = ninf nofpexcept FCVTSHr [[COPY]], implicit $fpcr
+  ; CHECK-CVT-NEXT:   [[FCVTSHr1:%[0-9]+]]:fpr32 = ninf nofpexcept FCVTSHr [[COPY1]], implicit $fpcr
+  ; CHECK-CVT-NEXT:   [[FADDSrr:%[0-9]+]]:fpr32 = ninf nofpexcept FADDSrr killed [[FCVTSHr1]], killed [[FCVTSHr]], implicit $fpcr
+  ; CHECK-CVT-NEXT:   [[FCVTHSr:%[0-9]+]]:fpr16 = ninf nofpexcept FCVTHSr killed [[FADDSrr]], implicit $fpcr
+  ; CHECK-CVT-NEXT:   $h0 = COPY [[FCVTHSr]]
+  ; CHECK-CVT-NEXT:   RET_ReallyLR implicit $h0
+  ;
+  ; CHECK-FP16-LABEL: name: ninf_fadd
+  ; CHECK-FP16: bb.0.entry:
+  ; CHECK-FP16-NEXT:   liveins: $h0, $h1
+  ; CHECK-FP16-NEXT: {{  $}}
+  ; CHECK-FP16-NEXT:   [[COPY:%[0-9]+]]:fpr16 = COPY $h1
+  ; CHECK-FP16-NEXT:   [[COPY1:%[0-9]+]]:fpr16 = COPY $h0
+  ; CHECK-FP16-NEXT:   [[FADDHrr:%[0-9]+]]:fpr16 = ninf nofpexcept FADDHrr [[COPY1]], [[COPY]], implicit $fpcr
+  ; CHECK-FP16-NEXT:   $h0 = COPY [[FADDHrr]]
+  ; CHECK-FP16-NEXT:   RET_ReallyLR implicit $h0
+entry:
+  %add = fadd ninf half %x, %y
+  ret half %add
+}
diff --git a/llvm/test/CodeGen/AArch64/landingpad-ifcvt.ll b/llvm/test/CodeGen/AArch64/landingpad-ifcvt.ll
index 367b4975e7558..c6dd152d7ff54 100644
--- a/llvm/test/CodeGen/AArch64/landingpad-ifcvt.ll
+++ b/llvm/test/CodeGen/AArch64/landingpad-ifcvt.ll
@@ -2,6 +2,7 @@
 
 ; Make sure this doesn't crash (and the output is sane).
 ; CHECK: // %__except.ret
+; CHECK-NEXT: $ehgcr_0_2:
 ; CHECK-NEXT: mov     x0, xzr
 
 target datalayout = "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128"
diff --git a/llvm/test/CodeGen/AArch64/patchable-function-entry-empty.mir b/llvm/test/CodeGen/AArch64/patchable-function-entry-empty.mir
index 2db89d210316d..f562a4fce3d74 100644
--- a/llvm/test/CodeGen/AArch64/patchable-function-entry-empty.mir
+++ b/llvm/test/CodeGen/AArch64/patchable-function-entry-empty.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=aarch64 -run-pass=patchable-function %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64 -passes=patchable-function %s -o - | FileCheck %s
 
 ## The initial .loc covers PATCHABLE_FUNCTION_ENTER.
 ## Emitting a new .loc may create a prologue_end prematurely.
diff --git a/llvm/test/CodeGen/AArch64/sve-fcopysign.ll b/llvm/test/CodeGen/AArch64/sve-fcopysign.ll
index 6f11bf963c1d1..39535a11e4d12 100644
--- a/llvm/test/CodeGen/AArch64/sve-fcopysign.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fcopysign.ll
@@ -185,6 +185,19 @@ define <vscale x 2 x float> @copysign_nxv2f32_nxv2f64(<vscale x 2 x float> %a, <
   ret <vscale x 2 x float> %r
 }
 
+define <vscale x 2 x float> @copysign_nxv2f32_nxv2bf16(<vscale x 2 x float> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: copysign_nxv2f32_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z1.s, z1.s, #16
+; CHECK-NEXT:    and z0.s, z0.s, #0x7fffffff
+; CHECK-NEXT:    and z1.s, z1.s, #0x80000000
+; CHECK-NEXT:    orr z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %tmp0 = fpext <vscale x 2 x bfloat> %b to <vscale x 2 x float>
+  %r = call <vscale x 2 x float> @llvm.copysign.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %tmp0)
+  ret <vscale x 2 x float> %r
+}
+
 ;
 ; llvm.copysign.nxv4f32
 ;
@@ -230,6 +243,19 @@ define <vscale x 4 x float> @copysign_nxv4f32_nxv4f64(<vscale x 4 x float> %a, <
   ret <vscale x 4 x float> %r
 }
 
+define <vscale x 4 x float> @copysign_nxv4f32_nxv4bf16(<vscale x 4 x float> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: copysign_nxv4f32_nxv4bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z1.s, z1.s, #16
+; CHECK-NEXT:    and z0.s, z0.s, #0x7fffffff
+; CHECK-NEXT:    and z1.s, z1.s, #0x80000000
+; CHECK-NEXT:    orr z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %tmp0 = fpext <vscale x 4 x bfloat> %b to <vscale x 4 x float>
+  %r = call <vscale x 4 x float> @llvm.copysign.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %tmp0)
+  ret <vscale x 4 x float> %r
+}
+
 ;
 ; llvm.copysign.nxv2f64
 ;
@@ -273,9 +299,137 @@ define <vscale x 2 x double> @copysign_nxv2f64_nxv2f64(<vscale x 2 x double> %a,
   ret <vscale x 2 x double> %r
 }
 
+define <vscale x 2 x double> @copysign_nxv2f64_nxv2bf16(<vscale x 2 x double> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: copysign_nxv2f64_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z1.s, z1.s, #16
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    and z0.d, z0.d, #0x7fffffffffffffff
+; CHECK-NEXT:    fcvt z1.d, p0/m, z1.s
+; CHECK-NEXT:    and z1.d, z1.d, #0x8000000000000000
+; CHECK-NEXT:    orr z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %b.ext = fpext <vscale x 2 x bfloat> %b to <vscale x 2 x double>
+  %r = call <vscale x 2 x double> @llvm.copysign.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b.ext)
+  ret <vscale x 2 x double> %r
+}
+
+;
+; llvm.copysign.nxv2bf16
+;
+
+define <vscale x 2 x bfloat> @copysign_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: copysign_nxv2bf16_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.h, z1.h, #0x8000
+; CHECK-NEXT:    and z0.h, z0.h, #0x7fff
+; CHECK-NEXT:    orr z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %r = call <vscale x 2 x bfloat> @llvm.copysign.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b)
+  ret <vscale x 2 x bfloat> %r
+}
+
+define <vscale x 2 x bfloat> @copysign_nxv2bf16_nxv2f32(<vscale x 2 x bfloat> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: copysign_nxv2bf16_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    and z0.h, z0.h, #0x7fff
+; CHECK-NEXT:    bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT:    and z1.h, z1.h, #0x8000
+; CHECK-NEXT:    orr z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %tmp0 = fptrunc <vscale x 2 x float> %b to <vscale x 2 x bfloat>
+  %r = call <vscale x 2 x bfloat> @llvm.copysign.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %tmp0)
+  ret <vscale x 2 x bfloat> %r
+}
+
+; TODO: Cannot downconvert from double to bfloat
+;define <vscale x 2 x bfloat> @copysign_nxv2bf16_nxv2f64(<vscale x 2 x bfloat> %a, <vscale x 2 x double> %b) {
+;  %tmp0 = fptrunc <vscale x 2 x double> %b to <vscale x 2 x bfloat>
+;  %r = call <vscale x 2 x bfloat> @llvm.copysign.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %tmp0)
+;  ret <vscale x 2 x bfloat> %r
+;}
+
+;
+; llvm.copysign.nxv2bf16
+;
+
+define <vscale x 4 x bfloat> @copysign_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: copysign_nxv4bf16_nxv4bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.h, z1.h, #0x8000
+; CHECK-NEXT:    and z0.h, z0.h, #0x7fff
+; CHECK-NEXT:    orr z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %r = call <vscale x 4 x bfloat> @llvm.copysign.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b)
+  ret <vscale x 4 x bfloat> %r
+}
+
+define <vscale x 4 x bfloat> @copysign_nxv4bf16_nxv4f32(<vscale x 4 x bfloat> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: copysign_nxv4bf16_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    and z0.h, z0.h, #0x7fff
+; CHECK-NEXT:    bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT:    and z1.h, z1.h, #0x8000
+; CHECK-NEXT:    orr z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %b.trunc = fptrunc <vscale x 4 x float> %b to <vscale x 4 x bfloat>
+  %r = call <vscale x 4 x bfloat> @llvm.copysign.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b.trunc)
+  ret <vscale x 4 x bfloat> %r
+}
+
+; TODO: Cannot downconvert from double to bfloat
+;define <vscale x 4 x bfloat> @copysign_nxv4bf16_nxv4f64(<vscale x 4 x bfloat> %a, <vscale x 4 x double> %b) {
+;  %b.trunc = fptrunc <vscale x 4 x double> %b to <vscale x 4 x bfloat>
+;  %r = call <vscale x 4 x bfloat> @llvm.copysign.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b.trunc)
+;  ret <vscale x 4 x bfloat> %r
+;}
+
+;
+; llvm.copysign.nxv8bf16
+;
+
+define <vscale x 8 x bfloat> @copysign_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: copysign_nxv8bf16_nxv8bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.h, z1.h, #0x8000
+; CHECK-NEXT:    and z0.h, z0.h, #0x7fff
+; CHECK-NEXT:    orr z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %r = call <vscale x 8 x bfloat> @llvm.copysign.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b)
+  ret <vscale x 8 x bfloat> %r
+}
+
+define <vscale x 8 x bfloat> @copysign_nxv8bf16_nxv8f32(<vscale x 8 x bfloat> %a, <vscale x 8 x float> %b) {
+; CHECK-LABEL: copysign_nxv8bf16_nxv8f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    and z0.h, z0.h, #0x7fff
+; CHECK-NEXT:    bfcvt z2.h, p0/m, z2.s
+; CHECK-NEXT:    bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT:    uzp1 z1.h, z1.h, z2.h
+; CHECK-NEXT:    and z1.h, z1.h, #0x8000
+; CHECK-NEXT:    orr z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %b.trunc = fptrunc <vscale x 8 x float> %b to <vscale x 8 x bfloat>
+  %r = call <vscale x 8 x bfloat> @llvm.copysign.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b.trunc)
+  ret <vscale x 8 x bfloat> %r
+}
+
+; TODO: Cannot downconvert from double to bfloat
+;define <vscale x 8 x bfloat> @copysign_nxv8bf16_nxv8f64(<vscale x 8 x bfloat> %a, <vscale x 8 x double> %b) {
+;  %b.trunc = fptrunc <vscale x 8 x double> %b to <vscale x 8 x bfloat>
+;  %r = call <vscale x 8 x bfloat> @llvm.copysign.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b.trunc)
+;  ret <vscale x 8 x bfloat> %r
+;}
+
 declare <vscale x 2 x half> @llvm.copysign.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> %b)
 declare <vscale x 4 x half> @llvm.copysign.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %b)
 declare <vscale x 8 x half> @llvm.copysign.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b)
 declare <vscale x 2 x float> @llvm.copysign.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b)
 declare <vscale x 4 x float> @llvm.copysign.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b)
 declare <vscale x 2 x double> @llvm.copysign.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b)
+declare <vscale x 2 x bfloat> @llvm.copysign.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b)
+declare <vscale x 4 x bfloat> @llvm.copysign.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b)
+declare <vscale x 8 x bfloat> @llvm.copysign.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-offsets.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-offsets.ll
deleted file mode 100644
index 700bbe4f060ca..0000000000000
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-offsets.ll
+++ /dev/null
@@ -1,362 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -aarch64-sve-vector-bits-min=128 -aarch64-sve-vector-bits-max=128 < %s | FileCheck %s --check-prefix=CHECK-128
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -aarch64-sve-vector-bits-min=256 -aarch64-sve-vector-bits-max=256 < %s | FileCheck %s --check-prefix=CHECK-256
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -aarch64-sve-vector-bits-min=512 -aarch64-sve-vector-bits-max=512 < %s | FileCheck %s --check-prefix=CHECK-512
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -aarch64-sve-vector-bits-min=1024 -aarch64-sve-vector-bits-max=1024 < %s | FileCheck %s --check-prefix=CHECK-1024
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -aarch64-sve-vector-bits-min=2048 -aarch64-sve-vector-bits-max=2048 < %s | FileCheck %s --check-prefix=CHECK-2048
-
-define void @nxv16i8(ptr %ldptr, ptr %stptr) {
-; CHECK-LABEL: nxv16i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b
-; CHECK-NEXT:    mov w8, #256 // =0x100
-; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0, x8]
-; CHECK-NEXT:    st1b { z0.b }, p0, [x1, x8]
-; CHECK-NEXT:    ret
-;
-; CHECK-128-LABEL: nxv16i8:
-; CHECK-128:       // %bb.0:
-; CHECK-128-NEXT:    ldr z0, [x0, #16, mul vl]
-; CHECK-128-NEXT:    str z0, [x1, #16, mul vl]
-; CHECK-128-NEXT:    ret
-;
-; CHECK-256-LABEL: nxv16i8:
-; CHECK-256:       // %bb.0:
-; CHECK-256-NEXT:    ldr z0, [x0, #8, mul vl]
-; CHECK-256-NEXT:    str z0, [x1, #8, mul vl]
-; CHECK-256-NEXT:    ret
-;
-; CHECK-512-LABEL: nxv16i8:
-; CHECK-512:       // %bb.0:
-; CHECK-512-NEXT:    ldr z0, [x0, #4, mul vl]
-; CHECK-512-NEXT:    str z0, [x1, #4, mul vl]
-; CHECK-512-NEXT:    ret
-;
-; CHECK-1024-LABEL: nxv16i8:
-; CHECK-1024:       // %bb.0:
-; CHECK-1024-NEXT:    ldr z0, [x0, #2, mul vl]
-; CHECK-1024-NEXT:    str z0, [x1, #2, mul vl]
-; CHECK-1024-NEXT:    ret
-;
-; CHECK-2048-LABEL: nxv16i8:
-; CHECK-2048:       // %bb.0:
-; CHECK-2048-NEXT:    ldr z0, [x0, #1, mul vl]
-; CHECK-2048-NEXT:    str z0, [x1, #1, mul vl]
-; CHECK-2048-NEXT:    ret
-  %ldoff = getelementptr inbounds nuw i8, ptr %ldptr, i64 256
-  %stoff = getelementptr inbounds nuw i8, ptr %stptr, i64 256
-  %x = load <vscale x 16 x i8>, ptr %ldoff, align 1
-  store <vscale x 16 x i8> %x, ptr %stoff, align 1
-  ret void
-}
-
-define void @nxv8i16(ptr %ldptr, ptr %stptr) {
-; CHECK-LABEL: nxv8i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.h
-; CHECK-NEXT:    mov x8, #128 // =0x80
-; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
-; CHECK-NEXT:    st1h { z0.h }, p0, [x1, x8, lsl #1]
-; CHECK-NEXT:    ret
-;
-; CHECK-128-LABEL: nxv8i16:
-; CHECK-128:       // %bb.0:
-; CHECK-128-NEXT:    ldr z0, [x0, #16, mul vl]
-; CHECK-128-NEXT:    str z0, [x1, #16, mul vl]
-; CHECK-128-NEXT:    ret
-;
-; CHECK-256-LABEL: nxv8i16:
-; CHECK-256:       // %bb.0:
-; CHECK-256-NEXT:    ldr z0, [x0, #8, mul vl]
-; CHECK-256-NEXT:    str z0, [x1, #8, mul vl]
-; CHECK-256-NEXT:    ret
-;
-; CHECK-512-LABEL: nxv8i16:
-; CHECK-512:       // %bb.0:
-; CHECK-512-NEXT:    ldr z0, [x0, #4, mul vl]
-; CHECK-512-NEXT:    str z0, [x1, #4, mul vl]
-; CHECK-512-NEXT:    ret
-;
-; CHECK-1024-LABEL: nxv8i16:
-; CHECK-1024:       // %bb.0:
-; CHECK-1024-NEXT:    ldr z0, [x0, #2, mul vl]
-; CHECK-1024-NEXT:    str z0, [x1, #2, mul vl]
-; CHECK-1024-NEXT:    ret
-;
-; CHECK-2048-LABEL: nxv8i16:
-; CHECK-2048:       // %bb.0:
-; CHECK-2048-NEXT:    ldr z0, [x0, #1, mul vl]
-; CHECK-2048-NEXT:    str z0, [x1, #1, mul vl]
-; CHECK-2048-NEXT:    ret
-  %ldoff = getelementptr inbounds nuw i16, ptr %ldptr, i64 128
-  %stoff = getelementptr inbounds nuw i16, ptr %stptr, i64 128
-  %x = load <vscale x 8 x i16>, ptr %ldoff, align 2
-  store <vscale x 8 x i16> %x, ptr %stoff, align 2
-  ret void
-}
-
-define void @nxv4i32(ptr %ldptr, ptr %stptr) {
-; CHECK-LABEL: nxv4i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    mov x8, #64 // =0x40
-; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
-; CHECK-NEXT:    st1w { z0.s }, p0, [x1, x8, lsl #2]
-; CHECK-NEXT:    ret
-;
-; CHECK-128-LABEL: nxv4i32:
-; CHECK-128:       // %bb.0:
-; CHECK-128-NEXT:    ldr z0, [x0, #16, mul vl]
-; CHECK-128-NEXT:    str z0, [x1, #16, mul vl]
-; CHECK-128-NEXT:    ret
-;
-; CHECK-256-LABEL: nxv4i32:
-; CHECK-256:       // %bb.0:
-; CHECK-256-NEXT:    ldr z0, [x0, #8, mul vl]
-; CHECK-256-NEXT:    str z0, [x1, #8, mul vl]
-; CHECK-256-NEXT:    ret
-;
-; CHECK-512-LABEL: nxv4i32:
-; CHECK-512:       // %bb.0:
-; CHECK-512-NEXT:    ldr z0, [x0, #4, mul vl]
-; CHECK-512-NEXT:    str z0, [x1, #4, mul vl]
-; CHECK-512-NEXT:    ret
-;
-; CHECK-1024-LABEL: nxv4i32:
-; CHECK-1024:       // %bb.0:
-; CHECK-1024-NEXT:    ldr z0, [x0, #2, mul vl]
-; CHECK-1024-NEXT:    str z0, [x1, #2, mul vl]
-; CHECK-1024-NEXT:    ret
-;
-; CHECK-2048-LABEL: nxv4i32:
-; CHECK-2048:       // %bb.0:
-; CHECK-2048-NEXT:    ldr z0, [x0, #1, mul vl]
-; CHECK-2048-NEXT:    str z0, [x1, #1, mul vl]
-; CHECK-2048-NEXT:    ret
-  %ldoff = getelementptr inbounds nuw i32, ptr %ldptr, i64 64
-  %stoff = getelementptr inbounds nuw i32, ptr %stptr, i64 64
-  %x = load <vscale x 4 x i32>, ptr %ldoff, align 4
-  store <vscale x 4 x i32> %x, ptr %stoff, align 4
-  ret void
-}
-
-define void @nxv2i64(ptr %ldptr, ptr %stptr) {
-; CHECK-LABEL: nxv2i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    mov x8, #32 // =0x20
-; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
-; CHECK-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
-; CHECK-NEXT:    ret
-;
-; CHECK-128-LABEL: nxv2i64:
-; CHECK-128:       // %bb.0:
-; CHECK-128-NEXT:    ldr z0, [x0, #16, mul vl]
-; CHECK-128-NEXT:    str z0, [x1, #16, mul vl]
-; CHECK-128-NEXT:    ret
-;
-; CHECK-256-LABEL: nxv2i64:
-; CHECK-256:       // %bb.0:
-; CHECK-256-NEXT:    ldr z0, [x0, #8, mul vl]
-; CHECK-256-NEXT:    str z0, [x1, #8, mul vl]
-; CHECK-256-NEXT:    ret
-;
-; CHECK-512-LABEL: nxv2i64:
-; CHECK-512:       // %bb.0:
-; CHECK-512-NEXT:    ldr z0, [x0, #4, mul vl]
-; CHECK-512-NEXT:    str z0, [x1, #4, mul vl]
-; CHECK-512-NEXT:    ret
-;
-; CHECK-1024-LABEL: nxv2i64:
-; CHECK-1024:       // %bb.0:
-; CHECK-1024-NEXT:    ldr z0, [x0, #2, mul vl]
-; CHECK-1024-NEXT:    str z0, [x1, #2, mul vl]
-; CHECK-1024-NEXT:    ret
-;
-; CHECK-2048-LABEL: nxv2i64:
-; CHECK-2048:       // %bb.0:
-; CHECK-2048-NEXT:    ldr z0, [x0, #1, mul vl]
-; CHECK-2048-NEXT:    str z0, [x1, #1, mul vl]
-; CHECK-2048-NEXT:    ret
-  %ldoff = getelementptr inbounds nuw i64, ptr %ldptr, i64 32
-  %stoff = getelementptr inbounds nuw i64, ptr %stptr, i64 32
-  %x = load <vscale x 2 x i64>, ptr %ldoff, align 8
-  store <vscale x 2 x i64> %x, ptr %stoff, align 8
-  ret void
-}
-
-define void @nxv4i8(ptr %ldptr, ptr %stptr) {
-; CHECK-LABEL: nxv4i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    mov w8, #32 // =0x20
-; CHECK-NEXT:    ld1b { z0.s }, p0/z, [x0, x8]
-; CHECK-NEXT:    st1b { z0.s }, p0, [x1, x8]
-; CHECK-NEXT:    ret
-;
-; CHECK-128-LABEL: nxv4i8:
-; CHECK-128:       // %bb.0:
-; CHECK-128-NEXT:    ptrue p0.s
-; CHECK-128-NEXT:    mov w8, #32 // =0x20
-; CHECK-128-NEXT:    ld1b { z0.s }, p0/z, [x0, x8]
-; CHECK-128-NEXT:    st1b { z0.s }, p0, [x1, x8]
-; CHECK-128-NEXT:    ret
-;
-; CHECK-256-LABEL: nxv4i8:
-; CHECK-256:       // %bb.0:
-; CHECK-256-NEXT:    ptrue p0.s
-; CHECK-256-NEXT:    ld1b { z0.s }, p0/z, [x0, #4, mul vl]
-; CHECK-256-NEXT:    st1b { z0.s }, p0, [x1, #4, mul vl]
-; CHECK-256-NEXT:    ret
-;
-; CHECK-512-LABEL: nxv4i8:
-; CHECK-512:       // %bb.0:
-; CHECK-512-NEXT:    ptrue p0.s
-; CHECK-512-NEXT:    ld1b { z0.s }, p0/z, [x0, #2, mul vl]
-; CHECK-512-NEXT:    st1b { z0.s }, p0, [x1, #2, mul vl]
-; CHECK-512-NEXT:    ret
-;
-; CHECK-1024-LABEL: nxv4i8:
-; CHECK-1024:       // %bb.0:
-; CHECK-1024-NEXT:    ptrue p0.s
-; CHECK-1024-NEXT:    ld1b { z0.s }, p0/z, [x0, #1, mul vl]
-; CHECK-1024-NEXT:    st1b { z0.s }, p0, [x1, #1, mul vl]
-; CHECK-1024-NEXT:    ret
-;
-; CHECK-2048-LABEL: nxv4i8:
-; CHECK-2048:       // %bb.0:
-; CHECK-2048-NEXT:    ptrue p0.s
-; CHECK-2048-NEXT:    mov w8, #32 // =0x20
-; CHECK-2048-NEXT:    ld1b { z0.s }, p0/z, [x0, x8]
-; CHECK-2048-NEXT:    st1b { z0.s }, p0, [x1, x8]
-; CHECK-2048-NEXT:    ret
-  %ldoff = getelementptr inbounds nuw i8, ptr %ldptr, i64 32
-  %stoff = getelementptr inbounds nuw i8, ptr %stptr, i64 32
-  %x = load <vscale x 4 x i8>, ptr %ldoff, align 1
-  store <vscale x 4 x i8> %x, ptr %stoff, align 1
-  ret void
-}
-
-define void @nxv2f32(ptr %ldptr, ptr %stptr) {
-; CHECK-LABEL: nxv2f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    mov x8, #16 // =0x10
-; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0, x8, lsl #2]
-; CHECK-NEXT:    st1w { z0.d }, p0, [x1, x8, lsl #2]
-; CHECK-NEXT:    ret
-;
-; CHECK-128-LABEL: nxv2f32:
-; CHECK-128:       // %bb.0:
-; CHECK-128-NEXT:    ptrue p0.d
-; CHECK-128-NEXT:    mov x8, #16 // =0x10
-; CHECK-128-NEXT:    ld1w { z0.d }, p0/z, [x0, x8, lsl #2]
-; CHECK-128-NEXT:    st1w { z0.d }, p0, [x1, x8, lsl #2]
-; CHECK-128-NEXT:    ret
-;
-; CHECK-256-LABEL: nxv2f32:
-; CHECK-256:       // %bb.0:
-; CHECK-256-NEXT:    ptrue p0.d
-; CHECK-256-NEXT:    ld1w { z0.d }, p0/z, [x0, #4, mul vl]
-; CHECK-256-NEXT:    st1w { z0.d }, p0, [x1, #4, mul vl]
-; CHECK-256-NEXT:    ret
-;
-; CHECK-512-LABEL: nxv2f32:
-; CHECK-512:       // %bb.0:
-; CHECK-512-NEXT:    ptrue p0.d
-; CHECK-512-NEXT:    ld1w { z0.d }, p0/z, [x0, #2, mul vl]
-; CHECK-512-NEXT:    st1w { z0.d }, p0, [x1, #2, mul vl]
-; CHECK-512-NEXT:    ret
-;
-; CHECK-1024-LABEL: nxv2f32:
-; CHECK-1024:       // %bb.0:
-; CHECK-1024-NEXT:    ptrue p0.d
-; CHECK-1024-NEXT:    ld1w { z0.d }, p0/z, [x0, #1, mul vl]
-; CHECK-1024-NEXT:    st1w { z0.d }, p0, [x1, #1, mul vl]
-; CHECK-1024-NEXT:    ret
-;
-; CHECK-2048-LABEL: nxv2f32:
-; CHECK-2048:       // %bb.0:
-; CHECK-2048-NEXT:    ptrue p0.d
-; CHECK-2048-NEXT:    mov x8, #16 // =0x10
-; CHECK-2048-NEXT:    ld1w { z0.d }, p0/z, [x0, x8, lsl #2]
-; CHECK-2048-NEXT:    st1w { z0.d }, p0, [x1, x8, lsl #2]
-; CHECK-2048-NEXT:    ret
-  %ldoff = getelementptr inbounds nuw i8, ptr %ldptr, i64 64
-  %stoff = getelementptr inbounds nuw i8, ptr %stptr, i64 64
-  %x = load <vscale x 2 x float>, ptr %ldoff, align 1
-  store <vscale x 2 x float> %x, ptr %stoff, align 1
-  ret void
-}
-
-define void @nxv4f64(ptr %ldptr, ptr %stptr) {
-; CHECK-LABEL: nxv4f64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    mov x8, #16 // =0x10
-; CHECK-NEXT:    add x9, x0, #128
-; CHECK-NEXT:    ldr z1, [x9, #1, mul vl]
-; CHECK-NEXT:    add x9, x1, #128
-; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
-; CHECK-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
-; CHECK-NEXT:    str z1, [x9, #1, mul vl]
-; CHECK-NEXT:    ret
-;
-; CHECK-128-LABEL: nxv4f64:
-; CHECK-128:       // %bb.0:
-; CHECK-128-NEXT:    add x8, x0, #128
-; CHECK-128-NEXT:    ldr z1, [x0, #8, mul vl]
-; CHECK-128-NEXT:    ldr z0, [x8, #1, mul vl]
-; CHECK-128-NEXT:    add x8, x1, #128
-; CHECK-128-NEXT:    str z0, [x8, #1, mul vl]
-; CHECK-128-NEXT:    str z1, [x1, #8, mul vl]
-; CHECK-128-NEXT:    ret
-;
-; CHECK-256-LABEL: nxv4f64:
-; CHECK-256:       // %bb.0:
-; CHECK-256-NEXT:    add x8, x0, #128
-; CHECK-256-NEXT:    ldr z1, [x0, #4, mul vl]
-; CHECK-256-NEXT:    ldr z0, [x8, #1, mul vl]
-; CHECK-256-NEXT:    add x8, x1, #128
-; CHECK-256-NEXT:    str z0, [x8, #1, mul vl]
-; CHECK-256-NEXT:    str z1, [x1, #4, mul vl]
-; CHECK-256-NEXT:    ret
-;
-; CHECK-512-LABEL: nxv4f64:
-; CHECK-512:       // %bb.0:
-; CHECK-512-NEXT:    add x8, x0, #128
-; CHECK-512-NEXT:    ldr z1, [x0, #2, mul vl]
-; CHECK-512-NEXT:    ldr z0, [x8, #1, mul vl]
-; CHECK-512-NEXT:    add x8, x1, #128
-; CHECK-512-NEXT:    str z0, [x8, #1, mul vl]
-; CHECK-512-NEXT:    str z1, [x1, #2, mul vl]
-; CHECK-512-NEXT:    ret
-;
-; CHECK-1024-LABEL: nxv4f64:
-; CHECK-1024:       // %bb.0:
-; CHECK-1024-NEXT:    add x8, x0, #128
-; CHECK-1024-NEXT:    ldr z1, [x0, #1, mul vl]
-; CHECK-1024-NEXT:    ldr z0, [x8, #1, mul vl]
-; CHECK-1024-NEXT:    add x8, x1, #128
-; CHECK-1024-NEXT:    str z0, [x8, #1, mul vl]
-; CHECK-1024-NEXT:    str z1, [x1, #1, mul vl]
-; CHECK-1024-NEXT:    ret
-;
-; CHECK-2048-LABEL: nxv4f64:
-; CHECK-2048:       // %bb.0:
-; CHECK-2048-NEXT:    ptrue p0.d
-; CHECK-2048-NEXT:    mov x8, #16 // =0x10
-; CHECK-2048-NEXT:    add x9, x0, #128
-; CHECK-2048-NEXT:    ldr z1, [x9, #1, mul vl]
-; CHECK-2048-NEXT:    add x9, x1, #128
-; CHECK-2048-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
-; CHECK-2048-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
-; CHECK-2048-NEXT:    str z1, [x9, #1, mul vl]
-; CHECK-2048-NEXT:    ret
-  %ldoff = getelementptr inbounds nuw i8, ptr %ldptr, i64 128
-  %stoff = getelementptr inbounds nuw i8, ptr %stptr, i64 128
-  %x = load <vscale x 4 x double>, ptr %ldoff, align 1
-  store <vscale x 4 x double> %x, ptr %stoff, align 1
-  ret void
-}
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
index 2d4cdfa7278b9..e33bc8da97c05 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
@@ -30,64 +30,64 @@ define void @crash_when_lowering_extract_shuffle(ptr %dst, i1 %cond) vscale_rang
 ; CHECK-NEXT:  // %bb.1: // %vector.body
 ; CHECK-NEXT:    mov z0.b, #0 // =0x0
 ; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    mov x9, #24 // =0x18
+; CHECK-NEXT:    mov x9, #8 // =0x8
+; CHECK-NEXT:    mov x10, #24 // =0x18
 ; CHECK-NEXT:    umov w8, v0.b[8]
-; CHECK-NEXT:    mov v2.16b, v0.16b
-; CHECK-NEXT:    mov z3.d, z0.d
-; CHECK-NEXT:    mov v2.b[1], v0.b[1]
-; CHECK-NEXT:    ext z3.b, z3.b, z0.b, #16
-; CHECK-NEXT:    fmov s1, w8
-; CHECK-NEXT:    mov x8, #8 // =0x8
-; CHECK-NEXT:    ext v4.16b, v3.16b, v3.16b, #8
-; CHECK-NEXT:    mov v1.b[1], v0.b[9]
-; CHECK-NEXT:    mov v2.b[2], v0.b[2]
-; CHECK-NEXT:    mov v1.b[2], v0.b[10]
-; CHECK-NEXT:    mov v2.b[3], v0.b[3]
-; CHECK-NEXT:    mov v1.b[3], v0.b[11]
-; CHECK-NEXT:    mov v2.b[4], v0.b[4]
-; CHECK-NEXT:    mov v1.b[4], v0.b[12]
-; CHECK-NEXT:    mov v2.b[5], v0.b[5]
-; CHECK-NEXT:    mov v1.b[5], v0.b[13]
-; CHECK-NEXT:    mov v2.b[6], v0.b[6]
-; CHECK-NEXT:    mov v1.b[6], v0.b[14]
-; CHECK-NEXT:    mov v2.b[7], v0.b[7]
-; CHECK-NEXT:    mov v1.b[7], v0.b[15]
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    mov v1.b[1], v0.b[1]
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    mov x8, #16 // =0x10
+; CHECK-NEXT:    mov v2.b[1], v0.b[9]
+; CHECK-NEXT:    mov v1.b[2], v0.b[2]
+; CHECK-NEXT:    mov v2.b[2], v0.b[10]
+; CHECK-NEXT:    mov v1.b[3], v0.b[3]
+; CHECK-NEXT:    mov v2.b[3], v0.b[11]
+; CHECK-NEXT:    mov v1.b[4], v0.b[4]
+; CHECK-NEXT:    mov v2.b[4], v0.b[12]
+; CHECK-NEXT:    mov v1.b[5], v0.b[5]
+; CHECK-NEXT:    mov v2.b[5], v0.b[13]
+; CHECK-NEXT:    mov v1.b[6], v0.b[6]
+; CHECK-NEXT:    mov v2.b[6], v0.b[14]
+; CHECK-NEXT:    mov v1.b[7], v0.b[7]
+; CHECK-NEXT:    mov v2.b[7], v0.b[15]
+; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #16
+; CHECK-NEXT:    uunpklo z1.h, z1.b
+; CHECK-NEXT:    ext v3.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    uunpklo z0.h, z0.b
 ; CHECK-NEXT:    uunpklo z2.h, z2.b
-; CHECK-NEXT:    uunpklo z0.h, z1.b
-; CHECK-NEXT:    uunpklo z1.h, z3.b
-; CHECK-NEXT:    uunpklo z3.h, z4.b
-; CHECK-NEXT:    uunpklo z2.s, z2.h
-; CHECK-NEXT:    uunpklo z0.s, z0.h
 ; CHECK-NEXT:    uunpklo z1.s, z1.h
+; CHECK-NEXT:    uunpklo z3.h, z3.b
+; CHECK-NEXT:    uunpklo z0.s, z0.h
+; CHECK-NEXT:    uunpklo z2.s, z2.h
+; CHECK-NEXT:    lsl z1.s, z1.s, #31
 ; CHECK-NEXT:    uunpklo z3.s, z3.h
-; CHECK-NEXT:    lsl z2.s, z2.s, #31
 ; CHECK-NEXT:    lsl z0.s, z0.s, #31
-; CHECK-NEXT:    lsl z1.s, z1.s, #31
+; CHECK-NEXT:    asr z1.s, z1.s, #31
+; CHECK-NEXT:    lsl z2.s, z2.s, #31
+; CHECK-NEXT:    asr z0.s, z0.s, #31
+; CHECK-NEXT:    and z1.s, z1.s, #0x1
 ; CHECK-NEXT:    lsl z3.s, z3.s, #31
 ; CHECK-NEXT:    asr z2.s, z2.s, #31
-; CHECK-NEXT:    asr z0.s, z0.s, #31
-; CHECK-NEXT:    asr z1.s, z1.s, #31
+; CHECK-NEXT:    and z0.s, z0.s, #0x1
+; CHECK-NEXT:    cmpne p4.s, p0/z, z1.s, #0
+; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x0]
 ; CHECK-NEXT:    asr z3.s, z3.s, #31
 ; CHECK-NEXT:    and z2.s, z2.s, #0x1
-; CHECK-NEXT:    and z0.s, z0.s, #0x1
-; CHECK-NEXT:    and z1.s, z1.s, #0x1
-; CHECK-NEXT:    and z3.s, z3.s, #0x1
-; CHECK-NEXT:    cmpne p4.s, p0/z, z2.s, #0
-; CHECK-NEXT:    ld1w { z2.s }, p0/z, [x0]
 ; CHECK-NEXT:    cmpne p1.s, p0/z, z0.s, #0
-; CHECK-NEXT:    cmpne p2.s, p0/z, z1.s, #0
-; CHECK-NEXT:    cmpne p3.s, p0/z, z3.s, #0
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
-; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x0, #1, mul vl]
-; CHECK-NEXT:    ld1w { z3.s }, p0/z, [x0, x9, lsl #2]
-; CHECK-NEXT:    mov z2.s, p4/m, #0 // =0x0
+; CHECK-NEXT:    and z3.s, z3.s, #0x1
+; CHECK-NEXT:    cmpne p2.s, p0/z, z2.s, #0
+; CHECK-NEXT:    ld1w { z2.s }, p0/z, [x0, x9, lsl #2]
+; CHECK-NEXT:    mov z1.s, p4/m, #0 // =0x0
+; CHECK-NEXT:    cmpne p3.s, p0/z, z3.s, #0
+; CHECK-NEXT:    ld1w { z3.s }, p0/z, [x0, x10, lsl #2]
 ; CHECK-NEXT:    mov z0.s, p1/m, #0 // =0x0
-; CHECK-NEXT:    mov z1.s, p2/m, #0 // =0x0
-; CHECK-NEXT:    mov z3.s, p3/m, #0 // =0x0
-; CHECK-NEXT:    st1w { z2.s }, p0, [x0]
+; CHECK-NEXT:    mov z2.s, p2/m, #0 // =0x0
+; CHECK-NEXT:    st1w { z1.s }, p0, [x0]
 ; CHECK-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
-; CHECK-NEXT:    st1w { z1.s }, p0, [x0, #1, mul vl]
-; CHECK-NEXT:    st1w { z3.s }, p0, [x0, x9, lsl #2]
+; CHECK-NEXT:    mov z3.s, p3/m, #0 // =0x0
+; CHECK-NEXT:    st1w { z2.s }, p0, [x0, x9, lsl #2]
+; CHECK-NEXT:    st1w { z3.s }, p0, [x0, x10, lsl #2]
 ; CHECK-NEXT:  .LBB1_2: // %exit
 ; CHECK-NEXT:    ret
   %broadcast.splat = shufflevector <32 x i1> zeroinitializer, <32 x i1> zeroinitializer, <32 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
index 79921e25caf53..381c67c6d749e 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
@@ -55,40 +55,21 @@ define void @test_copysign_f16(ptr %ap, ptr %bp) {
 define void @test_copysign_bf16(ptr %ap, ptr %bp) {
 ; SVE-LABEL: test_copysign_bf16:
 ; SVE:       // %bb.0:
-; SVE-NEXT:    sub sp, sp, #16
-; SVE-NEXT:    .cfi_def_cfa_offset 16
-; SVE-NEXT:    ldr h0, [x0]
-; SVE-NEXT:    ldr h1, [x1]
-; SVE-NEXT:    fmov w8, s0
-; SVE-NEXT:    str h1, [sp, #12]
-; SVE-NEXT:    ldrb w9, [sp, #13]
-; SVE-NEXT:    and w8, w8, #0x7fff
-; SVE-NEXT:    tst w9, #0x80
-; SVE-NEXT:    fmov s0, w8
-; SVE-NEXT:    eor w8, w8, #0x8000
-; SVE-NEXT:    fmov s1, w8
-; SVE-NEXT:    fcsel h0, h1, h0, ne
+; SVE-NEXT:    ldr h0, [x1]
+; SVE-NEXT:    ldr h1, [x0]
+; SVE-NEXT:    and z0.h, z0.h, #0x8000
+; SVE-NEXT:    and z1.h, z1.h, #0x7fff
+; SVE-NEXT:    orr z0.d, z1.d, z0.d
 ; SVE-NEXT:    str h0, [x0]
-; SVE-NEXT:    add sp, sp, #16
 ; SVE-NEXT:    ret
 ;
 ; SVE2-LABEL: test_copysign_bf16:
 ; SVE2:       // %bb.0:
-; SVE2-NEXT:    sub sp, sp, #16
-; SVE2-NEXT:    .cfi_def_cfa_offset 16
-; SVE2-NEXT:    ldr h0, [x0]
+; SVE2-NEXT:    mov z0.h, #32767 // =0x7fff
 ; SVE2-NEXT:    ldr h1, [x1]
-; SVE2-NEXT:    fmov w8, s0
-; SVE2-NEXT:    str h1, [sp, #12]
-; SVE2-NEXT:    ldrb w9, [sp, #13]
-; SVE2-NEXT:    and w8, w8, #0x7fff
-; SVE2-NEXT:    tst w9, #0x80
-; SVE2-NEXT:    fmov s0, w8
-; SVE2-NEXT:    eor w8, w8, #0x8000
-; SVE2-NEXT:    fmov s1, w8
-; SVE2-NEXT:    fcsel h0, h1, h0, ne
-; SVE2-NEXT:    str h0, [x0]
-; SVE2-NEXT:    add sp, sp, #16
+; SVE2-NEXT:    ldr h2, [x0]
+; SVE2-NEXT:    bsl z2.d, z2.d, z1.d, z0.d
+; SVE2-NEXT:    str h2, [x0]
 ; SVE2-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: test_copysign_bf16:
diff --git a/llvm/test/CodeGen/AArch64/sve2-fcopysign.ll b/llvm/test/CodeGen/AArch64/sve2-fcopysign.ll
index 624e4ef03f440..8510f16ba52c9 100644
--- a/llvm/test/CodeGen/AArch64/sve2-fcopysign.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-fcopysign.ll
@@ -173,6 +173,18 @@ define <vscale x 2 x float> @copysign_nxv2f32_nxv2f64(<vscale x 2 x float> %a, <
   ret <vscale x 2 x float> %r
 }
 
+define <vscale x 2 x float> @copysign_nxv2f32_nxv2bf16(<vscale x 2 x float> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: copysign_nxv2f32_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z2.s, #0x7fffffff
+; CHECK-NEXT:    lsl z1.s, z1.s, #16
+; CHECK-NEXT:    bsl z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT:    ret
+  %tmp0 = fpext <vscale x 2 x bfloat> %b to <vscale x 2 x float>
+  %r = call <vscale x 2 x float> @llvm.copysign.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %tmp0)
+  ret <vscale x 2 x float> %r
+}
+
 ;
 ; llvm.copysign.nxv4f32
 ;
@@ -215,6 +227,18 @@ define <vscale x 4 x float> @copysign_nxv4f32_nxv4f64(<vscale x 4 x float> %a, <
   ret <vscale x 4 x float> %r
 }
 
+define <vscale x 4 x float> @copysign_nxv4f32_nxv4bf16(<vscale x 4 x float> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: copysign_nxv4f32_nxv4bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z2.s, #0x7fffffff
+; CHECK-NEXT:    lsl z1.s, z1.s, #16
+; CHECK-NEXT:    bsl z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT:    ret
+  %tmp0 = fpext <vscale x 4 x bfloat> %b to <vscale x 4 x float>
+  %r = call <vscale x 4 x float> @llvm.copysign.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %tmp0)
+  ret <vscale x 4 x float> %r
+}
+
 ;
 ; llvm.copysign.nxv2f64
 ;
@@ -255,9 +279,163 @@ define <vscale x 2 x double> @copysign_nxv2f64_nxv2f64(<vscale x 2 x double> %a,
   ret <vscale x 2 x double> %r
 }
 
+define <vscale x 2 x double> @copysign_nxv2f64_nxv2bf16(<vscale x 2 x double> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: copysign_nxv2f64_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z1.s, z1.s, #16
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    fcvt z1.d, p0/m, z1.s
+; CHECK-NEXT:    bsl z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT:    ret
+  %b.ext = fpext <vscale x 2 x bfloat> %b to <vscale x 2 x double>
+  %r = call <vscale x 2 x double> @llvm.copysign.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b.ext)
+  ret <vscale x 2 x double> %r
+}
+
+;
+; llvm.copysign.nxv2bf16
+;
+
+define <vscale x 2 x bfloat> @copysign_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: copysign_nxv2bf16_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z2.h, #32767 // =0x7fff
+; CHECK-NEXT:    bsl z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT:    ret
+  %r = call <vscale x 2 x bfloat> @llvm.copysign.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b)
+  ret <vscale x 2 x bfloat> %r
+}
+
+define <vscale x 2 x bfloat> @copysign_nxv2bf16_nxv2f32(<vscale x 2 x bfloat> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: copysign_nxv2bf16_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov z2.h, #32767 // =0x7fff
+; CHECK-NEXT:    bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT:    bsl z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT:    ret
+  %tmp0 = fptrunc <vscale x 2 x float> %b to <vscale x 2 x bfloat>
+  %r = call <vscale x 2 x bfloat> @llvm.copysign.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %tmp0)
+  ret <vscale x 2 x bfloat> %r
+}
+
+define <vscale x 2 x bfloat> @copysign_nxv2bf16_nxv2f64(<vscale x 2 x bfloat> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: copysign_nxv2bf16_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov z2.h, #32767 // =0x7fff
+; CHECK-NEXT:    fcvtx z1.s, p0/m, z1.d
+; CHECK-NEXT:    bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT:    bsl z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT:    ret
+  %tmp0 = fptrunc <vscale x 2 x double> %b to <vscale x 2 x bfloat>
+  %r = call <vscale x 2 x bfloat> @llvm.copysign.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %tmp0)
+  ret <vscale x 2 x bfloat> %r
+}
+
+;
+; llvm.copysign.nxv2bf16
+;
+
+define <vscale x 4 x bfloat> @copysign_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: copysign_nxv4bf16_nxv4bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z2.h, #32767 // =0x7fff
+; CHECK-NEXT:    bsl z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT:    ret
+  %r = call <vscale x 4 x bfloat> @llvm.copysign.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b)
+  ret <vscale x 4 x bfloat> %r
+}
+
+define <vscale x 4 x bfloat> @copysign_nxv4bf16_nxv4f32(<vscale x 4 x bfloat> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: copysign_nxv4bf16_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    mov z2.h, #32767 // =0x7fff
+; CHECK-NEXT:    bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT:    bsl z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT:    ret
+  %b.trunc = fptrunc <vscale x 4 x float> %b to <vscale x 4 x bfloat>
+  %r = call <vscale x 4 x bfloat> @llvm.copysign.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b.trunc)
+  ret <vscale x 4 x bfloat> %r
+}
+
+define <vscale x 4 x bfloat> @copysign_nxv4bf16_nxv4f64(<vscale x 4 x bfloat> %a, <vscale x 4 x double> %b) {
+; CHECK-LABEL: copysign_nxv4bf16_nxv4f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov z3.h, #32767 // =0x7fff
+; CHECK-NEXT:    fcvtx z2.s, p0/m, z2.d
+; CHECK-NEXT:    fcvtx z1.s, p0/m, z1.d
+; CHECK-NEXT:    bfcvt z2.h, p0/m, z2.s
+; CHECK-NEXT:    bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT:    uzp1 z1.s, z1.s, z2.s
+; CHECK-NEXT:    bsl z0.d, z0.d, z1.d, z3.d
+; CHECK-NEXT:    ret
+  %b.trunc = fptrunc <vscale x 4 x double> %b to <vscale x 4 x bfloat>
+  %r = call <vscale x 4 x bfloat> @llvm.copysign.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b.trunc)
+  ret <vscale x 4 x bfloat> %r
+}
+
+;
+; llvm.copysign.nxv8bf16
+;
+
+define <vscale x 8 x bfloat> @copysign_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: copysign_nxv8bf16_nxv8bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z2.h, #32767 // =0x7fff
+; CHECK-NEXT:    bsl z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT:    ret
+  %r = call <vscale x 8 x bfloat> @llvm.copysign.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b)
+  ret <vscale x 8 x bfloat> %r
+}
+
+define <vscale x 8 x bfloat> @copysign_nxv8bf16_nxv8f32(<vscale x 8 x bfloat> %a, <vscale x 8 x float> %b) {
+; CHECK-LABEL: copysign_nxv8bf16_nxv8f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    mov z3.h, #32767 // =0x7fff
+; CHECK-NEXT:    bfcvt z2.h, p0/m, z2.s
+; CHECK-NEXT:    bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT:    uzp1 z1.h, z1.h, z2.h
+; CHECK-NEXT:    bsl z0.d, z0.d, z1.d, z3.d
+; CHECK-NEXT:    ret
+  %b.trunc = fptrunc <vscale x 8 x float> %b to <vscale x 8 x bfloat>
+  %r = call <vscale x 8 x bfloat> @llvm.copysign.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b.trunc)
+  ret <vscale x 8 x bfloat> %r
+}
+
+define <vscale x 8 x bfloat> @copysign_nxv8bf16_nxv8f64(<vscale x 8 x bfloat> %a, <vscale x 8 x double> %b) {
+; CHECK-LABEL: copysign_nxv8bf16_nxv8f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fcvtx z4.s, p0/m, z4.d
+; CHECK-NEXT:    fcvtx z3.s, p0/m, z3.d
+; CHECK-NEXT:    fcvtx z2.s, p0/m, z2.d
+; CHECK-NEXT:    fcvtx z1.s, p0/m, z1.d
+; CHECK-NEXT:    bfcvt z4.h, p0/m, z4.s
+; CHECK-NEXT:    bfcvt z3.h, p0/m, z3.s
+; CHECK-NEXT:    bfcvt z2.h, p0/m, z2.s
+; CHECK-NEXT:    bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT:    uzp1 z3.s, z3.s, z4.s
+; CHECK-NEXT:    uzp1 z1.s, z1.s, z2.s
+; CHECK-NEXT:    mov z2.h, #32767 // =0x7fff
+; CHECK-NEXT:    uzp1 z1.h, z1.h, z3.h
+; CHECK-NEXT:    bsl z0.d, z0.d, z1.d, z2.d
+; CHECK-NEXT:    ret
+  %b.trunc = fptrunc <vscale x 8 x double> %b to <vscale x 8 x bfloat>
+  %r = call <vscale x 8 x bfloat> @llvm.copysign.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b.trunc)
+  ret <vscale x 8 x bfloat> %r
+}
+
 declare <vscale x 2 x half> @llvm.copysign.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> %b)
 declare <vscale x 4 x half> @llvm.copysign.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %b)
 declare <vscale x 8 x half> @llvm.copysign.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b)
 declare <vscale x 2 x float> @llvm.copysign.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b)
 declare <vscale x 4 x float> @llvm.copysign.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b)
 declare <vscale x 2 x double> @llvm.copysign.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b)
+declare <vscale x 2 x bfloat> @llvm.copysign.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b)
+declare <vscale x 4 x bfloat> @llvm.copysign.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b)
+declare <vscale x 8 x bfloat> @llvm.copysign.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
index 89078f20f1d47..1a94429b1b5a1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
@@ -64,7 +64,7 @@ define amdgpu_ps { i32, i32 } @s_andn2_i32_multi_use(i32 inreg %src0, i32 inreg
 ; GFX11-NEXT:    ; return to shader part epilog
   %not.src1 = xor i32 %src1, -1
   %and = and i32 %src0, %not.src1
-  %insert.0 = insertvalue { i32, i32 } undef, i32 %and, 0
+  %insert.0 = insertvalue { i32, i32 } poison, i32 %and, 0
   %insert.1 = insertvalue { i32, i32 } %insert.0, i32 %not.src1, 1
   ret { i32, i32 } %insert.1
 }
@@ -90,7 +90,7 @@ define amdgpu_ps { i32, i32 } @s_andn2_i32_multi_foldable_use(i32 inreg %src0, i
   %not.src2 = xor i32 %src2, -1
   %and0 = and i32 %src0, %not.src2
   %and1 = and i32 %src1, %not.src2
-  %insert.0 = insertvalue { i32, i32 } undef, i32 %and0, 0
+  %insert.0 = insertvalue { i32, i32 } poison, i32 %and0, 0
   %insert.1 = insertvalue { i32, i32 } %insert.0, i32 %and1, 1
   ret { i32, i32 } %insert.1
 }
@@ -211,7 +211,7 @@ define amdgpu_ps { i64, i64 } @s_andn2_i64_multi_foldable_use(i64 inreg %src0, i
   %not.src2 = xor i64 %src2, -1
   %and0 = and i64 %src0, %not.src2
   %and1 = and i64 %src1, %not.src2
-  %insert.0 = insertvalue { i64, i64 } undef, i64 %and0, 0
+  %insert.0 = insertvalue { i64, i64 } poison, i64 %and0, 0
   %insert.1 = insertvalue { i64, i64 } %insert.0, i64 %and1, 1
   ret { i64, i64 } %insert.1
 }
@@ -238,7 +238,7 @@ define amdgpu_ps { i64, i64 } @s_andn2_i64_multi_use(i64 inreg %src0, i64 inreg
 ; GFX11-NEXT:    ; return to shader part epilog
   %not.src1 = xor i64 %src1, -1
   %and = and i64 %src0, %not.src1
-  %insert.0 = insertvalue { i64, i64 } undef, i64 %and, 0
+  %insert.0 = insertvalue { i64, i64 } poison, i64 %and, 0
   %insert.1 = insertvalue { i64, i64 } %insert.0, i64 %not.src1, 1
   ret { i64, i64 } %insert.1
 }
@@ -408,7 +408,7 @@ define amdgpu_ps { i16, i16 } @s_andn2_i16_multi_use(i16 inreg %src0, i16 inreg
 ; GFX11-NEXT:    ; return to shader part epilog
   %not.src1 = xor i16 %src1, -1
   %and = and i16 %src0, %not.src1
-  %insert.0 = insertvalue { i16, i16 } undef, i16 %and, 0
+  %insert.0 = insertvalue { i16, i16 } poison, i16 %and, 0
   %insert.1 = insertvalue { i16, i16 } %insert.0, i16 %not.src1, 1
   ret { i16, i16 } %insert.1
 }
@@ -434,7 +434,7 @@ define amdgpu_ps { i16, i16 } @s_andn2_i16_multi_foldable_use(i16 inreg %src0, i
   %not.src2 = xor i16 %src2, -1
   %and0 = and i16 %src0, %not.src2
   %and1 = and i16 %src1, %not.src2
-  %insert.0 = insertvalue { i16, i16 } undef, i16 %and0, 0
+  %insert.0 = insertvalue { i16, i16 } poison, i16 %and0, 0
   %insert.1 = insertvalue { i16, i16 } %insert.0, i16 %and1, 1
   ret { i16, i16 } %insert.1
 }
@@ -601,7 +601,7 @@ define amdgpu_ps { i32, i32 } @s_andn2_v2i16_multi_use(<2 x i16> inreg %src0, <2
 
   %cast.0 = bitcast <2 x i16> %and to i32
   %cast.1 = bitcast <2 x i16> %not.src1 to i32
-  %insert.0 = insertvalue { i32, i32 } undef, i32 %cast.0, 0
+  %insert.0 = insertvalue { i32, i32 } poison, i32 %cast.0, 0
   %insert.1 = insertvalue { i32, i32 } %insert.0, i32 %cast.1, 1
   ret { i32, i32 } %insert.1
 }
@@ -646,7 +646,7 @@ define amdgpu_ps { i32, i32 } @s_andn2_v2i16_multi_foldable_use(<2 x i16> inreg
 
   %cast.0 = bitcast <2 x i16> %and0 to i32
   %cast.1 = bitcast <2 x i16> %and1 to i32
-  %insert.0 = insertvalue { i32, i32 } undef, i32 %cast.0, 0
+  %insert.0 = insertvalue { i32, i32 } poison, i32 %cast.0, 0
   %insert.1 = insertvalue { i32, i32 } %insert.0, i32 %cast.1, 1
   ret { i32, i32 } %insert.1
 }
@@ -857,7 +857,7 @@ define amdgpu_ps { i48, i48 } @s_andn2_v3i16_multi_use(<3 x i16> inreg %src0, <3
   %and = and <3 x i16> %src0, %not.src1
   %cast.0 = bitcast <3 x i16> %and to i48
   %cast.1 = bitcast <3 x i16> %not.src1 to i48
-  %insert.0 = insertvalue { i48, i48 } undef, i48 %cast.0, 0
+  %insert.0 = insertvalue { i48, i48 } poison, i48 %cast.0, 0
   %insert.1 = insertvalue { i48, i48 } %insert.0, i48 %cast.1, 1
   ret { i48, i48 } %insert.1
 }
@@ -1028,7 +1028,7 @@ define amdgpu_ps { i64, i64 } @s_andn2_v4i16_multi_use(<4 x i16> inreg %src0, <4
 
   %cast.0 = bitcast <4 x i16> %and to i64
   %cast.1 = bitcast <4 x i16> %not.src1 to i64
-  %insert.0 = insertvalue { i64, i64 } undef, i64 %cast.0, 0
+  %insert.0 = insertvalue { i64, i64 } poison, i64 %cast.0, 0
   %insert.1 = insertvalue { i64, i64 } %insert.0, i64 %cast.1, 1
   ret { i64, i64 } %insert.1
 }
@@ -1082,7 +1082,7 @@ define amdgpu_ps { i64, i64 } @s_andn2_v4i16_multi_foldable_use(<4 x i16> inreg
 
   %cast.0 = bitcast <4 x i16> %and0 to i64
   %cast.1 = bitcast <4 x i16> %and1 to i64
-  %insert.0 = insertvalue { i64, i64 } undef, i64 %cast.0, 0
+  %insert.0 = insertvalue { i64, i64 } poison, i64 %cast.0, 0
   %insert.1 = insertvalue { i64, i64 } %insert.0, i64 %cast.1, 1
   ret { i64, i64 } %insert.1
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/bug-legalization-artifact-combiner-dead-def.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/bug-legalization-artifact-combiner-dead-def.ll
index 39423ea22cbb4..d317a3ef54162 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/bug-legalization-artifact-combiner-dead-def.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/bug-legalization-artifact-combiner-dead-def.ll
@@ -12,7 +12,7 @@ define void @value_finder_bug(ptr addrspace(5) %store_ptr, ptr addrspace(4) %ptr
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %vec = load <4 x float>, ptr addrspace(4) %ptr, align 4
   %vec.3 = extractelement <4 x float> %vec, i32 3
-  %shuffle = shufflevector <4 x float> %vec, <4 x float> undef, <2 x i32> <i32 2, i32 undef>
+  %shuffle = shufflevector <4 x float> %vec, <4 x float> poison, <2 x i32> <i32 2, i32 poison>
   %new_vec = insertelement <2 x float> %shuffle, float %vec.3, i32 1
   store <2 x float> %new_vec, ptr addrspace(5) %store_ptr, align 8
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll
index 4ddbb0afd7fc5..357e9d6530ce8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll
@@ -266,7 +266,7 @@ define <4 x float> @v_uitofp_unpack_i32_to_v4f32(i32 %arg0) nounwind {
   %mask.lshr.24 = and i32 %lshr.24, 255
   %cvt3 = uitofp i32 %mask.lshr.24 to float
 
-  %ins.0 = insertelement <4 x float> undef, float %cvt0, i32 0
+  %ins.0 = insertelement <4 x float> poison, float %cvt0, i32 0
   %ins.1 = insertelement <4 x float> %ins.0, float %cvt1, i32 1
   %ins.2 = insertelement <4 x float> %ins.1, float %cvt2, i32 2
   %ins.3 = insertelement <4 x float> %ins.2, float %cvt3, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll
index c5ded11c7d323..11acd451d98d7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -new-reg-bank-select -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
 
 ; Divergent phis that don't require lowering using lane mask merging
 
@@ -101,27 +101,28 @@ define void @divergent_i1_phi_used_inside_loop(float %val, ptr %addr) {
 ; GFX10-LABEL: divergent_i1_phi_used_inside_loop:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s5, 0
-; GFX10-NEXT:    v_mov_b32_e32 v3, 1
-; GFX10-NEXT:    v_mov_b32_e32 v4, s5
-; GFX10-NEXT:    ; implicit-def: $sgpr6
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    s_mov_b32 s5, 1
+; GFX10-NEXT:    s_mov_b32 s6, 0
+; GFX10-NEXT:    ; implicit-def: $sgpr7
 ; GFX10-NEXT:  .LBB2_1: ; %loop
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    v_xor_b32_e32 v3, 1, v3
-; GFX10-NEXT:    v_cvt_f32_u32_e32 v5, v4
-; GFX10-NEXT:    v_add_nc_u32_e32 v4, 1, v4
-; GFX10-NEXT:    v_and_b32_e32 v6, 1, v3
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v0
-; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, v6
-; GFX10-NEXT:    s_or_b32 s5, vcc_lo, s5
-; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
-; GFX10-NEXT:    s_and_b32 s4, exec_lo, s4
-; GFX10-NEXT:    s_or_b32 s6, s6, s4
-; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v3, s6
+; GFX10-NEXT:    s_xor_b32 s5, s5, 1
+; GFX10-NEXT:    s_and_b32 s8, s5, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v0
+; GFX10-NEXT:    s_cselect_b32 s8, exec_lo, 0
+; GFX10-NEXT:    s_add_i32 s6, s6, 1
+; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT:    s_andn2_b32 s7, s7, exec_lo
+; GFX10-NEXT:    s_and_b32 s8, exec_lo, s8
+; GFX10-NEXT:    s_or_b32 s7, s7, s8
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
 ; GFX10-NEXT:    s_cbranch_execnz .LBB2_1
 ; GFX10-NEXT:  ; %bb.2: ; %exit
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s6
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s7
 ; GFX10-NEXT:    flat_store_dword v[1:2], v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -147,46 +148,46 @@ define void @divergent_i1_phi_used_inside_loop_bigger_loop_body(float %val, floa
 ; GFX10-LABEL: divergent_i1_phi_used_inside_loop_bigger_loop_body:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s4, 0
-; GFX10-NEXT:    v_cmp_lt_f32_e64 s5, 1.0, v1
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s4, 1.0, v1
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0x3e8
-; GFX10-NEXT:    v_mov_b32_e32 v8, s4
-; GFX10-NEXT:    ; implicit-def: $sgpr6
+; GFX10-NEXT:    s_mov_b32 s5, 0
+; GFX10-NEXT:    s_mov_b32 s6, 0
+; GFX10-NEXT:    ; implicit-def: $sgpr7
 ; GFX10-NEXT:    s_branch .LBB3_2
 ; GFX10-NEXT:  .LBB3_1: ; %loop_body
 ; GFX10-NEXT:    ; in Loop: Header=BB3_2 Depth=1
-; GFX10-NEXT:    v_cvt_f32_u32_e32 v9, v8
-; GFX10-NEXT:    s_xor_b32 s5, s5, -1
-; GFX10-NEXT:    v_add_nc_u32_e32 v8, 1, v8
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v9, v0
-; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
-; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
-; GFX10-NEXT:    s_and_b32 s7, exec_lo, s5
-; GFX10-NEXT:    s_or_b32 s6, s6, s7
-; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v8, s6
+; GFX10-NEXT:    s_mov_b32 s8, exec_lo
+; GFX10-NEXT:    s_add_i32 s6, s6, 1
+; GFX10-NEXT:    s_xor_b32 s4, s4, s8
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v0
+; GFX10-NEXT:    s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT:    s_andn2_b32 s7, s7, exec_lo
+; GFX10-NEXT:    s_and_b32 s8, exec_lo, s4
+; GFX10-NEXT:    s_or_b32 s7, s7, s8
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s5
 ; GFX10-NEXT:    s_cbranch_execz .LBB3_6
 ; GFX10-NEXT:  .LBB3_2: ; %loop_start
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    s_mov_b32 s7, 1
-; GFX10-NEXT:    v_cmp_ge_i32_e32 vcc_lo, 0x3e8, v8
-; GFX10-NEXT:    s_cbranch_vccz .LBB3_4
+; GFX10-NEXT:    s_cmpk_le_i32 s6, 0x3e8
+; GFX10-NEXT:    s_mov_b32 s8, 1
+; GFX10-NEXT:    s_cbranch_scc0 .LBB3_4
 ; GFX10-NEXT:  ; %bb.3: ; %else
 ; GFX10-NEXT:    ; in Loop: Header=BB3_2 Depth=1
-; GFX10-NEXT:    s_mov_b32 s7, 0
+; GFX10-NEXT:    s_mov_b32 s8, 0
 ; GFX10-NEXT:    flat_store_dword v[6:7], v1
 ; GFX10-NEXT:  .LBB3_4: ; %Flow
 ; GFX10-NEXT:    ; in Loop: Header=BB3_2 Depth=1
-; GFX10-NEXT:    s_xor_b32 s7, s7, 1
-; GFX10-NEXT:    s_and_b32 s7, s7, 1
-; GFX10-NEXT:    s_cmp_lg_u32 s7, 0
+; GFX10-NEXT:    s_xor_b32 s8, s8, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
 ; GFX10-NEXT:    s_cbranch_scc1 .LBB3_1
 ; GFX10-NEXT:  ; %bb.5: ; %if
 ; GFX10-NEXT:    ; in Loop: Header=BB3_2 Depth=1
 ; GFX10-NEXT:    flat_store_dword v[4:5], v1
 ; GFX10-NEXT:    s_branch .LBB3_1
 ; GFX10-NEXT:  .LBB3_6: ; %exit
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s6
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s7
 ; GFX10-NEXT:    flat_store_dword v[2:3], v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -234,45 +235,47 @@ define amdgpu_cs void @single_lane_execution_attribute(i32 inreg %.userdata0, <3
 ; GFX10-NEXT:    s_mov_b32 s1, 0
 ; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v1, -1, 0
 ; GFX10-NEXT:    s_or_b64 s[12:13], s[4:5], s[0:1]
-; GFX10-NEXT:    s_mov_b32 s3, -1
 ; GFX10-NEXT:    s_load_dwordx8 s[4:11], s[12:13], 0x0
 ; GFX10-NEXT:    v_mbcnt_hi_u32_b32 v1, -1, v1
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v1
-; GFX10-NEXT:    v_xor_b32_e32 v3, 1, v1
-; GFX10-NEXT:    v_and_b32_e32 v3, 1, v3
+; GFX10-NEXT:    v_and_b32_e32 v3, 1, v1
 ; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v3
-; GFX10-NEXT:    ; implicit-def: $vgpr3
+; GFX10-NEXT:    s_xor_b32 s3, vcc_lo, exec_lo
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    buffer_load_dword v2, v2, s[4:7], 0 offen
+; GFX10-NEXT:    s_and_b32 vcc_lo, exec_lo, s3
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 0, v2
 ; GFX10-NEXT:    s_cbranch_vccnz .LBB4_4
 ; GFX10-NEXT:  ; %bb.1: ; %.preheader.preheader
-; GFX10-NEXT:    v_mov_b32_e32 v3, s1
-; GFX10-NEXT:    v_mov_b32_e32 v4, s1
+; GFX10-NEXT:    s_mov_b32 s3, 0
 ; GFX10-NEXT:  .LBB4_2: ; %.preheader
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    buffer_load_dword v5, v3, s[4:7], 0 offen
+; GFX10-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX10-NEXT:    v_add_nc_u32_e32 v1, -1, v1
-; GFX10-NEXT:    v_add_nc_u32_e32 v3, 4, v3
+; GFX10-NEXT:    s_add_i32 s1, s1, 4
+; GFX10-NEXT:    buffer_load_dword v3, v3, s[4:7], 0 offen
 ; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v1
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_add_nc_u32_e32 v4, v5, v4
+; GFX10-NEXT:    v_readfirstlane_b32 s12, v3
+; GFX10-NEXT:    s_add_i32 s3, s12, s3
 ; GFX10-NEXT:    s_cbranch_vccnz .LBB4_2
 ; GFX10-NEXT:  ; %bb.3: ; %.preheader._crit_edge
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v4, v2
-; GFX10-NEXT:    s_mov_b32 s3, 0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, s3, v2
 ; GFX10-NEXT:    s_or_b32 s1, s0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s1
-; GFX10-NEXT:  .LBB4_4: ; %Flow
-; GFX10-NEXT:    s_and_b32 vcc_lo, exec_lo, s3
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s1
+; GFX10-NEXT:    s_branch .LBB4_6
+; GFX10-NEXT:  .LBB4_4:
+; GFX10-NEXT:    s_mov_b32 s1, exec_lo
+; GFX10-NEXT:    ; implicit-def: $vgpr1
+; GFX10-NEXT:    s_and_b32 vcc_lo, exec_lo, s1
 ; GFX10-NEXT:    s_cbranch_vccz .LBB4_6
 ; GFX10-NEXT:  ; %bb.5: ; %.19
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX10-NEXT:    v_or_b32_e32 v3, 2, v1
+; GFX10-NEXT:    v_or_b32_e32 v1, 2, v1
 ; GFX10-NEXT:  .LBB4_6: ; %.22
 ; GFX10-NEXT:    v_add_lshl_u32 v0, v0, s2, 2
-; GFX10-NEXT:    buffer_store_dword v3, v0, s[8:11], 0 offen
+; GFX10-NEXT:    buffer_store_dword v1, v0, s[8:11], 0 offen
 ; GFX10-NEXT:    s_endpgm
 .entry:
   %.0 = call i64 @llvm.amdgcn.s.getpc()
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.mir
index 6594d7f504212..c0fbdb541ab9f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.mir
@@ -6,7 +6,7 @@
   define void @divergent_i1_phi_uniform_branch_simple() {ret void}
   define void @divergent_i1_phi_used_inside_loop() {ret void}
   define void @divergent_i1_phi_used_inside_loop_bigger_loop_body() {ret void}
-  define void @_amdgpu_cs_main() #0 {ret void}
+  define void @single_lane_execution_attribute() #0 {ret void}
 
   attributes #0 = {"amdgpu-flat-work-group-size"="1,1"}
 ...
@@ -60,10 +60,10 @@ body: |
   ; GFX10-NEXT:   G_BR %bb.2
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[COPY7]](s1)
   ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY7]](s1), [[C5]], [[C4]]
-  ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SEXT]], [[C4]]
+  ; GFX10-NEXT:   G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
   ; GFX10-NEXT:   S_ENDPGM 0
   bb.0:
     successors: %bb.1(0x30000000), %bb.2(0x50000000)
@@ -105,9 +105,9 @@ body: |
     G_BR %bb.2
 
   bb.4:
-    %16:_(s32) = G_CONSTANT i32 2
-    %17:_(s32) = G_CONSTANT i32 1
-    %18:_(s32) = G_SELECT %13(s1), %17, %16
+    %16:_(s32) = G_SEXT %13(s1)
+    %17:_(s32) = G_CONSTANT i32 2
+    %18:_(s32) = G_ADD %16, %17
     G_STORE %18(s32), %2(p1) :: (store (s32), addrspace 1)
     S_ENDPGM 0
 ...
@@ -149,10 +149,10 @@ body: |
   ; GFX10-NEXT: bb.2:
   ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[COPY4]](s1), %bb.0, [[S_OR_B32_]](s1), %bb.1
   ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
+  ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[COPY7]](s1)
   ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY7]](s1), [[C4]], [[C3]]
-  ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SEXT]], [[C3]]
+  ; GFX10-NEXT:   G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
   ; GFX10-NEXT:   S_ENDPGM 0
   bb.0:
     successors: %bb.1(0x30000000), %bb.2(0x50000000)
@@ -178,9 +178,9 @@ body: |
 
   bb.2:
     %11:_(s1) = G_PHI %6(s1), %bb.0, %10(s1), %bb.1
-    %12:_(s32) = G_CONSTANT i32 2
-    %13:_(s32) = G_CONSTANT i32 1
-    %14:_(s32) = G_SELECT %11(s1), %13, %12
+    %12:_(s32) = G_SEXT %11(s1)
+    %13:_(s32) = G_CONSTANT i32 2
+    %14:_(s32) = G_ADD %12, %13
     G_STORE %14(s32), %2(p1) :: (store (s32), addrspace 1)
     S_ENDPGM 0
 ...
@@ -199,17 +199,17 @@ body: |
   ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
-  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.1:
   ; GFX10-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %22(s1), %bb.1
-  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI %7(s32), %bb.1, [[C1]](s32), %bb.0
-  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.0, %9(s32), %bb.1
-  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[C]](s1), %bb.0, %11(s1), %bb.1
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %19(s1), %bb.1
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI %7(s32), %bb.1, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %9(s32), %bb.1
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[C1]](s1), %bb.0, %11(s1), %bb.1
   ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
   ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
   ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[PHI3]], [[C2]]
@@ -226,12 +226,10 @@ body: |
   ; GFX10-NEXT:   G_BR %bb.2
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.2:
-  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.1
-  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
-  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI4]](s32)
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
   ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
   ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
-  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY5]](s1), [[C5]], [[C4]]
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[S_OR_B32_]](s1), [[C5]], [[C4]]
   ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p0) :: (store (s32))
   ; GFX10-NEXT:   SI_RETURN
   bb.0:
@@ -242,15 +240,15 @@ body: |
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = COPY $vgpr2
     %3:_(p0) = G_MERGE_VALUES %1(s32), %2(s32)
-    %4:_(s1) = G_CONSTANT i1 true
-    %5:_(s32) = G_CONSTANT i32 0
+    %4:_(s32) = G_CONSTANT i32 0
+    %5:_(s1) = G_CONSTANT i1 true
 
   bb.1:
     successors: %bb.2(0x04000000), %bb.1(0x7c000000)
 
-    %6:_(s32) = G_PHI %7(s32), %bb.1, %5(s32), %bb.0
-    %8:_(s32) = G_PHI %5(s32), %bb.0, %9(s32), %bb.1
-    %10:_(s1) = G_PHI %4(s1), %bb.0, %11(s1), %bb.1
+    %6:_(s32) = G_PHI %7(s32), %bb.1, %4(s32), %bb.0
+    %8:_(s32) = G_PHI %4(s32), %bb.0, %9(s32), %bb.1
+    %10:_(s1) = G_PHI %5(s1), %bb.0, %11(s1), %bb.1
     %12:_(s1) = G_CONSTANT i1 true
     %11:_(s1) = G_XOR %10, %12
     %13:_(s32) = G_UITOFP %8(s32)
@@ -262,13 +260,11 @@ body: |
     G_BR %bb.2
 
   bb.2:
-    %16:_(s1) = G_PHI %11(s1), %bb.1
-    %17:_(s32) = G_PHI %7(s32), %bb.1
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %17(s32)
-    %18:_(s32) = G_FCONSTANT float 0.000000e+00
-    %19:_(s32) = G_FCONSTANT float 1.000000e+00
-    %20:_(s32) = G_SELECT %16(s1), %19, %18
-    G_STORE %20(s32), %3(p0) :: (store (s32))
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %7(s32)
+    %16:_(s32) = G_FCONSTANT float 0.000000e+00
+    %17:_(s32) = G_FCONSTANT float 1.000000e+00
+    %18:_(s32) = G_SELECT %11(s1), %17, %16
+    G_STORE %18(s32), %3(p0) :: (store (s32))
     SI_RETURN
 ...
 
@@ -293,31 +289,31 @@ body: |
   ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
   ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
   ; GFX10-NEXT:   [[MV2:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
-  ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[COPY1]](s32), [[C1]]
-  ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[FCMP]](s1)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[COPY1]](s32), [[C]]
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+  ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[FCMP]](s1)
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.1:
   ; GFX10-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %42(s1), %bb.5
-  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[COPY8]](s1), %bb.0, %39(s1), %bb.5
-  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI %15(s32), %bb.5, [[C]](s32), %bb.0
-  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %17(s32), %bb.5
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[COPY8]](s1), %bb.0, %44(s1), %bb.5
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %36(s1), %bb.5
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI %15(s32), %bb.5, [[C1]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.0, %17(s32), %bb.5
   ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
   ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1)
-  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1000
-  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(sle), [[PHI3]](s32), [[C3]]
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1000
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(sle), [[PHI3]](s32), [[C2]]
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
   ; GFX10-NEXT:   G_BRCOND [[ICMP]](s1), %bb.4
   ; GFX10-NEXT:   G_BR %bb.2
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.2:
   ; GFX10-NEXT:   successors: %bb.3(0x40000000), %bb.5(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s1) = G_PHI %24(s1), %bb.4, [[C2]](s1), %bb.1
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s1) = G_PHI %24(s1), %bb.4, [[C3]](s1), %bb.1
   ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
   ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[PHI4]], [[C4]]
   ; GFX10-NEXT:   G_BRCOND [[XOR]](s1), %bb.5
@@ -333,36 +329,34 @@ body: |
   ; GFX10-NEXT: bb.4:
   ; GFX10-NEXT:   successors: %bb.2(0x80000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
-  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 1000
-  ; GFX10-NEXT:   G_STORE [[C7]](s32), [[MV2]](p0) :: (store (s32))
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1000
+  ; GFX10-NEXT:   G_STORE [[C6]](s32), [[MV2]](p0) :: (store (s32))
+  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
   ; GFX10-NEXT:   G_BR %bb.2
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.5:
   ; GFX10-NEXT:   successors: %bb.6(0x04000000), %bb.1(0x7c000000)
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT:   [[C8:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[COPY10]], [[C8]]
+  ; GFX10-NEXT:   [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[COPY9]], [[C8]]
   ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[XOR1]](s1)
   ; GFX10-NEXT:   [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[PHI3]](s32)
   ; GFX10-NEXT:   [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
   ; GFX10-NEXT:   [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
   ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI3]], [[C9]]
   ; GFX10-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP1]](s1), [[PHI2]](s32)
-  ; GFX10-NEXT:   [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[XOR1]](s1)
-  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY9]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY10]](s1), $exec_lo, implicit-def $scc
   ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY11]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[XOR1]](s1)
   ; GFX10-NEXT:   SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.6
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.6:
-  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.5
-  ; GFX10-NEXT:   [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
-  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI5]](s32)
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
   ; GFX10-NEXT:   [[C10:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
   ; GFX10-NEXT:   [[C11:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
-  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY13]](s1), [[C11]], [[C10]]
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[S_OR_B32_]](s1), [[C11]], [[C10]]
   ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p0) :: (store (s32))
   ; GFX10-NEXT:   SI_RETURN
   bb.0:
@@ -380,26 +374,26 @@ body: |
     %8:_(s32) = COPY $vgpr6
     %9:_(s32) = COPY $vgpr7
     %10:_(p0) = G_MERGE_VALUES %8(s32), %9(s32)
-    %11:_(s32) = G_CONSTANT i32 0
-    %12:_(s32) = G_FCONSTANT float 1.000000e+00
-    %13:_(s1) = G_FCMP floatpred(ogt), %1(s32), %12
+    %11:_(s32) = G_FCONSTANT float 1.000000e+00
+    %12:_(s1) = G_FCMP floatpred(ogt), %1(s32), %11
+    %13:_(s32) = G_CONSTANT i32 0
 
   bb.1:
     successors: %bb.4(0x40000000), %bb.2(0x40000000)
 
-    %14:_(s32) = G_PHI %15(s32), %bb.5, %11(s32), %bb.0
-    %16:_(s32) = G_PHI %11(s32), %bb.0, %17(s32), %bb.5
-    %18:_(s1) = G_PHI %13(s1), %bb.0, %19(s1), %bb.5
-    %20:_(s1) = G_CONSTANT i1 true
-    %21:_(s32) = G_CONSTANT i32 1000
-    %22:_(s1) = G_ICMP intpred(sle), %16(s32), %21
-    G_BRCOND %22(s1), %bb.4
+    %14:_(s32) = G_PHI %15(s32), %bb.5, %13(s32), %bb.0
+    %16:_(s32) = G_PHI %13(s32), %bb.0, %17(s32), %bb.5
+    %18:_(s1) = G_PHI %12(s1), %bb.0, %19(s1), %bb.5
+    %20:_(s32) = G_CONSTANT i32 1000
+    %21:_(s1) = G_ICMP intpred(sle), %16(s32), %20
+    %22:_(s1) = G_CONSTANT i1 true
+    G_BRCOND %21(s1), %bb.4
     G_BR %bb.2
 
   bb.2:
     successors: %bb.3(0x40000000), %bb.5(0x40000000)
 
-    %23:_(s1) = G_PHI %24(s1), %bb.4, %20(s1), %bb.1
+    %23:_(s1) = G_PHI %24(s1), %bb.4, %22(s1), %bb.1
     %25:_(s1) = G_CONSTANT i1 true
     %26:_(s1) = G_XOR %23, %25
     G_BRCOND %26(s1), %bb.5
@@ -415,9 +409,9 @@ body: |
   bb.4:
     successors: %bb.2(0x80000000)
 
-    %24:_(s1) = G_CONSTANT i1 false
     %28:_(s32) = G_CONSTANT i32 1000
     G_STORE %28(s32), %10(p0) :: (store (s32))
+    %24:_(s1) = G_CONSTANT i1 false
     G_BR %bb.2
 
   bb.5:
@@ -434,22 +428,20 @@ body: |
     G_BR %bb.6
 
   bb.6:
-    %33:_(s1) = G_PHI %19(s1), %bb.5
-    %34:_(s32) = G_PHI %15(s32), %bb.5
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %34(s32)
-    %35:_(s32) = G_FCONSTANT float 0.000000e+00
-    %36:_(s32) = G_FCONSTANT float 1.000000e+00
-    %37:_(s32) = G_SELECT %33(s1), %36, %35
-    G_STORE %37(s32), %4(p0) :: (store (s32))
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %15(s32)
+    %33:_(s32) = G_FCONSTANT float 0.000000e+00
+    %34:_(s32) = G_FCONSTANT float 1.000000e+00
+    %35:_(s32) = G_SELECT %19(s1), %34, %33
+    G_STORE %35(s32), %4(p0) :: (store (s32))
     SI_RETURN
 ...
 
 ---
-name: _amdgpu_cs_main
+name: single_lane_execution_attribute
 legalized: true
 tracksRegLiveness: true
 body: |
-  ; GFX10-LABEL: name: _amdgpu_cs_main
+  ; GFX10-LABEL: name: single_lane_execution_attribute
   ; GFX10: bb.0:
   ; GFX10-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
   ; GFX10-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2
@@ -464,24 +456,22 @@ body: |
   ; GFX10-NEXT:   [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY]](s32)
   ; GFX10-NEXT:   [[OR:%[0-9]+]]:_(s64) = G_OR [[AND]], [[ZEXT]]
   ; GFX10-NEXT:   [[INTTOPTR:%[0-9]+]]:_(p4) = G_INTTOPTR [[OR]](s64)
-  ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(<8 x s32>) = G_LOAD [[INTTOPTR]](p4) :: (load (<8 x s32>))
-  ; GFX10-NEXT:   [[BITCAST:%[0-9]+]]:_(s256) = G_BITCAST [[LOAD]](<8 x s32>)
-  ; GFX10-NEXT:   [[TRUNC:%[0-9]+]]:_(s128) = G_TRUNC [[BITCAST]](s256)
-  ; GFX10-NEXT:   [[BITCAST1:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[TRUNC]](s128)
+  ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(<8 x s32>) = G_LOAD [[INTTOPTR]](p4) :: (invariant load (<8 x s32>), align 16, addrspace 4)
   ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-  ; GFX10-NEXT:   [[INT1:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.mbcnt.lo), [[C2]](s32), [[C1]](s32)
-  ; GFX10-NEXT:   [[INT2:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.mbcnt.hi), [[C2]](s32), [[INT1]](s32)
-  ; GFX10-NEXT:   [[FREEZE:%[0-9]+]]:_(s32) = G_FREEZE [[INT2]]
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
   ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(<4 x s32>), [[UV1:%[0-9]+]]:_(<4 x s32>) = G_UNMERGE_VALUES [[LOAD]](<8 x s32>)
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+  ; GFX10-NEXT:   [[INT1:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.mbcnt.lo), [[C4]](s32), [[C1]](s32)
+  ; GFX10-NEXT:   [[INT2:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.mbcnt.hi), [[C4]](s32), [[INT1]](s32)
+  ; GFX10-NEXT:   [[FREEZE:%[0-9]+]]:_(s32) = G_FREEZE [[INT2]]
   ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[FREEZE]], [[C3]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_BUFFER_LOAD [[BITCAST1]](<4 x s32>), [[C1]](s32), [[SHL]], [[C1]], 0, 0, 0 :: (load (s32), align 1, addrspace 8)
+  ; GFX10-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_BUFFER_LOAD [[UV]](<4 x s32>), [[C1]](s32), [[SHL]], [[C1]], 0, 0, 0 :: (dereferenceable load (s32), align 1, addrspace 8)
   ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[AMDGPU_BUFFER_LOAD]](s32), [[C1]]
-  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-  ; GFX10-NEXT:   [[AND1:%[0-9]+]]:_(s32) = G_AND [[FREEZE]], [[C4]]
-  ; GFX10-NEXT:   [[TRUNC1:%[0-9]+]]:_(s1) = G_TRUNC [[AND1]](s32)
+  ; GFX10-NEXT:   [[AND1:%[0-9]+]]:_(s32) = G_AND [[FREEZE]], [[C2]]
+  ; GFX10-NEXT:   [[TRUNC:%[0-9]+]]:_(s1) = G_TRUNC [[AND1]](s32)
   ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[TRUNC1]], [[C5]]
+  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[TRUNC]], [[C5]]
   ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:sreg_32(s1) = COPY [[C5]](s1)
   ; GFX10-NEXT:   G_BRCOND [[XOR]](s1), %bb.2
   ; GFX10-NEXT:   G_BR %bb.1
@@ -495,8 +485,8 @@ body: |
   ; GFX10-NEXT: bb.2:
   ; GFX10-NEXT:   successors: %bb.5(0x40000000), %bb.6(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[COPY3]](s1), %bb.0, %58(s1), %bb.4
-  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI %30(s32), %bb.4, [[DEF]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[COPY3]](s1), %bb.0, %57(s1), %bb.4
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI %29(s32), %bb.4, [[DEF]](s32), %bb.0
   ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
   ; GFX10-NEXT:   G_BRCOND [[COPY4]](s1), %bb.5
   ; GFX10-NEXT:   G_BR %bb.6
@@ -504,12 +494,13 @@ body: |
   ; GFX10-NEXT: bb.3:
   ; GFX10-NEXT:   successors: %bb.4(0x04000000), %bb.3(0x7c000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI %34(s32), %bb.3, [[C6]](s32), %bb.1
-  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI %36(s32), %bb.3, [[FREEZE]](s32), %bb.1
-  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI %38(s32), %bb.3, [[C6]](s32), %bb.1
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI %33(s32), %bb.3, [[C6]](s32), %bb.1
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI %35(s32), %bb.3, [[FREEZE]](s32), %bb.1
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI %37(s32), %bb.3, [[C6]](s32), %bb.1
   ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; GFX10-NEXT:   [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:_(s32) = G_AMDGPU_BUFFER_LOAD [[BITCAST1]](<4 x s32>), [[C7]](s32), [[PHI2]], [[C7]], 0, 0, 0 :: (load (s32), align 1, addrspace 8)
+  ; GFX10-NEXT:   [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:_(s32) = G_AMDGPU_BUFFER_LOAD [[UV]](<4 x s32>), [[C7]](s32), [[PHI2]], [[C7]], 0, 0, 0 :: (dereferenceable load (s32), align 1, addrspace 8)
   ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AMDGPU_BUFFER_LOAD1]], [[PHI4]]
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY [[ADD]](s32), implicit $exec_lo
   ; GFX10-NEXT:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
   ; GFX10-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI3]], [[C8]]
   ; GFX10-NEXT:   [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
@@ -521,12 +512,11 @@ body: |
   ; GFX10-NEXT: bb.4:
   ; GFX10-NEXT:   successors: %bb.2(0x80000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:_(s32) = G_PHI [[ADD]](s32), %bb.3
-  ; GFX10-NEXT:   [[C10:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
-  ; GFX10-NEXT:   [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[PHI5]](s32), [[AMDGPU_BUFFER_LOAD]]
+  ; GFX10-NEXT:   [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY5]](s32), [[AMDGPU_BUFFER_LOAD]]
   ; GFX10-NEXT:   [[OR1:%[0-9]+]]:_(s1) = G_OR [[ICMP]], [[ICMP2]]
   ; GFX10-NEXT:   [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s1)
-  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[C10]](s1)
+  ; GFX10-NEXT:   [[C10:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[C10]](s1)
   ; GFX10-NEXT:   G_BR %bb.2
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.5:
@@ -537,13 +527,13 @@ body: |
   ; GFX10-NEXT:   [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[C11]]
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.6:
-  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:_(s32) = G_PHI [[PHI1]](s32), %bb.2, [[OR2]](s32), %bb.5
-  ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(<4 x s32>), [[UV1:%[0-9]+]]:_(<4 x s32>) = G_UNMERGE_VALUES [[LOAD]](<8 x s32>)
+  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:_(s32) = G_PHI [[PHI1]](s32), %bb.2, [[OR2]](s32), %bb.5
+  ; GFX10-NEXT:   [[UV2:%[0-9]+]]:_(<4 x s32>), [[UV3:%[0-9]+]]:_(<4 x s32>) = G_UNMERGE_VALUES [[LOAD]](<8 x s32>)
   ; GFX10-NEXT:   [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[COPY2]], [[COPY1]]
   ; GFX10-NEXT:   [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
   ; GFX10-NEXT:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ADD3]], [[C12]](s32)
   ; GFX10-NEXT:   [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; GFX10-NEXT:   G_AMDGPU_BUFFER_STORE [[PHI6]](s32), [[UV1]](<4 x s32>), [[C13]](s32), [[SHL1]], [[C13]], 0, 0, 0 :: (store (s32), align 1, addrspace 8)
+  ; GFX10-NEXT:   G_AMDGPU_BUFFER_STORE [[PHI5]](s32), [[UV3]](<4 x s32>), [[C13]](s32), [[SHL1]], [[C13]], 0, 0, 0 :: (dereferenceable store (s32), align 1, addrspace 8)
   ; GFX10-NEXT:   S_ENDPGM 0
   bb.0:
     successors: %bb.1(0x40000000), %bb.2(0x40000000)
@@ -559,82 +549,79 @@ body: |
     %7:_(s64) = G_ZEXT %0(s32)
     %8:_(s64) = G_OR %6, %7
     %9:_(p4) = G_INTTOPTR %8(s64)
-    %10:_(<8 x s32>) = G_LOAD %9(p4) :: (load (<8 x s32>))
-    %11:_(s256) = G_BITCAST %10(<8 x s32>)
-    %12:_(s128) = G_TRUNC %11(s256)
-    %13:_(<4 x s32>) = G_BITCAST %12(s128)
-    %15:_(s32) = G_CONSTANT i32 0
-    %14:_(s32) = G_CONSTANT i32 -1
-    %16:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.mbcnt.lo), %14(s32), %15(s32)
-    %17:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.mbcnt.hi), %14(s32), %16(s32)
-    %18:_(s32) = G_FREEZE %17
-    %19:_(s32) = G_CONSTANT i32 2
-    %20:_(s32) = G_SHL %18, %19(s32)
-    %21:_(s32) = G_AMDGPU_BUFFER_LOAD %13(<4 x s32>), %15(s32), %20, %15, 0, 0, 0 :: (load (s32), align 1, addrspace 8)
-    %22:_(s1) = G_ICMP intpred(eq), %21(s32), %15
-    %23:_(s32) = G_CONSTANT i32 1
-    %24:_(s32) = G_AND %18, %23
-    %25:_(s1) = G_TRUNC %24(s32)
-    %26:_(s1) = G_CONSTANT i1 true
-    %27:_(s1) = G_XOR %25, %26
-    G_BRCOND %27(s1), %bb.2
+    %10:_(<8 x s32>) = G_LOAD %9(p4) :: (invariant load (<8 x s32>), align 16, addrspace 4)
+    %11:_(s32) = G_CONSTANT i32 0
+    %12:_(s32) = G_CONSTANT i32 1
+    %13:_(s32) = G_CONSTANT i32 2
+    %14:_(<4 x s32>), %15:_(<4 x s32>) = G_UNMERGE_VALUES %10(<8 x s32>)
+    %16:_(s32) = G_CONSTANT i32 -1
+    %17:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.mbcnt.lo), %16(s32), %11(s32)
+    %18:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.mbcnt.hi), %16(s32), %17(s32)
+    %19:_(s32) = G_FREEZE %18
+    %20:_(s32) = G_SHL %19, %13(s32)
+    %21:_(s32) = G_AMDGPU_BUFFER_LOAD %14(<4 x s32>), %11(s32), %20, %11, 0, 0, 0 :: (dereferenceable load (s32), align 1, addrspace 8)
+    %22:_(s1) = G_ICMP intpred(eq), %21(s32), %11
+    %23:_(s32) = G_AND %19, %12
+    %24:_(s1) = G_TRUNC %23(s32)
+    %25:_(s1) = G_CONSTANT i1 true
+    %26:_(s1) = G_XOR %24, %25
+    G_BRCOND %26(s1), %bb.2
     G_BR %bb.1
 
   bb.1:
     successors: %bb.3(0x80000000)
 
-    %28:_(s32) = G_CONSTANT i32 0
+    %27:_(s32) = G_CONSTANT i32 0
     G_BR %bb.3
 
   bb.2:
     successors: %bb.5(0x40000000), %bb.6(0x40000000)
 
-    %29:_(s32) = G_PHI %30(s32), %bb.4, %3(s32), %bb.0
-    %31:_(s1) = G_PHI %32(s1), %bb.4, %26(s1), %bb.0
-    G_BRCOND %31(s1), %bb.5
+    %28:_(s32) = G_PHI %29(s32), %bb.4, %3(s32), %bb.0
+    %30:_(s1) = G_PHI %31(s1), %bb.4, %25(s1), %bb.0
+    G_BRCOND %30(s1), %bb.5
     G_BR %bb.6
 
   bb.3:
     successors: %bb.4(0x04000000), %bb.3(0x7c000000)
 
-    %33:_(s32) = G_PHI %34(s32), %bb.3, %28(s32), %bb.1
-    %35:_(s32) = G_PHI %36(s32), %bb.3, %18(s32), %bb.1
-    %37:_(s32) = G_PHI %38(s32), %bb.3, %28(s32), %bb.1
-    %39:_(s32) = G_CONSTANT i32 0
-    %40:_(s32) = G_AMDGPU_BUFFER_LOAD %13(<4 x s32>), %39(s32), %33, %39, 0, 0, 0 :: (load (s32), align 1, addrspace 8)
-    %38:_(s32) = G_ADD %40, %37
-    %41:_(s32) = G_CONSTANT i32 -1
-    %36:_(s32) = G_ADD %35, %41
-    %42:_(s32) = G_CONSTANT i32 4
-    %34:_(s32) = G_ADD %33, %42
-    %43:_(s1) = G_ICMP intpred(ne), %36(s32), %39
-    G_BRCOND %43(s1), %bb.3
+    %32:_(s32) = G_PHI %33(s32), %bb.3, %27(s32), %bb.1
+    %34:_(s32) = G_PHI %35(s32), %bb.3, %19(s32), %bb.1
+    %36:_(s32) = G_PHI %37(s32), %bb.3, %27(s32), %bb.1
+    %38:_(s32) = G_CONSTANT i32 0
+    %39:_(s32) = G_AMDGPU_BUFFER_LOAD %14(<4 x s32>), %38(s32), %32, %38, 0, 0, 0 :: (dereferenceable load (s32), align 1, addrspace 8)
+    %37:_(s32) = G_ADD %39, %36
+    %40:_(s32) = G_CONSTANT i32 -1
+    %35:_(s32) = G_ADD %34, %40
+    %41:_(s32) = G_CONSTANT i32 4
+    %33:_(s32) = G_ADD %32, %41
+    %42:_(s1) = G_ICMP intpred(ne), %35(s32), %38
+    G_BRCOND %42(s1), %bb.3
     G_BR %bb.4
 
   bb.4:
     successors: %bb.2(0x80000000)
 
-    %44:_(s32) = G_PHI %38(s32), %bb.3
-    %32:_(s1) = G_CONSTANT i1 false
-    %45:_(s1) = G_ICMP intpred(eq), %44(s32), %21
-    %46:_(s1) = G_OR %22, %45
-    %30:_(s32) = G_ZEXT %46(s1)
+    %43:_(s1) = G_ICMP intpred(eq), %37(s32), %21
+    %44:_(s1) = G_OR %22, %43
+    %29:_(s32) = G_ZEXT %44(s1)
+    %31:_(s1) = G_CONSTANT i1 false
     G_BR %bb.2
 
   bb.5:
     successors: %bb.6(0x80000000)
 
-    %47:_(s32) = G_ZEXT %22(s1)
-    %48:_(s32) = G_CONSTANT i32 2
-    %49:_(s32) = G_OR %47, %48
+    %45:_(s32) = G_ZEXT %22(s1)
+    %46:_(s32) = G_CONSTANT i32 2
+    %47:_(s32) = G_OR %45, %46
 
   bb.6:
-    %50:_(s32) = G_PHI %29(s32), %bb.2, %49(s32), %bb.5
-    %51:_(<4 x s32>), %52:_(<4 x s32>) = G_UNMERGE_VALUES %10(<8 x s32>)
-    %53:_(s32) = G_ADD %2, %1
-    %54:_(s32) = G_CONSTANT i32 2
-    %55:_(s32) = G_SHL %53, %54(s32)
-    %56:_(s32) = G_CONSTANT i32 0
-    G_AMDGPU_BUFFER_STORE %50(s32), %52(<4 x s32>), %56(s32), %55, %56, 0, 0, 0 :: (store (s32), align 1, addrspace 8)
+    %48:_(s32) = G_PHI %28(s32), %bb.2, %47(s32), %bb.5
+    %49:_(<4 x s32>), %50:_(<4 x s32>) = G_UNMERGE_VALUES %10(<8 x s32>)
+    %51:_(s32) = G_ADD %2, %1
+    %52:_(s32) = G_CONSTANT i32 2
+    %53:_(s32) = G_SHL %51, %52(s32)
+    %54:_(s32) = G_CONSTANT i32 0
+    G_AMDGPU_BUFFER_STORE %48(s32), %50(<4 x s32>), %54(s32), %53, %54, 0, 0, 0 :: (dereferenceable store (s32), align 1, addrspace 8)
     S_ENDPGM 0
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
index a66c21feb1cbc..be90b02a6dd65 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
@@ -13,32 +13,30 @@ define void @divergent_i1_phi_used_outside_loop(float %val, float %pre.cond.val,
 ; GFX10-LABEL: divergent_i1_phi_used_outside_loop:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, 1.0, v1
 ; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s5, 1.0, v1
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s4
-; GFX10-NEXT:    s_andn2_b32 s5, s4, exec_lo
-; GFX10-NEXT:    s_and_b32 s6, exec_lo, vcc_lo
-; GFX10-NEXT:    s_or_b32 s6, s5, s6
-; GFX10-NEXT:    ; implicit-def: $sgpr5
+; GFX10-NEXT:    ; implicit-def: $sgpr6
+; GFX10-NEXT:    ; implicit-def: $sgpr7
 ; GFX10-NEXT:  .LBB0_1: ; %loop
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX10-NEXT:    v_cvt_f32_u32_e32 v4, v1
-; GFX10-NEXT:    s_xor_b32 s7, s6, -1
+; GFX10-NEXT:    s_xor_b32 s8, s5, -1
 ; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v0
 ; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
-; GFX10-NEXT:    s_andn2_b32 s8, s6, exec_lo
-; GFX10-NEXT:    s_and_b32 s7, exec_lo, s7
-; GFX10-NEXT:    s_andn2_b32 s5, s5, exec_lo
-; GFX10-NEXT:    s_and_b32 s6, exec_lo, s6
-; GFX10-NEXT:    s_or_b32 s7, s8, s7
-; GFX10-NEXT:    s_or_b32 s5, s5, s6
-; GFX10-NEXT:    s_mov_b32 s6, s7
+; GFX10-NEXT:    s_andn2_b32 s7, s7, exec_lo
+; GFX10-NEXT:    s_and_b32 s5, exec_lo, s5
+; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
+; GFX10-NEXT:    s_or_b32 s7, s7, s5
+; GFX10-NEXT:    s_mov_b32 s5, s8
+; GFX10-NEXT:    s_and_b32 s9, exec_lo, s7
+; GFX10-NEXT:    s_or_b32 s6, s6, s9
 ; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
 ; GFX10-NEXT:    s_cbranch_execnz .LBB0_1
 ; GFX10-NEXT:  ; %bb.2: ; %exit
 ; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s6
 ; GFX10-NEXT:    flat_store_dword v[2:3], v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -141,6 +139,7 @@ define void @divergent_i1_xor_used_outside_loop(float %val, float %pre.cond.val,
 ; GFX10-NEXT:    v_cmp_lt_f32_e64 s5, 1.0, v1
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s4
 ; GFX10-NEXT:    ; implicit-def: $sgpr6
+; GFX10-NEXT:    ; implicit-def: $sgpr7
 ; GFX10-NEXT:  .LBB2_1: ; %loop
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX10-NEXT:    v_cvt_f32_u32_e32 v4, v1
@@ -148,9 +147,12 @@ define void @divergent_i1_xor_used_outside_loop(float %val, float %pre.cond.val,
 ; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
 ; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v0
 ; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT:    s_andn2_b32 s7, s7, exec_lo
+; GFX10-NEXT:    s_and_b32 s8, exec_lo, s5
 ; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
-; GFX10-NEXT:    s_and_b32 s7, exec_lo, s5
-; GFX10-NEXT:    s_or_b32 s6, s6, s7
+; GFX10-NEXT:    s_or_b32 s7, s7, s8
+; GFX10-NEXT:    s_and_b32 s8, exec_lo, s7
+; GFX10-NEXT:    s_or_b32 s6, s6, s8
 ; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
 ; GFX10-NEXT:    s_cbranch_execnz .LBB2_1
 ; GFX10-NEXT:  ; %bb.2: ; %exit
@@ -178,6 +180,59 @@ exit:
   ret void
 }
 
+define void @divergent_i1_xor_used_outside_loop_twice(float %val, float %pre.cond.val, ptr %addr, ptr %addr2) {
+; GFX10-LABEL: divergent_i1_xor_used_outside_loop_twice:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s5, 1.0, v1
+; GFX10-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-NEXT:    ; implicit-def: $sgpr6
+; GFX10-NEXT:    ; implicit-def: $sgpr7
+; GFX10-NEXT:  .LBB3_1: ; %loop
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v6, v1
+; GFX10-NEXT:    s_xor_b32 s5, s5, -1
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v0
+; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT:    s_andn2_b32 s7, s7, exec_lo
+; GFX10-NEXT:    s_and_b32 s8, exec_lo, s5
+; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
+; GFX10-NEXT:    s_or_b32 s7, s7, s8
+; GFX10-NEXT:    s_and_b32 s8, exec_lo, s7
+; GFX10-NEXT:    s_or_b32 s6, s6, s8
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_cbranch_execnz .LBB3_1
+; GFX10-NEXT:  ; %bb.2: ; %exit
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, -1.0, 2.0, s6
+; GFX10-NEXT:    flat_store_dword v[2:3], v0
+; GFX10-NEXT:    flat_store_dword v[4:5], v1
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %pre.cond = fcmp ogt float %pre.cond.val, 1.0
+  br label %loop
+
+loop:
+  %counter = phi i32 [ 0, %entry ], [ %counter.plus.1, %loop ]
+  %bool.counter = phi i1 [ %pre.cond, %entry ], [ %neg.bool.counter, %loop ]
+  %neg.bool.counter = xor i1 %bool.counter, true
+  %f.counter = uitofp i32 %counter to float
+  %cond = fcmp ogt float %f.counter, %val
+  %counter.plus.1 = add i32 %counter, 1
+  br i1 %cond, label %exit, label %loop
+
+exit:
+  %select = select i1 %neg.bool.counter, float 1.000000e+00, float 0.000000e+00
+  store float %select, ptr %addr
+  %select2 = select i1 %neg.bool.counter, float 2.000000e+00, float -1.000000e+00
+  store float %select2, ptr %addr2
+  ret void
+}
+
 ;void xor(int num_elts, int* a, int* addr) {
 ;for(int i=0; i<num_elts; ++i) {
 ;  if(a[i]==0)
@@ -195,65 +250,69 @@ define void @divergent_i1_xor_used_outside_loop_larger_loop_body(i32 %num.elts,
 ; GFX10-NEXT:    s_mov_b32 s5, 0
 ; GFX10-NEXT:    s_mov_b32 s6, -1
 ; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
-; GFX10-NEXT:    s_cbranch_execz .LBB3_6
+; GFX10-NEXT:    s_cbranch_execz .LBB4_6
 ; GFX10-NEXT:  ; %bb.1: ; %loop.start.preheader
 ; GFX10-NEXT:    v_mov_b32_e32 v5, s5
 ; GFX10-NEXT:    ; implicit-def: $sgpr6
-; GFX10-NEXT:    ; implicit-def: $sgpr7
 ; GFX10-NEXT:    ; implicit-def: $sgpr8
-; GFX10-NEXT:    s_branch .LBB3_3
-; GFX10-NEXT:  .LBB3_2: ; %Flow
-; GFX10-NEXT:    ; in Loop: Header=BB3_3 Depth=1
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s9
-; GFX10-NEXT:    s_xor_b32 s9, s8, -1
-; GFX10-NEXT:    s_and_b32 s10, exec_lo, s7
-; GFX10-NEXT:    s_or_b32 s5, s10, s5
+; GFX10-NEXT:    ; implicit-def: $sgpr9
+; GFX10-NEXT:    ; implicit-def: $sgpr7
+; GFX10-NEXT:    s_branch .LBB4_3
+; GFX10-NEXT:  .LBB4_2: ; %Flow
+; GFX10-NEXT:    ; in Loop: Header=BB4_3 Depth=1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s10
+; GFX10-NEXT:    s_xor_b32 s10, s9, -1
+; GFX10-NEXT:    s_and_b32 s11, exec_lo, s8
+; GFX10-NEXT:    s_or_b32 s5, s11, s5
+; GFX10-NEXT:    s_andn2_b32 s7, s7, exec_lo
+; GFX10-NEXT:    s_and_b32 s10, exec_lo, s10
 ; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
-; GFX10-NEXT:    s_and_b32 s9, exec_lo, s9
-; GFX10-NEXT:    s_or_b32 s6, s6, s9
+; GFX10-NEXT:    s_or_b32 s7, s7, s10
+; GFX10-NEXT:    s_and_b32 s10, exec_lo, s7
+; GFX10-NEXT:    s_or_b32 s6, s6, s10
 ; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT:    s_cbranch_execz .LBB3_5
-; GFX10-NEXT:  .LBB3_3: ; %loop.start
+; GFX10-NEXT:    s_cbranch_execz .LBB4_5
+; GFX10-NEXT:  .LBB4_3: ; %loop.start
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
+; GFX10-NEXT:    s_andn2_b32 s9, s9, exec_lo
+; GFX10-NEXT:    s_and_b32 s10, exec_lo, -1
 ; GFX10-NEXT:    s_andn2_b32 s8, s8, exec_lo
-; GFX10-NEXT:    s_and_b32 s9, exec_lo, -1
-; GFX10-NEXT:    s_andn2_b32 s7, s7, exec_lo
-; GFX10-NEXT:    s_or_b32 s8, s8, s9
+; GFX10-NEXT:    s_or_b32 s9, s9, s10
 ; GFX10-NEXT:    v_lshlrev_b64 v[6:7], 2, v[5:6]
-; GFX10-NEXT:    s_or_b32 s7, s7, s9
+; GFX10-NEXT:    s_or_b32 s8, s8, s10
 ; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v1, v6
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v2, v7, vcc_lo
 ; GFX10-NEXT:    global_load_dword v6, v[6:7], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    s_and_saveexec_b32 s9, vcc_lo
-; GFX10-NEXT:    s_cbranch_execz .LBB3_2
+; GFX10-NEXT:    s_and_saveexec_b32 s10, vcc_lo
+; GFX10-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX10-NEXT:  ; %bb.4: ; %loop.cond
-; GFX10-NEXT:    ; in Loop: Header=BB3_3 Depth=1
+; GFX10-NEXT:    ; in Loop: Header=BB4_3 Depth=1
 ; GFX10-NEXT:    v_add_nc_u32_e32 v6, 1, v5
 ; GFX10-NEXT:    v_cmp_lt_i32_e32 vcc_lo, v5, v0
+; GFX10-NEXT:    s_andn2_b32 s9, s9, exec_lo
+; GFX10-NEXT:    s_and_b32 s11, exec_lo, 0
 ; GFX10-NEXT:    s_andn2_b32 s8, s8, exec_lo
-; GFX10-NEXT:    s_and_b32 s10, exec_lo, 0
-; GFX10-NEXT:    s_andn2_b32 s7, s7, exec_lo
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v6
-; GFX10-NEXT:    s_and_b32 s11, exec_lo, vcc_lo
-; GFX10-NEXT:    s_or_b32 s8, s8, s10
-; GFX10-NEXT:    s_or_b32 s7, s7, s11
-; GFX10-NEXT:    s_branch .LBB3_2
-; GFX10-NEXT:  .LBB3_5: ; %loop.exit.guard
+; GFX10-NEXT:    s_and_b32 s12, exec_lo, vcc_lo
+; GFX10-NEXT:    s_or_b32 s9, s9, s11
+; GFX10-NEXT:    s_or_b32 s8, s8, s12
+; GFX10-NEXT:    s_branch .LBB4_2
+; GFX10-NEXT:  .LBB4_5: ; %loop.exit.guard
 ; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
 ; GFX10-NEXT:    s_andn2_b32 s5, -1, exec_lo
 ; GFX10-NEXT:    s_and_b32 s6, exec_lo, s6
 ; GFX10-NEXT:    s_or_b32 s6, s5, s6
-; GFX10-NEXT:  .LBB3_6: ; %Flow1
+; GFX10-NEXT:  .LBB4_6: ; %Flow1
 ; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
 ; GFX10-NEXT:    s_and_saveexec_b32 s4, s6
-; GFX10-NEXT:    s_cbranch_execz .LBB3_8
+; GFX10-NEXT:    s_cbranch_execz .LBB4_8
 ; GFX10-NEXT:  ; %bb.7: ; %block.after.loop
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 5
 ; GFX10-NEXT:    flat_store_dword v[3:4], v0
-; GFX10-NEXT:  .LBB3_8: ; %exit
+; GFX10-NEXT:  .LBB4_8: ; %exit
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -296,56 +355,60 @@ define void @divergent_i1_icmp_used_outside_loop(i32 %v0, i32 %v1, ptr addrspace
 ; GFX10-LABEL: divergent_i1_icmp_used_outside_loop:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s5, 0
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    ; implicit-def: $sgpr5
 ; GFX10-NEXT:    ; implicit-def: $sgpr6
-; GFX10-NEXT:    v_mov_b32_e32 v5, s5
-; GFX10-NEXT:    s_branch .LBB4_2
-; GFX10-NEXT:  .LBB4_1: ; %Flow
-; GFX10-NEXT:    ; in Loop: Header=BB4_2 Depth=1
+; GFX10-NEXT:    v_mov_b32_e32 v5, s4
+; GFX10-NEXT:    s_branch .LBB5_2
+; GFX10-NEXT:  .LBB5_1: ; %Flow
+; GFX10-NEXT:    ; in Loop: Header=BB5_2 Depth=1
 ; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT:    s_and_b32 s4, exec_lo, s7
-; GFX10-NEXT:    s_or_b32 s5, s4, s5
-; GFX10-NEXT:    s_andn2_b32 s4, s6, exec_lo
-; GFX10-NEXT:    s_and_b32 s6, exec_lo, vcc_lo
-; GFX10-NEXT:    s_or_b32 s6, s4, s6
-; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT:    s_cbranch_execz .LBB4_6
-; GFX10-NEXT:  .LBB4_2: ; %cond.block.0
+; GFX10-NEXT:    s_and_b32 s7, exec_lo, s7
+; GFX10-NEXT:    s_or_b32 s4, s7, s4
+; GFX10-NEXT:    s_andn2_b32 s5, s5, exec_lo
+; GFX10-NEXT:    s_and_b32 s7, exec_lo, s6
+; GFX10-NEXT:    s_or_b32 s5, s5, s7
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_cbranch_execz .LBB5_6
+; GFX10-NEXT:  .LBB5_2: ; %cond.block.0
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX10-NEXT:    v_mov_b32_e32 v4, v5
+; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT:    s_and_b32 s7, exec_lo, vcc_lo
+; GFX10-NEXT:    s_or_b32 s6, s6, s7
 ; GFX10-NEXT:    s_and_saveexec_b32 s7, vcc_lo
-; GFX10-NEXT:    s_cbranch_execz .LBB4_4
+; GFX10-NEXT:    s_cbranch_execz .LBB5_4
 ; GFX10-NEXT:  ; %bb.3: ; %if.block.0
-; GFX10-NEXT:    ; in Loop: Header=BB4_2 Depth=1
+; GFX10-NEXT:    ; in Loop: Header=BB5_2 Depth=1
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v5, 31, v4
 ; GFX10-NEXT:    v_lshlrev_b64 v[8:9], 2, v[4:5]
-; GFX10-NEXT:    v_add_co_u32 v8, s4, v2, v8
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v9, s4, v3, v9, s4
+; GFX10-NEXT:    v_add_co_u32 v8, vcc_lo, v2, v8
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v3, v9, vcc_lo
 ; GFX10-NEXT:    global_store_dword v[8:9], v4, off
-; GFX10-NEXT:  .LBB4_4: ; %loop.break.block
-; GFX10-NEXT:    ; in Loop: Header=BB4_2 Depth=1
+; GFX10-NEXT:  .LBB5_4: ; %loop.break.block
+; GFX10-NEXT:    ; in Loop: Header=BB5_2 Depth=1
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s7
-; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, v1, v4
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v4
 ; GFX10-NEXT:    s_mov_b32 s7, -1
 ; GFX10-NEXT:    ; implicit-def: $vgpr5
-; GFX10-NEXT:    s_and_saveexec_b32 s8, s4
-; GFX10-NEXT:    s_cbranch_execz .LBB4_1
+; GFX10-NEXT:    s_and_saveexec_b32 s8, vcc_lo
+; GFX10-NEXT:    s_cbranch_execz .LBB5_1
 ; GFX10-NEXT:  ; %bb.5: ; %loop.cond
-; GFX10-NEXT:    ; in Loop: Header=BB4_2 Depth=1
+; GFX10-NEXT:    ; in Loop: Header=BB5_2 Depth=1
 ; GFX10-NEXT:    v_add_nc_u32_e32 v5, 1, v4
-; GFX10-NEXT:    s_andn2_b32 s4, -1, exec_lo
-; GFX10-NEXT:    s_and_b32 s7, exec_lo, 0
-; GFX10-NEXT:    s_or_b32 s7, s4, s7
-; GFX10-NEXT:    s_branch .LBB4_1
-; GFX10-NEXT:  .LBB4_6: ; %cond.block.1
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT:    s_and_saveexec_b32 s4, s6
-; GFX10-NEXT:    s_cbranch_execz .LBB4_8
+; GFX10-NEXT:    s_andn2_b32 s7, -1, exec_lo
+; GFX10-NEXT:    s_and_b32 s9, exec_lo, 0
+; GFX10-NEXT:    s_or_b32 s7, s7, s9
+; GFX10-NEXT:    s_branch .LBB5_1
+; GFX10-NEXT:  .LBB5_6: ; %cond.block.1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_and_saveexec_b32 s4, s5
+; GFX10-NEXT:    s_cbranch_execz .LBB5_8
 ; GFX10-NEXT:  ; %bb.7: ; %if.block.1
 ; GFX10-NEXT:    global_store_dword v[6:7], v4, off
-; GFX10-NEXT:  .LBB4_8: ; %exit
+; GFX10-NEXT:  .LBB5_8: ; %exit
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -406,48 +469,52 @@ define amdgpu_ps void @divergent_i1_freeze_used_outside_loop(i32 %n, ptr addrspa
 ; GFX10-LABEL: divergent_i1_freeze_used_outside_loop:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_mov_b32 s0, 0
-; GFX10-NEXT:    s_mov_b32 s3, -1
+; GFX10-NEXT:    s_mov_b32 s4, -1
 ; GFX10-NEXT:    v_mov_b32_e32 v5, s0
+; GFX10-NEXT:    ; implicit-def: $sgpr3
 ; GFX10-NEXT:    ; implicit-def: $sgpr1
 ; GFX10-NEXT:    ; implicit-def: $sgpr2
-; GFX10-NEXT:    s_branch .LBB5_2
-; GFX10-NEXT:  .LBB5_1: ; %loop.cond
-; GFX10-NEXT:    ; in Loop: Header=BB5_2 Depth=1
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_branch .LBB6_2
+; GFX10-NEXT:  .LBB6_1: ; %loop.cond
+; GFX10-NEXT:    ; in Loop: Header=BB6_2 Depth=1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
 ; GFX10-NEXT:    v_cmp_lt_i32_e32 vcc_lo, v5, v0
 ; GFX10-NEXT:    v_add_nc_u32_e32 v5, 1, v5
 ; GFX10-NEXT:    s_or_b32 s0, vcc_lo, s0
+; GFX10-NEXT:    s_andn2_b32 s2, s2, exec_lo
+; GFX10-NEXT:    s_and_b32 s5, exec_lo, s1
+; GFX10-NEXT:    s_andn2_b32 s4, s4, exec_lo
+; GFX10-NEXT:    s_or_b32 s2, s2, s5
 ; GFX10-NEXT:    s_andn2_b32 s3, s3, exec_lo
-; GFX10-NEXT:    s_and_b32 s4, exec_lo, s2
-; GFX10-NEXT:    s_andn2_b32 s1, s1, exec_lo
-; GFX10-NEXT:    s_or_b32 s3, s3, s4
-; GFX10-NEXT:    s_or_b32 s1, s1, s4
+; GFX10-NEXT:    s_and_b32 s6, exec_lo, s2
+; GFX10-NEXT:    s_or_b32 s4, s4, s5
+; GFX10-NEXT:    s_or_b32 s3, s3, s6
 ; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
-; GFX10-NEXT:    s_cbranch_execz .LBB5_4
-; GFX10-NEXT:  .LBB5_2: ; %loop.start
+; GFX10-NEXT:    s_cbranch_execz .LBB6_4
+; GFX10-NEXT:  .LBB6_2: ; %loop.start
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    s_andn2_b32 s2, s2, exec_lo
-; GFX10-NEXT:    s_and_b32 s4, exec_lo, s3
-; GFX10-NEXT:    s_or_b32 s2, s2, s4
-; GFX10-NEXT:    s_and_saveexec_b32 s4, s3
-; GFX10-NEXT:    s_cbranch_execz .LBB5_1
+; GFX10-NEXT:    s_andn2_b32 s1, s1, exec_lo
+; GFX10-NEXT:    s_and_b32 s5, exec_lo, s4
+; GFX10-NEXT:    s_or_b32 s1, s1, s5
+; GFX10-NEXT:    s_and_saveexec_b32 s5, s4
+; GFX10-NEXT:    s_cbranch_execz .LBB6_1
 ; GFX10-NEXT:  ; %bb.3: ; %is.eq.zero
-; GFX10-NEXT:    ; in Loop: Header=BB5_2 Depth=1
+; GFX10-NEXT:    ; in Loop: Header=BB6_2 Depth=1
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
-; GFX10-NEXT:    s_andn2_b32 s2, s2, exec_lo
+; GFX10-NEXT:    s_andn2_b32 s1, s1, exec_lo
 ; GFX10-NEXT:    v_lshlrev_b64 v[6:7], 2, v[5:6]
 ; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v1, v6
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v2, v7, vcc_lo
 ; GFX10-NEXT:    global_load_dword v6, v[6:7], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    s_and_b32 s3, exec_lo, vcc_lo
-; GFX10-NEXT:    s_or_b32 s2, s2, s3
-; GFX10-NEXT:    ; implicit-def: $sgpr3
-; GFX10-NEXT:    s_branch .LBB5_1
-; GFX10-NEXT:  .LBB5_4: ; %exit
+; GFX10-NEXT:    s_and_b32 s4, exec_lo, vcc_lo
+; GFX10-NEXT:    s_or_b32 s1, s1, s4
+; GFX10-NEXT:    ; implicit-def: $sgpr4
+; GFX10-NEXT:    s_branch .LBB6_1
+; GFX10-NEXT:  .LBB6_4: ; %exit
 ; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s1
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s3
 ; GFX10-NEXT:    flat_store_dword v[3:4], v0
 ; GFX10-NEXT:    s_endpgm
 entry:
@@ -483,64 +550,68 @@ define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, ptr addrspace(1) %a
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_mov_b32 s0, 0
 ; GFX10-NEXT:    ; implicit-def: $sgpr1
-; GFX10-NEXT:    ; implicit-def: $sgpr2
 ; GFX10-NEXT:    ; implicit-def: $sgpr3
+; GFX10-NEXT:    ; implicit-def: $sgpr4
+; GFX10-NEXT:    ; implicit-def: $sgpr2
 ; GFX10-NEXT:    v_mov_b32_e32 v6, s0
-; GFX10-NEXT:    s_branch .LBB6_2
-; GFX10-NEXT:  .LBB6_1: ; %Flow
-; GFX10-NEXT:    ; in Loop: Header=BB6_2 Depth=1
+; GFX10-NEXT:    s_branch .LBB7_2
+; GFX10-NEXT:  .LBB7_1: ; %Flow
+; GFX10-NEXT:    ; in Loop: Header=BB7_2 Depth=1
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT:    s_and_b32 s4, exec_lo, s2
-; GFX10-NEXT:    s_or_b32 s0, s4, s0
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    s_and_b32 s5, exec_lo, s3
+; GFX10-NEXT:    s_or_b32 s0, s5, s0
+; GFX10-NEXT:    s_andn2_b32 s2, s2, exec_lo
+; GFX10-NEXT:    s_and_b32 s5, exec_lo, s4
 ; GFX10-NEXT:    s_andn2_b32 s1, s1, exec_lo
-; GFX10-NEXT:    s_and_b32 s4, exec_lo, s3
-; GFX10-NEXT:    s_or_b32 s1, s1, s4
+; GFX10-NEXT:    s_or_b32 s2, s2, s5
+; GFX10-NEXT:    s_and_b32 s5, exec_lo, s2
+; GFX10-NEXT:    s_or_b32 s1, s1, s5
 ; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
-; GFX10-NEXT:    s_cbranch_execz .LBB6_4
-; GFX10-NEXT:  .LBB6_2: ; %A
+; GFX10-NEXT:    s_cbranch_execz .LBB7_4
+; GFX10-NEXT:  .LBB7_2: ; %A
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
+; GFX10-NEXT:    s_andn2_b32 s4, s4, exec_lo
+; GFX10-NEXT:    s_and_b32 s5, exec_lo, -1
 ; GFX10-NEXT:    s_andn2_b32 s3, s3, exec_lo
-; GFX10-NEXT:    s_and_b32 s4, exec_lo, -1
-; GFX10-NEXT:    s_andn2_b32 s2, s2, exec_lo
-; GFX10-NEXT:    s_or_b32 s3, s3, s4
+; GFX10-NEXT:    s_or_b32 s4, s4, s5
 ; GFX10-NEXT:    v_lshlrev_b64 v[7:8], 2, v[6:7]
-; GFX10-NEXT:    s_or_b32 s2, s2, s4
+; GFX10-NEXT:    s_or_b32 s3, s3, s5
 ; GFX10-NEXT:    v_add_co_u32 v9, vcc_lo, v2, v7
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, v3, v8, vcc_lo
 ; GFX10-NEXT:    global_load_dword v9, v[9:10], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
-; GFX10-NEXT:    s_cbranch_execz .LBB6_1
+; GFX10-NEXT:    s_and_saveexec_b32 s5, vcc_lo
+; GFX10-NEXT:    s_cbranch_execz .LBB7_1
 ; GFX10-NEXT:  ; %bb.3: ; %loop.body
-; GFX10-NEXT:    ; in Loop: Header=BB6_2 Depth=1
+; GFX10-NEXT:    ; in Loop: Header=BB7_2 Depth=1
 ; GFX10-NEXT:    v_add_co_u32 v7, vcc_lo, v0, v7
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, v1, v8, vcc_lo
 ; GFX10-NEXT:    v_add_nc_u32_e32 v10, 1, v6
 ; GFX10-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 0x64, v6
-; GFX10-NEXT:    s_andn2_b32 s3, s3, exec_lo
+; GFX10-NEXT:    s_andn2_b32 s4, s4, exec_lo
 ; GFX10-NEXT:    global_load_dword v9, v[7:8], off
-; GFX10-NEXT:    s_and_b32 s5, exec_lo, 0
+; GFX10-NEXT:    s_and_b32 s6, exec_lo, 0
 ; GFX10-NEXT:    v_mov_b32_e32 v6, v10
-; GFX10-NEXT:    s_andn2_b32 s2, s2, exec_lo
-; GFX10-NEXT:    s_and_b32 s6, exec_lo, vcc_lo
-; GFX10-NEXT:    s_or_b32 s3, s3, s5
-; GFX10-NEXT:    s_or_b32 s2, s2, s6
+; GFX10-NEXT:    s_andn2_b32 s3, s3, exec_lo
+; GFX10-NEXT:    s_and_b32 s7, exec_lo, vcc_lo
+; GFX10-NEXT:    s_or_b32 s4, s4, s6
+; GFX10-NEXT:    s_or_b32 s3, s3, s7
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_add_nc_u32_e32 v9, 1, v9
 ; GFX10-NEXT:    global_store_dword v[7:8], v9, off
-; GFX10-NEXT:    s_branch .LBB6_1
-; GFX10-NEXT:  .LBB6_4: ; %loop.exit.guard
+; GFX10-NEXT:    s_branch .LBB7_1
+; GFX10-NEXT:  .LBB7_4: ; %loop.exit.guard
 ; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX10-NEXT:    s_and_saveexec_b32 s0, s1
 ; GFX10-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX10-NEXT:    s_cbranch_execz .LBB6_6
+; GFX10-NEXT:    s_cbranch_execz .LBB7_6
 ; GFX10-NEXT:  ; %bb.5: ; %break.body
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 10
 ; GFX10-NEXT:    global_store_dword v[4:5], v0, off
-; GFX10-NEXT:  .LBB6_6: ; %exit
+; GFX10-NEXT:  .LBB7_6: ; %exit
 ; GFX10-NEXT:    s_endpgm
 entry:
   br label %A
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.mir
index 5bbe3e4886899..e800cb2e24a7a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.mir
@@ -16,52 +16,41 @@ body: |
   ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
   ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
-  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
-  ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[COPY1]](s32), [[C1]]
-  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[FCMP]](s1)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[COPY1]](s32), [[C]]
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
-  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[DEF]](s1)
-  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY5]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY4]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[FCMP]](s1)
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.1:
   ; GFX10-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %36(s1), %bb.1
-  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.0, %24(s1), %bb.1
-  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI %9(s32), %bb.1, [[C]](s32), %bb.0
-  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %11(s32), %bb.1
-  ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[COPY4]](s1), %bb.0, %29(s1), %bb.1
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %21(s1), %bb.1
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI %9(s32), %bb.1, [[C1]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.0, %11(s32), %bb.1
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
+  ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[COPY5]](s1)
   ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1)
-  ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[COPY7]](s1)
-  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1)
   ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[COPY7]], [[C2]]
-  ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1)
+  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[COPY5]], [[C2]]
   ; GFX10-NEXT:   [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[PHI3]](s32)
   ; GFX10-NEXT:   [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
   ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
   ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI3]], [[C3]]
   ; GFX10-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP1]](s1), [[PHI2]](s32)
-  ; GFX10-NEXT:   [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY9]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY10]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY8]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY7]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY6]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1)
   ; GFX10-NEXT:   SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.2
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.2:
-  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.1
-  ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_2]](s1)
-  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI4]](s32)
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
   ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
   ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
-  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY11]](s1), [[C5]], [[C4]]
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[S_OR_B32_]](s1), [[C5]], [[C4]]
   ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p0) :: (store (s32))
   ; GFX10-NEXT:   SI_RETURN
   bb.0:
@@ -73,16 +62,16 @@ body: |
     %2:_(s32) = COPY $vgpr2
     %3:_(s32) = COPY $vgpr3
     %4:_(p0) = G_MERGE_VALUES %2(s32), %3(s32)
-    %5:_(s32) = G_CONSTANT i32 0
-    %6:_(s32) = G_FCONSTANT float 1.000000e+00
-    %7:_(s1) = G_FCMP floatpred(ogt), %1(s32), %6
+    %5:_(s32) = G_FCONSTANT float 1.000000e+00
+    %6:_(s1) = G_FCMP floatpred(ogt), %1(s32), %5
+    %7:_(s32) = G_CONSTANT i32 0
 
   bb.1:
     successors: %bb.2(0x04000000), %bb.1(0x7c000000)
 
-    %8:_(s32) = G_PHI %9(s32), %bb.1, %5(s32), %bb.0
-    %10:_(s32) = G_PHI %5(s32), %bb.0, %11(s32), %bb.1
-    %12:_(s1) = G_PHI %7(s1), %bb.0, %13(s1), %bb.1
+    %8:_(s32) = G_PHI %9(s32), %bb.1, %7(s32), %bb.0
+    %10:_(s32) = G_PHI %7(s32), %bb.0, %11(s32), %bb.1
+    %12:_(s1) = G_PHI %6(s1), %bb.0, %13(s1), %bb.1
     %14:_(s1) = G_CONSTANT i1 true
     %13:_(s1) = G_XOR %12, %14
     %15:_(s32) = G_UITOFP %10(s32)
@@ -94,13 +83,11 @@ body: |
     G_BR %bb.2
 
   bb.2:
-    %18:_(s1) = G_PHI %12(s1), %bb.1
-    %19:_(s32) = G_PHI %9(s32), %bb.1
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %19(s32)
-    %20:_(s32) = G_FCONSTANT float 0.000000e+00
-    %21:_(s32) = G_FCONSTANT float 1.000000e+00
-    %22:_(s32) = G_SELECT %18(s1), %21, %20
-    G_STORE %22(s32), %4(p0) :: (store (s32))
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %9(s32)
+    %18:_(s32) = G_FCONSTANT float 0.000000e+00
+    %19:_(s32) = G_FCONSTANT float 1.000000e+00
+    %20:_(s32) = G_SELECT %12(s1), %19, %18
+    G_STORE %20(s32), %4(p0) :: (store (s32))
     SI_RETURN
 ...
 
@@ -120,9 +107,9 @@ body: |
   ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr3
   ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr4
   ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
-  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
-  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[C1]](s1)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[C]](s1)
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:sreg_32_xm0_xexec(s1) = IMPLICIT_DEF
   ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[DEF]](s1)
   ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY5]](s1), $exec_lo, implicit-def $scc
@@ -133,9 +120,9 @@ body: |
   ; GFX10-NEXT: bb.1:
   ; GFX10-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %41(s1), %bb.3
-  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[S_OR_B32_]](s1), %bb.0, %27(s1), %bb.3
-  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %9(s32), %bb.3
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %40(s1), %bb.3
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[S_OR_B32_]](s1), %bb.0, %26(s1), %bb.3
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.0, %9(s32), %bb.3
   ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(p1) = G_PHI [[MV]](p1), %bb.0, %11(p1), %bb.3
   ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
   ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI1]](s1)
@@ -175,15 +162,13 @@ body: |
   ; GFX10-NEXT:   [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY9]](s1), $exec_lo, implicit-def $scc
   ; GFX10-NEXT:   [[S_AND_B32_3:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY13]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_3:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_3]](s1), [[S_AND_B32_3]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[COPY7]](s1)
   ; GFX10-NEXT:   G_BRCOND [[ICMP1]](s1), %bb.1
   ; GFX10-NEXT:   G_BR %bb.4
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.4:
-  ; GFX10-NEXT:   [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[COPY14]](s1)
   ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
   ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
-  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY15]](s1), [[C7]], [[C6]]
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY7]](s1), [[C7]], [[C6]]
   ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV1]](p0) :: (store (s32))
   ; GFX10-NEXT:   SI_RETURN
   bb.0:
@@ -196,15 +181,15 @@ body: |
     %3:_(s32) = COPY $vgpr3
     %4:_(s32) = COPY $vgpr4
     %5:_(p0) = G_MERGE_VALUES %3(s32), %4(s32)
-    %6:_(s32) = G_CONSTANT i32 -1
-    %7:_(s1) = G_CONSTANT i1 true
+    %6:_(s1) = G_CONSTANT i1 true
+    %7:_(s32) = G_CONSTANT i32 -1
 
   bb.1:
     successors: %bb.2(0x40000000), %bb.3(0x40000000)
 
-    %8:_(s32) = G_PHI %6(s32), %bb.0, %9(s32), %bb.3
+    %8:_(s32) = G_PHI %7(s32), %bb.0, %9(s32), %bb.3
     %10:_(p1) = G_PHI %2(p1), %bb.0, %11(p1), %bb.3
-    %12:sreg_32_xm0_xexec(s1) = G_PHI %7(s1), %bb.0, %13(s1), %bb.3
+    %12:sreg_32_xm0_xexec(s1) = G_PHI %6(s1), %bb.0, %13(s1), %bb.3
     %14:sreg_32_xm0_xexec(s32) = SI_IF %12(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.2
 
@@ -230,11 +215,10 @@ body: |
     G_BR %bb.4
 
   bb.4:
-    %22:_(s1) = G_PHI %12(s1), %bb.3
-    %23:_(s32) = G_FCONSTANT float 0.000000e+00
-    %24:_(s32) = G_FCONSTANT float 1.000000e+00
-    %25:_(s32) = G_SELECT %22(s1), %24, %23
-    G_STORE %25(s32), %5(p0) :: (store (s32))
+    %22:_(s32) = G_FCONSTANT float 0.000000e+00
+    %23:_(s32) = G_FCONSTANT float 1.000000e+00
+    %24:_(s32) = G_SELECT %12(s1), %23, %22
+    G_STORE %24(s32), %5(p0) :: (store (s32))
     SI_RETURN
 ...
 
@@ -253,43 +237,41 @@ body: |
   ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
   ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
-  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
-  ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[COPY1]](s32), [[C1]]
-  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[FCMP]](s1)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[COPY1]](s32), [[C]]
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[FCMP]](s1)
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.1:
   ; GFX10-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %27(s1), %bb.1
-  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[COPY4]](s1), %bb.0, %24(s1), %bb.1
-  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI %9(s32), %bb.1, [[C]](s32), %bb.0
-  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %11(s32), %bb.1
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[COPY4]](s1), %bb.0, %29(s1), %bb.1
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %21(s1), %bb.1
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI %9(s32), %bb.1, [[C1]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.0, %11(s32), %bb.1
   ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
   ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1)
   ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[COPY6]], [[C2]]
+  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[COPY5]], [[C2]]
   ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1)
   ; GFX10-NEXT:   [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[PHI3]](s32)
   ; GFX10-NEXT:   [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
   ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
   ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI3]], [[C3]]
   ; GFX10-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP1]](s1), [[PHI2]](s32)
-  ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1)
-  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY5]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
   ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY7]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1)
   ; GFX10-NEXT:   SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.2
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.2:
-  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.1
-  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
-  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI4]](s32)
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
   ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
   ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
-  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY9]](s1), [[C5]], [[C4]]
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[S_OR_B32_]](s1), [[C5]], [[C4]]
   ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p0) :: (store (s32))
   ; GFX10-NEXT:   SI_RETURN
   bb.0:
@@ -301,16 +283,16 @@ body: |
     %2:_(s32) = COPY $vgpr2
     %3:_(s32) = COPY $vgpr3
     %4:_(p0) = G_MERGE_VALUES %2(s32), %3(s32)
-    %5:_(s32) = G_CONSTANT i32 0
-    %6:_(s32) = G_FCONSTANT float 1.000000e+00
-    %7:_(s1) = G_FCMP floatpred(ogt), %1(s32), %6
+    %5:_(s32) = G_FCONSTANT float 1.000000e+00
+    %6:_(s1) = G_FCMP floatpred(ogt), %1(s32), %5
+    %7:_(s32) = G_CONSTANT i32 0
 
   bb.1:
     successors: %bb.2(0x04000000), %bb.1(0x7c000000)
 
-    %8:_(s32) = G_PHI %9(s32), %bb.1, %5(s32), %bb.0
-    %10:_(s32) = G_PHI %5(s32), %bb.0, %11(s32), %bb.1
-    %12:_(s1) = G_PHI %7(s1), %bb.0, %13(s1), %bb.1
+    %8:_(s32) = G_PHI %9(s32), %bb.1, %7(s32), %bb.0
+    %10:_(s32) = G_PHI %7(s32), %bb.0, %11(s32), %bb.1
+    %12:_(s1) = G_PHI %6(s1), %bb.0, %13(s1), %bb.1
     %14:_(s1) = G_CONSTANT i1 true
     %13:_(s1) = G_XOR %12, %14
     %15:_(s32) = G_UITOFP %10(s32)
@@ -322,13 +304,116 @@ body: |
     G_BR %bb.2
 
   bb.2:
-    %18:_(s1) = G_PHI %13(s1), %bb.1
-    %19:_(s32) = G_PHI %9(s32), %bb.1
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %19(s32)
-    %20:_(s32) = G_FCONSTANT float 0.000000e+00
-    %21:_(s32) = G_FCONSTANT float 1.000000e+00
-    %22:_(s32) = G_SELECT %18(s1), %21, %20
-    G_STORE %22(s32), %4(p0) :: (store (s32))
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %9(s32)
+    %18:_(s32) = G_FCONSTANT float 0.000000e+00
+    %19:_(s32) = G_FCONSTANT float 1.000000e+00
+    %20:_(s32) = G_SELECT %13(s1), %19, %18
+    G_STORE %20(s32), %4(p0) :: (store (s32))
+    SI_RETURN
+...
+
+
+---
+name: divergent_i1_xor_used_outside_loop_twice
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: divergent_i1_xor_used_outside_loop_twice
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[COPY1]](s32), [[C]]
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[DEF:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+  ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[FCMP]](s1)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[COPY6]](s1), %bb.0, %35(s1), %bb.1
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %27(s1), %bb.1
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI %12(s32), %bb.1, [[C1]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.0, %14(s32), %bb.1
+  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
+  ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1)
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[COPY7]], [[C2]]
+  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1)
+  ; GFX10-NEXT:   [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[PHI3]](s32)
+  ; GFX10-NEXT:   [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI3]], [[C3]]
+  ; GFX10-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP1]](s1), [[PHI2]](s32)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY9]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1)
+  ; GFX10-NEXT:   SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[S_OR_B32_]](s1), [[C5]], [[C4]]
+  ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p0) :: (store (s32))
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_FCONSTANT float -1.000000e+00
+  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_FCONSTANT float 2.000000e+00
+  ; GFX10-NEXT:   [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[S_OR_B32_]](s1), [[C7]], [[C6]]
+  ; GFX10-NEXT:   G_STORE [[SELECT1]](s32), [[MV1]](p0) :: (store (s32))
+  ; GFX10-NEXT:   SI_RETURN
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(p0) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s32) = COPY $vgpr4
+    %6:_(s32) = COPY $vgpr5
+    %7:_(p0) = G_MERGE_VALUES %5(s32), %6(s32)
+    %8:_(s32) = G_FCONSTANT float 1.000000e+00
+    %9:_(s1) = G_FCMP floatpred(ogt), %1(s32), %8
+    %10:_(s32) = G_CONSTANT i32 0
+
+  bb.1:
+    successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+
+    %11:_(s32) = G_PHI %12(s32), %bb.1, %10(s32), %bb.0
+    %13:_(s32) = G_PHI %10(s32), %bb.0, %14(s32), %bb.1
+    %15:_(s1) = G_PHI %9(s1), %bb.0, %16(s1), %bb.1
+    %17:_(s1) = G_CONSTANT i1 true
+    %16:_(s1) = G_XOR %15, %17
+    %18:_(s32) = G_UITOFP %13(s32)
+    %19:_(s1) = G_FCMP floatpred(ogt), %18(s32), %0
+    %20:_(s32) = G_CONSTANT i32 1
+    %14:_(s32) = G_ADD %13, %20
+    %12:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %19(s1), %11(s32)
+    SI_LOOP %12(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %12(s32)
+    %21:_(s32) = G_FCONSTANT float 0.000000e+00
+    %22:_(s32) = G_FCONSTANT float 1.000000e+00
+    %23:_(s32) = G_SELECT %16(s1), %22, %21
+    G_STORE %23(s32), %4(p0) :: (store (s32))
+    %24:_(s32) = G_FCONSTANT float -1.000000e+00
+    %25:_(s32) = G_FCONSTANT float 2.000000e+00
+    %26:_(s32) = G_SELECT %16(s1), %25, %24
+    G_STORE %26(s32), %7(p0) :: (store (s32))
     SI_RETURN
 ...
 
@@ -370,7 +455,7 @@ body: |
   ; GFX10-NEXT: bb.2:
   ; GFX10-NEXT:   successors: %bb.5(0x40000000), %bb.6(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[COPY5]](s1), %bb.0, %40(s1), %bb.8
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[COPY5]](s1), %bb.0, %45(s1), %bb.8
   ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI]](s1)
   ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
   ; GFX10-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY7]](s1), %bb.6, implicit-def $exec, implicit-def $scc, implicit $exec
@@ -379,29 +464,29 @@ body: |
   ; GFX10-NEXT: bb.3:
   ; GFX10-NEXT:   successors: %bb.4(0x40000000), %bb.7(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF3]](s1), %bb.1, %73(s1), %bb.7
-  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:sreg_32(s1) = PHI [[DEF2]](s1), %bb.1, %62(s1), %bb.7
-  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.1, %49(s1), %bb.7
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF3]](s1), %bb.1, %67(s1), %bb.7
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:sreg_32(s1) = PHI [[DEF2]](s1), %bb.1, %54(s1), %bb.7
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.1, %37(s1), %bb.7
   ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[C2]](s32), %bb.1, %17(s32), %bb.7
   ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:_(s32) = G_PHI %19(s32), %bb.7, [[C2]](s32), %bb.1
   ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1)
   ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[PHI2]](s1)
   ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1)
-  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1)
-  ; GFX10-NEXT:   [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1)
   ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI5]](s32)
-  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C4]](s32)
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C3]](s32)
   ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV]], [[SHL]](s64)
   ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
-  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C5]]
-  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY10]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C4]]
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[C5]](s1)
+  ; GFX10-NEXT:   [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[C5]](s1)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY9]](s1), $exec_lo, implicit-def $scc
   ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY12]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
-  ; GFX10-NEXT:   [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY9]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc
   ; GFX10-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY11]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_1]](s1)
@@ -411,17 +496,17 @@ body: |
   ; GFX10-NEXT: bb.4:
   ; GFX10-NEXT:   successors: %bb.7(0x80000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
-  ; GFX10-NEXT:   [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[C6]](s1)
-  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI5]], [[C7]]
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI5]], [[C6]]
   ; GFX10-NEXT:   [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[PHI5]](s32), [[COPY]]
-  ; GFX10-NEXT:   [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1)
+  ; GFX10-NEXT:   [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1)
+  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; GFX10-NEXT:   [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[C7]](s1)
   ; GFX10-NEXT:   [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY13]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY15]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY16]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY14]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY16]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY15]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_3:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_3]](s1), [[S_AND_B32_3]](s1), implicit-def $scc
   ; GFX10-NEXT:   G_BR %bb.7
   ; GFX10-NEXT: {{  $}}
@@ -448,21 +533,19 @@ body: |
   ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[COPY18]], [[C9]]
   ; GFX10-NEXT:   [[COPY19:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1)
   ; GFX10-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[COPY17]](s1), [[PHI4]](s32)
-  ; GFX10-NEXT:   [[S_ANDN2_B32_4:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_ANDN2_B32_4:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY10]](s1), $exec_lo, implicit-def $scc
   ; GFX10-NEXT:   [[S_AND_B32_4:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY19]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_4:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_4]](s1), [[S_AND_B32_4]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[COPY20:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[S_OR_B32_4]](s1)
   ; GFX10-NEXT:   SI_LOOP [[INT]](s32), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.8
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.8:
   ; GFX10-NEXT:   successors: %bb.2(0x80000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI9:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.7
-  ; GFX10-NEXT:   [[COPY20:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_4]](s1)
-  ; GFX10-NEXT:   [[COPY21:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[COPY20]](s1)
-  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI9]](s32)
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
   ; GFX10-NEXT:   [[S_ANDN2_B32_5:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_5:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY21]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_5:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY20]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_5:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_5]](s1), [[S_AND_B32_5]](s1), implicit-def $scc
   ; GFX10-NEXT:   G_BR %bb.2
   bb.0:
@@ -502,24 +585,24 @@ body: |
 
     %16:_(s32) = G_PHI %12(s32), %bb.1, %17(s32), %bb.7
     %18:_(s32) = G_PHI %19(s32), %bb.7, %12(s32), %bb.1
-    %20:_(s1) = G_CONSTANT i1 true
-    %21:_(s64) = G_SEXT %18(s32)
-    %22:_(s32) = G_CONSTANT i32 2
-    %23:_(s64) = G_SHL %21, %22(s32)
-    %24:_(p1) = G_PTR_ADD %3, %23(s64)
-    %25:_(s32) = G_LOAD %24(p1) :: (load (s32), addrspace 1)
-    %26:_(s32) = G_CONSTANT i32 0
-    %27:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %25(s32), %26
-    %28:sreg_32_xm0_xexec(s32) = SI_IF %27(s1), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec
+    %20:_(s64) = G_SEXT %18(s32)
+    %21:_(s32) = G_CONSTANT i32 2
+    %22:_(s64) = G_SHL %20, %21(s32)
+    %23:_(p1) = G_PTR_ADD %3, %22(s64)
+    %24:_(s32) = G_LOAD %23(p1) :: (load (s32), addrspace 1)
+    %25:_(s32) = G_CONSTANT i32 0
+    %26:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %24(s32), %25
+    %27:_(s1) = G_CONSTANT i1 true
+    %28:sreg_32_xm0_xexec(s32) = SI_IF %26(s1), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.4
 
   bb.4:
     successors: %bb.7(0x80000000)
 
-    %29:_(s1) = G_CONSTANT i1 false
-    %30:_(s32) = G_CONSTANT i32 1
-    %31:_(s32) = G_ADD %18, %30
-    %32:_(s1) = G_ICMP intpred(slt), %18(s32), %0
+    %29:_(s32) = G_CONSTANT i32 1
+    %30:_(s32) = G_ADD %18, %29
+    %31:_(s1) = G_ICMP intpred(slt), %18(s32), %0
+    %32:_(s1) = G_CONSTANT i1 false
     G_BR %bb.7
 
   bb.5:
@@ -535,12 +618,12 @@ body: |
   bb.7:
     successors: %bb.8(0x04000000), %bb.3(0x7c000000)
 
-    %19:_(s32) = G_PHI %31(s32), %bb.4, %7(s32), %bb.3
-    %34:_(s1) = G_PHI %29(s1), %bb.4, %20(s1), %bb.3
-    %35:_(s1) = G_PHI %32(s1), %bb.4, %20(s1), %bb.3
+    %19:_(s32) = G_PHI %30(s32), %bb.4, %7(s32), %bb.3
+    %34:_(s1) = G_PHI %32(s1), %bb.4, %27(s1), %bb.3
+    %35:_(s1) = G_PHI %31(s1), %bb.4, %27(s1), %bb.3
     G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %28(s32)
     %36:_(s1) = G_CONSTANT i1 true
-    %37:_(s1) = G_XOR %34, %36
+    %14:_(s1) = G_XOR %34, %36
     %17:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %35(s1), %16(s32)
     SI_LOOP %17(s32), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.8
@@ -548,9 +631,7 @@ body: |
   bb.8:
     successors: %bb.2(0x80000000)
 
-    %14:_(s1) = G_PHI %37(s1), %bb.7
-    %38:_(s32) = G_PHI %17(s32), %bb.7
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %38(s32)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %17(s32)
     G_BR %bb.2
 ...
 
@@ -572,23 +653,27 @@ body: |
   ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr6
   ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr7
   ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
-  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; GFX10-NEXT:   [[DEF1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = IMPLICIT_DEF
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.1:
   ; GFX10-NEXT:   successors: %bb.2(0x80000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[DEF1]](s1), %bb.0, %39(s1), %bb.6
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %29(s1), %bb.6
   ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI %11(s32), %bb.6, [[C]](s32), %bb.0
   ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %13(s32), %bb.6
-  ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI]](s1)
+  ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
+  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:_(s32) = COPY [[PHI2]](s32), implicit $exec_lo
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.2:
   ; GFX10-NEXT:   successors: %bb.3(0x40000000), %bb.4(0x40000000)
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[PHI2]]
-  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[ICMP]](s1)
+  ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP]](s1)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY8]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.3
   ; GFX10-NEXT: {{  $}}
@@ -604,53 +689,47 @@ body: |
   ; GFX10-NEXT: bb.4:
   ; GFX10-NEXT:   successors: %bb.5(0x40000000), %bb.6(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
   ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
   ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[COPY1]](s32), [[PHI2]]
-  ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[C2]](s1)
-  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[COPY8]](s1)
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[C2]](s1)
+  ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[COPY9]](s1)
   ; GFX10-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP1]](s1), %bb.6, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.5
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.5:
   ; GFX10-NEXT:   successors: %bb.6(0x80000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
-  ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1)
-  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI2]], [[C4]]
-  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY9]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY10]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI2]], [[C3]]
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[C4]](s1)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY10]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY11]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.6:
   ; GFX10-NEXT:   successors: %bb.7(0x04000000), %bb.1(0x7c000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[COPY8]](s1), %bb.4, [[S_OR_B32_]](s1), %bb.5
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[COPY9]](s1), %bb.4, [[S_OR_B32_1]](s1), %bb.5
   ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[ADD]](s32), %bb.5, [[DEF]](s32), %bb.4
-  ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1)
+  ; GFX10-NEXT:   [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1)
   ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF1]](s32)
-  ; GFX10-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[COPY11]](s1), [[PHI1]](s32)
-  ; GFX10-NEXT:   [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY7]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[S_OR_B32_1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[COPY12]](s1), [[PHI1]](s32)
   ; GFX10-NEXT:   SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.7
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.7:
   ; GFX10-NEXT:   successors: %bb.8(0x40000000), %bb.9(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.6
-  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:_(s32) = G_PHI [[PHI2]](s32), %bb.6
-  ; GFX10-NEXT:   [[COPY12:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[S_OR_B32_1]](s1)
-  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI5]](s32)
-  ; GFX10-NEXT:   [[SI_IF2:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY12]](s1), %bb.9, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
+  ; GFX10-NEXT:   [[SI_IF2:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[S_OR_B32_]](s1), %bb.9, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.8
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.8:
   ; GFX10-NEXT:   successors: %bb.9(0x80000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   G_STORE [[PHI6]](s32), [[MV1]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   G_STORE [[COPY7]](s32), [[MV1]](p1) :: (store (s32), addrspace 1)
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.9:
   ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF2]](s32)
@@ -667,14 +746,14 @@ body: |
     %5:_(s32) = COPY $vgpr6
     %6:_(s32) = COPY $vgpr7
     %7:_(p1) = G_MERGE_VALUES %5(s32), %6(s32)
-    %8:_(s32) = G_CONSTANT i32 0
-    %9:_(s32) = G_IMPLICIT_DEF
+    %8:_(s32) = G_IMPLICIT_DEF
+    %9:_(s32) = G_CONSTANT i32 0
 
   bb.1:
     successors: %bb.2(0x80000000)
 
-    %10:_(s32) = G_PHI %11(s32), %bb.6, %8(s32), %bb.0
-    %12:_(s32) = G_PHI %8(s32), %bb.0, %13(s32), %bb.6
+    %10:_(s32) = G_PHI %11(s32), %bb.6, %9(s32), %bb.0
+    %12:_(s32) = G_PHI %9(s32), %bb.0, %13(s32), %bb.6
 
   bb.2:
     successors: %bb.3(0x40000000), %bb.4(0x40000000)
@@ -695,24 +774,24 @@ body: |
   bb.4:
     successors: %bb.5(0x40000000), %bb.6(0x40000000)
 
-    %20:_(s1) = G_CONSTANT i1 true
     G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %15(s32)
-    %21:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %1(s32), %12
-    %22:sreg_32_xm0_xexec(s32) = SI_IF %21(s1), %bb.6, implicit-def $exec, implicit-def $scc, implicit $exec
+    %20:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %1(s32), %12
+    %21:_(s1) = G_CONSTANT i1 true
+    %22:sreg_32_xm0_xexec(s32) = SI_IF %20(s1), %bb.6, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.5
 
   bb.5:
     successors: %bb.6(0x80000000)
 
-    %23:_(s1) = G_CONSTANT i1 false
-    %24:_(s32) = G_CONSTANT i32 1
-    %25:_(s32) = G_ADD %12, %24
+    %23:_(s32) = G_CONSTANT i32 1
+    %24:_(s32) = G_ADD %12, %23
+    %25:_(s1) = G_CONSTANT i1 false
 
   bb.6:
     successors: %bb.7(0x04000000), %bb.1(0x7c000000)
 
-    %13:_(s32) = G_PHI %25(s32), %bb.5, %9(s32), %bb.4
-    %26:_(s1) = G_PHI %23(s1), %bb.5, %20(s1), %bb.4
+    %13:_(s32) = G_PHI %24(s32), %bb.5, %8(s32), %bb.4
+    %26:_(s1) = G_PHI %25(s1), %bb.5, %21(s1), %bb.4
     G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %22(s32)
     %11:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %26(s1), %10(s32)
     SI_LOOP %11(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
@@ -721,20 +800,17 @@ body: |
   bb.7:
     successors: %bb.8(0x40000000), %bb.9(0x40000000)
 
-    %27:_(s32) = G_PHI %11(s32), %bb.6
-    %28:sreg_32_xm0_xexec(s1) = G_PHI %14(s1), %bb.6
-    %29:_(s32) = G_PHI %12(s32), %bb.6
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %27(s32)
-    %30:sreg_32_xm0_xexec(s32) = SI_IF %28(s1), %bb.9, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %11(s32)
+    %27:sreg_32_xm0_xexec(s32) = SI_IF %14(s1), %bb.9, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.8
 
   bb.8:
     successors: %bb.9(0x80000000)
 
-    G_STORE %29(s32), %7(p1) :: (store (s32), addrspace 1)
+    G_STORE %12(s32), %7(p1) :: (store (s32), addrspace 1)
 
   bb.9:
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %30(s32)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %27(s32)
     SI_RETURN
 ...
 
@@ -755,29 +831,29 @@ body: |
   ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
   ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
   ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
-  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[C1]](s1)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[C]](s1)
   ; GFX10-NEXT:   [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.1:
   ; GFX10-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %54(s1), %bb.3
-  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %43(s1), %bb.3
-  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[COPY5]](s1), %bb.0, %33(s1), %bb.3
-  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI %10(s32), %bb.3, [[C]](s32), %bb.0
-  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %12(s32), %bb.3
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %48(s1), %bb.3
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[COPY5]](s1), %bb.0, %38(s1), %bb.3
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %29(s1), %bb.3
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI %10(s32), %bb.3, [[C1]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.0, %12(s32), %bb.3
   ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
-  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1)
-  ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI2]](s1)
-  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[COPY8]](s1)
-  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY7]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY9]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI1]](s1)
+  ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[COPY7]](s1)
+  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[PHI2]](s1)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY8]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
-  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY8]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY7]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.2
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.2:
@@ -800,33 +876,31 @@ body: |
   ; GFX10-NEXT:   successors: %bb.4(0x04000000), %bb.1(0x7c000000)
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, [[S_OR_B32_1]](s1), %bb.2
-  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[PHI2]](s1), %bb.1, [[DEF2]](s1), %bb.2
+  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[PHI1]](s1), %bb.1, [[DEF2]](s1), %bb.2
   ; GFX10-NEXT:   [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1)
   ; GFX10-NEXT:   [[COPY13:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI6]](s1)
   ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
   ; GFX10-NEXT:   [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE [[COPY12]]
-  ; GFX10-NEXT:   [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[FREEZE]](s1)
-  ; GFX10-NEXT:   [[COPY15:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[FREEZE]](s1)
+  ; GFX10-NEXT:   [[COPY14:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[FREEZE]](s1)
+  ; GFX10-NEXT:   [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[FREEZE]](s1)
   ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
   ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI4]], [[C4]]
   ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[PHI4]](s32), [[COPY]]
   ; GFX10-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[ICMP1]](s1), [[PHI3]](s32)
-  ; GFX10-NEXT:   [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY13]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY15]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[S_OR_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY14]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[S_OR_B32_3:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_3]](s1), [[S_AND_B32_3]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY9]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY15]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY13]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_3:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY14]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_3:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_3]](s1), [[S_AND_B32_3]](s1), implicit-def $scc
   ; GFX10-NEXT:   SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.4
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.4:
-  ; GFX10-NEXT:   [[PHI7:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.3
-  ; GFX10-NEXT:   [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_3]](s1)
-  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI7]](s32)
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
   ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
   ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
-  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY16]](s1), [[C6]], [[C5]]
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[S_OR_B32_2]](s1), [[C6]], [[C5]]
   ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV1]](p0) :: (store (s32))
   ; GFX10-NEXT:   S_ENDPGM 0
   bb.0:
@@ -840,15 +914,15 @@ body: |
     %4:_(s32) = COPY $vgpr3
     %5:_(s32) = COPY $vgpr4
     %6:_(p0) = G_MERGE_VALUES %4(s32), %5(s32)
-    %7:_(s32) = G_CONSTANT i32 0
-    %8:_(s1) = G_CONSTANT i1 true
+    %7:_(s1) = G_CONSTANT i1 true
+    %8:_(s32) = G_CONSTANT i32 0
 
   bb.1:
     successors: %bb.2(0x40000000), %bb.3(0x40000000)
 
-    %9:_(s32) = G_PHI %10(s32), %bb.3, %7(s32), %bb.0
-    %11:_(s32) = G_PHI %7(s32), %bb.0, %12(s32), %bb.3
-    %13:sreg_32_xm0_xexec(s1) = G_PHI %8(s1), %bb.0, %14(s1), %bb.3
+    %9:_(s32) = G_PHI %10(s32), %bb.3, %8(s32), %bb.0
+    %11:_(s32) = G_PHI %8(s32), %bb.0, %12(s32), %bb.3
+    %13:sreg_32_xm0_xexec(s1) = G_PHI %7(s1), %bb.0, %14(s1), %bb.3
     %15:sreg_32_xm0_xexec(s32) = SI_IF %13(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.2
 
@@ -877,13 +951,11 @@ body: |
     G_BR %bb.4
 
   bb.4:
-    %26:_(s1) = G_PHI %14(s1), %bb.3
-    %27:_(s32) = G_PHI %10(s32), %bb.3
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %27(s32)
-    %28:_(s32) = G_FCONSTANT float 0.000000e+00
-    %29:_(s32) = G_FCONSTANT float 1.000000e+00
-    %30:_(s32) = G_SELECT %26(s1), %29, %28
-    G_STORE %30(s32), %6(p0) :: (store (s32))
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %10(s32)
+    %26:_(s32) = G_FCONSTANT float 0.000000e+00
+    %27:_(s32) = G_FCONSTANT float 1.000000e+00
+    %28:_(s32) = G_SELECT %14(s1), %27, %26
+    G_STORE %28(s32), %6(p0) :: (store (s32))
     S_ENDPGM 0
 ...
 
@@ -906,38 +978,38 @@ body: |
   ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
   ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
   ; GFX10-NEXT:   [[MV2:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
-  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
-  ; GFX10-NEXT:   [[DEF2:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
-  ; GFX10-NEXT:   [[DEF3:%[0-9]+]]:sreg_32_xm0_xexec(s1) = IMPLICIT_DEF
+  ; GFX10-NEXT:   [[DEF2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = IMPLICIT_DEF
+  ; GFX10-NEXT:   [[DEF3:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.1:
   ; GFX10-NEXT:   successors: %bb.3(0x40000000), %bb.5(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[DEF3]](s1), %bb.0, %67(s1), %bb.5
-  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF2]](s1), %bb.0, %56(s1), %bb.5
-  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %43(s1), %bb.5
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF3]](s1), %bb.0, %61(s1), %bb.5
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[DEF2]](s1), %bb.0, %48(s1), %bb.5
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %38(s1), %bb.5
   ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI %12(s32), %bb.5, [[C]](s32), %bb.0
   ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %14(s32), %bb.5
-  ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI]](s1)
-  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1)
+  ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
+  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI1]](s1)
   ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[PHI2]](s1)
-  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[C1]](s1)
-  ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[C1]](s1)
   ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI4]](s32)
-  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C2]](s32)
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C1]](s32)
   ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV1]], [[SHL]](s64)
   ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
-  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C3]]
-  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY10]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
-  ; GFX10-NEXT:   [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY7]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C2]]
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1)
+  ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[C3]](s1)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY7]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY10]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[S_OR_B32_]](s1)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
   ; GFX10-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY9]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_1]](s1)
@@ -954,24 +1026,24 @@ body: |
   ; GFX10-NEXT: bb.3:
   ; GFX10-NEXT:   successors: %bb.5(0x80000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
-  ; GFX10-NEXT:   [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[C5]](s1)
-  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-  ; GFX10-NEXT:   [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C6]](s32)
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C5]](s32)
   ; GFX10-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV]], [[SHL1]](s64)
   ; GFX10-NEXT:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1)
-  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD1]], [[C7]]
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD1]], [[C6]]
   ; GFX10-NEXT:   G_STORE [[ADD]](s32), [[PTR_ADD1]](p1) :: (store (s32), addrspace 1)
-  ; GFX10-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI4]], [[C7]]
-  ; GFX10-NEXT:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 100
-  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[PHI4]](s32), [[C8]]
-  ; GFX10-NEXT:   [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP1]](s1)
-  ; GFX10-NEXT:   [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY11]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY13]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI4]], [[C6]]
+  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 100
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[PHI4]](s32), [[C7]]
+  ; GFX10-NEXT:   [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP1]](s1)
+  ; GFX10-NEXT:   [[C8:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; GFX10-NEXT:   [[COPY14:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[C8]](s1)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY11]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY14]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY12]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY14]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY13]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_3:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_3]](s1), [[S_AND_B32_3]](s1), implicit-def $scc
   ; GFX10-NEXT:   G_BR %bb.5
   ; GFX10-NEXT: {{  $}}
@@ -983,26 +1055,24 @@ body: |
   ; GFX10-NEXT:   successors: %bb.6(0x04000000), %bb.1(0x7c000000)
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_1]](s1), %bb.1, [[S_OR_B32_3]](s1), %bb.3
-  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, [[S_OR_B32_2]](s1), %bb.3
+  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[S_OR_B32_]](s1), %bb.1, [[S_OR_B32_2]](s1), %bb.3
   ; GFX10-NEXT:   [[PHI7:%[0-9]+]]:_(s32) = G_PHI [[ADD1]](s32), %bb.3, [[DEF]](s32), %bb.1
   ; GFX10-NEXT:   [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1)
-  ; GFX10-NEXT:   [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[PHI6]](s1)
-  ; GFX10-NEXT:   [[COPY17:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[COPY16]](s1)
+  ; GFX10-NEXT:   [[COPY16:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI6]](s1)
+  ; GFX10-NEXT:   [[COPY17:%[0-9]+]]:sreg_32(s1) = COPY [[COPY16]](s1)
   ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
   ; GFX10-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[COPY15]](s1), [[PHI3]](s32)
-  ; GFX10-NEXT:   [[S_ANDN2_B32_4:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_4:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY17]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[S_OR_B32_4:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_4]](s1), [[S_AND_B32_4]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_ANDN2_B32_4:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_4:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY17]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_4:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_4]](s1), [[S_AND_B32_4]](s1), implicit-def $scc
   ; GFX10-NEXT:   SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.6
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.6:
   ; GFX10-NEXT:   successors: %bb.2(0x40000000), %bb.4(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI8:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.5
-  ; GFX10-NEXT:   [[COPY18:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[S_OR_B32_4]](s1)
-  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI8]](s32)
-  ; GFX10-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY18]](s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
+  ; GFX10-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[S_OR_B32_4]](s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.2
   bb.0:
     successors: %bb.1(0x80000000)
@@ -1017,23 +1087,23 @@ body: |
     %6:_(s32) = COPY $vgpr4
     %7:_(s32) = COPY $vgpr5
     %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
-    %9:_(s32) = G_CONSTANT i32 0
-    %10:_(s32) = G_IMPLICIT_DEF
+    %9:_(s32) = G_IMPLICIT_DEF
+    %10:_(s32) = G_CONSTANT i32 0
 
   bb.1:
     successors: %bb.3(0x40000000), %bb.5(0x40000000)
 
-    %11:_(s32) = G_PHI %12(s32), %bb.5, %9(s32), %bb.0
-    %13:_(s32) = G_PHI %9(s32), %bb.0, %14(s32), %bb.5
-    %15:_(s1) = G_CONSTANT i1 true
-    %16:_(s64) = G_SEXT %13(s32)
-    %17:_(s32) = G_CONSTANT i32 2
-    %18:_(s64) = G_SHL %16, %17(s32)
-    %19:_(p1) = G_PTR_ADD %5, %18(s64)
-    %20:_(s32) = G_LOAD %19(p1) :: (load (s32), addrspace 1)
-    %21:_(s32) = G_CONSTANT i32 0
-    %22:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %20(s32), %21
-    %23:sreg_32_xm0_xexec(s32) = SI_IF %22(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+    %11:_(s32) = G_PHI %12(s32), %bb.5, %10(s32), %bb.0
+    %13:_(s32) = G_PHI %10(s32), %bb.0, %14(s32), %bb.5
+    %15:_(s64) = G_SEXT %13(s32)
+    %16:_(s32) = G_CONSTANT i32 2
+    %17:_(s64) = G_SHL %15, %16(s32)
+    %18:_(p1) = G_PTR_ADD %5, %17(s64)
+    %19:_(s32) = G_LOAD %18(p1) :: (load (s32), addrspace 1)
+    %20:_(s32) = G_CONSTANT i32 0
+    %21:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %19(s32), %20
+    %22:_(s1) = G_CONSTANT i1 true
+    %23:sreg_32_xm0_xexec(s32) = SI_IF %21(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.3
 
   bb.2:
@@ -1046,17 +1116,17 @@ body: |
   bb.3:
     successors: %bb.5(0x80000000)
 
-    %25:_(s1) = G_CONSTANT i1 false
-    %26:_(s32) = G_CONSTANT i32 2
-    %27:_(s64) = G_SHL %16, %26(s32)
-    %28:_(p1) = G_PTR_ADD %2, %27(s64)
-    %29:_(s32) = G_LOAD %28(p1) :: (load (s32), addrspace 1)
-    %30:_(s32) = G_CONSTANT i32 1
-    %31:_(s32) = G_ADD %29, %30
-    G_STORE %31(s32), %28(p1) :: (store (s32), addrspace 1)
-    %32:_(s32) = G_ADD %13, %30
-    %33:_(s32) = G_CONSTANT i32 100
-    %34:_(s1) = G_ICMP intpred(ult), %13(s32), %33
+    %25:_(s32) = G_CONSTANT i32 2
+    %26:_(s64) = G_SHL %15, %25(s32)
+    %27:_(p1) = G_PTR_ADD %2, %26(s64)
+    %28:_(s32) = G_LOAD %27(p1) :: (load (s32), addrspace 1)
+    %29:_(s32) = G_CONSTANT i32 1
+    %30:_(s32) = G_ADD %28, %29
+    G_STORE %30(s32), %27(p1) :: (store (s32), addrspace 1)
+    %31:_(s32) = G_ADD %13, %29
+    %32:_(s32) = G_CONSTANT i32 100
+    %33:_(s1) = G_ICMP intpred(ult), %13(s32), %32
+    %34:_(s1) = G_CONSTANT i1 false
     G_BR %bb.5
 
   bb.4:
@@ -1066,9 +1136,9 @@ body: |
   bb.5:
     successors: %bb.6(0x04000000), %bb.1(0x7c000000)
 
-    %14:_(s32) = G_PHI %32(s32), %bb.3, %10(s32), %bb.1
-    %36:_(s1) = G_PHI %25(s1), %bb.3, %15(s1), %bb.1
-    %37:_(s1) = G_PHI %34(s1), %bb.3, %15(s1), %bb.1
+    %14:_(s32) = G_PHI %31(s32), %bb.3, %9(s32), %bb.1
+    %36:sreg_32_xm0_xexec(s1) = G_PHI %34(s1), %bb.3, %22(s1), %bb.1
+    %37:_(s1) = G_PHI %33(s1), %bb.3, %22(s1), %bb.1
     G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %23(s32)
     %12:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %37(s1), %11(s32)
     SI_LOOP %12(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
@@ -1077,9 +1147,7 @@ body: |
   bb.6:
     successors: %bb.2(0x40000000), %bb.4(0x40000000)
 
-    %38:sreg_32_xm0_xexec(s1) = G_PHI %36(s1), %bb.5
-    %39:_(s32) = G_PHI %12(s32), %bb.5
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %39(s32)
-    %35:sreg_32_xm0_xexec(s32) = SI_IF %38(s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %12(s32)
+    %35:sreg_32_xm0_xexec(s32) = SI_IF %36(s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.2
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll
index 1698f84eea518..e31077dd1986f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -new-reg-bank-select -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
 
 ; Simples case, if - then, that requires lane mask merging,
 ; %phi lane mask will hold %val_A at %A. Lanes that are active in %B
@@ -41,9 +41,10 @@ exit:
 define amdgpu_ps void @divergent_i1_phi_if_else(ptr addrspace(1) %out, i32 %tid, i32 %cond) {
 ; GFX10-LABEL: divergent_i1_phi_if_else:
 ; GFX10:       ; %bb.0: ; %entry
-; GFX10-NEXT:    s_and_b32 s0, 1, s0
+; GFX10-NEXT:    s_and_b32 s0, s0, 1
 ; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v3
-; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, s0
+; GFX10-NEXT:    s_cmp_lg_u32 s0, 0
+; GFX10-NEXT:    s_cselect_b32 s0, exec_lo, 0
 ; GFX10-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX10-NEXT:    s_xor_b32 s1, exec_lo, s1
 ; GFX10-NEXT:  ; %bb.1: ; %B
@@ -105,46 +106,51 @@ exit:
 define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, ptr addrspace(1) %a) {
 ; GFX10-LABEL: loop_with_1break:
 ; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_mov_b32 s4, 0
 ; GFX10-NEXT:    s_mov_b32 s0, 0
-; GFX10-NEXT:    ; implicit-def: $sgpr1
-; GFX10-NEXT:    v_mov_b32_e32 v4, s0
+; GFX10-NEXT:    ; implicit-def: $sgpr5
 ; GFX10-NEXT:    s_branch .LBB2_2
 ; GFX10-NEXT:  .LBB2_1: ; %Flow
 ; GFX10-NEXT:    ; in Loop: Header=BB2_2 Depth=1
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s2
-; GFX10-NEXT:    s_and_b32 s2, exec_lo, s1
-; GFX10-NEXT:    s_or_b32 s0, s2, s0
-; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX10-NEXT:    s_and_b32 s1, exec_lo, s5
+; GFX10-NEXT:    s_or_b32 s4, s1, s4
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
 ; GFX10-NEXT:    s_cbranch_execz .LBB2_4
 ; GFX10-NEXT:  .LBB2_2: ; %A
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    v_ashrrev_i32_e32 v5, 31, v4
-; GFX10-NEXT:    s_andn2_b32 s1, s1, exec_lo
-; GFX10-NEXT:    s_and_b32 s2, exec_lo, -1
-; GFX10-NEXT:    s_or_b32 s1, s1, s2
-; GFX10-NEXT:    v_lshlrev_b64 v[5:6], 2, v[4:5]
-; GFX10-NEXT:    v_add_co_u32 v7, vcc_lo, v2, v5
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, v3, v6, vcc_lo
-; GFX10-NEXT:    global_load_dword v7, v[7:8], off
+; GFX10-NEXT:    s_ashr_i32 s1, s0, 31
+; GFX10-NEXT:    s_lshl_b64 s[2:3], s[0:1], 2
+; GFX10-NEXT:    s_andn2_b32 s1, s5, exec_lo
+; GFX10-NEXT:    v_mov_b32_e32 v5, s3
+; GFX10-NEXT:    v_mov_b32_e32 v4, s2
+; GFX10-NEXT:    s_and_b32 s5, exec_lo, exec_lo
+; GFX10-NEXT:    s_or_b32 s5, s1, s5
+; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, v3, v5, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[4:5], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v7
-; GFX10-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v4
+; GFX10-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX10-NEXT:    s_cbranch_execz .LBB2_1
 ; GFX10-NEXT:  ; %bb.3: ; %loop.body
 ; GFX10-NEXT:    ; in Loop: Header=BB2_2 Depth=1
-; GFX10-NEXT:    v_add_co_u32 v5, vcc_lo, v0, v5
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, v1, v6, vcc_lo
-; GFX10-NEXT:    v_add_nc_u32_e32 v8, 1, v4
-; GFX10-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 0x64, v4
-; GFX10-NEXT:    s_andn2_b32 s1, s1, exec_lo
-; GFX10-NEXT:    global_load_dword v7, v[5:6], off
-; GFX10-NEXT:    v_mov_b32_e32 v4, v8
-; GFX10-NEXT:    s_and_b32 s3, exec_lo, vcc_lo
-; GFX10-NEXT:    s_or_b32 s1, s1, s3
+; GFX10-NEXT:    v_mov_b32_e32 v5, s3
+; GFX10-NEXT:    v_mov_b32_e32 v4, s2
+; GFX10-NEXT:    s_add_i32 s2, s0, 1
+; GFX10-NEXT:    s_cmpk_lt_u32 s0, 0x64
+; GFX10-NEXT:    s_cselect_b32 s0, exec_lo, 0
+; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, v1, v5, vcc_lo
+; GFX10-NEXT:    s_andn2_b32 s3, s5, exec_lo
+; GFX10-NEXT:    s_and_b32 s0, exec_lo, s0
+; GFX10-NEXT:    s_or_b32 s5, s3, s0
+; GFX10-NEXT:    global_load_dword v6, v[4:5], off
+; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_add_nc_u32_e32 v7, 1, v7
-; GFX10-NEXT:    global_store_dword v[5:6], v7, off
+; GFX10-NEXT:    v_add_nc_u32_e32 v6, 1, v6
+; GFX10-NEXT:    global_store_dword v[4:5], v6, off
 ; GFX10-NEXT:    s_branch .LBB2_1
 ; GFX10-NEXT:  .LBB2_4: ; %exit
 ; GFX10-NEXT:    s_endpgm
@@ -174,62 +180,69 @@ exit:
 define amdgpu_cs void @loop_with_2breaks(ptr addrspace(1) %x, ptr addrspace(1) %a, ptr addrspace(1) %b) {
 ; GFX10-LABEL: loop_with_2breaks:
 ; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_mov_b32 s4, 0
 ; GFX10-NEXT:    s_mov_b32 s0, 0
-; GFX10-NEXT:    ; implicit-def: $sgpr1
-; GFX10-NEXT:    v_mov_b32_e32 v6, s0
+; GFX10-NEXT:    ; implicit-def: $sgpr5
 ; GFX10-NEXT:    s_branch .LBB3_3
 ; GFX10-NEXT:  .LBB3_1: ; %Flow3
 ; GFX10-NEXT:    ; in Loop: Header=BB3_3 Depth=1
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s3
-; GFX10-NEXT:    s_andn2_b32 s1, s1, exec_lo
-; GFX10-NEXT:    s_and_b32 s3, exec_lo, s4
-; GFX10-NEXT:    s_or_b32 s1, s1, s3
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s7
+; GFX10-NEXT:    s_andn2_b32 s2, s5, exec_lo
+; GFX10-NEXT:    s_and_b32 s3, exec_lo, s6
+; GFX10-NEXT:    s_or_b32 s5, s2, s3
 ; GFX10-NEXT:  .LBB3_2: ; %Flow
 ; GFX10-NEXT:    ; in Loop: Header=BB3_3 Depth=1
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s2
-; GFX10-NEXT:    s_and_b32 s2, exec_lo, s1
-; GFX10-NEXT:    s_or_b32 s0, s2, s0
-; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX10-NEXT:    s_and_b32 s1, exec_lo, s5
+; GFX10-NEXT:    s_or_b32 s4, s1, s4
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
 ; GFX10-NEXT:    s_cbranch_execz .LBB3_6
 ; GFX10-NEXT:  .LBB3_3: ; %A
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
-; GFX10-NEXT:    s_andn2_b32 s1, s1, exec_lo
-; GFX10-NEXT:    s_and_b32 s2, exec_lo, -1
-; GFX10-NEXT:    s_or_b32 s1, s1, s2
-; GFX10-NEXT:    v_lshlrev_b64 v[7:8], 2, v[6:7]
-; GFX10-NEXT:    v_add_co_u32 v9, vcc_lo, v2, v7
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, v3, v8, vcc_lo
-; GFX10-NEXT:    global_load_dword v9, v[9:10], off
+; GFX10-NEXT:    s_ashr_i32 s1, s0, 31
+; GFX10-NEXT:    s_lshl_b64 s[2:3], s[0:1], 2
+; GFX10-NEXT:    s_andn2_b32 s1, s5, exec_lo
+; GFX10-NEXT:    v_mov_b32_e32 v7, s3
+; GFX10-NEXT:    v_mov_b32_e32 v6, s2
+; GFX10-NEXT:    s_and_b32 s5, exec_lo, exec_lo
+; GFX10-NEXT:    s_or_b32 s5, s1, s5
+; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v2, v6
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v3, v7, vcc_lo
+; GFX10-NEXT:    global_load_dword v6, v[6:7], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v6
+; GFX10-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX10-NEXT:    s_cbranch_execz .LBB3_2
 ; GFX10-NEXT:  ; %bb.4: ; %B
 ; GFX10-NEXT:    ; in Loop: Header=BB3_3 Depth=1
-; GFX10-NEXT:    v_add_co_u32 v9, vcc_lo, v4, v7
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, v5, v8, vcc_lo
-; GFX10-NEXT:    s_mov_b32 s4, -1
-; GFX10-NEXT:    global_load_dword v9, v[9:10], off
+; GFX10-NEXT:    v_mov_b32_e32 v7, s3
+; GFX10-NEXT:    v_mov_b32_e32 v6, s2
+; GFX10-NEXT:    s_mov_b32 s6, exec_lo
+; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v4, v6
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v5, v7, vcc_lo
+; GFX10-NEXT:    global_load_dword v6, v[6:7], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    s_and_saveexec_b32 s3, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v6
+; GFX10-NEXT:    s_and_saveexec_b32 s7, vcc_lo
 ; GFX10-NEXT:    s_cbranch_execz .LBB3_1
 ; GFX10-NEXT:  ; %bb.5: ; %loop.body
 ; GFX10-NEXT:    ; in Loop: Header=BB3_3 Depth=1
-; GFX10-NEXT:    v_add_co_u32 v7, vcc_lo, v0, v7
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, v1, v8, vcc_lo
-; GFX10-NEXT:    v_add_nc_u32_e32 v10, 1, v6
-; GFX10-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 0x64, v6
-; GFX10-NEXT:    s_andn2_b32 s4, -1, exec_lo
-; GFX10-NEXT:    global_load_dword v9, v[7:8], off
-; GFX10-NEXT:    v_mov_b32_e32 v6, v10
-; GFX10-NEXT:    s_and_b32 s5, exec_lo, vcc_lo
-; GFX10-NEXT:    s_or_b32 s4, s4, s5
+; GFX10-NEXT:    v_mov_b32_e32 v7, s3
+; GFX10-NEXT:    v_mov_b32_e32 v6, s2
+; GFX10-NEXT:    s_add_i32 s2, s0, 1
+; GFX10-NEXT:    s_cmpk_lt_u32 s0, 0x64
+; GFX10-NEXT:    s_cselect_b32 s0, exec_lo, 0
+; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v0, v6
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v1, v7, vcc_lo
+; GFX10-NEXT:    s_andn2_b32 s3, s6, exec_lo
+; GFX10-NEXT:    s_and_b32 s0, exec_lo, s0
+; GFX10-NEXT:    s_or_b32 s6, s3, s0
+; GFX10-NEXT:    global_load_dword v8, v[6:7], off
+; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_add_nc_u32_e32 v9, 1, v9
-; GFX10-NEXT:    global_store_dword v[7:8], v9, off
+; GFX10-NEXT:    v_add_nc_u32_e32 v8, 1, v8
+; GFX10-NEXT:    global_store_dword v[6:7], v8, off
 ; GFX10-NEXT:    s_branch .LBB3_1
 ; GFX10-NEXT:  .LBB3_6: ; %exit
 ; GFX10-NEXT:    s_endpgm
@@ -265,78 +278,87 @@ exit:
 define amdgpu_cs void @loop_with_3breaks(ptr addrspace(1) %x, ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c) {
 ; GFX10-LABEL: loop_with_3breaks:
 ; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_mov_b32 s4, 0
 ; GFX10-NEXT:    s_mov_b32 s0, 0
-; GFX10-NEXT:    ; implicit-def: $sgpr1
-; GFX10-NEXT:    v_mov_b32_e32 v8, s0
+; GFX10-NEXT:    ; implicit-def: $sgpr5
 ; GFX10-NEXT:    s_branch .LBB4_4
 ; GFX10-NEXT:  .LBB4_1: ; %Flow5
 ; GFX10-NEXT:    ; in Loop: Header=BB4_4 Depth=1
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT:    s_andn2_b32 s4, -1, exec_lo
-; GFX10-NEXT:    s_and_b32 s5, exec_lo, s5
-; GFX10-NEXT:    s_or_b32 s4, s4, s5
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s9
+; GFX10-NEXT:    s_andn2_b32 s2, s6, exec_lo
+; GFX10-NEXT:    s_and_b32 s3, exec_lo, s8
+; GFX10-NEXT:    s_or_b32 s6, s2, s3
 ; GFX10-NEXT:  .LBB4_2: ; %Flow4
 ; GFX10-NEXT:    ; in Loop: Header=BB4_4 Depth=1
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s3
-; GFX10-NEXT:    s_andn2_b32 s1, s1, exec_lo
-; GFX10-NEXT:    s_and_b32 s3, exec_lo, s4
-; GFX10-NEXT:    s_or_b32 s1, s1, s3
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s7
+; GFX10-NEXT:    s_andn2_b32 s2, s5, exec_lo
+; GFX10-NEXT:    s_and_b32 s3, exec_lo, s6
+; GFX10-NEXT:    s_or_b32 s5, s2, s3
 ; GFX10-NEXT:  .LBB4_3: ; %Flow
 ; GFX10-NEXT:    ; in Loop: Header=BB4_4 Depth=1
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s2
-; GFX10-NEXT:    s_and_b32 s2, exec_lo, s1
-; GFX10-NEXT:    s_or_b32 s0, s2, s0
-; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX10-NEXT:    s_and_b32 s1, exec_lo, s5
+; GFX10-NEXT:    s_or_b32 s4, s1, s4
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
 ; GFX10-NEXT:    s_cbranch_execz .LBB4_8
 ; GFX10-NEXT:  .LBB4_4: ; %A
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    v_ashrrev_i32_e32 v9, 31, v8
-; GFX10-NEXT:    s_andn2_b32 s1, s1, exec_lo
-; GFX10-NEXT:    s_and_b32 s2, exec_lo, -1
-; GFX10-NEXT:    s_or_b32 s1, s1, s2
-; GFX10-NEXT:    v_lshlrev_b64 v[9:10], 2, v[8:9]
-; GFX10-NEXT:    v_add_co_u32 v11, vcc_lo, v2, v9
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v12, vcc_lo, v3, v10, vcc_lo
-; GFX10-NEXT:    global_load_dword v11, v[11:12], off
+; GFX10-NEXT:    s_ashr_i32 s1, s0, 31
+; GFX10-NEXT:    s_lshl_b64 s[2:3], s[0:1], 2
+; GFX10-NEXT:    s_andn2_b32 s1, s5, exec_lo
+; GFX10-NEXT:    v_mov_b32_e32 v9, s3
+; GFX10-NEXT:    v_mov_b32_e32 v8, s2
+; GFX10-NEXT:    s_and_b32 s5, exec_lo, exec_lo
+; GFX10-NEXT:    s_or_b32 s5, s1, s5
+; GFX10-NEXT:    v_add_co_u32 v8, vcc_lo, v2, v8
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v3, v9, vcc_lo
+; GFX10-NEXT:    global_load_dword v8, v[8:9], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v11
-; GFX10-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v8
+; GFX10-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX10-NEXT:    s_cbranch_execz .LBB4_3
 ; GFX10-NEXT:  ; %bb.5: ; %B
 ; GFX10-NEXT:    ; in Loop: Header=BB4_4 Depth=1
-; GFX10-NEXT:    v_add_co_u32 v11, vcc_lo, v4, v9
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v12, vcc_lo, v5, v10, vcc_lo
-; GFX10-NEXT:    s_mov_b32 s4, -1
-; GFX10-NEXT:    global_load_dword v11, v[11:12], off
+; GFX10-NEXT:    v_mov_b32_e32 v9, s3
+; GFX10-NEXT:    v_mov_b32_e32 v8, s2
+; GFX10-NEXT:    s_mov_b32 s6, exec_lo
+; GFX10-NEXT:    v_add_co_u32 v8, vcc_lo, v4, v8
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v5, v9, vcc_lo
+; GFX10-NEXT:    global_load_dword v8, v[8:9], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v11
-; GFX10-NEXT:    s_and_saveexec_b32 s3, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v8
+; GFX10-NEXT:    s_and_saveexec_b32 s7, vcc_lo
 ; GFX10-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX10-NEXT:  ; %bb.6: ; %C
 ; GFX10-NEXT:    ; in Loop: Header=BB4_4 Depth=1
-; GFX10-NEXT:    v_add_co_u32 v11, vcc_lo, v6, v9
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v12, vcc_lo, v7, v10, vcc_lo
-; GFX10-NEXT:    s_mov_b32 s5, -1
-; GFX10-NEXT:    global_load_dword v11, v[11:12], off
+; GFX10-NEXT:    v_mov_b32_e32 v9, s3
+; GFX10-NEXT:    v_mov_b32_e32 v8, s2
+; GFX10-NEXT:    s_mov_b32 s8, exec_lo
+; GFX10-NEXT:    v_add_co_u32 v8, vcc_lo, v6, v8
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v7, v9, vcc_lo
+; GFX10-NEXT:    global_load_dword v8, v[8:9], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v11
-; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v8
+; GFX10-NEXT:    s_and_saveexec_b32 s9, vcc_lo
 ; GFX10-NEXT:    s_cbranch_execz .LBB4_1
 ; GFX10-NEXT:  ; %bb.7: ; %loop.body
 ; GFX10-NEXT:    ; in Loop: Header=BB4_4 Depth=1
-; GFX10-NEXT:    v_add_co_u32 v9, vcc_lo, v0, v9
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, v1, v10, vcc_lo
-; GFX10-NEXT:    v_add_nc_u32_e32 v12, 1, v8
-; GFX10-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 0x64, v8
-; GFX10-NEXT:    s_andn2_b32 s5, -1, exec_lo
-; GFX10-NEXT:    global_load_dword v11, v[9:10], off
-; GFX10-NEXT:    v_mov_b32_e32 v8, v12
-; GFX10-NEXT:    s_and_b32 s6, exec_lo, vcc_lo
-; GFX10-NEXT:    s_or_b32 s5, s5, s6
+; GFX10-NEXT:    v_mov_b32_e32 v9, s3
+; GFX10-NEXT:    v_mov_b32_e32 v8, s2
+; GFX10-NEXT:    s_add_i32 s2, s0, 1
+; GFX10-NEXT:    s_cmpk_lt_u32 s0, 0x64
+; GFX10-NEXT:    s_cselect_b32 s0, exec_lo, 0
+; GFX10-NEXT:    v_add_co_u32 v8, vcc_lo, v0, v8
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v1, v9, vcc_lo
+; GFX10-NEXT:    s_andn2_b32 s3, s8, exec_lo
+; GFX10-NEXT:    s_and_b32 s0, exec_lo, s0
+; GFX10-NEXT:    s_or_b32 s8, s3, s0
+; GFX10-NEXT:    global_load_dword v10, v[8:9], off
+; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_add_nc_u32_e32 v11, 1, v11
-; GFX10-NEXT:    global_store_dword v[9:10], v11, off
+; GFX10-NEXT:    v_add_nc_u32_e32 v10, 1, v10
+; GFX10-NEXT:    global_store_dword v[8:9], v10, off
 ; GFX10-NEXT:    s_branch .LBB4_1
 ; GFX10-NEXT:  .LBB4_8: ; %exit
 ; GFX10-NEXT:    s_endpgm
@@ -382,60 +404,67 @@ exit:
 define amdgpu_cs void @loop_with_div_break_with_body(ptr addrspace(1) %x, ptr addrspace(1) %a, ptr addrspace(1) %a.break) {
 ; GFX10-LABEL: loop_with_div_break_with_body:
 ; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_mov_b32 s4, 0
 ; GFX10-NEXT:    s_mov_b32 s0, 0
-; GFX10-NEXT:    ; implicit-def: $sgpr1
-; GFX10-NEXT:    ; implicit-def: $sgpr2
-; GFX10-NEXT:    ; implicit-def: $sgpr3
-; GFX10-NEXT:    v_mov_b32_e32 v6, s0
+; GFX10-NEXT:    ; implicit-def: $sgpr6
+; GFX10-NEXT:    ; implicit-def: $sgpr7
+; GFX10-NEXT:    ; implicit-def: $sgpr5
 ; GFX10-NEXT:    s_branch .LBB5_2
 ; GFX10-NEXT:  .LBB5_1: ; %Flow
 ; GFX10-NEXT:    ; in Loop: Header=BB5_2 Depth=1
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT:    s_and_b32 s4, exec_lo, s2
-; GFX10-NEXT:    s_or_b32 s0, s4, s0
-; GFX10-NEXT:    s_andn2_b32 s1, s1, exec_lo
-; GFX10-NEXT:    s_and_b32 s4, exec_lo, s3
-; GFX10-NEXT:    s_or_b32 s1, s1, s4
-; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX10-NEXT:    s_and_b32 s1, exec_lo, s6
+; GFX10-NEXT:    s_or_b32 s4, s1, s4
+; GFX10-NEXT:    s_andn2_b32 s1, s5, exec_lo
+; GFX10-NEXT:    s_and_b32 s2, exec_lo, s7
+; GFX10-NEXT:    s_or_b32 s5, s1, s2
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
 ; GFX10-NEXT:    s_cbranch_execz .LBB5_4
 ; GFX10-NEXT:  .LBB5_2: ; %A
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
-; GFX10-NEXT:    s_andn2_b32 s3, s3, exec_lo
-; GFX10-NEXT:    s_and_b32 s4, exec_lo, -1
-; GFX10-NEXT:    s_andn2_b32 s2, s2, exec_lo
-; GFX10-NEXT:    s_or_b32 s3, s3, s4
-; GFX10-NEXT:    v_lshlrev_b64 v[7:8], 2, v[6:7]
-; GFX10-NEXT:    s_or_b32 s2, s2, s4
-; GFX10-NEXT:    v_add_co_u32 v9, vcc_lo, v2, v7
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, v3, v8, vcc_lo
-; GFX10-NEXT:    global_load_dword v9, v[9:10], off
+; GFX10-NEXT:    s_ashr_i32 s1, s0, 31
+; GFX10-NEXT:    s_mov_b32 s8, exec_lo
+; GFX10-NEXT:    s_lshl_b64 s[2:3], s[0:1], 2
+; GFX10-NEXT:    s_andn2_b32 s1, s7, exec_lo
+; GFX10-NEXT:    v_mov_b32_e32 v7, s3
+; GFX10-NEXT:    v_mov_b32_e32 v6, s2
+; GFX10-NEXT:    s_and_b32 s7, exec_lo, s8
+; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
+; GFX10-NEXT:    s_and_b32 s8, exec_lo, exec_lo
+; GFX10-NEXT:    s_or_b32 s7, s1, s7
+; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v2, v6
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v3, v7, vcc_lo
+; GFX10-NEXT:    s_or_b32 s6, s6, s8
+; GFX10-NEXT:    global_load_dword v6, v[6:7], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v6
+; GFX10-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX10-NEXT:    s_cbranch_execz .LBB5_1
 ; GFX10-NEXT:  ; %bb.3: ; %loop.body
 ; GFX10-NEXT:    ; in Loop: Header=BB5_2 Depth=1
-; GFX10-NEXT:    v_add_co_u32 v7, vcc_lo, v0, v7
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, v1, v8, vcc_lo
-; GFX10-NEXT:    v_add_nc_u32_e32 v10, 1, v6
-; GFX10-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 0x64, v6
-; GFX10-NEXT:    s_andn2_b32 s3, s3, exec_lo
-; GFX10-NEXT:    global_load_dword v9, v[7:8], off
-; GFX10-NEXT:    s_and_b32 s5, exec_lo, 0
-; GFX10-NEXT:    v_mov_b32_e32 v6, v10
-; GFX10-NEXT:    s_andn2_b32 s2, s2, exec_lo
-; GFX10-NEXT:    s_and_b32 s6, exec_lo, vcc_lo
-; GFX10-NEXT:    s_or_b32 s3, s3, s5
-; GFX10-NEXT:    s_or_b32 s2, s2, s6
+; GFX10-NEXT:    v_mov_b32_e32 v7, s3
+; GFX10-NEXT:    v_mov_b32_e32 v6, s2
+; GFX10-NEXT:    s_add_i32 s2, s0, 1
+; GFX10-NEXT:    s_cmpk_lt_u32 s0, 0x64
+; GFX10-NEXT:    s_cselect_b32 s0, exec_lo, 0
+; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v0, v6
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v1, v7, vcc_lo
+; GFX10-NEXT:    s_andn2_b32 s3, s7, exec_lo
+; GFX10-NEXT:    s_and_b32 s7, exec_lo, 0
+; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
+; GFX10-NEXT:    global_load_dword v8, v[6:7], off
+; GFX10-NEXT:    s_and_b32 s0, exec_lo, s0
+; GFX10-NEXT:    s_or_b32 s7, s3, s7
+; GFX10-NEXT:    s_or_b32 s6, s6, s0
+; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_add_nc_u32_e32 v9, 1, v9
-; GFX10-NEXT:    global_store_dword v[7:8], v9, off
+; GFX10-NEXT:    v_add_nc_u32_e32 v8, 1, v8
+; GFX10-NEXT:    global_store_dword v[6:7], v8, off
 ; GFX10-NEXT:    s_branch .LBB5_1
 ; GFX10-NEXT:  .LBB5_4: ; %loop.exit.guard
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX10-NEXT:    s_and_saveexec_b32 s0, s1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_and_saveexec_b32 s0, s5
 ; GFX10-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX10-NEXT:    s_cbranch_execz .LBB5_6
 ; GFX10-NEXT:  ; %bb.5: ; %break.body
@@ -472,8 +501,7 @@ exit:
 }
 
 ; Snippet from test generated by the GraphicsFuzz tool, frontend generates ir
-; with irreducible control flow graph. FixIrreducible converts it into natural
-; loop and in the process creates i1 phi with three incoming values.
+; with irreducible control flow graph.
 
 ; int loop(int x, int y, int a0, int a1, int a2, int a3, int a4) {
 ;   do {
@@ -488,27 +516,100 @@ exit:
 ;   return a0;
 ; }
 
-; This test is also interesting because it has phi with three incomings
-;define amdgpu_ps i32 @irreducible_cfg(i32 %x, i32 %y, i32 %a0, i32 %a1, i32 %a2, i32 %a3) {
-;.entry:
-; %.y_lt_a2 = icmp sgt i32 %a2, %y
-; %.x_lt_a2 = icmp sgt i32 %a2, %x
-; %.x_lt_a3 = icmp sgt i32 %a3, %x
-; br i1 %.y_lt_a2, label %.preheader, label %.loopexit ; first iteration, jump to inner loop if 'y < a2' or start with 'if (x < a3)'
-;
-;.preheader: ; if (y < a2),
-; br label %.inner_loop
-;
-;.inner_loop: ; do while x < a2
-; br i1 %.x_lt_a2, label %.inner_loop, label %.loopexit
-;
-;.loopexit: ; if x < a3
-; %not.inner_loop = xor i1 %.y_lt_a2, true
-; %brmerge = select i1 %.x_lt_a3, i1 true, i1 %not.inner_loop ; exit loop if 'x < a3' or 'loop ends since !(y < a2)'
-; %.ret = select i1 %.x_lt_a3, i32 %a1, i32 %a0               ; select retrun value a1 'x < a3' or a0 'loop ends'
-; br i1 %brmerge, label %.exit, label %.preheader
-;
-;.exit:
-; ret i32 %.ret
-;}
+; This test is also interesting because it had phi with three incomings
+; After fa4cc9ddd58eb9fef2497e678873ff3b495340a3, FixIrreducible does not
+; generate phi with three incomings. There is a mir test with such phi.
+define amdgpu_ps i32 @irreducible_cfg(i32 %x, i32 %y, i32 %a0, i32 %a1, i32 %a2, i32 %a3) {
+; GFX10-LABEL: irreducible_cfg:
+; GFX10:       ; %bb.0: ; %.entry
+; GFX10-NEXT:    v_cmp_gt_i32_e32 vcc_lo, v4, v1
+; GFX10-NEXT:    s_mov_b32 s0, exec_lo
+; GFX10-NEXT:    s_mov_b32 s1, 0
+; GFX10-NEXT:    s_and_b32 s2, s0, 1
+; GFX10-NEXT:    s_xor_b32 s0, vcc_lo, s0
+; GFX10-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX10-NEXT:    ; implicit-def: $sgpr2
+; GFX10-NEXT:    s_cselect_b32 s3, exec_lo, 0
+; GFX10-NEXT:    s_branch .LBB6_2
+; GFX10-NEXT:  .LBB6_1: ; %Flow2
+; GFX10-NEXT:    ; in Loop: Header=BB6_2 Depth=1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT:    s_and_b32 s4, exec_lo, s5
+; GFX10-NEXT:    s_mov_b32 s0, exec_lo
+; GFX10-NEXT:    s_or_b32 s1, s4, s1
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s1
+; GFX10-NEXT:    s_cbranch_execz .LBB6_8
+; GFX10-NEXT:  .LBB6_2: ; %irr.guard
+; GFX10-NEXT:    ; =>This Loop Header: Depth=1
+; GFX10-NEXT:    ; Child Loop BB6_6 Depth 2
+; GFX10-NEXT:    s_mov_b32 s4, exec_lo
+; GFX10-NEXT:    s_and_saveexec_b32 s5, s0
+; GFX10-NEXT:    s_xor_b32 s5, exec_lo, s5
+; GFX10-NEXT:  ; %bb.3: ; %.loopexit
+; GFX10-NEXT:    ; in Loop: Header=BB6_2 Depth=1
+; GFX10-NEXT:    v_cmp_gt_i32_e64 s0, v5, v0
+; GFX10-NEXT:    s_mov_b32 s6, exec_lo
+; GFX10-NEXT:    s_mov_b32 s7, exec_lo
+; GFX10-NEXT:    s_xor_b32 s6, vcc_lo, s6
+; GFX10-NEXT:    s_andn2_b32 s3, s3, exec_lo
+; GFX10-NEXT:    s_or_b32 s6, s0, s6
+; GFX10-NEXT:    s_and_b32 s0, exec_lo, s0
+; GFX10-NEXT:    s_xor_b32 s6, s6, s7
+; GFX10-NEXT:    s_andn2_b32 s4, s4, exec_lo
+; GFX10-NEXT:    s_and_b32 s6, exec_lo, s6
+; GFX10-NEXT:    s_or_b32 s3, s3, s0
+; GFX10-NEXT:    s_or_b32 s4, s4, s6
+; GFX10-NEXT:  ; %bb.4: ; %Flow1
+; GFX10-NEXT:    ; in Loop: Header=BB6_2 Depth=1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    s_andn2_b32 s0, s2, exec_lo
+; GFX10-NEXT:    s_and_b32 s2, exec_lo, s3
+; GFX10-NEXT:    s_mov_b32 s5, exec_lo
+; GFX10-NEXT:    s_or_b32 s2, s0, s2
+; GFX10-NEXT:    s_and_saveexec_b32 s6, s4
+; GFX10-NEXT:    s_cbranch_execz .LBB6_1
+; GFX10-NEXT:  ; %bb.5: ; %.preheader
+; GFX10-NEXT:    ; in Loop: Header=BB6_2 Depth=1
+; GFX10-NEXT:    v_cmp_le_i32_e64 s0, v4, v0
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:  .LBB6_6: ; %.inner_loop
+; GFX10-NEXT:    ; Parent Loop BB6_2 Depth=1
+; GFX10-NEXT:    ; => This Inner Loop Header: Depth=2
+; GFX10-NEXT:    s_and_b32 s7, exec_lo, s0
+; GFX10-NEXT:    s_or_b32 s4, s7, s4
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_cbranch_execnz .LBB6_6
+; GFX10-NEXT:  ; %bb.7: ; %Flow
+; GFX10-NEXT:    ; in Loop: Header=BB6_2 Depth=1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_andn2_b32 s0, s5, exec_lo
+; GFX10-NEXT:    s_and_b32 s4, exec_lo, 0
+; GFX10-NEXT:    s_or_b32 s5, s0, s4
+; GFX10-NEXT:    s_branch .LBB6_1
+; GFX10-NEXT:  .LBB6_8: ; %.exit
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v2, v3, s2
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+.entry:
+ %.y_lt_a2 = icmp sgt i32 %a2, %y
+ %.x_lt_a2 = icmp sgt i32 %a2, %x
+ %.x_lt_a3 = icmp sgt i32 %a3, %x
+ br i1 %.y_lt_a2, label %.preheader, label %.loopexit ; first iteration, jump to inner loop if 'y < a2' or start with 'if (x < a3)'
+
+.preheader: ; if (y < a2),
+ br label %.inner_loop
+
+.inner_loop: ; do while x < a2
+ br i1 %.x_lt_a2, label %.inner_loop, label %.loopexit
+
+.loopexit: ; if x < a3
+ %not.inner_loop = xor i1 %.y_lt_a2, true
+ %brmerge = select i1 %.x_lt_a3, i1 true, i1 %not.inner_loop ; exit loop if 'x < a3' or 'loop ends since !(y < a2)'
+ %.ret = select i1 %.x_lt_a3, i32 %a1, i32 %a0               ; select retrun value a1 'x < a3' or a0 'loop ends'
+ br i1 %brmerge, label %.exit, label %.preheader
+
+.exit:
+ ret i32 %.ret
+}
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.mir
index 39ebf66411cc6..b76d421c16172 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.mir
@@ -39,10 +39,10 @@ body: |
   ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[COPY4]](s1), %bb.0, [[S_OR_B32_]](s1), %bb.1
   ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
   ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[COPY7]](s1)
   ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY7]](s1), [[C4]], [[C3]]
-  ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SEXT]], [[C3]]
+  ; GFX10-NEXT:   G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
   ; GFX10-NEXT:   S_ENDPGM 0
   bb.0:
     successors: %bb.1(0x40000000), %bb.2(0x40000000)
@@ -69,9 +69,9 @@ body: |
   bb.2:
     %12:_(s1) = G_PHI %6(s1), %bb.0, %11(s1), %bb.1
     G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %9(s32)
-    %13:_(s32) = G_CONSTANT i32 2
-    %14:_(s32) = G_CONSTANT i32 1
-    %15:_(s32) = G_SELECT %12(s1), %14, %13
+    %13:_(s32) = G_SEXT %12(s1)
+    %14:_(s32) = G_CONSTANT i32 2
+    %15:_(s32) = G_ADD %13, %14
     G_STORE %15(s32), %2(p1) :: (store (s32), addrspace 1)
     S_ENDPGM 0
 ...
@@ -135,10 +135,10 @@ body: |
   ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[COPY7]](s1), %bb.1, [[S_OR_B32_]](s1), %bb.2
   ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1)
   ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_ELSE]](s32)
-  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY11]](s1), [[C3]], [[C4]]
-  ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[COPY11]](s1)
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SEXT]], [[C3]]
+  ; GFX10-NEXT:   G_STORE [[ADD]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
   ; GFX10-NEXT:   S_ENDPGM 0
   bb.0:
     successors: %bb.3(0x40000000), %bb.1(0x40000000)
@@ -179,9 +179,9 @@ body: |
   bb.4:
     %15:_(s1) = G_PHI %9(s1), %bb.1, %13(s1), %bb.2
     G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %11(s32)
-    %16:_(s32) = G_CONSTANT i32 1
+    %16:_(s32) = G_SEXT %15(s1)
     %17:_(s32) = G_CONSTANT i32 2
-    %18:_(s32) = G_SELECT %15(s1), %16, %17
+    %18:_(s32) = G_ADD %16, %17
     G_STORE %18(s32), %2(p1) :: (store (s32), addrspace 1)
     S_ENDPGM 0
 ...
@@ -202,26 +202,26 @@ body: |
   ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
   ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
-  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.1:
   ; GFX10-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %35(s1), %bb.3
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %34(s1), %bb.3
   ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI %9(s32), %bb.3, [[C]](s32), %bb.0
   ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %11(s32), %bb.3
   ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
-  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[C1]](s1)
   ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI2]](s32)
-  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C2]](s32)
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C1]](s32)
   ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV1]], [[SHL]](s64)
   ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
-  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C3]]
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C2]]
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1)
   ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY4]](s1), $exec_lo, implicit-def $scc
   ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY5]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
@@ -259,8 +259,7 @@ body: |
   ; GFX10-NEXT:   G_BR %bb.4
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.4:
-  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.3
-  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI5]](s32)
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
   ; GFX10-NEXT:   S_ENDPGM 0
   bb.0:
     successors: %bb.1(0x80000000)
@@ -272,30 +271,30 @@ body: |
     %3:_(s32) = COPY $vgpr2
     %4:_(s32) = COPY $vgpr3
     %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
-    %6:_(s32) = G_CONSTANT i32 0
-    %7:_(s32) = G_IMPLICIT_DEF
+    %6:_(s32) = G_IMPLICIT_DEF
+    %7:_(s32) = G_CONSTANT i32 0
 
   bb.1:
     successors: %bb.2(0x40000000), %bb.3(0x40000000)
 
-    %8:_(s32) = G_PHI %9(s32), %bb.3, %6(s32), %bb.0
-    %10:_(s32) = G_PHI %6(s32), %bb.0, %11(s32), %bb.3
-    %12:_(s1) = G_CONSTANT i1 true
-    %13:_(s64) = G_SEXT %10(s32)
-    %14:_(s32) = G_CONSTANT i32 2
-    %15:_(s64) = G_SHL %13, %14(s32)
-    %16:_(p1) = G_PTR_ADD %5, %15(s64)
-    %17:_(s32) = G_LOAD %16(p1) :: (load (s32), addrspace 1)
-    %18:_(s32) = G_CONSTANT i32 0
-    %19:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %17(s32), %18
-    %20:sreg_32_xm0_xexec(s32) = SI_IF %19(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+    %8:_(s32) = G_PHI %9(s32), %bb.3, %7(s32), %bb.0
+    %10:_(s32) = G_PHI %7(s32), %bb.0, %11(s32), %bb.3
+    %12:_(s64) = G_SEXT %10(s32)
+    %13:_(s32) = G_CONSTANT i32 2
+    %14:_(s64) = G_SHL %12, %13(s32)
+    %15:_(p1) = G_PTR_ADD %5, %14(s64)
+    %16:_(s32) = G_LOAD %15(p1) :: (load (s32), addrspace 1)
+    %17:_(s32) = G_CONSTANT i32 0
+    %18:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %16(s32), %17
+    %19:_(s1) = G_CONSTANT i1 true
+    %20:sreg_32_xm0_xexec(s32) = SI_IF %18(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.2
 
   bb.2:
     successors: %bb.3(0x80000000)
 
     %21:_(s32) = G_CONSTANT i32 2
-    %22:_(s64) = G_SHL %13, %21(s32)
+    %22:_(s64) = G_SHL %12, %21(s32)
     %23:_(p1) = G_PTR_ADD %2, %22(s64)
     %24:_(s32) = G_LOAD %23(p1) :: (load (s32), addrspace 1)
     %25:_(s32) = G_CONSTANT i32 1
@@ -308,16 +307,15 @@ body: |
   bb.3:
     successors: %bb.4(0x04000000), %bb.1(0x7c000000)
 
-    %11:_(s32) = G_PHI %27(s32), %bb.2, %7(s32), %bb.1
-    %30:_(s1) = G_PHI %29(s1), %bb.2, %12(s1), %bb.1
+    %11:_(s32) = G_PHI %27(s32), %bb.2, %6(s32), %bb.1
+    %30:_(s1) = G_PHI %29(s1), %bb.2, %19(s1), %bb.1
     G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %20(s32)
     %9:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %30(s1), %8(s32)
     SI_LOOP %9(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.4
 
   bb.4:
-    %31:_(s32) = G_PHI %9(s32), %bb.3
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %31(s32)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %9(s32)
     S_ENDPGM 0
 ...
 
@@ -340,26 +338,26 @@ body: |
   ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
   ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
   ; GFX10-NEXT:   [[MV2:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
-  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.1:
   ; GFX10-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %48(s1), %bb.3
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %47(s1), %bb.3
   ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI %12(s32), %bb.3, [[C]](s32), %bb.0
   ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %14(s32), %bb.3
   ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
-  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[C1]](s1)
   ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI2]](s32)
-  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C2]](s32)
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C1]](s32)
   ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV1]], [[SHL]](s64)
   ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
-  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C3]]
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C2]]
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1)
   ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
   ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY7]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
@@ -370,14 +368,14 @@ body: |
   ; GFX10-NEXT: bb.2:
   ; GFX10-NEXT:   successors: %bb.4(0x40000000), %bb.5(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-  ; GFX10-NEXT:   [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C5]](s32)
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C4]](s32)
   ; GFX10-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV2]], [[SHL1]](s64)
   ; GFX10-NEXT:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1)
-  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD1]](s32), [[C6]]
-  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[C4]](s1)
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD1]](s32), [[C5]]
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[C6]](s1)
   ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[COPY9]](s1)
   ; GFX10-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP1]](s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.4
@@ -385,7 +383,7 @@ body: |
   ; GFX10-NEXT: bb.3:
   ; GFX10-NEXT:   successors: %bb.6(0x04000000), %bb.1(0x7c000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, %47(s1), %bb.5
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, %46(s1), %bb.5
   ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI %32(s32), %bb.5, [[DEF]](s32), %bb.1
   ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1)
   ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
@@ -425,8 +423,7 @@ body: |
   ; GFX10-NEXT:   G_BR %bb.3
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.6:
-  ; GFX10-NEXT:   [[PHI7:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.3
-  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI7]](s32)
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
   ; GFX10-NEXT:   S_ENDPGM 0
   bb.0:
     successors: %bb.1(0x80000000)
@@ -441,43 +438,43 @@ body: |
     %6:_(s32) = COPY $vgpr4
     %7:_(s32) = COPY $vgpr5
     %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
-    %9:_(s32) = G_CONSTANT i32 0
-    %10:_(s32) = G_IMPLICIT_DEF
+    %9:_(s32) = G_IMPLICIT_DEF
+    %10:_(s32) = G_CONSTANT i32 0
 
   bb.1:
     successors: %bb.2(0x40000000), %bb.3(0x40000000)
 
-    %11:_(s32) = G_PHI %12(s32), %bb.3, %9(s32), %bb.0
-    %13:_(s32) = G_PHI %9(s32), %bb.0, %14(s32), %bb.3
-    %15:_(s1) = G_CONSTANT i1 true
-    %16:_(s64) = G_SEXT %13(s32)
-    %17:_(s32) = G_CONSTANT i32 2
-    %18:_(s64) = G_SHL %16, %17(s32)
-    %19:_(p1) = G_PTR_ADD %5, %18(s64)
-    %20:_(s32) = G_LOAD %19(p1) :: (load (s32), addrspace 1)
-    %21:_(s32) = G_CONSTANT i32 0
-    %22:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %20(s32), %21
-    %23:sreg_32_xm0_xexec(s32) = SI_IF %22(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+    %11:_(s32) = G_PHI %12(s32), %bb.3, %10(s32), %bb.0
+    %13:_(s32) = G_PHI %10(s32), %bb.0, %14(s32), %bb.3
+    %15:_(s64) = G_SEXT %13(s32)
+    %16:_(s32) = G_CONSTANT i32 2
+    %17:_(s64) = G_SHL %15, %16(s32)
+    %18:_(p1) = G_PTR_ADD %5, %17(s64)
+    %19:_(s32) = G_LOAD %18(p1) :: (load (s32), addrspace 1)
+    %20:_(s32) = G_CONSTANT i32 0
+    %21:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %19(s32), %20
+    %22:_(s1) = G_CONSTANT i1 true
+    %23:sreg_32_xm0_xexec(s32) = SI_IF %21(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.2
 
   bb.2:
     successors: %bb.4(0x40000000), %bb.5(0x40000000)
 
-    %24:_(s1) = G_CONSTANT i1 true
-    %25:_(s32) = G_CONSTANT i32 2
-    %26:_(s64) = G_SHL %16, %25(s32)
-    %27:_(p1) = G_PTR_ADD %8, %26(s64)
-    %28:_(s32) = G_LOAD %27(p1) :: (load (s32), addrspace 1)
-    %29:_(s32) = G_CONSTANT i32 0
-    %30:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %28(s32), %29
-    %31:sreg_32_xm0_xexec(s32) = SI_IF %30(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+    %24:_(s32) = G_CONSTANT i32 2
+    %25:_(s64) = G_SHL %15, %24(s32)
+    %26:_(p1) = G_PTR_ADD %8, %25(s64)
+    %27:_(s32) = G_LOAD %26(p1) :: (load (s32), addrspace 1)
+    %28:_(s32) = G_CONSTANT i32 0
+    %29:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %27(s32), %28
+    %30:_(s1) = G_CONSTANT i1 true
+    %31:sreg_32_xm0_xexec(s32) = SI_IF %29(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.4
 
   bb.3:
     successors: %bb.6(0x04000000), %bb.1(0x7c000000)
 
-    %14:_(s32) = G_PHI %32(s32), %bb.5, %10(s32), %bb.1
-    %33:_(s1) = G_PHI %34(s1), %bb.5, %15(s1), %bb.1
+    %14:_(s32) = G_PHI %32(s32), %bb.5, %9(s32), %bb.1
+    %33:_(s1) = G_PHI %34(s1), %bb.5, %22(s1), %bb.1
     G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %23(s32)
     %12:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %33(s1), %11(s32)
     SI_LOOP %12(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
@@ -487,7 +484,7 @@ body: |
     successors: %bb.5(0x80000000)
 
     %35:_(s32) = G_CONSTANT i32 2
-    %36:_(s64) = G_SHL %16, %35(s32)
+    %36:_(s64) = G_SHL %15, %35(s32)
     %37:_(p1) = G_PTR_ADD %2, %36(s64)
     %38:_(s32) = G_LOAD %37(p1) :: (load (s32), addrspace 1)
     %39:_(s32) = G_CONSTANT i32 1
@@ -500,14 +497,13 @@ body: |
   bb.5:
     successors: %bb.3(0x80000000)
 
-    %32:_(s32) = G_PHI %41(s32), %bb.4, %10(s32), %bb.2
-    %34:_(s1) = G_PHI %43(s1), %bb.4, %24(s1), %bb.2
+    %32:_(s32) = G_PHI %41(s32), %bb.4, %9(s32), %bb.2
+    %34:_(s1) = G_PHI %43(s1), %bb.4, %30(s1), %bb.2
     G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %31(s32)
     G_BR %bb.3
 
   bb.6:
-    %44:_(s32) = G_PHI %12(s32), %bb.3
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %44(s32)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %12(s32)
     S_ENDPGM 0
 ...
 
@@ -533,26 +529,26 @@ body: |
   ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
   ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
   ; GFX10-NEXT:   [[MV3:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.1:
   ; GFX10-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %61(s1), %bb.3
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %60(s1), %bb.3
   ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI %15(s32), %bb.3, [[C]](s32), %bb.0
   ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %17(s32), %bb.3
   ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
-  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[C1]](s1)
   ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI2]](s32)
-  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C2]](s32)
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C1]](s32)
   ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV1]], [[SHL]](s64)
   ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
-  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C3]]
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C2]]
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1)
   ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc
   ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY9]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
@@ -563,14 +559,14 @@ body: |
   ; GFX10-NEXT: bb.2:
   ; GFX10-NEXT:   successors: %bb.4(0x40000000), %bb.5(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-  ; GFX10-NEXT:   [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C5]](s32)
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C4]](s32)
   ; GFX10-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV2]], [[SHL1]](s64)
   ; GFX10-NEXT:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1)
-  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD1]](s32), [[C6]]
-  ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[C4]](s1)
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD1]](s32), [[C5]]
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[C6]](s1)
   ; GFX10-NEXT:   [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[COPY11]](s1)
   ; GFX10-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP1]](s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.4
@@ -578,7 +574,7 @@ body: |
   ; GFX10-NEXT: bb.3:
   ; GFX10-NEXT:   successors: %bb.8(0x04000000), %bb.1(0x7c000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, %60(s1), %bb.5
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, %59(s1), %bb.5
   ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI %35(s32), %bb.5, [[DEF]](s32), %bb.1
   ; GFX10-NEXT:   [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1)
   ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
@@ -589,14 +585,14 @@ body: |
   ; GFX10-NEXT: bb.4:
   ; GFX10-NEXT:   successors: %bb.6(0x40000000), %bb.7(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-  ; GFX10-NEXT:   [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C8]](s32)
+  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C7]](s32)
   ; GFX10-NEXT:   [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV3]], [[SHL2]](s64)
   ; GFX10-NEXT:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p1) :: (load (s32), addrspace 1)
-  ; GFX10-NEXT:   [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; GFX10-NEXT:   [[ICMP2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD2]](s32), [[C9]]
-  ; GFX10-NEXT:   [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[C7]](s1)
+  ; GFX10-NEXT:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD2]](s32), [[C8]]
+  ; GFX10-NEXT:   [[C9:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[C9]](s1)
   ; GFX10-NEXT:   [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[COPY14]](s1)
   ; GFX10-NEXT:   [[SI_IF2:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP2]](s1), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.6
@@ -604,7 +600,7 @@ body: |
   ; GFX10-NEXT: bb.5:
   ; GFX10-NEXT:   successors: %bb.3(0x80000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[COPY11]](s1), %bb.2, %72(s1), %bb.7
+  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[COPY11]](s1), %bb.2, %71(s1), %bb.7
   ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:_(s32) = G_PHI %46(s32), %bb.7, [[DEF]](s32), %bb.2
   ; GFX10-NEXT:   [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1)
   ; GFX10-NEXT:   [[COPY17:%[0-9]+]]:sreg_32(s1) = COPY [[COPY16]](s1)
@@ -646,8 +642,7 @@ body: |
   ; GFX10-NEXT:   G_BR %bb.5
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.8:
-  ; GFX10-NEXT:   [[PHI9:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.3
-  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI9]](s32)
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
   ; GFX10-NEXT:   S_ENDPGM 0
   bb.0:
     successors: %bb.1(0x80000000)
@@ -665,43 +660,43 @@ body: |
     %9:_(s32) = COPY $vgpr6
     %10:_(s32) = COPY $vgpr7
     %11:_(p1) = G_MERGE_VALUES %9(s32), %10(s32)
-    %12:_(s32) = G_CONSTANT i32 0
-    %13:_(s32) = G_IMPLICIT_DEF
+    %12:_(s32) = G_IMPLICIT_DEF
+    %13:_(s32) = G_CONSTANT i32 0
 
   bb.1:
     successors: %bb.2(0x40000000), %bb.3(0x40000000)
 
-    %14:_(s32) = G_PHI %15(s32), %bb.3, %12(s32), %bb.0
-    %16:_(s32) = G_PHI %12(s32), %bb.0, %17(s32), %bb.3
-    %18:_(s1) = G_CONSTANT i1 true
-    %19:_(s64) = G_SEXT %16(s32)
-    %20:_(s32) = G_CONSTANT i32 2
-    %21:_(s64) = G_SHL %19, %20(s32)
-    %22:_(p1) = G_PTR_ADD %5, %21(s64)
-    %23:_(s32) = G_LOAD %22(p1) :: (load (s32), addrspace 1)
-    %24:_(s32) = G_CONSTANT i32 0
-    %25:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %23(s32), %24
-    %26:sreg_32_xm0_xexec(s32) = SI_IF %25(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+    %14:_(s32) = G_PHI %15(s32), %bb.3, %13(s32), %bb.0
+    %16:_(s32) = G_PHI %13(s32), %bb.0, %17(s32), %bb.3
+    %18:_(s64) = G_SEXT %16(s32)
+    %19:_(s32) = G_CONSTANT i32 2
+    %20:_(s64) = G_SHL %18, %19(s32)
+    %21:_(p1) = G_PTR_ADD %5, %20(s64)
+    %22:_(s32) = G_LOAD %21(p1) :: (load (s32), addrspace 1)
+    %23:_(s32) = G_CONSTANT i32 0
+    %24:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %22(s32), %23
+    %25:_(s1) = G_CONSTANT i1 true
+    %26:sreg_32_xm0_xexec(s32) = SI_IF %24(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.2
 
   bb.2:
     successors: %bb.4(0x40000000), %bb.5(0x40000000)
 
-    %27:_(s1) = G_CONSTANT i1 true
-    %28:_(s32) = G_CONSTANT i32 2
-    %29:_(s64) = G_SHL %19, %28(s32)
-    %30:_(p1) = G_PTR_ADD %8, %29(s64)
-    %31:_(s32) = G_LOAD %30(p1) :: (load (s32), addrspace 1)
-    %32:_(s32) = G_CONSTANT i32 0
-    %33:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %31(s32), %32
-    %34:sreg_32_xm0_xexec(s32) = SI_IF %33(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+    %27:_(s32) = G_CONSTANT i32 2
+    %28:_(s64) = G_SHL %18, %27(s32)
+    %29:_(p1) = G_PTR_ADD %8, %28(s64)
+    %30:_(s32) = G_LOAD %29(p1) :: (load (s32), addrspace 1)
+    %31:_(s32) = G_CONSTANT i32 0
+    %32:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %30(s32), %31
+    %33:_(s1) = G_CONSTANT i1 true
+    %34:sreg_32_xm0_xexec(s32) = SI_IF %32(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.4
 
   bb.3:
     successors: %bb.8(0x04000000), %bb.1(0x7c000000)
 
-    %17:_(s32) = G_PHI %35(s32), %bb.5, %13(s32), %bb.1
-    %36:_(s1) = G_PHI %37(s1), %bb.5, %18(s1), %bb.1
+    %17:_(s32) = G_PHI %35(s32), %bb.5, %12(s32), %bb.1
+    %36:_(s1) = G_PHI %37(s1), %bb.5, %25(s1), %bb.1
     G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %26(s32)
     %15:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %36(s1), %14(s32)
     SI_LOOP %15(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
@@ -710,21 +705,21 @@ body: |
   bb.4:
     successors: %bb.6(0x40000000), %bb.7(0x40000000)
 
-    %38:_(s1) = G_CONSTANT i1 true
-    %39:_(s32) = G_CONSTANT i32 2
-    %40:_(s64) = G_SHL %19, %39(s32)
-    %41:_(p1) = G_PTR_ADD %11, %40(s64)
-    %42:_(s32) = G_LOAD %41(p1) :: (load (s32), addrspace 1)
-    %43:_(s32) = G_CONSTANT i32 0
-    %44:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %42(s32), %43
-    %45:sreg_32_xm0_xexec(s32) = SI_IF %44(s1), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec
+    %38:_(s32) = G_CONSTANT i32 2
+    %39:_(s64) = G_SHL %18, %38(s32)
+    %40:_(p1) = G_PTR_ADD %11, %39(s64)
+    %41:_(s32) = G_LOAD %40(p1) :: (load (s32), addrspace 1)
+    %42:_(s32) = G_CONSTANT i32 0
+    %43:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %41(s32), %42
+    %44:_(s1) = G_CONSTANT i1 true
+    %45:sreg_32_xm0_xexec(s32) = SI_IF %43(s1), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.6
 
   bb.5:
     successors: %bb.3(0x80000000)
 
-    %35:_(s32) = G_PHI %46(s32), %bb.7, %13(s32), %bb.2
-    %37:_(s1) = G_PHI %47(s1), %bb.7, %27(s1), %bb.2
+    %35:_(s32) = G_PHI %46(s32), %bb.7, %12(s32), %bb.2
+    %37:_(s1) = G_PHI %47(s1), %bb.7, %33(s1), %bb.2
     G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %34(s32)
     G_BR %bb.3
 
@@ -732,7 +727,7 @@ body: |
     successors: %bb.7(0x80000000)
 
     %48:_(s32) = G_CONSTANT i32 2
-    %49:_(s64) = G_SHL %19, %48(s32)
+    %49:_(s64) = G_SHL %18, %48(s32)
     %50:_(p1) = G_PTR_ADD %2, %49(s64)
     %51:_(s32) = G_LOAD %50(p1) :: (load (s32), addrspace 1)
     %52:_(s32) = G_CONSTANT i32 1
@@ -745,14 +740,13 @@ body: |
   bb.7:
     successors: %bb.5(0x80000000)
 
-    %46:_(s32) = G_PHI %54(s32), %bb.6, %13(s32), %bb.4
-    %47:_(s1) = G_PHI %56(s1), %bb.6, %38(s1), %bb.4
+    %46:_(s32) = G_PHI %54(s32), %bb.6, %12(s32), %bb.4
+    %47:_(s1) = G_PHI %56(s1), %bb.6, %44(s1), %bb.4
     G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %45(s32)
     G_BR %bb.5
 
   bb.8:
-    %57:_(s32) = G_PHI %15(s32), %bb.3
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %57(s32)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %15(s32)
     S_ENDPGM 0
 ...
 
@@ -775,38 +769,38 @@ body: |
   ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
   ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
   ; GFX10-NEXT:   [[MV2:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
-  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
-  ; GFX10-NEXT:   [[DEF2:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
-  ; GFX10-NEXT:   [[DEF3:%[0-9]+]]:sreg_32_xm0_xexec(s1) = IMPLICIT_DEF
+  ; GFX10-NEXT:   [[DEF2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = IMPLICIT_DEF
+  ; GFX10-NEXT:   [[DEF3:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.1:
   ; GFX10-NEXT:   successors: %bb.3(0x40000000), %bb.5(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[DEF3]](s1), %bb.0, %67(s1), %bb.5
-  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF2]](s1), %bb.0, %56(s1), %bb.5
-  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %43(s1), %bb.5
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF3]](s1), %bb.0, %61(s1), %bb.5
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[DEF2]](s1), %bb.0, %48(s1), %bb.5
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %38(s1), %bb.5
   ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI %12(s32), %bb.5, [[C]](s32), %bb.0
   ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %14(s32), %bb.5
-  ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI]](s1)
-  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1)
+  ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
+  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI1]](s1)
   ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[PHI2]](s1)
-  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[C1]](s1)
-  ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[C1]](s1)
   ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI4]](s32)
-  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C2]](s32)
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C1]](s32)
   ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV1]], [[SHL]](s64)
   ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
-  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C3]]
-  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY10]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
-  ; GFX10-NEXT:   [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY7]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C2]]
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1)
+  ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[C3]](s1)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY7]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY10]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[S_OR_B32_]](s1)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
   ; GFX10-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY9]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_1]](s1)
@@ -823,24 +817,24 @@ body: |
   ; GFX10-NEXT: bb.3:
   ; GFX10-NEXT:   successors: %bb.5(0x80000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
-  ; GFX10-NEXT:   [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[C5]](s1)
-  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-  ; GFX10-NEXT:   [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C6]](s32)
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C5]](s32)
   ; GFX10-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV]], [[SHL1]](s64)
   ; GFX10-NEXT:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1)
-  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD1]], [[C7]]
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD1]], [[C6]]
   ; GFX10-NEXT:   G_STORE [[ADD]](s32), [[PTR_ADD1]](p1) :: (store (s32), addrspace 1)
-  ; GFX10-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI4]], [[C7]]
-  ; GFX10-NEXT:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 100
-  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[PHI4]](s32), [[C8]]
-  ; GFX10-NEXT:   [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP1]](s1)
-  ; GFX10-NEXT:   [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY11]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY13]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI4]], [[C6]]
+  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 100
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[PHI4]](s32), [[C7]]
+  ; GFX10-NEXT:   [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP1]](s1)
+  ; GFX10-NEXT:   [[C8:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; GFX10-NEXT:   [[COPY14:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[C8]](s1)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY11]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY14]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY12]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY14]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY13]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_3:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_3]](s1), [[S_AND_B32_3]](s1), implicit-def $scc
   ; GFX10-NEXT:   G_BR %bb.5
   ; GFX10-NEXT: {{  $}}
@@ -852,26 +846,24 @@ body: |
   ; GFX10-NEXT:   successors: %bb.6(0x04000000), %bb.1(0x7c000000)
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_1]](s1), %bb.1, [[S_OR_B32_3]](s1), %bb.3
-  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, [[S_OR_B32_2]](s1), %bb.3
+  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[S_OR_B32_]](s1), %bb.1, [[S_OR_B32_2]](s1), %bb.3
   ; GFX10-NEXT:   [[PHI7:%[0-9]+]]:_(s32) = G_PHI [[ADD1]](s32), %bb.3, [[DEF]](s32), %bb.1
   ; GFX10-NEXT:   [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1)
-  ; GFX10-NEXT:   [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[PHI6]](s1)
-  ; GFX10-NEXT:   [[COPY17:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[COPY16]](s1)
+  ; GFX10-NEXT:   [[COPY16:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI6]](s1)
+  ; GFX10-NEXT:   [[COPY17:%[0-9]+]]:sreg_32(s1) = COPY [[COPY16]](s1)
   ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
   ; GFX10-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[COPY15]](s1), [[PHI3]](s32)
-  ; GFX10-NEXT:   [[S_ANDN2_B32_4:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY6]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_4:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY17]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[S_OR_B32_4:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_4]](s1), [[S_AND_B32_4]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_ANDN2_B32_4:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_4:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY17]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_4:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_4]](s1), [[S_AND_B32_4]](s1), implicit-def $scc
   ; GFX10-NEXT:   SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.6
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.6:
   ; GFX10-NEXT:   successors: %bb.2(0x40000000), %bb.4(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI8:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.5
-  ; GFX10-NEXT:   [[COPY18:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[S_OR_B32_4]](s1)
-  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI8]](s32)
-  ; GFX10-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY18]](s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
+  ; GFX10-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[S_OR_B32_4]](s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.2
   bb.0:
     successors: %bb.1(0x80000000)
@@ -886,23 +878,23 @@ body: |
     %6:_(s32) = COPY $vgpr4
     %7:_(s32) = COPY $vgpr5
     %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
-    %9:_(s32) = G_CONSTANT i32 0
-    %10:_(s32) = G_IMPLICIT_DEF
+    %9:_(s32) = G_IMPLICIT_DEF
+    %10:_(s32) = G_CONSTANT i32 0
 
   bb.1:
     successors: %bb.3(0x40000000), %bb.5(0x40000000)
 
-    %11:_(s32) = G_PHI %12(s32), %bb.5, %9(s32), %bb.0
-    %13:_(s32) = G_PHI %9(s32), %bb.0, %14(s32), %bb.5
-    %15:_(s1) = G_CONSTANT i1 true
-    %16:_(s64) = G_SEXT %13(s32)
-    %17:_(s32) = G_CONSTANT i32 2
-    %18:_(s64) = G_SHL %16, %17(s32)
-    %19:_(p1) = G_PTR_ADD %5, %18(s64)
-    %20:_(s32) = G_LOAD %19(p1) :: (load (s32), addrspace 1)
-    %21:_(s32) = G_CONSTANT i32 0
-    %22:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %20(s32), %21
-    %23:sreg_32_xm0_xexec(s32) = SI_IF %22(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+    %11:_(s32) = G_PHI %12(s32), %bb.5, %10(s32), %bb.0
+    %13:_(s32) = G_PHI %10(s32), %bb.0, %14(s32), %bb.5
+    %15:_(s64) = G_SEXT %13(s32)
+    %16:_(s32) = G_CONSTANT i32 2
+    %17:_(s64) = G_SHL %15, %16(s32)
+    %18:_(p1) = G_PTR_ADD %5, %17(s64)
+    %19:_(s32) = G_LOAD %18(p1) :: (load (s32), addrspace 1)
+    %20:_(s32) = G_CONSTANT i32 0
+    %21:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %19(s32), %20
+    %22:_(s1) = G_CONSTANT i1 true
+    %23:sreg_32_xm0_xexec(s32) = SI_IF %21(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.3
 
   bb.2:
@@ -915,17 +907,17 @@ body: |
   bb.3:
     successors: %bb.5(0x80000000)
 
-    %25:_(s1) = G_CONSTANT i1 false
-    %26:_(s32) = G_CONSTANT i32 2
-    %27:_(s64) = G_SHL %16, %26(s32)
-    %28:_(p1) = G_PTR_ADD %2, %27(s64)
-    %29:_(s32) = G_LOAD %28(p1) :: (load (s32), addrspace 1)
-    %30:_(s32) = G_CONSTANT i32 1
-    %31:_(s32) = G_ADD %29, %30
-    G_STORE %31(s32), %28(p1) :: (store (s32), addrspace 1)
-    %32:_(s32) = G_ADD %13, %30
-    %33:_(s32) = G_CONSTANT i32 100
-    %34:_(s1) = G_ICMP intpred(ult), %13(s32), %33
+    %25:_(s32) = G_CONSTANT i32 2
+    %26:_(s64) = G_SHL %15, %25(s32)
+    %27:_(p1) = G_PTR_ADD %2, %26(s64)
+    %28:_(s32) = G_LOAD %27(p1) :: (load (s32), addrspace 1)
+    %29:_(s32) = G_CONSTANT i32 1
+    %30:_(s32) = G_ADD %28, %29
+    G_STORE %30(s32), %27(p1) :: (store (s32), addrspace 1)
+    %31:_(s32) = G_ADD %13, %29
+    %32:_(s32) = G_CONSTANT i32 100
+    %33:_(s1) = G_ICMP intpred(ult), %13(s32), %32
+    %34:_(s1) = G_CONSTANT i1 false
     G_BR %bb.5
 
   bb.4:
@@ -935,9 +927,9 @@ body: |
   bb.5:
     successors: %bb.6(0x04000000), %bb.1(0x7c000000)
 
-    %14:_(s32) = G_PHI %32(s32), %bb.3, %10(s32), %bb.1
-    %36:_(s1) = G_PHI %25(s1), %bb.3, %15(s1), %bb.1
-    %37:_(s1) = G_PHI %34(s1), %bb.3, %15(s1), %bb.1
+    %14:_(s32) = G_PHI %31(s32), %bb.3, %9(s32), %bb.1
+    %36:sreg_32_xm0_xexec(s1) = G_PHI %34(s1), %bb.3, %22(s1), %bb.1
+    %37:_(s1) = G_PHI %33(s1), %bb.3, %22(s1), %bb.1
     G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %23(s32)
     %12:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %37(s1), %11(s32)
     SI_LOOP %12(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
@@ -946,10 +938,8 @@ body: |
   bb.6:
     successors: %bb.2(0x40000000), %bb.4(0x40000000)
 
-    %38:sreg_32_xm0_xexec(s1) = G_PHI %36(s1), %bb.5
-    %39:_(s32) = G_PHI %12(s32), %bb.5
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %39(s32)
-    %35:sreg_32_xm0_xexec(s32) = SI_IF %38(s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %12(s32)
+    %35:sreg_32_xm0_xexec(s32) = SI_IF %36(s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.2
 ...
 
@@ -960,6 +950,224 @@ tracksRegLiveness: true
 body: |
   ; GFX10-LABEL: name: irreducible_cfg
   ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.8(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[COPY4]](s32), [[COPY1]]
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP]], [[C]]
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+  ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[DEF]](s1)
+  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1)
+  ; GFX10-NEXT:   G_BR %bb.8
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI %52(s1), %bb.5, %51(s1), %bb.8
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI %45(s1), %bb.5, %44(s1), %bb.8
+  ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI]](s1)
+  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1)
+  ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[COPY9]](s1)
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %17(s32)
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 %40(s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY10]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[C2]](s1)
+  ; GFX10-NEXT:   [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[COPY11]](s1)
+  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY8]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.4(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sle), [[COPY4]](s32), [[COPY]]
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   G_BR %bb.4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.6(0x04000000), %bb.8(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:sreg_32(s1) = PHI [[COPY11]](s1), %bb.1, %59(s1), %bb.7
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI %23(s1), %bb.7, [[DEF]](s1), %bb.1
+  ; GFX10-NEXT:   [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[PHI2]](s1)
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; GFX10-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[COPY13]](s1), %27(s32)
+  ; GFX10-NEXT:   [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[COPY9]](s1)
+  ; GFX10-NEXT:   [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1)
+  ; GFX10-NEXT:   SI_LOOP [[INT]](s32), %bb.8, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.6
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   successors: %bb.7(0x04000000), %bb.4(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[C3]](s32), %bb.2, %29(s32), %bb.4
+  ; GFX10-NEXT:   [[INT1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[ICMP1]](s1), [[PHI4]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INT1]](s32), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.7
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.5:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[COPY5]](s32), [[COPY]]
+  ; GFX10-NEXT:   [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1)
+  ; GFX10-NEXT:   [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE [[ICMP]]
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[FREEZE]], [[C4]]
+  ; GFX10-NEXT:   [[OR:%[0-9]+]]:_(s1) = G_OR [[ICMP2]], [[XOR1]]
+  ; GFX10-NEXT:   [[XOR2:%[0-9]+]]:_(s1) = G_XOR [[OR]], [[C4]]
+  ; GFX10-NEXT:   [[COPY17:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[XOR2]](s1)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 %46(s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY16]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 %53(s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY17]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
+  ; GFX10-NEXT:   G_BR %bb.1
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.6:
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[S_OR_B32_]](s1), [[COPY3]], [[COPY2]]
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[SELECT]](s32)
+  ; GFX10-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
+  ; GFX10-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.7:
+  ; GFX10-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT1]](s32)
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; GFX10-NEXT:   [[COPY18:%[0-9]+]]:sreg_32(s1) = COPY [[C5]](s1)
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY12]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY18]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_3:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_3]](s1), [[S_AND_B32_3]](s1), implicit-def $scc
+  ; GFX10-NEXT:   G_BR %bb.3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.8:
+  ; GFX10-NEXT:   successors: %bb.5(0x40000000), %bb.1(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[COPY7]](s1), %bb.0, [[COPY15]](s1), %bb.3
+  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:sreg_32(s1) = PHI [[COPY6]](s1), %bb.0, [[COPY14]](s1), %bb.3
+  ; GFX10-NEXT:   [[PHI7:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, [[S_OR_B32_]](s1), %bb.3
+  ; GFX10-NEXT:   [[PHI8:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.3, [[C1]](s32), %bb.0
+  ; GFX10-NEXT:   [[COPY19:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI5]](s1)
+  ; GFX10-NEXT:   [[COPY20:%[0-9]+]]:sreg_32(s1) = COPY [[PHI6]](s1)
+  ; GFX10-NEXT:   [[COPY21:%[0-9]+]]:sreg_32(s1) = COPY [[PHI7]](s1)
+  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[COPY22:%[0-9]+]]:sreg_32(s1) = COPY [[COPY20]](s1)
+  ; GFX10-NEXT:   [[COPY23:%[0-9]+]]:sreg_32(s1) = COPY [[COPY22]](s1)
+  ; GFX10-NEXT:   [[COPY24:%[0-9]+]]:sreg_32(s1) = COPY [[C7]](s1)
+  ; GFX10-NEXT:   [[COPY25:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[COPY24]](s1)
+  ; GFX10-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY19]](s1), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.5
+  bb.0:
+    successors: %bb.8(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(s32) = COPY $vgpr5
+    %6:_(s1) = G_IMPLICIT_DEF
+    %7:_(s1) = G_ICMP intpred(sgt), %4(s32), %1
+    %8:_(s1) = G_CONSTANT i1 true
+    %9:_(s1) = G_XOR %7, %8
+    %10:_(s32) = G_CONSTANT i32 0
+    G_BR %bb.8
+
+  bb.1:
+    successors: %bb.2(0x40000000), %bb.3(0x40000000)
+
+    %11:_(s1) = G_PHI %12(s1), %bb.5, %13(s1), %bb.8
+    %14:sreg_32_xm0_xexec(s1) = G_PHI %15(s1), %bb.5, %16(s1), %bb.8
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %17(s32)
+    %18:_(s1) = G_CONSTANT i1 true
+    %19:sreg_32_xm0_xexec(s32) = SI_IF %14(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    successors: %bb.4(0x80000000)
+
+    %20:_(s1) = G_ICMP intpred(sle), %4(s32), %0
+    %21:_(s32) = G_CONSTANT i32 0
+    G_BR %bb.4
+
+  bb.3:
+    successors: %bb.6(0x04000000), %bb.8(0x7c000000)
+
+    %22:_(s1) = G_PHI %23(s1), %bb.7, %6(s1), %bb.1
+    %24:_(s1) = G_PHI %25(s1), %bb.7, %18(s1), %bb.1
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %19(s32)
+    %26:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %24(s1), %27(s32)
+    SI_LOOP %26(s32), %bb.8, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.6
+
+  bb.4:
+    successors: %bb.7(0x04000000), %bb.4(0x7c000000)
+
+    %28:_(s32) = G_PHI %21(s32), %bb.2, %29(s32), %bb.4
+    %29:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %20(s1), %28(s32)
+    SI_LOOP %29(s32), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.7
+
+  bb.5:
+    successors: %bb.1(0x80000000)
+
+    %12:_(s1) = G_ICMP intpred(sgt), %5(s32), %0
+    %30:_(s1) = G_FREEZE %7
+    %31:_(s1) = G_CONSTANT i1 true
+    %32:_(s1) = G_XOR %30, %31
+    %33:_(s1) = G_OR %12, %32
+    %15:_(s1) = G_XOR %33, %31
+    G_BR %bb.1
+
+  bb.6:
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %26(s32)
+    %34:_(s32) = G_SELECT %11(s1), %3, %2
+    %35:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), %34(s32)
+    $sgpr0 = COPY %35(s32)
+    SI_RETURN_TO_EPILOG implicit $sgpr0
+
+  bb.7:
+    successors: %bb.3(0x80000000)
+
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %29(s32)
+    %25:_(s1) = G_CONSTANT i1 false
+    %23:_(s1) = G_CONSTANT i1 true
+    G_BR %bb.3
+
+  bb.8:
+    successors: %bb.5(0x40000000), %bb.1(0x40000000)
+
+    %27:_(s32) = G_PHI %26(s32), %bb.3, %10(s32), %bb.0
+    %13:_(s1) = G_PHI %6(s1), %bb.0, %11(s1), %bb.3
+    %36:sreg_32_xm0_xexec(s1) = G_PHI %9(s1), %bb.0, %22(s1), %bb.3
+    %16:_(s1) = G_CONSTANT i1 true
+    %17:sreg_32_xm0_xexec(s32) = SI_IF %36(s1), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.5
+...
+
+# Keep original test that had phi with three incomings.
+# FixIrreducible no longer generates such phi.
+---
+name: irreducible_cfg_phi_with_three_incomings
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: irreducible_cfg_phi_with_three_incomings
+  ; GFX10: bb.0:
   ; GFX10-NEXT:   successors: %bb.7(0x80000000)
   ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
   ; GFX10-NEXT: {{  $}}
@@ -969,9 +1177,9 @@ body: |
   ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
   ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
   ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
-  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF
   ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[COPY4]](s32), [[COPY1]]
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
   ; GFX10-NEXT:   [[DEF2:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
   ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP]](s1)
@@ -980,15 +1188,15 @@ body: |
   ; GFX10-NEXT: bb.1:
   ; GFX10-NEXT:   successors: %bb.3(0x80000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sle), [[COPY4]](s32), [[COPY]]
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   G_BR %bb.3
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.2:
   ; GFX10-NEXT:   successors: %bb.4(0x40000000), %bb.7(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI %53(s1), %bb.6, %57(s1), %bb.7
-  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI %35(s1), %bb.6, %34(s1), %bb.7
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI %52(s1), %bb.6, %56(s1), %bb.7
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI %41(s1), %bb.6, %40(s1), %bb.7
   ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s1) = G_PHI %12(s1), %bb.6, [[DEF]](s1), %bb.7
   ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI2]](s1)
   ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI]](s1)
@@ -1016,15 +1224,17 @@ body: |
   ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
   ; GFX10-NEXT:   [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[COPY5]](s32), [[COPY]]
   ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP2]](s1)
+  ; GFX10-NEXT:   [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE [[ICMP]]
   ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
   ; GFX10-NEXT:   [[COPY12:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[C2]](s1)
-  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP]], [[C2]]
+  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[FREEZE]], [[C2]]
   ; GFX10-NEXT:   [[OR:%[0-9]+]]:_(s1) = G_OR [[ICMP2]], [[XOR]]
-  ; GFX10-NEXT:   [[INT2:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[OR]](s1), %25(s32)
-  ; GFX10-NEXT:   [[DEF3:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
-  ; GFX10-NEXT:   [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 %49(s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[INT2:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[OR]](s1), %26(s32)
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 %34(s1), $exec_lo, implicit-def $scc
   ; GFX10-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY11]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[DEF3:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
   ; GFX10-NEXT:   [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY10]](s1), $exec_lo, implicit-def $scc
   ; GFX10-NEXT:   [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY12]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
@@ -1032,10 +1242,8 @@ body: |
   ; GFX10-NEXT:   G_BR %bb.5
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.5:
-  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[INT2]](s32), %bb.4
-  ; GFX10-NEXT:   [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_1]](s1)
-  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI4]](s32)
-  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY13]](s1), [[COPY3]], [[COPY2]]
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT2]](s32)
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[S_OR_B32_1]](s1), [[COPY3]], [[COPY2]]
   ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[SELECT]](s32)
   ; GFX10-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT]](s32)
   ; GFX10-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0
@@ -1043,12 +1251,11 @@ body: |
   ; GFX10-NEXT: bb.6:
   ; GFX10-NEXT:   successors: %bb.2(0x80000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:_(s32) = G_PHI [[INT1]](s32), %bb.3
-  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
-  ; GFX10-NEXT:   [[COPY14:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1)
-  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI5]](s32)
-  ; GFX10-NEXT:   [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 %42(s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY14]](s1), implicit-def $scc
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT1]](s32)
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; GFX10-NEXT:   [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[C4]](s1)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_3:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 %48(s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_3:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY13]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_3:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_3]](s1), [[S_AND_B32_3]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[DEF4:%[0-9]+]]:sreg_32_xm0_xexec(s1) = IMPLICIT_DEF
   ; GFX10-NEXT:   G_BR %bb.2
@@ -1056,21 +1263,21 @@ body: |
   ; GFX10-NEXT: bb.7:
   ; GFX10-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[COPY6]](s1), %bb.0, [[S_OR_B32_]](s1), %bb.2, [[S_OR_B32_2]](s1), %bb.4
-  ; GFX10-NEXT:   [[PHI7:%[0-9]+]]:sreg_32(s1) = PHI [[DEF2]](s1), %bb.0, [[PHI7]](s1), %bb.2, [[S_OR_B32_1]](s1), %bb.4
-  ; GFX10-NEXT:   [[PHI8:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, [[PHI1]](s1), %bb.2, [[DEF3]](s1), %bb.4
-  ; GFX10-NEXT:   [[PHI9:%[0-9]+]]:_(s32) = G_PHI [[INT2]](s32), %bb.4, [[PHI9]](s32), %bb.2, [[C]](s32), %bb.0
-  ; GFX10-NEXT:   [[PHI10:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.4, [[INT]](s32), %bb.2, [[C]](s32), %bb.0
-  ; GFX10-NEXT:   [[COPY15:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI6]](s1)
-  ; GFX10-NEXT:   [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[PHI7]](s1)
-  ; GFX10-NEXT:   [[COPY17:%[0-9]+]]:sreg_32(s1) = COPY [[PHI8]](s1)
-  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[COPY18:%[0-9]+]]:sreg_32(s1) = COPY [[C4]](s1)
-  ; GFX10-NEXT:   [[S_ANDN2_B32_4:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY17]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_4:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY18]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[COPY6]](s1), %bb.0, [[S_OR_B32_]](s1), %bb.2, [[S_OR_B32_2]](s1), %bb.4
+  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:sreg_32(s1) = PHI [[DEF2]](s1), %bb.0, [[PHI1]](s1), %bb.2, [[DEF3]](s1), %bb.4
+  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, [[PHI6]](s1), %bb.2, [[S_OR_B32_1]](s1), %bb.4
+  ; GFX10-NEXT:   [[PHI7:%[0-9]+]]:_(s32) = G_PHI [[INT2]](s32), %bb.4, [[PHI7]](s32), %bb.2, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI8:%[0-9]+]]:_(s32) = G_PHI [[C3]](s32), %bb.4, [[INT]](s32), %bb.2, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[COPY14:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI4]](s1)
+  ; GFX10-NEXT:   [[COPY15:%[0-9]+]]:sreg_32(s1) = COPY [[PHI5]](s1)
+  ; GFX10-NEXT:   [[COPY16:%[0-9]+]]:sreg_32(s1) = COPY [[PHI6]](s1)
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[COPY17:%[0-9]+]]:sreg_32(s1) = COPY [[C5]](s1)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_4:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY15]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_4:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY17]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_4:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_4]](s1), [[S_AND_B32_4]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[COPY19:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_4]](s1)
-  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY15]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   [[COPY18:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_4]](s1)
+  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY14]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.1
   bb.0:
     successors: %bb.7(0x80000000)
@@ -1082,22 +1289,22 @@ body: |
     %3:_(s32) = COPY $vgpr3
     %4:_(s32) = COPY $vgpr4
     %5:_(s32) = COPY $vgpr5
-    %6:_(s32) = G_CONSTANT i32 0
-    %7:_(s1) = G_IMPLICIT_DEF
-    %8:_(s1) = G_ICMP intpred(sgt), %4(s32), %1
+    %6:_(s1) = G_IMPLICIT_DEF
+    %7:_(s1) = G_ICMP intpred(sgt), %4(s32), %1
+    %8:_(s32) = G_CONSTANT i32 0
     G_BR %bb.7
 
   bb.1:
     successors: %bb.3(0x80000000)
 
-    %9:_(s32) = G_CONSTANT i32 0
-    %10:_(s1) = G_ICMP intpred(sle), %4(s32), %0
+    %9:_(s1) = G_ICMP intpred(sle), %4(s32), %0
+    %10:_(s32) = G_CONSTANT i32 0
     G_BR %bb.3
 
   bb.2:
     successors: %bb.4(0x40000000), %bb.7(0x40000000)
 
-    %11:_(s1) = G_PHI %12(s1), %bb.6, %7(s1), %bb.7
+    %11:_(s1) = G_PHI %12(s1), %bb.6, %6(s1), %bb.7
     %13:_(s1) = G_PHI %12(s1), %bb.6, %14(s1), %bb.7
     G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %15(s32)
     %16:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %13(s1), %17(s32)
@@ -1107,8 +1314,8 @@ body: |
   bb.3:
     successors: %bb.6(0x04000000), %bb.3(0x7c000000)
 
-    %18:_(s32) = G_PHI %9(s32), %bb.1, %19(s32), %bb.3
-    %19:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %10(s1), %18(s32)
+    %18:_(s32) = G_PHI %10(s32), %bb.1, %19(s32), %bb.3
+    %19:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %9(s1), %18(s32)
     SI_LOOP %19(s32), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.6
 
@@ -1117,18 +1324,18 @@ body: |
 
     G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %16(s32)
     %20:_(s1) = G_ICMP intpred(sgt), %5(s32), %0
-    %21:_(s1) = G_CONSTANT i1 true
-    %22:_(s1) = G_XOR %8, %21
-    %23:_(s1) = G_OR %20, %22
-    %24:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %23(s1), %25(s32)
-    SI_LOOP %24(s32), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec
+    %21:_(s1) = G_FREEZE %7
+    %22:_(s1) = G_CONSTANT i1 true
+    %23:_(s1) = G_XOR %21, %22
+    %24:_(s1) = G_OR %20, %23
+    %25:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %24(s1), %26(s32)
+    %27:_(s32) = G_CONSTANT i32 0
+    SI_LOOP %25(s32), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.5
 
   bb.5:
-    %26:_(s1) = G_PHI %20(s1), %bb.4
-    %27:_(s32) = G_PHI %24(s32), %bb.4
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %27(s32)
-    %28:_(s32) = G_SELECT %26(s1), %3, %2
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %25(s32)
+    %28:_(s32) = G_SELECT %20(s1), %3, %2
     %29:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), %28(s32)
     $sgpr0 = COPY %29(s32)
     SI_RETURN_TO_EPILOG implicit $sgpr0
@@ -1136,18 +1343,17 @@ body: |
   bb.6:
     successors: %bb.2(0x80000000)
 
-    %30:_(s32) = G_PHI %19(s32), %bb.3
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %19(s32)
     %12:_(s1) = G_CONSTANT i1 false
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %30(s32)
     G_BR %bb.2
 
   bb.7:
     successors: %bb.1(0x40000000), %bb.2(0x40000000)
 
-    %25:_(s32) = G_PHI %24(s32), %bb.4, %25(s32), %bb.2, %6(s32), %bb.0
-    %17:_(s32) = G_PHI %6(s32), %bb.4, %16(s32), %bb.2, %6(s32), %bb.0
-    %31:sreg_32_xm0_xexec(s1) = G_PHI %8(s1), %bb.0, %11(s1), %bb.2, %21(s1), %bb.4
+    %26:_(s32) = G_PHI %25(s32), %bb.4, %26(s32), %bb.2, %8(s32), %bb.0
+    %17:_(s32) = G_PHI %27(s32), %bb.4, %16(s32), %bb.2, %8(s32), %bb.0
+    %30:sreg_32_xm0_xexec(s1) = G_PHI %7(s1), %bb.0, %11(s1), %bb.2, %22(s1), %bb.4
     %14:_(s1) = G_CONSTANT i1 true
-    %15:sreg_32_xm0_xexec(s32) = SI_IF %31(s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+    %15:sreg_32_xm0_xexec(s32) = SI_IF %30(s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.1
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll
index 1855ede0483de..0da25260e2317 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll
@@ -1,31 +1,32 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -new-reg-bank-select -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
 
 define void @temporal_divergent_i1_phi(float %val, ptr %addr) {
 ; GFX10-LABEL: temporal_divergent_i1_phi:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s5, 0
-; GFX10-NEXT:    v_mov_b32_e32 v3, 1
-; GFX10-NEXT:    v_mov_b32_e32 v4, s5
-; GFX10-NEXT:    ; implicit-def: $sgpr6
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    s_mov_b32 s5, 1
+; GFX10-NEXT:    s_mov_b32 s6, 0
+; GFX10-NEXT:    ; implicit-def: $sgpr7
 ; GFX10-NEXT:  .LBB0_1: ; %loop
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    v_cvt_f32_u32_e32 v5, v4
-; GFX10-NEXT:    v_and_b32_e32 v6, 1, v3
-; GFX10-NEXT:    v_xor_b32_e32 v3, 1, v3
-; GFX10-NEXT:    v_add_nc_u32_e32 v4, 1, v4
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v0
-; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, v6
-; GFX10-NEXT:    s_or_b32 s5, vcc_lo, s5
-; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
-; GFX10-NEXT:    s_and_b32 s4, exec_lo, s4
-; GFX10-NEXT:    s_or_b32 s6, s6, s4
-; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v3, s6
+; GFX10-NEXT:    s_and_b32 s8, s5, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX10-NEXT:    s_cselect_b32 s8, exec_lo, 0
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v0
+; GFX10-NEXT:    s_xor_b32 s5, s5, 1
+; GFX10-NEXT:    s_add_i32 s6, s6, 1
+; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT:    s_andn2_b32 s7, s7, exec_lo
+; GFX10-NEXT:    s_and_b32 s8, exec_lo, s8
+; GFX10-NEXT:    s_or_b32 s7, s7, s8
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
 ; GFX10-NEXT:    s_cbranch_execnz .LBB0_1
 ; GFX10-NEXT:  ; %bb.2: ; %exit
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s6
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s7
 ; GFX10-NEXT:    flat_store_dword v[1:2], v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -51,27 +52,28 @@ define void @temporal_divergent_i1_non_phi(float %val, ptr %addr) {
 ; GFX10-LABEL: temporal_divergent_i1_non_phi:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s5, 0
-; GFX10-NEXT:    v_mov_b32_e32 v3, 1
-; GFX10-NEXT:    v_mov_b32_e32 v4, s5
-; GFX10-NEXT:    ; implicit-def: $sgpr6
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    s_mov_b32 s5, 1
+; GFX10-NEXT:    s_mov_b32 s6, 0
+; GFX10-NEXT:    ; implicit-def: $sgpr7
 ; GFX10-NEXT:  .LBB1_1: ; %loop
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    v_xor_b32_e32 v3, 1, v3
-; GFX10-NEXT:    v_cvt_f32_u32_e32 v5, v4
-; GFX10-NEXT:    v_add_nc_u32_e32 v4, 1, v4
-; GFX10-NEXT:    v_and_b32_e32 v6, 1, v3
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v0
-; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, v6
-; GFX10-NEXT:    s_or_b32 s5, vcc_lo, s5
-; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
-; GFX10-NEXT:    s_and_b32 s4, exec_lo, s4
-; GFX10-NEXT:    s_or_b32 s6, s6, s4
-; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v3, s6
+; GFX10-NEXT:    s_xor_b32 s5, s5, 1
+; GFX10-NEXT:    s_and_b32 s8, s5, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v0
+; GFX10-NEXT:    s_cselect_b32 s8, exec_lo, 0
+; GFX10-NEXT:    s_add_i32 s6, s6, 1
+; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT:    s_andn2_b32 s7, s7, exec_lo
+; GFX10-NEXT:    s_and_b32 s8, exec_lo, s8
+; GFX10-NEXT:    s_or_b32 s7, s7, s8
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
 ; GFX10-NEXT:    s_cbranch_execnz .LBB1_1
 ; GFX10-NEXT:  ; %bb.2: ; %exit
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s6
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s7
 ; GFX10-NEXT:    flat_store_dword v[1:2], v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -98,60 +100,64 @@ exit:
 define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, i32 %x.size, ptr addrspace(1) inreg %a, ptr addrspace(1) inreg %a.break) {
 ; GFX10-LABEL: loop_with_1break:
 ; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    v_mov_b32_e32 v3, 0
+; GFX10-NEXT:    s_mov_b32 s8, 0
 ; GFX10-NEXT:    s_mov_b32 s4, 0
-; GFX10-NEXT:    v_mov_b32_e32 v4, s1
-; GFX10-NEXT:    v_mov_b32_e32 v3, s0
-; GFX10-NEXT:    v_mov_b32_e32 v5, s4
-; GFX10-NEXT:    ; implicit-def: $sgpr0
-; GFX10-NEXT:    ; implicit-def: $sgpr1
+; GFX10-NEXT:    ; implicit-def: $sgpr10
+; GFX10-NEXT:    ; implicit-def: $sgpr9
 ; GFX10-NEXT:    s_branch .LBB2_3
 ; GFX10-NEXT:  .LBB2_1: ; %loop.body
 ; GFX10-NEXT:    ; in Loop: Header=BB2_3 Depth=1
-; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v0, v6
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v1, v7, vcc_lo
-; GFX10-NEXT:    v_add_nc_u32_e32 v9, 1, v5
-; GFX10-NEXT:    v_cmp_lt_u32_e32 vcc_lo, v5, v2
-; GFX10-NEXT:    s_andn2_b32 s1, s1, exec_lo
-; GFX10-NEXT:    global_load_dword v8, v[6:7], off
+; GFX10-NEXT:    v_mov_b32_e32 v4, s6
+; GFX10-NEXT:    v_mov_b32_e32 v5, s7
+; GFX10-NEXT:    s_andn2_b32 s6, s10, exec_lo
 ; GFX10-NEXT:    s_mov_b32 s5, 0
-; GFX10-NEXT:    v_mov_b32_e32 v5, v9
-; GFX10-NEXT:    s_and_b32 s6, exec_lo, vcc_lo
-; GFX10-NEXT:    s_or_b32 s1, s1, s6
+; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, v1, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u32_e32 vcc_lo, s4, v2
+; GFX10-NEXT:    s_add_i32 s4, s4, 1
+; GFX10-NEXT:    global_load_dword v6, v[4:5], off
+; GFX10-NEXT:    s_and_b32 s7, exec_lo, vcc_lo
+; GFX10-NEXT:    s_or_b32 s10, s6, s7
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_add_nc_u32_e32 v8, 1, v8
-; GFX10-NEXT:    global_store_dword v[6:7], v8, off
+; GFX10-NEXT:    v_add_nc_u32_e32 v6, 1, v6
+; GFX10-NEXT:    global_store_dword v[4:5], v6, off
 ; GFX10-NEXT:  .LBB2_2: ; %Flow
 ; GFX10-NEXT:    ; in Loop: Header=BB2_3 Depth=1
-; GFX10-NEXT:    s_and_b32 s5, 1, s5
-; GFX10-NEXT:    s_and_b32 s6, exec_lo, s1
-; GFX10-NEXT:    v_cmp_ne_u32_e64 s5, 0, s5
-; GFX10-NEXT:    s_or_b32 s4, s6, s4
-; GFX10-NEXT:    s_andn2_b32 s0, s0, exec_lo
+; GFX10-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX10-NEXT:    s_cselect_b32 s5, exec_lo, 0
+; GFX10-NEXT:    s_and_b32 s6, exec_lo, s10
+; GFX10-NEXT:    s_or_b32 s8, s6, s8
+; GFX10-NEXT:    s_andn2_b32 s6, s9, exec_lo
 ; GFX10-NEXT:    s_and_b32 s5, exec_lo, s5
-; GFX10-NEXT:    s_or_b32 s0, s0, s5
-; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_or_b32 s9, s6, s5
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s8
 ; GFX10-NEXT:    s_cbranch_execz .LBB2_5
 ; GFX10-NEXT:  .LBB2_3: ; %A
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
-; GFX10-NEXT:    s_andn2_b32 s1, s1, exec_lo
-; GFX10-NEXT:    s_and_b32 s5, exec_lo, -1
-; GFX10-NEXT:    s_or_b32 s1, s1, s5
-; GFX10-NEXT:    v_lshlrev_b64 v[6:7], 2, v[5:6]
-; GFX10-NEXT:    v_add_co_u32 v8, vcc_lo, v3, v6
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v4, v7, vcc_lo
-; GFX10-NEXT:    global_load_dword v8, v[8:9], off
+; GFX10-NEXT:    s_ashr_i32 s5, s4, 31
+; GFX10-NEXT:    s_lshl_b64 s[6:7], s[4:5], 2
+; GFX10-NEXT:    s_add_u32 s12, s0, s6
+; GFX10-NEXT:    s_addc_u32 s13, s1, s7
+; GFX10-NEXT:    global_load_dword v4, v3, s[12:13]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v8
-; GFX10-NEXT:    s_cbranch_vccnz .LBB2_1
+; GFX10-NEXT:    v_readfirstlane_b32 s5, v4
+; GFX10-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX10-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX10-NEXT:    s_andn2_b32 s10, s10, exec_lo
+; GFX10-NEXT:    s_and_b32 s11, exec_lo, exec_lo
+; GFX10-NEXT:    s_or_b32 s10, s10, s11
+; GFX10-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX10-NEXT:    s_cbranch_scc1 .LBB2_1
 ; GFX10-NEXT:  ; %bb.4: ; in Loop: Header=BB2_3 Depth=1
 ; GFX10-NEXT:    s_mov_b32 s5, 1
-; GFX10-NEXT:    ; implicit-def: $vgpr5
+; GFX10-NEXT:    ; implicit-def: $sgpr4
 ; GFX10-NEXT:    s_branch .LBB2_2
 ; GFX10-NEXT:  .LBB2_5: ; %loop.exit.guard
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT:    s_and_saveexec_b32 s1, s0
-; GFX10-NEXT:    s_xor_b32 s1, exec_lo, s1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; GFX10-NEXT:    s_and_saveexec_b32 s0, s9
+; GFX10-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX10-NEXT:    s_cbranch_execz .LBB2_7
 ; GFX10-NEXT:  ; %bb.6: ; %break.body
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 10
@@ -186,3 +192,279 @@ exit:
   ret void
 }
 
+; Temporal divergence i1 across inner and outer nested loops
+define void @nested_loops_temporal_divergence_inner(float %pre.cond.val, i32 %n.i, ptr %mat, ptr %mat.oe, ptr %arr) {
+; GFX10-LABEL: nested_loops_temporal_divergence_inner:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s8, 1.0, v0
+; GFX10-NEXT:    s_mov_b32 s5, 0
+; GFX10-NEXT:    s_mov_b32 s6, 0
+; GFX10-NEXT:  .LBB3_1: ; %OuterHeader
+; GFX10-NEXT:    ; =>This Loop Header: Depth=1
+; GFX10-NEXT:    ; Child Loop BB3_2 Depth 2
+; GFX10-NEXT:    s_ashr_i32 s7, s6, 31
+; GFX10-NEXT:    s_mov_b32 s4, s8
+; GFX10-NEXT:    s_lshl_b64 s[10:11], s[6:7], 2
+; GFX10-NEXT:    ; implicit-def: $sgpr9
+; GFX10-NEXT:    v_mov_b32_e32 v6, s10
+; GFX10-NEXT:    v_mov_b32_e32 v7, s11
+; GFX10-NEXT:    s_mov_b32 s10, 0
+; GFX10-NEXT:    s_mov_b32 s11, 0
+; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v2, v6
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v3, v7, vcc_lo
+; GFX10-NEXT:    flat_load_dword v0, v[6:7]
+; GFX10-NEXT:  .LBB3_2: ; %InnerHeader
+; GFX10-NEXT:    ; Parent Loop BB3_1 Depth=1
+; GFX10-NEXT:    ; => This Inner Loop Header: Depth=2
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v6, s11
+; GFX10-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10-NEXT:    s_add_i32 s11, s11, 1
+; GFX10-NEXT:    s_xor_b32 s4, s4, s12
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v0
+; GFX10-NEXT:    s_or_b32 s10, vcc_lo, s10
+; GFX10-NEXT:    s_andn2_b32 s9, s9, exec_lo
+; GFX10-NEXT:    s_and_b32 s12, exec_lo, s4
+; GFX10-NEXT:    s_or_b32 s9, s9, s12
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s10
+; GFX10-NEXT:    s_cbranch_execnz .LBB3_2
+; GFX10-NEXT:  ; %bb.3: ; %UseInst
+; GFX10-NEXT:    ; in Loop: Header=BB3_1 Depth=1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s10
+; GFX10-NEXT:    v_mov_b32_e32 v6, s6
+; GFX10-NEXT:    v_mov_b32_e32 v7, s7
+; GFX10-NEXT:    v_cmp_lt_u32_e32 vcc_lo, s6, v1
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s9
+; GFX10-NEXT:    s_add_i32 s6, s6, 1
+; GFX10-NEXT:    v_add_co_u32 v6, s4, v4, v6
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v7, s4, v5, v7, s4
+; GFX10-NEXT:    s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT:    flat_store_byte v[6:7], v0
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    s_cbranch_execnz .LBB3_1
+; GFX10-NEXT:  ; %bb.4: ; %exit
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %pre.cond = fcmp ogt float %pre.cond.val, 1.0
+  br label %OuterHeader
+
+OuterHeader:
+  %i = phi i32 [ 0, %entry ], [ %i.plus.1, %OuterLatch ]
+  %mat.i = getelementptr float, ptr %mat, i32 %i
+  %val.i = load float, ptr %mat.i
+  br label %InnerHeader
+
+InnerHeader:
+  %j = phi i32 [ 0, %OuterHeader ], [ %j.plus.1, %InnerHeader ]
+  %bool.counter = phi i1 [ %pre.cond, %OuterHeader ], [ %odd.even.counter, %InnerHeader ]
+
+  %odd.even.counter = xor i1 %bool.counter, true
+
+  %f.j = uitofp i32 %j to float
+  %j.plus.1 = add i32 %j, 1
+  %cond.j = fcmp ogt float %f.j, %val.i
+  br i1 %cond.j, label %UseInst, label %InnerHeader
+
+UseInst:
+  %mat.oe.i = getelementptr i1, ptr %mat.oe, i32 %i
+  ; mat.oe[i] = oddOReven(mat[i])
+  store i1 %odd.even.counter, ptr %mat.oe.i
+  br label %OuterLatch
+
+OuterLatch:
+  %cond.i = icmp ult i32 %i, %n.i
+  %i.plus.1 = add i32 %i, 1
+  br i1 %cond.i, label %exit, label %OuterHeader
+
+exit:
+  ret void
+}
+
+define void @nested_loops_temporal_divergence_outer(float %pre.cond.val, i32 %n.i, ptr %mat, ptr %mat.oe, ptr %arr) {
+; GFX10-LABEL: nested_loops_temporal_divergence_outer:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s8, 1.0, v0
+; GFX10-NEXT:    s_mov_b32 s5, 0
+; GFX10-NEXT:    s_mov_b32 s6, 0
+; GFX10-NEXT:  .LBB4_1: ; %OuterHeader
+; GFX10-NEXT:    ; =>This Loop Header: Depth=1
+; GFX10-NEXT:    ; Child Loop BB4_2 Depth 2
+; GFX10-NEXT:    s_ashr_i32 s7, s6, 31
+; GFX10-NEXT:    s_mov_b32 s4, s8
+; GFX10-NEXT:    s_lshl_b64 s[10:11], s[6:7], 2
+; GFX10-NEXT:    ; implicit-def: $sgpr9
+; GFX10-NEXT:    v_mov_b32_e32 v6, s10
+; GFX10-NEXT:    v_mov_b32_e32 v7, s11
+; GFX10-NEXT:    s_mov_b32 s10, 0
+; GFX10-NEXT:    s_mov_b32 s11, 0
+; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v2, v6
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v3, v7, vcc_lo
+; GFX10-NEXT:    flat_load_dword v0, v[6:7]
+; GFX10-NEXT:  .LBB4_2: ; %InnerHeader
+; GFX10-NEXT:    ; Parent Loop BB4_1 Depth=1
+; GFX10-NEXT:    ; => This Inner Loop Header: Depth=2
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v6, s11
+; GFX10-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10-NEXT:    s_add_i32 s11, s11, 1
+; GFX10-NEXT:    s_xor_b32 s4, s4, s12
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v0
+; GFX10-NEXT:    s_or_b32 s10, vcc_lo, s10
+; GFX10-NEXT:    s_andn2_b32 s9, s9, exec_lo
+; GFX10-NEXT:    s_and_b32 s12, exec_lo, s4
+; GFX10-NEXT:    s_or_b32 s9, s9, s12
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s10
+; GFX10-NEXT:    s_cbranch_execnz .LBB4_2
+; GFX10-NEXT:  ; %bb.3: ; %UseInst
+; GFX10-NEXT:    ; in Loop: Header=BB4_1 Depth=1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s10
+; GFX10-NEXT:    v_mov_b32_e32 v6, s6
+; GFX10-NEXT:    v_mov_b32_e32 v7, s7
+; GFX10-NEXT:    v_cmp_lt_u32_e32 vcc_lo, s6, v1
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s9
+; GFX10-NEXT:    s_add_i32 s6, s6, 1
+; GFX10-NEXT:    v_add_co_u32 v6, s4, v4, v6
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v7, s4, v5, v7, s4
+; GFX10-NEXT:    s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT:    flat_store_byte v[6:7], v0
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    s_cbranch_execnz .LBB4_1
+; GFX10-NEXT:  ; %bb.4: ; %exit
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %pre.cond = fcmp ogt float %pre.cond.val, 1.0
+  br label %OuterHeader
+
+OuterHeader:
+  %i = phi i32 [ 0, %entry ], [ %i.plus.1, %OuterLatch ]
+  %mat.i = getelementptr float, ptr %mat, i32 %i
+  %val.i = load float, ptr %mat.i
+  br label %InnerHeader
+
+InnerHeader:
+  %j = phi i32 [ 0, %OuterHeader ], [ %j.plus.1, %InnerHeader ]
+  %bool.counter = phi i1 [ %pre.cond, %OuterHeader ], [ %odd.even.counter, %InnerHeader ]
+
+  %odd.even.counter = xor i1 %bool.counter, true
+
+  %f.j = uitofp i32 %j to float
+  %j.plus.1 = add i32 %j, 1
+  %cond.j = fcmp ogt float %f.j, %val.i
+  br i1 %cond.j, label %UseInst, label %InnerHeader
+
+UseInst:
+  %mat.oe.i = getelementptr i1, ptr %mat.oe, i32 %i
+  ; mat.oe[i] = oddOReven(mat[i])
+  store i1 %odd.even.counter, ptr %mat.oe.i
+  br label %OuterLatch
+
+OuterLatch:
+  %cond.i = icmp ult i32 %i, %n.i
+  %i.plus.1 = add i32 %i, 1
+  br i1 %cond.i, label %exit, label %OuterHeader
+
+exit:
+  ret void
+}
+
+define void @nested_loops_temporal_divergence_both(float %pre.cond.val, i32 %n.i, ptr %mat, ptr %mat.oe, ptr %arr) {
+; GFX10-LABEL: nested_loops_temporal_divergence_both:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cmp_lt_f32_e64 s8, 1.0, v0
+; GFX10-NEXT:    s_mov_b32 s5, 0
+; GFX10-NEXT:    s_mov_b32 s6, 0
+; GFX10-NEXT:    ; implicit-def: $sgpr9
+; GFX10-NEXT:  .LBB5_1: ; %OuterHeader
+; GFX10-NEXT:    ; =>This Loop Header: Depth=1
+; GFX10-NEXT:    ; Child Loop BB5_2 Depth 2
+; GFX10-NEXT:    s_ashr_i32 s7, s6, 31
+; GFX10-NEXT:    s_mov_b32 s4, s8
+; GFX10-NEXT:    s_lshl_b64 s[10:11], s[6:7], 2
+; GFX10-NEXT:    v_mov_b32_e32 v8, s10
+; GFX10-NEXT:    v_mov_b32_e32 v9, s11
+; GFX10-NEXT:    s_mov_b32 s10, 0
+; GFX10-NEXT:    s_mov_b32 s11, 0
+; GFX10-NEXT:    v_add_co_u32 v8, vcc_lo, v2, v8
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v3, v9, vcc_lo
+; GFX10-NEXT:    flat_load_dword v0, v[8:9]
+; GFX10-NEXT:  .LBB5_2: ; %InnerHeader
+; GFX10-NEXT:    ; Parent Loop BB5_1 Depth=1
+; GFX10-NEXT:    ; => This Inner Loop Header: Depth=2
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v8, s11
+; GFX10-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10-NEXT:    s_add_i32 s11, s11, 1
+; GFX10-NEXT:    s_xor_b32 s4, s4, s12
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v8, v0
+; GFX10-NEXT:    s_or_b32 s10, vcc_lo, s10
+; GFX10-NEXT:    s_andn2_b32 s9, s9, exec_lo
+; GFX10-NEXT:    s_and_b32 s12, exec_lo, s4
+; GFX10-NEXT:    s_or_b32 s9, s9, s12
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s10
+; GFX10-NEXT:    s_cbranch_execnz .LBB5_2
+; GFX10-NEXT:  ; %bb.3: ; %UseInst
+; GFX10-NEXT:    ; in Loop: Header=BB5_1 Depth=1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s10
+; GFX10-NEXT:    v_mov_b32_e32 v9, s7
+; GFX10-NEXT:    v_mov_b32_e32 v8, s6
+; GFX10-NEXT:    v_cmp_lt_u32_e32 vcc_lo, s6, v1
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s9
+; GFX10-NEXT:    s_add_i32 s6, s6, 1
+; GFX10-NEXT:    v_add_co_u32 v8, s4, v4, v8
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v9, s4, v5, v9, s4
+; GFX10-NEXT:    s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT:    flat_store_byte v[8:9], v0
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    s_cbranch_execnz .LBB5_1
+; GFX10-NEXT:  ; %bb.4: ; %exit
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    flat_store_byte v[6:7], v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %pre.cond = fcmp ogt float %pre.cond.val, 1.0
+  br label %OuterHeader
+
+OuterHeader:
+  %i = phi i32 [ 0, %entry ], [ %i.plus.1, %OuterLatch ]
+  %mat.i = getelementptr float, ptr %mat, i32 %i
+  %val.i = load float, ptr %mat.i
+  br label %InnerHeader
+
+InnerHeader:
+  %j = phi i32 [ 0, %OuterHeader ], [ %j.plus.1, %InnerHeader ]
+  %bool.counter = phi i1 [ %pre.cond, %OuterHeader ], [ %odd.even.counter, %InnerHeader ]
+
+  %odd.even.counter = xor i1 %bool.counter, true
+
+  %f.j = uitofp i32 %j to float
+  %j.plus.1 = add i32 %j, 1
+  %cond.j = fcmp ogt float %f.j, %val.i
+  br i1 %cond.j, label %UseInst, label %InnerHeader
+
+UseInst:
+  %mat.oe.i = getelementptr i1, ptr %mat.oe, i32 %i
+  ; mat.oe[i] = oddOReven(mat[i])
+  store i1 %odd.even.counter, ptr %mat.oe.i
+  br label %OuterLatch
+
+OuterLatch:
+  %cond.i = icmp ult i32 %i, %n.i
+  %i.plus.1 = add i32 %i, 1
+  br i1 %cond.i, label %exit, label %OuterHeader
+
+exit:
+  ; arr = oddOReven(mat[n.i - 1])
+  store i1 %odd.even.counter, ptr %arr
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.mir
index fb436623bed2d..30c1c9f51c628 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.mir
@@ -15,17 +15,17 @@ body: |
   ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
-  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.1:
   ; GFX10-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %22(s1), %bb.1
-  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI %7(s32), %bb.1, [[C1]](s32), %bb.0
-  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.0, %9(s32), %bb.1
-  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[C]](s1), %bb.0, %11(s1), %bb.1
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %19(s1), %bb.1
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI %7(s32), %bb.1, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %9(s32), %bb.1
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[C1]](s1), %bb.0, %11(s1), %bb.1
   ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1)
   ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
   ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
@@ -42,12 +42,10 @@ body: |
   ; GFX10-NEXT:   G_BR %bb.2
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.2:
-  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.1
-  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
-  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI4]](s32)
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
   ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
   ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
-  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY5]](s1), [[C5]], [[C4]]
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[S_OR_B32_]](s1), [[C5]], [[C4]]
   ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p0) :: (store (s32))
   ; GFX10-NEXT:   SI_RETURN
   bb.0:
@@ -58,15 +56,15 @@ body: |
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = COPY $vgpr2
     %3:_(p0) = G_MERGE_VALUES %1(s32), %2(s32)
-    %4:_(s1) = G_CONSTANT i1 true
-    %5:_(s32) = G_CONSTANT i32 0
+    %4:_(s32) = G_CONSTANT i32 0
+    %5:_(s1) = G_CONSTANT i1 true
 
   bb.1:
     successors: %bb.2(0x04000000), %bb.1(0x7c000000)
 
-    %6:_(s32) = G_PHI %7(s32), %bb.1, %5(s32), %bb.0
-    %8:_(s32) = G_PHI %5(s32), %bb.0, %9(s32), %bb.1
-    %10:_(s1) = G_PHI %4(s1), %bb.0, %11(s1), %bb.1
+    %6:_(s32) = G_PHI %7(s32), %bb.1, %4(s32), %bb.0
+    %8:_(s32) = G_PHI %4(s32), %bb.0, %9(s32), %bb.1
+    %10:_(s1) = G_PHI %5(s1), %bb.0, %11(s1), %bb.1
     %12:_(s1) = G_CONSTANT i1 true
     %11:_(s1) = G_XOR %10, %12
     %13:_(s32) = G_UITOFP %8(s32)
@@ -78,13 +76,11 @@ body: |
     G_BR %bb.2
 
   bb.2:
-    %16:_(s1) = G_PHI %10(s1), %bb.1
-    %17:_(s32) = G_PHI %7(s32), %bb.1
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %17(s32)
-    %18:_(s32) = G_FCONSTANT float 0.000000e+00
-    %19:_(s32) = G_FCONSTANT float 1.000000e+00
-    %20:_(s32) = G_SELECT %16(s1), %19, %18
-    G_STORE %20(s32), %3(p0) :: (store (s32))
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %7(s32)
+    %16:_(s32) = G_FCONSTANT float 0.000000e+00
+    %17:_(s32) = G_FCONSTANT float 1.000000e+00
+    %18:_(s32) = G_SELECT %10(s1), %17, %16
+    G_STORE %18(s32), %3(p0) :: (store (s32))
     SI_RETURN
 ...
 
@@ -102,17 +98,17 @@ body: |
   ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
-  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.1:
   ; GFX10-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %22(s1), %bb.1
-  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI %7(s32), %bb.1, [[C1]](s32), %bb.0
-  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.0, %9(s32), %bb.1
-  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[C]](s1), %bb.0, %11(s1), %bb.1
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %19(s1), %bb.1
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI %7(s32), %bb.1, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %9(s32), %bb.1
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[C1]](s1), %bb.0, %11(s1), %bb.1
   ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
   ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
   ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[PHI3]], [[C2]]
@@ -129,12 +125,10 @@ body: |
   ; GFX10-NEXT:   G_BR %bb.2
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.2:
-  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.1
-  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
-  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI4]](s32)
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
   ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
   ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
-  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[COPY5]](s1), [[C5]], [[C4]]
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[S_OR_B32_]](s1), [[C5]], [[C4]]
   ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p0) :: (store (s32))
   ; GFX10-NEXT:   SI_RETURN
   bb.0:
@@ -145,15 +139,15 @@ body: |
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = COPY $vgpr2
     %3:_(p0) = G_MERGE_VALUES %1(s32), %2(s32)
-    %4:_(s1) = G_CONSTANT i1 true
-    %5:_(s32) = G_CONSTANT i32 0
+    %4:_(s32) = G_CONSTANT i32 0
+    %5:_(s1) = G_CONSTANT i1 true
 
   bb.1:
     successors: %bb.2(0x04000000), %bb.1(0x7c000000)
 
-    %6:_(s32) = G_PHI %7(s32), %bb.1, %5(s32), %bb.0
-    %8:_(s32) = G_PHI %5(s32), %bb.0, %9(s32), %bb.1
-    %10:_(s1) = G_PHI %4(s1), %bb.0, %11(s1), %bb.1
+    %6:_(s32) = G_PHI %7(s32), %bb.1, %4(s32), %bb.0
+    %8:_(s32) = G_PHI %4(s32), %bb.0, %9(s32), %bb.1
+    %10:_(s1) = G_PHI %5(s1), %bb.0, %11(s1), %bb.1
     %12:_(s1) = G_CONSTANT i1 true
     %11:_(s1) = G_XOR %10, %12
     %13:_(s32) = G_UITOFP %8(s32)
@@ -165,13 +159,11 @@ body: |
     G_BR %bb.2
 
   bb.2:
-    %16:_(s1) = G_PHI %11(s1), %bb.1
-    %17:_(s32) = G_PHI %7(s32), %bb.1
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %17(s32)
-    %18:_(s32) = G_FCONSTANT float 0.000000e+00
-    %19:_(s32) = G_FCONSTANT float 1.000000e+00
-    %20:_(s32) = G_SELECT %16(s1), %19, %18
-    G_STORE %20(s32), %3(p0) :: (store (s32))
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %7(s32)
+    %16:_(s32) = G_FCONSTANT float 0.000000e+00
+    %17:_(s32) = G_FCONSTANT float 1.000000e+00
+    %18:_(s32) = G_SELECT %11(s1), %17, %16
+    G_STORE %18(s32), %3(p0) :: (store (s32))
     SI_RETURN
 ...
 
@@ -195,30 +187,30 @@ body: |
   ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr2
   ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr3
   ; GFX10-NEXT:   [[MV2:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY5]](s32), [[COPY6]](s32)
-  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT:   [[DEF1:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
-  ; GFX10-NEXT:   [[DEF2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = IMPLICIT_DEF
+  ; GFX10-NEXT:   [[DEF2:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.1:
   ; GFX10-NEXT:   successors: %bb.3(0x50000000), %bb.5(0x30000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32_xm0_xexec(s1) = PHI [[DEF2]](s1), %bb.0, %53(s1), %bb.5
-  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %42(s1), %bb.5
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF2]](s1), %bb.0, %47(s1), %bb.5
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %37(s1), %bb.5
   ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI %13(s32), %bb.5, [[C]](s32), %bb.0
   ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %15(s32), %bb.5
-  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI]](s1)
+  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
   ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[PHI1]](s1)
-  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
-  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[C1]](s1)
   ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI3]](s32)
-  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C2]](s32)
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C1]](s32)
   ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV1]], [[SHL]](s64)
   ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
-  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C3]]
-  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C2]]
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[C3]](s1)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY7]](s1), $exec_lo, implicit-def $scc
   ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY9]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[S_OR_B32_]](s1)
@@ -235,17 +227,17 @@ body: |
   ; GFX10-NEXT: bb.3:
   ; GFX10-NEXT:   successors: %bb.5(0x80000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
-  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-  ; GFX10-NEXT:   [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C6]](s32)
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C5]](s32)
   ; GFX10-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV]], [[SHL1]](s64)
   ; GFX10-NEXT:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1)
-  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD1]], [[C7]]
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD1]], [[C6]]
   ; GFX10-NEXT:   G_STORE [[ADD]](s32), [[PTR_ADD1]](p1) :: (store (s32), addrspace 1)
-  ; GFX10-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI3]], [[C7]]
+  ; GFX10-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI3]], [[C6]]
   ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[PHI3]](s32), [[COPY2]]
   ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[ICMP1]](s1)
+  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
   ; GFX10-NEXT:   [[S_ANDN2_B32_1:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY10]](s1), $exec_lo, implicit-def $scc
   ; GFX10-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY11]](s1), implicit-def $scc
   ; GFX10-NEXT:   [[S_OR_B32_1:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_1]](s1), [[S_AND_B32_1]](s1), implicit-def $scc
@@ -260,23 +252,21 @@ body: |
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:sreg_32(s1) = PHI [[S_OR_B32_]](s1), %bb.1, [[S_OR_B32_1]](s1), %bb.3
   ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:_(s32) = G_PHI [[ADD1]](s32), %bb.3, [[DEF]](s32), %bb.1
-  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:_(s1) = G_PHI [[C5]](s1), %bb.3, [[C1]](s1), %bb.1
-  ; GFX10-NEXT:   [[COPY12:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[PHI6]](s1)
-  ; GFX10-NEXT:   [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[PHI4]](s1)
-  ; GFX10-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[COPY13]](s1), [[PHI2]](s32)
-  ; GFX10-NEXT:   [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_ANDN2_B32 [[COPY7]](s1), $exec_lo, implicit-def $scc
-  ; GFX10-NEXT:   [[S_AND_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_AND_B32 $exec_lo, [[COPY12]](s1), implicit-def $scc
-  ; GFX10-NEXT:   [[S_OR_B32_2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_PHI [[C7]](s1), %bb.3, [[C3]](s1), %bb.1
+  ; GFX10-NEXT:   [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[PHI4]](s1)
+  ; GFX10-NEXT:   [[COPY13:%[0-9]+]]:sreg_32(s1) = COPY [[PHI6]](s1)
+  ; GFX10-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[COPY12]](s1), [[PHI2]](s32)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_2:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_2:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY13]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_2:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_2]](s1), [[S_AND_B32_2]](s1), implicit-def $scc
   ; GFX10-NEXT:   SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.6
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.6:
   ; GFX10-NEXT:   successors: %bb.2(0x40000000), %bb.4(0x40000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI7:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.5
-  ; GFX10-NEXT:   [[COPY14:%[0-9]+]]:sreg_32_xm0_xexec(s1) = COPY [[S_OR_B32_2]](s1)
-  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI7]](s32)
-  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[COPY14]](s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
+  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[S_OR_B32_2]](s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX10-NEXT:   G_BR %bb.2
   bb.0:
     successors: %bb.1(0x80000000)
@@ -292,23 +282,23 @@ body: |
     %7:_(s32) = COPY $sgpr2
     %8:_(s32) = COPY $sgpr3
     %9:_(p1) = G_MERGE_VALUES %7(s32), %8(s32)
-    %10:_(s32) = G_CONSTANT i32 0
-    %11:_(s32) = G_IMPLICIT_DEF
+    %10:_(s32) = G_IMPLICIT_DEF
+    %11:_(s32) = G_CONSTANT i32 0
 
   bb.1:
     successors: %bb.3(0x50000000), %bb.5(0x30000000)
 
-    %12:_(s32) = G_PHI %13(s32), %bb.5, %10(s32), %bb.0
-    %14:_(s32) = G_PHI %10(s32), %bb.0, %15(s32), %bb.5
-    %16:_(s1) = G_CONSTANT i1 true
-    %17:_(s64) = G_SEXT %14(s32)
-    %18:_(s32) = G_CONSTANT i32 2
-    %19:_(s64) = G_SHL %17, %18(s32)
-    %20:_(p1) = G_PTR_ADD %6, %19(s64)
-    %21:_(s32) = G_LOAD %20(p1) :: (load (s32), addrspace 1)
-    %22:_(s32) = G_CONSTANT i32 0
-    %23:_(s1) = G_ICMP intpred(ne), %21(s32), %22
-    G_BRCOND %23(s1), %bb.3
+    %12:_(s32) = G_PHI %13(s32), %bb.5, %11(s32), %bb.0
+    %14:_(s32) = G_PHI %11(s32), %bb.0, %15(s32), %bb.5
+    %16:_(s64) = G_SEXT %14(s32)
+    %17:_(s32) = G_CONSTANT i32 2
+    %18:_(s64) = G_SHL %16, %17(s32)
+    %19:_(p1) = G_PTR_ADD %6, %18(s64)
+    %20:_(s32) = G_LOAD %19(p1) :: (load (s32), addrspace 1)
+    %21:_(s32) = G_CONSTANT i32 0
+    %22:_(s1) = G_ICMP intpred(ne), %20(s32), %21
+    %23:_(s1) = G_CONSTANT i1 true
+    G_BRCOND %22(s1), %bb.3
     G_BR %bb.5
 
   bb.2:
@@ -321,16 +311,16 @@ body: |
   bb.3:
     successors: %bb.5(0x80000000)
 
-    %25:_(s1) = G_CONSTANT i1 false
-    %26:_(s32) = G_CONSTANT i32 2
-    %27:_(s64) = G_SHL %17, %26(s32)
-    %28:_(p1) = G_PTR_ADD %2, %27(s64)
-    %29:_(s32) = G_LOAD %28(p1) :: (load (s32), addrspace 1)
-    %30:_(s32) = G_CONSTANT i32 1
-    %31:_(s32) = G_ADD %29, %30
-    G_STORE %31(s32), %28(p1) :: (store (s32), addrspace 1)
-    %32:_(s32) = G_ADD %14, %30
-    %33:_(s1) = G_ICMP intpred(ult), %14(s32), %3
+    %25:_(s32) = G_CONSTANT i32 2
+    %26:_(s64) = G_SHL %16, %25(s32)
+    %27:_(p1) = G_PTR_ADD %2, %26(s64)
+    %28:_(s32) = G_LOAD %27(p1) :: (load (s32), addrspace 1)
+    %29:_(s32) = G_CONSTANT i32 1
+    %30:_(s32) = G_ADD %28, %29
+    G_STORE %30(s32), %27(p1) :: (store (s32), addrspace 1)
+    %31:_(s32) = G_ADD %14, %29
+    %32:_(s1) = G_ICMP intpred(ult), %14(s32), %3
+    %33:_(s1) = G_CONSTANT i1 false
     G_BR %bb.5
 
   bb.4:
@@ -340,9 +330,9 @@ body: |
   bb.5:
     successors: %bb.6(0x04000000), %bb.1(0x7c000000)
 
-    %15:_(s32) = G_PHI %32(s32), %bb.3, %11(s32), %bb.1
-    %35:_(s1) = G_PHI %25(s1), %bb.3, %16(s1), %bb.1
-    %36:_(s1) = G_PHI %33(s1), %bb.3, %16(s1), %bb.1
+    %15:_(s32) = G_PHI %31(s32), %bb.3, %10(s32), %bb.1
+    %35:sreg_32_xm0_xexec(s1) = G_PHI %33(s1), %bb.3, %23(s1), %bb.1
+    %36:_(s1) = G_PHI %32(s1), %bb.3, %23(s1), %bb.1
     %13:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %36(s1), %12(s32)
     SI_LOOP %13(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.6
@@ -350,9 +340,429 @@ body: |
   bb.6:
     successors: %bb.2(0x40000000), %bb.4(0x40000000)
 
-    %37:sreg_32_xm0_xexec(s1) = G_PHI %35(s1), %bb.5
-    %38:_(s32) = G_PHI %13(s32), %bb.5
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %38(s32)
-    %34:sreg_32_xm0_xexec(s32) = SI_IF %37(s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %13(s32)
+    %34:sreg_32_xm0_xexec(s32) = SI_IF %35(s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
     G_BR %bb.2
 ...
+
+---
+name: nested_loops_temporal_divergence_inner
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: nested_loops_temporal_divergence_inner
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[COPY]](s32), [[C]]
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %12(s32), %bb.3, [[C1]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.0, %14(s32), %bb.3
+  ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI1]](s32)
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C2]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[MV]], [[SHL]](s64)
+  ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32))
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[DEF:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+  ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[FCMP]](s1)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.3(0x04000000), %bb.2(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:sreg_32(s1) = PHI [[COPY6]](s1), %bb.1, %43(s1), %bb.2
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.1, %35(s1), %bb.2
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[C3]](s32), %bb.1, %22(s32), %bb.2
+  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:_(s32) = G_PHI %24(s32), %bb.2, [[C3]](s32), %bb.1
+  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[PHI2]](s1)
+  ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1)
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[COPY7]], [[C4]]
+  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1)
+  ; GFX10-NEXT:   [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[PHI5]](s32)
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI5]], [[C5]]
+  ; GFX10-NEXT:   [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[LOAD]]
+  ; GFX10-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP1]](s1), [[PHI4]](s32)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY9]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1)
+  ; GFX10-NEXT:   SI_LOOP [[INT]](s32), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.4(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[MV1]], [[SEXT]](s64)
+  ; GFX10-NEXT:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[S_OR_B32_]](s1)
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   G_STORE [[ZEXT]](s32), [[PTR_ADD1]](p0) :: (store (s8))
+  ; GFX10-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C6]]
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[PHI1]](s32), [[COPY1]]
+  ; GFX10-NEXT:   [[INT1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[ICMP]](s1), [[PHI]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INT1]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT1]](s32)
+  ; GFX10-NEXT:   SI_RETURN
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(p0) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s32) = COPY $vgpr4
+    %6:_(s32) = COPY $vgpr5
+    %7:_(p0) = G_MERGE_VALUES %5(s32), %6(s32)
+    %8:_(s32) = G_FCONSTANT float 1.000000e+00
+    %9:_(s1) = G_FCMP floatpred(ogt), %0(s32), %8
+    %10:_(s32) = G_CONSTANT i32 0
+
+  bb.1:
+    %11:_(s32) = G_PHI %12(s32), %bb.3, %10(s32), %bb.0
+    %13:_(s32) = G_PHI %10(s32), %bb.0, %14(s32), %bb.3
+    %15:_(s64) = G_SEXT %13(s32)
+    %16:_(s32) = G_CONSTANT i32 2
+    %17:_(s64) = G_SHL %15, %16(s32)
+    %18:_(p0) = G_PTR_ADD %4, %17(s64)
+    %19:_(s32) = G_LOAD %18(p0) :: (load (s32))
+    %20:_(s32) = G_CONSTANT i32 0
+
+  bb.2:
+    successors: %bb.3(0x04000000), %bb.2(0x7c000000)
+
+    %21:_(s32) = G_PHI %20(s32), %bb.1, %22(s32), %bb.2
+    %23:_(s32) = G_PHI %24(s32), %bb.2, %20(s32), %bb.1
+    %25:_(s1) = G_PHI %26(s1), %bb.2, %9(s1), %bb.1
+    %27:_(s1) = G_CONSTANT i1 true
+    %26:_(s1) = G_XOR %25, %27
+    %28:_(s32) = G_UITOFP %23(s32)
+    %29:_(s32) = G_CONSTANT i32 1
+    %24:_(s32) = G_ADD %23, %29
+    %30:_(s1) = G_FCMP floatpred(ogt), %28(s32), %19
+    %22:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %30(s1), %21(s32)
+    SI_LOOP %22(s32), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.3
+
+  bb.3:
+    successors: %bb.4(0x04000000), %bb.1(0x7c000000)
+
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %22(s32)
+    %31:_(p0) = G_PTR_ADD %7, %15(s64)
+    %32:_(s32) = G_ZEXT %26(s1)
+    %33:_(s32) = G_CONSTANT i32 1
+    G_STORE %32(s32), %31(p0) :: (store (s8))
+    %14:_(s32) = G_ADD %13, %33
+    %34:_(s1) = G_ICMP intpred(ult), %13(s32), %1
+    %12:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %34(s1), %11(s32)
+    SI_LOOP %12(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.4
+
+  bb.4:
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %12(s32)
+    SI_RETURN
+...
+
+---
+name: nested_loops_temporal_divergence_outer
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: nested_loops_temporal_divergence_outer
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[COPY]](s32), [[C]]
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %12(s32), %bb.3, [[C1]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.0, %14(s32), %bb.3
+  ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI1]](s32)
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C2]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[MV]], [[SHL]](s64)
+  ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32))
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[DEF:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+  ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[FCMP]](s1)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.3(0x04000000), %bb.2(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:sreg_32(s1) = PHI [[COPY6]](s1), %bb.1, %43(s1), %bb.2
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.1, %35(s1), %bb.2
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[C3]](s32), %bb.1, %22(s32), %bb.2
+  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:_(s32) = G_PHI %24(s32), %bb.2, [[C3]](s32), %bb.1
+  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:sreg_32(s1) = COPY [[PHI2]](s1)
+  ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1)
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[COPY7]], [[C4]]
+  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1)
+  ; GFX10-NEXT:   [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[PHI5]](s32)
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI5]], [[C5]]
+  ; GFX10-NEXT:   [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[LOAD]]
+  ; GFX10-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP1]](s1), [[PHI4]](s32)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY8]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY9]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1)
+  ; GFX10-NEXT:   SI_LOOP [[INT]](s32), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.4(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[MV1]], [[SEXT]](s64)
+  ; GFX10-NEXT:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[S_OR_B32_]](s1)
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   G_STORE [[ZEXT]](s32), [[PTR_ADD1]](p0) :: (store (s8))
+  ; GFX10-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C6]]
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[PHI1]](s32), [[COPY1]]
+  ; GFX10-NEXT:   [[INT1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[ICMP]](s1), [[PHI]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INT1]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT1]](s32)
+  ; GFX10-NEXT:   SI_RETURN
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(p0) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s32) = COPY $vgpr4
+    %6:_(s32) = COPY $vgpr5
+    %7:_(p0) = G_MERGE_VALUES %5(s32), %6(s32)
+    %8:_(s32) = G_FCONSTANT float 1.000000e+00
+    %9:_(s1) = G_FCMP floatpred(ogt), %0(s32), %8
+    %10:_(s32) = G_CONSTANT i32 0
+
+  bb.1:
+    %11:_(s32) = G_PHI %12(s32), %bb.3, %10(s32), %bb.0
+    %13:_(s32) = G_PHI %10(s32), %bb.0, %14(s32), %bb.3
+    %15:_(s64) = G_SEXT %13(s32)
+    %16:_(s32) = G_CONSTANT i32 2
+    %17:_(s64) = G_SHL %15, %16(s32)
+    %18:_(p0) = G_PTR_ADD %4, %17(s64)
+    %19:_(s32) = G_LOAD %18(p0) :: (load (s32))
+    %20:_(s32) = G_CONSTANT i32 0
+
+  bb.2:
+    successors: %bb.3(0x04000000), %bb.2(0x7c000000)
+
+    %21:_(s32) = G_PHI %20(s32), %bb.1, %22(s32), %bb.2
+    %23:_(s32) = G_PHI %24(s32), %bb.2, %20(s32), %bb.1
+    %25:_(s1) = G_PHI %26(s1), %bb.2, %9(s1), %bb.1
+    %27:_(s1) = G_CONSTANT i1 true
+    %26:_(s1) = G_XOR %25, %27
+    %28:_(s32) = G_UITOFP %23(s32)
+    %29:_(s32) = G_CONSTANT i32 1
+    %24:_(s32) = G_ADD %23, %29
+    %30:_(s1) = G_FCMP floatpred(ogt), %28(s32), %19
+    %22:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %30(s1), %21(s32)
+    SI_LOOP %22(s32), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.3
+
+  bb.3:
+    successors: %bb.4(0x04000000), %bb.1(0x7c000000)
+
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %22(s32)
+    %31:_(p0) = G_PTR_ADD %7, %15(s64)
+    %32:_(s32) = G_ZEXT %26(s1)
+    %33:_(s32) = G_CONSTANT i32 1
+    G_STORE %32(s32), %31(p0) :: (store (s8))
+    %14:_(s32) = G_ADD %13, %33
+    %34:_(s1) = G_ICMP intpred(ult), %13(s32), %1
+    %12:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %34(s1), %11(s32)
+    SI_LOOP %12(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.4
+
+  bb.4:
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %12(s32)
+    SI_RETURN
+...
+
+---
+name: nested_loops_temporal_divergence_both
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: nested_loops_temporal_divergence_both
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; GFX10-NEXT:   [[MV2:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[COPY]](s32), [[C]]
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[DEF:%[0-9]+]]:sreg_32(s1) = IMPLICIT_DEF
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF]](s1), %bb.0, %39(s1), %bb.3
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI %15(s32), %bb.3, [[C1]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.0, %17(s32), %bb.3
+  ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI2]](s32)
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C2]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[MV]], [[SHL]](s64)
+  ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32))
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[COPY8:%[0-9]+]]:sreg_32(s1) = COPY [[FCMP]](s1)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.3(0x04000000), %bb.2(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:sreg_32(s1) = PHI [[COPY8]](s1), %bb.1, %48(s1), %bb.2
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:sreg_32(s1) = PHI [[PHI]](s1), %bb.1, %39(s1), %bb.2
+  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:_(s32) = G_PHI [[C3]](s32), %bb.1, %25(s32), %bb.2
+  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:_(s32) = G_PHI %27(s32), %bb.2, [[C3]](s32), %bb.1
+  ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:sreg_32(s1) = COPY [[PHI3]](s1)
+  ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:sreg_32(s1) = COPY [[PHI4]](s1)
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[COPY9]], [[C4]]
+  ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1)
+  ; GFX10-NEXT:   [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[PHI6]](s32)
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI6]], [[C5]]
+  ; GFX10-NEXT:   [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[LOAD]]
+  ; GFX10-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP1]](s1), [[PHI5]](s32)
+  ; GFX10-NEXT:   [[S_ANDN2_B32_:%[0-9]+]]:sreg_32(s1) = S_ANDN2_B32 [[COPY10]](s1), $exec_lo, implicit-def $scc
+  ; GFX10-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32(s1) = S_AND_B32 $exec_lo, [[COPY11]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[S_OR_B32_:%[0-9]+]]:sreg_32(s1) = S_OR_B32 [[S_ANDN2_B32_]](s1), [[S_AND_B32_]](s1), implicit-def $scc
+  ; GFX10-NEXT:   [[COPY12:%[0-9]+]]:sreg_32(s1) = COPY [[XOR]](s1)
+  ; GFX10-NEXT:   SI_LOOP [[INT]](s32), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.4(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[MV1]], [[SEXT]](s64)
+  ; GFX10-NEXT:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[S_OR_B32_]](s1)
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   G_STORE [[ZEXT]](s32), [[PTR_ADD1]](p0) :: (store (s8))
+  ; GFX10-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI2]], [[C6]]
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[PHI2]](s32), [[COPY1]]
+  ; GFX10-NEXT:   [[INT1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[ICMP]](s1), [[PHI1]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INT1]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT1]](s32)
+  ; GFX10-NEXT:   [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[S_OR_B32_]](s1)
+  ; GFX10-NEXT:   G_STORE [[ZEXT1]](s32), [[MV2]](p0) :: (store (s8))
+  ; GFX10-NEXT:   SI_RETURN
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(p0) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s32) = COPY $vgpr4
+    %6:_(s32) = COPY $vgpr5
+    %7:_(p0) = G_MERGE_VALUES %5(s32), %6(s32)
+    %8:_(s32) = COPY $vgpr6
+    %9:_(s32) = COPY $vgpr7
+    %10:_(p0) = G_MERGE_VALUES %8(s32), %9(s32)
+    %11:_(s32) = G_FCONSTANT float 1.000000e+00
+    %12:_(s1) = G_FCMP floatpred(ogt), %0(s32), %11
+    %13:_(s32) = G_CONSTANT i32 0
+
+  bb.1:
+    %14:_(s32) = G_PHI %15(s32), %bb.3, %13(s32), %bb.0
+    %16:_(s32) = G_PHI %13(s32), %bb.0, %17(s32), %bb.3
+    %18:_(s64) = G_SEXT %16(s32)
+    %19:_(s32) = G_CONSTANT i32 2
+    %20:_(s64) = G_SHL %18, %19(s32)
+    %21:_(p0) = G_PTR_ADD %4, %20(s64)
+    %22:_(s32) = G_LOAD %21(p0) :: (load (s32))
+    %23:_(s32) = G_CONSTANT i32 0
+
+  bb.2:
+    successors: %bb.3(0x04000000), %bb.2(0x7c000000)
+
+    %24:_(s32) = G_PHI %23(s32), %bb.1, %25(s32), %bb.2
+    %26:_(s32) = G_PHI %27(s32), %bb.2, %23(s32), %bb.1
+    %28:_(s1) = G_PHI %29(s1), %bb.2, %12(s1), %bb.1
+    %30:_(s1) = G_CONSTANT i1 true
+    %29:_(s1) = G_XOR %28, %30
+    %31:_(s32) = G_UITOFP %26(s32)
+    %32:_(s32) = G_CONSTANT i32 1
+    %27:_(s32) = G_ADD %26, %32
+    %33:_(s1) = G_FCMP floatpred(ogt), %31(s32), %22
+    %25:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %33(s1), %24(s32)
+    SI_LOOP %25(s32), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.3
+
+  bb.3:
+    successors: %bb.4(0x04000000), %bb.1(0x7c000000)
+
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %25(s32)
+    %34:_(p0) = G_PTR_ADD %7, %18(s64)
+    %35:_(s32) = G_ZEXT %29(s1)
+    %36:_(s32) = G_CONSTANT i32 1
+    G_STORE %35(s32), %34(p0) :: (store (s8))
+    %17:_(s32) = G_ADD %16, %36
+    %37:_(s1) = G_ICMP intpred(ult), %16(s32), %1
+    %15:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %37(s1), %14(s32)
+    SI_LOOP %15(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.4
+
+  bb.4:
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %15(s32)
+    %38:_(s32) = G_ZEXT %29(s1)
+    G_STORE %38(s32), %10(p0) :: (store (s8))
+    SI_RETURN
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.ll
index 1934958ea8f37..136f095b57875 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.ll
@@ -1,18 +1,18 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -new-reg-bank-select -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
 
 define void @temporal_divergent_i32(float %val, ptr %addr) {
 ; GFX10-LABEL: temporal_divergent_i32:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s4, -1
-; GFX10-NEXT:    v_mov_b32_e32 v3, s4
+; GFX10-NEXT:    s_mov_b32 s5, -1
 ; GFX10-NEXT:    s_mov_b32 s4, 0
 ; GFX10-NEXT:  .LBB0_1: ; %loop
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    v_add_nc_u32_e32 v3, 1, v3
-; GFX10-NEXT:    v_cvt_f32_u32_e32 v4, v3
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v0
+; GFX10-NEXT:    s_add_i32 s5, s5, 1
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v3, s5
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v0
+; GFX10-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
 ; GFX10-NEXT:    s_cbranch_execnz .LBB0_1
@@ -35,3 +35,40 @@ exit:
   store i32 %counter, ptr %addr
   ret void
 }
+
+define void @temporal_divergent_i32_multiple_use(float %val, ptr %addr, ptr %addr2) {
+; GFX10-LABEL: temporal_divergent_i32_multiple_use:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_mov_b32 s5, -1
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:  .LBB1_1: ; %loop
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    s_add_i32 s5, s5, 1
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v5, s5
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v0
+; GFX10-NEXT:    v_mov_b32_e32 v5, s5
+; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_cbranch_execnz .LBB1_1
+; GFX10-NEXT:  ; %bb.2: ; %exit
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    flat_store_dword v[1:2], v5
+; GFX10-NEXT:    flat_store_dword v[3:4], v5
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  br label %loop
+
+loop:
+  %counter = phi i32 [ 0, %entry ], [ %counter.plus.1, %loop ]
+  %f.counter = uitofp i32 %counter to float
+  %cond = fcmp ogt float %f.counter, %val
+  %counter.plus.1 = add i32 %counter, 1
+  br i1 %cond, label %exit, label %loop
+
+exit:
+  store i32 %counter, ptr %addr
+  store i32 %counter, ptr %addr2
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.mir
index d1b473f2f41d8..ac36dc4a9b441 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.mir
@@ -15,16 +15,17 @@ body: |
   ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
-  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.1:
   ; GFX10-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
   ; GFX10-NEXT: {{  $}}
-  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %7(s32), %bb.1, [[C]](s32), %bb.0
-  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.0, %9(s32), %bb.1
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %7(s32), %bb.1, [[C1]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %9(s32), %bb.1
   ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
   ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C2]]
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY [[ADD]](s32), implicit $exec_lo
   ; GFX10-NEXT:   [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[ADD]](s32)
   ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
   ; GFX10-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP]](s1), [[PHI]](s32)
@@ -32,10 +33,8 @@ body: |
   ; GFX10-NEXT:   G_BR %bb.2
   ; GFX10-NEXT: {{  $}}
   ; GFX10-NEXT: bb.2:
-  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[ADD]](s32), %bb.1
-  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI [[INT]](s32), %bb.1
-  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI3]](s32)
-  ; GFX10-NEXT:   G_STORE [[PHI2]](s32), [[MV]](p0) :: (store (s32))
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
+  ; GFX10-NEXT:   G_STORE [[COPY3]](s32), [[MV]](p0) :: (store (s32))
   ; GFX10-NEXT:   SI_RETURN
   bb.0:
     successors: %bb.1(0x80000000)
@@ -45,14 +44,14 @@ body: |
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = COPY $vgpr2
     %3:_(p0) = G_MERGE_VALUES %1(s32), %2(s32)
-    %4:_(s32) = G_CONSTANT i32 0
-    %5:_(s32) = G_CONSTANT i32 -1
+    %4:_(s32) = G_CONSTANT i32 -1
+    %5:_(s32) = G_CONSTANT i32 0
 
   bb.1:
     successors: %bb.2(0x04000000), %bb.1(0x7c000000)
 
-    %6:_(s32) = G_PHI %7(s32), %bb.1, %4(s32), %bb.0
-    %8:_(s32) = G_PHI %5(s32), %bb.0, %9(s32), %bb.1
+    %6:_(s32) = G_PHI %7(s32), %bb.1, %5(s32), %bb.0
+    %8:_(s32) = G_PHI %4(s32), %bb.0, %9(s32), %bb.1
     %10:_(s32) = G_CONSTANT i32 1
     %9:_(s32) = G_ADD %8, %10
     %11:_(s32) = G_UITOFP %9(s32)
@@ -62,9 +61,79 @@ body: |
     G_BR %bb.2
 
   bb.2:
-    %13:_(s32) = G_PHI %9(s32), %bb.1
-    %14:_(s32) = G_PHI %7(s32), %bb.1
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %14(s32)
-    G_STORE %13(s32), %3(p0) :: (store (s32))
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %7(s32)
+    G_STORE %9(s32), %3(p0) :: (store (s32))
+    SI_RETURN
+...
+
+---
+name: temporal_divergent_i32_multiple_use
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: temporal_divergent_i32_multiple_use
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %10(s32), %bb.1, [[C1]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %12(s32), %bb.1
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C2]]
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY [[ADD]](s32), implicit $exec_lo
+  ; GFX10-NEXT:   [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[ADD]](s32)
+  ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
+  ; GFX10-NEXT:   [[INT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), [[FCMP]](s1), [[PHI]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
+  ; GFX10-NEXT:   G_STORE [[COPY5]](s32), [[MV]](p0) :: (store (s32))
+  ; GFX10-NEXT:   G_STORE [[COPY5]](s32), [[MV1]](p0) :: (store (s32))
+  ; GFX10-NEXT:   SI_RETURN
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(p0) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s32) = COPY $vgpr3
+    %5:_(s32) = COPY $vgpr4
+    %6:_(p0) = G_MERGE_VALUES %4(s32), %5(s32)
+    %7:_(s32) = G_CONSTANT i32 -1
+    %8:_(s32) = G_CONSTANT i32 0
+
+  bb.1:
+    successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+
+    %9:_(s32) = G_PHI %10(s32), %bb.1, %8(s32), %bb.0
+    %11:_(s32) = G_PHI %7(s32), %bb.0, %12(s32), %bb.1
+    %13:_(s32) = G_CONSTANT i32 1
+    %12:_(s32) = G_ADD %11, %13
+    %14:_(s32) = G_UITOFP %12(s32)
+    %15:_(s1) = G_FCMP floatpred(ogt), %14(s32), %0
+    %10:sreg_32_xm0_xexec(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %15(s1), %9(s32)
+    SI_LOOP %10(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %10(s32)
+    G_STORE %12(s32), %3(p0) :: (store (s32))
+    G_STORE %12(s32), %6(p0) :: (store (s32))
     SI_RETURN
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergent-control-flow.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergent-control-flow.ll
index e7ddfda2875db..5fa991cd27785 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergent-control-flow.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergent-control-flow.ll
@@ -26,7 +26,7 @@ if.true:
   br label %endif
 
 endif:
-  %v = phi i32 [ %val, %if.true ], [ undef, %entry ]
+  %v = phi i32 [ %val, %if.true ], [ poison, %entry ]
   ret i32 %v
 }
 
@@ -49,7 +49,7 @@ entry:
   br i1 %c, label %if.true, label %endif
 
 endif:
-  %v = phi i32 [ %val, %if.true ], [ undef, %entry ]
+  %v = phi i32 [ %val, %if.true ], [ poison, %entry ]
   ret i32 %v
 
 if.true:
@@ -82,7 +82,7 @@ if.true:
   br label %endif
 
 endif:
-  %v = phi i32 [ %val, %if.true ], [ undef, %entry ]
+  %v = phi i32 [ %val, %if.true ], [ poison, %entry ]
   ret i32 %v
 }
 
@@ -114,7 +114,7 @@ if.true:
   br label %endif
 
 endif:
-  %v = phi i32 [ %val, %if.true ], [ undef, %entry ]
+  %v = phi i32 [ %val, %if.true ], [ poison, %entry ]
   ret i32 %v
 }
 
@@ -199,33 +199,31 @@ define amdgpu_kernel void @break_loop(i32 %arg) {
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_subrev_u32_e32 v0, s0, v0
 ; CHECK-NEXT:    s_mov_b64 s[0:1], 0
-; CHECK-NEXT:    s_branch .LBB5_3
-; CHECK-NEXT:  .LBB5_1: ; %bb4
-; CHECK-NEXT:    ; in Loop: Header=BB5_3 Depth=1
-; CHECK-NEXT:    global_load_dword v2, v[0:1], off glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
-; CHECK-NEXT:    v_cmp_ge_i32_e32 vcc, v0, v2
-; CHECK-NEXT:    s_and_b64 s[4:5], exec, vcc
-; CHECK-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
-; CHECK-NEXT:  .LBB5_2: ; %Flow
-; CHECK-NEXT:    ; in Loop: Header=BB5_3 Depth=1
+; CHECK-NEXT:    s_branch .LBB5_2
+; CHECK-NEXT:  .LBB5_1: ; %Flow
+; CHECK-NEXT:    ; in Loop: Header=BB5_2 Depth=1
 ; CHECK-NEXT:    s_and_b64 s[4:5], exec, s[2:3]
 ; CHECK-NEXT:    s_or_b64 s[0:1], s[4:5], s[0:1]
 ; CHECK-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; CHECK-NEXT:    s_cbranch_execz .LBB5_5
-; CHECK-NEXT:  .LBB5_3: ; %bb1
+; CHECK-NEXT:    s_cbranch_execz .LBB5_4
+; CHECK-NEXT:  .LBB5_2: ; %bb1
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    v_add_u32_e32 v1, 1, v1
 ; CHECK-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
 ; CHECK-NEXT:    s_and_b64 s[4:5], exec, -1
 ; CHECK-NEXT:    v_cmp_le_i32_e32 vcc, 0, v1
 ; CHECK-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
-; CHECK-NEXT:    s_cbranch_vccz .LBB5_1
-; CHECK-NEXT:  ; %bb.4: ; in Loop: Header=BB5_3 Depth=1
-; CHECK-NEXT:    ; implicit-def: $vgpr1
-; CHECK-NEXT:    s_branch .LBB5_2
-; CHECK-NEXT:  .LBB5_5: ; %bb9
+; CHECK-NEXT:    s_cbranch_vccnz .LBB5_1
+; CHECK-NEXT:  ; %bb.3: ; %bb4
+; CHECK-NEXT:    ; in Loop: Header=BB5_2 Depth=1
+; CHECK-NEXT:    global_load_dword v2, v[0:1], off glc
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
+; CHECK-NEXT:    v_cmp_ge_i32_e32 vcc, v0, v2
+; CHECK-NEXT:    s_and_b64 s[4:5], exec, vcc
+; CHECK-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; CHECK-NEXT:    s_branch .LBB5_1
+; CHECK-NEXT:  .LBB5_4: ; %bb9
 ; CHECK-NEXT:    s_endpgm
 bb:
   %id = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.ll
index 1f4330afb58d2..80a9fc509d6ea 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/function-returns.ll
@@ -1080,10 +1080,10 @@ define { <3 x i32>, i32 } @v3i32_struct_func_void_wasted_reg() #0 {
   %load2 = load volatile i32, ptr addrspace(3) undef
   %load3 = load volatile i32, ptr addrspace(3) undef
 
-  %insert.0 = insertelement <3 x i32> undef, i32 %load0, i32 0
+  %insert.0 = insertelement <3 x i32> poison, i32 %load0, i32 0
   %insert.1 = insertelement <3 x i32> %insert.0, i32 %load1, i32 1
   %insert.2 = insertelement <3 x i32> %insert.1, i32 %load2, i32 2
-  %insert.3 = insertvalue { <3 x i32>, i32 } undef, <3 x i32> %insert.2, 0
+  %insert.3 = insertvalue { <3 x i32>, i32 } poison, <3 x i32> %insert.2, 0
   %insert.4 = insertvalue { <3 x i32>, i32 } %insert.3, i32 %load3, 1
   ret { <3 x i32>, i32 } %insert.4
 }
@@ -1115,10 +1115,10 @@ define { <3 x float>, i32 } @v3f32_struct_func_void_wasted_reg() #0 {
   %load2 = load volatile float, ptr addrspace(3) undef
   %load3 = load volatile i32, ptr addrspace(3) undef
 
-  %insert.0 = insertelement <3 x float> undef, float %load0, i32 0
+  %insert.0 = insertelement <3 x float> poison, float %load0, i32 0
   %insert.1 = insertelement <3 x float> %insert.0, float %load1, i32 1
   %insert.2 = insertelement <3 x float> %insert.1, float %load2, i32 2
-  %insert.3 = insertvalue { <3 x float>, i32 } undef, <3 x float> %insert.2, 0
+  %insert.3 = insertvalue { <3 x float>, i32 } poison, <3 x float> %insert.2, 0
   %insert.4 = insertvalue { <3 x float>, i32 } %insert.3, i32 %load3, 1
   ret { <3 x float>, i32 } %insert.4
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
index 298dfcf048fc4..ece08e40ae453 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
@@ -870,10 +870,10 @@ define void @dyn_insertelement_v8f64_const_s_v_v(double %val, i32 %idx) {
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %insert = insertelement <8 x double> <double 1.0, double 2.0, double 3.0, double 4.0, double 5.0, double 6.0, double 7.0, double 8.0>, double %val, i32 %idx
-  %vec.0 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 0, i32 1>
-  %vec.1 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 2, i32 3>
-  %vec.2 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 4, i32 5>
-  %vec.3 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 6, i32 7>
+  %vec.0 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 0, i32 1>
+  %vec.1 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 2, i32 3>
+  %vec.2 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 4, i32 5>
+  %vec.3 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 6, i32 7>
   store volatile <2 x double> %vec.0, ptr addrspace(1) undef
   store volatile <2 x double> %vec.1, ptr addrspace(1) undef
   store volatile <2 x double> %vec.2, ptr addrspace(1) undef
@@ -1081,10 +1081,10 @@ define amdgpu_ps void @dyn_insertelement_v8f64_s_s_v(<8 x double> inreg %vec, do
 ; GFX11-NEXT:    s_endpgm
 entry:
   %insert = insertelement <8 x double> %vec, double %val, i32 %idx
-  %vec.0 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 0, i32 1>
-  %vec.1 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 2, i32 3>
-  %vec.2 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 4, i32 5>
-  %vec.3 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 6, i32 7>
+  %vec.0 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 0, i32 1>
+  %vec.1 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 2, i32 3>
+  %vec.2 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 4, i32 5>
+  %vec.3 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 6, i32 7>
   store volatile <2 x double> %vec.0, ptr addrspace(1) undef
   store volatile <2 x double> %vec.1, ptr addrspace(1) undef
   store volatile <2 x double> %vec.2, ptr addrspace(1) undef
@@ -1229,10 +1229,10 @@ define amdgpu_ps void @dyn_insertelement_v8f64_s_v_s(<8 x double> inreg %vec, do
 ; GFX11-NEXT:    s_endpgm
 entry:
   %insert = insertelement <8 x double> %vec, double %val, i32 %idx
-  %vec.0 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 0, i32 1>
-  %vec.1 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 2, i32 3>
-  %vec.2 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 4, i32 5>
-  %vec.3 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 6, i32 7>
+  %vec.0 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 0, i32 1>
+  %vec.1 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 2, i32 3>
+  %vec.2 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 4, i32 5>
+  %vec.3 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 6, i32 7>
   store volatile <2 x double> %vec.0, ptr addrspace(1) undef
   store volatile <2 x double> %vec.1, ptr addrspace(1) undef
   store volatile <2 x double> %vec.2, ptr addrspace(1) undef
@@ -1289,10 +1289,10 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_s_s(<8 x double> %vec, double i
 ; GFX11-NEXT:    s_endpgm
 entry:
   %insert = insertelement <8 x double> %vec, double %val, i32 %idx
-  %vec.0 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 0, i32 1>
-  %vec.1 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 2, i32 3>
-  %vec.2 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 4, i32 5>
-  %vec.3 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 6, i32 7>
+  %vec.0 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 0, i32 1>
+  %vec.1 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 2, i32 3>
+  %vec.2 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 4, i32 5>
+  %vec.3 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 6, i32 7>
   store volatile <2 x double> %vec.0, ptr addrspace(1) undef
   store volatile <2 x double> %vec.1, ptr addrspace(1) undef
   store volatile <2 x double> %vec.2, ptr addrspace(1) undef
@@ -1494,10 +1494,10 @@ define amdgpu_ps void @dyn_insertelement_v8f64_s_v_v(<8 x double> inreg %vec, do
 ; GFX11-NEXT:    s_endpgm
 entry:
   %insert = insertelement <8 x double> %vec, double %val, i32 %idx
-  %vec.0 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 0, i32 1>
-  %vec.1 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 2, i32 3>
-  %vec.2 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 4, i32 5>
-  %vec.3 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 6, i32 7>
+  %vec.0 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 0, i32 1>
+  %vec.1 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 2, i32 3>
+  %vec.2 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 4, i32 5>
+  %vec.3 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 6, i32 7>
   store volatile <2 x double> %vec.0, ptr addrspace(1) undef
   store volatile <2 x double> %vec.1, ptr addrspace(1) undef
   store volatile <2 x double> %vec.2, ptr addrspace(1) undef
@@ -1617,10 +1617,10 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_s_v(<8 x double> %vec, double i
 ; GFX11-NEXT:    s_endpgm
 entry:
   %insert = insertelement <8 x double> %vec, double %val, i32 %idx
-  %vec.0 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 0, i32 1>
-  %vec.1 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 2, i32 3>
-  %vec.2 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 4, i32 5>
-  %vec.3 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 6, i32 7>
+  %vec.0 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 0, i32 1>
+  %vec.1 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 2, i32 3>
+  %vec.2 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 4, i32 5>
+  %vec.3 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 6, i32 7>
   store volatile <2 x double> %vec.0, ptr addrspace(1) undef
   store volatile <2 x double> %vec.1, ptr addrspace(1) undef
   store volatile <2 x double> %vec.2, ptr addrspace(1) undef
@@ -1677,10 +1677,10 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_v_s(<8 x double> %vec, double %
 ; GFX11-NEXT:    s_endpgm
 entry:
   %insert = insertelement <8 x double> %vec, double %val, i32 %idx
-  %vec.0 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 0, i32 1>
-  %vec.1 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 2, i32 3>
-  %vec.2 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 4, i32 5>
-  %vec.3 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 6, i32 7>
+  %vec.0 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 0, i32 1>
+  %vec.1 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 2, i32 3>
+  %vec.2 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 4, i32 5>
+  %vec.3 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 6, i32 7>
   store volatile <2 x double> %vec.0, ptr addrspace(1) undef
   store volatile <2 x double> %vec.1, ptr addrspace(1) undef
   store volatile <2 x double> %vec.2, ptr addrspace(1) undef
@@ -1794,10 +1794,10 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_v_v(<8 x double> %vec, double %
 ; GFX11-NEXT:    s_endpgm
 entry:
   %insert = insertelement <8 x double> %vec, double %val, i32 %idx
-  %vec.0 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 0, i32 1>
-  %vec.1 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 2, i32 3>
-  %vec.2 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 4, i32 5>
-  %vec.3 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 6, i32 7>
+  %vec.0 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 0, i32 1>
+  %vec.1 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 2, i32 3>
+  %vec.2 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 4, i32 5>
+  %vec.3 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 6, i32 7>
   store volatile <2 x double> %vec.0, ptr addrspace(1) undef
   store volatile <2 x double> %vec.1, ptr addrspace(1) undef
   store volatile <2 x double> %vec.2, ptr addrspace(1) undef
@@ -2401,10 +2401,10 @@ define amdgpu_ps void @dyn_insertelement_v8f64_s_s_s_add_1(<8 x double> inreg %v
 entry:
   %idx.add = add i32 %idx, 1
   %insert = insertelement <8 x double> %vec, double %val, i32 %idx.add
-  %vec.0 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 0, i32 1>
-  %vec.1 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 2, i32 3>
-  %vec.2 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 4, i32 5>
-  %vec.3 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 6, i32 7>
+  %vec.0 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 0, i32 1>
+  %vec.1 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 2, i32 3>
+  %vec.2 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 4, i32 5>
+  %vec.3 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 6, i32 7>
   store volatile <2 x double> %vec.0, ptr addrspace(1) undef
   store volatile <2 x double> %vec.1, ptr addrspace(1) undef
   store volatile <2 x double> %vec.2, ptr addrspace(1) undef
@@ -2525,10 +2525,10 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_v_v_add_1(<8 x double> %vec, do
 entry:
   %idx.add = add i32 %idx, 1
   %insert = insertelement <8 x double> %vec, double %val, i32 %idx.add
-  %vec.0 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 0, i32 1>
-  %vec.1 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 2, i32 3>
-  %vec.2 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 4, i32 5>
-  %vec.3 = shufflevector <8 x double> %insert, <8 x double> undef, <2 x i32> <i32 6, i32 7>
+  %vec.0 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 0, i32 1>
+  %vec.1 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 2, i32 3>
+  %vec.2 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 4, i32 5>
+  %vec.3 = shufflevector <8 x double> %insert, <8 x double> poison, <2 x i32> <i32 6, i32 7>
   store volatile <2 x double> %vec.0, ptr addrspace(1) undef
   store volatile <2 x double> %vec.1, ptr addrspace(1) undef
   store volatile <2 x double> %vec.2, ptr addrspace(1) undef
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_ps.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_ps.ll
index e98f298adf58e..e30d3c2b26bf3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_ps.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_ps.ll
@@ -104,7 +104,7 @@ define amdgpu_ps { i32, i32 } @sgpr_struct_return_i32_i32(i32 %vgpr0, i32 %vgpr1
   ; CHECK-NEXT:   [[INTRINSIC_CONVERGENT1:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](s32)
   ; CHECK-NEXT:   $sgpr1 = COPY [[INTRINSIC_CONVERGENT1]](s32)
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
-  %insertvalue0 = insertvalue { i32, i32 } undef, i32 %vgpr0, 0
+  %insertvalue0 = insertvalue { i32, i32 } poison, i32 %vgpr0, 0
   %value = insertvalue { i32, i32 } %insertvalue0, i32 %vgpr1, 1
   ret { i32, i32 } %value
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll
index 875b725a3c76c..b4eb7e2f887e3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll
@@ -88,7 +88,7 @@ define amdgpu_vs <{ i32, i32 }> @ret_struct(i32 inreg %arg0, i32 inreg %arg1) {
   ; CHECK-NEXT:   $sgpr1 = COPY [[INTRINSIC_CONVERGENT1]](s32)
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
 main_body:
-  %tmp0 = insertvalue <{ i32, i32 }> undef, i32 %arg0, 0
+  %tmp0 = insertvalue <{ i32, i32 }> poison, i32 %arg0, 0
   %tmp1 = insertvalue <{ i32, i32 }> %tmp0, i32 %arg1, 1
   ret <{ i32, i32 }> %tmp1
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
index 78f33a174980d..a7b063427bc8d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
@@ -1519,7 +1519,7 @@ define amdgpu_kernel void @test_call_external_void_func_v3i64() #0 {
   ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
   ; CHECK-NEXT:   S_ENDPGM 0
   %load = load <2 x i64>, ptr addrspace(1) null
-  %val = shufflevector <2 x i64> %load, <2 x i64> <i64 8589934593, i64 undef>, <3 x i32> <i32 0, i32 1, i32 2>
+  %val = shufflevector <2 x i64> %load, <2 x i64> <i64 8589934593, i64 poison>, <3 x i32> <i32 0, i32 1, i32 2>
 
   call void @external_void_func_v3i64(<3 x i64> %val)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-misaligned-bug.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-misaligned-bug.ll
index c35952c5b77ed..b68cc982731ed 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-misaligned-bug.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-misaligned-bug.ll
@@ -17,7 +17,7 @@ bb:
   %load = load <2 x i32>, ptr addrspace(3) %gep, align 4
   %v1 = extractelement <2 x i32> %load, i32 0
   %v2 = extractelement <2 x i32> %load, i32 1
-  %v3 = insertelement <2 x i32> undef, i32 %v2, i32 0
+  %v3 = insertelement <2 x i32> poison, i32 %v2, i32 0
   %v4 = insertelement <2 x i32> %v3, i32 %v1, i32 1
   store <2 x i32> %v4, ptr addrspace(3) %gep, align 4
   ret void
@@ -39,7 +39,7 @@ bb:
   %v2 = extractelement <4 x i32> %load, i32 1
   %v3 = extractelement <4 x i32> %load, i32 2
   %v4 = extractelement <4 x i32> %load, i32 3
-  %v5 = insertelement <4 x i32> undef, i32 %v4, i32 0
+  %v5 = insertelement <4 x i32> poison, i32 %v4, i32 0
   %v6 = insertelement <4 x i32> %v5, i32 %v3, i32 1
   %v7 = insertelement <4 x i32> %v6, i32 %v2, i32 2
   %v8 = insertelement <4 x i32> %v7, i32 %v1, i32 3
@@ -62,7 +62,7 @@ bb:
   %v1 = extractelement <3 x i32> %load, i32 0
   %v2 = extractelement <3 x i32> %load, i32 1
   %v3 = extractelement <3 x i32> %load, i32 2
-  %v5 = insertelement <3 x i32> undef, i32 %v3, i32 0
+  %v5 = insertelement <3 x i32> poison, i32 %v3, i32 0
   %v6 = insertelement <3 x i32> %v5, i32 %v1, i32 1
   %v7 = insertelement <3 x i32> %v6, i32 %v2, i32 2
   store <3 x i32> %v7, ptr addrspace(3) %gep, align 4
@@ -79,7 +79,7 @@ bb:
   %load = load <2 x i32>, ptr addrspace(3) %gep, align 8
   %v1 = extractelement <2 x i32> %load, i32 0
   %v2 = extractelement <2 x i32> %load, i32 1
-  %v3 = insertelement <2 x i32> undef, i32 %v2, i32 0
+  %v3 = insertelement <2 x i32> poison, i32 %v2, i32 0
   %v4 = insertelement <2 x i32> %v3, i32 %v1, i32 1
   store <2 x i32> %v4, ptr addrspace(3) %gep, align 8
   ret void
@@ -96,7 +96,7 @@ bb:
   %v1 = extractelement <3 x i32> %load, i32 0
   %v2 = extractelement <3 x i32> %load, i32 1
   %v3 = extractelement <3 x i32> %load, i32 2
-  %v5 = insertelement <3 x i32> undef, i32 %v3, i32 0
+  %v5 = insertelement <3 x i32> poison, i32 %v3, i32 0
   %v6 = insertelement <3 x i32> %v5, i32 %v1, i32 1
   %v7 = insertelement <3 x i32> %v6, i32 %v2, i32 2
   store <3 x i32> %v7, ptr addrspace(3) %gep, align 16
@@ -121,7 +121,7 @@ bb:
   %v2 = extractelement <4 x i32> %load, i32 1
   %v3 = extractelement <4 x i32> %load, i32 2
   %v4 = extractelement <4 x i32> %load, i32 3
-  %v5 = insertelement <4 x i32> undef, i32 %v4, i32 0
+  %v5 = insertelement <4 x i32> poison, i32 %v4, i32 0
   %v6 = insertelement <4 x i32> %v5, i32 %v3, i32 1
   %v7 = insertelement <4 x i32> %v6, i32 %v2, i32 2
   %v8 = insertelement <4 x i32> %v7, i32 %v1, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
index b26ddbdd7a342..bba0e08ee6341 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
@@ -33,13 +33,13 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_flat(i32 %node_ptr, float
 ; GCN-NEXT:    image_bvh_intersect_ray v[0:3], v[0:10], s[0:3]
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    ; return to shader part epilog
-  %ray_origin0 = insertelement <3 x float> undef, float %ray_origin_x, i32 0
+  %ray_origin0 = insertelement <3 x float> poison, float %ray_origin_x, i32 0
   %ray_origin1 = insertelement <3 x float> %ray_origin0, float %ray_origin_y, i32 1
   %ray_origin = insertelement <3 x float> %ray_origin1, float %ray_origin_z, i32 2
-  %ray_dir0 = insertelement <3 x float> undef, float %ray_dir_x, i32 0
+  %ray_dir0 = insertelement <3 x float> poison, float %ray_dir_x, i32 0
   %ray_dir1 = insertelement <3 x float> %ray_dir0, float %ray_dir_y, i32 1
   %ray_dir = insertelement <3 x float> %ray_dir1, float %ray_dir_z, i32 2
-  %ray_inv_dir0 = insertelement <3 x float> undef, float %ray_inv_dir_x, i32 0
+  %ray_inv_dir0 = insertelement <3 x float> poison, float %ray_inv_dir_x, i32 0
   %ray_inv_dir1 = insertelement <3 x float> %ray_inv_dir0, float %ray_inv_dir_y, i32 1
   %ray_inv_dir = insertelement <3 x float> %ray_inv_dir1, float %ray_inv_dir_z, i32 2
   %v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i32.v4f32(i32 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x float> %ray_dir, <3 x float> %ray_inv_dir, <4 x i32> %tdescr)
@@ -96,13 +96,13 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_flat(<2 x i32> %node_ptr
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    ; return to shader part epilog
   %node_ptr = bitcast <2 x i32> %node_ptr_vec to i64
-  %ray_origin0 = insertelement <3 x float> undef, float %ray_origin_x, i32 0
+  %ray_origin0 = insertelement <3 x float> poison, float %ray_origin_x, i32 0
   %ray_origin1 = insertelement <3 x float> %ray_origin0, float %ray_origin_y, i32 1
   %ray_origin = insertelement <3 x float> %ray_origin1, float %ray_origin_z, i32 2
-  %ray_dir0 = insertelement <3 x float> undef, float %ray_dir_x, i32 0
+  %ray_dir0 = insertelement <3 x float> poison, float %ray_dir_x, i32 0
   %ray_dir1 = insertelement <3 x float> %ray_dir0, float %ray_dir_y, i32 1
   %ray_dir = insertelement <3 x float> %ray_dir1, float %ray_dir_z, i32 2
-  %ray_inv_dir0 = insertelement <3 x float> undef, float %ray_inv_dir_x, i32 0
+  %ray_inv_dir0 = insertelement <3 x float> poison, float %ray_inv_dir_x, i32 0
   %ray_inv_dir1 = insertelement <3 x float> %ray_inv_dir0, float %ray_inv_dir_y, i32 1
   %ray_inv_dir = insertelement <3 x float> %ray_inv_dir1, float %ray_inv_dir_z, i32 2
   %v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i64.v4f32(i64 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x float> %ray_dir, <3 x float> %ray_inv_dir, <4 x i32> %tdescr)
@@ -725,13 +725,13 @@ define amdgpu_kernel void @image_bvh_intersect_ray_nsa_reassign(ptr %p_node_ptr,
   %node_ptr = load i32, ptr %gep_node_ptr, align 4
   %gep_ray = getelementptr inbounds float, ptr %p_ray, i32 %lid
   %ray_extent = load float, ptr %gep_ray, align 4
-  %ray_origin0 = insertelement <3 x float> undef, float 0.0, i32 0
+  %ray_origin0 = insertelement <3 x float> poison, float 0.0, i32 0
   %ray_origin1 = insertelement <3 x float> %ray_origin0, float 1.0, i32 1
   %ray_origin = insertelement <3 x float> %ray_origin1, float 2.0, i32 2
-  %ray_dir0 = insertelement <3 x float> undef, float 3.0, i32 0
+  %ray_dir0 = insertelement <3 x float> poison, float 3.0, i32 0
   %ray_dir1 = insertelement <3 x float> %ray_dir0, float 4.0, i32 1
   %ray_dir = insertelement <3 x float> %ray_dir1, float 5.0, i32 2
-  %ray_inv_dir0 = insertelement <3 x float> undef, float 6.0, i32 0
+  %ray_inv_dir0 = insertelement <3 x float> poison, float 6.0, i32 0
   %ray_inv_dir1 = insertelement <3 x float> %ray_inv_dir0, float 7.0, i32 1
   %ray_inv_dir = insertelement <3 x float> %ray_inv_dir1, float 8.0, i32 2
   %v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i32.v4f32(i32 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x float> %ray_dir, <3 x float> %ray_inv_dir, <4 x i32> %tdescr)
@@ -829,13 +829,13 @@ define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_
   %node_ptr = load i32, ptr %gep_node_ptr, align 4
   %gep_ray = getelementptr inbounds float, ptr %p_ray, i32 %lid
   %ray_extent = load float, ptr %gep_ray, align 4
-  %ray_origin0 = insertelement <3 x float> undef, float 0.0, i32 0
+  %ray_origin0 = insertelement <3 x float> poison, float 0.0, i32 0
   %ray_origin1 = insertelement <3 x float> %ray_origin0, float 1.0, i32 1
   %ray_origin = insertelement <3 x float> %ray_origin1, float 2.0, i32 2
-  %ray_dir0 = insertelement <3 x half> undef, half 3.0, i32 0
+  %ray_dir0 = insertelement <3 x half> poison, half 3.0, i32 0
   %ray_dir1 = insertelement <3 x half> %ray_dir0, half 4.0, i32 1
   %ray_dir = insertelement <3 x half> %ray_dir1, half 5.0, i32 2
-  %ray_inv_dir0 = insertelement <3 x half> undef, half 6.0, i32 0
+  %ray_inv_dir0 = insertelement <3 x half> poison, half 6.0, i32 0
   %ray_inv_dir1 = insertelement <3 x half> %ray_inv_dir0, half 7.0, i32 1
   %ray_inv_dir = insertelement <3 x half> %ray_inv_dir1, half 8.0, i32 2
   %v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i32.v4f16(i32 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x half> %ray_dir, <3 x half> %ray_inv_dir, <4 x i32> %tdescr)
@@ -911,13 +911,13 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_nsa_reassign(ptr %p_ray, <4
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep_ray = getelementptr inbounds float, ptr %p_ray, i32 %lid
   %ray_extent = load float, ptr %gep_ray, align 4
-  %ray_origin0 = insertelement <3 x float> undef, float 0.0, i32 0
+  %ray_origin0 = insertelement <3 x float> poison, float 0.0, i32 0
   %ray_origin1 = insertelement <3 x float> %ray_origin0, float 1.0, i32 1
   %ray_origin = insertelement <3 x float> %ray_origin1, float 2.0, i32 2
-  %ray_dir0 = insertelement <3 x float> undef, float 3.0, i32 0
+  %ray_dir0 = insertelement <3 x float> poison, float 3.0, i32 0
   %ray_dir1 = insertelement <3 x float> %ray_dir0, float 4.0, i32 1
   %ray_dir = insertelement <3 x float> %ray_dir1, float 5.0, i32 2
-  %ray_inv_dir0 = insertelement <3 x float> undef, float 6.0, i32 0
+  %ray_inv_dir0 = insertelement <3 x float> poison, float 6.0, i32 0
   %ray_inv_dir1 = insertelement <3 x float> %ray_inv_dir0, float 7.0, i32 1
   %ray_inv_dir = insertelement <3 x float> %ray_inv_dir1, float 8.0, i32 2
   %v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i64.v4f32(i64 1111111111111, float %ray_extent, <3 x float> %ray_origin, <3 x float> %ray_dir, <3 x float> %ray_inv_dir, <4 x i32> %tdescr)
@@ -985,13 +985,13 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep_ray = getelementptr inbounds float, ptr %p_ray, i32 %lid
   %ray_extent = load float, ptr %gep_ray, align 4
-  %ray_origin0 = insertelement <3 x float> undef, float 0.0, i32 0
+  %ray_origin0 = insertelement <3 x float> poison, float 0.0, i32 0
   %ray_origin1 = insertelement <3 x float> %ray_origin0, float 1.0, i32 1
   %ray_origin = insertelement <3 x float> %ray_origin1, float 2.0, i32 2
-  %ray_dir0 = insertelement <3 x half> undef, half 3.0, i32 0
+  %ray_dir0 = insertelement <3 x half> poison, half 3.0, i32 0
   %ray_dir1 = insertelement <3 x half> %ray_dir0, half 4.0, i32 1
   %ray_dir = insertelement <3 x half> %ray_dir1, half 5.0, i32 2
-  %ray_inv_dir0 = insertelement <3 x half> undef, half 6.0, i32 0
+  %ray_inv_dir0 = insertelement <3 x half> poison, half 6.0, i32 0
   %ray_inv_dir1 = insertelement <3 x half> %ray_inv_dir0, half 7.0, i32 1
   %ray_inv_dir = insertelement <3 x half> %ray_inv_dir1, half 8.0, i32 2
   %v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i64.v4f16(i64 1111111111110, float %ray_extent, <3 x float> %ray_origin, <3 x half> %ray_dir, <3 x half> %ray_inv_dir, <4 x i32> %tdescr)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
index e2dab03e410aa..744a5b7feb48d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
@@ -322,7 +322,7 @@ define i32 @v_sdot2_shuffle10_a(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-NEXT:    v_alignbit_b32 v0, v0, v0, 16
 ; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
-  %shuf.a = shufflevector <2 x i16> %a, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
+  %shuf.a = shufflevector <2 x i16> %a, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> %shuf.a, <2 x i16> %b, i32 %c, i1 false)
   ret i32 %r
 }
@@ -349,7 +349,7 @@ define i32 @v_sdot2_shuffle10_b(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-NEXT:    v_alignbit_b32 v1, v1, v1, 16
 ; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
-  %shuf.b = shufflevector <2 x i16> %b, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
+  %shuf.b = shufflevector <2 x i16> %b, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> %shuf.b, i32 %c, i1 false)
   ret i32 %r
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
index 3acff52874dd9..9e623494a5a04 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
@@ -306,7 +306,7 @@ define i32 @v_udot2_shuffle10_a(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-NEXT:    v_alignbit_b32 v0, v0, v0, 16
 ; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
-  %shuf.a = shufflevector <2 x i16> %a, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
+  %shuf.a = shufflevector <2 x i16> %a, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> %shuf.a, <2 x i16> %b, i32 %c, i1 false)
   ret i32 %r
 }
@@ -332,7 +332,7 @@ define i32 @v_udot2_shuffle10_b(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-NEXT:    v_alignbit_b32 v1, v1, v1, 16
 ; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
-  %shuf.b = shufflevector <2 x i16> %b, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
+  %shuf.b = shufflevector <2 x i16> %b, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> %shuf.b, i32 %c, i1 false)
   ret i32 %r
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll
index 065fadf3b5ef3..2afdef4c8bf9e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll
@@ -64,7 +64,7 @@ define amdgpu_ps { i32, i32 } @s_orn2_i32_multi_use(i32 inreg %src0, i32 inreg %
 ; GFX11-NEXT:    ; return to shader part epilog
   %not.src1 = xor i32 %src1, -1
   %or = or i32 %src0, %not.src1
-  %insert.0 = insertvalue { i32, i32 } undef, i32 %or, 0
+  %insert.0 = insertvalue { i32, i32 } poison, i32 %or, 0
   %insert.1 = insertvalue { i32, i32 } %insert.0, i32 %not.src1, 1
   ret { i32, i32 } %insert.1
 }
@@ -90,7 +90,7 @@ define amdgpu_ps { i32, i32 } @s_orn2_i32_multi_foldable_use(i32 inreg %src0, i3
   %not.src2 = xor i32 %src2, -1
   %or0 = or i32 %src0, %not.src2
   %or1 = or i32 %src1, %not.src2
-  %insert.0 = insertvalue { i32, i32 } undef, i32 %or0, 0
+  %insert.0 = insertvalue { i32, i32 } poison, i32 %or0, 0
   %insert.1 = insertvalue { i32, i32 } %insert.0, i32 %or1, 1
   ret { i32, i32 } %insert.1
 }
@@ -211,7 +211,7 @@ define amdgpu_ps { i64, i64 } @s_orn2_i64_multi_foldable_use(i64 inreg %src0, i6
   %not.src2 = xor i64 %src2, -1
   %or0 = or i64 %src0, %not.src2
   %or1 = or i64 %src1, %not.src2
-  %insert.0 = insertvalue { i64, i64 } undef, i64 %or0, 0
+  %insert.0 = insertvalue { i64, i64 } poison, i64 %or0, 0
   %insert.1 = insertvalue { i64, i64 } %insert.0, i64 %or1, 1
   ret { i64, i64 } %insert.1
 }
@@ -238,7 +238,7 @@ define amdgpu_ps { i64, i64 } @s_orn2_i64_multi_use(i64 inreg %src0, i64 inreg %
 ; GFX11-NEXT:    ; return to shader part epilog
   %not.src1 = xor i64 %src1, -1
   %or = or i64 %src0, %not.src1
-  %insert.0 = insertvalue { i64, i64 } undef, i64 %or, 0
+  %insert.0 = insertvalue { i64, i64 } poison, i64 %or, 0
   %insert.1 = insertvalue { i64, i64 } %insert.0, i64 %not.src1, 1
   ret { i64, i64 } %insert.1
 }
@@ -408,7 +408,7 @@ define amdgpu_ps { i16, i16 } @s_orn2_i16_multi_use(i16 inreg %src0, i16 inreg %
 ; GFX11-NEXT:    ; return to shader part epilog
   %not.src1 = xor i16 %src1, -1
   %or = or i16 %src0, %not.src1
-  %insert.0 = insertvalue { i16, i16 } undef, i16 %or, 0
+  %insert.0 = insertvalue { i16, i16 } poison, i16 %or, 0
   %insert.1 = insertvalue { i16, i16 } %insert.0, i16 %not.src1, 1
   ret { i16, i16 } %insert.1
 }
@@ -434,7 +434,7 @@ define amdgpu_ps { i16, i16 } @s_orn2_i16_multi_foldable_use(i16 inreg %src0, i1
   %not.src2 = xor i16 %src2, -1
   %or0 = or i16 %src0, %not.src2
   %or1 = or i16 %src1, %not.src2
-  %insert.0 = insertvalue { i16, i16 } undef, i16 %or0, 0
+  %insert.0 = insertvalue { i16, i16 } poison, i16 %or0, 0
   %insert.1 = insertvalue { i16, i16 } %insert.0, i16 %or1, 1
   ret { i16, i16 } %insert.1
 }
@@ -601,7 +601,7 @@ define amdgpu_ps { i32, i32 } @s_orn2_v2i16_multi_use(<2 x i16> inreg %src0, <2
 
   %cast.0 = bitcast <2 x i16> %or to i32
   %cast.1 = bitcast <2 x i16> %not.src1 to i32
-  %insert.0 = insertvalue { i32, i32 } undef, i32 %cast.0, 0
+  %insert.0 = insertvalue { i32, i32 } poison, i32 %cast.0, 0
   %insert.1 = insertvalue { i32, i32 } %insert.0, i32 %cast.1, 1
   ret { i32, i32 } %insert.1
 }
@@ -646,7 +646,7 @@ define amdgpu_ps { i32, i32 } @s_orn2_v2i16_multi_foldable_use(<2 x i16> inreg %
 
   %cast.0 = bitcast <2 x i16> %or0 to i32
   %cast.1 = bitcast <2 x i16> %or1 to i32
-  %insert.0 = insertvalue { i32, i32 } undef, i32 %cast.0, 0
+  %insert.0 = insertvalue { i32, i32 } poison, i32 %cast.0, 0
   %insert.1 = insertvalue { i32, i32 } %insert.0, i32 %cast.1, 1
   ret { i32, i32 } %insert.1
 }
@@ -856,7 +856,7 @@ define amdgpu_ps { i48, i48 } @s_orn2_v3i16_multi_use(<3 x i16> inreg %src0, <3
   %or = or <3 x i16> %src0, %not.src1
   %cast.0 = bitcast <3 x i16> %or to i48
   %cast.1 = bitcast <3 x i16> %not.src1 to i48
-  %insert.0 = insertvalue { i48, i48 } undef, i48 %cast.0, 0
+  %insert.0 = insertvalue { i48, i48 } poison, i48 %cast.0, 0
   %insert.1 = insertvalue { i48, i48 } %insert.0, i48 %cast.1, 1
   ret { i48, i48 } %insert.1
 }
@@ -1027,7 +1027,7 @@ define amdgpu_ps { i64, i64 } @s_orn2_v4i16_multi_use(<4 x i16> inreg %src0, <4
 
   %cast.0 = bitcast <4 x i16> %or to i64
   %cast.1 = bitcast <4 x i16> %not.src1 to i64
-  %insert.0 = insertvalue { i64, i64 } undef, i64 %cast.0, 0
+  %insert.0 = insertvalue { i64, i64 } poison, i64 %cast.0, 0
   %insert.1 = insertvalue { i64, i64 } %insert.0, i64 %cast.1, 1
   ret { i64, i64 } %insert.1
 }
@@ -1081,7 +1081,7 @@ define amdgpu_ps { i64, i64 } @s_orn2_v4i16_multi_foldable_use(<4 x i16> inreg %
 
   %cast.0 = bitcast <4 x i16> %or0 to i64
   %cast.1 = bitcast <4 x i16> %or1 to i64
-  %insert.0 = insertvalue { i64, i64 } undef, i64 %cast.0, 0
+  %insert.0 = insertvalue { i64, i64 } poison, i64 %cast.0, 0
   %insert.1 = insertvalue { i64, i64 } %insert.0, i64 %cast.1, 1
   ret { i64, i64 } %insert.1
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbankselect.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbankselect.mir
index 8f3495ea87eec..97dcd5084cacc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbankselect.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui-regbankselect.mir
@@ -597,7 +597,7 @@ body: |
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:sgpr(s32) = G_PHI %17(s32), %bb.1, [[C1]](s32), %bb.0
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:sgpr(s32) = G_PHI %15(s32), %bb.1, [[C1]](s32), %bb.0
   ; CHECK-NEXT:   [[PHI1:%[0-9]+]]:sgpr(s32) = G_PHI [[C]](s32), %bb.0, %9(s32), %bb.1
   ; CHECK-NEXT:   [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
   ; CHECK-NEXT:   [[ADD:%[0-9]+]]:sgpr(s32) = G_ADD [[PHI1]], [[C2]]
@@ -609,11 +609,9 @@ body: |
   ; CHECK-NEXT:   G_BR %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   [[PHI2:%[0-9]+]]:vgpr(s32) = G_PHI [[ADD]](s32), %bb.1
-  ; CHECK-NEXT:   [[PHI3:%[0-9]+]]:sgpr(s32) = G_PHI [[INT]](s32), %bb.1
-  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI3]](s32)
+  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
   ; CHECK-NEXT:   [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
-  ; CHECK-NEXT:   [[MUL:%[0-9]+]]:vgpr(s32) = G_MUL [[PHI2]], [[C3]]
+  ; CHECK-NEXT:   [[MUL:%[0-9]+]]:vgpr(s32) = G_MUL [[ADD]], [[C3]]
   ; CHECK-NEXT:   G_STORE [[MUL]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
   ; CHECK-NEXT:   S_ENDPGM 0
   bb.0:
@@ -641,12 +639,10 @@ body: |
     G_BR %bb.2
 
   bb.2:
-    %13:_(s32) = G_PHI %9(s32), %bb.1
-    %14:_(s32) = G_PHI %7(s32), %bb.1
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %14(s32)
-    %15:_(s32) = G_CONSTANT i32 10
-    %16:_(s32) = G_MUL %13, %15
-    G_STORE %16(s32), %3(p1) :: (store (s32), addrspace 1)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %7(s32)
+    %13:_(s32) = G_CONSTANT i32 10
+    %14:_(s32) = G_MUL %9, %13
+    G_STORE %14(s32), %3(p1) :: (store (s32), addrspace 1)
     S_ENDPGM 0
 ...
 
@@ -677,7 +673,7 @@ body: |
   ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %13(s1), %bb.3
-  ; CHECK-NEXT:   [[PHI1:%[0-9]+]]:sgpr(s32) = G_PHI %68(s32), %bb.3, [[C]](s32), %bb.0
+  ; CHECK-NEXT:   [[PHI1:%[0-9]+]]:sgpr(s32) = G_PHI %67(s32), %bb.3, [[C]](s32), %bb.0
   ; CHECK-NEXT:   [[PHI2:%[0-9]+]]:sgpr(s32) = G_PHI [[C]](s32), %bb.0, %17(s32), %bb.3
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
   ; CHECK-NEXT:   [[SEXT:%[0-9]+]]:sgpr(s64) = G_SEXT [[PHI2]](s32)
@@ -760,8 +756,7 @@ body: |
   ; CHECK-NEXT:   G_BR %bb.3
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.6:
-  ; CHECK-NEXT:   [[PHI7:%[0-9]+]]:sgpr(s32) = G_PHI [[INT]](s32), %bb.3
-  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI7]](s32)
+  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
   ; CHECK-NEXT:   S_ENDPGM 0
   bb.0:
     successors: %bb.1(0x80000000)
@@ -861,7 +856,6 @@ body: |
     G_BR %bb.3
 
   bb.6:
-    %64:_(s32) = G_PHI %15(s32), %bb.3
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %64(s32)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %15(s32)
     S_ENDPGM 0
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.ll
index 951182f703a4e..755eb13a61e14 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.ll
@@ -450,20 +450,20 @@ define amdgpu_ps void @divergent_because_of_temporal_divergent_use(float %val, p
 ;
 ; NEW_RBS-LABEL: divergent_because_of_temporal_divergent_use:
 ; NEW_RBS:       ; %bb.0: ; %entry
-; NEW_RBS-NEXT:    s_mov_b32 s0, -1
-; NEW_RBS-NEXT:    s_mov_b32 s1, 0
+; NEW_RBS-NEXT:    s_mov_b32 s1, -1
+; NEW_RBS-NEXT:    s_mov_b32 s0, 0
 ; NEW_RBS-NEXT:  .LBB15_1: ; %loop
 ; NEW_RBS-NEXT:    ; =>This Inner Loop Header: Depth=1
-; NEW_RBS-NEXT:    s_add_i32 s0, s0, 1
-; NEW_RBS-NEXT:    v_cvt_f32_u32_e32 v3, s0
+; NEW_RBS-NEXT:    s_add_i32 s1, s1, 1
+; NEW_RBS-NEXT:    v_cvt_f32_u32_e32 v3, s1
 ; NEW_RBS-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v3, v0
-; NEW_RBS-NEXT:    s_or_b32 s1, vcc_lo, s1
-; NEW_RBS-NEXT:    s_andn2_b32 exec_lo, exec_lo, s1
+; NEW_RBS-NEXT:    v_mov_b32_e32 v3, s1
+; NEW_RBS-NEXT:    s_or_b32 s0, vcc_lo, s0
+; NEW_RBS-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
 ; NEW_RBS-NEXT:    s_cbranch_execnz .LBB15_1
 ; NEW_RBS-NEXT:  ; %bb.2: ; %exit
-; NEW_RBS-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; NEW_RBS-NEXT:    v_mov_b32_e32 v0, s0
-; NEW_RBS-NEXT:    v_mul_lo_u32 v0, v0, 10
+; NEW_RBS-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; NEW_RBS-NEXT:    v_mul_lo_u32 v0, v3, 10
 ; NEW_RBS-NEXT:    global_store_dword v[1:2], v0, off
 ; NEW_RBS-NEXT:    s_endpgm
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.mir
index 1b22ee4b3fffc..06b0b7269b224 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-mui.mir
@@ -971,12 +971,10 @@ body: |
   ; OLD_RBS-NEXT:   G_BR %bb.2
   ; OLD_RBS-NEXT: {{  $}}
   ; OLD_RBS-NEXT: bb.2:
-  ; OLD_RBS-NEXT:   [[PHI2:%[0-9]+]]:vgpr(s32) = G_PHI [[ADD]](s32), %bb.1
-  ; OLD_RBS-NEXT:   [[PHI3:%[0-9]+]]:sgpr(s32) = G_PHI [[INT]](s32), %bb.1
-  ; OLD_RBS-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI3]](s32)
+  ; OLD_RBS-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
   ; OLD_RBS-NEXT:   [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
   ; OLD_RBS-NEXT:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[C3]](s32)
-  ; OLD_RBS-NEXT:   [[MUL:%[0-9]+]]:vgpr(s32) = G_MUL [[PHI2]], [[COPY4]]
+  ; OLD_RBS-NEXT:   [[MUL:%[0-9]+]]:vgpr(s32) = G_MUL [[ADD]], [[COPY4]]
   ; OLD_RBS-NEXT:   G_STORE [[MUL]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
   ; OLD_RBS-NEXT:   S_ENDPGM 0
   ;
@@ -995,7 +993,7 @@ body: |
   ; NEW_RBS-NEXT: bb.1:
   ; NEW_RBS-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
   ; NEW_RBS-NEXT: {{  $}}
-  ; NEW_RBS-NEXT:   [[PHI:%[0-9]+]]:sgpr(s32) = G_PHI %17(s32), %bb.1, [[C1]](s32), %bb.0
+  ; NEW_RBS-NEXT:   [[PHI:%[0-9]+]]:sgpr(s32) = G_PHI %15(s32), %bb.1, [[C1]](s32), %bb.0
   ; NEW_RBS-NEXT:   [[PHI1:%[0-9]+]]:sgpr(s32) = G_PHI [[C]](s32), %bb.0, %9(s32), %bb.1
   ; NEW_RBS-NEXT:   [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
   ; NEW_RBS-NEXT:   [[ADD:%[0-9]+]]:sgpr(s32) = G_ADD [[PHI1]], [[C2]]
@@ -1008,12 +1006,11 @@ body: |
   ; NEW_RBS-NEXT:   G_BR %bb.2
   ; NEW_RBS-NEXT: {{  $}}
   ; NEW_RBS-NEXT: bb.2:
-  ; NEW_RBS-NEXT:   [[PHI2:%[0-9]+]]:vgpr(s32) = G_PHI [[ADD]](s32), %bb.1
-  ; NEW_RBS-NEXT:   [[PHI3:%[0-9]+]]:sgpr(s32) = G_PHI [[INT]](s32), %bb.1
-  ; NEW_RBS-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI3]](s32)
+  ; NEW_RBS-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
   ; NEW_RBS-NEXT:   [[C3:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 10
-  ; NEW_RBS-NEXT:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[C3]](s32)
-  ; NEW_RBS-NEXT:   [[MUL:%[0-9]+]]:vgpr(s32) = G_MUL [[PHI2]], [[COPY5]]
+  ; NEW_RBS-NEXT:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[ADD]](s32)
+  ; NEW_RBS-NEXT:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[C3]](s32)
+  ; NEW_RBS-NEXT:   [[MUL:%[0-9]+]]:vgpr(s32) = G_MUL [[COPY5]], [[COPY6]]
   ; NEW_RBS-NEXT:   G_STORE [[MUL]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
   ; NEW_RBS-NEXT:   S_ENDPGM 0
   bb.0:
@@ -1041,12 +1038,10 @@ body: |
     G_BR %bb.2
 
   bb.2:
-    %13:_(s32) = G_PHI %9(s32), %bb.1
-    %14:_(s32) = G_PHI %7(s32), %bb.1
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %14(s32)
-    %15:_(s32) = G_CONSTANT i32 10
-    %16:_(s32) = G_MUL %13, %15
-    G_STORE %16(s32), %3(p1) :: (store (s32), addrspace 1)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %7(s32)
+    %13:_(s32) = G_CONSTANT i32 10
+    %14:_(s32) = G_MUL %9, %13
+    G_STORE %14(s32), %3(p1) :: (store (s32), addrspace 1)
     S_ENDPGM 0
 ...
 
@@ -1167,8 +1162,7 @@ body: |
   ; OLD_RBS-NEXT:   G_BR %bb.3
   ; OLD_RBS-NEXT: {{  $}}
   ; OLD_RBS-NEXT: bb.6:
-  ; OLD_RBS-NEXT:   [[PHI7:%[0-9]+]]:sgpr(s32) = G_PHI [[INT]](s32), %bb.3
-  ; OLD_RBS-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI7]](s32)
+  ; OLD_RBS-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
   ; OLD_RBS-NEXT:   S_ENDPGM 0
   ;
   ; NEW_RBS-LABEL: name: loop_with_2breaks
@@ -1193,7 +1187,7 @@ body: |
   ; NEW_RBS-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
   ; NEW_RBS-NEXT: {{  $}}
   ; NEW_RBS-NEXT:   [[PHI:%[0-9]+]]:sreg_32(s1) = PHI [[DEF1]](s1), %bb.0, %13(s1), %bb.3
-  ; NEW_RBS-NEXT:   [[PHI1:%[0-9]+]]:sgpr(s32) = G_PHI %68(s32), %bb.3, [[C]](s32), %bb.0
+  ; NEW_RBS-NEXT:   [[PHI1:%[0-9]+]]:sgpr(s32) = G_PHI %67(s32), %bb.3, [[C]](s32), %bb.0
   ; NEW_RBS-NEXT:   [[PHI2:%[0-9]+]]:sgpr(s32) = G_PHI [[C]](s32), %bb.0, %17(s32), %bb.3
   ; NEW_RBS-NEXT:   [[COPY6:%[0-9]+]]:sreg_32(s1) = COPY [[PHI]](s1)
   ; NEW_RBS-NEXT:   [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 31
@@ -1285,8 +1279,7 @@ body: |
   ; NEW_RBS-NEXT:   G_BR %bb.3
   ; NEW_RBS-NEXT: {{  $}}
   ; NEW_RBS-NEXT: bb.6:
-  ; NEW_RBS-NEXT:   [[PHI7:%[0-9]+]]:sgpr(s32) = G_PHI [[INT]](s32), %bb.3
-  ; NEW_RBS-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI7]](s32)
+  ; NEW_RBS-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INT]](s32)
   ; NEW_RBS-NEXT:   S_ENDPGM 0
   bb.0:
     successors: %bb.1(0x80000000)
@@ -1386,7 +1379,6 @@ body: |
     G_BR %bb.3
 
   bb.6:
-    %64:_(s32) = G_PHI %15(s32), %bb.3
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %64(s32)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %15(s32)
     S_ENDPGM 0
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
index 0c9ff3eee8231..2a200259a93d2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
@@ -368,7 +368,7 @@ define amdgpu_ps i64 @s_sdiv_i64(i64 inreg %num, i64 inreg %den) {
   %elt.1 = extractelement <2 x i32> %cast, i32 1
   %res.0 = call i32 @llvm.amdgcn.readfirstlane(i32 %elt.0)
   %res.1 = call i32 @llvm.amdgcn.readfirstlane(i32 %elt.1)
-  %ins.0 = insertelement <2 x i32> undef, i32 %res.0, i32 0
+  %ins.0 = insertelement <2 x i32> poison, i32 %res.0, i32 0
   %ins.1 = insertelement <2 x i32> %ins.0, i32 %res.0, i32 1
   %cast.back = bitcast <2 x i32> %ins.1 to i64
   ret i64 %cast.back
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/shlN_add.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/shlN_add.ll
index 507985bd626a0..91f71a89c7d0d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/shlN_add.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/shlN_add.ll
@@ -476,7 +476,7 @@ define amdgpu_ps { i32, i32 } @s_shl4_add_u32_multi_use(i32 inreg %src0, i32 inr
 ; GCN-NEXT:    ; return to shader part epilog
   %shl = shl i32 %src0, 4
   %add = add i32 %shl, %src1
-  %insert0 = insertvalue { i32, i32 } undef, i32 %shl, 0
+  %insert0 = insertvalue { i32, i32 } poison, i32 %shl, 0
   %insert1 = insertvalue { i32, i32 } %insert0, i32 %add, 1
   ret { i32, i32 } %insert1
 }
@@ -489,7 +489,7 @@ define amdgpu_ps { i32, i32 } @s_shl3_add_u32_multi_use(i32 inreg %src0, i32 inr
 ; GCN-NEXT:    ; return to shader part epilog
   %shl = shl i32 %src0, 3
   %add = add i32 %shl, %src1
-  %insert0 = insertvalue { i32, i32 } undef, i32 %shl, 0
+  %insert0 = insertvalue { i32, i32 } poison, i32 %shl, 0
   %insert1 = insertvalue { i32, i32 } %insert0, i32 %add, 1
   ret { i32, i32 } %insert1
 }
@@ -502,7 +502,7 @@ define amdgpu_ps { i32, i32 } @s_shl2_add_u32_multi_use(i32 inreg %src0, i32 inr
 ; GCN-NEXT:    ; return to shader part epilog
   %shl = shl i32 %src0, 2
   %add = add i32 %shl, %src1
-  %insert0 = insertvalue { i32, i32 } undef, i32 %shl, 0
+  %insert0 = insertvalue { i32, i32 } poison, i32 %shl, 0
   %insert1 = insertvalue { i32, i32 } %insert0, i32 %add, 1
   ret { i32, i32 } %insert1
 }
@@ -516,7 +516,7 @@ define amdgpu_ps { i32, i32 } @s_shl1_add_u32_multi_use(i32 inreg %src0, i32 inr
 ; GCN-NEXT:    ; return to shader part epilog
   %shl = shl i32 %src0, 1
   %add = add i32 %shl, %src1
-  %insert0 = insertvalue { i32, i32 } undef, i32 %shl, 0
+  %insert0 = insertvalue { i32, i32 } poison, i32 %shl, 0
   %insert1 = insertvalue { i32, i32 } %insert0, i32 %add, 1
   ret { i32, i32 } %insert1
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
index df645888626c6..2bb42308d935c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
@@ -358,7 +358,7 @@ define amdgpu_ps i64 @s_srem_i64(i64 inreg %num, i64 inreg %den) {
   %elt.1 = extractelement <2 x i32> %cast, i32 1
   %res.0 = call i32 @llvm.amdgcn.readfirstlane(i32 %elt.0)
   %res.1 = call i32 @llvm.amdgcn.readfirstlane(i32 %elt.1)
-  %ins.0 = insertelement <2 x i32> undef, i32 %res.0, i32 0
+  %ins.0 = insertelement <2 x i32> poison, i32 %res.0, i32 0
   %ins.1 = insertelement <2 x i32> %ins.0, i32 %res.0, i32 1
   %cast.back = bitcast <2 x i32> %ins.1 to i64
   ret i64 %cast.back
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/trunc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/trunc.ll
index 0cb346c35552c..569ed35b150d8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/trunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/trunc.ll
@@ -121,7 +121,7 @@ define amdgpu_ps i32 @s_trunc_v2i32_to_v2i16(<2 x i32> inreg %src) {
 ; ; FIXME: G_INSERT mishandled
 ; define <2 x i32> @v_trunc_v3i32_to_v3i16(<3 x i32> %src) {
 ;   %trunc = trunc <3 x i32> %src to <3 x i16>
-;   %ext = shufflevector <3 x i16> %trunc, <3 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+;   %ext = shufflevector <3 x i16> %trunc, <3 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ;   %cast = bitcast <4 x i16> %ext to <2 x i32>
 ;   ret <2 x i32> %cast
 ; }
@@ -129,7 +129,7 @@ define amdgpu_ps i32 @s_trunc_v2i32_to_v2i16(<2 x i32> inreg %src) {
 ; ; FIXME: G_INSERT mishandled
 ; define amdgpu_ps <2 x i32> @s_trunc_v3i32_to_v3i16(<3 x i32> inreg %src) {
 ;   %trunc = trunc <3 x i32> %src to <3 x i16>
-;   %ext = shufflevector <3 x i16> %trunc, <3 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+;   %ext = shufflevector <3 x i16> %trunc, <3 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ;   %cast = bitcast <4 x i16> %ext to <2 x i32>
 ;   ret <2 x i32> %cast
 ; }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
index f5a901b024ef5..a292266fbbf0d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
@@ -355,7 +355,7 @@ define amdgpu_ps i64 @s_udiv_i64(i64 inreg %num, i64 inreg %den) {
   %elt.1 = extractelement <2 x i32> %cast, i32 1
   %res.0 = call i32 @llvm.amdgcn.readfirstlane(i32 %elt.0)
   %res.1 = call i32 @llvm.amdgcn.readfirstlane(i32 %elt.1)
-  %ins.0 = insertelement <2 x i32> undef, i32 %res.0, i32 0
+  %ins.0 = insertelement <2 x i32> poison, i32 %res.0, i32 0
   %ins.1 = insertelement <2 x i32> %ins.0, i32 %res.0, i32 1
   %cast.back = bitcast <2 x i32> %ins.1 to i64
   ret i64 %cast.back
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
index 2be4b52198b45..f29c2c2484456 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
@@ -349,7 +349,7 @@ define amdgpu_ps i64 @s_urem_i64(i64 inreg %num, i64 inreg %den) {
   %elt.1 = extractelement <2 x i32> %cast, i32 1
   %res.0 = call i32 @llvm.amdgcn.readfirstlane(i32 %elt.0)
   %res.1 = call i32 @llvm.amdgcn.readfirstlane(i32 %elt.1)
-  %ins.0 = insertelement <2 x i32> undef, i32 %res.0, i32 0
+  %ins.0 = insertelement <2 x i32> poison, i32 %res.0, i32 0
   %ins.1 = insertelement <2 x i32> %ins.0, i32 %res.0, i32 1
   %cast.back = bitcast <2 x i32> %ins.1 to i64
   ret i64 %cast.back
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll
index 9cf9839e69d5d..e12ac25867adb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll
@@ -426,7 +426,7 @@ define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negC_pack(<8 x half> %A, <8 x
 ; GFX12-NEXT:    s_endpgm
 bb:
   %C = load <16 x half>, ptr %Caddr
-  %C_shuffle = shufflevector <16 x half> %C, <16 x half> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %C_shuffle = shufflevector <16 x half> %C, <16 x half> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %fneg.C_shuffle = fneg <8 x half> %C_shuffle
   %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<8 x half> %A, <8 x half> %B, <8 x half> %fneg.C_shuffle , i1 0)
   store <8 x half> %res, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll
index be1761227f802..b871ac7976222 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll
@@ -381,7 +381,7 @@ define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negC_pack(<4 x half> %A, <4 x
 ; GFX12-NEXT:    s_endpgm
 bb:
   %C = load <8 x half>, ptr %Caddr
-  %C_shuffle = shufflevector <8 x half> %C, <8 x half> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %C_shuffle = shufflevector <8 x half> %C, <8 x half> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %fneg.C_shuffle = fneg <4 x half> %C_shuffle
   %res = call <4 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<4 x half> %A, <4 x half> %B, <4 x half> %fneg.C_shuffle , i1 0)
   store <4 x half> %res, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/xnor.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/xnor.ll
index 43322b1e23412..17b6f5072116d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/xnor.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/xnor.ll
@@ -90,7 +90,7 @@ entry:
   %xor = xor i32 %a, %b
   %r0.val = xor i32 %xor, -1
   %r1.val = add i32 %xor, %a
-  %ins0 = insertelement <2 x i32> undef, i32 %r0.val, i32 0
+  %ins0 = insertelement <2 x i32> poison, i32 %r0.val, i32 0
   %ins1 = insertelement <2 x i32> %ins0, i32 %r1.val, i32 1
   ret <2 x i32> %ins1
 }
@@ -196,7 +196,7 @@ define amdgpu_ps <2 x i64> @scalar_xnor_i64_mul_use(i64 inreg %a, i64 inreg %b)
   %xor = xor i64 %a, %b
   %r0.val = xor i64 %xor, -1
   %r1.val = add i64 %xor, %a
-  %ins0 = insertelement <2 x i64> undef, i64 %r0.val, i32 0
+  %ins0 = insertelement <2 x i64> poison, i64 %r0.val, i32 0
   %ins1 = insertelement <2 x i64> %ins0, i64 %r1.val, i32 1
   ret <2 x i64> %ins1
 }
diff --git a/llvm/test/CodeGen/AMDGPU/add3.ll b/llvm/test/CodeGen/AMDGPU/add3.ll
index d3f9c2d0fbc54..0d80296bb67b6 100644
--- a/llvm/test/CodeGen/AMDGPU/add3.ll
+++ b/llvm/test/CodeGen/AMDGPU/add3.ll
@@ -181,7 +181,7 @@ define amdgpu_ps <2 x float> @add3_multiuse_outer(i32 %a, i32 %b, i32 %c, i32 %x
   %inner = add i32 %a, %b
   %outer = add i32 %inner, %c
   %x1 = mul i32 %outer, %x
-  %r1 = insertelement <2 x i32> undef, i32 %outer, i32 0
+  %r1 = insertelement <2 x i32> poison, i32 %outer, i32 0
   %r0 = insertelement <2 x i32> %r1, i32 %x1, i32 1
   %bc = bitcast <2 x i32> %r0 to <2 x float>
   ret <2 x float> %bc
@@ -207,7 +207,7 @@ define amdgpu_ps <2 x float> @add3_multiuse_inner(i32 %a, i32 %b, i32 %c) {
 ; GFX10-NEXT:    ; return to shader part epilog
   %inner = add i32 %a, %b
   %outer = add i32 %inner, %c
-  %r1 = insertelement <2 x i32> undef, i32 %inner, i32 0
+  %r1 = insertelement <2 x i32> poison, i32 %inner, i32 0
   %r0 = insertelement <2 x i32> %r1, i32 %outer, i32 1
   %bc = bitcast <2 x i32> %r0 to <2 x float>
   ret <2 x float> %bc
diff --git a/llvm/test/CodeGen/AMDGPU/adjust-writemask-invalid-copy.ll b/llvm/test/CodeGen/AMDGPU/adjust-writemask-invalid-copy.ll
index ec4e2dda10d3a..d0c7a803ed6b5 100644
--- a/llvm/test/CodeGen/AMDGPU/adjust-writemask-invalid-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/adjust-writemask-invalid-copy.ll
@@ -9,7 +9,7 @@ define amdgpu_ps void @adjust_writemask_crash_0_nochain() #0 {
 main_body:
   %tmp = call <2 x float> @llvm.amdgcn.image.getlod.1d.v2f32.f32(i32 3, float undef, <8 x i32> undef, <4 x i32> undef, i1 0, i32 0, i32 0)
   %tmp1 = bitcast <2 x float> %tmp to <2 x i32>
-  %tmp2 = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %tmp2 = shufflevector <2 x i32> %tmp1, <2 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
   %tmp3 = bitcast <4 x i32> %tmp2 to <4 x float>
   %tmp4 = extractelement <4 x float> %tmp3, i32 0
   store volatile float %tmp4, ptr addrspace(1) undef
@@ -25,7 +25,7 @@ define amdgpu_ps void @adjust_writemask_crash_1_nochain() #0 {
 main_body:
   %tmp = call <2 x float> @llvm.amdgcn.image.getlod.1d.v2f32.f32(i32 3, float undef, <8 x i32> undef, <4 x i32> undef, i1 0, i32 0, i32 0)
   %tmp1 = bitcast <2 x float> %tmp to <2 x i32>
-  %tmp2 = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>
+  %tmp2 = shufflevector <2 x i32> %tmp1, <2 x i32> poison, <4 x i32> <i32 1, i32 0, i32 poison, i32 poison>
   %tmp3 = bitcast <4 x i32> %tmp2 to <4 x float>
   %tmp4 = extractelement <4 x float> %tmp3, i32 1
   store volatile float %tmp4, ptr addrspace(1) undef
@@ -41,7 +41,7 @@ define amdgpu_ps void @adjust_writemask_crash_0_chain() #0 {
 main_body:
   %tmp = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 3, float undef, <8 x i32> undef, <4 x i32> undef, i1 0, i32 0, i32 0)
   %tmp1 = bitcast <2 x float> %tmp to <2 x i32>
-  %tmp2 = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %tmp2 = shufflevector <2 x i32> %tmp1, <2 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
   %tmp3 = bitcast <4 x i32> %tmp2 to <4 x float>
   %tmp4 = extractelement <4 x float> %tmp3, i32 0
   store volatile float %tmp4, ptr addrspace(1) undef
@@ -57,7 +57,7 @@ define amdgpu_ps void @adjust_writemask_crash_1_chain() #0 {
 main_body:
   %tmp = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 3, float undef, <8 x i32> undef, <4 x i32> undef, i1 0, i32 0, i32 0)
   %tmp1 = bitcast <2 x float> %tmp to <2 x i32>
-  %tmp2 = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>
+  %tmp2 = shufflevector <2 x i32> %tmp1, <2 x i32> poison, <4 x i32> <i32 1, i32 0, i32 poison, i32 poison>
   %tmp3 = bitcast <4 x i32> %tmp2 to <4 x float>
   %tmp4 = extractelement <4 x float> %tmp3, i32 1
   store volatile float %tmp4, ptr addrspace(1) undef
@@ -68,7 +68,7 @@ define amdgpu_ps void @adjust_writemask_crash_0_v4() #0 {
 main_body:
   %tmp = call <4 x float> @llvm.amdgcn.image.getlod.1d.v4f32.f32(i32 5, float undef, <8 x i32> undef, <4 x i32> undef, i1 0, i32 0, i32 0)
   %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
-  %tmp2 = shufflevector <4 x i32> %tmp1, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %tmp2 = shufflevector <4 x i32> %tmp1, <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
   %tmp3 = bitcast <4 x i32> %tmp2 to <4 x float>
   %tmp4 = extractelement <4 x float> %tmp3, i32 0
   store volatile float %tmp4, ptr addrspace(1) undef
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
index e70e34fa0ba5d..3116b5d59a097 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
@@ -646,13 +646,13 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX908-NEXT:    v_add_f32_e32 v9, v9, v15
 ; GFX908-NEXT:    v_add_f32_e32 v10, v10, v12
 ; GFX908-NEXT:    v_add_f32_e32 v11, v11, v13
-; GFX908-NEXT:    s_mov_b64 s[20:21], -1
 ; GFX908-NEXT:    s_branch .LBB3_4
 ; GFX908-NEXT:  .LBB3_7: ; in Loop: Header=BB3_5 Depth=2
 ; GFX908-NEXT:    s_mov_b64 s[20:21], s[16:17]
 ; GFX908-NEXT:    s_andn2_b64 vcc, exec, s[20:21]
 ; GFX908-NEXT:    s_cbranch_vccz .LBB3_4
 ; GFX908-NEXT:  ; %bb.8: ; in Loop: Header=BB3_2 Depth=1
+; GFX908-NEXT:    s_mov_b64 s[20:21], -1
 ; GFX908-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GFX908-NEXT:    ; implicit-def: $sgpr18_sgpr19
 ; GFX908-NEXT:  .LBB3_9: ; %loop.exit.guard
@@ -798,13 +798,13 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX90A-NEXT:    v_pk_add_f32 v[8:9], v[8:9], v[26:27]
 ; GFX90A-NEXT:    v_pk_add_f32 v[10:11], v[10:11], v[16:17]
 ; GFX90A-NEXT:    v_pk_add_f32 v[12:13], v[12:13], v[14:15]
-; GFX90A-NEXT:    s_mov_b64 s[20:21], -1
 ; GFX90A-NEXT:    s_branch .LBB3_4
 ; GFX90A-NEXT:  .LBB3_7: ; in Loop: Header=BB3_5 Depth=2
 ; GFX90A-NEXT:    s_mov_b64 s[20:21], s[16:17]
 ; GFX90A-NEXT:    s_andn2_b64 vcc, exec, s[20:21]
 ; GFX90A-NEXT:    s_cbranch_vccz .LBB3_4
 ; GFX90A-NEXT:  ; %bb.8: ; in Loop: Header=BB3_2 Depth=1
+; GFX90A-NEXT:    s_mov_b64 s[20:21], -1
 ; GFX90A-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GFX90A-NEXT:    ; implicit-def: $sgpr18_sgpr19
 ; GFX90A-NEXT:  .LBB3_9: ; %loop.exit.guard
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-break-large-phis-heuristics.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-break-large-phis-heuristics.ll
index 651b030bd3f79..c94b33334646d 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-break-large-phis-heuristics.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-break-large-phis-heuristics.ll
@@ -477,7 +477,7 @@ entry:
 
 then:
   %x.1 = insertelement <5 x double> <double 3.140000e+00, double poison, double poison, double poison, double poison>, double %x, i32 %idx
-  %0 = shufflevector <5 x double> %x.1, <5 x double> <double poison, double poison, double poison, double 6.140000e+00, double 9.900000e+00>, <5 x i32> <i32 0, i32 1, i32 undef, i32 8, i32 9>
+  %0 = shufflevector <5 x double> %x.1, <5 x double> <double poison, double poison, double poison, double 6.140000e+00, double 9.900000e+00>, <5 x i32> <i32 0, i32 1, i32 poison, i32 8, i32 9>
   %x.4 = insertelement <5 x double> %0, double %x, i64 2
   br label %finally
 
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-break-large-phis.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-break-large-phis.ll
index 11772d252a16f..dc2a6137dba3b 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-break-large-phis.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-break-large-phis.ll
@@ -643,11 +643,11 @@ define amdgpu_kernel void @phi_v15i8_random_constant_init(<15 x i8> %in, ptr %ou
 ; OPT-NEXT:    br label [[FINALLY]]
 ; OPT:       finally:
 ; OPT-NEXT:    [[TMP0:%.*]] = phi <4 x i8> [ <i8 poison, i8 1, i8 2, i8 3>, [[THEN]] ], [ [[LARGEPHI_EXTRACTSLICE1]], [[ELSE]] ]
-; OPT-NEXT:    [[TMP1:%.*]] = phi <4 x i8> [ <i8 4, i8 undef, i8 6, i8 7>, [[THEN]] ], [ [[LARGEPHI_EXTRACTSLICE3]], [[ELSE]] ]
+; OPT-NEXT:    [[TMP1:%.*]] = phi <4 x i8> [ <i8 4, i8 poison, i8 6, i8 7>, [[THEN]] ], [ [[LARGEPHI_EXTRACTSLICE3]], [[ELSE]] ]
 ; OPT-NEXT:    [[TMP2:%.*]] = phi <4 x i8> [ <i8 9, i8 10, i8 11, i8 12>, [[THEN]] ], [ [[LARGEPHI_EXTRACTSLICE5]], [[ELSE]] ]
 ; OPT-NEXT:    [[TMP3:%.*]] = phi i8 [ 13, [[THEN]] ], [ [[LARGEPHI_EXTRACTSLICE7]], [[ELSE]] ]
 ; OPT-NEXT:    [[TMP4:%.*]] = phi i8 [ 14, [[THEN]] ], [ [[LARGEPHI_EXTRACTSLICE9]], [[ELSE]] ]
-; OPT-NEXT:    [[TMP5:%.*]] = phi i8 [ undef, [[THEN]] ], [ [[LARGEPHI_EXTRACTSLICE11]], [[ELSE]] ]
+; OPT-NEXT:    [[TMP5:%.*]] = phi i8 [ poison, [[THEN]] ], [ [[LARGEPHI_EXTRACTSLICE11]], [[ELSE]] ]
 ; OPT-NEXT:    [[LARGEPHI_INSERTSLICE0:%.*]] = call <15 x i8> @llvm.vector.insert.v15i8.v4i8(<15 x i8> poison, <4 x i8> [[TMP0]], i64 0)
 ; OPT-NEXT:    [[LARGEPHI_INSERTSLICE1:%.*]] = call <15 x i8> @llvm.vector.insert.v15i8.v4i8(<15 x i8> [[LARGEPHI_INSERTSLICE0]], <4 x i8> [[TMP1]], i64 4)
 ; OPT-NEXT:    [[LARGEPHI_INSERTSLICE2:%.*]] = call <15 x i8> @llvm.vector.insert.v15i8.v4i8(<15 x i8> [[LARGEPHI_INSERTSLICE1]], <4 x i8> [[TMP2]], i64 8)
@@ -666,7 +666,7 @@ define amdgpu_kernel void @phi_v15i8_random_constant_init(<15 x i8> %in, ptr %ou
 ; NOOPT-NEXT:    [[Y:%.*]] = insertelement <15 x i8> [[IN:%.*]], i8 64, i32 6
 ; NOOPT-NEXT:    br label [[FINALLY]]
 ; NOOPT:       finally:
-; NOOPT-NEXT:    [[VAL:%.*]] = phi <15 x i8> [ <i8 poison, i8 1, i8 2, i8 3, i8 4, i8 undef, i8 6, i8 7, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 undef>, [[THEN]] ], [ [[Y]], [[ELSE]] ]
+; NOOPT-NEXT:    [[VAL:%.*]] = phi <15 x i8> [ <i8 poison, i8 1, i8 2, i8 3, i8 4, i8 poison, i8 6, i8 7, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 poison>, [[THEN]] ], [ [[Y]], [[ELSE]] ]
 ; NOOPT-NEXT:    store <15 x i8> [[VAL]], ptr [[OUT:%.*]], align 1
 ; NOOPT-NEXT:    ret void
 ;
@@ -678,7 +678,7 @@ else:
   %y = insertelement <15 x i8> %in, i8 64, i32 6
   br label %finally
 finally:
-  %val = phi <15 x i8> [<i8 poison, i8 1, i8 2, i8 3, i8 4, i8 undef, i8 6, i8 7, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 undef>, %then], [%y, %else]
+  %val = phi <15 x i8> [<i8 poison, i8 1, i8 2, i8 3, i8 4, i8 poison, i8 6, i8 7, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 poison>, %then], [%y, %else]
   store <15 x i8> %val, ptr %out, align 1
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-export-kernel-runtime-handles.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-export-kernel-runtime-handles.ll
new file mode 100644
index 0000000000000..58b7441d85955
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-export-kernel-runtime-handles.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-attributes --check-globals all --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-export-kernel-runtime-handles < %s | FileCheck %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-export-kernel-runtime-handles < %s | FileCheck %s
+
+%block.runtime.handle.t = type { ptr addrspace(1), i32, i32 }
+
+; associated globals without the correct section should be ignored.
+@block.handle = internal addrspace(1) externally_initialized constant %block.runtime.handle.t zeroinitializer, section ".amdgpu.kernel.runtime.handle"
+@not.a.block.handle = internal addrspace(1) externally_initialized constant %block.runtime.handle.t zeroinitializer
+
+;.
+; CHECK: @block.handle = addrspace(1) externally_initialized constant %block.runtime.handle.t zeroinitializer, section ".amdgpu.kernel.runtime.handle"
+; CHECK: @not.a.block.handle = internal addrspace(1) externally_initialized constant %block.runtime.handle.t zeroinitializer
+;.
+define internal amdgpu_kernel void @block_kernel() !associated !0 {
+; CHECK-LABEL: define protected amdgpu_kernel void @block_kernel(
+; CHECK-SAME: ) !associated [[META0:![0-9]+]] {
+; CHECK-NEXT:    ret void
+;
+  ret void
+}
+
+define internal dso_local amdgpu_kernel void @dso_local_block_kernel() !associated !0 {
+; CHECK-LABEL: define protected amdgpu_kernel void @dso_local_block_kernel(
+; CHECK-SAME: ) !associated [[META0]] {
+; CHECK-NEXT:    ret void
+;
+  ret void
+}
+
+define internal amdgpu_kernel void @not_block_kernel() !associated !1 {
+; CHECK-LABEL: define internal amdgpu_kernel void @not_block_kernel(
+; CHECK-SAME: ) !associated [[META1:![0-9]+]] {
+; CHECK-NEXT:    ret void
+;
+  ret void
+}
+
+define internal amdgpu_kernel void @associated_null() !associated !2 {
+; CHECK-LABEL: define internal amdgpu_kernel void @associated_null(
+; CHECK-SAME: ) !associated [[META2:![0-9]+]] {
+; CHECK-NEXT:    ret void
+;
+  ret void
+}
+
+define internal amdgpu_kernel void @no_metadata() {
+; CHECK-LABEL: define internal amdgpu_kernel void @no_metadata() {
+; CHECK-NEXT:    ret void
+;
+  ret void
+}
+
+!0 = !{ptr addrspace(1) @block.handle }
+!1 = !{ptr addrspace(1) @not.a.block.handle }
+!2 = !{ptr addrspace(1) null }
+
+;.
+; CHECK: [[META0]] = !{ptr addrspace(1) @block.handle}
+; CHECK: [[META1]] = !{ptr addrspace(1) @not.a.block.handle}
+; CHECK: [[META2]] = !{ptr addrspace(1) null}
+;.
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow-codegen.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow-codegen.ll
index 54b4888120e5f..5bda853b76727 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow-codegen.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-pow-codegen.ll
@@ -127,29 +127,29 @@ define double @test_pow_fast_f64__integral_y(double %x, i32 %y.i) {
 ; CHECK-NEXT:    v_writelane_b32 v43, s38, 6
 ; CHECK-NEXT:    v_writelane_b32 v43, s39, 7
 ; CHECK-NEXT:    s_addk_i32 s32, 0x800
-; CHECK-NEXT:    v_writelane_b32 v43, s40, 8
-; CHECK-NEXT:    v_writelane_b32 v43, s41, 9
-; CHECK-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; CHECK-NEXT:    v_writelane_b32 v43, s48, 8
+; CHECK-NEXT:    v_writelane_b32 v43, s49, 9
+; CHECK-NEXT:    s_mov_b64 s[48:49], s[4:5]
 ; CHECK-NEXT:    s_getpc_b64 s[4:5]
 ; CHECK-NEXT:    s_add_u32 s4, s4, _Z4log2d@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, _Z4log2d@gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
-; CHECK-NEXT:    v_writelane_b32 v43, s42, 10
+; CHECK-NEXT:    v_writelane_b32 v43, s50, 10
 ; CHECK-NEXT:    buffer_store_dword v40, off, s[0:3], s33 offset:8 ; 4-byte Folded Spill
 ; CHECK-NEXT:    buffer_store_dword v41, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
 ; CHECK-NEXT:    buffer_store_dword v42, off, s[0:3], s33 ; 4-byte Folded Spill
-; CHECK-NEXT:    v_writelane_b32 v43, s43, 11
+; CHECK-NEXT:    v_writelane_b32 v43, s51, 11
 ; CHECK-NEXT:    v_mov_b32_e32 v42, v1
-; CHECK-NEXT:    v_writelane_b32 v43, s44, 12
+; CHECK-NEXT:    v_writelane_b32 v43, s52, 12
 ; CHECK-NEXT:    v_and_b32_e32 v1, 0x7fffffff, v42
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; CHECK-NEXT:    v_writelane_b32 v43, s45, 13
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; CHECK-NEXT:    v_writelane_b32 v43, s53, 13
 ; CHECK-NEXT:    v_mov_b32_e32 v40, v31
 ; CHECK-NEXT:    v_mov_b32_e32 v41, v2
-; CHECK-NEXT:    s_mov_b32 s42, s15
-; CHECK-NEXT:    s_mov_b32 s43, s14
-; CHECK-NEXT:    s_mov_b32 s44, s13
-; CHECK-NEXT:    s_mov_b32 s45, s12
+; CHECK-NEXT:    s_mov_b32 s50, s15
+; CHECK-NEXT:    s_mov_b32 s51, s14
+; CHECK-NEXT:    s_mov_b32 s52, s13
+; CHECK-NEXT:    s_mov_b32 s53, s12
 ; CHECK-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; CHECK-NEXT:    s_mov_b64 s[36:37], s[8:9]
 ; CHECK-NEXT:    s_mov_b64 s[38:39], s[6:7]
@@ -160,15 +160,15 @@ define double @test_pow_fast_f64__integral_y(double %x, i32 %y.i) {
 ; CHECK-NEXT:    s_add_u32 s4, s4, _Z4exp2d@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, _Z4exp2d@gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[36:37]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; CHECK-NEXT:    s_mov_b32 s12, s45
-; CHECK-NEXT:    s_mov_b32 s13, s44
-; CHECK-NEXT:    s_mov_b32 s14, s43
-; CHECK-NEXT:    s_mov_b32 s15, s42
+; CHECK-NEXT:    s_mov_b32 s12, s53
+; CHECK-NEXT:    s_mov_b32 s13, s52
+; CHECK-NEXT:    s_mov_b32 s14, s51
+; CHECK-NEXT:    s_mov_b32 s15, s50
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -178,12 +178,12 @@ define double @test_pow_fast_f64__integral_y(double %x, i32 %y.i) {
 ; CHECK-NEXT:    buffer_load_dword v41, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
 ; CHECK-NEXT:    buffer_load_dword v40, off, s[0:3], s33 offset:8 ; 4-byte Folded Reload
 ; CHECK-NEXT:    v_or_b32_e32 v1, v2, v1
-; CHECK-NEXT:    v_readlane_b32 s45, v43, 13
-; CHECK-NEXT:    v_readlane_b32 s44, v43, 12
-; CHECK-NEXT:    v_readlane_b32 s43, v43, 11
-; CHECK-NEXT:    v_readlane_b32 s42, v43, 10
-; CHECK-NEXT:    v_readlane_b32 s41, v43, 9
-; CHECK-NEXT:    v_readlane_b32 s40, v43, 8
+; CHECK-NEXT:    v_readlane_b32 s53, v43, 13
+; CHECK-NEXT:    v_readlane_b32 s52, v43, 12
+; CHECK-NEXT:    v_readlane_b32 s51, v43, 11
+; CHECK-NEXT:    v_readlane_b32 s50, v43, 10
+; CHECK-NEXT:    v_readlane_b32 s49, v43, 9
+; CHECK-NEXT:    v_readlane_b32 s48, v43, 8
 ; CHECK-NEXT:    v_readlane_b32 s39, v43, 7
 ; CHECK-NEXT:    v_readlane_b32 s38, v43, 6
 ; CHECK-NEXT:    v_readlane_b32 s37, v43, 5
@@ -267,28 +267,28 @@ define double @test_powr_fast_f64(double %x, double %y) {
 ; CHECK-NEXT:    v_writelane_b32 v43, s38, 6
 ; CHECK-NEXT:    v_writelane_b32 v43, s39, 7
 ; CHECK-NEXT:    s_addk_i32 s32, 0x800
-; CHECK-NEXT:    v_writelane_b32 v43, s40, 8
-; CHECK-NEXT:    v_writelane_b32 v43, s41, 9
-; CHECK-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; CHECK-NEXT:    v_writelane_b32 v43, s48, 8
+; CHECK-NEXT:    v_writelane_b32 v43, s49, 9
+; CHECK-NEXT:    s_mov_b64 s[48:49], s[4:5]
 ; CHECK-NEXT:    s_getpc_b64 s[4:5]
 ; CHECK-NEXT:    s_add_u32 s4, s4, _Z4log2d@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, _Z4log2d@gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
-; CHECK-NEXT:    v_writelane_b32 v43, s42, 10
-; CHECK-NEXT:    v_writelane_b32 v43, s43, 11
-; CHECK-NEXT:    v_writelane_b32 v43, s44, 12
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    v_writelane_b32 v43, s50, 10
+; CHECK-NEXT:    v_writelane_b32 v43, s51, 11
+; CHECK-NEXT:    v_writelane_b32 v43, s52, 12
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    buffer_store_dword v40, off, s[0:3], s33 offset:8 ; 4-byte Folded Spill
 ; CHECK-NEXT:    buffer_store_dword v41, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
 ; CHECK-NEXT:    buffer_store_dword v42, off, s[0:3], s33 ; 4-byte Folded Spill
-; CHECK-NEXT:    v_writelane_b32 v43, s45, 13
+; CHECK-NEXT:    v_writelane_b32 v43, s53, 13
 ; CHECK-NEXT:    v_mov_b32_e32 v42, v31
 ; CHECK-NEXT:    v_mov_b32_e32 v41, v3
 ; CHECK-NEXT:    v_mov_b32_e32 v40, v2
-; CHECK-NEXT:    s_mov_b32 s42, s15
-; CHECK-NEXT:    s_mov_b32 s43, s14
-; CHECK-NEXT:    s_mov_b32 s44, s13
-; CHECK-NEXT:    s_mov_b32 s45, s12
+; CHECK-NEXT:    s_mov_b32 s50, s15
+; CHECK-NEXT:    s_mov_b32 s51, s14
+; CHECK-NEXT:    s_mov_b32 s52, s13
+; CHECK-NEXT:    s_mov_b32 s53, s12
 ; CHECK-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; CHECK-NEXT:    s_mov_b64 s[36:37], s[8:9]
 ; CHECK-NEXT:    s_mov_b64 s[38:39], s[6:7]
@@ -299,26 +299,26 @@ define double @test_powr_fast_f64(double %x, double %y) {
 ; CHECK-NEXT:    s_add_u32 s4, s4, _Z4exp2d@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, _Z4exp2d@gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[36:37]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; CHECK-NEXT:    s_mov_b32 s12, s45
-; CHECK-NEXT:    s_mov_b32 s13, s44
-; CHECK-NEXT:    s_mov_b32 s14, s43
-; CHECK-NEXT:    s_mov_b32 s15, s42
+; CHECK-NEXT:    s_mov_b32 s12, s53
+; CHECK-NEXT:    s_mov_b32 s13, s52
+; CHECK-NEXT:    s_mov_b32 s14, s51
+; CHECK-NEXT:    s_mov_b32 s15, s50
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v42
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    buffer_load_dword v42, off, s[0:3], s33 ; 4-byte Folded Reload
 ; CHECK-NEXT:    buffer_load_dword v41, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
 ; CHECK-NEXT:    buffer_load_dword v40, off, s[0:3], s33 offset:8 ; 4-byte Folded Reload
-; CHECK-NEXT:    v_readlane_b32 s45, v43, 13
-; CHECK-NEXT:    v_readlane_b32 s44, v43, 12
-; CHECK-NEXT:    v_readlane_b32 s43, v43, 11
-; CHECK-NEXT:    v_readlane_b32 s42, v43, 10
-; CHECK-NEXT:    v_readlane_b32 s41, v43, 9
-; CHECK-NEXT:    v_readlane_b32 s40, v43, 8
+; CHECK-NEXT:    v_readlane_b32 s53, v43, 13
+; CHECK-NEXT:    v_readlane_b32 s52, v43, 12
+; CHECK-NEXT:    v_readlane_b32 s51, v43, 11
+; CHECK-NEXT:    v_readlane_b32 s50, v43, 10
+; CHECK-NEXT:    v_readlane_b32 s49, v43, 9
+; CHECK-NEXT:    v_readlane_b32 s48, v43, 8
 ; CHECK-NEXT:    v_readlane_b32 s39, v43, 7
 ; CHECK-NEXT:    v_readlane_b32 s38, v43, 6
 ; CHECK-NEXT:    v_readlane_b32 s37, v43, 5
@@ -409,29 +409,29 @@ define double @test_pown_fast_f64(double %x, i32 %y) {
 ; CHECK-NEXT:    v_writelane_b32 v43, s38, 6
 ; CHECK-NEXT:    v_writelane_b32 v43, s39, 7
 ; CHECK-NEXT:    s_addk_i32 s32, 0x800
-; CHECK-NEXT:    v_writelane_b32 v43, s40, 8
-; CHECK-NEXT:    v_writelane_b32 v43, s41, 9
-; CHECK-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; CHECK-NEXT:    v_writelane_b32 v43, s48, 8
+; CHECK-NEXT:    v_writelane_b32 v43, s49, 9
+; CHECK-NEXT:    s_mov_b64 s[48:49], s[4:5]
 ; CHECK-NEXT:    s_getpc_b64 s[4:5]
 ; CHECK-NEXT:    s_add_u32 s4, s4, _Z4log2d@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, _Z4log2d@gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
-; CHECK-NEXT:    v_writelane_b32 v43, s42, 10
+; CHECK-NEXT:    v_writelane_b32 v43, s50, 10
 ; CHECK-NEXT:    buffer_store_dword v40, off, s[0:3], s33 offset:8 ; 4-byte Folded Spill
 ; CHECK-NEXT:    buffer_store_dword v41, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
 ; CHECK-NEXT:    buffer_store_dword v42, off, s[0:3], s33 ; 4-byte Folded Spill
-; CHECK-NEXT:    v_writelane_b32 v43, s43, 11
+; CHECK-NEXT:    v_writelane_b32 v43, s51, 11
 ; CHECK-NEXT:    v_mov_b32_e32 v42, v1
-; CHECK-NEXT:    v_writelane_b32 v43, s44, 12
+; CHECK-NEXT:    v_writelane_b32 v43, s52, 12
 ; CHECK-NEXT:    v_and_b32_e32 v1, 0x7fffffff, v42
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; CHECK-NEXT:    v_writelane_b32 v43, s45, 13
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; CHECK-NEXT:    v_writelane_b32 v43, s53, 13
 ; CHECK-NEXT:    v_mov_b32_e32 v40, v31
 ; CHECK-NEXT:    v_mov_b32_e32 v41, v2
-; CHECK-NEXT:    s_mov_b32 s42, s15
-; CHECK-NEXT:    s_mov_b32 s43, s14
-; CHECK-NEXT:    s_mov_b32 s44, s13
-; CHECK-NEXT:    s_mov_b32 s45, s12
+; CHECK-NEXT:    s_mov_b32 s50, s15
+; CHECK-NEXT:    s_mov_b32 s51, s14
+; CHECK-NEXT:    s_mov_b32 s52, s13
+; CHECK-NEXT:    s_mov_b32 s53, s12
 ; CHECK-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; CHECK-NEXT:    s_mov_b64 s[36:37], s[8:9]
 ; CHECK-NEXT:    s_mov_b64 s[38:39], s[6:7]
@@ -442,15 +442,15 @@ define double @test_pown_fast_f64(double %x, i32 %y) {
 ; CHECK-NEXT:    s_add_u32 s4, s4, _Z4exp2d@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, _Z4exp2d@gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[36:37]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; CHECK-NEXT:    s_mov_b32 s12, s45
-; CHECK-NEXT:    s_mov_b32 s13, s44
-; CHECK-NEXT:    s_mov_b32 s14, s43
-; CHECK-NEXT:    s_mov_b32 s15, s42
+; CHECK-NEXT:    s_mov_b32 s12, s53
+; CHECK-NEXT:    s_mov_b32 s13, s52
+; CHECK-NEXT:    s_mov_b32 s14, s51
+; CHECK-NEXT:    s_mov_b32 s15, s50
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -460,12 +460,12 @@ define double @test_pown_fast_f64(double %x, i32 %y) {
 ; CHECK-NEXT:    buffer_load_dword v41, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
 ; CHECK-NEXT:    buffer_load_dword v40, off, s[0:3], s33 offset:8 ; 4-byte Folded Reload
 ; CHECK-NEXT:    v_or_b32_e32 v1, v2, v1
-; CHECK-NEXT:    v_readlane_b32 s45, v43, 13
-; CHECK-NEXT:    v_readlane_b32 s44, v43, 12
-; CHECK-NEXT:    v_readlane_b32 s43, v43, 11
-; CHECK-NEXT:    v_readlane_b32 s42, v43, 10
-; CHECK-NEXT:    v_readlane_b32 s41, v43, 9
-; CHECK-NEXT:    v_readlane_b32 s40, v43, 8
+; CHECK-NEXT:    v_readlane_b32 s53, v43, 13
+; CHECK-NEXT:    v_readlane_b32 s52, v43, 12
+; CHECK-NEXT:    v_readlane_b32 s51, v43, 11
+; CHECK-NEXT:    v_readlane_b32 s50, v43, 10
+; CHECK-NEXT:    v_readlane_b32 s49, v43, 9
+; CHECK-NEXT:    v_readlane_b32 s48, v43, 8
 ; CHECK-NEXT:    v_readlane_b32 s39, v43, 7
 ; CHECK-NEXT:    v_readlane_b32 s38, v43, 6
 ; CHECK-NEXT:    v_readlane_b32 s37, v43, 5
@@ -551,26 +551,26 @@ define double @test_pown_fast_f64_known_even(double %x, i32 %y.arg) {
 ; CHECK-NEXT:    v_writelane_b32 v42, s38, 6
 ; CHECK-NEXT:    v_writelane_b32 v42, s39, 7
 ; CHECK-NEXT:    s_addk_i32 s32, 0x400
-; CHECK-NEXT:    v_writelane_b32 v42, s40, 8
-; CHECK-NEXT:    v_writelane_b32 v42, s41, 9
-; CHECK-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; CHECK-NEXT:    v_writelane_b32 v42, s48, 8
+; CHECK-NEXT:    v_writelane_b32 v42, s49, 9
+; CHECK-NEXT:    s_mov_b64 s[48:49], s[4:5]
 ; CHECK-NEXT:    s_getpc_b64 s[4:5]
 ; CHECK-NEXT:    s_add_u32 s4, s4, _Z4log2d@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, _Z4log2d@gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
-; CHECK-NEXT:    v_writelane_b32 v42, s42, 10
-; CHECK-NEXT:    v_writelane_b32 v42, s43, 11
-; CHECK-NEXT:    v_writelane_b32 v42, s44, 12
+; CHECK-NEXT:    v_writelane_b32 v42, s50, 10
+; CHECK-NEXT:    v_writelane_b32 v42, s51, 11
+; CHECK-NEXT:    v_writelane_b32 v42, s52, 12
 ; CHECK-NEXT:    v_and_b32_e32 v1, 0x7fffffff, v1
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    buffer_store_dword v40, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
 ; CHECK-NEXT:    buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
-; CHECK-NEXT:    v_writelane_b32 v42, s45, 13
+; CHECK-NEXT:    v_writelane_b32 v42, s53, 13
 ; CHECK-NEXT:    v_mov_b32_e32 v40, v31
-; CHECK-NEXT:    s_mov_b32 s42, s15
-; CHECK-NEXT:    s_mov_b32 s43, s14
-; CHECK-NEXT:    s_mov_b32 s44, s13
-; CHECK-NEXT:    s_mov_b32 s45, s12
+; CHECK-NEXT:    s_mov_b32 s50, s15
+; CHECK-NEXT:    s_mov_b32 s51, s14
+; CHECK-NEXT:    s_mov_b32 s52, s13
+; CHECK-NEXT:    s_mov_b32 s53, s12
 ; CHECK-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; CHECK-NEXT:    s_mov_b64 s[36:37], s[8:9]
 ; CHECK-NEXT:    s_mov_b64 s[38:39], s[6:7]
@@ -582,26 +582,26 @@ define double @test_pown_fast_f64_known_even(double %x, i32 %y.arg) {
 ; CHECK-NEXT:    s_add_u32 s4, s4, _Z4exp2d@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, _Z4exp2d@gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[36:37]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; CHECK-NEXT:    s_mov_b32 s12, s45
-; CHECK-NEXT:    s_mov_b32 s13, s44
-; CHECK-NEXT:    s_mov_b32 s14, s43
-; CHECK-NEXT:    s_mov_b32 s15, s42
+; CHECK-NEXT:    s_mov_b32 s12, s53
+; CHECK-NEXT:    s_mov_b32 s13, s52
+; CHECK-NEXT:    s_mov_b32 s14, s51
+; CHECK-NEXT:    s_mov_b32 s15, s50
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload
 ; CHECK-NEXT:    buffer_load_dword v40, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
-; CHECK-NEXT:    v_readlane_b32 s45, v42, 13
-; CHECK-NEXT:    v_readlane_b32 s44, v42, 12
-; CHECK-NEXT:    v_readlane_b32 s43, v42, 11
-; CHECK-NEXT:    v_readlane_b32 s42, v42, 10
-; CHECK-NEXT:    v_readlane_b32 s41, v42, 9
-; CHECK-NEXT:    v_readlane_b32 s40, v42, 8
+; CHECK-NEXT:    v_readlane_b32 s53, v42, 13
+; CHECK-NEXT:    v_readlane_b32 s52, v42, 12
+; CHECK-NEXT:    v_readlane_b32 s51, v42, 11
+; CHECK-NEXT:    v_readlane_b32 s50, v42, 10
+; CHECK-NEXT:    v_readlane_b32 s49, v42, 9
+; CHECK-NEXT:    v_readlane_b32 s48, v42, 8
 ; CHECK-NEXT:    v_readlane_b32 s39, v42, 7
 ; CHECK-NEXT:    v_readlane_b32 s38, v42, 6
 ; CHECK-NEXT:    v_readlane_b32 s37, v42, 5
@@ -692,28 +692,28 @@ define double @test_pown_fast_f64_known_odd(double %x, i32 %y.arg) {
 ; CHECK-NEXT:    v_writelane_b32 v43, s38, 6
 ; CHECK-NEXT:    v_writelane_b32 v43, s39, 7
 ; CHECK-NEXT:    s_addk_i32 s32, 0x800
-; CHECK-NEXT:    v_writelane_b32 v43, s40, 8
-; CHECK-NEXT:    v_writelane_b32 v43, s41, 9
-; CHECK-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; CHECK-NEXT:    v_writelane_b32 v43, s48, 8
+; CHECK-NEXT:    v_writelane_b32 v43, s49, 9
+; CHECK-NEXT:    s_mov_b64 s[48:49], s[4:5]
 ; CHECK-NEXT:    s_getpc_b64 s[4:5]
 ; CHECK-NEXT:    s_add_u32 s4, s4, _Z4log2d@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, _Z4log2d@gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
-; CHECK-NEXT:    v_writelane_b32 v43, s42, 10
+; CHECK-NEXT:    v_writelane_b32 v43, s50, 10
 ; CHECK-NEXT:    buffer_store_dword v40, off, s[0:3], s33 offset:8 ; 4-byte Folded Spill
 ; CHECK-NEXT:    buffer_store_dword v41, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
 ; CHECK-NEXT:    buffer_store_dword v42, off, s[0:3], s33 ; 4-byte Folded Spill
-; CHECK-NEXT:    v_writelane_b32 v43, s43, 11
+; CHECK-NEXT:    v_writelane_b32 v43, s51, 11
 ; CHECK-NEXT:    v_mov_b32_e32 v41, v1
-; CHECK-NEXT:    v_writelane_b32 v43, s44, 12
+; CHECK-NEXT:    v_writelane_b32 v43, s52, 12
 ; CHECK-NEXT:    v_and_b32_e32 v1, 0x7fffffff, v41
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; CHECK-NEXT:    v_writelane_b32 v43, s45, 13
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; CHECK-NEXT:    v_writelane_b32 v43, s53, 13
 ; CHECK-NEXT:    v_mov_b32_e32 v40, v31
-; CHECK-NEXT:    s_mov_b32 s42, s15
-; CHECK-NEXT:    s_mov_b32 s43, s14
-; CHECK-NEXT:    s_mov_b32 s44, s13
-; CHECK-NEXT:    s_mov_b32 s45, s12
+; CHECK-NEXT:    s_mov_b32 s50, s15
+; CHECK-NEXT:    s_mov_b32 s51, s14
+; CHECK-NEXT:    s_mov_b32 s52, s13
+; CHECK-NEXT:    s_mov_b32 s53, s12
 ; CHECK-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; CHECK-NEXT:    s_mov_b64 s[36:37], s[8:9]
 ; CHECK-NEXT:    s_mov_b64 s[38:39], s[6:7]
@@ -725,15 +725,15 @@ define double @test_pown_fast_f64_known_odd(double %x, i32 %y.arg) {
 ; CHECK-NEXT:    s_add_u32 s4, s4, _Z4exp2d@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, _Z4exp2d@gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[36:37]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; CHECK-NEXT:    s_mov_b32 s12, s45
-; CHECK-NEXT:    s_mov_b32 s13, s44
-; CHECK-NEXT:    s_mov_b32 s14, s43
-; CHECK-NEXT:    s_mov_b32 s15, s42
+; CHECK-NEXT:    s_mov_b32 s12, s53
+; CHECK-NEXT:    s_mov_b32 s13, s52
+; CHECK-NEXT:    s_mov_b32 s14, s51
+; CHECK-NEXT:    s_mov_b32 s15, s50
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -742,12 +742,12 @@ define double @test_pown_fast_f64_known_odd(double %x, i32 %y.arg) {
 ; CHECK-NEXT:    buffer_load_dword v41, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
 ; CHECK-NEXT:    buffer_load_dword v40, off, s[0:3], s33 offset:8 ; 4-byte Folded Reload
 ; CHECK-NEXT:    v_or_b32_e32 v1, v2, v1
-; CHECK-NEXT:    v_readlane_b32 s45, v43, 13
-; CHECK-NEXT:    v_readlane_b32 s44, v43, 12
-; CHECK-NEXT:    v_readlane_b32 s43, v43, 11
-; CHECK-NEXT:    v_readlane_b32 s42, v43, 10
-; CHECK-NEXT:    v_readlane_b32 s41, v43, 9
-; CHECK-NEXT:    v_readlane_b32 s40, v43, 8
+; CHECK-NEXT:    v_readlane_b32 s53, v43, 13
+; CHECK-NEXT:    v_readlane_b32 s52, v43, 12
+; CHECK-NEXT:    v_readlane_b32 s51, v43, 11
+; CHECK-NEXT:    v_readlane_b32 s50, v43, 10
+; CHECK-NEXT:    v_readlane_b32 s49, v43, 9
+; CHECK-NEXT:    v_readlane_b32 s48, v43, 8
 ; CHECK-NEXT:    v_readlane_b32 s39, v43, 7
 ; CHECK-NEXT:    v_readlane_b32 s38, v43, 6
 ; CHECK-NEXT:    v_readlane_b32 s37, v43, 5
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu.private-memory.ll b/llvm/test/CodeGen/AMDGPU/amdgpu.private-memory.ll
index 82832277b1aba..a663d451cad35 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu.private-memory.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu.private-memory.ll
@@ -258,7 +258,7 @@ entry:
 ; FUNC-LABEL: {{^}}no_overlap:
 ;
 ; A total of 5 bytes should be allocated and used.
-; SI: buffer_store_byte v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], 0 ;
+; SI-ALLOCA: buffer_store_byte v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], 0 ;
 define amdgpu_kernel void @no_overlap(ptr addrspace(1) %out, i32 %in) #0 {
 entry:
   %0 = alloca [3 x i8], align 1, addrspace(5)
@@ -281,6 +281,7 @@ entry:
   ret void
 }
 
+; FUNC-LABEL: {{^}}char_array_array:
 define amdgpu_kernel void @char_array_array(ptr addrspace(1) %out, i32 %index) #0 {
 entry:
   %alloca = alloca [2 x [2 x i8]], addrspace(5)
@@ -294,6 +295,7 @@ entry:
   ret void
 }
 
+; FUNC-LABEL: {{^}}i32_array_array:
 define amdgpu_kernel void @i32_array_array(ptr addrspace(1) %out, i32 %index) #0 {
 entry:
   %alloca = alloca [2 x [2 x i32]], addrspace(5)
@@ -306,6 +308,7 @@ entry:
   ret void
 }
 
+; FUNC-LABEL: {{^}}i64_array_array:
 define amdgpu_kernel void @i64_array_array(ptr addrspace(1) %out, i32 %index) #0 {
 entry:
   %alloca = alloca [2 x [2 x i64]], addrspace(5)
@@ -319,7 +322,7 @@ entry:
 }
 
 %struct.pair32 = type { i32, i32 }
-
+; FUNC-LABEL: {{^}}struct_array_array:
 define amdgpu_kernel void @struct_array_array(ptr addrspace(1) %out, i32 %index) #0 {
 entry:
   %alloca = alloca [2 x [2 x %struct.pair32]], addrspace(5)
@@ -333,6 +336,7 @@ entry:
   ret void
 }
 
+; FUNC-LABEL: {{^}}struct_pair32_array:
 define amdgpu_kernel void @struct_pair32_array(ptr addrspace(1) %out, i32 %index) #0 {
 entry:
   %alloca = alloca [2 x %struct.pair32], addrspace(5)
@@ -346,6 +350,7 @@ entry:
   ret void
 }
 
+; FUNC-LABEL: {{^}}select_private:
 define amdgpu_kernel void @select_private(ptr addrspace(1) %out, i32 %in) nounwind {
 entry:
   %tmp = alloca [2 x i32], addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/amdpal_scratch_mergedshader.ll b/llvm/test/CodeGen/AMDGPU/amdpal_scratch_mergedshader.ll
index 20e1d6abc7978..2ddb2fea5ddc6 100644
--- a/llvm/test/CodeGen/AMDGPU/amdpal_scratch_mergedshader.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdpal_scratch_mergedshader.ll
@@ -17,8 +17,8 @@ define amdgpu_hs void @_amdgpu_hs_main(i32 inreg %arg, i32 inreg %arg1, i32 inre
 
 .beginls:                                         ; preds = %.entry
   %tmp15 = extractelement <6 x i32> %arg8, i32 3
-  %.0.vec.insert.i = insertelement <2 x i32> undef, i32 %tmp15, i32 0
-  %.4.vec.insert.i = shufflevector <2 x i32> %.0.vec.insert.i, <2 x i32> undef, <2 x i32> <i32 0, i32 3>
+  %.0.vec.insert.i = insertelement <2 x i32> poison, i32 %tmp15, i32 0
+  %.4.vec.insert.i = shufflevector <2 x i32> %.0.vec.insert.i, <2 x i32> poison, <2 x i32> <i32 0, i32 3>
   %tmp16 = bitcast <2 x i32> %.4.vec.insert.i to i64
   br label %.endls
 
@@ -28,7 +28,7 @@ define amdgpu_hs void @_amdgpu_hs_main(i32 inreg %arg, i32 inreg %arg1, i32 inre
   br label %bb
 
 bb:                                               ; preds = %bb, %.endls
-  %lsr.iv182 = phi ptr addrspace(5) [ undef, %bb ], [ %__llpc_global_proxy_7.i, %.endls ]
+  %lsr.iv182 = phi ptr addrspace(5) [ poison, %bb ], [ %__llpc_global_proxy_7.i, %.endls ]
   %scevgep183 = getelementptr [3 x <4 x float>], ptr addrspace(5) %lsr.iv182, i32 0, i32 1
   br label %bb
 }
diff --git a/llvm/test/CodeGen/AMDGPU/any_extend_vector_inreg.ll b/llvm/test/CodeGen/AMDGPU/any_extend_vector_inreg.ll
index 07e39d798f58d..8bcef24c8e23d 100644
--- a/llvm/test/CodeGen/AMDGPU/any_extend_vector_inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/any_extend_vector_inreg.ll
@@ -36,7 +36,7 @@ bb:
   %tmp21 = getelementptr inbounds <8 x i8>, ptr addrspace(1) %arg, i64 4
   %tmp23 = load <16 x i8>, ptr addrspace(1) %tmp21, align 16
   %tmp24 = extractelement <16 x i8> %tmp23, i64 3
-  %tmp1 = insertelement <16 x i8> undef, i8 %tmp3, i32 2
+  %tmp1 = insertelement <16 x i8> poison, i8 %tmp3, i32 2
   %tmp4 = insertelement <16 x i8> %tmp1, i8 0, i32 3
   %tmp5 = insertelement <16 x i8> %tmp4, i8 0, i32 4
   %tmp7 = insertelement <16 x i8> %tmp5, i8 %tmp6, i32 5
diff --git a/llvm/test/CodeGen/AMDGPU/anyext.ll b/llvm/test/CodeGen/AMDGPU/anyext.ll
index 6e16c90033273..f0aa141e65a5c 100644
--- a/llvm/test/CodeGen/AMDGPU/anyext.ll
+++ b/llvm/test/CodeGen/AMDGPU/anyext.ll
@@ -187,7 +187,7 @@ define amdgpu_kernel void @anyext_v2i16_to_v2i32() #0 {
 ; GFX9-NEXT:    s_endpgm
 bb:
   %tmp = load i16, ptr addrspace(1) undef, align 2
-  %tmp2 = insertelement <2 x i16> undef, i16 %tmp, i32 1
+  %tmp2 = insertelement <2 x i16> poison, i16 %tmp, i32 1
   %tmp4 = and <2 x i16> %tmp2, <i16 -32767, i16 -32767>
   %tmp5 = zext <2 x i16> %tmp4 to <2 x i32>
   %tmp6 = shl nuw <2 x i32> %tmp5, <i32 16, i32 16>
diff --git a/llvm/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll b/llvm/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll
index ac8782e1e542f..e1bbc243344b0 100644
--- a/llvm/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll
@@ -22,13 +22,7 @@ declare void @llvm.amdgcn.s.barrier() #2
 ; SI-ALLOCA: s_barrier
 ; SI-ALLOCA: buffer_load_dword {{v[0-9]+}}, [[PTRREG]], s[{{[0-9]+:[0-9]+}}], 0 offen offset:64
 ;
-; FIXME: The AMDGPUPromoteAlloca pass should be able to convert this
-; alloca to a vector.  It currently fails because it does not know how
-; to interpret:
-; getelementptr inbounds [16 x i32], ptr addrspace(5) %alloca, i32 1, i32 %b
-
-; SI-PROMOTE: v_add_i32_e32 [[PTRREG:v[0-9]+]], vcc, 64
-; SI-PROMOTE: ds_write_b32 [[PTRREG]]
+; SI-PROMOTE: LDSByteSize: 0
 define amdgpu_kernel void @test_private_array_ptr_calc(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %inA, ptr addrspace(1) noalias %inB) #0 {
   %alloca = alloca [16 x i32], align 16, addrspace(5)
   %mbcnt.lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0);
diff --git a/llvm/test/CodeGen/AMDGPU/av_movimm_pseudo_expansion.mir b/llvm/test/CodeGen/AMDGPU/av_movimm_pseudo_expansion.mir
new file mode 100644
index 0000000000000..1f9d49073da2c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/av_movimm_pseudo_expansion.mir
@@ -0,0 +1,56 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=postrapseudos %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass=postrapseudos %s -o - | FileCheck %s
+
+---
+name: av_mov_b32_imm_pseudo_agpr_0
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: av_mov_b32_imm_pseudo_agpr_0
+    ; CHECK: $agpr0 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+    $agpr0 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+...
+
+---
+name: av_mov_b32_imm_pseudo_agpr_64
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: av_mov_b32_imm_pseudo_agpr_64
+    ; CHECK: $agpr0 = V_ACCVGPR_WRITE_B32_e64 64, implicit $exec
+    $agpr0 = AV_MOV_B32_IMM_PSEUDO 64, implicit $exec
+...
+
+---
+name: av_mov_b32_imm_pseudo_vgpr_0
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: av_mov_b32_imm_pseudo_vgpr_0
+    ; CHECK: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+    $vgpr0 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+...
+
+---
+name: av_mov_b32_imm_pseudo_vgpr_64
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: av_mov_b32_imm_pseudo_vgpr_64
+    ; CHECK: $vgpr0 = V_MOV_B32_e32 64, implicit $exec
+    $vgpr0 = AV_MOV_B32_IMM_PSEUDO 64, implicit $exec
+...
+
+---
+name: av_mov_b32_imm_pseudo_agpr_vgpr
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: av_mov_b32_imm_pseudo_agpr_vgpr
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $agpr1 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec
+    $agpr1 = AV_MOV_B32_IMM_PSEUDO $vgpr0, implicit $exec
+...
diff --git a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
index 0b5d47df2cc35..4c01e583713a7 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
@@ -189,7 +189,7 @@ define amdgpu_ps float @fptrunc_f32_f32_to_v2bf16(float %a, float %b) {
 entry:
   %a.cvt = fptrunc float %a to bfloat
   %b.cvt = fptrunc float %b to bfloat
-  %v2.1 = insertelement <2 x bfloat> undef, bfloat %a.cvt, i32 0
+  %v2.1 = insertelement <2 x bfloat> poison, bfloat %a.cvt, i32 0
   %v2.2 = insertelement <2 x bfloat> %v2.1, bfloat %b.cvt, i32 1
   %ret = bitcast <2 x bfloat> %v2.2 to float
   ret float %ret
@@ -226,7 +226,7 @@ entry:
   %a.cvt = fptrunc float %a.neg to bfloat
   %b.abs = call float @llvm.fabs.f32(float %b)
   %b.cvt = fptrunc float %b.abs to bfloat
-  %v2.1 = insertelement <2 x bfloat> undef, bfloat %a.cvt, i32 0
+  %v2.1 = insertelement <2 x bfloat> poison, bfloat %a.cvt, i32 0
   %v2.2 = insertelement <2 x bfloat> %v2.1, bfloat %b.cvt, i32 1
   %ret = bitcast <2 x bfloat> %v2.2 to float
   ret float %ret
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index aafdb1c8cc36f..91598496eb984 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -40671,14 +40671,6 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
 ; GCN-LABEL: v_vselect_v16bf16:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GCN-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
-; GCN-NEXT:    s_mov_b64 exec, s[4:5]
-; GCN-NEXT:    s_waitcnt expcnt(0)
-; GCN-NEXT:    v_writelane_b32 v31, s30, 0
-; GCN-NEXT:    v_writelane_b32 v31, s31, 1
-; GCN-NEXT:    v_writelane_b32 v31, s34, 2
-; GCN-NEXT:    v_writelane_b32 v31, s35, 3
 ; GCN-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
 ; GCN-NEXT:    v_and_b32_e32 v0, 1, v1
@@ -40716,21 +40708,21 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
 ; GCN-NEXT:    v_and_b32_e32 v8, 1, v14
 ; GCN-NEXT:    v_cmp_eq_u32_e64 s[28:29], 1, v7
 ; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[30:31], 1, v8
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[40:41], 1, v8
 ; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:64
 ; GCN-NEXT:    v_and_b32_e32 v9, 1, v15
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[34:35], 1, v9
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[42:43], 1, v9
 ; GCN-NEXT:    buffer_load_dword v9, off, s[0:3], s32 offset:60
 ; GCN-NEXT:    s_waitcnt vmcnt(2)
 ; GCN-NEXT:    v_mul_f32_e32 v7, 1.0, v7
 ; GCN-NEXT:    s_waitcnt vmcnt(1)
 ; GCN-NEXT:    v_mul_f32_e32 v8, 1.0, v8
-; GCN-NEXT:    v_cndmask_b32_e64 v15, v8, v7, s[34:35]
+; GCN-NEXT:    v_cndmask_b32_e64 v15, v8, v7, s[42:43]
 ; GCN-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:56
 ; GCN-NEXT:    v_mul_f32_e32 v8, 1.0, v30
 ; GCN-NEXT:    s_waitcnt vmcnt(1)
 ; GCN-NEXT:    v_mul_f32_e32 v9, 1.0, v9
-; GCN-NEXT:    v_cndmask_b32_e64 v14, v9, v8, s[30:31]
+; GCN-NEXT:    v_cndmask_b32_e64 v14, v9, v8, s[40:41]
 ; GCN-NEXT:    buffer_load_dword v8, off, s[0:3], s32 offset:52
 ; GCN-NEXT:    v_mul_f32_e32 v9, 1.0, v29
 ; GCN-NEXT:    s_waitcnt vmcnt(1)
@@ -40806,14 +40798,6 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
 ; GCN-NEXT:    v_and_b32_e32 v13, 0xffff0000, v13
 ; GCN-NEXT:    v_and_b32_e32 v14, 0xffff0000, v14
 ; GCN-NEXT:    v_and_b32_e32 v15, 0xffff0000, v15
-; GCN-NEXT:    v_readlane_b32 s35, v31, 3
-; GCN-NEXT:    v_readlane_b32 s34, v31, 2
-; GCN-NEXT:    v_readlane_b32 s31, v31, 1
-; GCN-NEXT:    v_readlane_b32 s30, v31, 0
-; GCN-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GCN-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; GCN-NEXT:    s_mov_b64 exec, s[4:5]
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: v_vselect_v16bf16:
@@ -40954,9 +40938,6 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
 ; GFX8-LABEL: v_vselect_v16bf16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v1
@@ -40982,17 +40963,13 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v11
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[24:25], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v12
-; GFX8-NEXT:    v_writelane_b32 v31, s30, 0
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[26:27], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v13
-; GFX8-NEXT:    v_writelane_b32 v31, s31, 1
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[28:29], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v14
-; GFX8-NEXT:    v_writelane_b32 v31, s34, 2
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[30:31], 1, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[40:41], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v15
-; GFX8-NEXT:    v_writelane_b32 v31, s35, 3
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[34:35], 1, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[42:43], 1, v0
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 16, v22
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v30
 ; GFX8-NEXT:    v_cndmask_b32_e64 v6, v1, v0, s[28:29]
@@ -41018,9 +40995,9 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
 ; GFX8-NEXT:    v_or_b32_sdwa v4, v9, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_sdwa v5, v8, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_cndmask_b32_e64 v10, v0, v23, s[30:31]
+; GFX8-NEXT:    v_cndmask_b32_e64 v10, v0, v23, s[40:41]
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX8-NEXT:    v_cndmask_b32_e64 v11, v0, v1, s[34:35]
+; GFX8-NEXT:    v_cndmask_b32_e64 v11, v0, v1, s[42:43]
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 16, v19
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v27
 ; GFX8-NEXT:    v_cndmask_b32_e64 v3, v1, v0, s[16:17]
@@ -41043,14 +41020,6 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
 ; GFX8-NEXT:    v_or_b32_sdwa v2, v13, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_sdwa v3, v12, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_sdwa v7, v10, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT:    v_readlane_b32 s35, v31, 3
-; GFX8-NEXT:    v_readlane_b32 s34, v31, 2
-; GFX8-NEXT:    v_readlane_b32 s31, v31, 1
-; GFX8-NEXT:    v_readlane_b32 s30, v31, 0
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_vselect_v16bf16:
@@ -42030,108 +41999,80 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
 ; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
 ; GFX8-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
 ; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX8-NEXT:    v_writelane_b32 v34, s30, 0
-; GFX8-NEXT:    v_writelane_b32 v34, s31, 1
-; GFX8-NEXT:    v_writelane_b32 v34, s34, 2
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX8-NEXT:    v_writelane_b32 v34, s35, 3
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v1
-; GFX8-NEXT:    v_writelane_b32 v34, s36, 4
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v2
-; GFX8-NEXT:    v_writelane_b32 v34, s37, 5
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[6:7], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v3
-; GFX8-NEXT:    v_writelane_b32 v34, s38, 6
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[8:9], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v4
-; GFX8-NEXT:    v_writelane_b32 v34, s39, 7
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[10:11], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v5
-; GFX8-NEXT:    v_writelane_b32 v34, s40, 8
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[12:13], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v6
-; GFX8-NEXT:    v_writelane_b32 v34, s41, 9
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[14:15], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v7
-; GFX8-NEXT:    v_writelane_b32 v34, s42, 10
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[16:17], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v8
-; GFX8-NEXT:    v_writelane_b32 v34, s43, 11
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[18:19], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v9
-; GFX8-NEXT:    v_writelane_b32 v34, s44, 12
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[20:21], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v10
-; GFX8-NEXT:    v_writelane_b32 v34, s45, 13
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[22:23], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v11
-; GFX8-NEXT:    v_writelane_b32 v34, s46, 14
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[24:25], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v12
-; GFX8-NEXT:    v_writelane_b32 v34, s47, 15
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[26:27], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v13
-; GFX8-NEXT:    v_writelane_b32 v34, s48, 16
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[28:29], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v14
-; GFX8-NEXT:    v_writelane_b32 v34, s49, 17
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[30:31], 1, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[40:41], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v15
-; GFX8-NEXT:    v_writelane_b32 v34, s50, 18
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[34:35], 1, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[42:43], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v16
-; GFX8-NEXT:    v_writelane_b32 v34, s51, 19
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[36:37], 1, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[44:45], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v17
-; GFX8-NEXT:    v_writelane_b32 v34, s52, 20
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[38:39], 1, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[46:47], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v18
-; GFX8-NEXT:    v_writelane_b32 v34, s53, 21
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[40:41], 1, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[56:57], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v19
-; GFX8-NEXT:    v_writelane_b32 v34, s54, 22
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[42:43], 1, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[58:59], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v20
-; GFX8-NEXT:    v_writelane_b32 v34, s55, 23
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[44:45], 1, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[60:61], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v21
-; GFX8-NEXT:    v_writelane_b32 v34, s56, 24
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[46:47], 1, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[62:63], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v22
-; GFX8-NEXT:    v_writelane_b32 v34, s57, 25
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[48:49], 1, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[72:73], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v23
-; GFX8-NEXT:    v_writelane_b32 v34, s58, 26
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[50:51], 1, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[74:75], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v24
-; GFX8-NEXT:    v_writelane_b32 v34, s59, 27
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[52:53], 1, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[76:77], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v25
-; GFX8-NEXT:    v_writelane_b32 v34, s60, 28
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[54:55], 1, v0
+; GFX8-NEXT:    v_writelane_b32 v34, s30, 0
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[78:79], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v26
-; GFX8-NEXT:    v_writelane_b32 v34, s61, 29
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[56:57], 1, v0
+; GFX8-NEXT:    v_writelane_b32 v34, s31, 1
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[88:89], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v27
-; GFX8-NEXT:    v_writelane_b32 v34, s62, 30
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[58:59], 1, v0
+; GFX8-NEXT:    v_writelane_b32 v34, s34, 2
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[90:91], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v28
-; GFX8-NEXT:    v_writelane_b32 v34, s63, 31
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[60:61], 1, v0
+; GFX8-NEXT:    v_writelane_b32 v34, s35, 3
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[30:31], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v29
-; GFX8-NEXT:    v_writelane_b32 v34, s64, 32
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[62:63], 1, v0
+; GFX8-NEXT:    v_writelane_b32 v34, s36, 4
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[34:35], 1, v0
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v30
-; GFX8-NEXT:    v_writelane_b32 v34, s65, 33
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[64:65], 1, v0
+; GFX8-NEXT:    v_writelane_b32 v34, s37, 5
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[36:37], 1, v0
 ; GFX8-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
-; GFX8-NEXT:    v_writelane_b32 v34, s66, 34
-; GFX8-NEXT:    v_writelane_b32 v34, s67, 35
+; GFX8-NEXT:    v_writelane_b32 v34, s38, 6
+; GFX8-NEXT:    v_writelane_b32 v34, s39, 7
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX8-NEXT:    v_cmp_eq_u32_e64 s[66:67], 1, v0
+; GFX8-NEXT:    v_cmp_eq_u32_e64 s[38:39], 1, v0
 ; GFX8-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:68
 ; GFX8-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:4
 ; GFX8-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:72
@@ -42168,40 +42109,40 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v33, 16, v29
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v28, 16, v32
-; GFX8-NEXT:    v_cndmask_b32_e64 v28, v33, v28, s[66:67]
-; GFX8-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[64:65]
+; GFX8-NEXT:    v_cndmask_b32_e64 v28, v33, v28, s[38:39]
+; GFX8-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[36:37]
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v32, 16, v31
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v33, 16, v30
-; GFX8-NEXT:    v_cndmask_b32_e64 v32, v33, v32, s[62:63]
-; GFX8-NEXT:    v_cndmask_b32_e64 v30, v30, v31, s[60:61]
+; GFX8-NEXT:    v_cndmask_b32_e64 v32, v33, v32, s[34:35]
+; GFX8-NEXT:    v_cndmask_b32_e64 v30, v30, v31, s[30:31]
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v31, 16, v27
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v33, 16, v26
-; GFX8-NEXT:    v_cndmask_b32_e64 v31, v33, v31, s[58:59]
-; GFX8-NEXT:    v_cndmask_b32_e64 v26, v26, v27, s[56:57]
+; GFX8-NEXT:    v_cndmask_b32_e64 v31, v33, v31, s[90:91]
+; GFX8-NEXT:    v_cndmask_b32_e64 v26, v26, v27, s[88:89]
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v27, 16, v25
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v33, 16, v24
-; GFX8-NEXT:    v_cndmask_b32_e64 v27, v33, v27, s[54:55]
-; GFX8-NEXT:    v_cndmask_b32_e64 v24, v24, v25, s[52:53]
+; GFX8-NEXT:    v_cndmask_b32_e64 v27, v33, v27, s[78:79]
+; GFX8-NEXT:    v_cndmask_b32_e64 v24, v24, v25, s[76:77]
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v25, 16, v23
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v33, 16, v22
-; GFX8-NEXT:    v_cndmask_b32_e64 v25, v33, v25, s[50:51]
-; GFX8-NEXT:    v_cndmask_b32_e64 v22, v22, v23, s[48:49]
+; GFX8-NEXT:    v_cndmask_b32_e64 v25, v33, v25, s[74:75]
+; GFX8-NEXT:    v_cndmask_b32_e64 v22, v22, v23, s[72:73]
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v23, 16, v21
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v33, 16, v20
-; GFX8-NEXT:    v_cndmask_b32_e64 v23, v33, v23, s[46:47]
-; GFX8-NEXT:    v_cndmask_b32_e64 v20, v20, v21, s[44:45]
+; GFX8-NEXT:    v_cndmask_b32_e64 v23, v33, v23, s[62:63]
+; GFX8-NEXT:    v_cndmask_b32_e64 v20, v20, v21, s[60:61]
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v21, 16, v19
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v33, 16, v18
-; GFX8-NEXT:    v_cndmask_b32_e64 v21, v33, v21, s[42:43]
-; GFX8-NEXT:    v_cndmask_b32_e64 v18, v18, v19, s[40:41]
+; GFX8-NEXT:    v_cndmask_b32_e64 v21, v33, v21, s[58:59]
+; GFX8-NEXT:    v_cndmask_b32_e64 v18, v18, v19, s[56:57]
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v19, 16, v17
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v33, 16, v16
-; GFX8-NEXT:    v_cndmask_b32_e64 v19, v33, v19, s[38:39]
-; GFX8-NEXT:    v_cndmask_b32_e64 v16, v16, v17, s[36:37]
+; GFX8-NEXT:    v_cndmask_b32_e64 v19, v33, v19, s[46:47]
+; GFX8-NEXT:    v_cndmask_b32_e64 v16, v16, v17, s[44:45]
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v17, 16, v15
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v33, 16, v14
-; GFX8-NEXT:    v_cndmask_b32_e64 v17, v33, v17, s[34:35]
-; GFX8-NEXT:    v_cndmask_b32_e64 v14, v14, v15, s[30:31]
+; GFX8-NEXT:    v_cndmask_b32_e64 v17, v33, v17, s[42:43]
+; GFX8-NEXT:    v_cndmask_b32_e64 v14, v14, v15, s[40:41]
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v15, 16, v13
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v33, 16, v12
 ; GFX8-NEXT:    v_cndmask_b32_e64 v15, v33, v15, s[28:29]
@@ -42262,34 +42203,6 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
 ; GFX8-NEXT:    v_or_b32_sdwa v13, v26, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_sdwa v14, v30, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_sdwa v15, v29, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT:    v_readlane_b32 s67, v34, 35
-; GFX8-NEXT:    v_readlane_b32 s66, v34, 34
-; GFX8-NEXT:    v_readlane_b32 s65, v34, 33
-; GFX8-NEXT:    v_readlane_b32 s64, v34, 32
-; GFX8-NEXT:    v_readlane_b32 s63, v34, 31
-; GFX8-NEXT:    v_readlane_b32 s62, v34, 30
-; GFX8-NEXT:    v_readlane_b32 s61, v34, 29
-; GFX8-NEXT:    v_readlane_b32 s60, v34, 28
-; GFX8-NEXT:    v_readlane_b32 s59, v34, 27
-; GFX8-NEXT:    v_readlane_b32 s58, v34, 26
-; GFX8-NEXT:    v_readlane_b32 s57, v34, 25
-; GFX8-NEXT:    v_readlane_b32 s56, v34, 24
-; GFX8-NEXT:    v_readlane_b32 s55, v34, 23
-; GFX8-NEXT:    v_readlane_b32 s54, v34, 22
-; GFX8-NEXT:    v_readlane_b32 s53, v34, 21
-; GFX8-NEXT:    v_readlane_b32 s52, v34, 20
-; GFX8-NEXT:    v_readlane_b32 s51, v34, 19
-; GFX8-NEXT:    v_readlane_b32 s50, v34, 18
-; GFX8-NEXT:    v_readlane_b32 s49, v34, 17
-; GFX8-NEXT:    v_readlane_b32 s48, v34, 16
-; GFX8-NEXT:    v_readlane_b32 s47, v34, 15
-; GFX8-NEXT:    v_readlane_b32 s46, v34, 14
-; GFX8-NEXT:    v_readlane_b32 s45, v34, 13
-; GFX8-NEXT:    v_readlane_b32 s44, v34, 12
-; GFX8-NEXT:    v_readlane_b32 s43, v34, 11
-; GFX8-NEXT:    v_readlane_b32 s42, v34, 10
-; GFX8-NEXT:    v_readlane_b32 s41, v34, 9
-; GFX8-NEXT:    v_readlane_b32 s40, v34, 8
 ; GFX8-NEXT:    v_readlane_b32 s39, v34, 7
 ; GFX8-NEXT:    v_readlane_b32 s38, v34, 6
 ; GFX8-NEXT:    v_readlane_b32 s37, v34, 5
@@ -42310,108 +42223,76 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
 ; GFX9-NEXT:    s_xor_saveexec_b64 s[4:5], -1
 ; GFX9-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX9-NEXT:    v_writelane_b32 v33, s30, 0
-; GFX9-NEXT:    v_writelane_b32 v33, s31, 1
-; GFX9-NEXT:    v_writelane_b32 v33, s34, 2
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_writelane_b32 v33, s35, 3
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v3
-; GFX9-NEXT:    v_writelane_b32 v33, s36, 4
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[6:7], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v2
-; GFX9-NEXT:    v_writelane_b32 v33, s37, 5
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[8:9], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v5
-; GFX9-NEXT:    v_writelane_b32 v33, s38, 6
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[10:11], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v4
-; GFX9-NEXT:    v_writelane_b32 v33, s39, 7
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[12:13], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v7
-; GFX9-NEXT:    v_writelane_b32 v33, s40, 8
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[14:15], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v6
-; GFX9-NEXT:    v_writelane_b32 v33, s41, 9
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[16:17], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v9
-; GFX9-NEXT:    v_writelane_b32 v33, s42, 10
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[18:19], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v8
-; GFX9-NEXT:    v_writelane_b32 v33, s43, 11
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[20:21], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v11
-; GFX9-NEXT:    v_writelane_b32 v33, s44, 12
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[22:23], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v10
-; GFX9-NEXT:    v_writelane_b32 v33, s45, 13
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[24:25], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v13
-; GFX9-NEXT:    v_writelane_b32 v33, s46, 14
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[26:27], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v12
-; GFX9-NEXT:    v_writelane_b32 v33, s47, 15
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[28:29], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v15
-; GFX9-NEXT:    v_writelane_b32 v33, s48, 16
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[30:31], 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[40:41], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v14
-; GFX9-NEXT:    v_writelane_b32 v33, s49, 17
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[34:35], 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[42:43], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v17
-; GFX9-NEXT:    v_writelane_b32 v33, s50, 18
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[36:37], 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[44:45], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v16
-; GFX9-NEXT:    v_writelane_b32 v33, s51, 19
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[38:39], 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[46:47], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v19
-; GFX9-NEXT:    v_writelane_b32 v33, s52, 20
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[40:41], 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[56:57], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v18
-; GFX9-NEXT:    v_writelane_b32 v33, s53, 21
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[42:43], 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[58:59], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v21
-; GFX9-NEXT:    v_writelane_b32 v33, s54, 22
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[44:45], 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[60:61], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v20
-; GFX9-NEXT:    v_writelane_b32 v33, s55, 23
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[46:47], 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[62:63], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v23
-; GFX9-NEXT:    v_writelane_b32 v33, s56, 24
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[48:49], 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[72:73], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v22
-; GFX9-NEXT:    v_writelane_b32 v33, s57, 25
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[50:51], 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[74:75], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v25
-; GFX9-NEXT:    v_writelane_b32 v33, s58, 26
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[52:53], 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[76:77], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v24
-; GFX9-NEXT:    v_writelane_b32 v33, s59, 27
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[54:55], 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[78:79], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v27
-; GFX9-NEXT:    v_writelane_b32 v33, s60, 28
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[56:57], 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[88:89], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v26
-; GFX9-NEXT:    v_writelane_b32 v33, s61, 29
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[58:59], 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[90:91], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v29
-; GFX9-NEXT:    v_writelane_b32 v33, s62, 30
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[60:61], 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[92:93], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v28
-; GFX9-NEXT:    v_writelane_b32 v33, s63, 31
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[62:63], 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[94:95], 1, v0
 ; GFX9-NEXT:    buffer_load_ushort v0, off, s[0:3], s32
-; GFX9-NEXT:    v_writelane_b32 v33, s64, 32
-; GFX9-NEXT:    v_writelane_b32 v33, s65, 33
-; GFX9-NEXT:    v_writelane_b32 v33, s66, 34
+; GFX9-NEXT:    v_writelane_b32 v33, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v33, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v33, s34, 2
 ; GFX9-NEXT:    v_and_b32_e32 v1, 1, v1
-; GFX9-NEXT:    v_writelane_b32 v33, s67, 35
+; GFX9-NEXT:    v_writelane_b32 v33, s35, 3
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[64:65], 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[30:31], 1, v0
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v30
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[66:67], 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[34:35], 1, v0
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:68
 ; GFX9-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:4
 ; GFX9-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:72
@@ -42445,42 +42326,42 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
 ; GFX9-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:128
 ; GFX9-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:64
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_cndmask_b32_e64 v29, v31, v32, s[66:67]
+; GFX9-NEXT:    v_cndmask_b32_e64 v29, v31, v32, s[34:35]
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v32, 16, v32
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v31, 16, v31
-; GFX9-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[64:65]
-; GFX9-NEXT:    v_cndmask_b32_e64 v32, v28, v30, s[62:63]
+; GFX9-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[30:31]
+; GFX9-NEXT:    v_cndmask_b32_e64 v32, v28, v30, s[94:95]
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v30, 16, v30
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v28, 16, v28
-; GFX9-NEXT:    v_cndmask_b32_e64 v28, v28, v30, s[60:61]
-; GFX9-NEXT:    v_cndmask_b32_e64 v30, v26, v27, s[58:59]
+; GFX9-NEXT:    v_cndmask_b32_e64 v28, v28, v30, s[92:93]
+; GFX9-NEXT:    v_cndmask_b32_e64 v30, v26, v27, s[90:91]
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v27, 16, v27
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v26, 16, v26
-; GFX9-NEXT:    v_cndmask_b32_e64 v26, v26, v27, s[56:57]
-; GFX9-NEXT:    v_cndmask_b32_e64 v27, v24, v25, s[54:55]
+; GFX9-NEXT:    v_cndmask_b32_e64 v26, v26, v27, s[88:89]
+; GFX9-NEXT:    v_cndmask_b32_e64 v27, v24, v25, s[78:79]
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v25, 16, v25
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v24, 16, v24
-; GFX9-NEXT:    v_cndmask_b32_e64 v24, v24, v25, s[52:53]
-; GFX9-NEXT:    v_cndmask_b32_e64 v25, v22, v23, s[50:51]
+; GFX9-NEXT:    v_cndmask_b32_e64 v24, v24, v25, s[76:77]
+; GFX9-NEXT:    v_cndmask_b32_e64 v25, v22, v23, s[74:75]
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v23, 16, v23
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v22, 16, v22
-; GFX9-NEXT:    v_cndmask_b32_e64 v22, v22, v23, s[48:49]
-; GFX9-NEXT:    v_cndmask_b32_e64 v23, v20, v21, s[46:47]
+; GFX9-NEXT:    v_cndmask_b32_e64 v22, v22, v23, s[72:73]
+; GFX9-NEXT:    v_cndmask_b32_e64 v23, v20, v21, s[62:63]
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v21, 16, v21
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v20, 16, v20
-; GFX9-NEXT:    v_cndmask_b32_e64 v20, v20, v21, s[44:45]
-; GFX9-NEXT:    v_cndmask_b32_e64 v21, v18, v19, s[42:43]
+; GFX9-NEXT:    v_cndmask_b32_e64 v20, v20, v21, s[60:61]
+; GFX9-NEXT:    v_cndmask_b32_e64 v21, v18, v19, s[58:59]
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v19, 16, v19
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v18, 16, v18
-; GFX9-NEXT:    v_cndmask_b32_e64 v18, v18, v19, s[40:41]
-; GFX9-NEXT:    v_cndmask_b32_e64 v19, v16, v17, s[38:39]
+; GFX9-NEXT:    v_cndmask_b32_e64 v18, v18, v19, s[56:57]
+; GFX9-NEXT:    v_cndmask_b32_e64 v19, v16, v17, s[46:47]
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v17, 16, v17
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v16, 16, v16
-; GFX9-NEXT:    v_cndmask_b32_e64 v16, v16, v17, s[36:37]
-; GFX9-NEXT:    v_cndmask_b32_e64 v17, v14, v15, s[34:35]
+; GFX9-NEXT:    v_cndmask_b32_e64 v16, v16, v17, s[44:45]
+; GFX9-NEXT:    v_cndmask_b32_e64 v17, v14, v15, s[42:43]
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 16, v14
-; GFX9-NEXT:    v_cndmask_b32_e64 v14, v14, v15, s[30:31]
+; GFX9-NEXT:    v_cndmask_b32_e64 v14, v14, v15, s[40:41]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v15, v12, v13, s[28:29]
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v13, 16, v13
 ; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
@@ -42526,38 +42407,6 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
 ; GFX9-NEXT:    v_perm_b32 v13, v26, v30, s4
 ; GFX9-NEXT:    v_perm_b32 v14, v28, v32, s4
 ; GFX9-NEXT:    v_perm_b32 v15, v31, v29, s4
-; GFX9-NEXT:    v_readlane_b32 s67, v33, 35
-; GFX9-NEXT:    v_readlane_b32 s66, v33, 34
-; GFX9-NEXT:    v_readlane_b32 s65, v33, 33
-; GFX9-NEXT:    v_readlane_b32 s64, v33, 32
-; GFX9-NEXT:    v_readlane_b32 s63, v33, 31
-; GFX9-NEXT:    v_readlane_b32 s62, v33, 30
-; GFX9-NEXT:    v_readlane_b32 s61, v33, 29
-; GFX9-NEXT:    v_readlane_b32 s60, v33, 28
-; GFX9-NEXT:    v_readlane_b32 s59, v33, 27
-; GFX9-NEXT:    v_readlane_b32 s58, v33, 26
-; GFX9-NEXT:    v_readlane_b32 s57, v33, 25
-; GFX9-NEXT:    v_readlane_b32 s56, v33, 24
-; GFX9-NEXT:    v_readlane_b32 s55, v33, 23
-; GFX9-NEXT:    v_readlane_b32 s54, v33, 22
-; GFX9-NEXT:    v_readlane_b32 s53, v33, 21
-; GFX9-NEXT:    v_readlane_b32 s52, v33, 20
-; GFX9-NEXT:    v_readlane_b32 s51, v33, 19
-; GFX9-NEXT:    v_readlane_b32 s50, v33, 18
-; GFX9-NEXT:    v_readlane_b32 s49, v33, 17
-; GFX9-NEXT:    v_readlane_b32 s48, v33, 16
-; GFX9-NEXT:    v_readlane_b32 s47, v33, 15
-; GFX9-NEXT:    v_readlane_b32 s46, v33, 14
-; GFX9-NEXT:    v_readlane_b32 s45, v33, 13
-; GFX9-NEXT:    v_readlane_b32 s44, v33, 12
-; GFX9-NEXT:    v_readlane_b32 s43, v33, 11
-; GFX9-NEXT:    v_readlane_b32 s42, v33, 10
-; GFX9-NEXT:    v_readlane_b32 s41, v33, 9
-; GFX9-NEXT:    v_readlane_b32 s40, v33, 8
-; GFX9-NEXT:    v_readlane_b32 s39, v33, 7
-; GFX9-NEXT:    v_readlane_b32 s38, v33, 6
-; GFX9-NEXT:    v_readlane_b32 s37, v33, 5
-; GFX9-NEXT:    v_readlane_b32 s36, v33, 4
 ; GFX9-NEXT:    v_readlane_b32 s35, v33, 3
 ; GFX9-NEXT:    v_readlane_b32 s34, v33, 2
 ; GFX9-NEXT:    v_readlane_b32 s31, v33, 1
diff --git a/llvm/test/CodeGen/AMDGPU/bfi_nested.ll b/llvm/test/CodeGen/AMDGPU/bfi_nested.ll
index 4b38215ebc597..3d52c158a6017 100644
--- a/llvm/test/CodeGen/AMDGPU/bfi_nested.ll
+++ b/llvm/test/CodeGen/AMDGPU/bfi_nested.ll
@@ -297,7 +297,7 @@ define amdgpu_kernel void @v_bfi_dont_applied_for_scalar_ops(ptr addrspace(1) %o
 ; GCN-NEXT:    s_endpgm
   %shift = lshr i32 %b, 16
   %tr = trunc i32 %shift to i16
-  %tmp = insertelement <2 x i16> undef, i16 %a, i32 0
+  %tmp = insertelement <2 x i16> poison, i16 %a, i32 0
   %vec = insertelement <2 x i16> %tmp, i16 %tr, i32 1
   %val = bitcast <2 x i16> %vec to i32
   store i32 %val, ptr addrspace(1) %out, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/big_alu.ll b/llvm/test/CodeGen/AMDGPU/big_alu.ll
index 0daa14a63f21a..b3b6259838341 100644
--- a/llvm/test/CodeGen/AMDGPU/big_alu.ll
+++ b/llvm/test/CodeGen/AMDGPU/big_alu.ll
@@ -90,11 +90,11 @@ main_body:
   br i1 %tmp81, label %IF137, label %ENDIF136
 
 IF137:                                            ; preds = %main_body
-  %tmp82 = insertelement <4 x float> undef, float %tmp30, i32 0
+  %tmp82 = insertelement <4 x float> poison, float %tmp30, i32 0
   %tmp83 = insertelement <4 x float> %tmp82, float %tmp31, i32 1
   %tmp84 = insertelement <4 x float> %tmp83, float %tmp32, i32 2
   %tmp85 = insertelement <4 x float> %tmp84, float 0.000000e+00, i32 3
-  %tmp86 = insertelement <4 x float> undef, float %tmp30, i32 0
+  %tmp86 = insertelement <4 x float> poison, float %tmp30, i32 0
   %tmp87 = insertelement <4 x float> %tmp86, float %tmp31, i32 1
   %tmp88 = insertelement <4 x float> %tmp87, float %tmp32, i32 2
   %tmp89 = insertelement <4 x float> %tmp88, float 0.000000e+00, i32 3
@@ -103,20 +103,20 @@ IF137:                                            ; preds = %main_body
   %tmp92 = fmul float %tmp30, %tmp91
   %tmp93 = fmul float %tmp31, %tmp91
   %tmp94 = fmul float %tmp32, %tmp91
-  %tmp95 = insertelement <4 x float> undef, float %tmp92, i32 0
+  %tmp95 = insertelement <4 x float> poison, float %tmp92, i32 0
   %tmp96 = insertelement <4 x float> %tmp95, float %tmp93, i32 1
   %tmp97 = insertelement <4 x float> %tmp96, float %tmp94, i32 2
   %tmp98 = insertelement <4 x float> %tmp97, float 0.000000e+00, i32 3
-  %tmp99 = insertelement <4 x float> undef, float %tmp37, i32 0
+  %tmp99 = insertelement <4 x float> poison, float %tmp37, i32 0
   %tmp100 = insertelement <4 x float> %tmp99, float %tmp38, i32 1
   %tmp101 = insertelement <4 x float> %tmp100, float %tmp39, i32 2
   %tmp102 = insertelement <4 x float> %tmp101, float 0.000000e+00, i32 3
   %tmp103 = call float @llvm.r600.dot4(<4 x float> %tmp98, <4 x float> %tmp102)
-  %tmp104 = insertelement <4 x float> undef, float %tmp92, i32 0
+  %tmp104 = insertelement <4 x float> poison, float %tmp92, i32 0
   %tmp105 = insertelement <4 x float> %tmp104, float %tmp93, i32 1
   %tmp106 = insertelement <4 x float> %tmp105, float %tmp94, i32 2
   %tmp107 = insertelement <4 x float> %tmp106, float 0.000000e+00, i32 3
-  %tmp108 = insertelement <4 x float> undef, float %tmp40, i32 0
+  %tmp108 = insertelement <4 x float> poison, float %tmp40, i32 0
   %tmp109 = insertelement <4 x float> %tmp108, float %tmp41, i32 1
   %tmp110 = insertelement <4 x float> %tmp109, float %tmp42, i32 2
   %tmp111 = insertelement <4 x float> %tmp110, float 0.000000e+00, i32 3
@@ -124,11 +124,11 @@ IF137:                                            ; preds = %main_body
   %tmp113 = fsub float -0.000000e+00, %tmp92
   %tmp114 = fsub float -0.000000e+00, %tmp93
   %tmp115 = fsub float -0.000000e+00, %tmp94
-  %tmp116 = insertelement <4 x float> undef, float %tmp34, i32 0
+  %tmp116 = insertelement <4 x float> poison, float %tmp34, i32 0
   %tmp117 = insertelement <4 x float> %tmp116, float %tmp35, i32 1
   %tmp118 = insertelement <4 x float> %tmp117, float %tmp36, i32 2
   %tmp119 = insertelement <4 x float> %tmp118, float 0.000000e+00, i32 3
-  %tmp120 = insertelement <4 x float> undef, float %tmp113, i32 0
+  %tmp120 = insertelement <4 x float> poison, float %tmp113, i32 0
   %tmp121 = insertelement <4 x float> %tmp120, float %tmp114, i32 1
   %tmp122 = insertelement <4 x float> %tmp121, float %tmp115, i32 2
   %tmp123 = insertelement <4 x float> %tmp122, float 0.000000e+00, i32 3
@@ -155,7 +155,7 @@ ENDIF136:                                         ; preds = %ENDIF154, %main_bod
   %tmp138 = fmul float %tmp26, 0x3F847AE140000000
   %tmp139 = fmul float %tmp27, 0x3F847AE140000000
   %tmp140 = fmul float %tmp28, 0x3F847AE140000000
-  %tmp141 = insertelement <4 x float> undef, float %tmp138, i32 0
+  %tmp141 = insertelement <4 x float> poison, float %tmp138, i32 0
   %tmp142 = insertelement <4 x float> %tmp141, float %tmp139, i32 1
   %tmp143 = insertelement <4 x float> %tmp142, float %tmp140, i32 2
   %tmp144 = insertelement <4 x float> %tmp143, float 0.000000e+00, i32 3
@@ -163,7 +163,7 @@ ENDIF136:                                         ; preds = %ENDIF154, %main_bod
   %tmp146 = extractelement <4 x float> %tmp144, i32 1
   %tmp147 = extractelement <4 x float> %tmp144, i32 2
   %tmp148 = extractelement <4 x float> %tmp144, i32 3
-  %tmp149 = insertelement <4 x float> undef, float %tmp145, i32 0
+  %tmp149 = insertelement <4 x float> poison, float %tmp145, i32 0
   %tmp150 = insertelement <4 x float> %tmp149, float %tmp146, i32 1
   %tmp151 = insertelement <4 x float> %tmp150, float %tmp147, i32 2
   %tmp152 = insertelement <4 x float> %tmp151, float %tmp148, i32 3
@@ -176,7 +176,7 @@ ENDIF136:                                         ; preds = %ENDIF154, %main_bod
   %tmp159 = fmul float %tmp26, 0x3F45A07B40000000
   %tmp160 = fmul float %tmp27, 0x3F45A07B40000000
   %tmp161 = fmul float %tmp28, 0x3F45A07B40000000
-  %tmp162 = insertelement <4 x float> undef, float %tmp159, i32 0
+  %tmp162 = insertelement <4 x float> poison, float %tmp159, i32 0
   %tmp163 = insertelement <4 x float> %tmp162, float %tmp160, i32 1
   %tmp164 = insertelement <4 x float> %tmp163, float %tmp161, i32 2
   %tmp165 = insertelement <4 x float> %tmp164, float 0.000000e+00, i32 3
@@ -184,7 +184,7 @@ ENDIF136:                                         ; preds = %ENDIF154, %main_bod
   %tmp167 = extractelement <4 x float> %tmp165, i32 1
   %tmp168 = extractelement <4 x float> %tmp165, i32 2
   %tmp169 = extractelement <4 x float> %tmp165, i32 3
-  %tmp170 = insertelement <4 x float> undef, float %tmp166, i32 0
+  %tmp170 = insertelement <4 x float> poison, float %tmp166, i32 0
   %tmp171 = insertelement <4 x float> %tmp170, float %tmp167, i32 1
   %tmp172 = insertelement <4 x float> %tmp171, float %tmp168, i32 2
   %tmp173 = insertelement <4 x float> %tmp172, float %tmp169, i32 3
@@ -251,28 +251,28 @@ ENDIF136:                                         ; preds = %ENDIF154, %main_bod
   %tmp227 = fmul float %clamp.i14, %tmp226
   %tmp228 = fmul float %tmp26, 0x3F368B5CC0000000
   %tmp229 = fmul float %tmp27, 0x3F368B5CC0000000
-  %tmp230 = insertelement <4 x float> undef, float %tmp228, i32 0
+  %tmp230 = insertelement <4 x float> poison, float %tmp228, i32 0
   %tmp231 = insertelement <4 x float> %tmp230, float %tmp229, i32 1
   %tmp232 = insertelement <4 x float> %tmp231, float 0.000000e+00, i32 2
   %tmp233 = insertelement <4 x float> %tmp232, float 0.000000e+00, i32 3
   %tmp234 = extractelement <4 x float> %tmp233, i32 0
   %tmp235 = extractelement <4 x float> %tmp233, i32 1
-  %tmp236 = insertelement <4 x float> undef, float %tmp234, i32 0
+  %tmp236 = insertelement <4 x float> poison, float %tmp234, i32 0
   %tmp237 = insertelement <4 x float> %tmp236, float %tmp235, i32 1
-  %tmp238 = insertelement <4 x float> %tmp237, float undef, i32 2
-  %tmp239 = insertelement <4 x float> %tmp238, float undef, i32 3
+  %tmp238 = insertelement <4 x float> %tmp237, float poison, i32 2
+  %tmp239 = insertelement <4 x float> %tmp238, float poison, i32 3
   %tmp240 = shufflevector <4 x float> %tmp239, <4 x float> %tmp239, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %tmp241 = call <4 x float> @llvm.r600.tex(<4 x float> %tmp240, i32 0, i32 0, i32 0, i32 17, i32 1, i32 1, i32 1, i32 1, i32 1)
   %tmp242 = extractelement <4 x float> %tmp241, i32 0
-  %tmp243 = insertelement <4 x float> undef, float %tmp242, i32 0
+  %tmp243 = insertelement <4 x float> poison, float %tmp242, i32 0
   %tmp244 = insertelement <4 x float> %tmp243, float %tmp229, i32 1
   %tmp245 = insertelement <4 x float> %tmp244, float 0.000000e+00, i32 2
   %tmp246 = insertelement <4 x float> %tmp245, float 0.000000e+00, i32 3
   %tmp247 = extractelement <4 x float> %tmp246, i32 0
-  %tmp248 = insertelement <4 x float> undef, float %tmp247, i32 0
-  %tmp249 = insertelement <4 x float> %tmp248, float undef, i32 1
-  %tmp250 = insertelement <4 x float> %tmp249, float undef, i32 2
-  %tmp251 = insertelement <4 x float> %tmp250, float undef, i32 3
+  %tmp248 = insertelement <4 x float> poison, float %tmp247, i32 0
+  %tmp249 = insertelement <4 x float> %tmp248, float poison, i32 1
+  %tmp250 = insertelement <4 x float> %tmp249, float poison, i32 2
+  %tmp251 = insertelement <4 x float> %tmp250, float poison, i32 3
   %tmp252 = shufflevector <4 x float> %tmp251, <4 x float> %tmp251, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %tmp253 = call <4 x float> @llvm.r600.tex(<4 x float> %tmp252, i32 0, i32 0, i32 0, i32 18, i32 2, i32 1, i32 1, i32 1, i32 1)
   %tmp254 = extractelement <4 x float> %tmp253, i32 0
@@ -312,16 +312,16 @@ ENDIF136:                                         ; preds = %ENDIF154, %main_bod
   %tmp285 = fmul float %clamp.i8, %tmp284
   %tmp286 = fmul float %tmp26, 0x3F22DFD6A0000000
   %tmp287 = fmul float %tmp27, 0x3F22DFD6A0000000
-  %tmp288 = insertelement <4 x float> undef, float %tmp286, i32 0
+  %tmp288 = insertelement <4 x float> poison, float %tmp286, i32 0
   %tmp289 = insertelement <4 x float> %tmp288, float %tmp287, i32 1
   %tmp290 = insertelement <4 x float> %tmp289, float 0.000000e+00, i32 2
   %tmp291 = insertelement <4 x float> %tmp290, float 0.000000e+00, i32 3
   %tmp292 = extractelement <4 x float> %tmp291, i32 0
   %tmp293 = extractelement <4 x float> %tmp291, i32 1
-  %tmp294 = insertelement <4 x float> undef, float %tmp292, i32 0
+  %tmp294 = insertelement <4 x float> poison, float %tmp292, i32 0
   %tmp295 = insertelement <4 x float> %tmp294, float %tmp293, i32 1
-  %tmp296 = insertelement <4 x float> %tmp295, float undef, i32 2
-  %tmp297 = insertelement <4 x float> %tmp296, float undef, i32 3
+  %tmp296 = insertelement <4 x float> %tmp295, float poison, i32 2
+  %tmp297 = insertelement <4 x float> %tmp296, float poison, i32 3
   %tmp298 = shufflevector <4 x float> %tmp297, <4 x float> %tmp297, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %tmp299 = call <4 x float> @llvm.r600.tex(<4 x float> %tmp298, i32 0, i32 0, i32 0, i32 19, i32 3, i32 1, i32 1, i32 1, i32 1)
   %tmp300 = extractelement <4 x float> %tmp299, i32 0
@@ -347,11 +347,11 @@ ENDIF136:                                         ; preds = %ENDIF154, %main_bod
   %tmp320 = fadd float %tmp319, %tmp314
   %tmp321 = fmul float %temp70.0, %tmp36
   %tmp322 = fadd float %tmp321, %tmp316
-  %tmp323 = insertelement <4 x float> undef, float %tmp318, i32 0
+  %tmp323 = insertelement <4 x float> poison, float %tmp318, i32 0
   %tmp324 = insertelement <4 x float> %tmp323, float %tmp320, i32 1
   %tmp325 = insertelement <4 x float> %tmp324, float %tmp322, i32 2
   %tmp326 = insertelement <4 x float> %tmp325, float 0.000000e+00, i32 3
-  %tmp327 = insertelement <4 x float> undef, float %tmp318, i32 0
+  %tmp327 = insertelement <4 x float> poison, float %tmp318, i32 0
   %tmp328 = insertelement <4 x float> %tmp327, float %tmp320, i32 1
   %tmp329 = insertelement <4 x float> %tmp328, float %tmp322, i32 2
   %tmp330 = insertelement <4 x float> %tmp329, float 0.000000e+00, i32 3
@@ -380,11 +380,11 @@ ENDIF136:                                         ; preds = %ENDIF154, %main_bod
   %tmp353 = fsub float -0.000000e+00, %result.i
   %tmp354 = fsub float -0.000000e+00, %tmp347
   %tmp355 = fadd float %tmp353, %tmp354
-  %tmp356 = insertelement <4 x float> undef, float %tmp43, i32 0
+  %tmp356 = insertelement <4 x float> poison, float %tmp43, i32 0
   %tmp357 = insertelement <4 x float> %tmp356, float %tmp44, i32 1
   %tmp358 = insertelement <4 x float> %tmp357, float %tmp45, i32 2
   %tmp359 = insertelement <4 x float> %tmp358, float 0.000000e+00, i32 3
-  %tmp360 = insertelement <4 x float> undef, float %tmp43, i32 0
+  %tmp360 = insertelement <4 x float> poison, float %tmp43, i32 0
   %tmp361 = insertelement <4 x float> %tmp360, float %tmp44, i32 1
   %tmp362 = insertelement <4 x float> %tmp361, float %tmp45, i32 2
   %tmp363 = insertelement <4 x float> %tmp362, float 0.000000e+00, i32 3
@@ -407,11 +407,11 @@ ENDIF136:                                         ; preds = %ENDIF154, %main_bod
   %tmp379 = fsub float -0.000000e+00, %result.i
   %tmp380 = fsub float -0.000000e+00, %tmp347
   %tmp381 = fadd float %tmp379, %tmp380
-  %tmp382 = insertelement <4 x float> undef, float %tmp43, i32 0
+  %tmp382 = insertelement <4 x float> poison, float %tmp43, i32 0
   %tmp383 = insertelement <4 x float> %tmp382, float %tmp44, i32 1
   %tmp384 = insertelement <4 x float> %tmp383, float %tmp45, i32 2
   %tmp385 = insertelement <4 x float> %tmp384, float 0.000000e+00, i32 3
-  %tmp386 = insertelement <4 x float> undef, float %tmp43, i32 0
+  %tmp386 = insertelement <4 x float> poison, float %tmp43, i32 0
   %tmp387 = insertelement <4 x float> %tmp386, float %tmp44, i32 1
   %tmp388 = insertelement <4 x float> %tmp387, float %tmp45, i32 2
   %tmp389 = insertelement <4 x float> %tmp388, float 0.000000e+00, i32 3
@@ -492,16 +492,16 @@ IF140:                                            ; preds = %LOOP
   %tmp422 = fadd float %tmp421, %tmp22
   %tmp423 = fmul float %tmp130, %temp92.0
   %tmp424 = fadd float %tmp423, %tmp23
-  %tmp425 = insertelement <4 x float> undef, float %tmp422, i32 0
+  %tmp425 = insertelement <4 x float> poison, float %tmp422, i32 0
   %tmp426 = insertelement <4 x float> %tmp425, float %tmp424, i32 1
   %tmp427 = insertelement <4 x float> %tmp426, float 0.000000e+00, i32 2
   %tmp428 = insertelement <4 x float> %tmp427, float 0.000000e+00, i32 3
   %tmp429 = extractelement <4 x float> %tmp428, i32 0
   %tmp430 = extractelement <4 x float> %tmp428, i32 1
-  %tmp431 = insertelement <4 x float> undef, float %tmp429, i32 0
+  %tmp431 = insertelement <4 x float> poison, float %tmp429, i32 0
   %tmp432 = insertelement <4 x float> %tmp431, float %tmp430, i32 1
-  %tmp433 = insertelement <4 x float> %tmp432, float undef, i32 2
-  %tmp434 = insertelement <4 x float> %tmp433, float undef, i32 3
+  %tmp433 = insertelement <4 x float> %tmp432, float poison, i32 2
+  %tmp434 = insertelement <4 x float> %tmp433, float poison, i32 3
   %tmp435 = shufflevector <4 x float> %tmp434, <4 x float> %tmp434, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %tmp436 = call <4 x float> @llvm.r600.tex(<4 x float> %tmp435, i32 0, i32 0, i32 0, i32 20, i32 4, i32 1, i32 1, i32 1, i32 1)
   %tmp437 = extractelement <4 x float> %tmp436, i32 3
@@ -518,16 +518,16 @@ ENDIF139:                                         ; preds = %LOOP
   %tmp445 = fadd float %tmp444, %tmp22
   %tmp446 = fmul float %tmp130, %tmp443
   %tmp447 = fadd float %tmp446, %tmp23
-  %tmp448 = insertelement <4 x float> undef, float %tmp445, i32 0
+  %tmp448 = insertelement <4 x float> poison, float %tmp445, i32 0
   %tmp449 = insertelement <4 x float> %tmp448, float %tmp447, i32 1
   %tmp450 = insertelement <4 x float> %tmp449, float 0.000000e+00, i32 2
   %tmp451 = insertelement <4 x float> %tmp450, float 0.000000e+00, i32 3
   %tmp452 = extractelement <4 x float> %tmp451, i32 0
   %tmp453 = extractelement <4 x float> %tmp451, i32 1
-  %tmp454 = insertelement <4 x float> undef, float %tmp452, i32 0
+  %tmp454 = insertelement <4 x float> poison, float %tmp452, i32 0
   %tmp455 = insertelement <4 x float> %tmp454, float %tmp453, i32 1
-  %tmp456 = insertelement <4 x float> %tmp455, float undef, i32 2
-  %tmp457 = insertelement <4 x float> %tmp456, float undef, i32 3
+  %tmp456 = insertelement <4 x float> %tmp455, float poison, i32 2
+  %tmp457 = insertelement <4 x float> %tmp456, float poison, i32 3
   %tmp458 = shufflevector <4 x float> %tmp457, <4 x float> %tmp457, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %tmp459 = call <4 x float> @llvm.r600.tex(<4 x float> %tmp458, i32 0, i32 0, i32 0, i32 20, i32 4, i32 1, i32 1, i32 1, i32 1)
   %tmp460 = extractelement <4 x float> %tmp459, i32 3
@@ -563,16 +563,16 @@ ENDIF145:                                         ; preds = %IF146, %IF140
   %tmp482 = fadd float %tmp481, %tmp22
   %tmp483 = fmul float %tmp130, %tmp479
   %tmp484 = fadd float %tmp483, %tmp23
-  %tmp485 = insertelement <4 x float> undef, float %tmp482, i32 0
+  %tmp485 = insertelement <4 x float> poison, float %tmp482, i32 0
   %tmp486 = insertelement <4 x float> %tmp485, float %tmp484, i32 1
   %tmp487 = insertelement <4 x float> %tmp486, float 0.000000e+00, i32 2
   %tmp488 = insertelement <4 x float> %tmp487, float %tmp437, i32 3
   %tmp489 = extractelement <4 x float> %tmp488, i32 0
   %tmp490 = extractelement <4 x float> %tmp488, i32 1
-  %tmp491 = insertelement <4 x float> undef, float %tmp489, i32 0
+  %tmp491 = insertelement <4 x float> poison, float %tmp489, i32 0
   %tmp492 = insertelement <4 x float> %tmp491, float %tmp490, i32 1
-  %tmp493 = insertelement <4 x float> %tmp492, float undef, i32 2
-  %tmp494 = insertelement <4 x float> %tmp493, float undef, i32 3
+  %tmp493 = insertelement <4 x float> %tmp492, float poison, i32 2
+  %tmp494 = insertelement <4 x float> %tmp493, float poison, i32 3
   %tmp495 = shufflevector <4 x float> %tmp494, <4 x float> %tmp494, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %tmp496 = call <4 x float> @llvm.r600.tex(<4 x float> %tmp495, i32 0, i32 0, i32 0, i32 20, i32 4, i32 1, i32 1, i32 1, i32 1)
   %tmp497 = extractelement <4 x float> %tmp496, i32 3
@@ -598,16 +598,16 @@ ENDIF148:                                         ; preds = %IF149, %ENDIF145
   %tmp509 = fadd float %tmp508, %tmp22
   %tmp510 = fmul float %tmp130, %tmp506
   %tmp511 = fadd float %tmp510, %tmp23
-  %tmp512 = insertelement <4 x float> undef, float %tmp509, i32 0
+  %tmp512 = insertelement <4 x float> poison, float %tmp509, i32 0
   %tmp513 = insertelement <4 x float> %tmp512, float %tmp511, i32 1
   %tmp514 = insertelement <4 x float> %tmp513, float 0.000000e+00, i32 2
   %tmp515 = insertelement <4 x float> %tmp514, float %tmp497, i32 3
   %tmp516 = extractelement <4 x float> %tmp515, i32 0
   %tmp517 = extractelement <4 x float> %tmp515, i32 1
-  %tmp518 = insertelement <4 x float> undef, float %tmp516, i32 0
+  %tmp518 = insertelement <4 x float> poison, float %tmp516, i32 0
   %tmp519 = insertelement <4 x float> %tmp518, float %tmp517, i32 1
-  %tmp520 = insertelement <4 x float> %tmp519, float undef, i32 2
-  %tmp521 = insertelement <4 x float> %tmp520, float undef, i32 3
+  %tmp520 = insertelement <4 x float> %tmp519, float poison, i32 2
+  %tmp521 = insertelement <4 x float> %tmp520, float poison, i32 3
   %tmp522 = shufflevector <4 x float> %tmp521, <4 x float> %tmp521, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %tmp523 = call <4 x float> @llvm.r600.tex(<4 x float> %tmp522, i32 0, i32 0, i32 0, i32 20, i32 4, i32 1, i32 1, i32 1, i32 1)
   %tmp524 = extractelement <4 x float> %tmp523, i32 3
@@ -633,16 +633,16 @@ ENDIF151:                                         ; preds = %IF152, %ENDIF148
   %tmp536 = fadd float %tmp535, %tmp22
   %tmp537 = fmul float %tmp130, %tmp533
   %tmp538 = fadd float %tmp537, %tmp23
-  %tmp539 = insertelement <4 x float> undef, float %tmp536, i32 0
+  %tmp539 = insertelement <4 x float> poison, float %tmp536, i32 0
   %tmp540 = insertelement <4 x float> %tmp539, float %tmp538, i32 1
   %tmp541 = insertelement <4 x float> %tmp540, float 0.000000e+00, i32 2
   %tmp542 = insertelement <4 x float> %tmp541, float %tmp524, i32 3
   %tmp543 = extractelement <4 x float> %tmp542, i32 0
   %tmp544 = extractelement <4 x float> %tmp542, i32 1
-  %tmp545 = insertelement <4 x float> undef, float %tmp543, i32 0
+  %tmp545 = insertelement <4 x float> poison, float %tmp543, i32 0
   %tmp546 = insertelement <4 x float> %tmp545, float %tmp544, i32 1
-  %tmp547 = insertelement <4 x float> %tmp546, float undef, i32 2
-  %tmp548 = insertelement <4 x float> %tmp547, float undef, i32 3
+  %tmp547 = insertelement <4 x float> %tmp546, float poison, i32 2
+  %tmp548 = insertelement <4 x float> %tmp547, float poison, i32 3
   %tmp549 = shufflevector <4 x float> %tmp548, <4 x float> %tmp548, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %tmp550 = call <4 x float> @llvm.r600.tex(<4 x float> %tmp549, i32 0, i32 0, i32 0, i32 20, i32 4, i32 1, i32 1, i32 1, i32 1)
   %tmp551 = extractelement <4 x float> %tmp550, i32 3
@@ -667,16 +667,16 @@ ENDIF154:                                         ; preds = %IF155, %ENDIF151
   %tmp562 = fadd float %tmp561, %tmp22
   %tmp563 = fmul float %tmp130, %tmp560
   %tmp564 = fadd float %tmp563, %tmp23
-  %tmp565 = insertelement <4 x float> undef, float %tmp562, i32 0
+  %tmp565 = insertelement <4 x float> poison, float %tmp562, i32 0
   %tmp566 = insertelement <4 x float> %tmp565, float %tmp564, i32 1
   %tmp567 = insertelement <4 x float> %tmp566, float 0.000000e+00, i32 2
   %tmp568 = insertelement <4 x float> %tmp567, float %tmp551, i32 3
   %tmp569 = extractelement <4 x float> %tmp568, i32 0
   %tmp570 = extractelement <4 x float> %tmp568, i32 1
-  %tmp571 = insertelement <4 x float> undef, float %tmp569, i32 0
+  %tmp571 = insertelement <4 x float> poison, float %tmp569, i32 0
   %tmp572 = insertelement <4 x float> %tmp571, float %tmp570, i32 1
-  %tmp573 = insertelement <4 x float> %tmp572, float undef, i32 2
-  %tmp574 = insertelement <4 x float> %tmp573, float undef, i32 3
+  %tmp573 = insertelement <4 x float> %tmp572, float poison, i32 2
+  %tmp574 = insertelement <4 x float> %tmp573, float poison, i32 3
   %tmp575 = shufflevector <4 x float> %tmp574, <4 x float> %tmp574, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %tmp576 = call <4 x float> @llvm.r600.tex(<4 x float> %tmp575, i32 0, i32 0, i32 0, i32 20, i32 4, i32 1, i32 1, i32 1, i32 1)
   %tmp577 = extractelement <4 x float> %tmp576, i32 3
@@ -690,16 +690,16 @@ ENDIF154:                                         ; preds = %IF155, %ENDIF151
   %tmp584 = fadd float %tmp583, %tmp22
   %tmp585 = fmul float %tmp130, %.temp92.4
   %tmp586 = fadd float %tmp585, %tmp23
-  %tmp587 = insertelement <4 x float> undef, float %tmp584, i32 0
+  %tmp587 = insertelement <4 x float> poison, float %tmp584, i32 0
   %tmp588 = insertelement <4 x float> %tmp587, float %tmp586, i32 1
   %tmp589 = insertelement <4 x float> %tmp588, float 0.000000e+00, i32 2
   %tmp590 = insertelement <4 x float> %tmp589, float %tmp577, i32 3
   %tmp591 = extractelement <4 x float> %tmp590, i32 0
   %tmp592 = extractelement <4 x float> %tmp590, i32 1
-  %tmp593 = insertelement <4 x float> undef, float %tmp591, i32 0
+  %tmp593 = insertelement <4 x float> poison, float %tmp591, i32 0
   %tmp594 = insertelement <4 x float> %tmp593, float %tmp592, i32 1
-  %tmp595 = insertelement <4 x float> %tmp594, float undef, i32 2
-  %tmp596 = insertelement <4 x float> %tmp595, float undef, i32 3
+  %tmp595 = insertelement <4 x float> %tmp594, float poison, i32 2
+  %tmp596 = insertelement <4 x float> %tmp595, float poison, i32 3
   %tmp597 = shufflevector <4 x float> %tmp596, <4 x float> %tmp596, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %tmp598 = call <4 x float> @llvm.r600.tex(<4 x float> %tmp597, i32 0, i32 0, i32 0, i32 20, i32 4, i32 1, i32 1, i32 1, i32 1)
   %tmp599 = extractelement <4 x float> %tmp598, i32 0
@@ -735,11 +735,11 @@ IF161:                                            ; preds = %ENDIF136
   %one.sub.ac.i126 = fmul float %one.sub.a.i125, %tmp307
   %mul.i127 = fmul float %result.i160, %tmp307
   %result.i128 = fadd float %mul.i127, %one.sub.ac.i126
-  %tmp613 = insertelement <4 x float> undef, float %tmp333, i32 0
+  %tmp613 = insertelement <4 x float> poison, float %tmp333, i32 0
   %tmp614 = insertelement <4 x float> %tmp613, float %tmp334, i32 1
   %tmp615 = insertelement <4 x float> %tmp614, float %tmp335, i32 2
   %tmp616 = insertelement <4 x float> %tmp615, float 0.000000e+00, i32 3
-  %tmp617 = insertelement <4 x float> undef, float %tmp63, i32 0
+  %tmp617 = insertelement <4 x float> poison, float %tmp63, i32 0
   %tmp618 = insertelement <4 x float> %tmp617, float %tmp65, i32 1
   %tmp619 = insertelement <4 x float> %tmp618, float %tmp67, i32 2
   %tmp620 = insertelement <4 x float> %tmp619, float 0.000000e+00, i32 3
@@ -749,11 +749,11 @@ IF161:                                            ; preds = %ENDIF136
   %tmp624 = fmul float %tmp8, %tmp623
   %tmp625 = fmul float %tmp13, %tmp623
   %tmp626 = fmul float %tmp18, %tmp623
-  %tmp627 = insertelement <4 x float> undef, float %tmp34, i32 0
+  %tmp627 = insertelement <4 x float> poison, float %tmp34, i32 0
   %tmp628 = insertelement <4 x float> %tmp627, float %tmp35, i32 1
   %tmp629 = insertelement <4 x float> %tmp628, float %tmp36, i32 2
   %tmp630 = insertelement <4 x float> %tmp629, float 0.000000e+00, i32 3
-  %tmp631 = insertelement <4 x float> undef, float %tmp63, i32 0
+  %tmp631 = insertelement <4 x float> poison, float %tmp63, i32 0
   %tmp632 = insertelement <4 x float> %tmp631, float %tmp65, i32 1
   %tmp633 = insertelement <4 x float> %tmp632, float %tmp67, i32 2
   %tmp634 = insertelement <4 x float> %tmp633, float 0.000000e+00, i32 3
@@ -809,11 +809,11 @@ IF164:                                            ; preds = %ENDIF160
   %one.sub.ac.i110 = fmul float %one.sub.a.i109, %tmp307
   %mul.i111 = fmul float %result.i160, %tmp307
   %result.i112 = fadd float %mul.i111, %one.sub.ac.i110
-  %tmp658 = insertelement <4 x float> undef, float %tmp333, i32 0
+  %tmp658 = insertelement <4 x float> poison, float %tmp333, i32 0
   %tmp659 = insertelement <4 x float> %tmp658, float %tmp334, i32 1
   %tmp660 = insertelement <4 x float> %tmp659, float %tmp335, i32 2
   %tmp661 = insertelement <4 x float> %tmp660, float 0.000000e+00, i32 3
-  %tmp662 = insertelement <4 x float> undef, float %tmp63, i32 0
+  %tmp662 = insertelement <4 x float> poison, float %tmp63, i32 0
   %tmp663 = insertelement <4 x float> %tmp662, float %tmp65, i32 1
   %tmp664 = insertelement <4 x float> %tmp663, float %tmp67, i32 2
   %tmp665 = insertelement <4 x float> %tmp664, float 0.000000e+00, i32 3
@@ -823,11 +823,11 @@ IF164:                                            ; preds = %ENDIF160
   %tmp669 = fmul float %tmp8, %tmp668
   %tmp670 = fmul float %tmp13, %tmp668
   %tmp671 = fmul float %tmp18, %tmp668
-  %tmp672 = insertelement <4 x float> undef, float %tmp34, i32 0
+  %tmp672 = insertelement <4 x float> poison, float %tmp34, i32 0
   %tmp673 = insertelement <4 x float> %tmp672, float %tmp35, i32 1
   %tmp674 = insertelement <4 x float> %tmp673, float %tmp36, i32 2
   %tmp675 = insertelement <4 x float> %tmp674, float 0.000000e+00, i32 3
-  %tmp676 = insertelement <4 x float> undef, float %tmp63, i32 0
+  %tmp676 = insertelement <4 x float> poison, float %tmp63, i32 0
   %tmp677 = insertelement <4 x float> %tmp676, float %tmp65, i32 1
   %tmp678 = insertelement <4 x float> %tmp677, float %tmp67, i32 2
   %tmp679 = insertelement <4 x float> %tmp678, float 0.000000e+00, i32 3
@@ -883,11 +883,11 @@ IF167:                                            ; preds = %ENDIF163
   %one.sub.ac.i94 = fmul float %one.sub.a.i93, %tmp307
   %mul.i95 = fmul float %result.i144, %tmp307
   %result.i96 = fadd float %mul.i95, %one.sub.ac.i94
-  %tmp703 = insertelement <4 x float> undef, float %tmp333, i32 0
+  %tmp703 = insertelement <4 x float> poison, float %tmp333, i32 0
   %tmp704 = insertelement <4 x float> %tmp703, float %tmp334, i32 1
   %tmp705 = insertelement <4 x float> %tmp704, float %tmp335, i32 2
   %tmp706 = insertelement <4 x float> %tmp705, float 0.000000e+00, i32 3
-  %tmp707 = insertelement <4 x float> undef, float %tmp63, i32 0
+  %tmp707 = insertelement <4 x float> poison, float %tmp63, i32 0
   %tmp708 = insertelement <4 x float> %tmp707, float %tmp65, i32 1
   %tmp709 = insertelement <4 x float> %tmp708, float %tmp67, i32 2
   %tmp710 = insertelement <4 x float> %tmp709, float 0.000000e+00, i32 3
@@ -897,11 +897,11 @@ IF167:                                            ; preds = %ENDIF163
   %tmp714 = fmul float %tmp8, %tmp713
   %tmp715 = fmul float %tmp13, %tmp713
   %tmp716 = fmul float %tmp18, %tmp713
-  %tmp717 = insertelement <4 x float> undef, float %tmp34, i32 0
+  %tmp717 = insertelement <4 x float> poison, float %tmp34, i32 0
   %tmp718 = insertelement <4 x float> %tmp717, float %tmp35, i32 1
   %tmp719 = insertelement <4 x float> %tmp718, float %tmp36, i32 2
   %tmp720 = insertelement <4 x float> %tmp719, float 0.000000e+00, i32 3
-  %tmp721 = insertelement <4 x float> undef, float %tmp63, i32 0
+  %tmp721 = insertelement <4 x float> poison, float %tmp63, i32 0
   %tmp722 = insertelement <4 x float> %tmp721, float %tmp65, i32 1
   %tmp723 = insertelement <4 x float> %tmp722, float %tmp67, i32 2
   %tmp724 = insertelement <4 x float> %tmp723, float 0.000000e+00, i32 3
@@ -957,11 +957,11 @@ IF170:                                            ; preds = %ENDIF166
   %one.sub.ac.i78 = fmul float %one.sub.a.i77, %tmp307
   %mul.i79 = fmul float %result.i144, %tmp307
   %result.i80 = fadd float %mul.i79, %one.sub.ac.i78
-  %tmp748 = insertelement <4 x float> undef, float %tmp333, i32 0
+  %tmp748 = insertelement <4 x float> poison, float %tmp333, i32 0
   %tmp749 = insertelement <4 x float> %tmp748, float %tmp334, i32 1
   %tmp750 = insertelement <4 x float> %tmp749, float %tmp335, i32 2
   %tmp751 = insertelement <4 x float> %tmp750, float 0.000000e+00, i32 3
-  %tmp752 = insertelement <4 x float> undef, float %tmp63, i32 0
+  %tmp752 = insertelement <4 x float> poison, float %tmp63, i32 0
   %tmp753 = insertelement <4 x float> %tmp752, float %tmp65, i32 1
   %tmp754 = insertelement <4 x float> %tmp753, float %tmp67, i32 2
   %tmp755 = insertelement <4 x float> %tmp754, float 0.000000e+00, i32 3
@@ -971,11 +971,11 @@ IF170:                                            ; preds = %ENDIF166
   %tmp759 = fmul float %tmp8, %tmp758
   %tmp760 = fmul float %tmp13, %tmp758
   %tmp761 = fmul float %tmp18, %tmp758
-  %tmp762 = insertelement <4 x float> undef, float %tmp34, i32 0
+  %tmp762 = insertelement <4 x float> poison, float %tmp34, i32 0
   %tmp763 = insertelement <4 x float> %tmp762, float %tmp35, i32 1
   %tmp764 = insertelement <4 x float> %tmp763, float %tmp36, i32 2
   %tmp765 = insertelement <4 x float> %tmp764, float 0.000000e+00, i32 3
-  %tmp766 = insertelement <4 x float> undef, float %tmp63, i32 0
+  %tmp766 = insertelement <4 x float> poison, float %tmp63, i32 0
   %tmp767 = insertelement <4 x float> %tmp766, float %tmp65, i32 1
   %tmp768 = insertelement <4 x float> %tmp767, float %tmp67, i32 2
   %tmp769 = insertelement <4 x float> %tmp768, float 0.000000e+00, i32 3
@@ -1031,11 +1031,11 @@ IF173:                                            ; preds = %ENDIF169
   %one.sub.ac.i62 = fmul float %one.sub.a.i61, %tmp307
   %mul.i63 = fmul float %result.i160, %tmp307
   %result.i64 = fadd float %mul.i63, %one.sub.ac.i62
-  %tmp793 = insertelement <4 x float> undef, float %tmp333, i32 0
+  %tmp793 = insertelement <4 x float> poison, float %tmp333, i32 0
   %tmp794 = insertelement <4 x float> %tmp793, float %tmp334, i32 1
   %tmp795 = insertelement <4 x float> %tmp794, float %tmp335, i32 2
   %tmp796 = insertelement <4 x float> %tmp795, float 0.000000e+00, i32 3
-  %tmp797 = insertelement <4 x float> undef, float %tmp63, i32 0
+  %tmp797 = insertelement <4 x float> poison, float %tmp63, i32 0
   %tmp798 = insertelement <4 x float> %tmp797, float %tmp65, i32 1
   %tmp799 = insertelement <4 x float> %tmp798, float %tmp67, i32 2
   %tmp800 = insertelement <4 x float> %tmp799, float 0.000000e+00, i32 3
@@ -1045,11 +1045,11 @@ IF173:                                            ; preds = %ENDIF169
   %tmp804 = fmul float %tmp8, %tmp803
   %tmp805 = fmul float %tmp13, %tmp803
   %tmp806 = fmul float %tmp18, %tmp803
-  %tmp807 = insertelement <4 x float> undef, float %tmp34, i32 0
+  %tmp807 = insertelement <4 x float> poison, float %tmp34, i32 0
   %tmp808 = insertelement <4 x float> %tmp807, float %tmp35, i32 1
   %tmp809 = insertelement <4 x float> %tmp808, float %tmp36, i32 2
   %tmp810 = insertelement <4 x float> %tmp809, float 0.000000e+00, i32 3
-  %tmp811 = insertelement <4 x float> undef, float %tmp63, i32 0
+  %tmp811 = insertelement <4 x float> poison, float %tmp63, i32 0
   %tmp812 = insertelement <4 x float> %tmp811, float %tmp65, i32 1
   %tmp813 = insertelement <4 x float> %tmp812, float %tmp67, i32 2
   %tmp814 = insertelement <4 x float> %tmp813, float 0.000000e+00, i32 3
@@ -1098,11 +1098,11 @@ IF176:                                            ; preds = %ENDIF172
   %one.sub.ac.i46 = fmul float %one.sub.a.i45, %tmp307
   %mul.i47 = fmul float %result.i160, %tmp307
   %result.i48 = fadd float %mul.i47, %one.sub.ac.i46
-  %tmp831 = insertelement <4 x float> undef, float %tmp333, i32 0
+  %tmp831 = insertelement <4 x float> poison, float %tmp333, i32 0
   %tmp832 = insertelement <4 x float> %tmp831, float %tmp334, i32 1
   %tmp833 = insertelement <4 x float> %tmp832, float %tmp335, i32 2
   %tmp834 = insertelement <4 x float> %tmp833, float 0.000000e+00, i32 3
-  %tmp835 = insertelement <4 x float> undef, float %tmp63, i32 0
+  %tmp835 = insertelement <4 x float> poison, float %tmp63, i32 0
   %tmp836 = insertelement <4 x float> %tmp835, float %tmp65, i32 1
   %tmp837 = insertelement <4 x float> %tmp836, float %tmp67, i32 2
   %tmp838 = insertelement <4 x float> %tmp837, float 0.000000e+00, i32 3
@@ -1112,11 +1112,11 @@ IF176:                                            ; preds = %ENDIF172
   %tmp842 = fmul float %tmp8, %tmp841
   %tmp843 = fmul float %tmp13, %tmp841
   %tmp844 = fmul float %tmp18, %tmp841
-  %tmp845 = insertelement <4 x float> undef, float %tmp34, i32 0
+  %tmp845 = insertelement <4 x float> poison, float %tmp34, i32 0
   %tmp846 = insertelement <4 x float> %tmp845, float %tmp35, i32 1
   %tmp847 = insertelement <4 x float> %tmp846, float %tmp36, i32 2
   %tmp848 = insertelement <4 x float> %tmp847, float 0.000000e+00, i32 3
-  %tmp849 = insertelement <4 x float> undef, float %tmp63, i32 0
+  %tmp849 = insertelement <4 x float> poison, float %tmp63, i32 0
   %tmp850 = insertelement <4 x float> %tmp849, float %tmp65, i32 1
   %tmp851 = insertelement <4 x float> %tmp850, float %tmp67, i32 2
   %tmp852 = insertelement <4 x float> %tmp851, float 0.000000e+00, i32 3
@@ -1149,11 +1149,11 @@ IF179:                                            ; preds = %ENDIF175
   %tmp866 = fadd float %result.i, 1.000000e+00
   %tmp867 = fadd float %result.i, 1.000000e+00
   %tmp868 = fadd float %result.i, 1.000000e+00
-  %tmp869 = insertelement <4 x float> undef, float %tmp43, i32 0
+  %tmp869 = insertelement <4 x float> poison, float %tmp43, i32 0
   %tmp870 = insertelement <4 x float> %tmp869, float %tmp44, i32 1
   %tmp871 = insertelement <4 x float> %tmp870, float %tmp45, i32 2
   %tmp872 = insertelement <4 x float> %tmp871, float 0.000000e+00, i32 3
-  %tmp873 = insertelement <4 x float> undef, float %tmp43, i32 0
+  %tmp873 = insertelement <4 x float> poison, float %tmp43, i32 0
   %tmp874 = insertelement <4 x float> %tmp873, float %tmp44, i32 1
   %tmp875 = insertelement <4 x float> %tmp874, float %tmp45, i32 2
   %tmp876 = insertelement <4 x float> %tmp875, float 0.000000e+00, i32 3
@@ -1209,11 +1209,11 @@ IF179:                                            ; preds = %ENDIF175
   %one.sub.ac.i14 = fmul float %one.sub.a.i13, %temp87.5
   %mul.i15 = fmul float %result.i32, %temp87.5
   %result.i16 = fadd float %mul.i15, %one.sub.ac.i14
-  %tmp896 = insertelement <4 x float> undef, float %tmp333, i32 0
+  %tmp896 = insertelement <4 x float> poison, float %tmp333, i32 0
   %tmp897 = insertelement <4 x float> %tmp896, float %tmp334, i32 1
   %tmp898 = insertelement <4 x float> %tmp897, float %tmp335, i32 2
   %tmp899 = insertelement <4 x float> %tmp898, float 0.000000e+00, i32 3
-  %tmp900 = insertelement <4 x float> undef, float %tmp63, i32 0
+  %tmp900 = insertelement <4 x float> poison, float %tmp63, i32 0
   %tmp901 = insertelement <4 x float> %tmp900, float %tmp65, i32 1
   %tmp902 = insertelement <4 x float> %tmp901, float %tmp67, i32 2
   %tmp903 = insertelement <4 x float> %tmp902, float 0.000000e+00, i32 3
@@ -1223,11 +1223,11 @@ IF179:                                            ; preds = %ENDIF175
   %tmp907 = fmul float %tmp8, %tmp906
   %tmp908 = fmul float %tmp13, %tmp906
   %tmp909 = fmul float %tmp18, %tmp906
-  %tmp910 = insertelement <4 x float> undef, float %tmp34, i32 0
+  %tmp910 = insertelement <4 x float> poison, float %tmp34, i32 0
   %tmp911 = insertelement <4 x float> %tmp910, float %tmp35, i32 1
   %tmp912 = insertelement <4 x float> %tmp911, float %tmp36, i32 2
   %tmp913 = insertelement <4 x float> %tmp912, float 0.000000e+00, i32 3
-  %tmp914 = insertelement <4 x float> undef, float %tmp63, i32 0
+  %tmp914 = insertelement <4 x float> poison, float %tmp63, i32 0
   %tmp915 = insertelement <4 x float> %tmp914, float %tmp65, i32 1
   %tmp916 = insertelement <4 x float> %tmp915, float %tmp67, i32 2
   %tmp917 = insertelement <4 x float> %tmp916, float 0.000000e+00, i32 3
@@ -1289,7 +1289,7 @@ ENDIF178:                                         ; preds = %IF179, %ENDIF175
   %one.sub.ac.i2 = fmul float %one.sub.a.i1, %tmp51
   %mul.i3 = fmul float %tmp930, %tmp51
   %result.i4 = fadd float %mul.i3, %one.sub.ac.i2
-  %tmp953 = insertelement <4 x float> undef, float %result.i12, i32 0
+  %tmp953 = insertelement <4 x float> poison, float %result.i12, i32 0
   %tmp954 = insertelement <4 x float> %tmp953, float %result.i8, i32 1
   %tmp955 = insertelement <4 x float> %tmp954, float %result.i4, i32 2
   %tmp956 = insertelement <4 x float> %tmp955, float %tmp931, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll b/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll
index f9ffa5ae57f3e..a6af63b816573 100644
--- a/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll
+++ b/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll
@@ -9,53 +9,53 @@ define amdgpu_kernel void @blender_no_live_segment_at_def_error(<4 x float> %ext
 ; CHECK-NEXT:    s_addc_u32 s13, s13, 0
 ; CHECK-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
 ; CHECK-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
-; CHECK-NEXT:    s_load_dwordx8 s[36:43], s[8:9], 0x0
+; CHECK-NEXT:    s_load_dwordx8 s[48:55], s[8:9], 0x0
 ; CHECK-NEXT:    s_add_u32 s0, s0, s17
 ; CHECK-NEXT:    s_addc_u32 s1, s1, 0
 ; CHECK-NEXT:    s_mov_b32 s12, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    s_cmp_lg_u32 s40, 0
-; CHECK-NEXT:    s_cbranch_scc1 .LBB0_8
+; CHECK-NEXT:    s_cmp_lg_u32 s52, 0
+; CHECK-NEXT:    s_cbranch_scc1 .LBB0_9
 ; CHECK-NEXT:  ; %bb.1: ; %if.end13.i.i
-; CHECK-NEXT:    s_cmp_eq_u32 s42, 0
+; CHECK-NEXT:    s_cmp_eq_u32 s54, 0
 ; CHECK-NEXT:    s_cbranch_scc1 .LBB0_4
 ; CHECK-NEXT:  ; %bb.2: ; %if.else251.i.i
-; CHECK-NEXT:    s_cmp_lg_u32 s43, 0
+; CHECK-NEXT:    s_cmp_lg_u32 s55, 0
 ; CHECK-NEXT:    s_mov_b32 s17, 0
 ; CHECK-NEXT:    s_cselect_b32 s12, -1, 0
 ; CHECK-NEXT:    s_and_b32 vcc_lo, exec_lo, s12
 ; CHECK-NEXT:    s_cbranch_vccz .LBB0_5
 ; CHECK-NEXT:  ; %bb.3:
-; CHECK-NEXT:    s_mov_b32 s36, 0
-; CHECK-NEXT:    s_andn2_b32 vcc_lo, exec_lo, s12
-; CHECK-NEXT:    s_cbranch_vccz .LBB0_6
-; CHECK-NEXT:    s_branch .LBB0_7
+; CHECK-NEXT:    s_mov_b32 s18, 0
+; CHECK-NEXT:    s_branch .LBB0_6
 ; CHECK-NEXT:  .LBB0_4:
 ; CHECK-NEXT:    s_mov_b32 s14, s12
 ; CHECK-NEXT:    s_mov_b32 s15, s12
 ; CHECK-NEXT:    s_mov_b32 s13, s12
-; CHECK-NEXT:    s_mov_b64 s[38:39], s[14:15]
-; CHECK-NEXT:    s_mov_b64 s[36:37], s[12:13]
-; CHECK-NEXT:    s_branch .LBB0_7
+; CHECK-NEXT:    s_mov_b64 s[50:51], s[14:15]
+; CHECK-NEXT:    s_mov_b64 s[48:49], s[12:13]
+; CHECK-NEXT:    s_branch .LBB0_8
 ; CHECK-NEXT:  .LBB0_5: ; %if.then263.i.i
-; CHECK-NEXT:    v_cmp_lt_f32_e64 s12, s41, 0
-; CHECK-NEXT:    s_mov_b32 s36, 1.0
+; CHECK-NEXT:    v_cmp_lt_f32_e64 s12, s53, 0
+; CHECK-NEXT:    s_mov_b32 s18, 1.0
 ; CHECK-NEXT:    s_mov_b32 s17, 0x7fc00000
-; CHECK-NEXT:    s_mov_b32 s37, s36
-; CHECK-NEXT:    s_mov_b32 s38, s36
-; CHECK-NEXT:    s_mov_b32 s39, s36
+; CHECK-NEXT:  .LBB0_6: ; %Flow
+; CHECK-NEXT:    s_mov_b32 s48, 1.0
 ; CHECK-NEXT:    s_andn2_b32 vcc_lo, exec_lo, s12
-; CHECK-NEXT:    s_cbranch_vccnz .LBB0_7
-; CHECK-NEXT:  .LBB0_6: ; %if.end273.i.i
+; CHECK-NEXT:    s_mov_b32 s49, s48
+; CHECK-NEXT:    s_mov_b32 s50, s48
+; CHECK-NEXT:    s_mov_b32 s51, s48
+; CHECK-NEXT:    s_cbranch_vccnz .LBB0_8
+; CHECK-NEXT:  ; %bb.7: ; %if.end273.i.i
 ; CHECK-NEXT:    s_add_u32 s12, s8, 40
 ; CHECK-NEXT:    s_addc_u32 s13, s9, 0
-; CHECK-NEXT:    s_getpc_b64 s[18:19]
-; CHECK-NEXT:    s_add_u32 s18, s18, _Z3dotDv3_fS_@gotpcrel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s19, s19, _Z3dotDv3_fS_@gotpcrel32@hi+12
+; CHECK-NEXT:    s_getpc_b64 s[20:21]
+; CHECK-NEXT:    s_add_u32 s20, s20, _Z3dotDv3_fS_@gotpcrel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s21, s21, _Z3dotDv3_fS_@gotpcrel32@hi+12
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; CHECK-NEXT:    s_load_dwordx2 s[18:19], s[18:19], 0x0
+; CHECK-NEXT:    s_load_dwordx2 s[20:21], s[20:21], 0x0
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v3, 10, v1
-; CHECK-NEXT:    v_add_f32_e64 v1, s17, s36
+; CHECK-NEXT:    v_add_f32_e64 v1, s17, s18
 ; CHECK-NEXT:    s_mov_b64 s[34:35], s[8:9]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[12:13]
 ; CHECK-NEXT:    s_mov_b32 s12, s14
@@ -65,26 +65,26 @@ define amdgpu_kernel void @blender_no_live_segment_at_def_error(<4 x float> %ext
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 0
 ; CHECK-NEXT:    s_mov_b32 s13, s15
 ; CHECK-NEXT:    s_mov_b32 s14, s16
-; CHECK-NEXT:    s_mov_b32 s36, 0
+; CHECK-NEXT:    s_mov_b32 s48, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    s_swappc_b64 s[30:31], s[18:19]
+; CHECK-NEXT:    s_swappc_b64 s[30:31], s[20:21]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[34:35]
-; CHECK-NEXT:    s_mov_b32 s37, s36
-; CHECK-NEXT:    s_mov_b32 s38, s36
-; CHECK-NEXT:    s_mov_b32 s39, s36
-; CHECK-NEXT:  .LBB0_7: ; %if.end294.i.i
+; CHECK-NEXT:    s_mov_b32 s49, s48
+; CHECK-NEXT:    s_mov_b32 s50, s48
+; CHECK-NEXT:    s_mov_b32 s51, s48
+; CHECK-NEXT:  .LBB0_8: ; %if.end294.i.i
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    buffer_store_dword v0, off, s[0:3], 0 offset:12
 ; CHECK-NEXT:    buffer_store_dword v0, off, s[0:3], 0 offset:8
 ; CHECK-NEXT:    buffer_store_dword v0, off, s[0:3], 0 offset:4
 ; CHECK-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; CHECK-NEXT:  .LBB0_8: ; %kernel_direct_lighting.exit
+; CHECK-NEXT:  .LBB0_9: ; %kernel_direct_lighting.exit
 ; CHECK-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x20
-; CHECK-NEXT:    v_mov_b32_e32 v0, s36
+; CHECK-NEXT:    v_mov_b32_e32 v0, s48
 ; CHECK-NEXT:    v_mov_b32_e32 v4, 0
-; CHECK-NEXT:    v_mov_b32_e32 v1, s37
-; CHECK-NEXT:    v_mov_b32_e32 v2, s38
-; CHECK-NEXT:    v_mov_b32_e32 v3, s39
+; CHECK-NEXT:    v_mov_b32_e32 v1, s49
+; CHECK-NEXT:    v_mov_b32_e32 v2, s50
+; CHECK-NEXT:    v_mov_b32_e32 v3, s51
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[4:5]
 ; CHECK-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
index 98136347ab702..266216c4d8b50 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
@@ -13,16 +13,16 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   $sgpr0 = S_ADD_U32 $sgpr0, $sgpr17, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT:   $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT:   renamable $vgpr31 = COPY $vgpr0, implicit $exec
-  ; GFX90A-NEXT:   renamable $sgpr33 = S_LOAD_DWORD_IMM renamable $sgpr8_sgpr9, 24, 0 :: (dereferenceable invariant load (s32) from %ir.arg4.kernarg.offset.align.down, align 8, addrspace 4)
+  ; GFX90A-NEXT:   renamable $sgpr17 = S_LOAD_DWORD_IMM renamable $sgpr8_sgpr9, 24, 0 :: (dereferenceable invariant load (s32) from %ir.arg4.kernarg.offset.align.down, align 8, addrspace 4)
   ; GFX90A-NEXT:   early-clobber renamable $sgpr20_sgpr21_sgpr22_sgpr23 = S_LOAD_DWORDX4_IMM_ec renamable $sgpr8_sgpr9, 24, 0 :: (dereferenceable invariant load (s128) from %ir.arg6.kernarg.offset.align.down, align 8, addrspace 4)
-  ; GFX90A-NEXT:   renamable $sgpr17 = S_LOAD_DWORD_IMM renamable $sgpr8_sgpr9, 40, 0 :: (dereferenceable invariant load (s32) from %ir.arg6.kernarg.offset.align.down + 16, align 8, addrspace 4)
+  ; GFX90A-NEXT:   renamable $sgpr33 = S_LOAD_DWORD_IMM renamable $sgpr8_sgpr9, 40, 0 :: (dereferenceable invariant load (s32) from %ir.arg6.kernarg.offset.align.down + 16, align 8, addrspace 4)
   ; GFX90A-NEXT:   renamable $sgpr24_sgpr25_sgpr26_sgpr27 = S_LOAD_DWORDX4_IMM renamable $sgpr8_sgpr9, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_LOAD_DWORDX2_IMM renamable $sgpr8_sgpr9, 16, 0 :: (dereferenceable invariant load (s64) from %ir.arg.kernarg.offset1 + 16, align 16, addrspace 4)
-  ; GFX90A-NEXT:   S_BITCMP1_B32 renamable $sgpr33, 0, implicit-def $scc
+  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_LOAD_DWORDX2_IMM renamable $sgpr8_sgpr9, 16, 0 :: (dereferenceable invariant load (s64) from %ir.arg.kernarg.offset1 + 16, align 16, addrspace 4)
+  ; GFX90A-NEXT:   S_BITCMP1_B32 renamable $sgpr17, 0, implicit-def $scc
   ; GFX90A-NEXT:   renamable $sgpr12_sgpr13 = S_CSELECT_B64 -1, 0, implicit killed $scc
   ; GFX90A-NEXT:   renamable $sgpr34_sgpr35 = S_MOV_B64 -1
   ; GFX90A-NEXT:   renamable $sgpr28_sgpr29 = S_XOR_B64 renamable $sgpr12_sgpr13, -1, implicit-def dead $scc
-  ; GFX90A-NEXT:   S_BITCMP1_B32 renamable $sgpr33, 8, implicit-def $scc
+  ; GFX90A-NEXT:   S_BITCMP1_B32 renamable $sgpr17, 8, implicit-def $scc
   ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_CSELECT_B64 -1, 0, implicit killed $scc
   ; GFX90A-NEXT:   renamable $sgpr30_sgpr31 = S_XOR_B64 killed renamable $sgpr18_sgpr19, -1, implicit-def dead $scc
   ; GFX90A-NEXT:   renamable $vgpr3 = V_MOV_B32_e32 0, implicit $exec
@@ -33,7 +33,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.1.bb103:
   ; GFX90A-NEXT:   successors: %bb.58(0x40000000), %bb.2(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x00000000000000FF, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr46_sgpr47:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x00000000000000FF, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $sgpr34_sgpr35 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, renamable $sgpr30_sgpr31, implicit-def dead $scc
@@ -41,10 +41,8 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.2:
   ; GFX90A-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8, $sgpr9, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr56, $sgpr57, $sgpr20_sgpr21_sgpr22, $sgpr24_sgpr25_sgpr26, $sgpr26_sgpr27, $vgpr2, $vgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8, $sgpr9, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr46, $sgpr47, $sgpr20_sgpr21_sgpr22, $sgpr22_sgpr23, $sgpr24_sgpr25_sgpr26, $sgpr26_sgpr27, $vgpr2, $vgpr3
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $sgpr23 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
   ; GFX90A-NEXT:   renamable $vgpr19 = IMPLICIT_DEF implicit-def $vgpr18
   ; GFX90A-NEXT:   renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
@@ -54,15 +52,17 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.3.Flow17:
   ; GFX90A-NEXT:   successors: %bb.4(0x40000000), %bb.57(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr18_vgpr19:0x000000000000000F, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr46_sgpr47:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr18_vgpr19:0x000000000000000F, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr30 = V_AND_B32_e32 1023, $vgpr31, implicit $exec
   ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr34_sgpr35, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $vgpr15 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr17 = V_MOV_B32_e32 0, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_VCCZ %bb.57, implicit $vcc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.4.bb15:
   ; GFX90A-NEXT:   successors: %bb.35(0x40000000), %bb.5(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr18_sgpr19
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr46_sgpr47:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr18_sgpr19
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr0_vgpr1 = V_LSHLREV_B64_e64 2, $vgpr2_vgpr3, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr4 = COPY renamable $sgpr25, implicit $exec
@@ -79,15 +79,15 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr18_sgpr19
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $sgpr34_sgpr35 = S_MOV_B64 -1
-  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr40_sgpr41 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr66_sgpr67 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr64_sgpr65 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_MOV_B64 0
-  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_MOV_B64 0
-  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_MOV_B64 0
-  ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_MOV_B64 0
-  ; GFX90A-NEXT:   renamable $sgpr40_sgpr41 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $sgpr38_sgpr39 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr6_vgpr7 = IMPLICIT_DEF
@@ -107,32 +107,31 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
   ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
-  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.6.Flow20:
   ; GFX90A-NEXT:   successors: %bb.7(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   renamable $vgpr19 = COPY renamable $sgpr17, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr18 = COPY $sgpr17, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr21 = COPY $sgpr17, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr20 = COPY $sgpr17, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr23 = COPY $sgpr17, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr22 = COPY $sgpr17, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr25 = COPY $sgpr17, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr24 = COPY $sgpr17, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr19 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr18 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr21 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr20 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr23 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr22 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr25 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr24 = V_MOV_B32_e32 0, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.7.Flow19:
   ; GFX90A-NEXT:   successors: %bb.62(0x40000000), %bb.8(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x000000000000000F, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x000000000000000F, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = S_MOV_B64 0
   ; GFX90A-NEXT:   $sgpr24_sgpr25 = S_AND_SAVEEXEC_B64 $sgpr36_sgpr37, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.62, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.8.Flow32:
   ; GFX90A-NEXT:   successors: %bb.9(0x40000000), %bb.10(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr58_sgpr59, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr24_sgpr25, implicit-def $scc
   ; GFX90A-NEXT:   $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $sgpr18_sgpr19, implicit-def $exec, implicit-def $scc, implicit $exec
@@ -141,58 +140,58 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.9.bb89:
   ; GFX90A-NEXT:   successors: %bb.10(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr58_sgpr59, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr9, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr8, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_OR_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.10.Flow33:
   ; GFX90A-NEXT:   successors: %bb.11(0x40000000), %bb.12(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr58_sgpr59, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def $scc
-  ; GFX90A-NEXT:   $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $sgpr58_sgpr59, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $sgpr44_sgpr45, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr12_sgpr13 = S_XOR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
   ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.12, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.11.bb84:
   ; GFX90A-NEXT:   successors: %bb.12(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr7, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr6, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_OR_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.12.Flow34:
   ; GFX90A-NEXT:   successors: %bb.13(0x40000000), %bb.14(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def $scc
-  ; GFX90A-NEXT:   $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $sgpr54_sgpr55, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $sgpr42_sgpr43, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr12_sgpr13 = S_XOR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
   ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.14, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.13.bb79:
   ; GFX90A-NEXT:   successors: %bb.14(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr56_sgpr57, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_OR_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.14.Flow35:
   ; GFX90A-NEXT:   successors: %bb.15(0x40000000), %bb.16(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr56_sgpr57, $vgpr0_vgpr1:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def $scc
-  ; GFX90A-NEXT:   $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $sgpr52_sgpr53, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $sgpr40_sgpr41, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr36_sgpr37 = S_XOR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
   ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.16, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.15.bb72:
   ; GFX90A-NEXT:   successors: %bb.16(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr56_sgpr57, $vgpr0_vgpr1:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $sgpr8 = S_ADD_U32 renamable $sgpr8, 48, implicit-def $scc
   ; GFX90A-NEXT:   renamable $sgpr9 = S_ADDC_U32 killed renamable $sgpr9, 0, implicit-def dead $scc, implicit killed $scc
@@ -202,113 +201,113 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   $sgpr13 = COPY killed renamable $sgpr15
   ; GFX90A-NEXT:   $sgpr14 = COPY killed renamable $sgpr16
   ; GFX90A-NEXT:   dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr18_sgpr19, @f2, csr_amdgpu_gfx90ainsts, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit undef $sgpr15, implicit $vgpr31, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_OR_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.16.Flow36:
   ; GFX90A-NEXT:   successors: %bb.17(0x40000000), %bb.18(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr56_sgpr57, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr36_sgpr37, implicit-def $scc
-  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr50_sgpr51, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr66_sgpr67, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
   ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.18, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.17.bb67:
   ; GFX90A-NEXT:   successors: %bb.18(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr56_sgpr57, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr47, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr46, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_OR_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.18.Flow37:
   ; GFX90A-NEXT:   successors: %bb.19(0x40000000), %bb.20(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr56_sgpr57, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
-  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr48_sgpr49, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr64_sgpr65, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
   ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.20, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.19.bb62:
   ; GFX90A-NEXT:   successors: %bb.20(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr56_sgpr57, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr63, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr62, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_OR_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.20.Flow38:
   ; GFX90A-NEXT:   successors: %bb.21(0x40000000), %bb.22(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr56_sgpr57, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
-  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr46_sgpr47, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr54_sgpr55, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
   ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.22, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.21.bb54:
   ; GFX90A-NEXT:   successors: %bb.22(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr56_sgpr57, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr61, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr60, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_OR_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.22.Flow39:
   ; GFX90A-NEXT:   successors: %bb.23(0x40000000), %bb.24(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr56_sgpr57, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
-  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr44_sgpr45, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr52_sgpr53, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
   ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.24, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.23.bb47:
   ; GFX90A-NEXT:   successors: %bb.24(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr56_sgpr57, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr59, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr58, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_OR_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.24.Flow40:
   ; GFX90A-NEXT:   successors: %bb.25(0x40000000), %bb.26(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr56_sgpr57, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
-  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr42_sgpr43, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr50_sgpr51, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
   ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.26, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.25.bb40:
   ; GFX90A-NEXT:   successors: %bb.26(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr56_sgpr57, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr57, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr56, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_OR_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.26.Flow41:
   ; GFX90A-NEXT:   successors: %bb.27(0x40000000), %bb.28(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr56_sgpr57, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
-  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr40_sgpr41, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr48_sgpr49, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
   ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.28, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.27.bb33:
   ; GFX90A-NEXT:   successors: %bb.28(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr56_sgpr57, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr45, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr44, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_OR_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.28.Flow42:
   ; GFX90A-NEXT:   successors: %bb.34(0x40000000), %bb.29(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr56_sgpr57, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
   ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr38_sgpr39, implicit-def $exec, implicit-def $scc, implicit $exec
@@ -317,7 +316,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.29.Flow43:
   ; GFX90A-NEXT:   successors: %bb.30(0x40000000), %bb.31(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr56_sgpr57, $vgpr40_vgpr41:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
   ; GFX90A-NEXT:   $vcc = S_ANDN2_B64 $exec, killed renamable $sgpr34_sgpr35, implicit-def dead $scc
@@ -325,17 +324,17 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.30.bb19:
   ; GFX90A-NEXT:   successors: %bb.31(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr56_sgpr57, $vgpr40_vgpr41:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr41, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr40, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_OR_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.31.Flow44:
   ; GFX90A-NEXT:   successors: %bb.32(0x40000000), %bb.33(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr56_sgpr57, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr68_sgpr69, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr56_sgpr57, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr68_sgpr69, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.33, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.32.UnifiedUnreachableBlock:
@@ -351,32 +350,32 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.34.bb26:
   ; GFX90A-NEXT:   successors: %bb.29(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr56_sgpr57, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr43, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
   ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr42, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_OR_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
   ; GFX90A-NEXT:   S_BRANCH %bb.29
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.35.bb20:
   ; GFX90A-NEXT:   successors: %bb.37(0x40000000), %bb.36(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr18_sgpr19
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr46_sgpr47:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr18_sgpr19
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr0 = GLOBAL_LOAD_SBYTE renamable $vgpr40_vgpr41, 1024, 0, implicit $exec :: (load (s8) from %ir.i21, addrspace 1)
   ; GFX90A-NEXT:   renamable $vgpr42 = V_ADD_CO_U32_e32 1024, $vgpr40, implicit-def $vcc, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr34_sgpr35 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $sgpr38_sgpr39 = S_MOV_B64 -1
-  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr40_sgpr41 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr66_sgpr67 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr64_sgpr65 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_MOV_B64 0
-  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_MOV_B64 0
-  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_MOV_B64 0
-  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_MOV_B64 0
-  ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $vgpr43, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vcc = V_CMP_LT_I16_e64 0, killed $vgpr0, implicit $exec
-  ; GFX90A-NEXT:   renamable $sgpr40_sgpr41 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr6_vgpr7 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr4_vgpr5 = IMPLICIT_DEF
@@ -394,25 +393,27 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
   ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
-  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
   ; GFX90A-NEXT:   $sgpr24_sgpr25 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.37, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.36.Flow21:
   ; GFX90A-NEXT:   successors: %bb.6(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr24_sgpr25, implicit-def $scc
   ; GFX90A-NEXT:   S_BRANCH %bb.6
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.37.bb27:
   ; GFX90A-NEXT:   successors: %bb.39(0x40000000), %bb.38(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr18_sgpr19, $sgpr58_sgpr59, $sgpr54_sgpr55, $sgpr52_sgpr53, $sgpr50_sgpr51, $sgpr48_sgpr49, $sgpr46_sgpr47, $sgpr44_sgpr45, $sgpr42_sgpr43
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr46_sgpr47:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr18_sgpr19, $sgpr44_sgpr45, $sgpr42_sgpr43, $sgpr54_sgpr55, $sgpr52_sgpr53, $sgpr64_sgpr65, $sgpr50_sgpr51, $sgpr66_sgpr67
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr0 = GLOBAL_LOAD_UBYTE renamable $vgpr40_vgpr41, 2048, 0, implicit $exec :: (load (s8) from %ir.i28, addrspace 1)
   ; GFX90A-NEXT:   renamable $vgpr44 = V_ADD_CO_U32_e32 2048, $vgpr40, implicit-def $vcc, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr40_sgpr41 = S_MOV_B64 -1
-  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = COPY renamable $sgpr36_sgpr37
+  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = COPY renamable $sgpr36_sgpr37
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr62_sgpr63 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $vgpr45, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
@@ -431,39 +432,39 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
   ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
-  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
   ; GFX90A-NEXT:   $sgpr38_sgpr39 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.39, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.38.Flow22:
   ; GFX90A-NEXT:   successors: %bb.36(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr38_sgpr39, implicit-def $scc
   ; GFX90A-NEXT:   renamable $sgpr38_sgpr39 = S_XOR_B64 $exec, -1, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr40_sgpr41 = S_AND_B64 killed renamable $sgpr40_sgpr41, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_AND_B64 killed renamable $sgpr42_sgpr43, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_AND_B64 killed renamable $sgpr44_sgpr45, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_AND_B64 killed renamable $sgpr46_sgpr47, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr48_sgpr49, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr40_sgpr41, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr64_sgpr65, $exec, implicit-def dead $scc
   ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
   ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_AND_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr64_sgpr65 = S_AND_B64 killed renamable $sgpr62_sgpr63, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr66_sgpr67 = S_AND_B64 killed renamable $sgpr60_sgpr61, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr40_sgpr41 = S_AND_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_AND_B64 killed renamable $sgpr42_sgpr43, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_AND_B64 killed renamable $sgpr44_sgpr45, $exec, implicit-def dead $scc
   ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_AND_B64 killed renamable $sgpr18_sgpr19, $exec, implicit-def dead $scc
   ; GFX90A-NEXT:   renamable $sgpr36_sgpr37 = S_ANDN2_B64 killed renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr60_sgpr61, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr36_sgpr37 = S_OR_B64 killed renamable $sgpr36_sgpr37, killed renamable $sgpr56_sgpr57, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_AND_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr36_sgpr37 = S_OR_B64 killed renamable $sgpr36_sgpr37, killed renamable $sgpr46_sgpr47, implicit-def dead $scc
   ; GFX90A-NEXT:   S_BRANCH %bb.36
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.39.bb34:
   ; GFX90A-NEXT:   successors: %bb.41(0x40000000), %bb.40(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr18_sgpr19, $sgpr58_sgpr59, $sgpr54_sgpr55, $sgpr52_sgpr53, $sgpr50_sgpr51, $sgpr48_sgpr49, $sgpr46_sgpr47, $sgpr44_sgpr45
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr46_sgpr47:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr18_sgpr19, $sgpr44_sgpr45, $sgpr50_sgpr51, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr54_sgpr55, $sgpr62_sgpr63, $sgpr52_sgpr53, $sgpr64_sgpr65, $sgpr66_sgpr67
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr0 = GLOBAL_LOAD_UBYTE renamable $vgpr40_vgpr41, 3072, 0, implicit $exec :: (load (s8) from %ir.i35, addrspace 1)
   ; GFX90A-NEXT:   renamable $vgpr56 = V_ADD_CO_U32_e32 3072, $vgpr40, implicit-def $vcc, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_MOV_B64 -1
-  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = COPY renamable $sgpr36_sgpr37
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = COPY renamable $sgpr36_sgpr37
+  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $vgpr57, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
@@ -481,33 +482,32 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
   ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
-  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
   ; GFX90A-NEXT:   $sgpr40_sgpr41 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.41, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.40.Flow23:
   ; GFX90A-NEXT:   successors: %bb.38(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr40_sgpr41, implicit-def $scc
   ; GFX90A-NEXT:   renamable $sgpr40_sgpr41 = S_XOR_B64 $exec, -1, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_AND_B64 killed renamable $sgpr42_sgpr43, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_AND_B64 killed renamable $sgpr44_sgpr45, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_AND_B64 killed renamable $sgpr46_sgpr47, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr48_sgpr49, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr64_sgpr65 = S_AND_B64 killed renamable $sgpr42_sgpr43, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr62_sgpr63, $exec, implicit-def dead $scc
   ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_AND_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr62_sgpr63 = S_AND_B64 killed renamable $sgpr60_sgpr61, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = S_AND_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_AND_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_AND_B64 killed renamable $sgpr44_sgpr45, $exec, implicit-def dead $scc
   ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_AND_B64 killed renamable $sgpr18_sgpr19, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = S_AND_B64 killed renamable $sgpr60_sgpr61, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = S_OR_B64 killed renamable $sgpr56_sgpr57, killed renamable $sgpr60_sgpr61, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr48_sgpr49, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_OR_B64 killed renamable $sgpr46_sgpr47, killed renamable $sgpr48_sgpr49, implicit-def dead $scc
   ; GFX90A-NEXT:   S_BRANCH %bb.38
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.41.bb41:
   ; GFX90A-NEXT:   successors: %bb.46(0x40000000), %bb.42(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr58_sgpr59, $sgpr54_sgpr55, $sgpr52_sgpr53, $sgpr50_sgpr51, $sgpr48_sgpr49, $sgpr46_sgpr47
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr46_sgpr47:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr52_sgpr53, $sgpr50_sgpr51, $sgpr56_sgpr57, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr58 = V_ADD_CO_U32_e32 4096, $vgpr40, implicit-def $vcc, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = COPY $vcc
@@ -515,10 +515,9 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr0 = GLOBAL_LOAD_UBYTE renamable $vgpr58_vgpr59, 0, 0, implicit $exec :: (load (s8) from %ir.i42, addrspace 1)
   ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_MOV_B64 -1
-  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = COPY renamable $sgpr36_sgpr37
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = COPY renamable $sgpr36_sgpr37
   ; GFX90A-NEXT:   renamable $vgpr18, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr0, implicit $exec
-  ; GFX90A-NEXT:   renamable $sgpr62_sgpr63 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr6_vgpr7 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr4_vgpr5 = IMPLICIT_DEF
@@ -533,47 +532,46 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
   ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
-  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
   ; GFX90A-NEXT:   $sgpr42_sgpr43 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.46, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.42.Flow24:
   ; GFX90A-NEXT:   successors: %bb.40(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr15, $vgpr17, $vgpr18, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr18, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr42_sgpr43, implicit-def $scc
   ; GFX90A-NEXT:   renamable $vgpr59 = COPY killed renamable $vgpr18, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_XOR_B64 $exec, -1, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_AND_B64 killed renamable $sgpr44_sgpr45, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_AND_B64 killed renamable $sgpr62_sgpr63, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr48_sgpr49, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr62_sgpr63 = S_AND_B64 killed renamable $sgpr44_sgpr45, $exec, implicit-def dead $scc
   ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = S_AND_B64 killed renamable $sgpr60_sgpr61, $exec, implicit-def dead $scc
   ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_AND_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
   ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_AND_B64 killed renamable $sgpr18_sgpr19, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = S_AND_B64 killed renamable $sgpr60_sgpr61, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = S_OR_B64 killed renamable $sgpr56_sgpr57, killed renamable $sgpr60_sgpr61, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr48_sgpr49, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_OR_B64 killed renamable $sgpr46_sgpr47, killed renamable $sgpr48_sgpr49, implicit-def dead $scc
   ; GFX90A-NEXT:   S_BRANCH %bb.40
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.43.bb55:
   ; GFX90A-NEXT:   successors: %bb.48(0x40000000), %bb.44(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr44_sgpr45, $sgpr52_sgpr53, $sgpr58_sgpr59, $sgpr54_sgpr55, $sgpr46_sgpr47
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr46_sgpr47:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr44_sgpr45, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr56_sgpr57, $sgpr48_sgpr49
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   S_BITCMP1_B32 killed renamable $sgpr33, 16, implicit-def $scc
+  ; GFX90A-NEXT:   S_BITCMP1_B32 killed renamable $sgpr17, 16, implicit-def $scc
   ; GFX90A-NEXT:   renamable $sgpr64_sgpr65 = S_CSELECT_B64 -1, 0, implicit killed $scc
-  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_XOR_B64 renamable $sgpr64_sgpr65, -1, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_XOR_B64 renamable $sgpr64_sgpr65, -1, implicit-def dead $scc
   ; GFX90A-NEXT:   renamable $vgpr62 = V_ADD_CO_U32_e32 6144, $vgpr40, implicit-def $vcc, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr63, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, renamable $sgpr48_sgpr49, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, renamable $sgpr50_sgpr51, implicit-def dead $scc
   ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.48, implicit $vcc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.44:
   ; GFX90A-NEXT:   successors: %bb.45(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr56, $vgpr47, $vgpr18, $vgpr30, $vgpr31, $vgpr58, $vgpr61, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8, $sgpr9, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $vgpr57, $vgpr63, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr20_sgpr21_sgpr22, $sgpr22_sgpr23, $sgpr24_sgpr25_sgpr26, $sgpr26_sgpr27, $vgpr46, $vgpr45, $vgpr2, $vgpr3, $vgpr44, $vgpr43, $vgpr42, $vgpr41, $vgpr40, $vgpr60, $vgpr62
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr57, $vgpr56, $vgpr18, $vgpr30, $vgpr31, $vgpr60, $vgpr62, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8, $sgpr9, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $vgpr61, $vgpr58, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr58_sgpr59, $sgpr20_sgpr21_sgpr22, $sgpr22_sgpr23, $sgpr24_sgpr25_sgpr26, $sgpr26_sgpr27, $vgpr47, $vgpr46, $vgpr2, $vgpr3, $vgpr45, $vgpr44, $vgpr43, $vgpr42, $vgpr41, $vgpr40, $vgpr63
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = COPY renamable $sgpr36_sgpr37
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = COPY renamable $sgpr36_sgpr37
   ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr6_vgpr7 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr4_vgpr5 = IMPLICIT_DEF
@@ -586,37 +584,36 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
   ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
-  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_MOV_B64 0
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.45.Flow26:
   ; GFX90A-NEXT:   successors: %bb.47(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr15, $vgpr17, $vgpr18, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr18, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr58_sgpr59, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = S_XOR_B64 $exec, -1, implicit-def dead $scc
   ; GFX90A-NEXT:   renamable $sgpr70_sgpr71 = S_AND_B64 killed renamable $sgpr44_sgpr45, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = S_AND_B64 killed renamable $sgpr46_sgpr47, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr66_sgpr67 = S_AND_B64 killed renamable $sgpr48_sgpr49, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_AND_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = S_AND_B64 killed renamable $sgpr48_sgpr49, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr66_sgpr67 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr64_sgpr65 = S_AND_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
   ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr64_sgpr65 = S_OR_B64 killed renamable $sgpr44_sgpr45, killed renamable $sgpr48_sgpr49, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr62_sgpr63 = S_OR_B64 killed renamable $sgpr44_sgpr45, killed renamable $sgpr46_sgpr47, implicit-def dead $scc
   ; GFX90A-NEXT:   S_BRANCH %bb.47
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.46.bb48:
   ; GFX90A-NEXT:   successors: %bb.43(0x40000000), %bb.47(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr46_sgpr47, $sgpr58_sgpr59, $sgpr54_sgpr55, $sgpr44_sgpr45, $sgpr52_sgpr53
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr46_sgpr47:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr64_sgpr65, $sgpr50_sgpr51, $sgpr66_sgpr67, $sgpr44_sgpr45, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr56_sgpr57
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr60 = V_ADD_CO_U32_e32 5120, $vgpr40, implicit-def $vcc, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = COPY $vcc
   ; GFX90A-NEXT:   renamable $vgpr0 = V_ADD_CO_U32_e32 4096, $vgpr40, implicit-def $vcc, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr1, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr0 = GLOBAL_LOAD_UBYTE killed renamable $vgpr0_vgpr1, 1024, 0, implicit $exec :: (load (s8) from %ir.i49, addrspace 1)
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = S_MOV_B64 -1
-  ; GFX90A-NEXT:   renamable $sgpr64_sgpr65 = COPY renamable $sgpr36_sgpr37
-  ; GFX90A-NEXT:   renamable $sgpr66_sgpr67 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr62_sgpr63 = COPY renamable $sgpr36_sgpr37
   ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $vgpr61, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $sgpr18_sgpr19, 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr0, implicit $exec
@@ -634,57 +631,56 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
   ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
-  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
   ; GFX90A-NEXT:   $sgpr18_sgpr19 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.43, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.47.Flow25:
   ; GFX90A-NEXT:   successors: %bb.42(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr15, $vgpr17, $vgpr18, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr46_sgpr47, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr70_sgpr71, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr18, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr70_sgpr71, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr18_sgpr19, implicit-def $scc
   ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_XOR_B64 $exec, -1, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr62_sgpr63 = S_AND_B64 killed renamable $sgpr60_sgpr61, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr70_sgpr71, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr66_sgpr67, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_AND_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_AND_B64 killed renamable $sgpr46_sgpr47, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr60_sgpr61, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = S_AND_B64 killed renamable $sgpr70_sgpr71, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_AND_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr66_sgpr67, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr64_sgpr65, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_AND_B64 killed renamable $sgpr48_sgpr49, $exec, implicit-def dead $scc
   ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr64_sgpr65, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = S_OR_B64 killed renamable $sgpr46_sgpr47, killed renamable $sgpr56_sgpr57, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr62_sgpr63, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_OR_B64 killed renamable $sgpr46_sgpr47, killed renamable $sgpr48_sgpr49, implicit-def dead $scc
   ; GFX90A-NEXT:   S_BRANCH %bb.42
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.48.bb63:
   ; GFX90A-NEXT:   successors: %bb.50(0x40000000), %bb.49(0x40000000)
-  ; GFX90A-NEXT:   liveins: $vcc, $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr48_sgpr49, $sgpr56_sgpr57:0x000000000000000F, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr52_sgpr53, $sgpr58_sgpr59, $sgpr54_sgpr55, $sgpr46_sgpr47
+  ; GFX90A-NEXT:   liveins: $vcc, $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr46_sgpr47:0x000000000000000F, $sgpr50_sgpr51, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr56_sgpr57, $sgpr48_sgpr49
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_MOV_B64 0
   ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.50, implicit $vcc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.49:
   ; GFX90A-NEXT:   successors: %bb.44(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr52_sgpr53, $sgpr58_sgpr59, $sgpr54_sgpr55
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr56_sgpr57
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_MOV_B64 -1
   ; GFX90A-NEXT:   S_BRANCH %bb.44
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.50.bb68:
   ; GFX90A-NEXT:   successors: %bb.54(0x40000000), %bb.51(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr56_sgpr57:0x000000000000000F, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr46_sgpr47, $sgpr52_sgpr53, $sgpr58_sgpr59, $sgpr54_sgpr55
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47:0x000000000000000F, $sgpr50_sgpr51, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr56_sgpr57
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr0 = nuw nsw V_LSHLREV_B32_e32 3, $vgpr30, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-  ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr48_sgpr49, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr50_sgpr51, implicit-def dead $scc
   ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.54, implicit $vcc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.51:
   ; GFX90A-NEXT:   successors: %bb.45(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr52_sgpr53, $sgpr58_sgpr59, $sgpr54_sgpr55
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr56_sgpr57
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_MOV_B64 -1
-  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = COPY renamable $sgpr36_sgpr37
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = COPY renamable $sgpr36_sgpr37
   ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr6_vgpr7 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr4_vgpr5 = IMPLICIT_DEF
@@ -696,25 +692,24 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
   ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
-  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
   ; GFX90A-NEXT:   S_BRANCH %bb.45
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.52.bb80:
   ; GFX90A-NEXT:   successors: %bb.59(0x40000000), %bb.53(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr56_sgpr57:0x000000000000000F, $sgpr60_sgpr61, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47:0x000000000000000F, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr60_sgpr61, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $sgpr17 = S_BFE_U32 renamable $sgpr20, 65560, implicit-def dead $scc
   ; GFX90A-NEXT:   S_CMP_EQ_U32 killed renamable $sgpr17, 0, implicit-def $scc
   ; GFX90A-NEXT:   renamable $vgpr6 = V_ADD_CO_U32_e32 4096, $vgpr0, implicit-def $vcc, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr7, dead renamable $sgpr50_sgpr51 = V_ADDC_U32_e64 0, 0, killed $vcc, 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr7, dead renamable $sgpr52_sgpr53 = V_ADDC_U32_e64 0, 0, killed $vcc, 0, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_SCC1 %bb.59, implicit killed $scc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.53:
   ; GFX90A-NEXT:   successors: %bb.61(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr60_sgpr61, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr60_sgpr61, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_MOV_B64 0
-  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_MOV_B64 -1
   ; GFX90A-NEXT:   renamable $sgpr62_sgpr63 = COPY renamable $sgpr36_sgpr37
   ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr17 = IMPLICIT_DEF
@@ -725,18 +720,17 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
   ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
-  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
   ; GFX90A-NEXT:   S_BRANCH %bb.61
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.54.bb73:
   ; GFX90A-NEXT:   successors: %bb.52(0x40000000), %bb.55(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr56_sgpr57:0x000000000000000F, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr52_sgpr53
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47:0x000000000000000F, $sgpr48_sgpr49, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr54_sgpr55
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr6 = GLOBAL_LOAD_UBYTE renamable $vgpr0_vgpr1, 2048, 0, implicit $exec :: (load (s8) from %ir.i74, addrspace 1)
   ; GFX90A-NEXT:   renamable $vgpr4 = V_ADD_CO_U32_e32 2048, $vgpr0, implicit-def $vcc, implicit $exec
-  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_MOV_B64 0
-  ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_MOV_B64 -1
-  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = COPY renamable $sgpr36_sgpr37
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = COPY renamable $sgpr36_sgpr37
   ; GFX90A-NEXT:   renamable $vgpr5, dead renamable $sgpr58_sgpr59 = V_ADDC_U32_e64 0, 0, killed $vcc, 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr6, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_MOV_B64 0
@@ -750,20 +744,19 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
   ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
-  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
   ; GFX90A-NEXT:   $sgpr60_sgpr61 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.52, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.55.Flow29:
   ; GFX90A-NEXT:   successors: %bb.45(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr15, $vgpr17, $vgpr18, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr18, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr60_sgpr61, implicit-def $scc
   ; GFX90A-NEXT:   S_BRANCH %bb.45
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.56.bb90:
   ; GFX90A-NEXT:   successors: %bb.60(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr52_sgpr53, $sgpr56_sgpr57:0x000000000000000F, $sgpr60_sgpr61, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47:0x000000000000000F, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr53 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $sgpr64_sgpr65, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr10 = V_MOV_B32_e32 0, implicit $exec
@@ -772,31 +765,29 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr16_vgpr17 = DS_READ_B64_gfx9 killed renamable $vgpr10, 0, 0, implicit $exec :: (load (s64) from %ir.7, addrspace 3)
   ; GFX90A-NEXT:   renamable $vgpr10 = COPY renamable $sgpr22, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr12_vgpr13 = DS_READ_B64_gfx9 killed renamable $vgpr10, 0, 0, implicit $exec :: (load (s64) from %ir.8, addrspace 3)
-  ; GFX90A-NEXT:   renamable $vgpr10 = COPY renamable $sgpr56, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr11 = V_ALIGNBIT_B32_e64 killed $sgpr57, killed $vgpr10, 1, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr10 = COPY renamable $sgpr46, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr11 = V_ALIGNBIT_B32_e64 killed $sgpr47, killed $vgpr10, 1, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr52 = V_ALIGNBIT_B32_e64 $vgpr17, $vgpr16, 1, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr17 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr12_sgpr13, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr15 = V_ALIGNBIT_B32_e64 $vgpr15, $vgpr14, 1, implicit $exec
-  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_XOR_B64 $exec, -1, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_XOR_B64 $exec, -1, implicit-def dead $scc
   ; GFX90A-NEXT:   renamable $sgpr62_sgpr63 = S_OR_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
   ; GFX90A-NEXT:   renamable $vgpr10 = COPY renamable $vgpr14, implicit $exec
   ; GFX90A-NEXT:   S_BRANCH %bb.60
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.57:
   ; GFX90A-NEXT:   successors: %bb.7(0x80000000)
-  ; GFX90A-NEXT:   liveins: $exec, $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr23, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr18_vgpr19:0x000000000000000F, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr18_vgpr19:0x000000000000000F, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   renamable $vgpr15 = COPY killed renamable $sgpr23, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr17 = COPY killed renamable $sgpr17, implicit $exec
-  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr40_sgpr41 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr66_sgpr67 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr64_sgpr65 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_MOV_B64 0
-  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_MOV_B64 0
-  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_MOV_B64 0
-  ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_MOV_B64 0
-  ; GFX90A-NEXT:   renamable $sgpr40_sgpr41 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $sgpr38_sgpr39 = S_MOV_B64 0
   ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr6_vgpr7 = IMPLICIT_DEF
@@ -810,43 +801,40 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr42_vgpr43 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr40_vgpr41 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr46_vgpr47 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   renamable $vgpr14 = COPY renamable $vgpr15, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr52 = COPY renamable $vgpr15, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr16 = COPY renamable $vgpr15, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr53 = COPY renamable $vgpr17, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr13 = COPY renamable $vgpr15, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr12 = COPY renamable $vgpr15, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr14 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr52 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr16 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr53 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr13 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr12 = V_MOV_B32_e32 0, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr34_sgpr35 = S_MOV_B64 0
   ; GFX90A-NEXT:   S_BRANCH %bb.7
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.58.bb105:
   ; GFX90A-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x00000000000000FF, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr46_sgpr47:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x00000000000000FF, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr22_vgpr23 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
   ; GFX90A-NEXT:   renamable $vgpr0 = COPY renamable $sgpr23, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr20_vgpr21 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.434, addrspace 3)
+  ; GFX90A-NEXT:   renamable $vgpr20_vgpr21 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.419, addrspace 3)
   ; GFX90A-NEXT:   renamable $vgpr0 = COPY renamable $sgpr21, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr18_vgpr19 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.7, addrspace 3)
-  ; GFX90A-NEXT:   renamable $vgpr0 = COPY killed renamable $sgpr17, implicit $exec
-  ; GFX90A-NEXT:   renamable $vgpr10_vgpr11 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.435, addrspace 3)
+  ; GFX90A-NEXT:   renamable $vgpr0 = COPY killed renamable $sgpr33, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr10_vgpr11 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.420, addrspace 3)
   ; GFX90A-NEXT:   renamable $vgpr0 = COPY renamable $sgpr22, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr24_vgpr25 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.8, addrspace 3)
   ; GFX90A-NEXT:   renamable $sgpr36_sgpr37 = S_MOV_B64 -1
-  ; GFX90A-NEXT:   renamable $sgpr23 = S_MOV_B32 0
-  ; GFX90A-NEXT:   renamable $sgpr17 = S_MOV_B32 0
   ; GFX90A-NEXT:   S_BRANCH %bb.3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.59.bb85:
   ; GFX90A-NEXT:   successors: %bb.56(0x40000000), %bb.60(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr56_sgpr57:0x000000000000000F, $sgpr60_sgpr61, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47:0x000000000000000F, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr60_sgpr61, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr8 = V_OR_B32_e32 1, $vgpr6, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr9 = COPY renamable $vgpr7, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr10 = FLAT_LOAD_UBYTE renamable $vgpr8_vgpr9, 0, 0, implicit $exec, implicit $flat_scr :: (load (s8) from %ir.i86)
-  ; GFX90A-NEXT:   renamable $sgpr17 = S_MOV_B32 0
-  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_MOV_B64 -1
   ; GFX90A-NEXT:   renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr10, implicit $exec
   ; GFX90A-NEXT:   renamable $sgpr62_sgpr63 = COPY renamable $sgpr36_sgpr37
   ; GFX90A-NEXT:   renamable $vgpr17 = IMPLICIT_DEF
@@ -857,31 +845,31 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT:   renamable $vgpr53 = IMPLICIT_DEF
   ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF implicit-def $vgpr12
   ; GFX90A-NEXT:   renamable $vgpr11 = IMPLICIT_DEF implicit-def $vgpr10
-  ; GFX90A-NEXT:   $sgpr52_sgpr53 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   $sgpr54_sgpr55 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.56, implicit $exec
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.60.Flow31:
   ; GFX90A-NEXT:   successors: %bb.61(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr15, $vgpr17, $vgpr18, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr18, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr52_sgpr53, implicit-def $scc
-  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_MOV_B64 0
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr54_sgpr55, implicit-def $scc
+  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_MOV_B64 0
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.61.Flow30:
   ; GFX90A-NEXT:   successors: %bb.55(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr15, $vgpr17, $vgpr18, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
-  ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_XOR_B64 $exec, -1, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr62_sgpr63, $exec, implicit-def dead $scc
-  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_OR_B64 killed renamable $sgpr50_sgpr51, killed renamable $sgpr56_sgpr57, implicit-def dead $scc
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr18, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_XOR_B64 $exec, -1, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_AND_B64 killed renamable $sgpr46_sgpr47, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr62_sgpr63, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_OR_B64 killed renamable $sgpr46_sgpr47, killed renamable $sgpr52_sgpr53, implicit-def dead $scc
   ; GFX90A-NEXT:   S_BRANCH %bb.55
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.62.bb140:
   ; GFX90A-NEXT:   successors: %bb.68(0x40000000), %bb.63(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x000000000000000F, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x000000000000000F, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $sgpr36_sgpr37 = S_MOV_B64 -1
   ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr30_sgpr31, implicit-def dead $scc
@@ -889,14 +877,14 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.63.Flow13:
   ; GFX90A-NEXT:   successors: %bb.64(0x40000000), %bb.66(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000C, $vgpr12_vgpr13:0x000000000000000C, $vgpr18_vgpr19:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000C, $vgpr12_vgpr13:0x000000000000000C, $vgpr18_vgpr19:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $vcc = S_ANDN2_B64 $exec, killed renamable $sgpr36_sgpr37, implicit-def dead $scc
   ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.66, implicit $vcc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.64.bb159:
   ; GFX90A-NEXT:   successors: %bb.67(0x40000000), %bb.65(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000C, $vgpr12_vgpr13:0x000000000000000C, $vgpr18_vgpr19:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000C, $vgpr12_vgpr13:0x000000000000000C, $vgpr18_vgpr19:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vcc = V_CMP_NE_U32_e64 0, killed $vgpr30, implicit $exec
   ; GFX90A-NEXT:   $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
@@ -905,21 +893,21 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.65.Flow10:
   ; GFX90A-NEXT:   successors: %bb.66(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $sgpr12_sgpr13 = S_ANDN2_SAVEEXEC_B64 $sgpr12_sgpr13, implicit-def $exec, implicit-def $scc, implicit $exec
   ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def $scc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.66.Flow14:
   ; GFX90A-NEXT:   successors: %bb.8(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = COPY $exec
+  ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = COPY $exec
   ; GFX90A-NEXT:   S_BRANCH %bb.8
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.67.bb161:
   ; GFX90A-NEXT:   successors: %bb.65(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000C, $vgpr12_vgpr13:0x000000000000000C, $vgpr18_vgpr19:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000C, $vgpr12_vgpr13:0x000000000000000C, $vgpr18_vgpr19:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr2 = V_OR_B32_e32 killed $vgpr21, killed $vgpr23, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr2 = V_OR_B32_e32 killed $vgpr2, killed $vgpr25, implicit $exec
@@ -938,7 +926,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.68.bb174:
   ; GFX90A-NEXT:   successors: %bb.72(0x40000000), %bb.69(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x000000000000000F, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x000000000000000F, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr26 = V_OR_B32_e32 1, $vgpr24, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr48 = V_OR_B32_e32 $vgpr26, $vgpr22, implicit $exec
@@ -954,14 +942,14 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.69.Flow:
   ; GFX90A-NEXT:   successors: %bb.70(0x40000000), %bb.71(0x40000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000C, $vgpr12_vgpr13:0x000000000000000C, $vgpr18_vgpr19:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x0000000000000003, $vgpr28_vgpr29:0x0000000000000003, $vgpr32_vgpr33:0x0000000000000003, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000C, $vgpr12_vgpr13:0x000000000000000C, $vgpr18_vgpr19:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x0000000000000003, $vgpr28_vgpr29:0x0000000000000003, $vgpr32_vgpr33:0x0000000000000003, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   $vcc = S_ANDN2_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
   ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.71, implicit $vcc
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.70.bb186:
   ; GFX90A-NEXT:   successors: %bb.71(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000C, $vgpr12_vgpr13:0x000000000000000C, $vgpr18_vgpr19:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x0000000000000003, $vgpr28_vgpr29:0x0000000000000003, $vgpr32_vgpr33:0x0000000000000003, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000C, $vgpr12_vgpr13:0x000000000000000C, $vgpr18_vgpr19:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x0000000000000003, $vgpr28_vgpr29:0x0000000000000003, $vgpr32_vgpr33:0x0000000000000003, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr2_vgpr3 = V_LSHLREV_B64_e64 3, killed $vgpr2_vgpr3, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr10 = COPY renamable $sgpr27, implicit $exec
@@ -990,14 +978,14 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.71.Flow9:
   ; GFX90A-NEXT:   successors: %bb.63(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000C, $vgpr12_vgpr13:0x000000000000000C, $vgpr18_vgpr19:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000C, $vgpr12_vgpr13:0x000000000000000C, $vgpr18_vgpr19:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $sgpr36_sgpr37 = S_MOV_B64 0
   ; GFX90A-NEXT:   S_BRANCH %bb.63
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.72.bb196:
   ; GFX90A-NEXT:   successors: %bb.69(0x80000000)
-  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000C, $vgpr12_vgpr13:0x000000000000000C, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x0000000000000003, $vgpr28_vgpr29:0x0000000000000003, $vgpr32_vgpr33:0x0000000000000003, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000C, $vgpr12_vgpr13:0x000000000000000C, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x0000000000000003, $vgpr28_vgpr29:0x0000000000000003, $vgpr32_vgpr33:0x0000000000000003, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   renamable $vgpr10 = V_OR_B32_e32 $vgpr50, killed $vgpr16, implicit $exec
   ; GFX90A-NEXT:   renamable $vgpr54 = V_OR_B32_e32 killed $vgpr10, killed $vgpr14, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/branch-relax-spill.ll b/llvm/test/CodeGen/AMDGPU/branch-relax-spill.ll
index 05c2e0077f4ae..83ab6c32aee96 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-relax-spill.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-relax-spill.ll
@@ -900,9 +900,8 @@ define void @spill_func(ptr addrspace(1) %arg) #0 {
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_xor_saveexec_b64 s[4:5], -1
 ; CHECK-NEXT:    buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill
-; CHECK-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
 ; CHECK-NEXT:    s_mov_b64 exec, s[4:5]
-; CHECK-NEXT:    s_waitcnt expcnt(1)
+; CHECK-NEXT:    s_waitcnt expcnt(0)
 ; CHECK-NEXT:    v_writelane_b32 v0, s30, 0
 ; CHECK-NEXT:    v_writelane_b32 v0, s31, 1
 ; CHECK-NEXT:    v_writelane_b32 v0, s33, 2
@@ -912,71 +911,38 @@ define void @spill_func(ptr addrspace(1) %arg) #0 {
 ; CHECK-NEXT:    v_writelane_b32 v0, s37, 6
 ; CHECK-NEXT:    v_writelane_b32 v0, s38, 7
 ; CHECK-NEXT:    v_writelane_b32 v0, s39, 8
-; CHECK-NEXT:    v_writelane_b32 v0, s40, 9
-; CHECK-NEXT:    v_writelane_b32 v0, s41, 10
-; CHECK-NEXT:    v_writelane_b32 v0, s42, 11
-; CHECK-NEXT:    v_writelane_b32 v0, s43, 12
-; CHECK-NEXT:    v_writelane_b32 v0, s44, 13
-; CHECK-NEXT:    v_writelane_b32 v0, s45, 14
-; CHECK-NEXT:    v_writelane_b32 v0, s46, 15
-; CHECK-NEXT:    v_writelane_b32 v0, s47, 16
-; CHECK-NEXT:    v_writelane_b32 v0, s48, 17
-; CHECK-NEXT:    v_writelane_b32 v0, s49, 18
-; CHECK-NEXT:    v_writelane_b32 v0, s50, 19
-; CHECK-NEXT:    v_writelane_b32 v0, s51, 20
-; CHECK-NEXT:    v_writelane_b32 v0, s52, 21
-; CHECK-NEXT:    v_writelane_b32 v0, s53, 22
-; CHECK-NEXT:    v_writelane_b32 v0, s54, 23
-; CHECK-NEXT:    v_writelane_b32 v0, s55, 24
-; CHECK-NEXT:    v_writelane_b32 v0, s56, 25
-; CHECK-NEXT:    v_writelane_b32 v0, s57, 26
-; CHECK-NEXT:    v_writelane_b32 v0, s58, 27
-; CHECK-NEXT:    v_writelane_b32 v0, s59, 28
-; CHECK-NEXT:    v_writelane_b32 v0, s60, 29
-; CHECK-NEXT:    v_writelane_b32 v0, s61, 30
-; CHECK-NEXT:    v_writelane_b32 v0, s62, 31
-; CHECK-NEXT:    v_writelane_b32 v0, s63, 32
-; CHECK-NEXT:    v_writelane_b32 v0, s64, 33
-; CHECK-NEXT:    v_writelane_b32 v0, s65, 34
-; CHECK-NEXT:    v_writelane_b32 v0, s66, 35
-; CHECK-NEXT:    v_writelane_b32 v0, s67, 36
-; CHECK-NEXT:    v_writelane_b32 v0, s68, 37
-; CHECK-NEXT:    v_writelane_b32 v0, s69, 38
-; CHECK-NEXT:    v_writelane_b32 v0, s70, 39
-; CHECK-NEXT:    v_writelane_b32 v0, s71, 40
-; CHECK-NEXT:    v_writelane_b32 v0, s72, 41
-; CHECK-NEXT:    v_writelane_b32 v0, s73, 42
-; CHECK-NEXT:    v_writelane_b32 v0, s74, 43
-; CHECK-NEXT:    v_writelane_b32 v0, s75, 44
-; CHECK-NEXT:    v_writelane_b32 v0, s76, 45
-; CHECK-NEXT:    v_writelane_b32 v0, s77, 46
-; CHECK-NEXT:    v_writelane_b32 v0, s78, 47
-; CHECK-NEXT:    v_writelane_b32 v0, s79, 48
-; CHECK-NEXT:    v_writelane_b32 v0, s80, 49
-; CHECK-NEXT:    v_writelane_b32 v0, s81, 50
-; CHECK-NEXT:    v_writelane_b32 v0, s82, 51
-; CHECK-NEXT:    v_writelane_b32 v0, s83, 52
-; CHECK-NEXT:    v_writelane_b32 v0, s84, 53
-; CHECK-NEXT:    v_writelane_b32 v0, s85, 54
-; CHECK-NEXT:    v_writelane_b32 v0, s86, 55
-; CHECK-NEXT:    v_writelane_b32 v0, s87, 56
-; CHECK-NEXT:    v_writelane_b32 v0, s88, 57
-; CHECK-NEXT:    s_waitcnt expcnt(0)
-; CHECK-NEXT:    v_writelane_b32 v1, s95, 0
-; CHECK-NEXT:    v_writelane_b32 v0, s89, 58
-; CHECK-NEXT:    v_writelane_b32 v1, s96, 1
-; CHECK-NEXT:    v_writelane_b32 v0, s90, 59
-; CHECK-NEXT:    v_writelane_b32 v1, s97, 2
-; CHECK-NEXT:    v_writelane_b32 v0, s91, 60
-; CHECK-NEXT:    v_writelane_b32 v1, s98, 3
-; CHECK-NEXT:    v_writelane_b32 v0, s92, 61
-; CHECK-NEXT:    v_writelane_b32 v1, s99, 4
-; CHECK-NEXT:    s_mov_b32 s31, s12
-; CHECK-NEXT:    v_writelane_b32 v0, s93, 62
-; CHECK-NEXT:    v_writelane_b32 v1, s100, 5
-; CHECK-NEXT:    s_cmp_eq_u32 s31, 0
-; CHECK-NEXT:    v_writelane_b32 v0, s94, 63
-; CHECK-NEXT:    v_writelane_b32 v1, s101, 6
+; CHECK-NEXT:    v_writelane_b32 v0, s48, 9
+; CHECK-NEXT:    v_writelane_b32 v0, s49, 10
+; CHECK-NEXT:    v_writelane_b32 v0, s50, 11
+; CHECK-NEXT:    v_writelane_b32 v0, s51, 12
+; CHECK-NEXT:    v_writelane_b32 v0, s52, 13
+; CHECK-NEXT:    v_writelane_b32 v0, s53, 14
+; CHECK-NEXT:    v_writelane_b32 v0, s54, 15
+; CHECK-NEXT:    v_writelane_b32 v0, s55, 16
+; CHECK-NEXT:    v_writelane_b32 v0, s64, 17
+; CHECK-NEXT:    v_writelane_b32 v0, s65, 18
+; CHECK-NEXT:    v_writelane_b32 v0, s66, 19
+; CHECK-NEXT:    v_writelane_b32 v0, s67, 20
+; CHECK-NEXT:    v_writelane_b32 v0, s68, 21
+; CHECK-NEXT:    v_writelane_b32 v0, s69, 22
+; CHECK-NEXT:    v_writelane_b32 v0, s70, 23
+; CHECK-NEXT:    v_writelane_b32 v0, s71, 24
+; CHECK-NEXT:    v_writelane_b32 v0, s80, 25
+; CHECK-NEXT:    v_writelane_b32 v0, s81, 26
+; CHECK-NEXT:    v_writelane_b32 v0, s82, 27
+; CHECK-NEXT:    v_writelane_b32 v0, s83, 28
+; CHECK-NEXT:    v_writelane_b32 v0, s84, 29
+; CHECK-NEXT:    v_writelane_b32 v0, s85, 30
+; CHECK-NEXT:    v_writelane_b32 v0, s86, 31
+; CHECK-NEXT:    v_writelane_b32 v0, s87, 32
+; CHECK-NEXT:    v_writelane_b32 v0, s96, 33
+; CHECK-NEXT:    v_writelane_b32 v0, s97, 34
+; CHECK-NEXT:    v_writelane_b32 v0, s98, 35
+; CHECK-NEXT:    v_writelane_b32 v0, s99, 36
+; CHECK-NEXT:    s_mov_b32 s40, s12
+; CHECK-NEXT:    v_writelane_b32 v0, s100, 37
+; CHECK-NEXT:    s_cmp_eq_u32 s40, 0
+; CHECK-NEXT:    v_writelane_b32 v0, s101, 38
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    s_mov_b32 s0, 0
 ; CHECK-NEXT:    ;;#ASMEND
@@ -1292,9 +1258,9 @@ define void @spill_func(ptr addrspace(1) %arg) #0 {
 ; CHECK-NEXT:    s_cbranch_scc0 .LBB1_1
 ; CHECK-NEXT:  ; %bb.3: ; %entry
 ; CHECK-NEXT:    s_not_b64 exec, exec
-; CHECK-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:8
-; CHECK-NEXT:    v_writelane_b32 v2, s0, 0
-; CHECK-NEXT:    v_writelane_b32 v2, s1, 1
+; CHECK-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:4
+; CHECK-NEXT:    v_writelane_b32 v1, s0, 0
+; CHECK-NEXT:    v_writelane_b32 v1, s1, 1
 ; CHECK-NEXT:    s_getpc_b64 s[0:1]
 ; CHECK-NEXT:  .Lpost_getpc1:
 ; CHECK-NEXT:    s_add_u32 s0, s0, (.LBB1_4-.Lpost_getpc1)&4294967295
@@ -1313,9 +1279,9 @@ define void @spill_func(ptr addrspace(1) %arg) #0 {
 ; CHECK-NEXT:    ;;#ASMEND
 ; CHECK-NEXT:    s_branch .LBB1_2
 ; CHECK-NEXT:  .LBB1_4: ; %bb3
-; CHECK-NEXT:    v_readlane_b32 s0, v2, 0
-; CHECK-NEXT:    v_readlane_b32 s1, v2, 1
-; CHECK-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8
+; CHECK-NEXT:    v_readlane_b32 s0, v1, 0
+; CHECK-NEXT:    v_readlane_b32 s1, v1, 1
+; CHECK-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:4
 ; CHECK-NEXT:    s_not_b64 exec, exec
 ; CHECK-NEXT:  .LBB1_2: ; %bb3
 ; CHECK-NEXT:    ;;#ASMSTART
@@ -1630,68 +1596,36 @@ define void @spill_func(ptr addrspace(1) %arg) #0 {
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ; reg use vcc_hi
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_readlane_b32 s101, v1, 6
-; CHECK-NEXT:    v_readlane_b32 s100, v1, 5
-; CHECK-NEXT:    v_readlane_b32 s99, v1, 4
-; CHECK-NEXT:    v_readlane_b32 s98, v1, 3
-; CHECK-NEXT:    v_readlane_b32 s97, v1, 2
-; CHECK-NEXT:    v_readlane_b32 s96, v1, 1
-; CHECK-NEXT:    v_readlane_b32 s95, v1, 0
-; CHECK-NEXT:    v_readlane_b32 s94, v0, 63
-; CHECK-NEXT:    v_readlane_b32 s93, v0, 62
-; CHECK-NEXT:    v_readlane_b32 s92, v0, 61
-; CHECK-NEXT:    v_readlane_b32 s91, v0, 60
-; CHECK-NEXT:    v_readlane_b32 s90, v0, 59
-; CHECK-NEXT:    v_readlane_b32 s89, v0, 58
-; CHECK-NEXT:    v_readlane_b32 s88, v0, 57
-; CHECK-NEXT:    v_readlane_b32 s87, v0, 56
-; CHECK-NEXT:    v_readlane_b32 s86, v0, 55
-; CHECK-NEXT:    v_readlane_b32 s85, v0, 54
-; CHECK-NEXT:    v_readlane_b32 s84, v0, 53
-; CHECK-NEXT:    v_readlane_b32 s83, v0, 52
-; CHECK-NEXT:    v_readlane_b32 s82, v0, 51
-; CHECK-NEXT:    v_readlane_b32 s81, v0, 50
-; CHECK-NEXT:    v_readlane_b32 s80, v0, 49
-; CHECK-NEXT:    v_readlane_b32 s79, v0, 48
-; CHECK-NEXT:    v_readlane_b32 s78, v0, 47
-; CHECK-NEXT:    v_readlane_b32 s77, v0, 46
-; CHECK-NEXT:    v_readlane_b32 s76, v0, 45
-; CHECK-NEXT:    v_readlane_b32 s75, v0, 44
-; CHECK-NEXT:    v_readlane_b32 s74, v0, 43
-; CHECK-NEXT:    v_readlane_b32 s73, v0, 42
-; CHECK-NEXT:    v_readlane_b32 s72, v0, 41
-; CHECK-NEXT:    v_readlane_b32 s71, v0, 40
-; CHECK-NEXT:    v_readlane_b32 s70, v0, 39
-; CHECK-NEXT:    v_readlane_b32 s69, v0, 38
-; CHECK-NEXT:    v_readlane_b32 s68, v0, 37
-; CHECK-NEXT:    v_readlane_b32 s67, v0, 36
-; CHECK-NEXT:    v_readlane_b32 s66, v0, 35
-; CHECK-NEXT:    v_readlane_b32 s65, v0, 34
-; CHECK-NEXT:    v_readlane_b32 s64, v0, 33
-; CHECK-NEXT:    v_readlane_b32 s63, v0, 32
-; CHECK-NEXT:    v_readlane_b32 s62, v0, 31
-; CHECK-NEXT:    v_readlane_b32 s61, v0, 30
-; CHECK-NEXT:    v_readlane_b32 s60, v0, 29
-; CHECK-NEXT:    v_readlane_b32 s59, v0, 28
-; CHECK-NEXT:    v_readlane_b32 s58, v0, 27
-; CHECK-NEXT:    v_readlane_b32 s57, v0, 26
-; CHECK-NEXT:    v_readlane_b32 s56, v0, 25
-; CHECK-NEXT:    v_readlane_b32 s55, v0, 24
-; CHECK-NEXT:    v_readlane_b32 s54, v0, 23
-; CHECK-NEXT:    v_readlane_b32 s53, v0, 22
-; CHECK-NEXT:    v_readlane_b32 s52, v0, 21
-; CHECK-NEXT:    v_readlane_b32 s51, v0, 20
-; CHECK-NEXT:    v_readlane_b32 s50, v0, 19
-; CHECK-NEXT:    v_readlane_b32 s49, v0, 18
-; CHECK-NEXT:    v_readlane_b32 s48, v0, 17
-; CHECK-NEXT:    v_readlane_b32 s47, v0, 16
-; CHECK-NEXT:    v_readlane_b32 s46, v0, 15
-; CHECK-NEXT:    v_readlane_b32 s45, v0, 14
-; CHECK-NEXT:    v_readlane_b32 s44, v0, 13
-; CHECK-NEXT:    v_readlane_b32 s43, v0, 12
-; CHECK-NEXT:    v_readlane_b32 s42, v0, 11
-; CHECK-NEXT:    v_readlane_b32 s41, v0, 10
-; CHECK-NEXT:    v_readlane_b32 s40, v0, 9
+; CHECK-NEXT:    v_readlane_b32 s101, v0, 38
+; CHECK-NEXT:    v_readlane_b32 s100, v0, 37
+; CHECK-NEXT:    v_readlane_b32 s99, v0, 36
+; CHECK-NEXT:    v_readlane_b32 s98, v0, 35
+; CHECK-NEXT:    v_readlane_b32 s97, v0, 34
+; CHECK-NEXT:    v_readlane_b32 s96, v0, 33
+; CHECK-NEXT:    v_readlane_b32 s87, v0, 32
+; CHECK-NEXT:    v_readlane_b32 s86, v0, 31
+; CHECK-NEXT:    v_readlane_b32 s85, v0, 30
+; CHECK-NEXT:    v_readlane_b32 s84, v0, 29
+; CHECK-NEXT:    v_readlane_b32 s83, v0, 28
+; CHECK-NEXT:    v_readlane_b32 s82, v0, 27
+; CHECK-NEXT:    v_readlane_b32 s81, v0, 26
+; CHECK-NEXT:    v_readlane_b32 s80, v0, 25
+; CHECK-NEXT:    v_readlane_b32 s71, v0, 24
+; CHECK-NEXT:    v_readlane_b32 s70, v0, 23
+; CHECK-NEXT:    v_readlane_b32 s69, v0, 22
+; CHECK-NEXT:    v_readlane_b32 s68, v0, 21
+; CHECK-NEXT:    v_readlane_b32 s67, v0, 20
+; CHECK-NEXT:    v_readlane_b32 s66, v0, 19
+; CHECK-NEXT:    v_readlane_b32 s65, v0, 18
+; CHECK-NEXT:    v_readlane_b32 s64, v0, 17
+; CHECK-NEXT:    v_readlane_b32 s55, v0, 16
+; CHECK-NEXT:    v_readlane_b32 s54, v0, 15
+; CHECK-NEXT:    v_readlane_b32 s53, v0, 14
+; CHECK-NEXT:    v_readlane_b32 s52, v0, 13
+; CHECK-NEXT:    v_readlane_b32 s51, v0, 12
+; CHECK-NEXT:    v_readlane_b32 s50, v0, 11
+; CHECK-NEXT:    v_readlane_b32 s49, v0, 10
+; CHECK-NEXT:    v_readlane_b32 s48, v0, 9
 ; CHECK-NEXT:    v_readlane_b32 s39, v0, 8
 ; CHECK-NEXT:    v_readlane_b32 s38, v0, 7
 ; CHECK-NEXT:    v_readlane_b32 s37, v0, 6
@@ -1703,7 +1637,6 @@ define void @spill_func(ptr addrspace(1) %arg) #0 {
 ; CHECK-NEXT:    v_readlane_b32 s30, v0, 0
 ; CHECK-NEXT:    s_xor_saveexec_b64 s[4:5], -1
 ; CHECK-NEXT:    buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
-; CHECK-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
 ; CHECK-NEXT:    s_mov_b64 exec, s[4:5]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll b/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll
index a03ad4daab014..2cd15210eb8e5 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll
@@ -1529,7 +1529,7 @@ bb14:                                             ; preds = %bb13, %bb9
   br label %bb19
 
 bb19:                                             ; preds = %bb14, %bb13, %bb9
-  %tmp20 = phi i32 [ undef, %bb9 ], [ undef, %bb13 ], [ %tmp18, %bb14 ]
+  %tmp20 = phi i32 [ poison, %bb9 ], [ poison, %bb13 ], [ %tmp18, %bb14 ]
   %tmp21 = getelementptr inbounds i32, ptr addrspace(1) %arg, i64 %arg5
   store i32 %tmp20, ptr addrspace(1) %tmp21, align 4
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll b/llvm/test/CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll
index ce55e9171c818..84a4b579fef0c 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-rsrc-ptr-ops.ll
@@ -49,7 +49,7 @@ main_body:
   %buf1.int = ptrtoint ptr addrspace(8) %buf1 to i128
   %buf1.vec = bitcast i128 %buf1.int to <4 x i32>
   call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> %buf1.vec, ptr addrspace(8) %buf2, i32 0, i32 0, i32 0)
-  %shuffled = shufflevector <2 x ptr addrspace(8)> %buffers, <2 x ptr addrspace(8)> undef, <2 x i32> <i32 1, i32 0>
+  %shuffled = shufflevector <2 x ptr addrspace(8)> %buffers, <2 x ptr addrspace(8)> poison, <2 x i32> <i32 1, i32 0>
   %somewhere.next = getelementptr <2 x ptr addrspace(8)>, ptr addrspace(1) %somewhere, i64 1
   store <2 x ptr addrspace(8)> %shuffled, ptr addrspace(1) %somewhere.next
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/bug-deadlanes.ll b/llvm/test/CodeGen/AMDGPU/bug-deadlanes.ll
index 98590c227859b..95f97adcada31 100644
--- a/llvm/test/CodeGen/AMDGPU/bug-deadlanes.ll
+++ b/llvm/test/CodeGen/AMDGPU/bug-deadlanes.ll
@@ -18,7 +18,7 @@ define dllexport amdgpu_ps void @_amdgpu_ps_main(i32 %descTable2) #0 {
 
 bb1750:                                           ; preds = %bb1897, %.entry
   %__llpc_global_proxy_r3.12.vec.extract2358295 = phi i32 [ 0, %.entry ], [ %__llpc_global_proxy_r3.12.vec.extract2358, %bb1897 ]
-  %__llpc_global_proxy_r13.20293 = phi <4 x i32> [ undef, %.entry ], [ %__llpc_global_proxy_r13.22, %bb1897 ]
+  %__llpc_global_proxy_r13.20293 = phi <4 x i32> [ poison, %.entry ], [ %__llpc_global_proxy_r13.22, %bb1897 ]
   %__llpc_global_proxy_r10.19291 = phi <4 x i32> [ poison, %.entry ], [ %i1914, %bb1897 ]
   %i1751 = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %i1746, i32 poison, i32 0, i32 0, i32 0)
   %i1754 = shufflevector <4 x i32> %__llpc_global_proxy_r10.19291, <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
@@ -45,19 +45,19 @@ bb1789:                                           ; preds = %bb1750
   %i1879 = bitcast <3 x i32> %i1878 to <3 x float>
   %i1881 = fmul reassoc nnan nsz arcp contract afn <3 x float> %i1540, %i1879
   %i1882 = call <3 x i32> @llvm.amdgcn.s.buffer.load.v3i32(<4 x i32> poison, i32 poison, i32 0)
-  %i1883 = shufflevector <3 x i32> %i1882, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %i1883 = shufflevector <3 x i32> %i1882, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
   %i1884 = bitcast <4 x i32> %i1883 to <4 x float>
   %i1885 = shufflevector <4 x float> %i1884, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
-  %i1886 = insertelement <3 x i32> undef, i32 %i1819, i64 0
+  %i1886 = insertelement <3 x i32> poison, i32 %i1819, i64 0
   %i1887 = bitcast <3 x i32> %i1886 to <3 x float>
-  %i1888 = insertelement <3 x i32> undef, i32 %i1801, i64 0
+  %i1888 = insertelement <3 x i32> poison, i32 %i1801, i64 0
   %i1889 = bitcast <3 x i32> %i1888 to <3 x float>
   %i1890 = fmul reassoc nnan nsz arcp contract afn <3 x float> %i1887, %i1889
   %i1891 = shufflevector <3 x float> %i1890, <3 x float> poison, <3 x i32> zeroinitializer
   %i1892 = fmul reassoc nnan nsz arcp contract afn <3 x float> %i1885, %i1891
   %i1893 = fmul reassoc nnan nsz arcp contract afn <3 x float> %i1892, %i1881
   %i1894 = bitcast <3 x float> %i1893 to <3 x i32>
-  %i1895 = shufflevector <3 x i32> %i1894, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %i1895 = shufflevector <3 x i32> %i1894, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
   %i1896 = insertelement <4 x i32> %i1895, i32 %i1819, i64 3
   br label %bb1897
 
@@ -70,7 +70,7 @@ bb1897:                                           ; preds = %bb1789, %bb1787
   %i1901 = bitcast <3 x i32> %i1900 to <3 x float>
   %i1902 = fadd reassoc nnan nsz arcp contract afn <3 x float> %i1901, %i1899
   %i1903 = bitcast <3 x float> %i1902 to <3 x i32>
-  %i1907 = shufflevector <3 x i32> %i1903, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %i1907 = shufflevector <3 x i32> %i1903, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
   %i1908 = shufflevector <4 x i32> %i1907, <4 x i32> %__llpc_global_proxy_r11.19, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
   %i1914 = shufflevector <4 x i32> %i1908, <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
   %__llpc_global_proxy_r3.12.vec.extract2358 = extractelement <2 x i32> zeroinitializer, i64 1
diff --git a/llvm/test/CodeGen/AMDGPU/bug-v4f64-subvector.ll b/llvm/test/CodeGen/AMDGPU/bug-v4f64-subvector.ll
index 2acd2355965a5..99166aed6b215 100644
--- a/llvm/test/CodeGen/AMDGPU/bug-v4f64-subvector.ll
+++ b/llvm/test/CodeGen/AMDGPU/bug-v4f64-subvector.ll
@@ -15,15 +15,15 @@ entry:
   %tmp2 = getelementptr inbounds double, ptr addrspace(1) %tmp1, i64 undef
   %tmp4 = load <3 x double>, ptr addrspace(1) %tmp2, align 8, !tbaa !6
   %tmp5 = extractelement <3 x double> %tmp4, i32 1
-  %tmp6 = insertelement <3 x double> undef, double %tmp5, i32 1
-  %tmp7 = insertelement <3 x double> %tmp6, double undef, i32 2
+  %tmp6 = insertelement <3 x double> poison, double %tmp5, i32 1
+  %tmp7 = insertelement <3 x double> %tmp6, double poison, i32 2
   %tmp8 = load <3 x double>, ptr addrspace(1) undef, align 8, !tbaa !6
   %tmp9 = extractelement <3 x double> %tmp8, i32 2
-  %tmp10 = insertelement <3 x double> undef, double %tmp9, i32 2
+  %tmp10 = insertelement <3 x double> poison, double %tmp9, i32 2
   %tmp11 = fcmp olt <3 x double> %tmp10, %tmp7
   %tmp12 = select <3 x i1> %tmp11, <3 x double> zeroinitializer, <3 x double> <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
   %tmp13 = extractelement <3 x double> %tmp12, i64 1
-  %tmp14 = insertelement <2 x double> undef, double %tmp13, i32 1
+  %tmp14 = insertelement <2 x double> poison, double %tmp13, i32 1
   store <2 x double> %tmp14, ptr addrspace(1) undef, align 8, !tbaa !6
   ret void
 }
@@ -42,12 +42,12 @@ entry:
   %tmp2 = getelementptr inbounds double, ptr addrspace(1) %tmp1, i64 undef
   %tmp4 = load <3 x double>, ptr addrspace(1) %tmp2, align 8, !tbaa !6
   %tmp5 = extractelement <3 x double> %tmp4, i32 1
-  %tmp6 = insertelement <3 x double> undef, double %tmp5, i32 1
-  %tmp7 = insertelement <3 x double> %tmp6, double undef, i32 2
+  %tmp6 = insertelement <3 x double> poison, double %tmp5, i32 1
+  %tmp7 = insertelement <3 x double> %tmp6, double poison, i32 2
   %tmp8 = load <3 x double>, ptr addrspace(1) undef, align 8, !tbaa !6
   %tmp9 = extractelement <3 x double> %tmp8, i32 1
-  %tmp10 = insertelement <3 x double> undef, double %tmp9, i32 1
-  %tmp11 = insertelement <3 x double> %tmp10, double undef, i32 2
+  %tmp10 = insertelement <3 x double> poison, double %tmp9, i32 1
+  %tmp11 = insertelement <3 x double> %tmp10, double poison, i32 2
   %tmp12 = fcmp olt <3 x double> %tmp11, %tmp7
   %tmp13 = select <3 x i1> %tmp12, <3 x double> zeroinitializer, <3 x double> <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
   %tmp14 = extractelement <3 x double> %tmp13, i64 2
diff --git a/llvm/test/CodeGen/AMDGPU/build-vector-insert-elt-infloop.ll b/llvm/test/CodeGen/AMDGPU/build-vector-insert-elt-infloop.ll
index efe8f9303e2dd..f70b3fd9548ae 100644
--- a/llvm/test/CodeGen/AMDGPU/build-vector-insert-elt-infloop.ll
+++ b/llvm/test/CodeGen/AMDGPU/build-vector-insert-elt-infloop.ll
@@ -15,7 +15,7 @@ bb1:
   %tmp2 = phi half [ 0xH0000, %bb ], [ %tmp8, %bb1 ]
   %tmp3 = load volatile half, ptr null, align 536870912
   %tmp4 = bitcast half %tmp3 to i16
-  %tmp5 = insertelement <2 x i16> <i16 0, i16 undef>, i16 %tmp4, i32 1
+  %tmp5 = insertelement <2 x i16> <i16 0, i16 poison>, i16 %tmp4, i32 1
   store volatile half %tmp2, ptr %arg, align 2
   %tmp7 = bitcast <2 x i16> %tmp to <2 x half>
   %tmp8 = extractelement <2 x half> %tmp7, i32 0
diff --git a/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll b/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
index 46b2f82d9de2a..5c7172ff8d047 100644
--- a/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
@@ -20,7 +20,7 @@ define void @undef_lo_v2i16(i16 %arg0) {
 ; GFX8-NEXT:    ; use v0
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
-  %undef.lo = insertelement <2 x i16> undef, i16 %arg0, i32 1
+  %undef.lo = insertelement <2 x i16> poison, i16 %arg0, i32 1
   call void asm sideeffect "; use $0", "v"(<2 x i16> %undef.lo);
   ret void
 }
@@ -43,7 +43,7 @@ define void @undef_lo_v2f16(half %arg0) {
 ; GFX8-NEXT:    ; use v0
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
-  %undef.lo = insertelement <2 x half> undef, half %arg0, i32 1
+  %undef.lo = insertelement <2 x half> poison, half %arg0, i32 1
   call void asm sideeffect "; use $0", "v"(<2 x half> %undef.lo);
   ret void
 }
@@ -69,7 +69,7 @@ define void @undef_lo_op_v2f16(half %arg0) {
 ; GFX8-NEXT:    ; use v0
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
-  %undef.lo = insertelement <2 x half> undef, half %arg0, i32 1
+  %undef.lo = insertelement <2 x half> poison, half %arg0, i32 1
   %op = fadd <2 x half> %undef.lo, <half 1.0, half 1.0>
   call void asm sideeffect "; use $0", "v"(<2 x half> %op);
   ret void
@@ -96,7 +96,7 @@ define void @undef_lo_op_v2i16(i16 %arg0) {
 ; GFX8-NEXT:    ; use v0
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
-  %undef.lo = insertelement <2 x i16> undef, i16 %arg0, i32 1
+  %undef.lo = insertelement <2 x i16> poison, i16 %arg0, i32 1
   %op = add <2 x i16> %undef.lo, <i16 99, i16 99>
   call void asm sideeffect "; use $0", "v"(<2 x i16> %op);
   ret void
@@ -120,7 +120,7 @@ define void @undef_lo3_v4i16(i16 %arg0) {
 ; GFX8-NEXT:    ; use v[0:1]
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
-  %undef.lo = insertelement <4 x i16> undef, i16 %arg0, i32 1
+  %undef.lo = insertelement <4 x i16> poison, i16 %arg0, i32 1
   call void asm sideeffect "; use $0", "v"(<4 x i16> %undef.lo);
   ret void
 }
@@ -143,7 +143,7 @@ define void @undef_lo3_v4f16(half %arg0) {
 ; GFX8-NEXT:    ; use v[0:1]
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
-  %undef.lo = insertelement <4 x half> undef, half %arg0, i32 1
+  %undef.lo = insertelement <4 x half> poison, half %arg0, i32 1
   call void asm sideeffect "; use $0", "v"(<4 x half> %undef.lo);
   ret void
 }
@@ -168,7 +168,7 @@ define void @undef_lo2_v4i16(<2 x i16> %arg0) {
 ; GFX8-NEXT:    ; use v[0:1]
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
-  %undef.lo = shufflevector <2 x i16> %arg0, <2 x i16> undef, <4 x i32> <i32 1, i32 1, i32 2, i32 3>
+  %undef.lo = shufflevector <2 x i16> %arg0, <2 x i16> poison, <4 x i32> <i32 1, i32 1, i32 2, i32 3>
   call void asm sideeffect "; use $0", "v"(<4 x i16> %undef.lo);
   ret void
 }
@@ -193,7 +193,7 @@ define void @undef_lo2_v4f16(<2 x half> %arg0) {
 ; GFX8-NEXT:    ; use v[0:1]
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
-  %undef.lo = shufflevector <2 x half> %arg0, <2 x half> undef, <4 x i32> <i32 1, i32 1, i32 2, i32 3>
+  %undef.lo = shufflevector <2 x half> %arg0, <2 x half> poison, <4 x i32> <i32 1, i32 1, i32 2, i32 3>
   call void asm sideeffect "; use $0", "v"(<4 x half> %undef.lo);
   ret void
 }
@@ -214,7 +214,7 @@ define void @undef_hi_v2i16(i16 %arg0) {
 ; GFX8-NEXT:    ; use v0
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
-  %undef.hi = insertelement <2 x i16> undef, i16 %arg0, i32 0
+  %undef.hi = insertelement <2 x i16> poison, i16 %arg0, i32 0
   call void asm sideeffect "; use $0", "v"(<2 x i16> %undef.hi);
   ret void
 }
@@ -235,7 +235,7 @@ define void @undef_hi_v2f16(half %arg0) {
 ; GFX8-NEXT:    ; use v0
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
-  %undef.hi = insertelement <2 x half> undef, half %arg0, i32 0
+  %undef.hi = insertelement <2 x half> poison, half %arg0, i32 0
   call void asm sideeffect "; use $0", "v"(<2 x half> %undef.hi);
   ret void
 }
@@ -259,7 +259,7 @@ define void @undef_hi_op_v2f16(half %arg0) {
 ; GFX8-NEXT:    ; use v0
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
-  %undef.hi = insertelement <2 x half> undef, half %arg0, i32 0
+  %undef.hi = insertelement <2 x half> poison, half %arg0, i32 0
   %op = fadd <2 x half> %undef.hi, <half 1.0, half 1.0>
   call void asm sideeffect "; use $0", "v"(<2 x half> %op);
   ret void
@@ -284,7 +284,7 @@ define void @undef_hi_op_v2i16(i16 %arg0) {
 ; GFX8-NEXT:    ; use v0
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
-  %undef.hi = insertelement <2 x i16> undef, i16 %arg0, i32 0
+  %undef.hi = insertelement <2 x i16> poison, i16 %arg0, i32 0
   %op = add <2 x i16> %undef.hi, <i16 99, i16 99>
   call void asm sideeffect "; use $0", "v"(<2 x i16> %op);
   ret void
@@ -306,7 +306,7 @@ define void @undef_hi3_v4i16(i16 %arg0) {
 ; GFX8-NEXT:    ; use v[0:1]
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
-  %undef.hi = insertelement <4 x i16> undef, i16 %arg0, i32 0
+  %undef.hi = insertelement <4 x i16> poison, i16 %arg0, i32 0
   call void asm sideeffect "; use $0", "v"(<4 x i16> %undef.hi);
   ret void
 }
@@ -327,7 +327,7 @@ define void @undef_hi3_v4f16(half %arg0) {
 ; GFX8-NEXT:    ; use v[0:1]
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
-  %undef.hi = insertelement <4 x half> undef, half %arg0, i32 0
+  %undef.hi = insertelement <4 x half> poison, half %arg0, i32 0
   call void asm sideeffect "; use $0", "v"(<4 x half> %undef.hi);
   ret void
 }
@@ -348,7 +348,7 @@ define void @undef_hi2_v4i16(<2 x i16> %arg0) {
 ; GFX8-NEXT:    ; use v[0:1]
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
-  %undef.hi = shufflevector <2 x i16> %arg0, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %undef.hi = shufflevector <2 x i16> %arg0, <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   call void asm sideeffect "; use $0", "v"(<4 x i16> %undef.hi);
   ret void
 }
@@ -369,7 +369,7 @@ define void @undef_hi2_v4f16(<2 x half> %arg0) {
 ; GFX8-NEXT:    ; use v[0:1]
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
-  %undef.hi = shufflevector <2 x half> %arg0, <2 x half> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %undef.hi = shufflevector <2 x half> %arg0, <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   call void asm sideeffect "; use $0", "v"(<4 x half> %undef.hi);
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/build_vector-r600.ll b/llvm/test/CodeGen/AMDGPU/build_vector-r600.ll
index 2abcbbcdd1bc6..df26fd8430774 100644
--- a/llvm/test/CodeGen/AMDGPU/build_vector-r600.ll
+++ b/llvm/test/CodeGen/AMDGPU/build_vector-r600.ll
@@ -72,7 +72,7 @@ define amdgpu_kernel void @build_vector_v2i16_trunc (ptr addrspace(1) %out, i32
 ; R600-NEXT:    327680(4.591775e-40), 2(2.802597e-45)
   %srl = lshr i32 %a, 16
   %trunc = trunc i32 %srl to i16
-  %ins.0 = insertelement <2 x i16> undef, i16 %trunc, i32 0
+  %ins.0 = insertelement <2 x i16> poison, i16 %trunc, i32 0
   %ins.1 = insertelement <2 x i16> %ins.0, i16 5, i32 1
   store <2 x i16> %ins.1, ptr addrspace(1) %out
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/build_vector.ll b/llvm/test/CodeGen/AMDGPU/build_vector.ll
index b79984f41114e..7208eaeff8eb1 100644
--- a/llvm/test/CodeGen/AMDGPU/build_vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/build_vector.ll
@@ -245,7 +245,7 @@ define amdgpu_kernel void @build_vector_v2i16_trunc (ptr addrspace(1) %out, i32
 ; GFX942-NEXT:    s_endpgm
   %srl = lshr i32 %a, 16
   %trunc = trunc i32 %srl to i16
-  %ins.0 = insertelement <2 x i16> undef, i16 %trunc, i32 0
+  %ins.0 = insertelement <2 x i16> poison, i16 %trunc, i32 0
   %ins.1 = insertelement <2 x i16> %ins.0, i16 5, i32 1
   store <2 x i16> %ins.1, ptr addrspace(1) %out
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/bypass-div.ll b/llvm/test/CodeGen/AMDGPU/bypass-div.ll
index 5dde193528aa4..3cf70c42390c2 100644
--- a/llvm/test/CodeGen/AMDGPU/bypass-div.ll
+++ b/llvm/test/CodeGen/AMDGPU/bypass-div.ll
@@ -860,7 +860,7 @@ define <2 x i64> @sdivrem64(i64 %a, i64 %b) {
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %d = sdiv i64 %a, %b
   %r = srem i64 %a, %b
-  %ins.0 = insertelement <2 x i64> undef, i64 %d, i32 0
+  %ins.0 = insertelement <2 x i64> poison, i64 %d, i32 0
   %ins.1 = insertelement <2 x i64> %ins.0, i64 %r, i32 1
   ret <2 x i64> %ins.1
 }
@@ -1012,7 +1012,7 @@ define <2 x i64> @udivrem64(i64 %a, i64 %b) {
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %d = udiv i64 %a, %b
   %r = urem i64 %a, %b
-  %ins.0 = insertelement <2 x i64> undef, i64 %d, i32 0
+  %ins.0 = insertelement <2 x i64> poison, i64 %d, i32 0
   %ins.1 = insertelement <2 x i64> %ins.0, i64 %r, i32 1
   ret <2 x i64> %ins.1
 }
diff --git a/llvm/test/CodeGen/AMDGPU/call-args-inreg-no-sgpr-for-csrspill-xfail.ll b/llvm/test/CodeGen/AMDGPU/call-args-inreg-no-sgpr-for-csrspill-xfail.ll
index d4c50cf2c7e4a..34f4476f7fd6a 100644
--- a/llvm/test/CodeGen/AMDGPU/call-args-inreg-no-sgpr-for-csrspill-xfail.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-args-inreg-no-sgpr-for-csrspill-xfail.ll
@@ -1,6 +1,6 @@
-; RUN: not --crash llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs=0 -filetype=null %s 2>&1 | FileCheck -enable-var-scope %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs=0 -filetype=null %s 2>&1 | FileCheck -enable-var-scope %s
 
-; CHECK: LLVM ERROR: failed to find free scratch register
+; CHECK: illegal VGPR to SGPR copy
 
 declare hidden void @external_void_func_a15i32_inreg([15 x i32] inreg) #0
 declare hidden void @external_void_func_a16i32_inreg([16 x i32] inreg) #0
diff --git a/llvm/test/CodeGen/AMDGPU/call-args-inreg.ll b/llvm/test/CodeGen/AMDGPU/call-args-inreg.ll
index 0b8ad359ccb94..c10cb0ae6d336 100644
--- a/llvm/test/CodeGen/AMDGPU/call-args-inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-args-inreg.ll
@@ -1385,15 +1385,15 @@ define void @test_call_external_void_func_a15i32_inreg([13 x i32] inreg %arg0) #
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_mov_b32 s29, s33
 ; GFX9-NEXT:    s_mov_b32 s33, s32
-; GFX9-NEXT:    s_or_saveexec_b64 vcc, -1
+; GFX9-NEXT:    s_or_saveexec_b64 s[40:41], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_mov_b64 exec, vcc
+; GFX9-NEXT:    s_mov_b64 exec, s[40:41]
 ; GFX9-NEXT:    v_writelane_b32 v40, s29, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_getpc_b64 vcc
-; GFX9-NEXT:    s_add_u32 vcc_lo, vcc_lo, external_void_func_a15i32_inreg@rel32@lo+4
-; GFX9-NEXT:    s_addc_u32 vcc_hi, vcc_hi, external_void_func_a15i32_inreg@rel32@hi+12
+; GFX9-NEXT:    s_getpc_b64 s[40:41]
+; GFX9-NEXT:    s_add_u32 s40, s40, external_void_func_a15i32_inreg@rel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s41, s41, external_void_func_a15i32_inreg@rel32@hi+12
 ; GFX9-NEXT:    s_mov_b32 s3, s19
 ; GFX9-NEXT:    s_mov_b32 s2, s18
 ; GFX9-NEXT:    s_mov_b32 s1, s17
@@ -1408,7 +1408,7 @@ define void @test_call_external_void_func_a15i32_inreg([13 x i32] inreg %arg0) #
 ; GFX9-NEXT:    s_mov_b32 s23, s27
 ; GFX9-NEXT:    s_mov_b32 s24, s28
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX9-NEXT:    s_swappc_b64 s[30:31], vcc
+; GFX9-NEXT:    s_swappc_b64 s[30:31], s[40:41]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
 ; GFX9-NEXT:    s_mov_b32 s32, s33
diff --git a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
index 9bbecacd6c774..775cad3f3cec1 100644
--- a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
@@ -1454,7 +1454,7 @@ define amdgpu_kernel void @test_call_external_void_func_v3i64() #0 {
 ; HSA-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; HSA-NEXT:    s_endpgm
   %load = load <2 x i64>, ptr addrspace(1) null
-  %val = shufflevector <2 x i64> %load, <2 x i64> <i64 8589934593, i64 undef>, <3 x i32> <i32 0, i32 1, i32 2>
+  %val = shufflevector <2 x i64> %load, <2 x i64> <i64 8589934593, i64 poison>, <3 x i32> <i32 0, i32 1, i32 2>
 
   call void @external_void_func_v3i64(<3 x i64> %val)
   ret void
@@ -4542,19 +4542,19 @@ define amdgpu_kernel void @test_call_external_void_func_v32i32_i32(i32) #0 {
 define amdgpu_kernel void @test_call_external_i32_func_i32_imm(ptr addrspace(1) %out) #0 {
 ; VI-LABEL: test_call_external_i32_func_i32_imm:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; VI-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; VI-NEXT:    s_mov_b32 s42, -1
-; VI-NEXT:    s_mov_b32 s43, 0xe80000
-; VI-NEXT:    s_add_u32 s40, s40, s5
+; VI-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; VI-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; VI-NEXT:    s_mov_b32 s50, -1
+; VI-NEXT:    s_mov_b32 s51, 0xe80000
+; VI-NEXT:    s_add_u32 s48, s48, s5
 ; VI-NEXT:    s_load_dwordx2 s[36:37], s[2:3], 0x24
-; VI-NEXT:    s_addc_u32 s41, s41, 0
+; VI-NEXT:    s_addc_u32 s49, s49, 0
 ; VI-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; VI-NEXT:    s_mov_b64 s[0:1], s[40:41]
+; VI-NEXT:    s_mov_b64 s[0:1], s[48:49]
 ; VI-NEXT:    s_getpc_b64 s[4:5]
 ; VI-NEXT:    s_add_u32 s4, s4, external_i32_func_i32@rel32@lo+4
 ; VI-NEXT:    s_addc_u32 s5, s5, external_i32_func_i32@rel32@hi+12
-; VI-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; VI-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; VI-NEXT:    v_mov_b32_e32 v0, 42
 ; VI-NEXT:    s_mov_b32 s32, 0
 ; VI-NEXT:    s_mov_b32 s39, 0xf000
@@ -4566,19 +4566,19 @@ define amdgpu_kernel void @test_call_external_i32_func_i32_imm(ptr addrspace(1)
 ;
 ; CI-LABEL: test_call_external_i32_func_i32_imm:
 ; CI:       ; %bb.0:
-; CI-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; CI-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; CI-NEXT:    s_mov_b32 s42, -1
-; CI-NEXT:    s_mov_b32 s43, 0xe8f000
-; CI-NEXT:    s_add_u32 s40, s40, s5
+; CI-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; CI-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; CI-NEXT:    s_mov_b32 s50, -1
+; CI-NEXT:    s_mov_b32 s51, 0xe8f000
+; CI-NEXT:    s_add_u32 s48, s48, s5
 ; CI-NEXT:    s_load_dwordx2 s[36:37], s[2:3], 0x9
-; CI-NEXT:    s_addc_u32 s41, s41, 0
+; CI-NEXT:    s_addc_u32 s49, s49, 0
 ; CI-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; CI-NEXT:    s_mov_b64 s[0:1], s[40:41]
+; CI-NEXT:    s_mov_b64 s[0:1], s[48:49]
 ; CI-NEXT:    s_getpc_b64 s[4:5]
 ; CI-NEXT:    s_add_u32 s4, s4, external_i32_func_i32@rel32@lo+4
 ; CI-NEXT:    s_addc_u32 s5, s5, external_i32_func_i32@rel32@hi+12
-; CI-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; CI-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; CI-NEXT:    v_mov_b32_e32 v0, 42
 ; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_mov_b32 s39, 0xf000
@@ -4590,19 +4590,19 @@ define amdgpu_kernel void @test_call_external_i32_func_i32_imm(ptr addrspace(1)
 ;
 ; GFX9-LABEL: test_call_external_i32_func_i32_imm:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s42, -1
-; GFX9-NEXT:    s_mov_b32 s43, 0xe00000
-; GFX9-NEXT:    s_add_u32 s40, s40, s5
+; GFX9-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s50, -1
+; GFX9-NEXT:    s_mov_b32 s51, 0xe00000
+; GFX9-NEXT:    s_add_u32 s48, s48, s5
 ; GFX9-NEXT:    s_load_dwordx2 s[36:37], s[2:3], 0x24
-; GFX9-NEXT:    s_addc_u32 s41, s41, 0
+; GFX9-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[0:1]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[40:41]
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
 ; GFX9-NEXT:    s_getpc_b64 s[4:5]
 ; GFX9-NEXT:    s_add_u32 s4, s4, external_i32_func_i32@rel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s5, s5, external_i32_func_i32@rel32@hi+12
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 42
 ; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_mov_b32 s39, 0xf000
diff --git a/llvm/test/CodeGen/AMDGPU/call-preserved-registers.ll b/llvm/test/CodeGen/AMDGPU/call-preserved-registers.ll
index db9ce56ecc3cc..67a70cdeb1ecc 100644
--- a/llvm/test/CodeGen/AMDGPU/call-preserved-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-preserved-registers.ll
@@ -528,15 +528,16 @@ define void @callee_saved_sgpr_func() #2 {
 ; MUBUF-NEXT:    s_getpc_b64 s[4:5]
 ; MUBUF-NEXT:    s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
 ; MUBUF-NEXT:    s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
-; MUBUF-NEXT:    v_writelane_b32 v40, s40, 2
+; MUBUF-NEXT:    v_writelane_b32 v40, s34, 2
 ; MUBUF-NEXT:    ;;#ASMSTART
 ; MUBUF-NEXT:    ; def s40
 ; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    s_mov_b32 s34, s40
 ; MUBUF-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; MUBUF-NEXT:    ;;#ASMSTART
-; MUBUF-NEXT:    ; use s40
+; MUBUF-NEXT:    ; use s34
 ; MUBUF-NEXT:    ;;#ASMEND
-; MUBUF-NEXT:    v_readlane_b32 s40, v40, 2
+; MUBUF-NEXT:    v_readlane_b32 s34, v40, 2
 ; MUBUF-NEXT:    v_readlane_b32 s31, v40, 1
 ; MUBUF-NEXT:    v_readlane_b32 s30, v40, 0
 ; MUBUF-NEXT:    s_mov_b32 s32, s33
@@ -563,15 +564,16 @@ define void @callee_saved_sgpr_func() #2 {
 ; FLATSCR-NEXT:    s_getpc_b64 s[0:1]
 ; FLATSCR-NEXT:    s_add_u32 s0, s0, external_void_func_void@rel32@lo+4
 ; FLATSCR-NEXT:    s_addc_u32 s1, s1, external_void_func_void@rel32@hi+12
-; FLATSCR-NEXT:    v_writelane_b32 v40, s40, 2
+; FLATSCR-NEXT:    v_writelane_b32 v40, s34, 2
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ; def s40
 ; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    s_mov_b32 s34, s40
 ; FLATSCR-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; FLATSCR-NEXT:    ;;#ASMSTART
-; FLATSCR-NEXT:    ; use s40
+; FLATSCR-NEXT:    ; use s34
 ; FLATSCR-NEXT:    ;;#ASMEND
-; FLATSCR-NEXT:    v_readlane_b32 s40, v40, 2
+; FLATSCR-NEXT:    v_readlane_b32 s34, v40, 2
 ; FLATSCR-NEXT:    v_readlane_b32 s31, v40, 1
 ; FLATSCR-NEXT:    v_readlane_b32 s30, v40, 0
 ; FLATSCR-NEXT:    s_mov_b32 s32, s33
@@ -600,9 +602,10 @@ define amdgpu_kernel void @callee_saved_sgpr_kernel() #2 {
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ; def s40
 ; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    s_mov_b32 s33, s40
 ; FLATSCR-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; FLATSCR-NEXT:    ;;#ASMSTART
-; FLATSCR-NEXT:    ; use s40
+; FLATSCR-NEXT:    ; use s33
 ; FLATSCR-NEXT:    ;;#ASMEND
 ; FLATSCR-NEXT:    s_endpgm
   %s40 = call i32 asm sideeffect "; def s40", "={s40}"() #0
@@ -629,22 +632,23 @@ define void @callee_saved_sgpr_vgpr_func() #2 {
 ; MUBUF-NEXT:    s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
 ; MUBUF-NEXT:    s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
 ; MUBUF-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
-; MUBUF-NEXT:    v_writelane_b32 v41, s40, 2
+; MUBUF-NEXT:    v_writelane_b32 v41, s34, 2
 ; MUBUF-NEXT:    ;;#ASMSTART
 ; MUBUF-NEXT:    ; def s40
 ; MUBUF-NEXT:    ;;#ASMEND
+; MUBUF-NEXT:    s_mov_b32 s34, s40
 ; MUBUF-NEXT:    ;;#ASMSTART
 ; MUBUF-NEXT:    ; def v40
 ; MUBUF-NEXT:    ;;#ASMEND
 ; MUBUF-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; MUBUF-NEXT:    ;;#ASMSTART
-; MUBUF-NEXT:    ; use s40
+; MUBUF-NEXT:    ; use s34
 ; MUBUF-NEXT:    ;;#ASMEND
 ; MUBUF-NEXT:    ;;#ASMSTART
 ; MUBUF-NEXT:    ; use v40
 ; MUBUF-NEXT:    ;;#ASMEND
 ; MUBUF-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
-; MUBUF-NEXT:    v_readlane_b32 s40, v41, 2
+; MUBUF-NEXT:    v_readlane_b32 s34, v41, 2
 ; MUBUF-NEXT:    v_readlane_b32 s31, v41, 1
 ; MUBUF-NEXT:    v_readlane_b32 s30, v41, 0
 ; MUBUF-NEXT:    s_mov_b32 s32, s33
@@ -672,22 +676,23 @@ define void @callee_saved_sgpr_vgpr_func() #2 {
 ; FLATSCR-NEXT:    s_add_u32 s0, s0, external_void_func_void@rel32@lo+4
 ; FLATSCR-NEXT:    s_addc_u32 s1, s1, external_void_func_void@rel32@hi+12
 ; FLATSCR-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
-; FLATSCR-NEXT:    v_writelane_b32 v41, s40, 2
+; FLATSCR-NEXT:    v_writelane_b32 v41, s34, 2
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ; def s40
 ; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    s_mov_b32 s34, s40
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ; def v40
 ; FLATSCR-NEXT:    ;;#ASMEND
 ; FLATSCR-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; FLATSCR-NEXT:    ;;#ASMSTART
-; FLATSCR-NEXT:    ; use s40
+; FLATSCR-NEXT:    ; use s34
 ; FLATSCR-NEXT:    ;;#ASMEND
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ; use v40
 ; FLATSCR-NEXT:    ;;#ASMEND
 ; FLATSCR-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
-; FLATSCR-NEXT:    v_readlane_b32 s40, v41, 2
+; FLATSCR-NEXT:    v_readlane_b32 s34, v41, 2
 ; FLATSCR-NEXT:    v_readlane_b32 s31, v41, 1
 ; FLATSCR-NEXT:    v_readlane_b32 s30, v41, 0
 ; FLATSCR-NEXT:    s_mov_b32 s32, s33
@@ -718,13 +723,14 @@ define amdgpu_kernel void @callee_saved_sgpr_vgpr_kernel() #2 {
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ; def s40
 ; FLATSCR-NEXT:    ;;#ASMEND
+; FLATSCR-NEXT:    s_mov_b32 s33, s40
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ; def v32
 ; FLATSCR-NEXT:    ;;#ASMEND
 ; FLATSCR-NEXT:    v_mov_b32_e32 v40, v32
 ; FLATSCR-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; FLATSCR-NEXT:    ;;#ASMSTART
-; FLATSCR-NEXT:    ; use s40
+; FLATSCR-NEXT:    ; use s33
 ; FLATSCR-NEXT:    ;;#ASMEND
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ; use v40
diff --git a/llvm/test/CodeGen/AMDGPU/callee-frame-setup.ll b/llvm/test/CodeGen/AMDGPU/callee-frame-setup.ll
index 4c6f2d22080e0..9561aa555c80e 100644
--- a/llvm/test/CodeGen/AMDGPU/callee-frame-setup.ll
+++ b/llvm/test/CodeGen/AMDGPU/callee-frame-setup.ll
@@ -257,50 +257,26 @@ define void @callee_func_sgpr_spill_no_calls(i32 %in) #0 {
 ; MUBUF-NEXT:    v_writelane_b32 v40, s37, 1
 ; MUBUF-NEXT:    v_writelane_b32 v40, s38, 2
 ; MUBUF-NEXT:    v_writelane_b32 v40, s39, 3
-; MUBUF-NEXT:    v_writelane_b32 v40, s40, 4
-; MUBUF-NEXT:    v_writelane_b32 v40, s41, 5
-; MUBUF-NEXT:    v_writelane_b32 v40, s42, 6
-; MUBUF-NEXT:    v_writelane_b32 v40, s43, 7
-; MUBUF-NEXT:    v_writelane_b32 v40, s44, 8
-; MUBUF-NEXT:    v_writelane_b32 v40, s45, 9
-; MUBUF-NEXT:    v_writelane_b32 v40, s46, 10
-; MUBUF-NEXT:    v_writelane_b32 v40, s47, 11
-; MUBUF-NEXT:    v_writelane_b32 v40, s48, 12
-; MUBUF-NEXT:    v_writelane_b32 v40, s49, 13
-; MUBUF-NEXT:    v_writelane_b32 v40, s50, 14
-; MUBUF-NEXT:    v_writelane_b32 v40, s51, 15
-; MUBUF-NEXT:    v_writelane_b32 v40, s52, 16
-; MUBUF-NEXT:    v_writelane_b32 v40, s53, 17
-; MUBUF-NEXT:    v_writelane_b32 v40, s54, 18
-; MUBUF-NEXT:    v_writelane_b32 v40, s55, 19
-; MUBUF-NEXT:    v_writelane_b32 v40, s56, 20
-; MUBUF-NEXT:    v_writelane_b32 v40, s57, 21
-; MUBUF-NEXT:    v_writelane_b32 v40, s58, 22
-; MUBUF-NEXT:    v_writelane_b32 v40, s59, 23
-; MUBUF-NEXT:    v_writelane_b32 v40, s60, 24
-; MUBUF-NEXT:    v_writelane_b32 v40, s61, 25
-; MUBUF-NEXT:    v_writelane_b32 v40, s62, 26
-; MUBUF-NEXT:    v_writelane_b32 v40, s63, 27
-; MUBUF-NEXT:    v_writelane_b32 v40, s64, 28
-; MUBUF-NEXT:    v_writelane_b32 v40, s65, 29
-; MUBUF-NEXT:    v_writelane_b32 v40, s66, 30
-; MUBUF-NEXT:    v_writelane_b32 v40, s67, 31
-; MUBUF-NEXT:    v_writelane_b32 v40, s68, 32
-; MUBUF-NEXT:    v_writelane_b32 v40, s69, 33
-; MUBUF-NEXT:    v_writelane_b32 v40, s70, 34
-; MUBUF-NEXT:    v_writelane_b32 v40, s71, 35
-; MUBUF-NEXT:    v_writelane_b32 v40, s72, 36
-; MUBUF-NEXT:    v_writelane_b32 v40, s73, 37
-; MUBUF-NEXT:    v_writelane_b32 v40, s74, 38
-; MUBUF-NEXT:    v_writelane_b32 v40, s75, 39
-; MUBUF-NEXT:    v_writelane_b32 v40, s76, 40
-; MUBUF-NEXT:    v_writelane_b32 v40, s77, 41
-; MUBUF-NEXT:    v_writelane_b32 v40, s78, 42
-; MUBUF-NEXT:    v_writelane_b32 v40, s79, 43
-; MUBUF-NEXT:    v_writelane_b32 v40, s80, 44
-; MUBUF-NEXT:    v_writelane_b32 v40, s81, 45
-; MUBUF-NEXT:    v_writelane_b32 v40, s82, 46
-; MUBUF-NEXT:    v_writelane_b32 v40, s83, 47
+; MUBUF-NEXT:    v_writelane_b32 v40, s48, 4
+; MUBUF-NEXT:    v_writelane_b32 v40, s49, 5
+; MUBUF-NEXT:    v_writelane_b32 v40, s50, 6
+; MUBUF-NEXT:    v_writelane_b32 v40, s51, 7
+; MUBUF-NEXT:    v_writelane_b32 v40, s52, 8
+; MUBUF-NEXT:    v_writelane_b32 v40, s53, 9
+; MUBUF-NEXT:    v_writelane_b32 v40, s54, 10
+; MUBUF-NEXT:    v_writelane_b32 v40, s55, 11
+; MUBUF-NEXT:    v_writelane_b32 v40, s64, 12
+; MUBUF-NEXT:    v_writelane_b32 v40, s65, 13
+; MUBUF-NEXT:    v_writelane_b32 v40, s66, 14
+; MUBUF-NEXT:    v_writelane_b32 v40, s67, 15
+; MUBUF-NEXT:    v_writelane_b32 v40, s68, 16
+; MUBUF-NEXT:    v_writelane_b32 v40, s69, 17
+; MUBUF-NEXT:    v_writelane_b32 v40, s70, 18
+; MUBUF-NEXT:    v_writelane_b32 v40, s71, 19
+; MUBUF-NEXT:    v_writelane_b32 v40, s80, 20
+; MUBUF-NEXT:    v_writelane_b32 v40, s81, 21
+; MUBUF-NEXT:    v_writelane_b32 v40, s82, 22
+; MUBUF-NEXT:    v_writelane_b32 v40, s83, 23
 ; MUBUF-NEXT:    ;;#ASMSTART
 ; MUBUF-NEXT:    ;;#ASMEND
 ; MUBUF-NEXT:    ;;#ASMSTART
@@ -347,50 +323,26 @@ define void @callee_func_sgpr_spill_no_calls(i32 %in) #0 {
 ; MUBUF-NEXT:    ;;#ASMSTART
 ; MUBUF-NEXT:    ; use s[4:19]
 ; MUBUF-NEXT:    ;;#ASMEND
-; MUBUF-NEXT:    v_readlane_b32 s83, v40, 47
-; MUBUF-NEXT:    v_readlane_b32 s82, v40, 46
-; MUBUF-NEXT:    v_readlane_b32 s81, v40, 45
-; MUBUF-NEXT:    v_readlane_b32 s80, v40, 44
-; MUBUF-NEXT:    v_readlane_b32 s79, v40, 43
-; MUBUF-NEXT:    v_readlane_b32 s78, v40, 42
-; MUBUF-NEXT:    v_readlane_b32 s77, v40, 41
-; MUBUF-NEXT:    v_readlane_b32 s76, v40, 40
-; MUBUF-NEXT:    v_readlane_b32 s75, v40, 39
-; MUBUF-NEXT:    v_readlane_b32 s74, v40, 38
-; MUBUF-NEXT:    v_readlane_b32 s73, v40, 37
-; MUBUF-NEXT:    v_readlane_b32 s72, v40, 36
-; MUBUF-NEXT:    v_readlane_b32 s71, v40, 35
-; MUBUF-NEXT:    v_readlane_b32 s70, v40, 34
-; MUBUF-NEXT:    v_readlane_b32 s69, v40, 33
-; MUBUF-NEXT:    v_readlane_b32 s68, v40, 32
-; MUBUF-NEXT:    v_readlane_b32 s67, v40, 31
-; MUBUF-NEXT:    v_readlane_b32 s66, v40, 30
-; MUBUF-NEXT:    v_readlane_b32 s65, v40, 29
-; MUBUF-NEXT:    v_readlane_b32 s64, v40, 28
-; MUBUF-NEXT:    v_readlane_b32 s63, v40, 27
-; MUBUF-NEXT:    v_readlane_b32 s62, v40, 26
-; MUBUF-NEXT:    v_readlane_b32 s61, v40, 25
-; MUBUF-NEXT:    v_readlane_b32 s60, v40, 24
-; MUBUF-NEXT:    v_readlane_b32 s59, v40, 23
-; MUBUF-NEXT:    v_readlane_b32 s58, v40, 22
-; MUBUF-NEXT:    v_readlane_b32 s57, v40, 21
-; MUBUF-NEXT:    v_readlane_b32 s56, v40, 20
-; MUBUF-NEXT:    v_readlane_b32 s55, v40, 19
-; MUBUF-NEXT:    v_readlane_b32 s54, v40, 18
-; MUBUF-NEXT:    v_readlane_b32 s53, v40, 17
-; MUBUF-NEXT:    v_readlane_b32 s52, v40, 16
-; MUBUF-NEXT:    v_readlane_b32 s51, v40, 15
-; MUBUF-NEXT:    v_readlane_b32 s50, v40, 14
-; MUBUF-NEXT:    v_readlane_b32 s49, v40, 13
-; MUBUF-NEXT:    v_readlane_b32 s48, v40, 12
-; MUBUF-NEXT:    v_readlane_b32 s47, v40, 11
-; MUBUF-NEXT:    v_readlane_b32 s46, v40, 10
-; MUBUF-NEXT:    v_readlane_b32 s45, v40, 9
-; MUBUF-NEXT:    v_readlane_b32 s44, v40, 8
-; MUBUF-NEXT:    v_readlane_b32 s43, v40, 7
-; MUBUF-NEXT:    v_readlane_b32 s42, v40, 6
-; MUBUF-NEXT:    v_readlane_b32 s41, v40, 5
-; MUBUF-NEXT:    v_readlane_b32 s40, v40, 4
+; MUBUF-NEXT:    v_readlane_b32 s83, v40, 23
+; MUBUF-NEXT:    v_readlane_b32 s82, v40, 22
+; MUBUF-NEXT:    v_readlane_b32 s81, v40, 21
+; MUBUF-NEXT:    v_readlane_b32 s80, v40, 20
+; MUBUF-NEXT:    v_readlane_b32 s71, v40, 19
+; MUBUF-NEXT:    v_readlane_b32 s70, v40, 18
+; MUBUF-NEXT:    v_readlane_b32 s69, v40, 17
+; MUBUF-NEXT:    v_readlane_b32 s68, v40, 16
+; MUBUF-NEXT:    v_readlane_b32 s67, v40, 15
+; MUBUF-NEXT:    v_readlane_b32 s66, v40, 14
+; MUBUF-NEXT:    v_readlane_b32 s65, v40, 13
+; MUBUF-NEXT:    v_readlane_b32 s64, v40, 12
+; MUBUF-NEXT:    v_readlane_b32 s55, v40, 11
+; MUBUF-NEXT:    v_readlane_b32 s54, v40, 10
+; MUBUF-NEXT:    v_readlane_b32 s53, v40, 9
+; MUBUF-NEXT:    v_readlane_b32 s52, v40, 8
+; MUBUF-NEXT:    v_readlane_b32 s51, v40, 7
+; MUBUF-NEXT:    v_readlane_b32 s50, v40, 6
+; MUBUF-NEXT:    v_readlane_b32 s49, v40, 5
+; MUBUF-NEXT:    v_readlane_b32 s48, v40, 4
 ; MUBUF-NEXT:    v_readlane_b32 s39, v40, 3
 ; MUBUF-NEXT:    v_readlane_b32 s38, v40, 2
 ; MUBUF-NEXT:    v_readlane_b32 s37, v40, 1
@@ -409,48 +361,22 @@ define void @callee_func_sgpr_spill_no_calls(i32 %in) #0 {
 ; FLATSCR-NEXT:    s_mov_b64 exec, s[0:1]
 ; FLATSCR-NEXT:    v_writelane_b32 v40, s30, 0
 ; FLATSCR-NEXT:    v_writelane_b32 v40, s31, 1
-; FLATSCR-NEXT:    v_writelane_b32 v40, s34, 2
-; FLATSCR-NEXT:    v_writelane_b32 v40, s35, 3
-; FLATSCR-NEXT:    v_writelane_b32 v40, s36, 4
-; FLATSCR-NEXT:    v_writelane_b32 v40, s37, 5
-; FLATSCR-NEXT:    v_writelane_b32 v40, s38, 6
-; FLATSCR-NEXT:    v_writelane_b32 v40, s39, 7
-; FLATSCR-NEXT:    v_writelane_b32 v40, s40, 8
-; FLATSCR-NEXT:    v_writelane_b32 v40, s41, 9
-; FLATSCR-NEXT:    v_writelane_b32 v40, s42, 10
-; FLATSCR-NEXT:    v_writelane_b32 v40, s43, 11
-; FLATSCR-NEXT:    v_writelane_b32 v40, s44, 12
-; FLATSCR-NEXT:    v_writelane_b32 v40, s45, 13
-; FLATSCR-NEXT:    v_writelane_b32 v40, s46, 14
-; FLATSCR-NEXT:    v_writelane_b32 v40, s47, 15
-; FLATSCR-NEXT:    v_writelane_b32 v40, s48, 16
-; FLATSCR-NEXT:    v_writelane_b32 v40, s49, 17
-; FLATSCR-NEXT:    v_writelane_b32 v40, s50, 18
-; FLATSCR-NEXT:    v_writelane_b32 v40, s51, 19
-; FLATSCR-NEXT:    v_writelane_b32 v40, s52, 20
-; FLATSCR-NEXT:    v_writelane_b32 v40, s53, 21
-; FLATSCR-NEXT:    v_writelane_b32 v40, s54, 22
-; FLATSCR-NEXT:    v_writelane_b32 v40, s55, 23
-; FLATSCR-NEXT:    v_writelane_b32 v40, s56, 24
-; FLATSCR-NEXT:    v_writelane_b32 v40, s57, 25
-; FLATSCR-NEXT:    v_writelane_b32 v40, s58, 26
-; FLATSCR-NEXT:    v_writelane_b32 v40, s59, 27
-; FLATSCR-NEXT:    v_writelane_b32 v40, s60, 28
-; FLATSCR-NEXT:    v_writelane_b32 v40, s61, 29
-; FLATSCR-NEXT:    v_writelane_b32 v40, s62, 30
-; FLATSCR-NEXT:    v_writelane_b32 v40, s63, 31
-; FLATSCR-NEXT:    v_writelane_b32 v40, s64, 32
-; FLATSCR-NEXT:    v_writelane_b32 v40, s65, 33
-; FLATSCR-NEXT:    v_writelane_b32 v40, s66, 34
-; FLATSCR-NEXT:    v_writelane_b32 v40, s67, 35
-; FLATSCR-NEXT:    v_writelane_b32 v40, s68, 36
-; FLATSCR-NEXT:    v_writelane_b32 v40, s69, 37
-; FLATSCR-NEXT:    v_writelane_b32 v40, s70, 38
-; FLATSCR-NEXT:    v_writelane_b32 v40, s71, 39
-; FLATSCR-NEXT:    v_writelane_b32 v40, s72, 40
-; FLATSCR-NEXT:    v_writelane_b32 v40, s73, 41
-; FLATSCR-NEXT:    v_writelane_b32 v40, s74, 42
-; FLATSCR-NEXT:    v_writelane_b32 v40, s75, 43
+; FLATSCR-NEXT:    v_writelane_b32 v40, s36, 2
+; FLATSCR-NEXT:    v_writelane_b32 v40, s37, 3
+; FLATSCR-NEXT:    v_writelane_b32 v40, s38, 4
+; FLATSCR-NEXT:    v_writelane_b32 v40, s39, 5
+; FLATSCR-NEXT:    v_writelane_b32 v40, s48, 6
+; FLATSCR-NEXT:    v_writelane_b32 v40, s49, 7
+; FLATSCR-NEXT:    v_writelane_b32 v40, s50, 8
+; FLATSCR-NEXT:    v_writelane_b32 v40, s51, 9
+; FLATSCR-NEXT:    v_writelane_b32 v40, s52, 10
+; FLATSCR-NEXT:    v_writelane_b32 v40, s53, 11
+; FLATSCR-NEXT:    v_writelane_b32 v40, s54, 12
+; FLATSCR-NEXT:    v_writelane_b32 v40, s55, 13
+; FLATSCR-NEXT:    v_writelane_b32 v40, s64, 14
+; FLATSCR-NEXT:    v_writelane_b32 v40, s65, 15
+; FLATSCR-NEXT:    v_writelane_b32 v40, s66, 16
+; FLATSCR-NEXT:    v_writelane_b32 v40, s67, 17
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ;;#ASMEND
 ; FLATSCR-NEXT:    ;;#ASMSTART
@@ -474,10 +400,10 @@ define void @callee_func_sgpr_spill_no_calls(i32 %in) #0 {
 ; FLATSCR-NEXT:    ; def s[0:15]
 ; FLATSCR-NEXT:    ;;#ASMEND
 ; FLATSCR-NEXT:    ;;#ASMSTART
-; FLATSCR-NEXT:    ; def s[68:75]
+; FLATSCR-NEXT:    ; def s[72:79]
 ; FLATSCR-NEXT:    ;;#ASMEND
 ; FLATSCR-NEXT:    ;;#ASMSTART
-; FLATSCR-NEXT:    ; def s[34:35]
+; FLATSCR-NEXT:    ; def s[88:89]
 ; FLATSCR-NEXT:    ;;#ASMEND
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ; use s[52:67]
@@ -489,56 +415,30 @@ define void @callee_func_sgpr_spill_no_calls(i32 %in) #0 {
 ; FLATSCR-NEXT:    ; use s[16:31]
 ; FLATSCR-NEXT:    ;;#ASMEND
 ; FLATSCR-NEXT:    ;;#ASMSTART
-; FLATSCR-NEXT:    ; use s[68:75]
+; FLATSCR-NEXT:    ; use s[72:79]
 ; FLATSCR-NEXT:    ;;#ASMEND
 ; FLATSCR-NEXT:    ;;#ASMSTART
-; FLATSCR-NEXT:    ; use s[34:35]
+; FLATSCR-NEXT:    ; use s[88:89]
 ; FLATSCR-NEXT:    ;;#ASMEND
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ; use s[0:15]
 ; FLATSCR-NEXT:    ;;#ASMEND
-; FLATSCR-NEXT:    v_readlane_b32 s75, v40, 43
-; FLATSCR-NEXT:    v_readlane_b32 s74, v40, 42
-; FLATSCR-NEXT:    v_readlane_b32 s73, v40, 41
-; FLATSCR-NEXT:    v_readlane_b32 s72, v40, 40
-; FLATSCR-NEXT:    v_readlane_b32 s71, v40, 39
-; FLATSCR-NEXT:    v_readlane_b32 s70, v40, 38
-; FLATSCR-NEXT:    v_readlane_b32 s69, v40, 37
-; FLATSCR-NEXT:    v_readlane_b32 s68, v40, 36
-; FLATSCR-NEXT:    v_readlane_b32 s67, v40, 35
-; FLATSCR-NEXT:    v_readlane_b32 s66, v40, 34
-; FLATSCR-NEXT:    v_readlane_b32 s65, v40, 33
-; FLATSCR-NEXT:    v_readlane_b32 s64, v40, 32
-; FLATSCR-NEXT:    v_readlane_b32 s63, v40, 31
-; FLATSCR-NEXT:    v_readlane_b32 s62, v40, 30
-; FLATSCR-NEXT:    v_readlane_b32 s61, v40, 29
-; FLATSCR-NEXT:    v_readlane_b32 s60, v40, 28
-; FLATSCR-NEXT:    v_readlane_b32 s59, v40, 27
-; FLATSCR-NEXT:    v_readlane_b32 s58, v40, 26
-; FLATSCR-NEXT:    v_readlane_b32 s57, v40, 25
-; FLATSCR-NEXT:    v_readlane_b32 s56, v40, 24
-; FLATSCR-NEXT:    v_readlane_b32 s55, v40, 23
-; FLATSCR-NEXT:    v_readlane_b32 s54, v40, 22
-; FLATSCR-NEXT:    v_readlane_b32 s53, v40, 21
-; FLATSCR-NEXT:    v_readlane_b32 s52, v40, 20
-; FLATSCR-NEXT:    v_readlane_b32 s51, v40, 19
-; FLATSCR-NEXT:    v_readlane_b32 s50, v40, 18
-; FLATSCR-NEXT:    v_readlane_b32 s49, v40, 17
-; FLATSCR-NEXT:    v_readlane_b32 s48, v40, 16
-; FLATSCR-NEXT:    v_readlane_b32 s47, v40, 15
-; FLATSCR-NEXT:    v_readlane_b32 s46, v40, 14
-; FLATSCR-NEXT:    v_readlane_b32 s45, v40, 13
-; FLATSCR-NEXT:    v_readlane_b32 s44, v40, 12
-; FLATSCR-NEXT:    v_readlane_b32 s43, v40, 11
-; FLATSCR-NEXT:    v_readlane_b32 s42, v40, 10
-; FLATSCR-NEXT:    v_readlane_b32 s41, v40, 9
-; FLATSCR-NEXT:    v_readlane_b32 s40, v40, 8
-; FLATSCR-NEXT:    v_readlane_b32 s39, v40, 7
-; FLATSCR-NEXT:    v_readlane_b32 s38, v40, 6
-; FLATSCR-NEXT:    v_readlane_b32 s37, v40, 5
-; FLATSCR-NEXT:    v_readlane_b32 s36, v40, 4
-; FLATSCR-NEXT:    v_readlane_b32 s35, v40, 3
-; FLATSCR-NEXT:    v_readlane_b32 s34, v40, 2
+; FLATSCR-NEXT:    v_readlane_b32 s67, v40, 17
+; FLATSCR-NEXT:    v_readlane_b32 s66, v40, 16
+; FLATSCR-NEXT:    v_readlane_b32 s65, v40, 15
+; FLATSCR-NEXT:    v_readlane_b32 s64, v40, 14
+; FLATSCR-NEXT:    v_readlane_b32 s55, v40, 13
+; FLATSCR-NEXT:    v_readlane_b32 s54, v40, 12
+; FLATSCR-NEXT:    v_readlane_b32 s53, v40, 11
+; FLATSCR-NEXT:    v_readlane_b32 s52, v40, 10
+; FLATSCR-NEXT:    v_readlane_b32 s51, v40, 9
+; FLATSCR-NEXT:    v_readlane_b32 s50, v40, 8
+; FLATSCR-NEXT:    v_readlane_b32 s49, v40, 7
+; FLATSCR-NEXT:    v_readlane_b32 s48, v40, 6
+; FLATSCR-NEXT:    v_readlane_b32 s39, v40, 5
+; FLATSCR-NEXT:    v_readlane_b32 s38, v40, 4
+; FLATSCR-NEXT:    v_readlane_b32 s37, v40, 3
+; FLATSCR-NEXT:    v_readlane_b32 s36, v40, 2
 ; FLATSCR-NEXT:    v_readlane_b32 s31, v40, 1
 ; FLATSCR-NEXT:    v_readlane_b32 s30, v40, 0
 ; FLATSCR-NEXT:    s_or_saveexec_b64 s[0:1], -1
@@ -571,39 +471,13 @@ define void @callee_func_sgpr_spill_no_calls(i32 %in) #0 {
 ; Has no spilled CSR VGPRs used for SGPR spilling, so no need to
 ; enable all lanes and restore.
 define void @spill_only_csr_sgpr() {
-; MUBUF-LABEL: spill_only_csr_sgpr:
-; MUBUF:       ; %bb.0:
-; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; MUBUF-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill
-; MUBUF-NEXT:    s_mov_b64 exec, s[4:5]
-; MUBUF-NEXT:    v_writelane_b32 v0, s42, 0
-; MUBUF-NEXT:    ;;#ASMSTART
-; MUBUF-NEXT:    ; clobber s42
-; MUBUF-NEXT:    ;;#ASMEND
-; MUBUF-NEXT:    v_readlane_b32 s42, v0, 0
-; MUBUF-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; MUBUF-NEXT:    buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
-; MUBUF-NEXT:    s_mov_b64 exec, s[4:5]
-; MUBUF-NEXT:    s_waitcnt vmcnt(0)
-; MUBUF-NEXT:    s_setpc_b64 s[30:31]
-;
-; FLATSCR-LABEL: spill_only_csr_sgpr:
-; FLATSCR:       ; %bb.0:
-; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; FLATSCR-NEXT:    s_xor_saveexec_b64 s[0:1], -1
-; FLATSCR-NEXT:    scratch_store_dword off, v0, s32 ; 4-byte Folded Spill
-; FLATSCR-NEXT:    s_mov_b64 exec, s[0:1]
-; FLATSCR-NEXT:    v_writelane_b32 v0, s42, 0
-; FLATSCR-NEXT:    ;;#ASMSTART
-; FLATSCR-NEXT:    ; clobber s42
-; FLATSCR-NEXT:    ;;#ASMEND
-; FLATSCR-NEXT:    v_readlane_b32 s42, v0, 0
-; FLATSCR-NEXT:    s_xor_saveexec_b64 s[0:1], -1
-; FLATSCR-NEXT:    scratch_load_dword v0, off, s32 ; 4-byte Folded Reload
-; FLATSCR-NEXT:    s_mov_b64 exec, s[0:1]
-; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: spill_only_csr_sgpr:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    ;;#ASMSTART
+; GCN-NEXT:    ; clobber s42
+; GCN-NEXT:    ;;#ASMEND
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   call void asm sideeffect "; clobber s42", "~{s42}"()
   ret void
 }
@@ -663,143 +537,79 @@ define void @last_lane_vgpr_for_fp_csr() #1 {
 ; MUBUF-NEXT:    s_xor_saveexec_b64 s[6:7], -1
 ; MUBUF-NEXT:    buffer_store_dword v1, off, s[0:3], s33 offset:8 ; 4-byte Folded Spill
 ; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
-; MUBUF-NEXT:    v_writelane_b32 v1, s40, 0
-; MUBUF-NEXT:    v_writelane_b32 v1, s41, 1
-; MUBUF-NEXT:    v_writelane_b32 v1, s42, 2
-; MUBUF-NEXT:    v_writelane_b32 v1, s43, 3
-; MUBUF-NEXT:    v_writelane_b32 v1, s44, 4
-; MUBUF-NEXT:    v_writelane_b32 v1, s45, 5
-; MUBUF-NEXT:    v_writelane_b32 v1, s46, 6
-; MUBUF-NEXT:    v_writelane_b32 v1, s47, 7
-; MUBUF-NEXT:    v_writelane_b32 v1, s48, 8
-; MUBUF-NEXT:    v_writelane_b32 v1, s49, 9
-; MUBUF-NEXT:    v_writelane_b32 v1, s50, 10
-; MUBUF-NEXT:    v_writelane_b32 v1, s51, 11
-; MUBUF-NEXT:    v_writelane_b32 v1, s52, 12
-; MUBUF-NEXT:    v_writelane_b32 v1, s53, 13
-; MUBUF-NEXT:    v_writelane_b32 v1, s54, 14
-; MUBUF-NEXT:    v_writelane_b32 v1, s55, 15
-; MUBUF-NEXT:    v_writelane_b32 v1, s56, 16
-; MUBUF-NEXT:    v_writelane_b32 v1, s57, 17
-; MUBUF-NEXT:    v_writelane_b32 v1, s58, 18
-; MUBUF-NEXT:    v_writelane_b32 v1, s59, 19
-; MUBUF-NEXT:    v_writelane_b32 v1, s60, 20
-; MUBUF-NEXT:    v_writelane_b32 v1, s61, 21
-; MUBUF-NEXT:    v_writelane_b32 v1, s62, 22
-; MUBUF-NEXT:    v_writelane_b32 v1, s63, 23
-; MUBUF-NEXT:    v_writelane_b32 v1, s64, 24
-; MUBUF-NEXT:    v_writelane_b32 v1, s65, 25
-; MUBUF-NEXT:    v_writelane_b32 v1, s66, 26
-; MUBUF-NEXT:    v_writelane_b32 v1, s67, 27
-; MUBUF-NEXT:    v_writelane_b32 v1, s68, 28
-; MUBUF-NEXT:    v_writelane_b32 v1, s69, 29
-; MUBUF-NEXT:    v_writelane_b32 v1, s70, 30
-; MUBUF-NEXT:    v_writelane_b32 v1, s71, 31
-; MUBUF-NEXT:    v_writelane_b32 v1, s72, 32
-; MUBUF-NEXT:    v_writelane_b32 v1, s73, 33
-; MUBUF-NEXT:    v_writelane_b32 v1, s74, 34
-; MUBUF-NEXT:    v_writelane_b32 v1, s75, 35
-; MUBUF-NEXT:    v_writelane_b32 v1, s76, 36
-; MUBUF-NEXT:    v_writelane_b32 v1, s77, 37
-; MUBUF-NEXT:    v_writelane_b32 v1, s78, 38
-; MUBUF-NEXT:    v_writelane_b32 v1, s79, 39
-; MUBUF-NEXT:    v_writelane_b32 v1, s80, 40
-; MUBUF-NEXT:    v_writelane_b32 v1, s81, 41
-; MUBUF-NEXT:    v_writelane_b32 v1, s82, 42
-; MUBUF-NEXT:    v_writelane_b32 v1, s83, 43
-; MUBUF-NEXT:    v_writelane_b32 v1, s84, 44
-; MUBUF-NEXT:    v_writelane_b32 v1, s85, 45
-; MUBUF-NEXT:    v_writelane_b32 v1, s86, 46
-; MUBUF-NEXT:    v_writelane_b32 v1, s87, 47
-; MUBUF-NEXT:    v_writelane_b32 v1, s88, 48
-; MUBUF-NEXT:    v_writelane_b32 v1, s89, 49
-; MUBUF-NEXT:    v_writelane_b32 v1, s90, 50
-; MUBUF-NEXT:    v_writelane_b32 v1, s91, 51
-; MUBUF-NEXT:    v_writelane_b32 v1, s92, 52
-; MUBUF-NEXT:    v_writelane_b32 v1, s93, 53
-; MUBUF-NEXT:    v_writelane_b32 v1, s94, 54
-; MUBUF-NEXT:    v_writelane_b32 v1, s95, 55
-; MUBUF-NEXT:    v_writelane_b32 v1, s96, 56
-; MUBUF-NEXT:    v_writelane_b32 v1, s97, 57
-; MUBUF-NEXT:    v_writelane_b32 v1, s98, 58
-; MUBUF-NEXT:    v_writelane_b32 v1, s99, 59
-; MUBUF-NEXT:    v_writelane_b32 v1, s100, 60
+; MUBUF-NEXT:    v_writelane_b32 v1, s48, 0
+; MUBUF-NEXT:    v_writelane_b32 v1, s49, 1
+; MUBUF-NEXT:    v_writelane_b32 v1, s50, 2
+; MUBUF-NEXT:    v_writelane_b32 v1, s51, 3
+; MUBUF-NEXT:    v_writelane_b32 v1, s52, 4
+; MUBUF-NEXT:    v_writelane_b32 v1, s53, 5
+; MUBUF-NEXT:    v_writelane_b32 v1, s54, 6
+; MUBUF-NEXT:    v_writelane_b32 v1, s55, 7
+; MUBUF-NEXT:    v_writelane_b32 v1, s64, 8
+; MUBUF-NEXT:    v_writelane_b32 v1, s65, 9
+; MUBUF-NEXT:    v_writelane_b32 v1, s66, 10
+; MUBUF-NEXT:    v_writelane_b32 v1, s67, 11
+; MUBUF-NEXT:    v_writelane_b32 v1, s68, 12
+; MUBUF-NEXT:    v_writelane_b32 v1, s69, 13
+; MUBUF-NEXT:    v_writelane_b32 v1, s70, 14
+; MUBUF-NEXT:    v_writelane_b32 v1, s71, 15
+; MUBUF-NEXT:    v_writelane_b32 v1, s80, 16
+; MUBUF-NEXT:    v_writelane_b32 v1, s81, 17
+; MUBUF-NEXT:    v_writelane_b32 v1, s82, 18
+; MUBUF-NEXT:    v_writelane_b32 v1, s83, 19
+; MUBUF-NEXT:    v_writelane_b32 v1, s84, 20
+; MUBUF-NEXT:    v_writelane_b32 v1, s85, 21
+; MUBUF-NEXT:    v_writelane_b32 v1, s86, 22
+; MUBUF-NEXT:    v_writelane_b32 v1, s87, 23
+; MUBUF-NEXT:    v_writelane_b32 v1, s96, 24
+; MUBUF-NEXT:    v_writelane_b32 v1, s97, 25
+; MUBUF-NEXT:    v_writelane_b32 v1, s98, 26
+; MUBUF-NEXT:    v_writelane_b32 v1, s99, 27
+; MUBUF-NEXT:    v_writelane_b32 v1, s100, 28
 ; MUBUF-NEXT:    v_mov_b32_e32 v0, 0
 ; MUBUF-NEXT:    buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
-; MUBUF-NEXT:    v_writelane_b32 v1, s101, 61
+; MUBUF-NEXT:    v_writelane_b32 v1, s101, 29
 ; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s33 offset:4
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
 ; MUBUF-NEXT:    ;;#ASMSTART
 ; MUBUF-NEXT:    ; clobber v41
 ; MUBUF-NEXT:    ;;#ASMEND
-; MUBUF-NEXT:    v_writelane_b32 v1, s102, 62
+; MUBUF-NEXT:    v_writelane_b32 v1, s102, 30
 ; MUBUF-NEXT:    ;;#ASMSTART
 ; MUBUF-NEXT:    ;;#ASMEND
 ; MUBUF-NEXT:    buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload
 ; MUBUF-NEXT:    s_addk_i32 s32, 0x400
-; MUBUF-NEXT:    v_readlane_b32 s102, v1, 62
-; MUBUF-NEXT:    v_readlane_b32 s101, v1, 61
-; MUBUF-NEXT:    v_readlane_b32 s100, v1, 60
-; MUBUF-NEXT:    v_readlane_b32 s99, v1, 59
-; MUBUF-NEXT:    v_readlane_b32 s98, v1, 58
-; MUBUF-NEXT:    v_readlane_b32 s97, v1, 57
-; MUBUF-NEXT:    v_readlane_b32 s96, v1, 56
-; MUBUF-NEXT:    v_readlane_b32 s95, v1, 55
-; MUBUF-NEXT:    v_readlane_b32 s94, v1, 54
-; MUBUF-NEXT:    v_readlane_b32 s93, v1, 53
-; MUBUF-NEXT:    v_readlane_b32 s92, v1, 52
-; MUBUF-NEXT:    v_readlane_b32 s91, v1, 51
-; MUBUF-NEXT:    v_readlane_b32 s90, v1, 50
-; MUBUF-NEXT:    v_readlane_b32 s89, v1, 49
-; MUBUF-NEXT:    v_readlane_b32 s88, v1, 48
-; MUBUF-NEXT:    v_readlane_b32 s87, v1, 47
-; MUBUF-NEXT:    v_readlane_b32 s86, v1, 46
-; MUBUF-NEXT:    v_readlane_b32 s85, v1, 45
-; MUBUF-NEXT:    v_readlane_b32 s84, v1, 44
-; MUBUF-NEXT:    v_readlane_b32 s83, v1, 43
-; MUBUF-NEXT:    v_readlane_b32 s82, v1, 42
-; MUBUF-NEXT:    v_readlane_b32 s81, v1, 41
-; MUBUF-NEXT:    v_readlane_b32 s80, v1, 40
-; MUBUF-NEXT:    v_readlane_b32 s79, v1, 39
-; MUBUF-NEXT:    v_readlane_b32 s78, v1, 38
-; MUBUF-NEXT:    v_readlane_b32 s77, v1, 37
-; MUBUF-NEXT:    v_readlane_b32 s76, v1, 36
-; MUBUF-NEXT:    v_readlane_b32 s75, v1, 35
-; MUBUF-NEXT:    v_readlane_b32 s74, v1, 34
-; MUBUF-NEXT:    v_readlane_b32 s73, v1, 33
-; MUBUF-NEXT:    v_readlane_b32 s72, v1, 32
-; MUBUF-NEXT:    v_readlane_b32 s71, v1, 31
-; MUBUF-NEXT:    v_readlane_b32 s70, v1, 30
-; MUBUF-NEXT:    v_readlane_b32 s69, v1, 29
-; MUBUF-NEXT:    v_readlane_b32 s68, v1, 28
-; MUBUF-NEXT:    v_readlane_b32 s67, v1, 27
-; MUBUF-NEXT:    v_readlane_b32 s66, v1, 26
-; MUBUF-NEXT:    v_readlane_b32 s65, v1, 25
-; MUBUF-NEXT:    v_readlane_b32 s64, v1, 24
-; MUBUF-NEXT:    v_readlane_b32 s63, v1, 23
-; MUBUF-NEXT:    v_readlane_b32 s62, v1, 22
-; MUBUF-NEXT:    v_readlane_b32 s61, v1, 21
-; MUBUF-NEXT:    v_readlane_b32 s60, v1, 20
-; MUBUF-NEXT:    v_readlane_b32 s59, v1, 19
-; MUBUF-NEXT:    v_readlane_b32 s58, v1, 18
-; MUBUF-NEXT:    v_readlane_b32 s57, v1, 17
-; MUBUF-NEXT:    v_readlane_b32 s56, v1, 16
-; MUBUF-NEXT:    v_readlane_b32 s55, v1, 15
-; MUBUF-NEXT:    v_readlane_b32 s54, v1, 14
-; MUBUF-NEXT:    v_readlane_b32 s53, v1, 13
-; MUBUF-NEXT:    v_readlane_b32 s52, v1, 12
-; MUBUF-NEXT:    v_readlane_b32 s51, v1, 11
-; MUBUF-NEXT:    v_readlane_b32 s50, v1, 10
-; MUBUF-NEXT:    v_readlane_b32 s49, v1, 9
-; MUBUF-NEXT:    v_readlane_b32 s48, v1, 8
-; MUBUF-NEXT:    v_readlane_b32 s47, v1, 7
-; MUBUF-NEXT:    v_readlane_b32 s46, v1, 6
-; MUBUF-NEXT:    v_readlane_b32 s45, v1, 5
-; MUBUF-NEXT:    v_readlane_b32 s44, v1, 4
-; MUBUF-NEXT:    v_readlane_b32 s43, v1, 3
-; MUBUF-NEXT:    v_readlane_b32 s42, v1, 2
-; MUBUF-NEXT:    v_readlane_b32 s41, v1, 1
-; MUBUF-NEXT:    v_readlane_b32 s40, v1, 0
+; MUBUF-NEXT:    v_readlane_b32 s102, v1, 30
+; MUBUF-NEXT:    v_readlane_b32 s101, v1, 29
+; MUBUF-NEXT:    v_readlane_b32 s100, v1, 28
+; MUBUF-NEXT:    v_readlane_b32 s99, v1, 27
+; MUBUF-NEXT:    v_readlane_b32 s98, v1, 26
+; MUBUF-NEXT:    v_readlane_b32 s97, v1, 25
+; MUBUF-NEXT:    v_readlane_b32 s96, v1, 24
+; MUBUF-NEXT:    v_readlane_b32 s87, v1, 23
+; MUBUF-NEXT:    v_readlane_b32 s86, v1, 22
+; MUBUF-NEXT:    v_readlane_b32 s85, v1, 21
+; MUBUF-NEXT:    v_readlane_b32 s84, v1, 20
+; MUBUF-NEXT:    v_readlane_b32 s83, v1, 19
+; MUBUF-NEXT:    v_readlane_b32 s82, v1, 18
+; MUBUF-NEXT:    v_readlane_b32 s81, v1, 17
+; MUBUF-NEXT:    v_readlane_b32 s80, v1, 16
+; MUBUF-NEXT:    v_readlane_b32 s71, v1, 15
+; MUBUF-NEXT:    v_readlane_b32 s70, v1, 14
+; MUBUF-NEXT:    v_readlane_b32 s69, v1, 13
+; MUBUF-NEXT:    v_readlane_b32 s68, v1, 12
+; MUBUF-NEXT:    v_readlane_b32 s67, v1, 11
+; MUBUF-NEXT:    v_readlane_b32 s66, v1, 10
+; MUBUF-NEXT:    v_readlane_b32 s65, v1, 9
+; MUBUF-NEXT:    v_readlane_b32 s64, v1, 8
+; MUBUF-NEXT:    v_readlane_b32 s55, v1, 7
+; MUBUF-NEXT:    v_readlane_b32 s54, v1, 6
+; MUBUF-NEXT:    v_readlane_b32 s53, v1, 5
+; MUBUF-NEXT:    v_readlane_b32 s52, v1, 4
+; MUBUF-NEXT:    v_readlane_b32 s51, v1, 3
+; MUBUF-NEXT:    v_readlane_b32 s50, v1, 2
+; MUBUF-NEXT:    v_readlane_b32 s49, v1, 1
+; MUBUF-NEXT:    v_readlane_b32 s48, v1, 0
 ; MUBUF-NEXT:    s_mov_b32 s32, s33
 ; MUBUF-NEXT:    s_xor_saveexec_b64 s[6:7], -1
 ; MUBUF-NEXT:    buffer_load_dword v1, off, s[0:3], s33 offset:8 ; 4-byte Folded Reload
@@ -816,143 +626,79 @@ define void @last_lane_vgpr_for_fp_csr() #1 {
 ; FLATSCR-NEXT:    s_xor_saveexec_b64 s[2:3], -1
 ; FLATSCR-NEXT:    scratch_store_dword off, v1, s33 offset:8 ; 4-byte Folded Spill
 ; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
-; FLATSCR-NEXT:    v_writelane_b32 v1, s40, 0
-; FLATSCR-NEXT:    v_writelane_b32 v1, s41, 1
-; FLATSCR-NEXT:    v_writelane_b32 v1, s42, 2
-; FLATSCR-NEXT:    v_writelane_b32 v1, s43, 3
-; FLATSCR-NEXT:    v_writelane_b32 v1, s44, 4
-; FLATSCR-NEXT:    v_writelane_b32 v1, s45, 5
-; FLATSCR-NEXT:    v_writelane_b32 v1, s46, 6
-; FLATSCR-NEXT:    v_writelane_b32 v1, s47, 7
-; FLATSCR-NEXT:    v_writelane_b32 v1, s48, 8
-; FLATSCR-NEXT:    v_writelane_b32 v1, s49, 9
-; FLATSCR-NEXT:    v_writelane_b32 v1, s50, 10
-; FLATSCR-NEXT:    v_writelane_b32 v1, s51, 11
-; FLATSCR-NEXT:    v_writelane_b32 v1, s52, 12
-; FLATSCR-NEXT:    v_writelane_b32 v1, s53, 13
-; FLATSCR-NEXT:    v_writelane_b32 v1, s54, 14
-; FLATSCR-NEXT:    v_writelane_b32 v1, s55, 15
-; FLATSCR-NEXT:    v_writelane_b32 v1, s56, 16
-; FLATSCR-NEXT:    v_writelane_b32 v1, s57, 17
-; FLATSCR-NEXT:    v_writelane_b32 v1, s58, 18
-; FLATSCR-NEXT:    v_writelane_b32 v1, s59, 19
-; FLATSCR-NEXT:    v_writelane_b32 v1, s60, 20
-; FLATSCR-NEXT:    v_writelane_b32 v1, s61, 21
-; FLATSCR-NEXT:    v_writelane_b32 v1, s62, 22
-; FLATSCR-NEXT:    v_writelane_b32 v1, s63, 23
-; FLATSCR-NEXT:    v_writelane_b32 v1, s64, 24
-; FLATSCR-NEXT:    v_writelane_b32 v1, s65, 25
-; FLATSCR-NEXT:    v_writelane_b32 v1, s66, 26
-; FLATSCR-NEXT:    v_writelane_b32 v1, s67, 27
-; FLATSCR-NEXT:    v_writelane_b32 v1, s68, 28
-; FLATSCR-NEXT:    v_writelane_b32 v1, s69, 29
-; FLATSCR-NEXT:    v_writelane_b32 v1, s70, 30
-; FLATSCR-NEXT:    v_writelane_b32 v1, s71, 31
-; FLATSCR-NEXT:    v_writelane_b32 v1, s72, 32
-; FLATSCR-NEXT:    v_writelane_b32 v1, s73, 33
-; FLATSCR-NEXT:    v_writelane_b32 v1, s74, 34
-; FLATSCR-NEXT:    v_writelane_b32 v1, s75, 35
-; FLATSCR-NEXT:    v_writelane_b32 v1, s76, 36
-; FLATSCR-NEXT:    v_writelane_b32 v1, s77, 37
-; FLATSCR-NEXT:    v_writelane_b32 v1, s78, 38
-; FLATSCR-NEXT:    v_writelane_b32 v1, s79, 39
-; FLATSCR-NEXT:    v_writelane_b32 v1, s80, 40
-; FLATSCR-NEXT:    v_writelane_b32 v1, s81, 41
-; FLATSCR-NEXT:    v_writelane_b32 v1, s82, 42
-; FLATSCR-NEXT:    v_writelane_b32 v1, s83, 43
-; FLATSCR-NEXT:    v_writelane_b32 v1, s84, 44
-; FLATSCR-NEXT:    v_writelane_b32 v1, s85, 45
-; FLATSCR-NEXT:    v_writelane_b32 v1, s86, 46
-; FLATSCR-NEXT:    v_writelane_b32 v1, s87, 47
-; FLATSCR-NEXT:    v_writelane_b32 v1, s88, 48
-; FLATSCR-NEXT:    v_writelane_b32 v1, s89, 49
-; FLATSCR-NEXT:    v_writelane_b32 v1, s90, 50
-; FLATSCR-NEXT:    v_writelane_b32 v1, s91, 51
-; FLATSCR-NEXT:    v_writelane_b32 v1, s92, 52
-; FLATSCR-NEXT:    v_writelane_b32 v1, s93, 53
-; FLATSCR-NEXT:    v_writelane_b32 v1, s94, 54
-; FLATSCR-NEXT:    v_writelane_b32 v1, s95, 55
-; FLATSCR-NEXT:    v_writelane_b32 v1, s96, 56
-; FLATSCR-NEXT:    v_writelane_b32 v1, s97, 57
-; FLATSCR-NEXT:    v_writelane_b32 v1, s98, 58
-; FLATSCR-NEXT:    v_writelane_b32 v1, s99, 59
-; FLATSCR-NEXT:    v_writelane_b32 v1, s100, 60
+; FLATSCR-NEXT:    v_writelane_b32 v1, s48, 0
+; FLATSCR-NEXT:    v_writelane_b32 v1, s49, 1
+; FLATSCR-NEXT:    v_writelane_b32 v1, s50, 2
+; FLATSCR-NEXT:    v_writelane_b32 v1, s51, 3
+; FLATSCR-NEXT:    v_writelane_b32 v1, s52, 4
+; FLATSCR-NEXT:    v_writelane_b32 v1, s53, 5
+; FLATSCR-NEXT:    v_writelane_b32 v1, s54, 6
+; FLATSCR-NEXT:    v_writelane_b32 v1, s55, 7
+; FLATSCR-NEXT:    v_writelane_b32 v1, s64, 8
+; FLATSCR-NEXT:    v_writelane_b32 v1, s65, 9
+; FLATSCR-NEXT:    v_writelane_b32 v1, s66, 10
+; FLATSCR-NEXT:    v_writelane_b32 v1, s67, 11
+; FLATSCR-NEXT:    v_writelane_b32 v1, s68, 12
+; FLATSCR-NEXT:    v_writelane_b32 v1, s69, 13
+; FLATSCR-NEXT:    v_writelane_b32 v1, s70, 14
+; FLATSCR-NEXT:    v_writelane_b32 v1, s71, 15
+; FLATSCR-NEXT:    v_writelane_b32 v1, s80, 16
+; FLATSCR-NEXT:    v_writelane_b32 v1, s81, 17
+; FLATSCR-NEXT:    v_writelane_b32 v1, s82, 18
+; FLATSCR-NEXT:    v_writelane_b32 v1, s83, 19
+; FLATSCR-NEXT:    v_writelane_b32 v1, s84, 20
+; FLATSCR-NEXT:    v_writelane_b32 v1, s85, 21
+; FLATSCR-NEXT:    v_writelane_b32 v1, s86, 22
+; FLATSCR-NEXT:    v_writelane_b32 v1, s87, 23
+; FLATSCR-NEXT:    v_writelane_b32 v1, s96, 24
+; FLATSCR-NEXT:    v_writelane_b32 v1, s97, 25
+; FLATSCR-NEXT:    v_writelane_b32 v1, s98, 26
+; FLATSCR-NEXT:    v_writelane_b32 v1, s99, 27
+; FLATSCR-NEXT:    v_writelane_b32 v1, s100, 28
 ; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
 ; FLATSCR-NEXT:    scratch_store_dword off, v41, s33 ; 4-byte Folded Spill
-; FLATSCR-NEXT:    v_writelane_b32 v1, s101, 61
+; FLATSCR-NEXT:    v_writelane_b32 v1, s101, 29
 ; FLATSCR-NEXT:    scratch_store_dword off, v0, s33 offset:4
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ; clobber v41
 ; FLATSCR-NEXT:    ;;#ASMEND
-; FLATSCR-NEXT:    v_writelane_b32 v1, s102, 62
+; FLATSCR-NEXT:    v_writelane_b32 v1, s102, 30
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ;;#ASMEND
 ; FLATSCR-NEXT:    scratch_load_dword v41, off, s33 ; 4-byte Folded Reload
 ; FLATSCR-NEXT:    s_add_i32 s32, s32, 16
-; FLATSCR-NEXT:    v_readlane_b32 s102, v1, 62
-; FLATSCR-NEXT:    v_readlane_b32 s101, v1, 61
-; FLATSCR-NEXT:    v_readlane_b32 s100, v1, 60
-; FLATSCR-NEXT:    v_readlane_b32 s99, v1, 59
-; FLATSCR-NEXT:    v_readlane_b32 s98, v1, 58
-; FLATSCR-NEXT:    v_readlane_b32 s97, v1, 57
-; FLATSCR-NEXT:    v_readlane_b32 s96, v1, 56
-; FLATSCR-NEXT:    v_readlane_b32 s95, v1, 55
-; FLATSCR-NEXT:    v_readlane_b32 s94, v1, 54
-; FLATSCR-NEXT:    v_readlane_b32 s93, v1, 53
-; FLATSCR-NEXT:    v_readlane_b32 s92, v1, 52
-; FLATSCR-NEXT:    v_readlane_b32 s91, v1, 51
-; FLATSCR-NEXT:    v_readlane_b32 s90, v1, 50
-; FLATSCR-NEXT:    v_readlane_b32 s89, v1, 49
-; FLATSCR-NEXT:    v_readlane_b32 s88, v1, 48
-; FLATSCR-NEXT:    v_readlane_b32 s87, v1, 47
-; FLATSCR-NEXT:    v_readlane_b32 s86, v1, 46
-; FLATSCR-NEXT:    v_readlane_b32 s85, v1, 45
-; FLATSCR-NEXT:    v_readlane_b32 s84, v1, 44
-; FLATSCR-NEXT:    v_readlane_b32 s83, v1, 43
-; FLATSCR-NEXT:    v_readlane_b32 s82, v1, 42
-; FLATSCR-NEXT:    v_readlane_b32 s81, v1, 41
-; FLATSCR-NEXT:    v_readlane_b32 s80, v1, 40
-; FLATSCR-NEXT:    v_readlane_b32 s79, v1, 39
-; FLATSCR-NEXT:    v_readlane_b32 s78, v1, 38
-; FLATSCR-NEXT:    v_readlane_b32 s77, v1, 37
-; FLATSCR-NEXT:    v_readlane_b32 s76, v1, 36
-; FLATSCR-NEXT:    v_readlane_b32 s75, v1, 35
-; FLATSCR-NEXT:    v_readlane_b32 s74, v1, 34
-; FLATSCR-NEXT:    v_readlane_b32 s73, v1, 33
-; FLATSCR-NEXT:    v_readlane_b32 s72, v1, 32
-; FLATSCR-NEXT:    v_readlane_b32 s71, v1, 31
-; FLATSCR-NEXT:    v_readlane_b32 s70, v1, 30
-; FLATSCR-NEXT:    v_readlane_b32 s69, v1, 29
-; FLATSCR-NEXT:    v_readlane_b32 s68, v1, 28
-; FLATSCR-NEXT:    v_readlane_b32 s67, v1, 27
-; FLATSCR-NEXT:    v_readlane_b32 s66, v1, 26
-; FLATSCR-NEXT:    v_readlane_b32 s65, v1, 25
-; FLATSCR-NEXT:    v_readlane_b32 s64, v1, 24
-; FLATSCR-NEXT:    v_readlane_b32 s63, v1, 23
-; FLATSCR-NEXT:    v_readlane_b32 s62, v1, 22
-; FLATSCR-NEXT:    v_readlane_b32 s61, v1, 21
-; FLATSCR-NEXT:    v_readlane_b32 s60, v1, 20
-; FLATSCR-NEXT:    v_readlane_b32 s59, v1, 19
-; FLATSCR-NEXT:    v_readlane_b32 s58, v1, 18
-; FLATSCR-NEXT:    v_readlane_b32 s57, v1, 17
-; FLATSCR-NEXT:    v_readlane_b32 s56, v1, 16
-; FLATSCR-NEXT:    v_readlane_b32 s55, v1, 15
-; FLATSCR-NEXT:    v_readlane_b32 s54, v1, 14
-; FLATSCR-NEXT:    v_readlane_b32 s53, v1, 13
-; FLATSCR-NEXT:    v_readlane_b32 s52, v1, 12
-; FLATSCR-NEXT:    v_readlane_b32 s51, v1, 11
-; FLATSCR-NEXT:    v_readlane_b32 s50, v1, 10
-; FLATSCR-NEXT:    v_readlane_b32 s49, v1, 9
-; FLATSCR-NEXT:    v_readlane_b32 s48, v1, 8
-; FLATSCR-NEXT:    v_readlane_b32 s47, v1, 7
-; FLATSCR-NEXT:    v_readlane_b32 s46, v1, 6
-; FLATSCR-NEXT:    v_readlane_b32 s45, v1, 5
-; FLATSCR-NEXT:    v_readlane_b32 s44, v1, 4
-; FLATSCR-NEXT:    v_readlane_b32 s43, v1, 3
-; FLATSCR-NEXT:    v_readlane_b32 s42, v1, 2
-; FLATSCR-NEXT:    v_readlane_b32 s41, v1, 1
-; FLATSCR-NEXT:    v_readlane_b32 s40, v1, 0
+; FLATSCR-NEXT:    v_readlane_b32 s102, v1, 30
+; FLATSCR-NEXT:    v_readlane_b32 s101, v1, 29
+; FLATSCR-NEXT:    v_readlane_b32 s100, v1, 28
+; FLATSCR-NEXT:    v_readlane_b32 s99, v1, 27
+; FLATSCR-NEXT:    v_readlane_b32 s98, v1, 26
+; FLATSCR-NEXT:    v_readlane_b32 s97, v1, 25
+; FLATSCR-NEXT:    v_readlane_b32 s96, v1, 24
+; FLATSCR-NEXT:    v_readlane_b32 s87, v1, 23
+; FLATSCR-NEXT:    v_readlane_b32 s86, v1, 22
+; FLATSCR-NEXT:    v_readlane_b32 s85, v1, 21
+; FLATSCR-NEXT:    v_readlane_b32 s84, v1, 20
+; FLATSCR-NEXT:    v_readlane_b32 s83, v1, 19
+; FLATSCR-NEXT:    v_readlane_b32 s82, v1, 18
+; FLATSCR-NEXT:    v_readlane_b32 s81, v1, 17
+; FLATSCR-NEXT:    v_readlane_b32 s80, v1, 16
+; FLATSCR-NEXT:    v_readlane_b32 s71, v1, 15
+; FLATSCR-NEXT:    v_readlane_b32 s70, v1, 14
+; FLATSCR-NEXT:    v_readlane_b32 s69, v1, 13
+; FLATSCR-NEXT:    v_readlane_b32 s68, v1, 12
+; FLATSCR-NEXT:    v_readlane_b32 s67, v1, 11
+; FLATSCR-NEXT:    v_readlane_b32 s66, v1, 10
+; FLATSCR-NEXT:    v_readlane_b32 s65, v1, 9
+; FLATSCR-NEXT:    v_readlane_b32 s64, v1, 8
+; FLATSCR-NEXT:    v_readlane_b32 s55, v1, 7
+; FLATSCR-NEXT:    v_readlane_b32 s54, v1, 6
+; FLATSCR-NEXT:    v_readlane_b32 s53, v1, 5
+; FLATSCR-NEXT:    v_readlane_b32 s52, v1, 4
+; FLATSCR-NEXT:    v_readlane_b32 s51, v1, 3
+; FLATSCR-NEXT:    v_readlane_b32 s50, v1, 2
+; FLATSCR-NEXT:    v_readlane_b32 s49, v1, 1
+; FLATSCR-NEXT:    v_readlane_b32 s48, v1, 0
 ; FLATSCR-NEXT:    s_mov_b32 s32, s33
 ; FLATSCR-NEXT:    s_xor_saveexec_b64 s[2:3], -1
 ; FLATSCR-NEXT:    scratch_load_dword v1, off, s33 offset:8 ; 4-byte Folded Reload
@@ -986,143 +732,79 @@ define void @no_new_vgpr_for_fp_csr() #1 {
 ; MUBUF-NEXT:    buffer_store_dword v1, off, s[0:3], s33 offset:8 ; 4-byte Folded Spill
 ; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
 ; MUBUF-NEXT:    v_writelane_b32 v1, s39, 0
-; MUBUF-NEXT:    v_writelane_b32 v1, s40, 1
-; MUBUF-NEXT:    v_writelane_b32 v1, s41, 2
-; MUBUF-NEXT:    v_writelane_b32 v1, s42, 3
-; MUBUF-NEXT:    v_writelane_b32 v1, s43, 4
-; MUBUF-NEXT:    v_writelane_b32 v1, s44, 5
-; MUBUF-NEXT:    v_writelane_b32 v1, s45, 6
-; MUBUF-NEXT:    v_writelane_b32 v1, s46, 7
-; MUBUF-NEXT:    v_writelane_b32 v1, s47, 8
-; MUBUF-NEXT:    v_writelane_b32 v1, s48, 9
-; MUBUF-NEXT:    v_writelane_b32 v1, s49, 10
-; MUBUF-NEXT:    v_writelane_b32 v1, s50, 11
-; MUBUF-NEXT:    v_writelane_b32 v1, s51, 12
-; MUBUF-NEXT:    v_writelane_b32 v1, s52, 13
-; MUBUF-NEXT:    v_writelane_b32 v1, s53, 14
-; MUBUF-NEXT:    v_writelane_b32 v1, s54, 15
-; MUBUF-NEXT:    v_writelane_b32 v1, s55, 16
-; MUBUF-NEXT:    v_writelane_b32 v1, s56, 17
-; MUBUF-NEXT:    v_writelane_b32 v1, s57, 18
-; MUBUF-NEXT:    v_writelane_b32 v1, s58, 19
-; MUBUF-NEXT:    v_writelane_b32 v1, s59, 20
-; MUBUF-NEXT:    v_writelane_b32 v1, s60, 21
-; MUBUF-NEXT:    v_writelane_b32 v1, s61, 22
-; MUBUF-NEXT:    v_writelane_b32 v1, s62, 23
-; MUBUF-NEXT:    v_writelane_b32 v1, s63, 24
-; MUBUF-NEXT:    v_writelane_b32 v1, s64, 25
-; MUBUF-NEXT:    v_writelane_b32 v1, s65, 26
-; MUBUF-NEXT:    v_writelane_b32 v1, s66, 27
-; MUBUF-NEXT:    v_writelane_b32 v1, s67, 28
-; MUBUF-NEXT:    v_writelane_b32 v1, s68, 29
-; MUBUF-NEXT:    v_writelane_b32 v1, s69, 30
-; MUBUF-NEXT:    v_writelane_b32 v1, s70, 31
-; MUBUF-NEXT:    v_writelane_b32 v1, s71, 32
-; MUBUF-NEXT:    v_writelane_b32 v1, s72, 33
-; MUBUF-NEXT:    v_writelane_b32 v1, s73, 34
-; MUBUF-NEXT:    v_writelane_b32 v1, s74, 35
-; MUBUF-NEXT:    v_writelane_b32 v1, s75, 36
-; MUBUF-NEXT:    v_writelane_b32 v1, s76, 37
-; MUBUF-NEXT:    v_writelane_b32 v1, s77, 38
-; MUBUF-NEXT:    v_writelane_b32 v1, s78, 39
-; MUBUF-NEXT:    v_writelane_b32 v1, s79, 40
-; MUBUF-NEXT:    v_writelane_b32 v1, s80, 41
-; MUBUF-NEXT:    v_writelane_b32 v1, s81, 42
-; MUBUF-NEXT:    v_writelane_b32 v1, s82, 43
-; MUBUF-NEXT:    v_writelane_b32 v1, s83, 44
-; MUBUF-NEXT:    v_writelane_b32 v1, s84, 45
-; MUBUF-NEXT:    v_writelane_b32 v1, s85, 46
-; MUBUF-NEXT:    v_writelane_b32 v1, s86, 47
-; MUBUF-NEXT:    v_writelane_b32 v1, s87, 48
-; MUBUF-NEXT:    v_writelane_b32 v1, s88, 49
-; MUBUF-NEXT:    v_writelane_b32 v1, s89, 50
-; MUBUF-NEXT:    v_writelane_b32 v1, s90, 51
-; MUBUF-NEXT:    v_writelane_b32 v1, s91, 52
-; MUBUF-NEXT:    v_writelane_b32 v1, s92, 53
-; MUBUF-NEXT:    v_writelane_b32 v1, s93, 54
-; MUBUF-NEXT:    v_writelane_b32 v1, s94, 55
-; MUBUF-NEXT:    v_writelane_b32 v1, s95, 56
-; MUBUF-NEXT:    v_writelane_b32 v1, s96, 57
-; MUBUF-NEXT:    v_writelane_b32 v1, s97, 58
-; MUBUF-NEXT:    v_writelane_b32 v1, s98, 59
-; MUBUF-NEXT:    v_writelane_b32 v1, s99, 60
-; MUBUF-NEXT:    v_writelane_b32 v1, s100, 61
+; MUBUF-NEXT:    v_writelane_b32 v1, s48, 1
+; MUBUF-NEXT:    v_writelane_b32 v1, s49, 2
+; MUBUF-NEXT:    v_writelane_b32 v1, s50, 3
+; MUBUF-NEXT:    v_writelane_b32 v1, s51, 4
+; MUBUF-NEXT:    v_writelane_b32 v1, s52, 5
+; MUBUF-NEXT:    v_writelane_b32 v1, s53, 6
+; MUBUF-NEXT:    v_writelane_b32 v1, s54, 7
+; MUBUF-NEXT:    v_writelane_b32 v1, s55, 8
+; MUBUF-NEXT:    v_writelane_b32 v1, s64, 9
+; MUBUF-NEXT:    v_writelane_b32 v1, s65, 10
+; MUBUF-NEXT:    v_writelane_b32 v1, s66, 11
+; MUBUF-NEXT:    v_writelane_b32 v1, s67, 12
+; MUBUF-NEXT:    v_writelane_b32 v1, s68, 13
+; MUBUF-NEXT:    v_writelane_b32 v1, s69, 14
+; MUBUF-NEXT:    v_writelane_b32 v1, s70, 15
+; MUBUF-NEXT:    v_writelane_b32 v1, s71, 16
+; MUBUF-NEXT:    v_writelane_b32 v1, s80, 17
+; MUBUF-NEXT:    v_writelane_b32 v1, s81, 18
+; MUBUF-NEXT:    v_writelane_b32 v1, s82, 19
+; MUBUF-NEXT:    v_writelane_b32 v1, s83, 20
+; MUBUF-NEXT:    v_writelane_b32 v1, s84, 21
+; MUBUF-NEXT:    v_writelane_b32 v1, s85, 22
+; MUBUF-NEXT:    v_writelane_b32 v1, s86, 23
+; MUBUF-NEXT:    v_writelane_b32 v1, s87, 24
+; MUBUF-NEXT:    v_writelane_b32 v1, s96, 25
+; MUBUF-NEXT:    v_writelane_b32 v1, s97, 26
+; MUBUF-NEXT:    v_writelane_b32 v1, s98, 27
+; MUBUF-NEXT:    v_writelane_b32 v1, s99, 28
+; MUBUF-NEXT:    v_writelane_b32 v1, s100, 29
 ; MUBUF-NEXT:    v_mov_b32_e32 v0, 0
 ; MUBUF-NEXT:    buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
-; MUBUF-NEXT:    v_writelane_b32 v1, s101, 62
+; MUBUF-NEXT:    v_writelane_b32 v1, s101, 30
 ; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s33 offset:4
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
 ; MUBUF-NEXT:    ;;#ASMSTART
 ; MUBUF-NEXT:    ; clobber v41
 ; MUBUF-NEXT:    ;;#ASMEND
-; MUBUF-NEXT:    v_writelane_b32 v1, s102, 63
+; MUBUF-NEXT:    v_writelane_b32 v1, s102, 31
 ; MUBUF-NEXT:    ;;#ASMSTART
 ; MUBUF-NEXT:    ;;#ASMEND
 ; MUBUF-NEXT:    buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload
 ; MUBUF-NEXT:    s_addk_i32 s32, 0x400
-; MUBUF-NEXT:    v_readlane_b32 s102, v1, 63
-; MUBUF-NEXT:    v_readlane_b32 s101, v1, 62
-; MUBUF-NEXT:    v_readlane_b32 s100, v1, 61
-; MUBUF-NEXT:    v_readlane_b32 s99, v1, 60
-; MUBUF-NEXT:    v_readlane_b32 s98, v1, 59
-; MUBUF-NEXT:    v_readlane_b32 s97, v1, 58
-; MUBUF-NEXT:    v_readlane_b32 s96, v1, 57
-; MUBUF-NEXT:    v_readlane_b32 s95, v1, 56
-; MUBUF-NEXT:    v_readlane_b32 s94, v1, 55
-; MUBUF-NEXT:    v_readlane_b32 s93, v1, 54
-; MUBUF-NEXT:    v_readlane_b32 s92, v1, 53
-; MUBUF-NEXT:    v_readlane_b32 s91, v1, 52
-; MUBUF-NEXT:    v_readlane_b32 s90, v1, 51
-; MUBUF-NEXT:    v_readlane_b32 s89, v1, 50
-; MUBUF-NEXT:    v_readlane_b32 s88, v1, 49
-; MUBUF-NEXT:    v_readlane_b32 s87, v1, 48
-; MUBUF-NEXT:    v_readlane_b32 s86, v1, 47
-; MUBUF-NEXT:    v_readlane_b32 s85, v1, 46
-; MUBUF-NEXT:    v_readlane_b32 s84, v1, 45
-; MUBUF-NEXT:    v_readlane_b32 s83, v1, 44
-; MUBUF-NEXT:    v_readlane_b32 s82, v1, 43
-; MUBUF-NEXT:    v_readlane_b32 s81, v1, 42
-; MUBUF-NEXT:    v_readlane_b32 s80, v1, 41
-; MUBUF-NEXT:    v_readlane_b32 s79, v1, 40
-; MUBUF-NEXT:    v_readlane_b32 s78, v1, 39
-; MUBUF-NEXT:    v_readlane_b32 s77, v1, 38
-; MUBUF-NEXT:    v_readlane_b32 s76, v1, 37
-; MUBUF-NEXT:    v_readlane_b32 s75, v1, 36
-; MUBUF-NEXT:    v_readlane_b32 s74, v1, 35
-; MUBUF-NEXT:    v_readlane_b32 s73, v1, 34
-; MUBUF-NEXT:    v_readlane_b32 s72, v1, 33
-; MUBUF-NEXT:    v_readlane_b32 s71, v1, 32
-; MUBUF-NEXT:    v_readlane_b32 s70, v1, 31
-; MUBUF-NEXT:    v_readlane_b32 s69, v1, 30
-; MUBUF-NEXT:    v_readlane_b32 s68, v1, 29
-; MUBUF-NEXT:    v_readlane_b32 s67, v1, 28
-; MUBUF-NEXT:    v_readlane_b32 s66, v1, 27
-; MUBUF-NEXT:    v_readlane_b32 s65, v1, 26
-; MUBUF-NEXT:    v_readlane_b32 s64, v1, 25
-; MUBUF-NEXT:    v_readlane_b32 s63, v1, 24
-; MUBUF-NEXT:    v_readlane_b32 s62, v1, 23
-; MUBUF-NEXT:    v_readlane_b32 s61, v1, 22
-; MUBUF-NEXT:    v_readlane_b32 s60, v1, 21
-; MUBUF-NEXT:    v_readlane_b32 s59, v1, 20
-; MUBUF-NEXT:    v_readlane_b32 s58, v1, 19
-; MUBUF-NEXT:    v_readlane_b32 s57, v1, 18
-; MUBUF-NEXT:    v_readlane_b32 s56, v1, 17
-; MUBUF-NEXT:    v_readlane_b32 s55, v1, 16
-; MUBUF-NEXT:    v_readlane_b32 s54, v1, 15
-; MUBUF-NEXT:    v_readlane_b32 s53, v1, 14
-; MUBUF-NEXT:    v_readlane_b32 s52, v1, 13
-; MUBUF-NEXT:    v_readlane_b32 s51, v1, 12
-; MUBUF-NEXT:    v_readlane_b32 s50, v1, 11
-; MUBUF-NEXT:    v_readlane_b32 s49, v1, 10
-; MUBUF-NEXT:    v_readlane_b32 s48, v1, 9
-; MUBUF-NEXT:    v_readlane_b32 s47, v1, 8
-; MUBUF-NEXT:    v_readlane_b32 s46, v1, 7
-; MUBUF-NEXT:    v_readlane_b32 s45, v1, 6
-; MUBUF-NEXT:    v_readlane_b32 s44, v1, 5
-; MUBUF-NEXT:    v_readlane_b32 s43, v1, 4
-; MUBUF-NEXT:    v_readlane_b32 s42, v1, 3
-; MUBUF-NEXT:    v_readlane_b32 s41, v1, 2
-; MUBUF-NEXT:    v_readlane_b32 s40, v1, 1
+; MUBUF-NEXT:    v_readlane_b32 s102, v1, 31
+; MUBUF-NEXT:    v_readlane_b32 s101, v1, 30
+; MUBUF-NEXT:    v_readlane_b32 s100, v1, 29
+; MUBUF-NEXT:    v_readlane_b32 s99, v1, 28
+; MUBUF-NEXT:    v_readlane_b32 s98, v1, 27
+; MUBUF-NEXT:    v_readlane_b32 s97, v1, 26
+; MUBUF-NEXT:    v_readlane_b32 s96, v1, 25
+; MUBUF-NEXT:    v_readlane_b32 s87, v1, 24
+; MUBUF-NEXT:    v_readlane_b32 s86, v1, 23
+; MUBUF-NEXT:    v_readlane_b32 s85, v1, 22
+; MUBUF-NEXT:    v_readlane_b32 s84, v1, 21
+; MUBUF-NEXT:    v_readlane_b32 s83, v1, 20
+; MUBUF-NEXT:    v_readlane_b32 s82, v1, 19
+; MUBUF-NEXT:    v_readlane_b32 s81, v1, 18
+; MUBUF-NEXT:    v_readlane_b32 s80, v1, 17
+; MUBUF-NEXT:    v_readlane_b32 s71, v1, 16
+; MUBUF-NEXT:    v_readlane_b32 s70, v1, 15
+; MUBUF-NEXT:    v_readlane_b32 s69, v1, 14
+; MUBUF-NEXT:    v_readlane_b32 s68, v1, 13
+; MUBUF-NEXT:    v_readlane_b32 s67, v1, 12
+; MUBUF-NEXT:    v_readlane_b32 s66, v1, 11
+; MUBUF-NEXT:    v_readlane_b32 s65, v1, 10
+; MUBUF-NEXT:    v_readlane_b32 s64, v1, 9
+; MUBUF-NEXT:    v_readlane_b32 s55, v1, 8
+; MUBUF-NEXT:    v_readlane_b32 s54, v1, 7
+; MUBUF-NEXT:    v_readlane_b32 s53, v1, 6
+; MUBUF-NEXT:    v_readlane_b32 s52, v1, 5
+; MUBUF-NEXT:    v_readlane_b32 s51, v1, 4
+; MUBUF-NEXT:    v_readlane_b32 s50, v1, 3
+; MUBUF-NEXT:    v_readlane_b32 s49, v1, 2
+; MUBUF-NEXT:    v_readlane_b32 s48, v1, 1
 ; MUBUF-NEXT:    v_readlane_b32 s39, v1, 0
 ; MUBUF-NEXT:    s_mov_b32 s32, s33
 ; MUBUF-NEXT:    s_xor_saveexec_b64 s[6:7], -1
@@ -1141,143 +823,79 @@ define void @no_new_vgpr_for_fp_csr() #1 {
 ; FLATSCR-NEXT:    scratch_store_dword off, v1, s33 offset:8 ; 4-byte Folded Spill
 ; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
 ; FLATSCR-NEXT:    v_writelane_b32 v1, s39, 0
-; FLATSCR-NEXT:    v_writelane_b32 v1, s40, 1
-; FLATSCR-NEXT:    v_writelane_b32 v1, s41, 2
-; FLATSCR-NEXT:    v_writelane_b32 v1, s42, 3
-; FLATSCR-NEXT:    v_writelane_b32 v1, s43, 4
-; FLATSCR-NEXT:    v_writelane_b32 v1, s44, 5
-; FLATSCR-NEXT:    v_writelane_b32 v1, s45, 6
-; FLATSCR-NEXT:    v_writelane_b32 v1, s46, 7
-; FLATSCR-NEXT:    v_writelane_b32 v1, s47, 8
-; FLATSCR-NEXT:    v_writelane_b32 v1, s48, 9
-; FLATSCR-NEXT:    v_writelane_b32 v1, s49, 10
-; FLATSCR-NEXT:    v_writelane_b32 v1, s50, 11
-; FLATSCR-NEXT:    v_writelane_b32 v1, s51, 12
-; FLATSCR-NEXT:    v_writelane_b32 v1, s52, 13
-; FLATSCR-NEXT:    v_writelane_b32 v1, s53, 14
-; FLATSCR-NEXT:    v_writelane_b32 v1, s54, 15
-; FLATSCR-NEXT:    v_writelane_b32 v1, s55, 16
-; FLATSCR-NEXT:    v_writelane_b32 v1, s56, 17
-; FLATSCR-NEXT:    v_writelane_b32 v1, s57, 18
-; FLATSCR-NEXT:    v_writelane_b32 v1, s58, 19
-; FLATSCR-NEXT:    v_writelane_b32 v1, s59, 20
-; FLATSCR-NEXT:    v_writelane_b32 v1, s60, 21
-; FLATSCR-NEXT:    v_writelane_b32 v1, s61, 22
-; FLATSCR-NEXT:    v_writelane_b32 v1, s62, 23
-; FLATSCR-NEXT:    v_writelane_b32 v1, s63, 24
-; FLATSCR-NEXT:    v_writelane_b32 v1, s64, 25
-; FLATSCR-NEXT:    v_writelane_b32 v1, s65, 26
-; FLATSCR-NEXT:    v_writelane_b32 v1, s66, 27
-; FLATSCR-NEXT:    v_writelane_b32 v1, s67, 28
-; FLATSCR-NEXT:    v_writelane_b32 v1, s68, 29
-; FLATSCR-NEXT:    v_writelane_b32 v1, s69, 30
-; FLATSCR-NEXT:    v_writelane_b32 v1, s70, 31
-; FLATSCR-NEXT:    v_writelane_b32 v1, s71, 32
-; FLATSCR-NEXT:    v_writelane_b32 v1, s72, 33
-; FLATSCR-NEXT:    v_writelane_b32 v1, s73, 34
-; FLATSCR-NEXT:    v_writelane_b32 v1, s74, 35
-; FLATSCR-NEXT:    v_writelane_b32 v1, s75, 36
-; FLATSCR-NEXT:    v_writelane_b32 v1, s76, 37
-; FLATSCR-NEXT:    v_writelane_b32 v1, s77, 38
-; FLATSCR-NEXT:    v_writelane_b32 v1, s78, 39
-; FLATSCR-NEXT:    v_writelane_b32 v1, s79, 40
-; FLATSCR-NEXT:    v_writelane_b32 v1, s80, 41
-; FLATSCR-NEXT:    v_writelane_b32 v1, s81, 42
-; FLATSCR-NEXT:    v_writelane_b32 v1, s82, 43
-; FLATSCR-NEXT:    v_writelane_b32 v1, s83, 44
-; FLATSCR-NEXT:    v_writelane_b32 v1, s84, 45
-; FLATSCR-NEXT:    v_writelane_b32 v1, s85, 46
-; FLATSCR-NEXT:    v_writelane_b32 v1, s86, 47
-; FLATSCR-NEXT:    v_writelane_b32 v1, s87, 48
-; FLATSCR-NEXT:    v_writelane_b32 v1, s88, 49
-; FLATSCR-NEXT:    v_writelane_b32 v1, s89, 50
-; FLATSCR-NEXT:    v_writelane_b32 v1, s90, 51
-; FLATSCR-NEXT:    v_writelane_b32 v1, s91, 52
-; FLATSCR-NEXT:    v_writelane_b32 v1, s92, 53
-; FLATSCR-NEXT:    v_writelane_b32 v1, s93, 54
-; FLATSCR-NEXT:    v_writelane_b32 v1, s94, 55
-; FLATSCR-NEXT:    v_writelane_b32 v1, s95, 56
-; FLATSCR-NEXT:    v_writelane_b32 v1, s96, 57
-; FLATSCR-NEXT:    v_writelane_b32 v1, s97, 58
-; FLATSCR-NEXT:    v_writelane_b32 v1, s98, 59
-; FLATSCR-NEXT:    v_writelane_b32 v1, s99, 60
-; FLATSCR-NEXT:    v_writelane_b32 v1, s100, 61
+; FLATSCR-NEXT:    v_writelane_b32 v1, s48, 1
+; FLATSCR-NEXT:    v_writelane_b32 v1, s49, 2
+; FLATSCR-NEXT:    v_writelane_b32 v1, s50, 3
+; FLATSCR-NEXT:    v_writelane_b32 v1, s51, 4
+; FLATSCR-NEXT:    v_writelane_b32 v1, s52, 5
+; FLATSCR-NEXT:    v_writelane_b32 v1, s53, 6
+; FLATSCR-NEXT:    v_writelane_b32 v1, s54, 7
+; FLATSCR-NEXT:    v_writelane_b32 v1, s55, 8
+; FLATSCR-NEXT:    v_writelane_b32 v1, s64, 9
+; FLATSCR-NEXT:    v_writelane_b32 v1, s65, 10
+; FLATSCR-NEXT:    v_writelane_b32 v1, s66, 11
+; FLATSCR-NEXT:    v_writelane_b32 v1, s67, 12
+; FLATSCR-NEXT:    v_writelane_b32 v1, s68, 13
+; FLATSCR-NEXT:    v_writelane_b32 v1, s69, 14
+; FLATSCR-NEXT:    v_writelane_b32 v1, s70, 15
+; FLATSCR-NEXT:    v_writelane_b32 v1, s71, 16
+; FLATSCR-NEXT:    v_writelane_b32 v1, s80, 17
+; FLATSCR-NEXT:    v_writelane_b32 v1, s81, 18
+; FLATSCR-NEXT:    v_writelane_b32 v1, s82, 19
+; FLATSCR-NEXT:    v_writelane_b32 v1, s83, 20
+; FLATSCR-NEXT:    v_writelane_b32 v1, s84, 21
+; FLATSCR-NEXT:    v_writelane_b32 v1, s85, 22
+; FLATSCR-NEXT:    v_writelane_b32 v1, s86, 23
+; FLATSCR-NEXT:    v_writelane_b32 v1, s87, 24
+; FLATSCR-NEXT:    v_writelane_b32 v1, s96, 25
+; FLATSCR-NEXT:    v_writelane_b32 v1, s97, 26
+; FLATSCR-NEXT:    v_writelane_b32 v1, s98, 27
+; FLATSCR-NEXT:    v_writelane_b32 v1, s99, 28
+; FLATSCR-NEXT:    v_writelane_b32 v1, s100, 29
 ; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
 ; FLATSCR-NEXT:    scratch_store_dword off, v41, s33 ; 4-byte Folded Spill
-; FLATSCR-NEXT:    v_writelane_b32 v1, s101, 62
+; FLATSCR-NEXT:    v_writelane_b32 v1, s101, 30
 ; FLATSCR-NEXT:    scratch_store_dword off, v0, s33 offset:4
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ; clobber v41
 ; FLATSCR-NEXT:    ;;#ASMEND
-; FLATSCR-NEXT:    v_writelane_b32 v1, s102, 63
+; FLATSCR-NEXT:    v_writelane_b32 v1, s102, 31
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ;;#ASMEND
 ; FLATSCR-NEXT:    scratch_load_dword v41, off, s33 ; 4-byte Folded Reload
 ; FLATSCR-NEXT:    s_add_i32 s32, s32, 16
-; FLATSCR-NEXT:    v_readlane_b32 s102, v1, 63
-; FLATSCR-NEXT:    v_readlane_b32 s101, v1, 62
-; FLATSCR-NEXT:    v_readlane_b32 s100, v1, 61
-; FLATSCR-NEXT:    v_readlane_b32 s99, v1, 60
-; FLATSCR-NEXT:    v_readlane_b32 s98, v1, 59
-; FLATSCR-NEXT:    v_readlane_b32 s97, v1, 58
-; FLATSCR-NEXT:    v_readlane_b32 s96, v1, 57
-; FLATSCR-NEXT:    v_readlane_b32 s95, v1, 56
-; FLATSCR-NEXT:    v_readlane_b32 s94, v1, 55
-; FLATSCR-NEXT:    v_readlane_b32 s93, v1, 54
-; FLATSCR-NEXT:    v_readlane_b32 s92, v1, 53
-; FLATSCR-NEXT:    v_readlane_b32 s91, v1, 52
-; FLATSCR-NEXT:    v_readlane_b32 s90, v1, 51
-; FLATSCR-NEXT:    v_readlane_b32 s89, v1, 50
-; FLATSCR-NEXT:    v_readlane_b32 s88, v1, 49
-; FLATSCR-NEXT:    v_readlane_b32 s87, v1, 48
-; FLATSCR-NEXT:    v_readlane_b32 s86, v1, 47
-; FLATSCR-NEXT:    v_readlane_b32 s85, v1, 46
-; FLATSCR-NEXT:    v_readlane_b32 s84, v1, 45
-; FLATSCR-NEXT:    v_readlane_b32 s83, v1, 44
-; FLATSCR-NEXT:    v_readlane_b32 s82, v1, 43
-; FLATSCR-NEXT:    v_readlane_b32 s81, v1, 42
-; FLATSCR-NEXT:    v_readlane_b32 s80, v1, 41
-; FLATSCR-NEXT:    v_readlane_b32 s79, v1, 40
-; FLATSCR-NEXT:    v_readlane_b32 s78, v1, 39
-; FLATSCR-NEXT:    v_readlane_b32 s77, v1, 38
-; FLATSCR-NEXT:    v_readlane_b32 s76, v1, 37
-; FLATSCR-NEXT:    v_readlane_b32 s75, v1, 36
-; FLATSCR-NEXT:    v_readlane_b32 s74, v1, 35
-; FLATSCR-NEXT:    v_readlane_b32 s73, v1, 34
-; FLATSCR-NEXT:    v_readlane_b32 s72, v1, 33
-; FLATSCR-NEXT:    v_readlane_b32 s71, v1, 32
-; FLATSCR-NEXT:    v_readlane_b32 s70, v1, 31
-; FLATSCR-NEXT:    v_readlane_b32 s69, v1, 30
-; FLATSCR-NEXT:    v_readlane_b32 s68, v1, 29
-; FLATSCR-NEXT:    v_readlane_b32 s67, v1, 28
-; FLATSCR-NEXT:    v_readlane_b32 s66, v1, 27
-; FLATSCR-NEXT:    v_readlane_b32 s65, v1, 26
-; FLATSCR-NEXT:    v_readlane_b32 s64, v1, 25
-; FLATSCR-NEXT:    v_readlane_b32 s63, v1, 24
-; FLATSCR-NEXT:    v_readlane_b32 s62, v1, 23
-; FLATSCR-NEXT:    v_readlane_b32 s61, v1, 22
-; FLATSCR-NEXT:    v_readlane_b32 s60, v1, 21
-; FLATSCR-NEXT:    v_readlane_b32 s59, v1, 20
-; FLATSCR-NEXT:    v_readlane_b32 s58, v1, 19
-; FLATSCR-NEXT:    v_readlane_b32 s57, v1, 18
-; FLATSCR-NEXT:    v_readlane_b32 s56, v1, 17
-; FLATSCR-NEXT:    v_readlane_b32 s55, v1, 16
-; FLATSCR-NEXT:    v_readlane_b32 s54, v1, 15
-; FLATSCR-NEXT:    v_readlane_b32 s53, v1, 14
-; FLATSCR-NEXT:    v_readlane_b32 s52, v1, 13
-; FLATSCR-NEXT:    v_readlane_b32 s51, v1, 12
-; FLATSCR-NEXT:    v_readlane_b32 s50, v1, 11
-; FLATSCR-NEXT:    v_readlane_b32 s49, v1, 10
-; FLATSCR-NEXT:    v_readlane_b32 s48, v1, 9
-; FLATSCR-NEXT:    v_readlane_b32 s47, v1, 8
-; FLATSCR-NEXT:    v_readlane_b32 s46, v1, 7
-; FLATSCR-NEXT:    v_readlane_b32 s45, v1, 6
-; FLATSCR-NEXT:    v_readlane_b32 s44, v1, 5
-; FLATSCR-NEXT:    v_readlane_b32 s43, v1, 4
-; FLATSCR-NEXT:    v_readlane_b32 s42, v1, 3
-; FLATSCR-NEXT:    v_readlane_b32 s41, v1, 2
-; FLATSCR-NEXT:    v_readlane_b32 s40, v1, 1
+; FLATSCR-NEXT:    v_readlane_b32 s102, v1, 31
+; FLATSCR-NEXT:    v_readlane_b32 s101, v1, 30
+; FLATSCR-NEXT:    v_readlane_b32 s100, v1, 29
+; FLATSCR-NEXT:    v_readlane_b32 s99, v1, 28
+; FLATSCR-NEXT:    v_readlane_b32 s98, v1, 27
+; FLATSCR-NEXT:    v_readlane_b32 s97, v1, 26
+; FLATSCR-NEXT:    v_readlane_b32 s96, v1, 25
+; FLATSCR-NEXT:    v_readlane_b32 s87, v1, 24
+; FLATSCR-NEXT:    v_readlane_b32 s86, v1, 23
+; FLATSCR-NEXT:    v_readlane_b32 s85, v1, 22
+; FLATSCR-NEXT:    v_readlane_b32 s84, v1, 21
+; FLATSCR-NEXT:    v_readlane_b32 s83, v1, 20
+; FLATSCR-NEXT:    v_readlane_b32 s82, v1, 19
+; FLATSCR-NEXT:    v_readlane_b32 s81, v1, 18
+; FLATSCR-NEXT:    v_readlane_b32 s80, v1, 17
+; FLATSCR-NEXT:    v_readlane_b32 s71, v1, 16
+; FLATSCR-NEXT:    v_readlane_b32 s70, v1, 15
+; FLATSCR-NEXT:    v_readlane_b32 s69, v1, 14
+; FLATSCR-NEXT:    v_readlane_b32 s68, v1, 13
+; FLATSCR-NEXT:    v_readlane_b32 s67, v1, 12
+; FLATSCR-NEXT:    v_readlane_b32 s66, v1, 11
+; FLATSCR-NEXT:    v_readlane_b32 s65, v1, 10
+; FLATSCR-NEXT:    v_readlane_b32 s64, v1, 9
+; FLATSCR-NEXT:    v_readlane_b32 s55, v1, 8
+; FLATSCR-NEXT:    v_readlane_b32 s54, v1, 7
+; FLATSCR-NEXT:    v_readlane_b32 s53, v1, 6
+; FLATSCR-NEXT:    v_readlane_b32 s52, v1, 5
+; FLATSCR-NEXT:    v_readlane_b32 s51, v1, 4
+; FLATSCR-NEXT:    v_readlane_b32 s50, v1, 3
+; FLATSCR-NEXT:    v_readlane_b32 s49, v1, 2
+; FLATSCR-NEXT:    v_readlane_b32 s48, v1, 1
 ; FLATSCR-NEXT:    v_readlane_b32 s39, v1, 0
 ; FLATSCR-NEXT:    s_mov_b32 s32, s33
 ; FLATSCR-NEXT:    s_xor_saveexec_b64 s[2:3], -1
@@ -1346,7 +964,7 @@ define void @no_unused_non_csr_sgpr_for_fp() #1 {
 ; MUBUF-LABEL: no_unused_non_csr_sgpr_for_fp:
 ; MUBUF:       ; %bb.0:
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; MUBUF-NEXT:    s_mov_b32 vcc_lo, s33
+; MUBUF-NEXT:    s_mov_b32 s40, s33
 ; MUBUF-NEXT:    s_mov_b32 s33, s32
 ; MUBUF-NEXT:    s_xor_saveexec_b64 s[4:5], -1
 ; MUBUF-NEXT:    buffer_store_dword v1, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
@@ -1365,14 +983,14 @@ define void @no_unused_non_csr_sgpr_for_fp() #1 {
 ; MUBUF-NEXT:    s_xor_saveexec_b64 s[4:5], -1
 ; MUBUF-NEXT:    buffer_load_dword v1, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
 ; MUBUF-NEXT:    s_mov_b64 exec, s[4:5]
-; MUBUF-NEXT:    s_mov_b32 s33, vcc_lo
+; MUBUF-NEXT:    s_mov_b32 s33, s40
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
 ; MUBUF-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; FLATSCR-LABEL: no_unused_non_csr_sgpr_for_fp:
 ; FLATSCR:       ; %bb.0:
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; FLATSCR-NEXT:    s_mov_b32 vcc_lo, s33
+; FLATSCR-NEXT:    s_mov_b32 s40, s33
 ; FLATSCR-NEXT:    s_mov_b32 s33, s32
 ; FLATSCR-NEXT:    s_xor_saveexec_b64 s[0:1], -1
 ; FLATSCR-NEXT:    scratch_store_dword off, v1, s33 offset:4 ; 4-byte Folded Spill
@@ -1391,7 +1009,7 @@ define void @no_unused_non_csr_sgpr_for_fp() #1 {
 ; FLATSCR-NEXT:    s_xor_saveexec_b64 s[0:1], -1
 ; FLATSCR-NEXT:    scratch_load_dword v1, off, s33 offset:4 ; 4-byte Folded Reload
 ; FLATSCR-NEXT:    s_mov_b64 exec, s[0:1]
-; FLATSCR-NEXT:    s_mov_b32 s33, vcc_lo
+; FLATSCR-NEXT:    s_mov_b32 s33, s40
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca i32, addrspace(5)
@@ -1412,7 +1030,7 @@ define void @no_unused_non_csr_sgpr_for_fp_no_scratch_vgpr() #1 {
 ; MUBUF-LABEL: no_unused_non_csr_sgpr_for_fp_no_scratch_vgpr:
 ; MUBUF:       ; %bb.0:
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; MUBUF-NEXT:    s_mov_b32 vcc_lo, s33
+; MUBUF-NEXT:    s_mov_b32 s40, s33
 ; MUBUF-NEXT:    s_mov_b32 s33, s32
 ; MUBUF-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; MUBUF-NEXT:    buffer_store_dword v40, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
@@ -1434,14 +1052,14 @@ define void @no_unused_non_csr_sgpr_for_fp_no_scratch_vgpr() #1 {
 ; MUBUF-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; MUBUF-NEXT:    buffer_load_dword v40, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
 ; MUBUF-NEXT:    s_mov_b64 exec, s[4:5]
-; MUBUF-NEXT:    s_mov_b32 s33, vcc_lo
+; MUBUF-NEXT:    s_mov_b32 s33, s40
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
 ; MUBUF-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; FLATSCR-LABEL: no_unused_non_csr_sgpr_for_fp_no_scratch_vgpr:
 ; FLATSCR:       ; %bb.0:
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; FLATSCR-NEXT:    s_mov_b32 vcc_lo, s33
+; FLATSCR-NEXT:    s_mov_b32 s40, s33
 ; FLATSCR-NEXT:    s_mov_b32 s33, s32
 ; FLATSCR-NEXT:    s_or_saveexec_b64 s[0:1], -1
 ; FLATSCR-NEXT:    scratch_store_dword off, v40, s33 offset:4 ; 4-byte Folded Spill
@@ -1463,7 +1081,7 @@ define void @no_unused_non_csr_sgpr_for_fp_no_scratch_vgpr() #1 {
 ; FLATSCR-NEXT:    s_or_saveexec_b64 s[0:1], -1
 ; FLATSCR-NEXT:    scratch_load_dword v40, off, s33 offset:4 ; 4-byte Folded Reload
 ; FLATSCR-NEXT:    s_mov_b64 exec, s[0:1]
-; FLATSCR-NEXT:    s_mov_b32 s33, vcc_lo
+; FLATSCR-NEXT:    s_mov_b32 s33, s40
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca i32, addrspace(5)
@@ -1491,7 +1109,7 @@ define void @scratch_reg_needed_mubuf_offset(ptr addrspace(5) byval([4096 x i8])
 ; MUBUF-LABEL: scratch_reg_needed_mubuf_offset:
 ; MUBUF:       ; %bb.0:
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; MUBUF-NEXT:    s_mov_b32 vcc_lo, s33
+; MUBUF-NEXT:    s_mov_b32 s40, s33
 ; MUBUF-NEXT:    s_mov_b32 s33, s32
 ; MUBUF-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; MUBUF-NEXT:    s_add_i32 s6, s33, 0x40100
@@ -1517,14 +1135,14 @@ define void @scratch_reg_needed_mubuf_offset(ptr addrspace(5) byval([4096 x i8])
 ; MUBUF-NEXT:    s_add_i32 s6, s33, 0x40100
 ; MUBUF-NEXT:    buffer_load_dword v40, off, s[0:3], s6 ; 4-byte Folded Reload
 ; MUBUF-NEXT:    s_mov_b64 exec, s[4:5]
-; MUBUF-NEXT:    s_mov_b32 s33, vcc_lo
+; MUBUF-NEXT:    s_mov_b32 s33, s40
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
 ; MUBUF-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; FLATSCR-LABEL: scratch_reg_needed_mubuf_offset:
 ; FLATSCR:       ; %bb.0:
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; FLATSCR-NEXT:    s_mov_b32 vcc_lo, s33
+; FLATSCR-NEXT:    s_mov_b32 s40, s33
 ; FLATSCR-NEXT:    s_mov_b32 s33, s32
 ; FLATSCR-NEXT:    s_or_saveexec_b64 s[0:1], -1
 ; FLATSCR-NEXT:    s_add_i32 s2, s33, 0x1004
@@ -1550,7 +1168,7 @@ define void @scratch_reg_needed_mubuf_offset(ptr addrspace(5) byval([4096 x i8])
 ; FLATSCR-NEXT:    s_add_i32 s2, s33, 0x1004
 ; FLATSCR-NEXT:    scratch_load_dword v40, off, s2 ; 4-byte Folded Reload
 ; FLATSCR-NEXT:    s_mov_b64 exec, s[0:1]
-; FLATSCR-NEXT:    s_mov_b32 s33, vcc_lo
+; FLATSCR-NEXT:    s_mov_b32 s33, s40
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca i32, addrspace(5)
@@ -1650,22 +1268,15 @@ define void @callee_need_to_spill_fp_to_memory() #3 {
 ; MUBUF-LABEL: callee_need_to_spill_fp_to_memory:
 ; MUBUF:       ; %bb.0:
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; MUBUF-NEXT:    s_mov_b32 s4, s33
+; MUBUF-NEXT:    s_mov_b32 s40, s33
 ; MUBUF-NEXT:    s_mov_b32 s33, s32
-; MUBUF-NEXT:    v_mov_b32_e32 v0, s4
-; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s33 ; 4-byte Folded Spill
 ; MUBUF-NEXT:    ;;#ASMSTART
 ; MUBUF-NEXT:    ; clobber nonpreserved SGPRs
 ; MUBUF-NEXT:    ;;#ASMEND
 ; MUBUF-NEXT:    ;;#ASMSTART
 ; MUBUF-NEXT:    ; clobber all VGPRs
 ; MUBUF-NEXT:    ;;#ASMEND
-; MUBUF-NEXT:    buffer_load_dword v0, off, s[0:3], s33 ; 4-byte Folded Reload
-; MUBUF-NEXT:    s_addk_i32 s32, 0x200
-; MUBUF-NEXT:    s_mov_b32 s32, s33
-; MUBUF-NEXT:    s_waitcnt vmcnt(0)
-; MUBUF-NEXT:    v_readfirstlane_b32 s4, v0
-; MUBUF-NEXT:    s_mov_b32 s33, s4
+; MUBUF-NEXT:    s_mov_b32 s33, s40
 ; MUBUF-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; FLATSCR-LABEL: callee_need_to_spill_fp_to_memory:
@@ -1707,147 +1318,80 @@ define void @callee_need_to_spill_fp_to_memory_full_reserved_vgpr() #3 {
 ; MUBUF-NEXT:    s_xor_saveexec_b64 s[6:7], -1
 ; MUBUF-NEXT:    buffer_store_dword v39, off, s[0:3], s33 ; 4-byte Folded Spill
 ; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
+; MUBUF-NEXT:    v_writelane_b32 v39, s4, 32
 ; MUBUF-NEXT:    v_writelane_b32 v39, s39, 0
-; MUBUF-NEXT:    v_writelane_b32 v39, s40, 1
-; MUBUF-NEXT:    v_writelane_b32 v39, s41, 2
-; MUBUF-NEXT:    v_writelane_b32 v39, s42, 3
-; MUBUF-NEXT:    v_writelane_b32 v39, s43, 4
-; MUBUF-NEXT:    v_writelane_b32 v39, s44, 5
-; MUBUF-NEXT:    v_writelane_b32 v39, s45, 6
-; MUBUF-NEXT:    v_writelane_b32 v39, s46, 7
-; MUBUF-NEXT:    v_writelane_b32 v39, s47, 8
-; MUBUF-NEXT:    v_writelane_b32 v39, s48, 9
-; MUBUF-NEXT:    v_writelane_b32 v39, s49, 10
-; MUBUF-NEXT:    v_writelane_b32 v39, s50, 11
-; MUBUF-NEXT:    v_writelane_b32 v39, s51, 12
-; MUBUF-NEXT:    v_writelane_b32 v39, s52, 13
-; MUBUF-NEXT:    v_writelane_b32 v39, s53, 14
-; MUBUF-NEXT:    v_writelane_b32 v39, s54, 15
-; MUBUF-NEXT:    v_writelane_b32 v39, s55, 16
-; MUBUF-NEXT:    v_writelane_b32 v39, s56, 17
-; MUBUF-NEXT:    v_writelane_b32 v39, s57, 18
-; MUBUF-NEXT:    v_writelane_b32 v39, s58, 19
-; MUBUF-NEXT:    v_writelane_b32 v39, s59, 20
-; MUBUF-NEXT:    v_writelane_b32 v39, s60, 21
-; MUBUF-NEXT:    v_writelane_b32 v39, s61, 22
-; MUBUF-NEXT:    v_writelane_b32 v39, s62, 23
-; MUBUF-NEXT:    v_writelane_b32 v39, s63, 24
-; MUBUF-NEXT:    v_writelane_b32 v39, s64, 25
-; MUBUF-NEXT:    v_writelane_b32 v39, s65, 26
-; MUBUF-NEXT:    v_writelane_b32 v39, s66, 27
-; MUBUF-NEXT:    v_writelane_b32 v39, s67, 28
-; MUBUF-NEXT:    v_writelane_b32 v39, s68, 29
-; MUBUF-NEXT:    v_writelane_b32 v39, s69, 30
-; MUBUF-NEXT:    v_writelane_b32 v39, s70, 31
-; MUBUF-NEXT:    v_writelane_b32 v39, s71, 32
-; MUBUF-NEXT:    v_writelane_b32 v39, s72, 33
-; MUBUF-NEXT:    v_writelane_b32 v39, s73, 34
-; MUBUF-NEXT:    v_writelane_b32 v39, s74, 35
-; MUBUF-NEXT:    v_writelane_b32 v39, s75, 36
-; MUBUF-NEXT:    v_writelane_b32 v39, s76, 37
-; MUBUF-NEXT:    v_writelane_b32 v39, s77, 38
-; MUBUF-NEXT:    v_writelane_b32 v39, s78, 39
-; MUBUF-NEXT:    v_writelane_b32 v39, s79, 40
-; MUBUF-NEXT:    v_writelane_b32 v39, s80, 41
-; MUBUF-NEXT:    v_writelane_b32 v39, s81, 42
-; MUBUF-NEXT:    v_writelane_b32 v39, s82, 43
-; MUBUF-NEXT:    v_writelane_b32 v39, s83, 44
-; MUBUF-NEXT:    v_writelane_b32 v39, s84, 45
-; MUBUF-NEXT:    v_writelane_b32 v39, s85, 46
-; MUBUF-NEXT:    v_writelane_b32 v39, s86, 47
-; MUBUF-NEXT:    v_writelane_b32 v39, s87, 48
-; MUBUF-NEXT:    v_writelane_b32 v39, s88, 49
-; MUBUF-NEXT:    v_writelane_b32 v39, s89, 50
-; MUBUF-NEXT:    v_writelane_b32 v39, s90, 51
-; MUBUF-NEXT:    v_writelane_b32 v39, s91, 52
-; MUBUF-NEXT:    v_writelane_b32 v39, s92, 53
-; MUBUF-NEXT:    v_writelane_b32 v39, s93, 54
-; MUBUF-NEXT:    v_writelane_b32 v39, s94, 55
-; MUBUF-NEXT:    v_writelane_b32 v39, s95, 56
-; MUBUF-NEXT:    v_writelane_b32 v39, s96, 57
-; MUBUF-NEXT:    v_writelane_b32 v39, s97, 58
-; MUBUF-NEXT:    v_writelane_b32 v39, s98, 59
-; MUBUF-NEXT:    v_writelane_b32 v39, s99, 60
-; MUBUF-NEXT:    v_writelane_b32 v39, s100, 61
-; MUBUF-NEXT:    v_mov_b32_e32 v0, s4
-; MUBUF-NEXT:    v_writelane_b32 v39, s101, 62
-; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
-; MUBUF-NEXT:    v_writelane_b32 v39, s102, 63
+; MUBUF-NEXT:    v_writelane_b32 v39, s48, 1
+; MUBUF-NEXT:    v_writelane_b32 v39, s49, 2
+; MUBUF-NEXT:    v_writelane_b32 v39, s50, 3
+; MUBUF-NEXT:    v_writelane_b32 v39, s51, 4
+; MUBUF-NEXT:    v_writelane_b32 v39, s52, 5
+; MUBUF-NEXT:    v_writelane_b32 v39, s53, 6
+; MUBUF-NEXT:    v_writelane_b32 v39, s54, 7
+; MUBUF-NEXT:    v_writelane_b32 v39, s55, 8
+; MUBUF-NEXT:    v_writelane_b32 v39, s64, 9
+; MUBUF-NEXT:    v_writelane_b32 v39, s65, 10
+; MUBUF-NEXT:    v_writelane_b32 v39, s66, 11
+; MUBUF-NEXT:    v_writelane_b32 v39, s67, 12
+; MUBUF-NEXT:    v_writelane_b32 v39, s68, 13
+; MUBUF-NEXT:    v_writelane_b32 v39, s69, 14
+; MUBUF-NEXT:    v_writelane_b32 v39, s70, 15
+; MUBUF-NEXT:    v_writelane_b32 v39, s71, 16
+; MUBUF-NEXT:    v_writelane_b32 v39, s80, 17
+; MUBUF-NEXT:    v_writelane_b32 v39, s81, 18
+; MUBUF-NEXT:    v_writelane_b32 v39, s82, 19
+; MUBUF-NEXT:    v_writelane_b32 v39, s83, 20
+; MUBUF-NEXT:    v_writelane_b32 v39, s84, 21
+; MUBUF-NEXT:    v_writelane_b32 v39, s85, 22
+; MUBUF-NEXT:    v_writelane_b32 v39, s86, 23
+; MUBUF-NEXT:    v_writelane_b32 v39, s87, 24
+; MUBUF-NEXT:    v_writelane_b32 v39, s96, 25
+; MUBUF-NEXT:    v_writelane_b32 v39, s97, 26
+; MUBUF-NEXT:    v_writelane_b32 v39, s98, 27
+; MUBUF-NEXT:    v_writelane_b32 v39, s99, 28
+; MUBUF-NEXT:    v_writelane_b32 v39, s100, 29
+; MUBUF-NEXT:    v_writelane_b32 v39, s101, 30
+; MUBUF-NEXT:    s_addk_i32 s32, 0x200
+; MUBUF-NEXT:    v_writelane_b32 v39, s102, 31
 ; MUBUF-NEXT:    ;;#ASMSTART
 ; MUBUF-NEXT:    ; clobber nonpreserved SGPRs and 64 CSRs
 ; MUBUF-NEXT:    ;;#ASMEND
 ; MUBUF-NEXT:    ;;#ASMSTART
 ; MUBUF-NEXT:    ; clobber all VGPRs except CSR v40
 ; MUBUF-NEXT:    ;;#ASMEND
-; MUBUF-NEXT:    buffer_load_dword v0, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
-; MUBUF-NEXT:    s_addk_i32 s32, 0x300
-; MUBUF-NEXT:    v_readlane_b32 s102, v39, 63
-; MUBUF-NEXT:    v_readlane_b32 s101, v39, 62
-; MUBUF-NEXT:    v_readlane_b32 s100, v39, 61
-; MUBUF-NEXT:    v_readlane_b32 s99, v39, 60
-; MUBUF-NEXT:    v_readlane_b32 s98, v39, 59
-; MUBUF-NEXT:    v_readlane_b32 s97, v39, 58
-; MUBUF-NEXT:    v_readlane_b32 s96, v39, 57
-; MUBUF-NEXT:    v_readlane_b32 s95, v39, 56
-; MUBUF-NEXT:    v_readlane_b32 s94, v39, 55
-; MUBUF-NEXT:    v_readlane_b32 s93, v39, 54
-; MUBUF-NEXT:    v_readlane_b32 s92, v39, 53
-; MUBUF-NEXT:    v_readlane_b32 s91, v39, 52
-; MUBUF-NEXT:    v_readlane_b32 s90, v39, 51
-; MUBUF-NEXT:    v_readlane_b32 s89, v39, 50
-; MUBUF-NEXT:    v_readlane_b32 s88, v39, 49
-; MUBUF-NEXT:    v_readlane_b32 s87, v39, 48
-; MUBUF-NEXT:    v_readlane_b32 s86, v39, 47
-; MUBUF-NEXT:    v_readlane_b32 s85, v39, 46
-; MUBUF-NEXT:    v_readlane_b32 s84, v39, 45
-; MUBUF-NEXT:    v_readlane_b32 s83, v39, 44
-; MUBUF-NEXT:    v_readlane_b32 s82, v39, 43
-; MUBUF-NEXT:    v_readlane_b32 s81, v39, 42
-; MUBUF-NEXT:    v_readlane_b32 s80, v39, 41
-; MUBUF-NEXT:    v_readlane_b32 s79, v39, 40
-; MUBUF-NEXT:    v_readlane_b32 s78, v39, 39
-; MUBUF-NEXT:    v_readlane_b32 s77, v39, 38
-; MUBUF-NEXT:    v_readlane_b32 s76, v39, 37
-; MUBUF-NEXT:    v_readlane_b32 s75, v39, 36
-; MUBUF-NEXT:    v_readlane_b32 s74, v39, 35
-; MUBUF-NEXT:    v_readlane_b32 s73, v39, 34
-; MUBUF-NEXT:    v_readlane_b32 s72, v39, 33
-; MUBUF-NEXT:    v_readlane_b32 s71, v39, 32
-; MUBUF-NEXT:    v_readlane_b32 s70, v39, 31
-; MUBUF-NEXT:    v_readlane_b32 s69, v39, 30
-; MUBUF-NEXT:    v_readlane_b32 s68, v39, 29
-; MUBUF-NEXT:    v_readlane_b32 s67, v39, 28
-; MUBUF-NEXT:    v_readlane_b32 s66, v39, 27
-; MUBUF-NEXT:    v_readlane_b32 s65, v39, 26
-; MUBUF-NEXT:    v_readlane_b32 s64, v39, 25
-; MUBUF-NEXT:    v_readlane_b32 s63, v39, 24
-; MUBUF-NEXT:    v_readlane_b32 s62, v39, 23
-; MUBUF-NEXT:    v_readlane_b32 s61, v39, 22
-; MUBUF-NEXT:    v_readlane_b32 s60, v39, 21
-; MUBUF-NEXT:    v_readlane_b32 s59, v39, 20
-; MUBUF-NEXT:    v_readlane_b32 s58, v39, 19
-; MUBUF-NEXT:    v_readlane_b32 s57, v39, 18
-; MUBUF-NEXT:    v_readlane_b32 s56, v39, 17
-; MUBUF-NEXT:    v_readlane_b32 s55, v39, 16
-; MUBUF-NEXT:    v_readlane_b32 s54, v39, 15
-; MUBUF-NEXT:    v_readlane_b32 s53, v39, 14
-; MUBUF-NEXT:    v_readlane_b32 s52, v39, 13
-; MUBUF-NEXT:    v_readlane_b32 s51, v39, 12
-; MUBUF-NEXT:    v_readlane_b32 s50, v39, 11
-; MUBUF-NEXT:    v_readlane_b32 s49, v39, 10
-; MUBUF-NEXT:    v_readlane_b32 s48, v39, 9
-; MUBUF-NEXT:    v_readlane_b32 s47, v39, 8
-; MUBUF-NEXT:    v_readlane_b32 s46, v39, 7
-; MUBUF-NEXT:    v_readlane_b32 s45, v39, 6
-; MUBUF-NEXT:    v_readlane_b32 s44, v39, 5
-; MUBUF-NEXT:    v_readlane_b32 s43, v39, 4
-; MUBUF-NEXT:    v_readlane_b32 s42, v39, 3
-; MUBUF-NEXT:    v_readlane_b32 s41, v39, 2
-; MUBUF-NEXT:    v_readlane_b32 s40, v39, 1
+; MUBUF-NEXT:    v_readlane_b32 s102, v39, 31
+; MUBUF-NEXT:    v_readlane_b32 s101, v39, 30
+; MUBUF-NEXT:    v_readlane_b32 s100, v39, 29
+; MUBUF-NEXT:    v_readlane_b32 s99, v39, 28
+; MUBUF-NEXT:    v_readlane_b32 s98, v39, 27
+; MUBUF-NEXT:    v_readlane_b32 s97, v39, 26
+; MUBUF-NEXT:    v_readlane_b32 s96, v39, 25
+; MUBUF-NEXT:    v_readlane_b32 s87, v39, 24
+; MUBUF-NEXT:    v_readlane_b32 s86, v39, 23
+; MUBUF-NEXT:    v_readlane_b32 s85, v39, 22
+; MUBUF-NEXT:    v_readlane_b32 s84, v39, 21
+; MUBUF-NEXT:    v_readlane_b32 s83, v39, 20
+; MUBUF-NEXT:    v_readlane_b32 s82, v39, 19
+; MUBUF-NEXT:    v_readlane_b32 s81, v39, 18
+; MUBUF-NEXT:    v_readlane_b32 s80, v39, 17
+; MUBUF-NEXT:    v_readlane_b32 s71, v39, 16
+; MUBUF-NEXT:    v_readlane_b32 s70, v39, 15
+; MUBUF-NEXT:    v_readlane_b32 s69, v39, 14
+; MUBUF-NEXT:    v_readlane_b32 s68, v39, 13
+; MUBUF-NEXT:    v_readlane_b32 s67, v39, 12
+; MUBUF-NEXT:    v_readlane_b32 s66, v39, 11
+; MUBUF-NEXT:    v_readlane_b32 s65, v39, 10
+; MUBUF-NEXT:    v_readlane_b32 s64, v39, 9
+; MUBUF-NEXT:    v_readlane_b32 s55, v39, 8
+; MUBUF-NEXT:    v_readlane_b32 s54, v39, 7
+; MUBUF-NEXT:    v_readlane_b32 s53, v39, 6
+; MUBUF-NEXT:    v_readlane_b32 s52, v39, 5
+; MUBUF-NEXT:    v_readlane_b32 s51, v39, 4
+; MUBUF-NEXT:    v_readlane_b32 s50, v39, 3
+; MUBUF-NEXT:    v_readlane_b32 s49, v39, 2
+; MUBUF-NEXT:    v_readlane_b32 s48, v39, 1
 ; MUBUF-NEXT:    v_readlane_b32 s39, v39, 0
 ; MUBUF-NEXT:    s_mov_b32 s32, s33
-; MUBUF-NEXT:    s_waitcnt vmcnt(0)
-; MUBUF-NEXT:    v_readfirstlane_b32 s4, v0
+; MUBUF-NEXT:    v_readlane_b32 s4, v39, 32
 ; MUBUF-NEXT:    s_xor_saveexec_b64 s[6:7], -1
 ; MUBUF-NEXT:    buffer_load_dword v39, off, s[0:3], s33 ; 4-byte Folded Reload
 ; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
@@ -1864,139 +1408,75 @@ define void @callee_need_to_spill_fp_to_memory_full_reserved_vgpr() #3 {
 ; FLATSCR-NEXT:    scratch_store_dword off, v39, s33 ; 4-byte Folded Spill
 ; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
 ; FLATSCR-NEXT:    v_writelane_b32 v39, s39, 0
-; FLATSCR-NEXT:    v_writelane_b32 v39, s40, 1
-; FLATSCR-NEXT:    v_writelane_b32 v39, s41, 2
-; FLATSCR-NEXT:    v_writelane_b32 v39, s42, 3
-; FLATSCR-NEXT:    v_writelane_b32 v39, s43, 4
-; FLATSCR-NEXT:    v_writelane_b32 v39, s44, 5
-; FLATSCR-NEXT:    v_writelane_b32 v39, s45, 6
-; FLATSCR-NEXT:    v_writelane_b32 v39, s46, 7
-; FLATSCR-NEXT:    v_writelane_b32 v39, s47, 8
-; FLATSCR-NEXT:    v_writelane_b32 v39, s48, 9
-; FLATSCR-NEXT:    v_writelane_b32 v39, s49, 10
-; FLATSCR-NEXT:    v_writelane_b32 v39, s50, 11
-; FLATSCR-NEXT:    v_writelane_b32 v39, s51, 12
-; FLATSCR-NEXT:    v_writelane_b32 v39, s52, 13
-; FLATSCR-NEXT:    v_writelane_b32 v39, s53, 14
-; FLATSCR-NEXT:    v_writelane_b32 v39, s54, 15
-; FLATSCR-NEXT:    v_writelane_b32 v39, s55, 16
-; FLATSCR-NEXT:    v_writelane_b32 v39, s56, 17
-; FLATSCR-NEXT:    v_writelane_b32 v39, s57, 18
-; FLATSCR-NEXT:    v_writelane_b32 v39, s58, 19
-; FLATSCR-NEXT:    v_writelane_b32 v39, s59, 20
-; FLATSCR-NEXT:    v_writelane_b32 v39, s60, 21
-; FLATSCR-NEXT:    v_writelane_b32 v39, s61, 22
-; FLATSCR-NEXT:    v_writelane_b32 v39, s62, 23
-; FLATSCR-NEXT:    v_writelane_b32 v39, s63, 24
-; FLATSCR-NEXT:    v_writelane_b32 v39, s64, 25
-; FLATSCR-NEXT:    v_writelane_b32 v39, s65, 26
-; FLATSCR-NEXT:    v_writelane_b32 v39, s66, 27
-; FLATSCR-NEXT:    v_writelane_b32 v39, s67, 28
-; FLATSCR-NEXT:    v_writelane_b32 v39, s68, 29
-; FLATSCR-NEXT:    v_writelane_b32 v39, s69, 30
-; FLATSCR-NEXT:    v_writelane_b32 v39, s70, 31
-; FLATSCR-NEXT:    v_writelane_b32 v39, s71, 32
-; FLATSCR-NEXT:    v_writelane_b32 v39, s72, 33
-; FLATSCR-NEXT:    v_writelane_b32 v39, s73, 34
-; FLATSCR-NEXT:    v_writelane_b32 v39, s74, 35
-; FLATSCR-NEXT:    v_writelane_b32 v39, s75, 36
-; FLATSCR-NEXT:    v_writelane_b32 v39, s76, 37
-; FLATSCR-NEXT:    v_writelane_b32 v39, s77, 38
-; FLATSCR-NEXT:    v_writelane_b32 v39, s78, 39
-; FLATSCR-NEXT:    v_writelane_b32 v39, s79, 40
-; FLATSCR-NEXT:    v_writelane_b32 v39, s80, 41
-; FLATSCR-NEXT:    v_writelane_b32 v39, s81, 42
-; FLATSCR-NEXT:    v_writelane_b32 v39, s82, 43
-; FLATSCR-NEXT:    v_writelane_b32 v39, s83, 44
-; FLATSCR-NEXT:    v_writelane_b32 v39, s84, 45
-; FLATSCR-NEXT:    v_writelane_b32 v39, s85, 46
-; FLATSCR-NEXT:    v_writelane_b32 v39, s86, 47
-; FLATSCR-NEXT:    v_writelane_b32 v39, s87, 48
-; FLATSCR-NEXT:    v_writelane_b32 v39, s88, 49
-; FLATSCR-NEXT:    v_writelane_b32 v39, s89, 50
-; FLATSCR-NEXT:    v_writelane_b32 v39, s90, 51
-; FLATSCR-NEXT:    v_writelane_b32 v39, s91, 52
-; FLATSCR-NEXT:    v_writelane_b32 v39, s92, 53
-; FLATSCR-NEXT:    v_writelane_b32 v39, s93, 54
-; FLATSCR-NEXT:    v_writelane_b32 v39, s94, 55
-; FLATSCR-NEXT:    v_writelane_b32 v39, s95, 56
-; FLATSCR-NEXT:    v_writelane_b32 v39, s96, 57
-; FLATSCR-NEXT:    v_writelane_b32 v39, s97, 58
-; FLATSCR-NEXT:    v_writelane_b32 v39, s98, 59
-; FLATSCR-NEXT:    v_writelane_b32 v39, s99, 60
-; FLATSCR-NEXT:    v_writelane_b32 v39, s100, 61
-; FLATSCR-NEXT:    v_writelane_b32 v39, s101, 62
+; FLATSCR-NEXT:    v_writelane_b32 v39, s48, 1
+; FLATSCR-NEXT:    v_writelane_b32 v39, s49, 2
+; FLATSCR-NEXT:    v_writelane_b32 v39, s50, 3
+; FLATSCR-NEXT:    v_writelane_b32 v39, s51, 4
+; FLATSCR-NEXT:    v_writelane_b32 v39, s52, 5
+; FLATSCR-NEXT:    v_writelane_b32 v39, s53, 6
+; FLATSCR-NEXT:    v_writelane_b32 v39, s54, 7
+; FLATSCR-NEXT:    v_writelane_b32 v39, s55, 8
+; FLATSCR-NEXT:    v_writelane_b32 v39, s64, 9
+; FLATSCR-NEXT:    v_writelane_b32 v39, s65, 10
+; FLATSCR-NEXT:    v_writelane_b32 v39, s66, 11
+; FLATSCR-NEXT:    v_writelane_b32 v39, s67, 12
+; FLATSCR-NEXT:    v_writelane_b32 v39, s68, 13
+; FLATSCR-NEXT:    v_writelane_b32 v39, s69, 14
+; FLATSCR-NEXT:    v_writelane_b32 v39, s70, 15
+; FLATSCR-NEXT:    v_writelane_b32 v39, s71, 16
+; FLATSCR-NEXT:    v_writelane_b32 v39, s80, 17
+; FLATSCR-NEXT:    v_writelane_b32 v39, s81, 18
+; FLATSCR-NEXT:    v_writelane_b32 v39, s82, 19
+; FLATSCR-NEXT:    v_writelane_b32 v39, s83, 20
+; FLATSCR-NEXT:    v_writelane_b32 v39, s84, 21
+; FLATSCR-NEXT:    v_writelane_b32 v39, s85, 22
+; FLATSCR-NEXT:    v_writelane_b32 v39, s86, 23
+; FLATSCR-NEXT:    v_writelane_b32 v39, s87, 24
+; FLATSCR-NEXT:    v_writelane_b32 v39, s96, 25
+; FLATSCR-NEXT:    v_writelane_b32 v39, s97, 26
+; FLATSCR-NEXT:    v_writelane_b32 v39, s98, 27
+; FLATSCR-NEXT:    v_writelane_b32 v39, s99, 28
+; FLATSCR-NEXT:    v_writelane_b32 v39, s100, 29
+; FLATSCR-NEXT:    v_writelane_b32 v39, s101, 30
 ; FLATSCR-NEXT:    s_add_i32 s32, s32, 8
-; FLATSCR-NEXT:    v_writelane_b32 v39, s102, 63
+; FLATSCR-NEXT:    v_writelane_b32 v39, s102, 31
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ; clobber nonpreserved SGPRs and 64 CSRs
 ; FLATSCR-NEXT:    ;;#ASMEND
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ; clobber all VGPRs except CSR v40
 ; FLATSCR-NEXT:    ;;#ASMEND
-; FLATSCR-NEXT:    v_readlane_b32 s102, v39, 63
-; FLATSCR-NEXT:    v_readlane_b32 s101, v39, 62
-; FLATSCR-NEXT:    v_readlane_b32 s100, v39, 61
-; FLATSCR-NEXT:    v_readlane_b32 s99, v39, 60
-; FLATSCR-NEXT:    v_readlane_b32 s98, v39, 59
-; FLATSCR-NEXT:    v_readlane_b32 s97, v39, 58
-; FLATSCR-NEXT:    v_readlane_b32 s96, v39, 57
-; FLATSCR-NEXT:    v_readlane_b32 s95, v39, 56
-; FLATSCR-NEXT:    v_readlane_b32 s94, v39, 55
-; FLATSCR-NEXT:    v_readlane_b32 s93, v39, 54
-; FLATSCR-NEXT:    v_readlane_b32 s92, v39, 53
-; FLATSCR-NEXT:    v_readlane_b32 s91, v39, 52
-; FLATSCR-NEXT:    v_readlane_b32 s90, v39, 51
-; FLATSCR-NEXT:    v_readlane_b32 s89, v39, 50
-; FLATSCR-NEXT:    v_readlane_b32 s88, v39, 49
-; FLATSCR-NEXT:    v_readlane_b32 s87, v39, 48
-; FLATSCR-NEXT:    v_readlane_b32 s86, v39, 47
-; FLATSCR-NEXT:    v_readlane_b32 s85, v39, 46
-; FLATSCR-NEXT:    v_readlane_b32 s84, v39, 45
-; FLATSCR-NEXT:    v_readlane_b32 s83, v39, 44
-; FLATSCR-NEXT:    v_readlane_b32 s82, v39, 43
-; FLATSCR-NEXT:    v_readlane_b32 s81, v39, 42
-; FLATSCR-NEXT:    v_readlane_b32 s80, v39, 41
-; FLATSCR-NEXT:    v_readlane_b32 s79, v39, 40
-; FLATSCR-NEXT:    v_readlane_b32 s78, v39, 39
-; FLATSCR-NEXT:    v_readlane_b32 s77, v39, 38
-; FLATSCR-NEXT:    v_readlane_b32 s76, v39, 37
-; FLATSCR-NEXT:    v_readlane_b32 s75, v39, 36
-; FLATSCR-NEXT:    v_readlane_b32 s74, v39, 35
-; FLATSCR-NEXT:    v_readlane_b32 s73, v39, 34
-; FLATSCR-NEXT:    v_readlane_b32 s72, v39, 33
-; FLATSCR-NEXT:    v_readlane_b32 s71, v39, 32
-; FLATSCR-NEXT:    v_readlane_b32 s70, v39, 31
-; FLATSCR-NEXT:    v_readlane_b32 s69, v39, 30
-; FLATSCR-NEXT:    v_readlane_b32 s68, v39, 29
-; FLATSCR-NEXT:    v_readlane_b32 s67, v39, 28
-; FLATSCR-NEXT:    v_readlane_b32 s66, v39, 27
-; FLATSCR-NEXT:    v_readlane_b32 s65, v39, 26
-; FLATSCR-NEXT:    v_readlane_b32 s64, v39, 25
-; FLATSCR-NEXT:    v_readlane_b32 s63, v39, 24
-; FLATSCR-NEXT:    v_readlane_b32 s62, v39, 23
-; FLATSCR-NEXT:    v_readlane_b32 s61, v39, 22
-; FLATSCR-NEXT:    v_readlane_b32 s60, v39, 21
-; FLATSCR-NEXT:    v_readlane_b32 s59, v39, 20
-; FLATSCR-NEXT:    v_readlane_b32 s58, v39, 19
-; FLATSCR-NEXT:    v_readlane_b32 s57, v39, 18
-; FLATSCR-NEXT:    v_readlane_b32 s56, v39, 17
-; FLATSCR-NEXT:    v_readlane_b32 s55, v39, 16
-; FLATSCR-NEXT:    v_readlane_b32 s54, v39, 15
-; FLATSCR-NEXT:    v_readlane_b32 s53, v39, 14
-; FLATSCR-NEXT:    v_readlane_b32 s52, v39, 13
-; FLATSCR-NEXT:    v_readlane_b32 s51, v39, 12
-; FLATSCR-NEXT:    v_readlane_b32 s50, v39, 11
-; FLATSCR-NEXT:    v_readlane_b32 s49, v39, 10
-; FLATSCR-NEXT:    v_readlane_b32 s48, v39, 9
-; FLATSCR-NEXT:    v_readlane_b32 s47, v39, 8
-; FLATSCR-NEXT:    v_readlane_b32 s46, v39, 7
-; FLATSCR-NEXT:    v_readlane_b32 s45, v39, 6
-; FLATSCR-NEXT:    v_readlane_b32 s44, v39, 5
-; FLATSCR-NEXT:    v_readlane_b32 s43, v39, 4
-; FLATSCR-NEXT:    v_readlane_b32 s42, v39, 3
-; FLATSCR-NEXT:    v_readlane_b32 s41, v39, 2
-; FLATSCR-NEXT:    v_readlane_b32 s40, v39, 1
+; FLATSCR-NEXT:    v_readlane_b32 s102, v39, 31
+; FLATSCR-NEXT:    v_readlane_b32 s101, v39, 30
+; FLATSCR-NEXT:    v_readlane_b32 s100, v39, 29
+; FLATSCR-NEXT:    v_readlane_b32 s99, v39, 28
+; FLATSCR-NEXT:    v_readlane_b32 s98, v39, 27
+; FLATSCR-NEXT:    v_readlane_b32 s97, v39, 26
+; FLATSCR-NEXT:    v_readlane_b32 s96, v39, 25
+; FLATSCR-NEXT:    v_readlane_b32 s87, v39, 24
+; FLATSCR-NEXT:    v_readlane_b32 s86, v39, 23
+; FLATSCR-NEXT:    v_readlane_b32 s85, v39, 22
+; FLATSCR-NEXT:    v_readlane_b32 s84, v39, 21
+; FLATSCR-NEXT:    v_readlane_b32 s83, v39, 20
+; FLATSCR-NEXT:    v_readlane_b32 s82, v39, 19
+; FLATSCR-NEXT:    v_readlane_b32 s81, v39, 18
+; FLATSCR-NEXT:    v_readlane_b32 s80, v39, 17
+; FLATSCR-NEXT:    v_readlane_b32 s71, v39, 16
+; FLATSCR-NEXT:    v_readlane_b32 s70, v39, 15
+; FLATSCR-NEXT:    v_readlane_b32 s69, v39, 14
+; FLATSCR-NEXT:    v_readlane_b32 s68, v39, 13
+; FLATSCR-NEXT:    v_readlane_b32 s67, v39, 12
+; FLATSCR-NEXT:    v_readlane_b32 s66, v39, 11
+; FLATSCR-NEXT:    v_readlane_b32 s65, v39, 10
+; FLATSCR-NEXT:    v_readlane_b32 s64, v39, 9
+; FLATSCR-NEXT:    v_readlane_b32 s55, v39, 8
+; FLATSCR-NEXT:    v_readlane_b32 s54, v39, 7
+; FLATSCR-NEXT:    v_readlane_b32 s53, v39, 6
+; FLATSCR-NEXT:    v_readlane_b32 s52, v39, 5
+; FLATSCR-NEXT:    v_readlane_b32 s51, v39, 4
+; FLATSCR-NEXT:    v_readlane_b32 s50, v39, 3
+; FLATSCR-NEXT:    v_readlane_b32 s49, v39, 2
+; FLATSCR-NEXT:    v_readlane_b32 s48, v39, 1
 ; FLATSCR-NEXT:    v_readlane_b32 s39, v39, 0
 ; FLATSCR-NEXT:    s_mov_b32 s32, s33
 ; FLATSCR-NEXT:    s_xor_saveexec_b64 s[2:3], -1
@@ -2037,149 +1517,83 @@ define void @callee_need_to_spill_fp_to_reg() #1 {
 ; MUBUF-NEXT:    s_mov_b32 s33, s32
 ; MUBUF-NEXT:    s_or_saveexec_b64 s[6:7], -1
 ; MUBUF-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
-; MUBUF-NEXT:    buffer_store_dword v41, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
 ; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
+; MUBUF-NEXT:    v_writelane_b32 v40, s4, 32
 ; MUBUF-NEXT:    v_writelane_b32 v40, s39, 0
-; MUBUF-NEXT:    v_writelane_b32 v40, s40, 1
-; MUBUF-NEXT:    v_writelane_b32 v40, s41, 2
-; MUBUF-NEXT:    v_writelane_b32 v40, s42, 3
-; MUBUF-NEXT:    v_writelane_b32 v40, s43, 4
-; MUBUF-NEXT:    v_writelane_b32 v40, s44, 5
-; MUBUF-NEXT:    v_writelane_b32 v40, s45, 6
-; MUBUF-NEXT:    v_writelane_b32 v40, s46, 7
-; MUBUF-NEXT:    v_writelane_b32 v40, s47, 8
-; MUBUF-NEXT:    v_writelane_b32 v40, s48, 9
-; MUBUF-NEXT:    v_writelane_b32 v40, s49, 10
-; MUBUF-NEXT:    v_writelane_b32 v40, s50, 11
-; MUBUF-NEXT:    v_writelane_b32 v40, s51, 12
-; MUBUF-NEXT:    v_writelane_b32 v40, s52, 13
-; MUBUF-NEXT:    v_writelane_b32 v40, s53, 14
-; MUBUF-NEXT:    v_writelane_b32 v40, s54, 15
-; MUBUF-NEXT:    v_writelane_b32 v40, s55, 16
-; MUBUF-NEXT:    v_writelane_b32 v40, s56, 17
-; MUBUF-NEXT:    v_writelane_b32 v40, s57, 18
-; MUBUF-NEXT:    v_writelane_b32 v40, s58, 19
-; MUBUF-NEXT:    v_writelane_b32 v40, s59, 20
-; MUBUF-NEXT:    v_writelane_b32 v40, s60, 21
-; MUBUF-NEXT:    v_writelane_b32 v40, s61, 22
-; MUBUF-NEXT:    v_writelane_b32 v40, s62, 23
-; MUBUF-NEXT:    v_writelane_b32 v40, s63, 24
-; MUBUF-NEXT:    v_writelane_b32 v40, s64, 25
-; MUBUF-NEXT:    v_writelane_b32 v40, s65, 26
-; MUBUF-NEXT:    v_writelane_b32 v40, s66, 27
-; MUBUF-NEXT:    v_writelane_b32 v40, s67, 28
-; MUBUF-NEXT:    v_writelane_b32 v40, s68, 29
-; MUBUF-NEXT:    v_writelane_b32 v40, s69, 30
-; MUBUF-NEXT:    v_writelane_b32 v40, s70, 31
-; MUBUF-NEXT:    v_writelane_b32 v40, s71, 32
-; MUBUF-NEXT:    v_writelane_b32 v40, s72, 33
-; MUBUF-NEXT:    v_writelane_b32 v40, s73, 34
-; MUBUF-NEXT:    v_writelane_b32 v40, s74, 35
-; MUBUF-NEXT:    v_writelane_b32 v40, s75, 36
-; MUBUF-NEXT:    v_writelane_b32 v40, s76, 37
-; MUBUF-NEXT:    v_writelane_b32 v40, s77, 38
-; MUBUF-NEXT:    v_writelane_b32 v40, s78, 39
-; MUBUF-NEXT:    v_writelane_b32 v40, s79, 40
-; MUBUF-NEXT:    v_writelane_b32 v40, s80, 41
-; MUBUF-NEXT:    v_writelane_b32 v40, s81, 42
-; MUBUF-NEXT:    v_writelane_b32 v40, s82, 43
-; MUBUF-NEXT:    v_writelane_b32 v40, s83, 44
-; MUBUF-NEXT:    v_writelane_b32 v40, s84, 45
-; MUBUF-NEXT:    v_writelane_b32 v40, s85, 46
-; MUBUF-NEXT:    v_writelane_b32 v40, s86, 47
-; MUBUF-NEXT:    v_writelane_b32 v40, s87, 48
-; MUBUF-NEXT:    v_writelane_b32 v40, s88, 49
-; MUBUF-NEXT:    v_writelane_b32 v40, s89, 50
-; MUBUF-NEXT:    v_writelane_b32 v40, s90, 51
-; MUBUF-NEXT:    v_writelane_b32 v40, s91, 52
-; MUBUF-NEXT:    v_writelane_b32 v40, s92, 53
-; MUBUF-NEXT:    v_writelane_b32 v40, s93, 54
-; MUBUF-NEXT:    v_writelane_b32 v40, s94, 55
-; MUBUF-NEXT:    v_writelane_b32 v40, s95, 56
-; MUBUF-NEXT:    v_writelane_b32 v40, s96, 57
-; MUBUF-NEXT:    v_writelane_b32 v40, s97, 58
-; MUBUF-NEXT:    v_writelane_b32 v40, s98, 59
-; MUBUF-NEXT:    v_writelane_b32 v40, s99, 60
-; MUBUF-NEXT:    v_writelane_b32 v40, s100, 61
-; MUBUF-NEXT:    v_writelane_b32 v40, s101, 62
-; MUBUF-NEXT:    v_writelane_b32 v41, s4, 0
-; MUBUF-NEXT:    s_addk_i32 s32, 0x300
-; MUBUF-NEXT:    v_writelane_b32 v40, s102, 63
+; MUBUF-NEXT:    v_writelane_b32 v40, s48, 1
+; MUBUF-NEXT:    v_writelane_b32 v40, s49, 2
+; MUBUF-NEXT:    v_writelane_b32 v40, s50, 3
+; MUBUF-NEXT:    v_writelane_b32 v40, s51, 4
+; MUBUF-NEXT:    v_writelane_b32 v40, s52, 5
+; MUBUF-NEXT:    v_writelane_b32 v40, s53, 6
+; MUBUF-NEXT:    v_writelane_b32 v40, s54, 7
+; MUBUF-NEXT:    v_writelane_b32 v40, s55, 8
+; MUBUF-NEXT:    v_writelane_b32 v40, s64, 9
+; MUBUF-NEXT:    v_writelane_b32 v40, s65, 10
+; MUBUF-NEXT:    v_writelane_b32 v40, s66, 11
+; MUBUF-NEXT:    v_writelane_b32 v40, s67, 12
+; MUBUF-NEXT:    v_writelane_b32 v40, s68, 13
+; MUBUF-NEXT:    v_writelane_b32 v40, s69, 14
+; MUBUF-NEXT:    v_writelane_b32 v40, s70, 15
+; MUBUF-NEXT:    v_writelane_b32 v40, s71, 16
+; MUBUF-NEXT:    v_writelane_b32 v40, s80, 17
+; MUBUF-NEXT:    v_writelane_b32 v40, s81, 18
+; MUBUF-NEXT:    v_writelane_b32 v40, s82, 19
+; MUBUF-NEXT:    v_writelane_b32 v40, s83, 20
+; MUBUF-NEXT:    v_writelane_b32 v40, s84, 21
+; MUBUF-NEXT:    v_writelane_b32 v40, s85, 22
+; MUBUF-NEXT:    v_writelane_b32 v40, s86, 23
+; MUBUF-NEXT:    v_writelane_b32 v40, s87, 24
+; MUBUF-NEXT:    v_writelane_b32 v40, s96, 25
+; MUBUF-NEXT:    v_writelane_b32 v40, s97, 26
+; MUBUF-NEXT:    v_writelane_b32 v40, s98, 27
+; MUBUF-NEXT:    v_writelane_b32 v40, s99, 28
+; MUBUF-NEXT:    v_writelane_b32 v40, s100, 29
+; MUBUF-NEXT:    v_writelane_b32 v40, s101, 30
+; MUBUF-NEXT:    s_addk_i32 s32, 0x200
+; MUBUF-NEXT:    v_writelane_b32 v40, s102, 31
 ; MUBUF-NEXT:    ;;#ASMSTART
 ; MUBUF-NEXT:    ; clobber nonpreserved SGPRs and 64 CSRs
 ; MUBUF-NEXT:    ;;#ASMEND
 ; MUBUF-NEXT:    ;;#ASMSTART
 ; MUBUF-NEXT:    ; clobber all VGPRs except CSR v40
 ; MUBUF-NEXT:    ;;#ASMEND
-; MUBUF-NEXT:    v_readlane_b32 s102, v40, 63
-; MUBUF-NEXT:    v_readlane_b32 s101, v40, 62
-; MUBUF-NEXT:    v_readlane_b32 s100, v40, 61
-; MUBUF-NEXT:    v_readlane_b32 s99, v40, 60
-; MUBUF-NEXT:    v_readlane_b32 s98, v40, 59
-; MUBUF-NEXT:    v_readlane_b32 s97, v40, 58
-; MUBUF-NEXT:    v_readlane_b32 s96, v40, 57
-; MUBUF-NEXT:    v_readlane_b32 s95, v40, 56
-; MUBUF-NEXT:    v_readlane_b32 s94, v40, 55
-; MUBUF-NEXT:    v_readlane_b32 s93, v40, 54
-; MUBUF-NEXT:    v_readlane_b32 s92, v40, 53
-; MUBUF-NEXT:    v_readlane_b32 s91, v40, 52
-; MUBUF-NEXT:    v_readlane_b32 s90, v40, 51
-; MUBUF-NEXT:    v_readlane_b32 s89, v40, 50
-; MUBUF-NEXT:    v_readlane_b32 s88, v40, 49
-; MUBUF-NEXT:    v_readlane_b32 s87, v40, 48
-; MUBUF-NEXT:    v_readlane_b32 s86, v40, 47
-; MUBUF-NEXT:    v_readlane_b32 s85, v40, 46
-; MUBUF-NEXT:    v_readlane_b32 s84, v40, 45
-; MUBUF-NEXT:    v_readlane_b32 s83, v40, 44
-; MUBUF-NEXT:    v_readlane_b32 s82, v40, 43
-; MUBUF-NEXT:    v_readlane_b32 s81, v40, 42
-; MUBUF-NEXT:    v_readlane_b32 s80, v40, 41
-; MUBUF-NEXT:    v_readlane_b32 s79, v40, 40
-; MUBUF-NEXT:    v_readlane_b32 s78, v40, 39
-; MUBUF-NEXT:    v_readlane_b32 s77, v40, 38
-; MUBUF-NEXT:    v_readlane_b32 s76, v40, 37
-; MUBUF-NEXT:    v_readlane_b32 s75, v40, 36
-; MUBUF-NEXT:    v_readlane_b32 s74, v40, 35
-; MUBUF-NEXT:    v_readlane_b32 s73, v40, 34
-; MUBUF-NEXT:    v_readlane_b32 s72, v40, 33
-; MUBUF-NEXT:    v_readlane_b32 s71, v40, 32
-; MUBUF-NEXT:    v_readlane_b32 s70, v40, 31
-; MUBUF-NEXT:    v_readlane_b32 s69, v40, 30
-; MUBUF-NEXT:    v_readlane_b32 s68, v40, 29
-; MUBUF-NEXT:    v_readlane_b32 s67, v40, 28
-; MUBUF-NEXT:    v_readlane_b32 s66, v40, 27
-; MUBUF-NEXT:    v_readlane_b32 s65, v40, 26
-; MUBUF-NEXT:    v_readlane_b32 s64, v40, 25
-; MUBUF-NEXT:    v_readlane_b32 s63, v40, 24
-; MUBUF-NEXT:    v_readlane_b32 s62, v40, 23
-; MUBUF-NEXT:    v_readlane_b32 s61, v40, 22
-; MUBUF-NEXT:    v_readlane_b32 s60, v40, 21
-; MUBUF-NEXT:    v_readlane_b32 s59, v40, 20
-; MUBUF-NEXT:    v_readlane_b32 s58, v40, 19
-; MUBUF-NEXT:    v_readlane_b32 s57, v40, 18
-; MUBUF-NEXT:    v_readlane_b32 s56, v40, 17
-; MUBUF-NEXT:    v_readlane_b32 s55, v40, 16
-; MUBUF-NEXT:    v_readlane_b32 s54, v40, 15
-; MUBUF-NEXT:    v_readlane_b32 s53, v40, 14
-; MUBUF-NEXT:    v_readlane_b32 s52, v40, 13
-; MUBUF-NEXT:    v_readlane_b32 s51, v40, 12
-; MUBUF-NEXT:    v_readlane_b32 s50, v40, 11
-; MUBUF-NEXT:    v_readlane_b32 s49, v40, 10
-; MUBUF-NEXT:    v_readlane_b32 s48, v40, 9
-; MUBUF-NEXT:    v_readlane_b32 s47, v40, 8
-; MUBUF-NEXT:    v_readlane_b32 s46, v40, 7
-; MUBUF-NEXT:    v_readlane_b32 s45, v40, 6
-; MUBUF-NEXT:    v_readlane_b32 s44, v40, 5
-; MUBUF-NEXT:    v_readlane_b32 s43, v40, 4
-; MUBUF-NEXT:    v_readlane_b32 s42, v40, 3
-; MUBUF-NEXT:    v_readlane_b32 s41, v40, 2
-; MUBUF-NEXT:    v_readlane_b32 s40, v40, 1
+; MUBUF-NEXT:    v_readlane_b32 s102, v40, 31
+; MUBUF-NEXT:    v_readlane_b32 s101, v40, 30
+; MUBUF-NEXT:    v_readlane_b32 s100, v40, 29
+; MUBUF-NEXT:    v_readlane_b32 s99, v40, 28
+; MUBUF-NEXT:    v_readlane_b32 s98, v40, 27
+; MUBUF-NEXT:    v_readlane_b32 s97, v40, 26
+; MUBUF-NEXT:    v_readlane_b32 s96, v40, 25
+; MUBUF-NEXT:    v_readlane_b32 s87, v40, 24
+; MUBUF-NEXT:    v_readlane_b32 s86, v40, 23
+; MUBUF-NEXT:    v_readlane_b32 s85, v40, 22
+; MUBUF-NEXT:    v_readlane_b32 s84, v40, 21
+; MUBUF-NEXT:    v_readlane_b32 s83, v40, 20
+; MUBUF-NEXT:    v_readlane_b32 s82, v40, 19
+; MUBUF-NEXT:    v_readlane_b32 s81, v40, 18
+; MUBUF-NEXT:    v_readlane_b32 s80, v40, 17
+; MUBUF-NEXT:    v_readlane_b32 s71, v40, 16
+; MUBUF-NEXT:    v_readlane_b32 s70, v40, 15
+; MUBUF-NEXT:    v_readlane_b32 s69, v40, 14
+; MUBUF-NEXT:    v_readlane_b32 s68, v40, 13
+; MUBUF-NEXT:    v_readlane_b32 s67, v40, 12
+; MUBUF-NEXT:    v_readlane_b32 s66, v40, 11
+; MUBUF-NEXT:    v_readlane_b32 s65, v40, 10
+; MUBUF-NEXT:    v_readlane_b32 s64, v40, 9
+; MUBUF-NEXT:    v_readlane_b32 s55, v40, 8
+; MUBUF-NEXT:    v_readlane_b32 s54, v40, 7
+; MUBUF-NEXT:    v_readlane_b32 s53, v40, 6
+; MUBUF-NEXT:    v_readlane_b32 s52, v40, 5
+; MUBUF-NEXT:    v_readlane_b32 s51, v40, 4
+; MUBUF-NEXT:    v_readlane_b32 s50, v40, 3
+; MUBUF-NEXT:    v_readlane_b32 s49, v40, 2
+; MUBUF-NEXT:    v_readlane_b32 s48, v40, 1
 ; MUBUF-NEXT:    v_readlane_b32 s39, v40, 0
 ; MUBUF-NEXT:    s_mov_b32 s32, s33
-; MUBUF-NEXT:    v_readlane_b32 s4, v41, 0
+; MUBUF-NEXT:    v_readlane_b32 s4, v40, 32
 ; MUBUF-NEXT:    s_or_saveexec_b64 s[6:7], -1
 ; MUBUF-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
-; MUBUF-NEXT:    buffer_load_dword v41, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
 ; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
 ; MUBUF-NEXT:    s_mov_b32 s33, s4
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
@@ -2194,139 +1608,75 @@ define void @callee_need_to_spill_fp_to_reg() #1 {
 ; FLATSCR-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
 ; FLATSCR-NEXT:    v_writelane_b32 v40, s39, 0
-; FLATSCR-NEXT:    v_writelane_b32 v40, s40, 1
-; FLATSCR-NEXT:    v_writelane_b32 v40, s41, 2
-; FLATSCR-NEXT:    v_writelane_b32 v40, s42, 3
-; FLATSCR-NEXT:    v_writelane_b32 v40, s43, 4
-; FLATSCR-NEXT:    v_writelane_b32 v40, s44, 5
-; FLATSCR-NEXT:    v_writelane_b32 v40, s45, 6
-; FLATSCR-NEXT:    v_writelane_b32 v40, s46, 7
-; FLATSCR-NEXT:    v_writelane_b32 v40, s47, 8
-; FLATSCR-NEXT:    v_writelane_b32 v40, s48, 9
-; FLATSCR-NEXT:    v_writelane_b32 v40, s49, 10
-; FLATSCR-NEXT:    v_writelane_b32 v40, s50, 11
-; FLATSCR-NEXT:    v_writelane_b32 v40, s51, 12
-; FLATSCR-NEXT:    v_writelane_b32 v40, s52, 13
-; FLATSCR-NEXT:    v_writelane_b32 v40, s53, 14
-; FLATSCR-NEXT:    v_writelane_b32 v40, s54, 15
-; FLATSCR-NEXT:    v_writelane_b32 v40, s55, 16
-; FLATSCR-NEXT:    v_writelane_b32 v40, s56, 17
-; FLATSCR-NEXT:    v_writelane_b32 v40, s57, 18
-; FLATSCR-NEXT:    v_writelane_b32 v40, s58, 19
-; FLATSCR-NEXT:    v_writelane_b32 v40, s59, 20
-; FLATSCR-NEXT:    v_writelane_b32 v40, s60, 21
-; FLATSCR-NEXT:    v_writelane_b32 v40, s61, 22
-; FLATSCR-NEXT:    v_writelane_b32 v40, s62, 23
-; FLATSCR-NEXT:    v_writelane_b32 v40, s63, 24
-; FLATSCR-NEXT:    v_writelane_b32 v40, s64, 25
-; FLATSCR-NEXT:    v_writelane_b32 v40, s65, 26
-; FLATSCR-NEXT:    v_writelane_b32 v40, s66, 27
-; FLATSCR-NEXT:    v_writelane_b32 v40, s67, 28
-; FLATSCR-NEXT:    v_writelane_b32 v40, s68, 29
-; FLATSCR-NEXT:    v_writelane_b32 v40, s69, 30
-; FLATSCR-NEXT:    v_writelane_b32 v40, s70, 31
-; FLATSCR-NEXT:    v_writelane_b32 v40, s71, 32
-; FLATSCR-NEXT:    v_writelane_b32 v40, s72, 33
-; FLATSCR-NEXT:    v_writelane_b32 v40, s73, 34
-; FLATSCR-NEXT:    v_writelane_b32 v40, s74, 35
-; FLATSCR-NEXT:    v_writelane_b32 v40, s75, 36
-; FLATSCR-NEXT:    v_writelane_b32 v40, s76, 37
-; FLATSCR-NEXT:    v_writelane_b32 v40, s77, 38
-; FLATSCR-NEXT:    v_writelane_b32 v40, s78, 39
-; FLATSCR-NEXT:    v_writelane_b32 v40, s79, 40
-; FLATSCR-NEXT:    v_writelane_b32 v40, s80, 41
-; FLATSCR-NEXT:    v_writelane_b32 v40, s81, 42
-; FLATSCR-NEXT:    v_writelane_b32 v40, s82, 43
-; FLATSCR-NEXT:    v_writelane_b32 v40, s83, 44
-; FLATSCR-NEXT:    v_writelane_b32 v40, s84, 45
-; FLATSCR-NEXT:    v_writelane_b32 v40, s85, 46
-; FLATSCR-NEXT:    v_writelane_b32 v40, s86, 47
-; FLATSCR-NEXT:    v_writelane_b32 v40, s87, 48
-; FLATSCR-NEXT:    v_writelane_b32 v40, s88, 49
-; FLATSCR-NEXT:    v_writelane_b32 v40, s89, 50
-; FLATSCR-NEXT:    v_writelane_b32 v40, s90, 51
-; FLATSCR-NEXT:    v_writelane_b32 v40, s91, 52
-; FLATSCR-NEXT:    v_writelane_b32 v40, s92, 53
-; FLATSCR-NEXT:    v_writelane_b32 v40, s93, 54
-; FLATSCR-NEXT:    v_writelane_b32 v40, s94, 55
-; FLATSCR-NEXT:    v_writelane_b32 v40, s95, 56
-; FLATSCR-NEXT:    v_writelane_b32 v40, s96, 57
-; FLATSCR-NEXT:    v_writelane_b32 v40, s97, 58
-; FLATSCR-NEXT:    v_writelane_b32 v40, s98, 59
-; FLATSCR-NEXT:    v_writelane_b32 v40, s99, 60
-; FLATSCR-NEXT:    v_writelane_b32 v40, s100, 61
-; FLATSCR-NEXT:    v_writelane_b32 v40, s101, 62
+; FLATSCR-NEXT:    v_writelane_b32 v40, s48, 1
+; FLATSCR-NEXT:    v_writelane_b32 v40, s49, 2
+; FLATSCR-NEXT:    v_writelane_b32 v40, s50, 3
+; FLATSCR-NEXT:    v_writelane_b32 v40, s51, 4
+; FLATSCR-NEXT:    v_writelane_b32 v40, s52, 5
+; FLATSCR-NEXT:    v_writelane_b32 v40, s53, 6
+; FLATSCR-NEXT:    v_writelane_b32 v40, s54, 7
+; FLATSCR-NEXT:    v_writelane_b32 v40, s55, 8
+; FLATSCR-NEXT:    v_writelane_b32 v40, s64, 9
+; FLATSCR-NEXT:    v_writelane_b32 v40, s65, 10
+; FLATSCR-NEXT:    v_writelane_b32 v40, s66, 11
+; FLATSCR-NEXT:    v_writelane_b32 v40, s67, 12
+; FLATSCR-NEXT:    v_writelane_b32 v40, s68, 13
+; FLATSCR-NEXT:    v_writelane_b32 v40, s69, 14
+; FLATSCR-NEXT:    v_writelane_b32 v40, s70, 15
+; FLATSCR-NEXT:    v_writelane_b32 v40, s71, 16
+; FLATSCR-NEXT:    v_writelane_b32 v40, s80, 17
+; FLATSCR-NEXT:    v_writelane_b32 v40, s81, 18
+; FLATSCR-NEXT:    v_writelane_b32 v40, s82, 19
+; FLATSCR-NEXT:    v_writelane_b32 v40, s83, 20
+; FLATSCR-NEXT:    v_writelane_b32 v40, s84, 21
+; FLATSCR-NEXT:    v_writelane_b32 v40, s85, 22
+; FLATSCR-NEXT:    v_writelane_b32 v40, s86, 23
+; FLATSCR-NEXT:    v_writelane_b32 v40, s87, 24
+; FLATSCR-NEXT:    v_writelane_b32 v40, s96, 25
+; FLATSCR-NEXT:    v_writelane_b32 v40, s97, 26
+; FLATSCR-NEXT:    v_writelane_b32 v40, s98, 27
+; FLATSCR-NEXT:    v_writelane_b32 v40, s99, 28
+; FLATSCR-NEXT:    v_writelane_b32 v40, s100, 29
+; FLATSCR-NEXT:    v_writelane_b32 v40, s101, 30
 ; FLATSCR-NEXT:    s_add_i32 s32, s32, 8
-; FLATSCR-NEXT:    v_writelane_b32 v40, s102, 63
+; FLATSCR-NEXT:    v_writelane_b32 v40, s102, 31
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ; clobber nonpreserved SGPRs and 64 CSRs
 ; FLATSCR-NEXT:    ;;#ASMEND
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ; clobber all VGPRs except CSR v40
 ; FLATSCR-NEXT:    ;;#ASMEND
-; FLATSCR-NEXT:    v_readlane_b32 s102, v40, 63
-; FLATSCR-NEXT:    v_readlane_b32 s101, v40, 62
-; FLATSCR-NEXT:    v_readlane_b32 s100, v40, 61
-; FLATSCR-NEXT:    v_readlane_b32 s99, v40, 60
-; FLATSCR-NEXT:    v_readlane_b32 s98, v40, 59
-; FLATSCR-NEXT:    v_readlane_b32 s97, v40, 58
-; FLATSCR-NEXT:    v_readlane_b32 s96, v40, 57
-; FLATSCR-NEXT:    v_readlane_b32 s95, v40, 56
-; FLATSCR-NEXT:    v_readlane_b32 s94, v40, 55
-; FLATSCR-NEXT:    v_readlane_b32 s93, v40, 54
-; FLATSCR-NEXT:    v_readlane_b32 s92, v40, 53
-; FLATSCR-NEXT:    v_readlane_b32 s91, v40, 52
-; FLATSCR-NEXT:    v_readlane_b32 s90, v40, 51
-; FLATSCR-NEXT:    v_readlane_b32 s89, v40, 50
-; FLATSCR-NEXT:    v_readlane_b32 s88, v40, 49
-; FLATSCR-NEXT:    v_readlane_b32 s87, v40, 48
-; FLATSCR-NEXT:    v_readlane_b32 s86, v40, 47
-; FLATSCR-NEXT:    v_readlane_b32 s85, v40, 46
-; FLATSCR-NEXT:    v_readlane_b32 s84, v40, 45
-; FLATSCR-NEXT:    v_readlane_b32 s83, v40, 44
-; FLATSCR-NEXT:    v_readlane_b32 s82, v40, 43
-; FLATSCR-NEXT:    v_readlane_b32 s81, v40, 42
-; FLATSCR-NEXT:    v_readlane_b32 s80, v40, 41
-; FLATSCR-NEXT:    v_readlane_b32 s79, v40, 40
-; FLATSCR-NEXT:    v_readlane_b32 s78, v40, 39
-; FLATSCR-NEXT:    v_readlane_b32 s77, v40, 38
-; FLATSCR-NEXT:    v_readlane_b32 s76, v40, 37
-; FLATSCR-NEXT:    v_readlane_b32 s75, v40, 36
-; FLATSCR-NEXT:    v_readlane_b32 s74, v40, 35
-; FLATSCR-NEXT:    v_readlane_b32 s73, v40, 34
-; FLATSCR-NEXT:    v_readlane_b32 s72, v40, 33
-; FLATSCR-NEXT:    v_readlane_b32 s71, v40, 32
-; FLATSCR-NEXT:    v_readlane_b32 s70, v40, 31
-; FLATSCR-NEXT:    v_readlane_b32 s69, v40, 30
-; FLATSCR-NEXT:    v_readlane_b32 s68, v40, 29
-; FLATSCR-NEXT:    v_readlane_b32 s67, v40, 28
-; FLATSCR-NEXT:    v_readlane_b32 s66, v40, 27
-; FLATSCR-NEXT:    v_readlane_b32 s65, v40, 26
-; FLATSCR-NEXT:    v_readlane_b32 s64, v40, 25
-; FLATSCR-NEXT:    v_readlane_b32 s63, v40, 24
-; FLATSCR-NEXT:    v_readlane_b32 s62, v40, 23
-; FLATSCR-NEXT:    v_readlane_b32 s61, v40, 22
-; FLATSCR-NEXT:    v_readlane_b32 s60, v40, 21
-; FLATSCR-NEXT:    v_readlane_b32 s59, v40, 20
-; FLATSCR-NEXT:    v_readlane_b32 s58, v40, 19
-; FLATSCR-NEXT:    v_readlane_b32 s57, v40, 18
-; FLATSCR-NEXT:    v_readlane_b32 s56, v40, 17
-; FLATSCR-NEXT:    v_readlane_b32 s55, v40, 16
-; FLATSCR-NEXT:    v_readlane_b32 s54, v40, 15
-; FLATSCR-NEXT:    v_readlane_b32 s53, v40, 14
-; FLATSCR-NEXT:    v_readlane_b32 s52, v40, 13
-; FLATSCR-NEXT:    v_readlane_b32 s51, v40, 12
-; FLATSCR-NEXT:    v_readlane_b32 s50, v40, 11
-; FLATSCR-NEXT:    v_readlane_b32 s49, v40, 10
-; FLATSCR-NEXT:    v_readlane_b32 s48, v40, 9
-; FLATSCR-NEXT:    v_readlane_b32 s47, v40, 8
-; FLATSCR-NEXT:    v_readlane_b32 s46, v40, 7
-; FLATSCR-NEXT:    v_readlane_b32 s45, v40, 6
-; FLATSCR-NEXT:    v_readlane_b32 s44, v40, 5
-; FLATSCR-NEXT:    v_readlane_b32 s43, v40, 4
-; FLATSCR-NEXT:    v_readlane_b32 s42, v40, 3
-; FLATSCR-NEXT:    v_readlane_b32 s41, v40, 2
-; FLATSCR-NEXT:    v_readlane_b32 s40, v40, 1
+; FLATSCR-NEXT:    v_readlane_b32 s102, v40, 31
+; FLATSCR-NEXT:    v_readlane_b32 s101, v40, 30
+; FLATSCR-NEXT:    v_readlane_b32 s100, v40, 29
+; FLATSCR-NEXT:    v_readlane_b32 s99, v40, 28
+; FLATSCR-NEXT:    v_readlane_b32 s98, v40, 27
+; FLATSCR-NEXT:    v_readlane_b32 s97, v40, 26
+; FLATSCR-NEXT:    v_readlane_b32 s96, v40, 25
+; FLATSCR-NEXT:    v_readlane_b32 s87, v40, 24
+; FLATSCR-NEXT:    v_readlane_b32 s86, v40, 23
+; FLATSCR-NEXT:    v_readlane_b32 s85, v40, 22
+; FLATSCR-NEXT:    v_readlane_b32 s84, v40, 21
+; FLATSCR-NEXT:    v_readlane_b32 s83, v40, 20
+; FLATSCR-NEXT:    v_readlane_b32 s82, v40, 19
+; FLATSCR-NEXT:    v_readlane_b32 s81, v40, 18
+; FLATSCR-NEXT:    v_readlane_b32 s80, v40, 17
+; FLATSCR-NEXT:    v_readlane_b32 s71, v40, 16
+; FLATSCR-NEXT:    v_readlane_b32 s70, v40, 15
+; FLATSCR-NEXT:    v_readlane_b32 s69, v40, 14
+; FLATSCR-NEXT:    v_readlane_b32 s68, v40, 13
+; FLATSCR-NEXT:    v_readlane_b32 s67, v40, 12
+; FLATSCR-NEXT:    v_readlane_b32 s66, v40, 11
+; FLATSCR-NEXT:    v_readlane_b32 s65, v40, 10
+; FLATSCR-NEXT:    v_readlane_b32 s64, v40, 9
+; FLATSCR-NEXT:    v_readlane_b32 s55, v40, 8
+; FLATSCR-NEXT:    v_readlane_b32 s54, v40, 7
+; FLATSCR-NEXT:    v_readlane_b32 s53, v40, 6
+; FLATSCR-NEXT:    v_readlane_b32 s52, v40, 5
+; FLATSCR-NEXT:    v_readlane_b32 s51, v40, 4
+; FLATSCR-NEXT:    v_readlane_b32 s50, v40, 3
+; FLATSCR-NEXT:    v_readlane_b32 s49, v40, 2
+; FLATSCR-NEXT:    v_readlane_b32 s48, v40, 1
 ; FLATSCR-NEXT:    v_readlane_b32 s39, v40, 0
 ; FLATSCR-NEXT:    s_mov_b32 s32, s33
 ; FLATSCR-NEXT:    s_or_saveexec_b64 s[2:3], -1
@@ -2367,75 +1717,42 @@ define void @spill_fp_to_memory_scratch_reg_needed_mubuf_offset(ptr addrspace(5)
 ; MUBUF-NEXT:    s_add_i32 s5, s33, 0x40100
 ; MUBUF-NEXT:    buffer_store_dword v39, off, s[0:3], s5 ; 4-byte Folded Spill
 ; MUBUF-NEXT:    s_mov_b64 exec, s[6:7]
+; MUBUF-NEXT:    v_writelane_b32 v39, s4, 32
 ; MUBUF-NEXT:    v_writelane_b32 v39, s39, 0
-; MUBUF-NEXT:    v_writelane_b32 v39, s40, 1
-; MUBUF-NEXT:    v_writelane_b32 v39, s41, 2
-; MUBUF-NEXT:    v_writelane_b32 v39, s42, 3
-; MUBUF-NEXT:    v_writelane_b32 v39, s43, 4
-; MUBUF-NEXT:    v_writelane_b32 v39, s44, 5
-; MUBUF-NEXT:    v_writelane_b32 v39, s45, 6
-; MUBUF-NEXT:    v_writelane_b32 v39, s46, 7
-; MUBUF-NEXT:    v_writelane_b32 v39, s47, 8
-; MUBUF-NEXT:    v_writelane_b32 v39, s48, 9
-; MUBUF-NEXT:    v_writelane_b32 v39, s49, 10
-; MUBUF-NEXT:    v_writelane_b32 v39, s50, 11
-; MUBUF-NEXT:    v_writelane_b32 v39, s51, 12
-; MUBUF-NEXT:    v_writelane_b32 v39, s52, 13
-; MUBUF-NEXT:    v_writelane_b32 v39, s53, 14
-; MUBUF-NEXT:    v_writelane_b32 v39, s54, 15
-; MUBUF-NEXT:    v_writelane_b32 v39, s55, 16
-; MUBUF-NEXT:    v_writelane_b32 v39, s56, 17
-; MUBUF-NEXT:    v_writelane_b32 v39, s57, 18
-; MUBUF-NEXT:    v_writelane_b32 v39, s58, 19
-; MUBUF-NEXT:    v_writelane_b32 v39, s59, 20
-; MUBUF-NEXT:    v_writelane_b32 v39, s60, 21
-; MUBUF-NEXT:    v_writelane_b32 v39, s61, 22
-; MUBUF-NEXT:    v_writelane_b32 v39, s62, 23
-; MUBUF-NEXT:    v_writelane_b32 v39, s63, 24
-; MUBUF-NEXT:    v_writelane_b32 v39, s64, 25
-; MUBUF-NEXT:    v_writelane_b32 v39, s65, 26
-; MUBUF-NEXT:    v_writelane_b32 v39, s66, 27
-; MUBUF-NEXT:    v_writelane_b32 v39, s67, 28
-; MUBUF-NEXT:    v_writelane_b32 v39, s68, 29
-; MUBUF-NEXT:    v_writelane_b32 v39, s69, 30
-; MUBUF-NEXT:    v_writelane_b32 v39, s70, 31
-; MUBUF-NEXT:    v_writelane_b32 v39, s71, 32
-; MUBUF-NEXT:    v_writelane_b32 v39, s72, 33
-; MUBUF-NEXT:    v_writelane_b32 v39, s73, 34
-; MUBUF-NEXT:    v_writelane_b32 v39, s74, 35
-; MUBUF-NEXT:    v_writelane_b32 v39, s75, 36
-; MUBUF-NEXT:    v_writelane_b32 v39, s76, 37
-; MUBUF-NEXT:    v_writelane_b32 v39, s77, 38
-; MUBUF-NEXT:    v_writelane_b32 v39, s78, 39
-; MUBUF-NEXT:    v_writelane_b32 v39, s79, 40
-; MUBUF-NEXT:    v_writelane_b32 v39, s80, 41
-; MUBUF-NEXT:    v_writelane_b32 v39, s81, 42
-; MUBUF-NEXT:    v_writelane_b32 v39, s82, 43
-; MUBUF-NEXT:    v_writelane_b32 v39, s83, 44
-; MUBUF-NEXT:    v_writelane_b32 v39, s84, 45
-; MUBUF-NEXT:    v_writelane_b32 v39, s85, 46
-; MUBUF-NEXT:    v_writelane_b32 v39, s86, 47
-; MUBUF-NEXT:    v_writelane_b32 v39, s87, 48
-; MUBUF-NEXT:    v_writelane_b32 v39, s88, 49
-; MUBUF-NEXT:    v_writelane_b32 v39, s89, 50
-; MUBUF-NEXT:    v_writelane_b32 v39, s90, 51
-; MUBUF-NEXT:    v_writelane_b32 v39, s91, 52
-; MUBUF-NEXT:    v_writelane_b32 v39, s92, 53
-; MUBUF-NEXT:    v_writelane_b32 v39, s93, 54
-; MUBUF-NEXT:    v_writelane_b32 v39, s94, 55
-; MUBUF-NEXT:    v_writelane_b32 v39, s95, 56
-; MUBUF-NEXT:    v_writelane_b32 v39, s96, 57
-; MUBUF-NEXT:    v_writelane_b32 v39, s97, 58
-; MUBUF-NEXT:    v_writelane_b32 v39, s98, 59
-; MUBUF-NEXT:    v_writelane_b32 v39, s99, 60
-; MUBUF-NEXT:    v_mov_b32_e32 v0, s4
-; MUBUF-NEXT:    s_add_i32 s5, s33, 0x40200
-; MUBUF-NEXT:    v_writelane_b32 v39, s100, 61
-; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s5 ; 4-byte Folded Spill
-; MUBUF-NEXT:    v_writelane_b32 v39, s101, 62
+; MUBUF-NEXT:    v_writelane_b32 v39, s48, 1
+; MUBUF-NEXT:    v_writelane_b32 v39, s49, 2
+; MUBUF-NEXT:    v_writelane_b32 v39, s50, 3
+; MUBUF-NEXT:    v_writelane_b32 v39, s51, 4
+; MUBUF-NEXT:    v_writelane_b32 v39, s52, 5
+; MUBUF-NEXT:    v_writelane_b32 v39, s53, 6
+; MUBUF-NEXT:    v_writelane_b32 v39, s54, 7
+; MUBUF-NEXT:    v_writelane_b32 v39, s55, 8
+; MUBUF-NEXT:    v_writelane_b32 v39, s64, 9
+; MUBUF-NEXT:    v_writelane_b32 v39, s65, 10
+; MUBUF-NEXT:    v_writelane_b32 v39, s66, 11
+; MUBUF-NEXT:    v_writelane_b32 v39, s67, 12
+; MUBUF-NEXT:    v_writelane_b32 v39, s68, 13
+; MUBUF-NEXT:    v_writelane_b32 v39, s69, 14
+; MUBUF-NEXT:    v_writelane_b32 v39, s70, 15
+; MUBUF-NEXT:    v_writelane_b32 v39, s71, 16
+; MUBUF-NEXT:    v_writelane_b32 v39, s80, 17
+; MUBUF-NEXT:    v_writelane_b32 v39, s81, 18
+; MUBUF-NEXT:    v_writelane_b32 v39, s82, 19
+; MUBUF-NEXT:    v_writelane_b32 v39, s83, 20
+; MUBUF-NEXT:    v_writelane_b32 v39, s84, 21
+; MUBUF-NEXT:    v_writelane_b32 v39, s85, 22
+; MUBUF-NEXT:    v_writelane_b32 v39, s86, 23
+; MUBUF-NEXT:    v_writelane_b32 v39, s87, 24
+; MUBUF-NEXT:    v_writelane_b32 v39, s96, 25
+; MUBUF-NEXT:    v_writelane_b32 v39, s97, 26
+; MUBUF-NEXT:    v_writelane_b32 v39, s98, 27
+; MUBUF-NEXT:    v_writelane_b32 v39, s99, 28
+; MUBUF-NEXT:    v_writelane_b32 v39, s100, 29
+; MUBUF-NEXT:    v_writelane_b32 v39, s101, 30
 ; MUBUF-NEXT:    v_mov_b32_e32 v0, 0
 ; MUBUF-NEXT:    v_mov_b32_e32 v1, 0x1000
-; MUBUF-NEXT:    v_writelane_b32 v39, s102, 63
+; MUBUF-NEXT:    s_add_i32 s32, s32, 0x40300
+; MUBUF-NEXT:    v_writelane_b32 v39, s102, 31
 ; MUBUF-NEXT:    buffer_store_dword v0, v1, s[0:3], s33 offen
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
 ; MUBUF-NEXT:    ;;#ASMSTART
@@ -2444,76 +1761,40 @@ define void @spill_fp_to_memory_scratch_reg_needed_mubuf_offset(ptr addrspace(5)
 ; MUBUF-NEXT:    ;;#ASMSTART
 ; MUBUF-NEXT:    ; clobber all VGPRs except CSR v40
 ; MUBUF-NEXT:    ;;#ASMEND
-; MUBUF-NEXT:    s_add_i32 s5, s33, 0x40200
-; MUBUF-NEXT:    buffer_load_dword v0, off, s[0:3], s5 ; 4-byte Folded Reload
-; MUBUF-NEXT:    s_add_i32 s32, s32, 0x40400
-; MUBUF-NEXT:    v_readlane_b32 s102, v39, 63
-; MUBUF-NEXT:    v_readlane_b32 s101, v39, 62
-; MUBUF-NEXT:    v_readlane_b32 s100, v39, 61
-; MUBUF-NEXT:    v_readlane_b32 s99, v39, 60
-; MUBUF-NEXT:    v_readlane_b32 s98, v39, 59
-; MUBUF-NEXT:    v_readlane_b32 s97, v39, 58
-; MUBUF-NEXT:    v_readlane_b32 s96, v39, 57
-; MUBUF-NEXT:    v_readlane_b32 s95, v39, 56
-; MUBUF-NEXT:    v_readlane_b32 s94, v39, 55
-; MUBUF-NEXT:    v_readlane_b32 s93, v39, 54
-; MUBUF-NEXT:    v_readlane_b32 s92, v39, 53
-; MUBUF-NEXT:    v_readlane_b32 s91, v39, 52
-; MUBUF-NEXT:    v_readlane_b32 s90, v39, 51
-; MUBUF-NEXT:    v_readlane_b32 s89, v39, 50
-; MUBUF-NEXT:    v_readlane_b32 s88, v39, 49
-; MUBUF-NEXT:    v_readlane_b32 s87, v39, 48
-; MUBUF-NEXT:    v_readlane_b32 s86, v39, 47
-; MUBUF-NEXT:    v_readlane_b32 s85, v39, 46
-; MUBUF-NEXT:    v_readlane_b32 s84, v39, 45
-; MUBUF-NEXT:    v_readlane_b32 s83, v39, 44
-; MUBUF-NEXT:    v_readlane_b32 s82, v39, 43
-; MUBUF-NEXT:    v_readlane_b32 s81, v39, 42
-; MUBUF-NEXT:    v_readlane_b32 s80, v39, 41
-; MUBUF-NEXT:    v_readlane_b32 s79, v39, 40
-; MUBUF-NEXT:    v_readlane_b32 s78, v39, 39
-; MUBUF-NEXT:    v_readlane_b32 s77, v39, 38
-; MUBUF-NEXT:    v_readlane_b32 s76, v39, 37
-; MUBUF-NEXT:    v_readlane_b32 s75, v39, 36
-; MUBUF-NEXT:    v_readlane_b32 s74, v39, 35
-; MUBUF-NEXT:    v_readlane_b32 s73, v39, 34
-; MUBUF-NEXT:    v_readlane_b32 s72, v39, 33
-; MUBUF-NEXT:    v_readlane_b32 s71, v39, 32
-; MUBUF-NEXT:    v_readlane_b32 s70, v39, 31
-; MUBUF-NEXT:    v_readlane_b32 s69, v39, 30
-; MUBUF-NEXT:    v_readlane_b32 s68, v39, 29
-; MUBUF-NEXT:    v_readlane_b32 s67, v39, 28
-; MUBUF-NEXT:    v_readlane_b32 s66, v39, 27
-; MUBUF-NEXT:    v_readlane_b32 s65, v39, 26
-; MUBUF-NEXT:    v_readlane_b32 s64, v39, 25
-; MUBUF-NEXT:    v_readlane_b32 s63, v39, 24
-; MUBUF-NEXT:    v_readlane_b32 s62, v39, 23
-; MUBUF-NEXT:    v_readlane_b32 s61, v39, 22
-; MUBUF-NEXT:    v_readlane_b32 s60, v39, 21
-; MUBUF-NEXT:    v_readlane_b32 s59, v39, 20
-; MUBUF-NEXT:    v_readlane_b32 s58, v39, 19
-; MUBUF-NEXT:    v_readlane_b32 s57, v39, 18
-; MUBUF-NEXT:    v_readlane_b32 s56, v39, 17
-; MUBUF-NEXT:    v_readlane_b32 s55, v39, 16
-; MUBUF-NEXT:    v_readlane_b32 s54, v39, 15
-; MUBUF-NEXT:    v_readlane_b32 s53, v39, 14
-; MUBUF-NEXT:    v_readlane_b32 s52, v39, 13
-; MUBUF-NEXT:    v_readlane_b32 s51, v39, 12
-; MUBUF-NEXT:    v_readlane_b32 s50, v39, 11
-; MUBUF-NEXT:    v_readlane_b32 s49, v39, 10
-; MUBUF-NEXT:    v_readlane_b32 s48, v39, 9
-; MUBUF-NEXT:    v_readlane_b32 s47, v39, 8
-; MUBUF-NEXT:    v_readlane_b32 s46, v39, 7
-; MUBUF-NEXT:    v_readlane_b32 s45, v39, 6
-; MUBUF-NEXT:    v_readlane_b32 s44, v39, 5
-; MUBUF-NEXT:    v_readlane_b32 s43, v39, 4
-; MUBUF-NEXT:    v_readlane_b32 s42, v39, 3
-; MUBUF-NEXT:    v_readlane_b32 s41, v39, 2
-; MUBUF-NEXT:    v_readlane_b32 s40, v39, 1
+; MUBUF-NEXT:    v_readlane_b32 s102, v39, 31
+; MUBUF-NEXT:    v_readlane_b32 s101, v39, 30
+; MUBUF-NEXT:    v_readlane_b32 s100, v39, 29
+; MUBUF-NEXT:    v_readlane_b32 s99, v39, 28
+; MUBUF-NEXT:    v_readlane_b32 s98, v39, 27
+; MUBUF-NEXT:    v_readlane_b32 s97, v39, 26
+; MUBUF-NEXT:    v_readlane_b32 s96, v39, 25
+; MUBUF-NEXT:    v_readlane_b32 s87, v39, 24
+; MUBUF-NEXT:    v_readlane_b32 s86, v39, 23
+; MUBUF-NEXT:    v_readlane_b32 s85, v39, 22
+; MUBUF-NEXT:    v_readlane_b32 s84, v39, 21
+; MUBUF-NEXT:    v_readlane_b32 s83, v39, 20
+; MUBUF-NEXT:    v_readlane_b32 s82, v39, 19
+; MUBUF-NEXT:    v_readlane_b32 s81, v39, 18
+; MUBUF-NEXT:    v_readlane_b32 s80, v39, 17
+; MUBUF-NEXT:    v_readlane_b32 s71, v39, 16
+; MUBUF-NEXT:    v_readlane_b32 s70, v39, 15
+; MUBUF-NEXT:    v_readlane_b32 s69, v39, 14
+; MUBUF-NEXT:    v_readlane_b32 s68, v39, 13
+; MUBUF-NEXT:    v_readlane_b32 s67, v39, 12
+; MUBUF-NEXT:    v_readlane_b32 s66, v39, 11
+; MUBUF-NEXT:    v_readlane_b32 s65, v39, 10
+; MUBUF-NEXT:    v_readlane_b32 s64, v39, 9
+; MUBUF-NEXT:    v_readlane_b32 s55, v39, 8
+; MUBUF-NEXT:    v_readlane_b32 s54, v39, 7
+; MUBUF-NEXT:    v_readlane_b32 s53, v39, 6
+; MUBUF-NEXT:    v_readlane_b32 s52, v39, 5
+; MUBUF-NEXT:    v_readlane_b32 s51, v39, 4
+; MUBUF-NEXT:    v_readlane_b32 s50, v39, 3
+; MUBUF-NEXT:    v_readlane_b32 s49, v39, 2
+; MUBUF-NEXT:    v_readlane_b32 s48, v39, 1
 ; MUBUF-NEXT:    v_readlane_b32 s39, v39, 0
 ; MUBUF-NEXT:    s_mov_b32 s32, s33
-; MUBUF-NEXT:    s_waitcnt vmcnt(0)
-; MUBUF-NEXT:    v_readfirstlane_b32 s4, v0
+; MUBUF-NEXT:    v_readlane_b32 s4, v39, 32
 ; MUBUF-NEXT:    s_xor_saveexec_b64 s[6:7], -1
 ; MUBUF-NEXT:    s_add_i32 s5, s33, 0x40100
 ; MUBUF-NEXT:    buffer_load_dword v39, off, s[0:3], s5 ; 4-byte Folded Reload
@@ -2532,72 +1813,40 @@ define void @spill_fp_to_memory_scratch_reg_needed_mubuf_offset(ptr addrspace(5)
 ; FLATSCR-NEXT:    scratch_store_dword off, v39, s1 ; 4-byte Folded Spill
 ; FLATSCR-NEXT:    s_mov_b64 exec, s[2:3]
 ; FLATSCR-NEXT:    v_writelane_b32 v39, s39, 0
-; FLATSCR-NEXT:    v_writelane_b32 v39, s40, 1
-; FLATSCR-NEXT:    v_writelane_b32 v39, s41, 2
-; FLATSCR-NEXT:    v_writelane_b32 v39, s42, 3
-; FLATSCR-NEXT:    v_writelane_b32 v39, s43, 4
-; FLATSCR-NEXT:    v_writelane_b32 v39, s44, 5
-; FLATSCR-NEXT:    v_writelane_b32 v39, s45, 6
-; FLATSCR-NEXT:    v_writelane_b32 v39, s46, 7
-; FLATSCR-NEXT:    v_writelane_b32 v39, s47, 8
-; FLATSCR-NEXT:    v_writelane_b32 v39, s48, 9
-; FLATSCR-NEXT:    v_writelane_b32 v39, s49, 10
-; FLATSCR-NEXT:    v_writelane_b32 v39, s50, 11
-; FLATSCR-NEXT:    v_writelane_b32 v39, s51, 12
-; FLATSCR-NEXT:    v_writelane_b32 v39, s52, 13
-; FLATSCR-NEXT:    v_writelane_b32 v39, s53, 14
-; FLATSCR-NEXT:    v_writelane_b32 v39, s54, 15
-; FLATSCR-NEXT:    v_writelane_b32 v39, s55, 16
-; FLATSCR-NEXT:    v_writelane_b32 v39, s56, 17
-; FLATSCR-NEXT:    v_writelane_b32 v39, s57, 18
-; FLATSCR-NEXT:    v_writelane_b32 v39, s58, 19
-; FLATSCR-NEXT:    v_writelane_b32 v39, s59, 20
-; FLATSCR-NEXT:    v_writelane_b32 v39, s60, 21
-; FLATSCR-NEXT:    v_writelane_b32 v39, s61, 22
-; FLATSCR-NEXT:    v_writelane_b32 v39, s62, 23
-; FLATSCR-NEXT:    v_writelane_b32 v39, s63, 24
-; FLATSCR-NEXT:    v_writelane_b32 v39, s64, 25
-; FLATSCR-NEXT:    v_writelane_b32 v39, s65, 26
-; FLATSCR-NEXT:    v_writelane_b32 v39, s66, 27
-; FLATSCR-NEXT:    v_writelane_b32 v39, s67, 28
-; FLATSCR-NEXT:    v_writelane_b32 v39, s68, 29
-; FLATSCR-NEXT:    v_writelane_b32 v39, s69, 30
-; FLATSCR-NEXT:    v_writelane_b32 v39, s70, 31
-; FLATSCR-NEXT:    v_writelane_b32 v39, s71, 32
-; FLATSCR-NEXT:    v_writelane_b32 v39, s72, 33
-; FLATSCR-NEXT:    v_writelane_b32 v39, s73, 34
-; FLATSCR-NEXT:    v_writelane_b32 v39, s74, 35
-; FLATSCR-NEXT:    v_writelane_b32 v39, s75, 36
-; FLATSCR-NEXT:    v_writelane_b32 v39, s76, 37
-; FLATSCR-NEXT:    v_writelane_b32 v39, s77, 38
-; FLATSCR-NEXT:    v_writelane_b32 v39, s78, 39
-; FLATSCR-NEXT:    v_writelane_b32 v39, s79, 40
-; FLATSCR-NEXT:    v_writelane_b32 v39, s80, 41
-; FLATSCR-NEXT:    v_writelane_b32 v39, s81, 42
-; FLATSCR-NEXT:    v_writelane_b32 v39, s82, 43
-; FLATSCR-NEXT:    v_writelane_b32 v39, s83, 44
-; FLATSCR-NEXT:    v_writelane_b32 v39, s84, 45
-; FLATSCR-NEXT:    v_writelane_b32 v39, s85, 46
-; FLATSCR-NEXT:    v_writelane_b32 v39, s86, 47
-; FLATSCR-NEXT:    v_writelane_b32 v39, s87, 48
-; FLATSCR-NEXT:    v_writelane_b32 v39, s88, 49
-; FLATSCR-NEXT:    v_writelane_b32 v39, s89, 50
-; FLATSCR-NEXT:    v_writelane_b32 v39, s90, 51
-; FLATSCR-NEXT:    v_writelane_b32 v39, s91, 52
-; FLATSCR-NEXT:    v_writelane_b32 v39, s92, 53
-; FLATSCR-NEXT:    v_writelane_b32 v39, s93, 54
-; FLATSCR-NEXT:    v_writelane_b32 v39, s94, 55
-; FLATSCR-NEXT:    v_writelane_b32 v39, s95, 56
-; FLATSCR-NEXT:    v_writelane_b32 v39, s96, 57
-; FLATSCR-NEXT:    v_writelane_b32 v39, s97, 58
-; FLATSCR-NEXT:    v_writelane_b32 v39, s98, 59
-; FLATSCR-NEXT:    v_writelane_b32 v39, s99, 60
+; FLATSCR-NEXT:    v_writelane_b32 v39, s48, 1
+; FLATSCR-NEXT:    v_writelane_b32 v39, s49, 2
+; FLATSCR-NEXT:    v_writelane_b32 v39, s50, 3
+; FLATSCR-NEXT:    v_writelane_b32 v39, s51, 4
+; FLATSCR-NEXT:    v_writelane_b32 v39, s52, 5
+; FLATSCR-NEXT:    v_writelane_b32 v39, s53, 6
+; FLATSCR-NEXT:    v_writelane_b32 v39, s54, 7
+; FLATSCR-NEXT:    v_writelane_b32 v39, s55, 8
+; FLATSCR-NEXT:    v_writelane_b32 v39, s64, 9
+; FLATSCR-NEXT:    v_writelane_b32 v39, s65, 10
+; FLATSCR-NEXT:    v_writelane_b32 v39, s66, 11
+; FLATSCR-NEXT:    v_writelane_b32 v39, s67, 12
+; FLATSCR-NEXT:    v_writelane_b32 v39, s68, 13
+; FLATSCR-NEXT:    v_writelane_b32 v39, s69, 14
+; FLATSCR-NEXT:    v_writelane_b32 v39, s70, 15
+; FLATSCR-NEXT:    v_writelane_b32 v39, s71, 16
+; FLATSCR-NEXT:    v_writelane_b32 v39, s80, 17
+; FLATSCR-NEXT:    v_writelane_b32 v39, s81, 18
+; FLATSCR-NEXT:    v_writelane_b32 v39, s82, 19
+; FLATSCR-NEXT:    v_writelane_b32 v39, s83, 20
+; FLATSCR-NEXT:    v_writelane_b32 v39, s84, 21
+; FLATSCR-NEXT:    v_writelane_b32 v39, s85, 22
+; FLATSCR-NEXT:    v_writelane_b32 v39, s86, 23
+; FLATSCR-NEXT:    v_writelane_b32 v39, s87, 24
+; FLATSCR-NEXT:    v_writelane_b32 v39, s96, 25
+; FLATSCR-NEXT:    v_writelane_b32 v39, s97, 26
+; FLATSCR-NEXT:    v_writelane_b32 v39, s98, 27
+; FLATSCR-NEXT:    v_writelane_b32 v39, s99, 28
 ; FLATSCR-NEXT:    s_addk_i32 s32, 0x100c
-; FLATSCR-NEXT:    v_writelane_b32 v39, s100, 61
-; FLATSCR-NEXT:    v_writelane_b32 v39, s101, 62
+; FLATSCR-NEXT:    v_writelane_b32 v39, s100, 29
+; FLATSCR-NEXT:    v_writelane_b32 v39, s101, 30
 ; FLATSCR-NEXT:    v_mov_b32_e32 v0, 0
 ; FLATSCR-NEXT:    s_add_i32 s1, s33, 0x1000
-; FLATSCR-NEXT:    v_writelane_b32 v39, s102, 63
+; FLATSCR-NEXT:    v_writelane_b32 v39, s102, 31
 ; FLATSCR-NEXT:    scratch_store_dword off, v0, s1
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    ;;#ASMSTART
@@ -2606,69 +1855,37 @@ define void @spill_fp_to_memory_scratch_reg_needed_mubuf_offset(ptr addrspace(5)
 ; FLATSCR-NEXT:    ;;#ASMSTART
 ; FLATSCR-NEXT:    ; clobber all VGPRs except CSR v40
 ; FLATSCR-NEXT:    ;;#ASMEND
-; FLATSCR-NEXT:    v_readlane_b32 s102, v39, 63
-; FLATSCR-NEXT:    v_readlane_b32 s101, v39, 62
-; FLATSCR-NEXT:    v_readlane_b32 s100, v39, 61
-; FLATSCR-NEXT:    v_readlane_b32 s99, v39, 60
-; FLATSCR-NEXT:    v_readlane_b32 s98, v39, 59
-; FLATSCR-NEXT:    v_readlane_b32 s97, v39, 58
-; FLATSCR-NEXT:    v_readlane_b32 s96, v39, 57
-; FLATSCR-NEXT:    v_readlane_b32 s95, v39, 56
-; FLATSCR-NEXT:    v_readlane_b32 s94, v39, 55
-; FLATSCR-NEXT:    v_readlane_b32 s93, v39, 54
-; FLATSCR-NEXT:    v_readlane_b32 s92, v39, 53
-; FLATSCR-NEXT:    v_readlane_b32 s91, v39, 52
-; FLATSCR-NEXT:    v_readlane_b32 s90, v39, 51
-; FLATSCR-NEXT:    v_readlane_b32 s89, v39, 50
-; FLATSCR-NEXT:    v_readlane_b32 s88, v39, 49
-; FLATSCR-NEXT:    v_readlane_b32 s87, v39, 48
-; FLATSCR-NEXT:    v_readlane_b32 s86, v39, 47
-; FLATSCR-NEXT:    v_readlane_b32 s85, v39, 46
-; FLATSCR-NEXT:    v_readlane_b32 s84, v39, 45
-; FLATSCR-NEXT:    v_readlane_b32 s83, v39, 44
-; FLATSCR-NEXT:    v_readlane_b32 s82, v39, 43
-; FLATSCR-NEXT:    v_readlane_b32 s81, v39, 42
-; FLATSCR-NEXT:    v_readlane_b32 s80, v39, 41
-; FLATSCR-NEXT:    v_readlane_b32 s79, v39, 40
-; FLATSCR-NEXT:    v_readlane_b32 s78, v39, 39
-; FLATSCR-NEXT:    v_readlane_b32 s77, v39, 38
-; FLATSCR-NEXT:    v_readlane_b32 s76, v39, 37
-; FLATSCR-NEXT:    v_readlane_b32 s75, v39, 36
-; FLATSCR-NEXT:    v_readlane_b32 s74, v39, 35
-; FLATSCR-NEXT:    v_readlane_b32 s73, v39, 34
-; FLATSCR-NEXT:    v_readlane_b32 s72, v39, 33
-; FLATSCR-NEXT:    v_readlane_b32 s71, v39, 32
-; FLATSCR-NEXT:    v_readlane_b32 s70, v39, 31
-; FLATSCR-NEXT:    v_readlane_b32 s69, v39, 30
-; FLATSCR-NEXT:    v_readlane_b32 s68, v39, 29
-; FLATSCR-NEXT:    v_readlane_b32 s67, v39, 28
-; FLATSCR-NEXT:    v_readlane_b32 s66, v39, 27
-; FLATSCR-NEXT:    v_readlane_b32 s65, v39, 26
-; FLATSCR-NEXT:    v_readlane_b32 s64, v39, 25
-; FLATSCR-NEXT:    v_readlane_b32 s63, v39, 24
-; FLATSCR-NEXT:    v_readlane_b32 s62, v39, 23
-; FLATSCR-NEXT:    v_readlane_b32 s61, v39, 22
-; FLATSCR-NEXT:    v_readlane_b32 s60, v39, 21
-; FLATSCR-NEXT:    v_readlane_b32 s59, v39, 20
-; FLATSCR-NEXT:    v_readlane_b32 s58, v39, 19
-; FLATSCR-NEXT:    v_readlane_b32 s57, v39, 18
-; FLATSCR-NEXT:    v_readlane_b32 s56, v39, 17
-; FLATSCR-NEXT:    v_readlane_b32 s55, v39, 16
-; FLATSCR-NEXT:    v_readlane_b32 s54, v39, 15
-; FLATSCR-NEXT:    v_readlane_b32 s53, v39, 14
-; FLATSCR-NEXT:    v_readlane_b32 s52, v39, 13
-; FLATSCR-NEXT:    v_readlane_b32 s51, v39, 12
-; FLATSCR-NEXT:    v_readlane_b32 s50, v39, 11
-; FLATSCR-NEXT:    v_readlane_b32 s49, v39, 10
-; FLATSCR-NEXT:    v_readlane_b32 s48, v39, 9
-; FLATSCR-NEXT:    v_readlane_b32 s47, v39, 8
-; FLATSCR-NEXT:    v_readlane_b32 s46, v39, 7
-; FLATSCR-NEXT:    v_readlane_b32 s45, v39, 6
-; FLATSCR-NEXT:    v_readlane_b32 s44, v39, 5
-; FLATSCR-NEXT:    v_readlane_b32 s43, v39, 4
-; FLATSCR-NEXT:    v_readlane_b32 s42, v39, 3
-; FLATSCR-NEXT:    v_readlane_b32 s41, v39, 2
-; FLATSCR-NEXT:    v_readlane_b32 s40, v39, 1
+; FLATSCR-NEXT:    v_readlane_b32 s102, v39, 31
+; FLATSCR-NEXT:    v_readlane_b32 s101, v39, 30
+; FLATSCR-NEXT:    v_readlane_b32 s100, v39, 29
+; FLATSCR-NEXT:    v_readlane_b32 s99, v39, 28
+; FLATSCR-NEXT:    v_readlane_b32 s98, v39, 27
+; FLATSCR-NEXT:    v_readlane_b32 s97, v39, 26
+; FLATSCR-NEXT:    v_readlane_b32 s96, v39, 25
+; FLATSCR-NEXT:    v_readlane_b32 s87, v39, 24
+; FLATSCR-NEXT:    v_readlane_b32 s86, v39, 23
+; FLATSCR-NEXT:    v_readlane_b32 s85, v39, 22
+; FLATSCR-NEXT:    v_readlane_b32 s84, v39, 21
+; FLATSCR-NEXT:    v_readlane_b32 s83, v39, 20
+; FLATSCR-NEXT:    v_readlane_b32 s82, v39, 19
+; FLATSCR-NEXT:    v_readlane_b32 s81, v39, 18
+; FLATSCR-NEXT:    v_readlane_b32 s80, v39, 17
+; FLATSCR-NEXT:    v_readlane_b32 s71, v39, 16
+; FLATSCR-NEXT:    v_readlane_b32 s70, v39, 15
+; FLATSCR-NEXT:    v_readlane_b32 s69, v39, 14
+; FLATSCR-NEXT:    v_readlane_b32 s68, v39, 13
+; FLATSCR-NEXT:    v_readlane_b32 s67, v39, 12
+; FLATSCR-NEXT:    v_readlane_b32 s66, v39, 11
+; FLATSCR-NEXT:    v_readlane_b32 s65, v39, 10
+; FLATSCR-NEXT:    v_readlane_b32 s64, v39, 9
+; FLATSCR-NEXT:    v_readlane_b32 s55, v39, 8
+; FLATSCR-NEXT:    v_readlane_b32 s54, v39, 7
+; FLATSCR-NEXT:    v_readlane_b32 s53, v39, 6
+; FLATSCR-NEXT:    v_readlane_b32 s52, v39, 5
+; FLATSCR-NEXT:    v_readlane_b32 s51, v39, 4
+; FLATSCR-NEXT:    v_readlane_b32 s50, v39, 3
+; FLATSCR-NEXT:    v_readlane_b32 s49, v39, 2
+; FLATSCR-NEXT:    v_readlane_b32 s48, v39, 1
 ; FLATSCR-NEXT:    v_readlane_b32 s39, v39, 0
 ; FLATSCR-NEXT:    s_mov_b32 s32, s33
 ; FLATSCR-NEXT:    s_xor_saveexec_b64 s[2:3], -1
diff --git a/llvm/test/CodeGen/AMDGPU/cc-sgpr-limit.ll b/llvm/test/CodeGen/AMDGPU/cc-sgpr-limit.ll
index fe173bf897539..e3fa683c55660 100644
--- a/llvm/test/CodeGen/AMDGPU/cc-sgpr-limit.ll
+++ b/llvm/test/CodeGen/AMDGPU/cc-sgpr-limit.ll
@@ -62,7 +62,7 @@ define amdgpu_gs { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i
   %56 =  add i32 %55, %27
   %57 =  add i32 %56, %28
   %58 =  add i32 %57, %29
-  %59 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } undef, i32 %30, 0
+  %59 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } poison, i32 %30, 0
   %60 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %59, i32 %31, 1
   %61 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %60, i32 %32, 2
   %62 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %61, i32 %33, 3
diff --git a/llvm/test/CodeGen/AMDGPU/cc-sgpr-over-limit.ll b/llvm/test/CodeGen/AMDGPU/cc-sgpr-over-limit.ll
index b427aee3fc370..8c34c12eed535 100644
--- a/llvm/test/CodeGen/AMDGPU/cc-sgpr-over-limit.ll
+++ b/llvm/test/CodeGen/AMDGPU/cc-sgpr-over-limit.ll
@@ -50,7 +50,7 @@ define amdgpu_gs { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i
   %88 =  add i32 %87, %43
   %89 =  add i32 %88, %44
   %90 =  add i32 %89, %45
-  %91 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } undef, i32 %46, 0
+  %91 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } poison, i32 %46, 0
   %92 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %91, i32 %47, 1
   %93 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %92, i32 %48, 2
   %94 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %93, i32 %49, 3
diff --git a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
index b6eca494fe690..55f62058ec7af 100644
--- a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
@@ -72,7 +72,7 @@ bb:
   %load_lo = load half, ptr addrspace(5) %gep_lo
   %load_hi = load half, ptr addrspace(5) null
 
-  %temp = insertelement <2 x half> undef, half %load_lo, i32 0
+  %temp = insertelement <2 x half> poison, half %load_lo, i32 0
   %result = insertelement <2 x half> %temp, half %load_hi, i32 1
 
   ret <2 x half> %result
@@ -133,7 +133,7 @@ bb:
   %load_lo = load half, ptr addrspace(5) %base_lo
   %load_hi = load half, ptr addrspace(5) %base_hi
 
-  %temp = insertelement <2 x half> undef, half %load_lo, i32 0
+  %temp = insertelement <2 x half> poison, half %load_lo, i32 0
   %result = insertelement <2 x half> %temp, half %load_hi, i32 1
 
   ret <2 x half> %result
@@ -197,7 +197,7 @@ bb:
   %arith_lo = fadd half %in, 1.0
   %load_hi = load half, ptr addrspace(5) %base
 
-  %temp = insertelement <2 x half> undef, half %arith_lo, i32 0
+  %temp = insertelement <2 x half> poison, half %arith_lo, i32 0
   %result = insertelement <2 x half> %temp, half %load_hi, i32 1
 
   ret <2 x half> %result
@@ -238,7 +238,7 @@ bb:
   %load_lo = load half, ptr addrspace(3) %gep_lo
   %load_hi = load half, ptr addrspace(3) null
 
-  %temp = insertelement <2 x half> undef, half %load_lo, i32 0
+  %temp = insertelement <2 x half> poison, half %load_lo, i32 0
   %result = insertelement <2 x half> %temp, half %load_hi, i32 1
 
   ret <2 x half> %result
@@ -275,7 +275,7 @@ bb:
   %load_lo = load half, ptr addrspace(3) %base_lo
   %load_hi = load half, ptr addrspace(3) %base_hi
 
-  %temp = insertelement <2 x half> undef, half %load_lo, i32 0
+  %temp = insertelement <2 x half> poison, half %load_lo, i32 0
   %result = insertelement <2 x half> %temp, half %load_hi, i32 1
 
   ret <2 x half> %result
@@ -334,7 +334,7 @@ bb:
   %load_lo = load half, ptr addrspace(1) %gep_lo
   %load_hi = load half, ptr addrspace(1) null
 
-  %temp = insertelement <2 x half> undef, half %load_lo, i32 0
+  %temp = insertelement <2 x half> poison, half %load_lo, i32 0
   %result = insertelement <2 x half> %temp, half %load_hi, i32 1
 
   ret <2 x half> %result
@@ -377,7 +377,7 @@ bb:
   %load_lo = load half, ptr addrspace(1) %base_lo
   %load_hi = load half, ptr addrspace(1) %base_hi
 
-  %temp = insertelement <2 x half> undef, half %load_lo, i32 0
+  %temp = insertelement <2 x half> poison, half %load_lo, i32 0
   %result = insertelement <2 x half> %temp, half %load_hi, i32 1
 
   ret <2 x half> %result
@@ -440,7 +440,7 @@ bb:
   %load_lo = load half, ptr %gep_lo
   %load_hi = load half, ptr null
 
-  %temp = insertelement <2 x half> undef, half %load_lo, i32 0
+  %temp = insertelement <2 x half> poison, half %load_lo, i32 0
   %result = insertelement <2 x half> %temp, half %load_hi, i32 1
 
   ret <2 x half> %result
@@ -486,7 +486,7 @@ bb:
   %load_lo = load half, ptr %base_lo
   %load_hi = load half, ptr %base_hi
 
-  %temp = insertelement <2 x half> undef, half %load_lo, i32 0
+  %temp = insertelement <2 x half> poison, half %load_lo, i32 0
   %result = insertelement <2 x half> %temp, half %load_hi, i32 1
 
   ret <2 x half> %result
@@ -740,7 +740,7 @@ bb:
   %gep_lo = getelementptr inbounds i16, ptr addrspace(3) %ptr, i64 1
   %load_lo = load i16, ptr addrspace(3) %gep_lo
   %load_hi = load i16, ptr addrspace(3) %ptr
-  %to.hi = insertelement <2 x i16> undef, i16 %load_hi, i32 1
+  %to.hi = insertelement <2 x i16> poison, i16 %load_hi, i32 1
   %op.hi = add <2 x i16> %to.hi, <i16 12, i16 12>
   %result = insertelement <2 x i16> %op.hi, i16 %load_lo, i32 0
   ret <2 x i16> %result
@@ -794,7 +794,7 @@ bb:
   %gep_lo = getelementptr inbounds i16, ptr addrspace(3) %ptr, i64 1
   %load_lo = load volatile i16, ptr addrspace(3) %gep_lo
   %load_hi = load volatile i16, ptr addrspace(3) %ptr
-  %to.hi = insertelement <2 x i16> undef, i16 %load_hi, i32 1
+  %to.hi = insertelement <2 x i16> poison, i16 %load_hi, i32 1
   %op.hi = add <2 x i16> %to.hi, <i16 12, i16 12>
   %result = insertelement <2 x i16> %op.hi, i16 %load_lo, i32 0
   ret <2 x i16> %result
@@ -859,7 +859,7 @@ bb:
   %gep_lo = getelementptr inbounds i16, ptr addrspace(5) %ptr, i64 1
   %load_lo = load i16, ptr addrspace(5) %gep_lo
   %load_hi = load i16, ptr addrspace(5) %ptr
-  %to.hi = insertelement <2 x i16> undef, i16 %load_hi, i32 1
+  %to.hi = insertelement <2 x i16> poison, i16 %load_hi, i32 1
   %op.hi = add <2 x i16> %to.hi, <i16 12, i16 12>
   %result = insertelement <2 x i16> %op.hi, i16 %load_lo, i32 0
   ret <2 x i16> %result
@@ -929,7 +929,7 @@ bb:
   %gep_lo = getelementptr inbounds i16, ptr addrspace(1) %ptr, i64 1
   %load_lo = load volatile i16, ptr addrspace(1) %gep_lo
   %load_hi = load volatile i16, ptr addrspace(1) %ptr
-  %to.hi = insertelement <2 x i16> undef, i16 %load_hi, i32 1
+  %to.hi = insertelement <2 x i16> poison, i16 %load_hi, i32 1
   %op.hi = add <2 x i16> %to.hi, <i16 12, i16 12>
   %result = insertelement <2 x i16> %op.hi, i16 %load_lo, i32 0
   ret <2 x i16> %result
@@ -1004,7 +1004,7 @@ bb:
   %gep_lo = getelementptr inbounds i16, ptr addrspace(0) %ptr, i64 1
   %load_lo = load volatile i16, ptr addrspace(0) %gep_lo
   %load_hi = load volatile i16, ptr addrspace(0) %ptr
-  %to.hi = insertelement <2 x i16> undef, i16 %load_hi, i32 1
+  %to.hi = insertelement <2 x i16> poison, i16 %load_hi, i32 1
   %op.hi = add <2 x i16> %to.hi, <i16 12, i16 12>
   %result = insertelement <2 x i16> %op.hi, i16 %load_lo, i32 0
   ret <2 x i16> %result
@@ -1073,7 +1073,7 @@ bb:
   store i16 123, ptr addrspace(3) %may.alias
   %load_lo = load i16, ptr addrspace(3) %gep_lo
 
-  %to.hi = insertelement <2 x i16> undef, i16 %load_hi, i32 1
+  %to.hi = insertelement <2 x i16> poison, i16 %load_hi, i32 1
   %result = insertelement <2 x i16> %to.hi, i16 %load_lo, i32 0
   ret <2 x i16> %result
 }
diff --git a/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll b/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
index 6571d515cfef2..49d6254dcd51a 100644
--- a/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
+++ b/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
@@ -1293,7 +1293,7 @@ define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm_shuf(ptr addrspace(1) %o
   %out.gep = getelementptr <2 x half>, ptr addrspace(1) %out, i32 %tid
   %a = load <2 x half>, ptr addrspace(1) %gep0
   %add = fadd <2 x half> %a, <half 1.0, half 1.0>
-  %shuf = shufflevector <2 x half> %add, <2 x half> undef, <2 x i32> <i32 1, i32 0>
+  %shuf = shufflevector <2 x half> %add, <2 x half> poison, <2 x i32> <i32 1, i32 0>
 
   %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %shuf, <2 x half> zeroinitializer)
   %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
diff --git a/llvm/test/CodeGen/AMDGPU/clamp.ll b/llvm/test/CodeGen/AMDGPU/clamp.ll
index 73ed23ab681f0..b3b7ce55a56e6 100644
--- a/llvm/test/CodeGen/AMDGPU/clamp.ll
+++ b/llvm/test/CodeGen/AMDGPU/clamp.ll
@@ -3728,7 +3728,7 @@ define amdgpu_kernel void @v_clamp_v2f16_shuffle(ptr addrspace(1) %out, ptr addr
   %gep0 = getelementptr <2 x half>, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr <2 x half>, ptr addrspace(1) %out, i32 %tid
   %a = load <2 x half>, ptr addrspace(1) %gep0
-  %shuf = shufflevector <2 x half> %a, <2 x half> undef, <2 x i32> <i32 1, i32 0>
+  %shuf = shufflevector <2 x half> %a, <2 x half> poison, <2 x i32> <i32 1, i32 0>
   %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %shuf, <2 x half> zeroinitializer)
   %med = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
 
diff --git a/llvm/test/CodeGen/AMDGPU/coalesce-copy-to-agpr-to-av-registers.mir b/llvm/test/CodeGen/AMDGPU/coalesce-copy-to-agpr-to-av-registers.mir
new file mode 100644
index 0000000000000..e3346918f8fee
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/coalesce-copy-to-agpr-to-av-registers.mir
@@ -0,0 +1,519 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -run-pass=register-coalescer -verify-coalescing -o - %s | FileCheck %s
+
+# Test coalescing situations which can use av_* registers to handle
+# copies between VGPRs and AGPRs.
+
+
+# Should coalesce %0 and %1 into subregisters of the av_64 common
+# class
+---
+name: copy_vgpr32_to_areg64_coalesce_with_av64
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: copy_vgpr32_to_areg64_coalesce_with_av64
+    ; CHECK: liveins: $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; CHECK-NEXT: undef [[COPY2:%[0-9]+]].sub0:areg_64 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]].sub1:areg_64 = COPY [[COPY1]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 3473417 /* reguse:AReg_64 */, [[COPY2]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    undef %2.sub0:areg_64 = COPY %0
+    %2.sub1:areg_64 = COPY %1
+    INLINEASM &"; use $0", 0 /* attdialect */, 3473417 /* reguse:AReg_64 */, killed %2
+    SI_RETURN
+
+...
+
+---
+name: copy_vgpr32_to_areg64_align2_coalesce_with_av64_align2
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: copy_vgpr32_to_areg64_align2_coalesce_with_av64_align2
+    ; CHECK: liveins: $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; CHECK-NEXT: undef [[COPY2:%[0-9]+]].sub0:areg_64_align2 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]].sub1:areg_64_align2 = COPY [[COPY1]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 3735561 /* reguse:AReg_64_Align2 */, [[COPY2]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    undef %2.sub0:areg_64_align2 = COPY %0
+    %2.sub1:areg_64_align2 = COPY %1
+    INLINEASM &"; use $0", 0 /* attdialect */, 3735561 /* reguse:AReg_64_Align2 */, %2
+    SI_RETURN
+
+...
+
+---
+name: copy_vgpr32_to_areg96_coalesce_with_av96
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    ; CHECK-LABEL: name: copy_vgpr32_to_areg96_coalesce_with_av96
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; CHECK-NEXT: undef [[COPY3:%[0-9]+]].sub0:areg_96 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]].sub1:areg_96 = COPY [[COPY1]]
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]].sub2:areg_96 = COPY [[COPY2]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 4587529 /* reguse:AReg_96 */, [[COPY3]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    %2:vgpr_32 = COPY $vgpr2
+    undef %3.sub0:areg_96 = COPY %0
+    %3.sub1:areg_96 = COPY %1
+    %3.sub2:areg_96 = COPY %2
+    INLINEASM &"; use $0", 0 /* attdialect */, 4587529 /* reguse:AReg_96 */, %3
+    SI_RETURN
+
+...
+
+---
+name: copy_vgpr32_to_areg96_coalesce_with_av96_align2
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    ; CHECK-LABEL: name: copy_vgpr32_to_areg96_coalesce_with_av96_align2
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; CHECK-NEXT: undef [[COPY3:%[0-9]+]].sub0:areg_96_align2 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]].sub1:areg_96_align2 = COPY [[COPY1]]
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]].sub2:areg_96_align2 = COPY [[COPY2]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 4915209 /* reguse:AReg_96_Align2 */, [[COPY3]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    %2:vgpr_32 = COPY $vgpr2
+    undef %3.sub0:areg_96_align2 = COPY %0
+    %3.sub1:areg_96_align2 = COPY %1
+    %3.sub2:areg_96_align2 = COPY %2
+    INLINEASM &"; use $0", 0 /* attdialect */, 4915209 /* reguse:AReg_96_Align2 */, %3
+    SI_RETURN
+
+...
+
+---
+name: copy_vgpr64_to_areg64_coalesce_with_av128
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: copy_vgpr64_to_areg64_coalesce_with_av128
+    ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: undef [[COPY2:%[0-9]+]].sub0_sub1:areg_128 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]].sub2_sub3:areg_128 = COPY [[COPY1]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 6225929 /* reguse:AReg_128 */, [[COPY2]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64 = COPY $vgpr0_vgpr1
+    %1:vreg_64 = COPY $vgpr2_vgpr3
+    undef %2.sub0_sub1:areg_128 = COPY %0
+    %2.sub2_sub3:areg_128 = COPY %1
+    INLINEASM &"; use $0", 0 /* attdialect */, 6225929 /* reguse:AReg_128 */, killed %2
+    SI_RETURN
+
+...
+
+---
+name: copy_vgpr64_to_areg64_align2_coalesce_with_av128_align2
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: copy_vgpr64_to_areg64_align2_coalesce_with_av128_align2
+    ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: undef [[COPY2:%[0-9]+]].sub0_sub1:areg_128_align2 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]].sub2_sub3:areg_128_align2 = COPY [[COPY1]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 6488073 /* reguse:AReg_128_Align2 */, [[COPY2]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64 = COPY $vgpr0_vgpr1
+    %1:vreg_64 = COPY $vgpr2_vgpr3
+    undef %2.sub0_sub1:areg_128_align2 = COPY %0
+    %2.sub2_sub3:areg_128_align2 = COPY %1
+    INLINEASM &"; use $0", 0 /* attdialect */, 6488073 /* reguse:AReg_128_Align2 */, %2
+    SI_RETURN
+
+...
+
+---
+name: copy_sgpr32_to_areg64_align2
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr8, $sgpr9
+
+    ; CHECK-LABEL: name: copy_sgpr32_to_areg64_align2
+    ; CHECK: liveins: $sgpr8, $sgpr9
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr8
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr9
+    ; CHECK-NEXT: undef [[COPY2:%[0-9]+]].sub0:areg_64_align2 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]].sub1:areg_64_align2 = COPY [[COPY1]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 3735561 /* reguse:AReg_64_Align2 */, [[COPY2]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:sgpr_32 = COPY $sgpr8
+    %1:sgpr_32 = COPY $sgpr9
+    undef %2.sub0:areg_64_align2 = COPY %0
+    %2.sub1:areg_64_align2 = COPY %1
+    INLINEASM &"; use $0", 0 /* attdialect */, 3735561 /* reguse:AReg_64_Align2 */, %2
+    SI_RETURN
+
+...
+
+---
+name: copy_vgpr32_vgpr64_to_areg96_coalesce_with_av96
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: copy_vgpr32_vgpr64_to_areg96_coalesce_with_av96
+    ; CHECK: liveins: $vgpr0, $vgpr1_vgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+    ; CHECK-NEXT: undef [[COPY2:%[0-9]+]].sub0:areg_96 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]].sub1_sub2:areg_96 = COPY [[COPY1]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 4587529 /* reguse:AReg_96 */, [[COPY2]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vreg_64 = COPY $vgpr1_vgpr2
+    undef %2.sub0:areg_96 = COPY %0
+    %2.sub1_sub2:areg_96 = COPY %1
+    INLINEASM &"; use $0", 0 /* attdialect */, 4587529 /* reguse:AReg_96 */, %2
+    SI_RETURN
+
+...
+
+---
+name: copy_vgpr32_vgpr64_to_areg96_coalesce_with_av96_align2
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: copy_vgpr32_vgpr64_to_areg96_coalesce_with_av96_align2
+    ; CHECK: liveins: $vgpr0, $vgpr1_vgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr1_vgpr2
+    ; CHECK-NEXT: undef [[COPY2:%[0-9]+]].sub0:areg_96_align2 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]].sub1_sub2:areg_96_align2 = COPY [[COPY1]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 3735561 /* reguse:AReg_64_Align2 */, [[COPY2]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vreg_64 = COPY $vgpr1_vgpr2
+    undef %2.sub0:areg_96_align2 = COPY %0
+    %2.sub1_sub2:areg_96_align2 = COPY %1
+    INLINEASM &"; use $0", 0 /* attdialect */, 3735561 /* reguse:AReg_64_Align2 */, %2
+    SI_RETURN
+
+...
+
+---
+name: copy_vgpr64_vgpr32_to_areg96_coalesce_with_av96
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    ; CHECK-LABEL: name: copy_vgpr64_vgpr32_to_areg96_coalesce_with_av96
+    ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; CHECK-NEXT: undef [[COPY2:%[0-9]+]].sub0_sub1:areg_96 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]].sub2:areg_96 = COPY [[COPY1]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 4587529 /* reguse:AReg_96 */, [[COPY2]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64 = COPY $vgpr0_vgpr1
+    %1:vgpr_32 = COPY $vgpr2
+    undef %2.sub0_sub1:areg_96 = COPY %0
+    %2.sub2:areg_96 = COPY %1
+    INLINEASM &"; use $0", 0 /* attdialect */, 4587529 /* reguse:AReg_96 */, %2
+    SI_RETURN
+
+...
+
+---
+name: copy_vgpr64_vgpr32_to_areg96_coalesce_with_av96_align2
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: copy_vgpr64_vgpr32_to_areg96_coalesce_with_av96_align2
+    ; CHECK: liveins: $vgpr0, $vgpr1_vgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; CHECK-NEXT: undef [[COPY2:%[0-9]+]].sub0_sub1:areg_96_align2 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]].sub2:areg_96_align2 = COPY [[COPY1]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 3735561 /* reguse:AReg_64_Align2 */, [[COPY2]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64 = COPY $vgpr0_vgpr1
+    %1:vgpr_32 = COPY $vgpr2
+    undef %2.sub0_sub1:areg_96_align2 = COPY %0
+    %2.sub2:areg_96_align2 = COPY %1
+    INLINEASM &"; use $0", 0 /* attdialect */, 3735561 /* reguse:AReg_64_Align2 */, %2
+    SI_RETURN
+
+...
+
+---
+name: copy_vgpr32_x2_to_areg64
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: copy_vgpr32_x2_to_areg64
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK-NEXT: undef [[COPY1:%[0-9]+]].sub0:areg_64 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]].sub1:areg_64 = COPY [[COPY]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 3473417 /* reguse:AReg_64 */, [[COPY1]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vgpr_32 = COPY $vgpr0
+    undef %2.sub0:areg_64 = COPY %0
+    %2.sub1:areg_64 = COPY %0
+    INLINEASM &"; use $0", 0 /* attdialect */, 3473417 /* reguse:AReg_64 */, killed %2
+    SI_RETURN
+
+...
+
+---
+name: copy_vgpr32_x2_to_areg64_coalesce_with_av64_align2
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: copy_vgpr32_x2_to_areg64_coalesce_with_av64_align2
+    ; CHECK: liveins: $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; CHECK-NEXT: undef [[COPY2:%[0-9]+]].sub0:areg_64_align2 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]].sub1:areg_64_align2 = COPY [[COPY1]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 3735561 /* reguse:AReg_64_Align2 */, [[COPY2]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    undef %2.sub0:areg_64_align2 = COPY %0
+    %2.sub1:areg_64_align2 = COPY %1
+    INLINEASM &"; use $0", 0 /* attdialect */, 3735561 /* reguse:AReg_64_Align2 */, %2
+    SI_RETURN
+
+...
+
+---
+name: copy_vgpr32_x3_to_areg96
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: copy_vgpr32_x3_to_areg96
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK-NEXT: undef [[COPY1:%[0-9]+]].sub0:areg_96 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]].sub1:areg_96 = COPY [[COPY]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 4587529 /* reguse:AReg_96 */, [[COPY1]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vgpr_32 = COPY $vgpr0
+    undef %1.sub0:areg_96 = COPY %0
+    %1.sub1:areg_96 = COPY %0
+    INLINEASM &"; use $0", 0 /* attdialect */, 4587529 /* reguse:AReg_96 */, %1
+    SI_RETURN
+
+...
+
+---
+name: copy_vgpr32_x3_to_areg96_align2
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: copy_vgpr32_x3_to_areg96_align2
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK-NEXT: undef [[COPY1:%[0-9]+]].sub0:areg_96_align2 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]].sub1:areg_96_align2 = COPY [[COPY]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 4915209 /* reguse:AReg_96_Align2 */, [[COPY1]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vgpr_32 = COPY $vgpr0
+    undef %1.sub0:areg_96_align2 = COPY %0
+    %1.sub1:areg_96_align2 = COPY %0
+    INLINEASM &"; use $0", 0 /* attdialect */, 4915209 /* reguse:AReg_96_Align2 */, %1
+    SI_RETURN
+
+...
+
+---
+name: copy_vgpr32_x4_to_areg128
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: copy_vgpr32_x4_to_areg128
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK-NEXT: undef [[COPY1:%[0-9]+]].sub0:areg_128 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]].sub1:areg_128 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]].sub2:areg_128 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]].sub3:areg_128 = COPY [[COPY]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 6225929 /* reguse:AReg_128 */, [[COPY1]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vgpr_32 = COPY $vgpr0
+    undef %1.sub0:areg_128 = COPY %0
+    %1.sub1:areg_128 = COPY %0
+    %1.sub2:areg_128 = COPY %0
+    %1.sub3:areg_128 = COPY %0
+    INLINEASM &"; use $0", 0 /* attdialect */, 6225929 /* reguse:AReg_128 */, killed %1
+    SI_RETURN
+
+...
+
+---
+name: copy_vgpr32_x4_to_areg128_align2
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: copy_vgpr32_x4_to_areg128_align2
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK-NEXT: undef [[COPY1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]].sub1:areg_128_align2 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]].sub2:areg_128_align2 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]].sub3:areg_128_align2 = COPY [[COPY]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 6488073 /* reguse:AReg_128_Align2 */, [[COPY1]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vgpr_32 = COPY $vgpr0
+    undef %1.sub0:areg_128_align2 = COPY %0
+    %1.sub1:areg_128_align2 = COPY %0
+    %1.sub2:areg_128_align2 = COPY %0
+    %1.sub3:areg_128_align2 = COPY %0
+    INLINEASM &"; use $0", 0 /* attdialect */, 6488073 /* reguse:AReg_128_Align2 */, %1
+    SI_RETURN
+
+...
+
+---
+name: copy_vgpr32_to_areg64_coalesce_with_av64_other_v_use
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: copy_vgpr32_to_areg64_coalesce_with_av64_other_v_use
+    ; CHECK: liveins: $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; CHECK-NEXT: undef [[COPY2:%[0-9]+]].sub0:areg_64 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]].sub1:areg_64 = COPY [[COPY1]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 3473417 /* reguse:AReg_64 */, [[COPY2]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 2228233 /* reguse:VGPR_32 */, [[COPY]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    undef %2.sub0:areg_64 = COPY %0
+    %2.sub1:areg_64 = COPY %1
+    INLINEASM &"; use $0", 0 /* attdialect */, 3473417 /* reguse:AReg_64 */, killed %2
+    INLINEASM &"; use $0", 0 /* attdialect */, 2228233 /* reguse:VGPR_32 */, killed %0
+    SI_RETURN
+
+...
+
+---
+name: copy_vgpr32_x2_to_areg64_coalesce_with_av64_other_v_use
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: copy_vgpr32_x2_to_areg64_coalesce_with_av64_other_v_use
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK-NEXT: undef [[COPY1:%[0-9]+]].sub0:areg_64 = COPY [[COPY]]
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]].sub1:areg_64 = COPY [[COPY]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 3473417 /* reguse:AReg_64 */, [[COPY1]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 2228233 /* reguse:VGPR_32 */, [[COPY]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vgpr_32 = COPY $vgpr0
+    undef %1.sub0:areg_64 = COPY %0
+    %1.sub1:areg_64 = COPY %0
+    INLINEASM &"; use $0", 0 /* attdialect */, 3473417 /* reguse:AReg_64 */, killed %1
+    INLINEASM &"; use $0", 0 /* attdialect */, 2228233 /* reguse:VGPR_32 */, killed %0
+    SI_RETURN
+
+...
+
+---
+name: copy_vgpr32_x2_to_areg64_and_vreg64_coalesce_with_av64_other_v_use
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: copy_vgpr32_x2_to_areg64_and_vreg64_coalesce_with_av64_other_v_use
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: undef [[COPY:%[0-9]+]].sub0:vreg_64 = COPY $vgpr0
+    ; CHECK-NEXT: undef [[COPY1:%[0-9]+]].sub0:areg_64 = COPY [[COPY]].sub0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]].sub1:areg_64 = COPY [[COPY]].sub0
+    ; CHECK-NEXT: [[COPY:%[0-9]+]].sub1:vreg_64 = COPY [[COPY]].sub0
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 3473417 /* reguse:AReg_64 */, [[COPY1]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 0 /* attdialect */, 3538953 /* reguse:VReg_64 */, [[COPY]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vgpr_32 = COPY $vgpr0
+    undef %1.sub0:areg_64 = COPY %0
+    %1.sub1:areg_64 = COPY %0
+    undef %2.sub0:vreg_64 = COPY %0
+    %2.sub1:vreg_64 = COPY %0
+    INLINEASM &"; use $0", 0 /* attdialect */, 3473417 /* reguse:AReg_64 */, killed %1
+    INLINEASM &"; use $0", 0 /* attdialect */, 3538953 /* reguse:VReg_64 */, killed %2
+    SI_RETURN
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/coalesce-vgpr-alignment.ll b/llvm/test/CodeGen/AMDGPU/coalesce-vgpr-alignment.ll
index dcd088e2bd988..380f14b125ead 100644
--- a/llvm/test/CodeGen/AMDGPU/coalesce-vgpr-alignment.ll
+++ b/llvm/test/CodeGen/AMDGPU/coalesce-vgpr-alignment.ll
@@ -14,7 +14,7 @@ bb:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep1 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg, i32 %lid
   %load = load <4 x i32>, ptr addrspace(1) %gep1, align 16
-  %shuffle = shufflevector <4 x i32> %load, <4 x i32> undef, <2 x i32> <i32 1, i32 3>
+  %shuffle = shufflevector <4 x i32> %load, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
   %gep2 = getelementptr inbounds <2 x i32>, ptr addrspace(1) %arg1, i32 %lid
   store <2 x i32> %shuffle, ptr addrspace(1) %gep2, align 8
   ret void
@@ -27,9 +27,9 @@ bb:
 define amdgpu_kernel void @test_vector_creation() #0 {
 entry:
   %tmp231 = load <4 x i16>, ptr addrspace(1) undef, align 2
-  %vext466 = shufflevector <4 x i16> %tmp231, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-  %vecinit467 = shufflevector <8 x i16> undef, <8 x i16> %vext466, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef>
-  %vecinit471 = shufflevector <8 x i16> %vecinit467, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+  %vext466 = shufflevector <4 x i16> %tmp231, <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+  %vecinit467 = shufflevector <8 x i16> poison, <8 x i16> %vext466, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison>
+  %vecinit471 = shufflevector <8 x i16> %vecinit467, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
   store <8 x i16> %vecinit471, ptr addrspace(1) undef, align 16
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/coalescer_distribute.ll b/llvm/test/CodeGen/AMDGPU/coalescer_distribute.ll
index 2d8ef336f52d6..8f4d10411ed73 100644
--- a/llvm/test/CodeGen/AMDGPU/coalescer_distribute.ll
+++ b/llvm/test/CodeGen/AMDGPU/coalescer_distribute.ll
@@ -15,7 +15,7 @@ bb6:
   br label %bb8
 
 bb8:
-  %tmp9 = phi i64 [ %tmp7, %bb6 ], [ undef, %bb2 ]
+  %tmp9 = phi i64 [ %tmp7, %bb6 ], [ poison, %bb2 ]
   %tmp10 = icmp eq i32 %tmp, 0
   br i1 %tmp10, label %bb11, label %bb23
 
@@ -26,16 +26,16 @@ bb17:
   br label %bb20
 
 bb20:
-  %tmp21 = phi i64 [ undef, %bb17 ], [ %tmp9, %bb11 ]
+  %tmp21 = phi i64 [ poison, %bb17 ], [ %tmp9, %bb11 ]
   %tmp22 = trunc i64 %tmp21 to i32
   br label %bb23
 
 bb23:
-  %tmp24 = phi i32 [ %tmp22, %bb20 ], [ undef, %bb8 ], [ undef, %bb ]
+  %tmp24 = phi i32 [ %tmp22, %bb20 ], [ poison, %bb8 ], [ poison, %bb ]
   br label %bb25
 
 bb25:
-  %tmp26 = phi i32 [ %tmp24, %bb23 ], [ undef, %bb25 ]
+  %tmp26 = phi i32 [ %tmp24, %bb23 ], [ poison, %bb25 ]
   br i1 %c3, label %bb25, label %bb30
 
 bb30:
diff --git a/llvm/test/CodeGen/AMDGPU/coalescer_remat.ll b/llvm/test/CodeGen/AMDGPU/coalescer_remat.ll
index 12057caec6e84..61830f18ad7a7 100644
--- a/llvm/test/CodeGen/AMDGPU/coalescer_remat.ll
+++ b/llvm/test/CodeGen/AMDGPU/coalescer_remat.ll
@@ -26,7 +26,7 @@ loop:
   %v3 = phi float [0.0, %entry], [%fma.3, %loop]
 
   ; Try to get the 0 constant to get coalesced into a wide register
-  %blup = insertelement <4 x float> undef, float %v0, i32 0
+  %blup = insertelement <4 x float> poison, float %v0, i32 0
   store <4 x float> %blup, ptr addrspace(1) %out
 
   %load = load <4 x float>, ptr addrspace(1) %in
@@ -48,7 +48,7 @@ exit:
   %ev1 = phi float [0.0, %entry], [%fma.1, %loop]
   %ev2 = phi float [0.0, %entry], [%fma.2, %loop]
   %ev3 = phi float [0.0, %entry], [%fma.3, %loop]
-  %dst.0 = insertelement <4 x float> undef,  float %ev0, i32 0
+  %dst.0 = insertelement <4 x float> poison,  float %ev0, i32 0
   %dst.1 = insertelement <4 x float> %dst.0, float %ev1, i32 1
   %dst.2 = insertelement <4 x float> %dst.1, float %ev2, i32 2
   %dst.3 = insertelement <4 x float> %dst.2, float %ev3, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll b/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
index a47ecb2c5d7f2..c8a4f2d2e6b7b 100644
--- a/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
+++ b/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
@@ -1365,7 +1365,7 @@ bb4:                                              ; preds = %bb2
   br i1 %tmp7, label %bb8, label %Flow
 
 bb8:                                              ; preds = %bb4
-  %tmp9 = insertelement <4 x float> undef, float 0.0, i32 1
+  %tmp9 = insertelement <4 x float> poison, float 0.0, i32 1
   br label %Flow
 
 Flow:                                             ; preds = %bb8, %bb4
diff --git a/llvm/test/CodeGen/AMDGPU/combine-add-zext-xor.ll b/llvm/test/CodeGen/AMDGPU/combine-add-zext-xor.ll
index a25078230385e..b42542db6dbd8 100644
--- a/llvm/test/CodeGen/AMDGPU/combine-add-zext-xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine-add-zext-xor.ll
@@ -71,7 +71,7 @@ bb:                                               ; preds = %.a
   br label %bb9
 
 bb9:                                              ; preds = %bb, %.a
-  %.2.0.in.in = phi i1 [ %i5, %bb ], [ undef, %.a ]
+  %.2.0.in.in = phi i1 [ %i5, %bb ], [ poison, %.a ]
   %.2.0.in = xor i1 %.2.0.in.in, true
   %.2.0 = zext i1 %.2.0.in to i32
   %i11 = add i32 %.2, %.2.0
@@ -151,7 +151,7 @@ bb:                                               ; preds = %.a
   br label %bb9
 
 bb9:                                              ; preds = %bb, %.a
-  %.2.0.in.in = phi i1 [ %i5, %bb ], [ undef, %.a ]
+  %.2.0.in.in = phi i1 [ %i5, %bb ], [ poison, %.a ]
   %.2.0.in = xor i1 %.2.0.in.in, true
   %.2.0 = zext i1 %.2.0.in to i32
   %i11 = sub i32 %.2, %.2.0
@@ -234,7 +234,7 @@ bb:                                               ; preds = %.a
   br label %bb9
 
 bb9:                                              ; preds = %bb, %.a
-  %.2.0.in.in = phi i1 [ %i5, %bb ], [ undef, %.a ]
+  %.2.0.in.in = phi i1 [ %i5, %bb ], [ poison, %.a ]
   %t = icmp sgt i32 %.2, -1050
   %.2.0.in = or i1 %.2.0.in.in, %t
   %.2.0 = zext i1 %.2.0.in to i32
@@ -318,7 +318,7 @@ bb:                                               ; preds = %.a
   br label %bb9
 
 bb9:                                              ; preds = %bb, %.a
-  %.2.0.in.in = phi i1 [ %i5, %bb ], [ undef, %.a ]
+  %.2.0.in.in = phi i1 [ %i5, %bb ], [ poison, %.a ]
   %t = icmp sgt i32 %.2, -1050
   %.2.0.in = or i1 %.2.0.in.in, %t
   %.2.0 = zext i1 %.2.0.in to i32
@@ -397,7 +397,7 @@ bb:                                               ; preds = %.a
   br label %bb9
 
 bb9:                                              ; preds = %bb, %.a
-  %.2.0.in.in = phi i1 [ %i5, %bb ], [ undef, %.a ]
+  %.2.0.in.in = phi i1 [ %i5, %bb ], [ poison, %.a ]
   %t = icmp sgt i32 %.2, -1050
   %.2.0.in = and i1 %.2.0.in.in, %t
   %.2.0 = zext i1 %.2.0.in to i32
@@ -476,7 +476,7 @@ bb:                                               ; preds = %.a
   br label %bb9
 
 bb9:                                              ; preds = %bb, %.a
-  %.2.0.in.in = phi i1 [ %i5, %bb ], [ undef, %.a ]
+  %.2.0.in.in = phi i1 [ %i5, %bb ], [ poison, %.a ]
   %t = icmp sgt i32 %.2, -1050
   %.2.0.in = and i1 %.2.0.in.in, %t
   %.2.0 = zext i1 %.2.0.in to i32
diff --git a/llvm/test/CodeGen/AMDGPU/combine_vloads.ll b/llvm/test/CodeGen/AMDGPU/combine_vloads.ll
index 42a9b80b134cc..e2e37047912a1 100644
--- a/llvm/test/CodeGen/AMDGPU/combine_vloads.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine_vloads.ll
@@ -23,12 +23,12 @@ for.body:                                         ; preds = %for.body, %entry
   %i.01 = phi i32 [ 0, %entry ], [ %tmp19, %for.body ]
   %vecload2 = load <8 x i32>, ptr addrspace(1) %src, align 32
   %0 = bitcast <8 x i32> %vecload2 to <32 x i8>
-  %tmp5 = shufflevector <32 x i8> %0, <32 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-  %tmp8 = shufflevector <32 x i8> %0, <32 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %tmp5 = shufflevector <32 x i8> %0, <32 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %tmp8 = shufflevector <32 x i8> %0, <32 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %tmp9 = add nsw <8 x i8> %tmp5, %tmp8
-  %tmp12 = shufflevector <32 x i8> %0, <32 x i8> undef, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+  %tmp12 = shufflevector <32 x i8> %0, <32 x i8> poison, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
   %tmp13 = add nsw <8 x i8> %tmp9, %tmp12
-  %tmp16 = shufflevector <32 x i8> %0, <32 x i8> undef, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %tmp16 = shufflevector <32 x i8> %0, <32 x i8> poison, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
   %tmp17 = add nsw <8 x i8> %tmp13, %tmp16
   %scevgep = getelementptr <8 x i8>, ptr addrspace(1) %result, i32 %i.01
   %1 = bitcast <8 x i8> %tmp17 to <2 x i32>
diff --git a/llvm/test/CodeGen/AMDGPU/complex-folding.ll b/llvm/test/CodeGen/AMDGPU/complex-folding.ll
index cf19b92e3fd8c..3d124e8cb7f4a 100644
--- a/llvm/test/CodeGen/AMDGPU/complex-folding.ll
+++ b/llvm/test/CodeGen/AMDGPU/complex-folding.ll
@@ -8,7 +8,7 @@ entry:
   %1 = call float @fabsf(float %0)
   %2 = fptoui float %1 to i32
   %3 = bitcast i32 %2 to float
-  %4 = insertelement <4 x float> undef, float %3, i32 0
+  %4 = insertelement <4 x float> poison, float %3, i32 0
   call void @llvm.r600.store.swizzle(<4 x float> %4, i32 0, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/computeKnownBits-scalar-to-vector-crash.ll b/llvm/test/CodeGen/AMDGPU/computeKnownBits-scalar-to-vector-crash.ll
index 350997c9d8674..e15e701f9e805 100644
--- a/llvm/test/CodeGen/AMDGPU/computeKnownBits-scalar-to-vector-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/computeKnownBits-scalar-to-vector-crash.ll
@@ -3,7 +3,7 @@
 ; CHECK: s_waitcnt
 define <2 x i16> @main(<2 x float>) #0 {
   %2 = bitcast <2 x float> %0 to <4 x i16>
-  %3 = shufflevector <4 x i16> %2, <4 x i16> undef, <2 x i32> <i32 0, i32 undef>
+  %3 = shufflevector <4 x i16> %2, <4 x i16> poison, <2 x i32> <i32 0, i32 poison>
   %4 = extractelement <4 x i16> %2, i32 0
   %5 = insertelement <2 x i16> %3, i16 %4, i32 0
   ret <2 x i16> %5
diff --git a/llvm/test/CodeGen/AMDGPU/concat_vectors.ll b/llvm/test/CodeGen/AMDGPU/concat_vectors.ll
index 2998bd3475cbe..a9e4c8991bb2b 100644
--- a/llvm/test/CodeGen/AMDGPU/concat_vectors.ll
+++ b/llvm/test/CodeGen/AMDGPU/concat_vectors.ll
@@ -289,8 +289,8 @@ define amdgpu_kernel void @test_concat_v16i16(ptr addrspace(1) %out, <16 x i16>
 define amdgpu_kernel void @concat_vector_crash(ptr addrspace(1) %out, ptr addrspace(1) %in) {
 bb:
   %tmp = load <2 x float>, ptr addrspace(1) %in, align 4
-  %tmp1 = shufflevector <2 x float> %tmp, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-  %tmp2 = shufflevector <8 x float> undef, <8 x float> %tmp1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+  %tmp1 = shufflevector <2 x float> %tmp, <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %tmp2 = shufflevector <8 x float> poison, <8 x float> %tmp1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
   store <8 x float> %tmp2, ptr addrspace(1) %out, align 32
   ret void
 }
@@ -301,8 +301,8 @@ define amdgpu_kernel void @concat_vector_crash2(ptr addrspace(1) %out, ptr addrs
   %tmp = load i32, ptr addrspace(1) %in, align 1
   %tmp1 = trunc i32 %tmp to i24
   %tmp2 = bitcast i24 %tmp1 to <3 x i8>
-  %tmp3 = shufflevector <3 x i8> %tmp2, <3 x i8> undef, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 1, i32 undef, i32 undef>
-  %tmp4 = shufflevector <8 x i8> %tmp3, <8 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 7, i8 8>, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
+  %tmp3 = shufflevector <3 x i8> %tmp2, <3 x i8> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 poison, i32 poison>
+  %tmp4 = shufflevector <8 x i8> %tmp3, <8 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 7, i8 8>, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
   store <8 x i8> %tmp4, ptr addrspace(1) %out, align 8
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/constant-address-space-32bit.ll b/llvm/test/CodeGen/AMDGPU/constant-address-space-32bit.ll
index 8cb7d6651a08c..4f3cff4b32ea3 100644
--- a/llvm/test/CodeGen/AMDGPU/constant-address-space-32bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/constant-address-space-32bit.ll
@@ -247,7 +247,7 @@ main_body:
   %34 = extractelement <4 x float> %31, i32 2
   %35 = extractelement <4 x float> %31, i32 3
   %36 = bitcast float %4 to i32
-  %37 = insertvalue <{ i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> undef, i32 %36, 4
+  %37 = insertvalue <{ i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> poison, i32 %36, 4
   %38 = insertvalue <{ i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %37, float %32, 5
   %39 = insertvalue <{ i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %38, float %33, 6
   %40 = insertvalue <{ i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %39, float %34, 7
@@ -279,7 +279,7 @@ main_body:
   %34 = extractelement <4 x float> %31, i32 2
   %35 = extractelement <4 x float> %31, i32 3
   %36 = bitcast float %4 to i32
-  %37 = insertvalue <{ i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> undef, i32 %36, 4
+  %37 = insertvalue <{ i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> poison, i32 %36, 4
   %38 = insertvalue <{ i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %37, float %32, 5
   %39 = insertvalue <{ i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %38, float %33, 6
   %40 = insertvalue <{ i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %39, float %34, 7
diff --git a/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll b/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll
index 9558d9f0bc4c9..b5616501900dd 100644
--- a/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll
+++ b/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll
@@ -23,15 +23,64 @@ entry:
   br label %loop
 
 loop:
-  %promotealloca = phi <16 x i32> [ undef, %entry ], [ %0, %loop ]
+  %promotealloca = phi <16 x i32> [ poison, %entry ], [ %insert, %loop ]
   %inc = phi i32 [ 0, %entry ], [ %inc.i, %loop ]
-  %0 = insertelement <16 x i32> %promotealloca, i32 %inc, i32 %inc
+  %insert = insertelement <16 x i32> %promotealloca, i32 %inc, i32 %inc
   %inc.i = add i32 %inc, %B
   %cnd = icmp uge i32 %inc.i, 16
   br i1 %cnd, label %done, label %loop
 
 done:
-  %1 = extractelement <16 x i32> %0, i32 0
-  store i32 %1, ptr addrspace(1) %out, align 4
+  %extract.0 = extractelement <16 x i32> %insert, i32 0
+  store i32 %extract.0, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+; When creating registers for %divergent.alloca.phi, we should report
+; the CopyToReg as divergent values (not uniform just because the
+; alloca is uniform)
+define void @phi_with_alloca_and_divergent_copy_to_reg(ptr addrspace(5) %divergent.private, ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: phi_with_alloca_and_divergent_copy_to_reg:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v7, v2
+; CHECK-NEXT:    v_mov_b32_e32 v6, v1
+; CHECK-NEXT:    s_mov_b64 s[4:5], 0
+; CHECK-NEXT:    v_lshrrev_b32_e64 v2, 6, s32
+; CHECK-NEXT:  .LBB1_1: ; %loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    v_mov_b32_e32 v1, v2
+; CHECK-NEXT:    v_lshl_add_u32 v2, v3, 2, v1
+; CHECK-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen
+; CHECK-NEXT:    v_add_u32_e32 v2, 1, v3
+; CHECK-NEXT:    v_cmp_lt_u32_e32 vcc, 15, v2
+; CHECK-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; CHECK-NEXT:    v_mov_b32_e32 v3, v4
+; CHECK-NEXT:    v_mov_b32_e32 v2, v0
+; CHECK-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; CHECK-NEXT:    s_cbranch_execnz .LBB1_1
+; CHECK-NEXT:  ; %bb.2: ; %done
+; CHECK-NEXT:    s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT:    buffer_load_dword v0, v1, s[0:3], 0 offen
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dword v[6:7], v0, off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %alloca0 = alloca [16 x i32], addrspace(5)
+  br label %loop
+
+loop:
+  %inc = phi i32 [%a, %entry], [%b, %loop]
+  %divergent.alloca.phi = phi ptr addrspace(5) [ %alloca0, %entry ], [ %divergent.private, %loop ]
+  %ptr = getelementptr [16 x i32], ptr addrspace(5) %divergent.alloca.phi, i32 0, i32 %inc
+  store i32 %inc, ptr addrspace(5) %ptr
+  %inc.i = add i32 %inc, 1
+  %cnd = icmp uge i32 %inc.i, 16
+  br i1 %cnd, label %done, label %loop
+
+done:
+  %tmp1 = load i32, ptr addrspace(5) %divergent.alloca.phi
+  store i32 %tmp1, ptr addrspace(1) %out
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/copy-to-reg.ll b/llvm/test/CodeGen/AMDGPU/copy-to-reg.ll
index e517f1c8e6628..931a14473c340 100644
--- a/llvm/test/CodeGen/AMDGPU/copy-to-reg.ll
+++ b/llvm/test/CodeGen/AMDGPU/copy-to-reg.ll
@@ -1,12 +1,69 @@
-; RUN: llc -mtriple=amdgcn -mattr=-promote-alloca -verify-machineinstrs < %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -mattr=-promote-alloca -verify-machineinstrs < %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=hawaii -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds < %s | FileCheck -check-prefix=GFX7 %s
+; RUN: llc -mtriple=amdgcn -mcpu=tonga -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds < %s | FileCheck -check-prefix=GFX8 %s
 
 ; Test that CopyToReg instructions don't have non-register operands prior
 ; to being emitted.
 
 ; Make sure this doesn't crash
-; CHECK-LABEL: {{^}}copy_to_reg_frameindex:
+
 define amdgpu_kernel void @copy_to_reg_frameindex(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c) {
+; GFX7-LABEL: copy_to_reg_frameindex:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX7-NEXT:    s_mov_b32 s14, -1
+; GFX7-NEXT:    s_mov_b32 s15, 0xe8f000
+; GFX7-NEXT:    s_add_u32 s12, s12, s11
+; GFX7-NEXT:    s_addc_u32 s13, s13, 0
+; GFX7-NEXT:    s_mov_b32 s0, 0
+; GFX7-NEXT:    s_mov_b32 s1, 0
+; GFX7-NEXT:  .LBB0_1: ; %loop
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    s_add_i32 s1, s1, 1
+; GFX7-NEXT:    s_add_i32 s0, s0, 4
+; GFX7-NEXT:    s_cmp_lt_u32 s1, 16
+; GFX7-NEXT:    buffer_store_dword v0, v1, s[12:15], 0 offen
+; GFX7-NEXT:    s_cbranch_scc1 .LBB0_1
+; GFX7-NEXT:  ; %bb.2: ; %done
+; GFX7-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    s_mov_b32 s2, -1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: copy_to_reg_frameindex:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
+; GFX8-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
+; GFX8-NEXT:    s_mov_b32 s90, -1
+; GFX8-NEXT:    s_mov_b32 s91, 0xe80000
+; GFX8-NEXT:    s_add_u32 s88, s88, s11
+; GFX8-NEXT:    s_addc_u32 s89, s89, 0
+; GFX8-NEXT:    s_mov_b32 s0, 0
+; GFX8-NEXT:    s_mov_b32 s1, 0
+; GFX8-NEXT:  .LBB0_1: ; %loop
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    s_add_i32 s1, s1, 1
+; GFX8-NEXT:    s_add_i32 s0, s0, 4
+; GFX8-NEXT:    s_cmp_lt_u32 s1, 16
+; GFX8-NEXT:    buffer_store_dword v0, v1, s[88:91], 0 offen
+; GFX8-NEXT:    s_cbranch_scc1 .LBB0_1
+; GFX8-NEXT:  ; %bb.2: ; %done
+; GFX8-NEXT:    buffer_load_dword v2, off, s[88:91], 0
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
 entry:
   %alloca = alloca [16 x i32], addrspace(5)
   br label %loop
diff --git a/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll b/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
index 58f7dd0784db0..346c418b4fe74 100644
--- a/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
@@ -168,7 +168,7 @@ bb1:
   %val1 = extractvalue { <4 x i32>, <4 x half> } %split.ret.type, 1
   %extract0 = extractelement <4 x i32> %val0, i32 0
   %extract1 = extractelement <4 x half> %val1, i32 0
-  %ins0 = insertvalue { i32, half } undef, i32 %extract0, 0
+  %ins0 = insertvalue { i32, half } poison, i32 %extract0, 0
   %ins1 = insertvalue { i32, half } %ins0, half %extract1, 1
   ret { i32, half } %ins1
 }
diff --git a/llvm/test/CodeGen/AMDGPU/cse-phi-incoming-val.ll b/llvm/test/CodeGen/AMDGPU/cse-phi-incoming-val.ll
index 4f0d040e3217c..c73a3b69d3716 100644
--- a/llvm/test/CodeGen/AMDGPU/cse-phi-incoming-val.ll
+++ b/llvm/test/CodeGen/AMDGPU/cse-phi-incoming-val.ll
@@ -30,7 +30,7 @@ bb9:                                              ; preds = %bb5
   br label %bb10
 
 bb10:                                             ; preds = %bb9, %bb5, %bb3, %bb
-  %tmp11 = phi float [ 1.000000e+00, %bb3 ], [ 0.000000e+00, %bb9 ], [ 1.000000e+00, %bb ], [ undef, %bb5 ]
+  %tmp11 = phi float [ 1.000000e+00, %bb3 ], [ 0.000000e+00, %bb9 ], [ 1.000000e+00, %bb ], [ poison, %bb5 ]
   call void @llvm.amdgcn.exp.f32(i32 40, i32 15, float %tmp11, float undef, float undef, float undef, i1 false, i1 false) #0
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/csr-sgpr-spill-live-ins.mir b/llvm/test/CodeGen/AMDGPU/csr-sgpr-spill-live-ins.mir
index a14d515688a8b..6504f48333485 100644
--- a/llvm/test/CodeGen/AMDGPU/csr-sgpr-spill-live-ins.mir
+++ b/llvm/test/CodeGen/AMDGPU/csr-sgpr-spill-live-ins.mir
@@ -14,15 +14,7 @@ body: |
   ; CHECK-LABEL: name: def_csr_sgpr
   ; CHECK: bb.0:
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
-  ; CHECK-NEXT:   liveins: $sgpr42, $sgpr43, $sgpr46, $sgpr47
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   $sgpr4_sgpr5 = S_XOR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, implicit $exec :: (store (s32) into %stack.4, addrspace 5)
-  ; CHECK-NEXT:   $exec = S_MOV_B64 killed $sgpr4_sgpr5
-  ; CHECK-NEXT:   $vgpr0 = SI_SPILL_S32_TO_VGPR $sgpr42, 0, $vgpr0
-  ; CHECK-NEXT:   $vgpr0 = SI_SPILL_S32_TO_VGPR $sgpr43, 1, $vgpr0
-  ; CHECK-NEXT:   $vgpr0 = SI_SPILL_S32_TO_VGPR $sgpr46, 2, $vgpr0
-  ; CHECK-NEXT:   $vgpr0 = SI_SPILL_S32_TO_VGPR $sgpr47, 3, $vgpr0
   ; CHECK-NEXT:   S_NOP 0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
diff --git a/llvm/test/CodeGen/AMDGPU/cube.ll b/llvm/test/CodeGen/AMDGPU/cube.ll
index 49d8276eda418..72711df2ad445 100644
--- a/llvm/test/CodeGen/AMDGPU/cube.ll
+++ b/llvm/test/CodeGen/AMDGPU/cube.ll
@@ -18,7 +18,7 @@ define amdgpu_kernel void @cube(ptr addrspace(1) %out, float %a, float %b, float
   %cubetc = call float @llvm.amdgcn.cubetc(float %a, float %b, float %c)
   %cubema = call float @llvm.amdgcn.cubema(float %a, float %b, float %c)
 
-  %vec0 = insertelement <4 x float> undef, float %cubeid, i32 0
+  %vec0 = insertelement <4 x float> poison, float %cubeid, i32 0
   %vec1 = insertelement <4 x float> %vec0, float %cubesc, i32 1
   %vec2 = insertelement <4 x float> %vec1, float %cubetc, i32 2
   %vec3 = insertelement <4 x float> %vec2, float %cubema, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index fec020a296b9b..5a301aaacb6f2 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -456,7 +456,7 @@ define <4 x float> @v_uitofp_unpack_i32_to_v4f32(i32 %arg0) nounwind {
   %mask.lshr.24 = and i32 %lshr.24, 255
   %cvt3 = uitofp i32 %mask.lshr.24 to float
 
-  %ins.0 = insertelement <4 x float> undef, float %cvt0, i32 0
+  %ins.0 = insertelement <4 x float> poison, float %cvt0, i32 0
   %ins.1 = insertelement <4 x float> %ins.0, float %cvt1, i32 1
   %ins.2 = insertelement <4 x float> %ins.1, float %cvt2, i32 2
   %ins.3 = insertelement <4 x float> %ins.2, float %cvt3, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/dagcomb-shuffle-vecextend-non2.ll b/llvm/test/CodeGen/AMDGPU/dagcomb-shuffle-vecextend-non2.ll
index 009590ed9e410..09607c982ed7f 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcomb-shuffle-vecextend-non2.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcomb-shuffle-vecextend-non2.ll
@@ -16,17 +16,17 @@ define amdgpu_ps void @main(i32 %in1, i32 inreg %arg) local_unnamed_addr {
   br i1 %cond, label %bb12, label %bb
 
 bb:
-  %__llpc_global_proxy_r5.12.vec.insert = insertelement <4 x i32> undef, i32 %in1, i32 3
-  %tmp3 = shufflevector <4 x i32> %__llpc_global_proxy_r5.12.vec.insert, <4 x i32> undef, <3 x i32> <i32 undef, i32 undef, i32 1>
+  %__llpc_global_proxy_r5.12.vec.insert = insertelement <4 x i32> poison, i32 %in1, i32 3
+  %tmp3 = shufflevector <4 x i32> %__llpc_global_proxy_r5.12.vec.insert, <4 x i32> poison, <3 x i32> <i32 poison, i32 poison, i32 1>
   %tmp4 = bitcast <3 x i32> %tmp3 to <3 x float>
   %a2.i123 = extractelement <3 x float> %tmp4, i32 2
   %tmp5 = bitcast float %a2.i123 to i32
-  %__llpc_global_proxy_r2.0.vec.insert196 = insertelement <4 x i32> undef, i32 %tmp5, i32 0
+  %__llpc_global_proxy_r2.0.vec.insert196 = insertelement <4 x i32> poison, i32 %tmp5, i32 0
   br label %bb12
 
 bb12:
-  %__llpc_global_proxy_r2.0 = phi <4 x i32> [ %__llpc_global_proxy_r2.0.vec.insert196, %bb ], [ undef, %.entry ]
-  %tmp6 = shufflevector <4 x i32> %__llpc_global_proxy_r2.0, <4 x i32> undef, <3 x i32> <i32 1, i32 2, i32 3>
+  %__llpc_global_proxy_r2.0 = phi <4 x i32> [ %__llpc_global_proxy_r2.0.vec.insert196, %bb ], [ poison, %.entry ]
+  %tmp6 = shufflevector <4 x i32> %__llpc_global_proxy_r2.0, <4 x i32> poison, <3 x i32> <i32 1, i32 2, i32 3>
   %tmp7 = bitcast <3 x i32> %tmp6 to <3 x float>
   %a0.i = extractelement <3 x float> %tmp7, i32 0
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll
index 29e4bd55c4b90..142494a803755 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll
@@ -64,8 +64,8 @@ bb2:
   br label %bb11
 
 bb11:
-  %i12 = phi float [ %i6, %bb2 ], [ undef, %bb ]
-  %i13 = phi float [ %i10, %bb2 ], [ undef, %bb ]
+  %i12 = phi float [ %i6, %bb2 ], [ poison, %bb ]
+  %i13 = phi float [ %i10, %bb2 ], [ poison, %bb ]
   %i14 = phi i1 [ false, %bb2 ], [ true, %bb ]
   br i1 %i14, label %bb15, label %bb17
 
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll
index f88d3f1261f0b..2464275a87992 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll
@@ -146,7 +146,7 @@ define amdgpu_ps float @_amdgpu_ps_main() #0 {
   %0 = call <3 x float> @llvm.amdgcn.image.sample.2d.v3f32.f32(i32 7, float undef, float undef, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0)
   %.i2243 = extractelement <3 x float> %0, i32 2
   %1 = call <3 x i32> @llvm.amdgcn.s.buffer.load.v3i32(<4 x i32> undef, i32 0, i32 0)
-  %2 = shufflevector <3 x i32> %1, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %2 = shufflevector <3 x i32> %1, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
   %3 = bitcast <4 x i32> %2 to <4 x float>
   %.i2248 = extractelement <4 x float> %3, i32 2
   %.i2249 = fmul reassoc nnan nsz arcp contract afn float %.i2243, %.i2248
@@ -159,17 +159,17 @@ define amdgpu_ps float @_amdgpu_ps_main() #0 {
   %.i0364 = extractelement <2 x float> %7, i32 0
   %8 = call float @llvm.amdgcn.image.sample.2d.f32.f32(i32 1, float undef, float undef, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0)
   %9 = call <3 x i32> @llvm.amdgcn.s.buffer.load.v3i32(<4 x i32> undef, i32 112, i32 0)
-  %10 = shufflevector <3 x i32> %9, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %10 = shufflevector <3 x i32> %9, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
   %11 = bitcast <4 x i32> %10 to <4 x float>
   %.i2360 = extractelement <4 x float> %11, i32 2
   %.i2363 = fmul reassoc nnan nsz arcp contract afn float %.i2360, %8
   %12 = call <3 x i32> @llvm.amdgcn.s.buffer.load.v3i32(<4 x i32> undef, i32 96, i32 0)
-  %13 = shufflevector <3 x i32> %12, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %13 = shufflevector <3 x i32> %12, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
   %14 = bitcast <4 x i32> %13 to <4 x float>
   %.i2367 = extractelement <4 x float> %14, i32 2
   %.i2370 = fmul reassoc nnan nsz arcp contract afn float %.i0364, %.i2367
   %15 = call <3 x i32> @llvm.amdgcn.s.buffer.load.v3i32(<4 x i32> undef, i32 32, i32 0)
-  %16 = shufflevector <3 x i32> %15, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %16 = shufflevector <3 x i32> %15, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
   %17 = bitcast <4 x i32> %16 to <4 x float>
   %.i2373 = extractelement <4 x float> %17, i32 2
   %.i2376 = fsub reassoc nnan nsz arcp contract afn float %.i2373, %.i2370
@@ -212,12 +212,12 @@ define amdgpu_ps float @_amdgpu_ps_main() #0 {
   %.i2466 = fmul reassoc nnan nsz arcp contract afn float %.i2465, %43
   %.i2469 = fmul reassoc nnan nsz arcp contract afn float %.i2415, %.i2466
   %45 = call <3 x i32> @llvm.amdgcn.s.buffer.load.v3i32(<4 x i32> undef, i32 64, i32 0)
-  %46 = shufflevector <3 x i32> %45, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %46 = shufflevector <3 x i32> %45, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
   %47 = bitcast <4 x i32> %46 to <4 x float>
   %.i2476 = extractelement <4 x float> %47, i32 2
   %.i2479 = fmul reassoc nnan nsz arcp contract afn float %.i2476, %18
   %48 = call <3 x i32> @llvm.amdgcn.s.buffer.load.v3i32(<4 x i32> undef, i32 80, i32 0)
-  %49 = shufflevector <3 x i32> %48, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %49 = shufflevector <3 x i32> %48, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
   %50 = bitcast <4 x i32> %49 to <4 x float>
   %.i2482 = extractelement <4 x float> %50, i32 2
   %.i2485 = fsub reassoc nnan nsz arcp contract afn float %.i2482, %.i2479
@@ -230,7 +230,7 @@ define amdgpu_ps float @_amdgpu_ps_main() #0 {
   %.i2522 = fadd reassoc nnan nsz arcp contract afn float %.i2521, %.i2516
   %.i2525 = fmul reassoc nnan nsz arcp contract afn float %.i2522, %43
   %52 = call <3 x i32> @llvm.amdgcn.s.buffer.load.v3i32(<4 x i32> undef, i32 16, i32 0)
-  %53 = shufflevector <3 x i32> %52, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %53 = shufflevector <3 x i32> %52, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
   %54 = bitcast <4 x i32> %53 to <4 x float>
   %.i2530 = extractelement <4 x float> %54, i32 2
   %.i2531 = fmul reassoc nnan nsz arcp contract afn float %.i2333, %.i2530
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombiner-bug-illegal-vec4-int-to-fp.ll b/llvm/test/CodeGen/AMDGPU/dagcombiner-bug-illegal-vec4-int-to-fp.ll
index 9333c7a5d43dc..f5bf1793b9f40 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombiner-bug-illegal-vec4-int-to-fp.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombiner-bug-illegal-vec4-int-to-fp.ll
@@ -15,8 +15,8 @@ entry:
   %ptr = getelementptr i32, ptr addrspace(1) %in, i32 1
   %sint = load i32, ptr addrspace(1) %in
   %conv = sitofp i32 %sint to float
-  %0 = insertelement <4 x float> undef, float %conv, i32 0
-  %splat = shufflevector <4 x float> %0, <4 x float> undef, <4 x i32> zeroinitializer
+  %0 = insertelement <4 x float> poison, float %conv, i32 0
+  %splat = shufflevector <4 x float> %0, <4 x float> poison, <4 x i32> zeroinitializer
   store <4 x float> %splat, ptr addrspace(1) %out
   ret void
 }
@@ -29,8 +29,8 @@ entry:
   %ptr = getelementptr i32, ptr addrspace(1) %in, i32 1
   %uint = load i32, ptr addrspace(1) %in
   %conv = uitofp i32 %uint to float
-  %0 = insertelement <4 x float> undef, float %conv, i32 0
-  %splat = shufflevector <4 x float> %0, <4 x float> undef, <4 x i32> zeroinitializer
+  %0 = insertelement <4 x float> poison, float %conv, i32 0
+  %splat = shufflevector <4 x float> %0, <4 x float> poison, <4 x i32> zeroinitializer
   store <4 x float> %splat, ptr addrspace(1) %out
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/dbg-info-inline-at.ll b/llvm/test/CodeGen/AMDGPU/dbg-info-inline-at.ll
new file mode 100644
index 0000000000000..ed609f85918f9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/dbg-info-inline-at.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 < %s | FileCheck %s
+
+define amdgpu_kernel void @_Z3fooPiiii(ptr addrspace(1) nocapture noundef writeonly %c.coerce, i32 noundef %a, i32 noundef %b, i32 noundef %d) !dbg !9 {
+; CHECK-LABEL: _Z3fooPiiii:
+; CHECK:       .Lfunc_begin0:
+; CHECK-NEXT:    .file 0 "test" "a.hip" md5 0x004a28df8cfd98cdd2c71d5d814d9c6b
+; CHECK-NEXT:    .cfi_sections .debug_frame
+; CHECK-NEXT:    .cfi_startproc
+; CHECK-NEXT:  ; %bb.0: ; %entry
+; CHECK-NEXT:    .file 1 "." "a.h"
+; CHECK-NEXT:    .loc 1 5 12 prologue_end ; ./a.h:5:12 @[ a.hip:12:8 ]
+; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x8
+; CHECK-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_add_i32 s1, s1, s0
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    .loc 1 5 12 is_stmt 0 ; ./a.h:5:12 @[ a.hip:13:9 ]
+; CHECK-NEXT:    s_add_i32 s0, s2, s0
+; CHECK-NEXT:  .Ltmp1:
+; CHECK-NEXT:    .file 2 "a.hip"
+; CHECK-NEXT:    .loc 2 13 6 is_stmt 1 ; a.hip:13:6
+; CHECK-NEXT:    s_mul_i32 s0, s0, s1
+; CHECK-NEXT:    v_mov_b32_e32 v1, s0
+; CHECK-NEXT:    global_store_dword v0, v1, s[4:5]
+; CHECK-NEXT:    .loc 2 14 1 ; a.hip:14:1
+; CHECK-NEXT:    s_endpgm
+; CHECK-NEXT:  .Ltmp2:
+entry:
+  %add.i = add nsw i32 %b, %a, !dbg !13
+  %add.i3 = add nsw i32 %d, %a, !dbg !17
+  %mul = mul nsw i32 %add.i3, %add.i, !dbg !19
+  store i32 %mul, ptr addrspace(1) %c.coerce, align 4, !dbg !19, !tbaa !20
+  ret void, !dbg !24
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6, !7}
+!llvm.ident = !{!8}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 20.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "a.hip", directory: "test", checksumkind: CSK_MD5, checksum: "004a28df8cfd98cdd2c71d5d814d9c6b")
+!2 = !{i32 1, !"amdhsa_code_object_version", i32 500}
+!3 = !{i32 1, !"amdgpu_printf_kind", !"hostcall"}
+!4 = !{i32 7, !"Dwarf Version", i32 5}
+!5 = !{i32 2, !"Debug Info Version", i32 3}
+!6 = !{i32 1, !"wchar_size", i32 4}
+!7 = !{i32 8, !"PIC Level", i32 2}
+!8 = !{!"clang version 20.0.0"}
+!9 = distinct !DISubprogram(name: "foo", scope: !10, file: !10, line: 11, type: !11, scopeLine: 11, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!10 = !DIFile(filename: "a.hip", directory: "test")
+!11 = !DISubroutineType(types: !12)
+!12 = !{}
+!13 = !DILocation(line: 5, column: 12, scope: !14, inlinedAt: !16)
+!14 = distinct !DISubprogram(name: "bar", scope: !15, file: !15, line: 4, type: !11, scopeLine: 4, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!15 = !DIFile(filename: "./a.h", directory: "test")
+!16 = distinct !DILocation(line: 12, column: 8, scope: !9)
+!17 = !DILocation(line: 5, column: 12, scope: !14, inlinedAt: !18)
+!18 = distinct !DILocation(line: 13, column: 9, scope: !9)
+!19 = !DILocation(line: 13, column: 6, scope: !9)
+!20 = !{!21, !21, i64 0}
+!21 = !{!"int", !22, i64 0}
+!22 = !{!"omnipotent char", !23, i64 0}
+!23 = !{!"Simple C++ TBAA"}
+!24 = !DILocation(line: 14, column: 1, scope: !9)
diff --git a/llvm/test/CodeGen/AMDGPU/dead-machine-elim-after-dead-lane.ll b/llvm/test/CodeGen/AMDGPU/dead-machine-elim-after-dead-lane.ll
index d3e42260cd5cb..e9d109b4aa1a9 100644
--- a/llvm/test/CodeGen/AMDGPU/dead-machine-elim-after-dead-lane.ll
+++ b/llvm/test/CodeGen/AMDGPU/dead-machine-elim-after-dead-lane.ll
@@ -14,7 +14,7 @@ entry:
 sw.bb4:
   %x = load i64, ptr addrspace(1) undef, align 8
   %c = sitofp i64 %x to float
-  %v = insertelement <2 x float> <float undef, float 0.000000e+00>, float %c, i32 0
+  %v = insertelement <2 x float> <float poison, float 0.000000e+00>, float %c, i32 0
   br label %foo.exit
 
 sw.bb10:
diff --git a/llvm/test/CodeGen/AMDGPU/debug-value.ll b/llvm/test/CodeGen/AMDGPU/debug-value.ll
index 127b23ed3bb23..e7b56cbe2e4d4 100644
--- a/llvm/test/CodeGen/AMDGPU/debug-value.ll
+++ b/llvm/test/CodeGen/AMDGPU/debug-value.ll
@@ -7,7 +7,7 @@ bb:
   %tmp = load i32, ptr addrspace(1) undef, align 4
   %tmp1 = load <4 x float>, ptr addrspace(1) undef, align 16
   %tmp2 = sext i32 %tmp to i64
-  %tmp3 = shufflevector <4 x float> undef, <4 x float> %tmp1, <2 x i32> <i32 3, i32 7>
+  %tmp3 = shufflevector <4 x float> poison, <4 x float> %tmp1, <2 x i32> <i32 3, i32 7>
   %tmp4 = call float @barney() #2
   %tmp9 = getelementptr inbounds %struct.wombat, ptr addrspace(1) %arg, i64 %tmp2, i32 2, i64 0
   %tmp10 = load i32, ptr addrspace(1) %tmp9, align 4
@@ -29,8 +29,8 @@ bb21:                                             ; preds = %bb
   br label %bb28
 
 bb25:                                             ; preds = %bb
-  %tmp26 = insertelement <4 x float> undef, float 0.000000e+00, i32 1
-  %tmp27 = insertelement <4 x float> %tmp26, float undef, i32 2
+  %tmp26 = insertelement <4 x float> poison, float 0.000000e+00, i32 1
+  %tmp27 = insertelement <4 x float> %tmp26, float poison, i32 2
   br label %bb28
 
 bb28:                                             ; preds = %bb25, %bb21
@@ -52,8 +52,8 @@ bb28:                                             ; preds = %bb25, %bb21
   %tmp44 = fsub float %tmp43, undef
   %tmp45 = fadd float undef, undef
   %tmp46 = fdiv float %tmp44, %tmp45
-  %tmp47 = insertelement <4 x float> undef, float %tmp46, i32 0
-  %tmp48 = shufflevector <4 x float> %tmp47, <4 x float> undef, <4 x i32> zeroinitializer
+  %tmp47 = insertelement <4 x float> poison, float %tmp46, i32 0
+  %tmp48 = shufflevector <4 x float> %tmp47, <4 x float> poison, <4 x i32> zeroinitializer
   %tmp49 = fsub <4 x float> %tmp48, %tmp40
   %tmp50 = extractelement <4 x float> %tmp41, i32 1
   %tmp51 = extractelement <4 x float> %tmp42, i32 2
@@ -70,8 +70,8 @@ bb28:                                             ; preds = %bb25, %bb21
   ; CHECK-NOT: ;DEBUG_VALUE:
   call void @llvm.dbg.value(metadata <4 x float> %tmp29, metadata !3, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)) #2, !dbg !5
   %tmp59 = bitcast i64 %tmp35 to <2 x float>
-  %tmp60 = insertelement <2 x float> undef, float %tmp58, i32 0
-  %tmp61 = shufflevector <2 x float> %tmp60, <2 x float> undef, <2 x i32> zeroinitializer
+  %tmp60 = insertelement <2 x float> poison, float %tmp58, i32 0
+  %tmp61 = shufflevector <2 x float> %tmp60, <2 x float> poison, <2 x i32> zeroinitializer
   %tmp62 = fmul <2 x float> %tmp61, undef
   %tmp63 = fsub <2 x float> %tmp62, %tmp59
   %tmp64 = extractelement <2 x float> %tmp63, i64 0
diff --git a/llvm/test/CodeGen/AMDGPU/debug-value2.ll b/llvm/test/CodeGen/AMDGPU/debug-value2.ll
index b09d540dd6d7b..3a16476adf5c1 100644
--- a/llvm/test/CodeGen/AMDGPU/debug-value2.ll
+++ b/llvm/test/CodeGen/AMDGPU/debug-value2.ll
@@ -25,8 +25,8 @@ entry:
   %tmp1 = load <4 x float>, ptr addrspace(1) %m_angularMotion, align 16
   %m_scaleMotion = getelementptr inbounds %struct.ShapeData, ptr addrspace(1) %call, i64 0, i32 4
   %tmp2 = load <4 x float>, ptr addrspace(1) %m_scaleMotion, align 16
-  %splat.splatinsert = insertelement <4 x float> undef, float %time, i32 0
-  %splat.splat = shufflevector <4 x float> %splat.splatinsert, <4 x float> undef, <4 x i32> zeroinitializer
+  %splat.splatinsert = insertelement <4 x float> poison, float %time, i32 0
+  %splat.splat = shufflevector <4 x float> %splat.splatinsert, <4 x float> poison, <4 x i32> zeroinitializer
   %tmp3 = tail call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %tmp2, <4 x float> %splat.splat, <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>)
   %tmp4 = load <4 x float>, ptr addrspace(1) %call, align 16
   %m_quaternion = getelementptr inbounds %struct.ShapeData, ptr addrspace(1) %call, i64 0, i32 1
@@ -36,7 +36,7 @@ entry:
   %mul = fmul <4 x float> %tmp6, %v
   %tmp7 = extractelement <4 x float> %tmp5, i64 0
   %sub.i.i = fsub float -0.000000e+00, %tmp7
-  %vecinit.i.i = insertelement <4 x float> undef, float %sub.i.i, i32 0
+  %vecinit.i.i = insertelement <4 x float> poison, float %sub.i.i, i32 0
   %tmp8 = extractelement <4 x float> %tmp5, i64 1
   %sub1.i.i = fsub float -0.000000e+00, %tmp8
   %vecinit2.i.i = insertelement <4 x float> %vecinit.i.i, float %sub1.i.i, i32 1
@@ -57,12 +57,12 @@ entry:
   %tmp20 = fmul float %tmp8, %tmp16
   %tmp21 = fsub float -0.000000e+00, %tmp20
   %tmp22 = tail call float @llvm.fmuladd.f32(float %tmp7, float %tmp12, float %tmp21)
-  %tmp23 = insertelement <4 x float> <float undef, float undef, float undef, float 0.000000e+00>, float %tmp15, i32 0
+  %tmp23 = insertelement <4 x float> <float poison, float poison, float poison, float 0.000000e+00>, float %tmp15, i32 0
   %tmp24 = insertelement <4 x float> %tmp23, float %tmp19, i32 1
   %tmp25 = insertelement <4 x float> %tmp24, float %tmp22, i32 2
   %tmp26 = extractelement <4 x float> %tmp5, i64 3
-  %splat.splat.i8.i = shufflevector <4 x float> %tmp5, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
-  %splat.splat2.i9.i = shufflevector <4 x float> %tmp10, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %splat.splat.i8.i = shufflevector <4 x float> %tmp5, <4 x float> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %splat.splat2.i9.i = shufflevector <4 x float> %tmp10, <4 x float> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
   %mul3.i10.i = fmul <4 x float> %tmp5, %splat.splat2.i9.i
   %tmp27 = tail call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %splat.splat.i8.i, <4 x float> %tmp10, <4 x float> %mul3.i10.i)
   %add.i11.i = fadd <4 x float> %tmp27, %tmp25
@@ -91,10 +91,10 @@ entry:
   %tmp49 = fmul float %tmp40, %sub.i.i
   %tmp50 = fsub float -0.000000e+00, %tmp49
   %tmp51 = tail call float @llvm.fmuladd.f32(float %tmp45, float %sub1.i.i, float %tmp50)
-  %tmp52 = insertelement <4 x float> <float undef, float undef, float undef, float 0.000000e+00>, float %tmp44, i32 0
+  %tmp52 = insertelement <4 x float> <float poison, float poison, float poison, float 0.000000e+00>, float %tmp44, i32 0
   %tmp53 = insertelement <4 x float> %tmp52, float %tmp48, i32 1
   %tmp54 = insertelement <4 x float> %tmp53, float %tmp51, i32 2
-  %splat.splat.i.i = shufflevector <4 x float> %tmp39, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+  %splat.splat.i.i = shufflevector <4 x float> %tmp39, <4 x float> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
   %tmp55 = extractelement <4 x float> %tmp5, i32 3
   %mul3.i.i = fmul <4 x float> %splat.splat.i8.i, %tmp39
   %tmp56 = tail call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %splat.splat.i.i, <4 x float> %vecinit5.i.i, <4 x float> %mul3.i.i)
@@ -113,12 +113,12 @@ entry:
   %tmp66 = extractelement <4 x float> %tmp1, i64 3
   %mul3 = fmul float %tmp66, %time
   %tmp67 = insertelement <4 x float> %tmp1, float 0.000000e+00, i32 3
-  %tmp68 = shufflevector <4 x float> %tmp67, <4 x float> %tmp1, <4 x i32> <i32 0, i32 5, i32 undef, i32 3>
+  %tmp68 = shufflevector <4 x float> %tmp67, <4 x float> %tmp1, <4 x i32> <i32 0, i32 5, i32 poison, i32 3>
   %vecinit3.i.i = shufflevector <4 x float> %tmp68, <4 x float> %tmp1, <4 x i32> <i32 0, i32 1, i32 6, i32 3>
   %tmp69 = fcmp oeq <4 x float> %vecinit3.i.i, zeroinitializer
   %tmp70 = sext <4 x i1> %tmp69 to <4 x i32>
-  %tmp71 = shufflevector <4 x i32> %tmp70, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
-  %tmp72 = shufflevector <4 x i32> %tmp70, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %tmp71 = shufflevector <4 x i32> %tmp70, <4 x i32> poison, <2 x i32> <i32 2, i32 3>
+  %tmp72 = shufflevector <4 x i32> %tmp70, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
   %tmp73 = and <2 x i32> %tmp71, %tmp72
   %tmp74 = extractelement <2 x i32> %tmp73, i64 0
   %tmp75 = extractelement <2 x i32> %tmp73, i64 1
@@ -169,7 +169,7 @@ bb98:                                             ; preds = %bb96
 bb109:                                            ; preds = %bb98
   %tmp110 = tail call zeroext i1 @llvm.amdgcn.class.f32(float %tmp103, i32 516)
   %tmp111 = sext i1 %tmp110 to i32
-  %tmp112 = insertelement <4 x i32> undef, i32 %tmp111, i32 0
+  %tmp112 = insertelement <4 x i32> poison, i32 %tmp111, i32 0
   %tmp113 = tail call zeroext i1 @llvm.amdgcn.class.f32(float %tmp102, i32 516)
   %tmp114 = sext i1 %tmp113 to i32
   %tmp115 = insertelement <4 x i32> %tmp112, i32 %tmp114, i32 1
@@ -184,7 +184,7 @@ bb109:                                            ; preds = %bb98
   %tmp124 = bitcast <4 x i32> %tmp123 to <4 x float>
   %tmp125 = extractelement <4 x float> %tmp124, i64 0
   %tmp126 = tail call float @llvm.copysign.f32(float %tmp125, float %tmp103)
-  %tmp127 = insertelement <4 x float> undef, float %tmp126, i32 0
+  %tmp127 = insertelement <4 x float> poison, float %tmp126, i32 0
   %tmp128 = extractelement <4 x float> %tmp124, i64 1
   %tmp129 = tail call float @llvm.copysign.f32(float %tmp128, float %tmp102)
   %tmp130 = insertelement <4 x float> %tmp127, float %tmp129, i32 1
@@ -204,8 +204,8 @@ bb141:                                            ; preds = %bb109, %bb98, %bb96
   %tmp142 = phi <4 x float> [ %tmp87, %bb86 ], [ %tmp136, %bb109 ], [ %tmp99, %bb98 ], [ %vecinit3.i.i, %bb96 ]
   %tmp143 = phi float [ %tmp95, %bb86 ], [ %tmp140, %bb109 ], [ %tmp107, %bb98 ], [ %tmp84, %bb96 ]
   %tmp144 = tail call float @llvm.amdgcn.rsq.f32(float %tmp143)
-  %tmp145 = insertelement <4 x float> undef, float %tmp144, i32 0
-  %tmp146 = shufflevector <4 x float> %tmp145, <4 x float> undef, <4 x i32> zeroinitializer
+  %tmp145 = insertelement <4 x float> poison, float %tmp144, i32 0
+  %tmp146 = shufflevector <4 x float> %tmp145, <4 x float> poison, <4 x i32> zeroinitializer
   %tmp147 = fmul <4 x float> %tmp142, %tmp146
   br label %qtSet.exit
 
@@ -231,7 +231,7 @@ qtSet.exit:                                       ; preds = %bb141, %entry
   %tmp151 = tail call float @llvm.fmuladd.f32(float %tmp150, float 0x3FCCCCCCC0000000, float %tmp149)
   %tmp152 = extractelement <4 x float> %tmp148, i64 0
   %mul.i = fmul float %tmp151, %tmp152
-  %tmp153 = insertelement <4 x float> undef, float %mul.i, i64 0
+  %tmp153 = insertelement <4 x float> poison, float %mul.i, i64 0
   %tmp154 = extractelement <4 x float> %tmp148, i64 1
   %mul2.i = fmul float %tmp151, %tmp154
   %tmp155 = insertelement <4 x float> %tmp153, float %mul2.i, i64 1
@@ -258,7 +258,7 @@ qtSet.exit:                                       ; preds = %bb141, %entry
   %tmp160 = tail call float @llvm.fmuladd.f32(float %tmp159, float 0x3FCCCCCCC0000000, float %tmp158)
   %tmp161 = insertelement <4 x float> %tmp157, float %tmp160, i64 3
   %sub.i.i32 = fsub float -0.000000e+00, %mul.i
-  %vecinit.i.i33 = insertelement <4 x float> undef, float %sub.i.i32, i32 0
+  %vecinit.i.i33 = insertelement <4 x float> poison, float %sub.i.i32, i32 0
   %sub1.i.i34 = fsub float -0.000000e+00, %mul2.i
   %vecinit2.i.i35 = insertelement <4 x float> %vecinit.i.i33, float %sub1.i.i34, i32 1
   %sub3.i.i36 = fsub float -0.000000e+00, %mul3.i
diff --git a/llvm/test/CodeGen/AMDGPU/detect-dead-lanes.mir b/llvm/test/CodeGen/AMDGPU/detect-dead-lanes.mir
index f58c7d357b29f..df64d247d15ce 100644
--- a/llvm/test/CodeGen/AMDGPU/detect-dead-lanes.mir
+++ b/llvm/test/CodeGen/AMDGPU/detect-dead-lanes.mir
@@ -1,4 +1,5 @@
 # RUN: llc -mtriple=amdgcn -run-pass detect-dead-lanes -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -passes detect-dead-lanes -o - %s | FileCheck %s
 ...
 ---
 # Combined use/def transfer check, the basics.
diff --git a/llvm/test/CodeGen/AMDGPU/divergence-driven-abs.ll b/llvm/test/CodeGen/AMDGPU/divergence-driven-abs.ll
index 4ea553b8aaad0..68ae9854bd7d2 100644
--- a/llvm/test/CodeGen/AMDGPU/divergence-driven-abs.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergence-driven-abs.ll
@@ -34,9 +34,9 @@ define amdgpu_kernel void @v_abs_i32(ptr addrspace(1) %out, ptr addrspace(1) %sr
 ; GCN: S_ABS_I32
 ; GCN: S_ABS_I32
 define amdgpu_kernel void @s_abs_v2i32(ptr addrspace(1) %out, <2 x i32> %val) nounwind {
-  %z0 = insertelement <2 x i32> undef, i32 0, i32 0
+  %z0 = insertelement <2 x i32> poison, i32 0, i32 0
   %z1 = insertelement <2 x i32> %z0, i32 0, i32 1
-  %t0 = insertelement <2 x i32> undef, i32 2, i32 0
+  %t0 = insertelement <2 x i32> poison, i32 2, i32 0
   %t1 = insertelement <2 x i32> %t0, i32 2, i32 1
   %neg = sub <2 x i32> %z1, %val
   %cond = icmp sgt <2 x i32> %val, %neg
@@ -52,9 +52,9 @@ define amdgpu_kernel void @s_abs_v2i32(ptr addrspace(1) %out, <2 x i32> %val) no
 ; GCN: V_MAX_I32_e64
 ; GCN: V_MAX_I32_e64
 define amdgpu_kernel void @v_abs_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %src) nounwind {
-  %z0 = insertelement <2 x i32> undef, i32 0, i32 0
+  %z0 = insertelement <2 x i32> poison, i32 0, i32 0
   %z1 = insertelement <2 x i32> %z0, i32 0, i32 1
-  %t0 = insertelement <2 x i32> undef, i32 2, i32 0
+  %t0 = insertelement <2 x i32> poison, i32 2, i32 0
   %t1 = insertelement <2 x i32> %t0, i32 2, i32 1
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.in = getelementptr inbounds <2 x i32>, ptr addrspace(1) %src, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll b/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
index 142ec2f926e80..ada3f017f45cf 100644
--- a/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
@@ -50,7 +50,7 @@ define amdgpu_kernel void @uniform_vec_0_i16(ptr addrspace(1) %out, i16 %a) {
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
 ; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX11-NEXT:    s_endpgm
-  %tmp = insertelement <2 x i16> undef, i16 0, i32 0
+  %tmp = insertelement <2 x i16> poison, i16 0, i32 0
   %vec = insertelement <2 x i16> %tmp, i16 %a, i32 1
   %val = bitcast <2 x i16> %vec to i32
   store i32 %val, ptr addrspace(1) %out, align 4
@@ -81,7 +81,7 @@ define i32 @divergent_vec_0_i16(i16 %a) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
-  %tmp = insertelement <2 x i16> undef, i16 0, i32 0
+  %tmp = insertelement <2 x i16> poison, i16 0, i32 0
   %vec = insertelement <2 x i16> %tmp, i16 %a, i32 1
   %val = bitcast <2 x i16> %vec to i32
   ret i32 %val
@@ -133,7 +133,7 @@ define amdgpu_kernel void @uniform_vec_i16_0(ptr addrspace(1) %out, i16 %a) {
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
 ; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX11-NEXT:    s_endpgm
-  %tmp = insertelement <2 x i16> undef, i16 %a, i32 0
+  %tmp = insertelement <2 x i16> poison, i16 %a, i32 0
   %vec = insertelement <2 x i16> %tmp, i16 0, i32 1
   %val = bitcast <2 x i16> %vec to i32
   store i32 %val, ptr addrspace(1) %out, align 4
@@ -164,7 +164,7 @@ define i32 @divergent_vec_i16_0(i16 %a) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
-  %tmp = insertelement <2 x i16> undef, i16 %a, i32 0
+  %tmp = insertelement <2 x i16> poison, i16 %a, i32 0
   %vec = insertelement <2 x i16> %tmp, i16 0, i32 1
   %val = bitcast <2 x i16> %vec to i32
   ret i32 %val
@@ -216,7 +216,7 @@ define amdgpu_kernel void @uniform_vec_f16_0(ptr addrspace(1) %out, half %a) {
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
 ; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX11-NEXT:    s_endpgm
-  %tmp = insertelement <2 x half> undef, half %a, i32 0
+  %tmp = insertelement <2 x half> poison, half %a, i32 0
   %vec = insertelement <2 x half> %tmp, half 0.0, i32 1
   %val = bitcast <2 x half> %vec to float
   store float %val, ptr addrspace(1) %out, align 4
@@ -247,7 +247,7 @@ define float @divergent_vec_f16_0(half %a) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
-  %tmp = insertelement <2 x half> undef, half %a, i32 0
+  %tmp = insertelement <2 x half> poison, half %a, i32 0
   %vec = insertelement <2 x half> %tmp, half 0.0, i32 1
   %val = bitcast <2 x half> %vec to float
   ret float %val
@@ -311,7 +311,7 @@ define amdgpu_kernel void @uniform_vec_i16_LL(ptr addrspace(4) %in0, ptr addrspa
   %val1 = load volatile i32, ptr addrspace(4) %in1
   %lo = trunc i32 %val0 to i16
   %hi = trunc i32 %val1 to i16
-  %vec.0 = insertelement <2 x i16> undef, i16 %lo, i32 0
+  %vec.0 = insertelement <2 x i16> poison, i16 %lo, i32 0
   %vec.1 = insertelement <2 x i16> %vec.0, i16 %hi, i32 1
   %vec.i32 = bitcast <2 x i16> %vec.1 to i32
   call void asm sideeffect "; use $0", "s"(i32 %vec.i32) #0
@@ -346,7 +346,7 @@ define i32 @divergent_vec_i16_LL(i16 %a, i16 %b) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
-  %tmp = insertelement <2 x i16> undef, i16 %a, i32 0
+  %tmp = insertelement <2 x i16> poison, i16 %a, i32 0
   %vec = insertelement <2 x i16> %tmp, i16 %b, i32 1
   %val = bitcast <2 x i16> %vec to i32
   ret i32 %val
@@ -399,7 +399,7 @@ define amdgpu_kernel void @uniform_vec_i16_LH(ptr addrspace(1) %out, i16 %a, i32
 ; GFX11-NEXT:    s_endpgm
   %shift = lshr i32 %b, 16
   %tr = trunc i32 %shift to i16
-  %tmp = insertelement <2 x i16> undef, i16 %a, i32 0
+  %tmp = insertelement <2 x i16> poison, i16 %a, i32 0
   %vec = insertelement <2 x i16> %tmp, i16 %tr, i32 1
   %val = bitcast <2 x i16> %vec to i32
   store i32 %val, ptr addrspace(1) %out, align 4
@@ -435,7 +435,7 @@ define i32 @divergent_vec_i16_LH(i16 %a, i32 %b) {
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %shift = lshr i32 %b, 16
   %tr = trunc i32 %shift to i16
-  %tmp = insertelement <2 x i16> undef, i16 %a, i32 0
+  %tmp = insertelement <2 x i16> poison, i16 %a, i32 0
   %vec = insertelement <2 x i16> %tmp, i16 %tr, i32 1
   %val = bitcast <2 x i16> %vec to i32
   ret i32 %val
@@ -489,7 +489,7 @@ define amdgpu_kernel void @uniform_vec_i16_HH(ptr addrspace(1) %out, i32 %a, i32
   %tr_a = trunc i32 %shift_a to i16
   %shift_b = lshr i32 %b, 16
   %tr_b = trunc i32 %shift_b to i16
-  %tmp = insertelement <2 x i16> undef, i16 %tr_a, i32 0
+  %tmp = insertelement <2 x i16> poison, i16 %tr_a, i32 0
   %vec = insertelement <2 x i16> %tmp, i16 %tr_b, i32 1
   %val = bitcast <2 x i16> %vec to i32
   store i32 %val, ptr addrspace(1) %out, align 4
@@ -527,7 +527,7 @@ define i32 @divergent_vec_i16_HH(i32 %a, i32 %b) {
   %tr_a = trunc i32 %shift_a to i16
   %shift_b = lshr i32 %b, 16
   %tr_b = trunc i32 %shift_b to i16
-  %tmp = insertelement <2 x i16> undef, i16 %tr_a, i32 0
+  %tmp = insertelement <2 x i16> poison, i16 %tr_a, i32 0
   %vec = insertelement <2 x i16> %tmp, i16 %tr_b, i32 1
   %val = bitcast <2 x i16> %vec to i32
   ret i32 %val
@@ -593,7 +593,7 @@ define amdgpu_kernel void @uniform_vec_f16_LL(ptr addrspace(4) %in0, ptr addrspa
   %hi.i = trunc i32 %val1 to i16
   %lo = bitcast i16 %lo.i to half
   %hi = bitcast i16 %hi.i to half
-  %vec.0 = insertelement <2 x half> undef, half %lo, i32 0
+  %vec.0 = insertelement <2 x half> poison, half %lo, i32 0
   %vec.1 = insertelement <2 x half> %vec.0, half %hi, i32 1
   %vec.i32 = bitcast <2 x half> %vec.1 to i32
 
@@ -630,7 +630,7 @@ define float @divergent_vec_f16_LL(half %a, half %b) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
-  %tmp = insertelement <2 x half> undef, half %a, i32 0
+  %tmp = insertelement <2 x half> poison, half %a, i32 0
   %vec = insertelement <2 x half> %tmp, half %b, i32 1
   %val = bitcast <2 x half> %vec to float
   ret float %val
@@ -667,7 +667,7 @@ define <2 x i16> @build_vec_v2i16_undeflo_divergent(ptr addrspace(3) %in) #0 {
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i16, ptr addrspace(3) %in
-  %build = insertelement <2 x i16> undef, i16 %load, i32 0
+  %build = insertelement <2 x i16> poison, i16 %load, i32 0
   ret <2 x i16> %build
 }
 
@@ -723,7 +723,7 @@ define amdgpu_kernel void @build_vec_v2i16_undeflo_uniform(ptr addrspace(3) %in,
 ; GFX11-NEXT:    s_endpgm
 entry:
   %load = load i16, ptr addrspace(3) %in
-  %build = insertelement <2 x i16> undef, i16 %load, i32 0
+  %build = insertelement <2 x i16> poison, i16 %load, i32 0
   %result = bitcast <2 x i16> %build to i32
   store i32 %result, ptr addrspace(1) %out
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll b/llvm/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll
index db7897d70fa93..827cb4ac2589a 100644
--- a/llvm/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll
@@ -78,7 +78,7 @@ loop:                                             ; preds = %Flow, %start
   br i1 %2, label %endif1, label %Flow
 
 Flow1:                                            ; preds = %endif2, %endif1
-  %3 = phi i32 [ %v5, %endif2 ], [ undef, %endif1 ]
+  %3 = phi i32 [ %v5, %endif2 ], [ poison, %endif1 ]
   %4 = phi i1 [ false, %endif2 ], [ true, %endif1 ]
   br label %Flow
 
@@ -107,7 +107,7 @@ endif1:                                           ; preds = %loop
   br i1 %5, label %endif2, label %Flow1
 
 Flow:                                             ; preds = %Flow1, %loop
-  %6 = phi i32 [ %3, %Flow1 ], [ undef, %loop ]
+  %6 = phi i32 [ %3, %Flow1 ], [ poison, %loop ]
   %7 = phi i1 [ %4, %Flow1 ], [ true, %loop ]
   %8 = phi i1 [ false, %Flow1 ], [ true, %loop ]
   br i1 %7, label %Flow2, label %loop
diff --git a/llvm/test/CodeGen/AMDGPU/ds-combine-with-dependence.ll b/llvm/test/CodeGen/AMDGPU/ds-combine-with-dependence.ll
index 7d75f1947b51a..418023b1501b7 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-combine-with-dependence.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds-combine-with-dependence.ll
@@ -15,7 +15,7 @@ define amdgpu_kernel void @ds_combine_nodep(ptr addrspace(1) %out, ptr addrspace
   %load0 = load <3 x float>, ptr addrspace(3) %addr0, align 4
   %v0 = extractelement <3 x float> %load0, i32 2
 
-  %tmp1 = insertelement <2 x float> undef, float 1.0, i32 0
+  %tmp1 = insertelement <2 x float> poison, float 1.0, i32 0
   %data = insertelement <2 x float> %tmp1, float 2.0, i32 1
 
   %tmp2 = getelementptr float, ptr addrspace(3) %inptr, i32 26
@@ -43,7 +43,7 @@ define amdgpu_kernel void @ds_combine_WAR(ptr addrspace(1) %out, ptr addrspace(3
   %load0 = load <3 x float>, ptr addrspace(3) %addr0, align 4
   %v0 = extractelement <3 x float> %load0, i32 2
 
-  %tmp1 = insertelement <2 x float> undef, float 1.0, i32 0
+  %tmp1 = insertelement <2 x float> poison, float 1.0, i32 0
   %data = insertelement <2 x float> %tmp1, float 2.0, i32 1
 
   %tmp2 = getelementptr float, ptr addrspace(3) %inptr, i32 26
@@ -73,7 +73,7 @@ define amdgpu_kernel void @ds_combine_RAW(ptr addrspace(1) %out, ptr addrspace(3
   %load0 = load <3 x float>, ptr addrspace(3) %addr0, align 4
   %v0 = extractelement <3 x float> %load0, i32 2
 
-  %tmp1 = insertelement <2 x float> undef, float 1.0, i32 0
+  %tmp1 = insertelement <2 x float> poison, float 1.0, i32 0
   %data = insertelement <2 x float> %tmp1, float 2.0, i32 1
 
   %tmp2 = getelementptr float, ptr addrspace(3) %inptr, i32 26
@@ -102,7 +102,7 @@ define amdgpu_kernel void @ds_combine_WAR_RAW(ptr addrspace(1) %out, ptr addrspa
   %load0 = load <3 x float>, ptr addrspace(3) %addr0, align 4
   %v0 = extractelement <3 x float> %load0, i32 2
 
-  %tmp1 = insertelement <2 x float> undef, float 1.0, i32 0
+  %tmp1 = insertelement <2 x float> poison, float 1.0, i32 0
   %data = insertelement <2 x float> %tmp1, float 2.0, i32 1
 
   %tmp2 = getelementptr float, ptr addrspace(3) %inptr, i32 26
diff --git a/llvm/test/CodeGen/AMDGPU/ds_read2.ll b/llvm/test/CodeGen/AMDGPU/ds_read2.ll
index 4d9c85ef99dcd..c37c7777f617f 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_read2.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_read2.ll
@@ -331,7 +331,7 @@ define amdgpu_kernel void @read2_ptr_is_subreg_arg_f32(ptr addrspace(1) %out, <2
 ; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
-  %index.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0
+  %index.0 = insertelement <2 x i32> poison, i32 %x.i, i32 0
   %index.1 = insertelement <2 x i32> %index.0, i32 8, i32 0
   %gep = getelementptr inbounds float, <2 x ptr addrspace(3)> %lds.ptr, <2 x i32> %index.1
   %gep.0 = extractelement <2 x ptr addrspace(3)> %gep, i32 0
@@ -382,7 +382,7 @@ define amdgpu_kernel void @read2_ptr_is_subreg_arg_offset_f32(ptr addrspace(1) %
 ; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-NEXT:    s_endpgm
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
-  %index.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0
+  %index.0 = insertelement <2 x i32> poison, i32 %x.i, i32 0
   %index.1 = insertelement <2 x i32> %index.0, i32 8, i32 0
   %gep = getelementptr inbounds float, <2 x ptr addrspace(3)> %lds.ptr, <2 x i32> %index.1
   %gep.0 = extractelement <2 x ptr addrspace(3)> %gep, i32 0
@@ -425,9 +425,9 @@ define amdgpu_kernel void @read2_ptr_is_subreg_f32(ptr addrspace(1) %out) #0 {
 ; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]
 ; GFX9-NEXT:    s_endpgm
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
-  %ptr.0 = insertelement <2 x ptr addrspace(3)> undef, ptr addrspace(3) @lds, i32 0
+  %ptr.0 = insertelement <2 x ptr addrspace(3)> poison, ptr addrspace(3) @lds, i32 0
   %ptr.1 = insertelement <2 x ptr addrspace(3)> %ptr.0, ptr addrspace(3) @lds, i32 1
-  %x.i.v.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0
+  %x.i.v.0 = insertelement <2 x i32> poison, i32 %x.i, i32 0
   %x.i.v.1 = insertelement <2 x i32> %x.i.v.0, i32 %x.i, i32 1
   %idx = add <2 x i32> %x.i.v.1, <i32 0, i32 8>
   %gep = getelementptr inbounds [512 x float], <2 x ptr addrspace(3)> %ptr.1, <2 x i32> <i32 0, i32 0>, <2 x i32> %idx
@@ -1321,19 +1321,19 @@ bb:
 define amdgpu_kernel void @ds_read_call_read(ptr addrspace(1) %out, ptr addrspace(3) %arg) {
 ; CI-LABEL: ds_read_call_read:
 ; CI:       ; %bb.0:
-; CI-NEXT:    s_getpc_b64 s[40:41]
-; CI-NEXT:    s_mov_b32 s40, s0
-; CI-NEXT:    s_load_dwordx4 s[40:43], s[40:41], 0x0
+; CI-NEXT:    s_getpc_b64 s[48:49]
+; CI-NEXT:    s_mov_b32 s48, s0
+; CI-NEXT:    s_load_dwordx4 s[48:51], s[48:49], 0x0
 ; CI-NEXT:    s_mov_b32 s14, s10
 ; CI-NEXT:    v_lshlrev_b32_e32 v3, 2, v0
 ; CI-NEXT:    s_mov_b32 m0, -1
 ; CI-NEXT:    s_mov_b32 s12, s8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-NEXT:    s_add_u32 s40, s40, s11
+; CI-NEXT:    s_add_u32 s48, s48, s11
 ; CI-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; CI-NEXT:    s_load_dwordx2 s[36:37], s[4:5], 0x0
 ; CI-NEXT:    s_load_dword s6, s[4:5], 0x2
-; CI-NEXT:    s_addc_u32 s41, s41, 0
+; CI-NEXT:    s_addc_u32 s49, s49, 0
 ; CI-NEXT:    s_add_u32 s8, s4, 12
 ; CI-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; CI-NEXT:    s_mov_b32 s13, s9
@@ -1345,11 +1345,11 @@ define amdgpu_kernel void @ds_read_call_read(ptr addrspace(1) %out, ptr addrspac
 ; CI-NEXT:    v_or_b32_e32 v0, v0, v1
 ; CI-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; CI-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; CI-NEXT:    s_mov_b64 s[0:1], s[40:41]
+; CI-NEXT:    s_mov_b64 s[0:1], s[48:49]
 ; CI-NEXT:    s_mov_b32 s17, void_func_void@abs32@hi
 ; CI-NEXT:    s_mov_b32 s16, void_func_void@abs32@lo
 ; CI-NEXT:    v_or_b32_e32 v31, v0, v2
-; CI-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; CI-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; CI-NEXT:    s_mov_b32 s32, 0
 ; CI-NEXT:    s_mov_b32 s39, 0xf000
 ; CI-NEXT:    s_mov_b32 s38, -1
@@ -1435,7 +1435,7 @@ define amdgpu_ps <2 x float> @ds_read_interp_read(i32 inreg %prims, ptr addrspac
   %ptr1 = getelementptr float, ptr addrspace(3) %inptr, i32 4
   %v1 = load float, ptr addrspace(3) %ptr1, align 4
   %v1b = fadd float %v1, %intrp
-  %r0 = insertelement <2 x float> undef, float %v0, i32 0
+  %r0 = insertelement <2 x float> poison, float %v0, i32 0
   %r1 = insertelement <2 x float> %r0, float %v1b, i32 1
   ret <2 x float> %r1
 }
diff --git a/llvm/test/CodeGen/AMDGPU/ds_read2_superreg.ll b/llvm/test/CodeGen/AMDGPU/ds_read2_superreg.ll
index 3aff5ccff4280..bdc31d9161388 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_read2_superreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_read2_superreg.ll
@@ -161,7 +161,7 @@ define amdgpu_kernel void @simple_read2_v2f32_superreg_scalar_loads_align4(ptr a
   %val0 = load float, ptr addrspace(3) %arrayidx0
   %val1 = load float, ptr addrspace(3) %arrayidx1
 
-  %vec.0 = insertelement <2 x float> undef, float %val0, i32 0
+  %vec.0 = insertelement <2 x float> poison, float %val0, i32 0
   %vec.1 = insertelement <2 x float> %vec.0, float %val1, i32 1
 
   %out.gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %out, i32 %x.i
@@ -188,7 +188,7 @@ define amdgpu_kernel void @simple_read2_v4f32_superreg_scalar_loads_align4(ptr a
   %val2 = load float, ptr addrspace(3) %arrayidx2
   %val3 = load float, ptr addrspace(3) %arrayidx3
 
-  %vec.0 = insertelement <4 x float> undef, float %val0, i32 0
+  %vec.0 = insertelement <4 x float> poison, float %val0, i32 0
   %vec.1 = insertelement <4 x float> %vec.0, float %val1, i32 1
   %vec.2 = insertelement <4 x float> %vec.1, float %val2, i32 2
   %vec.3 = insertelement <4 x float> %vec.2, float %val3, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/ds_write2.ll b/llvm/test/CodeGen/AMDGPU/ds_write2.ll
index 1f805b6d07f71..e49b181049cc5 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_write2.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_write2.ll
@@ -548,7 +548,7 @@ define amdgpu_kernel void @write2_ptr_subreg_arg_two_val_f32(ptr addrspace(1) %C
   %val0 = load float, ptr addrspace(1) %in0.gep, align 4
   %val1 = load float, ptr addrspace(1) %in1.gep, align 4
 
-  %index.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0
+  %index.0 = insertelement <2 x i32> poison, i32 %x.i, i32 0
   %index.1 = insertelement <2 x i32> %index.0, i32 8, i32 0
   %gep = getelementptr inbounds float, <2 x ptr addrspace(3)> %lds.ptr, <2 x i32> %index.1
   %gep.0 = extractelement <2 x ptr addrspace(3)> %gep, i32 0
diff --git a/llvm/test/CodeGen/AMDGPU/dwarf-multi-register-use-crash.ll b/llvm/test/CodeGen/AMDGPU/dwarf-multi-register-use-crash.ll
index 8b02bdbb70b7b..40cdfd76d6af6 100644
--- a/llvm/test/CodeGen/AMDGPU/dwarf-multi-register-use-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/dwarf-multi-register-use-crash.ll
@@ -30,59 +30,59 @@ define weak_odr void @test(i32 %0) !dbg !34 {
 ; CHECK-NEXT:    v_writelane_b32 v41, s37, 5
 ; CHECK-NEXT:    v_writelane_b32 v41, s38, 6
 ; CHECK-NEXT:    v_writelane_b32 v41, s39, 7
-; CHECK-NEXT:    v_writelane_b32 v41, s40, 8
-; CHECK-NEXT:    v_writelane_b32 v41, s41, 9
-; CHECK-NEXT:    v_writelane_b32 v41, s42, 10
-; CHECK-NEXT:    v_writelane_b32 v41, s43, 11
-; CHECK-NEXT:    v_writelane_b32 v41, s44, 12
+; CHECK-NEXT:    v_writelane_b32 v41, s48, 8
+; CHECK-NEXT:    v_writelane_b32 v41, s49, 9
+; CHECK-NEXT:    v_writelane_b32 v41, s50, 10
+; CHECK-NEXT:    v_writelane_b32 v41, s51, 11
+; CHECK-NEXT:    v_writelane_b32 v41, s52, 12
 ; CHECK-NEXT:    s_addk_i32 s32, 0x400
-; CHECK-NEXT:    v_writelane_b32 v41, s45, 13
-; CHECK-NEXT:    v_writelane_b32 v41, s46, 14
-; CHECK-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; CHECK-NEXT:    v_writelane_b32 v41, s53, 13
+; CHECK-NEXT:    v_writelane_b32 v41, s54, 14
+; CHECK-NEXT:    s_mov_b64 s[48:49], s[4:5]
 ; CHECK-NEXT:    ;DEBUG_VALUE: dummy:dummy <- undef
 ; CHECK-NEXT:  .Ltmp0:
 ; CHECK-NEXT:    .loc 1 49 9 prologue_end ; dummy:49:9
 ; CHECK-NEXT:    s_getpc_b64 s[4:5]
 ; CHECK-NEXT:    s_add_u32 s4, s4, __kmpc_alloc_shared@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, __kmpc_alloc_shared@gotpcrel32@hi+12
-; CHECK-NEXT:    v_writelane_b32 v41, s47, 15
-; CHECK-NEXT:    s_load_dwordx2 s[46:47], s[4:5], 0x0
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    v_writelane_b32 v41, s55, 15
+; CHECK-NEXT:    s_load_dwordx2 s[54:55], s[4:5], 0x0
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; CHECK-NEXT:    v_mov_b32_e32 v40, v31
-; CHECK-NEXT:    s_mov_b32 s42, s15
-; CHECK-NEXT:    s_mov_b32 s43, s14
-; CHECK-NEXT:    s_mov_b32 s44, s13
-; CHECK-NEXT:    s_mov_b32 s45, s12
+; CHECK-NEXT:    s_mov_b32 s50, s15
+; CHECK-NEXT:    s_mov_b32 s51, s14
+; CHECK-NEXT:    s_mov_b32 s52, s13
+; CHECK-NEXT:    s_mov_b32 s53, s12
 ; CHECK-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; CHECK-NEXT:    s_mov_b64 s[36:37], s[8:9]
 ; CHECK-NEXT:    s_mov_b64 s[38:39], s[6:7]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    s_swappc_b64 s[30:31], s[46:47]
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_swappc_b64 s[30:31], s[54:55]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[36:37]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; CHECK-NEXT:    s_mov_b32 s12, s45
-; CHECK-NEXT:    s_mov_b32 s13, s44
-; CHECK-NEXT:    s_mov_b32 s14, s43
-; CHECK-NEXT:    s_mov_b32 s15, s42
+; CHECK-NEXT:    s_mov_b32 s12, s53
+; CHECK-NEXT:    s_mov_b32 s13, s52
+; CHECK-NEXT:    s_mov_b32 s14, s51
+; CHECK-NEXT:    s_mov_b32 s15, s50
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
-; CHECK-NEXT:    s_swappc_b64 s[30:31], s[46:47]
+; CHECK-NEXT:    s_swappc_b64 s[30:31], s[54:55]
 ; CHECK-NEXT:  .Ltmp1:
 ; CHECK-NEXT:    ;DEBUG_VALUE: dummy:dummy <- [$vgpr0_vgpr1+0]
 ; CHECK-NEXT:    .loc 1 0 9 is_stmt 0 ; dummy:0:9
 ; CHECK-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 0
 ; CHECK-NEXT:    flat_store_dword v[0:1], v2
-; CHECK-NEXT:    v_readlane_b32 s47, v41, 15
-; CHECK-NEXT:    v_readlane_b32 s46, v41, 14
-; CHECK-NEXT:    v_readlane_b32 s45, v41, 13
-; CHECK-NEXT:    v_readlane_b32 s44, v41, 12
-; CHECK-NEXT:    v_readlane_b32 s43, v41, 11
-; CHECK-NEXT:    v_readlane_b32 s42, v41, 10
-; CHECK-NEXT:    v_readlane_b32 s41, v41, 9
-; CHECK-NEXT:    v_readlane_b32 s40, v41, 8
+; CHECK-NEXT:    v_readlane_b32 s55, v41, 15
+; CHECK-NEXT:    v_readlane_b32 s54, v41, 14
+; CHECK-NEXT:    v_readlane_b32 s53, v41, 13
+; CHECK-NEXT:    v_readlane_b32 s52, v41, 12
+; CHECK-NEXT:    v_readlane_b32 s51, v41, 11
+; CHECK-NEXT:    v_readlane_b32 s50, v41, 10
+; CHECK-NEXT:    v_readlane_b32 s49, v41, 9
+; CHECK-NEXT:    v_readlane_b32 s48, v41, 8
 ; CHECK-NEXT:    v_readlane_b32 s39, v41, 7
 ; CHECK-NEXT:    v_readlane_b32 s38, v41, 6
 ; CHECK-NEXT:    v_readlane_b32 s37, v41, 5
diff --git a/llvm/test/CodeGen/AMDGPU/early-if-convert.ll b/llvm/test/CodeGen/AMDGPU/early-if-convert.ll
index 028cd9110eb2b..8be5d1a3fde7c 100644
--- a/llvm/test/CodeGen/AMDGPU/early-if-convert.ll
+++ b/llvm/test/CodeGen/AMDGPU/early-if-convert.ll
@@ -331,7 +331,7 @@ if:
 
 endif:
   %r = phi <3 x i32> [ %v, %entry ], [ %u, %if ]
-  %r.ext = shufflevector <3 x i32> %r, <3 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %r.ext = shufflevector <3 x i32> %r, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   call void asm sideeffect "; reg use $0", "s"(<4 x i32> %r.ext) #0
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/eliminate-frame-index-s-mov-b32.mir b/llvm/test/CodeGen/AMDGPU/eliminate-frame-index-s-mov-b32.mir
index 0714def30053d..7f370b2cca658 100644
--- a/llvm/test/CodeGen/AMDGPU/eliminate-frame-index-s-mov-b32.mir
+++ b/llvm/test/CodeGen/AMDGPU/eliminate-frame-index-s-mov-b32.mir
@@ -142,9 +142,8 @@ body:             |
     ; GFX1100-NEXT: renamable $sgpr20 = S_MOV_B32 killed $sgpr22
     ; GFX1100-NEXT: undef renamable $sgpr22 = COPY killed undef renamable $sgpr22, implicit-def $sgpr22_sgpr23
     ; GFX1100-NEXT: undef renamable $sgpr26 = COPY killed undef renamable $sgpr26, implicit-def $sgpr26_sgpr27
-    ; GFX1100-NEXT: $sgpr32 = S_ADD_I32 $sgpr32, 8, implicit-def $scc
-    ; GFX1100-NEXT: renamable $sgpr31 = S_MOV_B32 $sgpr32
-    ; GFX1100-NEXT: $sgpr32 = S_ADD_I32 $sgpr32, -8, implicit-def $scc
+    ; GFX1100-NEXT: $sgpr40 = S_ADD_I32 $sgpr32, 8, implicit-def $scc
+    ; GFX1100-NEXT: renamable $sgpr31 = S_MOV_B32 killed $sgpr40
     ; GFX1100-NEXT: renamable $vgpr3 = COPY killed renamable $sgpr30, implicit $exec
     ; GFX1100-NEXT: renamable $vgpr0_vgpr1 = COPY renamable $sgpr28_sgpr29, implicit $exec
     ; GFX1100-NEXT: renamable $vgpr0_vgpr1 = COPY killed renamable $vcc, implicit $exec
@@ -168,9 +167,8 @@ body:             |
     ; GFX1200-NEXT: renamable $sgpr20 = S_MOV_B32 killed $sgpr22
     ; GFX1200-NEXT: undef renamable $sgpr22 = COPY killed undef renamable $sgpr22, implicit-def $sgpr22_sgpr23
     ; GFX1200-NEXT: undef renamable $sgpr26 = COPY killed undef renamable $sgpr26, implicit-def $sgpr26_sgpr27
-    ; GFX1200-NEXT: $sgpr32 = S_ADD_I32 $sgpr32, 8, implicit-def $scc
-    ; GFX1200-NEXT: renamable $sgpr31 = S_MOV_B32 $sgpr32
-    ; GFX1200-NEXT: $sgpr32 = S_ADD_I32 $sgpr32, -8, implicit-def $scc
+    ; GFX1200-NEXT: $sgpr40 = S_ADD_I32 $sgpr32, 8, implicit-def $scc
+    ; GFX1200-NEXT: renamable $sgpr31 = S_MOV_B32 killed $sgpr40
     ; GFX1200-NEXT: renamable $vgpr3 = COPY killed renamable $sgpr30, implicit $exec
     ; GFX1200-NEXT: renamable $vgpr0_vgpr1 = COPY renamable $sgpr28_sgpr29, implicit $exec
     ; GFX1200-NEXT: renamable $vgpr0_vgpr1 = COPY killed renamable $vcc, implicit $exec
@@ -706,8 +704,9 @@ body:   |
     ; GFX8-NEXT: S_NOP 0, implicit-def $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63
     ; GFX8-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
     ; GFX8-NEXT: S_CMP_EQ_I32 $sgpr4, $sgpr5, implicit-def $scc
-    ; GFX8-NEXT: $vgpr0 = V_MAD_U32_U24_e64 24, 64, $sgpr32, 0, implicit $exec
-    ; GFX8-NEXT: $vgpr0 = V_LSHRREV_B32_e64 6, $vgpr0, implicit $exec
+    ; GFX8-NEXT: $vgpr0 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX8-NEXT: $sgpr4 = S_MOV_B32 24
+    ; GFX8-NEXT: $vgpr0, dead $sgpr72_sgpr73 = V_ADD_CO_U32_e64 killed $sgpr4, killed $vgpr0, 0, implicit $exec
     ; GFX8-NEXT: $sgpr4 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec
     ; GFX8-NEXT: S_NOP 0, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX8-NEXT: S_NOP 0, implicit $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
@@ -810,10 +809,10 @@ body:   |
     ; GFX1100-NEXT: S_NOP 0, implicit-def $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63
     ; GFX1100-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
     ; GFX1100-NEXT: S_CMP_EQ_I32 $sgpr4, $sgpr5, implicit-def $scc
-    ; GFX1100-NEXT: $vcc_hi = S_ADDC_U32 $sgpr32, 24, implicit-def $scc, implicit $scc
-    ; GFX1100-NEXT: S_BITCMP1_B32 $vcc_hi, 0, implicit-def $scc
-    ; GFX1100-NEXT: $vcc_hi = S_BITSET0_B32 0, $vcc_hi
-    ; GFX1100-NEXT: renamable $sgpr4 = S_MOV_B32 killed $vcc_hi
+    ; GFX1100-NEXT: $sgpr72 = S_ADDC_U32 $sgpr32, 24, implicit-def $scc, implicit $scc
+    ; GFX1100-NEXT: S_BITCMP1_B32 $sgpr72, 0, implicit-def $scc
+    ; GFX1100-NEXT: $sgpr72 = S_BITSET0_B32 0, $sgpr72
+    ; GFX1100-NEXT: renamable $sgpr4 = S_MOV_B32 killed $sgpr72
     ; GFX1100-NEXT: S_NOP 0, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX1100-NEXT: S_NOP 0, implicit $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GFX1100-NEXT: S_NOP 0, implicit $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23
@@ -837,10 +836,10 @@ body:   |
     ; GFX1200-NEXT: S_NOP 0, implicit-def $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63
     ; GFX1200-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
     ; GFX1200-NEXT: S_CMP_EQ_I32 $sgpr4, $sgpr5, implicit-def $scc
-    ; GFX1200-NEXT: $vcc_hi = S_ADDC_U32 $sgpr32, 24, implicit-def $scc, implicit $scc
-    ; GFX1200-NEXT: S_BITCMP1_B32 $vcc_hi, 0, implicit-def $scc
-    ; GFX1200-NEXT: $vcc_hi = S_BITSET0_B32 0, $vcc_hi
-    ; GFX1200-NEXT: renamable $sgpr4 = S_MOV_B32 killed $vcc_hi
+    ; GFX1200-NEXT: $sgpr72 = S_ADDC_U32 $sgpr32, 24, implicit-def $scc, implicit $scc
+    ; GFX1200-NEXT: S_BITCMP1_B32 $sgpr72, 0, implicit-def $scc
+    ; GFX1200-NEXT: $sgpr72 = S_BITSET0_B32 0, $sgpr72
+    ; GFX1200-NEXT: renamable $sgpr4 = S_MOV_B32 killed $sgpr72
     ; GFX1200-NEXT: S_NOP 0, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX1200-NEXT: S_NOP 0, implicit $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GFX1200-NEXT: S_NOP 0, implicit $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23
@@ -900,9 +899,9 @@ body:             |
     ; GFX8-NEXT: S_NOP 0, implicit-def $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63
     ; GFX8-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
     ; GFX8-NEXT: S_CMP_EQ_I32 $sgpr4, $sgpr5, implicit-def $scc
-    ; GFX8-NEXT: $vgpr0 = V_MOV_B32_e32 68, implicit $exec
-    ; GFX8-NEXT: $vgpr0 = V_MAD_U32_U24_e64 killed $vgpr0, 64, $sgpr32, 0, implicit $exec
-    ; GFX8-NEXT: $vgpr0 = V_LSHRREV_B32_e64 6, $vgpr0, implicit $exec
+    ; GFX8-NEXT: $vgpr0 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
+    ; GFX8-NEXT: $sgpr4 = S_MOV_B32 68
+    ; GFX8-NEXT: $vgpr0, dead $sgpr72_sgpr73 = V_ADD_CO_U32_e64 killed $sgpr4, killed $vgpr0, 0, implicit $exec
     ; GFX8-NEXT: $sgpr4 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec
     ; GFX8-NEXT: S_NOP 0, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX8-NEXT: S_NOP 0, implicit $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
@@ -1005,10 +1004,10 @@ body:             |
     ; GFX1100-NEXT: S_NOP 0, implicit-def $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63
     ; GFX1100-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
     ; GFX1100-NEXT: S_CMP_EQ_I32 $sgpr4, $sgpr5, implicit-def $scc
-    ; GFX1100-NEXT: $vcc_hi = S_ADDC_U32 $sgpr32, 68, implicit-def $scc, implicit $scc
-    ; GFX1100-NEXT: S_BITCMP1_B32 $vcc_hi, 0, implicit-def $scc
-    ; GFX1100-NEXT: $vcc_hi = S_BITSET0_B32 0, $vcc_hi
-    ; GFX1100-NEXT: renamable $sgpr4 = S_MOV_B32 killed $vcc_hi
+    ; GFX1100-NEXT: $sgpr72 = S_ADDC_U32 $sgpr32, 68, implicit-def $scc, implicit $scc
+    ; GFX1100-NEXT: S_BITCMP1_B32 $sgpr72, 0, implicit-def $scc
+    ; GFX1100-NEXT: $sgpr72 = S_BITSET0_B32 0, $sgpr72
+    ; GFX1100-NEXT: renamable $sgpr4 = S_MOV_B32 killed $sgpr72
     ; GFX1100-NEXT: S_NOP 0, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX1100-NEXT: S_NOP 0, implicit $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GFX1100-NEXT: S_NOP 0, implicit $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23
@@ -1032,10 +1031,10 @@ body:             |
     ; GFX1200-NEXT: S_NOP 0, implicit-def $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63
     ; GFX1200-NEXT: V_CMP_EQ_U32_e32 0, killed $vgpr0, implicit-def $vcc_lo, implicit $exec
     ; GFX1200-NEXT: S_CMP_EQ_I32 $sgpr4, $sgpr5, implicit-def $scc
-    ; GFX1200-NEXT: $vcc_hi = S_ADDC_U32 $sgpr32, 68, implicit-def $scc, implicit $scc
-    ; GFX1200-NEXT: S_BITCMP1_B32 $vcc_hi, 0, implicit-def $scc
-    ; GFX1200-NEXT: $vcc_hi = S_BITSET0_B32 0, $vcc_hi
-    ; GFX1200-NEXT: renamable $sgpr4 = S_MOV_B32 killed $vcc_hi
+    ; GFX1200-NEXT: $sgpr72 = S_ADDC_U32 $sgpr32, 68, implicit-def $scc, implicit $scc
+    ; GFX1200-NEXT: S_BITCMP1_B32 $sgpr72, 0, implicit-def $scc
+    ; GFX1200-NEXT: $sgpr72 = S_BITSET0_B32 0, $sgpr72
+    ; GFX1200-NEXT: renamable $sgpr4 = S_MOV_B32 killed $sgpr72
     ; GFX1200-NEXT: S_NOP 0, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX1200-NEXT: S_NOP 0, implicit $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GFX1200-NEXT: S_NOP 0, implicit $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23
diff --git a/llvm/test/CodeGen/AMDGPU/enqueue-kernel.ll b/llvm/test/CodeGen/AMDGPU/enqueue-kernel.ll
deleted file mode 100644
index d7c8e47f98883..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/enqueue-kernel.ll
+++ /dev/null
@@ -1,215 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-attributes --check-globals --include-generated-funcs
-; RUN: opt -data-layout=A5 -amdgpu-lower-enqueued-block -S < %s | FileCheck %s
-; RUN: opt -data-layout=A5 -mtriple=amdgcn -passes=amdgpu-lower-enqueued-block -S < %s | FileCheck %s
-
-%struct.ndrange_t = type { i32 }
-%opencl.queue_t = type opaque
-
-define amdgpu_kernel void @non_caller(ptr addrspace(1) %a, i8 %b, ptr addrspace(1) %c, i64 %d) {
-  ret void
-}
-
-define amdgpu_kernel void @caller(ptr addrspace(1) %a, i8 %b, ptr addrspace(1) %c, i64 %d) {
-entry:
-  %block = alloca <{ i32, i32, ptr addrspace(1), i8 }>, align 8, addrspace(5)
-  %inst = alloca %struct.ndrange_t, align 4, addrspace(5)
-  %block2 = alloca <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, align 8, addrspace(5)
-  %inst3 = alloca %struct.ndrange_t, align 4, addrspace(5)
-  %block.size = getelementptr inbounds <{ i32, i32, ptr addrspace(1), i8 }>, ptr addrspace(5) %block, i32 0, i32 0
-  store i32 25, ptr addrspace(5) %block.size, align 8
-  %block.align = getelementptr inbounds <{ i32, i32, ptr addrspace(1), i8 }>, ptr addrspace(5) %block, i32 0, i32 1
-  store i32 8, ptr addrspace(5) %block.align, align 4
-  %block.captured = getelementptr inbounds <{ i32, i32, ptr addrspace(1), i8 }>, ptr addrspace(5) %block, i32 0, i32 2
-  store ptr addrspace(1) %a, ptr addrspace(5) %block.captured, align 8
-  %block.captured1 = getelementptr inbounds <{ i32, i32, ptr addrspace(1), i8 }>, ptr addrspace(5) %block, i32 0, i32 3
-  store i8 %b, ptr addrspace(5) %block.captured1, align 8
-  %inst4 = addrspacecast ptr addrspace(5) %block to ptr
-  %inst5 = call i32 @__enqueue_kernel_basic(ptr addrspace(1) undef, i32 0, ptr addrspace(5) byval(%struct.ndrange_t) nonnull %inst,
-  ptr @__test_block_invoke_kernel, ptr nonnull %inst4) #2
-  %inst10 = call i32 @__enqueue_kernel_basic(ptr addrspace(1) undef, i32 0, ptr addrspace(5) byval(%struct.ndrange_t) nonnull %inst,
-  ptr @__test_block_invoke_kernel, ptr nonnull %inst4) #2
-  %inst11 = call i32 @__enqueue_kernel_basic(ptr addrspace(1) undef, i32 0, ptr addrspace(5) byval(%struct.ndrange_t) nonnull %inst,
-  ptr @0, ptr nonnull %inst4) #2
-  %inst12 = call i32 @__enqueue_kernel_basic(ptr addrspace(1) undef, i32 0, ptr addrspace(5) byval(%struct.ndrange_t) nonnull %inst,
-  ptr @1, ptr nonnull %inst4) #2
-  %block.size4 = getelementptr inbounds <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr addrspace(5) %block2, i32 0, i32 0
-  store i32 41, ptr addrspace(5) %block.size4, align 8
-  %block.align5 = getelementptr inbounds <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr addrspace(5) %block2, i32 0, i32 1
-  store i32 8, ptr addrspace(5) %block.align5, align 4
-  %block.captured7 = getelementptr inbounds <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr addrspace(5) %block2, i32 0, i32 2
-  store ptr addrspace(1) %a, ptr addrspace(5) %block.captured7, align 8
-  %block.captured8 = getelementptr inbounds <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr addrspace(5) %block2, i32 0, i32 5
-  store i8 %b, ptr addrspace(5) %block.captured8, align 8
-  %block.captured9 = getelementptr inbounds <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr addrspace(5) %block2, i32 0, i32 3
-  store ptr addrspace(1) %c, ptr addrspace(5) %block.captured9, align 8
-  %block.captured10 = getelementptr inbounds <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr addrspace(5) %block2, i32 0, i32 4
-  store i64 %d, ptr addrspace(5) %block.captured10, align 8
-  %inst8 = addrspacecast ptr addrspace(5) %block2 to ptr
-  %inst9 = call i32 @__enqueue_kernel_basic(ptr addrspace(1) undef, i32 0, ptr addrspace(5) byval(%struct.ndrange_t) nonnull %inst3,
-  ptr @__test_block_invoke_2_kernel, ptr nonnull %inst8) #2
-  ret void
-}
-
-; __enqueue_kernel* functions may get inlined
-define amdgpu_kernel void @inlined_caller(ptr addrspace(1) %a, i8 %b, ptr addrspace(1) %c, i64 %d) {
-entry:
-  %inst = load i64, ptr addrspace(1) addrspacecast (ptr @__test_block_invoke_kernel to ptr addrspace(1))
-  store i64 %inst, ptr addrspace(1) %c
-  ret void
-}
-
-define internal amdgpu_kernel void @__test_block_invoke_kernel(<{ i32, i32, ptr addrspace(1), i8 }> %arg) #0 {
-entry:
-  %.fca.3.extract = extractvalue <{ i32, i32, ptr addrspace(1), i8 }> %arg, 2
-  %.fca.4.extract = extractvalue <{ i32, i32, ptr addrspace(1), i8 }> %arg, 3
-  store i8 %.fca.4.extract, ptr addrspace(1) %.fca.3.extract, align 1
-  ret void
-}
-
-declare i32 @__enqueue_kernel_basic(ptr addrspace(1), i32, ptr addrspace(5), ptr, ptr) local_unnamed_addr
-
-define internal amdgpu_kernel void @__test_block_invoke_2_kernel(<{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }> %arg) #0 {
-entry:
-  %.fca.3.extract = extractvalue <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }> %arg, 2
-  %.fca.4.extract = extractvalue <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }> %arg, 3
-  %.fca.5.extract = extractvalue <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }> %arg, 4
-  %.fca.6.extract = extractvalue <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }> %arg, 5
-  store i8 %.fca.6.extract, ptr addrspace(1) %.fca.3.extract, align 1
-  store i64 %.fca.5.extract, ptr addrspace(1) %.fca.4.extract, align 8
-  ret void
-}
-
-@kernel_address_user = global [1 x ptr] [ ptr @block_has_used_kernel_address ]
-
-define internal amdgpu_kernel void @block_has_used_kernel_address(<{ i32, i32, ptr addrspace(1), i8 }> %arg) #0 {
-entry:
-  %.fca.3.extract = extractvalue <{ i32, i32, ptr addrspace(1), i8 }> %arg, 2
-  %.fca.4.extract = extractvalue <{ i32, i32, ptr addrspace(1), i8 }> %arg, 3
-  store i8 %.fca.4.extract, ptr addrspace(1) %.fca.3.extract, align 1
-  ret void
-}
-
-define amdgpu_kernel void @user_of_kernel_address(ptr addrspace(1) %arg) {
-  store ptr @block_has_used_kernel_address, ptr addrspace(1) %arg
-  ret void
-}
-
-define internal amdgpu_kernel void @0(<{ i32, i32, ptr addrspace(1), i8 }> %arg) #0 {
-  ret void
-}
-
-define internal amdgpu_kernel void @1(<{ i32, i32, ptr addrspace(1), i8 }> %arg) #0 {
-  ret void
-}
-
-attributes #0 = { "enqueued-block" }
-;.
-; CHECK: @[[KERNEL_ADDRESS_USER:[a-zA-Z0-9_$"\\.-]+]] = global [1 x ptr] [ptr addrspacecast (ptr addrspace(1) @block_has_used_kernel_address.runtime_handle to ptr)]
-; CHECK: @[[__TEST_BLOCK_INVOKE_KERNEL_RUNTIME_HANDLE:[a-zA-Z0-9_$"\\.-]+]] = addrspace(1) externally_initialized constant [[BLOCK_RUNTIME_HANDLE_T:%.*]] zeroinitializer
-; CHECK: @[[__TEST_BLOCK_INVOKE_2_KERNEL_RUNTIME_HANDLE:[a-zA-Z0-9_$"\\.-]+]] = addrspace(1) externally_initialized constant [[BLOCK_RUNTIME_HANDLE_T:%.*]] zeroinitializer
-; CHECK: @[[BLOCK_HAS_USED_KERNEL_ADDRESS_RUNTIME_HANDLE:[a-zA-Z0-9_$"\\.-]+]] = addrspace(1) externally_initialized constant [[BLOCK_RUNTIME_HANDLE_T:%.*]] zeroinitializer
-; CHECK: @[[__AMDGPU_ENQUEUED_KERNEL_RUNTIME_HANDLE:[a-zA-Z0-9_$"\\.-]+]] = addrspace(1) externally_initialized constant [[BLOCK_RUNTIME_HANDLE_T:%.*]] zeroinitializer
-; CHECK: @[[__AMDGPU_ENQUEUED_KERNEL_1_RUNTIME_HANDLE:[a-zA-Z0-9_$"\\.-]+]] = addrspace(1) externally_initialized constant [[BLOCK_RUNTIME_HANDLE_T:%.*]] zeroinitializer
-;.
-; CHECK-LABEL: define {{[^@]+}}@non_caller
-; CHECK-SAME: (ptr addrspace(1) [[A:%.*]], i8 [[B:%.*]], ptr addrspace(1) [[C:%.*]], i64 [[D:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-;
-; CHECK-LABEL: define {{[^@]+}}@caller
-; CHECK-SAME: (ptr addrspace(1) [[A:%.*]], i8 [[B:%.*]], ptr addrspace(1) [[C:%.*]], i64 [[D:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[BLOCK:%.*]] = alloca <{ i32, i32, ptr addrspace(1), i8 }>, align 8, addrspace(5)
-; CHECK-NEXT:    [[INST:%.*]] = alloca [[STRUCT_NDRANGE_T:%.*]], align 4, addrspace(5)
-; CHECK-NEXT:    [[BLOCK2:%.*]] = alloca <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, align 8, addrspace(5)
-; CHECK-NEXT:    [[INST3:%.*]] = alloca [[STRUCT_NDRANGE_T]], align 4, addrspace(5)
-; CHECK-NEXT:    [[BLOCK_SIZE:%.*]] = getelementptr inbounds <{ i32, i32, ptr addrspace(1), i8 }>, ptr addrspace(5) [[BLOCK]], i32 0, i32 0
-; CHECK-NEXT:    store i32 25, ptr addrspace(5) [[BLOCK_SIZE]], align 8
-; CHECK-NEXT:    [[BLOCK_ALIGN:%.*]] = getelementptr inbounds <{ i32, i32, ptr addrspace(1), i8 }>, ptr addrspace(5) [[BLOCK]], i32 0, i32 1
-; CHECK-NEXT:    store i32 8, ptr addrspace(5) [[BLOCK_ALIGN]], align 4
-; CHECK-NEXT:    [[BLOCK_CAPTURED:%.*]] = getelementptr inbounds <{ i32, i32, ptr addrspace(1), i8 }>, ptr addrspace(5) [[BLOCK]], i32 0, i32 2
-; CHECK-NEXT:    store ptr addrspace(1) [[A]], ptr addrspace(5) [[BLOCK_CAPTURED]], align 8
-; CHECK-NEXT:    [[BLOCK_CAPTURED1:%.*]] = getelementptr inbounds <{ i32, i32, ptr addrspace(1), i8 }>, ptr addrspace(5) [[BLOCK]], i32 0, i32 3
-; CHECK-NEXT:    store i8 [[B]], ptr addrspace(5) [[BLOCK_CAPTURED1]], align 8
-; CHECK-NEXT:    [[INST4:%.*]] = addrspacecast ptr addrspace(5) [[BLOCK]] to ptr
-; CHECK-NEXT:    [[INST5:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) undef, i32 0, ptr addrspace(5) nonnull byval([[STRUCT_NDRANGE_T]]) [[INST]], ptr addrspacecast (ptr addrspace(1) @__test_block_invoke_kernel.runtime_handle to ptr), ptr nonnull [[INST4]])
-; CHECK-NEXT:    [[INST10:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) undef, i32 0, ptr addrspace(5) nonnull byval([[STRUCT_NDRANGE_T]]) [[INST]], ptr addrspacecast (ptr addrspace(1) @__test_block_invoke_kernel.runtime_handle to ptr), ptr nonnull [[INST4]])
-; CHECK-NEXT:    [[INST11:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) undef, i32 0, ptr addrspace(5) nonnull byval([[STRUCT_NDRANGE_T]]) [[INST]], ptr addrspacecast (ptr addrspace(1) @__amdgpu_enqueued_kernel.runtime_handle to ptr), ptr nonnull [[INST4]])
-; CHECK-NEXT:    [[INST12:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) undef, i32 0, ptr addrspace(5) nonnull byval([[STRUCT_NDRANGE_T]]) [[INST]], ptr addrspacecast (ptr addrspace(1) @__amdgpu_enqueued_kernel.1.runtime_handle to ptr), ptr nonnull [[INST4]])
-; CHECK-NEXT:    [[BLOCK_SIZE4:%.*]] = getelementptr inbounds <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr addrspace(5) [[BLOCK2]], i32 0, i32 0
-; CHECK-NEXT:    store i32 41, ptr addrspace(5) [[BLOCK_SIZE4]], align 8
-; CHECK-NEXT:    [[BLOCK_ALIGN5:%.*]] = getelementptr inbounds <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr addrspace(5) [[BLOCK2]], i32 0, i32 1
-; CHECK-NEXT:    store i32 8, ptr addrspace(5) [[BLOCK_ALIGN5]], align 4
-; CHECK-NEXT:    [[BLOCK_CAPTURED7:%.*]] = getelementptr inbounds <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr addrspace(5) [[BLOCK2]], i32 0, i32 2
-; CHECK-NEXT:    store ptr addrspace(1) [[A]], ptr addrspace(5) [[BLOCK_CAPTURED7]], align 8
-; CHECK-NEXT:    [[BLOCK_CAPTURED8:%.*]] = getelementptr inbounds <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr addrspace(5) [[BLOCK2]], i32 0, i32 5
-; CHECK-NEXT:    store i8 [[B]], ptr addrspace(5) [[BLOCK_CAPTURED8]], align 8
-; CHECK-NEXT:    [[BLOCK_CAPTURED9:%.*]] = getelementptr inbounds <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr addrspace(5) [[BLOCK2]], i32 0, i32 3
-; CHECK-NEXT:    store ptr addrspace(1) [[C]], ptr addrspace(5) [[BLOCK_CAPTURED9]], align 8
-; CHECK-NEXT:    [[BLOCK_CAPTURED10:%.*]] = getelementptr inbounds <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr addrspace(5) [[BLOCK2]], i32 0, i32 4
-; CHECK-NEXT:    store i64 [[D]], ptr addrspace(5) [[BLOCK_CAPTURED10]], align 8
-; CHECK-NEXT:    [[INST8:%.*]] = addrspacecast ptr addrspace(5) [[BLOCK2]] to ptr
-; CHECK-NEXT:    [[INST9:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) undef, i32 0, ptr addrspace(5) nonnull byval([[STRUCT_NDRANGE_T]]) [[INST3]], ptr addrspacecast (ptr addrspace(1) @__test_block_invoke_2_kernel.runtime_handle to ptr), ptr nonnull [[INST8]])
-; CHECK-NEXT:    ret void
-;
-;
-; CHECK-LABEL: define {{[^@]+}}@inlined_caller
-; CHECK-SAME: (ptr addrspace(1) [[A:%.*]], i8 [[B:%.*]], ptr addrspace(1) [[C:%.*]], i64 [[D:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[INST:%.*]] = load i64, ptr addrspace(1) @__test_block_invoke_kernel.runtime_handle, align 4
-; CHECK-NEXT:    store i64 [[INST]], ptr addrspace(1) [[C]], align 4
-; CHECK-NEXT:    ret void
-;
-;
-; CHECK-LABEL: define {{[^@]+}}@__test_block_invoke_kernel
-; CHECK-SAME: (<{ i32, i32, ptr addrspace(1), i8 }> [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[DOTFCA_3_EXTRACT:%.*]] = extractvalue <{ i32, i32, ptr addrspace(1), i8 }> [[ARG]], 2
-; CHECK-NEXT:    [[DOTFCA_4_EXTRACT:%.*]] = extractvalue <{ i32, i32, ptr addrspace(1), i8 }> [[ARG]], 3
-; CHECK-NEXT:    store i8 [[DOTFCA_4_EXTRACT]], ptr addrspace(1) [[DOTFCA_3_EXTRACT]], align 1
-; CHECK-NEXT:    ret void
-;
-;
-; CHECK-LABEL: define {{[^@]+}}@__test_block_invoke_2_kernel
-; CHECK-SAME: (<{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[ARG:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[DOTFCA_3_EXTRACT:%.*]] = extractvalue <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[ARG]], 2
-; CHECK-NEXT:    [[DOTFCA_4_EXTRACT:%.*]] = extractvalue <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[ARG]], 3
-; CHECK-NEXT:    [[DOTFCA_5_EXTRACT:%.*]] = extractvalue <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[ARG]], 4
-; CHECK-NEXT:    [[DOTFCA_6_EXTRACT:%.*]] = extractvalue <{ i32, i32, ptr addrspace(1), ptr addrspace(1), i64, i8 }> [[ARG]], 5
-; CHECK-NEXT:    store i8 [[DOTFCA_6_EXTRACT]], ptr addrspace(1) [[DOTFCA_3_EXTRACT]], align 1
-; CHECK-NEXT:    store i64 [[DOTFCA_5_EXTRACT]], ptr addrspace(1) [[DOTFCA_4_EXTRACT]], align 8
-; CHECK-NEXT:    ret void
-;
-;
-; CHECK-LABEL: define {{[^@]+}}@block_has_used_kernel_address
-; CHECK-SAME: (<{ i32, i32, ptr addrspace(1), i8 }> [[ARG:%.*]]) #[[ATTR2:[0-9]+]] {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[DOTFCA_3_EXTRACT:%.*]] = extractvalue <{ i32, i32, ptr addrspace(1), i8 }> [[ARG]], 2
-; CHECK-NEXT:    [[DOTFCA_4_EXTRACT:%.*]] = extractvalue <{ i32, i32, ptr addrspace(1), i8 }> [[ARG]], 3
-; CHECK-NEXT:    store i8 [[DOTFCA_4_EXTRACT]], ptr addrspace(1) [[DOTFCA_3_EXTRACT]], align 1
-; CHECK-NEXT:    ret void
-;
-;
-; CHECK-LABEL: define {{[^@]+}}@user_of_kernel_address
-; CHECK-SAME: (ptr addrspace(1) [[ARG:%.*]]) {
-; CHECK-NEXT:    store ptr addrspacecast (ptr addrspace(1) @block_has_used_kernel_address.runtime_handle to ptr), ptr addrspace(1) [[ARG]], align 8
-; CHECK-NEXT:    ret void
-;
-;
-; CHECK-LABEL: define {{[^@]+}}@__amdgpu_enqueued_kernel
-; CHECK-SAME: (<{ i32, i32, ptr addrspace(1), i8 }> [[ARG:%.*]]) #[[ATTR3:[0-9]+]] {
-; CHECK-NEXT:    ret void
-;
-;
-; CHECK-LABEL: define {{[^@]+}}@__amdgpu_enqueued_kernel.1
-; CHECK-SAME: (<{ i32, i32, ptr addrspace(1), i8 }> [[ARG:%.*]]) #[[ATTR4:[0-9]+]] {
-; CHECK-NEXT:    ret void
-;
-;.
-; CHECK: attributes #[[ATTR0]] = { "enqueued-block" "runtime-handle"="__test_block_invoke_kernel.runtime_handle" }
-; CHECK: attributes #[[ATTR1]] = { "enqueued-block" "runtime-handle"="__test_block_invoke_2_kernel.runtime_handle" }
-; CHECK: attributes #[[ATTR2]] = { "enqueued-block" "runtime-handle"="block_has_used_kernel_address.runtime_handle" }
-; CHECK: attributes #[[ATTR3]] = { "enqueued-block" "runtime-handle"="__amdgpu_enqueued_kernel.runtime_handle" }
-; CHECK: attributes #[[ATTR4]] = { "enqueued-block" "runtime-handle"="__amdgpu_enqueued_kernel.1.runtime_handle" }
-;.
diff --git a/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll b/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
index 6dabd8c0b83ea..24dc5b5bb3150 100644
--- a/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
@@ -156,7 +156,7 @@ F:
 
 exit:
   %m = phi <8 x i16> [ %t, %T ], [ %f, %F ]
-  %v2 = shufflevector <8 x i16> %m, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
+  %v2 = shufflevector <8 x i16> %m, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
   %b2 = icmp sgt <4 x i16> %v2, <i16 -1, i16 -1, i16 -1, i16 -1>
   %r2 = select <4 x i1> %b2, <4 x i16> <i16 -32768, i16 -32768, i16 -32768, i16 -32768>, <4 x i16> <i16 -1, i16 -1, i16 -1, i16 -1>
   ret <4 x i16> %r2
@@ -317,7 +317,7 @@ F:
 
 exit:
   %m = phi <8 x i16> [ %t, %T ], [ %f, %F ]
-  %v2 = shufflevector <8 x i16> %m, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %v2 = shufflevector <8 x i16> %m, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %b2 = icmp sgt <4 x i16> %v2, <i16 -1, i16 -1, i16 -1, i16 -1>
   %r2 = select <4 x i1> %b2, <4 x i16> <i16 -32768, i16 -32768, i16 -32768, i16 -32768>, <4 x i16> <i16 -1, i16 -1, i16 -1, i16 -1>
   ret <4 x i16> %r2
@@ -482,7 +482,7 @@ F:
 
 exit:
   %m = phi <8 x half> [ %t, %T ], [ %f, %F ]
-  %v2 = shufflevector <8 x half> %m, <8 x half> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
+  %v2 = shufflevector <8 x half> %m, <8 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
   %b2 = fcmp ugt <4 x half> %v2, <half 0xH3800, half 0xH3800, half 0xH3800, half 0xH3800>
   %r2 = select <4 x i1> %b2, <4 x half> <half 0xH3900, half 0xH3900, half 0xH3900, half 0xH3900>, <4 x half> <half 0xH3D00, half 0xH3D00, half 0xH3D00, half 0xH3D00>
   ret <4 x half> %r2
@@ -685,7 +685,7 @@ F:
 
 exit:
   %m = phi <16 x i16> [ %t, %T ], [ %f, %F ]
-  %v2 = shufflevector <16 x i16> %m, <16 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
+  %v2 = shufflevector <16 x i16> %m, <16 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
   %b2 = icmp sgt <4 x i16> %v2, <i16 -1, i16 -1, i16 -1, i16 -1>
   %r2 = select <4 x i1> %b2, <4 x i16> <i16 -32768, i16 -32768, i16 -32768, i16 -32768>, <4 x i16> <i16 -1, i16 -1, i16 -1, i16 -1>
   ret <4 x i16> %r2
@@ -890,7 +890,7 @@ F:
 
 exit:
   %m = phi <16 x i16> [ %t, %T ], [ %f, %F ]
-  %v2 = shufflevector <16 x i16> %m, <16 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %v2 = shufflevector <16 x i16> %m, <16 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %b2 = icmp sgt <4 x i16> %v2, <i16 -1, i16 -1, i16 -1, i16 -1>
   %r2 = select <4 x i1> %b2, <4 x i16> <i16 -32768, i16 -32768, i16 -32768, i16 -32768>, <4 x i16> <i16 -1, i16 -1, i16 -1, i16 -1>
   ret <4 x i16> %r2
@@ -1099,7 +1099,7 @@ F:
 
 exit:
   %m = phi <16 x half> [ %t, %T ], [ %f, %F ]
-  %v2 = shufflevector <16 x half> %m, <16 x half> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
+  %v2 = shufflevector <16 x half> %m, <16 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
   %b2 = fcmp ugt <4 x half> %v2, <half 0xH3800, half 0xH3800, half 0xH3800, half 0xH3800>
   %r2 = select <4 x i1> %b2, <4 x half> <half 0xH3900, half 0xH3900, half 0xH3900, half 0xH3900>, <4 x half> <half 0xH3D00, half 0xH3D00, half 0xH3D00, half 0xH3D00>
   ret <4 x half> %r2
@@ -1184,8 +1184,8 @@ define <8 x i16> @large_vector(ptr addrspace(3) %p, i32 %idxp) {
   %x.7 = load i16, ptr addrspace(3) %p.7, align 2
   %v3 = insertelement <8 x i16> %v3p, i16 %x.7, i32 1
 
-  %z.1 = shufflevector <8 x i16> %v0, <8 x i16> %v1, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef>
-  %z.2 = shufflevector <8 x i16> %z.1, <8 x i16> %v2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 undef, i32 undef>
+  %z.1 = shufflevector <8 x i16> %v0, <8 x i16> %v1, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
+  %z.2 = shufflevector <8 x i16> %z.1, <8 x i16> %v2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
   %z.3 = shufflevector <8 x i16> %z.2, <8 x i16> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
   ret <8 x i16> %z.3
 }
@@ -1464,7 +1464,7 @@ F:
 
 exit:
   %m = phi <16 x i16> [ %t, %T ], [ %f, %F ]
-  %v2 = shufflevector <16 x i16> %m, <16 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %v2 = shufflevector <16 x i16> %m, <16 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %b2 = icmp ugt <8 x i16> %v2, <i16 u0x3800, i16 u0x3800, i16 u0x3800, i16 u0x3800, i16 u0x3800, i16 u0x3800, i16 u0x3800, i16 u0x3800>
   %r2 = select <8 x i1> %b2, <8 x i16> <i16 u0x3900, i16 u0x3900, i16 u0x3900, i16 u0x3900, i16 u0x3900, i16 u0x3900, i16 u0x3900, i16 u0x3900>, <8 x i16> <i16 u0x3D00, i16 u0x3D00, i16 u0x3D00, i16 u0x3D00, i16 u0x3D00, i16 u0x3D00, i16 u0x3D00, i16 u0x3D00>
   ret <8 x i16> %r2
@@ -1755,7 +1755,7 @@ F:
 
 exit:
   %m = phi <16 x half> [ %t, %T ], [ %f, %F ]
-  %v2 = shufflevector <16 x half> %m, <16 x half> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %v2 = shufflevector <16 x half> %m, <16 x half> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %b2 = fcmp ugt <8 x half> %v2, <half 0xH3800, half 0xH3800, half 0xH3800, half 0xH3800, half 0xH3800, half 0xH3800, half 0xH3800, half 0xH3800>
   %r2 = select <8 x i1> %b2, <8 x half> <half 0xH3900, half 0xH3900, half 0xH3900, half 0xH3900, half 0xH3900, half 0xH3900, half 0xH3900, half 0xH3900>, <8 x half> <half 0xH3D00, half 0xH3D00, half 0xH3D00, half 0xH3D00, half 0xH3D00, half 0xH3D00, half 0xH3D00, half 0xH3D00>
   ret <8 x half> %r2
diff --git a/llvm/test/CodeGen/AMDGPU/extract-subvector-equal-length.ll b/llvm/test/CodeGen/AMDGPU/extract-subvector-equal-length.ll
index d3b5d45470eb7..4cd39590bfc34 100644
--- a/llvm/test/CodeGen/AMDGPU/extract-subvector-equal-length.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract-subvector-equal-length.ll
@@ -13,10 +13,10 @@ define <3 x i32> @quux() {
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 1
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 bb:
-  %tmp = shufflevector <4 x i8> <i8 1, i8 2, i8 3, i8 4>, <4 x i8> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  %tmp = shufflevector <4 x i8> <i8 1, i8 2, i8 3, i8 4>, <4 x i8> poison, <3 x i32> <i32 0, i32 1, i32 2>
   %tmp1 = extractelement <3 x i8> %tmp, i64 0
   %tmp2 = zext i8 %tmp1 to i32
-  %tmp3 = insertelement <3 x i32> undef, i32 %tmp2, i32 0
+  %tmp3 = insertelement <3 x i32> poison, i32 %tmp2, i32 0
   %tmp4 = extractelement <3 x i8> %tmp, i64 1
   %tmp5 = zext i8 %tmp4 to i32
   %tmp6 = insertelement <3 x i32> %tmp3, i32 %tmp5, i32 1
diff --git a/llvm/test/CodeGen/AMDGPU/extract-subvector.ll b/llvm/test/CodeGen/AMDGPU/extract-subvector.ll
index 1e86842be4e5e..61c0b8b861d5b 100644
--- a/llvm/test/CodeGen/AMDGPU/extract-subvector.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract-subvector.ll
@@ -90,7 +90,7 @@ F:
 
 exit:
   %m = phi <8 x i16> [ %t, %T ], [ %f, %F ]
-  %v2 = shufflevector <8 x i16> %m, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+  %v2 = shufflevector <8 x i16> %m, <8 x i16> poison, <2 x i32> <i32 0, i32 1>
   %b2 = icmp sgt <2 x i16> %v2, <i16 -1, i16 -1>
   %r2 = select <2 x i1> %b2, <2 x i16> <i16 -32768, i16 -32768>, <2 x i16> <i16 -1, i16 -1>
   ret <2 x i16> %r2
@@ -161,7 +161,7 @@ F:
 
 exit:
   %m = phi <8 x i64> [ %t, %T ], [ %f, %F ]
-  %v2 = shufflevector <8 x i64> %m, <8 x i64> undef, <2 x i32> <i32 0, i32 1>
+  %v2 = shufflevector <8 x i64> %m, <8 x i64> poison, <2 x i32> <i32 0, i32 1>
   %b2 = icmp sgt <2 x i64> %v2, <i64 -1, i64 -1>
   %r2 = select <2 x i1> %b2, <2 x i64> <i64 -32768, i64 -32768>, <2 x i64> <i64 -1, i64 -1>
   ret <2 x i64> %r2
@@ -238,7 +238,7 @@ F:
 
 exit:
   %m = phi <8 x i64> [ %t, %T ], [ %f, %F ]
-  %v2 = shufflevector <8 x i64> %m, <8 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %v2 = shufflevector <8 x i64> %m, <8 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %b2 = icmp sgt <4 x i64> %v2, <i64 -1, i64 -1, i64 -1, i64 -1>
   %r2 = select <4 x i1> %b2, <4 x i64> <i64 -32768, i64 -32768, i64 -32768, i64 -32768>, <4 x i64> <i64 -1, i64 -1, i64 -1, i64 -1>
   ret <4 x i64> %r2
@@ -342,7 +342,7 @@ F:
 
 exit:
   %m = phi <16 x i64> [ %t, %T ], [ %f, %F ]
-  %v2 = shufflevector <16 x i64> %m, <16 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %v2 = shufflevector <16 x i64> %m, <16 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %b2 = icmp sgt <8 x i64> %v2, <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
   %r2 = select <8 x i1> %b2, <8 x i64> <i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768>, <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
   ret <8 x i64> %r2
@@ -413,7 +413,7 @@ F:
 
 exit:
   %m = phi <8 x double> [ %t, %T ], [ %f, %F ]
-  %v2 = shufflevector <8 x double> %m, <8 x double> undef, <2 x i32> <i32 0, i32 1>
+  %v2 = shufflevector <8 x double> %m, <8 x double> poison, <2 x i32> <i32 0, i32 1>
   %b2 = fcmp ogt <2 x double> %v2, <double -1.0, double -1.0>
   %r2 = select <2 x i1> %b2, <2 x double> <double -2.0, double -2.0>, <2 x double> <double -1.0, double -1.0>
   ret <2 x double> %r2
@@ -490,7 +490,7 @@ F:
 
 exit:
   %m = phi <8 x double> [ %t, %T ], [ %f, %F ]
-  %v2 = shufflevector <8 x double> %m, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %v2 = shufflevector <8 x double> %m, <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %b2 = fcmp ogt <4 x double> %v2, <double -1.0, double -1.0, double -1.0, double -1.0>
   %r2 = select <4 x i1> %b2, <4 x double> <double -2.0, double -2.0, double -2.0, double -2.0>, <4 x double> <double -1.0, double -1.0, double -1.0, double -1.0>
   ret <4 x double> %r2
@@ -594,7 +594,7 @@ F:
 
 exit:
   %m = phi <16 x double> [ %t, %T ], [ %f, %F ]
-  %v2 = shufflevector <16 x double> %m, <16 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %v2 = shufflevector <16 x double> %m, <16 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %b2 = fcmp ogt <8 x double> %v2, <double -1.0, double -1.0, double -1.0, double -1.0, double -1.0, double -1.0, double -1.0, double -1.0>
   %r2 = select <8 x i1> %b2, <8 x double> <double -2.0, double -2.0, double -2.0, double -2.0, double -2.0, double -2.0, double -2.0, double -2.0>, <8 x double> <double -1.0, double -1.0, double -1.0, double -1.0, double -1.0, double -1.0, double -1.0, double -1.0>
   ret <8 x double> %r2
diff --git a/llvm/test/CodeGen/AMDGPU/extract-vector-elt-build-vector-combine.ll b/llvm/test/CodeGen/AMDGPU/extract-vector-elt-build-vector-combine.ll
index df31ff0be84ce..cca0dd62aebdd 100644
--- a/llvm/test/CodeGen/AMDGPU/extract-vector-elt-build-vector-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract-vector-elt-build-vector-combine.ll
@@ -22,7 +22,7 @@ define amdgpu_kernel void @store_build_vector_multiple_uses_v4i32(ptr addrspace(
   %elt2 = load volatile i32, ptr addrspace(1) %in
   %elt3 = load volatile i32, ptr addrspace(1) %in
 
-  %vec0 = insertelement <4 x i32> undef, i32 %elt0, i32 0
+  %vec0 = insertelement <4 x i32> poison, i32 %elt0, i32 0
   %vec1 = insertelement <4 x i32> %vec0, i32 %elt1, i32 1
   %vec2 = insertelement <4 x i32> %vec1, i32 %elt2, i32 2
   %vec3 = insertelement <4 x i32> %vec2, i32 %elt3, i32 3
@@ -64,7 +64,7 @@ define amdgpu_kernel void @store_build_vector_multiple_extract_uses_v4i32(ptr ad
   %elt2 = load volatile i32, ptr addrspace(1) %in
   %elt3 = load volatile i32, ptr addrspace(1) %in
 
-  %vec0 = insertelement <4 x i32> undef, i32 %elt0, i32 0
+  %vec0 = insertelement <4 x i32> poison, i32 %elt0, i32 0
   %vec1 = insertelement <4 x i32> %vec0, i32 %elt1, i32 1
   %vec2 = insertelement <4 x i32> %vec1, i32 %elt2, i32 2
   %vec3 = insertelement <4 x i32> %vec2, i32 %elt3, i32 3
@@ -108,7 +108,7 @@ define amdgpu_kernel void @store_build_vector_multiple_uses_v4i32_bitcast_to_v2i
   %elt2 = load volatile i32, ptr addrspace(1) %in
   %elt3 = load volatile i32, ptr addrspace(1) %in
 
-  %vec0 = insertelement <4 x i32> undef, i32 %elt0, i32 0
+  %vec0 = insertelement <4 x i32> poison, i32 %elt0, i32 0
   %vec1 = insertelement <4 x i32> %vec0, i32 %elt1, i32 1
   %vec2 = insertelement <4 x i32> %vec1, i32 %elt2, i32 2
   %vec3 = insertelement <4 x i32> %vec2, i32 %elt3, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll b/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll
index 1319c526868c3..a44231f39ae6f 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll
@@ -27,7 +27,7 @@ define amdgpu_hs void @main(ptr addrspace(6) inreg %arg) {
 main_body:
   %tmp25 = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) undef, i32 undef, i32 0, i32 0)
   %tmp27 = bitcast <4 x float> %tmp25 to <16 x i8>
-  %tmp28 = shufflevector <16 x i8> %tmp27, <16 x i8> undef, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+  %tmp28 = shufflevector <16 x i8> %tmp27, <16 x i8> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
   %tmp29 = bitcast <12 x i8> %tmp28 to <3 x i32>
   call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> %tmp29, ptr addrspace(8) undef, i32 undef, i32 0, i32 0) #3
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll
index e1981972f58d1..32315560f560d 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll
@@ -740,7 +740,7 @@ define <2 x half> @v_test_canonicalize_build_vector_v2f16(<2 x half> %vec) {
   %hi = extractelement <2 x half> %vec, i32 1
   %lo.op = fadd half %lo, 1.0
   %hi.op = fmul half %lo, 4.0
-  %ins0 = insertelement <2 x half> undef, half %lo.op, i32 0
+  %ins0 = insertelement <2 x half> poison, half %lo.op, i32 0
   %ins1 = insertelement <2 x half> %ins0, half %hi.op, i32 1
   %canonicalized = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> %ins1)
   ret <2 x half> %canonicalized
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
index e72f3d3ce993a..d48b75a666db7 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
@@ -238,7 +238,7 @@ define <2 x half> @v_test_canonicalize_build_vector_v2f16(half %lo, half %hi) #1
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_pk_max_f16 v0, v0, v0
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
-  %ins0 = insertelement <2 x half> undef, half %lo, i32 0
+  %ins0 = insertelement <2 x half> poison, half %lo, i32 0
   %ins1 = insertelement <2 x half> %ins0, half %hi, i32 1
   %canonicalized = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> %ins1)
   ret <2 x half> %canonicalized
@@ -2581,7 +2581,7 @@ define <2 x half> @v_test_canonicalize_reg_undef_v2f16(half %val) #1 {
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_pack_b32_f16 v0, v0, 0
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
-  %vec = insertelement <2 x half> undef, half %val, i32 0
+  %vec = insertelement <2 x half> poison, half %val, i32 0
   %canonicalized = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> %vec)
   ret <2 x half> %canonicalized
 }
@@ -2622,7 +2622,7 @@ define <2 x half> @v_test_canonicalize_undef_reg_v2f16(half %val) #1 {
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
-  %vec = insertelement <2 x half> undef, half %val, i32 1
+  %vec = insertelement <2 x half> poison, half %val, i32 1
   %canonicalized = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> %vec)
   ret <2 x half> %canonicalized
 }
@@ -2785,7 +2785,7 @@ define <2 x half> @v_test_canonicalize_reg_k_v2f16(half %val) #1 {
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_pack_b32_f16 v0, v0, 2.0
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
-  %vec0 = insertelement <2 x half> undef, half %val, i32 0
+  %vec0 = insertelement <2 x half> poison, half %val, i32 0
   %vec1 = insertelement <2 x half> %vec0, half 2.0, i32 1
   %canonicalized = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> %vec1)
   ret <2 x half> %canonicalized
@@ -2829,7 +2829,7 @@ define <2 x half> @v_test_canonicalize_k_reg_v2f16(half %val) #1 {
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_pack_b32_f16 v0, 2.0, v0
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
-  %vec0 = insertelement <2 x half> undef, half 2.0, i32 0
+  %vec0 = insertelement <2 x half> poison, half 2.0, i32 0
   %vec1 = insertelement <2 x half> %vec0, half %val, i32 1
   %canonicalized = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> %vec1)
   ret <2 x half> %canonicalized
@@ -2925,7 +2925,7 @@ define <4 x half> @v_test_canonicalize_reg_undef_undef_undef_v4f16(half %val) #1
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_pack_b32_f16 v0, v0, 0
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
-  %vec = insertelement <4 x half> undef, half %val, i32 0
+  %vec = insertelement <4 x half> poison, half %val, i32 0
   %canonicalized = call <4 x half> @llvm.canonicalize.v4f16(<4 x half> %vec)
   ret <4 x half> %canonicalized
 }
@@ -2977,7 +2977,7 @@ define <4 x half> @v_test_canonicalize_reg_reg_undef_undef_v4f16(half %val0, hal
 ; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-FAKE16-NEXT:    v_pk_max_f16 v0, v0, v0
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
-  %vec0 = insertelement <4 x half> undef, half %val0, i32 0
+  %vec0 = insertelement <4 x half> poison, half %val0, i32 0
   %vec1 = insertelement <4 x half> %vec0, half %val1, i32 1
   %canonicalized = call <4 x half> @llvm.canonicalize.v4f16(<4 x half> %vec1)
   ret <4 x half> %canonicalized
@@ -3035,7 +3035,7 @@ define <4 x half> @v_test_canonicalize_reg_undef_reg_reg_v4f16(half %val0, half
 ; GFX11-FAKE16-NEXT:    v_pack_b32_f16 v0, v0, 0
 ; GFX11-FAKE16-NEXT:    v_pk_max_f16 v1, v1, v1
 ; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
-  %vec0 = insertelement <4 x half> undef, half %val0, i32 0
+  %vec0 = insertelement <4 x half> poison, half %val0, i32 0
   %vec1 = insertelement <4 x half> %vec0, half %val1, i32 2
   %vec2 = insertelement <4 x half> %vec1, half %val2, i32 3
   %canonicalized = call <4 x half> @llvm.canonicalize.v4f16(<4 x half> %vec2)
diff --git a/llvm/test/CodeGen/AMDGPU/flat-offset-bug.ll b/llvm/test/CodeGen/AMDGPU/flat-offset-bug.ll
index 90fbb6c12382a..54343fa820cba 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-offset-bug.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-offset-bug.ll
@@ -32,7 +32,7 @@ define void @global_inst_offset(ptr addrspace(1) nocapture %p) {
 define amdgpu_kernel void @load_i16_lo(ptr %arg, ptr %out) {
   %gep = getelementptr inbounds i16, ptr %arg, i32 4
   %ld = load i16, ptr %gep, align 2
-  %vec = insertelement <2 x i16> <i16 undef, i16 0>, i16 %ld, i32 0
+  %vec = insertelement <2 x i16> <i16 poison, i16 0>, i16 %ld, i32 0
   %v = add <2 x i16> %vec, %vec
   store <2 x i16> %v, ptr %out, align 4
   ret void
@@ -44,7 +44,7 @@ define amdgpu_kernel void @load_i16_lo(ptr %arg, ptr %out) {
 define amdgpu_kernel void @load_i16_hi(ptr %arg, ptr %out) {
   %gep = getelementptr inbounds i16, ptr %arg, i32 4
   %ld = load i16, ptr %gep, align 2
-  %vec = insertelement <2 x i16> <i16 0, i16 undef>, i16 %ld, i32 1
+  %vec = insertelement <2 x i16> <i16 0, i16 poison>, i16 %ld, i32 1
   %v = add <2 x i16> %vec, %vec
   store <2 x i16> %v, ptr %out, align 4
   ret void
@@ -56,7 +56,7 @@ define amdgpu_kernel void @load_i16_hi(ptr %arg, ptr %out) {
 define amdgpu_kernel void @load_half_lo(ptr %arg, ptr %out) {
   %gep = getelementptr inbounds half, ptr %arg, i32 4
   %ld = load half, ptr %gep, align 2
-  %vec = insertelement <2 x half> <half undef, half 0xH0000>, half %ld, i32 0
+  %vec = insertelement <2 x half> <half poison, half 0xH0000>, half %ld, i32 0
   %v = fadd <2 x half> %vec, %vec
   store <2 x half> %v, ptr %out, align 4
   ret void
@@ -68,7 +68,7 @@ define amdgpu_kernel void @load_half_lo(ptr %arg, ptr %out) {
 define amdgpu_kernel void @load_half_hi(ptr %arg, ptr %out) {
   %gep = getelementptr inbounds half, ptr %arg, i32 4
   %ld = load half, ptr %gep, align 2
-  %vec = insertelement <2 x half> <half 0xH0000, half undef>, half %ld, i32 1
+  %vec = insertelement <2 x half> <half 0xH0000, half poison>, half %ld, i32 1
   %v = fadd <2 x half> %vec, %vec
   store <2 x half> %v, ptr %out, align 4
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/floor.ll b/llvm/test/CodeGen/AMDGPU/floor.ll
index cc23289b20b0a..adfcd98ba3681 100644
--- a/llvm/test/CodeGen/AMDGPU/floor.ll
+++ b/llvm/test/CodeGen/AMDGPU/floor.ll
@@ -4,7 +4,7 @@
 define amdgpu_ps void @test(<4 x float> inreg %reg0) {
    %r0 = extractelement <4 x float> %reg0, i32 0
    %r1 = call float @floorf(float %r0)
-   %vec = insertelement <4 x float> undef, float %r1, i32 0
+   %vec = insertelement <4 x float> poison, float %r1, i32 0
    call void @llvm.r600.store.swizzle(<4 x float> %vec, i32 0, i32 0)
    ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll b/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll
index 4f49d4ce3a040..827e5da15e50c 100644
--- a/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmac.sdwa.ll
@@ -38,8 +38,8 @@ bb14:                                             ; preds = %bb14, %bb11
   %tmp24 = getelementptr inbounds float, ptr addrspace(4) %arg1, i64 %tmp23
   %tmp25 = load float, ptr addrspace(4) %tmp24, align 4
   %tmp26 = fptrunc float %tmp25 to half
-  %tmp27 = insertelement <4 x half> undef, half %tmp26, i32 0
-  %tmp28 = shufflevector <4 x half> %tmp27, <4 x half> undef, <4 x i32> zeroinitializer
+  %tmp27 = insertelement <4 x half> poison, half %tmp26, i32 0
+  %tmp28 = shufflevector <4 x half> %tmp27, <4 x half> poison, <4 x i32> zeroinitializer
   %vec.A.0 = extractelement <4 x half> %tmp21, i32 0
   %vec.B.0 = extractelement <4 x half> %tmp28, i32 0
   %vec.C.0 = extractelement <4 x half> %tmp15, i32 0
@@ -56,7 +56,7 @@ bb14:                                             ; preds = %bb14, %bb11
   %vec.B.3 = extractelement <4 x half> %tmp28, i32 3
   %vec.C.3 = extractelement <4 x half> %tmp15, i32 3
   %vec.res.3 = tail call half @llvm.fmuladd.f16(half %vec.A.3, half %vec.B.3, half %vec.C.3)
-  %full.res.0 = insertelement <4 x half> undef, half %vec.res.0, i32 0
+  %full.res.0 = insertelement <4 x half> poison, half %vec.res.0, i32 0
   %full.res.1 = insertelement <4 x half> %full.res.0, half %vec.res.1, i32 1
   %full.res.2 = insertelement <4 x half> %full.res.1, half %vec.res.2, i32 2
   %tmp29 = insertelement <4 x half> %full.res.2, half %vec.res.3, i32 3
@@ -80,7 +80,7 @@ define linkonce_odr hidden <4 x half> @_Z13convert_half4Dv4_h(<4 x i8> %arg) loc
 bb:
   %tmp = extractelement <4 x i8> %arg, i64 0
   %tmp1 = uitofp i8 %tmp to half
-  %tmp2 = insertelement <4 x half> undef, half %tmp1, i32 0
+  %tmp2 = insertelement <4 x half> poison, half %tmp1, i32 0
   %tmp3 = extractelement <4 x i8> %arg, i64 1
   %tmp4 = uitofp i8 %tmp3 to half
   %tmp5 = insertelement <4 x half> %tmp2, half %tmp4, i32 1
diff --git a/llvm/test/CodeGen/AMDGPU/fmad-formation-fmul-distribute-denormal-mode.ll b/llvm/test/CodeGen/AMDGPU/fmad-formation-fmul-distribute-denormal-mode.ll
index 25776bb8380ae..92eb4a642dc92 100644
--- a/llvm/test/CodeGen/AMDGPU/fmad-formation-fmul-distribute-denormal-mode.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmad-formation-fmul-distribute-denormal-mode.ll
@@ -236,7 +236,7 @@ define <2 x float> @unsafe_fast_fmul_fadd_distribute_post_legalize_f32(float %ar
 ; FMADGFX10-NEXT:    v_mad_f32 v0, v0, v1, v1
 ; FMADGFX10-NEXT:    s_setpc_b64 s[30:31]
   %add = fadd fast float %arg0, 1.0
-  %splat = insertelement <2 x float> undef, float %add, i32 0
+  %splat = insertelement <2 x float> poison, float %add, i32 0
   %tmp1 = fmul fast <2 x float> %arg1, %splat
   ret <2 x float> %tmp1
 }
@@ -279,7 +279,7 @@ define <2 x float> @unsafe_fast_fmul_fsub_ditribute_post_legalize(float %arg0, <
 ; FMADGFX10-NEXT:    v_mad_f32 v0, -v0, v1, v1
 ; FMADGFX10-NEXT:    s_setpc_b64 s[30:31]
   %sub = fsub fast float 1.0, %arg0
-  %splat = insertelement <2 x float> undef, float %sub, i32 0
+  %splat = insertelement <2 x float> poison, float %sub, i32 0
   %tmp1 = fmul fast <2 x float> %arg1, %splat
   ret <2 x float> %tmp1
 }
diff --git a/llvm/test/CodeGen/AMDGPU/fmad.ll b/llvm/test/CodeGen/AMDGPU/fmad.ll
index a1d62228fd4b6..f21ad11b1b06f 100644
--- a/llvm/test/CodeGen/AMDGPU/fmad.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmad.ll
@@ -8,7 +8,7 @@ define amdgpu_ps void @test(<4 x float> inreg %reg0) {
    %r2 = extractelement <4 x float> %reg0, i32 2
    %r3 = fmul float %r0, %r1
    %r4 = fadd float %r3, %r2
-   %vec = insertelement <4 x float> undef, float %r4, i32 0
+   %vec = insertelement <4 x float> poison, float %r4, i32 0
    call void @llvm.r600.store.swizzle(<4 x float> %vec, i32 0, i32 0)
    ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/fmax.ll b/llvm/test/CodeGen/AMDGPU/fmax.ll
index 825ad93b896c5..d94b4cbf28e06 100644
--- a/llvm/test/CodeGen/AMDGPU/fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmax.ll
@@ -7,7 +7,7 @@ define amdgpu_ps void @test(<4 x float> inreg %reg0) {
    %r1 = extractelement <4 x float> %reg0, i32 1
    %r2 = fcmp oge float %r0, %r1
    %r3 = select i1 %r2, float %r0, float %r1
-   %vec = insertelement <4 x float> undef, float %r3, i32 0
+   %vec = insertelement <4 x float> poison, float %r3, i32 0
    call void @llvm.r600.store.swizzle(<4 x float> %vec, i32 0, i32 0)
    ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/fmin.ll b/llvm/test/CodeGen/AMDGPU/fmin.ll
index e7cb2d7532483..530636af96038 100644
--- a/llvm/test/CodeGen/AMDGPU/fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmin.ll
@@ -7,7 +7,7 @@ define amdgpu_ps void @test(<4 x float> inreg %reg0) {
    %r1 = extractelement <4 x float> %reg0, i32 1
    %r2 = fcmp uge float %r0, %r1
    %r3 = select i1 %r2, float %r1, float %r0
-   %vec = insertelement <4 x float> undef, float %r3, i32 0
+   %vec = insertelement <4 x float> poison, float %r3, i32 0
    call void @llvm.r600.store.swizzle(<4 x float> %vec, i32 0, i32 0)
    ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll
index f4c5ebd8b3cf5..78afde138944e 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll
@@ -1466,7 +1466,7 @@ define <2 x half> @v_fneg_minnum_multi_use_minnum_f16_no_ieee(half %a, half %b)
   %min = call half @llvm.minnum.f16(half %a, half %b)
   %fneg = fneg half %min
   %use1 = fmul half %min, 4.0
-  %ins0 = insertelement <2 x half> undef, half %fneg, i32 0
+  %ins0 = insertelement <2 x half> poison, half %fneg, i32 0
   %ins1 = insertelement <2 x half> %ins0, half %use1, i32 1
   ret <2 x half> %ins1
 }
@@ -1904,7 +1904,7 @@ define <2 x half> @v_fneg_maxnum_multi_use_maxnum_f16_no_ieee(half %a, half %b)
   %max = call half @llvm.maxnum.f16(half %a, half %b)
   %fneg = fneg half %max
   %use1 = fmul half %max, 4.0
-  %ins0 = insertelement <2 x half> undef, half %fneg, i32 0
+  %ins0 = insertelement <2 x half> poison, half %fneg, i32 0
   %ins1 = insertelement <2 x half> %ins0, half %use1, i32 1
   ret <2 x half> %ins1
 }
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.ll
index 0cb4b8c960bbf..3a178b73ebd6c 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-combines.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.ll
@@ -853,7 +853,7 @@ define amdgpu_ps <2 x float> @v_fneg_minnum_multi_use_minnum_f32_no_ieee(float %
   %min = call float @llvm.minnum.f32(float %a, float %b)
   %fneg = fneg float %min
   %use1 = fmul float %min, 4.0
-  %ins0 = insertelement <2 x float> undef, float %fneg, i32 0
+  %ins0 = insertelement <2 x float> poison, float %fneg, i32 0
   %ins1 = insertelement <2 x float> %ins0, float %use1, i32 1
   ret <2 x float> %ins1
 }
@@ -1093,7 +1093,7 @@ define amdgpu_ps <2 x float> @v_fneg_maxnum_multi_use_maxnum_f32_no_ieee(float %
   %max = call float @llvm.maxnum.f32(float %a, float %b)
   %fneg = fneg float %max
   %use1 = fmul float %max, 4.0
-  %ins0 = insertelement <2 x float> undef, float %fneg, i32 0
+  %ins0 = insertelement <2 x float> poison, float %fneg, i32 0
   %ins1 = insertelement <2 x float> %ins0, float %use1, i32 1
   ret <2 x float> %ins1
 }
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll
index 9a72fe96b5c3a..14f7cbcd0f438 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll
@@ -1186,7 +1186,7 @@ define <2 x float> @v_fneg_minnum_multi_use_minnum_f32_no_ieee(float %a, float %
   %min = call float @llvm.minnum.f32(float %a, float %b)
   %fneg = fneg float %min
   %use1 = fmul float %min, 4.0
-  %ins0 = insertelement <2 x float> undef, float %fneg, i32 0
+  %ins0 = insertelement <2 x float> poison, float %fneg, i32 0
   %ins1 = insertelement <2 x float> %ins0, float %use1, i32 1
   ret <2 x float> %ins1
 }
@@ -1376,7 +1376,7 @@ define <2 x float> @v_fneg_maxnum_multi_use_maxnum_f32_no_ieee(float %a, float %
   %max = call float @llvm.maxnum.f32(float %a, float %b)
   %fneg = fneg float %max
   %use1 = fmul float %max, 4.0
-  %ins0 = insertelement <2 x float> undef, float %fneg, i32 0
+  %ins0 = insertelement <2 x float> poison, float %fneg, i32 0
   %ins1 = insertelement <2 x float> %ins0, float %use1, i32 1
   ret <2 x float> %ins1
 }
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fold-legalize-dag-increase-insts.ll b/llvm/test/CodeGen/AMDGPU/fneg-fold-legalize-dag-increase-insts.ll
index 0a2cb54877b80..89d11d57c7a9e 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-fold-legalize-dag-increase-insts.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-fold-legalize-dag-increase-insts.ll
@@ -30,7 +30,7 @@ bb:
   %tmp3 = fsub fast double %arg2, %arg
   %tmp4 = fadd fast double %tmp3, %tmp
   %tmp5 = fsub fast double %tmp, %tmp3
-  %tmp6 = insertvalue { double, double } undef, double %tmp4, 0
+  %tmp6 = insertvalue { double, double } poison, double %tmp4, 0
   %tmp7 = insertvalue { double, double } %tmp6, double %tmp5, 1
   ret { double, double } %tmp7
 }
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll b/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
index e1791daa3aa0c..781a2ca3146f5 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
@@ -1585,7 +1585,7 @@ define amdgpu_kernel void @fnge_select_f32_multi_use_regression(float %.i2369) {
 
 bb:                                               ; preds = %.entry
   %i2 = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> zeroinitializer, i32 1, i32 0)
-  %i3 = shufflevector <2 x i32> %i2, <2 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %i3 = shufflevector <2 x i32> %i2, <2 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   %i4 = bitcast <4 x i32> %i3 to <4 x float>
   %.i0753 = extractelement <4 x float> %i4, i64 0
   br label %bb5
diff --git a/llvm/test/CodeGen/AMDGPU/fold-fabs.ll b/llvm/test/CodeGen/AMDGPU/fold-fabs.ll
index a04bf44549325..6a8594a168f03 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-fabs.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-fabs.ll
@@ -179,7 +179,7 @@ entry:
   br i1 %4, label %header, label %exit
 
 header:
-  %h.fabs.phi = phi float [ undef, %entry ], [ %l.fabs, %l ]
+  %h.fabs.phi = phi float [ poison, %entry ], [ %l.fabs, %l ]
   %h.fmul = fmul reassoc nnan nsz arcp contract afn float %h.fabs.phi, 2.000000e+00
   %l.1 = fmul reassoc nnan nsz arcp contract afn float %h.fabs.phi, 3.000000e+00
   br label %l
diff --git a/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir b/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir
index 5d68c3135a026..dce11209d0514 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir
@@ -109,3 +109,128 @@ body:             |
     S_ENDPGM 0
 
 ...
+
+---
+name: av_mov_b32_imm_pseudo_copy_av_32_to_physreg_agpr
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: av_mov_b32_imm_pseudo_copy_av_32_to_physreg_agpr
+    ; GCN: [[AV_MOV_:%[0-9]+]]:av_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+    ; GCN-NEXT: $agpr0 = COPY [[AV_MOV_]]
+    ; GCN-NEXT: S_ENDPGM 0, implicit $agpr0
+    %0:av_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+    $agpr0 = COPY %0
+    S_ENDPGM 0, implicit $agpr0
+
+...
+
+---
+name: av_mov_b32_imm_pseudo_copy_av_32_to_physreg_vgpr
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: av_mov_b32_imm_pseudo_copy_av_32_to_physreg_vgpr
+    ; GCN: [[AV_MOV_:%[0-9]+]]:av_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+    ; GCN-NEXT: $vgpr0 = COPY [[AV_MOV_]]
+    ; GCN-NEXT: S_ENDPGM 0, implicit $vgpr0
+    %0:av_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+    $vgpr0 = COPY %0
+    S_ENDPGM 0, implicit $vgpr0
+
+...
+
+---
+name: av_mov_b32_imm_pseudo_copy_av_32_to_virtreg_agpr
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: av_mov_b32_imm_pseudo_copy_av_32_to_virtreg_agpr
+    ; GCN: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+    ; GCN-NEXT: S_ENDPGM 0, implicit [[V_ACCVGPR_WRITE_B32_e64_]]
+    %0:av_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+    %1:agpr_32 = COPY %0
+    S_ENDPGM 0, implicit %1
+
+...
+
+---
+name: av_mov_b32_imm_pseudo_copy_av_32_to_virtreg_vgpr
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: av_mov_b32_imm_pseudo_copy_av_32_to_virtreg_vgpr
+    ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
+    %0:av_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+    %1:vgpr_32 = COPY %0
+    S_ENDPGM 0, implicit %1
+
+...
+
+---
+name: v_mov_b32_imm_literal_copy_v_to_agpr_32
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: v_mov_b32_imm_literal_copy_v_to_agpr_32
+    ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 999, implicit $exec
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:agpr_32 = COPY [[V_MOV_B32_e32_]]
+    ; GCN-NEXT: $agpr0 = COPY [[COPY]]
+    ; GCN-NEXT: S_ENDPGM 0
+    %0:vgpr_32 = V_MOV_B32_e32 999, implicit $exec
+    %1:agpr_32 = COPY %0
+    $agpr0 = COPY %1
+    S_ENDPGM 0
+
+...
+
+# FIXME: Register class restrictions of av register not respected,
+# issue 130020
+
+# ---
+# name: s_mov_b32_inlineimm_copy_s_to_av_32
+# tracksRegLiveness: true
+# body:             |
+#   bb.0:
+#     %0:sreg_32 = S_MOV_B32 32
+#     %1:av_32 = COPY %0
+#     $agpr0 = COPY %1
+#     S_ENDPGM 0
+
+# ...
+
+# ---
+# name: v_mov_b32_inlineimm_copy_v_to_av_32
+# tracksRegLiveness: true
+# body:             |
+#  bb.0:
+#    %0:vgpr_32 = V_MOV_B32_e32 32, implicit $exec
+#    %1:av_32 = COPY %0
+#    $agpr0 = COPY %1
+#    S_ENDPGM 0
+# ...
+
+# ---
+# name: s_mov_b32_imm_literal_copy_s_to_av_32
+# tracksRegLiveness: true
+# body:             |
+#   bb.0:
+#     %0:sreg_32 = S_MOV_B32 999
+#     %1:av_32 = COPY %0
+#     $agpr0 = COPY %1
+#     S_ENDPGM 0
+
+# ...
+
+# ---
+# name: v_mov_b32_imm_literal_copy_v_to_av_32
+# tracksRegLiveness: true
+# body:             |
+#   bb.0:
+#     %0:vgpr_32 = V_MOV_B32_e32 999, implicit $exec
+#     %1:av_32 = COPY %0
+#     $agpr0 = COPY %1
+#     S_ENDPGM 0
+
+# ...
diff --git a/llvm/test/CodeGen/AMDGPU/fold-sgpr-multi-imm.mir b/llvm/test/CodeGen/AMDGPU/fold-sgpr-multi-imm.mir
index d85d77e338870..cc4314263bcba 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-sgpr-multi-imm.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-sgpr-multi-imm.mir
@@ -268,3 +268,18 @@ body: |
     %0:sreg_32 = S_MOV_B32 80
     S_CMP_EQ_U32 %0, 64, implicit-def $scc
 ...
+
+# Ignore regmask operands
+# GCN-LABEL: name: test_si_cs_chain_fold_with_regmask{{$}}
+# GCN: SI_CS_CHAIN_TC_W32 %0, 0, 0, 1234, amdgpu_allvgprs, implicit $sgpr0, implicit $vgpr8
+---
+name: test_si_cs_chain_fold_with_regmask
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr2_sgpr3, $vgpr8
+
+    %1:ccr_sgpr_64 = COPY $sgpr2_sgpr3
+    %2:sreg_32 = S_MOV_B32 1234
+    SI_CS_CHAIN_TC_W32 %1:ccr_sgpr_64, 0, 0, %2:sreg_32, amdgpu_allvgprs, implicit $sgpr0, implicit $vgpr8
+...
diff --git a/llvm/test/CodeGen/AMDGPU/frame-index-elimination.ll b/llvm/test/CodeGen/AMDGPU/frame-index-elimination.ll
index 7125e7740c10a..aa5f72a15c2f3 100644
--- a/llvm/test/CodeGen/AMDGPU/frame-index-elimination.ll
+++ b/llvm/test/CodeGen/AMDGPU/frame-index-elimination.ll
@@ -254,7 +254,7 @@ declare void @func(ptr addrspace(5) nocapture) #0
 define void @undefined_stack_store_reg(float %arg, i32 %arg1) #0 {
 bb:
   %tmp = alloca <4 x float>, align 16, addrspace(5)
-  %tmp2 = insertelement <4 x float> undef, float %arg, i32 0
+  %tmp2 = insertelement <4 x float> poison, float %arg, i32 0
   store <4 x float> %tmp2, ptr addrspace(5) undef
   %tmp3 = icmp eq i32 %arg1, 0
   br i1 %tmp3, label %bb4, label %bb5
diff --git a/llvm/test/CodeGen/AMDGPU/function-args-inreg.ll b/llvm/test/CodeGen/AMDGPU/function-args-inreg.ll
index 32f255df82499..dc20ae3765069 100644
--- a/llvm/test/CodeGen/AMDGPU/function-args-inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-args-inreg.ll
@@ -2060,9 +2060,9 @@ define void @void_func_a13i32_inreg([13  x i32] inreg %arg0, ptr addrspace(1) %p
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    s_mov_b32 s29, s33
 ; GFX9-NEXT:    s_mov_b32 s33, s32
-; GFX9-NEXT:    s_or_saveexec_b64 vcc, -1
+; GFX9-NEXT:    s_or_saveexec_b64 s[40:41], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_mov_b64 exec, vcc
+; GFX9-NEXT:    s_mov_b64 exec, s[40:41]
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s28
 ; GFX9-NEXT:    global_store_dword v[0:1], v2, off offset:48
 ; GFX9-NEXT:    v_mov_b32_e32 v5, s27
diff --git a/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll b/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll
index 22257d3eba7d6..512d58d3f996d 100644
--- a/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-resource-usage.ll
@@ -202,18 +202,18 @@ define void @indirect_use_50_vgpr() #0 {
 }
 
 ; GCN-LABEL: {{^}}use_80_sgpr:
-; GCN:	.set use_80_sgpr.num_vgpr, 1
+; GCN:	.set use_80_sgpr.num_vgpr, 0
 ; GCN:	.set use_80_sgpr.num_agpr, 0
 ; GCN:	.set use_80_sgpr.numbered_sgpr, 80
-; GCN:	.set use_80_sgpr.private_seg_size, 8
+; GCN:	.set use_80_sgpr.private_seg_size, 0
 ; GCN:	.set use_80_sgpr.uses_vcc, 0
 ; GCN:	.set use_80_sgpr.uses_flat_scratch, 0
 ; GCN:	.set use_80_sgpr.has_dyn_sized_stack, 0
 ; GCN:	.set use_80_sgpr.has_recursion, 0
 ; GCN:	.set use_80_sgpr.has_indirect_call, 0
 ; GCN: TotalNumSgprs: 84
-; GCN: NumVgprs: 1
-; GCN: ScratchSize: 8
+; GCN: NumVgprs: 0
+; GCN: ScratchSize: 0
 define void @use_80_sgpr() #1 {
   call void asm sideeffect "", "~{s79}"() #0
   ret void
@@ -231,7 +231,7 @@ define void @use_80_sgpr() #1 {
 ; GCN:	.set indirect_use_80_sgpr.has_indirect_call, or(0, use_80_sgpr.has_indirect_call)
 ; GCN: TotalNumSgprs: 84
 ; GCN: NumVgprs: 41
-; GCN: ScratchSize: 24
+; GCN: ScratchSize: 16
 define void @indirect_use_80_sgpr() #1 {
   call void @use_80_sgpr()
   ret void
@@ -249,7 +249,7 @@ define void @indirect_use_80_sgpr() #1 {
 ; GCN:	.set indirect_2_level_use_80_sgpr.has_indirect_call, or(0, indirect_use_80_sgpr.has_indirect_call)
 ; GCN: TotalNumSgprs: 86
 ; GCN: NumVgprs: 41
-; GCN: ScratchSize: 24
+; GCN: ScratchSize: 16
 define amdgpu_kernel void @indirect_2_level_use_80_sgpr() #0 {
   call void @indirect_use_80_sgpr()
   ret void
@@ -336,14 +336,14 @@ define amdgpu_kernel void @indirect_2_level_use_stack() #0 {
 ; GCN-LABEL: {{^}}multi_call_use_use_stack:
 ; GCN:	.set multi_call_use_use_stack.num_vgpr, max(41, use_stack0.num_vgpr, use_stack1.num_vgpr)
 ; GCN:	.set multi_call_use_use_stack.num_agpr, max(0, use_stack0.num_agpr, use_stack1.num_agpr)
-; GCN:	.set multi_call_use_use_stack.numbered_sgpr, max(44, use_stack0.numbered_sgpr, use_stack1.numbered_sgpr)
+; GCN:	.set multi_call_use_use_stack.numbered_sgpr, max(52, use_stack0.numbered_sgpr, use_stack1.numbered_sgpr)
 ; GCN:	.set multi_call_use_use_stack.private_seg_size, 0+(max(use_stack0.private_seg_size, use_stack1.private_seg_size))
 ; GCN:	.set multi_call_use_use_stack.uses_vcc, or(1, use_stack0.uses_vcc, use_stack1.uses_vcc)
 ; GCN:	.set multi_call_use_use_stack.uses_flat_scratch, or(1, use_stack0.uses_flat_scratch, use_stack1.uses_flat_scratch)
 ; GCN:	.set multi_call_use_use_stack.has_dyn_sized_stack, or(0, use_stack0.has_dyn_sized_stack, use_stack1.has_dyn_sized_stack)
 ; GCN:	.set multi_call_use_use_stack.has_recursion, or(0, use_stack0.has_recursion, use_stack1.has_recursion)
 ; GCN:	.set multi_call_use_use_stack.has_indirect_call, or(0, use_stack0.has_indirect_call, use_stack1.has_indirect_call)
-; GCN: TotalNumSgprs: 50
+; GCN: TotalNumSgprs: 58
 ; GCN: NumVgprs: 41
 ; GCN: ScratchSize: 2052
 define amdgpu_kernel void @multi_call_use_use_stack() #0 {
@@ -357,7 +357,7 @@ declare void @external() #0
 ; GCN-LABEL: {{^}}multi_call_with_external:
 ; GCN:	.set multi_call_with_external.num_vgpr, max(41, amdgpu.max_num_vgpr)
 ; GCN:	.set multi_call_with_external.num_agpr, max(0, amdgpu.max_num_agpr)
-; GCN:	.set multi_call_with_external.numbered_sgpr, max(44, amdgpu.max_num_sgpr)
+; GCN:	.set multi_call_with_external.numbered_sgpr, max(52, amdgpu.max_num_sgpr)
 ; GCN:	.set multi_call_with_external.private_seg_size, 0+(max(use_stack0.private_seg_size, use_stack1.private_seg_size))
 ; GCN:	.set multi_call_with_external.uses_vcc, 1
 ; GCN:	.set multi_call_with_external.uses_flat_scratch, 1
@@ -377,7 +377,7 @@ define amdgpu_kernel void @multi_call_with_external() #0 {
 ; GCN-LABEL: {{^}}multi_call_with_external_and_duplicates:
 ; GCN:	.set multi_call_with_external_and_duplicates.num_vgpr, max(41, amdgpu.max_num_vgpr)
 ; GCN:	.set multi_call_with_external_and_duplicates.num_agpr, max(0, amdgpu.max_num_agpr)
-; GCN:	.set multi_call_with_external_and_duplicates.numbered_sgpr, max(46, amdgpu.max_num_sgpr)
+; GCN:	.set multi_call_with_external_and_duplicates.numbered_sgpr, max(54, amdgpu.max_num_sgpr)
 ; GCN:	.set multi_call_with_external_and_duplicates.private_seg_size, 0+(max(use_stack0.private_seg_size, use_stack1.private_seg_size))
 ; GCN:	.set multi_call_with_external_and_duplicates.uses_vcc, 1
 ; GCN:	.set multi_call_with_external_and_duplicates.uses_flat_scratch, 1
@@ -594,7 +594,7 @@ define amdgpu_kernel void @usage_multi_stage_recurse_noattrs(i32 %n) #0 {
 ; GCN-LABEL: {{^}}multi_call_with_multi_stage_recurse:
 ; GCN:  .set multi_call_with_multi_stage_recurse.num_vgpr, max(41, use_stack0.num_vgpr, use_stack1.num_vgpr, multi_stage_recurse1.num_vgpr)
 ; GCN:  .set multi_call_with_multi_stage_recurse.num_agpr, max(0, use_stack0.num_agpr, use_stack1.num_agpr, multi_stage_recurse1.num_agpr)
-; GCN:  .set multi_call_with_multi_stage_recurse.numbered_sgpr, max(45, use_stack0.numbered_sgpr, use_stack1.numbered_sgpr, multi_stage_recurse1.numbered_sgpr)
+; GCN:  .set multi_call_with_multi_stage_recurse.numbered_sgpr, max(53, use_stack0.numbered_sgpr, use_stack1.numbered_sgpr, multi_stage_recurse1.numbered_sgpr)
 ; GCN:  .set multi_call_with_multi_stage_recurse.private_seg_size, 0+(max(use_stack0.private_seg_size, use_stack1.private_seg_size, multi_stage_recurse1.private_seg_size))
 ; GCN:  .set multi_call_with_multi_stage_recurse.uses_vcc, or(1, use_stack0.uses_vcc, use_stack1.uses_vcc, multi_stage_recurse1.uses_vcc)
 ; GCN:  .set multi_call_with_multi_stage_recurse.uses_flat_scratch, or(1, use_stack0.uses_flat_scratch, use_stack1.uses_flat_scratch, multi_stage_recurse1.uses_flat_scratch)
diff --git a/llvm/test/CodeGen/AMDGPU/function-returns.ll b/llvm/test/CodeGen/AMDGPU/function-returns.ll
index ac9f56d1ee7b1..128f40a34de89 100644
--- a/llvm/test/CodeGen/AMDGPU/function-returns.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-returns.ll
@@ -2207,10 +2207,10 @@ define { <3 x i32>, i32 } @v3i32_struct_func_void_wasted_reg() #0 {
   %load2 = load volatile i32, ptr addrspace(3) undef
   %load3 = load volatile i32, ptr addrspace(3) undef
 
-  %insert.0 = insertelement <3 x i32> undef, i32 %load0, i32 0
+  %insert.0 = insertelement <3 x i32> poison, i32 %load0, i32 0
   %insert.1 = insertelement <3 x i32> %insert.0, i32 %load1, i32 1
   %insert.2 = insertelement <3 x i32> %insert.1, i32 %load2, i32 2
-  %insert.3 = insertvalue { <3 x i32>, i32 } undef, <3 x i32> %insert.2, 0
+  %insert.3 = insertvalue { <3 x i32>, i32 } poison, <3 x i32> %insert.2, 0
   %insert.4 = insertvalue { <3 x i32>, i32 } %insert.3, i32 %load3, 1
   ret { <3 x i32>, i32 } %insert.4
 }
@@ -2266,10 +2266,10 @@ define { <3 x float>, i32 } @v3f32_struct_func_void_wasted_reg() #0 {
   %load2 = load volatile float, ptr addrspace(3) undef
   %load3 = load volatile i32, ptr addrspace(3) undef
 
-  %insert.0 = insertelement <3 x float> undef, float %load0, i32 0
+  %insert.0 = insertelement <3 x float> poison, float %load0, i32 0
   %insert.1 = insertelement <3 x float> %insert.0, float %load1, i32 1
   %insert.2 = insertelement <3 x float> %insert.1, float %load2, i32 2
-  %insert.3 = insertvalue { <3 x float>, i32 } undef, <3 x float> %insert.2, 0
+  %insert.3 = insertvalue { <3 x float>, i32 } poison, <3 x float> %insert.2, 0
   %insert.4 = insertvalue { <3 x float>, i32 } %insert.3, i32 %load3, 1
   ret { <3 x float>, i32 } %insert.4
 }
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-call-non-gfx-func.ll b/llvm/test/CodeGen/AMDGPU/gfx-call-non-gfx-func.ll
index 1ad365df2e8a8..8ac187eacf1fe 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-call-non-gfx-func.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-call-non-gfx-func.ll
@@ -40,12 +40,44 @@ define amdgpu_gfx void @gfx_func() {
 ; SDAG-NEXT:    v_writelane_b32 v40, s28, 24
 ; SDAG-NEXT:    v_writelane_b32 v40, s29, 25
 ; SDAG-NEXT:    v_writelane_b32 v40, s30, 26
+; SDAG-NEXT:    v_writelane_b32 v40, s31, 27
+; SDAG-NEXT:    v_writelane_b32 v40, s72, 28
+; SDAG-NEXT:    v_writelane_b32 v40, s73, 29
+; SDAG-NEXT:    v_writelane_b32 v40, s74, 30
+; SDAG-NEXT:    v_writelane_b32 v40, s75, 31
+; SDAG-NEXT:    v_writelane_b32 v40, s76, 32
+; SDAG-NEXT:    v_writelane_b32 v40, s77, 33
+; SDAG-NEXT:    v_writelane_b32 v40, s78, 34
+; SDAG-NEXT:    v_writelane_b32 v40, s79, 35
+; SDAG-NEXT:    v_writelane_b32 v40, s88, 36
+; SDAG-NEXT:    v_writelane_b32 v40, s89, 37
+; SDAG-NEXT:    v_writelane_b32 v40, s90, 38
+; SDAG-NEXT:    v_writelane_b32 v40, s91, 39
+; SDAG-NEXT:    v_writelane_b32 v40, s92, 40
+; SDAG-NEXT:    v_writelane_b32 v40, s93, 41
+; SDAG-NEXT:    v_writelane_b32 v40, s94, 42
 ; SDAG-NEXT:    s_mov_b32 s35, extern_c_func@abs32@hi
 ; SDAG-NEXT:    s_mov_b32 s34, extern_c_func@abs32@lo
 ; SDAG-NEXT:    s_mov_b64 s[8:9], 0
 ; SDAG-NEXT:    s_addk_i32 s32, 0x400
-; SDAG-NEXT:    v_writelane_b32 v40, s31, 27
+; SDAG-NEXT:    v_writelane_b32 v40, s95, 43
 ; SDAG-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; SDAG-NEXT:    v_readlane_b32 s95, v40, 43
+; SDAG-NEXT:    v_readlane_b32 s94, v40, 42
+; SDAG-NEXT:    v_readlane_b32 s93, v40, 41
+; SDAG-NEXT:    v_readlane_b32 s92, v40, 40
+; SDAG-NEXT:    v_readlane_b32 s91, v40, 39
+; SDAG-NEXT:    v_readlane_b32 s90, v40, 38
+; SDAG-NEXT:    v_readlane_b32 s89, v40, 37
+; SDAG-NEXT:    v_readlane_b32 s88, v40, 36
+; SDAG-NEXT:    v_readlane_b32 s79, v40, 35
+; SDAG-NEXT:    v_readlane_b32 s78, v40, 34
+; SDAG-NEXT:    v_readlane_b32 s77, v40, 33
+; SDAG-NEXT:    v_readlane_b32 s76, v40, 32
+; SDAG-NEXT:    v_readlane_b32 s75, v40, 31
+; SDAG-NEXT:    v_readlane_b32 s74, v40, 30
+; SDAG-NEXT:    v_readlane_b32 s73, v40, 29
+; SDAG-NEXT:    v_readlane_b32 s72, v40, 28
 ; SDAG-NEXT:    v_readlane_b32 s31, v40, 27
 ; SDAG-NEXT:    v_readlane_b32 s30, v40, 26
 ; SDAG-NEXT:    v_readlane_b32 s29, v40, 25
@@ -117,12 +149,44 @@ define amdgpu_gfx void @gfx_func() {
 ; GISEL-NEXT:    v_writelane_b32 v40, s28, 24
 ; GISEL-NEXT:    v_writelane_b32 v40, s29, 25
 ; GISEL-NEXT:    v_writelane_b32 v40, s30, 26
+; GISEL-NEXT:    v_writelane_b32 v40, s31, 27
+; GISEL-NEXT:    v_writelane_b32 v40, s72, 28
+; GISEL-NEXT:    v_writelane_b32 v40, s73, 29
+; GISEL-NEXT:    v_writelane_b32 v40, s74, 30
+; GISEL-NEXT:    v_writelane_b32 v40, s75, 31
+; GISEL-NEXT:    v_writelane_b32 v40, s76, 32
+; GISEL-NEXT:    v_writelane_b32 v40, s77, 33
+; GISEL-NEXT:    v_writelane_b32 v40, s78, 34
+; GISEL-NEXT:    v_writelane_b32 v40, s79, 35
+; GISEL-NEXT:    v_writelane_b32 v40, s88, 36
+; GISEL-NEXT:    v_writelane_b32 v40, s89, 37
+; GISEL-NEXT:    v_writelane_b32 v40, s90, 38
+; GISEL-NEXT:    v_writelane_b32 v40, s91, 39
+; GISEL-NEXT:    v_writelane_b32 v40, s92, 40
+; GISEL-NEXT:    v_writelane_b32 v40, s93, 41
+; GISEL-NEXT:    v_writelane_b32 v40, s94, 42
 ; GISEL-NEXT:    s_mov_b32 s34, extern_c_func@abs32@lo
 ; GISEL-NEXT:    s_mov_b32 s35, extern_c_func@abs32@hi
 ; GISEL-NEXT:    s_mov_b64 s[8:9], 0
 ; GISEL-NEXT:    s_addk_i32 s32, 0x400
-; GISEL-NEXT:    v_writelane_b32 v40, s31, 27
+; GISEL-NEXT:    v_writelane_b32 v40, s95, 43
 ; GISEL-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GISEL-NEXT:    v_readlane_b32 s95, v40, 43
+; GISEL-NEXT:    v_readlane_b32 s94, v40, 42
+; GISEL-NEXT:    v_readlane_b32 s93, v40, 41
+; GISEL-NEXT:    v_readlane_b32 s92, v40, 40
+; GISEL-NEXT:    v_readlane_b32 s91, v40, 39
+; GISEL-NEXT:    v_readlane_b32 s90, v40, 38
+; GISEL-NEXT:    v_readlane_b32 s89, v40, 37
+; GISEL-NEXT:    v_readlane_b32 s88, v40, 36
+; GISEL-NEXT:    v_readlane_b32 s79, v40, 35
+; GISEL-NEXT:    v_readlane_b32 s78, v40, 34
+; GISEL-NEXT:    v_readlane_b32 s77, v40, 33
+; GISEL-NEXT:    v_readlane_b32 s76, v40, 32
+; GISEL-NEXT:    v_readlane_b32 s75, v40, 31
+; GISEL-NEXT:    v_readlane_b32 s74, v40, 30
+; GISEL-NEXT:    v_readlane_b32 s73, v40, 29
+; GISEL-NEXT:    v_readlane_b32 s72, v40, 28
 ; GISEL-NEXT:    v_readlane_b32 s31, v40, 27
 ; GISEL-NEXT:    v_readlane_b32 s30, v40, 26
 ; GISEL-NEXT:    v_readlane_b32 s29, v40, 25
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
index 2e3ca34af4c74..c4ea8fc67122c 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
@@ -1841,7 +1841,7 @@ define amdgpu_gfx void @test_call_external_void_func_v3i64() #0 {
 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-SCRATCH-NEXT:    s_setpc_b64 s[30:31]
   %load = load <2 x i64>, ptr addrspace(1) null
-  %val = shufflevector <2 x i64> %load, <2 x i64> <i64 8589934593, i64 undef>, <3 x i32> <i32 0, i32 1, i32 2>
+  %val = shufflevector <2 x i64> %load, <2 x i64> <i64 8589934593, i64 poison>, <3 x i32> <i32 0, i32 1, i32 2>
 
   call amdgpu_gfx void @external_void_func_v3i64(<3 x i64> %val)
   ret void
@@ -9093,62 +9093,30 @@ define void @tail_call_byval_align16(<32 x i32> %val, double %tmp) #0 {
 ; GFX9-NEXT:    v_writelane_b32 v40, s37, 5
 ; GFX9-NEXT:    v_writelane_b32 v40, s38, 6
 ; GFX9-NEXT:    v_writelane_b32 v40, s39, 7
-; GFX9-NEXT:    v_writelane_b32 v40, s40, 8
-; GFX9-NEXT:    v_writelane_b32 v40, s41, 9
-; GFX9-NEXT:    v_writelane_b32 v40, s42, 10
-; GFX9-NEXT:    v_writelane_b32 v40, s43, 11
-; GFX9-NEXT:    v_writelane_b32 v40, s44, 12
-; GFX9-NEXT:    v_writelane_b32 v40, s45, 13
-; GFX9-NEXT:    v_writelane_b32 v40, s46, 14
-; GFX9-NEXT:    v_writelane_b32 v40, s47, 15
-; GFX9-NEXT:    v_writelane_b32 v40, s48, 16
-; GFX9-NEXT:    v_writelane_b32 v40, s49, 17
-; GFX9-NEXT:    v_writelane_b32 v40, s50, 18
-; GFX9-NEXT:    v_writelane_b32 v40, s51, 19
-; GFX9-NEXT:    v_writelane_b32 v40, s52, 20
-; GFX9-NEXT:    v_writelane_b32 v40, s53, 21
-; GFX9-NEXT:    v_writelane_b32 v40, s54, 22
-; GFX9-NEXT:    v_writelane_b32 v40, s55, 23
-; GFX9-NEXT:    v_writelane_b32 v40, s56, 24
-; GFX9-NEXT:    v_writelane_b32 v40, s57, 25
-; GFX9-NEXT:    v_writelane_b32 v40, s58, 26
-; GFX9-NEXT:    v_writelane_b32 v40, s59, 27
-; GFX9-NEXT:    v_writelane_b32 v40, s60, 28
-; GFX9-NEXT:    v_writelane_b32 v40, s61, 29
+; GFX9-NEXT:    v_writelane_b32 v40, s48, 8
+; GFX9-NEXT:    v_writelane_b32 v40, s49, 9
+; GFX9-NEXT:    v_writelane_b32 v40, s50, 10
+; GFX9-NEXT:    v_writelane_b32 v40, s51, 11
+; GFX9-NEXT:    v_writelane_b32 v40, s52, 12
+; GFX9-NEXT:    v_writelane_b32 v40, s53, 13
 ; GFX9-NEXT:    s_addk_i32 s32, 0x800
-; GFX9-NEXT:    v_writelane_b32 v40, s62, 30
+; GFX9-NEXT:    v_writelane_b32 v40, s54, 14
 ; GFX9-NEXT:    s_mov_b32 s5, byval_align16_f64_arg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s4, byval_align16_f64_arg@abs32@lo
-; GFX9-NEXT:    v_writelane_b32 v40, s63, 31
+; GFX9-NEXT:    v_writelane_b32 v40, s55, 15
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v32, off, s[0:3], s32
 ; GFX9-NEXT:    s_waitcnt vmcnt(2)
 ; GFX9-NEXT:    buffer_store_dword v33, off, s[0:3], s32 offset:4
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[4:5]
-; GFX9-NEXT:    v_readlane_b32 s63, v40, 31
-; GFX9-NEXT:    v_readlane_b32 s62, v40, 30
-; GFX9-NEXT:    v_readlane_b32 s61, v40, 29
-; GFX9-NEXT:    v_readlane_b32 s60, v40, 28
-; GFX9-NEXT:    v_readlane_b32 s59, v40, 27
-; GFX9-NEXT:    v_readlane_b32 s58, v40, 26
-; GFX9-NEXT:    v_readlane_b32 s57, v40, 25
-; GFX9-NEXT:    v_readlane_b32 s56, v40, 24
-; GFX9-NEXT:    v_readlane_b32 s55, v40, 23
-; GFX9-NEXT:    v_readlane_b32 s54, v40, 22
-; GFX9-NEXT:    v_readlane_b32 s53, v40, 21
-; GFX9-NEXT:    v_readlane_b32 s52, v40, 20
-; GFX9-NEXT:    v_readlane_b32 s51, v40, 19
-; GFX9-NEXT:    v_readlane_b32 s50, v40, 18
-; GFX9-NEXT:    v_readlane_b32 s49, v40, 17
-; GFX9-NEXT:    v_readlane_b32 s48, v40, 16
-; GFX9-NEXT:    v_readlane_b32 s47, v40, 15
-; GFX9-NEXT:    v_readlane_b32 s46, v40, 14
-; GFX9-NEXT:    v_readlane_b32 s45, v40, 13
-; GFX9-NEXT:    v_readlane_b32 s44, v40, 12
-; GFX9-NEXT:    v_readlane_b32 s43, v40, 11
-; GFX9-NEXT:    v_readlane_b32 s42, v40, 10
-; GFX9-NEXT:    v_readlane_b32 s41, v40, 9
-; GFX9-NEXT:    v_readlane_b32 s40, v40, 8
+; GFX9-NEXT:    v_readlane_b32 s55, v40, 15
+; GFX9-NEXT:    v_readlane_b32 s54, v40, 14
+; GFX9-NEXT:    v_readlane_b32 s53, v40, 13
+; GFX9-NEXT:    v_readlane_b32 s52, v40, 12
+; GFX9-NEXT:    v_readlane_b32 s51, v40, 11
+; GFX9-NEXT:    v_readlane_b32 s50, v40, 10
+; GFX9-NEXT:    v_readlane_b32 s49, v40, 9
+; GFX9-NEXT:    v_readlane_b32 s48, v40, 8
 ; GFX9-NEXT:    v_readlane_b32 s39, v40, 7
 ; GFX9-NEXT:    v_readlane_b32 s38, v40, 6
 ; GFX9-NEXT:    v_readlane_b32 s37, v40, 5
@@ -9193,55 +9161,23 @@ define void @tail_call_byval_align16(<32 x i32> %val, double %tmp) #0 {
 ; GFX10-NEXT:    v_writelane_b32 v40, s37, 5
 ; GFX10-NEXT:    v_writelane_b32 v40, s38, 6
 ; GFX10-NEXT:    v_writelane_b32 v40, s39, 7
-; GFX10-NEXT:    v_writelane_b32 v40, s40, 8
-; GFX10-NEXT:    v_writelane_b32 v40, s41, 9
-; GFX10-NEXT:    v_writelane_b32 v40, s42, 10
-; GFX10-NEXT:    v_writelane_b32 v40, s43, 11
-; GFX10-NEXT:    v_writelane_b32 v40, s44, 12
-; GFX10-NEXT:    v_writelane_b32 v40, s45, 13
-; GFX10-NEXT:    v_writelane_b32 v40, s46, 14
-; GFX10-NEXT:    v_writelane_b32 v40, s47, 15
-; GFX10-NEXT:    v_writelane_b32 v40, s48, 16
-; GFX10-NEXT:    v_writelane_b32 v40, s49, 17
-; GFX10-NEXT:    v_writelane_b32 v40, s50, 18
-; GFX10-NEXT:    v_writelane_b32 v40, s51, 19
-; GFX10-NEXT:    v_writelane_b32 v40, s52, 20
-; GFX10-NEXT:    v_writelane_b32 v40, s53, 21
-; GFX10-NEXT:    v_writelane_b32 v40, s54, 22
-; GFX10-NEXT:    v_writelane_b32 v40, s55, 23
-; GFX10-NEXT:    v_writelane_b32 v40, s56, 24
-; GFX10-NEXT:    v_writelane_b32 v40, s57, 25
-; GFX10-NEXT:    v_writelane_b32 v40, s58, 26
-; GFX10-NEXT:    v_writelane_b32 v40, s59, 27
-; GFX10-NEXT:    v_writelane_b32 v40, s60, 28
-; GFX10-NEXT:    v_writelane_b32 v40, s61, 29
-; GFX10-NEXT:    v_writelane_b32 v40, s62, 30
-; GFX10-NEXT:    v_writelane_b32 v40, s63, 31
+; GFX10-NEXT:    v_writelane_b32 v40, s48, 8
+; GFX10-NEXT:    v_writelane_b32 v40, s49, 9
+; GFX10-NEXT:    v_writelane_b32 v40, s50, 10
+; GFX10-NEXT:    v_writelane_b32 v40, s51, 11
+; GFX10-NEXT:    v_writelane_b32 v40, s52, 12
+; GFX10-NEXT:    v_writelane_b32 v40, s53, 13
+; GFX10-NEXT:    v_writelane_b32 v40, s54, 14
+; GFX10-NEXT:    v_writelane_b32 v40, s55, 15
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
-; GFX10-NEXT:    v_readlane_b32 s63, v40, 31
-; GFX10-NEXT:    v_readlane_b32 s62, v40, 30
-; GFX10-NEXT:    v_readlane_b32 s61, v40, 29
-; GFX10-NEXT:    v_readlane_b32 s60, v40, 28
-; GFX10-NEXT:    v_readlane_b32 s59, v40, 27
-; GFX10-NEXT:    v_readlane_b32 s58, v40, 26
-; GFX10-NEXT:    v_readlane_b32 s57, v40, 25
-; GFX10-NEXT:    v_readlane_b32 s56, v40, 24
-; GFX10-NEXT:    v_readlane_b32 s55, v40, 23
-; GFX10-NEXT:    v_readlane_b32 s54, v40, 22
-; GFX10-NEXT:    v_readlane_b32 s53, v40, 21
-; GFX10-NEXT:    v_readlane_b32 s52, v40, 20
-; GFX10-NEXT:    v_readlane_b32 s51, v40, 19
-; GFX10-NEXT:    v_readlane_b32 s50, v40, 18
-; GFX10-NEXT:    v_readlane_b32 s49, v40, 17
-; GFX10-NEXT:    v_readlane_b32 s48, v40, 16
-; GFX10-NEXT:    v_readlane_b32 s47, v40, 15
-; GFX10-NEXT:    v_readlane_b32 s46, v40, 14
-; GFX10-NEXT:    v_readlane_b32 s45, v40, 13
-; GFX10-NEXT:    v_readlane_b32 s44, v40, 12
-; GFX10-NEXT:    v_readlane_b32 s43, v40, 11
-; GFX10-NEXT:    v_readlane_b32 s42, v40, 10
-; GFX10-NEXT:    v_readlane_b32 s41, v40, 9
-; GFX10-NEXT:    v_readlane_b32 s40, v40, 8
+; GFX10-NEXT:    v_readlane_b32 s55, v40, 15
+; GFX10-NEXT:    v_readlane_b32 s54, v40, 14
+; GFX10-NEXT:    v_readlane_b32 s53, v40, 13
+; GFX10-NEXT:    v_readlane_b32 s52, v40, 12
+; GFX10-NEXT:    v_readlane_b32 s51, v40, 11
+; GFX10-NEXT:    v_readlane_b32 s50, v40, 10
+; GFX10-NEXT:    v_readlane_b32 s49, v40, 9
+; GFX10-NEXT:    v_readlane_b32 s48, v40, 8
 ; GFX10-NEXT:    v_readlane_b32 s39, v40, 7
 ; GFX10-NEXT:    v_readlane_b32 s38, v40, 6
 ; GFX10-NEXT:    v_readlane_b32 s37, v40, 5
@@ -9281,57 +9217,25 @@ define void @tail_call_byval_align16(<32 x i32> %val, double %tmp) #0 {
 ; GFX11-NEXT:    v_writelane_b32 v40, s37, 5
 ; GFX11-NEXT:    v_writelane_b32 v40, s38, 6
 ; GFX11-NEXT:    v_writelane_b32 v40, s39, 7
-; GFX11-NEXT:    v_writelane_b32 v40, s40, 8
-; GFX11-NEXT:    v_writelane_b32 v40, s41, 9
-; GFX11-NEXT:    v_writelane_b32 v40, s42, 10
-; GFX11-NEXT:    v_writelane_b32 v40, s43, 11
-; GFX11-NEXT:    v_writelane_b32 v40, s44, 12
-; GFX11-NEXT:    v_writelane_b32 v40, s45, 13
-; GFX11-NEXT:    v_writelane_b32 v40, s46, 14
-; GFX11-NEXT:    v_writelane_b32 v40, s47, 15
-; GFX11-NEXT:    v_writelane_b32 v40, s48, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s49, 17
-; GFX11-NEXT:    v_writelane_b32 v40, s50, 18
-; GFX11-NEXT:    v_writelane_b32 v40, s51, 19
-; GFX11-NEXT:    v_writelane_b32 v40, s52, 20
-; GFX11-NEXT:    v_writelane_b32 v40, s53, 21
-; GFX11-NEXT:    v_writelane_b32 v40, s54, 22
-; GFX11-NEXT:    v_writelane_b32 v40, s55, 23
-; GFX11-NEXT:    v_writelane_b32 v40, s56, 24
-; GFX11-NEXT:    v_writelane_b32 v40, s57, 25
-; GFX11-NEXT:    v_writelane_b32 v40, s58, 26
-; GFX11-NEXT:    v_writelane_b32 v40, s59, 27
-; GFX11-NEXT:    v_writelane_b32 v40, s60, 28
-; GFX11-NEXT:    v_writelane_b32 v40, s61, 29
-; GFX11-NEXT:    v_writelane_b32 v40, s62, 30
-; GFX11-NEXT:    v_writelane_b32 v40, s63, 31
+; GFX11-NEXT:    v_writelane_b32 v40, s48, 8
+; GFX11-NEXT:    v_writelane_b32 v40, s49, 9
+; GFX11-NEXT:    v_writelane_b32 v40, s50, 10
+; GFX11-NEXT:    v_writelane_b32 v40, s51, 11
+; GFX11-NEXT:    v_writelane_b32 v40, s52, 12
+; GFX11-NEXT:    v_writelane_b32 v40, s53, 13
+; GFX11-NEXT:    v_writelane_b32 v40, s54, 14
+; GFX11-NEXT:    v_writelane_b32 v40, s55, 15
 ; GFX11-NEXT:    s_waitcnt vmcnt(1)
 ; GFX11-NEXT:    scratch_store_b64 off, v[32:33], s32
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX11-NEXT:    v_readlane_b32 s63, v40, 31
-; GFX11-NEXT:    v_readlane_b32 s62, v40, 30
-; GFX11-NEXT:    v_readlane_b32 s61, v40, 29
-; GFX11-NEXT:    v_readlane_b32 s60, v40, 28
-; GFX11-NEXT:    v_readlane_b32 s59, v40, 27
-; GFX11-NEXT:    v_readlane_b32 s58, v40, 26
-; GFX11-NEXT:    v_readlane_b32 s57, v40, 25
-; GFX11-NEXT:    v_readlane_b32 s56, v40, 24
-; GFX11-NEXT:    v_readlane_b32 s55, v40, 23
-; GFX11-NEXT:    v_readlane_b32 s54, v40, 22
-; GFX11-NEXT:    v_readlane_b32 s53, v40, 21
-; GFX11-NEXT:    v_readlane_b32 s52, v40, 20
-; GFX11-NEXT:    v_readlane_b32 s51, v40, 19
-; GFX11-NEXT:    v_readlane_b32 s50, v40, 18
-; GFX11-NEXT:    v_readlane_b32 s49, v40, 17
-; GFX11-NEXT:    v_readlane_b32 s48, v40, 16
-; GFX11-NEXT:    v_readlane_b32 s47, v40, 15
-; GFX11-NEXT:    v_readlane_b32 s46, v40, 14
-; GFX11-NEXT:    v_readlane_b32 s45, v40, 13
-; GFX11-NEXT:    v_readlane_b32 s44, v40, 12
-; GFX11-NEXT:    v_readlane_b32 s43, v40, 11
-; GFX11-NEXT:    v_readlane_b32 s42, v40, 10
-; GFX11-NEXT:    v_readlane_b32 s41, v40, 9
-; GFX11-NEXT:    v_readlane_b32 s40, v40, 8
+; GFX11-NEXT:    v_readlane_b32 s55, v40, 15
+; GFX11-NEXT:    v_readlane_b32 s54, v40, 14
+; GFX11-NEXT:    v_readlane_b32 s53, v40, 13
+; GFX11-NEXT:    v_readlane_b32 s52, v40, 12
+; GFX11-NEXT:    v_readlane_b32 s51, v40, 11
+; GFX11-NEXT:    v_readlane_b32 s50, v40, 10
+; GFX11-NEXT:    v_readlane_b32 s49, v40, 9
+; GFX11-NEXT:    v_readlane_b32 s48, v40, 8
 ; GFX11-NEXT:    v_readlane_b32 s39, v40, 7
 ; GFX11-NEXT:    v_readlane_b32 s38, v40, 6
 ; GFX11-NEXT:    v_readlane_b32 s37, v40, 5
@@ -9371,57 +9275,25 @@ define void @tail_call_byval_align16(<32 x i32> %val, double %tmp) #0 {
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s37, 5
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s38, 6
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s39, 7
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s40, 8
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s41, 9
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s42, 10
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s43, 11
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s44, 12
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s45, 13
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s46, 14
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s47, 15
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s48, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s49, 17
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s50, 18
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s51, 19
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s52, 20
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s53, 21
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s54, 22
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s55, 23
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s56, 24
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s57, 25
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s58, 26
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s59, 27
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s60, 28
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s61, 29
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s62, 30
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s63, 31
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s48, 8
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s49, 9
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s50, 10
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s51, 11
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s52, 12
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s53, 13
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s54, 14
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s55, 15
 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(1)
 ; GFX10-SCRATCH-NEXT:    scratch_store_dwordx2 off, v[32:33], s32
 ; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s63, v40, 31
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s62, v40, 30
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s61, v40, 29
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s60, v40, 28
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s59, v40, 27
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s58, v40, 26
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s57, v40, 25
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s56, v40, 24
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s55, v40, 23
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s54, v40, 22
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s53, v40, 21
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s52, v40, 20
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s51, v40, 19
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s50, v40, 18
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s49, v40, 17
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s48, v40, 16
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s47, v40, 15
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s46, v40, 14
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s45, v40, 13
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s44, v40, 12
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s43, v40, 11
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s42, v40, 10
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s41, v40, 9
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s40, v40, 8
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s55, v40, 15
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s54, v40, 14
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s53, v40, 13
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s52, v40, 12
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s51, v40, 11
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s50, v40, 10
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s49, v40, 9
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s48, v40, 8
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s39, v40, 7
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s38, v40, 6
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s37, v40, 5
@@ -10567,7 +10439,7 @@ define amdgpu_gfx void @test_call_external_void_func_v3i64_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-SCRATCH-NEXT:    s_setpc_b64 s[30:31]
   %load = load <2 x i64>, ptr addrspace(4) null
-  %val = shufflevector <2 x i64> %load, <2 x i64> <i64 8589934593, i64 undef>, <3 x i32> <i32 0, i32 1, i32 2>
+  %val = shufflevector <2 x i64> %load, <2 x i64> <i64 8589934593, i64 poison>, <3 x i32> <i32 0, i32 1, i32 2>
 
   call amdgpu_gfx void @external_void_func_v3i64_inreg(<3 x i64> inreg %val)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/global-load-saddr-to-vaddr.ll b/llvm/test/CodeGen/AMDGPU/global-load-saddr-to-vaddr.ll
index b9592a9ff9073..e2d33df64dec2 100644
--- a/llvm/test/CodeGen/AMDGPU/global-load-saddr-to-vaddr.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-load-saddr-to-vaddr.ll
@@ -85,7 +85,7 @@ bb3:                                              ; preds = %bb3, %bb
   %i4 = zext i32 %i to i64
   %i5 = getelementptr inbounds i16, ptr addrspace(1) %arg, i64 %i4
   %i6 = load volatile i16, ptr addrspace(1) %i5, align 4
-  %insertelt = insertelement <2 x i16> undef, i16 %i6, i32 1
+  %insertelt = insertelement <2 x i16> poison, i16 %i6, i32 1
   %i8 =  bitcast <2 x i16> %insertelt to i32
   %i9 = icmp eq i32 %i8, 256
   br i1 %i9, label %bb2, label %bb3
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
index bc49f70cbee11..0cd75f3fc208c 100644
--- a/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
@@ -3849,7 +3849,7 @@ define amdgpu_ps <2 x half> @global_load_saddr_i16_d16lo_undef_hi(ptr addrspace(
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %load = load i16, ptr addrspace(1) %gep0
-  %build = insertelement <2 x i16> undef, i16 %load, i32 0
+  %build = insertelement <2 x i16> poison, i16 %load, i32 0
   %cast = bitcast <2 x i16> %build to <2 x half>
   ret <2 x half> %cast
 }
@@ -3882,7 +3882,7 @@ define amdgpu_ps <2 x half> @global_load_saddr_i16_d16lo_undef_hi_immneg128(ptr
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
   %load = load i16, ptr addrspace(1) %gep1
-  %build = insertelement <2 x i16> undef, i16 %load, i32 0
+  %build = insertelement <2 x i16> poison, i16 %load, i32 0
   %cast = bitcast <2 x i16> %build to <2 x half>
   ret <2 x half> %cast
 }
@@ -4225,7 +4225,7 @@ define amdgpu_ps <2 x half> @global_load_saddr_i16_d16hi_undef_hi(ptr addrspace(
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %load = load i16, ptr addrspace(1) %gep0
-  %build = insertelement <2 x i16> undef, i16 %load, i32 1
+  %build = insertelement <2 x i16> poison, i16 %load, i32 1
   %cast = bitcast <2 x i16> %build to <2 x half>
   ret <2 x half> %cast
 }
@@ -4259,7 +4259,7 @@ define amdgpu_ps <2 x half> @global_load_saddr_i16_d16hi_undef_hi_immneg128(ptr
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
   %load = load i16, ptr addrspace(1) %gep1
-  %build = insertelement <2 x i16> undef, i16 %load, i32 1
+  %build = insertelement <2 x i16> poison, i16 %load, i32 1
   %cast = bitcast <2 x i16> %build to <2 x half>
   ret <2 x half> %cast
 }
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
index 15be44a335a1d..4f5c46d5f424f 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
@@ -735,12 +735,12 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -760,8 +760,8 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[36:39], 0
@@ -1955,12 +1955,12 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_uni_address_div_value_one_as_scope_unsafe_structfp:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -1980,8 +1980,8 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[36:39], 0
@@ -3235,12 +3235,12 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_uni_address_div_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -3260,8 +3260,8 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[36:39], 0
@@ -4011,12 +4011,12 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_uni_address_div_value_agent_scope_unsafe_structfp:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -4036,8 +4036,8 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[36:39], 0
@@ -5316,12 +5316,12 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_uni_address_div_value_default_scope_strictfp:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -5341,8 +5341,8 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[36:39], 0
@@ -5747,14 +5747,14 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
 ; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, s0, 0
 ; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, s1, v3
@@ -5763,16 +5763,16 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX7LESS-NEXT:  ; %bb.1:
 ; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX7LESS-NEXT:    v_cvt_f64_u32_e32 v[1:2], s2
 ; GFX7LESS-NEXT:    v_or_b32_e32 v4, v0, v4
@@ -5785,11 +5785,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:8
+; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
+; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
 ; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
@@ -5802,42 +5802,42 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX7LESS-NEXT:  .LBB9_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s50, -1
-; GFX9-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s66, -1
+; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    s_mov_b32 s51, 0xe00000
+; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, s0, 0
-; GFX9-NEXT:    s_add_u32 s48, s48, s11
+; GFX9-NEXT:    s_add_u32 s64, s64, s11
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, s1, v3
-; GFX9-NEXT:    s_addc_u32 s49, s49, 0
+; GFX9-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX9-NEXT:    s_movk_i32 s32, 0x800
@@ -5845,21 +5845,21 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX9-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX9-NEXT:  ; %bb.1:
 ; GFX9-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX9-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX9-NEXT:    s_mov_b32 s33, s10
 ; GFX9-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX9-NEXT:    s_mov_b32 s42, s9
-; GFX9-NEXT:    s_mov_b32 s43, s8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX9-NEXT:    s_mov_b32 s50, s9
+; GFX9-NEXT:    s_mov_b32 s51, s8
 ; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s1
-; GFX9-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s0
 ; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-NEXT:  .LBB9_2: ; %atomicrmw.start
@@ -5872,68 +5872,68 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v3, off, s[48:51], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; GFX9-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:4
+; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v3, off, s[48:51], 0
-; GFX9-NEXT:    buffer_load_dword v4, off, s[48:51], 0 offset:4
+; GFX9-NEXT:    buffer_load_dword v3, off, s[64:67], 0
+; GFX9-NEXT:    buffer_load_dword v4, off, s[64:67], 0 offset:4
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX9-NEXT:  .LBB9_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s50, -1
-; GFX1064-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s66, -1
+; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-NEXT:    s_mov_b32 s33, s10
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1064-NEXT:    s_addc_u32 s49, s49, 0
+; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, s10, 0
-; GFX1064-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, s11, v3
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1064-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1064-NEXT:    s_mov_b32 s42, s9
+; GFX1064-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1064-NEXT:    s_mov_b32 s43, s8
+; GFX1064-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1064-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
@@ -5950,69 +5950,69 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s12, s51
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1064-NEXT:  .LBB9_3:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s42, s9
+; GFX1032-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-NEXT:    s_mov_b32 s9, exec_lo
-; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
-; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s66, -1
+; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1032-NEXT:    s_mov_b32 s46, 0
+; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1032-NEXT:    s_bcnt1_i32_b32 s0, s9
 ; GFX1032-NEXT:    s_mov_b32 s33, s10
 ; GFX1032-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1032-NEXT:    s_mov_b32 s43, s8
+; GFX1032-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1032-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
@@ -6029,37 +6029,37 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s12, s51
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1032-NEXT:  .LBB9_3:
 ; GFX1032-NEXT:    s_endpgm
@@ -6070,7 +6070,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], exec
 ; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, s10, 0
-; GFX1164-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -6079,16 +6079,16 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX1164-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1164-NEXT:  ; %bb.1:
 ; GFX1164-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1164-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1164-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-NEXT:    s_mov_b32 s42, s9
-; GFX1164-NEXT:    s_mov_b32 s43, s8
+; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1164-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
@@ -6111,18 +6111,18 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -6130,8 +6130,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1164-NEXT:  .LBB9_3:
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -6142,24 +6142,24 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX1132-NEXT:    s_mov_b32 s8, exec_lo
 ; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX1132-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1132-NEXT:    s_mov_b32 s46, 0
+; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1132-NEXT:  ; %bb.1:
 ; GFX1132-NEXT:    s_bcnt1_i32_b32 s0, s8
-; GFX1132-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1132-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
 ; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s42, s14
-; GFX1132-NEXT:    s_mov_b32 s43, s13
+; GFX1132-NEXT:    s_mov_b32 s50, s14
+; GFX1132-NEXT:    s_mov_b32 s51, s13
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
@@ -6179,25 +6179,25 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s43
-; GFX1132-NEXT:    s_mov_b32 s13, s42
+; GFX1132-NEXT:    s_mov_b32 s12, s51
+; GFX1132-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1132-NEXT:  .LBB9_3:
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -6206,14 +6206,14 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, s0, 0
 ; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, s1, v3
@@ -6222,16 +6222,16 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
 ; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_cvt_f64_u32_e32 v[1:2], s2
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v4, v0, v4
@@ -6244,11 +6244,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], v[41:42]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:8
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
@@ -6261,42 +6261,42 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX7LESS-DPP-NEXT:  .LBB9_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX9-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT:    s_mov_b32 s51, 0xe00000
+; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s0, 0
-; GFX9-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, s1, v3
-; GFX9-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
@@ -6304,21 +6304,21 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
 ; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX9-DPP-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
 ; GFX9-DPP-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX9-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s0
 ; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
@@ -6331,68 +6331,68 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX9-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v3, off, s[48:51], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v4, off, s[48:51], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_load_dword v3, off, s[64:67], 0
+; GFX9-DPP-NEXT:    buffer_load_dword v4, off, s[64:67], 0 offset:4
 ; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX9-DPP-NEXT:  .LBB9_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1064-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s10, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, s11, v3
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1064-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-DPP-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1064-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1064-DPP-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
@@ -6409,69 +6409,69 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1064-DPP-NEXT:  .LBB9_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-DPP-NEXT:    s_mov_b32 s9, exec_lo
-; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s0, s9
 ; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
 ; GFX1032-DPP-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1032-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1032-DPP-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
@@ -6488,37 +6488,37 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1032-DPP-NEXT:  .LBB9_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -6529,7 +6529,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], exec
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s10, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -6538,16 +6538,16 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
 ; GFX1164-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1164-DPP-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1164-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1164-DPP-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -6570,18 +6570,18 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -6589,8 +6589,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1164-DPP-NEXT:  .LBB9_3:
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -6601,24 +6601,24 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX1132-DPP-NEXT:    s_mov_b32 s8, exec_lo
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1132-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
 ; GFX1132-DPP-NEXT:    s_bcnt1_i32_b32 s0, s8
-; GFX1132-DPP-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1132-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s42, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s43, s13
+; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
+; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-DPP-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -6638,25 +6638,25 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1132-DPP-NEXT:  .LBB9_3:
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -6669,19 +6669,19 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX7LESS:       ; %bb.0:
 ; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
@@ -6692,15 +6692,15 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
@@ -6725,21 +6725,21 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[44:45], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s47, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s46, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s54, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[44:47], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:8
+; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
+; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
 ; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
@@ -6752,44 +6752,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX7LESS-NEXT:  .LBB10_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s50, -1
-; GFX9-NEXT:    s_mov_b32 s51, 0xe00000
-; GFX9-NEXT:    s_add_u32 s48, s48, s11
-; GFX9-NEXT:    s_addc_u32 s49, s49, 0
+; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s66, -1
+; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
+; GFX9-NEXT:    s_add_u32 s64, s64, s11
+; GFX9-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s43, s8
+; GFX9-NEXT:    s_mov_b32 s51, s8
 ; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s42, s9
+; GFX9-NEXT:    s_mov_b32 s50, s9
 ; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -6801,14 +6801,14 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -6833,11 +6833,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[44:45], s[36:37], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX9-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX9-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -6848,53 +6848,53 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX9-NEXT:  .LBB10_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s50, -1
-; GFX1064-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s66, -1
+; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-NEXT:    s_mov_b32 s43, s8
+; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s42, s9
+; GFX1064-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -6906,14 +6906,14 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
+; GFX1064-NEXT:    s_mov_b32 s12, s51
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -6938,11 +6938,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1064-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1064-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1064-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
@@ -6952,55 +6952,55 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s12, s51
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1064-NEXT:  .LBB10_5:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s66, -1
+; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-NEXT:    s_mov_b32 s43, s8
+; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s42, s9
+; GFX1032-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -7012,14 +7012,14 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
+; GFX1032-NEXT:    s_mov_b32 s12, s51
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -7038,16 +7038,16 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB10_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s46, 0
+; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1032-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1032-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1032-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
@@ -7057,37 +7057,37 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s12, s51
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1032-NEXT:  .LBB10_5:
 ; GFX1032-NEXT:    s_endpgm
@@ -7095,11 +7095,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s43, s8
+; GFX1164-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s42, s9
+; GFX1164-NEXT:    s_mov_b32 s50, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1164-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1164-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -7107,11 +7107,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1164-NEXT:    s_mov_b32 s33, s10
 ; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
@@ -7143,11 +7143,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1164-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1164-NEXT:    .p2align 6
 ; GFX1164-NEXT:  .LBB10_4: ; %atomicrmw.start
@@ -7164,18 +7164,18 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -7183,8 +7183,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1164-NEXT:  .LBB10_5:
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -7193,7 +7193,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
 ; GFX1132-NEXT:    s_getpc_b64 s[0:1]
@@ -7202,9 +7202,9 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s42, s14
-; GFX1132-NEXT:    s_mov_b32 s43, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b32 s50, s14
+; GFX1132-NEXT:    s_mov_b32 s51, s13
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
@@ -7233,17 +7233,17 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB10_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s46, 0
+; GFX1132-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1132-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1132-NEXT:    .p2align 6
 ; GFX1132-NEXT:  .LBB10_4: ; %atomicrmw.start
@@ -7258,25 +7258,25 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s43
-; GFX1132-NEXT:    s_mov_b32 s13, s42
+; GFX1132-NEXT:    s_mov_b32 s12, s51
+; GFX1132-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1132-NEXT:  .LBB10_5:
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -7285,22 +7285,22 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s47, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s46, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
@@ -7311,30 +7311,30 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v42, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v41, v1
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[0:1], off, s[44:47], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-DPP-NEXT:  .LBB10_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], v[40:41]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:8
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
@@ -7347,44 +7347,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB10_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s52, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s53, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s55, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s52, s52, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s53, s53, 0
+; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
 ; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -7396,14 +7396,14 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[52:53]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[54:55]
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
 ; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -7450,74 +7450,74 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s45, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s44, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[46:47], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[46:47]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
 ; GFX9-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], s[44:45]
+; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], s[52:53]
 ; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[52:53]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[52:55], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[52:55], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[52:55], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[52:55], 0 offset:8
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
+; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
+; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[54:55]
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s46
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s47
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[52:55], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[52:55], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
+; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
 ; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[48:49], vcc, s[48:49]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[48:49]
+; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX9-DPP-NEXT:  .LBB10_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -7529,14 +7529,14 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -7581,10 +7581,10 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1064-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
@@ -7594,55 +7594,55 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1064-DPP-NEXT:  .LBB10_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -7654,14 +7654,14 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -7696,14 +7696,14 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1032-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
@@ -7713,37 +7713,37 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1032-DPP-NEXT:  .LBB10_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -7751,11 +7751,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -7763,11 +7763,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
@@ -7825,10 +7825,10 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1164-DPP-NEXT:    .p2align 6
 ; GFX1164-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
@@ -7845,18 +7845,18 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -7864,8 +7864,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1164-DPP-NEXT:  .LBB10_3:
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -7874,7 +7874,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
 ; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
@@ -7883,9 +7883,9 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s42, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s43, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
+; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
@@ -7936,14 +7936,14 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1132-DPP-NEXT:    .p2align 6
 ; GFX1132-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
@@ -7958,25 +7958,25 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1132-DPP-NEXT:  .LBB10_3:
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -8922,12 +8922,12 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -8947,8 +8947,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[4:5], off, s[36:39], 0
@@ -10355,12 +10355,12 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -10380,8 +10380,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[4:5], off, s[36:39], 0
@@ -11270,12 +11270,12 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -11295,8 +11295,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[4:5], off, s[36:39], 0
@@ -11771,13 +11771,13 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX7LESS:       ; %bb.0:
 ; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-NEXT:    s_addc_u32 s49, s49, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
+; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v5, exec_lo, 0
 ; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s0, exec
 ; GFX7LESS-NEXT:    s_mov_b32 s1, 0x43300000
@@ -11791,15 +11791,15 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX7LESS-NEXT:  ; %bb.1:
 ; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX7LESS-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v3, v0, v1
@@ -11811,11 +11811,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:8
+; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
+; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
 ; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
@@ -11828,40 +11828,40 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX7LESS-NEXT:  .LBB16_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s50, -1
-; GFX9-NEXT:    s_mov_b32 s51, 0xe00000
-; GFX9-NEXT:    s_add_u32 s48, s48, s11
+; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s66, -1
+; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
+; GFX9-NEXT:    s_add_u32 s64, s64, s11
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-NEXT:    s_addc_u32 s49, s49, 0
-; GFX9-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-NEXT:    s_addc_u32 s65, s65, 0
+; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-NEXT:    s_bcnt1_i32_b64 s0, exec
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0xc3300000
 ; GFX9-NEXT:    s_mov_b32 s1, 0x43300000
@@ -11874,20 +11874,20 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b32 s42, s9
+; GFX9-NEXT:    s_mov_b32 s50, s9
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX9-NEXT:    s_mov_b32 s43, s8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX9-NEXT:    s_mov_b32 s51, s8
 ; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX9-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX9-NEXT:  .LBB16_2: ; %atomicrmw.start
@@ -11900,48 +11900,48 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX9-NEXT:  .LBB16_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s50, -1
-; GFX1064-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s48, s48, s11
-; GFX1064-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s66, -1
+; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-NEXT:    s_bcnt1_i32_b64 s0, exec
 ; GFX1064-NEXT:    s_mov_b32 s1, 0x43300000
 ; GFX1064-NEXT:    s_movk_i32 s32, 0x800
@@ -11953,19 +11953,19 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b32 s42, s9
-; GFX1064-NEXT:    s_mov_b32 s43, s8
+; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
@@ -11978,53 +11978,53 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s12, s51
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1064-NEXT:  .LBB16_3:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s48, s48, s11
-; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s66, -1
+; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
 ; GFX1032-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT:    s_mov_b32 s46, 0
+; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
@@ -12033,18 +12033,18 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b32 s42, s9
-; GFX1032-NEXT:    s_mov_b32 s43, s8
+; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
@@ -12057,44 +12057,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s12, s51
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1032-NEXT:  .LBB16_3:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1164:       ; %bb.0:
-; GFX1164-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_bcnt1_i32_b64 s0, exec
 ; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 0x43300000
@@ -12115,16 +12115,16 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b32 s42, s9
-; GFX1164-NEXT:    s_mov_b32 s43, s8
+; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
@@ -12145,18 +12145,18 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -12164,8 +12164,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1164-NEXT:  .LBB16_3:
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -12173,12 +12173,12 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ;
 ; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1132:       ; %bb.0:
-; GFX1132-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX1132-NEXT:    v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0
 ; GFX1132-NEXT:    v_mov_b32_e32 v0, 0x43300000
-; GFX1132-NEXT:    s_mov_b32 s46, 0
+; GFX1132-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-NEXT:    s_clause 0x1
 ; GFX1132-NEXT:    scratch_store_b32 off, v0, off offset:20
 ; GFX1132-NEXT:    scratch_store_b32 off, v1, off offset:16
@@ -12193,15 +12193,15 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s42, s14
-; GFX1132-NEXT:    s_mov_b32 s43, s13
+; GFX1132-NEXT:    s_mov_b32 s50, s14
+; GFX1132-NEXT:    s_mov_b32 s51, s13
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
@@ -12219,25 +12219,25 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s43
-; GFX1132-NEXT:    s_mov_b32 s13, s42
+; GFX1132-NEXT:    s_mov_b32 s12, s51
+; GFX1132-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1132-NEXT:  .LBB16_3:
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -12246,13 +12246,13 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v5, exec_lo, 0
 ; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s1, 0x43300000
@@ -12266,15 +12266,15 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v3, v0, v1
@@ -12286,11 +12286,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], v[41:42]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:8
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
@@ -12303,40 +12303,40 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX7LESS-DPP-NEXT:  .LBB16_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s51, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0xc3300000
 ; GFX9-DPP-NEXT:    s_mov_b32 s1, 0x43300000
@@ -12349,20 +12349,20 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX9-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
@@ -12375,48 +12375,48 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX9-DPP-NEXT:  .LBB16_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX1064-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
 ; GFX1064-DPP-NEXT:    s_mov_b32 s1, 0x43300000
 ; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
@@ -12428,19 +12428,19 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1064-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
@@ -12453,53 +12453,53 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1064-DPP-NEXT:  .LBB16_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
 ; GFX1032-DPP-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
@@ -12508,18 +12508,18 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
@@ -12532,44 +12532,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1032-DPP-NEXT:  .LBB16_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1164-DPP:       ; %bb.0:
-; GFX1164-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0x43300000
@@ -12590,16 +12590,16 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
@@ -12620,18 +12620,18 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -12639,8 +12639,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1164-DPP-NEXT:  .LBB16_3:
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -12648,12 +12648,12 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1132-DPP:       ; %bb.0:
-; GFX1132-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-DPP-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, 0x43300000
-; GFX1132-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-DPP-NEXT:    s_clause 0x1
 ; GFX1132-DPP-NEXT:    scratch_store_b32 off, v0, off offset:20
 ; GFX1132-DPP-NEXT:    scratch_store_b32 off, v1, off offset:16
@@ -12668,15 +12668,15 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s42, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s43, s13
+; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
+; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
@@ -12694,25 +12694,25 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1132-DPP-NEXT:  .LBB16_3:
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -12725,19 +12725,19 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX7LESS:       ; %bb.0:
 ; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
@@ -12748,15 +12748,15 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
@@ -12781,21 +12781,21 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[44:45], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s47, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s46, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s54, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[44:47], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:8
+; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
+; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
 ; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
@@ -12808,44 +12808,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX7LESS-NEXT:  .LBB17_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s50, -1
-; GFX9-NEXT:    s_mov_b32 s51, 0xe00000
-; GFX9-NEXT:    s_add_u32 s48, s48, s11
-; GFX9-NEXT:    s_addc_u32 s49, s49, 0
+; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s66, -1
+; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
+; GFX9-NEXT:    s_add_u32 s64, s64, s11
+; GFX9-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s43, s8
+; GFX9-NEXT:    s_mov_b32 s51, s8
 ; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s42, s9
+; GFX9-NEXT:    s_mov_b32 s50, s9
 ; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -12857,14 +12857,14 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -12889,11 +12889,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[44:45], s[36:37], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX9-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX9-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -12904,53 +12904,53 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX9-NEXT:  .LBB17_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s50, -1
-; GFX1064-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s66, -1
+; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-NEXT:    s_mov_b32 s43, s8
+; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s42, s9
+; GFX1064-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -12962,14 +12962,14 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
+; GFX1064-NEXT:    s_mov_b32 s12, s51
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -12994,11 +12994,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1064-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1064-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1064-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
@@ -13008,55 +13008,55 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s12, s51
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1064-NEXT:  .LBB17_5:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s66, -1
+; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-NEXT:    s_mov_b32 s43, s8
+; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s42, s9
+; GFX1032-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -13068,14 +13068,14 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
+; GFX1032-NEXT:    s_mov_b32 s12, s51
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -13094,16 +13094,16 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB17_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s46, 0
+; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1032-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1032-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1032-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
@@ -13113,37 +13113,37 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s12, s51
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1032-NEXT:  .LBB17_5:
 ; GFX1032-NEXT:    s_endpgm
@@ -13151,11 +13151,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s43, s8
+; GFX1164-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s42, s9
+; GFX1164-NEXT:    s_mov_b32 s50, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1164-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1164-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -13163,11 +13163,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1164-NEXT:    s_mov_b32 s33, s10
 ; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
@@ -13199,11 +13199,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1164-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1164-NEXT:    .p2align 6
 ; GFX1164-NEXT:  .LBB17_4: ; %atomicrmw.start
@@ -13220,18 +13220,18 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -13239,8 +13239,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1164-NEXT:  .LBB17_5:
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -13249,7 +13249,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
 ; GFX1132-NEXT:    s_getpc_b64 s[0:1]
@@ -13258,9 +13258,9 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s42, s14
-; GFX1132-NEXT:    s_mov_b32 s43, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b32 s50, s14
+; GFX1132-NEXT:    s_mov_b32 s51, s13
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
@@ -13289,17 +13289,17 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB17_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s46, 0
+; GFX1132-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1132-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1132-NEXT:    .p2align 6
 ; GFX1132-NEXT:  .LBB17_4: ; %atomicrmw.start
@@ -13314,25 +13314,25 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s43
-; GFX1132-NEXT:    s_mov_b32 s13, s42
+; GFX1132-NEXT:    s_mov_b32 s12, s51
+; GFX1132-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1132-NEXT:  .LBB17_5:
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -13341,22 +13341,22 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s47, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s46, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
@@ -13367,30 +13367,30 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v42, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v41, v1
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[0:1], off, s[44:47], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-DPP-NEXT:  .LBB17_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], v[40:41]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:8
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
@@ -13403,44 +13403,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB17_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s52, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s53, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s55, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s52, s52, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s53, s53, 0
+; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
 ; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -13452,14 +13452,14 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[52:53]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[54:55]
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
 ; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -13506,74 +13506,74 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s45, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s44, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[46:47], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[46:47]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
 ; GFX9-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], s[44:45]
+; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], s[52:53]
 ; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[52:53]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[52:55], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[52:55], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[52:55], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[52:55], 0 offset:8
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
+; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
+; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[54:55]
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s46
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s47
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[52:55], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[52:55], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
+; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
 ; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[48:49], vcc, s[48:49]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[48:49]
+; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX9-DPP-NEXT:  .LBB17_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -13585,14 +13585,14 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -13637,10 +13637,10 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1064-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
@@ -13650,55 +13650,55 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1064-DPP-NEXT:  .LBB17_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -13710,14 +13710,14 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -13752,14 +13752,14 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1032-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
@@ -13769,37 +13769,37 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1032-DPP-NEXT:  .LBB17_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -13807,11 +13807,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -13819,11 +13819,11 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
@@ -13881,10 +13881,10 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1164-DPP-NEXT:    .p2align 6
 ; GFX1164-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
@@ -13901,18 +13901,18 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -13920,8 +13920,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1164-DPP-NEXT:  .LBB17_3:
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -13930,7 +13930,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
 ; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
@@ -13939,9 +13939,9 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s42, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s43, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
+; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
@@ -13992,14 +13992,14 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1132-DPP-NEXT:    .p2align 6
 ; GFX1132-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
@@ -14014,25 +14014,25 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1132-DPP-NEXT:  .LBB17_3:
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
index a4410bb9ed2d0..e1ba4a2b0bf2a 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
@@ -635,12 +635,12 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
 ; GFX7LESS-DPP-LABEL: global_atomic_fmax_uni_address_div_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -660,8 +660,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[36:39], 0
@@ -1674,12 +1674,12 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
 ; GFX7LESS-DPP-LABEL: global_atomic_fmax_uni_address_div_value_one_as_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -1699,8 +1699,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[36:39], 0
@@ -2713,12 +2713,12 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
 ; GFX7LESS-DPP-LABEL: global_atomic_fmax_uni_address_div_value_default_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -2738,8 +2738,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[36:39], 0
@@ -3135,13 +3135,13 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
 ; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-NEXT:    s_addc_u32 s49, s49, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
+; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
 ; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
@@ -3149,15 +3149,15 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX7LESS-NEXT:  ; %bb.1:
 ; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX7LESS-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v3, v0, v1
@@ -3169,8 +3169,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], 4.0
@@ -3178,8 +3178,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
@@ -3188,59 +3188,59 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX7LESS-NEXT:  .LBB6_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s50, -1
-; GFX9-NEXT:    s_mov_b32 s51, 0xe00000
+; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s66, -1
+; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-NEXT:    s_add_u32 s48, s48, s11
+; GFX9-NEXT:    s_add_u32 s64, s64, s11
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-NEXT:    s_addc_u32 s49, s49, 0
-; GFX9-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-NEXT:    s_addc_u32 s65, s65, 0
+; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX9-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b32 s42, s9
+; GFX9-NEXT:    s_mov_b32 s50, s9
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX9-NEXT:    s_mov_b32 s43, s8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX9-NEXT:    s_mov_b32 s51, s8
 ; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX9-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX9-NEXT:  .LBB6_2: ; %atomicrmw.start
@@ -3253,36 +3253,36 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX9-NEXT:  .LBB6_3:
 ; GFX9-NEXT:    s_endpgm
@@ -3290,32 +3290,32 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1064-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064:       ; %bb.0:
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s50, -1
-; GFX1064-NEXT:    s_mov_b32 s51, 0x31e16000
+; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s66, -1
+; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-NEXT:    s_add_u32 s48, s48, s11
-; GFX1064-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b32 s42, s9
-; GFX1064-NEXT:    s_mov_b32 s43, s8
+; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
@@ -3328,38 +3328,38 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s12, s51
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1064-NEXT:  .LBB6_3:
 ; GFX1064-NEXT:    s_endpgm
@@ -3367,31 +3367,31 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
+; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s66, -1
+; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_add_u32 s48, s48, s11
-; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1032-NEXT:    s_mov_b32 s46, 0
+; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b32 s42, s9
-; GFX1032-NEXT:    s_mov_b32 s43, s8
+; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
@@ -3404,38 +3404,38 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s12, s51
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1032-NEXT:  .LBB6_3:
 ; GFX1032-NEXT:    s_endpgm
@@ -3444,7 +3444,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -3452,16 +3452,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b32 s42, s9
-; GFX1164-NEXT:    s_mov_b32 s43, s8
+; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
@@ -3483,18 +3483,18 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
 ; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -3502,8 +3502,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1164-NEXT:  .LBB6_3:
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -3513,22 +3513,22 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1132-NEXT:    s_mov_b32 s46, 0
+; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s42, s14
-; GFX1132-NEXT:    s_mov_b32 s43, s13
+; GFX1132-NEXT:    s_mov_b32 s50, s14
+; GFX1132-NEXT:    s_mov_b32 s51, s13
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
@@ -3547,26 +3547,26 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s43
-; GFX1132-NEXT:    s_mov_b32 s13, s42
+; GFX1132-NEXT:    s_mov_b32 s12, s51
+; GFX1132-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX1132-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
 ; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1132-NEXT:  .LBB6_3:
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -3575,13 +3575,13 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX7LESS-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
 ; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
@@ -3589,15 +3589,15 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v3, v0, v1
@@ -3609,8 +3609,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], 4.0
@@ -3618,8 +3618,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
@@ -3628,59 +3628,59 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX7LESS-DPP-NEXT:  .LBB6_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s51, 0xe00000
+; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX9-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
@@ -3693,36 +3693,36 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX9-DPP-NEXT:  .LBB6_3:
 ; GFX9-DPP-NEXT:    s_endpgm
@@ -3730,32 +3730,32 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, 0x31e16000
+; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
 ; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX1064-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1064-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
@@ -3768,38 +3768,38 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1064-DPP-NEXT:  .LBB6_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
@@ -3807,31 +3807,31 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
+; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
@@ -3844,38 +3844,38 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1032-DPP-NEXT:  .LBB6_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -3884,7 +3884,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -3892,16 +3892,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
@@ -3923,18 +3923,18 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -3942,8 +3942,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1164-DPP-NEXT:  .LBB6_3:
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -3953,22 +3953,22 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1132-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s42, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s43, s13
+; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
+; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
@@ -3987,26 +3987,26 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1132-DPP-NEXT:  .LBB6_3:
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -4019,19 +4019,19 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
 ; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
@@ -4042,15 +4042,15 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
@@ -4077,19 +4077,19 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[44:45], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s47, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s46, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s54, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[44:47], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
 ; GFX7LESS-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[41:42]
@@ -4097,8 +4097,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
@@ -4107,43 +4107,43 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX7LESS-NEXT:  .LBB7_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s50, -1
-; GFX9-NEXT:    s_mov_b32 s51, 0xe00000
-; GFX9-NEXT:    s_add_u32 s48, s48, s11
-; GFX9-NEXT:    s_addc_u32 s49, s49, 0
+; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s66, -1
+; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
+; GFX9-NEXT:    s_add_u32 s64, s64, s11
+; GFX9-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s43, s8
+; GFX9-NEXT:    s_mov_b32 s51, s8
 ; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s42, s9
+; GFX9-NEXT:    s_mov_b32 s50, s9
 ; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -4155,14 +4155,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -4189,12 +4189,12 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[44:45], s[36:37], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX9-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[4:5], v0, s[44:45]
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
 ; GFX9-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -4205,54 +4205,54 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v5, off, s[48:51], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v4, off, s[48:51], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
+; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX9-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v4, off, s[48:51], 0
-; GFX9-NEXT:    buffer_load_dword v5, off, s[48:51], 0 offset:4
+; GFX9-NEXT:    buffer_load_dword v4, off, s[64:67], 0
+; GFX9-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX9-NEXT:  .LBB7_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s50, -1
-; GFX1064-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s66, -1
+; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-NEXT:    s_mov_b32 s43, s8
+; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s42, s9
+; GFX1064-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -4264,14 +4264,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
+; GFX1064-NEXT:    s_mov_b32 s12, s51
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -4298,12 +4298,12 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1064-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1064-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[4:5], v0, s[44:45]
+; GFX1064-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
 ; GFX1064-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
@@ -4313,56 +4313,56 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v5, off, s[48:51], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[48:51], 0
+; GFX1064-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
+; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s44
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s12, s51
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v4, off, s[48:51], 0
-; GFX1064-NEXT:    buffer_load_dword v5, off, s[48:51], 0 offset:4
+; GFX1064-NEXT:    buffer_load_dword v4, off, s[64:67], 0
+; GFX1064-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
 ; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX1064-NEXT:  .LBB7_5:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s66, -1
+; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-NEXT:    s_mov_b32 s43, s8
+; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s42, s9
+; GFX1032-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -4374,14 +4374,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
+; GFX1032-NEXT:    s_mov_b32 s12, s51
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -4402,17 +4402,17 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s46, 0
+; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1032-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[4:5], v0, s[44:45]
+; GFX1032-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
 ; GFX1032-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
@@ -4422,38 +4422,38 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v5, off, s[48:51], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[48:51], 0
+; GFX1032-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
+; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0
 ; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s44
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s12, s51
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v4, off, s[48:51], 0
-; GFX1032-NEXT:    buffer_load_dword v5, off, s[48:51], 0 offset:4
+; GFX1032-NEXT:    buffer_load_dword v4, off, s[64:67], 0
+; GFX1032-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
 ; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX1032-NEXT:  .LBB7_5:
 ; GFX1032-NEXT:    s_endpgm
@@ -4461,11 +4461,11 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1164-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s43, s8
+; GFX1164-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s42, s9
+; GFX1164-NEXT:    s_mov_b32 s50, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1164-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1164-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -4473,11 +4473,11 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1164-NEXT:    s_mov_b32 s33, s10
 ; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
@@ -4511,12 +4511,12 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1164-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[4:5], v0, s[44:45]
+; GFX1164-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1164-NEXT:    .p2align 6
 ; GFX1164-NEXT:  .LBB7_4: ; %atomicrmw.start
@@ -4530,15 +4530,15 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s44
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
 ; GFX1164-NEXT:    scratch_store_b64 off, v[4:5], off
@@ -4553,8 +4553,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX1164-NEXT:  .LBB7_5:
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -4563,7 +4563,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1132-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
 ; GFX1132-NEXT:    s_getpc_b64 s[0:1]
@@ -4572,9 +4572,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s42, s14
-; GFX1132-NEXT:    s_mov_b32 s43, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b32 s50, s14
+; GFX1132-NEXT:    s_mov_b32 s51, s13
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
@@ -4606,19 +4606,19 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s46, 0
+; GFX1132-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX1132-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[4:5], v0, s[44:45]
+; GFX1132-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1132-NEXT:    .p2align 6
 ; GFX1132-NEXT:  .LBB7_4: ; %atomicrmw.start
@@ -4632,16 +4632,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX1132-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1132-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s43
-; GFX1132-NEXT:    s_mov_b32 s13, s42
+; GFX1132-NEXT:    s_mov_b32 s12, s51
+; GFX1132-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1132-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1132-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
 ; GFX1132-NEXT:    scratch_store_b64 off, v[4:5], off
 ; GFX1132-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 8
@@ -4653,8 +4653,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX1132-NEXT:  .LBB7_5:
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -4663,22 +4663,22 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s47, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s46, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
@@ -4689,34 +4689,34 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[44:47], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[52:55], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[41:42], v[0:1], v[0:1]
 ; GFX7LESS-DPP-NEXT:  .LBB7_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
@@ -4725,43 +4725,43 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v3, off, s[48:51], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v3, off, s[64:67], 0 offset:4
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB7_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s52, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s53, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s55, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s52, s52, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s53, s53, 0
+; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
 ; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -4773,14 +4773,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[52:53]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[54:55]
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
 ; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -4834,20 +4834,20 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s45, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s44, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[46:47], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[46:47]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
 ; GFX9-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], s[44:45], s[44:45]
+; GFX9-DPP-NEXT:    v_max_f64 v[3:4], s[52:53], s[52:53]
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
 ; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
@@ -4856,54 +4856,54 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[52:53]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[52:55], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[52:55], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
+; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[52:55], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[52:55], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[54:55]
+; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
+; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s46
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s47
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[52:55], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[52:55], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
+; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
 ; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[48:49], vcc, s[48:49]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[48:49]
+; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX9-DPP-NEXT:  .LBB7_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -4915,14 +4915,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -4975,10 +4975,10 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1064-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[41:42], v[41:42]
@@ -4989,56 +4989,56 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX1064-DPP-NEXT:  .LBB7_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -5050,14 +5050,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -5098,15 +5098,15 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1032-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
@@ -5116,38 +5116,38 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[41:42]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX1032-DPP-NEXT:  .LBB7_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -5155,11 +5155,11 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -5167,11 +5167,11 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
@@ -5239,11 +5239,11 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1164-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1164-DPP-NEXT:    .p2align 6
 ; GFX1164-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
@@ -5261,18 +5261,18 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[41:42]
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -5280,8 +5280,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX1164-DPP-NEXT:  .LBB7_3:
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -5290,7 +5290,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
 ; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
@@ -5299,9 +5299,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s42, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s43, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
+; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
@@ -5356,16 +5356,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v8
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1132-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1132-DPP-NEXT:    .p2align 6
 ; GFX1132-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
@@ -5381,26 +5381,26 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[41:42]
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX1132-DPP-NEXT:  .LBB7_3:
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -6162,12 +6162,12 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
 ; GFX7LESS-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -6187,8 +6187,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[36:39], 0
@@ -6683,13 +6683,13 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
 ; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-NEXT:    s_addc_u32 s49, s49, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
+; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
 ; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
@@ -6697,15 +6697,15 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX7LESS-NEXT:  ; %bb.1:
 ; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX7LESS-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v3, v0, v1
@@ -6717,8 +6717,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], 4.0
@@ -6726,8 +6726,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
@@ -6736,59 +6736,59 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX7LESS-NEXT:  .LBB10_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s50, -1
-; GFX9-NEXT:    s_mov_b32 s51, 0xe00000
+; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s66, -1
+; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-NEXT:    s_add_u32 s48, s48, s11
+; GFX9-NEXT:    s_add_u32 s64, s64, s11
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-NEXT:    s_addc_u32 s49, s49, 0
-; GFX9-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-NEXT:    s_addc_u32 s65, s65, 0
+; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX9-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b32 s42, s9
+; GFX9-NEXT:    s_mov_b32 s50, s9
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX9-NEXT:    s_mov_b32 s43, s8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX9-NEXT:    s_mov_b32 s51, s8
 ; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX9-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX9-NEXT:  .LBB10_2: ; %atomicrmw.start
@@ -6801,36 +6801,36 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX9-NEXT:  .LBB10_3:
 ; GFX9-NEXT:    s_endpgm
@@ -6838,32 +6838,32 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1064-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1064:       ; %bb.0:
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s50, -1
-; GFX1064-NEXT:    s_mov_b32 s51, 0x31e16000
+; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s66, -1
+; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-NEXT:    s_add_u32 s48, s48, s11
-; GFX1064-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b32 s42, s9
-; GFX1064-NEXT:    s_mov_b32 s43, s8
+; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
@@ -6876,38 +6876,38 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s12, s51
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1064-NEXT:  .LBB10_3:
 ; GFX1064-NEXT:    s_endpgm
@@ -6915,31 +6915,31 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1032:       ; %bb.0:
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
+; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s66, -1
+; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_add_u32 s48, s48, s11
-; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1032-NEXT:    s_mov_b32 s46, 0
+; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b32 s42, s9
-; GFX1032-NEXT:    s_mov_b32 s43, s8
+; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
@@ -6952,38 +6952,38 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s12, s51
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1032-NEXT:  .LBB10_3:
 ; GFX1032-NEXT:    s_endpgm
@@ -6992,7 +6992,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -7000,16 +7000,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b32 s42, s9
-; GFX1164-NEXT:    s_mov_b32 s43, s8
+; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
@@ -7031,18 +7031,18 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
 ; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -7050,8 +7050,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1164-NEXT:  .LBB10_3:
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -7061,22 +7061,22 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1132-NEXT:    s_mov_b32 s46, 0
+; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s42, s14
-; GFX1132-NEXT:    s_mov_b32 s43, s13
+; GFX1132-NEXT:    s_mov_b32 s50, s14
+; GFX1132-NEXT:    s_mov_b32 s51, s13
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
@@ -7095,26 +7095,26 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s43
-; GFX1132-NEXT:    s_mov_b32 s13, s42
+; GFX1132-NEXT:    s_mov_b32 s12, s51
+; GFX1132-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX1132-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
 ; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1132-NEXT:  .LBB10_3:
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -7123,13 +7123,13 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX7LESS-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
 ; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
@@ -7137,15 +7137,15 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v3, v0, v1
@@ -7157,8 +7157,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], 4.0
@@ -7166,8 +7166,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
@@ -7176,59 +7176,59 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX7LESS-DPP-NEXT:  .LBB10_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s51, 0xe00000
+; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX9-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
@@ -7241,36 +7241,36 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX9-DPP-NEXT:  .LBB10_3:
 ; GFX9-DPP-NEXT:    s_endpgm
@@ -7278,32 +7278,32 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, 0x31e16000
+; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
 ; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX1064-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1064-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
@@ -7316,38 +7316,38 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1064-DPP-NEXT:  .LBB10_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
@@ -7355,31 +7355,31 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
+; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
@@ -7392,38 +7392,38 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1032-DPP-NEXT:  .LBB10_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -7432,7 +7432,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -7440,16 +7440,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
@@ -7471,18 +7471,18 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -7490,8 +7490,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1164-DPP-NEXT:  .LBB10_3:
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -7501,22 +7501,22 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1132-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s42, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s43, s13
+; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
+; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
@@ -7535,26 +7535,26 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1132-DPP-NEXT:  .LBB10_3:
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -7567,19 +7567,19 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
 ; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
@@ -7590,15 +7590,15 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
@@ -7625,19 +7625,19 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[44:45], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s47, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s46, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s54, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[44:47], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
 ; GFX7LESS-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[41:42]
@@ -7645,8 +7645,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
@@ -7655,43 +7655,43 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX7LESS-NEXT:  .LBB11_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s50, -1
-; GFX9-NEXT:    s_mov_b32 s51, 0xe00000
-; GFX9-NEXT:    s_add_u32 s48, s48, s11
-; GFX9-NEXT:    s_addc_u32 s49, s49, 0
+; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s66, -1
+; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
+; GFX9-NEXT:    s_add_u32 s64, s64, s11
+; GFX9-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s43, s8
+; GFX9-NEXT:    s_mov_b32 s51, s8
 ; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s42, s9
+; GFX9-NEXT:    s_mov_b32 s50, s9
 ; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -7703,14 +7703,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -7737,12 +7737,12 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[44:45], s[36:37], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX9-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[4:5], v0, s[44:45]
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
 ; GFX9-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -7753,54 +7753,54 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v5, off, s[48:51], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v4, off, s[48:51], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
+; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX9-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v4, off, s[48:51], 0
-; GFX9-NEXT:    buffer_load_dword v5, off, s[48:51], 0 offset:4
+; GFX9-NEXT:    buffer_load_dword v4, off, s[64:67], 0
+; GFX9-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX9-NEXT:  .LBB11_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s50, -1
-; GFX1064-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s66, -1
+; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-NEXT:    s_mov_b32 s43, s8
+; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s42, s9
+; GFX1064-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -7812,14 +7812,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
+; GFX1064-NEXT:    s_mov_b32 s12, s51
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -7846,12 +7846,12 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1064-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1064-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[4:5], v0, s[44:45]
+; GFX1064-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
 ; GFX1064-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
@@ -7861,56 +7861,56 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v5, off, s[48:51], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[48:51], 0
+; GFX1064-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
+; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s44
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s12, s51
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v4, off, s[48:51], 0
-; GFX1064-NEXT:    buffer_load_dword v5, off, s[48:51], 0 offset:4
+; GFX1064-NEXT:    buffer_load_dword v4, off, s[64:67], 0
+; GFX1064-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
 ; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX1064-NEXT:  .LBB11_5:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s66, -1
+; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-NEXT:    s_mov_b32 s43, s8
+; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s42, s9
+; GFX1032-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -7922,14 +7922,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
+; GFX1032-NEXT:    s_mov_b32 s12, s51
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -7950,17 +7950,17 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s46, 0
+; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1032-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[4:5], v0, s[44:45]
+; GFX1032-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
 ; GFX1032-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
@@ -7970,38 +7970,38 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v5, off, s[48:51], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[48:51], 0
+; GFX1032-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
+; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0
 ; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s44
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s12, s51
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v4, off, s[48:51], 0
-; GFX1032-NEXT:    buffer_load_dword v5, off, s[48:51], 0 offset:4
+; GFX1032-NEXT:    buffer_load_dword v4, off, s[64:67], 0
+; GFX1032-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
 ; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX1032-NEXT:  .LBB11_5:
 ; GFX1032-NEXT:    s_endpgm
@@ -8009,11 +8009,11 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1164-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s43, s8
+; GFX1164-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s42, s9
+; GFX1164-NEXT:    s_mov_b32 s50, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1164-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1164-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -8021,11 +8021,11 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1164-NEXT:    s_mov_b32 s33, s10
 ; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
@@ -8059,12 +8059,12 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1164-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[4:5], v0, s[44:45]
+; GFX1164-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1164-NEXT:    .p2align 6
 ; GFX1164-NEXT:  .LBB11_4: ; %atomicrmw.start
@@ -8078,15 +8078,15 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s44
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
 ; GFX1164-NEXT:    scratch_store_b64 off, v[4:5], off
@@ -8101,8 +8101,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX1164-NEXT:  .LBB11_5:
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -8111,7 +8111,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1132-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
 ; GFX1132-NEXT:    s_getpc_b64 s[0:1]
@@ -8120,9 +8120,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s42, s14
-; GFX1132-NEXT:    s_mov_b32 s43, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b32 s50, s14
+; GFX1132-NEXT:    s_mov_b32 s51, s13
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
@@ -8154,19 +8154,19 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s46, 0
+; GFX1132-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX1132-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[4:5], v0, s[44:45]
+; GFX1132-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1132-NEXT:    .p2align 6
 ; GFX1132-NEXT:  .LBB11_4: ; %atomicrmw.start
@@ -8180,16 +8180,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX1132-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1132-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s43
-; GFX1132-NEXT:    s_mov_b32 s13, s42
+; GFX1132-NEXT:    s_mov_b32 s12, s51
+; GFX1132-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1132-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1132-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
 ; GFX1132-NEXT:    scratch_store_b64 off, v[4:5], off
 ; GFX1132-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 8
@@ -8201,8 +8201,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX1132-NEXT:  .LBB11_5:
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -8211,22 +8211,22 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX7LESS-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s47, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s46, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
@@ -8237,34 +8237,34 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[44:47], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[52:55], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[41:42], v[0:1], v[0:1]
 ; GFX7LESS-DPP-NEXT:  .LBB11_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
@@ -8273,43 +8273,43 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v3, off, s[48:51], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v3, off, s[64:67], 0 offset:4
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB11_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s52, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s53, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s55, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s52, s52, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s53, s53, 0
+; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
 ; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -8321,14 +8321,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[52:53]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[54:55]
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
 ; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -8382,20 +8382,20 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s45, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s44, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[46:47], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[46:47]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
 ; GFX9-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], s[44:45], s[44:45]
+; GFX9-DPP-NEXT:    v_max_f64 v[3:4], s[52:53], s[52:53]
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
 ; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
@@ -8404,54 +8404,54 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[52:53]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[52:55], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[52:55], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
+; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[52:55], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[52:55], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[54:55]
+; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
+; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s46
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s47
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[52:55], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[52:55], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
+; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
 ; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[48:49], vcc, s[48:49]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[48:49]
+; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX9-DPP-NEXT:  .LBB11_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -8463,14 +8463,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -8523,10 +8523,10 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1064-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[41:42], v[41:42]
@@ -8537,56 +8537,56 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX1064-DPP-NEXT:  .LBB11_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -8598,14 +8598,14 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -8646,15 +8646,15 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1032-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
@@ -8664,38 +8664,38 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[41:42]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX1032-DPP-NEXT:  .LBB11_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -8703,11 +8703,11 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -8715,11 +8715,11 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
@@ -8787,11 +8787,11 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1164-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1164-DPP-NEXT:    .p2align 6
 ; GFX1164-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
@@ -8809,18 +8809,18 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[41:42]
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -8828,8 +8828,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX1164-DPP-NEXT:  .LBB11_3:
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -8838,7 +8838,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
 ; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
@@ -8847,9 +8847,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s42, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s43, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
+; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
@@ -8904,16 +8904,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v8
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1132-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1132-DPP-NEXT:    .p2align 6
 ; GFX1132-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
@@ -8929,26 +8929,26 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[41:42]
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX1132-DPP-NEXT:  .LBB11_3:
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
index 68d7dcc60506c..6b1d5253e178f 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
@@ -635,12 +635,12 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
 ; GFX7LESS-DPP-LABEL: global_atomic_fmin_uni_address_div_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -660,8 +660,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[36:39], 0
@@ -1674,12 +1674,12 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
 ; GFX7LESS-DPP-LABEL: global_atomic_fmin_uni_address_div_value_one_as_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -1699,8 +1699,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[36:39], 0
@@ -2713,12 +2713,12 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
 ; GFX7LESS-DPP-LABEL: global_atomic_fmin_uni_address_div_value_default_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -2738,8 +2738,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[36:39], 0
@@ -3135,13 +3135,13 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
 ; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-NEXT:    s_addc_u32 s49, s49, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
+; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
 ; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
@@ -3149,15 +3149,15 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX7LESS-NEXT:  ; %bb.1:
 ; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX7LESS-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v3, v0, v1
@@ -3169,8 +3169,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[2:3], 4.0
@@ -3178,8 +3178,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
@@ -3188,59 +3188,59 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX7LESS-NEXT:  .LBB6_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s50, -1
-; GFX9-NEXT:    s_mov_b32 s51, 0xe00000
+; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s66, -1
+; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-NEXT:    s_add_u32 s48, s48, s11
+; GFX9-NEXT:    s_add_u32 s64, s64, s11
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-NEXT:    s_addc_u32 s49, s49, 0
-; GFX9-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-NEXT:    s_addc_u32 s65, s65, 0
+; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX9-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b32 s42, s9
+; GFX9-NEXT:    s_mov_b32 s50, s9
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX9-NEXT:    s_mov_b32 s43, s8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX9-NEXT:    s_mov_b32 s51, s8
 ; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX9-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX9-NEXT:  .LBB6_2: ; %atomicrmw.start
@@ -3253,36 +3253,36 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX9-NEXT:  .LBB6_3:
 ; GFX9-NEXT:    s_endpgm
@@ -3290,32 +3290,32 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1064-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064:       ; %bb.0:
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s50, -1
-; GFX1064-NEXT:    s_mov_b32 s51, 0x31e16000
+; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s66, -1
+; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-NEXT:    s_add_u32 s48, s48, s11
-; GFX1064-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b32 s42, s9
-; GFX1064-NEXT:    s_mov_b32 s43, s8
+; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
@@ -3328,38 +3328,38 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s12, s51
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1064-NEXT:  .LBB6_3:
 ; GFX1064-NEXT:    s_endpgm
@@ -3367,31 +3367,31 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
+; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s66, -1
+; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_add_u32 s48, s48, s11
-; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1032-NEXT:    s_mov_b32 s46, 0
+; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b32 s42, s9
-; GFX1032-NEXT:    s_mov_b32 s43, s8
+; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
@@ -3404,38 +3404,38 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s12, s51
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1032-NEXT:  .LBB6_3:
 ; GFX1032-NEXT:    s_endpgm
@@ -3444,7 +3444,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -3452,16 +3452,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b32 s42, s9
-; GFX1164-NEXT:    s_mov_b32 s43, s8
+; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
@@ -3483,18 +3483,18 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
 ; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -3502,8 +3502,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1164-NEXT:  .LBB6_3:
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -3513,22 +3513,22 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1132-NEXT:    s_mov_b32 s46, 0
+; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s42, s14
-; GFX1132-NEXT:    s_mov_b32 s43, s13
+; GFX1132-NEXT:    s_mov_b32 s50, s14
+; GFX1132-NEXT:    s_mov_b32 s51, s13
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
@@ -3547,26 +3547,26 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s43
-; GFX1132-NEXT:    s_mov_b32 s13, s42
+; GFX1132-NEXT:    s_mov_b32 s12, s51
+; GFX1132-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX1132-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
 ; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1132-NEXT:  .LBB6_3:
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -3575,13 +3575,13 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX7LESS-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
 ; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
@@ -3589,15 +3589,15 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v3, v0, v1
@@ -3609,8 +3609,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[2:3], 4.0
@@ -3618,8 +3618,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
@@ -3628,59 +3628,59 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX7LESS-DPP-NEXT:  .LBB6_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s51, 0xe00000
+; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX9-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
@@ -3693,36 +3693,36 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX9-DPP-NEXT:  .LBB6_3:
 ; GFX9-DPP-NEXT:    s_endpgm
@@ -3730,32 +3730,32 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, 0x31e16000
+; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
 ; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX1064-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1064-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
@@ -3768,38 +3768,38 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1064-DPP-NEXT:  .LBB6_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
@@ -3807,31 +3807,31 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
+; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
@@ -3844,38 +3844,38 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1032-DPP-NEXT:  .LBB6_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -3884,7 +3884,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -3892,16 +3892,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
@@ -3923,18 +3923,18 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -3942,8 +3942,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1164-DPP-NEXT:  .LBB6_3:
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -3953,22 +3953,22 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1132-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s42, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s43, s13
+; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
+; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
@@ -3987,26 +3987,26 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX1132-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1132-DPP-NEXT:  .LBB6_3:
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -4019,19 +4019,19 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
 ; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
@@ -4042,15 +4042,15 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
@@ -4077,19 +4077,19 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[44:45], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s47, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s46, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s54, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[44:47], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
 ; GFX7LESS-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[2:3], v[41:42]
@@ -4097,8 +4097,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
@@ -4107,43 +4107,43 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX7LESS-NEXT:  .LBB7_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s50, -1
-; GFX9-NEXT:    s_mov_b32 s51, 0xe00000
-; GFX9-NEXT:    s_add_u32 s48, s48, s11
-; GFX9-NEXT:    s_addc_u32 s49, s49, 0
+; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s66, -1
+; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
+; GFX9-NEXT:    s_add_u32 s64, s64, s11
+; GFX9-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s43, s8
+; GFX9-NEXT:    s_mov_b32 s51, s8
 ; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s42, s9
+; GFX9-NEXT:    s_mov_b32 s50, s9
 ; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -4155,14 +4155,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -4189,12 +4189,12 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[44:45], s[36:37], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX9-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[4:5], v0, s[44:45]
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
 ; GFX9-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -4205,54 +4205,54 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v5, off, s[48:51], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v4, off, s[48:51], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
+; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX9-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v4, off, s[48:51], 0
-; GFX9-NEXT:    buffer_load_dword v5, off, s[48:51], 0 offset:4
+; GFX9-NEXT:    buffer_load_dword v4, off, s[64:67], 0
+; GFX9-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX9-NEXT:  .LBB7_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s50, -1
-; GFX1064-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s66, -1
+; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-NEXT:    s_mov_b32 s43, s8
+; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s42, s9
+; GFX1064-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -4264,14 +4264,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
+; GFX1064-NEXT:    s_mov_b32 s12, s51
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -4298,12 +4298,12 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1064-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1064-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[4:5], v0, s[44:45]
+; GFX1064-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
 ; GFX1064-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
@@ -4313,56 +4313,56 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v5, off, s[48:51], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[48:51], 0
+; GFX1064-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
+; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s44
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s12, s51
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v4, off, s[48:51], 0
-; GFX1064-NEXT:    buffer_load_dword v5, off, s[48:51], 0 offset:4
+; GFX1064-NEXT:    buffer_load_dword v4, off, s[64:67], 0
+; GFX1064-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
 ; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX1064-NEXT:  .LBB7_5:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s66, -1
+; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-NEXT:    s_mov_b32 s43, s8
+; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s42, s9
+; GFX1032-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -4374,14 +4374,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
+; GFX1032-NEXT:    s_mov_b32 s12, s51
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -4402,17 +4402,17 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s46, 0
+; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1032-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[4:5], v0, s[44:45]
+; GFX1032-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
 ; GFX1032-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
@@ -4422,38 +4422,38 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v5, off, s[48:51], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[48:51], 0
+; GFX1032-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
+; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0
 ; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s44
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s12, s51
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v4, off, s[48:51], 0
-; GFX1032-NEXT:    buffer_load_dword v5, off, s[48:51], 0 offset:4
+; GFX1032-NEXT:    buffer_load_dword v4, off, s[64:67], 0
+; GFX1032-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
 ; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX1032-NEXT:  .LBB7_5:
 ; GFX1032-NEXT:    s_endpgm
@@ -4461,11 +4461,11 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1164-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s43, s8
+; GFX1164-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s42, s9
+; GFX1164-NEXT:    s_mov_b32 s50, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1164-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1164-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -4473,11 +4473,11 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1164-NEXT:    s_mov_b32 s33, s10
 ; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
@@ -4511,12 +4511,12 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1164-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[4:5], v0, s[44:45]
+; GFX1164-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1164-NEXT:    .p2align 6
 ; GFX1164-NEXT:  .LBB7_4: ; %atomicrmw.start
@@ -4530,15 +4530,15 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s44
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
 ; GFX1164-NEXT:    scratch_store_b64 off, v[4:5], off
@@ -4553,8 +4553,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX1164-NEXT:  .LBB7_5:
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -4563,7 +4563,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1132-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
 ; GFX1132-NEXT:    s_getpc_b64 s[0:1]
@@ -4572,9 +4572,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s42, s14
-; GFX1132-NEXT:    s_mov_b32 s43, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b32 s50, s14
+; GFX1132-NEXT:    s_mov_b32 s51, s13
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
@@ -4606,19 +4606,19 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s46, 0
+; GFX1132-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX1132-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[4:5], v0, s[44:45]
+; GFX1132-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1132-NEXT:    .p2align 6
 ; GFX1132-NEXT:  .LBB7_4: ; %atomicrmw.start
@@ -4632,16 +4632,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX1132-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1132-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s43
-; GFX1132-NEXT:    s_mov_b32 s13, s42
+; GFX1132-NEXT:    s_mov_b32 s12, s51
+; GFX1132-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1132-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1132-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
 ; GFX1132-NEXT:    scratch_store_b64 off, v[4:5], off
 ; GFX1132-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 8
@@ -4653,8 +4653,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX1132-NEXT:  .LBB7_5:
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -4663,22 +4663,22 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s47, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s46, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
@@ -4689,34 +4689,34 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[44:47], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[52:55], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[41:42], v[0:1], v[0:1]
 ; GFX7LESS-DPP-NEXT:  .LBB7_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
@@ -4725,43 +4725,43 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v3, off, s[48:51], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v3, off, s[64:67], 0 offset:4
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB7_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s52, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s53, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s55, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s52, s52, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s53, s53, 0
+; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
 ; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -4773,14 +4773,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[52:53]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[54:55]
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
 ; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -4834,20 +4834,20 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s45, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s44, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[46:47], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[46:47]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
 ; GFX9-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], s[44:45], s[44:45]
+; GFX9-DPP-NEXT:    v_max_f64 v[3:4], s[52:53], s[52:53]
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
 ; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
@@ -4856,54 +4856,54 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[52:53]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[52:55], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[52:55], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
+; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[52:55], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[52:55], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[54:55]
+; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
+; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s46
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s47
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[52:55], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[52:55], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
+; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
 ; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[48:49], vcc, s[48:49]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[48:49]
+; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX9-DPP-NEXT:  .LBB7_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -4915,14 +4915,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -4975,10 +4975,10 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1064-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[41:42], v[41:42]
@@ -4989,56 +4989,56 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX1064-DPP-NEXT:  .LBB7_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -5050,14 +5050,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -5098,15 +5098,15 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1032-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
@@ -5116,38 +5116,38 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[41:42]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX1032-DPP-NEXT:  .LBB7_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -5155,11 +5155,11 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -5167,11 +5167,11 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
@@ -5239,11 +5239,11 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1164-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1164-DPP-NEXT:    .p2align 6
 ; GFX1164-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
@@ -5261,18 +5261,18 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[41:42]
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -5280,8 +5280,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX1164-DPP-NEXT:  .LBB7_3:
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -5290,7 +5290,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
 ; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
@@ -5299,9 +5299,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s42, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s43, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
+; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
@@ -5356,16 +5356,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v8
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1132-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1132-DPP-NEXT:    .p2align 6
 ; GFX1132-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
@@ -5381,26 +5381,26 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX1132-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[41:42]
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX1132-DPP-NEXT:  .LBB7_3:
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -6162,12 +6162,12 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
 ; GFX7LESS-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -6187,8 +6187,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[36:39], 0
@@ -6683,13 +6683,13 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
 ; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-NEXT:    s_addc_u32 s49, s49, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
+; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
 ; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
@@ -6697,15 +6697,15 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX7LESS-NEXT:  ; %bb.1:
 ; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX7LESS-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v3, v0, v1
@@ -6717,8 +6717,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[2:3], 4.0
@@ -6726,8 +6726,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
@@ -6736,59 +6736,59 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX7LESS-NEXT:  .LBB10_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s50, -1
-; GFX9-NEXT:    s_mov_b32 s51, 0xe00000
+; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s66, -1
+; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-NEXT:    s_add_u32 s48, s48, s11
+; GFX9-NEXT:    s_add_u32 s64, s64, s11
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-NEXT:    s_addc_u32 s49, s49, 0
-; GFX9-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-NEXT:    s_addc_u32 s65, s65, 0
+; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX9-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b32 s42, s9
+; GFX9-NEXT:    s_mov_b32 s50, s9
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX9-NEXT:    s_mov_b32 s43, s8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX9-NEXT:    s_mov_b32 s51, s8
 ; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX9-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX9-NEXT:  .LBB10_2: ; %atomicrmw.start
@@ -6801,36 +6801,36 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX9-NEXT:  .LBB10_3:
 ; GFX9-NEXT:    s_endpgm
@@ -6838,32 +6838,32 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1064-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1064:       ; %bb.0:
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s50, -1
-; GFX1064-NEXT:    s_mov_b32 s51, 0x31e16000
+; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s66, -1
+; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-NEXT:    s_add_u32 s48, s48, s11
-; GFX1064-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b32 s42, s9
-; GFX1064-NEXT:    s_mov_b32 s43, s8
+; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
@@ -6876,38 +6876,38 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s12, s51
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1064-NEXT:  .LBB10_3:
 ; GFX1064-NEXT:    s_endpgm
@@ -6915,31 +6915,31 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1032:       ; %bb.0:
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
+; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s66, -1
+; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_add_u32 s48, s48, s11
-; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1032-NEXT:    s_mov_b32 s46, 0
+; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b32 s42, s9
-; GFX1032-NEXT:    s_mov_b32 s43, s8
+; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
@@ -6952,38 +6952,38 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s12, s51
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1032-NEXT:  .LBB10_3:
 ; GFX1032-NEXT:    s_endpgm
@@ -6992,7 +6992,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -7000,16 +7000,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b32 s42, s9
-; GFX1164-NEXT:    s_mov_b32 s43, s8
+; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
@@ -7031,18 +7031,18 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
 ; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -7050,8 +7050,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1164-NEXT:  .LBB10_3:
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -7061,22 +7061,22 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1132-NEXT:    s_mov_b32 s46, 0
+; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s42, s14
-; GFX1132-NEXT:    s_mov_b32 s43, s13
+; GFX1132-NEXT:    s_mov_b32 s50, s14
+; GFX1132-NEXT:    s_mov_b32 s51, s13
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
@@ -7095,26 +7095,26 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s43
-; GFX1132-NEXT:    s_mov_b32 s13, s42
+; GFX1132-NEXT:    s_mov_b32 s12, s51
+; GFX1132-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX1132-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
 ; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1132-NEXT:  .LBB10_3:
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -7123,13 +7123,13 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX7LESS-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
 ; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
@@ -7137,15 +7137,15 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v3, v0, v1
@@ -7157,8 +7157,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[2:3], 4.0
@@ -7166,8 +7166,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
@@ -7176,59 +7176,59 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX7LESS-DPP-NEXT:  .LBB10_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s51, 0xe00000
+; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX9-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
@@ -7241,36 +7241,36 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX9-DPP-NEXT:  .LBB10_3:
 ; GFX9-DPP-NEXT:    s_endpgm
@@ -7278,32 +7278,32 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, 0x31e16000
+; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
 ; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX1064-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1064-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
@@ -7316,38 +7316,38 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1064-DPP-NEXT:  .LBB10_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
@@ -7355,31 +7355,31 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
+; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
@@ -7392,38 +7392,38 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1032-DPP-NEXT:  .LBB10_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -7432,7 +7432,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -7440,16 +7440,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
@@ -7471,18 +7471,18 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -7490,8 +7490,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1164-DPP-NEXT:  .LBB10_3:
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -7501,22 +7501,22 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1132-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s42, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s43, s13
+; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
+; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
@@ -7535,26 +7535,26 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX1132-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1132-DPP-NEXT:  .LBB10_3:
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -7567,19 +7567,19 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
 ; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
@@ -7590,15 +7590,15 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
@@ -7625,19 +7625,19 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[44:45], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s47, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s46, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s54, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[44:47], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
 ; GFX7LESS-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[2:3], v[41:42]
@@ -7645,8 +7645,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
@@ -7655,43 +7655,43 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX7LESS-NEXT:  .LBB11_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s50, -1
-; GFX9-NEXT:    s_mov_b32 s51, 0xe00000
-; GFX9-NEXT:    s_add_u32 s48, s48, s11
-; GFX9-NEXT:    s_addc_u32 s49, s49, 0
+; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s66, -1
+; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
+; GFX9-NEXT:    s_add_u32 s64, s64, s11
+; GFX9-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s43, s8
+; GFX9-NEXT:    s_mov_b32 s51, s8
 ; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s42, s9
+; GFX9-NEXT:    s_mov_b32 s50, s9
 ; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -7703,14 +7703,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -7737,12 +7737,12 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[44:45], s[36:37], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX9-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[4:5], v0, s[44:45]
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
 ; GFX9-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -7753,54 +7753,54 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v5, off, s[48:51], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v4, off, s[48:51], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
+; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX9-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v4, off, s[48:51], 0
-; GFX9-NEXT:    buffer_load_dword v5, off, s[48:51], 0 offset:4
+; GFX9-NEXT:    buffer_load_dword v4, off, s[64:67], 0
+; GFX9-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX9-NEXT:  .LBB11_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s50, -1
-; GFX1064-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s66, -1
+; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-NEXT:    s_mov_b32 s43, s8
+; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s42, s9
+; GFX1064-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -7812,14 +7812,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
+; GFX1064-NEXT:    s_mov_b32 s12, s51
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -7846,12 +7846,12 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1064-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1064-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[4:5], v0, s[44:45]
+; GFX1064-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
 ; GFX1064-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
@@ -7861,56 +7861,56 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v5, off, s[48:51], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[48:51], 0
+; GFX1064-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
+; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s44
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s12, s51
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX1064-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v4, off, s[48:51], 0
-; GFX1064-NEXT:    buffer_load_dword v5, off, s[48:51], 0 offset:4
+; GFX1064-NEXT:    buffer_load_dword v4, off, s[64:67], 0
+; GFX1064-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
 ; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX1064-NEXT:  .LBB11_5:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s66, -1
+; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-NEXT:    s_mov_b32 s43, s8
+; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s42, s9
+; GFX1032-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -7922,14 +7922,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
+; GFX1032-NEXT:    s_mov_b32 s12, s51
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -7950,17 +7950,17 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s46, 0
+; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1032-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[4:5], v0, s[44:45]
+; GFX1032-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
 ; GFX1032-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
@@ -7970,38 +7970,38 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v5, off, s[48:51], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[48:51], 0
+; GFX1032-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
+; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0
 ; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s44
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s12, s51
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX1032-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v4, off, s[48:51], 0
-; GFX1032-NEXT:    buffer_load_dword v5, off, s[48:51], 0 offset:4
+; GFX1032-NEXT:    buffer_load_dword v4, off, s[64:67], 0
+; GFX1032-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
 ; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX1032-NEXT:  .LBB11_5:
 ; GFX1032-NEXT:    s_endpgm
@@ -8009,11 +8009,11 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1164-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s43, s8
+; GFX1164-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s42, s9
+; GFX1164-NEXT:    s_mov_b32 s50, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1164-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1164-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -8021,11 +8021,11 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1164-NEXT:    s_mov_b32 s33, s10
 ; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
@@ -8059,12 +8059,12 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1164-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[4:5], v0, s[44:45]
+; GFX1164-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1164-NEXT:    .p2align 6
 ; GFX1164-NEXT:  .LBB11_4: ; %atomicrmw.start
@@ -8078,15 +8078,15 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s44
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
 ; GFX1164-NEXT:    scratch_store_b64 off, v[4:5], off
@@ -8101,8 +8101,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX1164-NEXT:  .LBB11_5:
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -8111,7 +8111,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1132-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
 ; GFX1132-NEXT:    s_getpc_b64 s[0:1]
@@ -8120,9 +8120,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s42, s14
-; GFX1132-NEXT:    s_mov_b32 s43, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b32 s50, s14
+; GFX1132-NEXT:    s_mov_b32 s51, s13
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
@@ -8154,19 +8154,19 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s46, 0
+; GFX1132-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX1132-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[4:5], v0, s[44:45]
+; GFX1132-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1132-NEXT:    .p2align 6
 ; GFX1132-NEXT:  .LBB11_4: ; %atomicrmw.start
@@ -8180,16 +8180,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX1132-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1132-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s43
-; GFX1132-NEXT:    s_mov_b32 s13, s42
+; GFX1132-NEXT:    s_mov_b32 s12, s51
+; GFX1132-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1132-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1132-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
 ; GFX1132-NEXT:    scratch_store_b64 off, v[4:5], off
 ; GFX1132-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 8
@@ -8201,8 +8201,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX1132-NEXT:  .LBB11_5:
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -8211,22 +8211,22 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX7LESS-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s47, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s46, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
@@ -8237,34 +8237,34 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[44:47], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[52:55], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[41:42], v[0:1], v[0:1]
 ; GFX7LESS-DPP-NEXT:  .LBB11_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
@@ -8273,43 +8273,43 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v3, off, s[48:51], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v3, off, s[64:67], 0 offset:4
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB11_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s52, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s53, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s55, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s52, s52, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s53, s53, 0
+; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
 ; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -8321,14 +8321,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[52:53]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[54:55]
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
 ; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -8382,20 +8382,20 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s45, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s44, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[46:47], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[46:47]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
 ; GFX9-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], s[44:45], s[44:45]
+; GFX9-DPP-NEXT:    v_max_f64 v[3:4], s[52:53], s[52:53]
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
 ; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
@@ -8404,54 +8404,54 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[52:53]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[52:55], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[52:55], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
+; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[52:55], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[52:55], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[54:55]
+; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
+; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s46
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s47
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[52:55], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[52:55], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
+; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
 ; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[48:49], vcc, s[48:49]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[48:49]
+; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX9-DPP-NEXT:  .LBB11_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -8463,14 +8463,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -8523,10 +8523,10 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1064-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[41:42], v[41:42]
@@ -8537,56 +8537,56 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX1064-DPP-NEXT:  .LBB11_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -8598,14 +8598,14 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -8646,15 +8646,15 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1032-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
@@ -8664,38 +8664,38 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[41:42]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX1032-DPP-NEXT:  .LBB11_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -8703,11 +8703,11 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
 ; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
@@ -8715,11 +8715,11 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
@@ -8787,11 +8787,11 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1164-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1164-DPP-NEXT:    .p2align 6
 ; GFX1164-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
@@ -8809,18 +8809,18 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[41:42]
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -8828,8 +8828,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX1164-DPP-NEXT:  .LBB11_3:
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -8838,7 +8838,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
 ; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
@@ -8847,9 +8847,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s42, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s43, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
+; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
@@ -8904,16 +8904,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v8
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1132-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1132-DPP-NEXT:    .p2align 6
 ; GFX1132-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
@@ -8929,26 +8929,26 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX1132-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[41:42]
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX1132-DPP-NEXT:  .LBB11_3:
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
index 7126680525b87..d575605f102b7 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
@@ -821,12 +821,12 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -846,8 +846,8 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[36:39], 0
@@ -2153,12 +2153,12 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_uni_address_div_value_one_as_scope_unsafe_structfp:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -2178,8 +2178,8 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[36:39], 0
@@ -3485,12 +3485,12 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_uni_address_div_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -3510,8 +3510,8 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[36:39], 0
@@ -4313,12 +4313,12 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_uni_address_div_value_agent_scope_unsafe_structfp:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -4338,8 +4338,8 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[36:39], 0
@@ -5644,12 +5644,12 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_uni_address_div_value_default_scope_strictfp:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -5669,8 +5669,8 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[36:39], 0
@@ -6075,14 +6075,14 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
 ; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, s0, 0
 ; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, s1, v3
@@ -6091,16 +6091,16 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX7LESS-NEXT:  ; %bb.1:
 ; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX7LESS-NEXT:    v_cvt_f64_u32_e32 v[1:2], s2
 ; GFX7LESS-NEXT:    v_or_b32_e32 v4, v0, v4
@@ -6113,11 +6113,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], -v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:8
+; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
+; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
 ; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
@@ -6130,42 +6130,42 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX7LESS-NEXT:  .LBB9_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s50, -1
-; GFX9-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s66, -1
+; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    s_mov_b32 s51, 0xe00000
+; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, s0, 0
-; GFX9-NEXT:    s_add_u32 s48, s48, s11
+; GFX9-NEXT:    s_add_u32 s64, s64, s11
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, s1, v3
-; GFX9-NEXT:    s_addc_u32 s49, s49, 0
+; GFX9-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX9-NEXT:    s_movk_i32 s32, 0x800
@@ -6173,21 +6173,21 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX9-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX9-NEXT:  ; %bb.1:
 ; GFX9-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX9-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX9-NEXT:    s_mov_b32 s33, s10
 ; GFX9-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX9-NEXT:    s_mov_b32 s42, s9
-; GFX9-NEXT:    s_mov_b32 s43, s8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX9-NEXT:    s_mov_b32 s50, s9
+; GFX9-NEXT:    s_mov_b32 s51, s8
 ; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s1
-; GFX9-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s0
 ; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-NEXT:  .LBB9_2: ; %atomicrmw.start
@@ -6200,68 +6200,68 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v3, off, s[48:51], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; GFX9-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:4
+; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX9-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v3, off, s[48:51], 0
-; GFX9-NEXT:    buffer_load_dword v4, off, s[48:51], 0 offset:4
+; GFX9-NEXT:    buffer_load_dword v3, off, s[64:67], 0
+; GFX9-NEXT:    buffer_load_dword v4, off, s[64:67], 0 offset:4
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX9-NEXT:  .LBB9_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s50, -1
-; GFX1064-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s66, -1
+; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-NEXT:    s_mov_b32 s33, s10
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1064-NEXT:    s_addc_u32 s49, s49, 0
+; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, s10, 0
-; GFX1064-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, s11, v3
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1064-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1064-NEXT:    s_mov_b32 s42, s9
+; GFX1064-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1064-NEXT:    s_mov_b32 s43, s8
+; GFX1064-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1064-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
@@ -6278,69 +6278,69 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s12, s51
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1064-NEXT:  .LBB9_3:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s42, s9
+; GFX1032-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-NEXT:    s_mov_b32 s9, exec_lo
-; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
-; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s66, -1
+; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1032-NEXT:    s_mov_b32 s46, 0
+; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1032-NEXT:    s_bcnt1_i32_b32 s0, s9
 ; GFX1032-NEXT:    s_mov_b32 s33, s10
 ; GFX1032-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1032-NEXT:    s_mov_b32 s43, s8
+; GFX1032-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1032-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
@@ -6357,37 +6357,37 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s12, s51
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1032-NEXT:  .LBB9_3:
 ; GFX1032-NEXT:    s_endpgm
@@ -6398,7 +6398,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], exec
 ; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, s10, 0
-; GFX1164-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -6407,16 +6407,16 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX1164-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1164-NEXT:  ; %bb.1:
 ; GFX1164-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1164-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1164-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-NEXT:    s_mov_b32 s42, s9
-; GFX1164-NEXT:    s_mov_b32 s43, s8
+; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1164-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
@@ -6439,18 +6439,18 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -6458,8 +6458,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1164-NEXT:  .LBB9_3:
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -6470,24 +6470,24 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX1132-NEXT:    s_mov_b32 s8, exec_lo
 ; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX1132-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1132-NEXT:    s_mov_b32 s46, 0
+; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1132-NEXT:  ; %bb.1:
 ; GFX1132-NEXT:    s_bcnt1_i32_b32 s0, s8
-; GFX1132-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1132-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
 ; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s42, s14
-; GFX1132-NEXT:    s_mov_b32 s43, s13
+; GFX1132-NEXT:    s_mov_b32 s50, s14
+; GFX1132-NEXT:    s_mov_b32 s51, s13
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
@@ -6507,25 +6507,25 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s43
-; GFX1132-NEXT:    s_mov_b32 s13, s42
+; GFX1132-NEXT:    s_mov_b32 s12, s51
+; GFX1132-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1132-NEXT:  .LBB9_3:
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -6534,14 +6534,14 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, s0, 0
 ; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, s1, v3
@@ -6550,16 +6550,16 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
 ; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_cvt_f64_u32_e32 v[1:2], s2
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v4, v0, v4
@@ -6572,11 +6572,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], -v[41:42]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:8
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
@@ -6589,42 +6589,42 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX7LESS-DPP-NEXT:  .LBB9_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX9-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT:    s_mov_b32 s51, 0xe00000
+; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s0, 0
-; GFX9-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, s1, v3
-; GFX9-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
@@ -6632,21 +6632,21 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
 ; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX9-DPP-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
 ; GFX9-DPP-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX9-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s0
 ; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
@@ -6659,68 +6659,68 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0 offset:8
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
+; GFX9-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v3, off, s[48:51], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v4, off, s[48:51], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_load_dword v3, off, s[64:67], 0
+; GFX9-DPP-NEXT:    buffer_load_dword v4, off, s[64:67], 0 offset:4
 ; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX9-DPP-NEXT:  .LBB9_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1064-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s10, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, s11, v3
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1064-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-DPP-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1064-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1064-DPP-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
@@ -6737,69 +6737,69 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1064-DPP-NEXT:  .LBB9_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-DPP-NEXT:    s_mov_b32 s9, exec_lo
-; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
 ; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s0, s9
 ; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
 ; GFX1032-DPP-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1032-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1032-DPP-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
@@ -6816,37 +6816,37 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1032-DPP-NEXT:  .LBB9_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -6857,7 +6857,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], exec
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s10, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -6866,16 +6866,16 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
 ; GFX1164-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1164-DPP-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1164-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1164-DPP-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -6898,18 +6898,18 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -6917,8 +6917,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1164-DPP-NEXT:  .LBB9_3:
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -6929,24 +6929,24 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX1132-DPP-NEXT:    s_mov_b32 s8, exec_lo
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX1132-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
 ; GFX1132-DPP-NEXT:    s_bcnt1_i32_b32 s0, s8
-; GFX1132-DPP-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1132-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s42, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s43, s13
+; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
+; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-DPP-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
@@ -6966,25 +6966,25 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1132-DPP-NEXT:  .LBB9_3:
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -6997,19 +6997,19 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX7LESS:       ; %bb.0:
 ; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
@@ -7020,15 +7020,15 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
@@ -7053,21 +7053,21 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[44:45], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s47, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s46, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s54, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[44:47], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], -v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:8
+; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
+; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
 ; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
@@ -7080,44 +7080,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX7LESS-NEXT:  .LBB10_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s50, -1
-; GFX9-NEXT:    s_mov_b32 s51, 0xe00000
-; GFX9-NEXT:    s_add_u32 s48, s48, s11
-; GFX9-NEXT:    s_addc_u32 s49, s49, 0
+; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s66, -1
+; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
+; GFX9-NEXT:    s_add_u32 s64, s64, s11
+; GFX9-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s43, s8
+; GFX9-NEXT:    s_mov_b32 s51, s8
 ; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s42, s9
+; GFX9-NEXT:    s_mov_b32 s50, s9
 ; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -7129,14 +7129,14 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -7161,11 +7161,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[44:45], s[36:37], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX9-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX9-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -7176,53 +7176,53 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX9-NEXT:  .LBB10_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s50, -1
-; GFX1064-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s66, -1
+; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-NEXT:    s_mov_b32 s43, s8
+; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s42, s9
+; GFX1064-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -7234,14 +7234,14 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
+; GFX1064-NEXT:    s_mov_b32 s12, s51
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -7266,11 +7266,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1064-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1064-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1064-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
@@ -7280,55 +7280,55 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s12, s51
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1064-NEXT:  .LBB10_5:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s66, -1
+; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-NEXT:    s_mov_b32 s43, s8
+; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s42, s9
+; GFX1032-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -7340,14 +7340,14 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
+; GFX1032-NEXT:    s_mov_b32 s12, s51
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -7366,16 +7366,16 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB10_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s46, 0
+; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1032-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1032-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1032-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
@@ -7385,37 +7385,37 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s12, s51
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1032-NEXT:  .LBB10_5:
 ; GFX1032-NEXT:    s_endpgm
@@ -7423,11 +7423,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s43, s8
+; GFX1164-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s42, s9
+; GFX1164-NEXT:    s_mov_b32 s50, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1164-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1164-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -7435,11 +7435,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1164-NEXT:    s_mov_b32 s33, s10
 ; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
@@ -7471,11 +7471,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1164-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1164-NEXT:    .p2align 6
 ; GFX1164-NEXT:  .LBB10_4: ; %atomicrmw.start
@@ -7492,18 +7492,18 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -7511,8 +7511,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1164-NEXT:  .LBB10_5:
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -7521,7 +7521,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
 ; GFX1132-NEXT:    s_getpc_b64 s[0:1]
@@ -7530,9 +7530,9 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s42, s14
-; GFX1132-NEXT:    s_mov_b32 s43, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b32 s50, s14
+; GFX1132-NEXT:    s_mov_b32 s51, s13
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
@@ -7561,17 +7561,17 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB10_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s46, 0
+; GFX1132-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1132-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1132-NEXT:    .p2align 6
 ; GFX1132-NEXT:  .LBB10_4: ; %atomicrmw.start
@@ -7586,25 +7586,25 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s43
-; GFX1132-NEXT:    s_mov_b32 s13, s42
+; GFX1132-NEXT:    s_mov_b32 s12, s51
+; GFX1132-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1132-NEXT:  .LBB10_5:
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -7613,22 +7613,22 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s47, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s46, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
@@ -7639,30 +7639,30 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v42, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v41, v1
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[0:1], off, s[44:47], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-DPP-NEXT:  .LBB10_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], -v[40:41]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:8
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
@@ -7675,44 +7675,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB10_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s52, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s53, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s55, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s52, s52, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s53, s53, 0
+; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
 ; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -7724,14 +7724,14 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[52:53]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[54:55]
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
 ; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -7778,74 +7778,74 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s45, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s44, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[46:47], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[46:47]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
 ; GFX9-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -s[44:45]
+; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -s[52:53]
 ; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[52:53]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[52:55], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[52:55], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[52:55], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[52:55], 0 offset:8
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
+; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
+; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[54:55]
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s46
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s47
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[52:55], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[52:55], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
+; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
 ; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[48:49], vcc, s[48:49]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[48:49]
+; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX9-DPP-NEXT:  .LBB10_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -7857,14 +7857,14 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -7909,10 +7909,10 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1064-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
@@ -7922,55 +7922,55 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1064-DPP-NEXT:  .LBB10_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -7982,14 +7982,14 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -8024,14 +8024,14 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1032-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
@@ -8041,37 +8041,37 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1032-DPP-NEXT:  .LBB10_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -8079,11 +8079,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -8091,11 +8091,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
@@ -8153,10 +8153,10 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1164-DPP-NEXT:    .p2align 6
 ; GFX1164-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
@@ -8173,18 +8173,18 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -8192,8 +8192,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1164-DPP-NEXT:  .LBB10_3:
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -8202,7 +8202,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
 ; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
@@ -8211,9 +8211,9 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s42, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s43, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
+; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
@@ -8264,14 +8264,14 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1132-DPP-NEXT:    .p2align 6
 ; GFX1132-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
@@ -8286,25 +8286,25 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1132-DPP-NEXT:  .LBB10_3:
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -9249,12 +9249,12 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -9274,8 +9274,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[4:5], off, s[36:39], 0
@@ -10682,12 +10682,12 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -10707,8 +10707,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[4:5], off, s[36:39], 0
@@ -11597,12 +11597,12 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s40, s40, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s41, s41, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
@@ -11622,8 +11622,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[4:5], off, s[36:39], 0
@@ -12097,13 +12097,13 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX7LESS:       ; %bb.0:
 ; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-NEXT:    s_addc_u32 s49, s49, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
+; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v5, exec_lo, 0
 ; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s0, exec
 ; GFX7LESS-NEXT:    s_mov_b32 s1, 0x43300000
@@ -12117,15 +12117,15 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX7LESS-NEXT:  ; %bb.1:
 ; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX7LESS-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v3, v0, v1
@@ -12137,11 +12137,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], -v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:8
+; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
+; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
 ; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
@@ -12154,40 +12154,40 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX7LESS-NEXT:  .LBB16_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s50, -1
-; GFX9-NEXT:    s_mov_b32 s51, 0xe00000
-; GFX9-NEXT:    s_add_u32 s48, s48, s11
+; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s66, -1
+; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
+; GFX9-NEXT:    s_add_u32 s64, s64, s11
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-NEXT:    s_addc_u32 s49, s49, 0
-; GFX9-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-NEXT:    s_addc_u32 s65, s65, 0
+; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-NEXT:    s_bcnt1_i32_b64 s0, exec
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0xc3300000
 ; GFX9-NEXT:    s_mov_b32 s1, 0x43300000
@@ -12200,20 +12200,20 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b32 s42, s9
+; GFX9-NEXT:    s_mov_b32 s50, s9
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX9-NEXT:    s_mov_b32 s43, s8
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX9-NEXT:    s_mov_b32 s51, s8
 ; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX9-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX9-NEXT:  .LBB16_2: ; %atomicrmw.start
@@ -12226,48 +12226,48 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX9-NEXT:  .LBB16_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s50, -1
-; GFX1064-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s48, s48, s11
-; GFX1064-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s66, -1
+; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-NEXT:    s_bcnt1_i32_b64 s0, exec
 ; GFX1064-NEXT:    s_mov_b32 s1, 0x43300000
 ; GFX1064-NEXT:    s_movk_i32 s32, 0x800
@@ -12279,19 +12279,19 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b32 s42, s9
-; GFX1064-NEXT:    s_mov_b32 s43, s8
+; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
@@ -12304,53 +12304,53 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s12, s51
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1064-NEXT:  .LBB16_3:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s48, s48, s11
-; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s66, -1
+; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
 ; GFX1032-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT:    s_mov_b32 s46, 0
+; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
@@ -12359,18 +12359,18 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b32 s42, s9
-; GFX1032-NEXT:    s_mov_b32 s43, s8
+; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
@@ -12383,44 +12383,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s12, s51
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1032-NEXT:  .LBB16_3:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1164:       ; %bb.0:
-; GFX1164-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_bcnt1_i32_b64 s0, exec
 ; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 0x43300000
@@ -12441,16 +12441,16 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b32 s42, s9
-; GFX1164-NEXT:    s_mov_b32 s43, s8
+; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
@@ -12471,18 +12471,18 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -12490,8 +12490,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1164-NEXT:  .LBB16_3:
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -12499,12 +12499,12 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ;
 ; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1132:       ; %bb.0:
-; GFX1132-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX1132-NEXT:    v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0
 ; GFX1132-NEXT:    v_mov_b32_e32 v0, 0x43300000
-; GFX1132-NEXT:    s_mov_b32 s46, 0
+; GFX1132-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-NEXT:    s_clause 0x1
 ; GFX1132-NEXT:    scratch_store_b32 off, v0, off offset:20
 ; GFX1132-NEXT:    scratch_store_b32 off, v1, off offset:16
@@ -12519,15 +12519,15 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s42, s14
-; GFX1132-NEXT:    s_mov_b32 s43, s13
+; GFX1132-NEXT:    s_mov_b32 s50, s14
+; GFX1132-NEXT:    s_mov_b32 s51, s13
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
@@ -12545,25 +12545,25 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s43
-; GFX1132-NEXT:    s_mov_b32 s13, s42
+; GFX1132-NEXT:    s_mov_b32 s12, s51
+; GFX1132-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1132-NEXT:  .LBB16_3:
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -12572,13 +12572,13 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v5, exec_lo, 0
 ; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s1, 0x43300000
@@ -12592,15 +12592,15 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v3, v0, v1
@@ -12612,11 +12612,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], -v[41:42]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:8
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
@@ -12629,40 +12629,40 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX7LESS-DPP-NEXT:  .LBB16_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s51, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0xc3300000
 ; GFX9-DPP-NEXT:    s_mov_b32 s1, 0x43300000
@@ -12675,20 +12675,20 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX9-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
@@ -12701,48 +12701,48 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX9-DPP-NEXT:  .LBB16_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX1064-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
 ; GFX1064-DPP-NEXT:    s_mov_b32 s1, 0x43300000
 ; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
@@ -12754,19 +12754,19 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1064-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
@@ -12779,53 +12779,53 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1064-DPP-NEXT:  .LBB16_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
 ; GFX1032-DPP-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
@@ -12834,18 +12834,18 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
 ; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[44:45], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
@@ -12858,44 +12858,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1032-DPP-NEXT:  .LBB16_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1164-DPP:       ; %bb.0:
-; GFX1164-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0x43300000
@@ -12916,16 +12916,16 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
@@ -12946,18 +12946,18 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -12965,8 +12965,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1164-DPP-NEXT:  .LBB16_3:
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -12974,12 +12974,12 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1132-DPP:       ; %bb.0:
-; GFX1132-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-DPP-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, 0x43300000
-; GFX1132-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-DPP-NEXT:    s_clause 0x1
 ; GFX1132-DPP-NEXT:    scratch_store_b32 off, v0, off offset:20
 ; GFX1132-DPP-NEXT:    scratch_store_b32 off, v1, off offset:16
@@ -12994,15 +12994,15 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[44:45], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
 ; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s42, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s43, s13
+; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
+; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[44:45], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
@@ -13020,25 +13020,25 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1132-DPP-NEXT:  .LBB16_3:
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -13051,19 +13051,19 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX7LESS:       ; %bb.0:
 ; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
@@ -13074,15 +13074,15 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
@@ -13107,21 +13107,21 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[44:45], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s47, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s46, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s54, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[44:47], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], -v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:8
+; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
+; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
 ; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
@@ -13134,44 +13134,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX7LESS-NEXT:  .LBB17_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s50, -1
-; GFX9-NEXT:    s_mov_b32 s51, 0xe00000
-; GFX9-NEXT:    s_add_u32 s48, s48, s11
-; GFX9-NEXT:    s_addc_u32 s49, s49, 0
+; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s66, -1
+; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
+; GFX9-NEXT:    s_add_u32 s64, s64, s11
+; GFX9-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s43, s8
+; GFX9-NEXT:    s_mov_b32 s51, s8
 ; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s42, s9
+; GFX9-NEXT:    s_mov_b32 s50, s9
 ; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -13183,14 +13183,14 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -13215,11 +13215,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[44:45], s[36:37], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    s_mov_b64 s[46:47], 0
+; GFX9-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX9-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX9-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -13230,53 +13230,53 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s43
-; GFX9-NEXT:    s_mov_b32 s13, s42
+; GFX9-NEXT:    s_mov_b32 s12, s51
+; GFX9-NEXT:    s_mov_b32 s13, s50
 ; GFX9-NEXT:    s_mov_b32 s14, s33
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s44
-; GFX9-NEXT:    v_mov_b32_e32 v3, s45
+; GFX9-NEXT:    v_mov_b32_e32 v2, s52
+; GFX9-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX9-NEXT:  .LBB17_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s50, -1
-; GFX1064-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s66, -1
+; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-NEXT:    s_mov_b32 s43, s8
+; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s42, s9
+; GFX1064-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -13288,14 +13288,14 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
+; GFX1064-NEXT:    s_mov_b32 s12, s51
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -13320,11 +13320,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1064-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1064-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1064-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
@@ -13334,55 +13334,55 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s43
-; GFX1064-NEXT:    s_mov_b32 s13, s42
+; GFX1064-NEXT:    s_mov_b32 s12, s51
+; GFX1064-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1064-NEXT:  .LBB17_5:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s50, -1
-; GFX1032-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s66, -1
+; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-NEXT:    s_mov_b32 s43, s8
+; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s42, s9
+; GFX1032-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -13394,14 +13394,14 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
+; GFX1032-NEXT:    s_mov_b32 s12, s51
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -13420,16 +13420,16 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB17_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s46, 0
+; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1032-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1032-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1032-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
@@ -13439,37 +13439,37 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s43
-; GFX1032-NEXT:    s_mov_b32 s13, s42
+; GFX1032-NEXT:    s_mov_b32 s12, s51
+; GFX1032-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1032-NEXT:  .LBB17_5:
 ; GFX1032-NEXT:    s_endpgm
@@ -13477,11 +13477,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s43, s8
+; GFX1164-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s42, s9
+; GFX1164-NEXT:    s_mov_b32 s50, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1164-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1164-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -13489,11 +13489,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1164-NEXT:    s_mov_b32 s33, s10
 ; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
@@ -13525,11 +13525,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1164-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1164-NEXT:    .p2align 6
 ; GFX1164-NEXT:  .LBB17_4: ; %atomicrmw.start
@@ -13546,18 +13546,18 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s43
-; GFX1164-NEXT:    s_mov_b32 s13, s42
+; GFX1164-NEXT:    s_mov_b32 s12, s51
+; GFX1164-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -13565,8 +13565,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1164-NEXT:  .LBB17_5:
 ; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -13575,7 +13575,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
 ; GFX1132-NEXT:    s_getpc_b64 s[0:1]
@@ -13584,9 +13584,9 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s42, s14
-; GFX1132-NEXT:    s_mov_b32 s43, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b32 s50, s14
+; GFX1132-NEXT:    s_mov_b32 s51, s13
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
@@ -13615,17 +13615,17 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB17_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s46, 0
+; GFX1132-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1132-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1132-NEXT:    .p2align 6
 ; GFX1132-NEXT:  .LBB17_4: ; %atomicrmw.start
@@ -13640,25 +13640,25 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s43
-; GFX1132-NEXT:    s_mov_b32 s13, s42
+; GFX1132-NEXT:    s_mov_b32 s12, s51
+; GFX1132-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1132-NEXT:  .LBB17_5:
 ; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -13667,22 +13667,22 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX7LESS-DPP:       ; %bb.0:
 ; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s42, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[44:45], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s47, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s46, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
@@ -13693,30 +13693,30 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v42, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v41, v1
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[0:1], off, s[44:47], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX7LESS-DPP-NEXT:  .LBB17_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], -v[40:41]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[48:51], 0
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:8
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
+; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
 ; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
@@ -13729,44 +13729,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s44
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[48:51], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0 offset:4
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
 ; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB17_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s52, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s53, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s55, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s52, s52, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s53, s53, 0
+; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
 ; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -13778,14 +13778,14 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[52:53]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[54:55]
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
 ; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -13832,74 +13832,74 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s45, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s44, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[46:47], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[46:47]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
 ; GFX9-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -s[44:45]
+; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -s[52:53]
 ; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
 ; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
 ; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
 ; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[52:53]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[52:55], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[52:55], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[52:55], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[52:55], 0 offset:8
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
+; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
+; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[54:55]
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s46
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s47
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[52:55], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[52:55], 0 offset:4
+; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
+; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
 ; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[48:49], vcc, s[48:49]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[48:49]
+; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX9-DPP-NEXT:  .LBB17_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -13911,14 +13911,14 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -13963,10 +13963,10 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1064-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
@@ -13976,55 +13976,55 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[46:47]
+; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1064-DPP-NEXT:  .LBB17_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s49, s49, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -14036,14 +14036,14 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
 ; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -14078,14 +14078,14 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[44:45], s[34:35], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[44:45]
+; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
 ; GFX1032-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
@@ -14095,37 +14095,37 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
 ; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[48:51], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[48:51], 0
+; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
 ; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[48:51], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[48:51], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
+; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[48:51], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[48:51], 0 offset:4
+; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
+; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
 ; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s46
+; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1032-DPP-NEXT:  .LBB17_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -14133,11 +14133,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s43, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s42, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
 ; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
 ; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
@@ -14145,11 +14145,11 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
@@ -14207,10 +14207,10 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b64 s[46:47], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1164-DPP-NEXT:    .p2align 6
 ; GFX1164-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
@@ -14227,18 +14227,18 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s44
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s45
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
@@ -14246,8 +14246,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[46:47], vcc, s[46:47]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[46:47]
+; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1164-DPP-NEXT:  .LBB17_3:
 ; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
@@ -14256,7 +14256,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[40:41], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
 ; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
@@ -14265,9 +14265,9 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s42, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s43, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
+; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
@@ -14318,14 +14318,14 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s46, 0
+; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[44:45], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[44:45]
+; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX1132-DPP-NEXT:    .p2align 6
 ; GFX1132-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
@@ -14340,25 +14340,25 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
 ; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s44
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
 ; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
 ; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s43
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s42
+; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
+; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
 ; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s45 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
 ; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s46, vcc_lo, s46
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s46
+; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1132-DPP-NEXT:  .LBB17_3:
 ; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
diff --git a/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir b/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir
index dde84af57ed25..da1175c02e94a 100644
--- a/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir
+++ b/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir
@@ -31,102 +31,105 @@ body:             |
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $sgpr34_sgpr35 = COPY $sgpr8_sgpr9
   ; CHECK-NEXT:   renamable $sgpr33 = COPY $sgpr15
-  ; CHECK-NEXT:   renamable $sgpr42 = COPY $sgpr14
+  ; CHECK-NEXT:   renamable $sgpr50 = COPY $sgpr14
   ; CHECK-NEXT:   renamable $sgpr36_sgpr37 = COPY $sgpr10_sgpr11
   ; CHECK-NEXT:   renamable $sgpr38_sgpr39 = COPY $sgpr6_sgpr7
-  ; CHECK-NEXT:   renamable $sgpr40_sgpr41 = COPY $sgpr4_sgpr5
-  ; CHECK-NEXT:   renamable $sgpr66_sgpr67 = S_LOAD_DWORDX2_IMM renamable $sgpr34_sgpr35, 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
-  ; CHECK-NEXT:   renamable $sgpr44 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr45 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr46 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr47 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr48 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr49 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr50 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr51 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr52 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr53 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr54 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr55 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr56 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr57 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr58 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr59 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr60 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr61 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr62 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr63 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr64 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr68_sgpr69 = IMPLICIT_DEF
+  ; CHECK-NEXT:   renamable $sgpr48_sgpr49 = COPY $sgpr4_sgpr5
+  ; CHECK-NEXT:   renamable $sgpr64_sgpr65 = S_LOAD_DWORDX2_IMM renamable $sgpr34_sgpr35, 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+  ; CHECK-NEXT:   renamable $sgpr68 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr69 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr70 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr71 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr72 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr73 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr74 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr75 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr76 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr77 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr78 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr79 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr80 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr81 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr82 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr83 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr84 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr85 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr86 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr87 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr88 = S_MOV_B32 0
+  ; CHECK-NEXT:   SI_SPILL_S1024_SAVE renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99, %stack.0, implicit $exec, implicit $sgpr32 :: (store (s1024) into %stack.0, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr52_sgpr53 = IMPLICIT_DEF
   ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
-  ; CHECK-NEXT:   dead $sgpr30_sgpr31 = SI_CALL renamable $sgpr68_sgpr69, 0, csr_amdgpu, implicit $sgpr0_sgpr1_sgpr2_sgpr3
+  ; CHECK-NEXT:   dead $sgpr30_sgpr31 = SI_CALL renamable $sgpr52_sgpr53, 0, csr_amdgpu, implicit $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
   ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
-  ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY killed renamable $sgpr40_sgpr41
+  ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY killed renamable $sgpr48_sgpr49
   ; CHECK-NEXT:   $sgpr6_sgpr7 = COPY killed renamable $sgpr38_sgpr39
   ; CHECK-NEXT:   $sgpr8_sgpr9 = COPY killed renamable $sgpr34_sgpr35
   ; CHECK-NEXT:   $sgpr10_sgpr11 = COPY killed renamable $sgpr36_sgpr37
-  ; CHECK-NEXT:   $sgpr12 = COPY killed renamable $sgpr42
+  ; CHECK-NEXT:   $sgpr12 = COPY killed renamable $sgpr50
   ; CHECK-NEXT:   $sgpr13 = COPY killed renamable $sgpr33
-  ; CHECK-NEXT:   dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr68_sgpr69, 0, csr_amdgpu, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr0_sgpr1_sgpr2_sgpr3
+  ; CHECK-NEXT:   dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr52_sgpr53, 0, csr_amdgpu, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
   ; CHECK-NEXT:   renamable $sgpr4_sgpr5 = COPY $exec, implicit-def $exec
   ; CHECK-NEXT:   dead renamable $sgpr6_sgpr7 = IMPLICIT_DEF
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.4(0x40000000)
-  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5, $sgpr66_sgpr67:0x000000000000000F, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75:0x000003FFFFFFFFFF
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5, $sgpr64_sgpr65:0x000000000000000F
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $sgpr6_sgpr7 = COPY $exec, implicit-def $exec
   ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.4, implicit $exec
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   successors: %bb.3(0x80000000)
-  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5, $sgpr66_sgpr67:0x000000000000000F, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75:0x000003FFFFFFFFFF
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5, $sgpr64_sgpr65:0x000000000000000F
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vreg_1024 = COPY renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75
-  ; CHECK-NEXT:   renamable $sgpr6 = S_LSHL_B32 renamable $sgpr67, 1, implicit-def dead $scc
+  ; CHECK-NEXT:   renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.0, align 4, addrspace 5)
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vreg_1024 = COPY renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99
+  ; CHECK-NEXT:   renamable $sgpr6 = S_LSHL_B32 renamable $sgpr65, 1, implicit-def dead $scc
   ; CHECK-NEXT:   dead [[COPY:%[0-9]+]]:vreg_1024 = V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32 [[COPY]], 0, killed $sgpr6, 3, implicit-def $m0, implicit $m0, implicit $exec
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.5(0x40000000), %bb.1(0x40000000)
-  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5, $sgpr66_sgpr67:0x000000000000000F, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75:0x000003FFFFFFFFFF
+  ; CHECK-NEXT:   liveins: $sgpr4_sgpr5, $sgpr64_sgpr65:0x000000000000000F, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99:0x0000000000000003
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $sgpr6_sgpr7 = S_OR_SAVEEXEC_B64 renamable $sgpr4_sgpr5, implicit-def $exec, implicit-def $scc, implicit $exec
-  ; CHECK-NEXT:   renamable $sgpr68 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr69 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr70 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr71 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr72 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr73 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr74 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr75 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr76 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr77 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr78 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr79 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr80 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr81 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr82 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr83 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr84 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr85 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr86 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr87 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr88 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   renamable $sgpr89 = COPY renamable $sgpr44
-  ; CHECK-NEXT:   dead [[COPY1:%[0-9]+]]:vreg_1024 = COPY renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr36 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr37 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr38 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr39 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr40 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr41 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr42 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr43 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr44 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr45 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr46 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr47 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr48 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr49 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr50 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr51 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr52 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr53 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr54 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr55 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr56 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr57 = COPY killed renamable $sgpr68
+  ; CHECK-NEXT:   dead [[COPY1:%[0-9]+]]:vreg_1024 = COPY renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, implicit $exec
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, killed renamable $sgpr6_sgpr7, implicit-def $scc
   ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.5, implicit $exec
   ; CHECK-NEXT:   S_BRANCH %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.4:
   ; CHECK-NEXT:   successors: %bb.5(0x80000000)
-  ; CHECK-NEXT:   liveins: $sgpr6_sgpr7, $sgpr66_sgpr67:0x0000000000000003, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75:0x000003FFFFFFFFFF
+  ; CHECK-NEXT:   liveins: $sgpr6_sgpr7, $sgpr64_sgpr65:0x0000000000000003
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr6_sgpr7, implicit-def $scc
-  ; CHECK-NEXT:   dead renamable $sgpr4 = S_LSHL_B32 killed renamable $sgpr66, 1, implicit-def dead $scc
-  ; CHECK-NEXT:   dead [[COPY2:%[0-9]+]]:vreg_1024 = COPY renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75
+  ; CHECK-NEXT:   dead renamable $sgpr4 = S_LSHL_B32 killed renamable $sgpr64, 1, implicit-def dead $scc
+  ; CHECK-NEXT:   renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67 = SI_SPILL_S1024_RESTORE %stack.0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.0, align 4, addrspace 5)
+  ; CHECK-NEXT:   dead [[COPY2:%[0-9]+]]:vreg_1024 = COPY killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.5:
   bb.0:
@@ -211,7 +214,6 @@ body:             |
     %15.sub19:sgpr_1024 = COPY %7.sub0
     %15.sub20:sgpr_1024 = COPY %7.sub0
     %15.sub21:sgpr_1024 = COPY %7.sub0
-    ; Spill code ends up getting inserted here, and we end up with many unspillable sgpr1024 ranges
     %16:vreg_1024 = COPY %15, implicit $exec
     $exec = S_XOR_B64_term $exec, %14, implicit-def $scc
     S_CBRANCH_EXECZ %bb.5, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/greedy-reverse-local-assignment.ll b/llvm/test/CodeGen/AMDGPU/greedy-reverse-local-assignment.ll
index 6c921441c972d..6e82a294243d2 100644
--- a/llvm/test/CodeGen/AMDGPU/greedy-reverse-local-assignment.ll
+++ b/llvm/test/CodeGen/AMDGPU/greedy-reverse-local-assignment.ll
@@ -48,6 +48,6 @@ define <4 x half> @shuffle_v4f16_234u(ptr addrspace(1) %arg0, ptr addrspace(1) %
 ; NOXNACK-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x half>, ptr addrspace(1) %arg0
   %val1 = load <4 x half>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 4, i32 undef>
+  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 4, i32 poison>
   ret <4 x half> %shuffle
 }
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll
index 28246d7f9e6fb..f0c3a493d05f1 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll
@@ -14,7 +14,8 @@
 %struct.B = type { ptr addrspace(1) }
 %opencl.clk_event_t = type opaque
 
-@__test_block_invoke_kernel_runtime_handle = external addrspace(1) externally_initialized constant ptr addrspace(1)
+@__test_block_invoke_kernel_runtime_handle = external addrspace(1) externally_initialized constant ptr addrspace(1), section ".amdgpu.kernel.runtime.handle"
+@not.a.handle = external addrspace(1) externally_initialized constant ptr addrspace(1)
 
 ; CHECK:              ---
 ; CHECK-NEXT: amdhsa.kernels:
@@ -1678,7 +1679,7 @@ define amdgpu_kernel void @test_pointee_align_attribute(ptr addrspace(1) align 1
 ; CHECK:          .name:           __test_block_invoke_kernel
 ; CHECK:          .symbol:         __test_block_invoke_kernel.kd
 define amdgpu_kernel void @__test_block_invoke_kernel(
-    <{ i32, i32, ptr, ptr addrspace(1), i8 }> %arg) #1
+    <{ i32, i32, ptr, ptr addrspace(1), i8 }> %arg) #1 !associated !112
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !110
     !kernel_arg_base_type !110 !kernel_arg_type_qual !4 {
   ret void
@@ -1734,6 +1735,29 @@ define amdgpu_kernel void @unknown_addrspace_kernarg(ptr addrspace(12345) %ptr)
   ret void
 }
 
+; Make sure the device_enqueue_symbol is not reported
+; CHECK: - .args:           []
+; CHECK-NEXT: .group_segment_fixed_size: 0
+; CHECK-NEXT: .kernarg_segment_align: 4
+; CHECK-NEXT: .kernarg_segment_size: 0
+; CHECK-NEXT: .language:       OpenCL C
+; CHECK-NEXT: .language_version:
+; CHECK-NEXT: - 2
+; CHECK-NEXT: - 0
+; CHECK-NEXT: .max_flat_workgroup_size: 1024
+; CHECK-NEXT: .name:           associated_global_not_handle
+; CHECK-NEXT: .private_segment_fixed_size: 0
+; CHECK-NEXT: .sgpr_count:
+; CHECK-NEXT: .sgpr_spill_count: 0
+; CHECK-NEXT: .symbol:         associated_global_not_handle.kd
+; CHECK-NEXT: .vgpr_count:
+; CHECK-NEXT: .vgpr_spill_count: 0
+; CHECK-NEXT: .wavefront_size: 64
+; CHECK-NOT: device_enqueue_symbol
+define amdgpu_kernel void @associated_global_not_handle() #3 !associated !113 {
+  ret void
+}
+
 ; CHECK:  amdhsa.printf:
 ; CHECK-NEXT: - '1:1:4:%d\n'
 ; CHECK-NEXT: - '2:1:8:%g\n'
@@ -1744,6 +1768,7 @@ define amdgpu_kernel void @unknown_addrspace_kernarg(ptr addrspace(12345) %ptr)
 attributes #0 = { optnone noinline "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-implicitarg-num-bytes"="56" }
 attributes #1 = { optnone noinline "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-implicitarg-num-bytes"="56" "runtime-handle"="__test_block_invoke_kernel_runtime_handle" }
 attributes #2 = { optnone noinline "amdgpu-implicitarg-num-bytes"="56" }
+attributes #3 = { optnone noinline "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" }
 
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"amdhsa_code_object_version", i32 400}
@@ -1803,5 +1828,7 @@ attributes #2 = { optnone noinline "amdgpu-implicitarg-num-bytes"="56" }
 !101 = !{!"2:1:8:%g\5Cn"}
 !110 = !{!"__block_literal"}
 !111 = !{!"char", !"char"}
+!112 = !{ptr addrspace(1) @__test_block_invoke_kernel_runtime_handle }
+!113 = !{ptr addrspace(1) @not.a.handle }
 
 ; PARSER: AMDGPU HSA Metadata Parser Test: PASS
diff --git a/llvm/test/CodeGen/AMDGPU/i1-copy-phi.ll b/llvm/test/CodeGen/AMDGPU/i1-copy-phi.ll
index 80aa6ee0ab103..2c062bbd81867 100644
--- a/llvm/test/CodeGen/AMDGPU/i1-copy-phi.ll
+++ b/llvm/test/CodeGen/AMDGPU/i1-copy-phi.ll
@@ -52,7 +52,7 @@ false:
   br label %exit
 
 exit:
-  %c = phi <2 x i1> [ undef, %entry ], [ %cmp, %false ]
+  %c = phi <2 x i1> [ poison, %entry ], [ %cmp, %false ]
   %ret = select <2 x i1> %c, <2 x float> <float 2.0, float 2.0>, <2 x float> <float 4.0, float 4.0>
   ret <2 x float> %ret
 }
diff --git a/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll b/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll
new file mode 100644
index 0000000000000..59b0ba2469a20
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll
@@ -0,0 +1,2109 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=amdgcn -mcpu=hawaii | FileCheck %s -check-prefix=GFX7
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx942 | FileCheck %s -check-prefix=GFX9
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 | FileCheck %s -check-prefix=GFX11
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 | FileCheck %s -check-prefix=GFX12
+
+define bfloat @v_uitofp_i1_to_bf16(i1 %num) {
+; GFX7-LABEL: v_uitofp_i1_to_bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc
+; GFX7-NEXT:    v_and_b32_e32 v0, 0x7fff0000, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uitofp_i1_to_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX9-NEXT:    s_movk_i32 s0, 0x7fff
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT:    v_add3_u32 v1, v1, v0, s0
+; GFX9-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_uitofp_i1_to_bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
+; GFX11-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v_uitofp_i1_to_bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
+; GFX12-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %op = uitofp i1 %num to bfloat
+  ret bfloat %op
+}
+
+define amdgpu_ps i32 @s_uitofp_i1_to_bf16(i1 inreg %num) {
+; GFX7-LABEL: s_uitofp_i1_to_bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s[0:1]
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX7-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uitofp_i1_to_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s[0:1]
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    s_bfe_u32 s1, s0, 0x10010
+; GFX9-NEXT:    s_or_b32 s2, s0, 0x400000
+; GFX9-NEXT:    s_add_i32 s0, s1, s0
+; GFX9-NEXT:    s_add_i32 s3, s0, 0x7fff
+; GFX9-NEXT:    s_and_b64 s[0:1], vcc, exec
+; GFX9-NEXT:    s_cselect_b32 s0, s2, s3
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: s_uitofp_i1_to_bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s0
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_bfe_u32 s1, s0, 0x10010
+; GFX11-NEXT:    s_add_i32 s1, s1, s0
+; GFX11-NEXT:    s_bitset1_b32 s0, 22
+; GFX11-NEXT:    s_addk_i32 s1, 0x7fff
+; GFX11-NEXT:    s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-NEXT:    s_cselect_b32 s0, s0, s1
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX11-NEXT:    ; return to shader part epilog
+;
+; GFX12-LABEL: s_uitofp_i1_to_bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s0
+; GFX12-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT:    s_or_b32 s2, s0, 0x400000
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_add_co_i32 s1, s1, s0
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT:    s_cmp_u_f32 s0, s0
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_cselect_b32 s0, s2, s1
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    ; return to shader part epilog
+  %op = uitofp i1 %num to bfloat
+  %b16 = bitcast bfloat %op to i16
+  %b32 = zext i16 %b16 to i32
+  ret i32 %b32
+}
+
+define <2 x bfloat> @v_uitofp_v2i1_to_v2bf16(<2 x i1> %num) {
+; GFX7-LABEL: v_uitofp_v2i1_to_v2bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc
+; GFX7-NEXT:    v_and_b32_e32 v0, 0x7fff0000, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, 0x7fff0000, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uitofp_v2i1_to_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX9-NEXT:    s_movk_i32 s0, 0x7fff
+; GFX9-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT:    v_add3_u32 v2, v2, v0, s0
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT:    v_add3_u32 v2, v2, v1, s0
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    s_mov_b32 s0, 0x7060302
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT:    v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_uitofp_v2i1_to_v2bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
+; GFX11-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-NEXT:    v_perm_b32 v0, v1, v0, 0x7060302
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v_uitofp_v2i1_to_v2bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
+; GFX12-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX12-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-NEXT:    v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX12-NEXT:    v_perm_b32 v0, v1, v0, 0x7060302
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %op = uitofp <2 x i1> %num to <2 x bfloat>
+  ret <2 x bfloat> %op
+}
+
+define amdgpu_ps <2 x i32> @s_uitofp_v2i1_to_v2bf16(<2 x i1> inreg %num) {
+; GFX7-LABEL: s_uitofp_v2i1_to_v2bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_and_b32 s2, 1, s0
+; GFX7-NEXT:    s_bitcmp1_b32 s1, 0
+; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX7-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s[0:1]
+; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s[0:1]
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX7-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uitofp_v2i1_to_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s2, 1, s0
+; GFX9-NEXT:    s_bitcmp1_b32 s1, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s[0:1]
+; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s[0:1]
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX9-NEXT:    s_bfe_u32 s1, s0, 0x10010
+; GFX9-NEXT:    s_or_b32 s2, s0, 0x400000
+; GFX9-NEXT:    s_add_i32 s0, s1, s0
+; GFX9-NEXT:    s_add_i32 s3, s0, 0x7fff
+; GFX9-NEXT:    s_and_b64 s[0:1], vcc, exec
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX9-NEXT:    s_cselect_b32 s0, s2, s3
+; GFX9-NEXT:    s_bfe_u32 s2, s1, 0x10010
+; GFX9-NEXT:    s_or_b32 s4, s1, 0x400000
+; GFX9-NEXT:    s_add_i32 s1, s2, s1
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX9-NEXT:    s_addk_i32 s1, 0x7fff
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    s_and_b64 s[2:3], vcc, exec
+; GFX9-NEXT:    s_cselect_b32 s1, s4, s1
+; GFX9-NEXT:    s_lshr_b32 s1, s1, 16
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: s_uitofp_v2i1_to_v2bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_and_b32 s0, 1, s0
+; GFX11-NEXT:    s_bitcmp1_b32 s1, 0
+; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s0, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s1
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s0
+; GFX11-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX11-NEXT:    v_cmp_u_f32_e64 s1, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    s_bfe_u32 s3, s0, 0x10010
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_i32 s3, s3, s0
+; GFX11-NEXT:    s_bitset1_b32 s0, 22
+; GFX11-NEXT:    s_addk_i32 s3, 0x7fff
+; GFX11-NEXT:    s_and_b32 s4, vcc_lo, exec_lo
+; GFX11-NEXT:    s_cselect_b32 s0, s0, s3
+; GFX11-NEXT:    s_bfe_u32 s3, s2, 0x10010
+; GFX11-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX11-NEXT:    s_add_i32 s3, s3, s2
+; GFX11-NEXT:    s_bitset1_b32 s2, 22
+; GFX11-NEXT:    s_addk_i32 s3, 0x7fff
+; GFX11-NEXT:    s_and_b32 s1, s1, exec_lo
+; GFX11-NEXT:    s_cselect_b32 s1, s2, s3
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_lshr_b32 s1, s1, 16
+; GFX11-NEXT:    ; return to shader part epilog
+;
+; GFX12-LABEL: s_uitofp_v2i1_to_v2bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_and_b32 s0, 1, s0
+; GFX12-NEXT:    s_bitcmp1_b32 s1, 0
+; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s0, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s1
+; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX12-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT:    s_or_b32 s3, s0, 0x400000
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_add_co_i32 s1, s1, s0
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT:    s_cmp_u_f32 s0, s0
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_cselect_b32 s0, s3, s1
+; GFX12-NEXT:    s_bfe_u32 s1, s2, 0x10010
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX12-NEXT:    s_add_co_i32 s1, s1, s2
+; GFX12-NEXT:    s_or_b32 s3, s2, 0x400000
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT:    s_cmp_u_f32 s2, s2
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_lshr_b32 s1, s1, 16
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    ; return to shader part epilog
+  %op = uitofp <2 x i1> %num to <2 x bfloat>
+  %b16 = bitcast <2 x bfloat> %op to <2 x i16>
+  %b32 = zext <2 x i16> %b16 to <2 x i32>
+  ret <2 x i32> %b32
+}
+
+define <3 x bfloat> @v_uitofp_v3i1_to_v3bf16(<3 x i1> %num) {
+; GFX7-LABEL: v_uitofp_v3i1_to_v3bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX7-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc
+; GFX7-NEXT:    v_and_b32_e32 v0, 0x7fff0000, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, 0x7fff0000, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, 0x7fff0000, v2
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uitofp_v3i1_to_v3bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
+; GFX9-NEXT:    s_movk_i32 s0, 0x7fff
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT:    v_add3_u32 v3, v3, v2, s0
+; GFX9-NEXT:    v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT:    v_add3_u32 v3, v3, v0, s0
+; GFX9-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT:    v_add3_u32 v3, v3, v1, s0
+; GFX9-NEXT:    v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    s_mov_b32 s0, 0x7060302
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT:    v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT:    v_alignbit_b32 v1, s0, v2, 16
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_uitofp_v3i1_to_v3bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v2
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_bfe_u32 v4, v2, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
+; GFX11-NEXT:    v_or_b32_e32 v6, 0x400000, v2
+; GFX11-NEXT:    v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v7, 0x400000, v0
+; GFX11-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    v_bfe_u32 v5, v1, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v1
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add3_u32 v5, v5, v1, 0x7fff
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v8, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_perm_b32 v0, v1, v0, 0x7060302
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-NEXT:    v_alignbit_b32 v1, s0, v2, 16
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v_uitofp_v3i1_to_v3bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX12-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, vcc_lo
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_bfe_u32 v4, v2, 16, 1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
+; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v2
+; GFX12-NEXT:    v_add3_u32 v4, v4, v2, 0x7fff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v7, 0x400000, v0
+; GFX12-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v1
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-NEXT:    v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v5, v8, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-NEXT:    v_perm_b32 v0, v1, v0, 0x7060302
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_alignbit_b32 v1, s0, v2, 16
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %op = uitofp <3 x i1> %num to <3 x bfloat>
+  ret <3 x bfloat> %op
+}
+
+define amdgpu_ps <3 x i32> @s_uitofp_v3i1_to_v3bf16(<3 x i1> inreg %num) {
+; GFX7-LABEL: s_uitofp_v3i1_to_v3bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_and_b32 s4, 1, s0
+; GFX7-NEXT:    s_and_b32 s3, 1, s1
+; GFX7-NEXT:    s_bitcmp1_b32 s2, 0
+; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX7-NEXT:    s_cmp_eq_u32 s3, 1
+; GFX7-NEXT:    s_cselect_b64 s[2:3], -1, 0
+; GFX7-NEXT:    s_cmp_eq_u32 s4, 1
+; GFX7-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, s[0:1]
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX7-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uitofp_v3i1_to_v3bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s4, 1, s0
+; GFX9-NEXT:    s_and_b32 s3, 1, s1
+; GFX9-NEXT:    s_bitcmp1_b32 s2, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s3, 1
+; GFX9-NEXT:    s_cselect_b64 s[2:3], -1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s4, 1
+; GFX9-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s[4:5]
+; GFX9-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v0
+; GFX9-NEXT:    v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT:    v_add_u32_e32 v2, 0x7fff, v2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s[2:3]
+; GFX9-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v1
+; GFX9-NEXT:    v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT:    v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, s[0:1]
+; GFX9-NEXT:    v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v4, v4, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v2
+; GFX9-NEXT:    v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v3, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: s_uitofp_v3i1_to_v3bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_and_b32 s0, 1, s0
+; GFX11-NEXT:    s_and_b32 s1, 1, s1
+; GFX11-NEXT:    s_bitcmp1_b32 s2, 0
+; GFX11-NEXT:    s_cselect_b32 s2, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, s2
+; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s0, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s1
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX11-NEXT:    v_bfe_u32 v5, v2, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s0
+; GFX11-NEXT:    v_or_b32_e32 v6, 0x400000, v2
+; GFX11-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v7, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, v5, v2
+; GFX11-NEXT:    v_bfe_u32 v4, v1, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v0
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, v4, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 0x7fff, v4
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v8, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX11-NEXT:    ; return to shader part epilog
+;
+; GFX12-LABEL: s_uitofp_v3i1_to_v3bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_and_b32 s0, 1, s0
+; GFX12-NEXT:    s_and_b32 s1, 1, s1
+; GFX12-NEXT:    s_bitcmp1_b32 s2, 0
+; GFX12-NEXT:    s_cselect_b32 s2, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, s2
+; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s0, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s1
+; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX12-NEXT:    v_bfe_u32 v5, v2, 16, 1
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s0
+; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v2
+; GFX12-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v7, 0x400000, v0
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    v_add_nc_u32_e32 v5, v5, v2
+; GFX12-NEXT:    v_bfe_u32 v4, v1, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX12-NEXT:    v_add_nc_u32_e32 v3, v3, v0
+; GFX12-NEXT:    v_add_nc_u32_e32 v4, v4, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX12-NEXT:    v_add_nc_u32_e32 v4, 0x7fff, v4
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v4, v8, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX12-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    ; return to shader part epilog
+  %op = uitofp <3 x i1> %num to <3 x bfloat>
+  %b16 = bitcast <3 x bfloat> %op to <3 x i16>
+  %b32 = zext <3 x i16> %b16 to <3 x i32>
+  ret <3 x i32> %b32
+}
+
+define <4 x bfloat> @v_uitofp_v4i1_to_v4bf16(<4 x i1> %num) {
+; GFX7-LABEL: v_uitofp_v4i1_to_v4bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v3, 1, v3
+; GFX7-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v3
+; GFX7-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, 0, 1.0, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc
+; GFX7-NEXT:    v_and_b32_e32 v0, 0x7fff0000, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, 0x7fff0000, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, 0x7fff0000, v2
+; GFX7-NEXT:    v_and_b32_e32 v3, 0x7fff0000, v3
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uitofp_v4i1_to_v4bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
+; GFX9-NEXT:    s_movk_i32 s0, 0x7fff
+; GFX9-NEXT:    v_and_b32_e32 v3, 1, v3
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT:    v_add3_u32 v4, v4, v2, s0
+; GFX9-NEXT:    v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v5, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v3
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, 1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT:    v_add3_u32 v4, v4, v3, s0
+; GFX9-NEXT:    v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT:    v_add3_u32 v4, v4, v0, s0
+; GFX9-NEXT:    v_or_b32_e32 v5, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT:    v_add3_u32 v4, v4, v1, s0
+; GFX9-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    s_mov_b32 s0, 0x7060302
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-NEXT:    v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT:    v_perm_b32 v1, v3, v2, s0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_uitofp_v4i1_to_v4bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX11-NEXT:    v_and_b32_e32 v3, 1, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v2
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_bfe_u32 v4, v2, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0, 1.0, vcc_lo
+; GFX11-NEXT:    v_or_b32_e32 v7, 0x400000, v2
+; GFX11-NEXT:    v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_bfe_u32 v5, v3, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v9, 0x400000, v3
+; GFX11-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX11-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-NEXT:    v_bfe_u32 v6, v0, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-NEXT:    v_bfe_u32 v8, v1, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_add3_u32 v7, v8, v1, 0x7fff
+; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v1
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v6, v4, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v7, v8, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT:    v_perm_b32 v0, v1, v0, 0x7060302
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, v5, v9, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_perm_b32 v1, v3, v2, 0x7060302
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v_uitofp_v4i1_to_v4bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX12-NEXT:    v_and_b32_e32 v3, 1, v3
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, vcc_lo
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v3
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_bfe_u32 v4, v2, 16, 1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v3, 0, 1.0, vcc_lo
+; GFX12-NEXT:    v_or_b32_e32 v7, 0x400000, v2
+; GFX12-NEXT:    v_add3_u32 v4, v4, v2, 0x7fff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_bfe_u32 v5, v3, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
+; GFX12-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX12-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX12-NEXT:    v_bfe_u32 v6, v0, 16, 1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_add3_u32 v6, v6, v0, 0x7fff
+; GFX12-NEXT:    v_bfe_u32 v8, v1, 16, 1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    v_add3_u32 v7, v8, v1, 0x7fff
+; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v6, v4, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v7, v8, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_perm_b32 v0, v1, v0, 0x7060302
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v3, v5, v9, vcc_lo
+; GFX12-NEXT:    v_perm_b32 v1, v3, v2, 0x7060302
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %op = uitofp <4 x i1> %num to <4 x bfloat>
+  ret <4 x bfloat> %op
+}
+
+define amdgpu_ps <4 x i32> @s_uitofp_v4i1_to_v4bf16(<4 x i1> inreg %num) {
+; GFX7-LABEL: s_uitofp_v4i1_to_v4bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_and_b32 s6, 1, s0
+; GFX7-NEXT:    s_and_b32 s4, 1, s1
+; GFX7-NEXT:    s_and_b32 s2, 1, s2
+; GFX7-NEXT:    s_bitcmp1_b32 s3, 0
+; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX7-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX7-NEXT:    s_cselect_b64 s[2:3], -1, 0
+; GFX7-NEXT:    s_cmp_eq_u32 s4, 1
+; GFX7-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; GFX7-NEXT:    s_cmp_eq_u32 s6, 1
+; GFX7-NEXT:    s_cselect_b64 s[6:7], -1, 0
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, s[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, 0, 1.0, s[0:1]
+; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX7-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uitofp_v4i1_to_v4bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s6, 1, s0
+; GFX9-NEXT:    s_and_b32 s4, 1, s1
+; GFX9-NEXT:    s_and_b32 s2, 1, s2
+; GFX9-NEXT:    s_bitcmp1_b32 s3, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX9-NEXT:    s_cselect_b64 s[2:3], -1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s4, 1
+; GFX9-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s6, 1
+; GFX9-NEXT:    s_cselect_b64 s[6:7], -1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s[6:7]
+; GFX9-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v0
+; GFX9-NEXT:    v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT:    v_add_u32_e32 v2, 0x7fff, v2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s[4:5]
+; GFX9-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v1
+; GFX9-NEXT:    v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT:    v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1.0, s[2:3]
+; GFX9-NEXT:    v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v4, v4, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v2
+; GFX9-NEXT:    v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, 1.0, s[0:1]
+; GFX9-NEXT:    v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v5, v5, v3
+; GFX9-NEXT:    v_or_b32_e32 v4, 0x400000, v3
+; GFX9-NEXT:    v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v4, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: s_uitofp_v4i1_to_v4bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_and_b32 s0, 1, s0
+; GFX11-NEXT:    s_and_b32 s1, 1, s1
+; GFX11-NEXT:    s_and_b32 s2, 1, s2
+; GFX11-NEXT:    s_bitcmp1_b32 s3, 0
+; GFX11-NEXT:    s_cselect_b32 s3, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s3
+; GFX11-NEXT:    s_cselect_b32 s2, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, 0, 1.0, s2
+; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s0, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0, 1.0, s1
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX11-NEXT:    v_bfe_u32 v7, v4, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s0
+; GFX11-NEXT:    v_bfe_u32 v5, v1, 16, 1
+; GFX11-NEXT:    v_bfe_u32 v6, v3, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v9, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, v7, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, v5, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, v6, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 0x7fff, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, v2, v0
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v9, vcc_lo
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v6
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT:    v_or_b32_e32 v6, 0x400000, v4
+; GFX11-NEXT:    v_or_b32_e32 v9, 0x400000, v1
+; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, v7, v6, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX11-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v9, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_readfirstlane_b32 s2, v3
+; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX11-NEXT:    ; return to shader part epilog
+;
+; GFX12-LABEL: s_uitofp_v4i1_to_v4bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_and_b32 s0, 1, s0
+; GFX12-NEXT:    s_and_b32 s1, 1, s1
+; GFX12-NEXT:    s_and_b32 s2, 1, s2
+; GFX12-NEXT:    s_bitcmp1_b32 s3, 0
+; GFX12-NEXT:    s_cselect_b32 s3, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, 1.0, s3
+; GFX12-NEXT:    s_cselect_b32 s2, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    v_cndmask_b32_e64 v4, 0, 1.0, s2
+; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s0, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v3, 0, 1.0, s1
+; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX12-NEXT:    v_bfe_u32 v7, v4, 16, 1
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s0
+; GFX12-NEXT:    v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT:    v_bfe_u32 v6, v3, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v3
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v0
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    v_add_nc_u32_e32 v7, v7, v4
+; GFX12-NEXT:    v_add_nc_u32_e32 v5, v5, v1
+; GFX12-NEXT:    v_add_nc_u32_e32 v6, v6, v3
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-NEXT:    v_add_nc_u32_e32 v7, 0x7fff, v7
+; GFX12-NEXT:    v_add_nc_u32_e32 v2, v2, v0
+; GFX12-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v9, vcc_lo
+; GFX12-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v6
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v4
+; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX12-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v3, v7, v6, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX12-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v5, v9, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_readfirstlane_b32 s2, v3
+; GFX12-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    ; return to shader part epilog
+  %op = uitofp <4 x i1> %num to <4 x bfloat>
+  %b16 = bitcast <4 x bfloat> %op to <4 x i16>
+  %b32 = zext <4 x i16> %b16 to <4 x i32>
+  ret <4 x i32> %b32
+}
+
+define bfloat @v_sitofp_i1_to_bf16(i1 %num) {
+; GFX7-LABEL: v_sitofp_i1_to_bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_sitofp_i1_to_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX9-NEXT:    s_movk_i32 s0, 0x7fff
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT:    v_add3_u32 v1, v1, v0, s0
+; GFX9-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_sitofp_i1_to_bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
+; GFX11-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v_sitofp_i1_to_bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
+; GFX12-NEXT:    v_bfe_u32 v1, v0, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v2, 0x400000, v0
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %op = sitofp i1 %num to bfloat
+  ret bfloat %op
+}
+
+define amdgpu_ps i32 @s_sitofp_i1_to_bf16(i1 inreg %num) {
+; GFX7-LABEL: s_sitofp_i1_to_bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s[0:1]
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    s_ashr_i32 s0, s0, 16
+; GFX7-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_sitofp_i1_to_bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s[0:1]
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    s_bfe_u32 s1, s0, 0x10010
+; GFX9-NEXT:    s_or_b32 s2, s0, 0x400000
+; GFX9-NEXT:    s_add_i32 s0, s1, s0
+; GFX9-NEXT:    s_add_i32 s3, s0, 0x7fff
+; GFX9-NEXT:    s_and_b64 s[0:1], vcc, exec
+; GFX9-NEXT:    s_cselect_b32 s0, s2, s3
+; GFX9-NEXT:    s_ashr_i32 s0, s0, 16
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: s_sitofp_i1_to_bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s0
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_bfe_u32 s1, s0, 0x10010
+; GFX11-NEXT:    s_add_i32 s1, s1, s0
+; GFX11-NEXT:    s_bitset1_b32 s0, 22
+; GFX11-NEXT:    s_addk_i32 s1, 0x7fff
+; GFX11-NEXT:    s_and_b32 s2, vcc_lo, exec_lo
+; GFX11-NEXT:    s_cselect_b32 s0, s0, s1
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_ashr_i32 s0, s0, 16
+; GFX11-NEXT:    ; return to shader part epilog
+;
+; GFX12-LABEL: s_sitofp_i1_to_bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s0
+; GFX12-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT:    s_or_b32 s2, s0, 0x400000
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_add_co_i32 s1, s1, s0
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT:    s_cmp_u_f32 s0, s0
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_cselect_b32 s0, s2, s1
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_ashr_i32 s0, s0, 16
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    ; return to shader part epilog
+  %op = sitofp i1 %num to bfloat
+  %b16 = bitcast bfloat %op to i16
+  %b32 = sext i16 %b16 to i32
+  ret i32 %b32
+}
+
+define <2 x bfloat> @v_sitofp_v2i1_to_v2bf16(<2 x i1> %num) {
+; GFX7-LABEL: v_sitofp_v2i1_to_v2bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX7-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_sitofp_v2i1_to_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX9-NEXT:    s_movk_i32 s0, 0x7fff
+; GFX9-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT:    v_add3_u32 v2, v2, v0, s0
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v2, v1, 16, 1
+; GFX9-NEXT:    v_add3_u32 v2, v2, v1, s0
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    s_mov_b32 s0, 0x7060302
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GFX9-NEXT:    v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_sitofp_v2i1_to_v2bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
+; GFX11-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-NEXT:    v_perm_b32 v0, v1, v0, 0x7060302
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v_sitofp_v2i1_to_v2bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
+; GFX12-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX12-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-NEXT:    v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX12-NEXT:    v_perm_b32 v0, v1, v0, 0x7060302
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %op = sitofp <2 x i1> %num to <2 x bfloat>
+  ret <2 x bfloat> %op
+}
+
+define amdgpu_ps <2 x i32> @s_sitofp_v2i1_to_v2bf16(<2 x i1> inreg %num) {
+; GFX7-LABEL: s_sitofp_v2i1_to_v2bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_and_b32 s2, 1, s1
+; GFX7-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX7-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s[0:1]
+; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s[0:1]
+; GFX7-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX7-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_sitofp_v2i1_to_v2bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s2, 1, s1
+; GFX9-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s[0:1]
+; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s[0:1]
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX9-NEXT:    s_bfe_u32 s1, s0, 0x10010
+; GFX9-NEXT:    s_or_b32 s2, s0, 0x400000
+; GFX9-NEXT:    s_add_i32 s0, s1, s0
+; GFX9-NEXT:    s_add_i32 s3, s0, 0x7fff
+; GFX9-NEXT:    s_and_b64 s[0:1], vcc, exec
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX9-NEXT:    s_bfe_u32 s1, s0, 0x10010
+; GFX9-NEXT:    s_or_b32 s3, s0, 0x400000
+; GFX9-NEXT:    s_add_i32 s0, s1, s0
+; GFX9-NEXT:    s_add_i32 s4, s0, 0x7fff
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    s_and_b64 s[0:1], vcc, exec
+; GFX9-NEXT:    s_cselect_b32 s0, s3, s4
+; GFX9-NEXT:    s_ashr_i32 s0, s0, 16
+; GFX9-NEXT:    s_ashr_i32 s1, s2, 16
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: s_sitofp_v2i1_to_v2bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_and_b32 s1, 1, s1
+; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s0
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s0
+; GFX11-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX11-NEXT:    v_cmp_u_f32_e64 s0, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    s_bfe_u32 s3, s1, 0x10010
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_i32 s3, s3, s1
+; GFX11-NEXT:    s_bitset1_b32 s1, 22
+; GFX11-NEXT:    s_addk_i32 s3, 0x7fff
+; GFX11-NEXT:    s_and_b32 s4, vcc_lo, exec_lo
+; GFX11-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX11-NEXT:    s_bfe_u32 s3, s2, 0x10010
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_i32 s3, s3, s2
+; GFX11-NEXT:    s_bitset1_b32 s2, 22
+; GFX11-NEXT:    s_addk_i32 s3, 0x7fff
+; GFX11-NEXT:    s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT:    s_cselect_b32 s0, s2, s3
+; GFX11-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX11-NEXT:    s_ashr_i32 s0, s0, 16
+; GFX11-NEXT:    ; return to shader part epilog
+;
+; GFX12-LABEL: s_sitofp_v2i1_to_v2bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_and_b32 s1, 1, s1
+; GFX12-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s0
+; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX12-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    s_bfe_u32 s1, s0, 0x10010
+; GFX12-NEXT:    s_or_b32 s3, s0, 0x400000
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_add_co_i32 s1, s1, s0
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_addk_co_i32 s1, 0x7fff
+; GFX12-NEXT:    s_cmp_u_f32 s0, s0
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX12-NEXT:    s_bfe_u32 s0, s2, 0x10010
+; GFX12-NEXT:    s_or_b32 s3, s2, 0x400000
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_add_co_i32 s0, s0, s2
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_addk_co_i32 s0, 0x7fff
+; GFX12-NEXT:    s_cmp_u_f32 s2, s2
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_cselect_b32 s0, s3, s0
+; GFX12-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_ashr_i32 s0, s0, 16
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    ; return to shader part epilog
+  %op = sitofp <2 x i1> %num to <2 x bfloat>
+  %b16 = bitcast <2 x bfloat> %op to <2 x i16>
+  %b32 = sext <2 x i16> %b16 to <2 x i32>
+  ret <2 x i32> %b32
+}
+
+define <3 x bfloat> @v_sitofp_v3i1_to_v3bf16(<3 x i1> %num) {
+; GFX7-LABEL: v_sitofp_v3i1_to_v3bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX7-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_sitofp_v3i1_to_v3bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
+; GFX9-NEXT:    s_movk_i32 s0, 0x7fff
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v3, v2, 16, 1
+; GFX9-NEXT:    v_add3_u32 v3, v3, v2, s0
+; GFX9-NEXT:    v_or_b32_e32 v4, 0x400000, v2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT:    v_add3_u32 v3, v3, v0, s0
+; GFX9-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT:    v_add3_u32 v3, v3, v1, s0
+; GFX9-NEXT:    v_or_b32_e32 v4, 0x400000, v1
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    s_mov_b32 s0, 0x7060302
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
+; GFX9-NEXT:    v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT:    v_alignbit_b32 v1, s0, v2, 16
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_sitofp_v3i1_to_v3bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v2
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_bfe_u32 v4, v2, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
+; GFX11-NEXT:    v_or_b32_e32 v6, 0x400000, v2
+; GFX11-NEXT:    v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v7, 0x400000, v0
+; GFX11-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    v_bfe_u32 v5, v1, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v1
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add3_u32 v5, v5, v1, 0x7fff
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v5, v8, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_perm_b32 v0, v1, v0, 0x7060302
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-NEXT:    v_alignbit_b32 v1, s0, v2, 16
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v_sitofp_v3i1_to_v3bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX12-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, vcc_lo
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_bfe_u32 v4, v2, 16, 1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
+; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v2
+; GFX12-NEXT:    v_add3_u32 v4, v4, v2, 0x7fff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v7, 0x400000, v0
+; GFX12-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v1
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-NEXT:    v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v5, v8, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-NEXT:    v_perm_b32 v0, v1, v0, 0x7060302
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_alignbit_b32 v1, s0, v2, 16
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %op = sitofp <3 x i1> %num to <3 x bfloat>
+  ret <3 x bfloat> %op
+}
+
+define amdgpu_ps <3 x i32> @s_sitofp_v3i1_to_v3bf16(<3 x i1> inreg %num) {
+; GFX7-LABEL: s_sitofp_v3i1_to_v3bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_and_b32 s4, 1, s2
+; GFX7-NEXT:    s_and_b32 s2, 1, s1
+; GFX7-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX7-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX7-NEXT:    s_cselect_b64 s[2:3], -1, 0
+; GFX7-NEXT:    s_cmp_eq_u32 s4, 1
+; GFX7-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, s[0:1]
+; GFX7-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
+; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX7-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX7-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_sitofp_v3i1_to_v3bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s4, 1, s2
+; GFX9-NEXT:    s_and_b32 s2, 1, s1
+; GFX9-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX9-NEXT:    s_cselect_b64 s[2:3], -1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s4, 1
+; GFX9-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s[4:5]
+; GFX9-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v0
+; GFX9-NEXT:    v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT:    v_add_u32_e32 v2, 0x7fff, v2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s[2:3]
+; GFX9-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v1
+; GFX9-NEXT:    v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT:    v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, s[0:1]
+; GFX9-NEXT:    v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v4, v4, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v2
+; GFX9-NEXT:    v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v3, vcc
+; GFX9-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: s_sitofp_v3i1_to_v3bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_and_b32 s2, 1, s2
+; GFX11-NEXT:    s_and_b32 s1, 1, s1
+; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s0
+; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s1
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX11-NEXT:    v_bfe_u32 v4, v1, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, s0
+; GFX11-NEXT:    v_or_b32_e32 v6, 0x400000, v1
+; GFX11-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v7, 0x400000, v0
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, v4, v1
+; GFX11-NEXT:    v_bfe_u32 v5, v2, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v2
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, v3, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    v_add_nc_u32_e32 v4, 0x7fff, v4
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-NEXT:    v_dual_cndmask_b32 v0, v3, v7 :: v_dual_add_nc_u32 v5, v5, v2
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX11-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v4, v6, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX11-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, v5, v8, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX11-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX11-NEXT:    ; return to shader part epilog
+;
+; GFX12-LABEL: s_sitofp_v3i1_to_v3bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_and_b32 s2, 1, s2
+; GFX12-NEXT:    s_and_b32 s1, 1, s1
+; GFX12-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s0
+; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s1
+; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX12-NEXT:    v_bfe_u32 v4, v1, 16, 1
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, s0
+; GFX12-NEXT:    v_or_b32_e32 v6, 0x400000, v1
+; GFX12-NEXT:    v_bfe_u32 v3, v0, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v7, 0x400000, v0
+; GFX12-NEXT:    v_add_nc_u32_e32 v4, v4, v1
+; GFX12-NEXT:    v_bfe_u32 v5, v2, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v2
+; GFX12-NEXT:    v_add_nc_u32_e32 v3, v3, v0
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    v_add_nc_u32_e32 v4, 0x7fff, v4
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX12-NEXT:    v_dual_cndmask_b32 v0, v3, v7 :: v_dual_add_nc_u32 v5, v5, v2
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX12-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v4, v6, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_readfirstlane_b32 s1, v0
+; GFX12-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v2, v5, v8, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX12-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    ; return to shader part epilog
+  %op = sitofp <3 x i1> %num to <3 x bfloat>
+  %b16 = bitcast <3 x bfloat> %op to <3 x i16>
+  %b32 = sext <3 x i16> %b16 to <3 x i32>
+  ret <3 x i32> %b32
+}
+
+define <4 x bfloat> @v_sitofp_v4i1_to_v4bf16(<4 x i1> %num) {
+; GFX7-LABEL: v_sitofp_v4i1_to_v4bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v3, 1, v3
+; GFX7-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v3
+; GFX7-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, 0, -1.0, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
+; GFX7-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, vcc
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
+; GFX7-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_sitofp_v4i1_to_v4bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
+; GFX9-NEXT:    s_movk_i32 s0, 0x7fff
+; GFX9-NEXT:    v_and_b32_e32 v3, 1, v3
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT:    v_add3_u32 v4, v4, v2, s0
+; GFX9-NEXT:    v_or_b32_e32 v5, 0x400000, v2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v5, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v3
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, -1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v4, v3, 16, 1
+; GFX9-NEXT:    v_add3_u32 v4, v4, v3, s0
+; GFX9-NEXT:    v_or_b32_e32 v5, 0x400000, v3
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v4, v0, 16, 1
+; GFX9-NEXT:    v_add3_u32 v4, v4, v0, s0
+; GFX9-NEXT:    v_or_b32_e32 v5, 0x400000, v0
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, vcc
+; GFX9-NEXT:    v_bfe_u32 v4, v1, 16, 1
+; GFX9-NEXT:    v_add3_u32 v4, v4, v1, s0
+; GFX9-NEXT:    v_or_b32_e32 v5, 0x400000, v1
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    s_mov_b32 s0, 0x7060302
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v5, vcc
+; GFX9-NEXT:    v_perm_b32 v0, v1, v0, s0
+; GFX9-NEXT:    v_perm_b32 v1, v3, v2, s0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_sitofp_v4i1_to_v4bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX11-NEXT:    v_and_b32_e32 v3, 1, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v2
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_bfe_u32 v4, v2, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0, -1.0, vcc_lo
+; GFX11-NEXT:    v_or_b32_e32 v7, 0x400000, v2
+; GFX11-NEXT:    v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_bfe_u32 v5, v3, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v9, 0x400000, v3
+; GFX11-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX11-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-NEXT:    v_bfe_u32 v6, v0, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-NEXT:    v_bfe_u32 v8, v1, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_add3_u32 v7, v8, v1, 0x7fff
+; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v1
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v6, v4, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, v7, v8, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT:    v_perm_b32 v0, v1, v0, 0x7060302
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, v5, v9, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_perm_b32 v1, v3, v2, 0x7060302
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v_sitofp_v4i1_to_v4bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_expcnt 0x0
+; GFX12-NEXT:    s_wait_samplecnt 0x0
+; GFX12-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_and_b32_e32 v2, 1, v2
+; GFX12-NEXT:    v_and_b32_e32 v3, 1, v3
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v2
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, vcc_lo
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v3
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_bfe_u32 v4, v2, 16, 1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v3, 0, -1.0, vcc_lo
+; GFX12-NEXT:    v_or_b32_e32 v7, 0x400000, v2
+; GFX12-NEXT:    v_add3_u32 v4, v4, v2, 0x7fff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_bfe_u32 v5, v3, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v3
+; GFX12-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX12-NEXT:    v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
+; GFX12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX12-NEXT:    v_bfe_u32 v6, v0, 16, 1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_add3_u32 v6, v6, v0, 0x7fff
+; GFX12-NEXT:    v_bfe_u32 v8, v1, 16, 1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX12-NEXT:    v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    v_add3_u32 v7, v8, v1, 0x7fff
+; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v6, v4, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v1, v7, v8, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_perm_b32 v0, v1, v0, 0x7060302
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v3, v5, v9, vcc_lo
+; GFX12-NEXT:    v_perm_b32 v1, v3, v2, 0x7060302
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
+  %op = sitofp <4 x i1> %num to <4 x bfloat>
+  ret <4 x bfloat> %op
+}
+
+define amdgpu_ps <4 x i32> @s_sitofp_v4i1_to_v4bf16(<4 x i1> inreg %num) {
+; GFX7-LABEL: s_sitofp_v4i1_to_v4bf16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_and_b32 s6, 1, s3
+; GFX7-NEXT:    s_and_b32 s4, 1, s2
+; GFX7-NEXT:    s_and_b32 s2, 1, s1
+; GFX7-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX7-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX7-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX7-NEXT:    s_cselect_b64 s[2:3], -1, 0
+; GFX7-NEXT:    s_cmp_eq_u32 s4, 1
+; GFX7-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; GFX7-NEXT:    s_cmp_eq_u32 s6, 1
+; GFX7-NEXT:    s_cselect_b64 s[6:7], -1, 0
+; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, s[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e64 v3, 0, -1.0, s[0:1]
+; GFX7-NEXT:    v_ashrrev_i32_e32 v3, 16, v3
+; GFX7-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
+; GFX7-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX7-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX7-NEXT:    v_readfirstlane_b32 s0, v3
+; GFX7-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX7-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX7-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX7-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_sitofp_v4i1_to_v4bf16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_and_b32 s6, 1, s3
+; GFX9-NEXT:    s_and_b32 s4, 1, s2
+; GFX9-NEXT:    s_and_b32 s2, 1, s1
+; GFX9-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX9-NEXT:    s_cselect_b64 s[2:3], -1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s4, 1
+; GFX9-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s6, 1
+; GFX9-NEXT:    s_cselect_b64 s[6:7], -1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s[6:7]
+; GFX9-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v0
+; GFX9-NEXT:    v_or_b32_e32 v1, 0x400000, v0
+; GFX9-NEXT:    v_add_u32_e32 v2, 0x7fff, v2
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT:    s_nop 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s[4:5]
+; GFX9-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v1
+; GFX9-NEXT:    v_or_b32_e32 v2, 0x400000, v1
+; GFX9-NEXT:    v_add_u32_e32 v3, 0x7fff, v3
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX9-NEXT:    s_nop 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, -1.0, s[2:3]
+; GFX9-NEXT:    v_bfe_u32 v4, v2, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v4, v4, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, 0x400000, v2
+; GFX9-NEXT:    v_add_u32_e32 v4, 0x7fff, v4
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
+; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, -1.0, s[0:1]
+; GFX9-NEXT:    v_bfe_u32 v5, v3, 16, 1
+; GFX9-NEXT:    v_add_u32_e32 v5, v5, v3
+; GFX9-NEXT:    v_or_b32_e32 v4, 0x400000, v3
+; GFX9-NEXT:    v_add_u32_e32 v5, 0x7fff, v5
+; GFX9-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
+; GFX9-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v4, vcc
+; GFX9-NEXT:    v_ashrrev_i32_e32 v3, 16, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: s_sitofp_v4i1_to_v4bf16:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_and_b32 s3, 1, s3
+; GFX11-NEXT:    s_and_b32 s2, 1, s2
+; GFX11-NEXT:    s_and_b32 s1, 1, s1
+; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s1, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, 0, -1.0, s0
+; GFX11-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, 0, -1.0, s1
+; GFX11-NEXT:    s_cselect_b32 s2, -1, 0
+; GFX11-NEXT:    s_cmp_eq_u32 s3, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s2
+; GFX11-NEXT:    s_cselect_b32 s3, -1, 0
+; GFX11-NEXT:    v_bfe_u32 v7, v4, 16, 1
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s3
+; GFX11-NEXT:    v_bfe_u32 v6, v3, 16, 1
+; GFX11-NEXT:    v_bfe_u32 v5, v1, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v10, 0x400000, v4
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, v7, v4
+; GFX11-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX11-NEXT:    v_or_b32_e32 v8, 0x400000, v0
+; GFX11-NEXT:    v_or_b32_e32 v9, 0x400000, v1
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, v6, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v7, 0x7fff, v7
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, v5, v1
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-NEXT:    v_or_b32_e32 v11, 0x400000, v3
+; GFX11-NEXT:    v_add_nc_u32_e32 v6, 0x7fff, v6
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX11-NEXT:    v_dual_cndmask_b32 v1, v5, v9 :: v_dual_add_nc_u32 v2, v2, v0
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX11-NEXT:    v_cndmask_b32_e32 v3, v6, v11, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX11-NEXT:    v_cndmask_b32_e32 v4, v7, v10, vcc_lo
+; GFX11-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v8, vcc_lo
+; GFX11-NEXT:    v_ashrrev_i32_e32 v2, 16, v3
+; GFX11-NEXT:    v_ashrrev_i32_e32 v3, 16, v4
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v3
+; GFX11-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX11-NEXT:    ; return to shader part epilog
+;
+; GFX12-LABEL: s_sitofp_v4i1_to_v4bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_and_b32 s3, 1, s3
+; GFX12-NEXT:    s_and_b32 s2, 1, s2
+; GFX12-NEXT:    s_and_b32 s1, 1, s1
+; GFX12-NEXT:    s_bitcmp1_b32 s0, 0
+; GFX12-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s1, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v3, 0, -1.0, s0
+; GFX12-NEXT:    s_cselect_b32 s1, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    v_cndmask_b32_e64 v4, 0, -1.0, s1
+; GFX12-NEXT:    s_cselect_b32 s2, -1, 0
+; GFX12-NEXT:    s_cmp_eq_u32 s3, 1
+; GFX12-NEXT:    v_cndmask_b32_e64 v1, 0, -1.0, s2
+; GFX12-NEXT:    s_cselect_b32 s3, -1, 0
+; GFX12-NEXT:    v_bfe_u32 v7, v4, 16, 1
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, s3
+; GFX12-NEXT:    v_bfe_u32 v6, v3, 16, 1
+; GFX12-NEXT:    v_bfe_u32 v5, v1, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v10, 0x400000, v4
+; GFX12-NEXT:    v_add_nc_u32_e32 v7, v7, v4
+; GFX12-NEXT:    v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT:    v_or_b32_e32 v8, 0x400000, v0
+; GFX12-NEXT:    v_or_b32_e32 v9, 0x400000, v1
+; GFX12-NEXT:    v_add_nc_u32_e32 v6, v6, v3
+; GFX12-NEXT:    v_add_nc_u32_e32 v7, 0x7fff, v7
+; GFX12-NEXT:    v_add_nc_u32_e32 v5, v5, v1
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-NEXT:    v_or_b32_e32 v11, 0x400000, v3
+; GFX12-NEXT:    v_add_nc_u32_e32 v6, 0x7fff, v6
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX12-NEXT:    v_dual_cndmask_b32 v1, v5, v9 :: v_dual_add_nc_u32 v2, v2, v0
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX12-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v3, v6, v11, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-NEXT:    v_readfirstlane_b32 s2, v1
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v4, v7, v10, vcc_lo
+; GFX12-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    v_cndmask_b32_e32 v0, v2, v8, vcc_lo
+; GFX12-NEXT:    v_ashrrev_i32_e32 v2, 16, v3
+; GFX12-NEXT:    v_ashrrev_i32_e32 v3, 16, v4
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX12-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT:    v_readfirstlane_b32 s1, v3
+; GFX12-NEXT:    v_readfirstlane_b32 s3, v0
+; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    ; return to shader part epilog
+  %op = sitofp <4 x i1> %num to <4 x bfloat>
+  %b16 = bitcast <4 x bfloat> %op to <4 x i16>
+  %b32 = sext <4 x i16> %b16 to <4 x i32>
+  ret <4 x i32> %b32
+}
diff --git a/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll b/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll
index 81eac63ae5bdf..8dbd6c5d133ea 100644
--- a/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll
+++ b/llvm/test/CodeGen/AMDGPU/identical-subrange-spill-infloop.ll
@@ -16,28 +16,20 @@ define void @main(i1 %arg) #0 {
 ; CHECK-NEXT:    v_writelane_b32 v5, s36, 4
 ; CHECK-NEXT:    v_writelane_b32 v5, s37, 5
 ; CHECK-NEXT:    v_writelane_b32 v5, s38, 6
-; CHECK-NEXT:    v_writelane_b32 v5, s39, 7
-; CHECK-NEXT:    v_writelane_b32 v5, s40, 8
-; CHECK-NEXT:    v_writelane_b32 v5, s41, 9
-; CHECK-NEXT:    v_writelane_b32 v5, s42, 10
-; CHECK-NEXT:    v_writelane_b32 v5, s43, 11
-; CHECK-NEXT:    v_writelane_b32 v5, s44, 12
-; CHECK-NEXT:    v_writelane_b32 v5, s45, 13
-; CHECK-NEXT:    v_writelane_b32 v5, s46, 14
 ; CHECK-NEXT:    s_getpc_b64 s[24:25]
-; CHECK-NEXT:    v_writelane_b32 v5, s47, 15
+; CHECK-NEXT:    v_writelane_b32 v5, s39, 7
 ; CHECK-NEXT:    s_movk_i32 s20, 0xf0
 ; CHECK-NEXT:    s_mov_b32 s21, s24
-; CHECK-NEXT:    v_writelane_b32 v5, s48, 16
+; CHECK-NEXT:    v_writelane_b32 v5, s48, 8
 ; CHECK-NEXT:    s_load_dwordx16 s[4:19], s[20:21], 0x0
 ; CHECK-NEXT:    s_mov_b64 s[20:21], 0
-; CHECK-NEXT:    v_writelane_b32 v5, s49, 17
+; CHECK-NEXT:    v_writelane_b32 v5, s49, 9
 ; CHECK-NEXT:    s_load_dwordx4 s[20:23], s[20:21], 0x0
-; CHECK-NEXT:    v_writelane_b32 v5, s50, 18
+; CHECK-NEXT:    v_writelane_b32 v5, s50, 10
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_movk_i32 s22, 0x130
 ; CHECK-NEXT:    s_mov_b32 s23, s24
-; CHECK-NEXT:    v_writelane_b32 v5, s51, 19
+; CHECK-NEXT:    v_writelane_b32 v5, s51, 11
 ; CHECK-NEXT:    s_load_dwordx16 s[36:51], s[22:23], 0x0
 ; CHECK-NEXT:    s_mov_b32 s28, 0
 ; CHECK-NEXT:    v_mov_b32_e32 v1, 0
@@ -49,60 +41,51 @@ define void @main(i1 %arg) #0 {
 ; CHECK-NEXT:    image_sample_lz v3, v[2:3], s[12:19], s[28:31] dmask:0x1
 ; CHECK-NEXT:    v_mov_b32_e32 v2, v1
 ; CHECK-NEXT:    ; implicit-def: $vgpr6 : SGPR spill to VGPR lane
-; CHECK-NEXT:    v_writelane_b32 v5, s52, 20
+; CHECK-NEXT:    v_writelane_b32 v5, s52, 12
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_writelane_b32 v6, s36, 0
-; CHECK-NEXT:    v_writelane_b32 v5, s53, 21
-; CHECK-NEXT:    v_writelane_b32 v5, s54, 22
-; CHECK-NEXT:    v_writelane_b32 v5, s55, 23
-; CHECK-NEXT:    v_writelane_b32 v5, s56, 24
 ; CHECK-NEXT:    v_writelane_b32 v6, s37, 1
-; CHECK-NEXT:    image_sample_lz v4, v[1:2], s[36:43], s[28:31] dmask:0x1
-; CHECK-NEXT:    v_writelane_b32 v5, s57, 25
 ; CHECK-NEXT:    v_writelane_b32 v6, s38, 2
-; CHECK-NEXT:    v_writelane_b32 v5, s58, 26
 ; CHECK-NEXT:    v_writelane_b32 v6, s39, 3
-; CHECK-NEXT:    v_writelane_b32 v5, s59, 27
 ; CHECK-NEXT:    v_writelane_b32 v6, s40, 4
-; CHECK-NEXT:    v_writelane_b32 v5, s60, 28
 ; CHECK-NEXT:    v_writelane_b32 v6, s41, 5
-; CHECK-NEXT:    v_writelane_b32 v5, s61, 29
+; CHECK-NEXT:    image_sample_lz v4, v[1:2], s[36:43], s[28:31] dmask:0x1
 ; CHECK-NEXT:    v_writelane_b32 v6, s42, 6
-; CHECK-NEXT:    v_writelane_b32 v5, s62, 30
 ; CHECK-NEXT:    v_writelane_b32 v6, s43, 7
-; CHECK-NEXT:    v_writelane_b32 v5, s63, 31
 ; CHECK-NEXT:    v_writelane_b32 v6, s44, 8
-; CHECK-NEXT:    v_writelane_b32 v5, s64, 32
 ; CHECK-NEXT:    v_writelane_b32 v6, s45, 9
-; CHECK-NEXT:    v_writelane_b32 v5, s65, 33
+; CHECK-NEXT:    v_writelane_b32 v5, s53, 13
 ; CHECK-NEXT:    v_writelane_b32 v6, s46, 10
-; CHECK-NEXT:    v_writelane_b32 v5, s66, 34
+; CHECK-NEXT:    v_writelane_b32 v5, s54, 14
 ; CHECK-NEXT:    v_writelane_b32 v6, s47, 11
-; CHECK-NEXT:    v_writelane_b32 v5, s67, 35
+; CHECK-NEXT:    v_writelane_b32 v5, s55, 15
 ; CHECK-NEXT:    v_writelane_b32 v6, s48, 12
-; CHECK-NEXT:    v_writelane_b32 v5, s68, 36
+; CHECK-NEXT:    v_writelane_b32 v5, s64, 16
 ; CHECK-NEXT:    v_writelane_b32 v6, s49, 13
-; CHECK-NEXT:    v_writelane_b32 v5, s69, 37
+; CHECK-NEXT:    v_writelane_b32 v5, s65, 17
 ; CHECK-NEXT:    v_writelane_b32 v6, s50, 14
-; CHECK-NEXT:    s_mov_b32 s34, 48
-; CHECK-NEXT:    s_movk_i32 s52, 0x1f0
-; CHECK-NEXT:    s_movk_i32 s68, 0x2f0
-; CHECK-NEXT:    s_mov_b32 s35, s24
-; CHECK-NEXT:    s_mov_b32 s53, s24
-; CHECK-NEXT:    s_mov_b32 s69, s24
+; CHECK-NEXT:    v_writelane_b32 v5, s66, 18
 ; CHECK-NEXT:    v_writelane_b32 v6, s51, 15
-; CHECK-NEXT:    s_load_dwordx8 s[20:27], s[34:35], 0x0
-; CHECK-NEXT:    s_load_dwordx16 s[36:51], s[52:53], 0x0
+; CHECK-NEXT:    s_mov_b32 s40, 48
+; CHECK-NEXT:    s_movk_i32 s56, 0x1f0
+; CHECK-NEXT:    s_movk_i32 s34, 0x2f0
+; CHECK-NEXT:    s_mov_b32 s41, s24
+; CHECK-NEXT:    s_mov_b32 s57, s24
+; CHECK-NEXT:    s_mov_b32 s35, s24
+; CHECK-NEXT:    v_writelane_b32 v5, s67, 19
+; CHECK-NEXT:    s_load_dwordx8 s[20:27], s[40:41], 0x0
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    s_load_dwordx16 s[36:51], s[56:57], 0x0
 ; CHECK-NEXT:    v_and_b32_e32 v0, 1, v0
-; CHECK-NEXT:    s_load_dwordx16 s[52:67], s[68:69], 0x0
+; CHECK-NEXT:    s_load_dwordx16 s[52:67], s[34:35], 0x0
 ; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; CHECK-NEXT:    v_writelane_b32 v5, s70, 38
-; CHECK-NEXT:    s_xor_b64 s[34:35], vcc, -1
-; CHECK-NEXT:    v_writelane_b32 v5, s71, 39
+; CHECK-NEXT:    v_writelane_b32 v5, s68, 20
+; CHECK-NEXT:    s_xor_b64 s[72:73], vcc, -1
+; CHECK-NEXT:    v_writelane_b32 v5, s69, 21
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    v_mul_f32_e32 v0, v4, v3
-; CHECK-NEXT:    s_and_saveexec_b64 vcc, s[34:35]
-; CHECK-NEXT:    s_xor_b64 s[68:69], exec, vcc
+; CHECK-NEXT:    s_and_saveexec_b64 vcc, s[72:73]
+; CHECK-NEXT:    s_xor_b64 s[34:35], exec, vcc
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_3
 ; CHECK-NEXT:  ; %bb.1: ; %bb48
 ; CHECK-NEXT:    image_sample_lz v3, v[1:2], s[12:19], s[28:31] dmask:0x1
@@ -124,10 +107,10 @@ define void @main(i1 %arg) #0 {
 ; CHECK-NEXT:    s_mov_b64 vcc, vcc
 ; CHECK-NEXT:    s_cbranch_vccnz .LBB0_2
 ; CHECK-NEXT:  .LBB0_3: ; %Flow14
-; CHECK-NEXT:    s_andn2_saveexec_b64 s[12:13], s[68:69]
+; CHECK-NEXT:    s_andn2_saveexec_b64 s[12:13], s[34:35]
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_10
 ; CHECK-NEXT:  ; %bb.4: ; %bb32
-; CHECK-NEXT:    s_and_saveexec_b64 s[14:15], s[34:35]
+; CHECK-NEXT:    s_and_saveexec_b64 s[14:15], s[72:73]
 ; CHECK-NEXT:    s_xor_b64 s[14:15], exec, s[14:15]
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_6
 ; CHECK-NEXT:  ; %bb.5: ; %bb43
@@ -201,39 +184,21 @@ define void @main(i1 %arg) #0 {
 ; CHECK-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; CHECK-NEXT:  .LBB0_10: ; %UnifiedReturnBlock
 ; CHECK-NEXT:    s_or_b64 exec, exec, s[12:13]
-; CHECK-NEXT:    v_readlane_b32 s71, v5, 39
-; CHECK-NEXT:    v_readlane_b32 s70, v5, 38
-; CHECK-NEXT:    v_readlane_b32 s69, v5, 37
-; CHECK-NEXT:    v_readlane_b32 s68, v5, 36
+; CHECK-NEXT:    v_readlane_b32 s69, v5, 21
+; CHECK-NEXT:    v_readlane_b32 s68, v5, 20
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_readlane_b32 s67, v5, 35
-; CHECK-NEXT:    v_readlane_b32 s66, v5, 34
-; CHECK-NEXT:    v_readlane_b32 s65, v5, 33
-; CHECK-NEXT:    v_readlane_b32 s64, v5, 32
-; CHECK-NEXT:    v_readlane_b32 s63, v5, 31
-; CHECK-NEXT:    v_readlane_b32 s62, v5, 30
-; CHECK-NEXT:    v_readlane_b32 s61, v5, 29
-; CHECK-NEXT:    v_readlane_b32 s60, v5, 28
-; CHECK-NEXT:    v_readlane_b32 s59, v5, 27
-; CHECK-NEXT:    v_readlane_b32 s58, v5, 26
-; CHECK-NEXT:    v_readlane_b32 s57, v5, 25
-; CHECK-NEXT:    v_readlane_b32 s56, v5, 24
-; CHECK-NEXT:    v_readlane_b32 s55, v5, 23
-; CHECK-NEXT:    v_readlane_b32 s54, v5, 22
-; CHECK-NEXT:    v_readlane_b32 s53, v5, 21
-; CHECK-NEXT:    v_readlane_b32 s52, v5, 20
-; CHECK-NEXT:    v_readlane_b32 s51, v5, 19
-; CHECK-NEXT:    v_readlane_b32 s50, v5, 18
-; CHECK-NEXT:    v_readlane_b32 s49, v5, 17
-; CHECK-NEXT:    v_readlane_b32 s48, v5, 16
-; CHECK-NEXT:    v_readlane_b32 s47, v5, 15
-; CHECK-NEXT:    v_readlane_b32 s46, v5, 14
-; CHECK-NEXT:    v_readlane_b32 s45, v5, 13
-; CHECK-NEXT:    v_readlane_b32 s44, v5, 12
-; CHECK-NEXT:    v_readlane_b32 s43, v5, 11
-; CHECK-NEXT:    v_readlane_b32 s42, v5, 10
-; CHECK-NEXT:    v_readlane_b32 s41, v5, 9
-; CHECK-NEXT:    v_readlane_b32 s40, v5, 8
+; CHECK-NEXT:    v_readlane_b32 s67, v5, 19
+; CHECK-NEXT:    v_readlane_b32 s66, v5, 18
+; CHECK-NEXT:    v_readlane_b32 s65, v5, 17
+; CHECK-NEXT:    v_readlane_b32 s64, v5, 16
+; CHECK-NEXT:    v_readlane_b32 s55, v5, 15
+; CHECK-NEXT:    v_readlane_b32 s54, v5, 14
+; CHECK-NEXT:    v_readlane_b32 s53, v5, 13
+; CHECK-NEXT:    v_readlane_b32 s52, v5, 12
+; CHECK-NEXT:    v_readlane_b32 s51, v5, 11
+; CHECK-NEXT:    v_readlane_b32 s50, v5, 10
+; CHECK-NEXT:    v_readlane_b32 s49, v5, 9
+; CHECK-NEXT:    v_readlane_b32 s48, v5, 8
 ; CHECK-NEXT:    v_readlane_b32 s39, v5, 7
 ; CHECK-NEXT:    v_readlane_b32 s38, v5, 6
 ; CHECK-NEXT:    v_readlane_b32 s37, v5, 5
diff --git a/llvm/test/CodeGen/AMDGPU/image-schedule.ll b/llvm/test/CodeGen/AMDGPU/image-schedule.ll
index dbd9efc58e59d..09e819de074ff 100644
--- a/llvm/test/CodeGen/AMDGPU/image-schedule.ll
+++ b/llvm/test/CodeGen/AMDGPU/image-schedule.ll
@@ -14,7 +14,7 @@ define dllexport amdgpu_cs void @_amdgpu_cs_main(i32 inreg %arg, i32 inreg %arg1
 .entry:
   %tmp = call i64 @llvm.amdgcn.s.getpc() #1
   %tmp6 = bitcast i64 %tmp to <2 x i32>
-  %.0.vec.insert = insertelement <2 x i32> undef, i32 %arg2, i32 0
+  %.0.vec.insert = insertelement <2 x i32> poison, i32 %arg2, i32 0
   %.4.vec.insert = shufflevector <2 x i32> %.0.vec.insert, <2 x i32> %tmp6, <2 x i32> <i32 0, i32 3>
   %tmp7 = bitcast <2 x i32> %.4.vec.insert to i64
   %tmp8 = inttoptr i64 %tmp7 to ptr addrspace(4)
diff --git a/llvm/test/CodeGen/AMDGPU/implicit-def-muse.ll b/llvm/test/CodeGen/AMDGPU/implicit-def-muse.ll
index aabe4e930f708..495e8a281b05b 100644
--- a/llvm/test/CodeGen/AMDGPU/implicit-def-muse.ll
+++ b/llvm/test/CodeGen/AMDGPU/implicit-def-muse.ll
@@ -18,7 +18,7 @@ false:
   br label %exit
 
 exit:
-  %c = phi <2 x i1> [ undef, %entry ], [ %cmp, %false ]
+  %c = phi <2 x i1> [ poison, %entry ], [ %cmp, %false ]
   %ret = select <2 x i1> %c, <2 x float> <float 2.0, float 2.0>, <2 x float> <float 4.0, float 4.0>
   ret <2 x float> %ret
 }
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
index 56a3ce7281030..a7e31375f425e 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
@@ -1880,7 +1880,7 @@ define amdgpu_kernel void @insert_undef_offset_sgpr_vector_src(ptr addrspace(1)
 ; GFX9-IDXMODE-NEXT:    s_endpgm
 entry:
   %ld = load <4 x i32>, ptr addrspace(1) %in
-  %value = insertelement <4 x i32> %ld, i32 5, i32 undef
+  %value = insertelement <4 x i32> %ld, i32 5, i32 poison
   store <4 x i32> %value, ptr addrspace(1) %out
   ret void
 }
@@ -7699,13 +7699,13 @@ bb:
 
 bb1:
   %tmp2 = load volatile <4 x float>, ptr addrspace(1) undef
-  %tmp3 = insertelement <4 x float> %tmp2, float %val0, i32 undef
+  %tmp3 = insertelement <4 x float> %tmp2, float %val0, i32 poison
   call void asm sideeffect "; reg use $0", "v"(<4 x float> %tmp3) ; Prevent block optimize out
   br label %bb7
 
 bb4:
   %tmp5 = load volatile <4 x float>, ptr addrspace(1) undef
-  %tmp6 = insertelement <4 x float> %tmp5, float %val0, i32 undef
+  %tmp6 = insertelement <4 x float> %tmp5, float %val0, i32 poison
   call void asm sideeffect "; reg use $0", "v"(<4 x float> %tmp6) ; Prevent block optimize out
   br label %bb7
 
@@ -9459,7 +9459,7 @@ bb2:
 
 bb4:
   %vgpr = load volatile i32, ptr addrspace(1) undef
-  %tmp5 = insertelement <16 x i32> undef, i32 undef, i32 %vgpr
+  %tmp5 = insertelement <16 x i32> poison, i32 poison, i32 %vgpr
   %tmp6 = insertelement <16 x i32> %tmp5, i32 %arg1, i32 %vgpr
   %tmp7 = extractelement <16 x i32> %tmp6, i32 0
   br label %bb2
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-call.ll b/llvm/test/CodeGen/AMDGPU/indirect-call.ll
index 55da485b91f67..d7c4f6afbdade 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-call.ll
@@ -136,55 +136,55 @@ define void @test_indirect_call_vgpr_ptr(ptr %fptr) {
 ; GCN-NEXT:    v_writelane_b32 v40, s37, 5
 ; GCN-NEXT:    v_writelane_b32 v40, s38, 6
 ; GCN-NEXT:    v_writelane_b32 v40, s39, 7
-; GCN-NEXT:    v_writelane_b32 v40, s40, 8
-; GCN-NEXT:    v_writelane_b32 v40, s41, 9
-; GCN-NEXT:    v_writelane_b32 v40, s42, 10
-; GCN-NEXT:    v_writelane_b32 v40, s43, 11
-; GCN-NEXT:    v_writelane_b32 v40, s44, 12
-; GCN-NEXT:    v_writelane_b32 v40, s45, 13
-; GCN-NEXT:    v_writelane_b32 v40, s46, 14
-; GCN-NEXT:    v_writelane_b32 v40, s47, 15
-; GCN-NEXT:    v_writelane_b32 v40, s48, 16
-; GCN-NEXT:    v_writelane_b32 v40, s49, 17
-; GCN-NEXT:    s_mov_b32 s42, s15
-; GCN-NEXT:    s_mov_b32 s43, s14
-; GCN-NEXT:    s_mov_b32 s44, s13
-; GCN-NEXT:    s_mov_b32 s45, s12
+; GCN-NEXT:    v_writelane_b32 v40, s48, 8
+; GCN-NEXT:    v_writelane_b32 v40, s49, 9
+; GCN-NEXT:    v_writelane_b32 v40, s50, 10
+; GCN-NEXT:    v_writelane_b32 v40, s51, 11
+; GCN-NEXT:    v_writelane_b32 v40, s52, 12
+; GCN-NEXT:    v_writelane_b32 v40, s53, 13
+; GCN-NEXT:    v_writelane_b32 v40, s54, 14
+; GCN-NEXT:    v_writelane_b32 v40, s55, 15
+; GCN-NEXT:    v_writelane_b32 v40, s64, 16
+; GCN-NEXT:    v_writelane_b32 v40, s65, 17
+; GCN-NEXT:    s_mov_b32 s50, s15
+; GCN-NEXT:    s_mov_b32 s51, s14
+; GCN-NEXT:    s_mov_b32 s52, s13
+; GCN-NEXT:    s_mov_b32 s53, s12
 ; GCN-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; GCN-NEXT:    s_mov_b64 s[36:37], s[8:9]
 ; GCN-NEXT:    s_mov_b64 s[38:39], s[6:7]
-; GCN-NEXT:    s_mov_b64 s[40:41], s[4:5]
-; GCN-NEXT:    s_mov_b64 s[46:47], exec
+; GCN-NEXT:    s_mov_b64 s[48:49], s[4:5]
+; GCN-NEXT:    s_mov_b64 s[54:55], exec
 ; GCN-NEXT:  .LBB2_1: ; =>This Inner Loop Header: Depth=1
 ; GCN-NEXT:    v_readfirstlane_b32 s16, v0
 ; GCN-NEXT:    v_readfirstlane_b32 s17, v1
 ; GCN-NEXT:    v_cmp_eq_u64_e32 vcc, s[16:17], v[0:1]
-; GCN-NEXT:    s_and_saveexec_b64 s[48:49], vcc
-; GCN-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GCN-NEXT:    s_and_saveexec_b64 s[64:65], vcc
+; GCN-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GCN-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GCN-NEXT:    s_mov_b64 s[8:9], s[36:37]
 ; GCN-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GCN-NEXT:    s_mov_b32 s12, s45
-; GCN-NEXT:    s_mov_b32 s13, s44
-; GCN-NEXT:    s_mov_b32 s14, s43
-; GCN-NEXT:    s_mov_b32 s15, s42
+; GCN-NEXT:    s_mov_b32 s12, s53
+; GCN-NEXT:    s_mov_b32 s13, s52
+; GCN-NEXT:    s_mov_b32 s14, s51
+; GCN-NEXT:    s_mov_b32 s15, s50
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN-NEXT:    ; implicit-def: $vgpr31
-; GCN-NEXT:    s_xor_b64 exec, exec, s[48:49]
+; GCN-NEXT:    s_xor_b64 exec, exec, s[64:65]
 ; GCN-NEXT:    s_cbranch_execnz .LBB2_1
 ; GCN-NEXT:  ; %bb.2:
-; GCN-NEXT:    s_mov_b64 exec, s[46:47]
-; GCN-NEXT:    v_readlane_b32 s49, v40, 17
-; GCN-NEXT:    v_readlane_b32 s48, v40, 16
-; GCN-NEXT:    v_readlane_b32 s47, v40, 15
-; GCN-NEXT:    v_readlane_b32 s46, v40, 14
-; GCN-NEXT:    v_readlane_b32 s45, v40, 13
-; GCN-NEXT:    v_readlane_b32 s44, v40, 12
-; GCN-NEXT:    v_readlane_b32 s43, v40, 11
-; GCN-NEXT:    v_readlane_b32 s42, v40, 10
-; GCN-NEXT:    v_readlane_b32 s41, v40, 9
-; GCN-NEXT:    v_readlane_b32 s40, v40, 8
+; GCN-NEXT:    s_mov_b64 exec, s[54:55]
+; GCN-NEXT:    v_readlane_b32 s65, v40, 17
+; GCN-NEXT:    v_readlane_b32 s64, v40, 16
+; GCN-NEXT:    v_readlane_b32 s55, v40, 15
+; GCN-NEXT:    v_readlane_b32 s54, v40, 14
+; GCN-NEXT:    v_readlane_b32 s53, v40, 13
+; GCN-NEXT:    v_readlane_b32 s52, v40, 12
+; GCN-NEXT:    v_readlane_b32 s51, v40, 11
+; GCN-NEXT:    v_readlane_b32 s50, v40, 10
+; GCN-NEXT:    v_readlane_b32 s49, v40, 9
+; GCN-NEXT:    v_readlane_b32 s48, v40, 8
 ; GCN-NEXT:    v_readlane_b32 s39, v40, 7
 ; GCN-NEXT:    v_readlane_b32 s38, v40, 6
 ; GCN-NEXT:    v_readlane_b32 s37, v40, 5
@@ -220,55 +220,55 @@ define void @test_indirect_call_vgpr_ptr(ptr %fptr) {
 ; GISEL-NEXT:    v_writelane_b32 v40, s37, 5
 ; GISEL-NEXT:    v_writelane_b32 v40, s38, 6
 ; GISEL-NEXT:    v_writelane_b32 v40, s39, 7
-; GISEL-NEXT:    v_writelane_b32 v40, s40, 8
-; GISEL-NEXT:    v_writelane_b32 v40, s41, 9
-; GISEL-NEXT:    v_writelane_b32 v40, s42, 10
-; GISEL-NEXT:    v_writelane_b32 v40, s43, 11
-; GISEL-NEXT:    v_writelane_b32 v40, s44, 12
-; GISEL-NEXT:    v_writelane_b32 v40, s45, 13
-; GISEL-NEXT:    v_writelane_b32 v40, s46, 14
-; GISEL-NEXT:    v_writelane_b32 v40, s47, 15
-; GISEL-NEXT:    v_writelane_b32 v40, s48, 16
-; GISEL-NEXT:    v_writelane_b32 v40, s49, 17
-; GISEL-NEXT:    s_mov_b32 s42, s15
-; GISEL-NEXT:    s_mov_b32 s43, s14
-; GISEL-NEXT:    s_mov_b32 s44, s13
-; GISEL-NEXT:    s_mov_b32 s45, s12
+; GISEL-NEXT:    v_writelane_b32 v40, s48, 8
+; GISEL-NEXT:    v_writelane_b32 v40, s49, 9
+; GISEL-NEXT:    v_writelane_b32 v40, s50, 10
+; GISEL-NEXT:    v_writelane_b32 v40, s51, 11
+; GISEL-NEXT:    v_writelane_b32 v40, s52, 12
+; GISEL-NEXT:    v_writelane_b32 v40, s53, 13
+; GISEL-NEXT:    v_writelane_b32 v40, s54, 14
+; GISEL-NEXT:    v_writelane_b32 v40, s55, 15
+; GISEL-NEXT:    v_writelane_b32 v40, s64, 16
+; GISEL-NEXT:    v_writelane_b32 v40, s65, 17
+; GISEL-NEXT:    s_mov_b32 s50, s15
+; GISEL-NEXT:    s_mov_b32 s51, s14
+; GISEL-NEXT:    s_mov_b32 s52, s13
+; GISEL-NEXT:    s_mov_b32 s53, s12
 ; GISEL-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; GISEL-NEXT:    s_mov_b64 s[36:37], s[8:9]
 ; GISEL-NEXT:    s_mov_b64 s[38:39], s[6:7]
-; GISEL-NEXT:    s_mov_b64 s[40:41], s[4:5]
-; GISEL-NEXT:    s_mov_b64 s[46:47], exec
+; GISEL-NEXT:    s_mov_b64 s[48:49], s[4:5]
+; GISEL-NEXT:    s_mov_b64 s[54:55], exec
 ; GISEL-NEXT:  .LBB2_1: ; =>This Inner Loop Header: Depth=1
 ; GISEL-NEXT:    v_readfirstlane_b32 s16, v0
 ; GISEL-NEXT:    v_readfirstlane_b32 s17, v1
 ; GISEL-NEXT:    v_cmp_eq_u64_e32 vcc, s[16:17], v[0:1]
-; GISEL-NEXT:    s_and_saveexec_b64 s[48:49], vcc
-; GISEL-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GISEL-NEXT:    s_and_saveexec_b64 s[64:65], vcc
+; GISEL-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GISEL-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GISEL-NEXT:    s_mov_b64 s[8:9], s[36:37]
 ; GISEL-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GISEL-NEXT:    s_mov_b32 s12, s45
-; GISEL-NEXT:    s_mov_b32 s13, s44
-; GISEL-NEXT:    s_mov_b32 s14, s43
-; GISEL-NEXT:    s_mov_b32 s15, s42
+; GISEL-NEXT:    s_mov_b32 s12, s53
+; GISEL-NEXT:    s_mov_b32 s13, s52
+; GISEL-NEXT:    s_mov_b32 s14, s51
+; GISEL-NEXT:    s_mov_b32 s15, s50
 ; GISEL-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GISEL-NEXT:    ; implicit-def: $vgpr0
 ; GISEL-NEXT:    ; implicit-def: $vgpr31
-; GISEL-NEXT:    s_xor_b64 exec, exec, s[48:49]
+; GISEL-NEXT:    s_xor_b64 exec, exec, s[64:65]
 ; GISEL-NEXT:    s_cbranch_execnz .LBB2_1
 ; GISEL-NEXT:  ; %bb.2:
-; GISEL-NEXT:    s_mov_b64 exec, s[46:47]
-; GISEL-NEXT:    v_readlane_b32 s49, v40, 17
-; GISEL-NEXT:    v_readlane_b32 s48, v40, 16
-; GISEL-NEXT:    v_readlane_b32 s47, v40, 15
-; GISEL-NEXT:    v_readlane_b32 s46, v40, 14
-; GISEL-NEXT:    v_readlane_b32 s45, v40, 13
-; GISEL-NEXT:    v_readlane_b32 s44, v40, 12
-; GISEL-NEXT:    v_readlane_b32 s43, v40, 11
-; GISEL-NEXT:    v_readlane_b32 s42, v40, 10
-; GISEL-NEXT:    v_readlane_b32 s41, v40, 9
-; GISEL-NEXT:    v_readlane_b32 s40, v40, 8
+; GISEL-NEXT:    s_mov_b64 exec, s[54:55]
+; GISEL-NEXT:    v_readlane_b32 s65, v40, 17
+; GISEL-NEXT:    v_readlane_b32 s64, v40, 16
+; GISEL-NEXT:    v_readlane_b32 s55, v40, 15
+; GISEL-NEXT:    v_readlane_b32 s54, v40, 14
+; GISEL-NEXT:    v_readlane_b32 s53, v40, 13
+; GISEL-NEXT:    v_readlane_b32 s52, v40, 12
+; GISEL-NEXT:    v_readlane_b32 s51, v40, 11
+; GISEL-NEXT:    v_readlane_b32 s50, v40, 10
+; GISEL-NEXT:    v_readlane_b32 s49, v40, 9
+; GISEL-NEXT:    v_readlane_b32 s48, v40, 8
 ; GISEL-NEXT:    v_readlane_b32 s39, v40, 7
 ; GISEL-NEXT:    v_readlane_b32 s38, v40, 6
 ; GISEL-NEXT:    v_readlane_b32 s37, v40, 5
@@ -308,58 +308,58 @@ define void @test_indirect_call_vgpr_ptr_arg(ptr %fptr) {
 ; GCN-NEXT:    v_writelane_b32 v40, s37, 5
 ; GCN-NEXT:    v_writelane_b32 v40, s38, 6
 ; GCN-NEXT:    v_writelane_b32 v40, s39, 7
-; GCN-NEXT:    v_writelane_b32 v40, s40, 8
-; GCN-NEXT:    v_writelane_b32 v40, s41, 9
-; GCN-NEXT:    v_writelane_b32 v40, s42, 10
-; GCN-NEXT:    v_writelane_b32 v40, s43, 11
-; GCN-NEXT:    v_writelane_b32 v40, s44, 12
-; GCN-NEXT:    v_writelane_b32 v40, s45, 13
-; GCN-NEXT:    v_writelane_b32 v40, s46, 14
-; GCN-NEXT:    v_writelane_b32 v40, s47, 15
-; GCN-NEXT:    v_writelane_b32 v40, s48, 16
-; GCN-NEXT:    v_writelane_b32 v40, s49, 17
-; GCN-NEXT:    s_mov_b32 s42, s15
-; GCN-NEXT:    s_mov_b32 s43, s14
-; GCN-NEXT:    s_mov_b32 s44, s13
-; GCN-NEXT:    s_mov_b32 s45, s12
+; GCN-NEXT:    v_writelane_b32 v40, s48, 8
+; GCN-NEXT:    v_writelane_b32 v40, s49, 9
+; GCN-NEXT:    v_writelane_b32 v40, s50, 10
+; GCN-NEXT:    v_writelane_b32 v40, s51, 11
+; GCN-NEXT:    v_writelane_b32 v40, s52, 12
+; GCN-NEXT:    v_writelane_b32 v40, s53, 13
+; GCN-NEXT:    v_writelane_b32 v40, s54, 14
+; GCN-NEXT:    v_writelane_b32 v40, s55, 15
+; GCN-NEXT:    v_writelane_b32 v40, s64, 16
+; GCN-NEXT:    v_writelane_b32 v40, s65, 17
+; GCN-NEXT:    s_mov_b32 s50, s15
+; GCN-NEXT:    s_mov_b32 s51, s14
+; GCN-NEXT:    s_mov_b32 s52, s13
+; GCN-NEXT:    s_mov_b32 s53, s12
 ; GCN-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; GCN-NEXT:    s_mov_b64 s[36:37], s[8:9]
 ; GCN-NEXT:    s_mov_b64 s[38:39], s[6:7]
-; GCN-NEXT:    s_mov_b64 s[40:41], s[4:5]
-; GCN-NEXT:    s_mov_b64 s[46:47], exec
+; GCN-NEXT:    s_mov_b64 s[48:49], s[4:5]
+; GCN-NEXT:    s_mov_b64 s[54:55], exec
 ; GCN-NEXT:    v_mov_b32_e32 v2, 0x7b
 ; GCN-NEXT:  .LBB3_1: ; =>This Inner Loop Header: Depth=1
 ; GCN-NEXT:    v_readfirstlane_b32 s16, v0
 ; GCN-NEXT:    v_readfirstlane_b32 s17, v1
 ; GCN-NEXT:    v_cmp_eq_u64_e32 vcc, s[16:17], v[0:1]
-; GCN-NEXT:    s_and_saveexec_b64 s[48:49], vcc
-; GCN-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GCN-NEXT:    s_and_saveexec_b64 s[64:65], vcc
+; GCN-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GCN-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GCN-NEXT:    s_mov_b64 s[8:9], s[36:37]
 ; GCN-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GCN-NEXT:    s_mov_b32 s12, s45
-; GCN-NEXT:    s_mov_b32 s13, s44
-; GCN-NEXT:    s_mov_b32 s14, s43
-; GCN-NEXT:    s_mov_b32 s15, s42
+; GCN-NEXT:    s_mov_b32 s12, s53
+; GCN-NEXT:    s_mov_b32 s13, s52
+; GCN-NEXT:    s_mov_b32 s14, s51
+; GCN-NEXT:    s_mov_b32 s15, s50
 ; GCN-NEXT:    v_mov_b32_e32 v0, v2
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN-NEXT:    ; implicit-def: $vgpr31
 ; GCN-NEXT:    ; implicit-def: $vgpr2
-; GCN-NEXT:    s_xor_b64 exec, exec, s[48:49]
+; GCN-NEXT:    s_xor_b64 exec, exec, s[64:65]
 ; GCN-NEXT:    s_cbranch_execnz .LBB3_1
 ; GCN-NEXT:  ; %bb.2:
-; GCN-NEXT:    s_mov_b64 exec, s[46:47]
-; GCN-NEXT:    v_readlane_b32 s49, v40, 17
-; GCN-NEXT:    v_readlane_b32 s48, v40, 16
-; GCN-NEXT:    v_readlane_b32 s47, v40, 15
-; GCN-NEXT:    v_readlane_b32 s46, v40, 14
-; GCN-NEXT:    v_readlane_b32 s45, v40, 13
-; GCN-NEXT:    v_readlane_b32 s44, v40, 12
-; GCN-NEXT:    v_readlane_b32 s43, v40, 11
-; GCN-NEXT:    v_readlane_b32 s42, v40, 10
-; GCN-NEXT:    v_readlane_b32 s41, v40, 9
-; GCN-NEXT:    v_readlane_b32 s40, v40, 8
+; GCN-NEXT:    s_mov_b64 exec, s[54:55]
+; GCN-NEXT:    v_readlane_b32 s65, v40, 17
+; GCN-NEXT:    v_readlane_b32 s64, v40, 16
+; GCN-NEXT:    v_readlane_b32 s55, v40, 15
+; GCN-NEXT:    v_readlane_b32 s54, v40, 14
+; GCN-NEXT:    v_readlane_b32 s53, v40, 13
+; GCN-NEXT:    v_readlane_b32 s52, v40, 12
+; GCN-NEXT:    v_readlane_b32 s51, v40, 11
+; GCN-NEXT:    v_readlane_b32 s50, v40, 10
+; GCN-NEXT:    v_readlane_b32 s49, v40, 9
+; GCN-NEXT:    v_readlane_b32 s48, v40, 8
 ; GCN-NEXT:    v_readlane_b32 s39, v40, 7
 ; GCN-NEXT:    v_readlane_b32 s38, v40, 6
 ; GCN-NEXT:    v_readlane_b32 s37, v40, 5
@@ -395,56 +395,56 @@ define void @test_indirect_call_vgpr_ptr_arg(ptr %fptr) {
 ; GISEL-NEXT:    v_writelane_b32 v40, s37, 5
 ; GISEL-NEXT:    v_writelane_b32 v40, s38, 6
 ; GISEL-NEXT:    v_writelane_b32 v40, s39, 7
-; GISEL-NEXT:    v_writelane_b32 v40, s40, 8
-; GISEL-NEXT:    v_writelane_b32 v40, s41, 9
-; GISEL-NEXT:    v_writelane_b32 v40, s42, 10
-; GISEL-NEXT:    v_writelane_b32 v40, s43, 11
-; GISEL-NEXT:    v_writelane_b32 v40, s44, 12
-; GISEL-NEXT:    v_writelane_b32 v40, s45, 13
-; GISEL-NEXT:    v_writelane_b32 v40, s46, 14
-; GISEL-NEXT:    v_writelane_b32 v40, s47, 15
-; GISEL-NEXT:    v_writelane_b32 v40, s48, 16
-; GISEL-NEXT:    v_writelane_b32 v40, s49, 17
-; GISEL-NEXT:    s_mov_b32 s42, s15
-; GISEL-NEXT:    s_mov_b32 s43, s14
-; GISEL-NEXT:    s_mov_b32 s44, s13
-; GISEL-NEXT:    s_mov_b32 s45, s12
+; GISEL-NEXT:    v_writelane_b32 v40, s48, 8
+; GISEL-NEXT:    v_writelane_b32 v40, s49, 9
+; GISEL-NEXT:    v_writelane_b32 v40, s50, 10
+; GISEL-NEXT:    v_writelane_b32 v40, s51, 11
+; GISEL-NEXT:    v_writelane_b32 v40, s52, 12
+; GISEL-NEXT:    v_writelane_b32 v40, s53, 13
+; GISEL-NEXT:    v_writelane_b32 v40, s54, 14
+; GISEL-NEXT:    v_writelane_b32 v40, s55, 15
+; GISEL-NEXT:    v_writelane_b32 v40, s64, 16
+; GISEL-NEXT:    v_writelane_b32 v40, s65, 17
+; GISEL-NEXT:    s_mov_b32 s50, s15
+; GISEL-NEXT:    s_mov_b32 s51, s14
+; GISEL-NEXT:    s_mov_b32 s52, s13
+; GISEL-NEXT:    s_mov_b32 s53, s12
 ; GISEL-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; GISEL-NEXT:    s_mov_b64 s[36:37], s[8:9]
 ; GISEL-NEXT:    s_mov_b64 s[38:39], s[6:7]
-; GISEL-NEXT:    s_mov_b64 s[40:41], s[4:5]
-; GISEL-NEXT:    s_mov_b64 s[46:47], exec
+; GISEL-NEXT:    s_mov_b64 s[48:49], s[4:5]
+; GISEL-NEXT:    s_mov_b64 s[54:55], exec
 ; GISEL-NEXT:  .LBB3_1: ; =>This Inner Loop Header: Depth=1
 ; GISEL-NEXT:    v_readfirstlane_b32 s16, v0
 ; GISEL-NEXT:    v_readfirstlane_b32 s17, v1
 ; GISEL-NEXT:    v_cmp_eq_u64_e32 vcc, s[16:17], v[0:1]
-; GISEL-NEXT:    s_and_saveexec_b64 s[48:49], vcc
+; GISEL-NEXT:    s_and_saveexec_b64 s[64:65], vcc
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0x7b
-; GISEL-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GISEL-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GISEL-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GISEL-NEXT:    s_mov_b64 s[8:9], s[36:37]
 ; GISEL-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GISEL-NEXT:    s_mov_b32 s12, s45
-; GISEL-NEXT:    s_mov_b32 s13, s44
-; GISEL-NEXT:    s_mov_b32 s14, s43
-; GISEL-NEXT:    s_mov_b32 s15, s42
+; GISEL-NEXT:    s_mov_b32 s12, s53
+; GISEL-NEXT:    s_mov_b32 s13, s52
+; GISEL-NEXT:    s_mov_b32 s14, s51
+; GISEL-NEXT:    s_mov_b32 s15, s50
 ; GISEL-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GISEL-NEXT:    ; implicit-def: $vgpr0
 ; GISEL-NEXT:    ; implicit-def: $vgpr31
-; GISEL-NEXT:    s_xor_b64 exec, exec, s[48:49]
+; GISEL-NEXT:    s_xor_b64 exec, exec, s[64:65]
 ; GISEL-NEXT:    s_cbranch_execnz .LBB3_1
 ; GISEL-NEXT:  ; %bb.2:
-; GISEL-NEXT:    s_mov_b64 exec, s[46:47]
-; GISEL-NEXT:    v_readlane_b32 s49, v40, 17
-; GISEL-NEXT:    v_readlane_b32 s48, v40, 16
-; GISEL-NEXT:    v_readlane_b32 s47, v40, 15
-; GISEL-NEXT:    v_readlane_b32 s46, v40, 14
-; GISEL-NEXT:    v_readlane_b32 s45, v40, 13
-; GISEL-NEXT:    v_readlane_b32 s44, v40, 12
-; GISEL-NEXT:    v_readlane_b32 s43, v40, 11
-; GISEL-NEXT:    v_readlane_b32 s42, v40, 10
-; GISEL-NEXT:    v_readlane_b32 s41, v40, 9
-; GISEL-NEXT:    v_readlane_b32 s40, v40, 8
+; GISEL-NEXT:    s_mov_b64 exec, s[54:55]
+; GISEL-NEXT:    v_readlane_b32 s65, v40, 17
+; GISEL-NEXT:    v_readlane_b32 s64, v40, 16
+; GISEL-NEXT:    v_readlane_b32 s55, v40, 15
+; GISEL-NEXT:    v_readlane_b32 s54, v40, 14
+; GISEL-NEXT:    v_readlane_b32 s53, v40, 13
+; GISEL-NEXT:    v_readlane_b32 s52, v40, 12
+; GISEL-NEXT:    v_readlane_b32 s51, v40, 11
+; GISEL-NEXT:    v_readlane_b32 s50, v40, 10
+; GISEL-NEXT:    v_readlane_b32 s49, v40, 9
+; GISEL-NEXT:    v_readlane_b32 s48, v40, 8
 ; GISEL-NEXT:    v_readlane_b32 s39, v40, 7
 ; GISEL-NEXT:    v_readlane_b32 s38, v40, 6
 ; GISEL-NEXT:    v_readlane_b32 s37, v40, 5
@@ -484,57 +484,57 @@ define i32 @test_indirect_call_vgpr_ptr_ret(ptr %fptr) {
 ; GCN-NEXT:    v_writelane_b32 v40, s37, 5
 ; GCN-NEXT:    v_writelane_b32 v40, s38, 6
 ; GCN-NEXT:    v_writelane_b32 v40, s39, 7
-; GCN-NEXT:    v_writelane_b32 v40, s40, 8
-; GCN-NEXT:    v_writelane_b32 v40, s41, 9
-; GCN-NEXT:    v_writelane_b32 v40, s42, 10
-; GCN-NEXT:    v_writelane_b32 v40, s43, 11
-; GCN-NEXT:    v_writelane_b32 v40, s44, 12
-; GCN-NEXT:    v_writelane_b32 v40, s45, 13
-; GCN-NEXT:    v_writelane_b32 v40, s46, 14
-; GCN-NEXT:    v_writelane_b32 v40, s47, 15
-; GCN-NEXT:    v_writelane_b32 v40, s48, 16
-; GCN-NEXT:    v_writelane_b32 v40, s49, 17
-; GCN-NEXT:    s_mov_b32 s42, s15
-; GCN-NEXT:    s_mov_b32 s43, s14
-; GCN-NEXT:    s_mov_b32 s44, s13
-; GCN-NEXT:    s_mov_b32 s45, s12
+; GCN-NEXT:    v_writelane_b32 v40, s48, 8
+; GCN-NEXT:    v_writelane_b32 v40, s49, 9
+; GCN-NEXT:    v_writelane_b32 v40, s50, 10
+; GCN-NEXT:    v_writelane_b32 v40, s51, 11
+; GCN-NEXT:    v_writelane_b32 v40, s52, 12
+; GCN-NEXT:    v_writelane_b32 v40, s53, 13
+; GCN-NEXT:    v_writelane_b32 v40, s54, 14
+; GCN-NEXT:    v_writelane_b32 v40, s55, 15
+; GCN-NEXT:    v_writelane_b32 v40, s64, 16
+; GCN-NEXT:    v_writelane_b32 v40, s65, 17
+; GCN-NEXT:    s_mov_b32 s50, s15
+; GCN-NEXT:    s_mov_b32 s51, s14
+; GCN-NEXT:    s_mov_b32 s52, s13
+; GCN-NEXT:    s_mov_b32 s53, s12
 ; GCN-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; GCN-NEXT:    s_mov_b64 s[36:37], s[8:9]
 ; GCN-NEXT:    s_mov_b64 s[38:39], s[6:7]
-; GCN-NEXT:    s_mov_b64 s[40:41], s[4:5]
-; GCN-NEXT:    s_mov_b64 s[46:47], exec
+; GCN-NEXT:    s_mov_b64 s[48:49], s[4:5]
+; GCN-NEXT:    s_mov_b64 s[54:55], exec
 ; GCN-NEXT:  .LBB4_1: ; =>This Inner Loop Header: Depth=1
 ; GCN-NEXT:    v_readfirstlane_b32 s16, v0
 ; GCN-NEXT:    v_readfirstlane_b32 s17, v1
 ; GCN-NEXT:    v_cmp_eq_u64_e32 vcc, s[16:17], v[0:1]
-; GCN-NEXT:    s_and_saveexec_b64 s[48:49], vcc
-; GCN-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GCN-NEXT:    s_and_saveexec_b64 s[64:65], vcc
+; GCN-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GCN-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GCN-NEXT:    s_mov_b64 s[8:9], s[36:37]
 ; GCN-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GCN-NEXT:    s_mov_b32 s12, s45
-; GCN-NEXT:    s_mov_b32 s13, s44
-; GCN-NEXT:    s_mov_b32 s14, s43
-; GCN-NEXT:    s_mov_b32 s15, s42
+; GCN-NEXT:    s_mov_b32 s12, s53
+; GCN-NEXT:    s_mov_b32 s13, s52
+; GCN-NEXT:    s_mov_b32 s14, s51
+; GCN-NEXT:    s_mov_b32 s15, s50
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GCN-NEXT:    v_mov_b32_e32 v2, v0
 ; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN-NEXT:    ; implicit-def: $vgpr31
-; GCN-NEXT:    s_xor_b64 exec, exec, s[48:49]
+; GCN-NEXT:    s_xor_b64 exec, exec, s[64:65]
 ; GCN-NEXT:    s_cbranch_execnz .LBB4_1
 ; GCN-NEXT:  ; %bb.2:
-; GCN-NEXT:    s_mov_b64 exec, s[46:47]
+; GCN-NEXT:    s_mov_b64 exec, s[54:55]
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, 1, v2
-; GCN-NEXT:    v_readlane_b32 s49, v40, 17
-; GCN-NEXT:    v_readlane_b32 s48, v40, 16
-; GCN-NEXT:    v_readlane_b32 s47, v40, 15
-; GCN-NEXT:    v_readlane_b32 s46, v40, 14
-; GCN-NEXT:    v_readlane_b32 s45, v40, 13
-; GCN-NEXT:    v_readlane_b32 s44, v40, 12
-; GCN-NEXT:    v_readlane_b32 s43, v40, 11
-; GCN-NEXT:    v_readlane_b32 s42, v40, 10
-; GCN-NEXT:    v_readlane_b32 s41, v40, 9
-; GCN-NEXT:    v_readlane_b32 s40, v40, 8
+; GCN-NEXT:    v_readlane_b32 s65, v40, 17
+; GCN-NEXT:    v_readlane_b32 s64, v40, 16
+; GCN-NEXT:    v_readlane_b32 s55, v40, 15
+; GCN-NEXT:    v_readlane_b32 s54, v40, 14
+; GCN-NEXT:    v_readlane_b32 s53, v40, 13
+; GCN-NEXT:    v_readlane_b32 s52, v40, 12
+; GCN-NEXT:    v_readlane_b32 s51, v40, 11
+; GCN-NEXT:    v_readlane_b32 s50, v40, 10
+; GCN-NEXT:    v_readlane_b32 s49, v40, 9
+; GCN-NEXT:    v_readlane_b32 s48, v40, 8
 ; GCN-NEXT:    v_readlane_b32 s39, v40, 7
 ; GCN-NEXT:    v_readlane_b32 s38, v40, 6
 ; GCN-NEXT:    v_readlane_b32 s37, v40, 5
@@ -570,57 +570,57 @@ define i32 @test_indirect_call_vgpr_ptr_ret(ptr %fptr) {
 ; GISEL-NEXT:    v_writelane_b32 v40, s37, 5
 ; GISEL-NEXT:    v_writelane_b32 v40, s38, 6
 ; GISEL-NEXT:    v_writelane_b32 v40, s39, 7
-; GISEL-NEXT:    v_writelane_b32 v40, s40, 8
-; GISEL-NEXT:    v_writelane_b32 v40, s41, 9
-; GISEL-NEXT:    v_writelane_b32 v40, s42, 10
-; GISEL-NEXT:    v_writelane_b32 v40, s43, 11
-; GISEL-NEXT:    v_writelane_b32 v40, s44, 12
-; GISEL-NEXT:    v_writelane_b32 v40, s45, 13
-; GISEL-NEXT:    v_writelane_b32 v40, s46, 14
-; GISEL-NEXT:    v_writelane_b32 v40, s47, 15
-; GISEL-NEXT:    v_writelane_b32 v40, s48, 16
-; GISEL-NEXT:    v_writelane_b32 v40, s49, 17
-; GISEL-NEXT:    s_mov_b32 s42, s15
-; GISEL-NEXT:    s_mov_b32 s43, s14
-; GISEL-NEXT:    s_mov_b32 s44, s13
-; GISEL-NEXT:    s_mov_b32 s45, s12
+; GISEL-NEXT:    v_writelane_b32 v40, s48, 8
+; GISEL-NEXT:    v_writelane_b32 v40, s49, 9
+; GISEL-NEXT:    v_writelane_b32 v40, s50, 10
+; GISEL-NEXT:    v_writelane_b32 v40, s51, 11
+; GISEL-NEXT:    v_writelane_b32 v40, s52, 12
+; GISEL-NEXT:    v_writelane_b32 v40, s53, 13
+; GISEL-NEXT:    v_writelane_b32 v40, s54, 14
+; GISEL-NEXT:    v_writelane_b32 v40, s55, 15
+; GISEL-NEXT:    v_writelane_b32 v40, s64, 16
+; GISEL-NEXT:    v_writelane_b32 v40, s65, 17
+; GISEL-NEXT:    s_mov_b32 s50, s15
+; GISEL-NEXT:    s_mov_b32 s51, s14
+; GISEL-NEXT:    s_mov_b32 s52, s13
+; GISEL-NEXT:    s_mov_b32 s53, s12
 ; GISEL-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; GISEL-NEXT:    s_mov_b64 s[36:37], s[8:9]
 ; GISEL-NEXT:    s_mov_b64 s[38:39], s[6:7]
-; GISEL-NEXT:    s_mov_b64 s[40:41], s[4:5]
-; GISEL-NEXT:    s_mov_b64 s[46:47], exec
+; GISEL-NEXT:    s_mov_b64 s[48:49], s[4:5]
+; GISEL-NEXT:    s_mov_b64 s[54:55], exec
 ; GISEL-NEXT:  .LBB4_1: ; =>This Inner Loop Header: Depth=1
 ; GISEL-NEXT:    v_readfirstlane_b32 s16, v0
 ; GISEL-NEXT:    v_readfirstlane_b32 s17, v1
 ; GISEL-NEXT:    v_cmp_eq_u64_e32 vcc, s[16:17], v[0:1]
-; GISEL-NEXT:    s_and_saveexec_b64 s[48:49], vcc
-; GISEL-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GISEL-NEXT:    s_and_saveexec_b64 s[64:65], vcc
+; GISEL-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GISEL-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GISEL-NEXT:    s_mov_b64 s[8:9], s[36:37]
 ; GISEL-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GISEL-NEXT:    s_mov_b32 s12, s45
-; GISEL-NEXT:    s_mov_b32 s13, s44
-; GISEL-NEXT:    s_mov_b32 s14, s43
-; GISEL-NEXT:    s_mov_b32 s15, s42
+; GISEL-NEXT:    s_mov_b32 s12, s53
+; GISEL-NEXT:    s_mov_b32 s13, s52
+; GISEL-NEXT:    s_mov_b32 s14, s51
+; GISEL-NEXT:    s_mov_b32 s15, s50
 ; GISEL-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GISEL-NEXT:    v_mov_b32_e32 v1, v0
 ; GISEL-NEXT:    ; implicit-def: $vgpr0
 ; GISEL-NEXT:    ; implicit-def: $vgpr31
-; GISEL-NEXT:    s_xor_b64 exec, exec, s[48:49]
+; GISEL-NEXT:    s_xor_b64 exec, exec, s[64:65]
 ; GISEL-NEXT:    s_cbranch_execnz .LBB4_1
 ; GISEL-NEXT:  ; %bb.2:
-; GISEL-NEXT:    s_mov_b64 exec, s[46:47]
+; GISEL-NEXT:    s_mov_b64 exec, s[54:55]
 ; GISEL-NEXT:    v_add_i32_e32 v0, vcc, 1, v1
-; GISEL-NEXT:    v_readlane_b32 s49, v40, 17
-; GISEL-NEXT:    v_readlane_b32 s48, v40, 16
-; GISEL-NEXT:    v_readlane_b32 s47, v40, 15
-; GISEL-NEXT:    v_readlane_b32 s46, v40, 14
-; GISEL-NEXT:    v_readlane_b32 s45, v40, 13
-; GISEL-NEXT:    v_readlane_b32 s44, v40, 12
-; GISEL-NEXT:    v_readlane_b32 s43, v40, 11
-; GISEL-NEXT:    v_readlane_b32 s42, v40, 10
-; GISEL-NEXT:    v_readlane_b32 s41, v40, 9
-; GISEL-NEXT:    v_readlane_b32 s40, v40, 8
+; GISEL-NEXT:    v_readlane_b32 s65, v40, 17
+; GISEL-NEXT:    v_readlane_b32 s64, v40, 16
+; GISEL-NEXT:    v_readlane_b32 s55, v40, 15
+; GISEL-NEXT:    v_readlane_b32 s54, v40, 14
+; GISEL-NEXT:    v_readlane_b32 s53, v40, 13
+; GISEL-NEXT:    v_readlane_b32 s52, v40, 12
+; GISEL-NEXT:    v_readlane_b32 s51, v40, 11
+; GISEL-NEXT:    v_readlane_b32 s50, v40, 10
+; GISEL-NEXT:    v_readlane_b32 s49, v40, 9
+; GISEL-NEXT:    v_readlane_b32 s48, v40, 8
 ; GISEL-NEXT:    v_readlane_b32 s39, v40, 7
 ; GISEL-NEXT:    v_readlane_b32 s38, v40, 6
 ; GISEL-NEXT:    v_readlane_b32 s37, v40, 5
@@ -661,66 +661,66 @@ define void @test_indirect_call_vgpr_ptr_in_branch(ptr %fptr, i1 %cond) {
 ; GCN-NEXT:    v_writelane_b32 v40, s37, 5
 ; GCN-NEXT:    v_writelane_b32 v40, s38, 6
 ; GCN-NEXT:    v_writelane_b32 v40, s39, 7
-; GCN-NEXT:    v_writelane_b32 v40, s40, 8
-; GCN-NEXT:    v_writelane_b32 v40, s41, 9
-; GCN-NEXT:    v_writelane_b32 v40, s42, 10
-; GCN-NEXT:    v_writelane_b32 v40, s43, 11
-; GCN-NEXT:    v_writelane_b32 v40, s44, 12
-; GCN-NEXT:    v_writelane_b32 v40, s45, 13
-; GCN-NEXT:    v_writelane_b32 v40, s46, 14
-; GCN-NEXT:    v_writelane_b32 v40, s47, 15
-; GCN-NEXT:    v_writelane_b32 v40, s48, 16
-; GCN-NEXT:    v_writelane_b32 v40, s49, 17
-; GCN-NEXT:    v_writelane_b32 v40, s50, 18
-; GCN-NEXT:    v_writelane_b32 v40, s51, 19
-; GCN-NEXT:    s_mov_b32 s42, s15
-; GCN-NEXT:    s_mov_b32 s43, s14
-; GCN-NEXT:    s_mov_b32 s44, s13
-; GCN-NEXT:    s_mov_b32 s45, s12
+; GCN-NEXT:    v_writelane_b32 v40, s48, 8
+; GCN-NEXT:    v_writelane_b32 v40, s49, 9
+; GCN-NEXT:    v_writelane_b32 v40, s50, 10
+; GCN-NEXT:    v_writelane_b32 v40, s51, 11
+; GCN-NEXT:    v_writelane_b32 v40, s52, 12
+; GCN-NEXT:    v_writelane_b32 v40, s53, 13
+; GCN-NEXT:    v_writelane_b32 v40, s54, 14
+; GCN-NEXT:    v_writelane_b32 v40, s55, 15
+; GCN-NEXT:    v_writelane_b32 v40, s64, 16
+; GCN-NEXT:    v_writelane_b32 v40, s65, 17
+; GCN-NEXT:    v_writelane_b32 v40, s66, 18
+; GCN-NEXT:    v_writelane_b32 v40, s67, 19
+; GCN-NEXT:    s_mov_b32 s50, s15
+; GCN-NEXT:    s_mov_b32 s51, s14
+; GCN-NEXT:    s_mov_b32 s52, s13
+; GCN-NEXT:    s_mov_b32 s53, s12
 ; GCN-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; GCN-NEXT:    s_mov_b64 s[36:37], s[8:9]
 ; GCN-NEXT:    s_mov_b64 s[38:39], s[6:7]
-; GCN-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; GCN-NEXT:    s_mov_b64 s[48:49], s[4:5]
 ; GCN-NEXT:    v_and_b32_e32 v2, 1, v2
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GCN-NEXT:    s_and_saveexec_b64 s[46:47], vcc
+; GCN-NEXT:    s_and_saveexec_b64 s[54:55], vcc
 ; GCN-NEXT:    s_cbranch_execz .LBB5_4
 ; GCN-NEXT:  ; %bb.1: ; %bb1
-; GCN-NEXT:    s_mov_b64 s[48:49], exec
+; GCN-NEXT:    s_mov_b64 s[64:65], exec
 ; GCN-NEXT:  .LBB5_2: ; =>This Inner Loop Header: Depth=1
 ; GCN-NEXT:    v_readfirstlane_b32 s16, v0
 ; GCN-NEXT:    v_readfirstlane_b32 s17, v1
 ; GCN-NEXT:    v_cmp_eq_u64_e32 vcc, s[16:17], v[0:1]
-; GCN-NEXT:    s_and_saveexec_b64 s[50:51], vcc
-; GCN-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GCN-NEXT:    s_and_saveexec_b64 s[66:67], vcc
+; GCN-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GCN-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GCN-NEXT:    s_mov_b64 s[8:9], s[36:37]
 ; GCN-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GCN-NEXT:    s_mov_b32 s12, s45
-; GCN-NEXT:    s_mov_b32 s13, s44
-; GCN-NEXT:    s_mov_b32 s14, s43
-; GCN-NEXT:    s_mov_b32 s15, s42
+; GCN-NEXT:    s_mov_b32 s12, s53
+; GCN-NEXT:    s_mov_b32 s13, s52
+; GCN-NEXT:    s_mov_b32 s14, s51
+; GCN-NEXT:    s_mov_b32 s15, s50
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN-NEXT:    ; implicit-def: $vgpr31
-; GCN-NEXT:    s_xor_b64 exec, exec, s[50:51]
+; GCN-NEXT:    s_xor_b64 exec, exec, s[66:67]
 ; GCN-NEXT:    s_cbranch_execnz .LBB5_2
 ; GCN-NEXT:  ; %bb.3:
-; GCN-NEXT:    s_mov_b64 exec, s[48:49]
+; GCN-NEXT:    s_mov_b64 exec, s[64:65]
 ; GCN-NEXT:  .LBB5_4: ; %bb2
-; GCN-NEXT:    s_or_b64 exec, exec, s[46:47]
-; GCN-NEXT:    v_readlane_b32 s51, v40, 19
-; GCN-NEXT:    v_readlane_b32 s50, v40, 18
-; GCN-NEXT:    v_readlane_b32 s49, v40, 17
-; GCN-NEXT:    v_readlane_b32 s48, v40, 16
-; GCN-NEXT:    v_readlane_b32 s47, v40, 15
-; GCN-NEXT:    v_readlane_b32 s46, v40, 14
-; GCN-NEXT:    v_readlane_b32 s45, v40, 13
-; GCN-NEXT:    v_readlane_b32 s44, v40, 12
-; GCN-NEXT:    v_readlane_b32 s43, v40, 11
-; GCN-NEXT:    v_readlane_b32 s42, v40, 10
-; GCN-NEXT:    v_readlane_b32 s41, v40, 9
-; GCN-NEXT:    v_readlane_b32 s40, v40, 8
+; GCN-NEXT:    s_or_b64 exec, exec, s[54:55]
+; GCN-NEXT:    v_readlane_b32 s67, v40, 19
+; GCN-NEXT:    v_readlane_b32 s66, v40, 18
+; GCN-NEXT:    v_readlane_b32 s65, v40, 17
+; GCN-NEXT:    v_readlane_b32 s64, v40, 16
+; GCN-NEXT:    v_readlane_b32 s55, v40, 15
+; GCN-NEXT:    v_readlane_b32 s54, v40, 14
+; GCN-NEXT:    v_readlane_b32 s53, v40, 13
+; GCN-NEXT:    v_readlane_b32 s52, v40, 12
+; GCN-NEXT:    v_readlane_b32 s51, v40, 11
+; GCN-NEXT:    v_readlane_b32 s50, v40, 10
+; GCN-NEXT:    v_readlane_b32 s49, v40, 9
+; GCN-NEXT:    v_readlane_b32 s48, v40, 8
 ; GCN-NEXT:    v_readlane_b32 s39, v40, 7
 ; GCN-NEXT:    v_readlane_b32 s38, v40, 6
 ; GCN-NEXT:    v_readlane_b32 s37, v40, 5
@@ -756,66 +756,66 @@ define void @test_indirect_call_vgpr_ptr_in_branch(ptr %fptr, i1 %cond) {
 ; GISEL-NEXT:    v_writelane_b32 v40, s37, 5
 ; GISEL-NEXT:    v_writelane_b32 v40, s38, 6
 ; GISEL-NEXT:    v_writelane_b32 v40, s39, 7
-; GISEL-NEXT:    v_writelane_b32 v40, s40, 8
-; GISEL-NEXT:    v_writelane_b32 v40, s41, 9
-; GISEL-NEXT:    v_writelane_b32 v40, s42, 10
-; GISEL-NEXT:    v_writelane_b32 v40, s43, 11
-; GISEL-NEXT:    v_writelane_b32 v40, s44, 12
-; GISEL-NEXT:    v_writelane_b32 v40, s45, 13
-; GISEL-NEXT:    v_writelane_b32 v40, s46, 14
-; GISEL-NEXT:    v_writelane_b32 v40, s47, 15
-; GISEL-NEXT:    v_writelane_b32 v40, s48, 16
-; GISEL-NEXT:    v_writelane_b32 v40, s49, 17
-; GISEL-NEXT:    v_writelane_b32 v40, s50, 18
-; GISEL-NEXT:    v_writelane_b32 v40, s51, 19
-; GISEL-NEXT:    s_mov_b32 s42, s15
-; GISEL-NEXT:    s_mov_b32 s43, s14
-; GISEL-NEXT:    s_mov_b32 s44, s13
-; GISEL-NEXT:    s_mov_b32 s45, s12
+; GISEL-NEXT:    v_writelane_b32 v40, s48, 8
+; GISEL-NEXT:    v_writelane_b32 v40, s49, 9
+; GISEL-NEXT:    v_writelane_b32 v40, s50, 10
+; GISEL-NEXT:    v_writelane_b32 v40, s51, 11
+; GISEL-NEXT:    v_writelane_b32 v40, s52, 12
+; GISEL-NEXT:    v_writelane_b32 v40, s53, 13
+; GISEL-NEXT:    v_writelane_b32 v40, s54, 14
+; GISEL-NEXT:    v_writelane_b32 v40, s55, 15
+; GISEL-NEXT:    v_writelane_b32 v40, s64, 16
+; GISEL-NEXT:    v_writelane_b32 v40, s65, 17
+; GISEL-NEXT:    v_writelane_b32 v40, s66, 18
+; GISEL-NEXT:    v_writelane_b32 v40, s67, 19
+; GISEL-NEXT:    s_mov_b32 s50, s15
+; GISEL-NEXT:    s_mov_b32 s51, s14
+; GISEL-NEXT:    s_mov_b32 s52, s13
+; GISEL-NEXT:    s_mov_b32 s53, s12
 ; GISEL-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; GISEL-NEXT:    s_mov_b64 s[36:37], s[8:9]
 ; GISEL-NEXT:    s_mov_b64 s[38:39], s[6:7]
-; GISEL-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; GISEL-NEXT:    s_mov_b64 s[48:49], s[4:5]
 ; GISEL-NEXT:    v_and_b32_e32 v2, 1, v2
 ; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; GISEL-NEXT:    s_and_saveexec_b64 s[46:47], vcc
+; GISEL-NEXT:    s_and_saveexec_b64 s[54:55], vcc
 ; GISEL-NEXT:    s_cbranch_execz .LBB5_4
 ; GISEL-NEXT:  ; %bb.1: ; %bb1
-; GISEL-NEXT:    s_mov_b64 s[48:49], exec
+; GISEL-NEXT:    s_mov_b64 s[64:65], exec
 ; GISEL-NEXT:  .LBB5_2: ; =>This Inner Loop Header: Depth=1
 ; GISEL-NEXT:    v_readfirstlane_b32 s16, v0
 ; GISEL-NEXT:    v_readfirstlane_b32 s17, v1
 ; GISEL-NEXT:    v_cmp_eq_u64_e32 vcc, s[16:17], v[0:1]
-; GISEL-NEXT:    s_and_saveexec_b64 s[50:51], vcc
-; GISEL-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GISEL-NEXT:    s_and_saveexec_b64 s[66:67], vcc
+; GISEL-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GISEL-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GISEL-NEXT:    s_mov_b64 s[8:9], s[36:37]
 ; GISEL-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GISEL-NEXT:    s_mov_b32 s12, s45
-; GISEL-NEXT:    s_mov_b32 s13, s44
-; GISEL-NEXT:    s_mov_b32 s14, s43
-; GISEL-NEXT:    s_mov_b32 s15, s42
+; GISEL-NEXT:    s_mov_b32 s12, s53
+; GISEL-NEXT:    s_mov_b32 s13, s52
+; GISEL-NEXT:    s_mov_b32 s14, s51
+; GISEL-NEXT:    s_mov_b32 s15, s50
 ; GISEL-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GISEL-NEXT:    ; implicit-def: $vgpr0
 ; GISEL-NEXT:    ; implicit-def: $vgpr31
-; GISEL-NEXT:    s_xor_b64 exec, exec, s[50:51]
+; GISEL-NEXT:    s_xor_b64 exec, exec, s[66:67]
 ; GISEL-NEXT:    s_cbranch_execnz .LBB5_2
 ; GISEL-NEXT:  ; %bb.3:
-; GISEL-NEXT:    s_mov_b64 exec, s[48:49]
+; GISEL-NEXT:    s_mov_b64 exec, s[64:65]
 ; GISEL-NEXT:  .LBB5_4: ; %bb2
-; GISEL-NEXT:    s_or_b64 exec, exec, s[46:47]
-; GISEL-NEXT:    v_readlane_b32 s51, v40, 19
-; GISEL-NEXT:    v_readlane_b32 s50, v40, 18
-; GISEL-NEXT:    v_readlane_b32 s49, v40, 17
-; GISEL-NEXT:    v_readlane_b32 s48, v40, 16
-; GISEL-NEXT:    v_readlane_b32 s47, v40, 15
-; GISEL-NEXT:    v_readlane_b32 s46, v40, 14
-; GISEL-NEXT:    v_readlane_b32 s45, v40, 13
-; GISEL-NEXT:    v_readlane_b32 s44, v40, 12
-; GISEL-NEXT:    v_readlane_b32 s43, v40, 11
-; GISEL-NEXT:    v_readlane_b32 s42, v40, 10
-; GISEL-NEXT:    v_readlane_b32 s41, v40, 9
-; GISEL-NEXT:    v_readlane_b32 s40, v40, 8
+; GISEL-NEXT:    s_or_b64 exec, exec, s[54:55]
+; GISEL-NEXT:    v_readlane_b32 s67, v40, 19
+; GISEL-NEXT:    v_readlane_b32 s66, v40, 18
+; GISEL-NEXT:    v_readlane_b32 s65, v40, 17
+; GISEL-NEXT:    v_readlane_b32 s64, v40, 16
+; GISEL-NEXT:    v_readlane_b32 s55, v40, 15
+; GISEL-NEXT:    v_readlane_b32 s54, v40, 14
+; GISEL-NEXT:    v_readlane_b32 s53, v40, 13
+; GISEL-NEXT:    v_readlane_b32 s52, v40, 12
+; GISEL-NEXT:    v_readlane_b32 s51, v40, 11
+; GISEL-NEXT:    v_readlane_b32 s50, v40, 10
+; GISEL-NEXT:    v_readlane_b32 s49, v40, 9
+; GISEL-NEXT:    v_readlane_b32 s48, v40, 8
 ; GISEL-NEXT:    v_readlane_b32 s39, v40, 7
 ; GISEL-NEXT:    v_readlane_b32 s38, v40, 6
 ; GISEL-NEXT:    v_readlane_b32 s37, v40, 5
@@ -861,30 +861,14 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) {
 ; GCN-NEXT:    v_writelane_b32 v40, s37, 5
 ; GCN-NEXT:    v_writelane_b32 v40, s38, 6
 ; GCN-NEXT:    v_writelane_b32 v40, s39, 7
-; GCN-NEXT:    v_writelane_b32 v40, s40, 8
-; GCN-NEXT:    v_writelane_b32 v40, s41, 9
-; GCN-NEXT:    v_writelane_b32 v40, s42, 10
-; GCN-NEXT:    v_writelane_b32 v40, s43, 11
-; GCN-NEXT:    v_writelane_b32 v40, s44, 12
-; GCN-NEXT:    v_writelane_b32 v40, s45, 13
-; GCN-NEXT:    v_writelane_b32 v40, s46, 14
-; GCN-NEXT:    v_writelane_b32 v40, s47, 15
-; GCN-NEXT:    v_writelane_b32 v40, s48, 16
-; GCN-NEXT:    v_writelane_b32 v40, s49, 17
-; GCN-NEXT:    v_writelane_b32 v40, s50, 18
-; GCN-NEXT:    v_writelane_b32 v40, s51, 19
-; GCN-NEXT:    v_writelane_b32 v40, s52, 20
-; GCN-NEXT:    v_writelane_b32 v40, s53, 21
-; GCN-NEXT:    v_writelane_b32 v40, s54, 22
-; GCN-NEXT:    v_writelane_b32 v40, s55, 23
-; GCN-NEXT:    v_writelane_b32 v40, s56, 24
-; GCN-NEXT:    v_writelane_b32 v40, s57, 25
-; GCN-NEXT:    v_writelane_b32 v40, s58, 26
-; GCN-NEXT:    v_writelane_b32 v40, s59, 27
-; GCN-NEXT:    v_writelane_b32 v40, s60, 28
-; GCN-NEXT:    v_writelane_b32 v40, s61, 29
-; GCN-NEXT:    v_writelane_b32 v40, s62, 30
-; GCN-NEXT:    v_writelane_b32 v40, s63, 31
+; GCN-NEXT:    v_writelane_b32 v40, s48, 8
+; GCN-NEXT:    v_writelane_b32 v40, s49, 9
+; GCN-NEXT:    v_writelane_b32 v40, s50, 10
+; GCN-NEXT:    v_writelane_b32 v40, s51, 11
+; GCN-NEXT:    v_writelane_b32 v40, s52, 12
+; GCN-NEXT:    v_writelane_b32 v40, s53, 13
+; GCN-NEXT:    v_writelane_b32 v40, s54, 14
+; GCN-NEXT:    v_writelane_b32 v40, s55, 15
 ; GCN-NEXT:    s_mov_b64 s[6:7], exec
 ; GCN-NEXT:  .LBB6_1: ; =>This Inner Loop Header: Depth=1
 ; GCN-NEXT:    v_readfirstlane_b32 s8, v0
@@ -898,30 +882,14 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) {
 ; GCN-NEXT:    s_cbranch_execnz .LBB6_1
 ; GCN-NEXT:  ; %bb.2:
 ; GCN-NEXT:    s_mov_b64 exec, s[6:7]
-; GCN-NEXT:    v_readlane_b32 s63, v40, 31
-; GCN-NEXT:    v_readlane_b32 s62, v40, 30
-; GCN-NEXT:    v_readlane_b32 s61, v40, 29
-; GCN-NEXT:    v_readlane_b32 s60, v40, 28
-; GCN-NEXT:    v_readlane_b32 s59, v40, 27
-; GCN-NEXT:    v_readlane_b32 s58, v40, 26
-; GCN-NEXT:    v_readlane_b32 s57, v40, 25
-; GCN-NEXT:    v_readlane_b32 s56, v40, 24
-; GCN-NEXT:    v_readlane_b32 s55, v40, 23
-; GCN-NEXT:    v_readlane_b32 s54, v40, 22
-; GCN-NEXT:    v_readlane_b32 s53, v40, 21
-; GCN-NEXT:    v_readlane_b32 s52, v40, 20
-; GCN-NEXT:    v_readlane_b32 s51, v40, 19
-; GCN-NEXT:    v_readlane_b32 s50, v40, 18
-; GCN-NEXT:    v_readlane_b32 s49, v40, 17
-; GCN-NEXT:    v_readlane_b32 s48, v40, 16
-; GCN-NEXT:    v_readlane_b32 s47, v40, 15
-; GCN-NEXT:    v_readlane_b32 s46, v40, 14
-; GCN-NEXT:    v_readlane_b32 s45, v40, 13
-; GCN-NEXT:    v_readlane_b32 s44, v40, 12
-; GCN-NEXT:    v_readlane_b32 s43, v40, 11
-; GCN-NEXT:    v_readlane_b32 s42, v40, 10
-; GCN-NEXT:    v_readlane_b32 s41, v40, 9
-; GCN-NEXT:    v_readlane_b32 s40, v40, 8
+; GCN-NEXT:    v_readlane_b32 s55, v40, 15
+; GCN-NEXT:    v_readlane_b32 s54, v40, 14
+; GCN-NEXT:    v_readlane_b32 s53, v40, 13
+; GCN-NEXT:    v_readlane_b32 s52, v40, 12
+; GCN-NEXT:    v_readlane_b32 s51, v40, 11
+; GCN-NEXT:    v_readlane_b32 s50, v40, 10
+; GCN-NEXT:    v_readlane_b32 s49, v40, 9
+; GCN-NEXT:    v_readlane_b32 s48, v40, 8
 ; GCN-NEXT:    v_readlane_b32 s39, v40, 7
 ; GCN-NEXT:    v_readlane_b32 s38, v40, 6
 ; GCN-NEXT:    v_readlane_b32 s37, v40, 5
@@ -955,30 +923,14 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) {
 ; GISEL-NEXT:    v_writelane_b32 v40, s37, 5
 ; GISEL-NEXT:    v_writelane_b32 v40, s38, 6
 ; GISEL-NEXT:    v_writelane_b32 v40, s39, 7
-; GISEL-NEXT:    v_writelane_b32 v40, s40, 8
-; GISEL-NEXT:    v_writelane_b32 v40, s41, 9
-; GISEL-NEXT:    v_writelane_b32 v40, s42, 10
-; GISEL-NEXT:    v_writelane_b32 v40, s43, 11
-; GISEL-NEXT:    v_writelane_b32 v40, s44, 12
-; GISEL-NEXT:    v_writelane_b32 v40, s45, 13
-; GISEL-NEXT:    v_writelane_b32 v40, s46, 14
-; GISEL-NEXT:    v_writelane_b32 v40, s47, 15
-; GISEL-NEXT:    v_writelane_b32 v40, s48, 16
-; GISEL-NEXT:    v_writelane_b32 v40, s49, 17
-; GISEL-NEXT:    v_writelane_b32 v40, s50, 18
-; GISEL-NEXT:    v_writelane_b32 v40, s51, 19
-; GISEL-NEXT:    v_writelane_b32 v40, s52, 20
-; GISEL-NEXT:    v_writelane_b32 v40, s53, 21
-; GISEL-NEXT:    v_writelane_b32 v40, s54, 22
-; GISEL-NEXT:    v_writelane_b32 v40, s55, 23
-; GISEL-NEXT:    v_writelane_b32 v40, s56, 24
-; GISEL-NEXT:    v_writelane_b32 v40, s57, 25
-; GISEL-NEXT:    v_writelane_b32 v40, s58, 26
-; GISEL-NEXT:    v_writelane_b32 v40, s59, 27
-; GISEL-NEXT:    v_writelane_b32 v40, s60, 28
-; GISEL-NEXT:    v_writelane_b32 v40, s61, 29
-; GISEL-NEXT:    v_writelane_b32 v40, s62, 30
-; GISEL-NEXT:    v_writelane_b32 v40, s63, 31
+; GISEL-NEXT:    v_writelane_b32 v40, s48, 8
+; GISEL-NEXT:    v_writelane_b32 v40, s49, 9
+; GISEL-NEXT:    v_writelane_b32 v40, s50, 10
+; GISEL-NEXT:    v_writelane_b32 v40, s51, 11
+; GISEL-NEXT:    v_writelane_b32 v40, s52, 12
+; GISEL-NEXT:    v_writelane_b32 v40, s53, 13
+; GISEL-NEXT:    v_writelane_b32 v40, s54, 14
+; GISEL-NEXT:    v_writelane_b32 v40, s55, 15
 ; GISEL-NEXT:    s_mov_b64 s[6:7], exec
 ; GISEL-NEXT:  .LBB6_1: ; =>This Inner Loop Header: Depth=1
 ; GISEL-NEXT:    v_readfirstlane_b32 s8, v0
@@ -992,30 +944,14 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) {
 ; GISEL-NEXT:    s_cbranch_execnz .LBB6_1
 ; GISEL-NEXT:  ; %bb.2:
 ; GISEL-NEXT:    s_mov_b64 exec, s[6:7]
-; GISEL-NEXT:    v_readlane_b32 s63, v40, 31
-; GISEL-NEXT:    v_readlane_b32 s62, v40, 30
-; GISEL-NEXT:    v_readlane_b32 s61, v40, 29
-; GISEL-NEXT:    v_readlane_b32 s60, v40, 28
-; GISEL-NEXT:    v_readlane_b32 s59, v40, 27
-; GISEL-NEXT:    v_readlane_b32 s58, v40, 26
-; GISEL-NEXT:    v_readlane_b32 s57, v40, 25
-; GISEL-NEXT:    v_readlane_b32 s56, v40, 24
-; GISEL-NEXT:    v_readlane_b32 s55, v40, 23
-; GISEL-NEXT:    v_readlane_b32 s54, v40, 22
-; GISEL-NEXT:    v_readlane_b32 s53, v40, 21
-; GISEL-NEXT:    v_readlane_b32 s52, v40, 20
-; GISEL-NEXT:    v_readlane_b32 s51, v40, 19
-; GISEL-NEXT:    v_readlane_b32 s50, v40, 18
-; GISEL-NEXT:    v_readlane_b32 s49, v40, 17
-; GISEL-NEXT:    v_readlane_b32 s48, v40, 16
-; GISEL-NEXT:    v_readlane_b32 s47, v40, 15
-; GISEL-NEXT:    v_readlane_b32 s46, v40, 14
-; GISEL-NEXT:    v_readlane_b32 s45, v40, 13
-; GISEL-NEXT:    v_readlane_b32 s44, v40, 12
-; GISEL-NEXT:    v_readlane_b32 s43, v40, 11
-; GISEL-NEXT:    v_readlane_b32 s42, v40, 10
-; GISEL-NEXT:    v_readlane_b32 s41, v40, 9
-; GISEL-NEXT:    v_readlane_b32 s40, v40, 8
+; GISEL-NEXT:    v_readlane_b32 s55, v40, 15
+; GISEL-NEXT:    v_readlane_b32 s54, v40, 14
+; GISEL-NEXT:    v_readlane_b32 s53, v40, 13
+; GISEL-NEXT:    v_readlane_b32 s52, v40, 12
+; GISEL-NEXT:    v_readlane_b32 s51, v40, 11
+; GISEL-NEXT:    v_readlane_b32 s50, v40, 10
+; GISEL-NEXT:    v_readlane_b32 s49, v40, 9
+; GISEL-NEXT:    v_readlane_b32 s48, v40, 8
 ; GISEL-NEXT:    v_readlane_b32 s39, v40, 7
 ; GISEL-NEXT:    v_readlane_b32 s38, v40, 6
 ; GISEL-NEXT:    v_readlane_b32 s37, v40, 5
@@ -1054,30 +990,14 @@ define i32 @test_indirect_call_vgpr_ptr_arg_and_reuse(i32 %i, ptr %fptr) {
 ; GCN-NEXT:    v_writelane_b32 v41, s37, 5
 ; GCN-NEXT:    v_writelane_b32 v41, s38, 6
 ; GCN-NEXT:    v_writelane_b32 v41, s39, 7
-; GCN-NEXT:    v_writelane_b32 v41, s40, 8
-; GCN-NEXT:    v_writelane_b32 v41, s41, 9
-; GCN-NEXT:    v_writelane_b32 v41, s42, 10
-; GCN-NEXT:    v_writelane_b32 v41, s43, 11
-; GCN-NEXT:    v_writelane_b32 v41, s44, 12
-; GCN-NEXT:    v_writelane_b32 v41, s45, 13
-; GCN-NEXT:    v_writelane_b32 v41, s46, 14
-; GCN-NEXT:    v_writelane_b32 v41, s47, 15
-; GCN-NEXT:    v_writelane_b32 v41, s48, 16
-; GCN-NEXT:    v_writelane_b32 v41, s49, 17
-; GCN-NEXT:    v_writelane_b32 v41, s50, 18
-; GCN-NEXT:    v_writelane_b32 v41, s51, 19
-; GCN-NEXT:    v_writelane_b32 v41, s52, 20
-; GCN-NEXT:    v_writelane_b32 v41, s53, 21
-; GCN-NEXT:    v_writelane_b32 v41, s54, 22
-; GCN-NEXT:    v_writelane_b32 v41, s55, 23
-; GCN-NEXT:    v_writelane_b32 v41, s56, 24
-; GCN-NEXT:    v_writelane_b32 v41, s57, 25
-; GCN-NEXT:    v_writelane_b32 v41, s58, 26
-; GCN-NEXT:    v_writelane_b32 v41, s59, 27
-; GCN-NEXT:    v_writelane_b32 v41, s60, 28
-; GCN-NEXT:    v_writelane_b32 v41, s61, 29
-; GCN-NEXT:    v_writelane_b32 v41, s62, 30
-; GCN-NEXT:    v_writelane_b32 v41, s63, 31
+; GCN-NEXT:    v_writelane_b32 v41, s48, 8
+; GCN-NEXT:    v_writelane_b32 v41, s49, 9
+; GCN-NEXT:    v_writelane_b32 v41, s50, 10
+; GCN-NEXT:    v_writelane_b32 v41, s51, 11
+; GCN-NEXT:    v_writelane_b32 v41, s52, 12
+; GCN-NEXT:    v_writelane_b32 v41, s53, 13
+; GCN-NEXT:    v_writelane_b32 v41, s54, 14
+; GCN-NEXT:    v_writelane_b32 v41, s55, 15
 ; GCN-NEXT:    v_mov_b32_e32 v40, v0
 ; GCN-NEXT:    s_mov_b64 s[4:5], exec
 ; GCN-NEXT:  .LBB7_1: ; =>This Inner Loop Header: Depth=1
@@ -1093,30 +1013,14 @@ define i32 @test_indirect_call_vgpr_ptr_arg_and_reuse(i32 %i, ptr %fptr) {
 ; GCN-NEXT:  ; %bb.2:
 ; GCN-NEXT:    s_mov_b64 exec, s[4:5]
 ; GCN-NEXT:    v_mov_b32_e32 v0, v40
-; GCN-NEXT:    v_readlane_b32 s63, v41, 31
-; GCN-NEXT:    v_readlane_b32 s62, v41, 30
-; GCN-NEXT:    v_readlane_b32 s61, v41, 29
-; GCN-NEXT:    v_readlane_b32 s60, v41, 28
-; GCN-NEXT:    v_readlane_b32 s59, v41, 27
-; GCN-NEXT:    v_readlane_b32 s58, v41, 26
-; GCN-NEXT:    v_readlane_b32 s57, v41, 25
-; GCN-NEXT:    v_readlane_b32 s56, v41, 24
-; GCN-NEXT:    v_readlane_b32 s55, v41, 23
-; GCN-NEXT:    v_readlane_b32 s54, v41, 22
-; GCN-NEXT:    v_readlane_b32 s53, v41, 21
-; GCN-NEXT:    v_readlane_b32 s52, v41, 20
-; GCN-NEXT:    v_readlane_b32 s51, v41, 19
-; GCN-NEXT:    v_readlane_b32 s50, v41, 18
-; GCN-NEXT:    v_readlane_b32 s49, v41, 17
-; GCN-NEXT:    v_readlane_b32 s48, v41, 16
-; GCN-NEXT:    v_readlane_b32 s47, v41, 15
-; GCN-NEXT:    v_readlane_b32 s46, v41, 14
-; GCN-NEXT:    v_readlane_b32 s45, v41, 13
-; GCN-NEXT:    v_readlane_b32 s44, v41, 12
-; GCN-NEXT:    v_readlane_b32 s43, v41, 11
-; GCN-NEXT:    v_readlane_b32 s42, v41, 10
-; GCN-NEXT:    v_readlane_b32 s41, v41, 9
-; GCN-NEXT:    v_readlane_b32 s40, v41, 8
+; GCN-NEXT:    v_readlane_b32 s55, v41, 15
+; GCN-NEXT:    v_readlane_b32 s54, v41, 14
+; GCN-NEXT:    v_readlane_b32 s53, v41, 13
+; GCN-NEXT:    v_readlane_b32 s52, v41, 12
+; GCN-NEXT:    v_readlane_b32 s51, v41, 11
+; GCN-NEXT:    v_readlane_b32 s50, v41, 10
+; GCN-NEXT:    v_readlane_b32 s49, v41, 9
+; GCN-NEXT:    v_readlane_b32 s48, v41, 8
 ; GCN-NEXT:    v_readlane_b32 s39, v41, 7
 ; GCN-NEXT:    v_readlane_b32 s38, v41, 6
 ; GCN-NEXT:    v_readlane_b32 s37, v41, 5
@@ -1152,30 +1056,14 @@ define i32 @test_indirect_call_vgpr_ptr_arg_and_reuse(i32 %i, ptr %fptr) {
 ; GISEL-NEXT:    v_writelane_b32 v41, s37, 5
 ; GISEL-NEXT:    v_writelane_b32 v41, s38, 6
 ; GISEL-NEXT:    v_writelane_b32 v41, s39, 7
-; GISEL-NEXT:    v_writelane_b32 v41, s40, 8
-; GISEL-NEXT:    v_writelane_b32 v41, s41, 9
-; GISEL-NEXT:    v_writelane_b32 v41, s42, 10
-; GISEL-NEXT:    v_writelane_b32 v41, s43, 11
-; GISEL-NEXT:    v_writelane_b32 v41, s44, 12
-; GISEL-NEXT:    v_writelane_b32 v41, s45, 13
-; GISEL-NEXT:    v_writelane_b32 v41, s46, 14
-; GISEL-NEXT:    v_writelane_b32 v41, s47, 15
-; GISEL-NEXT:    v_writelane_b32 v41, s48, 16
-; GISEL-NEXT:    v_writelane_b32 v41, s49, 17
-; GISEL-NEXT:    v_writelane_b32 v41, s50, 18
-; GISEL-NEXT:    v_writelane_b32 v41, s51, 19
-; GISEL-NEXT:    v_writelane_b32 v41, s52, 20
-; GISEL-NEXT:    v_writelane_b32 v41, s53, 21
-; GISEL-NEXT:    v_writelane_b32 v41, s54, 22
-; GISEL-NEXT:    v_writelane_b32 v41, s55, 23
-; GISEL-NEXT:    v_writelane_b32 v41, s56, 24
-; GISEL-NEXT:    v_writelane_b32 v41, s57, 25
-; GISEL-NEXT:    v_writelane_b32 v41, s58, 26
-; GISEL-NEXT:    v_writelane_b32 v41, s59, 27
-; GISEL-NEXT:    v_writelane_b32 v41, s60, 28
-; GISEL-NEXT:    v_writelane_b32 v41, s61, 29
-; GISEL-NEXT:    v_writelane_b32 v41, s62, 30
-; GISEL-NEXT:    v_writelane_b32 v41, s63, 31
+; GISEL-NEXT:    v_writelane_b32 v41, s48, 8
+; GISEL-NEXT:    v_writelane_b32 v41, s49, 9
+; GISEL-NEXT:    v_writelane_b32 v41, s50, 10
+; GISEL-NEXT:    v_writelane_b32 v41, s51, 11
+; GISEL-NEXT:    v_writelane_b32 v41, s52, 12
+; GISEL-NEXT:    v_writelane_b32 v41, s53, 13
+; GISEL-NEXT:    v_writelane_b32 v41, s54, 14
+; GISEL-NEXT:    v_writelane_b32 v41, s55, 15
 ; GISEL-NEXT:    v_mov_b32_e32 v40, v0
 ; GISEL-NEXT:    s_mov_b64 s[4:5], exec
 ; GISEL-NEXT:  .LBB7_1: ; =>This Inner Loop Header: Depth=1
@@ -1191,30 +1079,14 @@ define i32 @test_indirect_call_vgpr_ptr_arg_and_reuse(i32 %i, ptr %fptr) {
 ; GISEL-NEXT:  ; %bb.2:
 ; GISEL-NEXT:    s_mov_b64 exec, s[4:5]
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v40
-; GISEL-NEXT:    v_readlane_b32 s63, v41, 31
-; GISEL-NEXT:    v_readlane_b32 s62, v41, 30
-; GISEL-NEXT:    v_readlane_b32 s61, v41, 29
-; GISEL-NEXT:    v_readlane_b32 s60, v41, 28
-; GISEL-NEXT:    v_readlane_b32 s59, v41, 27
-; GISEL-NEXT:    v_readlane_b32 s58, v41, 26
-; GISEL-NEXT:    v_readlane_b32 s57, v41, 25
-; GISEL-NEXT:    v_readlane_b32 s56, v41, 24
-; GISEL-NEXT:    v_readlane_b32 s55, v41, 23
-; GISEL-NEXT:    v_readlane_b32 s54, v41, 22
-; GISEL-NEXT:    v_readlane_b32 s53, v41, 21
-; GISEL-NEXT:    v_readlane_b32 s52, v41, 20
-; GISEL-NEXT:    v_readlane_b32 s51, v41, 19
-; GISEL-NEXT:    v_readlane_b32 s50, v41, 18
-; GISEL-NEXT:    v_readlane_b32 s49, v41, 17
-; GISEL-NEXT:    v_readlane_b32 s48, v41, 16
-; GISEL-NEXT:    v_readlane_b32 s47, v41, 15
-; GISEL-NEXT:    v_readlane_b32 s46, v41, 14
-; GISEL-NEXT:    v_readlane_b32 s45, v41, 13
-; GISEL-NEXT:    v_readlane_b32 s44, v41, 12
-; GISEL-NEXT:    v_readlane_b32 s43, v41, 11
-; GISEL-NEXT:    v_readlane_b32 s42, v41, 10
-; GISEL-NEXT:    v_readlane_b32 s41, v41, 9
-; GISEL-NEXT:    v_readlane_b32 s40, v41, 8
+; GISEL-NEXT:    v_readlane_b32 s55, v41, 15
+; GISEL-NEXT:    v_readlane_b32 s54, v41, 14
+; GISEL-NEXT:    v_readlane_b32 s53, v41, 13
+; GISEL-NEXT:    v_readlane_b32 s52, v41, 12
+; GISEL-NEXT:    v_readlane_b32 s51, v41, 11
+; GISEL-NEXT:    v_readlane_b32 s50, v41, 10
+; GISEL-NEXT:    v_readlane_b32 s49, v41, 9
+; GISEL-NEXT:    v_readlane_b32 s48, v41, 8
 ; GISEL-NEXT:    v_readlane_b32 s39, v41, 7
 ; GISEL-NEXT:    v_readlane_b32 s38, v41, 6
 ; GISEL-NEXT:    v_readlane_b32 s37, v41, 5
@@ -1257,30 +1129,14 @@ define i32 @test_indirect_call_vgpr_ptr_arg_and_return(i32 %i, ptr %fptr) {
 ; GCN-NEXT:    v_writelane_b32 v40, s37, 5
 ; GCN-NEXT:    v_writelane_b32 v40, s38, 6
 ; GCN-NEXT:    v_writelane_b32 v40, s39, 7
-; GCN-NEXT:    v_writelane_b32 v40, s40, 8
-; GCN-NEXT:    v_writelane_b32 v40, s41, 9
-; GCN-NEXT:    v_writelane_b32 v40, s42, 10
-; GCN-NEXT:    v_writelane_b32 v40, s43, 11
-; GCN-NEXT:    v_writelane_b32 v40, s44, 12
-; GCN-NEXT:    v_writelane_b32 v40, s45, 13
-; GCN-NEXT:    v_writelane_b32 v40, s46, 14
-; GCN-NEXT:    v_writelane_b32 v40, s47, 15
-; GCN-NEXT:    v_writelane_b32 v40, s48, 16
-; GCN-NEXT:    v_writelane_b32 v40, s49, 17
-; GCN-NEXT:    v_writelane_b32 v40, s50, 18
-; GCN-NEXT:    v_writelane_b32 v40, s51, 19
-; GCN-NEXT:    v_writelane_b32 v40, s52, 20
-; GCN-NEXT:    v_writelane_b32 v40, s53, 21
-; GCN-NEXT:    v_writelane_b32 v40, s54, 22
-; GCN-NEXT:    v_writelane_b32 v40, s55, 23
-; GCN-NEXT:    v_writelane_b32 v40, s56, 24
-; GCN-NEXT:    v_writelane_b32 v40, s57, 25
-; GCN-NEXT:    v_writelane_b32 v40, s58, 26
-; GCN-NEXT:    v_writelane_b32 v40, s59, 27
-; GCN-NEXT:    v_writelane_b32 v40, s60, 28
-; GCN-NEXT:    v_writelane_b32 v40, s61, 29
-; GCN-NEXT:    v_writelane_b32 v40, s62, 30
-; GCN-NEXT:    v_writelane_b32 v40, s63, 31
+; GCN-NEXT:    v_writelane_b32 v40, s48, 8
+; GCN-NEXT:    v_writelane_b32 v40, s49, 9
+; GCN-NEXT:    v_writelane_b32 v40, s50, 10
+; GCN-NEXT:    v_writelane_b32 v40, s51, 11
+; GCN-NEXT:    v_writelane_b32 v40, s52, 12
+; GCN-NEXT:    v_writelane_b32 v40, s53, 13
+; GCN-NEXT:    v_writelane_b32 v40, s54, 14
+; GCN-NEXT:    v_writelane_b32 v40, s55, 15
 ; GCN-NEXT:    s_mov_b64 s[4:5], exec
 ; GCN-NEXT:  .LBB8_1: ; =>This Inner Loop Header: Depth=1
 ; GCN-NEXT:    v_readfirstlane_b32 s8, v1
@@ -1296,30 +1152,14 @@ define i32 @test_indirect_call_vgpr_ptr_arg_and_return(i32 %i, ptr %fptr) {
 ; GCN-NEXT:  ; %bb.2:
 ; GCN-NEXT:    s_mov_b64 exec, s[4:5]
 ; GCN-NEXT:    v_mov_b32_e32 v0, v3
-; GCN-NEXT:    v_readlane_b32 s63, v40, 31
-; GCN-NEXT:    v_readlane_b32 s62, v40, 30
-; GCN-NEXT:    v_readlane_b32 s61, v40, 29
-; GCN-NEXT:    v_readlane_b32 s60, v40, 28
-; GCN-NEXT:    v_readlane_b32 s59, v40, 27
-; GCN-NEXT:    v_readlane_b32 s58, v40, 26
-; GCN-NEXT:    v_readlane_b32 s57, v40, 25
-; GCN-NEXT:    v_readlane_b32 s56, v40, 24
-; GCN-NEXT:    v_readlane_b32 s55, v40, 23
-; GCN-NEXT:    v_readlane_b32 s54, v40, 22
-; GCN-NEXT:    v_readlane_b32 s53, v40, 21
-; GCN-NEXT:    v_readlane_b32 s52, v40, 20
-; GCN-NEXT:    v_readlane_b32 s51, v40, 19
-; GCN-NEXT:    v_readlane_b32 s50, v40, 18
-; GCN-NEXT:    v_readlane_b32 s49, v40, 17
-; GCN-NEXT:    v_readlane_b32 s48, v40, 16
-; GCN-NEXT:    v_readlane_b32 s47, v40, 15
-; GCN-NEXT:    v_readlane_b32 s46, v40, 14
-; GCN-NEXT:    v_readlane_b32 s45, v40, 13
-; GCN-NEXT:    v_readlane_b32 s44, v40, 12
-; GCN-NEXT:    v_readlane_b32 s43, v40, 11
-; GCN-NEXT:    v_readlane_b32 s42, v40, 10
-; GCN-NEXT:    v_readlane_b32 s41, v40, 9
-; GCN-NEXT:    v_readlane_b32 s40, v40, 8
+; GCN-NEXT:    v_readlane_b32 s55, v40, 15
+; GCN-NEXT:    v_readlane_b32 s54, v40, 14
+; GCN-NEXT:    v_readlane_b32 s53, v40, 13
+; GCN-NEXT:    v_readlane_b32 s52, v40, 12
+; GCN-NEXT:    v_readlane_b32 s51, v40, 11
+; GCN-NEXT:    v_readlane_b32 s50, v40, 10
+; GCN-NEXT:    v_readlane_b32 s49, v40, 9
+; GCN-NEXT:    v_readlane_b32 s48, v40, 8
 ; GCN-NEXT:    v_readlane_b32 s39, v40, 7
 ; GCN-NEXT:    v_readlane_b32 s38, v40, 6
 ; GCN-NEXT:    v_readlane_b32 s37, v40, 5
@@ -1353,30 +1193,14 @@ define i32 @test_indirect_call_vgpr_ptr_arg_and_return(i32 %i, ptr %fptr) {
 ; GISEL-NEXT:    v_writelane_b32 v40, s37, 5
 ; GISEL-NEXT:    v_writelane_b32 v40, s38, 6
 ; GISEL-NEXT:    v_writelane_b32 v40, s39, 7
-; GISEL-NEXT:    v_writelane_b32 v40, s40, 8
-; GISEL-NEXT:    v_writelane_b32 v40, s41, 9
-; GISEL-NEXT:    v_writelane_b32 v40, s42, 10
-; GISEL-NEXT:    v_writelane_b32 v40, s43, 11
-; GISEL-NEXT:    v_writelane_b32 v40, s44, 12
-; GISEL-NEXT:    v_writelane_b32 v40, s45, 13
-; GISEL-NEXT:    v_writelane_b32 v40, s46, 14
-; GISEL-NEXT:    v_writelane_b32 v40, s47, 15
-; GISEL-NEXT:    v_writelane_b32 v40, s48, 16
-; GISEL-NEXT:    v_writelane_b32 v40, s49, 17
-; GISEL-NEXT:    v_writelane_b32 v40, s50, 18
-; GISEL-NEXT:    v_writelane_b32 v40, s51, 19
-; GISEL-NEXT:    v_writelane_b32 v40, s52, 20
-; GISEL-NEXT:    v_writelane_b32 v40, s53, 21
-; GISEL-NEXT:    v_writelane_b32 v40, s54, 22
-; GISEL-NEXT:    v_writelane_b32 v40, s55, 23
-; GISEL-NEXT:    v_writelane_b32 v40, s56, 24
-; GISEL-NEXT:    v_writelane_b32 v40, s57, 25
-; GISEL-NEXT:    v_writelane_b32 v40, s58, 26
-; GISEL-NEXT:    v_writelane_b32 v40, s59, 27
-; GISEL-NEXT:    v_writelane_b32 v40, s60, 28
-; GISEL-NEXT:    v_writelane_b32 v40, s61, 29
-; GISEL-NEXT:    v_writelane_b32 v40, s62, 30
-; GISEL-NEXT:    v_writelane_b32 v40, s63, 31
+; GISEL-NEXT:    v_writelane_b32 v40, s48, 8
+; GISEL-NEXT:    v_writelane_b32 v40, s49, 9
+; GISEL-NEXT:    v_writelane_b32 v40, s50, 10
+; GISEL-NEXT:    v_writelane_b32 v40, s51, 11
+; GISEL-NEXT:    v_writelane_b32 v40, s52, 12
+; GISEL-NEXT:    v_writelane_b32 v40, s53, 13
+; GISEL-NEXT:    v_writelane_b32 v40, s54, 14
+; GISEL-NEXT:    v_writelane_b32 v40, s55, 15
 ; GISEL-NEXT:    s_mov_b64 s[4:5], exec
 ; GISEL-NEXT:  .LBB8_1: ; =>This Inner Loop Header: Depth=1
 ; GISEL-NEXT:    v_readfirstlane_b32 s8, v1
@@ -1392,30 +1216,14 @@ define i32 @test_indirect_call_vgpr_ptr_arg_and_return(i32 %i, ptr %fptr) {
 ; GISEL-NEXT:  ; %bb.2:
 ; GISEL-NEXT:    s_mov_b64 exec, s[4:5]
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v2
-; GISEL-NEXT:    v_readlane_b32 s63, v40, 31
-; GISEL-NEXT:    v_readlane_b32 s62, v40, 30
-; GISEL-NEXT:    v_readlane_b32 s61, v40, 29
-; GISEL-NEXT:    v_readlane_b32 s60, v40, 28
-; GISEL-NEXT:    v_readlane_b32 s59, v40, 27
-; GISEL-NEXT:    v_readlane_b32 s58, v40, 26
-; GISEL-NEXT:    v_readlane_b32 s57, v40, 25
-; GISEL-NEXT:    v_readlane_b32 s56, v40, 24
-; GISEL-NEXT:    v_readlane_b32 s55, v40, 23
-; GISEL-NEXT:    v_readlane_b32 s54, v40, 22
-; GISEL-NEXT:    v_readlane_b32 s53, v40, 21
-; GISEL-NEXT:    v_readlane_b32 s52, v40, 20
-; GISEL-NEXT:    v_readlane_b32 s51, v40, 19
-; GISEL-NEXT:    v_readlane_b32 s50, v40, 18
-; GISEL-NEXT:    v_readlane_b32 s49, v40, 17
-; GISEL-NEXT:    v_readlane_b32 s48, v40, 16
-; GISEL-NEXT:    v_readlane_b32 s47, v40, 15
-; GISEL-NEXT:    v_readlane_b32 s46, v40, 14
-; GISEL-NEXT:    v_readlane_b32 s45, v40, 13
-; GISEL-NEXT:    v_readlane_b32 s44, v40, 12
-; GISEL-NEXT:    v_readlane_b32 s43, v40, 11
-; GISEL-NEXT:    v_readlane_b32 s42, v40, 10
-; GISEL-NEXT:    v_readlane_b32 s41, v40, 9
-; GISEL-NEXT:    v_readlane_b32 s40, v40, 8
+; GISEL-NEXT:    v_readlane_b32 s55, v40, 15
+; GISEL-NEXT:    v_readlane_b32 s54, v40, 14
+; GISEL-NEXT:    v_readlane_b32 s53, v40, 13
+; GISEL-NEXT:    v_readlane_b32 s52, v40, 12
+; GISEL-NEXT:    v_readlane_b32 s51, v40, 11
+; GISEL-NEXT:    v_readlane_b32 s50, v40, 10
+; GISEL-NEXT:    v_readlane_b32 s49, v40, 9
+; GISEL-NEXT:    v_readlane_b32 s48, v40, 8
 ; GISEL-NEXT:    v_readlane_b32 s39, v40, 7
 ; GISEL-NEXT:    v_readlane_b32 s38, v40, 6
 ; GISEL-NEXT:    v_readlane_b32 s37, v40, 5
@@ -1454,30 +1262,14 @@ define void @test_indirect_tail_call_vgpr_ptr(ptr %fptr) {
 ; GCN-NEXT:    v_writelane_b32 v40, s37, 5
 ; GCN-NEXT:    v_writelane_b32 v40, s38, 6
 ; GCN-NEXT:    v_writelane_b32 v40, s39, 7
-; GCN-NEXT:    v_writelane_b32 v40, s40, 8
-; GCN-NEXT:    v_writelane_b32 v40, s41, 9
-; GCN-NEXT:    v_writelane_b32 v40, s42, 10
-; GCN-NEXT:    v_writelane_b32 v40, s43, 11
-; GCN-NEXT:    v_writelane_b32 v40, s44, 12
-; GCN-NEXT:    v_writelane_b32 v40, s45, 13
-; GCN-NEXT:    v_writelane_b32 v40, s46, 14
-; GCN-NEXT:    v_writelane_b32 v40, s47, 15
-; GCN-NEXT:    v_writelane_b32 v40, s48, 16
-; GCN-NEXT:    v_writelane_b32 v40, s49, 17
-; GCN-NEXT:    v_writelane_b32 v40, s50, 18
-; GCN-NEXT:    v_writelane_b32 v40, s51, 19
-; GCN-NEXT:    v_writelane_b32 v40, s52, 20
-; GCN-NEXT:    v_writelane_b32 v40, s53, 21
-; GCN-NEXT:    v_writelane_b32 v40, s54, 22
-; GCN-NEXT:    v_writelane_b32 v40, s55, 23
-; GCN-NEXT:    v_writelane_b32 v40, s56, 24
-; GCN-NEXT:    v_writelane_b32 v40, s57, 25
-; GCN-NEXT:    v_writelane_b32 v40, s58, 26
-; GCN-NEXT:    v_writelane_b32 v40, s59, 27
-; GCN-NEXT:    v_writelane_b32 v40, s60, 28
-; GCN-NEXT:    v_writelane_b32 v40, s61, 29
-; GCN-NEXT:    v_writelane_b32 v40, s62, 30
-; GCN-NEXT:    v_writelane_b32 v40, s63, 31
+; GCN-NEXT:    v_writelane_b32 v40, s48, 8
+; GCN-NEXT:    v_writelane_b32 v40, s49, 9
+; GCN-NEXT:    v_writelane_b32 v40, s50, 10
+; GCN-NEXT:    v_writelane_b32 v40, s51, 11
+; GCN-NEXT:    v_writelane_b32 v40, s52, 12
+; GCN-NEXT:    v_writelane_b32 v40, s53, 13
+; GCN-NEXT:    v_writelane_b32 v40, s54, 14
+; GCN-NEXT:    v_writelane_b32 v40, s55, 15
 ; GCN-NEXT:    s_mov_b64 s[4:5], exec
 ; GCN-NEXT:  .LBB9_1: ; =>This Inner Loop Header: Depth=1
 ; GCN-NEXT:    v_readfirstlane_b32 s6, v0
@@ -1490,30 +1282,14 @@ define void @test_indirect_tail_call_vgpr_ptr(ptr %fptr) {
 ; GCN-NEXT:    s_cbranch_execnz .LBB9_1
 ; GCN-NEXT:  ; %bb.2:
 ; GCN-NEXT:    s_mov_b64 exec, s[4:5]
-; GCN-NEXT:    v_readlane_b32 s63, v40, 31
-; GCN-NEXT:    v_readlane_b32 s62, v40, 30
-; GCN-NEXT:    v_readlane_b32 s61, v40, 29
-; GCN-NEXT:    v_readlane_b32 s60, v40, 28
-; GCN-NEXT:    v_readlane_b32 s59, v40, 27
-; GCN-NEXT:    v_readlane_b32 s58, v40, 26
-; GCN-NEXT:    v_readlane_b32 s57, v40, 25
-; GCN-NEXT:    v_readlane_b32 s56, v40, 24
-; GCN-NEXT:    v_readlane_b32 s55, v40, 23
-; GCN-NEXT:    v_readlane_b32 s54, v40, 22
-; GCN-NEXT:    v_readlane_b32 s53, v40, 21
-; GCN-NEXT:    v_readlane_b32 s52, v40, 20
-; GCN-NEXT:    v_readlane_b32 s51, v40, 19
-; GCN-NEXT:    v_readlane_b32 s50, v40, 18
-; GCN-NEXT:    v_readlane_b32 s49, v40, 17
-; GCN-NEXT:    v_readlane_b32 s48, v40, 16
-; GCN-NEXT:    v_readlane_b32 s47, v40, 15
-; GCN-NEXT:    v_readlane_b32 s46, v40, 14
-; GCN-NEXT:    v_readlane_b32 s45, v40, 13
-; GCN-NEXT:    v_readlane_b32 s44, v40, 12
-; GCN-NEXT:    v_readlane_b32 s43, v40, 11
-; GCN-NEXT:    v_readlane_b32 s42, v40, 10
-; GCN-NEXT:    v_readlane_b32 s41, v40, 9
-; GCN-NEXT:    v_readlane_b32 s40, v40, 8
+; GCN-NEXT:    v_readlane_b32 s55, v40, 15
+; GCN-NEXT:    v_readlane_b32 s54, v40, 14
+; GCN-NEXT:    v_readlane_b32 s53, v40, 13
+; GCN-NEXT:    v_readlane_b32 s52, v40, 12
+; GCN-NEXT:    v_readlane_b32 s51, v40, 11
+; GCN-NEXT:    v_readlane_b32 s50, v40, 10
+; GCN-NEXT:    v_readlane_b32 s49, v40, 9
+; GCN-NEXT:    v_readlane_b32 s48, v40, 8
 ; GCN-NEXT:    v_readlane_b32 s39, v40, 7
 ; GCN-NEXT:    v_readlane_b32 s38, v40, 6
 ; GCN-NEXT:    v_readlane_b32 s37, v40, 5
@@ -1547,30 +1323,14 @@ define void @test_indirect_tail_call_vgpr_ptr(ptr %fptr) {
 ; GISEL-NEXT:    v_writelane_b32 v40, s37, 5
 ; GISEL-NEXT:    v_writelane_b32 v40, s38, 6
 ; GISEL-NEXT:    v_writelane_b32 v40, s39, 7
-; GISEL-NEXT:    v_writelane_b32 v40, s40, 8
-; GISEL-NEXT:    v_writelane_b32 v40, s41, 9
-; GISEL-NEXT:    v_writelane_b32 v40, s42, 10
-; GISEL-NEXT:    v_writelane_b32 v40, s43, 11
-; GISEL-NEXT:    v_writelane_b32 v40, s44, 12
-; GISEL-NEXT:    v_writelane_b32 v40, s45, 13
-; GISEL-NEXT:    v_writelane_b32 v40, s46, 14
-; GISEL-NEXT:    v_writelane_b32 v40, s47, 15
-; GISEL-NEXT:    v_writelane_b32 v40, s48, 16
-; GISEL-NEXT:    v_writelane_b32 v40, s49, 17
-; GISEL-NEXT:    v_writelane_b32 v40, s50, 18
-; GISEL-NEXT:    v_writelane_b32 v40, s51, 19
-; GISEL-NEXT:    v_writelane_b32 v40, s52, 20
-; GISEL-NEXT:    v_writelane_b32 v40, s53, 21
-; GISEL-NEXT:    v_writelane_b32 v40, s54, 22
-; GISEL-NEXT:    v_writelane_b32 v40, s55, 23
-; GISEL-NEXT:    v_writelane_b32 v40, s56, 24
-; GISEL-NEXT:    v_writelane_b32 v40, s57, 25
-; GISEL-NEXT:    v_writelane_b32 v40, s58, 26
-; GISEL-NEXT:    v_writelane_b32 v40, s59, 27
-; GISEL-NEXT:    v_writelane_b32 v40, s60, 28
-; GISEL-NEXT:    v_writelane_b32 v40, s61, 29
-; GISEL-NEXT:    v_writelane_b32 v40, s62, 30
-; GISEL-NEXT:    v_writelane_b32 v40, s63, 31
+; GISEL-NEXT:    v_writelane_b32 v40, s48, 8
+; GISEL-NEXT:    v_writelane_b32 v40, s49, 9
+; GISEL-NEXT:    v_writelane_b32 v40, s50, 10
+; GISEL-NEXT:    v_writelane_b32 v40, s51, 11
+; GISEL-NEXT:    v_writelane_b32 v40, s52, 12
+; GISEL-NEXT:    v_writelane_b32 v40, s53, 13
+; GISEL-NEXT:    v_writelane_b32 v40, s54, 14
+; GISEL-NEXT:    v_writelane_b32 v40, s55, 15
 ; GISEL-NEXT:    s_mov_b64 s[4:5], exec
 ; GISEL-NEXT:  .LBB9_1: ; =>This Inner Loop Header: Depth=1
 ; GISEL-NEXT:    v_readfirstlane_b32 s6, v0
@@ -1583,30 +1343,14 @@ define void @test_indirect_tail_call_vgpr_ptr(ptr %fptr) {
 ; GISEL-NEXT:    s_cbranch_execnz .LBB9_1
 ; GISEL-NEXT:  ; %bb.2:
 ; GISEL-NEXT:    s_mov_b64 exec, s[4:5]
-; GISEL-NEXT:    v_readlane_b32 s63, v40, 31
-; GISEL-NEXT:    v_readlane_b32 s62, v40, 30
-; GISEL-NEXT:    v_readlane_b32 s61, v40, 29
-; GISEL-NEXT:    v_readlane_b32 s60, v40, 28
-; GISEL-NEXT:    v_readlane_b32 s59, v40, 27
-; GISEL-NEXT:    v_readlane_b32 s58, v40, 26
-; GISEL-NEXT:    v_readlane_b32 s57, v40, 25
-; GISEL-NEXT:    v_readlane_b32 s56, v40, 24
-; GISEL-NEXT:    v_readlane_b32 s55, v40, 23
-; GISEL-NEXT:    v_readlane_b32 s54, v40, 22
-; GISEL-NEXT:    v_readlane_b32 s53, v40, 21
-; GISEL-NEXT:    v_readlane_b32 s52, v40, 20
-; GISEL-NEXT:    v_readlane_b32 s51, v40, 19
-; GISEL-NEXT:    v_readlane_b32 s50, v40, 18
-; GISEL-NEXT:    v_readlane_b32 s49, v40, 17
-; GISEL-NEXT:    v_readlane_b32 s48, v40, 16
-; GISEL-NEXT:    v_readlane_b32 s47, v40, 15
-; GISEL-NEXT:    v_readlane_b32 s46, v40, 14
-; GISEL-NEXT:    v_readlane_b32 s45, v40, 13
-; GISEL-NEXT:    v_readlane_b32 s44, v40, 12
-; GISEL-NEXT:    v_readlane_b32 s43, v40, 11
-; GISEL-NEXT:    v_readlane_b32 s42, v40, 10
-; GISEL-NEXT:    v_readlane_b32 s41, v40, 9
-; GISEL-NEXT:    v_readlane_b32 s40, v40, 8
+; GISEL-NEXT:    v_readlane_b32 s55, v40, 15
+; GISEL-NEXT:    v_readlane_b32 s54, v40, 14
+; GISEL-NEXT:    v_readlane_b32 s53, v40, 13
+; GISEL-NEXT:    v_readlane_b32 s52, v40, 12
+; GISEL-NEXT:    v_readlane_b32 s51, v40, 11
+; GISEL-NEXT:    v_readlane_b32 s50, v40, 10
+; GISEL-NEXT:    v_readlane_b32 s49, v40, 9
+; GISEL-NEXT:    v_readlane_b32 s48, v40, 8
 ; GISEL-NEXT:    v_readlane_b32 s39, v40, 7
 ; GISEL-NEXT:    v_readlane_b32 s38, v40, 6
 ; GISEL-NEXT:    v_readlane_b32 s37, v40, 5
diff --git a/llvm/test/CodeGen/AMDGPU/infinite-loop-evergreen.ll b/llvm/test/CodeGen/AMDGPU/infinite-loop-evergreen.ll
index 232a1d3789c9a..06fc4ce164bc2 100644
--- a/llvm/test/CodeGen/AMDGPU/infinite-loop-evergreen.ll
+++ b/llvm/test/CodeGen/AMDGPU/infinite-loop-evergreen.ll
@@ -1,6 +1,6 @@
 ; XFAIL: *
 ; REQUIRES: asserts
-; RUN: llc -mtriple=r600 -mcpu=cypress < %s | FileCheck %s
+; RUN: llc -mtriple=r600 -mcpu=cypress < %s
 
 define amdgpu_kernel void @inf_loop_irreducible_cfg() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir b/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir
new file mode 100644
index 0000000000000..2cc25d88347ee
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir
@@ -0,0 +1,113 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -simplify-mir -start-before=greedy,2 -stress-regalloc=3 -stop-after=postrapseudos -o - -verify-regalloc %s | FileCheck %s
+
+# Compare results of using V_MOV_B32 vs. AV_MOV_B32_IMM_PSEUDO during
+# allocation.
+
+---
+name: av_mov_b32_split
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr72_sgpr73_sgpr74_sgpr75'
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       7
+body:             |
+  bb.0:
+    liveins: $vgpr0, $sgpr4_sgpr5
+
+    ; CHECK-LABEL: name: av_mov_b32_split
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $agpr0 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+    ; CHECK-NEXT: renamable $agpr1 = V_ACCVGPR_WRITE_B32_e64 1, implicit $exec
+    ; CHECK-NEXT: renamable $agpr2 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+    ; CHECK-NEXT: $vgpr0 = V_ACCVGPR_READ_B32_e64 killed $agpr0, implicit $exec
+    ; CHECK-NEXT: renamable $agpr0 = V_ACCVGPR_WRITE_B32_e64 3, implicit $exec
+    ; CHECK-NEXT: $vgpr1 = V_ACCVGPR_READ_B32_e64 killed $agpr0, implicit $exec
+    ; CHECK-NEXT: renamable $agpr0 = V_ACCVGPR_WRITE_B32_e64 4, implicit $exec
+    ; CHECK-NEXT: $vgpr2 = V_ACCVGPR_READ_B32_e64 killed $agpr0, implicit $exec
+    ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec
+    ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr0
+    ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr1
+    ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr2
+    ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 $vgpr1, implicit $exec
+    ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr0
+    ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 $vgpr2, implicit $exec
+    ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr0
+    %0:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+    %1:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+    %2:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+    %3:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 3, implicit $exec
+    %4:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 4, implicit $exec
+
+    %5:agpr_32 = COPY %0
+    %6:agpr_32 = COPY %1
+    %7:agpr_32 = COPY %2
+    %8:agpr_32 = COPY %3
+    %9:agpr_32 = COPY %4
+
+    S_NOP 0, implicit %5
+    S_NOP 0, implicit %6
+    S_NOP 0, implicit %7
+    S_NOP 0, implicit %8
+    S_NOP 0, implicit %9
+
+...
+
+---
+name: v_mov_b32_split
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr72_sgpr73_sgpr74_sgpr75'
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       7
+body:             |
+  bb.0:
+    liveins: $vgpr0, $sgpr4_sgpr5
+
+    ; CHECK-LABEL: name: v_mov_b32_split
+    ; CHECK: liveins: $vgpr0, $vgpr3, $vgpr4, $vgpr5, $sgpr4_sgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr2 = V_MOV_B32_e32 2, implicit $exec
+    ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr0, implicit $exec, implicit $exec
+    ; CHECK-NEXT: $vgpr3 = V_ACCVGPR_READ_B32_e64 killed $agpr0, implicit $exec
+    ; CHECK-NEXT: $agpr1 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr1, implicit $exec, implicit $exec
+    ; CHECK-NEXT: $agpr2 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr2, implicit $exec, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 3, implicit $exec
+    ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr0, implicit $exec, implicit $exec
+    ; CHECK-NEXT: $vgpr4 = V_ACCVGPR_READ_B32_e64 killed $agpr0, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 4, implicit $exec
+    ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr0, implicit $exec, implicit $exec
+    ; CHECK-NEXT: $vgpr5 = V_ACCVGPR_READ_B32_e64 killed $agpr0, implicit $exec
+    ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 $vgpr3, implicit $exec
+    ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr0
+    ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr1
+    ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr2
+    ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 $vgpr4, implicit $exec
+    ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr0
+    ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 $vgpr5, implicit $exec
+    ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr0
+    %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %1:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    %2:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
+    %3:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+    %4:vgpr_32 = V_MOV_B32_e32 4, implicit $exec
+
+    %5:agpr_32 = COPY %0
+    %6:agpr_32 = COPY %1
+    %7:agpr_32 = COPY %2
+    %8:agpr_32 = COPY %3
+    %9:agpr_32 = COPY %4
+
+    S_NOP 0, implicit %5
+    S_NOP 0, implicit %6
+    S_NOP 0, implicit %7
+    S_NOP 0, implicit %8
+    S_NOP 0, implicit %9
+
+...
+
diff --git a/llvm/test/CodeGen/AMDGPU/inline-asm.ll b/llvm/test/CodeGen/AMDGPU/inline-asm.ll
index 881433f48c76f..4a6dd870fe62d 100644
--- a/llvm/test/CodeGen/AMDGPU/inline-asm.ll
+++ b/llvm/test/CodeGen/AMDGPU/inline-asm.ll
@@ -307,7 +307,7 @@ false:
   br label %exit
 
 exit:
-  %s1 = phi { i64, i64} [ undef, %entry ], [ %s0, %false]
+  %s1 = phi { i64, i64} [ poison, %entry ], [ %s0, %false]
   %v0 = extractvalue { i64, i64 } %s1, 0
   %v1 = extractvalue { i64, i64 } %s1, 1
   tail call void asm sideeffect "; use $0", "v"(i64 %v0)
diff --git a/llvm/test/CodeGen/AMDGPU/input-mods.r600.ll b/llvm/test/CodeGen/AMDGPU/input-mods.r600.ll
index 4db5e2a472ee7..633ad6acebd4f 100644
--- a/llvm/test/CodeGen/AMDGPU/input-mods.r600.ll
+++ b/llvm/test/CodeGen/AMDGPU/input-mods.r600.ll
@@ -43,7 +43,7 @@ define amdgpu_ps void @test(<4 x float> inreg %reg0) {
    %r1 = call float @llvm.fabs.f32(float %r0)
    %r2 = fsub float -0.000000e+00, %r1
    %r3 = call afn float @llvm.exp2.f32(float %r2)
-   %vec = insertelement <4 x float> undef, float %r3, i32 0
+   %vec = insertelement <4 x float> poison, float %r3, i32 0
    call void @llvm.r600.store.swizzle(<4 x float> %vec, i32 0, i32 0)
    ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
index add8c0f75bf33..6f0c850117208 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
@@ -48,7 +48,7 @@ define amdgpu_kernel void @float4_inselt_undef(ptr addrspace(1) %out, i32 %sel)
 ; GCN-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; GCN-NEXT:    s_endpgm
 entry:
-  %v = insertelement <4 x float> undef, float 1.000000e+00, i32 %sel
+  %v = insertelement <4 x float> poison, float 1.000000e+00, i32 %sel
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
index f213b0635c8ac..8720fda9646e2 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
@@ -2016,7 +2016,7 @@ define amdgpu_kernel void @insert_split_bb(ptr addrspace(1) %out, ptr addrspace(
 ; VI-NEXT:  .LBB42_4:
 ; VI-NEXT:    s_branch .LBB42_2
 entry:
-  %0 = insertelement <2 x i32> undef, i32 %a, i32 0
+  %0 = insertelement <2 x i32> poison, i32 %a, i32 0
   %1 = icmp eq i32 %a, 0
   br i1 %1, label %if, label %else
 
diff --git a/llvm/test/CodeGen/AMDGPU/issue48473.mir b/llvm/test/CodeGen/AMDGPU/issue48473.mir
index dd73e65de7cb6..654461a62fa9f 100644
--- a/llvm/test/CodeGen/AMDGPU/issue48473.mir
+++ b/llvm/test/CodeGen/AMDGPU/issue48473.mir
@@ -43,7 +43,7 @@
 # %25 to $sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
 
 # CHECK-LABEL: name: issue48473
-# CHECK: S_NOP 0, implicit killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed renamable $sgpr12_sgpr13_sgpr14_sgpr15, implicit killed renamable $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23, implicit killed renamable $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31, implicit killed renamable $sgpr84_sgpr85_sgpr86_sgpr87, implicit killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43, implicit killed renamable $sgpr4_sgpr5_sgpr6_sgpr7, implicit killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, implicit killed renamable $sgpr88_sgpr89_sgpr90_sgpr91, implicit killed renamable $sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, implicit undef $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed renamable $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59, implicit killed renamable $sgpr92_sgpr93_sgpr94_sgpr95, implicit killed renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75, implicit renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75, implicit killed renamable $sgpr96_sgpr97_sgpr98_sgpr99, implicit killed renamable $sgpr8_sgpr9_sgpr10_sgpr11, implicit killed renamable $sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+# CHECK: S_NOP 0, implicit killed renamable $sgpr20_sgpr21_sgpr22_sgpr23, implicit killed renamable $sgpr88_sgpr89_sgpr90_sgpr91, implicit killed renamable $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit killed renamable $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, implicit killed renamable $sgpr40_sgpr41_sgpr42_sgpr43, implicit killed renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63, implicit killed renamable $sgpr36_sgpr37_sgpr38_sgpr39, implicit killed renamable $sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79, implicit killed renamable $sgpr80_sgpr81_sgpr82_sgpr83, implicit killed renamable $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71, implicit killed renamable $sgpr16_sgpr17_sgpr18_sgpr19, implicit killed renamable $sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99, implicit killed renamable $sgpr84_sgpr85_sgpr86_sgpr87, implicit killed renamable $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55, implicit renamable $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55, implicit undef $sgpr0_sgpr1_sgpr2_sgpr3, implicit killed renamable $sgpr44_sgpr45_sgpr46_sgpr47, implicit killed renamable $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
 
 ---
 name:            issue48473
diff --git a/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll b/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll
index 3e6de32492457..44139fafbfe20 100644
--- a/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll
+++ b/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll
@@ -38,30 +38,30 @@ define bfloat @sitofp_i128_to_bf16(i128 %x) {
 ; GCN-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN-NEXT:    v_add_u32_e32 v6, 64, v6
 ; GCN-NEXT:    v_cndmask_b32_e32 v7, v6, v2, vcc
-; GCN-NEXT:    v_sub_u32_e32 v6, 0x80, v7
-; GCN-NEXT:    v_sub_u32_e32 v2, 0x7f, v7
-; GCN-NEXT:    v_cmp_gt_i32_e32 vcc, 25, v6
+; GCN-NEXT:    v_sub_u32_e32 v2, 0x80, v7
+; GCN-NEXT:    v_cmp_gt_i32_e32 vcc, 25, v2
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN-NEXT:  ; %bb.2: ; %itofp-if-else
-; GCN-NEXT:    v_add_u32_e32 v4, 0xffffff98, v7
-; GCN-NEXT:    v_lshlrev_b64 v[0:1], v4, v[0:1]
-; GCN-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v4
+; GCN-NEXT:    v_add_u32_e32 v2, 0xffffff98, v7
+; GCN-NEXT:    v_lshlrev_b64 v[0:1], v2, v[0:1]
+; GCN-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v2
 ; GCN-NEXT:    v_cndmask_b32_e32 v8, 0, v0, vcc
-; GCN-NEXT:    ; implicit-def: $vgpr6
+; GCN-NEXT:    ; implicit-def: $vgpr2
 ; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN-NEXT:    ; implicit-def: $vgpr7
 ; GCN-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN-NEXT:  ; %bb.3: ; %Flow3
-; GCN-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; GCN-NEXT:    s_or_saveexec_b64 s[8:9], s[4:5]
+; GCN-NEXT:    v_sub_u32_e32 v6, 0x7f, v7
+; GCN-NEXT:    s_xor_b64 exec, exec, s[8:9]
 ; GCN-NEXT:    s_cbranch_execz .LBB0_13
 ; GCN-NEXT:  ; %bb.4: ; %NodeBlock
-; GCN-NEXT:    v_cmp_lt_i32_e32 vcc, 25, v6
+; GCN-NEXT:    v_cmp_lt_i32_e32 vcc, 25, v2
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[10:11], exec, s[4:5]
 ; GCN-NEXT:    s_cbranch_execz .LBB0_8
 ; GCN-NEXT:  ; %bb.5: ; %LeafBlock
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v6
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v2
 ; GCN-NEXT:    s_and_saveexec_b64 s[12:13], vcc
 ; GCN-NEXT:    s_cbranch_execz .LBB0_7
 ; GCN-NEXT:  ; %bb.6: ; %itofp-sw-default
@@ -120,13 +120,13 @@ define bfloat @sitofp_i128_to_bf16(i128 %x) {
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:  ; %bb.11: ; %itofp-if-then20
 ; GCN-NEXT:    v_alignbit_b32 v8, v1, v0, 3
-; GCN-NEXT:    v_mov_b32_e32 v2, v6
+; GCN-NEXT:    v_mov_b32_e32 v6, v2
 ; GCN-NEXT:  ; %bb.12: ; %Flow
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:  .LBB0_13: ; %Flow4
 ; GCN-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GCN-NEXT:    v_and_b32_e32 v0, 0x80000000, v3
-; GCN-NEXT:    v_lshl_add_u32 v1, v2, 23, 1.0
+; GCN-NEXT:    v_lshl_add_u32 v1, v6, 23, 1.0
 ; GCN-NEXT:    v_and_b32_e32 v2, 0x7fffff, v8
 ; GCN-NEXT:    v_or3_b32 v0, v2, v0, v1
 ; GCN-NEXT:    v_bfe_u32 v1, v8, 16, 1
@@ -166,9 +166,8 @@ define bfloat @uitofp_i128_to_bf16(i128 %x) {
 ; GCN-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
 ; GCN-NEXT:    v_add_u32_e32 v5, 64, v5
 ; GCN-NEXT:    v_cndmask_b32_e32 v6, v5, v4, vcc
-; GCN-NEXT:    v_sub_u32_e32 v5, 0x80, v6
-; GCN-NEXT:    v_sub_u32_e32 v4, 0x7f, v6
-; GCN-NEXT:    v_cmp_gt_i32_e32 vcc, 25, v5
+; GCN-NEXT:    v_sub_u32_e32 v4, 0x80, v6
+; GCN-NEXT:    v_cmp_gt_i32_e32 vcc, 25, v4
 ; GCN-NEXT:    ; implicit-def: $vgpr7
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -177,20 +176,21 @@ define bfloat @uitofp_i128_to_bf16(i128 %x) {
 ; GCN-NEXT:    v_lshlrev_b64 v[0:1], v2, v[0:1]
 ; GCN-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v2
 ; GCN-NEXT:    v_cndmask_b32_e32 v7, 0, v0, vcc
-; GCN-NEXT:    ; implicit-def: $vgpr5
+; GCN-NEXT:    ; implicit-def: $vgpr4
 ; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN-NEXT:    ; implicit-def: $vgpr6
 ; GCN-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN-NEXT:  ; %bb.3: ; %Flow3
-; GCN-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; GCN-NEXT:    s_or_saveexec_b64 s[8:9], s[4:5]
+; GCN-NEXT:    v_sub_u32_e32 v5, 0x7f, v6
+; GCN-NEXT:    s_xor_b64 exec, exec, s[8:9]
 ; GCN-NEXT:    s_cbranch_execz .LBB1_13
 ; GCN-NEXT:  ; %bb.4: ; %NodeBlock
-; GCN-NEXT:    v_cmp_lt_i32_e32 vcc, 25, v5
+; GCN-NEXT:    v_cmp_lt_i32_e32 vcc, 25, v4
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:    s_xor_b64 s[10:11], exec, s[4:5]
 ; GCN-NEXT:    s_cbranch_execz .LBB1_8
 ; GCN-NEXT:  ; %bb.5: ; %LeafBlock
-; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v5
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v4
 ; GCN-NEXT:    s_and_saveexec_b64 s[12:13], vcc
 ; GCN-NEXT:    s_cbranch_execz .LBB1_7
 ; GCN-NEXT:  ; %bb.6: ; %itofp-sw-default
@@ -249,13 +249,13 @@ define bfloat @uitofp_i128_to_bf16(i128 %x) {
 ; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-NEXT:  ; %bb.11: ; %itofp-if-then20
 ; GCN-NEXT:    v_alignbit_b32 v7, v1, v0, 3
-; GCN-NEXT:    v_mov_b32_e32 v4, v5
+; GCN-NEXT:    v_mov_b32_e32 v5, v4
 ; GCN-NEXT:  ; %bb.12: ; %Flow
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN-NEXT:  .LBB1_13: ; %Flow4
 ; GCN-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GCN-NEXT:    v_and_b32_e32 v0, 0x7fffff, v7
-; GCN-NEXT:    v_lshl_or_b32 v0, v4, 23, v0
+; GCN-NEXT:    v_lshl_or_b32 v0, v5, 23, v0
 ; GCN-NEXT:    v_add_u32_e32 v0, 1.0, v0
 ; GCN-NEXT:    v_bfe_u32 v1, v0, 16, 1
 ; GCN-NEXT:    s_movk_i32 s4, 0x7fff
diff --git a/llvm/test/CodeGen/AMDGPU/itofp.i128.ll b/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
index f372a54894604..c316ec71863d0 100644
--- a/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
@@ -34,30 +34,30 @@ define float @sitofp_i128_to_f32(i128 %x) {
 ; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; SDAG-NEXT:    v_add_u32_e32 v6, 64, v6
 ; SDAG-NEXT:    v_cndmask_b32_e32 v7, v6, v2, vcc
-; SDAG-NEXT:    v_sub_u32_e32 v6, 0x80, v7
-; SDAG-NEXT:    v_sub_u32_e32 v2, 0x7f, v7
-; SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 25, v6
+; SDAG-NEXT:    v_sub_u32_e32 v2, 0x80, v7
+; SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 25, v2
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SDAG-NEXT:  ; %bb.2: ; %itofp-if-else
-; SDAG-NEXT:    v_add_u32_e32 v4, 0xffffff98, v7
-; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v4, v[0:1]
-; SDAG-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v4
+; SDAG-NEXT:    v_add_u32_e32 v2, 0xffffff98, v7
+; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v2, v[0:1]
+; SDAG-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v2
 ; SDAG-NEXT:    v_cndmask_b32_e32 v8, 0, v0, vcc
-; SDAG-NEXT:    ; implicit-def: $vgpr6
+; SDAG-NEXT:    ; implicit-def: $vgpr2
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; SDAG-NEXT:    ; implicit-def: $vgpr7
 ; SDAG-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; SDAG-NEXT:  ; %bb.3: ; %Flow3
-; SDAG-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; SDAG-NEXT:    s_or_saveexec_b64 s[8:9], s[4:5]
+; SDAG-NEXT:    v_sub_u32_e32 v6, 0x7f, v7
+; SDAG-NEXT:    s_xor_b64 exec, exec, s[8:9]
 ; SDAG-NEXT:    s_cbranch_execz .LBB0_13
 ; SDAG-NEXT:  ; %bb.4: ; %NodeBlock
-; SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, 25, v6
+; SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, 25, v2
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:    s_xor_b64 s[10:11], exec, s[4:5]
 ; SDAG-NEXT:    s_cbranch_execz .LBB0_8
 ; SDAG-NEXT:  ; %bb.5: ; %LeafBlock
-; SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v6
+; SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v2
 ; SDAG-NEXT:    s_and_saveexec_b64 s[12:13], vcc
 ; SDAG-NEXT:    s_cbranch_execz .LBB0_7
 ; SDAG-NEXT:  ; %bb.6: ; %itofp-sw-default
@@ -116,13 +116,13 @@ define float @sitofp_i128_to_f32(i128 %x) {
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:  ; %bb.11: ; %itofp-if-then20
 ; SDAG-NEXT:    v_alignbit_b32 v8, v1, v0, 3
-; SDAG-NEXT:    v_mov_b32_e32 v2, v6
+; SDAG-NEXT:    v_mov_b32_e32 v6, v2
 ; SDAG-NEXT:  ; %bb.12: ; %Flow
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SDAG-NEXT:  .LBB0_13: ; %Flow4
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SDAG-NEXT:    v_and_b32_e32 v0, 0x80000000, v3
-; SDAG-NEXT:    v_lshl_add_u32 v1, v2, 23, 1.0
+; SDAG-NEXT:    v_lshl_add_u32 v1, v6, 23, 1.0
 ; SDAG-NEXT:    v_and_b32_e32 v2, 0x7fffff, v8
 ; SDAG-NEXT:    v_or3_b32 v4, v2, v0, v1
 ; SDAG-NEXT:  .LBB0_14: ; %Flow5
@@ -161,9 +161,8 @@ define float @sitofp_i128_to_f32(i128 %x) {
 ; GISEL-NEXT:    v_add_u32_e32 v4, 64, v4
 ; GISEL-NEXT:    v_min_u32_e32 v5, v5, v7
 ; GISEL-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc
-; GISEL-NEXT:    v_sub_u32_e32 v8, 0x80, v5
-; GISEL-NEXT:    v_sub_u32_e32 v7, 0x7f, v5
-; GISEL-NEXT:    v_cmp_ge_i32_e32 vcc, 24, v8
+; GISEL-NEXT:    v_sub_u32_e32 v7, 0x80, v5
+; GISEL-NEXT:    v_cmp_ge_i32_e32 vcc, 24, v7
 ; GISEL-NEXT:    ; implicit-def: $vgpr4
 ; GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GISEL-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -172,20 +171,21 @@ define float @sitofp_i128_to_f32(i128 %x) {
 ; GISEL-NEXT:    v_lshlrev_b64 v[0:1], v2, v[0:1]
 ; GISEL-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v2
 ; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
-; GISEL-NEXT:    ; implicit-def: $vgpr8
+; GISEL-NEXT:    ; implicit-def: $vgpr7
 ; GISEL-NEXT:    ; implicit-def: $vgpr0
-; GISEL-NEXT:    ; implicit-def: $vgpr5
 ; GISEL-NEXT:    ; implicit-def: $vgpr2
 ; GISEL-NEXT:  ; %bb.3: ; %Flow3
-; GISEL-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT:    s_or_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT:    v_sub_u32_e32 v8, 0x7f, v5
+; GISEL-NEXT:    s_xor_b64 exec, exec, s[8:9]
 ; GISEL-NEXT:    s_cbranch_execz .LBB0_13
 ; GISEL-NEXT:  ; %bb.4: ; %NodeBlock
-; GISEL-NEXT:    v_cmp_le_i32_e32 vcc, 26, v8
+; GISEL-NEXT:    v_cmp_le_i32_e32 vcc, 26, v7
 ; GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GISEL-NEXT:    s_xor_b64 s[10:11], exec, s[4:5]
 ; GISEL-NEXT:    s_cbranch_execz .LBB0_8
 ; GISEL-NEXT:  ; %bb.5: ; %LeafBlock
-; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v8
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v7
 ; GISEL-NEXT:    s_and_saveexec_b64 s[12:13], vcc
 ; GISEL-NEXT:    s_cbranch_execz .LBB0_7
 ; GISEL-NEXT:  ; %bb.6: ; %itofp-sw-default
@@ -249,13 +249,13 @@ define float @sitofp_i128_to_f32(i128 %x) {
 ; GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GISEL-NEXT:  ; %bb.11: ; %itofp-if-then20
 ; GISEL-NEXT:    v_lshrrev_b64 v[4:5], 3, v[0:1]
-; GISEL-NEXT:    v_mov_b32_e32 v7, v8
+; GISEL-NEXT:    v_mov_b32_e32 v8, v7
 ; GISEL-NEXT:  ; %bb.12: ; %Flow
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GISEL-NEXT:  .LBB0_13: ; %Flow4
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GISEL-NEXT:    v_and_b32_e32 v0, 0x80000000, v6
-; GISEL-NEXT:    v_lshl_add_u32 v1, v7, 23, 1.0
+; GISEL-NEXT:    v_lshl_add_u32 v1, v8, 23, 1.0
 ; GISEL-NEXT:    v_and_b32_e32 v2, 0x7fffff, v4
 ; GISEL-NEXT:    v_or3_b32 v4, v2, v0, v1
 ; GISEL-NEXT:  .LBB0_14: ; %Flow5
@@ -288,9 +288,8 @@ define float @uitofp_i128_to_f32(i128 %x) {
 ; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
 ; SDAG-NEXT:    v_add_u32_e32 v5, 64, v5
 ; SDAG-NEXT:    v_cndmask_b32_e32 v6, v5, v4, vcc
-; SDAG-NEXT:    v_sub_u32_e32 v5, 0x80, v6
-; SDAG-NEXT:    v_sub_u32_e32 v4, 0x7f, v6
-; SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 25, v5
+; SDAG-NEXT:    v_sub_u32_e32 v4, 0x80, v6
+; SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 25, v4
 ; SDAG-NEXT:    ; implicit-def: $vgpr7
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -299,20 +298,21 @@ define float @uitofp_i128_to_f32(i128 %x) {
 ; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v2, v[0:1]
 ; SDAG-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v2
 ; SDAG-NEXT:    v_cndmask_b32_e32 v7, 0, v0, vcc
-; SDAG-NEXT:    ; implicit-def: $vgpr5
+; SDAG-NEXT:    ; implicit-def: $vgpr4
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; SDAG-NEXT:    ; implicit-def: $vgpr6
 ; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; SDAG-NEXT:  ; %bb.3: ; %Flow3
-; SDAG-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; SDAG-NEXT:    s_or_saveexec_b64 s[8:9], s[4:5]
+; SDAG-NEXT:    v_sub_u32_e32 v5, 0x7f, v6
+; SDAG-NEXT:    s_xor_b64 exec, exec, s[8:9]
 ; SDAG-NEXT:    s_cbranch_execz .LBB1_13
 ; SDAG-NEXT:  ; %bb.4: ; %NodeBlock
-; SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, 25, v5
+; SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, 25, v4
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:    s_xor_b64 s[10:11], exec, s[4:5]
 ; SDAG-NEXT:    s_cbranch_execz .LBB1_8
 ; SDAG-NEXT:  ; %bb.5: ; %LeafBlock
-; SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v5
+; SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v4
 ; SDAG-NEXT:    s_and_saveexec_b64 s[12:13], vcc
 ; SDAG-NEXT:    s_cbranch_execz .LBB1_7
 ; SDAG-NEXT:  ; %bb.6: ; %itofp-sw-default
@@ -371,13 +371,13 @@ define float @uitofp_i128_to_f32(i128 %x) {
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:  ; %bb.11: ; %itofp-if-then20
 ; SDAG-NEXT:    v_alignbit_b32 v7, v1, v0, 3
-; SDAG-NEXT:    v_mov_b32_e32 v4, v5
+; SDAG-NEXT:    v_mov_b32_e32 v5, v4
 ; SDAG-NEXT:  ; %bb.12: ; %Flow
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SDAG-NEXT:  .LBB1_13: ; %Flow4
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SDAG-NEXT:    v_and_b32_e32 v0, 0x7fffff, v7
-; SDAG-NEXT:    v_lshl_or_b32 v0, v4, 23, v0
+; SDAG-NEXT:    v_lshl_or_b32 v0, v5, 23, v0
 ; SDAG-NEXT:    v_add_u32_e32 v4, 1.0, v0
 ; SDAG-NEXT:  .LBB1_14: ; %Flow5
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[6:7]
@@ -406,9 +406,8 @@ define float @uitofp_i128_to_f32(i128 %x) {
 ; GISEL-NEXT:    v_add_u32_e32 v4, 64, v4
 ; GISEL-NEXT:    v_min_u32_e32 v5, v5, v6
 ; GISEL-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc
-; GISEL-NEXT:    v_sub_u32_e32 v7, 0x80, v5
-; GISEL-NEXT:    v_sub_u32_e32 v6, 0x7f, v5
-; GISEL-NEXT:    v_cmp_ge_i32_e32 vcc, 24, v7
+; GISEL-NEXT:    v_sub_u32_e32 v6, 0x80, v5
+; GISEL-NEXT:    v_cmp_ge_i32_e32 vcc, 24, v6
 ; GISEL-NEXT:    ; implicit-def: $vgpr4
 ; GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GISEL-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -417,20 +416,21 @@ define float @uitofp_i128_to_f32(i128 %x) {
 ; GISEL-NEXT:    v_lshlrev_b64 v[0:1], v2, v[0:1]
 ; GISEL-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v2
 ; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
-; GISEL-NEXT:    ; implicit-def: $vgpr7
+; GISEL-NEXT:    ; implicit-def: $vgpr6
 ; GISEL-NEXT:    ; implicit-def: $vgpr0
-; GISEL-NEXT:    ; implicit-def: $vgpr5
 ; GISEL-NEXT:    ; implicit-def: $vgpr2
 ; GISEL-NEXT:  ; %bb.3: ; %Flow3
-; GISEL-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT:    s_or_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT:    v_sub_u32_e32 v7, 0x7f, v5
+; GISEL-NEXT:    s_xor_b64 exec, exec, s[8:9]
 ; GISEL-NEXT:    s_cbranch_execz .LBB1_13
 ; GISEL-NEXT:  ; %bb.4: ; %NodeBlock
-; GISEL-NEXT:    v_cmp_le_i32_e32 vcc, 26, v7
+; GISEL-NEXT:    v_cmp_le_i32_e32 vcc, 26, v6
 ; GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GISEL-NEXT:    s_xor_b64 s[10:11], exec, s[4:5]
 ; GISEL-NEXT:    s_cbranch_execz .LBB1_8
 ; GISEL-NEXT:  ; %bb.5: ; %LeafBlock
-; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v7
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v6
 ; GISEL-NEXT:    s_and_saveexec_b64 s[12:13], vcc
 ; GISEL-NEXT:    s_cbranch_execz .LBB1_7
 ; GISEL-NEXT:  ; %bb.6: ; %itofp-sw-default
@@ -494,12 +494,12 @@ define float @uitofp_i128_to_f32(i128 %x) {
 ; GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GISEL-NEXT:  ; %bb.11: ; %itofp-if-then20
 ; GISEL-NEXT:    v_lshrrev_b64 v[4:5], 3, v[0:1]
-; GISEL-NEXT:    v_mov_b32_e32 v6, v7
+; GISEL-NEXT:    v_mov_b32_e32 v7, v6
 ; GISEL-NEXT:  ; %bb.12: ; %Flow
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GISEL-NEXT:  .LBB1_13: ; %Flow4
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GISEL-NEXT:    v_lshl_add_u32 v0, v6, 23, 1.0
+; GISEL-NEXT:    v_lshl_add_u32 v0, v7, 23, 1.0
 ; GISEL-NEXT:    v_mov_b32_e32 v1, 0x7fffff
 ; GISEL-NEXT:    v_and_or_b32 v4, v4, v1, v0
 ; GISEL-NEXT:  .LBB1_14: ; %Flow5
@@ -545,32 +545,32 @@ define double @sitofp_i128_to_f64(i128 %x) {
 ; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
 ; SDAG-NEXT:    v_add_u32_e32 v1, 64, v1
 ; SDAG-NEXT:    v_cndmask_b32_e32 v9, v1, v0, vcc
-; SDAG-NEXT:    v_sub_u32_e32 v8, 0x80, v9
-; SDAG-NEXT:    v_sub_u32_e32 v2, 0x7f, v9
-; SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 54, v8
+; SDAG-NEXT:    v_sub_u32_e32 v2, 0x80, v9
+; SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 54, v2
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SDAG-NEXT:  ; %bb.2: ; %itofp-if-else
-; SDAG-NEXT:    v_add_u32_e32 v6, 0xffffffb5, v9
-; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v6, v[4:5]
-; SDAG-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v6
+; SDAG-NEXT:    v_add_u32_e32 v2, 0xffffffb5, v9
+; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v2, v[4:5]
+; SDAG-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v2
 ; SDAG-NEXT:    v_cndmask_b32_e32 v10, 0, v1, vcc
 ; SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; SDAG-NEXT:    ; implicit-def: $vgpr8
+; SDAG-NEXT:    ; implicit-def: $vgpr2
 ; SDAG-NEXT:    ; implicit-def: $vgpr6_vgpr7
 ; SDAG-NEXT:    ; implicit-def: $vgpr4_vgpr5
-; SDAG-NEXT:    ; implicit-def: $vgpr9
 ; SDAG-NEXT:  ; %bb.3: ; %Flow3
-; SDAG-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; SDAG-NEXT:    s_or_saveexec_b64 s[8:9], s[4:5]
+; SDAG-NEXT:    v_sub_u32_e32 v8, 0x7f, v9
+; SDAG-NEXT:    s_xor_b64 exec, exec, s[8:9]
 ; SDAG-NEXT:    s_cbranch_execz .LBB2_13
 ; SDAG-NEXT:  ; %bb.4: ; %NodeBlock
-; SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, 54, v8
+; SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, 54, v2
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:    s_xor_b64 s[10:11], exec, s[4:5]
 ; SDAG-NEXT:    s_cbranch_execz .LBB2_8
 ; SDAG-NEXT:  ; %bb.5: ; %LeafBlock
-; SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 55, v8
+; SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 55, v2
 ; SDAG-NEXT:    s_and_saveexec_b64 s[12:13], vcc
 ; SDAG-NEXT:    s_cbranch_execz .LBB2_7
 ; SDAG-NEXT:  ; %bb.6: ; %itofp-sw-default
@@ -640,16 +640,16 @@ define double @sitofp_i128_to_f64(i128 %x) {
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:  ; %bb.11: ; %itofp-if-then20
 ; SDAG-NEXT:    v_lshrrev_b64 v[0:1], 3, v[4:5]
-; SDAG-NEXT:    v_lshlrev_b32_e32 v2, 29, v6
-; SDAG-NEXT:    v_or_b32_e32 v10, v1, v2
-; SDAG-NEXT:    v_mov_b32_e32 v2, v8
+; SDAG-NEXT:    v_lshlrev_b32_e32 v4, 29, v6
+; SDAG-NEXT:    v_or_b32_e32 v10, v1, v4
+; SDAG-NEXT:    v_mov_b32_e32 v8, v2
 ; SDAG-NEXT:  ; %bb.12: ; %Flow
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SDAG-NEXT:  .LBB2_13: ; %Flow4
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[8:9]
+; SDAG-NEXT:    v_mov_b32_e32 v2, 0x3ff00000
 ; SDAG-NEXT:    v_and_b32_e32 v1, 0x80000000, v3
-; SDAG-NEXT:    v_mov_b32_e32 v3, 0x3ff00000
-; SDAG-NEXT:    v_lshl_add_u32 v2, v2, 20, v3
+; SDAG-NEXT:    v_lshl_add_u32 v2, v8, 20, v2
 ; SDAG-NEXT:    v_and_b32_e32 v3, 0xfffff, v10
 ; SDAG-NEXT:    v_or3_b32 v1, v3, v1, v2
 ; SDAG-NEXT:  .LBB2_14: ; %Flow5
@@ -690,9 +690,8 @@ define double @sitofp_i128_to_f64(i128 %x) {
 ; GISEL-NEXT:    v_add_u32_e32 v0, 64, v0
 ; GISEL-NEXT:    v_min_u32_e32 v1, v1, v7
 ; GISEL-NEXT:    v_cndmask_b32_e32 v9, v1, v0, vcc
-; GISEL-NEXT:    v_sub_u32_e32 v8, 0x80, v9
-; GISEL-NEXT:    v_sub_u32_e32 v7, 0x7f, v9
-; GISEL-NEXT:    v_cmp_ge_i32_e32 vcc, 53, v8
+; GISEL-NEXT:    v_sub_u32_e32 v7, 0x80, v9
+; GISEL-NEXT:    v_cmp_ge_i32_e32 vcc, 53, v7
 ; GISEL-NEXT:    ; implicit-def: $vgpr10
 ; GISEL-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
@@ -703,19 +702,20 @@ define double @sitofp_i128_to_f64(i128 %x) {
 ; GISEL-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v4
 ; GISEL-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
 ; GISEL-NEXT:    v_cndmask_b32_e32 v10, 0, v1, vcc
-; GISEL-NEXT:    ; implicit-def: $vgpr8
+; GISEL-NEXT:    ; implicit-def: $vgpr7
 ; GISEL-NEXT:    ; implicit-def: $vgpr2
-; GISEL-NEXT:    ; implicit-def: $vgpr9
 ; GISEL-NEXT:  ; %bb.3: ; %Flow3
-; GISEL-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT:    s_or_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT:    v_sub_u32_e32 v8, 0x7f, v9
+; GISEL-NEXT:    s_xor_b64 exec, exec, s[8:9]
 ; GISEL-NEXT:    s_cbranch_execz .LBB2_13
 ; GISEL-NEXT:  ; %bb.4: ; %NodeBlock
-; GISEL-NEXT:    v_cmp_le_i32_e32 vcc, 55, v8
+; GISEL-NEXT:    v_cmp_le_i32_e32 vcc, 55, v7
 ; GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GISEL-NEXT:    s_xor_b64 s[10:11], exec, s[4:5]
 ; GISEL-NEXT:    s_cbranch_execz .LBB2_8
 ; GISEL-NEXT:  ; %bb.5: ; %LeafBlock
-; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 55, v8
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 55, v7
 ; GISEL-NEXT:    s_and_saveexec_b64 s[12:13], vcc
 ; GISEL-NEXT:    s_cbranch_execz .LBB2_7
 ; GISEL-NEXT:  ; %bb.6: ; %itofp-sw-default
@@ -791,7 +791,7 @@ define double @sitofp_i128_to_f64(i128 %x) {
 ; GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GISEL-NEXT:  ; %bb.11: ; %itofp-if-then20
 ; GISEL-NEXT:    v_lshrrev_b64 v[0:1], 3, v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v7, v8
+; GISEL-NEXT:    v_mov_b32_e32 v8, v7
 ; GISEL-NEXT:    v_lshl_or_b32 v10, v4, 29, v1
 ; GISEL-NEXT:  ; %bb.12: ; %Flow
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[4:5]
@@ -800,7 +800,7 @@ define double @sitofp_i128_to_f64(i128 %x) {
 ; GISEL-NEXT:    v_and_b32_e32 v1, 0x80000000, v6
 ; GISEL-NEXT:    v_mov_b32_e32 v2, 0x3ff00000
 ; GISEL-NEXT:    v_mov_b32_e32 v3, 0xfffff
-; GISEL-NEXT:    v_lshl_add_u32 v2, v7, 20, v2
+; GISEL-NEXT:    v_lshl_add_u32 v2, v8, 20, v2
 ; GISEL-NEXT:    v_and_or_b32 v1, v10, v3, v1
 ; GISEL-NEXT:    v_or3_b32 v1, v1, v2, 0
 ; GISEL-NEXT:  .LBB2_14: ; %Flow5
@@ -833,9 +833,8 @@ define double @uitofp_i128_to_f64(i128 %x) {
 ; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
 ; SDAG-NEXT:    v_add_u32_e32 v5, 64, v5
 ; SDAG-NEXT:    v_cndmask_b32_e32 v8, v5, v4, vcc
-; SDAG-NEXT:    v_sub_u32_e32 v7, 0x80, v8
-; SDAG-NEXT:    v_sub_u32_e32 v6, 0x7f, v8
-; SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 54, v7
+; SDAG-NEXT:    v_sub_u32_e32 v6, 0x80, v8
+; SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 54, v6
 ; SDAG-NEXT:    ; implicit-def: $vgpr9
 ; SDAG-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
@@ -846,20 +845,21 @@ define double @uitofp_i128_to_f64(i128 %x) {
 ; SDAG-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v2
 ; SDAG-NEXT:    v_cndmask_b32_e32 v9, 0, v1, vcc
 ; SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
-; SDAG-NEXT:    ; implicit-def: $vgpr7
+; SDAG-NEXT:    ; implicit-def: $vgpr6
 ; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; SDAG-NEXT:    ; implicit-def: $vgpr8
 ; SDAG-NEXT:  ; %bb.3: ; %Flow3
-; SDAG-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; SDAG-NEXT:    s_or_saveexec_b64 s[8:9], s[4:5]
+; SDAG-NEXT:    v_sub_u32_e32 v7, 0x7f, v8
+; SDAG-NEXT:    s_xor_b64 exec, exec, s[8:9]
 ; SDAG-NEXT:    s_cbranch_execz .LBB3_13
 ; SDAG-NEXT:  ; %bb.4: ; %NodeBlock
-; SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, 54, v7
+; SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, 54, v6
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:    s_xor_b64 s[10:11], exec, s[4:5]
 ; SDAG-NEXT:    s_cbranch_execz .LBB3_8
 ; SDAG-NEXT:  ; %bb.5: ; %LeafBlock
-; SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 55, v7
+; SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 55, v6
 ; SDAG-NEXT:    s_and_saveexec_b64 s[12:13], vcc
 ; SDAG-NEXT:    s_cbranch_execz .LBB3_7
 ; SDAG-NEXT:  ; %bb.6: ; %itofp-sw-default
@@ -929,13 +929,13 @@ define double @uitofp_i128_to_f64(i128 %x) {
 ; SDAG-NEXT:  ; %bb.11: ; %itofp-if-then20
 ; SDAG-NEXT:    v_lshrrev_b64 v[4:5], 3, v[0:1]
 ; SDAG-NEXT:    v_alignbit_b32 v9, v2, v1, 3
-; SDAG-NEXT:    v_mov_b32_e32 v6, v7
+; SDAG-NEXT:    v_mov_b32_e32 v7, v6
 ; SDAG-NEXT:  ; %bb.12: ; %Flow
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SDAG-NEXT:  .LBB3_13: ; %Flow4
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SDAG-NEXT:    v_and_b32_e32 v0, 0xfffff, v9
-; SDAG-NEXT:    v_lshl_or_b32 v0, v6, 20, v0
+; SDAG-NEXT:    v_lshl_or_b32 v0, v7, 20, v0
 ; SDAG-NEXT:    v_add_u32_e32 v5, 0x3ff00000, v0
 ; SDAG-NEXT:  .LBB3_14: ; %Flow5
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[6:7]
@@ -966,9 +966,8 @@ define double @uitofp_i128_to_f64(i128 %x) {
 ; GISEL-NEXT:    v_add_u32_e32 v4, 64, v4
 ; GISEL-NEXT:    v_min_u32_e32 v5, v5, v6
 ; GISEL-NEXT:    v_cndmask_b32_e32 v8, v5, v4, vcc
-; GISEL-NEXT:    v_sub_u32_e32 v7, 0x80, v8
-; GISEL-NEXT:    v_sub_u32_e32 v6, 0x7f, v8
-; GISEL-NEXT:    v_cmp_ge_i32_e32 vcc, 53, v7
+; GISEL-NEXT:    v_sub_u32_e32 v6, 0x80, v8
+; GISEL-NEXT:    v_cmp_ge_i32_e32 vcc, 53, v6
 ; GISEL-NEXT:    ; implicit-def: $vgpr9
 ; GISEL-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
@@ -979,19 +978,20 @@ define double @uitofp_i128_to_f64(i128 %x) {
 ; GISEL-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v2
 ; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
 ; GISEL-NEXT:    v_cndmask_b32_e32 v9, 0, v1, vcc
-; GISEL-NEXT:    ; implicit-def: $vgpr7
+; GISEL-NEXT:    ; implicit-def: $vgpr6
 ; GISEL-NEXT:    ; implicit-def: $vgpr0
-; GISEL-NEXT:    ; implicit-def: $vgpr8
 ; GISEL-NEXT:  ; %bb.3: ; %Flow3
-; GISEL-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT:    s_or_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT:    v_sub_u32_e32 v7, 0x7f, v8
+; GISEL-NEXT:    s_xor_b64 exec, exec, s[8:9]
 ; GISEL-NEXT:    s_cbranch_execz .LBB3_13
 ; GISEL-NEXT:  ; %bb.4: ; %NodeBlock
-; GISEL-NEXT:    v_cmp_le_i32_e32 vcc, 55, v7
+; GISEL-NEXT:    v_cmp_le_i32_e32 vcc, 55, v6
 ; GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GISEL-NEXT:    s_xor_b64 s[10:11], exec, s[4:5]
 ; GISEL-NEXT:    s_cbranch_execz .LBB3_8
 ; GISEL-NEXT:  ; %bb.5: ; %LeafBlock
-; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 55, v7
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 55, v6
 ; GISEL-NEXT:    s_and_saveexec_b64 s[12:13], vcc
 ; GISEL-NEXT:    s_cbranch_execz .LBB3_7
 ; GISEL-NEXT:  ; %bb.6: ; %itofp-sw-default
@@ -1074,13 +1074,13 @@ define double @uitofp_i128_to_f64(i128 %x) {
 ; GISEL-NEXT:    v_lshrrev_b64 v[4:5], 3, v[0:1]
 ; GISEL-NEXT:    v_lshrrev_b32_e32 v0, 3, v1
 ; GISEL-NEXT:    v_or_b32_e32 v9, v0, v2
-; GISEL-NEXT:    v_mov_b32_e32 v6, v7
+; GISEL-NEXT:    v_mov_b32_e32 v7, v6
 ; GISEL-NEXT:  ; %bb.12: ; %Flow
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GISEL-NEXT:  .LBB3_13: ; %Flow4
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0x3ff00000
-; GISEL-NEXT:    v_lshl_add_u32 v0, v6, 20, v0
+; GISEL-NEXT:    v_lshl_add_u32 v0, v7, 20, v0
 ; GISEL-NEXT:    v_and_b32_e32 v1, 0xfffff, v9
 ; GISEL-NEXT:    v_or3_b32 v5, v1, v0, 0
 ; GISEL-NEXT:  .LBB3_14: ; %Flow5
@@ -1124,30 +1124,30 @@ define half @sitofp_i128_to_f16(i128 %x) {
 ; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; SDAG-NEXT:    v_add_u32_e32 v6, 64, v6
 ; SDAG-NEXT:    v_cndmask_b32_e32 v7, v6, v2, vcc
-; SDAG-NEXT:    v_sub_u32_e32 v6, 0x80, v7
-; SDAG-NEXT:    v_sub_u32_e32 v2, 0x7f, v7
-; SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 25, v6
+; SDAG-NEXT:    v_sub_u32_e32 v2, 0x80, v7
+; SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 25, v2
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; SDAG-NEXT:  ; %bb.2: ; %itofp-if-else
-; SDAG-NEXT:    v_add_u32_e32 v4, 0xffffff98, v7
-; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v4, v[0:1]
-; SDAG-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v4
+; SDAG-NEXT:    v_add_u32_e32 v2, 0xffffff98, v7
+; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v2, v[0:1]
+; SDAG-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v2
 ; SDAG-NEXT:    v_cndmask_b32_e32 v8, 0, v0, vcc
-; SDAG-NEXT:    ; implicit-def: $vgpr6
+; SDAG-NEXT:    ; implicit-def: $vgpr2
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; SDAG-NEXT:    ; implicit-def: $vgpr7
 ; SDAG-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; SDAG-NEXT:  ; %bb.3: ; %Flow3
-; SDAG-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; SDAG-NEXT:    s_or_saveexec_b64 s[8:9], s[4:5]
+; SDAG-NEXT:    v_sub_u32_e32 v6, 0x7f, v7
+; SDAG-NEXT:    s_xor_b64 exec, exec, s[8:9]
 ; SDAG-NEXT:    s_cbranch_execz .LBB4_13
 ; SDAG-NEXT:  ; %bb.4: ; %NodeBlock
-; SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, 25, v6
+; SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, 25, v2
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:    s_xor_b64 s[10:11], exec, s[4:5]
 ; SDAG-NEXT:    s_cbranch_execz .LBB4_8
 ; SDAG-NEXT:  ; %bb.5: ; %LeafBlock
-; SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v6
+; SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v2
 ; SDAG-NEXT:    s_and_saveexec_b64 s[12:13], vcc
 ; SDAG-NEXT:    s_cbranch_execz .LBB4_7
 ; SDAG-NEXT:  ; %bb.6: ; %itofp-sw-default
@@ -1206,13 +1206,13 @@ define half @sitofp_i128_to_f16(i128 %x) {
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:  ; %bb.11: ; %itofp-if-then20
 ; SDAG-NEXT:    v_alignbit_b32 v8, v1, v0, 3
-; SDAG-NEXT:    v_mov_b32_e32 v2, v6
+; SDAG-NEXT:    v_mov_b32_e32 v6, v2
 ; SDAG-NEXT:  ; %bb.12: ; %Flow
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SDAG-NEXT:  .LBB4_13: ; %Flow4
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SDAG-NEXT:    v_and_b32_e32 v0, 0x80000000, v3
-; SDAG-NEXT:    v_lshl_add_u32 v1, v2, 23, 1.0
+; SDAG-NEXT:    v_lshl_add_u32 v1, v6, 23, 1.0
 ; SDAG-NEXT:    v_and_b32_e32 v2, 0x7fffff, v8
 ; SDAG-NEXT:    v_or3_b32 v0, v2, v0, v1
 ; SDAG-NEXT:    v_cvt_f16_f32_e32 v4, v0
@@ -1252,9 +1252,8 @@ define half @sitofp_i128_to_f16(i128 %x) {
 ; GISEL-NEXT:    v_add_u32_e32 v4, 64, v4
 ; GISEL-NEXT:    v_min_u32_e32 v5, v5, v7
 ; GISEL-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc
-; GISEL-NEXT:    v_sub_u32_e32 v8, 0x80, v5
-; GISEL-NEXT:    v_sub_u32_e32 v7, 0x7f, v5
-; GISEL-NEXT:    v_cmp_ge_i32_e32 vcc, 24, v8
+; GISEL-NEXT:    v_sub_u32_e32 v7, 0x80, v5
+; GISEL-NEXT:    v_cmp_ge_i32_e32 vcc, 24, v7
 ; GISEL-NEXT:    ; implicit-def: $vgpr4
 ; GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GISEL-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -1263,20 +1262,21 @@ define half @sitofp_i128_to_f16(i128 %x) {
 ; GISEL-NEXT:    v_lshlrev_b64 v[0:1], v2, v[0:1]
 ; GISEL-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v2
 ; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
-; GISEL-NEXT:    ; implicit-def: $vgpr8
+; GISEL-NEXT:    ; implicit-def: $vgpr7
 ; GISEL-NEXT:    ; implicit-def: $vgpr0
-; GISEL-NEXT:    ; implicit-def: $vgpr5
 ; GISEL-NEXT:    ; implicit-def: $vgpr2
 ; GISEL-NEXT:  ; %bb.3: ; %Flow3
-; GISEL-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT:    s_or_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT:    v_sub_u32_e32 v8, 0x7f, v5
+; GISEL-NEXT:    s_xor_b64 exec, exec, s[8:9]
 ; GISEL-NEXT:    s_cbranch_execz .LBB4_13
 ; GISEL-NEXT:  ; %bb.4: ; %NodeBlock
-; GISEL-NEXT:    v_cmp_le_i32_e32 vcc, 26, v8
+; GISEL-NEXT:    v_cmp_le_i32_e32 vcc, 26, v7
 ; GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GISEL-NEXT:    s_xor_b64 s[10:11], exec, s[4:5]
 ; GISEL-NEXT:    s_cbranch_execz .LBB4_8
 ; GISEL-NEXT:  ; %bb.5: ; %LeafBlock
-; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v8
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v7
 ; GISEL-NEXT:    s_and_saveexec_b64 s[12:13], vcc
 ; GISEL-NEXT:    s_cbranch_execz .LBB4_7
 ; GISEL-NEXT:  ; %bb.6: ; %itofp-sw-default
@@ -1340,13 +1340,13 @@ define half @sitofp_i128_to_f16(i128 %x) {
 ; GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GISEL-NEXT:  ; %bb.11: ; %itofp-if-then20
 ; GISEL-NEXT:    v_lshrrev_b64 v[4:5], 3, v[0:1]
-; GISEL-NEXT:    v_mov_b32_e32 v7, v8
+; GISEL-NEXT:    v_mov_b32_e32 v8, v7
 ; GISEL-NEXT:  ; %bb.12: ; %Flow
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GISEL-NEXT:  .LBB4_13: ; %Flow4
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GISEL-NEXT:    v_and_b32_e32 v0, 0x80000000, v6
-; GISEL-NEXT:    v_lshl_add_u32 v1, v7, 23, 1.0
+; GISEL-NEXT:    v_lshl_add_u32 v1, v8, 23, 1.0
 ; GISEL-NEXT:    v_and_b32_e32 v2, 0x7fffff, v4
 ; GISEL-NEXT:    v_or3_b32 v0, v2, v0, v1
 ; GISEL-NEXT:    v_cvt_f16_f32_e32 v4, v0
@@ -1380,9 +1380,8 @@ define half @uitofp_i128_to_f16(i128 %x) {
 ; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
 ; SDAG-NEXT:    v_add_u32_e32 v5, 64, v5
 ; SDAG-NEXT:    v_cndmask_b32_e32 v6, v5, v4, vcc
-; SDAG-NEXT:    v_sub_u32_e32 v5, 0x80, v6
-; SDAG-NEXT:    v_sub_u32_e32 v4, 0x7f, v6
-; SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 25, v5
+; SDAG-NEXT:    v_sub_u32_e32 v4, 0x80, v6
+; SDAG-NEXT:    v_cmp_gt_i32_e32 vcc, 25, v4
 ; SDAG-NEXT:    ; implicit-def: $vgpr7
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -1391,20 +1390,21 @@ define half @uitofp_i128_to_f16(i128 %x) {
 ; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v2, v[0:1]
 ; SDAG-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v2
 ; SDAG-NEXT:    v_cndmask_b32_e32 v7, 0, v0, vcc
-; SDAG-NEXT:    ; implicit-def: $vgpr5
+; SDAG-NEXT:    ; implicit-def: $vgpr4
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; SDAG-NEXT:    ; implicit-def: $vgpr6
 ; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; SDAG-NEXT:  ; %bb.3: ; %Flow3
-; SDAG-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; SDAG-NEXT:    s_or_saveexec_b64 s[8:9], s[4:5]
+; SDAG-NEXT:    v_sub_u32_e32 v5, 0x7f, v6
+; SDAG-NEXT:    s_xor_b64 exec, exec, s[8:9]
 ; SDAG-NEXT:    s_cbranch_execz .LBB5_13
 ; SDAG-NEXT:  ; %bb.4: ; %NodeBlock
-; SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, 25, v5
+; SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, 25, v4
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:    s_xor_b64 s[10:11], exec, s[4:5]
 ; SDAG-NEXT:    s_cbranch_execz .LBB5_8
 ; SDAG-NEXT:  ; %bb.5: ; %LeafBlock
-; SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v5
+; SDAG-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v4
 ; SDAG-NEXT:    s_and_saveexec_b64 s[12:13], vcc
 ; SDAG-NEXT:    s_cbranch_execz .LBB5_7
 ; SDAG-NEXT:  ; %bb.6: ; %itofp-sw-default
@@ -1463,13 +1463,13 @@ define half @uitofp_i128_to_f16(i128 %x) {
 ; SDAG-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; SDAG-NEXT:  ; %bb.11: ; %itofp-if-then20
 ; SDAG-NEXT:    v_alignbit_b32 v7, v1, v0, 3
-; SDAG-NEXT:    v_mov_b32_e32 v4, v5
+; SDAG-NEXT:    v_mov_b32_e32 v5, v4
 ; SDAG-NEXT:  ; %bb.12: ; %Flow
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SDAG-NEXT:  .LBB5_13: ; %Flow4
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SDAG-NEXT:    v_and_b32_e32 v0, 0x7fffff, v7
-; SDAG-NEXT:    v_lshl_or_b32 v0, v4, 23, v0
+; SDAG-NEXT:    v_lshl_or_b32 v0, v5, 23, v0
 ; SDAG-NEXT:    v_add_u32_e32 v0, 1.0, v0
 ; SDAG-NEXT:    v_cvt_f16_f32_e32 v4, v0
 ; SDAG-NEXT:  .LBB5_14: ; %Flow5
@@ -1499,9 +1499,8 @@ define half @uitofp_i128_to_f16(i128 %x) {
 ; GISEL-NEXT:    v_add_u32_e32 v4, 64, v4
 ; GISEL-NEXT:    v_min_u32_e32 v5, v5, v6
 ; GISEL-NEXT:    v_cndmask_b32_e32 v5, v5, v4, vcc
-; GISEL-NEXT:    v_sub_u32_e32 v7, 0x80, v5
-; GISEL-NEXT:    v_sub_u32_e32 v6, 0x7f, v5
-; GISEL-NEXT:    v_cmp_ge_i32_e32 vcc, 24, v7
+; GISEL-NEXT:    v_sub_u32_e32 v6, 0x80, v5
+; GISEL-NEXT:    v_cmp_ge_i32_e32 vcc, 24, v6
 ; GISEL-NEXT:    ; implicit-def: $vgpr4
 ; GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GISEL-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
@@ -1510,20 +1509,21 @@ define half @uitofp_i128_to_f16(i128 %x) {
 ; GISEL-NEXT:    v_lshlrev_b64 v[0:1], v2, v[0:1]
 ; GISEL-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v2
 ; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
-; GISEL-NEXT:    ; implicit-def: $vgpr7
+; GISEL-NEXT:    ; implicit-def: $vgpr6
 ; GISEL-NEXT:    ; implicit-def: $vgpr0
-; GISEL-NEXT:    ; implicit-def: $vgpr5
 ; GISEL-NEXT:    ; implicit-def: $vgpr2
 ; GISEL-NEXT:  ; %bb.3: ; %Flow3
-; GISEL-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT:    s_or_saveexec_b64 s[8:9], s[4:5]
+; GISEL-NEXT:    v_sub_u32_e32 v7, 0x7f, v5
+; GISEL-NEXT:    s_xor_b64 exec, exec, s[8:9]
 ; GISEL-NEXT:    s_cbranch_execz .LBB5_13
 ; GISEL-NEXT:  ; %bb.4: ; %NodeBlock
-; GISEL-NEXT:    v_cmp_le_i32_e32 vcc, 26, v7
+; GISEL-NEXT:    v_cmp_le_i32_e32 vcc, 26, v6
 ; GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GISEL-NEXT:    s_xor_b64 s[10:11], exec, s[4:5]
 ; GISEL-NEXT:    s_cbranch_execz .LBB5_8
 ; GISEL-NEXT:  ; %bb.5: ; %LeafBlock
-; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v7
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 26, v6
 ; GISEL-NEXT:    s_and_saveexec_b64 s[12:13], vcc
 ; GISEL-NEXT:    s_cbranch_execz .LBB5_7
 ; GISEL-NEXT:  ; %bb.6: ; %itofp-sw-default
@@ -1587,12 +1587,12 @@ define half @uitofp_i128_to_f16(i128 %x) {
 ; GISEL-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GISEL-NEXT:  ; %bb.11: ; %itofp-if-then20
 ; GISEL-NEXT:    v_lshrrev_b64 v[4:5], 3, v[0:1]
-; GISEL-NEXT:    v_mov_b32_e32 v6, v7
+; GISEL-NEXT:    v_mov_b32_e32 v7, v6
 ; GISEL-NEXT:  ; %bb.12: ; %Flow
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GISEL-NEXT:  .LBB5_13: ; %Flow4
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GISEL-NEXT:    v_lshl_add_u32 v0, v6, 23, 1.0
+; GISEL-NEXT:    v_lshl_add_u32 v0, v7, 23, 1.0
 ; GISEL-NEXT:    v_mov_b32_e32 v1, 0x7fffff
 ; GISEL-NEXT:    v_and_or_b32 v0, v4, v1, v0
 ; GISEL-NEXT:    v_cvt_f16_f32_e32 v4, v0
diff --git a/llvm/test/CodeGen/AMDGPU/jump-address.ll b/llvm/test/CodeGen/AMDGPU/jump-address.ll
index 0747133c0c581..d58db378e1384 100644
--- a/llvm/test/CodeGen/AMDGPU/jump-address.ll
+++ b/llvm/test/CodeGen/AMDGPU/jump-address.ll
@@ -32,7 +32,7 @@ ENDIF:                                            ; preds = %IF13, %ELSE, %main_
   %temp1.0 = phi float [ 0.000000e+00, %main_body ], [ %23, %IF13 ], [ 0.000000e+00, %ELSE ]
   %temp2.0 = phi float [ 1.000000e+00, %main_body ], [ 0.000000e+00, %ELSE ], [ 0.000000e+00, %IF13 ]
   %temp3.0 = phi float [ 5.000000e-01, %main_body ], [ 0.000000e+00, %ELSE ], [ 0.000000e+00, %IF13 ]
-  %16 = insertelement <4 x float> undef, float %temp.0, i32 0
+  %16 = insertelement <4 x float> poison, float %temp.0, i32 0
   %17 = insertelement <4 x float> %16, float %temp1.0, i32 1
   %18 = insertelement <4 x float> %17, float %temp2.0, i32 2
   %19 = insertelement <4 x float> %18, float %temp3.0, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/kcache-fold.ll b/llvm/test/CodeGen/AMDGPU/kcache-fold.ll
index c3b41af529ecd..f00b8db963337 100644
--- a/llvm/test/CodeGen/AMDGPU/kcache-fold.ll
+++ b/llvm/test/CodeGen/AMDGPU/kcache-fold.ll
@@ -44,7 +44,7 @@ main_body:
   %clamp.i4 = call float @llvm.minnum.f32(float %max.0.i3, float 1.000000e+00)
   %max.0.i1 = call float @llvm.maxnum.f32(float %tmp37, float 0.000000e+00)
   %clamp.i2 = call float @llvm.minnum.f32(float %max.0.i1, float 1.000000e+00)
-  %tmp38 = insertelement <4 x float> undef, float %clamp.i, i32 0
+  %tmp38 = insertelement <4 x float> poison, float %clamp.i, i32 0
   %tmp39 = insertelement <4 x float> %tmp38, float %clamp.i6, i32 1
   %tmp40 = insertelement <4 x float> %tmp39, float %clamp.i4, i32 2
   %tmp41 = insertelement <4 x float> %tmp40, float %clamp.i2, i32 3
@@ -96,7 +96,7 @@ main_body:
   %clamp.i4 = call float @llvm.minnum.f32(float %max.0.i3, float 1.000000e+00)
   %max.0.i1 = call float @llvm.maxnum.f32(float %tmp37, float 0.000000e+00)
   %clamp.i2 = call float @llvm.minnum.f32(float %max.0.i1, float 1.000000e+00)
-  %tmp38 = insertelement <4 x float> undef, float %clamp.i, i32 0
+  %tmp38 = insertelement <4 x float> poison, float %clamp.i, i32 0
   %tmp39 = insertelement <4 x float> %tmp38, float %clamp.i6, i32 1
   %tmp40 = insertelement <4 x float> %tmp39, float %clamp.i4, i32 2
   %tmp41 = insertelement <4 x float> %tmp40, float %clamp.i2, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/lds-bounds.ll b/llvm/test/CodeGen/AMDGPU/lds-bounds.ll
index 942b4f7cf4cbc..c7307cc6f552a 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-bounds.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-bounds.ll
@@ -24,7 +24,7 @@ entry:
   %v.0 = load i32, ptr addrspace(3) %ptr, align 8
   %v.1 = load i32, ptr addrspace(3) %ptr.gep.1
 
-  %r.0 = insertelement <2 x i32> undef, i32 %v.0, i32 0
+  %r.0 = insertelement <2 x i32> poison, i32 %v.0, i32 0
   %r.1 = insertelement <2 x i32> %r.0, i32 %v.1, i32 1
   %bc = bitcast <2 x i32> %r.1 to <2 x float>
   ret <2 x float> %bc
@@ -54,7 +54,7 @@ entry:
   %v.0 = load i32, ptr addrspace(3) %ptr.a
   %v.1 = load i32, ptr addrspace(3) %ptr.b
 
-  %r.0 = insertelement <2 x i32> undef, i32 %v.0, i32 0
+  %r.0 = insertelement <2 x i32> poison, i32 %v.0, i32 0
   %r.1 = insertelement <2 x i32> %r.0, i32 %v.1, i32 1
   %bc = bitcast <2 x i32> %r.1 to <2 x float>
   ret <2 x float> %bc
@@ -88,7 +88,7 @@ entry:
   %v.0 = load i32, ptr addrspace(3) %ptr.a
   %v.1 = load i32, ptr addrspace(3) %ptr.b
 
-  %r.0 = insertelement <2 x i32> undef, i32 %v.0, i32 0
+  %r.0 = insertelement <2 x i32> poison, i32 %v.0, i32 0
   %r.1 = insertelement <2 x i32> %r.0, i32 %v.1, i32 1
   %bc = bitcast <2 x i32> %r.1 to <2 x float>
   ret <2 x float> %bc
@@ -120,7 +120,7 @@ entry:
   %v.0 = load i32, ptr addrspace(3) %ptr.a
   %v.1 = load i32, ptr addrspace(3) %ptr.b
 
-  %r.0 = insertelement <2 x i32> undef, i32 %v.0, i32 0
+  %r.0 = insertelement <2 x i32> poison, i32 %v.0, i32 0
   %r.1 = insertelement <2 x i32> %r.0, i32 %v.1, i32 1
   %bc = bitcast <2 x i32> %r.1 to <2 x float>
   ret <2 x float> %bc
diff --git a/llvm/test/CodeGen/AMDGPU/lds-dma-waits.ll b/llvm/test/CodeGen/AMDGPU/lds-dma-waits.ll
index 5cb3ca0b80b66..3cf02be69d3fe 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-dma-waits.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-dma-waits.ll
@@ -32,7 +32,7 @@ main_body:
   %val.0 = load float, ptr addrspace(3) %gep.0, align 4
   call void @llvm.amdgcn.wave.barrier()
   %val.1 = load float, ptr addrspace(3) %gep.1, align 4
-  %tmp.0 = insertelement <2 x float> undef, float %val.0, i32 0
+  %tmp.0 = insertelement <2 x float> poison, float %val.0, i32 0
   %res = insertelement <2 x float> %tmp.0, float %val.1, i32 1
   store <2 x float> %res, ptr addrspace(1) %out
   ret void
@@ -60,7 +60,7 @@ main_body:
   %val.0 = load float, ptr addrspace(3) %gep.0, align 4
   call void @llvm.amdgcn.wave.barrier()
   %val.1 = load float, ptr addrspace(3) %gep.1, align 4
-  %tmp.0 = insertelement <2 x float> undef, float %val.0, i32 0
+  %tmp.0 = insertelement <2 x float> poison, float %val.0, i32 0
   %res = insertelement <2 x float> %tmp.0, float %val.1, i32 1
   store <2 x float> %res, ptr addrspace(1) %out
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll b/llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll
index 01af334652382..278ad63b0b76c 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll
@@ -18,7 +18,7 @@ bb:
   %load = load <2 x i32>, ptr addrspace(3) %gep, align 4
   %v1 = extractelement <2 x i32> %load, i32 0
   %v2 = extractelement <2 x i32> %load, i32 1
-  %v3 = insertelement <2 x i32> undef, i32 %v2, i32 0
+  %v3 = insertelement <2 x i32> poison, i32 %v2, i32 0
   %v4 = insertelement <2 x i32> %v3, i32 %v1, i32 1
   store <2 x i32> %v4, ptr addrspace(3) %gep, align 4
   ret void
@@ -38,7 +38,7 @@ bb:
   %v2 = extractelement <4 x i32> %load, i32 1
   %v3 = extractelement <4 x i32> %load, i32 2
   %v4 = extractelement <4 x i32> %load, i32 3
-  %v5 = insertelement <4 x i32> undef, i32 %v4, i32 0
+  %v5 = insertelement <4 x i32> poison, i32 %v4, i32 0
   %v6 = insertelement <4 x i32> %v5, i32 %v3, i32 1
   %v7 = insertelement <4 x i32> %v6, i32 %v2, i32 2
   %v8 = insertelement <4 x i32> %v7, i32 %v1, i32 3
@@ -59,7 +59,7 @@ bb:
   %v1 = extractelement <3 x i32> %load, i32 0
   %v2 = extractelement <3 x i32> %load, i32 1
   %v3 = extractelement <3 x i32> %load, i32 2
-  %v5 = insertelement <3 x i32> undef, i32 %v3, i32 0
+  %v5 = insertelement <3 x i32> poison, i32 %v3, i32 0
   %v6 = insertelement <3 x i32> %v5, i32 %v1, i32 1
   %v7 = insertelement <3 x i32> %v6, i32 %v2, i32 2
   store <3 x i32> %v7, ptr addrspace(3) %gep, align 4
@@ -80,7 +80,7 @@ bb:
   %load = load <2 x i32>, ptr %gep, align 4
   %v1 = extractelement <2 x i32> %load, i32 0
   %v2 = extractelement <2 x i32> %load, i32 1
-  %v3 = insertelement <2 x i32> undef, i32 %v2, i32 0
+  %v3 = insertelement <2 x i32> poison, i32 %v2, i32 0
   %v4 = insertelement <2 x i32> %v3, i32 %v1, i32 1
   store <2 x i32> %v4, ptr %gep, align 4
   ret void
@@ -106,7 +106,7 @@ bb:
   %v2 = extractelement <4 x i32> %load, i32 1
   %v3 = extractelement <4 x i32> %load, i32 2
   %v4 = extractelement <4 x i32> %load, i32 3
-  %v5 = insertelement <4 x i32> undef, i32 %v4, i32 0
+  %v5 = insertelement <4 x i32> poison, i32 %v4, i32 0
   %v6 = insertelement <4 x i32> %v5, i32 %v3, i32 1
   %v7 = insertelement <4 x i32> %v6, i32 %v2, i32 2
   %v8 = insertelement <4 x i32> %v7, i32 %v1, i32 3
@@ -131,7 +131,7 @@ bb:
   %v1 = extractelement <3 x i32> %load, i32 0
   %v2 = extractelement <3 x i32> %load, i32 1
   %v3 = extractelement <3 x i32> %load, i32 2
-  %v5 = insertelement <3 x i32> undef, i32 %v3, i32 0
+  %v5 = insertelement <3 x i32> poison, i32 %v3, i32 0
   %v6 = insertelement <3 x i32> %v5, i32 %v1, i32 1
   %v7 = insertelement <3 x i32> %v6, i32 %v2, i32 2
   store <3 x i32> %v7, ptr %gep, align 4
@@ -148,7 +148,7 @@ bb:
   %load = load <2 x i32>, ptr addrspace(3) %gep, align 8
   %v1 = extractelement <2 x i32> %load, i32 0
   %v2 = extractelement <2 x i32> %load, i32 1
-  %v3 = insertelement <2 x i32> undef, i32 %v2, i32 0
+  %v3 = insertelement <2 x i32> poison, i32 %v2, i32 0
   %v4 = insertelement <2 x i32> %v3, i32 %v1, i32 1
   store <2 x i32> %v4, ptr addrspace(3) %gep, align 8
   ret void
@@ -165,7 +165,7 @@ bb:
   %v1 = extractelement <3 x i32> %load, i32 0
   %v2 = extractelement <3 x i32> %load, i32 1
   %v3 = extractelement <3 x i32> %load, i32 2
-  %v5 = insertelement <3 x i32> undef, i32 %v3, i32 0
+  %v5 = insertelement <3 x i32> poison, i32 %v3, i32 0
   %v6 = insertelement <3 x i32> %v5, i32 %v1, i32 1
   %v7 = insertelement <3 x i32> %v6, i32 %v2, i32 2
   store <3 x i32> %v7, ptr addrspace(3) %gep, align 16
@@ -182,7 +182,7 @@ bb:
   %load = load <2 x i32>, ptr %gep, align 8
   %v1 = extractelement <2 x i32> %load, i32 0
   %v2 = extractelement <2 x i32> %load, i32 1
-  %v3 = insertelement <2 x i32> undef, i32 %v2, i32 0
+  %v3 = insertelement <2 x i32> poison, i32 %v2, i32 0
   %v4 = insertelement <2 x i32> %v3, i32 %v1, i32 1
   store <2 x i32> %v4, ptr %gep, align 8
   ret void
@@ -200,7 +200,7 @@ bb:
   %v2 = extractelement <4 x i32> %load, i32 1
   %v3 = extractelement <4 x i32> %load, i32 2
   %v4 = extractelement <4 x i32> %load, i32 3
-  %v5 = insertelement <4 x i32> undef, i32 %v4, i32 0
+  %v5 = insertelement <4 x i32> poison, i32 %v4, i32 0
   %v6 = insertelement <4 x i32> %v5, i32 %v3, i32 1
   %v7 = insertelement <4 x i32> %v6, i32 %v2, i32 2
   %v8 = insertelement <4 x i32> %v7, i32 %v1, i32 3
@@ -222,7 +222,7 @@ bb:
   %v2 = extractelement <4 x i32> %load, i32 1
   %v3 = extractelement <4 x i32> %load, i32 2
   %v4 = extractelement <4 x i32> %load, i32 3
-  %v5 = insertelement <4 x i32> undef, i32 %v4, i32 0
+  %v5 = insertelement <4 x i32> poison, i32 %v4, i32 0
   %v6 = insertelement <4 x i32> %v5, i32 %v3, i32 1
   %v7 = insertelement <4 x i32> %v6, i32 %v2, i32 2
   %v8 = insertelement <4 x i32> %v7, i32 %v1, i32 3
@@ -246,7 +246,7 @@ bb:
   %v2 = extractelement <4 x i32> %load, i32 1
   %v3 = extractelement <4 x i32> %load, i32 2
   %v4 = extractelement <4 x i32> %load, i32 3
-  %v5 = insertelement <4 x i32> undef, i32 %v4, i32 0
+  %v5 = insertelement <4 x i32> poison, i32 %v4, i32 0
   %v6 = insertelement <4 x i32> %v5, i32 %v3, i32 1
   %v7 = insertelement <4 x i32> %v6, i32 %v2, i32 2
   %v8 = insertelement <4 x i32> %v7, i32 %v1, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index d7f54f3b8e9e2..9aca7a5fc741f 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -38,7 +38,7 @@
 ; GCN-O0-NEXT:        Dominator Tree Construction
 ; GCN-O0-NEXT:        Basic Alias Analysis (stateless AA impl)
 ; GCN-O0-NEXT:        Function Alias Analysis Results
-; GCN-O0-NEXT:    Lower OpenCL enqueued blocks
+; GCN-O0-NEXT:    Externalize enqueued block runtime handles
 ; GCN-O0-NEXT:    AMDGPU Software lowering of LDS
 ; GCN-O0-NEXT:    Lower uses of LDS variables from non-kernel functions
 ; GCN-O0-NEXT:    FunctionPass Manager
@@ -188,7 +188,7 @@
 ; GCN-O1-NEXT:        Dominator Tree Construction
 ; GCN-O1-NEXT:        Basic Alias Analysis (stateless AA impl)
 ; GCN-O1-NEXT:        Function Alias Analysis Results
-; GCN-O1-NEXT:    Lower OpenCL enqueued blocks
+; GCN-O1-NEXT:    Externalize enqueued block runtime handles
 ; GCN-O1-NEXT:    AMDGPU Software lowering of LDS
 ; GCN-O1-NEXT:    Lower uses of LDS variables from non-kernel functions
 ; GCN-O1-NEXT:    FunctionPass Manager
@@ -473,7 +473,7 @@
 ; GCN-O1-OPTS-NEXT:        Dominator Tree Construction
 ; GCN-O1-OPTS-NEXT:        Basic Alias Analysis (stateless AA impl)
 ; GCN-O1-OPTS-NEXT:        Function Alias Analysis Results
-; GCN-O1-OPTS-NEXT:    Lower OpenCL enqueued blocks
+; GCN-O1-OPTS-NEXT:    Externalize enqueued block runtime handles
 ; GCN-O1-OPTS-NEXT:    AMDGPU Software lowering of LDS
 ; GCN-O1-OPTS-NEXT:    Lower uses of LDS variables from non-kernel functions
 ; GCN-O1-OPTS-NEXT:    FunctionPass Manager
@@ -788,7 +788,7 @@
 ; GCN-O2-NEXT:        Dominator Tree Construction
 ; GCN-O2-NEXT:        Basic Alias Analysis (stateless AA impl)
 ; GCN-O2-NEXT:        Function Alias Analysis Results
-; GCN-O2-NEXT:    Lower OpenCL enqueued blocks
+; GCN-O2-NEXT:    Externalize enqueued block runtime handles
 ; GCN-O2-NEXT:    AMDGPU Software lowering of LDS
 ; GCN-O2-NEXT:    Lower uses of LDS variables from non-kernel functions
 ; GCN-O2-NEXT:    FunctionPass Manager
@@ -1107,7 +1107,7 @@
 ; GCN-O3-NEXT:        Dominator Tree Construction
 ; GCN-O3-NEXT:        Basic Alias Analysis (stateless AA impl)
 ; GCN-O3-NEXT:        Function Alias Analysis Results
-; GCN-O3-NEXT:    Lower OpenCL enqueued blocks
+; GCN-O3-NEXT:    Externalize enqueued block runtime handles
 ; GCN-O3-NEXT:    AMDGPU Software lowering of LDS
 ; GCN-O3-NEXT:    Lower uses of LDS variables from non-kernel functions
 ; GCN-O3-NEXT:    FunctionPass Manager
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.swap.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.swap.ll
index cee5bbbe85f48..c9f089e248e76 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.swap.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.swap.ll
@@ -41,7 +41,7 @@ if-true:
   br label %endif
 
 endif:
-  %v = phi i32 [ %val, %if-true ], [ undef, %entry ]
+  %v = phi i32 [ %val, %if-true ], [ poison, %entry ]
   %r = bitcast i32 %v to float
   ret float %r
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.d16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.d16.dim.ll
index b4f05bce37680..dbd324b0334ae 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.d16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.d16.dim.ll
@@ -36,7 +36,7 @@ main_body:
 define amdgpu_ps <2 x float> @image_load_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
 main_body:
   %tex = call <3 x half> @llvm.amdgcn.image.load.2d.v3f16.i32(i32 7, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
-  %ext = shufflevector <3 x half> %tex, <3 x half> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %ext = shufflevector <3 x half> %tex, <3 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %r = bitcast <4 x half> %ext to <2 x float>
   ret <2 x float> %r
 }
@@ -90,7 +90,7 @@ main_body:
 define amdgpu_ps <2 x float> @image_load_3d_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %r) {
 main_body:
   %tex = call <3 x half> @llvm.amdgcn.image.load.3d.v3f16.i32(i32 7, i32 %s, i32 %t, i32 %r, <8 x i32> %rsrc, i32 0, i32 0)
-  %ext = shufflevector <3 x half> %tex, <3 x half> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %ext = shufflevector <3 x half> %tex, <3 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %res = bitcast <4 x half> %ext to <2 x float>
   ret <2 x float> %res
 }
@@ -129,7 +129,7 @@ main_body:
 define amdgpu_ps void @image_store_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <2 x float> %in) {
 main_body:
   %r = bitcast <2 x float> %in to <4 x half>
-  %data = shufflevector <4 x half> %r, <4 x half> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  %data = shufflevector <4 x half> %r, <4 x half> poison, <3 x i32> <i32 0, i32 1, i32 2>
   call void @llvm.amdgcn.image.store.2d.v3f16.i32(<3 x half> %data, i32 7, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.2dmsaa.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.2dmsaa.ll
index 4df6fcd584ff6..e51a4903e6697 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.2dmsaa.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.2dmsaa.ll
@@ -11,7 +11,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask1(<8 x i32> inreg %rsrc, i3
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -25,7 +25,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask1(<8 x i32> inreg %rsrc, i3
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
 ; MSAA-NEXT:    [[I3:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -36,7 +36,7 @@ main_body:
   %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [4 x float] undef, float %i, 0
+  %i4 = insertvalue [4 x float] poison, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
   %i7 = insertvalue [4 x float] %i6, float %i3, 3
@@ -51,7 +51,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask2(<8 x i32> inreg %rsrc, i3
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -65,7 +65,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask2(<8 x i32> inreg %rsrc, i3
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
 ; MSAA-NEXT:    [[I3:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -76,7 +76,7 @@ main_body:
   %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 2, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 2, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 2, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [4 x float] undef, float %i, 0
+  %i4 = insertvalue [4 x float] poison, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
   %i7 = insertvalue [4 x float] %i6, float %i3, 3
@@ -91,7 +91,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask4(<8 x i32> inreg %rsrc, i3
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 4, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 4, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 4, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -105,7 +105,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask4(<8 x i32> inreg %rsrc, i3
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
 ; MSAA-NEXT:    [[I3:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -116,7 +116,7 @@ main_body:
   %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 4, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 4, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 4, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [4 x float] undef, float %i, 0
+  %i4 = insertvalue [4 x float] poison, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
   %i7 = insertvalue [4 x float] %i6, float %i3, 3
@@ -131,7 +131,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask8(<8 x i32> inreg %rsrc, i3
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 8, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 8, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 8, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -145,7 +145,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask8(<8 x i32> inreg %rsrc, i3
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
 ; MSAA-NEXT:    [[I3:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -156,7 +156,7 @@ main_body:
   %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 8, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 8, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 8, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [4 x float] undef, float %i, 0
+  %i4 = insertvalue [4 x float] poison, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
   %i7 = insertvalue [4 x float] %i6, float %i3, 3
@@ -171,7 +171,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_reverse(<8 x i32> inreg %rsrc, i
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -185,7 +185,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_reverse(<8 x i32> inreg %rsrc, i
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I3:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -196,7 +196,7 @@ main_body:
   %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [4 x float] undef, float %i, 0
+  %i4 = insertvalue [4 x float] poison, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
   %i7 = insertvalue [4 x float] %i6, float %i3, 3
@@ -212,7 +212,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_vaddr(<8 x i32> inreg %rsrc, i32
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S1]], i32 [[T1]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S2]], i32 [[T2]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S3]], i32 [[T3]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -225,7 +225,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_vaddr(<8 x i32> inreg %rsrc, i32
 ; MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S1]], i32 [[T1]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S2]], i32 [[T2]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S3]], i32 [[T3]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -236,7 +236,7 @@ main_body:
   %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s1, i32 %t1, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s2, i32 %t2, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s3, i32 %t3, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [4 x float] undef, float %i, 0
+  %i4 = insertvalue [4 x float] poison, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
   %i7 = insertvalue [4 x float] %i6, float %i3, 3
@@ -255,7 +255,7 @@ define amdgpu_ps [8 x float] @load_2dmsaa_v8f32(<8 x i32> inreg %rsrc, i32 %s, i
 ; NO-MSAA-NEXT:    [[I5:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I6:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I7:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I8:%.*]] = insertvalue [8 x float] undef, float [[I]], 0
+; NO-MSAA-NEXT:    [[I8:%.*]] = insertvalue [8 x float] poison, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I9:%.*]] = insertvalue [8 x float] [[I8]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I10:%.*]] = insertvalue [8 x float] [[I9]], float [[I2]], 2
 ; NO-MSAA-NEXT:    [[I11:%.*]] = insertvalue [8 x float] [[I10]], float [[I3]], 3
@@ -277,7 +277,7 @@ define amdgpu_ps [8 x float] @load_2dmsaa_v8f32(<8 x i32> inreg %rsrc, i32 %s, i
 ; MSAA-NEXT:    [[I5:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I6:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
 ; MSAA-NEXT:    [[I7:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
-; MSAA-NEXT:    [[I8:%.*]] = insertvalue [8 x float] undef, float [[I]], 0
+; MSAA-NEXT:    [[I8:%.*]] = insertvalue [8 x float] poison, float [[I]], 0
 ; MSAA-NEXT:    [[I9:%.*]] = insertvalue [8 x float] [[I8]], float [[I1]], 1
 ; MSAA-NEXT:    [[I10:%.*]] = insertvalue [8 x float] [[I9]], float [[I2]], 2
 ; MSAA-NEXT:    [[I11:%.*]] = insertvalue [8 x float] [[I10]], float [[I3]], 3
@@ -296,7 +296,7 @@ main_body:
   %i5 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i6 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i7 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i8 = insertvalue [8 x float] undef, float %i, 0
+  %i8 = insertvalue [8 x float] poison, float %i, 0
   %i9 = insertvalue [8 x float] %i8, float %i1, 1
   %i10 = insertvalue [8 x float] %i9, float %i2, 2
   %i11 = insertvalue [8 x float] %i10, float %i3, 3
@@ -312,7 +312,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_interleaved(<8 x i32> inreg %rsr
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
 ; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; NO-MSAA-NEXT:    [[I1:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = insertvalue [4 x float] [[I1]], float [[I2]], 1
 ; NO-MSAA-NEXT:    [[I4:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
@@ -329,7 +329,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_interleaved(<8 x i32> inreg %rsr
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I4:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
 ; MSAA-NEXT:    [[I6:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
-; MSAA-NEXT:    [[I1:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; MSAA-NEXT:    [[I1:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; MSAA-NEXT:    [[I3:%.*]] = insertvalue [4 x float] [[I1]], float [[I2]], 1
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I3]], float [[I4]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I5]], float [[I6]], 3
@@ -337,7 +337,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_interleaved(<8 x i32> inreg %rsr
 ;
 main_body:
   %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = insertvalue [4 x float] undef, float %i, 0
+  %i1 = insertvalue [4 x float] poison, float %i, 0
   %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = insertvalue [4 x float] %i1, float %i2, 1
   %i4 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
@@ -353,7 +353,7 @@ define amdgpu_ps [2 x float] @load_2dmsaa_v2f32_fragId01(<8 x i32> inreg %rsrc,
 ; NO-MSAA-NEXT:  main_body:
 ; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = insertvalue [2 x float] undef, float [[I]], 0
+; NO-MSAA-NEXT:    [[I2:%.*]] = insertvalue [2 x float] poison, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I3:%.*]] = insertvalue [2 x float] [[I2]], float [[I1]], 1
 ; NO-MSAA-NEXT:    ret [2 x float] [[I3]]
 ;
@@ -363,14 +363,14 @@ define amdgpu_ps [2 x float] @load_2dmsaa_v2f32_fragId01(<8 x i32> inreg %rsrc,
 ; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
-; MSAA-NEXT:    [[I2:%.*]] = insertvalue [2 x float] undef, float [[I]], 0
+; MSAA-NEXT:    [[I2:%.*]] = insertvalue [2 x float] poison, float [[I]], 0
 ; MSAA-NEXT:    [[I3:%.*]] = insertvalue [2 x float] [[I2]], float [[I1]], 1
 ; MSAA-NEXT:    ret [2 x float] [[I3]]
 ;
 main_body:
   %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
   %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = insertvalue [2 x float] undef, float %i, 0
+  %i2 = insertvalue [2 x float] poison, float %i, 0
   %i3 = insertvalue [2 x float] %i2, float %i1, 1
   ret [2 x float] %i3
 }
@@ -381,7 +381,7 @@ define amdgpu_ps [2 x float] @load_2dmsaa_v2f32_fragId23(<8 x i32> inreg %rsrc,
 ; NO-MSAA-NEXT:  main_body:
 ; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = insertvalue [2 x float] undef, float [[I]], 0
+; NO-MSAA-NEXT:    [[I2:%.*]] = insertvalue [2 x float] poison, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I3:%.*]] = insertvalue [2 x float] [[I2]], float [[I1]], 1
 ; NO-MSAA-NEXT:    ret [2 x float] [[I3]]
 ;
@@ -391,14 +391,14 @@ define amdgpu_ps [2 x float] @load_2dmsaa_v2f32_fragId23(<8 x i32> inreg %rsrc,
 ; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
-; MSAA-NEXT:    [[I2:%.*]] = insertvalue [2 x float] undef, float [[I]], 0
+; MSAA-NEXT:    [[I2:%.*]] = insertvalue [2 x float] poison, float [[I]], 0
 ; MSAA-NEXT:    [[I3:%.*]] = insertvalue [2 x float] [[I2]], float [[I1]], 1
 ; MSAA-NEXT:    ret [2 x float] [[I3]]
 ;
 main_body:
   %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = insertvalue [2 x float] undef, float %i, 0
+  %i2 = insertvalue [2 x float] poison, float %i, 0
   %i3 = insertvalue [2 x float] %i2, float %i1, 1
   ret [2 x float] %i3
 }
@@ -411,7 +411,7 @@ define amdgpu_ps [2 x <2 x float>] @load_2dmsaa_v2v2f32_dmask3(<8 x i32> inreg %
 ; NO-MSAA-NEXT:  main_body:
 ; NO-MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [2 x <2 x float>] undef, <2 x float> [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [2 x <2 x float>] poison, <2 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [2 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    ret [2 x <2 x float>] [[I5]]
 ;
@@ -420,14 +420,14 @@ define amdgpu_ps [2 x <2 x float>] @load_2dmsaa_v2v2f32_dmask3(<8 x i32> inreg %
 ; MSAA-NEXT:  main_body:
 ; MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [2 x <2 x float>] undef, <2 x float> [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [2 x <2 x float>] poison, <2 x float> [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [2 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; MSAA-NEXT:    ret [2 x <2 x float>] [[I5]]
 ;
 main_body:
   %i = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
   %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [2 x <2 x float>] undef, <2 x float> %i, 0
+  %i4 = insertvalue [2 x <2 x float>] poison, <2 x float> %i, 0
   %i5 = insertvalue [2 x <2 x float>] %i4, <2 x float> %i1, 1
   ret [2 x <2 x float>] %i5
 }
@@ -443,7 +443,7 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask3(<8 x i32> inreg %
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] poison, <2 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x <2 x float>] [[I6]], <2 x float> [[I3]], 3
@@ -455,22 +455,22 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask3(<8 x i32> inreg %
 ; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
-; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> undef, float [[TMP2]], i64 0
+; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> poison, float [[TMP2]], i64 0
 ; MSAA-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
 ; MSAA-NEXT:    [[I:%.*]] = insertelement <2 x float> [[TMP3]], float [[TMP4]], i64 1
 ; MSAA-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
-; MSAA-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> undef, float [[TMP5]], i64 0
+; MSAA-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> poison, float [[TMP5]], i64 0
 ; MSAA-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP1]], i64 1
 ; MSAA-NEXT:    [[I1:%.*]] = insertelement <2 x float> [[TMP6]], float [[TMP7]], i64 1
 ; MSAA-NEXT:    [[TMP8:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
-; MSAA-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> undef, float [[TMP8]], i64 0
+; MSAA-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> poison, float [[TMP8]], i64 0
 ; MSAA-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[TMP1]], i64 2
 ; MSAA-NEXT:    [[I2:%.*]] = insertelement <2 x float> [[TMP9]], float [[TMP10]], i64 1
 ; MSAA-NEXT:    [[TMP11:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
-; MSAA-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> undef, float [[TMP11]], i64 0
+; MSAA-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> poison, float [[TMP11]], i64 0
 ; MSAA-NEXT:    [[TMP13:%.*]] = extractelement <4 x float> [[TMP1]], i64 3
 ; MSAA-NEXT:    [[I3:%.*]] = insertelement <2 x float> [[TMP12]], float [[TMP13]], i64 1
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] poison, <2 x float> [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x <2 x float>] [[I6]], <2 x float> [[I3]], 3
@@ -481,7 +481,7 @@ main_body:
   %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [4 x <2 x float>] undef, <2 x float> %i, 0
+  %i4 = insertvalue [4 x <2 x float>] poison, <2 x float> %i, 0
   %i5 = insertvalue [4 x <2 x float>] %i4, <2 x float> %i1, 1
   %i6 = insertvalue [4 x <2 x float>] %i5, <2 x float> %i2, 2
   %i7 = insertvalue [4 x <2 x float>] %i6, <2 x float> %i3, 3
@@ -499,7 +499,7 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask5(<8 x i32> inreg %
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 5, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 5, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 5, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] poison, <2 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x <2 x float>] [[I6]], <2 x float> [[I3]], 3
@@ -511,22 +511,22 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask5(<8 x i32> inreg %
 ; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 4, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
-; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> undef, float [[TMP2]], i64 0
+; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> poison, float [[TMP2]], i64 0
 ; MSAA-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
 ; MSAA-NEXT:    [[I:%.*]] = insertelement <2 x float> [[TMP3]], float [[TMP4]], i64 1
 ; MSAA-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
-; MSAA-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> undef, float [[TMP5]], i64 0
+; MSAA-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> poison, float [[TMP5]], i64 0
 ; MSAA-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP1]], i64 1
 ; MSAA-NEXT:    [[I1:%.*]] = insertelement <2 x float> [[TMP6]], float [[TMP7]], i64 1
 ; MSAA-NEXT:    [[TMP8:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
-; MSAA-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> undef, float [[TMP8]], i64 0
+; MSAA-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> poison, float [[TMP8]], i64 0
 ; MSAA-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[TMP1]], i64 2
 ; MSAA-NEXT:    [[I2:%.*]] = insertelement <2 x float> [[TMP9]], float [[TMP10]], i64 1
 ; MSAA-NEXT:    [[TMP11:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
-; MSAA-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> undef, float [[TMP11]], i64 0
+; MSAA-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> poison, float [[TMP11]], i64 0
 ; MSAA-NEXT:    [[TMP13:%.*]] = extractelement <4 x float> [[TMP1]], i64 3
 ; MSAA-NEXT:    [[I3:%.*]] = insertelement <2 x float> [[TMP12]], float [[TMP13]], i64 1
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] poison, <2 x float> [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x <2 x float>] [[I6]], <2 x float> [[I3]], 3
@@ -537,7 +537,7 @@ main_body:
   %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 5, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 5, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 5, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [4 x <2 x float>] undef, <2 x float> %i, 0
+  %i4 = insertvalue [4 x <2 x float>] poison, <2 x float> %i, 0
   %i5 = insertvalue [4 x <2 x float>] %i4, <2 x float> %i1, 1
   %i6 = insertvalue [4 x <2 x float>] %i5, <2 x float> %i2, 2
   %i7 = insertvalue [4 x <2 x float>] %i6, <2 x float> %i3, 3
@@ -555,7 +555,7 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask6(<8 x i32> inreg %
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 6, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 6, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 6, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] poison, <2 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x <2 x float>] [[I6]], <2 x float> [[I3]], 3
@@ -567,22 +567,22 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask6(<8 x i32> inreg %
 ; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 4, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
-; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> undef, float [[TMP2]], i64 0
+; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> poison, float [[TMP2]], i64 0
 ; MSAA-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
 ; MSAA-NEXT:    [[I:%.*]] = insertelement <2 x float> [[TMP3]], float [[TMP4]], i64 1
 ; MSAA-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
-; MSAA-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> undef, float [[TMP5]], i64 0
+; MSAA-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> poison, float [[TMP5]], i64 0
 ; MSAA-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP1]], i64 1
 ; MSAA-NEXT:    [[I1:%.*]] = insertelement <2 x float> [[TMP6]], float [[TMP7]], i64 1
 ; MSAA-NEXT:    [[TMP8:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
-; MSAA-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> undef, float [[TMP8]], i64 0
+; MSAA-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> poison, float [[TMP8]], i64 0
 ; MSAA-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[TMP1]], i64 2
 ; MSAA-NEXT:    [[I2:%.*]] = insertelement <2 x float> [[TMP9]], float [[TMP10]], i64 1
 ; MSAA-NEXT:    [[TMP11:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
-; MSAA-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> undef, float [[TMP11]], i64 0
+; MSAA-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> poison, float [[TMP11]], i64 0
 ; MSAA-NEXT:    [[TMP13:%.*]] = extractelement <4 x float> [[TMP1]], i64 3
 ; MSAA-NEXT:    [[I3:%.*]] = insertelement <2 x float> [[TMP12]], float [[TMP13]], i64 1
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] poison, <2 x float> [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x <2 x float>] [[I6]], <2 x float> [[I3]], 3
@@ -593,7 +593,7 @@ main_body:
   %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 6, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 6, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 6, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [4 x <2 x float>] undef, <2 x float> %i, 0
+  %i4 = insertvalue [4 x <2 x float>] poison, <2 x float> %i, 0
   %i5 = insertvalue [4 x <2 x float>] %i4, <2 x float> %i1, 1
   %i6 = insertvalue [4 x <2 x float>] %i5, <2 x float> %i2, 2
   %i7 = insertvalue [4 x <2 x float>] %i6, <2 x float> %i3, 3
@@ -611,7 +611,7 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask9(<8 x i32> inreg %
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 9, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 9, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 9, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] poison, <2 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x <2 x float>] [[I6]], <2 x float> [[I3]], 3
@@ -623,22 +623,22 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask9(<8 x i32> inreg %
 ; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 8, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
-; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> undef, float [[TMP2]], i64 0
+; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> poison, float [[TMP2]], i64 0
 ; MSAA-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
 ; MSAA-NEXT:    [[I:%.*]] = insertelement <2 x float> [[TMP3]], float [[TMP4]], i64 1
 ; MSAA-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
-; MSAA-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> undef, float [[TMP5]], i64 0
+; MSAA-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> poison, float [[TMP5]], i64 0
 ; MSAA-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP1]], i64 1
 ; MSAA-NEXT:    [[I1:%.*]] = insertelement <2 x float> [[TMP6]], float [[TMP7]], i64 1
 ; MSAA-NEXT:    [[TMP8:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
-; MSAA-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> undef, float [[TMP8]], i64 0
+; MSAA-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> poison, float [[TMP8]], i64 0
 ; MSAA-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[TMP1]], i64 2
 ; MSAA-NEXT:    [[I2:%.*]] = insertelement <2 x float> [[TMP9]], float [[TMP10]], i64 1
 ; MSAA-NEXT:    [[TMP11:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
-; MSAA-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> undef, float [[TMP11]], i64 0
+; MSAA-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> poison, float [[TMP11]], i64 0
 ; MSAA-NEXT:    [[TMP13:%.*]] = extractelement <4 x float> [[TMP1]], i64 3
 ; MSAA-NEXT:    [[I3:%.*]] = insertelement <2 x float> [[TMP12]], float [[TMP13]], i64 1
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] poison, <2 x float> [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x <2 x float>] [[I6]], <2 x float> [[I3]], 3
@@ -649,7 +649,7 @@ main_body:
   %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 9, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 9, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 9, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [4 x <2 x float>] undef, <2 x float> %i, 0
+  %i4 = insertvalue [4 x <2 x float>] poison, <2 x float> %i, 0
   %i5 = insertvalue [4 x <2 x float>] %i4, <2 x float> %i1, 1
   %i6 = insertvalue [4 x <2 x float>] %i5, <2 x float> %i2, 2
   %i7 = insertvalue [4 x <2 x float>] %i6, <2 x float> %i3, 3
@@ -667,7 +667,7 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask10(<8 x i32> inreg
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 10, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 10, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 10, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] poison, <2 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x <2 x float>] [[I6]], <2 x float> [[I3]], 3
@@ -679,22 +679,22 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask10(<8 x i32> inreg
 ; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 8, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
-; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> undef, float [[TMP2]], i64 0
+; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> poison, float [[TMP2]], i64 0
 ; MSAA-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
 ; MSAA-NEXT:    [[I:%.*]] = insertelement <2 x float> [[TMP3]], float [[TMP4]], i64 1
 ; MSAA-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
-; MSAA-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> undef, float [[TMP5]], i64 0
+; MSAA-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> poison, float [[TMP5]], i64 0
 ; MSAA-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP1]], i64 1
 ; MSAA-NEXT:    [[I1:%.*]] = insertelement <2 x float> [[TMP6]], float [[TMP7]], i64 1
 ; MSAA-NEXT:    [[TMP8:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
-; MSAA-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> undef, float [[TMP8]], i64 0
+; MSAA-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> poison, float [[TMP8]], i64 0
 ; MSAA-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[TMP1]], i64 2
 ; MSAA-NEXT:    [[I2:%.*]] = insertelement <2 x float> [[TMP9]], float [[TMP10]], i64 1
 ; MSAA-NEXT:    [[TMP11:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
-; MSAA-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> undef, float [[TMP11]], i64 0
+; MSAA-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> poison, float [[TMP11]], i64 0
 ; MSAA-NEXT:    [[TMP13:%.*]] = extractelement <4 x float> [[TMP1]], i64 3
 ; MSAA-NEXT:    [[I3:%.*]] = insertelement <2 x float> [[TMP12]], float [[TMP13]], i64 1
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] poison, <2 x float> [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x <2 x float>] [[I6]], <2 x float> [[I3]], 3
@@ -705,7 +705,7 @@ main_body:
   %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 10, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 10, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 10, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [4 x <2 x float>] undef, <2 x float> %i, 0
+  %i4 = insertvalue [4 x <2 x float>] poison, <2 x float> %i, 0
   %i5 = insertvalue [4 x <2 x float>] %i4, <2 x float> %i1, 1
   %i6 = insertvalue [4 x <2 x float>] %i5, <2 x float> %i2, 2
   %i7 = insertvalue [4 x <2 x float>] %i6, <2 x float> %i3, 3
@@ -723,7 +723,7 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask12(<8 x i32> inreg
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 12, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 12, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 12, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] poison, <2 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x <2 x float>] [[I6]], <2 x float> [[I3]], 3
@@ -735,22 +735,22 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask12(<8 x i32> inreg
 ; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 4, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 8, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
-; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> undef, float [[TMP2]], i64 0
+; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> poison, float [[TMP2]], i64 0
 ; MSAA-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
 ; MSAA-NEXT:    [[I:%.*]] = insertelement <2 x float> [[TMP3]], float [[TMP4]], i64 1
 ; MSAA-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
-; MSAA-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> undef, float [[TMP5]], i64 0
+; MSAA-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> poison, float [[TMP5]], i64 0
 ; MSAA-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP1]], i64 1
 ; MSAA-NEXT:    [[I1:%.*]] = insertelement <2 x float> [[TMP6]], float [[TMP7]], i64 1
 ; MSAA-NEXT:    [[TMP8:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
-; MSAA-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> undef, float [[TMP8]], i64 0
+; MSAA-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> poison, float [[TMP8]], i64 0
 ; MSAA-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[TMP1]], i64 2
 ; MSAA-NEXT:    [[I2:%.*]] = insertelement <2 x float> [[TMP9]], float [[TMP10]], i64 1
 ; MSAA-NEXT:    [[TMP11:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
-; MSAA-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> undef, float [[TMP11]], i64 0
+; MSAA-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> poison, float [[TMP11]], i64 0
 ; MSAA-NEXT:    [[TMP13:%.*]] = extractelement <4 x float> [[TMP1]], i64 3
 ; MSAA-NEXT:    [[I3:%.*]] = insertelement <2 x float> [[TMP12]], float [[TMP13]], i64 1
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] poison, <2 x float> [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x <2 x float>] [[I6]], <2 x float> [[I3]], 3
@@ -761,7 +761,7 @@ main_body:
   %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 12, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 12, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 12, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [4 x <2 x float>] undef, <2 x float> %i, 0
+  %i4 = insertvalue [4 x <2 x float>] poison, <2 x float> %i, 0
   %i5 = insertvalue [4 x <2 x float>] %i4, <2 x float> %i1, 1
   %i6 = insertvalue [4 x <2 x float>] %i5, <2 x float> %i2, 2
   %i7 = insertvalue [4 x <2 x float>] %i6, <2 x float> %i3, 3
@@ -774,7 +774,7 @@ define amdgpu_ps [2 x half] @load_2dmsaa_v2f16_fragId01(<8 x i32> inreg %rsrc, i
 ; NO-MSAA-NEXT:  main_body:
 ; NO-MSAA-NEXT:    [[I:%.*]] = call half @llvm.amdgcn.image.load.2dmsaa.f16.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call half @llvm.amdgcn.image.load.2dmsaa.f16.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = insertvalue [2 x half] undef, half [[I]], 0
+; NO-MSAA-NEXT:    [[I2:%.*]] = insertvalue [2 x half] poison, half [[I]], 0
 ; NO-MSAA-NEXT:    [[I3:%.*]] = insertvalue [2 x half] [[I2]], half [[I1]], 1
 ; NO-MSAA-NEXT:    ret [2 x half] [[I3]]
 ;
@@ -784,14 +784,14 @@ define amdgpu_ps [2 x half] @load_2dmsaa_v2f16_fragId01(<8 x i32> inreg %rsrc, i
 ; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x half> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f16.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I:%.*]] = extractelement <4 x half> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x half> [[TMP0]], i64 1
-; MSAA-NEXT:    [[I2:%.*]] = insertvalue [2 x half] undef, half [[I]], 0
+; MSAA-NEXT:    [[I2:%.*]] = insertvalue [2 x half] poison, half [[I]], 0
 ; MSAA-NEXT:    [[I3:%.*]] = insertvalue [2 x half] [[I2]], half [[I1]], 1
 ; MSAA-NEXT:    ret [2 x half] [[I3]]
 ;
 main_body:
   %i = call half @llvm.amdgcn.image.load.2dmsaa.f16.i32.v8i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
   %i1 = call half @llvm.amdgcn.image.load.2dmsaa.f16.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = insertvalue [2 x half] undef, half %i, 0
+  %i2 = insertvalue [2 x half] poison, half %i, 0
   %i3 = insertvalue [2 x half] %i2, half %i1, 1
   ret [2 x half] %i3
 }
@@ -804,7 +804,7 @@ define amdgpu_ps [4 x float] @load_2darraymsaa_v4f32_dmask1(<8 x i32> inreg %rsr
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -818,7 +818,7 @@ define amdgpu_ps [4 x float] @load_2darraymsaa_v4f32_dmask1(<8 x i32> inreg %rsr
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
 ; MSAA-NEXT:    [[I3:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -829,7 +829,7 @@ main_body:
   %i1 = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 %slice, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 %slice, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 %slice, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [4 x float] undef, float %i, 0
+  %i4 = insertvalue [4 x float] poison, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
   %i7 = insertvalue [4 x float] %i6, float %i3, 3
@@ -847,7 +847,7 @@ define amdgpu_ps [4 x <2 x float>] @load_2darraymsaa_v4v2f32_dmask3(<8 x i32> in
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] poison, <2 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x <2 x float>] [[I6]], <2 x float> [[I3]], 3
@@ -859,22 +859,22 @@ define amdgpu_ps [4 x <2 x float>] @load_2darraymsaa_v4v2f32_dmask3(<8 x i32> in
 ; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2darraymsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2darraymsaa.v4f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
-; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> undef, float [[TMP2]], i64 0
+; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> poison, float [[TMP2]], i64 0
 ; MSAA-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
 ; MSAA-NEXT:    [[I:%.*]] = insertelement <2 x float> [[TMP3]], float [[TMP4]], i64 1
 ; MSAA-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
-; MSAA-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> undef, float [[TMP5]], i64 0
+; MSAA-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> poison, float [[TMP5]], i64 0
 ; MSAA-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP1]], i64 1
 ; MSAA-NEXT:    [[I1:%.*]] = insertelement <2 x float> [[TMP6]], float [[TMP7]], i64 1
 ; MSAA-NEXT:    [[TMP8:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
-; MSAA-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> undef, float [[TMP8]], i64 0
+; MSAA-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> poison, float [[TMP8]], i64 0
 ; MSAA-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[TMP1]], i64 2
 ; MSAA-NEXT:    [[I2:%.*]] = insertelement <2 x float> [[TMP9]], float [[TMP10]], i64 1
 ; MSAA-NEXT:    [[TMP11:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
-; MSAA-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> undef, float [[TMP11]], i64 0
+; MSAA-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> poison, float [[TMP11]], i64 0
 ; MSAA-NEXT:    [[TMP13:%.*]] = extractelement <4 x float> [[TMP1]], i64 3
 ; MSAA-NEXT:    [[I3:%.*]] = insertelement <2 x float> [[TMP12]], float [[TMP13]], i64 1
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] poison, <2 x float> [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x <2 x float>] [[I6]], <2 x float> [[I3]], 3
@@ -885,7 +885,7 @@ main_body:
   %i1 = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 %slice, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 %slice, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 %slice, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [4 x <2 x float>] undef, <2 x float> %i, 0
+  %i4 = insertvalue [4 x <2 x float>] poison, <2 x float> %i, 0
   %i5 = insertvalue [4 x <2 x float>] %i4, <2 x float> %i1, 1
   %i6 = insertvalue [4 x <2 x float>] %i5, <2 x float> %i2, 2
   %i7 = insertvalue [4 x <2 x float>] %i6, <2 x float> %i3, 3
@@ -905,7 +905,7 @@ define amdgpu_ps [4 x <3 x float>] @load_2dmsaa_v4v3f32_dmask7(<8 x i32> inreg %
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <3 x float>] undef, <3 x float> [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <3 x float>] poison, <3 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <3 x float>] [[I4]], <3 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <3 x float>] [[I5]], <3 x float> [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x <3 x float>] [[I6]], <3 x float> [[I3]], 3
@@ -918,30 +918,30 @@ define amdgpu_ps [4 x <3 x float>] @load_2dmsaa_v4v3f32_dmask7(<8 x i32> inreg %
 ; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 4, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
-; MSAA-NEXT:    [[TMP4:%.*]] = insertelement <3 x float> undef, float [[TMP3]], i64 0
+; MSAA-NEXT:    [[TMP4:%.*]] = insertelement <3 x float> poison, float [[TMP3]], i64 0
 ; MSAA-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
 ; MSAA-NEXT:    [[TMP6:%.*]] = insertelement <3 x float> [[TMP4]], float [[TMP5]], i64 1
 ; MSAA-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP2]], i64 0
 ; MSAA-NEXT:    [[I:%.*]] = insertelement <3 x float> [[TMP6]], float [[TMP7]], i64 2
 ; MSAA-NEXT:    [[TMP8:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
-; MSAA-NEXT:    [[TMP9:%.*]] = insertelement <3 x float> undef, float [[TMP8]], i64 0
+; MSAA-NEXT:    [[TMP9:%.*]] = insertelement <3 x float> poison, float [[TMP8]], i64 0
 ; MSAA-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[TMP1]], i64 1
 ; MSAA-NEXT:    [[TMP11:%.*]] = insertelement <3 x float> [[TMP9]], float [[TMP10]], i64 1
 ; MSAA-NEXT:    [[TMP12:%.*]] = extractelement <4 x float> [[TMP2]], i64 1
 ; MSAA-NEXT:    [[I1:%.*]] = insertelement <3 x float> [[TMP11]], float [[TMP12]], i64 2
 ; MSAA-NEXT:    [[TMP13:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
-; MSAA-NEXT:    [[TMP14:%.*]] = insertelement <3 x float> undef, float [[TMP13]], i64 0
+; MSAA-NEXT:    [[TMP14:%.*]] = insertelement <3 x float> poison, float [[TMP13]], i64 0
 ; MSAA-NEXT:    [[TMP15:%.*]] = extractelement <4 x float> [[TMP1]], i64 2
 ; MSAA-NEXT:    [[TMP16:%.*]] = insertelement <3 x float> [[TMP14]], float [[TMP15]], i64 1
 ; MSAA-NEXT:    [[TMP17:%.*]] = extractelement <4 x float> [[TMP2]], i64 2
 ; MSAA-NEXT:    [[I2:%.*]] = insertelement <3 x float> [[TMP16]], float [[TMP17]], i64 2
 ; MSAA-NEXT:    [[TMP18:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
-; MSAA-NEXT:    [[TMP19:%.*]] = insertelement <3 x float> undef, float [[TMP18]], i64 0
+; MSAA-NEXT:    [[TMP19:%.*]] = insertelement <3 x float> poison, float [[TMP18]], i64 0
 ; MSAA-NEXT:    [[TMP20:%.*]] = extractelement <4 x float> [[TMP1]], i64 3
 ; MSAA-NEXT:    [[TMP21:%.*]] = insertelement <3 x float> [[TMP19]], float [[TMP20]], i64 1
 ; MSAA-NEXT:    [[TMP22:%.*]] = extractelement <4 x float> [[TMP2]], i64 3
 ; MSAA-NEXT:    [[I3:%.*]] = insertelement <3 x float> [[TMP21]], float [[TMP22]], i64 2
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <3 x float>] undef, <3 x float> [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <3 x float>] poison, <3 x float> [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <3 x float>] [[I4]], <3 x float> [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <3 x float>] [[I5]], <3 x float> [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x <3 x float>] [[I6]], <3 x float> [[I3]], 3
@@ -952,7 +952,7 @@ main_body:
   %i1 = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [4 x <3 x float>] undef, <3 x float> %i, 0
+  %i4 = insertvalue [4 x <3 x float>] poison, <3 x float> %i, 0
   %i5 = insertvalue [4 x <3 x float>] %i4, <3 x float> %i1, 1
   %i6 = insertvalue [4 x <3 x float>] %i5, <3 x float> %i2, 2
   %i7 = insertvalue [4 x <3 x float>] %i6, <3 x float> %i3, 3
@@ -972,7 +972,7 @@ define amdgpu_ps [4 x <3 x float>] @load_2dmsaa_v4v3f32_dmask7_group1(<8 x i32>
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 [[S]], i32 [[T]], i32 5, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 [[S]], i32 [[T]], i32 6, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 [[S]], i32 [[T]], i32 7, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <3 x float>] undef, <3 x float> [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <3 x float>] poison, <3 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <3 x float>] [[I4]], <3 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <3 x float>] [[I5]], <3 x float> [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x <3 x float>] [[I6]], <3 x float> [[I3]], 3
@@ -985,30 +985,30 @@ define amdgpu_ps [4 x <3 x float>] @load_2dmsaa_v4v3f32_dmask7_group1(<8 x i32>
 ; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 4, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 4, i32 [[S]], i32 [[T]], i32 4, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
-; MSAA-NEXT:    [[TMP4:%.*]] = insertelement <3 x float> undef, float [[TMP3]], i64 0
+; MSAA-NEXT:    [[TMP4:%.*]] = insertelement <3 x float> poison, float [[TMP3]], i64 0
 ; MSAA-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
 ; MSAA-NEXT:    [[TMP6:%.*]] = insertelement <3 x float> [[TMP4]], float [[TMP5]], i64 1
 ; MSAA-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP2]], i64 0
 ; MSAA-NEXT:    [[I:%.*]] = insertelement <3 x float> [[TMP6]], float [[TMP7]], i64 2
 ; MSAA-NEXT:    [[TMP8:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
-; MSAA-NEXT:    [[TMP9:%.*]] = insertelement <3 x float> undef, float [[TMP8]], i64 0
+; MSAA-NEXT:    [[TMP9:%.*]] = insertelement <3 x float> poison, float [[TMP8]], i64 0
 ; MSAA-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[TMP1]], i64 1
 ; MSAA-NEXT:    [[TMP11:%.*]] = insertelement <3 x float> [[TMP9]], float [[TMP10]], i64 1
 ; MSAA-NEXT:    [[TMP12:%.*]] = extractelement <4 x float> [[TMP2]], i64 1
 ; MSAA-NEXT:    [[I1:%.*]] = insertelement <3 x float> [[TMP11]], float [[TMP12]], i64 2
 ; MSAA-NEXT:    [[TMP13:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
-; MSAA-NEXT:    [[TMP14:%.*]] = insertelement <3 x float> undef, float [[TMP13]], i64 0
+; MSAA-NEXT:    [[TMP14:%.*]] = insertelement <3 x float> poison, float [[TMP13]], i64 0
 ; MSAA-NEXT:    [[TMP15:%.*]] = extractelement <4 x float> [[TMP1]], i64 2
 ; MSAA-NEXT:    [[TMP16:%.*]] = insertelement <3 x float> [[TMP14]], float [[TMP15]], i64 1
 ; MSAA-NEXT:    [[TMP17:%.*]] = extractelement <4 x float> [[TMP2]], i64 2
 ; MSAA-NEXT:    [[I2:%.*]] = insertelement <3 x float> [[TMP16]], float [[TMP17]], i64 2
 ; MSAA-NEXT:    [[TMP18:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
-; MSAA-NEXT:    [[TMP19:%.*]] = insertelement <3 x float> undef, float [[TMP18]], i64 0
+; MSAA-NEXT:    [[TMP19:%.*]] = insertelement <3 x float> poison, float [[TMP18]], i64 0
 ; MSAA-NEXT:    [[TMP20:%.*]] = extractelement <4 x float> [[TMP1]], i64 3
 ; MSAA-NEXT:    [[TMP21:%.*]] = insertelement <3 x float> [[TMP19]], float [[TMP20]], i64 1
 ; MSAA-NEXT:    [[TMP22:%.*]] = extractelement <4 x float> [[TMP2]], i64 3
 ; MSAA-NEXT:    [[I3:%.*]] = insertelement <3 x float> [[TMP21]], float [[TMP22]], i64 2
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <3 x float>] undef, <3 x float> [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <3 x float>] poison, <3 x float> [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <3 x float>] [[I4]], <3 x float> [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <3 x float>] [[I5]], <3 x float> [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x <3 x float>] [[I6]], <3 x float> [[I3]], 3
@@ -1019,7 +1019,7 @@ main_body:
   %i1 = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 %s, i32 %t, i32 5, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 %s, i32 %t, i32 6, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 %s, i32 %t, i32 7, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [4 x <3 x float>] undef, <3 x float> %i, 0
+  %i4 = insertvalue [4 x <3 x float>] poison, <3 x float> %i, 0
   %i5 = insertvalue [4 x <3 x float>] %i4, <3 x float> %i1, 1
   %i6 = insertvalue [4 x <3 x float>] %i5, <3 x float> %i2, 2
   %i7 = insertvalue [4 x <3 x float>] %i6, <3 x float> %i3, 3
@@ -1035,7 +1035,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_sections(<8 x i32> inreg %rsrc,
 ; NO-MSAA-NEXT:    call void @llvm.amdgcn.image.store.2dmsaa.f32.i32.v8i32(float [[VDATA]], i32 1, i32 [[S]], i32 [[T]], i32 [[FRAGID]], <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -1051,7 +1051,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_sections(<8 x i32> inreg %rsrc,
 ; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP1]], i64 2
 ; MSAA-NEXT:    [[I3:%.*]] = extractelement <4 x float> [[TMP1]], i64 3
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -1063,7 +1063,7 @@ main_body:
   call void @llvm.amdgcn.image.store.2dmsaa.f32.i32(float %vdata, i32 1, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [4 x float] undef, float %i, 0
+  %i4 = insertvalue [4 x float] poison, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
   %i7 = insertvalue [4 x float] %i6, float %i3, 3
@@ -1078,7 +1078,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_blocks(<8 x i32> inreg %rsrc, i3
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -1089,7 +1089,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_blocks(<8 x i32> inreg %rsrc, i3
 ; NO-MSAA-NEXT:    [[I10:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I11:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I12:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I13:%.*]] = insertvalue [4 x float] undef, float [[I9]], 0
+; NO-MSAA-NEXT:    [[I13:%.*]] = insertvalue [4 x float] poison, float [[I9]], 0
 ; NO-MSAA-NEXT:    [[I14:%.*]] = insertvalue [4 x float] [[I13]], float [[I10]], 1
 ; NO-MSAA-NEXT:    [[I15:%.*]] = insertvalue [4 x float] [[I14]], float [[I11]], 2
 ; NO-MSAA-NEXT:    [[I16:%.*]] = insertvalue [4 x float] [[I15]], float [[I12]], 3
@@ -1099,7 +1099,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_blocks(<8 x i32> inreg %rsrc, i3
 ; NO-MSAA-NEXT:    [[I18:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I19:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I20:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I21:%.*]] = insertvalue [4 x float] undef, float [[I17]], 0
+; NO-MSAA-NEXT:    [[I21:%.*]] = insertvalue [4 x float] poison, float [[I17]], 0
 ; NO-MSAA-NEXT:    [[I22:%.*]] = insertvalue [4 x float] [[I21]], float [[I18]], 1
 ; NO-MSAA-NEXT:    [[I23:%.*]] = insertvalue [4 x float] [[I22]], float [[I19]], 2
 ; NO-MSAA-NEXT:    [[I24:%.*]] = insertvalue [4 x float] [[I23]], float [[I20]], 3
@@ -1116,7 +1116,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_blocks(<8 x i32> inreg %rsrc, i3
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
 ; MSAA-NEXT:    [[I3:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -1128,7 +1128,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_blocks(<8 x i32> inreg %rsrc, i3
 ; MSAA-NEXT:    [[I10:%.*]] = extractelement <4 x float> [[TMP1]], i64 1
 ; MSAA-NEXT:    [[I11:%.*]] = extractelement <4 x float> [[TMP1]], i64 2
 ; MSAA-NEXT:    [[I12:%.*]] = extractelement <4 x float> [[TMP1]], i64 3
-; MSAA-NEXT:    [[I13:%.*]] = insertvalue [4 x float] undef, float [[I9]], 0
+; MSAA-NEXT:    [[I13:%.*]] = insertvalue [4 x float] poison, float [[I9]], 0
 ; MSAA-NEXT:    [[I14:%.*]] = insertvalue [4 x float] [[I13]], float [[I10]], 1
 ; MSAA-NEXT:    [[I15:%.*]] = insertvalue [4 x float] [[I14]], float [[I11]], 2
 ; MSAA-NEXT:    [[I16:%.*]] = insertvalue [4 x float] [[I15]], float [[I12]], 3
@@ -1139,7 +1139,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_blocks(<8 x i32> inreg %rsrc, i3
 ; MSAA-NEXT:    [[I18:%.*]] = extractelement <4 x float> [[TMP2]], i64 1
 ; MSAA-NEXT:    [[I19:%.*]] = extractelement <4 x float> [[TMP2]], i64 2
 ; MSAA-NEXT:    [[I20:%.*]] = extractelement <4 x float> [[TMP2]], i64 3
-; MSAA-NEXT:    [[I21:%.*]] = insertvalue [4 x float] undef, float [[I17]], 0
+; MSAA-NEXT:    [[I21:%.*]] = insertvalue [4 x float] poison, float [[I17]], 0
 ; MSAA-NEXT:    [[I22:%.*]] = insertvalue [4 x float] [[I21]], float [[I18]], 1
 ; MSAA-NEXT:    [[I23:%.*]] = insertvalue [4 x float] [[I22]], float [[I19]], 2
 ; MSAA-NEXT:    [[I24:%.*]] = insertvalue [4 x float] [[I23]], float [[I20]], 3
@@ -1153,7 +1153,7 @@ main_body:
   %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = insertvalue [4 x float] undef, float %i, 0
+  %i4 = insertvalue [4 x float] poison, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
   %i7 = insertvalue [4 x float] %i6, float %i3, 3
@@ -1164,7 +1164,7 @@ if_equal:
   %i10 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i11 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i12 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i13 = insertvalue [4 x float] undef, float %i9, 0
+  %i13 = insertvalue [4 x float] poison, float %i9, 0
   %i14 = insertvalue [4 x float] %i13, float %i10, 1
   %i15 = insertvalue [4 x float] %i14, float %i11, 2
   %i16 = insertvalue [4 x float] %i15, float %i12, 3
@@ -1174,7 +1174,7 @@ if_unequal:
   %i18 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i19 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i20 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i21 = insertvalue [4 x float] undef, float %i17, 0
+  %i21 = insertvalue [4 x float] poison, float %i17, 0
   %i22 = insertvalue [4 x float] %i21, float %i18, 1
   %i23 = insertvalue [4 x float] %i22, float %i19, 2
   %i24 = insertvalue [4 x float] %i23, float %i20, 3
@@ -1192,7 +1192,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask1_different_rsrc(<8 x i32>
 ; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC1]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC2]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC2]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -1207,7 +1207,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask1_different_rsrc(<8 x i32>
 ; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC2]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
 ; MSAA-NEXT:    [[I3:%.*]] = extractelement <4 x float> [[TMP1]], i64 1
-; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] poison, float [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
 ; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
@@ -1218,7 +1218,7 @@ main_body:
   %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc1, i32 0, i32 0)
   %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc2, i32 0, i32 0)
   %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc2, i32 0, i32 0)
-  %i4 = insertvalue [4 x float] undef, float %i, 0
+  %i4 = insertvalue [4 x float] poison, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
   %i7 = insertvalue [4 x float] %i6, float %i3, 3
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.nsa.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.nsa.ll
index 22b4c5a7362ad..2d1a055066fdf 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.nsa.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.nsa.ll
@@ -54,7 +54,7 @@ define amdgpu_ps <2 x float> @sample_contig_nsa(<8 x i32> inreg %rsrc, <4 x i32>
 main_body:
   %v1 = call float @llvm.amdgcn.image.sample.c.l.3d.f32.f32(i32 1, float %zcompare, float %s1, float %t1, float %r1, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   %v2 = call float @llvm.amdgcn.image.sample.3d.f32.f32(i32 1, float %s2, float %t2, float %r2, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
-  %r.0 = insertelement <2 x float> undef, float %v1, i32 0
+  %r.0 = insertelement <2 x float> poison, float %v1, i32 0
   %r = insertelement <2 x float> %r.0, float %v2, i32 1
   ret <2 x float> %r
 }
@@ -70,7 +70,7 @@ define amdgpu_ps <2 x float> @sample_nsa_nsa(<8 x i32> inreg %rsrc, <4 x i32> in
 main_body:
   %v1 = call float @llvm.amdgcn.image.sample.c.l.3d.f32.f32(i32 1, float %zcompare, float %s1, float %t1, float %r1, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   %v2 = call float @llvm.amdgcn.image.sample.3d.f32.f32(i32 1, float %s2, float %t2, float %r2, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
-  %r.0 = insertelement <2 x float> undef, float %v1, i32 0
+  %r.0 = insertelement <2 x float> poison, float %v1, i32 0
   %r = insertelement <2 x float> %r.0, float %v2, i32 1
   ret <2 x float> %r
 }
@@ -86,7 +86,7 @@ define amdgpu_ps <2 x float> @sample_nsa_contig(<8 x i32> inreg %rsrc, <4 x i32>
 main_body:
   %v1 = call float @llvm.amdgcn.image.sample.c.l.3d.f32.f32(i32 1, float %zcompare, float %s1, float %t1, float %r1, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   %v2 = call float @llvm.amdgcn.image.sample.3d.f32.f32(i32 1, float %s2, float %t2, float %r2, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
-  %r.0 = insertelement <2 x float> undef, float %v1, i32 0
+  %r.0 = insertelement <2 x float> poison, float %v1, i32 0
   %r = insertelement <2 x float> %r.0, float %v2, i32 1
   ret <2 x float> %r
 }
@@ -106,7 +106,7 @@ define amdgpu_ps <2 x float> @sample_contig_contig(<8 x i32> inreg %rsrc, <4 x i
 main_body:
   %v1 = call float @llvm.amdgcn.image.sample.c.l.3d.f32.f32(i32 1, float %zcompare, float %s1, float %t1, float %r1, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   %v2 = call float @llvm.amdgcn.image.sample.3d.f32.f32(i32 1, float %s2, float %t2, float %r2, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
-  %r.0 = insertelement <2 x float> undef, float %v1, i32 0
+  %r.0 = insertelement <2 x float> poison, float %v1, i32 0
   %r = insertelement <2 x float> %r.0, float %v2, i32 1
   ret <2 x float> %r
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll
index be66d6516f438..8861ff4c78137 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll
@@ -259,7 +259,7 @@ main_body:
   %tex.vec = extractvalue {<2 x half>, i32} %tex, 0
   %tex.err = extractvalue {<2 x half>, i32} %tex, 1
   %tex.vecf = bitcast <2 x half> %tex.vec to float
-  %r.0 = insertelement <2 x float> undef, float %tex.vecf, i32 0
+  %r.0 = insertelement <2 x float> poison, float %tex.vecf, i32 0
   %tex.errf = bitcast i32 %tex.err to float
   %r = insertelement <2 x float> %r.0, float %tex.errf, i32 1
   ret <2 x float> %r
@@ -315,7 +315,7 @@ define amdgpu_ps <2 x float> @image_sample_b_2d_v3f16(<8 x i32> inreg %rsrc, <4
 ; GFX12-NEXT:    ; return to shader part epilog
 main_body:
   %tex = call <3 x half> @llvm.amdgcn.image.sample.b.2d.v3f16.f32.f32(i32 7, float %bias, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
-  %tex_wide = shufflevector <3 x half> %tex, <3 x half> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %tex_wide = shufflevector <3 x half> %tex, <3 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %r = bitcast <4 x half> %tex_wide to <2 x float>
   ret <2 x float> %r
 }
@@ -410,12 +410,12 @@ define amdgpu_ps <4 x float> @image_sample_b_2d_v3f16_tfe(<8 x i32> inreg %rsrc,
 main_body:
   %tex = call {<3 x half>,i32} @llvm.amdgcn.image.sample.b.2d.v3f16i32.f32.f32(i32 7, float %bias, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 1, i32 0)
   %tex.vec = extractvalue {<3 x half>, i32} %tex, 0
-  %tex.vec_wide = shufflevector <3 x half> %tex.vec, <3 x half> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %tex.vec_wide = shufflevector <3 x half> %tex.vec, <3 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %tex.err = extractvalue {<3 x half>, i32} %tex, 1
   %tex.vecf = bitcast <4 x half> %tex.vec_wide to <2 x float>
   %tex.vecf.0 = extractelement <2 x float> %tex.vecf, i32 0
   %tex.vecf.1 = extractelement <2 x float> %tex.vecf, i32 1
-  %r.0 = insertelement <4 x float> undef, float %tex.vecf.0, i32 0
+  %r.0 = insertelement <4 x float> poison, float %tex.vecf.0, i32 0
   %r.1 = insertelement <4 x float> %r.0, float %tex.vecf.1, i32 1
   %tex.errf = bitcast i32 %tex.err to float
   %r = insertelement <4 x float> %r.1, float %tex.errf, i32 2
@@ -571,7 +571,7 @@ main_body:
   %tex.vecf = bitcast <4 x half> %tex.vec to <2 x float>
   %tex.vecf.0 = extractelement <2 x float> %tex.vecf, i32 0
   %tex.vecf.1 = extractelement <2 x float> %tex.vecf, i32 1
-  %r.0 = insertelement <4 x float> undef, float %tex.vecf.0, i32 0
+  %r.0 = insertelement <4 x float> poison, float %tex.vecf.0, i32 0
   %r.1 = insertelement <4 x float> %r.0, float %tex.vecf.1, i32 1
   %tex.errf = bitcast i32 %tex.err to float
   %r = insertelement <4 x float> %r.1, float %tex.errf, i32 2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.ll
index c8421c66f97c3..a713b1db119b8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.ll
@@ -207,7 +207,7 @@ main_body:
   %res.f = extractelement <4 x float> %res.vec, i32 0
   %res.err = extractvalue {<4 x float>,i32} %v, 1
   %res.errf = bitcast i32 %res.err to float
-  %res.tmp = insertelement <2 x float> undef, float %res.f, i32 0
+  %res.tmp = insertelement <2 x float> poison, float %res.f, i32 0
   %res = insertelement <2 x float> %res.tmp, float %res.errf, i32 1
   ret <2 x float> %res
 }
@@ -266,7 +266,7 @@ main_body:
   %res.f = extractelement <4 x float> %res.vec, i32 1
   %res.err = extractvalue {<4 x float>,i32} %v, 1
   %res.errf = bitcast i32 %res.err to float
-  %res.tmp = insertelement <2 x float> undef, float %res.f, i32 0
+  %res.tmp = insertelement <2 x float> poison, float %res.f, i32 0
   %res = insertelement <2 x float> %res.tmp, float %res.errf, i32 1
   ret <2 x float> %res
 }
@@ -325,7 +325,7 @@ main_body:
   %res.f = extractelement <4 x float> %res.vec, i32 2
   %res.err = extractvalue {<4 x float>,i32} %v, 1
   %res.errf = bitcast i32 %res.err to float
-  %res.tmp = insertelement <2 x float> undef, float %res.f, i32 0
+  %res.tmp = insertelement <2 x float> poison, float %res.f, i32 0
   %res = insertelement <2 x float> %res.tmp, float %res.errf, i32 1
   ret <2 x float> %res
 }
@@ -384,7 +384,7 @@ main_body:
   %res.f = extractelement <4 x float> %res.vec, i32 3
   %res.err = extractvalue {<4 x float>,i32} %v, 1
   %res.errf = bitcast i32 %res.err to float
-  %res.tmp = insertelement <2 x float> undef, float %res.f, i32 0
+  %res.tmp = insertelement <2 x float> poison, float %res.f, i32 0
   %res = insertelement <2 x float> %res.tmp, float %res.errf, i32 1
   ret <2 x float> %res
 }
@@ -458,7 +458,7 @@ main_body:
   %res.f2 = extractelement <4 x float> %res.vec, i32 1
   %res.err = extractvalue {<4 x float>,i32} %v, 1
   %res.errf = bitcast i32 %res.err to float
-  %res.tmp1 = insertelement <4 x float> undef, float %res.f1, i32 0
+  %res.tmp1 = insertelement <4 x float> poison, float %res.f1, i32 0
   %res.tmp2 = insertelement <4 x float> %res.tmp1, float %res.f2, i32 1
   %res = insertelement <4 x float> %res.tmp2, float %res.errf, i32 2
   ret <4 x float> %res
@@ -533,7 +533,7 @@ main_body:
   %res.f2 = extractelement <4 x float> %res.vec, i32 3
   %res.err = extractvalue {<4 x float>,i32} %v, 1
   %res.errf = bitcast i32 %res.err to float
-  %res.tmp1 = insertelement <4 x float> undef, float %res.f1, i32 0
+  %res.tmp1 = insertelement <4 x float> poison, float %res.f1, i32 0
   %res.tmp2 = insertelement <4 x float> %res.tmp1, float %res.f2, i32 1
   %res = insertelement <4 x float> %res.tmp2, float %res.errf, i32 2
   ret <4 x float> %res
@@ -602,7 +602,7 @@ main_body:
   %res.f3 = extractelement <4 x float> %res.vec, i32 3
   %res.err = extractvalue {<4 x float>,i32} %v, 1
   %res.errf = bitcast i32 %res.err to float
-  %res.tmp1 = insertelement <4 x float> undef, float %res.f1, i32 0
+  %res.tmp1 = insertelement <4 x float> poison, float %res.f1, i32 0
   %res.tmp2 = insertelement <4 x float> %res.tmp1, float %res.f2, i32 1
   %res.tmp3 = insertelement <4 x float> %res.tmp2, float %res.f3, i32 2
   %res = insertelement <4 x float> %res.tmp3, float %res.errf, i32 3
@@ -2144,7 +2144,7 @@ main_body:
   %v.f2 = extractelement <2 x float> %v.vec, i32 1
   %v.err = extractvalue {<2 x float>, i32} %v, 1
   %v.errf = bitcast i32 %v.err to float
-  %res.0 = insertelement <4 x float> undef, float %v.f1, i32 0
+  %res.0 = insertelement <4 x float> poison, float %v.f1, i32 0
   %res.1 = insertelement <4 x float> %res.0, float %v.f2, i32 1
   %res.2 = insertelement <4 x float> %res.1, float %v.errf, i32 2
   ret <4 x float> %res.2
@@ -2394,7 +2394,7 @@ define amdgpu_ps <2 x float> @adjust_writemask_sample_01(<8 x i32> inreg %rsrc,
 ; GFX12-NEXT:    ; return to shader part epilog
 main_body:
   %r = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
-  %out = shufflevector <4 x float> %r, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  %out = shufflevector <4 x float> %r, <4 x float> poison, <2 x i32> <i32 0, i32 1>
   ret <2 x float> %out
 }
 
@@ -2436,7 +2436,7 @@ define amdgpu_ps <3 x float> @adjust_writemask_sample_012(<8 x i32> inreg %rsrc,
 ; GFX12-NEXT:    ; return to shader part epilog
 main_body:
   %r = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
-  %out = shufflevector <4 x float> %r, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  %out = shufflevector <4 x float> %r, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
   ret <3 x float> %out
 }
 
@@ -2478,7 +2478,7 @@ define amdgpu_ps <2 x float> @adjust_writemask_sample_12(<8 x i32> inreg %rsrc,
 ; GFX12-NEXT:    ; return to shader part epilog
 main_body:
   %r = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
-  %out = shufflevector <4 x float> %r, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+  %out = shufflevector <4 x float> %r, <4 x float> poison, <2 x i32> <i32 1, i32 2>
   ret <2 x float> %out
 }
 
@@ -2520,7 +2520,7 @@ define amdgpu_ps <2 x float> @adjust_writemask_sample_03(<8 x i32> inreg %rsrc,
 ; GFX12-NEXT:    ; return to shader part epilog
 main_body:
   %r = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
-  %out = shufflevector <4 x float> %r, <4 x float> undef, <2 x i32> <i32 0, i32 3>
+  %out = shufflevector <4 x float> %r, <4 x float> poison, <2 x i32> <i32 0, i32 3>
   ret <2 x float> %out
 }
 
@@ -2562,7 +2562,7 @@ define amdgpu_ps <2 x float> @adjust_writemask_sample_13(<8 x i32> inreg %rsrc,
 ; GFX12-NEXT:    ; return to shader part epilog
 main_body:
   %r = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
-  %out = shufflevector <4 x float> %r, <4 x float> undef, <2 x i32> <i32 1, i32 3>
+  %out = shufflevector <4 x float> %r, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   ret <2 x float> %out
 }
 
@@ -2604,7 +2604,7 @@ define amdgpu_ps <3 x float> @adjust_writemask_sample_123(<8 x i32> inreg %rsrc,
 ; GFX12-NEXT:    ; return to shader part epilog
 main_body:
   %r = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
-  %out = shufflevector <4 x float> %r, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+  %out = shufflevector <4 x float> %r, <4 x float> poison, <3 x i32> <i32 1, i32 2, i32 3>
   ret <3 x float> %out
 }
 
@@ -2667,7 +2667,7 @@ define amdgpu_ps <2 x float> @adjust_writemask_sample_123_to_12(<8 x i32> inreg
 ; GFX12-NEXT:    ; return to shader part epilog
 main_body:
   %r = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 14, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
-  %out = shufflevector <4 x float> %r, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  %out = shufflevector <4 x float> %r, <4 x float> poison, <2 x i32> <i32 0, i32 1>
   ret <2 x float> %out
 }
 
@@ -2709,7 +2709,7 @@ define amdgpu_ps <2 x float> @adjust_writemask_sample_013_to_13(<8 x i32> inreg
 ; GFX12-NEXT:    ; return to shader part epilog
 main_body:
   %r = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 11, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
-  %out = shufflevector <4 x float> %r, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+  %out = shufflevector <4 x float> %r, <4 x float> poison, <2 x i32> <i32 1, i32 2>
   ret <2 x float> %out
 }
 
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
index 8af5db9f62908..33a0ed2caa9e6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
@@ -33,13 +33,13 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray(i32 %node_ptr, float %ray_
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    ; return to shader part epilog
 main_body:
-  %ray_origin0 = insertelement <3 x float> undef, float %ray_origin_x, i32 0
+  %ray_origin0 = insertelement <3 x float> poison, float %ray_origin_x, i32 0
   %ray_origin1 = insertelement <3 x float> %ray_origin0, float %ray_origin_y, i32 1
   %ray_origin = insertelement <3 x float> %ray_origin1, float %ray_origin_z, i32 2
-  %ray_dir0 = insertelement <3 x float> undef, float %ray_dir_x, i32 0
+  %ray_dir0 = insertelement <3 x float> poison, float %ray_dir_x, i32 0
   %ray_dir1 = insertelement <3 x float> %ray_dir0, float %ray_dir_y, i32 1
   %ray_dir = insertelement <3 x float> %ray_dir1, float %ray_dir_z, i32 2
-  %ray_inv_dir0 = insertelement <3 x float> undef, float %ray_inv_dir_x, i32 0
+  %ray_inv_dir0 = insertelement <3 x float> poison, float %ray_inv_dir_x, i32 0
   %ray_inv_dir1 = insertelement <3 x float> %ray_inv_dir0, float %ray_inv_dir_y, i32 1
   %ray_inv_dir = insertelement <3 x float> %ray_inv_dir1, float %ray_inv_dir_z, i32 2
   %v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i32.v4f32(i32 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x float> %ray_dir, <3 x float> %ray_inv_dir, <4 x i32> %tdescr)
@@ -152,13 +152,13 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray(<2 x i32> %node_ptr_vec,
 ; GFX12-NEXT:    ; return to shader part epilog
 main_body:
   %node_ptr = bitcast <2 x i32> %node_ptr_vec to i64
-  %ray_origin0 = insertelement <3 x float> undef, float %ray_origin_x, i32 0
+  %ray_origin0 = insertelement <3 x float> poison, float %ray_origin_x, i32 0
   %ray_origin1 = insertelement <3 x float> %ray_origin0, float %ray_origin_y, i32 1
   %ray_origin = insertelement <3 x float> %ray_origin1, float %ray_origin_z, i32 2
-  %ray_dir0 = insertelement <3 x float> undef, float %ray_dir_x, i32 0
+  %ray_dir0 = insertelement <3 x float> poison, float %ray_dir_x, i32 0
   %ray_dir1 = insertelement <3 x float> %ray_dir0, float %ray_dir_y, i32 1
   %ray_dir = insertelement <3 x float> %ray_dir1, float %ray_dir_z, i32 2
-  %ray_inv_dir0 = insertelement <3 x float> undef, float %ray_inv_dir_x, i32 0
+  %ray_inv_dir0 = insertelement <3 x float> poison, float %ray_inv_dir_x, i32 0
   %ray_inv_dir1 = insertelement <3 x float> %ray_inv_dir0, float %ray_inv_dir_y, i32 1
   %ray_inv_dir = insertelement <3 x float> %ray_inv_dir1, float %ray_inv_dir_z, i32 2
   %v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i64.v4f32(i64 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x float> %ray_dir, <3 x float> %ray_inv_dir, <4 x i32> %tdescr)
@@ -411,13 +411,13 @@ main_body:
   %node_ptr = load i32, ptr %gep_node_ptr, align 4
   %gep_ray = getelementptr inbounds float, ptr %p_ray, i32 %lid
   %ray_extent = load float, ptr %gep_ray, align 4
-  %ray_origin0 = insertelement <3 x float> undef, float 0.0, i32 0
+  %ray_origin0 = insertelement <3 x float> poison, float 0.0, i32 0
   %ray_origin1 = insertelement <3 x float> %ray_origin0, float 1.0, i32 1
   %ray_origin = insertelement <3 x float> %ray_origin1, float 2.0, i32 2
-  %ray_dir0 = insertelement <3 x float> undef, float 3.0, i32 0
+  %ray_dir0 = insertelement <3 x float> poison, float 3.0, i32 0
   %ray_dir1 = insertelement <3 x float> %ray_dir0, float 4.0, i32 1
   %ray_dir = insertelement <3 x float> %ray_dir1, float 5.0, i32 2
-  %ray_inv_dir0 = insertelement <3 x float> undef, float 6.0, i32 0
+  %ray_inv_dir0 = insertelement <3 x float> poison, float 6.0, i32 0
   %ray_inv_dir1 = insertelement <3 x float> %ray_inv_dir0, float 7.0, i32 1
   %ray_inv_dir = insertelement <3 x float> %ray_inv_dir1, float 8.0, i32 2
   %v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i32.v4f32(i32 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x float> %ray_dir, <3 x float> %ray_inv_dir, <4 x i32> %tdescr)
@@ -559,13 +559,13 @@ main_body:
   %node_ptr = load i32, ptr %gep_node_ptr, align 4
   %gep_ray = getelementptr inbounds float, ptr %p_ray, i32 %lid
   %ray_extent = load float, ptr %gep_ray, align 4
-  %ray_origin0 = insertelement <3 x float> undef, float 0.0, i32 0
+  %ray_origin0 = insertelement <3 x float> poison, float 0.0, i32 0
   %ray_origin1 = insertelement <3 x float> %ray_origin0, float 1.0, i32 1
   %ray_origin = insertelement <3 x float> %ray_origin1, float 2.0, i32 2
-  %ray_dir0 = insertelement <3 x half> undef, half 3.0, i32 0
+  %ray_dir0 = insertelement <3 x half> poison, half 3.0, i32 0
   %ray_dir1 = insertelement <3 x half> %ray_dir0, half 4.0, i32 1
   %ray_dir = insertelement <3 x half> %ray_dir1, half 5.0, i32 2
-  %ray_inv_dir0 = insertelement <3 x half> undef, half 6.0, i32 0
+  %ray_inv_dir0 = insertelement <3 x half> poison, half 6.0, i32 0
   %ray_inv_dir1 = insertelement <3 x half> %ray_inv_dir0, half 7.0, i32 1
   %ray_inv_dir = insertelement <3 x half> %ray_inv_dir1, half 8.0, i32 2
   %v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i32.v4f16(i32 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x half> %ray_dir, <3 x half> %ray_inv_dir, <4 x i32> %tdescr)
@@ -720,13 +720,13 @@ main_body:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep_ray = getelementptr inbounds float, ptr %p_ray, i32 %lid
   %ray_extent = load float, ptr %gep_ray, align 4
-  %ray_origin0 = insertelement <3 x float> undef, float 0.0, i32 0
+  %ray_origin0 = insertelement <3 x float> poison, float 0.0, i32 0
   %ray_origin1 = insertelement <3 x float> %ray_origin0, float 1.0, i32 1
   %ray_origin = insertelement <3 x float> %ray_origin1, float 2.0, i32 2
-  %ray_dir0 = insertelement <3 x float> undef, float 3.0, i32 0
+  %ray_dir0 = insertelement <3 x float> poison, float 3.0, i32 0
   %ray_dir1 = insertelement <3 x float> %ray_dir0, float 4.0, i32 1
   %ray_dir = insertelement <3 x float> %ray_dir1, float 5.0, i32 2
-  %ray_inv_dir0 = insertelement <3 x float> undef, float 6.0, i32 0
+  %ray_inv_dir0 = insertelement <3 x float> poison, float 6.0, i32 0
   %ray_inv_dir1 = insertelement <3 x float> %ray_inv_dir0, float 7.0, i32 1
   %ray_inv_dir = insertelement <3 x float> %ray_inv_dir1, float 8.0, i32 2
   %v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i64.v4f32(i64 1111111111111, float %ray_extent, <3 x float> %ray_origin, <3 x float> %ray_dir, <3 x float> %ray_inv_dir, <4 x i32> %tdescr)
@@ -870,13 +870,13 @@ main_body:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep_ray = getelementptr inbounds float, ptr %p_ray, i32 %lid
   %ray_extent = load float, ptr %gep_ray, align 4
-  %ray_origin0 = insertelement <3 x float> undef, float 0.0, i32 0
+  %ray_origin0 = insertelement <3 x float> poison, float 0.0, i32 0
   %ray_origin1 = insertelement <3 x float> %ray_origin0, float 1.0, i32 1
   %ray_origin = insertelement <3 x float> %ray_origin1, float 2.0, i32 2
-  %ray_dir0 = insertelement <3 x half> undef, half 3.0, i32 0
+  %ray_dir0 = insertelement <3 x half> poison, half 3.0, i32 0
   %ray_dir1 = insertelement <3 x half> %ray_dir0, half 4.0, i32 1
   %ray_dir = insertelement <3 x half> %ray_dir1, half 5.0, i32 2
-  %ray_inv_dir0 = insertelement <3 x half> undef, half 6.0, i32 0
+  %ray_inv_dir0 = insertelement <3 x half> poison, half 6.0, i32 0
   %ray_inv_dir1 = insertelement <3 x half> %ray_inv_dir0, half 7.0, i32 1
   %ray_inv_dir = insertelement <3 x half> %ray_inv_dir1, half 8.0, i32 2
   %v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i64.v4f16(i64 1111111111110, float %ray_extent, <3 x float> %ray_origin, <3 x half> %ray_dir, <3 x half> %ray_inv_dir, <4 x i32> %tdescr)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.format.ll
index 29efdddac39b2..5e84ea59816af 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.format.ll
@@ -11,7 +11,7 @@ main_body:
   %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 0, i32 0)
   %data_glc = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 0, i32 1)
   %data_slc = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 0, i32 2)
-  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %data, 0
+  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %data, 0
   %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %data_glc, 1
   %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %data_slc, 2
   ret {<4 x float>, <4 x float>, <4 x float>} %r2
@@ -30,7 +30,7 @@ main_body:
   %fdata = bitcast <4 x i32> %data to <4 x float>
   %fdata_glc = bitcast <4 x i32> %data_glc to <4 x float>
   %fdata_slc = bitcast <4 x i32> %data_slc to <4 x float>
-  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %fdata, 0
+  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %fdata, 0
   %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %fdata_glc, 1
   %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %fdata_slc, 2
   ret {<4 x float>, <4 x float>, <4 x float>} %r2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
index b42ba7d75094a..8ae33553c29d8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
@@ -45,7 +45,7 @@ main_body:
   %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 0, i32 0)
   %data_glc = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 0, i32 1)
   %data_slc = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 0, i32 2)
-  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %data, 0
+  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %data, 0
   %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %data_glc, 1
   %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %data_slc, 2
   ret {<4 x float>, <4 x float>, <4 x float>} %r2
@@ -90,7 +90,7 @@ main_body:
   %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 0, i32 4)
   %data_glc = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 0, i32 5)
   %data_slc = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 0, i32 6)
-  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %data, 0
+  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %data, 0
   %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %data_glc, 1
   %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %data_slc, 2
   ret {<4 x float>, <4 x float>, <4 x float>} %r2
@@ -936,7 +936,7 @@ main_body:
   %fdata = bitcast <4 x i32> %data to <4 x float>
   %fdata_glc = bitcast <2 x i32> %data_glc to <2 x float>
   %fdata_slc = bitcast i32 %data_slc to float
-  %r0 = insertvalue {<4 x float>, <2 x float>, float} undef, <4 x float> %fdata, 0
+  %r0 = insertvalue {<4 x float>, <2 x float>, float} poison, <4 x float> %fdata, 0
   %r1 = insertvalue {<4 x float>, <2 x float>, float} %r0, <2 x float> %fdata_glc, 1
   %r2 = insertvalue {<4 x float>, <2 x float>, float} %r1, float %fdata_slc, 2
   ret {<4 x float>, <2 x float>, float} %r2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.format.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.format.d16.ll
index ca03d954d5b2a..5b4e2e4ce171b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.format.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.format.d16.ll
@@ -47,7 +47,7 @@ main_body:
 ; PACKED: buffer_store_format_d16_xyz v[[[LO]]:[[HI]]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen
 define amdgpu_kernel void @buffer_store_format_d16_xyz(<4 x i32> %rsrc, <4 x half> %data, i32 %voffset) {
 main_body:
-  %data_subvec = shufflevector <4 x half> %data, <4 x half> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  %data_subvec = shufflevector <4 x half> %data, <4 x half> poison, <3 x i32> <i32 0, i32 1, i32 2>
   call void @llvm.amdgcn.raw.buffer.store.format.v3f16(<3 x half> %data_subvec, <4 x i32> %rsrc, i32 %voffset, i32 0, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.format.ll
index 81aee1e9c4488..8021391a2c02c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.format.ll
@@ -11,7 +11,7 @@ main_body:
   %data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0)
   %data_glc = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 1)
   %data_slc = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 2)
-  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %data, 0
+  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %data, 0
   %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %data_glc, 1
   %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %data_slc, 2
   ret {<4 x float>, <4 x float>, <4 x float>} %r2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll
index d600d6e759124..05116d556d475 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll
@@ -34,7 +34,7 @@ main_body:
   %data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0)
   %data_glc = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 1)
   %data_slc = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 2)
-  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %data, 0
+  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %data, 0
   %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %data_glc, 1
   %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %data_slc, 2
   ret {<4 x float>, <4 x float>, <4 x float>} %r2
@@ -70,7 +70,7 @@ main_body:
   %data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 4)
   %data_glc = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 5)
   %data_slc = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 6)
-  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %data, 0
+  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %data, 0
   %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %data_glc, 1
   %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %data_slc, 2
   ret {<4 x float>, <4 x float>, <4 x float>} %r2
@@ -106,7 +106,7 @@ main_body:
   %data = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 -2147483648)
   %data_glc = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 -2147483647)
   %data_slc = call <4 x float> @llvm.amdgcn.raw.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 -2147483646)
-  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %data, 0
+  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %data, 0
   %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %data_glc, 1
   %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %data_slc, 2
   ret {<4 x float>, <4 x float>, <4 x float>} %r2
@@ -768,7 +768,7 @@ main_body:
   %fdata = bitcast <4 x i32> %data to <4 x float>
   %fdata_glc = bitcast <2 x i32> %data_glc to <2 x float>
   %fdata_slc = bitcast i32 %data_slc to float
-  %r0 = insertvalue {<4 x float>, <2 x float>, float} undef, <4 x float> %fdata, 0
+  %r0 = insertvalue {<4 x float>, <2 x float>, float} poison, <4 x float> %fdata, 0
   %r1 = insertvalue {<4 x float>, <2 x float>, float} %r0, <2 x float> %fdata_glc, 1
   %r2 = insertvalue {<4 x float>, <2 x float>, float} %r1, float %fdata_slc, 2
   ret {<4 x float>, <2 x float>, float} %r2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.store.format.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.store.format.d16.ll
index 8e278c101057a..b1a2747cb5784 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.store.format.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.store.format.d16.ll
@@ -47,7 +47,7 @@ main_body:
 ; PACKED: buffer_store_format_d16_xyz v[[[LO]]:[[HI]]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen
 define amdgpu_kernel void @buffer_store_format_d16_xyz(ptr addrspace(8) %rsrc, <4 x half> %data, i32 %voffset) {
 main_body:
-  %data_subvec = shufflevector <4 x half> %data, <4 x half> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  %data_subvec = shufflevector <4 x half> %data, <4 x half> poison, <3 x i32> <i32 0, i32 1, i32 2>
   call void @llvm.amdgcn.raw.ptr.buffer.store.format.v3f16(<3 x half> %data_subvec, ptr addrspace(8) %rsrc, i32 %voffset, i32 0, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.ll
index 8bac51994cc48..cce9af9ed4262 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.ll
@@ -41,7 +41,7 @@ main_body:
     %vdata.f     = bitcast <4 x i32> %vdata to <4 x float>
     %vdata_glc.f = bitcast <4 x i32> %vdata_glc to <4 x float>
     %vdata_slc.f = bitcast <4 x i32> %vdata_slc to <4 x float>
-    %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %vdata.f, 0
+    %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %vdata.f, 0
     %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %vdata_glc.f, 1
     %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %vdata_slc.f, 2
     %r3 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} %r2, <4 x float> %vdata_f32, 3
@@ -231,7 +231,7 @@ define amdgpu_vs {<4 x float>, <4 x float>, <4 x float>} @tbuffer_load_immoffs_l
     %vdata.f     = bitcast <4 x i32> %vdata to <4 x float>
     %vdata_glc.f = bitcast <4 x i32> %vdata_glc to <4 x float>
     %vdata_slc.f = bitcast <4 x i32> %vdata_slc to <4 x float>
-    %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %vdata.f, 0
+    %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %vdata.f, 0
     %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %vdata_glc.f, 1
     %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %vdata_slc.f, 2
     ret {<4 x float>, <4 x float>, <4 x float>} %r2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.d16.ll
index 02fc82de5d7bc..9c576039ff6ac 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.store.d16.ll
@@ -144,7 +144,7 @@ define amdgpu_kernel void @tbuffer_store_d16_xyz(ptr addrspace(8) %rsrc, <4 x ha
 ; GFX11-PACKED-NEXT:    tbuffer_store_d16_format_xyz v[0:1], off, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM]
 ; GFX11-PACKED-NEXT:    s_endpgm
 main_body:
-  %data_subvec = shufflevector <4 x half> %data, <4 x half> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  %data_subvec = shufflevector <4 x half> %data, <4 x half> poison, <3 x i32> <i32 0, i32 1, i32 2>
   call void @llvm.amdgcn.raw.ptr.tbuffer.store.v3f16(<3 x half> %data_subvec, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 33, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll
index 73a27dc62695c..c59f8bc40d0c7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.ll
@@ -53,7 +53,7 @@ main_body:
     %vdata.f     = bitcast <4 x i32> %vdata to <4 x float>
     %vdata_glc.f = bitcast <4 x i32> %vdata_glc to <4 x float>
     %vdata_slc.f = bitcast <4 x i32> %vdata_slc to <4 x float>
-    %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %vdata.f, 0
+    %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %vdata.f, 0
     %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %vdata_glc.f, 1
     %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %vdata_slc.f, 2
     %r3 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} %r2, <4 x float> %vdata_f32, 3
@@ -290,7 +290,7 @@ define amdgpu_vs {<4 x float>, <4 x float>, <4 x float>} @tbuffer_load_immoffs_l
     %vdata.f     = bitcast <4 x i32> %vdata to <4 x float>
     %vdata_glc.f = bitcast <4 x i32> %vdata_glc to <4 x float>
     %vdata_slc.f = bitcast <4 x i32> %vdata_slc to <4 x float>
-    %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %vdata.f, 0
+    %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %vdata.f, 0
     %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %vdata_glc.f, 1
     %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %vdata_slc.f, 2
     ret {<4 x float>, <4 x float>, <4 x float>} %r2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.d16.ll
index 63b139bb25e77..cde2627008e27 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.store.d16.ll
@@ -191,7 +191,7 @@ define amdgpu_kernel void @tbuffer_store_d16_xyz(<4 x i32> %rsrc, <4 x half> %da
 ; GFX12-PACKED-GISEL-NEXT:    tbuffer_store_d16_format_xyzw v[0:1], off, s[0:3], null format:[BUF_FMT_10_10_10_2_SNORM]
 ; GFX12-PACKED-GISEL-NEXT:    s_endpgm
 main_body:
-  %data_subvec = shufflevector <4 x half> %data, <4 x half> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  %data_subvec = shufflevector <4 x half> %data, <4 x half> poison, <3 x i32> <i32 0, i32 1, i32 2>
   call void @llvm.amdgcn.raw.tbuffer.store.v3f16(<3 x half> %data_subvec, <4 x i32> %rsrc, i32 0, i32 0, i32 33, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
index a3bd0aabd5c3f..55fa02a0c582c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
@@ -815,50 +815,27 @@ define void @test_readfirstlane_v32f32(ptr addrspace(1) %out, <32 x float> %src)
 ; CHECK-SDAG-NEXT:    s_xor_saveexec_b64 s[4:5], -1
 ; CHECK-SDAG-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
 ; CHECK-SDAG-NEXT:    s_mov_b64 exec, s[4:5]
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s36, 0
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s37, 1
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s38, 2
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s39, 3
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s40, 4
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s41, 5
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s42, 6
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s43, 7
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s44, 8
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s45, 9
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s46, 10
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s47, 11
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s48, 12
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s49, 13
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s50, 14
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s51, 15
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s52, 16
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s53, 17
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s54, 18
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s55, 19
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s56, 20
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s57, 21
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s58, 22
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s59, 23
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s60, 24
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s61, 25
 ; CHECK-SDAG-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:4
 ; CHECK-SDAG-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:8
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s61, v27
 ; CHECK-SDAG-NEXT:    buffer_load_dword v27, off, s[0:3], s32
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s62, 26
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s63, 27
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s64, 28
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s65, 29
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s66, 30
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s67, 31
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s36, 0
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s37, 1
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s38, 2
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s39, 3
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s48, 4
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s49, 5
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s50, 6
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s51, 7
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s52, 8
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s53, 9
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s54, 10
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s55, 11
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s64, 12
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s65, 13
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s66, 14
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s67, 15
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s64, v30
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s63, v29
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s62, v28
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s60, v26
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s59, v25
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s58, v24
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s57, v23
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s56, v22
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s55, v21
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s54, v20
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s53, v19
@@ -867,6 +844,17 @@ define void @test_readfirstlane_v32f32(ptr addrspace(1) %out, <32 x float> %src)
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s50, v16
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s49, v15
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s48, v14
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s39, v5
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s38, v4
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s37, v3
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s36, v2
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s63, v29
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s62, v28
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s60, v26
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s59, v25
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s58, v24
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s57, v23
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s56, v22
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s47, v13
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s46, v12
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s45, v11
@@ -875,10 +863,6 @@ define void @test_readfirstlane_v32f32(ptr addrspace(1) %out, <32 x float> %src)
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s42, v8
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s41, v7
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s40, v6
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s39, v5
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s38, v4
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s37, v3
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s36, v2
 ; CHECK-SDAG-NEXT:    s_waitcnt vmcnt(2)
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s66, v0
 ; CHECK-SDAG-NEXT:    s_waitcnt vmcnt(1)
@@ -888,34 +872,18 @@ define void @test_readfirstlane_v32f32(ptr addrspace(1) %out, <32 x float> %src)
 ; CHECK-SDAG-NEXT:    ;;#ASMSTART
 ; CHECK-SDAG-NEXT:    ; use s[36:67]
 ; CHECK-SDAG-NEXT:    ;;#ASMEND
-; CHECK-SDAG-NEXT:    v_readlane_b32 s67, v31, 31
-; CHECK-SDAG-NEXT:    v_readlane_b32 s66, v31, 30
-; CHECK-SDAG-NEXT:    v_readlane_b32 s65, v31, 29
-; CHECK-SDAG-NEXT:    v_readlane_b32 s64, v31, 28
-; CHECK-SDAG-NEXT:    v_readlane_b32 s63, v31, 27
-; CHECK-SDAG-NEXT:    v_readlane_b32 s62, v31, 26
-; CHECK-SDAG-NEXT:    v_readlane_b32 s61, v31, 25
-; CHECK-SDAG-NEXT:    v_readlane_b32 s60, v31, 24
-; CHECK-SDAG-NEXT:    v_readlane_b32 s59, v31, 23
-; CHECK-SDAG-NEXT:    v_readlane_b32 s58, v31, 22
-; CHECK-SDAG-NEXT:    v_readlane_b32 s57, v31, 21
-; CHECK-SDAG-NEXT:    v_readlane_b32 s56, v31, 20
-; CHECK-SDAG-NEXT:    v_readlane_b32 s55, v31, 19
-; CHECK-SDAG-NEXT:    v_readlane_b32 s54, v31, 18
-; CHECK-SDAG-NEXT:    v_readlane_b32 s53, v31, 17
-; CHECK-SDAG-NEXT:    v_readlane_b32 s52, v31, 16
-; CHECK-SDAG-NEXT:    v_readlane_b32 s51, v31, 15
-; CHECK-SDAG-NEXT:    v_readlane_b32 s50, v31, 14
-; CHECK-SDAG-NEXT:    v_readlane_b32 s49, v31, 13
-; CHECK-SDAG-NEXT:    v_readlane_b32 s48, v31, 12
-; CHECK-SDAG-NEXT:    v_readlane_b32 s47, v31, 11
-; CHECK-SDAG-NEXT:    v_readlane_b32 s46, v31, 10
-; CHECK-SDAG-NEXT:    v_readlane_b32 s45, v31, 9
-; CHECK-SDAG-NEXT:    v_readlane_b32 s44, v31, 8
-; CHECK-SDAG-NEXT:    v_readlane_b32 s43, v31, 7
-; CHECK-SDAG-NEXT:    v_readlane_b32 s42, v31, 6
-; CHECK-SDAG-NEXT:    v_readlane_b32 s41, v31, 5
-; CHECK-SDAG-NEXT:    v_readlane_b32 s40, v31, 4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s67, v31, 15
+; CHECK-SDAG-NEXT:    v_readlane_b32 s66, v31, 14
+; CHECK-SDAG-NEXT:    v_readlane_b32 s65, v31, 13
+; CHECK-SDAG-NEXT:    v_readlane_b32 s64, v31, 12
+; CHECK-SDAG-NEXT:    v_readlane_b32 s55, v31, 11
+; CHECK-SDAG-NEXT:    v_readlane_b32 s54, v31, 10
+; CHECK-SDAG-NEXT:    v_readlane_b32 s53, v31, 9
+; CHECK-SDAG-NEXT:    v_readlane_b32 s52, v31, 8
+; CHECK-SDAG-NEXT:    v_readlane_b32 s51, v31, 7
+; CHECK-SDAG-NEXT:    v_readlane_b32 s50, v31, 6
+; CHECK-SDAG-NEXT:    v_readlane_b32 s49, v31, 5
+; CHECK-SDAG-NEXT:    v_readlane_b32 s48, v31, 4
 ; CHECK-SDAG-NEXT:    v_readlane_b32 s39, v31, 3
 ; CHECK-SDAG-NEXT:    v_readlane_b32 s38, v31, 2
 ; CHECK-SDAG-NEXT:    v_readlane_b32 s37, v31, 1
@@ -940,45 +908,21 @@ define void @test_readfirstlane_v32f32(ptr addrspace(1) %out, <32 x float> %src)
 ; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s37, 1
 ; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s38, 2
 ; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s39, 3
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s40, 4
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s41, 5
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s42, 6
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s43, 7
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s44, 8
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s45, 9
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s46, 10
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s47, 11
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s48, 12
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s49, 13
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s50, 14
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s51, 15
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s52, 16
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s53, 17
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s54, 18
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s55, 19
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s56, 20
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s57, 21
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s58, 22
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s59, 23
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s60, 24
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s61, 25
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s62, 26
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s63, 27
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s64, 28
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s65, 29
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s66, 30
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s67, 31
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s48, 4
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s49, 5
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s50, 6
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s51, 7
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s52, 8
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s53, 9
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s54, 10
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s55, 11
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s64, 12
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s65, 13
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s66, 14
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s67, 15
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s37, v3
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s38, v4
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s39, v5
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s40, v6
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s41, v7
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s42, v8
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s43, v9
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s44, v10
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s45, v11
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s46, v12
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s47, v13
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s48, v14
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s49, v15
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s50, v16
@@ -987,6 +931,15 @@ define void @test_readfirstlane_v32f32(ptr addrspace(1) %out, <32 x float> %src)
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s53, v19
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s54, v20
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s55, v21
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s64, v30
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s40, v6
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s41, v7
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s42, v8
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s43, v9
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s44, v10
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s45, v11
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s46, v12
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s47, v13
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s56, v22
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s57, v23
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s58, v24
@@ -995,7 +948,6 @@ define void @test_readfirstlane_v32f32(ptr addrspace(1) %out, <32 x float> %src)
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s61, v27
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s62, v28
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s63, v29
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s64, v30
 ; CHECK-GISEL-NEXT:    s_waitcnt vmcnt(2)
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s65, v0
 ; CHECK-GISEL-NEXT:    s_waitcnt vmcnt(1)
@@ -1005,34 +957,18 @@ define void @test_readfirstlane_v32f32(ptr addrspace(1) %out, <32 x float> %src)
 ; CHECK-GISEL-NEXT:    ;;#ASMSTART
 ; CHECK-GISEL-NEXT:    ; use s[36:67]
 ; CHECK-GISEL-NEXT:    ;;#ASMEND
-; CHECK-GISEL-NEXT:    v_readlane_b32 s67, v31, 31
-; CHECK-GISEL-NEXT:    v_readlane_b32 s66, v31, 30
-; CHECK-GISEL-NEXT:    v_readlane_b32 s65, v31, 29
-; CHECK-GISEL-NEXT:    v_readlane_b32 s64, v31, 28
-; CHECK-GISEL-NEXT:    v_readlane_b32 s63, v31, 27
-; CHECK-GISEL-NEXT:    v_readlane_b32 s62, v31, 26
-; CHECK-GISEL-NEXT:    v_readlane_b32 s61, v31, 25
-; CHECK-GISEL-NEXT:    v_readlane_b32 s60, v31, 24
-; CHECK-GISEL-NEXT:    v_readlane_b32 s59, v31, 23
-; CHECK-GISEL-NEXT:    v_readlane_b32 s58, v31, 22
-; CHECK-GISEL-NEXT:    v_readlane_b32 s57, v31, 21
-; CHECK-GISEL-NEXT:    v_readlane_b32 s56, v31, 20
-; CHECK-GISEL-NEXT:    v_readlane_b32 s55, v31, 19
-; CHECK-GISEL-NEXT:    v_readlane_b32 s54, v31, 18
-; CHECK-GISEL-NEXT:    v_readlane_b32 s53, v31, 17
-; CHECK-GISEL-NEXT:    v_readlane_b32 s52, v31, 16
-; CHECK-GISEL-NEXT:    v_readlane_b32 s51, v31, 15
-; CHECK-GISEL-NEXT:    v_readlane_b32 s50, v31, 14
-; CHECK-GISEL-NEXT:    v_readlane_b32 s49, v31, 13
-; CHECK-GISEL-NEXT:    v_readlane_b32 s48, v31, 12
-; CHECK-GISEL-NEXT:    v_readlane_b32 s47, v31, 11
-; CHECK-GISEL-NEXT:    v_readlane_b32 s46, v31, 10
-; CHECK-GISEL-NEXT:    v_readlane_b32 s45, v31, 9
-; CHECK-GISEL-NEXT:    v_readlane_b32 s44, v31, 8
-; CHECK-GISEL-NEXT:    v_readlane_b32 s43, v31, 7
-; CHECK-GISEL-NEXT:    v_readlane_b32 s42, v31, 6
-; CHECK-GISEL-NEXT:    v_readlane_b32 s41, v31, 5
-; CHECK-GISEL-NEXT:    v_readlane_b32 s40, v31, 4
+; CHECK-GISEL-NEXT:    v_readlane_b32 s67, v31, 15
+; CHECK-GISEL-NEXT:    v_readlane_b32 s66, v31, 14
+; CHECK-GISEL-NEXT:    v_readlane_b32 s65, v31, 13
+; CHECK-GISEL-NEXT:    v_readlane_b32 s64, v31, 12
+; CHECK-GISEL-NEXT:    v_readlane_b32 s55, v31, 11
+; CHECK-GISEL-NEXT:    v_readlane_b32 s54, v31, 10
+; CHECK-GISEL-NEXT:    v_readlane_b32 s53, v31, 9
+; CHECK-GISEL-NEXT:    v_readlane_b32 s52, v31, 8
+; CHECK-GISEL-NEXT:    v_readlane_b32 s51, v31, 7
+; CHECK-GISEL-NEXT:    v_readlane_b32 s50, v31, 6
+; CHECK-GISEL-NEXT:    v_readlane_b32 s49, v31, 5
+; CHECK-GISEL-NEXT:    v_readlane_b32 s48, v31, 4
 ; CHECK-GISEL-NEXT:    v_readlane_b32 s39, v31, 3
 ; CHECK-GISEL-NEXT:    v_readlane_b32 s38, v31, 2
 ; CHECK-GISEL-NEXT:    v_readlane_b32 s37, v31, 1
@@ -1324,50 +1260,27 @@ define void @test_readfirstlane_v32i32(ptr addrspace(1) %out, <32 x i32> %src) {
 ; CHECK-SDAG-NEXT:    s_xor_saveexec_b64 s[4:5], -1
 ; CHECK-SDAG-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
 ; CHECK-SDAG-NEXT:    s_mov_b64 exec, s[4:5]
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s36, 0
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s37, 1
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s38, 2
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s39, 3
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s40, 4
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s41, 5
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s42, 6
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s43, 7
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s44, 8
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s45, 9
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s46, 10
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s47, 11
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s48, 12
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s49, 13
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s50, 14
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s51, 15
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s52, 16
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s53, 17
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s54, 18
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s55, 19
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s56, 20
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s57, 21
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s58, 22
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s59, 23
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s60, 24
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s61, 25
 ; CHECK-SDAG-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:4
 ; CHECK-SDAG-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:8
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s61, v27
 ; CHECK-SDAG-NEXT:    buffer_load_dword v27, off, s[0:3], s32
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s62, 26
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s63, 27
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s64, 28
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s65, 29
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s66, 30
-; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s67, 31
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s36, 0
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s37, 1
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s38, 2
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s39, 3
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s48, 4
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s49, 5
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s50, 6
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s51, 7
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s52, 8
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s53, 9
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s54, 10
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s55, 11
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s64, 12
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s65, 13
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s66, 14
+; CHECK-SDAG-NEXT:    v_writelane_b32 v31, s67, 15
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s64, v30
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s63, v29
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s62, v28
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s60, v26
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s59, v25
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s58, v24
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s57, v23
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s56, v22
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s55, v21
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s54, v20
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s53, v19
@@ -1376,6 +1289,17 @@ define void @test_readfirstlane_v32i32(ptr addrspace(1) %out, <32 x i32> %src) {
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s50, v16
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s49, v15
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s48, v14
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s39, v5
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s38, v4
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s37, v3
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s36, v2
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s63, v29
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s62, v28
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s60, v26
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s59, v25
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s58, v24
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s57, v23
+; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s56, v22
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s47, v13
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s46, v12
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s45, v11
@@ -1384,10 +1308,6 @@ define void @test_readfirstlane_v32i32(ptr addrspace(1) %out, <32 x i32> %src) {
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s42, v8
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s41, v7
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s40, v6
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s39, v5
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s38, v4
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s37, v3
-; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s36, v2
 ; CHECK-SDAG-NEXT:    s_waitcnt vmcnt(2)
 ; CHECK-SDAG-NEXT:    v_readfirstlane_b32 s66, v0
 ; CHECK-SDAG-NEXT:    s_waitcnt vmcnt(1)
@@ -1397,34 +1317,18 @@ define void @test_readfirstlane_v32i32(ptr addrspace(1) %out, <32 x i32> %src) {
 ; CHECK-SDAG-NEXT:    ;;#ASMSTART
 ; CHECK-SDAG-NEXT:    ; use s[36:67]
 ; CHECK-SDAG-NEXT:    ;;#ASMEND
-; CHECK-SDAG-NEXT:    v_readlane_b32 s67, v31, 31
-; CHECK-SDAG-NEXT:    v_readlane_b32 s66, v31, 30
-; CHECK-SDAG-NEXT:    v_readlane_b32 s65, v31, 29
-; CHECK-SDAG-NEXT:    v_readlane_b32 s64, v31, 28
-; CHECK-SDAG-NEXT:    v_readlane_b32 s63, v31, 27
-; CHECK-SDAG-NEXT:    v_readlane_b32 s62, v31, 26
-; CHECK-SDAG-NEXT:    v_readlane_b32 s61, v31, 25
-; CHECK-SDAG-NEXT:    v_readlane_b32 s60, v31, 24
-; CHECK-SDAG-NEXT:    v_readlane_b32 s59, v31, 23
-; CHECK-SDAG-NEXT:    v_readlane_b32 s58, v31, 22
-; CHECK-SDAG-NEXT:    v_readlane_b32 s57, v31, 21
-; CHECK-SDAG-NEXT:    v_readlane_b32 s56, v31, 20
-; CHECK-SDAG-NEXT:    v_readlane_b32 s55, v31, 19
-; CHECK-SDAG-NEXT:    v_readlane_b32 s54, v31, 18
-; CHECK-SDAG-NEXT:    v_readlane_b32 s53, v31, 17
-; CHECK-SDAG-NEXT:    v_readlane_b32 s52, v31, 16
-; CHECK-SDAG-NEXT:    v_readlane_b32 s51, v31, 15
-; CHECK-SDAG-NEXT:    v_readlane_b32 s50, v31, 14
-; CHECK-SDAG-NEXT:    v_readlane_b32 s49, v31, 13
-; CHECK-SDAG-NEXT:    v_readlane_b32 s48, v31, 12
-; CHECK-SDAG-NEXT:    v_readlane_b32 s47, v31, 11
-; CHECK-SDAG-NEXT:    v_readlane_b32 s46, v31, 10
-; CHECK-SDAG-NEXT:    v_readlane_b32 s45, v31, 9
-; CHECK-SDAG-NEXT:    v_readlane_b32 s44, v31, 8
-; CHECK-SDAG-NEXT:    v_readlane_b32 s43, v31, 7
-; CHECK-SDAG-NEXT:    v_readlane_b32 s42, v31, 6
-; CHECK-SDAG-NEXT:    v_readlane_b32 s41, v31, 5
-; CHECK-SDAG-NEXT:    v_readlane_b32 s40, v31, 4
+; CHECK-SDAG-NEXT:    v_readlane_b32 s67, v31, 15
+; CHECK-SDAG-NEXT:    v_readlane_b32 s66, v31, 14
+; CHECK-SDAG-NEXT:    v_readlane_b32 s65, v31, 13
+; CHECK-SDAG-NEXT:    v_readlane_b32 s64, v31, 12
+; CHECK-SDAG-NEXT:    v_readlane_b32 s55, v31, 11
+; CHECK-SDAG-NEXT:    v_readlane_b32 s54, v31, 10
+; CHECK-SDAG-NEXT:    v_readlane_b32 s53, v31, 9
+; CHECK-SDAG-NEXT:    v_readlane_b32 s52, v31, 8
+; CHECK-SDAG-NEXT:    v_readlane_b32 s51, v31, 7
+; CHECK-SDAG-NEXT:    v_readlane_b32 s50, v31, 6
+; CHECK-SDAG-NEXT:    v_readlane_b32 s49, v31, 5
+; CHECK-SDAG-NEXT:    v_readlane_b32 s48, v31, 4
 ; CHECK-SDAG-NEXT:    v_readlane_b32 s39, v31, 3
 ; CHECK-SDAG-NEXT:    v_readlane_b32 s38, v31, 2
 ; CHECK-SDAG-NEXT:    v_readlane_b32 s37, v31, 1
@@ -1449,45 +1353,21 @@ define void @test_readfirstlane_v32i32(ptr addrspace(1) %out, <32 x i32> %src) {
 ; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s37, 1
 ; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s38, 2
 ; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s39, 3
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s40, 4
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s41, 5
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s42, 6
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s43, 7
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s44, 8
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s45, 9
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s46, 10
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s47, 11
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s48, 12
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s49, 13
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s50, 14
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s51, 15
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s52, 16
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s53, 17
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s54, 18
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s55, 19
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s56, 20
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s57, 21
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s58, 22
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s59, 23
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s60, 24
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s61, 25
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s62, 26
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s63, 27
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s64, 28
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s65, 29
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s66, 30
-; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s67, 31
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s48, 4
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s49, 5
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s50, 6
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s51, 7
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s52, 8
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s53, 9
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s54, 10
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s55, 11
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s64, 12
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s65, 13
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s66, 14
+; CHECK-GISEL-NEXT:    v_writelane_b32 v31, s67, 15
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s37, v3
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s38, v4
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s39, v5
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s40, v6
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s41, v7
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s42, v8
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s43, v9
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s44, v10
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s45, v11
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s46, v12
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s47, v13
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s48, v14
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s49, v15
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s50, v16
@@ -1496,6 +1376,15 @@ define void @test_readfirstlane_v32i32(ptr addrspace(1) %out, <32 x i32> %src) {
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s53, v19
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s54, v20
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s55, v21
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s64, v30
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s40, v6
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s41, v7
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s42, v8
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s43, v9
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s44, v10
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s45, v11
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s46, v12
+; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s47, v13
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s56, v22
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s57, v23
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s58, v24
@@ -1504,7 +1393,6 @@ define void @test_readfirstlane_v32i32(ptr addrspace(1) %out, <32 x i32> %src) {
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s61, v27
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s62, v28
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s63, v29
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s64, v30
 ; CHECK-GISEL-NEXT:    s_waitcnt vmcnt(2)
 ; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s65, v0
 ; CHECK-GISEL-NEXT:    s_waitcnt vmcnt(1)
@@ -1514,34 +1402,18 @@ define void @test_readfirstlane_v32i32(ptr addrspace(1) %out, <32 x i32> %src) {
 ; CHECK-GISEL-NEXT:    ;;#ASMSTART
 ; CHECK-GISEL-NEXT:    ; use s[36:67]
 ; CHECK-GISEL-NEXT:    ;;#ASMEND
-; CHECK-GISEL-NEXT:    v_readlane_b32 s67, v31, 31
-; CHECK-GISEL-NEXT:    v_readlane_b32 s66, v31, 30
-; CHECK-GISEL-NEXT:    v_readlane_b32 s65, v31, 29
-; CHECK-GISEL-NEXT:    v_readlane_b32 s64, v31, 28
-; CHECK-GISEL-NEXT:    v_readlane_b32 s63, v31, 27
-; CHECK-GISEL-NEXT:    v_readlane_b32 s62, v31, 26
-; CHECK-GISEL-NEXT:    v_readlane_b32 s61, v31, 25
-; CHECK-GISEL-NEXT:    v_readlane_b32 s60, v31, 24
-; CHECK-GISEL-NEXT:    v_readlane_b32 s59, v31, 23
-; CHECK-GISEL-NEXT:    v_readlane_b32 s58, v31, 22
-; CHECK-GISEL-NEXT:    v_readlane_b32 s57, v31, 21
-; CHECK-GISEL-NEXT:    v_readlane_b32 s56, v31, 20
-; CHECK-GISEL-NEXT:    v_readlane_b32 s55, v31, 19
-; CHECK-GISEL-NEXT:    v_readlane_b32 s54, v31, 18
-; CHECK-GISEL-NEXT:    v_readlane_b32 s53, v31, 17
-; CHECK-GISEL-NEXT:    v_readlane_b32 s52, v31, 16
-; CHECK-GISEL-NEXT:    v_readlane_b32 s51, v31, 15
-; CHECK-GISEL-NEXT:    v_readlane_b32 s50, v31, 14
-; CHECK-GISEL-NEXT:    v_readlane_b32 s49, v31, 13
-; CHECK-GISEL-NEXT:    v_readlane_b32 s48, v31, 12
-; CHECK-GISEL-NEXT:    v_readlane_b32 s47, v31, 11
-; CHECK-GISEL-NEXT:    v_readlane_b32 s46, v31, 10
-; CHECK-GISEL-NEXT:    v_readlane_b32 s45, v31, 9
-; CHECK-GISEL-NEXT:    v_readlane_b32 s44, v31, 8
-; CHECK-GISEL-NEXT:    v_readlane_b32 s43, v31, 7
-; CHECK-GISEL-NEXT:    v_readlane_b32 s42, v31, 6
-; CHECK-GISEL-NEXT:    v_readlane_b32 s41, v31, 5
-; CHECK-GISEL-NEXT:    v_readlane_b32 s40, v31, 4
+; CHECK-GISEL-NEXT:    v_readlane_b32 s67, v31, 15
+; CHECK-GISEL-NEXT:    v_readlane_b32 s66, v31, 14
+; CHECK-GISEL-NEXT:    v_readlane_b32 s65, v31, 13
+; CHECK-GISEL-NEXT:    v_readlane_b32 s64, v31, 12
+; CHECK-GISEL-NEXT:    v_readlane_b32 s55, v31, 11
+; CHECK-GISEL-NEXT:    v_readlane_b32 s54, v31, 10
+; CHECK-GISEL-NEXT:    v_readlane_b32 s53, v31, 9
+; CHECK-GISEL-NEXT:    v_readlane_b32 s52, v31, 8
+; CHECK-GISEL-NEXT:    v_readlane_b32 s51, v31, 7
+; CHECK-GISEL-NEXT:    v_readlane_b32 s50, v31, 6
+; CHECK-GISEL-NEXT:    v_readlane_b32 s49, v31, 5
+; CHECK-GISEL-NEXT:    v_readlane_b32 s48, v31, 4
 ; CHECK-GISEL-NEXT:    v_readlane_b32 s39, v31, 3
 ; CHECK-GISEL-NEXT:    v_readlane_b32 s38, v31, 2
 ; CHECK-GISEL-NEXT:    v_readlane_b32 s37, v31, 1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll
index ba3d306cc0cfd..e81fee9c9bedb 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll
@@ -58,7 +58,7 @@ main_body:
   %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 0, i32 0, i32 0)
   %data_glc = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 0, i32 0, i32 1)
   %data_slc = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 0, i32 0, i32 2)
-  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %data, 0
+  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %data, 0
   %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %data_glc, 1
   %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %data_slc, 2
   ret {<4 x float>, <4 x float>, <4 x float>} %r2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.ll
index 5e03748bee08f..9290b51df102d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.ll
@@ -11,7 +11,7 @@ main_body:
   %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 0, i32 0, i32 0)
   %data_glc = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 0, i32 0, i32 1)
   %data_slc = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 0, i32 0, i32 2)
-  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %data, 0
+  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %data, 0
   %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %data_glc, 1
   %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %data_slc, 2
   ret {<4 x float>, <4 x float>, <4 x float>} %r2
@@ -138,7 +138,7 @@ main_body:
   %fdata = bitcast <4 x i32> %data to <4 x float>
   %fdata_glc = bitcast <2 x i32> %data_glc to <2 x float>
   %fdata_slc = bitcast i32 %data_slc to float
-  %r0 = insertvalue {<4 x float>, <2 x float>, float} undef, <4 x float> %fdata, 0
+  %r0 = insertvalue {<4 x float>, <2 x float>, float} poison, <4 x float> %fdata, 0
   %r1 = insertvalue {<4 x float>, <2 x float>, float} %r0, <2 x float> %fdata_glc, 1
   %r2 = insertvalue {<4 x float>, <2 x float>, float} %r1, float %fdata_slc, 2
   ret {<4 x float>, <2 x float>, float} %r2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.format.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.format.d16.ll
index 73ac61a77a94e..aa7064dad9e95 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.format.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.format.d16.ll
@@ -47,7 +47,7 @@ main_body:
 ; PACKED: buffer_store_format_d16_xyz v[[[LO]]:[[HI]]], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 idxen
 define amdgpu_kernel void @buffer_store_format_d16_xyz(<4 x i32> %rsrc, <4 x half> %data, i32 %index) {
 main_body:
-  %data_subvec = shufflevector <4 x half> %data, <4 x half> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  %data_subvec = shufflevector <4 x half> %data, <4 x half> poison, <3 x i32> <i32 0, i32 1, i32 2>
   call void @llvm.amdgcn.struct.buffer.store.format.v3f16(<3 x half> %data_subvec, <4 x i32> %rsrc, i32 %index, i32 0, i32 0, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll
index c5202b84fa1ef..5b73d58287146 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll
@@ -46,7 +46,7 @@ main_body:
   %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0, i32 0)
   %data_glc = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0, i32 1)
   %data_slc = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0, i32 2)
-  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %data, 0
+  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %data, 0
   %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %data_glc, 1
   %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %data_slc, 2
   ret {<4 x float>, <4 x float>, <4 x float>} %r2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.ll
index 55600cab8432b..bfbc765a0fbc5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.ll
@@ -15,7 +15,7 @@ main_body:
   %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0, i32 0)
   %data_glc = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0, i32 1)
   %data_slc = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0, i32 2)
-  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %data, 0
+  %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %data, 0
   %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %data_glc, 1
   %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %data_slc, 2
   ret {<4 x float>, <4 x float>, <4 x float>} %r2
@@ -204,7 +204,7 @@ main_body:
   %fdata = bitcast <4 x i32> %data to <4 x float>
   %fdata_glc = bitcast <2 x i32> %data_glc to <2 x float>
   %fdata_slc = bitcast i32 %data_slc to float
-  %r0 = insertvalue {<4 x float>, <2 x float>, float} undef, <4 x float> %fdata, 0
+  %r0 = insertvalue {<4 x float>, <2 x float>, float} poison, <4 x float> %fdata, 0
   %r1 = insertvalue {<4 x float>, <2 x float>, float} %r0, <2 x float> %fdata_glc, 1
   %r2 = insertvalue {<4 x float>, <2 x float>, float} %r1, float %fdata_slc, 2
   ret {<4 x float>, <2 x float>, float} %r2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.store.format.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.store.format.d16.ll
index 8a281376965d1..51d3687ac5185 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.store.format.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.store.format.d16.ll
@@ -77,7 +77,7 @@ define amdgpu_kernel void @buffer_store_format_d16_xyz(ptr addrspace(8) %rsrc, <
 ; PACKED-NEXT:    buffer_store_format_d16_xyz v[0:1], v2, s[0:3], 0 idxen
 ; PACKED-NEXT:    s_endpgm
 main_body:
-  %data_subvec = shufflevector <4 x half> %data, <4 x half> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  %data_subvec = shufflevector <4 x half> %data, <4 x half> poison, <3 x i32> <i32 0, i32 1, i32 2>
   call void @llvm.amdgcn.struct.ptr.buffer.store.format.v3f16(<3 x half> %data_subvec, ptr addrspace(8) %rsrc, i32 %index, i32 0, i32 0, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.load.ll
index 84f818a168a4f..b144e3779fe27 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.load.ll
@@ -44,7 +44,7 @@ main_body:
     %vdata.f     = bitcast <4 x i32> %vdata to <4 x float>
     %vdata_glc.f = bitcast <4 x i32> %vdata_glc to <4 x float>
     %vdata_slc.f = bitcast <4 x i32> %vdata_slc to <4 x float>
-    %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %vdata.f, 0
+    %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %vdata.f, 0
     %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %vdata_glc.f, 1
     %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %vdata_slc.f, 2
     %r3 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} %r2, <4 x float> %vdata_f32, 3
@@ -113,7 +113,7 @@ define amdgpu_vs {<4 x float>, <4 x float>, <4 x float>} @tbuffer_load_immoffs_l
     %vdata.f     = bitcast <4 x i32> %vdata to <4 x float>
     %vdata_glc.f = bitcast <4 x i32> %vdata_glc to <4 x float>
     %vdata_slc.f = bitcast <4 x i32> %vdata_slc to <4 x float>
-    %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %vdata.f, 0
+    %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %vdata.f, 0
     %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %vdata_glc.f, 1
     %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %vdata_slc.f, 2
     ret {<4 x float>, <4 x float>, <4 x float>} %r2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.store.d16.ll
index e5eae03bb5bde..7e92f0d2c5973 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.store.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.store.d16.ll
@@ -160,7 +160,7 @@ define amdgpu_kernel void @tbuffer_store_d16_xyz(ptr addrspace(8) %rsrc, <4 x ha
 ; GFX11-PACKED-NEXT:    tbuffer_store_d16_format_xyz v[0:1], v2, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM] idxen
 ; GFX11-PACKED-NEXT:    s_endpgm
 main_body:
-  %data_subvec = shufflevector <4 x half> %data, <4 x half> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  %data_subvec = shufflevector <4 x half> %data, <4 x half> poison, <3 x i32> <i32 0, i32 1, i32 2>
   call void @llvm.amdgcn.struct.ptr.tbuffer.store.v3f16(<3 x half> %data_subvec, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0, i32 33, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.ll
index 0c0f50bd49557..04539ffe4a12c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.ll
@@ -57,7 +57,7 @@ main_body:
     %vdata.f     = bitcast <4 x i32> %vdata to <4 x float>
     %vdata_glc.f = bitcast <4 x i32> %vdata_glc to <4 x float>
     %vdata_slc.f = bitcast <4 x i32> %vdata_slc to <4 x float>
-    %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %vdata.f, 0
+    %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %vdata.f, 0
     %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %vdata_glc.f, 1
     %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %vdata_slc.f, 2
     %r3 = insertvalue {<4 x float>, <4 x float>, <4 x float>, <4 x float>} %r2, <4 x float> %vdata_f32, 3
@@ -144,7 +144,7 @@ define amdgpu_vs {<4 x float>, <4 x float>, <4 x float>} @tbuffer_load_immoffs_l
     %vdata.f     = bitcast <4 x i32> %vdata to <4 x float>
     %vdata_glc.f = bitcast <4 x i32> %vdata_glc to <4 x float>
     %vdata_slc.f = bitcast <4 x i32> %vdata_slc to <4 x float>
-    %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} undef, <4 x float> %vdata.f, 0
+    %r0 = insertvalue {<4 x float>, <4 x float>, <4 x float>} poison, <4 x float> %vdata.f, 0
     %r1 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r0, <4 x float> %vdata_glc.f, 1
     %r2 = insertvalue {<4 x float>, <4 x float>, <4 x float>} %r1, <4 x float> %vdata_slc.f, 2
     ret {<4 x float>, <4 x float>, <4 x float>} %r2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll
index 17ebb1a835462..4a02973941ec5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll
@@ -211,7 +211,7 @@ define amdgpu_kernel void @tbuffer_store_d16_xyz(<4 x i32> %rsrc, <4 x half> %da
 ; GFX12-PACKED-GISEL-NEXT:    tbuffer_store_d16_format_xyzw v[0:1], v2, s[0:3], null format:[BUF_FMT_10_10_10_2_SNORM] idxen
 ; GFX12-PACKED-GISEL-NEXT:    s_endpgm
 main_body:
-  %data_subvec = shufflevector <4 x half> %data, <4 x half> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  %data_subvec = shufflevector <4 x half> %data, <4 x half> poison, <3 x i32> <i32 0, i32 1, i32 2>
   call void @llvm.amdgcn.struct.tbuffer.store.v3f16(<3 x half> %data_subvec, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 33, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.udot2.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.udot2.ll
index 2346cf0ba6303..33ef08273c8b5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.udot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.udot2.ll
@@ -58,7 +58,7 @@ entry:
   %b.val = load <2 x i16>, ptr addrspace(1) %b.gep
   %b.elt0 = extractelement <2 x i16> %b.val, i32 0
   %b.elt1 = extractelement <2 x i16> %b.val, i32 1
-  %b0 = insertelement <2 x i16> undef, i16 %b.elt1, i32 0
+  %b0 = insertelement <2 x i16> poison, i16 %b.elt1, i32 0
   %b1 = insertelement <2 x i16> %b0, i16 %b.elt0, i32 1
   %r.val = call i32 @llvm.amdgcn.udot2(<2 x i16> <i16 1, i16 1>, <2 x i16> %b1, i32 %c, i1 0)
   store i32 %r.val, ptr addrspace(1) %r
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f32.ll
index 584dd2700c419..8b1ba393c8de8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f32.ll
@@ -1727,14 +1727,9 @@ define <16 x float> @v_maximum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX7-LABEL: v_maximum_v16f32:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX7-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; GFX7-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v17
 ; GFX7-NEXT:    v_max_f32_e32 v1, v1, v17
 ; GFX7-NEXT:    buffer_load_dword v17, off, s[0:3], s32
-; GFX7-NEXT:    v_writelane_b32 v31, s30, 0
-; GFX7-NEXT:    v_writelane_b32 v31, s31, 1
 ; GFX7-NEXT:    v_cmp_o_f32_e64 s[4:5], v2, v18
 ; GFX7-NEXT:    v_max_f32_e32 v2, v2, v18
 ; GFX7-NEXT:    v_cmp_o_f32_e64 s[6:7], v3, v19
@@ -1743,7 +1738,7 @@ define <16 x float> @v_maximum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX7-NEXT:    v_max_f32_e32 v19, v0, v16
 ; GFX7-NEXT:    v_cmp_o_f32_e64 s[28:29], v0, v16
 ; GFX7-NEXT:    v_max_f32_e32 v16, v14, v30
-; GFX7-NEXT:    v_cmp_o_f32_e64 s[30:31], v14, v30
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[40:41], v14, v30
 ; GFX7-NEXT:    v_cmp_o_f32_e64 s[8:9], v4, v20
 ; GFX7-NEXT:    v_max_f32_e32 v4, v4, v20
 ; GFX7-NEXT:    v_cmp_o_f32_e64 s[10:11], v5, v21
@@ -1765,7 +1760,7 @@ define <16 x float> @v_maximum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX7-NEXT:    v_cmp_o_f32_e64 s[26:27], v13, v29
 ; GFX7-NEXT:    v_max_f32_e32 v13, v13, v29
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v18, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v14, v18, v16, s[30:31]
+; GFX7-NEXT:    v_cndmask_b32_e64 v14, v18, v16, s[40:41]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v0, v18, v19, s[28:29]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s[4:5]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v3, v18, v3, s[6:7]
@@ -1779,29 +1774,18 @@ define <16 x float> @v_maximum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX7-NEXT:    v_cndmask_b32_e64 v11, v18, v11, s[22:23]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v12, v18, v12, s[24:25]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v13, v18, v13, s[26:27]
-; GFX7-NEXT:    v_readlane_b32 s31, v31, 1
-; GFX7-NEXT:    v_readlane_b32 s30, v31, 0
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    v_max_f32_e32 v16, v15, v17
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v15, v17
 ; GFX7-NEXT:    v_cndmask_b32_e32 v15, v18, v16, vcc
-; GFX7-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX7-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v16f32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v1, v17
 ; GFX8-NEXT:    v_max_f32_e32 v1, v1, v17
 ; GFX8-NEXT:    buffer_load_dword v17, off, s[0:3], s32
-; GFX8-NEXT:    v_writelane_b32 v31, s30, 0
-; GFX8-NEXT:    v_writelane_b32 v31, s31, 1
 ; GFX8-NEXT:    v_cmp_o_f32_e64 s[4:5], v2, v18
 ; GFX8-NEXT:    v_max_f32_e32 v2, v2, v18
 ; GFX8-NEXT:    v_cmp_o_f32_e64 s[6:7], v3, v19
@@ -1810,7 +1794,7 @@ define <16 x float> @v_maximum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX8-NEXT:    v_max_f32_e32 v19, v0, v16
 ; GFX8-NEXT:    v_cmp_o_f32_e64 s[28:29], v0, v16
 ; GFX8-NEXT:    v_max_f32_e32 v16, v14, v30
-; GFX8-NEXT:    v_cmp_o_f32_e64 s[30:31], v14, v30
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[40:41], v14, v30
 ; GFX8-NEXT:    v_cmp_o_f32_e64 s[8:9], v4, v20
 ; GFX8-NEXT:    v_max_f32_e32 v4, v4, v20
 ; GFX8-NEXT:    v_cmp_o_f32_e64 s[10:11], v5, v21
@@ -1832,7 +1816,7 @@ define <16 x float> @v_maximum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX8-NEXT:    v_cmp_o_f32_e64 s[26:27], v13, v29
 ; GFX8-NEXT:    v_max_f32_e32 v13, v13, v29
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v18, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v14, v18, v16, s[30:31]
+; GFX8-NEXT:    v_cndmask_b32_e64 v14, v18, v16, s[40:41]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v0, v18, v19, s[28:29]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s[4:5]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v3, v18, v3, s[6:7]
@@ -1846,29 +1830,18 @@ define <16 x float> @v_maximum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX8-NEXT:    v_cndmask_b32_e64 v11, v18, v11, s[22:23]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v12, v18, v12, s[24:25]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v13, v18, v13, s[26:27]
-; GFX8-NEXT:    v_readlane_b32 s31, v31, 1
-; GFX8-NEXT:    v_readlane_b32 s30, v31, 0
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    v_max_f32_e32 v16, v15, v17
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v15, v17
 ; GFX8-NEXT:    v_cndmask_b32_e32 v15, v18, v16, vcc
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_maximum_v16f32:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX900-NEXT:    v_cmp_o_f32_e32 vcc, v1, v17
 ; GFX900-NEXT:    v_max_f32_e32 v1, v1, v17
 ; GFX900-NEXT:    buffer_load_dword v17, off, s[0:3], s32
-; GFX900-NEXT:    v_writelane_b32 v31, s30, 0
-; GFX900-NEXT:    v_writelane_b32 v31, s31, 1
 ; GFX900-NEXT:    v_cmp_o_f32_e64 s[4:5], v2, v18
 ; GFX900-NEXT:    v_max_f32_e32 v2, v2, v18
 ; GFX900-NEXT:    v_cmp_o_f32_e64 s[6:7], v3, v19
@@ -1877,7 +1850,7 @@ define <16 x float> @v_maximum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX900-NEXT:    v_max_f32_e32 v19, v0, v16
 ; GFX900-NEXT:    v_cmp_o_f32_e64 s[28:29], v0, v16
 ; GFX900-NEXT:    v_max_f32_e32 v16, v14, v30
-; GFX900-NEXT:    v_cmp_o_f32_e64 s[30:31], v14, v30
+; GFX900-NEXT:    v_cmp_o_f32_e64 s[40:41], v14, v30
 ; GFX900-NEXT:    v_cmp_o_f32_e64 s[8:9], v4, v20
 ; GFX900-NEXT:    v_max_f32_e32 v4, v4, v20
 ; GFX900-NEXT:    v_cmp_o_f32_e64 s[10:11], v5, v21
@@ -1899,7 +1872,7 @@ define <16 x float> @v_maximum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX900-NEXT:    v_cmp_o_f32_e64 s[26:27], v13, v29
 ; GFX900-NEXT:    v_max_f32_e32 v13, v13, v29
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v18, v1, vcc
-; GFX900-NEXT:    v_cndmask_b32_e64 v14, v18, v16, s[30:31]
+; GFX900-NEXT:    v_cndmask_b32_e64 v14, v18, v16, s[40:41]
 ; GFX900-NEXT:    v_cndmask_b32_e64 v0, v18, v19, s[28:29]
 ; GFX900-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s[4:5]
 ; GFX900-NEXT:    v_cndmask_b32_e64 v3, v18, v3, s[6:7]
@@ -1913,16 +1886,10 @@ define <16 x float> @v_maximum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX900-NEXT:    v_cndmask_b32_e64 v11, v18, v11, s[22:23]
 ; GFX900-NEXT:    v_cndmask_b32_e64 v12, v18, v12, s[24:25]
 ; GFX900-NEXT:    v_cndmask_b32_e64 v13, v18, v13, s[26:27]
-; GFX900-NEXT:    v_readlane_b32 s31, v31, 1
-; GFX900-NEXT:    v_readlane_b32 s30, v31, 0
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    v_max_f32_e32 v16, v15, v17
 ; GFX900-NEXT:    v_cmp_o_f32_e32 vcc, v15, v17
 ; GFX900-NEXT:    v_cndmask_b32_e32 v15, v18, v16, vcc
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX950-LABEL: v_maximum_v16f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll
index e354ec6fb3dd7..3344c73f9eb6f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f64.ll
@@ -2008,15 +2008,8 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX7-LABEL: v_maximum_v16f64:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX7-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; GFX7-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:8
 ; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX7-NEXT:    v_writelane_b32 v34, s30, 0
-; GFX7-NEXT:    v_writelane_b32 v34, s31, 1
-; GFX7-NEXT:    v_writelane_b32 v34, s34, 2
-; GFX7-NEXT:    v_writelane_b32 v34, s35, 3
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[31:32]
 ; GFX7-NEXT:    v_max_f64 v[0:1], v[0:1], v[31:32]
@@ -2102,14 +2095,14 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:116
 ; GFX7-NEXT:    v_cndmask_b32_e64 v26, v26, 0, s[28:29]
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_cmp_u_f64_e64 s[30:31], v[28:29], v[31:32]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[40:41], v[28:29], v[31:32]
 ; GFX7-NEXT:    v_max_f64 v[28:29], v[28:29], v[31:32]
 ; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX7-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
 ; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:124
-; GFX7-NEXT:    v_cndmask_b32_e64 v28, v28, 0, s[30:31]
+; GFX7-NEXT:    v_cndmask_b32_e64 v28, v28, 0, s[40:41]
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_cmp_u_f64_e64 s[34:35], v[30:31], v[32:33]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[42:43], v[30:31], v[32:33]
 ; GFX7-NEXT:    v_max_f64 v[30:31], v[30:31], v[32:33]
 ; GFX7-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v32, vcc
@@ -2126,31 +2119,16 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX7-NEXT:    v_cndmask_b32_e64 v23, v23, v32, s[24:25]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v25, v25, v32, s[26:27]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v27, v27, v32, s[28:29]
-; GFX7-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[30:31]
-; GFX7-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[34:35]
-; GFX7-NEXT:    v_cndmask_b32_e64 v30, v30, 0, s[34:35]
-; GFX7-NEXT:    v_readlane_b32 s35, v34, 3
-; GFX7-NEXT:    v_readlane_b32 s34, v34, 2
-; GFX7-NEXT:    v_readlane_b32 s31, v34, 1
-; GFX7-NEXT:    v_readlane_b32 s30, v34, 0
-; GFX7-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX7-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX7-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[40:41]
+; GFX7-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[42:43]
+; GFX7-NEXT:    v_cndmask_b32_e64 v30, v30, 0, s[42:43]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_maximum_v16f64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:8
 ; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX8-NEXT:    v_writelane_b32 v34, s30, 0
-; GFX8-NEXT:    v_writelane_b32 v34, s31, 1
-; GFX8-NEXT:    v_writelane_b32 v34, s34, 2
-; GFX8-NEXT:    v_writelane_b32 v34, s35, 3
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[31:32]
 ; GFX8-NEXT:    v_max_f64 v[0:1], v[0:1], v[31:32]
@@ -2236,14 +2214,14 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:116
 ; GFX8-NEXT:    v_cndmask_b32_e64 v26, v26, 0, s[28:29]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_cmp_u_f64_e64 s[30:31], v[28:29], v[31:32]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[40:41], v[28:29], v[31:32]
 ; GFX8-NEXT:    v_max_f64 v[28:29], v[28:29], v[31:32]
 ; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX8-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
 ; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:124
-; GFX8-NEXT:    v_cndmask_b32_e64 v28, v28, 0, s[30:31]
+; GFX8-NEXT:    v_cndmask_b32_e64 v28, v28, 0, s[40:41]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_cmp_u_f64_e64 s[34:35], v[30:31], v[32:33]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[42:43], v[30:31], v[32:33]
 ; GFX8-NEXT:    v_max_f64 v[30:31], v[30:31], v[32:33]
 ; GFX8-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v32, vcc
@@ -2260,31 +2238,16 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX8-NEXT:    v_cndmask_b32_e64 v23, v23, v32, s[24:25]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v25, v25, v32, s[26:27]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v27, v27, v32, s[28:29]
-; GFX8-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[30:31]
-; GFX8-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[34:35]
-; GFX8-NEXT:    v_cndmask_b32_e64 v30, v30, 0, s[34:35]
-; GFX8-NEXT:    v_readlane_b32 s35, v34, 3
-; GFX8-NEXT:    v_readlane_b32 s34, v34, 2
-; GFX8-NEXT:    v_readlane_b32 s31, v34, 1
-; GFX8-NEXT:    v_readlane_b32 s30, v34, 0
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[40:41]
+; GFX8-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[42:43]
+; GFX8-NEXT:    v_cndmask_b32_e64 v30, v30, 0, s[42:43]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_maximum_v16f64:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX900-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:8
 ; GFX900-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX900-NEXT:    v_writelane_b32 v34, s30, 0
-; GFX900-NEXT:    v_writelane_b32 v34, s31, 1
-; GFX900-NEXT:    v_writelane_b32 v34, s34, 2
-; GFX900-NEXT:    v_writelane_b32 v34, s35, 3
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[31:32]
 ; GFX900-NEXT:    v_max_f64 v[0:1], v[0:1], v[31:32]
@@ -2370,14 +2333,14 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX900-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:116
 ; GFX900-NEXT:    v_cndmask_b32_e64 v26, v26, 0, s[28:29]
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
-; GFX900-NEXT:    v_cmp_u_f64_e64 s[30:31], v[28:29], v[31:32]
+; GFX900-NEXT:    v_cmp_u_f64_e64 s[40:41], v[28:29], v[31:32]
 ; GFX900-NEXT:    v_max_f64 v[28:29], v[28:29], v[31:32]
 ; GFX900-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX900-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
 ; GFX900-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:124
-; GFX900-NEXT:    v_cndmask_b32_e64 v28, v28, 0, s[30:31]
+; GFX900-NEXT:    v_cndmask_b32_e64 v28, v28, 0, s[40:41]
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
-; GFX900-NEXT:    v_cmp_u_f64_e64 s[34:35], v[30:31], v[32:33]
+; GFX900-NEXT:    v_cmp_u_f64_e64 s[42:43], v[30:31], v[32:33]
 ; GFX900-NEXT:    v_max_f64 v[30:31], v[30:31], v[32:33]
 ; GFX900-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v32, vcc
@@ -2394,17 +2357,9 @@ define <16 x double> @v_maximum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX900-NEXT:    v_cndmask_b32_e64 v23, v23, v32, s[24:25]
 ; GFX900-NEXT:    v_cndmask_b32_e64 v25, v25, v32, s[26:27]
 ; GFX900-NEXT:    v_cndmask_b32_e64 v27, v27, v32, s[28:29]
-; GFX900-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[30:31]
-; GFX900-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[34:35]
-; GFX900-NEXT:    v_cndmask_b32_e64 v30, v30, 0, s[34:35]
-; GFX900-NEXT:    v_readlane_b32 s35, v34, 3
-; GFX900-NEXT:    v_readlane_b32 s34, v34, 2
-; GFX900-NEXT:    v_readlane_b32 s31, v34, 1
-; GFX900-NEXT:    v_readlane_b32 s30, v34, 0
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[40:41]
+; GFX900-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[42:43]
+; GFX900-NEXT:    v_cndmask_b32_e64 v30, v30, 0, s[42:43]
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX950-LABEL: v_maximum_v16f64:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f32.ll
index 9962433134073..7b2998cbd242f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f32.ll
@@ -1727,14 +1727,9 @@ define <16 x float> @v_minimum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX7-LABEL: v_minimum_v16f32:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX7-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; GFX7-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v1, v17
 ; GFX7-NEXT:    v_min_f32_e32 v1, v1, v17
 ; GFX7-NEXT:    buffer_load_dword v17, off, s[0:3], s32
-; GFX7-NEXT:    v_writelane_b32 v31, s30, 0
-; GFX7-NEXT:    v_writelane_b32 v31, s31, 1
 ; GFX7-NEXT:    v_cmp_o_f32_e64 s[4:5], v2, v18
 ; GFX7-NEXT:    v_min_f32_e32 v2, v2, v18
 ; GFX7-NEXT:    v_cmp_o_f32_e64 s[6:7], v3, v19
@@ -1743,7 +1738,7 @@ define <16 x float> @v_minimum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX7-NEXT:    v_min_f32_e32 v19, v0, v16
 ; GFX7-NEXT:    v_cmp_o_f32_e64 s[28:29], v0, v16
 ; GFX7-NEXT:    v_min_f32_e32 v16, v14, v30
-; GFX7-NEXT:    v_cmp_o_f32_e64 s[30:31], v14, v30
+; GFX7-NEXT:    v_cmp_o_f32_e64 s[40:41], v14, v30
 ; GFX7-NEXT:    v_cmp_o_f32_e64 s[8:9], v4, v20
 ; GFX7-NEXT:    v_min_f32_e32 v4, v4, v20
 ; GFX7-NEXT:    v_cmp_o_f32_e64 s[10:11], v5, v21
@@ -1765,7 +1760,7 @@ define <16 x float> @v_minimum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX7-NEXT:    v_cmp_o_f32_e64 s[26:27], v13, v29
 ; GFX7-NEXT:    v_min_f32_e32 v13, v13, v29
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v18, v1, vcc
-; GFX7-NEXT:    v_cndmask_b32_e64 v14, v18, v16, s[30:31]
+; GFX7-NEXT:    v_cndmask_b32_e64 v14, v18, v16, s[40:41]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v0, v18, v19, s[28:29]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s[4:5]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v3, v18, v3, s[6:7]
@@ -1779,29 +1774,18 @@ define <16 x float> @v_minimum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX7-NEXT:    v_cndmask_b32_e64 v11, v18, v11, s[22:23]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v12, v18, v12, s[24:25]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v13, v18, v13, s[26:27]
-; GFX7-NEXT:    v_readlane_b32 s31, v31, 1
-; GFX7-NEXT:    v_readlane_b32 s30, v31, 0
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    v_min_f32_e32 v16, v15, v17
 ; GFX7-NEXT:    v_cmp_o_f32_e32 vcc, v15, v17
 ; GFX7-NEXT:    v_cndmask_b32_e32 v15, v18, v16, vcc
-; GFX7-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX7-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v16f32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v1, v17
 ; GFX8-NEXT:    v_min_f32_e32 v1, v1, v17
 ; GFX8-NEXT:    buffer_load_dword v17, off, s[0:3], s32
-; GFX8-NEXT:    v_writelane_b32 v31, s30, 0
-; GFX8-NEXT:    v_writelane_b32 v31, s31, 1
 ; GFX8-NEXT:    v_cmp_o_f32_e64 s[4:5], v2, v18
 ; GFX8-NEXT:    v_min_f32_e32 v2, v2, v18
 ; GFX8-NEXT:    v_cmp_o_f32_e64 s[6:7], v3, v19
@@ -1810,7 +1794,7 @@ define <16 x float> @v_minimum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX8-NEXT:    v_min_f32_e32 v19, v0, v16
 ; GFX8-NEXT:    v_cmp_o_f32_e64 s[28:29], v0, v16
 ; GFX8-NEXT:    v_min_f32_e32 v16, v14, v30
-; GFX8-NEXT:    v_cmp_o_f32_e64 s[30:31], v14, v30
+; GFX8-NEXT:    v_cmp_o_f32_e64 s[40:41], v14, v30
 ; GFX8-NEXT:    v_cmp_o_f32_e64 s[8:9], v4, v20
 ; GFX8-NEXT:    v_min_f32_e32 v4, v4, v20
 ; GFX8-NEXT:    v_cmp_o_f32_e64 s[10:11], v5, v21
@@ -1832,7 +1816,7 @@ define <16 x float> @v_minimum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX8-NEXT:    v_cmp_o_f32_e64 s[26:27], v13, v29
 ; GFX8-NEXT:    v_min_f32_e32 v13, v13, v29
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v18, v1, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v14, v18, v16, s[30:31]
+; GFX8-NEXT:    v_cndmask_b32_e64 v14, v18, v16, s[40:41]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v0, v18, v19, s[28:29]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s[4:5]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v3, v18, v3, s[6:7]
@@ -1846,29 +1830,18 @@ define <16 x float> @v_minimum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX8-NEXT:    v_cndmask_b32_e64 v11, v18, v11, s[22:23]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v12, v18, v12, s[24:25]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v13, v18, v13, s[26:27]
-; GFX8-NEXT:    v_readlane_b32 s31, v31, 1
-; GFX8-NEXT:    v_readlane_b32 s30, v31, 0
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    v_min_f32_e32 v16, v15, v17
 ; GFX8-NEXT:    v_cmp_o_f32_e32 vcc, v15, v17
 ; GFX8-NEXT:    v_cndmask_b32_e32 v15, v18, v16, vcc
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_minimum_v16f32:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX900-NEXT:    v_cmp_o_f32_e32 vcc, v1, v17
 ; GFX900-NEXT:    v_min_f32_e32 v1, v1, v17
 ; GFX900-NEXT:    buffer_load_dword v17, off, s[0:3], s32
-; GFX900-NEXT:    v_writelane_b32 v31, s30, 0
-; GFX900-NEXT:    v_writelane_b32 v31, s31, 1
 ; GFX900-NEXT:    v_cmp_o_f32_e64 s[4:5], v2, v18
 ; GFX900-NEXT:    v_min_f32_e32 v2, v2, v18
 ; GFX900-NEXT:    v_cmp_o_f32_e64 s[6:7], v3, v19
@@ -1877,7 +1850,7 @@ define <16 x float> @v_minimum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX900-NEXT:    v_min_f32_e32 v19, v0, v16
 ; GFX900-NEXT:    v_cmp_o_f32_e64 s[28:29], v0, v16
 ; GFX900-NEXT:    v_min_f32_e32 v16, v14, v30
-; GFX900-NEXT:    v_cmp_o_f32_e64 s[30:31], v14, v30
+; GFX900-NEXT:    v_cmp_o_f32_e64 s[40:41], v14, v30
 ; GFX900-NEXT:    v_cmp_o_f32_e64 s[8:9], v4, v20
 ; GFX900-NEXT:    v_min_f32_e32 v4, v4, v20
 ; GFX900-NEXT:    v_cmp_o_f32_e64 s[10:11], v5, v21
@@ -1899,7 +1872,7 @@ define <16 x float> @v_minimum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX900-NEXT:    v_cmp_o_f32_e64 s[26:27], v13, v29
 ; GFX900-NEXT:    v_min_f32_e32 v13, v13, v29
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v18, v1, vcc
-; GFX900-NEXT:    v_cndmask_b32_e64 v14, v18, v16, s[30:31]
+; GFX900-NEXT:    v_cndmask_b32_e64 v14, v18, v16, s[40:41]
 ; GFX900-NEXT:    v_cndmask_b32_e64 v0, v18, v19, s[28:29]
 ; GFX900-NEXT:    v_cndmask_b32_e64 v2, v18, v2, s[4:5]
 ; GFX900-NEXT:    v_cndmask_b32_e64 v3, v18, v3, s[6:7]
@@ -1913,16 +1886,10 @@ define <16 x float> @v_minimum_v16f32(<16 x float> %src0, <16 x float> %src1) {
 ; GFX900-NEXT:    v_cndmask_b32_e64 v11, v18, v11, s[22:23]
 ; GFX900-NEXT:    v_cndmask_b32_e64 v12, v18, v12, s[24:25]
 ; GFX900-NEXT:    v_cndmask_b32_e64 v13, v18, v13, s[26:27]
-; GFX900-NEXT:    v_readlane_b32 s31, v31, 1
-; GFX900-NEXT:    v_readlane_b32 s30, v31, 0
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    v_min_f32_e32 v16, v15, v17
 ; GFX900-NEXT:    v_cmp_o_f32_e32 vcc, v15, v17
 ; GFX900-NEXT:    v_cndmask_b32_e32 v15, v18, v16, vcc
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX950-LABEL: v_minimum_v16f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll
index 71fdd691a1512..1d1673315f6ff 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f64.ll
@@ -2008,15 +2008,8 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX7-LABEL: v_minimum_v16f64:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX7-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; GFX7-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:8
 ; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX7-NEXT:    v_writelane_b32 v34, s30, 0
-; GFX7-NEXT:    v_writelane_b32 v34, s31, 1
-; GFX7-NEXT:    v_writelane_b32 v34, s34, 2
-; GFX7-NEXT:    v_writelane_b32 v34, s35, 3
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[31:32]
 ; GFX7-NEXT:    v_min_f64 v[0:1], v[0:1], v[31:32]
@@ -2102,14 +2095,14 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:116
 ; GFX7-NEXT:    v_cndmask_b32_e64 v26, v26, 0, s[28:29]
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_cmp_u_f64_e64 s[30:31], v[28:29], v[31:32]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[40:41], v[28:29], v[31:32]
 ; GFX7-NEXT:    v_min_f64 v[28:29], v[28:29], v[31:32]
 ; GFX7-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX7-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
 ; GFX7-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:124
-; GFX7-NEXT:    v_cndmask_b32_e64 v28, v28, 0, s[30:31]
+; GFX7-NEXT:    v_cndmask_b32_e64 v28, v28, 0, s[40:41]
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_cmp_u_f64_e64 s[34:35], v[30:31], v[32:33]
+; GFX7-NEXT:    v_cmp_u_f64_e64 s[42:43], v[30:31], v[32:33]
 ; GFX7-NEXT:    v_min_f64 v[30:31], v[30:31], v[32:33]
 ; GFX7-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v32, vcc
@@ -2126,31 +2119,16 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX7-NEXT:    v_cndmask_b32_e64 v23, v23, v32, s[24:25]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v25, v25, v32, s[26:27]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v27, v27, v32, s[28:29]
-; GFX7-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[30:31]
-; GFX7-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[34:35]
-; GFX7-NEXT:    v_cndmask_b32_e64 v30, v30, 0, s[34:35]
-; GFX7-NEXT:    v_readlane_b32 s35, v34, 3
-; GFX7-NEXT:    v_readlane_b32 s34, v34, 2
-; GFX7-NEXT:    v_readlane_b32 s31, v34, 1
-; GFX7-NEXT:    v_readlane_b32 s30, v34, 0
-; GFX7-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX7-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX7-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[40:41]
+; GFX7-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[42:43]
+; GFX7-NEXT:    v_cndmask_b32_e64 v30, v30, 0, s[42:43]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_minimum_v16f64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:8
 ; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX8-NEXT:    v_writelane_b32 v34, s30, 0
-; GFX8-NEXT:    v_writelane_b32 v34, s31, 1
-; GFX8-NEXT:    v_writelane_b32 v34, s34, 2
-; GFX8-NEXT:    v_writelane_b32 v34, s35, 3
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[31:32]
 ; GFX8-NEXT:    v_min_f64 v[0:1], v[0:1], v[31:32]
@@ -2236,14 +2214,14 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:116
 ; GFX8-NEXT:    v_cndmask_b32_e64 v26, v26, 0, s[28:29]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_cmp_u_f64_e64 s[30:31], v[28:29], v[31:32]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[40:41], v[28:29], v[31:32]
 ; GFX8-NEXT:    v_min_f64 v[28:29], v[28:29], v[31:32]
 ; GFX8-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX8-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
 ; GFX8-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:124
-; GFX8-NEXT:    v_cndmask_b32_e64 v28, v28, 0, s[30:31]
+; GFX8-NEXT:    v_cndmask_b32_e64 v28, v28, 0, s[40:41]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_cmp_u_f64_e64 s[34:35], v[30:31], v[32:33]
+; GFX8-NEXT:    v_cmp_u_f64_e64 s[42:43], v[30:31], v[32:33]
 ; GFX8-NEXT:    v_min_f64 v[30:31], v[30:31], v[32:33]
 ; GFX8-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v32, vcc
@@ -2260,31 +2238,16 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX8-NEXT:    v_cndmask_b32_e64 v23, v23, v32, s[24:25]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v25, v25, v32, s[26:27]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v27, v27, v32, s[28:29]
-; GFX8-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[30:31]
-; GFX8-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[34:35]
-; GFX8-NEXT:    v_cndmask_b32_e64 v30, v30, 0, s[34:35]
-; GFX8-NEXT:    v_readlane_b32 s35, v34, 3
-; GFX8-NEXT:    v_readlane_b32 s34, v34, 2
-; GFX8-NEXT:    v_readlane_b32 s31, v34, 1
-; GFX8-NEXT:    v_readlane_b32 s30, v34, 0
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[40:41]
+; GFX8-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[42:43]
+; GFX8-NEXT:    v_cndmask_b32_e64 v30, v30, 0, s[42:43]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: v_minimum_v16f64:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    buffer_store_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX900-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:8
 ; GFX900-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX900-NEXT:    v_writelane_b32 v34, s30, 0
-; GFX900-NEXT:    v_writelane_b32 v34, s31, 1
-; GFX900-NEXT:    v_writelane_b32 v34, s34, 2
-; GFX900-NEXT:    v_writelane_b32 v34, s35, 3
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    v_cmp_u_f64_e32 vcc, v[0:1], v[31:32]
 ; GFX900-NEXT:    v_min_f64 v[0:1], v[0:1], v[31:32]
@@ -2370,14 +2333,14 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX900-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:116
 ; GFX900-NEXT:    v_cndmask_b32_e64 v26, v26, 0, s[28:29]
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
-; GFX900-NEXT:    v_cmp_u_f64_e64 s[30:31], v[28:29], v[31:32]
+; GFX900-NEXT:    v_cmp_u_f64_e64 s[40:41], v[28:29], v[31:32]
 ; GFX900-NEXT:    v_min_f64 v[28:29], v[28:29], v[31:32]
 ; GFX900-NEXT:    buffer_load_dword v31, off, s[0:3], s32
 ; GFX900-NEXT:    buffer_load_dword v33, off, s[0:3], s32 offset:128
 ; GFX900-NEXT:    buffer_load_dword v32, off, s[0:3], s32 offset:124
-; GFX900-NEXT:    v_cndmask_b32_e64 v28, v28, 0, s[30:31]
+; GFX900-NEXT:    v_cndmask_b32_e64 v28, v28, 0, s[40:41]
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
-; GFX900-NEXT:    v_cmp_u_f64_e64 s[34:35], v[30:31], v[32:33]
+; GFX900-NEXT:    v_cmp_u_f64_e64 s[42:43], v[30:31], v[32:33]
 ; GFX900-NEXT:    v_min_f64 v[30:31], v[30:31], v[32:33]
 ; GFX900-NEXT:    v_mov_b32_e32 v32, 0x7ff80000
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v1, v32, vcc
@@ -2394,17 +2357,9 @@ define <16 x double> @v_minimum_v16f64(<16 x double> %src0, <16 x double> %src1)
 ; GFX900-NEXT:    v_cndmask_b32_e64 v23, v23, v32, s[24:25]
 ; GFX900-NEXT:    v_cndmask_b32_e64 v25, v25, v32, s[26:27]
 ; GFX900-NEXT:    v_cndmask_b32_e64 v27, v27, v32, s[28:29]
-; GFX900-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[30:31]
-; GFX900-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[34:35]
-; GFX900-NEXT:    v_cndmask_b32_e64 v30, v30, 0, s[34:35]
-; GFX900-NEXT:    v_readlane_b32 s35, v34, 3
-; GFX900-NEXT:    v_readlane_b32 s34, v34, 2
-; GFX900-NEXT:    v_readlane_b32 s31, v34, 1
-; GFX900-NEXT:    v_readlane_b32 s30, v34, 0
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    buffer_load_dword v34, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    v_cndmask_b32_e64 v29, v29, v32, s[40:41]
+; GFX900-NEXT:    v_cndmask_b32_e64 v31, v31, v32, s[42:43]
+; GFX900-NEXT:    v_cndmask_b32_e64 v30, v30, 0, s[42:43]
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX950-LABEL: v_minimum_v16f64:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.pow.ll b/llvm/test/CodeGen/AMDGPU/llvm.pow.ll
index 88b0526b174b9..55c2ad3483c4c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.pow.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.pow.ll
@@ -9,7 +9,7 @@ define amdgpu_ps void @test1(<4 x float> inreg %reg0) {
    %r0 = extractelement <4 x float> %reg0, i32 0
    %r1 = extractelement <4 x float> %reg0, i32 1
    %r2 = call float @llvm.pow.f32( float %r0, float %r1)
-   %vec = insertelement <4 x float> undef, float %r2, i32 0
+   %vec = insertelement <4 x float> poison, float %r2, i32 0
    call void @llvm.r600.store.swizzle(<4 x float> %vec, i32 0, i32 0)
    ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.prefetch.ll b/llvm/test/CodeGen/AMDGPU/llvm.prefetch.ll
index 2b517736ecff3..72260e0b99715 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.prefetch.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.prefetch.ll
@@ -1,34 +1,36 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX12,GFX12-SDAG %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX11 %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX12,GFX12-GISEL %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX11 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,NOSPREFETCH %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+safe-smem-prefetch -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,SPREFETCH,SPREFETCH-SDAG %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,NOSPREFETCH %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,NOSPREFETCH %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+safe-smem-prefetch -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,SPREFETCH,SPREFETCH-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,NOSPREFETCH %s
 
 ; Scalar data prefetch
 
 define amdgpu_ps void @prefetch_data_sgpr(ptr addrspace(4) inreg %ptr) {
-; GFX12-LABEL: prefetch_data_sgpr:
-; GFX12:       ; %bb.0: ; %entry
-; GFX12-NEXT:    s_prefetch_data s[0:1], 0x0, null, 0
-; GFX12-NEXT:    s_endpgm
+; NOSPREFETCH-LABEL: prefetch_data_sgpr:
+; NOSPREFETCH:       ; %bb.0: ; %entry
+; NOSPREFETCH-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: prefetch_data_sgpr:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_endpgm
+; SPREFETCH-LABEL: prefetch_data_sgpr:
+; SPREFETCH:       ; %bb.0: ; %entry
+; SPREFETCH-NEXT:    s_prefetch_data s[0:1], 0x0, null, 0
+; SPREFETCH-NEXT:    s_endpgm
 entry:
   tail call void @llvm.prefetch.p4(ptr addrspace(4) %ptr, i32 0, i32 0, i32 1)
   ret void
 }
 
 define amdgpu_ps void @prefetch_data_sgpr_offset(ptr addrspace(4) inreg %ptr) {
-; GFX12-LABEL: prefetch_data_sgpr_offset:
-; GFX12:       ; %bb.0: ; %entry
-; GFX12-NEXT:    s_prefetch_data s[0:1], 0x200, null, 0
-; GFX12-NEXT:    s_endpgm
+; NOSPREFETCH-LABEL: prefetch_data_sgpr_offset:
+; NOSPREFETCH:       ; %bb.0: ; %entry
+; NOSPREFETCH-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: prefetch_data_sgpr_offset:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_endpgm
+; SPREFETCH-LABEL: prefetch_data_sgpr_offset:
+; SPREFETCH:       ; %bb.0: ; %entry
+; SPREFETCH-NEXT:    s_prefetch_data s[0:1], 0x200, null, 0
+; SPREFETCH-NEXT:    s_endpgm
 entry:
   %gep = getelementptr float, ptr addrspace(4) %ptr, i32 128
   tail call void @llvm.prefetch.p4(ptr addrspace(4) %gep, i32 0, i32 0, i32 1)
@@ -38,14 +40,14 @@ entry:
 ; Check large offsets
 
 define amdgpu_ps void @prefetch_data_sgpr_max_offset(ptr addrspace(4) inreg %ptr) {
-; GFX12-LABEL: prefetch_data_sgpr_max_offset:
-; GFX12:       ; %bb.0: ; %entry
-; GFX12-NEXT:    s_prefetch_data s[0:1], 0x7fffff, null, 0
-; GFX12-NEXT:    s_endpgm
+; NOSPREFETCH-LABEL: prefetch_data_sgpr_max_offset:
+; NOSPREFETCH:       ; %bb.0: ; %entry
+; NOSPREFETCH-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: prefetch_data_sgpr_max_offset:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_endpgm
+; SPREFETCH-LABEL: prefetch_data_sgpr_max_offset:
+; SPREFETCH:       ; %bb.0: ; %entry
+; SPREFETCH-NEXT:    s_prefetch_data s[0:1], 0x7fffff, null, 0
+; SPREFETCH-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i8, ptr addrspace(4) %ptr, i32 8388607
   tail call void @llvm.prefetch.p4(ptr addrspace(4) %gep, i32 0, i32 0, i32 1)
@@ -53,25 +55,25 @@ entry:
 }
 
 define amdgpu_ps void @prefetch_data_sgpr_min_offset(ptr addrspace(4) inreg %ptr) {
-; GFX12-SDAG-LABEL: prefetch_data_sgpr_min_offset:
-; GFX12-SDAG:       ; %bb.0: ; %entry
-; GFX12-SDAG-NEXT:    s_mov_b32 s2, 0xff800000
-; GFX12-SDAG-NEXT:    s_mov_b32 s3, -1
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-SDAG-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[2:3]
-; GFX12-SDAG-NEXT:    s_prefetch_data s[0:1], 0x0, null, 0
-; GFX12-SDAG-NEXT:    s_endpgm
+; NOSPREFETCH-LABEL: prefetch_data_sgpr_min_offset:
+; NOSPREFETCH:       ; %bb.0: ; %entry
+; NOSPREFETCH-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: prefetch_data_sgpr_min_offset:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_endpgm
+; SPREFETCH-SDAG-LABEL: prefetch_data_sgpr_min_offset:
+; SPREFETCH-SDAG:       ; %bb.0: ; %entry
+; SPREFETCH-SDAG-NEXT:    s_mov_b32 s2, 0xff800000
+; SPREFETCH-SDAG-NEXT:    s_mov_b32 s3, -1
+; SPREFETCH-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; SPREFETCH-SDAG-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[2:3]
+; SPREFETCH-SDAG-NEXT:    s_prefetch_data s[0:1], 0x0, null, 0
+; SPREFETCH-SDAG-NEXT:    s_endpgm
 ;
-; GFX12-GISEL-LABEL: prefetch_data_sgpr_min_offset:
-; GFX12-GISEL:       ; %bb.0: ; %entry
-; GFX12-GISEL-NEXT:    s_add_co_u32 s0, s0, 0xff800000
-; GFX12-GISEL-NEXT:    s_add_co_ci_u32 s1, s1, -1
-; GFX12-GISEL-NEXT:    s_prefetch_data s[0:1], 0x0, null, 0
-; GFX12-GISEL-NEXT:    s_endpgm
+; SPREFETCH-GISEL-LABEL: prefetch_data_sgpr_min_offset:
+; SPREFETCH-GISEL:       ; %bb.0: ; %entry
+; SPREFETCH-GISEL-NEXT:    s_add_co_u32 s0, s0, 0xff800000
+; SPREFETCH-GISEL-NEXT:    s_add_co_ci_u32 s1, s1, -1
+; SPREFETCH-GISEL-NEXT:    s_prefetch_data s[0:1], 0x0, null, 0
+; SPREFETCH-GISEL-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i8, ptr addrspace(4) %ptr, i32 -8388608
   tail call void @llvm.prefetch.p4(ptr addrspace(4) %gep, i32 0, i32 0, i32 1)
@@ -79,22 +81,22 @@ entry:
 }
 
 define amdgpu_ps void @prefetch_data_sgpr_too_large_offset(ptr addrspace(4) inreg %ptr) {
-; GFX12-SDAG-LABEL: prefetch_data_sgpr_too_large_offset:
-; GFX12-SDAG:       ; %bb.0: ; %entry
-; GFX12-SDAG-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 0x800000
-; GFX12-SDAG-NEXT:    s_prefetch_data s[0:1], 0x0, null, 0
-; GFX12-SDAG-NEXT:    s_endpgm
+; NOSPREFETCH-LABEL: prefetch_data_sgpr_too_large_offset:
+; NOSPREFETCH:       ; %bb.0: ; %entry
+; NOSPREFETCH-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: prefetch_data_sgpr_too_large_offset:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_endpgm
+; SPREFETCH-SDAG-LABEL: prefetch_data_sgpr_too_large_offset:
+; SPREFETCH-SDAG:       ; %bb.0: ; %entry
+; SPREFETCH-SDAG-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 0x800000
+; SPREFETCH-SDAG-NEXT:    s_prefetch_data s[0:1], 0x0, null, 0
+; SPREFETCH-SDAG-NEXT:    s_endpgm
 ;
-; GFX12-GISEL-LABEL: prefetch_data_sgpr_too_large_offset:
-; GFX12-GISEL:       ; %bb.0: ; %entry
-; GFX12-GISEL-NEXT:    s_add_co_u32 s0, s0, 0x800000
-; GFX12-GISEL-NEXT:    s_add_co_ci_u32 s1, s1, 0
-; GFX12-GISEL-NEXT:    s_prefetch_data s[0:1], 0x0, null, 0
-; GFX12-GISEL-NEXT:    s_endpgm
+; SPREFETCH-GISEL-LABEL: prefetch_data_sgpr_too_large_offset:
+; SPREFETCH-GISEL:       ; %bb.0: ; %entry
+; SPREFETCH-GISEL-NEXT:    s_add_co_u32 s0, s0, 0x800000
+; SPREFETCH-GISEL-NEXT:    s_add_co_ci_u32 s1, s1, 0
+; SPREFETCH-GISEL-NEXT:    s_prefetch_data s[0:1], 0x0, null, 0
+; SPREFETCH-GISEL-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i8, ptr addrspace(4) %ptr, i32 8388608
   tail call void @llvm.prefetch.p4(ptr addrspace(4) %gep, i32 0, i32 0, i32 1)
@@ -135,43 +137,43 @@ entry:
 ; Check supported address spaces
 
 define amdgpu_ps void @prefetch_data_sgpr_flat(ptr inreg %ptr) {
-; GFX12-LABEL: prefetch_data_sgpr_flat:
-; GFX12:       ; %bb.0: ; %entry
-; GFX12-NEXT:    s_prefetch_data s[0:1], 0x0, null, 0
-; GFX12-NEXT:    s_endpgm
+; NOSPREFETCH-LABEL: prefetch_data_sgpr_flat:
+; NOSPREFETCH:       ; %bb.0: ; %entry
+; NOSPREFETCH-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: prefetch_data_sgpr_flat:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_endpgm
+; SPREFETCH-LABEL: prefetch_data_sgpr_flat:
+; SPREFETCH:       ; %bb.0: ; %entry
+; SPREFETCH-NEXT:    s_prefetch_data s[0:1], 0x0, null, 0
+; SPREFETCH-NEXT:    s_endpgm
 entry:
   tail call void @llvm.prefetch.pf(ptr %ptr, i32 0, i32 0, i32 1)
   ret void
 }
 
 define amdgpu_ps void @prefetch_data_sgpr_global(ptr addrspace(1) inreg %ptr) {
-; GFX12-LABEL: prefetch_data_sgpr_global:
-; GFX12:       ; %bb.0: ; %entry
-; GFX12-NEXT:    s_prefetch_data s[0:1], 0x0, null, 0
-; GFX12-NEXT:    s_endpgm
+; NOSPREFETCH-LABEL: prefetch_data_sgpr_global:
+; NOSPREFETCH:       ; %bb.0: ; %entry
+; NOSPREFETCH-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: prefetch_data_sgpr_global:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_endpgm
+; SPREFETCH-LABEL: prefetch_data_sgpr_global:
+; SPREFETCH:       ; %bb.0: ; %entry
+; SPREFETCH-NEXT:    s_prefetch_data s[0:1], 0x0, null, 0
+; SPREFETCH-NEXT:    s_endpgm
 entry:
   tail call void @llvm.prefetch.p1(ptr addrspace(1) %ptr, i32 0, i32 0, i32 1)
   ret void
 }
 
 define amdgpu_ps void @prefetch_data_sgpr_constant_32bit(ptr addrspace(6) inreg %ptr) {
-; GFX12-LABEL: prefetch_data_sgpr_constant_32bit:
-; GFX12:       ; %bb.0: ; %entry
-; GFX12-NEXT:    s_mov_b32 s1, 0
-; GFX12-NEXT:    s_prefetch_data s[0:1], 0x0, null, 0
-; GFX12-NEXT:    s_endpgm
+; NOSPREFETCH-LABEL: prefetch_data_sgpr_constant_32bit:
+; NOSPREFETCH:       ; %bb.0: ; %entry
+; NOSPREFETCH-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: prefetch_data_sgpr_constant_32bit:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_endpgm
+; SPREFETCH-LABEL: prefetch_data_sgpr_constant_32bit:
+; SPREFETCH:       ; %bb.0: ; %entry
+; SPREFETCH-NEXT:    s_mov_b32 s1, 0
+; SPREFETCH-NEXT:    s_prefetch_data s[0:1], 0x0, null, 0
+; SPREFETCH-NEXT:    s_endpgm
 entry:
   tail call void @llvm.prefetch.p6(ptr addrspace(6) %ptr, i32 0, i32 0, i32 1)
   ret void
@@ -180,28 +182,28 @@ entry:
 ; I$ prefetch
 
 define amdgpu_ps void @prefetch_inst_sgpr(ptr addrspace(4) inreg %ptr) {
-; GFX12-LABEL: prefetch_inst_sgpr:
-; GFX12:       ; %bb.0: ; %entry
-; GFX12-NEXT:    s_prefetch_inst s[0:1], 0x0, null, 0
-; GFX12-NEXT:    s_endpgm
+; NOSPREFETCH-LABEL: prefetch_inst_sgpr:
+; NOSPREFETCH:       ; %bb.0: ; %entry
+; NOSPREFETCH-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: prefetch_inst_sgpr:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_endpgm
+; SPREFETCH-LABEL: prefetch_inst_sgpr:
+; SPREFETCH:       ; %bb.0: ; %entry
+; SPREFETCH-NEXT:    s_prefetch_inst s[0:1], 0x0, null, 0
+; SPREFETCH-NEXT:    s_endpgm
 entry:
   tail call void @llvm.prefetch.p4(ptr addrspace(4) %ptr, i32 0, i32 0, i32 0)
   ret void
 }
 
 define amdgpu_ps void @prefetch_inst_sgpr_offset(ptr addrspace(4) inreg %ptr) {
-; GFX12-LABEL: prefetch_inst_sgpr_offset:
-; GFX12:       ; %bb.0: ; %entry
-; GFX12-NEXT:    s_prefetch_inst s[0:1], 0x80, null, 0
-; GFX12-NEXT:    s_endpgm
+; NOSPREFETCH-LABEL: prefetch_inst_sgpr_offset:
+; NOSPREFETCH:       ; %bb.0: ; %entry
+; NOSPREFETCH-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: prefetch_inst_sgpr_offset:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_endpgm
+; SPREFETCH-LABEL: prefetch_inst_sgpr_offset:
+; SPREFETCH:       ; %bb.0: ; %entry
+; SPREFETCH-NEXT:    s_prefetch_inst s[0:1], 0x80, null, 0
+; SPREFETCH-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i8, ptr addrspace(4) %ptr, i32 128
   tail call void @llvm.prefetch.p4(ptr addrspace(4) %gep, i32 0, i32 0, i32 0)
@@ -211,14 +213,14 @@ entry:
 ; Check large offsets
 
 define amdgpu_ps void @prefetch_inst_sgpr_max_offset(ptr addrspace(4) inreg %ptr) {
-; GFX12-LABEL: prefetch_inst_sgpr_max_offset:
-; GFX12:       ; %bb.0: ; %entry
-; GFX12-NEXT:    s_prefetch_inst s[0:1], 0x7fffff, null, 0
-; GFX12-NEXT:    s_endpgm
+; NOSPREFETCH-LABEL: prefetch_inst_sgpr_max_offset:
+; NOSPREFETCH:       ; %bb.0: ; %entry
+; NOSPREFETCH-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: prefetch_inst_sgpr_max_offset:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_endpgm
+; SPREFETCH-LABEL: prefetch_inst_sgpr_max_offset:
+; SPREFETCH:       ; %bb.0: ; %entry
+; SPREFETCH-NEXT:    s_prefetch_inst s[0:1], 0x7fffff, null, 0
+; SPREFETCH-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i8, ptr addrspace(4) %ptr, i32 8388607
   tail call void @llvm.prefetch.p4(ptr addrspace(4) %gep, i32 0, i32 0, i32 0)
@@ -226,25 +228,25 @@ entry:
 }
 
 define amdgpu_ps void @prefetch_inst_sgpr_min_offset(ptr addrspace(4) inreg %ptr) {
-; GFX12-SDAG-LABEL: prefetch_inst_sgpr_min_offset:
-; GFX12-SDAG:       ; %bb.0: ; %entry
-; GFX12-SDAG-NEXT:    s_mov_b32 s2, 0xff800000
-; GFX12-SDAG-NEXT:    s_mov_b32 s3, -1
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-SDAG-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[2:3]
-; GFX12-SDAG-NEXT:    s_prefetch_inst s[0:1], 0x0, null, 0
-; GFX12-SDAG-NEXT:    s_endpgm
+; NOSPREFETCH-LABEL: prefetch_inst_sgpr_min_offset:
+; NOSPREFETCH:       ; %bb.0: ; %entry
+; NOSPREFETCH-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: prefetch_inst_sgpr_min_offset:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_endpgm
+; SPREFETCH-SDAG-LABEL: prefetch_inst_sgpr_min_offset:
+; SPREFETCH-SDAG:       ; %bb.0: ; %entry
+; SPREFETCH-SDAG-NEXT:    s_mov_b32 s2, 0xff800000
+; SPREFETCH-SDAG-NEXT:    s_mov_b32 s3, -1
+; SPREFETCH-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; SPREFETCH-SDAG-NEXT:    s_add_nc_u64 s[0:1], s[0:1], s[2:3]
+; SPREFETCH-SDAG-NEXT:    s_prefetch_inst s[0:1], 0x0, null, 0
+; SPREFETCH-SDAG-NEXT:    s_endpgm
 ;
-; GFX12-GISEL-LABEL: prefetch_inst_sgpr_min_offset:
-; GFX12-GISEL:       ; %bb.0: ; %entry
-; GFX12-GISEL-NEXT:    s_add_co_u32 s0, s0, 0xff800000
-; GFX12-GISEL-NEXT:    s_add_co_ci_u32 s1, s1, -1
-; GFX12-GISEL-NEXT:    s_prefetch_inst s[0:1], 0x0, null, 0
-; GFX12-GISEL-NEXT:    s_endpgm
+; SPREFETCH-GISEL-LABEL: prefetch_inst_sgpr_min_offset:
+; SPREFETCH-GISEL:       ; %bb.0: ; %entry
+; SPREFETCH-GISEL-NEXT:    s_add_co_u32 s0, s0, 0xff800000
+; SPREFETCH-GISEL-NEXT:    s_add_co_ci_u32 s1, s1, -1
+; SPREFETCH-GISEL-NEXT:    s_prefetch_inst s[0:1], 0x0, null, 0
+; SPREFETCH-GISEL-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i8, ptr addrspace(4) %ptr, i32 -8388608
   tail call void @llvm.prefetch.p4(ptr addrspace(4) %gep, i32 0, i32 0, i32 0)
@@ -252,22 +254,22 @@ entry:
 }
 
 define amdgpu_ps void @prefetch_inst_sgpr_too_large_offset(ptr addrspace(4) inreg %ptr) {
-; GFX12-SDAG-LABEL: prefetch_inst_sgpr_too_large_offset:
-; GFX12-SDAG:       ; %bb.0: ; %entry
-; GFX12-SDAG-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 0x800000
-; GFX12-SDAG-NEXT:    s_prefetch_inst s[0:1], 0x0, null, 0
-; GFX12-SDAG-NEXT:    s_endpgm
+; NOSPREFETCH-LABEL: prefetch_inst_sgpr_too_large_offset:
+; NOSPREFETCH:       ; %bb.0: ; %entry
+; NOSPREFETCH-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: prefetch_inst_sgpr_too_large_offset:
-; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_endpgm
+; SPREFETCH-SDAG-LABEL: prefetch_inst_sgpr_too_large_offset:
+; SPREFETCH-SDAG:       ; %bb.0: ; %entry
+; SPREFETCH-SDAG-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 0x800000
+; SPREFETCH-SDAG-NEXT:    s_prefetch_inst s[0:1], 0x0, null, 0
+; SPREFETCH-SDAG-NEXT:    s_endpgm
 ;
-; GFX12-GISEL-LABEL: prefetch_inst_sgpr_too_large_offset:
-; GFX12-GISEL:       ; %bb.0: ; %entry
-; GFX12-GISEL-NEXT:    s_add_co_u32 s0, s0, 0x800000
-; GFX12-GISEL-NEXT:    s_add_co_ci_u32 s1, s1, 0
-; GFX12-GISEL-NEXT:    s_prefetch_inst s[0:1], 0x0, null, 0
-; GFX12-GISEL-NEXT:    s_endpgm
+; SPREFETCH-GISEL-LABEL: prefetch_inst_sgpr_too_large_offset:
+; SPREFETCH-GISEL:       ; %bb.0: ; %entry
+; SPREFETCH-GISEL-NEXT:    s_add_co_u32 s0, s0, 0x800000
+; SPREFETCH-GISEL-NEXT:    s_add_co_ci_u32 s1, s1, 0
+; SPREFETCH-GISEL-NEXT:    s_prefetch_inst s[0:1], 0x0, null, 0
+; SPREFETCH-GISEL-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i8, ptr addrspace(4) %ptr, i32 8388608
   tail call void @llvm.prefetch.p4(ptr addrspace(4) %gep, i32 0, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.r600.cube.ll b/llvm/test/CodeGen/AMDGPU/llvm.r600.cube.ll
index 7ec7d9bdb400f..f471f4549823e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.r600.cube.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.r600.cube.ll
@@ -18,7 +18,7 @@ main_body:
   %tmp8 = load <4 x float>, ptr addrspace(8) getelementptr ([1024 x <4 x float>], ptr addrspace(8) null, i64 0, i32 9)
   %tmp9 = extractelement <4 x float> %tmp8, i32 2
   %tmp10 = fdiv float %tmp9, %tmp1
-  %tmp11 = insertelement <4 x float> undef, float %tmp4, i32 0
+  %tmp11 = insertelement <4 x float> poison, float %tmp4, i32 0
   %tmp12 = insertelement <4 x float> %tmp11, float %tmp7, i32 1
   %tmp13 = insertelement <4 x float> %tmp12, float %tmp10, i32 2
   %tmp14 = insertelement <4 x float> %tmp13, float 1.000000e+00, i32 3
@@ -33,7 +33,7 @@ main_body:
   %tmp23 = fadd float %tmp22, 1.500000e+00
   %tmp24 = fmul float %tmp17, %tmp21
   %tmp25 = fadd float %tmp24, 1.500000e+00
-  %tmp26 = insertelement <4 x float> undef, float %tmp25, i32 0
+  %tmp26 = insertelement <4 x float> poison, float %tmp25, i32 0
   %tmp27 = insertelement <4 x float> %tmp26, float %tmp23, i32 1
   %tmp28 = insertelement <4 x float> %tmp27, float %tmp19, i32 2
   %tmp29 = insertelement <4 x float> %tmp28, float %tmp25, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/load-hi16.ll b/llvm/test/CodeGen/AMDGPU/load-hi16.ll
index 0c61c58ef0619..c4eafe5d999a4 100644
--- a/llvm/test/CodeGen/AMDGPU/load-hi16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-hi16.ll
@@ -66,7 +66,7 @@ entry:
   %load.lo = load i16, ptr addrspace(3) %in
   %load.hi = load i16, ptr addrspace(3) %gep
   store i16 %load.lo, ptr addrspace(3) null
-  %build0 = insertelement <2 x i16> undef, i16 %load.lo, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %load.lo, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load.hi, i32 1
   ret <2 x i16> %build1
 }
@@ -133,7 +133,7 @@ entry:
   %load.lo = load i16, ptr addrspace(3) %in
   %load.hi = load i16, ptr addrspace(3) %gep
   store i16 %load.hi, ptr addrspace(3) null
-  %build0 = insertelement <2 x i16> undef, i16 %load.lo, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %load.lo, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load.hi, i32 1
   ret <2 x i16> %build1
 }
@@ -201,7 +201,7 @@ entry:
   %load.hi = load i16, ptr addrspace(3) %gep
   store i16 %load.lo, ptr addrspace(3) %out0
   store i16 %load.hi, ptr addrspace(3) %out1
-  %build0 = insertelement <2 x i16> undef, i16 %load.lo, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %load.lo, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load.hi, i32 1
   ret <2 x i16> %build1
 }
@@ -239,7 +239,7 @@ define <2 x i16> @load_local_hi_v2i16_undeflo(ptr addrspace(3) %in) #0 {
 ; GFX900-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i16, ptr addrspace(3) %in
-  %build = insertelement <2 x i16> undef, i16 %load, i32 1
+  %build = insertelement <2 x i16> poison, i16 %load, i32 1
   ret <2 x i16> %build
 }
 
@@ -280,7 +280,7 @@ define <2 x i16> @load_local_hi_v2i16_reglo(ptr addrspace(3) %in, i16 %reg) #0 {
 ; GFX900-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i16, ptr addrspace(3) %in
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
   ret <2 x i16> %build1
 }
@@ -328,7 +328,7 @@ define void @load_local_hi_v2i16_reglo_vreg(ptr addrspace(3) %in, i16 %reg) #0 {
 ; GFX900-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i16, ptr addrspace(3) %in
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -459,7 +459,7 @@ define void @load_local_hi_v2f16_reglo_vreg(ptr addrspace(3) %in, half %reg) #0
 ; GFX900-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load half, ptr addrspace(3) %in
-  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build0 = insertelement <2 x half> poison, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %load, i32 1
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -509,7 +509,7 @@ define void @load_local_hi_v2i16_reglo_vreg_zexti8(ptr addrspace(3) %in, i16 %re
 entry:
   %load = load i8, ptr addrspace(3) %in
   %ext = zext i8 %load to i16
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -559,7 +559,7 @@ define void @load_local_hi_v2i16_reglo_vreg_sexti8(ptr addrspace(3) %in, i16 %re
 entry:
   %load = load i8, ptr addrspace(3) %in
   %ext = sext i8 %load to i16
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -611,7 +611,7 @@ entry:
   %ext = zext i8 %load to i16
   %bitcast = bitcast i16 %ext to half
 
-  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build0 = insertelement <2 x half> poison, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %bitcast, i32 1
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -663,7 +663,7 @@ entry:
   %ext = sext i8 %load to i16
   %bitcast = bitcast i16 %ext to half
 
-  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build0 = insertelement <2 x half> poison, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %bitcast, i32 1
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -714,7 +714,7 @@ define void @load_global_hi_v2i16_reglo_vreg(ptr addrspace(1) %in, i16 %reg) #0
 entry:
   %gep = getelementptr inbounds i16, ptr addrspace(1) %in, i64 -2047
   %load = load i16, ptr addrspace(1) %gep
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -765,7 +765,7 @@ define void @load_global_hi_v2f16_reglo_vreg(ptr addrspace(1) %in, half %reg) #0
 entry:
   %gep = getelementptr inbounds half, ptr addrspace(1) %in, i64 -2047
   %load = load half, ptr addrspace(1) %gep
-  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build0 = insertelement <2 x half> poison, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %load, i32 1
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -817,7 +817,7 @@ entry:
   %gep = getelementptr inbounds i8, ptr addrspace(1) %in, i64 -4095
   %load = load i8, ptr addrspace(1) %gep
   %ext = zext i8 %load to i16
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -869,7 +869,7 @@ entry:
   %gep = getelementptr inbounds i8, ptr addrspace(1) %in, i64 -4095
   %load = load i8, ptr addrspace(1) %gep
   %ext = sext i8 %load to i16
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -922,7 +922,7 @@ entry:
   %load = load i8, ptr addrspace(1) %gep
   %ext = sext i8 %load to i16
   %bitcast = bitcast i16 %ext to half
-  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build0 = insertelement <2 x half> poison, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %bitcast, i32 1
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -975,7 +975,7 @@ entry:
   %load = load i8, ptr addrspace(1) %gep
   %ext = zext i8 %load to i16
   %bitcast = bitcast i16 %ext to half
-  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build0 = insertelement <2 x half> poison, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %bitcast, i32 1
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -1023,7 +1023,7 @@ define void @load_flat_hi_v2i16_reglo_vreg(ptr %in, i16 %reg) #0 {
 ; GFX900-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i16, ptr %in
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -1071,7 +1071,7 @@ define void @load_flat_hi_v2f16_reglo_vreg(ptr %in, half %reg) #0 {
 ; GFX900-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load half, ptr %in
-  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build0 = insertelement <2 x half> poison, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %load, i32 1
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -1120,7 +1120,7 @@ define void @load_flat_hi_v2i16_reglo_vreg_zexti8(ptr %in, i16 %reg) #0 {
 entry:
   %load = load i8, ptr %in
   %ext = zext i8 %load to i16
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -1169,7 +1169,7 @@ define void @load_flat_hi_v2i16_reglo_vreg_sexti8(ptr %in, i16 %reg) #0 {
 entry:
   %load = load i8, ptr %in
   %ext = sext i8 %load to i16
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -1219,7 +1219,7 @@ entry:
   %load = load i8, ptr %in
   %ext = zext i8 %load to i16
   %bitcast = bitcast i16 %ext to half
-  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build0 = insertelement <2 x half> poison, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %bitcast, i32 1
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -1269,7 +1269,7 @@ entry:
   %load = load i8, ptr %in
   %ext = sext i8 %load to i16
   %bitcast = bitcast i16 %ext to half
-  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build0 = insertelement <2 x half> poison, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %bitcast, i32 1
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -1318,7 +1318,7 @@ define void @load_private_hi_v2i16_reglo_vreg(ptr addrspace(5) byval(i16) %in, i
 entry:
   %gep = getelementptr inbounds i16, ptr addrspace(5) %in, i64 2047
   %load = load i16, ptr addrspace(5) %gep
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -1367,7 +1367,7 @@ define void @load_private_hi_v2f16_reglo_vreg(ptr addrspace(5) byval(half) %in,
 entry:
   %gep = getelementptr inbounds half, ptr addrspace(5) %in, i64 2047
   %load = load half, ptr addrspace(5) %gep
-  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build0 = insertelement <2 x half> poison, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %load, i32 1
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -1416,7 +1416,7 @@ define void @load_private_hi_v2i16_reglo_vreg_nooff(ptr addrspace(5) byval(i16)
 ; GFX900-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load volatile i16, ptr addrspace(5) inttoptr (i32 4094 to ptr addrspace(5))
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -1465,7 +1465,7 @@ define void @load_private_hi_v2f16_reglo_vreg_nooff(ptr addrspace(5) %in, half %
 ; GFX900-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load volatile half, ptr addrspace(5) inttoptr (i32 4094 to ptr addrspace(5))
-  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build0 = insertelement <2 x half> poison, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %load, i32 1
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -1515,7 +1515,7 @@ entry:
   %gep = getelementptr inbounds i8, ptr addrspace(5) %in, i64 4095
   %load = load i8, ptr addrspace(5) %gep
   %ext = zext i8 %load to i16
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -1566,7 +1566,7 @@ entry:
   %load = load i8, ptr addrspace(5) %gep
   %ext = zext i8 %load to i16
   %bitcast = bitcast i16 %ext to half
-  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build0 = insertelement <2 x half> poison, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %bitcast, i32 1
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -1617,7 +1617,7 @@ entry:
   %load = load i8, ptr addrspace(5) %gep
   %ext = sext i8 %load to i16
   %bitcast = bitcast i16 %ext to half
-  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build0 = insertelement <2 x half> poison, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %bitcast, i32 1
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -1667,7 +1667,7 @@ entry:
   %gep = getelementptr inbounds i8, ptr addrspace(5) %in, i64 4095
   %load = load i8, ptr addrspace(5) %gep
   %ext = sext i8 %load to i16
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -1717,7 +1717,7 @@ define void @load_private_hi_v2i16_reglo_vreg_nooff_zexti8(ptr addrspace(5) %in,
 entry:
   %load = load volatile i8, ptr addrspace(5) inttoptr (i32 4094 to ptr addrspace(5))
   %ext = zext i8 %load to i16
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -1767,7 +1767,7 @@ define void @load_private_hi_v2i16_reglo_vreg_nooff_sexti8(ptr addrspace(5) %in,
 entry:
   %load = load volatile i8, ptr addrspace(5) inttoptr (i32 4094 to ptr addrspace(5))
   %ext = sext i8 %load to i16
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -1818,7 +1818,7 @@ entry:
   %load = load volatile i8, ptr addrspace(5) inttoptr (i32 4094 to ptr addrspace(5))
   %ext = zext i8 %load to i16
   %bc.ext = bitcast i16 %ext to half
-  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build0 = insertelement <2 x half> poison, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %bc.ext, i32 1
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -1869,7 +1869,7 @@ define void @load_constant_hi_v2i16_reglo_vreg(ptr addrspace(4) %in, i16 %reg) #
 entry:
   %gep = getelementptr inbounds i16, ptr addrspace(4) %in, i64 -2047
   %load = load i16, ptr addrspace(4) %gep
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -1920,7 +1920,7 @@ define void @load_constant_hi_v2f16_reglo_vreg(ptr addrspace(4) %in, half %reg)
 entry:
   %gep = getelementptr inbounds half, ptr addrspace(4) %in, i64 -2047
   %load = load half, ptr addrspace(4) %gep
-  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build0 = insertelement <2 x half> poison, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %load, i32 1
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -1973,7 +1973,7 @@ entry:
   %load = load i8, ptr addrspace(4) %gep
   %ext = sext i8 %load to i16
   %bitcast = bitcast i16 %ext to half
-  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build0 = insertelement <2 x half> poison, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %bitcast, i32 1
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -2026,7 +2026,7 @@ entry:
   %load = load i8, ptr addrspace(4) %gep
   %ext = zext i8 %load to i16
   %bitcast = bitcast i16 %ext to half
-  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build0 = insertelement <2 x half> poison, half %reg, i32 0
   %build1 = insertelement <2 x half> %build0, half %bitcast, i32 1
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -2092,7 +2092,7 @@ entry:
   store volatile i32 123, ptr addrspace(5) %obj0
   %gep = getelementptr inbounds [4096 x i16], ptr addrspace(5) %obj1, i32 0, i32 2027
   %load = load i16, ptr addrspace(5) %gep
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -2156,7 +2156,7 @@ entry:
   %gep = getelementptr inbounds [4096 x i8], ptr addrspace(5) %obj1, i32 0, i32 4055
   %load = load i8, ptr addrspace(5) %gep
   %ext = sext i8 %load to i16
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -2220,7 +2220,7 @@ entry:
   %gep = getelementptr inbounds [4096 x i8], ptr addrspace(5) %obj1, i32 0, i32 4055
   %load = load i8, ptr addrspace(5) %gep
   %ext = zext i8 %load to i16
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -2273,7 +2273,7 @@ entry:
   %gep = getelementptr inbounds i16, ptr addrspace(3) %in, i32 1
   %load0 = load volatile i16, ptr addrspace(3) %in
   %load1 = load volatile i16, ptr addrspace(3) %gep
-  %build0 = insertelement <2 x i16> undef, i16 %load0, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %load0, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
   ret <2 x i16> %build1
 }
@@ -2324,7 +2324,7 @@ entry:
   %gep = getelementptr inbounds i16, ptr addrspace(3) %in, i32 8
   %load.lo = load i16, ptr addrspace(3) %in
   %load.hi = load i16, ptr addrspace(3) %gep
-  %build0 = insertelement <2 x i16> undef, i16 %load.lo, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %load.lo, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load.hi, i32 1
   ret <2 x i16> %build1
 }
@@ -2370,7 +2370,7 @@ define <2 x i16> @load_local_v2i16_broadcast(ptr addrspace(3) %in) #0 {
 entry:
   %gep = getelementptr inbounds i16, ptr addrspace(3) %in, i32 1
   %load0 = load i16, ptr addrspace(3) %in
-  %build0 = insertelement <2 x i16> undef, i16 %load0, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %load0, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load0, i32 1
   ret <2 x i16> %build1
 }
@@ -2429,7 +2429,7 @@ entry:
   %load.lo = load i16, ptr addrspace(3) %in
   store i16 123, ptr addrspace(3) %may.alias
   %load.hi = load i16, ptr addrspace(3) %gep
-  %build0 = insertelement <2 x i16> undef, i16 %load.lo, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %load.lo, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load.hi, i32 1
   ret <2 x i16> %build1
 }
@@ -2483,7 +2483,7 @@ entry:
   %gep = getelementptr inbounds i16, ptr addrspace(1) %in, i64 1
   %load0 = load volatile i16, ptr addrspace(1) %in
   %load1 = load volatile i16, ptr addrspace(1) %gep
-  %build0 = insertelement <2 x i16> undef, i16 %load0, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %load0, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
   ret <2 x i16> %build1
 }
@@ -2538,7 +2538,7 @@ entry:
   %gep = getelementptr inbounds i16, ptr %in, i64 1
   %load0 = load volatile i16, ptr %in
   %load1 = load volatile i16, ptr %gep
-  %build0 = insertelement <2 x i16> undef, i16 %load0, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %load0, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
   ret <2 x i16> %build1
 }
@@ -2590,7 +2590,7 @@ entry:
   %gep = getelementptr inbounds i16, ptr addrspace(4) %in, i64 1
   %load0 = load volatile i16, ptr addrspace(4) %in
   %load1 = load volatile i16, ptr addrspace(4) %gep
-  %build0 = insertelement <2 x i16> undef, i16 %load0, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %load0, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
   ret <2 x i16> %build1
 }
@@ -2641,7 +2641,7 @@ entry:
   %gep = getelementptr inbounds i16, ptr addrspace(5) %in, i32 1
   %load0 = load volatile i16, ptr addrspace(5) %in
   %load1 = load volatile i16, ptr addrspace(5) %gep
-  %build0 = insertelement <2 x i16> undef, i16 %load0, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %load0, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
   ret <2 x i16> %build1
 }
@@ -2698,7 +2698,7 @@ define <2 x i16> @load_local_hi_v2i16_store_local_lo(i16 %reg, ptr addrspace(3)
 ; GFX900-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i16, ptr addrspace(3) %in
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
   store volatile i16 %reg, ptr addrspace(3) %in
   ret <2 x i16> %build1
diff --git a/llvm/test/CodeGen/AMDGPU/load-input-fold.ll b/llvm/test/CodeGen/AMDGPU/load-input-fold.ll
index a243e6f538722..7d34f6b4abfbd 100644
--- a/llvm/test/CodeGen/AMDGPU/load-input-fold.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-input-fold.ll
@@ -80,16 +80,16 @@ main_body:
   %75 = extractelement <4 x float> %74, i32 1
   %76 = load <4 x float>, ptr addrspace(8) getelementptr ([1024 x <4 x float>], ptr addrspace(8) null, i64 0, i32 4)
   %77 = extractelement <4 x float> %76, i32 2
-  %78 = insertelement <4 x float> undef, float %4, i32 0
+  %78 = insertelement <4 x float> poison, float %4, i32 0
   %79 = insertelement <4 x float> %78, float %5, i32 1
   %80 = insertelement <4 x float> %79, float %6, i32 2
   %81 = insertelement <4 x float> %80, float 0.000000e+00, i32 3
-  %82 = insertelement <4 x float> undef, float %73, i32 0
+  %82 = insertelement <4 x float> poison, float %73, i32 0
   %83 = insertelement <4 x float> %82, float %75, i32 1
   %84 = insertelement <4 x float> %83, float %77, i32 2
   %85 = insertelement <4 x float> %84, float 0.000000e+00, i32 3
   %86 = call float @llvm.r600.dot4(<4 x float> %81, <4 x float> %85)
-  %87 = insertelement <4 x float> undef, float %86, i32 0
+  %87 = insertelement <4 x float> poison, float %86, i32 0
   call void @llvm.r600.store.swizzle(<4 x float> %87, i32 2, i32 2)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/load-lo16.ll b/llvm/test/CodeGen/AMDGPU/load-lo16.ll
index 3ef86c13e150a..ab1ca9093b551 100644
--- a/llvm/test/CodeGen/AMDGPU/load-lo16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-lo16.ll
@@ -28,7 +28,7 @@ define <2 x i16> @load_local_lo_v2i16_undeflo(ptr addrspace(3) %in) #0 {
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i16, ptr addrspace(3) %in
-  %build = insertelement <2 x i16> undef, i16 %load, i32 0
+  %build = insertelement <2 x i16> poison, i16 %load, i32 0
   ret <2 x i16> %build
 }
 
@@ -71,7 +71,7 @@ define <2 x i16> @load_local_lo_v2i16_reglo(ptr addrspace(3) %in, i16 %reg) #0 {
 ; GFX900-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i16, ptr addrspace(3) %in
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 1
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 1
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 0
   ret <2 x i16> %build1
 }
@@ -124,7 +124,7 @@ define void @load_local_lo_v2i16_reglo_vreg(ptr addrspace(3) %in, i16 %reg) #0 {
 ; GFX900-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load i16, ptr addrspace(3) %in
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 1
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 1
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 0
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -282,7 +282,7 @@ define void @load_local_lo_v2f16_reglo_vreg(ptr addrspace(3) %in, half %reg) #0
 ; GFX900-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load = load half, ptr addrspace(3) %in
-  %build0 = insertelement <2 x half> undef, half %reg, i32 1
+  %build0 = insertelement <2 x half> poison, half %reg, i32 1
   %build1 = insertelement <2 x half> %build0, half %load, i32 0
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -377,7 +377,7 @@ define void @load_local_lo_v2i16_reglo_vreg_zexti8(ptr addrspace(3) %in, i16 %re
 entry:
   %load = load i8, ptr addrspace(3) %in
   %ext = zext i8 %load to i16
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 1
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 1
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 0
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -472,7 +472,7 @@ define void @load_local_lo_v2i16_reglo_vreg_sexti8(ptr addrspace(3) %in, i16 %re
 entry:
   %load = load i8, ptr addrspace(3) %in
   %ext = sext i8 %load to i16
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 1
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 1
   %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 0
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
@@ -527,7 +527,7 @@ entry:
   %load = load i8, ptr addrspace(3) %in
   %ext = zext i8 %load to i16
   %bitcast = bitcast i16 %ext to half
-  %build0 = insertelement <2 x half> undef, half %reg, i32 1
+  %build0 = insertelement <2 x half> poison, half %reg, i32 1
   %build1 = insertelement <2 x half> %build0, half %bitcast, i32 0
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -582,7 +582,7 @@ entry:
   %load = load i8, ptr addrspace(3) %in
   %ext = sext i8 %load to i16
   %bitcast = bitcast i16 %ext to half
-  %build0 = insertelement <2 x half> undef, half %reg, i32 1
+  %build0 = insertelement <2 x half> poison, half %reg, i32 1
   %build1 = insertelement <2 x half> %build0, half %bitcast, i32 0
   store <2 x half> %build1, ptr addrspace(1) undef
   ret void
@@ -1363,7 +1363,7 @@ define void @load_private_lo_v2i16_reghi_vreg(ptr addrspace(5) byval(i16) %in, i
 entry:
   %gep = getelementptr inbounds i16, ptr addrspace(5) %in, i64 2047
   %load = load i16, ptr addrspace(5) %gep
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 1
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 1
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 0
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll b/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
index 01bab28a49858..2cee66f902b6d 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
@@ -22,7 +22,7 @@ define amdgpu_vs void @test(ptr addrspace(8) inreg %arg1, ptr addrspace(3) %arg2
 ; CHECK-NEXT:    s_endpgm
   call void @llvm.amdgcn.exp.f32(i32 0, i32 0, float undef, float undef, float undef, float undef, i1 false, i1 false)
   %var1 = load <6 x float>, ptr addrspace(3) %arg2, align 4
-  %var2 = shufflevector <6 x float> %var1, <6 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %var2 = shufflevector <6 x float> %var1, <6 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   call void @llvm.amdgcn.struct.ptr.tbuffer.store.v4f32(<4 x float> %var2, ptr addrspace(8) %arg1, i32 0, i32 0, i32 0, i32 126, i32 0)
   ret void
 }
@@ -52,9 +52,9 @@ define amdgpu_vs void @test_2(ptr addrspace(8) inreg %arg1, i32 %arg2, i32 inreg
 ; CHECK-NEXT:    tbuffer_store_format_xyzw v[2:5], v0, s[0:3], s4 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:16 glc slc
 ; CHECK-NEXT:    s_endpgm
   %load = load <8 x float>, ptr addrspace(3) %arg4, align 4
-  %vec1 = shufflevector <8 x float> %load, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %vec1 = shufflevector <8 x float> %load, <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   call void @llvm.amdgcn.struct.ptr.tbuffer.store.v4f32(<4 x float> %vec1, ptr addrspace(8) %arg1, i32 %arg2, i32 0, i32 %arg3, i32 77, i32 3)
-  %vec2 = shufflevector <8 x float> %load, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %vec2 = shufflevector <8 x float> %load, <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   call void @llvm.amdgcn.struct.ptr.tbuffer.store.v4f32(<4 x float> %vec2, ptr addrspace(8) %arg1, i32 %arg2, i32 16, i32 %arg3, i32 77, i32 3)
   ret void
 }
@@ -102,17 +102,17 @@ define amdgpu_vs void @test_3(i32 inreg %arg1, i32 inreg %arg2, ptr addrspace(8)
 ; CHECK-NEXT:    tbuffer_store_format_xy v[0:1], v9, s[4:7], s1 format:[BUF_DATA_FORMAT_INVALID,BUF_NUM_FORMAT_UINT] idxen offset:256 glc slc
 ; CHECK-NEXT:    s_endpgm
   %load1 = load <6 x float>, ptr addrspace(3) %arg5, align 4
-  %vec11 = shufflevector <6 x float> %load1, <6 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %vec11 = shufflevector <6 x float> %load1, <6 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   call void @llvm.amdgcn.struct.ptr.tbuffer.store.v4f32(<4 x float> %vec11, ptr addrspace(8) %arg3, i32 %arg1, i32 264, i32 %arg2, i32 77, i32 3)
-  %vec12 = shufflevector <6 x float> %load1, <6 x float> undef, <2 x i32> <i32 4, i32 5>
+  %vec12 = shufflevector <6 x float> %load1, <6 x float> poison, <2 x i32> <i32 4, i32 5>
   call void @llvm.amdgcn.struct.ptr.tbuffer.store.v2f32(<2 x float> %vec12, ptr addrspace(8) %arg3, i32 %arg1, i32 280, i32 %arg2, i32 64, i32 3)
 
   call void @llvm.amdgcn.exp.f32(i32 0, i32 0, float undef, float undef, float undef, float undef, i1 false, i1 false)
 
   %load2 = load <6 x float>, ptr addrspace(3) %arg6, align 4
-  %vec21 = shufflevector <6 x float> %load2, <6 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %vec21 = shufflevector <6 x float> %load2, <6 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   call void @llvm.amdgcn.struct.ptr.tbuffer.store.v4f32(<4 x float> %vec21, ptr addrspace(8) %arg3, i32 %arg1, i32 240, i32 %arg2, i32 77, i32 3)
-  %vec22 = shufflevector <6 x float> %load2, <6 x float> undef, <2 x i32> <i32 4, i32 5>
+  %vec22 = shufflevector <6 x float> %load2, <6 x float> poison, <2 x i32> <i32 4, i32 5>
   call void @llvm.amdgcn.struct.ptr.tbuffer.store.v2f32(<2 x float> %vec22, ptr addrspace(8) %arg3, i32 %arg1, i32 256, i32 %arg2, i32 64, i32 3)
 
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/loop-live-out-copy-undef-subrange.ll b/llvm/test/CodeGen/AMDGPU/loop-live-out-copy-undef-subrange.ll
index 2d3c03bbe5317..dec86d41769f6 100644
--- a/llvm/test/CodeGen/AMDGPU/loop-live-out-copy-undef-subrange.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop-live-out-copy-undef-subrange.ll
@@ -35,8 +35,8 @@ bb1:                                              ; preds = %bb3, %bb
   %i2 = extractelement <3 x float> %i, i64 2
   %i3 = fmul float %i2, 1.000000e+00
   %i4 = fmul nsz <3 x float> %arg, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
-  %i5 = insertelement <3 x float> undef, float %i3, i32 0
-  %i6 = shufflevector <3 x float> %i5, <3 x float> undef, <3 x i32> zeroinitializer
+  %i5 = insertelement <3 x float> poison, float %i3, i32 0
+  %i6 = shufflevector <3 x float> %i5, <3 x float> poison, <3 x i32> zeroinitializer
   %i7 = fmul <3 x float> %i4, %i6
   %i8 = fcmp oeq float %i3, 0.000000e+00
   br i1 %i8, label %bb3, label %bb2
diff --git a/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll b/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll
index dd77e575e7505..874dece6b728d 100644
--- a/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll
@@ -1,33 +1,60 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -amdgpu-loop-prefetch < %s | FileCheck --check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -amdgpu-loop-prefetch < %s | FileCheck --check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -amdgpu-loop-prefetch -mattr=+safe-smem-prefetch < %s | FileCheck --check-prefix=GFX12-SPREFETCH %s
 
 define amdgpu_kernel void @copy_flat(ptr nocapture %d, ptr nocapture readonly %s, i32 %n) {
-; GCN-LABEL: copy_flat:
-; GCN:       ; %bb.0: ; %entry
-; GCN-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; GCN-NEXT:    s_wait_kmcnt 0x0
-; GCN-NEXT:    s_cmp_eq_u32 s6, 0
-; GCN-NEXT:    s_cbranch_scc1 .LBB0_3
-; GCN-NEXT:  ; %bb.1: ; %for.body.preheader
-; GCN-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GCN-NEXT:    s_wait_kmcnt 0x0
-; GCN-NEXT:    s_add_nc_u64 s[2:3], s[2:3], 0xb0
-; GCN-NEXT:  .LBB0_2: ; %for.body
-; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT:    s_wait_alu 0xfffe
-; GCN-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
-; GCN-NEXT:    s_prefetch_data s[2:3], 0x0, null, 0
-; GCN-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
-; GCN-NEXT:    s_add_co_i32 s6, s6, -1
-; GCN-NEXT:    flat_load_b128 v[0:3], v[0:1] offset:-176
-; GCN-NEXT:    s_add_nc_u64 s[2:3], s[2:3], 16
-; GCN-NEXT:    s_cmp_lg_u32 s6, 0
-; GCN-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 16
-; GCN-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GCN-NEXT:    flat_store_b128 v[4:5], v[0:3]
-; GCN-NEXT:    s_cbranch_scc1 .LBB0_2
-; GCN-NEXT:  .LBB0_3: ; %for.end
-; GCN-NEXT:    s_endpgm
+; GFX12-LABEL: copy_flat:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX12-NEXT:    s_cbranch_scc1 .LBB0_3
+; GFX12-NEXT:  ; %bb.1: ; %for.body.preheader
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], 0xb0
+; GFX12-NEXT:  .LBB0_2: ; %for.body
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
+; GFX12-NEXT:    s_add_co_i32 s6, s6, -1
+; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], 16
+; GFX12-NEXT:    flat_load_b128 v[0:3], v[0:1] offset:-176
+; GFX12-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 16
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    flat_store_b128 v[4:5], v[0:3]
+; GFX12-NEXT:    s_cbranch_scc1 .LBB0_2
+; GFX12-NEXT:  .LBB0_3: ; %for.end
+; GFX12-NEXT:    s_endpgm
+;
+; GFX12-SPREFETCH-LABEL: copy_flat:
+; GFX12-SPREFETCH:       ; %bb.0: ; %entry
+; GFX12-SPREFETCH-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-SPREFETCH-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SPREFETCH-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX12-SPREFETCH-NEXT:    s_cbranch_scc1 .LBB0_3
+; GFX12-SPREFETCH-NEXT:  ; %bb.1: ; %for.body.preheader
+; GFX12-SPREFETCH-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-SPREFETCH-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SPREFETCH-NEXT:    s_add_nc_u64 s[2:3], s[2:3], 0xb0
+; GFX12-SPREFETCH-NEXT:  .LBB0_2: ; %for.body
+; GFX12-SPREFETCH-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-SPREFETCH-NEXT:    s_wait_alu 0xfffe
+; GFX12-SPREFETCH-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-SPREFETCH-NEXT:    s_prefetch_data s[2:3], 0x0, null, 0
+; GFX12-SPREFETCH-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
+; GFX12-SPREFETCH-NEXT:    s_add_co_i32 s6, s6, -1
+; GFX12-SPREFETCH-NEXT:    flat_load_b128 v[0:3], v[0:1] offset:-176
+; GFX12-SPREFETCH-NEXT:    s_add_nc_u64 s[2:3], s[2:3], 16
+; GFX12-SPREFETCH-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX12-SPREFETCH-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 16
+; GFX12-SPREFETCH-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SPREFETCH-NEXT:    flat_store_b128 v[4:5], v[0:3]
+; GFX12-SPREFETCH-NEXT:    s_cbranch_scc1 .LBB0_2
+; GFX12-SPREFETCH-NEXT:  .LBB0_3: ; %for.end
+; GFX12-SPREFETCH-NEXT:    s_endpgm
 entry:
   %cmp6.not = icmp eq i32 %n, 0
   br i1 %cmp6.not, label %for.end, label %for.body
@@ -48,30 +75,54 @@ for.end:                                          ; preds = %for.body, %entry
 }
 
 define amdgpu_kernel void @copy_global(ptr addrspace(1) nocapture %d, ptr addrspace(1) nocapture readonly %s, i32 %n) {
-; GCN-LABEL: copy_global:
-; GCN:       ; %bb.0: ; %entry
-; GCN-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; GCN-NEXT:    s_wait_kmcnt 0x0
-; GCN-NEXT:    s_cmp_eq_u32 s6, 0
-; GCN-NEXT:    s_cbranch_scc1 .LBB1_3
-; GCN-NEXT:  ; %bb.1: ; %for.body.preheader
-; GCN-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GCN-NEXT:    v_mov_b32_e32 v0, 0
-; GCN-NEXT:    s_wait_kmcnt 0x0
-; GCN-NEXT:    s_add_nc_u64 s[2:3], s[2:3], 0xb0
-; GCN-NEXT:  .LBB1_2: ; %for.body
-; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT:    global_load_b128 v[1:4], v0, s[2:3] offset:-176
-; GCN-NEXT:    s_prefetch_data s[2:3], 0x0, null, 0
-; GCN-NEXT:    s_add_co_i32 s6, s6, -1
-; GCN-NEXT:    s_add_nc_u64 s[2:3], s[2:3], 16
-; GCN-NEXT:    s_cmp_lg_u32 s6, 0
-; GCN-NEXT:    s_wait_loadcnt 0x0
-; GCN-NEXT:    global_store_b128 v0, v[1:4], s[0:1]
-; GCN-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 16
-; GCN-NEXT:    s_cbranch_scc1 .LBB1_2
-; GCN-NEXT:  .LBB1_3: ; %for.end
-; GCN-NEXT:    s_endpgm
+; GFX12-LABEL: copy_global:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX12-NEXT:    s_cbranch_scc1 .LBB1_3
+; GFX12-NEXT:  ; %bb.1: ; %for.body.preheader
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], 0xb0
+; GFX12-NEXT:  .LBB1_2: ; %for.body
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    global_load_b128 v[1:4], v0, s[2:3] offset:-176
+; GFX12-NEXT:    s_add_co_i32 s6, s6, -1
+; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], 16
+; GFX12-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    global_store_b128 v0, v[1:4], s[0:1]
+; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 16
+; GFX12-NEXT:    s_cbranch_scc1 .LBB1_2
+; GFX12-NEXT:  .LBB1_3: ; %for.end
+; GFX12-NEXT:    s_endpgm
+;
+; GFX12-SPREFETCH-LABEL: copy_global:
+; GFX12-SPREFETCH:       ; %bb.0: ; %entry
+; GFX12-SPREFETCH-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-SPREFETCH-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SPREFETCH-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX12-SPREFETCH-NEXT:    s_cbranch_scc1 .LBB1_3
+; GFX12-SPREFETCH-NEXT:  ; %bb.1: ; %for.body.preheader
+; GFX12-SPREFETCH-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-SPREFETCH-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-SPREFETCH-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SPREFETCH-NEXT:    s_add_nc_u64 s[2:3], s[2:3], 0xb0
+; GFX12-SPREFETCH-NEXT:  .LBB1_2: ; %for.body
+; GFX12-SPREFETCH-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-SPREFETCH-NEXT:    global_load_b128 v[1:4], v0, s[2:3] offset:-176
+; GFX12-SPREFETCH-NEXT:    s_prefetch_data s[2:3], 0x0, null, 0
+; GFX12-SPREFETCH-NEXT:    s_add_co_i32 s6, s6, -1
+; GFX12-SPREFETCH-NEXT:    s_add_nc_u64 s[2:3], s[2:3], 16
+; GFX12-SPREFETCH-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX12-SPREFETCH-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SPREFETCH-NEXT:    global_store_b128 v0, v[1:4], s[0:1]
+; GFX12-SPREFETCH-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 16
+; GFX12-SPREFETCH-NEXT:    s_cbranch_scc1 .LBB1_2
+; GFX12-SPREFETCH-NEXT:  .LBB1_3: ; %for.end
+; GFX12-SPREFETCH-NEXT:    s_endpgm
 entry:
   %cmp6.not = icmp eq i32 %n, 0
   br i1 %cmp6.not, label %for.end, label %for.body
@@ -92,31 +143,56 @@ for.end:                                          ; preds = %for.body, %entry
 }
 
 define amdgpu_kernel void @copy_constant(ptr addrspace(1) nocapture %d, ptr addrspace(4) nocapture readonly %s, i32 %n) {
-; GCN-LABEL: copy_constant:
-; GCN:       ; %bb.0: ; %entry
-; GCN-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; GCN-NEXT:    s_wait_kmcnt 0x0
-; GCN-NEXT:    s_cmp_eq_u32 s6, 0
-; GCN-NEXT:    s_cbranch_scc1 .LBB2_3
-; GCN-NEXT:  ; %bb.1: ; %for.body.preheader
-; GCN-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GCN-NEXT:    v_mov_b32_e32 v0, 0
-; GCN-NEXT:  .LBB2_2: ; %for.body
-; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT:    s_wait_kmcnt 0x0
-; GCN-NEXT:    s_load_b128 s[8:11], s[2:3], 0x0
-; GCN-NEXT:    s_prefetch_data s[2:3], 0xb0, null, 0
-; GCN-NEXT:    s_add_co_i32 s6, s6, -1
-; GCN-NEXT:    s_add_nc_u64 s[2:3], s[2:3], 16
-; GCN-NEXT:    s_cmp_lg_u32 s6, 0
-; GCN-NEXT:    s_wait_kmcnt 0x0
-; GCN-NEXT:    v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s9
-; GCN-NEXT:    v_dual_mov_b32 v3, s10 :: v_dual_mov_b32 v4, s11
-; GCN-NEXT:    global_store_b128 v0, v[1:4], s[0:1]
-; GCN-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 16
-; GCN-NEXT:    s_cbranch_scc1 .LBB2_2
-; GCN-NEXT:  .LBB2_3: ; %for.end
-; GCN-NEXT:    s_endpgm
+; GFX12-LABEL: copy_constant:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX12-NEXT:    s_cbranch_scc1 .LBB2_3
+; GFX12-NEXT:  ; %bb.1: ; %for.body.preheader
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-NEXT:  .LBB2_2: ; %for.body
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_load_b128 s[8:11], s[2:3], 0x0
+; GFX12-NEXT:    s_add_co_i32 s6, s6, -1
+; GFX12-NEXT:    s_add_nc_u64 s[2:3], s[2:3], 16
+; GFX12-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s9
+; GFX12-NEXT:    v_dual_mov_b32 v3, s10 :: v_dual_mov_b32 v4, s11
+; GFX12-NEXT:    global_store_b128 v0, v[1:4], s[0:1]
+; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 16
+; GFX12-NEXT:    s_cbranch_scc1 .LBB2_2
+; GFX12-NEXT:  .LBB2_3: ; %for.end
+; GFX12-NEXT:    s_endpgm
+;
+; GFX12-SPREFETCH-LABEL: copy_constant:
+; GFX12-SPREFETCH:       ; %bb.0: ; %entry
+; GFX12-SPREFETCH-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-SPREFETCH-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SPREFETCH-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX12-SPREFETCH-NEXT:    s_cbranch_scc1 .LBB2_3
+; GFX12-SPREFETCH-NEXT:  ; %bb.1: ; %for.body.preheader
+; GFX12-SPREFETCH-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-SPREFETCH-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-SPREFETCH-NEXT:  .LBB2_2: ; %for.body
+; GFX12-SPREFETCH-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-SPREFETCH-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SPREFETCH-NEXT:    s_load_b128 s[8:11], s[2:3], 0x0
+; GFX12-SPREFETCH-NEXT:    s_prefetch_data s[2:3], 0xb0, null, 0
+; GFX12-SPREFETCH-NEXT:    s_add_co_i32 s6, s6, -1
+; GFX12-SPREFETCH-NEXT:    s_add_nc_u64 s[2:3], s[2:3], 16
+; GFX12-SPREFETCH-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX12-SPREFETCH-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SPREFETCH-NEXT:    v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s9
+; GFX12-SPREFETCH-NEXT:    v_dual_mov_b32 v3, s10 :: v_dual_mov_b32 v4, s11
+; GFX12-SPREFETCH-NEXT:    global_store_b128 v0, v[1:4], s[0:1]
+; GFX12-SPREFETCH-NEXT:    s_add_nc_u64 s[0:1], s[0:1], 16
+; GFX12-SPREFETCH-NEXT:    s_cbranch_scc1 .LBB2_2
+; GFX12-SPREFETCH-NEXT:  .LBB2_3: ; %for.end
+; GFX12-SPREFETCH-NEXT:    s_endpgm
 entry:
   %cmp6.not = icmp eq i32 %n, 0
   br i1 %cmp6.not, label %for.end, label %for.body
@@ -137,30 +213,55 @@ for.end:                                          ; preds = %for.body, %entry
 }
 
 define amdgpu_kernel void @copy_local(ptr addrspace(3) nocapture %d, ptr addrspace(3) nocapture readonly %s, i32 %n) {
-; GCN-LABEL: copy_local:
-; GCN:       ; %bb.0: ; %entry
-; GCN-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
-; GCN-NEXT:    s_wait_kmcnt 0x0
-; GCN-NEXT:    s_cmp_eq_u32 s2, 0
-; GCN-NEXT:    s_cbranch_scc1 .LBB3_2
-; GCN-NEXT:  .LBB3_1: ; %for.body
-; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT:    s_wait_alu 0xfffe
-; GCN-NEXT:    v_mov_b32_e32 v2, s1
-; GCN-NEXT:    v_mov_b32_e32 v4, s0
-; GCN-NEXT:    s_add_co_i32 s2, s2, -1
-; GCN-NEXT:    s_add_co_i32 s0, s0, 16
-; GCN-NEXT:    s_add_co_i32 s1, s1, 16
-; GCN-NEXT:    ds_load_2addr_b32 v[0:1], v2 offset0:2 offset1:3
-; GCN-NEXT:    ds_load_2addr_b32 v[2:3], v2 offset1:1
-; GCN-NEXT:    s_cmp_lg_u32 s2, 0
-; GCN-NEXT:    s_wait_dscnt 0x1
-; GCN-NEXT:    ds_store_2addr_b32 v4, v0, v1 offset0:2 offset1:3
-; GCN-NEXT:    s_wait_dscnt 0x1
-; GCN-NEXT:    ds_store_2addr_b32 v4, v2, v3 offset1:1
-; GCN-NEXT:    s_cbranch_scc1 .LBB3_1
-; GCN-NEXT:  .LBB3_2: ; %for.end
-; GCN-NEXT:    s_endpgm
+; GFX12-LABEL: copy_local:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    s_cmp_eq_u32 s2, 0
+; GFX12-NEXT:    s_cbranch_scc1 .LBB3_2
+; GFX12-NEXT:  .LBB3_1: ; %for.body
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    v_mov_b32_e32 v2, s1
+; GFX12-NEXT:    v_mov_b32_e32 v4, s0
+; GFX12-NEXT:    s_add_co_i32 s2, s2, -1
+; GFX12-NEXT:    s_add_co_i32 s0, s0, 16
+; GFX12-NEXT:    s_add_co_i32 s1, s1, 16
+; GFX12-NEXT:    ds_load_2addr_b32 v[0:1], v2 offset0:2 offset1:3
+; GFX12-NEXT:    ds_load_2addr_b32 v[2:3], v2 offset1:1
+; GFX12-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX12-NEXT:    s_wait_dscnt 0x1
+; GFX12-NEXT:    ds_store_2addr_b32 v4, v0, v1 offset0:2 offset1:3
+; GFX12-NEXT:    s_wait_dscnt 0x1
+; GFX12-NEXT:    ds_store_2addr_b32 v4, v2, v3 offset1:1
+; GFX12-NEXT:    s_cbranch_scc1 .LBB3_1
+; GFX12-NEXT:  .LBB3_2: ; %for.end
+; GFX12-NEXT:    s_endpgm
+;
+; GFX12-SPREFETCH-LABEL: copy_local:
+; GFX12-SPREFETCH:       ; %bb.0: ; %entry
+; GFX12-SPREFETCH-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
+; GFX12-SPREFETCH-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SPREFETCH-NEXT:    s_cmp_eq_u32 s2, 0
+; GFX12-SPREFETCH-NEXT:    s_cbranch_scc1 .LBB3_2
+; GFX12-SPREFETCH-NEXT:  .LBB3_1: ; %for.body
+; GFX12-SPREFETCH-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-SPREFETCH-NEXT:    s_wait_alu 0xfffe
+; GFX12-SPREFETCH-NEXT:    v_mov_b32_e32 v2, s1
+; GFX12-SPREFETCH-NEXT:    v_mov_b32_e32 v4, s0
+; GFX12-SPREFETCH-NEXT:    s_add_co_i32 s2, s2, -1
+; GFX12-SPREFETCH-NEXT:    s_add_co_i32 s0, s0, 16
+; GFX12-SPREFETCH-NEXT:    s_add_co_i32 s1, s1, 16
+; GFX12-SPREFETCH-NEXT:    ds_load_2addr_b32 v[0:1], v2 offset0:2 offset1:3
+; GFX12-SPREFETCH-NEXT:    ds_load_2addr_b32 v[2:3], v2 offset1:1
+; GFX12-SPREFETCH-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX12-SPREFETCH-NEXT:    s_wait_dscnt 0x1
+; GFX12-SPREFETCH-NEXT:    ds_store_2addr_b32 v4, v0, v1 offset0:2 offset1:3
+; GFX12-SPREFETCH-NEXT:    s_wait_dscnt 0x1
+; GFX12-SPREFETCH-NEXT:    ds_store_2addr_b32 v4, v2, v3 offset1:1
+; GFX12-SPREFETCH-NEXT:    s_cbranch_scc1 .LBB3_1
+; GFX12-SPREFETCH-NEXT:  .LBB3_2: ; %for.end
+; GFX12-SPREFETCH-NEXT:    s_endpgm
 entry:
   %cmp6.not = icmp eq i32 %n, 0
   br i1 %cmp6.not, label %for.end, label %for.body
diff --git a/llvm/test/CodeGen/AMDGPU/loop_break.ll b/llvm/test/CodeGen/AMDGPU/loop_break.ll
index 6e53d40631086..223efc5dd1912 100644
--- a/llvm/test/CodeGen/AMDGPU/loop_break.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop_break.ll
@@ -5,28 +5,28 @@
 ; Uses llvm.amdgcn.break
 
 define amdgpu_kernel void @break_loop(i32 %arg) #0 {
-; OPT-LABEL: @break_loop(
-; OPT-NEXT:  bb:
+; OPT-LABEL: define amdgpu_kernel void @break_loop(
+; OPT-SAME: i32 [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
+; OPT-NEXT:  [[BB:.*]]:
 ; OPT-NEXT:    [[ID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; OPT-NEXT:    [[MY_TMP:%.*]] = sub i32 [[ID]], [[ARG:%.*]]
-; OPT-NEXT:    br label [[BB1:%.*]]
-; OPT:       bb1:
-; OPT-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP2:%.*]], [[FLOW:%.*]] ], [ 0, [[BB:%.*]] ]
-; OPT-NEXT:    [[LSR_IV:%.*]] = phi i32 [ undef, [[BB]] ], [ [[TMP0:%.*]], [[FLOW]] ]
-; OPT-NEXT:    [[LSR_IV_NEXT:%.*]] = add i32 [[LSR_IV]], 1
-; OPT-NEXT:    [[CMP0:%.*]] = icmp slt i32 [[LSR_IV_NEXT]], 0
-; OPT-NEXT:    br i1 [[CMP0]], label [[BB4:%.*]], label [[FLOW]]
-; OPT:       bb4:
+; OPT-NEXT:    [[MY_TMP:%.*]] = sub i32 [[ID]], [[ARG]]
+; OPT-NEXT:    br label %[[BB1:.*]]
+; OPT:       [[BB1]]:
+; OPT-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP2:%.*]], %[[FLOW:.*]] ], [ 0, %[[BB]] ]
+; OPT-NEXT:    [[LSR_IV:%.*]] = phi i32 [ undef, %[[BB]] ], [ [[TMP0:%.*]], %[[FLOW]] ]
+; OPT-NEXT:    [[TMP0]] = add i32 [[LSR_IV]], 1
+; OPT-NEXT:    [[CMP0:%.*]] = icmp slt i32 [[TMP0]], 0
+; OPT-NEXT:    br i1 [[CMP0]], label %[[BB4:.*]], label %[[FLOW]]
+; OPT:       [[BB4]]:
 ; OPT-NEXT:    [[LOAD:%.*]] = load volatile i32, ptr addrspace(1) undef, align 4
 ; OPT-NEXT:    [[CMP1:%.*]] = icmp sge i32 [[MY_TMP]], [[LOAD]]
-; OPT-NEXT:    br label [[FLOW]]
-; OPT:       Flow:
-; OPT-NEXT:    [[TMP0]] = phi i32 [ [[LSR_IV_NEXT]], [[BB4]] ], [ undef, [[BB1]] ]
-; OPT-NEXT:    [[TMP1:%.*]] = phi i1 [ [[CMP1]], [[BB4]] ], [ true, [[BB1]] ]
+; OPT-NEXT:    br label %[[FLOW]]
+; OPT:       [[FLOW]]:
+; OPT-NEXT:    [[TMP1:%.*]] = phi i1 [ [[CMP1]], %[[BB4]] ], [ true, %[[BB1]] ]
 ; OPT-NEXT:    [[TMP2]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP1]], i64 [[PHI_BROKEN]])
 ; OPT-NEXT:    [[TMP3:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP2]])
-; OPT-NEXT:    br i1 [[TMP3]], label [[BB9:%.*]], label [[BB1]]
-; OPT:       bb9:
+; OPT-NEXT:    br i1 [[TMP3]], label %[[BB9:.*]], label %[[BB1]]
+; OPT:       [[BB9]]:
 ; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP2]])
 ; OPT-NEXT:    ret void
 ;
@@ -45,11 +45,8 @@ define amdgpu_kernel void @break_loop(i32 %arg) #0 {
 ; GCN-NEXT:    s_add_i32 s6, s6, 1
 ; GCN-NEXT:    s_or_b64 s[4:5], s[4:5], exec
 ; GCN-NEXT:    s_cmp_gt_i32 s6, -1
-; GCN-NEXT:    s_cbranch_scc0 .LBB0_3
-; GCN-NEXT:  ; %bb.2: ; in Loop: Header=BB0_1 Depth=1
-; GCN-NEXT:    ; implicit-def: $sgpr6
-; GCN-NEXT:    s_branch .LBB0_4
-; GCN-NEXT:  .LBB0_3: ; %bb4
+; GCN-NEXT:    s_cbranch_scc1 .LBB0_3
+; GCN-NEXT:  ; %bb.2: ; %bb4
 ; GCN-NEXT:    ; in Loop: Header=BB0_1 Depth=1
 ; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], 0 glc
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
@@ -57,13 +54,13 @@ define amdgpu_kernel void @break_loop(i32 %arg) #0 {
 ; GCN-NEXT:    s_andn2_b64 s[4:5], s[4:5], exec
 ; GCN-NEXT:    s_and_b64 s[8:9], vcc, exec
 ; GCN-NEXT:    s_or_b64 s[4:5], s[4:5], s[8:9]
-; GCN-NEXT:  .LBB0_4: ; %Flow
+; GCN-NEXT:  .LBB0_3: ; %Flow
 ; GCN-NEXT:    ; in Loop: Header=BB0_1 Depth=1
 ; GCN-NEXT:    s_and_b64 s[8:9], exec, s[4:5]
 ; GCN-NEXT:    s_or_b64 s[0:1], s[8:9], s[0:1]
 ; GCN-NEXT:    s_andn2_b64 exec, exec, s[0:1]
 ; GCN-NEXT:    s_cbranch_execnz .LBB0_1
-; GCN-NEXT:  ; %bb.5: ; %bb9
+; GCN-NEXT:  ; %bb.4: ; %bb9
 ; GCN-NEXT:    s_endpgm
 bb:
   %id = call i32 @llvm.amdgcn.workitem.id.x()
@@ -86,28 +83,29 @@ bb9:
 }
 
 define amdgpu_kernel void @undef_phi_cond_break_loop(i32 %arg) #0 {
-; OPT-LABEL: @undef_phi_cond_break_loop(
-; OPT-NEXT:  bb:
+; OPT-LABEL: define amdgpu_kernel void @undef_phi_cond_break_loop(
+; OPT-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; OPT-NEXT:  [[BB:.*]]:
 ; OPT-NEXT:    [[ID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; OPT-NEXT:    [[MY_TMP:%.*]] = sub i32 [[ID]], [[ARG:%.*]]
-; OPT-NEXT:    br label [[BB1:%.*]]
-; OPT:       bb1:
-; OPT-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP0:%.*]], [[FLOW:%.*]] ], [ 0, [[BB:%.*]] ]
-; OPT-NEXT:    [[LSR_IV:%.*]] = phi i32 [ undef, [[BB]] ], [ [[MY_TMP2:%.*]], [[FLOW]] ]
+; OPT-NEXT:    [[MY_TMP:%.*]] = sub i32 [[ID]], [[ARG]]
+; OPT-NEXT:    br label %[[BB1:.*]]
+; OPT:       [[BB1]]:
+; OPT-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP0:%.*]], %[[FLOW:.*]] ], [ 0, %[[BB]] ]
+; OPT-NEXT:    [[LSR_IV:%.*]] = phi i32 [ undef, %[[BB]] ], [ [[MY_TMP2:%.*]], %[[FLOW]] ]
 ; OPT-NEXT:    [[LSR_IV_NEXT:%.*]] = add i32 [[LSR_IV]], 1
 ; OPT-NEXT:    [[CMP0:%.*]] = icmp slt i32 [[LSR_IV_NEXT]], 0
-; OPT-NEXT:    br i1 [[CMP0]], label [[BB4:%.*]], label [[FLOW]]
-; OPT:       bb4:
+; OPT-NEXT:    br i1 [[CMP0]], label %[[BB4:.*]], label %[[FLOW]]
+; OPT:       [[BB4]]:
 ; OPT-NEXT:    [[LOAD:%.*]] = load volatile i32, ptr addrspace(1) undef, align 4
 ; OPT-NEXT:    [[CMP1:%.*]] = icmp sge i32 [[MY_TMP]], [[LOAD]]
-; OPT-NEXT:    br label [[FLOW]]
-; OPT:       Flow:
-; OPT-NEXT:    [[MY_TMP2]] = phi i32 [ [[LSR_IV_NEXT]], [[BB4]] ], [ undef, [[BB1]] ]
-; OPT-NEXT:    [[MY_TMP3:%.*]] = phi i1 [ [[CMP1]], [[BB4]] ], [ undef, [[BB1]] ]
+; OPT-NEXT:    br label %[[FLOW]]
+; OPT:       [[FLOW]]:
+; OPT-NEXT:    [[MY_TMP2]] = phi i32 [ [[LSR_IV_NEXT]], %[[BB4]] ], [ undef, %[[BB1]] ]
+; OPT-NEXT:    [[MY_TMP3:%.*]] = phi i1 [ [[CMP1]], %[[BB4]] ], [ undef, %[[BB1]] ]
 ; OPT-NEXT:    [[TMP0]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[MY_TMP3]], i64 [[PHI_BROKEN]])
 ; OPT-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP0]])
-; OPT-NEXT:    br i1 [[TMP1]], label [[BB9:%.*]], label [[BB1]]
-; OPT:       bb9:
+; OPT-NEXT:    br i1 [[TMP1]], label %[[BB9:.*]], label %[[BB1]]
+; OPT:       [[BB9]]:
 ; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP0]])
 ; OPT-NEXT:    store volatile i32 7, ptr addrspace(3) undef, align 4
 ; OPT-NEXT:    ret void
@@ -174,29 +172,30 @@ bb9:                                              ; preds = %Flow
 @lds = addrspace(3) global i32 undef
 
 define amdgpu_kernel void @constexpr_phi_cond_break_loop(i32 %arg) #0 {
-; OPT-LABEL: @constexpr_phi_cond_break_loop(
-; OPT-NEXT:  bb:
+; OPT-LABEL: define amdgpu_kernel void @constexpr_phi_cond_break_loop(
+; OPT-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; OPT-NEXT:  [[BB:.*]]:
 ; OPT-NEXT:    [[ID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; OPT-NEXT:    [[MY_TMP:%.*]] = sub i32 [[ID]], [[ARG:%.*]]
-; OPT-NEXT:    br label [[BB1:%.*]]
-; OPT:       bb1:
-; OPT-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP0:%.*]], [[FLOW:%.*]] ], [ 0, [[BB:%.*]] ]
-; OPT-NEXT:    [[LSR_IV:%.*]] = phi i32 [ undef, [[BB]] ], [ [[MY_TMP2:%.*]], [[FLOW]] ]
+; OPT-NEXT:    [[MY_TMP:%.*]] = sub i32 [[ID]], [[ARG]]
+; OPT-NEXT:    br label %[[BB1:.*]]
+; OPT:       [[BB1]]:
+; OPT-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP0:%.*]], %[[FLOW:.*]] ], [ 0, %[[BB]] ]
+; OPT-NEXT:    [[LSR_IV:%.*]] = phi i32 [ undef, %[[BB]] ], [ [[MY_TMP2:%.*]], %[[FLOW]] ]
 ; OPT-NEXT:    [[LSR_IV_NEXT:%.*]] = add i32 [[LSR_IV]], 1
 ; OPT-NEXT:    [[CMP0:%.*]] = icmp slt i32 [[LSR_IV_NEXT]], 0
 ; OPT-NEXT:    [[CMP2:%.*]] = icmp ne ptr addrspace(3) inttoptr (i32 4 to ptr addrspace(3)), @lds
-; OPT-NEXT:    br i1 [[CMP0]], label [[BB4:%.*]], label [[FLOW]]
-; OPT:       bb4:
+; OPT-NEXT:    br i1 [[CMP0]], label %[[BB4:.*]], label %[[FLOW]]
+; OPT:       [[BB4]]:
 ; OPT-NEXT:    [[LOAD:%.*]] = load volatile i32, ptr addrspace(1) undef, align 4
 ; OPT-NEXT:    [[CMP1:%.*]] = icmp sge i32 [[MY_TMP]], [[LOAD]]
-; OPT-NEXT:    br label [[FLOW]]
-; OPT:       Flow:
-; OPT-NEXT:    [[MY_TMP2]] = phi i32 [ [[LSR_IV_NEXT]], [[BB4]] ], [ undef, [[BB1]] ]
-; OPT-NEXT:    [[MY_TMP3:%.*]] = phi i1 [ [[CMP1]], [[BB4]] ], [ [[CMP2]], [[BB1]] ]
+; OPT-NEXT:    br label %[[FLOW]]
+; OPT:       [[FLOW]]:
+; OPT-NEXT:    [[MY_TMP2]] = phi i32 [ [[LSR_IV_NEXT]], %[[BB4]] ], [ undef, %[[BB1]] ]
+; OPT-NEXT:    [[MY_TMP3:%.*]] = phi i1 [ [[CMP1]], %[[BB4]] ], [ [[CMP2]], %[[BB1]] ]
 ; OPT-NEXT:    [[TMP0]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[MY_TMP3]], i64 [[PHI_BROKEN]])
 ; OPT-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP0]])
-; OPT-NEXT:    br i1 [[TMP1]], label [[BB9:%.*]], label [[BB1]]
-; OPT:       bb9:
+; OPT-NEXT:    br i1 [[TMP1]], label %[[BB9:.*]], label %[[BB1]]
+; OPT:       [[BB9]]:
 ; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP0]])
 ; OPT-NEXT:    store volatile i32 7, ptr addrspace(3) undef, align 4
 ; OPT-NEXT:    ret void
@@ -265,28 +264,29 @@ bb9:                                              ; preds = %Flow
 }
 
 define amdgpu_kernel void @true_phi_cond_break_loop(i32 %arg) #0 {
-; OPT-LABEL: @true_phi_cond_break_loop(
-; OPT-NEXT:  bb:
+; OPT-LABEL: define amdgpu_kernel void @true_phi_cond_break_loop(
+; OPT-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; OPT-NEXT:  [[BB:.*]]:
 ; OPT-NEXT:    [[ID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; OPT-NEXT:    [[MY_TMP:%.*]] = sub i32 [[ID]], [[ARG:%.*]]
-; OPT-NEXT:    br label [[BB1:%.*]]
-; OPT:       bb1:
-; OPT-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP0:%.*]], [[FLOW:%.*]] ], [ 0, [[BB:%.*]] ]
-; OPT-NEXT:    [[LSR_IV:%.*]] = phi i32 [ undef, [[BB]] ], [ [[MY_TMP2:%.*]], [[FLOW]] ]
+; OPT-NEXT:    [[MY_TMP:%.*]] = sub i32 [[ID]], [[ARG]]
+; OPT-NEXT:    br label %[[BB1:.*]]
+; OPT:       [[BB1]]:
+; OPT-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP0:%.*]], %[[FLOW:.*]] ], [ 0, %[[BB]] ]
+; OPT-NEXT:    [[LSR_IV:%.*]] = phi i32 [ undef, %[[BB]] ], [ [[MY_TMP2:%.*]], %[[FLOW]] ]
 ; OPT-NEXT:    [[LSR_IV_NEXT:%.*]] = add i32 [[LSR_IV]], 1
 ; OPT-NEXT:    [[CMP0:%.*]] = icmp slt i32 [[LSR_IV_NEXT]], 0
-; OPT-NEXT:    br i1 [[CMP0]], label [[BB4:%.*]], label [[FLOW]]
-; OPT:       bb4:
+; OPT-NEXT:    br i1 [[CMP0]], label %[[BB4:.*]], label %[[FLOW]]
+; OPT:       [[BB4]]:
 ; OPT-NEXT:    [[LOAD:%.*]] = load volatile i32, ptr addrspace(1) undef, align 4
 ; OPT-NEXT:    [[CMP1:%.*]] = icmp sge i32 [[MY_TMP]], [[LOAD]]
-; OPT-NEXT:    br label [[FLOW]]
-; OPT:       Flow:
-; OPT-NEXT:    [[MY_TMP2]] = phi i32 [ [[LSR_IV_NEXT]], [[BB4]] ], [ undef, [[BB1]] ]
-; OPT-NEXT:    [[MY_TMP3:%.*]] = phi i1 [ [[CMP1]], [[BB4]] ], [ true, [[BB1]] ]
+; OPT-NEXT:    br label %[[FLOW]]
+; OPT:       [[FLOW]]:
+; OPT-NEXT:    [[MY_TMP2]] = phi i32 [ [[LSR_IV_NEXT]], %[[BB4]] ], [ undef, %[[BB1]] ]
+; OPT-NEXT:    [[MY_TMP3:%.*]] = phi i1 [ [[CMP1]], %[[BB4]] ], [ true, %[[BB1]] ]
 ; OPT-NEXT:    [[TMP0]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[MY_TMP3]], i64 [[PHI_BROKEN]])
 ; OPT-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP0]])
-; OPT-NEXT:    br i1 [[TMP1]], label [[BB9:%.*]], label [[BB1]]
-; OPT:       bb9:
+; OPT-NEXT:    br i1 [[TMP1]], label %[[BB9:.*]], label %[[BB1]]
+; OPT:       [[BB9]]:
 ; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP0]])
 ; OPT-NEXT:    store volatile i32 7, ptr addrspace(3) undef, align 4
 ; OPT-NEXT:    ret void
@@ -354,28 +354,29 @@ bb9:                                              ; preds = %Flow
 }
 
 define amdgpu_kernel void @false_phi_cond_break_loop(i32 %arg) #0 {
-; OPT-LABEL: @false_phi_cond_break_loop(
-; OPT-NEXT:  bb:
+; OPT-LABEL: define amdgpu_kernel void @false_phi_cond_break_loop(
+; OPT-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; OPT-NEXT:  [[BB:.*]]:
 ; OPT-NEXT:    [[ID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; OPT-NEXT:    [[MY_TMP:%.*]] = sub i32 [[ID]], [[ARG:%.*]]
-; OPT-NEXT:    br label [[BB1:%.*]]
-; OPT:       bb1:
-; OPT-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP0:%.*]], [[FLOW:%.*]] ], [ 0, [[BB:%.*]] ]
-; OPT-NEXT:    [[LSR_IV:%.*]] = phi i32 [ undef, [[BB]] ], [ [[MY_TMP2:%.*]], [[FLOW]] ]
+; OPT-NEXT:    [[MY_TMP:%.*]] = sub i32 [[ID]], [[ARG]]
+; OPT-NEXT:    br label %[[BB1:.*]]
+; OPT:       [[BB1]]:
+; OPT-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP0:%.*]], %[[FLOW:.*]] ], [ 0, %[[BB]] ]
+; OPT-NEXT:    [[LSR_IV:%.*]] = phi i32 [ undef, %[[BB]] ], [ [[MY_TMP2:%.*]], %[[FLOW]] ]
 ; OPT-NEXT:    [[LSR_IV_NEXT:%.*]] = add i32 [[LSR_IV]], 1
 ; OPT-NEXT:    [[CMP0:%.*]] = icmp slt i32 [[LSR_IV_NEXT]], 0
-; OPT-NEXT:    br i1 [[CMP0]], label [[BB4:%.*]], label [[FLOW]]
-; OPT:       bb4:
+; OPT-NEXT:    br i1 [[CMP0]], label %[[BB4:.*]], label %[[FLOW]]
+; OPT:       [[BB4]]:
 ; OPT-NEXT:    [[LOAD:%.*]] = load volatile i32, ptr addrspace(1) undef, align 4
 ; OPT-NEXT:    [[CMP1:%.*]] = icmp sge i32 [[MY_TMP]], [[LOAD]]
-; OPT-NEXT:    br label [[FLOW]]
-; OPT:       Flow:
-; OPT-NEXT:    [[MY_TMP2]] = phi i32 [ [[LSR_IV_NEXT]], [[BB4]] ], [ undef, [[BB1]] ]
-; OPT-NEXT:    [[MY_TMP3:%.*]] = phi i1 [ [[CMP1]], [[BB4]] ], [ false, [[BB1]] ]
+; OPT-NEXT:    br label %[[FLOW]]
+; OPT:       [[FLOW]]:
+; OPT-NEXT:    [[MY_TMP2]] = phi i32 [ [[LSR_IV_NEXT]], %[[BB4]] ], [ undef, %[[BB1]] ]
+; OPT-NEXT:    [[MY_TMP3:%.*]] = phi i1 [ [[CMP1]], %[[BB4]] ], [ false, %[[BB1]] ]
 ; OPT-NEXT:    [[TMP0]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[MY_TMP3]], i64 [[PHI_BROKEN]])
 ; OPT-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP0]])
-; OPT-NEXT:    br i1 [[TMP1]], label [[BB9:%.*]], label [[BB1]]
-; OPT:       bb9:
+; OPT-NEXT:    br i1 [[TMP1]], label %[[BB9:.*]], label %[[BB1]]
+; OPT:       [[BB9]]:
 ; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP0]])
 ; OPT-NEXT:    store volatile i32 7, ptr addrspace(3) undef, align 4
 ; OPT-NEXT:    ret void
@@ -446,29 +447,30 @@ bb9:                                              ; preds = %Flow
 ; continue.
 
 define amdgpu_kernel void @invert_true_phi_cond_break_loop(i32 %arg) #0 {
-; OPT-LABEL: @invert_true_phi_cond_break_loop(
-; OPT-NEXT:  bb:
+; OPT-LABEL: define amdgpu_kernel void @invert_true_phi_cond_break_loop(
+; OPT-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; OPT-NEXT:  [[BB:.*]]:
 ; OPT-NEXT:    [[ID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; OPT-NEXT:    [[MY_TMP:%.*]] = sub i32 [[ID]], [[ARG:%.*]]
-; OPT-NEXT:    br label [[BB1:%.*]]
-; OPT:       bb1:
-; OPT-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP0:%.*]], [[FLOW:%.*]] ], [ 0, [[BB:%.*]] ]
-; OPT-NEXT:    [[LSR_IV:%.*]] = phi i32 [ undef, [[BB]] ], [ [[MY_TMP2:%.*]], [[FLOW]] ]
+; OPT-NEXT:    [[MY_TMP:%.*]] = sub i32 [[ID]], [[ARG]]
+; OPT-NEXT:    br label %[[BB1:.*]]
+; OPT:       [[BB1]]:
+; OPT-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP0:%.*]], %[[FLOW:.*]] ], [ 0, %[[BB]] ]
+; OPT-NEXT:    [[LSR_IV:%.*]] = phi i32 [ undef, %[[BB]] ], [ [[MY_TMP2:%.*]], %[[FLOW]] ]
 ; OPT-NEXT:    [[LSR_IV_NEXT:%.*]] = add i32 [[LSR_IV]], 1
 ; OPT-NEXT:    [[CMP0:%.*]] = icmp slt i32 [[LSR_IV_NEXT]], 0
-; OPT-NEXT:    br i1 [[CMP0]], label [[BB4:%.*]], label [[FLOW]]
-; OPT:       bb4:
+; OPT-NEXT:    br i1 [[CMP0]], label %[[BB4:.*]], label %[[FLOW]]
+; OPT:       [[BB4]]:
 ; OPT-NEXT:    [[LOAD:%.*]] = load volatile i32, ptr addrspace(1) undef, align 4
 ; OPT-NEXT:    [[CMP1:%.*]] = icmp sge i32 [[MY_TMP]], [[LOAD]]
-; OPT-NEXT:    br label [[FLOW]]
-; OPT:       Flow:
-; OPT-NEXT:    [[MY_TMP2]] = phi i32 [ [[LSR_IV_NEXT]], [[BB4]] ], [ undef, [[BB1]] ]
-; OPT-NEXT:    [[MY_TMP3:%.*]] = phi i1 [ [[CMP1]], [[BB4]] ], [ true, [[BB1]] ]
+; OPT-NEXT:    br label %[[FLOW]]
+; OPT:       [[FLOW]]:
+; OPT-NEXT:    [[MY_TMP2]] = phi i32 [ [[LSR_IV_NEXT]], %[[BB4]] ], [ undef, %[[BB1]] ]
+; OPT-NEXT:    [[MY_TMP3:%.*]] = phi i1 [ [[CMP1]], %[[BB4]] ], [ true, %[[BB1]] ]
 ; OPT-NEXT:    [[MY_TMP3_INV:%.*]] = xor i1 [[MY_TMP3]], true
 ; OPT-NEXT:    [[TMP0]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[MY_TMP3_INV]], i64 [[PHI_BROKEN]])
 ; OPT-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP0]])
-; OPT-NEXT:    br i1 [[TMP1]], label [[BB9:%.*]], label [[BB1]]
-; OPT:       bb9:
+; OPT-NEXT:    br i1 [[TMP1]], label %[[BB9:.*]], label %[[BB1]]
+; OPT:       [[BB9]]:
 ; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP0]])
 ; OPT-NEXT:    store volatile i32 7, ptr addrspace(3) undef, align 4
 ; OPT-NEXT:    ret void
diff --git a/llvm/test/CodeGen/AMDGPU/lower-work-group-id-intrinsics-hsa.ll b/llvm/test/CodeGen/AMDGPU/lower-work-group-id-intrinsics-hsa.ll
index 989ef6f981d9d..018cdee038b91 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-work-group-id-intrinsics-hsa.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-work-group-id-intrinsics-hsa.ll
@@ -58,7 +58,7 @@ define amdgpu_kernel void @workgroup_ids_kernel() {
   %idx = call i32 @llvm.amdgcn.workgroup.id.x()
   %idy = call i32 @llvm.amdgcn.workgroup.id.y()
   %idz = call i32 @llvm.amdgcn.workgroup.id.z()
-  %ielemx = insertelement <3 x i32> undef, i32 %idx, i64 0
+  %ielemx = insertelement <3 x i32> poison, i32 %idx, i64 0
   %ielemy = insertelement <3 x i32> %ielemx, i32 %idy, i64 1
   %ielemz = insertelement <3 x i32> %ielemy, i32 %idz, i64 2
   call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> %ielemz, ptr addrspace(8) undef, i32 0, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/lower-work-group-id-intrinsics-pal.ll b/llvm/test/CodeGen/AMDGPU/lower-work-group-id-intrinsics-pal.ll
index 356439bad8f00..dcec7288686b1 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-work-group-id-intrinsics-pal.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-work-group-id-intrinsics-pal.ll
@@ -65,7 +65,7 @@ define amdgpu_cs void @_amdgpu_cs_main() {
   %idx = call i32 @llvm.amdgcn.workgroup.id.x()
   %idy = call i32 @llvm.amdgcn.workgroup.id.y()
   %idz = call i32 @llvm.amdgcn.workgroup.id.z()
-  %ielemx = insertelement <3 x i32> undef, i32 %idx, i64 0
+  %ielemx = insertelement <3 x i32> poison, i32 %idx, i64 0
   %ielemy = insertelement <3 x i32> %ielemx, i32 %idy, i64 1
   %ielemz = insertelement <3 x i32> %ielemy, i32 %idz, i64 2
   call void @llvm.amdgcn.raw.ptr.buffer.store.v3i32(<3 x i32> %ielemz, ptr addrspace(8) undef, i32 0, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
index acb706cee04d0..4fb6a0114b499 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
+++ b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
@@ -43,28 +43,28 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_addc_u32 s13, s13, 0
 ; CHECK-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
 ; CHECK-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
-; CHECK-NEXT:    s_load_dwordx8 s[48:55], s[8:9], 0x0
+; CHECK-NEXT:    s_load_dwordx8 s[64:71], s[8:9], 0x0
 ; CHECK-NEXT:    s_add_u32 s0, s0, s17
 ; CHECK-NEXT:    s_mov_b64 s[34:35], s[8:9]
 ; CHECK-NEXT:    s_addc_u32 s1, s1, 0
 ; CHECK-NEXT:    v_mov_b32_e32 v40, v0
-; CHECK-NEXT:    s_add_u32 s44, s34, 40
+; CHECK-NEXT:    s_add_u32 s52, s34, 40
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v0
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    s_mov_b32 s33, s16
-; CHECK-NEXT:    s_addc_u32 s45, s35, 0
-; CHECK-NEXT:    s_mov_b32 s43, s14
+; CHECK-NEXT:    s_addc_u32 s53, s35, 0
+; CHECK-NEXT:    s_mov_b32 s51, s14
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, _Z13get_global_idj@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, _Z13get_global_idj@rel32@hi+12
-; CHECK-NEXT:    s_mov_b64 s[8:9], s[44:45]
+; CHECK-NEXT:    s_mov_b64 s[8:9], s[52:53]
 ; CHECK-NEXT:    s_mov_b32 s12, s14
 ; CHECK-NEXT:    s_mov_b32 s13, s15
 ; CHECK-NEXT:    s_mov_b32 s14, s33
-; CHECK-NEXT:    s_mov_b32 s42, s15
+; CHECK-NEXT:    s_mov_b32 s50, s15
 ; CHECK-NEXT:    s_mov_b64 s[36:37], s[10:11]
 ; CHECK-NEXT:    s_mov_b64 s[38:39], s[6:7]
-; CHECK-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; CHECK-NEXT:    s_mov_b64 s[48:49], s[4:5]
 ; CHECK-NEXT:    v_mov_b32_e32 v45, 0
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_mov_b32_e32 v43, v0
@@ -73,12 +73,12 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, _Z12get_local_idj@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, _Z12get_local_idj@rel32@hi+12
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; CHECK-NEXT:    s_mov_b64 s[8:9], s[44:45]
+; CHECK-NEXT:    s_mov_b64 s[8:9], s[52:53]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; CHECK-NEXT:    s_mov_b32 s12, s43
-; CHECK-NEXT:    s_mov_b32 s13, s42
+; CHECK-NEXT:    s_mov_b32 s12, s51
+; CHECK-NEXT:    s_mov_b32 s13, s50
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_mov_b32_e32 v41, v0
@@ -87,12 +87,12 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, _Z7barrierj@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, _Z7barrierj@rel32@hi+12
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; CHECK-NEXT:    s_mov_b64 s[8:9], s[44:45]
+; CHECK-NEXT:    s_mov_b64 s[8:9], s[52:53]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; CHECK-NEXT:    s_mov_b32 s12, s43
-; CHECK-NEXT:    s_mov_b32 s13, s42
+; CHECK-NEXT:    s_mov_b32 s12, s51
+; CHECK-NEXT:    s_mov_b32 s13, s50
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    ds_write_b32 v45, v45 offset:15360
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -102,22 +102,22 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, _Z3minjj@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, _Z3minjj@rel32@hi+12
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    v_and_b32_e32 v0, 0x7ffffffc, v0
 ; CHECK-NEXT:    v_and_b32_e32 v1, 28, v1
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; CHECK-NEXT:    s_mov_b64 s[8:9], s[44:45]
+; CHECK-NEXT:    s_mov_b64 s[8:9], s[52:53]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; CHECK-NEXT:    global_load_dword v0, v0, s[52:53]
-; CHECK-NEXT:    s_mov_b32 s12, s43
-; CHECK-NEXT:    s_mov_b32 s13, s42
+; CHECK-NEXT:    global_load_dword v0, v0, s[68:69]
+; CHECK-NEXT:    s_mov_b32 s12, s51
+; CHECK-NEXT:    s_mov_b32 s13, s50
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    v_bfe_u32 v0, v0, v1, 4
 ; CHECK-NEXT:    v_mov_b32_e32 v1, 12
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_mov_b32_e32 v42, v0
-; CHECK-NEXT:    s_mov_b32 s44, exec_lo
+; CHECK-NEXT:    s_mov_b32 s52, exec_lo
 ; CHECK-NEXT:    v_cmpx_ne_u32_e32 0, v42
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_25
 ; CHECK-NEXT:  ; %bb.1: ; %.preheader5
@@ -136,7 +136,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:  ; %bb.3:
 ; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s4
 ; CHECK-NEXT:    v_add_nc_u32_e32 v45, -1, v42
-; CHECK-NEXT:    s_mov_b32 s45, 0
+; CHECK-NEXT:    s_mov_b32 s53, 0
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v45
 ; CHECK-NEXT:    s_and_b32 exec_lo, exec_lo, vcc_lo
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_25
@@ -144,46 +144,46 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v43, 10, v43
 ; CHECK-NEXT:    v_add_nc_u32_e32 v46, 0x3c05, v0
 ; CHECK-NEXT:    v_mov_b32_e32 v47, 0
-; CHECK-NEXT:    s_mov_b32 s47, 0
+; CHECK-NEXT:    s_mov_b32 s55, 0
 ; CHECK-NEXT:  .LBB0_5: ; =>This Loop Header: Depth=1
 ; CHECK-NEXT:    ; Child Loop BB0_8 Depth 2
 ; CHECK-NEXT:    ; Child Loop BB0_20 Depth 2
-; CHECK-NEXT:    v_add_nc_u32_e32 v0, s47, v44
-; CHECK-NEXT:    s_lshl_b32 s4, s47, 5
-; CHECK-NEXT:    s_add_i32 s46, s47, 1
-; CHECK-NEXT:    s_add_i32 s5, s47, 5
-; CHECK-NEXT:    v_or3_b32 v57, s4, v43, s46
+; CHECK-NEXT:    v_add_nc_u32_e32 v0, s55, v44
+; CHECK-NEXT:    s_lshl_b32 s4, s55, 5
+; CHECK-NEXT:    s_add_i32 s54, s55, 1
+; CHECK-NEXT:    s_add_i32 s5, s55, 5
+; CHECK-NEXT:    v_or3_b32 v57, s4, v43, s54
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    ds_read_u8 v56, v0
-; CHECK-NEXT:    v_mov_b32_e32 v58, s46
-; CHECK-NEXT:    s_mov_b32 s52, exec_lo
+; CHECK-NEXT:    v_mov_b32_e32 v58, s54
+; CHECK-NEXT:    s_mov_b32 s68, exec_lo
 ; CHECK-NEXT:    v_cmpx_lt_u32_e64 s5, v42
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_17
 ; CHECK-NEXT:  ; %bb.6: ; %.preheader2
 ; CHECK-NEXT:    ; in Loop: Header=BB0_5 Depth=1
-; CHECK-NEXT:    s_mov_b32 s53, 0
-; CHECK-NEXT:    s_mov_b32 s56, 0
+; CHECK-NEXT:    s_mov_b32 s69, 0
+; CHECK-NEXT:    s_mov_b32 s80, 0
 ; CHECK-NEXT:    s_branch .LBB0_8
 ; CHECK-NEXT:  .LBB0_7: ; in Loop: Header=BB0_8 Depth=2
-; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s57
-; CHECK-NEXT:    s_add_i32 s56, s56, 4
-; CHECK-NEXT:    s_add_i32 s4, s47, s56
-; CHECK-NEXT:    v_add_nc_u32_e32 v0, s56, v57
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s81
+; CHECK-NEXT:    s_add_i32 s80, s80, 4
+; CHECK-NEXT:    s_add_i32 s4, s55, s80
+; CHECK-NEXT:    v_add_nc_u32_e32 v0, s80, v57
 ; CHECK-NEXT:    s_add_i32 s5, s4, 5
 ; CHECK-NEXT:    s_add_i32 s4, s4, 1
 ; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc_lo, s5, v42
 ; CHECK-NEXT:    v_mov_b32_e32 v58, s4
-; CHECK-NEXT:    s_or_b32 s53, vcc_lo, s53
-; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s53
+; CHECK-NEXT:    s_or_b32 s69, vcc_lo, s69
+; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s69
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_16
 ; CHECK-NEXT:  .LBB0_8: ; Parent Loop BB0_5 Depth=1
 ; CHECK-NEXT:    ; => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    v_add_nc_u32_e32 v59, s56, v46
-; CHECK-NEXT:    v_add_nc_u32_e32 v58, s56, v57
+; CHECK-NEXT:    v_add_nc_u32_e32 v59, s80, v46
+; CHECK-NEXT:    v_add_nc_u32_e32 v58, s80, v57
 ; CHECK-NEXT:    ds_read_u8 v0, v59
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_cmp_eq_u16_sdwa s4, v56, v0 src0_sel:BYTE_0 src1_sel:DWORD
-; CHECK-NEXT:    s_and_saveexec_b32 s57, s4
+; CHECK-NEXT:    s_and_saveexec_b32 s81, s4
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_10
 ; CHECK-NEXT:  ; %bb.9: ; in Loop: Header=BB0_8 Depth=2
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
@@ -193,22 +193,22 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_incPU3AS3Vj@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_incPU3AS3Vj@rel32@hi+12
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; CHECK-NEXT:    s_mov_b32 s12, s43
-; CHECK-NEXT:    s_mov_b32 s13, s42
+; CHECK-NEXT:    s_mov_b32 s12, s51
+; CHECK-NEXT:    s_mov_b32 s13, s50
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    v_add_nc_u32_e32 v47, 1, v47
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; CHECK-NEXT:    ds_write_b32 v0, v58
 ; CHECK-NEXT:  .LBB0_10: ; in Loop: Header=BB0_8 Depth=2
-; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s57
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s81
 ; CHECK-NEXT:    ds_read_u8 v0, v59 offset:1
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_cmp_eq_u16_sdwa s4, v56, v0 src0_sel:BYTE_0 src1_sel:DWORD
-; CHECK-NEXT:    s_and_saveexec_b32 s57, s4
+; CHECK-NEXT:    s_and_saveexec_b32 s81, s4
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_12
 ; CHECK-NEXT:  ; %bb.11: ; in Loop: Header=BB0_8 Depth=2
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
@@ -218,11 +218,11 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_incPU3AS3Vj@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_incPU3AS3Vj@rel32@hi+12
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; CHECK-NEXT:    s_mov_b32 s12, s43
-; CHECK-NEXT:    s_mov_b32 s13, s42
+; CHECK-NEXT:    s_mov_b32 s12, s51
+; CHECK-NEXT:    s_mov_b32 s13, s50
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    v_add_nc_u32_e32 v60, 1, v58
 ; CHECK-NEXT:    v_add_nc_u32_e32 v47, 1, v47
@@ -230,11 +230,11 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; CHECK-NEXT:    ds_write_b32 v0, v60
 ; CHECK-NEXT:  .LBB0_12: ; in Loop: Header=BB0_8 Depth=2
-; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s57
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s81
 ; CHECK-NEXT:    ds_read_u8 v0, v59 offset:2
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_cmp_eq_u16_sdwa s4, v56, v0 src0_sel:BYTE_0 src1_sel:DWORD
-; CHECK-NEXT:    s_and_saveexec_b32 s57, s4
+; CHECK-NEXT:    s_and_saveexec_b32 s81, s4
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_14
 ; CHECK-NEXT:  ; %bb.13: ; in Loop: Header=BB0_8 Depth=2
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
@@ -244,11 +244,11 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_incPU3AS3Vj@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_incPU3AS3Vj@rel32@hi+12
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; CHECK-NEXT:    s_mov_b32 s12, s43
-; CHECK-NEXT:    s_mov_b32 s13, s42
+; CHECK-NEXT:    s_mov_b32 s12, s51
+; CHECK-NEXT:    s_mov_b32 s13, s50
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    v_add_nc_u32_e32 v60, 2, v58
 ; CHECK-NEXT:    v_add_nc_u32_e32 v47, 1, v47
@@ -256,11 +256,11 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; CHECK-NEXT:    ds_write_b32 v0, v60
 ; CHECK-NEXT:  .LBB0_14: ; in Loop: Header=BB0_8 Depth=2
-; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s57
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s81
 ; CHECK-NEXT:    ds_read_u8 v0, v59 offset:3
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_cmp_eq_u16_sdwa s4, v56, v0 src0_sel:BYTE_0 src1_sel:DWORD
-; CHECK-NEXT:    s_and_saveexec_b32 s57, s4
+; CHECK-NEXT:    s_and_saveexec_b32 s81, s4
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_7
 ; CHECK-NEXT:  ; %bb.15: ; in Loop: Header=BB0_8 Depth=2
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
@@ -270,11 +270,11 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_incPU3AS3Vj@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_incPU3AS3Vj@rel32@hi+12
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; CHECK-NEXT:    s_mov_b32 s12, s43
-; CHECK-NEXT:    s_mov_b32 s13, s42
+; CHECK-NEXT:    s_mov_b32 s12, s51
+; CHECK-NEXT:    s_mov_b32 s13, s50
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    v_add_nc_u32_e32 v58, 3, v58
 ; CHECK-NEXT:    v_add_nc_u32_e32 v47, 1, v47
@@ -284,27 +284,27 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_branch .LBB0_7
 ; CHECK-NEXT:  .LBB0_16: ; %Flow45
 ; CHECK-NEXT:    ; in Loop: Header=BB0_5 Depth=1
-; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s53
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s69
 ; CHECK-NEXT:    v_mov_b32_e32 v57, v0
 ; CHECK-NEXT:  .LBB0_17: ; %Flow46
 ; CHECK-NEXT:    ; in Loop: Header=BB0_5 Depth=1
-; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s52
-; CHECK-NEXT:    s_mov_b32 s47, exec_lo
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s68
+; CHECK-NEXT:    s_mov_b32 s55, exec_lo
 ; CHECK-NEXT:    v_cmpx_lt_u32_e64 v58, v42
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_23
 ; CHECK-NEXT:  ; %bb.18: ; %.preheader
 ; CHECK-NEXT:    ; in Loop: Header=BB0_5 Depth=1
-; CHECK-NEXT:    s_mov_b32 s52, 0
+; CHECK-NEXT:    s_mov_b32 s68, 0
 ; CHECK-NEXT:    s_inst_prefetch 0x1
 ; CHECK-NEXT:    s_branch .LBB0_20
 ; CHECK-NEXT:    .p2align 6
 ; CHECK-NEXT:  .LBB0_19: ; in Loop: Header=BB0_20 Depth=2
-; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s53
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s69
 ; CHECK-NEXT:    v_add_nc_u32_e32 v58, 1, v58
 ; CHECK-NEXT:    v_add_nc_u32_e32 v57, 1, v57
 ; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc_lo, v58, v42
-; CHECK-NEXT:    s_or_b32 s52, vcc_lo, s52
-; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s52
+; CHECK-NEXT:    s_or_b32 s68, vcc_lo, s68
+; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s68
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_22
 ; CHECK-NEXT:  .LBB0_20: ; Parent Loop BB0_5 Depth=1
 ; CHECK-NEXT:    ; => This Inner Loop Header: Depth=2
@@ -312,7 +312,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    ds_read_u8 v0, v0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_cmp_eq_u16_sdwa s4, v56, v0 src0_sel:BYTE_0 src1_sel:DWORD
-; CHECK-NEXT:    s_and_saveexec_b32 s53, s4
+; CHECK-NEXT:    s_and_saveexec_b32 s69, s4
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_19
 ; CHECK-NEXT:  ; %bb.21: ; in Loop: Header=BB0_20 Depth=2
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
@@ -322,11 +322,11 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_incPU3AS3Vj@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_incPU3AS3Vj@rel32@hi+12
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; CHECK-NEXT:    s_mov_b32 s12, s43
-; CHECK-NEXT:    s_mov_b32 s13, s42
+; CHECK-NEXT:    s_mov_b32 s12, s51
+; CHECK-NEXT:    s_mov_b32 s13, s50
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    v_add_nc_u32_e32 v47, 1, v47
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -336,22 +336,22 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:  .LBB0_22: ; %Flow43
 ; CHECK-NEXT:    ; in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:    s_inst_prefetch 0x2
-; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s52
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s68
 ; CHECK-NEXT:  .LBB0_23: ; %Flow44
 ; CHECK-NEXT:    ; in Loop: Header=BB0_5 Depth=1
-; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s47
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s55
 ; CHECK-NEXT:  ; %bb.24: ; in Loop: Header=BB0_5 Depth=1
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc_lo, s46, v45
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc_lo, s54, v45
 ; CHECK-NEXT:    v_cmp_lt_u32_e64 s4, 59, v47
 ; CHECK-NEXT:    v_add_nc_u32_e32 v46, 1, v46
-; CHECK-NEXT:    s_mov_b32 s47, s46
+; CHECK-NEXT:    s_mov_b32 s55, s54
 ; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; CHECK-NEXT:    s_and_b32 s4, exec_lo, s4
-; CHECK-NEXT:    s_or_b32 s45, s4, s45
-; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s45
+; CHECK-NEXT:    s_or_b32 s53, s4, s53
+; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s53
 ; CHECK-NEXT:    s_cbranch_execnz .LBB0_5
 ; CHECK-NEXT:  .LBB0_25: ; %Flow51
-; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s44
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s52
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 1
 ; CHECK-NEXT:    s_add_u32 s8, s34, 40
@@ -359,11 +359,11 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, _Z7barrierj@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, _Z7barrierj@rel32@hi+12
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; CHECK-NEXT:    s_mov_b32 s12, s43
-; CHECK-NEXT:    s_mov_b32 s13, s42
+; CHECK-NEXT:    s_mov_b32 s12, s51
+; CHECK-NEXT:    s_mov_b32 s13, s50
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
@@ -373,10 +373,10 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_cmpx_gt_u32_e64 v47, v41
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_33
 ; CHECK-NEXT:  ; %bb.26:
-; CHECK-NEXT:    s_mov_b32 s44, 0
+; CHECK-NEXT:    s_mov_b32 s52, 0
 ; CHECK-NEXT:    s_branch .LBB0_28
 ; CHECK-NEXT:  .LBB0_27: ; in Loop: Header=BB0_28 Depth=1
-; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s45
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s53
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    s_add_u32 s8, s34, 40
@@ -384,21 +384,21 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, _Z14get_local_sizej@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, _Z14get_local_sizej@rel32@hi+12
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; CHECK-NEXT:    s_mov_b32 s12, s43
-; CHECK-NEXT:    s_mov_b32 s13, s42
+; CHECK-NEXT:    s_mov_b32 s12, s51
+; CHECK-NEXT:    s_mov_b32 s13, s50
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_add_co_u32 v41, vcc_lo, v0, v41
 ; CHECK-NEXT:    v_cmp_le_u32_e32 vcc_lo, v47, v41
-; CHECK-NEXT:    s_or_b32 s44, vcc_lo, s44
-; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s44
+; CHECK-NEXT:    s_or_b32 s52, vcc_lo, s52
+; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s52
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_33
 ; CHECK-NEXT:  .LBB0_28: ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v41
-; CHECK-NEXT:    s_mov_b32 s45, exec_lo
+; CHECK-NEXT:    s_mov_b32 s53, exec_lo
 ; CHECK-NEXT:    ds_read_b32 v0, v0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v63, 10, v0
@@ -407,8 +407,8 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_mul_u32_u24_e32 v1, 0x180, v63
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 5, v62
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v4, 5, v72
-; CHECK-NEXT:    v_add_co_u32 v2, s4, s48, v1
-; CHECK-NEXT:    v_add_co_ci_u32_e64 v3, null, s49, 0, s4
+; CHECK-NEXT:    v_add_co_u32 v2, s4, s64, v1
+; CHECK-NEXT:    v_add_co_ci_u32_e64 v3, null, s65, 0, s4
 ; CHECK-NEXT:    v_add_co_u32 v0, vcc_lo, v2, v0
 ; CHECK-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v3, vcc_lo
 ; CHECK-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
@@ -445,8 +445,8 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_or3_b32 v2, v3, v2, v4
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; CHECK-NEXT:    s_mov_b32 s12, s43
-; CHECK-NEXT:    s_mov_b32 s13, s42
+; CHECK-NEXT:    s_mov_b32 s12, s51
+; CHECK-NEXT:    s_mov_b32 s13, s50
 ; CHECK-NEXT:    v_or3_b32 v73, v2, v0, v1
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v0, 1, v73
@@ -454,11 +454,11 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_and_b32_e32 v0, 0x7fffc, v0
 ; CHECK-NEXT:    v_lshlrev_b32_e64 v44, v1, 1
 ; CHECK-NEXT:    v_and_b32_e32 v74, 28, v1
-; CHECK-NEXT:    v_add_co_u32 v42, s4, s54, v0
-; CHECK-NEXT:    v_add_co_ci_u32_e64 v43, null, s55, 0, s4
+; CHECK-NEXT:    v_add_co_u32 v42, s4, s70, v0
+; CHECK-NEXT:    v_add_co_ci_u32_e64 v43, null, s71, 0, s4
 ; CHECK-NEXT:    v_mov_b32_e32 v2, v44
 ; CHECK-NEXT:    v_mov_b32_e32 v0, v42
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    v_mov_b32_e32 v1, v43
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_bfe_u32 v0, v0, v74, 4
@@ -469,7 +469,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:  ; %bb.30: ; in Loop: Header=BB0_28 Depth=1
 ; CHECK-NEXT:    v_xor_b32_e32 v4, v60, v58
 ; CHECK-NEXT:    v_lshrrev_b64 v[2:3], 16, v[56:57]
-; CHECK-NEXT:    v_mad_u64_u32 v[6:7], null, 0x180, v73, s[50:51]
+; CHECK-NEXT:    v_mad_u64_u32 v[6:7], null, 0x180, v73, s[66:67]
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v10, 5, v0
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v1, 16, v4
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v8, 6, v72
@@ -503,11 +503,11 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_subPU3AS1Vjj@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_subPU3AS1Vjj@rel32@hi+12
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; CHECK-NEXT:    s_mov_b32 s12, s43
-; CHECK-NEXT:    s_mov_b32 s13, s42
+; CHECK-NEXT:    s_mov_b32 s12, s51
+; CHECK-NEXT:    s_mov_b32 s13, s50
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    s_branch .LBB0_27
@@ -792,28 +792,28 @@ define protected amdgpu_kernel void @kernel_round1_short(ptr addrspace(1) nocapt
 ; CHECK-NEXT:    s_addc_u32 s13, s13, 0
 ; CHECK-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
 ; CHECK-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
-; CHECK-NEXT:    s_load_dwordx2 s[46:47], s[8:9], 0x10
+; CHECK-NEXT:    s_load_dwordx2 s[54:55], s[8:9], 0x10
 ; CHECK-NEXT:    s_add_u32 s0, s0, s17
 ; CHECK-NEXT:    s_mov_b64 s[38:39], s[8:9]
 ; CHECK-NEXT:    s_addc_u32 s1, s1, 0
 ; CHECK-NEXT:    v_mov_b32_e32 v40, v0
-; CHECK-NEXT:    s_add_u32 s44, s38, 40
+; CHECK-NEXT:    s_add_u32 s52, s38, 40
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v0
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    s_mov_b32 s33, s16
-; CHECK-NEXT:    s_addc_u32 s45, s39, 0
-; CHECK-NEXT:    s_mov_b32 s43, s14
+; CHECK-NEXT:    s_addc_u32 s53, s39, 0
+; CHECK-NEXT:    s_mov_b32 s51, s14
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, _Z13get_global_idj@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, _Z13get_global_idj@rel32@hi+12
-; CHECK-NEXT:    s_mov_b64 s[8:9], s[44:45]
+; CHECK-NEXT:    s_mov_b64 s[8:9], s[52:53]
 ; CHECK-NEXT:    s_mov_b32 s12, s14
 ; CHECK-NEXT:    s_mov_b32 s13, s15
 ; CHECK-NEXT:    s_mov_b32 s14, s33
-; CHECK-NEXT:    s_mov_b32 s42, s15
+; CHECK-NEXT:    s_mov_b32 s50, s15
 ; CHECK-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; CHECK-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; CHECK-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; CHECK-NEXT:    s_mov_b64 s[48:49], s[4:5]
 ; CHECK-NEXT:    v_mov_b32_e32 v43, 0
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_mov_b32_e32 v42, v0
@@ -822,12 +822,12 @@ define protected amdgpu_kernel void @kernel_round1_short(ptr addrspace(1) nocapt
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, _Z12get_local_idj@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, _Z12get_local_idj@rel32@hi+12
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[36:37]
-; CHECK-NEXT:    s_mov_b64 s[8:9], s[44:45]
+; CHECK-NEXT:    s_mov_b64 s[8:9], s[52:53]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; CHECK-NEXT:    s_mov_b32 s12, s43
-; CHECK-NEXT:    s_mov_b32 s13, s42
+; CHECK-NEXT:    s_mov_b32 s12, s51
+; CHECK-NEXT:    s_mov_b32 s13, s50
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_mul_lo_u32 v46, v0, 14
@@ -836,12 +836,12 @@ define protected amdgpu_kernel void @kernel_round1_short(ptr addrspace(1) nocapt
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, _Z7barrierj@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, _Z7barrierj@rel32@hi+12
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[36:37]
-; CHECK-NEXT:    s_mov_b64 s[8:9], s[44:45]
+; CHECK-NEXT:    s_mov_b64 s[8:9], s[52:53]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; CHECK-NEXT:    s_mov_b32 s12, s43
-; CHECK-NEXT:    s_mov_b32 s13, s42
+; CHECK-NEXT:    s_mov_b32 s12, s51
+; CHECK-NEXT:    s_mov_b32 s13, s50
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    ds_write_b32 v43, v43 offset:15360
 ; CHECK-NEXT:    v_add_nc_u32_e32 v44, 0x3c04, v46
@@ -852,15 +852,15 @@ define protected amdgpu_kernel void @kernel_round1_short(ptr addrspace(1) nocapt
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, _Z3minjj@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, _Z3minjj@rel32@hi+12
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    v_and_b32_e32 v0, 0x7ffffffc, v0
 ; CHECK-NEXT:    v_and_b32_e32 v1, 28, v1
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[36:37]
-; CHECK-NEXT:    s_mov_b64 s[8:9], s[44:45]
+; CHECK-NEXT:    s_mov_b64 s[8:9], s[52:53]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; CHECK-NEXT:    global_load_dword v0, v0, s[46:47]
-; CHECK-NEXT:    s_mov_b32 s12, s43
-; CHECK-NEXT:    s_mov_b32 s13, s42
+; CHECK-NEXT:    global_load_dword v0, v0, s[54:55]
+; CHECK-NEXT:    s_mov_b32 s12, s51
+; CHECK-NEXT:    s_mov_b32 s13, s50
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    v_bfe_u32 v0, v0, v1, 4
@@ -868,7 +868,7 @@ define protected amdgpu_kernel void @kernel_round1_short(ptr addrspace(1) nocapt
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    v_mov_b32_e32 v41, v0
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v42, 10, v42
-; CHECK-NEXT:    s_mov_b32 s44, 0
+; CHECK-NEXT:    s_mov_b32 s52, 0
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:    ds_write_b8 v46, v43 offset:15364
 ; CHECK-NEXT:    v_add_nc_u32_e32 v45, -1, v41
@@ -878,12 +878,12 @@ define protected amdgpu_kernel void @kernel_round1_short(ptr addrspace(1) nocapt
 ; CHECK-NEXT:    ; Child Loop BB1_8 Depth 2
 ; CHECK-NEXT:    v_add_nc_u32_e32 v0, s4, v44
 ; CHECK-NEXT:    s_lshl_b32 s5, s4, 5
-; CHECK-NEXT:    s_add_i32 s45, s4, 1
+; CHECK-NEXT:    s_add_i32 s53, s4, 1
 ; CHECK-NEXT:    s_add_i32 s6, s4, 5
-; CHECK-NEXT:    v_or3_b32 v47, s5, v42, s45
+; CHECK-NEXT:    v_or3_b32 v47, s5, v42, s53
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    ds_read_u8 v46, v0
-; CHECK-NEXT:    v_mov_b32_e32 v56, s45
+; CHECK-NEXT:    v_mov_b32_e32 v56, s53
 ; CHECK-NEXT:    s_mov_b32 s5, exec_lo
 ; CHECK-NEXT:    v_cmpx_lt_u32_e64 s6, v41
 ; CHECK-NEXT:    s_cbranch_execz .LBB1_5
@@ -912,23 +912,23 @@ define protected amdgpu_kernel void @kernel_round1_short(ptr addrspace(1) nocapt
 ; CHECK-NEXT:  .LBB1_5: ; %Flow4
 ; CHECK-NEXT:    ; in Loop: Header=BB1_1 Depth=1
 ; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; CHECK-NEXT:    s_mov_b32 s46, exec_lo
+; CHECK-NEXT:    s_mov_b32 s54, exec_lo
 ; CHECK-NEXT:    v_cmpx_lt_u32_e64 v56, v41
 ; CHECK-NEXT:    s_cbranch_execz .LBB1_11
 ; CHECK-NEXT:  ; %bb.6: ; %.103.preheader
 ; CHECK-NEXT:    ; in Loop: Header=BB1_1 Depth=1
-; CHECK-NEXT:    s_mov_b32 s47, 0
+; CHECK-NEXT:    s_mov_b32 s55, 0
 ; CHECK-NEXT:    s_inst_prefetch 0x1
 ; CHECK-NEXT:    s_branch .LBB1_8
 ; CHECK-NEXT:    .p2align 6
 ; CHECK-NEXT:  .LBB1_7: ; %.114
 ; CHECK-NEXT:    ; in Loop: Header=BB1_8 Depth=2
-; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s48
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s64
 ; CHECK-NEXT:    v_add_nc_u32_e32 v56, 1, v56
 ; CHECK-NEXT:    v_add_nc_u32_e32 v47, 1, v47
 ; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc_lo, v56, v41
-; CHECK-NEXT:    s_or_b32 s47, vcc_lo, s47
-; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s47
+; CHECK-NEXT:    s_or_b32 s55, vcc_lo, s55
+; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s55
 ; CHECK-NEXT:    s_cbranch_execz .LBB1_10
 ; CHECK-NEXT:  .LBB1_8: ; %.103
 ; CHECK-NEXT:    ; Parent Loop BB1_1 Depth=1
@@ -937,7 +937,7 @@ define protected amdgpu_kernel void @kernel_round1_short(ptr addrspace(1) nocapt
 ; CHECK-NEXT:    ds_read_u8 v0, v0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    v_cmp_eq_u16_sdwa s4, v46, v0 src0_sel:BYTE_0 src1_sel:DWORD
-; CHECK-NEXT:    s_and_saveexec_b32 s48, s4
+; CHECK-NEXT:    s_and_saveexec_b32 s64, s4
 ; CHECK-NEXT:    s_cbranch_execz .LBB1_7
 ; CHECK-NEXT:  ; %bb.9: ; %.110
 ; CHECK-NEXT:    ; in Loop: Header=BB1_8 Depth=2
@@ -948,11 +948,11 @@ define protected amdgpu_kernel void @kernel_round1_short(ptr addrspace(1) nocapt
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, _Z10atomic_incPU3AS3Vj@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, _Z10atomic_incPU3AS3Vj@rel32@hi+12
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; CHECK-NEXT:    s_mov_b32 s12, s43
-; CHECK-NEXT:    s_mov_b32 s13, s42
+; CHECK-NEXT:    s_mov_b32 s12, s51
+; CHECK-NEXT:    s_mov_b32 s13, s50
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    v_add_nc_u32_e32 v43, 1, v43
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -962,22 +962,22 @@ define protected amdgpu_kernel void @kernel_round1_short(ptr addrspace(1) nocapt
 ; CHECK-NEXT:  .LBB1_10: ; %Flow
 ; CHECK-NEXT:    ; in Loop: Header=BB1_1 Depth=1
 ; CHECK-NEXT:    s_inst_prefetch 0x2
-; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s47
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s55
 ; CHECK-NEXT:  .LBB1_11: ; %Flow2
 ; CHECK-NEXT:    ; in Loop: Header=BB1_1 Depth=1
-; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s46
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s54
 ; CHECK-NEXT:  ; %bb.12: ; %.32
 ; CHECK-NEXT:    ; in Loop: Header=BB1_1 Depth=1
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc_lo, s45, v45
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc_lo, s53, v45
 ; CHECK-NEXT:    v_cmp_lt_u32_e64 s4, 59, v43
 ; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; CHECK-NEXT:    s_and_b32 s4, exec_lo, s4
-; CHECK-NEXT:    s_or_b32 s44, s4, s44
-; CHECK-NEXT:    s_mov_b32 s4, s45
-; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s44
+; CHECK-NEXT:    s_or_b32 s52, s4, s52
+; CHECK-NEXT:    s_mov_b32 s4, s53
+; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s52
 ; CHECK-NEXT:    s_cbranch_execnz .LBB1_1
 ; CHECK-NEXT:  ; %bb.13: ; %.119
-; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s44
+; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s52
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 1
 ; CHECK-NEXT:    s_add_u32 s8, s38, 40
@@ -985,11 +985,11 @@ define protected amdgpu_kernel void @kernel_round1_short(ptr addrspace(1) nocapt
 ; CHECK-NEXT:    s_getpc_b64 s[16:17]
 ; CHECK-NEXT:    s_add_u32 s16, s16, _Z7barrierj@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s17, s17, _Z7barrierj@rel32@hi+12
-; CHECK-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; CHECK-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; CHECK-NEXT:    s_mov_b32 s12, s43
-; CHECK-NEXT:    s_mov_b32 s13, s42
+; CHECK-NEXT:    s_mov_b32 s12, s51
+; CHECK-NEXT:    s_mov_b32 s13, s50
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; CHECK-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
index a7060e4f198f1..4fffab84a29db 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
@@ -55,7 +55,7 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo(half %src0, half %s
   %src2.ext = fpext half %src2 to float
   %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
   %cvt.result = fptrunc float %result to half
-  %vec.result = insertelement <2 x half> undef, half %cvt.result, i32 1
+  %vec.result = insertelement <2 x half> poison, half %cvt.result, i32 1
   ret <2 x half> %vec.result
 }
 
@@ -112,7 +112,7 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_constlo(half %src0, half %s
   %src2.ext = fpext half %src2 to float
   %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
   %cvt.result = fptrunc float %result to half
-  %vec.result = insertelement <2 x half> <half 1.0, half undef>, half %cvt.result, i32 1
+  %vec.result = insertelement <2 x half> <half 1.0, half poison>, half %cvt.result, i32 1
   ret <2 x half> %vec.result
 }
 
@@ -167,7 +167,7 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo(half %src0, half %src
   %src2.ext = fpext half %src2 to float
   %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
   %cvt.result = fptrunc float %result to half
-  %vec = insertelement <2 x half> undef, half %lo, i32 0
+  %vec = insertelement <2 x half> poison, half %lo, i32 0
   %vec.result = insertelement <2 x half> %vec, half %cvt.result, i32 1
   ret <2 x half> %vec.result
 }
@@ -398,7 +398,7 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt(half %
   %max = call float @llvm.maxnum.f32(float %result, float 0.0)
   %clamp = call float @llvm.minnum.f32(float %max, float 1.0)
   %cvt.result = fptrunc float %clamp to half
-  %vec.result = insertelement <2 x half> undef, half %cvt.result, i32 1
+  %vec.result = insertelement <2 x half> poison, half %cvt.result, i32 1
   ret <2 x half> %vec.result
 }
 
@@ -457,7 +457,7 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt(half
   %cvt.result = fptrunc float %result to half
   %max = call half @llvm.maxnum.f16(half %cvt.result, half 0.0)
   %clamp = call half @llvm.minnum.f16(half %max, half 1.0)
-  %vec.result = insertelement <2 x half> undef, half %clamp, i32 1
+  %vec.result = insertelement <2 x half> poison, half %clamp, i32 1
   ret <2 x half> %vec.result
 }
 
@@ -534,7 +534,7 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt_multi
   store volatile half %cvt.result, ptr addrspace(1) undef
   %max = call half @llvm.maxnum.f16(half %cvt.result, half 0.0)
   %clamp = call half @llvm.minnum.f16(half %max, half 1.0)
-  %vec.result = insertelement <2 x half> undef, half %clamp, i32 1
+  %vec.result = insertelement <2 x half> poison, half %clamp, i32 1
   ret <2 x half> %vec.result
 }
 
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix.ll b/llvm/test/CodeGen/AMDGPU/mad-mix.ll
index 4c2a16c17b38a..a4568b58661db 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix.ll
@@ -456,9 +456,9 @@ define <2 x float> @v_mad_mix_v2f32_shuffle(<2 x half> %src0, <2 x half> %src1,
 ; GISEL-CI-NEXT:    v_mad_f32 v0, v4, v0, v1
 ; GISEL-CI-NEXT:    v_mac_f32_e32 v1, v5, v2
 ; GISEL-CI-NEXT:    s_setpc_b64 s[30:31]
-  %src0.shuf = shufflevector <2 x half> %src0, <2 x half> undef, <2 x i32> <i32 1, i32 0>
-  %src1.shuf = shufflevector <2 x half> %src1, <2 x half> undef, <2 x i32> <i32 0, i32 1>
-  %src2.shuf = shufflevector <2 x half> %src2, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+  %src0.shuf = shufflevector <2 x half> %src0, <2 x half> poison, <2 x i32> <i32 1, i32 0>
+  %src1.shuf = shufflevector <2 x half> %src1, <2 x half> poison, <2 x i32> <i32 0, i32 1>
+  %src2.shuf = shufflevector <2 x half> %src2, <2 x half> poison, <2 x i32> <i32 1, i32 1>
   %src0.ext = fpext <2 x half> %src0.shuf to <2 x float>
   %src1.ext = fpext <2 x half> %src1.shuf to <2 x float>
   %src2.ext = fpext <2 x half> %src2.shuf to <2 x float>
diff --git a/llvm/test/CodeGen/AMDGPU/mad_uint24.ll b/llvm/test/CodeGen/AMDGPU/mad_uint24.ll
index a71b083d22e2b..99d930bf73fc9 100644
--- a/llvm/test/CodeGen/AMDGPU/mad_uint24.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad_uint24.ll
@@ -298,7 +298,7 @@ bb19:                                             ; preds = %bb19, %bb
   %tmp39 = mul i32 %tmp38, %tmp
   %tmp40 = add i32 %tmp39, %arg5
   store i32 %tmp40, ptr addrspace(1) %arg7
-  %tmp41 = insertelement <4 x i32> undef, i32 %tmp40, i32 0
+  %tmp41 = insertelement <4 x i32> poison, i32 %tmp40, i32 0
   %tmp42 = and i32 %tmp29, 16777215
   %tmp43 = mul i32 %tmp42, %tmp11
   %tmp44 = add i32 %tmp43, %tmp10
diff --git a/llvm/test/CodeGen/AMDGPU/madmk.ll b/llvm/test/CodeGen/AMDGPU/madmk.ll
index d5ef866dc8a85..1769b7418b232 100644
--- a/llvm/test/CodeGen/AMDGPU/madmk.ll
+++ b/llvm/test/CodeGen/AMDGPU/madmk.ll
@@ -199,7 +199,7 @@ bb1:                                              ; preds = %bb2
   ret void
 
 bb2:                                              ; preds = %bb6, %bb
-  %tmp = phi float [ undef, %bb ], [ %tmp8, %bb6 ]
+  %tmp = phi float [ poison, %bb ], [ %tmp8, %bb6 ]
   %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #1
   %f_tid = bitcast i32 %tid to float
   %tmp3 = fsub float %f_tid, %tmp
diff --git a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.gfx10.ll b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.gfx10.ll
index b4682dfb8a26d..4ca00f2daf97a 100644
--- a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.gfx10.ll
+++ b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.gfx10.ll
@@ -12,13 +12,7 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc() #0 {
 ; GFX10_1-LABEL: scalar_mov_materializes_frame_index_unavailable_scc:
 ; GFX10_1:       ; %bb.0:
 ; GFX10_1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10_1-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_1-NEXT:    s_add_i32 s5, s32, 0x80880
-; GFX10_1-NEXT:    buffer_store_dword v1, off, s[0:3], s5 ; 4-byte Folded Spill
-; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10_1-NEXT:    s_mov_b32 exec_lo, s4
 ; GFX10_1-NEXT:    v_lshrrev_b32_e64 v0, 5, s32
-; GFX10_1-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX10_1-NEXT:    s_and_b32 s4, 0, exec_lo
 ; GFX10_1-NEXT:    v_add_nc_u32_e32 v0, 64, v0
 ; GFX10_1-NEXT:    ;;#ASMSTART
@@ -30,24 +24,12 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc() #0 {
 ; GFX10_1-NEXT:    ;;#ASMSTART
 ; GFX10_1-NEXT:    ; use s59, scc
 ; GFX10_1-NEXT:    ;;#ASMEND
-; GFX10_1-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX10_1-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_1-NEXT:    s_add_i32 s5, s32, 0x80880
-; GFX10_1-NEXT:    buffer_load_dword v1, off, s[0:3], s5 ; 4-byte Folded Reload
-; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10_1-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_1-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10_3-LABEL: scalar_mov_materializes_frame_index_unavailable_scc:
 ; GFX10_3:       ; %bb.0:
 ; GFX10_3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10_3-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_3-NEXT:    s_add_i32 s5, s32, 0x80880
-; GFX10_3-NEXT:    buffer_store_dword v1, off, s[0:3], s5 ; 4-byte Folded Spill
-; GFX10_3-NEXT:    s_mov_b32 exec_lo, s4
 ; GFX10_3-NEXT:    v_lshrrev_b32_e64 v0, 5, s32
-; GFX10_3-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX10_3-NEXT:    s_and_b32 s4, 0, exec_lo
 ; GFX10_3-NEXT:    v_add_nc_u32_e32 v0, 64, v0
 ; GFX10_3-NEXT:    ;;#ASMSTART
@@ -59,23 +41,13 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc() #0 {
 ; GFX10_3-NEXT:    ;;#ASMSTART
 ; GFX10_3-NEXT:    ; use s59, scc
 ; GFX10_3-NEXT:    ;;#ASMEND
-; GFX10_3-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX10_3-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_3-NEXT:    s_add_i32 s5, s32, 0x80880
-; GFX10_3-NEXT:    buffer_load_dword v1, off, s[0:3], s5 ; 4-byte Folded Reload
-; GFX10_3-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_3-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_3-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: scalar_mov_materializes_frame_index_unavailable_scc:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX11-NEXT:    s_add_i32 s1, s32, 0x4044
-; GFX11-NEXT:    scratch_store_b32 off, v1, s1 ; 4-byte Folded Spill
-; GFX11-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX11-NEXT:    s_add_i32 s0, s32, 64
-; GFX11-NEXT:    v_writelane_b32 v1, s59, 0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX11-NEXT:    s_and_b32 s0, 0, exec_lo
 ; GFX11-NEXT:    s_addc_u32 s0, s32, 0x4040
@@ -89,12 +61,6 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc() #0 {
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use s59, scc
 ; GFX11-NEXT:    ;;#ASMEND
-; GFX11-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX11-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX11-NEXT:    s_add_i32 s1, s32, 0x4044
-; GFX11-NEXT:    scratch_load_b32 v1, off, s1 ; 4-byte Folded Reload
-; GFX11-NEXT:    s_mov_b32 exec_lo, s0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: scalar_mov_materializes_frame_index_unavailable_scc:
@@ -104,13 +70,7 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc() #0 {
 ; GFX12-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX12-NEXT:    scratch_store_b32 off, v1, s32 offset:16388 ; 4-byte Folded Spill
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 exec_lo, s0
-; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX12-NEXT:    s_and_b32 s0, 0, exec_lo
-; GFX12-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX12-NEXT:    s_add_co_ci_u32 s0, s32, 0x4000
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s32
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -124,50 +84,30 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc() #0 {
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; use s59, scc
 ; GFX12-NEXT:    ;;#ASMEND
-; GFX12-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX12-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX12-NEXT:    scratch_load_b32 v1, off, s32 offset:16388 ; 4-byte Folded Reload
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 exec_lo, s0
-; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: scalar_mov_materializes_frame_index_unavailable_scc:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    s_add_i32 s6, s32, 0x101100
-; GFX8-NEXT:    buffer_store_dword v1, off, s[0:3], s6 ; 4-byte Folded Spill
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    v_lshrrev_b32_e64 v0, 6, s32
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 64, v0
-; GFX8-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use alloca0 v0
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    v_lshrrev_b32_e64 v0, 6, s32
 ; GFX8-NEXT:    s_movk_i32 s59, 0x4040
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s59, v0
-; GFX8-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX8-NEXT:    s_and_b64 s[4:5], 0, exec
+; GFX8-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use s59, scc
 ; GFX8-NEXT:    ;;#ASMEND
-; GFX8-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    s_add_i32 s6, s32, 0x101100
-; GFX8-NEXT:    buffer_load_dword v1, off, s[0:3], s6 ; 4-byte Folded Reload
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: scalar_mov_materializes_frame_index_unavailable_scc:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    s_add_i32 s6, s32, 0x101100
-; GFX900-NEXT:    buffer_store_dword v1, off, s[0:3], s6 ; 4-byte Folded Spill
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX900-NEXT:    v_lshrrev_b32_e64 v0, 6, s32
 ; GFX900-NEXT:    v_add_u32_e32 v0, 64, v0
 ; GFX900-NEXT:    ;;#ASMSTART
@@ -175,47 +115,29 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc() #0 {
 ; GFX900-NEXT:    ;;#ASMEND
 ; GFX900-NEXT:    v_lshrrev_b32_e64 v0, 6, s32
 ; GFX900-NEXT:    v_add_u32_e32 v0, 0x4040, v0
-; GFX900-NEXT:    v_writelane_b32 v1, s59, 0
-; GFX900-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX900-NEXT:    s_and_b64 s[4:5], 0, exec
+; GFX900-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s59, scc
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    s_add_i32 s6, s32, 0x101100
-; GFX900-NEXT:    buffer_load_dword v1, off, s[0:3], s6 ; 4-byte Folded Reload
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX942-LABEL: scalar_mov_materializes_frame_index_unavailable_scc:
 ; GFX942:       ; %bb.0:
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT:    s_xor_saveexec_b64 s[0:1], -1
-; GFX942-NEXT:    s_add_i32 s2, s32, 0x4044
-; GFX942-NEXT:    scratch_store_dword off, v1, s2 ; 4-byte Folded Spill
-; GFX942-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX942-NEXT:    s_add_i32 s0, s32, 64
 ; GFX942-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX942-NEXT:    s_and_b64 s[0:1], 0, exec
 ; GFX942-NEXT:    s_addc_u32 s0, s32, 0x4040
 ; GFX942-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX942-NEXT:    s_bitset0_b32 s0, 0
-; GFX942-NEXT:    v_writelane_b32 v1, s59, 0
-; GFX942-NEXT:    s_mov_b32 s59, s0
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; use alloca0 v0
 ; GFX942-NEXT:    ;;#ASMEND
+; GFX942-NEXT:    s_mov_b32 s59, s0
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; use s59, scc
 ; GFX942-NEXT:    ;;#ASMEND
-; GFX942-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX942-NEXT:    s_xor_saveexec_b64 s[0:1], -1
-; GFX942-NEXT:    s_add_i32 s2, s32, 0x4044
-; GFX942-NEXT:    scratch_load_dword v1, off, s2 ; 4-byte Folded Reload
-; GFX942-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %alloca0 = alloca [4096 x i32], align 64, addrspace(5)
   %alloca1 = alloca i32, align 4, addrspace(5)
@@ -230,12 +152,6 @@ define void @scalar_mov_materializes_frame_index_dead_scc() #0 {
 ; GFX10_1-LABEL: scalar_mov_materializes_frame_index_dead_scc:
 ; GFX10_1:       ; %bb.0:
 ; GFX10_1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10_1-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_1-NEXT:    s_add_i32 s5, s32, 0x80880
-; GFX10_1-NEXT:    buffer_store_dword v1, off, s[0:3], s5 ; 4-byte Folded Spill
-; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10_1-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_1-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX10_1-NEXT:    v_lshrrev_b32_e64 v0, 5, s32
 ; GFX10_1-NEXT:    s_lshr_b32 s59, s32, 5
 ; GFX10_1-NEXT:    s_addk_i32 s59, 0x4040
@@ -246,23 +162,11 @@ define void @scalar_mov_materializes_frame_index_dead_scc() #0 {
 ; GFX10_1-NEXT:    ;;#ASMSTART
 ; GFX10_1-NEXT:    ; use s59
 ; GFX10_1-NEXT:    ;;#ASMEND
-; GFX10_1-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX10_1-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_1-NEXT:    s_add_i32 s5, s32, 0x80880
-; GFX10_1-NEXT:    buffer_load_dword v1, off, s[0:3], s5 ; 4-byte Folded Reload
-; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10_1-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_1-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10_3-LABEL: scalar_mov_materializes_frame_index_dead_scc:
 ; GFX10_3:       ; %bb.0:
 ; GFX10_3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10_3-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_3-NEXT:    s_add_i32 s5, s32, 0x80880
-; GFX10_3-NEXT:    buffer_store_dword v1, off, s[0:3], s5 ; 4-byte Folded Spill
-; GFX10_3-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_3-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX10_3-NEXT:    v_lshrrev_b32_e64 v0, 5, s32
 ; GFX10_3-NEXT:    s_lshr_b32 s59, s32, 5
 ; GFX10_3-NEXT:    s_addk_i32 s59, 0x4040
@@ -273,22 +177,11 @@ define void @scalar_mov_materializes_frame_index_dead_scc() #0 {
 ; GFX10_3-NEXT:    ;;#ASMSTART
 ; GFX10_3-NEXT:    ; use s59
 ; GFX10_3-NEXT:    ;;#ASMEND
-; GFX10_3-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX10_3-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_3-NEXT:    s_add_i32 s5, s32, 0x80880
-; GFX10_3-NEXT:    buffer_load_dword v1, off, s[0:3], s5 ; 4-byte Folded Reload
-; GFX10_3-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_3-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_3-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: scalar_mov_materializes_frame_index_dead_scc:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX11-NEXT:    s_add_i32 s1, s32, 0x4044
-; GFX11-NEXT:    scratch_store_b32 off, v1, s1 ; 4-byte Folded Spill
-; GFX11-NEXT:    s_mov_b32 exec_lo, s0
-; GFX11-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX11-NEXT:    s_add_i32 s0, s32, 64
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    v_mov_b32_e32 v0, s0
@@ -300,12 +193,6 @@ define void @scalar_mov_materializes_frame_index_dead_scc() #0 {
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use s59
 ; GFX11-NEXT:    ;;#ASMEND
-; GFX11-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX11-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX11-NEXT:    s_add_i32 s1, s32, 0x4044
-; GFX11-NEXT:    scratch_load_b32 v1, off, s1 ; 4-byte Folded Reload
-; GFX11-NEXT:    s_mov_b32 exec_lo, s0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: scalar_mov_materializes_frame_index_dead_scc:
@@ -315,105 +202,62 @@ define void @scalar_mov_materializes_frame_index_dead_scc() #0 {
 ; GFX12-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX12-NEXT:    scratch_store_b32 off, v1, s32 offset:16388 ; 4-byte Folded Spill
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 exec_lo, s0
-; GFX12-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX12-NEXT:    s_add_co_i32 s0, s32, 0x4000
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s32
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 s59, s0
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; use alloca0 v0
 ; GFX12-NEXT:    ;;#ASMEND
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_mov_b32 s59, s0
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; use s59
 ; GFX12-NEXT:    ;;#ASMEND
-; GFX12-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX12-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX12-NEXT:    scratch_load_b32 v1, off, s32 offset:16388 ; 4-byte Folded Reload
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 exec_lo, s0
-; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: scalar_mov_materializes_frame_index_dead_scc:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    s_add_i32 s6, s32, 0x101100
-; GFX8-NEXT:    buffer_store_dword v1, off, s[0:3], s6 ; 4-byte Folded Spill
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX8-NEXT:    v_writelane_b32 v1, s59, 0
-; GFX8-NEXT:    s_lshr_b32 s59, s32, 6
 ; GFX8-NEXT:    v_lshrrev_b32_e64 v0, 6, s32
-; GFX8-NEXT:    s_addk_i32 s59, 0x4040
+; GFX8-NEXT:    s_lshr_b32 s59, s32, 6
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 64, v0
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use alloca0 v0
 ; GFX8-NEXT:    ;;#ASMEND
+; GFX8-NEXT:    s_addk_i32 s59, 0x4040
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use s59
 ; GFX8-NEXT:    ;;#ASMEND
-; GFX8-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    s_add_i32 s6, s32, 0x101100
-; GFX8-NEXT:    buffer_load_dword v1, off, s[0:3], s6 ; 4-byte Folded Reload
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: scalar_mov_materializes_frame_index_dead_scc:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    s_add_i32 s6, s32, 0x101100
-; GFX900-NEXT:    buffer_store_dword v1, off, s[0:3], s6 ; 4-byte Folded Spill
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX900-NEXT:    v_writelane_b32 v1, s59, 0
-; GFX900-NEXT:    s_lshr_b32 s59, s32, 6
 ; GFX900-NEXT:    v_lshrrev_b32_e64 v0, 6, s32
-; GFX900-NEXT:    s_addk_i32 s59, 0x4040
+; GFX900-NEXT:    s_lshr_b32 s59, s32, 6
 ; GFX900-NEXT:    v_add_u32_e32 v0, 64, v0
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use alloca0 v0
 ; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    s_addk_i32 s59, 0x4040
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s59
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    s_add_i32 s6, s32, 0x101100
-; GFX900-NEXT:    buffer_load_dword v1, off, s[0:3], s6 ; 4-byte Folded Reload
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX942-LABEL: scalar_mov_materializes_frame_index_dead_scc:
 ; GFX942:       ; %bb.0:
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT:    s_xor_saveexec_b64 s[0:1], -1
-; GFX942-NEXT:    s_add_i32 s2, s32, 0x4044
-; GFX942-NEXT:    scratch_store_dword off, v1, s2 ; 4-byte Folded Spill
-; GFX942-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX942-NEXT:    s_add_i32 s0, s32, 64
 ; GFX942-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX942-NEXT:    s_add_i32 s0, s32, 0x4040
-; GFX942-NEXT:    v_writelane_b32 v1, s59, 0
-; GFX942-NEXT:    s_mov_b32 s59, s0
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; use alloca0 v0
 ; GFX942-NEXT:    ;;#ASMEND
+; GFX942-NEXT:    s_mov_b32 s59, s0
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; use s59
 ; GFX942-NEXT:    ;;#ASMEND
-; GFX942-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX942-NEXT:    s_xor_saveexec_b64 s[0:1], -1
-; GFX942-NEXT:    s_add_i32 s2, s32, 0x4044
-; GFX942-NEXT:    scratch_load_dword v1, off, s2 ; 4-byte Folded Reload
-; GFX942-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %alloca0 = alloca [4096 x i32], align 64, addrspace(5)
   %alloca1 = alloca i32, align 4, addrspace(5)
@@ -428,14 +272,8 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_fp() #1 {
 ; GFX10_1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10_1-NEXT:    s_mov_b32 s5, s33
 ; GFX10_1-NEXT:    s_mov_b32 s33, s32
-; GFX10_1-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_1-NEXT:    s_add_i32 s6, s33, 0x80880
-; GFX10_1-NEXT:    buffer_store_dword v1, off, s[0:3], s6 ; 4-byte Folded Spill
-; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10_1-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_1-NEXT:    v_lshrrev_b32_e64 v0, 5, s33
-; GFX10_1-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX10_1-NEXT:    s_add_i32 s32, s32, 0x81000
+; GFX10_1-NEXT:    v_lshrrev_b32_e64 v0, 5, s33
 ; GFX10_1-NEXT:    s_and_b32 s4, 0, exec_lo
 ; GFX10_1-NEXT:    s_mov_b32 s32, s33
 ; GFX10_1-NEXT:    v_add_nc_u32_e32 v0, 64, v0
@@ -443,19 +281,12 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_fp() #1 {
 ; GFX10_1-NEXT:    ; use alloca0 v0
 ; GFX10_1-NEXT:    ;;#ASMEND
 ; GFX10_1-NEXT:    v_lshrrev_b32_e64 v0, 5, s33
+; GFX10_1-NEXT:    s_mov_b32 s33, s5
 ; GFX10_1-NEXT:    v_add_nc_u32_e32 v0, 0x4040, v0
 ; GFX10_1-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX10_1-NEXT:    ;;#ASMSTART
 ; GFX10_1-NEXT:    ; use s59, scc
 ; GFX10_1-NEXT:    ;;#ASMEND
-; GFX10_1-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX10_1-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_1-NEXT:    s_add_i32 s6, s33, 0x80880
-; GFX10_1-NEXT:    buffer_load_dword v1, off, s[0:3], s6 ; 4-byte Folded Reload
-; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10_1-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_1-NEXT:    s_mov_b32 s33, s5
-; GFX10_1-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10_3-LABEL: scalar_mov_materializes_frame_index_unavailable_scc_fp:
@@ -463,13 +294,8 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_fp() #1 {
 ; GFX10_3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10_3-NEXT:    s_mov_b32 s5, s33
 ; GFX10_3-NEXT:    s_mov_b32 s33, s32
-; GFX10_3-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_3-NEXT:    s_add_i32 s6, s33, 0x80880
-; GFX10_3-NEXT:    buffer_store_dword v1, off, s[0:3], s6 ; 4-byte Folded Spill
-; GFX10_3-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_3-NEXT:    v_lshrrev_b32_e64 v0, 5, s33
-; GFX10_3-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX10_3-NEXT:    s_add_i32 s32, s32, 0x81000
+; GFX10_3-NEXT:    v_lshrrev_b32_e64 v0, 5, s33
 ; GFX10_3-NEXT:    s_and_b32 s4, 0, exec_lo
 ; GFX10_3-NEXT:    s_mov_b32 s32, s33
 ; GFX10_3-NEXT:    v_add_nc_u32_e32 v0, 64, v0
@@ -477,18 +303,12 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_fp() #1 {
 ; GFX10_3-NEXT:    ; use alloca0 v0
 ; GFX10_3-NEXT:    ;;#ASMEND
 ; GFX10_3-NEXT:    v_lshrrev_b32_e64 v0, 5, s33
+; GFX10_3-NEXT:    s_mov_b32 s33, s5
 ; GFX10_3-NEXT:    v_add_nc_u32_e32 v0, 0x4040, v0
 ; GFX10_3-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX10_3-NEXT:    ;;#ASMSTART
 ; GFX10_3-NEXT:    ; use s59, scc
 ; GFX10_3-NEXT:    ;;#ASMEND
-; GFX10_3-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX10_3-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_3-NEXT:    s_add_i32 s6, s33, 0x80880
-; GFX10_3-NEXT:    buffer_load_dword v1, off, s[0:3], s6 ; 4-byte Folded Reload
-; GFX10_3-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_3-NEXT:    s_mov_b32 s33, s5
-; GFX10_3-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_3-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: scalar_mov_materializes_frame_index_unavailable_scc_fp:
@@ -496,13 +316,9 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_fp() #1 {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s1, s33
 ; GFX11-NEXT:    s_mov_b32 s33, s32
-; GFX11-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX11-NEXT:    s_add_i32 s2, s33, 0x4044
-; GFX11-NEXT:    scratch_store_b32 off, v1, s2 ; 4-byte Folded Spill
-; GFX11-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX11-NEXT:    s_addk_i32 s32, 0x4080
 ; GFX11-NEXT:    s_add_i32 s0, s33, 64
-; GFX11-NEXT:    v_writelane_b32 v1, s59, 0
+; GFX11-NEXT:    s_mov_b32 s32, s33
 ; GFX11-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX11-NEXT:    s_and_b32 s0, 0, exec_lo
 ; GFX11-NEXT:    s_addc_u32 s0, s33, 0x4040
@@ -511,18 +327,11 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_fp() #1 {
 ; GFX11-NEXT:    ;;#ASMEND
 ; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX11-NEXT:    s_bitset0_b32 s0, 0
-; GFX11-NEXT:    s_mov_b32 s32, s33
+; GFX11-NEXT:    s_mov_b32 s33, s1
 ; GFX11-NEXT:    s_mov_b32 s59, s0
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use s59, scc
 ; GFX11-NEXT:    ;;#ASMEND
-; GFX11-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX11-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX11-NEXT:    s_add_i32 s2, s33, 0x4044
-; GFX11-NEXT:    scratch_load_b32 v1, off, s2 ; 4-byte Folded Reload
-; GFX11-NEXT:    s_mov_b32 exec_lo, s0
-; GFX11-NEXT:    s_mov_b32 s33, s1
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: scalar_mov_materializes_frame_index_unavailable_scc_fp:
@@ -534,13 +343,9 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_fp() #1 {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    s_mov_b32 s1, s33
 ; GFX12-NEXT:    s_mov_b32 s33, s32
-; GFX12-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX12-NEXT:    scratch_store_b32 off, v1, s33 offset:16388 ; 4-byte Folded Spill
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX12-NEXT:    s_addk_co_i32 s32, 0x4040
 ; GFX12-NEXT:    s_and_b32 s0, 0, exec_lo
-; GFX12-NEXT:    v_writelane_b32 v1, s59, 0
+; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_add_co_ci_u32 s0, s33, 0x4000
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s33
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -554,14 +359,8 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_fp() #1 {
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; use s59, scc
 ; GFX12-NEXT:    ;;#ASMEND
-; GFX12-NEXT:    v_readlane_b32 s59, v1, 0
 ; GFX12-NEXT:    s_mov_b32 s32, s33
-; GFX12-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX12-NEXT:    scratch_load_b32 v1, off, s33 offset:16388 ; 4-byte Folded Reload
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX12-NEXT:    s_mov_b32 s33, s1
-; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -570,33 +369,22 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_fp() #1 {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    s_mov_b32 s6, s33
 ; GFX8-NEXT:    s_mov_b32 s33, s32
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    s_add_i32 s7, s33, 0x101100
-; GFX8-NEXT:    buffer_store_dword v1, off, s[0:3], s7 ; 4-byte Folded Spill
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    v_lshrrev_b32_e64 v0, 6, s33
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 64, v0
-; GFX8-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use alloca0 v0
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    v_lshrrev_b32_e64 v0, 6, s33
 ; GFX8-NEXT:    s_movk_i32 s59, 0x4040
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s59, v0
 ; GFX8-NEXT:    s_add_i32 s32, s32, 0x102000
-; GFX8-NEXT:    v_readfirstlane_b32 s59, v0
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s59, v0
 ; GFX8-NEXT:    s_and_b64 s[4:5], 0, exec
+; GFX8-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use s59, scc
 ; GFX8-NEXT:    ;;#ASMEND
-; GFX8-NEXT:    v_readlane_b32 s59, v1, 0
 ; GFX8-NEXT:    s_mov_b32 s32, s33
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    s_add_i32 s7, s33, 0x101100
-; GFX8-NEXT:    buffer_load_dword v1, off, s[0:3], s7 ; 4-byte Folded Reload
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    s_mov_b32 s33, s6
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: scalar_mov_materializes_frame_index_unavailable_scc_fp:
@@ -604,32 +392,21 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_fp() #1 {
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-NEXT:    s_mov_b32 s6, s33
 ; GFX900-NEXT:    s_mov_b32 s33, s32
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    s_add_i32 s7, s33, 0x101100
-; GFX900-NEXT:    buffer_store_dword v1, off, s[0:3], s7 ; 4-byte Folded Spill
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX900-NEXT:    v_lshrrev_b32_e64 v0, 6, s33
 ; GFX900-NEXT:    v_add_u32_e32 v0, 64, v0
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use alloca0 v0
 ; GFX900-NEXT:    ;;#ASMEND
 ; GFX900-NEXT:    v_lshrrev_b32_e64 v0, 6, s33
-; GFX900-NEXT:    v_add_u32_e32 v0, 0x4040, v0
 ; GFX900-NEXT:    s_add_i32 s32, s32, 0x102000
-; GFX900-NEXT:    v_writelane_b32 v1, s59, 0
-; GFX900-NEXT:    v_readfirstlane_b32 s59, v0
+; GFX900-NEXT:    v_add_u32_e32 v0, 0x4040, v0
 ; GFX900-NEXT:    s_and_b64 s[4:5], 0, exec
+; GFX900-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s59, scc
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s59, v1, 0
 ; GFX900-NEXT:    s_mov_b32 s32, s33
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    s_add_i32 s7, s33, 0x101100
-; GFX900-NEXT:    buffer_load_dword v1, off, s[0:3], s7 ; 4-byte Folded Reload
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX900-NEXT:    s_mov_b32 s33, s6
-; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX942-LABEL: scalar_mov_materializes_frame_index_unavailable_scc_fp:
@@ -637,10 +414,6 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_fp() #1 {
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX942-NEXT:    s_mov_b32 s2, s33
 ; GFX942-NEXT:    s_mov_b32 s33, s32
-; GFX942-NEXT:    s_xor_saveexec_b64 s[0:1], -1
-; GFX942-NEXT:    s_add_i32 s3, s33, 0x4044
-; GFX942-NEXT:    scratch_store_dword off, v1, s3 ; 4-byte Folded Spill
-; GFX942-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX942-NEXT:    s_addk_i32 s32, 0x4080
 ; GFX942-NEXT:    s_add_i32 s0, s33, 64
 ; GFX942-NEXT:    v_mov_b32_e32 v0, s0
@@ -648,22 +421,15 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_fp() #1 {
 ; GFX942-NEXT:    s_addc_u32 s0, s33, 0x4040
 ; GFX942-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX942-NEXT:    s_bitset0_b32 s0, 0
-; GFX942-NEXT:    v_writelane_b32 v1, s59, 0
-; GFX942-NEXT:    s_mov_b32 s59, s0
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; use alloca0 v0
 ; GFX942-NEXT:    ;;#ASMEND
+; GFX942-NEXT:    s_mov_b32 s59, s0
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; use s59, scc
 ; GFX942-NEXT:    ;;#ASMEND
-; GFX942-NEXT:    v_readlane_b32 s59, v1, 0
 ; GFX942-NEXT:    s_mov_b32 s32, s33
-; GFX942-NEXT:    s_xor_saveexec_b64 s[0:1], -1
-; GFX942-NEXT:    s_add_i32 s3, s33, 0x4044
-; GFX942-NEXT:    scratch_load_dword v1, off, s3 ; 4-byte Folded Reload
-; GFX942-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX942-NEXT:    s_mov_b32 s33, s2
-; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %alloca0 = alloca [4096 x i32], align 64, addrspace(5)
   %alloca1 = alloca i32, align 4, addrspace(5)
@@ -676,75 +442,39 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_small_offset()
 ; GFX10_1-LABEL: scalar_mov_materializes_frame_index_unavailable_scc_small_offset:
 ; GFX10_1:       ; %bb.0:
 ; GFX10_1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10_1-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_1-NEXT:    s_add_i32 s5, s32, 0x80800
-; GFX10_1-NEXT:    buffer_store_dword v0, off, s[0:3], s5 ; 4-byte Folded Spill
-; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10_1-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_1-NEXT:    v_lshrrev_b32_e64 v1, 5, s32
-; GFX10_1-NEXT:    v_writelane_b32 v0, s59, 0
+; GFX10_1-NEXT:    v_lshrrev_b32_e64 v0, 5, s32
 ; GFX10_1-NEXT:    s_and_b32 s4, 0, exec_lo
-; GFX10_1-NEXT:    v_add_nc_u32_e32 v1, 64, v1
-; GFX10_1-NEXT:    v_readfirstlane_b32 s59, v1
+; GFX10_1-NEXT:    v_add_nc_u32_e32 v0, 64, v0
+; GFX10_1-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX10_1-NEXT:    ;;#ASMSTART
 ; GFX10_1-NEXT:    ; use s59, scc
 ; GFX10_1-NEXT:    ;;#ASMEND
-; GFX10_1-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX10_1-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_1-NEXT:    s_add_i32 s5, s32, 0x80800
-; GFX10_1-NEXT:    buffer_load_dword v0, off, s[0:3], s5 ; 4-byte Folded Reload
-; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10_1-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_1-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10_3-LABEL: scalar_mov_materializes_frame_index_unavailable_scc_small_offset:
 ; GFX10_3:       ; %bb.0:
 ; GFX10_3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10_3-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_3-NEXT:    s_add_i32 s5, s32, 0x80800
-; GFX10_3-NEXT:    buffer_store_dword v0, off, s[0:3], s5 ; 4-byte Folded Spill
-; GFX10_3-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_3-NEXT:    v_lshrrev_b32_e64 v1, 5, s32
-; GFX10_3-NEXT:    v_writelane_b32 v0, s59, 0
+; GFX10_3-NEXT:    v_lshrrev_b32_e64 v0, 5, s32
 ; GFX10_3-NEXT:    s_and_b32 s4, 0, exec_lo
-; GFX10_3-NEXT:    v_add_nc_u32_e32 v1, 64, v1
-; GFX10_3-NEXT:    v_readfirstlane_b32 s59, v1
+; GFX10_3-NEXT:    v_add_nc_u32_e32 v0, 64, v0
+; GFX10_3-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX10_3-NEXT:    ;;#ASMSTART
 ; GFX10_3-NEXT:    ; use s59, scc
 ; GFX10_3-NEXT:    ;;#ASMEND
-; GFX10_3-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX10_3-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_3-NEXT:    s_add_i32 s5, s32, 0x80800
-; GFX10_3-NEXT:    buffer_load_dword v0, off, s[0:3], s5 ; 4-byte Folded Reload
-; GFX10_3-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_3-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_3-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: scalar_mov_materializes_frame_index_unavailable_scc_small_offset:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX11-NEXT:    s_add_i32 s1, s32, 0x4040
-; GFX11-NEXT:    scratch_store_b32 off, v0, s1 ; 4-byte Folded Spill
-; GFX11-NEXT:    s_mov_b32 exec_lo, s0
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_and_b32 s0, 0, exec_lo
-; GFX11-NEXT:    v_writelane_b32 v0, s59, 0
 ; GFX11-NEXT:    s_addc_u32 s0, s32, 64
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX11-NEXT:    s_bitset0_b32 s0, 0
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_mov_b32 s59, s0
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use s59, scc
 ; GFX11-NEXT:    ;;#ASMEND
-; GFX11-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX11-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX11-NEXT:    s_add_i32 s1, s32, 0x4040
-; GFX11-NEXT:    scratch_load_b32 v0, off, s1 ; 4-byte Folded Reload
-; GFX11-NEXT:    s_mov_b32 exec_lo, s0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: scalar_mov_materializes_frame_index_unavailable_scc_small_offset:
@@ -754,94 +484,50 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_small_offset()
 ; GFX12-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX12-NEXT:    scratch_store_b32 off, v0, s32 offset:16384 ; 4-byte Folded Spill
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 exec_lo, s0
-; GFX12-NEXT:    v_writelane_b32 v0, s59, 0
-; GFX12-NEXT:    s_mov_b32 s59, s32
 ; GFX12-NEXT:    s_and_b32 s0, 0, exec_lo
+; GFX12-NEXT:    s_mov_b32 s59, s32
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; use s59, scc
 ; GFX12-NEXT:    ;;#ASMEND
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX12-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX12-NEXT:    scratch_load_b32 v0, off, s32 offset:16384 ; 4-byte Folded Reload
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 exec_lo, s0
-; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: scalar_mov_materializes_frame_index_unavailable_scc_small_offset:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    s_add_i32 s6, s32, 0x101000
-; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], s6 ; 4-byte Folded Spill
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX8-NEXT:    v_writelane_b32 v0, s59, 0
-; GFX8-NEXT:    v_lshrrev_b32_e64 v1, 6, s32
+; GFX8-NEXT:    v_lshrrev_b32_e64 v0, 6, s32
 ; GFX8-NEXT:    s_mov_b32 s59, 64
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s59, v1
-; GFX8-NEXT:    v_readfirstlane_b32 s59, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s59, v0
 ; GFX8-NEXT:    s_and_b64 s[4:5], 0, exec
+; GFX8-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use s59, scc
 ; GFX8-NEXT:    ;;#ASMEND
-; GFX8-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    s_add_i32 s6, s32, 0x101000
-; GFX8-NEXT:    buffer_load_dword v0, off, s[0:3], s6 ; 4-byte Folded Reload
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: scalar_mov_materializes_frame_index_unavailable_scc_small_offset:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    s_add_i32 s6, s32, 0x101000
-; GFX900-NEXT:    buffer_store_dword v0, off, s[0:3], s6 ; 4-byte Folded Spill
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX900-NEXT:    v_lshrrev_b32_e64 v1, 6, s32
-; GFX900-NEXT:    v_add_u32_e32 v1, 64, v1
-; GFX900-NEXT:    v_writelane_b32 v0, s59, 0
-; GFX900-NEXT:    v_readfirstlane_b32 s59, v1
+; GFX900-NEXT:    v_lshrrev_b32_e64 v0, 6, s32
+; GFX900-NEXT:    v_add_u32_e32 v0, 64, v0
 ; GFX900-NEXT:    s_and_b64 s[4:5], 0, exec
+; GFX900-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s59, scc
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    s_add_i32 s6, s32, 0x101000
-; GFX900-NEXT:    buffer_load_dword v0, off, s[0:3], s6 ; 4-byte Folded Reload
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX942-LABEL: scalar_mov_materializes_frame_index_unavailable_scc_small_offset:
 ; GFX942:       ; %bb.0:
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT:    s_xor_saveexec_b64 s[0:1], -1
-; GFX942-NEXT:    s_add_i32 s2, s32, 0x4040
-; GFX942-NEXT:    scratch_store_dword off, v0, s2 ; 4-byte Folded Spill
-; GFX942-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX942-NEXT:    s_and_b64 s[0:1], 0, exec
 ; GFX942-NEXT:    s_addc_u32 s0, s32, 64
 ; GFX942-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX942-NEXT:    s_bitset0_b32 s0, 0
-; GFX942-NEXT:    v_writelane_b32 v0, s59, 0
 ; GFX942-NEXT:    s_mov_b32 s59, s0
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; use s59, scc
 ; GFX942-NEXT:    ;;#ASMEND
-; GFX942-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX942-NEXT:    s_xor_saveexec_b64 s[0:1], -1
-; GFX942-NEXT:    s_add_i32 s2, s32, 0x4040
-; GFX942-NEXT:    scratch_load_dword v0, off, s2 ; 4-byte Folded Reload
-; GFX942-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %alloca0 = alloca [4096 x i32], align 64, addrspace(5)
   call void asm sideeffect "; use $0, $1", "{s59},{scc}"(ptr addrspace(5) %alloca0, i32 0)
@@ -852,67 +538,32 @@ define void @scalar_mov_materializes_frame_index_available_scc_small_offset() #0
 ; GFX10_1-LABEL: scalar_mov_materializes_frame_index_available_scc_small_offset:
 ; GFX10_1:       ; %bb.0:
 ; GFX10_1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10_1-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_1-NEXT:    s_add_i32 s5, s32, 0x80800
-; GFX10_1-NEXT:    buffer_store_dword v0, off, s[0:3], s5 ; 4-byte Folded Spill
-; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10_1-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_1-NEXT:    v_writelane_b32 v0, s59, 0
 ; GFX10_1-NEXT:    s_lshr_b32 s59, s32, 5
 ; GFX10_1-NEXT:    s_add_i32 s59, s59, 64
 ; GFX10_1-NEXT:    ;;#ASMSTART
 ; GFX10_1-NEXT:    ; use s59
 ; GFX10_1-NEXT:    ;;#ASMEND
-; GFX10_1-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX10_1-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_1-NEXT:    s_add_i32 s5, s32, 0x80800
-; GFX10_1-NEXT:    buffer_load_dword v0, off, s[0:3], s5 ; 4-byte Folded Reload
-; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10_1-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_1-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10_3-LABEL: scalar_mov_materializes_frame_index_available_scc_small_offset:
 ; GFX10_3:       ; %bb.0:
 ; GFX10_3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10_3-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_3-NEXT:    s_add_i32 s5, s32, 0x80800
-; GFX10_3-NEXT:    buffer_store_dword v0, off, s[0:3], s5 ; 4-byte Folded Spill
-; GFX10_3-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_3-NEXT:    v_writelane_b32 v0, s59, 0
 ; GFX10_3-NEXT:    s_lshr_b32 s59, s32, 5
 ; GFX10_3-NEXT:    s_add_i32 s59, s59, 64
 ; GFX10_3-NEXT:    ;;#ASMSTART
 ; GFX10_3-NEXT:    ; use s59
 ; GFX10_3-NEXT:    ;;#ASMEND
-; GFX10_3-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX10_3-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_3-NEXT:    s_add_i32 s5, s32, 0x80800
-; GFX10_3-NEXT:    buffer_load_dword v0, off, s[0:3], s5 ; 4-byte Folded Reload
-; GFX10_3-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_3-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_3-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: scalar_mov_materializes_frame_index_available_scc_small_offset:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX11-NEXT:    s_add_i32 s1, s32, 0x4040
-; GFX11-NEXT:    scratch_store_b32 off, v0, s1 ; 4-byte Folded Spill
-; GFX11-NEXT:    s_mov_b32 exec_lo, s0
-; GFX11-NEXT:    v_writelane_b32 v0, s59, 0
 ; GFX11-NEXT:    s_add_i32 s0, s32, 64
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_mov_b32 s59, s0
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use s59
 ; GFX11-NEXT:    ;;#ASMEND
-; GFX11-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX11-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX11-NEXT:    s_add_i32 s1, s32, 0x4040
-; GFX11-NEXT:    scratch_load_b32 v0, off, s1 ; 4-byte Folded Reload
-; GFX11-NEXT:    s_mov_b32 exec_lo, s0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: scalar_mov_materializes_frame_index_available_scc_small_offset:
@@ -922,85 +573,41 @@ define void @scalar_mov_materializes_frame_index_available_scc_small_offset() #0
 ; GFX12-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX12-NEXT:    scratch_store_b32 off, v0, s32 offset:16384 ; 4-byte Folded Spill
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 exec_lo, s0
-; GFX12-NEXT:    v_writelane_b32 v0, s59, 0
 ; GFX12-NEXT:    s_mov_b32 s59, s32
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; use s59
 ; GFX12-NEXT:    ;;#ASMEND
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX12-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX12-NEXT:    scratch_load_b32 v0, off, s32 offset:16384 ; 4-byte Folded Reload
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 exec_lo, s0
-; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: scalar_mov_materializes_frame_index_available_scc_small_offset:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    s_add_i32 s6, s32, 0x101000
-; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], s6 ; 4-byte Folded Spill
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX8-NEXT:    v_writelane_b32 v0, s59, 0
 ; GFX8-NEXT:    s_lshr_b32 s59, s32, 6
 ; GFX8-NEXT:    s_add_i32 s59, s59, 64
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use s59
 ; GFX8-NEXT:    ;;#ASMEND
-; GFX8-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    s_add_i32 s6, s32, 0x101000
-; GFX8-NEXT:    buffer_load_dword v0, off, s[0:3], s6 ; 4-byte Folded Reload
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: scalar_mov_materializes_frame_index_available_scc_small_offset:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    s_add_i32 s6, s32, 0x101000
-; GFX900-NEXT:    buffer_store_dword v0, off, s[0:3], s6 ; 4-byte Folded Spill
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX900-NEXT:    v_writelane_b32 v0, s59, 0
 ; GFX900-NEXT:    s_lshr_b32 s59, s32, 6
 ; GFX900-NEXT:    s_add_i32 s59, s59, 64
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s59
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    s_add_i32 s6, s32, 0x101000
-; GFX900-NEXT:    buffer_load_dword v0, off, s[0:3], s6 ; 4-byte Folded Reload
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX942-LABEL: scalar_mov_materializes_frame_index_available_scc_small_offset:
 ; GFX942:       ; %bb.0:
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT:    s_xor_saveexec_b64 s[0:1], -1
-; GFX942-NEXT:    s_add_i32 s2, s32, 0x4040
-; GFX942-NEXT:    scratch_store_dword off, v0, s2 ; 4-byte Folded Spill
-; GFX942-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX942-NEXT:    s_add_i32 s0, s32, 64
-; GFX942-NEXT:    v_writelane_b32 v0, s59, 0
 ; GFX942-NEXT:    s_mov_b32 s59, s0
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; use s59
 ; GFX942-NEXT:    ;;#ASMEND
-; GFX942-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX942-NEXT:    s_xor_saveexec_b64 s[0:1], -1
-; GFX942-NEXT:    s_add_i32 s2, s32, 0x4040
-; GFX942-NEXT:    scratch_load_dword v0, off, s2 ; 4-byte Folded Reload
-; GFX942-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %alloca0 = alloca [4096 x i32], align 64, addrspace(5)
   call void asm sideeffect "; use $0", "{s59}"(ptr addrspace(5) %alloca0)
@@ -1013,29 +620,16 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_small_offset_fp
 ; GFX10_1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10_1-NEXT:    s_mov_b32 s5, s33
 ; GFX10_1-NEXT:    s_mov_b32 s33, s32
-; GFX10_1-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_1-NEXT:    s_add_i32 s6, s33, 0x80800
-; GFX10_1-NEXT:    buffer_store_dword v0, off, s[0:3], s6 ; 4-byte Folded Spill
-; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10_1-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_1-NEXT:    v_lshrrev_b32_e64 v1, 5, s33
-; GFX10_1-NEXT:    v_writelane_b32 v0, s59, 0
-; GFX10_1-NEXT:    s_add_i32 s32, s32, 0x81000
+; GFX10_1-NEXT:    s_add_i32 s32, s32, 0x80800
+; GFX10_1-NEXT:    v_lshrrev_b32_e64 v0, 5, s33
 ; GFX10_1-NEXT:    s_and_b32 s4, 0, exec_lo
 ; GFX10_1-NEXT:    s_mov_b32 s32, s33
-; GFX10_1-NEXT:    v_add_nc_u32_e32 v1, 64, v1
-; GFX10_1-NEXT:    v_readfirstlane_b32 s59, v1
+; GFX10_1-NEXT:    s_mov_b32 s33, s5
+; GFX10_1-NEXT:    v_add_nc_u32_e32 v0, 64, v0
+; GFX10_1-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX10_1-NEXT:    ;;#ASMSTART
 ; GFX10_1-NEXT:    ; use s59, scc
 ; GFX10_1-NEXT:    ;;#ASMEND
-; GFX10_1-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX10_1-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_1-NEXT:    s_add_i32 s6, s33, 0x80800
-; GFX10_1-NEXT:    buffer_load_dword v0, off, s[0:3], s6 ; 4-byte Folded Reload
-; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10_1-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_1-NEXT:    s_mov_b32 s33, s5
-; GFX10_1-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10_3-LABEL: scalar_mov_materializes_frame_index_unavailable_scc_small_offset_fp:
@@ -1043,27 +637,16 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_small_offset_fp
 ; GFX10_3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10_3-NEXT:    s_mov_b32 s5, s33
 ; GFX10_3-NEXT:    s_mov_b32 s33, s32
-; GFX10_3-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_3-NEXT:    s_add_i32 s6, s33, 0x80800
-; GFX10_3-NEXT:    buffer_store_dword v0, off, s[0:3], s6 ; 4-byte Folded Spill
-; GFX10_3-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_3-NEXT:    v_lshrrev_b32_e64 v1, 5, s33
-; GFX10_3-NEXT:    v_writelane_b32 v0, s59, 0
-; GFX10_3-NEXT:    s_add_i32 s32, s32, 0x81000
+; GFX10_3-NEXT:    s_add_i32 s32, s32, 0x80800
+; GFX10_3-NEXT:    v_lshrrev_b32_e64 v0, 5, s33
 ; GFX10_3-NEXT:    s_and_b32 s4, 0, exec_lo
 ; GFX10_3-NEXT:    s_mov_b32 s32, s33
-; GFX10_3-NEXT:    v_add_nc_u32_e32 v1, 64, v1
-; GFX10_3-NEXT:    v_readfirstlane_b32 s59, v1
+; GFX10_3-NEXT:    s_mov_b32 s33, s5
+; GFX10_3-NEXT:    v_add_nc_u32_e32 v0, 64, v0
+; GFX10_3-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX10_3-NEXT:    ;;#ASMSTART
 ; GFX10_3-NEXT:    ; use s59, scc
 ; GFX10_3-NEXT:    ;;#ASMEND
-; GFX10_3-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX10_3-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_3-NEXT:    s_add_i32 s6, s33, 0x80800
-; GFX10_3-NEXT:    buffer_load_dword v0, off, s[0:3], s6 ; 4-byte Folded Reload
-; GFX10_3-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_3-NEXT:    s_mov_b32 s33, s5
-; GFX10_3-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_3-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: scalar_mov_materializes_frame_index_unavailable_scc_small_offset_fp:
@@ -1071,29 +654,17 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_small_offset_fp
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s1, s33
 ; GFX11-NEXT:    s_mov_b32 s33, s32
-; GFX11-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX11-NEXT:    s_add_i32 s2, s33, 0x4040
-; GFX11-NEXT:    scratch_store_b32 off, v0, s2 ; 4-byte Folded Spill
-; GFX11-NEXT:    s_mov_b32 exec_lo, s0
-; GFX11-NEXT:    s_addk_i32 s32, 0x4080
+; GFX11-NEXT:    s_addk_i32 s32, 0x4040
 ; GFX11-NEXT:    s_and_b32 s0, 0, exec_lo
-; GFX11-NEXT:    v_writelane_b32 v0, s59, 0
 ; GFX11-NEXT:    s_addc_u32 s0, s33, 64
 ; GFX11-NEXT:    s_mov_b32 s32, s33
 ; GFX11-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX11-NEXT:    s_bitset0_b32 s0, 0
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_mov_b32 s33, s1
 ; GFX11-NEXT:    s_mov_b32 s59, s0
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use s59, scc
 ; GFX11-NEXT:    ;;#ASMEND
-; GFX11-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX11-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX11-NEXT:    s_add_i32 s2, s33, 0x4040
-; GFX11-NEXT:    scratch_load_b32 v0, off, s2 ; 4-byte Folded Reload
-; GFX11-NEXT:    s_mov_b32 exec_lo, s0
-; GFX11-NEXT:    s_mov_b32 s33, s1
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: scalar_mov_materializes_frame_index_unavailable_scc_small_offset_fp:
@@ -1105,25 +676,15 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_small_offset_fp
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    s_mov_b32 s1, s33
 ; GFX12-NEXT:    s_mov_b32 s33, s32
-; GFX12-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX12-NEXT:    scratch_store_b32 off, v0, s33 offset:16384 ; 4-byte Folded Spill
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 exec_lo, s0
-; GFX12-NEXT:    v_writelane_b32 v0, s59, 0
 ; GFX12-NEXT:    s_addk_co_i32 s32, 0x4040
-; GFX12-NEXT:    s_mov_b32 s59, s33
 ; GFX12-NEXT:    s_and_b32 s0, 0, exec_lo
+; GFX12-NEXT:    s_wait_alu 0xfffe
+; GFX12-NEXT:    s_mov_b32 s59, s33
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; use s59, scc
 ; GFX12-NEXT:    ;;#ASMEND
-; GFX12-NEXT:    v_readlane_b32 s59, v0, 0
 ; GFX12-NEXT:    s_mov_b32 s32, s33
-; GFX12-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX12-NEXT:    scratch_load_b32 v0, off, s33 offset:16384 ; 4-byte Folded Reload
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX12-NEXT:    s_mov_b32 s33, s1
-; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1132,28 +693,17 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_small_offset_fp
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    s_mov_b32 s6, s33
 ; GFX8-NEXT:    s_mov_b32 s33, s32
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    s_add_i32 s7, s33, 0x101000
-; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], s7 ; 4-byte Folded Spill
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX8-NEXT:    v_writelane_b32 v0, s59, 0
-; GFX8-NEXT:    v_lshrrev_b32_e64 v1, 6, s33
+; GFX8-NEXT:    v_lshrrev_b32_e64 v0, 6, s33
 ; GFX8-NEXT:    s_mov_b32 s59, 64
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s59, v1
-; GFX8-NEXT:    s_add_i32 s32, s32, 0x102000
-; GFX8-NEXT:    v_readfirstlane_b32 s59, v1
+; GFX8-NEXT:    s_add_i32 s32, s32, 0x101000
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s59, v0
 ; GFX8-NEXT:    s_and_b64 s[4:5], 0, exec
+; GFX8-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use s59, scc
 ; GFX8-NEXT:    ;;#ASMEND
-; GFX8-NEXT:    v_readlane_b32 s59, v0, 0
 ; GFX8-NEXT:    s_mov_b32 s32, s33
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    s_add_i32 s7, s33, 0x101000
-; GFX8-NEXT:    buffer_load_dword v0, off, s[0:3], s7 ; 4-byte Folded Reload
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    s_mov_b32 s33, s6
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: scalar_mov_materializes_frame_index_unavailable_scc_small_offset_fp:
@@ -1161,27 +711,16 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_small_offset_fp
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-NEXT:    s_mov_b32 s6, s33
 ; GFX900-NEXT:    s_mov_b32 s33, s32
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    s_add_i32 s7, s33, 0x101000
-; GFX900-NEXT:    buffer_store_dword v0, off, s[0:3], s7 ; 4-byte Folded Spill
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX900-NEXT:    v_lshrrev_b32_e64 v1, 6, s33
-; GFX900-NEXT:    v_add_u32_e32 v1, 64, v1
-; GFX900-NEXT:    s_add_i32 s32, s32, 0x102000
-; GFX900-NEXT:    v_writelane_b32 v0, s59, 0
-; GFX900-NEXT:    v_readfirstlane_b32 s59, v1
+; GFX900-NEXT:    v_lshrrev_b32_e64 v0, 6, s33
+; GFX900-NEXT:    s_add_i32 s32, s32, 0x101000
+; GFX900-NEXT:    v_add_u32_e32 v0, 64, v0
 ; GFX900-NEXT:    s_and_b64 s[4:5], 0, exec
+; GFX900-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s59, scc
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s59, v0, 0
 ; GFX900-NEXT:    s_mov_b32 s32, s33
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    s_add_i32 s7, s33, 0x101000
-; GFX900-NEXT:    buffer_load_dword v0, off, s[0:3], s7 ; 4-byte Folded Reload
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX900-NEXT:    s_mov_b32 s33, s6
-; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX942-LABEL: scalar_mov_materializes_frame_index_unavailable_scc_small_offset_fp:
@@ -1189,28 +728,17 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc_small_offset_fp
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX942-NEXT:    s_mov_b32 s2, s33
 ; GFX942-NEXT:    s_mov_b32 s33, s32
-; GFX942-NEXT:    s_xor_saveexec_b64 s[0:1], -1
-; GFX942-NEXT:    s_add_i32 s3, s33, 0x4040
-; GFX942-NEXT:    scratch_store_dword off, v0, s3 ; 4-byte Folded Spill
-; GFX942-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX942-NEXT:    s_addk_i32 s32, 0x4080
+; GFX942-NEXT:    s_addk_i32 s32, 0x4040
 ; GFX942-NEXT:    s_and_b64 s[0:1], 0, exec
 ; GFX942-NEXT:    s_addc_u32 s0, s33, 64
 ; GFX942-NEXT:    s_bitcmp1_b32 s0, 0
 ; GFX942-NEXT:    s_bitset0_b32 s0, 0
-; GFX942-NEXT:    v_writelane_b32 v0, s59, 0
 ; GFX942-NEXT:    s_mov_b32 s59, s0
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; use s59, scc
 ; GFX942-NEXT:    ;;#ASMEND
-; GFX942-NEXT:    v_readlane_b32 s59, v0, 0
 ; GFX942-NEXT:    s_mov_b32 s32, s33
-; GFX942-NEXT:    s_xor_saveexec_b64 s[0:1], -1
-; GFX942-NEXT:    s_add_i32 s3, s33, 0x4040
-; GFX942-NEXT:    scratch_load_dword v0, off, s3 ; 4-byte Folded Reload
-; GFX942-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX942-NEXT:    s_mov_b32 s33, s2
-; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %alloca0 = alloca [4096 x i32], align 64, addrspace(5)
   call void asm sideeffect "; use $0, $1", "{s59},{scc}"(ptr addrspace(5) %alloca0, i32 0)
@@ -1223,27 +751,14 @@ define void @scalar_mov_materializes_frame_index_available_scc_small_offset_fp()
 ; GFX10_1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10_1-NEXT:    s_mov_b32 s4, s33
 ; GFX10_1-NEXT:    s_mov_b32 s33, s32
-; GFX10_1-NEXT:    s_xor_saveexec_b32 s5, -1
-; GFX10_1-NEXT:    s_add_i32 s6, s33, 0x80800
-; GFX10_1-NEXT:    buffer_store_dword v0, off, s[0:3], s6 ; 4-byte Folded Spill
-; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10_1-NEXT:    s_mov_b32 exec_lo, s5
-; GFX10_1-NEXT:    v_writelane_b32 v0, s59, 0
-; GFX10_1-NEXT:    s_add_i32 s32, s32, 0x81000
+; GFX10_1-NEXT:    s_add_i32 s32, s32, 0x80800
 ; GFX10_1-NEXT:    s_lshr_b32 s59, s33, 5
 ; GFX10_1-NEXT:    s_mov_b32 s32, s33
 ; GFX10_1-NEXT:    s_add_i32 s59, s59, 64
 ; GFX10_1-NEXT:    ;;#ASMSTART
 ; GFX10_1-NEXT:    ; use s59
 ; GFX10_1-NEXT:    ;;#ASMEND
-; GFX10_1-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX10_1-NEXT:    s_xor_saveexec_b32 s5, -1
-; GFX10_1-NEXT:    s_add_i32 s6, s33, 0x80800
-; GFX10_1-NEXT:    buffer_load_dword v0, off, s[0:3], s6 ; 4-byte Folded Reload
-; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10_1-NEXT:    s_mov_b32 exec_lo, s5
 ; GFX10_1-NEXT:    s_mov_b32 s33, s4
-; GFX10_1-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10_3-LABEL: scalar_mov_materializes_frame_index_available_scc_small_offset_fp:
@@ -1251,25 +766,14 @@ define void @scalar_mov_materializes_frame_index_available_scc_small_offset_fp()
 ; GFX10_3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10_3-NEXT:    s_mov_b32 s4, s33
 ; GFX10_3-NEXT:    s_mov_b32 s33, s32
-; GFX10_3-NEXT:    s_xor_saveexec_b32 s5, -1
-; GFX10_3-NEXT:    s_add_i32 s6, s33, 0x80800
-; GFX10_3-NEXT:    buffer_store_dword v0, off, s[0:3], s6 ; 4-byte Folded Spill
-; GFX10_3-NEXT:    s_mov_b32 exec_lo, s5
-; GFX10_3-NEXT:    v_writelane_b32 v0, s59, 0
-; GFX10_3-NEXT:    s_add_i32 s32, s32, 0x81000
+; GFX10_3-NEXT:    s_add_i32 s32, s32, 0x80800
 ; GFX10_3-NEXT:    s_lshr_b32 s59, s33, 5
 ; GFX10_3-NEXT:    s_mov_b32 s32, s33
 ; GFX10_3-NEXT:    s_add_i32 s59, s59, 64
 ; GFX10_3-NEXT:    ;;#ASMSTART
 ; GFX10_3-NEXT:    ; use s59
 ; GFX10_3-NEXT:    ;;#ASMEND
-; GFX10_3-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX10_3-NEXT:    s_xor_saveexec_b32 s5, -1
-; GFX10_3-NEXT:    s_add_i32 s6, s33, 0x80800
-; GFX10_3-NEXT:    buffer_load_dword v0, off, s[0:3], s6 ; 4-byte Folded Reload
-; GFX10_3-NEXT:    s_mov_b32 exec_lo, s5
 ; GFX10_3-NEXT:    s_mov_b32 s33, s4
-; GFX10_3-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_3-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: scalar_mov_materializes_frame_index_available_scc_small_offset_fp:
@@ -1277,25 +781,14 @@ define void @scalar_mov_materializes_frame_index_available_scc_small_offset_fp()
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s0, s33
 ; GFX11-NEXT:    s_mov_b32 s33, s32
-; GFX11-NEXT:    s_xor_saveexec_b32 s1, -1
-; GFX11-NEXT:    s_add_i32 s2, s33, 0x4040
-; GFX11-NEXT:    scratch_store_b32 off, v0, s2 ; 4-byte Folded Spill
-; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v0, s59, 0
-; GFX11-NEXT:    s_addk_i32 s32, 0x4080
+; GFX11-NEXT:    s_addk_i32 s32, 0x4040
 ; GFX11-NEXT:    s_add_i32 s1, s33, 64
 ; GFX11-NEXT:    s_mov_b32 s32, s33
 ; GFX11-NEXT:    s_mov_b32 s59, s1
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use s59
 ; GFX11-NEXT:    ;;#ASMEND
-; GFX11-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX11-NEXT:    s_xor_saveexec_b32 s1, -1
-; GFX11-NEXT:    s_add_i32 s2, s33, 0x4040
-; GFX11-NEXT:    scratch_load_b32 v0, off, s2 ; 4-byte Folded Reload
-; GFX11-NEXT:    s_mov_b32 exec_lo, s1
 ; GFX11-NEXT:    s_mov_b32 s33, s0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: scalar_mov_materializes_frame_index_available_scc_small_offset_fp:
@@ -1307,24 +800,14 @@ define void @scalar_mov_materializes_frame_index_available_scc_small_offset_fp()
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    s_mov_b32 s0, s33
 ; GFX12-NEXT:    s_mov_b32 s33, s32
-; GFX12-NEXT:    s_xor_saveexec_b32 s1, -1
-; GFX12-NEXT:    scratch_store_b32 off, v0, s33 offset:16384 ; 4-byte Folded Spill
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 exec_lo, s1
-; GFX12-NEXT:    v_writelane_b32 v0, s59, 0
 ; GFX12-NEXT:    s_addk_co_i32 s32, 0x4040
+; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_mov_b32 s59, s33
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; use s59
 ; GFX12-NEXT:    ;;#ASMEND
 ; GFX12-NEXT:    s_mov_b32 s32, s33
-; GFX12-NEXT:    v_readlane_b32 s59, v0, 0
-; GFX12-NEXT:    s_xor_saveexec_b32 s1, -1
-; GFX12-NEXT:    scratch_load_b32 v0, off, s33 offset:16384 ; 4-byte Folded Reload
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 exec_lo, s1
 ; GFX12-NEXT:    s_mov_b32 s33, s0
-; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1333,25 +816,14 @@ define void @scalar_mov_materializes_frame_index_available_scc_small_offset_fp()
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    s_mov_b32 s4, s33
 ; GFX8-NEXT:    s_mov_b32 s33, s32
-; GFX8-NEXT:    s_xor_saveexec_b64 s[6:7], -1
-; GFX8-NEXT:    s_add_i32 s5, s33, 0x101000
-; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], s5 ; 4-byte Folded Spill
-; GFX8-NEXT:    s_mov_b64 exec, s[6:7]
-; GFX8-NEXT:    s_add_i32 s32, s32, 0x102000
-; GFX8-NEXT:    v_writelane_b32 v0, s59, 0
+; GFX8-NEXT:    s_add_i32 s32, s32, 0x101000
 ; GFX8-NEXT:    s_lshr_b32 s59, s33, 6
 ; GFX8-NEXT:    s_add_i32 s59, s59, 64
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use s59
 ; GFX8-NEXT:    ;;#ASMEND
-; GFX8-NEXT:    v_readlane_b32 s59, v0, 0
 ; GFX8-NEXT:    s_mov_b32 s32, s33
-; GFX8-NEXT:    s_xor_saveexec_b64 s[6:7], -1
-; GFX8-NEXT:    s_add_i32 s5, s33, 0x101000
-; GFX8-NEXT:    buffer_load_dword v0, off, s[0:3], s5 ; 4-byte Folded Reload
-; GFX8-NEXT:    s_mov_b64 exec, s[6:7]
 ; GFX8-NEXT:    s_mov_b32 s33, s4
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: scalar_mov_materializes_frame_index_available_scc_small_offset_fp:
@@ -1359,25 +831,14 @@ define void @scalar_mov_materializes_frame_index_available_scc_small_offset_fp()
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900-NEXT:    s_mov_b32 s4, s33
 ; GFX900-NEXT:    s_mov_b32 s33, s32
-; GFX900-NEXT:    s_xor_saveexec_b64 s[6:7], -1
-; GFX900-NEXT:    s_add_i32 s5, s33, 0x101000
-; GFX900-NEXT:    buffer_store_dword v0, off, s[0:3], s5 ; 4-byte Folded Spill
-; GFX900-NEXT:    s_mov_b64 exec, s[6:7]
-; GFX900-NEXT:    s_add_i32 s32, s32, 0x102000
-; GFX900-NEXT:    v_writelane_b32 v0, s59, 0
+; GFX900-NEXT:    s_add_i32 s32, s32, 0x101000
 ; GFX900-NEXT:    s_lshr_b32 s59, s33, 6
 ; GFX900-NEXT:    s_add_i32 s59, s59, 64
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s59
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s59, v0, 0
 ; GFX900-NEXT:    s_mov_b32 s32, s33
-; GFX900-NEXT:    s_xor_saveexec_b64 s[6:7], -1
-; GFX900-NEXT:    s_add_i32 s5, s33, 0x101000
-; GFX900-NEXT:    buffer_load_dword v0, off, s[0:3], s5 ; 4-byte Folded Reload
-; GFX900-NEXT:    s_mov_b64 exec, s[6:7]
 ; GFX900-NEXT:    s_mov_b32 s33, s4
-; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX942-LABEL: scalar_mov_materializes_frame_index_available_scc_small_offset_fp:
@@ -1385,25 +846,14 @@ define void @scalar_mov_materializes_frame_index_available_scc_small_offset_fp()
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX942-NEXT:    s_mov_b32 s0, s33
 ; GFX942-NEXT:    s_mov_b32 s33, s32
-; GFX942-NEXT:    s_xor_saveexec_b64 s[2:3], -1
-; GFX942-NEXT:    s_add_i32 s1, s33, 0x4040
-; GFX942-NEXT:    scratch_store_dword off, v0, s1 ; 4-byte Folded Spill
-; GFX942-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX942-NEXT:    s_addk_i32 s32, 0x4080
+; GFX942-NEXT:    s_addk_i32 s32, 0x4040
 ; GFX942-NEXT:    s_add_i32 s1, s33, 64
-; GFX942-NEXT:    v_writelane_b32 v0, s59, 0
 ; GFX942-NEXT:    s_mov_b32 s59, s1
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; use s59
 ; GFX942-NEXT:    ;;#ASMEND
-; GFX942-NEXT:    v_readlane_b32 s59, v0, 0
 ; GFX942-NEXT:    s_mov_b32 s32, s33
-; GFX942-NEXT:    s_xor_saveexec_b64 s[2:3], -1
-; GFX942-NEXT:    s_add_i32 s1, s33, 0x4040
-; GFX942-NEXT:    scratch_load_dword v0, off, s1 ; 4-byte Folded Reload
-; GFX942-NEXT:    s_mov_b64 exec, s[2:3]
 ; GFX942-NEXT:    s_mov_b32 s33, s0
-; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %alloca0 = alloca [4096 x i32], align 64, addrspace(5)
   call void asm sideeffect "; use $0", "{s59}"(ptr addrspace(5) %alloca0)
@@ -1414,12 +864,6 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_immoffset(
 ; GFX10_1-LABEL: scalar_mov_materializes_frame_index_unavailable_scc__gep_immoffset:
 ; GFX10_1:       ; %bb.0:
 ; GFX10_1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10_1-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_1-NEXT:    s_add_i32 s5, s32, 0x100800
-; GFX10_1-NEXT:    buffer_store_dword v1, off, s[0:3], s5 ; 4-byte Folded Spill
-; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10_1-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_1-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX10_1-NEXT:    v_lshrrev_b32_e64 v0, 5, s32
 ; GFX10_1-NEXT:    s_lshr_b32 s4, s32, 5
 ; GFX10_1-NEXT:    s_add_i32 s59, s4, 0x442c
@@ -1431,23 +875,11 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_immoffset(
 ; GFX10_1-NEXT:    ;;#ASMSTART
 ; GFX10_1-NEXT:    ; use s59, scc
 ; GFX10_1-NEXT:    ;;#ASMEND
-; GFX10_1-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX10_1-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_1-NEXT:    s_add_i32 s5, s32, 0x100800
-; GFX10_1-NEXT:    buffer_load_dword v1, off, s[0:3], s5 ; 4-byte Folded Reload
-; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10_1-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_1-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10_3-LABEL: scalar_mov_materializes_frame_index_unavailable_scc__gep_immoffset:
 ; GFX10_3:       ; %bb.0:
 ; GFX10_3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10_3-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_3-NEXT:    s_add_i32 s5, s32, 0x100800
-; GFX10_3-NEXT:    buffer_store_dword v1, off, s[0:3], s5 ; 4-byte Folded Spill
-; GFX10_3-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_3-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX10_3-NEXT:    v_lshrrev_b32_e64 v0, 5, s32
 ; GFX10_3-NEXT:    s_lshr_b32 s4, s32, 5
 ; GFX10_3-NEXT:    s_add_i32 s59, s4, 0x442c
@@ -1459,22 +891,11 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_immoffset(
 ; GFX10_3-NEXT:    ;;#ASMSTART
 ; GFX10_3-NEXT:    ; use s59, scc
 ; GFX10_3-NEXT:    ;;#ASMEND
-; GFX10_3-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX10_3-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_3-NEXT:    s_add_i32 s5, s32, 0x100800
-; GFX10_3-NEXT:    buffer_load_dword v1, off, s[0:3], s5 ; 4-byte Folded Reload
-; GFX10_3-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_3-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_3-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: scalar_mov_materializes_frame_index_unavailable_scc__gep_immoffset:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX11-NEXT:    s_add_i32 s1, s32, 0x8040
-; GFX11-NEXT:    scratch_store_b32 off, v1, s1 ; 4-byte Folded Spill
-; GFX11-NEXT:    s_mov_b32 exec_lo, s0
-; GFX11-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX11-NEXT:    s_add_i32 s0, s32, 64
 ; GFX11-NEXT:    s_add_i32 s59, s32, 0x442c
 ; GFX11-NEXT:    v_mov_b32_e32 v0, s0
@@ -1485,12 +906,6 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_immoffset(
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use s59, scc
 ; GFX11-NEXT:    ;;#ASMEND
-; GFX11-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX11-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX11-NEXT:    s_add_i32 s1, s32, 0x8040
-; GFX11-NEXT:    scratch_load_b32 v1, off, s1 ; 4-byte Folded Reload
-; GFX11-NEXT:    s_mov_b32 exec_lo, s0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: scalar_mov_materializes_frame_index_unavailable_scc__gep_immoffset:
@@ -1500,11 +915,6 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_immoffset(
 ; GFX12-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX12-NEXT:    scratch_store_b32 off, v1, s32 offset:32768 ; 4-byte Folded Spill
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 exec_lo, s0
-; GFX12-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX12-NEXT:    s_add_co_i32 s59, s32, 0x43ec
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s32
 ; GFX12-NEXT:    s_and_b32 s0, 0, exec_lo
@@ -1514,23 +924,13 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_immoffset(
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; use s59, scc
 ; GFX12-NEXT:    ;;#ASMEND
-; GFX12-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX12-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX12-NEXT:    scratch_load_b32 v1, off, s32 offset:32768 ; 4-byte Folded Reload
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 exec_lo, s0
-; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: scalar_mov_materializes_frame_index_unavailable_scc__gep_immoffset:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    s_add_i32 s6, s32, 0x201000
-; GFX8-NEXT:    buffer_store_dword v1, off, s[0:3], s6 ; 4-byte Folded Spill
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    s_lshr_b32 s4, s32, 6
-; GFX8-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX8-NEXT:    s_add_i32 s59, s4, 0x442c
 ; GFX8-NEXT:    v_lshrrev_b32_e64 v0, 6, s32
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 64, v0
@@ -1541,23 +941,12 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_immoffset(
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use s59, scc
 ; GFX8-NEXT:    ;;#ASMEND
-; GFX8-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    s_add_i32 s6, s32, 0x201000
-; GFX8-NEXT:    buffer_load_dword v1, off, s[0:3], s6 ; 4-byte Folded Reload
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: scalar_mov_materializes_frame_index_unavailable_scc__gep_immoffset:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    s_add_i32 s6, s32, 0x201000
-; GFX900-NEXT:    buffer_store_dword v1, off, s[0:3], s6 ; 4-byte Folded Spill
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX900-NEXT:    s_lshr_b32 s4, s32, 6
-; GFX900-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX900-NEXT:    s_add_i32 s59, s4, 0x442c
 ; GFX900-NEXT:    v_lshrrev_b32_e64 v0, 6, s32
 ; GFX900-NEXT:    v_add_u32_e32 v0, 64, v0
@@ -1568,22 +957,11 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_immoffset(
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s59, scc
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    s_add_i32 s6, s32, 0x201000
-; GFX900-NEXT:    buffer_load_dword v1, off, s[0:3], s6 ; 4-byte Folded Reload
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX942-LABEL: scalar_mov_materializes_frame_index_unavailable_scc__gep_immoffset:
 ; GFX942:       ; %bb.0:
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT:    s_xor_saveexec_b64 s[0:1], -1
-; GFX942-NEXT:    s_add_i32 s2, s32, 0x8040
-; GFX942-NEXT:    scratch_store_dword off, v1, s2 ; 4-byte Folded Spill
-; GFX942-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX942-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX942-NEXT:    s_add_i32 s59, s32, 0x442c
 ; GFX942-NEXT:    s_add_i32 s0, s32, 64
 ; GFX942-NEXT:    v_mov_b32_e32 v0, s0
@@ -1594,12 +972,6 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_immoffset(
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; use s59, scc
 ; GFX942-NEXT:    ;;#ASMEND
-; GFX942-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX942-NEXT:    s_xor_saveexec_b64 s[0:1], -1
-; GFX942-NEXT:    s_add_i32 s2, s32, 0x8040
-; GFX942-NEXT:    scratch_load_dword v1, off, s2 ; 4-byte Folded Reload
-; GFX942-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %alloca0 = alloca [4096 x i32], align 64, addrspace(5)
   %alloca1 = alloca [4096 x i32], align 4, addrspace(5)
@@ -1613,12 +985,6 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offse
 ; GFX10_1-LABEL: scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offset:
 ; GFX10_1:       ; %bb.0:
 ; GFX10_1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10_1-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_1-NEXT:    s_add_i32 s5, s32, 0x100800
-; GFX10_1-NEXT:    buffer_store_dword v1, off, s[0:3], s5 ; 4-byte Folded Spill
-; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10_1-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_1-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX10_1-NEXT:    v_lshrrev_b32_e64 v0, 5, s32
 ; GFX10_1-NEXT:    s_lshl_b32 s4, s16, 2
 ; GFX10_1-NEXT:    s_lshr_b32 s59, s32, 5
@@ -1632,23 +998,11 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offse
 ; GFX10_1-NEXT:    ;;#ASMSTART
 ; GFX10_1-NEXT:    ; use s59, scc
 ; GFX10_1-NEXT:    ;;#ASMEND
-; GFX10_1-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX10_1-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_1-NEXT:    s_add_i32 s5, s32, 0x100800
-; GFX10_1-NEXT:    buffer_load_dword v1, off, s[0:3], s5 ; 4-byte Folded Reload
-; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10_1-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_1-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10_3-LABEL: scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offset:
 ; GFX10_3:       ; %bb.0:
 ; GFX10_3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10_3-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_3-NEXT:    s_add_i32 s5, s32, 0x100800
-; GFX10_3-NEXT:    buffer_store_dword v1, off, s[0:3], s5 ; 4-byte Folded Spill
-; GFX10_3-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_3-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX10_3-NEXT:    v_lshrrev_b32_e64 v0, 5, s32
 ; GFX10_3-NEXT:    s_lshl_b32 s4, s16, 2
 ; GFX10_3-NEXT:    s_lshr_b32 s59, s32, 5
@@ -1662,23 +1016,12 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offse
 ; GFX10_3-NEXT:    ;;#ASMSTART
 ; GFX10_3-NEXT:    ; use s59, scc
 ; GFX10_3-NEXT:    ;;#ASMEND
-; GFX10_3-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX10_3-NEXT:    s_xor_saveexec_b32 s4, -1
-; GFX10_3-NEXT:    s_add_i32 s5, s32, 0x100800
-; GFX10_3-NEXT:    buffer_load_dword v1, off, s[0:3], s5 ; 4-byte Folded Reload
-; GFX10_3-NEXT:    s_mov_b32 exec_lo, s4
-; GFX10_3-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_3-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offset:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_xor_saveexec_b32 s1, -1
-; GFX11-NEXT:    s_add_i32 s2, s32, 0x8040
-; GFX11-NEXT:    scratch_store_b32 off, v1, s2 ; 4-byte Folded Spill
-; GFX11-NEXT:    s_mov_b32 exec_lo, s1
 ; GFX11-NEXT:    s_add_i32 s1, s32, 64
-; GFX11-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX11-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX11-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX11-NEXT:    s_add_i32 s59, s32, s0
@@ -1690,12 +1033,6 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offse
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use s59, scc
 ; GFX11-NEXT:    ;;#ASMEND
-; GFX11-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX11-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX11-NEXT:    s_add_i32 s1, s32, 0x8040
-; GFX11-NEXT:    scratch_load_b32 v1, off, s1 ; 4-byte Folded Reload
-; GFX11-NEXT:    s_mov_b32 exec_lo, s0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-LABEL: scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offset:
@@ -1705,11 +1042,6 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offse
 ; GFX12-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_xor_saveexec_b32 s1, -1
-; GFX12-NEXT:    scratch_store_b32 off, v1, s32 offset:32768 ; 4-byte Folded Spill
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 exec_lo, s1
-; GFX12-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX12-NEXT:    s_lshl_b32 s0, s0, 2
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s32
 ; GFX12-NEXT:    s_wait_alu 0xfffe
@@ -1723,22 +1055,12 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offse
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; use s59, scc
 ; GFX12-NEXT:    ;;#ASMEND
-; GFX12-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX12-NEXT:    s_xor_saveexec_b32 s0, -1
-; GFX12-NEXT:    scratch_load_b32 v1, off, s32 offset:32768 ; 4-byte Folded Reload
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 exec_lo, s0
-; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offset:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    s_add_i32 s6, s32, 0x201000
-; GFX8-NEXT:    buffer_store_dword v1, off, s[0:3], s6 ; 4-byte Folded Spill
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX8-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX8-NEXT:    s_lshl_b32 s4, s16, 2
 ; GFX8-NEXT:    s_lshr_b32 s59, s32, 6
 ; GFX8-NEXT:    s_add_i32 s59, s59, s4
@@ -1752,22 +1074,11 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offse
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use s59, scc
 ; GFX8-NEXT:    ;;#ASMEND
-; GFX8-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX8-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX8-NEXT:    s_add_i32 s6, s32, 0x201000
-; GFX8-NEXT:    buffer_load_dword v1, off, s[0:3], s6 ; 4-byte Folded Reload
-; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offset:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    s_add_i32 s6, s32, 0x201000
-; GFX900-NEXT:    buffer_store_dword v1, off, s[0:3], s6 ; 4-byte Folded Spill
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX900-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX900-NEXT:    s_lshl_b32 s4, s16, 2
 ; GFX900-NEXT:    s_lshr_b32 s59, s32, 6
 ; GFX900-NEXT:    s_add_i32 s59, s59, s4
@@ -1781,23 +1092,12 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offse
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s59, scc
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX900-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GFX900-NEXT:    s_add_i32 s6, s32, 0x201000
-; GFX900-NEXT:    buffer_load_dword v1, off, s[0:3], s6 ; 4-byte Folded Reload
-; GFX900-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX942-LABEL: scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offset:
 ; GFX942:       ; %bb.0:
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT:    s_xor_saveexec_b64 s[2:3], -1
-; GFX942-NEXT:    s_add_i32 s1, s32, 0x8040
-; GFX942-NEXT:    scratch_store_dword off, v1, s1 ; 4-byte Folded Spill
-; GFX942-NEXT:    s_mov_b64 exec, s[2:3]
 ; GFX942-NEXT:    s_lshl_b32 s0, s0, 2
-; GFX942-NEXT:    v_writelane_b32 v1, s59, 0
 ; GFX942-NEXT:    s_add_i32 s59, s32, s0
 ; GFX942-NEXT:    s_addk_i32 s59, 0x4040
 ; GFX942-NEXT:    s_add_i32 s0, s32, 64
@@ -1809,12 +1109,6 @@ define void @scalar_mov_materializes_frame_index_unavailable_scc__gep_sgpr_offse
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; use s59, scc
 ; GFX942-NEXT:    ;;#ASMEND
-; GFX942-NEXT:    v_readlane_b32 s59, v1, 0
-; GFX942-NEXT:    s_xor_saveexec_b64 s[0:1], -1
-; GFX942-NEXT:    s_add_i32 s2, s32, 0x8040
-; GFX942-NEXT:    scratch_load_dword v1, off, s2 ; 4-byte Folded Reload
-; GFX942-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %alloca0 = alloca [4096 x i32], align 64, addrspace(5)
   %alloca1 = alloca [4096 x i32], align 4, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
index 2420393b63ba9..e8dacc93a8f3c 100644
--- a/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/materialize-frame-index-sgpr.ll
@@ -46,28 +46,17 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0
 ; GFX7-NEXT:    v_writelane_b32 v23, s37, 6
 ; GFX7-NEXT:    v_writelane_b32 v23, s38, 7
 ; GFX7-NEXT:    v_writelane_b32 v23, s39, 8
-; GFX7-NEXT:    v_writelane_b32 v23, s40, 9
-; GFX7-NEXT:    v_writelane_b32 v23, s41, 10
-; GFX7-NEXT:    v_writelane_b32 v23, s42, 11
-; GFX7-NEXT:    v_writelane_b32 v23, s43, 12
-; GFX7-NEXT:    v_writelane_b32 v23, s44, 13
-; GFX7-NEXT:    v_writelane_b32 v23, s45, 14
-; GFX7-NEXT:    v_writelane_b32 v23, s46, 15
-; GFX7-NEXT:    v_writelane_b32 v23, s47, 16
-; GFX7-NEXT:    v_writelane_b32 v23, s48, 17
-; GFX7-NEXT:    v_writelane_b32 v23, s49, 18
-; GFX7-NEXT:    v_writelane_b32 v23, s50, 19
-; GFX7-NEXT:    v_writelane_b32 v23, s51, 20
-; GFX7-NEXT:    v_writelane_b32 v23, s52, 21
-; GFX7-NEXT:    v_writelane_b32 v23, s53, 22
-; GFX7-NEXT:    v_writelane_b32 v23, s54, 23
-; GFX7-NEXT:    v_writelane_b32 v23, s55, 24
-; GFX7-NEXT:    v_writelane_b32 v23, s56, 25
+; GFX7-NEXT:    v_writelane_b32 v23, s48, 9
+; GFX7-NEXT:    v_writelane_b32 v23, s49, 10
+; GFX7-NEXT:    v_writelane_b32 v23, s50, 11
+; GFX7-NEXT:    v_writelane_b32 v23, s51, 12
+; GFX7-NEXT:    v_writelane_b32 v23, s52, 13
+; GFX7-NEXT:    v_writelane_b32 v23, s53, 14
 ; GFX7-NEXT:    v_lshr_b32_e64 v0, s32, 6
-; GFX7-NEXT:    v_writelane_b32 v23, s57, 26
+; GFX7-NEXT:    v_writelane_b32 v23, s54, 15
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 64, v0
 ; GFX7-NEXT:    s_and_b64 s[4:5], 0, exec
-; GFX7-NEXT:    v_writelane_b32 v23, s58, 27
+; GFX7-NEXT:    v_writelane_b32 v23, s55, 16
 ; GFX7-NEXT:    ;;#ASMSTART
 ; GFX7-NEXT:    ; use alloca0 v0
 ; GFX7-NEXT:    ;;#ASMEND
@@ -78,33 +67,20 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0
 ; GFX7-NEXT:    v_mov_b32_e32 v0, 0x4040
 ; GFX7-NEXT:    v_mad_u32_u24 v0, v0, 64, s32
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 6, v0
-; GFX7-NEXT:    v_writelane_b32 v23, s59, 28
 ; GFX7-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX7-NEXT:    buffer_load_dword v0, off, s[0:3], s32
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    ;;#ASMSTART
 ; GFX7-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s59, scc
 ; GFX7-NEXT:    ;;#ASMEND
-; GFX7-NEXT:    v_readlane_b32 s59, v23, 28
-; GFX7-NEXT:    v_readlane_b32 s58, v23, 27
-; GFX7-NEXT:    v_readlane_b32 s57, v23, 26
-; GFX7-NEXT:    v_readlane_b32 s56, v23, 25
-; GFX7-NEXT:    v_readlane_b32 s55, v23, 24
-; GFX7-NEXT:    v_readlane_b32 s54, v23, 23
-; GFX7-NEXT:    v_readlane_b32 s53, v23, 22
-; GFX7-NEXT:    v_readlane_b32 s52, v23, 21
-; GFX7-NEXT:    v_readlane_b32 s51, v23, 20
-; GFX7-NEXT:    v_readlane_b32 s50, v23, 19
-; GFX7-NEXT:    v_readlane_b32 s49, v23, 18
-; GFX7-NEXT:    v_readlane_b32 s48, v23, 17
-; GFX7-NEXT:    v_readlane_b32 s47, v23, 16
-; GFX7-NEXT:    v_readlane_b32 s46, v23, 15
-; GFX7-NEXT:    v_readlane_b32 s45, v23, 14
-; GFX7-NEXT:    v_readlane_b32 s44, v23, 13
-; GFX7-NEXT:    v_readlane_b32 s43, v23, 12
-; GFX7-NEXT:    v_readlane_b32 s42, v23, 11
-; GFX7-NEXT:    v_readlane_b32 s41, v23, 10
-; GFX7-NEXT:    v_readlane_b32 s40, v23, 9
+; GFX7-NEXT:    v_readlane_b32 s55, v23, 16
+; GFX7-NEXT:    v_readlane_b32 s54, v23, 15
+; GFX7-NEXT:    v_readlane_b32 s53, v23, 14
+; GFX7-NEXT:    v_readlane_b32 s52, v23, 13
+; GFX7-NEXT:    v_readlane_b32 s51, v23, 12
+; GFX7-NEXT:    v_readlane_b32 s50, v23, 11
+; GFX7-NEXT:    v_readlane_b32 s49, v23, 10
+; GFX7-NEXT:    v_readlane_b32 s48, v23, 9
 ; GFX7-NEXT:    v_readlane_b32 s39, v23, 8
 ; GFX7-NEXT:    v_readlane_b32 s38, v23, 7
 ; GFX7-NEXT:    v_readlane_b32 s37, v23, 6
@@ -137,28 +113,17 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0
 ; GFX8-NEXT:    v_writelane_b32 v23, s37, 6
 ; GFX8-NEXT:    v_writelane_b32 v23, s38, 7
 ; GFX8-NEXT:    v_writelane_b32 v23, s39, 8
-; GFX8-NEXT:    v_writelane_b32 v23, s40, 9
-; GFX8-NEXT:    v_writelane_b32 v23, s41, 10
-; GFX8-NEXT:    v_writelane_b32 v23, s42, 11
-; GFX8-NEXT:    v_writelane_b32 v23, s43, 12
-; GFX8-NEXT:    v_writelane_b32 v23, s44, 13
-; GFX8-NEXT:    v_writelane_b32 v23, s45, 14
-; GFX8-NEXT:    v_writelane_b32 v23, s46, 15
-; GFX8-NEXT:    v_writelane_b32 v23, s47, 16
-; GFX8-NEXT:    v_writelane_b32 v23, s48, 17
-; GFX8-NEXT:    v_writelane_b32 v23, s49, 18
-; GFX8-NEXT:    v_writelane_b32 v23, s50, 19
-; GFX8-NEXT:    v_writelane_b32 v23, s51, 20
-; GFX8-NEXT:    v_writelane_b32 v23, s52, 21
-; GFX8-NEXT:    v_writelane_b32 v23, s53, 22
-; GFX8-NEXT:    v_writelane_b32 v23, s54, 23
-; GFX8-NEXT:    v_writelane_b32 v23, s55, 24
-; GFX8-NEXT:    v_writelane_b32 v23, s56, 25
+; GFX8-NEXT:    v_writelane_b32 v23, s48, 9
+; GFX8-NEXT:    v_writelane_b32 v23, s49, 10
+; GFX8-NEXT:    v_writelane_b32 v23, s50, 11
+; GFX8-NEXT:    v_writelane_b32 v23, s51, 12
+; GFX8-NEXT:    v_writelane_b32 v23, s52, 13
+; GFX8-NEXT:    v_writelane_b32 v23, s53, 14
 ; GFX8-NEXT:    v_lshrrev_b32_e64 v0, 6, s32
-; GFX8-NEXT:    v_writelane_b32 v23, s57, 26
+; GFX8-NEXT:    v_writelane_b32 v23, s54, 15
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 64, v0
 ; GFX8-NEXT:    s_and_b64 s[4:5], 0, exec
-; GFX8-NEXT:    v_writelane_b32 v23, s58, 27
+; GFX8-NEXT:    v_writelane_b32 v23, s55, 16
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use alloca0 v0
 ; GFX8-NEXT:    ;;#ASMEND
@@ -169,33 +134,20 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, 0x4040
 ; GFX8-NEXT:    v_mad_u32_u24 v0, v0, 64, s32
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 6, v0
-; GFX8-NEXT:    v_writelane_b32 v23, s59, 28
 ; GFX8-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX8-NEXT:    buffer_load_dword v0, off, s[0:3], s32
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s59, scc
 ; GFX8-NEXT:    ;;#ASMEND
-; GFX8-NEXT:    v_readlane_b32 s59, v23, 28
-; GFX8-NEXT:    v_readlane_b32 s58, v23, 27
-; GFX8-NEXT:    v_readlane_b32 s57, v23, 26
-; GFX8-NEXT:    v_readlane_b32 s56, v23, 25
-; GFX8-NEXT:    v_readlane_b32 s55, v23, 24
-; GFX8-NEXT:    v_readlane_b32 s54, v23, 23
-; GFX8-NEXT:    v_readlane_b32 s53, v23, 22
-; GFX8-NEXT:    v_readlane_b32 s52, v23, 21
-; GFX8-NEXT:    v_readlane_b32 s51, v23, 20
-; GFX8-NEXT:    v_readlane_b32 s50, v23, 19
-; GFX8-NEXT:    v_readlane_b32 s49, v23, 18
-; GFX8-NEXT:    v_readlane_b32 s48, v23, 17
-; GFX8-NEXT:    v_readlane_b32 s47, v23, 16
-; GFX8-NEXT:    v_readlane_b32 s46, v23, 15
-; GFX8-NEXT:    v_readlane_b32 s45, v23, 14
-; GFX8-NEXT:    v_readlane_b32 s44, v23, 13
-; GFX8-NEXT:    v_readlane_b32 s43, v23, 12
-; GFX8-NEXT:    v_readlane_b32 s42, v23, 11
-; GFX8-NEXT:    v_readlane_b32 s41, v23, 10
-; GFX8-NEXT:    v_readlane_b32 s40, v23, 9
+; GFX8-NEXT:    v_readlane_b32 s55, v23, 16
+; GFX8-NEXT:    v_readlane_b32 s54, v23, 15
+; GFX8-NEXT:    v_readlane_b32 s53, v23, 14
+; GFX8-NEXT:    v_readlane_b32 s52, v23, 13
+; GFX8-NEXT:    v_readlane_b32 s51, v23, 12
+; GFX8-NEXT:    v_readlane_b32 s50, v23, 11
+; GFX8-NEXT:    v_readlane_b32 s49, v23, 10
+; GFX8-NEXT:    v_readlane_b32 s48, v23, 9
 ; GFX8-NEXT:    v_readlane_b32 s39, v23, 8
 ; GFX8-NEXT:    v_readlane_b32 s38, v23, 7
 ; GFX8-NEXT:    v_readlane_b32 s37, v23, 6
@@ -228,28 +180,17 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0
 ; GFX900-NEXT:    v_writelane_b32 v23, s37, 6
 ; GFX900-NEXT:    v_writelane_b32 v23, s38, 7
 ; GFX900-NEXT:    v_writelane_b32 v23, s39, 8
-; GFX900-NEXT:    v_writelane_b32 v23, s40, 9
-; GFX900-NEXT:    v_writelane_b32 v23, s41, 10
-; GFX900-NEXT:    v_writelane_b32 v23, s42, 11
-; GFX900-NEXT:    v_writelane_b32 v23, s43, 12
-; GFX900-NEXT:    v_writelane_b32 v23, s44, 13
-; GFX900-NEXT:    v_writelane_b32 v23, s45, 14
-; GFX900-NEXT:    v_writelane_b32 v23, s46, 15
-; GFX900-NEXT:    v_writelane_b32 v23, s47, 16
-; GFX900-NEXT:    v_writelane_b32 v23, s48, 17
-; GFX900-NEXT:    v_writelane_b32 v23, s49, 18
-; GFX900-NEXT:    v_writelane_b32 v23, s50, 19
-; GFX900-NEXT:    v_writelane_b32 v23, s51, 20
-; GFX900-NEXT:    v_writelane_b32 v23, s52, 21
-; GFX900-NEXT:    v_writelane_b32 v23, s53, 22
-; GFX900-NEXT:    v_writelane_b32 v23, s54, 23
-; GFX900-NEXT:    v_writelane_b32 v23, s55, 24
-; GFX900-NEXT:    v_writelane_b32 v23, s56, 25
+; GFX900-NEXT:    v_writelane_b32 v23, s48, 9
+; GFX900-NEXT:    v_writelane_b32 v23, s49, 10
+; GFX900-NEXT:    v_writelane_b32 v23, s50, 11
+; GFX900-NEXT:    v_writelane_b32 v23, s51, 12
+; GFX900-NEXT:    v_writelane_b32 v23, s52, 13
+; GFX900-NEXT:    v_writelane_b32 v23, s53, 14
 ; GFX900-NEXT:    v_lshrrev_b32_e64 v0, 6, s32
-; GFX900-NEXT:    v_writelane_b32 v23, s57, 26
+; GFX900-NEXT:    v_writelane_b32 v23, s54, 15
 ; GFX900-NEXT:    v_add_u32_e32 v0, 64, v0
 ; GFX900-NEXT:    s_and_b64 s[4:5], 0, exec
-; GFX900-NEXT:    v_writelane_b32 v23, s58, 27
+; GFX900-NEXT:    v_writelane_b32 v23, s55, 16
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use alloca0 v0
 ; GFX900-NEXT:    ;;#ASMEND
@@ -259,33 +200,20 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0
 ; GFX900-NEXT:    buffer_store_dword v0, off, s[0:3], s32
 ; GFX900-NEXT:    v_lshrrev_b32_e64 v0, 6, s32
 ; GFX900-NEXT:    v_add_u32_e32 v0, 0x4040, v0
-; GFX900-NEXT:    v_writelane_b32 v23, s59, 28
 ; GFX900-NEXT:    v_readfirstlane_b32 s59, v0
 ; GFX900-NEXT:    buffer_load_dword v0, off, s[0:3], s32
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s59, scc
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s59, v23, 28
-; GFX900-NEXT:    v_readlane_b32 s58, v23, 27
-; GFX900-NEXT:    v_readlane_b32 s57, v23, 26
-; GFX900-NEXT:    v_readlane_b32 s56, v23, 25
-; GFX900-NEXT:    v_readlane_b32 s55, v23, 24
-; GFX900-NEXT:    v_readlane_b32 s54, v23, 23
-; GFX900-NEXT:    v_readlane_b32 s53, v23, 22
-; GFX900-NEXT:    v_readlane_b32 s52, v23, 21
-; GFX900-NEXT:    v_readlane_b32 s51, v23, 20
-; GFX900-NEXT:    v_readlane_b32 s50, v23, 19
-; GFX900-NEXT:    v_readlane_b32 s49, v23, 18
-; GFX900-NEXT:    v_readlane_b32 s48, v23, 17
-; GFX900-NEXT:    v_readlane_b32 s47, v23, 16
-; GFX900-NEXT:    v_readlane_b32 s46, v23, 15
-; GFX900-NEXT:    v_readlane_b32 s45, v23, 14
-; GFX900-NEXT:    v_readlane_b32 s44, v23, 13
-; GFX900-NEXT:    v_readlane_b32 s43, v23, 12
-; GFX900-NEXT:    v_readlane_b32 s42, v23, 11
-; GFX900-NEXT:    v_readlane_b32 s41, v23, 10
-; GFX900-NEXT:    v_readlane_b32 s40, v23, 9
+; GFX900-NEXT:    v_readlane_b32 s55, v23, 16
+; GFX900-NEXT:    v_readlane_b32 s54, v23, 15
+; GFX900-NEXT:    v_readlane_b32 s53, v23, 14
+; GFX900-NEXT:    v_readlane_b32 s52, v23, 13
+; GFX900-NEXT:    v_readlane_b32 s51, v23, 12
+; GFX900-NEXT:    v_readlane_b32 s50, v23, 11
+; GFX900-NEXT:    v_readlane_b32 s49, v23, 10
+; GFX900-NEXT:    v_readlane_b32 s48, v23, 9
 ; GFX900-NEXT:    v_readlane_b32 s39, v23, 8
 ; GFX900-NEXT:    v_readlane_b32 s38, v23, 7
 ; GFX900-NEXT:    v_readlane_b32 s37, v23, 6
@@ -318,31 +246,17 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0
 ; GFX942-NEXT:    v_writelane_b32 v23, s37, 6
 ; GFX942-NEXT:    v_writelane_b32 v23, s38, 7
 ; GFX942-NEXT:    v_writelane_b32 v23, s39, 8
-; GFX942-NEXT:    v_writelane_b32 v23, s40, 9
-; GFX942-NEXT:    v_writelane_b32 v23, s41, 10
-; GFX942-NEXT:    v_writelane_b32 v23, s42, 11
-; GFX942-NEXT:    v_writelane_b32 v23, s43, 12
-; GFX942-NEXT:    v_writelane_b32 v23, s44, 13
-; GFX942-NEXT:    v_writelane_b32 v23, s45, 14
-; GFX942-NEXT:    v_writelane_b32 v23, s46, 15
-; GFX942-NEXT:    v_writelane_b32 v23, s47, 16
-; GFX942-NEXT:    v_writelane_b32 v23, s48, 17
-; GFX942-NEXT:    v_writelane_b32 v23, s49, 18
-; GFX942-NEXT:    v_writelane_b32 v23, s50, 19
-; GFX942-NEXT:    v_writelane_b32 v23, s51, 20
-; GFX942-NEXT:    v_writelane_b32 v23, s52, 21
-; GFX942-NEXT:    v_writelane_b32 v23, s53, 22
-; GFX942-NEXT:    v_writelane_b32 v23, s54, 23
-; GFX942-NEXT:    v_writelane_b32 v23, s55, 24
-; GFX942-NEXT:    v_writelane_b32 v23, s56, 25
-; GFX942-NEXT:    v_writelane_b32 v23, s57, 26
-; GFX942-NEXT:    v_writelane_b32 v23, s58, 27
-; GFX942-NEXT:    v_writelane_b32 v23, s59, 28
-; GFX942-NEXT:    v_writelane_b32 v23, s60, 29
+; GFX942-NEXT:    v_writelane_b32 v23, s48, 9
+; GFX942-NEXT:    v_writelane_b32 v23, s49, 10
+; GFX942-NEXT:    v_writelane_b32 v23, s50, 11
+; GFX942-NEXT:    v_writelane_b32 v23, s51, 12
+; GFX942-NEXT:    v_writelane_b32 v23, s52, 13
+; GFX942-NEXT:    v_writelane_b32 v23, s53, 14
 ; GFX942-NEXT:    s_add_i32 s0, s32, 64
-; GFX942-NEXT:    v_writelane_b32 v23, s61, 30
+; GFX942-NEXT:    v_writelane_b32 v23, s54, 15
 ; GFX942-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX942-NEXT:    s_and_b64 s[60:61], 0, exec
+; GFX942-NEXT:    v_writelane_b32 v23, s55, 16
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; use alloca0 v0
 ; GFX942-NEXT:    ;;#ASMEND
@@ -356,28 +270,14 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s59, scc
 ; GFX942-NEXT:    ;;#ASMEND
-; GFX942-NEXT:    v_readlane_b32 s61, v23, 30
-; GFX942-NEXT:    v_readlane_b32 s60, v23, 29
-; GFX942-NEXT:    v_readlane_b32 s59, v23, 28
-; GFX942-NEXT:    v_readlane_b32 s58, v23, 27
-; GFX942-NEXT:    v_readlane_b32 s57, v23, 26
-; GFX942-NEXT:    v_readlane_b32 s56, v23, 25
-; GFX942-NEXT:    v_readlane_b32 s55, v23, 24
-; GFX942-NEXT:    v_readlane_b32 s54, v23, 23
-; GFX942-NEXT:    v_readlane_b32 s53, v23, 22
-; GFX942-NEXT:    v_readlane_b32 s52, v23, 21
-; GFX942-NEXT:    v_readlane_b32 s51, v23, 20
-; GFX942-NEXT:    v_readlane_b32 s50, v23, 19
-; GFX942-NEXT:    v_readlane_b32 s49, v23, 18
-; GFX942-NEXT:    v_readlane_b32 s48, v23, 17
-; GFX942-NEXT:    v_readlane_b32 s47, v23, 16
-; GFX942-NEXT:    v_readlane_b32 s46, v23, 15
-; GFX942-NEXT:    v_readlane_b32 s45, v23, 14
-; GFX942-NEXT:    v_readlane_b32 s44, v23, 13
-; GFX942-NEXT:    v_readlane_b32 s43, v23, 12
-; GFX942-NEXT:    v_readlane_b32 s42, v23, 11
-; GFX942-NEXT:    v_readlane_b32 s41, v23, 10
-; GFX942-NEXT:    v_readlane_b32 s40, v23, 9
+; GFX942-NEXT:    v_readlane_b32 s55, v23, 16
+; GFX942-NEXT:    v_readlane_b32 s54, v23, 15
+; GFX942-NEXT:    v_readlane_b32 s53, v23, 14
+; GFX942-NEXT:    v_readlane_b32 s52, v23, 13
+; GFX942-NEXT:    v_readlane_b32 s51, v23, 12
+; GFX942-NEXT:    v_readlane_b32 s50, v23, 11
+; GFX942-NEXT:    v_readlane_b32 s49, v23, 10
+; GFX942-NEXT:    v_readlane_b32 s48, v23, 9
 ; GFX942-NEXT:    v_readlane_b32 s39, v23, 8
 ; GFX942-NEXT:    v_readlane_b32 s38, v23, 7
 ; GFX942-NEXT:    v_readlane_b32 s37, v23, 6
@@ -417,55 +317,31 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0
 ; GFX10_1-NEXT:    v_writelane_b32 v23, s37, 6
 ; GFX10_1-NEXT:    v_writelane_b32 v23, s38, 7
 ; GFX10_1-NEXT:    v_writelane_b32 v23, s39, 8
-; GFX10_1-NEXT:    v_writelane_b32 v23, s40, 9
-; GFX10_1-NEXT:    v_writelane_b32 v23, s41, 10
-; GFX10_1-NEXT:    v_writelane_b32 v23, s42, 11
-; GFX10_1-NEXT:    v_writelane_b32 v23, s43, 12
-; GFX10_1-NEXT:    v_writelane_b32 v23, s44, 13
-; GFX10_1-NEXT:    v_writelane_b32 v23, s45, 14
-; GFX10_1-NEXT:    v_writelane_b32 v23, s46, 15
-; GFX10_1-NEXT:    v_writelane_b32 v23, s47, 16
-; GFX10_1-NEXT:    v_writelane_b32 v23, s48, 17
-; GFX10_1-NEXT:    v_writelane_b32 v23, s49, 18
-; GFX10_1-NEXT:    v_writelane_b32 v23, s50, 19
-; GFX10_1-NEXT:    v_writelane_b32 v23, s51, 20
-; GFX10_1-NEXT:    v_writelane_b32 v23, s52, 21
-; GFX10_1-NEXT:    v_writelane_b32 v23, s53, 22
-; GFX10_1-NEXT:    v_writelane_b32 v23, s54, 23
-; GFX10_1-NEXT:    v_writelane_b32 v23, s55, 24
-; GFX10_1-NEXT:    v_writelane_b32 v23, s56, 25
-; GFX10_1-NEXT:    v_writelane_b32 v23, s57, 26
-; GFX10_1-NEXT:    v_writelane_b32 v23, s58, 27
+; GFX10_1-NEXT:    v_writelane_b32 v23, s48, 9
+; GFX10_1-NEXT:    v_writelane_b32 v23, s49, 10
+; GFX10_1-NEXT:    v_writelane_b32 v23, s50, 11
+; GFX10_1-NEXT:    v_writelane_b32 v23, s51, 12
+; GFX10_1-NEXT:    v_writelane_b32 v23, s52, 13
+; GFX10_1-NEXT:    v_writelane_b32 v23, s53, 14
+; GFX10_1-NEXT:    v_writelane_b32 v23, s54, 15
+; GFX10_1-NEXT:    v_writelane_b32 v23, s55, 16
 ; GFX10_1-NEXT:    ;;#ASMSTART
 ; GFX10_1-NEXT:    ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
 ; GFX10_1-NEXT:    ;;#ASMEND
 ; GFX10_1-NEXT:    v_lshrrev_b32_e64 v24, 5, s32
-; GFX10_1-NEXT:    v_writelane_b32 v23, s59, 28
 ; GFX10_1-NEXT:    v_add_nc_u32_e32 v24, 0x4040, v24
 ; GFX10_1-NEXT:    v_readfirstlane_b32 s59, v24
 ; GFX10_1-NEXT:    ;;#ASMSTART
 ; GFX10_1-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s59, scc
 ; GFX10_1-NEXT:    ;;#ASMEND
-; GFX10_1-NEXT:    v_readlane_b32 s59, v23, 28
-; GFX10_1-NEXT:    v_readlane_b32 s58, v23, 27
-; GFX10_1-NEXT:    v_readlane_b32 s57, v23, 26
-; GFX10_1-NEXT:    v_readlane_b32 s56, v23, 25
-; GFX10_1-NEXT:    v_readlane_b32 s55, v23, 24
-; GFX10_1-NEXT:    v_readlane_b32 s54, v23, 23
-; GFX10_1-NEXT:    v_readlane_b32 s53, v23, 22
-; GFX10_1-NEXT:    v_readlane_b32 s52, v23, 21
-; GFX10_1-NEXT:    v_readlane_b32 s51, v23, 20
-; GFX10_1-NEXT:    v_readlane_b32 s50, v23, 19
-; GFX10_1-NEXT:    v_readlane_b32 s49, v23, 18
-; GFX10_1-NEXT:    v_readlane_b32 s48, v23, 17
-; GFX10_1-NEXT:    v_readlane_b32 s47, v23, 16
-; GFX10_1-NEXT:    v_readlane_b32 s46, v23, 15
-; GFX10_1-NEXT:    v_readlane_b32 s45, v23, 14
-; GFX10_1-NEXT:    v_readlane_b32 s44, v23, 13
-; GFX10_1-NEXT:    v_readlane_b32 s43, v23, 12
-; GFX10_1-NEXT:    v_readlane_b32 s42, v23, 11
-; GFX10_1-NEXT:    v_readlane_b32 s41, v23, 10
-; GFX10_1-NEXT:    v_readlane_b32 s40, v23, 9
+; GFX10_1-NEXT:    v_readlane_b32 s55, v23, 16
+; GFX10_1-NEXT:    v_readlane_b32 s54, v23, 15
+; GFX10_1-NEXT:    v_readlane_b32 s53, v23, 14
+; GFX10_1-NEXT:    v_readlane_b32 s52, v23, 13
+; GFX10_1-NEXT:    v_readlane_b32 s51, v23, 12
+; GFX10_1-NEXT:    v_readlane_b32 s50, v23, 11
+; GFX10_1-NEXT:    v_readlane_b32 s49, v23, 10
+; GFX10_1-NEXT:    v_readlane_b32 s48, v23, 9
 ; GFX10_1-NEXT:    v_readlane_b32 s39, v23, 8
 ; GFX10_1-NEXT:    v_readlane_b32 s38, v23, 7
 ; GFX10_1-NEXT:    v_readlane_b32 s37, v23, 6
@@ -505,55 +381,31 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0
 ; GFX10_3-NEXT:    v_writelane_b32 v23, s37, 6
 ; GFX10_3-NEXT:    v_writelane_b32 v23, s38, 7
 ; GFX10_3-NEXT:    v_writelane_b32 v23, s39, 8
-; GFX10_3-NEXT:    v_writelane_b32 v23, s40, 9
-; GFX10_3-NEXT:    v_writelane_b32 v23, s41, 10
-; GFX10_3-NEXT:    v_writelane_b32 v23, s42, 11
-; GFX10_3-NEXT:    v_writelane_b32 v23, s43, 12
-; GFX10_3-NEXT:    v_writelane_b32 v23, s44, 13
-; GFX10_3-NEXT:    v_writelane_b32 v23, s45, 14
-; GFX10_3-NEXT:    v_writelane_b32 v23, s46, 15
-; GFX10_3-NEXT:    v_writelane_b32 v23, s47, 16
-; GFX10_3-NEXT:    v_writelane_b32 v23, s48, 17
-; GFX10_3-NEXT:    v_writelane_b32 v23, s49, 18
-; GFX10_3-NEXT:    v_writelane_b32 v23, s50, 19
-; GFX10_3-NEXT:    v_writelane_b32 v23, s51, 20
-; GFX10_3-NEXT:    v_writelane_b32 v23, s52, 21
-; GFX10_3-NEXT:    v_writelane_b32 v23, s53, 22
-; GFX10_3-NEXT:    v_writelane_b32 v23, s54, 23
-; GFX10_3-NEXT:    v_writelane_b32 v23, s55, 24
-; GFX10_3-NEXT:    v_writelane_b32 v23, s56, 25
-; GFX10_3-NEXT:    v_writelane_b32 v23, s57, 26
-; GFX10_3-NEXT:    v_writelane_b32 v23, s58, 27
+; GFX10_3-NEXT:    v_writelane_b32 v23, s48, 9
+; GFX10_3-NEXT:    v_writelane_b32 v23, s49, 10
+; GFX10_3-NEXT:    v_writelane_b32 v23, s50, 11
+; GFX10_3-NEXT:    v_writelane_b32 v23, s51, 12
+; GFX10_3-NEXT:    v_writelane_b32 v23, s52, 13
+; GFX10_3-NEXT:    v_writelane_b32 v23, s53, 14
+; GFX10_3-NEXT:    v_writelane_b32 v23, s54, 15
+; GFX10_3-NEXT:    v_writelane_b32 v23, s55, 16
 ; GFX10_3-NEXT:    ;;#ASMSTART
 ; GFX10_3-NEXT:    ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
 ; GFX10_3-NEXT:    ;;#ASMEND
 ; GFX10_3-NEXT:    v_lshrrev_b32_e64 v24, 5, s32
-; GFX10_3-NEXT:    v_writelane_b32 v23, s59, 28
 ; GFX10_3-NEXT:    v_add_nc_u32_e32 v24, 0x4040, v24
 ; GFX10_3-NEXT:    v_readfirstlane_b32 s59, v24
 ; GFX10_3-NEXT:    ;;#ASMSTART
 ; GFX10_3-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s59, scc
 ; GFX10_3-NEXT:    ;;#ASMEND
-; GFX10_3-NEXT:    v_readlane_b32 s59, v23, 28
-; GFX10_3-NEXT:    v_readlane_b32 s58, v23, 27
-; GFX10_3-NEXT:    v_readlane_b32 s57, v23, 26
-; GFX10_3-NEXT:    v_readlane_b32 s56, v23, 25
-; GFX10_3-NEXT:    v_readlane_b32 s55, v23, 24
-; GFX10_3-NEXT:    v_readlane_b32 s54, v23, 23
-; GFX10_3-NEXT:    v_readlane_b32 s53, v23, 22
-; GFX10_3-NEXT:    v_readlane_b32 s52, v23, 21
-; GFX10_3-NEXT:    v_readlane_b32 s51, v23, 20
-; GFX10_3-NEXT:    v_readlane_b32 s50, v23, 19
-; GFX10_3-NEXT:    v_readlane_b32 s49, v23, 18
-; GFX10_3-NEXT:    v_readlane_b32 s48, v23, 17
-; GFX10_3-NEXT:    v_readlane_b32 s47, v23, 16
-; GFX10_3-NEXT:    v_readlane_b32 s46, v23, 15
-; GFX10_3-NEXT:    v_readlane_b32 s45, v23, 14
-; GFX10_3-NEXT:    v_readlane_b32 s44, v23, 13
-; GFX10_3-NEXT:    v_readlane_b32 s43, v23, 12
-; GFX10_3-NEXT:    v_readlane_b32 s42, v23, 11
-; GFX10_3-NEXT:    v_readlane_b32 s41, v23, 10
-; GFX10_3-NEXT:    v_readlane_b32 s40, v23, 9
+; GFX10_3-NEXT:    v_readlane_b32 s55, v23, 16
+; GFX10_3-NEXT:    v_readlane_b32 s54, v23, 15
+; GFX10_3-NEXT:    v_readlane_b32 s53, v23, 14
+; GFX10_3-NEXT:    v_readlane_b32 s52, v23, 13
+; GFX10_3-NEXT:    v_readlane_b32 s51, v23, 12
+; GFX10_3-NEXT:    v_readlane_b32 s50, v23, 11
+; GFX10_3-NEXT:    v_readlane_b32 s49, v23, 10
+; GFX10_3-NEXT:    v_readlane_b32 s48, v23, 9
 ; GFX10_3-NEXT:    v_readlane_b32 s39, v23, 8
 ; GFX10_3-NEXT:    v_readlane_b32 s38, v23, 7
 ; GFX10_3-NEXT:    v_readlane_b32 s37, v23, 6
@@ -593,61 +445,33 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0
 ; GFX11-NEXT:    v_writelane_b32 v23, s37, 6
 ; GFX11-NEXT:    v_writelane_b32 v23, s38, 7
 ; GFX11-NEXT:    v_writelane_b32 v23, s39, 8
-; GFX11-NEXT:    v_writelane_b32 v23, s40, 9
-; GFX11-NEXT:    v_writelane_b32 v23, s41, 10
-; GFX11-NEXT:    v_writelane_b32 v23, s42, 11
-; GFX11-NEXT:    v_writelane_b32 v23, s43, 12
-; GFX11-NEXT:    v_writelane_b32 v23, s44, 13
-; GFX11-NEXT:    v_writelane_b32 v23, s45, 14
-; GFX11-NEXT:    v_writelane_b32 v23, s46, 15
-; GFX11-NEXT:    v_writelane_b32 v23, s47, 16
-; GFX11-NEXT:    v_writelane_b32 v23, s48, 17
-; GFX11-NEXT:    v_writelane_b32 v23, s49, 18
-; GFX11-NEXT:    v_writelane_b32 v23, s50, 19
-; GFX11-NEXT:    v_writelane_b32 v23, s51, 20
-; GFX11-NEXT:    v_writelane_b32 v23, s52, 21
-; GFX11-NEXT:    v_writelane_b32 v23, s53, 22
-; GFX11-NEXT:    v_writelane_b32 v23, s54, 23
-; GFX11-NEXT:    v_writelane_b32 v23, s55, 24
-; GFX11-NEXT:    v_writelane_b32 v23, s56, 25
-; GFX11-NEXT:    v_writelane_b32 v23, s57, 26
-; GFX11-NEXT:    v_writelane_b32 v23, s58, 27
+; GFX11-NEXT:    v_writelane_b32 v23, s48, 9
+; GFX11-NEXT:    v_writelane_b32 v23, s49, 10
+; GFX11-NEXT:    v_writelane_b32 v23, s50, 11
+; GFX11-NEXT:    v_writelane_b32 v23, s51, 12
+; GFX11-NEXT:    v_writelane_b32 v23, s52, 13
+; GFX11-NEXT:    v_writelane_b32 v23, s53, 14
+; GFX11-NEXT:    v_writelane_b32 v23, s54, 15
+; GFX11-NEXT:    v_writelane_b32 v23, s55, 16
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
 ; GFX11-NEXT:    ;;#ASMEND
-; GFX11-NEXT:    s_addc_u32 s32, s32, 0x4040
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    s_bitcmp1_b32 s32, 0
-; GFX11-NEXT:    v_writelane_b32 v23, s59, 28
-; GFX11-NEXT:    s_bitset0_b32 s32, 0
-; GFX11-NEXT:    s_mov_b32 s59, s32
-; GFX11-NEXT:    s_addc_u32 s32, s32, 0xffffbfc0
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_bitcmp1_b32 s32, 0
-; GFX11-NEXT:    s_bitset0_b32 s32, 0
+; GFX11-NEXT:    s_addc_u32 s60, s32, 0x4040
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_bitcmp1_b32 s60, 0
+; GFX11-NEXT:    s_bitset0_b32 s60, 0
+; GFX11-NEXT:    s_mov_b32 s59, s60
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s59, scc
 ; GFX11-NEXT:    ;;#ASMEND
-; GFX11-NEXT:    v_readlane_b32 s59, v23, 28
-; GFX11-NEXT:    v_readlane_b32 s58, v23, 27
-; GFX11-NEXT:    v_readlane_b32 s57, v23, 26
-; GFX11-NEXT:    v_readlane_b32 s56, v23, 25
-; GFX11-NEXT:    v_readlane_b32 s55, v23, 24
-; GFX11-NEXT:    v_readlane_b32 s54, v23, 23
-; GFX11-NEXT:    v_readlane_b32 s53, v23, 22
-; GFX11-NEXT:    v_readlane_b32 s52, v23, 21
-; GFX11-NEXT:    v_readlane_b32 s51, v23, 20
-; GFX11-NEXT:    v_readlane_b32 s50, v23, 19
-; GFX11-NEXT:    v_readlane_b32 s49, v23, 18
-; GFX11-NEXT:    v_readlane_b32 s48, v23, 17
-; GFX11-NEXT:    v_readlane_b32 s47, v23, 16
-; GFX11-NEXT:    v_readlane_b32 s46, v23, 15
-; GFX11-NEXT:    v_readlane_b32 s45, v23, 14
-; GFX11-NEXT:    v_readlane_b32 s44, v23, 13
-; GFX11-NEXT:    v_readlane_b32 s43, v23, 12
-; GFX11-NEXT:    v_readlane_b32 s42, v23, 11
-; GFX11-NEXT:    v_readlane_b32 s41, v23, 10
-; GFX11-NEXT:    v_readlane_b32 s40, v23, 9
+; GFX11-NEXT:    v_readlane_b32 s55, v23, 16
+; GFX11-NEXT:    v_readlane_b32 s54, v23, 15
+; GFX11-NEXT:    v_readlane_b32 s53, v23, 14
+; GFX11-NEXT:    v_readlane_b32 s52, v23, 13
+; GFX11-NEXT:    v_readlane_b32 s51, v23, 12
+; GFX11-NEXT:    v_readlane_b32 s50, v23, 11
+; GFX11-NEXT:    v_readlane_b32 s49, v23, 10
+; GFX11-NEXT:    v_readlane_b32 s48, v23, 9
 ; GFX11-NEXT:    v_readlane_b32 s39, v23, 8
 ; GFX11-NEXT:    v_readlane_b32 s38, v23, 7
 ; GFX11-NEXT:    v_readlane_b32 s37, v23, 6
@@ -689,62 +513,34 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0
 ; GFX12-NEXT:    v_writelane_b32 v23, s37, 6
 ; GFX12-NEXT:    v_writelane_b32 v23, s38, 7
 ; GFX12-NEXT:    v_writelane_b32 v23, s39, 8
-; GFX12-NEXT:    v_writelane_b32 v23, s40, 9
-; GFX12-NEXT:    v_writelane_b32 v23, s41, 10
-; GFX12-NEXT:    v_writelane_b32 v23, s42, 11
-; GFX12-NEXT:    v_writelane_b32 v23, s43, 12
-; GFX12-NEXT:    v_writelane_b32 v23, s44, 13
-; GFX12-NEXT:    v_writelane_b32 v23, s45, 14
-; GFX12-NEXT:    v_writelane_b32 v23, s46, 15
-; GFX12-NEXT:    v_writelane_b32 v23, s47, 16
-; GFX12-NEXT:    v_writelane_b32 v23, s48, 17
-; GFX12-NEXT:    v_writelane_b32 v23, s49, 18
-; GFX12-NEXT:    v_writelane_b32 v23, s50, 19
-; GFX12-NEXT:    v_writelane_b32 v23, s51, 20
-; GFX12-NEXT:    v_writelane_b32 v23, s52, 21
-; GFX12-NEXT:    v_writelane_b32 v23, s53, 22
-; GFX12-NEXT:    v_writelane_b32 v23, s54, 23
-; GFX12-NEXT:    v_writelane_b32 v23, s55, 24
-; GFX12-NEXT:    v_writelane_b32 v23, s56, 25
-; GFX12-NEXT:    v_writelane_b32 v23, s57, 26
-; GFX12-NEXT:    v_writelane_b32 v23, s58, 27
+; GFX12-NEXT:    v_writelane_b32 v23, s48, 9
+; GFX12-NEXT:    v_writelane_b32 v23, s49, 10
+; GFX12-NEXT:    v_writelane_b32 v23, s50, 11
+; GFX12-NEXT:    v_writelane_b32 v23, s51, 12
+; GFX12-NEXT:    v_writelane_b32 v23, s52, 13
+; GFX12-NEXT:    v_writelane_b32 v23, s53, 14
+; GFX12-NEXT:    v_writelane_b32 v23, s54, 15
+; GFX12-NEXT:    v_writelane_b32 v23, s55, 16
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc
 ; GFX12-NEXT:    ;;#ASMEND
-; GFX12-NEXT:    s_add_co_ci_u32 s32, s32, 0x4000
+; GFX12-NEXT:    s_add_co_ci_u32 s60, s32, 0x4000
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_bitcmp1_b32 s32, 0
-; GFX12-NEXT:    v_writelane_b32 v23, s59, 28
-; GFX12-NEXT:    s_bitset0_b32 s32, 0
+; GFX12-NEXT:    s_bitcmp1_b32 s60, 0
+; GFX12-NEXT:    s_bitset0_b32 s60, 0
 ; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_mov_b32 s59, s32
-; GFX12-NEXT:    s_add_co_ci_u32 s32, s32, 0xffffc000
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_bitcmp1_b32 s32, 0
-; GFX12-NEXT:    s_bitset0_b32 s32, 0
+; GFX12-NEXT:    s_mov_b32 s59, s60
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:22], vcc, s59, scc
 ; GFX12-NEXT:    ;;#ASMEND
-; GFX12-NEXT:    v_readlane_b32 s59, v23, 28
-; GFX12-NEXT:    v_readlane_b32 s58, v23, 27
-; GFX12-NEXT:    v_readlane_b32 s57, v23, 26
-; GFX12-NEXT:    v_readlane_b32 s56, v23, 25
-; GFX12-NEXT:    v_readlane_b32 s55, v23, 24
-; GFX12-NEXT:    v_readlane_b32 s54, v23, 23
-; GFX12-NEXT:    v_readlane_b32 s53, v23, 22
-; GFX12-NEXT:    v_readlane_b32 s52, v23, 21
-; GFX12-NEXT:    v_readlane_b32 s51, v23, 20
-; GFX12-NEXT:    v_readlane_b32 s50, v23, 19
-; GFX12-NEXT:    v_readlane_b32 s49, v23, 18
-; GFX12-NEXT:    v_readlane_b32 s48, v23, 17
-; GFX12-NEXT:    v_readlane_b32 s47, v23, 16
-; GFX12-NEXT:    v_readlane_b32 s46, v23, 15
-; GFX12-NEXT:    v_readlane_b32 s45, v23, 14
-; GFX12-NEXT:    v_readlane_b32 s44, v23, 13
-; GFX12-NEXT:    v_readlane_b32 s43, v23, 12
-; GFX12-NEXT:    v_readlane_b32 s42, v23, 11
-; GFX12-NEXT:    v_readlane_b32 s41, v23, 10
-; GFX12-NEXT:    v_readlane_b32 s40, v23, 9
+; GFX12-NEXT:    v_readlane_b32 s55, v23, 16
+; GFX12-NEXT:    v_readlane_b32 s54, v23, 15
+; GFX12-NEXT:    v_readlane_b32 s53, v23, 14
+; GFX12-NEXT:    v_readlane_b32 s52, v23, 13
+; GFX12-NEXT:    v_readlane_b32 s51, v23, 12
+; GFX12-NEXT:    v_readlane_b32 s50, v23, 11
+; GFX12-NEXT:    v_readlane_b32 s49, v23, 10
+; GFX12-NEXT:    v_readlane_b32 s48, v23, 9
 ; GFX12-NEXT:    v_readlane_b32 s39, v23, 8
 ; GFX12-NEXT:    v_readlane_b32 s38, v23, 7
 ; GFX12-NEXT:    v_readlane_b32 s37, v23, 6
@@ -819,56 +615,32 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowe
 ; GFX7-NEXT:    v_writelane_b32 v21, s37, 6
 ; GFX7-NEXT:    v_writelane_b32 v21, s38, 7
 ; GFX7-NEXT:    v_writelane_b32 v21, s39, 8
-; GFX7-NEXT:    v_writelane_b32 v21, s40, 9
-; GFX7-NEXT:    v_writelane_b32 v21, s41, 10
-; GFX7-NEXT:    v_writelane_b32 v21, s42, 11
-; GFX7-NEXT:    v_writelane_b32 v21, s43, 12
-; GFX7-NEXT:    v_writelane_b32 v21, s44, 13
-; GFX7-NEXT:    v_writelane_b32 v21, s45, 14
-; GFX7-NEXT:    v_writelane_b32 v21, s46, 15
-; GFX7-NEXT:    v_writelane_b32 v21, s47, 16
-; GFX7-NEXT:    v_writelane_b32 v21, s48, 17
-; GFX7-NEXT:    v_writelane_b32 v21, s49, 18
-; GFX7-NEXT:    v_writelane_b32 v21, s50, 19
-; GFX7-NEXT:    v_writelane_b32 v21, s51, 20
-; GFX7-NEXT:    v_writelane_b32 v21, s52, 21
-; GFX7-NEXT:    v_writelane_b32 v21, s53, 22
-; GFX7-NEXT:    v_writelane_b32 v21, s54, 23
-; GFX7-NEXT:    v_writelane_b32 v21, s55, 24
-; GFX7-NEXT:    v_writelane_b32 v21, s56, 25
-; GFX7-NEXT:    v_writelane_b32 v21, s57, 26
+; GFX7-NEXT:    v_writelane_b32 v21, s48, 9
+; GFX7-NEXT:    v_writelane_b32 v21, s49, 10
+; GFX7-NEXT:    v_writelane_b32 v21, s50, 11
+; GFX7-NEXT:    v_writelane_b32 v21, s51, 12
+; GFX7-NEXT:    v_writelane_b32 v21, s52, 13
+; GFX7-NEXT:    v_writelane_b32 v21, s53, 14
+; GFX7-NEXT:    v_writelane_b32 v21, s54, 15
 ; GFX7-NEXT:    s_and_b64 s[4:5], 0, exec
-; GFX7-NEXT:    v_writelane_b32 v21, s58, 27
+; GFX7-NEXT:    v_writelane_b32 v21, s55, 16
 ; GFX7-NEXT:    ;;#ASMSTART
 ; GFX7-NEXT:    ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
 ; GFX7-NEXT:    ;;#ASMEND
 ; GFX7-NEXT:    v_mad_u32_u24 v22, 16, 64, s32
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v22, 6, v22
-; GFX7-NEXT:    v_writelane_b32 v21, s59, 28
 ; GFX7-NEXT:    v_readfirstlane_b32 s59, v22
 ; GFX7-NEXT:    ;;#ASMSTART
 ; GFX7-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s59, scc
 ; GFX7-NEXT:    ;;#ASMEND
-; GFX7-NEXT:    v_readlane_b32 s59, v21, 28
-; GFX7-NEXT:    v_readlane_b32 s58, v21, 27
-; GFX7-NEXT:    v_readlane_b32 s57, v21, 26
-; GFX7-NEXT:    v_readlane_b32 s56, v21, 25
-; GFX7-NEXT:    v_readlane_b32 s55, v21, 24
-; GFX7-NEXT:    v_readlane_b32 s54, v21, 23
-; GFX7-NEXT:    v_readlane_b32 s53, v21, 22
-; GFX7-NEXT:    v_readlane_b32 s52, v21, 21
-; GFX7-NEXT:    v_readlane_b32 s51, v21, 20
-; GFX7-NEXT:    v_readlane_b32 s50, v21, 19
-; GFX7-NEXT:    v_readlane_b32 s49, v21, 18
-; GFX7-NEXT:    v_readlane_b32 s48, v21, 17
-; GFX7-NEXT:    v_readlane_b32 s47, v21, 16
-; GFX7-NEXT:    v_readlane_b32 s46, v21, 15
-; GFX7-NEXT:    v_readlane_b32 s45, v21, 14
-; GFX7-NEXT:    v_readlane_b32 s44, v21, 13
-; GFX7-NEXT:    v_readlane_b32 s43, v21, 12
-; GFX7-NEXT:    v_readlane_b32 s42, v21, 11
-; GFX7-NEXT:    v_readlane_b32 s41, v21, 10
-; GFX7-NEXT:    v_readlane_b32 s40, v21, 9
+; GFX7-NEXT:    v_readlane_b32 s55, v21, 16
+; GFX7-NEXT:    v_readlane_b32 s54, v21, 15
+; GFX7-NEXT:    v_readlane_b32 s53, v21, 14
+; GFX7-NEXT:    v_readlane_b32 s52, v21, 13
+; GFX7-NEXT:    v_readlane_b32 s51, v21, 12
+; GFX7-NEXT:    v_readlane_b32 s50, v21, 11
+; GFX7-NEXT:    v_readlane_b32 s49, v21, 10
+; GFX7-NEXT:    v_readlane_b32 s48, v21, 9
 ; GFX7-NEXT:    v_readlane_b32 s39, v21, 8
 ; GFX7-NEXT:    v_readlane_b32 s38, v21, 7
 ; GFX7-NEXT:    v_readlane_b32 s37, v21, 6
@@ -901,56 +673,32 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowe
 ; GFX8-NEXT:    v_writelane_b32 v21, s37, 6
 ; GFX8-NEXT:    v_writelane_b32 v21, s38, 7
 ; GFX8-NEXT:    v_writelane_b32 v21, s39, 8
-; GFX8-NEXT:    v_writelane_b32 v21, s40, 9
-; GFX8-NEXT:    v_writelane_b32 v21, s41, 10
-; GFX8-NEXT:    v_writelane_b32 v21, s42, 11
-; GFX8-NEXT:    v_writelane_b32 v21, s43, 12
-; GFX8-NEXT:    v_writelane_b32 v21, s44, 13
-; GFX8-NEXT:    v_writelane_b32 v21, s45, 14
-; GFX8-NEXT:    v_writelane_b32 v21, s46, 15
-; GFX8-NEXT:    v_writelane_b32 v21, s47, 16
-; GFX8-NEXT:    v_writelane_b32 v21, s48, 17
-; GFX8-NEXT:    v_writelane_b32 v21, s49, 18
-; GFX8-NEXT:    v_writelane_b32 v21, s50, 19
-; GFX8-NEXT:    v_writelane_b32 v21, s51, 20
-; GFX8-NEXT:    v_writelane_b32 v21, s52, 21
-; GFX8-NEXT:    v_writelane_b32 v21, s53, 22
-; GFX8-NEXT:    v_writelane_b32 v21, s54, 23
-; GFX8-NEXT:    v_writelane_b32 v21, s55, 24
-; GFX8-NEXT:    v_writelane_b32 v21, s56, 25
-; GFX8-NEXT:    v_writelane_b32 v21, s57, 26
+; GFX8-NEXT:    v_writelane_b32 v21, s48, 9
+; GFX8-NEXT:    v_writelane_b32 v21, s49, 10
+; GFX8-NEXT:    v_writelane_b32 v21, s50, 11
+; GFX8-NEXT:    v_writelane_b32 v21, s51, 12
+; GFX8-NEXT:    v_writelane_b32 v21, s52, 13
+; GFX8-NEXT:    v_writelane_b32 v21, s53, 14
+; GFX8-NEXT:    v_writelane_b32 v21, s54, 15
 ; GFX8-NEXT:    s_and_b64 s[4:5], 0, exec
-; GFX8-NEXT:    v_writelane_b32 v21, s58, 27
+; GFX8-NEXT:    v_writelane_b32 v21, s55, 16
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
 ; GFX8-NEXT:    ;;#ASMEND
 ; GFX8-NEXT:    v_mad_u32_u24 v22, 16, 64, s32
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v22, 6, v22
-; GFX8-NEXT:    v_writelane_b32 v21, s59, 28
 ; GFX8-NEXT:    v_readfirstlane_b32 s59, v22
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s59, scc
 ; GFX8-NEXT:    ;;#ASMEND
-; GFX8-NEXT:    v_readlane_b32 s59, v21, 28
-; GFX8-NEXT:    v_readlane_b32 s58, v21, 27
-; GFX8-NEXT:    v_readlane_b32 s57, v21, 26
-; GFX8-NEXT:    v_readlane_b32 s56, v21, 25
-; GFX8-NEXT:    v_readlane_b32 s55, v21, 24
-; GFX8-NEXT:    v_readlane_b32 s54, v21, 23
-; GFX8-NEXT:    v_readlane_b32 s53, v21, 22
-; GFX8-NEXT:    v_readlane_b32 s52, v21, 21
-; GFX8-NEXT:    v_readlane_b32 s51, v21, 20
-; GFX8-NEXT:    v_readlane_b32 s50, v21, 19
-; GFX8-NEXT:    v_readlane_b32 s49, v21, 18
-; GFX8-NEXT:    v_readlane_b32 s48, v21, 17
-; GFX8-NEXT:    v_readlane_b32 s47, v21, 16
-; GFX8-NEXT:    v_readlane_b32 s46, v21, 15
-; GFX8-NEXT:    v_readlane_b32 s45, v21, 14
-; GFX8-NEXT:    v_readlane_b32 s44, v21, 13
-; GFX8-NEXT:    v_readlane_b32 s43, v21, 12
-; GFX8-NEXT:    v_readlane_b32 s42, v21, 11
-; GFX8-NEXT:    v_readlane_b32 s41, v21, 10
-; GFX8-NEXT:    v_readlane_b32 s40, v21, 9
+; GFX8-NEXT:    v_readlane_b32 s55, v21, 16
+; GFX8-NEXT:    v_readlane_b32 s54, v21, 15
+; GFX8-NEXT:    v_readlane_b32 s53, v21, 14
+; GFX8-NEXT:    v_readlane_b32 s52, v21, 13
+; GFX8-NEXT:    v_readlane_b32 s51, v21, 12
+; GFX8-NEXT:    v_readlane_b32 s50, v21, 11
+; GFX8-NEXT:    v_readlane_b32 s49, v21, 10
+; GFX8-NEXT:    v_readlane_b32 s48, v21, 9
 ; GFX8-NEXT:    v_readlane_b32 s39, v21, 8
 ; GFX8-NEXT:    v_readlane_b32 s38, v21, 7
 ; GFX8-NEXT:    v_readlane_b32 s37, v21, 6
@@ -983,56 +731,32 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowe
 ; GFX900-NEXT:    v_writelane_b32 v21, s37, 6
 ; GFX900-NEXT:    v_writelane_b32 v21, s38, 7
 ; GFX900-NEXT:    v_writelane_b32 v21, s39, 8
-; GFX900-NEXT:    v_writelane_b32 v21, s40, 9
-; GFX900-NEXT:    v_writelane_b32 v21, s41, 10
-; GFX900-NEXT:    v_writelane_b32 v21, s42, 11
-; GFX900-NEXT:    v_writelane_b32 v21, s43, 12
-; GFX900-NEXT:    v_writelane_b32 v21, s44, 13
-; GFX900-NEXT:    v_writelane_b32 v21, s45, 14
-; GFX900-NEXT:    v_writelane_b32 v21, s46, 15
-; GFX900-NEXT:    v_writelane_b32 v21, s47, 16
-; GFX900-NEXT:    v_writelane_b32 v21, s48, 17
-; GFX900-NEXT:    v_writelane_b32 v21, s49, 18
-; GFX900-NEXT:    v_writelane_b32 v21, s50, 19
-; GFX900-NEXT:    v_writelane_b32 v21, s51, 20
-; GFX900-NEXT:    v_writelane_b32 v21, s52, 21
-; GFX900-NEXT:    v_writelane_b32 v21, s53, 22
-; GFX900-NEXT:    v_writelane_b32 v21, s54, 23
-; GFX900-NEXT:    v_writelane_b32 v21, s55, 24
-; GFX900-NEXT:    v_writelane_b32 v21, s56, 25
-; GFX900-NEXT:    v_writelane_b32 v21, s57, 26
+; GFX900-NEXT:    v_writelane_b32 v21, s48, 9
+; GFX900-NEXT:    v_writelane_b32 v21, s49, 10
+; GFX900-NEXT:    v_writelane_b32 v21, s50, 11
+; GFX900-NEXT:    v_writelane_b32 v21, s51, 12
+; GFX900-NEXT:    v_writelane_b32 v21, s52, 13
+; GFX900-NEXT:    v_writelane_b32 v21, s53, 14
+; GFX900-NEXT:    v_writelane_b32 v21, s54, 15
 ; GFX900-NEXT:    s_and_b64 s[4:5], 0, exec
-; GFX900-NEXT:    v_writelane_b32 v21, s58, 27
+; GFX900-NEXT:    v_writelane_b32 v21, s55, 16
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
 ; GFX900-NEXT:    ;;#ASMEND
 ; GFX900-NEXT:    v_lshrrev_b32_e64 v22, 6, s32
 ; GFX900-NEXT:    v_add_u32_e32 v22, 16, v22
-; GFX900-NEXT:    v_writelane_b32 v21, s59, 28
 ; GFX900-NEXT:    v_readfirstlane_b32 s59, v22
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s59, scc
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s59, v21, 28
-; GFX900-NEXT:    v_readlane_b32 s58, v21, 27
-; GFX900-NEXT:    v_readlane_b32 s57, v21, 26
-; GFX900-NEXT:    v_readlane_b32 s56, v21, 25
-; GFX900-NEXT:    v_readlane_b32 s55, v21, 24
-; GFX900-NEXT:    v_readlane_b32 s54, v21, 23
-; GFX900-NEXT:    v_readlane_b32 s53, v21, 22
-; GFX900-NEXT:    v_readlane_b32 s52, v21, 21
-; GFX900-NEXT:    v_readlane_b32 s51, v21, 20
-; GFX900-NEXT:    v_readlane_b32 s50, v21, 19
-; GFX900-NEXT:    v_readlane_b32 s49, v21, 18
-; GFX900-NEXT:    v_readlane_b32 s48, v21, 17
-; GFX900-NEXT:    v_readlane_b32 s47, v21, 16
-; GFX900-NEXT:    v_readlane_b32 s46, v21, 15
-; GFX900-NEXT:    v_readlane_b32 s45, v21, 14
-; GFX900-NEXT:    v_readlane_b32 s44, v21, 13
-; GFX900-NEXT:    v_readlane_b32 s43, v21, 12
-; GFX900-NEXT:    v_readlane_b32 s42, v21, 11
-; GFX900-NEXT:    v_readlane_b32 s41, v21, 10
-; GFX900-NEXT:    v_readlane_b32 s40, v21, 9
+; GFX900-NEXT:    v_readlane_b32 s55, v21, 16
+; GFX900-NEXT:    v_readlane_b32 s54, v21, 15
+; GFX900-NEXT:    v_readlane_b32 s53, v21, 14
+; GFX900-NEXT:    v_readlane_b32 s52, v21, 13
+; GFX900-NEXT:    v_readlane_b32 s51, v21, 12
+; GFX900-NEXT:    v_readlane_b32 s50, v21, 11
+; GFX900-NEXT:    v_readlane_b32 s49, v21, 10
+; GFX900-NEXT:    v_readlane_b32 s48, v21, 9
 ; GFX900-NEXT:    v_readlane_b32 s39, v21, 8
 ; GFX900-NEXT:    v_readlane_b32 s38, v21, 7
 ; GFX900-NEXT:    v_readlane_b32 s37, v21, 6
@@ -1065,29 +789,15 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowe
 ; GFX942-NEXT:    v_writelane_b32 v21, s37, 6
 ; GFX942-NEXT:    v_writelane_b32 v21, s38, 7
 ; GFX942-NEXT:    v_writelane_b32 v21, s39, 8
-; GFX942-NEXT:    v_writelane_b32 v21, s40, 9
-; GFX942-NEXT:    v_writelane_b32 v21, s41, 10
-; GFX942-NEXT:    v_writelane_b32 v21, s42, 11
-; GFX942-NEXT:    v_writelane_b32 v21, s43, 12
-; GFX942-NEXT:    v_writelane_b32 v21, s44, 13
-; GFX942-NEXT:    v_writelane_b32 v21, s45, 14
-; GFX942-NEXT:    v_writelane_b32 v21, s46, 15
-; GFX942-NEXT:    v_writelane_b32 v21, s47, 16
-; GFX942-NEXT:    v_writelane_b32 v21, s48, 17
-; GFX942-NEXT:    v_writelane_b32 v21, s49, 18
-; GFX942-NEXT:    v_writelane_b32 v21, s50, 19
-; GFX942-NEXT:    v_writelane_b32 v21, s51, 20
-; GFX942-NEXT:    v_writelane_b32 v21, s52, 21
-; GFX942-NEXT:    v_writelane_b32 v21, s53, 22
-; GFX942-NEXT:    v_writelane_b32 v21, s54, 23
-; GFX942-NEXT:    v_writelane_b32 v21, s55, 24
-; GFX942-NEXT:    v_writelane_b32 v21, s56, 25
-; GFX942-NEXT:    v_writelane_b32 v21, s57, 26
-; GFX942-NEXT:    v_writelane_b32 v21, s58, 27
-; GFX942-NEXT:    v_writelane_b32 v21, s59, 28
-; GFX942-NEXT:    v_writelane_b32 v21, s60, 29
-; GFX942-NEXT:    v_writelane_b32 v21, s61, 30
+; GFX942-NEXT:    v_writelane_b32 v21, s48, 9
+; GFX942-NEXT:    v_writelane_b32 v21, s49, 10
+; GFX942-NEXT:    v_writelane_b32 v21, s50, 11
+; GFX942-NEXT:    v_writelane_b32 v21, s51, 12
+; GFX942-NEXT:    v_writelane_b32 v21, s52, 13
+; GFX942-NEXT:    v_writelane_b32 v21, s53, 14
+; GFX942-NEXT:    v_writelane_b32 v21, s54, 15
 ; GFX942-NEXT:    s_and_b64 s[60:61], 0, exec
+; GFX942-NEXT:    v_writelane_b32 v21, s55, 16
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
 ; GFX942-NEXT:    ;;#ASMEND
@@ -1098,28 +808,14 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowe
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s59, scc
 ; GFX942-NEXT:    ;;#ASMEND
-; GFX942-NEXT:    v_readlane_b32 s61, v21, 30
-; GFX942-NEXT:    v_readlane_b32 s60, v21, 29
-; GFX942-NEXT:    v_readlane_b32 s59, v21, 28
-; GFX942-NEXT:    v_readlane_b32 s58, v21, 27
-; GFX942-NEXT:    v_readlane_b32 s57, v21, 26
-; GFX942-NEXT:    v_readlane_b32 s56, v21, 25
-; GFX942-NEXT:    v_readlane_b32 s55, v21, 24
-; GFX942-NEXT:    v_readlane_b32 s54, v21, 23
-; GFX942-NEXT:    v_readlane_b32 s53, v21, 22
-; GFX942-NEXT:    v_readlane_b32 s52, v21, 21
-; GFX942-NEXT:    v_readlane_b32 s51, v21, 20
-; GFX942-NEXT:    v_readlane_b32 s50, v21, 19
-; GFX942-NEXT:    v_readlane_b32 s49, v21, 18
-; GFX942-NEXT:    v_readlane_b32 s48, v21, 17
-; GFX942-NEXT:    v_readlane_b32 s47, v21, 16
-; GFX942-NEXT:    v_readlane_b32 s46, v21, 15
-; GFX942-NEXT:    v_readlane_b32 s45, v21, 14
-; GFX942-NEXT:    v_readlane_b32 s44, v21, 13
-; GFX942-NEXT:    v_readlane_b32 s43, v21, 12
-; GFX942-NEXT:    v_readlane_b32 s42, v21, 11
-; GFX942-NEXT:    v_readlane_b32 s41, v21, 10
-; GFX942-NEXT:    v_readlane_b32 s40, v21, 9
+; GFX942-NEXT:    v_readlane_b32 s55, v21, 16
+; GFX942-NEXT:    v_readlane_b32 s54, v21, 15
+; GFX942-NEXT:    v_readlane_b32 s53, v21, 14
+; GFX942-NEXT:    v_readlane_b32 s52, v21, 13
+; GFX942-NEXT:    v_readlane_b32 s51, v21, 12
+; GFX942-NEXT:    v_readlane_b32 s50, v21, 11
+; GFX942-NEXT:    v_readlane_b32 s49, v21, 10
+; GFX942-NEXT:    v_readlane_b32 s48, v21, 9
 ; GFX942-NEXT:    v_readlane_b32 s39, v21, 8
 ; GFX942-NEXT:    v_readlane_b32 s38, v21, 7
 ; GFX942-NEXT:    v_readlane_b32 s37, v21, 6
@@ -1145,6 +841,7 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowe
 ; GFX10_1-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10_1-NEXT:    s_mov_b32 exec_lo, s4
 ; GFX10_1-NEXT:    v_writelane_b32 v21, s30, 0
+; GFX10_1-NEXT:    s_and_b32 s59, 0, exec_lo
 ; GFX10_1-NEXT:    v_writelane_b32 v21, s31, 1
 ; GFX10_1-NEXT:    v_writelane_b32 v21, s33, 2
 ; GFX10_1-NEXT:    v_writelane_b32 v21, s34, 3
@@ -1153,56 +850,31 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowe
 ; GFX10_1-NEXT:    v_writelane_b32 v21, s37, 6
 ; GFX10_1-NEXT:    v_writelane_b32 v21, s38, 7
 ; GFX10_1-NEXT:    v_writelane_b32 v21, s39, 8
-; GFX10_1-NEXT:    v_writelane_b32 v21, s40, 9
-; GFX10_1-NEXT:    v_writelane_b32 v21, s41, 10
-; GFX10_1-NEXT:    v_writelane_b32 v21, s42, 11
-; GFX10_1-NEXT:    v_writelane_b32 v21, s43, 12
-; GFX10_1-NEXT:    v_writelane_b32 v21, s44, 13
-; GFX10_1-NEXT:    v_writelane_b32 v21, s45, 14
-; GFX10_1-NEXT:    v_writelane_b32 v21, s46, 15
-; GFX10_1-NEXT:    v_writelane_b32 v21, s47, 16
-; GFX10_1-NEXT:    v_writelane_b32 v21, s48, 17
-; GFX10_1-NEXT:    v_writelane_b32 v21, s49, 18
-; GFX10_1-NEXT:    v_writelane_b32 v21, s50, 19
-; GFX10_1-NEXT:    v_writelane_b32 v21, s51, 20
-; GFX10_1-NEXT:    v_writelane_b32 v21, s52, 21
-; GFX10_1-NEXT:    v_writelane_b32 v21, s53, 22
-; GFX10_1-NEXT:    v_writelane_b32 v21, s54, 23
-; GFX10_1-NEXT:    v_writelane_b32 v21, s55, 24
-; GFX10_1-NEXT:    v_writelane_b32 v21, s56, 25
-; GFX10_1-NEXT:    v_writelane_b32 v21, s57, 26
-; GFX10_1-NEXT:    v_writelane_b32 v21, s58, 27
+; GFX10_1-NEXT:    v_writelane_b32 v21, s48, 9
+; GFX10_1-NEXT:    v_writelane_b32 v21, s49, 10
+; GFX10_1-NEXT:    v_writelane_b32 v21, s50, 11
+; GFX10_1-NEXT:    v_writelane_b32 v21, s51, 12
+; GFX10_1-NEXT:    v_writelane_b32 v21, s52, 13
+; GFX10_1-NEXT:    v_writelane_b32 v21, s53, 14
+; GFX10_1-NEXT:    v_writelane_b32 v21, s54, 15
+; GFX10_1-NEXT:    v_writelane_b32 v21, s55, 16
 ; GFX10_1-NEXT:    ;;#ASMSTART
 ; GFX10_1-NEXT:    ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
 ; GFX10_1-NEXT:    ;;#ASMEND
 ; GFX10_1-NEXT:    v_lshrrev_b32_e64 v22, 5, s32
-; GFX10_1-NEXT:    v_writelane_b32 v21, s59, 28
-; GFX10_1-NEXT:    s_and_b32 s59, 0, exec_lo
 ; GFX10_1-NEXT:    v_add_nc_u32_e32 v22, 16, v22
 ; GFX10_1-NEXT:    v_readfirstlane_b32 s59, v22
 ; GFX10_1-NEXT:    ;;#ASMSTART
 ; GFX10_1-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s59, scc
 ; GFX10_1-NEXT:    ;;#ASMEND
-; GFX10_1-NEXT:    v_readlane_b32 s59, v21, 28
-; GFX10_1-NEXT:    v_readlane_b32 s58, v21, 27
-; GFX10_1-NEXT:    v_readlane_b32 s57, v21, 26
-; GFX10_1-NEXT:    v_readlane_b32 s56, v21, 25
-; GFX10_1-NEXT:    v_readlane_b32 s55, v21, 24
-; GFX10_1-NEXT:    v_readlane_b32 s54, v21, 23
-; GFX10_1-NEXT:    v_readlane_b32 s53, v21, 22
-; GFX10_1-NEXT:    v_readlane_b32 s52, v21, 21
-; GFX10_1-NEXT:    v_readlane_b32 s51, v21, 20
-; GFX10_1-NEXT:    v_readlane_b32 s50, v21, 19
-; GFX10_1-NEXT:    v_readlane_b32 s49, v21, 18
-; GFX10_1-NEXT:    v_readlane_b32 s48, v21, 17
-; GFX10_1-NEXT:    v_readlane_b32 s47, v21, 16
-; GFX10_1-NEXT:    v_readlane_b32 s46, v21, 15
-; GFX10_1-NEXT:    v_readlane_b32 s45, v21, 14
-; GFX10_1-NEXT:    v_readlane_b32 s44, v21, 13
-; GFX10_1-NEXT:    v_readlane_b32 s43, v21, 12
-; GFX10_1-NEXT:    v_readlane_b32 s42, v21, 11
-; GFX10_1-NEXT:    v_readlane_b32 s41, v21, 10
-; GFX10_1-NEXT:    v_readlane_b32 s40, v21, 9
+; GFX10_1-NEXT:    v_readlane_b32 s55, v21, 16
+; GFX10_1-NEXT:    v_readlane_b32 s54, v21, 15
+; GFX10_1-NEXT:    v_readlane_b32 s53, v21, 14
+; GFX10_1-NEXT:    v_readlane_b32 s52, v21, 13
+; GFX10_1-NEXT:    v_readlane_b32 s51, v21, 12
+; GFX10_1-NEXT:    v_readlane_b32 s50, v21, 11
+; GFX10_1-NEXT:    v_readlane_b32 s49, v21, 10
+; GFX10_1-NEXT:    v_readlane_b32 s48, v21, 9
 ; GFX10_1-NEXT:    v_readlane_b32 s39, v21, 8
 ; GFX10_1-NEXT:    v_readlane_b32 s38, v21, 7
 ; GFX10_1-NEXT:    v_readlane_b32 s37, v21, 6
@@ -1228,6 +900,7 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowe
 ; GFX10_3-NEXT:    buffer_store_dword v21, off, s[0:3], s5 ; 4-byte Folded Spill
 ; GFX10_3-NEXT:    s_mov_b32 exec_lo, s4
 ; GFX10_3-NEXT:    v_writelane_b32 v21, s30, 0
+; GFX10_3-NEXT:    s_and_b32 s59, 0, exec_lo
 ; GFX10_3-NEXT:    v_writelane_b32 v21, s31, 1
 ; GFX10_3-NEXT:    v_writelane_b32 v21, s33, 2
 ; GFX10_3-NEXT:    v_writelane_b32 v21, s34, 3
@@ -1236,56 +909,31 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowe
 ; GFX10_3-NEXT:    v_writelane_b32 v21, s37, 6
 ; GFX10_3-NEXT:    v_writelane_b32 v21, s38, 7
 ; GFX10_3-NEXT:    v_writelane_b32 v21, s39, 8
-; GFX10_3-NEXT:    v_writelane_b32 v21, s40, 9
-; GFX10_3-NEXT:    v_writelane_b32 v21, s41, 10
-; GFX10_3-NEXT:    v_writelane_b32 v21, s42, 11
-; GFX10_3-NEXT:    v_writelane_b32 v21, s43, 12
-; GFX10_3-NEXT:    v_writelane_b32 v21, s44, 13
-; GFX10_3-NEXT:    v_writelane_b32 v21, s45, 14
-; GFX10_3-NEXT:    v_writelane_b32 v21, s46, 15
-; GFX10_3-NEXT:    v_writelane_b32 v21, s47, 16
-; GFX10_3-NEXT:    v_writelane_b32 v21, s48, 17
-; GFX10_3-NEXT:    v_writelane_b32 v21, s49, 18
-; GFX10_3-NEXT:    v_writelane_b32 v21, s50, 19
-; GFX10_3-NEXT:    v_writelane_b32 v21, s51, 20
-; GFX10_3-NEXT:    v_writelane_b32 v21, s52, 21
-; GFX10_3-NEXT:    v_writelane_b32 v21, s53, 22
-; GFX10_3-NEXT:    v_writelane_b32 v21, s54, 23
-; GFX10_3-NEXT:    v_writelane_b32 v21, s55, 24
-; GFX10_3-NEXT:    v_writelane_b32 v21, s56, 25
-; GFX10_3-NEXT:    v_writelane_b32 v21, s57, 26
-; GFX10_3-NEXT:    v_writelane_b32 v21, s58, 27
+; GFX10_3-NEXT:    v_writelane_b32 v21, s48, 9
+; GFX10_3-NEXT:    v_writelane_b32 v21, s49, 10
+; GFX10_3-NEXT:    v_writelane_b32 v21, s50, 11
+; GFX10_3-NEXT:    v_writelane_b32 v21, s51, 12
+; GFX10_3-NEXT:    v_writelane_b32 v21, s52, 13
+; GFX10_3-NEXT:    v_writelane_b32 v21, s53, 14
+; GFX10_3-NEXT:    v_writelane_b32 v21, s54, 15
+; GFX10_3-NEXT:    v_writelane_b32 v21, s55, 16
 ; GFX10_3-NEXT:    ;;#ASMSTART
 ; GFX10_3-NEXT:    ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
 ; GFX10_3-NEXT:    ;;#ASMEND
 ; GFX10_3-NEXT:    v_lshrrev_b32_e64 v22, 5, s32
-; GFX10_3-NEXT:    v_writelane_b32 v21, s59, 28
-; GFX10_3-NEXT:    s_and_b32 s59, 0, exec_lo
 ; GFX10_3-NEXT:    v_add_nc_u32_e32 v22, 16, v22
 ; GFX10_3-NEXT:    v_readfirstlane_b32 s59, v22
 ; GFX10_3-NEXT:    ;;#ASMSTART
 ; GFX10_3-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s59, scc
 ; GFX10_3-NEXT:    ;;#ASMEND
-; GFX10_3-NEXT:    v_readlane_b32 s59, v21, 28
-; GFX10_3-NEXT:    v_readlane_b32 s58, v21, 27
-; GFX10_3-NEXT:    v_readlane_b32 s57, v21, 26
-; GFX10_3-NEXT:    v_readlane_b32 s56, v21, 25
-; GFX10_3-NEXT:    v_readlane_b32 s55, v21, 24
-; GFX10_3-NEXT:    v_readlane_b32 s54, v21, 23
-; GFX10_3-NEXT:    v_readlane_b32 s53, v21, 22
-; GFX10_3-NEXT:    v_readlane_b32 s52, v21, 21
-; GFX10_3-NEXT:    v_readlane_b32 s51, v21, 20
-; GFX10_3-NEXT:    v_readlane_b32 s50, v21, 19
-; GFX10_3-NEXT:    v_readlane_b32 s49, v21, 18
-; GFX10_3-NEXT:    v_readlane_b32 s48, v21, 17
-; GFX10_3-NEXT:    v_readlane_b32 s47, v21, 16
-; GFX10_3-NEXT:    v_readlane_b32 s46, v21, 15
-; GFX10_3-NEXT:    v_readlane_b32 s45, v21, 14
-; GFX10_3-NEXT:    v_readlane_b32 s44, v21, 13
-; GFX10_3-NEXT:    v_readlane_b32 s43, v21, 12
-; GFX10_3-NEXT:    v_readlane_b32 s42, v21, 11
-; GFX10_3-NEXT:    v_readlane_b32 s41, v21, 10
-; GFX10_3-NEXT:    v_readlane_b32 s40, v21, 9
+; GFX10_3-NEXT:    v_readlane_b32 s55, v21, 16
+; GFX10_3-NEXT:    v_readlane_b32 s54, v21, 15
+; GFX10_3-NEXT:    v_readlane_b32 s53, v21, 14
+; GFX10_3-NEXT:    v_readlane_b32 s52, v21, 13
+; GFX10_3-NEXT:    v_readlane_b32 s51, v21, 12
+; GFX10_3-NEXT:    v_readlane_b32 s50, v21, 11
+; GFX10_3-NEXT:    v_readlane_b32 s49, v21, 10
+; GFX10_3-NEXT:    v_readlane_b32 s48, v21, 9
 ; GFX10_3-NEXT:    v_readlane_b32 s39, v21, 8
 ; GFX10_3-NEXT:    v_readlane_b32 s38, v21, 7
 ; GFX10_3-NEXT:    v_readlane_b32 s37, v21, 6
@@ -1310,6 +958,7 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowe
 ; GFX11-NEXT:    scratch_store_b32 off, v21, s1 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX11-NEXT:    v_writelane_b32 v21, s30, 0
+; GFX11-NEXT:    s_and_b32 s59, 0, exec_lo
 ; GFX11-NEXT:    v_writelane_b32 v21, s31, 1
 ; GFX11-NEXT:    v_writelane_b32 v21, s33, 2
 ; GFX11-NEXT:    v_writelane_b32 v21, s34, 3
@@ -1318,62 +967,33 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowe
 ; GFX11-NEXT:    v_writelane_b32 v21, s37, 6
 ; GFX11-NEXT:    v_writelane_b32 v21, s38, 7
 ; GFX11-NEXT:    v_writelane_b32 v21, s39, 8
-; GFX11-NEXT:    v_writelane_b32 v21, s40, 9
-; GFX11-NEXT:    v_writelane_b32 v21, s41, 10
-; GFX11-NEXT:    v_writelane_b32 v21, s42, 11
-; GFX11-NEXT:    v_writelane_b32 v21, s43, 12
-; GFX11-NEXT:    v_writelane_b32 v21, s44, 13
-; GFX11-NEXT:    v_writelane_b32 v21, s45, 14
-; GFX11-NEXT:    v_writelane_b32 v21, s46, 15
-; GFX11-NEXT:    v_writelane_b32 v21, s47, 16
-; GFX11-NEXT:    v_writelane_b32 v21, s48, 17
-; GFX11-NEXT:    v_writelane_b32 v21, s49, 18
-; GFX11-NEXT:    v_writelane_b32 v21, s50, 19
-; GFX11-NEXT:    v_writelane_b32 v21, s51, 20
-; GFX11-NEXT:    v_writelane_b32 v21, s52, 21
-; GFX11-NEXT:    v_writelane_b32 v21, s53, 22
-; GFX11-NEXT:    v_writelane_b32 v21, s54, 23
-; GFX11-NEXT:    v_writelane_b32 v21, s55, 24
-; GFX11-NEXT:    v_writelane_b32 v21, s56, 25
-; GFX11-NEXT:    v_writelane_b32 v21, s57, 26
-; GFX11-NEXT:    v_writelane_b32 v21, s58, 27
+; GFX11-NEXT:    v_writelane_b32 v21, s48, 9
+; GFX11-NEXT:    v_writelane_b32 v21, s49, 10
+; GFX11-NEXT:    v_writelane_b32 v21, s50, 11
+; GFX11-NEXT:    v_writelane_b32 v21, s51, 12
+; GFX11-NEXT:    v_writelane_b32 v21, s52, 13
+; GFX11-NEXT:    v_writelane_b32 v21, s53, 14
+; GFX11-NEXT:    v_writelane_b32 v21, s54, 15
+; GFX11-NEXT:    v_writelane_b32 v21, s55, 16
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
 ; GFX11-NEXT:    ;;#ASMEND
-; GFX11-NEXT:    v_writelane_b32 v21, s59, 28
-; GFX11-NEXT:    s_and_b32 s59, 0, exec_lo
-; GFX11-NEXT:    s_addc_u32 s32, s32, 16
+; GFX11-NEXT:    s_addc_u32 s60, s32, 16
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    s_bitcmp1_b32 s32, 0
-; GFX11-NEXT:    s_bitset0_b32 s32, 0
-; GFX11-NEXT:    s_mov_b32 s59, s32
-; GFX11-NEXT:    s_addc_u32 s32, s32, -16
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_bitcmp1_b32 s32, 0
-; GFX11-NEXT:    s_bitset0_b32 s32, 0
+; GFX11-NEXT:    s_bitcmp1_b32 s60, 0
+; GFX11-NEXT:    s_bitset0_b32 s60, 0
+; GFX11-NEXT:    s_mov_b32 s59, s60
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s59, scc
 ; GFX11-NEXT:    ;;#ASMEND
-; GFX11-NEXT:    v_readlane_b32 s59, v21, 28
-; GFX11-NEXT:    v_readlane_b32 s58, v21, 27
-; GFX11-NEXT:    v_readlane_b32 s57, v21, 26
-; GFX11-NEXT:    v_readlane_b32 s56, v21, 25
-; GFX11-NEXT:    v_readlane_b32 s55, v21, 24
-; GFX11-NEXT:    v_readlane_b32 s54, v21, 23
-; GFX11-NEXT:    v_readlane_b32 s53, v21, 22
-; GFX11-NEXT:    v_readlane_b32 s52, v21, 21
-; GFX11-NEXT:    v_readlane_b32 s51, v21, 20
-; GFX11-NEXT:    v_readlane_b32 s50, v21, 19
-; GFX11-NEXT:    v_readlane_b32 s49, v21, 18
-; GFX11-NEXT:    v_readlane_b32 s48, v21, 17
-; GFX11-NEXT:    v_readlane_b32 s47, v21, 16
-; GFX11-NEXT:    v_readlane_b32 s46, v21, 15
-; GFX11-NEXT:    v_readlane_b32 s45, v21, 14
-; GFX11-NEXT:    v_readlane_b32 s44, v21, 13
-; GFX11-NEXT:    v_readlane_b32 s43, v21, 12
-; GFX11-NEXT:    v_readlane_b32 s42, v21, 11
-; GFX11-NEXT:    v_readlane_b32 s41, v21, 10
-; GFX11-NEXT:    v_readlane_b32 s40, v21, 9
+; GFX11-NEXT:    v_readlane_b32 s55, v21, 16
+; GFX11-NEXT:    v_readlane_b32 s54, v21, 15
+; GFX11-NEXT:    v_readlane_b32 s53, v21, 14
+; GFX11-NEXT:    v_readlane_b32 s52, v21, 13
+; GFX11-NEXT:    v_readlane_b32 s51, v21, 12
+; GFX11-NEXT:    v_readlane_b32 s50, v21, 11
+; GFX11-NEXT:    v_readlane_b32 s49, v21, 10
+; GFX11-NEXT:    v_readlane_b32 s48, v21, 9
 ; GFX11-NEXT:    v_readlane_b32 s39, v21, 8
 ; GFX11-NEXT:    v_readlane_b32 s38, v21, 7
 ; GFX11-NEXT:    v_readlane_b32 s37, v21, 6
@@ -1402,6 +1022,7 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowe
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX12-NEXT:    v_writelane_b32 v21, s30, 0
+; GFX12-NEXT:    s_and_b32 s59, 0, exec_lo
 ; GFX12-NEXT:    v_writelane_b32 v21, s31, 1
 ; GFX12-NEXT:    v_writelane_b32 v21, s33, 2
 ; GFX12-NEXT:    v_writelane_b32 v21, s34, 3
@@ -1410,55 +1031,30 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs__lowe
 ; GFX12-NEXT:    v_writelane_b32 v21, s37, 6
 ; GFX12-NEXT:    v_writelane_b32 v21, s38, 7
 ; GFX12-NEXT:    v_writelane_b32 v21, s39, 8
-; GFX12-NEXT:    v_writelane_b32 v21, s40, 9
-; GFX12-NEXT:    v_writelane_b32 v21, s41, 10
-; GFX12-NEXT:    v_writelane_b32 v21, s42, 11
-; GFX12-NEXT:    v_writelane_b32 v21, s43, 12
-; GFX12-NEXT:    v_writelane_b32 v21, s44, 13
-; GFX12-NEXT:    v_writelane_b32 v21, s45, 14
-; GFX12-NEXT:    v_writelane_b32 v21, s46, 15
-; GFX12-NEXT:    v_writelane_b32 v21, s47, 16
-; GFX12-NEXT:    v_writelane_b32 v21, s48, 17
-; GFX12-NEXT:    v_writelane_b32 v21, s49, 18
-; GFX12-NEXT:    v_writelane_b32 v21, s50, 19
-; GFX12-NEXT:    v_writelane_b32 v21, s51, 20
-; GFX12-NEXT:    v_writelane_b32 v21, s52, 21
-; GFX12-NEXT:    v_writelane_b32 v21, s53, 22
-; GFX12-NEXT:    v_writelane_b32 v21, s54, 23
-; GFX12-NEXT:    v_writelane_b32 v21, s55, 24
-; GFX12-NEXT:    v_writelane_b32 v21, s56, 25
-; GFX12-NEXT:    v_writelane_b32 v21, s57, 26
-; GFX12-NEXT:    v_writelane_b32 v21, s58, 27
+; GFX12-NEXT:    v_writelane_b32 v21, s48, 9
+; GFX12-NEXT:    v_writelane_b32 v21, s49, 10
+; GFX12-NEXT:    v_writelane_b32 v21, s50, 11
+; GFX12-NEXT:    v_writelane_b32 v21, s51, 12
+; GFX12-NEXT:    v_writelane_b32 v21, s52, 13
+; GFX12-NEXT:    v_writelane_b32 v21, s53, 14
+; GFX12-NEXT:    v_writelane_b32 v21, s54, 15
+; GFX12-NEXT:    v_writelane_b32 v21, s55, 16
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc
 ; GFX12-NEXT:    ;;#ASMEND
-; GFX12-NEXT:    v_writelane_b32 v21, s59, 28
-; GFX12-NEXT:    s_and_b32 s59, 0, exec_lo
 ; GFX12-NEXT:    s_mov_b32 s59, s32
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], s58, v[0:15], v[16:20], vcc, s59, scc
 ; GFX12-NEXT:    ;;#ASMEND
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT:    v_readlane_b32 s59, v21, 28
-; GFX12-NEXT:    v_readlane_b32 s58, v21, 27
-; GFX12-NEXT:    v_readlane_b32 s57, v21, 26
-; GFX12-NEXT:    v_readlane_b32 s56, v21, 25
-; GFX12-NEXT:    v_readlane_b32 s55, v21, 24
-; GFX12-NEXT:    v_readlane_b32 s54, v21, 23
-; GFX12-NEXT:    v_readlane_b32 s53, v21, 22
-; GFX12-NEXT:    v_readlane_b32 s52, v21, 21
-; GFX12-NEXT:    v_readlane_b32 s51, v21, 20
-; GFX12-NEXT:    v_readlane_b32 s50, v21, 19
-; GFX12-NEXT:    v_readlane_b32 s49, v21, 18
-; GFX12-NEXT:    v_readlane_b32 s48, v21, 17
-; GFX12-NEXT:    v_readlane_b32 s47, v21, 16
-; GFX12-NEXT:    v_readlane_b32 s46, v21, 15
-; GFX12-NEXT:    v_readlane_b32 s45, v21, 14
-; GFX12-NEXT:    v_readlane_b32 s44, v21, 13
-; GFX12-NEXT:    v_readlane_b32 s43, v21, 12
-; GFX12-NEXT:    v_readlane_b32 s42, v21, 11
-; GFX12-NEXT:    v_readlane_b32 s41, v21, 10
-; GFX12-NEXT:    v_readlane_b32 s40, v21, 9
+; GFX12-NEXT:    v_readlane_b32 s55, v21, 16
+; GFX12-NEXT:    v_readlane_b32 s54, v21, 15
+; GFX12-NEXT:    v_readlane_b32 s53, v21, 14
+; GFX12-NEXT:    v_readlane_b32 s52, v21, 13
+; GFX12-NEXT:    v_readlane_b32 s51, v21, 12
+; GFX12-NEXT:    v_readlane_b32 s50, v21, 11
+; GFX12-NEXT:    v_readlane_b32 s49, v21, 10
+; GFX12-NEXT:    v_readlane_b32 s48, v21, 9
 ; GFX12-NEXT:    v_readlane_b32 s39, v21, 8
 ; GFX12-NEXT:    v_readlane_b32 s38, v21, 7
 ; GFX12-NEXT:    v_readlane_b32 s37, v21, 6
@@ -1523,8 +1119,8 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX7-NEXT:    s_add_i32 s6, s32, 0x201100
 ; GFX7-NEXT:    buffer_store_dword v22, off, s[0:3], s6 ; 4-byte Folded Spill
 ; GFX7-NEXT:    s_mov_b64 exec, s[4:5]
-; GFX7-NEXT:    v_writelane_b32 v23, s28, 28
-; GFX7-NEXT:    v_writelane_b32 v23, s29, 29
+; GFX7-NEXT:    v_writelane_b32 v23, s28, 17
+; GFX7-NEXT:    v_writelane_b32 v23, s29, 18
 ; GFX7-NEXT:    v_writelane_b32 v23, s30, 0
 ; GFX7-NEXT:    v_writelane_b32 v23, s31, 1
 ; GFX7-NEXT:    v_writelane_b32 v23, s33, 2
@@ -1534,32 +1130,21 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX7-NEXT:    v_writelane_b32 v23, s37, 6
 ; GFX7-NEXT:    v_writelane_b32 v23, s38, 7
 ; GFX7-NEXT:    v_writelane_b32 v23, s39, 8
-; GFX7-NEXT:    v_writelane_b32 v23, s40, 9
-; GFX7-NEXT:    v_writelane_b32 v23, s41, 10
-; GFX7-NEXT:    v_writelane_b32 v23, s42, 11
-; GFX7-NEXT:    v_writelane_b32 v23, s43, 12
-; GFX7-NEXT:    v_writelane_b32 v23, s44, 13
-; GFX7-NEXT:    v_writelane_b32 v23, s45, 14
-; GFX7-NEXT:    v_writelane_b32 v23, s46, 15
-; GFX7-NEXT:    v_writelane_b32 v23, s47, 16
-; GFX7-NEXT:    v_writelane_b32 v23, s48, 17
-; GFX7-NEXT:    v_writelane_b32 v23, s49, 18
-; GFX7-NEXT:    v_writelane_b32 v23, s50, 19
-; GFX7-NEXT:    v_writelane_b32 v23, s51, 20
-; GFX7-NEXT:    v_writelane_b32 v23, s52, 21
-; GFX7-NEXT:    v_writelane_b32 v23, s53, 22
-; GFX7-NEXT:    v_writelane_b32 v23, s54, 23
-; GFX7-NEXT:    v_writelane_b32 v23, s55, 24
+; GFX7-NEXT:    v_writelane_b32 v23, s48, 9
+; GFX7-NEXT:    v_writelane_b32 v23, s49, 10
+; GFX7-NEXT:    v_writelane_b32 v23, s50, 11
+; GFX7-NEXT:    v_writelane_b32 v23, s51, 12
+; GFX7-NEXT:    v_writelane_b32 v23, s52, 13
 ; GFX7-NEXT:    s_lshr_b32 s5, s32, 6
-; GFX7-NEXT:    v_writelane_b32 v23, s56, 25
+; GFX7-NEXT:    v_writelane_b32 v23, s53, 14
 ; GFX7-NEXT:    v_lshr_b32_e64 v0, s32, 6
 ; GFX7-NEXT:    s_add_i32 s4, s5, 0x4240
 ; GFX7-NEXT:    ; implicit-def: $vgpr22 : SGPR spill to VGPR lane
-; GFX7-NEXT:    v_writelane_b32 v23, s57, 26
+; GFX7-NEXT:    v_writelane_b32 v23, s54, 15
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 64, v0
 ; GFX7-NEXT:    v_writelane_b32 v22, s4, 0
 ; GFX7-NEXT:    s_and_b64 s[4:5], 0, exec
-; GFX7-NEXT:    v_writelane_b32 v23, s59, 27
+; GFX7-NEXT:    v_writelane_b32 v23, s55, 16
 ; GFX7-NEXT:    ;;#ASMSTART
 ; GFX7-NEXT:    ; use alloca0 v0
 ; GFX7-NEXT:    ;;#ASMEND
@@ -1570,25 +1155,14 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX7-NEXT:    ;;#ASMSTART
 ; GFX7-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s59, scc
 ; GFX7-NEXT:    ;;#ASMEND
-; GFX7-NEXT:    v_readlane_b32 s59, v23, 27
-; GFX7-NEXT:    v_readlane_b32 s57, v23, 26
-; GFX7-NEXT:    v_readlane_b32 s56, v23, 25
-; GFX7-NEXT:    v_readlane_b32 s55, v23, 24
-; GFX7-NEXT:    v_readlane_b32 s54, v23, 23
-; GFX7-NEXT:    v_readlane_b32 s53, v23, 22
-; GFX7-NEXT:    v_readlane_b32 s52, v23, 21
-; GFX7-NEXT:    v_readlane_b32 s51, v23, 20
-; GFX7-NEXT:    v_readlane_b32 s50, v23, 19
-; GFX7-NEXT:    v_readlane_b32 s49, v23, 18
-; GFX7-NEXT:    v_readlane_b32 s48, v23, 17
-; GFX7-NEXT:    v_readlane_b32 s47, v23, 16
-; GFX7-NEXT:    v_readlane_b32 s46, v23, 15
-; GFX7-NEXT:    v_readlane_b32 s45, v23, 14
-; GFX7-NEXT:    v_readlane_b32 s44, v23, 13
-; GFX7-NEXT:    v_readlane_b32 s43, v23, 12
-; GFX7-NEXT:    v_readlane_b32 s42, v23, 11
-; GFX7-NEXT:    v_readlane_b32 s41, v23, 10
-; GFX7-NEXT:    v_readlane_b32 s40, v23, 9
+; GFX7-NEXT:    v_readlane_b32 s55, v23, 16
+; GFX7-NEXT:    v_readlane_b32 s54, v23, 15
+; GFX7-NEXT:    v_readlane_b32 s53, v23, 14
+; GFX7-NEXT:    v_readlane_b32 s52, v23, 13
+; GFX7-NEXT:    v_readlane_b32 s51, v23, 12
+; GFX7-NEXT:    v_readlane_b32 s50, v23, 11
+; GFX7-NEXT:    v_readlane_b32 s49, v23, 10
+; GFX7-NEXT:    v_readlane_b32 s48, v23, 9
 ; GFX7-NEXT:    v_readlane_b32 s39, v23, 8
 ; GFX7-NEXT:    v_readlane_b32 s38, v23, 7
 ; GFX7-NEXT:    v_readlane_b32 s37, v23, 6
@@ -1598,8 +1172,8 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX7-NEXT:    v_readlane_b32 s33, v23, 2
 ; GFX7-NEXT:    v_readlane_b32 s31, v23, 1
 ; GFX7-NEXT:    v_readlane_b32 s30, v23, 0
-; GFX7-NEXT:    v_readlane_b32 s28, v23, 28
-; GFX7-NEXT:    v_readlane_b32 s29, v23, 29
+; GFX7-NEXT:    v_readlane_b32 s28, v23, 17
+; GFX7-NEXT:    v_readlane_b32 s29, v23, 18
 ; GFX7-NEXT:    s_xor_saveexec_b64 s[4:5], -1
 ; GFX7-NEXT:    s_add_i32 s6, s32, 0x201000
 ; GFX7-NEXT:    buffer_load_dword v23, off, s[0:3], s6 ; 4-byte Folded Reload
@@ -1625,30 +1199,19 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX8-NEXT:    v_writelane_b32 v22, s37, 6
 ; GFX8-NEXT:    v_writelane_b32 v22, s38, 7
 ; GFX8-NEXT:    v_writelane_b32 v22, s39, 8
-; GFX8-NEXT:    v_writelane_b32 v22, s40, 9
-; GFX8-NEXT:    v_writelane_b32 v22, s41, 10
-; GFX8-NEXT:    v_writelane_b32 v22, s42, 11
-; GFX8-NEXT:    v_writelane_b32 v22, s43, 12
-; GFX8-NEXT:    v_writelane_b32 v22, s44, 13
-; GFX8-NEXT:    v_writelane_b32 v22, s45, 14
-; GFX8-NEXT:    v_writelane_b32 v22, s46, 15
-; GFX8-NEXT:    v_writelane_b32 v22, s47, 16
-; GFX8-NEXT:    v_writelane_b32 v22, s48, 17
-; GFX8-NEXT:    v_writelane_b32 v22, s49, 18
-; GFX8-NEXT:    v_writelane_b32 v22, s50, 19
-; GFX8-NEXT:    v_writelane_b32 v22, s51, 20
-; GFX8-NEXT:    v_writelane_b32 v22, s52, 21
-; GFX8-NEXT:    v_writelane_b32 v22, s53, 22
-; GFX8-NEXT:    v_writelane_b32 v22, s54, 23
-; GFX8-NEXT:    v_writelane_b32 v22, s55, 24
-; GFX8-NEXT:    v_writelane_b32 v22, s56, 25
-; GFX8-NEXT:    v_writelane_b32 v22, s57, 26
+; GFX8-NEXT:    v_writelane_b32 v22, s48, 9
+; GFX8-NEXT:    v_writelane_b32 v22, s49, 10
+; GFX8-NEXT:    v_writelane_b32 v22, s50, 11
+; GFX8-NEXT:    v_writelane_b32 v22, s51, 12
+; GFX8-NEXT:    v_writelane_b32 v22, s52, 13
 ; GFX8-NEXT:    s_lshr_b32 s4, s32, 6
-; GFX8-NEXT:    v_writelane_b32 v22, s59, 27
+; GFX8-NEXT:    v_writelane_b32 v22, s53, 14
 ; GFX8-NEXT:    v_lshrrev_b32_e64 v0, 6, s32
 ; GFX8-NEXT:    s_add_i32 s59, s4, 0x4240
+; GFX8-NEXT:    v_writelane_b32 v22, s54, 15
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 64, v0
 ; GFX8-NEXT:    s_and_b64 s[4:5], 0, exec
+; GFX8-NEXT:    v_writelane_b32 v22, s55, 16
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use alloca0 v0
 ; GFX8-NEXT:    ;;#ASMEND
@@ -1658,25 +1221,14 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s59, scc
 ; GFX8-NEXT:    ;;#ASMEND
-; GFX8-NEXT:    v_readlane_b32 s59, v22, 27
-; GFX8-NEXT:    v_readlane_b32 s57, v22, 26
-; GFX8-NEXT:    v_readlane_b32 s56, v22, 25
-; GFX8-NEXT:    v_readlane_b32 s55, v22, 24
-; GFX8-NEXT:    v_readlane_b32 s54, v22, 23
-; GFX8-NEXT:    v_readlane_b32 s53, v22, 22
-; GFX8-NEXT:    v_readlane_b32 s52, v22, 21
-; GFX8-NEXT:    v_readlane_b32 s51, v22, 20
-; GFX8-NEXT:    v_readlane_b32 s50, v22, 19
-; GFX8-NEXT:    v_readlane_b32 s49, v22, 18
-; GFX8-NEXT:    v_readlane_b32 s48, v22, 17
-; GFX8-NEXT:    v_readlane_b32 s47, v22, 16
-; GFX8-NEXT:    v_readlane_b32 s46, v22, 15
-; GFX8-NEXT:    v_readlane_b32 s45, v22, 14
-; GFX8-NEXT:    v_readlane_b32 s44, v22, 13
-; GFX8-NEXT:    v_readlane_b32 s43, v22, 12
-; GFX8-NEXT:    v_readlane_b32 s42, v22, 11
-; GFX8-NEXT:    v_readlane_b32 s41, v22, 10
-; GFX8-NEXT:    v_readlane_b32 s40, v22, 9
+; GFX8-NEXT:    v_readlane_b32 s55, v22, 16
+; GFX8-NEXT:    v_readlane_b32 s54, v22, 15
+; GFX8-NEXT:    v_readlane_b32 s53, v22, 14
+; GFX8-NEXT:    v_readlane_b32 s52, v22, 13
+; GFX8-NEXT:    v_readlane_b32 s51, v22, 12
+; GFX8-NEXT:    v_readlane_b32 s50, v22, 11
+; GFX8-NEXT:    v_readlane_b32 s49, v22, 10
+; GFX8-NEXT:    v_readlane_b32 s48, v22, 9
 ; GFX8-NEXT:    v_readlane_b32 s39, v22, 8
 ; GFX8-NEXT:    v_readlane_b32 s38, v22, 7
 ; GFX8-NEXT:    v_readlane_b32 s37, v22, 6
@@ -1709,30 +1261,19 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX900-NEXT:    v_writelane_b32 v22, s37, 6
 ; GFX900-NEXT:    v_writelane_b32 v22, s38, 7
 ; GFX900-NEXT:    v_writelane_b32 v22, s39, 8
-; GFX900-NEXT:    v_writelane_b32 v22, s40, 9
-; GFX900-NEXT:    v_writelane_b32 v22, s41, 10
-; GFX900-NEXT:    v_writelane_b32 v22, s42, 11
-; GFX900-NEXT:    v_writelane_b32 v22, s43, 12
-; GFX900-NEXT:    v_writelane_b32 v22, s44, 13
-; GFX900-NEXT:    v_writelane_b32 v22, s45, 14
-; GFX900-NEXT:    v_writelane_b32 v22, s46, 15
-; GFX900-NEXT:    v_writelane_b32 v22, s47, 16
-; GFX900-NEXT:    v_writelane_b32 v22, s48, 17
-; GFX900-NEXT:    v_writelane_b32 v22, s49, 18
-; GFX900-NEXT:    v_writelane_b32 v22, s50, 19
-; GFX900-NEXT:    v_writelane_b32 v22, s51, 20
-; GFX900-NEXT:    v_writelane_b32 v22, s52, 21
-; GFX900-NEXT:    v_writelane_b32 v22, s53, 22
-; GFX900-NEXT:    v_writelane_b32 v22, s54, 23
-; GFX900-NEXT:    v_writelane_b32 v22, s55, 24
-; GFX900-NEXT:    v_writelane_b32 v22, s56, 25
-; GFX900-NEXT:    v_writelane_b32 v22, s57, 26
+; GFX900-NEXT:    v_writelane_b32 v22, s48, 9
+; GFX900-NEXT:    v_writelane_b32 v22, s49, 10
+; GFX900-NEXT:    v_writelane_b32 v22, s50, 11
+; GFX900-NEXT:    v_writelane_b32 v22, s51, 12
+; GFX900-NEXT:    v_writelane_b32 v22, s52, 13
 ; GFX900-NEXT:    s_lshr_b32 s4, s32, 6
-; GFX900-NEXT:    v_writelane_b32 v22, s59, 27
+; GFX900-NEXT:    v_writelane_b32 v22, s53, 14
 ; GFX900-NEXT:    v_lshrrev_b32_e64 v0, 6, s32
 ; GFX900-NEXT:    s_add_i32 s59, s4, 0x4240
+; GFX900-NEXT:    v_writelane_b32 v22, s54, 15
 ; GFX900-NEXT:    v_add_u32_e32 v0, 64, v0
 ; GFX900-NEXT:    s_and_b64 s[4:5], 0, exec
+; GFX900-NEXT:    v_writelane_b32 v22, s55, 16
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use alloca0 v0
 ; GFX900-NEXT:    ;;#ASMEND
@@ -1742,25 +1283,14 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s59, scc
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s59, v22, 27
-; GFX900-NEXT:    v_readlane_b32 s57, v22, 26
-; GFX900-NEXT:    v_readlane_b32 s56, v22, 25
-; GFX900-NEXT:    v_readlane_b32 s55, v22, 24
-; GFX900-NEXT:    v_readlane_b32 s54, v22, 23
-; GFX900-NEXT:    v_readlane_b32 s53, v22, 22
-; GFX900-NEXT:    v_readlane_b32 s52, v22, 21
-; GFX900-NEXT:    v_readlane_b32 s51, v22, 20
-; GFX900-NEXT:    v_readlane_b32 s50, v22, 19
-; GFX900-NEXT:    v_readlane_b32 s49, v22, 18
-; GFX900-NEXT:    v_readlane_b32 s48, v22, 17
-; GFX900-NEXT:    v_readlane_b32 s47, v22, 16
-; GFX900-NEXT:    v_readlane_b32 s46, v22, 15
-; GFX900-NEXT:    v_readlane_b32 s45, v22, 14
-; GFX900-NEXT:    v_readlane_b32 s44, v22, 13
-; GFX900-NEXT:    v_readlane_b32 s43, v22, 12
-; GFX900-NEXT:    v_readlane_b32 s42, v22, 11
-; GFX900-NEXT:    v_readlane_b32 s41, v22, 10
-; GFX900-NEXT:    v_readlane_b32 s40, v22, 9
+; GFX900-NEXT:    v_readlane_b32 s55, v22, 16
+; GFX900-NEXT:    v_readlane_b32 s54, v22, 15
+; GFX900-NEXT:    v_readlane_b32 s53, v22, 14
+; GFX900-NEXT:    v_readlane_b32 s52, v22, 13
+; GFX900-NEXT:    v_readlane_b32 s51, v22, 12
+; GFX900-NEXT:    v_readlane_b32 s50, v22, 11
+; GFX900-NEXT:    v_readlane_b32 s49, v22, 10
+; GFX900-NEXT:    v_readlane_b32 s48, v22, 9
 ; GFX900-NEXT:    v_readlane_b32 s39, v22, 8
 ; GFX900-NEXT:    v_readlane_b32 s38, v22, 7
 ; GFX900-NEXT:    v_readlane_b32 s37, v22, 6
@@ -1793,28 +1323,16 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX942-NEXT:    v_writelane_b32 v22, s37, 6
 ; GFX942-NEXT:    v_writelane_b32 v22, s38, 7
 ; GFX942-NEXT:    v_writelane_b32 v22, s39, 8
-; GFX942-NEXT:    v_writelane_b32 v22, s40, 9
-; GFX942-NEXT:    v_writelane_b32 v22, s41, 10
-; GFX942-NEXT:    v_writelane_b32 v22, s42, 11
-; GFX942-NEXT:    v_writelane_b32 v22, s43, 12
-; GFX942-NEXT:    v_writelane_b32 v22, s44, 13
-; GFX942-NEXT:    v_writelane_b32 v22, s45, 14
-; GFX942-NEXT:    v_writelane_b32 v22, s46, 15
-; GFX942-NEXT:    v_writelane_b32 v22, s47, 16
-; GFX942-NEXT:    v_writelane_b32 v22, s48, 17
-; GFX942-NEXT:    v_writelane_b32 v22, s49, 18
-; GFX942-NEXT:    v_writelane_b32 v22, s50, 19
-; GFX942-NEXT:    v_writelane_b32 v22, s51, 20
-; GFX942-NEXT:    v_writelane_b32 v22, s52, 21
-; GFX942-NEXT:    v_writelane_b32 v22, s53, 22
-; GFX942-NEXT:    v_writelane_b32 v22, s54, 23
-; GFX942-NEXT:    v_writelane_b32 v22, s55, 24
-; GFX942-NEXT:    v_writelane_b32 v22, s56, 25
-; GFX942-NEXT:    v_writelane_b32 v22, s57, 26
+; GFX942-NEXT:    v_writelane_b32 v22, s48, 9
+; GFX942-NEXT:    v_writelane_b32 v22, s49, 10
+; GFX942-NEXT:    v_writelane_b32 v22, s50, 11
+; GFX942-NEXT:    v_writelane_b32 v22, s51, 12
+; GFX942-NEXT:    v_writelane_b32 v22, s52, 13
+; GFX942-NEXT:    v_writelane_b32 v22, s53, 14
 ; GFX942-NEXT:    s_add_i32 s0, s32, 64
-; GFX942-NEXT:    v_writelane_b32 v22, s59, 27
+; GFX942-NEXT:    v_writelane_b32 v22, s54, 15
 ; GFX942-NEXT:    v_mov_b32_e32 v0, s0
-; GFX942-NEXT:    v_writelane_b32 v22, s60, 28
+; GFX942-NEXT:    v_writelane_b32 v22, s55, 16
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; use alloca0 v0
 ; GFX942-NEXT:    ;;#ASMEND
@@ -1822,32 +1340,18 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX942-NEXT:    ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
 ; GFX942-NEXT:    ;;#ASMEND
 ; GFX942-NEXT:    s_add_i32 s59, s32, 0x4240
-; GFX942-NEXT:    v_writelane_b32 v22, s61, 29
 ; GFX942-NEXT:    s_and_b64 s[60:61], 0, exec
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s59, scc
 ; GFX942-NEXT:    ;;#ASMEND
-; GFX942-NEXT:    v_readlane_b32 s61, v22, 29
-; GFX942-NEXT:    v_readlane_b32 s60, v22, 28
-; GFX942-NEXT:    v_readlane_b32 s59, v22, 27
-; GFX942-NEXT:    v_readlane_b32 s57, v22, 26
-; GFX942-NEXT:    v_readlane_b32 s56, v22, 25
-; GFX942-NEXT:    v_readlane_b32 s55, v22, 24
-; GFX942-NEXT:    v_readlane_b32 s54, v22, 23
-; GFX942-NEXT:    v_readlane_b32 s53, v22, 22
-; GFX942-NEXT:    v_readlane_b32 s52, v22, 21
-; GFX942-NEXT:    v_readlane_b32 s51, v22, 20
-; GFX942-NEXT:    v_readlane_b32 s50, v22, 19
-; GFX942-NEXT:    v_readlane_b32 s49, v22, 18
-; GFX942-NEXT:    v_readlane_b32 s48, v22, 17
-; GFX942-NEXT:    v_readlane_b32 s47, v22, 16
-; GFX942-NEXT:    v_readlane_b32 s46, v22, 15
-; GFX942-NEXT:    v_readlane_b32 s45, v22, 14
-; GFX942-NEXT:    v_readlane_b32 s44, v22, 13
-; GFX942-NEXT:    v_readlane_b32 s43, v22, 12
-; GFX942-NEXT:    v_readlane_b32 s42, v22, 11
-; GFX942-NEXT:    v_readlane_b32 s41, v22, 10
-; GFX942-NEXT:    v_readlane_b32 s40, v22, 9
+; GFX942-NEXT:    v_readlane_b32 s55, v22, 16
+; GFX942-NEXT:    v_readlane_b32 s54, v22, 15
+; GFX942-NEXT:    v_readlane_b32 s53, v22, 14
+; GFX942-NEXT:    v_readlane_b32 s52, v22, 13
+; GFX942-NEXT:    v_readlane_b32 s51, v22, 12
+; GFX942-NEXT:    v_readlane_b32 s50, v22, 11
+; GFX942-NEXT:    v_readlane_b32 s49, v22, 10
+; GFX942-NEXT:    v_readlane_b32 s48, v22, 9
 ; GFX942-NEXT:    v_readlane_b32 s39, v22, 8
 ; GFX942-NEXT:    v_readlane_b32 s38, v22, 7
 ; GFX942-NEXT:    v_readlane_b32 s37, v22, 6
@@ -1875,8 +1379,10 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX10_1-NEXT:    v_writelane_b32 v22, s30, 0
 ; GFX10_1-NEXT:    v_lshrrev_b32_e64 v0, 5, s32
 ; GFX10_1-NEXT:    s_lshr_b32 s4, s32, 5
+; GFX10_1-NEXT:    s_add_i32 s59, s4, 0x4240
 ; GFX10_1-NEXT:    v_writelane_b32 v22, s31, 1
 ; GFX10_1-NEXT:    v_add_nc_u32_e32 v0, 64, v0
+; GFX10_1-NEXT:    s_and_b32 s4, 0, exec_lo
 ; GFX10_1-NEXT:    ;;#ASMSTART
 ; GFX10_1-NEXT:    ; use alloca0 v0
 ; GFX10_1-NEXT:    ;;#ASMEND
@@ -1887,52 +1393,28 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX10_1-NEXT:    v_writelane_b32 v22, s37, 6
 ; GFX10_1-NEXT:    v_writelane_b32 v22, s38, 7
 ; GFX10_1-NEXT:    v_writelane_b32 v22, s39, 8
-; GFX10_1-NEXT:    v_writelane_b32 v22, s40, 9
-; GFX10_1-NEXT:    v_writelane_b32 v22, s41, 10
-; GFX10_1-NEXT:    v_writelane_b32 v22, s42, 11
-; GFX10_1-NEXT:    v_writelane_b32 v22, s43, 12
-; GFX10_1-NEXT:    v_writelane_b32 v22, s44, 13
-; GFX10_1-NEXT:    v_writelane_b32 v22, s45, 14
-; GFX10_1-NEXT:    v_writelane_b32 v22, s46, 15
-; GFX10_1-NEXT:    v_writelane_b32 v22, s47, 16
-; GFX10_1-NEXT:    v_writelane_b32 v22, s48, 17
-; GFX10_1-NEXT:    v_writelane_b32 v22, s49, 18
-; GFX10_1-NEXT:    v_writelane_b32 v22, s50, 19
-; GFX10_1-NEXT:    v_writelane_b32 v22, s51, 20
-; GFX10_1-NEXT:    v_writelane_b32 v22, s52, 21
-; GFX10_1-NEXT:    v_writelane_b32 v22, s53, 22
-; GFX10_1-NEXT:    v_writelane_b32 v22, s54, 23
-; GFX10_1-NEXT:    v_writelane_b32 v22, s55, 24
-; GFX10_1-NEXT:    v_writelane_b32 v22, s56, 25
-; GFX10_1-NEXT:    v_writelane_b32 v22, s57, 26
-; GFX10_1-NEXT:    v_writelane_b32 v22, s59, 27
-; GFX10_1-NEXT:    s_add_i32 s59, s4, 0x4240
-; GFX10_1-NEXT:    s_and_b32 s4, 0, exec_lo
+; GFX10_1-NEXT:    v_writelane_b32 v22, s48, 9
+; GFX10_1-NEXT:    v_writelane_b32 v22, s49, 10
+; GFX10_1-NEXT:    v_writelane_b32 v22, s50, 11
+; GFX10_1-NEXT:    v_writelane_b32 v22, s51, 12
+; GFX10_1-NEXT:    v_writelane_b32 v22, s52, 13
+; GFX10_1-NEXT:    v_writelane_b32 v22, s53, 14
+; GFX10_1-NEXT:    v_writelane_b32 v22, s54, 15
+; GFX10_1-NEXT:    v_writelane_b32 v22, s55, 16
 ; GFX10_1-NEXT:    ;;#ASMSTART
 ; GFX10_1-NEXT:    ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
 ; GFX10_1-NEXT:    ;;#ASMEND
 ; GFX10_1-NEXT:    ;;#ASMSTART
 ; GFX10_1-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s59, scc
 ; GFX10_1-NEXT:    ;;#ASMEND
-; GFX10_1-NEXT:    v_readlane_b32 s59, v22, 27
-; GFX10_1-NEXT:    v_readlane_b32 s57, v22, 26
-; GFX10_1-NEXT:    v_readlane_b32 s56, v22, 25
-; GFX10_1-NEXT:    v_readlane_b32 s55, v22, 24
-; GFX10_1-NEXT:    v_readlane_b32 s54, v22, 23
-; GFX10_1-NEXT:    v_readlane_b32 s53, v22, 22
-; GFX10_1-NEXT:    v_readlane_b32 s52, v22, 21
-; GFX10_1-NEXT:    v_readlane_b32 s51, v22, 20
-; GFX10_1-NEXT:    v_readlane_b32 s50, v22, 19
-; GFX10_1-NEXT:    v_readlane_b32 s49, v22, 18
-; GFX10_1-NEXT:    v_readlane_b32 s48, v22, 17
-; GFX10_1-NEXT:    v_readlane_b32 s47, v22, 16
-; GFX10_1-NEXT:    v_readlane_b32 s46, v22, 15
-; GFX10_1-NEXT:    v_readlane_b32 s45, v22, 14
-; GFX10_1-NEXT:    v_readlane_b32 s44, v22, 13
-; GFX10_1-NEXT:    v_readlane_b32 s43, v22, 12
-; GFX10_1-NEXT:    v_readlane_b32 s42, v22, 11
-; GFX10_1-NEXT:    v_readlane_b32 s41, v22, 10
-; GFX10_1-NEXT:    v_readlane_b32 s40, v22, 9
+; GFX10_1-NEXT:    v_readlane_b32 s55, v22, 16
+; GFX10_1-NEXT:    v_readlane_b32 s54, v22, 15
+; GFX10_1-NEXT:    v_readlane_b32 s53, v22, 14
+; GFX10_1-NEXT:    v_readlane_b32 s52, v22, 13
+; GFX10_1-NEXT:    v_readlane_b32 s51, v22, 12
+; GFX10_1-NEXT:    v_readlane_b32 s50, v22, 11
+; GFX10_1-NEXT:    v_readlane_b32 s49, v22, 10
+; GFX10_1-NEXT:    v_readlane_b32 s48, v22, 9
 ; GFX10_1-NEXT:    v_readlane_b32 s39, v22, 8
 ; GFX10_1-NEXT:    v_readlane_b32 s38, v22, 7
 ; GFX10_1-NEXT:    v_readlane_b32 s37, v22, 6
@@ -1960,8 +1442,10 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX10_3-NEXT:    v_writelane_b32 v22, s30, 0
 ; GFX10_3-NEXT:    v_lshrrev_b32_e64 v0, 5, s32
 ; GFX10_3-NEXT:    s_lshr_b32 s4, s32, 5
+; GFX10_3-NEXT:    s_add_i32 s59, s4, 0x4240
 ; GFX10_3-NEXT:    v_writelane_b32 v22, s31, 1
 ; GFX10_3-NEXT:    v_add_nc_u32_e32 v0, 64, v0
+; GFX10_3-NEXT:    s_and_b32 s4, 0, exec_lo
 ; GFX10_3-NEXT:    ;;#ASMSTART
 ; GFX10_3-NEXT:    ; use alloca0 v0
 ; GFX10_3-NEXT:    ;;#ASMEND
@@ -1972,52 +1456,28 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX10_3-NEXT:    v_writelane_b32 v22, s37, 6
 ; GFX10_3-NEXT:    v_writelane_b32 v22, s38, 7
 ; GFX10_3-NEXT:    v_writelane_b32 v22, s39, 8
-; GFX10_3-NEXT:    v_writelane_b32 v22, s40, 9
-; GFX10_3-NEXT:    v_writelane_b32 v22, s41, 10
-; GFX10_3-NEXT:    v_writelane_b32 v22, s42, 11
-; GFX10_3-NEXT:    v_writelane_b32 v22, s43, 12
-; GFX10_3-NEXT:    v_writelane_b32 v22, s44, 13
-; GFX10_3-NEXT:    v_writelane_b32 v22, s45, 14
-; GFX10_3-NEXT:    v_writelane_b32 v22, s46, 15
-; GFX10_3-NEXT:    v_writelane_b32 v22, s47, 16
-; GFX10_3-NEXT:    v_writelane_b32 v22, s48, 17
-; GFX10_3-NEXT:    v_writelane_b32 v22, s49, 18
-; GFX10_3-NEXT:    v_writelane_b32 v22, s50, 19
-; GFX10_3-NEXT:    v_writelane_b32 v22, s51, 20
-; GFX10_3-NEXT:    v_writelane_b32 v22, s52, 21
-; GFX10_3-NEXT:    v_writelane_b32 v22, s53, 22
-; GFX10_3-NEXT:    v_writelane_b32 v22, s54, 23
-; GFX10_3-NEXT:    v_writelane_b32 v22, s55, 24
-; GFX10_3-NEXT:    v_writelane_b32 v22, s56, 25
-; GFX10_3-NEXT:    v_writelane_b32 v22, s57, 26
-; GFX10_3-NEXT:    v_writelane_b32 v22, s59, 27
-; GFX10_3-NEXT:    s_add_i32 s59, s4, 0x4240
-; GFX10_3-NEXT:    s_and_b32 s4, 0, exec_lo
+; GFX10_3-NEXT:    v_writelane_b32 v22, s48, 9
+; GFX10_3-NEXT:    v_writelane_b32 v22, s49, 10
+; GFX10_3-NEXT:    v_writelane_b32 v22, s50, 11
+; GFX10_3-NEXT:    v_writelane_b32 v22, s51, 12
+; GFX10_3-NEXT:    v_writelane_b32 v22, s52, 13
+; GFX10_3-NEXT:    v_writelane_b32 v22, s53, 14
+; GFX10_3-NEXT:    v_writelane_b32 v22, s54, 15
+; GFX10_3-NEXT:    v_writelane_b32 v22, s55, 16
 ; GFX10_3-NEXT:    ;;#ASMSTART
 ; GFX10_3-NEXT:    ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
 ; GFX10_3-NEXT:    ;;#ASMEND
 ; GFX10_3-NEXT:    ;;#ASMSTART
 ; GFX10_3-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s59, scc
 ; GFX10_3-NEXT:    ;;#ASMEND
-; GFX10_3-NEXT:    v_readlane_b32 s59, v22, 27
-; GFX10_3-NEXT:    v_readlane_b32 s57, v22, 26
-; GFX10_3-NEXT:    v_readlane_b32 s56, v22, 25
-; GFX10_3-NEXT:    v_readlane_b32 s55, v22, 24
-; GFX10_3-NEXT:    v_readlane_b32 s54, v22, 23
-; GFX10_3-NEXT:    v_readlane_b32 s53, v22, 22
-; GFX10_3-NEXT:    v_readlane_b32 s52, v22, 21
-; GFX10_3-NEXT:    v_readlane_b32 s51, v22, 20
-; GFX10_3-NEXT:    v_readlane_b32 s50, v22, 19
-; GFX10_3-NEXT:    v_readlane_b32 s49, v22, 18
-; GFX10_3-NEXT:    v_readlane_b32 s48, v22, 17
-; GFX10_3-NEXT:    v_readlane_b32 s47, v22, 16
-; GFX10_3-NEXT:    v_readlane_b32 s46, v22, 15
-; GFX10_3-NEXT:    v_readlane_b32 s45, v22, 14
-; GFX10_3-NEXT:    v_readlane_b32 s44, v22, 13
-; GFX10_3-NEXT:    v_readlane_b32 s43, v22, 12
-; GFX10_3-NEXT:    v_readlane_b32 s42, v22, 11
-; GFX10_3-NEXT:    v_readlane_b32 s41, v22, 10
-; GFX10_3-NEXT:    v_readlane_b32 s40, v22, 9
+; GFX10_3-NEXT:    v_readlane_b32 s55, v22, 16
+; GFX10_3-NEXT:    v_readlane_b32 s54, v22, 15
+; GFX10_3-NEXT:    v_readlane_b32 s53, v22, 14
+; GFX10_3-NEXT:    v_readlane_b32 s52, v22, 13
+; GFX10_3-NEXT:    v_readlane_b32 s51, v22, 12
+; GFX10_3-NEXT:    v_readlane_b32 s50, v22, 11
+; GFX10_3-NEXT:    v_readlane_b32 s49, v22, 10
+; GFX10_3-NEXT:    v_readlane_b32 s48, v22, 9
 ; GFX10_3-NEXT:    v_readlane_b32 s39, v22, 8
 ; GFX10_3-NEXT:    v_readlane_b32 s38, v22, 7
 ; GFX10_3-NEXT:    v_readlane_b32 s37, v22, 6
@@ -2043,12 +1503,13 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX11-NEXT:    v_writelane_b32 v22, s30, 0
 ; GFX11-NEXT:    s_add_i32 s0, s32, 64
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_i32 s59, s32, 0x4240
 ; GFX11-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-NEXT:    s_and_b32 s0, 0, exec_lo
+; GFX11-NEXT:    v_writelane_b32 v22, s31, 1
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use alloca0 v0
 ; GFX11-NEXT:    ;;#ASMEND
-; GFX11-NEXT:    v_writelane_b32 v22, s31, 1
 ; GFX11-NEXT:    v_writelane_b32 v22, s33, 2
 ; GFX11-NEXT:    v_writelane_b32 v22, s34, 3
 ; GFX11-NEXT:    v_writelane_b32 v22, s35, 4
@@ -2056,52 +1517,29 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX11-NEXT:    v_writelane_b32 v22, s37, 6
 ; GFX11-NEXT:    v_writelane_b32 v22, s38, 7
 ; GFX11-NEXT:    v_writelane_b32 v22, s39, 8
-; GFX11-NEXT:    v_writelane_b32 v22, s40, 9
-; GFX11-NEXT:    v_writelane_b32 v22, s41, 10
-; GFX11-NEXT:    v_writelane_b32 v22, s42, 11
-; GFX11-NEXT:    v_writelane_b32 v22, s43, 12
-; GFX11-NEXT:    v_writelane_b32 v22, s44, 13
-; GFX11-NEXT:    v_writelane_b32 v22, s45, 14
-; GFX11-NEXT:    v_writelane_b32 v22, s46, 15
-; GFX11-NEXT:    v_writelane_b32 v22, s47, 16
-; GFX11-NEXT:    v_writelane_b32 v22, s48, 17
-; GFX11-NEXT:    v_writelane_b32 v22, s49, 18
-; GFX11-NEXT:    v_writelane_b32 v22, s50, 19
-; GFX11-NEXT:    v_writelane_b32 v22, s51, 20
-; GFX11-NEXT:    v_writelane_b32 v22, s52, 21
-; GFX11-NEXT:    v_writelane_b32 v22, s53, 22
-; GFX11-NEXT:    v_writelane_b32 v22, s54, 23
-; GFX11-NEXT:    v_writelane_b32 v22, s55, 24
-; GFX11-NEXT:    v_writelane_b32 v22, s56, 25
-; GFX11-NEXT:    v_writelane_b32 v22, s57, 26
-; GFX11-NEXT:    v_writelane_b32 v22, s59, 27
-; GFX11-NEXT:    s_add_i32 s59, s32, 0x4240
-; GFX11-NEXT:    s_and_b32 s0, 0, exec_lo
+; GFX11-NEXT:    v_writelane_b32 v22, s48, 9
+; GFX11-NEXT:    v_writelane_b32 v22, s49, 10
+; GFX11-NEXT:    v_writelane_b32 v22, s50, 11
+; GFX11-NEXT:    v_writelane_b32 v22, s51, 12
+; GFX11-NEXT:    v_writelane_b32 v22, s52, 13
+; GFX11-NEXT:    v_writelane_b32 v22, s53, 14
+; GFX11-NEXT:    v_writelane_b32 v22, s54, 15
+; GFX11-NEXT:    v_writelane_b32 v22, s55, 16
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
 ; GFX11-NEXT:    ;;#ASMEND
 ; GFX11-NEXT:    ;;#ASMSTART
 ; GFX11-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s59, scc
 ; GFX11-NEXT:    ;;#ASMEND
-; GFX11-NEXT:    v_readlane_b32 s59, v22, 27
-; GFX11-NEXT:    v_readlane_b32 s57, v22, 26
-; GFX11-NEXT:    v_readlane_b32 s56, v22, 25
-; GFX11-NEXT:    v_readlane_b32 s55, v22, 24
-; GFX11-NEXT:    v_readlane_b32 s54, v22, 23
-; GFX11-NEXT:    v_readlane_b32 s53, v22, 22
-; GFX11-NEXT:    v_readlane_b32 s52, v22, 21
-; GFX11-NEXT:    v_readlane_b32 s51, v22, 20
-; GFX11-NEXT:    v_readlane_b32 s50, v22, 19
-; GFX11-NEXT:    v_readlane_b32 s49, v22, 18
-; GFX11-NEXT:    v_readlane_b32 s48, v22, 17
-; GFX11-NEXT:    v_readlane_b32 s47, v22, 16
-; GFX11-NEXT:    v_readlane_b32 s46, v22, 15
-; GFX11-NEXT:    v_readlane_b32 s45, v22, 14
-; GFX11-NEXT:    v_readlane_b32 s44, v22, 13
-; GFX11-NEXT:    v_readlane_b32 s43, v22, 12
-; GFX11-NEXT:    v_readlane_b32 s42, v22, 11
-; GFX11-NEXT:    v_readlane_b32 s41, v22, 10
-; GFX11-NEXT:    v_readlane_b32 s40, v22, 9
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_readlane_b32 s55, v22, 16
+; GFX11-NEXT:    v_readlane_b32 s54, v22, 15
+; GFX11-NEXT:    v_readlane_b32 s53, v22, 14
+; GFX11-NEXT:    v_readlane_b32 s52, v22, 13
+; GFX11-NEXT:    v_readlane_b32 s51, v22, 12
+; GFX11-NEXT:    v_readlane_b32 s50, v22, 11
+; GFX11-NEXT:    v_readlane_b32 s49, v22, 10
+; GFX11-NEXT:    v_readlane_b32 s48, v22, 9
 ; GFX11-NEXT:    v_readlane_b32 s39, v22, 8
 ; GFX11-NEXT:    v_readlane_b32 s38, v22, 7
 ; GFX11-NEXT:    v_readlane_b32 s37, v22, 6
@@ -2130,7 +1568,9 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX12-NEXT:    s_wait_alu 0xfffe
 ; GFX12-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX12-NEXT:    v_writelane_b32 v22, s30, 0
+; GFX12-NEXT:    s_add_co_i32 s59, s32, 0x4200
 ; GFX12-NEXT:    v_mov_b32_e32 v0, s32
+; GFX12-NEXT:    s_and_b32 s0, 0, exec_lo
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; use alloca0 v0
 ; GFX12-NEXT:    ;;#ASMEND
@@ -2142,52 +1582,29 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs_gep_i
 ; GFX12-NEXT:    v_writelane_b32 v22, s37, 6
 ; GFX12-NEXT:    v_writelane_b32 v22, s38, 7
 ; GFX12-NEXT:    v_writelane_b32 v22, s39, 8
-; GFX12-NEXT:    v_writelane_b32 v22, s40, 9
-; GFX12-NEXT:    v_writelane_b32 v22, s41, 10
-; GFX12-NEXT:    v_writelane_b32 v22, s42, 11
-; GFX12-NEXT:    v_writelane_b32 v22, s43, 12
-; GFX12-NEXT:    v_writelane_b32 v22, s44, 13
-; GFX12-NEXT:    v_writelane_b32 v22, s45, 14
-; GFX12-NEXT:    v_writelane_b32 v22, s46, 15
-; GFX12-NEXT:    v_writelane_b32 v22, s47, 16
-; GFX12-NEXT:    v_writelane_b32 v22, s48, 17
-; GFX12-NEXT:    v_writelane_b32 v22, s49, 18
-; GFX12-NEXT:    v_writelane_b32 v22, s50, 19
-; GFX12-NEXT:    v_writelane_b32 v22, s51, 20
-; GFX12-NEXT:    v_writelane_b32 v22, s52, 21
-; GFX12-NEXT:    v_writelane_b32 v22, s53, 22
-; GFX12-NEXT:    v_writelane_b32 v22, s54, 23
-; GFX12-NEXT:    v_writelane_b32 v22, s55, 24
-; GFX12-NEXT:    v_writelane_b32 v22, s56, 25
-; GFX12-NEXT:    v_writelane_b32 v22, s57, 26
-; GFX12-NEXT:    v_writelane_b32 v22, s59, 27
-; GFX12-NEXT:    s_add_co_i32 s59, s32, 0x4200
-; GFX12-NEXT:    s_and_b32 s0, 0, exec_lo
+; GFX12-NEXT:    v_writelane_b32 v22, s48, 9
+; GFX12-NEXT:    v_writelane_b32 v22, s49, 10
+; GFX12-NEXT:    v_writelane_b32 v22, s50, 11
+; GFX12-NEXT:    v_writelane_b32 v22, s51, 12
+; GFX12-NEXT:    v_writelane_b32 v22, s52, 13
+; GFX12-NEXT:    v_writelane_b32 v22, s53, 14
+; GFX12-NEXT:    v_writelane_b32 v22, s54, 15
+; GFX12-NEXT:    v_writelane_b32 v22, s55, 16
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; def s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc
 ; GFX12-NEXT:    ;;#ASMEND
 ; GFX12-NEXT:    ;;#ASMSTART
 ; GFX12-NEXT:    ; use s[0:15], s[16:31], s[32:47], s[48:55], s[56:57], v[0:15], v[16:21], vcc, s59, scc
 ; GFX12-NEXT:    ;;#ASMEND
-; GFX12-NEXT:    v_readlane_b32 s59, v22, 27
-; GFX12-NEXT:    v_readlane_b32 s57, v22, 26
-; GFX12-NEXT:    v_readlane_b32 s56, v22, 25
-; GFX12-NEXT:    v_readlane_b32 s55, v22, 24
-; GFX12-NEXT:    v_readlane_b32 s54, v22, 23
-; GFX12-NEXT:    v_readlane_b32 s53, v22, 22
-; GFX12-NEXT:    v_readlane_b32 s52, v22, 21
-; GFX12-NEXT:    v_readlane_b32 s51, v22, 20
-; GFX12-NEXT:    v_readlane_b32 s50, v22, 19
-; GFX12-NEXT:    v_readlane_b32 s49, v22, 18
-; GFX12-NEXT:    v_readlane_b32 s48, v22, 17
-; GFX12-NEXT:    v_readlane_b32 s47, v22, 16
-; GFX12-NEXT:    v_readlane_b32 s46, v22, 15
-; GFX12-NEXT:    v_readlane_b32 s45, v22, 14
-; GFX12-NEXT:    v_readlane_b32 s44, v22, 13
-; GFX12-NEXT:    v_readlane_b32 s43, v22, 12
-; GFX12-NEXT:    v_readlane_b32 s42, v22, 11
-; GFX12-NEXT:    v_readlane_b32 s41, v22, 10
-; GFX12-NEXT:    v_readlane_b32 s40, v22, 9
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_readlane_b32 s55, v22, 16
+; GFX12-NEXT:    v_readlane_b32 s54, v22, 15
+; GFX12-NEXT:    v_readlane_b32 s53, v22, 14
+; GFX12-NEXT:    v_readlane_b32 s52, v22, 13
+; GFX12-NEXT:    v_readlane_b32 s51, v22, 12
+; GFX12-NEXT:    v_readlane_b32 s50, v22, 11
+; GFX12-NEXT:    v_readlane_b32 s49, v22, 10
+; GFX12-NEXT:    v_readlane_b32 s48, v22, 9
 ; GFX12-NEXT:    v_readlane_b32 s39, v22, 8
 ; GFX12-NEXT:    v_readlane_b32 s38, v22, 7
 ; GFX12-NEXT:    v_readlane_b32 s37, v22, 6
diff --git a/llvm/test/CodeGen/AMDGPU/max-literals.ll b/llvm/test/CodeGen/AMDGPU/max-literals.ll
index f1cbf43aa089a..2cfe9d528b66f 100644
--- a/llvm/test/CodeGen/AMDGPU/max-literals.ll
+++ b/llvm/test/CodeGen/AMDGPU/max-literals.ll
@@ -17,14 +17,14 @@ main_body:
   %9 = bitcast float %4 to i32
   %10 = mul i32 %9, 6
   %11 = bitcast i32 %10 to float
-  %12 = insertelement <4 x float> undef, float %5, i32 0
+  %12 = insertelement <4 x float> poison, float %5, i32 0
   %13 = insertelement <4 x float> %12, float %6, i32 1
   %14 = insertelement <4 x float> %13, float %7, i32 2
   %15 = insertelement <4 x float> %14, float %8, i32 3
   %16 = insertelement <4 x float> %15, float %11, i32 3
 
   %17 = call float @llvm.r600.dot4(<4 x float> %15,<4 x float> %16)
-  %18 = insertelement <4 x float> undef, float %17, i32 0
+  %18 = insertelement <4 x float> poison, float %17, i32 0
   call void @llvm.r600.store.swizzle(<4 x float> %18, i32 0, i32 2)
   ret void
 }
@@ -46,14 +46,14 @@ main_body:
   %9 = bitcast float %4 to i32
   %10 = mul i32 %9, 6
   %11 = bitcast i32 %10 to float
-  %12 = insertelement <4 x float> undef, float %5, i32 0
+  %12 = insertelement <4 x float> poison, float %5, i32 0
   %13 = insertelement <4 x float> %12, float %6, i32 1
   %14 = insertelement <4 x float> %13, float %7, i32 2
   %15 = insertelement <4 x float> %14, float %8, i32 3
   %16 = insertelement <4 x float> %15, float %11, i32 3
 
   %17 = call float @llvm.r600.dot4(<4 x float> %15,<4 x float> %16)
-  %18 = insertelement <4 x float> undef, float %17, i32 0
+  %18 = insertelement <4 x float> poison, float %17, i32 0
   call void @llvm.r600.store.swizzle(<4 x float> %18, i32 0, i32 2)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/mcexpr-knownbits-assign-crash-gh-issue-110930.ll b/llvm/test/CodeGen/AMDGPU/mcexpr-knownbits-assign-crash-gh-issue-110930.ll
index 0112453e32bfc..52f380b7f80a3 100644
--- a/llvm/test/CodeGen/AMDGPU/mcexpr-knownbits-assign-crash-gh-issue-110930.ll
+++ b/llvm/test/CodeGen/AMDGPU/mcexpr-knownbits-assign-crash-gh-issue-110930.ll
@@ -8,7 +8,7 @@
 ; CHECK-LABEL: I_Quit:
 ; CHECK: .set I_Quit.num_vgpr, max(41, amdgpu.max_num_vgpr)
 ; CHECK: .set I_Quit.num_agpr, max(0, amdgpu.max_num_agpr)
-; CHECK: .set I_Quit.numbered_sgpr, max(48, amdgpu.max_num_sgpr)
+; CHECK: .set I_Quit.numbered_sgpr, max(56, amdgpu.max_num_sgpr)
 ; CHECK: .set I_Quit.private_seg_size, 16
 ; CHECK: .set I_Quit.uses_vcc, 1
 ; CHECK: .set I_Quit.uses_flat_scratch, 1
@@ -80,7 +80,7 @@ define void @P_SetThingPosition() {
 ; CHECK-LABEL: P_SetupPsprites:
 ; CHECK: .set P_SetupPsprites.num_vgpr, max(41, amdgpu.max_num_vgpr)
 ; CHECK: .set P_SetupPsprites.num_agpr, max(0, amdgpu.max_num_agpr)
-; CHECK: .set P_SetupPsprites.numbered_sgpr, max(48, amdgpu.max_num_sgpr)
+; CHECK: .set P_SetupPsprites.numbered_sgpr, max(56, amdgpu.max_num_sgpr)
 ; CHECK: .set P_SetupPsprites.private_seg_size, 16
 ; CHECK: .set P_SetupPsprites.uses_vcc, 1
 ; CHECK: .set P_SetupPsprites.uses_flat_scratch, 1
@@ -110,7 +110,7 @@ define void @HU_Start() {
 ; CHECK-LABEL: P_SpawnPlayer:
 ; CHECK: .set P_SpawnPlayer.num_vgpr, max(43, G_PlayerReborn.num_vgpr, P_SetThingPosition.num_vgpr, P_SetupPsprites.num_vgpr, HU_Start.num_vgpr)
 ; CHECK: .set P_SpawnPlayer.num_agpr, max(0, G_PlayerReborn.num_agpr, P_SetThingPosition.num_agpr, P_SetupPsprites.num_agpr, HU_Start.num_agpr)
-; CHECK: .set P_SpawnPlayer.numbered_sgpr, max(60, G_PlayerReborn.numbered_sgpr, P_SetThingPosition.numbered_sgpr, P_SetupPsprites.numbered_sgpr, HU_Start.numbered_sgpr)
+; CHECK: .set P_SpawnPlayer.numbered_sgpr, max(84, G_PlayerReborn.numbered_sgpr, P_SetThingPosition.numbered_sgpr, P_SetupPsprites.numbered_sgpr, HU_Start.numbered_sgpr)
 ; CHECK: .set P_SpawnPlayer.private_seg_size, 16+(max(G_PlayerReborn.private_seg_size, P_SetThingPosition.private_seg_size, P_SetupPsprites.private_seg_size, HU_Start.private_seg_size))
 ; CHECK: .set P_SpawnPlayer.uses_vcc, or(1, G_PlayerReborn.uses_vcc, P_SetThingPosition.uses_vcc, P_SetupPsprites.uses_vcc, HU_Start.uses_vcc)
 ; CHECK: .set P_SpawnPlayer.uses_flat_scratch, or(0, G_PlayerReborn.uses_flat_scratch, P_SetThingPosition.uses_flat_scratch, P_SetupPsprites.uses_flat_scratch, HU_Start.uses_flat_scratch)
@@ -128,7 +128,7 @@ define void @P_SpawnPlayer() {
 ; CHECK-LABEL: I_Error:
 ; CHECK: .set I_Error.num_vgpr, max(41, amdgpu.max_num_vgpr)
 ; CHECK: .set I_Error.num_agpr, max(0, amdgpu.max_num_agpr)
-; CHECK: .set I_Error.numbered_sgpr, max(48, amdgpu.max_num_sgpr)
+; CHECK: .set I_Error.numbered_sgpr, max(56, amdgpu.max_num_sgpr)
 ; CHECK: .set I_Error.private_seg_size, 16
 ; CHECK: .set I_Error.uses_vcc, 1
 ; CHECK: .set I_Error.uses_flat_scratch, 1
@@ -144,7 +144,7 @@ define void @I_Error(...) {
 ; CHECK-LABEL: G_DoReborn:
 ; CHECK: .set G_DoReborn.num_vgpr, max(44, P_RemoveMobj.num_vgpr, P_SpawnMobj.num_vgpr, P_SpawnPlayer.num_vgpr, I_Error.num_vgpr)
 ; CHECK: .set G_DoReborn.num_agpr, max(0, P_RemoveMobj.num_agpr, P_SpawnMobj.num_agpr, P_SpawnPlayer.num_agpr, I_Error.num_agpr)
-; CHECK: .set G_DoReborn.numbered_sgpr, max(72, P_RemoveMobj.numbered_sgpr, P_SpawnMobj.numbered_sgpr, P_SpawnPlayer.numbered_sgpr, I_Error.numbered_sgpr)
+; CHECK: .set G_DoReborn.numbered_sgpr, max(104, P_RemoveMobj.numbered_sgpr, P_SpawnMobj.numbered_sgpr, P_SpawnPlayer.numbered_sgpr, I_Error.numbered_sgpr)
 ; CHECK: .set G_DoReborn.private_seg_size, 32+(max(P_RemoveMobj.private_seg_size, P_SpawnMobj.private_seg_size, P_SpawnPlayer.private_seg_size, I_Error.private_seg_size))
 ; CHECK: .set G_DoReborn.uses_vcc, or(1, P_RemoveMobj.uses_vcc, P_SpawnMobj.uses_vcc, P_SpawnPlayer.uses_vcc, I_Error.uses_vcc)
 ; CHECK: .set G_DoReborn.uses_flat_scratch, or(0, P_RemoveMobj.uses_flat_scratch, P_SpawnMobj.uses_flat_scratch, P_SpawnPlayer.uses_flat_scratch, I_Error.uses_flat_scratch)
@@ -218,7 +218,7 @@ define void @F_Ticker() {
 ; CHECK-LABEL: G_CheckDemoStatus:
 ; CHECK: .set G_CheckDemoStatus.num_vgpr, max(43, I_Quit.num_vgpr, D_AdvanceDemo.num_vgpr, I_Error.num_vgpr)
 ; CHECK: .set G_CheckDemoStatus.num_agpr, max(0, I_Quit.num_agpr, D_AdvanceDemo.num_agpr, I_Error.num_agpr)
-; CHECK: .set G_CheckDemoStatus.numbered_sgpr, max(60, I_Quit.numbered_sgpr, D_AdvanceDemo.numbered_sgpr, I_Error.numbered_sgpr)
+; CHECK: .set G_CheckDemoStatus.numbered_sgpr, max(84, I_Quit.numbered_sgpr, D_AdvanceDemo.numbered_sgpr, I_Error.numbered_sgpr)
 ; CHECK: .set G_CheckDemoStatus.private_seg_size, 32+(max(I_Quit.private_seg_size, D_AdvanceDemo.private_seg_size, I_Error.private_seg_size))
 ; CHECK: .set G_CheckDemoStatus.uses_vcc, or(1, I_Quit.uses_vcc, D_AdvanceDemo.uses_vcc, I_Error.uses_vcc)
 ; CHECK: .set G_CheckDemoStatus.uses_flat_scratch, or(0, I_Quit.uses_flat_scratch, D_AdvanceDemo.uses_flat_scratch, I_Error.uses_flat_scratch)
@@ -264,7 +264,7 @@ define ptr @P_SaveGameFile() {
 ; CHECK-LABEL: R_FlatNumForName:
 ; CHECK: .set R_FlatNumForName.num_vgpr, max(42, I_Error.num_vgpr)
 ; CHECK: .set R_FlatNumForName.num_agpr, max(0, I_Error.num_agpr)
-; CHECK: .set R_FlatNumForName.numbered_sgpr, max(48, I_Error.numbered_sgpr)
+; CHECK: .set R_FlatNumForName.numbered_sgpr, max(56, I_Error.numbered_sgpr)
 ; CHECK: .set R_FlatNumForName.private_seg_size, 16+(max(I_Error.private_seg_size))
 ; CHECK: .set R_FlatNumForName.uses_vcc, or(1, I_Error.uses_vcc)
 ; CHECK: .set R_FlatNumForName.uses_flat_scratch, or(0, I_Error.uses_flat_scratch)
@@ -279,7 +279,7 @@ define i32 @R_FlatNumForName() {
 ; CHECK-LABEL: R_TextureNumForName:
 ; CHECK: .set R_TextureNumForName.num_vgpr, max(42, R_FlatNumForName.num_vgpr)
 ; CHECK: .set R_TextureNumForName.num_agpr, max(0, R_FlatNumForName.num_agpr)
-; CHECK: .set R_TextureNumForName.numbered_sgpr, max(48, R_FlatNumForName.numbered_sgpr)
+; CHECK: .set R_TextureNumForName.numbered_sgpr, max(56, R_FlatNumForName.numbered_sgpr)
 ; CHECK: .set R_TextureNumForName.private_seg_size, 16+(max(R_FlatNumForName.private_seg_size))
 ; CHECK: .set R_TextureNumForName.uses_vcc, or(1, R_FlatNumForName.uses_vcc)
 ; CHECK: .set R_TextureNumForName.uses_flat_scratch, or(0, R_FlatNumForName.uses_flat_scratch)
@@ -292,10 +292,10 @@ define i32 @R_TextureNumForName() {
 }
 
 ; CHECK-LABEL: G_Ticker:
-; CHECK: .set G_Ticker.num_vgpr, max(46, G_DoReborn.num_vgpr, F_Ticker.num_vgpr, AM_Stop.num_vgpr, F_StartFinale.num_vgpr, D_AdvanceDemo.num_vgpr, R_FlatNumForName.num_vgpr, R_TextureNumForName.num_vgpr, P_TempSaveGameFile.num_vgpr, P_SaveGameFile.num_vgpr, I_Error.num_vgpr)
+; CHECK: .set G_Ticker.num_vgpr, max(47, G_DoReborn.num_vgpr, F_Ticker.num_vgpr, AM_Stop.num_vgpr, F_StartFinale.num_vgpr, D_AdvanceDemo.num_vgpr, R_FlatNumForName.num_vgpr, R_TextureNumForName.num_vgpr, P_TempSaveGameFile.num_vgpr, P_SaveGameFile.num_vgpr, I_Error.num_vgpr)
 ; CHECK: .set G_Ticker.num_agpr, max(0, G_DoReborn.num_agpr, F_Ticker.num_agpr, AM_Stop.num_agpr, F_StartFinale.num_agpr, D_AdvanceDemo.num_agpr, R_FlatNumForName.num_agpr, R_TextureNumForName.num_agpr, P_TempSaveGameFile.num_agpr, P_SaveGameFile.num_agpr, I_Error.num_agpr)
-; CHECK: .set G_Ticker.numbered_sgpr, max(84, G_DoReborn.numbered_sgpr, F_Ticker.numbered_sgpr, AM_Stop.numbered_sgpr, F_StartFinale.numbered_sgpr, D_AdvanceDemo.numbered_sgpr, R_FlatNumForName.numbered_sgpr, R_TextureNumForName.numbered_sgpr, P_TempSaveGameFile.numbered_sgpr, P_SaveGameFile.numbered_sgpr, I_Error.numbered_sgpr)
-; CHECK: .set G_Ticker.private_seg_size, 32+(max(G_DoReborn.private_seg_size, F_Ticker.private_seg_size, AM_Stop.private_seg_size, F_StartFinale.private_seg_size, D_AdvanceDemo.private_seg_size, R_FlatNumForName.private_seg_size, R_TextureNumForName.private_seg_size, P_TempSaveGameFile.private_seg_size, P_SaveGameFile.private_seg_size, I_Error.private_seg_size))
+; CHECK: .set G_Ticker.numbered_sgpr, max(105, G_DoReborn.numbered_sgpr, F_Ticker.numbered_sgpr, AM_Stop.numbered_sgpr, F_StartFinale.numbered_sgpr, D_AdvanceDemo.numbered_sgpr, R_FlatNumForName.numbered_sgpr, R_TextureNumForName.numbered_sgpr, P_TempSaveGameFile.numbered_sgpr, P_SaveGameFile.numbered_sgpr, I_Error.numbered_sgpr)
+; CHECK: .set G_Ticker.private_seg_size, 48+(max(G_DoReborn.private_seg_size, F_Ticker.private_seg_size, AM_Stop.private_seg_size, F_StartFinale.private_seg_size, D_AdvanceDemo.private_seg_size, R_FlatNumForName.private_seg_size, R_TextureNumForName.private_seg_size, P_TempSaveGameFile.private_seg_size, P_SaveGameFile.private_seg_size, I_Error.private_seg_size))
 ; CHECK: .set G_Ticker.uses_vcc, or(1, G_DoReborn.uses_vcc, F_Ticker.uses_vcc, AM_Stop.uses_vcc, F_StartFinale.uses_vcc, D_AdvanceDemo.uses_vcc, R_FlatNumForName.uses_vcc, R_TextureNumForName.uses_vcc, P_TempSaveGameFile.uses_vcc, P_SaveGameFile.uses_vcc, I_Error.uses_vcc)
 ; CHECK: .set G_Ticker.uses_flat_scratch, or(0, G_DoReborn.uses_flat_scratch, F_Ticker.uses_flat_scratch, AM_Stop.uses_flat_scratch, F_StartFinale.uses_flat_scratch, D_AdvanceDemo.uses_flat_scratch, R_FlatNumForName.uses_flat_scratch, R_TextureNumForName.uses_flat_scratch, P_TempSaveGameFile.uses_flat_scratch, P_SaveGameFile.uses_flat_scratch, I_Error.uses_flat_scratch)
 ; CHECK: .set G_Ticker.has_dyn_sized_stack, or(0, G_DoReborn.has_dyn_sized_stack, F_Ticker.has_dyn_sized_stack, AM_Stop.has_dyn_sized_stack, F_StartFinale.has_dyn_sized_stack, D_AdvanceDemo.has_dyn_sized_stack, R_FlatNumForName.has_dyn_sized_stack, R_TextureNumForName.has_dyn_sized_stack, P_TempSaveGameFile.has_dyn_sized_stack, P_SaveGameFile.has_dyn_sized_stack, I_Error.has_dyn_sized_stack)
@@ -316,9 +316,9 @@ define void @G_Ticker() {
 }
 
 ; CHECK-LABEL: RunTic:
-; CHECK: .set RunTic.num_vgpr, max(46, G_CheckDemoStatus.num_vgpr, D_AdvanceDemo.num_vgpr, G_Ticker.num_vgpr)
+; CHECK: .set RunTic.num_vgpr, max(47, G_CheckDemoStatus.num_vgpr, D_AdvanceDemo.num_vgpr, G_Ticker.num_vgpr)
 ; CHECK: .set RunTic.num_agpr, max(0, G_CheckDemoStatus.num_agpr, D_AdvanceDemo.num_agpr, G_Ticker.num_agpr)
-; CHECK: .set RunTic.numbered_sgpr, max(84, G_CheckDemoStatus.numbered_sgpr, D_AdvanceDemo.numbered_sgpr, G_Ticker.numbered_sgpr)
+; CHECK: .set RunTic.numbered_sgpr, max(105, G_CheckDemoStatus.numbered_sgpr, D_AdvanceDemo.numbered_sgpr, G_Ticker.numbered_sgpr)
 ; CHECK: .set RunTic.private_seg_size, 32+(max(G_CheckDemoStatus.private_seg_size, D_AdvanceDemo.private_seg_size, G_Ticker.private_seg_size))
 ; CHECK: .set RunTic.uses_vcc, or(1, G_CheckDemoStatus.uses_vcc, D_AdvanceDemo.uses_vcc, G_Ticker.uses_vcc)
 ; CHECK: .set RunTic.uses_flat_scratch, or(0, G_CheckDemoStatus.uses_flat_scratch, D_AdvanceDemo.uses_flat_scratch, G_Ticker.uses_flat_scratch)
diff --git a/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll b/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll
index 1d0533ca07b3e..6d4467a541203 100644
--- a/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll
@@ -48,7 +48,7 @@ if.then3.i:                                       ; preds = %if.end60
   br label %if.end5.i
 
 if.end5.i:                                        ; preds = %if.then3.i, %if.end60
-  %pS.addr.0.i = phi ptr addrspace(5) [ undef, %if.then3.i ], [ %runtimeVersionCopy, %if.end60 ]
+  %pS.addr.0.i = phi ptr addrspace(5) [ poison, %if.then3.i ], [ %runtimeVersionCopy, %if.end60 ]
   %2 = load i8, ptr addrspace(5) %pS.addr.0.i, align 1
   %conv612.i = sext i8 %2 to i32
   %sub13.i = add nsw i32 %conv612.i, -48
@@ -65,7 +65,7 @@ if.then3.i308:                                    ; preds = %if.end5.i
   br label %if.end5.i314
 
 if.end5.i314:                                     ; preds = %if.then3.i308, %if.end5.i
-  %pS.addr.0.i309 = phi ptr addrspace(5) [ undef, %if.then3.i308 ], [ %licenseVersionCopy, %if.end5.i ]
+  %pS.addr.0.i309 = phi ptr addrspace(5) [ poison, %if.then3.i308 ], [ %licenseVersionCopy, %if.end5.i ]
   %3 = load i8, ptr addrspace(5) %pS.addr.0.i309, align 1
   %conv612.i311 = sext i8 %3 to i32
   %sub13.i312 = add nsw i32 %conv612.i311, -48
@@ -82,7 +82,7 @@ if.then3.i332:                                    ; preds = %if.end5.i314
   br label %if.end5.i338
 
 if.end5.i338:                                     ; preds = %if.then3.i332, %if.end5.i314
-  %pS.addr.0.i333 = phi ptr addrspace(5) [ undef, %if.then3.i332 ], [ %arrayidx144260.5, %if.end5.i314 ]
+  %pS.addr.0.i333 = phi ptr addrspace(5) [ poison, %if.then3.i332 ], [ %arrayidx144260.5, %if.end5.i314 ]
   %4 = load i8, ptr addrspace(5) %pS.addr.0.i333, align 1
   %conv612.i335 = sext i8 %4 to i32
   %sub13.i336 = add nsw i32 %conv612.i335, -48
@@ -99,7 +99,7 @@ if.then3.i356:                                    ; preds = %if.end5.i338
   br label %if.end5.i362
 
 if.end5.i362:                                     ; preds = %if.then3.i356, %if.end5.i338
-  %pS.addr.0.i357 = phi ptr addrspace(5) [ undef, %if.then3.i356 ], [ %arrayidx156258.5, %if.end5.i338 ]
+  %pS.addr.0.i357 = phi ptr addrspace(5) [ poison, %if.then3.i356 ], [ %arrayidx156258.5, %if.end5.i338 ]
   %5 = load i8, ptr addrspace(5) %pS.addr.0.i357, align 1
   %conv612.i359 = sext i8 %5 to i32
   %sub13.i360 = add nsw i32 %conv612.i359, -48
@@ -120,7 +120,7 @@ if.then3.i394:                                    ; preds = %if.end5.i362
   br label %if.end5.i400
 
 if.end5.i400:                                     ; preds = %if.then3.i394, %if.end5.i362
-  %pS.addr.0.i395 = phi ptr addrspace(5) [ %arrayidx232250.1, %if.then3.i394 ], [ undef, %if.end5.i362 ]
+  %pS.addr.0.i395 = phi ptr addrspace(5) [ %arrayidx232250.1, %if.then3.i394 ], [ poison, %if.end5.i362 ]
   %7 = load i8, ptr addrspace(5) %pS.addr.0.i395, align 1
   %conv612.i397 = sext i8 %7 to i32
   %sub13.i398 = add nsw i32 %conv612.i397, -48
diff --git a/llvm/test/CodeGen/AMDGPU/memory_clause.ll b/llvm/test/CodeGen/AMDGPU/memory_clause.ll
index 15f31b4e86dbe..1324951d06334 100644
--- a/llvm/test/CodeGen/AMDGPU/memory_clause.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory_clause.ll
@@ -328,10 +328,10 @@ entry:
   %gep = getelementptr inbounds i16, ptr addrspace(1) %in, i64 32
   %load1 = load i16, ptr addrspace(1) %in
   %load2 = load i16, ptr addrspace(1) %gep
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
   store <2 x i16> %build1, ptr addrspace(1) %out
-  %build2 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build2 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build3 = insertelement <2 x i16> %build2, i16 %load2, i32 1
   %gep2 = getelementptr inbounds <2 x i16>, ptr addrspace(1) %out, i64 32
   store <2 x i16> %build3, ptr addrspace(1) %gep2
diff --git a/llvm/test/CodeGen/AMDGPU/merge-consecutive-wait-alus.mir b/llvm/test/CodeGen/AMDGPU/merge-consecutive-wait-alus.mir
new file mode 100644
index 0000000000000..d8f4c9c8f14b5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/merge-consecutive-wait-alus.mir
@@ -0,0 +1,79 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass amdgpu-wait-sgpr-hazards -o -  %s | FileCheck %s
+
+
+---
+name: merge_consecutive_wait_alus
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: merge_consecutive_wait_alus
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $sgpr0 = V_CMP_NE_U32_e64 0, $vgpr0, implicit $exec, implicit-def $vcc_lo, implicit-def $vcc_lo
+    ; CHECK-NEXT: S_WAITCNT_DEPCTR 61946
+    ; CHECK-NEXT: renamable $vgpr0 = V_CNDMASK_B32_e64 0, -1, 0, killed $vgpr0, killed $sgpr0, implicit $exec, implicit-def $vcc_lo
+    renamable $sgpr0 = V_CMP_NE_U32_e64 0, $vgpr0, implicit $exec, implicit-def $vcc_lo, implicit-def $vcc
+    S_WAITCNT_DEPCTR 65530
+    renamable $vgpr0 = V_CNDMASK_B32_e64 0, -1, 0, killed $vgpr0, killed $sgpr0, implicit $exec, implicit-def $vcc
+...
+---
+name: merge_consecutive_wait_alus_two_bb
+body:             |
+  ; CHECK-LABEL: name: merge_consecutive_wait_alus_two_bb
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $vgpr0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $sgpr0 = V_CMP_NE_U32_e64 0, $vgpr0, implicit $exec, implicit-def $vcc_lo, implicit-def $vcc_lo
+  ; CHECK-NEXT:   S_WAITCNT_DEPCTR 65530
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   liveins: $sgpr0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_WAITCNT_DEPCTR 61951
+  ; CHECK-NEXT:   renamable $vgpr0 = V_CNDMASK_B32_e64 0, -1, 0, killed $vgpr0, killed $sgpr0, implicit $exec, implicit-def $vcc_lo
+  bb.0:
+    liveins: $vgpr0
+
+    renamable $sgpr0 = V_CMP_NE_U32_e64 0, $vgpr0, implicit $exec, implicit-def $vcc_lo, implicit-def $vcc
+    S_WAITCNT_DEPCTR 65530
+
+  bb.1:
+    liveins: $sgpr0
+
+    renamable $vgpr0 = V_CNDMASK_B32_e64 0, -1, 0, killed $vgpr0, killed $sgpr0, implicit $exec, implicit-def $vcc
+...
+---
+name: meta_instructions
+machineFunctionInfo:
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: meta_instructions
+    ; CHECK: renamable $sgpr0 = V_CMP_NE_U32_e64 0, $vgpr0, implicit $exec, implicit-def $vcc_lo, implicit-def $vcc_lo
+    ; CHECK-NEXT: S_WAITCNT_DEPCTR 65530
+    ; CHECK-NEXT: SCHED_BARRIER 0
+    ; CHECK-NEXT: S_WAITCNT_DEPCTR 61951
+    ; CHECK-NEXT: renamable $vgpr0 = V_CNDMASK_B32_e64 0, -1, 0, killed $vgpr0, killed $sgpr0, implicit $exec, implicit-def $vcc_lo
+    renamable $sgpr0 = V_CMP_NE_U32_e64 0, $vgpr0, implicit $exec, implicit-def $vcc_lo, implicit-def $vcc
+    S_WAITCNT_DEPCTR 65530
+    SCHED_BARRIER 0
+    renamable $vgpr0 = V_CNDMASK_B32_e64 0, -1, 0, killed $vgpr0, killed $sgpr0, implicit $exec, implicit-def $vcc
+...
+---
+name: debug_instruction
+machineFunctionInfo:
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: debug_instruction
+    ; CHECK: renamable $sgpr0 = V_CMP_NE_U32_e64 0, $vgpr0, implicit $exec, implicit-def $vcc_lo, implicit-def $vcc_lo
+    ; CHECK-NEXT: S_WAITCNT_DEPCTR 61946
+    ; CHECK-NEXT: DBG_VALUE $sgpr0
+    ; CHECK-NEXT: renamable $vgpr0 = V_CNDMASK_B32_e64 0, -1, 0, killed $vgpr0, killed $sgpr0, implicit $exec, implicit-def $vcc_lo
+    renamable $sgpr0 = V_CMP_NE_U32_e64 0, $vgpr0, implicit $exec, implicit-def $vcc_lo, implicit-def $vcc
+    S_WAITCNT_DEPCTR 65530
+    DBG_VALUE $sgpr0
+    renamable $vgpr0 = V_CNDMASK_B32_e64 0, -1, 0, killed $vgpr0, killed $sgpr0, implicit $exec, implicit-def $vcc
+...
+
diff --git a/llvm/test/CodeGen/AMDGPU/merge-store-crash.ll b/llvm/test/CodeGen/AMDGPU/merge-store-crash.ll
index bcd80a2a03b05..0809891aec72b 100644
--- a/llvm/test/CodeGen/AMDGPU/merge-store-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/merge-store-crash.ll
@@ -22,10 +22,10 @@ main_body:
   %tmp5 = getelementptr [8192 x i32], ptr addrspace(3) @tess_lds, i64 0, i64 %tmp4
   store float %tmp1, ptr addrspace(3) %tmp5, align 4
   %tmp7 = bitcast float %tmp1 to i32
-  %tmp8 = insertelement <4 x i32> undef, i32 %tmp2, i32 0
+  %tmp8 = insertelement <4 x i32> poison, i32 %tmp2, i32 0
   %tmp9 = insertelement <4 x i32> %tmp8, i32 %tmp7, i32 1
-  %tmp10 = insertelement <4 x i32> %tmp9, i32 undef, i32 2
-  %tmp11 = insertelement <4 x i32> %tmp10, i32 undef, i32 3
+  %tmp10 = insertelement <4 x i32> %tmp9, i32 poison, i32 2
+  %tmp11 = insertelement <4 x i32> %tmp10, i32 poison, i32 3
   call void @llvm.amdgcn.struct.ptr.tbuffer.store.v4i32(<4 x i32> %tmp11, ptr addrspace(8) undef, i32 0, i32 0, i32 %arg, i32 78, i32 3) #2
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-loop.ll b/llvm/test/CodeGen/AMDGPU/mfma-loop.ll
index 4a635d6e7f59f..d2835f06d8e0e 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-loop.ll
@@ -264,7 +264,7 @@ define amdgpu_kernel void @test_mfma_loop_vgpr_init(ptr addrspace(1) %arg) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %init = bitcast i32 %tid to float
-  %tmp0 = insertelement <32 x float> undef, float %init, i32 0
+  %tmp0 = insertelement <32 x float> poison, float %init, i32 0
   %tmp1 = insertelement <32 x float> %tmp0, float %init, i32 1
   %tmp2 = insertelement <32 x float> %tmp1, float %init, i32 2
   %tmp3 = insertelement <32 x float> %tmp2, float %init, i32 3
@@ -332,7 +332,7 @@ exit:
 
 define amdgpu_kernel void @test_mfma_loop_sgpr_init(ptr addrspace(1) %arg, float %init) #0 {
 entry:
-  %tmp0 = insertelement <32 x float> undef, float %init, i32 0
+  %tmp0 = insertelement <32 x float> poison, float %init, i32 0
   %tmp1 = insertelement <32 x float> %tmp0, float %init, i32 1
   %tmp2 = insertelement <32 x float> %tmp1, float %init, i32 2
   %tmp3 = insertelement <32 x float> %tmp2, float %init, i32 3
@@ -527,7 +527,7 @@ define amdgpu_kernel void @test_mfma_loop_agpr_init(ptr addrspace(1) %arg) #0 {
 entry:
   %mai.0 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> zeroinitializer, i32 0, i32 0, i32 0)
   %init = extractelement <32 x float> %mai.0, i32 0
-  %tmp0 = insertelement <32 x float> undef, float %init, i32 0
+  %tmp0 = insertelement <32 x float> poison, float %init, i32 0
   %tmp1 = insertelement <32 x float> %tmp0, float %init, i32 1
   %tmp2 = insertelement <32 x float> %tmp1, float %init, i32 2
   %tmp3 = insertelement <32 x float> %tmp2, float %init, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-worklist.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-worklist.ll
index 9b7d851a0c74e..a4cb43e14c2e2 100644
--- a/llvm/test/CodeGen/AMDGPU/move-to-valu-worklist.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-worklist.ll
@@ -17,8 +17,8 @@ bb:
 br label %bb1
 
 bb1:                                              ; preds = %bb1, %bb
-	%tmp2 = phi i64 [ undef, %bb ], [ %tmp16, %bb1 ]
-	%tmp3 = phi i64 [ %tmp, %bb ], [ undef, %bb1 ]
+	%tmp2 = phi i64 [ poison, %bb ], [ %tmp16, %bb1 ]
+	%tmp3 = phi i64 [ %tmp, %bb ], [ poison, %bb1 ]
 	%tmp11 = shl i64 %tmp2, 14
 	%tmp13 = xor i64 %tmp11, %tmp2
 	%tmp15 = and i64 %tmp3, %tmp13
diff --git a/llvm/test/CodeGen/AMDGPU/mul_int24.ll b/llvm/test/CodeGen/AMDGPU/mul_int24.ll
index ad3f36dc3f216..11cf129b1e479 100644
--- a/llvm/test/CodeGen/AMDGPU/mul_int24.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul_int24.ll
@@ -795,8 +795,8 @@ bb:
   br i1 %cmp, label %bb11, label %bb7
 
 bb11:
-  %tmp14 = shufflevector <2 x i32> %arg1, <2 x i32> undef, <2 x i32> zeroinitializer
-  %tmp16 = shufflevector <2 x i32> %arg2, <2 x i32> undef, <2 x i32> zeroinitializer
+  %tmp14 = shufflevector <2 x i32> %arg1, <2 x i32> poison, <2 x i32> zeroinitializer
+  %tmp16 = shufflevector <2 x i32> %arg2, <2 x i32> poison, <2 x i32> zeroinitializer
   %tmp17 = shl <2 x i32> %tmp14, <i32 8, i32 8>
   %tmp18 = ashr <2 x i32> %tmp17, <i32 8, i32 8>
   %tmp19 = shl <2 x i32> %tmp16, <i32 8, i32 8>
diff --git a/llvm/test/CodeGen/AMDGPU/multi-divergent-exit-region.ll b/llvm/test/CodeGen/AMDGPU/multi-divergent-exit-region.ll
index 10d08032bf59a..a16bbeddde7f9 100644
--- a/llvm/test/CodeGen/AMDGPU/multi-divergent-exit-region.ll
+++ b/llvm/test/CodeGen/AMDGPU/multi-divergent-exit-region.ll
@@ -326,13 +326,12 @@ exit1:                                     ; preds = %LeafBlock, %LeafBlock1
 
 ; IR-LABEL: @multi_divergent_region_exit_ret_ret_return_value(
 ; IR: Flow2:
-; IR: %8 = phi float [ 2.000000e+00, %exit1 ], [ undef, %Flow1 ]
-; IR: %9 = phi i1 [ false, %exit1 ], [ %13, %Flow1 ]
-; IR: call void @llvm.amdgcn.end.cf.i64(i64 %17)
+; IR: %8 = phi i1 [ false, %exit1 ], [ %12, %Flow1 ]
+; IR: call void @llvm.amdgcn.end.cf.i64(i64 %16)
 
 ; IR: UnifiedReturnBlock:
-; IR: %UnifiedRetVal = phi float [ %8, %Flow2 ], [ 1.000000e+00, %exit0 ]
-; IR: call void @llvm.amdgcn.end.cf.i64(i64 %12)
+; IR: %UnifiedRetVal = phi float [ 2.000000e+00, %Flow2 ], [ 1.000000e+00, %exit0 ]
+; IR: call void @llvm.amdgcn.end.cf.i64(i64 %11)
 ; IR: ret float %UnifiedRetVal
 define amdgpu_ps float @multi_divergent_region_exit_ret_ret_return_value(i32 %vgpr) #0 {
 entry:
@@ -367,7 +366,7 @@ exit1:                                     ; preds = %LeafBlock, %LeafBlock1
 ; GCN: {{^}}[[FLOW]]:
 
 ; GCN: s_or_b64 exec, exec
-; GCN: v_mov_b32_e32 v0, s6
+; GCN: v_mov_b32_e32 v0, 2.0
 ; GCN-NOT: s_and_b64 exec, exec
 ; GCN: v_mov_b32_e32 v0, 1.0
 
diff --git a/llvm/test/CodeGen/AMDGPU/multilevel-break.ll b/llvm/test/CodeGen/AMDGPU/multilevel-break.ll
index 056cc9b963423..9fb2474571a12 100644
--- a/llvm/test/CodeGen/AMDGPU/multilevel-break.ll
+++ b/llvm/test/CodeGen/AMDGPU/multilevel-break.ll
@@ -5,42 +5,43 @@
 ; Ensure two if.break calls, for both the inner and outer loops
 ; FIXME: duplicate comparison
 define amdgpu_vs void @multi_else_break(<4 x float> %vec, i32 %ub, i32 %cont) {
-; OPT-LABEL: @multi_else_break(
-; OPT-NEXT:  main_body:
-; OPT-NEXT:    br label [[LOOP_OUTER:%.*]]
-; OPT:       LOOP.outer:
-; OPT-NEXT:    [[PHI_BROKEN2:%.*]] = phi i64 [ [[TMP8:%.*]], [[FLOW1:%.*]] ], [ 0, [[MAIN_BODY:%.*]] ]
-; OPT-NEXT:    [[TMP43:%.*]] = phi i32 [ 0, [[MAIN_BODY]] ], [ [[TMP3:%.*]], [[FLOW1]] ]
-; OPT-NEXT:    br label [[LOOP:%.*]]
-; OPT:       LOOP:
-; OPT-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP6:%.*]], [[FLOW:%.*]] ], [ 0, [[LOOP_OUTER]] ]
-; OPT-NEXT:    [[TMP45:%.*]] = phi i32 [ [[TMP43]], [[LOOP_OUTER]] ], [ [[TMP3]], [[FLOW]] ]
-; OPT-NEXT:    [[TMP48:%.*]] = icmp slt i32 [[TMP45]], [[UB:%.*]]
+; OPT-LABEL: define amdgpu_vs void @multi_else_break(
+; OPT-SAME: <4 x float> [[VEC:%.*]], i32 [[UB:%.*]], i32 [[CONT:%.*]]) {
+; OPT-NEXT:  [[MAIN_BODY:.*]]:
+; OPT-NEXT:    br label %[[LOOP_OUTER:.*]]
+; OPT:       [[LOOP_OUTER]]:
+; OPT-NEXT:    [[PHI_BROKEN2:%.*]] = phi i64 [ [[TMP8:%.*]], %[[FLOW1:.*]] ], [ 0, %[[MAIN_BODY]] ]
+; OPT-NEXT:    [[TMP43:%.*]] = phi i32 [ 0, %[[MAIN_BODY]] ], [ [[TMP3:%.*]], %[[FLOW1]] ]
+; OPT-NEXT:    br label %[[LOOP:.*]]
+; OPT:       [[LOOP]]:
+; OPT-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP6:%.*]], %[[FLOW:.*]] ], [ 0, %[[LOOP_OUTER]] ]
+; OPT-NEXT:    [[TMP45:%.*]] = phi i32 [ [[TMP43]], %[[LOOP_OUTER]] ], [ [[TMP3]], %[[FLOW]] ]
+; OPT-NEXT:    [[TMP48:%.*]] = icmp slt i32 [[TMP45]], [[UB]]
 ; OPT-NEXT:    [[TMP0:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[TMP48]])
 ; OPT-NEXT:    [[TMP1:%.*]] = extractvalue { i1, i64 } [[TMP0]], 0
 ; OPT-NEXT:    [[TMP2:%.*]] = extractvalue { i1, i64 } [[TMP0]], 1
-; OPT-NEXT:    br i1 [[TMP1]], label [[ENDIF:%.*]], label [[FLOW]]
-; OPT:       Flow:
-; OPT-NEXT:    [[TMP3]] = phi i32 [ [[TMP47:%.*]], [[ENDIF]] ], [ undef, [[LOOP]] ]
-; OPT-NEXT:    [[TMP4:%.*]] = phi i1 [ [[TMP51:%.*]], [[ENDIF]] ], [ true, [[LOOP]] ]
-; OPT-NEXT:    [[TMP5:%.*]] = phi i1 [ [[TMP51_INV:%.*]], [[ENDIF]] ], [ true, [[LOOP]] ]
+; OPT-NEXT:    br i1 [[TMP1]], label %[[ENDIF:.*]], label %[[FLOW]]
+; OPT:       [[FLOW]]:
+; OPT-NEXT:    [[TMP3]] = phi i32 [ [[TMP47:%.*]], %[[ENDIF]] ], [ poison, %[[LOOP]] ]
+; OPT-NEXT:    [[TMP4:%.*]] = phi i1 [ [[TMP51:%.*]], %[[ENDIF]] ], [ true, %[[LOOP]] ]
+; OPT-NEXT:    [[TMP5:%.*]] = phi i1 [ [[TMP51_INV:%.*]], %[[ENDIF]] ], [ true, %[[LOOP]] ]
 ; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP2]])
 ; OPT-NEXT:    [[TMP6]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP5]], i64 [[PHI_BROKEN]])
 ; OPT-NEXT:    [[TMP7:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP6]])
-; OPT-NEXT:    br i1 [[TMP7]], label [[FLOW1]], label [[LOOP]]
-; OPT:       Flow1:
+; OPT-NEXT:    br i1 [[TMP7]], label %[[FLOW1]], label %[[LOOP]]
+; OPT:       [[FLOW1]]:
 ; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP6]])
 ; OPT-NEXT:    [[TMP8]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP4]], i64 [[PHI_BROKEN2]])
 ; OPT-NEXT:    [[TMP9:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP8]])
-; OPT-NEXT:    br i1 [[TMP9]], label [[IF:%.*]], label [[LOOP_OUTER]]
-; OPT:       IF:
+; OPT-NEXT:    br i1 [[TMP9]], label %[[IF:.*]], label %[[LOOP_OUTER]]
+; OPT:       [[IF]]:
 ; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP8]])
 ; OPT-NEXT:    ret void
-; OPT:       ENDIF:
+; OPT:       [[ENDIF]]:
 ; OPT-NEXT:    [[TMP47]] = add i32 [[TMP45]], 1
-; OPT-NEXT:    [[TMP51]] = icmp eq i32 [[TMP47]], [[CONT:%.*]]
+; OPT-NEXT:    [[TMP51]] = icmp eq i32 [[TMP47]], [[CONT]]
 ; OPT-NEXT:    [[TMP51_INV]] = xor i1 [[TMP51]], true
-; OPT-NEXT:    br label [[FLOW]]
+; OPT-NEXT:    br label %[[FLOW]]
 ;
 ; GCN-LABEL: multi_else_break:
 ; GCN:       ; %bb.0: ; %main_body
@@ -113,55 +114,52 @@ ENDIF:                                            ; preds = %LOOP
 }
 
 define amdgpu_kernel void @multi_if_break_loop(i32 %arg) #0 {
-; OPT-LABEL: @multi_if_break_loop(
-; OPT-NEXT:  bb:
+; OPT-LABEL: define amdgpu_kernel void @multi_if_break_loop(
+; OPT-SAME: i32 [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
+; OPT-NEXT:  [[BB:.*]]:
 ; OPT-NEXT:    [[ID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; OPT-NEXT:    [[TMP:%.*]] = sub i32 [[ID]], [[ARG:%.*]]
-; OPT-NEXT:    br label [[BB1:%.*]]
-; OPT:       bb1:
-; OPT-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP4:%.*]], [[FLOW4:%.*]] ], [ 0, [[BB:%.*]] ]
-; OPT-NEXT:    [[LSR_IV:%.*]] = phi i32 [ undef, [[BB]] ], [ [[TMP2:%.*]], [[FLOW4]] ]
-; OPT-NEXT:    [[LSR_IV_NEXT:%.*]] = add i32 [[LSR_IV]], 1
-; OPT-NEXT:    [[CMP0:%.*]] = icmp slt i32 [[LSR_IV_NEXT]], 0
+; OPT-NEXT:    [[TMP:%.*]] = sub i32 [[ID]], [[ARG]]
+; OPT-NEXT:    br label %[[BB1:.*]]
+; OPT:       [[BB1]]:
+; OPT-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP4:%.*]], %[[FLOW4:.*]] ], [ 0, %[[BB]] ]
+; OPT-NEXT:    [[LSR_IV:%.*]] = phi i32 [ poison, %[[BB]] ], [ [[TMP2:%.*]], %[[FLOW4]] ]
+; OPT-NEXT:    [[TMP2]] = add i32 [[LSR_IV]], 1
+; OPT-NEXT:    [[CMP0:%.*]] = icmp slt i32 [[TMP2]], 0
 ; OPT-NEXT:    [[LOAD0:%.*]] = load volatile i32, ptr addrspace(1) undef, align 4
-; OPT-NEXT:    br label [[NODEBLOCK:%.*]]
-; OPT:       NodeBlock:
+; OPT-NEXT:    br label %[[NODEBLOCK:.*]]
+; OPT:       [[NODEBLOCK]]:
 ; OPT-NEXT:    [[PIVOT:%.*]] = icmp sge i32 [[LOAD0]], 1
-; OPT-NEXT:    br i1 [[PIVOT]], label [[LEAFBLOCK1:%.*]], label [[FLOW:%.*]]
-; OPT:       LeafBlock1:
+; OPT-NEXT:    br i1 [[PIVOT]], label %[[LEAFBLOCK1:.*]], label %[[FLOW:.*]]
+; OPT:       [[LEAFBLOCK1]]:
 ; OPT-NEXT:    [[SWITCHLEAF2:%.*]] = icmp eq i32 [[LOAD0]], 1
-; OPT-NEXT:    br i1 [[SWITCHLEAF2]], label [[CASE1:%.*]], label [[FLOW3:%.*]]
-; OPT:       Flow3:
-; OPT-NEXT:    [[TMP0:%.*]] = phi i32 [ [[LSR_IV_NEXT]], [[CASE1]] ], [ undef, [[LEAFBLOCK1]] ]
-; OPT-NEXT:    [[TMP1:%.*]] = phi i1 [ [[CMP2:%.*]], [[CASE1]] ], [ true, [[LEAFBLOCK1]] ]
-; OPT-NEXT:    br label [[FLOW]]
-; OPT:       LeafBlock:
+; OPT-NEXT:    br i1 [[SWITCHLEAF2]], label %[[CASE1:.*]], label %[[FLOW3:.*]]
+; OPT:       [[FLOW3]]:
+; OPT-NEXT:    [[TMP1:%.*]] = phi i1 [ [[CMP2:%.*]], %[[CASE1]] ], [ true, %[[LEAFBLOCK1]] ]
+; OPT-NEXT:    br label %[[FLOW]]
+; OPT:       [[LEAFBLOCK:.*]]:
 ; OPT-NEXT:    [[SWITCHLEAF:%.*]] = icmp eq i32 [[LOAD0]], 0
-; OPT-NEXT:    br i1 [[SWITCHLEAF]], label [[CASE0:%.*]], label [[FLOW5:%.*]]
-; OPT:       Flow4:
-; OPT-NEXT:    [[TMP2]] = phi i32 [ [[TMP9:%.*]], [[FLOW5]] ], [ [[TMP6:%.*]], [[FLOW]] ]
-; OPT-NEXT:    [[TMP3:%.*]] = phi i1 [ [[TMP10:%.*]], [[FLOW5]] ], [ [[TMP7:%.*]], [[FLOW]] ]
+; OPT-NEXT:    br i1 [[SWITCHLEAF]], label %[[CASE0:.*]], label %[[FLOW5:.*]]
+; OPT:       [[FLOW4]]:
+; OPT-NEXT:    [[TMP3:%.*]] = phi i1 [ [[TMP10:%.*]], %[[FLOW5]] ], [ [[TMP7:%.*]], %[[FLOW]] ]
 ; OPT-NEXT:    [[TMP4]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP3]], i64 [[PHI_BROKEN]])
 ; OPT-NEXT:    [[TMP5:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP4]])
-; OPT-NEXT:    br i1 [[TMP5]], label [[BB9:%.*]], label [[BB1]]
-; OPT:       case0:
+; OPT-NEXT:    br i1 [[TMP5]], label %[[BB9:.*]], label %[[BB1]]
+; OPT:       [[CASE0]]:
 ; OPT-NEXT:    [[LOAD1:%.*]] = load volatile i32, ptr addrspace(1) undef, align 4
 ; OPT-NEXT:    [[CMP1:%.*]] = icmp sge i32 [[TMP]], [[LOAD1]]
-; OPT-NEXT:    br label [[FLOW5]]
-; OPT:       Flow:
-; OPT-NEXT:    [[TMP6]] = phi i32 [ [[TMP0]], [[FLOW3]] ], [ undef, [[NODEBLOCK]] ]
-; OPT-NEXT:    [[TMP7]] = phi i1 [ [[TMP1]], [[FLOW3]] ], [ true, [[NODEBLOCK]] ]
-; OPT-NEXT:    [[TMP8:%.*]] = phi i1 [ false, [[FLOW3]] ], [ true, [[NODEBLOCK]] ]
-; OPT-NEXT:    br i1 [[TMP8]], label [[LEAFBLOCK:%.*]], label [[FLOW4]]
-; OPT:       case1:
+; OPT-NEXT:    br label %[[FLOW5]]
+; OPT:       [[FLOW]]:
+; OPT-NEXT:    [[TMP7]] = phi i1 [ [[TMP1]], %[[FLOW3]] ], [ true, %[[NODEBLOCK]] ]
+; OPT-NEXT:    [[TMP8:%.*]] = phi i1 [ false, %[[FLOW3]] ], [ true, %[[NODEBLOCK]] ]
+; OPT-NEXT:    br i1 [[TMP8]], label %[[LEAFBLOCK]], label %[[FLOW4]]
+; OPT:       [[CASE1]]:
 ; OPT-NEXT:    [[LOAD2:%.*]] = load volatile i32, ptr addrspace(1) undef, align 4
 ; OPT-NEXT:    [[CMP2]] = icmp sge i32 [[TMP]], [[LOAD2]]
-; OPT-NEXT:    br label [[FLOW3]]
-; OPT:       Flow5:
-; OPT-NEXT:    [[TMP9]] = phi i32 [ [[LSR_IV_NEXT]], [[CASE0]] ], [ undef, [[LEAFBLOCK]] ]
-; OPT-NEXT:    [[TMP10]] = phi i1 [ [[CMP1]], [[CASE0]] ], [ [[TMP7]], [[LEAFBLOCK]] ]
-; OPT-NEXT:    br label [[FLOW4]]
-; OPT:       bb9:
+; OPT-NEXT:    br label %[[FLOW3]]
+; OPT:       [[FLOW5]]:
+; OPT-NEXT:    [[TMP10]] = phi i1 [ [[CMP1]], %[[CASE0]] ], [ [[TMP7]], %[[LEAFBLOCK]] ]
+; OPT-NEXT:    br label %[[FLOW4]]
+; OPT:       [[BB9]]:
 ; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP4]])
 ; OPT-NEXT:    ret void
 ;
@@ -227,7 +225,7 @@ bb:
   br label %bb1
 
 bb1:
-  %lsr.iv = phi i32 [ undef, %bb ], [ %lsr.iv.next, %case0 ], [ %lsr.iv.next, %case1 ]
+  %lsr.iv = phi i32 [ poison, %bb ], [ %lsr.iv.next, %case0 ], [ %lsr.iv.next, %case1 ]
   %lsr.iv.next = add i32 %lsr.iv, 1
   %cmp0 = icmp slt i32 %lsr.iv.next, 0
   %load0 = load volatile i32, ptr addrspace(1) undef, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/nested-loop-conditions.ll b/llvm/test/CodeGen/AMDGPU/nested-loop-conditions.ll
index b8e451bfb74e7..9b383c435e160 100644
--- a/llvm/test/CodeGen/AMDGPU/nested-loop-conditions.ll
+++ b/llvm/test/CodeGen/AMDGPU/nested-loop-conditions.ll
@@ -13,28 +13,27 @@ define amdgpu_kernel void @reduced_nested_loop_conditions(ptr addrspace(3) captu
 ; GCN-NEXT:    s_load_dword s0, s[4:5], 0x9
 ; GCN-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; GCN-NEXT:    s_mov_b32 m0, -1
-; GCN-NEXT:    s_mov_b32 s2, 0
+; GCN-NEXT:    s_and_b64 s[2:3], exec, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
 ; GCN-NEXT:    ds_read_b64 v[0:1], v0
-; GCN-NEXT:    s_and_b64 vcc, exec, 0
+; GCN-NEXT:    s_and_b64 s[0:1], exec, 0
 ; GCN-NEXT:    s_branch .LBB0_2
 ; GCN-NEXT:  .LBB0_1: ; %Flow
 ; GCN-NEXT:    ; in Loop: Header=BB0_2 Depth=1
-; GCN-NEXT:    ; implicit-def: $sgpr2
-; GCN-NEXT:    s_mov_b64 vcc, vcc
+; GCN-NEXT:    s_mov_b64 vcc, s[0:1]
 ; GCN-NEXT:    s_cbranch_vccz .LBB0_4
 ; GCN-NEXT:  .LBB0_2: ; %bb5
 ; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT:    s_cmp_lg_u32 s2, 1
-; GCN-NEXT:    s_mov_b64 s[0:1], -1
-; GCN-NEXT:    s_cbranch_scc0 .LBB0_1
+; GCN-NEXT:    s_mov_b64 s[4:5], -1
+; GCN-NEXT:    s_mov_b64 vcc, s[2:3]
+; GCN-NEXT:    s_cbranch_vccz .LBB0_1
 ; GCN-NEXT:  ; %bb.3: ; %bb10
 ; GCN-NEXT:    ; in Loop: Header=BB0_2 Depth=1
-; GCN-NEXT:    s_mov_b64 s[0:1], 0
+; GCN-NEXT:    s_mov_b64 s[4:5], 0
 ; GCN-NEXT:    s_branch .LBB0_1
 ; GCN-NEXT:  .LBB0_4: ; %loop.exit.guard
-; GCN-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; GCN-NEXT:    s_and_b64 vcc, exec, s[4:5]
 ; GCN-NEXT:    s_cbranch_vccz .LBB0_7
 ; GCN-NEXT:  ; %bb.5: ; %bb8
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
@@ -74,7 +73,7 @@ define amdgpu_kernel void @reduced_nested_loop_conditions(ptr addrspace(3) captu
 ; IR-NEXT:    br i1 [[TMP3]], label %[[BB23:.*]], label %[[BB5]]
 ; IR:       [[FLOW]]:
 ; IR-NEXT:    [[TMP4:%.*]] = phi i1 [ [[MY_TMP22:%.*]], %[[BB4]] ], [ true, %[[BB5]] ]
-; IR-NEXT:    [[TMP5]] = phi i32 [ [[MY_TMP21:%.*]], %[[BB4]] ], [ undef, %[[BB5]] ]
+; IR-NEXT:    [[TMP5]] = phi i32 [ [[MY_TMP21:%.*]], %[[BB4]] ], [ poison, %[[BB5]] ]
 ; IR-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP2]])
 ; IR-NEXT:    [[TMP6]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP4]], i64 [[PHI_BROKEN]])
 ; IR-NEXT:    br label %[[BB10]]
@@ -94,6 +93,7 @@ define amdgpu_kernel void @reduced_nested_loop_conditions(ptr addrspace(3) captu
 ; IR:       [[BB23]]:
 ; IR-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP6]])
 ; IR-NEXT:    ret void
+;
 bb:
   %my.tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %my.tmp1 = getelementptr inbounds i64, ptr addrspace(3) %arg, i32 %my.tmp
@@ -118,7 +118,7 @@ bb9:                                              ; preds = %bb20, %bb9
   br i1 false, label %bb3, label %bb9
 
 bb10:                                             ; preds = %bb5, %bb4
-  %my.tmp11 = phi i32 [ %my.tmp21, %bb4 ], [ undef, %bb5 ]
+  %my.tmp11 = phi i32 [ %my.tmp21, %bb4 ], [ poison, %bb5 ]
   %my.tmp12 = phi i1 [ %my.tmp22, %bb4 ], [ true, %bb5 ]
   br i1 %my.tmp12, label %bb23, label %bb5
 
@@ -236,8 +236,8 @@ define amdgpu_kernel void @nested_loop_conditions(ptr addrspace(1) captures(none
 ; IR-NEXT:    [[MY_TMP17:%.*]] = bitcast i64 [[MY_TMP3]] to <2 x i32>
 ; IR-NEXT:    br label %[[BB18:.*]]
 ; IR:       [[FLOW1]]:
-; IR-NEXT:    [[TMP11]] = phi <4 x i32> [ [[MY_TMP9:%.*]], %[[BB21:.*]] ], [ undef, %[[BB14]] ]
-; IR-NEXT:    [[TMP12]] = phi i32 [ [[MY_TMP10:%.*]], %[[BB21]] ], [ undef, %[[BB14]] ]
+; IR-NEXT:    [[TMP11]] = phi <4 x i32> [ [[MY_TMP9:%.*]], %[[BB21:.*]] ], [ poison, %[[BB14]] ]
+; IR-NEXT:    [[TMP12]] = phi i32 [ [[MY_TMP10:%.*]], %[[BB21]] ], [ poison, %[[BB14]] ]
 ; IR-NEXT:    [[TMP13:%.*]] = phi i1 [ [[MY_TMP12:%.*]], %[[BB21]] ], [ true, %[[BB14]] ]
 ; IR-NEXT:    [[TMP14]] = phi i1 [ [[MY_TMP12]], %[[BB21]] ], [ false, %[[BB14]] ]
 ; IR-NEXT:    [[TMP15:%.*]] = phi i1 [ false, %[[BB21]] ], [ true, %[[BB14]] ]
@@ -278,6 +278,7 @@ define amdgpu_kernel void @nested_loop_conditions(ptr addrspace(1) captures(none
 ; IR-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP7]])
 ; IR-NEXT:    store volatile i32 0, ptr addrspace(1) undef, align 4
 ; IR-NEXT:    ret void
+;
 bb:
   %my.tmp1134 = load volatile i32, ptr addrspace(1) undef
   %my.tmp1235 = icmp slt i32 %my.tmp1134, 9
diff --git a/llvm/test/CodeGen/AMDGPU/nsa-reassign.ll b/llvm/test/CodeGen/AMDGPU/nsa-reassign.ll
index 6df702d61602a..ff80af3d700ba 100644
--- a/llvm/test/CodeGen/AMDGPU/nsa-reassign.ll
+++ b/llvm/test/CodeGen/AMDGPU/nsa-reassign.ll
@@ -15,7 +15,7 @@ main_body:
   %lod.1 = fadd float %lod, 1.0
   %v1 = call float @llvm.amdgcn.image.sample.c.l.3d.f32.f32(i32 1, float %zcompare.1, float %s1.1, float %t1.1, float %r1.1, float %lod.1, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   %v2 = call float @llvm.amdgcn.image.sample.3d.f32.f32(i32 1, float %s2.1, float %t2.1, float %r2.1, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
-  %r.0 = insertelement <2 x float> undef, float %v1, i32 0
+  %r.0 = insertelement <2 x float> poison, float %v1, i32 0
   %r = insertelement <2 x float> %r.0, float %v2, i32 1
   ret <2 x float> %r
 }
@@ -35,7 +35,7 @@ main_body:
   %lod.1 = fadd float %lod, 1.0
   %v1 = call float @llvm.amdgcn.image.sample.c.l.3d.f32.f32(i32 1, float %zcompare.1, float %s1.1, float %t1.1, float %r1.1, float %lod.1, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   %v2 = call float @llvm.amdgcn.image.sample.3d.f32.f32(i32 1, float %s2.1, float %t2.1, float %r2.1, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
-  %r.0 = insertelement <2 x float> undef, float %v1, i32 0
+  %r.0 = insertelement <2 x float> poison, float %v1, i32 0
   %r = insertelement <2 x float> %r.0, float %v2, i32 1
   ret <2 x float> %r
 }
@@ -55,7 +55,7 @@ main_body:
   %lod.1 = fadd float %lod, 1.0
   %v2 = call float @llvm.amdgcn.image.sample.3d.f32.f32(i32 1, float %s2.1, float %t2.1, float %r2.1, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   %v1 = call float @llvm.amdgcn.image.sample.3d.f32.f32(i32 1, float %t2.1, float %s2.1, float %r2.1, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
-  %r.0 = insertelement <2 x float> undef, float %v1, i32 0
+  %r.0 = insertelement <2 x float> poison, float %v1, i32 0
   %r = insertelement <2 x float> %r.0, float %v2, i32 1
   ret <2 x float> %r
 }
@@ -75,7 +75,7 @@ main_body:
   %lod.1 = fadd float %lod, 1.0
   %v2 = call float @llvm.amdgcn.image.sample.3d.f32.f32(i32 1, float %s2.1, float %t2.1, float %r2.1, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 1)
   %v1 = call float @llvm.amdgcn.image.sample.3d.f32.f32(i32 1, float %s2.1, float %t2.1, float %r2.1, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
-  %r.0 = insertelement <2 x float> undef, float %v1, i32 0
+  %r.0 = insertelement <2 x float> poison, float %v1, i32 0
   %r = insertelement <2 x float> %r.0, float %v2, i32 1
   ret <2 x float> %r
 }
diff --git a/llvm/test/CodeGen/AMDGPU/operand-folding.ll b/llvm/test/CodeGen/AMDGPU/operand-folding.ll
index 1e3299a023736..040f53fdf66bb 100644
--- a/llvm/test/CodeGen/AMDGPU/operand-folding.ll
+++ b/llvm/test/CodeGen/AMDGPU/operand-folding.ll
@@ -68,7 +68,7 @@ entry:
   %tmp1 = add i32 %tmp0, 1
   %tmp2 = add i32 %tmp0, 2
   %tmp3 = add i32 %tmp0, 3
-  %vec0 = insertelement <4 x i32> undef, i32 %tmp0, i32 0
+  %vec0 = insertelement <4 x i32> poison, i32 %tmp0, i32 0
   %vec1 = insertelement <4 x i32> %vec0, i32 %tmp1, i32 1
   %vec2 = insertelement <4 x i32> %vec1, i32 %tmp2, i32 2
   %vec3 = insertelement <4 x i32> %vec2, i32 %tmp3, i32 3
@@ -100,7 +100,7 @@ entry:
   %tmp1 = add i32 %tmp0, 1
   %tmp2 = add i32 %tmp0, 2
   %tmp3 = add i32 %tmp0, 3
-  %vec0 = insertelement <4 x i32> undef, i32 %tmp0, i32 0
+  %vec0 = insertelement <4 x i32> poison, i32 %tmp0, i32 0
   %vec1 = insertelement <4 x i32> %vec0, i32 %tmp1, i32 1
   %vec2 = insertelement <4 x i32> %vec1, i32 %tmp2, i32 2
   %vec3 = insertelement <4 x i32> %vec2, i32 %tmp3, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/optimize-negated-cond.ll b/llvm/test/CodeGen/AMDGPU/optimize-negated-cond.ll
index 3e45a2d0df43d..720eaeff2e1ec 100644
--- a/llvm/test/CodeGen/AMDGPU/optimize-negated-cond.ll
+++ b/llvm/test/CodeGen/AMDGPU/optimize-negated-cond.ll
@@ -40,15 +40,16 @@ define amdgpu_kernel void @negated_cond(ptr addrspace(1) %arg1) {
 ; GCN-NEXT:    s_lshl_b32 s12, s12, 5
 ; GCN-NEXT:    s_cbranch_vccz .LBB0_6
 ; GCN-NEXT:  ; %bb.5: ; in Loop: Header=BB0_4 Depth=2
-; GCN-NEXT:    s_mov_b64 s[14:15], s[2:3]
+; GCN-NEXT:    s_mov_b64 s[16:17], s[2:3]
 ; GCN-NEXT:    s_branch .LBB0_7
 ; GCN-NEXT:  .LBB0_6: ; %bb3
 ; GCN-NEXT:    ; in Loop: Header=BB0_4 Depth=2
 ; GCN-NEXT:    s_add_i32 s12, s12, 1
-; GCN-NEXT:    s_mov_b64 s[14:15], -1
+; GCN-NEXT:    s_mov_b64 s[16:17], -1
 ; GCN-NEXT:  .LBB0_7: ; %Flow
 ; GCN-NEXT:    ; in Loop: Header=BB0_4 Depth=2
-; GCN-NEXT:    s_andn2_b64 vcc, exec, s[14:15]
+; GCN-NEXT:    s_mov_b64 s[14:15], -1
+; GCN-NEXT:    s_andn2_b64 vcc, exec, s[16:17]
 ; GCN-NEXT:    s_mov_b64 s[16:17], -1
 ; GCN-NEXT:    s_cbranch_vccnz .LBB0_3
 ; GCN-NEXT:  ; %bb.8: ; %bb4
diff --git a/llvm/test/CodeGen/AMDGPU/pack.v2f16.ll b/llvm/test/CodeGen/AMDGPU/pack.v2f16.ll
index da6120812ac1d..ef9305e1fc358 100644
--- a/llvm/test/CodeGen/AMDGPU/pack.v2f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/pack.v2f16.ll
@@ -53,7 +53,7 @@ define amdgpu_kernel void @s_pack_v2f16(ptr addrspace(4) %in0, ptr addrspace(4)
   %hi.i = trunc i32 %val1 to i16
   %lo = bitcast i16 %lo.i to half
   %hi = bitcast i16 %hi.i to half
-  %vec.0 = insertelement <2 x half> undef, half %lo, i32 0
+  %vec.0 = insertelement <2 x half> poison, half %lo, i32 0
   %vec.1 = insertelement <2 x half> %vec.0, half %hi, i32 1
   %vec.i32 = bitcast <2 x half> %vec.1 to i32
 
@@ -102,7 +102,7 @@ define amdgpu_kernel void @s_pack_v2f16_imm_lo(ptr addrspace(4) %in1) #0 {
   %val1 = load i32, ptr addrspace(4) %in1
   %hi.i = trunc i32 %val1 to i16
   %hi = bitcast i16 %hi.i to half
-  %vec.0 = insertelement <2 x half> undef, half 0xH1234, i32 0
+  %vec.0 = insertelement <2 x half> poison, half 0xH1234, i32 0
   %vec.1 = insertelement <2 x half> %vec.0, half %hi, i32 1
   %vec.i32 = bitcast <2 x half> %vec.1 to i32
 
@@ -151,7 +151,7 @@ define amdgpu_kernel void @s_pack_v2f16_imm_hi(ptr addrspace(4) %in0) #0 {
   %val0 = load i32, ptr addrspace(4) %in0
   %lo.i = trunc i32 %val0 to i16
   %lo = bitcast i16 %lo.i to half
-  %vec.0 = insertelement <2 x half> undef, half %lo, i32 0
+  %vec.0 = insertelement <2 x half> poison, half %lo, i32 0
   %vec.1 = insertelement <2 x half> %vec.0, half 0xH1234, i32 1
   %vec.i32 = bitcast <2 x half> %vec.1 to i32
 
@@ -230,7 +230,7 @@ define amdgpu_kernel void @v_pack_v2f16(ptr addrspace(1) %in0, ptr addrspace(1)
   %hi.i = trunc i32 %val1 to i16
   %lo = bitcast i16 %lo.i to half
   %hi = bitcast i16 %hi.i to half
-  %vec.0 = insertelement <2 x half> undef, half %lo, i32 0
+  %vec.0 = insertelement <2 x half> poison, half %lo, i32 0
   %vec.1 = insertelement <2 x half> %vec.0, half %hi, i32 1
   %vec.i32 = bitcast <2 x half> %vec.1 to i32
   call void asm sideeffect "; use $0", "v"(i32 %vec.i32) #0
@@ -313,7 +313,7 @@ define amdgpu_kernel void @v_pack_v2f16_user(ptr addrspace(1) %in0, ptr addrspac
   %hi.i = trunc i32 %val1 to i16
   %lo = bitcast i16 %lo.i to half
   %hi = bitcast i16 %hi.i to half
-  %vec.0 = insertelement <2 x half> undef, half %lo, i32 0
+  %vec.0 = insertelement <2 x half> poison, half %lo, i32 0
   %vec.1 = insertelement <2 x half> %vec.0, half %hi, i32 1
   %vec.i32 = bitcast <2 x half> %vec.1 to i32
   %foo = add i32 %vec.i32, 9
@@ -376,7 +376,7 @@ define amdgpu_kernel void @v_pack_v2f16_imm_lo(ptr addrspace(1) %in1) #0 {
   %val1 = load volatile i32, ptr addrspace(1) %in1.gep
   %hi.i = trunc i32 %val1 to i16
   %hi = bitcast i16 %hi.i to half
-  %vec.0 = insertelement <2 x half> undef, half 0xH1234, i32 0
+  %vec.0 = insertelement <2 x half> poison, half 0xH1234, i32 0
   %vec.1 = insertelement <2 x half> %vec.0, half %hi, i32 1
   %vec.i32 = bitcast <2 x half> %vec.1 to i32
   call void asm sideeffect "; use $0", "v"(i32 %vec.i32) #0
@@ -438,7 +438,7 @@ define amdgpu_kernel void @v_pack_v2f16_inline_imm_lo(ptr addrspace(1) %in1) #0
   %val1 = load volatile i32, ptr addrspace(1) %in1.gep
   %hi.i = trunc i32 %val1 to i16
   %hi = bitcast i16 %hi.i to half
-  %vec.0 = insertelement <2 x half> undef, half 4.0, i32 0
+  %vec.0 = insertelement <2 x half> poison, half 4.0, i32 0
   %vec.1 = insertelement <2 x half> %vec.0, half %hi, i32 1
   %vec.i32 = bitcast <2 x half> %vec.1 to i32
   call void asm sideeffect "; use $0", "v"(i32 %vec.i32) #0
@@ -500,7 +500,7 @@ define amdgpu_kernel void @v_pack_v2f16_imm_hi(ptr addrspace(1) %in0) #0 {
   %val0 = load volatile i32, ptr addrspace(1) %in0.gep
   %lo.i = trunc i32 %val0 to i16
   %lo = bitcast i16 %lo.i to half
-  %vec.0 = insertelement <2 x half> undef, half %lo, i32 0
+  %vec.0 = insertelement <2 x half> poison, half %lo, i32 0
   %vec.1 = insertelement <2 x half> %vec.0, half 0xH1234, i32 1
   %vec.i32 = bitcast <2 x half> %vec.1 to i32
   call void asm sideeffect "; use $0", "v"(i32 %vec.i32) #0
@@ -562,7 +562,7 @@ define amdgpu_kernel void @v_pack_v2f16_inline_f16imm_hi(ptr addrspace(1) %in0)
   %val0 = load volatile i32, ptr addrspace(1) %in0.gep
   %lo.i = trunc i32 %val0 to i16
   %lo = bitcast i16 %lo.i to half
-  %vec.0 = insertelement <2 x half> undef, half %lo, i32 0
+  %vec.0 = insertelement <2 x half> poison, half %lo, i32 0
   %vec.1 = insertelement <2 x half> %vec.0, half 1.0, i32 1
   %vec.i32 = bitcast <2 x half> %vec.1 to i32
   call void asm sideeffect "; use $0", "v"(i32 %vec.i32) #0
@@ -623,7 +623,7 @@ define amdgpu_kernel void @v_pack_v2f16_inline_imm_hi(ptr addrspace(1) %in0) #0
   %val0 = load volatile i32, ptr addrspace(1) %in0.gep
   %lo.i = trunc i32 %val0 to i16
   %lo = bitcast i16 %lo.i to half
-  %vec.0 = insertelement <2 x half> undef, half %lo, i32 0
+  %vec.0 = insertelement <2 x half> poison, half %lo, i32 0
   %vec.1 = insertelement <2 x half> %vec.0, half 0xH0040, i32 1
   %vec.i32 = bitcast <2 x half> %vec.1 to i32
   call void asm sideeffect "; use $0", "v"(i32 %vec.i32) #0
diff --git a/llvm/test/CodeGen/AMDGPU/pack.v2i16.ll b/llvm/test/CodeGen/AMDGPU/pack.v2i16.ll
index 44128f0e0dcd8..e706661e2f199 100644
--- a/llvm/test/CodeGen/AMDGPU/pack.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/pack.v2i16.ll
@@ -51,7 +51,7 @@ define amdgpu_kernel void @s_pack_v2i16(ptr addrspace(4) %in0, ptr addrspace(4)
   %val1 = load volatile i32, ptr addrspace(4) %in1
   %lo = trunc i32 %val0 to i16
   %hi = trunc i32 %val1 to i16
-  %vec.0 = insertelement <2 x i16> undef, i16 %lo, i32 0
+  %vec.0 = insertelement <2 x i16> poison, i16 %lo, i32 0
   %vec.1 = insertelement <2 x i16> %vec.0, i16 %hi, i32 1
   %vec.i32 = bitcast <2 x i16> %vec.1 to i32
 
@@ -99,7 +99,7 @@ define amdgpu_kernel void @s_pack_v2i16_imm_lo(ptr addrspace(4) %in1) #0 {
 ; GFX7-NEXT:    s_endpgm
   %val1 = load i32, ptr addrspace(4) %in1
   %hi = trunc i32 %val1 to i16
-  %vec.0 = insertelement <2 x i16> undef, i16 456, i32 0
+  %vec.0 = insertelement <2 x i16> poison, i16 456, i32 0
   %vec.1 = insertelement <2 x i16> %vec.0, i16 %hi, i32 1
   %vec.i32 = bitcast <2 x i16> %vec.1 to i32
 
@@ -147,7 +147,7 @@ define amdgpu_kernel void @s_pack_v2i16_imm_hi(ptr addrspace(4) %in0) #0 {
 ; GFX7-NEXT:    s_endpgm
   %val0 = load i32, ptr addrspace(4) %in0
   %lo = trunc i32 %val0 to i16
-  %vec.0 = insertelement <2 x i16> undef, i16 %lo, i32 0
+  %vec.0 = insertelement <2 x i16> poison, i16 %lo, i32 0
   %vec.1 = insertelement <2 x i16> %vec.0, i16 456, i32 1
   %vec.i32 = bitcast <2 x i16> %vec.1 to i32
 
@@ -224,7 +224,7 @@ define amdgpu_kernel void @v_pack_v2i16(ptr addrspace(1) %in0, ptr addrspace(1)
   %val1 = load volatile i32, ptr addrspace(1) %in1.gep
   %lo = trunc i32 %val0 to i16
   %hi = trunc i32 %val1 to i16
-  %vec.0 = insertelement <2 x i16> undef, i16 %lo, i32 0
+  %vec.0 = insertelement <2 x i16> poison, i16 %lo, i32 0
   %vec.1 = insertelement <2 x i16> %vec.0, i16 %hi, i32 1
   %vec.i32 = bitcast <2 x i16> %vec.1 to i32
   call void asm sideeffect "; use $0", "v"(i32 %vec.i32) #0
@@ -305,7 +305,7 @@ define amdgpu_kernel void @v_pack_v2i16_user(ptr addrspace(1) %in0, ptr addrspac
   %val1 = load volatile i32, ptr addrspace(1) %in1.gep
   %lo = trunc i32 %val0 to i16
   %hi = trunc i32 %val1 to i16
-  %vec.0 = insertelement <2 x i16> undef, i16 %lo, i32 0
+  %vec.0 = insertelement <2 x i16> poison, i16 %lo, i32 0
   %vec.1 = insertelement <2 x i16> %vec.0, i16 %hi, i32 1
   %vec.i32 = bitcast <2 x i16> %vec.1 to i32
   %foo = add i32 %vec.i32, 9
@@ -367,7 +367,7 @@ define amdgpu_kernel void @v_pack_v2i16_imm_lo(ptr addrspace(1) %in1) #0 {
   %in1.gep = getelementptr inbounds i32, ptr addrspace(1) %in1, i64 %tid.ext
   %val1 = load volatile i32, ptr addrspace(1) %in1.gep
   %hi = trunc i32 %val1 to i16
-  %vec.0 = insertelement <2 x i16> undef, i16 123, i32 0
+  %vec.0 = insertelement <2 x i16> poison, i16 123, i32 0
   %vec.1 = insertelement <2 x i16> %vec.0, i16 %hi, i32 1
   %vec.i32 = bitcast <2 x i16> %vec.1 to i32
   call void asm sideeffect "; use $0", "v"(i32 %vec.i32) #0
@@ -427,7 +427,7 @@ define amdgpu_kernel void @v_pack_v2i16_inline_imm_lo(ptr addrspace(1) %in1) #0
   %in1.gep = getelementptr inbounds i32, ptr addrspace(1) %in1, i64 %tid.ext
   %val1 = load volatile i32, ptr addrspace(1) %in1.gep
   %hi = trunc i32 %val1 to i16
-  %vec.0 = insertelement <2 x i16> undef, i16 64, i32 0
+  %vec.0 = insertelement <2 x i16> poison, i16 64, i32 0
   %vec.1 = insertelement <2 x i16> %vec.0, i16 %hi, i32 1
   %vec.i32 = bitcast <2 x i16> %vec.1 to i32
   call void asm sideeffect "; use $0", "v"(i32 %vec.i32) #0
@@ -488,7 +488,7 @@ define amdgpu_kernel void @v_pack_v2i16_imm_hi(ptr addrspace(1) %in0) #0 {
   %in0.gep = getelementptr inbounds i32, ptr addrspace(1) %in0, i64 %tid.ext
   %val0 = load volatile i32, ptr addrspace(1) %in0.gep
   %lo = trunc i32 %val0 to i16
-  %vec.0 = insertelement <2 x i16> undef, i16 %lo, i32 0
+  %vec.0 = insertelement <2 x i16> poison, i16 %lo, i32 0
   %vec.1 = insertelement <2 x i16> %vec.0, i16 123, i32 1
   %vec.i32 = bitcast <2 x i16> %vec.1 to i32
   call void asm sideeffect "; use $0", "v"(i32 %vec.i32) #0
@@ -548,7 +548,7 @@ define amdgpu_kernel void @v_pack_v2i16_inline_imm_hi(ptr addrspace(1) %in0) #0
   %in0.gep = getelementptr inbounds i32, ptr addrspace(1) %in0, i64 %tid.ext
   %val0 = load volatile i32, ptr addrspace(1) %in0.gep
   %lo = trunc i32 %val0 to i16
-  %vec.0 = insertelement <2 x i16> undef, i16 %lo, i32 0
+  %vec.0 = insertelement <2 x i16> poison, i16 %lo, i32 0
   %vec.1 = insertelement <2 x i16> %vec.0, i16 7, i32 1
   %vec.i32 = bitcast <2 x i16> %vec.1 to i32
   call void asm sideeffect "; use $0", "v"(i32 %vec.i32) #0
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
index 9b03a72fd826d..866abc10b5777 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
@@ -77,7 +77,7 @@ define amdgpu_kernel void @fadd_v2_v_v_splat(ptr addrspace(1) %a) {
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
   %load = load <2 x float>, ptr addrspace(1) %gep, align 8
   %fid = bitcast i32 %id to float
-  %tmp1 = insertelement <2 x float> undef, float %fid, i64 0
+  %tmp1 = insertelement <2 x float> poison, float %fid, i64 0
   %k = insertelement <2 x float> %tmp1, float %fid, i64 1
   %add = fadd <2 x float> %load, %k
   store <2 x float> %add, ptr addrspace(1) %gep, align 8
@@ -152,7 +152,7 @@ define amdgpu_kernel void @fadd_v2_v_fneg(ptr addrspace(1) %a, float %x) {
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
   %load = load <2 x float>, ptr addrspace(1) %gep, align 8
   %fneg = fsub float -0.0, %x
-  %tmp1 = insertelement <2 x float> undef, float %fneg, i64 0
+  %tmp1 = insertelement <2 x float> poison, float %fneg, i64 0
   %k = insertelement <2 x float> %tmp1, float %fneg, i64 1
   %add = fadd <2 x float> %load, %k
   store <2 x float> %add, ptr addrspace(1) %gep, align 8
@@ -169,7 +169,7 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo(ptr addrspace(1) %a, float %x) {
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
   %load = load <2 x float>, ptr addrspace(1) %gep, align 8
   %fneg = fsub float -0.0, %x
-  %tmp1 = insertelement <2 x float> undef, float %fneg, i64 0
+  %tmp1 = insertelement <2 x float> poison, float %fneg, i64 0
   %k = insertelement <2 x float> %tmp1, float %x, i64 1
   %add = fadd <2 x float> %load, %k
   store <2 x float> %add, ptr addrspace(1) %gep, align 8
@@ -186,7 +186,7 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi(ptr addrspace(1) %a, float %x) {
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
   %load = load <2 x float>, ptr addrspace(1) %gep, align 8
   %fneg = fsub float -0.0, %x
-  %tmp1 = insertelement <2 x float> undef, float %x, i64 0
+  %tmp1 = insertelement <2 x float> poison, float %x, i64 0
   %k = insertelement <2 x float> %tmp1, float %fneg, i64 1
   %add = fadd <2 x float> %load, %k
   store <2 x float> %add, ptr addrspace(1) %gep, align 8
@@ -203,7 +203,7 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo2(ptr addrspace(1) %a, float %x, flo
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
   %load = load <2 x float>, ptr addrspace(1) %gep, align 8
   %fneg = fsub float -0.0, %x
-  %tmp1 = insertelement <2 x float> undef, float %fneg, i64 0
+  %tmp1 = insertelement <2 x float> poison, float %fneg, i64 0
   %k = insertelement <2 x float> %tmp1, float %y, i64 1
   %add = fadd <2 x float> %load, %k
   store <2 x float> %add, ptr addrspace(1) %gep, align 8
@@ -220,7 +220,7 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi2(ptr addrspace(1) %a, float %x, flo
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
   %load = load <2 x float>, ptr addrspace(1) %gep, align 8
   %fneg = fsub float -0.0, %x
-  %tmp1 = insertelement <2 x float> undef, float %y, i64 0
+  %tmp1 = insertelement <2 x float> poison, float %y, i64 0
   %k = insertelement <2 x float> %tmp1, float %fneg, i64 1
   %add = fadd <2 x float> %load, %k
   store <2 x float> %add, ptr addrspace(1) %gep, align 8
@@ -298,7 +298,7 @@ define amdgpu_kernel void @fmul_v2_v_v_splat(ptr addrspace(1) %a) {
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
   %load = load <2 x float>, ptr addrspace(1) %gep, align 8
   %fid = bitcast i32 %id to float
-  %tmp1 = insertelement <2 x float> undef, float %fid, i64 0
+  %tmp1 = insertelement <2 x float> poison, float %fid, i64 0
   %k = insertelement <2 x float> %tmp1, float %fid, i64 1
   %mul = fmul <2 x float> %load, %k
   store <2 x float> %mul, ptr addrspace(1) %gep, align 8
@@ -342,7 +342,7 @@ define amdgpu_kernel void @fmul_v2_v_fneg(ptr addrspace(1) %a, float %x) {
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
   %load = load <2 x float>, ptr addrspace(1) %gep, align 8
   %fneg = fsub float -0.0, %x
-  %tmp1 = insertelement <2 x float> undef, float %fneg, i64 0
+  %tmp1 = insertelement <2 x float> poison, float %fneg, i64 0
   %k = insertelement <2 x float> %tmp1, float %fneg, i64 1
   %mul = fmul <2 x float> %load, %k
   store <2 x float> %mul, ptr addrspace(1) %gep, align 8
@@ -422,7 +422,7 @@ define amdgpu_kernel void @fma_v2_v_v_splat(ptr addrspace(1) %a) {
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
   %load = load <2 x float>, ptr addrspace(1) %gep, align 8
   %fid = bitcast i32 %id to float
-  %tmp1 = insertelement <2 x float> undef, float %fid, i64 0
+  %tmp1 = insertelement <2 x float> poison, float %fid, i64 0
   %k = insertelement <2 x float> %tmp1, float %fid, i64 1
   %fma = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %load, <2 x float> %k, <2 x float> %k)
   store <2 x float> %fma, ptr addrspace(1) %gep, align 8
@@ -468,7 +468,7 @@ define amdgpu_kernel void @fma_v2_v_fneg(ptr addrspace(1) %a, float %x) {
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
   %load = load <2 x float>, ptr addrspace(1) %gep, align 8
   %fneg = fsub float -0.0, %x
-  %tmp1 = insertelement <2 x float> undef, float %fneg, i64 0
+  %tmp1 = insertelement <2 x float> poison, float %fneg, i64 0
   %k = insertelement <2 x float> %tmp1, float %fneg, i64 1
   %fma = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %load, <2 x float> %k, <2 x float> %k)
   store <2 x float> %fma, ptr addrspace(1) %gep, align 8
@@ -485,8 +485,8 @@ bb:
   %scalar0 = load volatile float, ptr addrspace(3) %arg2, align 4
   %neg.scalar0 = fsub float -0.0, %scalar0
 
-  %neg.scalar0.vec = insertelement <2 x float> undef, float %neg.scalar0, i32 0
-  %neg.scalar0.broadcast = shufflevector <2 x float> %neg.scalar0.vec, <2 x float> undef, <2 x i32> zeroinitializer
+  %neg.scalar0.vec = insertelement <2 x float> poison, float %neg.scalar0, i32 0
+  %neg.scalar0.broadcast = shufflevector <2 x float> %neg.scalar0.vec, <2 x float> poison, <2 x i32> zeroinitializer
 
   %result = fadd <2 x float> %vec0, %neg.scalar0.broadcast
   store <2 x float> %result, ptr addrspace(1) %out, align 4
@@ -508,7 +508,7 @@ bb:
   %scalar0 = load volatile float, ptr addrspace(3) %arg2, align 4
   %scalar1 = load volatile float, ptr addrspace(3) %arg2.gep, align 4
 
-  %vec.ins0 = insertelement <2 x float> undef, float %scalar0, i32 0
+  %vec.ins0 = insertelement <2 x float> poison, float %scalar0, i32 0
   %vec2 = insertelement <2 x float> %vec.ins0, float %scalar1, i32 1
   %neg.vec2 = fsub <2 x float> <float -0.0, float -0.0>, %vec2
 
@@ -526,7 +526,7 @@ bb:
   %vec0 = load volatile <2 x float>, ptr addrspace(3) %lds, align 8
   %lds.gep1 = getelementptr inbounds <2 x float>, ptr addrspace(3) %lds, i32 1
   %vec1 = load volatile <2 x float>, ptr addrspace(3) %lds.gep1, align 8
-  %vec1.swap = shufflevector <2 x float> %vec1, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+  %vec1.swap = shufflevector <2 x float> %vec1, <2 x float> poison, <2 x i32> <i32 1, i32 0>
   %result = fadd <2 x float> %vec0, %vec1.swap
   store <2 x float> %result, ptr addrspace(1) %out, align 8
   ret void
@@ -543,7 +543,7 @@ bb:
   %f32 = load volatile float, ptr addrspace(3) undef, align 8
   %vec1 = load volatile <2 x float>, ptr addrspace(3) %lds.gep1, align 8
   %vec1.neg = fsub <2 x float> <float -0.0, float -0.0>, %vec1
-  %vec1.neg.swap = shufflevector <2 x float> %vec1.neg, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+  %vec1.neg.swap = shufflevector <2 x float> %vec1.neg, <2 x float> poison, <2 x i32> <i32 1, i32 0>
   %result = fadd <2 x float> %vec0, %vec1.neg.swap
   store <2 x float> %result, ptr addrspace(1) %out, align 8
   ret void
@@ -598,7 +598,7 @@ bb:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <4 x float>, ptr addrspace(1) %arg, i32 %tid
   %in.1 = load <4 x float>, ptr addrspace(1) %gep
-  %shuf = shufflevector <4 x float> %in.1, <4 x float> undef, <4 x i32> zeroinitializer
+  %shuf = shufflevector <4 x float> %in.1, <4 x float> poison, <4 x i32> zeroinitializer
   %add.1 = fadd <4 x float> %in.1, %shuf
   store <4 x float> %add.1, ptr addrspace(1) %gep
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
index d6fc52bdb9322..081756a0b7816 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
@@ -23,8 +23,8 @@ bb:
   %vec1 = load volatile <2 x half>, ptr addrspace(3) %lds.gep1, align 4
   %scalar0 = load volatile half, ptr addrspace(3) %arg2, align 2
 
-  %scalar0.vec = insertelement <2 x half> undef, half %scalar0, i32 0
-  %scalar0.broadcast = shufflevector <2 x half> %scalar0.vec, <2 x half> undef, <2 x i32> zeroinitializer
+  %scalar0.vec = insertelement <2 x half> poison, half %scalar0, i32 0
+  %scalar0.broadcast = shufflevector <2 x half> %scalar0.vec, <2 x half> poison, <2 x i32> zeroinitializer
 
   %result = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %vec0, <2 x half> %vec1, <2 x half> %scalar0.broadcast)
   store <2 x half> %result, ptr addrspace(1) %out, align 4
@@ -54,8 +54,8 @@ bb:
   %vec1 = load volatile <2 x half>, ptr addrspace(3) %lds.gep1, align 4
   %scalar0 = load volatile half, ptr addrspace(3) %arg2, align 2
 
-  %scalar0.vec = insertelement <2 x half> undef, half %scalar0, i32 0
-  %scalar0.broadcast = shufflevector <2 x half> %scalar0.vec, <2 x half> undef, <2 x i32> zeroinitializer
+  %scalar0.vec = insertelement <2 x half> poison, half %scalar0, i32 0
+  %scalar0.broadcast = shufflevector <2 x half> %scalar0.vec, <2 x half> poison, <2 x i32> zeroinitializer
   %neg.scalar0.broadcast = fsub <2 x half> <half -0.0, half -0.0>, %scalar0.broadcast
 
   %result = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %vec0, <2 x half> %vec1, <2 x half> %neg.scalar0.broadcast)
@@ -87,8 +87,8 @@ bb:
   %scalar0 = load volatile half, ptr addrspace(3) %arg2, align 2
 
   %neg.scalar0 = fsub half -0.0, %scalar0
-  %neg.scalar0.vec = insertelement <2 x half> undef, half %neg.scalar0, i32 0
-  %neg.scalar0.broadcast = shufflevector <2 x half> %neg.scalar0.vec, <2 x half> undef, <2 x i32> zeroinitializer
+  %neg.scalar0.vec = insertelement <2 x half> poison, half %neg.scalar0, i32 0
+  %neg.scalar0.broadcast = shufflevector <2 x half> %neg.scalar0.vec, <2 x half> poison, <2 x i32> zeroinitializer
 
   %result = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %vec0, <2 x half> %vec1, <2 x half> %neg.scalar0.broadcast)
   store <2 x half> %result, ptr addrspace(1) %out, align 4
@@ -119,8 +119,8 @@ bb:
   %scalar0 = load volatile half, ptr addrspace(3) %arg2, align 2
 
   %neg.scalar0 = fsub half -0.0, %scalar0
-  %neg.scalar0.vec = insertelement <2 x half> undef, half %neg.scalar0, i32 0
-  %neg.scalar0.broadcast = shufflevector <2 x half> %neg.scalar0.vec, <2 x half> undef, <2 x i32> zeroinitializer
+  %neg.scalar0.vec = insertelement <2 x half> poison, half %neg.scalar0, i32 0
+  %neg.scalar0.broadcast = shufflevector <2 x half> %neg.scalar0.vec, <2 x half> poison, <2 x i32> zeroinitializer
   %neg.neg.scalar0.broadcast = fsub <2 x half> <half -0.0, half -0.0>, %neg.scalar0.broadcast
 
   %result = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %vec0, <2 x half> %vec1, <2 x half> %neg.neg.scalar0.broadcast)
@@ -152,7 +152,7 @@ bb:
   %scalar0 = load volatile half, ptr addrspace(3) %arg2, align 2
 
   %neg.scalar0 = fsub half -0.0, %scalar0
-  %neg.scalar0.vec = insertelement <2 x half> undef, half %neg.scalar0, i32 0
+  %neg.scalar0.vec = insertelement <2 x half> poison, half %neg.scalar0, i32 0
   %neg.scalar0.scalar0 = insertelement <2 x half> %neg.scalar0.vec, half %scalar0, i32 1
   %result = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %vec0, <2 x half> %vec1, <2 x half> %neg.scalar0.scalar0)
   store <2 x half> %result, ptr addrspace(1) %out, align 4
@@ -183,7 +183,7 @@ bb:
   %scalar0 = load volatile half, ptr addrspace(3) %arg2, align 2
 
   %neg.scalar0 = fsub half -0.0, %scalar0
-  %neg.scalar0.vec = insertelement <2 x half> undef, half %scalar0, i32 0
+  %neg.scalar0.vec = insertelement <2 x half> poison, half %scalar0, i32 0
   %scalar0.neg.scalar0 = insertelement <2 x half> %neg.scalar0.vec, half %neg.scalar0, i32 1
   %result = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %vec0, <2 x half> %vec1, <2 x half> %scalar0.neg.scalar0)
   store <2 x half> %result, ptr addrspace(1) %out, align 4
@@ -211,8 +211,8 @@ bb:
   %neg.scalar0 = fsub half -0.0, %scalar0
   %neg.scalar0.bc = bitcast half %neg.scalar0 to i16
 
-  %neg.scalar0.vec = insertelement <2 x i16> undef, i16 %neg.scalar0.bc, i32 0
-  %neg.scalar0.broadcast = shufflevector <2 x i16> %neg.scalar0.vec, <2 x i16> undef, <2 x i32> zeroinitializer
+  %neg.scalar0.vec = insertelement <2 x i16> poison, i16 %neg.scalar0.bc, i32 0
+  %neg.scalar0.broadcast = shufflevector <2 x i16> %neg.scalar0.vec, <2 x i16> poison, <2 x i32> zeroinitializer
 
   %result = add <2 x i16> %vec0, %neg.scalar0.broadcast
   store <2 x i16> %result, ptr addrspace(1) %out, align 4
@@ -250,7 +250,7 @@ bb:
   %scalar1 = load volatile half, ptr addrspace(3) %arg2.gep, align 2
 
   %neg.scalar1 = fsub half -0.0, %scalar1
-  %vec.ins0 = insertelement <2 x half> undef, half %scalar0, i32 0
+  %vec.ins0 = insertelement <2 x half> poison, half %scalar0, i32 0
   %vec2 = insertelement <2 x half> %vec.ins0, half %neg.scalar1, i32 1
   %result = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %vec0, <2 x half> %vec1, <2 x half> %vec2)
   store <2 x half> %result, ptr addrspace(1) %out, align 4
@@ -285,7 +285,7 @@ bb:
   %scalar0 = load volatile half, ptr addrspace(3) %arg2, align 2
   %scalar1 = load volatile half, ptr addrspace(3) %arg2.gep, align 2
 
-  %vec.ins0 = insertelement <2 x half> undef, half %scalar0, i32 0
+  %vec.ins0 = insertelement <2 x half> poison, half %scalar0, i32 0
   %vec2 = insertelement <2 x half> %vec.ins0, half %scalar1, i32 1
   %neg.vec2 = fsub <2 x half> <half -0.0, half -0.0>, %vec2
 
@@ -318,7 +318,7 @@ bb:
   %vec2 = load volatile <2 x half>, ptr addrspace(3) %lds.gep2, align 4
 
   %vec2.fneg = fsub <2 x half> <half -0.0, half -0.0>, %vec2
-  %vec2.fneg.elt1.broadcast = shufflevector <2 x half> %vec2.fneg, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+  %vec2.fneg.elt1.broadcast = shufflevector <2 x half> %vec2.fneg, <2 x half> poison, <2 x i32> <i32 1, i32 1>
 
   %result = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %vec0, <2 x half> %vec1, <2 x half> %vec2.fneg.elt1.broadcast)
   store <2 x half> %result, ptr addrspace(1) %out, align 4
@@ -377,7 +377,7 @@ bb:
   %vec0 = load volatile <2 x i16>, ptr addrspace(3) %lds, align 4
   %vec1 = load volatile <2 x i16>, ptr addrspace(3) %lds.gep1, align 4
 
-  %vec1.elt1.broadcast = shufflevector <2 x i16> %vec1, <2 x i16> undef, <2 x i32> <i32 1, i32 1>
+  %vec1.elt1.broadcast = shufflevector <2 x i16> %vec1, <2 x i16> poison, <2 x i32> <i32 1, i32 1>
   %result = add <2 x i16> %vec0, %vec1.elt1.broadcast
 
   store <2 x i16> %result, ptr addrspace(1) %out, align 4
@@ -407,7 +407,7 @@ bb:
   %vec1 = load volatile <2 x half>, ptr addrspace(3) %lds.gep1, align 4
   %vec2 = load volatile <2 x half>, ptr addrspace(3) %lds.gep2, align 4
 
-  %vec2.elt1.broadcast = shufflevector <2 x half> %vec2, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+  %vec2.elt1.broadcast = shufflevector <2 x half> %vec2, <2 x half> poison, <2 x i32> <i32 1, i32 1>
 
   %result = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %vec0, <2 x half> %vec1, <2 x half> %vec2.elt1.broadcast)
 
@@ -471,7 +471,7 @@ bb:
   %vec1 = load volatile <2 x half>, ptr addrspace(3) %lds.gep1, align 4
   %vec2 = load volatile <2 x half>, ptr addrspace(3) %lds.gep2, align 4
 
-  %vec2.swap = shufflevector <2 x half> %vec2, <2 x half> undef, <2 x i32> <i32 1, i32 0>
+  %vec2.swap = shufflevector <2 x half> %vec2, <2 x half> poison, <2 x i32> <i32 1, i32 0>
   %result = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %vec0, <2 x half> %vec1, <2 x half> %vec2.swap)
 
   store <2 x half> %result, ptr addrspace(1) %out, align 4
@@ -502,7 +502,7 @@ bb:
   %vec2 = load volatile <2 x half>, ptr addrspace(3) %lds.gep2, align 4
   %neg.vec2 = fsub <2 x half> <half -0.0, half -0.0>, %vec2
 
-  %neg.vec2.swap = shufflevector <2 x half> %neg.vec2, <2 x half> undef, <2 x i32> <i32 1, i32 0>
+  %neg.vec2.swap = shufflevector <2 x half> %neg.vec2, <2 x half> poison, <2 x i32> <i32 1, i32 0>
   %result = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %vec0, <2 x half> %vec1, <2 x half> %neg.vec2.swap)
 
   store <2 x half> %result, ptr addrspace(1) %out, align 4
@@ -678,7 +678,7 @@ bb:
   %f32 = load volatile float, ptr addrspace(3) undef, align 4
   %neg.f32 = fsub float -0.0, %f32
   %bc = bitcast float %neg.f32 to <2 x half>
-  %shuf = shufflevector <2 x half> %bc, <2 x half> undef, <2 x i32> <i32 1, i32 0>
+  %shuf = shufflevector <2 x half> %bc, <2 x half> poison, <2 x i32> <i32 1, i32 0>
   %result = fadd <2 x half> %vec0, %shuf
   store <2 x half> %result, ptr addrspace(1) %out, align 4
   ret void
@@ -710,7 +710,7 @@ bb:
   %hi = lshr i64 %i64, 16
   %elt1 = trunc i64 %hi to i16
 
-  %ins0 = insertelement <2 x i16> undef, i16 %elt1, i32 0
+  %ins0 = insertelement <2 x i16> poison, i16 %elt1, i32 0
   %ins1 = insertelement <2 x i16> %ins0, i16 %elt0, i32 1
   %result = add <2 x i16> %vec0, %ins1
   store <2 x i16> %result, ptr addrspace(1) %out, align 4
@@ -791,7 +791,7 @@ bb:
   %shl = shl i16 %scalar0, 1
   %shl.bc = bitcast i16 %shl to half
 
-  %insert0 = insertelement <2 x half> undef, half %shl.bc, i32 0
+  %insert0 = insertelement <2 x half> poison, half %shl.bc, i32 0
 
   %fadd = fadd <2 x half> %vec2, <half 2.0, half 2.0>
   %insert1 = shufflevector <2 x half> %fadd, <2 x half> %insert0, <2 x i32> <i32 1, i32 0>
diff --git a/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir b/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir
index ddeb45a48a6ee..afbbb05bfc3a5 100644
--- a/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir
+++ b/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir
@@ -563,3 +563,68 @@ body:             |
     SI_RETURN_TO_EPILOG %1
 
 ...
+
+---
+name:            fold_v_mov_b32_e32_literal_to_agpr
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: fold_v_mov_b32_e32_literal_to_agpr
+    ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 999, implicit $exec
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:agpr_32 = COPY killed [[V_MOV_B32_e32_]]
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG implicit [[COPY]]
+    %0:vgpr_32 = V_MOV_B32_e32 999, implicit $exec
+    %1:agpr_32 = COPY killed %0
+    SI_RETURN_TO_EPILOG implicit %1
+...
+
+---
+name:            fold_v_mov_b32_e32_inlineimm_to_agpr
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: fold_v_mov_b32_e32_inlineimm_to_agpr
+    ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 64, implicit $exec
+    ; GCN-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 64, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG implicit [[V_ACCVGPR_WRITE_B32_e64_]]
+    %0:vgpr_32 = V_MOV_B32_e32 64, implicit $exec
+    %1:agpr_32 = COPY killed %0
+    SI_RETURN_TO_EPILOG implicit %1
+...
+
+---
+name:            fold_av_mov_b32_imm_pseudo_inlineimm_to_vgpr
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: fold_av_mov_b32_imm_pseudo_inlineimm_to_vgpr
+    ; GCN: [[AV_MOV_:%[0-9]+]]:av_32 = AV_MOV_B32_IMM_PSEUDO 64, implicit $exec
+    ; GCN-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 64, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG implicit [[V_MOV_B32_e32_]]
+    %0:av_32 = AV_MOV_B32_IMM_PSEUDO 64, implicit $exec
+    %1:vgpr_32 = COPY killed %0
+    SI_RETURN_TO_EPILOG implicit %1
+...
+
+---
+name:            fold_av_mov_b32_imm_pseudo_inlineimm_to_agpr
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: fold_av_mov_b32_imm_pseudo_inlineimm_to_agpr
+    ; GCN: [[AV_MOV_:%[0-9]+]]:av_32 = AV_MOV_B32_IMM_PSEUDO 64, implicit $exec
+    ; GCN-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 64, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG implicit [[V_ACCVGPR_WRITE_B32_e64_]]
+    %0:av_32 = AV_MOV_B32_IMM_PSEUDO 64, implicit $exec
+    %1:agpr_32 = COPY killed %0
+    SI_RETURN_TO_EPILOG implicit %1
+...
+
+---
+name:            fold_av_mov_b32_imm_pseudo_inlineimm_to_av
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: fold_av_mov_b32_imm_pseudo_inlineimm_to_av
+    ; GCN: [[AV_MOV_:%[0-9]+]]:av_32 = AV_MOV_B32_IMM_PSEUDO 64, implicit $exec
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:av_32 = COPY killed [[AV_MOV_]]
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG implicit [[COPY]]
+    %0:av_32 = AV_MOV_B32_IMM_PSEUDO 64, implicit $exec
+    %1:av_32 = COPY killed %0
+    SI_RETURN_TO_EPILOG implicit %1
+...
diff --git a/llvm/test/CodeGen/AMDGPU/pei-scavenge-sgpr-carry-out.mir b/llvm/test/CodeGen/AMDGPU/pei-scavenge-sgpr-carry-out.mir
index ba6524caf668d..05cbd4c2a010d 100644
--- a/llvm/test/CodeGen/AMDGPU/pei-scavenge-sgpr-carry-out.mir
+++ b/llvm/test/CodeGen/AMDGPU/pei-scavenge-sgpr-carry-out.mir
@@ -27,39 +27,25 @@ body:             |
     liveins: $vgpr1
 
     ; CHECK-LABEL: name: scavenge_sgpr_pei_no_sgprs
-    ; CHECK: liveins: $vgpr1, $vgpr2
+    ; CHECK: liveins: $sgpr40, $sgpr41, $vgpr1
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: $sgpr4 = COPY $sgpr33
+    ; CHECK-NEXT: $sgpr40 = frame-setup COPY $sgpr33
     ; CHECK-NEXT: $sgpr33 = frame-setup S_ADD_I32 $sgpr32, 524224, implicit-def $scc
     ; CHECK-NEXT: $sgpr33 = frame-setup S_AND_B32 killed $sgpr33, 4294443008, implicit-def dead $scc
-    ; CHECK-NEXT: $sgpr6_sgpr7 = S_XOR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-    ; CHECK-NEXT: $sgpr5 = S_ADD_I32 $sgpr33, 1048832, implicit-def dead $scc
-    ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, killed $sgpr5, 0, 0, 0, implicit $exec :: (store (s32) into %stack.3, addrspace 5)
-    ; CHECK-NEXT: $exec = S_MOV_B64 killed $sgpr6_sgpr7
-    ; CHECK-NEXT: $vgpr2 = SI_SPILL_S32_TO_VGPR $sgpr4, 0, undef $vgpr2
-    ; CHECK-NEXT: $vgpr2 = SI_SPILL_S32_TO_VGPR $sgpr34, 1, undef $vgpr2
+    ; CHECK-NEXT: $sgpr41 = frame-setup COPY $sgpr34
     ; CHECK-NEXT: $sgpr34 = frame-setup COPY $sgpr32
     ; CHECK-NEXT: $sgpr32 = frame-setup S_ADD_I32 $sgpr32, 2097152, implicit-def dead $scc
     ; CHECK-NEXT: S_NOP 0, implicit-def $sgpr4, implicit-def $sgpr5, implicit-def $sgpr6, implicit-def $sgpr7, implicit-def $sgpr8, implicit-def $sgpr9, implicit-def $sgpr10, implicit-def $sgpr11, implicit-def $sgpr12, implicit-def $sgpr13, implicit-def $sgpr14, implicit-def $sgpr15, implicit-def $sgpr16, implicit-def $sgpr17, implicit-def $sgpr18, implicit-def $sgpr19, implicit-def $sgpr20, implicit-def $sgpr21, implicit-def $sgpr22, implicit-def $sgpr23, implicit-def $sgpr24, implicit-def $sgpr25, implicit-def $sgpr26, implicit-def $sgpr27, implicit-def $sgpr28, implicit-def $sgpr29, implicit-def $sgpr30, implicit-def $sgpr31, implicit-def $vcc
-    ; CHECK-NEXT: $sgpr33 = S_LSHR_B32 $sgpr33, 6, implicit-def $scc
-    ; CHECK-NEXT: $sgpr33 = S_ADD_I32 killed $sgpr33, 8192, implicit-def $scc
-    ; CHECK-NEXT: $vgpr0 = COPY killed $sgpr33
-    ; CHECK-NEXT: $sgpr33 = S_ADD_I32 killed $sgpr33, -8192, implicit-def $scc
-    ; CHECK-NEXT: $sgpr33 = S_LSHL_B32 $sgpr33, 6, implicit-def $scc
-    ; CHECK-NEXT: $sgpr33 = S_LSHR_B32 $sgpr33, 6, implicit-def $scc
-    ; CHECK-NEXT: $sgpr33 = S_ADD_I32 killed $sgpr33, 16384, implicit-def $scc
-    ; CHECK-NEXT: $vgpr3 = COPY killed $sgpr33
-    ; CHECK-NEXT: $sgpr33 = S_ADD_I32 killed $sgpr33, -16384, implicit-def $scc
-    ; CHECK-NEXT: $sgpr33 = S_LSHL_B32 $sgpr33, 6, implicit-def $scc
-    ; CHECK-NEXT: $vgpr0 = V_OR_B32_e32 killed $vgpr3, $vgpr1, implicit $exec, implicit $sgpr4, implicit $sgpr5, implicit $sgpr6, implicit $sgpr7, implicit $sgpr8, implicit $sgpr9, implicit $sgpr10, implicit $sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr16, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $sgpr28, implicit $sgpr29, implicit $sgpr30, implicit $sgpr31
+    ; CHECK-NEXT: $vgpr0 = V_LSHRREV_B32_e64 6, $sgpr33, implicit $exec
+    ; CHECK-NEXT: $sgpr42 = S_MOV_B32 8192
+    ; CHECK-NEXT: $vgpr0, dead $sgpr42_sgpr43 = V_ADD_CO_U32_e64 killed $sgpr42, killed $vgpr0, 0, implicit $exec
+    ; CHECK-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr33, implicit $exec
+    ; CHECK-NEXT: $sgpr42 = S_MOV_B32 16384
+    ; CHECK-NEXT: $vgpr2, dead $sgpr42_sgpr43 = V_ADD_CO_U32_e64 killed $sgpr42, killed $vgpr2, 0, implicit $exec
+    ; CHECK-NEXT: $vgpr0 = V_OR_B32_e32 killed $vgpr2, $vgpr1, implicit $exec, implicit $sgpr4, implicit $sgpr5, implicit $sgpr6, implicit $sgpr7, implicit $sgpr8, implicit $sgpr9, implicit $sgpr10, implicit $sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr16, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $sgpr28, implicit $sgpr29, implicit $sgpr30, implicit $sgpr31
     ; CHECK-NEXT: $sgpr32 = frame-destroy COPY $sgpr34
-    ; CHECK-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR $vgpr2, 0
-    ; CHECK-NEXT: $sgpr34 = SI_RESTORE_S32_FROM_VGPR $vgpr2, 1
-    ; CHECK-NEXT: $sgpr6_sgpr7 = S_XOR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-    ; CHECK-NEXT: $sgpr5 = S_ADD_I32 $sgpr33, 1048832, implicit-def dead $scc
-    ; CHECK-NEXT: $vgpr2 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, killed $sgpr5, 0, 0, 0, implicit $exec :: (load (s32) from %stack.3, addrspace 5)
-    ; CHECK-NEXT: $exec = S_MOV_B64 killed $sgpr6_sgpr7
-    ; CHECK-NEXT: $sgpr33 = COPY $sgpr4
+    ; CHECK-NEXT: $sgpr34 = frame-destroy COPY $sgpr41
+    ; CHECK-NEXT: $sgpr33 = frame-destroy COPY $sgpr40
     ; CHECK-NEXT: S_ENDPGM 0, implicit $vcc
     S_NOP 0, implicit-def $sgpr4, implicit-def $sgpr5, implicit-def $sgpr6, implicit-def $sgpr7, implicit-def $sgpr8, implicit-def $sgpr9, implicit-def $sgpr10, implicit-def $sgpr11, implicit-def $sgpr12, implicit-def $sgpr13, implicit-def $sgpr14, implicit-def $sgpr15, implicit-def $sgpr16, implicit-def $sgpr17, implicit-def $sgpr18, implicit-def $sgpr19, implicit-def $sgpr20, implicit-def $sgpr21, implicit-def $sgpr22, implicit-def $sgpr23, implicit-def $sgpr24, implicit-def $sgpr25, implicit-def $sgpr26, implicit-def $sgpr27, implicit-def $sgpr28, implicit-def $sgpr29, implicit-def $sgpr30, implicit-def $sgpr31, implicit-def $vcc
     $vgpr0 = V_MOV_B32_e32 %stack.0, implicit $exec
@@ -89,36 +75,24 @@ body:             |
     liveins: $vgpr1
 
     ; CHECK-LABEL: name: scavenge_sgpr_pei_one_sgpr
-    ; CHECK: liveins: $sgpr29, $vgpr1, $vgpr2
+    ; CHECK: liveins: $sgpr29, $sgpr40, $vgpr1
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: $sgpr29 = frame-setup COPY $sgpr33
     ; CHECK-NEXT: $sgpr33 = frame-setup S_ADD_I32 $sgpr32, 524224, implicit-def $scc
     ; CHECK-NEXT: $sgpr33 = frame-setup S_AND_B32 killed $sgpr33, 4294443008, implicit-def dead $scc
-    ; CHECK-NEXT: $sgpr4_sgpr5 = S_XOR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-    ; CHECK-NEXT: $sgpr6 = S_ADD_I32 $sgpr33, 1048832, implicit-def dead $scc
-    ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, killed $sgpr6, 0, 0, 0, implicit $exec :: (store (s32) into %stack.3, addrspace 5)
-    ; CHECK-NEXT: $exec = S_MOV_B64 killed $sgpr4_sgpr5
-    ; CHECK-NEXT: $vgpr2 = SI_SPILL_S32_TO_VGPR $sgpr34, 0, undef $vgpr2
+    ; CHECK-NEXT: $sgpr40 = frame-setup COPY $sgpr34
     ; CHECK-NEXT: $sgpr34 = frame-setup COPY $sgpr32
     ; CHECK-NEXT: $sgpr32 = frame-setup S_ADD_I32 $sgpr32, 2097152, implicit-def dead $scc
     ; CHECK-NEXT: S_NOP 0, implicit-def $sgpr4, implicit-def $sgpr5, implicit-def $sgpr6, implicit-def $sgpr7, implicit-def $sgpr8, implicit-def $sgpr9, implicit-def $sgpr10, implicit-def $sgpr11, implicit-def $sgpr12, implicit-def $sgpr13, implicit-def $sgpr14, implicit-def $sgpr15, implicit-def $sgpr16, implicit-def $sgpr17, implicit-def $sgpr18, implicit-def $sgpr19, implicit-def $sgpr20, implicit-def $sgpr21, implicit-def $sgpr22, implicit-def $sgpr23, implicit-def $sgpr24, implicit-def $sgpr25, implicit-def $sgpr26, implicit-def $sgpr27, implicit-def $sgpr28, implicit-def $sgpr30, implicit-def $sgpr31, implicit-def $vcc
-    ; CHECK-NEXT: $sgpr33 = S_LSHR_B32 $sgpr33, 6, implicit-def $scc
-    ; CHECK-NEXT: $sgpr33 = S_ADD_I32 killed $sgpr33, 8192, implicit-def $scc
-    ; CHECK-NEXT: $vgpr0 = COPY killed $sgpr33
-    ; CHECK-NEXT: $sgpr33 = S_ADD_I32 killed $sgpr33, -8192, implicit-def $scc
-    ; CHECK-NEXT: $sgpr33 = S_LSHL_B32 $sgpr33, 6, implicit-def $scc
-    ; CHECK-NEXT: $sgpr33 = S_LSHR_B32 $sgpr33, 6, implicit-def $scc
-    ; CHECK-NEXT: $sgpr33 = S_ADD_I32 killed $sgpr33, 16384, implicit-def $scc
-    ; CHECK-NEXT: $vgpr3 = COPY killed $sgpr33
-    ; CHECK-NEXT: $sgpr33 = S_ADD_I32 killed $sgpr33, -16384, implicit-def $scc
-    ; CHECK-NEXT: $sgpr33 = S_LSHL_B32 $sgpr33, 6, implicit-def $scc
-    ; CHECK-NEXT: $vgpr0 = V_OR_B32_e32 killed $vgpr3, $vgpr1, implicit $exec, implicit $sgpr4, implicit $sgpr5, implicit $sgpr6, implicit $sgpr7, implicit $sgpr8, implicit $sgpr9, implicit $sgpr10, implicit $sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr16, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $sgpr28, implicit $sgpr31
+    ; CHECK-NEXT: $vgpr0 = V_LSHRREV_B32_e64 6, $sgpr33, implicit $exec
+    ; CHECK-NEXT: $sgpr42 = S_MOV_B32 8192
+    ; CHECK-NEXT: $vgpr0, dead $sgpr42_sgpr43 = V_ADD_CO_U32_e64 killed $sgpr42, killed $vgpr0, 0, implicit $exec
+    ; CHECK-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr33, implicit $exec
+    ; CHECK-NEXT: $sgpr42 = S_MOV_B32 16384
+    ; CHECK-NEXT: $vgpr2, dead $sgpr42_sgpr43 = V_ADD_CO_U32_e64 killed $sgpr42, killed $vgpr2, 0, implicit $exec
+    ; CHECK-NEXT: $vgpr0 = V_OR_B32_e32 killed $vgpr2, $vgpr1, implicit $exec, implicit $sgpr4, implicit $sgpr5, implicit $sgpr6, implicit $sgpr7, implicit $sgpr8, implicit $sgpr9, implicit $sgpr10, implicit $sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr16, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $sgpr28, implicit $sgpr31
     ; CHECK-NEXT: $sgpr32 = frame-destroy COPY $sgpr34
-    ; CHECK-NEXT: $sgpr34 = SI_RESTORE_S32_FROM_VGPR $vgpr2, 0
-    ; CHECK-NEXT: $sgpr4_sgpr5 = S_XOR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-    ; CHECK-NEXT: $sgpr6 = S_ADD_I32 $sgpr33, 1048832, implicit-def dead $scc
-    ; CHECK-NEXT: $vgpr2 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, killed $sgpr6, 0, 0, 0, implicit $exec :: (load (s32) from %stack.3, addrspace 5)
-    ; CHECK-NEXT: $exec = S_MOV_B64 killed $sgpr4_sgpr5
+    ; CHECK-NEXT: $sgpr34 = frame-destroy COPY $sgpr40
     ; CHECK-NEXT: $sgpr33 = frame-destroy COPY $sgpr29
     ; CHECK-NEXT: S_ENDPGM 0, implicit $vcc
     S_NOP 0, implicit-def $sgpr4, implicit-def $sgpr5, implicit-def $sgpr6, implicit-def $sgpr7, implicit-def $sgpr8, implicit-def $sgpr9, implicit-def $sgpr10, implicit-def $sgpr11, implicit-def $sgpr12, implicit-def $sgpr13, implicit-def $sgpr14, implicit-def $sgpr15, implicit-def $sgpr16, implicit-def $sgpr17, implicit-def $sgpr18, implicit-def $sgpr19, implicit-def $sgpr20, implicit-def $sgpr21, implicit-def $sgpr22, implicit-def $sgpr23, implicit-def $sgpr24, implicit-def $sgpr25, implicit-def $sgpr26, implicit-def $sgpr27, implicit-def $sgpr28, implicit-def $sgpr30, implicit-def $sgpr31, implicit-def $vcc
@@ -158,16 +132,12 @@ body:             |
     ; CHECK-NEXT: $sgpr34 = frame-setup COPY $sgpr32
     ; CHECK-NEXT: $sgpr32 = frame-setup S_ADD_I32 $sgpr32, 2097152, implicit-def dead $scc
     ; CHECK-NEXT: S_NOP 0, implicit-def $sgpr4, implicit-def $sgpr5, implicit-def $sgpr6, implicit-def $sgpr7, implicit-def $sgpr8, implicit-def $sgpr9, implicit-def $sgpr10, implicit-def $sgpr11, implicit-def $sgpr12, implicit-def $sgpr13, implicit-def $sgpr14, implicit-def $sgpr15, implicit-def $sgpr16, implicit-def $sgpr17, implicit-def $sgpr18, implicit-def $sgpr19, implicit-def $sgpr20, implicit-def $sgpr21, implicit-def $sgpr22, implicit-def $sgpr23, implicit-def $sgpr24, implicit-def $sgpr25, implicit-def $sgpr26, implicit-def $sgpr27, implicit-def $sgpr30, implicit-def $sgpr31, implicit-def $vcc
-    ; CHECK-NEXT: $sgpr33 = S_LSHR_B32 $sgpr33, 6, implicit-def $scc
-    ; CHECK-NEXT: $sgpr33 = S_ADD_I32 killed $sgpr33, 8192, implicit-def $scc
-    ; CHECK-NEXT: $vgpr0 = COPY killed $sgpr33
-    ; CHECK-NEXT: $sgpr33 = S_ADD_I32 killed $sgpr33, -8192, implicit-def $scc
-    ; CHECK-NEXT: $sgpr33 = S_LSHL_B32 $sgpr33, 6, implicit-def $scc
-    ; CHECK-NEXT: $sgpr33 = S_LSHR_B32 $sgpr33, 6, implicit-def $scc
-    ; CHECK-NEXT: $sgpr33 = S_ADD_I32 killed $sgpr33, 16384, implicit-def $scc
-    ; CHECK-NEXT: $vgpr2 = COPY killed $sgpr33
-    ; CHECK-NEXT: $sgpr33 = S_ADD_I32 killed $sgpr33, -16384, implicit-def $scc
-    ; CHECK-NEXT: $sgpr33 = S_LSHL_B32 $sgpr33, 6, implicit-def $scc
+    ; CHECK-NEXT: $vgpr0 = V_LSHRREV_B32_e64 6, $sgpr33, implicit $exec
+    ; CHECK-NEXT: $sgpr40 = S_MOV_B32 8192
+    ; CHECK-NEXT: $vgpr0, dead $sgpr40_sgpr41 = V_ADD_CO_U32_e64 killed $sgpr40, killed $vgpr0, 0, implicit $exec
+    ; CHECK-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr33, implicit $exec
+    ; CHECK-NEXT: $sgpr40 = S_MOV_B32 16384
+    ; CHECK-NEXT: $vgpr2, dead $sgpr40_sgpr41 = V_ADD_CO_U32_e64 killed $sgpr40, killed $vgpr2, 0, implicit $exec
     ; CHECK-NEXT: $vgpr0 = V_OR_B32_e32 killed $vgpr2, $vgpr1, implicit $exec, implicit $sgpr4, implicit $sgpr5, implicit $sgpr6, implicit $sgpr7, implicit $sgpr8, implicit $sgpr9, implicit $sgpr10, implicit $sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr16, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $sgpr31
     ; CHECK-NEXT: $sgpr32 = frame-destroy COPY $sgpr34
     ; CHECK-NEXT: $sgpr34 = frame-destroy COPY $sgpr29
diff --git a/llvm/test/CodeGen/AMDGPU/pei-scavenge-sgpr-gfx9.mir b/llvm/test/CodeGen/AMDGPU/pei-scavenge-sgpr-gfx9.mir
index 162d12f651d4a..4f1c9a20fddc3 100644
--- a/llvm/test/CodeGen/AMDGPU/pei-scavenge-sgpr-gfx9.mir
+++ b/llvm/test/CodeGen/AMDGPU/pei-scavenge-sgpr-gfx9.mir
@@ -23,64 +23,42 @@ body:             |
     liveins: $vgpr1
 
     ; MUBUF-LABEL: name: scavenge_sgpr_pei_no_sgprs
-    ; MUBUF: liveins: $vgpr1, $vgpr2
+    ; MUBUF: liveins: $sgpr40, $sgpr41, $vgpr1
     ; MUBUF-NEXT: {{  $}}
-    ; MUBUF-NEXT: $sgpr4 = COPY $sgpr33
+    ; MUBUF-NEXT: $sgpr40 = frame-setup COPY $sgpr33
     ; MUBUF-NEXT: $sgpr33 = frame-setup S_ADD_I32 $sgpr32, 524224, implicit-def $scc
     ; MUBUF-NEXT: $sgpr33 = frame-setup S_AND_B32 killed $sgpr33, 4294443008, implicit-def dead $scc
-    ; MUBUF-NEXT: $sgpr6_sgpr7 = S_XOR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-    ; MUBUF-NEXT: $sgpr5 = S_ADD_I32 $sgpr33, 1048832, implicit-def dead $scc
-    ; MUBUF-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, killed $sgpr5, 0, 0, 0, implicit $exec :: (store (s32) into %stack.3, addrspace 5)
-    ; MUBUF-NEXT: $exec = S_MOV_B64 killed $sgpr6_sgpr7
-    ; MUBUF-NEXT: $vgpr2 = SI_SPILL_S32_TO_VGPR $sgpr4, 0, undef $vgpr2
-    ; MUBUF-NEXT: $vgpr2 = SI_SPILL_S32_TO_VGPR $sgpr34, 1, undef $vgpr2
+    ; MUBUF-NEXT: $sgpr41 = frame-setup COPY $sgpr34
     ; MUBUF-NEXT: $sgpr34 = frame-setup COPY $sgpr32
     ; MUBUF-NEXT: $sgpr32 = frame-setup S_ADD_I32 $sgpr32, 2097152, implicit-def dead $scc
     ; MUBUF-NEXT: S_NOP 0, implicit-def $sgpr4, implicit-def $sgpr5, implicit-def $sgpr6, implicit-def $sgpr7, implicit-def $sgpr8, implicit-def $sgpr9, implicit-def $sgpr10, implicit-def $sgpr11, implicit-def $sgpr12, implicit-def $sgpr13, implicit-def $sgpr14, implicit-def $sgpr15, implicit-def $sgpr16, implicit-def $sgpr17, implicit-def $sgpr18, implicit-def $sgpr19, implicit-def $sgpr20, implicit-def $sgpr21, implicit-def $sgpr22, implicit-def $sgpr23, implicit-def $sgpr24, implicit-def $sgpr25, implicit-def $sgpr26, implicit-def $sgpr27, implicit-def $sgpr28, implicit-def $sgpr29, implicit-def $sgpr30, implicit-def $sgpr31, implicit-def $vcc
     ; MUBUF-NEXT: $vgpr0 = V_LSHRREV_B32_e64 6, $sgpr33, implicit $exec
     ; MUBUF-NEXT: $vgpr0 = V_ADD_U32_e32 8192, killed $vgpr0, implicit $exec
-    ; MUBUF-NEXT: $vgpr3 = V_LSHRREV_B32_e64 6, $sgpr33, implicit $exec
-    ; MUBUF-NEXT: $vgpr3 = V_ADD_U32_e32 16384, killed $vgpr3, implicit $exec
-    ; MUBUF-NEXT: $vgpr0 = V_OR_B32_e32 killed $vgpr3, $vgpr1, implicit $exec, implicit $sgpr4, implicit $sgpr5, implicit $sgpr6, implicit $sgpr7, implicit $sgpr8, implicit $sgpr9, implicit $sgpr10, implicit $sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr16, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $sgpr28, implicit $sgpr29, implicit $sgpr30, implicit $sgpr31
+    ; MUBUF-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr33, implicit $exec
+    ; MUBUF-NEXT: $vgpr2 = V_ADD_U32_e32 16384, killed $vgpr2, implicit $exec
+    ; MUBUF-NEXT: $vgpr0 = V_OR_B32_e32 killed $vgpr2, $vgpr1, implicit $exec, implicit $sgpr4, implicit $sgpr5, implicit $sgpr6, implicit $sgpr7, implicit $sgpr8, implicit $sgpr9, implicit $sgpr10, implicit $sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr16, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $sgpr28, implicit $sgpr29, implicit $sgpr30, implicit $sgpr31
     ; MUBUF-NEXT: $sgpr32 = frame-destroy COPY $sgpr34
-    ; MUBUF-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR $vgpr2, 0
-    ; MUBUF-NEXT: $sgpr34 = SI_RESTORE_S32_FROM_VGPR $vgpr2, 1
-    ; MUBUF-NEXT: $sgpr6_sgpr7 = S_XOR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-    ; MUBUF-NEXT: $sgpr5 = S_ADD_I32 $sgpr33, 1048832, implicit-def dead $scc
-    ; MUBUF-NEXT: $vgpr2 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, killed $sgpr5, 0, 0, 0, implicit $exec :: (load (s32) from %stack.3, addrspace 5)
-    ; MUBUF-NEXT: $exec = S_MOV_B64 killed $sgpr6_sgpr7
-    ; MUBUF-NEXT: $sgpr33 = COPY $sgpr4
+    ; MUBUF-NEXT: $sgpr34 = frame-destroy COPY $sgpr41
+    ; MUBUF-NEXT: $sgpr33 = frame-destroy COPY $sgpr40
     ; MUBUF-NEXT: S_ENDPGM 0, implicit $vcc
     ;
     ; FLATSCR-LABEL: name: scavenge_sgpr_pei_no_sgprs
-    ; FLATSCR: liveins: $vgpr1, $vgpr2
+    ; FLATSCR: liveins: $sgpr40, $sgpr41, $vgpr1
     ; FLATSCR-NEXT: {{  $}}
-    ; FLATSCR-NEXT: $sgpr4 = COPY $sgpr33
+    ; FLATSCR-NEXT: $sgpr40 = frame-setup COPY $sgpr33
     ; FLATSCR-NEXT: $sgpr33 = frame-setup S_ADD_I32 $sgpr32, 8191, implicit-def $scc
     ; FLATSCR-NEXT: $sgpr33 = frame-setup S_AND_B32 killed $sgpr33, 4294959104, implicit-def dead $scc
-    ; FLATSCR-NEXT: $sgpr6_sgpr7 = S_XOR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-    ; FLATSCR-NEXT: $sgpr5 = S_ADD_I32 $sgpr33, 16388, implicit-def dead $scc
-    ; FLATSCR-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr2, killed $sgpr5, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %stack.3, addrspace 5)
-    ; FLATSCR-NEXT: $exec = S_MOV_B64 killed $sgpr6_sgpr7
-    ; FLATSCR-NEXT: $vgpr2 = SI_SPILL_S32_TO_VGPR $sgpr4, 0, undef $vgpr2
-    ; FLATSCR-NEXT: $vgpr2 = SI_SPILL_S32_TO_VGPR $sgpr34, 1, undef $vgpr2
+    ; FLATSCR-NEXT: $sgpr41 = frame-setup COPY $sgpr34
     ; FLATSCR-NEXT: $sgpr34 = frame-setup COPY $sgpr32
     ; FLATSCR-NEXT: $sgpr32 = frame-setup S_ADD_I32 $sgpr32, 32768, implicit-def dead $scc
     ; FLATSCR-NEXT: S_NOP 0, implicit-def $sgpr4, implicit-def $sgpr5, implicit-def $sgpr6, implicit-def $sgpr7, implicit-def $sgpr8, implicit-def $sgpr9, implicit-def $sgpr10, implicit-def $sgpr11, implicit-def $sgpr12, implicit-def $sgpr13, implicit-def $sgpr14, implicit-def $sgpr15, implicit-def $sgpr16, implicit-def $sgpr17, implicit-def $sgpr18, implicit-def $sgpr19, implicit-def $sgpr20, implicit-def $sgpr21, implicit-def $sgpr22, implicit-def $sgpr23, implicit-def $sgpr24, implicit-def $sgpr25, implicit-def $sgpr26, implicit-def $sgpr27, implicit-def $sgpr28, implicit-def $sgpr29, implicit-def $sgpr30, implicit-def $sgpr31, implicit-def $vcc
-    ; FLATSCR-NEXT: $sgpr33 = S_ADD_I32 $sgpr33, 8192, implicit-def $scc
-    ; FLATSCR-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr33, implicit $exec
-    ; FLATSCR-NEXT: $sgpr33 = S_ADD_I32 $sgpr33, -8192, implicit-def $scc
-    ; FLATSCR-NEXT: $sgpr33 = S_ADD_I32 $sgpr33, 16384, implicit-def $scc
-    ; FLATSCR-NEXT: $vgpr0 = V_OR_B32_e32 $sgpr33, $vgpr1, implicit $exec, implicit $sgpr4, implicit $sgpr5, implicit $sgpr6, implicit $sgpr7, implicit $sgpr8, implicit $sgpr9, implicit $sgpr10, implicit $sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr16, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $sgpr28, implicit $sgpr29, implicit $sgpr30, implicit $sgpr31
-    ; FLATSCR-NEXT: $sgpr33 = S_ADD_I32 $sgpr33, -16384, implicit-def $scc
+    ; FLATSCR-NEXT: $sgpr42 = S_ADD_I32 $sgpr33, 8192, implicit-def $scc
+    ; FLATSCR-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr42, implicit $exec
+    ; FLATSCR-NEXT: $sgpr42 = S_ADD_I32 $sgpr33, 16384, implicit-def $scc
+    ; FLATSCR-NEXT: $vgpr0 = V_OR_B32_e32 killed $sgpr42, $vgpr1, implicit $exec, implicit $sgpr4, implicit $sgpr5, implicit $sgpr6, implicit $sgpr7, implicit $sgpr8, implicit $sgpr9, implicit $sgpr10, implicit $sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr16, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $sgpr28, implicit $sgpr29, implicit $sgpr30, implicit $sgpr31
     ; FLATSCR-NEXT: $sgpr32 = frame-destroy COPY $sgpr34
-    ; FLATSCR-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR $vgpr2, 0
-    ; FLATSCR-NEXT: $sgpr34 = SI_RESTORE_S32_FROM_VGPR $vgpr2, 1
-    ; FLATSCR-NEXT: $sgpr6_sgpr7 = S_XOR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-    ; FLATSCR-NEXT: $sgpr5 = S_ADD_I32 $sgpr33, 16388, implicit-def dead $scc
-    ; FLATSCR-NEXT: $vgpr2 = SCRATCH_LOAD_DWORD_SADDR killed $sgpr5, 0, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %stack.3, addrspace 5)
-    ; FLATSCR-NEXT: $exec = S_MOV_B64 killed $sgpr6_sgpr7
-    ; FLATSCR-NEXT: $sgpr33 = COPY $sgpr4
+    ; FLATSCR-NEXT: $sgpr34 = frame-destroy COPY $sgpr41
+    ; FLATSCR-NEXT: $sgpr33 = frame-destroy COPY $sgpr40
     ; FLATSCR-NEXT: S_ENDPGM 0, implicit $vcc
     S_NOP 0, implicit-def $sgpr4, implicit-def $sgpr5, implicit-def $sgpr6, implicit-def $sgpr7, implicit-def $sgpr8, implicit-def $sgpr9, implicit-def $sgpr10, implicit-def $sgpr11, implicit-def $sgpr12, implicit-def $sgpr13, implicit-def $sgpr14, implicit-def $sgpr15, implicit-def $sgpr16, implicit-def $sgpr17, implicit-def $sgpr18, implicit-def $sgpr19, implicit-def $sgpr20, implicit-def $sgpr21, implicit-def $sgpr22, implicit-def $sgpr23, implicit-def $sgpr24, implicit-def $sgpr25, implicit-def $sgpr26, implicit-def $sgpr27, implicit-def $sgpr28, implicit-def $sgpr29, implicit-def $sgpr30, implicit-def $sgpr31, implicit-def $vcc
     $vgpr0 = V_MOV_B32_e32 %stack.0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/pei-scavenge-sgpr.mir b/llvm/test/CodeGen/AMDGPU/pei-scavenge-sgpr.mir
index a4f936a4d705c..480859a09a347 100644
--- a/llvm/test/CodeGen/AMDGPU/pei-scavenge-sgpr.mir
+++ b/llvm/test/CodeGen/AMDGPU/pei-scavenge-sgpr.mir
@@ -22,34 +22,22 @@ body:             |
     liveins: $vgpr1
 
     ; CHECK-LABEL: name: scavenge_sgpr_pei
-    ; CHECK: liveins: $vgpr1, $vgpr2
+    ; CHECK: liveins: $sgpr40, $sgpr41, $vgpr1
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: $sgpr4 = COPY $sgpr33
+    ; CHECK-NEXT: $sgpr40 = frame-setup COPY $sgpr33
     ; CHECK-NEXT: $sgpr33 = frame-setup S_ADD_I32 $sgpr32, 262080, implicit-def $scc
     ; CHECK-NEXT: $sgpr33 = frame-setup S_AND_B32 killed $sgpr33, 4294705152, implicit-def dead $scc
-    ; CHECK-NEXT: $sgpr6_sgpr7 = S_XOR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-    ; CHECK-NEXT: $sgpr5 = S_ADD_I32 $sgpr33, 262400, implicit-def dead $scc
-    ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, killed $sgpr5, 0, 0, 0, implicit $exec :: (store (s32) into %stack.2, addrspace 5)
-    ; CHECK-NEXT: $exec = S_MOV_B64 killed $sgpr6_sgpr7
-    ; CHECK-NEXT: $vgpr2 = SI_SPILL_S32_TO_VGPR $sgpr4, 0, undef $vgpr2
-    ; CHECK-NEXT: $vgpr2 = SI_SPILL_S32_TO_VGPR $sgpr34, 1, undef $vgpr2
+    ; CHECK-NEXT: $sgpr41 = frame-setup COPY $sgpr34
     ; CHECK-NEXT: $sgpr34 = frame-setup COPY $sgpr32
     ; CHECK-NEXT: $sgpr32 = frame-setup S_ADD_I32 $sgpr32, 786432, implicit-def dead $scc
     ; CHECK-NEXT: S_NOP 0, implicit-def $sgpr4, implicit-def $sgpr5, implicit-def $sgpr6, implicit-def $sgpr7, implicit-def $sgpr8, implicit-def $sgpr9, implicit-def $sgpr10, implicit-def $sgpr11, implicit-def $sgpr12, implicit-def $sgpr13, implicit-def $sgpr14, implicit-def $sgpr15, implicit-def $sgpr16, implicit-def $sgpr17, implicit-def $sgpr18, implicit-def $sgpr19, implicit-def $sgpr20, implicit-def $sgpr21, implicit-def $sgpr22, implicit-def $sgpr23, implicit-def $sgpr24, implicit-def $sgpr25, implicit-def $sgpr26, implicit-def $sgpr27, implicit-def $sgpr28, implicit-def $sgpr29, implicit-def $sgpr30, implicit-def $sgpr31, implicit-def $vcc
-    ; CHECK-NEXT: $sgpr33 = S_LSHR_B32 $sgpr33, 6, implicit-def $scc
-    ; CHECK-NEXT: $sgpr33 = S_ADD_I32 killed $sgpr33, 4096, implicit-def $scc
-    ; CHECK-NEXT: $vgpr3 = COPY killed $sgpr33
-    ; CHECK-NEXT: $sgpr33 = S_ADD_I32 killed $sgpr33, -4096, implicit-def $scc
-    ; CHECK-NEXT: $sgpr33 = S_LSHL_B32 $sgpr33, 6, implicit-def $scc
-    ; CHECK-NEXT: $vgpr0 = V_OR_B32_e32 killed $vgpr3, $vgpr1, implicit $exec, implicit $sgpr4, implicit $sgpr5, implicit $sgpr6, implicit $sgpr7, implicit $sgpr8, implicit $sgpr9, implicit $sgpr10, implicit $sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr16, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $sgpr28, implicit $sgpr29, implicit $sgpr30, implicit $sgpr31
+    ; CHECK-NEXT: $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr33, implicit $exec
+    ; CHECK-NEXT: $sgpr42 = S_MOV_B32 4096
+    ; CHECK-NEXT: $vgpr2, dead $sgpr42_sgpr43 = V_ADD_CO_U32_e64 killed $sgpr42, killed $vgpr2, 0, implicit $exec
+    ; CHECK-NEXT: $vgpr0 = V_OR_B32_e32 killed $vgpr2, $vgpr1, implicit $exec, implicit $sgpr4, implicit $sgpr5, implicit $sgpr6, implicit $sgpr7, implicit $sgpr8, implicit $sgpr9, implicit $sgpr10, implicit $sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr16, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $sgpr28, implicit $sgpr29, implicit $sgpr30, implicit $sgpr31
     ; CHECK-NEXT: $sgpr32 = frame-destroy COPY $sgpr34
-    ; CHECK-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR $vgpr2, 0
-    ; CHECK-NEXT: $sgpr34 = SI_RESTORE_S32_FROM_VGPR $vgpr2, 1
-    ; CHECK-NEXT: $sgpr6_sgpr7 = S_XOR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-    ; CHECK-NEXT: $sgpr5 = S_ADD_I32 $sgpr33, 262400, implicit-def dead $scc
-    ; CHECK-NEXT: $vgpr2 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, killed $sgpr5, 0, 0, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
-    ; CHECK-NEXT: $exec = S_MOV_B64 killed $sgpr6_sgpr7
-    ; CHECK-NEXT: $sgpr33 = COPY $sgpr4
+    ; CHECK-NEXT: $sgpr34 = frame-destroy COPY $sgpr41
+    ; CHECK-NEXT: $sgpr33 = frame-destroy COPY $sgpr40
     ; CHECK-NEXT: S_ENDPGM 0, implicit $vcc
     S_NOP 0, implicit-def $sgpr4, implicit-def $sgpr5, implicit-def $sgpr6, implicit-def $sgpr7, implicit-def $sgpr8, implicit-def $sgpr9, implicit-def $sgpr10, implicit-def $sgpr11, implicit-def $sgpr12, implicit-def $sgpr13, implicit-def $sgpr14, implicit-def $sgpr15, implicit-def $sgpr16, implicit-def $sgpr17, implicit-def $sgpr18, implicit-def $sgpr19, implicit-def $sgpr20, implicit-def $sgpr21, implicit-def $sgpr22, implicit-def $sgpr23, implicit-def $sgpr24, implicit-def $sgpr25, implicit-def $sgpr26, implicit-def $sgpr27, implicit-def $sgpr28, implicit-def $sgpr29, implicit-def $sgpr30, implicit-def $sgpr31, implicit-def $vcc
     $vgpr0 = V_OR_B32_e32 %stack.0, $vgpr1, implicit $exec, implicit $sgpr4, implicit $sgpr5, implicit $sgpr6, implicit $sgpr7, implicit $sgpr8, implicit $sgpr9, implicit $sgpr10, implicit $sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr16, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $sgpr28, implicit $sgpr29, implicit $sgpr30, implicit $sgpr31
diff --git a/llvm/test/CodeGen/AMDGPU/permute_i8.ll b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
index 312dfa3717c77..5e867deb56ebe 100644
--- a/llvm/test/CodeGen/AMDGPU/permute_i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
@@ -342,7 +342,7 @@ define hidden void @shuffle7330ud2(ptr addrspace(1) %in0, ptr addrspace(1) %out0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
-  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 7, i32 3, i32 3, i32 0>
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> poison, <4 x i32> <i32 7, i32 3, i32 3, i32 0>
   store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
   ret void
 }
@@ -367,7 +367,7 @@ define hidden void @shuffle5341ud2(ptr addrspace(1) %in0, ptr addrspace(1) %out0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
-  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 5, i32 3, i32 4, i32 1>
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> poison, <4 x i32> <i32 5, i32 3, i32 4, i32 1>
   store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
   ret void
 }
@@ -393,7 +393,7 @@ define hidden void @shuffle6106ud2(ptr addrspace(1) %in0, ptr addrspace(1) %out0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
-  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 6, i32 1, i32 0, i32 6>
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> poison, <4 x i32> <i32 6, i32 1, i32 0, i32 6>
   store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
   ret void
 }
@@ -420,7 +420,7 @@ define hidden void @shuffle4327ud2(ptr addrspace(1) %in0, ptr addrspace(1) %out0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
-  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 4, i32 3, i32 2, i32 7>
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> poison, <4 x i32> <i32 4, i32 3, i32 2, i32 7>
   store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
   ret void
 }
@@ -446,7 +446,7 @@ define hidden void @shuffle3263ud2(ptr addrspace(1) %in0, ptr addrspace(1) %out0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
-  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 3, i32 2, i32 6, i32 3>
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 6, i32 3>
   store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
   ret void
 }
@@ -472,7 +472,7 @@ define hidden void @shuffle2763ud2(ptr addrspace(1) %in0, ptr addrspace(1) %out0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
-  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 2, i32 7, i32 6, i32 3>
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> poison, <4 x i32> <i32 2, i32 7, i32 6, i32 3>
   store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
   ret void
 }
@@ -498,7 +498,7 @@ define hidden void @shuffle1327ud2(ptr addrspace(1) %in0, ptr addrspace(1) %out0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
-  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 1, i32 3, i32 2, i32 7>
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> poison, <4 x i32> <i32 1, i32 3, i32 2, i32 7>
   store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
   ret void
 }
@@ -524,7 +524,7 @@ define hidden void @shuffle0605ud2(ptr addrspace(1) %in0, ptr addrspace(1) %out0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
-  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 0, i32 6, i32 0, i32 5>
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> poison, <4 x i32> <i32 0, i32 6, i32 0, i32 5>
   store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
   ret void
 }
@@ -554,7 +554,7 @@ define hidden void @insertUsesOr(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
   %vec1 = load <4 x i8>, ptr addrspace(1) %in1, align 4
-  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 4>
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 4>
   %vecins = insertelement <4 x i8> %shuffle0_0, i8 %elt, i32 1
   store <4 x i8> %vecins, ptr addrspace(1) %out0
   ret void
@@ -598,7 +598,7 @@ define hidden void @addUsesOr(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
   %vec1 = load <4 x i8>, ptr addrspace(1) %in1, align 4
-  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 7, i32 0, i32 6, i32 3>
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> poison, <4 x i32> <i32 7, i32 0, i32 6, i32 3>
   %added = add <4 x i8> %shuffle0_0, %vec1
   store <4 x i8> %added, ptr addrspace(1) %out0
   ret void
@@ -783,7 +783,7 @@ define hidden void @add_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %el
   %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
   %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
   %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
-  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 4>
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 4>
   %vecins = add <4 x i8> %shuffle0_0, %vec1
   store <4 x i8> %vecins, ptr addrspace(1) %out0
   ret void
@@ -835,7 +835,7 @@ define hidden void @add_store(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
   %vec1 = load <4 x i8>, ptr addrspace(1) %in1, align 4
-  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 4>
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 4>
   %vecins = add <4 x i8> %shuffle0_0, %vec1
   store <4 x i8> %vecins, ptr addrspace(1) %out0
   store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
@@ -903,7 +903,7 @@ define hidden void @add_store_div_16(ptr addrspace(1) %in0, ptr addrspace(1) %in
   %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
   %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
   %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
-  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 4>
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 4>
   %vecins = add <4 x i8> %shuffle0_0, %vec1
   store <4 x i8> %vecins, ptr addrspace(1) %out0
   store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
diff --git a/llvm/test/CodeGen/AMDGPU/predicate-dp4.ll b/llvm/test/CodeGen/AMDGPU/predicate-dp4.ll
index 9e2778bb3e71a..e807765c2604b 100644
--- a/llvm/test/CodeGen/AMDGPU/predicate-dp4.ll
+++ b/llvm/test/CodeGen/AMDGPU/predicate-dp4.ll
@@ -16,7 +16,7 @@ IF:                                             ; preds = %main_body
 
 ENDIF:                                            ; preds = %IF, %main_body
   %5 = phi float [%4, %IF], [0.000000e+00, %main_body]
-  %6 = insertelement <4 x float> undef, float %5, i32 0
+  %6 = insertelement <4 x float> poison, float %5, i32 0
   call void @llvm.r600.store.swizzle(<4 x float> %6, i32 0, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-array-aggregate.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-array-aggregate.ll
index 885766d572345..a4a8a985df0bf 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-array-aggregate.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-array-aggregate.ll
@@ -134,7 +134,7 @@ define amdgpu_vs void @promote_load_from_store_aggr_varoff(<4 x i32> %input) {
 ; CHECK-NEXT:    [[FOO3_UNPACK2:%.*]] = load i32, ptr addrspace(1) getelementptr inbounds (i8, ptr addrspace(1) @block4, i64 8), align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <3 x i32> undef, i32 [[FOO3_UNPACK2]], i32 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <3 x i32> [[TMP1]], i32 [[FOO3_UNPACK2]]
-; CHECK-NEXT:    [[FOO12:%.*]] = insertelement <4 x i32> %input, i32 [[TMP2]], i64 3
+; CHECK-NEXT:    [[FOO12:%.*]] = insertelement <4 x i32> [[INPUT:%.*]], i32 [[TMP2]], i64 3
 ; CHECK-NEXT:    store <4 x i32> [[FOO12]], ptr addrspace(1) @pv1, align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -337,14 +337,14 @@ define amdgpu_ps void @promote_double_aggr() #0 {
 ; CHECK-NEXT:    [[FOO1:%.*]] = load double, ptr addrspace(1) [[FOO]], align 8
 ; CHECK-NEXT:    [[FOO2:%.*]] = getelementptr { [4 x double], <2 x double>, <3 x double>, <4 x double> }, ptr addrspace(1) @tmp_g, i32 0, i32 0, i32 1
 ; CHECK-NEXT:    [[FOO3:%.*]] = load double, ptr addrspace(1) [[FOO2]], align 8
-; CHECK-NEXT:    [[FOO4:%.*]] = insertvalue [2 x double] undef, double [[FOO1]], 0
+; CHECK-NEXT:    [[FOO4:%.*]] = insertvalue [2 x double] poison, double [[FOO1]], 0
 ; CHECK-NEXT:    [[FOO5:%.*]] = insertvalue [2 x double] [[FOO4]], double [[FOO3]], 1
 ; CHECK-NEXT:    [[FOO5_FCA_0_EXTRACT:%.*]] = extractvalue [2 x double] [[FOO5]], 0
 ; CHECK-NEXT:    [[FOO5_FCA_1_EXTRACT:%.*]] = extractvalue [2 x double] [[FOO5]], 1
 ; CHECK-NEXT:    [[FOO10:%.*]] = fadd double [[FOO5_FCA_1_EXTRACT]], [[FOO5_FCA_1_EXTRACT]]
 ; CHECK-NEXT:    [[FOO16:%.*]] = fadd double [[FOO10]], [[FOO5_FCA_1_EXTRACT]]
 ; CHECK-NEXT:    [[FOO17:%.*]] = fptrunc double [[FOO16]] to float
-; CHECK-NEXT:    [[FOO18:%.*]] = insertelement <4 x float> undef, float [[FOO17]], i32 0
+; CHECK-NEXT:    [[FOO18:%.*]] = insertelement <4 x float> poison, float [[FOO17]], i32 0
 ; CHECK-NEXT:    [[FOO19:%.*]] = insertelement <4 x float> [[FOO18]], float [[FOO17]], i32 1
 ; CHECK-NEXT:    [[FOO20:%.*]] = insertelement <4 x float> [[FOO19]], float [[FOO17]], i32 2
 ; CHECK-NEXT:    [[FOO21:%.*]] = insertelement <4 x float> [[FOO20]], float [[FOO17]], i32 3
@@ -356,7 +356,7 @@ define amdgpu_ps void @promote_double_aggr() #0 {
   %foo1 = load double, ptr addrspace(1) %foo
   %foo2 = getelementptr { [4 x double], <2 x double>, <3 x double>, <4 x double> }, ptr addrspace(1) @tmp_g, i32 0, i32 0, i32 1
   %foo3 = load double, ptr addrspace(1) %foo2
-  %foo4 = insertvalue [2 x double] undef, double %foo1, 0
+  %foo4 = insertvalue [2 x double] poison, double %foo1, 0
   %foo5 = insertvalue [2 x double] %foo4, double %foo3, 1
   store [2 x double] %foo5, ptr addrspace(5) %s
   %foo6 = getelementptr [2 x double], ptr addrspace(5) %s, i32 0, i32 1
@@ -370,7 +370,7 @@ define amdgpu_ps void @promote_double_aggr() #0 {
   %foo15 = load double, ptr addrspace(5) %foo14
   %foo16 = fadd double %foo13, %foo15
   %foo17 = fptrunc double %foo16 to float
-  %foo18 = insertelement <4 x float> undef, float %foo17, i32 0
+  %foo18 = insertelement <4 x float> poison, float %foo17, i32 0
   %foo19 = insertelement <4 x float> %foo18, float %foo17, i32 1
   %foo20 = insertelement <4 x float> %foo19, float %foo17, i32 2
   %foo21 = insertelement <4 x float> %foo20, float %foo17, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-loadstores.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-loadstores.ll
index 1e49500a243e1..119d3611e1007 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-loadstores.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-loadstores.ll
@@ -9,15 +9,15 @@ define amdgpu_kernel void @test_overwrite(i64 %val, i1 %cond) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[COND]], label [[LOOP:%.*]], label [[END:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[PROMOTEALLOCA:%.*]] = phi <3 x i64> [ [[TMP2:%.*]], [[LOOP]] ], [ <i64 43, i64 undef, i64 undef>, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x i64> [[PROMOTEALLOCA]], i32 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <3 x i64> [[PROMOTEALLOCA]], i64 68, i32 0
+; CHECK-NEXT:    [[PROMOTEALLOCA1:%.*]] = phi <3 x i64> [ [[TMP2:%.*]], [[LOOP]] ], [ <i64 43, i64 undef, i64 undef>, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x i64> [[PROMOTEALLOCA1]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <3 x i64> [[PROMOTEALLOCA1]], i64 68, i32 0
 ; CHECK-NEXT:    [[TMP2]] = insertelement <3 x i64> [[TMP1]], i64 32, i32 0
 ; CHECK-NEXT:    [[LOOP_CC:%.*]] = icmp ne i64 [[TMP0]], 68
 ; CHECK-NEXT:    br i1 [[LOOP_CC]], label [[LOOP]], label [[END]]
 ; CHECK:       end:
-; CHECK-NEXT:    [[PROMOTEALLOCA1:%.*]] = phi <3 x i64> [ [[TMP2]], [[LOOP]] ], [ <i64 43, i64 undef, i64 undef>, [[ENTRY]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <3 x i64> [[PROMOTEALLOCA1]], i32 0
+; CHECK-NEXT:    [[PROMOTEALLOCA:%.*]] = phi <3 x i64> [ [[TMP2]], [[LOOP]] ], [ <i64 43, i64 undef, i64 undef>, [[ENTRY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <3 x i64> [[PROMOTEALLOCA]], i32 0
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -64,15 +64,15 @@ define amdgpu_kernel void @test_no_overwrite(i64 %val, i1 %cond) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[COND]], label [[LOOP:%.*]], label [[END:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[PROMOTEALLOCA:%.*]] = phi <3 x i64> [ [[TMP1:%.*]], [[LOOP]] ], [ <i64 43, i64 undef, i64 undef>, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x i64> [[PROMOTEALLOCA]], i32 0
-; CHECK-NEXT:    [[TMP1]] = insertelement <3 x i64> [[PROMOTEALLOCA]], i64 32, i32 1
+; CHECK-NEXT:    [[PROMOTEALLOCA1:%.*]] = phi <3 x i64> [ [[TMP1:%.*]], [[LOOP]] ], [ <i64 43, i64 undef, i64 undef>, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x i64> [[PROMOTEALLOCA1]], i32 0
+; CHECK-NEXT:    [[TMP1]] = insertelement <3 x i64> [[PROMOTEALLOCA1]], i64 32, i32 1
 ; CHECK-NEXT:    [[LOOP_CC:%.*]] = icmp ne i64 [[TMP0]], 32
 ; CHECK-NEXT:    br i1 [[LOOP_CC]], label [[LOOP]], label [[END]]
 ; CHECK:       end:
-; CHECK-NEXT:    [[PROMOTEALLOCA1:%.*]] = phi <3 x i64> [ [[TMP1]], [[LOOP]] ], [ <i64 43, i64 undef, i64 undef>, [[ENTRY]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <3 x i64> [[PROMOTEALLOCA1]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <3 x i64> [[PROMOTEALLOCA1]], i32 1
+; CHECK-NEXT:    [[PROMOTEALLOCA:%.*]] = phi <3 x i64> [ [[TMP1]], [[LOOP]] ], [ <i64 43, i64 undef, i64 undef>, [[ENTRY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <3 x i64> [[PROMOTEALLOCA]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <3 x i64> [[PROMOTEALLOCA]], i32 1
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-max-regs.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-max-regs.ll
new file mode 100644
index 0000000000000..f9bdfc51f61ff
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-max-regs.ll
@@ -0,0 +1,236 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-promote-alloca -disable-promote-alloca-to-lds=1 < %s | FileCheck --check-prefix=BASE --check-prefix=DEFAULT %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-promote-alloca -disable-promote-alloca-to-lds=1 -amdgpu-promote-alloca-to-vector-max-regs=24 < %s | FileCheck --check-prefix=BASE %s --check-prefix=MAX24
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-promote-alloca -disable-promote-alloca-to-lds=1 -amdgpu-promote-alloca-to-vector-max-regs=32 < %s | FileCheck --check-prefix=BASE %s --check-prefix=MAX32
+
+define amdgpu_kernel void @i32_24_elements(ptr %out) #0 {
+; DEFAULT-LABEL: define amdgpu_kernel void @i32_24_elements(
+; DEFAULT-SAME: ptr [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; DEFAULT-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; DEFAULT-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; DEFAULT-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; DEFAULT-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; DEFAULT-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; DEFAULT-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; DEFAULT-NEXT:    [[ALLOCA:%.*]] = alloca [24 x i32], align 16, addrspace(5)
+; DEFAULT-NEXT:    call void @llvm.memset.p5.i32(ptr addrspace(5) [[ALLOCA]], i8 0, i32 96, i1 false)
+; DEFAULT-NEXT:    [[GEP_0:%.*]] = getelementptr inbounds [24 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 0
+; DEFAULT-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds [24 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 20
+; DEFAULT-NEXT:    store i32 42, ptr addrspace(5) [[GEP_0]], align 4
+; DEFAULT-NEXT:    store i32 43, ptr addrspace(5) [[GEP_1]], align 4
+; DEFAULT-NEXT:    [[GEP:%.*]] = getelementptr inbounds [24 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[SEL2]]
+; DEFAULT-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(5) [[GEP]], align 4
+; DEFAULT-NEXT:    store i32 [[LOAD]], ptr [[OUT]], align 4
+; DEFAULT-NEXT:    ret void
+;
+; MAX24-LABEL: define amdgpu_kernel void @i32_24_elements(
+; MAX24-SAME: ptr [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; MAX24-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; MAX24-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; MAX24-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; MAX24-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; MAX24-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; MAX24-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; MAX24-NEXT:    [[TMP1:%.*]] = extractelement <24 x i32> <i32 42, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 43, i32 0, i32 0, i32 0>, i32 [[SEL2]]
+; MAX24-NEXT:    store i32 [[TMP1]], ptr [[OUT]], align 4
+; MAX24-NEXT:    ret void
+;
+; MAX32-LABEL: define amdgpu_kernel void @i32_24_elements(
+; MAX32-SAME: ptr [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; MAX32-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; MAX32-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; MAX32-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; MAX32-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; MAX32-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; MAX32-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; MAX32-NEXT:    [[TMP1:%.*]] = extractelement <24 x i32> <i32 42, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 43, i32 0, i32 0, i32 0>, i32 [[SEL2]]
+; MAX32-NEXT:    store i32 [[TMP1]], ptr [[OUT]], align 4
+; MAX32-NEXT:    ret void
+;
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %c1 = icmp uge i32 %x, 3
+  %c2 = icmp uge i32 %y, 3
+  %sel1 = select i1 %c1, i32 1, i32 2
+  %sel2 = select i1 %c2, i32 0, i32 %sel1
+  %alloca = alloca [24 x i32], align 16, addrspace(5)
+  call void @llvm.memset.p5.i32(ptr addrspace(5) %alloca, i8 0, i32 96, i1 false)
+  %gep.0 = getelementptr inbounds [24 x i32], ptr addrspace(5) %alloca, i32 0, i32 0
+  %gep.1 = getelementptr inbounds [24 x i32], ptr addrspace(5) %alloca, i32 0, i32 20
+  store i32 42, ptr addrspace(5) %gep.0
+  store i32 43, ptr addrspace(5) %gep.1
+  %gep = getelementptr inbounds [24 x i32], ptr addrspace(5) %alloca, i32 0, i32 %sel2
+  %load = load i32, ptr addrspace(5) %gep
+  store i32 %load, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @i32_24_elements_attrib(ptr %out) #1 {
+; BASE-LABEL: define amdgpu_kernel void @i32_24_elements_attrib(
+; BASE-SAME: ptr [[OUT:%.*]]) #[[ATTR1:[0-9]+]] {
+; BASE-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; BASE-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; BASE-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; BASE-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; BASE-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; BASE-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; BASE-NEXT:    [[TMP1:%.*]] = extractelement <24 x i32> <i32 42, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 43, i32 0, i32 0, i32 0>, i32 [[SEL2]]
+; BASE-NEXT:    store i32 [[TMP1]], ptr [[OUT]], align 4
+; BASE-NEXT:    ret void
+;
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %c1 = icmp uge i32 %x, 3
+  %c2 = icmp uge i32 %y, 3
+  %sel1 = select i1 %c1, i32 1, i32 2
+  %sel2 = select i1 %c2, i32 0, i32 %sel1
+  %alloca = alloca [24 x i32], align 16, addrspace(5)
+  call void @llvm.memset.p5.i32(ptr addrspace(5) %alloca, i8 0, i32 96, i1 false)
+  %gep.0 = getelementptr inbounds [24 x i32], ptr addrspace(5) %alloca, i32 0, i32 0
+  %gep.1 = getelementptr inbounds [24 x i32], ptr addrspace(5) %alloca, i32 0, i32 20
+  store i32 42, ptr addrspace(5) %gep.0
+  store i32 43, ptr addrspace(5) %gep.1
+  %gep = getelementptr inbounds [24 x i32], ptr addrspace(5) %alloca, i32 0, i32 %sel2
+  %load = load i32, ptr addrspace(5) %gep
+  store i32 %load, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @i32_32_elements(ptr %out) #0 {
+; DEFAULT-LABEL: define amdgpu_kernel void @i32_32_elements(
+; DEFAULT-SAME: ptr [[OUT:%.*]]) #[[ATTR0]] {
+; DEFAULT-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; DEFAULT-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; DEFAULT-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; DEFAULT-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; DEFAULT-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; DEFAULT-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; DEFAULT-NEXT:    [[ALLOCA:%.*]] = alloca [32 x i32], align 16, addrspace(5)
+; DEFAULT-NEXT:    call void @llvm.memset.p5.i32(ptr addrspace(5) [[ALLOCA]], i8 0, i32 128, i1 false)
+; DEFAULT-NEXT:    [[GEP_0:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 0
+; DEFAULT-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 30
+; DEFAULT-NEXT:    store i32 42, ptr addrspace(5) [[GEP_0]], align 4
+; DEFAULT-NEXT:    store i32 43, ptr addrspace(5) [[GEP_1]], align 4
+; DEFAULT-NEXT:    [[GEP:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[SEL2]]
+; DEFAULT-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(5) [[GEP]], align 4
+; DEFAULT-NEXT:    store i32 [[LOAD]], ptr [[OUT]], align 4
+; DEFAULT-NEXT:    ret void
+;
+; MAX24-LABEL: define amdgpu_kernel void @i32_32_elements(
+; MAX24-SAME: ptr [[OUT:%.*]]) #[[ATTR0]] {
+; MAX24-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; MAX24-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; MAX24-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; MAX24-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; MAX24-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; MAX24-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; MAX24-NEXT:    [[ALLOCA:%.*]] = alloca [32 x i32], align 16, addrspace(5)
+; MAX24-NEXT:    call void @llvm.memset.p5.i32(ptr addrspace(5) [[ALLOCA]], i8 0, i32 128, i1 false)
+; MAX24-NEXT:    [[GEP_0:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 0
+; MAX24-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 30
+; MAX24-NEXT:    store i32 42, ptr addrspace(5) [[GEP_0]], align 4
+; MAX24-NEXT:    store i32 43, ptr addrspace(5) [[GEP_1]], align 4
+; MAX24-NEXT:    [[GEP:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[SEL2]]
+; MAX24-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(5) [[GEP]], align 4
+; MAX24-NEXT:    store i32 [[LOAD]], ptr [[OUT]], align 4
+; MAX24-NEXT:    ret void
+;
+; MAX32-LABEL: define amdgpu_kernel void @i32_32_elements(
+; MAX32-SAME: ptr [[OUT:%.*]]) #[[ATTR0]] {
+; MAX32-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; MAX32-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; MAX32-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; MAX32-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; MAX32-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; MAX32-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; MAX32-NEXT:    [[TMP1:%.*]] = extractelement <32 x i32> <i32 42, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 43, i32 0>, i32 [[SEL2]]
+; MAX32-NEXT:    store i32 [[TMP1]], ptr [[OUT]], align 4
+; MAX32-NEXT:    ret void
+;
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %c1 = icmp uge i32 %x, 3
+  %c2 = icmp uge i32 %y, 3
+  %sel1 = select i1 %c1, i32 1, i32 2
+  %sel2 = select i1 %c2, i32 0, i32 %sel1
+  %alloca = alloca [32 x i32], align 16, addrspace(5)
+  call void @llvm.memset.p5.i32(ptr addrspace(5) %alloca, i8 0, i32 128, i1 false)
+  %gep.0 = getelementptr inbounds [32 x i32], ptr addrspace(5) %alloca, i32 0, i32 0
+  %gep.1 = getelementptr inbounds [32 x i32], ptr addrspace(5) %alloca, i32 0, i32 30
+  store i32 42, ptr addrspace(5) %gep.0
+  store i32 43, ptr addrspace(5) %gep.1
+  %gep = getelementptr inbounds [32 x i32], ptr addrspace(5) %alloca, i32 0, i32 %sel2
+  %load = load i32, ptr addrspace(5) %gep
+  store i32 %load, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @i32_32_elements_attrib(ptr %out) #2 {
+; DEFAULT-LABEL: define amdgpu_kernel void @i32_32_elements_attrib(
+; DEFAULT-SAME: ptr [[OUT:%.*]]) #[[ATTR2:[0-9]+]] {
+; DEFAULT-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; DEFAULT-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; DEFAULT-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; DEFAULT-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; DEFAULT-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; DEFAULT-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; DEFAULT-NEXT:    [[TMP1:%.*]] = extractelement <32 x i32> <i32 42, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 43, i32 0>, i32 [[SEL2]]
+; DEFAULT-NEXT:    store i32 [[TMP1]], ptr [[OUT]], align 4
+; DEFAULT-NEXT:    ret void
+;
+; MAX24-LABEL: define amdgpu_kernel void @i32_32_elements_attrib(
+; MAX24-SAME: ptr [[OUT:%.*]]) #[[ATTR2:[0-9]+]] {
+; MAX24-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; MAX24-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; MAX24-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; MAX24-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; MAX24-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; MAX24-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; MAX24-NEXT:    [[ALLOCA:%.*]] = alloca [32 x i32], align 16, addrspace(5)
+; MAX24-NEXT:    call void @llvm.memset.p5.i32(ptr addrspace(5) [[ALLOCA]], i8 0, i32 128, i1 false)
+; MAX24-NEXT:    [[GEP_0:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 0
+; MAX24-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 30
+; MAX24-NEXT:    store i32 42, ptr addrspace(5) [[GEP_0]], align 4
+; MAX24-NEXT:    store i32 43, ptr addrspace(5) [[GEP_1]], align 4
+; MAX24-NEXT:    [[GEP:%.*]] = getelementptr inbounds [32 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[SEL2]]
+; MAX24-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(5) [[GEP]], align 4
+; MAX24-NEXT:    store i32 [[LOAD]], ptr [[OUT]], align 4
+; MAX24-NEXT:    ret void
+;
+; MAX32-LABEL: define amdgpu_kernel void @i32_32_elements_attrib(
+; MAX32-SAME: ptr [[OUT:%.*]]) #[[ATTR2:[0-9]+]] {
+; MAX32-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; MAX32-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; MAX32-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; MAX32-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; MAX32-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; MAX32-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; MAX32-NEXT:    [[TMP1:%.*]] = extractelement <32 x i32> <i32 42, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 43, i32 0>, i32 [[SEL2]]
+; MAX32-NEXT:    store i32 [[TMP1]], ptr [[OUT]], align 4
+; MAX32-NEXT:    ret void
+;
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %c1 = icmp uge i32 %x, 3
+  %c2 = icmp uge i32 %y, 3
+  %sel1 = select i1 %c1, i32 1, i32 2
+  %sel2 = select i1 %c2, i32 0, i32 %sel1
+  %alloca = alloca [32 x i32], align 16, addrspace(5)
+  call void @llvm.memset.p5.i32(ptr addrspace(5) %alloca, i8 0, i32 128, i1 false)
+  %gep.0 = getelementptr inbounds [32 x i32], ptr addrspace(5) %alloca, i32 0, i32 0
+  %gep.1 = getelementptr inbounds [32 x i32], ptr addrspace(5) %alloca, i32 0, i32 30
+  store i32 42, ptr addrspace(5) %gep.0
+  store i32 43, ptr addrspace(5) %gep.1
+  %gep = getelementptr inbounds [32 x i32], ptr addrspace(5) %alloca, i32 0, i32 %sel2
+  %load = load i32, ptr addrspace(5) %gep
+  store i32 %load, ptr %out
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
+declare i32 @llvm.amdgcn.workitem.id.y()
+declare void @llvm.memset.p5.i32(ptr addrspace(5) nocapture writeonly, i8, i32, i1 immarg)
+
+attributes #0 = { nounwind "amdgpu-flat-work-group-size"="64,64" "amdgpu-waves-per-eu"="1,3" }
+attributes #1 = { nounwind "amdgpu-flat-work-group-size"="64,64" "amdgpu-waves-per-eu"="1,3" "amdgpu-promote-alloca-to-vector-max-regs"="24" }
+attributes #2 = { nounwind "amdgpu-flat-work-group-size"="64,64" "amdgpu-waves-per-eu"="1,3" "amdgpu-promote-alloca-to-vector-max-regs"="32" }
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-memset.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-memset.ll
index 74651e10c7809..bb51c1d27b336 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-memset.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-memset.ll
@@ -110,10 +110,7 @@ define amdgpu_kernel void @memset_vector_ptr_alloca(ptr %out) {
 
 define amdgpu_kernel void @memset_array_of_array_ptr_alloca(ptr %out) {
 ; CHECK-LABEL: @memset_array_of_array_ptr_alloca(
-; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [2 x [3 x ptr]], align 16, addrspace(5)
-; CHECK-NEXT:    call void @llvm.memset.p5.i64(ptr addrspace(5) [[ALLOCA]], i8 0, i64 48, i1 false)
-; CHECK-NEXT:    [[LOAD:%.*]] = load i64, ptr addrspace(5) [[ALLOCA]], align 8
-; CHECK-NEXT:    store i64 [[LOAD]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
   %alloca = alloca [2 x [3 x ptr]], align 16, addrspace(5)
@@ -125,14 +122,11 @@ define amdgpu_kernel void @memset_array_of_array_ptr_alloca(ptr %out) {
 
 define amdgpu_kernel void @memset_array_of_vec_ptr_alloca(ptr %out) {
 ; CHECK-LABEL: @memset_array_of_vec_ptr_alloca(
-; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [2 x <3 x ptr>], align 16, addrspace(5)
-; CHECK-NEXT:    call void @llvm.memset.p5.i64(ptr addrspace(5) [[ALLOCA]], i8 0, i64 48, i1 false)
-; CHECK-NEXT:    [[LOAD:%.*]] = load i64, ptr addrspace(5) [[ALLOCA]], align 8
-; CHECK-NEXT:    store i64 [[LOAD]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
   %alloca = alloca [2 x <3 x ptr>], align 16, addrspace(5)
-  call void @llvm.memset.p5.i64(ptr addrspace(5) %alloca, i8 0, i64 48, i1 false)
+  call void @llvm.memset.p5.i64(ptr addrspace(5) %alloca, i8 0, i64 64, i1 false)
   %load = load i64, ptr addrspace(5) %alloca
   store i64 %load, ptr %out
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-multidim.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-multidim.ll
new file mode 100644
index 0000000000000..c5b9bff8b663a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-multidim.ll
@@ -0,0 +1,433 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-promote-alloca -disable-promote-alloca-to-lds=1 < %s | FileCheck %s
+
+define amdgpu_kernel void @i32_2d_load_store(ptr %out) {
+; CHECK-LABEL: define amdgpu_kernel void @i32_2d_load_store(
+; CHECK-SAME: ptr [[OUT:%.*]]) {
+; CHECK-NEXT:    store i32 3, ptr [[OUT]], align 4
+; CHECK-NEXT:    ret void
+;
+  %alloca = alloca [2 x [3 x i32]], align 16, addrspace(5)
+  %gep.00 = getelementptr inbounds [2 x [3 x i32]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 0
+  %gep.01 = getelementptr inbounds [2 x [3 x i32]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 1
+  %gep.02 = getelementptr inbounds [2 x [3 x i32]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 2
+  %gep.10 = getelementptr inbounds [2 x [3 x i32]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 0
+  %gep.11 = getelementptr inbounds [2 x [3 x i32]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 1
+  %gep.12 = getelementptr inbounds [2 x [3 x i32]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 2
+  store i32 0, ptr addrspace(5) %gep.00
+  store i32 1, ptr addrspace(5) %gep.01
+  store i32 2, ptr addrspace(5) %gep.02
+  store i32 3, ptr addrspace(5) %gep.10
+  store i32 4, ptr addrspace(5) %gep.11
+  store i32 5, ptr addrspace(5) %gep.12
+  %gep = getelementptr inbounds [2 x [3 x i32]], ptr addrspace(5) %alloca, i32 0, i32 1
+  %load = load i32, ptr addrspace(5) %gep
+  store i32 %load, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @i64_2d_load_store(ptr %out) {
+; CHECK-LABEL: define amdgpu_kernel void @i64_2d_load_store(
+; CHECK-SAME: ptr [[OUT:%.*]]) {
+; CHECK-NEXT:    store i64 3, ptr [[OUT]], align 8
+; CHECK-NEXT:    ret void
+;
+  %alloca = alloca [2 x [3 x i64]], align 16, addrspace(5)
+  %gep.00 = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 0
+  %gep.01 = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 1
+  %gep.02 = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 2
+  %gep.10 = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 0
+  %gep.11 = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 1
+  %gep.12 = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 2
+  store i64 0, ptr addrspace(5) %gep.00
+  store i64 1, ptr addrspace(5) %gep.01
+  store i64 2, ptr addrspace(5) %gep.02
+  store i64 3, ptr addrspace(5) %gep.10
+  store i64 4, ptr addrspace(5) %gep.11
+  store i64 5, ptr addrspace(5) %gep.12
+  %gep = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0, i32 1
+  %load = load i64, ptr addrspace(5) %gep
+  store i64 %load, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @i32_2d_alloca_store_partial(ptr addrspace(1) %out, ptr addrspace(3) %dummy_lds) {
+; CHECK-LABEL: define amdgpu_kernel void @i32_2d_alloca_store_partial(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(3) [[DUMMY_LDS:%.*]]) {
+; CHECK-NEXT:  [[BB:.*:]]
+; CHECK-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; CHECK-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; CHECK-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 {{.*}}, i32 {{.*}}, i32 {{.*}}, i32 {{.*}}>, i32 [[SEL2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32 [[TMP0]] to float
+; CHECK-NEXT:    store float [[TMP1]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT:    ret void
+;
+bb:
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %c1 = icmp uge i32 %x, 3
+  %c2 = icmp uge i32 %y, 3
+  %sel1 = select i1 %c1, i32 1, i32 2
+  %sel2 = select i1 %c2, i32 0, i32 %sel1
+  %alloca = alloca [2 x [4 x i32]], align 4, addrspace(5)
+  %gep = getelementptr inbounds <4 x i32>, ptr addrspace(5) %alloca, i32 0, i32 %sel2
+  store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, ptr addrspace(5) %alloca, align 4
+  %load = load float, ptr addrspace(5) %gep, align 4
+  store float %load, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+
+define amdgpu_kernel void @i64_2d_load_store_cast(ptr %out) {
+; CHECK-LABEL: define amdgpu_kernel void @i64_2d_load_store_cast(
+; CHECK-SAME: ptr [[OUT:%.*]]) {
+; CHECK-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; CHECK-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; CHECK-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <6 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5>, i32 [[SEL2]]
+; CHECK-NEXT:    store i64 [[TMP1]], ptr [[OUT]], align 8
+; CHECK-NEXT:    ret void
+;
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %c1 = icmp uge i32 %x, 3
+  %c2 = icmp uge i32 %y, 3
+  %sel1 = select i1 %c1, i32 1, i32 2
+  %sel2 = select i1 %c2, i32 0, i32 %sel1
+  %alloca = alloca [2 x [3 x i64]], align 16, addrspace(5)
+  %gep.00 = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 0
+  %gep.01 = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 1
+  %gep.02 = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 2
+  %gep.10 = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 0
+  %gep.11 = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 1
+  %gep.12 = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 2
+  store i64 0, ptr addrspace(5) %gep.00
+  store i64 1, ptr addrspace(5) %gep.01
+  store i64 2, ptr addrspace(5) %gep.02
+  store i64 3, ptr addrspace(5) %gep.10
+  store i64 4, ptr addrspace(5) %gep.11
+  store i64 5, ptr addrspace(5) %gep.12
+  %gep = getelementptr inbounds [6 x i64], ptr addrspace(5) %alloca, i32 0, i32 %sel2
+  %load = load i64, ptr addrspace(5) %gep
+  store i64 %load, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @i64_2d_load_store_subvec_1(ptr %out) {
+; CHECK-LABEL: define amdgpu_kernel void @i64_2d_load_store_subvec_1(
+; CHECK-SAME: ptr [[OUT:%.*]]) {
+; CHECK-NEXT:    [[ELEM:%.*]] = extractelement <3 x i64> <i64 3, i64 4, i64 5>, i32 2
+; CHECK-NEXT:    store i64 [[ELEM]], ptr [[OUT]], align 8
+; CHECK-NEXT:    ret void
+;
+  %alloca = alloca [2 x [3 x i64]], align 16, addrspace(5)
+  %gep.00 = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0
+  %gep.01 = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 0
+  store <3 x i64> <i64 0, i64 1, i64 2>, ptr addrspace(5) %gep.00
+  store <3 x i64> <i64 3, i64 4, i64 5>, ptr addrspace(5) %gep.01
+  %gep = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0, i32 1
+  %load = load <3 x i64>, ptr addrspace(5) %gep
+  %elem = extractelement <3 x i64> %load, i32 2
+  store i64 %elem, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @i64_2d_load_store_subvec_2(ptr %out) {
+; CHECK-LABEL: define amdgpu_kernel void @i64_2d_load_store_subvec_2(
+; CHECK-SAME: ptr [[OUT:%.*]]) {
+; CHECK-NEXT:    [[ELEM:%.*]] = extractelement <3 x i64> <i64 3, i64 4, i64 5>, i32 2
+; CHECK-NEXT:    store i64 [[ELEM]], ptr [[OUT]], align 8
+; CHECK-NEXT:    ret void
+;
+  %alloca = alloca [2 x <3 x i64>], align 16, addrspace(5)
+  %gep.00 = getelementptr inbounds [2 x <3 x i64>], ptr addrspace(5) %alloca, i32 0
+  %gep.01 = getelementptr inbounds [2 x <3 x i64>], ptr addrspace(5) %alloca, i32 0, i32 1, i32 0
+  store <3 x i64> <i64 0, i64 1, i64 2>, ptr addrspace(5) %gep.00
+  store <3 x i64> <i64 3, i64 4, i64 5>, ptr addrspace(5) %gep.01
+  %gep = getelementptr inbounds [2 x <3 x i64>], ptr addrspace(5) %alloca, i32 0, i32 1
+  %load = load <3 x i64>, ptr addrspace(5) %gep
+  %elem = extractelement <3 x i64> %load, i32 2
+  store i64 %elem, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @i64_2d_load_store_subvec_3(ptr %out) {
+; CHECK-LABEL: define amdgpu_kernel void @i64_2d_load_store_subvec_3(
+; CHECK-SAME: ptr [[OUT:%.*]]) {
+; CHECK-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; CHECK-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; CHECK-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[SEL2]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <6 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5>, i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <3 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <6 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5>, i32 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <3 x i64> [[TMP3]], i64 [[TMP5]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = add i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <6 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5>, i32 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <3 x i64> [[TMP6]], i64 [[TMP8]], i64 2
+; CHECK-NEXT:    [[ELEM:%.*]] = extractelement <3 x i64> [[TMP9]], i32 2
+; CHECK-NEXT:    store i64 [[ELEM]], ptr [[OUT]], align 8
+; CHECK-NEXT:    ret void
+;
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %c1 = icmp uge i32 %x, 3
+  %c2 = icmp uge i32 %y, 3
+  %sel1 = select i1 %c1, i32 1, i32 2
+  %sel2 = select i1 %c2, i32 0, i32 %sel1
+  %alloca = alloca [2 x [3 x i64]], align 16, addrspace(5)
+  %gep.00 = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0
+  %gep.01 = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 0
+  store <3 x i64> <i64 0, i64 1, i64 2>, ptr addrspace(5) %gep.00
+  store <3 x i64> <i64 3, i64 4, i64 5>, ptr addrspace(5) %gep.01
+  %gep = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0, i32 %sel2
+  %load = load <3 x i64>, ptr addrspace(5) %gep
+  %elem = extractelement <3 x i64> %load, i32 2
+  store i64 %elem, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @i64_2d_load_store_subvec_3_i64_offset(ptr %out) {
+; CHECK-LABEL: define amdgpu_kernel void @i64_2d_load_store_subvec_3_i64_offset(
+; CHECK-SAME: ptr [[OUT:%.*]]) {
+; CHECK-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; CHECK-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; CHECK-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; CHECK-NEXT:    [[SEL3:%.*]] = zext i32 [[SEL2]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[SEL3]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <6 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5>, i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <3 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <6 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5>, i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <3 x i64> [[TMP3]], i64 [[TMP5]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <6 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5>, i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <3 x i64> [[TMP6]], i64 [[TMP8]], i64 2
+; CHECK-NEXT:    [[ELEM:%.*]] = extractelement <3 x i64> [[TMP9]], i32 2
+; CHECK-NEXT:    store i64 [[ELEM]], ptr [[OUT]], align 8
+; CHECK-NEXT:    ret void
+;
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %c1 = icmp uge i32 %x, 3
+  %c2 = icmp uge i32 %y, 3
+  %sel1 = select i1 %c1, i32 1, i32 2
+  %sel2 = select i1 %c2, i32 0, i32 %sel1
+  %sel3 = zext i32 %sel2 to i64
+  %alloca = alloca [2 x [3 x i64]], align 16, addrspace(5)
+  %gep.00 = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0
+  %gep.01 = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 0
+  store <3 x i64> <i64 0, i64 1, i64 2>, ptr addrspace(5) %gep.00
+  store <3 x i64> <i64 3, i64 4, i64 5>, ptr addrspace(5) %gep.01
+  %gep = getelementptr inbounds [2 x [3 x i64]], ptr addrspace(5) %alloca, i64 0, i64 %sel3
+  %load = load <3 x i64>, ptr addrspace(5) %gep
+  %elem = extractelement <3 x i64> %load, i32 2
+  store i64 %elem, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @i64_2d_load_store_subvec_4(ptr %out) {
+; CHECK-LABEL: define amdgpu_kernel void @i64_2d_load_store_subvec_4(
+; CHECK-SAME: ptr [[OUT:%.*]]) {
+; CHECK-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; CHECK-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; CHECK-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[SEL2]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i64> <i64 0, i64 1, i64 2, i64 {{.*}}, i64 3, i64 4, i64 5, i64 {{.*}}>, i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <3 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i64> <i64 0, i64 1, i64 2, i64 {{.*}}, i64 3, i64 4, i64 5, i64 {{.*}}>, i32 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <3 x i64> [[TMP3]], i64 [[TMP5]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = add i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i64> <i64 0, i64 1, i64 2, i64 {{.*}}, i64 3, i64 4, i64 5, i64 {{.*}}>, i32 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <3 x i64> [[TMP6]], i64 [[TMP8]], i64 2
+; CHECK-NEXT:    [[ELEM:%.*]] = extractelement <3 x i64> [[TMP9]], i32 2
+; CHECK-NEXT:    store i64 [[ELEM]], ptr [[OUT]], align 8
+; CHECK-NEXT:    ret void
+;
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %c1 = icmp uge i32 %x, 3
+  %c2 = icmp uge i32 %y, 3
+  %sel1 = select i1 %c1, i32 1, i32 2
+  %sel2 = select i1 %c2, i32 0, i32 %sel1
+  %alloca = alloca [2 x <3 x i64>], align 16, addrspace(5)
+  %gep.00 = getelementptr inbounds [2 x <3 x i64>], ptr addrspace(5) %alloca, i32 0
+  %gep.01 = getelementptr inbounds [2 x <3 x i64>], ptr addrspace(5) %alloca, i32 0, i32 1, i32 0
+  store <3 x i64> <i64 0, i64 1, i64 2>, ptr addrspace(5) %gep.00
+  store <3 x i64> <i64 3, i64 4, i64 5>, ptr addrspace(5) %gep.01
+  %gep = getelementptr inbounds [2 x <3 x i64>], ptr addrspace(5) %alloca, i32 0, i32 %sel2
+  %load = load <3 x i64>, ptr addrspace(5) %gep
+  %elem = extractelement <3 x i64> %load, i32 2
+  store i64 %elem, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @i32_3d_load_store(ptr %out) {
+; CHECK-LABEL: define amdgpu_kernel void @i32_3d_load_store(
+; CHECK-SAME: ptr [[OUT:%.*]]) {
+; CHECK-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; CHECK-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; CHECK-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>, i32 [[SEL2]]
+; CHECK-NEXT:    store i32 [[TMP1]], ptr [[OUT]], align 4
+; CHECK-NEXT:    ret void
+;
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %c1 = icmp uge i32 %x, 3
+  %c2 = icmp uge i32 %y, 3
+  %sel1 = select i1 %c1, i32 1, i32 2
+  %sel2 = select i1 %c2, i32 0, i32 %sel1
+  %alloca = alloca [2 x [2 x [3 x i32]]], align 16, addrspace(5)
+  %gep.000 = getelementptr inbounds [2 x [2 x [3 x i32]]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 0, i32 0
+  %gep.001 = getelementptr inbounds [2 x [2 x [3 x i32]]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 0, i32 1
+  %gep.002 = getelementptr inbounds [2 x [2 x [3 x i32]]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 0, i32 2
+  %gep.010 = getelementptr inbounds [2 x [2 x [3 x i32]]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 1, i32 0
+  %gep.011 = getelementptr inbounds [2 x [2 x [3 x i32]]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 1, i32 1
+  %gep.012 = getelementptr inbounds [2 x [2 x [3 x i32]]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 1, i32 2
+  %gep.100 = getelementptr inbounds [2 x [2 x [3 x i32]]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 0, i32 0
+  %gep.101 = getelementptr inbounds [2 x [2 x [3 x i32]]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 0, i32 1
+  %gep.102 = getelementptr inbounds [2 x [2 x [3 x i32]]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 0, i32 2
+  %gep.110 = getelementptr inbounds [2 x [2 x [3 x i32]]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 1, i32 0
+  %gep.111 = getelementptr inbounds [2 x [2 x [3 x i32]]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 1, i32 1
+  %gep.112 = getelementptr inbounds [2 x [2 x [3 x i32]]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 1, i32 2
+  store i32 0, ptr addrspace(5) %gep.000
+  store i32 1, ptr addrspace(5) %gep.001
+  store i32 2, ptr addrspace(5) %gep.002
+  store i32 3, ptr addrspace(5) %gep.010
+  store i32 4, ptr addrspace(5) %gep.011
+  store i32 5, ptr addrspace(5) %gep.012
+  store i32 6, ptr addrspace(5) %gep.100
+  store i32 7, ptr addrspace(5) %gep.101
+  store i32 8, ptr addrspace(5) %gep.102
+  store i32 9, ptr addrspace(5) %gep.110
+  store i32 10, ptr addrspace(5) %gep.111
+  store i32 11, ptr addrspace(5) %gep.112
+  %gep = getelementptr inbounds [12 x i32], ptr addrspace(5) %alloca, i32 0, i32 %sel2
+  %load = load i32, ptr addrspace(5) %gep
+  store i32 %load, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @i16_2d_load_store(ptr %out, i32 %sel) {
+; CHECK-LABEL: define amdgpu_kernel void @i16_2d_load_store(
+; CHECK-SAME: ptr [[OUT:%.*]], i32 [[SEL:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 3, [[SEL]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <6 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5>, i32 [[TMP1]]
+; CHECK-NEXT:    store i16 [[TMP2]], ptr [[OUT]], align 2
+; CHECK-NEXT:    ret void
+;
+  %alloca = alloca [2 x [3 x i16]], align 16, addrspace(5)
+  %gep.00 = getelementptr inbounds [2 x [3 x i16]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 0
+  %gep.01 = getelementptr inbounds [2 x [3 x i16]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 1
+  %gep.02 = getelementptr inbounds [2 x [3 x i16]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 2
+  %gep.10 = getelementptr inbounds [2 x [3 x i16]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 0
+  %gep.11 = getelementptr inbounds [2 x [3 x i16]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 1
+  %gep.12 = getelementptr inbounds [2 x [3 x i16]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 2
+  store i16 0, ptr addrspace(5) %gep.00
+  store i16 1, ptr addrspace(5) %gep.01
+  store i16 2, ptr addrspace(5) %gep.02
+  store i16 3, ptr addrspace(5) %gep.10
+  store i16 4, ptr addrspace(5) %gep.11
+  store i16 5, ptr addrspace(5) %gep.12
+  %gep = getelementptr inbounds [2 x [3 x i16]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 %sel
+  %load = load i16, ptr addrspace(5) %gep
+  store i16 %load, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @float_2d_load_store(ptr %out, i32 %sel) {
+; CHECK-LABEL: define amdgpu_kernel void @float_2d_load_store(
+; CHECK-SAME: ptr [[OUT:%.*]], i32 [[SEL:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 3, [[SEL]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <6 x float> <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00, float 5.000000e+00>, i32 [[TMP1]]
+; CHECK-NEXT:    store float [[TMP2]], ptr [[OUT]], align 4
+; CHECK-NEXT:    ret void
+;
+  %alloca = alloca [2 x [3 x float]], align 16, addrspace(5)
+  %gep.00 = getelementptr inbounds [2 x [3 x float]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 0
+  %gep.01 = getelementptr inbounds [2 x [3 x float]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 1
+  %gep.02 = getelementptr inbounds [2 x [3 x float]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 2
+  %gep.10 = getelementptr inbounds [2 x [3 x float]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 0
+  %gep.11 = getelementptr inbounds [2 x [3 x float]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 1
+  %gep.12 = getelementptr inbounds [2 x [3 x float]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 2
+  store float 0.0, ptr addrspace(5) %gep.00
+  store float 1.0, ptr addrspace(5) %gep.01
+  store float 2.0, ptr addrspace(5) %gep.02
+  store float 3.0, ptr addrspace(5) %gep.10
+  store float 4.0, ptr addrspace(5) %gep.11
+  store float 5.0, ptr addrspace(5) %gep.12
+  %gep = getelementptr inbounds [2 x [3 x float]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 %sel
+  %load = load float, ptr addrspace(5) %gep
+  store float %load, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @ptr_2d_load_store(ptr %out, i32 %sel) {
+; CHECK-LABEL: define amdgpu_kernel void @ptr_2d_load_store(
+; CHECK-SAME: ptr [[OUT:%.*]], i32 [[SEL:%.*]]) {
+; CHECK-NEXT:    [[PTR_0:%.*]] = getelementptr inbounds ptr, ptr [[OUT]], i32 0
+; CHECK-NEXT:    [[PTR_1:%.*]] = getelementptr inbounds ptr, ptr [[OUT]], i32 1
+; CHECK-NEXT:    [[PTR_2:%.*]] = getelementptr inbounds ptr, ptr [[OUT]], i32 2
+; CHECK-NEXT:    [[PTR_3:%.*]] = getelementptr inbounds ptr, ptr [[OUT]], i32 3
+; CHECK-NEXT:    [[PTR_4:%.*]] = getelementptr inbounds ptr, ptr [[OUT]], i32 4
+; CHECK-NEXT:    [[PTR_5:%.*]] = getelementptr inbounds ptr, ptr [[OUT]], i32 5
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <6 x ptr> {{.*}}, ptr [[PTR_0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <6 x ptr> [[TMP1]], ptr [[PTR_1]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <6 x ptr> [[TMP2]], ptr [[PTR_2]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <6 x ptr> [[TMP3]], ptr [[PTR_3]], i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <6 x ptr> [[TMP4]], ptr [[PTR_4]], i32 4
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <6 x ptr> [[TMP5]], ptr [[PTR_5]], i32 5
+; CHECK-NEXT:    [[TMP7:%.*]] = add i32 3, [[SEL]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <6 x ptr> [[TMP6]], i32 [[TMP7]]
+; CHECK-NEXT:    store ptr [[TMP8]], ptr [[OUT]], align 8
+; CHECK-NEXT:    ret void
+;
+  %alloca = alloca [2 x [3 x ptr]], align 16, addrspace(5)
+  %gep.00 = getelementptr inbounds [2 x [3 x ptr]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 0
+  %gep.01 = getelementptr inbounds [2 x [3 x ptr]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 1
+  %gep.02 = getelementptr inbounds [2 x [3 x ptr]], ptr addrspace(5) %alloca, i32 0, i32 0, i32 2
+  %gep.10 = getelementptr inbounds [2 x [3 x ptr]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 0
+  %gep.11 = getelementptr inbounds [2 x [3 x ptr]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 1
+  %gep.12 = getelementptr inbounds [2 x [3 x ptr]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 2
+  %ptr.0 = getelementptr inbounds ptr, ptr %out, i32 0
+  %ptr.1 = getelementptr inbounds ptr, ptr %out, i32 1
+  %ptr.2 = getelementptr inbounds ptr, ptr %out, i32 2
+  %ptr.3 = getelementptr inbounds ptr, ptr %out, i32 3
+  %ptr.4 = getelementptr inbounds ptr, ptr %out, i32 4
+  %ptr.5 = getelementptr inbounds ptr, ptr %out, i32 5
+  store ptr %ptr.0, ptr addrspace(5) %gep.00
+  store ptr %ptr.1, ptr addrspace(5) %gep.01
+  store ptr %ptr.2, ptr addrspace(5) %gep.02
+  store ptr %ptr.3, ptr addrspace(5) %gep.10
+  store ptr %ptr.4, ptr addrspace(5) %gep.11
+  store ptr %ptr.5, ptr addrspace(5) %gep.12
+  %gep = getelementptr inbounds [2 x [3 x ptr]], ptr addrspace(5) %alloca, i32 0, i32 1, i32 %sel
+  %load = load ptr, ptr addrspace(5) %gep
+  store ptr %load, ptr %out
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
+declare i32 @llvm.amdgcn.workitem.id.y()
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-no-opts.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-no-opts.ll
index 28b923243b6db..ac28d9c20e2fa 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-no-opts.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-no-opts.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -mattr=+promote-alloca < %s | FileCheck -check-prefix=NOOPTS -check-prefix=ALL %s
-; RUN: llc -O1 -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -mattr=+promote-alloca < %s | FileCheck -check-prefix=OPTS -check-prefix=ALL %s
+; RUN: llc -O0 -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -disable-promote-alloca-to-vector=1 -mattr=+promote-alloca < %s | FileCheck -check-prefix=NOOPTS -check-prefix=ALL %s
+; RUN: llc -O1 -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -disable-promote-alloca-to-vector=1 -mattr=+promote-alloca < %s | FileCheck -check-prefix=OPTS -check-prefix=ALL %s
 
 ; ALL-LABEL: {{^}}promote_alloca_i32_array_array:
 ; NOOPTS: .amdhsa_group_segment_fixed_size 0
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-subvecs.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-subvecs.ll
index 7c5410004ed5b..e5d0c563b74c4 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-subvecs.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-subvecs.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
-; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -passes=amdgpu-promote-alloca -amdgpu-promote-alloca-to-vector-limit=512 < %s | FileCheck %s
+; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -passes=amdgpu-promote-alloca -amdgpu-promote-alloca-to-vector-limit=512 -amdgpu-promote-alloca-to-vector-max-regs=32 < %s | FileCheck %s
 
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-vgpr-ratio.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-vgpr-ratio.ll
new file mode 100644
index 0000000000000..e06b491f5986c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-vgpr-ratio.ll
@@ -0,0 +1,276 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-promote-alloca -disable-promote-alloca-to-lds=1 < %s | FileCheck --check-prefix=BASE --check-prefix=DEFAULT %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-promote-alloca -disable-promote-alloca-to-lds=1 -amdgpu-promote-alloca-to-vector-vgpr-ratio=2 < %s | FileCheck --check-prefix=BASE %s --check-prefix=RATIO2
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-promote-alloca -disable-promote-alloca-to-lds=1 -amdgpu-promote-alloca-to-vector-vgpr-ratio=8 < %s | FileCheck --check-prefix=BASE %s --check-prefix=RATIO8
+
+define amdgpu_kernel void @i32_24_elements(ptr %out) #0 {
+; DEFAULT-LABEL: define amdgpu_kernel void @i32_24_elements(
+; DEFAULT-SAME: ptr [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; DEFAULT-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; DEFAULT-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; DEFAULT-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; DEFAULT-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; DEFAULT-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; DEFAULT-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; DEFAULT-NEXT:    [[ALLOCA:%.*]] = alloca [24 x i32], align 16, addrspace(5)
+; DEFAULT-NEXT:    call void @llvm.memset.p5.i32(ptr addrspace(5) [[ALLOCA]], i8 0, i32 96, i1 false)
+; DEFAULT-NEXT:    [[GEP_0:%.*]] = getelementptr inbounds [24 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 0
+; DEFAULT-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds [24 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 20
+; DEFAULT-NEXT:    store i32 42, ptr addrspace(5) [[GEP_0]], align 4
+; DEFAULT-NEXT:    store i32 43, ptr addrspace(5) [[GEP_1]], align 4
+; DEFAULT-NEXT:    [[GEP:%.*]] = getelementptr inbounds [24 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[SEL2]]
+; DEFAULT-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(5) [[GEP]], align 4
+; DEFAULT-NEXT:    store i32 [[LOAD]], ptr [[OUT]], align 4
+; DEFAULT-NEXT:    ret void
+;
+; RATIO2-LABEL: define amdgpu_kernel void @i32_24_elements(
+; RATIO2-SAME: ptr [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; RATIO2-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; RATIO2-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; RATIO2-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; RATIO2-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; RATIO2-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; RATIO2-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; RATIO2-NEXT:    [[TMP1:%.*]] = extractelement <24 x i32> <i32 42, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 43, i32 0, i32 0, i32 0>, i32 [[SEL2]]
+; RATIO2-NEXT:    store i32 [[TMP1]], ptr [[OUT]], align 4
+; RATIO2-NEXT:    ret void
+;
+; RATIO8-LABEL: define amdgpu_kernel void @i32_24_elements(
+; RATIO8-SAME: ptr [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; RATIO8-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; RATIO8-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; RATIO8-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; RATIO8-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; RATIO8-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; RATIO8-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; RATIO8-NEXT:    [[ALLOCA:%.*]] = alloca [24 x i32], align 16, addrspace(5)
+; RATIO8-NEXT:    call void @llvm.memset.p5.i32(ptr addrspace(5) [[ALLOCA]], i8 0, i32 96, i1 false)
+; RATIO8-NEXT:    [[GEP_0:%.*]] = getelementptr inbounds [24 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 0
+; RATIO8-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds [24 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 20
+; RATIO8-NEXT:    store i32 42, ptr addrspace(5) [[GEP_0]], align 4
+; RATIO8-NEXT:    store i32 43, ptr addrspace(5) [[GEP_1]], align 4
+; RATIO8-NEXT:    [[GEP:%.*]] = getelementptr inbounds [24 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[SEL2]]
+; RATIO8-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(5) [[GEP]], align 4
+; RATIO8-NEXT:    store i32 [[LOAD]], ptr [[OUT]], align 4
+; RATIO8-NEXT:    ret void
+;
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %c1 = icmp uge i32 %x, 3
+  %c2 = icmp uge i32 %y, 3
+  %sel1 = select i1 %c1, i32 1, i32 2
+  %sel2 = select i1 %c2, i32 0, i32 %sel1
+  %alloca = alloca [24 x i32], align 16, addrspace(5)
+  call void @llvm.memset.p5.i32(ptr addrspace(5) %alloca, i8 0, i32 96, i1 false)
+  %gep.0 = getelementptr inbounds [24 x i32], ptr addrspace(5) %alloca, i32 0, i32 0
+  %gep.1 = getelementptr inbounds [24 x i32], ptr addrspace(5) %alloca, i32 0, i32 20
+  store i32 42, ptr addrspace(5) %gep.0
+  store i32 43, ptr addrspace(5) %gep.1
+  %gep = getelementptr inbounds [24 x i32], ptr addrspace(5) %alloca, i32 0, i32 %sel2
+  %load = load i32, ptr addrspace(5) %gep
+  store i32 %load, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @i32_24_elements_attrib(ptr %out) #1 {
+; DEFAULT-LABEL: define amdgpu_kernel void @i32_24_elements_attrib(
+; DEFAULT-SAME: ptr [[OUT:%.*]]) #[[ATTR1:[0-9]+]] {
+; DEFAULT-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; DEFAULT-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; DEFAULT-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; DEFAULT-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; DEFAULT-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; DEFAULT-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; DEFAULT-NEXT:    [[TMP1:%.*]] = extractelement <24 x i32> <i32 42, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 43, i32 0, i32 0, i32 0>, i32 [[SEL2]]
+; DEFAULT-NEXT:    store i32 [[TMP1]], ptr [[OUT]], align 4
+; DEFAULT-NEXT:    ret void
+;
+; RATIO2-LABEL: define amdgpu_kernel void @i32_24_elements_attrib(
+; RATIO2-SAME: ptr [[OUT:%.*]]) #[[ATTR1:[0-9]+]] {
+; RATIO2-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; RATIO2-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; RATIO2-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; RATIO2-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; RATIO2-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; RATIO2-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; RATIO2-NEXT:    [[TMP1:%.*]] = extractelement <24 x i32> <i32 42, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 43, i32 0, i32 0, i32 0>, i32 [[SEL2]]
+; RATIO2-NEXT:    store i32 [[TMP1]], ptr [[OUT]], align 4
+; RATIO2-NEXT:    ret void
+;
+; RATIO8-LABEL: define amdgpu_kernel void @i32_24_elements_attrib(
+; RATIO8-SAME: ptr [[OUT:%.*]]) #[[ATTR1:[0-9]+]] {
+; RATIO8-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; RATIO8-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; RATIO8-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; RATIO8-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; RATIO8-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; RATIO8-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; RATIO8-NEXT:    [[ALLOCA:%.*]] = alloca [24 x i32], align 16, addrspace(5)
+; RATIO8-NEXT:    call void @llvm.memset.p5.i32(ptr addrspace(5) [[ALLOCA]], i8 0, i32 96, i1 false)
+; RATIO8-NEXT:    [[GEP_0:%.*]] = getelementptr inbounds [24 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 0
+; RATIO8-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds [24 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 20
+; RATIO8-NEXT:    store i32 42, ptr addrspace(5) [[GEP_0]], align 4
+; RATIO8-NEXT:    store i32 43, ptr addrspace(5) [[GEP_1]], align 4
+; RATIO8-NEXT:    [[GEP:%.*]] = getelementptr inbounds [24 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[SEL2]]
+; RATIO8-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(5) [[GEP]], align 4
+; RATIO8-NEXT:    store i32 [[LOAD]], ptr [[OUT]], align 4
+; RATIO8-NEXT:    ret void
+;
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %c1 = icmp uge i32 %x, 3
+  %c2 = icmp uge i32 %y, 3
+  %sel1 = select i1 %c1, i32 1, i32 2
+  %sel2 = select i1 %c2, i32 0, i32 %sel1
+  %alloca = alloca [24 x i32], align 16, addrspace(5)
+  call void @llvm.memset.p5.i32(ptr addrspace(5) %alloca, i8 0, i32 96, i1 false)
+  %gep.0 = getelementptr inbounds [24 x i32], ptr addrspace(5) %alloca, i32 0, i32 0
+  %gep.1 = getelementptr inbounds [24 x i32], ptr addrspace(5) %alloca, i32 0, i32 20
+  store i32 42, ptr addrspace(5) %gep.0
+  store i32 43, ptr addrspace(5) %gep.1
+  %gep = getelementptr inbounds [24 x i32], ptr addrspace(5) %alloca, i32 0, i32 %sel2
+  %load = load i32, ptr addrspace(5) %gep
+  store i32 %load, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @i32_16_elements(ptr %out) #0 {
+; DEFAULT-LABEL: define amdgpu_kernel void @i32_16_elements(
+; DEFAULT-SAME: ptr [[OUT:%.*]]) #[[ATTR0]] {
+; DEFAULT-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; DEFAULT-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; DEFAULT-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; DEFAULT-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; DEFAULT-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; DEFAULT-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; DEFAULT-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> <i32 42, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 43>, i32 [[SEL2]]
+; DEFAULT-NEXT:    store i32 [[TMP1]], ptr [[OUT]], align 4
+; DEFAULT-NEXT:    ret void
+;
+; RATIO2-LABEL: define amdgpu_kernel void @i32_16_elements(
+; RATIO2-SAME: ptr [[OUT:%.*]]) #[[ATTR0]] {
+; RATIO2-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; RATIO2-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; RATIO2-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; RATIO2-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; RATIO2-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; RATIO2-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; RATIO2-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> <i32 42, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 43>, i32 [[SEL2]]
+; RATIO2-NEXT:    store i32 [[TMP1]], ptr [[OUT]], align 4
+; RATIO2-NEXT:    ret void
+;
+; RATIO8-LABEL: define amdgpu_kernel void @i32_16_elements(
+; RATIO8-SAME: ptr [[OUT:%.*]]) #[[ATTR0]] {
+; RATIO8-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; RATIO8-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; RATIO8-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; RATIO8-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; RATIO8-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; RATIO8-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; RATIO8-NEXT:    [[ALLOCA:%.*]] = alloca [16 x i32], align 16, addrspace(5)
+; RATIO8-NEXT:    call void @llvm.memset.p5.i32(ptr addrspace(5) [[ALLOCA]], i8 0, i32 64, i1 false)
+; RATIO8-NEXT:    [[GEP_0:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 0
+; RATIO8-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 15
+; RATIO8-NEXT:    store i32 42, ptr addrspace(5) [[GEP_0]], align 4
+; RATIO8-NEXT:    store i32 43, ptr addrspace(5) [[GEP_1]], align 4
+; RATIO8-NEXT:    [[GEP:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[SEL2]]
+; RATIO8-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(5) [[GEP]], align 4
+; RATIO8-NEXT:    store i32 [[LOAD]], ptr [[OUT]], align 4
+; RATIO8-NEXT:    ret void
+;
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %c1 = icmp uge i32 %x, 3
+  %c2 = icmp uge i32 %y, 3
+  %sel1 = select i1 %c1, i32 1, i32 2
+  %sel2 = select i1 %c2, i32 0, i32 %sel1
+  %alloca = alloca [16 x i32], align 16, addrspace(5)
+  call void @llvm.memset.p5.i32(ptr addrspace(5) %alloca, i8 0, i32 64, i1 false)
+  %gep.0 = getelementptr inbounds [16 x i32], ptr addrspace(5) %alloca, i32 0, i32 0
+  %gep.1 = getelementptr inbounds [16 x i32], ptr addrspace(5) %alloca, i32 0, i32 15
+  store i32 42, ptr addrspace(5) %gep.0
+  store i32 43, ptr addrspace(5) %gep.1
+  %gep = getelementptr inbounds [16 x i32], ptr addrspace(5) %alloca, i32 0, i32 %sel2
+  %load = load i32, ptr addrspace(5) %gep
+  store i32 %load, ptr %out
+  ret void
+}
+
+define amdgpu_kernel void @i32_16_elements_attrib(ptr %out) #2 {
+; DEFAULT-LABEL: define amdgpu_kernel void @i32_16_elements_attrib(
+; DEFAULT-SAME: ptr [[OUT:%.*]]) #[[ATTR2:[0-9]+]] {
+; DEFAULT-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; DEFAULT-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; DEFAULT-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; DEFAULT-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; DEFAULT-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; DEFAULT-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; DEFAULT-NEXT:    [[ALLOCA:%.*]] = alloca [16 x i32], align 16, addrspace(5)
+; DEFAULT-NEXT:    call void @llvm.memset.p5.i32(ptr addrspace(5) [[ALLOCA]], i8 0, i32 64, i1 false)
+; DEFAULT-NEXT:    [[GEP_0:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 0
+; DEFAULT-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 15
+; DEFAULT-NEXT:    store i32 42, ptr addrspace(5) [[GEP_0]], align 4
+; DEFAULT-NEXT:    store i32 43, ptr addrspace(5) [[GEP_1]], align 4
+; DEFAULT-NEXT:    [[GEP:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[SEL2]]
+; DEFAULT-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(5) [[GEP]], align 4
+; DEFAULT-NEXT:    store i32 [[LOAD]], ptr [[OUT]], align 4
+; DEFAULT-NEXT:    ret void
+;
+; RATIO2-LABEL: define amdgpu_kernel void @i32_16_elements_attrib(
+; RATIO2-SAME: ptr [[OUT:%.*]]) #[[ATTR2:[0-9]+]] {
+; RATIO2-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; RATIO2-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; RATIO2-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; RATIO2-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; RATIO2-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; RATIO2-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; RATIO2-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> <i32 42, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 43>, i32 [[SEL2]]
+; RATIO2-NEXT:    store i32 [[TMP1]], ptr [[OUT]], align 4
+; RATIO2-NEXT:    ret void
+;
+; RATIO8-LABEL: define amdgpu_kernel void @i32_16_elements_attrib(
+; RATIO8-SAME: ptr [[OUT:%.*]]) #[[ATTR2:[0-9]+]] {
+; RATIO8-NEXT:    [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; RATIO8-NEXT:    [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; RATIO8-NEXT:    [[C1:%.*]] = icmp uge i32 [[X]], 3
+; RATIO8-NEXT:    [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; RATIO8-NEXT:    [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; RATIO8-NEXT:    [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; RATIO8-NEXT:    [[ALLOCA:%.*]] = alloca [16 x i32], align 16, addrspace(5)
+; RATIO8-NEXT:    call void @llvm.memset.p5.i32(ptr addrspace(5) [[ALLOCA]], i8 0, i32 64, i1 false)
+; RATIO8-NEXT:    [[GEP_0:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 0
+; RATIO8-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 15
+; RATIO8-NEXT:    store i32 42, ptr addrspace(5) [[GEP_0]], align 4
+; RATIO8-NEXT:    store i32 43, ptr addrspace(5) [[GEP_1]], align 4
+; RATIO8-NEXT:    [[GEP:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[SEL2]]
+; RATIO8-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(5) [[GEP]], align 4
+; RATIO8-NEXT:    store i32 [[LOAD]], ptr [[OUT]], align 4
+; RATIO8-NEXT:    ret void
+;
+  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %c1 = icmp uge i32 %x, 3
+  %c2 = icmp uge i32 %y, 3
+  %sel1 = select i1 %c1, i32 1, i32 2
+  %sel2 = select i1 %c2, i32 0, i32 %sel1
+  %alloca = alloca [16 x i32], align 16, addrspace(5)
+  call void @llvm.memset.p5.i32(ptr addrspace(5) %alloca, i8 0, i32 64, i1 false)
+  %gep.0 = getelementptr inbounds [16 x i32], ptr addrspace(5) %alloca, i32 0, i32 0
+  %gep.1 = getelementptr inbounds [16 x i32], ptr addrspace(5) %alloca, i32 0, i32 15
+  store i32 42, ptr addrspace(5) %gep.0
+  store i32 43, ptr addrspace(5) %gep.1
+  %gep = getelementptr inbounds [16 x i32], ptr addrspace(5) %alloca, i32 0, i32 %sel2
+  %load = load i32, ptr addrspace(5) %gep
+  store i32 %load, ptr %out
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
+declare i32 @llvm.amdgcn.workitem.id.y()
+declare void @llvm.memset.p5.i32(ptr addrspace(5) nocapture writeonly, i8, i32, i1 immarg)
+
+attributes #0 = { nounwind "amdgpu-promote-alloca-to-vector-max-regs"="24" "amdgpu-waves-per-eu"="4,4" }
+attributes #1 = { nounwind "amdgpu-promote-alloca-to-vector-max-regs"="24" "amdgpu-waves-per-eu"="4,4" "amdgpu-promote-alloca-to-vector-vgpr-ratio"="2" }
+attributes #2 = { nounwind "amdgpu-promote-alloca-to-vector-max-regs"="24" "amdgpu-waves-per-eu"="4,4" "amdgpu-promote-alloca-to-vector-vgpr-ratio"="8" }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; BASE: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
index 33720ea9b28e6..ee89bf406c2a3 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
@@ -1776,20 +1776,20 @@ entry:
 define amdgpu_kernel void @DiffBase(ptr addrspace(1) %buffer1,
 ; GFX8-LABEL: DiffBase:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX8-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX8-NEXT:    s_mov_b32 s42, -1
-; GFX8-NEXT:    s_mov_b32 s43, 0xe80000
-; GFX8-NEXT:    s_add_u32 s40, s40, s11
-; GFX8-NEXT:    s_addc_u32 s41, s41, 0
+; GFX8-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX8-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX8-NEXT:    s_mov_b32 s50, -1
+; GFX8-NEXT:    s_mov_b32 s51, 0xe80000
+; GFX8-NEXT:    s_add_u32 s48, s48, s11
+; GFX8-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX8-NEXT:    s_getpc_b64 s[0:1]
 ; GFX8-NEXT:    s_add_u32 s0, s0, _Z13get_global_idj@gotpcrel32@lo+4
 ; GFX8-NEXT:    s_addc_u32 s1, s1, _Z13get_global_idj@gotpcrel32@hi+12
 ; GFX8-NEXT:    s_load_dwordx4 s[36:39], s[4:5], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX8-NEXT:    s_mov_b64 s[0:1], s[40:41]
+; GFX8-NEXT:    s_mov_b64 s[0:1], s[48:49]
 ; GFX8-NEXT:    v_mov_b32_e32 v31, v0
-; GFX8-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX8-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX8-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX8-NEXT:    s_mov_b32 s32, 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1839,20 +1839,20 @@ define amdgpu_kernel void @DiffBase(ptr addrspace(1) %buffer1,
 ;
 ; GFX9-LABEL: DiffBase:
 ; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s42, -1
-; GFX9-NEXT:    s_mov_b32 s43, 0xe00000
-; GFX9-NEXT:    s_add_u32 s40, s40, s11
-; GFX9-NEXT:    s_addc_u32 s41, s41, 0
+; GFX9-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s50, -1
+; GFX9-NEXT:    s_mov_b32 s51, 0xe00000
+; GFX9-NEXT:    s_add_u32 s48, s48, s11
+; GFX9-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX9-NEXT:    s_getpc_b64 s[0:1]
 ; GFX9-NEXT:    s_add_u32 s0, s0, _Z13get_global_idj@gotpcrel32@lo+4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, _Z13get_global_idj@gotpcrel32@hi+12
 ; GFX9-NEXT:    s_load_dwordx4 s[36:39], s[4:5], 0x24
 ; GFX9-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
@@ -1898,12 +1898,12 @@ define amdgpu_kernel void @DiffBase(ptr addrspace(1) %buffer1,
 ;
 ; GFX10-LABEL: DiffBase:
 ; GFX10:       ; %bb.0: ; %entry
-; GFX10-NEXT:    s_mov_b32 s40, SCRATCH_RSRC_DWORD0
-; GFX10-NEXT:    s_mov_b32 s41, SCRATCH_RSRC_DWORD1
-; GFX10-NEXT:    s_mov_b32 s42, -1
-; GFX10-NEXT:    s_mov_b32 s43, 0x31c16000
-; GFX10-NEXT:    s_add_u32 s40, s40, s11
-; GFX10-NEXT:    s_addc_u32 s41, s41, 0
+; GFX10-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX10-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX10-NEXT:    s_mov_b32 s50, -1
+; GFX10-NEXT:    s_mov_b32 s51, 0x31c16000
+; GFX10-NEXT:    s_add_u32 s48, s48, s11
+; GFX10-NEXT:    s_addc_u32 s49, s49, 0
 ; GFX10-NEXT:    s_getpc_b64 s[0:1]
 ; GFX10-NEXT:    s_add_u32 s0, s0, _Z13get_global_idj@gotpcrel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s1, s1, _Z13get_global_idj@gotpcrel32@hi+12
@@ -1911,8 +1911,8 @@ define amdgpu_kernel void @DiffBase(ptr addrspace(1) %buffer1,
 ; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
 ; GFX10-NEXT:    s_load_dwordx4 s[36:39], s[4:5], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
-; GFX10-NEXT:    s_mov_b64 s[0:1], s[40:41]
-; GFX10-NEXT:    s_mov_b64 s[2:3], s[42:43]
+; GFX10-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX10-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX10-NEXT:    s_mov_b32 s32, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
diff --git a/llvm/test/CodeGen/AMDGPU/ps-shader-arg-count.ll b/llvm/test/CodeGen/AMDGPU/ps-shader-arg-count.ll
index f3c3acc317ce1..5b9b0feea9900 100644
--- a/llvm/test/CodeGen/AMDGPU/ps-shader-arg-count.ll
+++ b/llvm/test/CodeGen/AMDGPU/ps-shader-arg-count.ll
@@ -6,8 +6,8 @@
 define dllexport amdgpu_ps { <4 x float> } @_amdgpu_ps_1_arg(i32 inreg %arg, i32 inreg %arg1, i32 inreg %arg2, <2 x float> %arg3, <2 x float> %arg4, <2 x float> %arg5, <3 x float> %arg6, <2 x float> %arg7, <2 x float> %arg8, <2 x float> %arg9, float %arg10, float %arg11, float %arg12, float %arg13, float %arg14, i32 %arg15, i32 %arg16, i32 %arg17, i32 %arg18) local_unnamed_addr #0 {
 .entry:
   %i1 = extractelement <2 x float> %arg3, i32 1
-  %ret1 = insertelement <4 x float> undef, float %i1, i32 0
-  %ret2 = insertvalue { <4 x float> } undef, <4 x float> %ret1, 0
+  %ret1 = insertelement <4 x float> poison, float %i1, i32 0
+  %ret2 = insertvalue { <4 x float> } poison, <4 x float> %ret1, 0
   ret { <4 x float> } %ret2
 }
 
@@ -18,10 +18,10 @@ define dllexport amdgpu_ps { <4 x float> } @_amdgpu_ps_3_arg(i32 inreg %arg, i32
   %i1 = extractelement <2 x float> %arg3, i32 1
   %i2 = extractelement <2 x float> %arg4, i32 0
   %i3 = extractelement <2 x float> %arg5, i32 1
-  %ret1 = insertelement <4 x float> undef, float %i1, i32 0
+  %ret1 = insertelement <4 x float> poison, float %i1, i32 0
   %ret1.1 = insertelement <4 x float> %ret1, float %i2, i32 1
   %ret1.2 = insertelement <4 x float> %ret1.1, float %i3, i32 2
-  %ret2 = insertvalue { <4 x float> } undef, <4 x float> %ret1.2, 0
+  %ret2 = insertvalue { <4 x float> } poison, <4 x float> %ret1.2, 0
   ret { <4 x float> } %ret2
 }
 
@@ -31,9 +31,9 @@ define dllexport amdgpu_ps { <4 x float> } @_amdgpu_ps_2_arg_gap(i32 inreg %arg,
 .entry:
   %i1 = extractelement <2 x float> %arg3, i32 1
   %i3 = extractelement <2 x float> %arg5, i32 1
-  %ret1 = insertelement <4 x float> undef, float %i1, i32 0
+  %ret1 = insertelement <4 x float> poison, float %i1, i32 0
   %ret1.2 = insertelement <4 x float> %ret1, float %i3, i32 1
-  %ret2 = insertvalue { <4 x float> } undef, <4 x float> %ret1.2, 0
+  %ret2 = insertvalue { <4 x float> } poison, <4 x float> %ret1.2, 0
   ret { <4 x float> } %ret2
 }
 
@@ -45,9 +45,9 @@ define dllexport amdgpu_ps { <4 x float> } @_amdgpu_ps_2_arg_no_pack(i32 inreg %
 .entry:
   %i1 = extractelement <2 x float> %arg3, i32 1
   %i3 = extractelement <2 x float> %arg5, i32 1
-  %ret1 = insertelement <4 x float> undef, float %i1, i32 0
+  %ret1 = insertelement <4 x float> poison, float %i1, i32 0
   %ret1.2 = insertelement <4 x float> %ret1, float %i3, i32 1
-  %ret2 = insertvalue { <4 x float> } undef, <4 x float> %ret1.2, 0
+  %ret2 = insertvalue { <4 x float> } poison, <4 x float> %ret1.2, 0
   ret { <4 x float> } %ret2
 }
 
@@ -63,17 +63,17 @@ define dllexport amdgpu_ps { <4 x float>, <4 x float>, <4 x float>, <4 x float>
   %i6 = extractelement <2 x float> %arg8, i32 0
   %i7 = extractelement <2 x float> %arg9, i32 1
 
-  %ret1 = insertelement <4 x float> undef, float %i1, i32 0
+  %ret1 = insertelement <4 x float> poison, float %i1, i32 0
   %ret1.1 = insertelement <4 x float> %ret1, float %i2, i32 1
   %ret1.2 = insertelement <4 x float> %ret1.1, float %i3, i32 2
   %ret1.3 = insertelement <4 x float> %ret1.2, float %i4, i32 3
 
-  %ret2 = insertelement <4 x float> undef, float %i5, i32 0
+  %ret2 = insertelement <4 x float> poison, float %i5, i32 0
   %ret2.1 = insertelement <4 x float> %ret2, float %i6, i32 1
   %ret2.2 = insertelement <4 x float> %ret2.1, float %i7, i32 2
   %ret2.3 = insertelement <4 x float> %ret2.2, float %arg10, i32 3
 
-  %ret3 = insertelement <4 x float> undef, float %arg11, i32 0
+  %ret3 = insertelement <4 x float> poison, float %arg11, i32 0
   %ret3.1 = insertelement <4 x float> %ret3, float %arg12, i32 1
   %ret3.2 = insertelement <4 x float> %ret3.1, float %arg13, i32 2
   %ret3.3 = insertelement <4 x float> %ret3.2, float %arg14, i32 3
@@ -83,12 +83,12 @@ define dllexport amdgpu_ps { <4 x float>, <4 x float>, <4 x float>, <4 x float>
   %arg17.f = bitcast i32 %arg17 to float
   %arg18.f = bitcast i32 %arg18 to float
 
-  %ret4 = insertelement <4 x float> undef, float %arg15.f, i32 0
+  %ret4 = insertelement <4 x float> poison, float %arg15.f, i32 0
   %ret4.1 = insertelement <4 x float> %ret4, float %arg16.f, i32 1
   %ret4.2 = insertelement <4 x float> %ret4.1, float %arg17.f, i32 2
   %ret4.3 = insertelement <4 x float> %ret4.2, float %arg18.f, i32 3
 
-  %ret.res1 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } undef, <4 x float> %ret1.3, 0
+  %ret.res1 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } poison, <4 x float> %ret1.3, 0
   %ret.res2 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } %ret.res1, <4 x float> %ret2.3, 1
   %ret.res3 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } %ret.res2, <4 x float> %ret3.3, 2
   %ret.res  = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } %ret.res3, <4 x float> %ret4.3, 3
@@ -109,17 +109,17 @@ define dllexport amdgpu_ps { <4 x float>, <4 x float>, <4 x float>, <4 x float>
   %i6 = extractelement <2 x float> %arg8, i32 0
   %i7 = extractelement <2 x float> %arg9, i32 1
 
-  %ret1 = insertelement <4 x float> undef, float %i1, i32 0
+  %ret1 = insertelement <4 x float> poison, float %i1, i32 0
   %ret1.1 = insertelement <4 x float> %ret1, float %i2, i32 1
   %ret1.2 = insertelement <4 x float> %ret1.1, float %i3, i32 2
   %ret1.3 = insertelement <4 x float> %ret1.2, float %i4, i32 3
 
-  %ret2 = insertelement <4 x float> undef, float %i5, i32 0
+  %ret2 = insertelement <4 x float> poison, float %i5, i32 0
   %ret2.1 = insertelement <4 x float> %ret2, float %i6, i32 1
   %ret2.2 = insertelement <4 x float> %ret2.1, float %i7, i32 2
   %ret2.3 = insertelement <4 x float> %ret2.2, float %arg10, i32 3
 
-  %ret3 = insertelement <4 x float> undef, float %arg11, i32 0
+  %ret3 = insertelement <4 x float> poison, float %arg11, i32 0
   %ret3.1 = insertelement <4 x float> %ret3, float %arg12, i32 1
   %ret3.2 = insertelement <4 x float> %ret3.1, float %arg13, i32 2
   %ret3.3 = insertelement <4 x float> %ret3.2, float %arg14, i32 3
@@ -129,12 +129,12 @@ define dllexport amdgpu_ps { <4 x float>, <4 x float>, <4 x float>, <4 x float>
   %arg17.f = bitcast i32 %arg17 to float
   %arg18.f = bitcast i32 %arg18 to float
 
-  %ret4 = insertelement <4 x float> undef, float %arg15.f, i32 0
+  %ret4 = insertelement <4 x float> poison, float %arg15.f, i32 0
   %ret4.1 = insertelement <4 x float> %ret4, float %arg16.f, i32 1
   %ret4.2 = insertelement <4 x float> %ret4.1, float %arg17.f, i32 2
   %ret4.3 = insertelement <4 x float> %ret4.2, float %arg18.f, i32 3
 
-  %ret.res1 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } undef, <4 x float> %ret1.3, 0
+  %ret.res1 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } poison, <4 x float> %ret1.3, 0
   %ret.res2 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } %ret.res1, <4 x float> %ret2.3, 1
   %ret.res3 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } %ret.res2, <4 x float> %ret3.3, 2
   %ret.res  = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } %ret.res3, <4 x float> %ret4.3, 3
@@ -155,17 +155,17 @@ define dllexport amdgpu_ps { <4 x float>, <4 x float>, <4 x float>, <4 x float>
   %i6 = extractelement <2 x float> %arg8, i32 0
   %i7 = extractelement <2 x float> %arg9, i32 1
 
-  %ret1 = insertelement <4 x float> undef, float %i1, i32 0
+  %ret1 = insertelement <4 x float> poison, float %i1, i32 0
   %ret1.1 = insertelement <4 x float> %ret1, float %i2, i32 1
   %ret1.2 = insertelement <4 x float> %ret1.1, float %i3, i32 2
   %ret1.3 = insertelement <4 x float> %ret1.2, float %i4, i32 3
 
-  %ret2 = insertelement <4 x float> undef, float %i5, i32 0
+  %ret2 = insertelement <4 x float> poison, float %i5, i32 0
   %ret2.1 = insertelement <4 x float> %ret2, float %i6, i32 1
   %ret2.2 = insertelement <4 x float> %ret2.1, float %i7, i32 2
   %ret2.3 = insertelement <4 x float> %ret2.2, float %arg10, i32 3
 
-  %ret3 = insertelement <4 x float> undef, float %arg11, i32 0
+  %ret3 = insertelement <4 x float> poison, float %arg11, i32 0
   %ret3.1 = insertelement <4 x float> %ret3, float %arg12, i32 1
   %ret3.2 = insertelement <4 x float> %ret3.1, float %arg13, i32 2
   %ret3.3 = insertelement <4 x float> %ret3.2, float %arg14, i32 3
@@ -178,12 +178,12 @@ define dllexport amdgpu_ps { <4 x float>, <4 x float>, <4 x float>, <4 x float>
   %arg15_16.f = fadd float %arg15.f, %arg16.f
   %arg17_18.f = fadd float %arg17.f, %arg18.f
 
-  %ret4 = insertelement <4 x float> undef, float %extra_arg1, i32 0
+  %ret4 = insertelement <4 x float> poison, float %extra_arg1, i32 0
   %ret4.1 = insertelement <4 x float> %ret4, float %extra_arg2, i32 1
   %ret4.2 = insertelement <4 x float> %ret4.1, float %arg15_16.f, i32 2
   %ret4.3 = insertelement <4 x float> %ret4.2, float %arg17_18.f, i32 3
 
-  %ret.res1 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } undef, <4 x float> %ret1.3, 0
+  %ret.res1 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } poison, <4 x float> %ret1.3, 0
   %ret.res2 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } %ret.res1, <4 x float> %ret2.3, 1
   %ret.res3 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } %ret.res2, <4 x float> %ret3.3, 2
   %ret.res  = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } %ret.res3, <4 x float> %ret4.3, 3
@@ -212,10 +212,10 @@ define dllexport amdgpu_ps { <4 x float> } @_amdgpu_ps_all_unused_ia0(i32 inreg
 ; CHECK: NumVgprs: 4
 define dllexport amdgpu_ps { <4 x float> } @_amdgpu_ps_all_unused_extra_used(i32 inreg %arg, i32 inreg %arg1, i32 inreg %arg2, <2 x float> %arg3, <2 x float> %arg4, <2 x float> %arg5, <3 x float> %arg6, <2 x float> %arg7, <2 x float> %arg8, <2 x float> %arg9, float %arg10, float %arg11, float %arg12, float %arg13, float %arg14, i32 %arg15, i32 %arg16, i32 %arg17, i32 %arg18, float %extra_arg1, float %extra_arg2) local_unnamed_addr #0 {
 .entry:
-  %ret4.1 = insertelement <4 x float> undef, float %extra_arg1, i32 0
+  %ret4.1 = insertelement <4 x float> poison, float %extra_arg1, i32 0
   %ret4.2 = insertelement <4 x float> %ret4.1, float %extra_arg2, i32 1
 
-  %ret.res  = insertvalue { <4 x float> } undef, <4 x float> %ret4.2, 0
+  %ret.res  = insertvalue { <4 x float> } poison, <4 x float> %ret4.2, 0
 
   ret { <4 x float> } %ret.res
 }
@@ -224,11 +224,11 @@ define dllexport amdgpu_ps { <4 x float> } @_amdgpu_ps_all_unused_extra_used(i32
 ; CHECK: NumVgprs: 5
 define dllexport amdgpu_ps { <4 x float> } @_amdgpu_ps_part_unused_extra_used(i32 inreg %arg, i32 inreg %arg1, i32 inreg %arg2, <2 x float> %arg3, <2 x float> %arg4, <2 x float> %arg5, <3 x float> %arg6, <2 x float> %arg7, <2 x float> %arg8, <2 x float> %arg9, float %arg10, float %arg11, float %arg12, float %arg13, float %arg14, i32 %arg15, i32 %arg16, i32 %arg17, i32 %arg18, float %extra_arg1, float %extra_arg2) local_unnamed_addr #0 {
 .entry:
-  %ret4.1 = insertelement <4 x float> undef, float %arg14, i32 0
+  %ret4.1 = insertelement <4 x float> poison, float %arg14, i32 0
   %ret4.2 = insertelement <4 x float> %ret4.1, float %extra_arg1, i32 1
   %ret4.3 = insertelement <4 x float> %ret4.2, float %extra_arg2, i32 2
 
-  %ret.res  = insertvalue { <4 x float> } undef, <4 x float> %ret4.3, 0
+  %ret.res  = insertvalue { <4 x float> } poison, <4 x float> %ret4.3, 0
 
   ret { <4 x float> } %ret.res
 }
@@ -237,11 +237,11 @@ define dllexport amdgpu_ps { <4 x float> } @_amdgpu_ps_part_unused_extra_used(i3
 ; CHECK: NumVgprs: 7
 define dllexport amdgpu_ps { <4 x float> } @_amdgpu_ps_part_unused_extra_unused(i32 inreg %arg, i32 inreg %arg1, i32 inreg %arg2, <2 x float> %arg3, <2 x float> %arg4, <2 x float> %arg5, <3 x float> %arg6, <2 x float> %arg7, <2 x float> %arg8, <2 x float> %arg9, float %arg10, float %arg11, float %arg12, float %arg13, float %arg14, i32 %arg15, i32 %arg16, i32 %arg17, i32 %arg18, float %extra_arg1, float %extra_arg2) local_unnamed_addr #0 {
 .entry:
-  %ret4.1 = insertelement <4 x float> undef, float %arg12, i32 0
+  %ret4.1 = insertelement <4 x float> poison, float %arg12, i32 0
   %ret4.2 = insertelement <4 x float> %ret4.1, float %arg13, i32 1
   %ret4.3 = insertelement <4 x float> %ret4.2, float %arg14, i32 2
 
-  %ret.res  = insertvalue { <4 x float> } undef, <4 x float> %ret4.3, 0
+  %ret.res  = insertvalue { <4 x float> } poison, <4 x float> %ret4.3, 0
 
   ret { <4 x float> } %ret.res
 }
@@ -259,10 +259,10 @@ define dllexport amdgpu_ps { <4 x float> } @_amdgpu_ps_all_unused_extra_unused(i
 ; CHECK: NumVgprs: 26
 define dllexport amdgpu_ps { <4 x float> } @_amdgpu_ps_all_unused_extra_used_no_packing(i32 inreg %arg, i32 inreg %arg1, i32 inreg %arg2, <2 x float> %arg3, <2 x float> %arg4, <2 x float> %arg5, <3 x float> %arg6, <2 x float> %arg7, <2 x float> %arg8, <2 x float> %arg9, float %arg10, float %arg11, float %arg12, float %arg13, float %arg14, i32 %arg15, i32 %arg16, i32 %arg17, i32 %arg18, float %extra_arg1, float %extra_arg2) local_unnamed_addr #2 {
 .entry:
-  %ret4.1 = insertelement <4 x float> undef, float %extra_arg1, i32 0
+  %ret4.1 = insertelement <4 x float> poison, float %extra_arg1, i32 0
   %ret4.2 = insertelement <4 x float> %ret4.1, float %extra_arg2, i32 1
 
-  %ret.res  = insertvalue { <4 x float> } undef, <4 x float> %ret4.2, 0
+  %ret.res  = insertvalue { <4 x float> } poison, <4 x float> %ret4.2, 0
 
   ret { <4 x float> } %ret.res
 }
@@ -287,17 +287,17 @@ define dllexport amdgpu_ps { <4 x float>, <4 x float>, <4 x float>, <4 x float>
   %i6 = extractelement <2 x float> %arg8, i32 0
   %i7 = extractelement <2 x float> %arg9, i32 1
 
-  %ret1 = insertelement <4 x float> undef, float %i1, i32 0
+  %ret1 = insertelement <4 x float> poison, float %i1, i32 0
   %ret1.1 = insertelement <4 x float> %ret1, float %i2, i32 1
   %ret1.2 = insertelement <4 x float> %ret1.1, float %i3, i32 2
   %ret1.3 = insertelement <4 x float> %ret1.2, float %i4, i32 3
 
-  %ret2 = insertelement <4 x float> undef, float %i5, i32 0
+  %ret2 = insertelement <4 x float> poison, float %i5, i32 0
   %ret2.1 = insertelement <4 x float> %ret2, float %i6, i32 1
   %ret2.2 = insertelement <4 x float> %ret2.1, float %i7, i32 2
   %ret2.3 = insertelement <4 x float> %ret2.2, float %arg10, i32 3
 
-  %ret3 = insertelement <4 x float> undef, float %arg11, i32 0
+  %ret3 = insertelement <4 x float> poison, float %arg11, i32 0
   %ret3.1 = insertelement <4 x float> %ret3, float %arg12, i32 1
   %ret3.2 = insertelement <4 x float> %ret3.1, float %arg13, i32 2
   %ret3.3 = insertelement <4 x float> %ret3.2, float %arg14, i32 3
@@ -305,12 +305,12 @@ define dllexport amdgpu_ps { <4 x float>, <4 x float>, <4 x float>, <4 x float>
   %arg15.f = bitcast i32 %arg15 to float
   %arg16.f = bitcast i32 %arg16 to float
 
-  %ret4 = insertelement <4 x float> undef, float %extra_arg1, i32 0
+  %ret4 = insertelement <4 x float> poison, float %extra_arg1, i32 0
   %ret4.1 = insertelement <4 x float> %ret4, float %extra_arg2, i32 1
   %ret4.2 = insertelement <4 x float> %ret4.1, float %arg15.f, i32 2
   %ret4.3 = insertelement <4 x float> %ret4.2, float %arg16.f, i32 3
 
-  %ret.res1 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } undef, <4 x float> %ret1.3, 0
+  %ret.res1 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } poison, <4 x float> %ret1.3, 0
   %ret.res2 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } %ret.res1, <4 x float> %ret2.3, 1
   %ret.res3 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } %ret.res2, <4 x float> %ret3.3, 2
   %ret.res  = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } %ret.res3, <4 x float> %ret4.3, 3
@@ -331,25 +331,25 @@ define dllexport amdgpu_ps { <4 x float>, <4 x float>, <4 x float>, <4 x float>
   %i6 = extractelement <2 x float> %arg8, i32 0
   %i7 = extractelement <2 x float> %arg9, i32 1
 
-  %ret1 = insertelement <4 x float> undef, float %i1, i32 0
+  %ret1 = insertelement <4 x float> poison, float %i1, i32 0
   %ret1.1 = insertelement <4 x float> %ret1, float %i2, i32 1
   %ret1.2 = insertelement <4 x float> %ret1.1, float %i3, i32 2
   %ret1.3 = insertelement <4 x float> %ret1.2, float %i4, i32 3
 
-  %ret2 = insertelement <4 x float> undef, float %i5, i32 0
+  %ret2 = insertelement <4 x float> poison, float %i5, i32 0
   %ret2.1 = insertelement <4 x float> %ret2, float %i6, i32 1
   %ret2.2 = insertelement <4 x float> %ret2.1, float %i7, i32 2
   %ret2.3 = insertelement <4 x float> %ret2.2, float %arg10, i32 3
 
-  %ret3 = insertelement <4 x float> undef, float %arg11, i32 0
+  %ret3 = insertelement <4 x float> poison, float %arg11, i32 0
   %ret3.1 = insertelement <4 x float> %ret3, float %arg12, i32 1
   %ret3.2 = insertelement <4 x float> %ret3.1, float %arg13, i32 2
   %ret3.3 = insertelement <4 x float> %ret3.2, float %arg14, i32 3
 
-  %ret4 = insertelement <4 x float> undef, float %extra_arg1, i32 0
+  %ret4 = insertelement <4 x float> poison, float %extra_arg1, i32 0
   %ret4.1 = insertelement <4 x float> %ret4, float %extra_arg2, i32 1
 
-  %ret.res1 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } undef, <4 x float> %ret1.3, 0
+  %ret.res1 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } poison, <4 x float> %ret1.3, 0
   %ret.res2 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } %ret.res1, <4 x float> %ret2.3, 1
   %ret.res3 = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } %ret.res2, <4 x float> %ret3.3, 2
   %ret.res  = insertvalue { < 4 x float>, <4 x float>, <4 x float>, <4 x float> } %ret.res3, <4 x float> %ret4.1, 3
diff --git a/llvm/test/CodeGen/AMDGPU/pv-packing.ll b/llvm/test/CodeGen/AMDGPU/pv-packing.ll
index 1d6c751278c68..a83b28d297905 100644
--- a/llvm/test/CodeGen/AMDGPU/pv-packing.ll
+++ b/llvm/test/CodeGen/AMDGPU/pv-packing.ll
@@ -42,12 +42,12 @@ main_body:
   %17 = fadd float %16, %8
   %18 = fmul float %11, %11
   %19 = fadd float %18, %0
-  %20 = insertelement <4 x float> undef, float %13, i32 0
+  %20 = insertelement <4 x float> poison, float %13, i32 0
   %21 = insertelement <4 x float> %20, float %15, i32 1
   %22 = insertelement <4 x float> %21, float %17, i32 2
   %23 = insertelement <4 x float> %22, float %19, i32 3
   %24 = call float @llvm.r600.dot4(<4 x float> %23, <4 x float> %10)
-  %25 = insertelement <4 x float> undef, float %24, i32 0
+  %25 = insertelement <4 x float> poison, float %24, i32 0
   call void @llvm.r600.store.swizzle(<4 x float> %25, i32 0, i32 2)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/pv.ll b/llvm/test/CodeGen/AMDGPU/pv.ll
index 856457aa58cbf..96db6ec436a2d 100644
--- a/llvm/test/CodeGen/AMDGPU/pv.ll
+++ b/llvm/test/CodeGen/AMDGPU/pv.ll
@@ -92,11 +92,11 @@ main_body:
   %tmp97 = extractelement <4 x float> %tmp96, i32 3
   %tmp98 = fmul float %tmp15, %tmp97
   %tmp99 = fadd float %tmp98, %tmp83
-  %tmp100 = insertelement <4 x float> undef, float %tmp16, i32 0
+  %tmp100 = insertelement <4 x float> poison, float %tmp16, i32 0
   %tmp101 = insertelement <4 x float> %tmp100, float %tmp17, i32 1
   %tmp102 = insertelement <4 x float> %tmp101, float %tmp18, i32 2
   %tmp103 = insertelement <4 x float> %tmp102, float 0.000000e+00, i32 3
-  %tmp104 = insertelement <4 x float> undef, float %tmp16, i32 0
+  %tmp104 = insertelement <4 x float> poison, float %tmp16, i32 0
   %tmp105 = insertelement <4 x float> %tmp104, float %tmp17, i32 1
   %tmp106 = insertelement <4 x float> %tmp105, float %tmp18, i32 2
   %tmp107 = insertelement <4 x float> %tmp106, float 0.000000e+00, i32 3
@@ -132,11 +132,11 @@ main_body:
   %tmp129 = extractelement <4 x float> %tmp128, i32 1
   %tmp130 = load <4 x float>, ptr addrspace(8) getelementptr ([1024 x <4 x float>], ptr addrspace(8) null, i64 0, i32 5)
   %tmp131 = extractelement <4 x float> %tmp130, i32 2
-  %tmp132 = insertelement <4 x float> undef, float %tmp111, i32 0
+  %tmp132 = insertelement <4 x float> poison, float %tmp111, i32 0
   %tmp133 = insertelement <4 x float> %tmp132, float %tmp112, i32 1
   %tmp134 = insertelement <4 x float> %tmp133, float %tmp113, i32 2
   %tmp135 = insertelement <4 x float> %tmp134, float 0.000000e+00, i32 3
-  %tmp136 = insertelement <4 x float> undef, float %tmp127, i32 0
+  %tmp136 = insertelement <4 x float> poison, float %tmp127, i32 0
   %tmp137 = insertelement <4 x float> %tmp136, float %tmp129, i32 1
   %tmp138 = insertelement <4 x float> %tmp137, float %tmp131, i32 2
   %tmp139 = insertelement <4 x float> %tmp138, float 0.000000e+00, i32 3
@@ -147,11 +147,11 @@ main_body:
   %tmp144 = extractelement <4 x float> %tmp143, i32 1
   %tmp145 = load <4 x float>, ptr addrspace(8) getelementptr ([1024 x <4 x float>], ptr addrspace(8) null, i64 0, i32 7)
   %tmp146 = extractelement <4 x float> %tmp145, i32 2
-  %tmp147 = insertelement <4 x float> undef, float %tmp111, i32 0
+  %tmp147 = insertelement <4 x float> poison, float %tmp111, i32 0
   %tmp148 = insertelement <4 x float> %tmp147, float %tmp112, i32 1
   %tmp149 = insertelement <4 x float> %tmp148, float %tmp113, i32 2
   %tmp150 = insertelement <4 x float> %tmp149, float 0.000000e+00, i32 3
-  %tmp151 = insertelement <4 x float> undef, float %tmp142, i32 0
+  %tmp151 = insertelement <4 x float> poison, float %tmp142, i32 0
   %tmp152 = insertelement <4 x float> %tmp151, float %tmp144, i32 1
   %tmp153 = insertelement <4 x float> %tmp152, float %tmp146, i32 2
   %tmp154 = insertelement <4 x float> %tmp153, float 0.000000e+00, i32 3
@@ -211,12 +211,12 @@ main_body:
   %clamp.i4 = call float @llvm.minnum.f32(float %max.0.i3, float 1.000000e+00)
   %max.0.i1 = call float @llvm.maxnum.f32(float %tmp204, float 0.000000e+00)
   %clamp.i2 = call float @llvm.minnum.f32(float %max.0.i1, float 1.000000e+00)
-  %tmp205 = insertelement <4 x float> undef, float %tmp87, i32 0
+  %tmp205 = insertelement <4 x float> poison, float %tmp87, i32 0
   %tmp206 = insertelement <4 x float> %tmp205, float %tmp91, i32 1
   %tmp207 = insertelement <4 x float> %tmp206, float %tmp95, i32 2
   %tmp208 = insertelement <4 x float> %tmp207, float %tmp99, i32 3
   call void @llvm.r600.store.swizzle(<4 x float> %tmp208, i32 60, i32 1)
-  %tmp209 = insertelement <4 x float> undef, float %clamp.i6, i32 0
+  %tmp209 = insertelement <4 x float> poison, float %clamp.i6, i32 0
   %tmp210 = insertelement <4 x float> %tmp209, float %clamp.i4, i32 1
   %tmp211 = insertelement <4 x float> %tmp210, float %clamp.i2, i32 2
   %tmp212 = insertelement <4 x float> %tmp211, float %clamp.i8, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/r600-encoding.ll b/llvm/test/CodeGen/AMDGPU/r600-encoding.ll
index 2580c58f56057..bd20af36c37c1 100644
--- a/llvm/test/CodeGen/AMDGPU/r600-encoding.ll
+++ b/llvm/test/CodeGen/AMDGPU/r600-encoding.ll
@@ -15,7 +15,7 @@ entry:
   %r0 = extractelement <4 x float> %reg0, i32 0
   %r1 = extractelement <4 x float> %reg0, i32 1
   %r2 = fmul float %r0, %r1
-  %vec = insertelement <4 x float> undef, float %r2, i32 0
+  %vec = insertelement <4 x float> poison, float %r2, i32 0
   call void @llvm.r600.store.swizzle(<4 x float> %vec, i32 0, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/r600-export-fix.ll b/llvm/test/CodeGen/AMDGPU/r600-export-fix.ll
index 21a7050c2c320..4740a21ed1313 100644
--- a/llvm/test/CodeGen/AMDGPU/r600-export-fix.ll
+++ b/llvm/test/CodeGen/AMDGPU/r600-export-fix.ll
@@ -125,42 +125,42 @@ main_body:
   %79 = extractelement <4 x float> %78, i32 1
   %80 = load <4 x float>, ptr addrspace(8) getelementptr ([1024 x <4 x float>], ptr addrspace(8) null, i64 0, i32 3)
   %81 = extractelement <4 x float> %80, i32 2
-  %82 = insertelement <4 x float> undef, float %51, i32 0
+  %82 = insertelement <4 x float> poison, float %51, i32 0
   %83 = insertelement <4 x float> %82, float %55, i32 1
   %84 = insertelement <4 x float> %83, float %59, i32 2
   %85 = insertelement <4 x float> %84, float %63, i32 3
   call void @llvm.r600.store.swizzle(<4 x float> %85, i32 60, i32 1)
-  %86 = insertelement <4 x float> undef, float 0.000000e+00, i32 0
+  %86 = insertelement <4 x float> poison, float 0.000000e+00, i32 0
   %87 = insertelement <4 x float> %86, float 0.000000e+00, i32 1
   %88 = insertelement <4 x float> %87, float 0.000000e+00, i32 2
   %89 = insertelement <4 x float> %88, float 0.000000e+00, i32 3
   call void @llvm.r600.store.swizzle(<4 x float> %89, i32 0, i32 2)
-  %90 = insertelement <4 x float> undef, float 0.000000e+00, i32 0
+  %90 = insertelement <4 x float> poison, float 0.000000e+00, i32 0
   %91 = insertelement <4 x float> %90, float 0.000000e+00, i32 1
   %92 = insertelement <4 x float> %91, float 0.000000e+00, i32 2
   %93 = insertelement <4 x float> %92, float 0.000000e+00, i32 3
   call void @llvm.r600.store.swizzle(<4 x float> %93, i32 1, i32 2)
-  %94 = insertelement <4 x float> undef, float 0.000000e+00, i32 0
+  %94 = insertelement <4 x float> poison, float 0.000000e+00, i32 0
   %95 = insertelement <4 x float> %94, float %65, i32 1
   %96 = insertelement <4 x float> %95, float %67, i32 2
   %97 = insertelement <4 x float> %96, float %69, i32 3
   call void @llvm.r600.store.swizzle(<4 x float> %97, i32 2, i32 2)
-  %98 = insertelement <4 x float> undef, float %77, i32 0
+  %98 = insertelement <4 x float> poison, float %77, i32 0
   %99 = insertelement <4 x float> %98, float %79, i32 1
   %100 = insertelement <4 x float> %99, float %81, i32 2
   %101 = insertelement <4 x float> %100, float %71, i32 3
   call void @llvm.r600.store.swizzle(<4 x float> %101, i32 3, i32 2)
-  %102 = insertelement <4 x float> undef, float %73, i32 0
+  %102 = insertelement <4 x float> poison, float %73, i32 0
   %103 = insertelement <4 x float> %102, float %75, i32 1
   %104 = insertelement <4 x float> %103, float 0.000000e+00, i32 2
   %105 = insertelement <4 x float> %104, float 0.000000e+00, i32 3
   call void @llvm.r600.store.swizzle(<4 x float> %105, i32 4, i32 2)
-  %106 = insertelement <4 x float> undef, float 0.000000e+00, i32 0
+  %106 = insertelement <4 x float> poison, float 0.000000e+00, i32 0
   %107 = insertelement <4 x float> %106, float 0.000000e+00, i32 1
   %108 = insertelement <4 x float> %107, float 0.000000e+00, i32 2
   %109 = insertelement <4 x float> %108, float 0.000000e+00, i32 3
   call void @llvm.r600.store.swizzle(<4 x float> %109, i32 5, i32 2)
-  %110 = insertelement <4 x float> undef, float 0.000000e+00, i32 0
+  %110 = insertelement <4 x float> poison, float 0.000000e+00, i32 0
   %111 = insertelement <4 x float> %110, float 0.000000e+00, i32 1
   %112 = insertelement <4 x float> %111, float 0.000000e+00, i32 2
   %113 = insertelement <4 x float> %112, float 0.000000e+00, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/r600-infinite-loop-bug-while-reorganizing-vector.ll b/llvm/test/CodeGen/AMDGPU/r600-infinite-loop-bug-while-reorganizing-vector.ll
index 54cc43ce36b9f..d815c18a289e1 100644
--- a/llvm/test/CodeGen/AMDGPU/r600-infinite-loop-bug-while-reorganizing-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/r600-infinite-loop-bug-while-reorganizing-vector.ll
@@ -6,7 +6,7 @@ main_body:
   %tmp2 = extractelement <4 x float> %arg, i32 1
   %tmp3 = extractelement <4 x float> %arg, i32 2
   %tmp4 = extractelement <4 x float> %arg, i32 3
-  %tmp5 = insertelement <4 x float> undef, float %tmp, i32 0
+  %tmp5 = insertelement <4 x float> poison, float %tmp, i32 0
   %tmp6 = insertelement <4 x float> %tmp5, float %tmp2, i32 1
   %tmp7 = insertelement <4 x float> %tmp6, float %tmp3, i32 2
   %tmp8 = insertelement <4 x float> %tmp7, float %tmp4, i32 3
@@ -21,7 +21,7 @@ main_body:
   %tmp17 = fadd float %tmp16, 1.500000e+00
   %tmp18 = fmul float %tmp11, %tmp15
   %tmp19 = fadd float %tmp18, 1.500000e+00
-  %tmp20 = insertelement <4 x float> undef, float %tmp19, i32 0
+  %tmp20 = insertelement <4 x float> poison, float %tmp19, i32 0
   %tmp21 = insertelement <4 x float> %tmp20, float %tmp17, i32 1
   %tmp22 = insertelement <4 x float> %tmp21, float %tmp13, i32 2
   %tmp23 = insertelement <4 x float> %tmp22, float %tmp4, i32 3
@@ -29,14 +29,14 @@ main_body:
   %tmp25 = extractelement <4 x float> %tmp23, i32 1
   %tmp26 = extractelement <4 x float> %tmp23, i32 2
   %tmp27 = extractelement <4 x float> %tmp23, i32 3
-  %tmp28 = insertelement <4 x float> undef, float %tmp24, i32 0
+  %tmp28 = insertelement <4 x float> poison, float %tmp24, i32 0
   %tmp29 = insertelement <4 x float> %tmp28, float %tmp25, i32 1
   %tmp30 = insertelement <4 x float> %tmp29, float %tmp26, i32 2
   %tmp31 = insertelement <4 x float> %tmp30, float %tmp27, i32 3
   %tmp32 = shufflevector <4 x float> %tmp31, <4 x float> %tmp31, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %tmp33 = call <4 x float> @llvm.r600.texc(<4 x float> %tmp32, i32 0, i32 0, i32 0, i32 16, i32 0, i32 1, i32 1, i32 1, i32 1)
   %tmp34 = extractelement <4 x float> %tmp33, i32 0
-  %tmp35 = insertelement <4 x float> undef, float %tmp34, i32 0
+  %tmp35 = insertelement <4 x float> poison, float %tmp34, i32 0
   %tmp36 = insertelement <4 x float> %tmp35, float %tmp34, i32 1
   %tmp37 = insertelement <4 x float> %tmp36, float %tmp34, i32 2
   %tmp38 = insertelement <4 x float> %tmp37, float 1.000000e+00, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/r600cfg.ll b/llvm/test/CodeGen/AMDGPU/r600cfg.ll
index 663b13a913ee2..e5a30298703e4 100644
--- a/llvm/test/CodeGen/AMDGPU/r600cfg.ll
+++ b/llvm/test/CodeGen/AMDGPU/r600cfg.ll
@@ -28,27 +28,27 @@ LOOP:                                             ; preds = %LOOP47, %main_body
   br i1 %15, label %IF41, label %ENDIF40
 
 IF41:                                             ; preds = %LOOP
-  %16 = insertelement <4 x float> undef, float %0, i32 0
+  %16 = insertelement <4 x float> poison, float %0, i32 0
   %17 = insertelement <4 x float> %16, float %temp8.0, i32 1
   %18 = insertelement <4 x float> %17, float %temp12.0, i32 2
   %19 = insertelement <4 x float> %18, float 0.000000e+00, i32 3
   call void @llvm.r600.store.stream.output(<4 x float> %19, i32 0, i32 0, i32 1)
-  %20 = insertelement <4 x float> undef, float %0, i32 0
+  %20 = insertelement <4 x float> poison, float %0, i32 0
   %21 = insertelement <4 x float> %20, float %temp8.0, i32 1
   %22 = insertelement <4 x float> %21, float %temp12.0, i32 2
   %23 = insertelement <4 x float> %22, float 0.000000e+00, i32 3
   call void @llvm.r600.store.stream.output(<4 x float> %23, i32 0, i32 0, i32 2)
-  %24 = insertelement <4 x float> undef, float %0, i32 0
+  %24 = insertelement <4 x float> poison, float %0, i32 0
   %25 = insertelement <4 x float> %24, float %temp8.0, i32 1
   %26 = insertelement <4 x float> %25, float %temp12.0, i32 2
   %27 = insertelement <4 x float> %26, float 0.000000e+00, i32 3
   call void @llvm.r600.store.stream.output(<4 x float> %27, i32 0, i32 0, i32 4)
-  %28 = insertelement <4 x float> undef, float 0.000000e+00, i32 0
+  %28 = insertelement <4 x float> poison, float 0.000000e+00, i32 0
   %29 = insertelement <4 x float> %28, float 0.000000e+00, i32 1
   %30 = insertelement <4 x float> %29, float 0.000000e+00, i32 2
   %31 = insertelement <4 x float> %30, float 0.000000e+00, i32 3
   call void @llvm.r600.store.swizzle(<4 x float> %31, i32 60, i32 1)
-  %32 = insertelement <4 x float> undef, float %0, i32 0
+  %32 = insertelement <4 x float> poison, float %0, i32 0
   %33 = insertelement <4 x float> %32, float %temp8.0, i32 1
   %34 = insertelement <4 x float> %33, float %temp12.0, i32 2
   %35 = insertelement <4 x float> %34, float 0.000000e+00, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
index c6ee557d970cd..4a0bb6ceccd3f 100644
--- a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
+++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
@@ -41,63 +41,102 @@ body:             |
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $sgpr34_sgpr35 = V_CMP_GT_I32_e64 1, undef %18:vgpr_32, implicit $exec
-  ; CHECK-NEXT:   renamable $sgpr36_sgpr37 = V_CMP_EQ_U32_e64 0, undef %18:vgpr_32, implicit $exec
-  ; CHECK-NEXT:   renamable $sgpr38_sgpr39 = V_CMP_NE_U32_e64 0, undef %18:vgpr_32, implicit $exec
-  ; CHECK-NEXT:   renamable $sgpr40_sgpr41 = V_CMP_GT_I32_e64 0, undef %18:vgpr_32, implicit $exec
-  ; CHECK-NEXT:   renamable $sgpr60 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $sgpr42_sgpr43 = V_CMP_EQ_U32_e64 undef $sgpr4, undef %18:vgpr_32, implicit $exec
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vreg_1024_align2 = COPY renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75, implicit $exec
-  ; CHECK-NEXT:   renamable $sgpr44_sgpr45 = V_CMP_NE_U32_e64 1, undef %18:vgpr_32, implicit $exec
-  ; CHECK-NEXT:   renamable $sgpr61 = S_MOV_B32 1083786240
+  ; CHECK-NEXT:   renamable $sgpr18_sgpr19 = V_CMP_GT_I32_e64 1, undef %18:vgpr_32, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr12_sgpr13 = V_CMP_EQ_U32_e64 0, undef %18:vgpr_32, implicit $exec
+  ; CHECK-NEXT:   SI_SPILL_S64_SAVE killed renamable $sgpr12_sgpr13, %stack.2, implicit $exec, implicit $sgpr32 :: (store (s64) into %stack.2, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr12_sgpr13 = V_CMP_NE_U32_e64 0, undef %18:vgpr_32, implicit $exec
+  ; CHECK-NEXT:   SI_SPILL_S64_SAVE killed renamable $sgpr12_sgpr13, %stack.3, implicit $exec, implicit $sgpr32 :: (store (s64) into %stack.3, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr34_sgpr35 = V_CMP_GT_I32_e64 0, undef %18:vgpr_32, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr56 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $sgpr12_sgpr13 = V_CMP_EQ_U32_e64 undef $sgpr4, undef %18:vgpr_32, implicit $exec
+  ; CHECK-NEXT:   SI_SPILL_S64_SAVE killed renamable $sgpr12_sgpr13, %stack.4, implicit $exec, implicit $sgpr32 :: (store (s64) into %stack.4, align 4, addrspace 5)
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vreg_1024_align2 = COPY renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr100_sgpr101 = V_CMP_NE_U32_e64 1, undef %18:vgpr_32, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr57 = S_MOV_B32 1083786240
+  ; CHECK-NEXT:   SI_SPILL_S1024_SAVE renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71, %stack.1, implicit $exec, implicit $sgpr32 :: (store (s1024) into %stack.1, align 4, addrspace 5)
   ; CHECK-NEXT:   S_BRANCH %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.17(0x40000000)
-  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75:0x0000000F00000000
+  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr100_sgpr101
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   $vcc = S_AND_B64 $exec, renamable $sgpr44_sgpr45, implicit-def dead $scc
+  ; CHECK-NEXT:   $vcc = S_AND_B64 $exec, renamable $sgpr100_sgpr101, implicit-def dead $scc
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vreg_1024_align2 = COPY [[COPY]]
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.2, implicit $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.17
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   successors: %bb.11(0x40000000), %bb.5(0x40000000)
-  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75:0x0000000F00000000
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $sgpr64 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr65 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr66 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr67 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr68 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr69 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr70 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr71 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr72 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr73 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr74 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr75 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr76 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr77 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr78 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr79 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr80 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr81 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr82 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr83 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr84 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr85 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr86 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr87 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr88 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr89 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr90 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr91 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr92 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr93 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr94 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   renamable $sgpr95 = COPY renamable $sgpr60
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vreg_1024_align2 = COPY killed renamable $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit $exec
+  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr100_sgpr101
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr40 = COPY renamable $sgpr72
+  ; CHECK-NEXT:   renamable $sgpr41 = COPY renamable $sgpr72
+  ; CHECK-NEXT:   renamable $sgpr42 = COPY renamable $sgpr72
+  ; CHECK-NEXT:   renamable $sgpr43 = COPY renamable $sgpr72
+  ; CHECK-NEXT:   renamable $sgpr44 = COPY renamable $sgpr72
+  ; CHECK-NEXT:   renamable $sgpr45 = COPY renamable $sgpr72
+  ; CHECK-NEXT:   renamable $sgpr46 = COPY renamable $sgpr72
+  ; CHECK-NEXT:   renamable $sgpr47 = COPY renamable $sgpr72
+  ; CHECK-NEXT:   renamable $sgpr48 = COPY renamable $sgpr72
+  ; CHECK-NEXT:   renamable $sgpr49 = COPY renamable $sgpr72
+  ; CHECK-NEXT:   renamable $sgpr50 = COPY renamable $sgpr72
+  ; CHECK-NEXT:   renamable $sgpr51 = COPY killed renamable $sgpr72
+  ; CHECK-NEXT:   renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 = COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+  ; CHECK-NEXT:   renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr56 = COPY killed renamable $sgpr72
+  ; CHECK-NEXT:   renamable $sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = COPY killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55
+  ; CHECK-NEXT:   renamable $sgpr52 = COPY renamable $sgpr56
+  ; CHECK-NEXT:   renamable $sgpr53 = COPY killed renamable $sgpr76
+  ; CHECK-NEXT:   renamable $sgpr56_sgpr57 = COPY renamable $sgpr52_sgpr53
+  ; CHECK-NEXT:   renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 = COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+  ; CHECK-NEXT:   renamable $sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = COPY killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55
+  ; CHECK-NEXT:   renamable $sgpr52_sgpr53 = COPY renamable $sgpr56_sgpr57
+  ; CHECK-NEXT:   renamable $sgpr54 = COPY killed renamable $sgpr76
+  ; CHECK-NEXT:   renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47 = COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51
+  ; CHECK-NEXT:   renamable $sgpr48_sgpr49_sgpr50 = COPY renamable $sgpr52_sgpr53_sgpr54
+  ; CHECK-NEXT:   renamable $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr52_sgpr53_sgpr54 = COPY renamable $sgpr48_sgpr49_sgpr50
+  ; CHECK-NEXT:   renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = COPY killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47
+  ; CHECK-NEXT:   renamable $sgpr55 = COPY killed renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr56 = COPY killed renamable $sgpr72
+  ; CHECK-NEXT:   renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr57 = COPY killed renamable $sgpr84
+  ; CHECK-NEXT:   renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr58 = COPY killed renamable $sgpr84
+  ; CHECK-NEXT:   renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr59 = COPY killed renamable $sgpr84
+  ; CHECK-NEXT:   renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr60 = COPY killed renamable $sgpr84
+  ; CHECK-NEXT:   renamable $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr61 = COPY killed renamable $sgpr80
+  ; CHECK-NEXT:   renamable $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr62 = COPY killed renamable $sgpr80
+  ; CHECK-NEXT:   renamable $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr63 = COPY killed renamable $sgpr80
+  ; CHECK-NEXT:   renamable $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr64 = COPY killed renamable $sgpr80
+  ; CHECK-NEXT:   renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr65 = COPY killed renamable $sgpr84
+  ; CHECK-NEXT:   renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr66 = COPY killed renamable $sgpr84
+  ; CHECK-NEXT:   renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr67 = COPY killed renamable $sgpr84
+  ; CHECK-NEXT:   renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr68 = COPY killed renamable $sgpr84
+  ; CHECK-NEXT:   renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = COPY killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55
+  ; CHECK-NEXT:   renamable $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63 = COPY renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
+  ; CHECK-NEXT:   renamable $sgpr64 = COPY renamable $sgpr68
+  ; CHECK-NEXT:   renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr65 = COPY killed renamable $sgpr84
+  ; CHECK-NEXT:   renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr66 = COPY killed renamable $sgpr84
+  ; CHECK-NEXT:   renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr67 = COPY killed renamable $sgpr84
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vreg_1024_align2 = COPY killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, implicit $exec
   ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.11, implicit $exec
   ; CHECK-NEXT:   S_BRANCH %bb.5
   ; CHECK-NEXT: {{  $}}
@@ -126,111 +165,117 @@ body:             |
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.5:
   ; CHECK-NEXT:   successors: %bb.12(0x40000000), %bb.6(0x40000000)
-  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75:0x0000000F00000000
+  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr100_sgpr101
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $sgpr12_sgpr13 = S_AND_B64 renamable $sgpr38_sgpr39, undef renamable $sgpr46_sgpr47, implicit-def dead $scc
-  ; CHECK-NEXT:   renamable $sgpr46_sgpr47 = V_CMP_GT_I32_e64 0, undef %18:vgpr_32, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr12_sgpr13 = SI_SPILL_S64_RESTORE %stack.3, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.3, align 4, addrspace 5)
+  ; CHECK-NEXT:   renamable $sgpr12_sgpr13 = S_AND_B64 killed renamable $sgpr12_sgpr13, undef renamable $sgpr54_sgpr55, implicit-def dead $scc
+  ; CHECK-NEXT:   renamable $sgpr54_sgpr55 = V_CMP_GT_I32_e64 0, undef %18:vgpr_32, implicit $exec
   ; CHECK-NEXT:   $exec = S_MOV_B64_term killed renamable $sgpr12_sgpr13
   ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.12, implicit $exec
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.6:
   ; CHECK-NEXT:   successors: %bb.7(0x80000000)
-  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75:0x0000000F00000000
+  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr54_sgpr55, $sgpr100_sgpr101
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   dead [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr40_sgpr41, implicit $exec
+  ; CHECK-NEXT:   dead [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr34_sgpr35, implicit $exec
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.7:
   ; CHECK-NEXT:   successors: %bb.8(0x80000000)
-  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75:0x0000000F00000000
+  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr54_sgpr55, $sgpr100_sgpr101
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $sgpr48_sgpr49 = nofpexcept V_CMP_NLT_F64_e64 0, undef $sgpr4_sgpr5, 0, undef %29:vreg_64_align2, 0, implicit $mode, implicit $exec
-  ; CHECK-NEXT:   renamable $sgpr50_sgpr51 = nofpexcept V_CMP_NLT_F64_e64 0, 4607182418800017408, 0, undef %29:vreg_64_align2, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr64_sgpr65 = nofpexcept V_CMP_NLT_F64_e64 0, undef $sgpr4_sgpr5, 0, undef %29:vreg_64_align2, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr66_sgpr67 = nofpexcept V_CMP_NLT_F64_e64 0, 4607182418800017408, 0, undef %29:vreg_64_align2, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   dead [[V_INDIRECT_REG_READ_GPR_IDX_B32_V32_:%[0-9]+]]:vgpr_32 = V_INDIRECT_REG_READ_GPR_IDX_B32_V32 [[COPY1]], undef $sgpr14, 11, implicit-def $m0, implicit $m0, implicit $exec
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.8:
   ; CHECK-NEXT:   successors: %bb.10(0x40000000), %bb.9(0x40000000)
-  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75:0x0000000F00000000
+  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr100_sgpr101
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   $vcc = S_AND_B64 $exec, renamable $sgpr48_sgpr49, implicit-def dead $scc
+  ; CHECK-NEXT:   $vcc = S_AND_B64 $exec, renamable $sgpr64_sgpr65, implicit-def dead $scc
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.10, implicit $vcc
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.9:
   ; CHECK-NEXT:   successors: %bb.10(0x40000000), %bb.17(0x40000000)
-  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75:0x0000000F00000000
+  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr100_sgpr101
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY renamable $sgpr60_sgpr61, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY killed renamable $sgpr84_sgpr85, implicit $exec
   ; CHECK-NEXT:   GLOBAL_STORE_DWORDX2_SADDR undef %18:vgpr_32, [[COPY2]], undef renamable $sgpr4_sgpr5, 0, 0, implicit $exec :: (store (s64), addrspace 1)
-  ; CHECK-NEXT:   [[V_CNDMASK_B32_e64_1:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr34_sgpr35, implicit $exec
+  ; CHECK-NEXT:   [[V_CNDMASK_B32_e64_1:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr18_sgpr19, implicit $exec
   ; CHECK-NEXT:   dead renamable $sgpr12_sgpr13 = V_CMP_NE_U32_e64 1, [[V_CNDMASK_B32_e64_1]], implicit $exec
-  ; CHECK-NEXT:   renamable $sgpr58 = S_ADD_U32 renamable $sgpr8, 32, implicit-def dead $scc
+  ; CHECK-NEXT:   renamable $sgpr82 = S_ADD_U32 renamable $sgpr8, 32, implicit-def dead $scc
   ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
-  ; CHECK-NEXT:   renamable $sgpr52_sgpr53 = COPY killed renamable $sgpr4_sgpr5
-  ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY renamable $sgpr52_sgpr53
-  ; CHECK-NEXT:   renamable $sgpr54_sgpr55 = COPY killed renamable $sgpr6_sgpr7
-  ; CHECK-NEXT:   $sgpr6_sgpr7 = COPY renamable $sgpr54_sgpr55
-  ; CHECK-NEXT:   renamable $sgpr56_sgpr57 = COPY killed renamable $sgpr10_sgpr11
-  ; CHECK-NEXT:   $sgpr10_sgpr11 = COPY renamable $sgpr56_sgpr57
+  ; CHECK-NEXT:   renamable $sgpr68_sgpr69 = COPY killed renamable $sgpr4_sgpr5
+  ; CHECK-NEXT:   $sgpr4_sgpr5 = COPY renamable $sgpr68_sgpr69
+  ; CHECK-NEXT:   renamable $sgpr70_sgpr71 = COPY killed renamable $sgpr6_sgpr7
+  ; CHECK-NEXT:   $sgpr6_sgpr7 = COPY renamable $sgpr70_sgpr71
+  ; CHECK-NEXT:   renamable $sgpr80_sgpr81 = COPY killed renamable $sgpr10_sgpr11
+  ; CHECK-NEXT:   $sgpr10_sgpr11 = COPY renamable $sgpr80_sgpr81
   ; CHECK-NEXT:   $sgpr12 = COPY renamable $sgpr14
   ; CHECK-NEXT:   $sgpr13 = COPY renamable $sgpr15
-  ; CHECK-NEXT:   renamable $sgpr62 = COPY killed renamable $sgpr8
+  ; CHECK-NEXT:   renamable $sgpr84 = COPY killed renamable $sgpr8
   ; CHECK-NEXT:   renamable $sgpr33 = COPY killed renamable $sgpr16
-  ; CHECK-NEXT:   renamable $sgpr59 = COPY killed renamable $sgpr15
-  ; CHECK-NEXT:   renamable $sgpr63 = COPY killed renamable $sgpr14
+  ; CHECK-NEXT:   renamable $sgpr83 = COPY killed renamable $sgpr15
+  ; CHECK-NEXT:   renamable $sgpr85 = COPY killed renamable $sgpr14
+  ; CHECK-NEXT:   renamable $sgpr48_sgpr49 = COPY killed renamable $sgpr18_sgpr19
   ; CHECK-NEXT:   dead $sgpr30_sgpr31 = SI_CALL undef renamable $sgpr12_sgpr13, 0, csr_amdgpu_gfx90ainsts, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13
   ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
   ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
-  ; CHECK-NEXT:   $sgpr8_sgpr9 = COPY renamable $sgpr58_sgpr59
+  ; CHECK-NEXT:   $sgpr8_sgpr9 = COPY renamable $sgpr82_sgpr83
   ; CHECK-NEXT:   dead $sgpr30_sgpr31 = SI_CALL undef renamable $sgpr12_sgpr13, 0, csr_amdgpu_gfx90ainsts, implicit $sgpr8_sgpr9
-  ; CHECK-NEXT:   renamable $sgpr14 = COPY killed renamable $sgpr63
-  ; CHECK-NEXT:   renamable $sgpr15 = COPY killed renamable $sgpr59
+  ; CHECK-NEXT:   renamable $sgpr18_sgpr19 = COPY killed renamable $sgpr48_sgpr49
+  ; CHECK-NEXT:   renamable $sgpr14 = COPY killed renamable $sgpr85
+  ; CHECK-NEXT:   renamable $sgpr15 = COPY killed renamable $sgpr83
   ; CHECK-NEXT:   renamable $sgpr16 = COPY killed renamable $sgpr33
-  ; CHECK-NEXT:   renamable $sgpr4_sgpr5 = COPY killed renamable $sgpr52_sgpr53
-  ; CHECK-NEXT:   renamable $sgpr6_sgpr7 = COPY killed renamable $sgpr54_sgpr55
-  ; CHECK-NEXT:   renamable $sgpr8 = COPY killed renamable $sgpr62
-  ; CHECK-NEXT:   renamable $sgpr10_sgpr11 = COPY killed renamable $sgpr56_sgpr57
+  ; CHECK-NEXT:   renamable $sgpr4_sgpr5 = COPY killed renamable $sgpr68_sgpr69
+  ; CHECK-NEXT:   renamable $sgpr6_sgpr7 = COPY killed renamable $sgpr70_sgpr71
+  ; CHECK-NEXT:   renamable $sgpr8 = COPY killed renamable $sgpr84
+  ; CHECK-NEXT:   renamable $sgpr10_sgpr11 = COPY killed renamable $sgpr80_sgpr81
   ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
-  ; CHECK-NEXT:   $exec = S_MOV_B64_term renamable $sgpr50_sgpr51
+  ; CHECK-NEXT:   $exec = S_MOV_B64_term renamable $sgpr66_sgpr67
   ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.10, implicit $exec
   ; CHECK-NEXT:   S_BRANCH %bb.17
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.10:
   ; CHECK-NEXT:   successors: %bb.8(0x40000000), %bb.12(0x40000000)
-  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75:0x0000000F00000000
+  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr100_sgpr101
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.8, implicit $exec
   ; CHECK-NEXT:   S_BRANCH %bb.12
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.11:
   ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.17(0x40000000)
-  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75:0x0000000F00000000
+  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr100_sgpr101
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.1, implicit $exec
   ; CHECK-NEXT:   S_BRANCH %bb.17
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.12:
   ; CHECK-NEXT:   successors: %bb.11(0x40000000), %bb.13(0x40000000)
-  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75:0x0000000F00000000
+  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr54_sgpr55, $sgpr100_sgpr101
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   $exec = S_MOV_B64_term killed renamable $sgpr46_sgpr47
+  ; CHECK-NEXT:   $exec = S_MOV_B64_term killed renamable $sgpr54_sgpr55
   ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.11, implicit $exec
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.13:
   ; CHECK-NEXT:   successors: %bb.15(0x40000000), %bb.14(0x40000000)
-  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75:0x0000000F00000000
+  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr100_sgpr101
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   $vcc = S_AND_B64 $exec, renamable $sgpr42_sgpr43, implicit-def dead $scc
+  ; CHECK-NEXT:   renamable $sgpr12_sgpr13 = SI_SPILL_S64_RESTORE %stack.4, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.4, align 4, addrspace 5)
+  ; CHECK-NEXT:   $vcc = S_AND_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.15, implicit $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.14
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.14:
   ; CHECK-NEXT:   successors: %bb.15(0x80000000)
-  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75:0x0000000F00000000
+  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr100_sgpr101
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.15:
   ; CHECK-NEXT:   successors: %bb.11(0x40000000), %bb.16(0x40000000)
-  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75:0x0000000F00000000
+  ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr100_sgpr101
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   $vcc = S_AND_B64 $exec, renamable $sgpr36_sgpr37, implicit-def dead $scc
+  ; CHECK-NEXT:   renamable $sgpr12_sgpr13 = SI_SPILL_S64_RESTORE %stack.2, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.2, align 4, addrspace 5)
+  ; CHECK-NEXT:   $vcc = S_AND_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.11, implicit $vcc
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.16:
diff --git a/llvm/test/CodeGen/AMDGPU/reassoc-scalar.ll b/llvm/test/CodeGen/AMDGPU/reassoc-scalar.ll
index 22fe358b3b6be..509b88250b4f1 100644
--- a/llvm/test/CodeGen/AMDGPU/reassoc-scalar.ll
+++ b/llvm/test/CodeGen/AMDGPU/reassoc-scalar.ll
@@ -56,7 +56,7 @@ define amdgpu_kernel void @reassoc_v2i32(ptr addrspace(1) %arg, <2 x i32> %x, <2
 bb:
   %t1 = tail call i32 @llvm.amdgcn.workitem.id.x()
   %t2 = tail call i32 @llvm.amdgcn.workitem.id.y()
-  %v1 = insertelement <2 x i32> undef, i32 %t1, i32 0
+  %v1 = insertelement <2 x i32> poison, i32 %t1, i32 0
   %v2 = insertelement <2 x i32> %v1, i32 %t2, i32 1
   %add1 = add <2 x i32> %x, %v2
   %add2 = add <2 x i32> %add1, %y
diff --git a/llvm/test/CodeGen/AMDGPU/reduction.ll b/llvm/test/CodeGen/AMDGPU/reduction.ll
index 53a036b617725..cd4b3c1d1b546 100644
--- a/llvm/test/CodeGen/AMDGPU/reduction.ll
+++ b/llvm/test/CodeGen/AMDGPU/reduction.ll
@@ -10,9 +10,9 @@
 ; VI-NEXT: v_add_f16_e32
 define half @reduction_fadd_v4f16(<4 x half> %vec4) {
 entry:
-  %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %bin.rdx = fadd <4 x half> %vec4, %rdx.shuf
-  %rdx.shuf1 = shufflevector <4 x half> %bin.rdx, <4 x half> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf1 = shufflevector <4 x half> %bin.rdx, <4 x half> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
   %bin.rdx2 = fadd <4 x half> %bin.rdx, %rdx.shuf1
   %res = extractelement <4 x half> %bin.rdx2, i32 0
   ret half %res
@@ -30,9 +30,9 @@ entry:
 ; VI-NEXT: s_setpc_b64
 define half @reduction_fsub_v4f16(<4 x half> %vec4) {
 entry:
-  %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %bin.rdx = fsub <4 x half> %vec4, %rdx.shuf
-  %rdx.shuf1 = shufflevector <4 x half> %bin.rdx, <4 x half> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf1 = shufflevector <4 x half> %bin.rdx, <4 x half> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
   %bin.rdx2 = fsub <4 x half> %bin.rdx, %rdx.shuf1
   %res = extractelement <4 x half> %bin.rdx2, i32 0
   ret half %res
@@ -52,9 +52,9 @@ entry:
 ; VI-NEXT: s_setpc_b64
 define half @reduction_fsub_v4f16_preserve_fmf(<4 x half> %vec4) {
 entry:
-  %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %bin.rdx = fsub nsz <4 x half> %vec4, %rdx.shuf
-  %rdx.shuf1 = shufflevector <4 x half> %bin.rdx, <4 x half> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf1 = shufflevector <4 x half> %bin.rdx, <4 x half> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
   %bin.rdx2 = fsub nsz <4 x half> %bin.rdx, %rdx.shuf1
   %res = extractelement <4 x half> %bin.rdx2, i32 0
   %neg.res = fsub half -0.0, %res
@@ -70,9 +70,9 @@ entry:
 ; VI-NEXT: v_mul_f16_e32
 define half @reduction_fmul_half4(<4 x half> %vec4) {
 entry:
-  %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %bin.rdx = fmul <4 x half> %vec4, %rdx.shuf
-  %rdx.shuf1 = shufflevector <4 x half> %bin.rdx, <4 x half> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf1 = shufflevector <4 x half> %bin.rdx, <4 x half> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
   %bin.rdx2 = fmul <4 x half> %bin.rdx, %rdx.shuf1
   %res = extractelement <4 x half> %bin.rdx2, i32 0
   ret half %res
@@ -87,9 +87,9 @@ entry:
 ; VI-NEXT: v_add_u16_e32
 define i16 @reduction_v4i16(<4 x i16> %vec4) {
 entry:
-  %rdx.shuf = shufflevector <4 x i16> %vec4, <4 x i16> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %rdx.shuf = shufflevector <4 x i16> %vec4, <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %bin.rdx = add <4 x i16> %vec4, %rdx.shuf
-  %rdx.shuf1 = shufflevector <4 x i16> %bin.rdx, <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf1 = shufflevector <4 x i16> %bin.rdx, <4 x i16> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
   %bin.rdx2 = add <4 x i16> %bin.rdx, %rdx.shuf1
   %res = extractelement <4 x i16> %bin.rdx2, i32 0
   ret i16 %res
@@ -111,11 +111,11 @@ entry:
 
 define half @reduction_half8(<8 x half> %vec8) {
 entry:
-  %rdx.shuf = shufflevector <8 x half> %vec8, <8 x half> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf = shufflevector <8 x half> %vec8, <8 x half> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
   %bin.rdx = fadd <8 x half> %vec8, %rdx.shuf
-  %rdx.shuf1 = shufflevector <8 x half> %bin.rdx, <8 x half> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf1 = shufflevector <8 x half> %bin.rdx, <8 x half> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   %bin.rdx2 = fadd <8 x half> %bin.rdx, %rdx.shuf1
-  %rdx.shuf3 = shufflevector <8 x half> %bin.rdx2, <8 x half> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf3 = shufflevector <8 x half> %bin.rdx2, <8 x half> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   %bin.rdx4 = fadd <8 x half> %bin.rdx2, %rdx.shuf3
   %res = extractelement <8 x half> %bin.rdx4, i32 0
   ret half %res
@@ -137,11 +137,11 @@ entry:
 
 define i16 @reduction_v8i16(<8 x i16> %vec8) {
 entry:
-  %rdx.shuf = shufflevector <8 x i16> %vec8, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf = shufflevector <8 x i16> %vec8, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
   %bin.rdx = add <8 x i16> %vec8, %rdx.shuf
-  %rdx.shuf1 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf1 = shufflevector <8 x i16> %bin.rdx, <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   %bin.rdx2 = add <8 x i16> %bin.rdx, %rdx.shuf1
-  %rdx.shuf3 = shufflevector <8 x i16> %bin.rdx2, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf3 = shufflevector <8 x i16> %bin.rdx2, <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   %bin.rdx4 = add <8 x i16> %bin.rdx2, %rdx.shuf3
   %res = extractelement <8 x i16> %bin.rdx4, i32 0
   ret i16 %res
@@ -175,13 +175,13 @@ entry:
 
 define half @reduction_half16(<16 x half> %vec16) {
 entry:
-  %rdx.shuf = shufflevector <16 x half> %vec16, <16 x half> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf = shufflevector <16 x half> %vec16, <16 x half> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   %bin.rdx = fadd <16 x half> %vec16, %rdx.shuf
-  %rdx.shuf1 = shufflevector <16 x half> %bin.rdx, <16 x half> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf1 = shufflevector <16 x half> %bin.rdx, <16 x half> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   %bin.rdx2 = fadd <16 x half> %bin.rdx, %rdx.shuf1
-  %rdx.shuf3 = shufflevector <16 x half> %bin.rdx2, <16 x half> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf3 = shufflevector <16 x half> %bin.rdx2, <16 x half> poison, <16 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   %bin.rdx4 = fadd <16 x half> %bin.rdx2, %rdx.shuf3
-  %rdx.shuf5 = shufflevector <16 x half> %bin.rdx4, <16 x half> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf5 = shufflevector <16 x half> %bin.rdx4, <16 x half> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   %bin.rdx6 = fadd <16 x half> %bin.rdx4, %rdx.shuf5
   %res = extractelement <16 x half> %bin.rdx6, i32 0
   ret half %res
@@ -196,10 +196,10 @@ entry:
 ; VI-NEXT: v_min_u16_e32
 define i16 @reduction_min_v4i16(<4 x i16> %vec4) {
 entry:
-  %rdx.shuf = shufflevector <4 x i16> %vec4, <4 x i16> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %rdx.shuf = shufflevector <4 x i16> %vec4, <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %rdx.minmax.cmp = icmp ult <4 x i16> %vec4, %rdx.shuf
   %rdx.minmax.select = select <4 x i1> %rdx.minmax.cmp, <4 x i16> %vec4, <4 x i16> %rdx.shuf
-  %rdx.shuf1 = shufflevector <4 x i16> %rdx.minmax.select, <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf1 = shufflevector <4 x i16> %rdx.minmax.select, <4 x i16> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
   %rdx.minmax.cmp2 = icmp ult <4 x i16> %rdx.minmax.select, %rdx.shuf1
   %rdx.minmax.select3 = select <4 x i1> %rdx.minmax.cmp2, <4 x i16> %rdx.minmax.select, <4 x i16> %rdx.shuf1
   %res = extractelement <4 x i16> %rdx.minmax.select3, i32 0
@@ -221,13 +221,13 @@ entry:
 ; VI-NEXT: v_min_u16_e32
 define i16 @reduction_umin_v8i16(<8 x i16> %vec8) {
 entry:
-  %rdx.shuf = shufflevector <8 x i16> %vec8, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf = shufflevector <8 x i16> %vec8, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
   %rdx.minmax.cmp = icmp ult <8 x i16> %vec8, %rdx.shuf
   %rdx.minmax.select = select <8 x i1> %rdx.minmax.cmp, <8 x i16> %vec8, <8 x i16> %rdx.shuf
-  %rdx.shuf1 = shufflevector <8 x i16> %rdx.minmax.select, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf1 = shufflevector <8 x i16> %rdx.minmax.select, <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   %rdx.minmax.cmp2 = icmp ult <8 x i16> %rdx.minmax.select, %rdx.shuf1
   %rdx.minmax.select3 = select <8 x i1> %rdx.minmax.cmp2, <8 x i16> %rdx.minmax.select, <8 x i16> %rdx.shuf1
-  %rdx.shuf4 = shufflevector <8 x i16> %rdx.minmax.select3, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf4 = shufflevector <8 x i16> %rdx.minmax.select3, <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   %rdx.minmax.cmp5 = icmp ult <8 x i16> %rdx.minmax.select3, %rdx.shuf4
   %rdx.minmax.select6 = select <8 x i1> %rdx.minmax.cmp5, <8 x i16> %rdx.minmax.select3, <8 x i16> %rdx.shuf4
   %res = extractelement <8 x i16> %rdx.minmax.select6, i32 0
@@ -301,16 +301,16 @@ entry:
 ; VI-NEXT: v_min_i16_e32
 define i16 @reduction_smin_v16i16(<16 x i16> %vec16) {
 entry:
-  %rdx.shuf = shufflevector <16 x i16> %vec16, <16 x i16> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf = shufflevector <16 x i16> %vec16, <16 x i16> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   %rdx.minmax.cmp = icmp slt <16 x i16> %vec16, %rdx.shuf
   %rdx.minmax.select = select <16 x i1> %rdx.minmax.cmp, <16 x i16> %vec16, <16 x i16> %rdx.shuf
-  %rdx.shuf1 = shufflevector <16 x i16> %rdx.minmax.select, <16 x i16> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf1 = shufflevector <16 x i16> %rdx.minmax.select, <16 x i16> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   %rdx.minmax.cmp2 = icmp slt <16 x i16> %rdx.minmax.select, %rdx.shuf1
   %rdx.minmax.select3 = select <16 x i1> %rdx.minmax.cmp2, <16 x i16> %rdx.minmax.select, <16 x i16> %rdx.shuf1
-  %rdx.shuf4 = shufflevector <16 x i16> %rdx.minmax.select3, <16 x i16> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf4 = shufflevector <16 x i16> %rdx.minmax.select3, <16 x i16> poison, <16 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   %rdx.minmax.cmp5 = icmp slt <16 x i16> %rdx.minmax.select3, %rdx.shuf4
   %rdx.minmax.select6 = select <16 x i1> %rdx.minmax.cmp5, <16 x i16> %rdx.minmax.select3, <16 x i16> %rdx.shuf4
-  %rdx.shuf7 = shufflevector <16 x i16> %rdx.minmax.select6, <16 x i16> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf7 = shufflevector <16 x i16> %rdx.minmax.select6, <16 x i16> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   %rdx.minmax.cmp8 = icmp slt <16 x i16> %rdx.minmax.select6, %rdx.shuf7
   %rdx.minmax.select9 = select <16 x i1> %rdx.minmax.cmp8, <16 x i16> %rdx.minmax.select6, <16 x i16> %rdx.shuf7
   %res = extractelement <16 x i16> %rdx.minmax.select9, i32 0
@@ -404,10 +404,10 @@ entry:
 ; VI-NEXT: v_max_u16_e32
 define i16 @reduction_umax_v4i16(<4 x i16> %vec4) {
 entry:
-  %rdx.shuf = shufflevector <4 x i16> %vec4, <4 x i16> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %rdx.shuf = shufflevector <4 x i16> %vec4, <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %rdx.minmax.cmp = icmp ugt <4 x i16> %vec4, %rdx.shuf
   %rdx.minmax.select = select <4 x i1> %rdx.minmax.cmp, <4 x i16> %vec4, <4 x i16> %rdx.shuf
-  %rdx.shuf1 = shufflevector <4 x i16> %rdx.minmax.select, <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf1 = shufflevector <4 x i16> %rdx.minmax.select, <4 x i16> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
   %rdx.minmax.cmp2 = icmp ugt <4 x i16> %rdx.minmax.select, %rdx.shuf1
   %rdx.minmax.select3 = select <4 x i1> %rdx.minmax.cmp2, <4 x i16> %rdx.minmax.select, <4 x i16> %rdx.shuf1
   %res = extractelement <4 x i16> %rdx.minmax.select3, i32 0
@@ -423,10 +423,10 @@ entry:
 ; VI-NEXT: v_max_i16_e32
 define i16 @reduction_smax_v4i16(<4 x i16> %vec4) #0 {
 entry:
-  %rdx.shuf = shufflevector <4 x i16> %vec4, <4 x i16> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %rdx.shuf = shufflevector <4 x i16> %vec4, <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %rdx.minmax.cmp = icmp sgt <4 x i16> %vec4, %rdx.shuf
   %rdx.minmax.select = select <4 x i1> %rdx.minmax.cmp, <4 x i16> %vec4, <4 x i16> %rdx.shuf
-  %rdx.shuf1 = shufflevector <4 x i16> %rdx.minmax.select, <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf1 = shufflevector <4 x i16> %rdx.minmax.select, <4 x i16> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
   %rdx.minmax.cmp2 = icmp sgt <4 x i16> %rdx.minmax.select, %rdx.shuf1
   %rdx.minmax.select3 = select <4 x i1> %rdx.minmax.cmp2, <4 x i16> %rdx.minmax.select, <4 x i16> %rdx.shuf1
   %res = extractelement <4 x i16> %rdx.minmax.select3, i32 0
@@ -451,9 +451,9 @@ entry:
 ; VI: v_max_f16_e32 v0, [[MAX1]], [[MAX0]]
 define half @reduction_maxnum_v4f16(<4 x half> %vec4) {
 entry:
-  %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %rdx.minmax = call <4 x half> @llvm.maxnum.v4f16(<4 x half> %vec4, <4 x half> %rdx.shuf)
-  %rdx.shuf1 = shufflevector <4 x half> %rdx.minmax, <4 x half> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf1 = shufflevector <4 x half> %rdx.minmax, <4 x half> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
   %rdx.minmax3 = call <4 x half> @llvm.maxnum.v4f16(<4 x half> %rdx.minmax, <4 x half> %rdx.shuf1)
   %res = extractelement <4 x half> %rdx.minmax3, i32 0
   ret half %res
@@ -476,9 +476,9 @@ entry:
 ; VI: v_min_f16_e32 v0, [[MAX1]], [[MAX0]]
 define half @reduction_minnum_v4f16(<4 x half> %vec4) {
 entry:
-  %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %rdx.minmax = call <4 x half> @llvm.minnum.v4f16(<4 x half> %vec4, <4 x half> %rdx.shuf)
-  %rdx.shuf1 = shufflevector <4 x half> %rdx.minmax, <4 x half> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf1 = shufflevector <4 x half> %rdx.minmax, <4 x half> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
   %rdx.minmax3 = call <4 x half> @llvm.minnum.v4f16(<4 x half> %rdx.minmax, <4 x half> %rdx.shuf1)
   %res = extractelement <4 x half> %rdx.minmax3, i32 0
   ret half %res
@@ -513,10 +513,10 @@ entry:
 ; VI: v_max_f16_e32 v0, [[MAX1]], [[MAX0]]
 define half @reduction_fast_max_pattern_v4f16(<4 x half> %vec4) {
 entry:
-  %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %rdx.minmax.cmp = fcmp nnan nsz ogt <4 x half> %vec4, %rdx.shuf
   %rdx.minmax.select = select <4 x i1> %rdx.minmax.cmp, <4 x half> %vec4, <4 x half> %rdx.shuf
-  %rdx.shuf1 = shufflevector <4 x half> %rdx.minmax.select, <4 x half> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf1 = shufflevector <4 x half> %rdx.minmax.select, <4 x half> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
   %rdx.minmax.cmp2 = fcmp nnan nsz ogt <4 x half> %rdx.minmax.select, %rdx.shuf1
   %rdx.minmax.select3 = select <4 x i1> %rdx.minmax.cmp2, <4 x half> %rdx.minmax.select, <4 x half> %rdx.shuf1
   %res = extractelement <4 x half> %rdx.minmax.select3, i32 0
@@ -552,10 +552,10 @@ entry:
 ; VI: v_min_f16_e32 v0, [[MAX1]], [[MAX0]]
 define half @reduction_fast_min_pattern_v4f16(<4 x half> %vec4) {
 entry:
-  %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   %rdx.minmax.cmp = fcmp nnan nsz olt <4 x half> %vec4, %rdx.shuf
   %rdx.minmax.select = select <4 x i1> %rdx.minmax.cmp, <4 x half> %vec4, <4 x half> %rdx.shuf
-  %rdx.shuf1 = shufflevector <4 x half> %rdx.minmax.select, <4 x half> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %rdx.shuf1 = shufflevector <4 x half> %rdx.minmax.select, <4 x half> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
   %rdx.minmax.cmp2 = fcmp nnan nsz olt <4 x half> %rdx.minmax.select, %rdx.shuf1
   %rdx.minmax.select3 = select <4 x i1> %rdx.minmax.cmp2, <4 x half> %rdx.minmax.select, <4 x half> %rdx.shuf1
   %res = extractelement <4 x half> %rdx.minmax.select3, i32 0
diff --git a/llvm/test/CodeGen/AMDGPU/ret.ll b/llvm/test/CodeGen/AMDGPU/ret.ll
index a0528825d24f1..365170e0e1f44 100644
--- a/llvm/test/CodeGen/AMDGPU/ret.ll
+++ b/llvm/test/CodeGen/AMDGPU/ret.ll
@@ -13,7 +13,7 @@ define amdgpu_vs { float, float } @vgpr(ptr addrspace(4) inreg %arg, i32 inreg %
 bb:
   call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %arg3, float %arg3, float %arg3, float %arg3, i1 true, i1 true) #0
   %x = fadd float %arg3, 1.000000e+00
-  %a = insertvalue { float, float } undef, float %x, 0
+  %a = insertvalue { float, float } poison, float %x, 0
   %b = insertvalue { float, float } %a, float %arg3, 1
   ret { float, float } %b
 }
@@ -54,7 +54,7 @@ bb:
   %f1 = bitcast i32 %i1 to float
   %f2 = bitcast i32 %i2 to float
   %f3 = bitcast i32 %i3 to float
-  %r0 = insertvalue { float, float, float, float, float } undef, float %f0, 0
+  %r0 = insertvalue { float, float, float, float, float } poison, float %f0, 0
   %r1 = insertvalue { float, float, float, float, float } %r0, float %f1, 1
   %r2 = insertvalue { float, float, float, float, float } %r1, float %f2, 2
   %r3 = insertvalue { float, float, float, float, float } %r2, float %f3, 3
@@ -86,7 +86,7 @@ bb:
 define amdgpu_ps { float, <2 x float> } @ps_input_ena_pos_w(ptr addrspace(4) inreg %arg, i32 inreg %arg1, i32 inreg %arg2, <2 x i32> %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <3 x i32> %arg6, <2 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, float %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18) #1 {
 bb:
   %f = bitcast <2 x i32> %arg8 to <2 x float>
-  %s = insertvalue { float, <2 x float> } undef, float %arg14, 0
+  %s = insertvalue { float, <2 x float> } poison, float %arg14, 0
   %s1 = insertvalue { float, <2 x float> } %s, <2 x float> %f, 1
   ret { float, <2 x float> } %s1
 }
@@ -112,7 +112,7 @@ bb:
   %f1 = bitcast i32 %i1 to float
   %f2 = bitcast i32 %i2 to float
   %f3 = bitcast i32 %i3 to float
-  %r0 = insertvalue { float, float, float, float, float } undef, float %f0, 0
+  %r0 = insertvalue { float, float, float, float, float } poison, float %f0, 0
   %r1 = insertvalue { float, float, float, float, float } %r0, float %f1, 1
   %r2 = insertvalue { float, float, float, float, float } %r1, float %f2, 2
   %r3 = insertvalue { float, float, float, float, float } %r2, float %f3, 3
@@ -141,7 +141,7 @@ bb:
   %f1 = bitcast i32 %i1 to float
   %f2 = bitcast i32 %i2 to float
   %f3 = bitcast i32 %i3 to float
-  %r0 = insertvalue { float, float, float, float, float } undef, float %f0, 0
+  %r0 = insertvalue { float, float, float, float, float } poison, float %f0, 0
   %r1 = insertvalue { float, float, float, float, float } %r0, float %f1, 1
   %r2 = insertvalue { float, float, float, float, float } %r1, float %f2, 2
   %r3 = insertvalue { float, float, float, float, float } %r2, float %f3, 3
@@ -170,7 +170,7 @@ bb:
   %f1 = bitcast i32 %i1 to float
   %f2 = bitcast i32 %i2 to float
   %f3 = bitcast i32 %i3 to float
-  %r0 = insertvalue { float, float, float, float, float } undef, float %f0, 0
+  %r0 = insertvalue { float, float, float, float, float } poison, float %f0, 0
   %r1 = insertvalue { float, float, float, float, float } %r0, float %f1, 1
   %r2 = insertvalue { float, float, float, float, float } %r1, float %f2, 2
   %r3 = insertvalue { float, float, float, float, float } %r2, float %f3, 3
@@ -185,7 +185,7 @@ bb:
 define amdgpu_vs { i32, i32, i32 } @sgpr(ptr addrspace(4) inreg %arg, i32 inreg %arg1, i32 inreg %arg2, float %arg3) #0 {
 bb:
   %x = add i32 %arg2, 2
-  %a = insertvalue { i32, i32, i32 } undef, i32 %x, 0
+  %a = insertvalue { i32, i32, i32 } poison, i32 %x, 0
   %b = insertvalue { i32, i32, i32 } %a, i32 %arg1, 1
   %c = insertvalue { i32, i32, i32 } %a, i32 %arg2, 2
   ret { i32, i32, i32 } %c
@@ -218,7 +218,7 @@ bb:
   call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %arg3, float %arg3, float %arg3, float %arg3, i1 true, i1 true) #0
   %v = fadd float %arg3, 1.000000e+00
   %s = add i32 %arg2, 2
-  %a0 = insertvalue { float, i32, float, i32, i32 } undef, float %v, 0
+  %a0 = insertvalue { float, i32, float, i32, i32 } poison, float %v, 0
   %a1 = insertvalue { float, i32, float, i32, i32 } %a0, i32 %s, 1
   %a2 = insertvalue { float, i32, float, i32, i32 } %a1, float %arg3, 2
   %a3 = insertvalue { float, i32, float, i32, i32 } %a2, i32 %arg1, 3
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-undef-for-phi.ll b/llvm/test/CodeGen/AMDGPU/rewrite-undef-for-phi.ll
index 4f6b9f474c131..d402df3553f62 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-undef-for-phi.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-undef-for-phi.ll
@@ -20,7 +20,7 @@ if:
   br label %end
 
 end:
-  %c2 = phi float [ undef, %if ], [ %c, %entry ]
+  %c2 = phi float [ poison, %if ], [ %c, %entry ]
   ret float %c2
 }
 
@@ -61,7 +61,7 @@ bb4:
   br label %end
 
 end:
-  %c2 = phi float [ undef, %bb2 ], [ %c, %bb3 ], [ undef, %bb4 ], [ %c, %entry ]
+  %c2 = phi float [ poison, %bb2 ], [ %c, %bb3 ], [ poison, %bb4 ], [ %c, %entry ]
   ret float %c2
 }
 
@@ -72,7 +72,7 @@ define amdgpu_ps float @exclude_backedge(float inreg %c, i32 %x) #0 {
 ; OPT-NEXT:    br i1 [[CC]], label [[END:%.*]], label [[LOOP:%.*]]
 ; OPT:       loop:
 ; OPT-NEXT:    [[IND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[LOOP]] ]
-; OPT-NEXT:    [[C2:%.*]] = phi float [ [[C:%.*]], [[ENTRY]] ], [ undef, [[LOOP]] ]
+; OPT-NEXT:    [[C2:%.*]] = phi float [ [[C:%.*]], [[ENTRY]] ], [ poison, [[LOOP]] ]
 ; OPT-NEXT:    [[INC]] = add i32 [[IND]], 1
 ; OPT-NEXT:    [[LOOP_CC:%.*]] = icmp slt i32 [[INC]], 5
 ; OPT-NEXT:    br i1 [[LOOP_CC]], label [[LOOP]], label [[LOOP_END:%.*]]
@@ -88,7 +88,7 @@ entry:
 
 loop:
   %ind = phi i32 [ 0, %entry ], [ %inc, %loop ]
-  %c2 = phi float [ %c, %entry ], [ undef, %loop ]
+  %c2 = phi float [ %c, %entry ], [ poison, %loop ]
   %inc = add i32 %ind, 1
   %loop_cc = icmp slt i32 %inc, 5
   br i1 %loop_cc, label %loop, label %loop_end
diff --git a/llvm/test/CodeGen/AMDGPU/rv7x0_count3.ll b/llvm/test/CodeGen/AMDGPU/rv7x0_count3.ll
index 30f32e1f3e336..cd7631c962944 100644
--- a/llvm/test/CodeGen/AMDGPU/rv7x0_count3.ll
+++ b/llvm/test/CodeGen/AMDGPU/rv7x0_count3.ll
@@ -7,7 +7,7 @@ bb:
   %tmp1 = extractelement <4 x float> %reg1, i32 1
   %tmp2 = extractelement <4 x float> %reg1, i32 2
   %tmp3 = extractelement <4 x float> %reg1, i32 3
-  %tmp4 = insertelement <4 x float> undef, float %tmp, i32 0
+  %tmp4 = insertelement <4 x float> poison, float %tmp, i32 0
   %tmp5 = insertelement <4 x float> %tmp4, float %tmp1, i32 1
   %tmp6 = insertelement <4 x float> %tmp5, float %tmp2, i32 2
   %tmp7 = insertelement <4 x float> %tmp6, float %tmp3, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
index e14666cdac5c2..90b1eb5500f4c 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
@@ -67,7 +67,7 @@ define amdgpu_kernel void @scalar_to_vector_v2i32(ptr addrspace(1) %out, ptr add
 ; GFX9-NEXT:    s_endpgm
   %tmp1 = load i32, ptr addrspace(1) %in, align 4
   %bc = bitcast i32 %tmp1 to <2 x i16>
-  %tmp2 = shufflevector <2 x i16> %bc, <2 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %tmp2 = shufflevector <2 x i16> %bc, <2 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
   store <4 x i16> %tmp2, ptr addrspace(1) %out, align 8
   ret void
 }
@@ -135,7 +135,7 @@ define amdgpu_kernel void @scalar_to_vector_v2f32(ptr addrspace(1) %out, ptr add
 ; GFX9-NEXT:    s_endpgm
   %tmp1 = load float, ptr addrspace(1) %in, align 4
   %bc = bitcast float %tmp1 to <2 x i16>
-  %tmp2 = shufflevector <2 x i16> %bc, <2 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %tmp2 = shufflevector <2 x i16> %bc, <2 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
   store <4 x i16> %tmp2, ptr addrspace(1) %out, align 8
   ret void
 }
@@ -193,7 +193,7 @@ define amdgpu_kernel void @scalar_to_vector_v4i16() {
 bb:
   %tmp = load <2 x i8>, ptr addrspace(1) undef, align 1
   %tmp1 = shufflevector <2 x i8> %tmp, <2 x i8> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-  %tmp2 = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> <i32 0, i32 9, i32 9, i32 9, i32 9, i32 9, i32 9, i32 9>
+  %tmp2 = shufflevector <8 x i8> %tmp1, <8 x i8> poison, <8 x i32> <i32 0, i32 9, i32 9, i32 9, i32 9, i32 9, i32 9, i32 9>
   store <8 x i8> %tmp2, ptr addrspace(1) undef, align 8
   ret void
 }
@@ -262,7 +262,7 @@ bb:
   %load = load half, ptr addrspace(1) undef, align 1
   %tmp = bitcast half %load to <2 x i8>
   %tmp1 = shufflevector <2 x i8> %tmp, <2 x i8> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-  %tmp2 = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> <i32 0, i32 9, i32 9, i32 9, i32 9, i32 9, i32 9, i32 9>
+  %tmp2 = shufflevector <8 x i8> %tmp1, <8 x i8> poison, <8 x i32> <i32 0, i32 9, i32 9, i32 9, i32 9, i32 9, i32 9, i32 9>
   store <8 x i8> %tmp2, ptr addrspace(1) undef, align 8
   ret void
 }
@@ -275,14 +275,14 @@ bb:
 ;   %tmp1 = load i32, ptr addrspace(1) %in, align 4
 ;   %bc = bitcast i32 %tmp1 to <4 x i8>
 
-;   %tmp2 = shufflevector <4 x i8> %bc, <4 x i8> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+;   %tmp2 = shufflevector <4 x i8> %bc, <4 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
 ;   store <8 x i8> %tmp2, ptr addrspace(1) %out, align 4
 ;   ret void
 ; }
 
 ; define amdgpu_kernel void @scalar_to_vector_test3(ptr addrspace(1) %out) nounwind {
-;   %newvec0 = insertelement <2 x i64> undef, i64 12345, i32 0
-;   %newvec1 = insertelement <2 x i64> %newvec0, i64 undef, i32 1
+;   %newvec0 = insertelement <2 x i64> poison, i64 12345, i32 0
+;   %newvec1 = insertelement <2 x i64> %newvec0, i64 poison, i32 1
 ;   %bc = bitcast <2 x i64> %newvec1 to <4 x i32>
 ;   %add = add <4 x i32> %bc, <i32 1, i32 2, i32 3, i32 4>
 ;   store <4 x i32> %add, ptr addrspace(1) %out, align 16
@@ -290,7 +290,7 @@ bb:
 ; }
 
 ; define amdgpu_kernel void @scalar_to_vector_test4(ptr addrspace(1) %out) nounwind {
-;   %newvec0 = insertelement <4 x i32> undef, i32 12345, i32 0
+;   %newvec0 = insertelement <4 x i32> poison, i32 12345, i32 0
 ;   %bc = bitcast <4 x i32> %newvec0 to <8 x i16>
 ;   %add = add <8 x i16> %bc, <i16 1, i16 2, i16 3, i16 4, i16 1, i16 2, i16 3, i16 4>
 ;   store <8 x i16> %add, ptr addrspace(1) %out, align 16
@@ -298,7 +298,7 @@ bb:
 ; }
 
 ; define amdgpu_kernel void @scalar_to_vector_test5(ptr addrspace(1) %out) nounwind {
-;   %newvec0 = insertelement <2 x i32> undef, i32 12345, i32 0
+;   %newvec0 = insertelement <2 x i32> poison, i32 12345, i32 0
 ;   %bc = bitcast <2 x i32> %newvec0 to <4 x i16>
 ;   %add = add <4 x i16> %bc, <i16 1, i16 2, i16 3, i16 4>
 ;   store <4 x i16> %add, ptr addrspace(1) %out, align 16
@@ -327,7 +327,7 @@ define amdgpu_kernel void @scalar_to_vector_test6(ptr addrspace(1) %out, i8 zero
 ; GFX89-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX89-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX89-NEXT:    s_endpgm
-  %newvec0 = insertelement <4 x i8> undef, i8 %val, i32 0
+  %newvec0 = insertelement <4 x i8> poison, i8 %val, i32 0
   %bc = bitcast <4 x i8> %newvec0 to <2 x half>
   store <2 x half> %bc, ptr addrspace(1) %out
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll
index 570ea4b7132aa..0d25bc97ff775 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg.ll
@@ -246,10 +246,10 @@ bb3:
 ; CHECK-LABEL: {{^}}spill_func:
 ; GCN:    NumSgprs: 104
 ; GCN-GCNTRACKERS:    NumSgprs: 104
-; GCN:    NumVgprs: 3
-; GCN-GCNTRACKERS:    NumVgprs: 4
-; GCN:    ScratchSize: 12
-; GCN-GCNTRACKERS:    ScratchSize: 16
+; GCN:    NumVgprs: 2
+; GCN-GCNTRACKERS:    NumVgprs: 3
+; GCN:    ScratchSize: 8
+; GCN-GCNTRACKERS:    ScratchSize: 12
 
 define void @spill_func(ptr addrspace(1) %arg) #0 {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-fs-loop-nested-if.ll b/llvm/test/CodeGen/AMDGPU/schedule-fs-loop-nested-if.ll
index ae779eb3410ff..63d75f3ad0b01 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-fs-loop-nested-if.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-fs-loop-nested-if.ll
@@ -50,7 +50,7 @@ ENDIF:                                            ; preds = %ELSE17, %ELSE, %IF
   %clamp.i4 = call float @llvm.minnum.f32(float %max.0.i3, float 1.000000e+00)
   %max.0.i1 = call float @llvm.maxnum.f32(float %temp2.0, float 0.000000e+00)
   %clamp.i2 = call float @llvm.minnum.f32(float %max.0.i1, float 1.000000e+00)
-  %tmp31 = insertelement <4 x float> undef, float %clamp.i, i32 0
+  %tmp31 = insertelement <4 x float> poison, float %clamp.i, i32 0
   %tmp32 = insertelement <4 x float> %tmp31, float %clamp.i4, i32 1
   %tmp33 = insertelement <4 x float> %tmp32, float %clamp.i2, i32 2
   %tmp34 = insertelement <4 x float> %tmp33, float 1.000000e+00, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-fs-loop-nested.ll b/llvm/test/CodeGen/AMDGPU/schedule-fs-loop-nested.ll
index 57d3c7fc8bd01..48caabd224930 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-fs-loop-nested.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-fs-loop-nested.ll
@@ -49,7 +49,7 @@ IF:                                               ; preds = %LOOP
   %clamp.i4 = call float @llvm.minnum.f32(float %max.0.i3, float 1.000000e+00)
   %max.0.i1 = call float @llvm.maxnum.f32(float %temp6.0, float 0.000000e+00)
   %clamp.i2 = call float @llvm.minnum.f32(float %max.0.i1, float 1.000000e+00)
-  %tmp34 = insertelement <4 x float> undef, float %clamp.i, i32 0
+  %tmp34 = insertelement <4 x float> poison, float %clamp.i, i32 0
   %tmp35 = insertelement <4 x float> %tmp34, float %clamp.i4, i32 1
   %tmp36 = insertelement <4 x float> %tmp35, float %clamp.i2, i32 2
   %tmp37 = insertelement <4 x float> %tmp36, float 1.000000e+00, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-fs-loop.ll b/llvm/test/CodeGen/AMDGPU/schedule-fs-loop.ll
index 9f0b2119d0f4e..8380bee02868f 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-fs-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-fs-loop.ll
@@ -50,7 +50,7 @@ ENDIF:                                            ; preds = %ELSE17, %ELSE, %IF
   %clamp.i4 = call float @llvm.minnum.f32(float %max.0.i3, float 1.000000e+00)
   %max.0.i1 = call float @llvm.maxnum.f32(float %temp2.0, float 0.000000e+00)
   %clamp.i2 = call float @llvm.minnum.f32(float %max.0.i1, float 1.000000e+00)
-  %tmp31 = insertelement <4 x float> undef, float %clamp.i, i32 0
+  %tmp31 = insertelement <4 x float> poison, float %clamp.i, i32 0
   %tmp32 = insertelement <4 x float> %tmp31, float %clamp.i4, i32 1
   %tmp33 = insertelement <4 x float> %tmp32, float %clamp.i2, i32 2
   %tmp34 = insertelement <4 x float> %tmp33, float 1.000000e+00, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-if-2.ll b/llvm/test/CodeGen/AMDGPU/schedule-if-2.ll
index 1888df188ac07..d6dc911151f25 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-if-2.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-if-2.ll
@@ -62,7 +62,7 @@ ENDIF:                                            ; preds = %IF23, %ELSE, %IF
   %temp5.0 = phi float [ %27, %IF ], [ %60, %IF23 ], [ 0.000000e+00, %ELSE ]
   %temp6.0 = phi float [ %29, %IF ], [ 0.000000e+00, %ELSE ], [ 0.000000e+00, %IF23 ]
   %temp7.0 = phi float [ %35, %IF ], [ 0.000000e+00, %ELSE ], [ 0.000000e+00, %IF23 ]
-  %44 = insertelement <4 x float> undef, float %temp4.0, i32 0
+  %44 = insertelement <4 x float> poison, float %temp4.0, i32 0
   %45 = insertelement <4 x float> %44, float %temp5.0, i32 1
   %46 = insertelement <4 x float> %45, float %temp6.0, i32 2
   %47 = insertelement <4 x float> %46, float %temp7.0, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-if.ll b/llvm/test/CodeGen/AMDGPU/schedule-if.ll
index 0e2a82d57044b..0d3891db5e4b3 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-if.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-if.ll
@@ -28,7 +28,7 @@ ENDIF:                                            ; preds = %IF13, %ELSE, %main_
   %temp.0 = phi float [ 1.000000e+03, %main_body ], [ 1.000000e+00, %IF13 ], [ 0.000000e+00, %ELSE ]
   %temp1.0 = phi float [ 0.000000e+00, %main_body ], [ %23, %IF13 ], [ 0.000000e+00, %ELSE ]
   %temp3.0 = phi float [ 1.000000e+00, %main_body ], [ 0.000000e+00, %ELSE ], [ 0.000000e+00, %IF13 ]
-  %16 = insertelement <4 x float> undef, float %temp.0, i32 0
+  %16 = insertelement <4 x float> poison, float %temp.0, i32 0
   %17 = insertelement <4 x float> %16, float %temp1.0, i32 1
   %18 = insertelement <4 x float> %17, float 0.000000e+00, i32 2
   %19 = insertelement <4 x float> %18, float %temp3.0, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-vs-if-nested-loop-failure.ll b/llvm/test/CodeGen/AMDGPU/schedule-vs-if-nested-loop-failure.ll
index f6441f34cb619..c5e04b324cd48 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-vs-if-nested-loop-failure.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-vs-if-nested-loop-failure.ll
@@ -18,11 +18,11 @@ main_body:
   br i1 %8, label %LOOP, label %ENDIF
 
 Flow1:                                            ; preds = %ENDIF19, %ENDIF16
-  %9 = phi float [ %115, %ENDIF19 ], [ undef, %ENDIF16 ]
-  %10 = phi float [ %114, %ENDIF19 ], [ undef, %ENDIF16 ]
-  %11 = phi float [ %113, %ENDIF19 ], [ undef, %ENDIF16 ]
-  %12 = phi float [ %112, %ENDIF19 ], [ undef, %ENDIF16 ]
-  %13 = phi float [ %111, %ENDIF19 ], [ undef, %ENDIF16 ]
+  %9 = phi float [ %115, %ENDIF19 ], [ poison, %ENDIF16 ]
+  %10 = phi float [ %114, %ENDIF19 ], [ poison, %ENDIF16 ]
+  %11 = phi float [ %113, %ENDIF19 ], [ poison, %ENDIF16 ]
+  %12 = phi float [ %112, %ENDIF19 ], [ poison, %ENDIF16 ]
+  %13 = phi float [ %111, %ENDIF19 ], [ poison, %ENDIF16 ]
   %14 = phi i1 [ false, %ENDIF19 ], [ true, %ENDIF16 ]
   br label %Flow
 
@@ -96,12 +96,12 @@ ENDIF:                                            ; preds = %main_body, %Flow2
   %74 = extractelement <4 x float> %73, i32 3
   %75 = fmul float %74, %16
   %76 = fadd float %75, %60
-  %77 = insertelement <4 x float> undef, float %64, i32 0
+  %77 = insertelement <4 x float> poison, float %64, i32 0
   %78 = insertelement <4 x float> %77, float %68, i32 1
   %79 = insertelement <4 x float> %78, float %72, i32 2
   %80 = insertelement <4 x float> %79, float %76, i32 3
   call void @llvm.amdgcn.s.barrier()
-  %81 = insertelement <4 x float> undef, float %temp.0, i32 0
+  %81 = insertelement <4 x float> poison, float %temp.0, i32 0
   %82 = insertelement <4 x float> %81, float %temp1.0, i32 1
   %83 = insertelement <4 x float> %82, float %temp2.0, i32 2
   %84 = insertelement <4 x float> %83, float %temp3.0, i32 3
@@ -140,11 +140,11 @@ Flow:                                             ; preds = %Flow1, %LOOP
   %102 = phi float [ %temp2.1, %Flow1 ], [ %temp2.1, %LOOP ]
   %103 = phi float [ %temp1.1, %Flow1 ], [ %temp1.1, %LOOP ]
   %104 = phi float [ %temp.1, %Flow1 ], [ %temp.1, %LOOP ]
-  %105 = phi float [ %9, %Flow1 ], [ undef, %LOOP ]
-  %106 = phi float [ %10, %Flow1 ], [ undef, %LOOP ]
-  %107 = phi float [ %11, %Flow1 ], [ undef, %LOOP ]
-  %108 = phi float [ %12, %Flow1 ], [ undef, %LOOP ]
-  %109 = phi float [ %13, %Flow1 ], [ undef, %LOOP ]
+  %105 = phi float [ %9, %Flow1 ], [ poison, %LOOP ]
+  %106 = phi float [ %10, %Flow1 ], [ poison, %LOOP ]
+  %107 = phi float [ %11, %Flow1 ], [ poison, %LOOP ]
+  %108 = phi float [ %12, %Flow1 ], [ poison, %LOOP ]
+  %109 = phi float [ %13, %Flow1 ], [ poison, %LOOP ]
   %110 = phi i1 [ %14, %Flow1 ], [ true, %LOOP ]
   br i1 %110, label %Flow2, label %LOOP
 
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-vs-if-nested-loop.ll b/llvm/test/CodeGen/AMDGPU/schedule-vs-if-nested-loop.ll
index b6c7a03b990fd..4066877bf1797 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-vs-if-nested-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-vs-if-nested-loop.ll
@@ -81,12 +81,12 @@ ENDIF:                                            ; preds = %ENDIF16, %LOOP, %ma
   %68 = extractelement <4 x float> %67, i32 3
   %69 = fmul float %68, %3
   %70 = fadd float %69, %54
-  %71 = insertelement <4 x float> undef, float %58, i32 0
+  %71 = insertelement <4 x float> poison, float %58, i32 0
   %72 = insertelement <4 x float> %71, float %62, i32 1
   %73 = insertelement <4 x float> %72, float %66, i32 2
   %74 = insertelement <4 x float> %73, float %70, i32 3
   call void @llvm.r600.store.swizzle(<4 x float> %74, i32 60, i32 1)
-  %75 = insertelement <4 x float> undef, float %temp.0, i32 0
+  %75 = insertelement <4 x float> poison, float %temp.0, i32 0
   %76 = insertelement <4 x float> %75, float %temp1.0, i32 1
   %77 = insertelement <4 x float> %76, float %temp2.0, i32 2
   %78 = insertelement <4 x float> %77, float %temp3.0, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/scheduler-rp-calc-one-successor-two-predecessors-bug.ll b/llvm/test/CodeGen/AMDGPU/scheduler-rp-calc-one-successor-two-predecessors-bug.ll
index 1718002a222c9..118c47e680709 100644
--- a/llvm/test/CodeGen/AMDGPU/scheduler-rp-calc-one-successor-two-predecessors-bug.ll
+++ b/llvm/test/CodeGen/AMDGPU/scheduler-rp-calc-one-successor-two-predecessors-bug.ll
@@ -15,7 +15,6 @@ define amdgpu_ps void @_amdgpu_ps_main(float %arg) {
 ; GFX900-NEXT:    s_mov_b32 s0, 0
 ; GFX900-NEXT:    v_cmp_ngt_f32_e32 vcc, 0, v1
 ; GFX900-NEXT:    ; implicit-def: $vgpr0
-; GFX900-NEXT:    ; implicit-def: $vgpr2
 ; GFX900-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX900-NEXT:    s_xor_b64 s[6:7], exec, s[2:3]
 ; GFX900-NEXT:    s_cbranch_execz .LBB0_2
@@ -33,11 +32,11 @@ define amdgpu_ps void @_amdgpu_ps_main(float %arg) {
 ; GFX900-NEXT:    s_mov_b32 s14, s0
 ; GFX900-NEXT:    s_mov_b32 s15, s0
 ; GFX900-NEXT:    image_sample v[0:1], v[0:1], s[8:15], s[0:3] dmask:0x3
-; GFX900-NEXT:    v_mov_b32_e32 v2, 1.0
 ; GFX900-NEXT:  .LBB0_2: ; %Flow
 ; GFX900-NEXT:    s_or_saveexec_b64 s[0:1], s[6:7]
 ; GFX900-NEXT:    s_and_b64 exec, exec, s[4:5]
 ; GFX900-NEXT:    s_and_b64 s[0:1], exec, s[0:1]
+; GFX900-NEXT:    v_mov_b32_e32 v2, 1.0
 ; GFX900-NEXT:    s_xor_b64 exec, exec, s[0:1]
 ; GFX900-NEXT:    s_cbranch_execz .LBB0_5
 ; GFX900-NEXT:  ; %bb.3: ; %bb5
diff --git a/llvm/test/CodeGen/AMDGPU/scheduler-subrange-crash.ll b/llvm/test/CodeGen/AMDGPU/scheduler-subrange-crash.ll
index 1c912d09c47d1..b84163aa427ae 100644
--- a/llvm/test/CodeGen/AMDGPU/scheduler-subrange-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/scheduler-subrange-crash.ll
@@ -19,11 +19,11 @@ main_body:
   %tmp1 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 24, i32 0)
   %tmp2 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 48, i32 0)
   %array_vector3 = insertelement <4 x float> zeroinitializer, float %tmp2, i32 3
-  %array_vector5 = insertelement <4 x float> <float 0.000000e+00, float undef, float undef, float undef>, float %tmp, i32 1
-  %array_vector6 = insertelement <4 x float> %array_vector5, float undef, i32 2
-  %array_vector9 = insertelement <4 x float> <float 0.000000e+00, float undef, float undef, float undef>, float %tmp1, i32 1
+  %array_vector5 = insertelement <4 x float> <float 0.000000e+00, float poison, float poison, float poison>, float %tmp, i32 1
+  %array_vector6 = insertelement <4 x float> %array_vector5, float poison, i32 2
+  %array_vector9 = insertelement <4 x float> <float 0.000000e+00, float poison, float poison, float poison>, float %tmp1, i32 1
   %array_vector10 = insertelement <4 x float> %array_vector9, float 0.000000e+00, i32 2
-  %array_vector11 = insertelement <4 x float> %array_vector10, float undef, i32 3
+  %array_vector11 = insertelement <4 x float> %array_vector10, float poison, i32 3
   %tmp3 = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> undef, i32 undef, i32 4864, i32 0)
   call void @llvm.amdgcn.raw.tbuffer.store.i32(i32 %tmp3, <4 x i32> undef, i32 36, i32 %arg, i32 68, i32 3)
   %bc = bitcast <4 x float> %array_vector3 to <4 x i32>
@@ -32,9 +32,9 @@ main_body:
   %bc49 = bitcast <4 x float> %array_vector11 to <4 x i32>
   %tmp5 = extractelement <4 x i32> %bc49, i32 undef
   call void @llvm.amdgcn.raw.tbuffer.store.i32(i32 %tmp5, <4 x i32> undef, i32 72, i32 %arg, i32 68, i32 3)
-  %array_vector21 = insertelement <4 x float> <float 0.000000e+00, float undef, float undef, float undef>, float %tmp, i32 1
-  %array_vector22 = insertelement <4 x float> %array_vector21, float undef, i32 2
-  %array_vector23 = insertelement <4 x float> %array_vector22, float undef, i32 3
+  %array_vector21 = insertelement <4 x float> <float 0.000000e+00, float poison, float poison, float poison>, float %tmp, i32 1
+  %array_vector22 = insertelement <4 x float> %array_vector21, float poison, i32 2
+  %array_vector23 = insertelement <4 x float> %array_vector22, float poison, i32 3
   call void @llvm.amdgcn.raw.tbuffer.store.i32(i32 undef, <4 x i32> undef, i32 28, i32 %arg, i32 68, i32 3)
   %bc52 = bitcast <4 x float> %array_vector23 to <4 x i32>
   %tmp6 = extractelement <4 x i32> %bc52, i32 undef
diff --git a/llvm/test/CodeGen/AMDGPU/scratch-simple.ll b/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
index 674c7a67303e4..7126cb50b3b6e 100644
--- a/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
+++ b/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
@@ -334,7 +334,7 @@ define amdgpu_hs <{i32, i32, i32, float}> @hs_ir_uses_scratch_offset(i32 inreg,
   %v1 = extractelement <81 x float> <float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0xBFEA477C60000000, float 0xBFEBE5DC60000000, float 0xBFEC71C720000000, float 0xBFEBE5DC60000000, float 0xBFEA477C60000000, float 0xBFE7A693C0000000, float 0xBFE41CFEA0000000, float 0x3FDF9B13E0000000, float 0x3FDF9B1380000000, float 0x3FD5C53B80000000, float 0x3FD5C53B00000000, float 0x3FC6326AC0000000, float 0x3FC63269E0000000, float 0xBEE05CEB00000000, float 0xBEE086A320000000, float 0xBFC63269E0000000, float 0xBFC6326AC0000000, float 0xBFD5C53B80000000, float 0xBFD5C53B80000000, float 0xBFDF9B13E0000000, float 0xBFDF9B1460000000, float 0xBFE41CFE80000000, float 0x3FE7A693C0000000, float 0x3FEA477C20000000, float 0x3FEBE5DC40000000, float 0x3FEC71C6E0000000, float 0x3FEBE5DC40000000, float 0x3FEA477C20000000, float 0x3FE7A693C0000000, float 0xBFE41CFE80000000>, i32 %idx
   %v2 = extractelement <81 x float> <float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0x3FE7A693C0000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFEBE5DC40000000, float 0x3FEBE5DC40000000, float 0xBFEC71C720000000, float 0x3FEC71C6E0000000, float 0xBFEBE5DC60000000, float 0x3FEBE5DC40000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFE7A693C0000000, float 0x3FE7A69380000000, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFE80000000>, i32 %idx
   %f = fadd float %v1, %v2
-  %r1 = insertvalue <{i32, i32, i32, float}> undef, i32 %swo, 2
+  %r1 = insertvalue <{i32, i32, i32, float}> poison, i32 %swo, 2
   %r2 = insertvalue <{i32, i32, i32, float}> %r1, float %f, 3
   ret <{i32, i32, i32, float}> %r2
 }
@@ -389,7 +389,7 @@ define amdgpu_gs <{i32, i32, i32, float}> @gs_ir_uses_scratch_offset(i32 inreg,
   %v1 = extractelement <81 x float> <float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0xBFEA477C60000000, float 0xBFEBE5DC60000000, float 0xBFEC71C720000000, float 0xBFEBE5DC60000000, float 0xBFEA477C60000000, float 0xBFE7A693C0000000, float 0xBFE41CFEA0000000, float 0x3FDF9B13E0000000, float 0x3FDF9B1380000000, float 0x3FD5C53B80000000, float 0x3FD5C53B00000000, float 0x3FC6326AC0000000, float 0x3FC63269E0000000, float 0xBEE05CEB00000000, float 0xBEE086A320000000, float 0xBFC63269E0000000, float 0xBFC6326AC0000000, float 0xBFD5C53B80000000, float 0xBFD5C53B80000000, float 0xBFDF9B13E0000000, float 0xBFDF9B1460000000, float 0xBFE41CFE80000000, float 0x3FE7A693C0000000, float 0x3FEA477C20000000, float 0x3FEBE5DC40000000, float 0x3FEC71C6E0000000, float 0x3FEBE5DC40000000, float 0x3FEA477C20000000, float 0x3FE7A693C0000000, float 0xBFE41CFE80000000>, i32 %idx
   %v2 = extractelement <81 x float> <float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0x3FE7A693C0000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFEBE5DC40000000, float 0x3FEBE5DC40000000, float 0xBFEC71C720000000, float 0x3FEC71C6E0000000, float 0xBFEBE5DC60000000, float 0x3FEBE5DC40000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFE7A693C0000000, float 0x3FE7A69380000000, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFE80000000>, i32 %idx
   %f = fadd float %v1, %v2
-  %r1 = insertvalue <{i32, i32, i32, float}> undef, i32 %swo, 2
+  %r1 = insertvalue <{i32, i32, i32, float}> poison, i32 %swo, 2
   %r2 = insertvalue <{i32, i32, i32, float}> %r1, float %f, 3
   ret <{i32, i32, i32, float}> %r2
 }
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
index 168128a82b7ec..80be621999931 100644
--- a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
@@ -1992,13 +1992,13 @@ entry:
   %tmp7 = extractelement <8 x i8> %tmp, i32 6
   %tmp8 = extractelement <8 x i8> %tmp, i32 7
 
-  %tmp9 = insertelement <2 x i8> undef, i8 %tmp1, i32 0
+  %tmp9 = insertelement <2 x i8> poison, i8 %tmp1, i32 0
   %tmp10 = insertelement <2 x i8> %tmp9, i8 %tmp2, i32 1
-  %tmp11 = insertelement <2 x i8> undef, i8 %tmp3, i32 0
+  %tmp11 = insertelement <2 x i8> poison, i8 %tmp3, i32 0
   %tmp12 = insertelement <2 x i8> %tmp11, i8 %tmp4, i32 1
-  %tmp13 = insertelement <2 x i8> undef, i8 %tmp5, i32 0
+  %tmp13 = insertelement <2 x i8> poison, i8 %tmp5, i32 0
   %tmp14 = insertelement <2 x i8> %tmp13, i8 %tmp6, i32 1
-  %tmp15 = insertelement <2 x i8> undef, i8 %tmp7, i32 0
+  %tmp15 = insertelement <2 x i8> poison, i8 %tmp7, i32 0
   %tmp16 = insertelement <2 x i8> %tmp15, i8 %tmp8, i32 1
 
   %tmp17 = shufflevector <2 x i8> %tmp10, <2 x i8> %tmp12, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -2082,7 +2082,7 @@ bb:
   br label %bb1
 
 bb1:                                              ; preds = %bb11, %bb
-  %tmp = phi <2 x i32> [ %tmp12, %bb11 ], [ undef, %bb ]
+  %tmp = phi <2 x i32> [ %tmp12, %bb11 ], [ poison, %bb ]
   br i1 true, label %bb2, label %bb11
 
 bb2:                                              ; preds = %bb1
diff --git a/llvm/test/CodeGen/AMDGPU/select-undef.ll b/llvm/test/CodeGen/AMDGPU/select-undef.ll
index 5b9866a3c9157..c7d85094f1259 100644
--- a/llvm/test/CodeGen/AMDGPU/select-undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/select-undef.ll
@@ -53,7 +53,7 @@ entry:
   br label %loop
 
 loop:
-  %phi = phi <6 x float> [ undef, %entry ], [ %add, %loop ]
+  %phi = phi <6 x float> [ poison, %entry ], [ %add, %loop ]
   %load = load volatile <6 x float>, ptr addrspace(3) undef
   %add = fadd <6 x float> %load, %phi
   br i1 %cond, label %loop, label %ret
@@ -72,7 +72,7 @@ entry:
   br label %loop
 
 loop:
-  %phi = phi <6 x i32> [ undef, %entry ], [ %add, %loop ]
+  %phi = phi <6 x i32> [ poison, %entry ], [ %add, %loop ]
   %load = load volatile <6 x i32>, ptr addrspace(3) undef
   %add = add <6 x i32> %load, %phi
   br i1 %cond, label %loop, label %ret
@@ -92,7 +92,7 @@ entry:
   br label %loop
 
 loop:
-  %phi = phi <5 x float> [ undef, %entry ], [ %add, %loop ]
+  %phi = phi <5 x float> [ poison, %entry ], [ %add, %loop ]
   %load = load volatile <5 x float>, ptr addrspace(3) undef
   %add = fadd <5 x float> %load, %phi
   br i1 %cond, label %loop, label %ret
@@ -111,7 +111,7 @@ entry:
   br label %loop
 
 loop:
-  %phi = phi <5 x i32> [ undef, %entry ], [ %add, %loop ]
+  %phi = phi <5 x i32> [ poison, %entry ], [ %add, %loop ]
   %load = load volatile <5 x i32>, ptr addrspace(3) undef
   %add = add <5 x i32> %load, %phi
   br i1 %cond, label %loop, label %ret
@@ -131,7 +131,7 @@ entry:
   br label %loop
 
 loop:
-  %phi = phi <3 x double> [ undef, %entry ], [ %add, %loop ]
+  %phi = phi <3 x double> [ poison, %entry ], [ %add, %loop ]
   %load = load volatile <3 x double>, ptr addrspace(3) %ptr
   %add = fadd <3 x double> %load, %phi
   br i1 %cond, label %loop, label %ret
@@ -150,7 +150,7 @@ entry:
   br label %loop
 
 loop:
-  %phi = phi <3 x i64> [ undef, %entry ], [ %add, %loop ]
+  %phi = phi <3 x i64> [ poison, %entry ], [ %add, %loop ]
   %load = load volatile <3 x i64>, ptr addrspace(3) %ptr
   %add = add <3 x i64> %load, %phi
   br i1 %cond, label %loop, label %ret
@@ -170,7 +170,7 @@ entry:
   br label %loop
 
 loop:
-  %phi = phi <4 x half> [ undef, %entry ], [ %add, %loop ]
+  %phi = phi <4 x half> [ poison, %entry ], [ %add, %loop ]
   %load = load volatile <4 x half>, ptr addrspace(3) %ptr
   %add = fadd <4 x half> %load, %phi
   br i1 %cond, label %loop, label %ret
@@ -189,7 +189,7 @@ entry:
   br label %loop
 
 loop:
-  %phi = phi <4 x i16> [ undef, %entry ], [ %add, %loop ]
+  %phi = phi <4 x i16> [ poison, %entry ], [ %add, %loop ]
   %load = load volatile <4 x i16>, ptr addrspace(3) %ptr
   %add = add <4 x i16> %load, %phi
   br i1 %cond, label %loop, label %ret
@@ -209,7 +209,7 @@ entry:
   br label %loop
 
 loop:
-  %phi = phi <2 x half> [ undef, %entry ], [ %add, %loop ]
+  %phi = phi <2 x half> [ poison, %entry ], [ %add, %loop ]
   %load = load volatile <2 x half>, ptr addrspace(3) %ptr
   %add = fadd <2 x half> %load, %phi
   br i1 %cond, label %loop, label %ret
@@ -228,7 +228,7 @@ entry:
   br label %loop
 
 loop:
-  %phi = phi <2 x i16> [ undef, %entry ], [ %add, %loop ]
+  %phi = phi <2 x i16> [ poison, %entry ], [ %add, %loop ]
   %load = load volatile <2 x i16>, ptr addrspace(3) %ptr
   %add = add <2 x i16> %load, %phi
   br i1 %cond, label %loop, label %ret
@@ -268,7 +268,7 @@ entry:
   br label %loop
 
 loop:
-  %phi = phi bfloat [ undef, %entry ], [ %add, %loop ]
+  %phi = phi bfloat [ poison, %entry ], [ %add, %loop ]
   %load = load volatile bfloat, ptr addrspace(3) undef
   %bc.0 = bitcast bfloat %load to i16
   %bc.1 = bitcast bfloat %phi to i16
@@ -290,7 +290,7 @@ entry:
   br label %loop
 
 loop:
-  %phi = phi <2 x bfloat> [ undef, %entry ], [ %add, %loop ]
+  %phi = phi <2 x bfloat> [ poison, %entry ], [ %add, %loop ]
   %load = load volatile <2 x bfloat>, ptr addrspace(3) undef
   %bc.0 = bitcast <2 x bfloat> %load to <2 x i16>
   %bc.1 = bitcast <2 x bfloat> %phi to <2 x i16>
@@ -312,7 +312,7 @@ entry:
   br label %loop
 
 loop:
-  %phi = phi <3 x bfloat> [ undef, %entry ], [ %add, %loop ]
+  %phi = phi <3 x bfloat> [ poison, %entry ], [ %add, %loop ]
   %load = load volatile <3 x bfloat>, ptr addrspace(3) undef
   %bc.0 = bitcast <3 x bfloat> %load to <3 x i16>
   %bc.1 = bitcast <3 x bfloat> %phi to <3 x i16>
@@ -334,7 +334,7 @@ entry:
   br label %loop
 
 loop:
-  %phi = phi <4 x bfloat> [ undef, %entry ], [ %add, %loop ]
+  %phi = phi <4 x bfloat> [ poison, %entry ], [ %add, %loop ]
   %load = load volatile <4 x bfloat>, ptr addrspace(3) undef
   %bc.0 = bitcast <4 x bfloat> %load to <4 x i16>
   %bc.1 = bitcast <4 x bfloat> %phi to <4 x i16>
@@ -356,7 +356,7 @@ entry:
   br label %loop
 
 loop:
-  %phi = phi <6 x bfloat> [ undef, %entry ], [ %add, %loop ]
+  %phi = phi <6 x bfloat> [ poison, %entry ], [ %add, %loop ]
   %load = load volatile <6 x bfloat>, ptr addrspace(3) undef
   %bc.0 = bitcast <6 x bfloat> %load to <6 x i16>
   %bc.1 = bitcast <6 x bfloat> %phi to <6 x i16>
@@ -378,7 +378,7 @@ entry:
   br label %loop
 
 loop:
-  %phi = phi <8 x bfloat> [ undef, %entry ], [ %add, %loop ]
+  %phi = phi <8 x bfloat> [ poison, %entry ], [ %add, %loop ]
   %load = load volatile <8 x bfloat>, ptr addrspace(3) undef
   %bc.0 = bitcast <8 x bfloat> %load to <8 x i16>
   %bc.1 = bitcast <8 x bfloat> %phi to <8 x i16>
@@ -400,7 +400,7 @@ entry:
   br label %loop
 
 loop:
-  %phi = phi <16 x bfloat> [ undef, %entry ], [ %add, %loop ]
+  %phi = phi <16 x bfloat> [ poison, %entry ], [ %add, %loop ]
   %load = load volatile <16 x bfloat>, ptr addrspace(3) undef
   %bc.0 = bitcast <16 x bfloat> %load to <16 x i16>
   %bc.1 = bitcast <16 x bfloat> %phi to <16 x i16>
@@ -422,7 +422,7 @@ entry:
   br label %loop
 
 loop:
-  %phi = phi <32 x bfloat> [ undef, %entry ], [ %add, %loop ]
+  %phi = phi <32 x bfloat> [ poison, %entry ], [ %add, %loop ]
   %load = load volatile <32 x bfloat>, ptr addrspace(3) undef
   %bc.0 = bitcast <32 x bfloat> %load to <32 x i16>
   %bc.1 = bitcast <32 x bfloat> %phi to <32 x i16>
diff --git a/llvm/test/CodeGen/AMDGPU/select.f16.ll b/llvm/test/CodeGen/AMDGPU/select.f16.ll
index d345b57d3d08b..e920fdee51815 100644
--- a/llvm/test/CodeGen/AMDGPU/select.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/select.f16.ll
@@ -1903,19 +1903,14 @@ define <16 x half> @v_vselect_v16f16(<16 x half> %a, <16 x half> %b, <16 x i32>
 ; VI-LABEL: v_vselect_v16f16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; VI-NEXT:    buffer_store_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; VI-NEXT:    s_mov_b64 exec, s[4:5]
-; VI-NEXT:    v_writelane_b32 v31, s30, 0
-; VI-NEXT:    v_writelane_b32 v31, s31, 1
 ; VI-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v16
 ; VI-NEXT:    v_cmp_eq_u32_e64 s[18:19], 0, v17
-; VI-NEXT:    v_cmp_eq_u32_e64 s[30:31], 0, v29
+; VI-NEXT:    v_cmp_eq_u32_e64 s[40:41], 0, v29
 ; VI-NEXT:    v_lshrrev_b32_e32 v16, 16, v6
 ; VI-NEXT:    v_lshrrev_b32_e32 v17, 16, v14
 ; VI-NEXT:    v_cmp_eq_u32_e64 s[6:7], 0, v18
 ; VI-NEXT:    v_cmp_eq_u32_e64 s[28:29], 0, v27
-; VI-NEXT:    v_cndmask_b32_e64 v16, v17, v16, s[30:31]
+; VI-NEXT:    v_cndmask_b32_e64 v16, v17, v16, s[40:41]
 ; VI-NEXT:    v_lshrrev_b32_e32 v17, 16, v5
 ; VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v13
 ; VI-NEXT:    v_cmp_eq_u32_e64 s[20:21], 0, v19
@@ -1957,8 +1952,6 @@ define <16 x half> @v_vselect_v16f16(<16 x half> %a, <16 x half> %b, <16 x i32>
 ; VI-NEXT:    v_cmp_eq_u32_e64 s[16:17], 0, v28
 ; VI-NEXT:    v_cndmask_b32_e64 v5, v13, v5, s[14:15]
 ; VI-NEXT:    v_cndmask_b32_e64 v6, v14, v6, s[16:17]
-; VI-NEXT:    v_readlane_b32 s31, v31, 1
-; VI-NEXT:    v_readlane_b32 s30, v31, 0
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v8
 ; VI-NEXT:    v_cndmask_b32_e32 v8, v10, v9, vcc
@@ -1976,10 +1969,6 @@ define <16 x half> @v_vselect_v16f16(<16 x half> %a, <16 x half> %b, <16 x i32>
 ; VI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; VI-NEXT:    v_or_b32_sdwa v6, v6, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; VI-NEXT:    buffer_load_dword v31, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; VI-NEXT:    s_mov_b64 exec, s[4:5]
-; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_vselect_v16f16:
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-copy-duplicate-operand.ll b/llvm/test/CodeGen/AMDGPU/sgpr-copy-duplicate-operand.ll
index 18a20c0bf3f85..849744856f207 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-copy-duplicate-operand.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-copy-duplicate-operand.ll
@@ -11,7 +11,7 @@ define amdgpu_kernel void @test_dup_operands(ptr addrspace(1) noalias %out, ptr
   %lo = extractelement <2 x i32> %a, i32 0
   %hi = extractelement <2 x i32> %a, i32 1
   %add = add i32 %lo, %lo
-  %vec0 = insertelement <2 x i32> undef, i32 %add, i32 0
+  %vec0 = insertelement <2 x i32> poison, i32 %add, i32 0
   %vec1 = insertelement <2 x i32> %vec0, i32 %hi, i32 1
   store <2 x i32> %vec1, ptr addrspace(1) %out, align 8
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/shared-op-cycle.ll b/llvm/test/CodeGen/AMDGPU/shared-op-cycle.ll
index e0308bc56d172..7cedbb38cacf7 100644
--- a/llvm/test/CodeGen/AMDGPU/shared-op-cycle.ll
+++ b/llvm/test/CodeGen/AMDGPU/shared-op-cycle.ll
@@ -26,11 +26,11 @@ define amdgpu_vs void @main(<4 x float> inreg %reg0, <4 x float> inreg %reg1, <4
    %r1 = fadd float %sq1, 2.0
    %sq2 = fmul float %w2, %w2
    %r2 = fadd float %sq2, 2.0
-   %v0 = insertelement <4 x float> undef, float %r0, i32 0
+   %v0 = insertelement <4 x float> poison, float %r0, i32 0
    %v1 = insertelement <4 x float> %v0, float %r1, i32 1
    %v2 = insertelement <4 x float> %v1, float %r2, i32 2
    %res = call float @llvm.r600.dot4(<4 x float> %v2, <4 x float> %v2)
-   %vecres = insertelement <4 x float> undef, float %res, i32 0
+   %vecres = insertelement <4 x float> poison, float %res, i32 0
    call void @llvm.r600.store.swizzle(<4 x float> %vecres, i32 0, i32 2)
    ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector-physreg-copy.ll b/llvm/test/CodeGen/AMDGPU/shufflevector-physreg-copy.ll
new file mode 100644
index 0000000000000..936118750ff10
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector-physreg-copy.ll
@@ -0,0 +1,796 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; FIXME: Fails expensive checks, should re-enable verifier, see issue #130884
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs=0 < %s | FileCheck -check-prefixes=GFX9,GFX900 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs=0 < %s | FileCheck -check-prefixes=GFX9,GFX90APLUS,GFX90A %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -verify-machineinstrs=0 < %s | FileCheck -check-prefixes=GFX9,GFX90APLUS,GFX940 %s
+
+; Test that we can form v_pk_mov_b32 in certain shuffles when they
+; originate from 32-bit physreg copy sequences.
+
+; TODO: Test 16-bit paired cases
+
+define void @shufflevector_v2i32_10_physreg_even_vgpr_pair_copy(ptr addrspace(1) inreg %ptr) {
+; GFX900-LABEL: shufflevector_v2i32_10_physreg_even_vgpr_pair_copy:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v0, 0
+; GFX900-NEXT:    ;;#ASMSTART
+; GFX900-NEXT:    ; def v4, v5
+; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    v_mov_b32_e32 v3, v5
+; GFX900-NEXT:    global_store_dwordx2 v0, v[3:4], s[16:17]
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: shufflevector_v2i32_10_physreg_even_vgpr_pair_copy:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v2, 0
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; def v4, v5
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    global_store_dwordx2 v2, v[0:1], s[16:17]
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shufflevector_v2i32_10_physreg_even_vgpr_pair_copy:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; def v4, v5
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+  %asm = call { i32, i32 } asm "; def $0, $1", "={v4},={v5}"()
+  %asm.0 = extractvalue { i32, i32 } %asm, 0
+  %asm.1 = extractvalue { i32, i32 } %asm, 1
+  %insert0 = insertelement <2 x i32> poison, i32 %asm.0, i32 0
+  %insert1 = insertelement <2 x i32> %insert0, i32 %asm.1, i32 1
+  %shuffle = shufflevector <2 x i32> %insert1, <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+  store <2 x i32> %shuffle, ptr addrspace(1) %ptr, align 8
+  ret void
+}
+
+define void @shufflevector_v2i32_10_physreg_odd_vgpr_pair_copy(ptr addrspace(1) inreg %ptr) {
+; GFX900-LABEL: shufflevector_v2i32_10_physreg_odd_vgpr_pair_copy:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v0, 0
+; GFX900-NEXT:    ;;#ASMSTART
+; GFX900-NEXT:    ; def v5, v6
+; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    v_mov_b32_e32 v4, v6
+; GFX900-NEXT:    global_store_dwordx2 v0, v[4:5], s[16:17]
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: shufflevector_v2i32_10_physreg_odd_vgpr_pair_copy:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; def v5, v6
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v4, v6
+; GFX90A-NEXT:    global_store_dwordx2 v0, v[4:5], s[16:17]
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shufflevector_v2i32_10_physreg_odd_vgpr_pair_copy:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; def v5, v6
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_mov_b32_e32 v4, v6
+; GFX940-NEXT:    global_store_dwordx2 v0, v[4:5], s[0:1]
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+  %asm = call { i32, i32 } asm "; def $0, $1", "={v5},={v6}"()
+  %asm.0 = extractvalue { i32, i32 } %asm, 0
+  %asm.1 = extractvalue { i32, i32 } %asm, 1
+  %insert0 = insertelement <2 x i32> poison, i32 %asm.0, i32 0
+  %insert1 = insertelement <2 x i32> %insert0, i32 %asm.1, i32 1
+  %shuffle = shufflevector <2 x i32> %insert1, <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+  store <2 x i32> %shuffle, ptr addrspace(1) %ptr, align 8
+  ret void
+}
+
+define void @shufflevector_v2i32_10_physreg_even_disjoint_even_vgpr_pair(ptr addrspace(1) inreg %ptr) {
+; GFX900-LABEL: shufflevector_v2i32_10_physreg_even_disjoint_even_vgpr_pair:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v0, 0
+; GFX900-NEXT:    ;;#ASMSTART
+; GFX900-NEXT:    ; def v4, v6
+; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    v_mov_b32_e32 v3, v6
+; GFX900-NEXT:    global_store_dwordx2 v0, v[3:4], s[16:17]
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: shufflevector_v2i32_10_physreg_even_disjoint_even_vgpr_pair:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; def v4, v6
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v7, v4
+; GFX90A-NEXT:    global_store_dwordx2 v0, v[6:7], s[16:17]
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shufflevector_v2i32_10_physreg_even_disjoint_even_vgpr_pair:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; def v4, v6
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_mov_b32_e32 v7, v4
+; GFX940-NEXT:    global_store_dwordx2 v0, v[6:7], s[0:1]
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+  %asm = call { i32, i32 } asm "; def $0, $1", "={v4},={v6}"()
+  %asm.0 = extractvalue { i32, i32 } %asm, 0
+  %asm.1 = extractvalue { i32, i32 } %asm, 1
+  %insert0 = insertelement <2 x i32> poison, i32 %asm.0, i32 0
+  %insert1 = insertelement <2 x i32> %insert0, i32 %asm.1, i32 1
+  %shuffle = shufflevector <2 x i32> %insert1, <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+  store <2 x i32> %shuffle, ptr addrspace(1) %ptr, align 8
+  ret void
+}
+
+define void @shufflevector_v2i32_00_physreg_even_vgpr_pair_copy(ptr addrspace(1) inreg %ptr) {
+; GFX900-LABEL: shufflevector_v2i32_00_physreg_even_vgpr_pair_copy:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ;;#ASMSTART
+; GFX900-NEXT:    ; def v4, v5
+; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    v_mov_b32_e32 v0, 0
+; GFX900-NEXT:    v_mov_b32_e32 v5, v4
+; GFX900-NEXT:    global_store_dwordx2 v0, v[4:5], s[16:17]
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: shufflevector_v2i32_00_physreg_even_vgpr_pair_copy:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; def v4, v5
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90A-NEXT:    v_mov_b32_e32 v5, v4
+; GFX90A-NEXT:    global_store_dwordx2 v0, v[4:5], s[16:17]
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shufflevector_v2i32_00_physreg_even_vgpr_pair_copy:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; def v4, v5
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v5, v4
+; GFX940-NEXT:    global_store_dwordx2 v0, v[4:5], s[0:1]
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+  %asm = call { i32, i32 } asm "; def $0, $1", "={v4},={v5}"()
+  %asm.0 = extractvalue { i32, i32 } %asm, 0
+  %asm.1 = extractvalue { i32, i32 } %asm, 1
+  %insert0 = insertelement <2 x i32> poison, i32 %asm.0, i32 0
+  %insert1 = insertelement <2 x i32> %insert0, i32 %asm.1, i32 1
+  %shuffle = shufflevector <2 x i32> %insert1, <2 x i32> poison, <2 x i32> zeroinitializer
+  store <2 x i32> %shuffle, ptr addrspace(1) %ptr, align 8
+  ret void
+}
+
+define void @shufflevector_v2i32_11_physreg_even_vgpr_pair_copy(ptr addrspace(1) inreg %ptr) {
+; GFX900-LABEL: shufflevector_v2i32_11_physreg_even_vgpr_pair_copy:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v0, 0
+; GFX900-NEXT:    ;;#ASMSTART
+; GFX900-NEXT:    ; def v4, v5
+; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    v_mov_b32_e32 v6, v5
+; GFX900-NEXT:    global_store_dwordx2 v0, v[5:6], s[16:17]
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: shufflevector_v2i32_11_physreg_even_vgpr_pair_copy:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v2, 0
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; def v4, v5
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
+; GFX90A-NEXT:    global_store_dwordx2 v2, v[0:1], s[16:17]
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shufflevector_v2i32_11_physreg_even_vgpr_pair_copy:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; def v4, v5
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v5
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+  %asm = call { i32, i32 } asm "; def $0, $1", "={v4},={v5}"()
+  %asm.0 = extractvalue { i32, i32 } %asm, 0
+  %asm.1 = extractvalue { i32, i32 } %asm, 1
+  %insert0 = insertelement <2 x i32> poison, i32 %asm.0, i32 0
+  %insert1 = insertelement <2 x i32> %insert0, i32 %asm.1, i32 1
+  %shuffle = shufflevector <2 x i32> %insert1, <2 x i32> poison, <2 x i32> splat (i32 1)
+  store <2 x i32> %shuffle, ptr addrspace(1) %ptr, align 8
+  ret void
+}
+
+define void @shufflevector_v4i32_3210_physreg_even_vgpr_quad_copy(ptr addrspace(1) inreg %ptr) {
+; GFX900-LABEL: shufflevector_v4i32_3210_physreg_even_vgpr_quad_copy:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v0, 0
+; GFX900-NEXT:    ;;#ASMSTART
+; GFX900-NEXT:    ; def v4, v5, v6, v7
+; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    v_mov_b32_e32 v3, v5
+; GFX900-NEXT:    v_mov_b32_e32 v2, v6
+; GFX900-NEXT:    v_mov_b32_e32 v1, v7
+; GFX900-NEXT:    global_store_dwordx4 v0, v[1:4], s[16:17]
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: shufflevector_v4i32_3210_physreg_even_vgpr_quad_copy:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; def v4, v5, v6, v7
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
+; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shufflevector_v4i32_3210_physreg_even_vgpr_quad_copy:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; def v4, v5, v6, v7
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_mov_b32_e32 v2, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v6
+; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    v_mov_b32_e32 v3, v4
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+  %asm = call { i32, i32, i32, i32 } asm "; def $0, $1, $2, $3", "={v4},={v5},={v6},={v7}"()
+  %asm.0 = extractvalue { i32, i32, i32, i32 } %asm, 0
+  %asm.1 = extractvalue { i32, i32, i32, i32 } %asm, 1
+  %asm.2 = extractvalue { i32, i32, i32, i32 } %asm, 2
+  %asm.3 = extractvalue { i32, i32, i32, i32 } %asm, 3
+
+  %insert0 = insertelement <4 x i32> poison, i32 %asm.0, i32 0
+  %insert1 = insertelement <4 x i32> %insert0, i32 %asm.1, i32 1
+  %insert2 = insertelement <4 x i32> %insert1, i32 %asm.2, i32 2
+  %insert3 = insertelement <4 x i32> %insert2, i32 %asm.3, i32 3
+
+  %shuffle = shufflevector <4 x i32> %insert3, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  store <4 x i32> %shuffle, ptr addrspace(1) %ptr, align 8
+  ret void
+}
+
+
+
+define void @shufflevector_v4i32_1032_physreg_even_vgpr_quad_copy(ptr addrspace(1) inreg %ptr) {
+; GFX900-LABEL: shufflevector_v4i32_1032_physreg_even_vgpr_quad_copy:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ;;#ASMSTART
+; GFX900-NEXT:    ; def v4, v5, v6, v7
+; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    v_mov_b32_e32 v0, 0
+; GFX900-NEXT:    v_mov_b32_e32 v3, v5
+; GFX900-NEXT:    v_mov_b32_e32 v5, v7
+; GFX900-NEXT:    global_store_dwordx4 v0, v[3:6], s[16:17]
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: shufflevector_v4i32_1032_physreg_even_vgpr_quad_copy:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; def v4, v5, v6, v7
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v3, v6
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v7
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v4
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shufflevector_v4i32_1032_physreg_even_vgpr_quad_copy:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; def v4, v5, v6, v7
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_mov_b32_e32 v3, v6
+; GFX940-NEXT:    v_mov_b32_e32 v2, v7
+; GFX940-NEXT:    v_mov_b32_e32 v1, v4
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+  %asm = call { i32, i32, i32, i32 } asm "; def $0, $1, $2, $3", "={v4},={v5},={v6},={v7}"()
+  %asm.0 = extractvalue { i32, i32, i32, i32 } %asm, 0
+  %asm.1 = extractvalue { i32, i32, i32, i32 } %asm, 1
+  %asm.2 = extractvalue { i32, i32, i32, i32 } %asm, 2
+  %asm.3 = extractvalue { i32, i32, i32, i32 } %asm, 3
+
+  %insert0 = insertelement <4 x i32> poison, i32 %asm.0, i32 0
+  %insert1 = insertelement <4 x i32> %insert0, i32 %asm.1, i32 1
+  %insert2 = insertelement <4 x i32> %insert1, i32 %asm.2, i32 2
+  %insert3 = insertelement <4 x i32> %insert2, i32 %asm.3, i32 3
+
+  %shuffle = shufflevector <4 x i32> %insert3, <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+  store <4 x i32> %shuffle, ptr addrspace(1) %ptr, align 8
+  ret void
+}
+
+define void @shufflevector_v4i32_1132_physreg_even_vgpr_quad_copy(ptr addrspace(1) inreg %ptr) {
+; GFX900-LABEL: shufflevector_v4i32_1132_physreg_even_vgpr_quad_copy:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ;;#ASMSTART
+; GFX900-NEXT:    ; def v4, v5, v6, v7
+; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    v_mov_b32_e32 v0, 0
+; GFX900-NEXT:    v_mov_b32_e32 v8, v6
+; GFX900-NEXT:    v_mov_b32_e32 v6, v5
+; GFX900-NEXT:    global_store_dwordx4 v0, v[5:8], s[16:17]
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: shufflevector_v4i32_1132_physreg_even_vgpr_quad_copy:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; def v4, v5, v6, v7
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[6:7] op_sel:[1,0]
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v5
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shufflevector_v4i32_1132_physreg_even_vgpr_quad_copy:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; def v4, v5, v6, v7
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    v_pk_mov_b32 v[2:3], v[6:7], v[6:7] op_sel:[1,0]
+; GFX940-NEXT:    v_mov_b32_e32 v1, v5
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+  %asm = call { i32, i32, i32, i32 } asm "; def $0, $1, $2, $3", "={v4},={v5},={v6},={v7}"()
+  %asm.0 = extractvalue { i32, i32, i32, i32 } %asm, 0
+  %asm.1 = extractvalue { i32, i32, i32, i32 } %asm, 1
+  %asm.2 = extractvalue { i32, i32, i32, i32 } %asm, 2
+  %asm.3 = extractvalue { i32, i32, i32, i32 } %asm, 3
+
+  %insert0 = insertelement <4 x i32> poison, i32 %asm.0, i32 0
+  %insert1 = insertelement <4 x i32> %insert0, i32 %asm.1, i32 1
+  %insert2 = insertelement <4 x i32> %insert1, i32 %asm.2, i32 2
+  %insert3 = insertelement <4 x i32> %insert2, i32 %asm.3, i32 3
+
+  %shuffle = shufflevector <4 x i32> %insert3, <4 x i32> poison, <4 x i32> <i32 1, i32 1, i32 3, i32 2>
+  store <4 x i32> %shuffle, ptr addrspace(1) %ptr, align 8
+  ret void
+}
+
+define void @shufflevector_v4i32_3201_physreg_even_vgpr_quad_copy(ptr addrspace(1) inreg %ptr) {
+; GFX900-LABEL: shufflevector_v4i32_3201_physreg_even_vgpr_quad_copy:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v0, 0
+; GFX900-NEXT:    ;;#ASMSTART
+; GFX900-NEXT:    ; def v4, v5, v6, v7
+; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    v_mov_b32_e32 v3, v6
+; GFX900-NEXT:    v_mov_b32_e32 v2, v7
+; GFX900-NEXT:    global_store_dwordx4 v0, v[2:5], s[16:17]
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: shufflevector_v4i32_3201_physreg_even_vgpr_quad_copy:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; def v4, v5, v6, v7
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v3, v6
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v7
+; GFX90A-NEXT:    global_store_dwordx4 v0, v[2:5], s[16:17]
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shufflevector_v4i32_3201_physreg_even_vgpr_quad_copy:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; def v4, v5, v6, v7
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_mov_b32_e32 v3, v6
+; GFX940-NEXT:    v_mov_b32_e32 v2, v7
+; GFX940-NEXT:    global_store_dwordx4 v0, v[2:5], s[0:1]
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+  %asm = call { i32, i32, i32, i32 } asm "; def $0, $1, $2, $3", "={v4},={v5},={v6},={v7}"()
+  %asm.0 = extractvalue { i32, i32, i32, i32 } %asm, 0
+  %asm.1 = extractvalue { i32, i32, i32, i32 } %asm, 1
+  %asm.2 = extractvalue { i32, i32, i32, i32 } %asm, 2
+  %asm.3 = extractvalue { i32, i32, i32, i32 } %asm, 3
+
+  %insert0 = insertelement <4 x i32> poison, i32 %asm.0, i32 0
+  %insert1 = insertelement <4 x i32> %insert0, i32 %asm.1, i32 1
+  %insert2 = insertelement <4 x i32> %insert1, i32 %asm.2, i32 2
+  %insert3 = insertelement <4 x i32> %insert2, i32 %asm.3, i32 3
+
+  %shuffle = shufflevector <4 x i32> %insert3, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 0, i32 1>
+  store <4 x i32> %shuffle, ptr addrspace(1) %ptr, align 8
+  ret void
+}
+
+define void @shufflevector_v2i32_10_physreg_even_sgpr_pair_copy() {
+; GFX900-LABEL: shufflevector_v2i32_10_physreg_even_sgpr_pair_copy:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ;;#ASMSTART
+; GFX900-NEXT:    ; def s4, s5
+; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    s_mov_b32 s7, s4
+; GFX900-NEXT:    s_mov_b32 s6, s5
+; GFX900-NEXT:    ;;#ASMSTART
+; GFX900-NEXT:    ; use s[6:7]
+; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: shufflevector_v2i32_10_physreg_even_sgpr_pair_copy:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; def s4, s5
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    s_mov_b32 s7, s4
+; GFX90A-NEXT:    s_mov_b32 s6, s5
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; use s[6:7]
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shufflevector_v2i32_10_physreg_even_sgpr_pair_copy:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; def s4, s5
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    s_mov_b32 s1, s4
+; GFX940-NEXT:    s_mov_b32 s0, s5
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; use s[0:1]
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+  %asm = call { i32, i32 } asm "; def $0, $1", "={s4},={s5}"()
+  %asm.0 = extractvalue { i32, i32 } %asm, 0
+  %asm.1 = extractvalue { i32, i32 } %asm, 1
+  %insert0 = insertelement <2 x i32> poison, i32 %asm.0, i32 0
+  %insert1 = insertelement <2 x i32> %insert0, i32 %asm.1, i32 1
+  %shuffle = shufflevector <2 x i32> %insert1, <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+  call void asm sideeffect "; use $0", "s"(<2 x i32> %shuffle)
+  ret void
+}
+
+define void @shufflevector_v2i32_10_physreg_odd_sgpr_pair_copy() {
+; GFX9-LABEL: shufflevector_v2i32_10_physreg_odd_sgpr_pair_copy:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ;;#ASMSTART
+; GFX9-NEXT:    ; def s5, s6
+; GFX9-NEXT:    ;;#ASMEND
+; GFX9-NEXT:    s_mov_b32 s7, s5
+; GFX9-NEXT:    ;;#ASMSTART
+; GFX9-NEXT:    ; use s[6:7]
+; GFX9-NEXT:    ;;#ASMEND
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %asm = call { i32, i32 } asm "; def $0, $1", "={s5},={s6}"()
+  %asm.0 = extractvalue { i32, i32 } %asm, 0
+  %asm.1 = extractvalue { i32, i32 } %asm, 1
+  %insert0 = insertelement <2 x i32> poison, i32 %asm.0, i32 0
+  %insert1 = insertelement <2 x i32> %insert0, i32 %asm.1, i32 1
+  %shuffle = shufflevector <2 x i32> %insert1, <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+  call void asm sideeffect "; use $0", "s"(<2 x i32> %shuffle)
+  ret void
+}
+
+define void @shufflevector_v2i32_10_physreg_even_agpr_pair_copy(ptr addrspace(1) inreg %ptr) {
+; GFX900-LABEL: shufflevector_v2i32_10_physreg_even_agpr_pair_copy:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v2, 0
+; GFX900-NEXT:    ;;#ASMSTART
+; GFX900-NEXT:    ; def a4, a5
+; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    v_accvgpr_read_b32 v0, a5
+; GFX900-NEXT:    v_accvgpr_read_b32 v1, a4
+; GFX900-NEXT:    global_store_dwordx2 v2, v[0:1], s[16:17]
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: shufflevector_v2i32_10_physreg_even_agpr_pair_copy:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; def a4, a5
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_accvgpr_mov_b32 a1, a4
+; GFX90A-NEXT:    v_accvgpr_mov_b32 a0, a5
+; GFX90A-NEXT:    global_store_dwordx2 v0, a[0:1], s[16:17]
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shufflevector_v2i32_10_physreg_even_agpr_pair_copy:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; def a4, a5
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_accvgpr_mov_b32 a1, a4
+; GFX940-NEXT:    v_accvgpr_mov_b32 a0, a5
+; GFX940-NEXT:    global_store_dwordx2 v0, a[0:1], s[0:1]
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+  %asm = call { i32, i32 } asm "; def $0, $1", "={a4},={a5}"()
+  %asm.0 = extractvalue { i32, i32 } %asm, 0
+  %asm.1 = extractvalue { i32, i32 } %asm, 1
+  %insert0 = insertelement <2 x i32> poison, i32 %asm.0, i32 0
+  %insert1 = insertelement <2 x i32> %insert0, i32 %asm.1, i32 1
+  %shuffle = shufflevector <2 x i32> %insert1, <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+  store <2 x i32> %shuffle, ptr addrspace(1) %ptr, align 8
+  ret void
+}
+
+define void @shufflevector_v2i32_10_physreg_odd_agpr_pair_copy(ptr addrspace(1) inreg %ptr) {
+; GFX900-LABEL: shufflevector_v2i32_10_physreg_odd_agpr_pair_copy:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v2, 0
+; GFX900-NEXT:    ;;#ASMSTART
+; GFX900-NEXT:    ; def a5, a6
+; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    v_accvgpr_read_b32 v0, a6
+; GFX900-NEXT:    v_accvgpr_read_b32 v1, a5
+; GFX900-NEXT:    global_store_dwordx2 v2, v[0:1], s[16:17]
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: shufflevector_v2i32_10_physreg_odd_agpr_pair_copy:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; def a5, a6
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_accvgpr_mov_b32 a4, a6
+; GFX90A-NEXT:    global_store_dwordx2 v0, a[4:5], s[16:17]
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shufflevector_v2i32_10_physreg_odd_agpr_pair_copy:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; def a5, a6
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_accvgpr_mov_b32 a4, a6
+; GFX940-NEXT:    global_store_dwordx2 v0, a[4:5], s[0:1]
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+  %asm = call { i32, i32 } asm "; def $0, $1", "={a5},={a6}"()
+  %asm.0 = extractvalue { i32, i32 } %asm, 0
+  %asm.1 = extractvalue { i32, i32 } %asm, 1
+  %insert0 = insertelement <2 x i32> poison, i32 %asm.0, i32 0
+  %insert1 = insertelement <2 x i32> %insert0, i32 %asm.1, i32 1
+  %shuffle = shufflevector <2 x i32> %insert1, <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+  store <2 x i32> %shuffle, ptr addrspace(1) %ptr, align 8
+  ret void
+}
+
+define i32 @shufflevector_v2i32_10_physreg_even_vgpr_pair_copy_other_use_elt0(ptr addrspace(1) inreg %ptr) {
+; GFX900-LABEL: shufflevector_v2i32_10_physreg_even_vgpr_pair_copy_other_use_elt0:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ;;#ASMSTART
+; GFX900-NEXT:    ; def v4, v5
+; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    v_mov_b32_e32 v0, v4
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: shufflevector_v2i32_10_physreg_even_vgpr_pair_copy_other_use_elt0:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; def v4, v5
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shufflevector_v2i32_10_physreg_even_vgpr_pair_copy_other_use_elt0:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; def v4, v5
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, v4
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+  %asm = call { i32, i32 } asm "; def $0, $1", "={v4},={v5}"()
+  %asm.0 = extractvalue { i32, i32 } %asm, 0
+  %asm.1 = extractvalue { i32, i32 } %asm, 1
+  %insert0 = insertelement <2 x i32> poison, i32 %asm.0, i32 0
+  %insert1 = insertelement <2 x i32> %insert0, i32 %asm.1, i32 1
+  %shuffle = shufflevector <2 x i32> %insert1, <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+  ret i32 %asm.0 ; other use of copy
+}
+
+define i32 @shufflevector_v2i32_10_physreg_even_vgpr_pair_copy_other_use_elt1(ptr addrspace(1) inreg %ptr) {
+; GFX900-LABEL: shufflevector_v2i32_10_physreg_even_vgpr_pair_copy_other_use_elt1:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    ;;#ASMSTART
+; GFX900-NEXT:    ; def v4, v5
+; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    v_mov_b32_e32 v0, v5
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: shufflevector_v2i32_10_physreg_even_vgpr_pair_copy_other_use_elt1:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; def v4, v5
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v5
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shufflevector_v2i32_10_physreg_even_vgpr_pair_copy_other_use_elt1:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; def v4, v5
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, v5
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+  %asm = call { i32, i32 } asm "; def $0, $1", "={v4},={v5}"()
+  %asm.0 = extractvalue { i32, i32 } %asm, 0
+  %asm.1 = extractvalue { i32, i32 } %asm, 1
+  %insert0 = insertelement <2 x i32> poison, i32 %asm.0, i32 0
+  %insert1 = insertelement <2 x i32> %insert0, i32 %asm.1, i32 1
+  %shuffle = shufflevector <2 x i32> %insert1, <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+  ret i32 %asm.1 ; other use of copy
+}
+
+define i32 @shufflevector_v4i32_3210_physreg_even_vgpr_quad_copy_other_use_elt(ptr addrspace(1) inreg %ptr) {
+; GFX900-LABEL: shufflevector_v4i32_3210_physreg_even_vgpr_quad_copy_other_use_elt:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v0, 0
+; GFX900-NEXT:    ;;#ASMSTART
+; GFX900-NEXT:    ; def v4, v5, v6, v7
+; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    v_mov_b32_e32 v9, v5
+; GFX900-NEXT:    v_mov_b32_e32 v8, v6
+; GFX900-NEXT:    v_mov_b32_e32 v10, v4
+; GFX900-NEXT:    global_store_dwordx4 v0, v[7:10], s[16:17]
+; GFX900-NEXT:    v_mov_b32_e32 v0, v6
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-LABEL: shufflevector_v4i32_3210_physreg_even_vgpr_quad_copy_other_use_elt:
+; GFX90A:       ; %bb.0:
+; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v8, 0
+; GFX90A-NEXT:    ;;#ASMSTART
+; GFX90A-NEXT:    ; def v4, v5, v6, v7
+; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    v_mov_b32_e32 v2, v5
+; GFX90A-NEXT:    v_mov_b32_e32 v1, v6
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v7
+; GFX90A-NEXT:    v_mov_b32_e32 v3, v4
+; GFX90A-NEXT:    global_store_dwordx4 v8, v[0:3], s[16:17]
+; GFX90A-NEXT:    s_nop 0
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v6
+; GFX90A-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: shufflevector_v4i32_3210_physreg_even_vgpr_quad_copy_other_use_elt:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v8, 0
+; GFX940-NEXT:    ;;#ASMSTART
+; GFX940-NEXT:    ; def v4, v5, v6, v7
+; GFX940-NEXT:    ;;#ASMEND
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_mov_b32_e32 v2, v5
+; GFX940-NEXT:    v_mov_b32_e32 v1, v6
+; GFX940-NEXT:    v_mov_b32_e32 v0, v7
+; GFX940-NEXT:    v_mov_b32_e32 v3, v4
+; GFX940-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_mov_b32_e32 v0, v6
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
+  %asm = call { i32, i32, i32, i32 } asm "; def $0, $1, $2, $3", "={v4},={v5},={v6},={v7}"()
+  %asm.0 = extractvalue { i32, i32, i32, i32 } %asm, 0
+  %asm.1 = extractvalue { i32, i32, i32, i32 } %asm, 1
+  %asm.2 = extractvalue { i32, i32, i32, i32 } %asm, 2
+  %asm.3 = extractvalue { i32, i32, i32, i32 } %asm, 3
+
+  %insert0 = insertelement <4 x i32> poison, i32 %asm.0, i32 0
+  %insert1 = insertelement <4 x i32> %insert0, i32 %asm.1, i32 1
+  %insert2 = insertelement <4 x i32> %insert1, i32 %asm.2, i32 2
+  %insert3 = insertelement <4 x i32> %insert2, i32 %asm.3, i32 3
+
+  %shuffle = shufflevector <4 x i32> %insert3, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  store <4 x i32> %shuffle, ptr addrspace(1) %ptr, align 8
+  ret i32 %asm.2
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX90APLUS: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v2i64.v8i64.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v2i64.v8i64.ll
index dc227f745aa9a..7ee7c83e0122d 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v2i64.v8i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v2i64.v8i64.ll
@@ -15008,18 +15008,10 @@ define void @s_shuffle_v2i64_v8i64__15_6() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -15033,18 +15025,10 @@ define void @s_shuffle_v2i64_v8i64__15_6() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -15065,18 +15049,10 @@ define void @s_shuffle_v2i64_v8i64__15_6() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -15090,18 +15066,10 @@ define void @s_shuffle_v2i64_v8i64__15_6() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -15158,21 +15126,13 @@ define void @s_shuffle_v2i64_v8i64__15_7() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[36:51]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -15183,18 +15143,10 @@ define void @s_shuffle_v2i64_v8i64__15_7() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -15215,21 +15167,13 @@ define void @s_shuffle_v2i64_v8i64__15_7() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[36:51]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -15240,18 +15184,10 @@ define void @s_shuffle_v2i64_v8i64__15_7() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -16122,18 +16058,10 @@ define void @s_shuffle_v2i64_v8i64__10_0() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[36:51]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -16145,18 +16073,10 @@ define void @s_shuffle_v2i64_v8i64__10_0() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -16177,18 +16097,10 @@ define void @s_shuffle_v2i64_v8i64__10_0() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[36:51]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -16200,18 +16112,10 @@ define void @s_shuffle_v2i64_v8i64__10_0() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -16883,18 +16787,10 @@ define void @s_shuffle_v2i64_v8i64__10_1() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[36:51]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -16906,18 +16802,10 @@ define void @s_shuffle_v2i64_v8i64__10_1() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -16938,18 +16826,10 @@ define void @s_shuffle_v2i64_v8i64__10_1() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[36:51]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -16961,18 +16841,10 @@ define void @s_shuffle_v2i64_v8i64__10_1() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -20983,18 +20855,10 @@ define void @s_shuffle_v2i64_v8i64__9_6() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -21008,18 +20872,10 @@ define void @s_shuffle_v2i64_v8i64__9_6() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -21040,18 +20896,10 @@ define void @s_shuffle_v2i64_v8i64__9_6() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -21065,18 +20913,10 @@ define void @s_shuffle_v2i64_v8i64__9_6() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -21189,18 +21029,10 @@ define void @s_shuffle_v2i64_v8i64__11_6() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -21214,18 +21046,10 @@ define void @s_shuffle_v2i64_v8i64__11_6() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -21246,18 +21070,10 @@ define void @s_shuffle_v2i64_v8i64__11_6() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -21271,18 +21087,10 @@ define void @s_shuffle_v2i64_v8i64__11_6() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -21339,18 +21147,10 @@ define void @s_shuffle_v2i64_v8i64__12_6() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -21364,18 +21164,10 @@ define void @s_shuffle_v2i64_v8i64__12_6() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -21396,18 +21188,10 @@ define void @s_shuffle_v2i64_v8i64__12_6() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -21421,18 +21205,10 @@ define void @s_shuffle_v2i64_v8i64__12_6() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -21477,18 +21253,10 @@ define void @s_shuffle_v2i64_v8i64__13_6() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -21502,18 +21270,10 @@ define void @s_shuffle_v2i64_v8i64__13_6() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -21534,18 +21294,10 @@ define void @s_shuffle_v2i64_v8i64__13_6() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -21559,18 +21311,10 @@ define void @s_shuffle_v2i64_v8i64__13_6() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -21627,18 +21371,10 @@ define void @s_shuffle_v2i64_v8i64__14_6() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -21652,18 +21388,10 @@ define void @s_shuffle_v2i64_v8i64__14_6() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -21684,18 +21412,10 @@ define void @s_shuffle_v2i64_v8i64__14_6() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -21709,18 +21429,10 @@ define void @s_shuffle_v2i64_v8i64__14_6() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -22203,21 +21915,13 @@ define void @s_shuffle_v2i64_v8i64__9_7() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[36:51]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -22228,18 +21932,10 @@ define void @s_shuffle_v2i64_v8i64__9_7() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -22260,21 +21956,13 @@ define void @s_shuffle_v2i64_v8i64__9_7() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[36:51]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -22285,18 +21973,10 @@ define void @s_shuffle_v2i64_v8i64__9_7() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -22409,21 +22089,13 @@ define void @s_shuffle_v2i64_v8i64__11_7() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[36:51]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -22434,18 +22106,10 @@ define void @s_shuffle_v2i64_v8i64__11_7() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -22466,21 +22130,13 @@ define void @s_shuffle_v2i64_v8i64__11_7() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[36:51]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -22491,18 +22147,10 @@ define void @s_shuffle_v2i64_v8i64__11_7() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -22559,18 +22207,10 @@ define void @s_shuffle_v2i64_v8i64__12_7() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -22584,18 +22224,10 @@ define void @s_shuffle_v2i64_v8i64__12_7() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -22616,18 +22248,10 @@ define void @s_shuffle_v2i64_v8i64__12_7() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -22641,18 +22265,10 @@ define void @s_shuffle_v2i64_v8i64__12_7() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -22697,21 +22313,13 @@ define void @s_shuffle_v2i64_v8i64__13_7() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[36:51]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -22722,18 +22330,10 @@ define void @s_shuffle_v2i64_v8i64__13_7() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -22754,21 +22354,13 @@ define void @s_shuffle_v2i64_v8i64__13_7() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[36:51]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -22779,18 +22371,10 @@ define void @s_shuffle_v2i64_v8i64__13_7() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -22847,18 +22431,10 @@ define void @s_shuffle_v2i64_v8i64__14_7() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -22872,18 +22448,10 @@ define void @s_shuffle_v2i64_v8i64__14_7() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -22904,18 +22472,10 @@ define void @s_shuffle_v2i64_v8i64__14_7() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -22929,18 +22489,10 @@ define void @s_shuffle_v2i64_v8i64__14_7() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -23861,18 +23413,10 @@ define void @s_shuffle_v2i64_v8i64__3_9() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[36:51]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -23884,18 +23428,10 @@ define void @s_shuffle_v2i64_v8i64__3_9() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -23916,18 +23452,10 @@ define void @s_shuffle_v2i64_v8i64__3_9() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[36:51]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -23939,18 +23467,10 @@ define void @s_shuffle_v2i64_v8i64__3_9() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -24166,21 +23686,13 @@ define void @s_shuffle_v2i64_v8i64__6_9() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[36:51]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -24191,18 +23703,10 @@ define void @s_shuffle_v2i64_v8i64__6_9() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -24223,21 +23727,13 @@ define void @s_shuffle_v2i64_v8i64__6_9() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[36:51]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -24248,18 +23744,10 @@ define void @s_shuffle_v2i64_v8i64__6_9() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -25042,21 +24530,13 @@ define void @s_shuffle_v2i64_v8i64__6_10() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[36:51]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -25067,18 +24547,10 @@ define void @s_shuffle_v2i64_v8i64__6_10() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -25099,21 +24571,13 @@ define void @s_shuffle_v2i64_v8i64__6_10() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[36:51]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -25124,18 +24588,10 @@ define void @s_shuffle_v2i64_v8i64__6_10() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -25192,18 +24648,10 @@ define void @s_shuffle_v2i64_v8i64__7_10() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -25217,18 +24665,10 @@ define void @s_shuffle_v2i64_v8i64__7_10() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -25249,18 +24689,10 @@ define void @s_shuffle_v2i64_v8i64__7_10() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -25274,18 +24706,10 @@ define void @s_shuffle_v2i64_v8i64__7_10() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -25737,18 +25161,10 @@ define void @s_shuffle_v2i64_v8i64__1_11() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[36:51]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -25760,18 +25176,10 @@ define void @s_shuffle_v2i64_v8i64__1_11() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -25792,18 +25200,10 @@ define void @s_shuffle_v2i64_v8i64__1_11() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[36:51]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -25815,18 +25215,10 @@ define void @s_shuffle_v2i64_v8i64__1_11() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -26154,21 +25546,13 @@ define void @s_shuffle_v2i64_v8i64__6_11() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[36:51]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -26179,18 +25563,10 @@ define void @s_shuffle_v2i64_v8i64__6_11() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -26211,21 +25587,13 @@ define void @s_shuffle_v2i64_v8i64__6_11() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[36:51]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -26236,18 +25604,10 @@ define void @s_shuffle_v2i64_v8i64__6_11() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -27030,21 +26390,13 @@ define void @s_shuffle_v2i64_v8i64__6_12() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[36:51]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -27055,18 +26407,10 @@ define void @s_shuffle_v2i64_v8i64__6_12() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -27087,21 +26431,13 @@ define void @s_shuffle_v2i64_v8i64__6_12() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[36:51]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -27112,18 +26448,10 @@ define void @s_shuffle_v2i64_v8i64__6_12() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -27180,18 +26508,10 @@ define void @s_shuffle_v2i64_v8i64__7_12() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -27205,18 +26525,10 @@ define void @s_shuffle_v2i64_v8i64__7_12() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -27237,18 +26549,10 @@ define void @s_shuffle_v2i64_v8i64__7_12() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -27262,18 +26566,10 @@ define void @s_shuffle_v2i64_v8i64__7_12() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -27989,21 +27285,13 @@ define void @s_shuffle_v2i64_v8i64__6_13() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[36:51]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -28014,18 +27302,10 @@ define void @s_shuffle_v2i64_v8i64__6_13() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -28046,21 +27326,13 @@ define void @s_shuffle_v2i64_v8i64__6_13() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[36:51]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -28071,18 +27343,10 @@ define void @s_shuffle_v2i64_v8i64__6_13() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -28139,18 +27403,10 @@ define void @s_shuffle_v2i64_v8i64__7_13() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[36:51]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -28164,18 +27420,10 @@ define void @s_shuffle_v2i64_v8i64__7_13() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -28196,18 +27444,10 @@ define void @s_shuffle_v2i64_v8i64__7_13() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[36:51]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -28221,18 +27461,10 @@ define void @s_shuffle_v2i64_v8i64__7_13() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -29011,21 +28243,13 @@ define void @s_shuffle_v2i64_v8i64__6_14() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[36:51]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -29036,18 +28260,10 @@ define void @s_shuffle_v2i64_v8i64__6_14() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -29068,21 +28284,13 @@ define void @s_shuffle_v2i64_v8i64__6_14() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[36:51]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -29093,18 +28301,10 @@ define void @s_shuffle_v2i64_v8i64__6_14() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -29161,18 +28361,10 @@ define void @s_shuffle_v2i64_v8i64__7_14() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -29186,18 +28378,10 @@ define void @s_shuffle_v2i64_v8i64__7_14() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -29218,18 +28402,10 @@ define void @s_shuffle_v2i64_v8i64__7_14() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -29243,18 +28419,10 @@ define void @s_shuffle_v2i64_v8i64__7_14() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -30057,21 +29225,13 @@ define void @s_shuffle_v2i64_v8i64__6_15() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[4:19]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[36:51]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -30082,18 +29242,10 @@ define void @s_shuffle_v2i64_v8i64__6_15() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -30114,21 +29266,13 @@ define void @s_shuffle_v2i64_v8i64__6_15() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[4:19]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[36:51]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -30139,18 +29283,10 @@ define void @s_shuffle_v2i64_v8i64__6_15() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
@@ -30207,18 +29343,10 @@ define void @s_shuffle_v2i64_v8i64__7_15() {
 ; GFX900-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX900-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX900-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX900-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def s[36:51]
 ; GFX900-NEXT:    ;;#ASMEND
@@ -30232,18 +29360,10 @@ define void @s_shuffle_v2i64_v8i64__7_15() {
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; use s[8:11]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX900-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v0, 1
@@ -30264,18 +29384,10 @@ define void @s_shuffle_v2i64_v8i64__7_15() {
 ; GFX90A-NEXT:    v_writelane_b32 v0, s37, 1
 ; GFX90A-NEXT:    v_writelane_b32 v0, s38, 2
 ; GFX90A-NEXT:    v_writelane_b32 v0, s39, 3
-; GFX90A-NEXT:    v_writelane_b32 v0, s40, 4
-; GFX90A-NEXT:    v_writelane_b32 v0, s41, 5
-; GFX90A-NEXT:    v_writelane_b32 v0, s42, 6
-; GFX90A-NEXT:    v_writelane_b32 v0, s43, 7
-; GFX90A-NEXT:    v_writelane_b32 v0, s44, 8
-; GFX90A-NEXT:    v_writelane_b32 v0, s45, 9
-; GFX90A-NEXT:    v_writelane_b32 v0, s46, 10
-; GFX90A-NEXT:    v_writelane_b32 v0, s47, 11
-; GFX90A-NEXT:    v_writelane_b32 v0, s48, 12
-; GFX90A-NEXT:    v_writelane_b32 v0, s49, 13
-; GFX90A-NEXT:    v_writelane_b32 v0, s50, 14
-; GFX90A-NEXT:    v_writelane_b32 v0, s51, 15
+; GFX90A-NEXT:    v_writelane_b32 v0, s48, 4
+; GFX90A-NEXT:    v_writelane_b32 v0, s49, 5
+; GFX90A-NEXT:    v_writelane_b32 v0, s50, 6
+; GFX90A-NEXT:    v_writelane_b32 v0, s51, 7
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def s[36:51]
 ; GFX90A-NEXT:    ;;#ASMEND
@@ -30289,18 +29401,10 @@ define void @s_shuffle_v2i64_v8i64__7_15() {
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; use s[8:11]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_readlane_b32 s51, v0, 15
-; GFX90A-NEXT:    v_readlane_b32 s50, v0, 14
-; GFX90A-NEXT:    v_readlane_b32 s49, v0, 13
-; GFX90A-NEXT:    v_readlane_b32 s48, v0, 12
-; GFX90A-NEXT:    v_readlane_b32 s47, v0, 11
-; GFX90A-NEXT:    v_readlane_b32 s46, v0, 10
-; GFX90A-NEXT:    v_readlane_b32 s45, v0, 9
-; GFX90A-NEXT:    v_readlane_b32 s44, v0, 8
-; GFX90A-NEXT:    v_readlane_b32 s43, v0, 7
-; GFX90A-NEXT:    v_readlane_b32 s42, v0, 6
-; GFX90A-NEXT:    v_readlane_b32 s41, v0, 5
-; GFX90A-NEXT:    v_readlane_b32 s40, v0, 4
+; GFX90A-NEXT:    v_readlane_b32 s51, v0, 7
+; GFX90A-NEXT:    v_readlane_b32 s50, v0, 6
+; GFX90A-NEXT:    v_readlane_b32 s49, v0, 5
+; GFX90A-NEXT:    v_readlane_b32 s48, v0, 4
 ; GFX90A-NEXT:    v_readlane_b32 s39, v0, 3
 ; GFX90A-NEXT:    v_readlane_b32 s38, v0, 2
 ; GFX90A-NEXT:    v_readlane_b32 s37, v0, 1
diff --git a/llvm/test/CodeGen/AMDGPU/si-annotate-dbg-info.ll b/llvm/test/CodeGen/AMDGPU/si-annotate-dbg-info.ll
index 15f6bb632f311..679bc069efbe2 100644
--- a/llvm/test/CodeGen/AMDGPU/si-annotate-dbg-info.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-annotate-dbg-info.ll
@@ -12,17 +12,16 @@ define amdgpu_ps i32 @if_else(i32 %0) !dbg !5 {
 ; OPT-NEXT:    [[TMP4:%.*]] = extractvalue { i1, i64 } [[TMP2]], 1, !dbg [[DBG14]]
 ; OPT-NEXT:    br i1 [[TMP3]], label [[FALSE:%.*]], label [[FLOW:%.*]], !dbg [[DBG14]]
 ; OPT:       Flow:
-; OPT-NEXT:    [[TMP5:%.*]] = phi i32 [ 33, [[FALSE]] ], [ undef, [[TMP1:%.*]] ]
-; OPT-NEXT:    [[TMP6:%.*]] = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 [[TMP4]]), !dbg [[DBG14]]
-; OPT-NEXT:    [[TMP7:%.*]] = extractvalue { i1, i64 } [[TMP6]], 0, !dbg [[DBG14]]
-; OPT-NEXT:    [[TMP8:%.*]] = extractvalue { i1, i64 } [[TMP6]], 1, !dbg [[DBG14]]
-; OPT-NEXT:    br i1 [[TMP7]], label [[TRUE:%.*]], label [[EXIT:%.*]], !dbg [[DBG14]]
+; OPT-NEXT:    [[TMP5:%.*]] = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 [[TMP4]]), !dbg [[DBG14]]
+; OPT-NEXT:    [[TMP6:%.*]] = extractvalue { i1, i64 } [[TMP5]], 0, !dbg [[DBG14]]
+; OPT-NEXT:    [[TMP8:%.*]] = extractvalue { i1, i64 } [[TMP5]], 1, !dbg [[DBG14]]
+; OPT-NEXT:    br i1 [[TMP6]], label [[TRUE:%.*]], label [[EXIT:%.*]], !dbg [[DBG14]]
 ; OPT:       true:
 ; OPT-NEXT:    br label [[EXIT]], !dbg [[DBG15:![0-9]+]]
 ; OPT:       false:
 ; OPT-NEXT:    br label [[FLOW]], !dbg [[DBG16:![0-9]+]]
 ; OPT:       exit:
-; OPT-NEXT:    [[RET:%.*]] = phi i32 [ [[TMP5]], [[FLOW]] ], [ 42, [[TRUE]] ], !dbg [[DBG17:![0-9]+]]
+; OPT-NEXT:    [[RET:%.*]] = phi i32 [ 33, [[FLOW]] ], [ 42, [[TRUE]] ], !dbg [[DBG17:![0-9]+]]
 ; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP8]])
 ; OPT-NEXT:      #dbg_value(i32 [[RET]], [[META11:![0-9]+]], !DIExpression(), [[DBG17]])
 ; OPT-NEXT:    ret i32 [[RET]], !dbg [[DBG18:![0-9]+]]
@@ -63,7 +62,7 @@ define amdgpu_ps void @loop_if_break(i32 %n) !dbg !19 {
 ; OPT-NEXT:      #dbg_value(i32 [[I_NEXT]], [[META23:![0-9]+]], !DIExpression(), [[DBG28]])
 ; OPT-NEXT:    br label [[FLOW]], !dbg [[DBG29:![0-9]+]]
 ; OPT:       Flow:
-; OPT-NEXT:    [[TMP3]] = phi i32 [ [[I_NEXT]], [[LOOP_BODY]] ], [ undef, [[LOOP]] ]
+; OPT-NEXT:    [[TMP3]] = phi i32 [ [[I_NEXT]], [[LOOP_BODY]] ], [ poison, [[LOOP]] ]
 ; OPT-NEXT:    [[TMP4:%.*]] = phi i1 [ false, [[LOOP_BODY]] ], [ true, [[LOOP]] ]
 ; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP2]])
 ; OPT-NEXT:    [[TMP5]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP4]], i64 [[PHI_BROKEN]]), !dbg [[DBG27]]
diff --git a/llvm/test/CodeGen/AMDGPU/si-sgpr-spill.ll b/llvm/test/CodeGen/AMDGPU/si-sgpr-spill.ll
index a14c456952db7..b662254fc3db0 100644
--- a/llvm/test/CodeGen/AMDGPU/si-sgpr-spill.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-sgpr-spill.ll
@@ -234,7 +234,7 @@ main_body:
   %tmp128 = load i32, ptr addrspace(3) %tmp118
   %tmp129 = bitcast i32 %tmp128 to float
   %tmp130 = fsub float %tmp129, %tmp127
-  %tmp131 = insertelement <4 x float> undef, float %tmp124, i32 0
+  %tmp131 = insertelement <4 x float> poison, float %tmp124, i32 0
   %tmp132 = insertelement <4 x float> %tmp131, float %tmp130, i32 1
   %tmp133 = insertelement <4 x float> %tmp132, float %tmp130, i32 2
   %tmp134 = insertelement <4 x float> %tmp133, float %tmp130, i32 3
@@ -290,7 +290,7 @@ main_body:
   %tmp174 = load i32, ptr addrspace(3) %tmp152
   %tmp175 = bitcast i32 %tmp174 to float
   %tmp176 = fsub float %tmp175, %tmp173
-  %tmp177 = insertelement <4 x float> undef, float %tmp158, i32 0
+  %tmp177 = insertelement <4 x float> poison, float %tmp158, i32 0
   %tmp178 = insertelement <4 x float> %tmp177, float %tmp164, i32 1
   %tmp179 = insertelement <4 x float> %tmp178, float %tmp170, i32 2
   %tmp180 = insertelement <4 x float> %tmp179, float %tmp176, i32 3
@@ -348,7 +348,7 @@ IF:                                               ; preds = %LOOP
   %tmp230 = bitcast float %tmp181 to i32
   %tmp231 = bitcast float %tmp136 to i32
   %tmp232 = bitcast float %tmp182 to i32
-  %tmp233 = insertelement <8 x i32> undef, i32 %tmp229, i32 0
+  %tmp233 = insertelement <8 x i32> poison, i32 %tmp229, i32 0
   %tmp234 = insertelement <8 x i32> %tmp233, i32 %tmp230, i32 1
   %tmp235 = insertelement <8 x i32> %tmp234, i32 %tmp231, i32 2
   %tmp236 = insertelement <8 x i32> %tmp235, i32 %tmp232, i32 3
@@ -492,7 +492,7 @@ IF67:                                             ; preds = %LOOP65
   %tmp445 = fadd float %tmp444, %tmp439
   %tmp446 = fmul float %tmp55, %tmp414
   %tmp447 = fadd float %tmp446, %tmp441
-  %tmp448 = insertelement <4 x float> undef, float %tmp443, i32 0
+  %tmp448 = insertelement <4 x float> poison, float %tmp443, i32 0
   %tmp449 = insertelement <4 x float> %tmp448, float %tmp445, i32 1
   %tmp450 = insertelement <4 x float> %tmp449, float %tmp447, i32 2
   %tmp451 = insertelement <4 x float> %tmp450, float %tmp194, i32 3
@@ -503,7 +503,7 @@ IF67:                                             ; preds = %LOOP65
   %cubesc = call float @llvm.amdgcn.cubesc(float %tmp451.x, float %tmp451.y, float %tmp451.z)
   %cubema = call float @llvm.amdgcn.cubema(float %tmp451.x, float %tmp451.y, float %tmp451.z)
   %cubeid = call float @llvm.amdgcn.cubeid(float %tmp451.x, float %tmp451.y, float %tmp451.z)
-  %tmp452.0 = insertelement <4 x float> undef, float %cubetc, i32 0
+  %tmp452.0 = insertelement <4 x float> poison, float %cubetc, i32 0
   %tmp452.1 = insertelement <4 x float> %tmp452.0, float %cubesc, i32 1
   %tmp452.2 = insertelement <4 x float> %tmp452.1, float %cubema, i32 2
   %tmp452 = insertelement <4 x float> %tmp452.2, float %cubeid, i32 3
@@ -1076,10 +1076,10 @@ LOOP:                                             ; preds = %LOOP, %main_body
   %temp170.0 = phi float [ %tmp252, %main_body ], [ %tmp286, %LOOP ]
   %tmp276 = bitcast float %temp168.0 to i32
   %tmp277 = bitcast float %temp169.0 to i32
-  %tmp278 = insertelement <4 x i32> undef, i32 %tmp276, i32 0
+  %tmp278 = insertelement <4 x i32> poison, i32 %tmp276, i32 0
   %tmp279 = insertelement <4 x i32> %tmp278, i32 %tmp277, i32 1
   %tmp280 = insertelement <4 x i32> %tmp279, i32 0, i32 2
-  %tmp281 = insertelement <4 x i32> %tmp280, i32 undef, i32 3
+  %tmp281 = insertelement <4 x i32> %tmp280, i32 poison, i32 3
   %tmp148.bc = bitcast <4 x i32> %tmp148 to <4 x i32>
   %tmp281.bc = bitcast <4 x i32> %tmp281 to <4 x float>
   %tmp282 = call <4 x float> @llvm.amdgcn.image.sample.l.2d.v4f32.f32(i32 15, float %temp168.0, float %temp169.0, float 0.0, <8 x i32> %tmp146, <4 x i32> %tmp148.bc, i1 0, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/si-spill-cf.ll b/llvm/test/CodeGen/AMDGPU/si-spill-cf.ll
index 7290b47658b3d..1e7bcafc4264c 100644
--- a/llvm/test/CodeGen/AMDGPU/si-spill-cf.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-spill-cf.ll
@@ -80,7 +80,7 @@ main_body:
 
 LOOP:                                             ; preds = %ENDIF2795, %main_body
   %temp894.0 = phi float [ 0.000000e+00, %main_body ], [ %temp894.1, %ENDIF2795 ]
-  %temp18.0 = phi float [ undef, %main_body ], [ %temp18.1, %ENDIF2795 ]
+  %temp18.0 = phi float [ poison, %main_body ], [ %temp18.1, %ENDIF2795 ]
   %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
   %tmp67 = icmp sgt i32 %tid, 4
   br i1 %tmp67, label %ENDLOOP, label %ENDIF
@@ -247,8 +247,8 @@ ELSE2566:                                         ; preds = %ENDIF
   br i1 %tmp201, label %ENDLOOP, label %ELSE2593
 
 ENDIF2564:                                        ; preds = %ENDIF2594, %ENDIF2588
-  %temp894.1 = phi float [ undef, %ENDIF2588 ], [ %temp894.2, %ENDIF2594 ]
-  %temp18.1 = phi float [ %tmp218, %ENDIF2588 ], [ undef, %ENDIF2594 ]
+  %temp894.1 = phi float [ poison, %ENDIF2588 ], [ %temp894.2, %ENDIF2594 ]
+  %temp18.1 = phi float [ %tmp218, %ENDIF2588 ], [ poison, %ENDIF2594 ]
   %tmp202 = fsub float %tmp5, undef
   %tmp203 = fmul float %tmp202, undef
   %tmp204 = call float @llvm.maxnum.f32(float undef, float %tmp203)
diff --git a/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll b/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
index 8fec79a7efd9b..1b4a3af796bcd 100644
--- a/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
@@ -316,7 +316,7 @@ define amdgpu_vs void @reorder_local_load_tbuffer_store_local_load(ptr addrspace
 
   %tmp1 = load i32, ptr addrspace(3) %ptr1, align 4
 
-  %vdata = insertelement <4 x i32> undef, i32 %a1, i32 0
+  %vdata = insertelement <4 x i32> poison, i32 %a1, i32 0
   %vaddr.add = add i32 %vaddr, 32
   call void @llvm.amdgcn.struct.ptr.tbuffer.store.v4i32(<4 x i32> %vdata, ptr addrspace(8) undef, i32 %vaddr.add, i32 0, i32 0, i32 228, i32 3)
 
diff --git a/llvm/test/CodeGen/AMDGPU/si-vector-hang.ll b/llvm/test/CodeGen/AMDGPU/si-vector-hang.ll
index 99efc0eb60d17..ee843dc2069d9 100644
--- a/llvm/test/CodeGen/AMDGPU/si-vector-hang.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-vector-hang.ll
@@ -15,7 +15,7 @@
 define amdgpu_kernel void @test_8_min_char(ptr addrspace(1) nocapture %out, ptr addrspace(1) nocapture readonly %in0, ptr addrspace(1) nocapture readonly %in1) #0 {
 entry:
   %0 = load i8, ptr addrspace(1) %in0, align 1
-  %1 = insertelement <8 x i8> undef, i8 %0, i32 0
+  %1 = insertelement <8 x i8> poison, i8 %0, i32 0
   %arrayidx2.i.i = getelementptr inbounds i8, ptr addrspace(1) %in0, i64 1
   %2 = load i8, ptr addrspace(1) %arrayidx2.i.i, align 1
   %3 = insertelement <8 x i8> %1, i8 %2, i32 1
@@ -27,7 +27,7 @@ entry:
   %7 = insertelement <8 x i8> %5, i8 %6, i32 3
   %arrayidx.i.i = getelementptr inbounds i8, ptr addrspace(1) %in0, i64 4
   %8 = load i8, ptr addrspace(1) %arrayidx.i.i, align 1
-  %9 = insertelement <8 x i8> undef, i8 %8, i32 0
+  %9 = insertelement <8 x i8> poison, i8 %8, i32 0
   %arrayidx2.i9.i = getelementptr inbounds i8, ptr addrspace(1) %in0, i64 5
   %10 = load i8, ptr addrspace(1) %arrayidx2.i9.i, align 1
   %11 = insertelement <8 x i8> %9, i8 %10, i32 1
@@ -39,7 +39,7 @@ entry:
   %15 = insertelement <8 x i8> %13, i8 %14, i32 3
   %vecinit5.i = shufflevector <8 x i8> %7, <8 x i8> %15, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
   %16 = load i8, ptr addrspace(1) %in1, align 1
-  %17 = insertelement <8 x i8> undef, i8 %16, i32 0
+  %17 = insertelement <8 x i8> poison, i8 %16, i32 0
   %arrayidx2.i.i4 = getelementptr inbounds i8, ptr addrspace(1) %in1, i64 1
   %18 = load i8, ptr addrspace(1) %arrayidx2.i.i4, align 1
   %19 = insertelement <8 x i8> %17, i8 %18, i32 1
@@ -51,7 +51,7 @@ entry:
   %23 = insertelement <8 x i8> %21, i8 %22, i32 3
   %arrayidx.i.i7 = getelementptr inbounds i8, ptr addrspace(1) %in1, i64 4
   %24 = load i8, ptr addrspace(1) %arrayidx.i.i7, align 1
-  %25 = insertelement <8 x i8> undef, i8 %24, i32 0
+  %25 = insertelement <8 x i8> poison, i8 %24, i32 0
   %arrayidx2.i9.i8 = getelementptr inbounds i8, ptr addrspace(1) %in1, i64 5
   %26 = load i8, ptr addrspace(1) %arrayidx2.i9.i8, align 1
   %27 = insertelement <8 x i8> %25, i8 %26, i32 1
diff --git a/llvm/test/CodeGen/AMDGPU/sibling-call.ll b/llvm/test/CodeGen/AMDGPU/sibling-call.ll
index b57adfe7d9306..0221bb0cf4f35 100644
--- a/llvm/test/CodeGen/AMDGPU/sibling-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/sibling-call.ll
@@ -612,40 +612,40 @@ define hidden fastcc i32 @indirect_divergent_sibling_call_i32_fastcc_i32_i32(ptr
 ; FIJI-NEXT:    v_writelane_b32 v40, s37, 5
 ; FIJI-NEXT:    v_writelane_b32 v40, s38, 6
 ; FIJI-NEXT:    v_writelane_b32 v40, s39, 7
-; FIJI-NEXT:    v_writelane_b32 v40, s40, 8
-; FIJI-NEXT:    v_writelane_b32 v40, s41, 9
-; FIJI-NEXT:    v_writelane_b32 v40, s42, 10
-; FIJI-NEXT:    v_writelane_b32 v40, s43, 11
-; FIJI-NEXT:    v_writelane_b32 v40, s44, 12
-; FIJI-NEXT:    v_writelane_b32 v40, s45, 13
-; FIJI-NEXT:    v_writelane_b32 v40, s46, 14
-; FIJI-NEXT:    v_writelane_b32 v40, s47, 15
-; FIJI-NEXT:    v_writelane_b32 v40, s48, 16
-; FIJI-NEXT:    s_mov_b32 s42, s15
-; FIJI-NEXT:    s_mov_b32 s43, s14
-; FIJI-NEXT:    s_mov_b32 s44, s13
-; FIJI-NEXT:    s_mov_b32 s45, s12
+; FIJI-NEXT:    v_writelane_b32 v40, s48, 8
+; FIJI-NEXT:    v_writelane_b32 v40, s49, 9
+; FIJI-NEXT:    v_writelane_b32 v40, s50, 10
+; FIJI-NEXT:    v_writelane_b32 v40, s51, 11
+; FIJI-NEXT:    v_writelane_b32 v40, s52, 12
+; FIJI-NEXT:    v_writelane_b32 v40, s53, 13
+; FIJI-NEXT:    v_writelane_b32 v40, s54, 14
+; FIJI-NEXT:    v_writelane_b32 v40, s55, 15
+; FIJI-NEXT:    v_writelane_b32 v40, s64, 16
+; FIJI-NEXT:    s_mov_b32 s50, s15
+; FIJI-NEXT:    s_mov_b32 s51, s14
+; FIJI-NEXT:    s_mov_b32 s52, s13
+; FIJI-NEXT:    s_mov_b32 s53, s12
 ; FIJI-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; FIJI-NEXT:    s_mov_b64 s[36:37], s[8:9]
 ; FIJI-NEXT:    s_mov_b64 s[38:39], s[6:7]
-; FIJI-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; FIJI-NEXT:    s_mov_b64 s[48:49], s[4:5]
 ; FIJI-NEXT:    v_add_u32_e32 v3, vcc, v3, v4
-; FIJI-NEXT:    s_mov_b64 s[46:47], exec
+; FIJI-NEXT:    s_mov_b64 s[54:55], exec
 ; FIJI-NEXT:    s_addk_i32 s32, 0x400
-; FIJI-NEXT:    v_writelane_b32 v40, s49, 17
+; FIJI-NEXT:    v_writelane_b32 v40, s65, 17
 ; FIJI-NEXT:  .LBB18_1: ; =>This Inner Loop Header: Depth=1
 ; FIJI-NEXT:    v_readfirstlane_b32 s16, v0
 ; FIJI-NEXT:    v_readfirstlane_b32 s17, v1
 ; FIJI-NEXT:    v_cmp_eq_u64_e32 vcc, s[16:17], v[0:1]
-; FIJI-NEXT:    s_and_saveexec_b64 s[48:49], vcc
-; FIJI-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; FIJI-NEXT:    s_and_saveexec_b64 s[64:65], vcc
+; FIJI-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; FIJI-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; FIJI-NEXT:    s_mov_b64 s[8:9], s[36:37]
 ; FIJI-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; FIJI-NEXT:    s_mov_b32 s12, s45
-; FIJI-NEXT:    s_mov_b32 s13, s44
-; FIJI-NEXT:    s_mov_b32 s14, s43
-; FIJI-NEXT:    s_mov_b32 s15, s42
+; FIJI-NEXT:    s_mov_b32 s12, s53
+; FIJI-NEXT:    s_mov_b32 s13, s52
+; FIJI-NEXT:    s_mov_b32 s14, s51
+; FIJI-NEXT:    s_mov_b32 s15, s50
 ; FIJI-NEXT:    v_mov_b32_e32 v0, v2
 ; FIJI-NEXT:    v_mov_b32_e32 v1, v3
 ; FIJI-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -654,21 +654,21 @@ define hidden fastcc i32 @indirect_divergent_sibling_call_i32_fastcc_i32_i32(ptr
 ; FIJI-NEXT:    ; implicit-def: $vgpr31
 ; FIJI-NEXT:    ; implicit-def: $vgpr2
 ; FIJI-NEXT:    ; implicit-def: $vgpr3
-; FIJI-NEXT:    s_xor_b64 exec, exec, s[48:49]
+; FIJI-NEXT:    s_xor_b64 exec, exec, s[64:65]
 ; FIJI-NEXT:    s_cbranch_execnz .LBB18_1
 ; FIJI-NEXT:  ; %bb.2:
-; FIJI-NEXT:    s_mov_b64 exec, s[46:47]
+; FIJI-NEXT:    s_mov_b64 exec, s[54:55]
 ; FIJI-NEXT:    v_mov_b32_e32 v0, v4
-; FIJI-NEXT:    v_readlane_b32 s49, v40, 17
-; FIJI-NEXT:    v_readlane_b32 s48, v40, 16
-; FIJI-NEXT:    v_readlane_b32 s47, v40, 15
-; FIJI-NEXT:    v_readlane_b32 s46, v40, 14
-; FIJI-NEXT:    v_readlane_b32 s45, v40, 13
-; FIJI-NEXT:    v_readlane_b32 s44, v40, 12
-; FIJI-NEXT:    v_readlane_b32 s43, v40, 11
-; FIJI-NEXT:    v_readlane_b32 s42, v40, 10
-; FIJI-NEXT:    v_readlane_b32 s41, v40, 9
-; FIJI-NEXT:    v_readlane_b32 s40, v40, 8
+; FIJI-NEXT:    v_readlane_b32 s65, v40, 17
+; FIJI-NEXT:    v_readlane_b32 s64, v40, 16
+; FIJI-NEXT:    v_readlane_b32 s55, v40, 15
+; FIJI-NEXT:    v_readlane_b32 s54, v40, 14
+; FIJI-NEXT:    v_readlane_b32 s53, v40, 13
+; FIJI-NEXT:    v_readlane_b32 s52, v40, 12
+; FIJI-NEXT:    v_readlane_b32 s51, v40, 11
+; FIJI-NEXT:    v_readlane_b32 s50, v40, 10
+; FIJI-NEXT:    v_readlane_b32 s49, v40, 9
+; FIJI-NEXT:    v_readlane_b32 s48, v40, 8
 ; FIJI-NEXT:    v_readlane_b32 s39, v40, 7
 ; FIJI-NEXT:    v_readlane_b32 s38, v40, 6
 ; FIJI-NEXT:    v_readlane_b32 s37, v40, 5
@@ -703,40 +703,40 @@ define hidden fastcc i32 @indirect_divergent_sibling_call_i32_fastcc_i32_i32(ptr
 ; HAWAII-NEXT:    v_writelane_b32 v40, s37, 5
 ; HAWAII-NEXT:    v_writelane_b32 v40, s38, 6
 ; HAWAII-NEXT:    v_writelane_b32 v40, s39, 7
-; HAWAII-NEXT:    v_writelane_b32 v40, s40, 8
-; HAWAII-NEXT:    v_writelane_b32 v40, s41, 9
-; HAWAII-NEXT:    v_writelane_b32 v40, s42, 10
-; HAWAII-NEXT:    v_writelane_b32 v40, s43, 11
-; HAWAII-NEXT:    v_writelane_b32 v40, s44, 12
-; HAWAII-NEXT:    v_writelane_b32 v40, s45, 13
-; HAWAII-NEXT:    v_writelane_b32 v40, s46, 14
-; HAWAII-NEXT:    v_writelane_b32 v40, s47, 15
-; HAWAII-NEXT:    v_writelane_b32 v40, s48, 16
-; HAWAII-NEXT:    s_mov_b32 s42, s15
-; HAWAII-NEXT:    s_mov_b32 s43, s14
-; HAWAII-NEXT:    s_mov_b32 s44, s13
-; HAWAII-NEXT:    s_mov_b32 s45, s12
+; HAWAII-NEXT:    v_writelane_b32 v40, s48, 8
+; HAWAII-NEXT:    v_writelane_b32 v40, s49, 9
+; HAWAII-NEXT:    v_writelane_b32 v40, s50, 10
+; HAWAII-NEXT:    v_writelane_b32 v40, s51, 11
+; HAWAII-NEXT:    v_writelane_b32 v40, s52, 12
+; HAWAII-NEXT:    v_writelane_b32 v40, s53, 13
+; HAWAII-NEXT:    v_writelane_b32 v40, s54, 14
+; HAWAII-NEXT:    v_writelane_b32 v40, s55, 15
+; HAWAII-NEXT:    v_writelane_b32 v40, s64, 16
+; HAWAII-NEXT:    s_mov_b32 s50, s15
+; HAWAII-NEXT:    s_mov_b32 s51, s14
+; HAWAII-NEXT:    s_mov_b32 s52, s13
+; HAWAII-NEXT:    s_mov_b32 s53, s12
 ; HAWAII-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; HAWAII-NEXT:    s_mov_b64 s[36:37], s[8:9]
 ; HAWAII-NEXT:    s_mov_b64 s[38:39], s[6:7]
-; HAWAII-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; HAWAII-NEXT:    s_mov_b64 s[48:49], s[4:5]
 ; HAWAII-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
-; HAWAII-NEXT:    s_mov_b64 s[46:47], exec
+; HAWAII-NEXT:    s_mov_b64 s[54:55], exec
 ; HAWAII-NEXT:    s_addk_i32 s32, 0x400
-; HAWAII-NEXT:    v_writelane_b32 v40, s49, 17
+; HAWAII-NEXT:    v_writelane_b32 v40, s65, 17
 ; HAWAII-NEXT:  .LBB18_1: ; =>This Inner Loop Header: Depth=1
 ; HAWAII-NEXT:    v_readfirstlane_b32 s16, v0
 ; HAWAII-NEXT:    v_readfirstlane_b32 s17, v1
 ; HAWAII-NEXT:    v_cmp_eq_u64_e32 vcc, s[16:17], v[0:1]
-; HAWAII-NEXT:    s_and_saveexec_b64 s[48:49], vcc
-; HAWAII-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; HAWAII-NEXT:    s_and_saveexec_b64 s[64:65], vcc
+; HAWAII-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; HAWAII-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; HAWAII-NEXT:    s_mov_b64 s[8:9], s[36:37]
 ; HAWAII-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; HAWAII-NEXT:    s_mov_b32 s12, s45
-; HAWAII-NEXT:    s_mov_b32 s13, s44
-; HAWAII-NEXT:    s_mov_b32 s14, s43
-; HAWAII-NEXT:    s_mov_b32 s15, s42
+; HAWAII-NEXT:    s_mov_b32 s12, s53
+; HAWAII-NEXT:    s_mov_b32 s13, s52
+; HAWAII-NEXT:    s_mov_b32 s14, s51
+; HAWAII-NEXT:    s_mov_b32 s15, s50
 ; HAWAII-NEXT:    v_mov_b32_e32 v0, v2
 ; HAWAII-NEXT:    v_mov_b32_e32 v1, v3
 ; HAWAII-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -745,21 +745,21 @@ define hidden fastcc i32 @indirect_divergent_sibling_call_i32_fastcc_i32_i32(ptr
 ; HAWAII-NEXT:    ; implicit-def: $vgpr31
 ; HAWAII-NEXT:    ; implicit-def: $vgpr2
 ; HAWAII-NEXT:    ; implicit-def: $vgpr3
-; HAWAII-NEXT:    s_xor_b64 exec, exec, s[48:49]
+; HAWAII-NEXT:    s_xor_b64 exec, exec, s[64:65]
 ; HAWAII-NEXT:    s_cbranch_execnz .LBB18_1
 ; HAWAII-NEXT:  ; %bb.2:
-; HAWAII-NEXT:    s_mov_b64 exec, s[46:47]
+; HAWAII-NEXT:    s_mov_b64 exec, s[54:55]
 ; HAWAII-NEXT:    v_mov_b32_e32 v0, v4
-; HAWAII-NEXT:    v_readlane_b32 s49, v40, 17
-; HAWAII-NEXT:    v_readlane_b32 s48, v40, 16
-; HAWAII-NEXT:    v_readlane_b32 s47, v40, 15
-; HAWAII-NEXT:    v_readlane_b32 s46, v40, 14
-; HAWAII-NEXT:    v_readlane_b32 s45, v40, 13
-; HAWAII-NEXT:    v_readlane_b32 s44, v40, 12
-; HAWAII-NEXT:    v_readlane_b32 s43, v40, 11
-; HAWAII-NEXT:    v_readlane_b32 s42, v40, 10
-; HAWAII-NEXT:    v_readlane_b32 s41, v40, 9
-; HAWAII-NEXT:    v_readlane_b32 s40, v40, 8
+; HAWAII-NEXT:    v_readlane_b32 s65, v40, 17
+; HAWAII-NEXT:    v_readlane_b32 s64, v40, 16
+; HAWAII-NEXT:    v_readlane_b32 s55, v40, 15
+; HAWAII-NEXT:    v_readlane_b32 s54, v40, 14
+; HAWAII-NEXT:    v_readlane_b32 s53, v40, 13
+; HAWAII-NEXT:    v_readlane_b32 s52, v40, 12
+; HAWAII-NEXT:    v_readlane_b32 s51, v40, 11
+; HAWAII-NEXT:    v_readlane_b32 s50, v40, 10
+; HAWAII-NEXT:    v_readlane_b32 s49, v40, 9
+; HAWAII-NEXT:    v_readlane_b32 s48, v40, 8
 ; HAWAII-NEXT:    v_readlane_b32 s39, v40, 7
 ; HAWAII-NEXT:    v_readlane_b32 s38, v40, 6
 ; HAWAII-NEXT:    v_readlane_b32 s37, v40, 5
@@ -794,40 +794,40 @@ define hidden fastcc i32 @indirect_divergent_sibling_call_i32_fastcc_i32_i32(ptr
 ; GFX9-NEXT:    v_writelane_b32 v40, s37, 5
 ; GFX9-NEXT:    v_writelane_b32 v40, s38, 6
 ; GFX9-NEXT:    v_writelane_b32 v40, s39, 7
-; GFX9-NEXT:    v_writelane_b32 v40, s40, 8
-; GFX9-NEXT:    v_writelane_b32 v40, s41, 9
-; GFX9-NEXT:    v_writelane_b32 v40, s42, 10
-; GFX9-NEXT:    v_writelane_b32 v40, s43, 11
-; GFX9-NEXT:    v_writelane_b32 v40, s44, 12
-; GFX9-NEXT:    v_writelane_b32 v40, s45, 13
-; GFX9-NEXT:    v_writelane_b32 v40, s46, 14
-; GFX9-NEXT:    v_writelane_b32 v40, s47, 15
-; GFX9-NEXT:    v_writelane_b32 v40, s48, 16
-; GFX9-NEXT:    s_mov_b32 s42, s15
-; GFX9-NEXT:    s_mov_b32 s43, s14
-; GFX9-NEXT:    s_mov_b32 s44, s13
-; GFX9-NEXT:    s_mov_b32 s45, s12
+; GFX9-NEXT:    v_writelane_b32 v40, s48, 8
+; GFX9-NEXT:    v_writelane_b32 v40, s49, 9
+; GFX9-NEXT:    v_writelane_b32 v40, s50, 10
+; GFX9-NEXT:    v_writelane_b32 v40, s51, 11
+; GFX9-NEXT:    v_writelane_b32 v40, s52, 12
+; GFX9-NEXT:    v_writelane_b32 v40, s53, 13
+; GFX9-NEXT:    v_writelane_b32 v40, s54, 14
+; GFX9-NEXT:    v_writelane_b32 v40, s55, 15
+; GFX9-NEXT:    v_writelane_b32 v40, s64, 16
+; GFX9-NEXT:    s_mov_b32 s50, s15
+; GFX9-NEXT:    s_mov_b32 s51, s14
+; GFX9-NEXT:    s_mov_b32 s52, s13
+; GFX9-NEXT:    s_mov_b32 s53, s12
 ; GFX9-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; GFX9-NEXT:    s_mov_b64 s[36:37], s[8:9]
 ; GFX9-NEXT:    s_mov_b64 s[38:39], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[48:49], s[4:5]
 ; GFX9-NEXT:    v_add_u32_e32 v3, v3, v4
-; GFX9-NEXT:    s_mov_b64 s[46:47], exec
+; GFX9-NEXT:    s_mov_b64 s[54:55], exec
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s49, 17
+; GFX9-NEXT:    v_writelane_b32 v40, s65, 17
 ; GFX9-NEXT:  .LBB18_1: ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    v_readfirstlane_b32 s16, v0
 ; GFX9-NEXT:    v_readfirstlane_b32 s17, v1
 ; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, s[16:17], v[0:1]
-; GFX9-NEXT:    s_and_saveexec_b64 s[48:49], vcc
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GFX9-NEXT:    s_and_saveexec_b64 s[64:65], vcc
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GFX9-NEXT:    s_mov_b64 s[8:9], s[36:37]
 ; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s45
-; GFX9-NEXT:    s_mov_b32 s13, s44
-; GFX9-NEXT:    s_mov_b32 s14, s43
-; GFX9-NEXT:    s_mov_b32 s15, s42
+; GFX9-NEXT:    s_mov_b32 s12, s53
+; GFX9-NEXT:    s_mov_b32 s13, s52
+; GFX9-NEXT:    s_mov_b32 s14, s51
+; GFX9-NEXT:    s_mov_b32 s15, s50
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
@@ -836,21 +836,21 @@ define hidden fastcc i32 @indirect_divergent_sibling_call_i32_fastcc_i32_i32(ptr
 ; GFX9-NEXT:    ; implicit-def: $vgpr31
 ; GFX9-NEXT:    ; implicit-def: $vgpr2
 ; GFX9-NEXT:    ; implicit-def: $vgpr3
-; GFX9-NEXT:    s_xor_b64 exec, exec, s[48:49]
+; GFX9-NEXT:    s_xor_b64 exec, exec, s[64:65]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB18_1
 ; GFX9-NEXT:  ; %bb.2:
-; GFX9-NEXT:    s_mov_b64 exec, s[46:47]
+; GFX9-NEXT:    s_mov_b64 exec, s[54:55]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v4
-; GFX9-NEXT:    v_readlane_b32 s49, v40, 17
-; GFX9-NEXT:    v_readlane_b32 s48, v40, 16
-; GFX9-NEXT:    v_readlane_b32 s47, v40, 15
-; GFX9-NEXT:    v_readlane_b32 s46, v40, 14
-; GFX9-NEXT:    v_readlane_b32 s45, v40, 13
-; GFX9-NEXT:    v_readlane_b32 s44, v40, 12
-; GFX9-NEXT:    v_readlane_b32 s43, v40, 11
-; GFX9-NEXT:    v_readlane_b32 s42, v40, 10
-; GFX9-NEXT:    v_readlane_b32 s41, v40, 9
-; GFX9-NEXT:    v_readlane_b32 s40, v40, 8
+; GFX9-NEXT:    v_readlane_b32 s65, v40, 17
+; GFX9-NEXT:    v_readlane_b32 s64, v40, 16
+; GFX9-NEXT:    v_readlane_b32 s55, v40, 15
+; GFX9-NEXT:    v_readlane_b32 s54, v40, 14
+; GFX9-NEXT:    v_readlane_b32 s53, v40, 13
+; GFX9-NEXT:    v_readlane_b32 s52, v40, 12
+; GFX9-NEXT:    v_readlane_b32 s51, v40, 11
+; GFX9-NEXT:    v_readlane_b32 s50, v40, 10
+; GFX9-NEXT:    v_readlane_b32 s49, v40, 9
+; GFX9-NEXT:    v_readlane_b32 s48, v40, 8
 ; GFX9-NEXT:    v_readlane_b32 s39, v40, 7
 ; GFX9-NEXT:    v_readlane_b32 s38, v40, 6
 ; GFX9-NEXT:    v_readlane_b32 s37, v40, 5
diff --git a/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll b/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll
index 4993df7e1ba48..6a88be6e55859 100644
--- a/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll
+++ b/llvm/test/CodeGen/AMDGPU/simplify-libcalls.ll
@@ -56,13 +56,13 @@ declare <2 x float> @_Z3cosDv2_f(<2 x float>)
 define amdgpu_kernel void @test_sincos_v3(ptr addrspace(1) nocapture %a) {
 entry:
   %loadVec4 = load <4 x float>, ptr addrspace(1) %a, align 16
-  %extractVec4 = shufflevector <4 x float> %loadVec4, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  %extractVec4 = shufflevector <4 x float> %loadVec4, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
   %call = call fast <3 x float> @_Z3sinDv3_f(<3 x float> %extractVec4)
-  %extractVec6 = shufflevector <3 x float> %call, <3 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %extractVec6 = shufflevector <3 x float> %call, <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
   store <4 x float> %extractVec6, ptr addrspace(1) %a, align 16
   %call11 = call fast <3 x float> @_Z3cosDv3_f(<3 x float> %extractVec4)
   %arrayidx12 = getelementptr inbounds <3 x float>, ptr addrspace(1) %a, i64 1
-  %extractVec13 = shufflevector <3 x float> %call11, <3 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %extractVec13 = shufflevector <3 x float> %call11, <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
   store <4 x float> %extractVec13, ptr addrspace(1) %arrayidx12, align 16
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/simplifydemandedbits-recursion.ll b/llvm/test/CodeGen/AMDGPU/simplifydemandedbits-recursion.ll
index fec2fcba83f4a..bdbe4f1b4b694 100644
--- a/llvm/test/CodeGen/AMDGPU/simplifydemandedbits-recursion.ll
+++ b/llvm/test/CodeGen/AMDGPU/simplifydemandedbits-recursion.ll
@@ -66,7 +66,7 @@ bb26:                                             ; preds = %.loopexit
   br label %bb31
 
 .preheader:                                       ; preds = %.preheader, %bb12
-  %tmp27 = phi i32 [ %tmp28, %.preheader ], [ undef, %bb12 ]
+  %tmp27 = phi i32 [ %tmp28, %.preheader ], [ poison, %bb12 ]
   %tmp28 = add nuw i32 %tmp27, 128
   %tmp29 = icmp ult i32 %tmp28, 1568
   br i1 %tmp29, label %.preheader, label %.loopexit145
@@ -75,7 +75,7 @@ bb30:                                             ; preds = %bb31
   br i1 %c4, label %bb11, label %bb12
 
 bb31:                                             ; preds = %bb31, %bb26
-  %tmp32 = phi i32 [ %tmp9, %bb26 ], [ undef, %bb31 ]
+  %tmp32 = phi i32 [ %tmp9, %bb26 ], [ poison, %bb31 ]
   %tmp33 = getelementptr inbounds [462 x float], ptr addrspace(3) @0, i32 0, i32 %tmp32
   %tmp34 = load float, ptr addrspace(3) %tmp33, align 4
   %tmp35 = tail call float @llvm.fmuladd.f32(float %tmp34, float undef, float undef)
diff --git a/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll b/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll
index 0501602bbd8f4..80db5a7f1d794 100644
--- a/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll
+++ b/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll
@@ -1690,7 +1690,7 @@ live:
   br label %export
 
 export:
-  %proxy = phi float [ undef, %kill ], [ %scale, %live ]
+  %proxy = phi float [ poison, %kill ], [ %scale, %live ]
   call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %proxy, float %proxy, float %proxy, float %proxy, i1 true, i1 true) #3
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/skip-promote-alloca-vector-users.ll b/llvm/test/CodeGen/AMDGPU/skip-promote-alloca-vector-users.ll
index e14ae06b80316..a266c6385e8c8 100644
--- a/llvm/test/CodeGen/AMDGPU/skip-promote-alloca-vector-users.ll
+++ b/llvm/test/CodeGen/AMDGPU/skip-promote-alloca-vector-users.ll
@@ -4,33 +4,33 @@
 
 ; CHECK-LABEL: @test_insertelement(
 ; CHECK:  %alloca = alloca i16
-; CHECK-NEXT:  insertelement <2 x ptr addrspace(5)> undef, ptr addrspace(5) %alloca, i32 0
+; CHECK-NEXT:  insertelement <2 x ptr addrspace(5)> poison, ptr addrspace(5) %alloca, i32 0
 define amdgpu_kernel void @test_insertelement() #0 {
 entry:
   %alloca = alloca i16, align 4, addrspace(5)
-  %in = insertelement <2 x ptr addrspace(5)> undef, ptr addrspace(5) %alloca, i32 0
+  %in = insertelement <2 x ptr addrspace(5)> poison, ptr addrspace(5) %alloca, i32 0
   store <2 x ptr addrspace(5)> %in, ptr undef, align 4
   ret void
 }
 
 ; CHECK-LABEL: @test_insertvalue(
 ; CHECK:  %alloca = alloca i16
-; CHECK-NEXT:  insertvalue { ptr addrspace(5) } undef, ptr addrspace(5) %alloca, 0
+; CHECK-NEXT:  insertvalue { ptr addrspace(5) } poison, ptr addrspace(5) %alloca, 0
 define amdgpu_kernel void @test_insertvalue() #0 {
 entry:
   %alloca = alloca i16, align 4, addrspace(5)
-  %in = insertvalue { ptr addrspace(5) } undef, ptr addrspace(5) %alloca, 0
+  %in = insertvalue { ptr addrspace(5) } poison, ptr addrspace(5) %alloca, 0
   store { ptr addrspace(5) } %in, ptr undef, align 4
   ret void
 }
 
 ; CHECK-LABEL: @test_insertvalue_array(
 ; CHECK:  %alloca = alloca i16
-; CHECK-NEXT:  insertvalue [2 x ptr addrspace(5)] undef, ptr addrspace(5) %alloca, 0
+; CHECK-NEXT:  insertvalue [2 x ptr addrspace(5)] poison, ptr addrspace(5) %alloca, 0
 define amdgpu_kernel void @test_insertvalue_array() #0 {
 entry:
   %alloca = alloca i16, align 4, addrspace(5)
-  %in = insertvalue [2 x ptr addrspace(5)] undef, ptr addrspace(5) %alloca, 0
+  %in = insertvalue [2 x ptr addrspace(5)] poison, ptr addrspace(5) %alloca, 0
   store [2 x ptr addrspace(5)] %in, ptr undef, align 4
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/smfmac_no_agprs.ll b/llvm/test/CodeGen/AMDGPU/smfmac_no_agprs.ll
index 899ec36e9b2fe..d8c015b85584a 100644
--- a/llvm/test/CodeGen/AMDGPU/smfmac_no_agprs.ll
+++ b/llvm/test/CodeGen/AMDGPU/smfmac_no_agprs.ll
@@ -35,9 +35,9 @@ entry:
   %1 = load i32, ptr addrspace(1) %arrayidx1
   %2 = load i32, ptr addrspace(1) %arrayidx2
   %3 = load i32, ptr addrspace(1) %arrayidx3
-  %src1.0 = insertelement <2 x i32> undef, i32 %0, i64 0
+  %src1.0 = insertelement <2 x i32> poison, i32 %0, i64 0
   %src1 = insertelement <2 x i32> %src1.0, i32 %1, i64 1
-  %src2.0 = insertelement <4 x i32> undef, i32 %2, i64 0
+  %src2.0 = insertelement <4 x i32> poison, i32 %2, i64 0
   %src2.1 = insertelement <4 x i32> %src2.0, i32 %3, i64 1
   %src2.2 = insertelement <4 x i32> %src2.1, i32 %3, i64 2
   %src2 = insertelement <4 x i32> %src2.2, i32 %3, i64 3
diff --git a/llvm/test/CodeGen/AMDGPU/sminmax.ll b/llvm/test/CodeGen/AMDGPU/sminmax.ll
index 65ae200275a0e..78ce539808306 100644
--- a/llvm/test/CodeGen/AMDGPU/sminmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/sminmax.ll
@@ -65,9 +65,9 @@ define amdgpu_kernel void @v_abs_i32_repeat_user(ptr addrspace(1) %out, ptr addr
 ; EG: MAX_INT
 ; EG: MAX_INT
 define amdgpu_kernel void @s_abs_v2i32(ptr addrspace(1) %out, <2 x i32> %val) nounwind {
-  %z0 = insertelement <2 x i32> undef, i32 0, i32 0
+  %z0 = insertelement <2 x i32> poison, i32 0, i32 0
   %z1 = insertelement <2 x i32> %z0, i32 0, i32 1
-  %t0 = insertelement <2 x i32> undef, i32 2, i32 0
+  %t0 = insertelement <2 x i32> poison, i32 2, i32 0
   %t1 = insertelement <2 x i32> %t0, i32 2, i32 1
   %neg = sub <2 x i32> %z1, %val
   %cond = icmp sgt <2 x i32> %val, %neg
@@ -96,9 +96,9 @@ define amdgpu_kernel void @s_abs_v2i32(ptr addrspace(1) %out, <2 x i32> %val) no
 ; EG: MAX_INT
 ; EG: MAX_INT
 define amdgpu_kernel void @v_abs_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %src) nounwind {
-  %z0 = insertelement <2 x i32> undef, i32 0, i32 0
+  %z0 = insertelement <2 x i32> poison, i32 0, i32 0
   %z1 = insertelement <2 x i32> %z0, i32 0, i32 1
-  %t0 = insertelement <2 x i32> undef, i32 2, i32 0
+  %t0 = insertelement <2 x i32> poison, i32 2, i32 0
   %t1 = insertelement <2 x i32> %t0, i32 2, i32 1
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.in = getelementptr inbounds <2 x i32>, ptr addrspace(1) %src, i32 %tid
@@ -128,11 +128,11 @@ define amdgpu_kernel void @v_abs_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %
 ; EG: MAX_INT
 ; EG: MAX_INT
 define amdgpu_kernel void @s_abs_v4i32(ptr addrspace(1) %out, <4 x i32> %val) nounwind {
-  %z0 = insertelement <4 x i32> undef, i32 0, i32 0
+  %z0 = insertelement <4 x i32> poison, i32 0, i32 0
   %z1 = insertelement <4 x i32> %z0, i32 0, i32 1
   %z2 = insertelement <4 x i32> %z1, i32 0, i32 2
   %z3 = insertelement <4 x i32> %z2, i32 0, i32 3
-  %t0 = insertelement <4 x i32> undef, i32 2, i32 0
+  %t0 = insertelement <4 x i32> poison, i32 2, i32 0
   %t1 = insertelement <4 x i32> %t0, i32 2, i32 1
   %t2 = insertelement <4 x i32> %t1, i32 2, i32 2
   %t3 = insertelement <4 x i32> %t2, i32 2, i32 3
@@ -176,11 +176,11 @@ define amdgpu_kernel void @s_abs_v4i32(ptr addrspace(1) %out, <4 x i32> %val) no
 ; EG: MAX_INT
 ; EG: MAX_INT
 define amdgpu_kernel void @v_abs_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %src) nounwind {
-  %z0 = insertelement <4 x i32> undef, i32 0, i32 0
+  %z0 = insertelement <4 x i32> poison, i32 0, i32 0
   %z1 = insertelement <4 x i32> %z0, i32 0, i32 1
   %z2 = insertelement <4 x i32> %z1, i32 0, i32 2
   %z3 = insertelement <4 x i32> %z2, i32 0, i32 3
-  %t0 = insertelement <4 x i32> undef, i32 2, i32 0
+  %t0 = insertelement <4 x i32> poison, i32 2, i32 0
   %t1 = insertelement <4 x i32> %t0, i32 2, i32 1
   %t2 = insertelement <4 x i32> %t1, i32 2, i32 2
   %t3 = insertelement <4 x i32> %t2, i32 2, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
index 71033cfd1a6f3..ff93d6f7c3961 100644
--- a/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
@@ -215,9 +215,9 @@ define amdgpu_kernel void @s_abs_v2i16_2(ptr addrspace(1) %out, <2 x i16> %val)
 ; CI-NEXT:    v_mov_b32_e32 v0, s4
 ; CI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; CI-NEXT:    s_endpgm
-  %z0 = insertelement <2 x i16> undef, i16 0, i16 0
+  %z0 = insertelement <2 x i16> poison, i16 0, i16 0
   %z1 = insertelement <2 x i16> %z0, i16 0, i16 1
-  %t0 = insertelement <2 x i16> undef, i16 2, i16 0
+  %t0 = insertelement <2 x i16> poison, i16 2, i16 0
   %t1 = insertelement <2 x i16> %t0, i16 2, i16 1
   %neg = sub <2 x i16> %z1, %val
   %cond = icmp sgt <2 x i16> %val, %neg
@@ -297,9 +297,9 @@ define amdgpu_kernel void @v_abs_v2i16_2(ptr addrspace(1) %out, ptr addrspace(1)
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, 0x20000, v0
 ; CI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; CI-NEXT:    s_endpgm
-  %z0 = insertelement <2 x i16> undef, i16 0, i16 0
+  %z0 = insertelement <2 x i16> poison, i16 0, i16 0
   %z1 = insertelement <2 x i16> %z0, i16 0, i16 1
-  %t0 = insertelement <2 x i16> undef, i16 2, i16 0
+  %t0 = insertelement <2 x i16> poison, i16 2, i16 0
   %t1 = insertelement <2 x i16> %t0, i16 2, i16 1
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.in = getelementptr inbounds <2 x i16>, ptr addrspace(1) %src, i32 %tid
@@ -406,11 +406,11 @@ define amdgpu_kernel void @s_abs_v4i16(ptr addrspace(1) %out, <4 x i16> %val) #0
 ; CI-NEXT:    v_mov_b32_e32 v1, s0
 ; CI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; CI-NEXT:    s_endpgm
-  %z0 = insertelement <4 x i16> undef, i16 0, i16 0
+  %z0 = insertelement <4 x i16> poison, i16 0, i16 0
   %z1 = insertelement <4 x i16> %z0, i16 0, i16 1
   %z2 = insertelement <4 x i16> %z1, i16 0, i16 2
   %z3 = insertelement <4 x i16> %z2, i16 0, i16 3
-  %t0 = insertelement <4 x i16> undef, i16 2, i16 0
+  %t0 = insertelement <4 x i16> poison, i16 2, i16 0
   %t1 = insertelement <4 x i16> %t0, i16 2, i16 1
   %t2 = insertelement <4 x i16> %t1, i16 2, i16 2
   %t3 = insertelement <4 x i16> %t2, i16 2, i16 3
@@ -518,11 +518,11 @@ define amdgpu_kernel void @v_abs_v4i16(ptr addrspace(1) %out, ptr addrspace(1) %
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, 0x20000, v0
 ; CI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; CI-NEXT:    s_endpgm
-  %z0 = insertelement <4 x i16> undef, i16 0, i16 0
+  %z0 = insertelement <4 x i16> poison, i16 0, i16 0
   %z1 = insertelement <4 x i16> %z0, i16 0, i16 1
   %z2 = insertelement <4 x i16> %z1, i16 0, i16 2
   %z3 = insertelement <4 x i16> %z2, i16 0, i16 3
-  %t0 = insertelement <4 x i16> undef, i16 2, i16 0
+  %t0 = insertelement <4 x i16> poison, i16 2, i16 0
   %t1 = insertelement <4 x i16> %t0, i16 2, i16 1
   %t2 = insertelement <4 x i16> %t1, i16 2, i16 2
   %t3 = insertelement <4 x i16> %t2, i16 2, i16 3
diff --git a/llvm/test/CodeGen/AMDGPU/smrd.ll b/llvm/test/CodeGen/AMDGPU/smrd.ll
index 52db7fea08e05..ae398cc800ff0 100644
--- a/llvm/test/CodeGen/AMDGPU/smrd.ll
+++ b/llvm/test/CodeGen/AMDGPU/smrd.ll
@@ -125,7 +125,7 @@ entry:
 ; GCN-NEXT: s_buffer_load_dword s0, s[0:3], 0x0
 define amdgpu_ps float @smrd_hazard(<4 x i32> inreg %desc) #0 {
 main_body:
-  %d0 = insertelement <4 x i32> undef, i32 0, i32 0
+  %d0 = insertelement <4 x i32> poison, i32 0, i32 0
   %d1 = insertelement <4 x i32> %d0, i32 1, i32 1
   %d2 = insertelement <4 x i32> %d1, i32 2, i32 2
   %d3 = insertelement <4 x i32> %d2, i32 3, i32 3
@@ -419,7 +419,7 @@ main_body:
   %v0.y = call nsz float @llvm.amdgcn.interp.p2(float %v0.y1, float %v, i32 0, i32 1, i32 %prim)
   %v0.z1 = call nsz float @llvm.amdgcn.interp.p1(float %u, i32 0, i32 2, i32 %prim)
   %v0.z = call nsz float @llvm.amdgcn.interp.p2(float %v0.z1, float %v, i32 0, i32 2, i32 %prim)
-  %v0.tmp0 = insertelement <3 x float> undef, float %v0.x, i32 0
+  %v0.tmp0 = insertelement <3 x float> poison, float %v0.x, i32 0
   %v0.tmp1 = insertelement <3 x float> %v0.tmp0, float %v0.y, i32 1
   %v0 = insertelement <3 x float> %v0.tmp1, float %v0.z, i32 2
   %a = extractelement <3 x float> %v0, i32 %idx1
@@ -430,7 +430,7 @@ main_body:
   %v1.y = call nsz float @llvm.amdgcn.interp.p2(float %v1.y1, float %v, i32 1, i32 1, i32 %prim)
   %v1.z1 = call nsz float @llvm.amdgcn.interp.p1(float %u, i32 1, i32 2, i32 %prim)
   %v1.z = call nsz float @llvm.amdgcn.interp.p2(float %v1.z1, float %v, i32 1, i32 2, i32 %prim)
-  %v1.tmp0 = insertelement <3 x float> undef, float %v0.x, i32 0
+  %v1.tmp0 = insertelement <3 x float> poison, float %v0.x, i32 0
   %v1.tmp1 = insertelement <3 x float> %v0.tmp0, float %v0.y, i32 1
   %v1 = insertelement <3 x float> %v0.tmp1, float %v0.z, i32 2
 
diff --git a/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir b/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir
index 080bd052a7391..cf23a9d1e8a57 100644
--- a/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir
+++ b/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir
@@ -34,57 +34,56 @@ body:             |
   ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
   ; CHECK-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $sgpr34_sgpr35 = IMPLICIT_DEF
   ; CHECK-NEXT:   dead [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
-  ; CHECK-NEXT:   renamable $sgpr41 = IMPLICIT_DEF
+  ; CHECK-NEXT:   renamable $sgpr35 = IMPLICIT_DEF
   ; CHECK-NEXT:   renamable $sgpr38_sgpr39 = COPY undef $sgpr8_sgpr9
   ; CHECK-NEXT:   renamable $sgpr36_sgpr37 = IMPLICIT_DEF
-  ; CHECK-NEXT:   renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = S_LOAD_DWORDX8_IMM renamable $sgpr38_sgpr39, 0, 0 :: (dereferenceable invariant load (s256), align 16, addrspace 4)
+  ; CHECK-NEXT:   renamable $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 = S_LOAD_DWORDX8_IMM renamable $sgpr38_sgpr39, 0, 0 :: (dereferenceable invariant load (s256), align 16, addrspace 4)
   ; CHECK-NEXT:   dead renamable $sgpr4 = S_LOAD_DWORD_IMM renamable $sgpr38_sgpr39, 48, 0 :: (dereferenceable invariant load (s32), align 16, addrspace 4)
-  ; CHECK-NEXT:   dead renamable $sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM renamable $sgpr44_sgpr45, 0, 0 :: (invariant load (s64), align 16, addrspace 4)
+  ; CHECK-NEXT:   dead renamable $sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM renamable $sgpr48_sgpr49, 0, 0 :: (invariant load (s64), align 16, addrspace 4)
   ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
-  ; CHECK-NEXT:   $vgpr1 = COPY renamable $sgpr51
+  ; CHECK-NEXT:   $vgpr1 = COPY renamable $sgpr55
   ; CHECK-NEXT:   dead $sgpr30_sgpr31 = SI_CALL undef renamable $sgpr4_sgpr5, 0, csr_amdgpu, implicit undef $sgpr15, implicit $vgpr31, implicit $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
-  ; CHECK-NEXT:   $vcc = COPY renamable $sgpr40_sgpr41
+  ; CHECK-NEXT:   $vcc = COPY renamable $sgpr34_sgpr35
   ; CHECK-NEXT:   S_CBRANCH_VCCZ %bb.2, implicit undef $vcc
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.3(0x80000000)
-  ; CHECK-NEXT:   liveins: $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FC00
+  ; CHECK-NEXT:   liveins: $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55:0x000000000000FC00
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11 = S_LOAD_DWORDX8_IMM renamable $sgpr38_sgpr39, 56, 0 :: (dereferenceable invariant load (s256), align 8, addrspace 4)
   ; CHECK-NEXT:   S_BRANCH %bb.3
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   successors: %bb.3(0x80000000)
-  ; CHECK-NEXT:   liveins: $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FC00
+  ; CHECK-NEXT:   liveins: $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55:0x000000000000FC00
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11 = S_LOAD_DWORDX8_IMM renamable $sgpr38_sgpr39, 56, 0 :: (dereferenceable invariant load (s256), align 8, addrspace 4)
   ; CHECK-NEXT:   S_CMP_LG_U64 renamable $sgpr4_sgpr5, 0, implicit-def $scc
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.5(0x40000000), %bb.4(0x40000000)
-  ; CHECK-NEXT:   liveins: $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11:0x00000000000003F0, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FC00
+  ; CHECK-NEXT:   liveins: $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11:0x00000000000003F0, $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55:0x000000000000FC00
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   S_CBRANCH_VCCZ %bb.5, implicit undef $vcc
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.4:
   ; CHECK-NEXT:   successors: %bb.5(0x80000000)
-  ; CHECK-NEXT:   liveins: $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11:0x00000000000003F0, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FC00
+  ; CHECK-NEXT:   liveins: $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11:0x00000000000003F0, $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55:0x000000000000FC00
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   S_CMP_EQ_U32 renamable $sgpr8, 0, implicit-def $scc
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.5:
-  ; CHECK-NEXT:   liveins: $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11:0x00000000000000F0, $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51:0x000000000000FC00
+  ; CHECK-NEXT:   liveins: $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11:0x00000000000000F0, $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55:0x000000000000FC00
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   dead renamable $sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM killed renamable $sgpr38_sgpr39, 40, 0 :: (dereferenceable invariant load (s64), addrspace 4)
   ; CHECK-NEXT:   GLOBAL_STORE_DWORD_SADDR undef [[DEF]], undef [[DEF]], killed renamable $sgpr6_sgpr7, 0, 0, implicit $exec :: (store (s32), addrspace 1)
-  ; CHECK-NEXT:   GLOBAL_STORE_DWORD_SADDR undef [[DEF]], undef [[DEF]], renamable $sgpr50_sgpr51, 0, 0, implicit $exec :: (store (s32), addrspace 1)
-  ; CHECK-NEXT:   dead [[COPY:%[0-9]+]]:vgpr_32 = COPY killed renamable $sgpr49
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORD_SADDR undef [[DEF]], undef [[DEF]], renamable $sgpr54_sgpr55, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+  ; CHECK-NEXT:   dead [[COPY:%[0-9]+]]:vgpr_32 = COPY killed renamable $sgpr53
   ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
   ; CHECK-NEXT:   $sgpr6_sgpr7 = COPY killed renamable $sgpr36_sgpr37
-  ; CHECK-NEXT:   $sgpr10_sgpr11 = COPY killed renamable $sgpr34_sgpr35
+  ; CHECK-NEXT:   renamable $sgpr10_sgpr11 = IMPLICIT_DEF
   ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
   ; CHECK-NEXT:   S_ENDPGM 0
   bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/spill-sgpr-to-virtual-vgpr.mir b/llvm/test/CodeGen/AMDGPU/spill-sgpr-to-virtual-vgpr.mir
index dff2bd7f7aef9..7f4f9489ea4b7 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-sgpr-to-virtual-vgpr.mir
+++ b/llvm/test/CodeGen/AMDGPU/spill-sgpr-to-virtual-vgpr.mir
@@ -53,7 +53,7 @@ body:             |
   bb.0:
     liveins: $sgpr30_sgpr31, $sgpr10, $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71, $sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79, $sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87, $sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95
     ; GCN-LABEL: name: sgpr_spill_lane_crossover
-    ; GCN: liveins: $sgpr10, $sgpr64, $sgpr65, $sgpr66, $sgpr67, $sgpr68, $sgpr69, $sgpr70, $sgpr71, $sgpr72, $sgpr73, $sgpr74, $sgpr75, $sgpr76, $sgpr77, $sgpr78, $sgpr79, $sgpr80, $sgpr81, $sgpr82, $sgpr83, $sgpr84, $sgpr85, $sgpr86, $sgpr87, $sgpr88, $sgpr89, $sgpr90, $sgpr91, $sgpr92, $sgpr93, $sgpr94, $sgpr95, $vgpr63, $sgpr30_sgpr31, $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71, $sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79, $sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87, $sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95
+    ; GCN: liveins: $sgpr10, $sgpr64, $sgpr65, $sgpr66, $sgpr67, $sgpr68, $sgpr69, $sgpr70, $sgpr71, $sgpr80, $sgpr81, $sgpr82, $sgpr83, $sgpr84, $sgpr85, $sgpr86, $sgpr87, $vgpr63, $sgpr30_sgpr31, $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71, $sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79, $sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87, $sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95
     ; GCN-NEXT: {{  $}}
     ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr64, 0, $vgpr63
     ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr65, 1, $vgpr63
@@ -63,30 +63,14 @@ body:             |
     ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr69, 5, $vgpr63
     ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr70, 6, $vgpr63
     ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr71, 7, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr72, 8, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr73, 9, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr74, 10, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr75, 11, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr76, 12, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr77, 13, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr78, 14, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr79, 15, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr80, 16, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr81, 17, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr82, 18, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr83, 19, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr84, 20, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr85, 21, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr86, 22, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr87, 23, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr88, 24, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr89, 25, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr90, 26, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr91, 27, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr92, 28, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr93, 29, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr94, 30, $vgpr63
-    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr95, 31, $vgpr63
+    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr80, 8, $vgpr63
+    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr81, 9, $vgpr63
+    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr82, 10, $vgpr63
+    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr83, 11, $vgpr63
+    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr84, 12, $vgpr63
+    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr85, 13, $vgpr63
+    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr86, 14, $vgpr63
+    ; GCN-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr87, 15, $vgpr63
     ; GCN-NEXT: S_NOP 0
     ; GCN-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; GCN-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = SI_SPILL_S32_TO_VGPR killed $sgpr10, 0, [[DEF]]
diff --git a/llvm/test/CodeGen/AMDGPU/spill-sgpr-used-for-exec-copy.mir b/llvm/test/CodeGen/AMDGPU/spill-sgpr-used-for-exec-copy.mir
index 9b0f52cb39b01..6e8a5126ca823 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-sgpr-used-for-exec-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/spill-sgpr-used-for-exec-copy.mir
@@ -56,20 +56,15 @@ body:             |
   bb.0:
     liveins: $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr18, $sgpr19, $sgpr20, $sgpr21, $sgpr22, $sgpr23, $sgpr24, $sgpr25, $sgpr26, $sgpr27, $sgpr30_sgpr31, $vcc, $vgpr0
     ; GCN-LABEL: name: spill_exec_copy_reserved_reg
-    ; GCN: liveins: $vcc, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr18, $sgpr19, $sgpr20, $sgpr21, $sgpr22, $sgpr23, $sgpr24, $sgpr25, $sgpr26, $sgpr27, $vgpr0, $vgpr2, $sgpr30_sgpr31
+    ; GCN: liveins: $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr18, $sgpr19, $sgpr20, $sgpr21, $sgpr22, $sgpr23, $sgpr24, $sgpr25, $sgpr26, $sgpr27, $sgpr30_sgpr31, $vcc, $vgpr0
     ; GCN-NEXT: {{  $}}
-    ; GCN-NEXT: $sgpr28_sgpr29 = S_XOR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-    ; GCN-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, implicit $exec :: (store (s32) into %stack.1, addrspace 5)
-    ; GCN-NEXT: $exec = S_MOV_B64 killed $sgpr28_sgpr29
-    ; GCN-NEXT: $vgpr2 = SI_SPILL_S32_TO_VGPR $sgpr34, 0, undef $vgpr2
-    ; GCN-NEXT: $vgpr2 = SI_SPILL_S32_TO_VGPR $sgpr35, 1, undef $vgpr2
     ; GCN-NEXT: $vgpr0 = SI_SPILL_S32_TO_VGPR $sgpr30, 0, killed $vgpr0
     ; GCN-NEXT: $vgpr0 = SI_SPILL_S32_TO_VGPR $sgpr31, 1, killed $vgpr0
-    ; GCN-NEXT: $sgpr34_sgpr35 = S_OR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
+    ; GCN-NEXT: $sgpr40_sgpr41 = S_OR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
     ; GCN-NEXT: $sgpr28_sgpr29 = IMPLICIT_DEF
     ; GCN-NEXT: $vgpr1 = COPY $vgpr0
     ; GCN-NEXT: S_NOP 0, implicit $sgpr28_sgpr29
-    ; GCN-NEXT: $exec = S_MOV_B64 killed $sgpr34_sgpr35
+    ; GCN-NEXT: $exec = S_MOV_B64 killed $sgpr40_sgpr41
     ; GCN-NEXT: $sgpr30 = SI_RESTORE_S32_FROM_VGPR $vgpr1, 0
     ; GCN-NEXT: $sgpr31 = SI_RESTORE_S32_FROM_VGPR $vgpr1, 1
     ; GCN-NEXT: S_SETPC_B64 $sgpr30_sgpr31, implicit $sgpr4, implicit $sgpr5, implicit $sgpr6, implicit $sgpr7, implicit $sgpr8_sgpr9_sgpr10, implicit $sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr15, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $vcc
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
index 03e8e28ef54db..aaf2e1dd604fa 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
@@ -18,7 +18,7 @@ define amdgpu_kernel void @test_spill_av_class(<4 x i32> %arg) #0 {
   ; GCN-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 3538953 /* reguse:VReg_64 */, %13
   ; GCN-NEXT:   S_ENDPGM 0
   %v0 = call i32 asm sideeffect "; def $0", "=v"()
-  %tmp = insertelement <2 x i32> undef, i32 %v0, i32 0
+  %tmp = insertelement <2 x i32> poison, i32 %v0, i32 0
   %mai = tail call <4 x i32> @llvm.amdgcn.mfma.i32.4x4x4i8(i32 1, i32 2, <4 x i32> %arg, i32 0, i32 0, i32 0)
   store volatile <4 x i32> %mai, ptr addrspace(1) undef
   call void asm sideeffect "; use $0", "v"(<2 x i32> %tmp);
diff --git a/llvm/test/CodeGen/AMDGPU/spill_more_than_wavesize_csr_sgprs.ll b/llvm/test/CodeGen/AMDGPU/spill_more_than_wavesize_csr_sgprs.ll
index 0d6bccad89d82..fba85455ef693 100644
--- a/llvm/test/CodeGen/AMDGPU/spill_more_than_wavesize_csr_sgprs.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill_more_than_wavesize_csr_sgprs.ll
@@ -7,141 +7,76 @@ define void @spill_more_than_wavesize_csr_sgprs() {
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_xor_saveexec_b64 s[4:5], -1
 ; CHECK-NEXT:    buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill
-; CHECK-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
 ; CHECK-NEXT:    s_mov_b64 exec, s[4:5]
 ; CHECK-NEXT:    v_writelane_b32 v0, s35, 0
 ; CHECK-NEXT:    v_writelane_b32 v0, s36, 1
 ; CHECK-NEXT:    v_writelane_b32 v0, s37, 2
 ; CHECK-NEXT:    v_writelane_b32 v0, s38, 3
 ; CHECK-NEXT:    v_writelane_b32 v0, s39, 4
-; CHECK-NEXT:    v_writelane_b32 v0, s40, 5
-; CHECK-NEXT:    v_writelane_b32 v0, s41, 6
-; CHECK-NEXT:    v_writelane_b32 v0, s42, 7
-; CHECK-NEXT:    v_writelane_b32 v0, s43, 8
-; CHECK-NEXT:    v_writelane_b32 v0, s44, 9
-; CHECK-NEXT:    v_writelane_b32 v0, s45, 10
-; CHECK-NEXT:    v_writelane_b32 v0, s46, 11
-; CHECK-NEXT:    v_writelane_b32 v0, s47, 12
-; CHECK-NEXT:    v_writelane_b32 v0, s48, 13
-; CHECK-NEXT:    v_writelane_b32 v0, s49, 14
-; CHECK-NEXT:    v_writelane_b32 v0, s50, 15
-; CHECK-NEXT:    v_writelane_b32 v0, s51, 16
-; CHECK-NEXT:    v_writelane_b32 v0, s52, 17
-; CHECK-NEXT:    v_writelane_b32 v0, s53, 18
-; CHECK-NEXT:    v_writelane_b32 v0, s54, 19
-; CHECK-NEXT:    v_writelane_b32 v0, s55, 20
-; CHECK-NEXT:    v_writelane_b32 v0, s56, 21
-; CHECK-NEXT:    v_writelane_b32 v0, s57, 22
-; CHECK-NEXT:    v_writelane_b32 v0, s58, 23
-; CHECK-NEXT:    v_writelane_b32 v0, s59, 24
-; CHECK-NEXT:    v_writelane_b32 v0, s60, 25
-; CHECK-NEXT:    v_writelane_b32 v0, s61, 26
-; CHECK-NEXT:    v_writelane_b32 v0, s62, 27
-; CHECK-NEXT:    v_writelane_b32 v0, s63, 28
-; CHECK-NEXT:    v_writelane_b32 v0, s64, 29
-; CHECK-NEXT:    v_writelane_b32 v0, s65, 30
-; CHECK-NEXT:    v_writelane_b32 v0, s66, 31
-; CHECK-NEXT:    v_writelane_b32 v0, s67, 32
-; CHECK-NEXT:    v_writelane_b32 v0, s68, 33
-; CHECK-NEXT:    v_writelane_b32 v0, s69, 34
-; CHECK-NEXT:    v_writelane_b32 v0, s70, 35
-; CHECK-NEXT:    v_writelane_b32 v0, s71, 36
-; CHECK-NEXT:    v_writelane_b32 v0, s72, 37
-; CHECK-NEXT:    v_writelane_b32 v0, s73, 38
-; CHECK-NEXT:    v_writelane_b32 v0, s74, 39
-; CHECK-NEXT:    v_writelane_b32 v0, s75, 40
-; CHECK-NEXT:    v_writelane_b32 v0, s76, 41
-; CHECK-NEXT:    v_writelane_b32 v0, s77, 42
-; CHECK-NEXT:    v_writelane_b32 v0, s78, 43
-; CHECK-NEXT:    v_writelane_b32 v0, s79, 44
-; CHECK-NEXT:    v_writelane_b32 v0, s80, 45
-; CHECK-NEXT:    v_writelane_b32 v0, s81, 46
-; CHECK-NEXT:    v_writelane_b32 v0, s82, 47
-; CHECK-NEXT:    v_writelane_b32 v0, s83, 48
-; CHECK-NEXT:    v_writelane_b32 v0, s84, 49
-; CHECK-NEXT:    v_writelane_b32 v0, s85, 50
-; CHECK-NEXT:    v_writelane_b32 v0, s86, 51
-; CHECK-NEXT:    v_writelane_b32 v0, s87, 52
-; CHECK-NEXT:    v_writelane_b32 v0, s88, 53
-; CHECK-NEXT:    v_writelane_b32 v0, s89, 54
-; CHECK-NEXT:    v_writelane_b32 v0, s90, 55
-; CHECK-NEXT:    v_writelane_b32 v0, s91, 56
-; CHECK-NEXT:    v_writelane_b32 v0, s92, 57
-; CHECK-NEXT:    v_writelane_b32 v0, s93, 58
-; CHECK-NEXT:    v_writelane_b32 v0, s94, 59
-; CHECK-NEXT:    v_writelane_b32 v0, s95, 60
-; CHECK-NEXT:    v_writelane_b32 v1, s99, 0
-; CHECK-NEXT:    v_writelane_b32 v0, s96, 61
-; CHECK-NEXT:    v_writelane_b32 v1, s100, 1
-; CHECK-NEXT:    v_writelane_b32 v0, s97, 62
-; CHECK-NEXT:    v_writelane_b32 v1, s101, 2
-; CHECK-NEXT:    v_writelane_b32 v0, s98, 63
-; CHECK-NEXT:    v_writelane_b32 v1, s102, 3
+; CHECK-NEXT:    v_writelane_b32 v0, s48, 5
+; CHECK-NEXT:    v_writelane_b32 v0, s49, 6
+; CHECK-NEXT:    v_writelane_b32 v0, s50, 7
+; CHECK-NEXT:    v_writelane_b32 v0, s51, 8
+; CHECK-NEXT:    v_writelane_b32 v0, s52, 9
+; CHECK-NEXT:    v_writelane_b32 v0, s53, 10
+; CHECK-NEXT:    v_writelane_b32 v0, s54, 11
+; CHECK-NEXT:    v_writelane_b32 v0, s55, 12
+; CHECK-NEXT:    v_writelane_b32 v0, s64, 13
+; CHECK-NEXT:    v_writelane_b32 v0, s65, 14
+; CHECK-NEXT:    v_writelane_b32 v0, s66, 15
+; CHECK-NEXT:    v_writelane_b32 v0, s67, 16
+; CHECK-NEXT:    v_writelane_b32 v0, s68, 17
+; CHECK-NEXT:    v_writelane_b32 v0, s69, 18
+; CHECK-NEXT:    v_writelane_b32 v0, s70, 19
+; CHECK-NEXT:    v_writelane_b32 v0, s71, 20
+; CHECK-NEXT:    v_writelane_b32 v0, s80, 21
+; CHECK-NEXT:    v_writelane_b32 v0, s81, 22
+; CHECK-NEXT:    v_writelane_b32 v0, s82, 23
+; CHECK-NEXT:    v_writelane_b32 v0, s83, 24
+; CHECK-NEXT:    v_writelane_b32 v0, s84, 25
+; CHECK-NEXT:    v_writelane_b32 v0, s85, 26
+; CHECK-NEXT:    v_writelane_b32 v0, s86, 27
+; CHECK-NEXT:    v_writelane_b32 v0, s87, 28
+; CHECK-NEXT:    v_writelane_b32 v0, s96, 29
+; CHECK-NEXT:    v_writelane_b32 v0, s97, 30
+; CHECK-NEXT:    v_writelane_b32 v0, s98, 31
+; CHECK-NEXT:    v_writelane_b32 v0, s99, 32
+; CHECK-NEXT:    v_writelane_b32 v0, s100, 33
+; CHECK-NEXT:    v_writelane_b32 v0, s101, 34
+; CHECK-NEXT:    v_writelane_b32 v0, s102, 35
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_readlane_b32 s102, v1, 3
-; CHECK-NEXT:    v_readlane_b32 s101, v1, 2
-; CHECK-NEXT:    v_readlane_b32 s100, v1, 1
-; CHECK-NEXT:    v_readlane_b32 s99, v1, 0
-; CHECK-NEXT:    v_readlane_b32 s98, v0, 63
-; CHECK-NEXT:    v_readlane_b32 s97, v0, 62
-; CHECK-NEXT:    v_readlane_b32 s96, v0, 61
-; CHECK-NEXT:    v_readlane_b32 s95, v0, 60
-; CHECK-NEXT:    v_readlane_b32 s94, v0, 59
-; CHECK-NEXT:    v_readlane_b32 s93, v0, 58
-; CHECK-NEXT:    v_readlane_b32 s92, v0, 57
-; CHECK-NEXT:    v_readlane_b32 s91, v0, 56
-; CHECK-NEXT:    v_readlane_b32 s90, v0, 55
-; CHECK-NEXT:    v_readlane_b32 s89, v0, 54
-; CHECK-NEXT:    v_readlane_b32 s88, v0, 53
-; CHECK-NEXT:    v_readlane_b32 s87, v0, 52
-; CHECK-NEXT:    v_readlane_b32 s86, v0, 51
-; CHECK-NEXT:    v_readlane_b32 s85, v0, 50
-; CHECK-NEXT:    v_readlane_b32 s84, v0, 49
-; CHECK-NEXT:    v_readlane_b32 s83, v0, 48
-; CHECK-NEXT:    v_readlane_b32 s82, v0, 47
-; CHECK-NEXT:    v_readlane_b32 s81, v0, 46
-; CHECK-NEXT:    v_readlane_b32 s80, v0, 45
-; CHECK-NEXT:    v_readlane_b32 s79, v0, 44
-; CHECK-NEXT:    v_readlane_b32 s78, v0, 43
-; CHECK-NEXT:    v_readlane_b32 s77, v0, 42
-; CHECK-NEXT:    v_readlane_b32 s76, v0, 41
-; CHECK-NEXT:    v_readlane_b32 s75, v0, 40
-; CHECK-NEXT:    v_readlane_b32 s74, v0, 39
-; CHECK-NEXT:    v_readlane_b32 s73, v0, 38
-; CHECK-NEXT:    v_readlane_b32 s72, v0, 37
-; CHECK-NEXT:    v_readlane_b32 s71, v0, 36
-; CHECK-NEXT:    v_readlane_b32 s70, v0, 35
-; CHECK-NEXT:    v_readlane_b32 s69, v0, 34
-; CHECK-NEXT:    v_readlane_b32 s68, v0, 33
-; CHECK-NEXT:    v_readlane_b32 s67, v0, 32
-; CHECK-NEXT:    v_readlane_b32 s66, v0, 31
-; CHECK-NEXT:    v_readlane_b32 s65, v0, 30
-; CHECK-NEXT:    v_readlane_b32 s64, v0, 29
-; CHECK-NEXT:    v_readlane_b32 s63, v0, 28
-; CHECK-NEXT:    v_readlane_b32 s62, v0, 27
-; CHECK-NEXT:    v_readlane_b32 s61, v0, 26
-; CHECK-NEXT:    v_readlane_b32 s60, v0, 25
-; CHECK-NEXT:    v_readlane_b32 s59, v0, 24
-; CHECK-NEXT:    v_readlane_b32 s58, v0, 23
-; CHECK-NEXT:    v_readlane_b32 s57, v0, 22
-; CHECK-NEXT:    v_readlane_b32 s56, v0, 21
-; CHECK-NEXT:    v_readlane_b32 s55, v0, 20
-; CHECK-NEXT:    v_readlane_b32 s54, v0, 19
-; CHECK-NEXT:    v_readlane_b32 s53, v0, 18
-; CHECK-NEXT:    v_readlane_b32 s52, v0, 17
-; CHECK-NEXT:    v_readlane_b32 s51, v0, 16
-; CHECK-NEXT:    v_readlane_b32 s50, v0, 15
-; CHECK-NEXT:    v_readlane_b32 s49, v0, 14
-; CHECK-NEXT:    v_readlane_b32 s48, v0, 13
-; CHECK-NEXT:    v_readlane_b32 s47, v0, 12
-; CHECK-NEXT:    v_readlane_b32 s46, v0, 11
-; CHECK-NEXT:    v_readlane_b32 s45, v0, 10
-; CHECK-NEXT:    v_readlane_b32 s44, v0, 9
-; CHECK-NEXT:    v_readlane_b32 s43, v0, 8
-; CHECK-NEXT:    v_readlane_b32 s42, v0, 7
-; CHECK-NEXT:    v_readlane_b32 s41, v0, 6
-; CHECK-NEXT:    v_readlane_b32 s40, v0, 5
+; CHECK-NEXT:    v_readlane_b32 s102, v0, 35
+; CHECK-NEXT:    v_readlane_b32 s101, v0, 34
+; CHECK-NEXT:    v_readlane_b32 s100, v0, 33
+; CHECK-NEXT:    v_readlane_b32 s99, v0, 32
+; CHECK-NEXT:    v_readlane_b32 s98, v0, 31
+; CHECK-NEXT:    v_readlane_b32 s97, v0, 30
+; CHECK-NEXT:    v_readlane_b32 s96, v0, 29
+; CHECK-NEXT:    v_readlane_b32 s87, v0, 28
+; CHECK-NEXT:    v_readlane_b32 s86, v0, 27
+; CHECK-NEXT:    v_readlane_b32 s85, v0, 26
+; CHECK-NEXT:    v_readlane_b32 s84, v0, 25
+; CHECK-NEXT:    v_readlane_b32 s83, v0, 24
+; CHECK-NEXT:    v_readlane_b32 s82, v0, 23
+; CHECK-NEXT:    v_readlane_b32 s81, v0, 22
+; CHECK-NEXT:    v_readlane_b32 s80, v0, 21
+; CHECK-NEXT:    v_readlane_b32 s71, v0, 20
+; CHECK-NEXT:    v_readlane_b32 s70, v0, 19
+; CHECK-NEXT:    v_readlane_b32 s69, v0, 18
+; CHECK-NEXT:    v_readlane_b32 s68, v0, 17
+; CHECK-NEXT:    v_readlane_b32 s67, v0, 16
+; CHECK-NEXT:    v_readlane_b32 s66, v0, 15
+; CHECK-NEXT:    v_readlane_b32 s65, v0, 14
+; CHECK-NEXT:    v_readlane_b32 s64, v0, 13
+; CHECK-NEXT:    v_readlane_b32 s55, v0, 12
+; CHECK-NEXT:    v_readlane_b32 s54, v0, 11
+; CHECK-NEXT:    v_readlane_b32 s53, v0, 10
+; CHECK-NEXT:    v_readlane_b32 s52, v0, 9
+; CHECK-NEXT:    v_readlane_b32 s51, v0, 8
+; CHECK-NEXT:    v_readlane_b32 s50, v0, 7
+; CHECK-NEXT:    v_readlane_b32 s49, v0, 6
+; CHECK-NEXT:    v_readlane_b32 s48, v0, 5
 ; CHECK-NEXT:    v_readlane_b32 s39, v0, 4
 ; CHECK-NEXT:    v_readlane_b32 s38, v0, 3
 ; CHECK-NEXT:    v_readlane_b32 s37, v0, 2
@@ -149,7 +84,6 @@ define void @spill_more_than_wavesize_csr_sgprs() {
 ; CHECK-NEXT:    v_readlane_b32 s35, v0, 0
 ; CHECK-NEXT:    s_xor_saveexec_b64 s[4:5], -1
 ; CHECK-NEXT:    buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
-; CHECK-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
 ; CHECK-NEXT:    s_mov_b64 exec, s[4:5]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
@@ -172,144 +106,79 @@ define void @spill_more_than_wavesize_csr_sgprs_with_stack_object() {
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    s_xor_saveexec_b64 s[4:5], -1
 ; CHECK-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
-; CHECK-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
 ; CHECK-NEXT:    s_mov_b64 exec, s[4:5]
 ; CHECK-NEXT:    v_writelane_b32 v1, s35, 0
 ; CHECK-NEXT:    v_writelane_b32 v1, s36, 1
 ; CHECK-NEXT:    v_writelane_b32 v1, s37, 2
 ; CHECK-NEXT:    v_writelane_b32 v1, s38, 3
 ; CHECK-NEXT:    v_writelane_b32 v1, s39, 4
-; CHECK-NEXT:    v_writelane_b32 v1, s40, 5
-; CHECK-NEXT:    v_writelane_b32 v1, s41, 6
-; CHECK-NEXT:    v_writelane_b32 v1, s42, 7
-; CHECK-NEXT:    v_writelane_b32 v1, s43, 8
-; CHECK-NEXT:    v_writelane_b32 v1, s44, 9
-; CHECK-NEXT:    v_writelane_b32 v1, s45, 10
-; CHECK-NEXT:    v_writelane_b32 v1, s46, 11
-; CHECK-NEXT:    v_writelane_b32 v1, s47, 12
-; CHECK-NEXT:    v_writelane_b32 v1, s48, 13
-; CHECK-NEXT:    v_writelane_b32 v1, s49, 14
-; CHECK-NEXT:    v_writelane_b32 v1, s50, 15
-; CHECK-NEXT:    v_writelane_b32 v1, s51, 16
-; CHECK-NEXT:    v_writelane_b32 v1, s52, 17
-; CHECK-NEXT:    v_writelane_b32 v1, s53, 18
-; CHECK-NEXT:    v_writelane_b32 v1, s54, 19
-; CHECK-NEXT:    v_writelane_b32 v1, s55, 20
-; CHECK-NEXT:    v_writelane_b32 v1, s56, 21
-; CHECK-NEXT:    v_writelane_b32 v1, s57, 22
-; CHECK-NEXT:    v_writelane_b32 v1, s58, 23
-; CHECK-NEXT:    v_writelane_b32 v1, s59, 24
-; CHECK-NEXT:    v_writelane_b32 v1, s60, 25
-; CHECK-NEXT:    v_writelane_b32 v1, s61, 26
-; CHECK-NEXT:    v_writelane_b32 v1, s62, 27
-; CHECK-NEXT:    v_writelane_b32 v1, s63, 28
-; CHECK-NEXT:    v_writelane_b32 v1, s64, 29
-; CHECK-NEXT:    v_writelane_b32 v1, s65, 30
-; CHECK-NEXT:    v_writelane_b32 v1, s66, 31
-; CHECK-NEXT:    v_writelane_b32 v1, s67, 32
-; CHECK-NEXT:    v_writelane_b32 v1, s68, 33
-; CHECK-NEXT:    v_writelane_b32 v1, s69, 34
-; CHECK-NEXT:    v_writelane_b32 v1, s70, 35
-; CHECK-NEXT:    v_writelane_b32 v1, s71, 36
-; CHECK-NEXT:    v_writelane_b32 v1, s72, 37
-; CHECK-NEXT:    v_writelane_b32 v1, s73, 38
-; CHECK-NEXT:    v_writelane_b32 v1, s74, 39
-; CHECK-NEXT:    v_writelane_b32 v1, s75, 40
-; CHECK-NEXT:    v_writelane_b32 v1, s76, 41
-; CHECK-NEXT:    v_writelane_b32 v1, s77, 42
-; CHECK-NEXT:    v_writelane_b32 v1, s78, 43
-; CHECK-NEXT:    v_writelane_b32 v1, s79, 44
-; CHECK-NEXT:    v_writelane_b32 v1, s80, 45
-; CHECK-NEXT:    v_writelane_b32 v1, s81, 46
-; CHECK-NEXT:    v_writelane_b32 v1, s82, 47
-; CHECK-NEXT:    v_writelane_b32 v1, s83, 48
-; CHECK-NEXT:    v_writelane_b32 v1, s84, 49
-; CHECK-NEXT:    v_writelane_b32 v1, s85, 50
-; CHECK-NEXT:    v_writelane_b32 v1, s86, 51
-; CHECK-NEXT:    v_writelane_b32 v1, s87, 52
-; CHECK-NEXT:    v_writelane_b32 v1, s88, 53
-; CHECK-NEXT:    v_writelane_b32 v1, s89, 54
-; CHECK-NEXT:    v_writelane_b32 v1, s90, 55
-; CHECK-NEXT:    v_writelane_b32 v1, s91, 56
-; CHECK-NEXT:    v_writelane_b32 v1, s92, 57
-; CHECK-NEXT:    v_writelane_b32 v1, s93, 58
-; CHECK-NEXT:    v_writelane_b32 v1, s94, 59
-; CHECK-NEXT:    v_writelane_b32 v1, s95, 60
-; CHECK-NEXT:    v_writelane_b32 v2, s99, 0
-; CHECK-NEXT:    v_writelane_b32 v1, s96, 61
-; CHECK-NEXT:    v_writelane_b32 v2, s100, 1
-; CHECK-NEXT:    v_writelane_b32 v1, s97, 62
-; CHECK-NEXT:    v_writelane_b32 v2, s101, 2
+; CHECK-NEXT:    v_writelane_b32 v1, s48, 5
+; CHECK-NEXT:    v_writelane_b32 v1, s49, 6
+; CHECK-NEXT:    v_writelane_b32 v1, s50, 7
+; CHECK-NEXT:    v_writelane_b32 v1, s51, 8
+; CHECK-NEXT:    v_writelane_b32 v1, s52, 9
+; CHECK-NEXT:    v_writelane_b32 v1, s53, 10
+; CHECK-NEXT:    v_writelane_b32 v1, s54, 11
+; CHECK-NEXT:    v_writelane_b32 v1, s55, 12
+; CHECK-NEXT:    v_writelane_b32 v1, s64, 13
+; CHECK-NEXT:    v_writelane_b32 v1, s65, 14
+; CHECK-NEXT:    v_writelane_b32 v1, s66, 15
+; CHECK-NEXT:    v_writelane_b32 v1, s67, 16
+; CHECK-NEXT:    v_writelane_b32 v1, s68, 17
+; CHECK-NEXT:    v_writelane_b32 v1, s69, 18
+; CHECK-NEXT:    v_writelane_b32 v1, s70, 19
+; CHECK-NEXT:    v_writelane_b32 v1, s71, 20
+; CHECK-NEXT:    v_writelane_b32 v1, s80, 21
+; CHECK-NEXT:    v_writelane_b32 v1, s81, 22
+; CHECK-NEXT:    v_writelane_b32 v1, s82, 23
+; CHECK-NEXT:    v_writelane_b32 v1, s83, 24
+; CHECK-NEXT:    v_writelane_b32 v1, s84, 25
+; CHECK-NEXT:    v_writelane_b32 v1, s85, 26
+; CHECK-NEXT:    v_writelane_b32 v1, s86, 27
+; CHECK-NEXT:    v_writelane_b32 v1, s87, 28
+; CHECK-NEXT:    v_writelane_b32 v1, s96, 29
+; CHECK-NEXT:    v_writelane_b32 v1, s97, 30
+; CHECK-NEXT:    v_writelane_b32 v1, s98, 31
+; CHECK-NEXT:    v_writelane_b32 v1, s99, 32
+; CHECK-NEXT:    v_writelane_b32 v1, s100, 33
+; CHECK-NEXT:    v_writelane_b32 v1, s101, 34
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
-; CHECK-NEXT:    v_writelane_b32 v1, s98, 63
-; CHECK-NEXT:    v_writelane_b32 v2, s102, 3
+; CHECK-NEXT:    v_writelane_b32 v1, s102, 35
 ; CHECK-NEXT:    buffer_store_dword v0, off, s[0:3], s32
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    ;;#ASMSTART
 ; CHECK-NEXT:    ;;#ASMEND
-; CHECK-NEXT:    v_readlane_b32 s102, v2, 3
-; CHECK-NEXT:    v_readlane_b32 s101, v2, 2
-; CHECK-NEXT:    v_readlane_b32 s100, v2, 1
-; CHECK-NEXT:    v_readlane_b32 s99, v2, 0
-; CHECK-NEXT:    v_readlane_b32 s98, v1, 63
-; CHECK-NEXT:    v_readlane_b32 s97, v1, 62
-; CHECK-NEXT:    v_readlane_b32 s96, v1, 61
-; CHECK-NEXT:    v_readlane_b32 s95, v1, 60
-; CHECK-NEXT:    v_readlane_b32 s94, v1, 59
-; CHECK-NEXT:    v_readlane_b32 s93, v1, 58
-; CHECK-NEXT:    v_readlane_b32 s92, v1, 57
-; CHECK-NEXT:    v_readlane_b32 s91, v1, 56
-; CHECK-NEXT:    v_readlane_b32 s90, v1, 55
-; CHECK-NEXT:    v_readlane_b32 s89, v1, 54
-; CHECK-NEXT:    v_readlane_b32 s88, v1, 53
-; CHECK-NEXT:    v_readlane_b32 s87, v1, 52
-; CHECK-NEXT:    v_readlane_b32 s86, v1, 51
-; CHECK-NEXT:    v_readlane_b32 s85, v1, 50
-; CHECK-NEXT:    v_readlane_b32 s84, v1, 49
-; CHECK-NEXT:    v_readlane_b32 s83, v1, 48
-; CHECK-NEXT:    v_readlane_b32 s82, v1, 47
-; CHECK-NEXT:    v_readlane_b32 s81, v1, 46
-; CHECK-NEXT:    v_readlane_b32 s80, v1, 45
-; CHECK-NEXT:    v_readlane_b32 s79, v1, 44
-; CHECK-NEXT:    v_readlane_b32 s78, v1, 43
-; CHECK-NEXT:    v_readlane_b32 s77, v1, 42
-; CHECK-NEXT:    v_readlane_b32 s76, v1, 41
-; CHECK-NEXT:    v_readlane_b32 s75, v1, 40
-; CHECK-NEXT:    v_readlane_b32 s74, v1, 39
-; CHECK-NEXT:    v_readlane_b32 s73, v1, 38
-; CHECK-NEXT:    v_readlane_b32 s72, v1, 37
-; CHECK-NEXT:    v_readlane_b32 s71, v1, 36
-; CHECK-NEXT:    v_readlane_b32 s70, v1, 35
-; CHECK-NEXT:    v_readlane_b32 s69, v1, 34
-; CHECK-NEXT:    v_readlane_b32 s68, v1, 33
-; CHECK-NEXT:    v_readlane_b32 s67, v1, 32
-; CHECK-NEXT:    v_readlane_b32 s66, v1, 31
-; CHECK-NEXT:    v_readlane_b32 s65, v1, 30
-; CHECK-NEXT:    v_readlane_b32 s64, v1, 29
-; CHECK-NEXT:    v_readlane_b32 s63, v1, 28
-; CHECK-NEXT:    v_readlane_b32 s62, v1, 27
-; CHECK-NEXT:    v_readlane_b32 s61, v1, 26
-; CHECK-NEXT:    v_readlane_b32 s60, v1, 25
-; CHECK-NEXT:    v_readlane_b32 s59, v1, 24
-; CHECK-NEXT:    v_readlane_b32 s58, v1, 23
-; CHECK-NEXT:    v_readlane_b32 s57, v1, 22
-; CHECK-NEXT:    v_readlane_b32 s56, v1, 21
-; CHECK-NEXT:    v_readlane_b32 s55, v1, 20
-; CHECK-NEXT:    v_readlane_b32 s54, v1, 19
-; CHECK-NEXT:    v_readlane_b32 s53, v1, 18
-; CHECK-NEXT:    v_readlane_b32 s52, v1, 17
-; CHECK-NEXT:    v_readlane_b32 s51, v1, 16
-; CHECK-NEXT:    v_readlane_b32 s50, v1, 15
-; CHECK-NEXT:    v_readlane_b32 s49, v1, 14
-; CHECK-NEXT:    v_readlane_b32 s48, v1, 13
-; CHECK-NEXT:    v_readlane_b32 s47, v1, 12
-; CHECK-NEXT:    v_readlane_b32 s46, v1, 11
-; CHECK-NEXT:    v_readlane_b32 s45, v1, 10
-; CHECK-NEXT:    v_readlane_b32 s44, v1, 9
-; CHECK-NEXT:    v_readlane_b32 s43, v1, 8
-; CHECK-NEXT:    v_readlane_b32 s42, v1, 7
-; CHECK-NEXT:    v_readlane_b32 s41, v1, 6
-; CHECK-NEXT:    v_readlane_b32 s40, v1, 5
+; CHECK-NEXT:    v_readlane_b32 s102, v1, 35
+; CHECK-NEXT:    v_readlane_b32 s101, v1, 34
+; CHECK-NEXT:    v_readlane_b32 s100, v1, 33
+; CHECK-NEXT:    v_readlane_b32 s99, v1, 32
+; CHECK-NEXT:    v_readlane_b32 s98, v1, 31
+; CHECK-NEXT:    v_readlane_b32 s97, v1, 30
+; CHECK-NEXT:    v_readlane_b32 s96, v1, 29
+; CHECK-NEXT:    v_readlane_b32 s87, v1, 28
+; CHECK-NEXT:    v_readlane_b32 s86, v1, 27
+; CHECK-NEXT:    v_readlane_b32 s85, v1, 26
+; CHECK-NEXT:    v_readlane_b32 s84, v1, 25
+; CHECK-NEXT:    v_readlane_b32 s83, v1, 24
+; CHECK-NEXT:    v_readlane_b32 s82, v1, 23
+; CHECK-NEXT:    v_readlane_b32 s81, v1, 22
+; CHECK-NEXT:    v_readlane_b32 s80, v1, 21
+; CHECK-NEXT:    v_readlane_b32 s71, v1, 20
+; CHECK-NEXT:    v_readlane_b32 s70, v1, 19
+; CHECK-NEXT:    v_readlane_b32 s69, v1, 18
+; CHECK-NEXT:    v_readlane_b32 s68, v1, 17
+; CHECK-NEXT:    v_readlane_b32 s67, v1, 16
+; CHECK-NEXT:    v_readlane_b32 s66, v1, 15
+; CHECK-NEXT:    v_readlane_b32 s65, v1, 14
+; CHECK-NEXT:    v_readlane_b32 s64, v1, 13
+; CHECK-NEXT:    v_readlane_b32 s55, v1, 12
+; CHECK-NEXT:    v_readlane_b32 s54, v1, 11
+; CHECK-NEXT:    v_readlane_b32 s53, v1, 10
+; CHECK-NEXT:    v_readlane_b32 s52, v1, 9
+; CHECK-NEXT:    v_readlane_b32 s51, v1, 8
+; CHECK-NEXT:    v_readlane_b32 s50, v1, 7
+; CHECK-NEXT:    v_readlane_b32 s49, v1, 6
+; CHECK-NEXT:    v_readlane_b32 s48, v1, 5
 ; CHECK-NEXT:    v_readlane_b32 s39, v1, 4
 ; CHECK-NEXT:    v_readlane_b32 s38, v1, 3
 ; CHECK-NEXT:    v_readlane_b32 s37, v1, 2
@@ -317,7 +186,6 @@ define void @spill_more_than_wavesize_csr_sgprs_with_stack_object() {
 ; CHECK-NEXT:    v_readlane_b32 s35, v1, 0
 ; CHECK-NEXT:    s_xor_saveexec_b64 s[4:5], -1
 ; CHECK-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; CHECK-NEXT:    buffer_load_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
 ; CHECK-NEXT:    s_mov_b64 exec, s[4:5]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/spillv16.ll b/llvm/test/CodeGen/AMDGPU/spillv16.ll
index 0e45df223465d..3d21860e2af40 100644
--- a/llvm/test/CodeGen/AMDGPU/spillv16.ll
+++ b/llvm/test/CodeGen/AMDGPU/spillv16.ll
@@ -61,8 +61,8 @@ define void @spill_i16_alu_two_vals() {
 ; GCN-TRUE16-NEXT:    scratch_load_d16_b16 v0, off, s32 offset:4 glc dlc
 ; GCN-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-TRUE16-NEXT:    scratch_load_d16_hi_b16 v0, off, s32 offset:6 ; 2-byte Folded Reload
-; GCN-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x7b, v0.l
+; GCN-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-TRUE16-NEXT:    scratch_store_d16_hi_b16 off, v0, s32 dlc
 ; GCN-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GCN-TRUE16-NEXT:    scratch_store_b16 off, v0, s32 offset:4 dlc
diff --git a/llvm/test/CodeGen/AMDGPU/split-arg-dbg-value.ll b/llvm/test/CodeGen/AMDGPU/split-arg-dbg-value.ll
index 6d91c33fd2876..109c7d638f924 100644
--- a/llvm/test/CodeGen/AMDGPU/split-arg-dbg-value.ll
+++ b/llvm/test/CodeGen/AMDGPU/split-arg-dbg-value.ll
@@ -52,7 +52,7 @@ define hidden <4 x float> @split_v4f32_multi_arg(<4 x float> %arg0, <2 x float>
 ; GCN:         .cfi_endproc
   call void @llvm.dbg.value(metadata <4 x float> %arg0, metadata !29, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !31
   call void @llvm.dbg.value(metadata <2 x float> %arg1, metadata !30, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)), !dbg !31
-  %tmp = shufflevector <2 x float> %arg1, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>, !dbg !32
+  %tmp = shufflevector <2 x float> %arg1, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>, !dbg !32
   %add = fadd <4 x float> %tmp, %arg0, !dbg !33
   ret <4 x float> %add, !dbg !34
 }
diff --git a/llvm/test/CodeGen/AMDGPU/split-scalar-i64-add.ll b/llvm/test/CodeGen/AMDGPU/split-scalar-i64-add.ll
index 22ed0f37c1b3d..00c2a9d96dc85 100644
--- a/llvm/test/CodeGen/AMDGPU/split-scalar-i64-add.ll
+++ b/llvm/test/CodeGen/AMDGPU/split-scalar-i64-add.ll
@@ -15,7 +15,7 @@ declare i32 @llvm.amdgcn.workitem.id.x() readnone
 ; SI: v_addc_u32_e32 v{{[0-9]+}}, vcc, 0, v{{[0-9]+}}, vcc
 define amdgpu_kernel void @imp_def_vcc_split_i64_add_0(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %s.val) {
   %v.val = load volatile i32, ptr addrspace(1) %in
-  %vec.0 = insertelement <2 x i32> undef, i32 %s.val, i32 0
+  %vec.0 = insertelement <2 x i32> poison, i32 %s.val, i32 0
   %vec.1 = insertelement <2 x i32> %vec.0, i32 %v.val, i32 1
   %bc = bitcast <2 x i32> %vec.1 to i64
   %add = add i64 %bc, 399
@@ -27,7 +27,7 @@ define amdgpu_kernel void @imp_def_vcc_split_i64_add_0(ptr addrspace(1) %out, pt
 ; SI: s_add_u32 {{s[0-9]+}}, {{s[0-9]+}}, 0x18f
 ; SI: s_addc_u32 {{s[0-9]+}}, 0xf423f, 0
 define amdgpu_kernel void @s_imp_def_vcc_split_i64_add_0(ptr addrspace(1) %out, i32 %val) {
-  %vec.0 = insertelement <2 x i32> undef, i32 %val, i32 0
+  %vec.0 = insertelement <2 x i32> poison, i32 %val, i32 0
   %vec.1 = insertelement <2 x i32> %vec.0, i32 999999, i32 1
   %bc = bitcast <2 x i32> %vec.1 to i64
   %add = add i64 %bc, 399
@@ -40,7 +40,7 @@ define amdgpu_kernel void @s_imp_def_vcc_split_i64_add_0(ptr addrspace(1) %out,
 ; SI: v_addc_u32
 define amdgpu_kernel void @imp_def_vcc_split_i64_add_1(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %val0, i64 %val1) {
   %v.val = load volatile i32, ptr addrspace(1) %in
-  %vec.0 = insertelement <2 x i32> undef, i32 %val0, i32 0
+  %vec.0 = insertelement <2 x i32> poison, i32 %val0, i32 0
   %vec.1 = insertelement <2 x i32> %vec.0, i32 %v.val, i32 1
   %bc = bitcast <2 x i32> %vec.1 to i64
   %add = add i64 %bc, %val1
@@ -52,7 +52,7 @@ define amdgpu_kernel void @imp_def_vcc_split_i64_add_1(ptr addrspace(1) %out, pt
 ; SI: s_add_u32 {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}
 ; SI: s_addc_u32 {{s[0-9]+}}, 0x1869f, {{s[0-9]+}}
 define amdgpu_kernel void @s_imp_def_vcc_split_i64_add_1(ptr addrspace(1) %out, i32 %val0, i64 %val1) {
-  %vec.0 = insertelement <2 x i32> undef, i32 %val0, i32 0
+  %vec.0 = insertelement <2 x i32> poison, i32 %val0, i32 0
   %vec.1 = insertelement <2 x i32> %vec.0, i32 99999, i32 1
   %bc = bitcast <2 x i32> %vec.1 to i64
   %add = add i64 %bc, %val1
@@ -68,7 +68,7 @@ define amdgpu_kernel void @imp_def_vcc_split_i64_add_2(ptr addrspace(1) %out, pt
   %tid = call i32 @llvm.amdgcn.workitem.id.x() readnone
   %gep = getelementptr i32, ptr addrspace(1) %in, i32 %tid
   %load = load i32, ptr addrspace(1) %gep
-  %vec.0 = insertelement <2 x i32> undef, i32 %val0, i32 0
+  %vec.0 = insertelement <2 x i32> poison, i32 %val0, i32 0
   %vec.1 = insertelement <2 x i32> %vec.0, i32 %load, i32 1
   %bc = bitcast <2 x i32> %vec.1 to i64
   %add = add i64 %bc, %val1
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-copy-bundle.mir b/llvm/test/CodeGen/AMDGPU/splitkit-copy-bundle.mir
index 8f53ec2f992da..359152e9d2b45 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-copy-bundle.mir
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-copy-bundle.mir
@@ -17,70 +17,78 @@ body:             |
   ; RA-NEXT:   [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
   ; RA-NEXT:   undef [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_1024 = S_MOV_B32 -1
   ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub0:sgpr_1024 = S_MOV_B32 -1
+  ; RA-NEXT:   SI_SPILL_S1024_SAVE [[S_MOV_B32_]], %stack.0, implicit $exec, implicit $sgpr32 :: (store (s1024) into %stack.0, align 4, addrspace 5)
   ; RA-NEXT:   undef [[S_MOV_B32_1:%[0-9]+]].sub0:sgpr_1024 = S_MOV_B32 0
+  ; RA-NEXT:   SI_SPILL_S1024_SAVE [[S_MOV_B32_1]], %stack.1, implicit $exec, implicit $sgpr32 :: (store (s1024) into %stack.1, align 4, addrspace 5)
   ; RA-NEXT: {{  $}}
   ; RA-NEXT: bb.1:
   ; RA-NEXT:   successors: %bb.2(0x80000000)
   ; RA-NEXT: {{  $}}
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub2:sgpr_1024 = COPY [[S_MOV_B32_]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub3:sgpr_1024 = COPY [[S_MOV_B32_]].sub1
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub4:sgpr_1024 = COPY [[S_MOV_B32_]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub5:sgpr_1024 = COPY [[S_MOV_B32_]].sub1
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub6:sgpr_1024 = COPY [[S_MOV_B32_]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub7:sgpr_1024 = COPY [[S_MOV_B32_]].sub1
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub8:sgpr_1024 = COPY [[S_MOV_B32_]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub9:sgpr_1024 = COPY [[S_MOV_B32_]].sub1
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub10:sgpr_1024 = COPY [[S_MOV_B32_]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub11:sgpr_1024 = COPY [[S_MOV_B32_]].sub1
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub12:sgpr_1024 = COPY [[S_MOV_B32_]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub13:sgpr_1024 = COPY [[S_MOV_B32_]].sub1
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub14:sgpr_1024 = COPY [[S_MOV_B32_]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub15:sgpr_1024 = COPY [[S_MOV_B32_]].sub1
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub16:sgpr_1024 = COPY [[S_MOV_B32_]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub17:sgpr_1024 = COPY [[S_MOV_B32_]].sub1
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub18:sgpr_1024 = COPY [[S_MOV_B32_]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub19:sgpr_1024 = COPY [[S_MOV_B32_]].sub1
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub20:sgpr_1024 = COPY [[S_MOV_B32_]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub21:sgpr_1024 = COPY [[S_MOV_B32_]].sub1
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub22:sgpr_1024 = COPY [[S_MOV_B32_]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub23:sgpr_1024 = COPY [[S_MOV_B32_]].sub1
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub24:sgpr_1024 = COPY [[S_MOV_B32_]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub25:sgpr_1024 = COPY [[S_MOV_B32_]].sub1
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub26:sgpr_1024 = COPY [[S_MOV_B32_]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub27:sgpr_1024 = COPY [[S_MOV_B32_]].sub1
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub28:sgpr_1024 = COPY [[S_MOV_B32_]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_:%[0-9]+]].sub29:sgpr_1024 = COPY [[S_MOV_B32_]].sub1
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub1:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub2:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub3:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub4:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub5:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub6:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub7:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub8:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub9:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub10:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub11:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub12:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub13:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub14:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub15:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub16:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub17:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub18:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub19:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub20:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub21:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub22:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub23:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub24:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub25:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub26:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub27:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub28:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub29:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub30:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
-  ; RA-NEXT:   [[S_MOV_B32_1:%[0-9]+]].sub31:sgpr_1024 = COPY [[S_MOV_B32_1]].sub0
+  ; RA-NEXT:   [[SI_SPILL_S1024_RESTORE:%[0-9]+]]:sgpr_1024 = SI_SPILL_S1024_RESTORE %stack.0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.0, align 4, addrspace 5)
+  ; RA-NEXT:   undef [[COPY:%[0-9]+]].sub0_sub1:sgpr_1024 = COPY [[SI_SPILL_S1024_RESTORE]].sub0_sub1
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub2:sgpr_1024 = COPY [[COPY]].sub0
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub3:sgpr_1024 = COPY [[COPY]].sub1
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub4:sgpr_1024 = COPY [[COPY]].sub0
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub5:sgpr_1024 = COPY [[COPY]].sub1
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub6:sgpr_1024 = COPY [[COPY]].sub0
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub7:sgpr_1024 = COPY [[COPY]].sub1
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub8:sgpr_1024 = COPY [[COPY]].sub0
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub9:sgpr_1024 = COPY [[COPY]].sub1
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub10:sgpr_1024 = COPY [[COPY]].sub0
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub11:sgpr_1024 = COPY [[COPY]].sub1
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub12:sgpr_1024 = COPY [[COPY]].sub0
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub13:sgpr_1024 = COPY [[COPY]].sub1
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub14:sgpr_1024 = COPY [[COPY]].sub0
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub15:sgpr_1024 = COPY [[COPY]].sub1
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub16:sgpr_1024 = COPY [[COPY]].sub0
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub17:sgpr_1024 = COPY [[COPY]].sub1
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub18:sgpr_1024 = COPY [[COPY]].sub0
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub19:sgpr_1024 = COPY [[COPY]].sub1
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub20:sgpr_1024 = COPY [[COPY]].sub0
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub21:sgpr_1024 = COPY [[COPY]].sub1
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub22:sgpr_1024 = COPY [[COPY]].sub0
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub23:sgpr_1024 = COPY [[COPY]].sub1
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub24:sgpr_1024 = COPY [[COPY]].sub0
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub25:sgpr_1024 = COPY [[COPY]].sub1
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub26:sgpr_1024 = COPY [[COPY]].sub0
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub27:sgpr_1024 = COPY [[COPY]].sub1
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub28:sgpr_1024 = COPY [[COPY]].sub0
+  ; RA-NEXT:   [[COPY:%[0-9]+]].sub29:sgpr_1024 = COPY [[COPY]].sub1
+  ; RA-NEXT:   SI_SPILL_S1024_SAVE [[COPY]], %stack.0, implicit $exec, implicit $sgpr32 :: (store (s1024) into %stack.0, align 4, addrspace 5)
+  ; RA-NEXT:   [[SI_SPILL_S1024_RESTORE1:%[0-9]+]]:sgpr_1024 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; RA-NEXT:   undef [[COPY1:%[0-9]+]].sub0:sgpr_1024 = COPY [[SI_SPILL_S1024_RESTORE1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub1:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub2:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub3:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub4:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub5:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub6:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub7:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub8:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub9:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub10:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub11:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub12:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub13:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub14:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub15:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub16:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub17:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub18:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub19:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub20:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub21:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub22:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub23:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub24:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub25:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub26:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub27:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub28:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub29:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub30:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   [[COPY1:%[0-9]+]].sub31:sgpr_1024 = COPY [[COPY1]].sub0
+  ; RA-NEXT:   SI_SPILL_S1024_SAVE [[COPY1]], %stack.1, implicit $exec, implicit $sgpr32 :: (store (s1024) into %stack.1, align 4, addrspace 5)
   ; RA-NEXT: {{  $}}
   ; RA-NEXT: bb.2:
   ; RA-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
@@ -95,14 +103,17 @@ body:             |
   ; VR-NEXT: {{  $}}
   ; VR-NEXT:   renamable $sgpr37 = S_MOV_B32 -1
   ; VR-NEXT:   renamable $sgpr36 = S_MOV_B32 -1
-  ; VR-NEXT:   renamable $sgpr68 = S_MOV_B32 0
+  ; VR-NEXT:   SI_SPILL_S1024_SAVE renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, %stack.0, implicit $exec, implicit $sgpr32 :: (store (s1024) into %stack.0, align 4, addrspace 5)
+  ; VR-NEXT:   renamable $sgpr36 = S_MOV_B32 0
   ; VR-NEXT:   renamable $sgpr30_sgpr31 = IMPLICIT_DEF
   ; VR-NEXT:   renamable $sgpr34_sgpr35 = IMPLICIT_DEF
+  ; VR-NEXT:   SI_SPILL_S1024_SAVE killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, %stack.1, implicit $exec, implicit $sgpr32 :: (store (s1024) into %stack.1, align 4, addrspace 5)
   ; VR-NEXT: {{  $}}
   ; VR-NEXT: bb.1:
   ; VR-NEXT:   successors: %bb.2(0x80000000)
-  ; VR-NEXT:   liveins: $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67:0x000000000000000F, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99:0x0000000000000003
+  ; VR-NEXT:   liveins: $sgpr30_sgpr31, $sgpr34_sgpr35
   ; VR-NEXT: {{  $}}
+  ; VR-NEXT:   renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67 = SI_SPILL_S1024_RESTORE %stack.0, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.0, align 4, addrspace 5)
   ; VR-NEXT:   renamable $sgpr38 = COPY renamable $sgpr36
   ; VR-NEXT:   renamable $sgpr39 = COPY renamable $sgpr37
   ; VR-NEXT:   renamable $sgpr40 = COPY renamable $sgpr36
@@ -131,41 +142,44 @@ body:             |
   ; VR-NEXT:   renamable $sgpr63 = COPY renamable $sgpr37
   ; VR-NEXT:   renamable $sgpr64 = COPY renamable $sgpr36
   ; VR-NEXT:   renamable $sgpr65 = COPY renamable $sgpr37
-  ; VR-NEXT:   renamable $sgpr69 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr70 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr71 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr72 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr73 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr74 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr75 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr76 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr77 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr78 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr79 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr80 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr81 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr82 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr83 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr84 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr85 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr86 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr87 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr88 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr89 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr90 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr91 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr92 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr93 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr94 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr95 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr96 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr97 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr98 = COPY renamable $sgpr68
-  ; VR-NEXT:   renamable $sgpr99 = COPY renamable $sgpr68
+  ; VR-NEXT:   SI_SPILL_S1024_SAVE killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, %stack.0, implicit $exec, implicit $sgpr32 :: (store (s1024) into %stack.0, align 4, addrspace 5)
+  ; VR-NEXT:   renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67 = SI_SPILL_S1024_RESTORE %stack.1, implicit $exec, implicit $sgpr32 :: (load (s1024) from %stack.1, align 4, addrspace 5)
+  ; VR-NEXT:   renamable $sgpr37 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr38 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr39 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr40 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr41 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr42 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr43 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr44 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr45 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr46 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr47 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr48 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr49 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr50 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr51 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr52 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr53 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr54 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr55 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr56 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr57 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr58 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr59 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr60 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr61 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr62 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr63 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr64 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr65 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr66 = COPY renamable $sgpr36
+  ; VR-NEXT:   renamable $sgpr67 = COPY renamable $sgpr36
+  ; VR-NEXT:   SI_SPILL_S1024_SAVE killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, %stack.1, implicit $exec, implicit $sgpr32 :: (store (s1024) into %stack.1, align 4, addrspace 5)
   ; VR-NEXT: {{  $}}
   ; VR-NEXT: bb.2:
   ; VR-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
-  ; VR-NEXT:   liveins: $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67:0x000000000000000F, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95_sgpr96_sgpr97_sgpr98_sgpr99:0x0000000000000003
+  ; VR-NEXT:   liveins: $sgpr30_sgpr31, $sgpr34_sgpr35
   ; VR-NEXT: {{  $}}
   ; VR-NEXT:   S_NOP 0, csr_amdgpu, implicit renamable $sgpr30_sgpr31, implicit renamable $sgpr34_sgpr35
   ; VR-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit undef $vcc
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir b/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir
index 9e2501a65c6e4..9b4031826dd22 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir
@@ -153,3 +153,146 @@ body:             |
     S_ENDPGM 0
 
 ...
+
+# Same, except uses AV_MOV_B32_IMM_PSEUDO which is able to
+# rematerialize in the inflated register class.
+---
+name:            temp_vgpr_to_agpr_should_not_undo_split_with_remat_av_pseudo
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
+  stackPtrOffsetReg: '$sgpr32'
+  argumentInfo:
+    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+    kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+    workGroupIDX:    { reg: '$sgpr6' }
+    privateSegmentWaveByteOffset: { reg: '$sgpr7' }
+    workItemIDX:     { reg: '$vgpr0' }
+  occupancy:       4
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  bb.0:
+    liveins: $vgpr0, $sgpr4_sgpr5
+
+    ; CHECK-LABEL: name: temp_vgpr_to_agpr_should_not_undo_split_with_remat_av_pseudo
+    ; CHECK: liveins: $vgpr0, $sgpr4_sgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: S_NOP 0, implicit-def $agpr0
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+    ; CHECK-NEXT: [[V_LSHLREV_B32_e32_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e32 7, [[COPY]], implicit $exec
+    ; CHECK-NEXT: undef [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub28_sub29_sub30_sub31:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, [[V_LSHLREV_B32_e32_]], 112, 0, implicit $exec :: (load (s128), addrspace 1)
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub24_sub25_sub26_sub27:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, [[V_LSHLREV_B32_e32_]], 96, 0, implicit $exec :: (load (s128), align 32, addrspace 1)
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub20_sub21_sub22_sub23:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, [[V_LSHLREV_B32_e32_]], 80, 0, implicit $exec :: (load (s128), addrspace 1)
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub16_sub17_sub18_sub19:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, [[V_LSHLREV_B32_e32_]], 64, 0, implicit $exec :: (load (s128), align 64, addrspace 1)
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub12_sub13_sub14_sub15:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, [[V_LSHLREV_B32_e32_]], 48, 0, implicit $exec :: (load (s128), addrspace 1)
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub8_sub9_sub10_sub11:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, [[V_LSHLREV_B32_e32_]], 32, 0, implicit $exec :: (load (s128), align 32, addrspace 1)
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub4_sub5_sub6_sub7:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, [[V_LSHLREV_B32_e32_]], 16, 0, implicit $exec :: (load (s128), addrspace 1)
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub0_sub1_sub2_sub3:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, [[V_LSHLREV_B32_e32_]], 0, 0, implicit $exec :: (load (s128), align 128, addrspace 1)
+    ; CHECK-NEXT: [[AV_MOV_:%[0-9]+]]:av_32 = AV_MOV_B32_IMM_PSEUDO 1065353216, implicit $exec
+    ; CHECK-NEXT: [[AV_MOV_1:%[0-9]+]]:av_32 = AV_MOV_B32_IMM_PSEUDO 1073741824, implicit $exec
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]]:vreg_1024_align2 = V_MFMA_F32_32X32X1F32_mac_vgprcd_e64 [[AV_MOV_]], [[AV_MOV_1]], [[GLOBAL_LOAD_DWORDX4_SADDR]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: [[AV_MOV_2:%[0-9]+]]:av_32 = AV_MOV_B32_IMM_PSEUDO 1073741824, implicit $exec
+    ; CHECK-NEXT: [[AV_MOV_3:%[0-9]+]]:av_32 = AV_MOV_B32_IMM_PSEUDO 1065353216, implicit $exec
+    ; CHECK-NEXT: early-clobber %5:vreg_1024_align2 = V_MFMA_F32_32X32X1F32_vgprcd_e64 [[AV_MOV_3]], [[AV_MOV_2]], [[GLOBAL_LOAD_DWORDX4_SADDR]], 0, 0, 0, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub2:vreg_1024_align2 = COPY %5.sub0
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub3:vreg_1024_align2 = COPY %5.sub1
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub4:vreg_1024_align2 = COPY %5.sub2
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub5:vreg_1024_align2 = COPY %5.sub3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub6:vreg_1024_align2 = COPY %5.sub4
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub7:vreg_1024_align2 = COPY %5.sub5
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub8:vreg_1024_align2 = COPY %5.sub6
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub9:vreg_1024_align2 = COPY %5.sub7
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub10:vreg_1024_align2 = COPY %5.sub8
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub11:vreg_1024_align2 = COPY %5.sub9
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub12:vreg_1024_align2 = COPY %5.sub10
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub13:vreg_1024_align2 = COPY %5.sub11
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub14:vreg_1024_align2 = COPY %5.sub12
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub15:vreg_1024_align2 = COPY %5.sub13
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub16:vreg_1024_align2 = COPY %5.sub14
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub17:vreg_1024_align2 = COPY %5.sub15
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub18:vreg_1024_align2 = COPY %5.sub16
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub19:vreg_1024_align2 = COPY %5.sub17
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub20:vreg_1024_align2 = COPY %5.sub18
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub21:vreg_1024_align2 = COPY %5.sub19
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub22:vreg_1024_align2 = COPY %5.sub20
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub23:vreg_1024_align2 = COPY %5.sub21
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub24:vreg_1024_align2 = COPY %5.sub22
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub25:vreg_1024_align2 = COPY %5.sub23
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub26:vreg_1024_align2 = COPY %5.sub24
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub27:vreg_1024_align2 = COPY %5.sub25
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub28:vreg_1024_align2 = COPY %5.sub26
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub29:vreg_1024_align2 = COPY %5.sub27
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub30:vreg_1024_align2 = COPY %5.sub28
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub31:vreg_1024_align2 = COPY %5.sub29
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]]:vreg_1024_align2 = V_MFMA_F32_32X32X1F32_mac_vgprcd_e64 [[AV_MOV_3]], [[AV_MOV_2]], [[GLOBAL_LOAD_DWORDX4_SADDR]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: [[AV_MOV_4:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR [[AV_MOV_4]], [[GLOBAL_LOAD_DWORDX4_SADDR]].sub24_sub25_sub26_sub27, renamable $sgpr0_sgpr1, 96, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR [[AV_MOV_4]], [[GLOBAL_LOAD_DWORDX4_SADDR]].sub28_sub29_sub30_sub31, renamable $sgpr0_sgpr1, 112, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR [[AV_MOV_4]], [[GLOBAL_LOAD_DWORDX4_SADDR]].sub16_sub17_sub18_sub19, renamable $sgpr0_sgpr1, 64, 0, implicit $exec :: (store (s128), align 64, addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR [[AV_MOV_4]], [[GLOBAL_LOAD_DWORDX4_SADDR]].sub20_sub21_sub22_sub23, renamable $sgpr0_sgpr1, 80, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR [[AV_MOV_4]], [[GLOBAL_LOAD_DWORDX4_SADDR]].sub8_sub9_sub10_sub11, renamable $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR [[AV_MOV_4]], [[GLOBAL_LOAD_DWORDX4_SADDR]].sub12_sub13_sub14_sub15, renamable $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR [[AV_MOV_4]], [[GLOBAL_LOAD_DWORDX4_SADDR]].sub0_sub1_sub2_sub3, renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR [[AV_MOV_4]], [[GLOBAL_LOAD_DWORDX4_SADDR]].sub4_sub5_sub6_sub7, killed renamable $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    S_NOP 0, implicit-def $agpr0
+    %0:vgpr_32 = COPY $vgpr0
+    renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+    %1:vgpr_32 = V_LSHLREV_B32_e32 7, %0, implicit $exec
+    undef %2.sub28_sub29_sub30_sub31:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, %1, 112, 0, implicit $exec :: (load (s128), addrspace 1)
+    %2.sub24_sub25_sub26_sub27:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, %1, 96, 0, implicit $exec :: (load (s128), align 32, addrspace 1)
+    %2.sub20_sub21_sub22_sub23:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, %1, 80, 0, implicit $exec :: (load (s128), addrspace 1)
+    %2.sub16_sub17_sub18_sub19:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, %1, 64, 0, implicit $exec :: (load (s128), align 64, addrspace 1)
+    %2.sub12_sub13_sub14_sub15:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, %1, 48, 0, implicit $exec :: (load (s128), addrspace 1)
+    %2.sub8_sub9_sub10_sub11:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, %1, 32, 0, implicit $exec :: (load (s128), align 32, addrspace 1)
+    %2.sub4_sub5_sub6_sub7:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, %1, 16, 0, implicit $exec :: (load (s128), addrspace 1)
+    %2.sub0_sub1_sub2_sub3:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, %1, 0, 0, implicit $exec :: (load (s128), align 128, addrspace 1)
+    %3:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1065353216, implicit $exec
+    %4:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1073741824, implicit $exec
+    %2:vreg_1024_align2 = V_MFMA_F32_32X32X1F32_mac_vgprcd_e64 %3, %4, %2, 0, 0, 0, implicit $mode, implicit $exec
+    early-clobber %5:vreg_1024_align2 = V_MFMA_F32_32X32X1F32_vgprcd_e64 %3, %4, %2, 0, 0, 0, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+    %2.sub2:vreg_1024_align2 = COPY %5.sub0
+    %2.sub3:vreg_1024_align2 = COPY %5.sub1
+    %2.sub4:vreg_1024_align2 = COPY %5.sub2
+    %2.sub5:vreg_1024_align2 = COPY %5.sub3
+    %2.sub6:vreg_1024_align2 = COPY %5.sub4
+    %2.sub7:vreg_1024_align2 = COPY %5.sub5
+    %2.sub8:vreg_1024_align2 = COPY %5.sub6
+    %2.sub9:vreg_1024_align2 = COPY %5.sub7
+    %2.sub10:vreg_1024_align2 = COPY %5.sub8
+    %2.sub11:vreg_1024_align2 = COPY %5.sub9
+    %2.sub12:vreg_1024_align2 = COPY %5.sub10
+    %2.sub13:vreg_1024_align2 = COPY %5.sub11
+    %2.sub14:vreg_1024_align2 = COPY %5.sub12
+    %2.sub15:vreg_1024_align2 = COPY %5.sub13
+    %2.sub16:vreg_1024_align2 = COPY %5.sub14
+    %2.sub17:vreg_1024_align2 = COPY %5.sub15
+    %2.sub18:vreg_1024_align2 = COPY %5.sub16
+    %2.sub19:vreg_1024_align2 = COPY %5.sub17
+    %2.sub20:vreg_1024_align2 = COPY %5.sub18
+    %2.sub21:vreg_1024_align2 = COPY %5.sub19
+    %2.sub22:vreg_1024_align2 = COPY %5.sub20
+    %2.sub23:vreg_1024_align2 = COPY %5.sub21
+    %2.sub24:vreg_1024_align2 = COPY %5.sub22
+    %2.sub25:vreg_1024_align2 = COPY %5.sub23
+    %2.sub26:vreg_1024_align2 = COPY %5.sub24
+    %2.sub27:vreg_1024_align2 = COPY %5.sub25
+    %2.sub28:vreg_1024_align2 = COPY %5.sub26
+    %2.sub29:vreg_1024_align2 = COPY %5.sub27
+    %2.sub30:vreg_1024_align2 = COPY %5.sub28
+    %2.sub31:vreg_1024_align2 = COPY %5.sub29
+    %2:vreg_1024_align2 = V_MFMA_F32_32X32X1F32_mac_vgprcd_e64 %3, %4, %2, 0, 0, 0, implicit $mode, implicit $exec
+    %6:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %6, %2.sub24_sub25_sub26_sub27, renamable $sgpr0_sgpr1, 96, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %6, %2.sub28_sub29_sub30_sub31, renamable $sgpr0_sgpr1, 112, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %6, %2.sub16_sub17_sub18_sub19, renamable $sgpr0_sgpr1, 64, 0, implicit $exec :: (store (s128), align 64, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %6, %2.sub20_sub21_sub22_sub23, renamable $sgpr0_sgpr1, 80, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %6, %2.sub8_sub9_sub10_sub11, renamable $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %6, %2.sub12_sub13_sub14_sub15, renamable $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %6, %2.sub0_sub1_sub2_sub3, renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %6, %2.sub4_sub5_sub6_sub7, killed renamable $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
index c611c4b502817..64bbdb34b7d14 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
@@ -375,28 +375,28 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   %12 = extractelement <31 x i32> %userData, i64 24
   %13 = extractelement <31 x i32> %userData, i64 26
   %14 = extractelement <31 x i32> %userData, i64 30
-  %15 = insertelement <2 x i32> undef, i32 %13, i32 0
+  %15 = insertelement <2 x i32> poison, i32 %13, i32 0
   %16 = bitcast <2 x i32> %15 to i64
   %17 = inttoptr i64 %16 to ptr addrspace(4)
-  %18 = insertelement <2 x i32> undef, i32 %12, i32 0
+  %18 = insertelement <2 x i32> poison, i32 %12, i32 0
   %19 = bitcast <2 x i32> %18 to i64
   %20 = inttoptr i64 %19 to ptr addrspace(4)
-  %21 = insertelement <2 x i32> undef, i32 %11, i32 0
+  %21 = insertelement <2 x i32> poison, i32 %11, i32 0
   %22 = bitcast <2 x i32> %21 to i64
-  %23 = insertelement <2 x i32> undef, i32 %10, i32 0
+  %23 = insertelement <2 x i32> poison, i32 %10, i32 0
   %24 = bitcast <2 x i32> %23 to i64
-  %25 = insertelement <2 x i32> undef, i32 %9, i32 0
+  %25 = insertelement <2 x i32> poison, i32 %9, i32 0
   %26 = bitcast <2 x i32> %25 to i64
   %27 = inttoptr i64 %26 to ptr addrspace(4)
-  %28 = insertelement <2 x i32> undef, i32 %8, i32 0
+  %28 = insertelement <2 x i32> poison, i32 %8, i32 0
   %29 = bitcast <2 x i32> %28 to i64
-  %30 = insertelement <2 x i32> undef, i32 %7, i32 0
+  %30 = insertelement <2 x i32> poison, i32 %7, i32 0
   %31 = bitcast <2 x i32> %30 to i64
   %32 = inttoptr i64 %31 to ptr addrspace(4)
-  %33 = insertelement <2 x i32> undef, i32 %6, i32 0
+  %33 = insertelement <2 x i32> poison, i32 %6, i32 0
   %34 = bitcast <2 x i32> %33 to i64
   %35 = inttoptr i64 %34 to ptr addrspace(4)
-  %36 = insertelement <2 x i32> undef, i32 %14, i32 0
+  %36 = insertelement <2 x i32> poison, i32 %14, i32 0
   %37 = bitcast <2 x i32> %36 to i64
   %38 = inttoptr i64 %37 to ptr addrspace(4)
   %39 = getelementptr i8, ptr addrspace(4) %38, i64 232
@@ -404,10 +404,10 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   %.i184.i = getelementptr i8, ptr addrspace(4) %38, i64 236
   %rootDesc58.ii1.i = load i32, ptr addrspace(4) %.i184.i, align 4
   %40 = and i32 %rootDesc58.ii1.i, 65535
-  %41 = insertelement <4 x i32> <i32 undef, i32 undef, i32 -1, i32 553734060>, i32 %rootDesc58.ii0.i, i32 0
+  %41 = insertelement <4 x i32> <i32 poison, i32 poison, i32 -1, i32 553734060>, i32 %rootDesc58.ii0.i, i32 0
   %42 = insertelement <4 x i32> %41, i32 %40, i32 1
   %43 = and i32 undef, 65535
-  %44 = insertelement <4 x i32> undef, i32 %43, i32 1
+  %44 = insertelement <4 x i32> poison, i32 %43, i32 1
   %45 = load <4 x i32>, ptr addrspace(4) undef, align 16
   %46 = call i32 @llvm.amdgcn.struct.buffer.load.format.i32(<4 x i32> %45, i32 0, i32 0, i32 0, i32 0)
   %47 = add i32 %46, -1
@@ -662,7 +662,7 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   %296 = getelementptr i8, ptr addrspace(4) %293, i64 %295
   %.ii0.i = load i32, ptr addrspace(4) %296, align 8
   %297 = and i32 undef, 65535
-  %298 = insertelement <4 x i32> <i32 undef, i32 undef, i32 -1, i32 553734060>, i32 %.ii0.i, i32 0
+  %298 = insertelement <4 x i32> <i32 poison, i32 poison, i32 -1, i32 553734060>, i32 %.ii0.i, i32 0
   %299 = insertelement <4 x i32> %298, i32 %297, i32 1
   %300 = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %299, i32 0, i32 0)
   %301 = add i32 %300, -467
@@ -674,7 +674,7 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   %.i191.i = getelementptr i8, ptr addrspace(4) %305, i64 4
   %.ii192.i = load i32, ptr addrspace(4) %.i191.i, align 4
   %306 = and i32 %.ii192.i, 65535
-  %307 = insertelement <4 x i32> <i32 undef, i32 undef, i32 -1, i32 553734060>, i32 %.ii090.i, i32 0
+  %307 = insertelement <4 x i32> <i32 poison, i32 poison, i32 -1, i32 553734060>, i32 %.ii090.i, i32 0
   %308 = insertelement <4 x i32> %307, i32 %306, i32 1
   %309 = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %308, i32 0, i32 0)
   %310 = add i32 %309, -468
@@ -686,7 +686,7 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   %.i197.i = getelementptr i8, ptr addrspace(4) %314, i64 4
   %.ii198.i = load i32, ptr addrspace(4) %.i197.i, align 4
   %315 = and i32 %.ii198.i, 65535
-  %316 = insertelement <4 x i32> <i32 undef, i32 undef, i32 -1, i32 553734060>, i32 %.ii096.i, i32 0
+  %316 = insertelement <4 x i32> <i32 poison, i32 poison, i32 -1, i32 553734060>, i32 %.ii096.i, i32 0
   %317 = insertelement <4 x i32> %316, i32 %315, i32 1
   %318 = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %317, i32 0, i32 0)
   %319 = add i32 %318, -469
@@ -698,7 +698,7 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   %.ii0102.i = load i32, ptr addrspace(4) %324, align 8
   %.ii1104.i = load i32, ptr addrspace(4) undef, align 4
   %325 = and i32 %.ii1104.i, 65535
-  %326 = insertelement <4 x i32> <i32 undef, i32 undef, i32 -1, i32 553734060>, i32 %.ii0102.i, i32 0
+  %326 = insertelement <4 x i32> <i32 poison, i32 poison, i32 -1, i32 553734060>, i32 %.ii0102.i, i32 0
   %327 = insertelement <4 x i32> %326, i32 %325, i32 1
   %328 = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %327, i32 0, i32 0)
   %329 = add i32 %328, -473
@@ -755,7 +755,7 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   %.not.i = icmp eq i32 %379, 0
   %380 = load <8 x i32>, ptr addrspace(4) undef, align 32
   %.i010.i = select i1 %.not.i, float 0x36A0000000000000, float 0.000000e+00
-  %381 = insertelement <4 x float> undef, float %.i010.i, i32 3
+  %381 = insertelement <4 x float> poison, float %.i010.i, i32 3
   call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %381, i32 15, i32 undef, i32 undef, <8 x i32> %380, i32 0, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/sram-ecc-default.ll b/llvm/test/CodeGen/AMDGPU/sram-ecc-default.ll
index b0154279f4ae1..88ca0306ae7d1 100644
--- a/llvm/test/CodeGen/AMDGPU/sram-ecc-default.ll
+++ b/llvm/test/CodeGen/AMDGPU/sram-ecc-default.ll
@@ -17,7 +17,7 @@ define void @load_global_hi_v2i16_reglo_vreg(ptr addrspace(1) %in, i16 %reg) {
 entry:
   %gep = getelementptr inbounds i16, ptr addrspace(1) %in, i64 -2047
   %load = load i16, ptr addrspace(1) %gep
-  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0
   %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
   store <2 x i16> %build1, ptr addrspace(1) undef
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/stack-realign.ll b/llvm/test/CodeGen/AMDGPU/stack-realign.ll
index 0e568e3071e99..4ddde7f297172 100644
--- a/llvm/test/CodeGen/AMDGPU/stack-realign.ll
+++ b/llvm/test/CodeGen/AMDGPU/stack-realign.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 ; Check that we properly realign the stack. While 4-byte access is all
@@ -415,28 +416,21 @@ define void @no_free_scratch_sgpr_for_bp_copy(<32 x i32> %a, i32 %b) #0 {
 ; GCN-LABEL: no_free_scratch_sgpr_for_bp_copy:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_mov_b32 vcc_lo, s33
-; GCN-NEXT:    s_add_i32 s33, s32, 0x1fc0
-; GCN-NEXT:    s_and_b32 s33, s33, 0xffffe000
-; GCN-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GCN-NEXT:    buffer_store_dword v1, off, s[0:3], s33 offset:132 ; 4-byte Folded Spill
-; GCN-NEXT:    s_mov_b64 exec, s[4:5]
-; GCN-NEXT:    v_writelane_b32 v1, s34, 0
+; GCN-NEXT:    s_mov_b32 s41, s34
 ; GCN-NEXT:    s_mov_b32 s34, s32
 ; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s34 offset:4
+; GCN-NEXT:    s_mov_b32 s40, s33
+; GCN-NEXT:    s_add_i32 s33, s32, 0x1fc0
+; GCN-NEXT:    s_and_b32 s33, s33, 0xffffe000
 ; GCN-NEXT:    s_addk_i32 s32, 0x6000
 ; GCN-NEXT:    s_mov_b32 s32, s34
-; GCN-NEXT:    v_readlane_b32 s34, v1, 0
+; GCN-NEXT:    s_mov_b32 s34, s41
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s33 offset:128
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    ;;#ASMSTART
 ; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    s_xor_saveexec_b64 s[4:5], -1
-; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], s33 offset:132 ; 4-byte Folded Reload
-; GCN-NEXT:    s_mov_b64 exec, s[4:5]
-; GCN-NEXT:    s_mov_b32 s33, vcc_lo
-; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_mov_b32 s33, s40
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %local_val = alloca i32, align 128, addrspace(5)
   store volatile i32 %b, ptr addrspace(5) %local_val, align 128
@@ -460,78 +454,45 @@ define void @no_free_regs_spill_bp_to_memory(<32 x i32> %a, i32 %b) #5 {
 ; GCN-NEXT:    s_xor_saveexec_b64 s[6:7], -1
 ; GCN-NEXT:    buffer_store_dword v39, off, s[0:3], s33 offset:132 ; 4-byte Folded Spill
 ; GCN-NEXT:    s_mov_b64 exec, s[6:7]
-; GCN-NEXT:    v_mov_b32_e32 v0, s4
-; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s33 offset:136 ; 4-byte Folded Spill
-; GCN-NEXT:    v_mov_b32_e32 v0, s34
+; GCN-NEXT:    v_writelane_b32 v39, s4, 32
+; GCN-NEXT:    v_writelane_b32 v39, s34, 33
 ; GCN-NEXT:    s_mov_b32 s34, s32
-; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s33 offset:140 ; 4-byte Folded Spill
 ; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s34 offset:4
 ; GCN-NEXT:    v_writelane_b32 v39, s39, 0
-; GCN-NEXT:    v_writelane_b32 v39, s40, 1
-; GCN-NEXT:    v_writelane_b32 v39, s41, 2
-; GCN-NEXT:    v_writelane_b32 v39, s42, 3
-; GCN-NEXT:    v_writelane_b32 v39, s43, 4
-; GCN-NEXT:    v_writelane_b32 v39, s44, 5
-; GCN-NEXT:    v_writelane_b32 v39, s45, 6
-; GCN-NEXT:    v_writelane_b32 v39, s46, 7
-; GCN-NEXT:    v_writelane_b32 v39, s47, 8
-; GCN-NEXT:    v_writelane_b32 v39, s48, 9
-; GCN-NEXT:    v_writelane_b32 v39, s49, 10
-; GCN-NEXT:    v_writelane_b32 v39, s50, 11
-; GCN-NEXT:    v_writelane_b32 v39, s51, 12
-; GCN-NEXT:    v_writelane_b32 v39, s52, 13
-; GCN-NEXT:    v_writelane_b32 v39, s53, 14
-; GCN-NEXT:    v_writelane_b32 v39, s54, 15
-; GCN-NEXT:    v_writelane_b32 v39, s55, 16
-; GCN-NEXT:    v_writelane_b32 v39, s56, 17
-; GCN-NEXT:    v_writelane_b32 v39, s57, 18
-; GCN-NEXT:    v_writelane_b32 v39, s58, 19
-; GCN-NEXT:    v_writelane_b32 v39, s59, 20
-; GCN-NEXT:    v_writelane_b32 v39, s60, 21
-; GCN-NEXT:    v_writelane_b32 v39, s61, 22
-; GCN-NEXT:    v_writelane_b32 v39, s62, 23
-; GCN-NEXT:    v_writelane_b32 v39, s63, 24
-; GCN-NEXT:    v_writelane_b32 v39, s64, 25
-; GCN-NEXT:    v_writelane_b32 v39, s65, 26
-; GCN-NEXT:    v_writelane_b32 v39, s66, 27
-; GCN-NEXT:    v_writelane_b32 v39, s67, 28
-; GCN-NEXT:    v_writelane_b32 v39, s68, 29
-; GCN-NEXT:    v_writelane_b32 v39, s69, 30
-; GCN-NEXT:    v_writelane_b32 v39, s70, 31
-; GCN-NEXT:    v_writelane_b32 v39, s71, 32
-; GCN-NEXT:    v_writelane_b32 v39, s72, 33
-; GCN-NEXT:    v_writelane_b32 v39, s73, 34
-; GCN-NEXT:    v_writelane_b32 v39, s74, 35
-; GCN-NEXT:    v_writelane_b32 v39, s75, 36
-; GCN-NEXT:    v_writelane_b32 v39, s76, 37
-; GCN-NEXT:    v_writelane_b32 v39, s77, 38
-; GCN-NEXT:    v_writelane_b32 v39, s78, 39
-; GCN-NEXT:    v_writelane_b32 v39, s79, 40
-; GCN-NEXT:    v_writelane_b32 v39, s80, 41
-; GCN-NEXT:    v_writelane_b32 v39, s81, 42
-; GCN-NEXT:    v_writelane_b32 v39, s82, 43
-; GCN-NEXT:    v_writelane_b32 v39, s83, 44
-; GCN-NEXT:    v_writelane_b32 v39, s84, 45
-; GCN-NEXT:    v_writelane_b32 v39, s85, 46
-; GCN-NEXT:    v_writelane_b32 v39, s86, 47
-; GCN-NEXT:    v_writelane_b32 v39, s87, 48
-; GCN-NEXT:    v_writelane_b32 v39, s88, 49
-; GCN-NEXT:    v_writelane_b32 v39, s89, 50
-; GCN-NEXT:    v_writelane_b32 v39, s90, 51
-; GCN-NEXT:    v_writelane_b32 v39, s91, 52
-; GCN-NEXT:    v_writelane_b32 v39, s92, 53
-; GCN-NEXT:    v_writelane_b32 v39, s93, 54
-; GCN-NEXT:    v_writelane_b32 v39, s94, 55
-; GCN-NEXT:    v_writelane_b32 v39, s95, 56
-; GCN-NEXT:    v_writelane_b32 v39, s96, 57
-; GCN-NEXT:    v_writelane_b32 v39, s97, 58
-; GCN-NEXT:    v_writelane_b32 v39, s98, 59
-; GCN-NEXT:    v_writelane_b32 v39, s99, 60
-; GCN-NEXT:    v_writelane_b32 v39, s100, 61
-; GCN-NEXT:    v_writelane_b32 v39, s101, 62
-; GCN-NEXT:    v_writelane_b32 v39, s102, 63
+; GCN-NEXT:    v_writelane_b32 v39, s48, 1
+; GCN-NEXT:    v_writelane_b32 v39, s49, 2
+; GCN-NEXT:    v_writelane_b32 v39, s50, 3
+; GCN-NEXT:    v_writelane_b32 v39, s51, 4
+; GCN-NEXT:    v_writelane_b32 v39, s52, 5
+; GCN-NEXT:    v_writelane_b32 v39, s53, 6
+; GCN-NEXT:    v_writelane_b32 v39, s54, 7
+; GCN-NEXT:    v_writelane_b32 v39, s55, 8
+; GCN-NEXT:    v_writelane_b32 v39, s64, 9
+; GCN-NEXT:    v_writelane_b32 v39, s65, 10
+; GCN-NEXT:    v_writelane_b32 v39, s66, 11
+; GCN-NEXT:    v_writelane_b32 v39, s67, 12
+; GCN-NEXT:    v_writelane_b32 v39, s68, 13
+; GCN-NEXT:    v_writelane_b32 v39, s69, 14
+; GCN-NEXT:    v_writelane_b32 v39, s70, 15
+; GCN-NEXT:    v_writelane_b32 v39, s71, 16
+; GCN-NEXT:    v_writelane_b32 v39, s80, 17
+; GCN-NEXT:    v_writelane_b32 v39, s81, 18
+; GCN-NEXT:    v_writelane_b32 v39, s82, 19
+; GCN-NEXT:    v_writelane_b32 v39, s83, 20
+; GCN-NEXT:    v_writelane_b32 v39, s84, 21
+; GCN-NEXT:    v_writelane_b32 v39, s85, 22
+; GCN-NEXT:    v_writelane_b32 v39, s86, 23
+; GCN-NEXT:    v_writelane_b32 v39, s87, 24
+; GCN-NEXT:    v_writelane_b32 v39, s96, 25
+; GCN-NEXT:    v_writelane_b32 v39, s97, 26
+; GCN-NEXT:    v_writelane_b32 v39, s98, 27
+; GCN-NEXT:    v_writelane_b32 v39, s99, 28
+; GCN-NEXT:    v_writelane_b32 v39, s100, 29
+; GCN-NEXT:    v_writelane_b32 v39, s101, 30
 ; GCN-NEXT:    s_addk_i32 s32, 0x6000
+; GCN-NEXT:    v_writelane_b32 v39, s102, 31
 ; GCN-NEXT:    s_mov_b32 s32, s34
+; GCN-NEXT:    v_readlane_b32 s34, v39, 33
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s33 offset:128
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
@@ -541,76 +502,39 @@ define void @no_free_regs_spill_bp_to_memory(<32 x i32> %a, i32 %b) #5 {
 ; GCN-NEXT:    ;;#ASMSTART
 ; GCN-NEXT:    ; clobber all VGPRs
 ; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s33 offset:136 ; 4-byte Folded Reload
-; GCN-NEXT:    v_readlane_b32 s102, v39, 63
-; GCN-NEXT:    v_readlane_b32 s101, v39, 62
-; GCN-NEXT:    v_readlane_b32 s100, v39, 61
-; GCN-NEXT:    v_readlane_b32 s99, v39, 60
-; GCN-NEXT:    v_readlane_b32 s98, v39, 59
-; GCN-NEXT:    v_readlane_b32 s97, v39, 58
-; GCN-NEXT:    v_readlane_b32 s96, v39, 57
-; GCN-NEXT:    v_readlane_b32 s95, v39, 56
-; GCN-NEXT:    v_readlane_b32 s94, v39, 55
-; GCN-NEXT:    v_readlane_b32 s93, v39, 54
-; GCN-NEXT:    v_readlane_b32 s92, v39, 53
-; GCN-NEXT:    v_readlane_b32 s91, v39, 52
-; GCN-NEXT:    v_readlane_b32 s90, v39, 51
-; GCN-NEXT:    v_readlane_b32 s89, v39, 50
-; GCN-NEXT:    v_readlane_b32 s88, v39, 49
-; GCN-NEXT:    v_readlane_b32 s87, v39, 48
-; GCN-NEXT:    v_readlane_b32 s86, v39, 47
-; GCN-NEXT:    v_readlane_b32 s85, v39, 46
-; GCN-NEXT:    v_readlane_b32 s84, v39, 45
-; GCN-NEXT:    v_readlane_b32 s83, v39, 44
-; GCN-NEXT:    v_readlane_b32 s82, v39, 43
-; GCN-NEXT:    v_readlane_b32 s81, v39, 42
-; GCN-NEXT:    v_readlane_b32 s80, v39, 41
-; GCN-NEXT:    v_readlane_b32 s79, v39, 40
-; GCN-NEXT:    v_readlane_b32 s78, v39, 39
-; GCN-NEXT:    v_readlane_b32 s77, v39, 38
-; GCN-NEXT:    v_readlane_b32 s76, v39, 37
-; GCN-NEXT:    v_readlane_b32 s75, v39, 36
-; GCN-NEXT:    v_readlane_b32 s74, v39, 35
-; GCN-NEXT:    v_readlane_b32 s73, v39, 34
-; GCN-NEXT:    v_readlane_b32 s72, v39, 33
-; GCN-NEXT:    v_readlane_b32 s71, v39, 32
-; GCN-NEXT:    v_readlane_b32 s70, v39, 31
-; GCN-NEXT:    v_readlane_b32 s69, v39, 30
-; GCN-NEXT:    v_readlane_b32 s68, v39, 29
-; GCN-NEXT:    v_readlane_b32 s67, v39, 28
-; GCN-NEXT:    v_readlane_b32 s66, v39, 27
-; GCN-NEXT:    v_readlane_b32 s65, v39, 26
-; GCN-NEXT:    v_readlane_b32 s64, v39, 25
-; GCN-NEXT:    v_readlane_b32 s63, v39, 24
-; GCN-NEXT:    v_readlane_b32 s62, v39, 23
-; GCN-NEXT:    v_readlane_b32 s61, v39, 22
-; GCN-NEXT:    v_readlane_b32 s60, v39, 21
-; GCN-NEXT:    v_readlane_b32 s59, v39, 20
-; GCN-NEXT:    v_readlane_b32 s58, v39, 19
-; GCN-NEXT:    v_readlane_b32 s57, v39, 18
-; GCN-NEXT:    v_readlane_b32 s56, v39, 17
-; GCN-NEXT:    v_readlane_b32 s55, v39, 16
-; GCN-NEXT:    v_readlane_b32 s54, v39, 15
-; GCN-NEXT:    v_readlane_b32 s53, v39, 14
-; GCN-NEXT:    v_readlane_b32 s52, v39, 13
-; GCN-NEXT:    v_readlane_b32 s51, v39, 12
-; GCN-NEXT:    v_readlane_b32 s50, v39, 11
-; GCN-NEXT:    v_readlane_b32 s49, v39, 10
-; GCN-NEXT:    v_readlane_b32 s48, v39, 9
-; GCN-NEXT:    v_readlane_b32 s47, v39, 8
-; GCN-NEXT:    v_readlane_b32 s46, v39, 7
-; GCN-NEXT:    v_readlane_b32 s45, v39, 6
-; GCN-NEXT:    v_readlane_b32 s44, v39, 5
-; GCN-NEXT:    v_readlane_b32 s43, v39, 4
-; GCN-NEXT:    v_readlane_b32 s42, v39, 3
-; GCN-NEXT:    v_readlane_b32 s41, v39, 2
-; GCN-NEXT:    v_readlane_b32 s40, v39, 1
+; GCN-NEXT:    v_readlane_b32 s102, v39, 31
+; GCN-NEXT:    v_readlane_b32 s101, v39, 30
+; GCN-NEXT:    v_readlane_b32 s100, v39, 29
+; GCN-NEXT:    v_readlane_b32 s99, v39, 28
+; GCN-NEXT:    v_readlane_b32 s98, v39, 27
+; GCN-NEXT:    v_readlane_b32 s97, v39, 26
+; GCN-NEXT:    v_readlane_b32 s96, v39, 25
+; GCN-NEXT:    v_readlane_b32 s87, v39, 24
+; GCN-NEXT:    v_readlane_b32 s86, v39, 23
+; GCN-NEXT:    v_readlane_b32 s85, v39, 22
+; GCN-NEXT:    v_readlane_b32 s84, v39, 21
+; GCN-NEXT:    v_readlane_b32 s83, v39, 20
+; GCN-NEXT:    v_readlane_b32 s82, v39, 19
+; GCN-NEXT:    v_readlane_b32 s81, v39, 18
+; GCN-NEXT:    v_readlane_b32 s80, v39, 17
+; GCN-NEXT:    v_readlane_b32 s71, v39, 16
+; GCN-NEXT:    v_readlane_b32 s70, v39, 15
+; GCN-NEXT:    v_readlane_b32 s69, v39, 14
+; GCN-NEXT:    v_readlane_b32 s68, v39, 13
+; GCN-NEXT:    v_readlane_b32 s67, v39, 12
+; GCN-NEXT:    v_readlane_b32 s66, v39, 11
+; GCN-NEXT:    v_readlane_b32 s65, v39, 10
+; GCN-NEXT:    v_readlane_b32 s64, v39, 9
+; GCN-NEXT:    v_readlane_b32 s55, v39, 8
+; GCN-NEXT:    v_readlane_b32 s54, v39, 7
+; GCN-NEXT:    v_readlane_b32 s53, v39, 6
+; GCN-NEXT:    v_readlane_b32 s52, v39, 5
+; GCN-NEXT:    v_readlane_b32 s51, v39, 4
+; GCN-NEXT:    v_readlane_b32 s50, v39, 3
+; GCN-NEXT:    v_readlane_b32 s49, v39, 2
+; GCN-NEXT:    v_readlane_b32 s48, v39, 1
 ; GCN-NEXT:    v_readlane_b32 s39, v39, 0
-; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_readfirstlane_b32 s4, v0
-; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s33 offset:140 ; 4-byte Folded Reload
-; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_readfirstlane_b32 s34, v0
+; GCN-NEXT:    v_readlane_b32 s4, v39, 32
 ; GCN-NEXT:    s_xor_saveexec_b64 s[6:7], -1
 ; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s33 offset:132 ; 4-byte Folded Reload
 ; GCN-NEXT:    s_mov_b64 exec, s[6:7]
@@ -653,81 +577,46 @@ define void @spill_bp_to_memory_scratch_reg_needed_mubuf_offset(<32 x i32> %a, i
 ; GCN-NEXT:    s_add_i32 s5, s33, 0x42100
 ; GCN-NEXT:    buffer_store_dword v39, off, s[0:3], s5 ; 4-byte Folded Spill
 ; GCN-NEXT:    s_mov_b64 exec, s[6:7]
-; GCN-NEXT:    v_mov_b32_e32 v0, s4
-; GCN-NEXT:    s_add_i32 s5, s33, 0x42200
-; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s5 ; 4-byte Folded Spill
-; GCN-NEXT:    v_mov_b32_e32 v0, s34
-; GCN-NEXT:    s_add_i32 s5, s33, 0x42300
+; GCN-NEXT:    v_writelane_b32 v39, s4, 32
+; GCN-NEXT:    v_writelane_b32 v39, s34, 33
 ; GCN-NEXT:    s_mov_b32 s34, s32
-; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], s5 ; 4-byte Folded Spill
 ; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s34 offset:4
 ; GCN-NEXT:    v_writelane_b32 v39, s39, 0
-; GCN-NEXT:    v_writelane_b32 v39, s40, 1
-; GCN-NEXT:    v_writelane_b32 v39, s41, 2
-; GCN-NEXT:    v_writelane_b32 v39, s42, 3
-; GCN-NEXT:    v_writelane_b32 v39, s43, 4
-; GCN-NEXT:    v_writelane_b32 v39, s44, 5
-; GCN-NEXT:    v_writelane_b32 v39, s45, 6
-; GCN-NEXT:    v_writelane_b32 v39, s46, 7
-; GCN-NEXT:    v_writelane_b32 v39, s47, 8
-; GCN-NEXT:    v_writelane_b32 v39, s48, 9
-; GCN-NEXT:    v_writelane_b32 v39, s49, 10
-; GCN-NEXT:    v_writelane_b32 v39, s50, 11
-; GCN-NEXT:    v_writelane_b32 v39, s51, 12
-; GCN-NEXT:    v_writelane_b32 v39, s52, 13
-; GCN-NEXT:    v_writelane_b32 v39, s53, 14
-; GCN-NEXT:    v_writelane_b32 v39, s54, 15
-; GCN-NEXT:    v_writelane_b32 v39, s55, 16
-; GCN-NEXT:    v_writelane_b32 v39, s56, 17
-; GCN-NEXT:    v_writelane_b32 v39, s57, 18
-; GCN-NEXT:    v_writelane_b32 v39, s58, 19
-; GCN-NEXT:    v_writelane_b32 v39, s59, 20
-; GCN-NEXT:    v_writelane_b32 v39, s60, 21
-; GCN-NEXT:    v_writelane_b32 v39, s61, 22
-; GCN-NEXT:    v_writelane_b32 v39, s62, 23
-; GCN-NEXT:    v_writelane_b32 v39, s63, 24
-; GCN-NEXT:    v_writelane_b32 v39, s64, 25
-; GCN-NEXT:    v_writelane_b32 v39, s65, 26
-; GCN-NEXT:    v_writelane_b32 v39, s66, 27
-; GCN-NEXT:    v_writelane_b32 v39, s67, 28
-; GCN-NEXT:    v_writelane_b32 v39, s68, 29
-; GCN-NEXT:    v_writelane_b32 v39, s69, 30
-; GCN-NEXT:    v_writelane_b32 v39, s70, 31
-; GCN-NEXT:    v_writelane_b32 v39, s71, 32
-; GCN-NEXT:    v_writelane_b32 v39, s72, 33
-; GCN-NEXT:    v_writelane_b32 v39, s73, 34
-; GCN-NEXT:    v_writelane_b32 v39, s74, 35
-; GCN-NEXT:    v_writelane_b32 v39, s75, 36
-; GCN-NEXT:    v_writelane_b32 v39, s76, 37
-; GCN-NEXT:    v_writelane_b32 v39, s77, 38
-; GCN-NEXT:    v_writelane_b32 v39, s78, 39
-; GCN-NEXT:    v_writelane_b32 v39, s79, 40
-; GCN-NEXT:    v_writelane_b32 v39, s80, 41
-; GCN-NEXT:    v_writelane_b32 v39, s81, 42
-; GCN-NEXT:    v_writelane_b32 v39, s82, 43
-; GCN-NEXT:    v_writelane_b32 v39, s83, 44
-; GCN-NEXT:    v_writelane_b32 v39, s84, 45
-; GCN-NEXT:    v_writelane_b32 v39, s85, 46
-; GCN-NEXT:    v_writelane_b32 v39, s86, 47
-; GCN-NEXT:    v_writelane_b32 v39, s87, 48
-; GCN-NEXT:    v_writelane_b32 v39, s88, 49
-; GCN-NEXT:    v_writelane_b32 v39, s89, 50
-; GCN-NEXT:    v_writelane_b32 v39, s90, 51
-; GCN-NEXT:    v_writelane_b32 v39, s91, 52
-; GCN-NEXT:    v_writelane_b32 v39, s92, 53
-; GCN-NEXT:    v_writelane_b32 v39, s93, 54
-; GCN-NEXT:    v_writelane_b32 v39, s94, 55
-; GCN-NEXT:    v_writelane_b32 v39, s95, 56
-; GCN-NEXT:    v_writelane_b32 v39, s96, 57
-; GCN-NEXT:    v_writelane_b32 v39, s97, 58
-; GCN-NEXT:    v_writelane_b32 v39, s98, 59
-; GCN-NEXT:    v_writelane_b32 v39, s99, 60
-; GCN-NEXT:    v_writelane_b32 v39, s100, 61
-; GCN-NEXT:    v_writelane_b32 v39, s101, 62
+; GCN-NEXT:    v_writelane_b32 v39, s48, 1
+; GCN-NEXT:    v_writelane_b32 v39, s49, 2
+; GCN-NEXT:    v_writelane_b32 v39, s50, 3
+; GCN-NEXT:    v_writelane_b32 v39, s51, 4
+; GCN-NEXT:    v_writelane_b32 v39, s52, 5
+; GCN-NEXT:    v_writelane_b32 v39, s53, 6
+; GCN-NEXT:    v_writelane_b32 v39, s54, 7
+; GCN-NEXT:    v_writelane_b32 v39, s55, 8
+; GCN-NEXT:    v_writelane_b32 v39, s64, 9
+; GCN-NEXT:    v_writelane_b32 v39, s65, 10
+; GCN-NEXT:    v_writelane_b32 v39, s66, 11
+; GCN-NEXT:    v_writelane_b32 v39, s67, 12
+; GCN-NEXT:    v_writelane_b32 v39, s68, 13
+; GCN-NEXT:    v_writelane_b32 v39, s69, 14
+; GCN-NEXT:    v_writelane_b32 v39, s70, 15
+; GCN-NEXT:    v_writelane_b32 v39, s71, 16
+; GCN-NEXT:    v_writelane_b32 v39, s80, 17
+; GCN-NEXT:    v_writelane_b32 v39, s81, 18
+; GCN-NEXT:    v_writelane_b32 v39, s82, 19
+; GCN-NEXT:    v_writelane_b32 v39, s83, 20
+; GCN-NEXT:    v_writelane_b32 v39, s84, 21
+; GCN-NEXT:    v_writelane_b32 v39, s85, 22
+; GCN-NEXT:    v_writelane_b32 v39, s86, 23
+; GCN-NEXT:    v_writelane_b32 v39, s87, 24
+; GCN-NEXT:    v_writelane_b32 v39, s96, 25
+; GCN-NEXT:    v_writelane_b32 v39, s97, 26
+; GCN-NEXT:    v_writelane_b32 v39, s98, 27
+; GCN-NEXT:    v_writelane_b32 v39, s99, 28
+; GCN-NEXT:    v_writelane_b32 v39, s100, 29
+; GCN-NEXT:    v_writelane_b32 v39, s101, 30
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0x1080
-; GCN-NEXT:    v_writelane_b32 v39, s102, 63
 ; GCN-NEXT:    s_add_i32 s32, s32, 0x46000
+; GCN-NEXT:    v_writelane_b32 v39, s102, 31
 ; GCN-NEXT:    s_mov_b32 s32, s34
+; GCN-NEXT:    v_readlane_b32 s34, v39, 33
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_store_dword v0, v1, s[0:3], s33 offen
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
@@ -737,78 +626,39 @@ define void @spill_bp_to_memory_scratch_reg_needed_mubuf_offset(<32 x i32> %a, i
 ; GCN-NEXT:    ;;#ASMSTART
 ; GCN-NEXT:    ; clobber all VGPRs
 ; GCN-NEXT:    ;;#ASMEND
-; GCN-NEXT:    s_add_i32 s5, s33, 0x42200
-; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s5 ; 4-byte Folded Reload
-; GCN-NEXT:    s_add_i32 s5, s33, 0x42300
-; GCN-NEXT:    v_readlane_b32 s102, v39, 63
-; GCN-NEXT:    v_readlane_b32 s101, v39, 62
-; GCN-NEXT:    v_readlane_b32 s100, v39, 61
-; GCN-NEXT:    v_readlane_b32 s99, v39, 60
-; GCN-NEXT:    v_readlane_b32 s98, v39, 59
-; GCN-NEXT:    v_readlane_b32 s97, v39, 58
-; GCN-NEXT:    v_readlane_b32 s96, v39, 57
-; GCN-NEXT:    v_readlane_b32 s95, v39, 56
-; GCN-NEXT:    v_readlane_b32 s94, v39, 55
-; GCN-NEXT:    v_readlane_b32 s93, v39, 54
-; GCN-NEXT:    v_readlane_b32 s92, v39, 53
-; GCN-NEXT:    v_readlane_b32 s91, v39, 52
-; GCN-NEXT:    v_readlane_b32 s90, v39, 51
-; GCN-NEXT:    v_readlane_b32 s89, v39, 50
-; GCN-NEXT:    v_readlane_b32 s88, v39, 49
-; GCN-NEXT:    v_readlane_b32 s87, v39, 48
-; GCN-NEXT:    v_readlane_b32 s86, v39, 47
-; GCN-NEXT:    v_readlane_b32 s85, v39, 46
-; GCN-NEXT:    v_readlane_b32 s84, v39, 45
-; GCN-NEXT:    v_readlane_b32 s83, v39, 44
-; GCN-NEXT:    v_readlane_b32 s82, v39, 43
-; GCN-NEXT:    v_readlane_b32 s81, v39, 42
-; GCN-NEXT:    v_readlane_b32 s80, v39, 41
-; GCN-NEXT:    v_readlane_b32 s79, v39, 40
-; GCN-NEXT:    v_readlane_b32 s78, v39, 39
-; GCN-NEXT:    v_readlane_b32 s77, v39, 38
-; GCN-NEXT:    v_readlane_b32 s76, v39, 37
-; GCN-NEXT:    v_readlane_b32 s75, v39, 36
-; GCN-NEXT:    v_readlane_b32 s74, v39, 35
-; GCN-NEXT:    v_readlane_b32 s73, v39, 34
-; GCN-NEXT:    v_readlane_b32 s72, v39, 33
-; GCN-NEXT:    v_readlane_b32 s71, v39, 32
-; GCN-NEXT:    v_readlane_b32 s70, v39, 31
-; GCN-NEXT:    v_readlane_b32 s69, v39, 30
-; GCN-NEXT:    v_readlane_b32 s68, v39, 29
-; GCN-NEXT:    v_readlane_b32 s67, v39, 28
-; GCN-NEXT:    v_readlane_b32 s66, v39, 27
-; GCN-NEXT:    v_readlane_b32 s65, v39, 26
-; GCN-NEXT:    v_readlane_b32 s64, v39, 25
-; GCN-NEXT:    v_readlane_b32 s63, v39, 24
-; GCN-NEXT:    v_readlane_b32 s62, v39, 23
-; GCN-NEXT:    v_readlane_b32 s61, v39, 22
-; GCN-NEXT:    v_readlane_b32 s60, v39, 21
-; GCN-NEXT:    v_readlane_b32 s59, v39, 20
-; GCN-NEXT:    v_readlane_b32 s58, v39, 19
-; GCN-NEXT:    v_readlane_b32 s57, v39, 18
-; GCN-NEXT:    v_readlane_b32 s56, v39, 17
-; GCN-NEXT:    v_readlane_b32 s55, v39, 16
-; GCN-NEXT:    v_readlane_b32 s54, v39, 15
-; GCN-NEXT:    v_readlane_b32 s53, v39, 14
-; GCN-NEXT:    v_readlane_b32 s52, v39, 13
-; GCN-NEXT:    v_readlane_b32 s51, v39, 12
-; GCN-NEXT:    v_readlane_b32 s50, v39, 11
-; GCN-NEXT:    v_readlane_b32 s49, v39, 10
-; GCN-NEXT:    v_readlane_b32 s48, v39, 9
-; GCN-NEXT:    v_readlane_b32 s47, v39, 8
-; GCN-NEXT:    v_readlane_b32 s46, v39, 7
-; GCN-NEXT:    v_readlane_b32 s45, v39, 6
-; GCN-NEXT:    v_readlane_b32 s44, v39, 5
-; GCN-NEXT:    v_readlane_b32 s43, v39, 4
-; GCN-NEXT:    v_readlane_b32 s42, v39, 3
-; GCN-NEXT:    v_readlane_b32 s41, v39, 2
-; GCN-NEXT:    v_readlane_b32 s40, v39, 1
+; GCN-NEXT:    v_readlane_b32 s102, v39, 31
+; GCN-NEXT:    v_readlane_b32 s101, v39, 30
+; GCN-NEXT:    v_readlane_b32 s100, v39, 29
+; GCN-NEXT:    v_readlane_b32 s99, v39, 28
+; GCN-NEXT:    v_readlane_b32 s98, v39, 27
+; GCN-NEXT:    v_readlane_b32 s97, v39, 26
+; GCN-NEXT:    v_readlane_b32 s96, v39, 25
+; GCN-NEXT:    v_readlane_b32 s87, v39, 24
+; GCN-NEXT:    v_readlane_b32 s86, v39, 23
+; GCN-NEXT:    v_readlane_b32 s85, v39, 22
+; GCN-NEXT:    v_readlane_b32 s84, v39, 21
+; GCN-NEXT:    v_readlane_b32 s83, v39, 20
+; GCN-NEXT:    v_readlane_b32 s82, v39, 19
+; GCN-NEXT:    v_readlane_b32 s81, v39, 18
+; GCN-NEXT:    v_readlane_b32 s80, v39, 17
+; GCN-NEXT:    v_readlane_b32 s71, v39, 16
+; GCN-NEXT:    v_readlane_b32 s70, v39, 15
+; GCN-NEXT:    v_readlane_b32 s69, v39, 14
+; GCN-NEXT:    v_readlane_b32 s68, v39, 13
+; GCN-NEXT:    v_readlane_b32 s67, v39, 12
+; GCN-NEXT:    v_readlane_b32 s66, v39, 11
+; GCN-NEXT:    v_readlane_b32 s65, v39, 10
+; GCN-NEXT:    v_readlane_b32 s64, v39, 9
+; GCN-NEXT:    v_readlane_b32 s55, v39, 8
+; GCN-NEXT:    v_readlane_b32 s54, v39, 7
+; GCN-NEXT:    v_readlane_b32 s53, v39, 6
+; GCN-NEXT:    v_readlane_b32 s52, v39, 5
+; GCN-NEXT:    v_readlane_b32 s51, v39, 4
+; GCN-NEXT:    v_readlane_b32 s50, v39, 3
+; GCN-NEXT:    v_readlane_b32 s49, v39, 2
+; GCN-NEXT:    v_readlane_b32 s48, v39, 1
 ; GCN-NEXT:    v_readlane_b32 s39, v39, 0
-; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_readfirstlane_b32 s4, v0
-; GCN-NEXT:    buffer_load_dword v0, off, s[0:3], s5 ; 4-byte Folded Reload
-; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_readfirstlane_b32 s34, v0
+; GCN-NEXT:    v_readlane_b32 s4, v39, 32
 ; GCN-NEXT:    s_xor_saveexec_b64 s[6:7], -1
 ; GCN-NEXT:    s_add_i32 s5, s33, 0x42100
 ; GCN-NEXT:    buffer_load_dword v39, off, s[0:3], s5 ; 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/AMDGPU/subreg-coalescer-crash.ll b/llvm/test/CodeGen/AMDGPU/subreg-coalescer-crash.ll
index 66c34414b6921..f6d67aa2d5df3 100644
--- a/llvm/test/CodeGen/AMDGPU/subreg-coalescer-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/subreg-coalescer-crash.ll
@@ -7,31 +7,31 @@ entry:
   br i1 undef, label %for.inc.1, label %do.body.preheader
 
 do.body.preheader:                                ; preds = %entry
-  %tmp = insertelement <4 x i32> zeroinitializer, i32 undef, i32 1
+  %tmp = insertelement <4 x i32> zeroinitializer, i32 poison, i32 1
   br i1 undef, label %do.body56.1, label %do.body90
 
 do.body90:                                        ; preds = %do.body56.2, %do.body56.1, %do.body.preheader
   %tmp1 = phi <4 x i32> [ %tmp6, %do.body56.2 ], [ %tmp5, %do.body56.1 ], [ %tmp, %do.body.preheader ]
-  %tmp2 = insertelement <4 x i32> %tmp1, i32 undef, i32 2
-  %tmp3 = insertelement <4 x i32> %tmp2, i32 undef, i32 3
+  %tmp2 = insertelement <4 x i32> %tmp1, i32 poison, i32 2
+  %tmp3 = insertelement <4 x i32> %tmp2, i32 poison, i32 3
   br i1 undef, label %do.body124.1, label %do.body.1562.preheader
 
 do.body.1562.preheader:                           ; preds = %do.body124.1, %do.body90
   %storemerge = phi <4 x i32> [ %tmp3, %do.body90 ], [ %tmp7, %do.body124.1 ]
-  %tmp4 = insertelement <4 x i32> undef, i32 undef, i32 1
+  %tmp4 = insertelement <4 x i32> poison, i32 poison, i32 1
   br label %for.inc.1
 
 do.body56.1:                                      ; preds = %do.body.preheader
-  %tmp5 = insertelement <4 x i32> %tmp, i32 undef, i32 1
+  %tmp5 = insertelement <4 x i32> %tmp, i32 poison, i32 1
   %or.cond472.1 = or i1 undef, undef
   br i1 %or.cond472.1, label %do.body56.2, label %do.body90
 
 do.body56.2:                                      ; preds = %do.body56.1
-  %tmp6 = insertelement <4 x i32> %tmp5, i32 undef, i32 1
+  %tmp6 = insertelement <4 x i32> %tmp5, i32 poison, i32 1
   br label %do.body90
 
 do.body124.1:                                     ; preds = %do.body90
-  %tmp7 = insertelement <4 x i32> %tmp3, i32 undef, i32 3
+  %tmp7 = insertelement <4 x i32> %tmp3, i32 poison, i32 3
   br label %do.body.1562.preheader
 
 for.inc.1:                                        ; preds = %do.body.1562.preheader, %entry
@@ -54,7 +54,7 @@ bb2:                                              ; preds = %bb4, %bb
   br i1 undef, label %bb9, label %bb13
 
 bb4:                                              ; preds = %bb7, %bb6, %bb1
-  %tmp5 = phi float [ undef, %bb1 ], [ undef, %bb6 ], [ %tmp8, %bb7 ]
+  %tmp5 = phi float [ poison, %bb1 ], [ poison, %bb6 ], [ %tmp8, %bb7 ]
   br label %bb2
 
 bb6:                                              ; preds = %bb1
@@ -74,8 +74,8 @@ bb13:                                             ; preds = %bb2
   br i1 undef, label %bb23, label %bb24
 
 bb14:                                             ; preds = %bb27, %bb24, %bb9
-  %tmp15 = phi float [ %tmp12, %bb9 ], [ undef, %bb27 ], [ 0.000000e+00, %bb24 ]
-  %tmp16 = phi float [ %tmp11, %bb9 ], [ undef, %bb27 ], [ %tmp25, %bb24 ]
+  %tmp15 = phi float [ %tmp12, %bb9 ], [ poison, %bb27 ], [ 0.000000e+00, %bb24 ]
+  %tmp16 = phi float [ %tmp11, %bb9 ], [ poison, %bb27 ], [ %tmp25, %bb24 ]
   %tmp17 = fmul float 1.050000e+01, %tmp16
   %tmp18 = fmul float 1.150000e+01, %tmp15
   call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp18, float %tmp17, float %tmp17, float %tmp17, i1 true, i1 true) #0
diff --git a/llvm/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll b/llvm/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll
index d4329aec2021c..4621be5cab450 100644
--- a/llvm/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll
+++ b/llvm/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll
@@ -40,13 +40,13 @@ define amdgpu_kernel void @foobar(float %a0, float %a1, ptr addrspace(1) %out) #
 ; Such a copies appear because the float4 vectors and their elements in the test are uniform
 ; but the PHI node in "ife" block is divergent because of the CF dependency (divergent branch in bb0)
 entry:
-  %v0 = insertelement <4 x float> undef, float %a0, i32 0
+  %v0 = insertelement <4 x float> poison, float %a0, i32 0
   %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0
   %cnd = icmp eq i32 %tid, 0
   br i1 %cnd, label %ift, label %ife
 
 ift:
-  %v1 = insertelement <4 x float> undef, float %a1, i32 0
+  %v1 = insertelement <4 x float> poison, float %a1, i32 0
   br label %ife
 
 ife:
diff --git a/llvm/test/CodeGen/AMDGPU/subreg-eliminate-dead.ll b/llvm/test/CodeGen/AMDGPU/subreg-eliminate-dead.ll
index f79ca18e7672d..b60eb219081e0 100644
--- a/llvm/test/CodeGen/AMDGPU/subreg-eliminate-dead.ll
+++ b/llvm/test/CodeGen/AMDGPU/subreg-eliminate-dead.ll
@@ -11,7 +11,7 @@ define amdgpu_kernel void @foobar() {
   %v4 = extractelement <4 x i32> %v3, i32 1
   %v5 = icmp ne i32 %v4, 0
   %v6 = select i1 %v5, i32 undef, i32 0
-  %v15 = insertelement <2 x i32> undef, i32 %v6, i32 1
+  %v15 = insertelement <2 x i32> poison, i32 %v6, i32 1
   store <2 x i32> %v15, ptr addrspace(1) undef, align 8
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/switch-default-block-unreachable.ll b/llvm/test/CodeGen/AMDGPU/switch-default-block-unreachable.ll
index a5e14234c3d41..6bd9f1cc70b41 100644
--- a/llvm/test/CodeGen/AMDGPU/switch-default-block-unreachable.ll
+++ b/llvm/test/CodeGen/AMDGPU/switch-default-block-unreachable.ll
@@ -47,7 +47,7 @@ define void @test(i1 %c0) #1 {
     br label %unreach.blk
 
   unreach.blk:                                      ; preds = %preheader.blk, %pre.false.blk
-    %phi.val = phi i32 [ %call.pre.false, %pre.false.blk ], [ undef, %preheader.blk ]
+    %phi.val = phi i32 [ %call.pre.false, %pre.false.blk ], [ poison, %preheader.blk ]
     store i32 %phi.val, ptr undef
     unreachable
 
diff --git a/llvm/test/CodeGen/AMDGPU/swizzle-export.ll b/llvm/test/CodeGen/AMDGPU/swizzle-export.ll
index 1adf549c76767..8d5b6568340db 100644
--- a/llvm/test/CodeGen/AMDGPU/swizzle-export.ll
+++ b/llvm/test/CodeGen/AMDGPU/swizzle-export.ll
@@ -64,27 +64,27 @@ main_body:
   %53 = load <4 x float>, ptr addrspace(8) null
   %54 = extractelement <4 x float> %53, i32 0
   %55 = fmul float 1.000000e+00, %54
-  %56 = insertelement <4 x float> undef, float %0, i32 0
+  %56 = insertelement <4 x float> poison, float %0, i32 0
   %57 = insertelement <4 x float> %56, float %1, i32 1
   %58 = insertelement <4 x float> %57, float %2, i32 2
   %59 = insertelement <4 x float> %58, float %3, i32 3
   call void @llvm.r600.store.swizzle(<4 x float> %59, i32 60, i32 1)
-  %60 = insertelement <4 x float> undef, float %10, i32 0
+  %60 = insertelement <4 x float> poison, float %10, i32 0
   %61 = insertelement <4 x float> %60, float %13, i32 1
   %62 = insertelement <4 x float> %61, float %16, i32 2
   %63 = insertelement <4 x float> %62, float %19, i32 3
   call void @llvm.r600.store.swizzle(<4 x float> %63, i32 0, i32 2)
-  %64 = insertelement <4 x float> undef, float %22, i32 0
+  %64 = insertelement <4 x float> poison, float %22, i32 0
   %65 = insertelement <4 x float> %64, float %25, i32 1
   %66 = insertelement <4 x float> %65, float %28, i32 2
   %67 = insertelement <4 x float> %66, float %31, i32 3
   call void @llvm.r600.store.swizzle(<4 x float> %67, i32 1, i32 2)
-  %68 = insertelement <4 x float> undef, float %34, i32 0
+  %68 = insertelement <4 x float> poison, float %34, i32 0
   %69 = insertelement <4 x float> %68, float %37, i32 1
   %70 = insertelement <4 x float> %69, float %40, i32 2
   %71 = insertelement <4 x float> %70, float %43, i32 3
   call void @llvm.r600.store.swizzle(<4 x float> %71, i32 2, i32 2)
-  %72 = insertelement <4 x float> undef, float %46, i32 0
+  %72 = insertelement <4 x float> poison, float %46, i32 0
   %73 = insertelement <4 x float> %72, float %49, i32 1
   %74 = insertelement <4 x float> %73, float %52, i32 2
   %75 = insertelement <4 x float> %74, float %55, i32 3
@@ -109,10 +109,10 @@ main_body:
   %8 = extractelement <4 x float> %7, i32 0
   %9 = load <4 x float>, ptr addrspace(8) null
   %10 = extractelement <4 x float> %9, i32 1
-  %11 = insertelement <4 x float> undef, float %2, i32 0
+  %11 = insertelement <4 x float> poison, float %2, i32 0
   %12 = insertelement <4 x float> %11, float %3, i32 1
   call void @llvm.r600.store.swizzle(<4 x float> %12, i32 60, i32 1)
-  %13 = insertelement <4 x float> undef, float %6, i32 0
+  %13 = insertelement <4 x float> poison, float %6, i32 0
   %14 = insertelement <4 x float> %13, float %8, i32 1
   %15 = insertelement <4 x float> %14, float %10, i32 2
   %16 = insertelement <4 x float> %15, float 0.000000e+00, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/tex-clause-antidep.ll b/llvm/test/CodeGen/AMDGPU/tex-clause-antidep.ll
index 6e906446ccc62..63dd9f7bbc4c2 100644
--- a/llvm/test/CodeGen/AMDGPU/tex-clause-antidep.ll
+++ b/llvm/test/CodeGen/AMDGPU/tex-clause-antidep.ll
@@ -8,7 +8,7 @@ define amdgpu_vs void @test(<4 x float> inreg %reg0) {
   %2 = extractelement <4 x float> %reg0, i32 1
   %3 = extractelement <4 x float> %reg0, i32 2
   %4 = extractelement <4 x float> %reg0, i32 3
-  %5 = insertelement <4 x float> undef, float %1, i32 0
+  %5 = insertelement <4 x float> poison, float %1, i32 0
   %6 = insertelement <4 x float> %5, float %2, i32 1
   %7 = insertelement <4 x float> %6, float %3, i32 2
   %8 = insertelement <4 x float> %7, float %4, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/texture-input-merge.ll b/llvm/test/CodeGen/AMDGPU/texture-input-merge.ll
index 92aeb6bd428ef..26dce4fe7f993 100644
--- a/llvm/test/CodeGen/AMDGPU/texture-input-merge.ll
+++ b/llvm/test/CodeGen/AMDGPU/texture-input-merge.ll
@@ -11,11 +11,11 @@ define amdgpu_vs void @test(<4 x float> inreg %reg0) {
   %6 = fmul float %2, 3.0
   %7 = fmul float %3, 3.0
   %8 = fmul float %4, 3.0
-  %9 = insertelement <4 x float> undef, float %5, i32 0
+  %9 = insertelement <4 x float> poison, float %5, i32 0
   %10 = insertelement <4 x float> %9, float %6, i32 1
-  %11 = insertelement <4 x float> undef, float %7, i32 0
+  %11 = insertelement <4 x float> poison, float %7, i32 0
   %12 = insertelement <4 x float> %11, float %5, i32 1
-  %13 = insertelement <4 x float> undef, float %8, i32 0
+  %13 = insertelement <4 x float> poison, float %8, i32 0
   %14 = call <4 x float> @llvm.r600.tex(<4 x float> %10, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
   %15 = call <4 x float> @llvm.r600.tex(<4 x float> %12, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
   %16 = call <4 x float> @llvm.r600.tex(<4 x float> %13, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/trunc-combine.ll b/llvm/test/CodeGen/AMDGPU/trunc-combine.ll
index 15d172eb17688..0c2adefd0be7c 100644
--- a/llvm/test/CodeGen/AMDGPU/trunc-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/trunc-combine.ll
@@ -45,7 +45,7 @@ define i16 @trunc_bitcast_v2i32_to_i16(<2 x i32> %bar) {
 ; VI-NEXT:    s_setpc_b64 s[30:31]
   %load0 = load i32, ptr addrspace(1) undef
   %load1 = load i32, ptr addrspace(1) null
-  %insert.0 = insertelement <2 x i32> undef, i32 %load0, i32 0
+  %insert.0 = insertelement <2 x i32> poison, i32 %load0, i32 0
   %insert.1 = insertelement <2 x i32> %insert.0, i32 99, i32 1
   %bc = bitcast <2 x i32> %insert.1 to i64
   %trunc = trunc i64 %bc to i16
@@ -74,7 +74,7 @@ define i16 @trunc_bitcast_v2f32_to_i16(<2 x float> %bar) {
 ; VI-NEXT:    s_setpc_b64 s[30:31]
   %load0 = load float, ptr addrspace(1) undef
   %load1 = load float, ptr addrspace(1) null
-  %insert.0 = insertelement <2 x float> undef, float %load0, i32 0
+  %insert.0 = insertelement <2 x float> poison, float %load0, i32 0
   %insert.1 = insertelement <2 x float> %insert.0, float 4.0, i32 1
   %bc = bitcast <2 x float> %insert.1 to i64
   %trunc = trunc i64 %bc to i16
@@ -128,8 +128,8 @@ bb:
   %tmp8 = extractelement <2 x i32> %tmp6, i64 0
   %tmp9 = extractelement <2 x i32> %tmp7, i64 0
   %tmp10 = mul nsw i32 %tmp9, %tmp8
-  %tmp11 = insertelement <2 x i32> undef, i32 %tmp10, i32 0
-  %tmp12 = insertelement <2 x i32> %tmp11, i32 undef, i32 1
+  %tmp11 = insertelement <2 x i32> poison, i32 %tmp10, i32 0
+  %tmp12 = insertelement <2 x i32> %tmp11, i32 poison, i32 1
   %tmp13 = lshr <2 x i32> %tmp12, <i32 16, i32 16>
   %tmp14 = trunc <2 x i32> %tmp13 to <2 x i16>
   %tmp15 = getelementptr inbounds <2 x i16>, ptr addrspace(1) %arg2, i64 undef
diff --git a/llvm/test/CodeGen/AMDGPU/trunc-store-vec-i16-to-i8.ll b/llvm/test/CodeGen/AMDGPU/trunc-store-vec-i16-to-i8.ll
index 7cb30fc8247e9..18bcd6cf880a4 100644
--- a/llvm/test/CodeGen/AMDGPU/trunc-store-vec-i16-to-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/trunc-store-vec-i16-to-i8.ll
@@ -15,8 +15,8 @@ entry:
 define protected amdgpu_kernel void @short2_char4(ptr addrspace(1) %out) {
 entry:
   %tmp = load <2 x i16>, ptr addrspace(1) undef, align 4
-  %vecinit = shufflevector <2 x i16> %tmp, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-  %vecinit2 = shufflevector <4 x i16> %vecinit, <4 x i16> <i16 undef, i16 undef, i16 0, i16 0>, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  %vecinit = shufflevector <2 x i16> %tmp, <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %vecinit2 = shufflevector <4 x i16> %vecinit, <4 x i16> <i16 poison, i16 poison, i16 0, i16 0>, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
   %tmp1 = trunc <4 x i16> %vecinit2 to <4 x i8>
   store <4 x i8> %tmp1, ptr addrspace(1) %out, align 4
   ret void
@@ -27,8 +27,8 @@ entry:
 define protected amdgpu_kernel void @short4_char8(ptr addrspace(1) %out) {
 entry:
   %tmp = load <4 x i16>, ptr addrspace(1) undef, align 8
-  %vecinit = shufflevector <4 x i16> %tmp, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-  %vecinit2 = shufflevector <8 x i16> %vecinit, <8 x i16> <i16 undef, i16 undef, i16 undef, i16 undef, i16 0, i16 0, i16 0, i16 0>, <8 x i32> <i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7>
+  %vecinit = shufflevector <4 x i16> %tmp, <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+  %vecinit2 = shufflevector <8 x i16> %vecinit, <8 x i16> <i16 poison, i16 poison, i16 poison, i16 poison, i16 0, i16 0, i16 0, i16 0>, <8 x i32> <i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7>
   %tmp1 = trunc <8 x i16> %vecinit2 to <8 x i8>
   store <8 x i8> %tmp1, ptr addrspace(1) %out, align 8
   ret void
@@ -39,8 +39,8 @@ entry:
 define protected amdgpu_kernel void @short8_char16(ptr addrspace(1) %out) {
 entry:
   %tmp = load <8 x i16>, ptr addrspace(1) undef, align 16
-  %vecinit = shufflevector <8 x i16> %tmp, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-  %vecinit2 = shufflevector <16 x i16> %vecinit, <16 x i16> <i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, <16 x i32> <i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7>
+  %vecinit = shufflevector <8 x i16> %tmp, <8 x i16> poison, <16 x i32> <i32 0, i32 1, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %vecinit2 = shufflevector <16 x i16> %vecinit, <16 x i16> <i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, <16 x i32> <i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7>
   %tmp1 = trunc <16 x i16> %vecinit2 to <16 x i8>
   store <16 x i8> %tmp1, ptr addrspace(1) %out, align 16
   ret void
@@ -52,8 +52,8 @@ entry:
 define protected amdgpu_kernel void @short16_char32(ptr addrspace(1) %out) {
 entry:
   %tmp = load <16 x i16>, ptr addrspace(1) undef, align 32
-  %vecinit = shufflevector <16 x i16> %tmp, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-  %vecinit2 = shufflevector <32 x i16> %vecinit, <32 x i16> <i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 0, i16 1, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, <32 x i32> <i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7>
+  %vecinit = shufflevector <16 x i16> %tmp, <16 x i16> poison, <32 x i32> <i32 0, i32 1, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %vecinit2 = shufflevector <32 x i16> %vecinit, <32 x i16> <i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 0, i16 1, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 poison, i16 poison, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, <32 x i32> <i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7, i32 0, i32 1, i32 6, i32 7>
   %tmp1 = trunc <32 x i16> %vecinit2 to <32 x i8>
   store <32 x i8> %tmp1, ptr addrspace(1) %out, align 32
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll b/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
index 54b1d1e6f3f43..474482b2d89ff 100644
--- a/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
+++ b/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
@@ -32,14 +32,14 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS1-LABEL: kernel:
 ; GLOBALNESS1:       ; %bb.0: ; %bb
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GLOBALNESS1-NEXT:    s_load_dwordx4 s[76:79], s[8:9], 0x0
+; GLOBALNESS1-NEXT:    s_load_dwordx4 s[52:55], s[8:9], 0x0
 ; GLOBALNESS1-NEXT:    s_load_dword s6, s[8:9], 0x14
 ; GLOBALNESS1-NEXT:    v_mov_b32_e32 v42, 0
 ; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[44:45], 0, 0
 ; GLOBALNESS1-NEXT:    global_store_dword v[44:45], v42, off
 ; GLOBALNESS1-NEXT:    s_waitcnt lgkmcnt(0)
-; GLOBALNESS1-NEXT:    global_load_dword v2, v42, s[76:77]
-; GLOBALNESS1-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; GLOBALNESS1-NEXT:    global_load_dword v2, v42, s[52:53]
+; GLOBALNESS1-NEXT:    s_mov_b64 s[48:49], s[4:5]
 ; GLOBALNESS1-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x18
 ; GLOBALNESS1-NEXT:    s_load_dword s7, s[8:9], 0x20
 ; GLOBALNESS1-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
@@ -49,7 +49,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS1-NEXT:    v_mov_b32_e32 v0, 0
 ; GLOBALNESS1-NEXT:    s_addc_u32 s1, s1, 0
 ; GLOBALNESS1-NEXT:    v_mov_b32_e32 v1, 0x40994400
-; GLOBALNESS1-NEXT:    s_bitcmp1_b32 s78, 0
+; GLOBALNESS1-NEXT:    s_bitcmp1_b32 s54, 0
 ; GLOBALNESS1-NEXT:    s_waitcnt lgkmcnt(0)
 ; GLOBALNESS1-NEXT:    v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
 ; GLOBALNESS1-NEXT:    v_cmp_ngt_f64_e64 s[4:5], s[4:5], 0
@@ -59,25 +59,28 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS1-NEXT:    s_xor_b64 s[4:5], s[4:5], -1
 ; GLOBALNESS1-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
 ; GLOBALNESS1-NEXT:    s_bitcmp1_b32 s6, 0
-; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[42:43], 1, v0
+; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[50:51], 1, v0
 ; GLOBALNESS1-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
 ; GLOBALNESS1-NEXT:    s_cselect_b64 s[4:5], -1, 0
 ; GLOBALNESS1-NEXT:    s_xor_b64 s[4:5], s[4:5], -1
 ; GLOBALNESS1-NEXT:    s_bitcmp1_b32 s7, 0
-; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[48:49], 1, v0
+; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[64:65], 1, v0
 ; GLOBALNESS1-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
 ; GLOBALNESS1-NEXT:    s_cselect_b64 s[4:5], -1, 0
 ; GLOBALNESS1-NEXT:    s_xor_b64 s[4:5], s[4:5], -1
-; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[50:51], 1, v0
+; GLOBALNESS1-NEXT:    s_mov_b64 s[38:39], s[8:9]
+; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[8:9], 1, v1
+; GLOBALNESS1-NEXT:    ; implicit-def: $vgpr59 : SGPR spill to VGPR lane
+; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[66:67], 1, v0
 ; GLOBALNESS1-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[52:53], 1, v0
-; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[44:45], 1, v1
-; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[46:47], 1, v3
+; GLOBALNESS1-NEXT:    v_writelane_b32 v59, s8, 0
+; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[68:69], 1, v0
+; GLOBALNESS1-NEXT:    v_writelane_b32 v59, s9, 1
+; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[70:71], 1, v3
 ; GLOBALNESS1-NEXT:    v_mov_b32_e32 v46, 0x80
-; GLOBALNESS1-NEXT:    s_mov_b32 s70, s16
-; GLOBALNESS1-NEXT:    s_mov_b64 s[38:39], s[8:9]
-; GLOBALNESS1-NEXT:    s_mov_b32 s71, s15
-; GLOBALNESS1-NEXT:    s_mov_b32 s72, s14
+; GLOBALNESS1-NEXT:    s_mov_b32 s82, s16
+; GLOBALNESS1-NEXT:    s_mov_b32 s83, s15
+; GLOBALNESS1-NEXT:    s_mov_b32 s84, s14
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; GLOBALNESS1-NEXT:    v_mov_b32_e32 v47, 0
 ; GLOBALNESS1-NEXT:    s_mov_b32 s32, 0
@@ -88,26 +91,35 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS1-NEXT:    v_cmp_gt_i32_e32 vcc, 1, v2
 ; GLOBALNESS1-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
 ; GLOBALNESS1-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
+; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, v0
 ; GLOBALNESS1-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; GLOBALNESS1-NEXT:    v_writelane_b32 v59, s4, 2
 ; GLOBALNESS1-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GLOBALNESS1-NEXT:    v_writelane_b32 v59, s5, 3
+; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, v3
 ; GLOBALNESS1-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[54:55], 1, v0
-; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[56:57], 1, v1
-; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[58:59], 1, v3
-; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[60:61], 1, v2
+; GLOBALNESS1-NEXT:    v_writelane_b32 v59, s4, 4
+; GLOBALNESS1-NEXT:    v_writelane_b32 v59, s5, 5
+; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, v2
+; GLOBALNESS1-NEXT:    v_writelane_b32 v59, s4, 6
+; GLOBALNESS1-NEXT:    v_writelane_b32 v59, s5, 7
+; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[80:81], 1, v1
+; GLOBALNESS1-NEXT:    v_writelane_b32 v59, s70, 8
+; GLOBALNESS1-NEXT:    v_writelane_b32 v59, s71, 9
 ; GLOBALNESS1-NEXT:    s_branch .LBB1_4
 ; GLOBALNESS1-NEXT:  .LBB1_1: ; %bb70.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[60:61]
+; GLOBALNESS1-NEXT:    v_readlane_b32 s6, v59, 6
+; GLOBALNESS1-NEXT:    v_readlane_b32 s7, v59, 7
+; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[6:7]
 ; GLOBALNESS1-NEXT:    s_cbranch_vccz .LBB1_28
 ; GLOBALNESS1-NEXT:  .LBB1_2: ; %Flow15
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
 ; GLOBALNESS1-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GLOBALNESS1-NEXT:    s_mov_b64 s[8:9], 0
-; GLOBALNESS1-NEXT:    ; implicit-def: $sgpr4_sgpr5
+; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], 0
 ; GLOBALNESS1-NEXT:  .LBB1_3: ; %Flow28
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[8:9]
+; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[6:7]
 ; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[56:57], v[0:1], v[0:1] op_sel:[0,1]
 ; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_29
 ; GLOBALNESS1-NEXT:  .LBB1_4: ; %bb5
@@ -122,57 +134,63 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS1-NEXT:    s_add_u32 s4, s4, wobble@gotpcrel32@lo+4
 ; GLOBALNESS1-NEXT:    s_addc_u32 s5, s5, wobble@gotpcrel32@hi+12
 ; GLOBALNESS1-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
-; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS1-NEXT:    s_mov_b32 s12, s72
-; GLOBALNESS1-NEXT:    s_mov_b32 s13, s71
-; GLOBALNESS1-NEXT:    s_mov_b32 s14, s70
+; GLOBALNESS1-NEXT:    s_mov_b32 s12, s84
+; GLOBALNESS1-NEXT:    s_mov_b32 s13, s83
+; GLOBALNESS1-NEXT:    s_mov_b32 s14, s82
 ; GLOBALNESS1-NEXT:    v_mov_b32_e32 v31, v41
 ; GLOBALNESS1-NEXT:    s_waitcnt lgkmcnt(0)
 ; GLOBALNESS1-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[46:47]
-; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], -1
-; GLOBALNESS1-NEXT:    ; implicit-def: $sgpr4_sgpr5
+; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[70:71]
+; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], -1
+; GLOBALNESS1-NEXT:    ; implicit-def: $sgpr8_sgpr9
 ; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_9
 ; GLOBALNESS1-NEXT:  ; %bb.5: ; %NodeBlock
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    s_cmp_lt_i32 s79, 1
+; GLOBALNESS1-NEXT:    s_mov_b64 s[8:9], -1
+; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], 0
+; GLOBALNESS1-NEXT:    s_cmp_lt_i32 s55, 1
+; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], -1
 ; GLOBALNESS1-NEXT:    s_cbranch_scc1 .LBB1_7
 ; GLOBALNESS1-NEXT:  ; %bb.6: ; %LeafBlock12
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s79, 1
-; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], -1
-; GLOBALNESS1-NEXT:    s_cselect_b64 s[6:7], -1, 0
-; GLOBALNESS1-NEXT:    s_cbranch_execz .LBB1_8
-; GLOBALNESS1-NEXT:    s_branch .LBB1_9
-; GLOBALNESS1-NEXT:  .LBB1_7: ; in Loop: Header=BB1_4 Depth=1
+; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s55, 1
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], 0
-; GLOBALNESS1-NEXT:    ; implicit-def: $sgpr4_sgpr5
-; GLOBALNESS1-NEXT:  .LBB1_8: ; %LeafBlock
+; GLOBALNESS1-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; GLOBALNESS1-NEXT:  .LBB1_7: ; %Flow26
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s79, 0
-; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], 0
-; GLOBALNESS1-NEXT:    s_cselect_b64 s[6:7], -1, 0
+; GLOBALNESS1-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
+; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_9
+; GLOBALNESS1-NEXT:  ; %bb.8: ; %LeafBlock
+; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
+; GLOBALNESS1-NEXT:    s_cmp_lg_u32 s55, 0
+; GLOBALNESS1-NEXT:    s_mov_b64 s[8:9], 0
+; GLOBALNESS1-NEXT:    s_cselect_b64 s[4:5], -1, 0
 ; GLOBALNESS1-NEXT:  .LBB1_9: ; %Flow25
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    s_mov_b64 s[8:9], -1
-; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[6:7]
+; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], -1
+; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[4:5]
 ; GLOBALNESS1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GLOBALNESS1-NEXT:    s_cbranch_vccz .LBB1_3
 ; GLOBALNESS1-NEXT:  ; %bb.10: ; %baz.exit.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
 ; GLOBALNESS1-NEXT:    flat_load_dword v0, v[44:45]
 ; GLOBALNESS1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GLOBALNESS1-NEXT:    v_cmp_gt_i32_e64 s[62:63], 0, v0
+; GLOBALNESS1-NEXT:    v_cmp_gt_i32_e64 s[86:87], 0, v0
 ; GLOBALNESS1-NEXT:    v_mov_b32_e32 v0, 0
 ; GLOBALNESS1-NEXT:    v_mov_b32_e32 v1, 0x3ff00000
-; GLOBALNESS1-NEXT:    s_and_saveexec_b64 s[74:75], s[62:63]
+; GLOBALNESS1-NEXT:    s_and_saveexec_b64 s[52:53], s[86:87]
 ; GLOBALNESS1-NEXT:    s_cbranch_execz .LBB1_25
 ; GLOBALNESS1-NEXT:  ; %bb.11: ; %bb33.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
 ; GLOBALNESS1-NEXT:    global_load_dwordx2 v[0:1], v[44:45], off
-; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[54:55]
+; GLOBALNESS1-NEXT:    v_writelane_b32 v59, s8, 10
+; GLOBALNESS1-NEXT:    v_writelane_b32 v59, s9, 11
+; GLOBALNESS1-NEXT:    v_readlane_b32 s4, v59, 2
+; GLOBALNESS1-NEXT:    v_readlane_b32 s5, v59, 3
+; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[4:5]
 ; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_13
 ; GLOBALNESS1-NEXT:  ; %bb.12: ; %bb39.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
@@ -185,69 +203,71 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS1-NEXT:    s_waitcnt vmcnt(0)
 ; GLOBALNESS1-NEXT:    v_cmp_nlt_f64_e32 vcc, 0, v[0:1]
 ; GLOBALNESS1-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; GLOBALNESS1-NEXT:    v_cmp_eq_u32_e64 s[64:65], 0, v2
-; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[66:67], 1, v0
+; GLOBALNESS1-NEXT:    v_cmp_eq_u32_e64 s[96:97], 0, v2
+; GLOBALNESS1-NEXT:    v_cmp_ne_u32_e64 s[98:99], 1, v0
 ; GLOBALNESS1-NEXT:    s_branch .LBB1_16
 ; GLOBALNESS1-NEXT:  .LBB1_14: ; %Flow16
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_16 Depth=2
 ; GLOBALNESS1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GLOBALNESS1-NEXT:  .LBB1_15: ; %bb63.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[52:53]
+; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[68:69]
 ; GLOBALNESS1-NEXT:    s_cbranch_vccz .LBB1_24
 ; GLOBALNESS1-NEXT:  .LBB1_16: ; %bb44.i
 ; GLOBALNESS1-NEXT:    ; Parent Loop BB1_4 Depth=1
 ; GLOBALNESS1-NEXT:    ; => This Inner Loop Header: Depth=2
-; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[48:49]
+; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[64:65]
 ; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_15
 ; GLOBALNESS1-NEXT:  ; %bb.17: ; %bb46.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[50:51]
+; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[66:67]
 ; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_15
 ; GLOBALNESS1-NEXT:  ; %bb.18: ; %bb50.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[42:43]
+; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[50:51]
 ; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_21
 ; GLOBALNESS1-NEXT:  ; %bb.19: ; %bb3.i.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[44:45]
+; GLOBALNESS1-NEXT:    v_readlane_b32 s4, v59, 0
+; GLOBALNESS1-NEXT:    v_readlane_b32 s5, v59, 1
+; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[4:5]
 ; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_21
 ; GLOBALNESS1-NEXT:  ; %bb.20: ; %bb6.i.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[66:67]
+; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[98:99]
 ; GLOBALNESS1-NEXT:  .LBB1_21: ; %spam.exit.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[56:57]
+; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[80:81]
 ; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_15
 ; GLOBALNESS1-NEXT:  ; %bb.22: ; %bb55.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS1-NEXT:    s_add_u32 s68, s38, 40
-; GLOBALNESS1-NEXT:    s_addc_u32 s69, s39, 0
+; GLOBALNESS1-NEXT:    s_add_u32 s70, s38, 40
+; GLOBALNESS1-NEXT:    s_addc_u32 s71, s39, 0
 ; GLOBALNESS1-NEXT:    s_getpc_b64 s[4:5]
 ; GLOBALNESS1-NEXT:    s_add_u32 s4, s4, wobble@gotpcrel32@lo+4
 ; GLOBALNESS1-NEXT:    s_addc_u32 s5, s5, wobble@gotpcrel32@hi+12
-; GLOBALNESS1-NEXT:    s_load_dwordx2 s[76:77], s[4:5], 0x0
-; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS1-NEXT:    s_load_dwordx2 s[54:55], s[4:5], 0x0
+; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], s[36:37]
-; GLOBALNESS1-NEXT:    s_mov_b64 s[8:9], s[68:69]
+; GLOBALNESS1-NEXT:    s_mov_b64 s[8:9], s[70:71]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS1-NEXT:    s_mov_b32 s12, s72
-; GLOBALNESS1-NEXT:    s_mov_b32 s13, s71
-; GLOBALNESS1-NEXT:    s_mov_b32 s14, s70
+; GLOBALNESS1-NEXT:    s_mov_b32 s12, s84
+; GLOBALNESS1-NEXT:    s_mov_b32 s13, s83
+; GLOBALNESS1-NEXT:    s_mov_b32 s14, s82
 ; GLOBALNESS1-NEXT:    v_mov_b32_e32 v31, v41
 ; GLOBALNESS1-NEXT:    s_waitcnt lgkmcnt(0)
-; GLOBALNESS1-NEXT:    s_swappc_b64 s[30:31], s[76:77]
-; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS1-NEXT:    s_swappc_b64 s[30:31], s[54:55]
+; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], s[36:37]
-; GLOBALNESS1-NEXT:    s_mov_b64 s[8:9], s[68:69]
+; GLOBALNESS1-NEXT:    s_mov_b64 s[8:9], s[70:71]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS1-NEXT:    s_mov_b32 s12, s72
-; GLOBALNESS1-NEXT:    s_mov_b32 s13, s71
-; GLOBALNESS1-NEXT:    s_mov_b32 s14, s70
+; GLOBALNESS1-NEXT:    s_mov_b32 s12, s84
+; GLOBALNESS1-NEXT:    s_mov_b32 s13, s83
+; GLOBALNESS1-NEXT:    s_mov_b32 s14, s82
 ; GLOBALNESS1-NEXT:    v_mov_b32_e32 v31, v41
 ; GLOBALNESS1-NEXT:    global_store_dwordx2 v[44:45], v[56:57], off
-; GLOBALNESS1-NEXT:    s_swappc_b64 s[30:31], s[76:77]
-; GLOBALNESS1-NEXT:    s_and_saveexec_b64 s[4:5], s[64:65]
+; GLOBALNESS1-NEXT:    s_swappc_b64 s[30:31], s[54:55]
+; GLOBALNESS1-NEXT:    s_and_saveexec_b64 s[4:5], s[96:97]
 ; GLOBALNESS1-NEXT:    s_cbranch_execz .LBB1_14
 ; GLOBALNESS1-NEXT:  ; %bb.23: ; %bb62.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_16 Depth=2
@@ -256,15 +276,24 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS1-NEXT:    s_branch .LBB1_14
 ; GLOBALNESS1-NEXT:  .LBB1_24: ; %Flow23
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
+; GLOBALNESS1-NEXT:    s_load_dwordx4 s[4:7], s[38:39], 0x0
+; GLOBALNESS1-NEXT:    v_readlane_b32 s70, v59, 8
+; GLOBALNESS1-NEXT:    v_readlane_b32 s8, v59, 10
 ; GLOBALNESS1-NEXT:    v_pk_mov_b32 v[0:1], 0, 0
+; GLOBALNESS1-NEXT:    v_readlane_b32 s71, v59, 9
+; GLOBALNESS1-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBALNESS1-NEXT:    s_mov_b32 s55, s7
+; GLOBALNESS1-NEXT:    v_readlane_b32 s9, v59, 11
 ; GLOBALNESS1-NEXT:  .LBB1_25: ; %Flow24
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    s_or_b64 exec, exec, s[74:75]
-; GLOBALNESS1-NEXT:    s_and_saveexec_b64 s[4:5], s[62:63]
+; GLOBALNESS1-NEXT:    s_or_b64 exec, exec, s[52:53]
+; GLOBALNESS1-NEXT:    s_and_saveexec_b64 s[4:5], s[86:87]
 ; GLOBALNESS1-NEXT:    s_cbranch_execz .LBB1_2
 ; GLOBALNESS1-NEXT:  ; %bb.26: ; %bb67.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[58:59]
+; GLOBALNESS1-NEXT:    v_readlane_b32 s6, v59, 4
+; GLOBALNESS1-NEXT:    v_readlane_b32 s7, v59, 5
+; GLOBALNESS1-NEXT:    s_and_b64 vcc, exec, s[6:7]
 ; GLOBALNESS1-NEXT:    s_cbranch_vccnz .LBB1_1
 ; GLOBALNESS1-NEXT:  ; %bb.27: ; %bb69.i
 ; GLOBALNESS1-NEXT:    ; in Loop: Header=BB1_4 Depth=1
@@ -277,7 +306,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS1-NEXT:    global_store_dwordx2 v[44:45], v[42:43], off
 ; GLOBALNESS1-NEXT:    s_branch .LBB1_2
 ; GLOBALNESS1-NEXT:  .LBB1_29: ; %loop.exit.guard
-; GLOBALNESS1-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
+; GLOBALNESS1-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], -1
 ; GLOBALNESS1-NEXT:    s_cbranch_vccz .LBB1_31
 ; GLOBALNESS1-NEXT:  ; %bb.30: ; %bb7.i.i
@@ -286,12 +315,12 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS1-NEXT:    s_getpc_b64 s[16:17]
 ; GLOBALNESS1-NEXT:    s_add_u32 s16, s16, widget@rel32@lo+4
 ; GLOBALNESS1-NEXT:    s_addc_u32 s17, s17, widget@rel32@hi+12
-; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS1-NEXT:    s_mov_b32 s12, s72
-; GLOBALNESS1-NEXT:    s_mov_b32 s13, s71
-; GLOBALNESS1-NEXT:    s_mov_b32 s14, s70
+; GLOBALNESS1-NEXT:    s_mov_b32 s12, s84
+; GLOBALNESS1-NEXT:    s_mov_b32 s13, s83
+; GLOBALNESS1-NEXT:    s_mov_b32 s14, s82
 ; GLOBALNESS1-NEXT:    v_mov_b32_e32 v31, v41
 ; GLOBALNESS1-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], 0
@@ -304,12 +333,12 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS1-NEXT:    s_getpc_b64 s[16:17]
 ; GLOBALNESS1-NEXT:    s_add_u32 s16, s16, widget@rel32@lo+4
 ; GLOBALNESS1-NEXT:    s_addc_u32 s17, s17, widget@rel32@hi+12
-; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS1-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; GLOBALNESS1-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS1-NEXT:    s_mov_b32 s12, s72
-; GLOBALNESS1-NEXT:    s_mov_b32 s13, s71
-; GLOBALNESS1-NEXT:    s_mov_b32 s14, s70
+; GLOBALNESS1-NEXT:    s_mov_b32 s12, s84
+; GLOBALNESS1-NEXT:    s_mov_b32 s13, s83
+; GLOBALNESS1-NEXT:    s_mov_b32 s14, s82
 ; GLOBALNESS1-NEXT:    v_mov_b32_e32 v31, v41
 ; GLOBALNESS1-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GLOBALNESS1-NEXT:  .LBB1_33: ; %UnifiedUnreachableBlock
@@ -317,14 +346,14 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS0-LABEL: kernel:
 ; GLOBALNESS0:       ; %bb.0: ; %bb
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GLOBALNESS0-NEXT:    s_load_dwordx4 s[72:75], s[8:9], 0x0
+; GLOBALNESS0-NEXT:    s_load_dwordx4 s[52:55], s[8:9], 0x0
 ; GLOBALNESS0-NEXT:    s_load_dword s6, s[8:9], 0x14
 ; GLOBALNESS0-NEXT:    v_mov_b32_e32 v42, 0
 ; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[44:45], 0, 0
 ; GLOBALNESS0-NEXT:    global_store_dword v[44:45], v42, off
 ; GLOBALNESS0-NEXT:    s_waitcnt lgkmcnt(0)
-; GLOBALNESS0-NEXT:    global_load_dword v2, v42, s[72:73]
-; GLOBALNESS0-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; GLOBALNESS0-NEXT:    global_load_dword v2, v42, s[52:53]
+; GLOBALNESS0-NEXT:    s_mov_b64 s[48:49], s[4:5]
 ; GLOBALNESS0-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x18
 ; GLOBALNESS0-NEXT:    s_load_dword s7, s[8:9], 0x20
 ; GLOBALNESS0-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
@@ -334,7 +363,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS0-NEXT:    v_mov_b32_e32 v0, 0
 ; GLOBALNESS0-NEXT:    s_addc_u32 s1, s1, 0
 ; GLOBALNESS0-NEXT:    v_mov_b32_e32 v1, 0x40994400
-; GLOBALNESS0-NEXT:    s_bitcmp1_b32 s74, 0
+; GLOBALNESS0-NEXT:    s_bitcmp1_b32 s54, 0
 ; GLOBALNESS0-NEXT:    s_waitcnt lgkmcnt(0)
 ; GLOBALNESS0-NEXT:    v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
 ; GLOBALNESS0-NEXT:    v_cmp_ngt_f64_e64 s[4:5], s[4:5], 0
@@ -344,25 +373,28 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS0-NEXT:    s_xor_b64 s[4:5], s[4:5], -1
 ; GLOBALNESS0-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
 ; GLOBALNESS0-NEXT:    s_bitcmp1_b32 s6, 0
-; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[42:43], 1, v0
+; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[50:51], 1, v0
 ; GLOBALNESS0-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
 ; GLOBALNESS0-NEXT:    s_cselect_b64 s[4:5], -1, 0
 ; GLOBALNESS0-NEXT:    s_xor_b64 s[4:5], s[4:5], -1
 ; GLOBALNESS0-NEXT:    s_bitcmp1_b32 s7, 0
-; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[48:49], 1, v0
+; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[64:65], 1, v0
 ; GLOBALNESS0-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
 ; GLOBALNESS0-NEXT:    s_cselect_b64 s[4:5], -1, 0
 ; GLOBALNESS0-NEXT:    s_xor_b64 s[4:5], s[4:5], -1
-; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[50:51], 1, v0
+; GLOBALNESS0-NEXT:    s_mov_b64 s[38:39], s[8:9]
+; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[8:9], 1, v1
+; GLOBALNESS0-NEXT:    ; implicit-def: $vgpr59 : SGPR spill to VGPR lane
+; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[66:67], 1, v0
 ; GLOBALNESS0-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[52:53], 1, v0
-; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[44:45], 1, v1
-; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[46:47], 1, v3
+; GLOBALNESS0-NEXT:    v_writelane_b32 v59, s8, 0
+; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[68:69], 1, v0
+; GLOBALNESS0-NEXT:    v_writelane_b32 v59, s9, 1
+; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[84:85], 1, v3
 ; GLOBALNESS0-NEXT:    v_mov_b32_e32 v46, 0x80
-; GLOBALNESS0-NEXT:    s_mov_b32 s68, s16
-; GLOBALNESS0-NEXT:    s_mov_b64 s[38:39], s[8:9]
-; GLOBALNESS0-NEXT:    s_mov_b32 s69, s15
-; GLOBALNESS0-NEXT:    s_mov_b32 s70, s14
+; GLOBALNESS0-NEXT:    s_mov_b32 s70, s16
+; GLOBALNESS0-NEXT:    s_mov_b32 s71, s15
+; GLOBALNESS0-NEXT:    s_mov_b32 s82, s14
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; GLOBALNESS0-NEXT:    v_mov_b32_e32 v47, 0
 ; GLOBALNESS0-NEXT:    s_mov_b32 s32, 0
@@ -373,26 +405,35 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS0-NEXT:    v_cmp_gt_i32_e32 vcc, 1, v2
 ; GLOBALNESS0-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
 ; GLOBALNESS0-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
+; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, v0
 ; GLOBALNESS0-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; GLOBALNESS0-NEXT:    v_writelane_b32 v59, s4, 2
 ; GLOBALNESS0-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GLOBALNESS0-NEXT:    v_writelane_b32 v59, s5, 3
+; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, v3
 ; GLOBALNESS0-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[54:55], 1, v0
-; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[56:57], 1, v1
-; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[58:59], 1, v3
-; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[60:61], 1, v2
+; GLOBALNESS0-NEXT:    v_writelane_b32 v59, s4, 4
+; GLOBALNESS0-NEXT:    v_writelane_b32 v59, s5, 5
+; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, v2
+; GLOBALNESS0-NEXT:    v_writelane_b32 v59, s4, 6
+; GLOBALNESS0-NEXT:    v_writelane_b32 v59, s5, 7
+; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[80:81], 1, v1
+; GLOBALNESS0-NEXT:    v_writelane_b32 v59, s84, 8
+; GLOBALNESS0-NEXT:    v_writelane_b32 v59, s85, 9
 ; GLOBALNESS0-NEXT:    s_branch .LBB1_4
 ; GLOBALNESS0-NEXT:  .LBB1_1: ; %bb70.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[60:61]
+; GLOBALNESS0-NEXT:    v_readlane_b32 s6, v59, 6
+; GLOBALNESS0-NEXT:    v_readlane_b32 s7, v59, 7
+; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[6:7]
 ; GLOBALNESS0-NEXT:    s_cbranch_vccz .LBB1_28
 ; GLOBALNESS0-NEXT:  .LBB1_2: ; %Flow15
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
 ; GLOBALNESS0-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GLOBALNESS0-NEXT:    s_mov_b64 s[8:9], 0
-; GLOBALNESS0-NEXT:    ; implicit-def: $sgpr4_sgpr5
+; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], 0
 ; GLOBALNESS0-NEXT:  .LBB1_3: ; %Flow28
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[8:9]
+; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[6:7]
 ; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[56:57], v[0:1], v[0:1] op_sel:[0,1]
 ; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_29
 ; GLOBALNESS0-NEXT:  .LBB1_4: ; %bb5
@@ -407,57 +448,64 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS0-NEXT:    s_add_u32 s4, s4, wobble@gotpcrel32@lo+4
 ; GLOBALNESS0-NEXT:    s_addc_u32 s5, s5, wobble@gotpcrel32@hi+12
 ; GLOBALNESS0-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
-; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS0-NEXT:    s_mov_b32 s12, s70
-; GLOBALNESS0-NEXT:    s_mov_b32 s13, s69
-; GLOBALNESS0-NEXT:    s_mov_b32 s14, s68
+; GLOBALNESS0-NEXT:    s_mov_b32 s12, s82
+; GLOBALNESS0-NEXT:    s_mov_b32 s13, s71
+; GLOBALNESS0-NEXT:    s_mov_b32 s14, s70
 ; GLOBALNESS0-NEXT:    v_mov_b32_e32 v31, v41
 ; GLOBALNESS0-NEXT:    s_waitcnt lgkmcnt(0)
 ; GLOBALNESS0-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[46:47]
-; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], -1
-; GLOBALNESS0-NEXT:    ; implicit-def: $sgpr4_sgpr5
+; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[84:85]
+; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], -1
+; GLOBALNESS0-NEXT:    ; implicit-def: $sgpr8_sgpr9
 ; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_9
 ; GLOBALNESS0-NEXT:  ; %bb.5: ; %NodeBlock
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    s_cmp_lt_i32 s75, 1
+; GLOBALNESS0-NEXT:    s_mov_b64 s[8:9], -1
+; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], 0
+; GLOBALNESS0-NEXT:    s_cmp_lt_i32 s55, 1
+; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], -1
 ; GLOBALNESS0-NEXT:    s_cbranch_scc1 .LBB1_7
 ; GLOBALNESS0-NEXT:  ; %bb.6: ; %LeafBlock12
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s75, 1
-; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], -1
-; GLOBALNESS0-NEXT:    s_cselect_b64 s[6:7], -1, 0
-; GLOBALNESS0-NEXT:    s_cbranch_execz .LBB1_8
-; GLOBALNESS0-NEXT:    s_branch .LBB1_9
-; GLOBALNESS0-NEXT:  .LBB1_7: ; in Loop: Header=BB1_4 Depth=1
+; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s55, 1
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], 0
-; GLOBALNESS0-NEXT:    ; implicit-def: $sgpr4_sgpr5
-; GLOBALNESS0-NEXT:  .LBB1_8: ; %LeafBlock
+; GLOBALNESS0-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; GLOBALNESS0-NEXT:  .LBB1_7: ; %Flow26
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s75, 0
-; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], 0
-; GLOBALNESS0-NEXT:    s_cselect_b64 s[6:7], -1, 0
+; GLOBALNESS0-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
+; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_9
+; GLOBALNESS0-NEXT:  ; %bb.8: ; %LeafBlock
+; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
+; GLOBALNESS0-NEXT:    s_cmp_lg_u32 s55, 0
+; GLOBALNESS0-NEXT:    s_mov_b64 s[8:9], 0
+; GLOBALNESS0-NEXT:    s_cselect_b64 s[4:5], -1, 0
 ; GLOBALNESS0-NEXT:  .LBB1_9: ; %Flow25
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    s_mov_b64 s[8:9], -1
-; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[6:7]
+; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], -1
+; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[4:5]
 ; GLOBALNESS0-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GLOBALNESS0-NEXT:    s_cbranch_vccz .LBB1_3
 ; GLOBALNESS0-NEXT:  ; %bb.10: ; %baz.exit.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
 ; GLOBALNESS0-NEXT:    flat_load_dword v0, v[44:45]
 ; GLOBALNESS0-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GLOBALNESS0-NEXT:    v_cmp_gt_i32_e64 s[62:63], 0, v0
+; GLOBALNESS0-NEXT:    v_cmp_gt_i32_e64 s[86:87], 0, v0
 ; GLOBALNESS0-NEXT:    v_mov_b32_e32 v0, 0
 ; GLOBALNESS0-NEXT:    v_mov_b32_e32 v1, 0x3ff00000
-; GLOBALNESS0-NEXT:    s_and_saveexec_b64 s[76:77], s[62:63]
+; GLOBALNESS0-NEXT:    s_and_saveexec_b64 s[52:53], s[86:87]
 ; GLOBALNESS0-NEXT:    s_cbranch_execz .LBB1_25
 ; GLOBALNESS0-NEXT:  ; %bb.11: ; %bb33.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
 ; GLOBALNESS0-NEXT:    global_load_dwordx2 v[0:1], v[44:45], off
-; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[54:55]
+; GLOBALNESS0-NEXT:    v_writelane_b32 v59, s8, 10
+; GLOBALNESS0-NEXT:    v_writelane_b32 v59, s9, 11
+; GLOBALNESS0-NEXT:    v_readlane_b32 s4, v59, 2
+; GLOBALNESS0-NEXT:    v_readlane_b32 s5, v59, 3
+; GLOBALNESS0-NEXT:    s_mov_b32 s83, s55
+; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[4:5]
 ; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_13
 ; GLOBALNESS0-NEXT:  ; %bb.12: ; %bb39.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
@@ -470,69 +518,71 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS0-NEXT:    s_waitcnt vmcnt(0)
 ; GLOBALNESS0-NEXT:    v_cmp_nlt_f64_e32 vcc, 0, v[0:1]
 ; GLOBALNESS0-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; GLOBALNESS0-NEXT:    v_cmp_eq_u32_e64 s[64:65], 0, v2
-; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[66:67], 1, v0
+; GLOBALNESS0-NEXT:    v_cmp_eq_u32_e64 s[96:97], 0, v2
+; GLOBALNESS0-NEXT:    v_cmp_ne_u32_e64 s[98:99], 1, v0
 ; GLOBALNESS0-NEXT:    s_branch .LBB1_16
 ; GLOBALNESS0-NEXT:  .LBB1_14: ; %Flow16
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_16 Depth=2
 ; GLOBALNESS0-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GLOBALNESS0-NEXT:  .LBB1_15: ; %bb63.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[52:53]
+; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[68:69]
 ; GLOBALNESS0-NEXT:    s_cbranch_vccz .LBB1_24
 ; GLOBALNESS0-NEXT:  .LBB1_16: ; %bb44.i
 ; GLOBALNESS0-NEXT:    ; Parent Loop BB1_4 Depth=1
 ; GLOBALNESS0-NEXT:    ; => This Inner Loop Header: Depth=2
-; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[48:49]
+; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[64:65]
 ; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_15
 ; GLOBALNESS0-NEXT:  ; %bb.17: ; %bb46.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[50:51]
+; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[66:67]
 ; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_15
 ; GLOBALNESS0-NEXT:  ; %bb.18: ; %bb50.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[42:43]
+; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[50:51]
 ; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_21
 ; GLOBALNESS0-NEXT:  ; %bb.19: ; %bb3.i.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[44:45]
+; GLOBALNESS0-NEXT:    v_readlane_b32 s4, v59, 0
+; GLOBALNESS0-NEXT:    v_readlane_b32 s5, v59, 1
+; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[4:5]
 ; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_21
 ; GLOBALNESS0-NEXT:  ; %bb.20: ; %bb6.i.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[66:67]
+; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[98:99]
 ; GLOBALNESS0-NEXT:  .LBB1_21: ; %spam.exit.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[56:57]
+; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[80:81]
 ; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_15
 ; GLOBALNESS0-NEXT:  ; %bb.22: ; %bb55.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS0-NEXT:    s_add_u32 s72, s38, 40
-; GLOBALNESS0-NEXT:    s_addc_u32 s73, s39, 0
+; GLOBALNESS0-NEXT:    s_add_u32 s84, s38, 40
+; GLOBALNESS0-NEXT:    s_addc_u32 s85, s39, 0
 ; GLOBALNESS0-NEXT:    s_getpc_b64 s[4:5]
 ; GLOBALNESS0-NEXT:    s_add_u32 s4, s4, wobble@gotpcrel32@lo+4
 ; GLOBALNESS0-NEXT:    s_addc_u32 s5, s5, wobble@gotpcrel32@hi+12
-; GLOBALNESS0-NEXT:    s_load_dwordx2 s[78:79], s[4:5], 0x0
-; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS0-NEXT:    s_load_dwordx2 s[54:55], s[4:5], 0x0
+; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], s[36:37]
-; GLOBALNESS0-NEXT:    s_mov_b64 s[8:9], s[72:73]
+; GLOBALNESS0-NEXT:    s_mov_b64 s[8:9], s[84:85]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS0-NEXT:    s_mov_b32 s12, s70
-; GLOBALNESS0-NEXT:    s_mov_b32 s13, s69
-; GLOBALNESS0-NEXT:    s_mov_b32 s14, s68
+; GLOBALNESS0-NEXT:    s_mov_b32 s12, s82
+; GLOBALNESS0-NEXT:    s_mov_b32 s13, s71
+; GLOBALNESS0-NEXT:    s_mov_b32 s14, s70
 ; GLOBALNESS0-NEXT:    v_mov_b32_e32 v31, v41
 ; GLOBALNESS0-NEXT:    s_waitcnt lgkmcnt(0)
-; GLOBALNESS0-NEXT:    s_swappc_b64 s[30:31], s[78:79]
-; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS0-NEXT:    s_swappc_b64 s[30:31], s[54:55]
+; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], s[36:37]
-; GLOBALNESS0-NEXT:    s_mov_b64 s[8:9], s[72:73]
+; GLOBALNESS0-NEXT:    s_mov_b64 s[8:9], s[84:85]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS0-NEXT:    s_mov_b32 s12, s70
-; GLOBALNESS0-NEXT:    s_mov_b32 s13, s69
-; GLOBALNESS0-NEXT:    s_mov_b32 s14, s68
+; GLOBALNESS0-NEXT:    s_mov_b32 s12, s82
+; GLOBALNESS0-NEXT:    s_mov_b32 s13, s71
+; GLOBALNESS0-NEXT:    s_mov_b32 s14, s70
 ; GLOBALNESS0-NEXT:    v_mov_b32_e32 v31, v41
 ; GLOBALNESS0-NEXT:    global_store_dwordx2 v[44:45], v[56:57], off
-; GLOBALNESS0-NEXT:    s_swappc_b64 s[30:31], s[78:79]
-; GLOBALNESS0-NEXT:    s_and_saveexec_b64 s[4:5], s[64:65]
+; GLOBALNESS0-NEXT:    s_swappc_b64 s[30:31], s[54:55]
+; GLOBALNESS0-NEXT:    s_and_saveexec_b64 s[4:5], s[96:97]
 ; GLOBALNESS0-NEXT:    s_cbranch_execz .LBB1_14
 ; GLOBALNESS0-NEXT:  ; %bb.23: ; %bb62.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_16 Depth=2
@@ -541,15 +591,22 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS0-NEXT:    s_branch .LBB1_14
 ; GLOBALNESS0-NEXT:  .LBB1_24: ; %Flow23
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
+; GLOBALNESS0-NEXT:    v_readlane_b32 s84, v59, 8
+; GLOBALNESS0-NEXT:    v_readlane_b32 s8, v59, 10
 ; GLOBALNESS0-NEXT:    v_pk_mov_b32 v[0:1], 0, 0
+; GLOBALNESS0-NEXT:    s_mov_b32 s55, s83
+; GLOBALNESS0-NEXT:    v_readlane_b32 s85, v59, 9
+; GLOBALNESS0-NEXT:    v_readlane_b32 s9, v59, 11
 ; GLOBALNESS0-NEXT:  .LBB1_25: ; %Flow24
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    s_or_b64 exec, exec, s[76:77]
-; GLOBALNESS0-NEXT:    s_and_saveexec_b64 s[4:5], s[62:63]
+; GLOBALNESS0-NEXT:    s_or_b64 exec, exec, s[52:53]
+; GLOBALNESS0-NEXT:    s_and_saveexec_b64 s[4:5], s[86:87]
 ; GLOBALNESS0-NEXT:    s_cbranch_execz .LBB1_2
 ; GLOBALNESS0-NEXT:  ; %bb.26: ; %bb67.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[58:59]
+; GLOBALNESS0-NEXT:    v_readlane_b32 s6, v59, 4
+; GLOBALNESS0-NEXT:    v_readlane_b32 s7, v59, 5
+; GLOBALNESS0-NEXT:    s_and_b64 vcc, exec, s[6:7]
 ; GLOBALNESS0-NEXT:    s_cbranch_vccnz .LBB1_1
 ; GLOBALNESS0-NEXT:  ; %bb.27: ; %bb69.i
 ; GLOBALNESS0-NEXT:    ; in Loop: Header=BB1_4 Depth=1
@@ -562,7 +619,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS0-NEXT:    global_store_dwordx2 v[44:45], v[42:43], off
 ; GLOBALNESS0-NEXT:    s_branch .LBB1_2
 ; GLOBALNESS0-NEXT:  .LBB1_29: ; %loop.exit.guard
-; GLOBALNESS0-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
+; GLOBALNESS0-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], -1
 ; GLOBALNESS0-NEXT:    s_cbranch_vccz .LBB1_31
 ; GLOBALNESS0-NEXT:  ; %bb.30: ; %bb7.i.i
@@ -571,12 +628,12 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS0-NEXT:    s_getpc_b64 s[16:17]
 ; GLOBALNESS0-NEXT:    s_add_u32 s16, s16, widget@rel32@lo+4
 ; GLOBALNESS0-NEXT:    s_addc_u32 s17, s17, widget@rel32@hi+12
-; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS0-NEXT:    s_mov_b32 s12, s70
-; GLOBALNESS0-NEXT:    s_mov_b32 s13, s69
-; GLOBALNESS0-NEXT:    s_mov_b32 s14, s68
+; GLOBALNESS0-NEXT:    s_mov_b32 s12, s82
+; GLOBALNESS0-NEXT:    s_mov_b32 s13, s71
+; GLOBALNESS0-NEXT:    s_mov_b32 s14, s70
 ; GLOBALNESS0-NEXT:    v_mov_b32_e32 v31, v41
 ; GLOBALNESS0-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], 0
@@ -589,12 +646,12 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
 ; GLOBALNESS0-NEXT:    s_getpc_b64 s[16:17]
 ; GLOBALNESS0-NEXT:    s_add_u32 s16, s16, widget@rel32@lo+4
 ; GLOBALNESS0-NEXT:    s_addc_u32 s17, s17, widget@rel32@hi+12
-; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GLOBALNESS0-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; GLOBALNESS0-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS0-NEXT:    s_mov_b32 s12, s70
-; GLOBALNESS0-NEXT:    s_mov_b32 s13, s69
-; GLOBALNESS0-NEXT:    s_mov_b32 s14, s68
+; GLOBALNESS0-NEXT:    s_mov_b32 s12, s82
+; GLOBALNESS0-NEXT:    s_mov_b32 s13, s71
+; GLOBALNESS0-NEXT:    s_mov_b32 s14, s70
 ; GLOBALNESS0-NEXT:    v_mov_b32_e32 v31, v41
 ; GLOBALNESS0-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GLOBALNESS0-NEXT:  .LBB1_33: ; %UnifiedUnreachableBlock
diff --git a/llvm/test/CodeGen/AMDGPU/unaligned-buffer.ll b/llvm/test/CodeGen/AMDGPU/unaligned-buffer.ll
new file mode 100644
index 0000000000000..72c6010a5a80b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/unaligned-buffer.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck -check-prefix=SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck -check-prefix=GISEL %s
+
+; Check that in strict OOB mode for buffers (relaxed-buffer-oob-mode attribute not set) the underaligned loads and stores get split.
+; FIXME: The loads/stores do not get split (extend amdgpu-lower-buffer-fat-pointers?).
+
+define amdgpu_ps void @split_underaligned_load(ptr addrspace(7) inreg %p, ptr addrspace(7) inreg %p2) #0 {
+; CHECK-LABEL: split_underaligned_load:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v2, s9
+; CHECK-NEXT:    s_mov_b32 s15, s8
+; CHECK-NEXT:    s_mov_b32 s14, s7
+; CHECK-NEXT:    s_mov_b32 s13, s6
+; CHECK-NEXT:    buffer_load_b64 v[0:1], v0, s[0:3], 0 offen
+; CHECK-NEXT:    s_mov_b32 s12, s5
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    buffer_store_b64 v[0:1], v2, s[12:15], 0 offen
+; CHECK-NEXT:    s_endpgm
+; SDAG-LABEL: split_underaligned_load:
+; SDAG:       ; %bb.0: ; %entry
+; SDAG-NEXT:    v_mov_b32_e32 v0, s4
+; SDAG-NEXT:    v_mov_b32_e32 v2, s9
+; SDAG-NEXT:    s_mov_b32 s15, s8
+; SDAG-NEXT:    s_mov_b32 s14, s7
+; SDAG-NEXT:    s_mov_b32 s13, s6
+; SDAG-NEXT:    buffer_load_b64 v[0:1], v0, s[0:3], 0 offen
+; SDAG-NEXT:    s_mov_b32 s12, s5
+; SDAG-NEXT:    s_waitcnt vmcnt(0)
+; SDAG-NEXT:    buffer_store_b64 v[0:1], v2, s[12:15], 0 offen
+; SDAG-NEXT:    s_endpgm
+;
+; GISEL-LABEL: split_underaligned_load:
+; GISEL:       ; %bb.0: ; %entry
+; GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; GISEL-NEXT:    v_mov_b32_e32 v2, s9
+; GISEL-NEXT:    s_mov_b32 s12, s5
+; GISEL-NEXT:    s_mov_b32 s13, s6
+; GISEL-NEXT:    s_mov_b32 s14, s7
+; GISEL-NEXT:    buffer_load_b64 v[0:1], v0, s[0:3], 0 offen
+; GISEL-NEXT:    s_mov_b32 s15, s8
+; GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GISEL-NEXT:    buffer_store_b64 v[0:1], v2, s[12:15], 0 offen
+; GISEL-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i8, ptr addrspace(7) %p, i32 0
+  %ld = load i64, ptr addrspace(7) %gep, align 4
+
+  %gep2 = getelementptr i8, ptr addrspace(7) %p2, i32 0
+  store i64 %ld, ptr addrspace(7) %gep2, align 4
+  ret void
+}
+
+; Check that in strict OOB mode for buffers (relaxed-buffer-oob-mode attribute not set) the naturally aligned loads and stores do not get split.
+
+define amdgpu_ps void @do_not_split_aligned_load(ptr addrspace(7) inreg %p, ptr addrspace(7) inreg %p2) #0 {
+; CHECK-LABEL: do_not_split_aligned_load:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v2, s9
+; CHECK-NEXT:    s_mov_b32 s15, s8
+; CHECK-NEXT:    s_mov_b32 s14, s7
+; CHECK-NEXT:    s_mov_b32 s13, s6
+; CHECK-NEXT:    buffer_load_b64 v[0:1], v0, s[0:3], 0 offen
+; CHECK-NEXT:    s_mov_b32 s12, s5
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    buffer_store_b64 v[0:1], v2, s[12:15], 0 offen
+; CHECK-NEXT:    s_endpgm
+; SDAG-LABEL: do_not_split_aligned_load:
+; SDAG:       ; %bb.0: ; %entry
+; SDAG-NEXT:    v_mov_b32_e32 v0, s4
+; SDAG-NEXT:    v_mov_b32_e32 v2, s9
+; SDAG-NEXT:    s_mov_b32 s15, s8
+; SDAG-NEXT:    s_mov_b32 s14, s7
+; SDAG-NEXT:    s_mov_b32 s13, s6
+; SDAG-NEXT:    buffer_load_b64 v[0:1], v0, s[0:3], 0 offen
+; SDAG-NEXT:    s_mov_b32 s12, s5
+; SDAG-NEXT:    s_waitcnt vmcnt(0)
+; SDAG-NEXT:    buffer_store_b64 v[0:1], v2, s[12:15], 0 offen
+; SDAG-NEXT:    s_endpgm
+;
+; GISEL-LABEL: do_not_split_aligned_load:
+; GISEL:       ; %bb.0: ; %entry
+; GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; GISEL-NEXT:    v_mov_b32_e32 v2, s9
+; GISEL-NEXT:    s_mov_b32 s12, s5
+; GISEL-NEXT:    s_mov_b32 s13, s6
+; GISEL-NEXT:    s_mov_b32 s14, s7
+; GISEL-NEXT:    buffer_load_b64 v[0:1], v0, s[0:3], 0 offen
+; GISEL-NEXT:    s_mov_b32 s15, s8
+; GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GISEL-NEXT:    buffer_store_b64 v[0:1], v2, s[12:15], 0 offen
+; GISEL-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i8, ptr addrspace(7) %p, i32 0
+  %ld = load i64, ptr addrspace(7) %gep, align 8
+
+  %gep2 = getelementptr i8, ptr addrspace(7) %p2, i32 0
+  store i64 %ld, ptr addrspace(7) %gep2, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/unallocatable-bundle-regression.mir b/llvm/test/CodeGen/AMDGPU/unallocatable-bundle-regression.mir
index b80c478c3761f..0df2e651a15e1 100644
--- a/llvm/test/CodeGen/AMDGPU/unallocatable-bundle-regression.mir
+++ b/llvm/test/CodeGen/AMDGPU/unallocatable-bundle-regression.mir
@@ -27,11 +27,11 @@ body:             |
     ; CHECK-NEXT: renamable $sgpr4 = COPY $sgpr0
     ; CHECK-NEXT: SI_SPILL_S128_SAVE $sgpr0_sgpr1_sgpr2_sgpr3, %stack.0, implicit $exec, implicit $sgpr32 :: (store (s128) into %stack.0, align 4, addrspace 5)
     ; CHECK-NEXT: renamable $sgpr5 = S_MOV_B32 0
-    ; CHECK-NEXT: renamable $sgpr76 = COPY renamable $sgpr5
-    ; CHECK-NEXT: renamable $sgpr77 = COPY renamable $sgpr5
-    ; CHECK-NEXT: renamable $sgpr78 = COPY renamable $sgpr5
+    ; CHECK-NEXT: renamable $sgpr36 = COPY renamable $sgpr5
+    ; CHECK-NEXT: renamable $sgpr37 = COPY renamable $sgpr5
+    ; CHECK-NEXT: renamable $sgpr38 = COPY renamable $sgpr5
     ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 1056964608
-    ; CHECK-NEXT: renamable $sgpr79 = COPY renamable $sgpr5
+    ; CHECK-NEXT: renamable $sgpr39 = COPY renamable $sgpr5
     ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0
     ; CHECK-NEXT: renamable $sgpr8 = COPY renamable $sgpr5
     ; CHECK-NEXT: renamable $sgpr9 = COPY renamable $sgpr5
@@ -43,46 +43,46 @@ body:             |
     ; CHECK-NEXT: renamable $sgpr15 = COPY renamable $sgpr5
     ; CHECK-NEXT: renamable $vgpr5_vgpr6 = COPY killed renamable $sgpr0_sgpr1
     ; CHECK-NEXT: renamable $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23 = S_LOAD_DWORDX8_IMM renamable $sgpr4_sgpr5, 1088, 0 :: (dereferenceable load (s256), addrspace 6)
-    ; CHECK-NEXT: renamable $sgpr80_sgpr81_sgpr82_sgpr83 = S_LOAD_DWORDX4_IMM renamable $sgpr4_sgpr5, 0, 0 :: (load (s128), addrspace 6)
+    ; CHECK-NEXT: renamable $sgpr64_sgpr65_sgpr66_sgpr67 = S_LOAD_DWORDX4_IMM renamable $sgpr4_sgpr5, 0, 0 :: (load (s128), addrspace 6)
     ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 1200
     ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr5
-    ; CHECK-NEXT: renamable $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31 = S_LOAD_DWORDX8_IMM renamable $sgpr4_sgpr5, 1152, 0 :: (dereferenceable load (s256), addrspace 6)
-    ; CHECK-NEXT: renamable $sgpr84_sgpr85_sgpr86_sgpr87 = S_LOAD_DWORDX4_IMM renamable $sgpr0_sgpr1, 0, 0 :: (load (s128), addrspace 6)
+    ; CHECK-NEXT: renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47 = S_LOAD_DWORDX8_IMM renamable $sgpr4_sgpr5, 1152, 0 :: (dereferenceable load (s256), addrspace 6)
+    ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71 = S_LOAD_DWORDX4_IMM renamable $sgpr0_sgpr1, 0, 0 :: (load (s128), addrspace 6)
     ; CHECK-NEXT: KILL killed renamable $sgpr0, renamable $sgpr1
     ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 1264
     ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr5
-    ; CHECK-NEXT: renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43 = S_LOAD_DWORDX8_IMM renamable $sgpr4_sgpr5, 1216, 0 :: (dereferenceable load (s256), addrspace 6)
-    ; CHECK-NEXT: renamable $sgpr88_sgpr89_sgpr90_sgpr91 = S_LOAD_DWORDX4_IMM killed renamable $sgpr0_sgpr1, 0, 0 :: (load (s128), addrspace 6)
+    ; CHECK-NEXT: renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63 = S_LOAD_DWORDX8_IMM renamable $sgpr4_sgpr5, 1216, 0 :: (dereferenceable load (s256), addrspace 6)
+    ; CHECK-NEXT: renamable $sgpr80_sgpr81_sgpr82_sgpr83 = S_LOAD_DWORDX4_IMM killed renamable $sgpr0_sgpr1, 0, 0 :: (load (s128), addrspace 6)
     ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 1328
     ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr5
-    ; CHECK-NEXT: renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51 = S_LOAD_DWORDX8_IMM renamable $sgpr4_sgpr5, 1280, 0 :: (dereferenceable load (s256), addrspace 6)
-    ; CHECK-NEXT: renamable $sgpr92_sgpr93_sgpr94_sgpr95 = S_LOAD_DWORDX4_IMM killed renamable $sgpr0_sgpr1, 0, 0 :: (load (s128), addrspace 6)
-    ; CHECK-NEXT: renamable $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59 = S_LOAD_DWORDX8_IMM renamable $sgpr4_sgpr5, 1344, 0 :: (dereferenceable load (s256), addrspace 6)
+    ; CHECK-NEXT: renamable $sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79 = S_LOAD_DWORDX8_IMM renamable $sgpr4_sgpr5, 1280, 0 :: (dereferenceable load (s256), addrspace 6)
+    ; CHECK-NEXT: renamable $sgpr84_sgpr85_sgpr86_sgpr87 = S_LOAD_DWORDX4_IMM killed renamable $sgpr0_sgpr1, 0, 0 :: (load (s128), addrspace 6)
+    ; CHECK-NEXT: renamable $sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95 = S_LOAD_DWORDX8_IMM renamable $sgpr4_sgpr5, 1344, 0 :: (dereferenceable load (s256), addrspace 6)
     ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 1392
     ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr5
-    ; CHECK-NEXT: renamable $sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67 = S_LOAD_DWORDX8_IMM renamable $sgpr4_sgpr5, 0, 0 :: (load (s256), addrspace 6)
+    ; CHECK-NEXT: renamable $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31 = S_LOAD_DWORDX8_IMM renamable $sgpr4_sgpr5, 0, 0 :: (load (s256), addrspace 6)
     ; CHECK-NEXT: renamable $sgpr2 = S_MOV_B32 1456
     ; CHECK-NEXT: renamable $sgpr3 = COPY renamable $sgpr5
-    ; CHECK-NEXT: renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75 = S_LOAD_DWORDX8_IMM renamable $sgpr4_sgpr5, 1472, 0 :: (dereferenceable load (s256), addrspace 6)
+    ; CHECK-NEXT: renamable $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55 = S_LOAD_DWORDX8_IMM renamable $sgpr4_sgpr5, 1472, 0 :: (dereferenceable load (s256), addrspace 6)
     ; CHECK-NEXT: renamable $sgpr4 = S_MOV_B32 1520
     ; CHECK-NEXT: renamable $sgpr96_sgpr97_sgpr98_sgpr99 = S_LOAD_DWORDX4_IMM killed renamable $sgpr2_sgpr3, 0, 0 :: (load (s128), addrspace 6)
     ; CHECK-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (load (s128), addrspace 6)
     ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr0_sgpr1, 0, 0 :: (load (s128), addrspace 6)
-    ; CHECK-NEXT: renamable $vgpr7 = IMAGE_SAMPLE_LZ_V1_V2_gfx10 renamable $vgpr5_vgpr6, killed renamable $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, killed renamable $sgpr76_sgpr77_sgpr78_sgpr79, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128))
-    ; CHECK-NEXT: renamable $vgpr8 = IMAGE_SAMPLE_LZ_V1_V2_gfx10 renamable $vgpr5_vgpr6, killed renamable $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23, killed renamable $sgpr80_sgpr81_sgpr82_sgpr83, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128))
-    ; CHECK-NEXT: renamable $vgpr9 = IMAGE_SAMPLE_LZ_V1_V2_gfx10 renamable $vgpr5_vgpr6, killed renamable $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31, killed renamable $sgpr84_sgpr85_sgpr86_sgpr87, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128))
-    ; CHECK-NEXT: renamable $vgpr10 = IMAGE_SAMPLE_LZ_V1_V2_gfx10 renamable $vgpr5_vgpr6, killed renamable $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43, killed renamable $sgpr88_sgpr89_sgpr90_sgpr91, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128))
-    ; CHECK-NEXT: renamable $vgpr11 = IMAGE_SAMPLE_LZ_V1_V2_gfx10 renamable $vgpr5_vgpr6, killed renamable $sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, killed renamable $sgpr92_sgpr93_sgpr94_sgpr95, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128))
-    ; CHECK-NEXT: renamable $vgpr12 = IMAGE_SAMPLE_LZ_V1_V2_gfx10 renamable $vgpr5_vgpr6, killed renamable $sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, killed renamable $sgpr96_sgpr97_sgpr98_sgpr99, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128))
-    ; CHECK-NEXT: renamable $vgpr13 = IMAGE_SAMPLE_LZ_V1_V2_gfx10 renamable $vgpr5_vgpr6, renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128))
-    ; CHECK-NEXT: renamable $vgpr14 = IMAGE_SAMPLE_LZ_V1_V2_gfx10 renamable $vgpr5_vgpr6, renamable $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128))
+    ; CHECK-NEXT: renamable $vgpr7 = IMAGE_SAMPLE_LZ_V1_V2_gfx10 renamable $vgpr5_vgpr6, killed renamable $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15, killed renamable $sgpr36_sgpr37_sgpr38_sgpr39, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128))
+    ; CHECK-NEXT: renamable $vgpr8 = IMAGE_SAMPLE_LZ_V1_V2_gfx10 renamable $vgpr5_vgpr6, killed renamable $sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23, killed renamable $sgpr64_sgpr65_sgpr66_sgpr67, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128))
+    ; CHECK-NEXT: renamable $vgpr9 = IMAGE_SAMPLE_LZ_V1_V2_gfx10 renamable $vgpr5_vgpr6, killed renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47, killed renamable $sgpr68_sgpr69_sgpr70_sgpr71, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128))
+    ; CHECK-NEXT: renamable $vgpr10 = IMAGE_SAMPLE_LZ_V1_V2_gfx10 renamable $vgpr5_vgpr6, killed renamable $sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63, killed renamable $sgpr80_sgpr81_sgpr82_sgpr83, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128))
+    ; CHECK-NEXT: renamable $vgpr11 = IMAGE_SAMPLE_LZ_V1_V2_gfx10 renamable $vgpr5_vgpr6, killed renamable $sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79, killed renamable $sgpr84_sgpr85_sgpr86_sgpr87, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128))
+    ; CHECK-NEXT: renamable $vgpr12 = IMAGE_SAMPLE_LZ_V1_V2_gfx10 renamable $vgpr5_vgpr6, killed renamable $sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31, killed renamable $sgpr96_sgpr97_sgpr98_sgpr99, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128))
+    ; CHECK-NEXT: renamable $vgpr13 = IMAGE_SAMPLE_LZ_V1_V2_gfx10 renamable $vgpr5_vgpr6, renamable $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128))
+    ; CHECK-NEXT: renamable $vgpr14 = IMAGE_SAMPLE_LZ_V1_V2_gfx10 renamable $vgpr5_vgpr6, renamable $sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128))
     ; CHECK-NEXT: renamable $sgpr8_sgpr9_sgpr10_sgpr11 = SI_SPILL_S128_RESTORE %stack.0, implicit $exec, implicit $sgpr32 :: (load (s128) from %stack.0, align 4, addrspace 5)
     ; CHECK-NEXT: renamable $vgpr1_vgpr2_vgpr3_vgpr4 = BUFFER_LOAD_FORMAT_XYZW_IDXEN renamable $vgpr0, renamable $sgpr8_sgpr9_sgpr10_sgpr11, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
     ; CHECK-NEXT: KILL killed renamable $sgpr4_sgpr5_sgpr6_sgpr7
-    ; CHECK-NEXT: KILL killed renamable $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75
+    ; CHECK-NEXT: KILL killed renamable $sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55
     ; CHECK-NEXT: KILL killed renamable $vgpr5_vgpr6
     ; CHECK-NEXT: KILL killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
-    ; CHECK-NEXT: KILL killed renamable $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59
+    ; CHECK-NEXT: KILL killed renamable $sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95
     ; CHECK-NEXT: KILL killed renamable $sgpr8_sgpr9_sgpr10_sgpr11
     ; CHECK-NEXT: KILL killed renamable $vgpr0
     ; CHECK-NEXT: renamable $vgpr0 = nofpexcept V_MAX_F32_e32 killed $vgpr7, killed $vgpr8, implicit $mode, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/undefined-subreg-liverange.ll b/llvm/test/CodeGen/AMDGPU/undefined-subreg-liverange.ll
index 0acee5bd5ac19..2809234ce8941 100644
--- a/llvm/test/CodeGen/AMDGPU/undefined-subreg-liverange.ll
+++ b/llvm/test/CodeGen/AMDGPU/undefined-subreg-liverange.ll
@@ -69,7 +69,7 @@ define amdgpu_ps float @valley_partially_undef_copy() #0 {
 bb:
   %tmp = load volatile i32, ptr addrspace(1) undef, align 4
   %tmp1 = load volatile i32, ptr addrspace(1) undef, align 4
-  %tmp2 = insertelement <4 x i32> undef, i32 %tmp1, i32 0
+  %tmp2 = insertelement <4 x i32> poison, i32 %tmp1, i32 0
   %tmp3 = bitcast i32 %tmp1 to float
   %tmp4 = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float %tmp3, float %tmp3, <8 x i32> undef, <4 x i32> undef, i1 0, i32 0, i32 0)
   %tmp5 = extractelement <4 x float> %tmp4, i32 0
@@ -115,7 +115,7 @@ define amdgpu_kernel void @partially_undef_copy() #0 {
   %tmp0 = call i32 asm sideeffect "v_mov_b32_e32 v5, 5", "={v5}"()
   %tmp1 = call i32 asm sideeffect "v_mov_b32_e32 v6, 6", "={v6}"()
 
-  %partially.undef.0 = insertelement <4 x i32> undef, i32 %tmp0, i32 0
+  %partially.undef.0 = insertelement <4 x i32> poison, i32 %tmp0, i32 0
   %partially.undef.1 = insertelement <4 x i32> %partially.undef.0, i32 %tmp1, i32 0
 
   store volatile <4 x i32> %partially.undef.1, ptr addrspace(1) undef, align 16
diff --git a/llvm/test/CodeGen/AMDGPU/uniform-phi-with-undef.ll b/llvm/test/CodeGen/AMDGPU/uniform-phi-with-undef.ll
index 64d4a0cf78501..ea127b7b3e440 100644
--- a/llvm/test/CodeGen/AMDGPU/uniform-phi-with-undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/uniform-phi-with-undef.ll
@@ -44,7 +44,7 @@ if:
 
 end:
   %v2 = phi float [ %v.if, %if ], [ %v, %entry ]
-  %c2 = phi float [ undef, %if ], [ %c, %entry ]
+  %c2 = phi float [ poison, %if ], [ %c, %entry ]
   %r = fadd float %v2, %c2
   ret float %r
 }
diff --git a/llvm/test/CodeGen/AMDGPU/unigine-liveness-crash.ll b/llvm/test/CodeGen/AMDGPU/unigine-liveness-crash.ll
index fd0c74a192d38..2d780c86c4566 100644
--- a/llvm/test/CodeGen/AMDGPU/unigine-liveness-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/unigine-liveness-crash.ll
@@ -44,7 +44,7 @@ main_body:
   %tmp50 = bitcast float %tmp27 to i32
   %tmp51 = bitcast float %tmp48 to i32
   %tmp52 = bitcast float %tmp49 to i32
-  %tmp53 = insertelement <4 x i32> undef, i32 %tmp50, i32 0
+  %tmp53 = insertelement <4 x i32> poison, i32 %tmp50, i32 0
   %tmp54 = insertelement <4 x i32> %tmp53, i32 %tmp51, i32 1
   %tmp55 = insertelement <4 x i32> %tmp54, i32 %tmp52, i32 2
   %tmp55.cast = bitcast <4 x i32> %tmp55 to <4 x float>
@@ -63,7 +63,7 @@ main_body:
 
 IF26:                                             ; preds = %main_body
   %tmp71 = bitcast float %tmp27 to i32
-  %tmp72 = insertelement <4 x i32> undef, i32 %tmp71, i32 0
+  %tmp72 = insertelement <4 x i32> poison, i32 %tmp71, i32 0
   br label %LOOP
 
 ENDIF25:                                          ; preds = %IF29, %main_body
@@ -74,16 +74,16 @@ ENDIF25:                                          ; preds = %IF29, %main_body
   %tmp75 = fmul float %clamp.i, %clamp.i
   %tmp76 = fmul float %tmp75, %tmp75
   %tmp77 = fadd float %tmp76, %tmp76
-  %tmp78 = insertvalue <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> undef, float %tmp77, 11
-  %tmp79 = insertvalue <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %tmp78, float undef, 12
-  %tmp80 = insertvalue <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %tmp79, float undef, 13
+  %tmp78 = insertvalue <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> poison, float %tmp77, 11
+  %tmp79 = insertvalue <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %tmp78, float poison, 12
+  %tmp80 = insertvalue <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %tmp79, float poison, 13
   %tmp81 = insertvalue <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %tmp80, float %tmp25, 14
-  %tmp82 = insertvalue <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %tmp81, float undef, 15
+  %tmp82 = insertvalue <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %tmp81, float poison, 15
   %tmp83 = insertvalue <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %tmp82, float %arg21, 24
   ret <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %tmp83
 
 LOOP:                                             ; preds = %ENDIF28, %IF26
-  %.5 = phi float [ undef, %IF26 ], [ %tmp89, %ENDIF28 ]
+  %.5 = phi float [ poison, %IF26 ], [ %tmp89, %ENDIF28 ]
   br i1 false, label %IF29, label %ENDIF28
 
 IF29:                                             ; preds = %LOOP
diff --git a/llvm/test/CodeGen/AMDGPU/unpack-half.ll b/llvm/test/CodeGen/AMDGPU/unpack-half.ll
index f1dbc7f07f595..fc3b35a6cb8bd 100644
--- a/llvm/test/CodeGen/AMDGPU/unpack-half.ll
+++ b/llvm/test/CodeGen/AMDGPU/unpack-half.ll
@@ -13,7 +13,7 @@ define amdgpu_gs void @main(i32 inreg %arg) local_unnamed_addr #0 {
   %tmp = load volatile float, ptr addrspace(1) undef
   %tmp1 = bitcast float %tmp to i32
   %im0.i = lshr i32 %tmp1, 16
-  %tmp2 = insertelement <2 x i32> undef, i32 %im0.i, i32 1
+  %tmp2 = insertelement <2 x i32> poison, i32 %im0.i, i32 1
   %tmp3 = trunc <2 x i32> %tmp2 to <2 x i16>
   %tmp4 = bitcast <2 x i16> %tmp3 to <2 x half>
   %tmp5 = fpext <2 x half> %tmp4 to <2 x float>
diff --git a/llvm/test/CodeGen/AMDGPU/unstructured-cfg-def-use-issue.ll b/llvm/test/CodeGen/AMDGPU/unstructured-cfg-def-use-issue.ll
index d9df80ce6c1c0..f08e5be0fd742 100644
--- a/llvm/test/CodeGen/AMDGPU/unstructured-cfg-def-use-issue.ll
+++ b/llvm/test/CodeGen/AMDGPU/unstructured-cfg-def-use-issue.ll
@@ -22,14 +22,14 @@ define hidden void @widget() {
 ; GCN-NEXT:    v_writelane_b32 v41, s37, 5
 ; GCN-NEXT:    v_writelane_b32 v41, s38, 6
 ; GCN-NEXT:    v_writelane_b32 v41, s39, 7
-; GCN-NEXT:    v_writelane_b32 v41, s40, 8
-; GCN-NEXT:    v_writelane_b32 v41, s41, 9
-; GCN-NEXT:    v_writelane_b32 v41, s42, 10
-; GCN-NEXT:    v_writelane_b32 v41, s43, 11
-; GCN-NEXT:    v_writelane_b32 v41, s44, 12
-; GCN-NEXT:    v_writelane_b32 v41, s45, 13
-; GCN-NEXT:    v_writelane_b32 v41, s46, 14
-; GCN-NEXT:    v_writelane_b32 v41, s47, 15
+; GCN-NEXT:    v_writelane_b32 v41, s48, 8
+; GCN-NEXT:    v_writelane_b32 v41, s49, 9
+; GCN-NEXT:    v_writelane_b32 v41, s50, 10
+; GCN-NEXT:    v_writelane_b32 v41, s51, 11
+; GCN-NEXT:    v_writelane_b32 v41, s52, 12
+; GCN-NEXT:    v_writelane_b32 v41, s53, 13
+; GCN-NEXT:    v_writelane_b32 v41, s54, 14
+; GCN-NEXT:    v_writelane_b32 v41, s55, 15
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0
 ; GCN-NEXT:    flat_load_dword v0, v[0:1]
@@ -37,7 +37,7 @@ define hidden void @widget() {
 ; GCN-NEXT:    s_mov_b64 s[16:17], 0
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_cmp_gt_i32_e32 vcc, 21, v0
-; GCN-NEXT:    s_mov_b64 s[46:47], 0
+; GCN-NEXT:    s_mov_b64 s[54:55], 0
 ; GCN-NEXT:    s_mov_b64 s[18:19], 0
 ; GCN-NEXT:    s_cbranch_vccz .LBB0_9
 ; GCN-NEXT:  ; %bb.1: ; %Flow
@@ -53,29 +53,29 @@ define hidden void @widget() {
 ; GCN-NEXT:    s_mov_b64 s[34:35], s[4:5]
 ; GCN-NEXT:    s_mov_b64 s[36:37], s[6:7]
 ; GCN-NEXT:    s_mov_b64 s[38:39], s[8:9]
-; GCN-NEXT:    s_mov_b64 s[40:41], s[10:11]
-; GCN-NEXT:    s_mov_b32 s42, s12
-; GCN-NEXT:    s_mov_b32 s43, s13
-; GCN-NEXT:    s_mov_b32 s44, s14
-; GCN-NEXT:    s_mov_b32 s45, s15
+; GCN-NEXT:    s_mov_b64 s[48:49], s[10:11]
+; GCN-NEXT:    s_mov_b32 s50, s12
+; GCN-NEXT:    s_mov_b32 s51, s13
+; GCN-NEXT:    s_mov_b32 s52, s14
+; GCN-NEXT:    s_mov_b32 s53, s15
 ; GCN-NEXT:    v_mov_b32_e32 v40, v31
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GCN-NEXT:    v_mov_b32_e32 v31, v40
-; GCN-NEXT:    s_mov_b32 s12, s42
-; GCN-NEXT:    s_mov_b32 s13, s43
-; GCN-NEXT:    s_mov_b32 s14, s44
-; GCN-NEXT:    s_mov_b32 s15, s45
+; GCN-NEXT:    s_mov_b32 s12, s50
+; GCN-NEXT:    s_mov_b32 s13, s51
+; GCN-NEXT:    s_mov_b32 s14, s52
+; GCN-NEXT:    s_mov_b32 s15, s53
 ; GCN-NEXT:    s_mov_b64 s[4:5], s[34:35]
 ; GCN-NEXT:    s_mov_b64 s[6:7], s[36:37]
 ; GCN-NEXT:    s_mov_b64 s[8:9], s[38:39]
-; GCN-NEXT:    s_mov_b64 s[10:11], s[40:41]
+; GCN-NEXT:    s_mov_b64 s[10:11], s[48:49]
 ; GCN-NEXT:    v_cmp_nlt_f32_e32 vcc, 0, v0
 ; GCN-NEXT:    s_mov_b64 s[16:17], 0
-; GCN-NEXT:    s_andn2_b64 s[18:19], s[46:47], exec
+; GCN-NEXT:    s_andn2_b64 s[18:19], s[54:55], exec
 ; GCN-NEXT:    s_and_b64 s[20:21], vcc, exec
-; GCN-NEXT:    s_or_b64 s[46:47], s[18:19], s[20:21]
+; GCN-NEXT:    s_or_b64 s[54:55], s[18:19], s[20:21]
 ; GCN-NEXT:  .LBB0_4: ; %Flow2
-; GCN-NEXT:    s_and_saveexec_b64 s[18:19], s[46:47]
+; GCN-NEXT:    s_and_saveexec_b64 s[18:19], s[54:55]
 ; GCN-NEXT:    s_xor_b64 s[18:19], exec, s[18:19]
 ; GCN-NEXT:    s_cbranch_execz .LBB0_6
 ; GCN-NEXT:  ; %bb.5: ; %bb12
@@ -93,14 +93,14 @@ define hidden void @widget() {
 ; GCN-NEXT:    s_addc_u32 s17, s17, wibble@rel32@hi+12
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GCN-NEXT:  .LBB0_8: ; %UnifiedReturnBlock
-; GCN-NEXT:    v_readlane_b32 s47, v41, 15
-; GCN-NEXT:    v_readlane_b32 s46, v41, 14
-; GCN-NEXT:    v_readlane_b32 s45, v41, 13
-; GCN-NEXT:    v_readlane_b32 s44, v41, 12
-; GCN-NEXT:    v_readlane_b32 s43, v41, 11
-; GCN-NEXT:    v_readlane_b32 s42, v41, 10
-; GCN-NEXT:    v_readlane_b32 s41, v41, 9
-; GCN-NEXT:    v_readlane_b32 s40, v41, 8
+; GCN-NEXT:    v_readlane_b32 s55, v41, 15
+; GCN-NEXT:    v_readlane_b32 s54, v41, 14
+; GCN-NEXT:    v_readlane_b32 s53, v41, 13
+; GCN-NEXT:    v_readlane_b32 s52, v41, 12
+; GCN-NEXT:    v_readlane_b32 s51, v41, 11
+; GCN-NEXT:    v_readlane_b32 s50, v41, 10
+; GCN-NEXT:    v_readlane_b32 s49, v41, 9
+; GCN-NEXT:    v_readlane_b32 s48, v41, 8
 ; GCN-NEXT:    v_readlane_b32 s39, v41, 7
 ; GCN-NEXT:    v_readlane_b32 s38, v41, 6
 ; GCN-NEXT:    v_readlane_b32 s37, v41, 5
@@ -119,7 +119,7 @@ define hidden void @widget() {
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ; GCN-NEXT:  .LBB0_9: ; %bb2
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[46:47], 21, v0
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[54:55], 21, v0
 ; GCN-NEXT:    v_cmp_ne_u32_e64 s[18:19], 21, v0
 ; GCN-NEXT:    s_mov_b64 vcc, exec
 ; GCN-NEXT:    s_cbranch_execnz .LBB0_2
@@ -274,51 +274,51 @@ define hidden void @blam() {
 ; GCN-NEXT:    v_writelane_b32 v45, s37, 5
 ; GCN-NEXT:    v_writelane_b32 v45, s38, 6
 ; GCN-NEXT:    v_writelane_b32 v45, s39, 7
-; GCN-NEXT:    v_writelane_b32 v45, s40, 8
-; GCN-NEXT:    v_writelane_b32 v45, s41, 9
-; GCN-NEXT:    v_writelane_b32 v45, s42, 10
-; GCN-NEXT:    v_writelane_b32 v45, s43, 11
-; GCN-NEXT:    v_writelane_b32 v45, s44, 12
-; GCN-NEXT:    v_writelane_b32 v45, s45, 13
-; GCN-NEXT:    v_writelane_b32 v45, s46, 14
-; GCN-NEXT:    v_writelane_b32 v45, s47, 15
-; GCN-NEXT:    v_writelane_b32 v45, s48, 16
-; GCN-NEXT:    v_writelane_b32 v45, s49, 17
-; GCN-NEXT:    v_writelane_b32 v45, s50, 18
-; GCN-NEXT:    v_writelane_b32 v45, s51, 19
-; GCN-NEXT:    v_writelane_b32 v45, s52, 20
-; GCN-NEXT:    v_writelane_b32 v45, s53, 21
-; GCN-NEXT:    v_writelane_b32 v45, s54, 22
-; GCN-NEXT:    v_writelane_b32 v45, s55, 23
-; GCN-NEXT:    v_writelane_b32 v45, s56, 24
-; GCN-NEXT:    v_writelane_b32 v45, s57, 25
+; GCN-NEXT:    v_writelane_b32 v45, s48, 8
+; GCN-NEXT:    v_writelane_b32 v45, s49, 9
+; GCN-NEXT:    v_writelane_b32 v45, s50, 10
+; GCN-NEXT:    v_writelane_b32 v45, s51, 11
+; GCN-NEXT:    v_writelane_b32 v45, s52, 12
+; GCN-NEXT:    v_writelane_b32 v45, s53, 13
+; GCN-NEXT:    v_writelane_b32 v45, s54, 14
+; GCN-NEXT:    v_writelane_b32 v45, s55, 15
+; GCN-NEXT:    v_writelane_b32 v45, s64, 16
+; GCN-NEXT:    v_writelane_b32 v45, s65, 17
+; GCN-NEXT:    v_writelane_b32 v45, s66, 18
+; GCN-NEXT:    v_writelane_b32 v45, s67, 19
+; GCN-NEXT:    v_writelane_b32 v45, s68, 20
+; GCN-NEXT:    v_writelane_b32 v45, s69, 21
+; GCN-NEXT:    v_writelane_b32 v45, s70, 22
+; GCN-NEXT:    v_writelane_b32 v45, s71, 23
+; GCN-NEXT:    v_writelane_b32 v45, s80, 24
+; GCN-NEXT:    v_writelane_b32 v45, s81, 25
 ; GCN-NEXT:    v_mov_b32_e32 v40, v31
-; GCN-NEXT:    s_mov_b32 s46, s15
-; GCN-NEXT:    s_mov_b32 s47, s14
-; GCN-NEXT:    s_mov_b32 s48, s13
-; GCN-NEXT:    s_mov_b32 s49, s12
+; GCN-NEXT:    s_mov_b32 s54, s15
+; GCN-NEXT:    s_mov_b32 s55, s14
+; GCN-NEXT:    s_mov_b32 s64, s13
+; GCN-NEXT:    s_mov_b32 s65, s12
 ; GCN-NEXT:    s_mov_b64 s[34:35], s[10:11]
 ; GCN-NEXT:    s_mov_b64 s[36:37], s[8:9]
 ; GCN-NEXT:    s_mov_b64 s[38:39], s[6:7]
-; GCN-NEXT:    s_mov_b64 s[40:41], s[4:5]
+; GCN-NEXT:    s_mov_b64 s[48:49], s[4:5]
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0
 ; GCN-NEXT:    v_and_b32_e32 v2, 0x3ff, v40
 ; GCN-NEXT:    flat_load_dword v43, v[0:1]
 ; GCN-NEXT:    v_mov_b32_e32 v42, 0
-; GCN-NEXT:    s_mov_b64 s[50:51], 0
+; GCN-NEXT:    s_mov_b64 s[66:67], 0
 ; GCN-NEXT:    v_lshlrev_b32_e32 v41, 2, v2
 ; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_cmp_eq_f32_e64 s[52:53], 0, v43
-; GCN-NEXT:    v_cmp_neq_f32_e64 s[42:43], 0, v43
+; GCN-NEXT:    v_cmp_eq_f32_e64 s[68:69], 0, v43
+; GCN-NEXT:    v_cmp_neq_f32_e64 s[50:51], 0, v43
 ; GCN-NEXT:    v_mov_b32_e32 v44, 0x7fc00000
 ; GCN-NEXT:    s_branch .LBB1_2
 ; GCN-NEXT:  .LBB1_1: ; %Flow7
 ; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
 ; GCN-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GCN-NEXT:    s_and_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT:    s_or_b64 s[50:51], s[4:5], s[50:51]
-; GCN-NEXT:    s_andn2_b64 exec, exec, s[50:51]
+; GCN-NEXT:    s_or_b64 s[66:67], s[4:5], s[66:67]
+; GCN-NEXT:    s_andn2_b64 exec, exec, s[66:67]
 ; GCN-NEXT:    s_cbranch_execz .LBB1_18
 ; GCN-NEXT:  .LBB1_2: ; %bb2
 ; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -329,26 +329,26 @@ define hidden void @blam() {
 ; GCN-NEXT:    v_cmp_lt_i32_e32 vcc, 2, v0
 ; GCN-NEXT:    s_mov_b64 s[4:5], -1
 ; GCN-NEXT:    s_and_saveexec_b64 s[8:9], vcc
-; GCN-NEXT:    s_xor_b64 s[54:55], exec, s[8:9]
+; GCN-NEXT:    s_xor_b64 s[70:71], exec, s[8:9]
 ; GCN-NEXT:    s_cbranch_execz .LBB1_12
 ; GCN-NEXT:  ; %bb.3: ; %bb6
 ; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
-; GCN-NEXT:    v_cmp_eq_u32_e64 s[44:45], 3, v0
-; GCN-NEXT:    s_and_saveexec_b64 s[56:57], s[44:45]
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[52:53], 3, v0
+; GCN-NEXT:    s_and_saveexec_b64 s[80:81], s[52:53]
 ; GCN-NEXT:    s_cbranch_execz .LBB1_11
 ; GCN-NEXT:  ; %bb.4: ; %bb11
 ; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
 ; GCN-NEXT:    s_getpc_b64 s[16:17]
 ; GCN-NEXT:    s_add_u32 s16, s16, spam@rel32@lo+4
 ; GCN-NEXT:    s_addc_u32 s17, s17, spam@rel32@hi+12
-; GCN-NEXT:    s_mov_b64 s[4:5], s[40:41]
+; GCN-NEXT:    s_mov_b64 s[4:5], s[48:49]
 ; GCN-NEXT:    s_mov_b64 s[6:7], s[38:39]
 ; GCN-NEXT:    s_mov_b64 s[8:9], s[36:37]
 ; GCN-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GCN-NEXT:    s_mov_b32 s12, s49
-; GCN-NEXT:    s_mov_b32 s13, s48
-; GCN-NEXT:    s_mov_b32 s14, s47
-; GCN-NEXT:    s_mov_b32 s15, s46
+; GCN-NEXT:    s_mov_b32 s12, s65
+; GCN-NEXT:    s_mov_b32 s13, s64
+; GCN-NEXT:    s_mov_b32 s14, s55
+; GCN-NEXT:    s_mov_b32 s15, s54
 ; GCN-NEXT:    v_mov_b32_e32 v31, v40
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GCN-NEXT:    v_cmp_neq_f32_e32 vcc, 0, v0
@@ -357,13 +357,13 @@ define hidden void @blam() {
 ; GCN-NEXT:    s_cbranch_execz .LBB1_10
 ; GCN-NEXT:  ; %bb.5: ; %bb14
 ; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
-; GCN-NEXT:    s_mov_b64 s[8:9], s[52:53]
-; GCN-NEXT:    s_and_saveexec_b64 s[6:7], s[42:43]
+; GCN-NEXT:    s_mov_b64 s[8:9], s[68:69]
+; GCN-NEXT:    s_and_saveexec_b64 s[6:7], s[50:51]
 ; GCN-NEXT:    s_cbranch_execz .LBB1_7
 ; GCN-NEXT:  ; %bb.6: ; %bb16
 ; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
 ; GCN-NEXT:    buffer_store_dword v44, off, s[0:3], 0
-; GCN-NEXT:    s_or_b64 s[8:9], s[52:53], exec
+; GCN-NEXT:    s_or_b64 s[8:9], s[68:69], exec
 ; GCN-NEXT:  .LBB1_7: ; %Flow3
 ; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
 ; GCN-NEXT:    s_or_b64 exec, exec, s[6:7]
@@ -382,19 +382,19 @@ define hidden void @blam() {
 ; GCN-NEXT:  .LBB1_10: ; %Flow2
 ; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT:    s_andn2_b64 s[4:5], s[44:45], exec
+; GCN-NEXT:    s_andn2_b64 s[4:5], s[52:53], exec
 ; GCN-NEXT:    s_and_b64 s[8:9], vcc, exec
-; GCN-NEXT:    s_or_b64 s[44:45], s[4:5], s[8:9]
+; GCN-NEXT:    s_or_b64 s[52:53], s[4:5], s[8:9]
 ; GCN-NEXT:    s_and_b64 s[6:7], s[6:7], exec
 ; GCN-NEXT:  .LBB1_11: ; %Flow1
 ; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
-; GCN-NEXT:    s_or_b64 exec, exec, s[56:57]
-; GCN-NEXT:    s_orn2_b64 s[4:5], s[44:45], exec
+; GCN-NEXT:    s_or_b64 exec, exec, s[80:81]
+; GCN-NEXT:    s_orn2_b64 s[4:5], s[52:53], exec
 ; GCN-NEXT:    s_and_b64 s[6:7], s[6:7], exec
 ; GCN-NEXT:    ; implicit-def: $vgpr0
 ; GCN-NEXT:  .LBB1_12: ; %Flow
 ; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
-; GCN-NEXT:    s_andn2_saveexec_b64 s[8:9], s[54:55]
+; GCN-NEXT:    s_andn2_saveexec_b64 s[8:9], s[70:71]
 ; GCN-NEXT:    s_cbranch_execz .LBB1_16
 ; GCN-NEXT:  ; %bb.13: ; %bb8
 ; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
@@ -426,25 +426,25 @@ define hidden void @blam() {
 ; GCN-NEXT:    s_andn2_b64 s[4:5], s[4:5], exec
 ; GCN-NEXT:    s_branch .LBB1_1
 ; GCN-NEXT:  .LBB1_18: ; %DummyReturnBlock
-; GCN-NEXT:    s_or_b64 exec, exec, s[50:51]
-; GCN-NEXT:    v_readlane_b32 s57, v45, 25
-; GCN-NEXT:    v_readlane_b32 s56, v45, 24
-; GCN-NEXT:    v_readlane_b32 s55, v45, 23
-; GCN-NEXT:    v_readlane_b32 s54, v45, 22
-; GCN-NEXT:    v_readlane_b32 s53, v45, 21
-; GCN-NEXT:    v_readlane_b32 s52, v45, 20
-; GCN-NEXT:    v_readlane_b32 s51, v45, 19
-; GCN-NEXT:    v_readlane_b32 s50, v45, 18
-; GCN-NEXT:    v_readlane_b32 s49, v45, 17
-; GCN-NEXT:    v_readlane_b32 s48, v45, 16
-; GCN-NEXT:    v_readlane_b32 s47, v45, 15
-; GCN-NEXT:    v_readlane_b32 s46, v45, 14
-; GCN-NEXT:    v_readlane_b32 s45, v45, 13
-; GCN-NEXT:    v_readlane_b32 s44, v45, 12
-; GCN-NEXT:    v_readlane_b32 s43, v45, 11
-; GCN-NEXT:    v_readlane_b32 s42, v45, 10
-; GCN-NEXT:    v_readlane_b32 s41, v45, 9
-; GCN-NEXT:    v_readlane_b32 s40, v45, 8
+; GCN-NEXT:    s_or_b64 exec, exec, s[66:67]
+; GCN-NEXT:    v_readlane_b32 s81, v45, 25
+; GCN-NEXT:    v_readlane_b32 s80, v45, 24
+; GCN-NEXT:    v_readlane_b32 s71, v45, 23
+; GCN-NEXT:    v_readlane_b32 s70, v45, 22
+; GCN-NEXT:    v_readlane_b32 s69, v45, 21
+; GCN-NEXT:    v_readlane_b32 s68, v45, 20
+; GCN-NEXT:    v_readlane_b32 s67, v45, 19
+; GCN-NEXT:    v_readlane_b32 s66, v45, 18
+; GCN-NEXT:    v_readlane_b32 s65, v45, 17
+; GCN-NEXT:    v_readlane_b32 s64, v45, 16
+; GCN-NEXT:    v_readlane_b32 s55, v45, 15
+; GCN-NEXT:    v_readlane_b32 s54, v45, 14
+; GCN-NEXT:    v_readlane_b32 s53, v45, 13
+; GCN-NEXT:    v_readlane_b32 s52, v45, 12
+; GCN-NEXT:    v_readlane_b32 s51, v45, 11
+; GCN-NEXT:    v_readlane_b32 s50, v45, 10
+; GCN-NEXT:    v_readlane_b32 s49, v45, 9
+; GCN-NEXT:    v_readlane_b32 s48, v45, 8
 ; GCN-NEXT:    v_readlane_b32 s39, v45, 7
 ; GCN-NEXT:    v_readlane_b32 s38, v45, 6
 ; GCN-NEXT:    v_readlane_b32 s37, v45, 5
diff --git a/llvm/test/CodeGen/AMDGPU/use_restore_frame_reg.mir b/llvm/test/CodeGen/AMDGPU/use_restore_frame_reg.mir
index 8a0bf26f81d22..1e815f76ee149 100644
--- a/llvm/test/CodeGen/AMDGPU/use_restore_frame_reg.mir
+++ b/llvm/test/CodeGen/AMDGPU/use_restore_frame_reg.mir
@@ -37,100 +37,74 @@ body:             |
   ; MUBUF-LABEL: name: use_restore_frame_reg
   ; MUBUF: bb.0:
   ; MUBUF-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
-  ; MUBUF-NEXT:   liveins: $vgpr1, $vgpr2
+  ; MUBUF-NEXT:   liveins: $sgpr40, $sgpr41, $vgpr1
   ; MUBUF-NEXT: {{  $}}
-  ; MUBUF-NEXT:   $sgpr4 = COPY $sgpr33
+  ; MUBUF-NEXT:   $sgpr40 = frame-setup COPY $sgpr33
   ; MUBUF-NEXT:   $sgpr33 = frame-setup S_ADD_I32 $sgpr32, 524224, implicit-def $scc
   ; MUBUF-NEXT:   $sgpr33 = frame-setup S_AND_B32 killed $sgpr33, 4294443008, implicit-def dead $scc
-  ; MUBUF-NEXT:   $sgpr6_sgpr7 = S_XOR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-  ; MUBUF-NEXT:   $sgpr5 = S_ADD_I32 $sgpr33, 9961728, implicit-def dead $scc
-  ; MUBUF-NEXT:   BUFFER_STORE_DWORD_OFFSET $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, killed $sgpr5, 0, 0, 0, implicit $exec :: (store (s32) into %stack.20, addrspace 5)
-  ; MUBUF-NEXT:   $exec = S_MOV_B64 killed $sgpr6_sgpr7
-  ; MUBUF-NEXT:   $vgpr2 = SI_SPILL_S32_TO_VGPR $sgpr4, 0, undef $vgpr2
-  ; MUBUF-NEXT:   $vgpr2 = SI_SPILL_S32_TO_VGPR $sgpr34, 1, undef $vgpr2
+  ; MUBUF-NEXT:   $sgpr41 = frame-setup COPY $sgpr34
   ; MUBUF-NEXT:   $sgpr34 = frame-setup COPY $sgpr32
   ; MUBUF-NEXT:   $sgpr32 = frame-setup S_ADD_I32 $sgpr32, 11010048, implicit-def dead $scc
   ; MUBUF-NEXT:   S_CMP_EQ_U32 0, 0, implicit-def $scc
   ; MUBUF-NEXT:   S_NOP 0, implicit-def $sgpr4, implicit-def $sgpr5, implicit-def $sgpr6, implicit-def $sgpr7, implicit-def $sgpr8, implicit-def $sgpr9, implicit-def $sgpr10, implicit-def $sgpr11, implicit-def $sgpr12, implicit-def $sgpr13, implicit-def $sgpr14, implicit-def $sgpr15, implicit-def $sgpr16, implicit-def $sgpr17, implicit-def $sgpr18, implicit-def $sgpr19, implicit-def $sgpr20, implicit-def $sgpr21, implicit-def $sgpr22, implicit-def $sgpr23, implicit-def $sgpr24, implicit-def $sgpr25, implicit-def $sgpr26, implicit-def $sgpr27, implicit-def $sgpr28, implicit-def $sgpr29, implicit-def $sgpr30, implicit-def $sgpr31, implicit-def $vcc
   ; MUBUF-NEXT:   $vgpr0 = V_LSHRREV_B32_e64 6, $sgpr33, implicit $exec
   ; MUBUF-NEXT:   $vgpr0 = V_ADD_U32_e32 8192, killed $vgpr0, implicit $exec
-  ; MUBUF-NEXT:   $vgpr3 = V_LSHRREV_B32_e64 6, $sgpr33, implicit $exec
-  ; MUBUF-NEXT:   $vgpr3 = V_ADD_U32_e32 155648, killed $vgpr3, implicit $exec
-  ; MUBUF-NEXT:   $vgpr0 = V_OR_B32_e32 killed $vgpr3, $vgpr1, implicit $exec, implicit $sgpr4, implicit $sgpr5, implicit $sgpr6, implicit $sgpr7, implicit $sgpr8, implicit $sgpr9, implicit $sgpr10, implicit $sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr16, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $sgpr28, implicit $sgpr29, implicit $sgpr30, implicit $sgpr31
+  ; MUBUF-NEXT:   $vgpr2 = V_LSHRREV_B32_e64 6, $sgpr33, implicit $exec
+  ; MUBUF-NEXT:   $vgpr2 = V_ADD_U32_e32 155648, killed $vgpr2, implicit $exec
+  ; MUBUF-NEXT:   $vgpr0 = V_OR_B32_e32 killed $vgpr2, $vgpr1, implicit $exec, implicit $sgpr4, implicit $sgpr5, implicit $sgpr6, implicit $sgpr7, implicit $sgpr8, implicit $sgpr9, implicit $sgpr10, implicit $sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr16, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $sgpr28, implicit $sgpr29, implicit $sgpr30, implicit $sgpr31
   ; MUBUF-NEXT:   S_CBRANCH_VCCNZ %bb.2, implicit $vcc
   ; MUBUF-NEXT:   S_CBRANCH_SCC1 %bb.2, implicit $scc
   ; MUBUF-NEXT: {{  $}}
   ; MUBUF-NEXT: bb.1:
   ; MUBUF-NEXT:   successors: %bb.2(0x80000000)
-  ; MUBUF-NEXT:   liveins: $vgpr2
+  ; MUBUF-NEXT:   liveins: $sgpr40, $sgpr41
   ; MUBUF-NEXT: {{  $}}
   ; MUBUF-NEXT:   S_NOP 0
   ; MUBUF-NEXT: {{  $}}
   ; MUBUF-NEXT: bb.2:
-  ; MUBUF-NEXT:   liveins: $vgpr2
+  ; MUBUF-NEXT:   liveins: $sgpr40, $sgpr41
   ; MUBUF-NEXT: {{  $}}
   ; MUBUF-NEXT:   $sgpr32 = frame-destroy COPY $sgpr34
-  ; MUBUF-NEXT:   $sgpr4 = SI_RESTORE_S32_FROM_VGPR $vgpr2, 0
-  ; MUBUF-NEXT:   $sgpr34 = SI_RESTORE_S32_FROM_VGPR $vgpr2, 1
-  ; MUBUF-NEXT:   $sgpr6_sgpr7 = S_XOR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-  ; MUBUF-NEXT:   $sgpr5 = S_ADD_I32 $sgpr33, 9961728, implicit-def dead $scc
-  ; MUBUF-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, killed $sgpr5, 0, 0, 0, implicit $exec :: (load (s32) from %stack.20, addrspace 5)
-  ; MUBUF-NEXT:   $exec = S_MOV_B64 killed $sgpr6_sgpr7
-  ; MUBUF-NEXT:   $sgpr33 = COPY $sgpr4
+  ; MUBUF-NEXT:   $sgpr34 = frame-destroy COPY $sgpr41
+  ; MUBUF-NEXT:   $sgpr33 = frame-destroy COPY $sgpr40
   ; MUBUF-NEXT:   S_ENDPGM 0
   ;
   ; FLATSCR-LABEL: name: use_restore_frame_reg
   ; FLATSCR: bb.0:
   ; FLATSCR-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
-  ; FLATSCR-NEXT:   liveins: $vgpr1, $vgpr2
+  ; FLATSCR-NEXT:   liveins: $sgpr40, $sgpr41, $vgpr1
   ; FLATSCR-NEXT: {{  $}}
-  ; FLATSCR-NEXT:   $sgpr4 = COPY $sgpr33
+  ; FLATSCR-NEXT:   $sgpr40 = frame-setup COPY $sgpr33
   ; FLATSCR-NEXT:   $sgpr33 = frame-setup S_ADD_I32 $sgpr32, 8191, implicit-def $scc
   ; FLATSCR-NEXT:   $sgpr33 = frame-setup S_AND_B32 killed $sgpr33, 4294959104, implicit-def dead $scc
-  ; FLATSCR-NEXT:   $sgpr6_sgpr7 = S_XOR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-  ; FLATSCR-NEXT:   $sgpr5 = S_ADD_I32 $sgpr33, 155652, implicit-def dead $scc
-  ; FLATSCR-NEXT:   SCRATCH_STORE_DWORD_SADDR $vgpr2, killed $sgpr5, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %stack.20, addrspace 5)
-  ; FLATSCR-NEXT:   $exec = S_MOV_B64 killed $sgpr6_sgpr7
-  ; FLATSCR-NEXT:   $vgpr2 = SI_SPILL_S32_TO_VGPR $sgpr4, 0, undef $vgpr2
-  ; FLATSCR-NEXT:   $vgpr2 = SI_SPILL_S32_TO_VGPR $sgpr34, 1, undef $vgpr2
+  ; FLATSCR-NEXT:   $sgpr41 = frame-setup COPY $sgpr34
   ; FLATSCR-NEXT:   $sgpr34 = frame-setup COPY $sgpr32
   ; FLATSCR-NEXT:   $sgpr32 = frame-setup S_ADD_I32 $sgpr32, 172032, implicit-def dead $scc
   ; FLATSCR-NEXT:   S_CMP_EQ_U32 0, 0, implicit-def $scc
   ; FLATSCR-NEXT:   S_NOP 0, implicit-def $sgpr4, implicit-def $sgpr5, implicit-def $sgpr6, implicit-def $sgpr7, implicit-def $sgpr8, implicit-def $sgpr9, implicit-def $sgpr10, implicit-def $sgpr11, implicit-def $sgpr12, implicit-def $sgpr13, implicit-def $sgpr14, implicit-def $sgpr15, implicit-def $sgpr16, implicit-def $sgpr17, implicit-def $sgpr18, implicit-def $sgpr19, implicit-def $sgpr20, implicit-def $sgpr21, implicit-def $sgpr22, implicit-def $sgpr23, implicit-def $sgpr24, implicit-def $sgpr25, implicit-def $sgpr26, implicit-def $sgpr27, implicit-def $sgpr28, implicit-def $sgpr29, implicit-def $sgpr30, implicit-def $sgpr31, implicit-def $vcc
-  ; FLATSCR-NEXT:   $sgpr33 = S_ADDC_U32 $sgpr33, 8192, implicit-def $scc, implicit $scc
-  ; FLATSCR-NEXT:   S_BITCMP1_B32 $sgpr33, 0, implicit-def $scc
-  ; FLATSCR-NEXT:   $sgpr33 = S_BITSET0_B32 0, $sgpr33
-  ; FLATSCR-NEXT:   $vgpr0 = V_MOV_B32_e32 $sgpr33, implicit $exec
-  ; FLATSCR-NEXT:   $sgpr33 = S_ADDC_U32 $sgpr33, -8192, implicit-def $scc, implicit $scc
-  ; FLATSCR-NEXT:   S_BITCMP1_B32 $sgpr33, 0, implicit-def $scc
-  ; FLATSCR-NEXT:   $sgpr33 = S_BITSET0_B32 0, $sgpr33
-  ; FLATSCR-NEXT:   $sgpr33 = S_ADDC_U32 $sgpr33, 155648, implicit-def $scc, implicit $scc
-  ; FLATSCR-NEXT:   S_BITCMP1_B32 $sgpr33, 0, implicit-def $scc
-  ; FLATSCR-NEXT:   $sgpr33 = S_BITSET0_B32 0, $sgpr33
-  ; FLATSCR-NEXT:   $vgpr0 = V_OR_B32_e32 $sgpr33, $vgpr1, implicit $exec, implicit $sgpr4, implicit $sgpr5, implicit $sgpr6, implicit $sgpr7, implicit $sgpr8, implicit $sgpr9, implicit $sgpr10, implicit $sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr16, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $sgpr28, implicit $sgpr29, implicit $sgpr30, implicit $sgpr31
-  ; FLATSCR-NEXT:   $sgpr33 = S_ADDC_U32 $sgpr33, -155648, implicit-def $scc, implicit $scc
-  ; FLATSCR-NEXT:   S_BITCMP1_B32 $sgpr33, 0, implicit-def $scc
-  ; FLATSCR-NEXT:   $sgpr33 = S_BITSET0_B32 0, $sgpr33
+  ; FLATSCR-NEXT:   $sgpr42 = S_ADDC_U32 $sgpr33, 8192, implicit-def $scc, implicit $scc
+  ; FLATSCR-NEXT:   S_BITCMP1_B32 $sgpr42, 0, implicit-def $scc
+  ; FLATSCR-NEXT:   $sgpr42 = S_BITSET0_B32 0, $sgpr42
+  ; FLATSCR-NEXT:   $vgpr0 = V_MOV_B32_e32 killed $sgpr42, implicit $exec
+  ; FLATSCR-NEXT:   $sgpr42 = S_ADDC_U32 $sgpr33, 155648, implicit-def $scc, implicit $scc
+  ; FLATSCR-NEXT:   S_BITCMP1_B32 $sgpr42, 0, implicit-def $scc
+  ; FLATSCR-NEXT:   $sgpr42 = S_BITSET0_B32 0, $sgpr42
+  ; FLATSCR-NEXT:   $vgpr0 = V_OR_B32_e32 killed $sgpr42, $vgpr1, implicit $exec, implicit $sgpr4, implicit $sgpr5, implicit $sgpr6, implicit $sgpr7, implicit $sgpr8, implicit $sgpr9, implicit $sgpr10, implicit $sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr16, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $sgpr28, implicit $sgpr29, implicit $sgpr30, implicit $sgpr31
   ; FLATSCR-NEXT:   S_CBRANCH_VCCNZ %bb.2, implicit $vcc
   ; FLATSCR-NEXT:   S_CBRANCH_SCC1 %bb.2, implicit $scc
   ; FLATSCR-NEXT: {{  $}}
   ; FLATSCR-NEXT: bb.1:
   ; FLATSCR-NEXT:   successors: %bb.2(0x80000000)
-  ; FLATSCR-NEXT:   liveins: $vgpr2
+  ; FLATSCR-NEXT:   liveins: $sgpr40, $sgpr41
   ; FLATSCR-NEXT: {{  $}}
   ; FLATSCR-NEXT:   S_NOP 0
   ; FLATSCR-NEXT: {{  $}}
   ; FLATSCR-NEXT: bb.2:
-  ; FLATSCR-NEXT:   liveins: $vgpr2
+  ; FLATSCR-NEXT:   liveins: $sgpr40, $sgpr41
   ; FLATSCR-NEXT: {{  $}}
   ; FLATSCR-NEXT:   $sgpr32 = frame-destroy COPY $sgpr34
-  ; FLATSCR-NEXT:   $sgpr4 = SI_RESTORE_S32_FROM_VGPR $vgpr2, 0
-  ; FLATSCR-NEXT:   $sgpr34 = SI_RESTORE_S32_FROM_VGPR $vgpr2, 1
-  ; FLATSCR-NEXT:   $sgpr6_sgpr7 = S_XOR_SAVEEXEC_B64 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-  ; FLATSCR-NEXT:   $sgpr5 = S_ADD_I32 $sgpr33, 155652, implicit-def dead $scc
-  ; FLATSCR-NEXT:   $vgpr2 = SCRATCH_LOAD_DWORD_SADDR killed $sgpr5, 0, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %stack.20, addrspace 5)
-  ; FLATSCR-NEXT:   $exec = S_MOV_B64 killed $sgpr6_sgpr7
-  ; FLATSCR-NEXT:   $sgpr33 = COPY $sgpr4
+  ; FLATSCR-NEXT:   $sgpr34 = frame-destroy COPY $sgpr41
+  ; FLATSCR-NEXT:   $sgpr33 = frame-destroy COPY $sgpr40
   ; FLATSCR-NEXT:   S_ENDPGM 0
   bb.0:
     liveins: $vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/v_pack.ll b/llvm/test/CodeGen/AMDGPU/v_pack.ll
index 7b60ea100a51c..8a8829832f688 100644
--- a/llvm/test/CodeGen/AMDGPU/v_pack.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_pack.ll
@@ -130,7 +130,7 @@ define amdgpu_kernel void @v_pack_b32_v2f16(ptr addrspace(1) %in0, ptr addrspace
   %v1 = load volatile half, ptr addrspace(1) %in1.gep
   %v0.add = fadd half %v0, 2.0
   %v1.add = fadd half %v1, 2.0
-  %vec.0 = insertelement <2 x half> undef, half %v0.add, i32 0
+  %vec.0 = insertelement <2 x half> poison, half %v0.add, i32 0
   %vec.1 = insertelement <2 x half> %vec.0, half %v1.add, i32 1
   %vec.i32 = bitcast <2 x half> %vec.1 to i32
   call void asm sideeffect "; use $0", "v"(i32 %vec.i32) #0
@@ -259,7 +259,7 @@ define amdgpu_kernel void @v_pack_b32_v2f16_sub(ptr addrspace(1) %in0, ptr addrs
   %v1 = load volatile half, ptr addrspace(1) %in1.gep
   %v0.add = fsub half %v0, 2.0
   %v1.add = fadd half %v1, 2.0
-  %vec.0 = insertelement <2 x half> undef, half %v0.add, i32 0
+  %vec.0 = insertelement <2 x half> poison, half %v0.add, i32 0
   %vec.1 = insertelement <2 x half> %vec.0, half %v1.add, i32 1
   %vec.i32 = bitcast <2 x half> %vec.1 to i32
   call void asm sideeffect "; use $0", "v"(i32 %vec.i32) #0
@@ -504,7 +504,7 @@ define amdgpu_kernel void @v_pack_b32.fabs(ptr addrspace(1) %in0, ptr addrspace(
   %v1.add = fadd half %v1, 2.0
   %v0.fabs = call half @llvm.fabs.f16(half %v0.add)
   %v1.fabs = call half @llvm.fabs.f16(half %v1.add)
-  %vec.0 = insertelement <2 x half> undef, half %v0.fabs, i32 0
+  %vec.0 = insertelement <2 x half> poison, half %v0.fabs, i32 0
   %vec.1 = insertelement <2 x half> %vec.0, half %v1.fabs, i32 1
   %vec.i32 = bitcast <2 x half> %vec.1 to i32
   call void asm sideeffect "; use $0", "v"(i32 %vec.i32) #0
@@ -635,7 +635,7 @@ define amdgpu_kernel void @v_pack_b32.fneg(ptr addrspace(1) %in0, ptr addrspace(
   %v1.add = fadd half %v1, 2.0
   %v0.fneg = fsub half -0.0, %v0.add
   %v1.fneg = fsub half -0.0, %v1.add
-  %vec.0 = insertelement <2 x half> undef, half %v0.fneg, i32 0
+  %vec.0 = insertelement <2 x half> poison, half %v0.fneg, i32 0
   %vec.1 = insertelement <2 x half> %vec.0, half %v1.fneg, i32 1
   %vec.i32 = bitcast <2 x half> %vec.1 to i32
   call void asm sideeffect "; use $0", "v"(i32 %vec.i32) #0
diff --git a/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll b/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
index 2d84e87722951..b81f5d0a19ba8 100644
--- a/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
@@ -263,7 +263,7 @@ define amdgpu_kernel void @basic_smax_smin_sgpr(ptr addrspace(1) %out, i32 inreg
   %src0.clamp = call i16 @llvm.smin.i16(i16 %src0.max, i16 255)
   %src1.max = call i16 @llvm.smax.i16(i16 %src1, i16 0)
   %src1.clamp = call i16 @llvm.smin.i16(i16 %src1.max, i16 255)
-  %insert.0 = insertelement <2 x i16> undef, i16 %src0.clamp, i32 0
+  %insert.0 = insertelement <2 x i16> poison, i16 %src0.clamp, i32 0
   %vec = insertelement <2 x i16> %insert.0, i16 %src1.clamp, i32 1
   store <2 x i16> %vec, ptr addrspace(1) %out
   ret void
@@ -362,7 +362,7 @@ define <2 x i16> @basic_smin_smax(i16 %src0, i16 %src1) {
   %src0.clamp = call i16 @llvm.smax.i16(i16 %src0.min, i16 0)
   %src1.min = call i16 @llvm.smin.i16(i16 %src1, i16 255)
   %src1.clamp = call i16 @llvm.smax.i16(i16 %src1.min, i16 0)
-  %insert.0 = insertelement <2 x i16> undef, i16 %src0.clamp, i32 0
+  %insert.0 = insertelement <2 x i16> poison, i16 %src0.clamp, i32 0
   %vec = insertelement <2 x i16> %insert.0, i16 %src1.clamp, i32 1
   ret <2 x i16> %vec
 }
@@ -460,7 +460,7 @@ define <2 x i16> @basic_smin_smax_combined(i16 %src0, i16 %src1) {
   %src0.clamp = call i16 @llvm.smax.i16(i16 %src0.min, i16 0)
   %src1.max = call i16 @llvm.smax.i16(i16 %src1, i16 0)
   %src1.clamp = call i16 @llvm.smin.i16(i16 %src1.max, i16 255)
-  %insert.0 = insertelement <2 x i16> undef, i16 %src0.clamp, i32 0
+  %insert.0 = insertelement <2 x i16> poison, i16 %src0.clamp, i32 0
   %vec = insertelement <2 x i16> %insert.0, i16 %src1.clamp, i32 1
   ret <2 x i16> %vec
 }
@@ -1066,7 +1066,7 @@ define i16 @basic_smax_smin_vec_cast(i16 %src0, i16 %src1) {
   %src0.clamp = call i16 @llvm.smin.i16(i16 %src0.max, i16 255)
   %src1.max = call i16 @llvm.smax.i16(i16 %src1, i16 0)
   %src1.clamp = call i16 @llvm.smin.i16(i16 %src1.max, i16 255)
-  %insert.0 = insertelement <2 x i16> undef, i16 %src0.clamp, i32 0
+  %insert.0 = insertelement <2 x i16> poison, i16 %src0.clamp, i32 0
   %vec = insertelement <2 x i16> %insert.0, i16 %src1.clamp, i32 1
   %vec.trunc = trunc <2 x i16> %vec to <2 x i8>
   %cast = bitcast <2 x i8> %vec.trunc to i16
diff --git a/llvm/test/CodeGen/AMDGPU/vector-alloca-limits.ll b/llvm/test/CodeGen/AMDGPU/vector-alloca-limits.ll
index 64d7c7868ca8d..fee045359e751 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-alloca-limits.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-alloca-limits.ll
@@ -1,5 +1,5 @@
-; RUN: opt -S -mtriple=amdgcn-- -passes='amdgpu-promote-alloca,sroa,instcombine' < %s | FileCheck -check-prefix=OPT %s
-; RUN: opt -S -mtriple=amdgcn-- -passes='amdgpu-promote-alloca,sroa,instcombine' -amdgpu-promote-alloca-to-vector-limit=32 < %s | FileCheck -check-prefix=LIMIT32 %s
+; RUN: opt -S -mtriple=amdgcn-- -passes='amdgpu-promote-alloca,sroa,instcombine' -amdgpu-promote-alloca-to-vector-max-regs=64 < %s | FileCheck -check-prefix=OPT %s
+; RUN: opt -S -mtriple=amdgcn-- -passes='amdgpu-promote-alloca,sroa,instcombine' -amdgpu-promote-alloca-to-vector-limit=32 -amdgpu-promote-alloca-to-vector-max-regs=64 < %s | FileCheck -check-prefix=LIMIT32 %s
 
 target datalayout = "A5"
 
diff --git a/llvm/test/CodeGen/AMDGPU/vector-legalizer-divergence.ll b/llvm/test/CodeGen/AMDGPU/vector-legalizer-divergence.ll
index e2ba9d0c087ad..1daba85ae0122 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-legalizer-divergence.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-legalizer-divergence.ll
@@ -11,7 +11,7 @@ define amdgpu_kernel void @spam(ptr addrspace(1) noalias %arg) {
   %tmp2 = getelementptr inbounds double, ptr addrspace(1) %arg, i64 %tmp1
   %tmp3 = load double, ptr addrspace(1) %tmp2, align 8
   %tmp4 = fadd double undef, 0.000000e+00
-  %tmp5 = insertelement <2 x double> undef, double %tmp4, i64 0
+  %tmp5 = insertelement <2 x double> poison, double %tmp4, i64 0
   %tmp6 = insertelement <2 x double> %tmp5, double %tmp3, i64 1
   %tmp7 = insertelement <2 x double> %tmp6, double 0.000000e+00, i64 1
   %tmp8 = fadd <2 x double> zeroinitializer, undef
diff --git a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
index e9ae1ebe4572b..bc8d0d1f8cd8f 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
@@ -27,7 +27,7 @@ define <4 x half> @shuffle_v4f16_23uu(ptr addrspace(1) %arg0, ptr addrspace(1) %
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x half>, ptr addrspace(1) %arg0
   %val1 = load <4 x half>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   ret <4 x half> %shuffle
 }
 
@@ -74,7 +74,7 @@ define <4 x half> @shuffle_v4f16_234u(ptr addrspace(1) %arg0, ptr addrspace(1) %
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x half>, ptr addrspace(1) %arg0
   %val1 = load <4 x half>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 4, i32 undef>
+  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 4, i32 poison>
   ret <4 x half> %shuffle
 }
 
@@ -101,7 +101,7 @@ define <4 x half> @shuffle_v4f16_u1u3(ptr addrspace(1) %arg0, ptr addrspace(1) %
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x half>, ptr addrspace(1) %arg0
   %val1 = load <4 x half>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 undef, i32 1, i32 undef, i32 3>
+  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 poison, i32 1, i32 poison, i32 3>
   ret <4 x half> %shuffle
 }
 
@@ -140,7 +140,7 @@ define <4 x half> @shuffle_v4f16_u3u1(ptr addrspace(1) %arg0, ptr addrspace(1) %
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x half>, ptr addrspace(1) %arg0
   %val1 = load <4 x half>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 undef, i32 3, i32 undef, i32 1>
+  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 poison, i32 3, i32 poison, i32 1>
   ret <4 x half> %shuffle
 }
 
@@ -167,7 +167,7 @@ define <4 x half> @shuffle_v4f16_u3uu(ptr addrspace(1) %arg0, ptr addrspace(1) %
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x half>, ptr addrspace(1) %arg0
   %val1 = load <4 x half>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 undef, i32 3, i32 undef, i32 undef>
+  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 poison, i32 3, i32 poison, i32 poison>
   ret <4 x half> %shuffle
 }
 
@@ -216,7 +216,7 @@ define <4 x half> @shuffle_v4f16_3u6u(ptr addrspace(1) %arg0, ptr addrspace(1) %
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x half>, ptr addrspace(1) %arg0
   %val1 = load <4 x half>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 undef, i32 6, i32 undef>
+  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 poison, i32 6, i32 poison>
   ret <4 x half> %shuffle
 }
 
@@ -265,7 +265,7 @@ define <4 x half> @shuffle_v4f16_3uu7(ptr addrspace(1) %arg0, ptr addrspace(1) %
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x half>, ptr addrspace(1) %arg0
   %val1 = load <4 x half>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 undef, i32 undef, i32 7>
+  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 poison, i32 poison, i32 7>
   ret <4 x half> %shuffle
 }
 
@@ -312,7 +312,7 @@ define <4 x half> @shuffle_v4f16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1) %
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x half>, ptr addrspace(1) %arg0
   %val1 = load <4 x half>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 5, i32 undef, i32 5>
+  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 5, i32 poison, i32 5>
   ret <4 x half> %shuffle
 }
 
@@ -364,7 +364,7 @@ define <4 x half> @shuffle_v4f16_357u(ptr addrspace(1) %arg0, ptr addrspace(1) %
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x half>, ptr addrspace(1) %arg0
   %val1 = load <4 x half>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 5, i32 7, i32 undef>
+  %shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 5, i32 7, i32 poison>
   ret <4 x half> %shuffle
 }
 
@@ -1881,21 +1881,21 @@ entry:
   %tmp15 = load <4 x half>, ptr addrspace(1) %arrayidx1, align 8
   %arrayidx2 = getelementptr inbounds <4 x half>, ptr addrspace(1) %C, i64 %tmp12
   %tmp16 = load <4 x half>, ptr addrspace(1) %arrayidx2, align 8
-  %tmp17 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> zeroinitializer
-  %tmp18 = shufflevector <4 x half> %tmp15, <4 x half> undef, <2 x i32> <i32 0, i32 1>
-  %tmp19 = shufflevector <4 x half> %tmp16, <4 x half> undef, <2 x i32> <i32 0, i32 1>
+  %tmp17 = shufflevector <4 x half> %tmp14, <4 x half> poison, <2 x i32> zeroinitializer
+  %tmp18 = shufflevector <4 x half> %tmp15, <4 x half> poison, <2 x i32> <i32 0, i32 1>
+  %tmp19 = shufflevector <4 x half> %tmp16, <4 x half> poison, <2 x i32> <i32 0, i32 1>
   %tmp20 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp17, <2 x half> %tmp18, <2 x half> %tmp19)
-  %tmp21 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> <i32 1, i32 1>
-  %tmp22 = shufflevector <4 x half> %tmp15, <4 x half> undef, <2 x i32> <i32 2, i32 3>
+  %tmp21 = shufflevector <4 x half> %tmp14, <4 x half> poison, <2 x i32> <i32 1, i32 1>
+  %tmp22 = shufflevector <4 x half> %tmp15, <4 x half> poison, <2 x i32> <i32 2, i32 3>
   %tmp23 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp21, <2 x half> %tmp22, <2 x half> %tmp20)
-  %tmp24 = shufflevector <2 x half> %tmp23, <2 x half> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %tmp24 = shufflevector <2 x half> %tmp23, <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   %tmp25 = shufflevector <4 x half> %tmp24, <4 x half> %tmp16, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-  %tmp26 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> <i32 2, i32 2>
-  %tmp27 = shufflevector <4 x half> %tmp25, <4 x half> undef, <2 x i32> <i32 2, i32 3>
+  %tmp26 = shufflevector <4 x half> %tmp14, <4 x half> poison, <2 x i32> <i32 2, i32 2>
+  %tmp27 = shufflevector <4 x half> %tmp25, <4 x half> poison, <2 x i32> <i32 2, i32 3>
   %tmp28 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp26, <2 x half> %tmp18, <2 x half> %tmp27)
-  %tmp29 = shufflevector <4 x half> %tmp14, <4 x half> undef, <2 x i32> <i32 3, i32 3>
+  %tmp29 = shufflevector <4 x half> %tmp14, <4 x half> poison, <2 x i32> <i32 3, i32 3>
   %tmp30 = tail call <2 x half> @llvm.fma.v2f16(<2 x half> %tmp29, <2 x half> %tmp22, <2 x half> %tmp28)
-  %tmp31 = shufflevector <2 x half> %tmp30, <2 x half> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %tmp31 = shufflevector <2 x half> %tmp30, <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   %tmp32 = shufflevector <4 x half> %tmp25, <4 x half> %tmp31, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
   store <4 x half> %tmp32, ptr addrspace(1) %arrayidx2, align 8
   ret void
@@ -2006,7 +2006,7 @@ define amdgpu_kernel void @shuffle_scalar_load_v8i32_0123(ptr addrspace(4) %in,
 ; GFX11-NEXT:    global_store_b128 v4, v[0:3], s[2:3]
 ; GFX11-NEXT:    s_endpgm
   %ld8 = load <8 x i32>, ptr addrspace(4) %in, align 16
-  %id = shufflevector <8 x i32> %ld8, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %id = shufflevector <8 x i32> %ld8, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   store <4 x i32> %id, ptr addrspace(1) %out, align 8
   ret void
 }
@@ -2052,7 +2052,7 @@ define <2 x half> @low16bits_v2f16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
 entry:
   %0 = load <2 x half>, ptr addrspace(1) %x0, align 4
   %1 = load <2 x half>, ptr addrspace(1) %x1, align 4
-  %vy1.0.vec.insert = shufflevector <2 x half> %0, <2 x half> poison, <2 x i32> <i32 0, i32 undef>
+  %vy1.0.vec.insert = shufflevector <2 x half> %0, <2 x half> poison, <2 x i32> <i32 0, i32 poison>
   %vy1.2.vec.insert = shufflevector <2 x half> %vy1.0.vec.insert, <2 x half> %1, <2 x i32> <i32 0, i32 2>
   ret <2 x half> %vy1.2.vec.insert
 }
@@ -2098,7 +2098,7 @@ define <2 x half> @hi16bits_v2f16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
 entry:
   %0 = load <2 x half>, ptr addrspace(1) %x0, align 4
   %1 = load <2 x half>, ptr addrspace(1) %x1, align 4
-  %vy1.0.vec.insert = shufflevector <2 x half> %0, <2 x half> poison, <2 x i32> <i32 1, i32 undef>
+  %vy1.0.vec.insert = shufflevector <2 x half> %0, <2 x half> poison, <2 x i32> <i32 1, i32 poison>
   %vy1.2.vec.insert = shufflevector <2 x half> %vy1.0.vec.insert, <2 x half> %1, <2 x i32> <i32 0, i32 3>
   ret <2 x half> %vy1.2.vec.insert
 }
@@ -2144,7 +2144,7 @@ define <2 x half> @low16hi16bits_v2f16(ptr addrspace(1) %x0, ptr addrspace(1) %x
 entry:
   %0 = load <2 x half>, ptr addrspace(1) %x0, align 4
   %1 = load <2 x half>, ptr addrspace(1) %x1, align 4
-  %vy1.0.vec.insert = shufflevector <2 x half> %0, <2 x half> poison, <2 x i32> <i32 0, i32 undef>
+  %vy1.0.vec.insert = shufflevector <2 x half> %0, <2 x half> poison, <2 x i32> <i32 0, i32 poison>
   %vy1.2.vec.insert = shufflevector <2 x half> %vy1.0.vec.insert, <2 x half> %1, <2 x i32> <i32 0, i32 3>
   ret <2 x half> %vy1.2.vec.insert
 }
@@ -2179,7 +2179,7 @@ define <2 x half> @hi16low16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1) %
 entry:
   %0 = load <2 x half>, ptr addrspace(1) %x0, align 4
   %1 = load <2 x half>, ptr addrspace(1) %x1, align 4
-  %vy1.0.vec.insert = shufflevector <2 x half> %0, <2 x half> poison, <2 x i32> <i32 1, i32 undef>
+  %vy1.0.vec.insert = shufflevector <2 x half> %0, <2 x half> poison, <2 x i32> <i32 1, i32 poison>
   %vy1.2.vec.insert = shufflevector <2 x half> %vy1.0.vec.insert, <2 x half> %1, <2 x i32> <i32 0, i32 2>
   ret <2 x half> %vy1.2.vec.insert
 }
@@ -2225,7 +2225,7 @@ define <2 x i16> @i16_low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
 entry:
   %0 = load <2 x i16>, ptr addrspace(1) %x0, align 4
   %1 = load <2 x i16>, ptr addrspace(1) %x1, align 4
-  %vy1.0.vec.insert = shufflevector <2 x i16> %0, <2 x i16> poison, <2 x i32> <i32 0, i32 undef>
+  %vy1.0.vec.insert = shufflevector <2 x i16> %0, <2 x i16> poison, <2 x i32> <i32 0, i32 poison>
   %vy1.2.vec.insert = shufflevector <2 x i16> %vy1.0.vec.insert, <2 x i16> %1, <2 x i32> <i32 0, i32 2>
   ret <2 x i16> %vy1.2.vec.insert
 }
@@ -2271,7 +2271,7 @@ define <2 x i16> @i16_low16hi16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1)
 entry:
   %0 = load <2 x i16>, ptr addrspace(1) %x0, align 4
   %1 = load <2 x i16>, ptr addrspace(1) %x1, align 4
-  %vy1.0.vec.insert = shufflevector <2 x i16> %0, <2 x i16> poison, <2 x i32> <i32 0, i32 undef>
+  %vy1.0.vec.insert = shufflevector <2 x i16> %0, <2 x i16> poison, <2 x i32> <i32 0, i32 poison>
   %vy1.2.vec.insert = shufflevector <2 x i16> %vy1.0.vec.insert, <2 x i16> %1, <2 x i32> <i32 0, i32 3>
   ret <2 x i16> %vy1.2.vec.insert
 }
@@ -2306,7 +2306,7 @@ define <2 x i16> @i16_hi16low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1)
 entry:
   %0 = load <2 x i16>, ptr addrspace(1) %x0, align 4
   %1 = load <2 x i16>, ptr addrspace(1) %x1, align 4
-  %vy1.0.vec.insert = shufflevector <2 x i16> %0, <2 x i16> poison, <2 x i32> <i32 1, i32 undef>
+  %vy1.0.vec.insert = shufflevector <2 x i16> %0, <2 x i16> poison, <2 x i32> <i32 1, i32 poison>
   %vy1.2.vec.insert = shufflevector <2 x i16> %vy1.0.vec.insert, <2 x i16> %1, <2 x i32> <i32 0, i32 2>
   ret <2 x i16> %vy1.2.vec.insert
 }
@@ -2352,7 +2352,7 @@ define <2 x i16> @i16_hi16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
 entry:
   %0 = load <2 x i16>, ptr addrspace(1) %x0, align 4
   %1 = load <2 x i16>, ptr addrspace(1) %x1, align 4
-  %vy1.0.vec.insert = shufflevector <2 x i16> %0, <2 x i16> poison, <2 x i32> <i32 1, i32 undef>
+  %vy1.0.vec.insert = shufflevector <2 x i16> %0, <2 x i16> poison, <2 x i32> <i32 1, i32 poison>
   %vy1.2.vec.insert = shufflevector <2 x i16> %vy1.0.vec.insert, <2 x i16> %1, <2 x i32> <i32 0, i32 3>
   ret <2 x i16> %vy1.2.vec.insert
 }
@@ -2383,7 +2383,7 @@ define <2 x i16> @v2i16_hi16bits(ptr addrspace(1) %x0) {
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load0 = load <2 x i16>, ptr addrspace(1) %x0, align 4
-  %insert1 = insertelement <2 x i16> undef, i16 0, i32 0
+  %insert1 = insertelement <2 x i16> poison, i16 0, i32 0
   %insert2 = insertelement <2 x i16> %insert1, i16 0, i32 1
   %vec.ret = shufflevector <2 x i16> %insert2, <2 x i16> %load0, <2 x i32> <i32 0, i32 3>
   ret <2 x i16> %vec.ret
@@ -2415,7 +2415,7 @@ define <2 x half> @v2half_hi16bits(ptr addrspace(1) %x0) {
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load0 = load <2 x half>, ptr addrspace(1) %x0, align 4
-  %insert1 = insertelement <2 x half> undef, half 0.0, i32 0
+  %insert1 = insertelement <2 x half> poison, half 0.0, i32 0
   %insert2 = insertelement <2 x half> %insert1, half 0.0, i32 1
   %vec.ret = shufflevector <2 x half> %insert2, <2 x half> %load0, <2 x i32> <i32 0, i32 3>
   ret <2 x half> %vec.ret
@@ -3020,7 +3020,7 @@ define <4 x bfloat> @shuffle_v4bf16_23uu(ptr addrspace(1) %arg0, ptr addrspace(1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
   %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
   ret <4 x bfloat> %shuffle
 }
 
@@ -3067,7 +3067,7 @@ define <4 x bfloat> @shuffle_v4bf16_234u(ptr addrspace(1) %arg0, ptr addrspace(1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
   %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 4, i32 undef>
+  %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 4, i32 poison>
   ret <4 x bfloat> %shuffle
 }
 
@@ -3094,7 +3094,7 @@ define <4 x bfloat> @shuffle_v4bf16_u1u3(ptr addrspace(1) %arg0, ptr addrspace(1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
   %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 undef, i32 1, i32 undef, i32 3>
+  %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 poison, i32 1, i32 poison, i32 3>
   ret <4 x bfloat> %shuffle
 }
 
@@ -3133,7 +3133,7 @@ define <4 x bfloat> @shuffle_v4bf16_u3u1(ptr addrspace(1) %arg0, ptr addrspace(1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
   %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 undef, i32 3, i32 undef, i32 1>
+  %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 poison, i32 3, i32 poison, i32 1>
   ret <4 x bfloat> %shuffle
 }
 
@@ -3160,7 +3160,7 @@ define <4 x bfloat> @shuffle_v4bf16_u3uu(ptr addrspace(1) %arg0, ptr addrspace(1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
   %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 undef, i32 3, i32 undef, i32 undef>
+  %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 poison, i32 3, i32 poison, i32 poison>
   ret <4 x bfloat> %shuffle
 }
 
@@ -3209,7 +3209,7 @@ define <4 x bfloat> @shuffle_v4bf16_3u6u(ptr addrspace(1) %arg0, ptr addrspace(1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
   %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 3, i32 undef, i32 6, i32 undef>
+  %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 3, i32 poison, i32 6, i32 poison>
   ret <4 x bfloat> %shuffle
 }
 
@@ -3258,7 +3258,7 @@ define <4 x bfloat> @shuffle_v4bf16_3uu7(ptr addrspace(1) %arg0, ptr addrspace(1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
   %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 3, i32 undef, i32 undef, i32 7>
+  %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 3, i32 poison, i32 poison, i32 7>
   ret <4 x bfloat> %shuffle
 }
 
@@ -3305,7 +3305,7 @@ define <4 x bfloat> @shuffle_v4bf16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
   %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 3, i32 5, i32 undef, i32 5>
+  %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 3, i32 5, i32 poison, i32 5>
   ret <4 x bfloat> %shuffle
 }
 
@@ -3357,7 +3357,7 @@ define <4 x bfloat> @shuffle_v4bf16_357u(ptr addrspace(1) %arg0, ptr addrspace(1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
   %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
-  %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 3, i32 5, i32 7, i32 undef>
+  %shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 3, i32 5, i32 7, i32 poison>
   ret <4 x bfloat> %shuffle
 }
 
@@ -5059,21 +5059,21 @@ entry:
   %tmp15 = load <4 x bfloat>, ptr addrspace(1) %arrayidx1, align 8
   %arrayidx2 = getelementptr inbounds <4 x bfloat>, ptr addrspace(1) %C, i64 %tmp12
   %tmp16 = load <4 x bfloat>, ptr addrspace(1) %arrayidx2, align 8
-  %tmp17 = shufflevector <4 x bfloat> %tmp14, <4 x bfloat> undef, <2 x i32> zeroinitializer
-  %tmp18 = shufflevector <4 x bfloat> %tmp15, <4 x bfloat> undef, <2 x i32> <i32 0, i32 1>
-  %tmp19 = shufflevector <4 x bfloat> %tmp16, <4 x bfloat> undef, <2 x i32> <i32 0, i32 1>
+  %tmp17 = shufflevector <4 x bfloat> %tmp14, <4 x bfloat> poison, <2 x i32> zeroinitializer
+  %tmp18 = shufflevector <4 x bfloat> %tmp15, <4 x bfloat> poison, <2 x i32> <i32 0, i32 1>
+  %tmp19 = shufflevector <4 x bfloat> %tmp16, <4 x bfloat> poison, <2 x i32> <i32 0, i32 1>
   %tmp20 = tail call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %tmp17, <2 x bfloat> %tmp18, <2 x bfloat> %tmp19)
-  %tmp21 = shufflevector <4 x bfloat> %tmp14, <4 x bfloat> undef, <2 x i32> <i32 1, i32 1>
-  %tmp22 = shufflevector <4 x bfloat> %tmp15, <4 x bfloat> undef, <2 x i32> <i32 2, i32 3>
+  %tmp21 = shufflevector <4 x bfloat> %tmp14, <4 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+  %tmp22 = shufflevector <4 x bfloat> %tmp15, <4 x bfloat> poison, <2 x i32> <i32 2, i32 3>
   %tmp23 = tail call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %tmp21, <2 x bfloat> %tmp22, <2 x bfloat> %tmp20)
-  %tmp24 = shufflevector <2 x bfloat> %tmp23, <2 x bfloat> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %tmp24 = shufflevector <2 x bfloat> %tmp23, <2 x bfloat> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   %tmp25 = shufflevector <4 x bfloat> %tmp24, <4 x bfloat> %tmp16, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-  %tmp26 = shufflevector <4 x bfloat> %tmp14, <4 x bfloat> undef, <2 x i32> <i32 2, i32 2>
-  %tmp27 = shufflevector <4 x bfloat> %tmp25, <4 x bfloat> undef, <2 x i32> <i32 2, i32 3>
+  %tmp26 = shufflevector <4 x bfloat> %tmp14, <4 x bfloat> poison, <2 x i32> <i32 2, i32 2>
+  %tmp27 = shufflevector <4 x bfloat> %tmp25, <4 x bfloat> poison, <2 x i32> <i32 2, i32 3>
   %tmp28 = tail call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %tmp26, <2 x bfloat> %tmp18, <2 x bfloat> %tmp27)
-  %tmp29 = shufflevector <4 x bfloat> %tmp14, <4 x bfloat> undef, <2 x i32> <i32 3, i32 3>
+  %tmp29 = shufflevector <4 x bfloat> %tmp14, <4 x bfloat> poison, <2 x i32> <i32 3, i32 3>
   %tmp30 = tail call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %tmp29, <2 x bfloat> %tmp22, <2 x bfloat> %tmp28)
-  %tmp31 = shufflevector <2 x bfloat> %tmp30, <2 x bfloat> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %tmp31 = shufflevector <2 x bfloat> %tmp30, <2 x bfloat> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   %tmp32 = shufflevector <4 x bfloat> %tmp25, <4 x bfloat> %tmp31, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
   store <4 x bfloat> %tmp32, ptr addrspace(1) %arrayidx2, align 8
   ret void
@@ -5172,7 +5172,7 @@ define <2 x bfloat> @low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
 entry:
   %0 = load <2 x bfloat>, ptr addrspace(1) %x0, align 4
   %1 = load <2 x bfloat>, ptr addrspace(1) %x1, align 4
-  %vy1.0.vec.insert = shufflevector <2 x bfloat> %0, <2 x bfloat> poison, <2 x i32> <i32 0, i32 undef>
+  %vy1.0.vec.insert = shufflevector <2 x bfloat> %0, <2 x bfloat> poison, <2 x i32> <i32 0, i32 poison>
   %vy1.2.vec.insert = shufflevector <2 x bfloat> %vy1.0.vec.insert, <2 x bfloat> %1, <2 x i32> <i32 0, i32 2>
   ret <2 x bfloat> %vy1.2.vec.insert
 }
@@ -5218,7 +5218,7 @@ define <2 x bfloat> @hi16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1) %x1)
 entry:
   %0 = load <2 x bfloat>, ptr addrspace(1) %x0, align 4
   %1 = load <2 x bfloat>, ptr addrspace(1) %x1, align 4
-  %vy1.0.vec.insert = shufflevector <2 x bfloat> %0, <2 x bfloat> poison, <2 x i32> <i32 1, i32 undef>
+  %vy1.0.vec.insert = shufflevector <2 x bfloat> %0, <2 x bfloat> poison, <2 x i32> <i32 1, i32 poison>
   %vy1.2.vec.insert = shufflevector <2 x bfloat> %vy1.0.vec.insert, <2 x bfloat> %1, <2 x i32> <i32 0, i32 3>
   ret <2 x bfloat> %vy1.2.vec.insert
 }
@@ -5264,7 +5264,7 @@ define <2 x bfloat> @low16hi16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1)
 entry:
   %0 = load <2 x bfloat>, ptr addrspace(1) %x0, align 4
   %1 = load <2 x bfloat>, ptr addrspace(1) %x1, align 4
-  %vy1.0.vec.insert = shufflevector <2 x bfloat> %0, <2 x bfloat> poison, <2 x i32> <i32 0, i32 undef>
+  %vy1.0.vec.insert = shufflevector <2 x bfloat> %0, <2 x bfloat> poison, <2 x i32> <i32 0, i32 poison>
   %vy1.2.vec.insert = shufflevector <2 x bfloat> %vy1.0.vec.insert, <2 x bfloat> %1, <2 x i32> <i32 0, i32 3>
   ret <2 x bfloat> %vy1.2.vec.insert
 }
@@ -5299,7 +5299,7 @@ define <2 x bfloat> @hi16low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
 entry:
   %0 = load <2 x bfloat>, ptr addrspace(1) %x0, align 4
   %1 = load <2 x bfloat>, ptr addrspace(1) %x1, align 4
-  %vy1.0.vec.insert = shufflevector <2 x bfloat> %0, <2 x bfloat> poison, <2 x i32> <i32 1, i32 undef>
+  %vy1.0.vec.insert = shufflevector <2 x bfloat> %0, <2 x bfloat> poison, <2 x i32> <i32 1, i32 poison>
   %vy1.2.vec.insert = shufflevector <2 x bfloat> %vy1.0.vec.insert, <2 x bfloat> %1, <2 x i32> <i32 0, i32 2>
   ret <2 x bfloat> %vy1.2.vec.insert
 }
@@ -5330,7 +5330,7 @@ define <2 x bfloat> @v2bfloat_hi16bits(ptr addrspace(1) %x0) {
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %load0 = load <2 x bfloat>, ptr addrspace(1) %x0, align 4
-  %insert1 = insertelement <2 x bfloat> undef, bfloat 0.0, i32 0
+  %insert1 = insertelement <2 x bfloat> poison, bfloat 0.0, i32 0
   %insert2 = insertelement <2 x bfloat> %insert1, bfloat 0.0, i32 1
   %vec.ret = shufflevector <2 x bfloat> %insert2, <2 x bfloat> %load0, <2 x i32> <i32 0, i32 3>
   ret <2 x bfloat> %vec.ret
diff --git a/llvm/test/CodeGen/AMDGPU/vectorize-buffer-fat-pointer.ll b/llvm/test/CodeGen/AMDGPU/vectorize-buffer-fat-pointer.ll
index 4aab097229a47..f34617e6efc55 100644
--- a/llvm/test/CodeGen/AMDGPU/vectorize-buffer-fat-pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/vectorize-buffer-fat-pointer.ll
@@ -7,11 +7,11 @@ entry:
   %a2 = getelementptr i32, ptr addrspace(7) %out, i32 2
   %a3 = getelementptr i32, ptr addrspace(7) %out, i32 3
 
-; OPT: store <4 x i32> <i32 0, i32 1, i32 2, i32 3>, ptr addrspace(7) %out, align 4
-  store i32 0, ptr addrspace(7) %out
-  store i32 1, ptr addrspace(7) %a1
-  store i32 2, ptr addrspace(7) %a2
-  store i32 3, ptr addrspace(7) %a3
+; OPT: store <4 x i32> <i32 0, i32 1, i32 2, i32 3>, ptr addrspace(7) %out, align 16
+  store i32 0, ptr addrspace(7) %out, align 16
+  store i32 1, ptr addrspace(7) %a1, align 4
+  store i32 2, ptr addrspace(7) %a2, align 8
+  store i32 3, ptr addrspace(7) %a3, align 4
   ret void
 }
 
@@ -22,10 +22,10 @@ entry:
   %a2 = getelementptr i32, ptr addrspace(9) %out, i32 2
   %a3 = getelementptr i32, ptr addrspace(9) %out, i32 3
 
-; OPT: store <4 x i32> <i32 0, i32 1, i32 2, i32 3>, ptr addrspace(9) %out, align 4
-  store i32 0, ptr addrspace(9) %out
-  store i32 1, ptr addrspace(9) %a1
-  store i32 2, ptr addrspace(9) %a2
-  store i32 3, ptr addrspace(9) %a3
+; OPT: store <4 x i32> <i32 0, i32 1, i32 2, i32 3>, ptr addrspace(9) %out, align 16
+  store i32 0, ptr addrspace(9) %out, align 16
+  store i32 1, ptr addrspace(9) %a1, align 4
+  store i32 2, ptr addrspace(9) %a2, align 8
+  store i32 3, ptr addrspace(9) %a3, align 4
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-large-tuple-alloc-error.ll b/llvm/test/CodeGen/AMDGPU/vgpr-large-tuple-alloc-error.ll
index 5ced02f28c977..2ee62d13fcc51 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-large-tuple-alloc-error.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-large-tuple-alloc-error.ll
@@ -35,34 +35,18 @@ define i32 @test_tuple(<16 x i64> %0) {
 ; GFX900-NEXT:    v_writelane_b32 v63, s37, 1
 ; GFX900-NEXT:    v_writelane_b32 v63, s38, 2
 ; GFX900-NEXT:    v_writelane_b32 v63, s39, 3
-; GFX900-NEXT:    v_writelane_b32 v63, s40, 4
-; GFX900-NEXT:    v_writelane_b32 v63, s41, 5
-; GFX900-NEXT:    v_writelane_b32 v63, s42, 6
-; GFX900-NEXT:    v_writelane_b32 v63, s43, 7
-; GFX900-NEXT:    v_writelane_b32 v63, s44, 8
-; GFX900-NEXT:    v_writelane_b32 v63, s45, 9
-; GFX900-NEXT:    v_writelane_b32 v63, s46, 10
-; GFX900-NEXT:    v_writelane_b32 v63, s47, 11
-; GFX900-NEXT:    v_writelane_b32 v63, s48, 12
-; GFX900-NEXT:    v_writelane_b32 v63, s49, 13
-; GFX900-NEXT:    v_writelane_b32 v63, s50, 14
-; GFX900-NEXT:    v_writelane_b32 v63, s51, 15
-; GFX900-NEXT:    v_writelane_b32 v63, s52, 16
-; GFX900-NEXT:    v_writelane_b32 v63, s53, 17
-; GFX900-NEXT:    v_writelane_b32 v63, s54, 18
-; GFX900-NEXT:    v_writelane_b32 v63, s55, 19
-; GFX900-NEXT:    v_writelane_b32 v63, s56, 20
-; GFX900-NEXT:    v_writelane_b32 v63, s57, 21
-; GFX900-NEXT:    v_writelane_b32 v63, s58, 22
-; GFX900-NEXT:    v_writelane_b32 v63, s59, 23
-; GFX900-NEXT:    v_writelane_b32 v63, s60, 24
-; GFX900-NEXT:    v_writelane_b32 v63, s61, 25
-; GFX900-NEXT:    v_writelane_b32 v63, s62, 26
-; GFX900-NEXT:    v_writelane_b32 v63, s63, 27
-; GFX900-NEXT:    v_writelane_b32 v63, s64, 28
-; GFX900-NEXT:    v_writelane_b32 v63, s65, 29
-; GFX900-NEXT:    v_writelane_b32 v63, s66, 30
-; GFX900-NEXT:    v_writelane_b32 v63, s67, 31
+; GFX900-NEXT:    v_writelane_b32 v63, s48, 4
+; GFX900-NEXT:    v_writelane_b32 v63, s49, 5
+; GFX900-NEXT:    v_writelane_b32 v63, s50, 6
+; GFX900-NEXT:    v_writelane_b32 v63, s51, 7
+; GFX900-NEXT:    v_writelane_b32 v63, s52, 8
+; GFX900-NEXT:    v_writelane_b32 v63, s53, 9
+; GFX900-NEXT:    v_writelane_b32 v63, s54, 10
+; GFX900-NEXT:    v_writelane_b32 v63, s55, 11
+; GFX900-NEXT:    v_writelane_b32 v63, s64, 12
+; GFX900-NEXT:    v_writelane_b32 v63, s65, 13
+; GFX900-NEXT:    v_writelane_b32 v63, s66, 14
+; GFX900-NEXT:    v_writelane_b32 v63, s67, 15
 ; GFX900-NEXT:    v_mov_b32_e32 v33, v30
 ; GFX900-NEXT:    v_mov_b32_e32 v34, v29
 ; GFX900-NEXT:    v_mov_b32_e32 v35, v28
@@ -160,34 +144,18 @@ define i32 @test_tuple(<16 x i64> %0) {
 ; GFX900-NEXT:    ; kill: def $vgpr31 killed $vgpr32 killed $exec
 ; GFX900-NEXT:    ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
 ; GFX900-NEXT:    v_mov_b32_e32 v0, 0
-; GFX900-NEXT:    v_readlane_b32 s67, v63, 31
-; GFX900-NEXT:    v_readlane_b32 s66, v63, 30
-; GFX900-NEXT:    v_readlane_b32 s65, v63, 29
-; GFX900-NEXT:    v_readlane_b32 s64, v63, 28
-; GFX900-NEXT:    v_readlane_b32 s63, v63, 27
-; GFX900-NEXT:    v_readlane_b32 s62, v63, 26
-; GFX900-NEXT:    v_readlane_b32 s61, v63, 25
-; GFX900-NEXT:    v_readlane_b32 s60, v63, 24
-; GFX900-NEXT:    v_readlane_b32 s59, v63, 23
-; GFX900-NEXT:    v_readlane_b32 s58, v63, 22
-; GFX900-NEXT:    v_readlane_b32 s57, v63, 21
-; GFX900-NEXT:    v_readlane_b32 s56, v63, 20
-; GFX900-NEXT:    v_readlane_b32 s55, v63, 19
-; GFX900-NEXT:    v_readlane_b32 s54, v63, 18
-; GFX900-NEXT:    v_readlane_b32 s53, v63, 17
-; GFX900-NEXT:    v_readlane_b32 s52, v63, 16
-; GFX900-NEXT:    v_readlane_b32 s51, v63, 15
-; GFX900-NEXT:    v_readlane_b32 s50, v63, 14
-; GFX900-NEXT:    v_readlane_b32 s49, v63, 13
-; GFX900-NEXT:    v_readlane_b32 s48, v63, 12
-; GFX900-NEXT:    v_readlane_b32 s47, v63, 11
-; GFX900-NEXT:    v_readlane_b32 s46, v63, 10
-; GFX900-NEXT:    v_readlane_b32 s45, v63, 9
-; GFX900-NEXT:    v_readlane_b32 s44, v63, 8
-; GFX900-NEXT:    v_readlane_b32 s43, v63, 7
-; GFX900-NEXT:    v_readlane_b32 s42, v63, 6
-; GFX900-NEXT:    v_readlane_b32 s41, v63, 5
-; GFX900-NEXT:    v_readlane_b32 s40, v63, 4
+; GFX900-NEXT:    v_readlane_b32 s67, v63, 15
+; GFX900-NEXT:    v_readlane_b32 s66, v63, 14
+; GFX900-NEXT:    v_readlane_b32 s65, v63, 13
+; GFX900-NEXT:    v_readlane_b32 s64, v63, 12
+; GFX900-NEXT:    v_readlane_b32 s55, v63, 11
+; GFX900-NEXT:    v_readlane_b32 s54, v63, 10
+; GFX900-NEXT:    v_readlane_b32 s53, v63, 9
+; GFX900-NEXT:    v_readlane_b32 s52, v63, 8
+; GFX900-NEXT:    v_readlane_b32 s51, v63, 7
+; GFX900-NEXT:    v_readlane_b32 s50, v63, 6
+; GFX900-NEXT:    v_readlane_b32 s49, v63, 5
+; GFX900-NEXT:    v_readlane_b32 s48, v63, 4
 ; GFX900-NEXT:    v_readlane_b32 s39, v63, 3
 ; GFX900-NEXT:    v_readlane_b32 s38, v63, 2
 ; GFX900-NEXT:    v_readlane_b32 s37, v63, 1
@@ -238,34 +206,18 @@ define i32 @test_tuple(<16 x i64> %0) {
 ; GFX906-NEXT:    v_writelane_b32 v63, s37, 1
 ; GFX906-NEXT:    v_writelane_b32 v63, s38, 2
 ; GFX906-NEXT:    v_writelane_b32 v63, s39, 3
-; GFX906-NEXT:    v_writelane_b32 v63, s40, 4
-; GFX906-NEXT:    v_writelane_b32 v63, s41, 5
-; GFX906-NEXT:    v_writelane_b32 v63, s42, 6
-; GFX906-NEXT:    v_writelane_b32 v63, s43, 7
-; GFX906-NEXT:    v_writelane_b32 v63, s44, 8
-; GFX906-NEXT:    v_writelane_b32 v63, s45, 9
-; GFX906-NEXT:    v_writelane_b32 v63, s46, 10
-; GFX906-NEXT:    v_writelane_b32 v63, s47, 11
-; GFX906-NEXT:    v_writelane_b32 v63, s48, 12
-; GFX906-NEXT:    v_writelane_b32 v63, s49, 13
-; GFX906-NEXT:    v_writelane_b32 v63, s50, 14
-; GFX906-NEXT:    v_writelane_b32 v63, s51, 15
-; GFX906-NEXT:    v_writelane_b32 v63, s52, 16
-; GFX906-NEXT:    v_writelane_b32 v63, s53, 17
-; GFX906-NEXT:    v_writelane_b32 v63, s54, 18
-; GFX906-NEXT:    v_writelane_b32 v63, s55, 19
-; GFX906-NEXT:    v_writelane_b32 v63, s56, 20
-; GFX906-NEXT:    v_writelane_b32 v63, s57, 21
-; GFX906-NEXT:    v_writelane_b32 v63, s58, 22
-; GFX906-NEXT:    v_writelane_b32 v63, s59, 23
-; GFX906-NEXT:    v_writelane_b32 v63, s60, 24
-; GFX906-NEXT:    v_writelane_b32 v63, s61, 25
-; GFX906-NEXT:    v_writelane_b32 v63, s62, 26
-; GFX906-NEXT:    v_writelane_b32 v63, s63, 27
-; GFX906-NEXT:    v_writelane_b32 v63, s64, 28
-; GFX906-NEXT:    v_writelane_b32 v63, s65, 29
-; GFX906-NEXT:    v_writelane_b32 v63, s66, 30
-; GFX906-NEXT:    v_writelane_b32 v63, s67, 31
+; GFX906-NEXT:    v_writelane_b32 v63, s48, 4
+; GFX906-NEXT:    v_writelane_b32 v63, s49, 5
+; GFX906-NEXT:    v_writelane_b32 v63, s50, 6
+; GFX906-NEXT:    v_writelane_b32 v63, s51, 7
+; GFX906-NEXT:    v_writelane_b32 v63, s52, 8
+; GFX906-NEXT:    v_writelane_b32 v63, s53, 9
+; GFX906-NEXT:    v_writelane_b32 v63, s54, 10
+; GFX906-NEXT:    v_writelane_b32 v63, s55, 11
+; GFX906-NEXT:    v_writelane_b32 v63, s64, 12
+; GFX906-NEXT:    v_writelane_b32 v63, s65, 13
+; GFX906-NEXT:    v_writelane_b32 v63, s66, 14
+; GFX906-NEXT:    v_writelane_b32 v63, s67, 15
 ; GFX906-NEXT:    v_mov_b32_e32 v33, v30
 ; GFX906-NEXT:    v_mov_b32_e32 v34, v29
 ; GFX906-NEXT:    v_mov_b32_e32 v35, v28
@@ -363,34 +315,18 @@ define i32 @test_tuple(<16 x i64> %0) {
 ; GFX906-NEXT:    ; kill: def $vgpr31 killed $vgpr32 killed $exec
 ; GFX906-NEXT:    ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
 ; GFX906-NEXT:    v_mov_b32_e32 v0, 0
-; GFX906-NEXT:    v_readlane_b32 s67, v63, 31
-; GFX906-NEXT:    v_readlane_b32 s66, v63, 30
-; GFX906-NEXT:    v_readlane_b32 s65, v63, 29
-; GFX906-NEXT:    v_readlane_b32 s64, v63, 28
-; GFX906-NEXT:    v_readlane_b32 s63, v63, 27
-; GFX906-NEXT:    v_readlane_b32 s62, v63, 26
-; GFX906-NEXT:    v_readlane_b32 s61, v63, 25
-; GFX906-NEXT:    v_readlane_b32 s60, v63, 24
-; GFX906-NEXT:    v_readlane_b32 s59, v63, 23
-; GFX906-NEXT:    v_readlane_b32 s58, v63, 22
-; GFX906-NEXT:    v_readlane_b32 s57, v63, 21
-; GFX906-NEXT:    v_readlane_b32 s56, v63, 20
-; GFX906-NEXT:    v_readlane_b32 s55, v63, 19
-; GFX906-NEXT:    v_readlane_b32 s54, v63, 18
-; GFX906-NEXT:    v_readlane_b32 s53, v63, 17
-; GFX906-NEXT:    v_readlane_b32 s52, v63, 16
-; GFX906-NEXT:    v_readlane_b32 s51, v63, 15
-; GFX906-NEXT:    v_readlane_b32 s50, v63, 14
-; GFX906-NEXT:    v_readlane_b32 s49, v63, 13
-; GFX906-NEXT:    v_readlane_b32 s48, v63, 12
-; GFX906-NEXT:    v_readlane_b32 s47, v63, 11
-; GFX906-NEXT:    v_readlane_b32 s46, v63, 10
-; GFX906-NEXT:    v_readlane_b32 s45, v63, 9
-; GFX906-NEXT:    v_readlane_b32 s44, v63, 8
-; GFX906-NEXT:    v_readlane_b32 s43, v63, 7
-; GFX906-NEXT:    v_readlane_b32 s42, v63, 6
-; GFX906-NEXT:    v_readlane_b32 s41, v63, 5
-; GFX906-NEXT:    v_readlane_b32 s40, v63, 4
+; GFX906-NEXT:    v_readlane_b32 s67, v63, 15
+; GFX906-NEXT:    v_readlane_b32 s66, v63, 14
+; GFX906-NEXT:    v_readlane_b32 s65, v63, 13
+; GFX906-NEXT:    v_readlane_b32 s64, v63, 12
+; GFX906-NEXT:    v_readlane_b32 s55, v63, 11
+; GFX906-NEXT:    v_readlane_b32 s54, v63, 10
+; GFX906-NEXT:    v_readlane_b32 s53, v63, 9
+; GFX906-NEXT:    v_readlane_b32 s52, v63, 8
+; GFX906-NEXT:    v_readlane_b32 s51, v63, 7
+; GFX906-NEXT:    v_readlane_b32 s50, v63, 6
+; GFX906-NEXT:    v_readlane_b32 s49, v63, 5
+; GFX906-NEXT:    v_readlane_b32 s48, v63, 4
 ; GFX906-NEXT:    v_readlane_b32 s39, v63, 3
 ; GFX906-NEXT:    v_readlane_b32 s38, v63, 2
 ; GFX906-NEXT:    v_readlane_b32 s37, v63, 1
@@ -440,34 +376,18 @@ define i32 @test_tuple(<16 x i64> %0) {
 ; GFX908-NEXT:    v_writelane_b32 v62, s37, 1
 ; GFX908-NEXT:    v_writelane_b32 v62, s38, 2
 ; GFX908-NEXT:    v_writelane_b32 v62, s39, 3
-; GFX908-NEXT:    v_writelane_b32 v62, s40, 4
-; GFX908-NEXT:    v_writelane_b32 v62, s41, 5
-; GFX908-NEXT:    v_writelane_b32 v62, s42, 6
-; GFX908-NEXT:    v_writelane_b32 v62, s43, 7
-; GFX908-NEXT:    v_writelane_b32 v62, s44, 8
-; GFX908-NEXT:    v_writelane_b32 v62, s45, 9
-; GFX908-NEXT:    v_writelane_b32 v62, s46, 10
-; GFX908-NEXT:    v_writelane_b32 v62, s47, 11
-; GFX908-NEXT:    v_writelane_b32 v62, s48, 12
-; GFX908-NEXT:    v_writelane_b32 v62, s49, 13
-; GFX908-NEXT:    v_writelane_b32 v62, s50, 14
-; GFX908-NEXT:    v_writelane_b32 v62, s51, 15
-; GFX908-NEXT:    v_writelane_b32 v62, s52, 16
-; GFX908-NEXT:    v_writelane_b32 v62, s53, 17
-; GFX908-NEXT:    v_writelane_b32 v62, s54, 18
-; GFX908-NEXT:    v_writelane_b32 v62, s55, 19
-; GFX908-NEXT:    v_writelane_b32 v62, s56, 20
-; GFX908-NEXT:    v_writelane_b32 v62, s57, 21
-; GFX908-NEXT:    v_writelane_b32 v62, s58, 22
-; GFX908-NEXT:    v_writelane_b32 v62, s59, 23
-; GFX908-NEXT:    v_writelane_b32 v62, s60, 24
-; GFX908-NEXT:    v_writelane_b32 v62, s61, 25
-; GFX908-NEXT:    v_writelane_b32 v62, s62, 26
-; GFX908-NEXT:    v_writelane_b32 v62, s63, 27
-; GFX908-NEXT:    v_writelane_b32 v62, s64, 28
-; GFX908-NEXT:    v_writelane_b32 v62, s65, 29
-; GFX908-NEXT:    v_writelane_b32 v62, s66, 30
-; GFX908-NEXT:    v_writelane_b32 v62, s67, 31
+; GFX908-NEXT:    v_writelane_b32 v62, s48, 4
+; GFX908-NEXT:    v_writelane_b32 v62, s49, 5
+; GFX908-NEXT:    v_writelane_b32 v62, s50, 6
+; GFX908-NEXT:    v_writelane_b32 v62, s51, 7
+; GFX908-NEXT:    v_writelane_b32 v62, s52, 8
+; GFX908-NEXT:    v_writelane_b32 v62, s53, 9
+; GFX908-NEXT:    v_writelane_b32 v62, s54, 10
+; GFX908-NEXT:    v_writelane_b32 v62, s55, 11
+; GFX908-NEXT:    v_writelane_b32 v62, s64, 12
+; GFX908-NEXT:    v_writelane_b32 v62, s65, 13
+; GFX908-NEXT:    v_writelane_b32 v62, s66, 14
+; GFX908-NEXT:    v_writelane_b32 v62, s67, 15
 ; GFX908-NEXT:    v_mov_b32_e32 v33, v30
 ; GFX908-NEXT:    v_mov_b32_e32 v34, v29
 ; GFX908-NEXT:    v_mov_b32_e32 v35, v28
@@ -569,34 +489,18 @@ define i32 @test_tuple(<16 x i64> %0) {
 ; GFX908-NEXT:    ; kill: def $vgpr31 killed $vgpr32 killed $exec
 ; GFX908-NEXT:    ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
 ; GFX908-NEXT:    v_mov_b32_e32 v0, 0
-; GFX908-NEXT:    v_readlane_b32 s67, v62, 31
-; GFX908-NEXT:    v_readlane_b32 s66, v62, 30
-; GFX908-NEXT:    v_readlane_b32 s65, v62, 29
-; GFX908-NEXT:    v_readlane_b32 s64, v62, 28
-; GFX908-NEXT:    v_readlane_b32 s63, v62, 27
-; GFX908-NEXT:    v_readlane_b32 s62, v62, 26
-; GFX908-NEXT:    v_readlane_b32 s61, v62, 25
-; GFX908-NEXT:    v_readlane_b32 s60, v62, 24
-; GFX908-NEXT:    v_readlane_b32 s59, v62, 23
-; GFX908-NEXT:    v_readlane_b32 s58, v62, 22
-; GFX908-NEXT:    v_readlane_b32 s57, v62, 21
-; GFX908-NEXT:    v_readlane_b32 s56, v62, 20
-; GFX908-NEXT:    v_readlane_b32 s55, v62, 19
-; GFX908-NEXT:    v_readlane_b32 s54, v62, 18
-; GFX908-NEXT:    v_readlane_b32 s53, v62, 17
-; GFX908-NEXT:    v_readlane_b32 s52, v62, 16
-; GFX908-NEXT:    v_readlane_b32 s51, v62, 15
-; GFX908-NEXT:    v_readlane_b32 s50, v62, 14
-; GFX908-NEXT:    v_readlane_b32 s49, v62, 13
-; GFX908-NEXT:    v_readlane_b32 s48, v62, 12
-; GFX908-NEXT:    v_readlane_b32 s47, v62, 11
-; GFX908-NEXT:    v_readlane_b32 s46, v62, 10
-; GFX908-NEXT:    v_readlane_b32 s45, v62, 9
-; GFX908-NEXT:    v_readlane_b32 s44, v62, 8
-; GFX908-NEXT:    v_readlane_b32 s43, v62, 7
-; GFX908-NEXT:    v_readlane_b32 s42, v62, 6
-; GFX908-NEXT:    v_readlane_b32 s41, v62, 5
-; GFX908-NEXT:    v_readlane_b32 s40, v62, 4
+; GFX908-NEXT:    v_readlane_b32 s67, v62, 15
+; GFX908-NEXT:    v_readlane_b32 s66, v62, 14
+; GFX908-NEXT:    v_readlane_b32 s65, v62, 13
+; GFX908-NEXT:    v_readlane_b32 s64, v62, 12
+; GFX908-NEXT:    v_readlane_b32 s55, v62, 11
+; GFX908-NEXT:    v_readlane_b32 s54, v62, 10
+; GFX908-NEXT:    v_readlane_b32 s53, v62, 9
+; GFX908-NEXT:    v_readlane_b32 s52, v62, 8
+; GFX908-NEXT:    v_readlane_b32 s51, v62, 7
+; GFX908-NEXT:    v_readlane_b32 s50, v62, 6
+; GFX908-NEXT:    v_readlane_b32 s49, v62, 5
+; GFX908-NEXT:    v_readlane_b32 s48, v62, 4
 ; GFX908-NEXT:    v_readlane_b32 s39, v62, 3
 ; GFX908-NEXT:    v_readlane_b32 s38, v62, 2
 ; GFX908-NEXT:    v_readlane_b32 s37, v62, 1
@@ -646,34 +550,18 @@ define i32 @test_tuple(<16 x i64> %0) {
 ; GFX90a-NEXT:    v_writelane_b32 v63, s37, 1
 ; GFX90a-NEXT:    v_writelane_b32 v63, s38, 2
 ; GFX90a-NEXT:    v_writelane_b32 v63, s39, 3
-; GFX90a-NEXT:    v_writelane_b32 v63, s40, 4
-; GFX90a-NEXT:    v_writelane_b32 v63, s41, 5
-; GFX90a-NEXT:    v_writelane_b32 v63, s42, 6
-; GFX90a-NEXT:    v_writelane_b32 v63, s43, 7
-; GFX90a-NEXT:    v_writelane_b32 v63, s44, 8
-; GFX90a-NEXT:    v_writelane_b32 v63, s45, 9
-; GFX90a-NEXT:    v_writelane_b32 v63, s46, 10
-; GFX90a-NEXT:    v_writelane_b32 v63, s47, 11
-; GFX90a-NEXT:    v_writelane_b32 v63, s48, 12
-; GFX90a-NEXT:    v_writelane_b32 v63, s49, 13
-; GFX90a-NEXT:    v_writelane_b32 v63, s50, 14
-; GFX90a-NEXT:    v_writelane_b32 v63, s51, 15
-; GFX90a-NEXT:    v_writelane_b32 v63, s52, 16
-; GFX90a-NEXT:    v_writelane_b32 v63, s53, 17
-; GFX90a-NEXT:    v_writelane_b32 v63, s54, 18
-; GFX90a-NEXT:    v_writelane_b32 v63, s55, 19
-; GFX90a-NEXT:    v_writelane_b32 v63, s56, 20
-; GFX90a-NEXT:    v_writelane_b32 v63, s57, 21
-; GFX90a-NEXT:    v_writelane_b32 v63, s58, 22
-; GFX90a-NEXT:    v_writelane_b32 v63, s59, 23
-; GFX90a-NEXT:    v_writelane_b32 v63, s60, 24
-; GFX90a-NEXT:    v_writelane_b32 v63, s61, 25
-; GFX90a-NEXT:    v_writelane_b32 v63, s62, 26
-; GFX90a-NEXT:    v_writelane_b32 v63, s63, 27
-; GFX90a-NEXT:    v_writelane_b32 v63, s64, 28
-; GFX90a-NEXT:    v_writelane_b32 v63, s65, 29
-; GFX90a-NEXT:    v_writelane_b32 v63, s66, 30
-; GFX90a-NEXT:    v_writelane_b32 v63, s67, 31
+; GFX90a-NEXT:    v_writelane_b32 v63, s48, 4
+; GFX90a-NEXT:    v_writelane_b32 v63, s49, 5
+; GFX90a-NEXT:    v_writelane_b32 v63, s50, 6
+; GFX90a-NEXT:    v_writelane_b32 v63, s51, 7
+; GFX90a-NEXT:    v_writelane_b32 v63, s52, 8
+; GFX90a-NEXT:    v_writelane_b32 v63, s53, 9
+; GFX90a-NEXT:    v_writelane_b32 v63, s54, 10
+; GFX90a-NEXT:    v_writelane_b32 v63, s55, 11
+; GFX90a-NEXT:    v_writelane_b32 v63, s64, 12
+; GFX90a-NEXT:    v_writelane_b32 v63, s65, 13
+; GFX90a-NEXT:    v_writelane_b32 v63, s66, 14
+; GFX90a-NEXT:    v_writelane_b32 v63, s67, 15
 ; GFX90a-NEXT:    v_mov_b32_e32 v33, v30
 ; GFX90a-NEXT:    v_mov_b32_e32 v34, v29
 ; GFX90a-NEXT:    v_mov_b32_e32 v35, v28
@@ -771,34 +659,18 @@ define i32 @test_tuple(<16 x i64> %0) {
 ; GFX90a-NEXT:    ; kill: def $vgpr31 killed $vgpr32 killed $exec
 ; GFX90a-NEXT:    ; implicit-def: $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67
 ; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NEXT:    v_readlane_b32 s67, v63, 31
-; GFX90a-NEXT:    v_readlane_b32 s66, v63, 30
-; GFX90a-NEXT:    v_readlane_b32 s65, v63, 29
-; GFX90a-NEXT:    v_readlane_b32 s64, v63, 28
-; GFX90a-NEXT:    v_readlane_b32 s63, v63, 27
-; GFX90a-NEXT:    v_readlane_b32 s62, v63, 26
-; GFX90a-NEXT:    v_readlane_b32 s61, v63, 25
-; GFX90a-NEXT:    v_readlane_b32 s60, v63, 24
-; GFX90a-NEXT:    v_readlane_b32 s59, v63, 23
-; GFX90a-NEXT:    v_readlane_b32 s58, v63, 22
-; GFX90a-NEXT:    v_readlane_b32 s57, v63, 21
-; GFX90a-NEXT:    v_readlane_b32 s56, v63, 20
-; GFX90a-NEXT:    v_readlane_b32 s55, v63, 19
-; GFX90a-NEXT:    v_readlane_b32 s54, v63, 18
-; GFX90a-NEXT:    v_readlane_b32 s53, v63, 17
-; GFX90a-NEXT:    v_readlane_b32 s52, v63, 16
-; GFX90a-NEXT:    v_readlane_b32 s51, v63, 15
-; GFX90a-NEXT:    v_readlane_b32 s50, v63, 14
-; GFX90a-NEXT:    v_readlane_b32 s49, v63, 13
-; GFX90a-NEXT:    v_readlane_b32 s48, v63, 12
-; GFX90a-NEXT:    v_readlane_b32 s47, v63, 11
-; GFX90a-NEXT:    v_readlane_b32 s46, v63, 10
-; GFX90a-NEXT:    v_readlane_b32 s45, v63, 9
-; GFX90a-NEXT:    v_readlane_b32 s44, v63, 8
-; GFX90a-NEXT:    v_readlane_b32 s43, v63, 7
-; GFX90a-NEXT:    v_readlane_b32 s42, v63, 6
-; GFX90a-NEXT:    v_readlane_b32 s41, v63, 5
-; GFX90a-NEXT:    v_readlane_b32 s40, v63, 4
+; GFX90a-NEXT:    v_readlane_b32 s67, v63, 15
+; GFX90a-NEXT:    v_readlane_b32 s66, v63, 14
+; GFX90a-NEXT:    v_readlane_b32 s65, v63, 13
+; GFX90a-NEXT:    v_readlane_b32 s64, v63, 12
+; GFX90a-NEXT:    v_readlane_b32 s55, v63, 11
+; GFX90a-NEXT:    v_readlane_b32 s54, v63, 10
+; GFX90a-NEXT:    v_readlane_b32 s53, v63, 9
+; GFX90a-NEXT:    v_readlane_b32 s52, v63, 8
+; GFX90a-NEXT:    v_readlane_b32 s51, v63, 7
+; GFX90a-NEXT:    v_readlane_b32 s50, v63, 6
+; GFX90a-NEXT:    v_readlane_b32 s49, v63, 5
+; GFX90a-NEXT:    v_readlane_b32 s48, v63, 4
 ; GFX90a-NEXT:    v_readlane_b32 s39, v63, 3
 ; GFX90a-NEXT:    v_readlane_b32 s38, v63, 2
 ; GFX90a-NEXT:    v_readlane_b32 s37, v63, 1
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
index b0b5850084697..30d9465c33e40 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
@@ -73,8 +73,7 @@ define amdgpu_ps float @else2(i32 %z, float %v) #0 {
   ; SI-NEXT: bb.1.Flow:
   ; SI-NEXT:   successors: %bb.2(0x40000000), %bb.4(0x40000000)
   ; SI-NEXT: {{  $}}
-  ; SI-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %11:vgpr_32, %bb.0, %5, %bb.3
-  ; SI-NEXT:   [[PHI1:%[0-9]+]]:vgpr_32 = PHI undef %11:vgpr_32, %bb.0, [[COPY]], %bb.3
+  ; SI-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %10:vgpr_32, %bb.0, %4, %bb.3
   ; SI-NEXT:   [[SI_ELSE:%[0-9]+]]:sreg_32 = SI_ELSE killed [[SI_IF]], %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
   ; SI-NEXT:   S_BRANCH %bb.2
   ; SI-NEXT: {{  $}}
@@ -91,10 +90,10 @@ define amdgpu_ps float @else2(i32 %z, float %v) #0 {
   ; SI-NEXT:   S_BRANCH %bb.1
   ; SI-NEXT: {{  $}}
   ; SI-NEXT: bb.4.end:
-  ; SI-NEXT:   [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[PHI1]], %bb.1, [[V_ADD_F32_e64_]], %bb.2
-  ; SI-NEXT:   [[PHI3:%[0-9]+]]:vgpr_32 = PHI [[PHI]], %bb.1, [[V_ADD_F32_e64_]], %bb.2
+  ; SI-NEXT:   [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.1, [[V_ADD_F32_e64_]], %bb.2
+  ; SI-NEXT:   [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[PHI]], %bb.1, [[V_ADD_F32_e64_]], %bb.2
   ; SI-NEXT:   SI_END_CF killed [[SI_ELSE]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
-  ; SI-NEXT:   [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[PHI2]], 0, killed [[PHI3]], 0, 0, implicit $mode, implicit $exec
+  ; SI-NEXT:   [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[PHI1]], 0, killed [[PHI2]], 0, 0, implicit $mode, implicit $exec
   ; SI-NEXT:   $vgpr0 = COPY killed [[V_ADD_F32_e64_1]]
   ; SI-NEXT:   SI_RETURN_TO_EPILOG killed $vgpr0
 main_body:
@@ -544,12 +543,12 @@ if.then9:                                         ; preds = %entry
 sw.bb:                                            ; preds = %if.then9
   %i17 = load i8, ptr addrspace(1) null, align 1
   %i18 = insertelement <4 x i8> zeroinitializer, i8 %i17, i64 0
-  %a.sroa.0.0.vecblend = shufflevector <4 x i8> %i18, <4 x i8> zeroinitializer, <4 x i32> <i32 0, i32 0, i32 0, i32 undef>
+  %a.sroa.0.0.vecblend = shufflevector <4 x i8> %i18, <4 x i8> zeroinitializer, <4 x i32> <i32 0, i32 0, i32 0, i32 poison>
   br label %sw.bb18
 
 sw.bb18:                                          ; preds = %sw.bb, %if.then9
-  %a.sroa.0.0 = phi <4 x i8> [ %a.sroa.0.0.vecblend, %sw.bb ], [ undef, %if.then9 ]
-  %a.sroa.0.0.vec.extract61 = shufflevector <4 x i8> %a.sroa.0.0, <4 x i8> zeroinitializer, <3 x i32> <i32 undef, i32 1, i32 undef>
+  %a.sroa.0.0 = phi <4 x i8> [ %a.sroa.0.0.vecblend, %sw.bb ], [ poison, %if.then9 ]
+  %a.sroa.0.0.vec.extract61 = shufflevector <4 x i8> %a.sroa.0.0, <4 x i8> zeroinitializer, <3 x i32> <i32 poison, i32 1, i32 poison>
   %i19 = insertelement <3 x i8> %a.sroa.0.0.vec.extract61, i8 0, i64 0
   %i20 = select <3 x i1> zeroinitializer, <3 x i8> zeroinitializer, <3 x i8> %i19
   %i21 = extractelement <3 x i8> %i20, i64 1
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-remat.mir b/llvm/test/CodeGen/AMDGPU/vgpr-remat.mir
index 0e7e4b9b10d44..9dad99f307c19 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-remat.mir
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-remat.mir
@@ -90,3 +90,48 @@ body: |
     %4.sub2_sub3:vreg_192 = COPY %2:vreg_64_align2
     S_ENDPGM 0, implicit %4
 ...
+
+---
+name: av_mov_imm_b32
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: av_mov_imm_b32
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $sgpr0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   undef [[AV_MOV_:%[0-9]+]].sub0:vreg_96 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; CHECK-NEXT:   [[AV_MOV_:%[0-9]+]].sub1:vreg_96 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0
+  ; CHECK-NEXT:   $exec = S_MOV_B64_term [[COPY]]
+  ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.2, implicit $exec
+  ; CHECK-NEXT:   S_BRANCH %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[AV_MOV_:%[0-9]+]].sub0:vreg_96 = V_MUL_F32_e32 [[AV_MOV_]].sub0, [[AV_MOV_]].sub0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[AV_MOV_:%[0-9]+]].sub1:vreg_96 = V_MUL_F32_e32 [[AV_MOV_]].sub1, [[AV_MOV_]].sub1, implicit $mode, implicit $exec
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   S_ENDPGM 0, implicit [[AV_MOV_]]
+  bb.0:
+    liveins: $sgpr0
+    %0:av_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+    %1:vgpr_32 = COPY %0
+    %2:vgpr_32 = COPY %0
+    %3:sreg_64 = COPY $sgpr0
+    $exec = S_MOV_B64_term %3:sreg_64
+    S_CBRANCH_EXECZ %bb.2, implicit $exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    %1:vgpr_32 = V_MUL_F32_e32 %1:vgpr_32, %1:vgpr_32, implicit $mode, implicit $exec
+    %2:vgpr_32 = V_MUL_F32_e32 %2:vgpr_32, %2:vgpr_32, implicit $mode, implicit $exec
+
+  bb.2:
+    undef %4.sub0:vreg_96 = COPY %1:vgpr_32
+    %4.sub1:vreg_96 = COPY %2:vgpr_32
+    S_ENDPGM 0, implicit %4
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot-compute.ll b/llvm/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot-compute.ll
index 65a7554bb66a5..a69ada244ded9 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot-compute.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot-compute.ll
@@ -347,7 +347,7 @@ bb145:                                            ; preds = %bb12
   %tmp147 = bitcast float %tmp95 to i32
   %tmp148 = add i32 %tmp11, %tmp147
   %tmp149 = bitcast i32 %tmp148 to float
-  %tmp150 = insertelement <128 x float> undef, float %tmp91, i32 0
+  %tmp150 = insertelement <128 x float> poison, float %tmp91, i32 0
   %tmp151 = insertelement <128 x float> %tmp150, float %tmp90, i32 1
   %tmp152 = insertelement <128 x float> %tmp151, float %tmp89, i32 2
   %tmp153 = insertelement <128 x float> %tmp152, float %tmp87, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot.ll b/llvm/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot.ll
index 0cabfa9aea0e4..8dfd841671730 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot.ll
@@ -220,7 +220,7 @@ bb157:                                            ; preds = %bb24
   %tmp159 = bitcast float %tmp107 to i32
   %tmp160 = add i32 %tmp23, %tmp159
   %tmp161 = bitcast i32 %tmp160 to float
-  %tmp162 = insertelement <128 x float> undef, float %tmp103, i32 0
+  %tmp162 = insertelement <128 x float> poison, float %tmp103, i32 0
   %tmp163 = insertelement <128 x float> %tmp162, float %tmp102, i32 1
   %tmp164 = insertelement <128 x float> %tmp163, float %tmp101, i32 2
   %tmp165 = insertelement <128 x float> %tmp164, float %tmp99, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/wave32.ll b/llvm/test/CodeGen/AMDGPU/wave32.ll
index e49dd9eab30f3..4962254545c3c 100644
--- a/llvm/test/CodeGen/AMDGPU/wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/wave32.ll
@@ -1610,7 +1610,7 @@ bb:
   br label %bb1
 
 bb1:                                              ; preds = %Flow, %bb
-  %lsr.iv = phi i32 [ undef, %bb ], [ %tmp2, %Flow ]
+  %lsr.iv = phi i32 [ poison, %bb ], [ %tmp2, %Flow ]
   %lsr.iv.next = add i32 %lsr.iv, 1
   %cmp0 = icmp slt i32 %lsr.iv.next, 0
   br i1 %cmp0, label %bb4, label %Flow
@@ -1621,7 +1621,7 @@ bb4:                                              ; preds = %bb1
   br label %Flow
 
 Flow:                                             ; preds = %bb4, %bb1
-  %tmp2 = phi i32 [ %lsr.iv.next, %bb4 ], [ undef, %bb1 ]
+  %tmp2 = phi i32 [ %lsr.iv.next, %bb4 ], [ poison, %bb1 ]
   %tmp3 = phi i1 [ %cmp1, %bb4 ], [ true, %bb1 ]
   br i1 %tmp3, label %bb1, label %bb9
 
diff --git a/llvm/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll b/llvm/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll
index 186f50480a34d..26be45e45697f 100644
--- a/llvm/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll
+++ b/llvm/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll
@@ -14,10 +14,10 @@ bb:
   %tmp = extractelement <4 x double> %arg, i64 0
   %tmp1 = fcmp uno double %tmp, 0.000000e+00
   %tmp2 = sext i1 %tmp1 to i64
-  %tmp3 = insertelement <4 x i64> undef, i64 %tmp2, i32 0
-  %tmp4 = insertelement <4 x i64> %tmp3, i64 undef, i32 1
-  %tmp5 = insertelement <4 x i64> %tmp4, i64 undef, i32 2
-  %tmp6 = insertelement <4 x i64> %tmp5, i64 undef, i32 3
+  %tmp3 = insertelement <4 x i64> poison, i64 %tmp2, i32 0
+  %tmp4 = insertelement <4 x i64> %tmp3, i64 poison, i32 1
+  %tmp5 = insertelement <4 x i64> %tmp4, i64 poison, i32 2
+  %tmp6 = insertelement <4 x i64> %tmp5, i64 poison, i32 3
   %tmp7 = fcmp une <4 x double> %arg, zeroinitializer
   %tmp8 = icmp sgt <4 x i64> %tmp6, <i64 -1, i64 -1, i64 -1, i64 -1>
   %tmp9 = and <4 x i1> %tmp8, %tmp7
@@ -36,10 +36,10 @@ bb:
   %tmp = extractelement <4 x i64> %arg, i64 0
   %tmp1 = icmp eq i64 %tmp, 0
   %tmp2 = sext i1 %tmp1 to i64
-  %tmp3 = insertelement <4 x i64> undef, i64 %tmp2, i32 0
-  %tmp4 = insertelement <4 x i64> %tmp3, i64 undef, i32 1
-  %tmp5 = insertelement <4 x i64> %tmp4, i64 undef, i32 2
-  %tmp6 = insertelement <4 x i64> %tmp5, i64 undef, i32 3
+  %tmp3 = insertelement <4 x i64> poison, i64 %tmp2, i32 0
+  %tmp4 = insertelement <4 x i64> %tmp3, i64 poison, i32 1
+  %tmp5 = insertelement <4 x i64> %tmp4, i64 poison, i32 2
+  %tmp6 = insertelement <4 x i64> %tmp5, i64 poison, i32 3
   %tmp7 = icmp ne <4 x i64> %arg, zeroinitializer
   %tmp8 = icmp sgt <4 x i64> %tmp6, <i64 -1, i64 -1, i64 -1, i64 -1>
   %tmp9 = and <4 x i1> %tmp8, %tmp7
diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll
index 48ab512a7ae0d..2833237c1ea37 100644
--- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll
+++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll
@@ -421,7 +421,7 @@ define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negC_pack(<8 x half> %A, <8 x
 ; GFX12-NEXT:    s_endpgm
 bb:
   %C = load <16 x half>, ptr %Caddr
-  %C_shuffle = shufflevector <16 x half> %C, <16 x half> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %C_shuffle = shufflevector <16 x half> %C, <16 x half> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %fneg.C_shuffle = fneg <8 x half> %C_shuffle
   %res = call <8 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<8 x half> %A, <8 x half> %B, <8 x half> %fneg.C_shuffle , i1 0)
   store <8 x half> %res, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll
index 8f7cd5cb2bb65..98da9ef2e8819 100644
--- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll
+++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-f16-f32-matrix-modifiers.ll
@@ -378,7 +378,7 @@ define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negC_pack(<4 x half> %A, <4 x
 ; GFX12-NEXT:    s_endpgm
 bb:
   %C = load <8 x half>, ptr %Caddr
-  %C_shuffle = shufflevector <8 x half> %C, <8 x half> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %C_shuffle = shufflevector <8 x half> %C, <8 x half> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %fneg.C_shuffle = fneg <4 x half> %C_shuffle
   %res = call <4 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16.v8f16.v8f16(<4 x half> %A, <4 x half> %B, <4 x half> %fneg.C_shuffle , i1 0)
   store <4 x half> %res, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/wmma_modifiers.ll b/llvm/test/CodeGen/AMDGPU/wmma_modifiers.ll
index 2efbd3277f209..692491457ae3d 100644
--- a/llvm/test/CodeGen/AMDGPU/wmma_modifiers.ll
+++ b/llvm/test/CodeGen/AMDGPU/wmma_modifiers.ll
@@ -12,9 +12,9 @@ define amdgpu_cs void @xyz () {
   br label %loop
 loop:
   %ld = load <8 x float>, ptr addrspace(5) null, align 32
-  %in_shuffle = shufflevector <8 x float> %ld, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %in_shuffle = shufflevector <8 x float> %ld, <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %wmma = call <4 x float> @llvm.amdgcn.wmma.f32.16x16x16.f16.v4f32.v16f16(<16 x half> undef, <16 x half> undef, <4 x float> %in_shuffle)
-  %out_shuffle = shufflevector <4 x float> %wmma, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %out_shuffle = shufflevector <4 x float> %wmma, <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
   store <8 x float> %out_shuffle, ptr addrspace(5) null, align 32
   br i1 false, label %.exit, label %loop
 .exit:
diff --git a/llvm/test/CodeGen/AMDGPU/wqm-gfx11.ll b/llvm/test/CodeGen/AMDGPU/wqm-gfx11.ll
index e68f232a9f506..82d276e758c7b 100644
--- a/llvm/test/CodeGen/AMDGPU/wqm-gfx11.ll
+++ b/llvm/test/CodeGen/AMDGPU/wqm-gfx11.ll
@@ -21,7 +21,7 @@ main_body:
   %a = call float @llvm.amdgcn.lds.param.load(i32 0, i32 0, i32 %attr) #1
   %b = call float @llvm.amdgcn.lds.param.load(i32 1, i32 0, i32 %attr) #1
   %c = call float @llvm.amdgcn.lds.param.load(i32 2, i32 0, i32 %attr) #1
-  %tmp_0 = insertelement <3 x float> undef, float %a, i32 0
+  %tmp_0 = insertelement <3 x float> poison, float %a, i32 0
   %tmp_1 = insertelement <3 x float> %tmp_0, float %b, i32 1
   %tmp_2 = insertelement <3 x float> %tmp_1, float %c, i32 2
   %res = fadd <3 x float> %tmp_2, %to_add
@@ -50,7 +50,7 @@ main_body:
   %a = call float @llvm.amdgcn.lds.direct.load(i32 %arg_0) #1
   %b = call float @llvm.amdgcn.lds.direct.load(i32 %arg_1) #1
   %c = call float @llvm.amdgcn.lds.direct.load(i32 %arg_2) #1
-  %tmp_0 = insertelement <3 x float> undef, float %a, i32 0
+  %tmp_0 = insertelement <3 x float> poison, float %a, i32 0
   %tmp_1 = insertelement <3 x float> %tmp_0, float %b, i32 1
   %tmp_2 = insertelement <3 x float> %tmp_1, float %c, i32 2
   %res = fadd <3 x float> %tmp_2, %to_add
diff --git a/llvm/test/CodeGen/AMDGPU/xor3.ll b/llvm/test/CodeGen/AMDGPU/xor3.ll
index 864dec3563a6b..6c5a4670b245e 100644
--- a/llvm/test/CodeGen/AMDGPU/xor3.ll
+++ b/llvm/test/CodeGen/AMDGPU/xor3.ll
@@ -108,7 +108,7 @@ define amdgpu_ps <2 x float> @xor3_multiuse_outer(i32 %a, i32 %b, i32 %c, i32 %x
   %inner = xor i32 %a, %b
   %outer = xor i32 %inner, %c
   %x1 = mul i32 %outer, %x
-  %r1 = insertelement <2 x i32> undef, i32 %outer, i32 0
+  %r1 = insertelement <2 x i32> poison, i32 %outer, i32 0
   %r0 = insertelement <2 x i32> %r1, i32 %x1, i32 1
   %bc = bitcast <2 x i32> %r0 to <2 x float>
   ret <2 x float> %bc
@@ -128,7 +128,7 @@ define amdgpu_ps <2 x float> @xor3_multiuse_inner(i32 %a, i32 %b, i32 %c) {
 ; GFX10-NEXT:    ; return to shader part epilog
   %inner = xor i32 %a, %b
   %outer = xor i32 %inner, %c
-  %r1 = insertelement <2 x i32> undef, i32 %inner, i32 0
+  %r1 = insertelement <2 x i32> poison, i32 %inner, i32 0
   %r0 = insertelement <2 x i32> %r1, i32 %outer, i32 1
   %bc = bitcast <2 x i32> %r0 to <2 x float>
   ret <2 x float> %bc
diff --git a/llvm/test/CodeGen/ARM/fp16_fast_math.ll b/llvm/test/CodeGen/ARM/fp16_fast_math.ll
new file mode 100644
index 0000000000000..b440bb97674b4
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/fp16_fast_math.ll
@@ -0,0 +1,112 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=arm -mattr=+vfp4d16sp,-fullfp16 -stop-after=finalize-isel | FileCheck %s --check-prefixes=CHECK-CVT
+; RUN: llc < %s -mtriple=arm -mattr=+vfp4d16sp,+fullfp16 -stop-after=finalize-isel | FileCheck %s --check-prefixes=CHECK-FP16
+
+; Check that the output instructions have the same fast math flags as the input
+; fadd, even when f16 is legalized to f32.
+; FIXME: We don't get fast math flags on VCVTBHS because they get lost during a
+; DAGCombine transformation.
+; FIXME: We don't get fast math flags on VCVTBSH because the outermost node in
+; the isel pattern is COPY_TO_REGCLASS and the fast math flags end up there.
+
+define half @normal_fadd(half %x, half %y) {
+  ; CHECK-CVT-LABEL: name: normal_fadd
+  ; CHECK-CVT: bb.0.entry:
+  ; CHECK-CVT-NEXT:   liveins: $r0, $r1
+  ; CHECK-CVT-NEXT: {{  $}}
+  ; CHECK-CVT-NEXT:   [[COPY:%[0-9]+]]:gpr = COPY $r1
+  ; CHECK-CVT-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $r0
+  ; CHECK-CVT-NEXT:   [[COPY2:%[0-9]+]]:spr = COPY [[COPY1]]
+  ; CHECK-CVT-NEXT:   [[COPY3:%[0-9]+]]:spr = COPY [[COPY]]
+  ; CHECK-CVT-NEXT:   [[VCVTBHS:%[0-9]+]]:spr = VCVTBHS killed [[COPY3]], 14 /* CC::al */, $noreg
+  ; CHECK-CVT-NEXT:   [[VCVTBHS1:%[0-9]+]]:spr = VCVTBHS killed [[COPY2]], 14 /* CC::al */, $noreg
+  ; CHECK-CVT-NEXT:   [[VADDS:%[0-9]+]]:spr = VADDS killed [[VCVTBHS1]], killed [[VCVTBHS]], 14 /* CC::al */, $noreg
+  ; CHECK-CVT-NEXT:   [[DEF:%[0-9]+]]:spr = IMPLICIT_DEF
+  ; CHECK-CVT-NEXT:   [[VCVTBSH:%[0-9]+]]:spr = VCVTBSH [[DEF]], killed [[VADDS]], 14 /* CC::al */, $noreg
+  ; CHECK-CVT-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY [[VCVTBSH]]
+  ; CHECK-CVT-NEXT:   $r0 = COPY [[COPY4]]
+  ; CHECK-CVT-NEXT:   MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
+  ;
+  ; CHECK-FP16-LABEL: name: normal_fadd
+  ; CHECK-FP16: bb.0.entry:
+  ; CHECK-FP16-NEXT:   liveins: $r0, $r1
+  ; CHECK-FP16-NEXT: {{  $}}
+  ; CHECK-FP16-NEXT:   [[COPY:%[0-9]+]]:rgpr = COPY $r1
+  ; CHECK-FP16-NEXT:   [[COPY1:%[0-9]+]]:rgpr = COPY $r0
+  ; CHECK-FP16-NEXT:   [[VMOVHR:%[0-9]+]]:hpr = VMOVHR [[COPY]], 14, $noreg
+  ; CHECK-FP16-NEXT:   [[VMOVHR1:%[0-9]+]]:hpr = VMOVHR [[COPY1]], 14, $noreg
+  ; CHECK-FP16-NEXT:   [[VADDH:%[0-9]+]]:hpr = VADDH killed [[VMOVHR1]], killed [[VMOVHR]], 14, $noreg
+  ; CHECK-FP16-NEXT:   $r0 = COPY [[VADDH]]
+  ; CHECK-FP16-NEXT:   MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
+entry:
+  %add = fadd half %x, %y
+  ret half %add
+}
+
+define half @fast_fadd(half %x, half %y) {
+  ; CHECK-CVT-LABEL: name: fast_fadd
+  ; CHECK-CVT: bb.0.entry:
+  ; CHECK-CVT-NEXT:   liveins: $r0, $r1
+  ; CHECK-CVT-NEXT: {{  $}}
+  ; CHECK-CVT-NEXT:   [[COPY:%[0-9]+]]:gpr = COPY $r1
+  ; CHECK-CVT-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $r0
+  ; CHECK-CVT-NEXT:   [[COPY2:%[0-9]+]]:spr = COPY [[COPY1]]
+  ; CHECK-CVT-NEXT:   [[COPY3:%[0-9]+]]:spr = COPY [[COPY]]
+  ; CHECK-CVT-NEXT:   [[VCVTBHS:%[0-9]+]]:spr = VCVTBHS killed [[COPY3]], 14 /* CC::al */, $noreg
+  ; CHECK-CVT-NEXT:   [[VCVTBHS1:%[0-9]+]]:spr = VCVTBHS killed [[COPY2]], 14 /* CC::al */, $noreg
+  ; CHECK-CVT-NEXT:   [[VADDS:%[0-9]+]]:spr = nnan ninf nsz arcp contract afn reassoc VADDS killed [[VCVTBHS1]], killed [[VCVTBHS]], 14 /* CC::al */, $noreg
+  ; CHECK-CVT-NEXT:   [[DEF:%[0-9]+]]:spr = IMPLICIT_DEF
+  ; CHECK-CVT-NEXT:   [[VCVTBSH:%[0-9]+]]:spr = VCVTBSH [[DEF]], killed [[VADDS]], 14 /* CC::al */, $noreg
+  ; CHECK-CVT-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY [[VCVTBSH]]
+  ; CHECK-CVT-NEXT:   $r0 = COPY [[COPY4]]
+  ; CHECK-CVT-NEXT:   MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
+  ;
+  ; CHECK-FP16-LABEL: name: fast_fadd
+  ; CHECK-FP16: bb.0.entry:
+  ; CHECK-FP16-NEXT:   liveins: $r0, $r1
+  ; CHECK-FP16-NEXT: {{  $}}
+  ; CHECK-FP16-NEXT:   [[COPY:%[0-9]+]]:rgpr = COPY $r1
+  ; CHECK-FP16-NEXT:   [[COPY1:%[0-9]+]]:rgpr = COPY $r0
+  ; CHECK-FP16-NEXT:   [[VMOVHR:%[0-9]+]]:hpr = VMOVHR [[COPY]], 14, $noreg
+  ; CHECK-FP16-NEXT:   [[VMOVHR1:%[0-9]+]]:hpr = VMOVHR [[COPY1]], 14, $noreg
+  ; CHECK-FP16-NEXT:   [[VADDH:%[0-9]+]]:hpr = nnan ninf nsz arcp contract afn reassoc VADDH killed [[VMOVHR1]], killed [[VMOVHR]], 14, $noreg
+  ; CHECK-FP16-NEXT:   $r0 = COPY [[VADDH]]
+  ; CHECK-FP16-NEXT:   MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
+entry:
+  %add = fadd fast half %x, %y
+  ret half %add
+}
+
+define half @ninf_fadd(half %x, half %y) {
+  ; CHECK-CVT-LABEL: name: ninf_fadd
+  ; CHECK-CVT: bb.0.entry:
+  ; CHECK-CVT-NEXT:   liveins: $r0, $r1
+  ; CHECK-CVT-NEXT: {{  $}}
+  ; CHECK-CVT-NEXT:   [[COPY:%[0-9]+]]:gpr = COPY $r1
+  ; CHECK-CVT-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $r0
+  ; CHECK-CVT-NEXT:   [[COPY2:%[0-9]+]]:spr = COPY [[COPY1]]
+  ; CHECK-CVT-NEXT:   [[COPY3:%[0-9]+]]:spr = COPY [[COPY]]
+  ; CHECK-CVT-NEXT:   [[VCVTBHS:%[0-9]+]]:spr = VCVTBHS killed [[COPY3]], 14 /* CC::al */, $noreg
+  ; CHECK-CVT-NEXT:   [[VCVTBHS1:%[0-9]+]]:spr = VCVTBHS killed [[COPY2]], 14 /* CC::al */, $noreg
+  ; CHECK-CVT-NEXT:   [[VADDS:%[0-9]+]]:spr = ninf VADDS killed [[VCVTBHS1]], killed [[VCVTBHS]], 14 /* CC::al */, $noreg
+  ; CHECK-CVT-NEXT:   [[DEF:%[0-9]+]]:spr = IMPLICIT_DEF
+  ; CHECK-CVT-NEXT:   [[VCVTBSH:%[0-9]+]]:spr = VCVTBSH [[DEF]], killed [[VADDS]], 14 /* CC::al */, $noreg
+  ; CHECK-CVT-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY [[VCVTBSH]]
+  ; CHECK-CVT-NEXT:   $r0 = COPY [[COPY4]]
+  ; CHECK-CVT-NEXT:   MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
+  ;
+  ; CHECK-FP16-LABEL: name: ninf_fadd
+  ; CHECK-FP16: bb.0.entry:
+  ; CHECK-FP16-NEXT:   liveins: $r0, $r1
+  ; CHECK-FP16-NEXT: {{  $}}
+  ; CHECK-FP16-NEXT:   [[COPY:%[0-9]+]]:rgpr = COPY $r1
+  ; CHECK-FP16-NEXT:   [[COPY1:%[0-9]+]]:rgpr = COPY $r0
+  ; CHECK-FP16-NEXT:   [[VMOVHR:%[0-9]+]]:hpr = VMOVHR [[COPY]], 14, $noreg
+  ; CHECK-FP16-NEXT:   [[VMOVHR1:%[0-9]+]]:hpr = VMOVHR [[COPY1]], 14, $noreg
+  ; CHECK-FP16-NEXT:   [[VADDH:%[0-9]+]]:hpr = ninf VADDH killed [[VMOVHR1]], killed [[VMOVHR]], 14, $noreg
+  ; CHECK-FP16-NEXT:   $r0 = COPY [[VADDH]]
+  ; CHECK-FP16-NEXT:   MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
+entry:
+  %add = fadd ninf half %x, %y
+  ret half %add
+}
diff --git a/llvm/test/CodeGen/ARM/vecreduce-fmax-legalization-soft-float.ll b/llvm/test/CodeGen/ARM/vecreduce-fmax-legalization-soft-float.ll
index d3518fe468607..5868bd267e26d 100644
--- a/llvm/test/CodeGen/ARM/vecreduce-fmax-legalization-soft-float.ll
+++ b/llvm/test/CodeGen/ARM/vecreduce-fmax-legalization-soft-float.ll
@@ -11,37 +11,49 @@ define half @test_v4f16(<4 x half> %a) nounwind {
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
 ; CHECK-NEXT:    push {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    mov r8, #255
-; CHECK-NEXT:    mov r4, r3
-; CHECK-NEXT:    orr r8, r8, #65280
+; CHECK-NEXT:    mov r4, #255
+; CHECK-NEXT:    mov r8, r3
+; CHECK-NEXT:    orr r4, r4, #65280
 ; CHECK-NEXT:    mov r5, r2
-; CHECK-NEXT:    and r0, r0, r8
+; CHECK-NEXT:    and r0, r0, r4
 ; CHECK-NEXT:    mov r6, r1
 ; CHECK-NEXT:    bl __aeabi_h2f
 ; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    and r0, r6, r8
+; CHECK-NEXT:    and r0, r6, r4
 ; CHECK-NEXT:    bl __aeabi_h2f
-; CHECK-NEXT:    mov r1, r0
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    bl fmaxf
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    bl __aeabi_fcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    movne r6, r7
+; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    bl __aeabi_f2h
 ; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    and r0, r5, r8
+; CHECK-NEXT:    and r0, r5, r4
 ; CHECK-NEXT:    bl __aeabi_h2f
 ; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    and r0, r6, r8
+; CHECK-NEXT:    and r0, r6, r4
 ; CHECK-NEXT:    bl __aeabi_h2f
 ; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    bl fmaxf
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    bl __aeabi_fcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    movne r5, r6
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    bl __aeabi_f2h
+; CHECK-NEXT:    and r0, r0, r4
+; CHECK-NEXT:    bl __aeabi_h2f
 ; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    and r0, r4, r8
+; CHECK-NEXT:    and r0, r8, r4
 ; CHECK-NEXT:    bl __aeabi_h2f
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    and r0, r5, r8
-; CHECK-NEXT:    bl __aeabi_h2f
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    bl fmaxf
+; CHECK-NEXT:    bl __aeabi_fcmpgt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    movne r4, r5
+; CHECK-NEXT:    mov r0, r4
 ; CHECK-NEXT:    bl __aeabi_f2h
 ; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, lr}
 ; CHECK-NEXT:    mov pc, lr
diff --git a/llvm/test/CodeGen/ARM/vecreduce-fmin-legalization-soft-float.ll b/llvm/test/CodeGen/ARM/vecreduce-fmin-legalization-soft-float.ll
index 14644e00c94b0..2f5645a887616 100644
--- a/llvm/test/CodeGen/ARM/vecreduce-fmin-legalization-soft-float.ll
+++ b/llvm/test/CodeGen/ARM/vecreduce-fmin-legalization-soft-float.ll
@@ -11,37 +11,49 @@ define half @test_v4f16(<4 x half> %a) nounwind {
 ; CHECK:       @ %bb.0:
 ; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
 ; CHECK-NEXT:    push {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    mov r8, #255
-; CHECK-NEXT:    mov r4, r3
-; CHECK-NEXT:    orr r8, r8, #65280
+; CHECK-NEXT:    mov r4, #255
+; CHECK-NEXT:    mov r8, r3
+; CHECK-NEXT:    orr r4, r4, #65280
 ; CHECK-NEXT:    mov r5, r2
-; CHECK-NEXT:    and r0, r0, r8
+; CHECK-NEXT:    and r0, r0, r4
 ; CHECK-NEXT:    mov r6, r1
 ; CHECK-NEXT:    bl __aeabi_h2f
 ; CHECK-NEXT:    mov r7, r0
-; CHECK-NEXT:    and r0, r6, r8
+; CHECK-NEXT:    and r0, r6, r4
 ; CHECK-NEXT:    bl __aeabi_h2f
-; CHECK-NEXT:    mov r1, r0
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    bl fminf
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    bl __aeabi_fcmplt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    movne r6, r7
+; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    bl __aeabi_f2h
 ; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    and r0, r5, r8
+; CHECK-NEXT:    and r0, r5, r4
 ; CHECK-NEXT:    bl __aeabi_h2f
 ; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    and r0, r6, r8
+; CHECK-NEXT:    and r0, r6, r4
 ; CHECK-NEXT:    bl __aeabi_h2f
 ; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    bl fminf
+; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    bl __aeabi_fcmplt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    movne r5, r6
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    bl __aeabi_f2h
+; CHECK-NEXT:    and r0, r0, r4
+; CHECK-NEXT:    bl __aeabi_h2f
 ; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    and r0, r4, r8
+; CHECK-NEXT:    and r0, r8, r4
 ; CHECK-NEXT:    bl __aeabi_h2f
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    and r0, r5, r8
-; CHECK-NEXT:    bl __aeabi_h2f
+; CHECK-NEXT:    mov r0, r5
 ; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    bl fminf
+; CHECK-NEXT:    bl __aeabi_fcmplt
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    movne r4, r5
+; CHECK-NEXT:    mov r0, r4
 ; CHECK-NEXT:    bl __aeabi_f2h
 ; CHECK-NEXT:    pop {r4, r5, r6, r7, r8, lr}
 ; CHECK-NEXT:    mov pc, lr
diff --git a/llvm/test/CodeGen/BPF/CORE/arena_bitcast.ll b/llvm/test/CodeGen/BPF/CORE/arena_bitcast.ll
new file mode 100644
index 0000000000000..bcd71c04d264d
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/arena_bitcast.ll
@@ -0,0 +1,80 @@
+; RUN: opt -O2 %s | llvm-dis > %t1
+; RUN: llc -mcpu=v3 -filetype=asm -o - %t1 | FileCheck %s
+; Source code:
+;   struct lock_t {
+;     int counter;
+;   } __attribute__((preserve_access_index));
+;
+;   #define __arena __attribute__((address_space(1)))
+;   int test(struct lock_t __arena *lock, unsigned val)
+;   {
+;     return __sync_val_compare_and_swap((&lock->counter), val, 1);
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm -Xclang -disable-llvm-passes arena_bitcast.c
+
+target triple = "bpf"
+
+%struct.lock_t = type { i32 }
+
+; Function Attrs: nounwind
+define dso_local i32 @test(ptr addrspace(1) noundef %lock, i32 noundef %val) #0 !dbg !7 {
+entry:
+  %lock.addr = alloca ptr addrspace(1), align 8
+  %val.addr = alloca i32, align 4
+  store ptr addrspace(1) %lock, ptr %lock.addr, align 8, !tbaa !19
+    #dbg_declare(ptr %lock.addr, !17, !DIExpression(), !24)
+  store i32 %val, ptr %val.addr, align 4, !tbaa !25
+    #dbg_declare(ptr %val.addr, !18, !DIExpression(), !27)
+  %0 = load ptr addrspace(1), ptr %lock.addr, align 8, !dbg !28, !tbaa !19
+  %1 = call ptr addrspace(1) @llvm.preserve.struct.access.index.p1.p1(ptr addrspace(1) elementtype(%struct.lock_t) %0, i32 0, i32 0), !dbg !29, !llvm.preserve.access.index !12
+  %2 = load i32, ptr %val.addr, align 4, !dbg !30, !tbaa !25
+  %3 = cmpxchg ptr addrspace(1) %1, i32 %2, i32 1 seq_cst seq_cst, align 4, !dbg !31
+  %4 = extractvalue { i32, i1 } %3, 0, !dbg !31
+  ret i32 %4, !dbg !32
+}
+; CHECK:  r1 = addr_space_cast(r1, 0, 1)
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none)
+declare ptr addrspace(1) @llvm.preserve.struct.access.index.p1.p1(ptr addrspace(1), i32 immarg, i32 immarg) #1
+
+attributes #0 = { nounwind "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { nocallback nofree nosync nounwind willreturn memory(none) }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 21.0.0git (https://github.com/llvm/llvm-project.git 7d4d8509cbec7eecd8aaf2510015b54bc5c173e1)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "arena_bitcast.c", directory: "/root/home/yhs/tests/arena/simple", checksumkind: CSK_MD5, checksum: "51cb51c1fc09d3033dbd9aee9044dc9b")
+!2 = !{i32 7, !"Dwarf Version", i32 5}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{i32 7, !"frame-pointer", i32 2}
+!6 = !{!"clang version 21.0.0git (https://github.com/llvm/llvm-project.git 7d4d8509cbec7eecd8aaf2510015b54bc5c173e1)"}
+!7 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 6, type: !8, scopeLine: 7, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !16)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !11, !15}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64)
+!12 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "lock_t", file: !1, line: 1, size: 32, elements: !13)
+!13 = !{!14}
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "counter", scope: !12, file: !1, line: 2, baseType: !10, size: 32)
+!15 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!16 = !{!17, !18}
+!17 = !DILocalVariable(name: "lock", arg: 1, scope: !7, file: !1, line: 6, type: !11)
+!18 = !DILocalVariable(name: "val", arg: 2, scope: !7, file: !1, line: 6, type: !15)
+!19 = !{!20, !20, i64 0}
+!20 = !{!"p1 _ZTS6lock_t", !21, i64 0}
+!21 = !{!"any pointer", !22, i64 0}
+!22 = !{!"omnipotent char", !23, i64 0}
+!23 = !{!"Simple C/C++ TBAA"}
+!24 = !DILocation(line: 6, column: 33, scope: !7)
+!25 = !{!26, !26, i64 0}
+!26 = !{!"int", !22, i64 0}
+!27 = !DILocation(line: 6, column: 48, scope: !7)
+!28 = !DILocation(line: 8, column: 40, scope: !7)
+!29 = !DILocation(line: 8, column: 46, scope: !7)
+!30 = !DILocation(line: 8, column: 56, scope: !7)
+!31 = !DILocation(line: 8, column: 10, scope: !7)
+!32 = !DILocation(line: 8, column: 3, scope: !7)
diff --git a/llvm/test/CodeGen/DirectX/Metadata/cbuffer_metadata.ll b/llvm/test/CodeGen/DirectX/Metadata/cbuffer_metadata.ll
new file mode 100644
index 0000000000000..816cf04df4780
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/Metadata/cbuffer_metadata.ll
@@ -0,0 +1,77 @@
+; RUN: opt -S -dxil-translate-metadata < %s | FileCheck %s
+; RUN: opt -S --passes="dxil-pretty-printer" < %s 2>&1 | FileCheck %s --check-prefix=PRINT
+; RUN: llc %s --filetype=asm -o - < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,PRINT
+
+target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64"
+target triple = "dxil-pc-shadermodel6.6-compute"
+
+%__cblayout_CB1 = type <{ float, i32, double, <2 x i32> }>
+@CB1.cb = external constant target("dx.CBuffer", target("dx.Layout", %__cblayout_CB1, 24, 0, 4, 8, 16))
+
+%__cblayout_CB2 = type <{ float, double, float, half, i16, i64, i32 }>
+@CB2.cb = external constant target("dx.CBuffer", target("dx.Layout", %__cblayout_CB2, 36, 0, 8, 16, 20, 22, 24, 32))
+
+%__cblayout_CB3 = type <{ double, <3 x float>, float, <3 x double>, half, <2 x double>, float, <3 x half>, <3 x half> }>
+@CB3.cb = external constant target("dx.CBuffer", target("dx.Layout", %__cblayout_CB3, 96, 0, 16, 28, 32, 56, 64, 80, 84, 90))
+
+; PRINT:; Resource Bindings:
+; PRINT-NEXT:;
+; PRINT-NEXT:; Name                                 Type  Format         Dim      ID      HLSL Bind  Count
+; PRINT-NEXT:; ------------------------------ ---------- ------- ----------- ------- -------------- ------
+; PRINT-NEXT:;                                   cbuffer      NA          NA     CB0            cb0     1
+; PRINT-NEXT:;                                   cbuffer      NA          NA     CB1            cb1     1
+; PRINT-NEXT:;                                   cbuffer      NA          NA     CB2    cb5,space15     1
+
+define void @test() #0 {
+
+  ; cbuffer CB1 : register(b0) {
+  ;   float a;
+  ;   int b;
+  ;   double c;
+  ;   int2 d;
+  ; }
+  %CB1.cb_h = call target("dx.CBuffer", target("dx.Layout", %__cblayout_CB1, 24, 0, 4, 8, 16))
+            @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, i1 false)
+  store target("dx.CBuffer", target("dx.Layout", %__cblayout_CB1, 24, 0, 4, 8, 16)) %CB1.cb_h, ptr @CB1.cb, align 4
+
+  ; cbuffer CB2 : register(b0) {
+  ;   float a;
+  ;   double b;
+  ;   float c;
+  ;   half d;
+  ;   uint16_t e;
+  ;   int64_t f;
+  ;   int g;
+  ;}
+
+  %CB2.cb_h = call target("dx.CBuffer", target("dx.Layout", %__cblayout_CB2, 36, 0, 8, 16, 20, 22, 24, 32))
+            @llvm.dx.resource.handlefrombinding(i32 0, i32 1, i32 1, i32 0, i1 false)
+  store target("dx.CBuffer", target("dx.Layout", %__cblayout_CB2, 36, 0, 8, 16, 20, 22, 24, 32)) %CB2.cb_h, ptr @CB2.cb, align 4
+
+  ; cbuffer CB3 : register(b5) {
+  ;   double B0;
+  ;   float3 B1;
+  ;   float B2;
+  ;   double3 B3;
+  ;   half B4;
+  ;   double2 B5;
+  ;   float B6;
+  ;   half3 B7;
+  ;   half3 B8;
+  ; }
+  %CB3.cb_h = call target("dx.CBuffer", target("dx.Layout", %__cblayout_CB3, 96, 0, 16, 28, 32, 56, 64, 80, 84, 90))
+            @llvm.dx.resource.handlefrombinding(i32 15, i32 5, i32 1, i32 0, i1 false)
+  store target("dx.CBuffer", target("dx.Layout", %__cblayout_CB3, 96, 0, 16, 28, 32, 56, 64, 80, 84, 90)) %CB3.cb_h, ptr @CB3.cb, align 4
+
+  ret void
+}
+
+attributes #0 = { noinline nounwind "hlsl.shader"="compute" }
+
+; CHECK: !dx.resources = !{[[ResList:[!][0-9]+]]}
+
+; CHECK: [[ResList]] = !{null, null, [[CBList:[!][0-9]+]], null}
+; CHECK: [[CBList]] = !{![[CB1:[0-9]+]], ![[CB2:[0-9]+]], ![[CB3:[0-9]+]]}
+; CHECK: ![[CB1]] = !{i32 0, ptr @0, !"", i32 0, i32 0, i32 1, i32 24, null}
+; CHECK: ![[CB2]] = !{i32 1, ptr @1, !"", i32 0, i32 1, i32 1, i32 36, null}
+; CHECK: ![[CB3]] = !{i32 2, ptr @2, !"", i32 15, i32 5, i32 1, i32 96, null}
diff --git a/llvm/test/CodeGen/DirectX/Metadata/srv_metadata.ll b/llvm/test/CodeGen/DirectX/Metadata/srv_metadata.ll
new file mode 100644
index 0000000000000..cfdaeb488179d
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/Metadata/srv_metadata.ll
@@ -0,0 +1,107 @@
+; RUN: opt -S -dxil-translate-metadata < %s | FileCheck %s
+; RUN: opt -S --passes="dxil-pretty-printer" < %s 2>&1 | FileCheck %s --check-prefix=PRINT
+; RUN: llc %s --filetype=asm -o - < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,PRINT
+
+target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64"
+target triple = "dxil-pc-shadermodel6.6-compute"
+
+%"class.hlsl::Buffer" = type { target("dx.TypedBuffer", <4 x half>, 0, 0, 0) }
+%"class.hlsl::Buffer.1" = type { target("dx.TypedBuffer", <2 x float>, 0, 0, 0) }
+%"class.hlsl::Buffer.2" = type { target("dx.TypedBuffer", double, 0, 0, 0) }
+%"class.hlsl::Buffer.3" = type { target("dx.TypedBuffer", i32, 0, 0, 1) }
+%"class.hlsl::ByteAddressBuffer" = type { target("dx.RawBuffer", i8, 0, 0) }
+%"class.hlsl::StructuredBuffer" = type { target("dx.RawBuffer", i16, 0, 0) }
+%"class.hlsl::Buffer.4" = type { target("dx.TypedBuffer", i64, 0, 0, 0) }
+
+@Zero = internal global %"class.hlsl::Buffer" poison, align 4
+@One = internal global %"class.hlsl::Buffer.1" poison, align 4
+@Two = internal global %"class.hlsl::Buffer.2" poison, align 4
+@Three = internal global %"class.hlsl::Buffer.3" poison, align 4
+@Four = internal global %"class.hlsl::ByteAddressBuffer" poison, align 4
+@Five = internal global %"class.hlsl::StructuredBuffer" poison, align 4
+@Six = internal global %"class.hlsl::Buffer.4" poison, align 4
+
+; PRINT:; Resource Bindings:
+; PRINT-NEXT:;
+; PRINT-NEXT:; Name                                 Type  Format         Dim      ID      HLSL Bind  Count
+; PRINT-NEXT:; ------------------------------ ---------- ------- ----------- ------- -------------- ------
+; PRINT-NEXT:;                                       SRV     f16         buf      T0             t0     1
+; PRINT-NEXT:;                                       SRV     f32         buf      T1             t1     1
+; PRINT-NEXT:;                                       SRV     f64         buf      T2             t2     1
+; PRINT-NEXT:;                                       SRV     i32         buf      T3             t3     1
+; PRINT-NEXT:;                                       SRV    byte         r/o      T4             t5     1
+; PRINT-NEXT:;                                       SRV  struct         r/o      T5             t6     1
+; PRINT-NEXT:;                                       SRV     u64         buf      T6     t10,space2     1
+; PRINT-NEXT:;                                       SRV     f32         buf      T7      t4,space3   100
+
+define void @test() #0 {
+  ; Buffer<half4> Buf : register(t0)
+  %Zero_h = call target("dx.TypedBuffer", <4 x half>, 0, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, i1 false)
+  store target("dx.TypedBuffer", <4 x half>, 0, 0, 0) %Zero_h, ptr @Zero, align 4
+ 
+  ; Buffer<float4> Buf : register(t1)
+  %One_h = call target("dx.TypedBuffer", <2 x float>, 0, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 0, i32 1, i32 1, i32 0, i1 false)
+  store target("dx.TypedBuffer", <2 x float>, 0, 0, 0) %One_h, ptr @One, align 4
+ 
+  ; Buffer<double> Two : register(t2);
+  %Two_h = call target("dx.TypedBuffer", double, 0, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 0, i32 2, i32 1, i32 0, i1 false)
+  store target("dx.TypedBuffer", double, 0, 0, 0) %Two_h, ptr @Two, align 4
+
+  ; Buffer<int4> Three : register(t3);
+  %Three_h = call target("dx.TypedBuffer", <4 x i32>, 0, 0, 1)
+            @llvm.dx.resource.handlefrombinding(i32 0, i32 3, i32 1, i32 0, i1 false)
+  store target("dx.TypedBuffer", <4 x i32>, 0, 0, 1) %Three_h, ptr @Three, align 4
+
+  ; ByteAddressBuffer Four : register(t4)
+  %Four_h = call target("dx.RawBuffer", i8, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 0, i32 5, i32 1, i32 0, i1 false)
+  store target("dx.RawBuffer", i8, 0, 0) %Four_h, ptr @Four, align 4
+
+  ; StructuredBuffer<int16_t> Five : register(t6);
+  %Five_h = call target("dx.RawBuffer", i16, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 0, i32 6, i32 1, i32 0, i1 false)
+  store target("dx.RawBuffer", i16, 0, 0) %Five_h, ptr @Five, align 4  
+  
+  ; Buffer<double> Six : register(t10, space2);
+  %Six_h = call target("dx.TypedBuffer", i64, 0, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 2, i32 10, i32 1, i32 0, i1 false)
+  store target("dx.TypedBuffer", i64, 0, 0, 0) %Six_h, ptr @Six, align 4
+
+  ; Buffer<float4> Array[100] : register(t4, space3);
+  ; Buffer<float4> B1 = Array[30];
+  ; Buffer<float4> B1 = Array[42];
+  ; resource array accesses should produce one metadata entry   
+  %Array_30_h = call target("dx.TypedBuffer", <4 x float>, 0, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 3, i32 4, i32 100, i32 30, i1 false)
+  %Array_42_h = call target("dx.TypedBuffer", <4 x float>, 0, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 3, i32 4, i32 100, i32 42, i1 false)
+
+  ret void
+}
+
+attributes #0 = { noinline nounwind "hlsl.shader"="compute" }
+
+; CHECK: !dx.resources = !{[[ResList:[!][0-9]+]]}
+
+; CHECK: [[ResList]] = !{[[SRVList:[!][0-9]+]], null, null, null}
+; CHECK: [[SRVList]] = !{![[Zero:[0-9]+]], ![[One:[0-9]+]], ![[Two:[0-9]+]],
+; CHECK-SAME: ![[Three:[0-9]+]], ![[Four:[0-9]+]], ![[Five:[0-9]+]],
+; CHECK-SAME: ![[Six:[0-9]+]], ![[Array:[0-9]+]]}
+
+; CHECK: ![[Zero]] = !{i32 0, ptr @0, !"", i32 0, i32 0, i32 1, i32 10, i32 0, ![[Half:[0-9]+]]}
+; CHECK: ![[Half]] = !{i32 0, i32 8}
+; CHECK: ![[One]] = !{i32 1, ptr @1, !"", i32 0, i32 1, i32 1, i32 10, i32 0, ![[Float:[0-9]+]]}
+; CHECK: ![[Float]] = !{i32 0, i32 9}
+; CHECK: ![[Two]] = !{i32 2, ptr @2, !"", i32 0, i32 2, i32 1, i32 10, i32 0, ![[Double:[0-9]+]]}
+; CHECK: ![[Double]] = !{i32 0, i32 10}
+; CHECK: ![[Three]] = !{i32 3, ptr @3, !"", i32 0, i32 3, i32 1, i32 10, i32 0, ![[I32:[0-9]+]]}
+; CHECK: ![[I32]] = !{i32 0, i32 4}
+; CHECK: ![[Four]] = !{i32 4, ptr @4, !"", i32 0, i32 5, i32 1, i32 11, i32 0, null}
+; CHECK: ![[Five]] = !{i32 5, ptr @5, !"", i32 0, i32 6, i32 1, i32 12, i32 0, ![[FiveStride:[0-9]+]]}
+; CHECK: ![[FiveStride]] = !{i32 1, i32 2}
+; CHECK: ![[Six]] = !{i32 6, ptr @6, !"", i32 2, i32 10, i32 1, i32 10, i32 0, ![[U64:[0-9]+]]}
+; CHECK: ![[U64]] = !{i32 0, i32 7}
+; CHECK: ![[Array]] = !{i32 7, ptr @7, !"", i32 3, i32 4, i32 100, i32 10, i32 0, ![[Float]]}
diff --git a/llvm/test/CodeGen/DirectX/Metadata/uav_metadata.ll b/llvm/test/CodeGen/DirectX/Metadata/uav_metadata.ll
new file mode 100644
index 0000000000000..e2f7cffb50f3c
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/Metadata/uav_metadata.ll
@@ -0,0 +1,136 @@
+; RUN: opt -S -dxil-translate-metadata < %s | FileCheck %s
+; RUN: opt -S --passes="dxil-pretty-printer" < %s 2>&1 | FileCheck %s --check-prefix=PRINT
+; RUN: llc %s --filetype=asm -o - < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,PRINT
+
+target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64"
+target triple = "dxil-pc-shadermodel6.6-compute"
+
+%"class.hlsl::RWBuffer" = type { target("dx.TypedBuffer", <4 x half>, 1, 0, 0) }
+%"class.hlsl::RWBuffer.1" = type { target("dx.TypedBuffer", <2 x float>, 1, 0, 0) }
+%"class.hlsl::RWBuffer.2" = type { target("dx.TypedBuffer", double, 1, 0, 0) }
+%"class.hlsl::RWBuffer.3" = type { target("dx.TypedBuffer", i32, 1, 0, 1) }
+%"class.hlsl::RWByteAddressBuffer" = type { target("dx.RawBuffer", i8, 1, 0) }
+%"class.hlsl::RWStructuredBuffer" = type { target("dx.RawBuffer", i16, 1, 0) }
+%"class.hlsl::RasterizerOrderedBuffer" = type { target("dx.TypedBuffer", <4 x i32>, 1, 1, 1) }
+%"class.hlsl::RasterizerOrderedStructuredBuffer" = type { target("dx.RawBuffer", <4 x i32>, 1, 1) }
+%"class.hlsl::RasterizerOrderedByteAddressBuffer" = type { target("dx.RawBuffer", i8, 1, 1) }
+%"class.hlsl::RWBuffer.4" = type { target("dx.TypedBuffer", i64, 1, 0, 0) }
+
+@Zero = internal global %"class.hlsl::RWBuffer" poison, align 4
+@One = internal global %"class.hlsl::RWBuffer.1" poison, align 4
+@Two = internal global %"class.hlsl::RWBuffer.2" poison, align 4
+@Three = internal global %"class.hlsl::RWBuffer.3" poison, align 4
+@Four = internal global %"class.hlsl::RWByteAddressBuffer" poison, align 4
+@Five = internal global %"class.hlsl::RWStructuredBuffer" poison, align 4
+@Six = internal global %"class.hlsl::RasterizerOrderedBuffer" poison, align 4
+@Seven = internal global %"class.hlsl::RasterizerOrderedStructuredBuffer" poison, align 4
+@Eight = internal global %"class.hlsl::RasterizerOrderedByteAddressBuffer" poison, align 4
+@Nine = internal global %"class.hlsl::RWBuffer.4" poison, align 4
+
+; PRINT:; Resource Bindings:
+; PRINT-NEXT:;
+; PRINT-NEXT:; Name                                 Type  Format         Dim      ID      HLSL Bind  Count
+; PRINT-NEXT:; ------------------------------ ---------- ------- ----------- ------- -------------- ------
+; PRINT-NEXT:;                                       UAV     f16         buf      U0             u0     1
+; PRINT-NEXT:;                                       UAV     f32         buf      U1             u1     1
+; PRINT-NEXT:;                                       UAV     f64         buf      U2             u2     1
+; PRINT-NEXT:;                                       UAV     i32         buf      U3             u3     1
+; PRINT-NEXT:;                                       UAV    byte         r/w      U4             u5     1
+; PRINT-NEXT:;                                       UAV  struct         r/w      U5             u6     1
+; PRINT-NEXT:;                                       UAV     i32         buf      U6             u7     1
+; PRINT-NEXT:;                                       UAV  struct         r/w      U7             u8     1
+; PRINT-NEXT:;                                       UAV    byte         r/w      U8             u9     1
+; PRINT-NEXT:;                                       UAV     u64         buf      U9     u10,space2     1
+; PRINT-NEXT:;                                       UAV     f32         buf     U10      u4,space3   100
+
+define void @test() #0 {
+  ; RWBuffer<half4> Zero : register(u0)
+  %Zero_h = call target("dx.TypedBuffer", <4 x half>, 1, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, i1 false)
+  store target("dx.TypedBuffer", <4 x half>, 1, 0, 0) %Zero_h, ptr @Zero, align 4
+ 
+  ; RWBuffer<float4> One : register(u1)
+  %One_h = call target("dx.TypedBuffer", <2 x float>, 1, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 0, i32 1, i32 1, i32 0, i1 false)
+  store target("dx.TypedBuffer", <2 x float>, 1, 0, 0) %One_h, ptr @One, align 4
+ 
+  ; RWBuffer<double> Two : register(u2);
+  %Two_h = call target("dx.TypedBuffer", double, 1, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 0, i32 2, i32 1, i32 0, i1 false)
+  store target("dx.TypedBuffer", double, 1, 0, 0) %Two_h, ptr @Two, align 4
+
+  ; RWBuffer<int4> Three : register(u3);
+  %Three_h = call target("dx.TypedBuffer", <4 x i32>, 1, 0, 1)
+            @llvm.dx.resource.handlefrombinding(i32 0, i32 3, i32 1, i32 0, i1 false)
+  store target("dx.TypedBuffer", <4 x i32>, 1, 0, 1) %Three_h, ptr @Three, align 4
+
+  ; ByteAddressBuffer Four : register(u5)
+  %Four_h = call target("dx.RawBuffer", i8, 1, 0)
+            @llvm.dx.resource.handlefrombinding(i32 0, i32 5, i32 1, i32 0, i1 false)
+  store target("dx.RawBuffer", i8, 1, 0) %Four_h, ptr @Four, align 4
+
+  ; RWStructuredBuffer<int16_t> Five : register(u6);
+  %Five_h = call target("dx.RawBuffer", i16, 1, 0)
+            @llvm.dx.resource.handlefrombinding(i32 0, i32 6, i32 1, i32 0, i1 false)
+  store target("dx.RawBuffer", i16, 1, 0) %Five_h, ptr @Five, align 4
+  
+  ; RasterizerOrderedBuffer<int4> Six : register(u7);
+  %Six_h = call target("dx.TypedBuffer", <4 x i32>, 1, 1, 1)
+            @llvm.dx.resource.handlefrombinding(i32 0, i32 7, i32 1, i32 0, i1 false)
+  store target("dx.TypedBuffer", <4 x i32>, 1, 1, 1) %Six_h, ptr @Six, align 4
+
+  ; RasterizerOrderedStructuredBuffer<uint4> Seven : register(u3, space10);
+  %Seven_h = call target("dx.RawBuffer", <4 x i32>, 1, 1)
+            @llvm.dx.resource.handlefrombinding(i32 0, i32 8, i32 1, i32 0, i1 false)
+  store target("dx.RawBuffer", <4 x i32>, 1, 1) %Seven_h, ptr @Seven, align 4
+
+  ; RasterizerOrderedByteAddressBuffer Eight : register(u9); 
+  %Eight_h = call target("dx.RawBuffer", i8, 1, 1) 
+            @llvm.dx.resource.handlefrombinding(i32 0, i32 9, i32 1, i32 0, i1 false)
+  store target("dx.RawBuffer", i8, 1, 1) %Eight_h, ptr @Eight, align 4
+
+  ; RWBuffer<double> Nine : register(u2);
+  %Nine_h = call target("dx.TypedBuffer", i64, 1, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 2, i32 10, i32 1, i32 0, i1 false)
+  store target("dx.TypedBuffer", i64, 1, 0, 0) %Nine_h, ptr @Nine, align 4
+
+  ; RWBuffer<float4> Array[100] : register(u4, space3);
+  ; RWBuffer<float4> B1 = Array[30];
+  ; RWBuffer<float4> B1 = Array[42];
+  ; resource array accesses should produce one metadata entry   
+  %Array_30_h = call target("dx.TypedBuffer", <4 x float>, 1, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 3, i32 4, i32 100, i32 30, i1 false)
+  %Array_42_h = call target("dx.TypedBuffer", <4 x float>, 1, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 3, i32 4, i32 100, i32 42, i1 false)
+
+  ret void
+}
+
+attributes #0 = { noinline nounwind "hlsl.shader"="compute" }
+
+; CHECK: !dx.resources = !{[[ResList:[!][0-9]+]]}
+
+; CHECK: [[ResList]] = !{null, [[UAVList:[!][0-9]+]], null, null}
+; CHECK: [[UAVList]] = !{![[Zero:[0-9]+]], ![[One:[0-9]+]], ![[Two:[0-9]+]],
+; CHECK-SAME: ![[Three:[0-9]+]], ![[Four:[0-9]+]], ![[Five:[0-9]+]],
+; CHECK-SAME: ![[Six:[0-9]+]], ![[Seven:[0-9]+]], ![[Eight:[0-9]+]],
+; CHECK-SAME: ![[Nine:[0-9]+]], ![[Array:[0-9]+]]}
+
+; CHECK: ![[Zero]] = !{i32 0, ptr @0, !"", i32 0, i32 0, i32 1, i32 10, i1 false, i1 false, i1 false, ![[Half:[0-9]+]]}
+; CHECK: ![[Half]] = !{i32 0, i32 8}
+; CHECK: ![[One]] = !{i32 1, ptr @1, !"", i32 0, i32 1, i32 1, i32 10, i1 false, i1 false, i1 false, ![[Float:[0-9]+]]}
+; CHECK: ![[Float]] = !{i32 0, i32 9}
+; CHECK: ![[Two]] = !{i32 2, ptr @2, !"", i32 0, i32 2, i32 1, i32 10, i1 false, i1 false, i1 false, ![[Double:[0-9]+]]}
+; CHECK: ![[Double]] = !{i32 0, i32 10}
+; CHECK: ![[Three]] = !{i32 3, ptr @3, !"", i32 0, i32 3, i32 1, i32 10, i1 false, i1 false, i1 false, ![[I32:[0-9]+]]}
+; CHECK: ![[I32]] = !{i32 0, i32 4}
+; CHECK: ![[Four]] = !{i32 4, ptr @4, !"", i32 0, i32 5, i32 1, i32 11, i1 false, i1 false, i1 false, null}
+; CHECK: ![[Five]] = !{i32 5, ptr @5, !"", i32 0, i32 6, i32 1, i32 12, i1 false, i1 false, i1 false, ![[FiveStride:[0-9]+]]}
+; CHECK: ![[FiveStride]] = !{i32 1, i32 2}
+; CHECK: ![[Six]] = !{i32 6, ptr @6, !"", i32 0, i32 7, i32 1, i32 10, i1 false, i1 false, i1 true, ![[I32]]}
+; CHECK: ![[Seven]] = !{i32 7, ptr @7, !"", i32 0, i32 8, i32 1, i32 12, i1 false, i1 false, i1 true, ![[SevenStride:[0-9]+]]}
+; CHECK: ![[SevenStride]] = !{i32 1, i32 16}
+; CHECK: ![[Eight]] = !{i32 8, ptr @8, !"", i32 0, i32 9, i32 1, i32 11, i1 false, i1 false, i1 true, null}
+; CHECK: ![[Nine]] = !{i32 9, ptr @9, !"", i32 2, i32 10, i32 1, i32 10, i1 false, i1 false, i1 false, ![[U64:[0-9]+]]}
+; CHECK: ![[U64]] = !{i32 0, i32 7}
+; CHECK: ![[Array]] = !{i32 10, ptr @10, !"", i32 3, i32 4, i32 100, i32 10, i1 false, i1 false, i1 false, ![[Float]]}
diff --git a/llvm/test/CodeGen/DirectX/UAVMetadata.ll b/llvm/test/CodeGen/DirectX/UAVMetadata.ll
deleted file mode 100644
index 2c242ec08eda5..0000000000000
--- a/llvm/test/CodeGen/DirectX/UAVMetadata.ll
+++ /dev/null
@@ -1,77 +0,0 @@
-; RUN: opt -S -dxil-translate-metadata < %s | FileCheck %s
-; RUN: opt -S --passes="dxil-pretty-printer" < %s 2>&1 | FileCheck %s --check-prefix=PRINT
-; RUN: llc %s --filetype=asm -o - < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,PRINT
-
-target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64"
-target triple = "dxil-pc-shadermodel6.0-library"
-
-%"class.hlsl::RWBuffer" = type { ptr }
-
-
-; PRINT:; Resource Bindings:
-; PRINT-NEXT:;
-; PRINT-NEXT:; Name                                 Type  Format         Dim      ID      HLSL Bind  Count
-; PRINT-NEXT:; ------------------------------ ---------- ------- ----------- ------- -------------- ------
-; PRINT-NEXT:;                                       UAV     f16         buf      U0             u0     1
-; PRINT-NEXT:;                                       UAV     f32         buf      U1             u1     1
-; PRINT-NEXT:;                                       UAV     f64         buf      U2             u2     1
-; PRINT-NEXT:;                                       UAV      i1         buf      U3             u3     2
-; PRINT-NEXT:;                                       UAV    byte         r/w      U4             u5     1
-; PRINT-NEXT:;                                       UAV  struct         r/w      U5             u6     1
-; PRINT-NEXT:;                                       UAV     i32         buf      U6             u7     1
-; PRINT-NEXT:;                                       UAV  struct         r/w      U7             u8     1
-; PRINT-NEXT:;                                       UAV    byte         r/w      U8             u9     1
-; PRINT-NEXT:;                                       UAV     u64         buf      U9     u10,space2     1
-
-@Zero = local_unnamed_addr global %"class.hlsl::RWBuffer" zeroinitializer, align 4
-@One = local_unnamed_addr global %"class.hlsl::RWBuffer" zeroinitializer, align 4
-@Two = local_unnamed_addr global %"class.hlsl::RWBuffer" zeroinitializer, align 4
-@Three = local_unnamed_addr global [2 x %"class.hlsl::RWBuffer"] zeroinitializer, align 4
-@Four = local_unnamed_addr global %"class.hlsl::RWBuffer" zeroinitializer, align 4
-@Five = local_unnamed_addr global %"class.hlsl::RWBuffer" zeroinitializer, align 4
-@Six = local_unnamed_addr global %"class.hlsl::RWBuffer" zeroinitializer, align 4
-@Seven = local_unnamed_addr global %"class.hlsl::RWBuffer" zeroinitializer, align 4
-@Eight = local_unnamed_addr global %"class.hlsl::RWBuffer" zeroinitializer, align 4
-@Nine = local_unnamed_addr global %"class.hlsl::RWBuffer" zeroinitializer, align 4
-
-
-!hlsl.uavs = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9}
-
-!0 = !{ptr @Zero, i32 10, i32 8, i1 false, i32 0, i32 0}
-!1 = !{ptr @One, i32 10, i32 9, i1 false, i32 1, i32 0}
-!2 = !{ptr @Two, i32 10, i32 10, i1 false, i32 2, i32 0}
-!3 = !{ptr @Three, i32 10, i32 1, i1 false, i32 3, i32 0}
-!4 = !{ptr @Four, i32 11, i32 2, i1 false, i32 5, i32 0}
-!5 = !{ptr @Five, i32 12, i32 3, i1 false, i32 6, i32 0}
-!6 = !{ptr @Six, i32 10, i32 4, i1 true, i32 7, i32 0}
-!7 = !{ptr @Seven, i32 12, i32 5, i1 true, i32 8, i32 0}
-!8 = !{ptr @Eight, i32 11, i32 6, i1 true, i32 9, i32 0}
-!9 = !{ptr @Nine, i32 10, i32 7, i1 false, i32 10, i32 2}
-
-; CHECK: !dx.resources = !{[[ResList:[!][0-9]+]]}
-
-; CHECK: [[ResList]] = !{null, [[UAVList:[!][0-9]+]], null, null}
-; CHECK: [[UAVList]] = !{[[Zero:[!][0-9]+]], [[One:[!][0-9]+]],
-; CHECK-SAME: [[Two:[!][0-9]+]], [[Three:[!][0-9]+]], [[Four:[!][0-9]+]],
-; CHECK-SAME: [[Five:[!][0-9]+]], [[Six:[!][0-9]+]], [[Seven:[!][0-9]+]],
-; CHECK-SAME: [[Eight:[!][0-9]+]], [[Nine:[!][0-9]+]]}
-; CHECK: [[Zero]] = !{i32 0, ptr @Zero, !"", i32 0, i32 0, i32 1, i32 10, i1 false, i1 false, i1 false, [[Half:[!][0-9]+]]}
-; CHECK: [[Half]] = !{i32 0, i32 8}
-; CHECK: [[One]] = !{i32 1, ptr @One, !"", i32 0, i32 1, i32 1, i32 10, i1 false, i1 false, i1 false, [[Float:[!][0-9]+]]}
-; CHECK: [[Float]] = !{i32 0, i32 9}
-; CHECK: [[Two]] = !{i32 2, ptr @Two, !"", i32 0, i32 2, i32 1, i32 10, i1 false, i1 false, i1 false, [[Double:[!][0-9]+]]}
-; CHECK: [[Double]] = !{i32 0, i32 10}
-; CHECK: [[Three]] = !{i32 3, ptr @Three, !"", i32 0, i32 3, i32 2, i32 10, i1 false, i1 false, i1 false, [[Bool:[!][0-9]+]]}
-; CHECK: [[Bool]] = !{i32 0, i32 1}
-; CHECK: [[Four]] = !{i32 4, ptr @Four, !"", i32 0, i32 5, i32 1, i32 11, i1 false, i1 false, i1 false, [[I16:[!][0-9]+]]}
-; CHECK: [[I16]] = !{i32 0, i32 2}
-; CHECK: [[Five]] = !{i32 5, ptr @Five, !"", i32 0, i32 6, i32 1, i32 12, i1 false, i1 false, i1 false, [[U16:[!][0-9]+]]}
-; CHECK: [[U16]] = !{i32 0, i32 3}
-; CHECK: [[Six]] = !{i32 6, ptr @Six, !"", i32 0, i32 7, i32 1, i32 10, i1 false, i1 false, i1 true, [[I32:[!][0-9]+]]}
-; CHECK: [[I32]] = !{i32 0, i32 4}
-; CHECK: [[Seven]] = !{i32 7, ptr @Seven, !"", i32 0, i32 8, i32 1, i32 12, i1 false, i1 false, i1 true, [[U32:[!][0-9]+]]}
-; CHECK: [[U32]] = !{i32 0, i32 5}
-; CHECK: [[Eight]] = !{i32 8, ptr @Eight, !"", i32 0, i32 9, i32 1, i32 11, i1 false, i1 false, i1 true, [[I64:[!][0-9]+]]}
-; CHECK: [[I64]] = !{i32 0, i32 6}
-; CHECK: [[Nine]] = !{i32 9, ptr @Nine, !"", i32 2, i32 10, i32 1, i32 10, i1 false, i1 false, i1 false, [[U64:[!][0-9]+]]}
-; CHECK: [[U64]] = !{i32 0, i32 7}
diff --git a/llvm/test/CodeGen/DirectX/cbuf.ll b/llvm/test/CodeGen/DirectX/cbuf.ll
deleted file mode 100644
index 7589da5e15bd7..0000000000000
--- a/llvm/test/CodeGen/DirectX/cbuf.ll
+++ /dev/null
@@ -1,37 +0,0 @@
-; RUN: opt -S -dxil-translate-metadata < %s | FileCheck %s --check-prefix=DXILMD
-; RUN: opt -S --passes="dxil-pretty-printer" < %s 2>&1 | FileCheck %s --check-prefix=PRINT
-
-target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64"
-target triple = "dxil-unknown-shadermodel6.7-library"
-
-; Make sure the size is 24 = 16 + 8 (float,i32,double -> 16 and int2 -> 8)
-; DXILMD:!{i32 0, ptr @A.cb., !"", i32 1, i32 2, i32 1, i32 24}
-
-; Make sure match register(b2, space1) with ID 0.
-; PRINT:cbuffer      NA          NA     CB0     cb2,space1     1
-
-@A.cb. = external constant { float, i32, double, <2 x i32> }
-
-; Function Attrs: noinline nounwind optnone
-define noundef float @"?foo@@YAMXZ"() #0 {
-entry:
-  %0 = load float, ptr @A.cb., align 4
-  %conv = fpext float %0 to double
-  %1 = load double, ptr getelementptr inbounds ({ float, i32, double, <2 x i32> }, ptr @A.cb., i32 0, i32 2), align 8
-  %2 = load <2 x i32>, ptr getelementptr inbounds ({ float, i32, double, <2 x i32> }, ptr @A.cb., i32 0, i32 3), align 8
-  %3 = extractelement <2 x i32> %2, i32 1
-  %conv1 = sitofp i32 %3 to double
-  %4 = call double @llvm.fmuladd.f64(double %1, double %conv1, double %conv)
-  %conv2 = fptrunc double %4 to float
-  ret float %conv2
-}
-
-; Function Attrs: nocallback nofree nosync nounwind readnone speculatable willreturn
-declare double @llvm.fmuladd.f64(double, double, double) #1
-
-attributes #0 = { noinline nounwind }
-attributes #1 = { nocallback nofree nosync nounwind readnone speculatable willreturn }
-
-!hlsl.cbufs = !{!1}
-
-!1 = !{ptr @A.cb., !"A.cb.ty", i32 13, i1 false, i32 2, i32 1}
diff --git a/llvm/test/CodeGen/DirectX/legacy_cb_layout_0.ll b/llvm/test/CodeGen/DirectX/legacy_cb_layout_0.ll
deleted file mode 100644
index 1a618092c5fed..0000000000000
--- a/llvm/test/CodeGen/DirectX/legacy_cb_layout_0.ll
+++ /dev/null
@@ -1,14 +0,0 @@
-; RUN: opt -S -dxil-translate-metadata < %s | FileCheck %s --check-prefix=DXILMD
-
-target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64"
-target triple = "dxil-unknown-shadermodel6.7-library"
-
-; Make sure the size is 36 = 16 + 16 + 4 (float, double -> 16, float, half, i16, i64 -> 16 and int -> 4)
-; DXILMD:!{i32 0, ptr @A.cb., !"", i32 0, i32 2, i32 1, i32 36}
-
-@A.cb. = external local_unnamed_addr constant { float, double, float, half, i16, i64, i32 }
-
-
-!hlsl.cbufs = !{!1}
-
-!1 = !{ptr @A.cb., !"A.cb.ty", i32 13, i1 false, i32 2, i32 0}
diff --git a/llvm/test/CodeGen/DirectX/legacy_cb_layout_1.ll b/llvm/test/CodeGen/DirectX/legacy_cb_layout_1.ll
deleted file mode 100644
index 6886f2690209d..0000000000000
--- a/llvm/test/CodeGen/DirectX/legacy_cb_layout_1.ll
+++ /dev/null
@@ -1,37 +0,0 @@
-; RUN: opt -S -dxil-translate-metadata < %s | FileCheck %s --check-prefix=DXILMD
-
-target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64"
-target triple = "dxil-unknown-shadermodel6.7-library"
-
-;
-; cbuffer B
-; {
-;
-;   struct B
-;   {
-;
-;       double B0;                                    ; Offset:    0
-;       float3 B1;                                    ; Offset:   16
-;       float B2;                                     ; Offset:   28
-;       double3 B3;                                   ; Offset:   32
-;       half B4;                                      ; Offset:   56
-;       double2 B5;                                   ; Offset:   64
-;       float B6;                                     ; Offset:   80
-;       half3 B7;                                     ; Offset:   84
-;       half3 B8;                                     ; Offset:   90
-;   
-;   } B;                                              ; Offset:    0 Size:    96
-;
-; }
-;
-
-
-; Make sure the size is 96
-; DXILMD:!{i32 0, ptr @B.cb., !"", i32 0, i32 1, i32 1, i32 96}
-
-@B.cb. = external local_unnamed_addr constant { double, <3 x float>, float, <3 x double>, half, <2 x double>, float, <3 x half>, <3 x half> }
-
-
-!hlsl.cbufs = !{!0}
-
-!0 = !{ptr @B.cb., !"B.cb.ty", i32 13, i1 false, i32 1, i32 0}
diff --git a/llvm/test/CodeGen/DirectX/legacy_cb_layout_2.ll b/llvm/test/CodeGen/DirectX/legacy_cb_layout_2.ll
deleted file mode 100644
index 3b08b25542201..0000000000000
--- a/llvm/test/CodeGen/DirectX/legacy_cb_layout_2.ll
+++ /dev/null
@@ -1,51 +0,0 @@
-; RUN: opt -S -dxil-translate-metadata < %s | FileCheck %s --check-prefix=DXILMD
-
-target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64"
-target triple = "dxil-unknown-shadermodel6.7-library"
-
-; cbuffer B
-; {
-;
-;   struct B
-;   {
-;
-;       double B0[2];                                 ; Offset:    0
-;       float3 B1[3];                                 ; Offset:   32
-;       float B2;                                     ; Offset:   76
-;       double B3[3];                                 ; Offset:   80
-;       half B4;                                      ; Offset:  120
-;       double2 B5[1];                                ; Offset:  128
-;       float B6;                                     ; Offset:  144
-;       half3 B7[2];                                  ; Offset:  160
-;       half3 B8;                                     ; Offset:  182
-;   
-;   } B;                                              ; Offset:    0 Size:   188
-;
-; }
-;
-; cbuffer B
-; {
-;
-;   struct B.0
-;   {
-;
-;       double3 B9[3];                                ; Offset:    0
-;       half3 B10;                                    ; Offset:   88
-;   
-;   } B;                                              ; Offset:    0 Size:    94
-;
-; }
-
-
-; Make sure the size is 188.
-; DXILMD:!{i32 0, ptr @B.cb., !"", i32 0, i32 1, i32 1, i32 188}
-; Make sure the size is 94.
-; DXILMD:!{i32 1, ptr @B.cb..1, !"", i32 0, i32 2, i32 1, i32 94}
-
-@B.cb. = external local_unnamed_addr constant { [2 x double], [3 x <3 x float>], float, [3 x double], half, [1 x <2 x double>], float, [2 x <3 x half>], <3 x half> }
-@B.cb..1 = external local_unnamed_addr constant { [3 x <3 x double>], <3 x half> }
-
-!hlsl.cbufs = !{!0, !1}
-
-!0 = !{ptr @B.cb., !"B.cb.ty", i32 13, i1 false, i32 1, i32 0}
-!1 = !{ptr @B.cb..1, !"B.cb.ty", i32 13, i1 false, i32 2, i32 0}
diff --git a/llvm/test/CodeGen/DirectX/legacy_cb_layout_3.ll b/llvm/test/CodeGen/DirectX/legacy_cb_layout_3.ll
deleted file mode 100644
index f01afbdab9673..0000000000000
--- a/llvm/test/CodeGen/DirectX/legacy_cb_layout_3.ll
+++ /dev/null
@@ -1,81 +0,0 @@
-; RUN: opt -S -dxil-translate-metadata < %s | FileCheck %s --check-prefix=DXILMD
-
-target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64"
-target triple = "dxil-unknown-shadermodel6.7-library"
-
-; cbuffer D
-; {
-;
-;   struct D
-;   {
-;
-;       int D0;                                       ; Offset:    0
-;       struct struct.B
-;       {
-;
-;           double B0;                                ; Offset:   16
-;           float3 B1;                                ; Offset:   32
-;           float B2;                                 ; Offset:   44
-;           double3 B3;                               ; Offset:   48
-;           half B4;                                  ; Offset:   72
-;           double2 B5;                               ; Offset:   80
-;           float B6;                                 ; Offset:   96
-;           half3 B7;                                 ; Offset:  100
-;           half3 B8;                                 ; Offset:  106
-;       
-;       } D1;                                         ; Offset:   16
-;
-;       half D2;                                      ; Offset:  112
-;       struct struct.C
-;       {
-;
-;           struct struct.A
-;           {
-;
-;               float A0;                             ; Offset:  128
-;               double A1;                            ; Offset:  136
-;               float A2;                             ; Offset:  144
-;               half A3;                              ; Offset:  148
-;               int16_t A4;                           ; Offset:  150
-;               int64_t A5;                           ; Offset:  152
-;               int A6;                               ; Offset:  160
-;           
-;           } C0;                                     ; Offset:  128
-;
-;           float C1[1];                              ; Offset:  176
-;           struct struct.B
-;           {
-;
-;               double B0;                            ; Offset:  192
-;               float3 B1;                            ; Offset:  208
-;               float B2;                             ; Offset:  220
-;               double3 B3;                           ; Offset:  224
-;               half B4;                              ; Offset:  248
-;               double2 B5;                           ; Offset:  256
-;               float B6;                             ; Offset:  272
-;               half3 B7;                             ; Offset:  276
-;               half3 B8;                             ; Offset:  282
-;           
-;           } C2[2];;                                 ; Offset:  192
-;
-;           half C3;                                  ; Offset:  384
-;       
-;       } D3;                                         ; Offset:  128
-;
-;       double D4;                                    ; Offset:  392
-;   
-;   } D;                                              ; Offset:    0 Size:   400
-
-
-; Make sure the size is 400
-; DXILMD:!{i32 0, ptr @D.cb., !"", i32 0, i32 1, i32 1, i32 400}
-
-
-%struct.B = type <{ double, <3 x float>, float, <3 x double>, half, <2 x double>, float, <3 x half>, <3 x half> }>
-%struct.C = type <{ %struct.A, [1 x float], [2 x %struct.B], half }>
-%struct.A = type <{ float, double, float, half, i16, i64, i32 }>
-
-@D.cb. = external local_unnamed_addr constant { i32, %struct.B, half, %struct.C, double }
-
-!hlsl.cbufs = !{!0}
-!0 = !{ptr @D.cb., !"D.cb.ty", i32 13, i1 false, i32 1, i32 0}
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/spill-phys-vgprs.mir b/llvm/test/CodeGen/MIR/AMDGPU/spill-phys-vgprs.mir
index 4d6e33cf0b68a..b427c5bdd7229 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/spill-phys-vgprs.mir
+++ b/llvm/test/CodeGen/MIR/AMDGPU/spill-phys-vgprs.mir
@@ -1,8 +1,7 @@
 # RUN: llc -mtriple=amdgcn-amd-amdhsa --start-before=si-lower-sgpr-spills --stop-after=prologepilog -o - %s | FileCheck %s
 
 # CHECK: csr_sgpr_spill
-# CHECK: spillPhysVGPRs
-# CHECK-NEXT: - '$vgpr0'
+# CHECK-NOT: spillPhysVGPRs
 ---
 name: csr_sgpr_spill
 tracksRegLiveness: true
diff --git a/llvm/test/CodeGen/NVPTX/st_bulk.ll b/llvm/test/CodeGen/NVPTX/st_bulk.ll
new file mode 100644
index 0000000000000..785f78a6f9519
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/st_bulk.ll
@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100 -mattr=+ptx86 | FileCheck --check-prefixes=CHECK,CHECK-PTX64 %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100 -mattr=+ptx86 --nvptx-short-ptr | FileCheck --check-prefixes=CHECK,CHECK-PTX-SHARED32 %s
+; RUN: %if ptxas-12.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_100 -mattr=+ptx86 | %ptxas-verify -arch=sm_100 %}
+; RUN: %if ptxas-12.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_100 -mattr=+ptx86 --nvptx-short-ptr | %ptxas-verify -arch=sm_100 %}
+
+declare void @llvm.nvvm.st.bulk(ptr, i64, i64)
+define void @st_bulk(ptr %dest_addr, i64 %size) {
+; CHECK-LABEL: st_bulk(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u64 %rd1, [st_bulk_param_0];
+; CHECK-NEXT:    ld.param.u64 %rd2, [st_bulk_param_1];
+; CHECK-NEXT:    st.bulk [%rd1], %rd2, 0;
+; CHECK-NEXT:    ret;
+  call void @llvm.nvvm.st.bulk(ptr %dest_addr, i64 %size, i64 0)
+  ret void
+}
+
+declare void @llvm.nvvm.st.bulk.shared.cta(ptr addrspace(3), i64, i64)
+define void @st_bulk_shared_cta(ptr addrspace(3) %dest_addr, i64 %size) {
+; CHECK-PTX64-LABEL: st_bulk_shared_cta(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<3>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.u64 %rd1, [st_bulk_shared_cta_param_0];
+; CHECK-PTX64-NEXT:    ld.param.u64 %rd2, [st_bulk_shared_cta_param_1];
+; CHECK-PTX64-NEXT:    st.bulk.shared::cta [%rd1], %rd2, 0;
+; CHECK-PTX64-NEXT:    ret;
+;
+; CHECK-PTX-SHARED32-LABEL: st_bulk_shared_cta(
+; CHECK-PTX-SHARED32:       {
+; CHECK-PTX-SHARED32-NEXT:    .reg .b32 %r<2>;
+; CHECK-PTX-SHARED32-NEXT:    .reg .b64 %rd<2>;
+; CHECK-PTX-SHARED32-EMPTY:
+; CHECK-PTX-SHARED32-NEXT:  // %bb.0:
+; CHECK-PTX-SHARED32-NEXT:    ld.param.u32 %r1, [st_bulk_shared_cta_param_0];
+; CHECK-PTX-SHARED32-NEXT:    ld.param.u64 %rd1, [st_bulk_shared_cta_param_1];
+; CHECK-PTX-SHARED32-NEXT:    st.bulk.shared::cta [%r1], %rd1, 0;
+; CHECK-PTX-SHARED32-NEXT:    ret;
+   call void @llvm.nvvm.st.bulk.shared.cta(ptr addrspace(3) %dest_addr, i64 %size, i64 0)
+   ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/constbarrier-rv32.ll b/llvm/test/CodeGen/RISCV/GlobalISel/constbarrier-rv32.ll
index 2fcb911c2654a..309ebf71127c4 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/constbarrier-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/constbarrier-rv32.ll
@@ -38,7 +38,6 @@ define void @constant_fold_barrier_i128(ptr %p) {
 ; RV32-NEXT:    seqz a7, a6
 ; RV32-NEXT:    and a1, a7, a1
 ; RV32-NEXT:    add a7, a4, zero
-; RV32-NEXT:    add a5, a5, zero
 ; RV32-NEXT:    sltu a4, a4, a4
 ; RV32-NEXT:    or a1, a3, a1
 ; RV32-NEXT:    add a7, a7, a1
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/constbarrier-rv64.ll b/llvm/test/CodeGen/RISCV/GlobalISel/constbarrier-rv64.ll
index 2c3e3faddc391..8e4c0376ad0a5 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/constbarrier-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/constbarrier-rv64.ll
@@ -26,7 +26,6 @@ define i128 @constant_fold_barrier_i128(i128 %x) {
 ; RV64-NEXT:    and a0, a0, a2
 ; RV64-NEXT:    add a0, a0, a2
 ; RV64-NEXT:    sltu a2, a0, a2
-; RV64-NEXT:    add a1, a1, zero
 ; RV64-NEXT:    add a1, a1, a2
 ; RV64-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-splatvector-s64-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-splatvector-s64-rv32.mir
index a80ffb7087547..f97e27b66b278 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-splatvector-s64-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-splatvector-s64-rv32.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=riscv32 -mattr=+v -run-pass=legalizer %s -o - | FileCheck --check-prefix=HasF64 %s
-# RUN: llc -mtriple=riscv32 -mattr=+Zve64x -run-pass=legalizer %s -o - | FileCheck --check-prefix=NoF64 %s
+# RUN: llc -mtriple=riscv32 -mattr=+zve64x -run-pass=legalizer %s -o - | FileCheck --check-prefix=NoF64 %s
 
 ---
 name:            splatvector_nxv1i64
diff --git a/llvm/test/CodeGen/RISCV/O3-pipeline.ll b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
index b67fbe1b5d3cd..976d1ee003a1f 100644
--- a/llvm/test/CodeGen/RISCV/O3-pipeline.ll
+++ b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
@@ -182,6 +182,7 @@
 ; CHECK-NEXT:       Post-RA pseudo instruction expansion pass
 ; CHECK-NEXT:       RISC-V post-regalloc pseudo instruction expansion pass
 ; CHECK-NEXT:       Insert KCFI indirect call checks
+; CHECK-NEXT:       RISC-V Load / Store Optimizer
 ; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       Machine Natural Loop Construction
 ; CHECK-NEXT:       PostRA Machine Instruction Scheduler
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index 85e5a71fc7b62..5d4d96a52a0fd 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -402,7 +402,7 @@
 ; RV32XTHEADMEMIDX: .attribute 5, "rv32i2p1_xtheadmemidx1p0"
 ; RV32XTHEADMEMPAIR: .attribute 5, "rv32i2p1_xtheadmempair1p0"
 ; RV32XTHEADSYNC: .attribute 5, "rv32i2p1_xtheadsync1p0"
-; RV32XWCHC: .attribute 5, "rv32i2p1_xwchc2p2"
+; RV32XWCHC: .attribute 5, "rv32i2p1_zca1p0_xwchc2p2"
 ; RV32XQCCMP: .attribute 5, "rv32i2p1_zca1p0_xqccmp0p1"
 ; RV32XQCIA: .attribute 5, "rv32i2p1_xqcia0p4"
 ; RV32XQCIAC: .attribute 5, "rv32i2p1_zca1p0_xqciac0p3"
@@ -411,7 +411,7 @@
 ; RV32XQCICM: .attribute 5, "rv32i2p1_zca1p0_xqcicm0p2"
 ; RV32XQCICS: .attribute 5, "rv32i2p1_xqcics0p2"
 ; RV32XQCICSR: .attribute 5, "rv32i2p1_xqcicsr0p2"
-; RV32XQCIINT: .attribute 5, "rv32i2p1_zca1p0_xqciint0p2"
+; RV32XQCIINT: .attribute 5, "rv32i2p1_zca1p0_xqciint0p4"
 ; RV32XQCILIA: .attribute 5, "rv32i2p1_zca1p0_xqcilia0p2"
 ; RV32XQCILO: .attribute 5, "rv32i2p1_zca1p0_xqcilo0p2"
 ; RV32XQCILSM: .attribute 5, "rv32i2p1_xqcilsm0p2"
diff --git a/llvm/test/CodeGen/RISCV/bfloat-convert.ll b/llvm/test/CodeGen/RISCV/bfloat-convert.ll
index 82359769c7c22..3422ea63e748a 100644
--- a/llvm/test/CodeGen/RISCV/bfloat-convert.ll
+++ b/llvm/test/CodeGen/RISCV/bfloat-convert.ll
@@ -1098,17 +1098,99 @@ define bfloat @fcvt_bf16_wu_load(ptr %p) nounwind {
   ret bfloat %1
 }
 
-; TODO: The following tests error on rv32 with zfbfmin enabled.
-
-; define bfloat @fcvt_bf16_l(i64 %a) nounwind {
-;   %1 = sitofp i64 %a to bfloat
-;   ret bfloat %1
-; }
+define bfloat @fcvt_bf16_l(i64 %a) nounwind {
+; CHECK32ZFBFMIN-LABEL: fcvt_bf16_l:
+; CHECK32ZFBFMIN:       # %bb.0:
+; CHECK32ZFBFMIN-NEXT:    addi sp, sp, -16
+; CHECK32ZFBFMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32ZFBFMIN-NEXT:    call __floatdibf
+; CHECK32ZFBFMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32ZFBFMIN-NEXT:    addi sp, sp, 16
+; CHECK32ZFBFMIN-NEXT:    ret
+;
+; RV32ID-LABEL: fcvt_bf16_l:
+; RV32ID:       # %bb.0:
+; RV32ID-NEXT:    addi sp, sp, -16
+; RV32ID-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32ID-NEXT:    call __floatdisf
+; RV32ID-NEXT:    call __truncsfbf2
+; RV32ID-NEXT:    fmv.x.w a0, fa0
+; RV32ID-NEXT:    lui a1, 1048560
+; RV32ID-NEXT:    or a0, a0, a1
+; RV32ID-NEXT:    fmv.w.x fa0, a0
+; RV32ID-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32ID-NEXT:    addi sp, sp, 16
+; RV32ID-NEXT:    ret
+;
+; CHECK64ZFBFMIN-LABEL: fcvt_bf16_l:
+; CHECK64ZFBFMIN:       # %bb.0:
+; CHECK64ZFBFMIN-NEXT:    fcvt.s.l fa5, a0
+; CHECK64ZFBFMIN-NEXT:    fcvt.bf16.s fa0, fa5
+; CHECK64ZFBFMIN-NEXT:    ret
+;
+; RV64ID-LABEL: fcvt_bf16_l:
+; RV64ID:       # %bb.0:
+; RV64ID-NEXT:    addi sp, sp, -16
+; RV64ID-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64ID-NEXT:    fcvt.s.l fa0, a0
+; RV64ID-NEXT:    call __truncsfbf2
+; RV64ID-NEXT:    fmv.x.w a0, fa0
+; RV64ID-NEXT:    lui a1, 1048560
+; RV64ID-NEXT:    or a0, a0, a1
+; RV64ID-NEXT:    fmv.w.x fa0, a0
+; RV64ID-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64ID-NEXT:    addi sp, sp, 16
+; RV64ID-NEXT:    ret
+  %1 = sitofp i64 %a to bfloat
+  ret bfloat %1
+}
 
-; define bfloat @fcvt_bf16_lu(i64 %a) nounwind {
-;   %1 = uitofp i64 %a to bfloat
-;   ret bfloat %1
-; }
+define bfloat @fcvt_bf16_lu(i64 %a) nounwind {
+; CHECK32ZFBFMIN-LABEL: fcvt_bf16_lu:
+; CHECK32ZFBFMIN:       # %bb.0:
+; CHECK32ZFBFMIN-NEXT:    addi sp, sp, -16
+; CHECK32ZFBFMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32ZFBFMIN-NEXT:    call __floatundibf
+; CHECK32ZFBFMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32ZFBFMIN-NEXT:    addi sp, sp, 16
+; CHECK32ZFBFMIN-NEXT:    ret
+;
+; RV32ID-LABEL: fcvt_bf16_lu:
+; RV32ID:       # %bb.0:
+; RV32ID-NEXT:    addi sp, sp, -16
+; RV32ID-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32ID-NEXT:    call __floatundisf
+; RV32ID-NEXT:    call __truncsfbf2
+; RV32ID-NEXT:    fmv.x.w a0, fa0
+; RV32ID-NEXT:    lui a1, 1048560
+; RV32ID-NEXT:    or a0, a0, a1
+; RV32ID-NEXT:    fmv.w.x fa0, a0
+; RV32ID-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32ID-NEXT:    addi sp, sp, 16
+; RV32ID-NEXT:    ret
+;
+; CHECK64ZFBFMIN-LABEL: fcvt_bf16_lu:
+; CHECK64ZFBFMIN:       # %bb.0:
+; CHECK64ZFBFMIN-NEXT:    fcvt.s.lu fa5, a0
+; CHECK64ZFBFMIN-NEXT:    fcvt.bf16.s fa0, fa5
+; CHECK64ZFBFMIN-NEXT:    ret
+;
+; RV64ID-LABEL: fcvt_bf16_lu:
+; RV64ID:       # %bb.0:
+; RV64ID-NEXT:    addi sp, sp, -16
+; RV64ID-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64ID-NEXT:    fcvt.s.lu fa0, a0
+; RV64ID-NEXT:    call __truncsfbf2
+; RV64ID-NEXT:    fmv.x.w a0, fa0
+; RV64ID-NEXT:    lui a1, 1048560
+; RV64ID-NEXT:    or a0, a0, a1
+; RV64ID-NEXT:    fmv.w.x fa0, a0
+; RV64ID-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64ID-NEXT:    addi sp, sp, 16
+; RV64ID-NEXT:    ret
+  %1 = uitofp i64 %a to bfloat
+  ret bfloat %1
+}
 
 define bfloat @fcvt_bf16_s(float %a) nounwind {
 ; CHECK32ZFBFMIN-LABEL: fcvt_bf16_s:
diff --git a/llvm/test/CodeGen/RISCV/load-store-pair.ll b/llvm/test/CodeGen/RISCV/load-store-pair.ll
new file mode 100644
index 0000000000000..4aad0a8a0d05f
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/load-store-pair.ll
@@ -0,0 +1,326 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32I
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+d -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32D
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64I
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+d -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64D
+; RUN: llc -mtriple=riscv32 -mattr=+Xmipslsp -use-riscv-mips-load-store-pairs=1 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32I_PAIR
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+d,+Xmipslsp -use-riscv-mips-load-store-pairs=1 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32D_PAIR
+; RUN: llc -mtriple=riscv64 -mattr=+Xmipslsp -use-riscv-mips-load-store-pairs=1 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64I_PAIR
+; RUN: llc -mtriple=riscv64 -mcpu=mips-p8700 -mattr=+Xmipslsp -use-riscv-mips-load-store-pairs=1 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64P_8700
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+d,+Xmipslsp -use-riscv-mips-load-store-pairs=1 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64D_PAIR
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+d -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64D_NOPAIR
+
+define void @testi(ptr %a) {
+; RV32I-LABEL: testi:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    .cfi_def_cfa_offset 16
+; RV32I-NEXT:    sw s2, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 0(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    .cfi_offset s2, -4
+; RV32I-NEXT:    .cfi_offset s3, -8
+; RV32I-NEXT:    .cfi_offset s4, -12
+; RV32I-NEXT:    .cfi_offset s5, -16
+; RV32I-NEXT:    lw s3, 0(a0)
+; RV32I-NEXT:    lw s2, 4(a0)
+; RV32I-NEXT:    lw s5, 8(a0)
+; RV32I-NEXT:    lw s4, 12(a0)
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    lw s2, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 0(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    .cfi_restore s2
+; RV32I-NEXT:    .cfi_restore s3
+; RV32I-NEXT:    .cfi_restore s4
+; RV32I-NEXT:    .cfi_restore s5
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    .cfi_def_cfa_offset 0
+; RV32I-NEXT:    ret
+;
+; RV32D-LABEL: testi:
+; RV32D:       # %bb.0: # %entry
+; RV32D-NEXT:    addi sp, sp, -16
+; RV32D-NEXT:    .cfi_def_cfa_offset 16
+; RV32D-NEXT:    sw s2, 12(sp) # 4-byte Folded Spill
+; RV32D-NEXT:    sw s3, 8(sp) # 4-byte Folded Spill
+; RV32D-NEXT:    sw s4, 4(sp) # 4-byte Folded Spill
+; RV32D-NEXT:    sw s5, 0(sp) # 4-byte Folded Spill
+; RV32D-NEXT:    .cfi_offset s2, -4
+; RV32D-NEXT:    .cfi_offset s3, -8
+; RV32D-NEXT:    .cfi_offset s4, -12
+; RV32D-NEXT:    .cfi_offset s5, -16
+; RV32D-NEXT:    lw s3, 0(a0)
+; RV32D-NEXT:    lw s2, 4(a0)
+; RV32D-NEXT:    lw s5, 8(a0)
+; RV32D-NEXT:    lw s4, 12(a0)
+; RV32D-NEXT:    #APP
+; RV32D-NEXT:    #NO_APP
+; RV32D-NEXT:    lw s2, 12(sp) # 4-byte Folded Reload
+; RV32D-NEXT:    lw s3, 8(sp) # 4-byte Folded Reload
+; RV32D-NEXT:    lw s4, 4(sp) # 4-byte Folded Reload
+; RV32D-NEXT:    lw s5, 0(sp) # 4-byte Folded Reload
+; RV32D-NEXT:    .cfi_restore s2
+; RV32D-NEXT:    .cfi_restore s3
+; RV32D-NEXT:    .cfi_restore s4
+; RV32D-NEXT:    .cfi_restore s5
+; RV32D-NEXT:    addi sp, sp, 16
+; RV32D-NEXT:    .cfi_def_cfa_offset 0
+; RV32D-NEXT:    ret
+;
+; RV64I-LABEL: testi:
+; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    .cfi_def_cfa_offset 32
+; RV64I-NEXT:    sd s2, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    .cfi_offset s2, -8
+; RV64I-NEXT:    .cfi_offset s3, -16
+; RV64I-NEXT:    .cfi_offset s4, -24
+; RV64I-NEXT:    .cfi_offset s5, -32
+; RV64I-NEXT:    ld s3, 0(a0)
+; RV64I-NEXT:    ld s2, 8(a0)
+; RV64I-NEXT:    ld s5, 16(a0)
+; RV64I-NEXT:    ld s4, 24(a0)
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ld s2, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    .cfi_restore s2
+; RV64I-NEXT:    .cfi_restore s3
+; RV64I-NEXT:    .cfi_restore s4
+; RV64I-NEXT:    .cfi_restore s5
+; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    .cfi_def_cfa_offset 0
+; RV64I-NEXT:    ret
+;
+; RV64D-LABEL: testi:
+; RV64D:       # %bb.0: # %entry
+; RV64D-NEXT:    addi sp, sp, -32
+; RV64D-NEXT:    .cfi_def_cfa_offset 32
+; RV64D-NEXT:    sd s2, 24(sp) # 8-byte Folded Spill
+; RV64D-NEXT:    sd s3, 16(sp) # 8-byte Folded Spill
+; RV64D-NEXT:    sd s4, 8(sp) # 8-byte Folded Spill
+; RV64D-NEXT:    sd s5, 0(sp) # 8-byte Folded Spill
+; RV64D-NEXT:    .cfi_offset s2, -8
+; RV64D-NEXT:    .cfi_offset s3, -16
+; RV64D-NEXT:    .cfi_offset s4, -24
+; RV64D-NEXT:    .cfi_offset s5, -32
+; RV64D-NEXT:    ld s3, 0(a0)
+; RV64D-NEXT:    ld s2, 8(a0)
+; RV64D-NEXT:    ld s5, 16(a0)
+; RV64D-NEXT:    ld s4, 24(a0)
+; RV64D-NEXT:    #APP
+; RV64D-NEXT:    #NO_APP
+; RV64D-NEXT:    ld s2, 24(sp) # 8-byte Folded Reload
+; RV64D-NEXT:    ld s3, 16(sp) # 8-byte Folded Reload
+; RV64D-NEXT:    ld s4, 8(sp) # 8-byte Folded Reload
+; RV64D-NEXT:    ld s5, 0(sp) # 8-byte Folded Reload
+; RV64D-NEXT:    .cfi_restore s2
+; RV64D-NEXT:    .cfi_restore s3
+; RV64D-NEXT:    .cfi_restore s4
+; RV64D-NEXT:    .cfi_restore s5
+; RV64D-NEXT:    addi sp, sp, 32
+; RV64D-NEXT:    .cfi_def_cfa_offset 0
+; RV64D-NEXT:    ret
+;
+; RV32I_PAIR-LABEL: testi:
+; RV32I_PAIR:       # %bb.0: # %entry
+; RV32I_PAIR-NEXT:    addi sp, sp, -16
+; RV32I_PAIR-NEXT:    .cfi_def_cfa_offset 16
+; RV32I_PAIR-NEXT:    sw s3, 8(sp) # 4-byte Folded Spill
+; RV32I_PAIR-NEXT:    sw s2, 12(sp) # 4-byte Folded Spill
+; RV32I_PAIR-NEXT:    sw s5, 0(sp) # 4-byte Folded Spill
+; RV32I_PAIR-NEXT:    sw s4, 4(sp) # 4-byte Folded Spill
+; RV32I_PAIR-NEXT:    .cfi_offset s2, -4
+; RV32I_PAIR-NEXT:    .cfi_offset s3, -8
+; RV32I_PAIR-NEXT:    .cfi_offset s4, -12
+; RV32I_PAIR-NEXT:    .cfi_offset s5, -16
+; RV32I_PAIR-NEXT:    mips.lwp s3, s2, 0(a0)
+; RV32I_PAIR-NEXT:    mips.lwp s5, s4, 8(a0)
+; RV32I_PAIR-NEXT:    #APP
+; RV32I_PAIR-NEXT:    #NO_APP
+; RV32I_PAIR-NEXT:    lw s3, 8(sp) # 4-byte Folded Reload
+; RV32I_PAIR-NEXT:    lw s2, 12(sp) # 4-byte Folded Reload
+; RV32I_PAIR-NEXT:    lw s5, 0(sp) # 4-byte Folded Reload
+; RV32I_PAIR-NEXT:    lw s4, 4(sp) # 4-byte Folded Reload
+; RV32I_PAIR-NEXT:    .cfi_restore s2
+; RV32I_PAIR-NEXT:    .cfi_restore s3
+; RV32I_PAIR-NEXT:    .cfi_restore s4
+; RV32I_PAIR-NEXT:    .cfi_restore s5
+; RV32I_PAIR-NEXT:    addi sp, sp, 16
+; RV32I_PAIR-NEXT:    .cfi_def_cfa_offset 0
+; RV32I_PAIR-NEXT:    ret
+;
+; RV32D_PAIR-LABEL: testi:
+; RV32D_PAIR:       # %bb.0: # %entry
+; RV32D_PAIR-NEXT:    addi sp, sp, -16
+; RV32D_PAIR-NEXT:    .cfi_def_cfa_offset 16
+; RV32D_PAIR-NEXT:    sw s3, 8(sp) # 4-byte Folded Spill
+; RV32D_PAIR-NEXT:    sw s2, 12(sp) # 4-byte Folded Spill
+; RV32D_PAIR-NEXT:    sw s5, 0(sp) # 4-byte Folded Spill
+; RV32D_PAIR-NEXT:    sw s4, 4(sp) # 4-byte Folded Spill
+; RV32D_PAIR-NEXT:    .cfi_offset s2, -4
+; RV32D_PAIR-NEXT:    .cfi_offset s3, -8
+; RV32D_PAIR-NEXT:    .cfi_offset s4, -12
+; RV32D_PAIR-NEXT:    .cfi_offset s5, -16
+; RV32D_PAIR-NEXT:    mips.lwp s3, s2, 0(a0)
+; RV32D_PAIR-NEXT:    mips.lwp s5, s4, 8(a0)
+; RV32D_PAIR-NEXT:    #APP
+; RV32D_PAIR-NEXT:    #NO_APP
+; RV32D_PAIR-NEXT:    lw s3, 8(sp) # 4-byte Folded Reload
+; RV32D_PAIR-NEXT:    lw s2, 12(sp) # 4-byte Folded Reload
+; RV32D_PAIR-NEXT:    lw s5, 0(sp) # 4-byte Folded Reload
+; RV32D_PAIR-NEXT:    lw s4, 4(sp) # 4-byte Folded Reload
+; RV32D_PAIR-NEXT:    .cfi_restore s2
+; RV32D_PAIR-NEXT:    .cfi_restore s3
+; RV32D_PAIR-NEXT:    .cfi_restore s4
+; RV32D_PAIR-NEXT:    .cfi_restore s5
+; RV32D_PAIR-NEXT:    addi sp, sp, 16
+; RV32D_PAIR-NEXT:    .cfi_def_cfa_offset 0
+; RV32D_PAIR-NEXT:    ret
+;
+; RV64I_PAIR-LABEL: testi:
+; RV64I_PAIR:       # %bb.0: # %entry
+; RV64I_PAIR-NEXT:    addi sp, sp, -32
+; RV64I_PAIR-NEXT:    .cfi_def_cfa_offset 32
+; RV64I_PAIR-NEXT:    sd s3, 16(sp) # 8-byte Folded Spill
+; RV64I_PAIR-NEXT:    sd s2, 24(sp) # 8-byte Folded Spill
+; RV64I_PAIR-NEXT:    sd s5, 0(sp) # 8-byte Folded Spill
+; RV64I_PAIR-NEXT:    sd s4, 8(sp) # 8-byte Folded Spill
+; RV64I_PAIR-NEXT:    .cfi_offset s2, -8
+; RV64I_PAIR-NEXT:    .cfi_offset s3, -16
+; RV64I_PAIR-NEXT:    .cfi_offset s4, -24
+; RV64I_PAIR-NEXT:    .cfi_offset s5, -32
+; RV64I_PAIR-NEXT:    mips.ldp s3, s2, 0(a0)
+; RV64I_PAIR-NEXT:    ld s5, 16(a0)
+; RV64I_PAIR-NEXT:    ld s4, 24(a0)
+; RV64I_PAIR-NEXT:    #APP
+; RV64I_PAIR-NEXT:    #NO_APP
+; RV64I_PAIR-NEXT:    ld s3, 16(sp) # 8-byte Folded Reload
+; RV64I_PAIR-NEXT:    ld s2, 24(sp) # 8-byte Folded Reload
+; RV64I_PAIR-NEXT:    ld s5, 0(sp) # 8-byte Folded Reload
+; RV64I_PAIR-NEXT:    ld s4, 8(sp) # 8-byte Folded Reload
+; RV64I_PAIR-NEXT:    .cfi_restore s2
+; RV64I_PAIR-NEXT:    .cfi_restore s3
+; RV64I_PAIR-NEXT:    .cfi_restore s4
+; RV64I_PAIR-NEXT:    .cfi_restore s5
+; RV64I_PAIR-NEXT:    addi sp, sp, 32
+; RV64I_PAIR-NEXT:    .cfi_def_cfa_offset 0
+; RV64I_PAIR-NEXT:    ret
+;
+; RV64P_8700-LABEL: testi:
+; RV64P_8700:       # %bb.0: # %entry
+; RV64P_8700-NEXT:    addi sp, sp, -32
+; RV64P_8700-NEXT:    .cfi_def_cfa_offset 32
+; RV64P_8700-NEXT:    sd s3, 16(sp) # 8-byte Folded Spill
+; RV64P_8700-NEXT:    sd s2, 24(sp) # 8-byte Folded Spill
+; RV64P_8700-NEXT:    sd s5, 0(sp) # 8-byte Folded Spill
+; RV64P_8700-NEXT:    sd s4, 8(sp) # 8-byte Folded Spill
+; RV64P_8700-NEXT:    .cfi_offset s2, -8
+; RV64P_8700-NEXT:    .cfi_offset s3, -16
+; RV64P_8700-NEXT:    .cfi_offset s4, -24
+; RV64P_8700-NEXT:    .cfi_offset s5, -32
+; RV64P_8700-NEXT:    mips.ldp s3, s2, 0(a0)
+; RV64P_8700-NEXT:    ld s5, 16(a0)
+; RV64P_8700-NEXT:    ld s4, 24(a0)
+; RV64P_8700-NEXT:    #APP
+; RV64P_8700-NEXT:    #NO_APP
+; RV64P_8700-NEXT:    ld s3, 16(sp) # 8-byte Folded Reload
+; RV64P_8700-NEXT:    ld s2, 24(sp) # 8-byte Folded Reload
+; RV64P_8700-NEXT:    ld s5, 0(sp) # 8-byte Folded Reload
+; RV64P_8700-NEXT:    ld s4, 8(sp) # 8-byte Folded Reload
+; RV64P_8700-NEXT:    .cfi_restore s2
+; RV64P_8700-NEXT:    .cfi_restore s3
+; RV64P_8700-NEXT:    .cfi_restore s4
+; RV64P_8700-NEXT:    .cfi_restore s5
+; RV64P_8700-NEXT:    addi sp, sp, 32
+; RV64P_8700-NEXT:    .cfi_def_cfa_offset 0
+; RV64P_8700-NEXT:    ret
+;
+; RV64D_PAIR-LABEL: testi:
+; RV64D_PAIR:       # %bb.0: # %entry
+; RV64D_PAIR-NEXT:    addi sp, sp, -32
+; RV64D_PAIR-NEXT:    .cfi_def_cfa_offset 32
+; RV64D_PAIR-NEXT:    sd s3, 16(sp) # 8-byte Folded Spill
+; RV64D_PAIR-NEXT:    sd s2, 24(sp) # 8-byte Folded Spill
+; RV64D_PAIR-NEXT:    sd s5, 0(sp) # 8-byte Folded Spill
+; RV64D_PAIR-NEXT:    sd s4, 8(sp) # 8-byte Folded Spill
+; RV64D_PAIR-NEXT:    .cfi_offset s2, -8
+; RV64D_PAIR-NEXT:    .cfi_offset s3, -16
+; RV64D_PAIR-NEXT:    .cfi_offset s4, -24
+; RV64D_PAIR-NEXT:    .cfi_offset s5, -32
+; RV64D_PAIR-NEXT:    mips.ldp s3, s2, 0(a0)
+; RV64D_PAIR-NEXT:    ld s5, 16(a0)
+; RV64D_PAIR-NEXT:    ld s4, 24(a0)
+; RV64D_PAIR-NEXT:    #APP
+; RV64D_PAIR-NEXT:    #NO_APP
+; RV64D_PAIR-NEXT:    ld s3, 16(sp) # 8-byte Folded Reload
+; RV64D_PAIR-NEXT:    ld s2, 24(sp) # 8-byte Folded Reload
+; RV64D_PAIR-NEXT:    ld s5, 0(sp) # 8-byte Folded Reload
+; RV64D_PAIR-NEXT:    ld s4, 8(sp) # 8-byte Folded Reload
+; RV64D_PAIR-NEXT:    .cfi_restore s2
+; RV64D_PAIR-NEXT:    .cfi_restore s3
+; RV64D_PAIR-NEXT:    .cfi_restore s4
+; RV64D_PAIR-NEXT:    .cfi_restore s5
+; RV64D_PAIR-NEXT:    addi sp, sp, 32
+; RV64D_PAIR-NEXT:    .cfi_def_cfa_offset 0
+; RV64D_PAIR-NEXT:    ret
+;
+; RV64D_NOPAIR-LABEL: testi:
+; RV64D_NOPAIR:       # %bb.0: # %entry
+; RV64D_NOPAIR-NEXT:    addi sp, sp, -32
+; RV64D_NOPAIR-NEXT:    .cfi_def_cfa_offset 32
+; RV64D_NOPAIR-NEXT:    sd s2, 24(sp) # 8-byte Folded Spill
+; RV64D_NOPAIR-NEXT:    sd s3, 16(sp) # 8-byte Folded Spill
+; RV64D_NOPAIR-NEXT:    sd s4, 8(sp) # 8-byte Folded Spill
+; RV64D_NOPAIR-NEXT:    sd s5, 0(sp) # 8-byte Folded Spill
+; RV64D_NOPAIR-NEXT:    .cfi_offset s2, -8
+; RV64D_NOPAIR-NEXT:    .cfi_offset s3, -16
+; RV64D_NOPAIR-NEXT:    .cfi_offset s4, -24
+; RV64D_NOPAIR-NEXT:    .cfi_offset s5, -32
+; RV64D_NOPAIR-NEXT:    ld s3, 0(a0)
+; RV64D_NOPAIR-NEXT:    ld s2, 8(a0)
+; RV64D_NOPAIR-NEXT:    ld s5, 16(a0)
+; RV64D_NOPAIR-NEXT:    ld s4, 24(a0)
+; RV64D_NOPAIR-NEXT:    #APP
+; RV64D_NOPAIR-NEXT:    #NO_APP
+; RV64D_NOPAIR-NEXT:    ld s2, 24(sp) # 8-byte Folded Reload
+; RV64D_NOPAIR-NEXT:    ld s3, 16(sp) # 8-byte Folded Reload
+; RV64D_NOPAIR-NEXT:    ld s4, 8(sp) # 8-byte Folded Reload
+; RV64D_NOPAIR-NEXT:    ld s5, 0(sp) # 8-byte Folded Reload
+; RV64D_NOPAIR-NEXT:    .cfi_restore s2
+; RV64D_NOPAIR-NEXT:    .cfi_restore s3
+; RV64D_NOPAIR-NEXT:    .cfi_restore s4
+; RV64D_NOPAIR-NEXT:    .cfi_restore s5
+; RV64D_NOPAIR-NEXT:    addi sp, sp, 32
+; RV64D_NOPAIR-NEXT:    .cfi_def_cfa_offset 0
+; RV64D_NOPAIR-NEXT:    ret
+entry:
+  %arrayidx = getelementptr inbounds ptr, ptr %a, i64 1
+  %0 = load ptr, ptr %arrayidx, align 16
+  %1 = load ptr, ptr %a, align 16
+  %arrayidx2 = getelementptr inbounds ptr, ptr %a, i64 3
+  %2 = load ptr, ptr %arrayidx2, align 16
+  %arrayidx3 = getelementptr inbounds ptr, ptr %a, i64 2
+  %3 = load ptr, ptr %arrayidx3, align 8
+  tail call void asm sideeffect "", "{x18},{x19},{x20},{x21}"(ptr %0, ptr %1, ptr %2, ptr %3)
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/machine-copyprop-noop-removal.mir b/llvm/test/CodeGen/RISCV/machine-copyprop-noop-removal.mir
new file mode 100644
index 0000000000000..d739537b50d05
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/machine-copyprop-noop-removal.mir
@@ -0,0 +1,74 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -o - %s -mtriple=riscv64 -run-pass=machine-cp -mcp-use-is-copy-instr | FileCheck %s
+
+## This test was added to capture a case where MachineCopyPropagation risks
+## leaving a no-op register move (add, x0, reg).
+
+---
+name: ham
+body: |
+  ; CHECK-LABEL: name: ham
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; CHECK-NEXT:   liveins: $x10
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $x11 = ANDI killed renamable $x10, 1
+  ; CHECK-NEXT:   renamable $x10 = ADDI $x0, 1
+  ; CHECK-NEXT:   BEQ killed renamable $x11, $x0, %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $x10
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   $x11 = ADDI $x0, 0
+  ; CHECK-NEXT:   BEQ renamable $x10, $x0, %bb.4
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   $x10 = ADDI $x0, 0
+  ; CHECK-NEXT:   PseudoRET implicit $x10
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $x10
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $x11 = ADDI $x0, 1
+  ; CHECK-NEXT:   renamable $x10 = ADD killed renamable $x11, killed renamable $x10
+  ; CHECK-NEXT:   BNE renamable $x10, $x0, %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.4:
+  ; CHECK-NEXT:   liveins: $x10
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   PseudoRET implicit $x10
+  bb.0:
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    liveins: $x10
+
+    renamable $x11 = ANDI killed renamable $x10, 1
+    renamable $x10 = ADDI $x0, 1
+    BEQ killed renamable $x11, $x0, %bb.2
+
+  bb.1:
+    successors: %bb.4(0x40000000), %bb.5(0x40000000)
+    liveins: $x10
+
+    $x11 = ADDI $x0, 0
+    renamable $x10 = ADD killed renamable $x11, killed renamable $x10
+    BEQ renamable $x10, $x0, %bb.4
+
+  bb.5:
+    $x10 = ADDI $x0, 0
+    PseudoRET implicit $x10
+
+  bb.2:
+    successors: %bb.4(0x40000000), %bb.5(0x40000000)
+    liveins: $x10
+
+    renamable $x11 = ADDI $x0, 1
+    renamable $x10 = ADD killed renamable $x11, killed renamable $x10
+    BNE renamable $x10, $x0, %bb.5
+
+  bb.4:
+    liveins: $x10
+
+    PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/qci-interrupt-attr.ll b/llvm/test/CodeGen/RISCV/qci-interrupt-attr.ll
index a6c90ae9e69c3..76d61eee07eef 100644
--- a/llvm/test/CodeGen/RISCV/qci-interrupt-attr.ll
+++ b/llvm/test/CodeGen/RISCV/qci-interrupt-attr.ll
@@ -65,7 +65,6 @@ define void @test_nest_empty() "interrupt"="qci-nest" {
 ; QCI-FP-NEXT:    .cfi_offset t4, -72
 ; QCI-FP-NEXT:    .cfi_offset t5, -76
 ; QCI-FP-NEXT:    .cfi_offset t6, -80
-; QCI-FP-NEXT:    addi s0, sp, 96
 ; QCI-FP-NEXT:    .cfi_def_cfa s0, 0
 ; QCI-FP-NEXT:    .cfi_def_cfa sp, 96
 ; QCI-FP-NEXT:    qc.c.mileaveret
@@ -140,7 +139,6 @@ define void @test_nonest_empty() "interrupt"="qci-nonest" {
 ; QCI-FP-NEXT:    .cfi_offset t4, -72
 ; QCI-FP-NEXT:    .cfi_offset t5, -76
 ; QCI-FP-NEXT:    .cfi_offset t6, -80
-; QCI-FP-NEXT:    addi s0, sp, 96
 ; QCI-FP-NEXT:    .cfi_def_cfa s0, 0
 ; QCI-FP-NEXT:    .cfi_def_cfa sp, 96
 ; QCI-FP-NEXT:    qc.c.mileaveret
@@ -223,7 +221,6 @@ define void @test_nest_asm() "interrupt"="qci-nest" {
 ; QCI-FP-NEXT:    .cfi_offset t4, -72
 ; QCI-FP-NEXT:    .cfi_offset t5, -76
 ; QCI-FP-NEXT:    .cfi_offset t6, -80
-; QCI-FP-NEXT:    addi s0, sp, 96
 ; QCI-FP-NEXT:    .cfi_def_cfa s0, 0
 ; QCI-FP-NEXT:    li a0, 1
 ; QCI-FP-NEXT:    #APP
@@ -315,7 +312,6 @@ define void @test_nonest_asm() "interrupt"="qci-nonest" {
 ; QCI-FP-NEXT:    .cfi_offset t4, -72
 ; QCI-FP-NEXT:    .cfi_offset t5, -76
 ; QCI-FP-NEXT:    .cfi_offset t6, -80
-; QCI-FP-NEXT:    addi s0, sp, 96
 ; QCI-FP-NEXT:    .cfi_def_cfa s0, 0
 ; QCI-FP-NEXT:    li a0, 1
 ; QCI-FP-NEXT:    #APP
@@ -423,7 +419,6 @@ define void @test_nest_call() "interrupt"="qci-nest" {
 ; QCI-FP-NEXT:    .cfi_offset t6, -80
 ; QCI-FP-NEXT:    addi sp, sp, -16
 ; QCI-FP-NEXT:    .cfi_def_cfa_offset 112
-; QCI-FP-NEXT:    addi s0, sp, 112
 ; QCI-FP-NEXT:    .cfi_def_cfa s0, 0
 ; QCI-FP-NEXT:    li a0, 4
 ; QCI-FP-NEXT:    li a2, 1
@@ -548,7 +543,6 @@ define void @test_nonest_call() "interrupt"="qci-nonest" {
 ; QCI-FP-NEXT:    .cfi_offset t6, -80
 ; QCI-FP-NEXT:    addi sp, sp, -16
 ; QCI-FP-NEXT:    .cfi_def_cfa_offset 112
-; QCI-FP-NEXT:    addi s0, sp, 112
 ; QCI-FP-NEXT:    .cfi_def_cfa s0, 0
 ; QCI-FP-NEXT:    li a0, 4
 ; QCI-FP-NEXT:    li a2, 1
@@ -807,7 +801,6 @@ define void @test_spill_nest() "interrupt"="qci-nest" {
 ; QCI-FP-NEXT:    .cfi_offset s9, -132
 ; QCI-FP-NEXT:    .cfi_offset s10, -136
 ; QCI-FP-NEXT:    .cfi_offset s11, -140
-; QCI-FP-NEXT:    addi s0, sp, 176
 ; QCI-FP-NEXT:    .cfi_def_cfa s0, 0
 ; QCI-FP-NEXT:    lui t1, %hi(var)
 ; QCI-FP-NEXT:    lw a0, %lo(var)(t1)
@@ -1264,7 +1257,6 @@ define void @test_spill_nonest() "interrupt"="qci-nonest" {
 ; QCI-FP-NEXT:    .cfi_offset s9, -132
 ; QCI-FP-NEXT:    .cfi_offset s10, -136
 ; QCI-FP-NEXT:    .cfi_offset s11, -140
-; QCI-FP-NEXT:    addi s0, sp, 176
 ; QCI-FP-NEXT:    .cfi_def_cfa s0, 0
 ; QCI-FP-NEXT:    lui t1, %hi(var)
 ; QCI-FP-NEXT:    lw a0, %lo(var)(t1)
@@ -1765,7 +1757,6 @@ define void @test_spill_call_nest() "interrupt"="qci-nest" {
 ; QCI-FP-NEXT:    .cfi_offset s9, -132
 ; QCI-FP-NEXT:    .cfi_offset s10, -136
 ; QCI-FP-NEXT:    .cfi_offset s11, -140
-; QCI-FP-NEXT:    addi s0, sp, 240
 ; QCI-FP-NEXT:    .cfi_def_cfa s0, 0
 ; QCI-FP-NEXT:    lui s6, %hi(var)
 ; QCI-FP-NEXT:    lw a0, %lo(var)(s6)
@@ -2356,7 +2347,6 @@ define void @test_spill_call_nonest() "interrupt"="qci-nonest" {
 ; QCI-FP-NEXT:    .cfi_offset s9, -132
 ; QCI-FP-NEXT:    .cfi_offset s10, -136
 ; QCI-FP-NEXT:    .cfi_offset s11, -140
-; QCI-FP-NEXT:    addi s0, sp, 240
 ; QCI-FP-NEXT:    .cfi_def_cfa s0, 0
 ; QCI-FP-NEXT:    lui s6, %hi(var)
 ; QCI-FP-NEXT:    lw a0, %lo(var)(s6)
@@ -2768,7 +2758,6 @@ define void @test_nest_explicit_s11() "interrupt"="qci-nest" {
 ; QCI-FP-NEXT:    .cfi_def_cfa_offset 112
 ; QCI-FP-NEXT:    sw s11, 12(sp) # 4-byte Folded Spill
 ; QCI-FP-NEXT:    .cfi_offset s11, -100
-; QCI-FP-NEXT:    addi s0, sp, 112
 ; QCI-FP-NEXT:    .cfi_def_cfa s0, 0
 ; QCI-FP-NEXT:    #APP
 ; QCI-FP-NEXT:    li s4, 0
@@ -2877,7 +2866,6 @@ define void @test_nonest_explicit_s11() "interrupt"="qci-nonest" {
 ; QCI-FP-NEXT:    .cfi_def_cfa_offset 112
 ; QCI-FP-NEXT:    sw s11, 12(sp) # 4-byte Folded Spill
 ; QCI-FP-NEXT:    .cfi_offset s11, -100
-; QCI-FP-NEXT:    addi s0, sp, 112
 ; QCI-FP-NEXT:    .cfi_def_cfa s0, 0
 ; QCI-FP-NEXT:    #APP
 ; QCI-FP-NEXT:    li s11, 0
diff --git a/llvm/test/CodeGen/RISCV/rv32i-rv64i-float-double.ll b/llvm/test/CodeGen/RISCV/rv32i-rv64i-float-double.ll
index cd7bce868eae0..b0726a5b6fd6b 100644
--- a/llvm/test/CodeGen/RISCV/rv32i-rv64i-float-double.ll
+++ b/llvm/test/CodeGen/RISCV/rv32i-rv64i-float-double.ll
@@ -9,8 +9,6 @@
 ; legalisation or DAG combines aren't incorrectly triggered when the F
 ; extension isn't enabled.
 
-; TODO: f32 parameters on RV64 with a soft-float ABI are anyext.
-
 define float @float_test(float %a, float %b) nounwind {
 ; RV32IF-LABEL: float_test:
 ; RV32IF:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rvv/ctlz-vp.ll b/llvm/test/CodeGen/RISCV/rvv/ctlz-vp.ll
index 6f515996677ee..ceca813782461 100644
--- a/llvm/test/CodeGen/RISCV/rvv/ctlz-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/ctlz-vp.ll
@@ -2585,8 +2585,7 @@ define <vscale x 1 x i9> @vp_ctlz_nxv1i9(<vscale x 1 x i9> %va, <vscale x 1 x i1
 ; CHECK-NEXT:    vrsub.vx v8, v8, a0, v0.t
 ; CHECK-NEXT:    li a0, 16
 ; CHECK-NEXT:    vminu.vx v8, v8, a0, v0.t
-; CHECK-NEXT:    li a0, 7
-; CHECK-NEXT:    vsub.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    vadd.vi v8, v8, -7, v0.t
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlz_nxv1i9:
@@ -2595,8 +2594,7 @@ define <vscale x 1 x i9> @vp_ctlz_nxv1i9(<vscale x 1 x i9> %va, <vscale x 1 x i1
 ; CHECK-ZVBB-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-ZVBB-NEXT:    vand.vx v8, v8, a1, v0.t
 ; CHECK-ZVBB-NEXT:    vclz.v v8, v8, v0.t
-; CHECK-ZVBB-NEXT:    li a0, 7
-; CHECK-ZVBB-NEXT:    vsub.vx v8, v8, a0, v0.t
+; CHECK-ZVBB-NEXT:    vadd.vi v8, v8, -7, v0.t
 ; CHECK-ZVBB-NEXT:    ret
   %v = call <vscale x 1 x i9> @llvm.vp.ctlz.nxv1i9(<vscale x 1 x i9> %va, i1 false, <vscale x 1 x i1> %m, i32 %evl)
   ret <vscale x 1 x i9> %v
@@ -2744,13 +2742,12 @@ define <vscale x 1 x i9> @vp_ctlo_zero_nxv1i9_unpredicated_ctlz_with_vp_xor(<vsc
 ; CHECK-NEXT:    li a0, 142
 ; CHECK-NEXT:    vsetvli a2, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vand.vx v8, v8, a1
-; CHECK-NEXT:    li a1, 16
 ; CHECK-NEXT:    vfwcvt.f.xu.v v9, v8
 ; CHECK-NEXT:    vnsrl.wi v8, v9, 23
 ; CHECK-NEXT:    vrsub.vx v8, v8, a0
-; CHECK-NEXT:    vminu.vx v8, v8, a1
-; CHECK-NEXT:    li a0, 7
-; CHECK-NEXT:    vsub.vx v8, v8, a0
+; CHECK-NEXT:    li a0, 16
+; CHECK-NEXT:    vminu.vx v8, v8, a0
+; CHECK-NEXT:    vadd.vi v8, v8, -7
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_ctlo_zero_nxv1i9_unpredicated_ctlz_with_vp_xor:
@@ -2761,8 +2758,7 @@ define <vscale x 1 x i9> @vp_ctlo_zero_nxv1i9_unpredicated_ctlz_with_vp_xor(<vsc
 ; CHECK-ZVBB-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; CHECK-ZVBB-NEXT:    vand.vx v8, v8, a1
 ; CHECK-ZVBB-NEXT:    vclz.v v8, v8
-; CHECK-ZVBB-NEXT:    li a0, 7
-; CHECK-ZVBB-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVBB-NEXT:    vadd.vi v8, v8, -7
 ; CHECK-ZVBB-NEXT:    ret
   %va.not = call <vscale x 1 x i9> @llvm.vp.xor.nxv1i9(<vscale x 1 x i9> %va, <vscale x 1 x i9> splat (i9 -1), <vscale x 1 x i1> %m, i32 %evl)
   %v = call <vscale x 1 x i9> @llvm.ctlz(<vscale x 1 x i9> %va.not, i1 false)
diff --git a/llvm/test/CodeGen/RISCV/rvv/cttz-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/cttz-sdnode.ll
index 5761ae0926eae..bd7a20f9ef590 100644
--- a/llvm/test/CodeGen/RISCV/rvv/cttz-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/cttz-sdnode.ll
@@ -11,12 +11,11 @@
 define <vscale x 1 x i8> @cttz_nxv1i8(<vscale x 1 x i8> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_nxv1i8:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v9, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v9, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    li a0, 85
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v9, v8
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v9
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v9, v9, a0
 ; CHECK-ZVE64X-NEXT:    li a0, 51
@@ -77,12 +76,11 @@ declare <vscale x 1 x i8> @llvm.cttz.nxv1i8(<vscale x 1 x i8>, i1)
 define <vscale x 2 x i8> @cttz_nxv2i8(<vscale x 2 x i8> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_nxv2i8:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v9, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e8, mf4, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v9, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    li a0, 85
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v9, v8
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v9
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v9, v9, a0
 ; CHECK-ZVE64X-NEXT:    li a0, 51
@@ -143,12 +141,11 @@ declare <vscale x 2 x i8> @llvm.cttz.nxv2i8(<vscale x 2 x i8>, i1)
 define <vscale x 4 x i8> @cttz_nxv4i8(<vscale x 4 x i8> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_nxv4i8:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v9, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v9, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    li a0, 85
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v9, v8
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v9
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v9, v9, a0
 ; CHECK-ZVE64X-NEXT:    li a0, 51
@@ -209,12 +206,11 @@ declare <vscale x 4 x i8> @llvm.cttz.nxv4i8(<vscale x 4 x i8>, i1)
 define <vscale x 8 x i8> @cttz_nxv8i8(<vscale x 8 x i8> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_nxv8i8:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v9, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v9, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    li a0, 85
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v9, v8
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v9
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v9, v9, a0
 ; CHECK-ZVE64X-NEXT:    li a0, 51
@@ -275,12 +271,11 @@ declare <vscale x 8 x i8> @llvm.cttz.nxv8i8(<vscale x 8 x i8>, i1)
 define <vscale x 16 x i8> @cttz_nxv16i8(<vscale x 16 x i8> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_nxv16i8:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e8, m2, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v10, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v10, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    li a0, 85
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v10, v8
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v10
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v10, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v10, v10, a0
 ; CHECK-ZVE64X-NEXT:    li a0, 51
@@ -341,12 +336,11 @@ declare <vscale x 16 x i8> @llvm.cttz.nxv16i8(<vscale x 16 x i8>, i1)
 define <vscale x 32 x i8> @cttz_nxv32i8(<vscale x 32 x i8> %va) {
 ; CHECK-LABEL: cttz_nxv32i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
-; CHECK-NEXT:    vsetvli a1, zero, e8, m4, ta, ma
-; CHECK-NEXT:    vnot.v v12, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
+; CHECK-NEXT:    vadd.vi v12, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v12, v8
+; CHECK-NEXT:    vand.vv v8, v8, v12
 ; CHECK-NEXT:    vsrl.vi v12, v8, 1
 ; CHECK-NEXT:    vand.vx v12, v12, a0
 ; CHECK-NEXT:    li a0, 51
@@ -373,12 +367,11 @@ declare <vscale x 32 x i8> @llvm.cttz.nxv32i8(<vscale x 32 x i8>, i1)
 define <vscale x 64 x i8> @cttz_nxv64i8(<vscale x 64 x i8> %va) {
 ; CHECK-LABEL: cttz_nxv64i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
-; CHECK-NEXT:    vsetvli a1, zero, e8, m8, ta, ma
-; CHECK-NEXT:    vnot.v v16, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vadd.vi v16, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v16, v8
+; CHECK-NEXT:    vand.vv v8, v8, v16
 ; CHECK-NEXT:    vsrl.vi v16, v8, 1
 ; CHECK-NEXT:    vand.vx v16, v16, a0
 ; CHECK-NEXT:    li a0, 51
@@ -405,13 +398,12 @@ declare <vscale x 64 x i8> @llvm.cttz.nxv64i8(<vscale x 64 x i8>, i1)
 define <vscale x 1 x i16> @cttz_nxv1i16(<vscale x 1 x i16> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_nxv1i16:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v9, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v9, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 5
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v9
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v9, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v9, v9, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 3
@@ -472,13 +464,12 @@ declare <vscale x 1 x i16> @llvm.cttz.nxv1i16(<vscale x 1 x i16>, i1)
 define <vscale x 2 x i16> @cttz_nxv2i16(<vscale x 2 x i16> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_nxv2i16:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v9, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v9, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 5
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v9
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v9, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v9, v9, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 3
@@ -539,13 +530,12 @@ declare <vscale x 2 x i16> @llvm.cttz.nxv2i16(<vscale x 2 x i16>, i1)
 define <vscale x 4 x i16> @cttz_nxv4i16(<vscale x 4 x i16> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_nxv4i16:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v9, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v9, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 5
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v9
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v9, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v9, v9, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 3
@@ -606,13 +596,12 @@ declare <vscale x 4 x i16> @llvm.cttz.nxv4i16(<vscale x 4 x i16>, i1)
 define <vscale x 8 x i16> @cttz_nxv8i16(<vscale x 8 x i16> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_nxv8i16:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v10, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v10, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 5
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v10
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v10, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v10, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v10, v10, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 3
@@ -673,13 +662,12 @@ declare <vscale x 8 x i16> @llvm.cttz.nxv8i16(<vscale x 8 x i16>, i1)
 define <vscale x 16 x i16> @cttz_nxv16i16(<vscale x 16 x i16> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_nxv16i16:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v12, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v12, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 5
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v12
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v12, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v12, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v12, v12, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 3
@@ -740,13 +728,12 @@ declare <vscale x 16 x i16> @llvm.cttz.nxv16i16(<vscale x 16 x i16>, i1)
 define <vscale x 32 x i16> @cttz_nxv32i16(<vscale x 32 x i16> %va) {
 ; CHECK-LABEL: cttz_nxv32i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
-; CHECK-NEXT:    vsetvli a1, zero, e16, m8, ta, ma
-; CHECK-NEXT:    vnot.v v16, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m8, ta, ma
+; CHECK-NEXT:    vadd.vi v16, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    vand.vv v8, v8, v16
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v16, v8
 ; CHECK-NEXT:    vsrl.vi v16, v8, 1
 ; CHECK-NEXT:    vand.vx v16, v16, a0
 ; CHECK-NEXT:    lui a0, 3
@@ -779,13 +766,12 @@ declare <vscale x 32 x i16> @llvm.cttz.nxv32i16(<vscale x 32 x i16>, i1)
 define <vscale x 1 x i32> @cttz_nxv1i32(<vscale x 1 x i32> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_nxv1i32:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v9, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v9, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 349525
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v9
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v9, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v9, v9, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 209715
@@ -850,13 +836,12 @@ declare <vscale x 1 x i32> @llvm.cttz.nxv1i32(<vscale x 1 x i32>, i1)
 define <vscale x 2 x i32> @cttz_nxv2i32(<vscale x 2 x i32> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_nxv2i32:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v9, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v9, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 349525
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v9
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v9, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v9, v9, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 209715
@@ -921,13 +906,12 @@ declare <vscale x 2 x i32> @llvm.cttz.nxv2i32(<vscale x 2 x i32>, i1)
 define <vscale x 4 x i32> @cttz_nxv4i32(<vscale x 4 x i32> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_nxv4i32:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v10, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v10, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 349525
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v10
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v10, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v10, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v10, v10, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 209715
@@ -992,13 +976,12 @@ declare <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32>, i1)
 define <vscale x 8 x i32> @cttz_nxv8i32(<vscale x 8 x i32> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_nxv8i32:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v12, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v12, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 349525
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v12
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v12, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v12, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v12, v12, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 209715
@@ -1063,13 +1046,12 @@ declare <vscale x 8 x i32> @llvm.cttz.nxv8i32(<vscale x 8 x i32>, i1)
 define <vscale x 16 x i32> @cttz_nxv16i32(<vscale x 16 x i32> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_nxv16i32:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v16, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v16, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 349525
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v16
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v16, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v16, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v16, v16, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 209715
@@ -1135,18 +1117,17 @@ declare <vscale x 16 x i32> @llvm.cttz.nxv16i32(<vscale x 16 x i32>, i1)
 define <vscale x 1 x i64> @cttz_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV32I-LABEL: cttz_nxv1i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    li a0, 1
-; RV32I-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; RV32I-NEXT:    vnot.v v9, v8
-; RV32I-NEXT:    vsub.vx v8, v8, a0
+; RV32I-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; RV32I-NEXT:    vadd.vi v9, v8, -1
+; RV32I-NEXT:    vnot.v v8, v8
 ; RV32I-NEXT:    lui a0, 349525
 ; RV32I-NEXT:    addi a0, a0, 1365
 ; RV32I-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
 ; RV32I-NEXT:    vmv.v.x v10, a0
 ; RV32I-NEXT:    lui a0, 209715
-; RV32I-NEXT:    addi a0, a0, 819
 ; RV32I-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; RV32I-NEXT:    vand.vv v8, v9, v8
+; RV32I-NEXT:    vand.vv v8, v8, v9
+; RV32I-NEXT:    addi a0, a0, 819
 ; RV32I-NEXT:    vsrl.vi v9, v8, 1
 ; RV32I-NEXT:    vand.vv v9, v9, v10
 ; RV32I-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
@@ -1177,17 +1158,14 @@ define <vscale x 1 x i64> @cttz_nxv1i64(<vscale x 1 x i64> %va) {
 ;
 ; RV64I-LABEL: cttz_nxv1i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    li a0, 1
-; RV64I-NEXT:    lui a1, 349525
-; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    lui a3, 61681
-; RV64I-NEXT:    lui a4, 4112
-; RV64I-NEXT:    vsetvli a5, zero, e64, m1, ta, ma
-; RV64I-NEXT:    vsub.vx v9, v8, a0
-; RV64I-NEXT:    addiw a0, a1, 1365
-; RV64I-NEXT:    addiw a1, a2, 819
-; RV64I-NEXT:    addiw a2, a3, -241
-; RV64I-NEXT:    addiw a3, a4, 257
+; RV64I-NEXT:    lui a0, 349525
+; RV64I-NEXT:    lui a1, 209715
+; RV64I-NEXT:    lui a2, 61681
+; RV64I-NEXT:    lui a3, 4112
+; RV64I-NEXT:    addiw a0, a0, 1365
+; RV64I-NEXT:    addiw a1, a1, 819
+; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addiw a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -1196,6 +1174,8 @@ define <vscale x 1 x i64> @cttz_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV64I-NEXT:    add a2, a2, a4
 ; RV64I-NEXT:    slli a4, a3, 32
 ; RV64I-NEXT:    add a3, a3, a4
+; RV64I-NEXT:    vsetvli a4, zero, e64, m1, ta, ma
+; RV64I-NEXT:    vadd.vi v9, v8, -1
 ; RV64I-NEXT:    vnot.v v8, v8
 ; RV64I-NEXT:    vand.vv v8, v8, v9
 ; RV64I-NEXT:    vsrl.vi v9, v8, 1
@@ -1261,18 +1241,17 @@ declare <vscale x 1 x i64> @llvm.cttz.nxv1i64(<vscale x 1 x i64>, i1)
 define <vscale x 2 x i64> @cttz_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV32I-LABEL: cttz_nxv2i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    li a0, 1
-; RV32I-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
-; RV32I-NEXT:    vnot.v v10, v8
-; RV32I-NEXT:    vsub.vx v8, v8, a0
+; RV32I-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
+; RV32I-NEXT:    vadd.vi v10, v8, -1
+; RV32I-NEXT:    vnot.v v8, v8
 ; RV32I-NEXT:    lui a0, 349525
 ; RV32I-NEXT:    addi a0, a0, 1365
 ; RV32I-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; RV32I-NEXT:    vmv.v.x v12, a0
 ; RV32I-NEXT:    lui a0, 209715
-; RV32I-NEXT:    addi a0, a0, 819
 ; RV32I-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
-; RV32I-NEXT:    vand.vv v8, v10, v8
+; RV32I-NEXT:    vand.vv v8, v8, v10
+; RV32I-NEXT:    addi a0, a0, 819
 ; RV32I-NEXT:    vsrl.vi v10, v8, 1
 ; RV32I-NEXT:    vand.vv v10, v10, v12
 ; RV32I-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
@@ -1303,17 +1282,14 @@ define <vscale x 2 x i64> @cttz_nxv2i64(<vscale x 2 x i64> %va) {
 ;
 ; RV64I-LABEL: cttz_nxv2i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    li a0, 1
-; RV64I-NEXT:    lui a1, 349525
-; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    lui a3, 61681
-; RV64I-NEXT:    lui a4, 4112
-; RV64I-NEXT:    vsetvli a5, zero, e64, m2, ta, ma
-; RV64I-NEXT:    vsub.vx v10, v8, a0
-; RV64I-NEXT:    addiw a0, a1, 1365
-; RV64I-NEXT:    addiw a1, a2, 819
-; RV64I-NEXT:    addiw a2, a3, -241
-; RV64I-NEXT:    addiw a3, a4, 257
+; RV64I-NEXT:    lui a0, 349525
+; RV64I-NEXT:    lui a1, 209715
+; RV64I-NEXT:    lui a2, 61681
+; RV64I-NEXT:    lui a3, 4112
+; RV64I-NEXT:    addiw a0, a0, 1365
+; RV64I-NEXT:    addiw a1, a1, 819
+; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addiw a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -1322,6 +1298,8 @@ define <vscale x 2 x i64> @cttz_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV64I-NEXT:    add a2, a2, a4
 ; RV64I-NEXT:    slli a4, a3, 32
 ; RV64I-NEXT:    add a3, a3, a4
+; RV64I-NEXT:    vsetvli a4, zero, e64, m2, ta, ma
+; RV64I-NEXT:    vadd.vi v10, v8, -1
 ; RV64I-NEXT:    vnot.v v8, v8
 ; RV64I-NEXT:    vand.vv v8, v8, v10
 ; RV64I-NEXT:    vsrl.vi v10, v8, 1
@@ -1387,18 +1365,17 @@ declare <vscale x 2 x i64> @llvm.cttz.nxv2i64(<vscale x 2 x i64>, i1)
 define <vscale x 4 x i64> @cttz_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV32I-LABEL: cttz_nxv4i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    li a0, 1
-; RV32I-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
-; RV32I-NEXT:    vnot.v v12, v8
-; RV32I-NEXT:    vsub.vx v8, v8, a0
+; RV32I-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; RV32I-NEXT:    vadd.vi v12, v8, -1
+; RV32I-NEXT:    vnot.v v8, v8
 ; RV32I-NEXT:    lui a0, 349525
 ; RV32I-NEXT:    addi a0, a0, 1365
 ; RV32I-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
 ; RV32I-NEXT:    vmv.v.x v16, a0
 ; RV32I-NEXT:    lui a0, 209715
-; RV32I-NEXT:    addi a0, a0, 819
 ; RV32I-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
-; RV32I-NEXT:    vand.vv v8, v12, v8
+; RV32I-NEXT:    vand.vv v8, v8, v12
+; RV32I-NEXT:    addi a0, a0, 819
 ; RV32I-NEXT:    vsrl.vi v12, v8, 1
 ; RV32I-NEXT:    vand.vv v12, v12, v16
 ; RV32I-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
@@ -1429,17 +1406,14 @@ define <vscale x 4 x i64> @cttz_nxv4i64(<vscale x 4 x i64> %va) {
 ;
 ; RV64I-LABEL: cttz_nxv4i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    li a0, 1
-; RV64I-NEXT:    lui a1, 349525
-; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    lui a3, 61681
-; RV64I-NEXT:    lui a4, 4112
-; RV64I-NEXT:    vsetvli a5, zero, e64, m4, ta, ma
-; RV64I-NEXT:    vsub.vx v12, v8, a0
-; RV64I-NEXT:    addiw a0, a1, 1365
-; RV64I-NEXT:    addiw a1, a2, 819
-; RV64I-NEXT:    addiw a2, a3, -241
-; RV64I-NEXT:    addiw a3, a4, 257
+; RV64I-NEXT:    lui a0, 349525
+; RV64I-NEXT:    lui a1, 209715
+; RV64I-NEXT:    lui a2, 61681
+; RV64I-NEXT:    lui a3, 4112
+; RV64I-NEXT:    addiw a0, a0, 1365
+; RV64I-NEXT:    addiw a1, a1, 819
+; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addiw a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -1448,6 +1422,8 @@ define <vscale x 4 x i64> @cttz_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV64I-NEXT:    add a2, a2, a4
 ; RV64I-NEXT:    slli a4, a3, 32
 ; RV64I-NEXT:    add a3, a3, a4
+; RV64I-NEXT:    vsetvli a4, zero, e64, m4, ta, ma
+; RV64I-NEXT:    vadd.vi v12, v8, -1
 ; RV64I-NEXT:    vnot.v v8, v8
 ; RV64I-NEXT:    vand.vv v8, v8, v12
 ; RV64I-NEXT:    vsrl.vi v12, v8, 1
@@ -1513,18 +1489,17 @@ declare <vscale x 4 x i64> @llvm.cttz.nxv4i64(<vscale x 4 x i64>, i1)
 define <vscale x 8 x i64> @cttz_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV32I-LABEL: cttz_nxv8i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    li a0, 1
-; RV32I-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV32I-NEXT:    vnot.v v16, v8
-; RV32I-NEXT:    vsub.vx v8, v8, a0
+; RV32I-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; RV32I-NEXT:    vadd.vi v16, v8, -1
+; RV32I-NEXT:    vnot.v v8, v8
 ; RV32I-NEXT:    lui a0, 349525
 ; RV32I-NEXT:    addi a0, a0, 1365
 ; RV32I-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
 ; RV32I-NEXT:    vmv.v.x v24, a0
 ; RV32I-NEXT:    lui a0, 209715
-; RV32I-NEXT:    addi a0, a0, 819
 ; RV32I-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV32I-NEXT:    vand.vv v8, v16, v8
+; RV32I-NEXT:    vand.vv v8, v8, v16
+; RV32I-NEXT:    addi a0, a0, 819
 ; RV32I-NEXT:    vsrl.vi v16, v8, 1
 ; RV32I-NEXT:    vand.vv v24, v16, v24
 ; RV32I-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
@@ -1555,17 +1530,14 @@ define <vscale x 8 x i64> @cttz_nxv8i64(<vscale x 8 x i64> %va) {
 ;
 ; RV64I-LABEL: cttz_nxv8i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    li a0, 1
-; RV64I-NEXT:    lui a1, 349525
-; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    lui a3, 61681
-; RV64I-NEXT:    lui a4, 4112
-; RV64I-NEXT:    vsetvli a5, zero, e64, m8, ta, ma
-; RV64I-NEXT:    vsub.vx v16, v8, a0
-; RV64I-NEXT:    addiw a0, a1, 1365
-; RV64I-NEXT:    addiw a1, a2, 819
-; RV64I-NEXT:    addiw a2, a3, -241
-; RV64I-NEXT:    addiw a3, a4, 257
+; RV64I-NEXT:    lui a0, 349525
+; RV64I-NEXT:    lui a1, 209715
+; RV64I-NEXT:    lui a2, 61681
+; RV64I-NEXT:    lui a3, 4112
+; RV64I-NEXT:    addiw a0, a0, 1365
+; RV64I-NEXT:    addiw a1, a1, 819
+; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addiw a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -1574,6 +1546,8 @@ define <vscale x 8 x i64> @cttz_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV64I-NEXT:    add a2, a2, a4
 ; RV64I-NEXT:    slli a4, a3, 32
 ; RV64I-NEXT:    add a3, a3, a4
+; RV64I-NEXT:    vsetvli a4, zero, e64, m8, ta, ma
+; RV64I-NEXT:    vadd.vi v16, v8, -1
 ; RV64I-NEXT:    vnot.v v8, v8
 ; RV64I-NEXT:    vand.vv v8, v8, v16
 ; RV64I-NEXT:    vsrl.vi v16, v8, 1
@@ -1639,12 +1613,11 @@ declare <vscale x 8 x i64> @llvm.cttz.nxv8i64(<vscale x 8 x i64>, i1)
 define <vscale x 1 x i8> @cttz_zero_undef_nxv1i8(<vscale x 1 x i8> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_zero_undef_nxv1i8:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v9, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v9, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    li a0, 85
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v9, v8
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v9
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v9, v9, a0
 ; CHECK-ZVE64X-NEXT:    li a0, 51
@@ -1700,12 +1673,11 @@ define <vscale x 1 x i8> @cttz_zero_undef_nxv1i8(<vscale x 1 x i8> %va) {
 define <vscale x 2 x i8> @cttz_zero_undef_nxv2i8(<vscale x 2 x i8> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_zero_undef_nxv2i8:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v9, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e8, mf4, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v9, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    li a0, 85
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v9, v8
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v9
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v9, v9, a0
 ; CHECK-ZVE64X-NEXT:    li a0, 51
@@ -1761,12 +1733,11 @@ define <vscale x 2 x i8> @cttz_zero_undef_nxv2i8(<vscale x 2 x i8> %va) {
 define <vscale x 4 x i8> @cttz_zero_undef_nxv4i8(<vscale x 4 x i8> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_zero_undef_nxv4i8:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v9, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v9, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    li a0, 85
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v9, v8
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v9
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v9, v9, a0
 ; CHECK-ZVE64X-NEXT:    li a0, 51
@@ -1822,12 +1793,11 @@ define <vscale x 4 x i8> @cttz_zero_undef_nxv4i8(<vscale x 4 x i8> %va) {
 define <vscale x 8 x i8> @cttz_zero_undef_nxv8i8(<vscale x 8 x i8> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_zero_undef_nxv8i8:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v9, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v9, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    li a0, 85
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v9, v8
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v9
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v9, v9, a0
 ; CHECK-ZVE64X-NEXT:    li a0, 51
@@ -1883,12 +1853,11 @@ define <vscale x 8 x i8> @cttz_zero_undef_nxv8i8(<vscale x 8 x i8> %va) {
 define <vscale x 16 x i8> @cttz_zero_undef_nxv16i8(<vscale x 16 x i8> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_zero_undef_nxv16i8:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e8, m2, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v10, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v10, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    li a0, 85
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v10, v8
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v10
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v10, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v10, v10, a0
 ; CHECK-ZVE64X-NEXT:    li a0, 51
@@ -1944,12 +1913,11 @@ define <vscale x 16 x i8> @cttz_zero_undef_nxv16i8(<vscale x 16 x i8> %va) {
 define <vscale x 32 x i8> @cttz_zero_undef_nxv32i8(<vscale x 32 x i8> %va) {
 ; CHECK-LABEL: cttz_zero_undef_nxv32i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
-; CHECK-NEXT:    vsetvli a1, zero, e8, m4, ta, ma
-; CHECK-NEXT:    vnot.v v12, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
+; CHECK-NEXT:    vadd.vi v12, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v12, v8
+; CHECK-NEXT:    vand.vv v8, v8, v12
 ; CHECK-NEXT:    vsrl.vi v12, v8, 1
 ; CHECK-NEXT:    vand.vx v12, v12, a0
 ; CHECK-NEXT:    li a0, 51
@@ -1975,12 +1943,11 @@ define <vscale x 32 x i8> @cttz_zero_undef_nxv32i8(<vscale x 32 x i8> %va) {
 define <vscale x 64 x i8> @cttz_zero_undef_nxv64i8(<vscale x 64 x i8> %va) {
 ; CHECK-LABEL: cttz_zero_undef_nxv64i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
-; CHECK-NEXT:    vsetvli a1, zero, e8, m8, ta, ma
-; CHECK-NEXT:    vnot.v v16, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vadd.vi v16, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v16, v8
+; CHECK-NEXT:    vand.vv v8, v8, v16
 ; CHECK-NEXT:    vsrl.vi v16, v8, 1
 ; CHECK-NEXT:    vand.vx v16, v16, a0
 ; CHECK-NEXT:    li a0, 51
@@ -2006,13 +1973,12 @@ define <vscale x 64 x i8> @cttz_zero_undef_nxv64i8(<vscale x 64 x i8> %va) {
 define <vscale x 1 x i16> @cttz_zero_undef_nxv1i16(<vscale x 1 x i16> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_zero_undef_nxv1i16:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v9, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v9, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 5
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v9
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v9, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v9, v9, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 3
@@ -2066,13 +2032,12 @@ define <vscale x 1 x i16> @cttz_zero_undef_nxv1i16(<vscale x 1 x i16> %va) {
 define <vscale x 2 x i16> @cttz_zero_undef_nxv2i16(<vscale x 2 x i16> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_zero_undef_nxv2i16:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v9, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v9, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 5
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v9
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v9, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v9, v9, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 3
@@ -2126,13 +2091,12 @@ define <vscale x 2 x i16> @cttz_zero_undef_nxv2i16(<vscale x 2 x i16> %va) {
 define <vscale x 4 x i16> @cttz_zero_undef_nxv4i16(<vscale x 4 x i16> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_zero_undef_nxv4i16:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v9, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v9, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 5
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v9
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v9, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v9, v9, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 3
@@ -2186,13 +2150,12 @@ define <vscale x 4 x i16> @cttz_zero_undef_nxv4i16(<vscale x 4 x i16> %va) {
 define <vscale x 8 x i16> @cttz_zero_undef_nxv8i16(<vscale x 8 x i16> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_zero_undef_nxv8i16:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v10, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v10, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 5
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v10
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v10, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v10, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v10, v10, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 3
@@ -2246,13 +2209,12 @@ define <vscale x 8 x i16> @cttz_zero_undef_nxv8i16(<vscale x 8 x i16> %va) {
 define <vscale x 16 x i16> @cttz_zero_undef_nxv16i16(<vscale x 16 x i16> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_zero_undef_nxv16i16:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v12, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v12, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 5
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v12
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v12, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v12, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v12, v12, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 3
@@ -2306,13 +2268,12 @@ define <vscale x 16 x i16> @cttz_zero_undef_nxv16i16(<vscale x 16 x i16> %va) {
 define <vscale x 32 x i16> @cttz_zero_undef_nxv32i16(<vscale x 32 x i16> %va) {
 ; CHECK-LABEL: cttz_zero_undef_nxv32i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
-; CHECK-NEXT:    vsetvli a1, zero, e16, m8, ta, ma
-; CHECK-NEXT:    vnot.v v16, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m8, ta, ma
+; CHECK-NEXT:    vadd.vi v16, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    vand.vv v8, v8, v16
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v16, v8
 ; CHECK-NEXT:    vsrl.vi v16, v8, 1
 ; CHECK-NEXT:    vand.vx v16, v16, a0
 ; CHECK-NEXT:    lui a0, 3
@@ -2344,13 +2305,12 @@ define <vscale x 32 x i16> @cttz_zero_undef_nxv32i16(<vscale x 32 x i16> %va) {
 define <vscale x 1 x i32> @cttz_zero_undef_nxv1i32(<vscale x 1 x i32> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_zero_undef_nxv1i32:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v9, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v9, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 349525
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v9
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v9, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v9, v9, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 209715
@@ -2408,13 +2368,12 @@ define <vscale x 1 x i32> @cttz_zero_undef_nxv1i32(<vscale x 1 x i32> %va) {
 define <vscale x 2 x i32> @cttz_zero_undef_nxv2i32(<vscale x 2 x i32> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_zero_undef_nxv2i32:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v9, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v9, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 349525
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v9
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v9, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v9, v9, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 209715
@@ -2472,13 +2431,12 @@ define <vscale x 2 x i32> @cttz_zero_undef_nxv2i32(<vscale x 2 x i32> %va) {
 define <vscale x 4 x i32> @cttz_zero_undef_nxv4i32(<vscale x 4 x i32> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_zero_undef_nxv4i32:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v10, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v10, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 349525
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v10
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v10, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v10, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v10, v10, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 209715
@@ -2536,13 +2494,12 @@ define <vscale x 4 x i32> @cttz_zero_undef_nxv4i32(<vscale x 4 x i32> %va) {
 define <vscale x 8 x i32> @cttz_zero_undef_nxv8i32(<vscale x 8 x i32> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_zero_undef_nxv8i32:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v12, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v12, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 349525
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v12
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v12, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v12, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v12, v12, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 209715
@@ -2600,13 +2557,12 @@ define <vscale x 8 x i32> @cttz_zero_undef_nxv8i32(<vscale x 8 x i32> %va) {
 define <vscale x 16 x i32> @cttz_zero_undef_nxv16i32(<vscale x 16 x i32> %va) {
 ; CHECK-ZVE64X-LABEL: cttz_zero_undef_nxv16i32:
 ; CHECK-ZVE64X:       # %bb.0:
-; CHECK-ZVE64X-NEXT:    li a0, 1
-; CHECK-ZVE64X-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
-; CHECK-ZVE64X-NEXT:    vnot.v v16, v8
-; CHECK-ZVE64X-NEXT:    vsub.vx v8, v8, a0
+; CHECK-ZVE64X-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-ZVE64X-NEXT:    vadd.vi v16, v8, -1
+; CHECK-ZVE64X-NEXT:    vnot.v v8, v8
 ; CHECK-ZVE64X-NEXT:    lui a0, 349525
+; CHECK-ZVE64X-NEXT:    vand.vv v8, v8, v16
 ; CHECK-ZVE64X-NEXT:    addi a0, a0, 1365
-; CHECK-ZVE64X-NEXT:    vand.vv v8, v16, v8
 ; CHECK-ZVE64X-NEXT:    vsrl.vi v16, v8, 1
 ; CHECK-ZVE64X-NEXT:    vand.vx v16, v16, a0
 ; CHECK-ZVE64X-NEXT:    lui a0, 209715
@@ -2665,18 +2621,17 @@ define <vscale x 16 x i32> @cttz_zero_undef_nxv16i32(<vscale x 16 x i32> %va) {
 define <vscale x 1 x i64> @cttz_zero_undef_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV32I-LABEL: cttz_zero_undef_nxv1i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    li a0, 1
-; RV32I-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; RV32I-NEXT:    vnot.v v9, v8
-; RV32I-NEXT:    vsub.vx v8, v8, a0
+; RV32I-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; RV32I-NEXT:    vadd.vi v9, v8, -1
+; RV32I-NEXT:    vnot.v v8, v8
 ; RV32I-NEXT:    lui a0, 349525
 ; RV32I-NEXT:    addi a0, a0, 1365
 ; RV32I-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
 ; RV32I-NEXT:    vmv.v.x v10, a0
 ; RV32I-NEXT:    lui a0, 209715
-; RV32I-NEXT:    addi a0, a0, 819
 ; RV32I-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; RV32I-NEXT:    vand.vv v8, v9, v8
+; RV32I-NEXT:    vand.vv v8, v8, v9
+; RV32I-NEXT:    addi a0, a0, 819
 ; RV32I-NEXT:    vsrl.vi v9, v8, 1
 ; RV32I-NEXT:    vand.vv v9, v9, v10
 ; RV32I-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
@@ -2707,17 +2662,14 @@ define <vscale x 1 x i64> @cttz_zero_undef_nxv1i64(<vscale x 1 x i64> %va) {
 ;
 ; RV64I-LABEL: cttz_zero_undef_nxv1i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    li a0, 1
-; RV64I-NEXT:    lui a1, 349525
-; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    lui a3, 61681
-; RV64I-NEXT:    lui a4, 4112
-; RV64I-NEXT:    vsetvli a5, zero, e64, m1, ta, ma
-; RV64I-NEXT:    vsub.vx v9, v8, a0
-; RV64I-NEXT:    addiw a0, a1, 1365
-; RV64I-NEXT:    addiw a1, a2, 819
-; RV64I-NEXT:    addiw a2, a3, -241
-; RV64I-NEXT:    addiw a3, a4, 257
+; RV64I-NEXT:    lui a0, 349525
+; RV64I-NEXT:    lui a1, 209715
+; RV64I-NEXT:    lui a2, 61681
+; RV64I-NEXT:    lui a3, 4112
+; RV64I-NEXT:    addiw a0, a0, 1365
+; RV64I-NEXT:    addiw a1, a1, 819
+; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addiw a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -2726,6 +2678,8 @@ define <vscale x 1 x i64> @cttz_zero_undef_nxv1i64(<vscale x 1 x i64> %va) {
 ; RV64I-NEXT:    add a2, a2, a4
 ; RV64I-NEXT:    slli a4, a3, 32
 ; RV64I-NEXT:    add a3, a3, a4
+; RV64I-NEXT:    vsetvli a4, zero, e64, m1, ta, ma
+; RV64I-NEXT:    vadd.vi v9, v8, -1
 ; RV64I-NEXT:    vnot.v v8, v8
 ; RV64I-NEXT:    vand.vv v8, v8, v9
 ; RV64I-NEXT:    vsrl.vi v9, v8, 1
@@ -2783,18 +2737,17 @@ define <vscale x 1 x i64> @cttz_zero_undef_nxv1i64(<vscale x 1 x i64> %va) {
 define <vscale x 2 x i64> @cttz_zero_undef_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV32I-LABEL: cttz_zero_undef_nxv2i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    li a0, 1
-; RV32I-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
-; RV32I-NEXT:    vnot.v v10, v8
-; RV32I-NEXT:    vsub.vx v8, v8, a0
+; RV32I-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
+; RV32I-NEXT:    vadd.vi v10, v8, -1
+; RV32I-NEXT:    vnot.v v8, v8
 ; RV32I-NEXT:    lui a0, 349525
 ; RV32I-NEXT:    addi a0, a0, 1365
 ; RV32I-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; RV32I-NEXT:    vmv.v.x v12, a0
 ; RV32I-NEXT:    lui a0, 209715
-; RV32I-NEXT:    addi a0, a0, 819
 ; RV32I-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
-; RV32I-NEXT:    vand.vv v8, v10, v8
+; RV32I-NEXT:    vand.vv v8, v8, v10
+; RV32I-NEXT:    addi a0, a0, 819
 ; RV32I-NEXT:    vsrl.vi v10, v8, 1
 ; RV32I-NEXT:    vand.vv v10, v10, v12
 ; RV32I-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
@@ -2825,17 +2778,14 @@ define <vscale x 2 x i64> @cttz_zero_undef_nxv2i64(<vscale x 2 x i64> %va) {
 ;
 ; RV64I-LABEL: cttz_zero_undef_nxv2i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    li a0, 1
-; RV64I-NEXT:    lui a1, 349525
-; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    lui a3, 61681
-; RV64I-NEXT:    lui a4, 4112
-; RV64I-NEXT:    vsetvli a5, zero, e64, m2, ta, ma
-; RV64I-NEXT:    vsub.vx v10, v8, a0
-; RV64I-NEXT:    addiw a0, a1, 1365
-; RV64I-NEXT:    addiw a1, a2, 819
-; RV64I-NEXT:    addiw a2, a3, -241
-; RV64I-NEXT:    addiw a3, a4, 257
+; RV64I-NEXT:    lui a0, 349525
+; RV64I-NEXT:    lui a1, 209715
+; RV64I-NEXT:    lui a2, 61681
+; RV64I-NEXT:    lui a3, 4112
+; RV64I-NEXT:    addiw a0, a0, 1365
+; RV64I-NEXT:    addiw a1, a1, 819
+; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addiw a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -2844,6 +2794,8 @@ define <vscale x 2 x i64> @cttz_zero_undef_nxv2i64(<vscale x 2 x i64> %va) {
 ; RV64I-NEXT:    add a2, a2, a4
 ; RV64I-NEXT:    slli a4, a3, 32
 ; RV64I-NEXT:    add a3, a3, a4
+; RV64I-NEXT:    vsetvli a4, zero, e64, m2, ta, ma
+; RV64I-NEXT:    vadd.vi v10, v8, -1
 ; RV64I-NEXT:    vnot.v v8, v8
 ; RV64I-NEXT:    vand.vv v8, v8, v10
 ; RV64I-NEXT:    vsrl.vi v10, v8, 1
@@ -2901,18 +2853,17 @@ define <vscale x 2 x i64> @cttz_zero_undef_nxv2i64(<vscale x 2 x i64> %va) {
 define <vscale x 4 x i64> @cttz_zero_undef_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV32I-LABEL: cttz_zero_undef_nxv4i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    li a0, 1
-; RV32I-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
-; RV32I-NEXT:    vnot.v v12, v8
-; RV32I-NEXT:    vsub.vx v8, v8, a0
+; RV32I-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; RV32I-NEXT:    vadd.vi v12, v8, -1
+; RV32I-NEXT:    vnot.v v8, v8
 ; RV32I-NEXT:    lui a0, 349525
 ; RV32I-NEXT:    addi a0, a0, 1365
 ; RV32I-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
 ; RV32I-NEXT:    vmv.v.x v16, a0
 ; RV32I-NEXT:    lui a0, 209715
-; RV32I-NEXT:    addi a0, a0, 819
 ; RV32I-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
-; RV32I-NEXT:    vand.vv v8, v12, v8
+; RV32I-NEXT:    vand.vv v8, v8, v12
+; RV32I-NEXT:    addi a0, a0, 819
 ; RV32I-NEXT:    vsrl.vi v12, v8, 1
 ; RV32I-NEXT:    vand.vv v12, v12, v16
 ; RV32I-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
@@ -2943,17 +2894,14 @@ define <vscale x 4 x i64> @cttz_zero_undef_nxv4i64(<vscale x 4 x i64> %va) {
 ;
 ; RV64I-LABEL: cttz_zero_undef_nxv4i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    li a0, 1
-; RV64I-NEXT:    lui a1, 349525
-; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    lui a3, 61681
-; RV64I-NEXT:    lui a4, 4112
-; RV64I-NEXT:    vsetvli a5, zero, e64, m4, ta, ma
-; RV64I-NEXT:    vsub.vx v12, v8, a0
-; RV64I-NEXT:    addiw a0, a1, 1365
-; RV64I-NEXT:    addiw a1, a2, 819
-; RV64I-NEXT:    addiw a2, a3, -241
-; RV64I-NEXT:    addiw a3, a4, 257
+; RV64I-NEXT:    lui a0, 349525
+; RV64I-NEXT:    lui a1, 209715
+; RV64I-NEXT:    lui a2, 61681
+; RV64I-NEXT:    lui a3, 4112
+; RV64I-NEXT:    addiw a0, a0, 1365
+; RV64I-NEXT:    addiw a1, a1, 819
+; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addiw a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -2962,6 +2910,8 @@ define <vscale x 4 x i64> @cttz_zero_undef_nxv4i64(<vscale x 4 x i64> %va) {
 ; RV64I-NEXT:    add a2, a2, a4
 ; RV64I-NEXT:    slli a4, a3, 32
 ; RV64I-NEXT:    add a3, a3, a4
+; RV64I-NEXT:    vsetvli a4, zero, e64, m4, ta, ma
+; RV64I-NEXT:    vadd.vi v12, v8, -1
 ; RV64I-NEXT:    vnot.v v8, v8
 ; RV64I-NEXT:    vand.vv v8, v8, v12
 ; RV64I-NEXT:    vsrl.vi v12, v8, 1
@@ -3019,18 +2969,17 @@ define <vscale x 4 x i64> @cttz_zero_undef_nxv4i64(<vscale x 4 x i64> %va) {
 define <vscale x 8 x i64> @cttz_zero_undef_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV32I-LABEL: cttz_zero_undef_nxv8i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    li a0, 1
-; RV32I-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV32I-NEXT:    vnot.v v16, v8
-; RV32I-NEXT:    vsub.vx v8, v8, a0
+; RV32I-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; RV32I-NEXT:    vadd.vi v16, v8, -1
+; RV32I-NEXT:    vnot.v v8, v8
 ; RV32I-NEXT:    lui a0, 349525
 ; RV32I-NEXT:    addi a0, a0, 1365
 ; RV32I-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
 ; RV32I-NEXT:    vmv.v.x v24, a0
 ; RV32I-NEXT:    lui a0, 209715
-; RV32I-NEXT:    addi a0, a0, 819
 ; RV32I-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; RV32I-NEXT:    vand.vv v8, v16, v8
+; RV32I-NEXT:    vand.vv v8, v8, v16
+; RV32I-NEXT:    addi a0, a0, 819
 ; RV32I-NEXT:    vsrl.vi v16, v8, 1
 ; RV32I-NEXT:    vand.vv v24, v16, v24
 ; RV32I-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
@@ -3061,17 +3010,14 @@ define <vscale x 8 x i64> @cttz_zero_undef_nxv8i64(<vscale x 8 x i64> %va) {
 ;
 ; RV64I-LABEL: cttz_zero_undef_nxv8i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    li a0, 1
-; RV64I-NEXT:    lui a1, 349525
-; RV64I-NEXT:    lui a2, 209715
-; RV64I-NEXT:    lui a3, 61681
-; RV64I-NEXT:    lui a4, 4112
-; RV64I-NEXT:    vsetvli a5, zero, e64, m8, ta, ma
-; RV64I-NEXT:    vsub.vx v16, v8, a0
-; RV64I-NEXT:    addiw a0, a1, 1365
-; RV64I-NEXT:    addiw a1, a2, 819
-; RV64I-NEXT:    addiw a2, a3, -241
-; RV64I-NEXT:    addiw a3, a4, 257
+; RV64I-NEXT:    lui a0, 349525
+; RV64I-NEXT:    lui a1, 209715
+; RV64I-NEXT:    lui a2, 61681
+; RV64I-NEXT:    lui a3, 4112
+; RV64I-NEXT:    addiw a0, a0, 1365
+; RV64I-NEXT:    addiw a1, a1, 819
+; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    addiw a3, a3, 257
 ; RV64I-NEXT:    slli a4, a0, 32
 ; RV64I-NEXT:    add a0, a0, a4
 ; RV64I-NEXT:    slli a4, a1, 32
@@ -3080,6 +3026,8 @@ define <vscale x 8 x i64> @cttz_zero_undef_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV64I-NEXT:    add a2, a2, a4
 ; RV64I-NEXT:    slli a4, a3, 32
 ; RV64I-NEXT:    add a3, a3, a4
+; RV64I-NEXT:    vsetvli a4, zero, e64, m8, ta, ma
+; RV64I-NEXT:    vadd.vi v16, v8, -1
 ; RV64I-NEXT:    vnot.v v8, v8
 ; RV64I-NEXT:    vand.vv v8, v8, v16
 ; RV64I-NEXT:    vsrl.vi v16, v8, 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll b/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll
index 766717d92a749..38ef54ff4fe41 100644
--- a/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/cttz-vp.ll
@@ -13,9 +13,8 @@ declare <vscale x 1 x i8> @llvm.vp.cttz.nxv1i8(<vscale x 1 x i8>, i1 immarg, <vs
 define <vscale x 1 x i8> @vp_cttz_nxv1i8(<vscale x 1 x i8> %va, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv1i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    li a0, 85
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    vand.vv v8, v8, v9, v0.t
@@ -44,12 +43,11 @@ define <vscale x 1 x i8> @vp_cttz_nxv1i8(<vscale x 1 x i8> %va, <vscale x 1 x i1
 define <vscale x 1 x i8> @vp_cttz_nxv1i8_unmasked(<vscale x 1 x i8> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv1i8_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v9, v8
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    li a0, 51
@@ -77,9 +75,8 @@ declare <vscale x 2 x i8> @llvm.vp.cttz.nxv2i8(<vscale x 2 x i8>, i1 immarg, <vs
 define <vscale x 2 x i8> @vp_cttz_nxv2i8(<vscale x 2 x i8> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv2i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    li a0, 85
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    vand.vv v8, v8, v9, v0.t
@@ -108,12 +105,11 @@ define <vscale x 2 x i8> @vp_cttz_nxv2i8(<vscale x 2 x i8> %va, <vscale x 2 x i1
 define <vscale x 2 x i8> @vp_cttz_nxv2i8_unmasked(<vscale x 2 x i8> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv2i8_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v9, v8
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    li a0, 51
@@ -141,9 +137,8 @@ declare <vscale x 4 x i8> @llvm.vp.cttz.nxv4i8(<vscale x 4 x i8>, i1 immarg, <vs
 define <vscale x 4 x i8> @vp_cttz_nxv4i8(<vscale x 4 x i8> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    li a0, 85
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    vand.vv v8, v8, v9, v0.t
@@ -172,12 +167,11 @@ define <vscale x 4 x i8> @vp_cttz_nxv4i8(<vscale x 4 x i8> %va, <vscale x 4 x i1
 define <vscale x 4 x i8> @vp_cttz_nxv4i8_unmasked(<vscale x 4 x i8> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv4i8_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v9, v8
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    li a0, 51
@@ -205,9 +199,8 @@ declare <vscale x 8 x i8> @llvm.vp.cttz.nxv8i8(<vscale x 8 x i8>, i1 immarg, <vs
 define <vscale x 8 x i8> @vp_cttz_nxv8i8(<vscale x 8 x i8> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv8i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    li a0, 85
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    vand.vv v8, v8, v9, v0.t
@@ -236,12 +229,11 @@ define <vscale x 8 x i8> @vp_cttz_nxv8i8(<vscale x 8 x i8> %va, <vscale x 8 x i1
 define <vscale x 8 x i8> @vp_cttz_nxv8i8_unmasked(<vscale x 8 x i8> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv8i8_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v9, v8
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    li a0, 51
@@ -269,9 +261,8 @@ declare <vscale x 16 x i8> @llvm.vp.cttz.nxv16i8(<vscale x 16 x i8>, i1 immarg,
 define <vscale x 16 x i8> @vp_cttz_nxv16i8(<vscale x 16 x i8> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv16i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; CHECK-NEXT:    vsub.vx v10, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v10, v8, -1, v0.t
 ; CHECK-NEXT:    li a0, 85
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    vand.vv v8, v8, v10, v0.t
@@ -300,12 +291,11 @@ define <vscale x 16 x i8> @vp_cttz_nxv16i8(<vscale x 16 x i8> %va, <vscale x 16
 define <vscale x 16 x i8> @vp_cttz_nxv16i8_unmasked(<vscale x 16 x i8> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv16i8_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; CHECK-NEXT:    vnot.v v10, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v10, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v10, v8
+; CHECK-NEXT:    vand.vv v8, v8, v10
 ; CHECK-NEXT:    vsrl.vi v10, v8, 1
 ; CHECK-NEXT:    vand.vx v10, v10, a0
 ; CHECK-NEXT:    li a0, 51
@@ -333,9 +323,8 @@ declare <vscale x 32 x i8> @llvm.vp.cttz.nxv32i8(<vscale x 32 x i8>, i1 immarg,
 define <vscale x 32 x i8> @vp_cttz_nxv32i8(<vscale x 32 x i8> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv32i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; CHECK-NEXT:    vsub.vx v12, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v12, v8, -1, v0.t
 ; CHECK-NEXT:    li a0, 85
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    vand.vv v8, v8, v12, v0.t
@@ -364,12 +353,11 @@ define <vscale x 32 x i8> @vp_cttz_nxv32i8(<vscale x 32 x i8> %va, <vscale x 32
 define <vscale x 32 x i8> @vp_cttz_nxv32i8_unmasked(<vscale x 32 x i8> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv32i8_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; CHECK-NEXT:    vnot.v v12, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v12, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v12, v8
+; CHECK-NEXT:    vand.vv v8, v8, v12
 ; CHECK-NEXT:    vsrl.vi v12, v8, 1
 ; CHECK-NEXT:    vand.vx v12, v12, a0
 ; CHECK-NEXT:    li a0, 51
@@ -397,9 +385,8 @@ declare <vscale x 64 x i8> @llvm.vp.cttz.nxv64i8(<vscale x 64 x i8>, i1 immarg,
 define <vscale x 64 x i8> @vp_cttz_nxv64i8(<vscale x 64 x i8> %va, <vscale x 64 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv64i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
-; CHECK-NEXT:    vsub.vx v16, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; CHECK-NEXT:    li a0, 85
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    vand.vv v8, v8, v16, v0.t
@@ -428,12 +415,11 @@ define <vscale x 64 x i8> @vp_cttz_nxv64i8(<vscale x 64 x i8> %va, <vscale x 64
 define <vscale x 64 x i8> @vp_cttz_nxv64i8_unmasked(<vscale x 64 x i8> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv64i8_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
-; CHECK-NEXT:    vnot.v v16, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v16, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v16, v8
+; CHECK-NEXT:    vand.vv v8, v8, v16
 ; CHECK-NEXT:    vsrl.vi v16, v8, 1
 ; CHECK-NEXT:    vand.vx v16, v16, a0
 ; CHECK-NEXT:    li a0, 51
@@ -461,9 +447,8 @@ declare <vscale x 1 x i16> @llvm.vp.cttz.nxv1i16(<vscale x 1 x i16>, i1 immarg,
 define <vscale x 1 x i16> @vp_cttz_nxv1i16(<vscale x 1 x i16> %va, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv1i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 5
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -499,13 +484,12 @@ define <vscale x 1 x i16> @vp_cttz_nxv1i16(<vscale x 1 x i16> %va, <vscale x 1 x
 define <vscale x 1 x i16> @vp_cttz_nxv1i16_unmasked(<vscale x 1 x i16> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv1i16_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v9, v8
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    lui a0, 3
@@ -539,9 +523,8 @@ declare <vscale x 2 x i16> @llvm.vp.cttz.nxv2i16(<vscale x 2 x i16>, i1 immarg,
 define <vscale x 2 x i16> @vp_cttz_nxv2i16(<vscale x 2 x i16> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv2i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 5
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -577,13 +560,12 @@ define <vscale x 2 x i16> @vp_cttz_nxv2i16(<vscale x 2 x i16> %va, <vscale x 2 x
 define <vscale x 2 x i16> @vp_cttz_nxv2i16_unmasked(<vscale x 2 x i16> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv2i16_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v9, v8
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    lui a0, 3
@@ -617,9 +599,8 @@ declare <vscale x 4 x i16> @llvm.vp.cttz.nxv4i16(<vscale x 4 x i16>, i1 immarg,
 define <vscale x 4 x i16> @vp_cttz_nxv4i16(<vscale x 4 x i16> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv4i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 5
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -655,13 +636,12 @@ define <vscale x 4 x i16> @vp_cttz_nxv4i16(<vscale x 4 x i16> %va, <vscale x 4 x
 define <vscale x 4 x i16> @vp_cttz_nxv4i16_unmasked(<vscale x 4 x i16> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv4i16_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v9, v8
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    lui a0, 3
@@ -695,9 +675,8 @@ declare <vscale x 8 x i16> @llvm.vp.cttz.nxv8i16(<vscale x 8 x i16>, i1 immarg,
 define <vscale x 8 x i16> @vp_cttz_nxv8i16(<vscale x 8 x i16> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv8i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vsub.vx v10, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v10, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 5
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -733,13 +712,12 @@ define <vscale x 8 x i16> @vp_cttz_nxv8i16(<vscale x 8 x i16> %va, <vscale x 8 x
 define <vscale x 8 x i16> @vp_cttz_nxv8i16_unmasked(<vscale x 8 x i16> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv8i16_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vnot.v v10, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v10, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    vand.vv v8, v8, v10
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v10, v8
 ; CHECK-NEXT:    vsrl.vi v10, v8, 1
 ; CHECK-NEXT:    vand.vx v10, v10, a0
 ; CHECK-NEXT:    lui a0, 3
@@ -773,9 +751,8 @@ declare <vscale x 16 x i16> @llvm.vp.cttz.nxv16i16(<vscale x 16 x i16>, i1 immar
 define <vscale x 16 x i16> @vp_cttz_nxv16i16(<vscale x 16 x i16> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv16i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vsub.vx v12, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v12, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 5
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -811,13 +788,12 @@ define <vscale x 16 x i16> @vp_cttz_nxv16i16(<vscale x 16 x i16> %va, <vscale x
 define <vscale x 16 x i16> @vp_cttz_nxv16i16_unmasked(<vscale x 16 x i16> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv16i16_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vnot.v v12, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v12, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    vand.vv v8, v8, v12
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v12, v8
 ; CHECK-NEXT:    vsrl.vi v12, v8, 1
 ; CHECK-NEXT:    vand.vx v12, v12, a0
 ; CHECK-NEXT:    lui a0, 3
@@ -851,9 +827,8 @@ declare <vscale x 32 x i16> @llvm.vp.cttz.nxv32i16(<vscale x 32 x i16>, i1 immar
 define <vscale x 32 x i16> @vp_cttz_nxv32i16(<vscale x 32 x i16> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv32i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    vsub.vx v16, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 5
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -889,13 +864,12 @@ define <vscale x 32 x i16> @vp_cttz_nxv32i16(<vscale x 32 x i16> %va, <vscale x
 define <vscale x 32 x i16> @vp_cttz_nxv32i16_unmasked(<vscale x 32 x i16> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv32i16_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    vnot.v v16, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v16, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    vand.vv v8, v8, v16
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v16, v8
 ; CHECK-NEXT:    vsrl.vi v16, v8, 1
 ; CHECK-NEXT:    vand.vx v16, v16, a0
 ; CHECK-NEXT:    lui a0, 3
@@ -929,9 +903,8 @@ declare <vscale x 1 x i32> @llvm.vp.cttz.nxv1i32(<vscale x 1 x i32>, i1 immarg,
 define <vscale x 1 x i32> @vp_cttz_nxv1i32(<vscale x 1 x i32> %va, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv1i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 349525
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -968,13 +941,12 @@ define <vscale x 1 x i32> @vp_cttz_nxv1i32(<vscale x 1 x i32> %va, <vscale x 1 x
 define <vscale x 1 x i32> @vp_cttz_nxv1i32_unmasked(<vscale x 1 x i32> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv1i32_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 349525
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v9, v8
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    lui a0, 209715
@@ -1009,9 +981,8 @@ declare <vscale x 2 x i32> @llvm.vp.cttz.nxv2i32(<vscale x 2 x i32>, i1 immarg,
 define <vscale x 2 x i32> @vp_cttz_nxv2i32(<vscale x 2 x i32> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv2i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 349525
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -1048,13 +1019,12 @@ define <vscale x 2 x i32> @vp_cttz_nxv2i32(<vscale x 2 x i32> %va, <vscale x 2 x
 define <vscale x 2 x i32> @vp_cttz_nxv2i32_unmasked(<vscale x 2 x i32> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv2i32_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 349525
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v9, v8
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    lui a0, 209715
@@ -1089,9 +1059,8 @@ declare <vscale x 4 x i32> @llvm.vp.cttz.nxv4i32(<vscale x 4 x i32>, i1 immarg,
 define <vscale x 4 x i32> @vp_cttz_nxv4i32(<vscale x 4 x i32> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    vsub.vx v10, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v10, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 349525
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -1128,13 +1097,12 @@ define <vscale x 4 x i32> @vp_cttz_nxv4i32(<vscale x 4 x i32> %va, <vscale x 4 x
 define <vscale x 4 x i32> @vp_cttz_nxv4i32_unmasked(<vscale x 4 x i32> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv4i32_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    vnot.v v10, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v10, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 349525
+; CHECK-NEXT:    vand.vv v8, v8, v10
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v10, v8
 ; CHECK-NEXT:    vsrl.vi v10, v8, 1
 ; CHECK-NEXT:    vand.vx v10, v10, a0
 ; CHECK-NEXT:    lui a0, 209715
@@ -1169,9 +1137,8 @@ declare <vscale x 8 x i32> @llvm.vp.cttz.nxv8i32(<vscale x 8 x i32>, i1 immarg,
 define <vscale x 8 x i32> @vp_cttz_nxv8i32(<vscale x 8 x i32> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv8i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    vsub.vx v12, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v12, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 349525
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -1208,13 +1175,12 @@ define <vscale x 8 x i32> @vp_cttz_nxv8i32(<vscale x 8 x i32> %va, <vscale x 8 x
 define <vscale x 8 x i32> @vp_cttz_nxv8i32_unmasked(<vscale x 8 x i32> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv8i32_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    vnot.v v12, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v12, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 349525
+; CHECK-NEXT:    vand.vv v8, v8, v12
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v12, v8
 ; CHECK-NEXT:    vsrl.vi v12, v8, 1
 ; CHECK-NEXT:    vand.vx v12, v12, a0
 ; CHECK-NEXT:    lui a0, 209715
@@ -1249,9 +1215,8 @@ declare <vscale x 16 x i32> @llvm.vp.cttz.nxv16i32(<vscale x 16 x i32>, i1 immar
 define <vscale x 16 x i32> @vp_cttz_nxv16i32(<vscale x 16 x i32> %va, <vscale x 16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv16i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    vsub.vx v16, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 349525
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -1288,13 +1253,12 @@ define <vscale x 16 x i32> @vp_cttz_nxv16i32(<vscale x 16 x i32> %va, <vscale x
 define <vscale x 16 x i32> @vp_cttz_nxv16i32_unmasked(<vscale x 16 x i32> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_nxv16i32_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    vnot.v v16, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v16, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 349525
+; CHECK-NEXT:    vand.vv v8, v8, v16
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v16, v8
 ; CHECK-NEXT:    vsrl.vi v16, v8, 1
 ; CHECK-NEXT:    vand.vx v16, v16, a0
 ; CHECK-NEXT:    lui a0, 209715
@@ -1329,9 +1293,8 @@ declare <vscale x 1 x i64> @llvm.vp.cttz.nxv1i64(<vscale x 1 x i64>, i1 immarg,
 define <vscale x 1 x i64> @vp_cttz_nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_nxv1i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; RV32-NEXT:    vsub.vx v9, v8, a1, v0.t
+; RV32-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
@@ -1371,39 +1334,38 @@ define <vscale x 1 x i64> @vp_cttz_nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x
 ;
 ; RV64-LABEL: vp_cttz_nxv1i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; RV64-NEXT:    vsub.vx v9, v8, a1, v0.t
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v9, v0.t
 ; RV64-NEXT:    vsrl.vi v9, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v9, v9, a0, v0.t
+; RV64-NEXT:    vand.vx v9, v9, a1, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v9, v0.t
-; RV64-NEXT:    vand.vx v9, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v9, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vadd.vv v8, v9, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v9, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v9, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    li a0, 56
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a4, v0.t
 ; RV64-NEXT:    vsrl.vx v8, v8, a0, v0.t
 ; RV64-NEXT:    ret
 ;
@@ -1419,18 +1381,17 @@ define <vscale x 1 x i64> @vp_cttz_nxv1i64(<vscale x 1 x i64> %va, <vscale x 1 x
 define <vscale x 1 x i64> @vp_cttz_nxv1i64_unmasked(<vscale x 1 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_nxv1i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; RV32-NEXT:    vnot.v v9, v8
-; RV32-NEXT:    vsub.vx v8, v8, a1
+; RV32-NEXT:    vadd.vi v9, v8, -1
+; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
 ; RV32-NEXT:    vmv.v.x v10, a1
 ; RV32-NEXT:    lui a1, 209715
-; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; RV32-NEXT:    vand.vv v8, v9, v8
+; RV32-NEXT:    vand.vv v8, v8, v9
+; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsrl.vi v9, v8, 1
 ; RV32-NEXT:    vand.vv v9, v9, v10
 ; RV32-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
@@ -1461,38 +1422,37 @@ define <vscale x 1 x i64> @vp_cttz_nxv1i64_unmasked(<vscale x 1 x i64> %va, i32
 ;
 ; RV64-LABEL: vp_cttz_nxv1i64_unmasked:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; RV64-NEXT:    vsub.vx v9, v8, a1
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v9, v8, -1
 ; RV64-NEXT:    vnot.v v8, v8
 ; RV64-NEXT:    vand.vv v8, v8, v9
 ; RV64-NEXT:    vsrl.vi v9, v8, 1
-; RV64-NEXT:    vand.vx v9, v9, a0
+; RV64-NEXT:    vand.vx v9, v9, a1
 ; RV64-NEXT:    vsub.vv v8, v8, v9
-; RV64-NEXT:    vand.vx v9, v8, a1
+; RV64-NEXT:    vand.vx v9, v8, a2
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vand.vx v8, v8, a1
+; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    vadd.vv v8, v9, v8
 ; RV64-NEXT:    vsrl.vi v9, v8, 4
 ; RV64-NEXT:    vadd.vv v8, v8, v9
-; RV64-NEXT:    vand.vx v8, v8, a2
-; RV64-NEXT:    vmul.vx v8, v8, a3
+; RV64-NEXT:    vand.vx v8, v8, a3
+; RV64-NEXT:    vmul.vx v8, v8, a4
 ; RV64-NEXT:    li a0, 56
 ; RV64-NEXT:    vsrl.vx v8, v8, a0
 ; RV64-NEXT:    ret
@@ -1511,9 +1471,8 @@ declare <vscale x 2 x i64> @llvm.vp.cttz.nxv2i64(<vscale x 2 x i64>, i1 immarg,
 define <vscale x 2 x i64> @vp_cttz_nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_nxv2i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; RV32-NEXT:    vsub.vx v10, v8, a1, v0.t
+; RV32-NEXT:    vadd.vi v10, v8, -1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
@@ -1553,39 +1512,38 @@ define <vscale x 2 x i64> @vp_cttz_nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x
 ;
 ; RV64-LABEL: vp_cttz_nxv2i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; RV64-NEXT:    vsub.vx v10, v8, a1, v0.t
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v10, v8, -1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v10, v0.t
 ; RV64-NEXT:    vsrl.vi v10, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v10, v10, a0, v0.t
+; RV64-NEXT:    vand.vx v10, v10, a1, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v10, v0.t
-; RV64-NEXT:    vand.vx v10, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v10, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vadd.vv v8, v10, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v10, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v10, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    li a0, 56
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a4, v0.t
 ; RV64-NEXT:    vsrl.vx v8, v8, a0, v0.t
 ; RV64-NEXT:    ret
 ;
@@ -1601,18 +1559,17 @@ define <vscale x 2 x i64> @vp_cttz_nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x
 define <vscale x 2 x i64> @vp_cttz_nxv2i64_unmasked(<vscale x 2 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_nxv2i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; RV32-NEXT:    vnot.v v10, v8
-; RV32-NEXT:    vsub.vx v8, v8, a1
+; RV32-NEXT:    vadd.vi v10, v8, -1
+; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
 ; RV32-NEXT:    vmv.v.x v12, a1
 ; RV32-NEXT:    lui a1, 209715
-; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; RV32-NEXT:    vand.vv v8, v10, v8
+; RV32-NEXT:    vand.vv v8, v8, v10
+; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsrl.vi v10, v8, 1
 ; RV32-NEXT:    vand.vv v10, v10, v12
 ; RV32-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
@@ -1643,38 +1600,37 @@ define <vscale x 2 x i64> @vp_cttz_nxv2i64_unmasked(<vscale x 2 x i64> %va, i32
 ;
 ; RV64-LABEL: vp_cttz_nxv2i64_unmasked:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; RV64-NEXT:    vsub.vx v10, v8, a1
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v10, v8, -1
 ; RV64-NEXT:    vnot.v v8, v8
 ; RV64-NEXT:    vand.vv v8, v8, v10
 ; RV64-NEXT:    vsrl.vi v10, v8, 1
-; RV64-NEXT:    vand.vx v10, v10, a0
+; RV64-NEXT:    vand.vx v10, v10, a1
 ; RV64-NEXT:    vsub.vv v8, v8, v10
-; RV64-NEXT:    vand.vx v10, v8, a1
+; RV64-NEXT:    vand.vx v10, v8, a2
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vand.vx v8, v8, a1
+; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    vadd.vv v8, v10, v8
 ; RV64-NEXT:    vsrl.vi v10, v8, 4
 ; RV64-NEXT:    vadd.vv v8, v8, v10
-; RV64-NEXT:    vand.vx v8, v8, a2
-; RV64-NEXT:    vmul.vx v8, v8, a3
+; RV64-NEXT:    vand.vx v8, v8, a3
+; RV64-NEXT:    vmul.vx v8, v8, a4
 ; RV64-NEXT:    li a0, 56
 ; RV64-NEXT:    vsrl.vx v8, v8, a0
 ; RV64-NEXT:    ret
@@ -1693,9 +1649,8 @@ declare <vscale x 4 x i64> @llvm.vp.cttz.nxv4i64(<vscale x 4 x i64>, i1 immarg,
 define <vscale x 4 x i64> @vp_cttz_nxv4i64(<vscale x 4 x i64> %va, <vscale x 4 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_nxv4i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; RV32-NEXT:    vsub.vx v12, v8, a1, v0.t
+; RV32-NEXT:    vadd.vi v12, v8, -1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetvli a2, zero, e32, m4, ta, ma
@@ -1735,39 +1690,38 @@ define <vscale x 4 x i64> @vp_cttz_nxv4i64(<vscale x 4 x i64> %va, <vscale x 4 x
 ;
 ; RV64-LABEL: vp_cttz_nxv4i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; RV64-NEXT:    vsub.vx v12, v8, a1, v0.t
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v12, v8, -1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v12, v0.t
 ; RV64-NEXT:    vsrl.vi v12, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v12, v12, a0, v0.t
+; RV64-NEXT:    vand.vx v12, v12, a1, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v12, v0.t
-; RV64-NEXT:    vand.vx v12, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v12, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vadd.vv v8, v12, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v12, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v12, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    li a0, 56
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a4, v0.t
 ; RV64-NEXT:    vsrl.vx v8, v8, a0, v0.t
 ; RV64-NEXT:    ret
 ;
@@ -1783,18 +1737,17 @@ define <vscale x 4 x i64> @vp_cttz_nxv4i64(<vscale x 4 x i64> %va, <vscale x 4 x
 define <vscale x 4 x i64> @vp_cttz_nxv4i64_unmasked(<vscale x 4 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_nxv4i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; RV32-NEXT:    vnot.v v12, v8
-; RV32-NEXT:    vsub.vx v8, v8, a1
+; RV32-NEXT:    vadd.vi v12, v8, -1
+; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetvli a2, zero, e32, m4, ta, ma
 ; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 209715
-; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; RV32-NEXT:    vand.vv v8, v12, v8
+; RV32-NEXT:    vand.vv v8, v8, v12
+; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsrl.vi v12, v8, 1
 ; RV32-NEXT:    vand.vv v12, v12, v16
 ; RV32-NEXT:    vsetvli a2, zero, e32, m4, ta, ma
@@ -1825,38 +1778,37 @@ define <vscale x 4 x i64> @vp_cttz_nxv4i64_unmasked(<vscale x 4 x i64> %va, i32
 ;
 ; RV64-LABEL: vp_cttz_nxv4i64_unmasked:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; RV64-NEXT:    vsub.vx v12, v8, a1
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v12, v8, -1
 ; RV64-NEXT:    vnot.v v8, v8
 ; RV64-NEXT:    vand.vv v8, v8, v12
 ; RV64-NEXT:    vsrl.vi v12, v8, 1
-; RV64-NEXT:    vand.vx v12, v12, a0
+; RV64-NEXT:    vand.vx v12, v12, a1
 ; RV64-NEXT:    vsub.vv v8, v8, v12
-; RV64-NEXT:    vand.vx v12, v8, a1
+; RV64-NEXT:    vand.vx v12, v8, a2
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vand.vx v8, v8, a1
+; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    vadd.vv v8, v12, v8
 ; RV64-NEXT:    vsrl.vi v12, v8, 4
 ; RV64-NEXT:    vadd.vv v8, v8, v12
-; RV64-NEXT:    vand.vx v8, v8, a2
-; RV64-NEXT:    vmul.vx v8, v8, a3
+; RV64-NEXT:    vand.vx v8, v8, a3
+; RV64-NEXT:    vmul.vx v8, v8, a4
 ; RV64-NEXT:    li a0, 56
 ; RV64-NEXT:    vsrl.vx v8, v8, a0
 ; RV64-NEXT:    ret
@@ -1875,9 +1827,8 @@ declare <vscale x 7 x i64> @llvm.vp.cttz.nxv7i64(<vscale x 7 x i64>, i1 immarg,
 define <vscale x 7 x i64> @vp_cttz_nxv7i64(<vscale x 7 x i64> %va, <vscale x 7 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_nxv7i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v16, v8, a1, v0.t
+; RV32-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
@@ -1917,39 +1868,38 @@ define <vscale x 7 x i64> @vp_cttz_nxv7i64(<vscale x 7 x i64> %va, <vscale x 7 x
 ;
 ; RV64-LABEL: vp_cttz_nxv7i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v16, v8, a1, v0.t
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v16, v16, a0, v0.t
+; RV64-NEXT:    vand.vx v16, v16, a1, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v16, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v16, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    li a0, 56
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a4, v0.t
 ; RV64-NEXT:    vsrl.vx v8, v8, a0, v0.t
 ; RV64-NEXT:    ret
 ;
@@ -1965,18 +1915,17 @@ define <vscale x 7 x i64> @vp_cttz_nxv7i64(<vscale x 7 x i64> %va, <vscale x 7 x
 define <vscale x 7 x i64> @vp_cttz_nxv7i64_unmasked(<vscale x 7 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_nxv7i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vnot.v v16, v8
-; RV32-NEXT:    vsub.vx v8, v8, a1
+; RV32-NEXT:    vadd.vi v16, v8, -1
+; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
 ; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    lui a1, 209715
-; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v8, v16, v8
+; RV32-NEXT:    vand.vv v8, v8, v16
+; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsrl.vi v16, v8, 1
 ; RV32-NEXT:    vand.vv v24, v16, v24
 ; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
@@ -2007,38 +1956,37 @@ define <vscale x 7 x i64> @vp_cttz_nxv7i64_unmasked(<vscale x 7 x i64> %va, i32
 ;
 ; RV64-LABEL: vp_cttz_nxv7i64_unmasked:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v16, v8, a1
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v16, v8, -1
 ; RV64-NEXT:    vnot.v v8, v8
 ; RV64-NEXT:    vand.vv v8, v8, v16
 ; RV64-NEXT:    vsrl.vi v16, v8, 1
-; RV64-NEXT:    vand.vx v16, v16, a0
+; RV64-NEXT:    vand.vx v16, v16, a1
 ; RV64-NEXT:    vsub.vv v8, v8, v16
-; RV64-NEXT:    vand.vx v16, v8, a1
+; RV64-NEXT:    vand.vx v16, v8, a2
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vand.vx v8, v8, a1
+; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    vadd.vv v8, v16, v8
 ; RV64-NEXT:    vsrl.vi v16, v8, 4
 ; RV64-NEXT:    vadd.vv v8, v8, v16
-; RV64-NEXT:    vand.vx v8, v8, a2
-; RV64-NEXT:    vmul.vx v8, v8, a3
+; RV64-NEXT:    vand.vx v8, v8, a3
+; RV64-NEXT:    vmul.vx v8, v8, a4
 ; RV64-NEXT:    li a0, 56
 ; RV64-NEXT:    vsrl.vx v8, v8, a0
 ; RV64-NEXT:    ret
@@ -2057,9 +2005,8 @@ declare <vscale x 8 x i64> @llvm.vp.cttz.nxv8i64(<vscale x 8 x i64>, i1 immarg,
 define <vscale x 8 x i64> @vp_cttz_nxv8i64(<vscale x 8 x i64> %va, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_nxv8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v16, v8, a1, v0.t
+; RV32-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
@@ -2099,39 +2046,38 @@ define <vscale x 8 x i64> @vp_cttz_nxv8i64(<vscale x 8 x i64> %va, <vscale x 8 x
 ;
 ; RV64-LABEL: vp_cttz_nxv8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v16, v8, a1, v0.t
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v16, v16, a0, v0.t
+; RV64-NEXT:    vand.vx v16, v16, a1, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v16, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v16, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    li a0, 56
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a4, v0.t
 ; RV64-NEXT:    vsrl.vx v8, v8, a0, v0.t
 ; RV64-NEXT:    ret
 ;
@@ -2147,18 +2093,17 @@ define <vscale x 8 x i64> @vp_cttz_nxv8i64(<vscale x 8 x i64> %va, <vscale x 8 x
 define <vscale x 8 x i64> @vp_cttz_nxv8i64_unmasked(<vscale x 8 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_nxv8i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vnot.v v16, v8
-; RV32-NEXT:    vsub.vx v8, v8, a1
+; RV32-NEXT:    vadd.vi v16, v8, -1
+; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
 ; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    lui a1, 209715
-; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v8, v16, v8
+; RV32-NEXT:    vand.vv v8, v8, v16
+; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsrl.vi v16, v8, 1
 ; RV32-NEXT:    vand.vv v24, v16, v24
 ; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
@@ -2189,38 +2134,37 @@ define <vscale x 8 x i64> @vp_cttz_nxv8i64_unmasked(<vscale x 8 x i64> %va, i32
 ;
 ; RV64-LABEL: vp_cttz_nxv8i64_unmasked:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v16, v8, a1
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v16, v8, -1
 ; RV64-NEXT:    vnot.v v8, v8
 ; RV64-NEXT:    vand.vv v8, v8, v16
 ; RV64-NEXT:    vsrl.vi v16, v8, 1
-; RV64-NEXT:    vand.vx v16, v16, a0
+; RV64-NEXT:    vand.vx v16, v16, a1
 ; RV64-NEXT:    vsub.vv v8, v8, v16
-; RV64-NEXT:    vand.vx v16, v8, a1
+; RV64-NEXT:    vand.vx v16, v8, a2
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vand.vx v8, v8, a1
+; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    vadd.vv v8, v16, v8
 ; RV64-NEXT:    vsrl.vi v16, v8, 4
 ; RV64-NEXT:    vadd.vv v8, v8, v16
-; RV64-NEXT:    vand.vx v8, v8, a2
-; RV64-NEXT:    vmul.vx v8, v8, a3
+; RV64-NEXT:    vand.vx v8, v8, a3
+; RV64-NEXT:    vmul.vx v8, v8, a4
 ; RV64-NEXT:    li a0, 56
 ; RV64-NEXT:    vsrl.vx v8, v8, a0
 ; RV64-NEXT:    ret
@@ -2254,138 +2198,131 @@ define <vscale x 16 x i64> @vp_cttz_nxv16i64(<vscale x 16 x i64> %va, <vscale x
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 1
-; RV32-NEXT:    srli a3, a1, 3
-; RV32-NEXT:    sub a4, a0, a1
-; RV32-NEXT:    vslidedown.vx v0, v0, a3
-; RV32-NEXT:    sltu a3, a0, a4
-; RV32-NEXT:    addi a3, a3, -1
-; RV32-NEXT:    and a3, a3, a4
-; RV32-NEXT:    lui a4, 349525
-; RV32-NEXT:    addi a4, a4, 1365
-; RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v8, v16, a2, v0.t
+; RV32-NEXT:    srli a2, a1, 3
+; RV32-NEXT:    sub a3, a0, a1
+; RV32-NEXT:    vslidedown.vx v0, v0, a2
+; RV32-NEXT:    sltu a2, a0, a3
+; RV32-NEXT:    addi a2, a2, -1
+; RV32-NEXT:    and a2, a2, a3
+; RV32-NEXT:    lui a3, 349525
+; RV32-NEXT:    addi a3, a3, 1365
+; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
+; RV32-NEXT:    vadd.vi v8, v16, -1, v0.t
 ; RV32-NEXT:    vnot.v v16, v16, v0.t
-; RV32-NEXT:    vand.vv v8, v16, v8, v0.t
-; RV32-NEXT:    csrr a5, vlenb
-; RV32-NEXT:    li a6, 48
-; RV32-NEXT:    mul a5, a5, a6
-; RV32-NEXT:    add a5, sp, a5
-; RV32-NEXT:    addi a5, a5, 16
-; RV32-NEXT:    vs8r.v v8, (a5) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetvli a5, zero, e32, m8, ta, ma
-; RV32-NEXT:    vmv.v.x v16, a4
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    li a5, 40
-; RV32-NEXT:    mul a4, a4, a5
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 16
-; RV32-NEXT:    vs8r.v v16, (a4) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    li a5, 48
-; RV32-NEXT:    mul a4, a4, a5
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 16
-; RV32-NEXT:    vl8r.v v8, (a4) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v8, 1, v0.t
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    li a5, 24
-; RV32-NEXT:    mul a4, a4, a5
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 16
-; RV32-NEXT:    vs8r.v v8, (a4) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    li a5, 40
-; RV32-NEXT:    mul a4, a4, a5
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 16
-; RV32-NEXT:    vl8r.v v16, (a4) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
 ; RV32-NEXT:    csrr a4, vlenb
 ; RV32-NEXT:    li a5, 24
 ; RV32-NEXT:    mul a4, a4, a5
 ; RV32-NEXT:    add a4, sp, a4
 ; RV32-NEXT:    addi a4, a4, 16
-; RV32-NEXT:    vl8r.v v8, (a4) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    li a5, 48
-; RV32-NEXT:    mul a4, a4, a5
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 16
-; RV32-NEXT:    vl8r.v v8, (a4) # Unknown-size Folded Reload
-; RV32-NEXT:    vsub.vv v16, v8, v16, v0.t
-; RV32-NEXT:    lui a4, 209715
-; RV32-NEXT:    addi a4, a4, 819
-; RV32-NEXT:    vsetvli a5, zero, e32, m8, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a4
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    li a5, 48
-; RV32-NEXT:    mul a4, a4, a5
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 16
-; RV32-NEXT:    vs8r.v v8, (a4) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    li a5, 48
-; RV32-NEXT:    mul a4, a4, a5
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 16
-; RV32-NEXT:    vl8r.v v8, (a4) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
+; RV32-NEXT:    vs8r.v v16, (a4) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a3
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    li a4, 40
+; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
+; RV32-NEXT:    vsrl.vi v8, v16, 1, v0.t
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    li a4, 48
+; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    li a4, 40
+; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    li a4, 48
+; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
 ; RV32-NEXT:    vand.vv v8, v16, v8, v0.t
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    li a5, 24
-; RV32-NEXT:    mul a4, a4, a5
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 16
-; RV32-NEXT:    vs8r.v v8, (a4) # Unknown-size Folded Spill
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    li a4, 24
+; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    vsub.vv v16, v16, v8, v0.t
+; RV32-NEXT:    lui a3, 209715
+; RV32-NEXT:    addi a3, a3, 819
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a3
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    li a4, 48
+; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    li a4, 48
+; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v8, v16, v8, v0.t
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    li a4, 24
+; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsrl.vi v16, v16, 2, v0.t
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    li a5, 48
-; RV32-NEXT:    mul a4, a4, a5
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 16
-; RV32-NEXT:    vl8r.v v8, (a4) # Unknown-size Folded Reload
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    li a4, 48
+; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
 ; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    li a5, 24
-; RV32-NEXT:    mul a4, a4, a5
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 16
-; RV32-NEXT:    vl8r.v v8, (a4) # Unknown-size Folded Reload
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    li a4, 24
+; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
 ; RV32-NEXT:    vadd.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV32-NEXT:    vadd.vv v8, v8, v16, v0.t
-; RV32-NEXT:    lui a4, 61681
-; RV32-NEXT:    addi a4, a4, -241
-; RV32-NEXT:    vsetvli a5, zero, e32, m8, ta, ma
-; RV32-NEXT:    vmv.v.x v16, a4
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    li a5, 24
-; RV32-NEXT:    mul a4, a4, a5
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 16
-; RV32-NEXT:    vs8r.v v16, (a4) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
+; RV32-NEXT:    lui a3, 61681
+; RV32-NEXT:    addi a3, a3, -241
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a3
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    li a4, 24
+; RV32-NEXT:    mul a3, a3, a4
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
-; RV32-NEXT:    lui a4, 4112
-; RV32-NEXT:    addi a4, a4, 257
-; RV32-NEXT:    vsetvli a5, zero, e32, m8, ta, ma
-; RV32-NEXT:    vmv.v.x v16, a4
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    slli a4, a4, 4
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 16
-; RV32-NEXT:    vs8r.v v16, (a4) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
+; RV32-NEXT:    lui a3, 4112
+; RV32-NEXT:    addi a3, a3, 257
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a3
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    slli a3, a3, 4
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
-; RV32-NEXT:    li a3, 56
-; RV32-NEXT:    vsrl.vx v8, v8, a3, v0.t
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    slli a4, a4, 3
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 16
-; RV32-NEXT:    vs8r.v v8, (a4) # Unknown-size Folded Spill
+; RV32-NEXT:    li a2, 56
+; RV32-NEXT:    vsrl.vx v8, v8, a2, v0.t
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    slli a3, a3, 3
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    bltu a0, a1, .LBB46_2
 ; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    mv a0, a1
@@ -2394,11 +2331,11 @@ define <vscale x 16 x i64> @vp_cttz_nxv16i64(<vscale x 16 x i64> %va, <vscale x
 ; RV32-NEXT:    slli a1, a1, 5
 ; RV32-NEXT:    add a1, sp, a1
 ; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v16, v8, a2, v0.t
-; RV32-NEXT:    vnot.v v8, v8, v0.t
-; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    vadd.vi v8, v16, -1, v0.t
+; RV32-NEXT:    vnot.v v16, v16, v0.t
+; RV32-NEXT:    vand.vv v8, v16, v8, v0.t
 ; RV32-NEXT:    addi a0, sp, 16
 ; RV32-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsrl.vi v8, v8, 1, v0.t
@@ -2476,7 +2413,7 @@ define <vscale x 16 x i64> @vp_cttz_nxv16i64(<vscale x 16 x i64> %va, <vscale x
 ; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vx v8, v8, a3, v0.t
+; RV32-NEXT:    vsrl.vx v8, v8, a2, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    slli a0, a0, 3
 ; RV32-NEXT:    add a0, sp, a0
@@ -2507,48 +2444,47 @@ define <vscale x 16 x i64> @vp_cttz_nxv16i64(<vscale x 16 x i64> %va, <vscale x
 ; RV64-NEXT:    addi a1, a1, 16
 ; RV64-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 1
-; RV64-NEXT:    lui a3, 349525
-; RV64-NEXT:    lui a4, 209715
-; RV64-NEXT:    lui a5, 61681
-; RV64-NEXT:    lui a6, 4112
-; RV64-NEXT:    srli a7, a1, 3
-; RV64-NEXT:    sub t0, a0, a1
-; RV64-NEXT:    addiw a3, a3, 1365
-; RV64-NEXT:    addiw a4, a4, 819
-; RV64-NEXT:    addiw a5, a5, -241
-; RV64-NEXT:    addiw t1, a6, 257
-; RV64-NEXT:    vslidedown.vx v0, v0, a7
-; RV64-NEXT:    slli a7, a3, 32
-; RV64-NEXT:    add a7, a3, a7
-; RV64-NEXT:    slli a6, a4, 32
-; RV64-NEXT:    add a6, a4, a6
-; RV64-NEXT:    slli a3, a5, 32
-; RV64-NEXT:    add a3, a5, a3
-; RV64-NEXT:    slli a4, t1, 32
-; RV64-NEXT:    add a4, t1, a4
-; RV64-NEXT:    sltu a5, a0, t0
-; RV64-NEXT:    addi a5, a5, -1
-; RV64-NEXT:    and t0, a5, t0
-; RV64-NEXT:    li a5, 56
-; RV64-NEXT:    vsetvli zero, t0, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v8, v16, a2, v0.t
+; RV64-NEXT:    lui a2, 349525
+; RV64-NEXT:    lui a3, 209715
+; RV64-NEXT:    lui a4, 61681
+; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    srli a6, a1, 3
+; RV64-NEXT:    sub a7, a0, a1
+; RV64-NEXT:    addiw a2, a2, 1365
+; RV64-NEXT:    addiw a3, a3, 819
+; RV64-NEXT:    addiw a4, a4, -241
+; RV64-NEXT:    addiw t0, a5, 257
+; RV64-NEXT:    vslidedown.vx v0, v0, a6
+; RV64-NEXT:    slli a6, a2, 32
+; RV64-NEXT:    add a6, a2, a6
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a5, a3, a5
+; RV64-NEXT:    slli a2, a4, 32
+; RV64-NEXT:    add a2, a4, a2
+; RV64-NEXT:    slli a3, t0, 32
+; RV64-NEXT:    add a3, t0, a3
+; RV64-NEXT:    sltu a4, a0, a7
+; RV64-NEXT:    addi a4, a4, -1
+; RV64-NEXT:    and a7, a4, a7
+; RV64-NEXT:    li a4, 56
+; RV64-NEXT:    vsetvli zero, a7, e64, m8, ta, ma
+; RV64-NEXT:    vadd.vi v8, v16, -1, v0.t
 ; RV64-NEXT:    vnot.v v16, v16, v0.t
 ; RV64-NEXT:    vand.vv v8, v16, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v16, v16, a7, v0.t
+; RV64-NEXT:    vand.vx v16, v16, a6, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v16, v8, a6, v0.t
+; RV64-NEXT:    vand.vx v16, v8, a5, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a6, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a5, v0.t
 ; RV64-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
-; RV64-NEXT:    vmul.vx v8, v8, a4, v0.t
-; RV64-NEXT:    vsrl.vx v8, v8, a5, v0.t
-; RV64-NEXT:    addi t0, sp, 16
-; RV64-NEXT:    vs8r.v v8, (t0) # Unknown-size Folded Spill
+; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vsrl.vx v8, v8, a4, v0.t
+; RV64-NEXT:    addi a7, sp, 16
+; RV64-NEXT:    vs8r.v v8, (a7) # Unknown-size Folded Spill
 ; RV64-NEXT:    bltu a0, a1, .LBB46_2
 ; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    mv a0, a1
@@ -2559,21 +2495,21 @@ define <vscale x 16 x i64> @vp_cttz_nxv16i64(<vscale x 16 x i64> %va, <vscale x
 ; RV64-NEXT:    addi a1, a1, 16
 ; RV64-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v16, v8, a2, v0.t
+; RV64-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v16, v16, a7, v0.t
+; RV64-NEXT:    vand.vx v16, v16, a6, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v16, v8, a6, v0.t
+; RV64-NEXT:    vand.vx v16, v8, a5, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a6, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a5, v0.t
 ; RV64-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
-; RV64-NEXT:    vmul.vx v8, v8, a4, v0.t
-; RV64-NEXT:    vsrl.vx v8, v8, a5, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vsrl.vx v8, v8, a4, v0.t
 ; RV64-NEXT:    addi a0, sp, 16
 ; RV64-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV64-NEXT:    csrr a0, vlenb
@@ -2619,71 +2555,70 @@ define <vscale x 16 x i64> @vp_cttz_nxv16i64_unmasked(<vscale x 16 x i64> %va, i
 ; RV32-NEXT:    sub sp, sp, a1
 ; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 1
-; RV32-NEXT:    lui a3, 349525
-; RV32-NEXT:    lui a4, 209715
-; RV32-NEXT:    sub a5, a0, a1
-; RV32-NEXT:    addi a3, a3, 1365
-; RV32-NEXT:    addi a4, a4, 819
-; RV32-NEXT:    vsetvli a6, zero, e32, m8, ta, ma
-; RV32-NEXT:    vmv.v.x v0, a3
-; RV32-NEXT:    sltu a3, a0, a5
-; RV32-NEXT:    addi a3, a3, -1
-; RV32-NEXT:    and a3, a3, a5
-; RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v24, v16, a2
+; RV32-NEXT:    lui a2, 349525
+; RV32-NEXT:    lui a3, 209715
+; RV32-NEXT:    sub a4, a0, a1
+; RV32-NEXT:    addi a2, a2, 1365
+; RV32-NEXT:    addi a3, a3, 819
+; RV32-NEXT:    vsetvli a5, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v0, a2
+; RV32-NEXT:    sltu a2, a0, a4
+; RV32-NEXT:    addi a2, a2, -1
+; RV32-NEXT:    and a2, a2, a4
+; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
+; RV32-NEXT:    vadd.vi v24, v16, -1
 ; RV32-NEXT:    vnot.v v16, v16
 ; RV32-NEXT:    vand.vv v16, v16, v24
 ; RV32-NEXT:    vsrl.vi v24, v16, 1
-; RV32-NEXT:    csrr a5, vlenb
-; RV32-NEXT:    li a6, 24
-; RV32-NEXT:    mul a5, a5, a6
-; RV32-NEXT:    add a5, sp, a5
-; RV32-NEXT:    addi a5, a5, 16
-; RV32-NEXT:    vs8r.v v0, (a5) # Unknown-size Folded Spill
-; RV32-NEXT:    vand.vv v24, v24, v0
-; RV32-NEXT:    vsub.vv v16, v16, v24
-; RV32-NEXT:    vsetvli a5, zero, e32, m8, ta, ma
-; RV32-NEXT:    vmv.v.x v0, a4
-; RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v24, v16, v0
-; RV32-NEXT:    vsrl.vi v16, v16, 2
 ; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    slli a4, a4, 4
+; RV32-NEXT:    li a5, 24
+; RV32-NEXT:    mul a4, a4, a5
 ; RV32-NEXT:    add a4, sp, a4
 ; RV32-NEXT:    addi a4, a4, 16
 ; RV32-NEXT:    vs8r.v v0, (a4) # Unknown-size Folded Spill
+; RV32-NEXT:    vand.vv v24, v24, v0
+; RV32-NEXT:    vsub.vv v16, v16, v24
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v0, a3
+; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v24, v16, v0
+; RV32-NEXT:    vsrl.vi v16, v16, 2
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    slli a3, a3, 4
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v0, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    vand.vv v16, v16, v0
 ; RV32-NEXT:    vadd.vv v16, v24, v16
 ; RV32-NEXT:    vsrl.vi v24, v16, 4
 ; RV32-NEXT:    vadd.vv v16, v16, v24
-; RV32-NEXT:    lui a4, 61681
-; RV32-NEXT:    lui a5, 4112
-; RV32-NEXT:    addi a4, a4, -241
-; RV32-NEXT:    addi a5, a5, 257
-; RV32-NEXT:    vsetvli a6, zero, e32, m8, ta, ma
-; RV32-NEXT:    vmv.v.x v24, a4
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    slli a4, a4, 3
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 16
-; RV32-NEXT:    vs8r.v v24, (a4) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
+; RV32-NEXT:    lui a3, 61681
+; RV32-NEXT:    lui a4, 4112
+; RV32-NEXT:    addi a3, a3, -241
+; RV32-NEXT:    addi a4, a4, 257
+; RV32-NEXT:    vsetvli a5, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a3
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    slli a3, a3, 3
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v24, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vand.vv v16, v16, v24
-; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
-; RV32-NEXT:    vmv.v.x v24, a5
-; RV32-NEXT:    addi a4, sp, 16
-; RV32-NEXT:    vs8r.v v24, (a4) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a4
+; RV32-NEXT:    addi a3, sp, 16
+; RV32-NEXT:    vs8r.v v24, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vmul.vv v16, v16, v24
-; RV32-NEXT:    li a3, 56
-; RV32-NEXT:    vsrl.vx v16, v16, a3
+; RV32-NEXT:    li a2, 56
+; RV32-NEXT:    vsrl.vx v16, v16, a2
 ; RV32-NEXT:    bltu a0, a1, .LBB47_2
 ; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    mv a0, a1
 ; RV32-NEXT:  .LBB47_2:
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v24, v8, a2
+; RV32-NEXT:    vadd.vi v24, v8, -1
 ; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    vand.vv v8, v8, v24
 ; RV32-NEXT:    vsrl.vi v24, v8, 1
@@ -2715,7 +2650,7 @@ define <vscale x 16 x i64> @vp_cttz_nxv16i64_unmasked(<vscale x 16 x i64> %va, i
 ; RV32-NEXT:    addi a0, sp, 16
 ; RV32-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vmul.vv v8, v8, v24
-; RV32-NEXT:    vsrl.vx v8, v8, a3
+; RV32-NEXT:    vsrl.vx v8, v8, a2
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    slli a0, a0, 5
 ; RV32-NEXT:    add sp, sp, a0
@@ -2727,64 +2662,63 @@ define <vscale x 16 x i64> @vp_cttz_nxv16i64_unmasked(<vscale x 16 x i64> %va, i
 ; RV64-LABEL: vp_cttz_nxv16i64_unmasked:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    li a2, 1
-; RV64-NEXT:    lui a3, 349525
-; RV64-NEXT:    lui a4, 209715
-; RV64-NEXT:    lui a5, 61681
-; RV64-NEXT:    lui a6, 4112
-; RV64-NEXT:    sub a7, a0, a1
-; RV64-NEXT:    addiw a3, a3, 1365
-; RV64-NEXT:    addiw a4, a4, 819
-; RV64-NEXT:    addiw t0, a5, -241
-; RV64-NEXT:    addiw t1, a6, 257
-; RV64-NEXT:    slli a6, a3, 32
-; RV64-NEXT:    add a6, a3, a6
-; RV64-NEXT:    slli a5, a4, 32
-; RV64-NEXT:    add a5, a4, a5
+; RV64-NEXT:    lui a2, 349525
+; RV64-NEXT:    lui a3, 209715
+; RV64-NEXT:    lui a4, 61681
+; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    sub a6, a0, a1
+; RV64-NEXT:    addiw a2, a2, 1365
+; RV64-NEXT:    addiw a3, a3, 819
+; RV64-NEXT:    addiw a7, a4, -241
+; RV64-NEXT:    addiw t0, a5, 257
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a5, a2, a5
+; RV64-NEXT:    slli a4, a3, 32
+; RV64-NEXT:    add a4, a3, a4
+; RV64-NEXT:    slli a2, a7, 32
+; RV64-NEXT:    add a2, a7, a2
 ; RV64-NEXT:    slli a3, t0, 32
 ; RV64-NEXT:    add a3, t0, a3
-; RV64-NEXT:    slli a4, t1, 32
-; RV64-NEXT:    add a4, t1, a4
-; RV64-NEXT:    sltu t0, a0, a7
-; RV64-NEXT:    addi t0, t0, -1
-; RV64-NEXT:    and a7, t0, a7
-; RV64-NEXT:    vsetvli zero, a7, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v24, v16, a2
+; RV64-NEXT:    sltu a7, a0, a6
+; RV64-NEXT:    addi a7, a7, -1
+; RV64-NEXT:    and a6, a7, a6
+; RV64-NEXT:    vsetvli zero, a6, e64, m8, ta, ma
+; RV64-NEXT:    vadd.vi v24, v16, -1
 ; RV64-NEXT:    vnot.v v16, v16
 ; RV64-NEXT:    vand.vv v16, v16, v24
 ; RV64-NEXT:    vsrl.vi v24, v16, 1
-; RV64-NEXT:    vand.vx v24, v24, a6
+; RV64-NEXT:    vand.vx v24, v24, a5
 ; RV64-NEXT:    vsub.vv v16, v16, v24
-; RV64-NEXT:    vand.vx v24, v16, a5
+; RV64-NEXT:    vand.vx v24, v16, a4
 ; RV64-NEXT:    vsrl.vi v16, v16, 2
-; RV64-NEXT:    vand.vx v16, v16, a5
+; RV64-NEXT:    vand.vx v16, v16, a4
 ; RV64-NEXT:    vadd.vv v16, v24, v16
 ; RV64-NEXT:    vsrl.vi v24, v16, 4
 ; RV64-NEXT:    vadd.vv v16, v16, v24
-; RV64-NEXT:    vand.vx v16, v16, a3
-; RV64-NEXT:    vmul.vx v16, v16, a4
-; RV64-NEXT:    li a7, 56
-; RV64-NEXT:    vsrl.vx v16, v16, a7
+; RV64-NEXT:    vand.vx v16, v16, a2
+; RV64-NEXT:    vmul.vx v16, v16, a3
+; RV64-NEXT:    li a6, 56
+; RV64-NEXT:    vsrl.vx v16, v16, a6
 ; RV64-NEXT:    bltu a0, a1, .LBB47_2
 ; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    mv a0, a1
 ; RV64-NEXT:  .LBB47_2:
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v24, v8, a2
+; RV64-NEXT:    vadd.vi v24, v8, -1
 ; RV64-NEXT:    vnot.v v8, v8
 ; RV64-NEXT:    vand.vv v8, v8, v24
 ; RV64-NEXT:    vsrl.vi v24, v8, 1
-; RV64-NEXT:    vand.vx v24, v24, a6
+; RV64-NEXT:    vand.vx v24, v24, a5
 ; RV64-NEXT:    vsub.vv v8, v8, v24
-; RV64-NEXT:    vand.vx v24, v8, a5
+; RV64-NEXT:    vand.vx v24, v8, a4
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vand.vx v8, v8, a5
+; RV64-NEXT:    vand.vx v8, v8, a4
 ; RV64-NEXT:    vadd.vv v8, v24, v8
 ; RV64-NEXT:    vsrl.vi v24, v8, 4
 ; RV64-NEXT:    vadd.vv v8, v8, v24
-; RV64-NEXT:    vand.vx v8, v8, a3
-; RV64-NEXT:    vmul.vx v8, v8, a4
-; RV64-NEXT:    vsrl.vx v8, v8, a7
+; RV64-NEXT:    vand.vx v8, v8, a2
+; RV64-NEXT:    vmul.vx v8, v8, a3
+; RV64-NEXT:    vsrl.vx v8, v8, a6
 ; RV64-NEXT:    ret
 ;
 ; CHECK-ZVBB-LABEL: vp_cttz_nxv16i64_unmasked:
@@ -3080,9 +3014,8 @@ define <vscale x 16 x i8> @vp_cttz_zero_undef_nxv16i8_unmasked(<vscale x 16 x i8
 define <vscale x 32 x i8> @vp_cttz_zero_undef_nxv32i8(<vscale x 32 x i8> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_nxv32i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; CHECK-NEXT:    vsub.vx v12, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v12, v8, -1, v0.t
 ; CHECK-NEXT:    li a0, 85
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    vand.vv v8, v8, v12, v0.t
@@ -3111,12 +3044,11 @@ define <vscale x 32 x i8> @vp_cttz_zero_undef_nxv32i8(<vscale x 32 x i8> %va, <v
 define <vscale x 32 x i8> @vp_cttz_zero_undef_nxv32i8_unmasked(<vscale x 32 x i8> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_nxv32i8_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; CHECK-NEXT:    vnot.v v12, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v12, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v12, v8
+; CHECK-NEXT:    vand.vv v8, v8, v12
 ; CHECK-NEXT:    vsrl.vi v12, v8, 1
 ; CHECK-NEXT:    vand.vx v12, v12, a0
 ; CHECK-NEXT:    li a0, 51
@@ -3143,9 +3075,8 @@ define <vscale x 32 x i8> @vp_cttz_zero_undef_nxv32i8_unmasked(<vscale x 32 x i8
 define <vscale x 64 x i8> @vp_cttz_zero_undef_nxv64i8(<vscale x 64 x i8> %va, <vscale x 64 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_nxv64i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
-; CHECK-NEXT:    vsub.vx v16, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; CHECK-NEXT:    li a0, 85
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    vand.vv v8, v8, v16, v0.t
@@ -3174,12 +3105,11 @@ define <vscale x 64 x i8> @vp_cttz_zero_undef_nxv64i8(<vscale x 64 x i8> %va, <v
 define <vscale x 64 x i8> @vp_cttz_zero_undef_nxv64i8_unmasked(<vscale x 64 x i8> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_nxv64i8_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
-; CHECK-NEXT:    vnot.v v16, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v16, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v16, v8
+; CHECK-NEXT:    vand.vv v8, v8, v16
 ; CHECK-NEXT:    vsrl.vi v16, v8, 1
 ; CHECK-NEXT:    vand.vx v16, v16, a0
 ; CHECK-NEXT:    li a0, 51
@@ -3436,9 +3366,8 @@ define <vscale x 16 x i16> @vp_cttz_zero_undef_nxv16i16_unmasked(<vscale x 16 x
 define <vscale x 32 x i16> @vp_cttz_zero_undef_nxv32i16(<vscale x 32 x i16> %va, <vscale x 32 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_nxv32i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    vsub.vx v16, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 5
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -3474,13 +3403,12 @@ define <vscale x 32 x i16> @vp_cttz_zero_undef_nxv32i16(<vscale x 32 x i16> %va,
 define <vscale x 32 x i16> @vp_cttz_zero_undef_nxv32i16_unmasked(<vscale x 32 x i16> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_nxv32i16_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; CHECK-NEXT:    vnot.v v16, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v16, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    vand.vv v8, v8, v16
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v16, v8
 ; CHECK-NEXT:    vsrl.vi v16, v8, 1
 ; CHECK-NEXT:    vand.vx v16, v16, a0
 ; CHECK-NEXT:    lui a0, 3
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse-vp.ll
index 5ea4924468595..a72e21c55fa95 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitreverse-vp.ll
@@ -1652,116 +1652,106 @@ declare <15 x i64> @llvm.vp.bitreverse.v15i64(<15 x i64>, <15 x i1>, i32)
 define <15 x i64> @vp_bitreverse_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_bitreverse_v15i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    li a2, 24
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 24 * vlenb
+; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb
+; RV32-NEXT:    lui a1, 1044480
+; RV32-NEXT:    li a2, 56
+; RV32-NEXT:    lui a3, 16
+; RV32-NEXT:    li a4, 40
+; RV32-NEXT:    addi a5, sp, 8
+; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vmv8r.v v24, v8
-; RV32-NEXT:    lui a2, 1044480
-; RV32-NEXT:    lui a3, 61681
-; RV32-NEXT:    lui a4, 209715
-; RV32-NEXT:    lui a5, 349525
-; RV32-NEXT:    li a1, 56
-; RV32-NEXT:    lui a6, 16
-; RV32-NEXT:    sw a2, 16(sp)
-; RV32-NEXT:    sw zero, 20(sp)
-; RV32-NEXT:    addi a2, a3, -241
-; RV32-NEXT:    sw a2, 40(sp)
-; RV32-NEXT:    sw a2, 44(sp)
-; RV32-NEXT:    li a2, 40
-; RV32-NEXT:    addi a3, a4, 819
-; RV32-NEXT:    sw a3, 32(sp)
-; RV32-NEXT:    sw a3, 36(sp)
-; RV32-NEXT:    addi a3, sp, 16
-; RV32-NEXT:    addi a4, a5, 1365
-; RV32-NEXT:    vsll.vx v16, v8, a1, v0.t
-; RV32-NEXT:    addi a5, a6, -256
-; RV32-NEXT:    sw a4, 24(sp)
-; RV32-NEXT:    sw a4, 28(sp)
-; RV32-NEXT:    vand.vx v8, v8, a5, v0.t
-; RV32-NEXT:    vsll.vx v8, v8, a2, v0.t
-; RV32-NEXT:    vor.vv v8, v16, v8, v0.t
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    slli a4, a4, 4
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 48
-; RV32-NEXT:    vs8r.v v8, (a4) # Unknown-size Folded Spill
+; RV32-NEXT:    vsll.vx v16, v8, a2, v0.t
+; RV32-NEXT:    addi a1, a3, -256
+; RV32-NEXT:    vand.vx v24, v8, a1, v0.t
+; RV32-NEXT:    vsll.vx v24, v24, a4, v0.t
+; RV32-NEXT:    vor.vv v16, v16, v24, v0.t
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    slli a3, a3, 4
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a3), zero
+; RV32-NEXT:    vlse64.v v16, (a5), zero
 ; RV32-NEXT:    csrr a3, vlenb
 ; RV32-NEXT:    slli a3, a3, 3
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    lui a3, 4080
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vx v16, v24, a3, v0.t
-; RV32-NEXT:    vsll.vi v16, v16, 24, v0.t
-; RV32-NEXT:    addi a4, sp, 48
-; RV32-NEXT:    vs8r.v v16, (a4) # Unknown-size Folded Spill
-; RV32-NEXT:    vand.vv v16, v24, v8, v0.t
-; RV32-NEXT:    vsll.vi v16, v16, 8, v0.t
-; RV32-NEXT:    vl8r.v v8, (a4) # Unknown-size Folded Reload
-; RV32-NEXT:    vor.vv v16, v8, v16, v0.t
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    slli a4, a4, 4
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 48
-; RV32-NEXT:    vl8r.v v8, (a4) # Unknown-size Folded Reload
-; RV32-NEXT:    vor.vv v16, v8, v16, v0.t
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    slli a4, a4, 4
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 48
-; RV32-NEXT:    vs8r.v v16, (a4) # Unknown-size Folded Spill
-; RV32-NEXT:    vsrl.vx v16, v24, a1, v0.t
-; RV32-NEXT:    vsrl.vx v8, v24, a2, v0.t
-; RV32-NEXT:    vand.vx v8, v8, a5, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    addi a1, sp, 48
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    vsrl.vi v8, v24, 24, v0.t
-; RV32-NEXT:    vand.vx v16, v8, a3, v0.t
-; RV32-NEXT:    vsrl.vi v8, v24, 8, v0.t
+; RV32-NEXT:    vand.vx v24, v8, a3, v0.t
+; RV32-NEXT:    vsll.vi v24, v24, 24, v0.t
+; RV32-NEXT:    addi a5, sp, 16
+; RV32-NEXT:    vs8r.v v24, (a5) # Unknown-size Folded Spill
+; RV32-NEXT:    vand.vv v24, v8, v16, v0.t
+; RV32-NEXT:    vsll.vi v16, v24, 8, v0.t
+; RV32-NEXT:    vl8r.v v24, (a5) # Unknown-size Folded Reload
+; RV32-NEXT:    vor.vv v16, v24, v16, v0.t
+; RV32-NEXT:    csrr a5, vlenb
+; RV32-NEXT:    slli a5, a5, 4
+; RV32-NEXT:    add a5, sp, a5
+; RV32-NEXT:    addi a5, a5, 16
+; RV32-NEXT:    vl8r.v v24, (a5) # Unknown-size Folded Reload
+; RV32-NEXT:    vor.vv v16, v24, v16, v0.t
+; RV32-NEXT:    csrr a5, vlenb
+; RV32-NEXT:    slli a5, a5, 4
+; RV32-NEXT:    add a5, sp, a5
+; RV32-NEXT:    addi a5, a5, 16
+; RV32-NEXT:    vs8r.v v16, (a5) # Unknown-size Folded Spill
+; RV32-NEXT:    vsrl.vx v16, v8, a2, v0.t
+; RV32-NEXT:    vsrl.vx v24, v8, a4, v0.t
+; RV32-NEXT:    vand.vx v24, v24, a1, v0.t
+; RV32-NEXT:    vor.vv v16, v24, v16, v0.t
+; RV32-NEXT:    addi a1, sp, 16
+; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    vsrl.vi v24, v8, 24, v0.t
+; RV32-NEXT:    vand.vx v24, v24, a3, v0.t
+; RV32-NEXT:    vsrl.vi v8, v8, 8, v0.t
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    slli a1, a1, 3
 ; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    addi a1, sp, 48
+; RV32-NEXT:    addi a1, a1, 16
+; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    addi a1, sp, 16
 ; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    addi a1, sp, 40
-; RV32-NEXT:    addi a2, sp, 32
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    lui a1, 61681
+; RV32-NEXT:    lui a2, 209715
+; RV32-NEXT:    lui a3, 349525
+; RV32-NEXT:    addi a1, a1, -241
+; RV32-NEXT:    addi a2, a2, 819
+; RV32-NEXT:    addi a3, a3, 1365
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    slli a1, a1, 4
+; RV32-NEXT:    add a1, sp, a1
+; RV32-NEXT:    addi a1, a1, 16
+; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vor.vv v8, v16, v8, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV32-NEXT:    vand.vv v16, v16, v24, v0.t
 ; RV32-NEXT:    vand.vv v24, v8, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a2), zero
+; RV32-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a2
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vi v24, v24, 4, v0.t
 ; RV32-NEXT:    vor.vv v24, v16, v24, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v24, 2, v0.t
 ; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
 ; RV32-NEXT:    vand.vv v24, v24, v8, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
+; RV32-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a3
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vi v24, v24, 2, v0.t
 ; RV32-NEXT:    vor.vv v16, v16, v24, v0.t
@@ -1774,8 +1764,8 @@ define <15 x i64> @vp_bitreverse_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroex
 ; RV32-NEXT:    li a1, 24
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
+; RV32-NEXT:    .cfi_def_cfa sp, 16
+; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
@@ -1863,91 +1853,82 @@ define <15 x i64> @vp_bitreverse_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroex
 define <15 x i64> @vp_bitreverse_v15i64_unmasked(<15 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_bitreverse_v15i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    slli a1, a1, 4
 ; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
+; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
 ; RV32-NEXT:    lui a1, 1044480
-; RV32-NEXT:    lui a2, 61681
-; RV32-NEXT:    lui a3, 209715
-; RV32-NEXT:    lui a4, 349525
-; RV32-NEXT:    li a5, 56
-; RV32-NEXT:    lui a6, 16
+; RV32-NEXT:    li a2, 56
+; RV32-NEXT:    lui a3, 16
+; RV32-NEXT:    li a4, 40
+; RV32-NEXT:    lui a5, 4080
+; RV32-NEXT:    addi a6, sp, 8
+; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsll.vx v16, v8, a5
-; RV32-NEXT:    vsrl.vx v24, v8, a5
-; RV32-NEXT:    li a5, 40
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw zero, 20(sp)
-; RV32-NEXT:    addi a1, a2, -241
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
-; RV32-NEXT:    lui a1, 4080
-; RV32-NEXT:    addi a2, a3, 819
-; RV32-NEXT:    sw a2, 32(sp)
-; RV32-NEXT:    sw a2, 36(sp)
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    addi a3, a4, 1365
-; RV32-NEXT:    addi a4, a6, -256
-; RV32-NEXT:    vsrl.vx v0, v8, a5
-; RV32-NEXT:    sw a3, 24(sp)
-; RV32-NEXT:    sw a3, 28(sp)
-; RV32-NEXT:    vand.vx v0, v0, a4
+; RV32-NEXT:    vsll.vx v16, v8, a2
+; RV32-NEXT:    addi a1, a3, -256
+; RV32-NEXT:    vsrl.vx v24, v8, a2
+; RV32-NEXT:    vsrl.vx v0, v8, a4
+; RV32-NEXT:    vand.vx v0, v0, a1
 ; RV32-NEXT:    vor.vv v24, v0, v24
-; RV32-NEXT:    addi a3, sp, 48
-; RV32-NEXT:    vs8r.v v24, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    vand.vx v0, v8, a4
-; RV32-NEXT:    vsll.vx v0, v0, a5
-; RV32-NEXT:    vor.vv v16, v16, v0
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vs8r.v v24, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    vand.vx v24, v8, a1
+; RV32-NEXT:    vsll.vx v24, v24, a4
+; RV32-NEXT:    vor.vv v16, v16, v24
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    slli a1, a1, 3
+; RV32-NEXT:    add a1, sp, a1
+; RV32-NEXT:    addi a1, a1, 16
+; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v0, (a2), zero
+; RV32-NEXT:    vlse64.v v24, (a6), zero
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsrl.vi v16, v8, 24
-; RV32-NEXT:    vand.vx v16, v16, a1
-; RV32-NEXT:    vsrl.vi v24, v8, 8
-; RV32-NEXT:    vand.vv v24, v24, v0
-; RV32-NEXT:    vor.vv v16, v24, v16
-; RV32-NEXT:    vand.vv v24, v8, v0
-; RV32-NEXT:    vand.vx v8, v8, a1
+; RV32-NEXT:    vand.vx v16, v16, a5
+; RV32-NEXT:    vsrl.vi v0, v8, 8
+; RV32-NEXT:    vand.vv v0, v0, v24
+; RV32-NEXT:    vor.vv v16, v0, v16
+; RV32-NEXT:    vand.vv v24, v8, v24
+; RV32-NEXT:    vand.vx v8, v8, a5
 ; RV32-NEXT:    vsll.vi v8, v8, 24
 ; RV32-NEXT:    vsll.vi v24, v24, 8
-; RV32-NEXT:    vor.vv v0, v8, v24
-; RV32-NEXT:    addi a1, sp, 48
+; RV32-NEXT:    vor.vv v24, v8, v24
+; RV32-NEXT:    addi a1, sp, 16
 ; RV32-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vor.vv v8, v16, v8
-; RV32-NEXT:    addi a1, sp, 40
-; RV32-NEXT:    addi a2, sp, 32
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vor.vv v24, v16, v0
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a1), zero
-; RV32-NEXT:    addi a1, sp, 24
+; RV32-NEXT:    lui a1, 61681
+; RV32-NEXT:    lui a2, 209715
+; RV32-NEXT:    lui a3, 349525
+; RV32-NEXT:    addi a1, a1, -241
+; RV32-NEXT:    addi a2, a2, 819
+; RV32-NEXT:    addi a3, a3, 1365
+; RV32-NEXT:    csrr a4, vlenb
+; RV32-NEXT:    slli a4, a4, 3
+; RV32-NEXT:    add a4, sp, a4
+; RV32-NEXT:    addi a4, a4, 16
+; RV32-NEXT:    vl8r.v v16, (a4) # Unknown-size Folded Reload
+; RV32-NEXT:    vor.vv v16, v16, v24
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vor.vv v8, v24, v8
-; RV32-NEXT:    vsrl.vi v24, v8, 4
-; RV32-NEXT:    vand.vv v8, v8, v16
-; RV32-NEXT:    vand.vv v16, v24, v16
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a2), zero
+; RV32-NEXT:    vor.vv v8, v16, v8
+; RV32-NEXT:    vsrl.vi v16, v8, 4
+; RV32-NEXT:    vand.vv v8, v8, v24
+; RV32-NEXT:    vand.vv v16, v16, v24
+; RV32-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a2
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vi v8, v8, 4
 ; RV32-NEXT:    vor.vv v8, v16, v8
 ; RV32-NEXT:    vsrl.vi v16, v8, 2
 ; RV32-NEXT:    vand.vv v8, v8, v24
 ; RV32-NEXT:    vand.vv v16, v16, v24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
+; RV32-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a3
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vi v8, v8, 2
 ; RV32-NEXT:    vor.vv v8, v16, v8
@@ -1959,8 +1940,8 @@ define <15 x i64> @vp_bitreverse_v15i64_unmasked(<15 x i64> %va, i32 zeroext %ev
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    slli a0, a0, 4
 ; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
+; RV32-NEXT:    .cfi_def_cfa sp, 16
+; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
@@ -2048,116 +2029,106 @@ declare <16 x i64> @llvm.vp.bitreverse.v16i64(<16 x i64>, <16 x i1>, i32)
 define <16 x i64> @vp_bitreverse_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_bitreverse_v16i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    li a2, 24
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 24 * vlenb
+; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb
+; RV32-NEXT:    lui a1, 1044480
+; RV32-NEXT:    li a2, 56
+; RV32-NEXT:    lui a3, 16
+; RV32-NEXT:    li a4, 40
+; RV32-NEXT:    addi a5, sp, 8
+; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vmv8r.v v24, v8
-; RV32-NEXT:    lui a2, 1044480
-; RV32-NEXT:    lui a3, 61681
-; RV32-NEXT:    lui a4, 209715
-; RV32-NEXT:    lui a5, 349525
-; RV32-NEXT:    li a1, 56
-; RV32-NEXT:    lui a6, 16
-; RV32-NEXT:    sw a2, 16(sp)
-; RV32-NEXT:    sw zero, 20(sp)
-; RV32-NEXT:    addi a2, a3, -241
-; RV32-NEXT:    sw a2, 40(sp)
-; RV32-NEXT:    sw a2, 44(sp)
-; RV32-NEXT:    li a2, 40
-; RV32-NEXT:    addi a3, a4, 819
-; RV32-NEXT:    sw a3, 32(sp)
-; RV32-NEXT:    sw a3, 36(sp)
-; RV32-NEXT:    addi a3, sp, 16
-; RV32-NEXT:    addi a4, a5, 1365
-; RV32-NEXT:    vsll.vx v16, v8, a1, v0.t
-; RV32-NEXT:    addi a5, a6, -256
-; RV32-NEXT:    sw a4, 24(sp)
-; RV32-NEXT:    sw a4, 28(sp)
-; RV32-NEXT:    vand.vx v8, v8, a5, v0.t
-; RV32-NEXT:    vsll.vx v8, v8, a2, v0.t
-; RV32-NEXT:    vor.vv v8, v16, v8, v0.t
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    slli a4, a4, 4
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 48
-; RV32-NEXT:    vs8r.v v8, (a4) # Unknown-size Folded Spill
+; RV32-NEXT:    vsll.vx v16, v8, a2, v0.t
+; RV32-NEXT:    addi a1, a3, -256
+; RV32-NEXT:    vand.vx v24, v8, a1, v0.t
+; RV32-NEXT:    vsll.vx v24, v24, a4, v0.t
+; RV32-NEXT:    vor.vv v16, v16, v24, v0.t
+; RV32-NEXT:    csrr a3, vlenb
+; RV32-NEXT:    slli a3, a3, 4
+; RV32-NEXT:    add a3, sp, a3
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a3), zero
+; RV32-NEXT:    vlse64.v v16, (a5), zero
 ; RV32-NEXT:    csrr a3, vlenb
 ; RV32-NEXT:    slli a3, a3, 3
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    lui a3, 4080
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vx v16, v24, a3, v0.t
-; RV32-NEXT:    vsll.vi v16, v16, 24, v0.t
-; RV32-NEXT:    addi a4, sp, 48
-; RV32-NEXT:    vs8r.v v16, (a4) # Unknown-size Folded Spill
-; RV32-NEXT:    vand.vv v16, v24, v8, v0.t
-; RV32-NEXT:    vsll.vi v16, v16, 8, v0.t
-; RV32-NEXT:    vl8r.v v8, (a4) # Unknown-size Folded Reload
-; RV32-NEXT:    vor.vv v16, v8, v16, v0.t
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    slli a4, a4, 4
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 48
-; RV32-NEXT:    vl8r.v v8, (a4) # Unknown-size Folded Reload
-; RV32-NEXT:    vor.vv v16, v8, v16, v0.t
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    slli a4, a4, 4
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 48
-; RV32-NEXT:    vs8r.v v16, (a4) # Unknown-size Folded Spill
-; RV32-NEXT:    vsrl.vx v16, v24, a1, v0.t
-; RV32-NEXT:    vsrl.vx v8, v24, a2, v0.t
-; RV32-NEXT:    vand.vx v8, v8, a5, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    addi a1, sp, 48
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    vsrl.vi v8, v24, 24, v0.t
-; RV32-NEXT:    vand.vx v16, v8, a3, v0.t
-; RV32-NEXT:    vsrl.vi v8, v24, 8, v0.t
+; RV32-NEXT:    vand.vx v24, v8, a3, v0.t
+; RV32-NEXT:    vsll.vi v24, v24, 24, v0.t
+; RV32-NEXT:    addi a5, sp, 16
+; RV32-NEXT:    vs8r.v v24, (a5) # Unknown-size Folded Spill
+; RV32-NEXT:    vand.vv v24, v8, v16, v0.t
+; RV32-NEXT:    vsll.vi v16, v24, 8, v0.t
+; RV32-NEXT:    vl8r.v v24, (a5) # Unknown-size Folded Reload
+; RV32-NEXT:    vor.vv v16, v24, v16, v0.t
+; RV32-NEXT:    csrr a5, vlenb
+; RV32-NEXT:    slli a5, a5, 4
+; RV32-NEXT:    add a5, sp, a5
+; RV32-NEXT:    addi a5, a5, 16
+; RV32-NEXT:    vl8r.v v24, (a5) # Unknown-size Folded Reload
+; RV32-NEXT:    vor.vv v16, v24, v16, v0.t
+; RV32-NEXT:    csrr a5, vlenb
+; RV32-NEXT:    slli a5, a5, 4
+; RV32-NEXT:    add a5, sp, a5
+; RV32-NEXT:    addi a5, a5, 16
+; RV32-NEXT:    vs8r.v v16, (a5) # Unknown-size Folded Spill
+; RV32-NEXT:    vsrl.vx v16, v8, a2, v0.t
+; RV32-NEXT:    vsrl.vx v24, v8, a4, v0.t
+; RV32-NEXT:    vand.vx v24, v24, a1, v0.t
+; RV32-NEXT:    vor.vv v16, v24, v16, v0.t
+; RV32-NEXT:    addi a1, sp, 16
+; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    vsrl.vi v24, v8, 24, v0.t
+; RV32-NEXT:    vand.vx v24, v24, a3, v0.t
+; RV32-NEXT:    vsrl.vi v8, v8, 8, v0.t
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    slli a1, a1, 3
 ; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    addi a1, sp, 48
+; RV32-NEXT:    addi a1, a1, 16
+; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    addi a1, sp, 16
 ; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    addi a1, sp, 40
-; RV32-NEXT:    addi a2, sp, 32
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    lui a1, 61681
+; RV32-NEXT:    lui a2, 209715
+; RV32-NEXT:    lui a3, 349525
+; RV32-NEXT:    addi a1, a1, -241
+; RV32-NEXT:    addi a2, a2, 819
+; RV32-NEXT:    addi a3, a3, 1365
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    slli a1, a1, 4
+; RV32-NEXT:    add a1, sp, a1
+; RV32-NEXT:    addi a1, a1, 16
+; RV32-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vor.vv v8, v16, v8, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV32-NEXT:    vand.vv v16, v16, v24, v0.t
 ; RV32-NEXT:    vand.vv v24, v8, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a2), zero
+; RV32-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a2
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vi v24, v24, 4, v0.t
 ; RV32-NEXT:    vor.vv v24, v16, v24, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v24, 2, v0.t
 ; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
 ; RV32-NEXT:    vand.vv v24, v24, v8, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
+; RV32-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a3
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vi v24, v24, 2, v0.t
 ; RV32-NEXT:    vor.vv v16, v16, v24, v0.t
@@ -2170,8 +2141,8 @@ define <16 x i64> @vp_bitreverse_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroex
 ; RV32-NEXT:    li a1, 24
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
+; RV32-NEXT:    .cfi_def_cfa sp, 16
+; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
@@ -2259,91 +2230,82 @@ define <16 x i64> @vp_bitreverse_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroex
 define <16 x i64> @vp_bitreverse_v16i64_unmasked(<16 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_bitreverse_v16i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    slli a1, a1, 4
 ; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
+; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
 ; RV32-NEXT:    lui a1, 1044480
-; RV32-NEXT:    lui a2, 61681
-; RV32-NEXT:    lui a3, 209715
-; RV32-NEXT:    lui a4, 349525
-; RV32-NEXT:    li a5, 56
-; RV32-NEXT:    lui a6, 16
+; RV32-NEXT:    li a2, 56
+; RV32-NEXT:    lui a3, 16
+; RV32-NEXT:    li a4, 40
+; RV32-NEXT:    lui a5, 4080
+; RV32-NEXT:    addi a6, sp, 8
+; RV32-NEXT:    sw a1, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsll.vx v16, v8, a5
-; RV32-NEXT:    vsrl.vx v24, v8, a5
-; RV32-NEXT:    li a5, 40
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw zero, 20(sp)
-; RV32-NEXT:    addi a1, a2, -241
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
-; RV32-NEXT:    lui a1, 4080
-; RV32-NEXT:    addi a2, a3, 819
-; RV32-NEXT:    sw a2, 32(sp)
-; RV32-NEXT:    sw a2, 36(sp)
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    addi a3, a4, 1365
-; RV32-NEXT:    addi a4, a6, -256
-; RV32-NEXT:    vsrl.vx v0, v8, a5
-; RV32-NEXT:    sw a3, 24(sp)
-; RV32-NEXT:    sw a3, 28(sp)
-; RV32-NEXT:    vand.vx v0, v0, a4
+; RV32-NEXT:    vsll.vx v16, v8, a2
+; RV32-NEXT:    addi a1, a3, -256
+; RV32-NEXT:    vsrl.vx v24, v8, a2
+; RV32-NEXT:    vsrl.vx v0, v8, a4
+; RV32-NEXT:    vand.vx v0, v0, a1
 ; RV32-NEXT:    vor.vv v24, v0, v24
-; RV32-NEXT:    addi a3, sp, 48
-; RV32-NEXT:    vs8r.v v24, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    vand.vx v0, v8, a4
-; RV32-NEXT:    vsll.vx v0, v0, a5
-; RV32-NEXT:    vor.vv v16, v16, v0
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vs8r.v v24, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    vand.vx v24, v8, a1
+; RV32-NEXT:    vsll.vx v24, v24, a4
+; RV32-NEXT:    vor.vv v16, v16, v24
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    slli a1, a1, 3
+; RV32-NEXT:    add a1, sp, a1
+; RV32-NEXT:    addi a1, a1, 16
+; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v0, (a2), zero
+; RV32-NEXT:    vlse64.v v24, (a6), zero
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsrl.vi v16, v8, 24
-; RV32-NEXT:    vand.vx v16, v16, a1
-; RV32-NEXT:    vsrl.vi v24, v8, 8
-; RV32-NEXT:    vand.vv v24, v24, v0
-; RV32-NEXT:    vor.vv v16, v24, v16
-; RV32-NEXT:    vand.vv v24, v8, v0
-; RV32-NEXT:    vand.vx v8, v8, a1
+; RV32-NEXT:    vand.vx v16, v16, a5
+; RV32-NEXT:    vsrl.vi v0, v8, 8
+; RV32-NEXT:    vand.vv v0, v0, v24
+; RV32-NEXT:    vor.vv v16, v0, v16
+; RV32-NEXT:    vand.vv v24, v8, v24
+; RV32-NEXT:    vand.vx v8, v8, a5
 ; RV32-NEXT:    vsll.vi v8, v8, 24
 ; RV32-NEXT:    vsll.vi v24, v24, 8
-; RV32-NEXT:    vor.vv v0, v8, v24
-; RV32-NEXT:    addi a1, sp, 48
+; RV32-NEXT:    vor.vv v24, v8, v24
+; RV32-NEXT:    addi a1, sp, 16
 ; RV32-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
 ; RV32-NEXT:    vor.vv v8, v16, v8
-; RV32-NEXT:    addi a1, sp, 40
-; RV32-NEXT:    addi a2, sp, 32
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vor.vv v24, v16, v0
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a1), zero
-; RV32-NEXT:    addi a1, sp, 24
+; RV32-NEXT:    lui a1, 61681
+; RV32-NEXT:    lui a2, 209715
+; RV32-NEXT:    lui a3, 349525
+; RV32-NEXT:    addi a1, a1, -241
+; RV32-NEXT:    addi a2, a2, 819
+; RV32-NEXT:    addi a3, a3, 1365
+; RV32-NEXT:    csrr a4, vlenb
+; RV32-NEXT:    slli a4, a4, 3
+; RV32-NEXT:    add a4, sp, a4
+; RV32-NEXT:    addi a4, a4, 16
+; RV32-NEXT:    vl8r.v v16, (a4) # Unknown-size Folded Reload
+; RV32-NEXT:    vor.vv v16, v16, v24
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vor.vv v8, v24, v8
-; RV32-NEXT:    vsrl.vi v24, v8, 4
-; RV32-NEXT:    vand.vv v8, v8, v16
-; RV32-NEXT:    vand.vv v16, v24, v16
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a2), zero
+; RV32-NEXT:    vor.vv v8, v16, v8
+; RV32-NEXT:    vsrl.vi v16, v8, 4
+; RV32-NEXT:    vand.vv v8, v8, v24
+; RV32-NEXT:    vand.vv v16, v16, v24
+; RV32-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a2
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vi v8, v8, 4
 ; RV32-NEXT:    vor.vv v8, v16, v8
 ; RV32-NEXT:    vsrl.vi v16, v8, 2
 ; RV32-NEXT:    vand.vv v8, v8, v24
 ; RV32-NEXT:    vand.vv v16, v16, v24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
+; RV32-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a3
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vsll.vi v8, v8, 2
 ; RV32-NEXT:    vor.vv v8, v16, v8
@@ -2355,8 +2317,8 @@ define <16 x i64> @vp_bitreverse_v16i64_unmasked(<16 x i64> %va, i32 zeroext %ev
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    slli a0, a0, 4
 ; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
+; RV32-NEXT:    .cfi_def_cfa sp, 16
+; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz-vp.ll
index 9d0d42cf754c5..a973fb7efff33 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctlz-vp.ll
@@ -1503,102 +1503,54 @@ declare <15 x i64> @llvm.vp.ctlz.v15i64(<15 x i64>, i1 immarg, <15 x i1>, i32)
 define <15 x i64> @vp_ctlz_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_ctlz_v15i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
-; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vsrl.vi v24, v8, 1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
+; RV32-NEXT:    li a1, 32
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v8, 2, v0.t
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v8, 4, v0.t
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v8, 8, v0.t
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v8, 16, v0.t
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vx v24, v8, a1, v0.t
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 32(sp)
-; RV32-NEXT:    sw a1, 36(sp)
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vnot.v v24, v8, v0.t
+; RV32-NEXT:    vsrl.vi v8, v24, 1, v0.t
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a1
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v8, 1, v0.t
+; RV32-NEXT:    vsub.vv v24, v24, v16, v0.t
+; RV32-NEXT:    vand.vv v16, v24, v8, v0.t
+; RV32-NEXT:    vsrl.vi v24, v24, 2, v0.t
+; RV32-NEXT:    vand.vv v24, v24, v8, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a1
+; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
-; RV32-NEXT:    addi a1, sp, 40
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    li a1, 32
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vi v16, v8, 2, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vi v16, v8, 8, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vi v16, v8, 16, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vx v16, v8, a1, v0.t
-; RV32-NEXT:    addi a1, sp, 32
-; RV32-NEXT:    vor.vv v16, v8, v16, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    addi a1, sp, 48
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vnot.v v16, v16, v0.t
-; RV32-NEXT:    vsrl.vi v8, v16, 1, v0.t
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    vsub.vv v8, v16, v8, v0.t
-; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v8, v24, v0.t
-; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV32-NEXT:    vand.vv v24, v8, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a1, sp, 16
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v24, v16, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a1), zero
+; RV32-NEXT:    vadd.vv v16, v16, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v16, 4, v0.t
+; RV32-NEXT:    vadd.vv v16, v16, v24, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
+; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v24, 4, v0.t
-; RV32-NEXT:    vadd.vv v8, v24, v8, v0.t
-; RV32-NEXT:    li a0, 56
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vx v8, v8, a0, v0.t
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vand.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vmul.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vx v8, v8, a1, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_ctlz_v15i64:
@@ -1655,68 +1607,54 @@ define <15 x i64> @vp_ctlz_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroext %evl
 define <15 x i64> @vp_ctlz_v15i64_unmasked(<15 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_ctlz_v15i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vsrl.vi v16, v8, 1
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
+; RV32-NEXT:    li a1, 32
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vi v16, v8, 2
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vi v16, v8, 4
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vi v16, v8, 8
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vi v16, v8, 16
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vx v16, v8, a1
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vnot.v v8, v8
+; RV32-NEXT:    vsrl.vi v16, v8, 1
+; RV32-NEXT:    vand.vv v24, v16, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 8(sp)
-; RV32-NEXT:    sw a1, 12(sp)
-; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v0, v8, 1
+; RV32-NEXT:    vsub.vv v8, v8, v24
+; RV32-NEXT:    vand.vv v24, v8, v16
+; RV32-NEXT:    vsrl.vi v8, v8, 2
+; RV32-NEXT:    vand.vv v8, v8, v16
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
+; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 0(sp)
-; RV32-NEXT:    sw a1, 4(sp)
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    addi a1, sp, 16
-; RV32-NEXT:    vlse64.v v16, (a1), zero
-; RV32-NEXT:    li a1, 32
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vi v0, v8, 2
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vi v0, v8, 4
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vi v0, v8, 8
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vi v0, v8, 16
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vx v0, v8, a1
-; RV32-NEXT:    addi a1, sp, 8
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vnot.v v0, v8
-; RV32-NEXT:    vsrl.vi v8, v0, 1
-; RV32-NEXT:    vand.vv v24, v8, v24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    mv a1, sp
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vv v24, v0, v24
-; RV32-NEXT:    vand.vv v0, v24, v16
-; RV32-NEXT:    vsrl.vi v24, v24, 2
-; RV32-NEXT:    vand.vv v16, v24, v16
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
+; RV32-NEXT:    vadd.vv v8, v24, v8
+; RV32-NEXT:    vsrl.vi v24, v8, 4
+; RV32-NEXT:    vadd.vv v8, v8, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v16, v0, v16
-; RV32-NEXT:    vsrl.vi v0, v16, 4
-; RV32-NEXT:    vadd.vv v16, v16, v0
-; RV32-NEXT:    vand.vv v8, v16, v8
+; RV32-NEXT:    vand.vv v8, v8, v16
 ; RV32-NEXT:    vmul.vv v8, v8, v24
 ; RV32-NEXT:    li a0, 56
 ; RV32-NEXT:    vsrl.vx v8, v8, a0
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_ctlz_v15i64_unmasked:
@@ -1775,102 +1713,54 @@ declare <16 x i64> @llvm.vp.ctlz.v16i64(<16 x i64>, i1 immarg, <16 x i1>, i32)
 define <16 x i64> @vp_ctlz_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_ctlz_v16i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
-; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vsrl.vi v24, v8, 1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
+; RV32-NEXT:    li a1, 32
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v8, 2, v0.t
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v8, 4, v0.t
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v8, 8, v0.t
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v8, 16, v0.t
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vx v24, v8, a1, v0.t
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 32(sp)
-; RV32-NEXT:    sw a1, 36(sp)
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vnot.v v24, v8, v0.t
+; RV32-NEXT:    vsrl.vi v8, v24, 1, v0.t
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a1
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v8, 1, v0.t
+; RV32-NEXT:    vsub.vv v24, v24, v16, v0.t
+; RV32-NEXT:    vand.vv v16, v24, v8, v0.t
+; RV32-NEXT:    vsrl.vi v24, v24, 2, v0.t
+; RV32-NEXT:    vand.vv v24, v24, v8, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a1
+; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
-; RV32-NEXT:    addi a1, sp, 40
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    li a1, 32
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vi v16, v8, 2, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vi v16, v8, 8, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vi v16, v8, 16, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vx v16, v8, a1, v0.t
-; RV32-NEXT:    addi a1, sp, 32
-; RV32-NEXT:    vor.vv v16, v8, v16, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    addi a1, sp, 48
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vnot.v v16, v16, v0.t
-; RV32-NEXT:    vsrl.vi v8, v16, 1, v0.t
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    vsub.vv v8, v16, v8, v0.t
-; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v8, v24, v0.t
-; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV32-NEXT:    vand.vv v24, v8, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a1, sp, 16
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v24, v16, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a1), zero
+; RV32-NEXT:    vadd.vv v16, v16, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v16, 4, v0.t
+; RV32-NEXT:    vadd.vv v16, v16, v24, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
+; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v24, 4, v0.t
-; RV32-NEXT:    vadd.vv v8, v24, v8, v0.t
-; RV32-NEXT:    li a0, 56
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vx v8, v8, a0, v0.t
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vand.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vmul.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vx v8, v8, a1, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_ctlz_v16i64:
@@ -1927,68 +1817,54 @@ define <16 x i64> @vp_ctlz_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroext %evl
 define <16 x i64> @vp_ctlz_v16i64_unmasked(<16 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_ctlz_v16i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vsrl.vi v16, v8, 1
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
+; RV32-NEXT:    li a1, 32
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vi v16, v8, 2
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vi v16, v8, 4
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vi v16, v8, 8
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vi v16, v8, 16
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vx v16, v8, a1
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vnot.v v8, v8
+; RV32-NEXT:    vsrl.vi v16, v8, 1
+; RV32-NEXT:    vand.vv v24, v16, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 8(sp)
-; RV32-NEXT:    sw a1, 12(sp)
-; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v0, v8, 1
+; RV32-NEXT:    vsub.vv v8, v8, v24
+; RV32-NEXT:    vand.vv v24, v8, v16
+; RV32-NEXT:    vsrl.vi v8, v8, 2
+; RV32-NEXT:    vand.vv v8, v8, v16
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
+; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 0(sp)
-; RV32-NEXT:    sw a1, 4(sp)
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    addi a1, sp, 16
-; RV32-NEXT:    vlse64.v v16, (a1), zero
-; RV32-NEXT:    li a1, 32
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vi v0, v8, 2
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vi v0, v8, 4
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vi v0, v8, 8
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vi v0, v8, 16
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vx v0, v8, a1
-; RV32-NEXT:    addi a1, sp, 8
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vnot.v v0, v8
-; RV32-NEXT:    vsrl.vi v8, v0, 1
-; RV32-NEXT:    vand.vv v24, v8, v24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    mv a1, sp
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vv v24, v0, v24
-; RV32-NEXT:    vand.vv v0, v24, v16
-; RV32-NEXT:    vsrl.vi v24, v24, 2
-; RV32-NEXT:    vand.vv v16, v24, v16
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
+; RV32-NEXT:    vadd.vv v8, v24, v8
+; RV32-NEXT:    vsrl.vi v24, v8, 4
+; RV32-NEXT:    vadd.vv v8, v8, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v16, v0, v16
-; RV32-NEXT:    vsrl.vi v0, v16, 4
-; RV32-NEXT:    vadd.vv v16, v16, v0
-; RV32-NEXT:    vand.vv v8, v16, v8
+; RV32-NEXT:    vand.vv v8, v8, v16
 ; RV32-NEXT:    vmul.vv v8, v8, v24
 ; RV32-NEXT:    li a0, 56
 ; RV32-NEXT:    vsrl.vx v8, v8, a0
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_ctlz_v16i64_unmasked:
@@ -2047,37 +1923,21 @@ declare <32 x i64> @llvm.vp.ctlz.v32i64(<32 x i64>, i1 immarg, <32 x i1>, i32)
 define <32 x i64> @vp_ctlz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_ctlz_v32i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    li a2, 56
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x38, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 56 * vlenb
+; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x38, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 56 * vlenb
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
+; RV32-NEXT:    slli a1, a1, 5
 ; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
+; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    li a1, 16
 ; RV32-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV32-NEXT:    vslidedown.vi v24, v0, 2
-; RV32-NEXT:    lui a1, 349525
-; RV32-NEXT:    lui a2, 209715
-; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
-; RV32-NEXT:    lui a1, 61681
-; RV32-NEXT:    addi a2, a2, 819
-; RV32-NEXT:    sw a2, 32(sp)
-; RV32-NEXT:    sw a2, 36(sp)
-; RV32-NEXT:    lui a2, 4112
-; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    li a1, 16
-; RV32-NEXT:    addi a2, a2, 257
-; RV32-NEXT:    sw a2, 16(sp)
-; RV32-NEXT:    sw a2, 20(sp)
 ; RV32-NEXT:    mv a2, a0
 ; RV32-NEXT:    bltu a0, a1, .LBB34_2
 ; RV32-NEXT:  # %bb.1:
@@ -2086,9 +1946,9 @@ define <32 x i64> @vp_ctlz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vsrl.vi v16, v8, 1, v0.t
 ; RV32-NEXT:    li a1, 32
-; RV32-NEXT:    addi a3, sp, 40
-; RV32-NEXT:    addi a4, sp, 32
+; RV32-NEXT:    lui a3, 349525
 ; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
+; RV32-NEXT:    addi a3, a3, 1365
 ; RV32-NEXT:    vsrl.vi v16, v8, 2, v0.t
 ; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
@@ -2099,144 +1959,119 @@ define <32 x i64> @vp_ctlz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vx v16, v8, a1, v0.t
 ; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a3), zero
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a3
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a5, 40
-; RV32-NEXT:    mul a3, a3, a5
+; RV32-NEXT:    li a4, 40
+; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; RV32-NEXT:    vnot.v v16, v8, v0.t
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 5
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a4), zero
+; RV32-NEXT:    vnot.v v8, v8, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 48
+; RV32-NEXT:    li a4, 24
 ; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v16, 1, v0.t
+; RV32-NEXT:    vsrl.vi v8, v8, 1, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
+; RV32-NEXT:    li a4, 48
 ; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a3, vlenb
 ; RV32-NEXT:    li a4, 40
 ; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
+; RV32-NEXT:    li a4, 48
 ; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 5
+; RV32-NEXT:    li a4, 24
+; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vsub.vv v16, v16, v8, v0.t
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    vsub.vv v8, v8, v16, v0.t
+; RV32-NEXT:    lui a3, 209715
+; RV32-NEXT:    addi a3, a3, 819
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a3
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 5
+; RV32-NEXT:    li a4, 48
+; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 5
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    csrr a3, vlenb
 ; RV32-NEXT:    li a4, 48
 ; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
 ; RV32-NEXT:    li a4, 24
 ; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 5
+; RV32-NEXT:    li a4, 48
+; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vsrl.vi v16, v16, 2, v0.t
-; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
+; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
 ; RV32-NEXT:    li a4, 24
 ; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    vadd.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV32-NEXT:    vadd.vv v8, v8, v16, v0.t
+; RV32-NEXT:    lui a3, 61681
+; RV32-NEXT:    addi a3, a3, -241
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a3
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a3, sp, 24
-; RV32-NEXT:    addi a4, sp, 16
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a3), zero
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a5, 24
-; RV32-NEXT:    mul a3, a3, a5
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    vlse64.v v8, (a4), zero
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 5
+; RV32-NEXT:    li a4, 24
+; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    lui a3, 4112
+; RV32-NEXT:    addi a3, a3, 257
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a3
 ; RV32-NEXT:    csrr a3, vlenb
 ; RV32-NEXT:    slli a3, a3, 3
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
-; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
-; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vadd.vv v16, v8, v16, v0.t
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    li a3, 24
-; RV32-NEXT:    mul a2, a2, a3
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    slli a2, a2, 5
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vmul.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    li a2, 56
 ; RV32-NEXT:    vsrl.vx v8, v8, a2, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
+; RV32-NEXT:    slli a3, a3, 4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    addi a3, a0, -16
 ; RV32-NEXT:    sltu a0, a0, a3
@@ -2244,15 +2079,15 @@ define <32 x i64> @vp_ctlz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    and a0, a0, a3
 ; RV32-NEXT:    vmv1r.v v0, v24
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 4
+; RV32-NEXT:    slli a3, a3, 5
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v16, 1, v0.t
+; RV32-NEXT:    vsrl.vi v16, v8, 1, v0.t
+; RV32-NEXT:    vor.vv v16, v8, v16, v0.t
+; RV32-NEXT:    vsrl.vi v8, v16, 2, v0.t
 ; RV32-NEXT:    vor.vv v8, v16, v8, v0.t
-; RV32-NEXT:    vsrl.vi v16, v8, 2, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v8, 8, v0.t
@@ -2262,71 +2097,55 @@ define <32 x i64> @vp_ctlz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    vsrl.vx v16, v8, a1, v0.t
 ; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    vnot.v v8, v8, v0.t
-; RV32-NEXT:    addi a0, sp, 48
+; RV32-NEXT:    addi a0, sp, 16
 ; RV32-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsrl.vi v8, v8, 1, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
+; RV32-NEXT:    slli a0, a0, 5
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    li a1, 40
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
+; RV32-NEXT:    slli a0, a0, 5
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
-; RV32-NEXT:    addi a0, sp, 48
+; RV32-NEXT:    addi a0, sp, 16
 ; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsub.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 40
-; RV32-NEXT:    mul a0, a0, a1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    li a1, 48
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 40
-; RV32-NEXT:    mul a0, a0, a1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    li a1, 40
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    li a1, 48
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
+; RV32-NEXT:    li a1, 40
+; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
@@ -2335,27 +2154,27 @@ define <32 x i64> @vp_ctlz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV32-NEXT:    li a1, 24
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 5
+; RV32-NEXT:    slli a0, a0, 3
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vx v16, v8, a2, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 3
+; RV32-NEXT:    slli a0, a0, 4
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
+; RV32-NEXT:    .cfi_def_cfa sp, 16
+; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
@@ -2478,52 +2297,38 @@ define <32 x i64> @vp_ctlz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 define <32 x i64> @vp_ctlz_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_ctlz_v32i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
-; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
-; RV32-NEXT:    lui a1, 349525
-; RV32-NEXT:    lui a2, 209715
-; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
-; RV32-NEXT:    lui a1, 61681
-; RV32-NEXT:    addi a2, a2, 819
-; RV32-NEXT:    sw a2, 32(sp)
-; RV32-NEXT:    sw a2, 36(sp)
-; RV32-NEXT:    lui a2, 4112
-; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    li a3, 16
-; RV32-NEXT:    addi a1, a2, 257
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
+; RV32-NEXT:    li a2, 16
 ; RV32-NEXT:    mv a1, a0
-; RV32-NEXT:    bltu a0, a3, .LBB35_2
+; RV32-NEXT:    bltu a0, a2, .LBB35_2
 ; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    li a1, 16
 ; RV32-NEXT:  .LBB35_2:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a2, a2, 4
+; RV32-NEXT:    sub sp, sp, a2
+; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV32-NEXT:    vsrl.vi v0, v8, 1
 ; RV32-NEXT:    li a2, 32
-; RV32-NEXT:    addi a3, sp, 40
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a3), zero
+; RV32-NEXT:    lui a3, 349525
+; RV32-NEXT:    addi a3, a3, 1365
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a3
 ; RV32-NEXT:    csrr a3, vlenb
 ; RV32-NEXT:    slli a3, a3, 3
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vs8r.v v24, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    addi a3, a0, -16
 ; RV32-NEXT:    sltu a0, a0, a3
 ; RV32-NEXT:    addi a0, a0, -1
 ; RV32-NEXT:    and a0, a0, a3
-; RV32-NEXT:    addi a3, sp, 32
+; RV32-NEXT:    lui a3, 209715
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV32-NEXT:    vor.vv v8, v8, v0
+; RV32-NEXT:    addi a3, a3, 819
 ; RV32-NEXT:    vsrl.vi v0, v8, 2
 ; RV32-NEXT:    vor.vv v8, v8, v0
 ; RV32-NEXT:    vsrl.vi v0, v8, 4
@@ -2550,90 +2355,74 @@ define <32 x i64> @vp_ctlz_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 ; RV32-NEXT:    vsrl.vi v0, v16, 8
 ; RV32-NEXT:    vor.vv v16, v16, v0
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vnot.v v0, v8
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v16, 16
-; RV32-NEXT:    vor.vv v16, v16, v8
-; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v0, 1
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    slli a4, a4, 3
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 48
-; RV32-NEXT:    vl8r.v v24, (a4) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v24, v8, v24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a3), zero
-; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vv v24, v0, v24
+; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vx v0, v16, a2
+; RV32-NEXT:    vsrl.vi v0, v16, 16
 ; RV32-NEXT:    vor.vv v16, v16, v0
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v0, v24, v8
-; RV32-NEXT:    vsrl.vi v24, v24, 2
-; RV32-NEXT:    vand.vv v24, v24, v8
-; RV32-NEXT:    vadd.vv v24, v0, v24
-; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vs8r.v v24, (a2) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vnot.v v16, v16
-; RV32-NEXT:    vsrl.vi v0, v16, 1
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    slli a2, a2, 3
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vsrl.vi v0, v8, 1
 ; RV32-NEXT:    vand.vv v0, v0, v24
-; RV32-NEXT:    addi a2, sp, 24
-; RV32-NEXT:    addi a3, sp, 16
-; RV32-NEXT:    vsub.vv v0, v16, v0
-; RV32-NEXT:    addi a4, sp, 48
-; RV32-NEXT:    vl8r.v v24, (a4) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v24, 4
-; RV32-NEXT:    vadd.vv v16, v24, v16
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    slli a4, a4, 3
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 48
-; RV32-NEXT:    vs8r.v v16, (a4) # Unknown-size Folded Spill
+; RV32-NEXT:    vsub.vv v0, v8, v0
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v24, v0, v8
+; RV32-NEXT:    vsrl.vx v8, v16, a2
+; RV32-NEXT:    vor.vv v24, v16, v8
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a3
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v8, v0, v16
 ; RV32-NEXT:    vsrl.vi v0, v0, 2
-; RV32-NEXT:    vand.vv v8, v0, v8
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v0, (a2), zero
+; RV32-NEXT:    vand.vv v0, v0, v16
+; RV32-NEXT:    vadd.vv v8, v8, v0
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v8, v24, v8
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a3), zero
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v8, 4
-; RV32-NEXT:    vadd.vv v8, v8, v16
+; RV32-NEXT:    vnot.v v24, v24
+; RV32-NEXT:    vsrl.vi v0, v24, 1
 ; RV32-NEXT:    csrr a2, vlenb
 ; RV32-NEXT:    slli a2, a2, 3
 ; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
+; RV32-NEXT:    addi a2, a2, 16
 ; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v0, v0, v16
+; RV32-NEXT:    vsub.vv v24, v24, v0
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vsrl.vi v0, v8, 4
+; RV32-NEXT:    vadd.vv v8, v8, v0
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v0, v24, v16
+; RV32-NEXT:    vsrl.vi v24, v24, 2
+; RV32-NEXT:    vand.vv v16, v24, v16
+; RV32-NEXT:    lui a2, 61681
+; RV32-NEXT:    lui a3, 4112
+; RV32-NEXT:    addi a2, a2, -241
+; RV32-NEXT:    addi a3, a3, 257
+; RV32-NEXT:    vadd.vv v16, v0, v16
+; RV32-NEXT:    vsrl.vi v24, v16, 4
+; RV32-NEXT:    vadd.vv v16, v16, v24
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v16, v16, v0
+; RV32-NEXT:    vand.vv v8, v8, v24
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v8, v8, v0
+; RV32-NEXT:    vand.vv v16, v16, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a3
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vmul.vv v16, v16, v24
+; RV32-NEXT:    vmul.vv v8, v8, v24
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vmul.vv v24, v8, v24
+; RV32-NEXT:    vmul.vv v16, v16, v24
 ; RV32-NEXT:    li a2, 56
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vx v8, v16, a2
+; RV32-NEXT:    vsrl.vx v8, v8, a2
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vx v16, v24, a2
+; RV32-NEXT:    vsrl.vx v16, v16, a2
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    slli a0, a0, 4
 ; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
+; RV32-NEXT:    .cfi_def_cfa sp, 16
+; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
@@ -4213,102 +4002,54 @@ define <8 x i64> @vp_ctlz_zero_undef_v8i64_unmasked(<8 x i64> %va, i32 zeroext %
 define <15 x i64> @vp_ctlz_zero_undef_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_ctlz_zero_undef_v15i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
-; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vsrl.vi v24, v8, 1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
+; RV32-NEXT:    li a1, 32
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v8, 2, v0.t
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v8, 4, v0.t
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v8, 8, v0.t
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v8, 16, v0.t
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vx v24, v8, a1, v0.t
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 32(sp)
-; RV32-NEXT:    sw a1, 36(sp)
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vnot.v v24, v8, v0.t
+; RV32-NEXT:    vsrl.vi v8, v24, 1, v0.t
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a1
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v8, 1, v0.t
+; RV32-NEXT:    vsub.vv v24, v24, v16, v0.t
+; RV32-NEXT:    vand.vv v16, v24, v8, v0.t
+; RV32-NEXT:    vsrl.vi v24, v24, 2, v0.t
+; RV32-NEXT:    vand.vv v24, v24, v8, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a1
+; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
-; RV32-NEXT:    addi a1, sp, 40
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    li a1, 32
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vi v16, v8, 2, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vi v16, v8, 8, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vi v16, v8, 16, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vx v16, v8, a1, v0.t
-; RV32-NEXT:    addi a1, sp, 32
-; RV32-NEXT:    vor.vv v16, v8, v16, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    addi a1, sp, 48
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vnot.v v16, v16, v0.t
-; RV32-NEXT:    vsrl.vi v8, v16, 1, v0.t
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    vsub.vv v8, v16, v8, v0.t
-; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v8, v24, v0.t
-; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV32-NEXT:    vand.vv v24, v8, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a1, sp, 16
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v24, v16, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a1), zero
+; RV32-NEXT:    vadd.vv v16, v16, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v16, 4, v0.t
+; RV32-NEXT:    vadd.vv v16, v16, v24, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
+; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v24, 4, v0.t
-; RV32-NEXT:    vadd.vv v8, v24, v8, v0.t
-; RV32-NEXT:    li a0, 56
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vx v8, v8, a0, v0.t
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vand.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vmul.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vx v8, v8, a1, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_ctlz_zero_undef_v15i64:
@@ -4365,68 +4106,54 @@ define <15 x i64> @vp_ctlz_zero_undef_v15i64(<15 x i64> %va, <15 x i1> %m, i32 z
 define <15 x i64> @vp_ctlz_zero_undef_v15i64_unmasked(<15 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_ctlz_zero_undef_v15i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vsrl.vi v16, v8, 1
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
+; RV32-NEXT:    li a1, 32
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vi v16, v8, 2
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vi v16, v8, 4
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vi v16, v8, 8
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vi v16, v8, 16
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vx v16, v8, a1
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vnot.v v8, v8
+; RV32-NEXT:    vsrl.vi v16, v8, 1
+; RV32-NEXT:    vand.vv v24, v16, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 8(sp)
-; RV32-NEXT:    sw a1, 12(sp)
-; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v0, v8, 1
+; RV32-NEXT:    vsub.vv v8, v8, v24
+; RV32-NEXT:    vand.vv v24, v8, v16
+; RV32-NEXT:    vsrl.vi v8, v8, 2
+; RV32-NEXT:    vand.vv v8, v8, v16
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
+; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 0(sp)
-; RV32-NEXT:    sw a1, 4(sp)
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    addi a1, sp, 16
-; RV32-NEXT:    vlse64.v v16, (a1), zero
-; RV32-NEXT:    li a1, 32
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vi v0, v8, 2
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vi v0, v8, 4
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vi v0, v8, 8
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vi v0, v8, 16
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vx v0, v8, a1
-; RV32-NEXT:    addi a1, sp, 8
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vnot.v v0, v8
-; RV32-NEXT:    vsrl.vi v8, v0, 1
-; RV32-NEXT:    vand.vv v24, v8, v24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    mv a1, sp
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vv v24, v0, v24
-; RV32-NEXT:    vand.vv v0, v24, v16
-; RV32-NEXT:    vsrl.vi v24, v24, 2
-; RV32-NEXT:    vand.vv v16, v24, v16
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
+; RV32-NEXT:    vadd.vv v8, v24, v8
+; RV32-NEXT:    vsrl.vi v24, v8, 4
+; RV32-NEXT:    vadd.vv v8, v8, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v16, v0, v16
-; RV32-NEXT:    vsrl.vi v0, v16, 4
-; RV32-NEXT:    vadd.vv v16, v16, v0
-; RV32-NEXT:    vand.vv v8, v16, v8
+; RV32-NEXT:    vand.vv v8, v8, v16
 ; RV32-NEXT:    vmul.vv v8, v8, v24
 ; RV32-NEXT:    li a0, 56
 ; RV32-NEXT:    vsrl.vx v8, v8, a0
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_ctlz_zero_undef_v15i64_unmasked:
@@ -4483,102 +4210,54 @@ define <15 x i64> @vp_ctlz_zero_undef_v15i64_unmasked(<15 x i64> %va, i32 zeroex
 define <16 x i64> @vp_ctlz_zero_undef_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_ctlz_zero_undef_v16i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
-; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vsrl.vi v24, v8, 1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
+; RV32-NEXT:    li a1, 32
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v8, 2, v0.t
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v8, 4, v0.t
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v8, 8, v0.t
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v8, 16, v0.t
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vx v24, v8, a1, v0.t
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 32(sp)
-; RV32-NEXT:    sw a1, 36(sp)
+; RV32-NEXT:    vor.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vnot.v v24, v8, v0.t
+; RV32-NEXT:    vsrl.vi v8, v24, 1, v0.t
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a1
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v8, 1, v0.t
+; RV32-NEXT:    vsub.vv v24, v24, v16, v0.t
+; RV32-NEXT:    vand.vv v16, v24, v8, v0.t
+; RV32-NEXT:    vsrl.vi v24, v24, 2, v0.t
+; RV32-NEXT:    vand.vv v24, v24, v8, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a1
+; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
-; RV32-NEXT:    addi a1, sp, 40
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    li a1, 32
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vi v16, v8, 2, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vi v16, v8, 8, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vi v16, v8, 16, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vx v16, v8, a1, v0.t
-; RV32-NEXT:    addi a1, sp, 32
-; RV32-NEXT:    vor.vv v16, v8, v16, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    addi a1, sp, 48
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vnot.v v16, v16, v0.t
-; RV32-NEXT:    vsrl.vi v8, v16, 1, v0.t
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    vsub.vv v8, v16, v8, v0.t
-; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v8, v24, v0.t
-; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV32-NEXT:    vand.vv v24, v8, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a1, sp, 16
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v24, v16, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a1), zero
+; RV32-NEXT:    vadd.vv v16, v16, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v16, 4, v0.t
+; RV32-NEXT:    vadd.vv v16, v16, v24, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
+; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v24, 4, v0.t
-; RV32-NEXT:    vadd.vv v8, v24, v8, v0.t
-; RV32-NEXT:    li a0, 56
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vx v8, v8, a0, v0.t
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vand.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vmul.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vx v8, v8, a1, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_ctlz_zero_undef_v16i64:
@@ -4635,68 +4314,54 @@ define <16 x i64> @vp_ctlz_zero_undef_v16i64(<16 x i64> %va, <16 x i1> %m, i32 z
 define <16 x i64> @vp_ctlz_zero_undef_v16i64_unmasked(<16 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_ctlz_zero_undef_v16i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vsrl.vi v16, v8, 1
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
+; RV32-NEXT:    li a1, 32
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vi v16, v8, 2
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vi v16, v8, 4
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vi v16, v8, 8
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vi v16, v8, 16
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vsrl.vx v16, v8, a1
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
+; RV32-NEXT:    vor.vv v8, v8, v16
+; RV32-NEXT:    vnot.v v8, v8
+; RV32-NEXT:    vsrl.vi v16, v8, 1
+; RV32-NEXT:    vand.vv v24, v16, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 8(sp)
-; RV32-NEXT:    sw a1, 12(sp)
-; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v0, v8, 1
+; RV32-NEXT:    vsub.vv v8, v8, v24
+; RV32-NEXT:    vand.vv v24, v8, v16
+; RV32-NEXT:    vsrl.vi v8, v8, 2
+; RV32-NEXT:    vand.vv v8, v8, v16
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
+; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 0(sp)
-; RV32-NEXT:    sw a1, 4(sp)
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    addi a1, sp, 16
-; RV32-NEXT:    vlse64.v v16, (a1), zero
-; RV32-NEXT:    li a1, 32
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vi v0, v8, 2
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vi v0, v8, 4
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vi v0, v8, 8
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vi v0, v8, 16
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vsrl.vx v0, v8, a1
-; RV32-NEXT:    addi a1, sp, 8
-; RV32-NEXT:    vor.vv v8, v8, v0
-; RV32-NEXT:    vnot.v v0, v8
-; RV32-NEXT:    vsrl.vi v8, v0, 1
-; RV32-NEXT:    vand.vv v24, v8, v24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    mv a1, sp
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vv v24, v0, v24
-; RV32-NEXT:    vand.vv v0, v24, v16
-; RV32-NEXT:    vsrl.vi v24, v24, 2
-; RV32-NEXT:    vand.vv v16, v24, v16
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
+; RV32-NEXT:    vadd.vv v8, v24, v8
+; RV32-NEXT:    vsrl.vi v24, v8, 4
+; RV32-NEXT:    vadd.vv v8, v8, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v16, v0, v16
-; RV32-NEXT:    vsrl.vi v0, v16, 4
-; RV32-NEXT:    vadd.vv v16, v16, v0
-; RV32-NEXT:    vand.vv v8, v16, v8
+; RV32-NEXT:    vand.vv v8, v8, v16
 ; RV32-NEXT:    vmul.vv v8, v8, v24
 ; RV32-NEXT:    li a0, 56
 ; RV32-NEXT:    vsrl.vx v8, v8, a0
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_ctlz_zero_undef_v16i64_unmasked:
@@ -4753,37 +4418,21 @@ define <16 x i64> @vp_ctlz_zero_undef_v16i64_unmasked(<16 x i64> %va, i32 zeroex
 define <32 x i64> @vp_ctlz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_ctlz_zero_undef_v32i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    li a2, 56
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x38, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 56 * vlenb
+; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x38, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 56 * vlenb
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
+; RV32-NEXT:    slli a1, a1, 5
 ; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
+; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    li a1, 16
 ; RV32-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV32-NEXT:    vslidedown.vi v24, v0, 2
-; RV32-NEXT:    lui a1, 349525
-; RV32-NEXT:    lui a2, 209715
-; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
-; RV32-NEXT:    lui a1, 61681
-; RV32-NEXT:    addi a2, a2, 819
-; RV32-NEXT:    sw a2, 32(sp)
-; RV32-NEXT:    sw a2, 36(sp)
-; RV32-NEXT:    lui a2, 4112
-; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    li a1, 16
-; RV32-NEXT:    addi a2, a2, 257
-; RV32-NEXT:    sw a2, 16(sp)
-; RV32-NEXT:    sw a2, 20(sp)
 ; RV32-NEXT:    mv a2, a0
 ; RV32-NEXT:    bltu a0, a1, .LBB70_2
 ; RV32-NEXT:  # %bb.1:
@@ -4792,9 +4441,9 @@ define <32 x i64> @vp_ctlz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV32-NEXT:    vsrl.vi v16, v8, 1, v0.t
 ; RV32-NEXT:    li a1, 32
-; RV32-NEXT:    addi a3, sp, 40
-; RV32-NEXT:    addi a4, sp, 32
+; RV32-NEXT:    lui a3, 349525
 ; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
+; RV32-NEXT:    addi a3, a3, 1365
 ; RV32-NEXT:    vsrl.vi v16, v8, 2, v0.t
 ; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
@@ -4805,144 +4454,119 @@ define <32 x i64> @vp_ctlz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vx v16, v8, a1, v0.t
 ; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a3), zero
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a3
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a5, 40
-; RV32-NEXT:    mul a3, a3, a5
+; RV32-NEXT:    li a4, 40
+; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; RV32-NEXT:    vnot.v v16, v8, v0.t
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 5
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a4), zero
+; RV32-NEXT:    vnot.v v8, v8, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 48
+; RV32-NEXT:    li a4, 24
 ; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v16, 1, v0.t
+; RV32-NEXT:    vsrl.vi v8, v8, 1, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
+; RV32-NEXT:    li a4, 48
 ; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a3, vlenb
 ; RV32-NEXT:    li a4, 40
 ; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
+; RV32-NEXT:    li a4, 48
 ; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 5
+; RV32-NEXT:    li a4, 24
+; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vsub.vv v16, v16, v8, v0.t
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    vsub.vv v8, v8, v16, v0.t
+; RV32-NEXT:    lui a3, 209715
+; RV32-NEXT:    addi a3, a3, 819
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a3
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 5
+; RV32-NEXT:    li a4, 48
+; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 5
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    csrr a3, vlenb
 ; RV32-NEXT:    li a4, 48
 ; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
 ; RV32-NEXT:    li a4, 24
 ; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 5
+; RV32-NEXT:    li a4, 48
+; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vsrl.vi v16, v16, 2, v0.t
-; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
+; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
 ; RV32-NEXT:    li a4, 24
 ; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    vadd.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV32-NEXT:    vadd.vv v8, v8, v16, v0.t
+; RV32-NEXT:    lui a3, 61681
+; RV32-NEXT:    addi a3, a3, -241
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a3
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a3, sp, 24
-; RV32-NEXT:    addi a4, sp, 16
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a3), zero
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a5, 24
-; RV32-NEXT:    mul a3, a3, a5
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    vlse64.v v8, (a4), zero
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 5
+; RV32-NEXT:    li a4, 24
+; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    lui a3, 4112
+; RV32-NEXT:    addi a3, a3, 257
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a3
 ; RV32-NEXT:    csrr a3, vlenb
 ; RV32-NEXT:    slli a3, a3, 3
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
-; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
-; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vadd.vv v16, v8, v16, v0.t
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    li a3, 24
-; RV32-NEXT:    mul a2, a2, a3
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    slli a2, a2, 5
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vmul.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    li a2, 56
 ; RV32-NEXT:    vsrl.vx v8, v8, a2, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
+; RV32-NEXT:    slli a3, a3, 4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    addi a3, a0, -16
 ; RV32-NEXT:    sltu a0, a0, a3
@@ -4950,15 +4574,15 @@ define <32 x i64> @vp_ctlz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV32-NEXT:    and a0, a0, a3
 ; RV32-NEXT:    vmv1r.v v0, v24
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 4
+; RV32-NEXT:    slli a3, a3, 5
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    addi a3, a3, 16
+; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v16, 1, v0.t
+; RV32-NEXT:    vsrl.vi v16, v8, 1, v0.t
+; RV32-NEXT:    vor.vv v16, v8, v16, v0.t
+; RV32-NEXT:    vsrl.vi v8, v16, 2, v0.t
 ; RV32-NEXT:    vor.vv v8, v16, v8, v0.t
-; RV32-NEXT:    vsrl.vi v16, v8, 2, v0.t
-; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v8, 8, v0.t
@@ -4968,71 +4592,55 @@ define <32 x i64> @vp_ctlz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV32-NEXT:    vsrl.vx v16, v8, a1, v0.t
 ; RV32-NEXT:    vor.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    vnot.v v8, v8, v0.t
-; RV32-NEXT:    addi a0, sp, 48
+; RV32-NEXT:    addi a0, sp, 16
 ; RV32-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsrl.vi v8, v8, 1, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
+; RV32-NEXT:    slli a0, a0, 5
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    li a1, 40
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
+; RV32-NEXT:    slli a0, a0, 5
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
-; RV32-NEXT:    addi a0, sp, 48
+; RV32-NEXT:    addi a0, sp, 16
 ; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsub.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 40
-; RV32-NEXT:    mul a0, a0, a1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    li a1, 48
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 40
-; RV32-NEXT:    mul a0, a0, a1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    li a1, 40
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    li a1, 48
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
+; RV32-NEXT:    li a1, 40
+; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
@@ -5041,27 +4649,27 @@ define <32 x i64> @vp_ctlz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV32-NEXT:    li a1, 24
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 5
+; RV32-NEXT:    slli a0, a0, 3
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vx v16, v8, a2, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 3
+; RV32-NEXT:    slli a0, a0, 4
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
+; RV32-NEXT:    .cfi_def_cfa sp, 16
+; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
@@ -5184,52 +4792,38 @@ define <32 x i64> @vp_ctlz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 define <32 x i64> @vp_ctlz_zero_undef_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_ctlz_zero_undef_v32i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
-; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
-; RV32-NEXT:    lui a1, 349525
-; RV32-NEXT:    lui a2, 209715
-; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
-; RV32-NEXT:    lui a1, 61681
-; RV32-NEXT:    addi a2, a2, 819
-; RV32-NEXT:    sw a2, 32(sp)
-; RV32-NEXT:    sw a2, 36(sp)
-; RV32-NEXT:    lui a2, 4112
-; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    li a3, 16
-; RV32-NEXT:    addi a1, a2, 257
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
+; RV32-NEXT:    li a2, 16
 ; RV32-NEXT:    mv a1, a0
-; RV32-NEXT:    bltu a0, a3, .LBB71_2
+; RV32-NEXT:    bltu a0, a2, .LBB71_2
 ; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    li a1, 16
 ; RV32-NEXT:  .LBB71_2:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a2, a2, 4
+; RV32-NEXT:    sub sp, sp, a2
+; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV32-NEXT:    vsrl.vi v0, v8, 1
 ; RV32-NEXT:    li a2, 32
-; RV32-NEXT:    addi a3, sp, 40
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a3), zero
+; RV32-NEXT:    lui a3, 349525
+; RV32-NEXT:    addi a3, a3, 1365
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a3
 ; RV32-NEXT:    csrr a3, vlenb
 ; RV32-NEXT:    slli a3, a3, 3
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vs8r.v v24, (a3) # Unknown-size Folded Spill
 ; RV32-NEXT:    addi a3, a0, -16
 ; RV32-NEXT:    sltu a0, a0, a3
 ; RV32-NEXT:    addi a0, a0, -1
 ; RV32-NEXT:    and a0, a0, a3
-; RV32-NEXT:    addi a3, sp, 32
+; RV32-NEXT:    lui a3, 209715
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV32-NEXT:    vor.vv v8, v8, v0
+; RV32-NEXT:    addi a3, a3, 819
 ; RV32-NEXT:    vsrl.vi v0, v8, 2
 ; RV32-NEXT:    vor.vv v8, v8, v0
 ; RV32-NEXT:    vsrl.vi v0, v8, 4
@@ -5256,90 +4850,74 @@ define <32 x i64> @vp_ctlz_zero_undef_v32i64_unmasked(<32 x i64> %va, i32 zeroex
 ; RV32-NEXT:    vsrl.vi v0, v16, 8
 ; RV32-NEXT:    vor.vv v16, v16, v0
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vnot.v v0, v8
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v16, 16
-; RV32-NEXT:    vor.vv v16, v16, v8
-; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v0, 1
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    slli a4, a4, 3
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 48
-; RV32-NEXT:    vl8r.v v24, (a4) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v24, v8, v24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a3), zero
-; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vv v24, v0, v24
+; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vx v0, v16, a2
+; RV32-NEXT:    vsrl.vi v0, v16, 16
 ; RV32-NEXT:    vor.vv v16, v16, v0
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v0, v24, v8
-; RV32-NEXT:    vsrl.vi v24, v24, 2
-; RV32-NEXT:    vand.vv v24, v24, v8
-; RV32-NEXT:    vadd.vv v24, v0, v24
-; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vs8r.v v24, (a2) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vnot.v v16, v16
-; RV32-NEXT:    vsrl.vi v0, v16, 1
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    slli a2, a2, 3
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vsrl.vi v0, v8, 1
 ; RV32-NEXT:    vand.vv v0, v0, v24
-; RV32-NEXT:    addi a2, sp, 24
-; RV32-NEXT:    addi a3, sp, 16
-; RV32-NEXT:    vsub.vv v0, v16, v0
-; RV32-NEXT:    addi a4, sp, 48
-; RV32-NEXT:    vl8r.v v24, (a4) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v24, 4
-; RV32-NEXT:    vadd.vv v16, v24, v16
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    slli a4, a4, 3
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 48
-; RV32-NEXT:    vs8r.v v16, (a4) # Unknown-size Folded Spill
+; RV32-NEXT:    vsub.vv v0, v8, v0
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v24, v0, v8
+; RV32-NEXT:    vsrl.vx v8, v16, a2
+; RV32-NEXT:    vor.vv v24, v16, v8
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a3
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v8, v0, v16
 ; RV32-NEXT:    vsrl.vi v0, v0, 2
-; RV32-NEXT:    vand.vv v8, v0, v8
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v0, (a2), zero
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v8, v24, v8
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a3), zero
+; RV32-NEXT:    vand.vv v0, v0, v16
+; RV32-NEXT:    vadd.vv v8, v8, v0
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v8, 4
-; RV32-NEXT:    vadd.vv v8, v8, v16
+; RV32-NEXT:    vnot.v v24, v24
+; RV32-NEXT:    vsrl.vi v0, v24, 1
 ; RV32-NEXT:    csrr a2, vlenb
 ; RV32-NEXT:    slli a2, a2, 3
 ; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
+; RV32-NEXT:    addi a2, a2, 16
 ; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v0, v0, v16
+; RV32-NEXT:    vsub.vv v24, v24, v0
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vsrl.vi v0, v8, 4
+; RV32-NEXT:    vadd.vv v8, v8, v0
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v0, v24, v16
+; RV32-NEXT:    vsrl.vi v24, v24, 2
+; RV32-NEXT:    vand.vv v16, v24, v16
+; RV32-NEXT:    lui a2, 61681
+; RV32-NEXT:    lui a3, 4112
+; RV32-NEXT:    addi a2, a2, -241
+; RV32-NEXT:    addi a3, a3, 257
+; RV32-NEXT:    vadd.vv v16, v0, v16
+; RV32-NEXT:    vsrl.vi v24, v16, 4
+; RV32-NEXT:    vadd.vv v16, v16, v24
+; RV32-NEXT:    vsetvli a4, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v16, v16, v0
+; RV32-NEXT:    vand.vv v8, v8, v24
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v8, v8, v0
+; RV32-NEXT:    vand.vv v16, v16, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a3
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vmul.vv v16, v16, v24
+; RV32-NEXT:    vmul.vv v8, v8, v24
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vmul.vv v24, v8, v24
+; RV32-NEXT:    vmul.vv v16, v16, v24
 ; RV32-NEXT:    li a2, 56
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vx v8, v16, a2
+; RV32-NEXT:    vsrl.vx v8, v8, a2
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vx v16, v24, a2
+; RV32-NEXT:    vsrl.vx v16, v16, a2
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    slli a0, a0, 4
 ; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
+; RV32-NEXT:    .cfi_def_cfa sp, 16
+; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop-vp.ll
index a5a1061842427..78421e8488600 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ctpop-vp.ll
@@ -1119,71 +1119,40 @@ declare <15 x i64> @llvm.vp.ctpop.v15i64(<15 x i64>, <15 x i1>, i32)
 define <15 x i64> @vp_ctpop_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_ctpop_v15i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 8 * vlenb
-; RV32-NEXT:    addi a1, sp, 48
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vsrl.vi v16, v8, 1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 32(sp)
-; RV32-NEXT:    sw a1, 36(sp)
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v24, v16, v24, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    lui a1, 4112
-; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
-; RV32-NEXT:    addi a1, sp, 40
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v8, 1, v0.t
-; RV32-NEXT:    addi a1, sp, 32
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a1), zero
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v24, v8, v24, v0.t
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsub.vv v8, v8, v24, v0.t
 ; RV32-NEXT:    vand.vv v24, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
 ; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    addi a1, sp, 48
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a1, sp, 16
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a1
+; RV32-NEXT:    lui a1, 4112
+; RV32-NEXT:    addi a1, a1, 257
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v24, v24, v16, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a1), zero
+; RV32-NEXT:    vadd.vv v16, v24, v16, v0.t
+; RV32-NEXT:    vsrl.vi v24, v16, 4, v0.t
+; RV32-NEXT:    vadd.vv v16, v16, v24, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
+; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v24, 4, v0.t
-; RV32-NEXT:    vadd.vv v8, v24, v8, v0.t
-; RV32-NEXT:    li a0, 56
-; RV32-NEXT:    addi a1, sp, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vx v8, v8, a0, v0.t
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 3
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vand.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vmul.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vx v8, v8, a1, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_ctpop_v15i64:
@@ -1226,54 +1195,40 @@ define <15 x i64> @vp_ctpop_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroext %ev
 define <15 x i64> @vp_ctpop_v15i64_unmasked(<15 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_ctpop_v15i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vsrl.vi v16, v8, 1
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v24, v16, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 8(sp)
-; RV32-NEXT:    sw a1, 12(sp)
-; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v8, 1
+; RV32-NEXT:    vsub.vv v8, v8, v24
+; RV32-NEXT:    vand.vv v24, v8, v16
+; RV32-NEXT:    vsrl.vi v8, v8, 2
+; RV32-NEXT:    vand.vv v8, v8, v16
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
+; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 0(sp)
-; RV32-NEXT:    sw a1, 4(sp)
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v0, (a1), zero
-; RV32-NEXT:    addi a1, sp, 16
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    addi a1, sp, 8
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v0, v16, v0
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a1), zero
-; RV32-NEXT:    mv a1, sp
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vv v8, v8, v0
-; RV32-NEXT:    vand.vv v0, v8, v24
-; RV32-NEXT:    vsrl.vi v8, v8, 2
-; RV32-NEXT:    vand.vv v8, v8, v24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
+; RV32-NEXT:    vadd.vv v8, v24, v8
+; RV32-NEXT:    vsrl.vi v24, v8, 4
+; RV32-NEXT:    vadd.vv v8, v8, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v8, v0, v8
-; RV32-NEXT:    vsrl.vi v0, v8, 4
-; RV32-NEXT:    vadd.vv v8, v8, v0
 ; RV32-NEXT:    vand.vv v8, v8, v16
 ; RV32-NEXT:    vmul.vv v8, v8, v24
 ; RV32-NEXT:    li a0, 56
 ; RV32-NEXT:    vsrl.vx v8, v8, a0
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_ctpop_v15i64_unmasked:
@@ -1318,71 +1273,40 @@ declare <16 x i64> @llvm.vp.ctpop.v16i64(<16 x i64>, <16 x i1>, i32)
 define <16 x i64> @vp_ctpop_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_ctpop_v16i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 8 * vlenb
-; RV32-NEXT:    addi a1, sp, 48
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vsrl.vi v16, v8, 1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 32(sp)
-; RV32-NEXT:    sw a1, 36(sp)
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v24, v16, v24, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    lui a1, 4112
-; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
-; RV32-NEXT:    addi a1, sp, 40
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v8, 1, v0.t
-; RV32-NEXT:    addi a1, sp, 32
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a1), zero
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v24, v8, v24, v0.t
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsub.vv v8, v8, v24, v0.t
 ; RV32-NEXT:    vand.vv v24, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
 ; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    addi a1, sp, 48
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a1, sp, 16
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a1
+; RV32-NEXT:    lui a1, 4112
+; RV32-NEXT:    addi a1, a1, 257
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v24, v24, v16, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a1), zero
+; RV32-NEXT:    vadd.vv v16, v24, v16, v0.t
+; RV32-NEXT:    vsrl.vi v24, v16, 4, v0.t
+; RV32-NEXT:    vadd.vv v16, v16, v24, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
+; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v24, 4, v0.t
-; RV32-NEXT:    vadd.vv v8, v24, v8, v0.t
-; RV32-NEXT:    li a0, 56
-; RV32-NEXT:    addi a1, sp, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vx v8, v8, a0, v0.t
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 3
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vand.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vmul.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vx v8, v8, a1, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_ctpop_v16i64:
@@ -1425,54 +1349,40 @@ define <16 x i64> @vp_ctpop_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroext %ev
 define <16 x i64> @vp_ctpop_v16i64_unmasked(<16 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_ctpop_v16i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vsrl.vi v16, v8, 1
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    lui a1, 209715
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v24, v16, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 8(sp)
-; RV32-NEXT:    sw a1, 12(sp)
-; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v8, 1
+; RV32-NEXT:    vsub.vv v8, v8, v24
+; RV32-NEXT:    vand.vv v24, v8, v16
+; RV32-NEXT:    vsrl.vi v8, v8, 2
+; RV32-NEXT:    vand.vv v8, v8, v16
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
+; RV32-NEXT:    lui a1, 4112
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 0(sp)
-; RV32-NEXT:    sw a1, 4(sp)
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v0, (a1), zero
-; RV32-NEXT:    addi a1, sp, 16
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    addi a1, sp, 8
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v0, v16, v0
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a1), zero
-; RV32-NEXT:    mv a1, sp
+; RV32-NEXT:    vadd.vv v8, v24, v8
+; RV32-NEXT:    vsrl.vi v24, v8, 4
+; RV32-NEXT:    vadd.vv v8, v8, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vv v8, v8, v0
-; RV32-NEXT:    vand.vv v0, v8, v24
-; RV32-NEXT:    vsrl.vi v8, v8, 2
-; RV32-NEXT:    vand.vv v8, v8, v24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v8, v0, v8
-; RV32-NEXT:    vsrl.vi v0, v8, 4
-; RV32-NEXT:    vadd.vv v8, v8, v0
 ; RV32-NEXT:    vand.vv v8, v8, v16
 ; RV32-NEXT:    vmul.vv v8, v8, v24
 ; RV32-NEXT:    li a0, 56
 ; RV32-NEXT:    vsrl.vx v8, v8, a0
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_ctpop_v16i64_unmasked:
@@ -1517,129 +1427,122 @@ declare <32 x i64> @llvm.vp.ctpop.v32i64(<32 x i64>, <32 x i1>, i32)
 define <32 x i64> @vp_ctpop_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_ctpop_v32i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    li a2, 48
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x30, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 48 * vlenb
+; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x30, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 48 * vlenb
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
+; RV32-NEXT:    li a2, 24
+; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
+; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    li a2, 16
 ; RV32-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV32-NEXT:    vslidedown.vi v7, v0, 2
-; RV32-NEXT:    lui a1, 349525
-; RV32-NEXT:    lui a2, 209715
-; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
-; RV32-NEXT:    lui a1, 61681
-; RV32-NEXT:    addi a2, a2, 819
-; RV32-NEXT:    sw a2, 32(sp)
-; RV32-NEXT:    sw a2, 36(sp)
-; RV32-NEXT:    lui a2, 4112
-; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    li a3, 16
-; RV32-NEXT:    addi a1, a2, 257
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
 ; RV32-NEXT:    mv a1, a0
-; RV32-NEXT:    bltu a0, a3, .LBB34_2
+; RV32-NEXT:    bltu a0, a2, .LBB34_2
 ; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    li a1, 16
 ; RV32-NEXT:  .LBB34_2:
-; RV32-NEXT:    addi a2, sp, 40
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a2), zero
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vsrl.vi v16, v8, 1, v0.t
 ; RV32-NEXT:    csrr a2, vlenb
 ; RV32-NEXT:    li a3, 40
 ; RV32-NEXT:    mul a2, a2, a3
 ; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
+; RV32-NEXT:    addi a2, a2, 16
 ; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a2, sp, 32
-; RV32-NEXT:    vlse64.v v16, (a2), zero
+; RV32-NEXT:    lui a2, 349525
+; RV32-NEXT:    addi a2, a2, 1365
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a2
 ; RV32-NEXT:    csrr a2, vlenb
 ; RV32-NEXT:    slli a2, a2, 5
 ; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
+; RV32-NEXT:    addi a2, a2, 16
 ; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v24, v8, 1, v0.t
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a2, a2, 5
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a2, vlenb
 ; RV32-NEXT:    li a3, 40
 ; RV32-NEXT:    mul a2, a2, a3
 ; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV32-NEXT:    vand.vv v16, v24, v16, v0.t
 ; RV32-NEXT:    vsub.vv v8, v8, v16, v0.t
+; RV32-NEXT:    lui a2, 209715
+; RV32-NEXT:    addi a2, a2, 819
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a2
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    slli a2, a2, 5
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v24, v8, v16, v0.t
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    li a3, 24
+; RV32-NEXT:    li a3, 40
 ; RV32-NEXT:    mul a2, a2, a3
 ; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vs8r.v v24, (a2) # Unknown-size Folded Spill
-; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    li a3, 24
+; RV32-NEXT:    li a3, 40
 ; RV32-NEXT:    mul a2, a2, a3
 ; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
+; RV32-NEXT:    addi a2, a2, 16
 ; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vadd.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    slli a2, a2, 3
+; RV32-NEXT:    slli a2, a2, 4
 ; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a2, sp, 24
-; RV32-NEXT:    addi a3, sp, 16
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a2), zero
-; RV32-NEXT:    addi a2, sp, 48
+; RV32-NEXT:    addi a2, a2, 16
 ; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
-; RV32-NEXT:    vlse64.v v8, (a3), zero
+; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    li a3, 24
+; RV32-NEXT:    li a3, 40
 ; RV32-NEXT:    mul a2, a2, a3
 ; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    slli a2, a2, 3
+; RV32-NEXT:    slli a2, a2, 4
 ; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vadd.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
+; RV32-NEXT:    vadd.vv v8, v8, v16, v0.t
+; RV32-NEXT:    lui a2, 61681
+; RV32-NEXT:    addi a2, a2, -241
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a2
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a2, a2, 4
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v24, v8, 4, v0.t
-; RV32-NEXT:    vadd.vv v8, v8, v24, v0.t
-; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 24
-; RV32-NEXT:    mul a1, a1, a2
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vmul.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    lui a2, 4112
+; RV32-NEXT:    addi a2, a2, 257
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a2
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    vsrl.vx v8, v8, a1, v0.t
 ; RV32-NEXT:    csrr a2, vlenb
 ; RV32-NEXT:    slli a2, a2, 3
 ; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
+; RV32-NEXT:    addi a2, a2, 16
 ; RV32-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
 ; RV32-NEXT:    addi a2, a0, -16
 ; RV32-NEXT:    sltu a0, a0, a2
@@ -1647,53 +1550,69 @@ define <32 x i64> @vp_ctpop_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %ev
 ; RV32-NEXT:    and a0, a0, a2
 ; RV32-NEXT:    vmv1r.v v0, v7
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    slli a2, a2, 4
+; RV32-NEXT:    li a3, 24
+; RV32-NEXT:    mul a2, a2, a3
 ; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v24, v16, 1, v0.t
+; RV32-NEXT:    vsrl.vi v24, v8, 1, v0.t
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 5
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v16, v24, v16, v0.t
+; RV32-NEXT:    vsub.vv v16, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    li a2, 40
 ; RV32-NEXT:    mul a0, a0, a2
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v24, v8, v0.t
-; RV32-NEXT:    vsub.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vand.vv v8, v16, v8, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    slli a0, a0, 5
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v24, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vadd.vv v8, v24, v8, v0.t
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
+; RV32-NEXT:    vsrl.vi v16, v16, 2, v0.t
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    li a2, 40
+; RV32-NEXT:    mul a0, a0, a2
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 5
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    vadd.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV32-NEXT:    vadd.vv v8, v8, v16, v0.t
-; RV32-NEXT:    addi a0, sp, 48
-; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a2, 24
-; RV32-NEXT:    mul a0, a0, a2
+; RV32-NEXT:    slli a0, a0, 4
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    addi a0, sp, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vx v16, v8, a1, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    slli a0, a0, 3
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    li a1, 48
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
+; RV32-NEXT:    .cfi_def_cfa sp, 16
+; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
@@ -1789,159 +1708,71 @@ define <32 x i64> @vp_ctpop_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %ev
 define <32 x i64> @vp_ctpop_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_ctpop_v32i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 24
-; RV32-NEXT:    mul a1, a1, a2
-; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 24 * vlenb
-; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
-; RV32-NEXT:    vmv8r.v v24, v16
-; RV32-NEXT:    lui a1, 349525
-; RV32-NEXT:    lui a2, 209715
-; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
-; RV32-NEXT:    lui a1, 61681
-; RV32-NEXT:    addi a2, a2, 819
-; RV32-NEXT:    sw a2, 32(sp)
-; RV32-NEXT:    sw a2, 36(sp)
-; RV32-NEXT:    lui a2, 4112
-; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    li a3, 16
-; RV32-NEXT:    addi a1, a2, 257
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
+; RV32-NEXT:    li a2, 16
 ; RV32-NEXT:    mv a1, a0
-; RV32-NEXT:    bltu a0, a3, .LBB35_2
+; RV32-NEXT:    bltu a0, a2, .LBB35_2
 ; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    li a1, 16
 ; RV32-NEXT:  .LBB35_2:
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v0, v8, 1
-; RV32-NEXT:    addi a2, sp, 40
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a2), zero
+; RV32-NEXT:    vsrl.vi v24, v8, 1
+; RV32-NEXT:    lui a2, 349525
+; RV32-NEXT:    addi a2, a2, 1365
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v0, a2
 ; RV32-NEXT:    addi a2, a0, -16
 ; RV32-NEXT:    sltu a0, a0, a2
 ; RV32-NEXT:    addi a0, a0, -1
 ; RV32-NEXT:    and a0, a0, a2
-; RV32-NEXT:    addi a2, sp, 32
+; RV32-NEXT:    lui a2, 209715
+; RV32-NEXT:    addi a2, a2, 819
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v0, v0, v16
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v0, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v0, (a2), zero
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    slli a2, a2, 3
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
-; RV32-NEXT:    vmv8r.v v8, v24
+; RV32-NEXT:    vand.vv v24, v24, v0
+; RV32-NEXT:    vsub.vv v8, v8, v24
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v24, v24, 1
-; RV32-NEXT:    vand.vv v16, v24, v16
-; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    slli a2, a2, 4
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    slli a2, a2, 3
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vv v16, v24, v16
-; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vsrl.vi v24, v16, 1
+; RV32-NEXT:    vand.vv v24, v24, v0
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v0, a2
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vv v8, v8, v24
+; RV32-NEXT:    vsub.vv v16, v16, v24
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v24, v16, v0
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    slli a2, a2, 3
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vs8r.v v24, (a2) # Unknown-size Folded Spill
-; RV32-NEXT:    vsrl.vi v16, v16, 2
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vand.vv v24, v8, v0
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    slli a2, a2, 4
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vs8r.v v24, (a2) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsrl.vi v8, v8, 2
-; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v16, v16, v0
+; RV32-NEXT:    vand.vv v8, v8, v0
+; RV32-NEXT:    vadd.vv v8, v24, v8
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v0, v8, v0
-; RV32-NEXT:    addi a2, sp, 24
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v16, v8, v16
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a2), zero
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v24, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v24, v24, v0
+; RV32-NEXT:    vand.vv v24, v16, v0
+; RV32-NEXT:    vsrl.vi v16, v16, 2
+; RV32-NEXT:    vand.vv v16, v16, v0
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v0, v16, 4
-; RV32-NEXT:    vadd.vv v16, v16, v0
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v0, (a2), zero
+; RV32-NEXT:    vsrl.vi v0, v8, 4
+; RV32-NEXT:    vadd.vv v8, v8, v0
+; RV32-NEXT:    lui a2, 61681
+; RV32-NEXT:    addi a2, a2, -241
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v24, 4
 ; RV32-NEXT:    vadd.vv v16, v24, v16
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    slli a2, a2, 4
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
-; RV32-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vsrl.vi v24, v16, 4
+; RV32-NEXT:    vadd.vv v16, v16, v24
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a2
+; RV32-NEXT:    lui a2, 4112
+; RV32-NEXT:    addi a2, a2, 257
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v24, v24, v8
+; RV32-NEXT:    vand.vv v8, v8, v24
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v8, v16, v8
+; RV32-NEXT:    vand.vv v16, v16, v24
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vmul.vv v16, v24, v0
+; RV32-NEXT:    vmul.vv v8, v8, v24
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vmul.vv v24, v8, v0
+; RV32-NEXT:    vmul.vv v16, v16, v24
 ; RV32-NEXT:    li a2, 56
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vx v8, v16, a2
+; RV32-NEXT:    vsrl.vx v8, v8, a2
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vx v16, v24, a2
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 24
-; RV32-NEXT:    mul a0, a0, a1
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vsrl.vx v16, v16, a2
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_ctpop_v32i64_unmasked:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz-vp.ll
index 0fb0fbe073e6b..96acd0aceab13 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz-vp.ll
@@ -9,9 +9,8 @@ declare <2 x i8> @llvm.vp.cttz.v2i8(<2 x i8>, i1 immarg, <2 x i1>, i32)
 define <2 x i8> @vp_cttz_v2i8(<2 x i8> %va, <2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v2i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    li a0, 85
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    vand.vv v8, v8, v9, v0.t
@@ -34,12 +33,11 @@ define <2 x i8> @vp_cttz_v2i8(<2 x i8> %va, <2 x i1> %m, i32 zeroext %evl) {
 define <2 x i8> @vp_cttz_v2i8_unmasked(<2 x i8> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v2i8_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v9, v8
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    li a0, 51
@@ -61,9 +59,8 @@ declare <4 x i8> @llvm.vp.cttz.v4i8(<4 x i8>, i1 immarg, <4 x i1>, i32)
 define <4 x i8> @vp_cttz_v4i8(<4 x i8> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    li a0, 85
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    vand.vv v8, v8, v9, v0.t
@@ -86,12 +83,11 @@ define <4 x i8> @vp_cttz_v4i8(<4 x i8> %va, <4 x i1> %m, i32 zeroext %evl) {
 define <4 x i8> @vp_cttz_v4i8_unmasked(<4 x i8> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v4i8_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v9, v8
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    li a0, 51
@@ -113,9 +109,8 @@ declare <8 x i8> @llvm.vp.cttz.v8i8(<8 x i8>, i1 immarg, <8 x i1>, i32)
 define <8 x i8> @vp_cttz_v8i8(<8 x i8> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v8i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    li a0, 85
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    vand.vv v8, v8, v9, v0.t
@@ -138,12 +133,11 @@ define <8 x i8> @vp_cttz_v8i8(<8 x i8> %va, <8 x i1> %m, i32 zeroext %evl) {
 define <8 x i8> @vp_cttz_v8i8_unmasked(<8 x i8> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v8i8_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v9, v8
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    li a0, 51
@@ -165,9 +159,8 @@ declare <16 x i8> @llvm.vp.cttz.v16i8(<16 x i8>, i1 immarg, <16 x i1>, i32)
 define <16 x i8> @vp_cttz_v16i8(<16 x i8> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v16i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    li a0, 85
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    vand.vv v8, v8, v9, v0.t
@@ -190,12 +183,11 @@ define <16 x i8> @vp_cttz_v16i8(<16 x i8> %va, <16 x i1> %m, i32 zeroext %evl) {
 define <16 x i8> @vp_cttz_v16i8_unmasked(<16 x i8> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v16i8_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v9, v8
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    li a0, 51
@@ -217,9 +209,8 @@ declare <2 x i16> @llvm.vp.cttz.v2i16(<2 x i16>, i1 immarg, <2 x i1>, i32)
 define <2 x i16> @vp_cttz_v2i16(<2 x i16> %va, <2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v2i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 5
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -249,13 +240,12 @@ define <2 x i16> @vp_cttz_v2i16(<2 x i16> %va, <2 x i1> %m, i32 zeroext %evl) {
 define <2 x i16> @vp_cttz_v2i16_unmasked(<2 x i16> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v2i16_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v9, v8
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    lui a0, 3
@@ -283,9 +273,8 @@ declare <4 x i16> @llvm.vp.cttz.v4i16(<4 x i16>, i1 immarg, <4 x i1>, i32)
 define <4 x i16> @vp_cttz_v4i16(<4 x i16> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v4i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 5
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -315,13 +304,12 @@ define <4 x i16> @vp_cttz_v4i16(<4 x i16> %va, <4 x i1> %m, i32 zeroext %evl) {
 define <4 x i16> @vp_cttz_v4i16_unmasked(<4 x i16> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v4i16_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v9, v8
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    lui a0, 3
@@ -349,9 +337,8 @@ declare <8 x i16> @llvm.vp.cttz.v8i16(<8 x i16>, i1 immarg, <8 x i1>, i32)
 define <8 x i16> @vp_cttz_v8i16(<8 x i16> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v8i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 5
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -381,13 +368,12 @@ define <8 x i16> @vp_cttz_v8i16(<8 x i16> %va, <8 x i1> %m, i32 zeroext %evl) {
 define <8 x i16> @vp_cttz_v8i16_unmasked(<8 x i16> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v8i16_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v9, v8
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    lui a0, 3
@@ -415,9 +401,8 @@ declare <16 x i16> @llvm.vp.cttz.v16i16(<16 x i16>, i1 immarg, <16 x i1>, i32)
 define <16 x i16> @vp_cttz_v16i16(<16 x i16> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v16i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vsub.vx v10, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v10, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 5
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -447,13 +432,12 @@ define <16 x i16> @vp_cttz_v16i16(<16 x i16> %va, <16 x i1> %m, i32 zeroext %evl
 define <16 x i16> @vp_cttz_v16i16_unmasked(<16 x i16> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v16i16_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vnot.v v10, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v10, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    vand.vv v8, v8, v10
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v10, v8
 ; CHECK-NEXT:    vsrl.vi v10, v8, 1
 ; CHECK-NEXT:    vand.vx v10, v10, a0
 ; CHECK-NEXT:    lui a0, 3
@@ -481,9 +465,8 @@ declare <2 x i32> @llvm.vp.cttz.v2i32(<2 x i32>, i1 immarg, <2 x i1>, i32)
 define <2 x i32> @vp_cttz_v2i32(<2 x i32> %va, <2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v2i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 349525
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -514,13 +497,12 @@ define <2 x i32> @vp_cttz_v2i32(<2 x i32> %va, <2 x i1> %m, i32 zeroext %evl) {
 define <2 x i32> @vp_cttz_v2i32_unmasked(<2 x i32> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v2i32_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 349525
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v9, v8
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    lui a0, 209715
@@ -549,9 +531,8 @@ declare <4 x i32> @llvm.vp.cttz.v4i32(<4 x i32>, i1 immarg, <4 x i1>, i32)
 define <4 x i32> @vp_cttz_v4i32(<4 x i32> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 349525
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -582,13 +563,12 @@ define <4 x i32> @vp_cttz_v4i32(<4 x i32> %va, <4 x i1> %m, i32 zeroext %evl) {
 define <4 x i32> @vp_cttz_v4i32_unmasked(<4 x i32> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v4i32_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 349525
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v9, v8
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    lui a0, 209715
@@ -617,9 +597,8 @@ declare <8 x i32> @llvm.vp.cttz.v8i32(<8 x i32>, i1 immarg, <8 x i1>, i32)
 define <8 x i32> @vp_cttz_v8i32(<8 x i32> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v8i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    vsub.vx v10, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v10, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 349525
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -650,13 +629,12 @@ define <8 x i32> @vp_cttz_v8i32(<8 x i32> %va, <8 x i1> %m, i32 zeroext %evl) {
 define <8 x i32> @vp_cttz_v8i32_unmasked(<8 x i32> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v8i32_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    vnot.v v10, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v10, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 349525
+; CHECK-NEXT:    vand.vv v8, v8, v10
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v10, v8
 ; CHECK-NEXT:    vsrl.vi v10, v8, 1
 ; CHECK-NEXT:    vand.vx v10, v10, a0
 ; CHECK-NEXT:    lui a0, 209715
@@ -685,9 +663,8 @@ declare <16 x i32> @llvm.vp.cttz.v16i32(<16 x i32>, i1 immarg, <16 x i1>, i32)
 define <16 x i32> @vp_cttz_v16i32(<16 x i32> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v16i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    vsub.vx v12, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v12, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 349525
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -718,13 +695,12 @@ define <16 x i32> @vp_cttz_v16i32(<16 x i32> %va, <16 x i1> %m, i32 zeroext %evl
 define <16 x i32> @vp_cttz_v16i32_unmasked(<16 x i32> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_v16i32_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    vnot.v v12, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v12, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 349525
+; CHECK-NEXT:    vand.vv v8, v8, v12
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v12, v8
 ; CHECK-NEXT:    vsrl.vi v12, v8, 1
 ; CHECK-NEXT:    vand.vx v12, v12, a0
 ; CHECK-NEXT:    lui a0, 209715
@@ -753,9 +729,8 @@ declare <2 x i64> @llvm.vp.cttz.v2i64(<2 x i64>, i1 immarg, <2 x i1>, i32)
 define <2 x i64> @vp_cttz_v2i64(<2 x i64> %va, <2 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_v2i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; RV32-NEXT:    vsub.vx v9, v8, a1, v0.t
+; RV32-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
@@ -795,39 +770,38 @@ define <2 x i64> @vp_cttz_v2i64(<2 x i64> %va, <2 x i1> %m, i32 zeroext %evl) {
 ;
 ; RV64-LABEL: vp_cttz_v2i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; RV64-NEXT:    vsub.vx v9, v8, a1, v0.t
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v9, v0.t
 ; RV64-NEXT:    vsrl.vi v9, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v9, v9, a0, v0.t
+; RV64-NEXT:    vand.vx v9, v9, a1, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v9, v0.t
-; RV64-NEXT:    vand.vx v9, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v9, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vadd.vv v8, v9, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v9, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v9, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    li a0, 56
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a4, v0.t
 ; RV64-NEXT:    vsrl.vx v8, v8, a0, v0.t
 ; RV64-NEXT:    ret
   %v = call <2 x i64> @llvm.vp.cttz.v2i64(<2 x i64> %va, i1 false, <2 x i1> %m, i32 %evl)
@@ -837,18 +811,17 @@ define <2 x i64> @vp_cttz_v2i64(<2 x i64> %va, <2 x i1> %m, i32 zeroext %evl) {
 define <2 x i64> @vp_cttz_v2i64_unmasked(<2 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_v2i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; RV32-NEXT:    vnot.v v9, v8
-; RV32-NEXT:    vsub.vx v8, v8, a1
+; RV32-NEXT:    vadd.vi v9, v8, -1
+; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV32-NEXT:    vmv.v.x v10, a1
 ; RV32-NEXT:    lui a1, 209715
-; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; RV32-NEXT:    vand.vv v8, v9, v8
+; RV32-NEXT:    vand.vv v8, v8, v9
+; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsrl.vi v9, v8, 1
 ; RV32-NEXT:    vand.vv v9, v9, v10
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
@@ -879,38 +852,37 @@ define <2 x i64> @vp_cttz_v2i64_unmasked(<2 x i64> %va, i32 zeroext %evl) {
 ;
 ; RV64-LABEL: vp_cttz_v2i64_unmasked:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; RV64-NEXT:    vsub.vx v9, v8, a1
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v9, v8, -1
 ; RV64-NEXT:    vnot.v v8, v8
 ; RV64-NEXT:    vand.vv v8, v8, v9
 ; RV64-NEXT:    vsrl.vi v9, v8, 1
-; RV64-NEXT:    vand.vx v9, v9, a0
+; RV64-NEXT:    vand.vx v9, v9, a1
 ; RV64-NEXT:    vsub.vv v8, v8, v9
-; RV64-NEXT:    vand.vx v9, v8, a1
+; RV64-NEXT:    vand.vx v9, v8, a2
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vand.vx v8, v8, a1
+; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    vadd.vv v8, v9, v8
 ; RV64-NEXT:    vsrl.vi v9, v8, 4
 ; RV64-NEXT:    vadd.vv v8, v8, v9
-; RV64-NEXT:    vand.vx v8, v8, a2
-; RV64-NEXT:    vmul.vx v8, v8, a3
+; RV64-NEXT:    vand.vx v8, v8, a3
+; RV64-NEXT:    vmul.vx v8, v8, a4
 ; RV64-NEXT:    li a0, 56
 ; RV64-NEXT:    vsrl.vx v8, v8, a0
 ; RV64-NEXT:    ret
@@ -923,9 +895,8 @@ declare <4 x i64> @llvm.vp.cttz.v4i64(<4 x i64>, i1 immarg, <4 x i1>, i32)
 define <4 x i64> @vp_cttz_v4i64(<4 x i64> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_v4i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; RV32-NEXT:    vsub.vx v10, v8, a1, v0.t
+; RV32-NEXT:    vadd.vi v10, v8, -1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
@@ -965,39 +936,38 @@ define <4 x i64> @vp_cttz_v4i64(<4 x i64> %va, <4 x i1> %m, i32 zeroext %evl) {
 ;
 ; RV64-LABEL: vp_cttz_v4i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; RV64-NEXT:    vsub.vx v10, v8, a1, v0.t
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v10, v8, -1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v10, v0.t
 ; RV64-NEXT:    vsrl.vi v10, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v10, v10, a0, v0.t
+; RV64-NEXT:    vand.vx v10, v10, a1, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v10, v0.t
-; RV64-NEXT:    vand.vx v10, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v10, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vadd.vv v8, v10, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v10, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v10, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    li a0, 56
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a4, v0.t
 ; RV64-NEXT:    vsrl.vx v8, v8, a0, v0.t
 ; RV64-NEXT:    ret
   %v = call <4 x i64> @llvm.vp.cttz.v4i64(<4 x i64> %va, i1 false, <4 x i1> %m, i32 %evl)
@@ -1007,18 +977,17 @@ define <4 x i64> @vp_cttz_v4i64(<4 x i64> %va, <4 x i1> %m, i32 zeroext %evl) {
 define <4 x i64> @vp_cttz_v4i64_unmasked(<4 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_v4i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; RV32-NEXT:    vnot.v v10, v8
-; RV32-NEXT:    vsub.vx v8, v8, a1
+; RV32-NEXT:    vadd.vi v10, v8, -1
+; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; RV32-NEXT:    vmv.v.x v12, a1
 ; RV32-NEXT:    lui a1, 209715
-; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; RV32-NEXT:    vand.vv v8, v10, v8
+; RV32-NEXT:    vand.vv v8, v8, v10
+; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsrl.vi v10, v8, 1
 ; RV32-NEXT:    vand.vv v10, v10, v12
 ; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
@@ -1049,38 +1018,37 @@ define <4 x i64> @vp_cttz_v4i64_unmasked(<4 x i64> %va, i32 zeroext %evl) {
 ;
 ; RV64-LABEL: vp_cttz_v4i64_unmasked:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; RV64-NEXT:    vsub.vx v10, v8, a1
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v10, v8, -1
 ; RV64-NEXT:    vnot.v v8, v8
 ; RV64-NEXT:    vand.vv v8, v8, v10
 ; RV64-NEXT:    vsrl.vi v10, v8, 1
-; RV64-NEXT:    vand.vx v10, v10, a0
+; RV64-NEXT:    vand.vx v10, v10, a1
 ; RV64-NEXT:    vsub.vv v8, v8, v10
-; RV64-NEXT:    vand.vx v10, v8, a1
+; RV64-NEXT:    vand.vx v10, v8, a2
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vand.vx v8, v8, a1
+; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    vadd.vv v8, v10, v8
 ; RV64-NEXT:    vsrl.vi v10, v8, 4
 ; RV64-NEXT:    vadd.vv v8, v8, v10
-; RV64-NEXT:    vand.vx v8, v8, a2
-; RV64-NEXT:    vmul.vx v8, v8, a3
+; RV64-NEXT:    vand.vx v8, v8, a3
+; RV64-NEXT:    vmul.vx v8, v8, a4
 ; RV64-NEXT:    li a0, 56
 ; RV64-NEXT:    vsrl.vx v8, v8, a0
 ; RV64-NEXT:    ret
@@ -1093,9 +1061,8 @@ declare <8 x i64> @llvm.vp.cttz.v8i64(<8 x i64>, i1 immarg, <8 x i1>, i32)
 define <8 x i64> @vp_cttz_v8i64(<8 x i64> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_v8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; RV32-NEXT:    vsub.vx v12, v8, a1, v0.t
+; RV32-NEXT:    vadd.vi v12, v8, -1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
@@ -1135,39 +1102,38 @@ define <8 x i64> @vp_cttz_v8i64(<8 x i64> %va, <8 x i1> %m, i32 zeroext %evl) {
 ;
 ; RV64-LABEL: vp_cttz_v8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; RV64-NEXT:    vsub.vx v12, v8, a1, v0.t
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v12, v8, -1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v12, v0.t
 ; RV64-NEXT:    vsrl.vi v12, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v12, v12, a0, v0.t
+; RV64-NEXT:    vand.vx v12, v12, a1, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v12, v0.t
-; RV64-NEXT:    vand.vx v12, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v12, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vadd.vv v8, v12, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v12, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v12, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    li a0, 56
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a4, v0.t
 ; RV64-NEXT:    vsrl.vx v8, v8, a0, v0.t
 ; RV64-NEXT:    ret
   %v = call <8 x i64> @llvm.vp.cttz.v8i64(<8 x i64> %va, i1 false, <8 x i1> %m, i32 %evl)
@@ -1177,18 +1143,17 @@ define <8 x i64> @vp_cttz_v8i64(<8 x i64> %va, <8 x i1> %m, i32 zeroext %evl) {
 define <8 x i64> @vp_cttz_v8i64_unmasked(<8 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_v8i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; RV32-NEXT:    vnot.v v12, v8
-; RV32-NEXT:    vsub.vx v8, v8, a1
+; RV32-NEXT:    vadd.vi v12, v8, -1
+; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 209715
-; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; RV32-NEXT:    vand.vv v8, v12, v8
+; RV32-NEXT:    vand.vv v8, v8, v12
+; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsrl.vi v12, v8, 1
 ; RV32-NEXT:    vand.vv v12, v12, v16
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
@@ -1219,38 +1184,37 @@ define <8 x i64> @vp_cttz_v8i64_unmasked(<8 x i64> %va, i32 zeroext %evl) {
 ;
 ; RV64-LABEL: vp_cttz_v8i64_unmasked:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; RV64-NEXT:    vsub.vx v12, v8, a1
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v12, v8, -1
 ; RV64-NEXT:    vnot.v v8, v8
 ; RV64-NEXT:    vand.vv v8, v8, v12
 ; RV64-NEXT:    vsrl.vi v12, v8, 1
-; RV64-NEXT:    vand.vx v12, v12, a0
+; RV64-NEXT:    vand.vx v12, v12, a1
 ; RV64-NEXT:    vsub.vv v8, v8, v12
-; RV64-NEXT:    vand.vx v12, v8, a1
+; RV64-NEXT:    vand.vx v12, v8, a2
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vand.vx v8, v8, a1
+; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    vadd.vv v8, v12, v8
 ; RV64-NEXT:    vsrl.vi v12, v8, 4
 ; RV64-NEXT:    vadd.vv v8, v8, v12
-; RV64-NEXT:    vand.vx v8, v8, a2
-; RV64-NEXT:    vmul.vx v8, v8, a3
+; RV64-NEXT:    vand.vx v8, v8, a3
+; RV64-NEXT:    vmul.vx v8, v8, a4
 ; RV64-NEXT:    li a0, 56
 ; RV64-NEXT:    vsrl.vx v8, v8, a0
 ; RV64-NEXT:    ret
@@ -1263,129 +1227,79 @@ declare <15 x i64> @llvm.vp.cttz.v15i64(<15 x i64>, i1 immarg, <15 x i1>, i32)
 define <15 x i64> @vp_cttz_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_v15i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
-; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v16, v8, a1, v0.t
+; RV32-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    lui a1, 209715
-; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 32(sp)
-; RV32-NEXT:    sw a1, 36(sp)
-; RV32-NEXT:    lui a1, 61681
-; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    lui a1, 4112
-; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
-; RV32-NEXT:    addi a1, sp, 40
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a1, sp, 32
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vnot.v v8, v8, v0.t
+; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vi v8, v16, 1, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    addi a1, sp, 48
-; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    vsub.vv v8, v16, v8, v0.t
-; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v8, v24, v0.t
-; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
 ; RV32-NEXT:    vand.vv v24, v8, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a1, sp, 16
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a1
+; RV32-NEXT:    lui a1, 61681
+; RV32-NEXT:    addi a1, a1, -241
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v24, v16, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a1), zero
+; RV32-NEXT:    vsub.vv v24, v16, v24, v0.t
+; RV32-NEXT:    vand.vv v16, v24, v8, v0.t
+; RV32-NEXT:    vsrl.vi v24, v24, 2, v0.t
+; RV32-NEXT:    vand.vv v24, v24, v8, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a1
+; RV32-NEXT:    lui a1, 4112
+; RV32-NEXT:    addi a1, a1, 257
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v24, 4, v0.t
-; RV32-NEXT:    vadd.vv v8, v24, v8, v0.t
-; RV32-NEXT:    li a0, 56
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vx v8, v8, a0, v0.t
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vadd.vv v16, v16, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v16, 4, v0.t
+; RV32-NEXT:    vadd.vv v16, v16, v24, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
+; RV32-NEXT:    li a1, 56
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vmul.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vx v8, v8, a1, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_cttz_v15i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v16, v8, a1, v0.t
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v16, v16, a0, v0.t
+; RV64-NEXT:    vand.vx v16, v16, a1, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v16, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v16, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    li a0, 56
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a4, v0.t
 ; RV64-NEXT:    vsrl.vx v8, v8, a0, v0.t
 ; RV64-NEXT:    ret
   %v = call <15 x i64> @llvm.vp.cttz.v15i64(<15 x i64> %va, i1 false, <15 x i1> %m, i32 %evl)
@@ -1395,94 +1309,78 @@ define <15 x i64> @vp_cttz_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroext %evl
 define <15 x i64> @vp_cttz_v15i64_unmasked(<15 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_v15i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v16, v8, a1
+; RV32-NEXT:    vadd.vi v16, v8, -1
+; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    lui a1, 209715
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v8, v8, v16
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
+; RV32-NEXT:    vsrl.vi v16, v8, 1
+; RV32-NEXT:    vand.vv v24, v16, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 8(sp)
-; RV32-NEXT:    sw a1, 12(sp)
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vsub.vv v8, v8, v24
+; RV32-NEXT:    vand.vv v24, v8, v16
+; RV32-NEXT:    vsrl.vi v8, v8, 2
+; RV32-NEXT:    vand.vv v8, v8, v16
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 4112
-; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 0(sp)
-; RV32-NEXT:    sw a1, 4(sp)
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v0, (a1), zero
-; RV32-NEXT:    addi a1, sp, 16
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    addi a1, sp, 8
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v16, v8, v16
-; RV32-NEXT:    vsrl.vi v8, v16, 1
-; RV32-NEXT:    vand.vv v0, v8, v0
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    mv a1, sp
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vv v16, v16, v0
-; RV32-NEXT:    vand.vv v0, v16, v24
-; RV32-NEXT:    vsrl.vi v16, v16, 2
-; RV32-NEXT:    vand.vv v16, v16, v24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
+; RV32-NEXT:    vadd.vv v8, v24, v8
+; RV32-NEXT:    vsrl.vi v24, v8, 4
+; RV32-NEXT:    vadd.vv v8, v8, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v16, v0, v16
-; RV32-NEXT:    vsrl.vi v0, v16, 4
-; RV32-NEXT:    vadd.vv v16, v16, v0
-; RV32-NEXT:    vand.vv v8, v16, v8
+; RV32-NEXT:    vand.vv v8, v8, v16
 ; RV32-NEXT:    vmul.vv v8, v8, v24
 ; RV32-NEXT:    li a0, 56
 ; RV32-NEXT:    vsrl.vx v8, v8, a0
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_cttz_v15i64_unmasked:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v16, v8, a1
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v16, v8, -1
 ; RV64-NEXT:    vnot.v v8, v8
 ; RV64-NEXT:    vand.vv v8, v8, v16
 ; RV64-NEXT:    vsrl.vi v16, v8, 1
-; RV64-NEXT:    vand.vx v16, v16, a0
+; RV64-NEXT:    vand.vx v16, v16, a1
 ; RV64-NEXT:    vsub.vv v8, v8, v16
-; RV64-NEXT:    vand.vx v16, v8, a1
+; RV64-NEXT:    vand.vx v16, v8, a2
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vand.vx v8, v8, a1
+; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    vadd.vv v8, v16, v8
 ; RV64-NEXT:    vsrl.vi v16, v8, 4
 ; RV64-NEXT:    vadd.vv v8, v8, v16
-; RV64-NEXT:    vand.vx v8, v8, a2
-; RV64-NEXT:    vmul.vx v8, v8, a3
+; RV64-NEXT:    vand.vx v8, v8, a3
+; RV64-NEXT:    vmul.vx v8, v8, a4
 ; RV64-NEXT:    li a0, 56
 ; RV64-NEXT:    vsrl.vx v8, v8, a0
 ; RV64-NEXT:    ret
@@ -1495,129 +1393,79 @@ declare <16 x i64> @llvm.vp.cttz.v16i64(<16 x i64>, i1 immarg, <16 x i1>, i32)
 define <16 x i64> @vp_cttz_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_v16i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
-; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v16, v8, a1, v0.t
+; RV32-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    lui a1, 209715
-; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 32(sp)
-; RV32-NEXT:    sw a1, 36(sp)
-; RV32-NEXT:    lui a1, 61681
-; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    lui a1, 4112
-; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
-; RV32-NEXT:    addi a1, sp, 40
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a1, sp, 32
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vnot.v v8, v8, v0.t
+; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vi v8, v16, 1, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    addi a1, sp, 48
-; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    vsub.vv v8, v16, v8, v0.t
-; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v8, v24, v0.t
-; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
 ; RV32-NEXT:    vand.vv v24, v8, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a1, sp, 16
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a1
+; RV32-NEXT:    lui a1, 61681
+; RV32-NEXT:    addi a1, a1, -241
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v24, v16, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a1), zero
+; RV32-NEXT:    vsub.vv v24, v16, v24, v0.t
+; RV32-NEXT:    vand.vv v16, v24, v8, v0.t
+; RV32-NEXT:    vsrl.vi v24, v24, 2, v0.t
+; RV32-NEXT:    vand.vv v24, v24, v8, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a1
+; RV32-NEXT:    lui a1, 4112
+; RV32-NEXT:    addi a1, a1, 257
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v24, 4, v0.t
-; RV32-NEXT:    vadd.vv v8, v24, v8, v0.t
-; RV32-NEXT:    li a0, 56
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vx v8, v8, a0, v0.t
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vadd.vv v16, v16, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v16, 4, v0.t
+; RV32-NEXT:    vadd.vv v16, v16, v24, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
+; RV32-NEXT:    li a1, 56
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vmul.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vx v8, v8, a1, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_cttz_v16i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v16, v8, a1, v0.t
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v16, v16, a0, v0.t
+; RV64-NEXT:    vand.vx v16, v16, a1, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v16, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v16, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    li a0, 56
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a4, v0.t
 ; RV64-NEXT:    vsrl.vx v8, v8, a0, v0.t
 ; RV64-NEXT:    ret
   %v = call <16 x i64> @llvm.vp.cttz.v16i64(<16 x i64> %va, i1 false, <16 x i1> %m, i32 %evl)
@@ -1627,94 +1475,78 @@ define <16 x i64> @vp_cttz_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroext %evl
 define <16 x i64> @vp_cttz_v16i64_unmasked(<16 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_v16i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v16, v8, a1
+; RV32-NEXT:    vadd.vi v16, v8, -1
+; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    lui a1, 209715
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v8, v8, v16
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
+; RV32-NEXT:    vsrl.vi v16, v8, 1
+; RV32-NEXT:    vand.vv v24, v16, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 8(sp)
-; RV32-NEXT:    sw a1, 12(sp)
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vsub.vv v8, v8, v24
+; RV32-NEXT:    vand.vv v24, v8, v16
+; RV32-NEXT:    vsrl.vi v8, v8, 2
+; RV32-NEXT:    vand.vv v8, v8, v16
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 4112
-; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 0(sp)
-; RV32-NEXT:    sw a1, 4(sp)
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v0, (a1), zero
-; RV32-NEXT:    addi a1, sp, 16
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    addi a1, sp, 8
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v16, v8, v16
-; RV32-NEXT:    vsrl.vi v8, v16, 1
-; RV32-NEXT:    vand.vv v0, v8, v0
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    mv a1, sp
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vv v16, v16, v0
-; RV32-NEXT:    vand.vv v0, v16, v24
-; RV32-NEXT:    vsrl.vi v16, v16, 2
-; RV32-NEXT:    vand.vv v16, v16, v24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
+; RV32-NEXT:    vadd.vv v8, v24, v8
+; RV32-NEXT:    vsrl.vi v24, v8, 4
+; RV32-NEXT:    vadd.vv v8, v8, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v16, v0, v16
-; RV32-NEXT:    vsrl.vi v0, v16, 4
-; RV32-NEXT:    vadd.vv v16, v16, v0
-; RV32-NEXT:    vand.vv v8, v16, v8
+; RV32-NEXT:    vand.vv v8, v8, v16
 ; RV32-NEXT:    vmul.vv v8, v8, v24
 ; RV32-NEXT:    li a0, 56
 ; RV32-NEXT:    vsrl.vx v8, v8, a0
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_cttz_v16i64_unmasked:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v16, v8, a1
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v16, v8, -1
 ; RV64-NEXT:    vnot.v v8, v8
 ; RV64-NEXT:    vand.vv v8, v8, v16
 ; RV64-NEXT:    vsrl.vi v16, v8, 1
-; RV64-NEXT:    vand.vx v16, v16, a0
+; RV64-NEXT:    vand.vx v16, v16, a1
 ; RV64-NEXT:    vsub.vv v8, v8, v16
-; RV64-NEXT:    vand.vx v16, v8, a1
+; RV64-NEXT:    vand.vx v16, v8, a2
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vand.vx v8, v8, a1
+; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    vadd.vv v8, v16, v8
 ; RV64-NEXT:    vsrl.vi v16, v8, 4
 ; RV64-NEXT:    vadd.vv v8, v8, v16
-; RV64-NEXT:    vand.vx v8, v8, a2
-; RV64-NEXT:    vmul.vx v8, v8, a3
+; RV64-NEXT:    vand.vx v8, v8, a3
+; RV64-NEXT:    vmul.vx v8, v8, a4
 ; RV64-NEXT:    li a0, 56
 ; RV64-NEXT:    vsrl.vx v8, v8, a0
 ; RV64-NEXT:    ret
@@ -1727,263 +1559,243 @@ declare <32 x i64> @llvm.vp.cttz.v32i64(<32 x i64>, i1 immarg, <32 x i1>, i32)
 define <32 x i64> @vp_cttz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_v32i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 48
+; RV32-NEXT:    li a2, 56
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x30, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 48 * vlenb
+; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x38, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 56 * vlenb
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
+; RV32-NEXT:    slli a1, a1, 5
 ; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
+; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    li a2, 16
 ; RV32-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
-; RV32-NEXT:    vslidedown.vi v7, v0, 2
-; RV32-NEXT:    lui a1, 349525
-; RV32-NEXT:    lui a2, 209715
-; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
-; RV32-NEXT:    lui a1, 61681
-; RV32-NEXT:    addi a2, a2, 819
-; RV32-NEXT:    sw a2, 32(sp)
-; RV32-NEXT:    sw a2, 36(sp)
-; RV32-NEXT:    lui a2, 4112
-; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    li a1, 16
-; RV32-NEXT:    addi a2, a2, 257
-; RV32-NEXT:    sw a2, 16(sp)
-; RV32-NEXT:    sw a2, 20(sp)
-; RV32-NEXT:    mv a2, a0
-; RV32-NEXT:    bltu a0, a1, .LBB34_2
+; RV32-NEXT:    vslidedown.vi v24, v0, 2
+; RV32-NEXT:    mv a1, a0
+; RV32-NEXT:    bltu a0, a2, .LBB34_2
 ; RV32-NEXT:  # %bb.1:
-; RV32-NEXT:    li a2, 16
+; RV32-NEXT:    li a1, 16
 ; RV32-NEXT:  .LBB34_2:
-; RV32-NEXT:    li a1, 1
-; RV32-NEXT:    addi a3, sp, 40
-; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v16, v8, a1, v0.t
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vadd.vi v16, v8, -1, v0.t
+; RV32-NEXT:    lui a2, 349525
 ; RV32-NEXT:    vnot.v v8, v8, v0.t
+; RV32-NEXT:    addi a2, a2, 1365
 ; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    li a5, 24
-; RV32-NEXT:    mul a4, a4, a5
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 48
-; RV32-NEXT:    vs8r.v v8, (a4) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a3), zero
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 40
-; RV32-NEXT:    mul a3, a3, a4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a3, sp, 32
-; RV32-NEXT:    vlse64.v v8, (a3), zero
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 5
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v16, 1, v0.t
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 40
-; RV32-NEXT:    mul a3, a3, a4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v24, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v24, v24, v16, v0.t
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vsub.vv v16, v16, v24, v0.t
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vsrl.vi v16, v16, 2, v0.t
-; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vadd.vv v8, v8, v16, v0.t
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a3, sp, 24
-; RV32-NEXT:    addi a4, sp, 16
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a3), zero
-; RV32-NEXT:    addi a3, sp, 48
-; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    vlse64.v v8, (a4), zero
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
+; RV32-NEXT:    li a4, 48
 ; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v24, v8, 4, v0.t
-; RV32-NEXT:    vadd.vv v8, v8, v24, v0.t
-; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a2
 ; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    li a3, 24
+; RV32-NEXT:    li a3, 40
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 48
 ; RV32-NEXT:    mul a2, a2, a3
 ; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
+; RV32-NEXT:    addi a2, a2, 16
 ; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vmul.vv v8, v16, v8, v0.t
-; RV32-NEXT:    li a2, 56
-; RV32-NEXT:    vsrl.vx v8, v8, a2, v0.t
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a3, a0, -16
-; RV32-NEXT:    sltu a0, a0, a3
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vsrl.vi v8, v8, 1, v0.t
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 24
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 40
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 24
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 48
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vsub.vv v8, v8, v16, v0.t
+; RV32-NEXT:    lui a2, 209715
+; RV32-NEXT:    addi a2, a2, 819
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a2
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 48
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 48
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 24
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 48
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 24
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vadd.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
+; RV32-NEXT:    vadd.vv v8, v8, v16, v0.t
+; RV32-NEXT:    lui a2, 61681
+; RV32-NEXT:    addi a2, a2, -241
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a2
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 24
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    lui a2, 4112
+; RV32-NEXT:    addi a2, a2, 257
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a2
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a2, a2, 3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
+; RV32-NEXT:    li a1, 56
+; RV32-NEXT:    vsrl.vx v8, v8, a1, v0.t
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a2, a2, 4
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    addi a2, a0, -16
+; RV32-NEXT:    sltu a0, a0, a2
 ; RV32-NEXT:    addi a0, a0, -1
-; RV32-NEXT:    and a0, a0, a3
-; RV32-NEXT:    vmv1r.v v0, v7
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    and a0, a0, a2
+; RV32-NEXT:    vmv1r.v v0, v24
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a2, a2, 5
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v8, v16, a1, v0.t
-; RV32-NEXT:    vnot.v v16, v16, v0.t
-; RV32-NEXT:    vand.vv v8, v16, v8, v0.t
-; RV32-NEXT:    vsrl.vi v24, v8, 1, v0.t
+; RV32-NEXT:    vadd.vi v16, v8, -1, v0.t
+; RV32-NEXT:    vnot.v v8, v8, v0.t
+; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
+; RV32-NEXT:    vsrl.vi v8, v8, 1, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 40
-; RV32-NEXT:    mul a0, a0, a1
+; RV32-NEXT:    slli a0, a0, 5
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v24, v16, v0.t
-; RV32-NEXT:    vsub.vv v8, v8, v16, v0.t
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 40
-; RV32-NEXT:    mul a0, a0, a1
+; RV32-NEXT:    li a2, 40
+; RV32-NEXT:    mul a0, a0, a2
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    slli a0, a0, 5
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
+; RV32-NEXT:    addi a0, sp, 16
 ; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    vsub.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 40
-; RV32-NEXT:    mul a0, a0, a1
+; RV32-NEXT:    li a2, 48
+; RV32-NEXT:    mul a0, a0, a2
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
+; RV32-NEXT:    li a2, 40
+; RV32-NEXT:    mul a0, a0, a2
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 40
-; RV32-NEXT:    mul a0, a0, a1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 5
+; RV32-NEXT:    li a2, 48
+; RV32-NEXT:    mul a0, a0, a2
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
+; RV32-NEXT:    li a2, 40
+; RV32-NEXT:    mul a0, a0, a2
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV32-NEXT:    vadd.vv v8, v8, v16, v0.t
-; RV32-NEXT:    addi a0, sp, 48
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    li a2, 24
+; RV32-NEXT:    mul a0, a0, a2
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 24
-; RV32-NEXT:    mul a0, a0, a1
+; RV32-NEXT:    slli a0, a0, 3
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vx v16, v8, a2, v0.t
+; RV32-NEXT:    vsrl.vx v16, v8, a1, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 3
+; RV32-NEXT:    slli a0, a0, 4
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 48
+; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
+; RV32-NEXT:    .cfi_def_cfa sp, 16
+; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
@@ -2000,75 +1812,74 @@ define <32 x i64> @vp_cttz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
 ; RV64-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; RV64-NEXT:    li a1, 16
+; RV64-NEXT:    li a2, 16
 ; RV64-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64-NEXT:    vslidedown.vi v24, v0, 2
-; RV64-NEXT:    mv a4, a0
-; RV64-NEXT:    bltu a0, a1, .LBB34_2
+; RV64-NEXT:    mv a1, a0
+; RV64-NEXT:    bltu a0, a2, .LBB34_2
 ; RV64-NEXT:  # %bb.1:
-; RV64-NEXT:    li a4, 16
+; RV64-NEXT:    li a1, 16
 ; RV64-NEXT:  .LBB34_2:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a5, 61681
-; RV64-NEXT:    lui a6, 4112
-; RV64-NEXT:    addiw a2, a2, 1365
-; RV64-NEXT:    addiw a3, a3, 819
-; RV64-NEXT:    addiw a7, a5, -241
-; RV64-NEXT:    addiw t0, a6, 257
-; RV64-NEXT:    slli a6, a2, 32
-; RV64-NEXT:    add a6, a2, a6
-; RV64-NEXT:    slli a5, a3, 32
-; RV64-NEXT:    add a5, a3, a5
-; RV64-NEXT:    slli a2, a7, 32
-; RV64-NEXT:    add a2, a7, a2
-; RV64-NEXT:    slli a3, t0, 32
-; RV64-NEXT:    add a3, t0, a3
-; RV64-NEXT:    addi a7, a0, -16
-; RV64-NEXT:    sltu a0, a0, a7
+; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV64-NEXT:    vadd.vi v16, v8, -1, v0.t
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a5, a3, -241
+; RV64-NEXT:    addiw a6, a4, 257
+; RV64-NEXT:    slli a4, a1, 32
+; RV64-NEXT:    add a4, a1, a4
+; RV64-NEXT:    slli a3, a2, 32
+; RV64-NEXT:    add a3, a2, a3
+; RV64-NEXT:    slli a1, a5, 32
+; RV64-NEXT:    add a1, a5, a1
+; RV64-NEXT:    slli a2, a6, 32
+; RV64-NEXT:    add a2, a6, a2
+; RV64-NEXT:    addi a5, a0, -16
+; RV64-NEXT:    sltu a0, a0, a5
 ; RV64-NEXT:    addi a0, a0, -1
-; RV64-NEXT:    and a7, a0, a7
+; RV64-NEXT:    and a5, a0, a5
 ; RV64-NEXT:    li a0, 56
-; RV64-NEXT:    vsetvli zero, a4, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v16, v8, a1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v16, v16, a6, v0.t
+; RV64-NEXT:    vand.vx v16, v16, a4, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v16, v8, a5, v0.t
+; RV64-NEXT:    vand.vx v16, v8, a3, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a5, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vx v8, v8, a0, v0.t
-; RV64-NEXT:    addi a4, sp, 16
-; RV64-NEXT:    vs8r.v v8, (a4) # Unknown-size Folded Spill
+; RV64-NEXT:    addi a6, sp, 16
+; RV64-NEXT:    vs8r.v v8, (a6) # Unknown-size Folded Spill
 ; RV64-NEXT:    vmv1r.v v0, v24
-; RV64-NEXT:    csrr a4, vlenb
-; RV64-NEXT:    slli a4, a4, 3
-; RV64-NEXT:    add a4, sp, a4
-; RV64-NEXT:    addi a4, a4, 16
-; RV64-NEXT:    vl8r.v v8, (a4) # Unknown-size Folded Reload
-; RV64-NEXT:    vsetvli zero, a7, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v16, v8, a1, v0.t
+; RV64-NEXT:    csrr a6, vlenb
+; RV64-NEXT:    slli a6, a6, 3
+; RV64-NEXT:    add a6, sp, a6
+; RV64-NEXT:    addi a6, a6, 16
+; RV64-NEXT:    vl8r.v v8, (a6) # Unknown-size Folded Reload
+; RV64-NEXT:    vsetvli zero, a5, e64, m8, ta, ma
+; RV64-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v16, v16, a6, v0.t
+; RV64-NEXT:    vand.vx v16, v16, a4, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v16, v8, a5, v0.t
+; RV64-NEXT:    vand.vx v16, v8, a3, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a5, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vx v16, v8, a0, v0.t
 ; RV64-NEXT:    addi a0, sp, 16
 ; RV64-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
@@ -2086,108 +1897,101 @@ define <32 x i64> @vp_cttz_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl
 define <32 x i64> @vp_cttz_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_v32i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 8 * vlenb
-; RV32-NEXT:    lui a1, 349525
-; RV32-NEXT:    lui a2, 209715
-; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
-; RV32-NEXT:    lui a1, 61681
-; RV32-NEXT:    addi a2, a2, 819
-; RV32-NEXT:    sw a2, 32(sp)
-; RV32-NEXT:    sw a2, 36(sp)
-; RV32-NEXT:    lui a2, 4112
-; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    li a3, 16
-; RV32-NEXT:    addi a1, a2, 257
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
+; RV32-NEXT:    li a2, 16
 ; RV32-NEXT:    mv a1, a0
-; RV32-NEXT:    bltu a0, a3, .LBB35_2
+; RV32-NEXT:    bltu a0, a2, .LBB35_2
 ; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    li a1, 16
 ; RV32-NEXT:  .LBB35_2:
-; RV32-NEXT:    li a2, 1
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a2, a2, 4
+; RV32-NEXT:    sub sp, sp, a2
+; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vadd.vi v24, v8, -1
 ; RV32-NEXT:    vnot.v v0, v8
-; RV32-NEXT:    addi a3, sp, 40
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a3), zero
-; RV32-NEXT:    addi a3, a0, -16
-; RV32-NEXT:    sltu a0, a0, a3
+; RV32-NEXT:    lui a2, 349525
+; RV32-NEXT:    addi a2, a2, 1365
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a2
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a2, a2, 3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    addi a2, a0, -16
+; RV32-NEXT:    sltu a0, a0, a2
 ; RV32-NEXT:    addi a0, a0, -1
-; RV32-NEXT:    and a0, a0, a3
-; RV32-NEXT:    addi a3, sp, 32
+; RV32-NEXT:    and a0, a0, a2
+; RV32-NEXT:    lui a2, 209715
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v8, v8, a2
-; RV32-NEXT:    vand.vv v8, v0, v8
+; RV32-NEXT:    vand.vv v24, v0, v24
+; RV32-NEXT:    addi a2, a2, 819
+; RV32-NEXT:    vsrl.vi v0, v24, 1
+; RV32-NEXT:    vand.vv v0, v0, v8
+; RV32-NEXT:    vsub.vv v24, v24, v0
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v0, v16, a2
+; RV32-NEXT:    vadd.vi v0, v16, -1
 ; RV32-NEXT:    vnot.v v16, v16
-; RV32-NEXT:    vand.vv v16, v16, v0
+; RV32-NEXT:    vand.vv v0, v16, v0
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a2
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v0, v8, 1
-; RV32-NEXT:    vand.vv v0, v0, v24
-; RV32-NEXT:    vsub.vv v0, v8, v0
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v16, 1
-; RV32-NEXT:    vand.vv v24, v8, v24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a3), zero
-; RV32-NEXT:    addi a2, sp, 24
-; RV32-NEXT:    addi a3, sp, 16
+; RV32-NEXT:    vand.vv v8, v24, v16
+; RV32-NEXT:    vsrl.vi v24, v24, 2
+; RV32-NEXT:    vand.vv v24, v24, v16
+; RV32-NEXT:    vadd.vv v8, v8, v24
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vv v16, v16, v24
+; RV32-NEXT:    vsrl.vi v24, v0, 1
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a2, a2, 3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v24, v24, v16
+; RV32-NEXT:    vsub.vv v24, v0, v24
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v24, v0, v8
-; RV32-NEXT:    vsrl.vi v0, v0, 2
-; RV32-NEXT:    vand.vv v0, v0, v8
-; RV32-NEXT:    vadd.vv v24, v24, v0
+; RV32-NEXT:    vsrl.vi v0, v8, 4
+; RV32-NEXT:    vadd.vv v8, v8, v0
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v0, v16, v8
-; RV32-NEXT:    vsrl.vi v16, v16, 2
-; RV32-NEXT:    vand.vv v8, v16, v8
+; RV32-NEXT:    vand.vv v0, v24, v16
+; RV32-NEXT:    vsrl.vi v24, v24, 2
+; RV32-NEXT:    vand.vv v16, v24, v16
+; RV32-NEXT:    lui a2, 61681
+; RV32-NEXT:    addi a2, a2, -241
+; RV32-NEXT:    vadd.vv v16, v0, v16
+; RV32-NEXT:    vsrl.vi v24, v16, 4
+; RV32-NEXT:    vadd.vv v16, v16, v24
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a2
+; RV32-NEXT:    lui a2, 4112
+; RV32-NEXT:    addi a2, a2, 257
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v24, 4
-; RV32-NEXT:    vadd.vv v16, v24, v16
-; RV32-NEXT:    addi a4, sp, 48
-; RV32-NEXT:    vs8r.v v16, (a4) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a2), zero
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v8, v0, v8
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v0, (a3), zero
+; RV32-NEXT:    vand.vv v8, v8, v24
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v8, 4
-; RV32-NEXT:    vadd.vv v8, v8, v16
-; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV32-NEXT:    vand.vv v16, v16, v24
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v8, v8, v24
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vmul.vv v16, v16, v0
+; RV32-NEXT:    vmul.vv v8, v8, v24
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vmul.vv v24, v8, v0
+; RV32-NEXT:    vmul.vv v16, v16, v24
 ; RV32-NEXT:    li a2, 56
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vx v8, v16, a2
+; RV32-NEXT:    vsrl.vx v8, v8, a2
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vx v16, v24, a2
+; RV32-NEXT:    vsrl.vx v16, v16, a2
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 3
+; RV32-NEXT:    slli a0, a0, 4
 ; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
+; RV32-NEXT:    .cfi_def_cfa sp, 16
+; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
@@ -2199,70 +2003,69 @@ define <32 x i64> @vp_cttz_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 ; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    li a1, 16
 ; RV64-NEXT:  .LBB35_2:
-; RV64-NEXT:    li a2, 1
 ; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vnot.v v24, v8
-; RV64-NEXT:    lui a3, 349525
-; RV64-NEXT:    lui a4, 209715
-; RV64-NEXT:    lui a5, 61681
-; RV64-NEXT:    lui a6, 4112
-; RV64-NEXT:    addiw a3, a3, 1365
-; RV64-NEXT:    addiw a4, a4, 819
-; RV64-NEXT:    addiw a5, a5, -241
-; RV64-NEXT:    addiw a6, a6, 257
-; RV64-NEXT:    slli a7, a3, 32
-; RV64-NEXT:    add a3, a3, a7
-; RV64-NEXT:    slli a7, a4, 32
-; RV64-NEXT:    add a4, a4, a7
-; RV64-NEXT:    slli a7, a5, 32
-; RV64-NEXT:    add a5, a5, a7
-; RV64-NEXT:    slli a7, a6, 32
-; RV64-NEXT:    add a6, a6, a7
-; RV64-NEXT:    addi a7, a0, -16
-; RV64-NEXT:    sltu a0, a0, a7
+; RV64-NEXT:    vadd.vi v24, v8, -1
+; RV64-NEXT:    vnot.v v8, v8
+; RV64-NEXT:    lui a2, 349525
+; RV64-NEXT:    lui a3, 209715
+; RV64-NEXT:    lui a4, 61681
+; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    addiw a2, a2, 1365
+; RV64-NEXT:    addiw a3, a3, 819
+; RV64-NEXT:    addiw a4, a4, -241
+; RV64-NEXT:    addiw a5, a5, 257
+; RV64-NEXT:    slli a6, a2, 32
+; RV64-NEXT:    add a2, a2, a6
+; RV64-NEXT:    slli a6, a3, 32
+; RV64-NEXT:    add a3, a3, a6
+; RV64-NEXT:    slli a6, a4, 32
+; RV64-NEXT:    add a4, a4, a6
+; RV64-NEXT:    slli a6, a5, 32
+; RV64-NEXT:    add a5, a5, a6
+; RV64-NEXT:    addi a6, a0, -16
+; RV64-NEXT:    sltu a0, a0, a6
 ; RV64-NEXT:    addi a0, a0, -1
-; RV64-NEXT:    and a0, a0, a7
-; RV64-NEXT:    li a7, 56
-; RV64-NEXT:    vsub.vx v8, v8, a2
-; RV64-NEXT:    vand.vv v8, v24, v8
+; RV64-NEXT:    and a0, a0, a6
+; RV64-NEXT:    li a6, 56
+; RV64-NEXT:    vand.vv v8, v8, v24
 ; RV64-NEXT:    vsrl.vi v24, v8, 1
-; RV64-NEXT:    vand.vx v24, v24, a3
+; RV64-NEXT:    vand.vx v24, v24, a2
 ; RV64-NEXT:    vsub.vv v8, v8, v24
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v24, v16, a2
+; RV64-NEXT:    vadd.vi v24, v16, -1
 ; RV64-NEXT:    vnot.v v16, v16
 ; RV64-NEXT:    vand.vv v16, v16, v24
 ; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vand.vx v24, v8, a4
+; RV64-NEXT:    vand.vx v24, v8, a3
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vand.vx v8, v8, a4
+; RV64-NEXT:    vand.vx v8, v8, a3
 ; RV64-NEXT:    vadd.vv v8, v24, v8
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vsrl.vi v24, v16, 1
-; RV64-NEXT:    vand.vx v24, v24, a3
+; RV64-NEXT:    vand.vx v24, v24, a2
 ; RV64-NEXT:    vsub.vv v16, v16, v24
 ; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV64-NEXT:    vsrl.vi v24, v8, 4
 ; RV64-NEXT:    vadd.vv v8, v8, v24
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV64-NEXT:    vand.vx v24, v16, a4
+; RV64-NEXT:    vand.vx v24, v16, a3
 ; RV64-NEXT:    vsrl.vi v16, v16, 2
-; RV64-NEXT:    vand.vx v16, v16, a4
+; RV64-NEXT:    vand.vx v16, v16, a3
 ; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vand.vx v8, v8, a5
+; RV64-NEXT:    vand.vx v8, v8, a4
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vadd.vv v16, v24, v16
 ; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vmul.vx v8, v8, a6
+; RV64-NEXT:    vmul.vx v8, v8, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vsrl.vi v24, v16, 4
 ; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsrl.vx v8, v8, a7
+; RV64-NEXT:    vsrl.vx v8, v8, a6
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vadd.vv v16, v16, v24
-; RV64-NEXT:    vand.vx v16, v16, a5
-; RV64-NEXT:    vmul.vx v16, v16, a6
-; RV64-NEXT:    vsrl.vx v16, v16, a7
+; RV64-NEXT:    vand.vx v16, v16, a4
+; RV64-NEXT:    vmul.vx v16, v16, a5
+; RV64-NEXT:    vsrl.vx v16, v16, a6
 ; RV64-NEXT:    ret
   %v = call <32 x i64> @llvm.vp.cttz.v32i64(<32 x i64> %va, i1 false, <32 x i1> splat (i1 true), i32 %evl)
   ret <32 x i64> %v
@@ -2271,9 +2074,8 @@ define <32 x i64> @vp_cttz_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 define <2 x i8> @vp_cttz_zero_undef_v2i8(<2 x i8> %va, <2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v2i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    li a0, 85
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    vand.vv v8, v8, v9, v0.t
@@ -2296,12 +2098,11 @@ define <2 x i8> @vp_cttz_zero_undef_v2i8(<2 x i8> %va, <2 x i1> %m, i32 zeroext
 define <2 x i8> @vp_cttz_zero_undef_v2i8_unmasked(<2 x i8> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v2i8_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v9, v8
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    li a0, 51
@@ -2321,9 +2122,8 @@ define <2 x i8> @vp_cttz_zero_undef_v2i8_unmasked(<2 x i8> %va, i32 zeroext %evl
 define <4 x i8> @vp_cttz_zero_undef_v4i8(<4 x i8> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    li a0, 85
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    vand.vv v8, v8, v9, v0.t
@@ -2346,12 +2146,11 @@ define <4 x i8> @vp_cttz_zero_undef_v4i8(<4 x i8> %va, <4 x i1> %m, i32 zeroext
 define <4 x i8> @vp_cttz_zero_undef_v4i8_unmasked(<4 x i8> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v4i8_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v9, v8
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    li a0, 51
@@ -2371,9 +2170,8 @@ define <4 x i8> @vp_cttz_zero_undef_v4i8_unmasked(<4 x i8> %va, i32 zeroext %evl
 define <8 x i8> @vp_cttz_zero_undef_v8i8(<8 x i8> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v8i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    li a0, 85
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    vand.vv v8, v8, v9, v0.t
@@ -2396,12 +2194,11 @@ define <8 x i8> @vp_cttz_zero_undef_v8i8(<8 x i8> %va, <8 x i1> %m, i32 zeroext
 define <8 x i8> @vp_cttz_zero_undef_v8i8_unmasked(<8 x i8> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v8i8_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v9, v8
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    li a0, 51
@@ -2421,9 +2218,8 @@ define <8 x i8> @vp_cttz_zero_undef_v8i8_unmasked(<8 x i8> %va, i32 zeroext %evl
 define <16 x i8> @vp_cttz_zero_undef_v16i8(<16 x i8> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v16i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    li a0, 85
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    vand.vv v8, v8, v9, v0.t
@@ -2446,12 +2242,11 @@ define <16 x i8> @vp_cttz_zero_undef_v16i8(<16 x i8> %va, <16 x i1> %m, i32 zero
 define <16 x i8> @vp_cttz_zero_undef_v16i8_unmasked(<16 x i8> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v16i8_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    li a0, 85
-; CHECK-NEXT:    vand.vv v8, v9, v8
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    li a0, 51
@@ -2471,9 +2266,8 @@ define <16 x i8> @vp_cttz_zero_undef_v16i8_unmasked(<16 x i8> %va, i32 zeroext %
 define <2 x i16> @vp_cttz_zero_undef_v2i16(<2 x i16> %va, <2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v2i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 5
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -2503,13 +2297,12 @@ define <2 x i16> @vp_cttz_zero_undef_v2i16(<2 x i16> %va, <2 x i1> %m, i32 zeroe
 define <2 x i16> @vp_cttz_zero_undef_v2i16_unmasked(<2 x i16> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v2i16_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v9, v8
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    lui a0, 3
@@ -2535,9 +2328,8 @@ define <2 x i16> @vp_cttz_zero_undef_v2i16_unmasked(<2 x i16> %va, i32 zeroext %
 define <4 x i16> @vp_cttz_zero_undef_v4i16(<4 x i16> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v4i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 5
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -2567,13 +2359,12 @@ define <4 x i16> @vp_cttz_zero_undef_v4i16(<4 x i16> %va, <4 x i1> %m, i32 zeroe
 define <4 x i16> @vp_cttz_zero_undef_v4i16_unmasked(<4 x i16> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v4i16_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v9, v8
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    lui a0, 3
@@ -2599,9 +2390,8 @@ define <4 x i16> @vp_cttz_zero_undef_v4i16_unmasked(<4 x i16> %va, i32 zeroext %
 define <8 x i16> @vp_cttz_zero_undef_v8i16(<8 x i16> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v8i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 5
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -2631,13 +2421,12 @@ define <8 x i16> @vp_cttz_zero_undef_v8i16(<8 x i16> %va, <8 x i1> %m, i32 zeroe
 define <8 x i16> @vp_cttz_zero_undef_v8i16_unmasked(<8 x i16> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v8i16_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v9, v8
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    lui a0, 3
@@ -2663,9 +2452,8 @@ define <8 x i16> @vp_cttz_zero_undef_v8i16_unmasked(<8 x i16> %va, i32 zeroext %
 define <16 x i16> @vp_cttz_zero_undef_v16i16(<16 x i16> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v16i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vsub.vx v10, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v10, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 5
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -2695,13 +2483,12 @@ define <16 x i16> @vp_cttz_zero_undef_v16i16(<16 x i16> %va, <16 x i1> %m, i32 z
 define <16 x i16> @vp_cttz_zero_undef_v16i16_unmasked(<16 x i16> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v16i16_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
-; CHECK-NEXT:    vnot.v v10, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v10, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    vand.vv v8, v8, v10
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v10, v8
 ; CHECK-NEXT:    vsrl.vi v10, v8, 1
 ; CHECK-NEXT:    vand.vx v10, v10, a0
 ; CHECK-NEXT:    lui a0, 3
@@ -2727,9 +2514,8 @@ define <16 x i16> @vp_cttz_zero_undef_v16i16_unmasked(<16 x i16> %va, i32 zeroex
 define <2 x i32> @vp_cttz_zero_undef_v2i32(<2 x i32> %va, <2 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v2i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 349525
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -2760,13 +2546,12 @@ define <2 x i32> @vp_cttz_zero_undef_v2i32(<2 x i32> %va, <2 x i1> %m, i32 zeroe
 define <2 x i32> @vp_cttz_zero_undef_v2i32_unmasked(<2 x i32> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v2i32_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 349525
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v9, v8
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    lui a0, 209715
@@ -2793,9 +2578,8 @@ define <2 x i32> @vp_cttz_zero_undef_v2i32_unmasked(<2 x i32> %va, i32 zeroext %
 define <4 x i32> @vp_cttz_zero_undef_v4i32(<4 x i32> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vsub.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 349525
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -2826,13 +2610,12 @@ define <4 x i32> @vp_cttz_zero_undef_v4i32(<4 x i32> %va, <4 x i1> %m, i32 zeroe
 define <4 x i32> @vp_cttz_zero_undef_v4i32_unmasked(<4 x i32> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v4i32_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vnot.v v9, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v9, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 349525
+; CHECK-NEXT:    vand.vv v8, v8, v9
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v9, v8
 ; CHECK-NEXT:    vsrl.vi v9, v8, 1
 ; CHECK-NEXT:    vand.vx v9, v9, a0
 ; CHECK-NEXT:    lui a0, 209715
@@ -2859,9 +2642,8 @@ define <4 x i32> @vp_cttz_zero_undef_v4i32_unmasked(<4 x i32> %va, i32 zeroext %
 define <8 x i32> @vp_cttz_zero_undef_v8i32(<8 x i32> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v8i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    vsub.vx v10, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v10, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 349525
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -2892,13 +2674,12 @@ define <8 x i32> @vp_cttz_zero_undef_v8i32(<8 x i32> %va, <8 x i1> %m, i32 zeroe
 define <8 x i32> @vp_cttz_zero_undef_v8i32_unmasked(<8 x i32> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v8i32_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; CHECK-NEXT:    vnot.v v10, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v10, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 349525
+; CHECK-NEXT:    vand.vv v8, v8, v10
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v10, v8
 ; CHECK-NEXT:    vsrl.vi v10, v8, 1
 ; CHECK-NEXT:    vand.vx v10, v10, a0
 ; CHECK-NEXT:    lui a0, 209715
@@ -2925,9 +2706,8 @@ define <8 x i32> @vp_cttz_zero_undef_v8i32_unmasked(<8 x i32> %va, i32 zeroext %
 define <16 x i32> @vp_cttz_zero_undef_v16i32(<16 x i32> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v16i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    vsub.vx v12, v8, a1, v0.t
+; CHECK-NEXT:    vadd.vi v12, v8, -1, v0.t
 ; CHECK-NEXT:    lui a0, 349525
 ; CHECK-NEXT:    vnot.v v8, v8, v0.t
 ; CHECK-NEXT:    addi a0, a0, 1365
@@ -2958,13 +2738,12 @@ define <16 x i32> @vp_cttz_zero_undef_v16i32(<16 x i32> %va, <16 x i1> %m, i32 z
 define <16 x i32> @vp_cttz_zero_undef_v16i32_unmasked(<16 x i32> %va, i32 zeroext %evl) {
 ; CHECK-LABEL: vp_cttz_zero_undef_v16i32_unmasked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 1
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
-; CHECK-NEXT:    vnot.v v12, v8
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v12, v8, -1
+; CHECK-NEXT:    vnot.v v8, v8
 ; CHECK-NEXT:    lui a0, 349525
+; CHECK-NEXT:    vand.vv v8, v8, v12
 ; CHECK-NEXT:    addi a0, a0, 1365
-; CHECK-NEXT:    vand.vv v8, v12, v8
 ; CHECK-NEXT:    vsrl.vi v12, v8, 1
 ; CHECK-NEXT:    vand.vx v12, v12, a0
 ; CHECK-NEXT:    lui a0, 209715
@@ -2991,9 +2770,8 @@ define <16 x i32> @vp_cttz_zero_undef_v16i32_unmasked(<16 x i32> %va, i32 zeroex
 define <2 x i64> @vp_cttz_zero_undef_v2i64(<2 x i64> %va, <2 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_zero_undef_v2i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; RV32-NEXT:    vsub.vx v9, v8, a1, v0.t
+; RV32-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
@@ -3033,39 +2811,38 @@ define <2 x i64> @vp_cttz_zero_undef_v2i64(<2 x i64> %va, <2 x i1> %m, i32 zeroe
 ;
 ; RV64-LABEL: vp_cttz_zero_undef_v2i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; RV64-NEXT:    vsub.vx v9, v8, a1, v0.t
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v9, v8, -1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v9, v0.t
 ; RV64-NEXT:    vsrl.vi v9, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v9, v9, a0, v0.t
+; RV64-NEXT:    vand.vx v9, v9, a1, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v9, v0.t
-; RV64-NEXT:    vand.vx v9, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v9, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vadd.vv v8, v9, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v9, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v9, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    li a0, 56
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a4, v0.t
 ; RV64-NEXT:    vsrl.vx v8, v8, a0, v0.t
 ; RV64-NEXT:    ret
   %v = call <2 x i64> @llvm.vp.cttz.v2i64(<2 x i64> %va, i1 true, <2 x i1> %m, i32 %evl)
@@ -3075,18 +2852,17 @@ define <2 x i64> @vp_cttz_zero_undef_v2i64(<2 x i64> %va, <2 x i1> %m, i32 zeroe
 define <2 x i64> @vp_cttz_zero_undef_v2i64_unmasked(<2 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_zero_undef_v2i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; RV32-NEXT:    vnot.v v9, v8
-; RV32-NEXT:    vsub.vx v8, v8, a1
+; RV32-NEXT:    vadd.vi v9, v8, -1
+; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV32-NEXT:    vmv.v.x v10, a1
 ; RV32-NEXT:    lui a1, 209715
-; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; RV32-NEXT:    vand.vv v8, v9, v8
+; RV32-NEXT:    vand.vv v8, v8, v9
+; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsrl.vi v9, v8, 1
 ; RV32-NEXT:    vand.vv v9, v9, v10
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
@@ -3117,38 +2893,37 @@ define <2 x i64> @vp_cttz_zero_undef_v2i64_unmasked(<2 x i64> %va, i32 zeroext %
 ;
 ; RV64-LABEL: vp_cttz_zero_undef_v2i64_unmasked:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; RV64-NEXT:    vsub.vx v9, v8, a1
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v9, v8, -1
 ; RV64-NEXT:    vnot.v v8, v8
 ; RV64-NEXT:    vand.vv v8, v8, v9
 ; RV64-NEXT:    vsrl.vi v9, v8, 1
-; RV64-NEXT:    vand.vx v9, v9, a0
+; RV64-NEXT:    vand.vx v9, v9, a1
 ; RV64-NEXT:    vsub.vv v8, v8, v9
-; RV64-NEXT:    vand.vx v9, v8, a1
+; RV64-NEXT:    vand.vx v9, v8, a2
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vand.vx v8, v8, a1
+; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    vadd.vv v8, v9, v8
 ; RV64-NEXT:    vsrl.vi v9, v8, 4
 ; RV64-NEXT:    vadd.vv v8, v8, v9
-; RV64-NEXT:    vand.vx v8, v8, a2
-; RV64-NEXT:    vmul.vx v8, v8, a3
+; RV64-NEXT:    vand.vx v8, v8, a3
+; RV64-NEXT:    vmul.vx v8, v8, a4
 ; RV64-NEXT:    li a0, 56
 ; RV64-NEXT:    vsrl.vx v8, v8, a0
 ; RV64-NEXT:    ret
@@ -3159,9 +2934,8 @@ define <2 x i64> @vp_cttz_zero_undef_v2i64_unmasked(<2 x i64> %va, i32 zeroext %
 define <4 x i64> @vp_cttz_zero_undef_v4i64(<4 x i64> %va, <4 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_zero_undef_v4i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; RV32-NEXT:    vsub.vx v10, v8, a1, v0.t
+; RV32-NEXT:    vadd.vi v10, v8, -1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
@@ -3201,39 +2975,38 @@ define <4 x i64> @vp_cttz_zero_undef_v4i64(<4 x i64> %va, <4 x i1> %m, i32 zeroe
 ;
 ; RV64-LABEL: vp_cttz_zero_undef_v4i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; RV64-NEXT:    vsub.vx v10, v8, a1, v0.t
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v10, v8, -1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v10, v0.t
 ; RV64-NEXT:    vsrl.vi v10, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v10, v10, a0, v0.t
+; RV64-NEXT:    vand.vx v10, v10, a1, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v10, v0.t
-; RV64-NEXT:    vand.vx v10, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v10, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vadd.vv v8, v10, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v10, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v10, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    li a0, 56
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a4, v0.t
 ; RV64-NEXT:    vsrl.vx v8, v8, a0, v0.t
 ; RV64-NEXT:    ret
   %v = call <4 x i64> @llvm.vp.cttz.v4i64(<4 x i64> %va, i1 true, <4 x i1> %m, i32 %evl)
@@ -3243,18 +3016,17 @@ define <4 x i64> @vp_cttz_zero_undef_v4i64(<4 x i64> %va, <4 x i1> %m, i32 zeroe
 define <4 x i64> @vp_cttz_zero_undef_v4i64_unmasked(<4 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_zero_undef_v4i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; RV32-NEXT:    vnot.v v10, v8
-; RV32-NEXT:    vsub.vx v8, v8, a1
+; RV32-NEXT:    vadd.vi v10, v8, -1
+; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; RV32-NEXT:    vmv.v.x v12, a1
 ; RV32-NEXT:    lui a1, 209715
-; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; RV32-NEXT:    vand.vv v8, v10, v8
+; RV32-NEXT:    vand.vv v8, v8, v10
+; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsrl.vi v10, v8, 1
 ; RV32-NEXT:    vand.vv v10, v10, v12
 ; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
@@ -3285,38 +3057,37 @@ define <4 x i64> @vp_cttz_zero_undef_v4i64_unmasked(<4 x i64> %va, i32 zeroext %
 ;
 ; RV64-LABEL: vp_cttz_zero_undef_v4i64_unmasked:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
-; RV64-NEXT:    vsub.vx v10, v8, a1
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v10, v8, -1
 ; RV64-NEXT:    vnot.v v8, v8
 ; RV64-NEXT:    vand.vv v8, v8, v10
 ; RV64-NEXT:    vsrl.vi v10, v8, 1
-; RV64-NEXT:    vand.vx v10, v10, a0
+; RV64-NEXT:    vand.vx v10, v10, a1
 ; RV64-NEXT:    vsub.vv v8, v8, v10
-; RV64-NEXT:    vand.vx v10, v8, a1
+; RV64-NEXT:    vand.vx v10, v8, a2
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vand.vx v8, v8, a1
+; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    vadd.vv v8, v10, v8
 ; RV64-NEXT:    vsrl.vi v10, v8, 4
 ; RV64-NEXT:    vadd.vv v8, v8, v10
-; RV64-NEXT:    vand.vx v8, v8, a2
-; RV64-NEXT:    vmul.vx v8, v8, a3
+; RV64-NEXT:    vand.vx v8, v8, a3
+; RV64-NEXT:    vmul.vx v8, v8, a4
 ; RV64-NEXT:    li a0, 56
 ; RV64-NEXT:    vsrl.vx v8, v8, a0
 ; RV64-NEXT:    ret
@@ -3327,9 +3098,8 @@ define <4 x i64> @vp_cttz_zero_undef_v4i64_unmasked(<4 x i64> %va, i32 zeroext %
 define <8 x i64> @vp_cttz_zero_undef_v8i64(<8 x i64> %va, <8 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_zero_undef_v8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; RV32-NEXT:    vsub.vx v12, v8, a1, v0.t
+; RV32-NEXT:    vadd.vi v12, v8, -1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
@@ -3369,39 +3139,38 @@ define <8 x i64> @vp_cttz_zero_undef_v8i64(<8 x i64> %va, <8 x i1> %m, i32 zeroe
 ;
 ; RV64-LABEL: vp_cttz_zero_undef_v8i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; RV64-NEXT:    vsub.vx v12, v8, a1, v0.t
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v12, v8, -1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v12, v0.t
 ; RV64-NEXT:    vsrl.vi v12, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v12, v12, a0, v0.t
+; RV64-NEXT:    vand.vx v12, v12, a1, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v12, v0.t
-; RV64-NEXT:    vand.vx v12, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v12, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vadd.vv v8, v12, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v12, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v12, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    li a0, 56
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a4, v0.t
 ; RV64-NEXT:    vsrl.vx v8, v8, a0, v0.t
 ; RV64-NEXT:    ret
   %v = call <8 x i64> @llvm.vp.cttz.v8i64(<8 x i64> %va, i1 true, <8 x i1> %m, i32 %evl)
@@ -3411,18 +3180,17 @@ define <8 x i64> @vp_cttz_zero_undef_v8i64(<8 x i64> %va, <8 x i1> %m, i32 zeroe
 define <8 x i64> @vp_cttz_zero_undef_v8i64_unmasked(<8 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_zero_undef_v8i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; RV32-NEXT:    vnot.v v12, v8
-; RV32-NEXT:    vsub.vx v8, v8, a1
+; RV32-NEXT:    vadd.vi v12, v8, -1
+; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 209715
-; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; RV32-NEXT:    vand.vv v8, v12, v8
+; RV32-NEXT:    vand.vv v8, v8, v12
+; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vsrl.vi v12, v8, 1
 ; RV32-NEXT:    vand.vv v12, v12, v16
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
@@ -3453,38 +3221,37 @@ define <8 x i64> @vp_cttz_zero_undef_v8i64_unmasked(<8 x i64> %va, i32 zeroext %
 ;
 ; RV64-LABEL: vp_cttz_zero_undef_v8i64_unmasked:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m4, ta, ma
-; RV64-NEXT:    vsub.vx v12, v8, a1
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v12, v8, -1
 ; RV64-NEXT:    vnot.v v8, v8
 ; RV64-NEXT:    vand.vv v8, v8, v12
 ; RV64-NEXT:    vsrl.vi v12, v8, 1
-; RV64-NEXT:    vand.vx v12, v12, a0
+; RV64-NEXT:    vand.vx v12, v12, a1
 ; RV64-NEXT:    vsub.vv v8, v8, v12
-; RV64-NEXT:    vand.vx v12, v8, a1
+; RV64-NEXT:    vand.vx v12, v8, a2
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vand.vx v8, v8, a1
+; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    vadd.vv v8, v12, v8
 ; RV64-NEXT:    vsrl.vi v12, v8, 4
 ; RV64-NEXT:    vadd.vv v8, v8, v12
-; RV64-NEXT:    vand.vx v8, v8, a2
-; RV64-NEXT:    vmul.vx v8, v8, a3
+; RV64-NEXT:    vand.vx v8, v8, a3
+; RV64-NEXT:    vmul.vx v8, v8, a4
 ; RV64-NEXT:    li a0, 56
 ; RV64-NEXT:    vsrl.vx v8, v8, a0
 ; RV64-NEXT:    ret
@@ -3495,129 +3262,79 @@ define <8 x i64> @vp_cttz_zero_undef_v8i64_unmasked(<8 x i64> %va, i32 zeroext %
 define <15 x i64> @vp_cttz_zero_undef_v15i64(<15 x i64> %va, <15 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_zero_undef_v15i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
-; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v16, v8, a1, v0.t
+; RV32-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    lui a1, 209715
-; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 32(sp)
-; RV32-NEXT:    sw a1, 36(sp)
-; RV32-NEXT:    lui a1, 61681
-; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    lui a1, 4112
-; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
-; RV32-NEXT:    addi a1, sp, 40
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a1, sp, 32
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vnot.v v8, v8, v0.t
+; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vi v8, v16, 1, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    addi a1, sp, 48
-; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    vsub.vv v8, v16, v8, v0.t
-; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v8, v24, v0.t
-; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
 ; RV32-NEXT:    vand.vv v24, v8, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a1, sp, 16
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a1
+; RV32-NEXT:    lui a1, 61681
+; RV32-NEXT:    addi a1, a1, -241
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v24, v16, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a1), zero
+; RV32-NEXT:    vsub.vv v24, v16, v24, v0.t
+; RV32-NEXT:    vand.vv v16, v24, v8, v0.t
+; RV32-NEXT:    vsrl.vi v24, v24, 2, v0.t
+; RV32-NEXT:    vand.vv v24, v24, v8, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a1
+; RV32-NEXT:    lui a1, 4112
+; RV32-NEXT:    addi a1, a1, 257
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v24, 4, v0.t
-; RV32-NEXT:    vadd.vv v8, v24, v8, v0.t
-; RV32-NEXT:    li a0, 56
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vx v8, v8, a0, v0.t
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vadd.vv v16, v16, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v16, 4, v0.t
+; RV32-NEXT:    vadd.vv v16, v16, v24, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
+; RV32-NEXT:    li a1, 56
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vmul.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vx v8, v8, a1, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_cttz_zero_undef_v15i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v16, v8, a1, v0.t
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v16, v16, a0, v0.t
+; RV64-NEXT:    vand.vx v16, v16, a1, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v16, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v16, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    li a0, 56
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a4, v0.t
 ; RV64-NEXT:    vsrl.vx v8, v8, a0, v0.t
 ; RV64-NEXT:    ret
   %v = call <15 x i64> @llvm.vp.cttz.v15i64(<15 x i64> %va, i1 true, <15 x i1> %m, i32 %evl)
@@ -3627,94 +3344,78 @@ define <15 x i64> @vp_cttz_zero_undef_v15i64(<15 x i64> %va, <15 x i1> %m, i32 z
 define <15 x i64> @vp_cttz_zero_undef_v15i64_unmasked(<15 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_zero_undef_v15i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v16, v8, a1
+; RV32-NEXT:    vadd.vi v16, v8, -1
+; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    lui a1, 209715
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v8, v8, v16
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
+; RV32-NEXT:    vsrl.vi v16, v8, 1
+; RV32-NEXT:    vand.vv v24, v16, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 8(sp)
-; RV32-NEXT:    sw a1, 12(sp)
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vsub.vv v8, v8, v24
+; RV32-NEXT:    vand.vv v24, v8, v16
+; RV32-NEXT:    vsrl.vi v8, v8, 2
+; RV32-NEXT:    vand.vv v8, v8, v16
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 4112
-; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 0(sp)
-; RV32-NEXT:    sw a1, 4(sp)
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v0, (a1), zero
-; RV32-NEXT:    addi a1, sp, 16
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    addi a1, sp, 8
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v16, v8, v16
-; RV32-NEXT:    vsrl.vi v8, v16, 1
-; RV32-NEXT:    vand.vv v0, v8, v0
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    mv a1, sp
+; RV32-NEXT:    vadd.vv v8, v24, v8
+; RV32-NEXT:    vsrl.vi v24, v8, 4
+; RV32-NEXT:    vadd.vv v8, v8, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vv v16, v16, v0
-; RV32-NEXT:    vand.vv v0, v16, v24
-; RV32-NEXT:    vsrl.vi v16, v16, 2
-; RV32-NEXT:    vand.vv v16, v16, v24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v16, v0, v16
-; RV32-NEXT:    vsrl.vi v0, v16, 4
-; RV32-NEXT:    vadd.vv v16, v16, v0
-; RV32-NEXT:    vand.vv v8, v16, v8
+; RV32-NEXT:    vand.vv v8, v8, v16
 ; RV32-NEXT:    vmul.vv v8, v8, v24
 ; RV32-NEXT:    li a0, 56
 ; RV32-NEXT:    vsrl.vx v8, v8, a0
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_cttz_zero_undef_v15i64_unmasked:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v16, v8, a1
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v16, v8, -1
 ; RV64-NEXT:    vnot.v v8, v8
 ; RV64-NEXT:    vand.vv v8, v8, v16
 ; RV64-NEXT:    vsrl.vi v16, v8, 1
-; RV64-NEXT:    vand.vx v16, v16, a0
+; RV64-NEXT:    vand.vx v16, v16, a1
 ; RV64-NEXT:    vsub.vv v8, v8, v16
-; RV64-NEXT:    vand.vx v16, v8, a1
+; RV64-NEXT:    vand.vx v16, v8, a2
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vand.vx v8, v8, a1
+; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    vadd.vv v8, v16, v8
 ; RV64-NEXT:    vsrl.vi v16, v8, 4
 ; RV64-NEXT:    vadd.vv v8, v8, v16
-; RV64-NEXT:    vand.vx v8, v8, a2
-; RV64-NEXT:    vmul.vx v8, v8, a3
+; RV64-NEXT:    vand.vx v8, v8, a3
+; RV64-NEXT:    vmul.vx v8, v8, a4
 ; RV64-NEXT:    li a0, 56
 ; RV64-NEXT:    vsrl.vx v8, v8, a0
 ; RV64-NEXT:    ret
@@ -3725,129 +3426,79 @@ define <15 x i64> @vp_cttz_zero_undef_v15i64_unmasked(<15 x i64> %va, i32 zeroex
 define <16 x i64> @vp_cttz_zero_undef_v16i64(<16 x i64> %va, <16 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_zero_undef_v16i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
-; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 16 * vlenb
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v16, v8, a1, v0.t
+; RV32-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    lui a1, 209715
-; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 32(sp)
-; RV32-NEXT:    sw a1, 36(sp)
-; RV32-NEXT:    lui a1, 61681
-; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    lui a1, 4112
-; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
-; RV32-NEXT:    addi a1, sp, 40
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a1, sp, 32
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV32-NEXT:    vnot.v v8, v8, v0.t
+; RV32-NEXT:    addi a1, a1, 819
 ; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
 ; RV32-NEXT:    vsrl.vi v8, v16, 1, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    addi a1, sp, 48
-; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    vsub.vv v8, v16, v8, v0.t
-; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vl8r.v v24, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v8, v24, v0.t
-; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
 ; RV32-NEXT:    vand.vv v24, v8, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a1, sp, 16
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a1
+; RV32-NEXT:    lui a1, 61681
+; RV32-NEXT:    addi a1, a1, -241
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v24, v16, v24, v0.t
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a1), zero
+; RV32-NEXT:    vsub.vv v24, v16, v24, v0.t
+; RV32-NEXT:    vand.vv v16, v24, v8, v0.t
+; RV32-NEXT:    vsrl.vi v24, v24, 2, v0.t
+; RV32-NEXT:    vand.vv v24, v24, v8, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a1
+; RV32-NEXT:    lui a1, 4112
+; RV32-NEXT:    addi a1, a1, 257
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v24, 4, v0.t
-; RV32-NEXT:    vadd.vv v8, v24, v8, v0.t
-; RV32-NEXT:    li a0, 56
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
-; RV32-NEXT:    vl8r.v v24, (a1) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v8, v8, v24, v0.t
-; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vx v8, v8, a0, v0.t
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vadd.vv v16, v16, v24, v0.t
+; RV32-NEXT:    vsrl.vi v24, v16, 4, v0.t
+; RV32-NEXT:    vadd.vv v16, v16, v24, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
+; RV32-NEXT:    li a1, 56
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vmul.vv v8, v8, v24, v0.t
+; RV32-NEXT:    vsrl.vx v8, v8, a1, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_cttz_zero_undef_v16i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v16, v8, a1, v0.t
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v16, v16, a0, v0.t
+; RV64-NEXT:    vand.vx v16, v16, a1, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v16, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v16, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    li a0, 56
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a4, v0.t
 ; RV64-NEXT:    vsrl.vx v8, v8, a0, v0.t
 ; RV64-NEXT:    ret
   %v = call <16 x i64> @llvm.vp.cttz.v16i64(<16 x i64> %va, i1 true, <16 x i1> %m, i32 %evl)
@@ -3857,94 +3508,78 @@ define <16 x i64> @vp_cttz_zero_undef_v16i64(<16 x i64> %va, <16 x i1> %m, i32 z
 define <16 x i64> @vp_cttz_zero_undef_v16i64_unmasked(<16 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_zero_undef_v16i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
-; RV32-NEXT:    li a1, 1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v16, v8, a1
+; RV32-NEXT:    vadd.vi v16, v8, -1
+; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    lui a1, 349525
 ; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    lui a1, 209715
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v8, v8, v16
 ; RV32-NEXT:    addi a1, a1, 819
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
+; RV32-NEXT:    vsrl.vi v16, v8, 1
+; RV32-NEXT:    vand.vv v24, v16, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 61681
 ; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 8(sp)
-; RV32-NEXT:    sw a1, 12(sp)
+; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; RV32-NEXT:    vsub.vv v8, v8, v24
+; RV32-NEXT:    vand.vv v24, v8, v16
+; RV32-NEXT:    vsrl.vi v8, v8, 2
+; RV32-NEXT:    vand.vv v8, v8, v16
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a1
 ; RV32-NEXT:    lui a1, 4112
-; RV32-NEXT:    vnot.v v8, v8
 ; RV32-NEXT:    addi a1, a1, 257
-; RV32-NEXT:    sw a1, 0(sp)
-; RV32-NEXT:    sw a1, 4(sp)
-; RV32-NEXT:    addi a1, sp, 24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v0, (a1), zero
-; RV32-NEXT:    addi a1, sp, 16
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    addi a1, sp, 8
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v16, v8, v16
-; RV32-NEXT:    vsrl.vi v8, v16, 1
-; RV32-NEXT:    vand.vv v0, v8, v0
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a1), zero
-; RV32-NEXT:    mv a1, sp
+; RV32-NEXT:    vadd.vv v8, v24, v8
+; RV32-NEXT:    vsrl.vi v24, v8, 4
+; RV32-NEXT:    vadd.vv v8, v8, v24
+; RV32-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a1
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vv v16, v16, v0
-; RV32-NEXT:    vand.vv v0, v16, v24
-; RV32-NEXT:    vsrl.vi v16, v16, 2
-; RV32-NEXT:    vand.vv v16, v16, v24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a1), zero
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v16, v0, v16
-; RV32-NEXT:    vsrl.vi v0, v16, 4
-; RV32-NEXT:    vadd.vv v16, v16, v0
-; RV32-NEXT:    vand.vv v8, v16, v8
+; RV32-NEXT:    vand.vv v8, v8, v16
 ; RV32-NEXT:    vmul.vv v8, v8, v24
 ; RV32-NEXT:    li a0, 56
 ; RV32-NEXT:    vsrl.vx v8, v8, a0
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vp_cttz_zero_undef_v16i64_unmasked:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a4, 61681
-; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a3, a3, -241
+; RV64-NEXT:    addiw a4, a4, 257
+; RV64-NEXT:    slli a5, a1, 32
+; RV64-NEXT:    add a1, a1, a5
+; RV64-NEXT:    slli a5, a2, 32
+; RV64-NEXT:    add a2, a2, a5
+; RV64-NEXT:    slli a5, a3, 32
+; RV64-NEXT:    add a3, a3, a5
+; RV64-NEXT:    slli a5, a4, 32
+; RV64-NEXT:    add a4, a4, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v16, v8, a1
-; RV64-NEXT:    addiw a0, a2, 1365
-; RV64-NEXT:    addiw a1, a3, 819
-; RV64-NEXT:    addiw a2, a4, -241
-; RV64-NEXT:    addiw a3, a5, 257
-; RV64-NEXT:    slli a4, a0, 32
-; RV64-NEXT:    add a0, a0, a4
-; RV64-NEXT:    slli a4, a1, 32
-; RV64-NEXT:    add a1, a1, a4
-; RV64-NEXT:    slli a4, a2, 32
-; RV64-NEXT:    add a2, a2, a4
-; RV64-NEXT:    slli a4, a3, 32
-; RV64-NEXT:    add a3, a3, a4
+; RV64-NEXT:    vadd.vi v16, v8, -1
 ; RV64-NEXT:    vnot.v v8, v8
 ; RV64-NEXT:    vand.vv v8, v8, v16
 ; RV64-NEXT:    vsrl.vi v16, v8, 1
-; RV64-NEXT:    vand.vx v16, v16, a0
+; RV64-NEXT:    vand.vx v16, v16, a1
 ; RV64-NEXT:    vsub.vv v8, v8, v16
-; RV64-NEXT:    vand.vx v16, v8, a1
+; RV64-NEXT:    vand.vx v16, v8, a2
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vand.vx v8, v8, a1
+; RV64-NEXT:    vand.vx v8, v8, a2
 ; RV64-NEXT:    vadd.vv v8, v16, v8
 ; RV64-NEXT:    vsrl.vi v16, v8, 4
 ; RV64-NEXT:    vadd.vv v8, v8, v16
-; RV64-NEXT:    vand.vx v8, v8, a2
-; RV64-NEXT:    vmul.vx v8, v8, a3
+; RV64-NEXT:    vand.vx v8, v8, a3
+; RV64-NEXT:    vmul.vx v8, v8, a4
 ; RV64-NEXT:    li a0, 56
 ; RV64-NEXT:    vsrl.vx v8, v8, a0
 ; RV64-NEXT:    ret
@@ -3955,263 +3590,243 @@ define <16 x i64> @vp_cttz_zero_undef_v16i64_unmasked(<16 x i64> %va, i32 zeroex
 define <32 x i64> @vp_cttz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_zero_undef_v32i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    li a2, 48
+; RV32-NEXT:    li a2, 56
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x30, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 48 * vlenb
+; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x38, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 56 * vlenb
 ; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
+; RV32-NEXT:    slli a1, a1, 5
 ; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 48
+; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    li a2, 16
 ; RV32-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
-; RV32-NEXT:    vslidedown.vi v7, v0, 2
-; RV32-NEXT:    lui a1, 349525
-; RV32-NEXT:    lui a2, 209715
-; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
-; RV32-NEXT:    lui a1, 61681
-; RV32-NEXT:    addi a2, a2, 819
-; RV32-NEXT:    sw a2, 32(sp)
-; RV32-NEXT:    sw a2, 36(sp)
-; RV32-NEXT:    lui a2, 4112
-; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    li a1, 16
-; RV32-NEXT:    addi a2, a2, 257
-; RV32-NEXT:    sw a2, 16(sp)
-; RV32-NEXT:    sw a2, 20(sp)
-; RV32-NEXT:    mv a2, a0
-; RV32-NEXT:    bltu a0, a1, .LBB70_2
+; RV32-NEXT:    vslidedown.vi v24, v0, 2
+; RV32-NEXT:    mv a1, a0
+; RV32-NEXT:    bltu a0, a2, .LBB70_2
 ; RV32-NEXT:  # %bb.1:
-; RV32-NEXT:    li a2, 16
+; RV32-NEXT:    li a1, 16
 ; RV32-NEXT:  .LBB70_2:
-; RV32-NEXT:    li a1, 1
-; RV32-NEXT:    addi a3, sp, 40
-; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v16, v8, a1, v0.t
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vadd.vi v16, v8, -1, v0.t
+; RV32-NEXT:    lui a2, 349525
 ; RV32-NEXT:    vnot.v v8, v8, v0.t
+; RV32-NEXT:    addi a2, a2, 1365
 ; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
-; RV32-NEXT:    csrr a4, vlenb
-; RV32-NEXT:    li a5, 24
-; RV32-NEXT:    mul a4, a4, a5
-; RV32-NEXT:    add a4, sp, a4
-; RV32-NEXT:    addi a4, a4, 48
-; RV32-NEXT:    vs8r.v v8, (a4) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a3), zero
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 40
-; RV32-NEXT:    mul a3, a3, a4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a3, sp, 32
-; RV32-NEXT:    vlse64.v v8, (a3), zero
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 5
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v16, 1, v0.t
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 40
-; RV32-NEXT:    mul a3, a3, a4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v24, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v24, v24, v16, v0.t
 ; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
+; RV32-NEXT:    li a4, 48
 ; RV32-NEXT:    mul a3, a3, a4
 ; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vsub.vv v16, v16, v24, v0.t
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vsrl.vi v16, v16, 2, v0.t
-; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vadd.vv v8, v8, v16, v0.t
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a3, sp, 24
-; RV32-NEXT:    addi a4, sp, 16
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v16, (a3), zero
-; RV32-NEXT:    addi a3, sp, 48
-; RV32-NEXT:    vs8r.v v16, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    vlse64.v v8, (a4), zero
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    li a4, 24
-; RV32-NEXT:    mul a3, a3, a4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
+; RV32-NEXT:    addi a3, a3, 16
 ; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v24, v8, 4, v0.t
-; RV32-NEXT:    vadd.vv v8, v8, v24, v0.t
-; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a2
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 40
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 48
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vsrl.vi v8, v8, 1, v0.t
 ; RV32-NEXT:    csrr a2, vlenb
 ; RV32-NEXT:    li a3, 24
 ; RV32-NEXT:    mul a2, a2, a3
 ; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 48
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 40
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 24
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
 ; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vmul.vv v8, v16, v8, v0.t
-; RV32-NEXT:    li a2, 56
-; RV32-NEXT:    vsrl.vx v8, v8, a2, v0.t
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 3
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vs8r.v v8, (a3) # Unknown-size Folded Spill
-; RV32-NEXT:    addi a3, a0, -16
-; RV32-NEXT:    sltu a0, a0, a3
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 48
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vsub.vv v8, v8, v16, v0.t
+; RV32-NEXT:    lui a2, 209715
+; RV32-NEXT:    addi a2, a2, 819
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a2
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 48
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 48
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 24
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 48
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 24
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vadd.vv v8, v16, v8, v0.t
+; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
+; RV32-NEXT:    vadd.vv v8, v8, v16, v0.t
+; RV32-NEXT:    lui a2, 61681
+; RV32-NEXT:    addi a2, a2, -241
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a2
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    li a3, 24
+; RV32-NEXT:    mul a2, a2, a3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    lui a2, 4112
+; RV32-NEXT:    addi a2, a2, 257
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a2
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a2, a2, 3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
+; RV32-NEXT:    li a1, 56
+; RV32-NEXT:    vsrl.vx v8, v8, a1, v0.t
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a2, a2, 4
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    addi a2, a0, -16
+; RV32-NEXT:    sltu a0, a0, a2
 ; RV32-NEXT:    addi a0, a0, -1
-; RV32-NEXT:    and a0, a0, a3
-; RV32-NEXT:    vmv1r.v v0, v7
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    slli a3, a3, 4
-; RV32-NEXT:    add a3, sp, a3
-; RV32-NEXT:    addi a3, a3, 48
-; RV32-NEXT:    vl8r.v v16, (a3) # Unknown-size Folded Reload
+; RV32-NEXT:    and a0, a0, a2
+; RV32-NEXT:    vmv1r.v v0, v24
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a2, a2, 5
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v8, v16, a1, v0.t
-; RV32-NEXT:    vnot.v v16, v16, v0.t
-; RV32-NEXT:    vand.vv v8, v16, v8, v0.t
-; RV32-NEXT:    vsrl.vi v24, v8, 1, v0.t
+; RV32-NEXT:    vadd.vi v16, v8, -1, v0.t
+; RV32-NEXT:    vnot.v v8, v8, v0.t
+; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
+; RV32-NEXT:    vsrl.vi v8, v8, 1, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 40
-; RV32-NEXT:    mul a0, a0, a1
+; RV32-NEXT:    slli a0, a0, 5
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v24, v16, v0.t
-; RV32-NEXT:    vsub.vv v8, v8, v16, v0.t
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 40
-; RV32-NEXT:    mul a0, a0, a1
+; RV32-NEXT:    li a2, 40
+; RV32-NEXT:    mul a0, a0, a2
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vs8r.v v8, (a0) # Unknown-size Folded Spill
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    slli a0, a0, 5
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
+; RV32-NEXT:    addi a0, sp, 16
 ; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    vsub.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 40
-; RV32-NEXT:    mul a0, a0, a1
+; RV32-NEXT:    li a2, 48
+; RV32-NEXT:    mul a0, a0, a2
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vand.vv v16, v16, v8, v0.t
+; RV32-NEXT:    vand.vv v16, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
+; RV32-NEXT:    li a2, 40
+; RV32-NEXT:    mul a0, a0, a2
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 40
-; RV32-NEXT:    mul a0, a0, a1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
-; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsrl.vi v8, v8, 2, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 5
+; RV32-NEXT:    li a2, 48
+; RV32-NEXT:    mul a0, a0, a2
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
+; RV32-NEXT:    li a2, 40
+; RV32-NEXT:    mul a0, a0, a2
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV32-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV32-NEXT:    vadd.vv v8, v8, v16, v0.t
-; RV32-NEXT:    addi a0, sp, 48
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    li a2, 24
+; RV32-NEXT:    mul a0, a0, a2
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 24
-; RV32-NEXT:    mul a0, a0, a1
+; RV32-NEXT:    slli a0, a0, 3
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v16, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    vmul.vv v8, v8, v16, v0.t
-; RV32-NEXT:    vsrl.vx v16, v8, a2, v0.t
+; RV32-NEXT:    vsrl.vx v16, v8, a1, v0.t
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 3
+; RV32-NEXT:    slli a0, a0, 4
 ; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 48
+; RV32-NEXT:    addi a0, a0, 16
 ; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    li a1, 48
+; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    mul a0, a0, a1
 ; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
+; RV32-NEXT:    .cfi_def_cfa sp, 16
+; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
@@ -4228,75 +3843,74 @@ define <32 x i64> @vp_cttz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
 ; RV64-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; RV64-NEXT:    li a1, 16
+; RV64-NEXT:    li a2, 16
 ; RV64-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
 ; RV64-NEXT:    vslidedown.vi v24, v0, 2
-; RV64-NEXT:    mv a4, a0
-; RV64-NEXT:    bltu a0, a1, .LBB70_2
+; RV64-NEXT:    mv a1, a0
+; RV64-NEXT:    bltu a0, a2, .LBB70_2
 ; RV64-NEXT:  # %bb.1:
-; RV64-NEXT:    li a4, 16
+; RV64-NEXT:    li a1, 16
 ; RV64-NEXT:  .LBB70_2:
-; RV64-NEXT:    li a1, 1
-; RV64-NEXT:    lui a2, 349525
-; RV64-NEXT:    lui a3, 209715
-; RV64-NEXT:    lui a5, 61681
-; RV64-NEXT:    lui a6, 4112
-; RV64-NEXT:    addiw a2, a2, 1365
-; RV64-NEXT:    addiw a3, a3, 819
-; RV64-NEXT:    addiw a7, a5, -241
-; RV64-NEXT:    addiw t0, a6, 257
-; RV64-NEXT:    slli a6, a2, 32
-; RV64-NEXT:    add a6, a2, a6
-; RV64-NEXT:    slli a5, a3, 32
-; RV64-NEXT:    add a5, a3, a5
-; RV64-NEXT:    slli a2, a7, 32
-; RV64-NEXT:    add a2, a7, a2
-; RV64-NEXT:    slli a3, t0, 32
-; RV64-NEXT:    add a3, t0, a3
-; RV64-NEXT:    addi a7, a0, -16
-; RV64-NEXT:    sltu a0, a0, a7
+; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV64-NEXT:    vadd.vi v16, v8, -1, v0.t
+; RV64-NEXT:    lui a1, 349525
+; RV64-NEXT:    lui a2, 209715
+; RV64-NEXT:    lui a3, 61681
+; RV64-NEXT:    lui a4, 4112
+; RV64-NEXT:    addiw a1, a1, 1365
+; RV64-NEXT:    addiw a2, a2, 819
+; RV64-NEXT:    addiw a5, a3, -241
+; RV64-NEXT:    addiw a6, a4, 257
+; RV64-NEXT:    slli a4, a1, 32
+; RV64-NEXT:    add a4, a1, a4
+; RV64-NEXT:    slli a3, a2, 32
+; RV64-NEXT:    add a3, a2, a3
+; RV64-NEXT:    slli a1, a5, 32
+; RV64-NEXT:    add a1, a5, a1
+; RV64-NEXT:    slli a2, a6, 32
+; RV64-NEXT:    add a2, a6, a2
+; RV64-NEXT:    addi a5, a0, -16
+; RV64-NEXT:    sltu a0, a0, a5
 ; RV64-NEXT:    addi a0, a0, -1
-; RV64-NEXT:    and a7, a0, a7
+; RV64-NEXT:    and a5, a0, a5
 ; RV64-NEXT:    li a0, 56
-; RV64-NEXT:    vsetvli zero, a4, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v16, v8, a1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v16, v16, a6, v0.t
+; RV64-NEXT:    vand.vx v16, v16, a4, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v16, v8, a5, v0.t
+; RV64-NEXT:    vand.vx v16, v8, a3, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a5, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vx v8, v8, a0, v0.t
-; RV64-NEXT:    addi a4, sp, 16
-; RV64-NEXT:    vs8r.v v8, (a4) # Unknown-size Folded Spill
+; RV64-NEXT:    addi a6, sp, 16
+; RV64-NEXT:    vs8r.v v8, (a6) # Unknown-size Folded Spill
 ; RV64-NEXT:    vmv1r.v v0, v24
-; RV64-NEXT:    csrr a4, vlenb
-; RV64-NEXT:    slli a4, a4, 3
-; RV64-NEXT:    add a4, sp, a4
-; RV64-NEXT:    addi a4, a4, 16
-; RV64-NEXT:    vl8r.v v8, (a4) # Unknown-size Folded Reload
-; RV64-NEXT:    vsetvli zero, a7, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v16, v8, a1, v0.t
+; RV64-NEXT:    csrr a6, vlenb
+; RV64-NEXT:    slli a6, a6, 3
+; RV64-NEXT:    add a6, sp, a6
+; RV64-NEXT:    addi a6, a6, 16
+; RV64-NEXT:    vl8r.v v8, (a6) # Unknown-size Folded Reload
+; RV64-NEXT:    vsetvli zero, a5, e64, m8, ta, ma
+; RV64-NEXT:    vadd.vi v16, v8, -1, v0.t
 ; RV64-NEXT:    vnot.v v8, v8, v0.t
 ; RV64-NEXT:    vand.vv v8, v8, v16, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 1, v0.t
-; RV64-NEXT:    vand.vx v16, v16, a6, v0.t
+; RV64-NEXT:    vand.vx v16, v16, a4, v0.t
 ; RV64-NEXT:    vsub.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v16, v8, a5, v0.t
+; RV64-NEXT:    vand.vx v16, v8, a3, v0.t
 ; RV64-NEXT:    vsrl.vi v8, v8, 2, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a5, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a3, v0.t
 ; RV64-NEXT:    vadd.vv v8, v16, v8, v0.t
 ; RV64-NEXT:    vsrl.vi v16, v8, 4, v0.t
 ; RV64-NEXT:    vadd.vv v8, v8, v16, v0.t
-; RV64-NEXT:    vand.vx v8, v8, a2, v0.t
-; RV64-NEXT:    vmul.vx v8, v8, a3, v0.t
+; RV64-NEXT:    vand.vx v8, v8, a1, v0.t
+; RV64-NEXT:    vmul.vx v8, v8, a2, v0.t
 ; RV64-NEXT:    vsrl.vx v16, v8, a0, v0.t
 ; RV64-NEXT:    addi a0, sp, 16
 ; RV64-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
@@ -4314,108 +3928,101 @@ define <32 x i64> @vp_cttz_zero_undef_v32i64(<32 x i64> %va, <32 x i1> %m, i32 z
 define <32 x i64> @vp_cttz_zero_undef_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 ; RV32-LABEL: vp_cttz_zero_undef_v32i64_unmasked:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -48
-; RV32-NEXT:    .cfi_def_cfa_offset 48
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 8 * vlenb
-; RV32-NEXT:    lui a1, 349525
-; RV32-NEXT:    lui a2, 209715
-; RV32-NEXT:    addi a1, a1, 1365
-; RV32-NEXT:    sw a1, 40(sp)
-; RV32-NEXT:    sw a1, 44(sp)
-; RV32-NEXT:    lui a1, 61681
-; RV32-NEXT:    addi a2, a2, 819
-; RV32-NEXT:    sw a2, 32(sp)
-; RV32-NEXT:    sw a2, 36(sp)
-; RV32-NEXT:    lui a2, 4112
-; RV32-NEXT:    addi a1, a1, -241
-; RV32-NEXT:    sw a1, 24(sp)
-; RV32-NEXT:    sw a1, 28(sp)
-; RV32-NEXT:    li a3, 16
-; RV32-NEXT:    addi a1, a2, 257
-; RV32-NEXT:    sw a1, 16(sp)
-; RV32-NEXT:    sw a1, 20(sp)
+; RV32-NEXT:    li a2, 16
 ; RV32-NEXT:    mv a1, a0
-; RV32-NEXT:    bltu a0, a3, .LBB71_2
+; RV32-NEXT:    bltu a0, a2, .LBB71_2
 ; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    li a1, 16
 ; RV32-NEXT:  .LBB71_2:
-; RV32-NEXT:    li a2, 1
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a2, a2, 4
+; RV32-NEXT:    sub sp, sp, a2
+; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
+; RV32-NEXT:    vadd.vi v24, v8, -1
 ; RV32-NEXT:    vnot.v v0, v8
-; RV32-NEXT:    addi a3, sp, 40
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a3), zero
-; RV32-NEXT:    addi a3, a0, -16
-; RV32-NEXT:    sltu a0, a0, a3
+; RV32-NEXT:    lui a2, 349525
+; RV32-NEXT:    addi a2, a2, 1365
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a2
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a2, a2, 3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vs8r.v v8, (a2) # Unknown-size Folded Spill
+; RV32-NEXT:    addi a2, a0, -16
+; RV32-NEXT:    sltu a0, a0, a2
 ; RV32-NEXT:    addi a0, a0, -1
-; RV32-NEXT:    and a0, a0, a3
-; RV32-NEXT:    addi a3, sp, 32
+; RV32-NEXT:    and a0, a0, a2
+; RV32-NEXT:    lui a2, 209715
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v8, v8, a2
-; RV32-NEXT:    vand.vv v8, v0, v8
+; RV32-NEXT:    vand.vv v24, v0, v24
+; RV32-NEXT:    addi a2, a2, 819
+; RV32-NEXT:    vsrl.vi v0, v24, 1
+; RV32-NEXT:    vand.vv v0, v0, v8
+; RV32-NEXT:    vsub.vv v24, v24, v0
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vx v0, v16, a2
+; RV32-NEXT:    vadd.vi v0, v16, -1
 ; RV32-NEXT:    vnot.v v16, v16
-; RV32-NEXT:    vand.vv v16, v16, v0
+; RV32-NEXT:    vand.vv v0, v16, v0
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v16, a2
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vs8r.v v16, (a2) # Unknown-size Folded Spill
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v0, v8, 1
-; RV32-NEXT:    vand.vv v0, v0, v24
-; RV32-NEXT:    vsub.vv v0, v8, v0
+; RV32-NEXT:    vand.vv v8, v24, v16
+; RV32-NEXT:    vsrl.vi v24, v24, 2
+; RV32-NEXT:    vand.vv v24, v24, v16
+; RV32-NEXT:    vadd.vv v8, v8, v24
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v8, v16, 1
-; RV32-NEXT:    vand.vv v24, v8, v24
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a3), zero
-; RV32-NEXT:    addi a2, sp, 24
-; RV32-NEXT:    addi a3, sp, 16
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsub.vv v16, v16, v24
+; RV32-NEXT:    vsrl.vi v24, v0, 1
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a2, a2, 3
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
+; RV32-NEXT:    vand.vv v24, v24, v16
+; RV32-NEXT:    vsub.vv v24, v0, v24
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v24, v0, v8
-; RV32-NEXT:    vsrl.vi v0, v0, 2
-; RV32-NEXT:    vand.vv v0, v0, v8
-; RV32-NEXT:    vadd.vv v24, v24, v0
+; RV32-NEXT:    vsrl.vi v0, v8, 4
+; RV32-NEXT:    vadd.vv v8, v8, v0
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v0, v16, v8
-; RV32-NEXT:    vsrl.vi v16, v16, 2
-; RV32-NEXT:    vand.vv v8, v16, v8
+; RV32-NEXT:    vand.vv v0, v24, v16
+; RV32-NEXT:    vsrl.vi v24, v24, 2
+; RV32-NEXT:    vand.vv v16, v24, v16
+; RV32-NEXT:    lui a2, 61681
+; RV32-NEXT:    addi a2, a2, -241
+; RV32-NEXT:    vadd.vv v16, v0, v16
+; RV32-NEXT:    vsrl.vi v24, v16, 4
+; RV32-NEXT:    vadd.vv v16, v16, v24
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a2
+; RV32-NEXT:    lui a2, 4112
+; RV32-NEXT:    addi a2, a2, 257
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v24, 4
-; RV32-NEXT:    vadd.vv v16, v24, v16
-; RV32-NEXT:    addi a4, sp, 48
-; RV32-NEXT:    vs8r.v v16, (a4) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v24, (a2), zero
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vadd.vv v8, v0, v8
-; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV32-NEXT:    vlse64.v v0, (a3), zero
+; RV32-NEXT:    vand.vv v8, v8, v24
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vi v16, v8, 4
-; RV32-NEXT:    vadd.vv v8, v8, v16
-; RV32-NEXT:    addi a2, sp, 48
-; RV32-NEXT:    vl8r.v v16, (a2) # Unknown-size Folded Reload
-; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV32-NEXT:    vand.vv v16, v16, v24
-; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vand.vv v8, v8, v24
+; RV32-NEXT:    vsetvli a3, zero, e32, m8, ta, ma
+; RV32-NEXT:    vmv.v.x v24, a2
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vmul.vv v16, v16, v0
+; RV32-NEXT:    vmul.vv v8, v8, v24
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vmul.vv v24, v8, v0
+; RV32-NEXT:    vmul.vv v16, v16, v24
 ; RV32-NEXT:    li a2, 56
 ; RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vx v8, v16, a2
+; RV32-NEXT:    vsrl.vx v8, v8, a2
 ; RV32-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV32-NEXT:    vsrl.vx v16, v24, a2
+; RV32-NEXT:    vsrl.vx v16, v16, a2
 ; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 3
+; RV32-NEXT:    slli a0, a0, 4
 ; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 48
-; RV32-NEXT:    addi sp, sp, 48
+; RV32-NEXT:    .cfi_def_cfa sp, 16
+; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
@@ -4427,70 +4034,69 @@ define <32 x i64> @vp_cttz_zero_undef_v32i64_unmasked(<32 x i64> %va, i32 zeroex
 ; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    li a1, 16
 ; RV64-NEXT:  .LBB71_2:
-; RV64-NEXT:    li a2, 1
 ; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vnot.v v24, v8
-; RV64-NEXT:    lui a3, 349525
-; RV64-NEXT:    lui a4, 209715
-; RV64-NEXT:    lui a5, 61681
-; RV64-NEXT:    lui a6, 4112
-; RV64-NEXT:    addiw a3, a3, 1365
-; RV64-NEXT:    addiw a4, a4, 819
-; RV64-NEXT:    addiw a5, a5, -241
-; RV64-NEXT:    addiw a6, a6, 257
-; RV64-NEXT:    slli a7, a3, 32
-; RV64-NEXT:    add a3, a3, a7
-; RV64-NEXT:    slli a7, a4, 32
-; RV64-NEXT:    add a4, a4, a7
-; RV64-NEXT:    slli a7, a5, 32
-; RV64-NEXT:    add a5, a5, a7
-; RV64-NEXT:    slli a7, a6, 32
-; RV64-NEXT:    add a6, a6, a7
-; RV64-NEXT:    addi a7, a0, -16
-; RV64-NEXT:    sltu a0, a0, a7
+; RV64-NEXT:    vadd.vi v24, v8, -1
+; RV64-NEXT:    vnot.v v8, v8
+; RV64-NEXT:    lui a2, 349525
+; RV64-NEXT:    lui a3, 209715
+; RV64-NEXT:    lui a4, 61681
+; RV64-NEXT:    lui a5, 4112
+; RV64-NEXT:    addiw a2, a2, 1365
+; RV64-NEXT:    addiw a3, a3, 819
+; RV64-NEXT:    addiw a4, a4, -241
+; RV64-NEXT:    addiw a5, a5, 257
+; RV64-NEXT:    slli a6, a2, 32
+; RV64-NEXT:    add a2, a2, a6
+; RV64-NEXT:    slli a6, a3, 32
+; RV64-NEXT:    add a3, a3, a6
+; RV64-NEXT:    slli a6, a4, 32
+; RV64-NEXT:    add a4, a4, a6
+; RV64-NEXT:    slli a6, a5, 32
+; RV64-NEXT:    add a5, a5, a6
+; RV64-NEXT:    addi a6, a0, -16
+; RV64-NEXT:    sltu a0, a0, a6
 ; RV64-NEXT:    addi a0, a0, -1
-; RV64-NEXT:    and a0, a0, a7
-; RV64-NEXT:    li a7, 56
-; RV64-NEXT:    vsub.vx v8, v8, a2
-; RV64-NEXT:    vand.vv v8, v24, v8
+; RV64-NEXT:    and a0, a0, a6
+; RV64-NEXT:    li a6, 56
+; RV64-NEXT:    vand.vv v8, v8, v24
 ; RV64-NEXT:    vsrl.vi v24, v8, 1
-; RV64-NEXT:    vand.vx v24, v24, a3
+; RV64-NEXT:    vand.vx v24, v24, a2
 ; RV64-NEXT:    vsub.vv v8, v8, v24
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV64-NEXT:    vsub.vx v24, v16, a2
+; RV64-NEXT:    vadd.vi v24, v16, -1
 ; RV64-NEXT:    vnot.v v16, v16
 ; RV64-NEXT:    vand.vv v16, v16, v24
 ; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vand.vx v24, v8, a4
+; RV64-NEXT:    vand.vx v24, v8, a3
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vand.vx v8, v8, a4
+; RV64-NEXT:    vand.vx v8, v8, a3
 ; RV64-NEXT:    vadd.vv v8, v24, v8
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vsrl.vi v24, v16, 1
-; RV64-NEXT:    vand.vx v24, v24, a3
+; RV64-NEXT:    vand.vx v24, v24, a2
 ; RV64-NEXT:    vsub.vv v16, v16, v24
 ; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; RV64-NEXT:    vsrl.vi v24, v8, 4
 ; RV64-NEXT:    vadd.vv v8, v8, v24
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
-; RV64-NEXT:    vand.vx v24, v16, a4
+; RV64-NEXT:    vand.vx v24, v16, a3
 ; RV64-NEXT:    vsrl.vi v16, v16, 2
-; RV64-NEXT:    vand.vx v16, v16, a4
+; RV64-NEXT:    vand.vx v16, v16, a3
 ; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vand.vx v8, v8, a5
+; RV64-NEXT:    vand.vx v8, v8, a4
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vadd.vv v16, v24, v16
 ; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vmul.vx v8, v8, a6
+; RV64-NEXT:    vmul.vx v8, v8, a5
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vsrl.vi v24, v16, 4
 ; RV64-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
-; RV64-NEXT:    vsrl.vx v8, v8, a7
+; RV64-NEXT:    vsrl.vx v8, v8, a6
 ; RV64-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; RV64-NEXT:    vadd.vv v16, v16, v24
-; RV64-NEXT:    vand.vx v16, v16, a5
-; RV64-NEXT:    vmul.vx v16, v16, a6
-; RV64-NEXT:    vsrl.vx v16, v16, a7
+; RV64-NEXT:    vand.vx v16, v16, a4
+; RV64-NEXT:    vmul.vx v16, v16, a5
+; RV64-NEXT:    vsrl.vx v16, v16, a6
 ; RV64-NEXT:    ret
   %v = call <32 x i64> @llvm.vp.cttz.v32i64(<32 x i64> %va, i1 true, <32 x i1> splat (i1 true), i32 %evl)
   ret <32 x i64> %v
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz.ll
index 57e0eeb92ee2f..1724b92a9ab48 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-cttz.ll
@@ -13,9 +13,8 @@ define void @cttz_v16i8(ptr %x, ptr %y) nounwind {
 ; RVI:       # %bb.0:
 ; RVI-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; RVI-NEXT:    vle8.v v8, (a0)
-; RVI-NEXT:    li a1, 1
-; RVI-NEXT:    vsub.vx v9, v8, a1
 ; RVI-NEXT:    li a1, 85
+; RVI-NEXT:    vadd.vi v9, v8, -1
 ; RVI-NEXT:    vnot.v v8, v8
 ; RVI-NEXT:    vand.vv v8, v8, v9
 ; RVI-NEXT:    vsrl.vi v9, v8, 1
@@ -90,10 +89,9 @@ define void @cttz_v8i16(ptr %x, ptr %y) nounwind {
 ; RVI:       # %bb.0:
 ; RVI-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; RVI-NEXT:    vle16.v v8, (a0)
-; RVI-NEXT:    li a1, 1
-; RVI-NEXT:    vsub.vx v9, v8, a1
 ; RVI-NEXT:    lui a1, 5
 ; RVI-NEXT:    addi a1, a1, 1365
+; RVI-NEXT:    vadd.vi v9, v8, -1
 ; RVI-NEXT:    vnot.v v8, v8
 ; RVI-NEXT:    vand.vv v8, v8, v9
 ; RVI-NEXT:    vsrl.vi v9, v8, 1
@@ -168,10 +166,9 @@ define void @cttz_v4i32(ptr %x, ptr %y) nounwind {
 ; RVI:       # %bb.0:
 ; RVI-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RVI-NEXT:    vle32.v v8, (a0)
-; RVI-NEXT:    li a1, 1
-; RVI-NEXT:    vsub.vx v9, v8, a1
 ; RVI-NEXT:    lui a1, 349525
 ; RVI-NEXT:    addi a1, a1, 1365
+; RVI-NEXT:    vadd.vi v9, v8, -1
 ; RVI-NEXT:    vnot.v v8, v8
 ; RVI-NEXT:    vand.vv v8, v8, v9
 ; RVI-NEXT:    vsrl.vi v9, v8, 1
@@ -254,11 +251,10 @@ define void @cttz_v2i64(ptr %x, ptr %y) nounwind {
 ; RV32I-NEXT:    addi a1, a1, 1365
 ; RV32I-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV32I-NEXT:    vmv.v.x v9, a1
-; RV32I-NEXT:    li a1, 1
-; RV32I-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV32I-NEXT:    vsub.vx v10, v8, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
+; RV32I-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; RV32I-NEXT:    vadd.vi v10, v8, -1
 ; RV32I-NEXT:    vnot.v v8, v8
 ; RV32I-NEXT:    vand.vv v8, v8, v10
 ; RV32I-NEXT:    vsrl.vi v10, v8, 1
@@ -310,8 +306,7 @@ define void @cttz_v2i64(ptr %x, ptr %y) nounwind {
 ; RV64I-NEXT:    add a3, a3, a5
 ; RV64I-NEXT:    slli a5, a4, 32
 ; RV64I-NEXT:    add a4, a4, a5
-; RV64I-NEXT:    li a5, 1
-; RV64I-NEXT:    vsub.vx v9, v8, a5
+; RV64I-NEXT:    vadd.vi v9, v8, -1
 ; RV64I-NEXT:    vnot.v v8, v8
 ; RV64I-NEXT:    vand.vv v8, v8, v9
 ; RV64I-NEXT:    vsrl.vi v9, v8, 1
@@ -392,9 +387,8 @@ define void @cttz_v32i8(ptr %x, ptr %y) nounwind {
 ; RVI-NEXT:    li a1, 32
 ; RVI-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; RVI-NEXT:    vle8.v v8, (a0)
-; RVI-NEXT:    li a1, 1
-; RVI-NEXT:    vsub.vx v10, v8, a1
 ; RVI-NEXT:    li a1, 85
+; RVI-NEXT:    vadd.vi v10, v8, -1
 ; RVI-NEXT:    vnot.v v8, v8
 ; RVI-NEXT:    vand.vv v8, v8, v10
 ; RVI-NEXT:    vsrl.vi v10, v8, 1
@@ -472,10 +466,9 @@ define void @cttz_v16i16(ptr %x, ptr %y) nounwind {
 ; RVI:       # %bb.0:
 ; RVI-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; RVI-NEXT:    vle16.v v8, (a0)
-; RVI-NEXT:    li a1, 1
-; RVI-NEXT:    vsub.vx v10, v8, a1
 ; RVI-NEXT:    lui a1, 5
 ; RVI-NEXT:    addi a1, a1, 1365
+; RVI-NEXT:    vadd.vi v10, v8, -1
 ; RVI-NEXT:    vnot.v v8, v8
 ; RVI-NEXT:    vand.vv v8, v8, v10
 ; RVI-NEXT:    vsrl.vi v10, v8, 1
@@ -550,10 +543,9 @@ define void @cttz_v8i32(ptr %x, ptr %y) nounwind {
 ; RVI:       # %bb.0:
 ; RVI-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; RVI-NEXT:    vle32.v v8, (a0)
-; RVI-NEXT:    li a1, 1
-; RVI-NEXT:    vsub.vx v10, v8, a1
 ; RVI-NEXT:    lui a1, 349525
 ; RVI-NEXT:    addi a1, a1, 1365
+; RVI-NEXT:    vadd.vi v10, v8, -1
 ; RVI-NEXT:    vnot.v v8, v8
 ; RVI-NEXT:    vand.vv v8, v8, v10
 ; RVI-NEXT:    vsrl.vi v10, v8, 1
@@ -636,11 +628,10 @@ define void @cttz_v4i64(ptr %x, ptr %y) nounwind {
 ; RV32I-NEXT:    addi a1, a1, 1365
 ; RV32I-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; RV32I-NEXT:    vmv.v.x v10, a1
-; RV32I-NEXT:    li a1, 1
-; RV32I-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; RV32I-NEXT:    vsub.vx v12, v8, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
+; RV32I-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV32I-NEXT:    vadd.vi v12, v8, -1
 ; RV32I-NEXT:    vnot.v v8, v8
 ; RV32I-NEXT:    vand.vv v8, v8, v12
 ; RV32I-NEXT:    vsrl.vi v12, v8, 1
@@ -692,8 +683,7 @@ define void @cttz_v4i64(ptr %x, ptr %y) nounwind {
 ; RV64I-NEXT:    add a3, a3, a5
 ; RV64I-NEXT:    slli a5, a4, 32
 ; RV64I-NEXT:    add a4, a4, a5
-; RV64I-NEXT:    li a5, 1
-; RV64I-NEXT:    vsub.vx v10, v8, a5
+; RV64I-NEXT:    vadd.vi v10, v8, -1
 ; RV64I-NEXT:    vnot.v v8, v8
 ; RV64I-NEXT:    vand.vv v8, v8, v10
 ; RV64I-NEXT:    vsrl.vi v10, v8, 1
@@ -773,9 +763,8 @@ define void @cttz_zero_undef_v16i8(ptr %x, ptr %y) nounwind {
 ; RVI:       # %bb.0:
 ; RVI-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; RVI-NEXT:    vle8.v v8, (a0)
-; RVI-NEXT:    li a1, 1
-; RVI-NEXT:    vsub.vx v9, v8, a1
 ; RVI-NEXT:    li a1, 85
+; RVI-NEXT:    vadd.vi v9, v8, -1
 ; RVI-NEXT:    vnot.v v8, v8
 ; RVI-NEXT:    vand.vv v8, v8, v9
 ; RVI-NEXT:    vsrl.vi v9, v8, 1
@@ -845,10 +834,9 @@ define void @cttz_zero_undef_v8i16(ptr %x, ptr %y) nounwind {
 ; RVI:       # %bb.0:
 ; RVI-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; RVI-NEXT:    vle16.v v8, (a0)
-; RVI-NEXT:    li a1, 1
-; RVI-NEXT:    vsub.vx v9, v8, a1
 ; RVI-NEXT:    lui a1, 5
 ; RVI-NEXT:    addi a1, a1, 1365
+; RVI-NEXT:    vadd.vi v9, v8, -1
 ; RVI-NEXT:    vnot.v v8, v8
 ; RVI-NEXT:    vand.vv v8, v8, v9
 ; RVI-NEXT:    vsrl.vi v9, v8, 1
@@ -916,10 +904,9 @@ define void @cttz_zero_undef_v4i32(ptr %x, ptr %y) nounwind {
 ; RVI:       # %bb.0:
 ; RVI-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RVI-NEXT:    vle32.v v8, (a0)
-; RVI-NEXT:    li a1, 1
-; RVI-NEXT:    vsub.vx v9, v8, a1
 ; RVI-NEXT:    lui a1, 349525
 ; RVI-NEXT:    addi a1, a1, 1365
+; RVI-NEXT:    vadd.vi v9, v8, -1
 ; RVI-NEXT:    vnot.v v8, v8
 ; RVI-NEXT:    vand.vv v8, v8, v9
 ; RVI-NEXT:    vsrl.vi v9, v8, 1
@@ -995,11 +982,10 @@ define void @cttz_zero_undef_v2i64(ptr %x, ptr %y) nounwind {
 ; RV32I-NEXT:    addi a1, a1, 1365
 ; RV32I-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV32I-NEXT:    vmv.v.x v9, a1
-; RV32I-NEXT:    li a1, 1
-; RV32I-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV32I-NEXT:    vsub.vx v10, v8, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
+; RV32I-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; RV32I-NEXT:    vadd.vi v10, v8, -1
 ; RV32I-NEXT:    vnot.v v8, v8
 ; RV32I-NEXT:    vand.vv v8, v8, v10
 ; RV32I-NEXT:    vsrl.vi v10, v8, 1
@@ -1051,8 +1037,7 @@ define void @cttz_zero_undef_v2i64(ptr %x, ptr %y) nounwind {
 ; RV64I-NEXT:    add a3, a3, a5
 ; RV64I-NEXT:    slli a5, a4, 32
 ; RV64I-NEXT:    add a4, a4, a5
-; RV64I-NEXT:    li a5, 1
-; RV64I-NEXT:    vsub.vx v9, v8, a5
+; RV64I-NEXT:    vadd.vi v9, v8, -1
 ; RV64I-NEXT:    vnot.v v8, v8
 ; RV64I-NEXT:    vand.vv v8, v8, v9
 ; RV64I-NEXT:    vsrl.vi v9, v8, 1
@@ -1123,9 +1108,8 @@ define void @cttz_zero_undef_v32i8(ptr %x, ptr %y) nounwind {
 ; RVI-NEXT:    li a1, 32
 ; RVI-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
 ; RVI-NEXT:    vle8.v v8, (a0)
-; RVI-NEXT:    li a1, 1
-; RVI-NEXT:    vsub.vx v10, v8, a1
 ; RVI-NEXT:    li a1, 85
+; RVI-NEXT:    vadd.vi v10, v8, -1
 ; RVI-NEXT:    vnot.v v8, v8
 ; RVI-NEXT:    vand.vv v8, v8, v10
 ; RVI-NEXT:    vsrl.vi v10, v8, 1
@@ -1198,10 +1182,9 @@ define void @cttz_zero_undef_v16i16(ptr %x, ptr %y) nounwind {
 ; RVI:       # %bb.0:
 ; RVI-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; RVI-NEXT:    vle16.v v8, (a0)
-; RVI-NEXT:    li a1, 1
-; RVI-NEXT:    vsub.vx v10, v8, a1
 ; RVI-NEXT:    lui a1, 5
 ; RVI-NEXT:    addi a1, a1, 1365
+; RVI-NEXT:    vadd.vi v10, v8, -1
 ; RVI-NEXT:    vnot.v v8, v8
 ; RVI-NEXT:    vand.vv v8, v8, v10
 ; RVI-NEXT:    vsrl.vi v10, v8, 1
@@ -1269,10 +1252,9 @@ define void @cttz_zero_undef_v8i32(ptr %x, ptr %y) nounwind {
 ; RVI:       # %bb.0:
 ; RVI-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; RVI-NEXT:    vle32.v v8, (a0)
-; RVI-NEXT:    li a1, 1
-; RVI-NEXT:    vsub.vx v10, v8, a1
 ; RVI-NEXT:    lui a1, 349525
 ; RVI-NEXT:    addi a1, a1, 1365
+; RVI-NEXT:    vadd.vi v10, v8, -1
 ; RVI-NEXT:    vnot.v v8, v8
 ; RVI-NEXT:    vand.vv v8, v8, v10
 ; RVI-NEXT:    vsrl.vi v10, v8, 1
@@ -1348,11 +1330,10 @@ define void @cttz_zero_undef_v4i64(ptr %x, ptr %y) nounwind {
 ; RV32I-NEXT:    addi a1, a1, 1365
 ; RV32I-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; RV32I-NEXT:    vmv.v.x v10, a1
-; RV32I-NEXT:    li a1, 1
-; RV32I-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; RV32I-NEXT:    vsub.vx v12, v8, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
+; RV32I-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV32I-NEXT:    vadd.vi v12, v8, -1
 ; RV32I-NEXT:    vnot.v v8, v8
 ; RV32I-NEXT:    vand.vv v8, v8, v12
 ; RV32I-NEXT:    vsrl.vi v12, v8, 1
@@ -1404,8 +1385,7 @@ define void @cttz_zero_undef_v4i64(ptr %x, ptr %y) nounwind {
 ; RV64I-NEXT:    add a3, a3, a5
 ; RV64I-NEXT:    slli a5, a4, 32
 ; RV64I-NEXT:    add a4, a4, a5
-; RV64I-NEXT:    li a5, 1
-; RV64I-NEXT:    vsub.vx v10, v8, a5
+; RV64I-NEXT:    vadd.vi v10, v8, -1
 ; RV64I-NEXT:    vnot.v v8, v8
 ; RV64I-NEXT:    vand.vv v8, v8, v10
 ; RV64I-NEXT:    vsrl.vi v10, v8, 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll
index 39fd70beb9ee2..0436a27409f81 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll
@@ -4031,8 +4031,7 @@ define void @sub_vi_v16i8(ptr %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    li a1, -1
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v8, v8, 1
 ; CHECK-NEXT:    vse8.v v8, (a0)
 ; CHECK-NEXT:    ret
   %a = load <16 x i8>, ptr %x
@@ -4046,8 +4045,7 @@ define void @sub_vi_v8i16(ptr %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    li a1, -1
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v8, v8, 1
 ; CHECK-NEXT:    vse16.v v8, (a0)
 ; CHECK-NEXT:    ret
   %a = load <8 x i16>, ptr %x
@@ -4061,8 +4059,7 @@ define void @sub_vi_v4i32(ptr %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vle32.v v8, (a0)
-; CHECK-NEXT:    li a1, -1
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v8, v8, 1
 ; CHECK-NEXT:    vse32.v v8, (a0)
 ; CHECK-NEXT:    ret
   %a = load <4 x i32>, ptr %x
@@ -4076,8 +4073,7 @@ define void @sub_vi_v2i64(ptr %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
-; CHECK-NEXT:    li a1, -1
-; CHECK-NEXT:    vsub.vx v8, v8, a1
+; CHECK-NEXT:    vadd.vi v8, v8, 1
 ; CHECK-NEXT:    vse64.v v8, (a0)
 ; CHECK-NEXT:    ret
   %a = load <2 x i64>, ptr %x
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
index 7b9a30187f5e3..0928a47757430 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
@@ -380,30 +380,18 @@ entry:
   ret void
 }
 
-; TODO: Allow an undef initial element
 define void @vnsrl_0_i8_undef3(ptr %in, ptr %out) {
 ; CHECK-LABEL: vnsrl_0_i8_undef3:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetivli zero, 16, e8, mf2, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    vmv.v.i v0, 8
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf4, ta, mu
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    li a0, -32
-; CHECK-NEXT:    vadd.vv v9, v9, v9
-; CHECK-NEXT:    vadd.vi v9, v9, -8
-; CHECK-NEXT:    vslidedown.vi v10, v8, 2
-; CHECK-NEXT:    vslidedown.vi v10, v8, 3, v0.t
-; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 8
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf4, ta, mu
-; CHECK-NEXT:    vrgather.vv v10, v8, v9, v0.t
-; CHECK-NEXT:    vse8.v v10, (a1)
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf4, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    vse8.v v8, (a1)
 ; CHECK-NEXT:    ret
 entry:
   %0 = load <16 x i8>, ptr %in, align 1
-  %shuffle.i5 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 undef, i32 undef, i32 4, i32 6, i32 6, i32 10, i32 12, i32 14>
+  %shuffle.i5 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 undef, i32 undef, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   store <8 x i8> %shuffle.i5, ptr %out, align 1
   ret void
 }
diff --git a/llvm/test/CodeGen/RISCV/rvv/fold-binop-into-select.ll b/llvm/test/CodeGen/RISCV/rvv/fold-binop-into-select.ll
index 3a8d08f306a51..f3ad54afa21c2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fold-binop-into-select.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fold-binop-into-select.ll
@@ -39,9 +39,8 @@ define <vscale x 2 x i32> @i1_zext_add_commuted(<vscale x 2 x i1> %a, <vscale x
 define <vscale x 2 x i32> @i1_zext_sub(<vscale x 2 x i1> %a, <vscale x 2 x i32> %b) {
 ; CHECK-LABEL: i1_zext_sub:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
-; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, mu
-; CHECK-NEXT:    vsub.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, mu
+; CHECK-NEXT:    vadd.vi v8, v8, -1, v0.t
 ; CHECK-NEXT:    ret
   %zext = zext <vscale x 2 x i1> %a to <vscale x 2 x i32>
   %sub = sub <vscale x 2 x i32> %b, %zext
diff --git a/llvm/test/CodeGen/RISCV/rvv/urem-seteq-vec.ll b/llvm/test/CodeGen/RISCV/rvv/urem-seteq-vec.ll
index 0bd82e654e021..d34b401529202 100644
--- a/llvm/test/CodeGen/RISCV/rvv/urem-seteq-vec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/urem-seteq-vec.ll
@@ -75,9 +75,8 @@ define <vscale x 1 x i16> @test_urem_vec_odd_divisor_eq0(<vscale x 1 x i16> %x)
 define <vscale x 1 x i16> @test_urem_vec_even_divisor_eq1(<vscale x 1 x i16> %x) nounwind {
 ; RV32-LABEL: test_urem_vec_even_divisor_eq1:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a0, 1
-; RV32-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; RV32-NEXT:    vsub.vx v8, v8, a0
+; RV32-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; RV32-NEXT:    vadd.vi v8, v8, -1
 ; RV32-NEXT:    lui a0, 1048571
 ; RV32-NEXT:    addi a0, a0, -1365
 ; RV32-NEXT:    vmul.vx v8, v8, a0
@@ -93,9 +92,8 @@ define <vscale x 1 x i16> @test_urem_vec_even_divisor_eq1(<vscale x 1 x i16> %x)
 ;
 ; RV64-LABEL: test_urem_vec_even_divisor_eq1:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a0, 1
-; RV64-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; RV64-NEXT:    vsub.vx v8, v8, a0
+; RV64-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; RV64-NEXT:    vadd.vi v8, v8, -1
 ; RV64-NEXT:    lui a0, 1048571
 ; RV64-NEXT:    addi a0, a0, -1365
 ; RV64-NEXT:    vmul.vx v8, v8, a0
@@ -117,9 +115,8 @@ define <vscale x 1 x i16> @test_urem_vec_even_divisor_eq1(<vscale x 1 x i16> %x)
 define <vscale x 1 x i16> @test_urem_vec_odd_divisor_eq1(<vscale x 1 x i16> %x) nounwind {
 ; RV32-LABEL: test_urem_vec_odd_divisor_eq1:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    li a0, 1
-; RV32-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; RV32-NEXT:    vsub.vx v8, v8, a0
+; RV32-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; RV32-NEXT:    vadd.vi v8, v8, -1
 ; RV32-NEXT:    lui a0, 1048573
 ; RV32-NEXT:    addi a0, a0, -819
 ; RV32-NEXT:    vmul.vx v8, v8, a0
@@ -132,9 +129,8 @@ define <vscale x 1 x i16> @test_urem_vec_odd_divisor_eq1(<vscale x 1 x i16> %x)
 ;
 ; RV64-LABEL: test_urem_vec_odd_divisor_eq1:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    li a0, 1
-; RV64-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; RV64-NEXT:    vsub.vx v8, v8, a0
+; RV64-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; RV64-NEXT:    vadd.vi v8, v8, -1
 ; RV64-NEXT:    lui a0, 1048573
 ; RV64-NEXT:    addi a0, a0, -819
 ; RV64-NEXT:    vmul.vx v8, v8, a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/vscale-power-of-two.ll b/llvm/test/CodeGen/RISCV/rvv/vscale-power-of-two.ll
index b2a719b1f7901..8eef133d0e76c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vscale-power-of-two.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vscale-power-of-two.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv64 -mattr=+v,+m -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+Zve64x,+m -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+zve64x,+m -verify-machineinstrs < %s | FileCheck %s
 
 declare i64 @llvm.vscale.i64()
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vscale-vw-web-simplification.ll b/llvm/test/CodeGen/RISCV/rvv/vscale-vw-web-simplification.ll
index 58b6f0253b99a..d2ef711fc0d74 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vscale-vw-web-simplification.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vscale-vw-web-simplification.ll
@@ -156,7 +156,6 @@ define <vscale x 2 x i32> @vwop_vscale_sext_i1i32_multiple_users(ptr %x, ptr %y,
 ; NO_FOLDING-NEXT:    vlm.v v9, (a1)
 ; NO_FOLDING-NEXT:    vlm.v v10, (a2)
 ; NO_FOLDING-NEXT:    vmv.v.i v11, 0
-; NO_FOLDING-NEXT:    li a0, 1
 ; NO_FOLDING-NEXT:    vmv.v.v v0, v8
 ; NO_FOLDING-NEXT:    vmerge.vim v12, v11, -1, v0
 ; NO_FOLDING-NEXT:    vmv.v.v v0, v9
@@ -166,7 +165,7 @@ define <vscale x 2 x i32> @vwop_vscale_sext_i1i32_multiple_users(ptr %x, ptr %y,
 ; NO_FOLDING-NEXT:    vmul.vv v9, v12, v9
 ; NO_FOLDING-NEXT:    vsub.vv v11, v12, v10
 ; NO_FOLDING-NEXT:    vmv.v.v v0, v8
-; NO_FOLDING-NEXT:    vsub.vx v10, v10, a0, v0.t
+; NO_FOLDING-NEXT:    vadd.vi v10, v10, -1, v0.t
 ; NO_FOLDING-NEXT:    vor.vv v8, v9, v10
 ; NO_FOLDING-NEXT:    vor.vv v8, v8, v11
 ; NO_FOLDING-NEXT:    ret
@@ -178,7 +177,6 @@ define <vscale x 2 x i32> @vwop_vscale_sext_i1i32_multiple_users(ptr %x, ptr %y,
 ; FOLDING-NEXT:    vlm.v v9, (a1)
 ; FOLDING-NEXT:    vlm.v v10, (a2)
 ; FOLDING-NEXT:    vmv.v.i v11, 0
-; FOLDING-NEXT:    li a0, 1
 ; FOLDING-NEXT:    vmv.v.v v0, v8
 ; FOLDING-NEXT:    vmerge.vim v12, v11, -1, v0
 ; FOLDING-NEXT:    vmv.v.v v0, v9
@@ -188,7 +186,7 @@ define <vscale x 2 x i32> @vwop_vscale_sext_i1i32_multiple_users(ptr %x, ptr %y,
 ; FOLDING-NEXT:    vmul.vv v9, v12, v9
 ; FOLDING-NEXT:    vsub.vv v11, v12, v10
 ; FOLDING-NEXT:    vmv.v.v v0, v8
-; FOLDING-NEXT:    vsub.vx v10, v10, a0, v0.t
+; FOLDING-NEXT:    vadd.vi v10, v10, -1, v0.t
 ; FOLDING-NEXT:    vor.vv v8, v9, v10
 ; FOLDING-NEXT:    vor.vv v8, v8, v11
 ; FOLDING-NEXT:    ret
@@ -214,7 +212,6 @@ define <vscale x 2 x i8> @vwop_vscale_sext_i1i8_multiple_users(ptr %x, ptr %y, p
 ; NO_FOLDING-NEXT:    vlm.v v9, (a1)
 ; NO_FOLDING-NEXT:    vlm.v v10, (a2)
 ; NO_FOLDING-NEXT:    vmv.v.i v11, 0
-; NO_FOLDING-NEXT:    li a0, 1
 ; NO_FOLDING-NEXT:    vmv1r.v v0, v8
 ; NO_FOLDING-NEXT:    vmerge.vim v12, v11, -1, v0
 ; NO_FOLDING-NEXT:    vmv1r.v v0, v9
@@ -224,7 +221,7 @@ define <vscale x 2 x i8> @vwop_vscale_sext_i1i8_multiple_users(ptr %x, ptr %y, p
 ; NO_FOLDING-NEXT:    vmul.vv v9, v12, v9
 ; NO_FOLDING-NEXT:    vsub.vv v11, v12, v10
 ; NO_FOLDING-NEXT:    vmv1r.v v0, v8
-; NO_FOLDING-NEXT:    vsub.vx v10, v10, a0, v0.t
+; NO_FOLDING-NEXT:    vadd.vi v10, v10, -1, v0.t
 ; NO_FOLDING-NEXT:    vor.vv v8, v9, v10
 ; NO_FOLDING-NEXT:    vor.vv v8, v8, v11
 ; NO_FOLDING-NEXT:    ret
@@ -236,7 +233,6 @@ define <vscale x 2 x i8> @vwop_vscale_sext_i1i8_multiple_users(ptr %x, ptr %y, p
 ; FOLDING-NEXT:    vlm.v v9, (a1)
 ; FOLDING-NEXT:    vlm.v v10, (a2)
 ; FOLDING-NEXT:    vmv.v.i v11, 0
-; FOLDING-NEXT:    li a0, 1
 ; FOLDING-NEXT:    vmv1r.v v0, v8
 ; FOLDING-NEXT:    vmerge.vim v12, v11, -1, v0
 ; FOLDING-NEXT:    vmv1r.v v0, v9
@@ -246,7 +242,7 @@ define <vscale x 2 x i8> @vwop_vscale_sext_i1i8_multiple_users(ptr %x, ptr %y, p
 ; FOLDING-NEXT:    vmul.vv v9, v12, v9
 ; FOLDING-NEXT:    vsub.vv v11, v12, v10
 ; FOLDING-NEXT:    vmv1r.v v0, v8
-; FOLDING-NEXT:    vsub.vx v10, v10, a0, v0.t
+; FOLDING-NEXT:    vadd.vi v10, v10, -1, v0.t
 ; FOLDING-NEXT:    vor.vv v8, v9, v10
 ; FOLDING-NEXT:    vor.vv v8, v8, v11
 ; FOLDING-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsub-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vsub-sdnode.ll
index 726b47c5a745f..7442be92fffcd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsub-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsub-sdnode.ll
@@ -27,11 +27,11 @@ define <vscale x 1 x i8> @vsub_vx_nxv1i8(<vscale x 1 x i8> %va, i8 signext %b) {
 define <vscale x 1 x i8> @vsub_vx_nxv1i8_0(<vscale x 1 x i8> %va) {
 ; CHECK-LABEL: vsub_vx_nxv1i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 1 x i8> %va, splat (i8 1)
+  %vc = sub <vscale x 1 x i8> %va, splat (i8 17)
   ret <vscale x 1 x i8> %vc
 }
 
@@ -71,11 +71,11 @@ define <vscale x 2 x i8> @vsub_vx_nxv2i8(<vscale x 2 x i8> %va, i8 signext %b) {
 define <vscale x 2 x i8> @vsub_vx_nxv2i8_0(<vscale x 2 x i8> %va) {
 ; CHECK-LABEL: vsub_vx_nxv2i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 2 x i8> %va, splat (i8 1)
+  %vc = sub <vscale x 2 x i8> %va, splat (i8 17)
   ret <vscale x 2 x i8> %vc
 }
 
@@ -104,11 +104,11 @@ define <vscale x 4 x i8> @vsub_vx_nxv4i8(<vscale x 4 x i8> %va, i8 signext %b) {
 define <vscale x 4 x i8> @vsub_vx_nxv4i8_0(<vscale x 4 x i8> %va) {
 ; CHECK-LABEL: vsub_vx_nxv4i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 4 x i8> %va, splat (i8 1)
+  %vc = sub <vscale x 4 x i8> %va, splat (i8 17)
   ret <vscale x 4 x i8> %vc
 }
 
@@ -137,11 +137,11 @@ define <vscale x 8 x i8> @vsub_vx_nxv8i8(<vscale x 8 x i8> %va, i8 signext %b) {
 define <vscale x 8 x i8> @vsub_vx_nxv8i8_0(<vscale x 8 x i8> %va) {
 ; CHECK-LABEL: vsub_vx_nxv8i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 8 x i8> %va, splat (i8 1)
+  %vc = sub <vscale x 8 x i8> %va, splat (i8 17)
   ret <vscale x 8 x i8> %vc
 }
 
@@ -170,11 +170,11 @@ define <vscale x 16 x i8> @vsub_vx_nxv16i8(<vscale x 16 x i8> %va, i8 signext %b
 define <vscale x 16 x i8> @vsub_vx_nxv16i8_0(<vscale x 16 x i8> %va) {
 ; CHECK-LABEL: vsub_vx_nxv16i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e8, m2, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 16 x i8> %va, splat (i8 1)
+  %vc = sub <vscale x 16 x i8> %va, splat (i8 17)
   ret <vscale x 16 x i8> %vc
 }
 
@@ -203,11 +203,11 @@ define <vscale x 32 x i8> @vsub_vx_nxv32i8(<vscale x 32 x i8> %va, i8 signext %b
 define <vscale x 32 x i8> @vsub_vx_nxv32i8_0(<vscale x 32 x i8> %va) {
 ; CHECK-LABEL: vsub_vx_nxv32i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e8, m4, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 32 x i8> %va, splat (i8 1)
+  %vc = sub <vscale x 32 x i8> %va, splat (i8 17)
   ret <vscale x 32 x i8> %vc
 }
 
@@ -236,11 +236,11 @@ define <vscale x 64 x i8> @vsub_vx_nxv64i8(<vscale x 64 x i8> %va, i8 signext %b
 define <vscale x 64 x i8> @vsub_vx_nxv64i8_0(<vscale x 64 x i8> %va) {
 ; CHECK-LABEL: vsub_vx_nxv64i8_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e8, m8, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 64 x i8> %va, splat (i8 1)
+  %vc = sub <vscale x 64 x i8> %va, splat (i8 17)
   ret <vscale x 64 x i8> %vc
 }
 
@@ -269,11 +269,11 @@ define <vscale x 1 x i16> @vsub_vx_nxv1i16(<vscale x 1 x i16> %va, i16 signext %
 define <vscale x 1 x i16> @vsub_vx_nxv1i16_0(<vscale x 1 x i16> %va) {
 ; CHECK-LABEL: vsub_vx_nxv1i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 1 x i16> %va, splat (i16 1)
+  %vc = sub <vscale x 1 x i16> %va, splat (i16 17)
   ret <vscale x 1 x i16> %vc
 }
 
@@ -302,11 +302,11 @@ define <vscale x 2 x i16> @vsub_vx_nxv2i16(<vscale x 2 x i16> %va, i16 signext %
 define <vscale x 2 x i16> @vsub_vx_nxv2i16_0(<vscale x 2 x i16> %va) {
 ; CHECK-LABEL: vsub_vx_nxv2i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 2 x i16> %va, splat (i16 1)
+  %vc = sub <vscale x 2 x i16> %va, splat (i16 17)
   ret <vscale x 2 x i16> %vc
 }
 
@@ -335,11 +335,11 @@ define <vscale x 4 x i16> @vsub_vx_nxv4i16(<vscale x 4 x i16> %va, i16 signext %
 define <vscale x 4 x i16> @vsub_vx_nxv4i16_0(<vscale x 4 x i16> %va) {
 ; CHECK-LABEL: vsub_vx_nxv4i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 4 x i16> %va, splat (i16 1)
+  %vc = sub <vscale x 4 x i16> %va, splat (i16 17)
   ret <vscale x 4 x i16> %vc
 }
 
@@ -368,11 +368,11 @@ define <vscale x 8 x i16> @vsub_vx_nxv8i16(<vscale x 8 x i16> %va, i16 signext %
 define <vscale x 8 x i16> @vsub_vx_nxv8i16_0(<vscale x 8 x i16> %va) {
 ; CHECK-LABEL: vsub_vx_nxv8i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 8 x i16> %va, splat (i16 1)
+  %vc = sub <vscale x 8 x i16> %va, splat (i16 17)
   ret <vscale x 8 x i16> %vc
 }
 
@@ -401,11 +401,11 @@ define <vscale x 16 x i16> @vsub_vx_nxv16i16(<vscale x 16 x i16> %va, i16 signex
 define <vscale x 16 x i16> @vsub_vx_nxv16i16_0(<vscale x 16 x i16> %va) {
 ; CHECK-LABEL: vsub_vx_nxv16i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 16 x i16> %va, splat (i16 1)
+  %vc = sub <vscale x 16 x i16> %va, splat (i16 17)
   ret <vscale x 16 x i16> %vc
 }
 
@@ -434,11 +434,11 @@ define <vscale x 32 x i16> @vsub_vx_nxv32i16(<vscale x 32 x i16> %va, i16 signex
 define <vscale x 32 x i16> @vsub_vx_nxv32i16_0(<vscale x 32 x i16> %va) {
 ; CHECK-LABEL: vsub_vx_nxv32i16_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m8, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 32 x i16> %va, splat (i16 1)
+  %vc = sub <vscale x 32 x i16> %va, splat (i16 17)
   ret <vscale x 32 x i16> %vc
 }
 
@@ -467,11 +467,11 @@ define <vscale x 1 x i32> @vsub_vx_nxv1i32(<vscale x 1 x i32> %va, i32 signext %
 define <vscale x 1 x i32> @vsub_vx_nxv1i32_0(<vscale x 1 x i32> %va) {
 ; CHECK-LABEL: vsub_vx_nxv1i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 1 x i32> %va, splat (i32 1)
+  %vc = sub <vscale x 1 x i32> %va, splat (i32 17)
   ret <vscale x 1 x i32> %vc
 }
 
@@ -500,11 +500,11 @@ define <vscale x 2 x i32> @vsub_vx_nxv2i32(<vscale x 2 x i32> %va, i32 signext %
 define <vscale x 2 x i32> @vsub_vx_nxv2i32_0(<vscale x 2 x i32> %va) {
 ; CHECK-LABEL: vsub_vx_nxv2i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 2 x i32> %va, splat (i32 1)
+  %vc = sub <vscale x 2 x i32> %va, splat (i32 17)
   ret <vscale x 2 x i32> %vc
 }
 
@@ -533,11 +533,11 @@ define <vscale x 4 x i32> @vsub_vx_nxv4i32(<vscale x 4 x i32> %va, i32 signext %
 define <vscale x 4 x i32> @vsub_vx_nxv4i32_0(<vscale x 4 x i32> %va) {
 ; CHECK-LABEL: vsub_vx_nxv4i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 4 x i32> %va, splat (i32 1)
+  %vc = sub <vscale x 4 x i32> %va, splat (i32 17)
   ret <vscale x 4 x i32> %vc
 }
 
@@ -566,11 +566,11 @@ define <vscale x 8 x i32> @vsub_vx_nxv8i32(<vscale x 8 x i32> %va, i32 signext %
 define <vscale x 8 x i32> @vsub_vx_nxv8i32_0(<vscale x 8 x i32> %va) {
 ; CHECK-LABEL: vsub_vx_nxv8i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 8 x i32> %va, splat (i32 1)
+  %vc = sub <vscale x 8 x i32> %va, splat (i32 17)
   ret <vscale x 8 x i32> %vc
 }
 
@@ -599,11 +599,11 @@ define <vscale x 16 x i32> @vsub_vx_nxv16i32(<vscale x 16 x i32> %va, i32 signex
 define <vscale x 16 x i32> @vsub_vx_nxv16i32_0(<vscale x 16 x i32> %va) {
 ; CHECK-LABEL: vsub_vx_nxv16i32_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 16 x i32> %va, splat (i32 1)
+  %vc = sub <vscale x 16 x i32> %va, splat (i32 17)
   ret <vscale x 16 x i32> %vc
 }
 
@@ -646,11 +646,11 @@ define <vscale x 1 x i64> @vsub_vx_nxv1i64(<vscale x 1 x i64> %va, i64 %b) {
 define <vscale x 1 x i64> @vsub_vx_nxv1i64_0(<vscale x 1 x i64> %va) {
 ; CHECK-LABEL: vsub_vx_nxv1i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 1 x i64> %va, splat (i64 1)
+  %vc = sub <vscale x 1 x i64> %va, splat (i64 17)
   ret <vscale x 1 x i64> %vc
 }
 
@@ -693,11 +693,11 @@ define <vscale x 2 x i64> @vsub_vx_nxv2i64(<vscale x 2 x i64> %va, i64 %b) {
 define <vscale x 2 x i64> @vsub_vx_nxv2i64_0(<vscale x 2 x i64> %va) {
 ; CHECK-LABEL: vsub_vx_nxv2i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 2 x i64> %va, splat (i64 1)
+  %vc = sub <vscale x 2 x i64> %va, splat (i64 17)
   ret <vscale x 2 x i64> %vc
 }
 
@@ -740,11 +740,11 @@ define <vscale x 4 x i64> @vsub_vx_nxv4i64(<vscale x 4 x i64> %va, i64 %b) {
 define <vscale x 4 x i64> @vsub_vx_nxv4i64_0(<vscale x 4 x i64> %va) {
 ; CHECK-LABEL: vsub_vx_nxv4i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 4 x i64> %va, splat (i64 1)
+  %vc = sub <vscale x 4 x i64> %va, splat (i64 17)
   ret <vscale x 4 x i64> %vc
 }
 
@@ -787,11 +787,11 @@ define <vscale x 8 x i64> @vsub_vx_nxv8i64(<vscale x 8 x i64> %va, i64 %b) {
 define <vscale x 8 x i64> @vsub_vx_nxv8i64_0(<vscale x 8 x i64> %va) {
 ; CHECK-LABEL: vsub_vx_nxv8i64_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    li a0, 17
 ; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vsub.vx v8, v8, a0
 ; CHECK-NEXT:    ret
-  %vc = sub <vscale x 8 x i64> %va, splat (i64 1)
+  %vc = sub <vscale x 8 x i64> %va, splat (i64 17)
   ret <vscale x 8 x i64> %vc
 }
 
@@ -850,12 +850,72 @@ define <vscale x 8 x i32> @vsub_vx_mask_nxv8i32(<vscale x 8 x i32> %va, i32 sign
   ret <vscale x 8 x i32> %vc
 }
 
+define <vscale x 8 x i32> @vsub_vi_nxv8i32_one(<vscale x 8 x i32> %va) {
+; CHECK-LABEL: vsub_vi_nxv8i32_one:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vadd.vi v8, v8, -1
+; CHECK-NEXT:    ret
+  %vc = sub <vscale x 8 x i32> %va, splat (i32 1)
+  ret <vscale x 8 x i32> %vc
+}
+
+define <vscale x 8 x i32> @vsub_vi_nxv8i32_minusone(<vscale x 8 x i32> %va) {
+; CHECK-LABEL: vsub_vi_nxv8i32_minusone:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vadd.vi v8, v8, 1
+; CHECK-NEXT:    ret
+  %vc = sub <vscale x 8 x i32> %va, splat (i32 -1)
+  ret <vscale x 8 x i32> %vc
+}
+
+define <vscale x 8 x i32> @vsub_vi_nxv8i32_15(<vscale x 8 x i32> %va) {
+; CHECK-LABEL: vsub_vi_nxv8i32_15:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vadd.vi v8, v8, -15
+; CHECK-NEXT:    ret
+  %vc = sub <vscale x 8 x i32> %va, splat (i32 15)
+  ret <vscale x 8 x i32> %vc
+}
+
+define <vscale x 8 x i32> @vsub_vi_nxv8i32_16(<vscale x 8 x i32> %va) {
+; CHECK-LABEL: vsub_vi_nxv8i32_16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vadd.vi v8, v8, -16
+; CHECK-NEXT:    ret
+  %vc = sub <vscale x 8 x i32> %va, splat (i32 16)
+  ret <vscale x 8 x i32> %vc
+}
+
+define <vscale x 8 x i32> @vsub_vi_nxv8i32_minus15(<vscale x 8 x i32> %va) {
+; CHECK-LABEL: vsub_vi_nxv8i32_minus15:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vadd.vi v8, v8, 15
+; CHECK-NEXT:    ret
+  %vc = sub <vscale x 8 x i32> %va, splat (i32 -15)
+  ret <vscale x 8 x i32> %vc
+}
+
+define <vscale x 8 x i32> @vsub_vi_nxv8i32_minus16(<vscale x 8 x i32> %va) {
+; CHECK-LABEL: vsub_vi_nxv8i32_minus16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, -16
+; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vsub.vx v8, v8, a0
+; CHECK-NEXT:    ret
+  %vc = sub <vscale x 8 x i32> %va, splat (i32 -16)
+  ret <vscale x 8 x i32> %vc
+}
+
 define <vscale x 8 x i32> @vsub_vi_mask_nxv8i32(<vscale x 8 x i32> %va, <vscale x 8 x i1> %mask) {
 ; CHECK-LABEL: vsub_vi_mask_nxv8i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 7
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, mu
-; CHECK-NEXT:    vsub.vx v8, v8, a0, v0.t
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vadd.vi v8, v8, -7, v0.t
 ; CHECK-NEXT:    ret
   %vs = select <vscale x 8 x i1> %mask, <vscale x 8 x i32> splat (i32 7), <vscale x 8 x i32> zeroinitializer
   %vc = sub <vscale x 8 x i32> %va, %vs
diff --git a/llvm/test/CodeGen/RISCV/sextw-removal.ll b/llvm/test/CodeGen/RISCV/sextw-removal.ll
index e0a16aa05cd00..49494608eee4d 100644
--- a/llvm/test/CodeGen/RISCV/sextw-removal.ll
+++ b/llvm/test/CodeGen/RISCV/sextw-removal.ll
@@ -1352,7 +1352,6 @@ define signext i32 @sextw_sh2add(i1 zeroext %0, ptr %1, i32 signext %2, i32 sign
 ; NOREMOVAL-LABEL: sextw_sh2add:
 ; NOREMOVAL:       # %bb.0:
 ; NOREMOVAL-NEXT:    sh2add a2, a2, a3
-; NOREMOVAL-NEXT:    mv a2, a2
 ; NOREMOVAL-NEXT:    beqz a0, .LBB22_2
 ; NOREMOVAL-NEXT:  # %bb.1:
 ; NOREMOVAL-NEXT:    sw a2, 0(a1)
diff --git a/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll b/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
new file mode 100644
index 0000000000000..b2be401b4676f
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
@@ -0,0 +1,95 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v | FileCheck %s
+
+declare i32 @llvm.experimental.constrained.fptosi.i32.f64(double, metadata)
+declare void @g()
+
+define void @f(ptr %m, ptr %n, ptr %p, ptr %q, ptr %r, ptr %s, double %t) {
+; CHECK-LABEL: f:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -48
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset ra, -8
+; CHECK-NEXT:    .cfi_offset s0, -16
+; CHECK-NEXT:    .cfi_offset s1, -24
+; CHECK-NEXT:    .cfi_offset s2, -32
+; CHECK-NEXT:    csrr a6, vlenb
+; CHECK-NEXT:    sub sp, sp, a6
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x01, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 1 * vlenb
+; CHECK-NEXT:    mv s0, a5
+; CHECK-NEXT:    mv s1, a4
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vle64.v v8, (a0)
+; CHECK-NEXT:    vse64.v v8, (a1)
+; CHECK-NEXT:    vle64.v v8, (a2)
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vs1r.v v8, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    mv s2, a3
+; CHECK-NEXT:    call g
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vse64.v v8, (s2)
+; CHECK-NEXT:    vle64.v v8, (s1)
+; CHECK-NEXT:    vse64.v v8, (s0)
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    .cfi_def_cfa sp, 48
+; CHECK-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    .cfi_restore ra
+; CHECK-NEXT:    .cfi_restore s0
+; CHECK-NEXT:    .cfi_restore s1
+; CHECK-NEXT:    .cfi_restore s2
+; CHECK-NEXT:    addi sp, sp, 48
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    ret
+  %z0 = load i64, ptr %m
+  %m.1 = getelementptr i64, ptr %m, i64 1
+  %z1 = load i64, ptr %m.1
+  store i64 %z0, ptr %n
+  %n.1 = getelementptr i64, ptr %n, i64 1
+  store i64 %z1, ptr %n.1
+
+  %x0 = load i64, ptr %p
+  %p.1 = getelementptr i64, ptr %p, i64 1
+  %x1 = load i64, ptr %p.1
+  call void @g()
+  store i64 %x0, ptr %q
+  %q.1 = getelementptr i64, ptr %q, i64 1
+  store i64 %x1, ptr %q.1
+
+  %y0 = load i64, ptr %r
+  %r.1 = getelementptr i64, ptr %r, i64 1
+  %y1 = load i64, ptr %r.1
+  store i64 %y0, ptr %s
+  %s.1 = getelementptr i64, ptr %s, i64 1
+  store i64 %y1, ptr %s.1
+
+  ret void
+}
+
+define void @f1(ptr %m, ptr %n, ptr %p, ptr %q, ptr %r, ptr %s, double %t) {
+; CHECK-LABEL: f1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vle64.v v8, (a2)
+; CHECK-NEXT:    fcvt.wu.d a0, fa0, rtz
+; CHECK-NEXT:    vse64.v v8, (a3)
+; CHECK-NEXT:    ret
+  %x0 = load i64, ptr %p
+  %p.1 = getelementptr i64, ptr %p, i64 1
+  %x1 = load i64, ptr %p.1
+  %t1 = call i32 @llvm.experimental.constrained.fptoui.i32.f64(double %t, metadata !"fpexcept.strict")
+  store i64 %x0, ptr %q
+  %q.1 = getelementptr i64, ptr %q, i64 1
+  store i64 %x1, ptr %q.1
+
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
index feda774ab0e65..82c8d50e518b0 100644
--- a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
@@ -223,18 +223,31 @@ entry:
 }
 
 define arm_aapcs_vfpcc <8 x i16> @shuffle3_i16(<8 x i16> %src) {
-; CHECK-LABEL: shuffle3_i16:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovx.f16 s5, s3
-; CHECK-NEXT:    vmovx.f16 s6, s1
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vins.f16 s1, s0
-; CHECK-NEXT:    vins.f16 s6, s4
-; CHECK-NEXT:    vins.f16 s5, s3
-; CHECK-NEXT:    vmov.f32 s4, s2
-; CHECK-NEXT:    vmov.f32 s7, s1
-; CHECK-NEXT:    vmov q0, q1
-; CHECK-NEXT:    bx lr
+; CHECK-LV-LABEL: shuffle3_i16:
+; CHECK-LV:       @ %bb.0: @ %entry
+; CHECK-LV-NEXT:    vmovx.f16 s5, s3
+; CHECK-LV-NEXT:    vmovx.f16 s6, s1
+; CHECK-LV-NEXT:    vmovx.f16 s4, s0
+; CHECK-LV-NEXT:    vins.f16 s1, s0
+; CHECK-LV-NEXT:    vins.f16 s6, s4
+; CHECK-LV-NEXT:    vins.f16 s5, s3
+; CHECK-LV-NEXT:    vmov.f32 s4, s2
+; CHECK-LV-NEXT:    vmov.f32 s7, s1
+; CHECK-LV-NEXT:    vmov q0, q1
+; CHECK-LV-NEXT:    bx lr
+
+; CHECK-LIS-LABEL: shuffle3_i16:
+; CHECK-LIS:       @ %bb.0: @ %entry
+; CHECK-LIS-NEXT:    vmov q1, q0
+; CHECK-LIS-NEXT:    vmovx.f16 s2, s5
+; CHECK-LIS-NEXT:    vmovx.f16 s0, s4
+; CHECK-LIS-NEXT:    vins.f16 s5, s4
+; CHECK-LIS-NEXT:    vins.f16 s2, s0
+; CHECK-LIS-NEXT:    vmov.f32 s0, s6
+; CHECK-LIS-NEXT:    vmovx.f16 s1, s7
+; CHECK-LIS-NEXT:    vmov.f32 s3, s5
+; CHECK-LIS-NEXT:    vins.f16 s1, s7
+; CHECK-LIS-NEXT:    bx lr
 entry:
   %out = shufflevector <8 x i16> %src, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 7, i32 6, i32 3, i32 1, i32 2, i32 0>
   ret <8 x i16> %out
@@ -1145,18 +1158,31 @@ entry:
 }
 
 define arm_aapcs_vfpcc <8 x half> @shuffle3_f16(<8 x half> %src) {
-; CHECK-LABEL: shuffle3_f16:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovx.f16 s5, s3
-; CHECK-NEXT:    vmovx.f16 s6, s1
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vins.f16 s1, s0
-; CHECK-NEXT:    vins.f16 s6, s4
-; CHECK-NEXT:    vins.f16 s5, s3
-; CHECK-NEXT:    vmov.f32 s4, s2
-; CHECK-NEXT:    vmov.f32 s7, s1
-; CHECK-NEXT:    vmov q0, q1
-; CHECK-NEXT:    bx lr
+; CHECK-LV-LABEL: shuffle3_f16:
+; CHECK-LV:       @ %bb.0: @ %entry
+; CHECK-LV-NEXT:    vmovx.f16 s5, s3
+; CHECK-LV-NEXT:    vmovx.f16 s6, s1
+; CHECK-LV-NEXT:    vmovx.f16 s4, s0
+; CHECK-LV-NEXT:    vins.f16 s1, s0
+; CHECK-LV-NEXT:    vins.f16 s6, s4
+; CHECK-LV-NEXT:    vins.f16 s5, s3
+; CHECK-LV-NEXT:    vmov.f32 s4, s2
+; CHECK-LV-NEXT:    vmov.f32 s7, s1
+; CHECK-LV-NEXT:    vmov q0, q1
+; CHECK-LV-NEXT:    bx lr
+
+; CHECK-LIS-LABEL: shuffle3_f16:
+; CHECK-LIS:       @ %bb.0: @ %entry
+; CHECK-LIS-NEXT:    vmov q1, q0
+; CHECK-LIS-NEXT:    vmovx.f16 s2, s5
+; CHECK-LIS-NEXT:    vmovx.f16 s0, s4
+; CHECK-LIS-NEXT:    vins.f16 s5, s4
+; CHECK-LIS-NEXT:    vins.f16 s2, s0
+; CHECK-LIS-NEXT:    vmov.f32 s0, s6
+; CHECK-LIS-NEXT:    vmovx.f16 s1, s7
+; CHECK-LIS-NEXT:    vmov.f32 s3, s5
+; CHECK-LIS-NEXT:    vins.f16 s1, s7
+; CHECK-LIS-NEXT:    bx lr
 entry:
   %out = shufflevector <8 x half> %src, <8 x half> undef, <8 x i32> <i32 4, i32 5, i32 7, i32 6, i32 3, i32 1, i32 2, i32 0>
   ret <8 x half> %out
@@ -1467,27 +1493,47 @@ entry:
   ret <2 x double> %out
 }
 define arm_aapcs_vfpcc <8 x double> @shuffle9_f64(<4 x double> %src1, <4 x double> %src2) {
-; CHECK-LABEL: shuffle9_f64:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    vmov q5, q2
-; CHECK-NEXT:    vmov.f32 s16, s0
-; CHECK-NEXT:    vmov.f32 s18, s20
-; CHECK-NEXT:    vmov.f32 s20, s2
-; CHECK-NEXT:    vmov.f32 s10, s12
-; CHECK-NEXT:    vmov.f32 s19, s21
-; CHECK-NEXT:    vmov.f32 s8, s4
-; CHECK-NEXT:    vmov.f32 s17, s1
-; CHECK-NEXT:    vmov.f32 s21, s3
-; CHECK-NEXT:    vmov q0, q4
-; CHECK-NEXT:    vmov.f32 s12, s6
-; CHECK-NEXT:    vmov.f32 s11, s13
-; CHECK-NEXT:    vmov.f32 s9, s5
-; CHECK-NEXT:    vmov.f32 s13, s7
-; CHECK-NEXT:    vmov q1, q5
-; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    bx lr
+; CHECK-LV-LABEL: shuffle9_f64:
+; CHECK-LV:       @ %bb.0: @ %entry
+; CHECK-LV-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-LV-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-LV-NEXT:    vmov q5, q2
+; CHECK-LV-NEXT:    vmov.f32 s16, s0
+; CHECK-LV-NEXT:    vmov.f32 s18, s20
+; CHECK-LV-NEXT:    vmov.f32 s20, s2
+; CHECK-LV-NEXT:    vmov.f32 s10, s12
+; CHECK-LV-NEXT:    vmov.f32 s19, s21
+; CHECK-LV-NEXT:    vmov.f32 s8, s4
+; CHECK-LV-NEXT:    vmov.f32 s17, s1
+; CHECK-LV-NEXT:    vmov.f32 s21, s3
+; CHECK-LV-NEXT:    vmov q0, q4
+; CHECK-LV-NEXT:    vmov.f32 s12, s6
+; CHECK-LV-NEXT:    vmov.f32 s11, s13
+; CHECK-LV-NEXT:    vmov.f32 s9, s5
+; CHECK-LV-NEXT:    vmov.f32 s13, s7
+; CHECK-LV-NEXT:    vmov q1, q5
+; CHECK-LV-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-LV-NEXT:    bx lr
+
+; CHECK-LIS-LABEL: shuffle9_f64:
+; CHECK-LIS:       @ %bb.0: @ %entry
+; CHECK-LIS-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-LIS-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-LIS-NEXT:    vmov q5, q2
+; CHECK-LIS-NEXT:    vmov q4, q0
+; CHECK-LIS-NEXT:    vmov.f32 s2, s20
+; CHECK-LIS-NEXT:    vmov.f32 s20, s18
+; CHECK-LIS-NEXT:    vmov.f32 s10, s12
+; CHECK-LIS-NEXT:    vmov.f32 s3, s21
+; CHECK-LIS-NEXT:    vmov.f32 s8, s4
+; CHECK-LIS-NEXT:    vmov.f32 s21, s19
+; CHECK-LIS-NEXT:    vmov.f32 s12, s6
+; CHECK-LIS-NEXT:    vmov.f32 s11, s13
+; CHECK-LIS-NEXT:    vmov.f32 s9, s5
+; CHECK-LIS-NEXT:    vmov.f32 s13, s7
+; CHECK-LIS-NEXT:    vmov q1, q5
+; CHECK-LIS-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-LIS-NEXT:    bx lr
 entry:
   %out = shufflevector <4 x double> %src1, <4 x double> %src2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
   ret <8 x double> %out
@@ -1560,27 +1606,47 @@ entry:
   ret <2 x i64> %out
 }
 define arm_aapcs_vfpcc <8 x i64> @shuffle9_i64(<4 x i64> %src1, <4 x i64> %src2) {
-; CHECK-LABEL: shuffle9_i64:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    vmov q5, q2
-; CHECK-NEXT:    vmov.f32 s16, s0
-; CHECK-NEXT:    vmov.f32 s18, s20
-; CHECK-NEXT:    vmov.f32 s20, s2
-; CHECK-NEXT:    vmov.f32 s10, s12
-; CHECK-NEXT:    vmov.f32 s19, s21
-; CHECK-NEXT:    vmov.f32 s8, s4
-; CHECK-NEXT:    vmov.f32 s17, s1
-; CHECK-NEXT:    vmov.f32 s21, s3
-; CHECK-NEXT:    vmov q0, q4
-; CHECK-NEXT:    vmov.f32 s12, s6
-; CHECK-NEXT:    vmov.f32 s11, s13
-; CHECK-NEXT:    vmov.f32 s9, s5
-; CHECK-NEXT:    vmov.f32 s13, s7
-; CHECK-NEXT:    vmov q1, q5
-; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    bx lr
+; CHECK-LV-LABEL: shuffle9_i64:
+; CHECK-LV:       @ %bb.0: @ %entry
+; CHECK-LV-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-LV-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-LV-NEXT:    vmov q5, q2
+; CHECK-LV-NEXT:    vmov.f32 s16, s0
+; CHECK-LV-NEXT:    vmov.f32 s18, s20
+; CHECK-LV-NEXT:    vmov.f32 s20, s2
+; CHECK-LV-NEXT:    vmov.f32 s10, s12
+; CHECK-LV-NEXT:    vmov.f32 s19, s21
+; CHECK-LV-NEXT:    vmov.f32 s8, s4
+; CHECK-LV-NEXT:    vmov.f32 s17, s1
+; CHECK-LV-NEXT:    vmov.f32 s21, s3
+; CHECK-LV-NEXT:    vmov q0, q4
+; CHECK-LV-NEXT:    vmov.f32 s12, s6
+; CHECK-LV-NEXT:    vmov.f32 s11, s13
+; CHECK-LV-NEXT:    vmov.f32 s9, s5
+; CHECK-LV-NEXT:    vmov.f32 s13, s7
+; CHECK-LV-NEXT:    vmov q1, q5
+; CHECK-LV-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-LV-NEXT:    bx lr
+
+; CHECK-LIS-LABEL: shuffle9_i64:
+; CHECK-LIS:       @ %bb.0: @ %entry
+; CHECK-LIS-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-LIS-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-LIS-NEXT:    vmov q5, q2
+; CHECK-LIS-NEXT:    vmov q4, q0
+; CHECK-LIS-NEXT:    vmov.f32 s2, s20
+; CHECK-LIS-NEXT:    vmov.f32 s20, s18
+; CHECK-LIS-NEXT:    vmov.f32 s10, s12
+; CHECK-LIS-NEXT:    vmov.f32 s3, s21
+; CHECK-LIS-NEXT:    vmov.f32 s8, s4
+; CHECK-LIS-NEXT:    vmov.f32 s21, s19
+; CHECK-LIS-NEXT:    vmov.f32 s12, s6
+; CHECK-LIS-NEXT:    vmov.f32 s11, s13
+; CHECK-LIS-NEXT:    vmov.f32 s9, s5
+; CHECK-LIS-NEXT:    vmov.f32 s13, s7
+; CHECK-LIS-NEXT:    vmov q1, q5
+; CHECK-LIS-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-LIS-NEXT:    bx lr
 entry:
   %out = shufflevector <4 x i64> %src1, <4 x i64> %src2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
   ret <8 x i64> %out
diff --git a/llvm/test/CodeGen/Thumb2/mve-vld3.ll b/llvm/test/CodeGen/Thumb2/mve-vld3.ll
index 0c58abaa1c86e..b6c8056891f83 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vld3.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vld3.ll
@@ -33,29 +33,53 @@ entry:
 }
 
 define void @vld3_v4i32(ptr %src, ptr %dst) {
-; CHECK-LABEL: vld3_v4i32:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
-; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    vldrw.u32 q4, [r0, #32]
-; CHECK-NEXT:    vmov.f32 s10, s2
-; CHECK-NEXT:    vmov.f32 s13, s0
-; CHECK-NEXT:    vmov.f32 s14, s3
-; CHECK-NEXT:    vmov.f32 s8, s4
-; CHECK-NEXT:    vmov.f32 s9, s7
-; CHECK-NEXT:    vmov.f32 s12, s5
-; CHECK-NEXT:    vmov.f32 s15, s18
-; CHECK-NEXT:    vmov.f32 s11, s17
-; CHECK-NEXT:    vadd.i32 q2, q2, q3
-; CHECK-NEXT:    vmov.f32 s0, s6
-; CHECK-NEXT:    vmov.f32 s2, s16
-; CHECK-NEXT:    vmov.f32 s3, s19
-; CHECK-NEXT:    vadd.i32 q0, q2, q0
-; CHECK-NEXT:    vstrw.32 q0, [r1]
-; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    bx lr
+; CHECK-LV-LABEL: vld3_v4i32:
+; CHECK-LV:       @ %bb.0: @ %entry
+; CHECK-LV-NEXT:    .vsave {d8, d9}
+; CHECK-LV-NEXT:    vpush {d8, d9}
+; CHECK-LV-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-LV-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-LV-NEXT:    vldrw.u32 q4, [r0, #32]
+; CHECK-LV-NEXT:    vmov.f32 s10, s2
+; CHECK-LV-NEXT:    vmov.f32 s13, s0
+; CHECK-LV-NEXT:    vmov.f32 s14, s3
+; CHECK-LV-NEXT:    vmov.f32 s8, s4
+; CHECK-LV-NEXT:    vmov.f32 s9, s7
+; CHECK-LV-NEXT:    vmov.f32 s12, s5
+; CHECK-LV-NEXT:    vmov.f32 s15, s18
+; CHECK-LV-NEXT:    vmov.f32 s11, s17
+; CHECK-LV-NEXT:    vadd.i32 q2, q2, q3
+; CHECK-LV-NEXT:    vmov.f32 s0, s6
+; CHECK-LV-NEXT:    vmov.f32 s2, s16
+; CHECK-LV-NEXT:    vmov.f32 s3, s19
+; CHECK-LV-NEXT:    vadd.i32 q0, q2, q0
+; CHECK-LV-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LV-NEXT:    vpop {d8, d9}
+; CHECK-LV-NEXT:    bx lr
+
+; CHECK-LIS-LABEL: vld3_v4i32:
+; CHECK-LIS:       @ %bb.0: @ %entry
+; CHECK-LIS-NEXT:    .vsave {d8, d9}
+; CHECK-LIS-NEXT:    vpush {d8, d9}
+; CHECK-LIS-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-LIS-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-LIS-NEXT:    vmov.f32 s10, s2
+; CHECK-LIS-NEXT:    vmov.f32 s17, s0
+; CHECK-LIS-NEXT:    vmov.f32 s18, s3
+; CHECK-LIS-NEXT:    vmov.f32 s8, s4
+; CHECK-LIS-NEXT:    vmov.f32 s9, s7
+; CHECK-LIS-NEXT:    vmov.f32 s16, s5
+; CHECK-LIS-NEXT:    vmov.f32 s19, s14
+; CHECK-LIS-NEXT:    vmov.f32 s11, s13
+; CHECK-LIS-NEXT:    vadd.i32 q2, q2, q4
+; CHECK-LIS-NEXT:    vmov.f32 s0, s6
+; CHECK-LIS-NEXT:    vmov.f32 s2, s12
+; CHECK-LIS-NEXT:    vmov.f32 s3, s15
+; CHECK-LIS-NEXT:    vadd.i32 q0, q2, q0
+; CHECK-LIS-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LIS-NEXT:    vpop {d8, d9}
+; CHECK-LIS-NEXT:    bx lr
 entry:
   %l1 = load <12 x i32>, ptr %src, align 4
   %s1 = shufflevector <12 x i32> %l1, <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
@@ -68,46 +92,87 @@ entry:
 }
 
 define void @vld3_v8i32(ptr %src, ptr %dst) {
-; CHECK-LABEL: vld3_v8i32:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
-; CHECK-NEXT:    vldrw.u32 q4, [r0, #80]
-; CHECK-NEXT:    vmov.f32 s10, s2
-; CHECK-NEXT:    vmov.f32 s13, s0
-; CHECK-NEXT:    vmov.f32 s14, s3
-; CHECK-NEXT:    vmov.f32 s8, s4
-; CHECK-NEXT:    vmov.f32 s9, s7
-; CHECK-NEXT:    vmov.f32 s12, s5
-; CHECK-NEXT:    vmov.f32 s15, s18
-; CHECK-NEXT:    vmov.f32 s11, s17
-; CHECK-NEXT:    vadd.i32 q2, q2, q3
-; CHECK-NEXT:    vmov.f32 s0, s6
-; CHECK-NEXT:    vmov.f32 s2, s16
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
-; CHECK-NEXT:    vmov.f32 s3, s19
-; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
-; CHECK-NEXT:    vadd.i32 q0, q2, q0
-; CHECK-NEXT:    vldrw.u32 q2, [r0]
-; CHECK-NEXT:    vmov.f32 s17, s4
-; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
-; CHECK-NEXT:    vmov.f32 s18, s7
-; CHECK-NEXT:    vmov.f32 s22, s6
-; CHECK-NEXT:    vmov.f32 s16, s9
-; CHECK-NEXT:    vmov.f32 s19, s14
-; CHECK-NEXT:    vmov.f32 s20, s8
-; CHECK-NEXT:    vmov.f32 s21, s11
-; CHECK-NEXT:    vmov.f32 s23, s13
-; CHECK-NEXT:    vadd.i32 q4, q5, q4
-; CHECK-NEXT:    vmov.f32 s4, s10
-; CHECK-NEXT:    vmov.f32 s6, s12
-; CHECK-NEXT:    vmov.f32 s7, s15
-; CHECK-NEXT:    vadd.i32 q1, q4, q1
-; CHECK-NEXT:    vstrw.32 q1, [r1]
-; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    bx lr
+; CHECK-LV-LABEL: vld3_v8i32:
+; CHECK-LV:       @ %bb.0: @ %entry
+; CHECK-LV-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-LV-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-LV-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-LV-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-LV-NEXT:    vldrw.u32 q4, [r0, #80]
+; CHECK-LV-NEXT:    vmov.f32 s10, s2
+; CHECK-LV-NEXT:    vmov.f32 s13, s0
+; CHECK-LV-NEXT:    vmov.f32 s14, s3
+; CHECK-LV-NEXT:    vmov.f32 s8, s4
+; CHECK-LV-NEXT:    vmov.f32 s9, s7
+; CHECK-LV-NEXT:    vmov.f32 s12, s5
+; CHECK-LV-NEXT:    vmov.f32 s15, s18
+; CHECK-LV-NEXT:    vmov.f32 s11, s17
+; CHECK-LV-NEXT:    vadd.i32 q2, q2, q3
+; CHECK-LV-NEXT:    vmov.f32 s0, s6
+; CHECK-LV-NEXT:    vmov.f32 s2, s16
+; CHECK-LV-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-LV-NEXT:    vmov.f32 s3, s19
+; CHECK-LV-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-LV-NEXT:    vadd.i32 q0, q2, q0
+; CHECK-LV-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-LV-NEXT:    vmov.f32 s17, s4
+; CHECK-LV-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-LV-NEXT:    vmov.f32 s18, s7
+; CHECK-LV-NEXT:    vmov.f32 s22, s6
+; CHECK-LV-NEXT:    vmov.f32 s16, s9
+; CHECK-LV-NEXT:    vmov.f32 s19, s14
+; CHECK-LV-NEXT:    vmov.f32 s20, s8
+; CHECK-LV-NEXT:    vmov.f32 s21, s11
+; CHECK-LV-NEXT:    vmov.f32 s23, s13
+; CHECK-LV-NEXT:    vadd.i32 q4, q5, q4
+; CHECK-LV-NEXT:    vmov.f32 s4, s10
+; CHECK-LV-NEXT:    vmov.f32 s6, s12
+; CHECK-LV-NEXT:    vmov.f32 s7, s15
+; CHECK-LV-NEXT:    vadd.i32 q1, q4, q1
+; CHECK-LV-NEXT:    vstrw.32 q1, [r1]
+; CHECK-LV-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-LV-NEXT:    bx lr
+
+; CHECK-LIS-LABEL: vld3_v8i32:
+; CHECK-LIS:       @ %bb.0: @ %entry
+; CHECK-LIS-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-LIS-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-LIS-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-LIS-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #80]
+; CHECK-LIS-NEXT:    vmov.f32 s10, s2
+; CHECK-LIS-NEXT:    vmov.f32 s17, s0
+; CHECK-LIS-NEXT:    vmov.f32 s18, s3
+; CHECK-LIS-NEXT:    vmov.f32 s8, s4
+; CHECK-LIS-NEXT:    vmov.f32 s9, s7
+; CHECK-LIS-NEXT:    vmov.f32 s16, s5
+; CHECK-LIS-NEXT:    vmov.f32 s19, s14
+; CHECK-LIS-NEXT:    vmov.f32 s11, s13
+; CHECK-LIS-NEXT:    vmov.f32 s0, s6
+; CHECK-LIS-NEXT:    vadd.i32 q2, q2, q4
+; CHECK-LIS-NEXT:    vmov.f32 s2, s12
+; CHECK-LIS-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-LIS-NEXT:    vmov.f32 s3, s15
+; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-LIS-NEXT:    vadd.i32 q0, q2, q0
+; CHECK-LIS-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-LIS-NEXT:    vmov.f32 s17, s4
+; CHECK-LIS-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-LIS-NEXT:    vmov.f32 s18, s7
+; CHECK-LIS-NEXT:    vmov.f32 s22, s6
+; CHECK-LIS-NEXT:    vmov.f32 s16, s9
+; CHECK-LIS-NEXT:    vmov.f32 s19, s14
+; CHECK-LIS-NEXT:    vmov.f32 s20, s8
+; CHECK-LIS-NEXT:    vmov.f32 s21, s11
+; CHECK-LIS-NEXT:    vmov.f32 s23, s13
+; CHECK-LIS-NEXT:    vadd.i32 q4, q5, q4
+; CHECK-LIS-NEXT:    vmov.f32 s4, s10
+; CHECK-LIS-NEXT:    vmov.f32 s6, s12
+; CHECK-LIS-NEXT:    vmov.f32 s7, s15
+; CHECK-LIS-NEXT:    vadd.i32 q1, q4, q1
+; CHECK-LIS-NEXT:    vstrw.32 q1, [r1]
+; CHECK-LIS-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-LIS-NEXT:    bx lr
 entry:
   %l1 = load <24 x i32>, ptr %src, align 4
   %s1 = shufflevector <24 x i32> %l1, <24 x i32> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
@@ -120,80 +185,155 @@ entry:
 }
 
 define void @vld3_v16i32(ptr %src, ptr %dst) {
-; CHECK-LABEL: vld3_v16i32:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
-; CHECK-NEXT:    vldrw.u32 q4, [r0, #80]
-; CHECK-NEXT:    vldrw.u32 q6, [r0, #176]
-; CHECK-NEXT:    vmov.f32 s10, s2
-; CHECK-NEXT:    vmov.f32 s13, s0
-; CHECK-NEXT:    vmov.f32 s14, s3
-; CHECK-NEXT:    vmov.f32 s8, s4
-; CHECK-NEXT:    vmov.f32 s9, s7
-; CHECK-NEXT:    vmov.f32 s12, s5
-; CHECK-NEXT:    vmov.f32 s15, s18
-; CHECK-NEXT:    vmov.f32 s11, s17
-; CHECK-NEXT:    vadd.i32 q2, q2, q3
-; CHECK-NEXT:    vmov.f32 s0, s6
-; CHECK-NEXT:    vmov.f32 s2, s16
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
-; CHECK-NEXT:    vmov.f32 s3, s19
-; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
-; CHECK-NEXT:    vadd.i32 q0, q2, q0
-; CHECK-NEXT:    vldrw.u32 q2, [r0]
-; CHECK-NEXT:    vmov.f32 s17, s4
-; CHECK-NEXT:    vmov.f32 s18, s7
-; CHECK-NEXT:    vmov.f32 s22, s6
-; CHECK-NEXT:    vmov.f32 s16, s9
-; CHECK-NEXT:    vmov.f32 s19, s14
-; CHECK-NEXT:    vmov.f32 s20, s8
-; CHECK-NEXT:    vmov.f32 s21, s11
-; CHECK-NEXT:    vmov.f32 s23, s13
-; CHECK-NEXT:    vmov.f32 s4, s10
-; CHECK-NEXT:    vldrw.u32 q2, [r0, #160]
-; CHECK-NEXT:    vmov.f32 s6, s12
-; CHECK-NEXT:    vadd.i32 q4, q5, q4
-; CHECK-NEXT:    vmov.f32 s7, s15
-; CHECK-NEXT:    vldrw.u32 q3, [r0, #144]
-; CHECK-NEXT:    vadd.i32 q1, q4, q1
-; CHECK-NEXT:    vmov.f32 s18, s10
-; CHECK-NEXT:    vmov.f32 s21, s8
-; CHECK-NEXT:    vmov.f32 s22, s11
-; CHECK-NEXT:    vmov.f32 s16, s12
-; CHECK-NEXT:    vmov.f32 s17, s15
-; CHECK-NEXT:    vmov.f32 s20, s13
-; CHECK-NEXT:    vmov.f32 s23, s26
-; CHECK-NEXT:    vmov.f32 s19, s25
-; CHECK-NEXT:    vadd.i32 q4, q4, q5
-; CHECK-NEXT:    vmov.f32 s8, s14
-; CHECK-NEXT:    vmov.f32 s10, s24
-; CHECK-NEXT:    vldrw.u32 q3, [r0, #112]
-; CHECK-NEXT:    vmov.f32 s11, s27
-; CHECK-NEXT:    vldrw.u32 q5, [r0, #128]
-; CHECK-NEXT:    vadd.i32 q2, q4, q2
-; CHECK-NEXT:    vldrw.u32 q4, [r0, #96]
-; CHECK-NEXT:    vmov.f32 s25, s12
-; CHECK-NEXT:    vstrw.32 q2, [r1, #48]
-; CHECK-NEXT:    vmov.f32 s26, s15
-; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
-; CHECK-NEXT:    vmov.f32 s30, s14
-; CHECK-NEXT:    vstrw.32 q1, [r1]
-; CHECK-NEXT:    vmov.f32 s24, s17
-; CHECK-NEXT:    vmov.f32 s27, s22
-; CHECK-NEXT:    vmov.f32 s28, s16
-; CHECK-NEXT:    vmov.f32 s29, s19
-; CHECK-NEXT:    vmov.f32 s31, s21
-; CHECK-NEXT:    vadd.i32 q6, q7, q6
-; CHECK-NEXT:    vmov.f32 s12, s18
-; CHECK-NEXT:    vmov.f32 s14, s20
-; CHECK-NEXT:    vmov.f32 s15, s23
-; CHECK-NEXT:    vadd.i32 q3, q6, q3
-; CHECK-NEXT:    vstrw.32 q3, [r1, #32]
-; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    bx lr
+; CHECK-LV-LABEL: vld3_v16i32:
+; CHECK-LV:       @ %bb.0: @ %entry
+; CHECK-LV-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-LV-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-LV-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-LV-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-LV-NEXT:    vldrw.u32 q4, [r0, #80]
+; CHECK-LV-NEXT:    vldrw.u32 q6, [r0, #176]
+; CHECK-LV-NEXT:    vmov.f32 s10, s2
+; CHECK-LV-NEXT:    vmov.f32 s13, s0
+; CHECK-LV-NEXT:    vmov.f32 s14, s3
+; CHECK-LV-NEXT:    vmov.f32 s8, s4
+; CHECK-LV-NEXT:    vmov.f32 s9, s7
+; CHECK-LV-NEXT:    vmov.f32 s12, s5
+; CHECK-LV-NEXT:    vmov.f32 s15, s18
+; CHECK-LV-NEXT:    vmov.f32 s11, s17
+; CHECK-LV-NEXT:    vadd.i32 q2, q2, q3
+; CHECK-LV-NEXT:    vmov.f32 s0, s6
+; CHECK-LV-NEXT:    vmov.f32 s2, s16
+; CHECK-LV-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-LV-NEXT:    vmov.f32 s3, s19
+; CHECK-LV-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-LV-NEXT:    vadd.i32 q0, q2, q0
+; CHECK-LV-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-LV-NEXT:    vmov.f32 s17, s4
+; CHECK-LV-NEXT:    vmov.f32 s18, s7
+; CHECK-LV-NEXT:    vmov.f32 s22, s6
+; CHECK-LV-NEXT:    vmov.f32 s16, s9
+; CHECK-LV-NEXT:    vmov.f32 s19, s14
+; CHECK-LV-NEXT:    vmov.f32 s20, s8
+; CHECK-LV-NEXT:    vmov.f32 s21, s11
+; CHECK-LV-NEXT:    vmov.f32 s23, s13
+; CHECK-LV-NEXT:    vmov.f32 s4, s10
+; CHECK-LV-NEXT:    vldrw.u32 q2, [r0, #160]
+; CHECK-LV-NEXT:    vmov.f32 s6, s12
+; CHECK-LV-NEXT:    vadd.i32 q4, q5, q4
+; CHECK-LV-NEXT:    vmov.f32 s7, s15
+; CHECK-LV-NEXT:    vldrw.u32 q3, [r0, #144]
+; CHECK-LV-NEXT:    vadd.i32 q1, q4, q1
+; CHECK-LV-NEXT:    vmov.f32 s18, s10
+; CHECK-LV-NEXT:    vmov.f32 s21, s8
+; CHECK-LV-NEXT:    vmov.f32 s22, s11
+; CHECK-LV-NEXT:    vmov.f32 s16, s12
+; CHECK-LV-NEXT:    vmov.f32 s17, s15
+; CHECK-LV-NEXT:    vmov.f32 s20, s13
+; CHECK-LV-NEXT:    vmov.f32 s23, s26
+; CHECK-LV-NEXT:    vmov.f32 s19, s25
+; CHECK-LV-NEXT:    vadd.i32 q4, q4, q5
+; CHECK-LV-NEXT:    vmov.f32 s8, s14
+; CHECK-LV-NEXT:    vmov.f32 s10, s24
+; CHECK-LV-NEXT:    vldrw.u32 q3, [r0, #112]
+; CHECK-LV-NEXT:    vmov.f32 s11, s27
+; CHECK-LV-NEXT:    vldrw.u32 q5, [r0, #128]
+; CHECK-LV-NEXT:    vadd.i32 q2, q4, q2
+; CHECK-LV-NEXT:    vldrw.u32 q4, [r0, #96]
+; CHECK-LV-NEXT:    vmov.f32 s25, s12
+; CHECK-LV-NEXT:    vstrw.32 q2, [r1, #48]
+; CHECK-LV-NEXT:    vmov.f32 s26, s15
+; CHECK-LV-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-LV-NEXT:    vmov.f32 s30, s14
+; CHECK-LV-NEXT:    vstrw.32 q1, [r1]
+; CHECK-LV-NEXT:    vmov.f32 s24, s17
+; CHECK-LV-NEXT:    vmov.f32 s27, s22
+; CHECK-LV-NEXT:    vmov.f32 s28, s16
+; CHECK-LV-NEXT:    vmov.f32 s29, s19
+; CHECK-LV-NEXT:    vmov.f32 s31, s21
+; CHECK-LV-NEXT:    vadd.i32 q6, q7, q6
+; CHECK-LV-NEXT:    vmov.f32 s12, s18
+; CHECK-LV-NEXT:    vmov.f32 s14, s20
+; CHECK-LV-NEXT:    vmov.f32 s15, s23
+; CHECK-LV-NEXT:    vadd.i32 q3, q6, q3
+; CHECK-LV-NEXT:    vstrw.32 q3, [r1, #32]
+; CHECK-LV-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-LV-NEXT:    bx lr
+
+; CHECK-LIS-LABEL: vld3_v16i32:
+; CHECK-LIS:       @ %bb.0: @ %entry
+; CHECK-LIS-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-LIS-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-LIS-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-LIS-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #80]
+; CHECK-LIS-NEXT:    vmov.f32 s10, s2
+; CHECK-LIS-NEXT:    vmov.f32 s17, s0
+; CHECK-LIS-NEXT:    vmov.f32 s18, s3
+; CHECK-LIS-NEXT:    vmov.f32 s8, s4
+; CHECK-LIS-NEXT:    vmov.f32 s9, s7
+; CHECK-LIS-NEXT:    vmov.f32 s16, s5
+; CHECK-LIS-NEXT:    vmov.f32 s19, s14
+; CHECK-LIS-NEXT:    vmov.f32 s11, s13
+; CHECK-LIS-NEXT:    vmov.f32 s0, s6
+; CHECK-LIS-NEXT:    vadd.i32 q2, q2, q4
+; CHECK-LIS-NEXT:    vmov.f32 s2, s12
+; CHECK-LIS-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-LIS-NEXT:    vmov.f32 s3, s15
+; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-LIS-NEXT:    vadd.i32 q0, q2, q0
+; CHECK-LIS-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-LIS-NEXT:    vmov.f32 s17, s4
+; CHECK-LIS-NEXT:    vmov.f32 s18, s7
+; CHECK-LIS-NEXT:    vmov.f32 s22, s6
+; CHECK-LIS-NEXT:    vmov.f32 s16, s9
+; CHECK-LIS-NEXT:    vmov.f32 s19, s14
+; CHECK-LIS-NEXT:    vmov.f32 s20, s8
+; CHECK-LIS-NEXT:    vmov.f32 s21, s11
+; CHECK-LIS-NEXT:    vmov.f32 s23, s13
+; CHECK-LIS-NEXT:    vadd.i32 q4, q5, q4
+; CHECK-LIS-NEXT:    vmov.f32 s4, s10
+; CHECK-LIS-NEXT:    vldrw.u32 q2, [r0, #160]
+; CHECK-LIS-NEXT:    vldrw.u32 q5, [r0, #176]
+; CHECK-LIS-NEXT:    vmov.f32 s6, s12
+; CHECK-LIS-NEXT:    vmov.f32 s7, s15
+; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #144]
+; CHECK-LIS-NEXT:    vadd.i32 q1, q4, q1
+; CHECK-LIS-NEXT:    vmov.f32 s18, s10
+; CHECK-LIS-NEXT:    vmov.f32 s25, s8
+; CHECK-LIS-NEXT:    vmov.f32 s26, s11
+; CHECK-LIS-NEXT:    vmov.f32 s16, s12
+; CHECK-LIS-NEXT:    vmov.f32 s17, s15
+; CHECK-LIS-NEXT:    vmov.f32 s24, s13
+; CHECK-LIS-NEXT:    vmov.f32 s27, s22
+; CHECK-LIS-NEXT:    vmov.f32 s19, s21
+; CHECK-LIS-NEXT:    vmov.f32 s8, s14
+; CHECK-LIS-NEXT:    vadd.i32 q4, q4, q6
+; CHECK-LIS-NEXT:    vmov.f32 s10, s20
+; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #112]
+; CHECK-LIS-NEXT:    vmov.f32 s11, s23
+; CHECK-LIS-NEXT:    vldrw.u32 q5, [r0, #128]
+; CHECK-LIS-NEXT:    vadd.i32 q2, q4, q2
+; CHECK-LIS-NEXT:    vldrw.u32 q4, [r0, #96]
+; CHECK-LIS-NEXT:    vmov.f32 s25, s12
+; CHECK-LIS-NEXT:    vstrw.32 q2, [r1, #48]
+; CHECK-LIS-NEXT:    vmov.f32 s26, s15
+; CHECK-LIS-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-LIS-NEXT:    vmov.f32 s30, s14
+; CHECK-LIS-NEXT:    vstrw.32 q1, [r1]
+; CHECK-LIS-NEXT:    vmov.f32 s24, s17
+; CHECK-LIS-NEXT:    vmov.f32 s27, s22
+; CHECK-LIS-NEXT:    vmov.f32 s28, s16
+; CHECK-LIS-NEXT:    vmov.f32 s29, s19
+; CHECK-LIS-NEXT:    vmov.f32 s31, s21
+; CHECK-LIS-NEXT:    vadd.i32 q6, q7, q6
+; CHECK-LIS-NEXT:    vmov.f32 s12, s18
+; CHECK-LIS-NEXT:    vmov.f32 s14, s20
+; CHECK-LIS-NEXT:    vmov.f32 s15, s23
+; CHECK-LIS-NEXT:    vadd.i32 q3, q6, q3
+; CHECK-LIS-NEXT:    vstrw.32 q3, [r1, #32]
+; CHECK-LIS-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-LIS-NEXT:    bx lr
 entry:
   %l1 = load <48 x i32>, ptr %src, align 4
   %s1 = shufflevector <48 x i32> %l1, <48 x i32> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
@@ -792,35 +932,65 @@ entry:
 ; i64
 
 define void @vld3_v2i64(ptr %src, ptr %dst) {
-; CHECK-LABEL: vld3_v2i64:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #32]
-; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
-; CHECK-NEXT:    vmov.f32 s12, s2
-; CHECK-NEXT:    vmov.f32 s13, s3
-; CHECK-NEXT:    vmov.f32 s2, s4
-; CHECK-NEXT:    vmov.f32 s3, s5
-; CHECK-NEXT:    vmov r0, r3, d5
-; CHECK-NEXT:    vmov r2, r4, d3
-; CHECK-NEXT:    vmov r6, r7, d0
-; CHECK-NEXT:    vmov r5, r8, d6
-; CHECK-NEXT:    vmov lr, r12, d1
-; CHECK-NEXT:    adds.w r0, r0, lr
-; CHECK-NEXT:    adc.w r3, r3, r12
-; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    adc.w r2, r3, r4
-; CHECK-NEXT:    vmov r3, r4, d4
-; CHECK-NEXT:    adds r6, r6, r5
-; CHECK-NEXT:    adc.w r7, r7, r8
-; CHECK-NEXT:    adds r3, r3, r6
-; CHECK-NEXT:    adcs r7, r4
-; CHECK-NEXT:    vmov q0[2], q0[0], r3, r0
-; CHECK-NEXT:    vmov q0[3], q0[1], r7, r2
-; CHECK-NEXT:    vstrw.32 q0, [r1]
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-LV-LABEL: vld3_v2i64:
+; CHECK-LV:       @ %bb.0: @ %entry
+; CHECK-LV-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; CHECK-LV-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-LV-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-LV-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-LV-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-LV-NEXT:    vmov.f32 s12, s2
+; CHECK-LV-NEXT:    vmov.f32 s13, s3
+; CHECK-LV-NEXT:    vmov.f32 s2, s4
+; CHECK-LV-NEXT:    vmov.f32 s3, s5
+; CHECK-LV-NEXT:    vmov r0, r3, d5
+; CHECK-LV-NEXT:    vmov r2, r4, d3
+; CHECK-LV-NEXT:    vmov r6, r7, d0
+; CHECK-LV-NEXT:    vmov r5, r8, d6
+; CHECK-LV-NEXT:    vmov lr, r12, d1
+; CHECK-LV-NEXT:    adds.w r0, r0, lr
+; CHECK-LV-NEXT:    adc.w r3, r3, r12
+; CHECK-LV-NEXT:    adds r0, r0, r2
+; CHECK-LV-NEXT:    adc.w r2, r3, r4
+; CHECK-LV-NEXT:    vmov r3, r4, d4
+; CHECK-LV-NEXT:    adds r6, r6, r5
+; CHECK-LV-NEXT:    adc.w r7, r7, r8
+; CHECK-LV-NEXT:    adds r3, r3, r6
+; CHECK-LV-NEXT:    adcs r7, r4
+; CHECK-LV-NEXT:    vmov q0[2], q0[0], r3, r0
+; CHECK-LV-NEXT:    vmov q0[3], q0[1], r7, r2
+; CHECK-LV-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LV-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+
+; CHECK-LIS-LABEL: vld3_v2i64:
+; CHECK-LIS:       @ %bb.0: @ %entry
+; CHECK-LIS-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; CHECK-LIS-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-LIS-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-LIS-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #16]
+; CHECK-LIS-NEXT:    vmov.f32 s8, s2
+; CHECK-LIS-NEXT:    vmov.f32 s9, s3
+; CHECK-LIS-NEXT:    vmov.f32 s2, s4
+; CHECK-LIS-NEXT:    vmov.f32 s3, s5
+; CHECK-LIS-NEXT:    vmov r0, r3, d7
+; CHECK-LIS-NEXT:    vmov r2, r4, d3
+; CHECK-LIS-NEXT:    vmov r6, r7, d0
+; CHECK-LIS-NEXT:    vmov r5, r8, d4
+; CHECK-LIS-NEXT:    vmov lr, r12, d1
+; CHECK-LIS-NEXT:    adds.w r0, r0, lr
+; CHECK-LIS-NEXT:    adc.w r3, r3, r12
+; CHECK-LIS-NEXT:    adds r0, r0, r2
+; CHECK-LIS-NEXT:    adc.w r2, r3, r4
+; CHECK-LIS-NEXT:    vmov r3, r4, d6
+; CHECK-LIS-NEXT:    adds r6, r6, r5
+; CHECK-LIS-NEXT:    adc.w r7, r7, r8
+; CHECK-LIS-NEXT:    adds r3, r3, r6
+; CHECK-LIS-NEXT:    adcs r7, r4
+; CHECK-LIS-NEXT:    vmov q0[2], q0[0], r3, r0
+; CHECK-LIS-NEXT:    vmov q0[3], q0[1], r7, r2
+; CHECK-LIS-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LIS-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
 entry:
   %l1 = load <6 x i64>, ptr %src, align 4
   %s1 = shufflevector <6 x i64> %l1, <6 x i64> undef, <2 x i32> <i32 0, i32 3>
@@ -917,28 +1087,28 @@ define void @vld3_v4i64(ptr %src, ptr %dst) {
 ; CHECK-LIS-NEXT:    vmov lr, r12, d1
 ; CHECK-LIS-NEXT:    vmov.f32 s2, s12
 ; CHECK-LIS-NEXT:    vmov.f32 s3, s13
-; CHECK-LIS-NEXT:    vmov r6, r7, d12
+; CHECK-LIS-NEXT:    vmov r7, r6, d12
 ; CHECK-LIS-NEXT:    adds.w r0, r5, lr
 ; CHECK-LIS-NEXT:    adc.w r5, r4, r12
 ; CHECK-LIS-NEXT:    adds.w lr, r0, r3
 ; CHECK-LIS-NEXT:    vmov r4, r2, d8
 ; CHECK-LIS-NEXT:    adc.w r12, r5, r8
 ; CHECK-LIS-NEXT:    vmov r5, r0, d10
-; CHECK-LIS-NEXT:    adds r6, r6, r4
-; CHECK-LIS-NEXT:    adcs r2, r7
-; CHECK-LIS-NEXT:    adds r6, r6, r5
+; CHECK-LIS-NEXT:    adds r7, r7, r4
+; CHECK-LIS-NEXT:    adcs r2, r6
+; CHECK-LIS-NEXT:    adds r7, r7, r5
 ; CHECK-LIS-NEXT:    adc.w r8, r2, r0
-; CHECK-LIS-NEXT:    vmov r7, r4, d1
+; CHECK-LIS-NEXT:    vmov r6, r4, d1
 ; CHECK-LIS-NEXT:    vmov r2, r5, d3
 ; CHECK-LIS-NEXT:    vmov r3, r0, d0
-; CHECK-LIS-NEXT:    adds r2, r2, r7
-; CHECK-LIS-NEXT:    adc.w r7, r5, r4
+; CHECK-LIS-NEXT:    adds r2, r2, r6
+; CHECK-LIS-NEXT:    adc.w r6, r5, r4
 ; CHECK-LIS-NEXT:    vmov r5, r4, d7
 ; CHECK-LIS-NEXT:    adds r2, r2, r5
-; CHECK-LIS-NEXT:    adcs r7, r4
+; CHECK-LIS-NEXT:    adcs r6, r4
 ; CHECK-LIS-NEXT:    vmov r5, r4, d2
-; CHECK-LIS-NEXT:    vmov q1[2], q1[0], r6, r2
-; CHECK-LIS-NEXT:    vmov q1[3], q1[1], r8, r7
+; CHECK-LIS-NEXT:    vmov q1[2], q1[0], r7, r2
+; CHECK-LIS-NEXT:    vmov q1[3], q1[1], r8, r6
 ; CHECK-LIS-NEXT:    vstrw.32 q1, [r1, #16]
 ; CHECK-LIS-NEXT:    adds r3, r3, r5
 ; CHECK-LIS-NEXT:    adcs r0, r4
@@ -964,19 +1134,33 @@ entry:
 ; f32
 
 define void @vld3_v2f32(ptr %src, ptr %dst) {
-; CHECK-LABEL: vld3_v2f32:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q2, [r0]
-; CHECK-NEXT:    vldr s1, [r0, #16]
-; CHECK-NEXT:    vldr s5, [r0, #20]
-; CHECK-NEXT:    vmov.f32 s12, s8
-; CHECK-NEXT:    vmov.f32 s13, s11
-; CHECK-NEXT:    vmov.f32 s0, s9
-; CHECK-NEXT:    vadd.f32 q0, q3, q0
-; CHECK-NEXT:    vmov.f32 s4, s10
-; CHECK-NEXT:    vadd.f32 q0, q0, q1
-; CHECK-NEXT:    vstmia r1, {s0, s1}
-; CHECK-NEXT:    bx lr
+; CHECK-LV-LABEL: vld3_v2f32:
+; CHECK-LV:       @ %bb.0: @ %entry
+; CHECK-LV-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-LV-NEXT:    vldr s1, [r0, #16]
+; CHECK-LV-NEXT:    vldr s5, [r0, #20]
+; CHECK-LV-NEXT:    vmov.f32 s12, s8
+; CHECK-LV-NEXT:    vmov.f32 s13, s11
+; CHECK-LV-NEXT:    vmov.f32 s0, s9
+; CHECK-LV-NEXT:    vadd.f32 q0, q3, q0
+; CHECK-LV-NEXT:    vmov.f32 s4, s10
+; CHECK-LV-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-LV-NEXT:    vstmia r1, {s0, s1}
+; CHECK-LV-NEXT:    bx lr
+
+; CHECK-LIS-LABEL: vld3_v2f32:
+; CHECK-LIS:       @ %bb.0: @ %entry
+; CHECK-LIS-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-LIS-NEXT:    vldr s5, [r0, #16]
+; CHECK-LIS-NEXT:    vldr s1, [r0, #20]
+; CHECK-LIS-NEXT:    vmov.f32 s12, s8
+; CHECK-LIS-NEXT:    vmov.f32 s13, s11
+; CHECK-LIS-NEXT:    vmov.f32 s4, s9
+; CHECK-LIS-NEXT:    vadd.f32 q1, q3, q1
+; CHECK-LIS-NEXT:    vmov.f32 s0, s10
+; CHECK-LIS-NEXT:    vadd.f32 q0, q1, q0
+; CHECK-LIS-NEXT:    vstmia r1, {s0, s1}
+; CHECK-LIS-NEXT:    bx lr
 entry:
   %l1 = load <6 x float>, ptr %src, align 4
   %s1 = shufflevector <6 x float> %l1, <6 x float> undef, <2 x i32> <i32 0, i32 3>
@@ -989,29 +1173,53 @@ entry:
 }
 
 define void @vld3_v4f32(ptr %src, ptr %dst) {
-; CHECK-LABEL: vld3_v4f32:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
-; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    vldrw.u32 q4, [r0, #32]
-; CHECK-NEXT:    vmov.f32 s10, s2
-; CHECK-NEXT:    vmov.f32 s13, s0
-; CHECK-NEXT:    vmov.f32 s14, s3
-; CHECK-NEXT:    vmov.f32 s8, s4
-; CHECK-NEXT:    vmov.f32 s9, s7
-; CHECK-NEXT:    vmov.f32 s12, s5
-; CHECK-NEXT:    vmov.f32 s15, s18
-; CHECK-NEXT:    vmov.f32 s11, s17
-; CHECK-NEXT:    vadd.f32 q2, q2, q3
-; CHECK-NEXT:    vmov.f32 s0, s6
-; CHECK-NEXT:    vmov.f32 s2, s16
-; CHECK-NEXT:    vmov.f32 s3, s19
-; CHECK-NEXT:    vadd.f32 q0, q2, q0
-; CHECK-NEXT:    vstrw.32 q0, [r1]
-; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    bx lr
+; CHECK-LV-LABEL: vld3_v4f32:
+; CHECK-LV:       @ %bb.0: @ %entry
+; CHECK-LV-NEXT:    .vsave {d8, d9}
+; CHECK-LV-NEXT:    vpush {d8, d9}
+; CHECK-LV-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-LV-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-LV-NEXT:    vldrw.u32 q4, [r0, #32]
+; CHECK-LV-NEXT:    vmov.f32 s10, s2
+; CHECK-LV-NEXT:    vmov.f32 s13, s0
+; CHECK-LV-NEXT:    vmov.f32 s14, s3
+; CHECK-LV-NEXT:    vmov.f32 s8, s4
+; CHECK-LV-NEXT:    vmov.f32 s9, s7
+; CHECK-LV-NEXT:    vmov.f32 s12, s5
+; CHECK-LV-NEXT:    vmov.f32 s15, s18
+; CHECK-LV-NEXT:    vmov.f32 s11, s17
+; CHECK-LV-NEXT:    vadd.f32 q2, q2, q3
+; CHECK-LV-NEXT:    vmov.f32 s0, s6
+; CHECK-LV-NEXT:    vmov.f32 s2, s16
+; CHECK-LV-NEXT:    vmov.f32 s3, s19
+; CHECK-LV-NEXT:    vadd.f32 q0, q2, q0
+; CHECK-LV-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LV-NEXT:    vpop {d8, d9}
+; CHECK-LV-NEXT:    bx lr
+
+; CHECK-LIS-LABEL: vld3_v4f32:
+; CHECK-LIS:       @ %bb.0: @ %entry
+; CHECK-LIS-NEXT:    .vsave {d8, d9}
+; CHECK-LIS-NEXT:    vpush {d8, d9}
+; CHECK-LIS-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-LIS-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-LIS-NEXT:    vldrw.u32 q4, [r0, #32]
+; CHECK-LIS-NEXT:    vmov.f32 s10, s2
+; CHECK-LIS-NEXT:    vmov.f32 s13, s0
+; CHECK-LIS-NEXT:    vmov.f32 s14, s3
+; CHECK-LIS-NEXT:    vmov.f32 s8, s4
+; CHECK-LIS-NEXT:    vmov.f32 s9, s7
+; CHECK-LIS-NEXT:    vmov.f32 s12, s5
+; CHECK-LIS-NEXT:    vmov.f32 s15, s18
+; CHECK-LIS-NEXT:    vmov.f32 s11, s17
+; CHECK-LIS-NEXT:    vadd.f32 q2, q2, q3
+; CHECK-LIS-NEXT:    vmov.f32 s0, s6
+; CHECK-LIS-NEXT:    vmov.f32 s2, s16
+; CHECK-LIS-NEXT:    vmov.f32 s3, s19
+; CHECK-LIS-NEXT:    vadd.f32 q0, q2, q0
+; CHECK-LIS-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LIS-NEXT:    vpop {d8, d9}
+; CHECK-LIS-NEXT:    bx lr
 entry:
   %l1 = load <12 x float>, ptr %src, align 4
   %s1 = shufflevector <12 x float> %l1, <12 x float> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
@@ -1024,46 +1232,87 @@ entry:
 }
 
 define void @vld3_v8f32(ptr %src, ptr %dst) {
-; CHECK-LABEL: vld3_v8f32:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
-; CHECK-NEXT:    vldrw.u32 q4, [r0, #80]
-; CHECK-NEXT:    vmov.f32 s10, s2
-; CHECK-NEXT:    vmov.f32 s13, s0
-; CHECK-NEXT:    vmov.f32 s14, s3
-; CHECK-NEXT:    vmov.f32 s8, s4
-; CHECK-NEXT:    vmov.f32 s9, s7
-; CHECK-NEXT:    vmov.f32 s12, s5
-; CHECK-NEXT:    vmov.f32 s15, s18
-; CHECK-NEXT:    vmov.f32 s11, s17
-; CHECK-NEXT:    vadd.f32 q2, q2, q3
-; CHECK-NEXT:    vmov.f32 s0, s6
-; CHECK-NEXT:    vmov.f32 s2, s16
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
-; CHECK-NEXT:    vmov.f32 s3, s19
-; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
-; CHECK-NEXT:    vadd.f32 q0, q2, q0
-; CHECK-NEXT:    vldrw.u32 q2, [r0]
-; CHECK-NEXT:    vmov.f32 s17, s4
-; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
-; CHECK-NEXT:    vmov.f32 s18, s7
-; CHECK-NEXT:    vmov.f32 s22, s6
-; CHECK-NEXT:    vmov.f32 s16, s9
-; CHECK-NEXT:    vmov.f32 s19, s14
-; CHECK-NEXT:    vmov.f32 s20, s8
-; CHECK-NEXT:    vmov.f32 s21, s11
-; CHECK-NEXT:    vmov.f32 s23, s13
-; CHECK-NEXT:    vadd.f32 q4, q5, q4
-; CHECK-NEXT:    vmov.f32 s4, s10
-; CHECK-NEXT:    vmov.f32 s6, s12
-; CHECK-NEXT:    vmov.f32 s7, s15
-; CHECK-NEXT:    vadd.f32 q1, q4, q1
-; CHECK-NEXT:    vstrw.32 q1, [r1]
-; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    bx lr
+; CHECK-LV-LABEL: vld3_v8f32:
+; CHECK-LV:       @ %bb.0: @ %entry
+; CHECK-LV-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-LV-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-LV-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-LV-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-LV-NEXT:    vldrw.u32 q4, [r0, #80]
+; CHECK-LV-NEXT:    vmov.f32 s10, s2
+; CHECK-LV-NEXT:    vmov.f32 s13, s0
+; CHECK-LV-NEXT:    vmov.f32 s14, s3
+; CHECK-LV-NEXT:    vmov.f32 s8, s4
+; CHECK-LV-NEXT:    vmov.f32 s9, s7
+; CHECK-LV-NEXT:    vmov.f32 s12, s5
+; CHECK-LV-NEXT:    vmov.f32 s15, s18
+; CHECK-LV-NEXT:    vmov.f32 s11, s17
+; CHECK-LV-NEXT:    vadd.f32 q2, q2, q3
+; CHECK-LV-NEXT:    vmov.f32 s0, s6
+; CHECK-LV-NEXT:    vmov.f32 s2, s16
+; CHECK-LV-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-LV-NEXT:    vmov.f32 s3, s19
+; CHECK-LV-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-LV-NEXT:    vadd.f32 q0, q2, q0
+; CHECK-LV-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-LV-NEXT:    vmov.f32 s17, s4
+; CHECK-LV-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-LV-NEXT:    vmov.f32 s18, s7
+; CHECK-LV-NEXT:    vmov.f32 s22, s6
+; CHECK-LV-NEXT:    vmov.f32 s16, s9
+; CHECK-LV-NEXT:    vmov.f32 s19, s14
+; CHECK-LV-NEXT:    vmov.f32 s20, s8
+; CHECK-LV-NEXT:    vmov.f32 s21, s11
+; CHECK-LV-NEXT:    vmov.f32 s23, s13
+; CHECK-LV-NEXT:    vadd.f32 q4, q5, q4
+; CHECK-LV-NEXT:    vmov.f32 s4, s10
+; CHECK-LV-NEXT:    vmov.f32 s6, s12
+; CHECK-LV-NEXT:    vmov.f32 s7, s15
+; CHECK-LV-NEXT:    vadd.f32 q1, q4, q1
+; CHECK-LV-NEXT:    vstrw.32 q1, [r1]
+; CHECK-LV-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-LV-NEXT:    bx lr
+
+; CHECK-LIS-LABEL: vld3_v8f32:
+; CHECK-LIS:       @ %bb.0: @ %entry
+; CHECK-LIS-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-LIS-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-LIS-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-LIS-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-LIS-NEXT:    vldrw.u32 q4, [r0, #80]
+; CHECK-LIS-NEXT:    vmov.f32 s10, s2
+; CHECK-LIS-NEXT:    vmov.f32 s13, s0
+; CHECK-LIS-NEXT:    vmov.f32 s14, s3
+; CHECK-LIS-NEXT:    vmov.f32 s8, s4
+; CHECK-LIS-NEXT:    vmov.f32 s9, s7
+; CHECK-LIS-NEXT:    vmov.f32 s12, s5
+; CHECK-LIS-NEXT:    vmov.f32 s15, s18
+; CHECK-LIS-NEXT:    vmov.f32 s11, s17
+; CHECK-LIS-NEXT:    vadd.f32 q2, q2, q3
+; CHECK-LIS-NEXT:    vmov.f32 s0, s6
+; CHECK-LIS-NEXT:    vmov.f32 s2, s16
+; CHECK-LIS-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-LIS-NEXT:    vmov.f32 s3, s19
+; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-LIS-NEXT:    vadd.f32 q0, q2, q0
+; CHECK-LIS-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-LIS-NEXT:    vmov.f32 s17, s4
+; CHECK-LIS-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-LIS-NEXT:    vmov.f32 s18, s7
+; CHECK-LIS-NEXT:    vmov.f32 s22, s6
+; CHECK-LIS-NEXT:    vmov.f32 s16, s9
+; CHECK-LIS-NEXT:    vmov.f32 s19, s14
+; CHECK-LIS-NEXT:    vmov.f32 s20, s8
+; CHECK-LIS-NEXT:    vmov.f32 s21, s11
+; CHECK-LIS-NEXT:    vmov.f32 s23, s13
+; CHECK-LIS-NEXT:    vadd.f32 q4, q5, q4
+; CHECK-LIS-NEXT:    vmov.f32 s4, s10
+; CHECK-LIS-NEXT:    vmov.f32 s6, s12
+; CHECK-LIS-NEXT:    vmov.f32 s7, s15
+; CHECK-LIS-NEXT:    vadd.f32 q1, q4, q1
+; CHECK-LIS-NEXT:    vstrw.32 q1, [r1]
+; CHECK-LIS-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-LIS-NEXT:    bx lr
 entry:
   %l1 = load <24 x float>, ptr %src, align 4
   %s1 = shufflevector <24 x float> %l1, <24 x float> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
@@ -1076,80 +1325,155 @@ entry:
 }
 
 define void @vld3_v16f32(ptr %src, ptr %dst) {
-; CHECK-LABEL: vld3_v16f32:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
-; CHECK-NEXT:    vldrw.u32 q4, [r0, #80]
-; CHECK-NEXT:    vldrw.u32 q6, [r0, #176]
-; CHECK-NEXT:    vmov.f32 s10, s2
-; CHECK-NEXT:    vmov.f32 s13, s0
-; CHECK-NEXT:    vmov.f32 s14, s3
-; CHECK-NEXT:    vmov.f32 s8, s4
-; CHECK-NEXT:    vmov.f32 s9, s7
-; CHECK-NEXT:    vmov.f32 s12, s5
-; CHECK-NEXT:    vmov.f32 s15, s18
-; CHECK-NEXT:    vmov.f32 s11, s17
-; CHECK-NEXT:    vadd.f32 q2, q2, q3
-; CHECK-NEXT:    vmov.f32 s0, s6
-; CHECK-NEXT:    vmov.f32 s2, s16
-; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
-; CHECK-NEXT:    vmov.f32 s3, s19
-; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
-; CHECK-NEXT:    vadd.f32 q0, q2, q0
-; CHECK-NEXT:    vldrw.u32 q2, [r0]
-; CHECK-NEXT:    vmov.f32 s17, s4
-; CHECK-NEXT:    vmov.f32 s18, s7
-; CHECK-NEXT:    vmov.f32 s22, s6
-; CHECK-NEXT:    vmov.f32 s16, s9
-; CHECK-NEXT:    vmov.f32 s19, s14
-; CHECK-NEXT:    vmov.f32 s20, s8
-; CHECK-NEXT:    vmov.f32 s21, s11
-; CHECK-NEXT:    vmov.f32 s23, s13
-; CHECK-NEXT:    vmov.f32 s4, s10
-; CHECK-NEXT:    vldrw.u32 q2, [r0, #160]
-; CHECK-NEXT:    vmov.f32 s6, s12
-; CHECK-NEXT:    vadd.f32 q4, q5, q4
-; CHECK-NEXT:    vmov.f32 s7, s15
-; CHECK-NEXT:    vldrw.u32 q3, [r0, #144]
-; CHECK-NEXT:    vadd.f32 q1, q4, q1
-; CHECK-NEXT:    vmov.f32 s18, s10
-; CHECK-NEXT:    vmov.f32 s21, s8
-; CHECK-NEXT:    vmov.f32 s22, s11
-; CHECK-NEXT:    vmov.f32 s16, s12
-; CHECK-NEXT:    vmov.f32 s17, s15
-; CHECK-NEXT:    vmov.f32 s20, s13
-; CHECK-NEXT:    vmov.f32 s23, s26
-; CHECK-NEXT:    vmov.f32 s19, s25
-; CHECK-NEXT:    vadd.f32 q4, q4, q5
-; CHECK-NEXT:    vmov.f32 s8, s14
-; CHECK-NEXT:    vmov.f32 s10, s24
-; CHECK-NEXT:    vldrw.u32 q3, [r0, #112]
-; CHECK-NEXT:    vmov.f32 s11, s27
-; CHECK-NEXT:    vldrw.u32 q5, [r0, #128]
-; CHECK-NEXT:    vadd.f32 q2, q4, q2
-; CHECK-NEXT:    vldrw.u32 q4, [r0, #96]
-; CHECK-NEXT:    vmov.f32 s25, s12
-; CHECK-NEXT:    vstrw.32 q2, [r1, #48]
-; CHECK-NEXT:    vmov.f32 s26, s15
-; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
-; CHECK-NEXT:    vmov.f32 s30, s14
-; CHECK-NEXT:    vstrw.32 q1, [r1]
-; CHECK-NEXT:    vmov.f32 s24, s17
-; CHECK-NEXT:    vmov.f32 s27, s22
-; CHECK-NEXT:    vmov.f32 s28, s16
-; CHECK-NEXT:    vmov.f32 s29, s19
-; CHECK-NEXT:    vmov.f32 s31, s21
-; CHECK-NEXT:    vadd.f32 q6, q7, q6
-; CHECK-NEXT:    vmov.f32 s12, s18
-; CHECK-NEXT:    vmov.f32 s14, s20
-; CHECK-NEXT:    vmov.f32 s15, s23
-; CHECK-NEXT:    vadd.f32 q3, q6, q3
-; CHECK-NEXT:    vstrw.32 q3, [r1, #32]
-; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    bx lr
+; CHECK-LV-LABEL: vld3_v16f32:
+; CHECK-LV:       @ %bb.0: @ %entry
+; CHECK-LV-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-LV-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-LV-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-LV-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-LV-NEXT:    vldrw.u32 q4, [r0, #80]
+; CHECK-LV-NEXT:    vldrw.u32 q6, [r0, #176]
+; CHECK-LV-NEXT:    vmov.f32 s10, s2
+; CHECK-LV-NEXT:    vmov.f32 s13, s0
+; CHECK-LV-NEXT:    vmov.f32 s14, s3
+; CHECK-LV-NEXT:    vmov.f32 s8, s4
+; CHECK-LV-NEXT:    vmov.f32 s9, s7
+; CHECK-LV-NEXT:    vmov.f32 s12, s5
+; CHECK-LV-NEXT:    vmov.f32 s15, s18
+; CHECK-LV-NEXT:    vmov.f32 s11, s17
+; CHECK-LV-NEXT:    vadd.f32 q2, q2, q3
+; CHECK-LV-NEXT:    vmov.f32 s0, s6
+; CHECK-LV-NEXT:    vmov.f32 s2, s16
+; CHECK-LV-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-LV-NEXT:    vmov.f32 s3, s19
+; CHECK-LV-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-LV-NEXT:    vadd.f32 q0, q2, q0
+; CHECK-LV-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-LV-NEXT:    vmov.f32 s17, s4
+; CHECK-LV-NEXT:    vmov.f32 s18, s7
+; CHECK-LV-NEXT:    vmov.f32 s22, s6
+; CHECK-LV-NEXT:    vmov.f32 s16, s9
+; CHECK-LV-NEXT:    vmov.f32 s19, s14
+; CHECK-LV-NEXT:    vmov.f32 s20, s8
+; CHECK-LV-NEXT:    vmov.f32 s21, s11
+; CHECK-LV-NEXT:    vmov.f32 s23, s13
+; CHECK-LV-NEXT:    vmov.f32 s4, s10
+; CHECK-LV-NEXT:    vldrw.u32 q2, [r0, #160]
+; CHECK-LV-NEXT:    vmov.f32 s6, s12
+; CHECK-LV-NEXT:    vadd.f32 q4, q5, q4
+; CHECK-LV-NEXT:    vmov.f32 s7, s15
+; CHECK-LV-NEXT:    vldrw.u32 q3, [r0, #144]
+; CHECK-LV-NEXT:    vadd.f32 q1, q4, q1
+; CHECK-LV-NEXT:    vmov.f32 s18, s10
+; CHECK-LV-NEXT:    vmov.f32 s21, s8
+; CHECK-LV-NEXT:    vmov.f32 s22, s11
+; CHECK-LV-NEXT:    vmov.f32 s16, s12
+; CHECK-LV-NEXT:    vmov.f32 s17, s15
+; CHECK-LV-NEXT:    vmov.f32 s20, s13
+; CHECK-LV-NEXT:    vmov.f32 s23, s26
+; CHECK-LV-NEXT:    vmov.f32 s19, s25
+; CHECK-LV-NEXT:    vadd.f32 q4, q4, q5
+; CHECK-LV-NEXT:    vmov.f32 s8, s14
+; CHECK-LV-NEXT:    vmov.f32 s10, s24
+; CHECK-LV-NEXT:    vldrw.u32 q3, [r0, #112]
+; CHECK-LV-NEXT:    vmov.f32 s11, s27
+; CHECK-LV-NEXT:    vldrw.u32 q5, [r0, #128]
+; CHECK-LV-NEXT:    vadd.f32 q2, q4, q2
+; CHECK-LV-NEXT:    vldrw.u32 q4, [r0, #96]
+; CHECK-LV-NEXT:    vmov.f32 s25, s12
+; CHECK-LV-NEXT:    vstrw.32 q2, [r1, #48]
+; CHECK-LV-NEXT:    vmov.f32 s26, s15
+; CHECK-LV-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-LV-NEXT:    vmov.f32 s30, s14
+; CHECK-LV-NEXT:    vstrw.32 q1, [r1]
+; CHECK-LV-NEXT:    vmov.f32 s24, s17
+; CHECK-LV-NEXT:    vmov.f32 s27, s22
+; CHECK-LV-NEXT:    vmov.f32 s28, s16
+; CHECK-LV-NEXT:    vmov.f32 s29, s19
+; CHECK-LV-NEXT:    vmov.f32 s31, s21
+; CHECK-LV-NEXT:    vadd.f32 q6, q7, q6
+; CHECK-LV-NEXT:    vmov.f32 s12, s18
+; CHECK-LV-NEXT:    vmov.f32 s14, s20
+; CHECK-LV-NEXT:    vmov.f32 s15, s23
+; CHECK-LV-NEXT:    vadd.f32 q3, q6, q3
+; CHECK-LV-NEXT:    vstrw.32 q3, [r1, #32]
+; CHECK-LV-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-LV-NEXT:    bx lr
+
+; CHECK-LIS-LABEL: vld3_v16f32:
+; CHECK-LIS:       @ %bb.0: @ %entry
+; CHECK-LIS-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-LIS-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-LIS-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-LIS-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-LIS-NEXT:    vldrw.u32 q4, [r0, #80]
+; CHECK-LIS-NEXT:    vldrw.u32 q6, [r0, #176]
+; CHECK-LIS-NEXT:    vmov.f32 s10, s2
+; CHECK-LIS-NEXT:    vmov.f32 s13, s0
+; CHECK-LIS-NEXT:    vmov.f32 s14, s3
+; CHECK-LIS-NEXT:    vmov.f32 s8, s4
+; CHECK-LIS-NEXT:    vmov.f32 s9, s7
+; CHECK-LIS-NEXT:    vmov.f32 s12, s5
+; CHECK-LIS-NEXT:    vmov.f32 s15, s18
+; CHECK-LIS-NEXT:    vmov.f32 s11, s17
+; CHECK-LIS-NEXT:    vadd.f32 q2, q2, q3
+; CHECK-LIS-NEXT:    vmov.f32 s0, s6
+; CHECK-LIS-NEXT:    vmov.f32 s2, s16
+; CHECK-LIS-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-LIS-NEXT:    vmov.f32 s3, s19
+; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-LIS-NEXT:    vadd.f32 q0, q2, q0
+; CHECK-LIS-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-LIS-NEXT:    vmov.f32 s17, s4
+; CHECK-LIS-NEXT:    vmov.f32 s18, s7
+; CHECK-LIS-NEXT:    vmov.f32 s22, s6
+; CHECK-LIS-NEXT:    vmov.f32 s16, s9
+; CHECK-LIS-NEXT:    vmov.f32 s19, s14
+; CHECK-LIS-NEXT:    vmov.f32 s20, s8
+; CHECK-LIS-NEXT:    vmov.f32 s21, s11
+; CHECK-LIS-NEXT:    vmov.f32 s23, s13
+; CHECK-LIS-NEXT:    vmov.f32 s4, s10
+; CHECK-LIS-NEXT:    vldrw.u32 q2, [r0, #160]
+; CHECK-LIS-NEXT:    vmov.f32 s6, s12
+; CHECK-LIS-NEXT:    vadd.f32 q4, q5, q4
+; CHECK-LIS-NEXT:    vmov.f32 s7, s15
+; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #144]
+; CHECK-LIS-NEXT:    vadd.f32 q1, q4, q1
+; CHECK-LIS-NEXT:    vmov.f32 s18, s10
+; CHECK-LIS-NEXT:    vmov.f32 s21, s8
+; CHECK-LIS-NEXT:    vmov.f32 s22, s11
+; CHECK-LIS-NEXT:    vmov.f32 s16, s12
+; CHECK-LIS-NEXT:    vmov.f32 s17, s15
+; CHECK-LIS-NEXT:    vmov.f32 s20, s13
+; CHECK-LIS-NEXT:    vmov.f32 s23, s26
+; CHECK-LIS-NEXT:    vmov.f32 s19, s25
+; CHECK-LIS-NEXT:    vadd.f32 q4, q4, q5
+; CHECK-LIS-NEXT:    vmov.f32 s8, s14
+; CHECK-LIS-NEXT:    vmov.f32 s10, s24
+; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #112]
+; CHECK-LIS-NEXT:    vmov.f32 s11, s27
+; CHECK-LIS-NEXT:    vldrw.u32 q5, [r0, #128]
+; CHECK-LIS-NEXT:    vadd.f32 q2, q4, q2
+; CHECK-LIS-NEXT:    vldrw.u32 q4, [r0, #96]
+; CHECK-LIS-NEXT:    vmov.f32 s25, s12
+; CHECK-LIS-NEXT:    vstrw.32 q2, [r1, #48]
+; CHECK-LIS-NEXT:    vmov.f32 s26, s15
+; CHECK-LIS-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-LIS-NEXT:    vmov.f32 s30, s14
+; CHECK-LIS-NEXT:    vstrw.32 q1, [r1]
+; CHECK-LIS-NEXT:    vmov.f32 s24, s17
+; CHECK-LIS-NEXT:    vmov.f32 s27, s22
+; CHECK-LIS-NEXT:    vmov.f32 s28, s16
+; CHECK-LIS-NEXT:    vmov.f32 s29, s19
+; CHECK-LIS-NEXT:    vmov.f32 s31, s21
+; CHECK-LIS-NEXT:    vadd.f32 q6, q7, q6
+; CHECK-LIS-NEXT:    vmov.f32 s12, s18
+; CHECK-LIS-NEXT:    vmov.f32 s14, s20
+; CHECK-LIS-NEXT:    vmov.f32 s15, s23
+; CHECK-LIS-NEXT:    vadd.f32 q3, q6, q3
+; CHECK-LIS-NEXT:    vstrw.32 q3, [r1, #32]
+; CHECK-LIS-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-LIS-NEXT:    bx lr
 entry:
   %l1 = load <48 x float>, ptr %src, align 4
   %s1 = shufflevector <48 x float> %l1, <48 x float> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
@@ -1288,86 +1612,167 @@ entry:
 }
 
 define void @vld3_v16f16(ptr %src, ptr %dst) {
-; CHECK-LABEL: vld3_v16f16:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .vsave {d8, d9}
-; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #48]
-; CHECK-NEXT:    vldrw.u32 q2, [r0, #64]
-; CHECK-NEXT:    vldrw.u32 q3, [r0, #80]
-; CHECK-NEXT:    vmovx.f16 s6, s2
-; CHECK-NEXT:    vmov.f32 s4, s1
-; CHECK-NEXT:    vins.f16 s4, s6
-; CHECK-NEXT:    vmovx.f16 s6, s9
-; CHECK-NEXT:    vmov.f32 s5, s8
-; CHECK-NEXT:    vmovx.f16 s7, s12
-; CHECK-NEXT:    vins.f16 s5, s6
-; CHECK-NEXT:    vmov.f32 s6, s11
-; CHECK-NEXT:    vins.f16 s6, s7
-; CHECK-NEXT:    vmovx.f16 s16, s15
-; CHECK-NEXT:    vmov.f32 s7, s14
-; CHECK-NEXT:    vmovx.f16 s17, s3
-; CHECK-NEXT:    vins.f16 s7, s16
-; CHECK-NEXT:    vmovx.f16 s16, s0
-; CHECK-NEXT:    vins.f16 s16, s2
-; CHECK-NEXT:    vmovx.f16 s2, s1
-; CHECK-NEXT:    vins.f16 s0, s2
-; CHECK-NEXT:    vmovx.f16 s2, s8
-; CHECK-NEXT:    vins.f16 s3, s2
-; CHECK-NEXT:    vmovx.f16 s2, s11
-; CHECK-NEXT:    vmovx.f16 s18, s10
-; CHECK-NEXT:    vins.f16 s10, s2
-; CHECK-NEXT:    vmovx.f16 s2, s14
-; CHECK-NEXT:    vmovx.f16 s19, s13
-; CHECK-NEXT:    vins.f16 s13, s2
-; CHECK-NEXT:    vmov.f32 s1, s3
-; CHECK-NEXT:    vins.f16 s18, s12
-; CHECK-NEXT:    vins.f16 s19, s15
-; CHECK-NEXT:    vins.f16 s17, s9
-; CHECK-NEXT:    vmov.f32 s2, s10
-; CHECK-NEXT:    vmov.f32 s3, s13
-; CHECK-NEXT:    vldrw.u32 q2, [r0, #16]
-; CHECK-NEXT:    vadd.f16 q0, q0, q4
-; CHECK-NEXT:    vadd.f16 q3, q0, q1
-; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #32]
-; CHECK-NEXT:    vstrw.32 q3, [r1, #16]
-; CHECK-NEXT:    vmovx.f16 s14, s6
-; CHECK-NEXT:    vmov.f32 s12, s5
-; CHECK-NEXT:    vins.f16 s12, s14
-; CHECK-NEXT:    vmovx.f16 s14, s9
-; CHECK-NEXT:    vmov.f32 s13, s8
-; CHECK-NEXT:    vmovx.f16 s18, s10
-; CHECK-NEXT:    vins.f16 s13, s14
-; CHECK-NEXT:    vmovx.f16 s15, s0
-; CHECK-NEXT:    vmov.f32 s14, s11
-; CHECK-NEXT:    vins.f16 s18, s0
-; CHECK-NEXT:    vins.f16 s14, s15
-; CHECK-NEXT:    vmovx.f16 s16, s3
-; CHECK-NEXT:    vmov.f32 s15, s2
-; CHECK-NEXT:    vmovx.f16 s0, s5
-; CHECK-NEXT:    vins.f16 s15, s16
-; CHECK-NEXT:    vmovx.f16 s16, s4
-; CHECK-NEXT:    vins.f16 s4, s0
-; CHECK-NEXT:    vmovx.f16 s0, s8
-; CHECK-NEXT:    vmovx.f16 s17, s7
-; CHECK-NEXT:    vins.f16 s7, s0
-; CHECK-NEXT:    vmovx.f16 s0, s11
-; CHECK-NEXT:    vmovx.f16 s19, s1
-; CHECK-NEXT:    vins.f16 s10, s0
-; CHECK-NEXT:    vmovx.f16 s0, s2
-; CHECK-NEXT:    vins.f16 s1, s0
-; CHECK-NEXT:    vins.f16 s16, s6
-; CHECK-NEXT:    vmov.f32 s5, s7
-; CHECK-NEXT:    vins.f16 s19, s3
-; CHECK-NEXT:    vins.f16 s17, s9
-; CHECK-NEXT:    vmov.f32 s6, s10
-; CHECK-NEXT:    vmov.f32 s7, s1
-; CHECK-NEXT:    vadd.f16 q0, q1, q4
-; CHECK-NEXT:    vadd.f16 q0, q0, q3
-; CHECK-NEXT:    vstrw.32 q0, [r1]
-; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    bx lr
+; CHECK-LV-LABEL: vld3_v16f16:
+; CHECK-LV:       @ %bb.0: @ %entry
+; CHECK-LV-NEXT:    .vsave {d8, d9}
+; CHECK-LV-NEXT:    vpush {d8, d9}
+; CHECK-LV-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-LV-NEXT:    vldrw.u32 q2, [r0, #64]
+; CHECK-LV-NEXT:    vldrw.u32 q3, [r0, #80]
+; CHECK-LV-NEXT:    vmovx.f16 s6, s2
+; CHECK-LV-NEXT:    vmov.f32 s4, s1
+; CHECK-LV-NEXT:    vins.f16 s4, s6
+; CHECK-LV-NEXT:    vmovx.f16 s6, s9
+; CHECK-LV-NEXT:    vmov.f32 s5, s8
+; CHECK-LV-NEXT:    vmovx.f16 s7, s12
+; CHECK-LV-NEXT:    vins.f16 s5, s6
+; CHECK-LV-NEXT:    vmov.f32 s6, s11
+; CHECK-LV-NEXT:    vins.f16 s6, s7
+; CHECK-LV-NEXT:    vmovx.f16 s16, s15
+; CHECK-LV-NEXT:    vmov.f32 s7, s14
+; CHECK-LV-NEXT:    vmovx.f16 s17, s3
+; CHECK-LV-NEXT:    vins.f16 s7, s16
+; CHECK-LV-NEXT:    vmovx.f16 s16, s0
+; CHECK-LV-NEXT:    vins.f16 s16, s2
+; CHECK-LV-NEXT:    vmovx.f16 s2, s1
+; CHECK-LV-NEXT:    vins.f16 s0, s2
+; CHECK-LV-NEXT:    vmovx.f16 s2, s8
+; CHECK-LV-NEXT:    vins.f16 s3, s2
+; CHECK-LV-NEXT:    vmovx.f16 s2, s11
+; CHECK-LV-NEXT:    vmovx.f16 s18, s10
+; CHECK-LV-NEXT:    vins.f16 s10, s2
+; CHECK-LV-NEXT:    vmovx.f16 s2, s14
+; CHECK-LV-NEXT:    vmovx.f16 s19, s13
+; CHECK-LV-NEXT:    vins.f16 s13, s2
+; CHECK-LV-NEXT:    vmov.f32 s1, s3
+; CHECK-LV-NEXT:    vins.f16 s18, s12
+; CHECK-LV-NEXT:    vins.f16 s19, s15
+; CHECK-LV-NEXT:    vins.f16 s17, s9
+; CHECK-LV-NEXT:    vmov.f32 s2, s10
+; CHECK-LV-NEXT:    vmov.f32 s3, s13
+; CHECK-LV-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-LV-NEXT:    vadd.f16 q0, q0, q4
+; CHECK-LV-NEXT:    vadd.f16 q3, q0, q1
+; CHECK-LV-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-LV-NEXT:    vldrw.u32 q0, [r0, #32]
+; CHECK-LV-NEXT:    vstrw.32 q3, [r1, #16]
+; CHECK-LV-NEXT:    vmovx.f16 s14, s6
+; CHECK-LV-NEXT:    vmov.f32 s12, s5
+; CHECK-LV-NEXT:    vins.f16 s12, s14
+; CHECK-LV-NEXT:    vmovx.f16 s14, s9
+; CHECK-LV-NEXT:    vmov.f32 s13, s8
+; CHECK-LV-NEXT:    vmovx.f16 s18, s10
+; CHECK-LV-NEXT:    vins.f16 s13, s14
+; CHECK-LV-NEXT:    vmovx.f16 s15, s0
+; CHECK-LV-NEXT:    vmov.f32 s14, s11
+; CHECK-LV-NEXT:    vins.f16 s18, s0
+; CHECK-LV-NEXT:    vins.f16 s14, s15
+; CHECK-LV-NEXT:    vmovx.f16 s16, s3
+; CHECK-LV-NEXT:    vmov.f32 s15, s2
+; CHECK-LV-NEXT:    vmovx.f16 s0, s5
+; CHECK-LV-NEXT:    vins.f16 s15, s16
+; CHECK-LV-NEXT:    vmovx.f16 s16, s4
+; CHECK-LV-NEXT:    vins.f16 s4, s0
+; CHECK-LV-NEXT:    vmovx.f16 s0, s8
+; CHECK-LV-NEXT:    vmovx.f16 s17, s7
+; CHECK-LV-NEXT:    vins.f16 s7, s0
+; CHECK-LV-NEXT:    vmovx.f16 s0, s11
+; CHECK-LV-NEXT:    vmovx.f16 s19, s1
+; CHECK-LV-NEXT:    vins.f16 s10, s0
+; CHECK-LV-NEXT:    vmovx.f16 s0, s2
+; CHECK-LV-NEXT:    vins.f16 s1, s0
+; CHECK-LV-NEXT:    vins.f16 s16, s6
+; CHECK-LV-NEXT:    vmov.f32 s5, s7
+; CHECK-LV-NEXT:    vins.f16 s19, s3
+; CHECK-LV-NEXT:    vins.f16 s17, s9
+; CHECK-LV-NEXT:    vmov.f32 s6, s10
+; CHECK-LV-NEXT:    vmov.f32 s7, s1
+; CHECK-LV-NEXT:    vadd.f16 q0, q1, q4
+; CHECK-LV-NEXT:    vadd.f16 q0, q0, q3
+; CHECK-LV-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LV-NEXT:    vpop {d8, d9}
+; CHECK-LV-NEXT:    bx lr
+
+; CHECK-LIS-LABEL: vld3_v16f16:
+; CHECK-LIS:       @ %bb.0: @ %entry
+; CHECK-LIS-NEXT:    .vsave {d8, d9}
+; CHECK-LIS-NEXT:    vpush {d8, d9}
+; CHECK-LIS-NEXT:    vldrw.u32 q0, [r0, #48]
+; CHECK-LIS-NEXT:    vldrw.u32 q2, [r0, #64]
+; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #80]
+; CHECK-LIS-NEXT:    vmovx.f16 s6, s2
+; CHECK-LIS-NEXT:    vmov.f32 s4, s1
+; CHECK-LIS-NEXT:    vins.f16 s4, s6
+; CHECK-LIS-NEXT:    vmovx.f16 s6, s9
+; CHECK-LIS-NEXT:    vmov.f32 s5, s8
+; CHECK-LIS-NEXT:    vmovx.f16 s7, s12
+; CHECK-LIS-NEXT:    vins.f16 s5, s6
+; CHECK-LIS-NEXT:    vmov.f32 s6, s11
+; CHECK-LIS-NEXT:    vins.f16 s6, s7
+; CHECK-LIS-NEXT:    vmovx.f16 s16, s15
+; CHECK-LIS-NEXT:    vmov.f32 s7, s14
+; CHECK-LIS-NEXT:    vmovx.f16 s17, s3
+; CHECK-LIS-NEXT:    vins.f16 s7, s16
+; CHECK-LIS-NEXT:    vmovx.f16 s16, s0
+; CHECK-LIS-NEXT:    vins.f16 s16, s2
+; CHECK-LIS-NEXT:    vmovx.f16 s2, s1
+; CHECK-LIS-NEXT:    vins.f16 s0, s2
+; CHECK-LIS-NEXT:    vmovx.f16 s2, s8
+; CHECK-LIS-NEXT:    vins.f16 s3, s2
+; CHECK-LIS-NEXT:    vmovx.f16 s2, s11
+; CHECK-LIS-NEXT:    vmovx.f16 s18, s10
+; CHECK-LIS-NEXT:    vins.f16 s10, s2
+; CHECK-LIS-NEXT:    vmovx.f16 s2, s14
+; CHECK-LIS-NEXT:    vmovx.f16 s19, s13
+; CHECK-LIS-NEXT:    vins.f16 s13, s2
+; CHECK-LIS-NEXT:    vmov.f32 s1, s3
+; CHECK-LIS-NEXT:    vins.f16 s18, s12
+; CHECK-LIS-NEXT:    vins.f16 s19, s15
+; CHECK-LIS-NEXT:    vins.f16 s17, s9
+; CHECK-LIS-NEXT:    vmov.f32 s2, s10
+; CHECK-LIS-NEXT:    vmov.f32 s3, s13
+; CHECK-LIS-NEXT:    vldrw.u32 q2, [r0, #16]
+; CHECK-LIS-NEXT:    vadd.f16 q0, q0, q4
+; CHECK-LIS-NEXT:    vadd.f16 q3, q0, q1
+; CHECK-LIS-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-LIS-NEXT:    vldrw.u32 q1, [r0, #32]
+; CHECK-LIS-NEXT:    vstrw.32 q3, [r1, #16]
+; CHECK-LIS-NEXT:    vmovx.f16 s14, s2
+; CHECK-LIS-NEXT:    vmov.f32 s12, s1
+; CHECK-LIS-NEXT:    vins.f16 s12, s14
+; CHECK-LIS-NEXT:    vmovx.f16 s14, s9
+; CHECK-LIS-NEXT:    vmov.f32 s13, s8
+; CHECK-LIS-NEXT:    vmovx.f16 s15, s4
+; CHECK-LIS-NEXT:    vins.f16 s13, s14
+; CHECK-LIS-NEXT:    vmov.f32 s14, s11
+; CHECK-LIS-NEXT:    vins.f16 s14, s15
+; CHECK-LIS-NEXT:    vmovx.f16 s16, s7
+; CHECK-LIS-NEXT:    vmov.f32 s15, s6
+; CHECK-LIS-NEXT:    vmovx.f16 s17, s3
+; CHECK-LIS-NEXT:    vins.f16 s15, s16
+; CHECK-LIS-NEXT:    vmovx.f16 s16, s0
+; CHECK-LIS-NEXT:    vins.f16 s16, s2
+; CHECK-LIS-NEXT:    vmovx.f16 s2, s1
+; CHECK-LIS-NEXT:    vins.f16 s0, s2
+; CHECK-LIS-NEXT:    vmovx.f16 s2, s8
+; CHECK-LIS-NEXT:    vins.f16 s3, s2
+; CHECK-LIS-NEXT:    vmovx.f16 s2, s11
+; CHECK-LIS-NEXT:    vmovx.f16 s18, s10
+; CHECK-LIS-NEXT:    vins.f16 s10, s2
+; CHECK-LIS-NEXT:    vmovx.f16 s2, s6
+; CHECK-LIS-NEXT:    vmovx.f16 s19, s5
+; CHECK-LIS-NEXT:    vins.f16 s5, s2
+; CHECK-LIS-NEXT:    vmov.f32 s1, s3
+; CHECK-LIS-NEXT:    vins.f16 s18, s4
+; CHECK-LIS-NEXT:    vins.f16 s19, s7
+; CHECK-LIS-NEXT:    vins.f16 s17, s9
+; CHECK-LIS-NEXT:    vmov.f32 s2, s10
+; CHECK-LIS-NEXT:    vmov.f32 s3, s5
+; CHECK-LIS-NEXT:    vadd.f16 q0, q0, q4
+; CHECK-LIS-NEXT:    vadd.f16 q0, q0, q3
+; CHECK-LIS-NEXT:    vstrw.32 q0, [r1]
+; CHECK-LIS-NEXT:    vpop {d8, d9}
+; CHECK-LIS-NEXT:    bx lr
 entry:
   %l1 = load <48 x half>, ptr %src, align 4
   %s1 = shufflevector <48 x half> %l1, <48 x half> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
@@ -1436,4 +1841,4 @@ entry:
   %a = fadd <4 x double> %a1, %s3
   store <4 x double> %a, ptr %dst
   ret void
-}
+}
\ No newline at end of file
diff --git a/llvm/test/CodeGen/WebAssembly/exception-legacy.ll b/llvm/test/CodeGen/WebAssembly/exception-legacy.ll
index 3fe45bcc4cd29..b4ffd185e3ca8 100644
--- a/llvm/test/CodeGen/WebAssembly/exception-legacy.ll
+++ b/llvm/test/CodeGen/WebAssembly/exception-legacy.ll
@@ -171,7 +171,7 @@ invoke.cont2:                                     ; preds = %ehcleanup
 
 terminate:                                        ; preds = %ehcleanup
   %6 = cleanuppad within %5 []
-  call void @_ZSt9terminatev() [ "funclet"(token %6) ]
+  call void @_ZSt9terminatev() #2 [ "funclet"(token %6) ]
   unreachable
 }
 
@@ -477,7 +477,7 @@ invoke.cont.i:                                    ; preds = %catch.start.i
 
 terminate.i:                                      ; preds = %catch.start.i, %catch.dispatch.i
   %6 = cleanuppad within %0 []
-  call void @_ZSt9terminatev() [ "funclet"(token %6) ]
+  call void @_ZSt9terminatev() #2 [ "funclet"(token %6) ]
   unreachable
 
 _ZN4TempD2Ev.exit:                                ; preds = %invoke.cont.i
@@ -501,6 +501,50 @@ unreachable:                                      ; preds = %entry
   unreachable
 }
 
+; Regression test for a bug where, when an invoke unwinds to a catchswitch, the
+; catchswitch's unwind destination was not included in the invoke's unwind
+; destination when there was no direct link from catch.start to there.
+
+; CHECK-LABEL: unwind_destinations:
+; CHECK: try
+; CHECK:   try
+; CHECK:     call  foo
+; CHECK:   catch  $0=, __cpp_exception
+; CHECK:     call  _ZSt9terminatev
+; CHECK:     unreachable
+; CHECK:   end_try
+; Note the below is "terminate" BB and should not be DCE'd
+; CHECK: catch_all
+; CHECK:   call  _ZSt9terminatev
+; CHECK:   unreachable
+; CHECK: end_try
+; CHECK: return
+define void @unwind_destinations() personality ptr @__gxx_wasm_personality_v0 {
+entry:
+  invoke void @foo()
+          to label %try.cont unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %entry
+  %0 = catchswitch within none [label %catch.start] unwind label %terminate
+
+catch.start:                                      ; preds = %catch.dispatch
+  %1 = catchpad within %0 [ptr null]
+  %2 = call ptr @llvm.wasm.get.exception(token %1)
+  %3 = call ptr @__cxa_begin_catch(ptr %2) #5 [ "funclet"(token %1) ]
+  call void @_ZSt9terminatev() #2 [ "funclet"(token %1) ]
+  unreachable
+
+; Even if there is no link from catch.start to this terminate BB, when there is
+; an exception that catch.start does not catch (e.g. a foreign exception), it
+; should end up here, so this BB should NOT be DCE'ed
+terminate:                                        ; preds = %catch.dispatch
+  %4 = cleanuppad within none []
+  call void @_ZSt9terminatev() #2 [ "funclet"(token %4) ]
+  unreachable
+
+try.cont:                                         ; preds = %entry
+  ret void
+}
 
 declare void @foo()
 declare void @bar(ptr)
@@ -527,5 +571,6 @@ declare ptr @_ZN4TempD2Ev(ptr returned)
 
 attributes #0 = { nounwind }
 attributes #1 = { noreturn }
+attributes #2 = { noreturn nounwind }
 
 ; CHECK: __cpp_exception:
diff --git a/llvm/test/CodeGen/WebAssembly/exception.ll b/llvm/test/CodeGen/WebAssembly/exception.ll
index 57d1f37c0039f..fc42f050a687e 100644
--- a/llvm/test/CodeGen/WebAssembly/exception.ll
+++ b/llvm/test/CodeGen/WebAssembly/exception.ll
@@ -207,7 +207,7 @@ invoke.cont2:                                     ; preds = %ehcleanup
 
 terminate:                                        ; preds = %ehcleanup
   %6 = cleanuppad within %5 []
-  call void @_ZSt9terminatev() [ "funclet"(token %6) ]
+  call void @_ZSt9terminatev() #2 [ "funclet"(token %6) ]
   unreachable
 }
 
@@ -542,7 +542,7 @@ invoke.cont.i:                                    ; preds = %catch.start.i
 
 terminate.i:                                      ; preds = %catch.start.i, %catch.dispatch.i
   %6 = cleanuppad within %0 []
-  call void @_ZSt9terminatev() [ "funclet"(token %6) ]
+  call void @_ZSt9terminatev() #2 [ "funclet"(token %6) ]
   unreachable
 
 _ZN4TempD2Ev.exit:                                ; preds = %invoke.cont.i
@@ -593,6 +593,56 @@ try.cont:                                         ; preds = %catch, %entry
   ret void
 }
 
+; Regression test for a bug where, when an invoke unwinds to a catchswitch, the
+; catchswitch's unwind destination was not included in the invoke's unwind
+; destination when there was no direct link from catch.start to there.
+
+; CHECK-LABEL: unwind_destinations:
+; CHECK: block
+; CHECK:   block
+; CHECK:     try_table    (catch_all 0)
+; CHECK:       block     i32
+; CHECK:         try_table    (catch __cpp_exception 0)
+; CHECK:           call  foo
+; CHECK:         end_try_table
+; CHECK:         unreachable
+; CHECK:       end_block
+; CHECK:       call  _ZSt9terminatev
+; CHECK:       unreachable
+; CHECK:     end_try_table
+; CHECK:     unreachable
+; CHECK:   end_block
+; Note the below is "terminate" BB and should not be DCE'd
+; CHECK:   call  _ZSt9terminatev
+; CHECK:   unreachable
+; CHECK: end_block
+define void @unwind_destinations() personality ptr @__gxx_wasm_personality_v0 {
+entry:
+  invoke void @foo()
+          to label %try.cont unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %entry
+  %0 = catchswitch within none [label %catch.start] unwind label %terminate
+
+catch.start:                                      ; preds = %catch.dispatch
+  %1 = catchpad within %0 [ptr null]
+  %2 = call ptr @llvm.wasm.get.exception(token %1)
+  %3 = call ptr @__cxa_begin_catch(ptr %2) #5 [ "funclet"(token %1) ]
+  call void @_ZSt9terminatev() #2 [ "funclet"(token %1) ]
+  unreachable
+
+; Even if there is no link from catch.start to this terminate BB, when there is
+; an exception that catch.start does not catch (e.g. a foreign exception), it
+; should end up here, so this BB should NOT be DCE'ed
+terminate:                                        ; preds = %catch.dispatch
+  %4 = cleanuppad within none []
+  call void @_ZSt9terminatev() #2 [ "funclet"(token %4) ]
+  unreachable
+
+try.cont:                                         ; preds = %entry
+  ret void
+}
+
 declare void @foo()
 declare void @bar(ptr)
 declare void @take_i32(i32)
@@ -618,5 +668,6 @@ declare ptr @_ZN4TempD2Ev(ptr returned)
 
 attributes #0 = { nounwind }
 attributes #1 = { noreturn }
+attributes #2 = { noreturn nounwind }
 
 ; CHECK: __cpp_exception:
diff --git a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
index 1102044038afe..2dfa515d9f05c 100644
--- a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
+++ b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
@@ -3519,9 +3519,7 @@ define void @vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12(ptr %i
 ; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512BW-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} xmm1 = [0,25,0,27,0,29,0,31]
-; AVX512BW-NEXT:    vpermw %zmm0, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpbroadcastw %xmm0, %ymm0
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpermw %zmm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)
 ; AVX512BW-NEXT:    vzeroupper
@@ -3651,9 +3649,7 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
 ; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512BW-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} xmm1 = [0,25,26,0,28,29,0,31]
-; AVX512BW-NEXT:    vpermw %zmm0, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpbroadcastw %xmm0, %xmm0
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpermw %zmm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)
 ; AVX512BW-NEXT:    vzeroupper
@@ -3776,9 +3772,7 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ; AVX512BW-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,25,26,27,0,29,30,31,0,25,26,27,0,29,30,31]
 ; AVX512BW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermw %zmm0, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpbroadcastw %xmm0, %ymm0
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpermw %zmm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)
 ; AVX512BW-NEXT:    vzeroupper
@@ -3909,9 +3903,7 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
 ; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512BW-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} xmm1 = [0,25,26,27,28,29,0,31]
-; AVX512BW-NEXT:    vpermw %zmm0, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpbroadcastw %xmm0, %ymm0
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpermw %zmm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)
 ; AVX512BW-NEXT:    vzeroupper
@@ -4401,10 +4393,8 @@ define void @vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4(ptr %in.
 ; AVX512BW-SLOW:       # %bb.0:
 ; AVX512BW-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512BW-SLOW-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-SLOW-NEXT:    vpmovsxbd {{.*#+}} xmm1 = [0,13,14,0]
-; AVX512BW-SLOW-NEXT:    vpermd %zmm0, %zmm1, %zmm1
-; AVX512BW-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
-; AVX512BW-SLOW-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm0
+; AVX512BW-SLOW-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,13,14,0,0,0,0,0,0,0,1,1,0,0,0,0]
+; AVX512BW-SLOW-NEXT:    vpermd %zmm0, %zmm1, %zmm0
 ; AVX512BW-SLOW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
 ; AVX512BW-SLOW-NEXT:    vmovdqa64 %zmm0, (%rcx)
 ; AVX512BW-SLOW-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/apx/ccmp.ll b/llvm/test/CodeGen/X86/apx/ccmp.ll
index 7bd8aeea8863b..e2e4e8df93149 100644
--- a/llvm/test/CodeGen/X86/apx/ccmp.ll
+++ b/llvm/test/CodeGen/X86/apx/ccmp.ll
@@ -1300,5 +1300,75 @@ if.end:                                           ; preds = %entry, %if.then
   ret void
 }
 
+define void @ccmp_continous_adjust_imm(i32 noundef %a, i32 noundef %b) {
+; CHECK-LABEL: ccmp_continous_adjust_imm:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl $2, %edi # encoding: [0x83,0xff,0x02]
+; CHECK-NEXT:    ccmpll {dfv=} $2, %esi # encoding: [0x62,0xf4,0x04,0x0c,0x83,0xfe,0x02]
+; CHECK-NEXT:    jg .LBB31_1 # encoding: [0x7f,A]
+; CHECK-NEXT:    # fixup A - offset: 1, value: .LBB31_1-1, kind: FK_PCRel_1
+; CHECK-NEXT:  # %bb.2: # %if.then
+; CHECK-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
+; CHECK-NEXT:    jmp foo # TAILCALL
+; CHECK-NEXT:    # encoding: [0xeb,A]
+; CHECK-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK-NEXT:  .LBB31_1: # %if.end
+; CHECK-NEXT:    retq # encoding: [0xc3]
+;
+; NDD-LABEL: ccmp_continous_adjust_imm:
+; NDD:       # %bb.0: # %entry
+; NDD-NEXT:    cmpl $2, %edi # encoding: [0x83,0xff,0x02]
+; NDD-NEXT:    ccmpll {dfv=} $2, %esi # encoding: [0x62,0xf4,0x04,0x0c,0x83,0xfe,0x02]
+; NDD-NEXT:    jg .LBB31_1 # encoding: [0x7f,A]
+; NDD-NEXT:    # fixup A - offset: 1, value: .LBB31_1-1, kind: FK_PCRel_1
+; NDD-NEXT:  # %bb.2: # %if.then
+; NDD-NEXT:    xorl %eax, %eax # encoding: [0x31,0xc0]
+; NDD-NEXT:    jmp foo # TAILCALL
+; NDD-NEXT:    # encoding: [0xeb,A]
+; NDD-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; NDD-NEXT:  .LBB31_1: # %if.end
+; NDD-NEXT:    retq # encoding: [0xc3]
+entry:
+  %cmp = icmp slt i32 %a, 2
+  %cmp1 = icmp slt i32 %b, 2
+  %or.cond = and i1 %cmp, %cmp1
+  %cmp3 = icmp slt i32 %b, 3
+  %or.cond4 = and i1 %or.cond, %cmp3
+  br i1 %or.cond4, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void (...) @foo()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
+
+define i32 @ccmp_continous_nobranch_adjust_imm(i32 noundef %a, i32 noundef %b) {
+; CHECK-LABEL: ccmp_continous_nobranch_adjust_imm:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl $2, %esi # encoding: [0x83,0xfe,0x02]
+; CHECK-NEXT:    ccmpgl {dfv=} $2, %edi # encoding: [0x62,0xf4,0x04,0x0f,0x83,0xff,0x02]
+; CHECK-NEXT:    setge %al # encoding: [0x0f,0x9d,0xc0]
+; CHECK-NEXT:    movzbl %al, %eax # encoding: [0x0f,0xb6,0xc0]
+; CHECK-NEXT:    retq # encoding: [0xc3]
+;
+; NDD-LABEL: ccmp_continous_nobranch_adjust_imm:
+; NDD:       # %bb.0: # %entry
+; NDD-NEXT:    cmpl $2, %esi # encoding: [0x83,0xfe,0x02]
+; NDD-NEXT:    ccmpgl {dfv=} $2, %edi # encoding: [0x62,0xf4,0x04,0x0f,0x83,0xff,0x02]
+; NDD-NEXT:    setge %al # encoding: [0x0f,0x9d,0xc0]
+; NDD-NEXT:    movzbl %al, %eax # encoding: [0x0f,0xb6,0xc0]
+; NDD-NEXT:    retq # encoding: [0xc3]
+entry:
+  %cmp = icmp sgt i32 %a, 1
+  %cmp1 = icmp slt i32 %b, 2
+  %cmp2 = icmp slt i32 %b, 3
+  %or1 = or i1 %cmp, %cmp1
+  %or2 = or i1 %or1, %cmp2
+  %. = zext i1 %or2 to i32
+  ret i32 %.
+}
+
 declare dso_local void @foo(...)
 declare {i64, i1} @llvm.ssub.with.overflow.i64(i64, i64) nounwind readnone
diff --git a/llvm/test/CodeGen/X86/apx/cf.ll b/llvm/test/CodeGen/X86/apx/cf.ll
index c71d7768834f3..8d104e5f3ced2 100644
--- a/llvm/test/CodeGen/X86/apx/cf.ll
+++ b/llvm/test/CodeGen/X86/apx/cf.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64 -mattr=+cf,+avx512f -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64 -mattr=+cf,+nf,+avx512f -verify-machineinstrs | FileCheck %s
 
 define void @basic(i32 %a, ptr %b, ptr %p, ptr %q) {
 ; CHECK-LABEL: basic:
@@ -124,3 +124,37 @@ entry:
   call void @llvm.masked.store.v4i64.p0(<4 x i64> %0, ptr %p, i32 8, <4 x i1> %cond2)
   ret void
 }
+
+define void @single_cmp(i32 %a, i32 %b, ptr %c, ptr %d) {
+; CHECK-LABEL: single_cmp:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl %esi, %edi
+; CHECK-NEXT:    cfcmovnew (%rdx), %ax
+; CHECK-NEXT:    cfcmovnew %ax, (%rcx)
+; CHECK-NEXT:    retq
+entry:
+  %0 = icmp ne i32 %a, %b
+  %1 = insertelement <1 x i1> poison, i1 %0, i64 0
+  %2 = tail call <1 x i16> @llvm.masked.load.v1i16.p0(ptr %c, i32 2, <1 x i1> %1, <1 x i16> poison)
+  tail call void @llvm.masked.store.v1i16.p0(<1 x i16> %2, ptr %d, i32 2, <1 x i1> %1)
+  ret void
+}
+
+define void @load_add_store(i32 %a, i32 %b, ptr %p) {
+; CHECK-LABEL: load_add_store:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl %esi, %edi
+; CHECK-NEXT:    cfcmovnew (%rdx), %ax
+; CHECK-NEXT:    {nf} incl %eax
+; CHECK-NEXT:    cfcmovnew %ax, (%rdx)
+; CHECK-NEXT:    retq
+entry:
+  %0 = icmp ne i32 %a, %b
+  %1 = insertelement <1 x i1> poison, i1 %0, i64 0
+  %2 = tail call <1 x i16> @llvm.masked.load.v1i16.p0(ptr %p, i32 2, <1 x i1> %1, <1 x i16> poison)
+  %3 = extractelement <1 x i16> %2, i64 0
+  %4 = add i16 %3, 1
+  %5 = insertelement <1 x i16> poison, i16 %4, i64 0
+  tail call void @llvm.masked.store.v1i16.p0(<1 x i16> %5, ptr %p, i32 2, <1 x i1> %1)
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/apx/optimize-compare-ccmp.mir b/llvm/test/CodeGen/X86/apx/optimize-compare-ccmp.mir
new file mode 100644
index 0000000000000..f766ec00cbc3c
--- /dev/null
+++ b/llvm/test/CodeGen/X86/apx/optimize-compare-ccmp.mir
@@ -0,0 +1,396 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -o - %s -mtriple=x86_64-- -run-pass peephole-opt | FileCheck %s
+
+---
+name: opt_redundant_flags_adjusted_imm_0
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: opt_redundant_flags_adjusted_imm_0
+    ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rsi
+    ; CHECK-NEXT: CCMP64ri32 [[COPY]], 1, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $cl = SETCCr 4, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 15, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 7, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 14, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 6, implicit $eflags
+    %0:gr64 = COPY $rsi
+    ; CCMP+SETCC   %0 == 1
+    CCMP64ri32 %0, 1, 2, 5, implicit-def $eflags, implicit $eflags
+    $cl = SETCCr 4, implicit $eflags
+    ; CCMP+SETCC   %0 >= 2; CCMP can be removed.
+    CCMP64ri32 %0, 2, 2, 5, implicit-def $eflags, implicit $eflags
+    ; %0 >=s 2  -->  %0 >s 1
+    $bl = SETCCr 13, implicit $eflags
+    ; %0 >=u 2  -->  %0 >u 1
+    $bl = SETCCr 3, implicit $eflags
+    ; %0 <s 2  -->  %0 <=s 1
+    $bl = SETCCr 12, implicit $eflags
+    ; %0 <u 2  -->  %0 <=u 1
+    $bl = SETCCr 2, implicit $eflags
+...
+---
+name: opt_redundant_flags_adjusted_imm_1
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: opt_redundant_flags_adjusted_imm_1
+    ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rsi
+    ; CHECK-NEXT: CCMP64ri32 [[COPY]], 42, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $cl = SETCCr 5, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 13, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 3, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 12, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 2, implicit $eflags
+    %0:gr64 = COPY $rsi
+    ; CCMP+SETCC   %0 != 42
+    CCMP64ri32 %0, 42, 2, 5, implicit-def $eflags, implicit $eflags
+    $cl = SETCCr 5, implicit $eflags
+    ; CCMP+SETCC   %0 > 41; CCMP can be removed.
+    CCMP64ri32 %0, 41, 2, 5, implicit-def $eflags, implicit $eflags
+    ; %0 >s 41  -->  %0 >=s 42
+    $bl = SETCCr 15, implicit $eflags
+    ; %0 >u 41  -->  %0 >=u 42
+    $bl = SETCCr 7, implicit $eflags
+    ; %0 <=s 41  -->  %0 <s 42
+    $bl = SETCCr 14, implicit $eflags
+    ; %0 <=u 41  -->  %0 <u 42
+    $bl = SETCCr 6, implicit $eflags
+...
+---
+name: opt_redundant_flags_adjusted_imm_2
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: opt_redundant_flags_adjusted_imm_2
+    ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rsi
+    ; CHECK-NEXT: CCMP64ri32 [[COPY]], 1, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $cl = SETCCr 4, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 15, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 7, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 14, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 6, implicit $eflags
+    %0:gr64 = COPY $rsi
+    %1:gr64 = MOV64ri 1
+    ; CCMP+SETCC   %0 == 1
+    ; CCMP64rr will be optimized to CCMP64ri32 in the peephole optimiztion pass
+    CCMP64rr %0, %1, 2, 5, implicit-def $eflags, implicit $eflags
+    $cl = SETCCr 4, implicit $eflags
+    ; CCMP+SETCC   %0 >= 2; CCMP can be removed.
+    CCMP64ri32 %0, 2, 2, 5, implicit-def $eflags, implicit $eflags
+    ; %0 >=s 2  -->  %0 >s 1
+    $bl = SETCCr 13, implicit $eflags
+    ; %0 >=u 2  -->  %0 >u 1
+    $bl = SETCCr 3, implicit $eflags
+    ; %0 <s 2  -->  %0 <=s 1
+    $bl = SETCCr 12, implicit $eflags
+    ; %0 <u 2  -->  %0 <=u 1
+    $bl = SETCCr 2, implicit $eflags
+...
+---
+name: opt_redundant_flags_adjusted_imm_3
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: opt_redundant_flags_adjusted_imm_3
+    ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rsi
+    ; CHECK-NEXT: CMP64ri32 [[COPY]], 1, implicit-def $eflags
+    ; CHECK-NEXT: $cl = SETCCr 4, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 15, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 7, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 14, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 6, implicit $eflags
+    %0:gr64 = COPY $rsi
+    ; CMP+SETCC   %0 == 1
+    CMP64ri32 %0, 1, implicit-def $eflags
+    $cl = SETCCr 4, implicit $eflags
+    ; CCMP+SETCC   %0 >= 2; CCMP can be removed.
+    CCMP64ri32 %0, 2, 2, 5, implicit-def $eflags, implicit $eflags
+    ; %0 >=s 2  -->  %0 >s 1
+    $bl = SETCCr 13, implicit $eflags
+    ; %0 >=u 2  -->  %0 >u 1
+    $bl = SETCCr 3, implicit $eflags
+    ; %0 <s 2  -->  %0 <=s 1
+    $bl = SETCCr 12, implicit $eflags
+    ; %0 <u 2  -->  %0 <=u 1
+    $bl = SETCCr 2, implicit $eflags
+...
+---
+name: opt_redundant_flags_adjusted_imm_noopt_0
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: opt_redundant_flags_adjusted_imm_noopt_0
+    ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rsi
+    ; CHECK-NEXT: CCMP64ri32 [[COPY]], 42, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $cl = SETCCr 4, implicit $eflags
+    ; CHECK-NEXT: CCMP64ri32 [[COPY]], 41, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 4, implicit $eflags
+    %0:gr64 = COPY $rsi
+    ; CCMP+SETCC   %0 <s 1
+    CCMP64ri32 %0, 42, 2, 5, implicit-def $eflags, implicit $eflags
+    $cl = SETCCr 4, implicit $eflags
+    ; CCMP should not be removed.
+    CCMP64ri32 %0, 41, 2, 5, implicit-def $eflags, implicit $eflags
+    ; %0 == 41
+    $bl = SETCCr 4, implicit $eflags
+...
+---
+name: opt_redundant_flags_adjusted_imm_noopt_1
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: opt_redundant_flags_adjusted_imm_noopt_1
+    ; CHECK: [[COPY:%[0-9]+]]:gr32 = COPY $esi
+    ; CHECK-NEXT: CCMP32ri [[COPY]], 2147483647, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: CCMP32ri [[COPY]], -2147483648, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 12, implicit $eflags
+    ; CHECK-NEXT: CCMP32ri [[COPY]], 4294967295, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: CCMP32ri [[COPY]], -2147483648, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 12, implicit $eflags
+    ; CHECK-NEXT: CCMP32ri [[COPY]], 2147483647, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: CCMP32ri [[COPY]], -2147483648, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 13, implicit $eflags
+    ; CHECK-NEXT: CCMP32ri [[COPY]], 4294967295, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: CCMP32ri [[COPY]], 0, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 2, implicit $eflags
+    ; CHECK-NEXT: CCMP32ri [[COPY]], 4294967295, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: CCMP32ri [[COPY]], 0, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 3, implicit $eflags
+    %0:gr32 = COPY $esi
+    ; CCMP+SETCC   %0 == INT32_MAX
+    CCMP32ri %0, 2147483647, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CCMP should not be removed.
+    CCMP32ri %0, -2147483648, 2, 5, implicit-def $eflags, implicit $eflags
+    ; %0 <s INT32_MIN
+    $bl = SETCCr 12, implicit $eflags
+
+    CCMP32ri %0, 4294967295, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CCMP should not be removed.
+    CCMP32ri %0, -2147483648, 2, 5, implicit-def $eflags, implicit $eflags
+    $bl = SETCCr 12, implicit $eflags
+
+    CCMP32ri %0, 2147483647, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CCMP should not be removed.
+    CCMP32ri %0, -2147483648, 2, 5, implicit-def $eflags, implicit $eflags
+    $bl = SETCCr 13, implicit $eflags
+
+    CCMP32ri %0, 4294967295, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CCMP should not be removed.
+    CCMP32ri %0, 0, 2, 5, implicit-def $eflags, implicit $eflags
+    $bl = SETCCr 2, implicit $eflags
+
+    CCMP32ri %0, 4294967295, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CCMP should not be removed.
+    CCMP32ri %0, 0, 2, 5, implicit-def $eflags, implicit $eflags
+    $bl = SETCCr 3, implicit $eflags
+...
+---
+name: opt_redundant_flags_adjusted_imm_noopt_2
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: opt_redundant_flags_adjusted_imm_noopt_2
+    ; CHECK: [[COPY:%[0-9]+]]:gr16 = COPY $cx
+    ; CHECK-NEXT: CCMP16ri [[COPY]], -32768, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: CCMP16ri [[COPY]], 32767, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 15, implicit $eflags
+    ; CHECK-NEXT: CCMP16ri [[COPY]], 65535, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: CCMP16ri [[COPY]], 32767, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 15, implicit $eflags
+    ; CHECK-NEXT: CCMP16ri [[COPY]], -32768, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: CCMP16ri [[COPY]], 32767, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 14, implicit $eflags
+    ; CHECK-NEXT: CCMP16ri [[COPY]], 0, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: CCMP16ri [[COPY]], 65535, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 4, implicit $eflags
+    ; CHECK-NEXT: CCMP16ri [[COPY]], 0, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: CCMP16ri [[COPY]], 65535, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 6, implicit $eflags
+    %0:gr16 = COPY $cx
+    ; CCMP+SETCC   %0 == INT16_MIN
+    CCMP16ri %0, -32768, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CCMP should not be removed.
+    CCMP16ri %0, 32767, 2, 5, implicit-def $eflags, implicit $eflags
+    ; %0 >s INT16_MAX
+    $bl = SETCCr 15, implicit $eflags
+
+    CCMP16ri %0, 65535, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CCMP should not be removed.
+    CCMP16ri %0, 32767, 2, 5, implicit-def $eflags, implicit $eflags
+    $bl = SETCCr 15, implicit $eflags
+
+    CCMP16ri %0, -32768, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CCMP should not be removed.
+    CCMP16ri %0, 32767, 2, 5, implicit-def $eflags, implicit $eflags
+    $bl = SETCCr 14, implicit $eflags
+
+    CCMP16ri %0, 0, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CCMP should not be removed.
+    CCMP16ri %0, 65535, 2, 5, implicit-def $eflags, implicit $eflags
+    $bl = SETCCr 4, implicit $eflags
+
+    CCMP16ri %0, 0, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CCMP should not be removed.
+    CCMP16ri %0, 65535, 2, 5, implicit-def $eflags, implicit $eflags
+    $bl = SETCCr 6, implicit $eflags
+...
+---
+name: opt_redundant_flags_adjusted_imm_noopt_3
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: opt_redundant_flags_adjusted_imm_noopt_3
+    ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rsi
+    ; CHECK-NEXT: CCMP64ri32 [[COPY]], 1, 2, 7, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $cl = SETCCr 4, implicit $eflags, implicit $eflags
+    ; CHECK-NEXT: CCMP64ri32 [[COPY]], 2, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 13, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 3, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 12, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 2, implicit $eflags
+    %0:gr64 = COPY $rsi
+    ; CCMP+SETCC   %0 == 1
+    CCMP64ri32 %0, 1, 2, 7, implicit-def $eflags, implicit $eflags
+    $cl = SETCCr 4, implicit $eflags, implicit $eflags
+    ; CCMP+SETCC   %0 >= 2; CCMP should not be removed as the scc and dfv is
+    ; different.
+    CCMP64ri32 %0, 2, 2, 5, implicit-def $eflags, implicit $eflags
+    $bl = SETCCr 13, implicit $eflags
+    $bl = SETCCr 3, implicit $eflags
+    $bl = SETCCr 12, implicit $eflags
+    $bl = SETCCr 2, implicit $eflags
+...
+---
+name: opt_redundant_flags_adjusted_imm_noopt_4
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: opt_redundant_flags_adjusted_imm_noopt_4
+    ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rsi
+    ; CHECK-NEXT: CCMP64ri32 [[COPY]], 1, 5, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $cl = SETCCr 4, implicit $eflags, implicit $eflags
+    ; CHECK-NEXT: CCMP64ri32 [[COPY]], 2, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 13, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 3, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 12, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 2, implicit $eflags
+    %0:gr64 = COPY $rsi
+    ; CCMP+SETCC   %0 == 1
+    CCMP64ri32 %0, 1, 5, 5, implicit-def $eflags, implicit $eflags
+    $cl = SETCCr 4, implicit $eflags, implicit $eflags
+    ; CCMP+SETCC   %0 >= 2; CCMP should not be removed as the scc and dfv is
+    ; different.
+    CCMP64ri32 %0, 2, 2, 5, implicit-def $eflags, implicit $eflags
+    $bl = SETCCr 13, implicit $eflags
+    $bl = SETCCr 3, implicit $eflags
+    $bl = SETCCr 12, implicit $eflags
+    $bl = SETCCr 2, implicit $eflags
+...
+---
+name: opt_redundant_flags_adjusted_imm_noopt_5
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: opt_redundant_flags_adjusted_imm_noopt_5
+    ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rsi
+    ; CHECK-NEXT: CCMP64ri32 [[COPY]], 1, 2, 5, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: $cl = SETCCr 4, implicit $eflags
+    ; CHECK-NEXT: CMP64ri32 [[COPY]], 2, implicit-def $eflags
+    ; CHECK-NEXT: $bl = SETCCr 13, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 3, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 12, implicit $eflags
+    ; CHECK-NEXT: $bl = SETCCr 2, implicit $eflags
+    %0:gr64 = COPY $rsi
+    ; CCMP+SETCC   %0 == 1
+    CCMP64ri32 %0, 1, 2, 5, implicit-def $eflags, implicit $eflags
+    $cl = SETCCr 4, implicit $eflags
+    ; CMP+SETCC   %0 >= 2; CMP cannot be removed.
+    CMP64ri32 %0, 2, implicit-def $eflags
+    ; %0 >=s 2  -->  %0 >s 1
+    $bl = SETCCr 13, implicit $eflags
+    ; %0 >=u 2  -->  %0 >u 1
+    $bl = SETCCr 3, implicit $eflags
+    ; %0 <s 2  -->  %0 <=s 1
+    $bl = SETCCr 12, implicit $eflags
+    ; %0 <u 2  -->  %0 <=u 1
+    $bl = SETCCr 2, implicit $eflags
+...
+---
+name: opt_adjusted_imm_multiple_blocks
+body: |
+  ; CHECK-LABEL: name: opt_adjusted_imm_multiple_blocks
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = COPY $eax
+  ; CHECK-NEXT:   CCMP32ri [[COPY]], 20, 2, 5, implicit-def $eflags, implicit $eflags
+  ; CHECK-NEXT:   JCC_1 %bb.1, 4, implicit $eflags
+  ; CHECK-NEXT:   JMP_1 %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; CHECK-NEXT:   liveins: $eflags
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   JCC_1 %bb.2, 15, implicit $eflags
+  ; CHECK-NEXT:   JMP_1 %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   JMP_1 %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   RET 0
+  bb.0:
+    %0:gr32 = COPY $eax
+    CCMP32ri %0, 20, 2, 5, implicit-def $eflags, implicit $eflags
+    JCC_1 %bb.1, 4, implicit $eflags
+    JMP_1 %bb.3
+
+  bb.1:
+    CCMP32ri %0, 21, 2, 5, implicit-def $eflags, implicit $eflags
+    JCC_1 %bb.2, 13, implicit $eflags
+    JMP_1 %bb.3
+
+  bb.2:
+    JMP_1 %bb.3
+
+  bb.3:
+    RET 0
+...
+---
+name: opt_adjusted_imm_multiple_blocks_noopt
+body: |
+  ; CHECK-LABEL: name: opt_adjusted_imm_multiple_blocks_noopt
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = COPY $eax
+  ; CHECK-NEXT:   CCMP32ri [[COPY]], 20, 2, 5, implicit-def $eflags, implicit $eflags
+  ; CHECK-NEXT:   JCC_1 %bb.1, 4, implicit $eflags
+  ; CHECK-NEXT:   JMP_1 %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   CCMP32ri [[COPY]], 21, 2, 5, implicit-def $eflags, implicit $eflags
+  ; CHECK-NEXT:   JCC_1 %bb.2, 13, implicit $eflags
+  ; CHECK-NEXT:   JMP_1 %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT:   liveins: $eflags
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   $al = SETCCr 4, implicit $eflags
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   RET 0
+  bb.0:
+    %0:gr32 = COPY $eax
+    CCMP32ri %0, 20, 2, 5, implicit-def $eflags, implicit $eflags
+    JCC_1 %bb.1, 4, implicit $eflags
+    JMP_1 %bb.3
+
+  bb.1:
+    CCMP32ri %0, 21, 2, 5, implicit-def $eflags, implicit $eflags
+    JCC_1 %bb.2, 13, implicit $eflags
+    JMP_1 %bb.3
+
+  bb.2:
+    liveins: $eflags
+    $al = SETCCr 4, implicit $eflags
+
+  bb.3:
+    RET 0
+...
diff --git a/llvm/test/CodeGen/X86/atomic-idempotent-syncscope.ll b/llvm/test/CodeGen/X86/atomic-idempotent-syncscope.ll
new file mode 100644
index 0000000000000..9e20fdb59f552
--- /dev/null
+++ b/llvm/test/CodeGen/X86/atomic-idempotent-syncscope.ll
@@ -0,0 +1,615 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs                           | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i686-- -verify-machineinstrs           -mattr=+sse2      | FileCheck %s --check-prefixes=X86,X86-GENERIC,X86-SSE2
+; RUN: llc < %s -mtriple=i686-- -verify-machineinstrs -mcpu=slm -mattr=-sse2      | FileCheck %s --check-prefixes=X86,X86-GENERIC,X86-SLM
+; RUN: llc < %s -mtriple=i686-- -verify-machineinstrs -mcpu=goldmont -mattr=-sse2 | FileCheck %s --check-prefixes=X86,X86-GENERIC,X86-SLM
+; RUN: llc < %s -mtriple=i686-- -verify-machineinstrs -mcpu=knl -mattr=-sse2      | FileCheck %s --check-prefixes=X86,X86-GENERIC,X86-SLM
+; RUN: llc < %s -mtriple=i686-- -verify-machineinstrs -mcpu=atom -mattr=-sse2     | FileCheck %s --check-prefixes=X86,X86-ATOM
+
+; On x86, an atomic rmw operation that does not modify the value in memory
+; (such as atomic add 0) can be replaced by an mfence followed by a mov.
+; This is explained (with the motivation for such an optimization) in
+; http://www.hpl.hp.com/techreports/2012/HPL-2012-68.pdf
+
+define i8 @add8(ptr %p) #0 {
+; X64-LABEL: add8:
+; X64:       # %bb.0:
+; X64-NEXT:    #MEMBARRIER
+; X64-NEXT:    movzbl (%rdi), %eax
+; X64-NEXT:    retq
+;
+; X86-GENERIC-LABEL: add8:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-GENERIC-NEXT:    #MEMBARRIER
+; X86-GENERIC-NEXT:    movzbl (%eax), %eax
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: add8:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-ATOM-NEXT:    #MEMBARRIER
+; X86-ATOM-NEXT:    movzbl (%eax), %eax
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
+  %1 = atomicrmw add ptr %p, i8 0 syncscope("singlethread") monotonic
+  ret i8 %1
+}
+
+define i16 @or16(ptr %p) #0 {
+; X64-LABEL: or16:
+; X64:       # %bb.0:
+; X64-NEXT:    #MEMBARRIER
+; X64-NEXT:    movzwl (%rdi), %eax
+; X64-NEXT:    retq
+;
+; X86-GENERIC-LABEL: or16:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-GENERIC-NEXT:    #MEMBARRIER
+; X86-GENERIC-NEXT:    movzwl (%eax), %eax
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: or16:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-ATOM-NEXT:    #MEMBARRIER
+; X86-ATOM-NEXT:    movzwl (%eax), %eax
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
+  %1 = atomicrmw or ptr %p, i16 0 syncscope("singlethread") acquire
+  ret i16 %1
+}
+
+define i32 @xor32(ptr %p) #0 {
+; X64-LABEL: xor32:
+; X64:       # %bb.0:
+; X64-NEXT:    #MEMBARRIER
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    retq
+;
+; X86-GENERIC-LABEL: xor32:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-GENERIC-NEXT:    #MEMBARRIER
+; X86-GENERIC-NEXT:    movl (%eax), %eax
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: xor32:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-ATOM-NEXT:    #MEMBARRIER
+; X86-ATOM-NEXT:    movl (%eax), %eax
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
+  %1 = atomicrmw xor ptr %p, i32 0 syncscope("singlethread") release
+  ret i32 %1
+}
+
+define i64 @sub64(ptr %p) #0 {
+; X64-LABEL: sub64:
+; X64:       # %bb.0:
+; X64-NEXT:    #MEMBARRIER
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: sub64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl (%esi), %eax
+; X86-NEXT:    movl 4(%esi), %edx
+; X86-NEXT:    .p2align 4
+; X86-NEXT:  .LBB3_1: # %atomicrmw.start
+; X86-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-NEXT:    movl %edx, %ecx
+; X86-NEXT:    movl %eax, %ebx
+; X86-NEXT:    lock cmpxchg8b (%esi)
+; X86-NEXT:    jne .LBB3_1
+; X86-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
+  %1 = atomicrmw sub ptr %p, i64 0 syncscope("singlethread") seq_cst
+  ret i64 %1
+}
+
+define i128 @or128(ptr %p) #0 {
+; X64-LABEL: or128:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    xorl %esi, %esi
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    xorl %ecx, %ecx
+; X64-NEXT:    callq __atomic_fetch_or_16@PLT
+; X64-NEXT:    popq %rcx
+; X64-NEXT:    retq
+;
+; X86-GENERIC-LABEL: or128:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    pushl %ebp
+; X86-GENERIC-NEXT:    movl %esp, %ebp
+; X86-GENERIC-NEXT:    pushl %ebx
+; X86-GENERIC-NEXT:    pushl %edi
+; X86-GENERIC-NEXT:    pushl %esi
+; X86-GENERIC-NEXT:    andl $-16, %esp
+; X86-GENERIC-NEXT:    subl $48, %esp
+; X86-GENERIC-NEXT:    movl 12(%ebp), %edi
+; X86-GENERIC-NEXT:    movl 12(%edi), %ecx
+; X86-GENERIC-NEXT:    movl 8(%edi), %edx
+; X86-GENERIC-NEXT:    movl (%edi), %ebx
+; X86-GENERIC-NEXT:    movl 4(%edi), %esi
+; X86-GENERIC-NEXT:    .p2align 4
+; X86-GENERIC-NEXT:  .LBB4_1: # %atomicrmw.start
+; X86-GENERIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-GENERIC-NEXT:    movl %ebx, (%esp)
+; X86-GENERIC-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-GENERIC-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-GENERIC-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-GENERIC-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-GENERIC-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-GENERIC-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-GENERIC-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-GENERIC-NEXT:    pushl $0
+; X86-GENERIC-NEXT:    pushl $0
+; X86-GENERIC-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-GENERIC-NEXT:    pushl %eax
+; X86-GENERIC-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-GENERIC-NEXT:    pushl %eax
+; X86-GENERIC-NEXT:    pushl %edi
+; X86-GENERIC-NEXT:    pushl $16
+; X86-GENERIC-NEXT:    calll __atomic_compare_exchange@PLT
+; X86-GENERIC-NEXT:    addl $24, %esp
+; X86-GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-GENERIC-NEXT:    movl (%esp), %ebx
+; X86-GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-GENERIC-NEXT:    testb %al, %al
+; X86-GENERIC-NEXT:    je .LBB4_1
+; X86-GENERIC-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-GENERIC-NEXT:    movl 8(%ebp), %eax
+; X86-GENERIC-NEXT:    movl %ebx, (%eax)
+; X86-GENERIC-NEXT:    movl %esi, 4(%eax)
+; X86-GENERIC-NEXT:    movl %edx, 8(%eax)
+; X86-GENERIC-NEXT:    movl %ecx, 12(%eax)
+; X86-GENERIC-NEXT:    leal -12(%ebp), %esp
+; X86-GENERIC-NEXT:    popl %esi
+; X86-GENERIC-NEXT:    popl %edi
+; X86-GENERIC-NEXT:    popl %ebx
+; X86-GENERIC-NEXT:    popl %ebp
+; X86-GENERIC-NEXT:    retl $4
+;
+; X86-ATOM-LABEL: or128:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    pushl %ebp
+; X86-ATOM-NEXT:    movl %esp, %ebp
+; X86-ATOM-NEXT:    pushl %ebx
+; X86-ATOM-NEXT:    pushl %edi
+; X86-ATOM-NEXT:    pushl %esi
+; X86-ATOM-NEXT:    andl $-16, %esp
+; X86-ATOM-NEXT:    leal -{{[0-9]+}}(%esp), %esp
+; X86-ATOM-NEXT:    movl 12(%ebp), %edi
+; X86-ATOM-NEXT:    movl 12(%edi), %ecx
+; X86-ATOM-NEXT:    movl 8(%edi), %edx
+; X86-ATOM-NEXT:    movl (%edi), %esi
+; X86-ATOM-NEXT:    movl 4(%edi), %ebx
+; X86-ATOM-NEXT:    .p2align 4
+; X86-ATOM-NEXT:  .LBB4_1: # %atomicrmw.start
+; X86-ATOM-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-ATOM-NEXT:    movl %esi, (%esp)
+; X86-ATOM-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-ATOM-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-ATOM-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-ATOM-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-ATOM-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-ATOM-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-ATOM-NEXT:    movl %esi, {{[0-9]+}}(%esp)
+; X86-ATOM-NEXT:    pushl $0
+; X86-ATOM-NEXT:    pushl $0
+; X86-ATOM-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-ATOM-NEXT:    pushl %eax
+; X86-ATOM-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-ATOM-NEXT:    pushl %eax
+; X86-ATOM-NEXT:    pushl %edi
+; X86-ATOM-NEXT:    pushl $16
+; X86-ATOM-NEXT:    calll __atomic_compare_exchange@PLT
+; X86-ATOM-NEXT:    leal {{[0-9]+}}(%esp), %esp
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-ATOM-NEXT:    testb %al, %al
+; X86-ATOM-NEXT:    movl (%esp), %esi
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-ATOM-NEXT:    je .LBB4_1
+; X86-ATOM-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-ATOM-NEXT:    movl 8(%ebp), %eax
+; X86-ATOM-NEXT:    movl %esi, (%eax)
+; X86-ATOM-NEXT:    movl %ebx, 4(%eax)
+; X86-ATOM-NEXT:    movl %edx, 8(%eax)
+; X86-ATOM-NEXT:    movl %ecx, 12(%eax)
+; X86-ATOM-NEXT:    leal -12(%ebp), %esp
+; X86-ATOM-NEXT:    popl %esi
+; X86-ATOM-NEXT:    popl %edi
+; X86-ATOM-NEXT:    popl %ebx
+; X86-ATOM-NEXT:    popl %ebp
+; X86-ATOM-NEXT:    retl $4
+  %1 = atomicrmw or ptr %p, i128 0 syncscope("singlethread") monotonic
+  ret i128 %1
+}
+
+; For 'and', the idempotent value is (-1)
+define i32 @and32 (ptr %p) #0 {
+; X64-LABEL: and32:
+; X64:       # %bb.0:
+; X64-NEXT:    #MEMBARRIER
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    retq
+;
+; X86-GENERIC-LABEL: and32:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-GENERIC-NEXT:    #MEMBARRIER
+; X86-GENERIC-NEXT:    movl (%eax), %eax
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: and32:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-ATOM-NEXT:    #MEMBARRIER
+; X86-ATOM-NEXT:    movl (%eax), %eax
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
+  %1 = atomicrmw and ptr %p, i32 -1 syncscope("singlethread") acq_rel
+  ret i32 %1
+}
+
+define void @or32_nouse_monotonic(ptr %p) #0 {
+; X64-LABEL: or32_nouse_monotonic:
+; X64:       # %bb.0:
+; X64-NEXT:    #MEMBARRIER
+; X64-NEXT:    retq
+;
+; X86-GENERIC-LABEL: or32_nouse_monotonic:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    #MEMBARRIER
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: or32_nouse_monotonic:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    #MEMBARRIER
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
+  atomicrmw or ptr %p, i32 0 syncscope("singlethread") monotonic
+  ret void
+}
+
+
+define void @or32_nouse_acquire(ptr %p) #0 {
+; X64-LABEL: or32_nouse_acquire:
+; X64:       # %bb.0:
+; X64-NEXT:    #MEMBARRIER
+; X64-NEXT:    retq
+;
+; X86-GENERIC-LABEL: or32_nouse_acquire:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    #MEMBARRIER
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: or32_nouse_acquire:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    #MEMBARRIER
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
+  atomicrmw or ptr %p, i32 0 acquire
+  ret void
+}
+
+define void @or32_nouse_release(ptr %p) #0 {
+; X64-LABEL: or32_nouse_release:
+; X64:       # %bb.0:
+; X64-NEXT:    #MEMBARRIER
+; X64-NEXT:    retq
+;
+; X86-GENERIC-LABEL: or32_nouse_release:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    #MEMBARRIER
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: or32_nouse_release:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    #MEMBARRIER
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
+  atomicrmw or ptr %p, i32 0 syncscope("singlethread") release
+  ret void
+}
+
+define void @or32_nouse_acq_rel(ptr %p) #0 {
+; X64-LABEL: or32_nouse_acq_rel:
+; X64:       # %bb.0:
+; X64-NEXT:    #MEMBARRIER
+; X64-NEXT:    retq
+;
+; X86-GENERIC-LABEL: or32_nouse_acq_rel:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    #MEMBARRIER
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: or32_nouse_acq_rel:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    #MEMBARRIER
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
+  atomicrmw or ptr %p, i32 0 syncscope("singlethread") acq_rel
+  ret void
+}
+
+define void @or32_nouse_seq_cst(ptr %p) #0 {
+; X64-LABEL: or32_nouse_seq_cst:
+; X64:       # %bb.0:
+; X64-NEXT:    #MEMBARRIER
+; X64-NEXT:    retq
+;
+; X86-GENERIC-LABEL: or32_nouse_seq_cst:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    #MEMBARRIER
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: or32_nouse_seq_cst:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    #MEMBARRIER
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
+  atomicrmw or ptr %p, i32 0 syncscope("singlethread") seq_cst
+  ret void
+}
+
+; TODO: The value isn't used on 32 bit, so the cmpxchg8b is unneeded
+define void @or64_nouse_seq_cst(ptr %p) #0 {
+; X64-LABEL: or64_nouse_seq_cst:
+; X64:       # %bb.0:
+; X64-NEXT:    #MEMBARRIER
+; X64-NEXT:    retq
+;
+; X86-LABEL: or64_nouse_seq_cst:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl (%esi), %eax
+; X86-NEXT:    movl 4(%esi), %edx
+; X86-NEXT:    .p2align 4
+; X86-NEXT:  .LBB11_1: # %atomicrmw.start
+; X86-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-NEXT:    movl %edx, %ecx
+; X86-NEXT:    movl %eax, %ebx
+; X86-NEXT:    lock cmpxchg8b (%esi)
+; X86-NEXT:    jne .LBB11_1
+; X86-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
+  atomicrmw or ptr %p, i64 0 syncscope("singlethread") seq_cst
+  ret void
+}
+
+; TODO: Don't need to lower as sync_and_fetch call
+define void @or128_nouse_seq_cst(ptr %p) #0 {
+; X64-LABEL: or128_nouse_seq_cst:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    xorl %esi, %esi
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    movl $5, %ecx
+; X64-NEXT:    callq __atomic_fetch_or_16@PLT
+; X64-NEXT:    popq %rax
+; X64-NEXT:    retq
+;
+; X86-GENERIC-LABEL: or128_nouse_seq_cst:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    pushl %ebp
+; X86-GENERIC-NEXT:    movl %esp, %ebp
+; X86-GENERIC-NEXT:    pushl %ebx
+; X86-GENERIC-NEXT:    pushl %edi
+; X86-GENERIC-NEXT:    pushl %esi
+; X86-GENERIC-NEXT:    andl $-16, %esp
+; X86-GENERIC-NEXT:    subl $48, %esp
+; X86-GENERIC-NEXT:    movl 8(%ebp), %esi
+; X86-GENERIC-NEXT:    movl 12(%esi), %ecx
+; X86-GENERIC-NEXT:    movl 8(%esi), %edi
+; X86-GENERIC-NEXT:    movl (%esi), %edx
+; X86-GENERIC-NEXT:    movl 4(%esi), %ebx
+; X86-GENERIC-NEXT:    .p2align 4
+; X86-GENERIC-NEXT:  .LBB12_1: # %atomicrmw.start
+; X86-GENERIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-GENERIC-NEXT:    movl %edx, (%esp)
+; X86-GENERIC-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-GENERIC-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-GENERIC-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-GENERIC-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-GENERIC-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-GENERIC-NEXT:    movl %ebx, {{[0-9]+}}(%esp)
+; X86-GENERIC-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-GENERIC-NEXT:    pushl $5
+; X86-GENERIC-NEXT:    pushl $5
+; X86-GENERIC-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-GENERIC-NEXT:    pushl %eax
+; X86-GENERIC-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-GENERIC-NEXT:    pushl %eax
+; X86-GENERIC-NEXT:    pushl %esi
+; X86-GENERIC-NEXT:    pushl $16
+; X86-GENERIC-NEXT:    calll __atomic_compare_exchange@PLT
+; X86-GENERIC-NEXT:    addl $24, %esp
+; X86-GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-GENERIC-NEXT:    movl (%esp), %edx
+; X86-GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-GENERIC-NEXT:    testb %al, %al
+; X86-GENERIC-NEXT:    je .LBB12_1
+; X86-GENERIC-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-GENERIC-NEXT:    leal -12(%ebp), %esp
+; X86-GENERIC-NEXT:    popl %esi
+; X86-GENERIC-NEXT:    popl %edi
+; X86-GENERIC-NEXT:    popl %ebx
+; X86-GENERIC-NEXT:    popl %ebp
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: or128_nouse_seq_cst:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    pushl %ebp
+; X86-ATOM-NEXT:    movl %esp, %ebp
+; X86-ATOM-NEXT:    pushl %ebx
+; X86-ATOM-NEXT:    pushl %edi
+; X86-ATOM-NEXT:    pushl %esi
+; X86-ATOM-NEXT:    andl $-16, %esp
+; X86-ATOM-NEXT:    leal -{{[0-9]+}}(%esp), %esp
+; X86-ATOM-NEXT:    movl 8(%ebp), %esi
+; X86-ATOM-NEXT:    movl %esp, %ebx
+; X86-ATOM-NEXT:    movl 12(%esi), %ecx
+; X86-ATOM-NEXT:    movl 8(%esi), %edx
+; X86-ATOM-NEXT:    movl (%esi), %eax
+; X86-ATOM-NEXT:    movl 4(%esi), %edi
+; X86-ATOM-NEXT:    .p2align 4
+; X86-ATOM-NEXT:  .LBB12_1: # %atomicrmw.start
+; X86-ATOM-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-ATOM-NEXT:    movl %eax, (%esp)
+; X86-ATOM-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-ATOM-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-ATOM-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-ATOM-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-ATOM-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; X86-ATOM-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-ATOM-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-ATOM-NEXT:    pushl $5
+; X86-ATOM-NEXT:    pushl $5
+; X86-ATOM-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-ATOM-NEXT:    pushl %eax
+; X86-ATOM-NEXT:    pushl %ebx
+; X86-ATOM-NEXT:    pushl %esi
+; X86-ATOM-NEXT:    pushl $16
+; X86-ATOM-NEXT:    calll __atomic_compare_exchange@PLT
+; X86-ATOM-NEXT:    leal {{[0-9]+}}(%esp), %esp
+; X86-ATOM-NEXT:    testb %al, %al
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-ATOM-NEXT:    movl (%esp), %eax
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-ATOM-NEXT:    je .LBB12_1
+; X86-ATOM-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-ATOM-NEXT:    leal -12(%ebp), %esp
+; X86-ATOM-NEXT:    popl %esi
+; X86-ATOM-NEXT:    popl %edi
+; X86-ATOM-NEXT:    popl %ebx
+; X86-ATOM-NEXT:    popl %ebp
+; X86-ATOM-NEXT:    retl
+  atomicrmw or ptr %p, i128 0 syncscope("singlethread") seq_cst
+  ret void
+}
+
+
+define void @or16_nouse_seq_cst(ptr %p) #0 {
+; X64-LABEL: or16_nouse_seq_cst:
+; X64:       # %bb.0:
+; X64-NEXT:    #MEMBARRIER
+; X64-NEXT:    retq
+;
+; X86-GENERIC-LABEL: or16_nouse_seq_cst:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    #MEMBARRIER
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: or16_nouse_seq_cst:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    #MEMBARRIER
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
+  atomicrmw or ptr %p, i16 0 syncscope("singlethread") seq_cst
+  ret void
+}
+
+define void @or8_nouse_seq_cst(ptr %p) #0 {
+; X64-LABEL: or8_nouse_seq_cst:
+; X64:       # %bb.0:
+; X64-NEXT:    #MEMBARRIER
+; X64-NEXT:    retq
+;
+; X86-GENERIC-LABEL: or8_nouse_seq_cst:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    #MEMBARRIER
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: or8_nouse_seq_cst:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    #MEMBARRIER
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
+  atomicrmw or ptr %p, i8 0 syncscope("singlethread") seq_cst
+  ret void
+}
+
+attributes #0 = { nounwind }
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; X86-SLM: {{.*}}
+; X86-SSE2: {{.*}}
diff --git a/llvm/test/CodeGen/X86/atomic-idempotent.ll b/llvm/test/CodeGen/X86/atomic-idempotent.ll
index 55b4d1af094f6..020f9eb793102 100644
--- a/llvm/test/CodeGen/X86/atomic-idempotent.ll
+++ b/llvm/test/CodeGen/X86/atomic-idempotent.ll
@@ -11,10 +11,10 @@
 ; This is explained (with the motivation for such an optimization) in
 ; http://www.hpl.hp.com/techreports/2012/HPL-2012-68.pdf
 
-define i8 @add8(ptr %p) {
+define i8 @add8(ptr %p) #0 {
 ; X64-LABEL: add8:
 ; X64:       # %bb.0:
-; X64-NEXT:    mfence
+; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    movzbl (%rdi), %eax
 ; X64-NEXT:    retq
 ;
@@ -27,18 +27,16 @@ define i8 @add8(ptr %p) {
 ;
 ; X86-SLM-LABEL: add8:
 ; X86-SLM:       # %bb.0:
-; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-SLM-NEXT:    xorl %eax, %eax
-; X86-SLM-NEXT:    lock xaddb %al, (%ecx)
-; X86-SLM-NEXT:    # kill: def $al killed $al killed $eax
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SLM-NEXT:    lock orl $0, (%esp)
+; X86-SLM-NEXT:    movzbl (%eax), %eax
 ; X86-SLM-NEXT:    retl
 ;
 ; X86-ATOM-LABEL: add8:
 ; X86-ATOM:       # %bb.0:
-; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-ATOM-NEXT:    xorl %eax, %eax
-; X86-ATOM-NEXT:    lock xaddb %al, (%ecx)
-; X86-ATOM-NEXT:    # kill: def $al killed $al killed $eax
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-ATOM-NEXT:    lock orl $0, (%esp)
+; X86-ATOM-NEXT:    movzbl (%eax), %eax
 ; X86-ATOM-NEXT:    nop
 ; X86-ATOM-NEXT:    nop
 ; X86-ATOM-NEXT:    retl
@@ -46,10 +44,10 @@ define i8 @add8(ptr %p) {
   ret i8 %1
 }
 
-define i16 @or16(ptr %p) {
+define i16 @or16(ptr %p) #0 {
 ; X64-LABEL: or16:
 ; X64:       # %bb.0:
-; X64-NEXT:    mfence
+; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    movzwl (%rdi), %eax
 ; X64-NEXT:    retq
 ;
@@ -62,35 +60,27 @@ define i16 @or16(ptr %p) {
 ;
 ; X86-SLM-LABEL: or16:
 ; X86-SLM:       # %bb.0:
-; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-SLM-NEXT:    movzwl (%ecx), %eax
-; X86-SLM-NEXT:    .p2align 4
-; X86-SLM-NEXT:  .LBB1_1: # %atomicrmw.start
-; X86-SLM-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-SLM-NEXT:    lock cmpxchgw %ax, (%ecx)
-; X86-SLM-NEXT:    jne .LBB1_1
-; X86-SLM-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SLM-NEXT:    lock orl $0, (%esp)
+; X86-SLM-NEXT:    movzwl (%eax), %eax
 ; X86-SLM-NEXT:    retl
 ;
 ; X86-ATOM-LABEL: or16:
 ; X86-ATOM:       # %bb.0:
-; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-ATOM-NEXT:    movzwl (%ecx), %eax
-; X86-ATOM-NEXT:    .p2align 4
-; X86-ATOM-NEXT:  .LBB1_1: # %atomicrmw.start
-; X86-ATOM-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-ATOM-NEXT:    lock cmpxchgw %ax, (%ecx)
-; X86-ATOM-NEXT:    jne .LBB1_1
-; X86-ATOM-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-ATOM-NEXT:    lock orl $0, (%esp)
+; X86-ATOM-NEXT:    movzwl (%eax), %eax
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
 ; X86-ATOM-NEXT:    retl
   %1 = atomicrmw or ptr %p, i16 0 acquire
   ret i16 %1
 }
 
-define i32 @xor32(ptr %p) {
+define i32 @xor32(ptr %p) #0 {
 ; X64-LABEL: xor32:
 ; X64:       # %bb.0:
-; X64-NEXT:    mfence
+; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    movl (%rdi), %eax
 ; X64-NEXT:    retq
 ;
@@ -103,46 +93,34 @@ define i32 @xor32(ptr %p) {
 ;
 ; X86-SLM-LABEL: xor32:
 ; X86-SLM:       # %bb.0:
-; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-SLM-NEXT:    movl (%ecx), %eax
-; X86-SLM-NEXT:    .p2align 4
-; X86-SLM-NEXT:  .LBB2_1: # %atomicrmw.start
-; X86-SLM-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-SLM-NEXT:    lock cmpxchgl %eax, (%ecx)
-; X86-SLM-NEXT:    jne .LBB2_1
-; X86-SLM-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SLM-NEXT:    lock orl $0, (%esp)
+; X86-SLM-NEXT:    movl (%eax), %eax
 ; X86-SLM-NEXT:    retl
 ;
 ; X86-ATOM-LABEL: xor32:
 ; X86-ATOM:       # %bb.0:
-; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-ATOM-NEXT:    movl (%ecx), %eax
-; X86-ATOM-NEXT:    .p2align 4
-; X86-ATOM-NEXT:  .LBB2_1: # %atomicrmw.start
-; X86-ATOM-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-ATOM-NEXT:    lock cmpxchgl %eax, (%ecx)
-; X86-ATOM-NEXT:    jne .LBB2_1
-; X86-ATOM-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-ATOM-NEXT:    lock orl $0, (%esp)
+; X86-ATOM-NEXT:    movl (%eax), %eax
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
 ; X86-ATOM-NEXT:    retl
   %1 = atomicrmw xor ptr %p, i32 0 release
   ret i32 %1
 }
 
-define i64 @sub64(ptr %p) {
+define i64 @sub64(ptr %p) #0 {
 ; X64-LABEL: sub64:
 ; X64:       # %bb.0:
-; X64-NEXT:    mfence
+; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    movq (%rdi), %rax
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: sub64:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %ebx
-; X86-NEXT:    .cfi_def_cfa_offset 8
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    .cfi_def_cfa_offset 12
-; X86-NEXT:    .cfi_offset %esi, -12
-; X86-NEXT:    .cfi_offset %ebx, -8
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    movl (%esi), %eax
 ; X86-NEXT:    movl 4(%esi), %edx
@@ -155,42 +133,32 @@ define i64 @sub64(ptr %p) {
 ; X86-NEXT:    jne .LBB3_1
 ; X86-NEXT:  # %bb.2: # %atomicrmw.end
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    .cfi_def_cfa_offset 8
 ; X86-NEXT:    popl %ebx
-; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl
   %1 = atomicrmw sub ptr %p, i64 0 seq_cst
   ret i64 %1
 }
 
-define i128 @or128(ptr %p) {
+define i128 @or128(ptr %p) #0 {
 ; X64-LABEL: or128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    pushq %rax
-; X64-NEXT:    .cfi_def_cfa_offset 16
 ; X64-NEXT:    xorl %esi, %esi
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    xorl %ecx, %ecx
 ; X64-NEXT:    callq __atomic_fetch_or_16@PLT
 ; X64-NEXT:    popq %rcx
-; X64-NEXT:    .cfi_def_cfa_offset 8
 ; X64-NEXT:    retq
 ;
 ; X86-GENERIC-LABEL: or128:
 ; X86-GENERIC:       # %bb.0:
 ; X86-GENERIC-NEXT:    pushl %ebp
-; X86-GENERIC-NEXT:    .cfi_def_cfa_offset 8
-; X86-GENERIC-NEXT:    .cfi_offset %ebp, -8
 ; X86-GENERIC-NEXT:    movl %esp, %ebp
-; X86-GENERIC-NEXT:    .cfi_def_cfa_register %ebp
 ; X86-GENERIC-NEXT:    pushl %ebx
 ; X86-GENERIC-NEXT:    pushl %edi
 ; X86-GENERIC-NEXT:    pushl %esi
 ; X86-GENERIC-NEXT:    andl $-16, %esp
 ; X86-GENERIC-NEXT:    subl $48, %esp
-; X86-GENERIC-NEXT:    .cfi_offset %esi, -20
-; X86-GENERIC-NEXT:    .cfi_offset %edi, -16
-; X86-GENERIC-NEXT:    .cfi_offset %ebx, -12
 ; X86-GENERIC-NEXT:    movl 12(%ebp), %edi
 ; X86-GENERIC-NEXT:    movl 12(%edi), %ecx
 ; X86-GENERIC-NEXT:    movl 8(%edi), %edx
@@ -234,24 +202,17 @@ define i128 @or128(ptr %p) {
 ; X86-GENERIC-NEXT:    popl %edi
 ; X86-GENERIC-NEXT:    popl %ebx
 ; X86-GENERIC-NEXT:    popl %ebp
-; X86-GENERIC-NEXT:    .cfi_def_cfa %esp, 4
 ; X86-GENERIC-NEXT:    retl $4
 ;
 ; X86-ATOM-LABEL: or128:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    pushl %ebp
-; X86-ATOM-NEXT:    .cfi_def_cfa_offset 8
-; X86-ATOM-NEXT:    .cfi_offset %ebp, -8
 ; X86-ATOM-NEXT:    movl %esp, %ebp
-; X86-ATOM-NEXT:    .cfi_def_cfa_register %ebp
 ; X86-ATOM-NEXT:    pushl %ebx
 ; X86-ATOM-NEXT:    pushl %edi
 ; X86-ATOM-NEXT:    pushl %esi
 ; X86-ATOM-NEXT:    andl $-16, %esp
 ; X86-ATOM-NEXT:    leal -{{[0-9]+}}(%esp), %esp
-; X86-ATOM-NEXT:    .cfi_offset %esi, -20
-; X86-ATOM-NEXT:    .cfi_offset %edi, -16
-; X86-ATOM-NEXT:    .cfi_offset %ebx, -12
 ; X86-ATOM-NEXT:    movl 12(%ebp), %edi
 ; X86-ATOM-NEXT:    movl 12(%edi), %ecx
 ; X86-ATOM-NEXT:    movl 8(%edi), %edx
@@ -295,17 +256,16 @@ define i128 @or128(ptr %p) {
 ; X86-ATOM-NEXT:    popl %edi
 ; X86-ATOM-NEXT:    popl %ebx
 ; X86-ATOM-NEXT:    popl %ebp
-; X86-ATOM-NEXT:    .cfi_def_cfa %esp, 4
 ; X86-ATOM-NEXT:    retl $4
   %1 = atomicrmw or ptr %p, i128 0 monotonic
   ret i128 %1
 }
 
 ; For 'and', the idempotent value is (-1)
-define i32 @and32 (ptr %p) {
+define i32 @and32 (ptr %p) #0 {
 ; X64-LABEL: and32:
 ; X64:       # %bb.0:
-; X64-NEXT:    mfence
+; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    movl (%rdi), %eax
 ; X64-NEXT:    retq
 ;
@@ -318,32 +278,24 @@ define i32 @and32 (ptr %p) {
 ;
 ; X86-SLM-LABEL: and32:
 ; X86-SLM:       # %bb.0:
-; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-SLM-NEXT:    movl (%ecx), %eax
-; X86-SLM-NEXT:    .p2align 4
-; X86-SLM-NEXT:  .LBB5_1: # %atomicrmw.start
-; X86-SLM-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-SLM-NEXT:    lock cmpxchgl %eax, (%ecx)
-; X86-SLM-NEXT:    jne .LBB5_1
-; X86-SLM-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SLM-NEXT:    lock orl $0, (%esp)
+; X86-SLM-NEXT:    movl (%eax), %eax
 ; X86-SLM-NEXT:    retl
 ;
 ; X86-ATOM-LABEL: and32:
 ; X86-ATOM:       # %bb.0:
-; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-ATOM-NEXT:    movl (%ecx), %eax
-; X86-ATOM-NEXT:    .p2align 4
-; X86-ATOM-NEXT:  .LBB5_1: # %atomicrmw.start
-; X86-ATOM-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-ATOM-NEXT:    lock cmpxchgl %eax, (%ecx)
-; X86-ATOM-NEXT:    jne .LBB5_1
-; X86-ATOM-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-ATOM-NEXT:    lock orl $0, (%esp)
+; X86-ATOM-NEXT:    movl (%eax), %eax
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
 ; X86-ATOM-NEXT:    retl
   %1 = atomicrmw and ptr %p, i32 -1 acq_rel
   ret i32 %1
 }
 
-define void @or32_nouse_monotonic(ptr %p) {
+define void @or32_nouse_monotonic(ptr %p) #0 {
 ; X64-LABEL: or32_nouse_monotonic:
 ; X64:       # %bb.0:
 ; X64-NEXT:    #MEMBARRIER
@@ -371,7 +323,7 @@ define void @or32_nouse_monotonic(ptr %p) {
 }
 
 
-define void @or32_nouse_acquire(ptr %p) {
+define void @or32_nouse_acquire(ptr %p) #0 {
 ; X64-LABEL: or32_nouse_acquire:
 ; X64:       # %bb.0:
 ; X64-NEXT:    #MEMBARRIER
@@ -398,7 +350,7 @@ define void @or32_nouse_acquire(ptr %p) {
   ret void
 }
 
-define void @or32_nouse_release(ptr %p) {
+define void @or32_nouse_release(ptr %p) #0 {
 ; X64-LABEL: or32_nouse_release:
 ; X64:       # %bb.0:
 ; X64-NEXT:    #MEMBARRIER
@@ -425,7 +377,7 @@ define void @or32_nouse_release(ptr %p) {
   ret void
 }
 
-define void @or32_nouse_acq_rel(ptr %p) {
+define void @or32_nouse_acq_rel(ptr %p) #0 {
 ; X64-LABEL: or32_nouse_acq_rel:
 ; X64:       # %bb.0:
 ; X64-NEXT:    #MEMBARRIER
@@ -452,7 +404,7 @@ define void @or32_nouse_acq_rel(ptr %p) {
   ret void
 }
 
-define void @or32_nouse_seq_cst(ptr %p) {
+define void @or32_nouse_seq_cst(ptr %p) #0 {
 ; X64-LABEL: or32_nouse_seq_cst:
 ; X64:       # %bb.0:
 ; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
@@ -478,7 +430,7 @@ define void @or32_nouse_seq_cst(ptr %p) {
 }
 
 ; TODO: The value isn't used on 32 bit, so the cmpxchg8b is unneeded
-define void @or64_nouse_seq_cst(ptr %p) {
+define void @or64_nouse_seq_cst(ptr %p) #0 {
 ; X64-LABEL: or64_nouse_seq_cst:
 ; X64:       # %bb.0:
 ; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
@@ -487,11 +439,7 @@ define void @or64_nouse_seq_cst(ptr %p) {
 ; X86-LABEL: or64_nouse_seq_cst:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %ebx
-; X86-NEXT:    .cfi_def_cfa_offset 8
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    .cfi_def_cfa_offset 12
-; X86-NEXT:    .cfi_offset %esi, -12
-; X86-NEXT:    .cfi_offset %ebx, -8
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    movl (%esi), %eax
 ; X86-NEXT:    movl 4(%esi), %edx
@@ -504,43 +452,33 @@ define void @or64_nouse_seq_cst(ptr %p) {
 ; X86-NEXT:    jne .LBB11_1
 ; X86-NEXT:  # %bb.2: # %atomicrmw.end
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    .cfi_def_cfa_offset 8
 ; X86-NEXT:    popl %ebx
-; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl
   atomicrmw or ptr %p, i64 0 seq_cst
   ret void
 }
 
 ; TODO: Don't need to lower as sync_and_fetch call
-define void @or128_nouse_seq_cst(ptr %p) {
+define void @or128_nouse_seq_cst(ptr %p) #0 {
 ; X64-LABEL: or128_nouse_seq_cst:
 ; X64:       # %bb.0:
 ; X64-NEXT:    pushq %rax
-; X64-NEXT:    .cfi_def_cfa_offset 16
 ; X64-NEXT:    xorl %esi, %esi
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    movl $5, %ecx
 ; X64-NEXT:    callq __atomic_fetch_or_16@PLT
 ; X64-NEXT:    popq %rax
-; X64-NEXT:    .cfi_def_cfa_offset 8
 ; X64-NEXT:    retq
 ;
 ; X86-GENERIC-LABEL: or128_nouse_seq_cst:
 ; X86-GENERIC:       # %bb.0:
 ; X86-GENERIC-NEXT:    pushl %ebp
-; X86-GENERIC-NEXT:    .cfi_def_cfa_offset 8
-; X86-GENERIC-NEXT:    .cfi_offset %ebp, -8
 ; X86-GENERIC-NEXT:    movl %esp, %ebp
-; X86-GENERIC-NEXT:    .cfi_def_cfa_register %ebp
 ; X86-GENERIC-NEXT:    pushl %ebx
 ; X86-GENERIC-NEXT:    pushl %edi
 ; X86-GENERIC-NEXT:    pushl %esi
 ; X86-GENERIC-NEXT:    andl $-16, %esp
 ; X86-GENERIC-NEXT:    subl $48, %esp
-; X86-GENERIC-NEXT:    .cfi_offset %esi, -20
-; X86-GENERIC-NEXT:    .cfi_offset %edi, -16
-; X86-GENERIC-NEXT:    .cfi_offset %ebx, -12
 ; X86-GENERIC-NEXT:    movl 8(%ebp), %esi
 ; X86-GENERIC-NEXT:    movl 12(%esi), %ecx
 ; X86-GENERIC-NEXT:    movl 8(%esi), %edi
@@ -579,24 +517,17 @@ define void @or128_nouse_seq_cst(ptr %p) {
 ; X86-GENERIC-NEXT:    popl %edi
 ; X86-GENERIC-NEXT:    popl %ebx
 ; X86-GENERIC-NEXT:    popl %ebp
-; X86-GENERIC-NEXT:    .cfi_def_cfa %esp, 4
 ; X86-GENERIC-NEXT:    retl
 ;
 ; X86-ATOM-LABEL: or128_nouse_seq_cst:
 ; X86-ATOM:       # %bb.0:
 ; X86-ATOM-NEXT:    pushl %ebp
-; X86-ATOM-NEXT:    .cfi_def_cfa_offset 8
-; X86-ATOM-NEXT:    .cfi_offset %ebp, -8
 ; X86-ATOM-NEXT:    movl %esp, %ebp
-; X86-ATOM-NEXT:    .cfi_def_cfa_register %ebp
 ; X86-ATOM-NEXT:    pushl %ebx
 ; X86-ATOM-NEXT:    pushl %edi
 ; X86-ATOM-NEXT:    pushl %esi
 ; X86-ATOM-NEXT:    andl $-16, %esp
 ; X86-ATOM-NEXT:    leal -{{[0-9]+}}(%esp), %esp
-; X86-ATOM-NEXT:    .cfi_offset %esi, -20
-; X86-ATOM-NEXT:    .cfi_offset %edi, -16
-; X86-ATOM-NEXT:    .cfi_offset %ebx, -12
 ; X86-ATOM-NEXT:    movl 8(%ebp), %esi
 ; X86-ATOM-NEXT:    movl %esp, %ebx
 ; X86-ATOM-NEXT:    movl 12(%esi), %ecx
@@ -635,14 +566,13 @@ define void @or128_nouse_seq_cst(ptr %p) {
 ; X86-ATOM-NEXT:    popl %edi
 ; X86-ATOM-NEXT:    popl %ebx
 ; X86-ATOM-NEXT:    popl %ebp
-; X86-ATOM-NEXT:    .cfi_def_cfa %esp, 4
 ; X86-ATOM-NEXT:    retl
   atomicrmw or ptr %p, i128 0 seq_cst
   ret void
 }
 
 
-define void @or16_nouse_seq_cst(ptr %p) {
+define void @or16_nouse_seq_cst(ptr %p) #0 {
 ; X64-LABEL: or16_nouse_seq_cst:
 ; X64:       # %bb.0:
 ; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
@@ -667,7 +597,7 @@ define void @or16_nouse_seq_cst(ptr %p) {
   ret void
 }
 
-define void @or8_nouse_seq_cst(ptr %p) {
+define void @or8_nouse_seq_cst(ptr %p) #0 {
 ; X64-LABEL: or8_nouse_seq_cst:
 ; X64:       # %bb.0:
 ; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
@@ -691,3 +621,5 @@ define void @or8_nouse_seq_cst(ptr %p) {
   atomicrmw or ptr %p, i8 0 seq_cst
   ret void
 }
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/X86/atomic-unordered.ll b/llvm/test/CodeGen/X86/atomic-unordered.ll
index 3fb994cdb751a..e8e0ee0b7ef49 100644
--- a/llvm/test/CodeGen/X86/atomic-unordered.ll
+++ b/llvm/test/CodeGen/X86/atomic-unordered.ll
@@ -2096,7 +2096,7 @@ define i64 @nofold_fence(ptr %p) {
 ; CHECK-LABEL: nofold_fence:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    mfence
+; CHECK-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    addq $15, %rax
 ; CHECK-NEXT:    retq
   %v = load atomic i64, ptr %p unordered, align 8
@@ -2170,7 +2170,7 @@ define i64 @fold_constant_fence(i64 %arg) {
 ; CHECK-LABEL: fold_constant_fence:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq Constant(%rip), %rax
-; CHECK-NEXT:    mfence
+; CHECK-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    addq %rdi, %rax
 ; CHECK-NEXT:    retq
   %v = load atomic i64, ptr @Constant unordered, align 8
@@ -2197,7 +2197,7 @@ define i64 @fold_invariant_fence(ptr dereferenceable(8) %p, i64 %arg) {
 ; CHECK-LABEL: fold_invariant_fence:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    mfence
+; CHECK-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    addq %rsi, %rax
 ; CHECK-NEXT:    retq
   %v = load atomic i64, ptr %p unordered, align 8, !invariant.load !{}
@@ -2321,7 +2321,7 @@ define i1 @fold_cmp_over_fence(ptr %p, i32 %v1) {
 ; CHECK-O0-LABEL: fold_cmp_over_fence:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movl (%rdi), %eax
-; CHECK-O0-NEXT:    mfence
+; CHECK-O0-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; CHECK-O0-NEXT:    cmpl %eax, %esi
 ; CHECK-O0-NEXT:    jne .LBB116_2
 ; CHECK-O0-NEXT:  # %bb.1: # %taken
@@ -2335,7 +2335,7 @@ define i1 @fold_cmp_over_fence(ptr %p, i32 %v1) {
 ; CHECK-O3-LABEL: fold_cmp_over_fence:
 ; CHECK-O3:       # %bb.0:
 ; CHECK-O3-NEXT:    movl (%rdi), %eax
-; CHECK-O3-NEXT:    mfence
+; CHECK-O3-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; CHECK-O3-NEXT:    cmpl %eax, %esi
 ; CHECK-O3-NEXT:    jne .LBB116_2
 ; CHECK-O3-NEXT:  # %bb.1: # %taken
diff --git a/llvm/test/CodeGen/X86/avx-shift.ll b/llvm/test/CodeGen/X86/avx-shift.ll
index 1da78ebd75b14..c9c09d7d10346 100644
--- a/llvm/test/CodeGen/X86/avx-shift.ll
+++ b/llvm/test/CodeGen/X86/avx-shift.ll
@@ -136,13 +136,11 @@ define <32 x i8> @vshift10(<32 x i8> %a) {
 define <32 x i8> @vshift11(<32 x i8> %a) {
 ; CHECK-LABEL: vshift11:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; CHECK-NEXT:    vpsrlw $2, %xmm1, %xmm1
-; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm2 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; CHECK-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; CHECK-NEXT:    vpsrlw $2, %xmm0, %xmm1
+; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; CHECK-NEXT:    vpsrlw $2, %xmm0, %xmm0
-; CHECK-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; CHECK-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; CHECK-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; CHECK-NEXT:    retq
   %s = lshr <32 x i8> %a, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
   ret <32 x i8> %s
@@ -151,13 +149,11 @@ define <32 x i8> @vshift11(<32 x i8> %a) {
 define <32 x i8> @vshift12(<32 x i8> %a) {
 ; CHECK-LABEL: vshift12:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; CHECK-NEXT:    vpsllw $2, %xmm1, %xmm1
-; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm2 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; CHECK-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; CHECK-NEXT:    vpsllw $2, %xmm0, %xmm1
+; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; CHECK-NEXT:    vpsllw $2, %xmm0, %xmm0
-; CHECK-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; CHECK-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; CHECK-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; CHECK-NEXT:    retq
   %s = shl <32 x i8> %a, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
   ret <32 x i8> %s
diff --git a/llvm/test/CodeGen/X86/avx10_2_512ni-intrinsics.ll b/llvm/test/CodeGen/X86/avx10_2_512ni-intrinsics.ll
index 07e86cb01e133..b2e7caa15944c 100644
--- a/llvm/test/CodeGen/X86/avx10_2_512ni-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx10_2_512ni-intrinsics.ll
@@ -422,3 +422,14 @@ define { <32 x i16>, <32 x i16>, <32 x i16> } @test_mm512_mask_mpsadbw(<64 x i8>
 }
 
 declare <32 x i16> @llvm.x86.avx10.vmpsadbw.512(<64 x i8>, <64 x i8>, i8)
+
+; Regression test
+
+define <8 x float> @avx_dp_ps(<8 x float> %a, <8 x float> %b) {
+; CHECK-LABEL: avx_dp_ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vdpps $255, %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe3,0x7d,0x40,0xc1,0xff]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %r = tail call <8 x float> @llvm.x86.avx.dp.ps.256(<8 x float> %a, <8 x float> %b, i8 -1)
+  ret <8 x float> %r
+}
diff --git a/llvm/test/CodeGen/X86/combine-pmadd.ll b/llvm/test/CodeGen/X86/combine-pmadd.ll
index eb627f13db08f..d9283aa8591fc 100644
--- a/llvm/test/CodeGen/X86/combine-pmadd.ll
+++ b/llvm/test/CodeGen/X86/combine-pmadd.ll
@@ -51,11 +51,9 @@ define <8 x i32> @combine_pmaddwd_concat(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>
 ;
 ; AVX2-LABEL: combine_pmaddwd_concat:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    # kill: def $xmm1 killed $xmm1 def $ymm1
-; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpmaddwd %xmm3, %xmm2, %xmm1
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
   %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a0, <8 x i16> %a1)
   %2 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a2, <8 x i16> %a3)
@@ -179,28 +177,57 @@ define <8 x i16> @combine_pmaddubsw_zero_commute(<16 x i8> %a0, <16 x i8> %a1) {
   ret <8 x i16> %1
 }
 
-define <16 x i16> @combine_pmaddubsw_concat(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2, <16 x i8> %a3) {
+define <16 x i16> @combine_pmaddubsw_concat(<32 x i8> %a0, <32 x i8> %a1) {
 ; SSE-LABEL: combine_pmaddubsw_concat:
 ; SSE:       # %bb.0:
+; SSE-NEXT:    pmaddubsw %xmm2, %xmm0
+; SSE-NEXT:    pmaddubsw %xmm3, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: combine_pmaddubsw_concat:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpmaddubsw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: combine_pmaddubsw_concat:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %lo0 = shufflevector <32 x i8> %a0, <32 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %lo1 = shufflevector <32 x i8> %a1, <32 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %hi0 = shufflevector <32 x i8> %a0, <32 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %hi1 = shufflevector <32 x i8> %a1, <32 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %lo = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %lo0, <16 x i8> %lo1)
+  %hi = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %hi0, <16 x i8> %hi1)
+  %res = shufflevector <8 x i16> %lo, <8 x i16> %hi, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i16> %res
+}
+
+; Not beneficial to concatenate both inputs just to create a 256-bit pmaddubsw
+define <16 x i16> @combine_pmaddubsw_concat_unecessary(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2, <16 x i8> %a3) {
+; SSE-LABEL: combine_pmaddubsw_concat_unecessary:
+; SSE:       # %bb.0:
 ; SSE-NEXT:    pmaddubsw %xmm1, %xmm0
 ; SSE-NEXT:    pmaddubsw %xmm3, %xmm2
 ; SSE-NEXT:    movdqa %xmm2, %xmm1
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: combine_pmaddubsw_concat:
+; AVX1-LABEL: combine_pmaddubsw_concat_unecessary:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmaddubsw %xmm3, %xmm2, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: combine_pmaddubsw_concat:
+; AVX2-LABEL: combine_pmaddubsw_concat_unecessary:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    # kill: def $xmm1 killed $xmm1 def $ymm1
-; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
-; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpmaddubsw %xmm3, %xmm2, %xmm1
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
   %1 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a0, <16 x i8> %a1)
   %2 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a2, <16 x i8> %a3)
diff --git a/llvm/test/CodeGen/X86/exp10-libcall-names.ll b/llvm/test/CodeGen/X86/exp10-libcall-names.ll
index 4f993cfe74a3b..96e3aae408e94 100644
--- a/llvm/test/CodeGen/X86/exp10-libcall-names.ll
+++ b/llvm/test/CodeGen/X86/exp10-libcall-names.ll
@@ -10,13 +10,15 @@
 ; RUN: llc -mtriple=x86_64-apple-xros8.0 < %s | FileCheck -check-prefix=APPLE %s
 ; RUN: llc -mtriple=x86_64-apple-driverkit < %s | FileCheck -check-prefix=APPLE %s
 ; RUN: llc -mtriple=x86_64-apple-driverkit24.0 < %s | FileCheck -check-prefix=APPLE %s
+; RUN: llc < %s -mtriple=i686-linux-gnu -global-isel -global-isel-abort=1 | FileCheck %s --check-prefix=GISEL-X86
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -global-isel -global-isel-abort=1 | FileCheck %s --check-prefix=GISEL-X64
 
 ; RUN: not llc -mtriple=x86_64-apple-macos10.8 -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR %s
 ; Check exp10/exp10f is emitted as __exp10/__exp10f on assorted systems.
 
 ; ERR: no libcall available for fexp10
 
-define float @test_exp10_f32(float %x) {
+define float @test_exp10_f32(float %x) nounwind {
 ; LINUX-LABEL: test_exp10_f32:
 ; LINUX:       # %bb.0:
 ; LINUX-NEXT:    jmp exp10f@PLT # TAILCALL
@@ -25,15 +27,26 @@ define float @test_exp10_f32(float %x) {
 ; APPLE:       ## %bb.0:
 ; APPLE-NEXT:    jmp ___exp10f ## TAILCALL
 ;
-; MISSED-LABEL: test_exp10_f32:
-; MISSED:       ## %bb.0:
-; MISSED-NEXT:    jmp _exp10f ## TAILCALL
-
+; GISEL-X86-LABEL: test_exp10_f32:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl %eax, (%esp)
+; GISEL-X86-NEXT:    calll exp10f
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: test_exp10_f32:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    callq exp10f
+; GISEL-X64-NEXT:    popq %rax
+; GISEL-X64-NEXT:    retq
   %ret = call float @llvm.exp10.f32(float %x)
   ret float %ret
 }
 
-define double @test_exp10_f64(double %x) {
+define double @test_exp10_f64(double %x) nounwind {
 ; LINUX-LABEL: test_exp10_f64:
 ; LINUX:       # %bb.0:
 ; LINUX-NEXT:    jmp exp10@PLT # TAILCALL
@@ -42,10 +55,66 @@ define double @test_exp10_f64(double %x) {
 ; APPLE:       ## %bb.0:
 ; APPLE-NEXT:    jmp ___exp10 ## TAILCALL
 ;
-; MISSED-LABEL: test_exp10_f64:
-; MISSED:       ## %bb.0:
-; MISSED-NEXT:    jmp _exp10 ## TAILCALL
+; GISEL-X86-LABEL: test_exp10_f64:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movl 4(%eax), %eax
+; GISEL-X86-NEXT:    xorl %edx, %edx
+; GISEL-X86-NEXT:    addl %esp, %edx
+; GISEL-X86-NEXT:    movl %ecx, (%esp)
+; GISEL-X86-NEXT:    movl %eax, 4(%edx)
+; GISEL-X86-NEXT:    calll exp10
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
 ;
+; GISEL-X64-LABEL: test_exp10_f64:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    callq exp10
+; GISEL-X64-NEXT:    popq %rax
+; GISEL-X64-NEXT:    retq
   %ret = call double @llvm.exp10.f64(double %x)
   ret double %ret
 }
+
+define x86_fp80 @test_exp10_f80(x86_fp80 %x) nounwind {
+; LINUX-LABEL: test_exp10_f80:
+; LINUX:       # %bb.0:
+; LINUX-NEXT:    subq $24, %rsp
+; LINUX-NEXT:    fldt {{[0-9]+}}(%rsp)
+; LINUX-NEXT:    fstpt (%rsp)
+; LINUX-NEXT:    callq exp10l@PLT
+; LINUX-NEXT:    addq $24, %rsp
+; LINUX-NEXT:    retq
+;
+; APPLE-LABEL: test_exp10_f80:
+; APPLE:       ## %bb.0:
+; APPLE-NEXT:    subq $24, %rsp
+; APPLE-NEXT:    fldt {{[0-9]+}}(%rsp)
+; APPLE-NEXT:    fstpt (%rsp)
+; APPLE-NEXT:    callq _exp10l
+; APPLE-NEXT:    addq $24, %rsp
+; APPLE-NEXT:    retq
+;
+; GISEL-X86-LABEL: test_exp10_f80:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fstpt (%esp)
+; GISEL-X86-NEXT:    calll exp10l
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: test_exp10_f80:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    subq $24, %rsp
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fstpt (%rsp)
+; GISEL-X64-NEXT:    callq exp10l
+; GISEL-X64-NEXT:    addq $24, %rsp
+; GISEL-X64-NEXT:    retq
+  %ret = call x86_fp80 @llvm.exp10.f80(x86_fp80 %x)
+  ret x86_fp80 %ret
+}
diff --git a/llvm/test/CodeGen/X86/finite-libcalls.ll b/llvm/test/CodeGen/X86/finite-libcalls.ll
index 6b5b68e321c7c..ea2695dc96997 100644
--- a/llvm/test/CodeGen/X86/finite-libcalls.ll
+++ b/llvm/test/CodeGen/X86/finite-libcalls.ll
@@ -2,6 +2,8 @@
 ; RUN: llc < %s -mtriple=x86_64-pc-linux-gnu     | FileCheck %s --check-prefix=GNU
 ; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc  | FileCheck %s --check-prefix=WIN
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin     | FileCheck %s --check-prefix=MAC
+; RUN: llc < %s -mtriple=i686-linux-gnu -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes=GISEL-X86
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes=GISEL-X64
 
 ; PR35672 - https://bugs.llvm.org/show_bug.cgi?id=35672
 ; FIXME: We would not need the function-level attributes if FMF were propagated to DAG nodes for this case.
@@ -18,6 +20,22 @@ define float @exp_f32(float %x) #0 {
 ; MAC-LABEL: exp_f32:
 ; MAC:       ## %bb.0:
 ; MAC-NEXT:    jmp _expf ## TAILCALL
+;
+; GISEL-X86-LABEL: exp_f32:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl %eax, (%esp)
+; GISEL-X86-NEXT:    calll expf
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: exp_f32:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    callq expf
+; GISEL-X64-NEXT:    popq %rax
+; GISEL-X64-NEXT:    retq
   %r = tail call nnan ninf float @llvm.exp.f32(float %x)
   ret float %r
 }
@@ -34,6 +52,27 @@ define double @exp_f64(double %x) #0 {
 ; MAC-LABEL: exp_f64:
 ; MAC:       ## %bb.0:
 ; MAC-NEXT:    jmp _exp ## TAILCALL
+;
+; GISEL-X86-LABEL: exp_f64:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movl 4(%eax), %eax
+; GISEL-X86-NEXT:    xorl %edx, %edx
+; GISEL-X86-NEXT:    addl %esp, %edx
+; GISEL-X86-NEXT:    movl %ecx, (%esp)
+; GISEL-X86-NEXT:    movl %eax, 4(%edx)
+; GISEL-X86-NEXT:    calll exp
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: exp_f64:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    callq exp
+; GISEL-X64-NEXT:    popq %rax
+; GISEL-X64-NEXT:    retq
   %r = tail call nnan ninf double @llvm.exp.f64(double %x)
   ret double %r
 }
@@ -73,6 +112,24 @@ define x86_fp80 @exp_f80(x86_fp80 %x) #0 {
 ; MAC-NEXT:    callq _expl
 ; MAC-NEXT:    addq $24, %rsp
 ; MAC-NEXT:    retq
+;
+; GISEL-X86-LABEL: exp_f80:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fstpt (%esp)
+; GISEL-X86-NEXT:    calll expl
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: exp_f80:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    subq $24, %rsp
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fstpt (%rsp)
+; GISEL-X64-NEXT:    callq expl
+; GISEL-X64-NEXT:    addq $24, %rsp
+; GISEL-X64-NEXT:    retq
   %r = tail call nnan ninf x86_fp80 @llvm.exp.f80(x86_fp80 %x)
   ret x86_fp80 %r
 }
@@ -89,6 +146,22 @@ define float @exp2_f32(float %x) #0 {
 ; MAC-LABEL: exp2_f32:
 ; MAC:       ## %bb.0:
 ; MAC-NEXT:    jmp _exp2f ## TAILCALL
+;
+; GISEL-X86-LABEL: exp2_f32:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl %eax, (%esp)
+; GISEL-X86-NEXT:    calll exp2f
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: exp2_f32:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    callq exp2f
+; GISEL-X64-NEXT:    popq %rax
+; GISEL-X64-NEXT:    retq
   %r = tail call nnan ninf float @llvm.exp2.f32(float %x)
   ret float %r
 }
@@ -105,6 +178,27 @@ define double @exp2_f64(double %x) #0 {
 ; MAC-LABEL: exp2_f64:
 ; MAC:       ## %bb.0:
 ; MAC-NEXT:    jmp _exp2 ## TAILCALL
+;
+; GISEL-X86-LABEL: exp2_f64:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movl 4(%eax), %eax
+; GISEL-X86-NEXT:    xorl %edx, %edx
+; GISEL-X86-NEXT:    addl %esp, %edx
+; GISEL-X86-NEXT:    movl %ecx, (%esp)
+; GISEL-X86-NEXT:    movl %eax, 4(%edx)
+; GISEL-X86-NEXT:    calll exp2
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: exp2_f64:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    callq exp2
+; GISEL-X64-NEXT:    popq %rax
+; GISEL-X64-NEXT:    retq
   %r = tail call nnan ninf double @llvm.exp2.f64(double %x)
   ret double %r
 }
@@ -144,6 +238,24 @@ define x86_fp80 @exp2_f80(x86_fp80 %x) #0 {
 ; MAC-NEXT:    callq _exp2l
 ; MAC-NEXT:    addq $24, %rsp
 ; MAC-NEXT:    retq
+;
+; GISEL-X86-LABEL: exp2_f80:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fstpt (%esp)
+; GISEL-X86-NEXT:    calll exp2l
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: exp2_f80:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    subq $24, %rsp
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fstpt (%rsp)
+; GISEL-X64-NEXT:    callq exp2l
+; GISEL-X64-NEXT:    addq $24, %rsp
+; GISEL-X64-NEXT:    retq
   %r = tail call nnan ninf x86_fp80 @llvm.exp2.f80(x86_fp80 %x)
   ret x86_fp80 %r
 }
@@ -160,6 +272,22 @@ define float @log_f32(float %x) #0 {
 ; MAC-LABEL: log_f32:
 ; MAC:       ## %bb.0:
 ; MAC-NEXT:    jmp _logf ## TAILCALL
+;
+; GISEL-X86-LABEL: log_f32:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl %eax, (%esp)
+; GISEL-X86-NEXT:    calll logf
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: log_f32:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    callq logf
+; GISEL-X64-NEXT:    popq %rax
+; GISEL-X64-NEXT:    retq
   %r = tail call nnan ninf float @llvm.log.f32(float %x)
   ret float %r
 }
@@ -176,6 +304,27 @@ define double @log_f64(double %x) #0 {
 ; MAC-LABEL: log_f64:
 ; MAC:       ## %bb.0:
 ; MAC-NEXT:    jmp _log ## TAILCALL
+;
+; GISEL-X86-LABEL: log_f64:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movl 4(%eax), %eax
+; GISEL-X86-NEXT:    xorl %edx, %edx
+; GISEL-X86-NEXT:    addl %esp, %edx
+; GISEL-X86-NEXT:    movl %ecx, (%esp)
+; GISEL-X86-NEXT:    movl %eax, 4(%edx)
+; GISEL-X86-NEXT:    calll log
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: log_f64:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    callq log
+; GISEL-X64-NEXT:    popq %rax
+; GISEL-X64-NEXT:    retq
   %r = tail call nnan ninf double @llvm.log.f64(double %x)
   ret double %r
 }
@@ -215,6 +364,24 @@ define x86_fp80 @log_f80(x86_fp80 %x) #0 {
 ; MAC-NEXT:    callq _logl
 ; MAC-NEXT:    addq $24, %rsp
 ; MAC-NEXT:    retq
+;
+; GISEL-X86-LABEL: log_f80:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fstpt (%esp)
+; GISEL-X86-NEXT:    calll logl
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: log_f80:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    subq $24, %rsp
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fstpt (%rsp)
+; GISEL-X64-NEXT:    callq logl
+; GISEL-X64-NEXT:    addq $24, %rsp
+; GISEL-X64-NEXT:    retq
   %r = tail call nnan ninf x86_fp80 @llvm.log.f80(x86_fp80 %x)
   ret x86_fp80 %r
 }
@@ -231,6 +398,22 @@ define float @log2_f32(float %x) #0 {
 ; MAC-LABEL: log2_f32:
 ; MAC:       ## %bb.0:
 ; MAC-NEXT:    jmp _log2f ## TAILCALL
+;
+; GISEL-X86-LABEL: log2_f32:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl %eax, (%esp)
+; GISEL-X86-NEXT:    calll log2f
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: log2_f32:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    callq log2f
+; GISEL-X64-NEXT:    popq %rax
+; GISEL-X64-NEXT:    retq
   %r = tail call nnan ninf float @llvm.log2.f32(float %x)
   ret float %r
 }
@@ -247,6 +430,27 @@ define double @log2_f64(double %x) #0 {
 ; MAC-LABEL: log2_f64:
 ; MAC:       ## %bb.0:
 ; MAC-NEXT:    jmp _log2 ## TAILCALL
+;
+; GISEL-X86-LABEL: log2_f64:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movl 4(%eax), %eax
+; GISEL-X86-NEXT:    xorl %edx, %edx
+; GISEL-X86-NEXT:    addl %esp, %edx
+; GISEL-X86-NEXT:    movl %ecx, (%esp)
+; GISEL-X86-NEXT:    movl %eax, 4(%edx)
+; GISEL-X86-NEXT:    calll log2
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: log2_f64:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    callq log2
+; GISEL-X64-NEXT:    popq %rax
+; GISEL-X64-NEXT:    retq
   %r = tail call nnan ninf double @llvm.log2.f64(double %x)
   ret double %r
 }
@@ -286,6 +490,24 @@ define x86_fp80 @log2_f80(x86_fp80 %x) #0 {
 ; MAC-NEXT:    callq _log2l
 ; MAC-NEXT:    addq $24, %rsp
 ; MAC-NEXT:    retq
+;
+; GISEL-X86-LABEL: log2_f80:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fstpt (%esp)
+; GISEL-X86-NEXT:    calll log2l
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: log2_f80:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    subq $24, %rsp
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fstpt (%rsp)
+; GISEL-X64-NEXT:    callq log2l
+; GISEL-X64-NEXT:    addq $24, %rsp
+; GISEL-X64-NEXT:    retq
   %r = tail call nnan ninf x86_fp80 @llvm.log2.f80(x86_fp80 %x)
   ret x86_fp80 %r
 }
@@ -302,6 +524,22 @@ define float @log10_f32(float %x) #0 {
 ; MAC-LABEL: log10_f32:
 ; MAC:       ## %bb.0:
 ; MAC-NEXT:    jmp _log10f ## TAILCALL
+;
+; GISEL-X86-LABEL: log10_f32:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl %eax, (%esp)
+; GISEL-X86-NEXT:    calll log10f
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: log10_f32:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    callq log10f
+; GISEL-X64-NEXT:    popq %rax
+; GISEL-X64-NEXT:    retq
   %r = tail call nnan ninf float @llvm.log10.f32(float %x)
   ret float %r
 }
@@ -318,6 +556,27 @@ define double @log10_f64(double %x) #0 {
 ; MAC-LABEL: log10_f64:
 ; MAC:       ## %bb.0:
 ; MAC-NEXT:    jmp _log10 ## TAILCALL
+;
+; GISEL-X86-LABEL: log10_f64:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movl 4(%eax), %eax
+; GISEL-X86-NEXT:    xorl %edx, %edx
+; GISEL-X86-NEXT:    addl %esp, %edx
+; GISEL-X86-NEXT:    movl %ecx, (%esp)
+; GISEL-X86-NEXT:    movl %eax, 4(%edx)
+; GISEL-X86-NEXT:    calll log10
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: log10_f64:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    callq log10
+; GISEL-X64-NEXT:    popq %rax
+; GISEL-X64-NEXT:    retq
   %r = tail call nnan ninf double @llvm.log10.f64(double %x)
   ret double %r
 }
@@ -357,6 +616,24 @@ define x86_fp80 @log10_f80(x86_fp80 %x) #0 {
 ; MAC-NEXT:    callq _log10l
 ; MAC-NEXT:    addq $24, %rsp
 ; MAC-NEXT:    retq
+;
+; GISEL-X86-LABEL: log10_f80:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fstpt (%esp)
+; GISEL-X86-NEXT:    calll log10l
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: log10_f80:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    subq $24, %rsp
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fstpt (%rsp)
+; GISEL-X64-NEXT:    callq log10l
+; GISEL-X64-NEXT:    addq $24, %rsp
+; GISEL-X64-NEXT:    retq
   %r = tail call nnan ninf x86_fp80 @llvm.log10.f80(x86_fp80 %x)
   ret x86_fp80 %r
 }
@@ -376,6 +653,24 @@ define float @pow_f32(float %x) #0 {
 ; MAC:       ## %bb.0:
 ; MAC-NEXT:    movaps %xmm0, %xmm1
 ; MAC-NEXT:    jmp _powf ## TAILCALL
+;
+; GISEL-X86-LABEL: pow_f32:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl %eax, (%esp)
+; GISEL-X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    calll powf
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: pow_f32:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    movaps %xmm0, %xmm1
+; GISEL-X64-NEXT:    callq powf
+; GISEL-X64-NEXT:    popq %rax
+; GISEL-X64-NEXT:    retq
   %r = tail call nnan ninf float @llvm.pow.f32(float %x, float %x)
   ret float %r
 }
@@ -395,6 +690,32 @@ define double @pow_f64(double %x) #0 {
 ; MAC:       ## %bb.0:
 ; MAC-NEXT:    movaps %xmm0, %xmm1
 ; MAC-NEXT:    jmp _pow ## TAILCALL
+;
+; GISEL-X86-LABEL: pow_f64:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $28, %esp
+; GISEL-X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    movl 4(%eax), %eax
+; GISEL-X86-NEXT:    xorl %edx, %edx
+; GISEL-X86-NEXT:    addl %esp, %edx
+; GISEL-X86-NEXT:    movl %ecx, (%esp)
+; GISEL-X86-NEXT:    movl %eax, 4(%edx)
+; GISEL-X86-NEXT:    movl $8, %edx
+; GISEL-X86-NEXT:    addl %esp, %edx
+; GISEL-X86-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    movl %eax, 4(%edx)
+; GISEL-X86-NEXT:    calll pow
+; GISEL-X86-NEXT:    addl $28, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: pow_f64:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    movaps %xmm0, %xmm1
+; GISEL-X64-NEXT:    callq pow
+; GISEL-X64-NEXT:    popq %rax
+; GISEL-X64-NEXT:    retq
   %r = tail call nnan ninf double @llvm.pow.f64(double %x, double %x)
   ret double %r
 }
@@ -441,6 +762,28 @@ define x86_fp80 @pow_f80(x86_fp80 %x) #0 {
 ; MAC-NEXT:    callq _powl
 ; MAC-NEXT:    addq $40, %rsp
 ; MAC-NEXT:    retq
+;
+; GISEL-X86-LABEL: pow_f80:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $28, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fld %st(0)
+; GISEL-X86-NEXT:    fstpt (%esp)
+; GISEL-X86-NEXT:    fstpt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    calll powl
+; GISEL-X86-NEXT:    addl $28, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: pow_f80:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    subq $40, %rsp
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fld %st(0)
+; GISEL-X64-NEXT:    fstpt (%rsp)
+; GISEL-X64-NEXT:    fstpt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    callq powl
+; GISEL-X64-NEXT:    addq $40, %rsp
+; GISEL-X64-NEXT:    retq
   %r = tail call nnan ninf x86_fp80 @llvm.pow.f80(x86_fp80 %x, x86_fp80 %x)
   ret x86_fp80 %r
 }
diff --git a/llvm/test/CodeGen/X86/gfni-rotates.ll b/llvm/test/CodeGen/X86/gfni-rotates.ll
index 5fd4dfa7cc262..967f26f70946a 100644
--- a/llvm/test/CodeGen/X86/gfni-rotates.ll
+++ b/llvm/test/CodeGen/X86/gfni-rotates.ll
@@ -255,9 +255,9 @@ define <16 x i8> @splatvar_rotr_v16i8(<16 x i8> %a, <16 x i8> %amt) nounwind {
 ;
 ; GFNIAVX512BW-LABEL: splatvar_rotr_v16i8:
 ; GFNIAVX512BW:       # %bb.0:
-; GFNIAVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; GFNIAVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; GFNIAVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
+; GFNIAVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; GFNIAVX512BW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; GFNIAVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; GFNIAVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
 ; GFNIAVX512BW-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; GFNIAVX512BW-NEXT:    vpmovwb %ymm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/gfni-shifts.ll b/llvm/test/CodeGen/X86/gfni-shifts.ll
index 38f6cabd21fba..5cd1a2c76762e 100644
--- a/llvm/test/CodeGen/X86/gfni-shifts.ll
+++ b/llvm/test/CodeGen/X86/gfni-shifts.ll
@@ -1232,21 +1232,19 @@ define <32 x i8> @constant_shl_v32i8(<32 x i8> %a) nounwind {
 ;
 ; GFNIAVX1-LABEL: constant_shl_v32i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; GFNIAVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = [1,4,16,64,128,32,8,2]
-; GFNIAVX1-NEXT:    vpmaddubsw %xmm2, %xmm1, %xmm3
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1]
-; GFNIAVX1-NEXT:    vpmaddubsw %xmm5, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1]
+; GFNIAVX1-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm2
+; GFNIAVX1-NEXT:    vpsllw $8, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; GFNIAVX1-NEXT:    vpmaddubsw %xmm1, %xmm3, %xmm1
 ; GFNIAVX1-NEXT:    vpsllw $8, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpor %xmm1, %xmm3, %xmm1
-; GFNIAVX1-NEXT:    vpmaddubsw %xmm2, %xmm0, %xmm2
-; GFNIAVX1-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpmaddubsw %xmm5, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsllw $8, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; GFNIAVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = [1,4,16,64,128,32,8,2]
+; GFNIAVX1-NEXT:    vpmaddubsw %xmm2, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vpmaddubsw %xmm2, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: constant_shl_v32i8:
@@ -2575,33 +2573,31 @@ define <64 x i8> @constant_shl_v64i8(<64 x i8> %a) nounwind {
 ;
 ; GFNIAVX1-LABEL: constant_shl_v64i8:
 ; GFNIAVX1:       # %bb.0:
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; GFNIAVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = [1,4,16,64,128,32,8,2]
-; GFNIAVX1-NEXT:    vpmaddubsw %xmm3, %xmm2, %xmm4
-; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1]
-; GFNIAVX1-NEXT:    vpmaddubsw %xmm6, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpsllw $8, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpor %xmm2, %xmm4, %xmm2
-; GFNIAVX1-NEXT:    vpmaddubsw %xmm3, %xmm0, %xmm4
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vpmaddubsw %xmm6, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpsllw $8, %xmm0, %xmm0
-; GFNIAVX1-NEXT:    vpor %xmm0, %xmm4, %xmm0
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; GFNIAVX1-NEXT:    vpmaddubsw %xmm3, %xmm2, %xmm4
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
-; GFNIAVX1-NEXT:    vpmaddubsw %xmm6, %xmm2, %xmm2
+; GFNIAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1]
+; GFNIAVX1-NEXT:    vpmaddubsw %xmm2, %xmm0, %xmm3
+; GFNIAVX1-NEXT:    vpsllw $8, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; GFNIAVX1-NEXT:    vpmaddubsw %xmm2, %xmm4, %xmm5
+; GFNIAVX1-NEXT:    vpsllw $8, %xmm5, %xmm5
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
+; GFNIAVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = [1,4,16,64,128,32,8,2]
+; GFNIAVX1-NEXT:    vpmaddubsw %xmm5, %xmm4, %xmm4
+; GFNIAVX1-NEXT:    vpmaddubsw %xmm5, %xmm0, %xmm0
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vbroadcastss {{.*#+}} ymm4 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; GFNIAVX1-NEXT:    vandps %ymm4, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vorps %ymm3, %ymm0, %ymm0
+; GFNIAVX1-NEXT:    vpmaddubsw %xmm2, %xmm1, %xmm3
+; GFNIAVX1-NEXT:    vpsllw $8, %xmm3, %xmm3
+; GFNIAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
+; GFNIAVX1-NEXT:    vpmaddubsw %xmm2, %xmm6, %xmm2
 ; GFNIAVX1-NEXT:    vpsllw $8, %xmm2, %xmm2
-; GFNIAVX1-NEXT:    vpor %xmm2, %xmm4, %xmm2
-; GFNIAVX1-NEXT:    vpmaddubsw %xmm3, %xmm1, %xmm3
-; GFNIAVX1-NEXT:    vpand %xmm5, %xmm3, %xmm3
-; GFNIAVX1-NEXT:    vpmaddubsw %xmm6, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpsllw $8, %xmm1, %xmm1
-; GFNIAVX1-NEXT:    vpor %xmm1, %xmm3, %xmm1
-; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; GFNIAVX1-NEXT:    vpmaddubsw %xmm5, %xmm6, %xmm3
+; GFNIAVX1-NEXT:    vpmaddubsw %xmm5, %xmm1, %xmm1
+; GFNIAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; GFNIAVX1-NEXT:    vandps %ymm4, %ymm1, %ymm1
+; GFNIAVX1-NEXT:    vorps %ymm2, %ymm1, %ymm1
 ; GFNIAVX1-NEXT:    retq
 ;
 ; GFNIAVX2-LABEL: constant_shl_v64i8:
diff --git a/llvm/test/CodeGen/X86/haddsub-undef.ll b/llvm/test/CodeGen/X86/haddsub-undef.ll
index 6aa53278d81ef..94fa81742ba71 100644
--- a/llvm/test/CodeGen/X86/haddsub-undef.ll
+++ b/llvm/test/CodeGen/X86/haddsub-undef.ll
@@ -694,9 +694,9 @@ define <4 x double> @add_pd_011(<4 x double> %0, <4 x double> %1) {
 ;
 ; AVX1-FAST-LABEL: add_pd_011:
 ; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm2
-; AVX1-FAST-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-FAST-NEXT:    vhaddpd %ymm2, %ymm0, %ymm0
+; AVX1-FAST-NEXT:    vhaddpd %xmm0, %xmm1, %xmm2
+; AVX1-FAST-NEXT:    vhaddpd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-FAST-NEXT:    retq
 ;
 ; AVX512-LABEL: add_pd_011:
@@ -1272,9 +1272,9 @@ define <4 x double> @PR34724_add_v4f64_012u(<4 x double> %0, <4 x double> %1) {
 ; AVX-FAST-LABEL: PR34724_add_v4f64_012u:
 ; AVX-FAST:       # %bb.0:
 ; AVX-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX-FAST-NEXT:    vhaddpd %xmm2, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vhaddpd %xmm1, %xmm1, %xmm1
 ; AVX-FAST-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX-FAST-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
-; AVX-FAST-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0
 ; AVX-FAST-NEXT:    retq
   %3 = shufflevector <4 x double> %0, <4 x double> undef, <2 x i32> <i32 0, i32 2>
   %4 = shufflevector <4 x double> %0, <4 x double> undef, <2 x i32> <i32 1, i32 3>
diff --git a/llvm/test/CodeGen/X86/implicit-null-check.ll b/llvm/test/CodeGen/X86/implicit-null-check.ll
index fc81f703f5d40..de63c9ae209df 100644
--- a/llvm/test/CodeGen/X86/implicit-null-check.ll
+++ b/llvm/test/CodeGen/X86/implicit-null-check.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -verify-machineinstrs -O3 -mtriple=x86_64-apple-macosx -enable-implicit-null-checks < %s | FileCheck %s
 
 define i32 @imp_null_check_load(ptr %x) {
@@ -465,7 +466,7 @@ define i32 @imp_null_check_load_fence2(ptr %x) {
 ; CHECK-NEXT:    testq %rdi, %rdi
 ; CHECK-NEXT:    je LBB17_1
 ; CHECK-NEXT:  ## %bb.2: ## %not_null
-; CHECK-NEXT:    mfence
+; CHECK-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movl (%rdi), %eax
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:  LBB17_1: ## %is_null
diff --git a/llvm/test/CodeGen/X86/known-bits-vector.ll b/llvm/test/CodeGen/X86/known-bits-vector.ll
index 2eef32eb61414..261908fafc06e 100644
--- a/llvm/test/CodeGen/X86/known-bits-vector.ll
+++ b/llvm/test/CodeGen/X86/known-bits-vector.ll
@@ -384,23 +384,19 @@ declare <4 x i32> @llvm.bswap.v4i32(<4 x i32>)
 define <8 x float> @knownbits_mask_concat_uitofp(<4 x i32> %a0, <4 x i32> %a1) nounwind {
 ; X86-LABEL: knownbits_mask_concat_uitofp:
 ; X86:       # %bb.0:
-; X86-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,3,1,3]
-; X86-NEXT:    vbroadcastss {{.*#+}} xmm2 = [131071,131071,131071,131071]
-; X86-NEXT:    vandps %xmm2, %xmm1, %xmm1
-; X86-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2,0,2]
-; X86-NEXT:    vandps %xmm2, %xmm0, %xmm0
+; X86-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; X86-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
+; X86-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,2,0,2,5,7,5,7]
 ; X86-NEXT:    vcvtdq2ps %ymm0, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: knownbits_mask_concat_uitofp:
 ; X64:       # %bb.0:
-; X64-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[1,3,1,3]
-; X64-NEXT:    vbroadcastss {{.*#+}} xmm2 = [131071,131071,131071,131071]
-; X64-NEXT:    vandps %xmm2, %xmm1, %xmm1
-; X64-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2,0,2]
-; X64-NEXT:    vandps %xmm2, %xmm0, %xmm0
+; X64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; X64-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X64-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,2,0,2,5,7,5,7]
 ; X64-NEXT:    vcvtdq2ps %ymm0, %ymm0
 ; X64-NEXT:    retq
   %1 = and <4 x i32> %a0, <i32 131071, i32 -1, i32 131071, i32 -1>
diff --git a/llvm/test/CodeGen/X86/llvm.modf-win32.ll b/llvm/test/CodeGen/X86/llvm.modf-win32.ll
new file mode 100644
index 0000000000000..70ce773dda482
--- /dev/null
+++ b/llvm/test/CodeGen/X86/llvm.modf-win32.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=i386-pc-win32 < %s | FileCheck -check-prefix=WIN32 %s
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s | FileCheck -check-prefixes=X64 %s
+
+; On 32-bit windows this should be promoted to a call to modf (not modff).
+define { float, float } @test_modf_f32(float %a) {
+; WIN32-LABEL: test_modf_f32:
+; WIN32:       # %bb.0:
+; WIN32-NEXT:    pushl %ebp
+; WIN32-NEXT:    movl %esp, %ebp
+; WIN32-NEXT:    andl $-8, %esp
+; WIN32-NEXT:    subl $32, %esp
+; WIN32-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; WIN32-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; WIN32-NEXT:    flds 8(%ebp)
+; WIN32-NEXT:    fstpl (%esp)
+; WIN32-NEXT:    calll _modf
+; WIN32-NEXT:    fstps {{[0-9]+}}(%esp)
+; WIN32-NEXT:    fldl {{[0-9]+}}(%esp)
+; WIN32-NEXT:    fstps {{[0-9]+}}(%esp)
+; WIN32-NEXT:    flds {{[0-9]+}}(%esp)
+; WIN32-NEXT:    flds {{[0-9]+}}(%esp)
+; WIN32-NEXT:    fxch %st(1)
+; WIN32-NEXT:    movl %ebp, %esp
+; WIN32-NEXT:    popl %ebp
+; WIN32-NEXT:    retl
+;
+; X64-LABEL: test_modf_f32:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    .cfi_def_cfa_offset 16
+; X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdi
+; X64-NEXT:    callq modff@PLT
+; X64-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X64-NEXT:    popq %rax
+; X64-NEXT:    .cfi_def_cfa_offset 8
+; X64-NEXT:    retq
+  %result = call { float, float } @llvm.modf.f32(float %a)
+  ret { float, float } %result
+}
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll b/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
index da4432bd88e7d..31ef44bd6b42b 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
@@ -289,31 +289,31 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, ptr %p, <8 x i32> %mask) {
 ; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [2147483647,2147483647]
-; AVX1-NEXT:    # xmm4 = mem[0,0]
+; AVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [2147483647,2147483647]
+; AVX1-NEXT:    # xmm3 = mem[0,0]
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm4
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm1, %xmm3, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm3, %xmm5
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm6
+; AVX1-NEXT:    vblendvpd %xmm6, %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [18446744071562067968,18446744071562067968]
+; AVX1-NEXT:    # xmm3 = mem[0,0]
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm0, %xmm6
+; AVX1-NEXT:    vblendvpd %xmm6, %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm5, %xmm6
+; AVX1-NEXT:    vblendvpd %xmm6, %xmm5, %xmm3, %xmm5
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm5[0,2],xmm0[0,2]
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm1, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm3, %xmm1
 ; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm4, %xmm6
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm5, %xmm4, %xmm5
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm4, %xmm6
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm6
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [18446744071562067968,18446744071562067968]
-; AVX1-NEXT:    # xmm4 = mem[0,0]
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm6
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm6
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm5, %xmm6
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm5, %xmm4, %xmm5
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm3, %xmm6
-; AVX1-NEXT:    vblendvpd %xmm6, %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm3[0,2],xmm1[0,2]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm3[0,2],ymm0[4,6],ymm3[4,6]
 ; AVX1-NEXT:    vmaskmovps %ymm0, %ymm2, (%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll b/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
index 1597e13f02719..590f090c59596 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
@@ -224,29 +224,29 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, ptr %p, <8 x i32> %mask) {
 ; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    # xmm3 = mem[0,0]
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm4
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [9223372041149743103,9223372041149743103]
-; AVX1-NEXT:    # xmm5 = mem[0,0]
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm5, %xmm4
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm6 = [4294967295,4294967295]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    # xmm4 = mem[0,0]
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm5
+; AVX1-NEXT:    vmovddup {{.*#+}} xmm6 = [9223372041149743103,9223372041149743103]
 ; AVX1-NEXT:    # xmm6 = mem[0,0]
-; AVX1-NEXT:    vblendvpd %xmm4, %xmm1, %xmm6, %xmm4
-; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm5, %xmm7
-; AVX1-NEXT:    vblendvpd %xmm7, %xmm0, %xmm6, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm8
-; AVX1-NEXT:    vpcmpgtq %xmm8, %xmm5, %xmm8
-; AVX1-NEXT:    vblendvpd %xmm8, %xmm1, %xmm6, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vblendvpd %xmm3, %xmm0, %xmm6, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm5
+; AVX1-NEXT:    vmovddup {{.*#+}} xmm7 = [4294967295,4294967295]
+; AVX1-NEXT:    # xmm7 = mem[0,0]
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm3, %xmm7, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm7, %xmm0
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm3, %xmm7, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm4
+; AVX1-NEXT:    vblendvpd %xmm4, %xmm1, %xmm7, %xmm1
+; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm3[0,2]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm7, %ymm1
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[0,2],ymm0[0,2],ymm1[4,6],ymm0[4,6]
 ; AVX1-NEXT:    vmaskmovps %ymm0, %ymm2, (%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/matrix-multiply.ll b/llvm/test/CodeGen/X86/matrix-multiply.ll
index bb7ab4a666859..d723ec849f328 100644
--- a/llvm/test/CodeGen/X86/matrix-multiply.ll
+++ b/llvm/test/CodeGen/X86/matrix-multiply.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -disable-peephole -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=SSE
-; RUN: llc < %s -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
-; RUN: llc < %s -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
-; RUN: llc < %s -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512,AVX512F
-; RUN: llc < %s -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512,AVX512VL
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX1OR2,AVX1
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX1OR2,AVX2
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
 
 ;
 ; Basic matrix multiply tests based on the pattern:
@@ -117,22 +117,38 @@ define <4 x double> @test_mul2x2_f64(<4 x double> %a0, <4 x double> %a1) nounwin
 ; SSE-NEXT:    movapd %xmm4, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: test_mul2x2_f64:
-; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX-NEXT:    vmovddup {{.*#+}} xmm3 = xmm1[0,0]
-; AVX-NEXT:    vmulpd %xmm3, %xmm0, %xmm3
-; AVX-NEXT:    vshufpd {{.*#+}} xmm4 = xmm1[1,1]
-; AVX-NEXT:    vmulpd %xmm4, %xmm2, %xmm4
-; AVX-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
-; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX-NEXT:    vmovddup {{.*#+}} xmm4 = xmm1[0,0]
-; AVX-NEXT:    vmulpd %xmm4, %xmm0, %xmm0
-; AVX-NEXT:    vshufpd {{.*#+}} xmm1 = xmm1[1,1]
-; AVX-NEXT:    vmulpd %xmm1, %xmm2, %xmm1
-; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: test_mul2x2_f64:
+; AVX1:       # %bb.0: # %entry
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm2 = ymm1[1,1,3,3]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm0[2,3,2,3]
+; AVX1-NEXT:    vmulpd %ymm2, %ymm3, %ymm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-NEXT:    vmovddup {{.*#+}} ymm1 = ymm1[0,0,2,2]
+; AVX1-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_mul2x2_f64:
+; AVX2:       # %bb.0: # %entry
+; AVX2-NEXT:    vshufpd {{.*#+}} ymm2 = ymm1[1,1,3,3]
+; AVX2-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm0[2,3,2,3]
+; AVX2-NEXT:    vmulpd %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vmovddup {{.*#+}} ymm1 = ymm1[0,0,2,2]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX2-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: test_mul2x2_f64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vshufpd {{.*#+}} ymm2 = ymm1[1,1,3,3]
+; AVX512-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm0[2,3,2,3]
+; AVX512-NEXT:    vmulpd %ymm2, %ymm3, %ymm2
+; AVX512-NEXT:    vmovddup {{.*#+}} ymm1 = ymm1[0,0,2,2]
+; AVX512-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX512-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
+; AVX512-NEXT:    retq
 entry:
   %split = shufflevector <4 x double> %a0, <4 x double> poison, <2 x i32> <i32 0, i32 1>
   %split1 = shufflevector <4 x double> %a0, <4 x double> poison, <2 x i32> <i32 2, i32 3>
@@ -960,225 +976,86 @@ define <16 x float> @test_mul4x4_f32(<16 x float> %a0, <16 x float> %a1) nounwin
 ;
 ; AVX1-LABEL: test_mul4x4_f32:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; AVX1-NEXT:    vshufps {{.*#+}} xmm6 = xmm2[0,0,0,0]
-; AVX1-NEXT:    vmulps %xmm6, %xmm0, %xmm6
-; AVX1-NEXT:    vshufps {{.*#+}} xmm7 = xmm2[1,1,1,1]
-; AVX1-NEXT:    vmulps %xmm7, %xmm5, %xmm7
-; AVX1-NEXT:    vaddps %xmm7, %xmm6, %xmm6
-; AVX1-NEXT:    vshufps {{.*#+}} xmm7 = xmm2[2,2,2,2]
-; AVX1-NEXT:    vmulps %xmm7, %xmm1, %xmm7
-; AVX1-NEXT:    vaddps %xmm7, %xmm6, %xmm6
-; AVX1-NEXT:    vshufps {{.*#+}} xmm7 = xmm2[3,3,3,3]
-; AVX1-NEXT:    vmulps %xmm7, %xmm4, %xmm7
-; AVX1-NEXT:    vaddps %xmm7, %xmm6, %xmm6
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm2
-; AVX1-NEXT:    vshufps {{.*#+}} xmm7 = xmm2[0,0,0,0]
-; AVX1-NEXT:    vmulps %xmm7, %xmm0, %xmm7
-; AVX1-NEXT:    vshufps {{.*#+}} xmm8 = xmm2[1,1,1,1]
-; AVX1-NEXT:    vmulps %xmm5, %xmm8, %xmm8
-; AVX1-NEXT:    vaddps %xmm7, %xmm8, %xmm7
-; AVX1-NEXT:    vshufps {{.*#+}} xmm8 = xmm2[2,2,2,2]
-; AVX1-NEXT:    vmulps %xmm1, %xmm8, %xmm8
-; AVX1-NEXT:    vaddps %xmm7, %xmm8, %xmm7
-; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX1-NEXT:    vmulps %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vaddps %xmm2, %xmm7, %xmm2
-; AVX1-NEXT:    vshufps {{.*#+}} xmm7 = xmm3[0,0,0,0]
-; AVX1-NEXT:    vmulps %xmm7, %xmm0, %xmm7
-; AVX1-NEXT:    vshufps {{.*#+}} xmm8 = xmm3[1,1,1,1]
-; AVX1-NEXT:    vmulps %xmm5, %xmm8, %xmm8
-; AVX1-NEXT:    vaddps %xmm7, %xmm8, %xmm7
-; AVX1-NEXT:    vshufps {{.*#+}} xmm8 = xmm3[2,2,2,2]
-; AVX1-NEXT:    vmulps %xmm1, %xmm8, %xmm8
-; AVX1-NEXT:    vaddps %xmm7, %xmm8, %xmm7
-; AVX1-NEXT:    vshufps {{.*#+}} xmm8 = xmm3[3,3,3,3]
-; AVX1-NEXT:    vmulps %xmm4, %xmm8, %xmm8
-; AVX1-NEXT:    vaddps %xmm7, %xmm8, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm3
-; AVX1-NEXT:    vshufps {{.*#+}} xmm8 = xmm3[0,0,0,0]
-; AVX1-NEXT:    vmulps %xmm0, %xmm8, %xmm0
-; AVX1-NEXT:    vshufps {{.*#+}} xmm8 = xmm3[1,1,1,1]
-; AVX1-NEXT:    vmulps %xmm5, %xmm8, %xmm5
-; AVX1-NEXT:    vaddps %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vshufps {{.*#+}} xmm5 = xmm3[2,2,2,2]
-; AVX1-NEXT:    vmulps %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vaddps %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm3[3,3,3,3]
-; AVX1-NEXT:    vmulps %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vaddps %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm6, %ymm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm7, %ymm1
+; AVX1-NEXT:    vshufps {{.*#+}} ymm4 = ymm2[1,1,1,1,5,5,5,5]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm0[2,3,2,3]
+; AVX1-NEXT:    vmulps %ymm4, %ymm5, %ymm4
+; AVX1-NEXT:    vshufps {{.*#+}} ymm6 = ymm2[0,0,0,0,4,4,4,4]
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm7
+; AVX1-NEXT:    vmulps %ymm6, %ymm7, %ymm0
+; AVX1-NEXT:    vaddps %ymm4, %ymm0, %ymm0
+; AVX1-NEXT:    vshufps {{.*#+}} ymm4 = ymm2[2,2,2,2,6,6,6,6]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm6
+; AVX1-NEXT:    vmulps %ymm4, %ymm6, %ymm4
+; AVX1-NEXT:    vaddps %ymm4, %ymm0, %ymm0
+; AVX1-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[3,3,3,3,7,7,7,7]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
+; AVX1-NEXT:    vmulps %ymm2, %ymm1, %ymm2
+; AVX1-NEXT:    vaddps %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    vshufps {{.*#+}} ymm2 = ymm3[1,1,1,1,5,5,5,5]
+; AVX1-NEXT:    vmulps %ymm2, %ymm5, %ymm2
+; AVX1-NEXT:    vshufps {{.*#+}} ymm4 = ymm3[0,0,0,0,4,4,4,4]
+; AVX1-NEXT:    vmulps %ymm4, %ymm7, %ymm4
+; AVX1-NEXT:    vaddps %ymm2, %ymm4, %ymm2
+; AVX1-NEXT:    vshufps {{.*#+}} ymm4 = ymm3[2,2,2,2,6,6,6,6]
+; AVX1-NEXT:    vmulps %ymm4, %ymm6, %ymm4
+; AVX1-NEXT:    vaddps %ymm4, %ymm2, %ymm2
+; AVX1-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[3,3,3,3,7,7,7,7]
+; AVX1-NEXT:    vmulps %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    vaddps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: test_mul4x4_f32:
 ; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX2-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; AVX2-NEXT:    vbroadcastss %xmm2, %xmm6
-; AVX2-NEXT:    vmulps %xmm6, %xmm0, %xmm6
-; AVX2-NEXT:    vshufps {{.*#+}} xmm7 = xmm2[1,1,1,1]
-; AVX2-NEXT:    vmulps %xmm7, %xmm5, %xmm7
-; AVX2-NEXT:    vaddps %xmm7, %xmm6, %xmm6
-; AVX2-NEXT:    vshufps {{.*#+}} xmm7 = xmm2[2,2,2,2]
-; AVX2-NEXT:    vmulps %xmm7, %xmm1, %xmm7
-; AVX2-NEXT:    vaddps %xmm7, %xmm6, %xmm6
-; AVX2-NEXT:    vshufps {{.*#+}} xmm7 = xmm2[3,3,3,3]
-; AVX2-NEXT:    vmulps %xmm7, %xmm4, %xmm7
-; AVX2-NEXT:    vaddps %xmm7, %xmm6, %xmm6
-; AVX2-NEXT:    vextractf128 $1, %ymm2, %xmm2
-; AVX2-NEXT:    vbroadcastss %xmm2, %xmm7
-; AVX2-NEXT:    vmulps %xmm7, %xmm0, %xmm7
-; AVX2-NEXT:    vshufps {{.*#+}} xmm8 = xmm2[1,1,1,1]
-; AVX2-NEXT:    vmulps %xmm5, %xmm8, %xmm8
-; AVX2-NEXT:    vaddps %xmm7, %xmm8, %xmm7
-; AVX2-NEXT:    vshufps {{.*#+}} xmm8 = xmm2[2,2,2,2]
-; AVX2-NEXT:    vmulps %xmm1, %xmm8, %xmm8
-; AVX2-NEXT:    vaddps %xmm7, %xmm8, %xmm7
-; AVX2-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX2-NEXT:    vmulps %xmm2, %xmm4, %xmm2
-; AVX2-NEXT:    vaddps %xmm2, %xmm7, %xmm2
-; AVX2-NEXT:    vbroadcastss %xmm3, %xmm7
-; AVX2-NEXT:    vmulps %xmm7, %xmm0, %xmm7
-; AVX2-NEXT:    vshufps {{.*#+}} xmm8 = xmm3[1,1,1,1]
-; AVX2-NEXT:    vmulps %xmm5, %xmm8, %xmm8
-; AVX2-NEXT:    vaddps %xmm7, %xmm8, %xmm7
-; AVX2-NEXT:    vshufps {{.*#+}} xmm8 = xmm3[2,2,2,2]
-; AVX2-NEXT:    vmulps %xmm1, %xmm8, %xmm8
-; AVX2-NEXT:    vaddps %xmm7, %xmm8, %xmm7
-; AVX2-NEXT:    vshufps {{.*#+}} xmm8 = xmm3[3,3,3,3]
-; AVX2-NEXT:    vmulps %xmm4, %xmm8, %xmm8
-; AVX2-NEXT:    vaddps %xmm7, %xmm8, %xmm7
-; AVX2-NEXT:    vextractf128 $1, %ymm3, %xmm3
-; AVX2-NEXT:    vbroadcastss %xmm3, %xmm8
-; AVX2-NEXT:    vmulps %xmm0, %xmm8, %xmm0
-; AVX2-NEXT:    vshufps {{.*#+}} xmm8 = xmm3[1,1,1,1]
-; AVX2-NEXT:    vmulps %xmm5, %xmm8, %xmm5
-; AVX2-NEXT:    vaddps %xmm5, %xmm0, %xmm0
-; AVX2-NEXT:    vshufps {{.*#+}} xmm5 = xmm3[2,2,2,2]
-; AVX2-NEXT:    vmulps %xmm5, %xmm1, %xmm1
-; AVX2-NEXT:    vaddps %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vshufps {{.*#+}} xmm1 = xmm3[3,3,3,3]
-; AVX2-NEXT:    vmulps %xmm1, %xmm4, %xmm1
-; AVX2-NEXT:    vaddps %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vinsertf128 $1, %xmm2, %ymm6, %ymm0
-; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm7, %ymm1
+; AVX2-NEXT:    vshufps {{.*#+}} ymm4 = ymm2[1,1,1,1,5,5,5,5]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm5 = ymm0[2,3,2,3]
+; AVX2-NEXT:    vmulps %ymm4, %ymm5, %ymm4
+; AVX2-NEXT:    vshufps {{.*#+}} ymm6 = ymm2[0,0,0,0,4,4,4,4]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm7 = ymm0[0,1,0,1]
+; AVX2-NEXT:    vmulps %ymm6, %ymm7, %ymm0
+; AVX2-NEXT:    vaddps %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vshufps {{.*#+}} ymm4 = ymm2[2,2,2,2,6,6,6,6]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm6 = ymm1[0,1,0,1]
+; AVX2-NEXT:    vmulps %ymm4, %ymm6, %ymm4
+; AVX2-NEXT:    vaddps %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[3,3,3,3,7,7,7,7]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[2,3,2,3]
+; AVX2-NEXT:    vmulps %ymm2, %ymm1, %ymm2
+; AVX2-NEXT:    vaddps %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vshufps {{.*#+}} ymm2 = ymm3[1,1,1,1,5,5,5,5]
+; AVX2-NEXT:    vmulps %ymm2, %ymm5, %ymm2
+; AVX2-NEXT:    vshufps {{.*#+}} ymm4 = ymm3[0,0,0,0,4,4,4,4]
+; AVX2-NEXT:    vmulps %ymm4, %ymm7, %ymm4
+; AVX2-NEXT:    vaddps %ymm2, %ymm4, %ymm2
+; AVX2-NEXT:    vshufps {{.*#+}} ymm4 = ymm3[2,2,2,2,6,6,6,6]
+; AVX2-NEXT:    vmulps %ymm4, %ymm6, %ymm4
+; AVX2-NEXT:    vaddps %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[3,3,3,3,7,7,7,7]
+; AVX2-NEXT:    vmulps %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vaddps %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: test_mul4x4_f32:
-; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; AVX512F-NEXT:    vextractf32x4 $2, %zmm0, %xmm3
-; AVX512F-NEXT:    vextractf32x4 $3, %zmm0, %xmm2
-; AVX512F-NEXT:    vbroadcastss %xmm1, %xmm5
-; AVX512F-NEXT:    vmulps %xmm5, %xmm0, %xmm5
-; AVX512F-NEXT:    vshufps {{.*#+}} xmm6 = xmm1[1,1,1,1]
-; AVX512F-NEXT:    vmulps %xmm6, %xmm4, %xmm6
-; AVX512F-NEXT:    vaddps %xmm6, %xmm5, %xmm5
-; AVX512F-NEXT:    vshufps {{.*#+}} xmm6 = xmm1[2,2,2,2]
-; AVX512F-NEXT:    vmulps %xmm6, %xmm3, %xmm6
-; AVX512F-NEXT:    vaddps %xmm6, %xmm5, %xmm5
-; AVX512F-NEXT:    vshufps {{.*#+}} xmm6 = xmm1[3,3,3,3]
-; AVX512F-NEXT:    vmulps %xmm6, %xmm2, %xmm6
-; AVX512F-NEXT:    vaddps %xmm6, %xmm5, %xmm5
-; AVX512F-NEXT:    vextractf128 $1, %ymm1, %xmm6
-; AVX512F-NEXT:    vbroadcastss %xmm6, %xmm7
-; AVX512F-NEXT:    vmulps %xmm7, %xmm0, %xmm7
-; AVX512F-NEXT:    vshufps {{.*#+}} xmm8 = xmm6[1,1,1,1]
-; AVX512F-NEXT:    vmulps %xmm4, %xmm8, %xmm8
-; AVX512F-NEXT:    vaddps %xmm7, %xmm8, %xmm7
-; AVX512F-NEXT:    vshufps {{.*#+}} xmm8 = xmm6[2,2,2,2]
-; AVX512F-NEXT:    vmulps %xmm3, %xmm8, %xmm8
-; AVX512F-NEXT:    vaddps %xmm7, %xmm8, %xmm7
-; AVX512F-NEXT:    vshufps {{.*#+}} xmm6 = xmm6[3,3,3,3]
-; AVX512F-NEXT:    vmulps %xmm6, %xmm2, %xmm6
-; AVX512F-NEXT:    vaddps %xmm6, %xmm7, %xmm6
-; AVX512F-NEXT:    vextractf32x4 $2, %zmm1, %xmm7
-; AVX512F-NEXT:    vbroadcastss %xmm7, %xmm8
-; AVX512F-NEXT:    vmulps %xmm0, %xmm8, %xmm8
-; AVX512F-NEXT:    vshufps {{.*#+}} xmm9 = xmm7[1,1,1,1]
-; AVX512F-NEXT:    vmulps %xmm4, %xmm9, %xmm9
-; AVX512F-NEXT:    vaddps %xmm9, %xmm8, %xmm8
-; AVX512F-NEXT:    vshufps {{.*#+}} xmm9 = xmm7[2,2,2,2]
-; AVX512F-NEXT:    vmulps %xmm3, %xmm9, %xmm9
-; AVX512F-NEXT:    vaddps %xmm9, %xmm8, %xmm8
-; AVX512F-NEXT:    vshufps {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; AVX512F-NEXT:    vmulps %xmm7, %xmm2, %xmm7
-; AVX512F-NEXT:    vaddps %xmm7, %xmm8, %xmm7
-; AVX512F-NEXT:    vextractf32x4 $3, %zmm1, %xmm1
-; AVX512F-NEXT:    vbroadcastss %xmm1, %xmm8
-; AVX512F-NEXT:    vmulps %xmm0, %xmm8, %xmm0
-; AVX512F-NEXT:    vshufps {{.*#+}} xmm8 = xmm1[1,1,1,1]
-; AVX512F-NEXT:    vmulps %xmm4, %xmm8, %xmm4
-; AVX512F-NEXT:    vaddps %xmm4, %xmm0, %xmm0
-; AVX512F-NEXT:    vshufps {{.*#+}} xmm4 = xmm1[2,2,2,2]
-; AVX512F-NEXT:    vmulps %xmm4, %xmm3, %xmm3
-; AVX512F-NEXT:    vaddps %xmm3, %xmm0, %xmm0
-; AVX512F-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX512F-NEXT:    vmulps %xmm1, %xmm2, %xmm1
-; AVX512F-NEXT:    vaddps %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm7, %ymm0
-; AVX512F-NEXT:    vinsertf128 $1, %xmm6, %ymm5, %ymm1
-; AVX512F-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: test_mul4x4_f32:
-; AVX512VL:       # %bb.0: # %entry
-; AVX512VL-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX512VL-NEXT:    vextractf32x4 $2, %zmm0, %xmm3
-; AVX512VL-NEXT:    vextractf32x4 $3, %zmm0, %xmm4
-; AVX512VL-NEXT:    vbroadcastss %xmm1, %xmm5
-; AVX512VL-NEXT:    vmulps %xmm5, %xmm0, %xmm5
-; AVX512VL-NEXT:    vshufps {{.*#+}} xmm6 = xmm1[1,1,1,1]
-; AVX512VL-NEXT:    vmulps %xmm6, %xmm2, %xmm6
-; AVX512VL-NEXT:    vaddps %xmm6, %xmm5, %xmm5
-; AVX512VL-NEXT:    vshufps {{.*#+}} xmm6 = xmm1[2,2,2,2]
-; AVX512VL-NEXT:    vmulps %xmm6, %xmm3, %xmm6
-; AVX512VL-NEXT:    vaddps %xmm6, %xmm5, %xmm5
-; AVX512VL-NEXT:    vshufps {{.*#+}} xmm6 = xmm1[3,3,3,3]
-; AVX512VL-NEXT:    vmulps %xmm6, %xmm4, %xmm6
-; AVX512VL-NEXT:    vaddps %xmm6, %xmm5, %xmm5
-; AVX512VL-NEXT:    vextractf128 $1, %ymm1, %xmm6
-; AVX512VL-NEXT:    vbroadcastss %xmm6, %xmm7
-; AVX512VL-NEXT:    vmulps %xmm7, %xmm0, %xmm7
-; AVX512VL-NEXT:    vshufps {{.*#+}} xmm8 = xmm6[1,1,1,1]
-; AVX512VL-NEXT:    vmulps %xmm2, %xmm8, %xmm8
-; AVX512VL-NEXT:    vaddps %xmm7, %xmm8, %xmm7
-; AVX512VL-NEXT:    vshufps {{.*#+}} xmm8 = xmm6[2,2,2,2]
-; AVX512VL-NEXT:    vmulps %xmm3, %xmm8, %xmm8
-; AVX512VL-NEXT:    vaddps %xmm7, %xmm8, %xmm7
-; AVX512VL-NEXT:    vshufps {{.*#+}} xmm6 = xmm6[3,3,3,3]
-; AVX512VL-NEXT:    vmulps %xmm6, %xmm4, %xmm6
-; AVX512VL-NEXT:    vaddps %xmm6, %xmm7, %xmm6
-; AVX512VL-NEXT:    vextractf32x4 $2, %zmm1, %xmm7
-; AVX512VL-NEXT:    vbroadcastss %xmm7, %xmm8
-; AVX512VL-NEXT:    vmulps %xmm0, %xmm8, %xmm8
-; AVX512VL-NEXT:    vshufps {{.*#+}} xmm9 = xmm7[1,1,1,1]
-; AVX512VL-NEXT:    vmulps %xmm2, %xmm9, %xmm9
-; AVX512VL-NEXT:    vaddps %xmm9, %xmm8, %xmm8
-; AVX512VL-NEXT:    vshufps {{.*#+}} xmm9 = xmm7[2,2,2,2]
-; AVX512VL-NEXT:    vmulps %xmm3, %xmm9, %xmm9
-; AVX512VL-NEXT:    vaddps %xmm9, %xmm8, %xmm8
-; AVX512VL-NEXT:    vshufps {{.*#+}} xmm7 = xmm7[3,3,3,3]
-; AVX512VL-NEXT:    vmulps %xmm7, %xmm4, %xmm7
-; AVX512VL-NEXT:    vaddps %xmm7, %xmm8, %xmm7
-; AVX512VL-NEXT:    vextractf32x4 $3, %zmm1, %xmm1
-; AVX512VL-NEXT:    vbroadcastss %xmm1, %xmm8
-; AVX512VL-NEXT:    vmulps %xmm0, %xmm8, %xmm0
-; AVX512VL-NEXT:    vshufps {{.*#+}} xmm8 = xmm1[1,1,1,1]
-; AVX512VL-NEXT:    vmulps %xmm2, %xmm8, %xmm2
-; AVX512VL-NEXT:    vaddps %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[2,2,2,2]
-; AVX512VL-NEXT:    vmulps %xmm2, %xmm3, %xmm2
-; AVX512VL-NEXT:    vaddps %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX512VL-NEXT:    vmulps %xmm1, %xmm4, %xmm1
-; AVX512VL-NEXT:    vaddps %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    vinsertf128 $1, %xmm0, %ymm7, %ymm0
-; AVX512VL-NEXT:    vinsertf128 $1, %xmm6, %ymm5, %ymm1
-; AVX512VL-NEXT:    vinsertf64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512VL-NEXT:    retq
+; AVX512-LABEL: test_mul4x4_f32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vextractf32x4 $2, %zmm1, %xmm2
+; AVX512-NEXT:    vextractf32x4 $3, %zmm1, %xmm3
+; AVX512-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX512-NEXT:    vinsertf64x4 $1, %ymm2, %zmm1, %zmm1
+; AVX512-NEXT:    vshufps {{.*#+}} zmm2 = zmm1[1,1,1,1,5,5,5,5,9,9,9,9,13,13,13,13]
+; AVX512-NEXT:    vshuff64x2 {{.*#+}} zmm3 = zmm0[2,3,2,3,2,3,2,3]
+; AVX512-NEXT:    vmulps %zmm2, %zmm3, %zmm2
+; AVX512-NEXT:    vshufps {{.*#+}} zmm3 = zmm1[0,0,0,0,4,4,4,4,8,8,8,8,12,12,12,12]
+; AVX512-NEXT:    vshuff64x2 {{.*#+}} zmm4 = zmm0[0,1,0,1,0,1,0,1]
+; AVX512-NEXT:    vmulps %zmm3, %zmm4, %zmm3
+; AVX512-NEXT:    vaddps %zmm2, %zmm3, %zmm2
+; AVX512-NEXT:    vshufps {{.*#+}} zmm3 = zmm1[2,2,2,2,6,6,6,6,10,10,10,10,14,14,14,14]
+; AVX512-NEXT:    vshuff64x2 {{.*#+}} zmm4 = zmm0[4,5,4,5,4,5,4,5]
+; AVX512-NEXT:    vmulps %zmm3, %zmm4, %zmm3
+; AVX512-NEXT:    vaddps %zmm3, %zmm2, %zmm2
+; AVX512-NEXT:    vshufps {{.*#+}} zmm1 = zmm1[3,3,3,3,7,7,7,7,11,11,11,11,15,15,15,15]
+; AVX512-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[6,7,6,7,6,7,6,7]
+; AVX512-NEXT:    vmulps %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vaddps %zmm0, %zmm2, %zmm0
+; AVX512-NEXT:    retq
 entry:
   %split = shufflevector <16 x float> %a0, <16 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %split1 = shufflevector <16 x float> %a0, <16 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -1472,113 +1349,42 @@ define <16 x double> @test_mul4x4_f64(<16 x double> %a0, <16 x double> %a1) noun
 ; AVX2-NEXT:    vmovapd %ymm6, %ymm2
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: test_mul4x4_f64:
-; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    vextractf64x4 $1, %zmm0, %ymm5
-; AVX512F-NEXT:    vextractf64x4 $1, %zmm1, %ymm4
-; AVX512F-NEXT:    vbroadcastsd %xmm2, %ymm6
-; AVX512F-NEXT:    vmulpd %ymm6, %ymm0, %ymm6
-; AVX512F-NEXT:    vpermpd {{.*#+}} ymm7 = ymm2[1,1,1,1]
-; AVX512F-NEXT:    vmulpd %ymm7, %ymm5, %ymm7
-; AVX512F-NEXT:    vaddpd %ymm7, %ymm6, %ymm6
-; AVX512F-NEXT:    vpermpd {{.*#+}} ymm7 = ymm2[2,2,2,2]
-; AVX512F-NEXT:    vmulpd %ymm7, %ymm1, %ymm7
-; AVX512F-NEXT:    vaddpd %ymm7, %ymm6, %ymm6
-; AVX512F-NEXT:    vpermpd {{.*#+}} ymm7 = ymm2[3,3,3,3]
-; AVX512F-NEXT:    vmulpd %ymm7, %ymm4, %ymm7
-; AVX512F-NEXT:    vaddpd %ymm7, %ymm6, %ymm6
-; AVX512F-NEXT:    vextractf64x4 $1, %zmm2, %ymm2
-; AVX512F-NEXT:    vbroadcastsd %xmm2, %ymm7
-; AVX512F-NEXT:    vmulpd %ymm7, %ymm0, %ymm7
-; AVX512F-NEXT:    vpermpd {{.*#+}} ymm8 = ymm2[1,1,1,1]
-; AVX512F-NEXT:    vmulpd %ymm5, %ymm8, %ymm8
-; AVX512F-NEXT:    vaddpd %ymm7, %ymm8, %ymm7
-; AVX512F-NEXT:    vpermpd {{.*#+}} ymm8 = ymm2[2,2,2,2]
-; AVX512F-NEXT:    vmulpd %ymm1, %ymm8, %ymm8
-; AVX512F-NEXT:    vaddpd %ymm7, %ymm8, %ymm7
-; AVX512F-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[3,3,3,3]
-; AVX512F-NEXT:    vmulpd %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vaddpd %ymm2, %ymm7, %ymm2
-; AVX512F-NEXT:    vbroadcastsd %xmm3, %ymm7
-; AVX512F-NEXT:    vmulpd %ymm7, %ymm0, %ymm7
-; AVX512F-NEXT:    vpermpd {{.*#+}} ymm8 = ymm3[1,1,1,1]
-; AVX512F-NEXT:    vmulpd %ymm5, %ymm8, %ymm8
-; AVX512F-NEXT:    vaddpd %ymm7, %ymm8, %ymm7
-; AVX512F-NEXT:    vpermpd {{.*#+}} ymm8 = ymm3[2,2,2,2]
-; AVX512F-NEXT:    vmulpd %ymm1, %ymm8, %ymm8
-; AVX512F-NEXT:    vaddpd %ymm7, %ymm8, %ymm7
-; AVX512F-NEXT:    vpermpd {{.*#+}} ymm8 = ymm3[3,3,3,3]
-; AVX512F-NEXT:    vmulpd %ymm4, %ymm8, %ymm8
-; AVX512F-NEXT:    vaddpd %ymm7, %ymm8, %ymm7
-; AVX512F-NEXT:    vextractf64x4 $1, %zmm3, %ymm3
-; AVX512F-NEXT:    vbroadcastsd %xmm3, %ymm8
-; AVX512F-NEXT:    vmulpd %ymm0, %ymm8, %ymm0
-; AVX512F-NEXT:    vpermpd {{.*#+}} ymm8 = ymm3[1,1,1,1]
-; AVX512F-NEXT:    vmulpd %ymm5, %ymm8, %ymm5
-; AVX512F-NEXT:    vaddpd %ymm5, %ymm0, %ymm0
-; AVX512F-NEXT:    vpermpd {{.*#+}} ymm5 = ymm3[2,2,2,2]
-; AVX512F-NEXT:    vmulpd %ymm5, %ymm1, %ymm1
-; AVX512F-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpermpd {{.*#+}} ymm1 = ymm3[3,3,3,3]
-; AVX512F-NEXT:    vmulpd %ymm1, %ymm4, %ymm1
-; AVX512F-NEXT:    vaddpd %ymm1, %ymm0, %ymm1
-; AVX512F-NEXT:    vinsertf64x4 $1, %ymm2, %zmm6, %zmm0
-; AVX512F-NEXT:    vinsertf64x4 $1, %ymm1, %zmm7, %zmm1
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: test_mul4x4_f64:
-; AVX512VL:       # %bb.0: # %entry
-; AVX512VL-NEXT:    vextractf64x4 $1, %zmm0, %ymm4
-; AVX512VL-NEXT:    vextractf64x4 $1, %zmm1, %ymm5
-; AVX512VL-NEXT:    vbroadcastsd %xmm2, %ymm6
-; AVX512VL-NEXT:    vmulpd %ymm6, %ymm0, %ymm6
-; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm7 = ymm2[1,1,1,1]
-; AVX512VL-NEXT:    vmulpd %ymm7, %ymm4, %ymm7
-; AVX512VL-NEXT:    vaddpd %ymm7, %ymm6, %ymm6
-; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm7 = ymm2[2,2,2,2]
-; AVX512VL-NEXT:    vmulpd %ymm7, %ymm1, %ymm7
-; AVX512VL-NEXT:    vaddpd %ymm7, %ymm6, %ymm6
-; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm7 = ymm2[3,3,3,3]
-; AVX512VL-NEXT:    vmulpd %ymm7, %ymm5, %ymm7
-; AVX512VL-NEXT:    vaddpd %ymm7, %ymm6, %ymm6
-; AVX512VL-NEXT:    vextractf64x4 $1, %zmm2, %ymm2
-; AVX512VL-NEXT:    vbroadcastsd %xmm2, %ymm7
-; AVX512VL-NEXT:    vmulpd %ymm7, %ymm0, %ymm7
-; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm8 = ymm2[1,1,1,1]
-; AVX512VL-NEXT:    vmulpd %ymm4, %ymm8, %ymm8
-; AVX512VL-NEXT:    vaddpd %ymm7, %ymm8, %ymm7
-; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm8 = ymm2[2,2,2,2]
-; AVX512VL-NEXT:    vmulpd %ymm1, %ymm8, %ymm8
-; AVX512VL-NEXT:    vaddpd %ymm7, %ymm8, %ymm7
-; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[3,3,3,3]
-; AVX512VL-NEXT:    vmulpd %ymm2, %ymm5, %ymm2
-; AVX512VL-NEXT:    vaddpd %ymm2, %ymm7, %ymm2
-; AVX512VL-NEXT:    vbroadcastsd %xmm3, %ymm7
-; AVX512VL-NEXT:    vmulpd %ymm7, %ymm0, %ymm7
-; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm8 = ymm3[1,1,1,1]
-; AVX512VL-NEXT:    vmulpd %ymm4, %ymm8, %ymm8
-; AVX512VL-NEXT:    vaddpd %ymm7, %ymm8, %ymm7
-; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm8 = ymm3[2,2,2,2]
-; AVX512VL-NEXT:    vmulpd %ymm1, %ymm8, %ymm8
-; AVX512VL-NEXT:    vaddpd %ymm7, %ymm8, %ymm7
-; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm8 = ymm3[3,3,3,3]
-; AVX512VL-NEXT:    vmulpd %ymm5, %ymm8, %ymm8
-; AVX512VL-NEXT:    vaddpd %ymm7, %ymm8, %ymm7
-; AVX512VL-NEXT:    vextractf64x4 $1, %zmm3, %ymm3
-; AVX512VL-NEXT:    vbroadcastsd %xmm3, %ymm8
-; AVX512VL-NEXT:    vmulpd %ymm0, %ymm8, %ymm0
-; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm8 = ymm3[1,1,1,1]
-; AVX512VL-NEXT:    vmulpd %ymm4, %ymm8, %ymm4
-; AVX512VL-NEXT:    vaddpd %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm4 = ymm3[2,2,2,2]
-; AVX512VL-NEXT:    vmulpd %ymm4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpermpd {{.*#+}} ymm1 = ymm3[3,3,3,3]
-; AVX512VL-NEXT:    vmulpd %ymm1, %ymm5, %ymm1
-; AVX512VL-NEXT:    vaddpd %ymm1, %ymm0, %ymm1
-; AVX512VL-NEXT:    vinsertf64x4 $1, %ymm2, %zmm6, %zmm0
-; AVX512VL-NEXT:    vinsertf64x4 $1, %ymm1, %zmm7, %zmm1
-; AVX512VL-NEXT:    retq
+; AVX512-LABEL: test_mul4x4_f64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    vbroadcastsd %xmm2, %ymm4
+; AVX512-NEXT:    vmulpd %ymm4, %ymm0, %ymm4
+; AVX512-NEXT:    vextractf64x4 $1, %zmm2, %ymm5
+; AVX512-NEXT:    vbroadcastsd %xmm5, %ymm5
+; AVX512-NEXT:    vmulpd %ymm5, %ymm0, %ymm5
+; AVX512-NEXT:    vbroadcastsd %xmm3, %ymm6
+; AVX512-NEXT:    vmulpd %ymm6, %ymm0, %ymm6
+; AVX512-NEXT:    vextractf64x4 $1, %zmm3, %ymm7
+; AVX512-NEXT:    vbroadcastsd %xmm7, %ymm7
+; AVX512-NEXT:    vmulpd %ymm7, %ymm0, %ymm7
+; AVX512-NEXT:    vinsertf64x4 $1, %ymm5, %zmm4, %zmm4
+; AVX512-NEXT:    vpermpd {{.*#+}} zmm5 = zmm2[1,1,1,1,5,5,5,5]
+; AVX512-NEXT:    vshuff64x2 {{.*#+}} zmm8 = zmm0[4,5,6,7,4,5,6,7]
+; AVX512-NEXT:    vmulpd %zmm5, %zmm8, %zmm0
+; AVX512-NEXT:    vaddpd %zmm0, %zmm4, %zmm0
+; AVX512-NEXT:    vinsertf64x4 $1, %ymm1, %zmm1, %zmm4
+; AVX512-NEXT:    vpermpd {{.*#+}} zmm5 = zmm2[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vmulpd %zmm5, %zmm4, %zmm5
+; AVX512-NEXT:    vaddpd %zmm5, %zmm0, %zmm0
+; AVX512-NEXT:    vpermpd {{.*#+}} zmm2 = zmm2[3,3,3,3,7,7,7,7]
+; AVX512-NEXT:    vshuff64x2 {{.*#+}} zmm1 = zmm1[4,5,6,7,4,5,6,7]
+; AVX512-NEXT:    vmulpd %zmm2, %zmm1, %zmm2
+; AVX512-NEXT:    vaddpd %zmm2, %zmm0, %zmm0
+; AVX512-NEXT:    vinsertf64x4 $1, %ymm7, %zmm6, %zmm2
+; AVX512-NEXT:    vpermpd {{.*#+}} zmm5 = zmm3[1,1,1,1,5,5,5,5]
+; AVX512-NEXT:    vmulpd %zmm5, %zmm8, %zmm5
+; AVX512-NEXT:    vaddpd %zmm5, %zmm2, %zmm2
+; AVX512-NEXT:    vpermpd {{.*#+}} zmm5 = zmm3[2,2,2,2,6,6,6,6]
+; AVX512-NEXT:    vmulpd %zmm5, %zmm4, %zmm4
+; AVX512-NEXT:    vaddpd %zmm4, %zmm2, %zmm2
+; AVX512-NEXT:    vpermpd {{.*#+}} zmm3 = zmm3[3,3,3,3,7,7,7,7]
+; AVX512-NEXT:    vmulpd %zmm3, %zmm1, %zmm1
+; AVX512-NEXT:    vaddpd %zmm1, %zmm2, %zmm1
+; AVX512-NEXT:    retq
 entry:
   %split = shufflevector <16 x double> %a0, <16 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %split1 = shufflevector <16 x double> %a0, <16 x double> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -2146,413 +1952,209 @@ define <64 x float> @test_mul8x8_f32(<64 x float> %a0, <64 x float> %a1) nounwin
 ; SSE-NEXT:    addq $120, %rsp
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: test_mul8x8_f32:
-; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    pushq %rbp
-; AVX1-NEXT:    movq %rsp, %rbp
-; AVX1-NEXT:    andq $-32, %rsp
-; AVX1-NEXT:    subq $32, %rsp
-; AVX1-NEXT:    movq %rdi, %rax
-; AVX1-NEXT:    vbroadcastss 16(%rbp), %ymm8
-; AVX1-NEXT:    vmulps %ymm0, %ymm8, %ymm8
-; AVX1-NEXT:    vbroadcastss 20(%rbp), %ymm9
-; AVX1-NEXT:    vmulps %ymm1, %ymm9, %ymm9
-; AVX1-NEXT:    vaddps %ymm9, %ymm8, %ymm8
-; AVX1-NEXT:    vbroadcastss 24(%rbp), %ymm9
-; AVX1-NEXT:    vmulps %ymm2, %ymm9, %ymm9
-; AVX1-NEXT:    vaddps %ymm9, %ymm8, %ymm8
-; AVX1-NEXT:    vbroadcastss 28(%rbp), %ymm9
-; AVX1-NEXT:    vmulps %ymm3, %ymm9, %ymm9
-; AVX1-NEXT:    vaddps %ymm9, %ymm8, %ymm8
-; AVX1-NEXT:    vbroadcastss 32(%rbp), %ymm9
-; AVX1-NEXT:    vmulps %ymm4, %ymm9, %ymm9
-; AVX1-NEXT:    vaddps %ymm9, %ymm8, %ymm8
-; AVX1-NEXT:    vbroadcastss 36(%rbp), %ymm9
-; AVX1-NEXT:    vmulps %ymm5, %ymm9, %ymm9
-; AVX1-NEXT:    vaddps %ymm9, %ymm8, %ymm8
-; AVX1-NEXT:    vbroadcastss 40(%rbp), %ymm9
-; AVX1-NEXT:    vmulps %ymm6, %ymm9, %ymm9
-; AVX1-NEXT:    vaddps %ymm9, %ymm8, %ymm8
-; AVX1-NEXT:    vbroadcastss 44(%rbp), %ymm9
-; AVX1-NEXT:    vmulps %ymm7, %ymm9, %ymm9
-; AVX1-NEXT:    vaddps %ymm9, %ymm8, %ymm8
-; AVX1-NEXT:    vbroadcastss 48(%rbp), %ymm9
-; AVX1-NEXT:    vmulps %ymm0, %ymm9, %ymm9
-; AVX1-NEXT:    vbroadcastss 52(%rbp), %ymm10
-; AVX1-NEXT:    vmulps %ymm1, %ymm10, %ymm10
-; AVX1-NEXT:    vaddps %ymm10, %ymm9, %ymm9
-; AVX1-NEXT:    vbroadcastss 56(%rbp), %ymm10
-; AVX1-NEXT:    vmulps %ymm2, %ymm10, %ymm10
-; AVX1-NEXT:    vaddps %ymm10, %ymm9, %ymm9
-; AVX1-NEXT:    vbroadcastss 60(%rbp), %ymm10
-; AVX1-NEXT:    vmulps %ymm3, %ymm10, %ymm10
-; AVX1-NEXT:    vaddps %ymm10, %ymm9, %ymm9
-; AVX1-NEXT:    vbroadcastss 64(%rbp), %ymm10
-; AVX1-NEXT:    vmulps %ymm4, %ymm10, %ymm10
-; AVX1-NEXT:    vaddps %ymm10, %ymm9, %ymm9
-; AVX1-NEXT:    vbroadcastss 68(%rbp), %ymm10
-; AVX1-NEXT:    vmulps %ymm5, %ymm10, %ymm10
-; AVX1-NEXT:    vaddps %ymm10, %ymm9, %ymm9
-; AVX1-NEXT:    vbroadcastss 72(%rbp), %ymm10
-; AVX1-NEXT:    vmulps %ymm6, %ymm10, %ymm10
-; AVX1-NEXT:    vaddps %ymm10, %ymm9, %ymm9
-; AVX1-NEXT:    vbroadcastss 76(%rbp), %ymm10
-; AVX1-NEXT:    vmulps %ymm7, %ymm10, %ymm10
-; AVX1-NEXT:    vaddps %ymm10, %ymm9, %ymm9
-; AVX1-NEXT:    vbroadcastss 80(%rbp), %ymm10
-; AVX1-NEXT:    vmulps %ymm0, %ymm10, %ymm10
-; AVX1-NEXT:    vbroadcastss 84(%rbp), %ymm11
-; AVX1-NEXT:    vmulps %ymm1, %ymm11, %ymm11
-; AVX1-NEXT:    vaddps %ymm11, %ymm10, %ymm10
-; AVX1-NEXT:    vbroadcastss 88(%rbp), %ymm11
-; AVX1-NEXT:    vmulps %ymm2, %ymm11, %ymm11
-; AVX1-NEXT:    vaddps %ymm11, %ymm10, %ymm10
-; AVX1-NEXT:    vbroadcastss 92(%rbp), %ymm11
-; AVX1-NEXT:    vmulps %ymm3, %ymm11, %ymm11
-; AVX1-NEXT:    vaddps %ymm11, %ymm10, %ymm10
-; AVX1-NEXT:    vbroadcastss 96(%rbp), %ymm11
-; AVX1-NEXT:    vmulps %ymm4, %ymm11, %ymm11
-; AVX1-NEXT:    vaddps %ymm11, %ymm10, %ymm10
-; AVX1-NEXT:    vbroadcastss 100(%rbp), %ymm11
-; AVX1-NEXT:    vmulps %ymm5, %ymm11, %ymm11
-; AVX1-NEXT:    vaddps %ymm11, %ymm10, %ymm10
-; AVX1-NEXT:    vbroadcastss 104(%rbp), %ymm11
-; AVX1-NEXT:    vmulps %ymm6, %ymm11, %ymm11
-; AVX1-NEXT:    vaddps %ymm11, %ymm10, %ymm10
-; AVX1-NEXT:    vbroadcastss 108(%rbp), %ymm11
-; AVX1-NEXT:    vmulps %ymm7, %ymm11, %ymm11
-; AVX1-NEXT:    vaddps %ymm11, %ymm10, %ymm10
-; AVX1-NEXT:    vbroadcastss 112(%rbp), %ymm11
-; AVX1-NEXT:    vmulps %ymm0, %ymm11, %ymm11
-; AVX1-NEXT:    vbroadcastss 116(%rbp), %ymm12
-; AVX1-NEXT:    vmulps %ymm1, %ymm12, %ymm12
-; AVX1-NEXT:    vaddps %ymm12, %ymm11, %ymm11
-; AVX1-NEXT:    vbroadcastss 120(%rbp), %ymm12
-; AVX1-NEXT:    vmulps %ymm2, %ymm12, %ymm12
-; AVX1-NEXT:    vaddps %ymm12, %ymm11, %ymm11
-; AVX1-NEXT:    vbroadcastss 124(%rbp), %ymm12
-; AVX1-NEXT:    vmulps %ymm3, %ymm12, %ymm12
-; AVX1-NEXT:    vaddps %ymm12, %ymm11, %ymm11
-; AVX1-NEXT:    vbroadcastss 128(%rbp), %ymm12
-; AVX1-NEXT:    vmulps %ymm4, %ymm12, %ymm12
-; AVX1-NEXT:    vaddps %ymm12, %ymm11, %ymm11
-; AVX1-NEXT:    vbroadcastss 132(%rbp), %ymm12
-; AVX1-NEXT:    vmulps %ymm5, %ymm12, %ymm12
-; AVX1-NEXT:    vaddps %ymm12, %ymm11, %ymm11
-; AVX1-NEXT:    vbroadcastss 136(%rbp), %ymm12
-; AVX1-NEXT:    vmulps %ymm6, %ymm12, %ymm12
-; AVX1-NEXT:    vaddps %ymm12, %ymm11, %ymm11
-; AVX1-NEXT:    vbroadcastss 140(%rbp), %ymm12
-; AVX1-NEXT:    vmulps %ymm7, %ymm12, %ymm12
-; AVX1-NEXT:    vaddps %ymm12, %ymm11, %ymm11
-; AVX1-NEXT:    vbroadcastss 144(%rbp), %ymm12
-; AVX1-NEXT:    vmulps %ymm0, %ymm12, %ymm12
-; AVX1-NEXT:    vbroadcastss 148(%rbp), %ymm13
-; AVX1-NEXT:    vmulps %ymm1, %ymm13, %ymm13
-; AVX1-NEXT:    vaddps %ymm13, %ymm12, %ymm12
-; AVX1-NEXT:    vbroadcastss 152(%rbp), %ymm13
-; AVX1-NEXT:    vmulps %ymm2, %ymm13, %ymm13
-; AVX1-NEXT:    vaddps %ymm13, %ymm12, %ymm12
-; AVX1-NEXT:    vbroadcastss 156(%rbp), %ymm13
-; AVX1-NEXT:    vmulps %ymm3, %ymm13, %ymm13
-; AVX1-NEXT:    vaddps %ymm13, %ymm12, %ymm12
-; AVX1-NEXT:    vbroadcastss 160(%rbp), %ymm13
-; AVX1-NEXT:    vmulps %ymm4, %ymm13, %ymm13
-; AVX1-NEXT:    vaddps %ymm13, %ymm12, %ymm12
-; AVX1-NEXT:    vbroadcastss 164(%rbp), %ymm13
-; AVX1-NEXT:    vmulps %ymm5, %ymm13, %ymm13
-; AVX1-NEXT:    vaddps %ymm13, %ymm12, %ymm12
-; AVX1-NEXT:    vbroadcastss 168(%rbp), %ymm13
-; AVX1-NEXT:    vmulps %ymm6, %ymm13, %ymm13
-; AVX1-NEXT:    vaddps %ymm13, %ymm12, %ymm12
-; AVX1-NEXT:    vbroadcastss 172(%rbp), %ymm13
-; AVX1-NEXT:    vmulps %ymm7, %ymm13, %ymm13
-; AVX1-NEXT:    vaddps %ymm13, %ymm12, %ymm12
-; AVX1-NEXT:    vbroadcastss 176(%rbp), %ymm13
-; AVX1-NEXT:    vmulps %ymm0, %ymm13, %ymm13
-; AVX1-NEXT:    vbroadcastss 180(%rbp), %ymm14
-; AVX1-NEXT:    vmulps %ymm1, %ymm14, %ymm14
-; AVX1-NEXT:    vaddps %ymm14, %ymm13, %ymm13
-; AVX1-NEXT:    vbroadcastss 184(%rbp), %ymm14
-; AVX1-NEXT:    vmulps %ymm2, %ymm14, %ymm14
-; AVX1-NEXT:    vaddps %ymm14, %ymm13, %ymm13
-; AVX1-NEXT:    vbroadcastss 188(%rbp), %ymm14
-; AVX1-NEXT:    vmulps %ymm3, %ymm14, %ymm14
-; AVX1-NEXT:    vaddps %ymm14, %ymm13, %ymm13
-; AVX1-NEXT:    vbroadcastss 192(%rbp), %ymm14
-; AVX1-NEXT:    vmulps %ymm4, %ymm14, %ymm14
-; AVX1-NEXT:    vaddps %ymm14, %ymm13, %ymm13
-; AVX1-NEXT:    vbroadcastss 196(%rbp), %ymm14
-; AVX1-NEXT:    vmulps %ymm5, %ymm14, %ymm14
-; AVX1-NEXT:    vaddps %ymm14, %ymm13, %ymm13
-; AVX1-NEXT:    vbroadcastss 200(%rbp), %ymm14
-; AVX1-NEXT:    vmulps %ymm6, %ymm14, %ymm14
-; AVX1-NEXT:    vaddps %ymm14, %ymm13, %ymm13
-; AVX1-NEXT:    vbroadcastss 204(%rbp), %ymm14
-; AVX1-NEXT:    vmulps %ymm7, %ymm14, %ymm14
-; AVX1-NEXT:    vaddps %ymm14, %ymm13, %ymm13
-; AVX1-NEXT:    vbroadcastss 208(%rbp), %ymm14
-; AVX1-NEXT:    vmulps %ymm0, %ymm14, %ymm14
-; AVX1-NEXT:    vbroadcastss 212(%rbp), %ymm15
-; AVX1-NEXT:    vmulps %ymm1, %ymm15, %ymm15
-; AVX1-NEXT:    vaddps %ymm15, %ymm14, %ymm14
-; AVX1-NEXT:    vbroadcastss 216(%rbp), %ymm15
-; AVX1-NEXT:    vmulps %ymm2, %ymm15, %ymm15
-; AVX1-NEXT:    vaddps %ymm15, %ymm14, %ymm14
-; AVX1-NEXT:    vbroadcastss 220(%rbp), %ymm15
-; AVX1-NEXT:    vmulps %ymm3, %ymm15, %ymm15
-; AVX1-NEXT:    vaddps %ymm15, %ymm14, %ymm14
-; AVX1-NEXT:    vbroadcastss 224(%rbp), %ymm15
-; AVX1-NEXT:    vmulps %ymm4, %ymm15, %ymm15
-; AVX1-NEXT:    vaddps %ymm15, %ymm14, %ymm14
-; AVX1-NEXT:    vbroadcastss 228(%rbp), %ymm15
-; AVX1-NEXT:    vmulps %ymm5, %ymm15, %ymm15
-; AVX1-NEXT:    vaddps %ymm15, %ymm14, %ymm14
-; AVX1-NEXT:    vbroadcastss 232(%rbp), %ymm15
-; AVX1-NEXT:    vmulps %ymm6, %ymm15, %ymm15
-; AVX1-NEXT:    vaddps %ymm15, %ymm14, %ymm14
-; AVX1-NEXT:    vbroadcastss 236(%rbp), %ymm15
-; AVX1-NEXT:    vmulps %ymm7, %ymm15, %ymm15
-; AVX1-NEXT:    vaddps %ymm15, %ymm14, %ymm14
-; AVX1-NEXT:    vbroadcastss 240(%rbp), %ymm15
-; AVX1-NEXT:    vmulps %ymm0, %ymm15, %ymm0
-; AVX1-NEXT:    vbroadcastss 244(%rbp), %ymm15
-; AVX1-NEXT:    vmulps %ymm1, %ymm15, %ymm1
-; AVX1-NEXT:    vaddps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vbroadcastss 248(%rbp), %ymm1
-; AVX1-NEXT:    vmulps %ymm1, %ymm2, %ymm1
-; AVX1-NEXT:    vaddps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vbroadcastss 252(%rbp), %ymm1
-; AVX1-NEXT:    vmulps %ymm1, %ymm3, %ymm1
-; AVX1-NEXT:    vaddps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vbroadcastss 256(%rbp), %ymm1
-; AVX1-NEXT:    vmulps %ymm1, %ymm4, %ymm1
-; AVX1-NEXT:    vaddps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vbroadcastss 260(%rbp), %ymm1
-; AVX1-NEXT:    vmulps %ymm1, %ymm5, %ymm1
-; AVX1-NEXT:    vaddps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vbroadcastss 264(%rbp), %ymm1
-; AVX1-NEXT:    vmulps %ymm1, %ymm6, %ymm1
-; AVX1-NEXT:    vaddps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vbroadcastss 268(%rbp), %ymm1
-; AVX1-NEXT:    vmulps %ymm1, %ymm7, %ymm1
-; AVX1-NEXT:    vaddps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vmovaps %ymm0, 224(%rdi)
-; AVX1-NEXT:    vmovaps %ymm14, 192(%rdi)
-; AVX1-NEXT:    vmovaps %ymm13, 160(%rdi)
-; AVX1-NEXT:    vmovaps %ymm12, 128(%rdi)
-; AVX1-NEXT:    vmovaps %ymm11, 96(%rdi)
-; AVX1-NEXT:    vmovaps %ymm10, 64(%rdi)
-; AVX1-NEXT:    vmovaps %ymm9, 32(%rdi)
-; AVX1-NEXT:    vmovaps %ymm8, (%rdi)
-; AVX1-NEXT:    movq %rbp, %rsp
-; AVX1-NEXT:    popq %rbp
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test_mul8x8_f32:
-; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    pushq %rbp
-; AVX2-NEXT:    movq %rsp, %rbp
-; AVX2-NEXT:    andq $-32, %rsp
-; AVX2-NEXT:    subq $32, %rsp
-; AVX2-NEXT:    movq %rdi, %rax
-; AVX2-NEXT:    vbroadcastss 16(%rbp), %ymm8
-; AVX2-NEXT:    vmulps %ymm0, %ymm8, %ymm8
-; AVX2-NEXT:    vbroadcastss 20(%rbp), %ymm9
-; AVX2-NEXT:    vmulps %ymm1, %ymm9, %ymm9
-; AVX2-NEXT:    vaddps %ymm9, %ymm8, %ymm8
-; AVX2-NEXT:    vbroadcastss 24(%rbp), %ymm9
-; AVX2-NEXT:    vmulps %ymm2, %ymm9, %ymm9
-; AVX2-NEXT:    vaddps %ymm9, %ymm8, %ymm8
-; AVX2-NEXT:    vbroadcastss 28(%rbp), %ymm9
-; AVX2-NEXT:    vmulps %ymm3, %ymm9, %ymm9
-; AVX2-NEXT:    vaddps %ymm9, %ymm8, %ymm8
-; AVX2-NEXT:    vbroadcastss 32(%rbp), %ymm9
-; AVX2-NEXT:    vmulps %ymm4, %ymm9, %ymm9
-; AVX2-NEXT:    vaddps %ymm9, %ymm8, %ymm8
-; AVX2-NEXT:    vbroadcastss 36(%rbp), %ymm9
-; AVX2-NEXT:    vmulps %ymm5, %ymm9, %ymm9
-; AVX2-NEXT:    vaddps %ymm9, %ymm8, %ymm8
-; AVX2-NEXT:    vbroadcastss 40(%rbp), %ymm9
-; AVX2-NEXT:    vmulps %ymm6, %ymm9, %ymm9
-; AVX2-NEXT:    vaddps %ymm9, %ymm8, %ymm8
-; AVX2-NEXT:    vbroadcastss 44(%rbp), %ymm9
-; AVX2-NEXT:    vmulps %ymm7, %ymm9, %ymm9
-; AVX2-NEXT:    vaddps %ymm9, %ymm8, %ymm8
-; AVX2-NEXT:    vbroadcastss 48(%rbp), %ymm9
-; AVX2-NEXT:    vmulps %ymm0, %ymm9, %ymm9
-; AVX2-NEXT:    vbroadcastss 52(%rbp), %ymm10
-; AVX2-NEXT:    vmulps %ymm1, %ymm10, %ymm10
-; AVX2-NEXT:    vaddps %ymm10, %ymm9, %ymm9
-; AVX2-NEXT:    vbroadcastss 56(%rbp), %ymm10
-; AVX2-NEXT:    vmulps %ymm2, %ymm10, %ymm10
-; AVX2-NEXT:    vaddps %ymm10, %ymm9, %ymm9
-; AVX2-NEXT:    vbroadcastss 60(%rbp), %ymm10
-; AVX2-NEXT:    vmulps %ymm3, %ymm10, %ymm10
-; AVX2-NEXT:    vaddps %ymm10, %ymm9, %ymm9
-; AVX2-NEXT:    vbroadcastss 64(%rbp), %ymm10
-; AVX2-NEXT:    vmulps %ymm4, %ymm10, %ymm10
-; AVX2-NEXT:    vaddps %ymm10, %ymm9, %ymm9
-; AVX2-NEXT:    vbroadcastss 68(%rbp), %ymm10
-; AVX2-NEXT:    vmulps %ymm5, %ymm10, %ymm10
-; AVX2-NEXT:    vaddps %ymm10, %ymm9, %ymm9
-; AVX2-NEXT:    vbroadcastss 72(%rbp), %ymm10
-; AVX2-NEXT:    vmulps %ymm6, %ymm10, %ymm10
-; AVX2-NEXT:    vaddps %ymm10, %ymm9, %ymm9
-; AVX2-NEXT:    vbroadcastss 76(%rbp), %ymm10
-; AVX2-NEXT:    vmulps %ymm7, %ymm10, %ymm10
-; AVX2-NEXT:    vaddps %ymm10, %ymm9, %ymm9
-; AVX2-NEXT:    vbroadcastss 80(%rbp), %ymm10
-; AVX2-NEXT:    vmulps %ymm0, %ymm10, %ymm10
-; AVX2-NEXT:    vbroadcastss 84(%rbp), %ymm11
-; AVX2-NEXT:    vmulps %ymm1, %ymm11, %ymm11
-; AVX2-NEXT:    vaddps %ymm11, %ymm10, %ymm10
-; AVX2-NEXT:    vbroadcastss 88(%rbp), %ymm11
-; AVX2-NEXT:    vmulps %ymm2, %ymm11, %ymm11
-; AVX2-NEXT:    vaddps %ymm11, %ymm10, %ymm10
-; AVX2-NEXT:    vbroadcastss 92(%rbp), %ymm11
-; AVX2-NEXT:    vmulps %ymm3, %ymm11, %ymm11
-; AVX2-NEXT:    vaddps %ymm11, %ymm10, %ymm10
-; AVX2-NEXT:    vbroadcastss 96(%rbp), %ymm11
-; AVX2-NEXT:    vmulps %ymm4, %ymm11, %ymm11
-; AVX2-NEXT:    vaddps %ymm11, %ymm10, %ymm10
-; AVX2-NEXT:    vbroadcastss 100(%rbp), %ymm11
-; AVX2-NEXT:    vmulps %ymm5, %ymm11, %ymm11
-; AVX2-NEXT:    vaddps %ymm11, %ymm10, %ymm10
-; AVX2-NEXT:    vbroadcastss 104(%rbp), %ymm11
-; AVX2-NEXT:    vmulps %ymm6, %ymm11, %ymm11
-; AVX2-NEXT:    vaddps %ymm11, %ymm10, %ymm10
-; AVX2-NEXT:    vbroadcastss 108(%rbp), %ymm11
-; AVX2-NEXT:    vmulps %ymm7, %ymm11, %ymm11
-; AVX2-NEXT:    vaddps %ymm11, %ymm10, %ymm10
-; AVX2-NEXT:    vbroadcastss 112(%rbp), %ymm11
-; AVX2-NEXT:    vmulps %ymm0, %ymm11, %ymm11
-; AVX2-NEXT:    vbroadcastss 116(%rbp), %ymm12
-; AVX2-NEXT:    vmulps %ymm1, %ymm12, %ymm12
-; AVX2-NEXT:    vaddps %ymm12, %ymm11, %ymm11
-; AVX2-NEXT:    vbroadcastss 120(%rbp), %ymm12
-; AVX2-NEXT:    vmulps %ymm2, %ymm12, %ymm12
-; AVX2-NEXT:    vaddps %ymm12, %ymm11, %ymm11
-; AVX2-NEXT:    vbroadcastss 124(%rbp), %ymm12
-; AVX2-NEXT:    vmulps %ymm3, %ymm12, %ymm12
-; AVX2-NEXT:    vaddps %ymm12, %ymm11, %ymm11
-; AVX2-NEXT:    vbroadcastss 128(%rbp), %ymm12
-; AVX2-NEXT:    vmulps %ymm4, %ymm12, %ymm12
-; AVX2-NEXT:    vaddps %ymm12, %ymm11, %ymm11
-; AVX2-NEXT:    vbroadcastss 132(%rbp), %ymm12
-; AVX2-NEXT:    vmulps %ymm5, %ymm12, %ymm12
-; AVX2-NEXT:    vaddps %ymm12, %ymm11, %ymm11
-; AVX2-NEXT:    vbroadcastss 136(%rbp), %ymm12
-; AVX2-NEXT:    vmulps %ymm6, %ymm12, %ymm12
-; AVX2-NEXT:    vaddps %ymm12, %ymm11, %ymm11
-; AVX2-NEXT:    vbroadcastss 140(%rbp), %ymm12
-; AVX2-NEXT:    vmulps %ymm7, %ymm12, %ymm12
-; AVX2-NEXT:    vaddps %ymm12, %ymm11, %ymm11
-; AVX2-NEXT:    vbroadcastss 144(%rbp), %ymm12
-; AVX2-NEXT:    vmulps %ymm0, %ymm12, %ymm12
-; AVX2-NEXT:    vbroadcastss 148(%rbp), %ymm13
-; AVX2-NEXT:    vmulps %ymm1, %ymm13, %ymm13
-; AVX2-NEXT:    vaddps %ymm13, %ymm12, %ymm12
-; AVX2-NEXT:    vbroadcastss 152(%rbp), %ymm13
-; AVX2-NEXT:    vmulps %ymm2, %ymm13, %ymm13
-; AVX2-NEXT:    vaddps %ymm13, %ymm12, %ymm12
-; AVX2-NEXT:    vbroadcastss 156(%rbp), %ymm13
-; AVX2-NEXT:    vmulps %ymm3, %ymm13, %ymm13
-; AVX2-NEXT:    vaddps %ymm13, %ymm12, %ymm12
-; AVX2-NEXT:    vbroadcastss 160(%rbp), %ymm13
-; AVX2-NEXT:    vmulps %ymm4, %ymm13, %ymm13
-; AVX2-NEXT:    vaddps %ymm13, %ymm12, %ymm12
-; AVX2-NEXT:    vbroadcastss 164(%rbp), %ymm13
-; AVX2-NEXT:    vmulps %ymm5, %ymm13, %ymm13
-; AVX2-NEXT:    vaddps %ymm13, %ymm12, %ymm12
-; AVX2-NEXT:    vbroadcastss 168(%rbp), %ymm13
-; AVX2-NEXT:    vmulps %ymm6, %ymm13, %ymm13
-; AVX2-NEXT:    vaddps %ymm13, %ymm12, %ymm12
-; AVX2-NEXT:    vbroadcastss 172(%rbp), %ymm13
-; AVX2-NEXT:    vmulps %ymm7, %ymm13, %ymm13
-; AVX2-NEXT:    vaddps %ymm13, %ymm12, %ymm12
-; AVX2-NEXT:    vbroadcastss 176(%rbp), %ymm13
-; AVX2-NEXT:    vmulps %ymm0, %ymm13, %ymm13
-; AVX2-NEXT:    vbroadcastss 180(%rbp), %ymm14
-; AVX2-NEXT:    vmulps %ymm1, %ymm14, %ymm14
-; AVX2-NEXT:    vaddps %ymm14, %ymm13, %ymm13
-; AVX2-NEXT:    vbroadcastss 184(%rbp), %ymm14
-; AVX2-NEXT:    vmulps %ymm2, %ymm14, %ymm14
-; AVX2-NEXT:    vaddps %ymm14, %ymm13, %ymm13
-; AVX2-NEXT:    vbroadcastss 188(%rbp), %ymm14
-; AVX2-NEXT:    vmulps %ymm3, %ymm14, %ymm14
-; AVX2-NEXT:    vaddps %ymm14, %ymm13, %ymm13
-; AVX2-NEXT:    vbroadcastss 192(%rbp), %ymm14
-; AVX2-NEXT:    vmulps %ymm4, %ymm14, %ymm14
-; AVX2-NEXT:    vaddps %ymm14, %ymm13, %ymm13
-; AVX2-NEXT:    vbroadcastss 196(%rbp), %ymm14
-; AVX2-NEXT:    vmulps %ymm5, %ymm14, %ymm14
-; AVX2-NEXT:    vaddps %ymm14, %ymm13, %ymm13
-; AVX2-NEXT:    vbroadcastss 200(%rbp), %ymm14
-; AVX2-NEXT:    vmulps %ymm6, %ymm14, %ymm14
-; AVX2-NEXT:    vaddps %ymm14, %ymm13, %ymm13
-; AVX2-NEXT:    vbroadcastss 204(%rbp), %ymm14
-; AVX2-NEXT:    vmulps %ymm7, %ymm14, %ymm14
-; AVX2-NEXT:    vaddps %ymm14, %ymm13, %ymm13
-; AVX2-NEXT:    vbroadcastss 208(%rbp), %ymm14
-; AVX2-NEXT:    vmulps %ymm0, %ymm14, %ymm14
-; AVX2-NEXT:    vbroadcastss 212(%rbp), %ymm15
-; AVX2-NEXT:    vmulps %ymm1, %ymm15, %ymm15
-; AVX2-NEXT:    vaddps %ymm15, %ymm14, %ymm14
-; AVX2-NEXT:    vbroadcastss 216(%rbp), %ymm15
-; AVX2-NEXT:    vmulps %ymm2, %ymm15, %ymm15
-; AVX2-NEXT:    vaddps %ymm15, %ymm14, %ymm14
-; AVX2-NEXT:    vbroadcastss 220(%rbp), %ymm15
-; AVX2-NEXT:    vmulps %ymm3, %ymm15, %ymm15
-; AVX2-NEXT:    vaddps %ymm15, %ymm14, %ymm14
-; AVX2-NEXT:    vbroadcastss 224(%rbp), %ymm15
-; AVX2-NEXT:    vmulps %ymm4, %ymm15, %ymm15
-; AVX2-NEXT:    vaddps %ymm15, %ymm14, %ymm14
-; AVX2-NEXT:    vbroadcastss 228(%rbp), %ymm15
-; AVX2-NEXT:    vmulps %ymm5, %ymm15, %ymm15
-; AVX2-NEXT:    vaddps %ymm15, %ymm14, %ymm14
-; AVX2-NEXT:    vbroadcastss 232(%rbp), %ymm15
-; AVX2-NEXT:    vmulps %ymm6, %ymm15, %ymm15
-; AVX2-NEXT:    vaddps %ymm15, %ymm14, %ymm14
-; AVX2-NEXT:    vbroadcastss 236(%rbp), %ymm15
-; AVX2-NEXT:    vmulps %ymm7, %ymm15, %ymm15
-; AVX2-NEXT:    vaddps %ymm15, %ymm14, %ymm14
-; AVX2-NEXT:    vbroadcastss 240(%rbp), %ymm15
-; AVX2-NEXT:    vmulps %ymm0, %ymm15, %ymm0
-; AVX2-NEXT:    vbroadcastss 244(%rbp), %ymm15
-; AVX2-NEXT:    vmulps %ymm1, %ymm15, %ymm1
-; AVX2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vbroadcastss 248(%rbp), %ymm1
-; AVX2-NEXT:    vmulps %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vbroadcastss 252(%rbp), %ymm1
-; AVX2-NEXT:    vmulps %ymm1, %ymm3, %ymm1
-; AVX2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vbroadcastss 256(%rbp), %ymm1
-; AVX2-NEXT:    vmulps %ymm1, %ymm4, %ymm1
-; AVX2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vbroadcastss 260(%rbp), %ymm1
-; AVX2-NEXT:    vmulps %ymm1, %ymm5, %ymm1
-; AVX2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vbroadcastss 264(%rbp), %ymm1
-; AVX2-NEXT:    vmulps %ymm1, %ymm6, %ymm1
-; AVX2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vbroadcastss 268(%rbp), %ymm1
-; AVX2-NEXT:    vmulps %ymm1, %ymm7, %ymm1
-; AVX2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vmovaps %ymm0, 224(%rdi)
-; AVX2-NEXT:    vmovaps %ymm14, 192(%rdi)
-; AVX2-NEXT:    vmovaps %ymm13, 160(%rdi)
-; AVX2-NEXT:    vmovaps %ymm12, 128(%rdi)
-; AVX2-NEXT:    vmovaps %ymm11, 96(%rdi)
-; AVX2-NEXT:    vmovaps %ymm10, 64(%rdi)
-; AVX2-NEXT:    vmovaps %ymm9, 32(%rdi)
-; AVX2-NEXT:    vmovaps %ymm8, (%rdi)
-; AVX2-NEXT:    movq %rbp, %rsp
-; AVX2-NEXT:    popq %rbp
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
+; AVX1OR2-LABEL: test_mul8x8_f32:
+; AVX1OR2:       # %bb.0: # %entry
+; AVX1OR2-NEXT:    pushq %rbp
+; AVX1OR2-NEXT:    movq %rsp, %rbp
+; AVX1OR2-NEXT:    andq $-32, %rsp
+; AVX1OR2-NEXT:    subq $32, %rsp
+; AVX1OR2-NEXT:    movq %rdi, %rax
+; AVX1OR2-NEXT:    vbroadcastss 16(%rbp), %ymm8
+; AVX1OR2-NEXT:    vmulps %ymm0, %ymm8, %ymm8
+; AVX1OR2-NEXT:    vbroadcastss 20(%rbp), %ymm9
+; AVX1OR2-NEXT:    vmulps %ymm1, %ymm9, %ymm9
+; AVX1OR2-NEXT:    vaddps %ymm9, %ymm8, %ymm8
+; AVX1OR2-NEXT:    vbroadcastss 24(%rbp), %ymm9
+; AVX1OR2-NEXT:    vmulps %ymm2, %ymm9, %ymm9
+; AVX1OR2-NEXT:    vaddps %ymm9, %ymm8, %ymm8
+; AVX1OR2-NEXT:    vbroadcastss 28(%rbp), %ymm9
+; AVX1OR2-NEXT:    vmulps %ymm3, %ymm9, %ymm9
+; AVX1OR2-NEXT:    vaddps %ymm9, %ymm8, %ymm8
+; AVX1OR2-NEXT:    vbroadcastss 32(%rbp), %ymm9
+; AVX1OR2-NEXT:    vmulps %ymm4, %ymm9, %ymm9
+; AVX1OR2-NEXT:    vaddps %ymm9, %ymm8, %ymm8
+; AVX1OR2-NEXT:    vbroadcastss 36(%rbp), %ymm9
+; AVX1OR2-NEXT:    vmulps %ymm5, %ymm9, %ymm9
+; AVX1OR2-NEXT:    vaddps %ymm9, %ymm8, %ymm8
+; AVX1OR2-NEXT:    vbroadcastss 40(%rbp), %ymm9
+; AVX1OR2-NEXT:    vmulps %ymm6, %ymm9, %ymm9
+; AVX1OR2-NEXT:    vaddps %ymm9, %ymm8, %ymm8
+; AVX1OR2-NEXT:    vbroadcastss 44(%rbp), %ymm9
+; AVX1OR2-NEXT:    vmulps %ymm7, %ymm9, %ymm9
+; AVX1OR2-NEXT:    vaddps %ymm9, %ymm8, %ymm8
+; AVX1OR2-NEXT:    vbroadcastss 48(%rbp), %ymm9
+; AVX1OR2-NEXT:    vmulps %ymm0, %ymm9, %ymm9
+; AVX1OR2-NEXT:    vbroadcastss 52(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulps %ymm1, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddps %ymm10, %ymm9, %ymm9
+; AVX1OR2-NEXT:    vbroadcastss 56(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulps %ymm2, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddps %ymm10, %ymm9, %ymm9
+; AVX1OR2-NEXT:    vbroadcastss 60(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulps %ymm3, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddps %ymm10, %ymm9, %ymm9
+; AVX1OR2-NEXT:    vbroadcastss 64(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulps %ymm4, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddps %ymm10, %ymm9, %ymm9
+; AVX1OR2-NEXT:    vbroadcastss 68(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulps %ymm5, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddps %ymm10, %ymm9, %ymm9
+; AVX1OR2-NEXT:    vbroadcastss 72(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulps %ymm6, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddps %ymm10, %ymm9, %ymm9
+; AVX1OR2-NEXT:    vbroadcastss 76(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulps %ymm7, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddps %ymm10, %ymm9, %ymm9
+; AVX1OR2-NEXT:    vbroadcastss 80(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulps %ymm0, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vbroadcastss 84(%rbp), %ymm11
+; AVX1OR2-NEXT:    vmulps %ymm1, %ymm11, %ymm11
+; AVX1OR2-NEXT:    vaddps %ymm11, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vbroadcastss 88(%rbp), %ymm11
+; AVX1OR2-NEXT:    vmulps %ymm2, %ymm11, %ymm11
+; AVX1OR2-NEXT:    vaddps %ymm11, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vbroadcastss 92(%rbp), %ymm11
+; AVX1OR2-NEXT:    vmulps %ymm3, %ymm11, %ymm11
+; AVX1OR2-NEXT:    vaddps %ymm11, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vbroadcastss 96(%rbp), %ymm11
+; AVX1OR2-NEXT:    vmulps %ymm4, %ymm11, %ymm11
+; AVX1OR2-NEXT:    vaddps %ymm11, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vbroadcastss 100(%rbp), %ymm11
+; AVX1OR2-NEXT:    vmulps %ymm5, %ymm11, %ymm11
+; AVX1OR2-NEXT:    vaddps %ymm11, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vbroadcastss 104(%rbp), %ymm11
+; AVX1OR2-NEXT:    vmulps %ymm6, %ymm11, %ymm11
+; AVX1OR2-NEXT:    vaddps %ymm11, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vbroadcastss 108(%rbp), %ymm11
+; AVX1OR2-NEXT:    vmulps %ymm7, %ymm11, %ymm11
+; AVX1OR2-NEXT:    vaddps %ymm11, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vbroadcastss 112(%rbp), %ymm11
+; AVX1OR2-NEXT:    vmulps %ymm0, %ymm11, %ymm11
+; AVX1OR2-NEXT:    vbroadcastss 116(%rbp), %ymm12
+; AVX1OR2-NEXT:    vmulps %ymm1, %ymm12, %ymm12
+; AVX1OR2-NEXT:    vaddps %ymm12, %ymm11, %ymm11
+; AVX1OR2-NEXT:    vbroadcastss 120(%rbp), %ymm12
+; AVX1OR2-NEXT:    vmulps %ymm2, %ymm12, %ymm12
+; AVX1OR2-NEXT:    vaddps %ymm12, %ymm11, %ymm11
+; AVX1OR2-NEXT:    vbroadcastss 124(%rbp), %ymm12
+; AVX1OR2-NEXT:    vmulps %ymm3, %ymm12, %ymm12
+; AVX1OR2-NEXT:    vaddps %ymm12, %ymm11, %ymm11
+; AVX1OR2-NEXT:    vbroadcastss 128(%rbp), %ymm12
+; AVX1OR2-NEXT:    vmulps %ymm4, %ymm12, %ymm12
+; AVX1OR2-NEXT:    vaddps %ymm12, %ymm11, %ymm11
+; AVX1OR2-NEXT:    vbroadcastss 132(%rbp), %ymm12
+; AVX1OR2-NEXT:    vmulps %ymm5, %ymm12, %ymm12
+; AVX1OR2-NEXT:    vaddps %ymm12, %ymm11, %ymm11
+; AVX1OR2-NEXT:    vbroadcastss 136(%rbp), %ymm12
+; AVX1OR2-NEXT:    vmulps %ymm6, %ymm12, %ymm12
+; AVX1OR2-NEXT:    vaddps %ymm12, %ymm11, %ymm11
+; AVX1OR2-NEXT:    vbroadcastss 140(%rbp), %ymm12
+; AVX1OR2-NEXT:    vmulps %ymm7, %ymm12, %ymm12
+; AVX1OR2-NEXT:    vaddps %ymm12, %ymm11, %ymm11
+; AVX1OR2-NEXT:    vbroadcastss 144(%rbp), %ymm12
+; AVX1OR2-NEXT:    vmulps %ymm0, %ymm12, %ymm12
+; AVX1OR2-NEXT:    vbroadcastss 148(%rbp), %ymm13
+; AVX1OR2-NEXT:    vmulps %ymm1, %ymm13, %ymm13
+; AVX1OR2-NEXT:    vaddps %ymm13, %ymm12, %ymm12
+; AVX1OR2-NEXT:    vbroadcastss 152(%rbp), %ymm13
+; AVX1OR2-NEXT:    vmulps %ymm2, %ymm13, %ymm13
+; AVX1OR2-NEXT:    vaddps %ymm13, %ymm12, %ymm12
+; AVX1OR2-NEXT:    vbroadcastss 156(%rbp), %ymm13
+; AVX1OR2-NEXT:    vmulps %ymm3, %ymm13, %ymm13
+; AVX1OR2-NEXT:    vaddps %ymm13, %ymm12, %ymm12
+; AVX1OR2-NEXT:    vbroadcastss 160(%rbp), %ymm13
+; AVX1OR2-NEXT:    vmulps %ymm4, %ymm13, %ymm13
+; AVX1OR2-NEXT:    vaddps %ymm13, %ymm12, %ymm12
+; AVX1OR2-NEXT:    vbroadcastss 164(%rbp), %ymm13
+; AVX1OR2-NEXT:    vmulps %ymm5, %ymm13, %ymm13
+; AVX1OR2-NEXT:    vaddps %ymm13, %ymm12, %ymm12
+; AVX1OR2-NEXT:    vbroadcastss 168(%rbp), %ymm13
+; AVX1OR2-NEXT:    vmulps %ymm6, %ymm13, %ymm13
+; AVX1OR2-NEXT:    vaddps %ymm13, %ymm12, %ymm12
+; AVX1OR2-NEXT:    vbroadcastss 172(%rbp), %ymm13
+; AVX1OR2-NEXT:    vmulps %ymm7, %ymm13, %ymm13
+; AVX1OR2-NEXT:    vaddps %ymm13, %ymm12, %ymm12
+; AVX1OR2-NEXT:    vbroadcastss 176(%rbp), %ymm13
+; AVX1OR2-NEXT:    vmulps %ymm0, %ymm13, %ymm13
+; AVX1OR2-NEXT:    vbroadcastss 180(%rbp), %ymm14
+; AVX1OR2-NEXT:    vmulps %ymm1, %ymm14, %ymm14
+; AVX1OR2-NEXT:    vaddps %ymm14, %ymm13, %ymm13
+; AVX1OR2-NEXT:    vbroadcastss 184(%rbp), %ymm14
+; AVX1OR2-NEXT:    vmulps %ymm2, %ymm14, %ymm14
+; AVX1OR2-NEXT:    vaddps %ymm14, %ymm13, %ymm13
+; AVX1OR2-NEXT:    vbroadcastss 188(%rbp), %ymm14
+; AVX1OR2-NEXT:    vmulps %ymm3, %ymm14, %ymm14
+; AVX1OR2-NEXT:    vaddps %ymm14, %ymm13, %ymm13
+; AVX1OR2-NEXT:    vbroadcastss 192(%rbp), %ymm14
+; AVX1OR2-NEXT:    vmulps %ymm4, %ymm14, %ymm14
+; AVX1OR2-NEXT:    vaddps %ymm14, %ymm13, %ymm13
+; AVX1OR2-NEXT:    vbroadcastss 196(%rbp), %ymm14
+; AVX1OR2-NEXT:    vmulps %ymm5, %ymm14, %ymm14
+; AVX1OR2-NEXT:    vaddps %ymm14, %ymm13, %ymm13
+; AVX1OR2-NEXT:    vbroadcastss 200(%rbp), %ymm14
+; AVX1OR2-NEXT:    vmulps %ymm6, %ymm14, %ymm14
+; AVX1OR2-NEXT:    vaddps %ymm14, %ymm13, %ymm13
+; AVX1OR2-NEXT:    vbroadcastss 204(%rbp), %ymm14
+; AVX1OR2-NEXT:    vmulps %ymm7, %ymm14, %ymm14
+; AVX1OR2-NEXT:    vaddps %ymm14, %ymm13, %ymm13
+; AVX1OR2-NEXT:    vbroadcastss 208(%rbp), %ymm14
+; AVX1OR2-NEXT:    vmulps %ymm0, %ymm14, %ymm14
+; AVX1OR2-NEXT:    vbroadcastss 212(%rbp), %ymm15
+; AVX1OR2-NEXT:    vmulps %ymm1, %ymm15, %ymm15
+; AVX1OR2-NEXT:    vaddps %ymm15, %ymm14, %ymm14
+; AVX1OR2-NEXT:    vbroadcastss 216(%rbp), %ymm15
+; AVX1OR2-NEXT:    vmulps %ymm2, %ymm15, %ymm15
+; AVX1OR2-NEXT:    vaddps %ymm15, %ymm14, %ymm14
+; AVX1OR2-NEXT:    vbroadcastss 220(%rbp), %ymm15
+; AVX1OR2-NEXT:    vmulps %ymm3, %ymm15, %ymm15
+; AVX1OR2-NEXT:    vaddps %ymm15, %ymm14, %ymm14
+; AVX1OR2-NEXT:    vbroadcastss 224(%rbp), %ymm15
+; AVX1OR2-NEXT:    vmulps %ymm4, %ymm15, %ymm15
+; AVX1OR2-NEXT:    vaddps %ymm15, %ymm14, %ymm14
+; AVX1OR2-NEXT:    vbroadcastss 228(%rbp), %ymm15
+; AVX1OR2-NEXT:    vmulps %ymm5, %ymm15, %ymm15
+; AVX1OR2-NEXT:    vaddps %ymm15, %ymm14, %ymm14
+; AVX1OR2-NEXT:    vbroadcastss 232(%rbp), %ymm15
+; AVX1OR2-NEXT:    vmulps %ymm6, %ymm15, %ymm15
+; AVX1OR2-NEXT:    vaddps %ymm15, %ymm14, %ymm14
+; AVX1OR2-NEXT:    vbroadcastss 236(%rbp), %ymm15
+; AVX1OR2-NEXT:    vmulps %ymm7, %ymm15, %ymm15
+; AVX1OR2-NEXT:    vaddps %ymm15, %ymm14, %ymm14
+; AVX1OR2-NEXT:    vbroadcastss 240(%rbp), %ymm15
+; AVX1OR2-NEXT:    vmulps %ymm0, %ymm15, %ymm0
+; AVX1OR2-NEXT:    vbroadcastss 244(%rbp), %ymm15
+; AVX1OR2-NEXT:    vmulps %ymm1, %ymm15, %ymm1
+; AVX1OR2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vbroadcastss 248(%rbp), %ymm1
+; AVX1OR2-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; AVX1OR2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vbroadcastss 252(%rbp), %ymm1
+; AVX1OR2-NEXT:    vmulps %ymm1, %ymm3, %ymm1
+; AVX1OR2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vbroadcastss 256(%rbp), %ymm1
+; AVX1OR2-NEXT:    vmulps %ymm1, %ymm4, %ymm1
+; AVX1OR2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vbroadcastss 260(%rbp), %ymm1
+; AVX1OR2-NEXT:    vmulps %ymm1, %ymm5, %ymm1
+; AVX1OR2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vbroadcastss 264(%rbp), %ymm1
+; AVX1OR2-NEXT:    vmulps %ymm1, %ymm6, %ymm1
+; AVX1OR2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vbroadcastss 268(%rbp), %ymm1
+; AVX1OR2-NEXT:    vmulps %ymm1, %ymm7, %ymm1
+; AVX1OR2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vmovaps %ymm0, 224(%rdi)
+; AVX1OR2-NEXT:    vmovaps %ymm14, 192(%rdi)
+; AVX1OR2-NEXT:    vmovaps %ymm13, 160(%rdi)
+; AVX1OR2-NEXT:    vmovaps %ymm12, 128(%rdi)
+; AVX1OR2-NEXT:    vmovaps %ymm11, 96(%rdi)
+; AVX1OR2-NEXT:    vmovaps %ymm10, 64(%rdi)
+; AVX1OR2-NEXT:    vmovaps %ymm9, 32(%rdi)
+; AVX1OR2-NEXT:    vmovaps %ymm8, (%rdi)
+; AVX1OR2-NEXT:    movq %rbp, %rsp
+; AVX1OR2-NEXT:    popq %rbp
+; AVX1OR2-NEXT:    vzeroupper
+; AVX1OR2-NEXT:    retq
 ;
 ; AVX512F-LABEL: test_mul8x8_f32:
 ; AVX512F:       # %bb.0: # %entry
@@ -4226,803 +3828,404 @@ define <64 x double> @test_mul8x8_f64(<64 x double> %a0, <64 x double> %a1) noun
 ; SSE-NEXT:    addq $328, %rsp # imm = 0x148
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: test_mul8x8_f64:
-; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    pushq %rbp
-; AVX1-NEXT:    movq %rsp, %rbp
-; AVX1-NEXT:    andq $-32, %rsp
-; AVX1-NEXT:    subq $448, %rsp # imm = 0x1C0
-; AVX1-NEXT:    vmovapd %ymm2, %ymm12
-; AVX1-NEXT:    vmovapd %ymm0, (%rsp) # 32-byte Spill
-; AVX1-NEXT:    movq %rdi, %rax
-; AVX1-NEXT:    vmovapd 144(%rbp), %ymm13
-; AVX1-NEXT:    vmovapd 112(%rbp), %ymm14
-; AVX1-NEXT:    vbroadcastsd 272(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm1, %ymm10, %ymm8
-; AVX1-NEXT:    vmovapd %ymm1, %ymm9
-; AVX1-NEXT:    vmulpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 280(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm3, %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm11, %ymm8, %ymm1
-; AVX1-NEXT:    vmulpd %ymm2, %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 288(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm4, %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX1-NEXT:    vmulpd %ymm5, %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX1-NEXT:    vbroadcastsd 296(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm7, %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmulpd %ymm6, %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 304(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd 16(%rbp), %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX1-NEXT:    vmulpd 48(%rbp), %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX1-NEXT:    vbroadcastsd 312(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm10, %ymm14, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmulpd 80(%rbp), %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 320(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm10, %ymm13, %ymm11
-; AVX1-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX1-NEXT:    vmulpd 176(%rbp), %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX1-NEXT:    vbroadcastsd 328(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd 240(%rbp), %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmovapd %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-NEXT:    vmulpd 208(%rbp), %ymm10, %ymm1
-; AVX1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vmovapd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-NEXT:    vbroadcastsd 336(%rbp), %ymm0
-; AVX1-NEXT:    vmulpd %ymm0, %ymm9, %ymm1
-; AVX1-NEXT:    vbroadcastsd 344(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm3, %ymm10, %ymm11
-; AVX1-NEXT:    vmovapd %ymm3, %ymm8
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmovapd (%rsp), %ymm15 # 32-byte Reload
-; AVX1-NEXT:    vmulpd %ymm0, %ymm15, %ymm0
-; AVX1-NEXT:    vmulpd %ymm2, %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 352(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm4, %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX1-NEXT:    vmulpd %ymm5, %ymm10, %ymm10
-; AVX1-NEXT:    vmovapd %ymm5, %ymm3
-; AVX1-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX1-NEXT:    vbroadcastsd 360(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm7, %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmulpd %ymm6, %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 368(%rbp), %ymm10
-; AVX1-NEXT:    vmovapd 16(%rbp), %ymm2
-; AVX1-NEXT:    vmulpd %ymm2, %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX1-NEXT:    vmulpd 48(%rbp), %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX1-NEXT:    vbroadcastsd 376(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm10, %ymm14, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmovapd 80(%rbp), %ymm2
-; AVX1-NEXT:    vmulpd %ymm2, %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 384(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm10, %ymm13, %ymm11
-; AVX1-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX1-NEXT:    vmovapd 176(%rbp), %ymm14
-; AVX1-NEXT:    vmulpd %ymm10, %ymm14, %ymm10
-; AVX1-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX1-NEXT:    vbroadcastsd 392(%rbp), %ymm10
-; AVX1-NEXT:    vmovapd 240(%rbp), %ymm2
-; AVX1-NEXT:    vmulpd %ymm2, %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmovapd %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-NEXT:    vmulpd 208(%rbp), %ymm10, %ymm1
-; AVX1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vmovapd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-NEXT:    vbroadcastsd 400(%rbp), %ymm0
-; AVX1-NEXT:    vmulpd %ymm0, %ymm9, %ymm1
-; AVX1-NEXT:    vbroadcastsd 408(%rbp), %ymm10
-; AVX1-NEXT:    vmovapd %ymm8, %ymm5
-; AVX1-NEXT:    vmulpd %ymm10, %ymm8, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmulpd %ymm0, %ymm15, %ymm0
-; AVX1-NEXT:    vmulpd %ymm10, %ymm12, %ymm10
-; AVX1-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 416(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm4, %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX1-NEXT:    vmulpd %ymm3, %ymm10, %ymm10
-; AVX1-NEXT:    vmovapd %ymm3, %ymm2
-; AVX1-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX1-NEXT:    vbroadcastsd 424(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm7, %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmulpd %ymm6, %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 432(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd 16(%rbp), %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX1-NEXT:    vmulpd 48(%rbp), %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX1-NEXT:    vbroadcastsd 440(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd 112(%rbp), %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmulpd 80(%rbp), %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 448(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm10, %ymm13, %ymm11
-; AVX1-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX1-NEXT:    vmulpd %ymm10, %ymm14, %ymm10
-; AVX1-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX1-NEXT:    vbroadcastsd 456(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd 240(%rbp), %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmovapd %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-NEXT:    vmulpd 208(%rbp), %ymm10, %ymm1
-; AVX1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vmovapd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-NEXT:    vbroadcastsd 464(%rbp), %ymm0
-; AVX1-NEXT:    vmulpd %ymm0, %ymm9, %ymm1
-; AVX1-NEXT:    vmovapd %ymm9, %ymm13
-; AVX1-NEXT:    vbroadcastsd 472(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm10, %ymm8, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmulpd %ymm0, %ymm15, %ymm0
-; AVX1-NEXT:    vmovapd %ymm15, %ymm9
-; AVX1-NEXT:    vmulpd %ymm10, %ymm12, %ymm10
-; AVX1-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 480(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm4, %ymm10, %ymm11
-; AVX1-NEXT:    vmovapd %ymm4, %ymm3
-; AVX1-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX1-NEXT:    vmovapd %ymm2, %ymm15
-; AVX1-NEXT:    vmulpd %ymm2, %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX1-NEXT:    vbroadcastsd 488(%rbp), %ymm10
-; AVX1-NEXT:    vmovapd %ymm7, %ymm8
-; AVX1-NEXT:    vmulpd %ymm7, %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmovapd %ymm6, %ymm7
-; AVX1-NEXT:    vmulpd %ymm6, %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 496(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd 16(%rbp), %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX1-NEXT:    vmovapd 48(%rbp), %ymm4
-; AVX1-NEXT:    vmulpd %ymm4, %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX1-NEXT:    vbroadcastsd 504(%rbp), %ymm10
-; AVX1-NEXT:    vmovapd 112(%rbp), %ymm2
-; AVX1-NEXT:    vmulpd %ymm2, %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmovapd 80(%rbp), %ymm14
-; AVX1-NEXT:    vmulpd %ymm10, %ymm14, %ymm10
-; AVX1-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 512(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd 144(%rbp), %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX1-NEXT:    vmovapd 176(%rbp), %ymm2
-; AVX1-NEXT:    vmulpd %ymm2, %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX1-NEXT:    vbroadcastsd 520(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd 240(%rbp), %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmovapd %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-NEXT:    vmulpd 208(%rbp), %ymm10, %ymm1
-; AVX1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vmovapd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-NEXT:    vbroadcastsd 528(%rbp), %ymm0
-; AVX1-NEXT:    vmulpd %ymm0, %ymm13, %ymm1
-; AVX1-NEXT:    vbroadcastsd 536(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm5, %ymm10, %ymm11
-; AVX1-NEXT:    vmovapd %ymm5, %ymm6
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmulpd %ymm0, %ymm9, %ymm0
-; AVX1-NEXT:    vmulpd %ymm10, %ymm12, %ymm10
-; AVX1-NEXT:    vmovapd %ymm12, %ymm5
-; AVX1-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 544(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm3, %ymm10, %ymm11
-; AVX1-NEXT:    vmovapd %ymm3, %ymm12
-; AVX1-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX1-NEXT:    vmulpd %ymm10, %ymm15, %ymm10
-; AVX1-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX1-NEXT:    vbroadcastsd 552(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm10, %ymm8, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmulpd %ymm7, %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 560(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd 16(%rbp), %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX1-NEXT:    vmulpd %ymm4, %ymm10, %ymm10
-; AVX1-NEXT:    vmovapd %ymm4, %ymm3
-; AVX1-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX1-NEXT:    vbroadcastsd 568(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd 112(%rbp), %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmulpd %ymm10, %ymm14, %ymm10
-; AVX1-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 576(%rbp), %ymm10
-; AVX1-NEXT:    vmovapd 144(%rbp), %ymm4
-; AVX1-NEXT:    vmulpd %ymm4, %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX1-NEXT:    vmulpd %ymm2, %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX1-NEXT:    vbroadcastsd 584(%rbp), %ymm10
-; AVX1-NEXT:    vmovapd 240(%rbp), %ymm14
-; AVX1-NEXT:    vmulpd %ymm10, %ymm14, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmovapd %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-NEXT:    vmovapd 208(%rbp), %ymm2
-; AVX1-NEXT:    vmulpd %ymm2, %ymm10, %ymm1
-; AVX1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vmovapd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-NEXT:    vbroadcastsd 592(%rbp), %ymm0
-; AVX1-NEXT:    vmulpd %ymm0, %ymm13, %ymm1
-; AVX1-NEXT:    vbroadcastsd 600(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm6, %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmulpd %ymm0, %ymm9, %ymm0
-; AVX1-NEXT:    vmulpd %ymm5, %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 608(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm10, %ymm12, %ymm11
-; AVX1-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX1-NEXT:    vmulpd %ymm10, %ymm15, %ymm10
-; AVX1-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX1-NEXT:    vbroadcastsd 616(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm10, %ymm8, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmulpd %ymm7, %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 624(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd 16(%rbp), %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX1-NEXT:    vmulpd %ymm3, %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX1-NEXT:    vbroadcastsd 632(%rbp), %ymm10
-; AVX1-NEXT:    vmovapd 112(%rbp), %ymm3
-; AVX1-NEXT:    vmulpd %ymm3, %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmovapd 80(%rbp), %ymm3
-; AVX1-NEXT:    vmulpd %ymm3, %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX1-NEXT:    vbroadcastsd 640(%rbp), %ymm10
-; AVX1-NEXT:    vmulpd %ymm4, %ymm10, %ymm11
-; AVX1-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX1-NEXT:    vmovapd 176(%rbp), %ymm3
-; AVX1-NEXT:    vmulpd %ymm3, %ymm10, %ymm10
-; AVX1-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX1-NEXT:    vbroadcastsd 648(%rbp), %ymm10
-; AVX1-NEXT:    vmovapd %ymm14, %ymm4
-; AVX1-NEXT:    vmulpd %ymm10, %ymm14, %ymm11
-; AVX1-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX1-NEXT:    vmovapd %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-NEXT:    vmulpd %ymm2, %ymm10, %ymm1
-; AVX1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vmovapd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-NEXT:    vbroadcastsd 656(%rbp), %ymm2
-; AVX1-NEXT:    vmovapd %ymm13, %ymm3
-; AVX1-NEXT:    vmulpd %ymm2, %ymm13, %ymm1
-; AVX1-NEXT:    vbroadcastsd 664(%rbp), %ymm0
-; AVX1-NEXT:    vmulpd %ymm0, %ymm6, %ymm14
-; AVX1-NEXT:    vmovapd %ymm6, %ymm10
-; AVX1-NEXT:    vaddpd %ymm1, %ymm14, %ymm1
-; AVX1-NEXT:    vmulpd %ymm2, %ymm9, %ymm2
-; AVX1-NEXT:    vmulpd %ymm0, %ymm5, %ymm0
-; AVX1-NEXT:    vmovapd %ymm5, %ymm6
-; AVX1-NEXT:    vaddpd %ymm0, %ymm2, %ymm0
-; AVX1-NEXT:    vbroadcastsd 672(%rbp), %ymm2
-; AVX1-NEXT:    vmulpd %ymm2, %ymm12, %ymm14
-; AVX1-NEXT:    vaddpd %ymm0, %ymm14, %ymm0
-; AVX1-NEXT:    vmulpd %ymm2, %ymm15, %ymm2
-; AVX1-NEXT:    vaddpd %ymm2, %ymm1, %ymm1
-; AVX1-NEXT:    vbroadcastsd 680(%rbp), %ymm2
-; AVX1-NEXT:    vmulpd %ymm2, %ymm8, %ymm14
-; AVX1-NEXT:    vaddpd %ymm1, %ymm14, %ymm1
-; AVX1-NEXT:    vmulpd %ymm2, %ymm7, %ymm2
-; AVX1-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
-; AVX1-NEXT:    vbroadcastsd 688(%rbp), %ymm2
-; AVX1-NEXT:    vmovapd 16(%rbp), %ymm11
-; AVX1-NEXT:    vmulpd %ymm2, %ymm11, %ymm14
-; AVX1-NEXT:    vaddpd %ymm0, %ymm14, %ymm0
-; AVX1-NEXT:    vmulpd 48(%rbp), %ymm2, %ymm2
-; AVX1-NEXT:    vaddpd %ymm2, %ymm1, %ymm1
-; AVX1-NEXT:    vbroadcastsd 696(%rbp), %ymm2
-; AVX1-NEXT:    vmovapd 112(%rbp), %ymm5
-; AVX1-NEXT:    vmulpd %ymm2, %ymm5, %ymm14
-; AVX1-NEXT:    vaddpd %ymm1, %ymm14, %ymm1
-; AVX1-NEXT:    vmovapd 80(%rbp), %ymm5
-; AVX1-NEXT:    vmulpd %ymm2, %ymm5, %ymm2
-; AVX1-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
-; AVX1-NEXT:    vbroadcastsd 704(%rbp), %ymm2
-; AVX1-NEXT:    vmulpd 144(%rbp), %ymm2, %ymm14
-; AVX1-NEXT:    vaddpd %ymm0, %ymm14, %ymm0
-; AVX1-NEXT:    vmovapd 176(%rbp), %ymm13
-; AVX1-NEXT:    vmulpd %ymm2, %ymm13, %ymm2
-; AVX1-NEXT:    vaddpd %ymm2, %ymm1, %ymm1
-; AVX1-NEXT:    vbroadcastsd 712(%rbp), %ymm2
-; AVX1-NEXT:    vmulpd %ymm2, %ymm4, %ymm14
-; AVX1-NEXT:    vaddpd %ymm1, %ymm14, %ymm1
-; AVX1-NEXT:    vmovapd 208(%rbp), %ymm14
-; AVX1-NEXT:    vmulpd %ymm2, %ymm14, %ymm2
-; AVX1-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
-; AVX1-NEXT:    vbroadcastsd 720(%rbp), %ymm2
-; AVX1-NEXT:    vmulpd %ymm2, %ymm3, %ymm3
-; AVX1-NEXT:    vmulpd %ymm2, %ymm9, %ymm2
-; AVX1-NEXT:    vbroadcastsd 728(%rbp), %ymm4
-; AVX1-NEXT:    vmulpd %ymm4, %ymm10, %ymm5
-; AVX1-NEXT:    vaddpd %ymm5, %ymm3, %ymm3
-; AVX1-NEXT:    vmulpd %ymm4, %ymm6, %ymm4
-; AVX1-NEXT:    vaddpd %ymm4, %ymm2, %ymm2
-; AVX1-NEXT:    vbroadcastsd 736(%rbp), %ymm4
-; AVX1-NEXT:    vmulpd %ymm4, %ymm12, %ymm5
-; AVX1-NEXT:    vaddpd %ymm5, %ymm2, %ymm2
-; AVX1-NEXT:    vmulpd %ymm4, %ymm15, %ymm4
-; AVX1-NEXT:    vaddpd %ymm4, %ymm3, %ymm3
-; AVX1-NEXT:    vbroadcastsd 744(%rbp), %ymm4
-; AVX1-NEXT:    vmulpd %ymm4, %ymm8, %ymm5
-; AVX1-NEXT:    vaddpd %ymm5, %ymm3, %ymm3
-; AVX1-NEXT:    vmulpd %ymm4, %ymm7, %ymm4
-; AVX1-NEXT:    vaddpd %ymm4, %ymm2, %ymm2
-; AVX1-NEXT:    vbroadcastsd 752(%rbp), %ymm4
-; AVX1-NEXT:    vmulpd %ymm4, %ymm11, %ymm5
-; AVX1-NEXT:    vaddpd %ymm5, %ymm2, %ymm2
-; AVX1-NEXT:    vmulpd 48(%rbp), %ymm4, %ymm4
-; AVX1-NEXT:    vaddpd %ymm4, %ymm3, %ymm3
-; AVX1-NEXT:    vbroadcastsd 760(%rbp), %ymm4
-; AVX1-NEXT:    vmulpd 112(%rbp), %ymm4, %ymm5
-; AVX1-NEXT:    vaddpd %ymm5, %ymm3, %ymm3
-; AVX1-NEXT:    vmulpd 80(%rbp), %ymm4, %ymm4
-; AVX1-NEXT:    vaddpd %ymm4, %ymm2, %ymm2
-; AVX1-NEXT:    vbroadcastsd 768(%rbp), %ymm4
-; AVX1-NEXT:    vmulpd 144(%rbp), %ymm4, %ymm5
-; AVX1-NEXT:    vaddpd %ymm5, %ymm2, %ymm2
-; AVX1-NEXT:    vmulpd %ymm4, %ymm13, %ymm4
-; AVX1-NEXT:    vaddpd %ymm4, %ymm3, %ymm3
-; AVX1-NEXT:    vbroadcastsd 776(%rbp), %ymm4
-; AVX1-NEXT:    vmulpd 240(%rbp), %ymm4, %ymm5
-; AVX1-NEXT:    vaddpd %ymm5, %ymm3, %ymm3
-; AVX1-NEXT:    vmulpd %ymm4, %ymm14, %ymm4
-; AVX1-NEXT:    vaddpd %ymm4, %ymm2, %ymm2
-; AVX1-NEXT:    vmovapd %ymm3, 480(%rdi)
-; AVX1-NEXT:    vmovapd %ymm2, 448(%rdi)
-; AVX1-NEXT:    vmovapd %ymm1, 416(%rdi)
-; AVX1-NEXT:    vmovapd %ymm0, 384(%rdi)
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-NEXT:    vmovaps %ymm0, 352(%rdi)
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-NEXT:    vmovaps %ymm0, 320(%rdi)
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-NEXT:    vmovaps %ymm0, 288(%rdi)
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-NEXT:    vmovaps %ymm0, 256(%rdi)
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-NEXT:    vmovaps %ymm0, 224(%rdi)
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-NEXT:    vmovaps %ymm0, 192(%rdi)
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-NEXT:    vmovaps %ymm0, 160(%rdi)
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-NEXT:    vmovaps %ymm0, 128(%rdi)
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-NEXT:    vmovaps %ymm0, 96(%rdi)
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-NEXT:    vmovaps %ymm0, 64(%rdi)
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-NEXT:    vmovaps %ymm0, 32(%rdi)
-; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-NEXT:    vmovaps %ymm0, (%rdi)
-; AVX1-NEXT:    movq %rbp, %rsp
-; AVX1-NEXT:    popq %rbp
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test_mul8x8_f64:
-; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    pushq %rbp
-; AVX2-NEXT:    movq %rsp, %rbp
-; AVX2-NEXT:    andq $-32, %rsp
-; AVX2-NEXT:    subq $448, %rsp # imm = 0x1C0
-; AVX2-NEXT:    vmovapd %ymm2, %ymm12
-; AVX2-NEXT:    vmovapd %ymm0, (%rsp) # 32-byte Spill
-; AVX2-NEXT:    movq %rdi, %rax
-; AVX2-NEXT:    vmovapd 144(%rbp), %ymm13
-; AVX2-NEXT:    vmovapd 112(%rbp), %ymm14
-; AVX2-NEXT:    vbroadcastsd 272(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm1, %ymm10, %ymm8
-; AVX2-NEXT:    vmovapd %ymm1, %ymm9
-; AVX2-NEXT:    vmulpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 280(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm3, %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm11, %ymm8, %ymm1
-; AVX2-NEXT:    vmulpd %ymm2, %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 288(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm4, %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX2-NEXT:    vmulpd %ymm5, %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX2-NEXT:    vbroadcastsd 296(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm7, %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmulpd %ymm6, %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 304(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd 16(%rbp), %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX2-NEXT:    vmulpd 48(%rbp), %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX2-NEXT:    vbroadcastsd 312(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm10, %ymm14, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmulpd 80(%rbp), %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 320(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm10, %ymm13, %ymm11
-; AVX2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX2-NEXT:    vmulpd 176(%rbp), %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX2-NEXT:    vbroadcastsd 328(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd 240(%rbp), %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmovapd %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-NEXT:    vmulpd 208(%rbp), %ymm10, %ymm1
-; AVX2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vmovapd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-NEXT:    vbroadcastsd 336(%rbp), %ymm0
-; AVX2-NEXT:    vmulpd %ymm0, %ymm9, %ymm1
-; AVX2-NEXT:    vbroadcastsd 344(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm3, %ymm10, %ymm11
-; AVX2-NEXT:    vmovapd %ymm3, %ymm8
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmovapd (%rsp), %ymm15 # 32-byte Reload
-; AVX2-NEXT:    vmulpd %ymm0, %ymm15, %ymm0
-; AVX2-NEXT:    vmulpd %ymm2, %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 352(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm4, %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX2-NEXT:    vmulpd %ymm5, %ymm10, %ymm10
-; AVX2-NEXT:    vmovapd %ymm5, %ymm3
-; AVX2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX2-NEXT:    vbroadcastsd 360(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm7, %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmulpd %ymm6, %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 368(%rbp), %ymm10
-; AVX2-NEXT:    vmovapd 16(%rbp), %ymm2
-; AVX2-NEXT:    vmulpd %ymm2, %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX2-NEXT:    vmulpd 48(%rbp), %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX2-NEXT:    vbroadcastsd 376(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm10, %ymm14, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmovapd 80(%rbp), %ymm2
-; AVX2-NEXT:    vmulpd %ymm2, %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 384(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm10, %ymm13, %ymm11
-; AVX2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX2-NEXT:    vmovapd 176(%rbp), %ymm14
-; AVX2-NEXT:    vmulpd %ymm10, %ymm14, %ymm10
-; AVX2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX2-NEXT:    vbroadcastsd 392(%rbp), %ymm10
-; AVX2-NEXT:    vmovapd 240(%rbp), %ymm2
-; AVX2-NEXT:    vmulpd %ymm2, %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmovapd %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-NEXT:    vmulpd 208(%rbp), %ymm10, %ymm1
-; AVX2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vmovapd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-NEXT:    vbroadcastsd 400(%rbp), %ymm0
-; AVX2-NEXT:    vmulpd %ymm0, %ymm9, %ymm1
-; AVX2-NEXT:    vbroadcastsd 408(%rbp), %ymm10
-; AVX2-NEXT:    vmovapd %ymm8, %ymm5
-; AVX2-NEXT:    vmulpd %ymm10, %ymm8, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmulpd %ymm0, %ymm15, %ymm0
-; AVX2-NEXT:    vmulpd %ymm10, %ymm12, %ymm10
-; AVX2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 416(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm4, %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX2-NEXT:    vmulpd %ymm3, %ymm10, %ymm10
-; AVX2-NEXT:    vmovapd %ymm3, %ymm2
-; AVX2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX2-NEXT:    vbroadcastsd 424(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm7, %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmulpd %ymm6, %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 432(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd 16(%rbp), %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX2-NEXT:    vmulpd 48(%rbp), %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX2-NEXT:    vbroadcastsd 440(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd 112(%rbp), %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmulpd 80(%rbp), %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 448(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm10, %ymm13, %ymm11
-; AVX2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX2-NEXT:    vmulpd %ymm10, %ymm14, %ymm10
-; AVX2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX2-NEXT:    vbroadcastsd 456(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd 240(%rbp), %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmovapd %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-NEXT:    vmulpd 208(%rbp), %ymm10, %ymm1
-; AVX2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vmovapd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-NEXT:    vbroadcastsd 464(%rbp), %ymm0
-; AVX2-NEXT:    vmulpd %ymm0, %ymm9, %ymm1
-; AVX2-NEXT:    vmovapd %ymm9, %ymm13
-; AVX2-NEXT:    vbroadcastsd 472(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm10, %ymm8, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmulpd %ymm0, %ymm15, %ymm0
-; AVX2-NEXT:    vmovapd %ymm15, %ymm9
-; AVX2-NEXT:    vmulpd %ymm10, %ymm12, %ymm10
-; AVX2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 480(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm4, %ymm10, %ymm11
-; AVX2-NEXT:    vmovapd %ymm4, %ymm3
-; AVX2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX2-NEXT:    vmovapd %ymm2, %ymm15
-; AVX2-NEXT:    vmulpd %ymm2, %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX2-NEXT:    vbroadcastsd 488(%rbp), %ymm10
-; AVX2-NEXT:    vmovapd %ymm7, %ymm8
-; AVX2-NEXT:    vmulpd %ymm7, %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmovapd %ymm6, %ymm7
-; AVX2-NEXT:    vmulpd %ymm6, %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 496(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd 16(%rbp), %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX2-NEXT:    vmovapd 48(%rbp), %ymm4
-; AVX2-NEXT:    vmulpd %ymm4, %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX2-NEXT:    vbroadcastsd 504(%rbp), %ymm10
-; AVX2-NEXT:    vmovapd 112(%rbp), %ymm2
-; AVX2-NEXT:    vmulpd %ymm2, %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmovapd 80(%rbp), %ymm14
-; AVX2-NEXT:    vmulpd %ymm10, %ymm14, %ymm10
-; AVX2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 512(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd 144(%rbp), %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX2-NEXT:    vmovapd 176(%rbp), %ymm2
-; AVX2-NEXT:    vmulpd %ymm2, %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX2-NEXT:    vbroadcastsd 520(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd 240(%rbp), %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmovapd %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-NEXT:    vmulpd 208(%rbp), %ymm10, %ymm1
-; AVX2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vmovapd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-NEXT:    vbroadcastsd 528(%rbp), %ymm0
-; AVX2-NEXT:    vmulpd %ymm0, %ymm13, %ymm1
-; AVX2-NEXT:    vbroadcastsd 536(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm5, %ymm10, %ymm11
-; AVX2-NEXT:    vmovapd %ymm5, %ymm6
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmulpd %ymm0, %ymm9, %ymm0
-; AVX2-NEXT:    vmulpd %ymm10, %ymm12, %ymm10
-; AVX2-NEXT:    vmovapd %ymm12, %ymm5
-; AVX2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 544(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm3, %ymm10, %ymm11
-; AVX2-NEXT:    vmovapd %ymm3, %ymm12
-; AVX2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX2-NEXT:    vmulpd %ymm10, %ymm15, %ymm10
-; AVX2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX2-NEXT:    vbroadcastsd 552(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm10, %ymm8, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmulpd %ymm7, %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 560(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd 16(%rbp), %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX2-NEXT:    vmulpd %ymm4, %ymm10, %ymm10
-; AVX2-NEXT:    vmovapd %ymm4, %ymm3
-; AVX2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX2-NEXT:    vbroadcastsd 568(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd 112(%rbp), %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmulpd %ymm10, %ymm14, %ymm10
-; AVX2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 576(%rbp), %ymm10
-; AVX2-NEXT:    vmovapd 144(%rbp), %ymm4
-; AVX2-NEXT:    vmulpd %ymm4, %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX2-NEXT:    vmulpd %ymm2, %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX2-NEXT:    vbroadcastsd 584(%rbp), %ymm10
-; AVX2-NEXT:    vmovapd 240(%rbp), %ymm14
-; AVX2-NEXT:    vmulpd %ymm10, %ymm14, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmovapd %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-NEXT:    vmovapd 208(%rbp), %ymm2
-; AVX2-NEXT:    vmulpd %ymm2, %ymm10, %ymm1
-; AVX2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vmovapd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-NEXT:    vbroadcastsd 592(%rbp), %ymm0
-; AVX2-NEXT:    vmulpd %ymm0, %ymm13, %ymm1
-; AVX2-NEXT:    vbroadcastsd 600(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm6, %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmulpd %ymm0, %ymm9, %ymm0
-; AVX2-NEXT:    vmulpd %ymm5, %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 608(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm10, %ymm12, %ymm11
-; AVX2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX2-NEXT:    vmulpd %ymm10, %ymm15, %ymm10
-; AVX2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX2-NEXT:    vbroadcastsd 616(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm10, %ymm8, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmulpd %ymm7, %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 624(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd 16(%rbp), %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX2-NEXT:    vmulpd %ymm3, %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX2-NEXT:    vbroadcastsd 632(%rbp), %ymm10
-; AVX2-NEXT:    vmovapd 112(%rbp), %ymm3
-; AVX2-NEXT:    vmulpd %ymm3, %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmovapd 80(%rbp), %ymm3
-; AVX2-NEXT:    vmulpd %ymm3, %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
-; AVX2-NEXT:    vbroadcastsd 640(%rbp), %ymm10
-; AVX2-NEXT:    vmulpd %ymm4, %ymm10, %ymm11
-; AVX2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
-; AVX2-NEXT:    vmovapd 176(%rbp), %ymm3
-; AVX2-NEXT:    vmulpd %ymm3, %ymm10, %ymm10
-; AVX2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
-; AVX2-NEXT:    vbroadcastsd 648(%rbp), %ymm10
-; AVX2-NEXT:    vmovapd %ymm14, %ymm4
-; AVX2-NEXT:    vmulpd %ymm10, %ymm14, %ymm11
-; AVX2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
-; AVX2-NEXT:    vmovapd %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-NEXT:    vmulpd %ymm2, %ymm10, %ymm1
-; AVX2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vmovapd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-NEXT:    vbroadcastsd 656(%rbp), %ymm2
-; AVX2-NEXT:    vmovapd %ymm13, %ymm3
-; AVX2-NEXT:    vmulpd %ymm2, %ymm13, %ymm1
-; AVX2-NEXT:    vbroadcastsd 664(%rbp), %ymm0
-; AVX2-NEXT:    vmulpd %ymm0, %ymm6, %ymm14
-; AVX2-NEXT:    vmovapd %ymm6, %ymm10
-; AVX2-NEXT:    vaddpd %ymm1, %ymm14, %ymm1
-; AVX2-NEXT:    vmulpd %ymm2, %ymm9, %ymm2
-; AVX2-NEXT:    vmulpd %ymm0, %ymm5, %ymm0
-; AVX2-NEXT:    vmovapd %ymm5, %ymm6
-; AVX2-NEXT:    vaddpd %ymm0, %ymm2, %ymm0
-; AVX2-NEXT:    vbroadcastsd 672(%rbp), %ymm2
-; AVX2-NEXT:    vmulpd %ymm2, %ymm12, %ymm14
-; AVX2-NEXT:    vaddpd %ymm0, %ymm14, %ymm0
-; AVX2-NEXT:    vmulpd %ymm2, %ymm15, %ymm2
-; AVX2-NEXT:    vaddpd %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vbroadcastsd 680(%rbp), %ymm2
-; AVX2-NEXT:    vmulpd %ymm2, %ymm8, %ymm14
-; AVX2-NEXT:    vaddpd %ymm1, %ymm14, %ymm1
-; AVX2-NEXT:    vmulpd %ymm2, %ymm7, %ymm2
-; AVX2-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vbroadcastsd 688(%rbp), %ymm2
-; AVX2-NEXT:    vmovapd 16(%rbp), %ymm11
-; AVX2-NEXT:    vmulpd %ymm2, %ymm11, %ymm14
-; AVX2-NEXT:    vaddpd %ymm0, %ymm14, %ymm0
-; AVX2-NEXT:    vmulpd 48(%rbp), %ymm2, %ymm2
-; AVX2-NEXT:    vaddpd %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vbroadcastsd 696(%rbp), %ymm2
-; AVX2-NEXT:    vmovapd 112(%rbp), %ymm5
-; AVX2-NEXT:    vmulpd %ymm2, %ymm5, %ymm14
-; AVX2-NEXT:    vaddpd %ymm1, %ymm14, %ymm1
-; AVX2-NEXT:    vmovapd 80(%rbp), %ymm5
-; AVX2-NEXT:    vmulpd %ymm2, %ymm5, %ymm2
-; AVX2-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vbroadcastsd 704(%rbp), %ymm2
-; AVX2-NEXT:    vmulpd 144(%rbp), %ymm2, %ymm14
-; AVX2-NEXT:    vaddpd %ymm0, %ymm14, %ymm0
-; AVX2-NEXT:    vmovapd 176(%rbp), %ymm13
-; AVX2-NEXT:    vmulpd %ymm2, %ymm13, %ymm2
-; AVX2-NEXT:    vaddpd %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vbroadcastsd 712(%rbp), %ymm2
-; AVX2-NEXT:    vmulpd %ymm2, %ymm4, %ymm14
-; AVX2-NEXT:    vaddpd %ymm1, %ymm14, %ymm1
-; AVX2-NEXT:    vmovapd 208(%rbp), %ymm14
-; AVX2-NEXT:    vmulpd %ymm2, %ymm14, %ymm2
-; AVX2-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vbroadcastsd 720(%rbp), %ymm2
-; AVX2-NEXT:    vmulpd %ymm2, %ymm3, %ymm3
-; AVX2-NEXT:    vmulpd %ymm2, %ymm9, %ymm2
-; AVX2-NEXT:    vbroadcastsd 728(%rbp), %ymm4
-; AVX2-NEXT:    vmulpd %ymm4, %ymm10, %ymm5
-; AVX2-NEXT:    vaddpd %ymm5, %ymm3, %ymm3
-; AVX2-NEXT:    vmulpd %ymm4, %ymm6, %ymm4
-; AVX2-NEXT:    vaddpd %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vbroadcastsd 736(%rbp), %ymm4
-; AVX2-NEXT:    vmulpd %ymm4, %ymm12, %ymm5
-; AVX2-NEXT:    vaddpd %ymm5, %ymm2, %ymm2
-; AVX2-NEXT:    vmulpd %ymm4, %ymm15, %ymm4
-; AVX2-NEXT:    vaddpd %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vbroadcastsd 744(%rbp), %ymm4
-; AVX2-NEXT:    vmulpd %ymm4, %ymm8, %ymm5
-; AVX2-NEXT:    vaddpd %ymm5, %ymm3, %ymm3
-; AVX2-NEXT:    vmulpd %ymm4, %ymm7, %ymm4
-; AVX2-NEXT:    vaddpd %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vbroadcastsd 752(%rbp), %ymm4
-; AVX2-NEXT:    vmulpd %ymm4, %ymm11, %ymm5
-; AVX2-NEXT:    vaddpd %ymm5, %ymm2, %ymm2
-; AVX2-NEXT:    vmulpd 48(%rbp), %ymm4, %ymm4
-; AVX2-NEXT:    vaddpd %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vbroadcastsd 760(%rbp), %ymm4
-; AVX2-NEXT:    vmulpd 112(%rbp), %ymm4, %ymm5
-; AVX2-NEXT:    vaddpd %ymm5, %ymm3, %ymm3
-; AVX2-NEXT:    vmulpd 80(%rbp), %ymm4, %ymm4
-; AVX2-NEXT:    vaddpd %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vbroadcastsd 768(%rbp), %ymm4
-; AVX2-NEXT:    vmulpd 144(%rbp), %ymm4, %ymm5
-; AVX2-NEXT:    vaddpd %ymm5, %ymm2, %ymm2
-; AVX2-NEXT:    vmulpd %ymm4, %ymm13, %ymm4
-; AVX2-NEXT:    vaddpd %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vbroadcastsd 776(%rbp), %ymm4
-; AVX2-NEXT:    vmulpd 240(%rbp), %ymm4, %ymm5
-; AVX2-NEXT:    vaddpd %ymm5, %ymm3, %ymm3
-; AVX2-NEXT:    vmulpd %ymm4, %ymm14, %ymm4
-; AVX2-NEXT:    vaddpd %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vmovapd %ymm3, 480(%rdi)
-; AVX2-NEXT:    vmovapd %ymm2, 448(%rdi)
-; AVX2-NEXT:    vmovapd %ymm1, 416(%rdi)
-; AVX2-NEXT:    vmovapd %ymm0, 384(%rdi)
-; AVX2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-NEXT:    vmovaps %ymm0, 352(%rdi)
-; AVX2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-NEXT:    vmovaps %ymm0, 320(%rdi)
-; AVX2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-NEXT:    vmovaps %ymm0, 288(%rdi)
-; AVX2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-NEXT:    vmovaps %ymm0, 256(%rdi)
-; AVX2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-NEXT:    vmovaps %ymm0, 224(%rdi)
-; AVX2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-NEXT:    vmovaps %ymm0, 192(%rdi)
-; AVX2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-NEXT:    vmovaps %ymm0, 160(%rdi)
-; AVX2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-NEXT:    vmovaps %ymm0, 128(%rdi)
-; AVX2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-NEXT:    vmovaps %ymm0, 96(%rdi)
-; AVX2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-NEXT:    vmovaps %ymm0, 64(%rdi)
-; AVX2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-NEXT:    vmovaps %ymm0, 32(%rdi)
-; AVX2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-NEXT:    vmovaps %ymm0, (%rdi)
-; AVX2-NEXT:    movq %rbp, %rsp
-; AVX2-NEXT:    popq %rbp
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
+; AVX1OR2-LABEL: test_mul8x8_f64:
+; AVX1OR2:       # %bb.0: # %entry
+; AVX1OR2-NEXT:    pushq %rbp
+; AVX1OR2-NEXT:    movq %rsp, %rbp
+; AVX1OR2-NEXT:    andq $-32, %rsp
+; AVX1OR2-NEXT:    subq $448, %rsp # imm = 0x1C0
+; AVX1OR2-NEXT:    vmovapd %ymm2, %ymm12
+; AVX1OR2-NEXT:    vmovapd %ymm0, (%rsp) # 32-byte Spill
+; AVX1OR2-NEXT:    movq %rdi, %rax
+; AVX1OR2-NEXT:    vmovapd 144(%rbp), %ymm13
+; AVX1OR2-NEXT:    vmovapd 112(%rbp), %ymm14
+; AVX1OR2-NEXT:    vbroadcastsd 272(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm1, %ymm10, %ymm8
+; AVX1OR2-NEXT:    vmovapd %ymm1, %ymm9
+; AVX1OR2-NEXT:    vmulpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 280(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm3, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm11, %ymm8, %ymm1
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 288(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm4, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm5, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 296(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm7, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmulpd %ymm6, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 304(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd 16(%rbp), %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
+; AVX1OR2-NEXT:    vmulpd 48(%rbp), %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 312(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm14, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmulpd 80(%rbp), %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 320(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm13, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
+; AVX1OR2-NEXT:    vmulpd 176(%rbp), %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 328(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd 240(%rbp), %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmovapd %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1OR2-NEXT:    vmulpd 208(%rbp), %ymm10, %ymm1
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vmovapd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1OR2-NEXT:    vbroadcastsd 336(%rbp), %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm0, %ymm9, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 344(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm3, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vmovapd %ymm3, %ymm8
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmovapd (%rsp), %ymm15 # 32-byte Reload
+; AVX1OR2-NEXT:    vmulpd %ymm0, %ymm15, %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 352(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm4, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm5, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vmovapd %ymm5, %ymm3
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 360(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm7, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmulpd %ymm6, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 368(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmovapd 16(%rbp), %ymm2
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
+; AVX1OR2-NEXT:    vmulpd 48(%rbp), %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 376(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm14, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmovapd 80(%rbp), %ymm2
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 384(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm13, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
+; AVX1OR2-NEXT:    vmovapd 176(%rbp), %ymm14
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm14, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 392(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmovapd 240(%rbp), %ymm2
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmovapd %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1OR2-NEXT:    vmulpd 208(%rbp), %ymm10, %ymm1
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vmovapd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1OR2-NEXT:    vbroadcastsd 400(%rbp), %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm0, %ymm9, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 408(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmovapd %ymm8, %ymm5
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm8, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmulpd %ymm0, %ymm15, %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm12, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 416(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm4, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm3, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vmovapd %ymm3, %ymm2
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 424(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm7, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmulpd %ymm6, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 432(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd 16(%rbp), %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
+; AVX1OR2-NEXT:    vmulpd 48(%rbp), %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 440(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd 112(%rbp), %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmulpd 80(%rbp), %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 448(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm13, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm14, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 456(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd 240(%rbp), %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmovapd %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1OR2-NEXT:    vmulpd 208(%rbp), %ymm10, %ymm1
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vmovapd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1OR2-NEXT:    vbroadcastsd 464(%rbp), %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm0, %ymm9, %ymm1
+; AVX1OR2-NEXT:    vmovapd %ymm9, %ymm13
+; AVX1OR2-NEXT:    vbroadcastsd 472(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm8, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmulpd %ymm0, %ymm15, %ymm0
+; AVX1OR2-NEXT:    vmovapd %ymm15, %ymm9
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm12, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 480(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm4, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vmovapd %ymm4, %ymm3
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
+; AVX1OR2-NEXT:    vmovapd %ymm2, %ymm15
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 488(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmovapd %ymm7, %ymm8
+; AVX1OR2-NEXT:    vmulpd %ymm7, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmovapd %ymm6, %ymm7
+; AVX1OR2-NEXT:    vmulpd %ymm6, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 496(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd 16(%rbp), %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
+; AVX1OR2-NEXT:    vmovapd 48(%rbp), %ymm4
+; AVX1OR2-NEXT:    vmulpd %ymm4, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 504(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmovapd 112(%rbp), %ymm2
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmovapd 80(%rbp), %ymm14
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm14, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 512(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd 144(%rbp), %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
+; AVX1OR2-NEXT:    vmovapd 176(%rbp), %ymm2
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 520(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd 240(%rbp), %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmovapd %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1OR2-NEXT:    vmulpd 208(%rbp), %ymm10, %ymm1
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vmovapd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1OR2-NEXT:    vbroadcastsd 528(%rbp), %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm0, %ymm13, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 536(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm5, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vmovapd %ymm5, %ymm6
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmulpd %ymm0, %ymm9, %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm12, %ymm10
+; AVX1OR2-NEXT:    vmovapd %ymm12, %ymm5
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 544(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm3, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vmovapd %ymm3, %ymm12
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm15, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 552(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm8, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmulpd %ymm7, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 560(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd 16(%rbp), %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm4, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vmovapd %ymm4, %ymm3
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 568(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd 112(%rbp), %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm14, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 576(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmovapd 144(%rbp), %ymm4
+; AVX1OR2-NEXT:    vmulpd %ymm4, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 584(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmovapd 240(%rbp), %ymm14
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm14, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmovapd %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1OR2-NEXT:    vmovapd 208(%rbp), %ymm2
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vmovapd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1OR2-NEXT:    vbroadcastsd 592(%rbp), %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm0, %ymm13, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 600(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm6, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmulpd %ymm0, %ymm9, %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm5, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 608(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm12, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm15, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 616(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm8, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmulpd %ymm7, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 624(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd 16(%rbp), %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm3, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 632(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmovapd 112(%rbp), %ymm3
+; AVX1OR2-NEXT:    vmulpd %ymm3, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmovapd 80(%rbp), %ymm3
+; AVX1OR2-NEXT:    vmulpd %ymm3, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm10, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 640(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmulpd %ymm4, %ymm10, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm11, %ymm0
+; AVX1OR2-NEXT:    vmovapd 176(%rbp), %ymm3
+; AVX1OR2-NEXT:    vmulpd %ymm3, %ymm10, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 648(%rbp), %ymm10
+; AVX1OR2-NEXT:    vmovapd %ymm14, %ymm4
+; AVX1OR2-NEXT:    vmulpd %ymm10, %ymm14, %ymm11
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm11, %ymm1
+; AVX1OR2-NEXT:    vmovapd %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm10, %ymm1
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vmovapd %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1OR2-NEXT:    vbroadcastsd 656(%rbp), %ymm2
+; AVX1OR2-NEXT:    vmovapd %ymm13, %ymm3
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm13, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 664(%rbp), %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm0, %ymm6, %ymm14
+; AVX1OR2-NEXT:    vmovapd %ymm6, %ymm10
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm14, %ymm1
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm9, %ymm2
+; AVX1OR2-NEXT:    vmulpd %ymm0, %ymm5, %ymm0
+; AVX1OR2-NEXT:    vmovapd %ymm5, %ymm6
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm2, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 672(%rbp), %ymm2
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm12, %ymm14
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm14, %ymm0
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm15, %ymm2
+; AVX1OR2-NEXT:    vaddpd %ymm2, %ymm1, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 680(%rbp), %ymm2
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm8, %ymm14
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm14, %ymm1
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm7, %ymm2
+; AVX1OR2-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 688(%rbp), %ymm2
+; AVX1OR2-NEXT:    vmovapd 16(%rbp), %ymm11
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm11, %ymm14
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm14, %ymm0
+; AVX1OR2-NEXT:    vmulpd 48(%rbp), %ymm2, %ymm2
+; AVX1OR2-NEXT:    vaddpd %ymm2, %ymm1, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 696(%rbp), %ymm2
+; AVX1OR2-NEXT:    vmovapd 112(%rbp), %ymm5
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm5, %ymm14
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm14, %ymm1
+; AVX1OR2-NEXT:    vmovapd 80(%rbp), %ymm5
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm5, %ymm2
+; AVX1OR2-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 704(%rbp), %ymm2
+; AVX1OR2-NEXT:    vmulpd 144(%rbp), %ymm2, %ymm14
+; AVX1OR2-NEXT:    vaddpd %ymm0, %ymm14, %ymm0
+; AVX1OR2-NEXT:    vmovapd 176(%rbp), %ymm13
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm13, %ymm2
+; AVX1OR2-NEXT:    vaddpd %ymm2, %ymm1, %ymm1
+; AVX1OR2-NEXT:    vbroadcastsd 712(%rbp), %ymm2
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm4, %ymm14
+; AVX1OR2-NEXT:    vaddpd %ymm1, %ymm14, %ymm1
+; AVX1OR2-NEXT:    vmovapd 208(%rbp), %ymm14
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm14, %ymm2
+; AVX1OR2-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vbroadcastsd 720(%rbp), %ymm2
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm3, %ymm3
+; AVX1OR2-NEXT:    vmulpd %ymm2, %ymm9, %ymm2
+; AVX1OR2-NEXT:    vbroadcastsd 728(%rbp), %ymm4
+; AVX1OR2-NEXT:    vmulpd %ymm4, %ymm10, %ymm5
+; AVX1OR2-NEXT:    vaddpd %ymm5, %ymm3, %ymm3
+; AVX1OR2-NEXT:    vmulpd %ymm4, %ymm6, %ymm4
+; AVX1OR2-NEXT:    vaddpd %ymm4, %ymm2, %ymm2
+; AVX1OR2-NEXT:    vbroadcastsd 736(%rbp), %ymm4
+; AVX1OR2-NEXT:    vmulpd %ymm4, %ymm12, %ymm5
+; AVX1OR2-NEXT:    vaddpd %ymm5, %ymm2, %ymm2
+; AVX1OR2-NEXT:    vmulpd %ymm4, %ymm15, %ymm4
+; AVX1OR2-NEXT:    vaddpd %ymm4, %ymm3, %ymm3
+; AVX1OR2-NEXT:    vbroadcastsd 744(%rbp), %ymm4
+; AVX1OR2-NEXT:    vmulpd %ymm4, %ymm8, %ymm5
+; AVX1OR2-NEXT:    vaddpd %ymm5, %ymm3, %ymm3
+; AVX1OR2-NEXT:    vmulpd %ymm4, %ymm7, %ymm4
+; AVX1OR2-NEXT:    vaddpd %ymm4, %ymm2, %ymm2
+; AVX1OR2-NEXT:    vbroadcastsd 752(%rbp), %ymm4
+; AVX1OR2-NEXT:    vmulpd %ymm4, %ymm11, %ymm5
+; AVX1OR2-NEXT:    vaddpd %ymm5, %ymm2, %ymm2
+; AVX1OR2-NEXT:    vmulpd 48(%rbp), %ymm4, %ymm4
+; AVX1OR2-NEXT:    vaddpd %ymm4, %ymm3, %ymm3
+; AVX1OR2-NEXT:    vbroadcastsd 760(%rbp), %ymm4
+; AVX1OR2-NEXT:    vmulpd 112(%rbp), %ymm4, %ymm5
+; AVX1OR2-NEXT:    vaddpd %ymm5, %ymm3, %ymm3
+; AVX1OR2-NEXT:    vmulpd 80(%rbp), %ymm4, %ymm4
+; AVX1OR2-NEXT:    vaddpd %ymm4, %ymm2, %ymm2
+; AVX1OR2-NEXT:    vbroadcastsd 768(%rbp), %ymm4
+; AVX1OR2-NEXT:    vmulpd 144(%rbp), %ymm4, %ymm5
+; AVX1OR2-NEXT:    vaddpd %ymm5, %ymm2, %ymm2
+; AVX1OR2-NEXT:    vmulpd %ymm4, %ymm13, %ymm4
+; AVX1OR2-NEXT:    vaddpd %ymm4, %ymm3, %ymm3
+; AVX1OR2-NEXT:    vbroadcastsd 776(%rbp), %ymm4
+; AVX1OR2-NEXT:    vmulpd 240(%rbp), %ymm4, %ymm5
+; AVX1OR2-NEXT:    vaddpd %ymm5, %ymm3, %ymm3
+; AVX1OR2-NEXT:    vmulpd %ymm4, %ymm14, %ymm4
+; AVX1OR2-NEXT:    vaddpd %ymm4, %ymm2, %ymm2
+; AVX1OR2-NEXT:    vmovapd %ymm3, 480(%rdi)
+; AVX1OR2-NEXT:    vmovapd %ymm2, 448(%rdi)
+; AVX1OR2-NEXT:    vmovapd %ymm1, 416(%rdi)
+; AVX1OR2-NEXT:    vmovapd %ymm0, 384(%rdi)
+; AVX1OR2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX1OR2-NEXT:    vmovaps %ymm0, 352(%rdi)
+; AVX1OR2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX1OR2-NEXT:    vmovaps %ymm0, 320(%rdi)
+; AVX1OR2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX1OR2-NEXT:    vmovaps %ymm0, 288(%rdi)
+; AVX1OR2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX1OR2-NEXT:    vmovaps %ymm0, 256(%rdi)
+; AVX1OR2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX1OR2-NEXT:    vmovaps %ymm0, 224(%rdi)
+; AVX1OR2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX1OR2-NEXT:    vmovaps %ymm0, 192(%rdi)
+; AVX1OR2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX1OR2-NEXT:    vmovaps %ymm0, 160(%rdi)
+; AVX1OR2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX1OR2-NEXT:    vmovaps %ymm0, 128(%rdi)
+; AVX1OR2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX1OR2-NEXT:    vmovaps %ymm0, 96(%rdi)
+; AVX1OR2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX1OR2-NEXT:    vmovaps %ymm0, 64(%rdi)
+; AVX1OR2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX1OR2-NEXT:    vmovaps %ymm0, 32(%rdi)
+; AVX1OR2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX1OR2-NEXT:    vmovaps %ymm0, (%rdi)
+; AVX1OR2-NEXT:    movq %rbp, %rsp
+; AVX1OR2-NEXT:    popq %rbp
+; AVX1OR2-NEXT:    vzeroupper
+; AVX1OR2-NEXT:    retq
 ;
 ; AVX512-LABEL: test_mul8x8_f64:
 ; AVX512:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/X86/membarrier.ll b/llvm/test/CodeGen/X86/membarrier.ll
index 55f2a2f210139..2773f01f7ab82 100644
--- a/llvm/test/CodeGen/X86/membarrier.ll
+++ b/llvm/test/CodeGen/X86/membarrier.ll
@@ -6,9 +6,9 @@ define i32 @t() {
 ; CHECK-LABEL: t:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl $1, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    mfence
+; CHECK-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    lock decl -{{[0-9]+}}(%rsp)
-; CHECK-NEXT:    mfence
+; CHECK-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    retq
   %i = alloca i32, align 4
diff --git a/llvm/test/CodeGen/X86/mfence.ll b/llvm/test/CodeGen/X86/mfence.ll
index f34657b3f240c..f2d3bc8217b1d 100644
--- a/llvm/test/CodeGen/X86/mfence.ll
+++ b/llvm/test/CodeGen/X86/mfence.ll
@@ -1,37 +1,47 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X32
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-sse2 | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-sse2 | FileCheck %s --check-prefixes=CHECK,X64
 
 ; It doesn't matter if an x86-64 target has specified "no-sse2"; we still can use mfence.
 
 define void @test() {
-; X32-LABEL: test:
-; X32:       # %bb.0:
-; X32-NEXT:    mfence
-; X32-NEXT:    retl
+; X86-LABEL: test:
+; X86:       # %bb.0:
+; X86-NEXT:    mfence
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: test:
 ; X64:       # %bb.0:
-; X64-NEXT:    mfence
+; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
   fence seq_cst
   ret void
 }
 
 define i32 @fence(ptr %ptr) {
-; X32-LABEL: fence:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    mfence
-; X32-NEXT:    movl (%eax), %eax
-; X32-NEXT:    retl
+; X86-LABEL: fence:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mfence
+; X86-NEXT:    movl (%eax), %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: fence:
 ; X64:       # %bb.0:
-; X64-NEXT:    mfence
+; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    movl (%rdi), %eax
 ; X64-NEXT:    retq
   %atomic = atomicrmw add ptr %ptr, i32 0 seq_cst
   ret i32 %atomic
 }
 
+define void @mfence() nounwind {
+; CHECK-LABEL: mfence:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mfence
+; CHECK-NEXT:    ret{{[l|q]}}
+  call void @llvm.x86.sse2.mfence()
+  ret void
+}
+declare void @llvm.x86.sse2.mfence() nounwind readnone
+
diff --git a/llvm/test/CodeGen/X86/mulvi32.ll b/llvm/test/CodeGen/X86/mulvi32.ll
index 46d5da54a7482..bbda4d68bb685 100644
--- a/llvm/test/CodeGen/X86/mulvi32.ll
+++ b/llvm/test/CodeGen/X86/mulvi32.ll
@@ -286,12 +286,8 @@ define <4 x i64> @_mul4xi32toi64c(<4 x i32>, <4 x i32>) {
 ;
 ; AVX2-LABEL: _mul4xi32toi64c:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
-; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm3, %ymm1
-; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
   %lower0 = shufflevector <4 x i32> %0, <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 1, i32 undef>
diff --git a/llvm/test/CodeGen/X86/optimize-compare.mir b/llvm/test/CodeGen/X86/optimize-compare.mir
index 36ab851a85153..789a5c0100b50 100644
--- a/llvm/test/CodeGen/X86/optimize-compare.mir
+++ b/llvm/test/CodeGen/X86/optimize-compare.mir
@@ -422,7 +422,7 @@ body: |
     ; CMP+SETCC   %0 != 42
     CMP64ri32 %0, 42, implicit-def $eflags
     $cl = SETCCr 5, implicit $eflags
-    ; CMP+SETCC   %0 >= 2; CMP can be removed.
+    ; CMP+SETCC   %0 > 41; CMP can be removed.
     CMP64ri32 %0, 41, implicit-def $eflags
     ; %0 >s 41  -->  %0 >=s 42
     $bl = SETCCr 15, implicit $eflags
diff --git a/llvm/test/CodeGen/X86/paddus.ll b/llvm/test/CodeGen/X86/paddus.ll
index 81e0a09d0e3fa..26c55d6371b51 100644
--- a/llvm/test/CodeGen/X86/paddus.ll
+++ b/llvm/test/CodeGen/X86/paddus.ll
@@ -450,7 +450,7 @@ define <64 x i8> @test13(<64 x i8> %x) {
 ;
 ; AVX512-LABEL: test13:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = -1
 ; AVX512-NEXT:    vpcmpneqb %zmm1, %zmm0, %k1
 ; AVX512-NEXT:    vpsubb %zmm1, %zmm0, %zmm1 {%k1}
 ; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0
@@ -659,7 +659,7 @@ define <64 x i8> @test17(<64 x i8> %x) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm1
 ; AVX512-NEXT:    vpcmpltub %zmm0, %zmm1, %k1
-; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = -1
 ; AVX512-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
 ; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512-NEXT:    retq
@@ -1230,7 +1230,7 @@ define <32 x i16> @test31(<32 x i16> %x) {
 ;
 ; AVX512-LABEL: test31:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = -1
 ; AVX512-NEXT:    vpcmpneqw %zmm1, %zmm0, %k1
 ; AVX512-NEXT:    vpsubw %zmm1, %zmm0, %zmm1 {%k1}
 ; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0
@@ -1513,7 +1513,7 @@ define <32 x i16> @test35(<32 x i16> %x) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm1
 ; AVX512-NEXT:    vpcmpltuw %zmm0, %zmm1, %k1
-; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0
+; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = -1
 ; AVX512-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
 ; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/pmul.ll b/llvm/test/CodeGen/X86/pmul.ll
index fe8a4fa163129..c7cc2acaf2627 100644
--- a/llvm/test/CodeGen/X86/pmul.ll
+++ b/llvm/test/CodeGen/X86/pmul.ll
@@ -959,7 +959,7 @@ define <64 x i8> @mul_v64i8(<64 x i8> %i, <64 x i8> %j) nounwind  {
 ;
 ; AVX512F-LABEL: mul_v64i8:
 ; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} zmm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} zmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
 ; AVX512F-NEXT:    vpand %ymm2, %ymm3, %ymm4
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
diff --git a/llvm/test/CodeGen/X86/seh-except-restore.ll b/llvm/test/CodeGen/X86/seh-except-restore.ll
index e362f01791d06..f428a83cc2aef 100644
--- a/llvm/test/CodeGen/X86/seh-except-restore.ll
+++ b/llvm/test/CodeGen/X86/seh-except-restore.ll
@@ -48,6 +48,7 @@ return:                                           ; preds = %entry, %__except
 ; CHECK: LBB0_2:                                 # %return
 
 ; CHECK: LBB0_1:                                 # %__except.ret
+; CHECK-NEXT:         $ehgcr_0_1:
 ; CHECK-NEXT:         movl    -24(%ebp), %esp
 ; CHECK-NEXT:         addl    $12, %ebp
 
diff --git a/llvm/test/CodeGen/X86/tail-dup-computed-goto.mir b/llvm/test/CodeGen/X86/tail-dup-computed-goto.mir
new file mode 100644
index 0000000000000..a472dc67d8d51
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tail-dup-computed-goto.mir
@@ -0,0 +1,255 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=early-tailduplication -tail-dup-pred-size=1 -tail-dup-succ-size=1 %s -o - | FileCheck %s
+# Check that only the computed goto is not be restrict by tail-dup-pred-size and tail-dup-succ-size.
+--- |
+  declare i64 @f0()
+  declare i64 @f1()
+  declare i64 @f2()
+  declare i64 @f3()
+  declare i64 @f4()
+  declare i64 @f5()
+  @computed_goto.dispatch = external global [5 x ptr]
+  define void @computed_goto() { ret void }
+  define void @jump_table() { ret void }
+...
+---
+name:            computed_goto
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: computed_goto
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x20000000), %bb.2(0x20000000), %bb.3(0x20000000), %bb.4(0x20000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   CALL64pcrel32 target-flags(x86-plt) @f0, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+  ; CHECK-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr64 = COPY $rax
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gr64_nosp = COPY [[COPY]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gr64_nosp = COPY [[COPY1]]
+  ; CHECK-NEXT:   JMP64m $noreg, 8, [[COPY1]], @computed_goto.dispatch, $noreg
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x20000000), %bb.2(0x20000000), %bb.3(0x20000000), %bb.4(0x20000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   CALL64pcrel32 target-flags(x86-plt) @f1, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+  ; CHECK-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gr64 = COPY $rax
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gr64_nosp = COPY [[COPY3]]
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:gr64_nosp = COPY [[COPY4]]
+  ; CHECK-NEXT:   JMP64m $noreg, 8, [[COPY4]], @computed_goto.dispatch, $noreg
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.1(0x20000000), %bb.2(0x20000000), %bb.3(0x20000000), %bb.4(0x20000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   CALL64pcrel32 target-flags(x86-plt) @f2, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+  ; CHECK-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:gr64 = COPY $rax
+  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:gr64_nosp = COPY [[COPY6]]
+  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:gr64_nosp = COPY [[COPY7]]
+  ; CHECK-NEXT:   JMP64m $noreg, 8, [[COPY7]], @computed_goto.dispatch, $noreg
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   successors: %bb.1(0x20000000), %bb.2(0x20000000), %bb.3(0x20000000), %bb.4(0x20000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   CALL64pcrel32 target-flags(x86-plt) @f3, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+  ; CHECK-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:gr64 = COPY $rax
+  ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:gr64_nosp = COPY [[COPY9]]
+  ; CHECK-NEXT:   [[COPY11:%[0-9]+]]:gr64_nosp = COPY [[COPY10]]
+  ; CHECK-NEXT:   JMP64m $noreg, 8, [[COPY10]], @computed_goto.dispatch, $noreg
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.4:
+  ; CHECK-NEXT:   successors: %bb.1(0x20000000), %bb.2(0x20000000), %bb.3(0x20000000), %bb.4(0x20000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   CALL64pcrel32 target-flags(x86-plt) @f4, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+  ; CHECK-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   [[COPY12:%[0-9]+]]:gr64 = COPY $rax
+  ; CHECK-NEXT:   [[COPY13:%[0-9]+]]:gr64_nosp = COPY [[COPY12]]
+  ; CHECK-NEXT:   [[COPY14:%[0-9]+]]:gr64_nosp = COPY [[COPY13]]
+  ; CHECK-NEXT:   JMP64m $noreg, 8, [[COPY13]], @computed_goto.dispatch, $noreg
+  bb.0:
+    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    CALL64pcrel32 target-flags(x86-plt) @f0, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    %6:gr64 = COPY $rax
+    %0:gr64 = COPY %6
+    JMP_1 %bb.5
+
+  bb.1:
+    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    CALL64pcrel32 target-flags(x86-plt) @f1, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    %10:gr64 = COPY $rax
+    %1:gr64 = COPY %10
+    JMP_1 %bb.5
+
+  bb.2:
+    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    CALL64pcrel32 target-flags(x86-plt) @f2, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    %9:gr64 = COPY $rax
+    %2:gr64 = COPY %9
+    JMP_1 %bb.5
+
+  bb.3:
+    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    CALL64pcrel32 target-flags(x86-plt) @f3, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    %8:gr64 = COPY $rax
+    %3:gr64 = COPY %8
+    JMP_1 %bb.5
+
+  bb.4:
+    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    CALL64pcrel32 target-flags(x86-plt) @f4, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    %7:gr64 = COPY $rax
+    %4:gr64 = COPY %7
+
+  bb.5:
+    successors: %bb.1, %bb.2, %bb.3, %bb.4
+
+    %5:gr64_nosp = PHI %0, %bb.0, %4, %bb.4, %3, %bb.3, %2, %bb.2, %1, %bb.1
+    JMP64m $noreg, 8, %5, @computed_goto.dispatch, $noreg
+
+...
+---
+name:            jump_table
+tracksRegLiveness: true
+jumpTable:
+  kind:            block-address
+  entries:
+    - id:              0
+      blocks:          [ '%bb.2', '%bb.3', '%bb.4', '%bb.5', '%bb.6' ]
+body:             |
+  ; CHECK-LABEL: name: jump_table
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   CALL64pcrel32 target-flags(x86-plt) @f0, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+  ; CHECK-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr64 = COPY $rax
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gr64 = COPY [[COPY]]
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.3(0x1999999a), %bb.4(0x1999999a), %bb.5(0x1999999a), %bb.6(0x1999999a), %bb.7(0x1999999a)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:gr64 = PHI [[COPY1]], %bb.0, %3, %bb.7, %4, %bb.6, %5, %bb.5, %6, %bb.4, %7, %bb.3
+  ; CHECK-NEXT:   [[DEC64r:%[0-9]+]]:gr64_nosp = DEC64r [[PHI]], implicit-def dead $eflags
+  ; CHECK-NEXT:   JMP64m $noreg, 8, [[DEC64r]], %jump-table.0, $noreg :: (load (s64) from jump-table)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   CALL64pcrel32 target-flags(x86-plt) @f1, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+  ; CHECK-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gr64 = COPY $rax
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gr64 = COPY [[COPY2]]
+  ; CHECK-NEXT:   JMP_1 %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.4:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   CALL64pcrel32 target-flags(x86-plt) @f2, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+  ; CHECK-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gr64 = COPY $rax
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:gr64 = COPY [[COPY4]]
+  ; CHECK-NEXT:   JMP_1 %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.5:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   CALL64pcrel32 target-flags(x86-plt) @f3, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+  ; CHECK-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:gr64 = COPY $rax
+  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:gr64 = COPY [[COPY6]]
+  ; CHECK-NEXT:   JMP_1 %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.6:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   CALL64pcrel32 target-flags(x86-plt) @f4, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+  ; CHECK-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:gr64 = COPY $rax
+  ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:gr64 = COPY [[COPY8]]
+  ; CHECK-NEXT:   JMP_1 %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.7:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   CALL64pcrel32 target-flags(x86-plt) @f5, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+  ; CHECK-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:gr64 = COPY $rax
+  ; CHECK-NEXT:   [[COPY11:%[0-9]+]]:gr64 = COPY [[COPY10]]
+  ; CHECK-NEXT:   JMP_1 %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.8:
+  bb.0:
+    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    CALL64pcrel32 target-flags(x86-plt) @f0, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    %7:gr64 = COPY $rax
+    %0:gr64 = COPY %7
+
+  bb.1:
+    %1:gr64 = PHI %0, %bb.0, %6, %bb.6, %5, %bb.5, %4, %bb.4, %3, %bb.3, %2, %bb.2
+    %8:gr64_nosp = DEC64r %1, implicit-def dead $eflags
+
+  bb.8:
+    successors: %bb.2(0x1999999a), %bb.3(0x1999999a), %bb.4(0x1999999a), %bb.5(0x1999999a), %bb.6(0x1999999a)
+
+    JMP64m $noreg, 8, %8, %jump-table.0, $noreg :: (load (s64) from jump-table)
+
+  bb.2:
+    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    CALL64pcrel32 target-flags(x86-plt) @f1, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    %13:gr64 = COPY $rax
+    %2:gr64 = COPY %13
+    JMP_1 %bb.1
+
+  bb.3:
+    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    CALL64pcrel32 target-flags(x86-plt) @f2, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    %12:gr64 = COPY $rax
+    %3:gr64 = COPY %12
+    JMP_1 %bb.1
+
+  bb.4:
+    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    CALL64pcrel32 target-flags(x86-plt) @f3, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    %11:gr64 = COPY $rax
+    %4:gr64 = COPY %11
+    JMP_1 %bb.1
+
+  bb.5:
+    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    CALL64pcrel32 target-flags(x86-plt) @f4, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    %10:gr64 = COPY $rax
+    %5:gr64 = COPY %10
+    JMP_1 %bb.1
+
+  bb.6:
+    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    CALL64pcrel32 target-flags(x86-plt) @f5, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit-def $rax
+    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+    %9:gr64 = COPY $rax
+    %6:gr64 = COPY %9
+    JMP_1 %bb.1
+
+  bb.7:
+
+...
diff --git a/llvm/test/CodeGen/X86/vector-fshl-256.ll b/llvm/test/CodeGen/X86/vector-fshl-256.ll
index 32ad72b2aa56a..fdd0d68b89003 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-256.ll
@@ -887,15 +887,15 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
 ; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
 ; AVX1-NEXT:    vpsllq %xmm2, %xmm5, %xmm5
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX1-NEXT:    vpsllq %xmm2, %xmm6, %xmm6
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm5
 ; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
 ; AVX1-NEXT:    vpsllq %xmm2, %xmm3, %xmm3
+; AVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[1,3],xmm5[1,3]
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX1-NEXT:    vpsllq %xmm2, %xmm4, %xmm4
 ; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX1-NEXT:    vpsllq %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm4[1,3]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm5[1,3],ymm0[5,7],ymm5[5,7]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v8i32:
@@ -970,15 +970,15 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; XOPAVX1-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
 ; XOPAVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsllq %xmm2, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; XOPAVX1-NEXT:    vpsllq %xmm2, %xmm6, %xmm6
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm5
 ; XOPAVX1-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
 ; XOPAVX1-NEXT:    vpsllq %xmm2, %xmm3, %xmm3
+; XOPAVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[1,3],xmm5[1,3]
+; XOPAVX1-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; XOPAVX1-NEXT:    vpsllq %xmm2, %xmm4, %xmm4
 ; XOPAVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; XOPAVX1-NEXT:    vpsllq %xmm2, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm4[1,3]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm5[1,3],ymm0[5,7],ymm5[5,7]
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_funnnel_v8i32:
@@ -2270,20 +2270,16 @@ define <16 x i16> @splatconstant_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y) no
 define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; AVX1-LABEL: splatconstant_funnnel_v32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm2
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
index c7e430de1af97..f0f3d6028c27a 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
@@ -668,15 +668,15 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %amt) nounwind
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[2,2,3,3]
 ; AVX1-NEXT:    vpsllq %xmm1, %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,2,3,3]
-; AVX1-NEXT:    vpsllq %xmm1, %xmm4, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
 ; AVX1-NEXT:    vpsllq %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[1,3],xmm3[1,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,2,3,3]
+; AVX1-NEXT:    vpsllq %xmm1, %xmm3, %xmm3
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
 ; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm3[1,3]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm3[1,3],ymm0[5,7],ymm3[5,7]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v8i32:
@@ -1623,19 +1623,17 @@ define <16 x i16> @splatconstant_funnnel_v16i16(<16 x i16> %x) nounwind {
 define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX1-LABEL: splatconstant_funnnel_v32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm2
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm2
-; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    vbroadcastss {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX1-NEXT:    vandnps %ymm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %ymm3, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatconstant_funnnel_v32i8:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-128.ll b/llvm/test/CodeGen/X86/vector-fshr-128.ll
index d830834a83ab4..a56b0a6351a3b 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-128.ll
@@ -1566,11 +1566,12 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ;
 ; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i8:
 ; AVX512VLVBMI2:       # %bb.0:
-; AVX512VLVBMI2-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; AVX512VLVBMI2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512VLVBMI2-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
-; AVX512VLVBMI2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm1
-; AVX512VLVBMI2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $ymm1
+; AVX512VLVBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,32,1,33,2,34,3,35,4,36,5,37,6,38,7,39,8,40,9,41,10,42,11,43,12,44,13,45,14,46,15,47]
+; AVX512VLVBMI2-NEXT:    vpermi2b %ymm0, %ymm1, %ymm3
+; AVX512VLVBMI2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm0
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm0, %ymm3, %ymm0
 ; AVX512VLVBMI2-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512VLVBMI2-NEXT:    vzeroupper
 ; AVX512VLVBMI2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-fshr-256.ll b/llvm/test/CodeGen/X86/vector-fshr-256.ll
index 9c259ed38321d..6d7d279a3637a 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-256.ll
@@ -918,15 +918,15 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
 ; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlq %xmm2, %xmm5, %xmm5
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX1-NEXT:    vpsrlq %xmm2, %xmm6, %xmm6
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm5
 ; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
 ; AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm3
+; AVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[0,2],xmm5[0,2]
+; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX1-NEXT:    vpsrlq %xmm2, %xmm4, %xmm4
 ; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX1-NEXT:    vpsrlq %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm4[0,2]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm5[0,2],ymm0[4,6],ymm5[4,6]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v8i32:
@@ -1002,15 +1002,15 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; XOPAVX1-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
 ; XOPAVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsrlq %xmm2, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; XOPAVX1-NEXT:    vpsrlq %xmm2, %xmm6, %xmm6
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm5
 ; XOPAVX1-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
 ; XOPAVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm3
+; XOPAVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[0,2],xmm5[0,2]
+; XOPAVX1-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; XOPAVX1-NEXT:    vpsrlq %xmm2, %xmm4, %xmm4
 ; XOPAVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; XOPAVX1-NEXT:    vpsrlq %xmm2, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm4[0,2]
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm5[0,2],ymm0[4,6],ymm5[4,6]
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_funnnel_v8i32:
@@ -1625,17 +1625,6 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; AVX1-NEXT:    vpmullw %xmm6, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackuswb %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT:    vpaddb %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = [128,32,8,2,128,2,8,32]
-; AVX1-NEXT:    vpmaddubsw %xmm7, %xmm4, %xmm8
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vpand %xmm9, %xmm8, %xmm8
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm10 = [0,64,0,16,0,4,0,1,0,1,0,4,0,16,0,64]
-; AVX1-NEXT:    vpmaddubsw %xmm10, %xmm4, %xmm4
-; AVX1-NEXT:    vpsllw $8, %xmm4, %xmm4
-; AVX1-NEXT:    vpor %xmm4, %xmm8, %xmm4
-; AVX1-NEXT:    vpor %xmm2, %xmm4, %xmm2
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm3[8],xmm1[9],xmm3[9],xmm1[10],xmm3[10],xmm1[11],xmm3[11],xmm1[12],xmm3[12],xmm1[13],xmm3[13],xmm1[14],xmm3[14],xmm1[15],xmm3[15]
 ; AVX1-NEXT:    vpmullw %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
@@ -1643,14 +1632,23 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; AVX1-NEXT:    vpmullw %xmm6, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,64,0,16,0,4,0,1,0,1,0,4,0,16,0,64]
+; AVX1-NEXT:    vpmaddubsw %xmm3, %xmm2, %xmm4
+; AVX1-NEXT:    vpsllw $8, %xmm4, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpmaddubsw %xmm7, %xmm0, %xmm3
-; AVX1-NEXT:    vpand %xmm3, %xmm9, %xmm3
-; AVX1-NEXT:    vpmaddubsw %xmm10, %xmm0, %xmm0
-; AVX1-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX1-NEXT:    vpor %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vpmaddubsw %xmm3, %xmm0, %xmm3
+; AVX1-NEXT:    vpsllw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
+; AVX1-NEXT:    vorps %ymm1, %ymm3, %ymm1
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = [128,32,8,2,128,2,8,32]
+; AVX1-NEXT:    vpmaddubsw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpmaddubsw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_funnnel_v32i8:
@@ -2058,20 +2056,16 @@ define <16 x i16> @splatconstant_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y) no
 define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; AVX1-LABEL: splatconstant_funnnel_v32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm2
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
index 399bdb7a248b8..9ce682306f18b 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
@@ -1194,9 +1194,9 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v16i8:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512VLBW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512VLBW-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
+; AVX512VLBW-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512VLBW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512VLBW-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512VLBW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
 ; AVX512VLBW-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    vpmovwb %ymm0, %xmm0
@@ -1218,9 +1218,9 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ;
 ; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i8:
 ; AVX512VLVBMI2:       # %bb.0:
-; AVX512VLVBMI2-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512VLVBMI2-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512VLVBMI2-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
+; AVX512VLVBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512VLVBMI2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512VLVBMI2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512VLVBMI2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
 ; AVX512VLVBMI2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; AVX512VLVBMI2-NEXT:    vpmovwb %ymm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
index d81d6ef7c9f28..c66a3f709c5ca 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
@@ -703,15 +703,15 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %amt) nounwind
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[2,2,3,3]
 ; AVX1-NEXT:    vpsrlq %xmm1, %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,2,3,3]
-; AVX1-NEXT:    vpsrlq %xmm1, %xmm4, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
 ; AVX1-NEXT:    vpsrlq %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,2,3,3]
+; AVX1-NEXT:    vpsrlq %xmm1, %xmm3, %xmm3
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
 ; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm3[0,2],ymm0[4,6],ymm3[4,6]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v8i32:
@@ -1674,19 +1674,17 @@ define <16 x i16> @splatconstant_funnnel_v16i16(<16 x i16> %x) nounwind {
 define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX1-LABEL: splatconstant_funnnel_v32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm2
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm2
-; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    vbroadcastss {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX1-NEXT:    vandnps %ymm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %ymm3, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatconstant_funnnel_v32i8:
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
index e97d7e5bb090c..c0c93646b5aaf 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
@@ -164,38 +164,36 @@ define <16 x i16> @test_div7_16i16(<16 x i16> %a) nounwind {
 define <32 x i8> @test_div7_32i8(<32 x i8> %a) nounwind {
 ; AVX1-LABEL: test_div7_32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [37,37,37,37,37,37,37,37]
-; AVX1-NEXT:    vpmullw %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm5
-; AVX1-NEXT:    vpackuswb %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vpsubb %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $1, %xmm1, %xmm1
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX1-NEXT:    vpand %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $2, %xmm1, %xmm1
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
-; AVX1-NEXT:    vpmullw %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [37,37,37,37,37,37,37,37]
+; AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmullw %xmm4, %xmm6, %xmm4
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpmullw %xmm3, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm4
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vpsubb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubb %xmm2, %xmm0, %xmm4
+; AVX1-NEXT:    vpsrlw $1, %xmm4, %xmm4
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpaddb %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpmullw %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2NOBW-LABEL: test_div7_32i8:
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll
index 9f69a3cf44189..81fe19c4d8b56 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-2.ll
@@ -1049,24 +1049,24 @@ define void @load_i8_stride2_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
 ; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %ymm3
 ; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %ymm4
 ; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm4, %ymm5
+; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm3, %ymm7
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm8 = [0,2,5,7]
+; AVX512-FCP-NEXT:    vpermt2q %ymm5, %ymm8, %ymm7
 ; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm3, %ymm6
-; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm1, %ymm5
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm5
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm6 = [0,2,9,11,4,6,13,15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm6, %zmm5
-; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm4, %ymm4
-; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm3, %ymm3
-; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm5
+; AVX512-FCP-NEXT:    vpermt2q %ymm0, %ymm8, %ymm5
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm5, %zmm0
+; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
+; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm4, %ymm4
+; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
+; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm3, %ymm3
+; AVX512-FCP-NEXT:    vpermt2q %ymm4, %ymm8, %ymm3
+; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm2, %ymm2
+; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
+; AVX512-FCP-NEXT:    vpermt2q %ymm2, %ymm8, %ymm1
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm6, %zmm1
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, (%rsi)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, (%rsi)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, (%rdx)
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
@@ -1110,24 +1110,24 @@ define void @load_i8_stride2_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1) nou
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %ymm3
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %ymm4
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm4, %ymm5
+; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm3, %ymm7
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm8 = [0,2,5,7]
+; AVX512DQ-FCP-NEXT:    vpermt2q %ymm5, %ymm8, %ymm7
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14,0,2,4,6,8,10,12,14]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm3, %ymm6
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm1, %ymm5
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm5
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm6 = [0,2,9,11,4,6,13,15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm6, %zmm5
-; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm4, %ymm4
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm3, %ymm3
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm5
+; AVX512DQ-FCP-NEXT:    vpermt2q %ymm0, %ymm8, %ymm5
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm5, %zmm0
+; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm4, %ymm4
+; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15,1,3,5,7,9,11,13,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm3, %ymm3
+; AVX512DQ-FCP-NEXT:    vpermt2q %ymm4, %ymm8, %ymm3
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm2, %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT:    vpermt2q %ymm2, %ymm8, %ymm1
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm6, %zmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, (%rsi)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, (%rsi)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, (%rdx)
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll
index 7996d8d55aae8..77e7397e2fc96 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll
@@ -1678,7 +1678,7 @@ define void @store_i16_stride3_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm5
 ; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [21,21,0,22,22,0,23,23,0,0,0,0,1,1,0,2]
 ; AVX512-NEXT:    vpermi2d (%rdx), %zmm5, %zmm6
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm4))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm4))
 ; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm4
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm8 = [128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128,20,21,128,128]
 ; AVX512-NEXT:    vpshufb %ymm8, %ymm4, %ymm4
@@ -1754,7 +1754,7 @@ define void @store_i16_stride3_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm5
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [21,21,0,22,22,0,23,23,0,0,0,0,1,1,0,2]
 ; AVX512-FCP-NEXT:    vpermi2d (%rdx), %zmm5, %zmm6
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm4))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm4))
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm4
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128,20,21,128,128]
 ; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm4, %ymm4
@@ -1830,7 +1830,7 @@ define void @store_i16_stride3_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm5
 ; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [21,21,0,22,22,0,23,23,0,0,0,0,1,1,0,2]
 ; AVX512DQ-NEXT:    vpermi2d (%rdx), %zmm5, %zmm6
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm4))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm4))
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm4
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm8 = [128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128,20,21,128,128]
 ; AVX512DQ-NEXT:    vpshufb %ymm8, %ymm4, %ymm4
@@ -1906,7 +1906,7 @@ define void @store_i16_stride3_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %ymm5
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [21,21,0,22,22,0,23,23,0,0,0,0,1,1,0,2]
 ; AVX512DQ-FCP-NEXT:    vpermi2d (%rdx), %zmm5, %zmm6
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm4))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm4))
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm4
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128,20,21,128,128]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm4, %ymm4
@@ -3133,7 +3133,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm20 = [5,5,0,6,6,0,7,7,0,16,16,0,17,17,0,18]
 ; AVX512-NEXT:    vpermt2d %zmm8, %zmm20, %zmm0
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm23 & (zmm0 ^ zmm6))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm23 & (zmm0 ^ zmm6))
 ; AVX512-NEXT:    vmovdqa 96(%rdi), %ymm6
 ; AVX512-NEXT:    vpshufb %ymm5, %ymm6, %ymm6
 ; AVX512-NEXT:    vmovdqa 96(%rsi), %ymm9
@@ -3210,7 +3210,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm3, %ymm3
 ; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm3[0,1,2,3],zmm2[0,1,2,3]
 ; AVX512-NEXT:    vpermt2d %zmm4, %zmm20, %zmm1
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm23 & (zmm1 ^ zmm2))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm23 & (zmm1 ^ zmm2))
 ; AVX512-NEXT:    vmovdqa64 %zmm1, 64(%rcx)
 ; AVX512-NEXT:    vmovdqa64 %zmm9, 128(%rcx)
 ; AVX512-NEXT:    vmovdqa64 %zmm6, 192(%rcx)
@@ -3278,7 +3278,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm20 = [5,5,0,6,6,0,7,7,0,16,16,0,17,17,0,18]
 ; AVX512-FCP-NEXT:    vpermt2d %zmm8, %zmm20, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm23 & (zmm0 ^ zmm6))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm23 & (zmm0 ^ zmm6))
 ; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %ymm6
 ; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm6, %ymm6
 ; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %ymm9
@@ -3355,7 +3355,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm3, %ymm3
 ; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm3[0,1,2,3],zmm2[0,1,2,3]
 ; AVX512-FCP-NEXT:    vpermt2d %zmm4, %zmm20, %zmm1
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm23 & (zmm1 ^ zmm2))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm23 & (zmm1 ^ zmm2))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 64(%rcx)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm9, 128(%rcx)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, 192(%rcx)
@@ -3423,7 +3423,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm20 = [5,5,0,6,6,0,7,7,0,16,16,0,17,17,0,18]
 ; AVX512DQ-NEXT:    vpermt2d %zmm8, %zmm20, %zmm0
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm23 & (zmm0 ^ zmm6))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm23 & (zmm0 ^ zmm6))
 ; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %ymm6
 ; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm6, %ymm6
 ; AVX512DQ-NEXT:    vmovdqa 96(%rsi), %ymm9
@@ -3500,7 +3500,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm3, %ymm3
 ; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm3[0,1,2,3],zmm2[0,1,2,3]
 ; AVX512DQ-NEXT:    vpermt2d %zmm4, %zmm20, %zmm1
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm23 & (zmm1 ^ zmm2))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm23 & (zmm1 ^ zmm2))
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 64(%rcx)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm9, 128(%rcx)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm6, 192(%rcx)
@@ -3568,7 +3568,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm20 = [5,5,0,6,6,0,7,7,0,16,16,0,17,17,0,18]
 ; AVX512DQ-FCP-NEXT:    vpermt2d %zmm8, %zmm20, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm23 & (zmm0 ^ zmm6))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm23 & (zmm0 ^ zmm6))
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %ymm6
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm6, %ymm6
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %ymm9
@@ -3645,7 +3645,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm3, %ymm3
 ; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm3[0,1,2,3],zmm2[0,1,2,3]
 ; AVX512DQ-FCP-NEXT:    vpermt2d %zmm4, %zmm20, %zmm1
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm23 & (zmm1 ^ zmm2))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm23 & (zmm1 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 64(%rcx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm9, 128(%rcx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, 192(%rcx)
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-4.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-4.ll
index cec376e729205..2f6452467a420 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-4.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-4.ll
@@ -921,26 +921,26 @@ define void @store_i16_stride4_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vmovdqa (%rdx), %xmm2
 ; AVX512-NEXT:    vmovdqa 16(%rdx), %xmm3
 ; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; AVX512-NEXT:    vmovdqa (%rsi), %xmm5
+; AVX512-NEXT:    vmovdqa 16(%rsi), %xmm6
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm7
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm8
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [0,8,1,9,2,10,3,11]
+; AVX512-NEXT:    vpermt2d %ymm4, %ymm10, %ymm9
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512-NEXT:    vinserti32x4 $2, %xmm4, %zmm1, %zmm1
-; AVX512-NEXT:    vmovdqa (%rsi), %xmm3
-; AVX512-NEXT:    vmovdqa 16(%rsi), %xmm4
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm5
-; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm6
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
-; AVX512-NEXT:    vinserti32x4 $2, %xmm7, %zmm4, %zmm4
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [0,16,1,17,2,18,3,19,8,24,9,25,10,26,11,27]
-; AVX512-NEXT:    vpermt2d %zmm1, %zmm6, %zmm4
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]
+; AVX512-NEXT:    vpermt2d %ymm1, %ymm10, %ymm3
+; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm3, %zmm1
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]
+; AVX512-NEXT:    vpermt2d %ymm3, %ymm10, %ymm4
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; AVX512-NEXT:    vinserti32x4 $2, %xmm1, %zmm2, %zmm1
-; AVX512-NEXT:    vpermt2d %zmm0, %zmm6, %zmm1
-; AVX512-NEXT:    vmovdqa64 %zmm1, (%r8)
-; AVX512-NEXT:    vmovdqa64 %zmm4, 64(%r8)
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]
+; AVX512-NEXT:    vpermt2d %ymm0, %ymm10, %ymm2
+; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm0
+; AVX512-NEXT:    vmovdqa64 %zmm0, (%r8)
+; AVX512-NEXT:    vmovdqa64 %zmm1, 64(%r8)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
@@ -951,26 +951,26 @@ define void @store_i16_stride4_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm2
 ; AVX512-FCP-NEXT:    vmovdqa 16(%rdx), %xmm3
 ; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm5
+; AVX512-FCP-NEXT:    vmovdqa 16(%rsi), %xmm6
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm7
+; AVX512-FCP-NEXT:    vmovdqa 16(%rdi), %xmm8
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [0,8,1,9,2,10,3,11]
+; AVX512-FCP-NEXT:    vpermt2d %ymm4, %ymm10, %ymm9
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm4, %zmm1, %zmm1
-; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm3
-; AVX512-FCP-NEXT:    vmovdqa 16(%rsi), %xmm4
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm5
-; AVX512-FCP-NEXT:    vmovdqa 16(%rdi), %xmm6
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm7, %zmm4, %zmm4
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [0,16,1,17,2,18,3,19,8,24,9,25,10,26,11,27]
-; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm6, %zmm4
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]
+; AVX512-FCP-NEXT:    vpermt2d %ymm1, %ymm10, %ymm3
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm3, %zmm1
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]
+; AVX512-FCP-NEXT:    vpermt2d %ymm3, %ymm10, %ymm4
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm2, %zmm1
-; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm6, %zmm1
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, (%r8)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, 64(%r8)
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]
+; AVX512-FCP-NEXT:    vpermt2d %ymm0, %ymm10, %ymm2
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm0
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, (%r8)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 64(%r8)
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
@@ -981,26 +981,26 @@ define void @store_i16_stride4_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm2
 ; AVX512DQ-NEXT:    vmovdqa 16(%rdx), %xmm3
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm5
+; AVX512DQ-NEXT:    vmovdqa 16(%rsi), %xmm6
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm7
+; AVX512DQ-NEXT:    vmovdqa 16(%rdi), %xmm8
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [0,8,1,9,2,10,3,11]
+; AVX512DQ-NEXT:    vpermt2d %ymm4, %ymm10, %ymm9
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm4, %zmm1, %zmm1
-; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm3
-; AVX512DQ-NEXT:    vmovdqa 16(%rsi), %xmm4
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm5
-; AVX512DQ-NEXT:    vmovdqa 16(%rdi), %xmm6
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm7, %zmm4, %zmm4
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [0,16,1,17,2,18,3,19,8,24,9,25,10,26,11,27]
-; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm6, %zmm4
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]
+; AVX512DQ-NEXT:    vpermt2d %ymm1, %ymm10, %ymm3
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm3, %zmm1
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]
+; AVX512DQ-NEXT:    vpermt2d %ymm3, %ymm10, %ymm4
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm1, %zmm2, %zmm1
-; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm6, %zmm1
-; AVX512DQ-NEXT:    vmovdqa64 %zmm1, (%r8)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 64(%r8)
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]
+; AVX512DQ-NEXT:    vpermt2d %ymm0, %ymm10, %ymm2
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm0
+; AVX512DQ-NEXT:    vmovdqa64 %zmm0, (%r8)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 64(%r8)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
@@ -1011,26 +1011,26 @@ define void @store_i16_stride4_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rdx), %xmm3
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rsi), %xmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rdi), %xmm8
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [0,8,1,9,2,10,3,11]
+; AVX512DQ-FCP-NEXT:    vpermt2d %ymm4, %ymm10, %ymm9
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm4, %zmm1, %zmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rsi), %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm5
-; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rdi), %xmm6
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm7, %zmm4, %zmm4
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [0,16,1,17,2,18,3,19,8,24,9,25,10,26,11,27]
-; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm6, %zmm4
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]
+; AVX512DQ-FCP-NEXT:    vpermt2d %ymm1, %ymm10, %ymm3
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm3, %zmm1
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]
+; AVX512DQ-FCP-NEXT:    vpermt2d %ymm3, %ymm10, %ymm4
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm2, %zmm1
-; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm6, %zmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, (%r8)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 64(%r8)
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]
+; AVX512DQ-FCP-NEXT:    vpermt2d %ymm0, %ymm10, %ymm2
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, (%r8)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 64(%r8)
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll
index 4dfcf85d29589..fe567b90b3bcd 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll
@@ -935,7 +935,7 @@ define void @store_i16_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,1,1]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm6, %zmm4
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm5))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm5))
 ; AVX512-NEXT:    vpsrlq $48, %xmm3, %xmm3
 ; AVX512-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm3[1]
 ; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
@@ -974,7 +974,7 @@ define void @store_i16_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,1,1]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm6, %zmm4
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm5))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm5))
 ; AVX512-FCP-NEXT:    vpsrlq $48, %xmm1, %xmm1
 ; AVX512-FCP-NEXT:    vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1]
 ; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
@@ -1019,7 +1019,7 @@ define void @store_i16_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,1,1]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm6, %zmm4
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm5))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm5))
 ; AVX512DQ-NEXT:    vpsrlq $48, %xmm3, %xmm3
 ; AVX512DQ-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm3[1]
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
@@ -1058,7 +1058,7 @@ define void @store_i16_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,1,1]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm6, %zmm4
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm5))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vpsrlq $48, %xmm1, %xmm1
 ; AVX512DQ-FCP-NEXT:    vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1]
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
@@ -1751,7 +1751,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm4[0,1,1,1]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm6, %zmm6
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm5 = ymm1[u,u,u,u,u,u,u,u,14,15,u,u,u,u,u,u,u,u,16,17,u,u,u,u,u,u,u,u,18,19,u,u]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm0[1,1,2,2]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm5 = ymm8[0],ymm5[1],ymm8[2,3],ymm5[4],ymm8[5],ymm5[6],ymm8[7,8],ymm5[9],ymm8[10,11],ymm5[12],ymm8[13],ymm5[14],ymm8[15]
@@ -1770,9 +1770,9 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,3,2,3]
 ; AVX512-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm7 & (ymm10 ^ ymm5))
 ; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm8[0,1,2,3],zmm10[0,1,2,3]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm7 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpandnq 16(%r8){1to4}, %ymm7, %ymm7
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,ymm4[12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[16,17],zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vpbroadcastq 16(%r8), %ymm7
+; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm7
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm4, %zmm4
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm5 & mem)
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[3,2,3,3,7,6,7,7]
@@ -1826,7 +1826,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm4[0,1,1,1]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm6, %zmm6
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm1[u,u,u,u,u,u,u,u,14,15,u,u,u,u,u,u,u,u,16,17,u,u,u,u,u,u,u,u,18,19,u,u]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm0[1,1,2,2]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm8[0],ymm5[1],ymm8[2,3],ymm5[4],ymm8[5],ymm5[6],ymm8[7,8],ymm5[9],ymm8[10,11],ymm5[12],ymm8[13],ymm5[14],ymm8[15]
@@ -1845,9 +1845,9 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,3,2,3]
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm7 & (ymm10 ^ ymm5))
 ; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm8[0,1,2,3],zmm10[0,1,2,3]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpandnq 16(%r8){1to4}, %ymm7, %ymm7
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,ymm4[12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[16,17],zero,zero,zero,zero,zero,zero
+; AVX512-FCP-NEXT:    vpbroadcastq 16(%r8), %ymm7
+; AVX512-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm7
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm4, %zmm4
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm5 & mem)
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[3,2,3,3,7,6,7,7]
@@ -1900,7 +1900,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm4[0,1,1,1]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm6, %zmm6
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm5 = ymm1[u,u,u,u,u,u,u,u,14,15,u,u,u,u,u,u,u,u,16,17,u,u,u,u,u,u,u,u,18,19,u,u]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm0[1,1,2,2]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm5 = ymm8[0],ymm5[1],ymm8[2,3],ymm5[4],ymm8[5],ymm5[6],ymm8[7,8],ymm5[9],ymm8[10,11],ymm5[12],ymm8[13],ymm5[14],ymm8[15]
@@ -1919,9 +1919,9 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,3,2,3]
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm7 & (ymm10 ^ ymm5))
 ; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm8[0,1,2,3],zmm10[0,1,2,3]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm7 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpandnq 16(%r8){1to4}, %ymm7, %ymm7
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,ymm4[12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[16,17],zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT:    vpbroadcastq 16(%r8), %ymm7
+; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm7
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm4, %zmm4
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm5 & mem)
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[3,2,3,3,7,6,7,7]
@@ -1975,7 +1975,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm4[0,1,1,1]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm6, %zmm6
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm1[u,u,u,u,u,u,u,u,14,15,u,u,u,u,u,u,u,u,16,17,u,u,u,u,u,u,u,u,18,19,u,u]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm0[1,1,2,2]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm8[0],ymm5[1],ymm8[2,3],ymm5[4],ymm8[5],ymm5[6],ymm8[7,8],ymm5[9],ymm8[10,11],ymm5[12],ymm8[13],ymm5[14],ymm8[15]
@@ -1994,9 +1994,9 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,3,2,3]
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm5 ^ (ymm7 & (ymm10 ^ ymm5))
 ; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm8[0,1,2,3],zmm10[0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpandnq 16(%r8){1to4}, %ymm7, %ymm7
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,ymm4[12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm4[16,17],zero,zero,zero,zero,zero,zero
+; AVX512DQ-FCP-NEXT:    vpbroadcastq 16(%r8), %ymm7
+; AVX512DQ-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm7
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm4, %zmm4
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm5 & mem)
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[3,2,3,3,7,6,7,7]
@@ -3349,7 +3349,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm13 = [4,5,2,3,2,3,0,1,10,11,8,9,4,5,6,7]
 ; AVX512-NEXT:    vpshufb %xmm13, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa64 (%rdx), %ymm17
-; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm6
+; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm5
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm17[3,2,3,3,7,6,7,7]
 ; AVX512-NEXT:    vmovdqa (%rcx), %ymm1
 ; AVX512-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm1[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
@@ -3366,8 +3366,8 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,5,6]
 ; AVX512-NEXT:    vmovdqa64 (%rdi), %ymm20
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm20[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vmovdqa (%rsi), %ymm5
-; AVX512-NEXT:    vpshufhw {{.*#+}} ymm7 = ymm5[0,1,2,3,7,6,5,7,8,9,10,11,15,14,13,15]
+; AVX512-NEXT:    vmovdqa (%rsi), %ymm6
+; AVX512-NEXT:    vpshufhw {{.*#+}} ymm7 = ymm6[0,1,2,3,7,6,5,7,8,9,10,11,15,14,13,15]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm7 = ymm7[2,3,2,2,6,7,6,6]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm7[0],ymm3[1],ymm7[2],ymm3[3],ymm7[4,5],ymm3[6],ymm7[7,8],ymm3[9],ymm7[10],ymm3[11],ymm7[12,13],ymm3[14],ymm7[15]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,3,2,2]
@@ -3377,7 +3377,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpbroadcastq 24(%r8), %ymm0
 ; AVX512-NEXT:    vpbroadcastq 32(%r8), %ymm2
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm19
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm19 = zmm19 ^ (mem & (zmm19 ^ zmm3))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm19 = zmm19 ^ (mem & (zmm19 ^ zmm3))
 ; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm3
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm2 = [u,u,0,1,u,u,u,u,14,15,u,u,2,3,u,u,u,u,16,17,u,u,u,u,30,31,u,u,18,19,u,u]
 ; AVX512-NEXT:    vpshufb %ymm2, %ymm3, %ymm0
@@ -3395,7 +3395,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [u,u,12,13,u,u,0,1,u,u,u,u,14,15,u,u,u,u,28,29,u,u,16,17,u,u,u,u,30,31,u,u]
 ; AVX512-NEXT:    vpshufb %ymm4, %ymm0, %ymm2
 ; AVX512-NEXT:    vmovdqa64 %ymm4, %ymm23
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm8 = ymm6[3,0,3,0,7,4,7,4]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm8 = ymm5[3,0,3,0,7,4,7,4]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm8[0],ymm2[1],ymm8[2],ymm2[3],ymm8[4,5],ymm2[6],ymm8[7,8],ymm2[9],ymm8[10],ymm2[11],ymm8[12,13],ymm2[14],ymm8[15]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm8 = [10,11,u,u,6,7,u,u,8,9,8,9,u,u,8,9]
 ; AVX512-NEXT:    vpshufb %xmm8, %xmm10, %xmm10
@@ -3404,15 +3404,15 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,0]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm9, %zmm2
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm11))
-; AVX512-NEXT:    vmovdqa (%r8), %ymm9
-; AVX512-NEXT:    vmovdqa 32(%r8), %ymm10
+; AVX512-NEXT:    vmovdqa (%r8), %ymm10
+; AVX512-NEXT:    vmovdqa 32(%r8), %ymm9
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm11 = [128,128,128,128,12,13,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128]
-; AVX512-NEXT:    vpshufb %ymm11, %ymm10, %ymm4
+; AVX512-NEXT:    vpshufb %ymm11, %ymm9, %ymm4
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,1,1,1]
-; AVX512-NEXT:    vpandnq %ymm10, %ymm21, %ymm10
-; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm10, %zmm10
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 | (zmm2 & zmm21)
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,1,1]
+; AVX512-NEXT:    vpandnq %ymm9, %ymm21, %ymm9
+; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm9, %zmm9
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 | (zmm2 & zmm21)
 ; AVX512-NEXT:    vmovdqa (%rdx), %xmm2
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm12[0],xmm2[0],xmm12[1],xmm2[1],xmm12[2],xmm2[2],xmm12[3],xmm2[3]
 ; AVX512-NEXT:    vpshufb %xmm13, %xmm4, %xmm4
@@ -3433,9 +3433,9 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535]
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm2 ^ (zmm7 & (zmm4 ^ zmm2))
 ; AVX512-NEXT:    vpbroadcastq (%r8), %ymm2
-; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm9[0,1,1,1]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm10[0,1,1,1]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm2, %zmm2
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm4))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm4))
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm18[0,1,2,1,4,5,6,5]
 ; AVX512-NEXT:    vprolq $16, %ymm3, %ymm8
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm4 = ymm8[0,1],ymm4[2],ymm8[3],ymm4[4],ymm8[5,6],ymm4[7],ymm8[8,9],ymm4[10],ymm8[11],ymm4[12],ymm8[13,14],ymm4[15]
@@ -3449,48 +3449,48 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [18,19,0,0,22,23,22,23,0,0,20,21,0,0,24,25,18,19,0,0,22,23,22,23,0,0,20,21,0,0,24,25]
 ; AVX512-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX512-NEXT:    vpshufb %ymm3, %ymm0, %ymm8
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm12 = ymm6[1,1,1,2,5,5,5,6]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm12 = ymm5[1,1,1,2,5,5,5,6]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm8 = ymm8[0],ymm12[1],ymm8[2,3],ymm12[4],ymm8[5],ymm12[6],ymm8[7,8],ymm12[9],ymm8[10,11],ymm12[12],ymm8[13],ymm12[14],ymm8[15]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,3,2,3]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[3,2,3,3,7,6,7,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[3,2,3,3,7,6,7,7]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm6[1],ymm0[2],ymm6[3,4],ymm0[5,6,7,8],ymm6[9],ymm0[10],ymm6[11,12],ymm0[13,14,15]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm5[1],ymm0[2],ymm5[3,4],ymm0[5,6,7,8],ymm5[9],ymm0[10],ymm5[11,12],ymm0[13,14,15]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,3,2]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm8, %zmm0
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm4 ^ (zmm7 & (zmm0 ^ zmm4))
 ; AVX512-NEXT:    vpbroadcastq 48(%r8), %ymm4
-; AVX512-NEXT:    vpbroadcastq 56(%r8), %ymm6
-; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm4
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm6 & (zmm4 ^ zmm0))
+; AVX512-NEXT:    vpbroadcastq 56(%r8), %ymm5
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm0))
 ; AVX512-NEXT:    vmovdqa64 %ymm22, %ymm0
-; AVX512-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm20[1,1,2,2]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm7[0],ymm0[1],ymm7[2,3],ymm0[4],ymm7[5],ymm0[6],ymm7[7,8],ymm0[9],ymm7[10,11],ymm0[12],ymm7[13],ymm0[14],ymm7[15]
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm7 = ymm20[0,1,2,1,4,5,6,5]
-; AVX512-NEXT:    vprolq $16, %ymm5, %ymm5
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0,1],ymm7[2],ymm5[3],ymm7[4],ymm5[5,6],ymm7[7],ymm5[8,9],ymm7[10],ymm5[11],ymm7[12],ymm5[13,14],ymm7[15]
+; AVX512-NEXT:    vpshufb %ymm0, %ymm6, %ymm0
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm20[1,1,2,2]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1],ymm5[2,3],ymm0[4],ymm5[5],ymm0[6],ymm5[7,8],ymm0[9],ymm5[10,11],ymm0[12],ymm5[13],ymm0[14],ymm5[15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm20[0,1,2,1,4,5,6,5]
+; AVX512-NEXT:    vprolq $16, %ymm6, %ymm6
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm5 = ymm6[0,1],ymm5[2],ymm6[3],ymm5[4],ymm6[5,6],ymm5[7],ymm6[8,9],ymm5[10],ymm6[11],ymm5[12],ymm6[13,14],ymm5[15]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,3,2,3]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm0
 ; AVX512-NEXT:    vmovdqa64 %ymm23, %ymm5
 ; AVX512-NEXT:    vpshufb %ymm5, %ymm1, %ymm5
-; AVX512-NEXT:    vpshufd {{.*#+}} ymm7 = ymm17[3,0,3,0,7,4,7,4]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm5 = ymm7[0],ymm5[1],ymm7[2],ymm5[3],ymm7[4,5],ymm5[6],ymm7[7,8],ymm5[9],ymm7[10],ymm5[11],ymm7[12,13],ymm5[14],ymm7[15]
+; AVX512-NEXT:    vpshufd {{.*#+}} ymm6 = ymm17[3,0,3,0,7,4,7,4]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4,5],ymm5[6],ymm6[7,8],ymm5[9],ymm6[10],ymm5[11],ymm6[12,13],ymm5[14],ymm6[15]
 ; AVX512-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm17[1,1,1,2,5,5,5,6]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5],ymm3[6],ymm1[7,8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13],ymm3[14],ymm1[15]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm5, %zmm1
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm0 ^ (zmm16 & (zmm1 ^ zmm0))
-; AVX512-NEXT:    vpandnq 16(%r8){1to4}, %ymm6, %ymm0
-; AVX512-NEXT:    vpshufb %ymm11, %ymm9, %ymm3
-; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm3, %zmm0
+; AVX512-NEXT:    vpshufb %ymm11, %ymm10, %ymm0
+; AVX512-NEXT:    vpbroadcastq 16(%r8), %ymm3
+; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
+; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm1 & mem)
 ; AVX512-NEXT:    vmovdqa64 %zmm0, 64(%r9)
 ; AVX512-NEXT:    vmovdqa64 %zmm4, 256(%r9)
 ; AVX512-NEXT:    vmovdqa64 %zmm2, (%r9)
-; AVX512-NEXT:    vmovdqa64 %zmm10, 192(%r9)
+; AVX512-NEXT:    vmovdqa64 %zmm9, 192(%r9)
 ; AVX512-NEXT:    vmovdqa64 %zmm19, 128(%r9)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -3500,7 +3500,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %ymm5
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [u,u,0,1,u,u,u,u,14,15,u,u,2,3,u,u,u,u,16,17,u,u,u,u,30,31,u,u,18,19,u,u]
 ; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm5, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm1, %ymm19
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm1, %ymm23
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm7
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm7[1,1,2,2]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
@@ -3508,7 +3508,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm2
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm12 = [6,7,u,u,10,11,6,7,u,u,8,9,u,u,12,13]
 ; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm2, %xmm1
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm20
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm19
 ; AVX512-FCP-NEXT:    vpbroadcastq 40(%rdi), %xmm2
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2,3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]
@@ -3549,18 +3549,18 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm15[0],xmm1[1],xmm15[1],xmm1[2],xmm15[2],xmm1[3],xmm15[3]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,2,3,2,3,0,1,10,11,8,9,4,5,6,7]
 ; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm21
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm20
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm14 = [0,1,0,1,8,9,8,8]
 ; AVX512-FCP-NEXT:    vpermi2q %zmm3, %zmm1, %zmm14
 ; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm10, %xmm1
 ; AVX512-FCP-NEXT:    vpbroadcastq 8(%rdi), %xmm3
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2,3],xmm3[4],xmm1[5],xmm3[6],xmm1[7]
 ; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm3
-; AVX512-FCP-NEXT:    vmovdqa64 32(%rdi), %xmm22
+; AVX512-FCP-NEXT:    vmovdqa64 32(%rdi), %xmm21
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,3,8,9,10,11,4,5,4,5,6,7,12,13]
 ; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm3, %xmm3
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm23
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm22
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[0,1,0,1,4,5,4,5]
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535]
@@ -3568,9 +3568,9 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpbroadcastq (%r8), %ymm3
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm0[0,1,1,1]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm3, %zmm10
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm1))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm1))
 ; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm15
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm19, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm23, %ymm1
 ; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm15, %ymm1
 ; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm14
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm14[1,1,2,2]
@@ -3591,61 +3591,61 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm12[1],ymm2[2,3],ymm12[4],ymm2[5],ymm12[6],ymm2[7,8],ymm12[9],ymm2[10,11],ymm12[12],ymm2[13],ymm12[14],ymm2[15]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,3,2,3]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm13, %zmm2
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm3 ^ (zmm19 & (zmm2 ^ zmm3))
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
-; AVX512-FCP-NEXT:    vpandnq 16(%r8){1to4}, %ymm3, %ymm13
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm25, %ymm12
-; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm0, %ymm0
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm0, %zmm0
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm3 ^ (zmm12 & (zmm2 ^ zmm3))
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm25, %ymm3
+; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
+; AVX512-FCP-NEXT:    vpbroadcastq 16(%r8), %ymm3
+; AVX512-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & mem)
 ; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [30,31,28,29,26,27,30,31,30,31,28,29,30,31,28,29,30,31,28,29,26,27,30,31,30,31,28,29,30,31,28,29]
 ; AVX512-FCP-NEXT:    # ymm13 = mem[0,1,0,1]
 ; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm5, %ymm2
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm12 = ymm7[2,3,2,3,6,7,6,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm12[1],ymm2[2],ymm12[3],ymm2[4,5],ymm12[6],ymm2[7,8],ymm12[9],ymm2[10],ymm12[11],ymm2[12,13],ymm12[14],ymm2[15]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm7 = ymm7[0,1,2,1,4,5,6,5]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm7[2,3,2,3,6,7,6,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7,8],ymm3[9],ymm2[10],ymm3[11],ymm2[12,13],ymm3[14],ymm2[15]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm7[0,1,2,1,4,5,6,5]
 ; AVX512-FCP-NEXT:    vprolq $16, %ymm5, %ymm5
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0,1],ymm7[2],ymm5[3],ymm7[4],ymm5[5,6],ymm7[7],ymm5[8,9],ymm7[10],ymm5[11],ymm7[12],ymm5[13,14],ymm7[15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm7 = [2,3,2,3,10,11,10,10]
-; AVX512-FCP-NEXT:    vpermi2q %zmm2, %zmm5, %zmm7
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0,1],ymm3[2],ymm5[3],ymm3[4],ymm5[5,6],ymm3[7],ymm5[8,9],ymm3[10],ymm5[11],ymm3[12],ymm5[13,14],ymm3[15]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [2,3,2,3,10,11,10,10]
+; AVX512-FCP-NEXT:    vpermi2q %zmm2, %zmm3, %zmm5
 ; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [26,27,28,29,30,31,30,31,26,27,28,29,30,31,30,31,26,27,28,29,30,31,30,31,26,27,28,29,30,31,30,31]
-; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm8, %ymm5
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm12 = ymm9[3,2,3,3,7,6,7,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2],ymm12[3,4],ymm5[5,6,7,8],ymm12[9],ymm5[10],ymm12[11,12],ymm5[13,14,15]
+; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm8, %ymm3
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm7 = ymm9[3,2,3,3,7,6,7,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm7[1],ymm3[2],ymm7[3,4],ymm3[5,6,7,8],ymm7[9],ymm3[10],ymm7[11,12],ymm3[13,14,15]
 ; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm8, %ymm1
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm8 = ymm9[1,1,1,2,5,5,5,6]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm8[1],ymm1[2,3],ymm8[4],ymm1[5],ymm8[6],ymm1[7,8],ymm8[9],ymm1[10,11],ymm8[12],ymm1[13],ymm8[14],ymm1[15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm8 = [2,3,2,3,10,10,11,10]
-; AVX512-FCP-NEXT:    vpermi2q %zmm5, %zmm1, %zmm8
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm7 ^ (zmm16 & (zmm8 ^ zmm7))
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm7 = ymm9[1,1,1,2,5,5,5,6]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm7[1],ymm1[2,3],ymm7[4],ymm1[5],ymm7[6],ymm1[7,8],ymm7[9],ymm1[10,11],ymm7[12],ymm1[13],ymm7[14],ymm1[15]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm7 = [2,3,2,3,10,10,11,10]
+; AVX512-FCP-NEXT:    vpermi2q %zmm3, %zmm1, %zmm7
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm5 ^ (zmm16 & (zmm7 ^ zmm5))
 ; AVX512-FCP-NEXT:    vpbroadcastq 48(%r8), %ymm1
-; AVX512-FCP-NEXT:    vpbroadcastq 56(%r8), %ymm5
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm1, %zmm1
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm3 & (zmm1 ^ zmm8))
+; AVX512-FCP-NEXT:    vpbroadcastq 56(%r8), %ymm3
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm7))
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm11[0],xmm6[0],xmm11[1],xmm6[1],xmm11[2],xmm6[2],xmm11[3],xmm6[3]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm21, %xmm5
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm20, %xmm5
 ; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm3, %xmm3
 ; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm4, %ymm2
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm17[3,2,3,3,7,6,7,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm4[1],ymm2[2],ymm4[3,4],ymm2[5,6,7,8],ymm4[9],ymm2[10],ymm4[11,12],ymm2[13,14,15]
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm4 = [2,2,3,2,8,9,8,9]
 ; AVX512-FCP-NEXT:    vpermi2q %zmm3, %zmm2, %zmm4
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm20, %xmm2
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm22, %xmm3
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm19, %xmm2
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm21, %xmm3
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm23, %xmm3
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm22, %xmm3
 ; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
 ; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm15, %ymm3
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm14[2,3,2,3,6,7,6,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4,5],ymm5[6],ymm3[7,8],ymm5[9],ymm3[10],ymm5[11],ymm3[12,13],ymm5[14],ymm3[15]
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [2,3,2,2,8,9,8,9]
 ; AVX512-FCP-NEXT:    vpermi2q %zmm2, %zmm3, %zmm5
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm4 ^ (zmm19 & (zmm5 ^ zmm4))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm4 ^ (zmm12 & (zmm5 ^ zmm4))
 ; AVX512-FCP-NEXT:    vpbroadcastq 24(%r8), %ymm2
 ; AVX512-FCP-NEXT:    vpbroadcastq 32(%r8), %ymm3
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm5))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm5))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, 128(%r9)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 256(%r9)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 64(%r9)
@@ -3663,7 +3663,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm13 = [4,5,2,3,2,3,0,1,10,11,8,9,4,5,6,7]
 ; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vmovdqa64 (%rdx), %ymm17
-; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm6
+; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm5
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm17[3,2,3,3,7,6,7,7]
 ; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm1
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm1[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
@@ -3680,8 +3680,8 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,5,6]
 ; AVX512DQ-NEXT:    vmovdqa64 (%rdi), %ymm20
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm3 = ymm20[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm5
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm7 = ymm5[0,1,2,3,7,6,5,7,8,9,10,11,15,14,13,15]
+; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm6
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm7 = ymm6[0,1,2,3,7,6,5,7,8,9,10,11,15,14,13,15]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm7 = ymm7[2,3,2,2,6,7,6,6]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm7[0],ymm3[1],ymm7[2],ymm3[3],ymm7[4,5],ymm3[6],ymm7[7,8],ymm3[9],ymm7[10],ymm3[11],ymm7[12,13],ymm3[14],ymm7[15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,3,2,2]
@@ -3691,7 +3691,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpbroadcastq 24(%r8), %ymm0
 ; AVX512DQ-NEXT:    vpbroadcastq 32(%r8), %ymm2
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm19
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm19 = zmm19 ^ (mem & (zmm19 ^ zmm3))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm19 = zmm19 ^ (mem & (zmm19 ^ zmm3))
 ; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm3
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [u,u,0,1,u,u,u,u,14,15,u,u,2,3,u,u,u,u,16,17,u,u,u,u,30,31,u,u,18,19,u,u]
 ; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm3, %ymm0
@@ -3709,7 +3709,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [u,u,12,13,u,u,0,1,u,u,u,u,14,15,u,u,u,u,28,29,u,u,16,17,u,u,u,u,30,31,u,u]
 ; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm0, %ymm2
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm4, %ymm23
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm8 = ymm6[3,0,3,0,7,4,7,4]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm8 = ymm5[3,0,3,0,7,4,7,4]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm8[0],ymm2[1],ymm8[2],ymm2[3],ymm8[4,5],ymm2[6],ymm8[7,8],ymm2[9],ymm8[10],ymm2[11],ymm8[12,13],ymm2[14],ymm8[15]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm8 = [10,11,u,u,6,7,u,u,8,9,8,9,u,u,8,9]
 ; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm10, %xmm10
@@ -3718,15 +3718,15 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,0]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm9, %zmm2
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm11))
-; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm9
-; AVX512DQ-NEXT:    vmovdqa 32(%r8), %ymm10
+; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm10
+; AVX512DQ-NEXT:    vmovdqa 32(%r8), %ymm9
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm11 = [128,128,128,128,12,13,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128]
-; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm10, %ymm4
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm9, %ymm4
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,1,1,1]
-; AVX512DQ-NEXT:    vpandnq %ymm10, %ymm21, %ymm10
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm10, %zmm10
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 | (zmm2 & zmm21)
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,1,1]
+; AVX512DQ-NEXT:    vpandnq %ymm9, %ymm21, %ymm9
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm9, %zmm9
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 | (zmm2 & zmm21)
 ; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm2
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm12[0],xmm2[0],xmm12[1],xmm2[1],xmm12[2],xmm2[2],xmm12[3],xmm2[3]
 ; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm4, %xmm4
@@ -3747,9 +3747,9 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535]
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm2 ^ (zmm7 & (zmm4 ^ zmm2))
 ; AVX512DQ-NEXT:    vpbroadcastq (%r8), %ymm2
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm9[0,1,1,1]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm10[0,1,1,1]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm2, %zmm2
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm4))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm4))
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm4 = ymm18[0,1,2,1,4,5,6,5]
 ; AVX512DQ-NEXT:    vprolq $16, %ymm3, %ymm8
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm4 = ymm8[0,1],ymm4[2],ymm8[3],ymm4[4],ymm8[5,6],ymm4[7],ymm8[8,9],ymm4[10],ymm8[11],ymm4[12],ymm8[13,14],ymm4[15]
@@ -3763,48 +3763,48 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [18,19,0,0,22,23,22,23,0,0,20,21,0,0,24,25,18,19,0,0,22,23,22,23,0,0,20,21,0,0,24,25]
 ; AVX512DQ-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm0, %ymm8
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm12 = ymm6[1,1,1,2,5,5,5,6]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm12 = ymm5[1,1,1,2,5,5,5,6]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm8 = ymm8[0],ymm12[1],ymm8[2,3],ymm12[4],ymm8[5],ymm12[6],ymm8[7,8],ymm12[9],ymm8[10,11],ymm12[12],ymm8[13],ymm12[14],ymm8[15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,3,2,3]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[3,2,3,3,7,6,7,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[3,2,3,3,7,6,7,7]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm6[1],ymm0[2],ymm6[3,4],ymm0[5,6,7,8],ymm6[9],ymm0[10],ymm6[11,12],ymm0[13,14,15]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm5[1],ymm0[2],ymm5[3,4],ymm0[5,6,7,8],ymm5[9],ymm0[10],ymm5[11,12],ymm0[13,14,15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,3,2]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm8, %zmm0
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm4 ^ (zmm7 & (zmm0 ^ zmm4))
 ; AVX512DQ-NEXT:    vpbroadcastq 48(%r8), %ymm4
-; AVX512DQ-NEXT:    vpbroadcastq 56(%r8), %ymm6
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm4
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm6 & (zmm4 ^ zmm0))
+; AVX512DQ-NEXT:    vpbroadcastq 56(%r8), %ymm5
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm0))
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm22, %ymm0
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm20[1,1,2,2]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm7[0],ymm0[1],ymm7[2,3],ymm0[4],ymm7[5],ymm0[6],ymm7[7,8],ymm0[9],ymm7[10,11],ymm0[12],ymm7[13],ymm0[14],ymm7[15]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm7 = ymm20[0,1,2,1,4,5,6,5]
-; AVX512DQ-NEXT:    vprolq $16, %ymm5, %ymm5
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0,1],ymm7[2],ymm5[3],ymm7[4],ymm5[5,6],ymm7[7],ymm5[8,9],ymm7[10],ymm5[11],ymm7[12],ymm5[13,14],ymm7[15]
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm6, %ymm0
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm20[1,1,2,2]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1],ymm5[2,3],ymm0[4],ymm5[5],ymm0[6],ymm5[7,8],ymm0[9],ymm5[10,11],ymm0[12],ymm5[13],ymm0[14],ymm5[15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm20[0,1,2,1,4,5,6,5]
+; AVX512DQ-NEXT:    vprolq $16, %ymm6, %ymm6
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm5 = ymm6[0,1],ymm5[2],ymm6[3],ymm5[4],ymm6[5,6],ymm5[7],ymm6[8,9],ymm5[10],ymm6[11],ymm5[12],ymm6[13,14],ymm5[15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,3,2,3]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm23, %ymm5
 ; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm1, %ymm5
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm7 = ymm17[3,0,3,0,7,4,7,4]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm5 = ymm7[0],ymm5[1],ymm7[2],ymm5[3],ymm7[4,5],ymm5[6],ymm7[7,8],ymm5[9],ymm7[10],ymm5[11],ymm7[12,13],ymm5[14],ymm7[15]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm6 = ymm17[3,0,3,0,7,4,7,4]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4,5],ymm5[6],ymm6[7,8],ymm5[9],ymm6[10],ymm5[11],ymm6[12,13],ymm5[14],ymm6[15]
 ; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm3 = ymm17[1,1,1,2,5,5,5,6]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5],ymm3[6],ymm1[7,8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13],ymm3[14],ymm1[15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm5, %zmm1
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm0 ^ (zmm16 & (zmm1 ^ zmm0))
-; AVX512DQ-NEXT:    vpandnq 16(%r8){1to4}, %ymm6, %ymm0
-; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm9, %ymm3
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm3, %zmm0
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm10, %ymm0
+; AVX512DQ-NEXT:    vpbroadcastq 16(%r8), %ymm3
+; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm1 & mem)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 64(%r9)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 256(%r9)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm2, (%r9)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm10, 192(%r9)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm9, 192(%r9)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm19, 128(%r9)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
@@ -3814,7 +3814,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %ymm5
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [u,u,0,1,u,u,u,u,14,15,u,u,2,3,u,u,u,u,16,17,u,u,u,u,30,31,u,u,18,19,u,u]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm5, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm1, %ymm19
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm1, %ymm23
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm7
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm7[1,1,2,2]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
@@ -3822,7 +3822,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm12 = [6,7,u,u,10,11,6,7,u,u,8,9,u,u,12,13]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm2, %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm20
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm19
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 40(%rdi), %xmm2
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2,3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]
@@ -3863,18 +3863,18 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm15[0],xmm1[1],xmm15[1],xmm1[2],xmm15[2],xmm1[3],xmm15[3]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,2,3,2,3,0,1,10,11,8,9,4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm21
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm20
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm14 = [0,1,0,1,8,9,8,8]
 ; AVX512DQ-FCP-NEXT:    vpermi2q %zmm3, %zmm1, %zmm14
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm10, %xmm1
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 8(%rdi), %xmm3
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2,3],xmm3[4],xmm1[5],xmm3[6],xmm1[7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 32(%rdi), %xmm22
+; AVX512DQ-FCP-NEXT:    vmovdqa64 32(%rdi), %xmm21
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,3,8,9,10,11,4,5,4,5,6,7,12,13]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm23
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm22
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[0,1,0,1,4,5,4,5]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535]
@@ -3882,9 +3882,9 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq (%r8), %ymm3
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm0[0,1,1,1]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm3, %zmm10
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm1))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm15
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm19, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm23, %ymm1
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm15, %ymm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm14
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm14[1,1,2,2]
@@ -3905,61 +3905,61 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm12[1],ymm2[2,3],ymm12[4],ymm2[5],ymm12[6],ymm2[7,8],ymm12[9],ymm2[10,11],ymm12[12],ymm2[13],ymm12[14],ymm2[15]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,3,2,3]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm13, %zmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm3 ^ (zmm19 & (zmm2 ^ zmm3))
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0]
-; AVX512DQ-FCP-NEXT:    vpandnq 16(%r8){1to4}, %ymm3, %ymm13
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm25, %ymm12
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm0, %ymm0
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm0, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm3 ^ (zmm12 & (zmm2 ^ zmm3))
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm25, %ymm3
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
+; AVX512DQ-FCP-NEXT:    vpbroadcastq 16(%r8), %ymm3
+; AVX512DQ-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & mem)
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [30,31,28,29,26,27,30,31,30,31,28,29,30,31,28,29,30,31,28,29,26,27,30,31,30,31,28,29,30,31,28,29]
 ; AVX512DQ-FCP-NEXT:    # ymm13 = mem[0,1,0,1]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm5, %ymm2
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm12 = ymm7[2,3,2,3,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm12[1],ymm2[2],ymm12[3],ymm2[4,5],ymm12[6],ymm2[7,8],ymm12[9],ymm2[10],ymm12[11],ymm2[12,13],ymm12[14],ymm2[15]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm7 = ymm7[0,1,2,1,4,5,6,5]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm7[2,3,2,3,6,7,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7,8],ymm3[9],ymm2[10],ymm3[11],ymm2[12,13],ymm3[14],ymm2[15]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm7[0,1,2,1,4,5,6,5]
 ; AVX512DQ-FCP-NEXT:    vprolq $16, %ymm5, %ymm5
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0,1],ymm7[2],ymm5[3],ymm7[4],ymm5[5,6],ymm7[7],ymm5[8,9],ymm7[10],ymm5[11],ymm7[12],ymm5[13,14],ymm7[15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm7 = [2,3,2,3,10,11,10,10]
-; AVX512DQ-FCP-NEXT:    vpermi2q %zmm2, %zmm5, %zmm7
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0,1],ymm3[2],ymm5[3],ymm3[4],ymm5[5,6],ymm3[7],ymm5[8,9],ymm3[10],ymm5[11],ymm3[12],ymm5[13,14],ymm3[15]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [2,3,2,3,10,11,10,10]
+; AVX512DQ-FCP-NEXT:    vpermi2q %zmm2, %zmm3, %zmm5
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [26,27,28,29,30,31,30,31,26,27,28,29,30,31,30,31,26,27,28,29,30,31,30,31,26,27,28,29,30,31,30,31]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm8, %ymm5
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm12 = ymm9[3,2,3,3,7,6,7,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2],ymm12[3,4],ymm5[5,6,7,8],ymm12[9],ymm5[10],ymm12[11,12],ymm5[13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm8, %ymm3
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm7 = ymm9[3,2,3,3,7,6,7,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm7[1],ymm3[2],ymm7[3,4],ymm3[5,6,7,8],ymm7[9],ymm3[10],ymm7[11,12],ymm3[13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm8, %ymm1
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm8 = ymm9[1,1,1,2,5,5,5,6]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm8[1],ymm1[2,3],ymm8[4],ymm1[5],ymm8[6],ymm1[7,8],ymm8[9],ymm1[10,11],ymm8[12],ymm1[13],ymm8[14],ymm1[15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm8 = [2,3,2,3,10,10,11,10]
-; AVX512DQ-FCP-NEXT:    vpermi2q %zmm5, %zmm1, %zmm8
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm7 ^ (zmm16 & (zmm8 ^ zmm7))
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm7 = ymm9[1,1,1,2,5,5,5,6]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm7[1],ymm1[2,3],ymm7[4],ymm1[5],ymm7[6],ymm1[7,8],ymm7[9],ymm1[10,11],ymm7[12],ymm1[13],ymm7[14],ymm1[15]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm7 = [2,3,2,3,10,10,11,10]
+; AVX512DQ-FCP-NEXT:    vpermi2q %zmm3, %zmm1, %zmm7
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm5 ^ (zmm16 & (zmm7 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 48(%r8), %ymm1
-; AVX512DQ-FCP-NEXT:    vpbroadcastq 56(%r8), %ymm5
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm1, %zmm1
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm3 & (zmm1 ^ zmm8))
+; AVX512DQ-FCP-NEXT:    vpbroadcastq 56(%r8), %ymm3
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm7))
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm11[0],xmm6[0],xmm11[1],xmm6[1],xmm11[2],xmm6[2],xmm11[3],xmm6[3]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm21, %xmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm20, %xmm5
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm3, %xmm3
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm4, %ymm2
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm17[3,2,3,3,7,6,7,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm4[1],ymm2[2],ymm4[3,4],ymm2[5,6,7,8],ymm4[9],ymm2[10],ymm4[11,12],ymm2[13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm4 = [2,2,3,2,8,9,8,9]
 ; AVX512DQ-FCP-NEXT:    vpermi2q %zmm3, %zmm2, %zmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm20, %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm22, %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm19, %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm21, %xmm3
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm23, %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm22, %xmm3
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm15, %ymm3
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm14[2,3,2,3,6,7,6,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4,5],ymm5[6],ymm3[7,8],ymm5[9],ymm3[10],ymm5[11],ymm3[12,13],ymm5[14],ymm3[15]
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [2,3,2,2,8,9,8,9]
 ; AVX512DQ-FCP-NEXT:    vpermi2q %zmm2, %zmm3, %zmm5
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm4 ^ (zmm19 & (zmm5 ^ zmm4))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm4 ^ (zmm12 & (zmm5 ^ zmm4))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 24(%r8), %ymm2
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 32(%r8), %ymm3
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm5))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, 128(%r9)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 256(%r9)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 64(%r9)
@@ -6978,12 +6978,12 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpbroadcastq 96(%r8), %ymm2
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm18 & (zmm1 ^ zmm31))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm18 & (zmm1 ^ zmm31))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm27 = zmm25 ^ (zmm19 & (zmm27 ^ zmm25))
 ; AVX512-NEXT:    vpbroadcastq 24(%r8), %ymm2
 ; AVX512-NEXT:    vpbroadcastq 32(%r8), %ymm25
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm25, %zmm2, %zmm2
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm18 & (zmm2 ^ zmm27))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm18 & (zmm2 ^ zmm27))
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0]
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
 ; AVX512-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm31 # 64-byte Folded Reload
@@ -7027,28 +7027,28 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpbroadcastq 64(%r8), %ymm18
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm18, %zmm8
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (zmm18 & (zmm8 ^ zmm21))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (zmm18 & (zmm8 ^ zmm21))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm28, %zmm27, %zmm21
 ; AVX512-NEXT:    vpermq $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Folded Reload
 ; AVX512-NEXT:    # zmm22 = mem[0,1,0,1,4,5,4,5]
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm22 = zmm21 ^ (zmm24 & (zmm22 ^ zmm21))
 ; AVX512-NEXT:    vpbroadcastq (%r8), %ymm21
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm21, %zmm10
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm10 = zmm10 ^ (zmm18 & (zmm10 ^ zmm22))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm18 & (zmm10 ^ zmm22))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm13, %zmm14, %zmm13
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm12, %zmm11
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm13 ^ (zmm24 & (zmm11 ^ zmm13))
 ; AVX512-NEXT:    vpbroadcastq 112(%r8), %ymm12
 ; AVX512-NEXT:    vpbroadcastq 120(%r8), %ymm13
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm12 = zmm12 ^ (zmm16 & (zmm12 ^ zmm11))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 ^ (zmm16 & (zmm12 ^ zmm11))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm15, %zmm7
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm6, %zmm5
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm7 ^ (zmm24 & (zmm5 ^ zmm7))
 ; AVX512-NEXT:    vpbroadcastq 48(%r8), %ymm6
 ; AVX512-NEXT:    vpbroadcastq 56(%r8), %ymm7
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm16 & (zmm6 ^ zmm5))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm16 & (zmm6 ^ zmm5))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm30 ^ (zmm19 & (zmm9 ^ zmm30))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm20 ^ (zmm19 & (zmm0 ^ zmm20))
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
@@ -7237,7 +7237,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpbroadcastq 112(%r8), %ymm0
 ; AVX512-FCP-NEXT:    vpbroadcastq 120(%r8), %ymm1
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm14
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm14 = zmm14 ^ (zmm12 & (zmm14 ^ zmm2))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (zmm12 & (zmm14 ^ zmm2))
 ; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %ymm8
 ; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm8, %ymm0
 ; AVX512-FCP-NEXT:    vmovdqa64 64(%rdi), %ymm23
@@ -7282,7 +7282,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpbroadcastq 48(%r8), %ymm7
 ; AVX512-FCP-NEXT:    vpbroadcastq 56(%r8), %ymm11
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm7, %zmm7
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (zmm12 & (zmm7 ^ zmm15))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm12 & (zmm7 ^ zmm15))
 ; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
 ; AVX512-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm11, %xmm11 # 16-byte Folded Reload
 ; AVX512-FCP-NEXT:    # xmm11 = xmm11[0],mem[0],xmm11[1],mem[1],xmm11[2],mem[2],xmm11[3],mem[3]
@@ -7319,7 +7319,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpbroadcastq 96(%r8), %ymm5
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm5 & (zmm0 ^ zmm8))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm5 & (zmm0 ^ zmm8))
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm8
 ; AVX512-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm8 # 16-byte Folded Reload
 ; AVX512-FCP-NEXT:    # xmm8 = xmm8[0],mem[0],xmm8[1],mem[1],xmm8[2],mem[2],xmm8[3],mem[3]
@@ -7334,7 +7334,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpbroadcastq 24(%r8), %ymm4
 ; AVX512-FCP-NEXT:    vpbroadcastq 32(%r8), %ymm8
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm4, %zmm4
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm5 & (zmm4 ^ zmm6))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm5 & (zmm4 ^ zmm6))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0]
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm3 # 64-byte Folded Reload
@@ -7348,10 +7348,10 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpbroadcastq 64(%r8), %ymm5
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm5, %zmm2
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm5 & (zmm2 ^ zmm20))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm5 & (zmm2 ^ zmm20))
 ; AVX512-FCP-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm6, %zmm6
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm5 & (zmm6 ^ zmm21))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm5 & (zmm6 ^ zmm21))
 ; AVX512-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm25 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    # zmm25 = mem ^ (zmm1 & (zmm25 ^ mem))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm19 ^ (zmm1 & (zmm9 ^ zmm19))
@@ -7608,12 +7608,12 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpbroadcastq 96(%r8), %ymm2
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm18 & (zmm1 ^ zmm31))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm18 & (zmm1 ^ zmm31))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm27 = zmm25 ^ (zmm19 & (zmm27 ^ zmm25))
 ; AVX512DQ-NEXT:    vpbroadcastq 24(%r8), %ymm2
 ; AVX512DQ-NEXT:    vpbroadcastq 32(%r8), %ymm25
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm25, %zmm2, %zmm2
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm18 & (zmm2 ^ zmm27))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm18 & (zmm2 ^ zmm27))
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0]
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
 ; AVX512DQ-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm31 # 64-byte Folded Reload
@@ -7657,28 +7657,28 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpbroadcastq 64(%r8), %ymm18
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm18, %zmm8
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (zmm18 & (zmm8 ^ zmm21))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (zmm18 & (zmm8 ^ zmm21))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm28, %zmm27, %zmm21
 ; AVX512DQ-NEXT:    vpermq $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    # zmm22 = mem[0,1,0,1,4,5,4,5]
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm22 = zmm21 ^ (zmm24 & (zmm22 ^ zmm21))
 ; AVX512DQ-NEXT:    vpbroadcastq (%r8), %ymm21
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm21, %zmm10
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm10 = zmm10 ^ (zmm18 & (zmm10 ^ zmm22))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm18 & (zmm10 ^ zmm22))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm13, %zmm14, %zmm13
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm12, %zmm11
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm13 ^ (zmm24 & (zmm11 ^ zmm13))
 ; AVX512DQ-NEXT:    vpbroadcastq 112(%r8), %ymm12
 ; AVX512DQ-NEXT:    vpbroadcastq 120(%r8), %ymm13
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm12 = zmm12 ^ (zmm16 & (zmm12 ^ zmm11))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 ^ (zmm16 & (zmm12 ^ zmm11))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm15, %zmm7
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm6, %zmm5
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm7 ^ (zmm24 & (zmm5 ^ zmm7))
 ; AVX512DQ-NEXT:    vpbroadcastq 48(%r8), %ymm6
 ; AVX512DQ-NEXT:    vpbroadcastq 56(%r8), %ymm7
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm16 & (zmm6 ^ zmm5))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm16 & (zmm6 ^ zmm5))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm30 ^ (zmm19 & (zmm9 ^ zmm30))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm20 ^ (zmm19 & (zmm0 ^ zmm20))
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535]
@@ -7867,7 +7867,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 112(%r8), %ymm0
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 120(%r8), %ymm1
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm14
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm14 = zmm14 ^ (zmm12 & (zmm14 ^ zmm2))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (zmm12 & (zmm14 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rsi), %ymm8
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm8, %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 64(%rdi), %ymm23
@@ -7912,7 +7912,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 48(%r8), %ymm7
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 56(%r8), %ymm11
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm7, %zmm7
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (zmm12 & (zmm7 ^ zmm15))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm12 & (zmm7 ^ zmm15))
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm11, %xmm11 # 16-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # xmm11 = xmm11[0],mem[0],xmm11[1],mem[1],xmm11[2],mem[2],xmm11[3],mem[3]
@@ -7949,7 +7949,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 96(%r8), %ymm5
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm5 & (zmm0 ^ zmm8))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm5 & (zmm0 ^ zmm8))
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm8
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm8 # 16-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # xmm8 = xmm8[0],mem[0],xmm8[1],mem[1],xmm8[2],mem[2],xmm8[3],mem[3]
@@ -7964,7 +7964,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 24(%r8), %ymm4
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 32(%r8), %ymm8
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm4, %zmm4
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm5 & (zmm4 ^ zmm6))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm5 & (zmm4 ^ zmm6))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0]
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm3 # 64-byte Folded Reload
@@ -7978,10 +7978,10 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq 64(%r8), %ymm5
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm5, %zmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm5 & (zmm2 ^ zmm20))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm5 & (zmm2 ^ zmm20))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq (%r8), %ymm6
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm6, %zmm6
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm5 & (zmm6 ^ zmm21))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm5 & (zmm6 ^ zmm21))
 ; AVX512DQ-FCP-NEXT:    vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm25 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # zmm25 = mem ^ (zmm1 & (zmm25 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm19 ^ (zmm1 & (zmm9 ^ zmm19))
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll
index 244408c456dc4..5980ac1e4e2a3 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll
@@ -4110,19 +4110,19 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm8, %zmm8
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (zmm9 & (zmm8 ^ zmm16))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (zmm9 & (zmm8 ^ zmm16))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm14, %zmm13, %zmm13
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm13 = zmm13 ^ (zmm9 & (zmm13 ^ zmm17))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (zmm9 & (zmm13 ^ zmm17))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm5, %zmm0
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm5 & (zmm0 ^ zmm2))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm5 & (zmm0 ^ zmm2))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm7, %zmm2
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm5 & (zmm2 ^ zmm4))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm5 & (zmm2 ^ zmm4))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm11, %zmm4
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm5 & (zmm4 ^ zmm12))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm5 & (zmm4 ^ zmm12))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm15, %zmm3
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (zmm5 & (zmm3 ^ zmm1))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm5 & (zmm3 ^ zmm1))
 ; AVX512-NEXT:    vmovdqa64 %zmm3, (%rax)
 ; AVX512-NEXT:    vmovdqa64 %zmm4, 192(%rax)
 ; AVX512-NEXT:    vmovdqa64 %zmm2, 256(%rax)
@@ -4180,7 +4180,7 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm9, %ymm10
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm9, %ymm28
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm9 = [2,2,0,3,10,0,10,11]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm9 = [2,2,2,3,10,9,10,11]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm8, %zmm9, %zmm10
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm15
@@ -4270,7 +4270,7 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpermi2d %ymm4, %ymm0, %ymm19
 ; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm2[2,1,3,3,4,5,6,7]
 ; AVX512-FCP-NEXT:    vpermt2d %zmm4, %zmm20, %zmm0
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [0,0,0,0,8,8,0,9]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [0,0,0,0,8,8,10,9]
 ; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm8
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]
 ; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm8, %xmm1
@@ -4306,7 +4306,7 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm7 = ymm1[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]
 ; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [0,0,0,1,0,10,10,0]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [0,1,0,1,10,10,10,10]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm7, %zmm20, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm23, %ymm9
 ; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
@@ -4332,21 +4332,21 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm3[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm20, %zmm1
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm3 & (zmm1 ^ zmm2))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm3 & (zmm1 ^ zmm2))
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 256(%rax)
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm3 & (zmm0 ^ zmm5))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm3 & (zmm0 ^ zmm5))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm0 & (zmm4 ^ zmm19))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm0 & (zmm4 ^ zmm19))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm0 & (zmm6 ^ zmm18))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm0 & (zmm6 ^ zmm18))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, 192(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm11 = zmm11 ^ (zmm0 & (zmm11 ^ zmm22))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (zmm0 & (zmm11 ^ zmm22))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm11, 128(%rax)
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    # zmm1 = zmm1 ^ (zmm0 & (zmm1 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 320(%rax)
 ; AVX512-FCP-NEXT:    addq $40, %rsp
@@ -4562,19 +4562,19 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm20, %zmm19, %zmm10
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm10 = zmm10 ^ (zmm11 & (zmm10 ^ zmm16))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm11 & (zmm10 ^ zmm16))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm23, %zmm21, %zmm12
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm12 = zmm12 ^ (zmm11 & (zmm12 ^ zmm17))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 ^ (zmm11 & (zmm12 ^ zmm17))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm4 & (zmm0 ^ zmm22))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm4 & (zmm0 ^ zmm22))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm7, %zmm7
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (zmm4 & (zmm7 ^ zmm18))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm4 & (zmm7 ^ zmm18))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm4
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm5 & (zmm4 ^ zmm2))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm5 & (zmm4 ^ zmm2))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm9, %zmm1
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm5 & (zmm1 ^ zmm3))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm5 & (zmm1 ^ zmm3))
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 256(%rax)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 64(%rax)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm12, (%rax)
@@ -4633,7 +4633,7 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm11 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm4, %xmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm4, %xmm25
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [0,0,0,1,0,10,10,0]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [0,1,0,1,10,10,10,10]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm18, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm14
@@ -4702,7 +4702,7 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25,16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25]
 ; AVX512DQ-FCP-NEXT:    # ymm7 = mem[0,1,0,1]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm6, %ymm11
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm27 = [2,2,0,3,10,0,10,11]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm27 = [2,2,2,3,10,9,10,11]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm5, %zmm27, %zmm11
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm28, %ymm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm30, %ymm6
@@ -4760,7 +4760,7 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm26, %xmm0
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm0, %xmm4
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,2,1,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm10 = [0,0,0,0,8,8,0,9]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm10 = [0,0,0,0,8,8,10,9]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm10, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm22, %xmm13
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm20, %xmm14
@@ -4782,22 +4782,22 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,0,2,1,4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm10, %zmm5
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm9 = zmm9 ^ (zmm4 & (zmm9 ^ zmm17))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 ^ (zmm4 & (zmm9 ^ zmm17))
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm9, 256(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm6 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm6 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # zmm6 = zmm6 ^ (zmm4 & (zmm6 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, 64(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (zmm4 & (zmm5 ^ zmm3))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm4 & (zmm5 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, (%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm4 & (zmm0 ^ zmm2))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm4 & (zmm0 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 192(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm12 = zmm12 ^ (zmm0 & (zmm12 ^ zmm1))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 ^ (zmm0 & (zmm12 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm12, 128(%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm11 = zmm11 ^ (zmm0 & (zmm11 ^ zmm18))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (zmm0 & (zmm11 ^ zmm18))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm11, 320(%rax)
 ; AVX512DQ-FCP-NEXT:    popq %rax
 ; AVX512DQ-FCP-NEXT:    vzeroupper
@@ -8675,7 +8675,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,2,1]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm25, %zmm24, %zmm24
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm25 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm24 # 64-byte Folded Reload
+; AVX512-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm24 # 64-byte Folded Reload
 ; AVX512-NEXT:    # zmm24 = zmm24 ^ (zmm25 & (zmm24 ^ mem))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm27, %zmm26, %zmm26
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[2,3,2,3]
@@ -8688,7 +8688,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm10 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm10[0,2,2,1,4,5,6,7]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,1,0,1]
-; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm26 # 64-byte Folded Reload
+; AVX512-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm26 # 64-byte Folded Reload
 ; AVX512-NEXT:    # zmm26 = zmm26 ^ (zmm25 & (zmm26 ^ mem))
 ; AVX512-NEXT:    vpshuflw {{.*#+}} ymm5 = ymm5[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,2,2]
@@ -8703,31 +8703,31 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,4,4]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]
-; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm17 # 64-byte Folded Reload
+; AVX512-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm17 # 64-byte Folded Reload
 ; AVX512-NEXT:    # zmm17 = zmm17 ^ (zmm25 & (zmm17 ^ mem))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm16, %zmm18, %zmm16
-; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm16 # 64-byte Folded Reload
+; AVX512-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm16 # 64-byte Folded Reload
 ; AVX512-NEXT:    # zmm16 = zmm16 ^ (zmm25 & (zmm16 ^ mem))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm29, %zmm23, %zmm18
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm23, %zmm18 # 64-byte Folded Reload
+; AVX512-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm23, %zmm18 # 64-byte Folded Reload
 ; AVX512-NEXT:    # zmm18 = zmm18 ^ (zmm23 & (zmm18 ^ mem))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm11, %zmm4
-; AVX512-NEXT:    vpternlogd $184, (%rsp), %zmm23, %zmm4 # 64-byte Folded Reload
+; AVX512-NEXT:    vpternlogq $184, (%rsp), %zmm23, %zmm4 # 64-byte Folded Reload
 ; AVX512-NEXT:    # zmm4 = zmm4 ^ (zmm23 & (zmm4 ^ mem))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm13, %zmm7
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (zmm23 & (zmm7 ^ zmm20))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm23 & (zmm7 ^ zmm20))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm10, %zmm5
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (zmm23 & (zmm5 ^ zmm19))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm23 & (zmm5 ^ zmm19))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm12, %zmm8
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (zmm10 & (zmm8 ^ zmm22))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (zmm10 & (zmm8 ^ zmm22))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm14, %zmm6
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm10 & (zmm6 ^ zmm28))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm10 & (zmm6 ^ zmm28))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm9, %zmm0
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm10 & (zmm0 ^ zmm15))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm10 & (zmm0 ^ zmm15))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm10 & (zmm2 ^ zmm1))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm10 & (zmm2 ^ zmm1))
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-NEXT:    vmovdqa64 %zmm2, (%rax)
 ; AVX512-NEXT:    vmovdqa64 %zmm0, 192(%rax)
@@ -8793,7 +8793,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm15 = [16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25,16,17,20,21,20,21,22,23,24,25,24,25,24,25,24,25]
 ; AVX512-FCP-NEXT:    # ymm15 = mem[0,1,0,1]
 ; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm2, %ymm2
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm25 = [2,2,0,3,10,0,10,11]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm25 = [2,2,2,3,10,9,10,11]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm25, %zmm2
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %ymm3
@@ -8870,7 +8870,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpermt2d %ymm3, %ymm29, %ymm2
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm30 = [0,0,0,0,8,8,0,9]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm30 = [0,0,0,0,8,8,10,9]
 ; AVX512-FCP-NEXT:    vmovdqa 96(%r9), %xmm2
 ; AVX512-FCP-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,1,4,5,4,5,6,7,8,9,8,9,8,9,8,9]
@@ -9060,7 +9060,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm12[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]
 ; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm1, %xmm2
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,0,0,1,0,10,10,0]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [0,1,0,1,10,10,10,10]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
 ; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX512-FCP-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
@@ -9145,42 +9145,42 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm7 = ymm7[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm7, %zmm1, %zmm4
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm1 & (zmm4 ^ zmm3))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm1 & (zmm4 ^ zmm3))
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, 256(%rax)
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm1 & (zmm0 ^ zmm9))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm1 & (zmm0 ^ zmm9))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 448(%rax)
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (zmm1 & (zmm5 ^ zmm27))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm1 & (zmm5 ^ zmm27))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, 640(%rax)
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm1 & (zmm2 ^ zmm26))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm1 & (zmm2 ^ zmm26))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, 64(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm0 & (zmm6 ^ zmm23))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm0 & (zmm6 ^ zmm23))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, (%rax)
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (zmm0 & (zmm8 ^ zmm22))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (zmm0 & (zmm8 ^ zmm22))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm8, 192(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm1 & (zmm2 ^ zmm21))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm1 & (zmm2 ^ zmm21))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, 128(%rax)
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm2 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm2 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    # zmm2 = zmm2 ^ (zmm1 & (zmm2 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, 320(%rax)
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    # zmm2 = zmm2 ^ (zmm0 & (zmm2 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, 384(%rax)
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    # zmm2 = zmm2 ^ (zmm0 & (zmm2 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, 576(%rax)
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    # zmm0 = zmm0 ^ (zmm1 & (zmm0 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 512(%rax)
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; AVX512-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    # zmm0 = zmm0 ^ (zmm1 & (zmm0 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 704(%rax)
 ; AVX512-FCP-NEXT:    addq $1320, %rsp # imm = 0x528
@@ -9525,7 +9525,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm16
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm16 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm16 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    # zmm16 = zmm16 ^ (zmm28 & (zmm16 ^ mem))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm4[2,2,2,3]
 ; AVX512DQ-NEXT:    vinserti32x8 $1, %ymm1, %zmm2, %zmm3 {%k1}
@@ -9602,7 +9602,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm3 = ymm10[2,3,2,3,6,7,6,7]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm3[0,2,2,1,4,5,6,7,8,10,10,9,12,13,14,15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,1,2,3]
-; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm1 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm1 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    # zmm1 = zmm1 ^ (zmm28 & (zmm1 ^ mem))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm0, %zmm12
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm15, %zmm2, %zmm2
@@ -9633,32 +9633,32 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,1,0,1]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm10 = ymm10[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,2]
-; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm12 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm12 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    # zmm12 = zmm12 ^ (zmm28 & (zmm12 ^ mem))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm30, %zmm29, %zmm17
-; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm17 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm28, %zmm17 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    # zmm17 = zmm17 ^ (zmm28 & (zmm17 ^ mem))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm31, %zmm24, %zmm22
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm22 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm22 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    # zmm22 = zmm22 ^ (zmm24 & (zmm22 ^ mem))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm18, %zmm7
-; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm7 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm7 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    # zmm7 = zmm7 ^ (zmm24 & (zmm7 ^ mem))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm5, %zmm3
-; AVX512DQ-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm3 # 64-byte Folded Reload
+; AVX512DQ-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm3 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    # zmm3 = zmm3 ^ (zmm24 & (zmm3 ^ mem))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm13, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm24 & (zmm0 ^ zmm23))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm24 & (zmm0 ^ zmm23))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm15, %zmm14, %zmm5
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (zmm13 & (zmm5 ^ zmm25))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm13 & (zmm5 ^ zmm25))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm6, %zmm6
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm13 & (zmm6 ^ zmm26))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm13 & (zmm6 ^ zmm26))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm9, %zmm4
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm13 & (zmm4 ^ zmm27))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm13 & (zmm4 ^ zmm27))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm8, %zmm8
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (zmm13 & (zmm8 ^ zmm2))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (zmm13 & (zmm8 ^ zmm2))
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm8, 256(%rax)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 448(%rax)
@@ -9727,7 +9727,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,9,12,13,12,13,10,11,8,9,10,11,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm1, %xmm1
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [0,0,0,1,0,10,10,0]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm20 = [0,1,0,1,10,10,10,10]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm20, %zmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %ymm1
@@ -9886,7 +9886,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm13, %ymm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm2, %ymm13
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm26 = [2,2,0,3,10,0,10,11]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm26 = [2,2,2,3,10,9,10,11]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm26, %zmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm14, %ymm0
@@ -9946,7 +9946,7 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm3, %xmm4
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,0,2,1,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm17 = [0,0,0,0,8,8,0,9]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm17 = [0,0,0,0,8,8,10,9]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm17, %zmm3
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm5, %xmm4
@@ -10086,40 +10086,40 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm17, %zmm12
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm4 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm4 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # zmm4 = zmm4 ^ (zmm2 & (zmm4 ^ mem))
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 256(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm4 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm4 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # zmm4 = zmm4 ^ (zmm2 & (zmm4 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 448(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm4 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm4 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # zmm4 = zmm4 ^ (zmm2 & (zmm4 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 640(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm4 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm4 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # zmm4 = zmm4 ^ (zmm2 & (zmm4 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 64(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm12 = zmm12 ^ (zmm2 & (zmm12 ^ zmm7))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 ^ (zmm2 & (zmm12 ^ zmm7))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm12, (%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm2 & (zmm1 ^ zmm5))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm2 & (zmm1 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 192(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm11 = zmm11 ^ (zmm1 & (zmm11 ^ zmm22))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (zmm1 & (zmm11 ^ zmm22))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm11, 128(%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm9 = zmm9 ^ (zmm1 & (zmm9 ^ zmm24))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 ^ (zmm1 & (zmm9 ^ zmm24))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm9, 320(%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm14 = zmm14 ^ (zmm2 & (zmm14 ^ zmm30))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (zmm2 & (zmm14 ^ zmm30))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm14, 384(%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (zmm2 & (zmm3 ^ zmm20))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm2 & (zmm3 ^ zmm20))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, 576(%rax)
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm1 & (zmm0 ^ zmm21))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm1 & (zmm0 ^ zmm21))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 512(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # zmm0 = zmm0 ^ (zmm1 & (zmm0 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 704(%rax)
 ; AVX512DQ-FCP-NEXT:    addq $1112, %rsp # imm = 0x458
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
index 9f0710f5db970..11d4534817b87 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
@@ -646,7 +646,7 @@ define void @store_i16_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[4,5,12,13,u,u,u,u,u,u],zero,zero,zero,zero,ymm3[6,7],zero,zero,ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
 ; AVX512-NEXT:    vporq %zmm1, %zmm2, %zmm1
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm0 ^ (mem & (zmm1 ^ zmm0))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm0 ^ (mem & (zmm1 ^ zmm0))
 ; AVX512-NEXT:    vextracti32x4 $2, %zmm1, 32(%rax)
 ; AVX512-NEXT:    vextracti32x4 $3, %zmm1, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, 48(%rax)
@@ -728,7 +728,7 @@ define void @store_i16_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[4,5,12,13,u,u,u,u,u,u],zero,zero,zero,zero,ymm3[6,7],zero,zero,ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
 ; AVX512DQ-NEXT:    vporq %zmm1, %zmm2, %zmm1
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm0 ^ (mem & (zmm1 ^ zmm0))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm0 ^ (mem & (zmm1 ^ zmm0))
 ; AVX512DQ-NEXT:    vextracti32x4 $2, %zmm1, 32(%rax)
 ; AVX512DQ-NEXT:    vextracti32x4 $3, %zmm1, %xmm0
 ; AVX512DQ-NEXT:    vmovq %xmm0, 48(%rax)
@@ -1379,7 +1379,7 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpbroadcastd 4(%r10), %ymm12
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm11
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm11 = (zmm11 & mem) | zmm10
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm11 = zmm11 ^ (mem & (zmm11 ^ zmm9))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (mem & (zmm11 ^ zmm9))
 ; AVX512-NEXT:    vpsrlq $48, %xmm4, %xmm4
 ; AVX512-NEXT:    vpunpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm4[1]
 ; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
@@ -1461,7 +1461,7 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpbroadcastd 4(%r10), %ymm5
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (zmm4 & mem) | zmm3
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm2))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm2))
 ; AVX512-FCP-NEXT:    vmovdqa %xmm0, 96(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)
 ; AVX512-FCP-NEXT:    vmovdqa %ymm1, 64(%rax)
@@ -1504,7 +1504,7 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpbroadcastd 4(%r10), %ymm12
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm11
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm11 = (zmm11 & mem) | zmm10
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm11 = zmm11 ^ (mem & (zmm11 ^ zmm9))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (mem & (zmm11 ^ zmm9))
 ; AVX512DQ-NEXT:    vpsrlq $48, %xmm4, %xmm4
 ; AVX512DQ-NEXT:    vpunpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm4[1]
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
@@ -1586,7 +1586,7 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 4(%r10), %ymm5
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = (zmm4 & mem) | zmm3
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm2))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, 96(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, 64(%rax)
@@ -2842,7 +2842,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vporq %ymm1, %ymm4, %ymm17
 ; AVX512-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512-NEXT:    vmovdqa (%rsi), %xmm4
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm5 = ymm3[u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[14,15,u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[16,17,u,u],zero,zero
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[16,17],zero,zero,zero,zero
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm10 = ymm2[u,u,u,u,u,u,u,u,12,13,14,15],zero,zero,ymm2[u,u,u,u,u,u,u,u,u,u,16,17],zero,zero,ymm2[u,u,u,u]
 ; AVX512-NEXT:    vporq %ymm5, %ymm10, %ymm19
 ; AVX512-NEXT:    vpbroadcastd 8(%rax), %ymm5
@@ -2892,7 +2892,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm11[0],xmm15[1],xmm11[1],xmm15[2],xmm11[2],xmm15[3],xmm11[3]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm11 = xmm0[0,1,2,3,4,5,7,6]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,2,4,5,6,7]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm15 = [0,1,0,1,0,1,1,3,16,18,19,19,19,19,0,0]
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm15 = [0,0,0,1,0,1,1,0,0,18,19,0,19,19,0,0]
 ; AVX512-NEXT:    vpermi2d %zmm11, %zmm0, %zmm15
 ; AVX512-NEXT:    vprold $16, %ymm3, %ymm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm11 = ymm2[1,2,2,3,5,6,6,7]
@@ -2929,22 +2929,22 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm16, %zmm7, %zmm7
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm17, %zmm9, %zmm9
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm9 = zmm7 ^ (mem & (zmm9 ^ zmm7))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm7 ^ (mem & (zmm9 ^ zmm7))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm19, %zmm4, %zmm4
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm4 & mem)
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm9))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm4
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm7 = zmm20[2,2,2,3,6,6,6,7]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm4 ^ (mem & (zmm7 ^ zmm4))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm4 ^ (mem & (zmm7 ^ zmm4))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm12, %zmm0
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm0))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm0))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm7))
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm0 = zmm1[0,0,1,1,4,4,5,5]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm18 ^ (mem & (zmm0 ^ zmm18))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm18 ^ (mem & (zmm0 ^ zmm18))
 ; AVX512-NEXT:    vpbroadcastd (%rax), %ymm1
 ; AVX512-NEXT:    vpbroadcastd 4(%rax), %ymm4
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm1
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm15))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm15))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm0))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm6 ^ (mem & (ymm6 ^ ymm8))
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm13[2,1,3,2]
@@ -2975,7 +2975,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm20
 ; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm12
 ; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm14
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm1[u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[14,15,u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[16,17,u,u],zero,zero
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[16,17],zero,zero,zero,zero
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[u,u,u,u,u,u,u,u,12,13,14,15],zero,zero,ymm0[u,u,u,u,u,u,u,u,u,u,16,17],zero,zero,ymm0[u,u,u,u]
 ; AVX512-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm21
 ; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm15
@@ -3002,7 +3002,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm13 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,u,u,18,19,20,21,u,u,20,21]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm13 = ymm13[0,1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7,8,9,10],ymm3[11],ymm13[12,13],ymm3[14],ymm13[15]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm16 = [2,2,3,3,10,9,11,10]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm16 = [2,2,3,3,10,0,11,10]
 ; AVX512-FCP-NEXT:    vpermi2q %zmm8, %zmm13, %zmm16
 ; AVX512-FCP-NEXT:    vprold $16, %xmm11, %xmm8
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm13 = xmm10[1,1,2,3]
@@ -3014,7 +3014,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = xmm8[0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm19 = [0,0,0,1,8,9,9,11]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm19 = [0,0,0,1,8,9,9,0]
 ; AVX512-FCP-NEXT:    vpermi2q %zmm13, %zmm8, %zmm19
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm8 = xmm12[u,u,4,5,u,u,u,u,6,7,u,u,u,u,8,9]
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm13 = xmm14[1,1,2,2]
@@ -3057,20 +3057,20 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm17, %zmm11, %zmm1
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm20, %zmm10, %zmm10
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm1))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm1))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm21, %zmm2, %zmm1
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm1 & mem)
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm10))
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm22[2,2,2,3,6,6,6,7]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm9 ^ (mem & (zmm1 ^ zmm9))
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm16))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm9 ^ (mem & (zmm1 ^ zmm9))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm16))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm1))
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm8[0,0,1,1,4,4,5,5]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm18 ^ (mem & (zmm1 ^ zmm18))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm18 ^ (mem & (zmm1 ^ zmm18))
 ; AVX512-FCP-NEXT:    vpbroadcastd (%rax), %ymm2
 ; AVX512-FCP-NEXT:    vpbroadcastd 4(%rax), %ymm8
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm2, %zmm2
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm19))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm19))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm1))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = ymm4 ^ (mem & (ymm4 ^ ymm5))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm7 ^ (mem & (ymm0 ^ ymm7))
@@ -3104,7 +3104,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vporq %ymm1, %ymm4, %ymm17
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm4
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm5 = ymm3[u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[14,15,u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[16,17,u,u],zero,zero
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[16,17],zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm10 = ymm2[u,u,u,u,u,u,u,u,12,13,14,15],zero,zero,ymm2[u,u,u,u,u,u,u,u,u,u,16,17],zero,zero,ymm2[u,u,u,u]
 ; AVX512DQ-NEXT:    vporq %ymm5, %ymm10, %ymm19
 ; AVX512DQ-NEXT:    vpbroadcastd 8(%rax), %ymm5
@@ -3154,7 +3154,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm11[0],xmm15[1],xmm11[1],xmm15[2],xmm11[2],xmm15[3],xmm11[3]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm11 = xmm0[0,1,2,3,4,5,7,6]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm15 = [0,1,0,1,0,1,1,3,16,18,19,19,19,19,0,0]
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm15 = [0,0,0,1,0,1,1,0,0,18,19,0,19,19,0,0]
 ; AVX512DQ-NEXT:    vpermi2d %zmm11, %zmm0, %zmm15
 ; AVX512DQ-NEXT:    vprold $16, %ymm3, %ymm0
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm11 = ymm2[1,2,2,3,5,6,6,7]
@@ -3191,22 +3191,22 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm16, %zmm7, %zmm7
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm17, %zmm9, %zmm9
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm9 = zmm7 ^ (mem & (zmm9 ^ zmm7))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm7 ^ (mem & (zmm9 ^ zmm7))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm19, %zmm4, %zmm4
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm4 & mem)
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm9))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm4
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm7 = zmm20[2,2,2,3,6,6,6,7]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm4 ^ (mem & (zmm7 ^ zmm4))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm4 ^ (mem & (zmm7 ^ zmm4))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm12, %zmm0
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm0))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm0))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm7))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm0 = zmm1[0,0,1,1,4,4,5,5]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm18 ^ (mem & (zmm0 ^ zmm18))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm18 ^ (mem & (zmm0 ^ zmm18))
 ; AVX512DQ-NEXT:    vpbroadcastd (%rax), %ymm1
 ; AVX512DQ-NEXT:    vpbroadcastd 4(%rax), %ymm4
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm1
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm15))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm15))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm0))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm6 = ymm6 ^ (mem & (ymm6 ^ ymm8))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm13[2,1,3,2]
@@ -3237,7 +3237,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm20
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm12
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm14
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm1[u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[14,15,u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[16,17,u,u],zero,zero
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[16,17],zero,zero,zero,zero
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[u,u,u,u,u,u,u,u,12,13,14,15],zero,zero,ymm0[u,u,u,u,u,u,u,u,u,u,16,17],zero,zero,ymm0[u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm21
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm15
@@ -3264,7 +3264,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm13 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,u,u,18,19,20,21,u,u,20,21]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm13 = ymm13[0,1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7,8,9,10],ymm3[11],ymm13[12,13],ymm3[14],ymm13[15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm16 = [2,2,3,3,10,9,11,10]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm16 = [2,2,3,3,10,0,11,10]
 ; AVX512DQ-FCP-NEXT:    vpermi2q %zmm8, %zmm13, %zmm16
 ; AVX512DQ-FCP-NEXT:    vprold $16, %xmm11, %xmm8
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm13 = xmm10[1,1,2,3]
@@ -3276,7 +3276,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = xmm8[0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm19 = [0,0,0,1,8,9,9,11]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm19 = [0,0,0,1,8,9,9,0]
 ; AVX512DQ-FCP-NEXT:    vpermi2q %zmm13, %zmm8, %zmm19
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm8 = xmm12[u,u,4,5,u,u,u,u,6,7,u,u,u,u,8,9]
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm13 = xmm14[1,1,2,2]
@@ -3319,20 +3319,20 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm17, %zmm11, %zmm1
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm20, %zmm10, %zmm10
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm1))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm21, %zmm2, %zmm1
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm1 & mem)
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm10))
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm22[2,2,2,3,6,6,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm9 ^ (mem & (zmm1 ^ zmm9))
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm16))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm9 ^ (mem & (zmm1 ^ zmm9))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm16))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm8[0,0,1,1,4,4,5,5]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm18 ^ (mem & (zmm1 ^ zmm18))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm18 ^ (mem & (zmm1 ^ zmm18))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd (%rax), %ymm2
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 4(%rax), %ymm8
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm2, %zmm2
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm19))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm19))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm4 = ymm4 ^ (mem & (ymm4 ^ ymm5))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm7 ^ (mem & (ymm0 ^ ymm7))
@@ -5898,7 +5898,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm10[4],xmm2[4],xmm10[5],xmm2[5],xmm10[6],xmm2[6],xmm10[7],xmm2[7]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,7,6]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [16,18,19,19,19,19,0,0,0,1,0,1,2,3,2,3]
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [0,18,19,0,19,19,0,0,0,1,0,1,2,0,0,3]
 ; AVX512-NEXT:    vpermi2d %zmm3, %zmm2, %zmm4
 ; AVX512-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm2
@@ -5934,7 +5934,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm6[0,1,2,3,4,5,7,6]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,2,4,5,6,7]
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm25 = [0,1,0,1,0,1,1,3,16,18,19,19,19,19,0,0]
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm25 = [0,0,0,1,0,1,1,0,0,18,19,0,19,19,0,0]
 ; AVX512-NEXT:    vpermi2d %zmm7, %zmm6, %zmm25
 ; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
 ; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -6147,13 +6147,13 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,1,3,2]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm6, %zmm5
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm29))
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm27 = zmm27 ^ (mem & (zmm27 ^ zmm5))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm27 = zmm27 ^ (mem & (zmm27 ^ zmm5))
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm15[0,0,0,1]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,3,2]
 ; AVX512-NEXT:    vpbroadcastd 32(%rax), %ymm5
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm1, %zmm1
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm4))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm4))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm9))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm3, %zmm3
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm14, %zmm12, %zmm4
@@ -6162,7 +6162,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpbroadcastd 40(%rax), %ymm3
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm3))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm3))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm4))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm17, %zmm3, %zmm3
@@ -6178,7 +6178,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpbroadcastd (%rax), %ymm4
 ; AVX512-NEXT:    vpbroadcastd 4(%rax), %ymm5
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm25))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm25))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm3))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm31, %zmm30, %zmm3
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm5
@@ -6186,7 +6186,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm8, %zmm3
 ; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
 ; AVX512-NEXT:    vpermd (%rax), %zmm6, %zmm6
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm3))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm3))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-NEXT:    vmovdqa64 %zmm6, 128(%rax)
@@ -6286,7 +6286,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm6[3,3,3,3,7,7,7,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm12 = ymm0[0,1],ymm2[2],ymm0[3,4],ymm2[5],ymm0[6,7,8,9],ymm2[10],ymm0[11,12],ymm2[13],ymm0[14,15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm23 = [2,2,2,3,8,8,8,9]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm23 = [2,2,2,3,0,8,8,9]
 ; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm2
 ; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm3
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
@@ -6306,7 +6306,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm1, %xmm1
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
 ; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm28 = [0,1,1,3,8,8,9,9]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm28 = [0,1,1,0,8,8,9,9]
 ; AVX512-FCP-NEXT:    vpermi2q %zmm1, %zmm0, %zmm28
 ; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm1
 ; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm0
@@ -6326,7 +6326,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm3, %xmm4
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm17, %xmm7
 ; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm3, %xmm3
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm29 = [0,0,0,1,8,9,9,11]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm29 = [0,0,0,1,8,9,9,0]
 ; AVX512-FCP-NEXT:    vpermi2q %zmm4, %zmm3, %zmm29
 ; AVX512-FCP-NEXT:    vprold $16, %ymm15, %ymm3
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm6[1,2,2,3,5,6,6,7]
@@ -6335,7 +6335,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm15, %ymm7
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[0,0,2,1,4,4,6,5]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm7[0,1,2],ymm6[3],ymm7[4,5],ymm6[6],ymm7[7,8,9,10],ymm6[11],ymm7[12,13],ymm6[14],ymm7[15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm30 = [2,2,3,3,10,9,11,10]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm30 = [2,2,3,3,10,0,11,10]
 ; AVX512-FCP-NEXT:    vpermi2q %zmm3, %zmm6, %zmm30
 ; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm0, %xmm18
@@ -6421,7 +6421,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm6))
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [4,5,2,2,6,6,6,6,6,7,3,3,7,7,6,7]
 ; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm6, %zmm0
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm10))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm10))
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm6 = [6,0,0,0,7,0,0,7]
 ; AVX512-FCP-NEXT:    vpermd %ymm15, %ymm6, %ymm6
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
@@ -6429,7 +6429,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpbroadcastd 36(%rax), %ymm12
 ; AVX512-FCP-NEXT:    vpbroadcastd 40(%rax), %ymm14
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm12, %zmm12
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm12 = zmm12 ^ (mem & (zmm12 ^ zmm28))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 ^ (mem & (zmm12 ^ zmm28))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 ^ (mem & (zmm12 ^ zmm5))
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm18, %xmm5
 ; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,2,3,3,4,5,6,7]
@@ -6451,7 +6451,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpermt2q %zmm8, %zmm16, %zmm13
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (mem & (zmm13 ^ zmm7))
 ; AVX512-FCP-NEXT:    vpermd (%rax), %zmm20, %zmm7
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm30))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm30))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm13))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm14, %zmm8 # 32-byte Folded Reload
@@ -6469,12 +6469,12 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm4))
 ; AVX512-FCP-NEXT:    vpbroadcastd 32(%rax), %ymm1
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm6, %zmm1
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm23))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm23))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm21))
 ; AVX512-FCP-NEXT:    vpbroadcastd (%rax), %ymm2
 ; AVX512-FCP-NEXT:    vpbroadcastd 4(%rax), %ymm4
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm29))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm29))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm27))
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm7, 128(%rax)
@@ -6525,7 +6525,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm10[4],xmm2[4],xmm10[5],xmm2[5],xmm10[6],xmm2[6],xmm10[7],xmm2[7]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,7,6]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [16,18,19,19,19,19,0,0,0,1,0,1,2,3,2,3]
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [0,18,19,0,19,19,0,0,0,1,0,1,2,0,0,3]
 ; AVX512DQ-NEXT:    vpermi2d %zmm3, %zmm2, %zmm4
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm2
@@ -6561,7 +6561,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm6[0,1,2,3,4,5,7,6]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm25 = [0,1,0,1,0,1,1,3,16,18,19,19,19,19,0,0]
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm25 = [0,0,0,1,0,1,1,0,0,18,19,0,19,19,0,0]
 ; AVX512DQ-NEXT:    vpermi2d %zmm7, %zmm6, %zmm25
 ; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
 ; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -6774,13 +6774,13 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,1,3,2]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm6, %zmm5
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm29))
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm27 = zmm27 ^ (mem & (zmm27 ^ zmm5))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm27 = zmm27 ^ (mem & (zmm27 ^ zmm5))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm15[0,0,0,1]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,3,2]
 ; AVX512DQ-NEXT:    vpbroadcastd 32(%rax), %ymm5
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm1, %zmm1
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm4))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm4))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm9))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm3, %zmm3
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm14, %zmm12, %zmm4
@@ -6789,7 +6789,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpbroadcastd 40(%rax), %ymm3
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm3))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm3))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm4))
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm17, %zmm3, %zmm3
@@ -6805,7 +6805,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpbroadcastd (%rax), %ymm4
 ; AVX512DQ-NEXT:    vpbroadcastd 4(%rax), %ymm5
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm4, %zmm4
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm25))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm25))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm3))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm31, %zmm30, %zmm3
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm5
@@ -6813,7 +6813,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm8, %zmm3
 ; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
 ; AVX512DQ-NEXT:    vpermd (%rax), %zmm6, %zmm6
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm3))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm3))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm6, 128(%rax)
@@ -6913,7 +6913,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm6[3,3,3,3,7,7,7,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm12 = ymm0[0,1],ymm2[2],ymm0[3,4],ymm2[5],ymm0[6,7,8,9],ymm2[10],ymm0[11,12],ymm2[13],ymm0[14,15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm23 = [2,2,2,3,8,8,8,9]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm23 = [2,2,2,3,0,8,8,9]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm3
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
@@ -6933,7 +6933,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm1, %xmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm28 = [0,1,1,3,8,8,9,9]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm28 = [0,1,1,0,8,8,9,9]
 ; AVX512DQ-FCP-NEXT:    vpermi2q %zmm1, %zmm0, %zmm28
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm0
@@ -6953,7 +6953,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm4, %xmm3, %xmm4
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm17, %xmm7
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm29 = [0,0,0,1,8,9,9,11]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm29 = [0,0,0,1,8,9,9,0]
 ; AVX512DQ-FCP-NEXT:    vpermi2q %zmm4, %zmm3, %zmm29
 ; AVX512DQ-FCP-NEXT:    vprold $16, %ymm15, %ymm3
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm6[1,2,2,3,5,6,6,7]
@@ -6962,7 +6962,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm15, %ymm7
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[0,0,2,1,4,4,6,5]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm7[0,1,2],ymm6[3],ymm7[4,5],ymm6[6],ymm7[7,8,9,10],ymm6[11],ymm7[12,13],ymm6[14],ymm7[15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm30 = [2,2,3,3,10,9,11,10]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm30 = [2,2,3,3,10,0,11,10]
 ; AVX512DQ-FCP-NEXT:    vpermi2q %zmm3, %zmm6, %zmm30
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm0, %xmm18
@@ -7048,7 +7048,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm6))
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [4,5,2,2,6,6,6,6,6,7,3,3,7,7,6,7]
 ; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm6, %zmm0
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm10))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm10))
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm6 = [6,0,0,0,7,0,0,7]
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm15, %ymm6, %ymm6
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
@@ -7056,7 +7056,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 36(%rax), %ymm12
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 40(%rax), %ymm14
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm12, %zmm12
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm12 = zmm12 ^ (mem & (zmm12 ^ zmm28))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 ^ (mem & (zmm12 ^ zmm28))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 ^ (mem & (zmm12 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm18, %xmm5
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,2,3,3,4,5,6,7]
@@ -7078,7 +7078,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm8, %zmm16, %zmm13
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (mem & (zmm13 ^ zmm7))
 ; AVX512DQ-FCP-NEXT:    vpermd (%rax), %zmm20, %zmm7
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm30))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm30))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm13))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm14, %zmm8 # 32-byte Folded Reload
@@ -7096,12 +7096,12 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm4))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 32(%rax), %ymm1
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm6, %zmm1
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm23))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm23))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm21))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd (%rax), %ymm2
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 4(%rax), %ymm4
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm29))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm29))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm27))
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, 128(%rax)
@@ -12917,7 +12917,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpbroadcastd 100(%rax), %ymm1
 ; AVX512-NEXT:    vpbroadcastd 104(%rax), %ymm2
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm31
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm31 = zmm31 ^ (mem & (zmm31 ^ zmm0))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm31 = zmm31 ^ (mem & (zmm31 ^ zmm0))
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3]
 ; AVX512-NEXT:    vpshufb %xmm6, %xmm9, %xmm1
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm7[1,1,2,2]
@@ -12946,7 +12946,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpbroadcastd 68(%rax), %ymm5
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm25
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm25 = zmm25 ^ (zmm19 & (zmm25 ^ zmm1))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm25 = zmm25 ^ (zmm19 & (zmm25 ^ zmm1))
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
 ; AVX512-NEXT:    vpshufb %xmm6, %xmm8, %xmm3
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm13[1,1,2,2]
@@ -12973,7 +12973,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpbroadcastd (%rax), %ymm1
 ; AVX512-NEXT:    vpbroadcastd 4(%rax), %ymm2
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm20
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm20 = zmm20 ^ (zmm19 & (zmm20 ^ zmm0))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 ^ (zmm19 & (zmm20 ^ zmm0))
 ; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
 ; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm6[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,2,1,4,4,6,5]
@@ -13045,7 +13045,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpbroadcastd 36(%rax), %ymm2
 ; AVX512-NEXT:    vpbroadcastd 40(%rax), %ymm4
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm13
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm13 = zmm13 ^ (mem & (zmm13 ^ zmm1))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (mem & (zmm13 ^ zmm1))
 ; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm7
 ; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm6
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
@@ -13107,11 +13107,11 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm30[0,1,2,3],zmm2[0,1,2,3]
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm30 = zmm30 ^ (zmm2 & (zmm30 ^ zmm1))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm30 = zmm30 ^ (zmm2 & (zmm30 ^ zmm1))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm11 = zmm11 ^ (zmm2 & (zmm11 ^ zmm1))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (zmm2 & (zmm11 ^ zmm1))
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535]
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm25 = zmm25 ^ (zmm1 & (zmm25 ^ zmm14))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 ^ (zmm1 & (zmm20 ^ zmm5))
@@ -13131,10 +13131,10 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm14 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
 ; AVX512-NEXT:    vpermd 64(%rax), %zmm14, %zmm5
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (zmm3 & (zmm5 ^ zmm1))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm3 & (zmm5 ^ zmm1))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm24, %zmm26, %zmm1
 ; AVX512-NEXT:    vpermd (%rax), %zmm14, %zmm14
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm14 = zmm14 ^ (zmm3 & (zmm14 ^ zmm1))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (zmm3 & (zmm14 ^ zmm1))
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0]
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm1 & (zmm5 ^ zmm2))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (zmm1 & (zmm14 ^ zmm22))
@@ -13197,13 +13197,13 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm10[2,1,3,2]
 ; AVX512-NEXT:    vpbroadcastd 96(%rax), %ymm10
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm9 = zmm9 ^ (zmm29 & (zmm9 ^ zmm7))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 ^ (zmm29 & (zmm9 ^ zmm7))
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,0,1]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm24, %zmm3
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm26[2,1,3,2]
 ; AVX512-NEXT:    vpbroadcastd 32(%rax), %ymm10
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm7, %zmm7
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (zmm29 & (zmm7 ^ zmm3))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm29 & (zmm7 ^ zmm3))
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535]
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 ^ (zmm3 & (zmm9 ^ zmm22))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm3 & (zmm7 ^ zmm8))
@@ -13239,7 +13239,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX512-FCP-LABEL: store_i16_stride7_vf64:
 ; AVX512-FCP:       # %bb.0:
-; AVX512-FCP-NEXT:    subq $1496, %rsp # imm = 0x5D8
+; AVX512-FCP-NEXT:    subq $1512, %rsp # imm = 0x5E8
 ; AVX512-FCP-NEXT:    vmovdqa 96(%rcx), %ymm4
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
 ; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm4, %ymm0
@@ -13257,27 +13257,27 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
 ; AVX512-FCP-NEXT:    vmovdqa 64(%r9), %ymm1
 ; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm1, %ymm23
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm1, %ymm28
 ; AVX512-FCP-NEXT:    vmovdqa 64(%r8), %ymm13
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm12 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm13, %ymm1
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm13, %ymm25
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm13, %ymm27
 ; AVX512-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa 64(%rcx), %ymm0
 ; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm0, %ymm13
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm26
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm30
 ; AVX512-FCP-NEXT:    vmovdqa 64(%rdx), %ymm0
 ; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm0, %ymm14
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm30
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm23
 ; AVX512-FCP-NEXT:    vpor %ymm13, %ymm14, %ymm13
 ; AVX512-FCP-NEXT:    vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %ymm0
 ; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm0, %ymm13
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm22
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm29
 ; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %ymm0
 ; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm0, %ymm14
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm27
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm0, %ymm17
 ; AVX512-FCP-NEXT:    vpor %ymm13, %ymm14, %ymm13
 ; AVX512-FCP-NEXT:    vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa (%r9), %ymm0
@@ -13320,7 +13320,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm8, %ymm10
 ; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %ymm9
 ; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm9, %ymm11
-; AVX512-FCP-NEXT:    vporq %ymm11, %ymm10, %ymm17
+; AVX512-FCP-NEXT:    vporq %ymm11, %ymm10, %ymm19
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm4[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm12 = ymm5[0,1,1,3,4,5,5,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm12[0,1],ymm10[2],ymm12[3,4],ymm10[5],ymm12[6,7,8,9],ymm10[10],ymm12[11,12],ymm10[13],ymm12[14,15]
@@ -13328,17 +13328,17 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
 ; AVX512-FCP-NEXT:    # ymm13 = mem[0,1,0,1]
 ; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm2, %ymm12
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm13, %ymm21
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm13, %ymm26
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm13 = ymm3[1,1,1,1,5,5,5,5]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm12 = ymm12[0,1],ymm13[2],ymm12[3,4],ymm13[5],ymm12[6,7,8,9],ymm13[10],ymm12[11,12],ymm13[13],ymm12[14,15]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,2,2,3]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm16, %zmm10
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm7, %zmm12
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm10 ^ (mem & (zmm12 ^ zmm10))
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
 ; AVX512-FCP-NEXT:    vmovdqa 96(%r8), %ymm7
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm7[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm7[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm7[16,17,u,u,u,u],zero,zero
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm10 | (ymm12 & ymm18)
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm10 | (ymm12 & ymm13)
 ; AVX512-FCP-NEXT:    vmovdqa 96(%r9), %ymm11
 ; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm11, %ymm6
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm11, %ymm16
@@ -13350,7 +13350,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpermd %ymm7, %ymm12, %ymm12
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm12 = ymm12 ^ (ymm11 & (ymm12 ^ ymm10))
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm11, %zmm20
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm11, %zmm21
 ; AVX512-FCP-NEXT:    vprold $16, %ymm16, %ymm10
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,2]
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
@@ -13358,10 +13358,12 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm6[0,1,2,3],zmm10[0,1,2,3]
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [0,1,4,5,4,5,5,7]
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm12 = [0,1,4,5,4,5,5,7,4,5,2,2,6,6,6,6]
 ; AVX512-FCP-NEXT:    vmovdqa 96(%rax), %ymm6
-; AVX512-FCP-NEXT:    vpermd %ymm6, %ymm10, %ymm10
-; AVX512-FCP-NEXT:    vpandnq %ymm10, %ymm18, %ymm10
+; AVX512-FCP-NEXT:    vpermd %ymm6, %ymm12, %ymm10
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm12, %zmm16
+; AVX512-FCP-NEXT:    vpandn %ymm10, %ymm13, %ymm10
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm13, %zmm25
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
 ; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm6, %ymm12
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm12, %zmm10
@@ -13372,30 +13374,31 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm12[0,1,2],ymm10[3],ymm12[4,5],ymm10[6],ymm12[7,8,9,10],ymm10[11],ymm12[12,13],ymm10[14],ymm12[15]
 ; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm12 = [151522058,0,421010202,421010202]
 ; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm2, %ymm2
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm12, %ymm19
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm12, %ymm22
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[2,2,2,2,6,6,6,6]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6,7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13,14,15]
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,2,2,3,10,9,11,11]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm10, %zmm3, %zmm2
 ; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm12 = [218894094,0,488382238,488382238]
 ; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm4, %ymm10
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm12, %ymm16
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm12, %ymm20
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm12 = ymm5[3,3,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm12[0],ymm10[1],ymm12[2,3],ymm10[4],ymm12[5,6,7,8],ymm10[9],ymm12[10,11],ymm10[12],ymm12[13,14,15]
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
-; AVX512-FCP-NEXT:    # ymm12 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm4, %ymm4
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm12 = ymm12[0],ymm10[1],ymm12[2,3],ymm10[4],ymm12[5,6,7,8],ymm10[9],ymm12[10,11],ymm10[12],ymm12[13,14,15]
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
+; AVX512-FCP-NEXT:    # ymm10 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm4, %ymm4
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm10, %ymm24
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[2,2,2,2,6,6,6,6]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7,8,9],ymm5[10],ymm4[11,12],ymm5[13],ymm4[14,15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm24 = [2,2,2,3,8,10,10,11]
-; AVX512-FCP-NEXT:    vpermt2q %zmm10, %zmm24, %zmm4
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm31 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm2 ^ (zmm31 & (zmm4 ^ zmm2))
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [2,2,2,3,8,10,10,11]
+; AVX512-FCP-NEXT:    vpermt2q %zmm12, %zmm18, %zmm4
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm2 ^ (zmm12 & (zmm4 ^ zmm2))
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm2 = [5,0,0,0,6,0,0,6]
 ; AVX512-FCP-NEXT:    vpermd %ymm7, %ymm2, %ymm2
 ; AVX512-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [0,5,0,0,0,6,0,0,6,0,0,0,7,0,0,7]
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [4,5,2,2,6,6,6,6,6,7,3,3,7,7,6,7]
 ; AVX512-FCP-NEXT:    vpermd %zmm6, %zmm2, %zmm4
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
@@ -13418,29 +13421,30 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm8[0,0,2,1,4,4,6,5]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1,2],ymm5[3],ymm4[4,5],ymm5[6],ymm4[7,8,9,10],ymm5[11],ymm4[12,13],ymm5[14],ymm4[15]
 ; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [0,1,0,3,10,10,11,11]
-; AVX512-FCP-NEXT:    vpermi2q %zmm4, %zmm17, %zmm5
+; AVX512-FCP-NEXT:    vpermi2q %zmm4, %zmm19, %zmm5
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm0, %ymm4
 ; AVX512-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm1[3,3,3,3,7,7,7,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3],ymm5[4,5],ymm4[6],ymm5[7,8,9,10],ymm4[11],ymm5[12,13],ymm4[14],ymm5[15]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm19, %ymm10
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm22, %ymm10
 ; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm0, %ymm5
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm1[2,2,2,2,6,6,6,6]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3],ymm5[4],ymm6[5,6,7,8],ymm5[9],ymm6[10,11],ymm5[12],ymm6[13,14,15]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm3, %zmm5
 ; AVX512-FCP-NEXT:    vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm16, %ymm7
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm20, %ymm7
 ; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm14, %ymm3
 ; AVX512-FCP-NEXT:    vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm15[3,3,3,3,7,7,7,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6,7,8],ymm3[9],ymm4[10,11],ymm3[12],ymm4[13,14,15]
-; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm14, %ymm4
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm24, %ymm1
+; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm14, %ymm4
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm15[2,2,2,2,6,6,6,6]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0,1],ymm6[2],ymm4[3,4],ymm6[5],ymm4[6,7,8,9],ymm6[10],ymm4[11,12],ymm6[13],ymm4[14,15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm24, %zmm0
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm5 ^ (zmm31 & (zmm0 ^ zmm5))
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm18, %zmm0
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm5 ^ (zmm12 & (zmm0 ^ zmm5))
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm8[3,3,3,3,7,7,7,7]
@@ -13448,107 +13452,105 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vprold $16, %ymm9, %ymm4
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm8[1,2,2,3,5,6,6,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7,8,9],ymm4[10],ymm5[11,12],ymm4[13],ymm5[14,15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm14 = [2,1,3,2,10,10,10,11]
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm14, %zmm4
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm22 = [2,1,3,2,10,10,10,11]
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm22, %zmm4
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %ymm0
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm2, %zmm0
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm20 & (zmm0 ^ zmm4))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm21 & (zmm0 ^ zmm4))
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm22, %ymm0
-; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm0, %ymm2
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm27[2,2,2,2,6,6,6,6]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm29, %ymm15
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm15, %ymm2
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm10, %ymm19
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm17[2,2,2,2,6,6,6,6]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6,7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13,14,15]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm21, %ymm13
-; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm0, %ymm3
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm22, %ymm21
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm27[1,1,1,1,5,5,5,5]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm26, %ymm9
+; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm15, %ymm3
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm17[1,1,1,1,5,5,5,5]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7,8,9],ymm4[10],ymm3[11,12],ymm4[13],ymm3[14,15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm24, %zmm3
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm26, %ymm0
-; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm0, %ymm2
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm30[2,2,2,2,6,6,6,6]
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm18, %zmm3
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm30, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm0, %ymm2
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm24, %ymm14
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm23[2,2,2,2,6,6,6,6]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm30[0,1,1,3,4,5,5,7]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm23[0,1,1,3,4,5,5,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7,8,9],ymm4[10],ymm5[11,12],ymm4[13],ymm5[14,15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm14, %zmm6
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm22 = [0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm3 ^ (zmm22 & (zmm6 ^ zmm3))
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm23, %ymm10
-; AVX512-FCP-NEXT:    vprold $16, %ymm23, %ymm2
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm25, %ymm20
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm25[1,2,2,3,5,6,6,7]
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm22, %zmm6
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm3 ^ (zmm30 & (zmm6 ^ zmm3))
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm28, %ymm10
+; AVX512-FCP-NEXT:    vprold $16, %ymm28, %ymm2
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm27[1,2,2,3,5,6,6,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1],ymm2[2],ymm3[3,4],ymm2[5],ymm3[6,7,8,9],ymm2[10],ymm3[11,12],ymm2[13],ymm3[14,15]
 ; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm10, %ymm3
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm23, %ymm19
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm25[0,0,2,1,4,4,6,5]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7,8,9,10],ymm4[11],ymm3[12,13],ymm4[14],ymm3[15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [2,2,3,3,10,9,11,10]
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm1, %zmm3
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
-; AVX512-FCP-NEXT:    vpermd 64(%rax), %zmm1, %zmm1
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm18 & (zmm1 ^ zmm3))
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm27[0,0,2,1,4,4,6,5]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3],ymm3[4,5],ymm5[6],ymm3[7,8,9,10],ymm5[11],ymm3[12,13],ymm5[14],ymm3[15]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm21 = [2,2,3,3,10,9,11,10]
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm21, %zmm3
+; AVX512-FCP-NEXT:    vpermd 64(%rax), %zmm16, %zmm1
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm25 & (zmm1 ^ zmm3))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm6))
 ; AVX512-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm0, %ymm1
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm0 = ymm30[3,3,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm9 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6,7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13,14,15]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm0 = ymm23[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm12 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6,7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13,14,15]
 ; AVX512-FCP-NEXT:    vmovdqa 64(%rcx), %xmm0
 ; AVX512-FCP-NEXT:    vmovdqa 64(%rdx), %xmm3
 ; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm10 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
 ; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm0, %xmm2
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[1,1,2,2]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm6[0],xmm2[1],xmm6[2,3],xmm2[4],xmm6[5,6],xmm2[7]
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; AVX512-FCP-NEXT:    vmovdqa %xmm4, (%rsp) # 16-byte Spill
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; AVX512-FCP-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,1,2,3,0,1,2,3,6,7,4,5,6,7,4,5]
-; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm0
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm17 = [0,1,1,3,8,8,9,9]
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm17, %zmm0
-; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %xmm4
-; AVX512-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %xmm6
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm11 = [0,1,2,3,0,1,2,3,6,7,4,5,6,7,4,5]
+; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm0, %xmm0
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm24 = [0,1,1,3,8,8,9,9]
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm24, %zmm0
+; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %xmm6
 ; AVX512-FCP-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vprold $16, %xmm6, %xmm2
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm3 = xmm4[1,1,2,3]
+; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %xmm7
+; AVX512-FCP-NEXT:    vmovdqa %xmm7, (%rsp) # 16-byte Spill
+; AVX512-FCP-NEXT:    vprold $16, %xmm7, %xmm2
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm3 = xmm6[1,1,2,3]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2],xmm3[3,4],xmm2[5],xmm3[6,7]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
-; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm3, %xmm3
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [0,0,1,1,8,8,10,9]
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm18, %zmm3
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
+; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm3, %xmm3
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm26 = [0,0,1,1,8,8,10,9]
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm26, %zmm3
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0]
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm0 ^ (zmm2 & (zmm3 ^ zmm0))
 ; AVX512-FCP-NEXT:    vmovdqa 64(%r9), %xmm0
 ; AVX512-FCP-NEXT:    vmovdqa 64(%r8), %xmm6
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
-; AVX512-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
+; AVX512-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm11 = [0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
-; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm0, %xmm6
-; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm0, %xmm0
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm7 = [0,0,0,1,8,9,9,11]
-; AVX512-FCP-NEXT:    vpermt2q %zmm6, %zmm7, %zmm0
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm7 = [0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
+; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm0, %xmm6
+; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm0, %xmm0
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm16 = [0,0,0,1,8,9,9,11]
+; AVX512-FCP-NEXT:    vpermt2q %zmm6, %zmm16, %zmm0
 ; AVX512-FCP-NEXT:    vpbroadcastd 64(%rax), %ymm6
 ; AVX512-FCP-NEXT:    vpbroadcastd 68(%rax), %ymm8
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm6, %zmm30
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm30 = zmm30 ^ (zmm12 & (zmm30 ^ zmm0))
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm6, %zmm25
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm25 = zmm25 ^ (zmm13 & (zmm25 ^ zmm0))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm30 = zmm30 ^ (zmm0 & (zmm30 ^ zmm3))
-; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm4
-; AVX512-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm8
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm25 = zmm25 ^ (zmm0 & (zmm25 ^ zmm3))
+; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm8
 ; AVX512-FCP-NEXT:    vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm4, %xmm3
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm8[1,1,2,2]
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm4
+; AVX512-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm8, %xmm3
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[1,1,2,2]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm6[0],xmm3[1],xmm6[2,3],xmm3[4],xmm6[5,6],xmm3[7]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3]
-; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm6, %xmm6
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm17, %zmm6
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm4[0],xmm8[0],xmm4[1],xmm8[1],xmm4[2],xmm8[2],xmm4[3],xmm8[3]
+; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm6, %xmm6
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm24, %zmm6
 ; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm4
 ; AVX512-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm10
@@ -13557,232 +13559,231 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm4[1,1,2,3]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm4[0],xmm10[0],xmm4[1],xmm10[1],xmm4[2],xmm10[2],xmm4[3],xmm10[3]
-; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm8, %xmm8
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, %xmm10
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm18, %zmm8
+; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm8, %xmm8
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm26, %zmm8
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm6 ^ (zmm2 & (zmm8 ^ zmm6))
 ; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm2
 ; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm3
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; AVX512-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm2, %xmm3
-; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm2, %xmm2
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm15, %xmm17
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm7, %zmm2
+; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm2, %xmm3
+; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm7, %xmm24
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm16, %zmm2
 ; AVX512-FCP-NEXT:    vpbroadcastd (%rax), %ymm3
 ; AVX512-FCP-NEXT:    vpbroadcastd 4(%rax), %ymm6
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm3, %zmm26
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm26 = zmm26 ^ (zmm12 & (zmm26 ^ zmm2))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm26 = zmm26 ^ (zmm13 & (zmm26 ^ zmm2))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm26 = zmm26 ^ (zmm0 & (zmm26 ^ zmm8))
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm15[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm11[2,2,2,2,6,6,6,6]
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm19, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm8, %ymm0
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm19 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm19[2,2,2,2,6,6,6,6]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6,7,8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13,14,15]
-; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm15, %ymm2
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm11[1,1,1,1,5,5,5,5]
+; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm8, %ymm2
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm19[1,1,1,1,5,5,5,5]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7,8,9],ymm3[10],ymm2[11,12],ymm3[13],ymm2[14,15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm24, %zmm2
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,u,u,24,25,26,27,u,u,26,27,26,27]
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm7[2,2,2,2,6,6,6,6]
+; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm18, %zmm2
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm9, %ymm0
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm16 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm16[2,2,2,2,6,6,6,6]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm0[0,1],ymm3[2],ymm0[3,4],ymm3[5],ymm0[6,7,8,9],ymm3[10],ymm0[11,12],ymm3[13],ymm0[14,15]
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm4[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm7[0,1,1,3,4,5,5,7]
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm9[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm16[0,1,1,3,4,5,5,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm6[0,1],ymm0[2],ymm6[3,4],ymm0[5],ymm6[6,7,8,9],ymm0[10],ymm6[11,12],ymm0[13],ymm6[14,15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm14, %zmm0
-; AVX512-FCP-NEXT:    vmovdqa 96(%rcx), %xmm13
-; AVX512-FCP-NEXT:    vmovdqa 96(%rdx), %xmm14
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
-; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm3, %xmm3
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm5, %xmm25
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm24 = [0,2,2,3,8,9,9,11]
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm9, %zmm1
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm24, %zmm1
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm12 = [u,u,u,u,u,u,u,u,u,u,u,u,14,15,12,13,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm21, %ymm3
-; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm3, %ymm3
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm27[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm22, %zmm0
+; AVX512-FCP-NEXT:    vmovdqa 96(%rcx), %xmm14
+; AVX512-FCP-NEXT:    vmovdqa 96(%rdx), %xmm13
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm3, %xmm3
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm22 = [0,2,2,3,8,9,9,11]
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm22, %zmm12
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [u,u,u,u,u,u,u,u,u,u,u,u,14,15,12,13,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm29, %ymm1
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm1, %ymm3
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm17[3,3,3,3,7,7,7,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1,2],ymm3[3],ymm6[4,5],ymm3[6],ymm6[7,8,9,10],ymm3[11],ymm6[12,13],ymm3[14],ymm6[15]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm2 ^ (zmm22 & (zmm0 ^ zmm2))
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX512-FCP-NEXT:    vprold $16, %ymm5, %ymm2
-; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm23 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm23[1,2,2,3,5,6,6,7]
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm2 ^ (zmm30 & (zmm0 ^ zmm2))
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512-FCP-NEXT:    vprold $16, %ymm4, %ymm2
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm17[1,2,2,3,5,6,6,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1],ymm2[2],ymm3[3,4],ymm2[5],ymm3[6,7,8,9],ymm2[10],ymm3[11,12],ymm2[13],ymm3[14,15]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm5, %ymm21
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm23[0,0,2,1,4,4,6,5]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3],ymm3[4,5],ymm5[6],ymm3[7,8,9,10],ymm5[11],ymm3[12,13],ymm5[14],ymm3[15]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [2,2,3,3,10,9,11,10]
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm5, %zmm3
-; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %xmm5
-; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm9
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
-; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm2, %xmm2
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [2,1,3,3,8,8,9,9]
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm18, %zmm6
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm1 ^ (zmm31 & (zmm6 ^ zmm1))
-; AVX512-FCP-NEXT:    vbroadcasti32x4 {{.*#+}} ymm16 = [26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
-; AVX512-FCP-NEXT:    # ymm16 = mem[0,1,2,3,0,1,2,3]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm16, %ymm1
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm19, %ymm2
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm4, %ymm31
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm17[0,0,2,1,4,4,6,5]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7,8,9,10],ymm4[11],ymm3[12,13],ymm4[14],ymm3[15]
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm21, %zmm3
+; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %xmm4
+; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm5
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512-FCP-NEXT:    vmovdqa %xmm15, %xmm7
+; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm15 = [2,1,3,3,8,8,9,9]
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm15, %zmm6
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm12 ^ (zmm18 & (zmm6 ^ zmm12))
+; AVX512-FCP-NEXT:    vbroadcasti32x4 {{.*#+}} ymm21 = [26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
+; AVX512-FCP-NEXT:    # ymm21 = mem[0,1,2,3,0,1,2,3]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm21, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm28, %ymm2
 ; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm20[3,3,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm8 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm27[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm12 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,1,4,5,4,5,5,7,4,5,2,2,6,6,6,6]
 ; AVX512-FCP-NEXT:    vpermd (%rax), %zmm1, %zmm20
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm20 = zmm20 ^ (mem & (zmm20 ^ zmm3))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 ^ (mem & (zmm20 ^ zmm3))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 ^ (mem & (zmm20 ^ zmm0))
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm1
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm2
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm29
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm1, %xmm28
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm25, %xmm1
-; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm0, %xmm3
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm4[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm7[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm0
+; AVX512-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm1, %xmm29
+; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm0, %xmm3
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm9[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm16[3,3,3,3,7,7,7,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6,7,8],ymm1[9],ymm2[10,11],ymm1[12],ymm2[13,14,15]
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm24, %zmm0
-; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm15, %ymm1
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm11[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm22, %zmm0
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm8, %ymm1
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm19[3,3,3,3,7,7,7,7]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm2[0,1,2],ymm1[3],ymm2[4,5],ymm1[6],ymm2[7,8,9,10],ymm1[11],ymm2[12,13],ymm1[14],ymm2[15]
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm12
-; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm11
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
-; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm2, %xmm2
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm10, %xmm19
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm18, %zmm3
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm24 = [2,2,2,3,8,8,8,9]
-; AVX512-FCP-NEXT:    vmovdqa 96(%r9), %xmm7
-; AVX512-FCP-NEXT:    vmovdqa 96(%r8), %xmm4
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm4[0],xmm7[0],xmm4[1],xmm7[1],xmm4[2],xmm7[2],xmm4[3],xmm7[3]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm17, %xmm1
-; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm10, %xmm15
-; AVX512-FCP-NEXT:    vpermt2q %zmm15, %zmm24, %zmm8
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm0 ^ (zmm31 & (zmm3 ^ zmm0))
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm15 = [6,7,3,3,7,7,6,7]
-; AVX512-FCP-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm0 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm11
+; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm9
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
+; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm7, %xmm22
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm15, %zmm3
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm30 = [2,2,2,3,8,8,8,9]
+; AVX512-FCP-NEXT:    vmovdqa 96(%r9), %xmm8
+; AVX512-FCP-NEXT:    vmovdqa 96(%r8), %xmm7
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm24, %xmm1
+; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm15, %xmm10
+; AVX512-FCP-NEXT:    vpermt2q %zmm10, %zmm30, %zmm12
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm0 ^ (zmm18 & (zmm3 ^ zmm0))
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [6,7,3,3,7,7,6,7]
+; AVX512-FCP-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm0 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    vpbroadcastd 96(%rax), %ymm18
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm18, %zmm0, %zmm18
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm31 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm18 = zmm18 ^ (zmm31 & (zmm18 ^ zmm8))
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm25 = [0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm18 = zmm18 ^ (zmm25 & (zmm18 ^ zmm6))
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm5[4],xmm9[4],xmm5[5],xmm9[5],xmm5[6],xmm9[6],xmm5[7],xmm9[7]
-; AVX512-FCP-NEXT:    vprold $16, %xmm5, %xmm0
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm9[1,1,2,3]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm5[0,1],xmm0[2],xmm5[3,4],xmm0[5],xmm5[6,7]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm16, %ymm1
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm21, %ymm5
-; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm23[3,3,3,3,7,7,7,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm5[2],ymm1[3,4],ymm5[5],ymm1[6,7,8,9],ymm5[10],ymm1[11,12],ymm5[13],ymm1[14,15]
-; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm6
-; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm5
-; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm17, %xmm9
-; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm8, %xmm9
-; AVX512-FCP-NEXT:    vpermt2q %zmm9, %zmm24, %zmm1
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm24 = [0,0,2,1,8,8,9,11]
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm24, %zmm0
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
-; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm16 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm16, %xmm9
-; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm13, %xmm13
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm14 = xmm14[1,1,2,2]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm13 = xmm14[0],xmm13[1],xmm14[2,3],xmm13[4],xmm14[5,6],xmm13[7]
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm18 = zmm18 ^ (zmm16 & (zmm18 ^ zmm12))
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm18 = zmm18 ^ (zmm27 & (zmm18 ^ zmm6))
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
+; AVX512-FCP-NEXT:    vprold $16, %xmm4, %xmm0
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm5[1,1,2,3]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm4[0,1],xmm0[2],xmm4[3,4],xmm0[5],xmm4[6,7]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm21, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm31, %ymm4
+; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm4, %ymm1
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm17[3,3,3,3,7,7,7,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm4[2],ymm1[3,4],ymm4[5],ymm1[6,7,8,9],ymm4[10],ymm1[11,12],ymm4[13],ymm1[14,15]
+; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm5
+; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm4
+; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm24, %xmm12
+; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm6, %xmm12
+; AVX512-FCP-NEXT:    vpermt2q %zmm12, %zmm30, %zmm1
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm12 = [4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
+; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm12, %xmm31
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm30 = [0,0,2,1,8,8,9,11]
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm30, %zmm0
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm13[4],xmm14[4],xmm13[5],xmm14[5],xmm13[6],xmm14[6],xmm13[7],xmm14[7]
+; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm21 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm21, %xmm12
+; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm14, %xmm14
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[1,1,2,2]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm13 = xmm13[0],xmm14[1],xmm13[2,3],xmm14[4],xmm13[5,6],xmm14[7]
 ; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,2,3,3,4,5,6,7]
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm22 = [0,0,1,1,8,8,10,9]
-; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm22, %zmm13
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm0 ^ (zmm23 & (zmm13 ^ zmm0))
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm4[4],xmm7[4],xmm4[5],xmm7[5],xmm4[6],xmm7[6],xmm4[7],xmm7[7]
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm23 = [0,0,1,1,8,8,10,9]
+; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm23, %zmm13
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm0 ^ (zmm24 & (zmm13 ^ zmm0))
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7]
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm14 = ymm14 ^ (ymm31 & (ymm14 ^ ymm2))
+; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm17 = ymm17 ^ (ymm16 & (ymm17 ^ ymm2))
 ; AVX512-FCP-NEXT:    vextracti64x4 $1, %zmm2, %ymm2
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm19, %xmm9
-; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm0, %xmm0
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm17 = [0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm17, %xmm4
-; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm10, %xmm4
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm21 = [0,1,1,3,8,8,9,9]
-; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm21, %zmm4
-; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
-; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm7[0,1,2,3,5,5,7,6,8,9,10,11,13,13,15,14]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm22, %xmm12
+; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm0, %xmm0
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm22 = [0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm22, %xmm7
+; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm15, %xmm7
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm19 = [0,1,1,3,8,8,9,9]
+; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm19, %zmm7
+; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm8[0,1,2,3,5,5,7,6,8,9,10,11,13,13,15,14]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[3,3,3,3]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25,22,23,22,23,u,u,u,u,u,u,u,u]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,2,2]
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm10, %xmm10
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm19 = ymm10[0,0,1,1]
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm10, %xmm10
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm9, %xmm27
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm10[0,0,1,1]
-; AVX512-FCP-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm10 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25,22,23,22,23,u,u,u,u,u,u,u,u]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,2,2]
+; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm14, %xmm14
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,0,1,1]
+; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; AVX512-FCP-NEXT:    vpshufb %xmm12, %xmm15, %xmm15
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm12, %xmm28
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm15[0,0,1,1]
+; AVX512-FCP-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    vpbroadcastd 32(%rax), %ymm15
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm15, %zmm10, %zmm10
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm10 = zmm10 ^ (zmm31 & (zmm10 ^ zmm1))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm16 & (zmm10 ^ zmm1))
 ; AVX512-FCP-NEXT:    vpbroadcastd 100(%rax), %ymm1
 ; AVX512-FCP-NEXT:    vpbroadcastd 104(%rax), %ymm15
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm15, %zmm1, %zmm15
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm31 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm15 = zmm15 ^ (zmm31 & (zmm15 ^ zmm4))
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm25 & (zmm10 ^ zmm3))
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = zmm15 ^ (zmm16 & (zmm15 ^ zmm7))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm27 & (zmm10 ^ zmm3))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0]
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = zmm15 ^ (zmm1 & (zmm15 ^ zmm13))
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]
-; AVX512-FCP-NEXT:    vprold $16, %xmm11, %xmm4
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm11 = xmm12[1,1,2,3]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm11[0,1],xmm4[2],xmm11[3,4],xmm4[5],xmm11[6,7]
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm25 = [4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm25, %xmm11
-; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm3, %xmm3
-; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm24, %zmm4
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm16, %xmm3
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm28, %xmm11
-; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm11, %xmm3
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm29, %xmm12
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]
-; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm12 = xmm29[1,1,2,2]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm12[0],xmm3[1],xmm12[2,3],xmm3[4],xmm12[5,6],xmm3[7]
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm11[0,2,3,3,4,5,6,7]
-; AVX512-FCP-NEXT:    vpermt2q %zmm11, %zmm22, %zmm3
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm4 ^ (zmm23 & (zmm3 ^ zmm4))
-; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm27, %xmm5
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm9[4],xmm11[4],xmm9[5],xmm11[5],xmm9[6],xmm11[6],xmm9[7],xmm11[7]
+; AVX512-FCP-NEXT:    vprold $16, %xmm9, %xmm7
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm11[1,1,2,3]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm9[0,1],xmm7[2],xmm9[3,4],xmm7[5],xmm9[6,7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm31, %xmm9
+; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
+; AVX512-FCP-NEXT:    vpermt2q %zmm3, %zmm30, %zmm7
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm21, %xmm3
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm29, %xmm9
+; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm9, %xmm3
+; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm11[4],xmm9[4],xmm11[5],xmm9[5],xmm11[6],xmm9[6],xmm11[7],xmm9[7]
+; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[1,1,2,2]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm11[0],xmm3[1],xmm11[2,3],xmm3[4],xmm11[5,6],xmm3[7]
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[0,2,3,3,4,5,6,7]
+; AVX512-FCP-NEXT:    vpermt2q %zmm9, %zmm23, %zmm3
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm7 ^ (zmm24 & (zmm3 ^ zmm7))
+; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm28, %xmm5
 ; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm17, %xmm5
-; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm8, %xmm5
-; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm21, %zmm5
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm22, %xmm5
+; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm6, %xmm5
+; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm19, %zmm5
 ; AVX512-FCP-NEXT:    vpbroadcastd 36(%rax), %ymm4
 ; AVX512-FCP-NEXT:    vpbroadcastd 40(%rax), %ymm6
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm4
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm31 & (zmm4 ^ zmm5))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm16 & (zmm4 ^ zmm5))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm1 & (zmm4 ^ zmm3))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
 ; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} ymm2 = mem ^ (ymm1 & (ymm2 ^ mem))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm0 ^ (ymm1 & (ymm0 ^ ymm2))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm7 = ymm7 ^ (ymm2 & (ymm7 ^ ymm14))
-; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm7[0,1,2,3],zmm0[0,1,2,3]
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm8 ^ (ymm2 & (ymm8 ^ ymm17))
+; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm8[0,1,2,3],zmm0[0,1,2,3]
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm13 = zmm13 ^ (mem & (zmm13 ^ zmm0))
-; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm19, %zmm0 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (mem & (zmm13 ^ zmm0))
+; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm14, %zmm0 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm0 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    # zmm0 = (zmm0 & zmm2) | mem
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm2 # 32-byte Folded Reload
+; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm2 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm2 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    # zmm2 = (zmm2 & zmm3) | mem
-; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; AVX512-FCP-NEXT:    vmovdqa (%rsp), %xmm3 # 16-byte Reload
 ; AVX512-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload
 ; AVX512-FCP-NEXT:    # xmm3 = xmm3[4],mem[4],xmm3[5],mem[5],xmm3[6],mem[6],xmm3[7],mem[7]
 ; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
@@ -13791,7 +13792,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
 ; AVX512-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm6 # 16-byte Folded Reload
 ; AVX512-FCP-NEXT:    # xmm6 = xmm6[4],mem[4],xmm6[5],mem[5],xmm6[6],mem[6],xmm6[7],mem[7]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm25, %xmm7
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm31, %xmm7
 ; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm3, %xmm3
 ; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm6, %xmm6
 ; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
@@ -13804,14 +13805,14 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    # ymm9 = mem[1,1,1,1,5,5,5,5]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm8 = ymm8[0,1],ymm9[2],ymm8[3,4],ymm9[5],ymm8[6,7,8,9],ymm9[10],ymm8[11,12],ymm9[13],ymm8[14,15]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm9 = [0,1,4,5,4,5,5,7]
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm9 = [0,1,4,5,4,5,5,7,4,5,2,2,6,6,6,6]
 ; AVX512-FCP-NEXT:    vpermd %ymm11, %ymm9, %ymm9
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[16,17],zero,zero
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
 ; AVX512-FCP-NEXT:    vpandn %ymm9, %ymm12, %ymm9
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm11, %zmm9
-; AVX512-FCP-NEXT:    vpshuflw $248, (%rsp), %xmm11 # 16-byte Folded Reload
+; AVX512-FCP-NEXT:    vpshuflw $248, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
 ; AVX512-FCP-NEXT:    # xmm11 = mem[0,2,3,3,4,5,6,7]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,0,2,1]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,1,3]
@@ -13825,10 +13826,10 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (mem & (zmm14 ^ zmm12))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm11 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm11 ^ (zmm23 & (zmm3 ^ zmm11))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm11 ^ (zmm24 & (zmm3 ^ zmm11))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 32-byte Folded Reload
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm5 ^ (zmm23 & (zmm6 ^ zmm5))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm5 ^ (zmm24 & (zmm6 ^ zmm5))
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535]
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm0 ^ (zmm5 & (zmm3 ^ zmm0))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm2 ^ (zmm5 & (zmm6 ^ zmm2))
@@ -13849,7 +13850,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm10, 192(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm20, 128(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm26, (%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm30, 448(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm25, 448(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm15, 704(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm18, 640(%rax)
 ; AVX512-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
@@ -13859,7 +13860,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, 512(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm13, 832(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 768(%rax)
-; AVX512-FCP-NEXT:    addq $1496, %rsp # imm = 0x5D8
+; AVX512-FCP-NEXT:    addq $1512, %rsp # imm = 0x5E8
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
@@ -14246,7 +14247,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpbroadcastd 100(%rax), %ymm1
 ; AVX512DQ-NEXT:    vpbroadcastd 104(%rax), %ymm2
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm31
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm31 = zmm31 ^ (mem & (zmm31 ^ zmm0))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm31 = zmm31 ^ (mem & (zmm31 ^ zmm0))
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3]
 ; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm9, %xmm1
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm7[1,1,2,2]
@@ -14275,7 +14276,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpbroadcastd 68(%rax), %ymm5
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm25
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm25 = zmm25 ^ (zmm19 & (zmm25 ^ zmm1))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm25 = zmm25 ^ (zmm19 & (zmm25 ^ zmm1))
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
 ; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm8, %xmm3
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm5 = xmm13[1,1,2,2]
@@ -14302,7 +14303,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpbroadcastd (%rax), %ymm1
 ; AVX512DQ-NEXT:    vpbroadcastd 4(%rax), %ymm2
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm20
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm20 = zmm20 ^ (zmm19 & (zmm20 ^ zmm0))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 ^ (zmm19 & (zmm20 ^ zmm0))
 ; AVX512DQ-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm6[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,0,2,1,4,4,6,5]
@@ -14374,7 +14375,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpbroadcastd 36(%rax), %ymm2
 ; AVX512DQ-NEXT:    vpbroadcastd 40(%rax), %ymm4
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm13
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm13 = zmm13 ^ (mem & (zmm13 ^ zmm1))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (mem & (zmm13 ^ zmm1))
 ; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm7
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm6
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
@@ -14436,11 +14437,11 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm30[0,1,2,3],zmm2[0,1,2,3]
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm30 = zmm30 ^ (zmm2 & (zmm30 ^ zmm1))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm30 = zmm30 ^ (zmm2 & (zmm30 ^ zmm1))
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm11 = zmm11 ^ (zmm2 & (zmm11 ^ zmm1))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (zmm2 & (zmm11 ^ zmm1))
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535]
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm25 = zmm25 ^ (zmm1 & (zmm25 ^ zmm14))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 ^ (zmm1 & (zmm20 ^ zmm5))
@@ -14460,10 +14461,10 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm14 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
 ; AVX512DQ-NEXT:    vpermd 64(%rax), %zmm14, %zmm5
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (zmm3 & (zmm5 ^ zmm1))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm3 & (zmm5 ^ zmm1))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm24, %zmm26, %zmm1
 ; AVX512DQ-NEXT:    vpermd (%rax), %zmm14, %zmm14
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm14 = zmm14 ^ (zmm3 & (zmm14 ^ zmm1))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (zmm3 & (zmm14 ^ zmm1))
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0]
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm1 & (zmm5 ^ zmm2))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (zmm1 & (zmm14 ^ zmm22))
@@ -14526,13 +14527,13 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm10[2,1,3,2]
 ; AVX512DQ-NEXT:    vpbroadcastd 96(%rax), %ymm10
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm9 = zmm9 ^ (zmm29 & (zmm9 ^ zmm7))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 ^ (zmm29 & (zmm9 ^ zmm7))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,0,1]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm24, %zmm3
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm26[2,1,3,2]
 ; AVX512DQ-NEXT:    vpbroadcastd 32(%rax), %ymm10
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm7, %zmm7
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (zmm29 & (zmm7 ^ zmm3))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm29 & (zmm7 ^ zmm3))
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535]
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 ^ (zmm3 & (zmm9 ^ zmm22))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (zmm3 & (zmm7 ^ zmm8))
@@ -14568,7 +14569,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX512DQ-FCP-LABEL: store_i16_stride7_vf64:
 ; AVX512DQ-FCP:       # %bb.0:
-; AVX512DQ-FCP-NEXT:    subq $1496, %rsp # imm = 0x5D8
+; AVX512DQ-FCP-NEXT:    subq $1512, %rsp # imm = 0x5E8
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rcx), %ymm4
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm4, %ymm0
@@ -14586,27 +14587,27 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r9), %ymm1
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm1, %ymm23
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm1, %ymm28
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r8), %ymm13
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm12 = [u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm13, %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm13, %ymm25
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm13, %ymm27
 ; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rcx), %ymm0
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm0, %ymm13
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm26
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm30
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdx), %ymm0
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm0, %ymm14
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm30
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm23
 ; AVX512DQ-FCP-NEXT:    vpor %ymm13, %ymm14, %ymm13
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rsi), %ymm0
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm0, %ymm13
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm22
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm29
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %ymm0
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm0, %ymm14
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm27
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm0, %ymm17
 ; AVX512DQ-FCP-NEXT:    vpor %ymm13, %ymm14, %ymm13
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %ymm0
@@ -14649,7 +14650,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm8, %ymm10
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %ymm9
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm9, %ymm11
-; AVX512DQ-FCP-NEXT:    vporq %ymm11, %ymm10, %ymm17
+; AVX512DQ-FCP-NEXT:    vporq %ymm11, %ymm10, %ymm19
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm4[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm12 = ymm5[0,1,1,3,4,5,5,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm12[0,1],ymm10[2],ymm12[3,4],ymm10[5],ymm12[6,7,8,9],ymm10[10],ymm12[11,12],ymm10[13],ymm12[14,15]
@@ -14657,17 +14658,17 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
 ; AVX512DQ-FCP-NEXT:    # ymm13 = mem[0,1,0,1]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm2, %ymm12
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm13, %ymm21
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm13, %ymm26
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm13 = ymm3[1,1,1,1,5,5,5,5]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm12 = ymm12[0,1],ymm13[2],ymm12[3,4],ymm13[5],ymm12[6,7,8,9],ymm13[10],ymm12[11,12],ymm13[13],ymm12[14,15]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,2,2,3]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm16, %zmm10
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm7, %zmm12
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm10 ^ (mem & (zmm12 ^ zmm10))
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r8), %ymm7
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm7[u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm7[14,15,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,ymm7[16,17,u,u,u,u],zero,zero
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm10 | (ymm12 & ymm18)
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm10 = ymm10 | (ymm12 & ymm13)
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r9), %ymm11
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm11, %ymm6
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm11, %ymm16
@@ -14679,7 +14680,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm7, %ymm12, %ymm12
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0]
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm12 = ymm12 ^ (ymm11 & (ymm12 ^ ymm10))
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm11, %zmm20
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm11, %zmm21
 ; AVX512DQ-FCP-NEXT:    vprold $16, %ymm16, %ymm10
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,2,2]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
@@ -14687,10 +14688,12 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm6[0,1,2,3],zmm10[0,1,2,3]
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [0,1,4,5,4,5,5,7]
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm12 = [0,1,4,5,4,5,5,7,4,5,2,2,6,6,6,6]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rax), %ymm6
-; AVX512DQ-FCP-NEXT:    vpermd %ymm6, %ymm10, %ymm10
-; AVX512DQ-FCP-NEXT:    vpandnq %ymm10, %ymm18, %ymm10
+; AVX512DQ-FCP-NEXT:    vpermd %ymm6, %ymm12, %ymm10
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm12, %zmm16
+; AVX512DQ-FCP-NEXT:    vpandn %ymm10, %ymm13, %ymm10
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm13, %zmm25
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm6, %ymm12
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm12, %zmm10
@@ -14701,30 +14704,31 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm12[0,1,2],ymm10[3],ymm12[4,5],ymm10[6],ymm12[7,8,9,10],ymm10[11],ymm12[12,13],ymm10[14],ymm12[15]
 ; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm12 = [151522058,0,421010202,421010202]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm2, %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm12, %ymm19
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm12, %ymm22
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[2,2,2,2,6,6,6,6]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6,7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm3 = [0,2,2,3,10,9,11,11]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm10, %zmm3, %zmm2
 ; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm12 = [218894094,0,488382238,488382238]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm4, %ymm10
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm12, %ymm16
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm12, %ymm20
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm12 = ymm5[3,3,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm12[0],ymm10[1],ymm12[2,3],ymm10[4],ymm12[5,6,7,8],ymm10[9],ymm12[10,11],ymm10[12],ymm12[13,14,15]
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
-; AVX512DQ-FCP-NEXT:    # ymm12 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm4, %ymm4
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm12 = ymm12[0],ymm10[1],ymm12[2,3],ymm10[4],ymm12[5,6,7,8],ymm10[9],ymm12[10,11],ymm10[12],ymm12[13,14,15]
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
+; AVX512DQ-FCP-NEXT:    # ymm10 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm4, %ymm4
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm10, %ymm24
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[2,2,2,2,6,6,6,6]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1],ymm5[2],ymm4[3,4],ymm5[5],ymm4[6,7,8,9],ymm5[10],ymm4[11,12],ymm5[13],ymm4[14,15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm24 = [2,2,2,3,8,10,10,11]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm10, %zmm24, %zmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm31 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm2 ^ (zmm31 & (zmm4 ^ zmm2))
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [2,2,2,3,8,10,10,11]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm12, %zmm18, %zmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm2 ^ (zmm12 & (zmm4 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm2 = [5,0,0,0,6,0,0,6]
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm7, %ymm2, %ymm2
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [0,5,0,0,0,6,0,0,6,0,0,0,7,0,0,7]
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [4,5,2,2,6,6,6,6,6,7,3,3,7,7,6,7]
 ; AVX512DQ-FCP-NEXT:    vpermd %zmm6, %zmm2, %zmm4
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
@@ -14747,29 +14751,30 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm8[0,0,2,1,4,4,6,5]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1,2],ymm5[3],ymm4[4,5],ymm5[6],ymm4[7,8,9,10],ymm5[11],ymm4[12,13],ymm5[14],ymm4[15]
 ; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [0,1,0,3,10,10,11,11]
-; AVX512DQ-FCP-NEXT:    vpermi2q %zmm4, %zmm17, %zmm5
+; AVX512DQ-FCP-NEXT:    vpermi2q %zmm4, %zmm19, %zmm5
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm0, %ymm4
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm1[3,3,3,3,7,7,7,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3],ymm5[4,5],ymm4[6],ymm5[7,8,9,10],ymm4[11],ymm5[12,13],ymm4[14],ymm5[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm19, %ymm10
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm22, %ymm10
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm0, %ymm5
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm1[2,2,2,2,6,6,6,6]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3],ymm5[4],ymm6[5,6,7,8],ymm5[9],ymm6[10,11],ymm5[12],ymm6[13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm3, %zmm5
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm16, %ymm7
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm20, %ymm7
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm14, %ymm3
 ; AVX512DQ-FCP-NEXT:    vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm15[3,3,3,3,7,7,7,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6,7,8],ymm3[9],ymm4[10,11],ymm3[12],ymm4[13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm14, %ymm4
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm24, %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm14, %ymm4
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm15[2,2,2,2,6,6,6,6]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0,1],ymm6[2],ymm4[3,4],ymm6[5],ymm4[6,7,8,9],ymm6[10],ymm4[11,12],ymm6[13],ymm4[14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm24, %zmm0
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm5 ^ (zmm31 & (zmm0 ^ zmm5))
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm18, %zmm0
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm5 ^ (zmm12 & (zmm0 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm8[3,3,3,3,7,7,7,7]
@@ -14777,107 +14782,105 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vprold $16, %ymm9, %ymm4
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm8[1,2,2,3,5,6,6,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7,8,9],ymm4[10],ymm5[11,12],ymm4[13],ymm5[14,15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm14 = [2,1,3,2,10,10,10,11]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm14, %zmm4
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm22 = [2,1,3,2,10,10,10,11]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm22, %zmm4
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm2, %zmm0
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm20 & (zmm0 ^ zmm4))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm21 & (zmm0 ^ zmm4))
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm22, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm0, %ymm2
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm27[2,2,2,2,6,6,6,6]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm29, %ymm15
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm15, %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm10, %ymm19
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm17[2,2,2,2,6,6,6,6]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6,7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13,14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm21, %ymm13
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm0, %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm22, %ymm21
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm27[1,1,1,1,5,5,5,5]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm26, %ymm9
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm15, %ymm3
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm17[1,1,1,1,5,5,5,5]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7,8,9],ymm4[10],ymm3[11,12],ymm4[13],ymm3[14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm24, %zmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm26, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm0, %ymm2
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm30[2,2,2,2,6,6,6,6]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm18, %zmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm30, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm0, %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm24, %ymm14
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm23[2,2,2,2,6,6,6,6]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm0[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm30[0,1,1,3,4,5,5,7]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm23[0,1,1,3,4,5,5,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7,8,9],ymm4[10],ymm5[11,12],ymm4[13],ymm5[14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm14, %zmm6
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm22 = [0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm3 ^ (zmm22 & (zmm6 ^ zmm3))
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm23, %ymm10
-; AVX512DQ-FCP-NEXT:    vprold $16, %ymm23, %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm25, %ymm20
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm25[1,2,2,3,5,6,6,7]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm22, %zmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm3 ^ (zmm30 & (zmm6 ^ zmm3))
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm28, %ymm10
+; AVX512DQ-FCP-NEXT:    vprold $16, %ymm28, %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm27[1,2,2,3,5,6,6,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1],ymm2[2],ymm3[3,4],ymm2[5],ymm3[6,7,8,9],ymm2[10],ymm3[11,12],ymm2[13],ymm3[14,15]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm10, %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm23, %ymm19
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm25[0,0,2,1,4,4,6,5]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7,8,9,10],ymm4[11],ymm3[12,13],ymm4[14],ymm3[15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm1 = [2,2,3,3,10,9,11,10]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm1, %zmm3
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
-; AVX512DQ-FCP-NEXT:    vpermd 64(%rax), %zmm1, %zmm1
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm18 & (zmm1 ^ zmm3))
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm27[0,0,2,1,4,4,6,5]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3],ymm3[4,5],ymm5[6],ymm3[7,8,9,10],ymm5[11],ymm3[12,13],ymm5[14],ymm3[15]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm21 = [2,2,3,3,10,9,11,10]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm21, %zmm3
+; AVX512DQ-FCP-NEXT:    vpermd 64(%rax), %zmm16, %zmm1
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm25 & (zmm1 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm6))
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm0, %ymm1
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm0 = ymm30[3,3,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm9 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6,7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm0 = ymm23[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm12 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6,7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13,14,15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rcx), %xmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdx), %xmm3
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm10 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm0, %xmm2
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[1,1,2,2]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm6[0],xmm2[1],xmm6[2,3],xmm2[4],xmm6[5,6],xmm2[7]
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, (%rsp) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,1,2,3,0,1,2,3,6,7,4,5,6,7,4,5]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm0, %xmm0
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm17 = [0,1,1,3,8,8,9,9]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm17, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rsi), %xmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm11 = [0,1,2,3,0,1,2,3,6,7,4,5,6,7,4,5]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm0, %xmm0
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm24 = [0,1,1,3,8,8,9,9]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm24, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %xmm6
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vprold $16, %xmm6, %xmm2
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm3 = xmm4[1,1,2,3]
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rsi), %xmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm7, (%rsp) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vprold $16, %xmm7, %xmm2
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm3 = xmm6[1,1,2,3]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2],xmm3[3,4],xmm2[5],xmm3[6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [0,0,1,1,8,8,10,9]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm18, %zmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm3, %xmm3
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm26 = [0,0,1,1,8,8,10,9]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm26, %zmm3
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0]
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm0 ^ (zmm2 & (zmm3 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r9), %xmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r8), %xmm6
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm11 = [0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm0, %xmm6
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm0, %xmm0
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm7 = [0,0,0,1,8,9,9,11]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm6, %zmm7, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm7 = [0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm0, %xmm6
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm0, %xmm0
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm16 = [0,0,0,1,8,9,9,11]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm6, %zmm16, %zmm0
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 64(%rax), %ymm6
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 68(%rax), %ymm8
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm6, %zmm30
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm30 = zmm30 ^ (zmm12 & (zmm30 ^ zmm0))
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm6, %zmm25
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm25 = zmm25 ^ (zmm13 & (zmm25 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm30 = zmm30 ^ (zmm0 & (zmm30 ^ zmm3))
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm8
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm25 = zmm25 ^ (zmm0 & (zmm25 ^ zmm3))
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm8
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm4, %xmm3
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm8[1,1,2,2]
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm8, %xmm3
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[1,1,2,2]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm6[0],xmm3[1],xmm6[2,3],xmm3[4],xmm6[5,6],xmm3[7]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm6, %xmm6
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm17, %zmm6
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm4[0],xmm8[0],xmm4[1],xmm8[1],xmm4[2],xmm8[2],xmm4[3],xmm8[3]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm6, %xmm6
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm24, %zmm6
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm4
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm10
@@ -14886,232 +14889,231 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm4[1,1,2,3]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm4[0],xmm10[0],xmm4[1],xmm10[1],xmm4[2],xmm10[2],xmm4[3],xmm10[3]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm8, %xmm8
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, %xmm10
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm18, %zmm8
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm8, %xmm8
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm26, %zmm8
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm6 ^ (zmm2 & (zmm8 ^ zmm6))
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm3
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm2, %xmm3
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm2, %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm15, %xmm17
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm7, %zmm2
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm2, %xmm3
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm2, %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm7, %xmm24
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm16, %zmm2
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd (%rax), %ymm3
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 4(%rax), %ymm6
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm3, %zmm26
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm26 = zmm26 ^ (zmm12 & (zmm26 ^ zmm2))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm26 = zmm26 ^ (zmm13 & (zmm26 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm26 = zmm26 ^ (zmm0 & (zmm26 ^ zmm8))
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm15[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm11[2,2,2,2,6,6,6,6]
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm19, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm8, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm19 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm19[2,2,2,2,6,6,6,6]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6,7,8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm15, %ymm2
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm11[1,1,1,1,5,5,5,5]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm8, %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm19[1,1,1,1,5,5,5,5]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7,8,9],ymm3[10],ymm2[11,12],ymm3[13],ymm2[14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm24, %zmm2
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,u,u,24,25,26,27,u,u,26,27,26,27]
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm7[2,2,2,2,6,6,6,6]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm18, %zmm2
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm9, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm16 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm16[2,2,2,2,6,6,6,6]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm0[0,1],ymm3[2],ymm0[3,4],ymm3[5],ymm0[6,7,8,9],ymm3[10],ymm0[11,12],ymm3[13],ymm0[14,15]
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm4[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm7[0,1,1,3,4,5,5,7]
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm9[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm16[0,1,1,3,4,5,5,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm6[0,1],ymm0[2],ymm6[3,4],ymm0[5],ymm6[6,7,8,9],ymm0[10],ymm6[11,12],ymm0[13],ymm6[14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm14, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rcx), %xmm13
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdx), %xmm14
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm5, %xmm25
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm24 = [0,2,2,3,8,9,9,11]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm9, %zmm1
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm24, %zmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm12 = [u,u,u,u,u,u,u,u,u,u,u,u,14,15,12,13,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm21, %ymm3
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm3, %ymm3
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm27[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm22, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rcx), %xmm14
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdx), %xmm13
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm3, %xmm3
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm22 = [0,2,2,3,8,9,9,11]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm22, %zmm12
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [u,u,u,u,u,u,u,u,u,u,u,u,14,15,12,13,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm29, %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm1, %ymm3
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm6 = ymm17[3,3,3,3,7,7,7,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1,2],ymm3[3],ymm6[4,5],ymm3[6],ymm6[7,8,9,10],ymm3[11],ymm6[12,13],ymm3[14],ymm6[15]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm2 ^ (zmm22 & (zmm0 ^ zmm2))
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vprold $16, %ymm5, %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm23 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm23[1,2,2,3,5,6,6,7]
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm2 ^ (zmm30 & (zmm0 ^ zmm2))
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vprold $16, %ymm4, %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm3 = ymm17[1,2,2,3,5,6,6,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1],ymm2[2],ymm3[3,4],ymm2[5],ymm3[6,7,8,9],ymm2[10],ymm3[11,12],ymm2[13],ymm3[14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm5, %ymm21
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm23[0,0,2,1,4,4,6,5]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3],ymm3[4,5],ymm5[6],ymm3[7,8,9,10],ymm5[11],ymm3[12,13],ymm5[14],ymm3[15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm5 = [2,2,3,3,10,9,11,10]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm5, %zmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %xmm5
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm9
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm2, %xmm2
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm18 = [2,1,3,3,8,8,9,9]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm18, %zmm6
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm1 ^ (zmm31 & (zmm6 ^ zmm1))
-; AVX512DQ-FCP-NEXT:    vbroadcasti32x4 {{.*#+}} ymm16 = [26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
-; AVX512DQ-FCP-NEXT:    # ymm16 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm16, %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm19, %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm4, %ymm31
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm17[0,0,2,1,4,4,6,5]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7,8,9,10],ymm4[11],ymm3[12,13],ymm4[14],ymm3[15]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm21, %zmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %xmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm5
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm15, %xmm7
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm2, %xmm2
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm15 = [2,1,3,3,8,8,9,9]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm15, %zmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm12 ^ (zmm18 & (zmm6 ^ zmm12))
+; AVX512DQ-FCP-NEXT:    vbroadcasti32x4 {{.*#+}} ymm21 = [26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
+; AVX512DQ-FCP-NEXT:    # ymm21 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm21, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm28, %ymm2
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm20[3,3,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm8 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,4,0,0,0,5,0,0,5,0,0,0,6,0,0]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm27[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm12 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,1,4,5,4,5,5,7,4,5,2,2,6,6,6,6]
 ; AVX512DQ-FCP-NEXT:    vpermd (%rax), %zmm1, %zmm20
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm20 = zmm20 ^ (mem & (zmm20 ^ zmm3))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 ^ (mem & (zmm20 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm20 = zmm20 ^ (mem & (zmm20 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm2
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm29
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm1, %xmm28
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm25, %xmm1
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm0, %xmm3
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm4[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm7[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm1, %xmm29
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm0, %xmm3
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm9[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm16[3,3,3,3,7,7,7,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6,7,8],ymm1[9],ymm2[10,11],ymm1[12],ymm2[13,14,15]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm24, %zmm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm15, %ymm1
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm11[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm22, %zmm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm8, %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm2 = ymm19[3,3,3,3,7,7,7,7]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm2[0,1,2],ymm1[3],ymm2[4,5],ymm1[6],ymm2[7,8,9,10],ymm1[11],ymm2[12,13],ymm1[14],ymm2[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm12
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm11
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm2, %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm10, %xmm19
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm18, %zmm3
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm24 = [2,2,2,3,8,8,8,9]
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r9), %xmm7
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r8), %xmm4
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm4[0],xmm7[0],xmm4[1],xmm7[1],xmm4[2],xmm7[2],xmm4[3],xmm7[3]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm17, %xmm1
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm10, %xmm15
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm15, %zmm24, %zmm8
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm0 ^ (zmm31 & (zmm3 ^ zmm0))
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm15 = [6,7,3,3,7,7,6,7]
-; AVX512DQ-FCP-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm0 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm11
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm9
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm2, %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm7, %xmm22
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm15, %zmm3
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm30 = [2,2,2,3,8,8,8,9]
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r9), %xmm8
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r8), %xmm7
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm24, %xmm1
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm15, %xmm10
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm10, %zmm30, %zmm12
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm0 ^ (zmm18 & (zmm3 ^ zmm0))
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [6,7,3,3,7,7,6,7]
+; AVX512DQ-FCP-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm0 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 96(%rax), %ymm18
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm18, %zmm0, %zmm18
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm31 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm18 = zmm18 ^ (zmm31 & (zmm18 ^ zmm8))
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm25 = [0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm18 = zmm18 ^ (zmm25 & (zmm18 ^ zmm6))
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm5[4],xmm9[4],xmm5[5],xmm9[5],xmm5[6],xmm9[6],xmm5[7],xmm9[7]
-; AVX512DQ-FCP-NEXT:    vprold $16, %xmm5, %xmm0
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm9[1,1,2,3]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm5[0,1],xmm0[2],xmm5[3,4],xmm0[5],xmm5[6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm16, %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm21, %ymm5
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm5 = ymm23[3,3,3,3,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm5[2],ymm1[3,4],ymm5[5],ymm1[6,7,8,9],ymm5[10],ymm1[11,12],ymm5[13],ymm1[14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm6
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm5
-; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm17, %xmm9
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm8, %xmm9
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm9, %zmm24, %zmm1
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm24 = [0,0,2,1,8,8,9,11]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm24, %zmm0
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
-; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm16 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm16, %xmm9
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm13, %xmm13
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm14 = xmm14[1,1,2,2]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm13 = xmm14[0],xmm13[1],xmm14[2,3],xmm13[4],xmm14[5,6],xmm13[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm18 = zmm18 ^ (zmm16 & (zmm18 ^ zmm12))
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm18 = zmm18 ^ (zmm27 & (zmm18 ^ zmm6))
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
+; AVX512DQ-FCP-NEXT:    vprold $16, %xmm4, %xmm0
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm5[1,1,2,3]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm4[0,1],xmm0[2],xmm4[3,4],xmm0[5],xmm4[6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm21, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm31, %ymm4
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm4, %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} ymm4 = ymm17[3,3,3,3,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm4[2],ymm1[3,4],ymm4[5],ymm1[6,7,8,9],ymm4[10],ymm1[11,12],ymm4[13],ymm1[14,15]
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm4
+; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm24, %xmm12
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm6, %xmm12
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm12, %zmm30, %zmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm12 = [4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm2, %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm12, %xmm31
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm30 = [0,0,2,1,8,8,9,11]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm30, %zmm0
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm13[4],xmm14[4],xmm13[5],xmm14[5],xmm13[6],xmm14[6],xmm13[7],xmm14[7]
+; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm21 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm21, %xmm12
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm14, %xmm14
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[1,1,2,2]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm13 = xmm13[0],xmm14[1],xmm13[2,3],xmm14[4],xmm13[5,6],xmm14[7]
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,2,3,3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm22 = [0,0,1,1,8,8,10,9]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm22, %zmm13
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm0 ^ (zmm23 & (zmm13 ^ zmm0))
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm4[4],xmm7[4],xmm4[5],xmm7[5],xmm4[6],xmm7[6],xmm4[7],xmm7[7]
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm23 = [0,0,1,1,8,8,10,9]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm23, %zmm13
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm0 ^ (zmm24 & (zmm13 ^ zmm0))
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7]
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm14 = ymm14 ^ (ymm31 & (ymm14 ^ ymm2))
+; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm17 = ymm17 ^ (ymm16 & (ymm17 ^ ymm2))
 ; AVX512DQ-FCP-NEXT:    vextracti64x4 $1, %zmm2, %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm19, %xmm9
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm0, %xmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm17 = [0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm17, %xmm4
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm4, %xmm10, %xmm4
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm21 = [0,1,1,3,8,8,9,9]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm21, %zmm4
-; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm7[0,1,2,3,5,5,7,6,8,9,10,11,13,13,15,14]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm22, %xmm12
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm0, %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm22 = [0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm22, %xmm7
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm15, %xmm7
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm19 = [0,1,1,3,8,8,9,9]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm19, %zmm7
+; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm8[0,1,2,3,5,5,7,6,8,9,10,11,13,13,15,14]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[3,3,3,3]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25,22,23,22,23,u,u,u,u,u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,2,2]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm10, %xmm10
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm19 = ymm10[0,0,1,1]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm10, %xmm10
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm9, %xmm27
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm10[0,0,1,1]
-; AVX512DQ-FCP-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm10 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25,22,23,22,23,u,u,u,u,u,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,2,2]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm14, %xmm14
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,0,1,1]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm12, %xmm15, %xmm15
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm12, %xmm28
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm15[0,0,1,1]
+; AVX512DQ-FCP-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 32(%rax), %ymm15
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm15, %zmm10, %zmm10
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm10 = zmm10 ^ (zmm31 & (zmm10 ^ zmm1))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm16 & (zmm10 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 100(%rax), %ymm1
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 104(%rax), %ymm15
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm15, %zmm1, %zmm15
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm31 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm15 = zmm15 ^ (zmm31 & (zmm15 ^ zmm4))
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm25 & (zmm10 ^ zmm3))
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535]
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = zmm15 ^ (zmm16 & (zmm15 ^ zmm7))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm27 & (zmm10 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0]
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = zmm15 ^ (zmm1 & (zmm15 ^ zmm13))
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]
-; AVX512DQ-FCP-NEXT:    vprold $16, %xmm11, %xmm4
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm11 = xmm12[1,1,2,3]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm11[0,1],xmm4[2],xmm11[3,4],xmm4[5],xmm11[6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm25 = [4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm25, %xmm11
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm24, %zmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm16, %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm28, %xmm11
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm11, %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm29, %xmm12
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]
-; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm12 = xmm29[1,1,2,2]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm12[0],xmm3[1],xmm12[2,3],xmm3[4],xmm12[5,6],xmm3[7]
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm11[0,2,3,3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm11, %zmm22, %zmm3
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm4 ^ (zmm23 & (zmm3 ^ zmm4))
-; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm27, %xmm5
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm9[4],xmm11[4],xmm9[5],xmm11[5],xmm9[6],xmm11[6],xmm9[7],xmm11[7]
+; AVX512DQ-FCP-NEXT:    vprold $16, %xmm9, %xmm7
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm11[1,1,2,3]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm9[0,1],xmm7[2],xmm9[3,4],xmm7[5],xmm9[6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm31, %xmm9
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm3, %zmm30, %zmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm21, %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm29, %xmm9
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm9, %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm11[4],xmm9[4],xmm11[5],xmm9[5],xmm11[6],xmm9[6],xmm11[7],xmm9[7]
+; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[1,1,2,2]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm11[0],xmm3[1],xmm11[2,3],xmm3[4],xmm11[5,6],xmm3[7]
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[0,2,3,3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm9, %zmm23, %zmm3
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm7 ^ (zmm24 & (zmm3 ^ zmm7))
+; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm28, %xmm5
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm17, %xmm5
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm8, %xmm5
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm21, %zmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm22, %xmm5
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm6, %xmm5
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm19, %zmm5
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 36(%rax), %ymm4
 ; AVX512DQ-FCP-NEXT:    vpbroadcastd 40(%rax), %ymm6
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm4
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm31 & (zmm4 ^ zmm5))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm16 & (zmm4 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm1 & (zmm4 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535]
 ; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} ymm2 = mem ^ (ymm1 & (ymm2 ^ mem))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535]
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm0 ^ (ymm1 & (ymm0 ^ ymm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm7 = ymm7 ^ (ymm2 & (ymm7 ^ ymm14))
-; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm7[0,1,2,3],zmm0[0,1,2,3]
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm8 ^ (ymm2 & (ymm8 ^ ymm17))
+; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm8[0,1,2,3],zmm0[0,1,2,3]
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm13 = zmm13 ^ (mem & (zmm13 ^ zmm0))
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm19, %zmm0 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (mem & (zmm13 ^ zmm0))
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm14, %zmm0 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm0 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # zmm0 = (zmm0 & zmm2) | mem
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm2 # 32-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm2 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vpternlogq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm2 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # zmm2 = (zmm2 & zmm3) | mem
-; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsp), %xmm3 # 16-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # xmm3 = xmm3[4],mem[4],xmm3[5],mem[5],xmm3[6],mem[6],xmm3[7],mem[7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
@@ -15120,7 +15122,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm6 # 16-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # xmm6 = xmm6[4],mem[4],xmm6[5],mem[5],xmm6[6],mem[6],xmm6[7],mem[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm25, %xmm7
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm31, %xmm7
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm3, %xmm3
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm6, %xmm6
 ; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
@@ -15133,14 +15135,14 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpshufd $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # ymm9 = mem[1,1,1,1,5,5,5,5]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm8 = ymm8[0,1],ymm9[2],ymm8[3,4],ymm9[5],ymm8[6,7,8,9],ymm9[10],ymm8[11,12],ymm9[13],ymm8[14,15]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm9 = [0,1,4,5,4,5,5,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm9 = [0,1,4,5,4,5,5,7,4,5,2,2,6,6,6,6]
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm11, %ymm9, %ymm9
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[16,17],zero,zero
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535]
 ; AVX512DQ-FCP-NEXT:    vpandn %ymm9, %ymm12, %ymm9
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm11, %zmm9
-; AVX512DQ-FCP-NEXT:    vpshuflw $248, (%rsp), %xmm11 # 16-byte Folded Reload
+; AVX512DQ-FCP-NEXT:    vpshuflw $248, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # xmm11 = mem[0,2,3,3,4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,0,2,1]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,1,3]
@@ -15154,10 +15156,10 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (mem & (zmm14 ^ zmm12))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm11 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm11 ^ (zmm23 & (zmm3 ^ zmm11))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm11 ^ (zmm24 & (zmm3 ^ zmm11))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 32-byte Folded Reload
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm5 ^ (zmm23 & (zmm6 ^ zmm5))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm5 ^ (zmm24 & (zmm6 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535]
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm0 ^ (zmm5 & (zmm3 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm2 ^ (zmm5 & (zmm6 ^ zmm2))
@@ -15178,7 +15180,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm10, 192(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm20, 128(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm26, (%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm30, 448(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm25, 448(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm15, 704(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm18, 640(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
@@ -15188,7 +15190,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, 512(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm13, 832(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 768(%rax)
-; AVX512DQ-FCP-NEXT:    addq $1496, %rsp # imm = 0x5D8
+; AVX512DQ-FCP-NEXT:    addq $1512, %rsp # imm = 0x5E8
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll
index ba1621c67f480..7f2210742e7f1 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll
@@ -671,15 +671,16 @@ define void @store_i8_stride3_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
 ; AVX2-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
 ; AVX2-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
-; AVX2-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
 ; AVX2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
-; AVX2-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
 ; AVX2-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
+; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
+; AVX2-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vmovdqa %xmm0, 16(%rcx)
-; AVX2-NEXT:    vmovdqa %xmm1, (%rcx)
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vmovdqa %xmm2, 32(%rcx)
+; AVX2-NEXT:    vmovdqa %ymm0, (%rcx)
+; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX2-FP-LABEL: store_i8_stride3_vf16:
@@ -693,15 +694,16 @@ define void @store_i8_stride3_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2-FP-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
 ; AVX2-FP-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
 ; AVX2-FP-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
-; AVX2-FP-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
-; AVX2-FP-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
 ; AVX2-FP-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
-; AVX2-FP-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
 ; AVX2-FP-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
+; AVX2-FP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
+; AVX2-FP-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX2-FP-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
-; AVX2-FP-NEXT:    vmovdqa %xmm0, 16(%rcx)
-; AVX2-FP-NEXT:    vmovdqa %xmm1, (%rcx)
+; AVX2-FP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-FP-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
 ; AVX2-FP-NEXT:    vmovdqa %xmm2, 32(%rcx)
+; AVX2-FP-NEXT:    vmovdqa %ymm0, (%rcx)
+; AVX2-FP-NEXT:    vzeroupper
 ; AVX2-FP-NEXT:    retq
 ;
 ; AVX2-FCP-LABEL: store_i8_stride3_vf16:
@@ -715,15 +717,16 @@ define void @store_i8_stride3_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2-FCP-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
 ; AVX2-FCP-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
 ; AVX2-FCP-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
-; AVX2-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
-; AVX2-FCP-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
 ; AVX2-FCP-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
-; AVX2-FCP-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
 ; AVX2-FCP-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
+; AVX2-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
+; AVX2-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX2-FCP-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
-; AVX2-FCP-NEXT:    vmovdqa %xmm0, 16(%rcx)
-; AVX2-FCP-NEXT:    vmovdqa %xmm1, (%rcx)
+; AVX2-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-FCP-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
 ; AVX2-FCP-NEXT:    vmovdqa %xmm2, 32(%rcx)
+; AVX2-FCP-NEXT:    vmovdqa %ymm0, (%rcx)
+; AVX2-FCP-NEXT:    vzeroupper
 ; AVX2-FCP-NEXT:    retq
 ;
 ; AVX512-LABEL: store_i8_stride3_vf16:
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
index 2d5aa0977e4b6..efb068d23b92d 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
@@ -1334,7 +1334,7 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vporq %zmm3, %zmm4, %zmm3
 ; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [0,0,0,0,0,0,1,1,1,1,0,2,2,2,2,0]
 ; AVX512-NEXT:    vpermd %zmm0, %zmm4, %zmm4
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm3))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm3))
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,3,2,3]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u],zero,zero,xmm2[5,13,u],zero,zero,xmm2[6,14,u],zero,zero,xmm2[7,15,u]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,3,2,3]
@@ -1370,7 +1370,7 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vporq %zmm3, %zmm4, %zmm3
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [0,0,0,0,0,0,1,1,1,1,0,2,2,2,2,0]
 ; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm4, %zmm4
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm3))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm3))
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,3,2,3]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u],zero,zero,xmm2[5,13,u],zero,zero,xmm2[6,14,u],zero,zero,xmm2[7,15,u]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,3,2,3]
@@ -1409,7 +1409,7 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vporq %zmm3, %zmm4, %zmm3
 ; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [0,0,0,0,0,0,1,1,1,1,0,2,2,2,2,0]
 ; AVX512DQ-NEXT:    vpermd %zmm0, %zmm4, %zmm4
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm3))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm3))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,3,2,3]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u],zero,zero,xmm2[5,13,u],zero,zero,xmm2[6,14,u],zero,zero,xmm2[7,15,u]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,3,2,3]
@@ -1445,7 +1445,7 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vporq %zmm3, %zmm4, %zmm3
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [0,0,0,0,0,0,1,1,1,1,0,2,2,2,2,0]
 ; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm4, %zmm4
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm3))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,3,2,3]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u],zero,zero,xmm2[5,13,u],zero,zero,xmm2[6,14,u],zero,zero,xmm2[7,15,u]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,3,2,3]
@@ -2421,7 +2421,7 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vmovdqa (%r8), %xmm6
 ; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm8 = [0,0,0,0,0,0,1,1,1,1,0,2,2,2,2,0]
 ; AVX512-NEXT:    vpermd %zmm6, %zmm8, %zmm6
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm5 = ymm2[u,u,u],zero,ymm2[13,u,u,u],zero,ymm2[14,u,u,u],zero,ymm2[15,u,u,u],zero,ymm2[16,u,u,u],zero,ymm2[17,u,u,u],zero,ymm2[18,u,u]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm8 = ymm1[u,u,u,13],zero,ymm1[u,u,u,14],zero,ymm1[u,u,u,15],zero,ymm1[u,u,u,16],zero,ymm1[u,u,u,17],zero,ymm1[u,u,u,18],zero,ymm1[u,u]
 ; AVX512-NEXT:    vpor %ymm5, %ymm8, %ymm5
@@ -2442,8 +2442,7 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm8 = ymm0[12],zero,zero,zero,zero,ymm0[13],zero,zero,zero,zero,ymm0[14],zero,zero,zero,zero,ymm0[15],zero,zero,zero,zero,ymm0[16],zero,zero,zero,zero,ymm0[17],zero,zero,zero,zero,ymm0[18],zero
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm9 = ymm0[0,2,1,1,4,6,5,5]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,3,3,2]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm10 = [255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255]
-; AVX512-NEXT:    vpandn %ymm9, %ymm10, %ymm9
+; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm9, %ymm9
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm8, %zmm8
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 | (zmm5 & mem)
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm3[26],zero,ymm3[28],zero,zero,ymm3[27],zero,ymm3[29],zero,ymm3[31],zero,zero,ymm3[30],zero
@@ -2496,7 +2495,7 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm5[0,1,2,3],zmm9[0,1,2,3]
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [0,0,0,0,0,0,1,1,1,1,0,2,2,2,2,0]
 ; AVX512-FCP-NEXT:    vpermd %zmm0, %zmm6, %zmm6
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm2[u,u,u],zero,ymm2[13,u,u,u],zero,ymm2[14,u,u,u],zero,ymm2[15,u,u,u],zero,ymm2[16,u,u,u],zero,ymm2[17,u,u,u],zero,ymm2[18,u,u]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = ymm1[u,u,u,13],zero,ymm1[u,u,u,14],zero,ymm1[u,u,u,15],zero,ymm1[u,u,u,16],zero,ymm1[u,u,u,17],zero,ymm1[u,u,u,18],zero,ymm1[u,u]
 ; AVX512-FCP-NEXT:    vpor %ymm5, %ymm8, %ymm5
@@ -2514,12 +2513,11 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,3,3]
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm9 ^ (mem & (ymm9 ^ ymm5))
 ; AVX512-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm8[0,1,2,3],zmm9[0,1,2,3]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm8 = [4,0,5,5,5,5,0,6]
-; AVX512-FCP-NEXT:    vpermd %ymm0, %ymm8, %ymm8
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255]
-; AVX512-FCP-NEXT:    vpandn %ymm8, %ymm9, %ymm8
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm0[12],zero,zero,zero,zero,ymm0[13],zero,zero,zero,zero,ymm0[14],zero,zero,zero,zero,ymm0[15],zero,zero,zero,zero,ymm0[16],zero,zero,zero,zero,ymm0[17],zero,zero,zero,zero,ymm0[18],zero
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm9, %zmm8
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = ymm0[12],zero,zero,zero,zero,ymm0[13],zero,zero,zero,zero,ymm0[14],zero,zero,zero,zero,ymm0[15],zero,zero,zero,zero,ymm0[16],zero,zero,zero,zero,ymm0[17],zero,zero,zero,zero,ymm0[18],zero
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm9 = [4,0,5,5,5,5,0,6]
+; AVX512-FCP-NEXT:    vpermd %ymm0, %ymm9, %ymm9
+; AVX512-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm9, %ymm9
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm8, %zmm8
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 | (zmm5 & mem)
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm3[26],zero,ymm3[28],zero,zero,ymm3[27],zero,ymm3[29],zero,ymm3[31],zero,zero,ymm3[30],zero
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,u],zero,ymm4[26],zero,ymm4[28,u],zero,ymm4[u],zero,ymm4[29],zero,ymm4[31,u],zero,ymm4[30]
@@ -2572,7 +2570,7 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm6
 ; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm8 = [0,0,0,0,0,0,1,1,1,1,0,2,2,2,2,0]
 ; AVX512DQ-NEXT:    vpermd %zmm6, %zmm8, %zmm6
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm5 = ymm2[u,u,u],zero,ymm2[13,u,u,u],zero,ymm2[14,u,u,u],zero,ymm2[15,u,u,u],zero,ymm2[16,u,u,u],zero,ymm2[17,u,u,u],zero,ymm2[18,u,u]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm8 = ymm1[u,u,u,13],zero,ymm1[u,u,u,14],zero,ymm1[u,u,u,15],zero,ymm1[u,u,u,16],zero,ymm1[u,u,u,17],zero,ymm1[u,u,u,18],zero,ymm1[u,u]
 ; AVX512DQ-NEXT:    vpor %ymm5, %ymm8, %ymm5
@@ -2593,8 +2591,7 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm8 = ymm0[12],zero,zero,zero,zero,ymm0[13],zero,zero,zero,zero,ymm0[14],zero,zero,zero,zero,ymm0[15],zero,zero,zero,zero,ymm0[16],zero,zero,zero,zero,ymm0[17],zero,zero,zero,zero,ymm0[18],zero
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm9 = ymm0[0,2,1,1,4,6,5,5]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,3,3,2]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm10 = [255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255]
-; AVX512DQ-NEXT:    vpandn %ymm9, %ymm10, %ymm9
+; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm9, %ymm9
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm8, %zmm8
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 | (zmm5 & mem)
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm3[26],zero,ymm3[28],zero,zero,ymm3[27],zero,ymm3[29],zero,ymm3[31],zero,zero,ymm3[30],zero
@@ -2647,7 +2644,7 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm5[0,1,2,3],zmm9[0,1,2,3]
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm6 = [0,0,0,0,0,0,1,1,1,1,0,2,2,2,2,0]
 ; AVX512DQ-FCP-NEXT:    vpermd %zmm0, %zmm6, %zmm6
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm2[u,u,u],zero,ymm2[13,u,u,u],zero,ymm2[14,u,u,u],zero,ymm2[15,u,u,u],zero,ymm2[16,u,u,u],zero,ymm2[17,u,u,u],zero,ymm2[18,u,u]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = ymm1[u,u,u,13],zero,ymm1[u,u,u,14],zero,ymm1[u,u,u,15],zero,ymm1[u,u,u,16],zero,ymm1[u,u,u,17],zero,ymm1[u,u,u,18],zero,ymm1[u,u]
 ; AVX512DQ-FCP-NEXT:    vpor %ymm5, %ymm8, %ymm5
@@ -2665,12 +2662,11 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,3,3]
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} ymm9 = ymm9 ^ (mem & (ymm9 ^ ymm5))
 ; AVX512DQ-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm8[0,1,2,3],zmm9[0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm8 = [4,0,5,5,5,5,0,6]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm0, %ymm8, %ymm8
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255]
-; AVX512DQ-FCP-NEXT:    vpandn %ymm8, %ymm9, %ymm8
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm0[12],zero,zero,zero,zero,ymm0[13],zero,zero,zero,zero,ymm0[14],zero,zero,zero,zero,ymm0[15],zero,zero,zero,zero,ymm0[16],zero,zero,zero,zero,ymm0[17],zero,zero,zero,zero,ymm0[18],zero
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm9, %zmm8
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = ymm0[12],zero,zero,zero,zero,ymm0[13],zero,zero,zero,zero,ymm0[14],zero,zero,zero,zero,ymm0[15],zero,zero,zero,zero,ymm0[16],zero,zero,zero,zero,ymm0[17],zero,zero,zero,zero,ymm0[18],zero
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm9 = [4,0,5,5,5,5,0,6]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm0, %ymm9, %ymm9
+; AVX512DQ-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm9, %ymm9
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm8, %zmm8
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 | (zmm5 & mem)
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm3[26],zero,ymm3[28],zero,zero,ymm3[27],zero,ymm3[29],zero,ymm3[31],zero,zero,ymm3[30],zero
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,u],zero,ymm4[26],zero,ymm4[28,u],zero,ymm4[u],zero,ymm4[29],zero,ymm4[31,u],zero,ymm4[30]
@@ -4690,38 +4686,38 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ;
 ; AVX512-LABEL: store_i8_stride5_vf64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm11
+; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm10
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128,128,128]
-; AVX512-NEXT:    vpshufb %ymm1, %ymm11, %ymm0
+; AVX512-NEXT:    vpshufb %ymm1, %ymm10, %ymm0
 ; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm18
 ; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm5
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm2 = [12,13,128,15,12,13,14,128,12,13,14,15,128,u,u,u,16,128,18,19,16,17,128,19,16,17,18,128,16,17,18,19]
 ; AVX512-NEXT:    vpshufb %ymm2, %ymm5, %ymm1
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm19
-; AVX512-NEXT:    vporq %ymm0, %ymm1, %ymm20
-; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm12
+; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm21
+; AVX512-NEXT:    vporq %ymm0, %ymm1, %ymm19
+; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm14
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11]
-; AVX512-NEXT:    vpshufb %xmm1, %xmm12, %xmm0
+; AVX512-NEXT:    vpshufb %xmm1, %xmm14, %xmm0
 ; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm28
-; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm10
+; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm13
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [128,8,u,128,7,128,9,u,11,u,128,10,128,12,u,128]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm10, %xmm1
+; AVX512-NEXT:    vpshufb %xmm2, %xmm13, %xmm1
 ; AVX512-NEXT:    vmovdqa64 %xmm2, %xmm29
-; AVX512-NEXT:    vporq %xmm0, %xmm1, %xmm21
+; AVX512-NEXT:    vporq %xmm0, %xmm1, %xmm20
 ; AVX512-NEXT:    vmovdqa 32(%rcx), %ymm15
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm8 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128]
-; AVX512-NEXT:    vpshufb %ymm8, %ymm15, %ymm0
-; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm13
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm7 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128]
+; AVX512-NEXT:    vpshufb %ymm7, %ymm15, %ymm0
+; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm12
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm3 = [u,u,12,13,128,u,u,u,14,128,u,u,14,15,128,u,u,u,16,128,u,u,16,17,128,u,u,u,18,128,u,u]
-; AVX512-NEXT:    vpshufb %ymm3, %ymm13, %ymm1
+; AVX512-NEXT:    vpshufb %ymm3, %ymm12, %ymm1
 ; AVX512-NEXT:    vporq %ymm0, %ymm1, %ymm22
 ; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm6
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,6,128,8,u,128,7,128,9,128,11,u,128,10,128,12]
 ; AVX512-NEXT:    vpshufb %xmm1, %xmm6, %xmm0
 ; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm30
-; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm7
+; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm8
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm7, %xmm1
+; AVX512-NEXT:    vpshufb %xmm2, %xmm8, %xmm1
 ; AVX512-NEXT:    vmovdqa64 %xmm2, %xmm31
 ; AVX512-NEXT:    vporq %xmm0, %xmm1, %xmm23
 ; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128,128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128]
@@ -4733,133 +4729,132 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm24
 ; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30,27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30]
 ; AVX512-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm5, %ymm11, %ymm1
+; AVX512-NEXT:    vpshufb %ymm5, %ymm10, %ymm1
 ; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [19,128,21,0,128,20,128,22,128,24,0,128,23,128,25,0,19,128,21,0,128,20,128,22,128,24,0,128,23,128,25,0]
 ; AVX512-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm2, %ymm11, %ymm11
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm11, %zmm26
-; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128]
-; AVX512-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm11, %ymm13, %ymm1
+; AVX512-NEXT:    vpshufb %ymm2, %ymm10, %ymm10
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm10, %zmm26
+; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128]
+; AVX512-NEXT:    # ymm10 = mem[0,1,0,1]
+; AVX512-NEXT:    vpshufb %ymm10, %ymm12, %ymm1
 ; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128,128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128]
 ; AVX512-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm0, %ymm15, %ymm14
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm14, %zmm25
+; AVX512-NEXT:    vpshufb %ymm0, %ymm15, %ymm11
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm11, %zmm25
 ; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [25,128,27,0,128,26,128,28,128,30,0,128,29,128,31,0,25,128,27,0,128,26,128,28,128,30,0,128,29,128,31,0]
 ; AVX512-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm1, %ymm15, %ymm14
+; AVX512-NEXT:    vpshufb %ymm1, %ymm15, %ymm11
 ; AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm15 = [18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25]
 ; AVX512-NEXT:    # ymm15 = mem[0,1,0,1]
-; AVX512-NEXT:    vpshufb %ymm15, %ymm13, %ymm13
-; AVX512-NEXT:    vinserti64x4 $1, %ymm14, %zmm13, %zmm27
-; AVX512-NEXT:    vmovdqa (%rcx), %ymm13
-; AVX512-NEXT:    vpshufb %ymm8, %ymm13, %ymm8
-; AVX512-NEXT:    vmovdqa (%rdx), %ymm14
-; AVX512-NEXT:    vpshufb %ymm3, %ymm14, %ymm3
-; AVX512-NEXT:    vporq %ymm8, %ymm3, %ymm16
-; AVX512-NEXT:    vpshufb %ymm0, %ymm13, %ymm0
-; AVX512-NEXT:    vpshufb %ymm15, %ymm14, %ymm3
+; AVX512-NEXT:    vpshufb %ymm15, %ymm12, %ymm12
+; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm12, %zmm27
+; AVX512-NEXT:    vmovdqa (%rcx), %ymm12
+; AVX512-NEXT:    vpshufb %ymm7, %ymm12, %ymm7
+; AVX512-NEXT:    vmovdqa (%rdx), %ymm11
+; AVX512-NEXT:    vpshufb %ymm3, %ymm11, %ymm3
+; AVX512-NEXT:    vporq %ymm7, %ymm3, %ymm16
+; AVX512-NEXT:    vpshufb %ymm0, %ymm12, %ymm0
+; AVX512-NEXT:    vpshufb %ymm15, %ymm11, %ymm3
 ; AVX512-NEXT:    vporq %ymm0, %ymm3, %ymm17
 ; AVX512-NEXT:    vmovdqa (%rsi), %ymm3
 ; AVX512-NEXT:    vmovdqa64 %ymm18, %ymm0
 ; AVX512-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
-; AVX512-NEXT:    vmovdqa (%rdi), %ymm8
-; AVX512-NEXT:    vmovdqa64 %ymm19, %ymm15
-; AVX512-NEXT:    vpshufb %ymm15, %ymm8, %ymm15
+; AVX512-NEXT:    vmovdqa (%rdi), %ymm7
+; AVX512-NEXT:    vmovdqa64 %ymm21, %ymm15
+; AVX512-NEXT:    vpshufb %ymm15, %ymm7, %ymm15
 ; AVX512-NEXT:    vporq %ymm0, %ymm15, %ymm18
-; AVX512-NEXT:    vpshufb %ymm4, %ymm8, %ymm0
+; AVX512-NEXT:    vpshufb %ymm4, %ymm7, %ymm0
 ; AVX512-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
-; AVX512-NEXT:    vporq %ymm0, %ymm2, %ymm19
-; AVX512-NEXT:    vpshufb %ymm11, %ymm14, %ymm0
-; AVX512-NEXT:    vpshufb %ymm1, %ymm13, %ymm1
-; AVX512-NEXT:    vpor %ymm0, %ymm1, %ymm0
-; AVX512-NEXT:    vpshufb %ymm9, %ymm8, %ymm1
+; AVX512-NEXT:    vpor %ymm0, %ymm2, %ymm4
+; AVX512-NEXT:    vpshufb %ymm10, %ymm11, %ymm0
+; AVX512-NEXT:    vpshufb %ymm1, %ymm12, %ymm1
+; AVX512-NEXT:    vpor %ymm0, %ymm1, %ymm1
+; AVX512-NEXT:    vpshufb %ymm9, %ymm7, %ymm0
 ; AVX512-NEXT:    vpshufb %ymm5, %ymm3, %ymm2
 ; AVX512-NEXT:    vmovdqa (%rdi), %xmm5
-; AVX512-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512-NEXT:    vporq %ymm0, %ymm2, %ymm21
 ; AVX512-NEXT:    vmovdqa (%rsi), %xmm9
-; AVX512-NEXT:    vmovdqa (%rcx), %xmm8
-; AVX512-NEXT:    vmovdqa64 %xmm28, %xmm2
-; AVX512-NEXT:    vpshufb %xmm2, %xmm5, %xmm2
-; AVX512-NEXT:    vmovdqa64 %xmm29, %xmm3
-; AVX512-NEXT:    vpshufb %xmm3, %xmm9, %xmm3
-; AVX512-NEXT:    vpor %xmm2, %xmm3, %xmm4
-; AVX512-NEXT:    vmovdqa (%rdx), %xmm3
-; AVX512-NEXT:    vmovdqa 32(%r8), %ymm11
+; AVX512-NEXT:    vmovdqa (%rcx), %xmm7
+; AVX512-NEXT:    vmovdqa64 %xmm28, %xmm0
+; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm0
+; AVX512-NEXT:    vmovdqa64 %xmm29, %xmm2
+; AVX512-NEXT:    vpshufb %xmm2, %xmm9, %xmm2
+; AVX512-NEXT:    vpor %xmm0, %xmm2, %xmm10
+; AVX512-NEXT:    vmovdqa (%rdx), %xmm0
+; AVX512-NEXT:    vmovdqa 32(%r8), %ymm3
 ; AVX512-NEXT:    vmovdqa64 %xmm30, %xmm2
-; AVX512-NEXT:    vpshufb %xmm2, %xmm8, %xmm2
-; AVX512-NEXT:    vmovdqa64 %xmm31, %xmm13
-; AVX512-NEXT:    vpshufb %xmm13, %xmm3, %xmm13
-; AVX512-NEXT:    vpor %xmm2, %xmm13, %xmm13
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm14 = [12,128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128]
-; AVX512-NEXT:    vpshufb %ymm14, %ymm11, %ymm2
+; AVX512-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
+; AVX512-NEXT:    vmovdqa64 %xmm31, %xmm11
+; AVX512-NEXT:    vpshufb %xmm11, %xmm0, %xmm11
+; AVX512-NEXT:    vpor %xmm2, %xmm11, %xmm11
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm12 = [12,128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128]
+; AVX512-NEXT:    vpshufb %ymm12, %ymm3, %ymm2
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm15 = mem[1,1,2,2]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[0,1,1,1]
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255]
 ; AVX512-NEXT:    vpandnq %ymm15, %ymm28, %ymm15
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm15, %zmm2
 ; AVX512-NEXT:    vmovdqa (%r8), %ymm15
-; AVX512-NEXT:    vpshufb %ymm14, %ymm15, %ymm14
+; AVX512-NEXT:    vpshufb %ymm12, %ymm15, %ymm12
 ; AVX512-NEXT:    vpshufd {{.*#+}} ymm15 = ymm15[0,2,1,1,4,6,5,5]
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[2,3,3,2]
-; AVX512-NEXT:    vpandnq %ymm15, %ymm29, %ymm15
-; AVX512-NEXT:    vinserti64x4 $1, %ymm15, %zmm14, %zmm14
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3],xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm7 = [2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8]
-; AVX512-NEXT:    vpshufb %xmm7, %xmm6, %xmm6
+; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm15, %ymm15
+; AVX512-NEXT:    vinserti64x4 $1, %ymm15, %zmm12, %zmm12
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1],xmm6[2],xmm8[2],xmm6[3],xmm8[3],xmm6[4],xmm8[4],xmm6[5],xmm8[5],xmm6[6],xmm8[6],xmm6[7],xmm8[7]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm8 = [2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8]
+; AVX512-NEXT:    vpshufb %xmm8, %xmm6, %xmm6
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,1,1]
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3],xmm12[4],xmm10[4],xmm12[5],xmm10[5],xmm12[6],xmm10[6],xmm12[7],xmm10[7]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm12 = [0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13]
-; AVX512-NEXT:    vpshufb %xmm12, %xmm10, %xmm10
-; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,1,1]
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3],xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13]
+; AVX512-NEXT:    vpshufb %xmm14, %xmm13, %xmm13
+; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,1,1]
 ; AVX512-NEXT:    vmovdqa64 (%r8), %zmm15
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm30 = [4,0,5,5,5,5,0,6,6,6,6,0,7,7,7,7]
-; AVX512-NEXT:    vpermd %zmm11, %zmm30, %zmm30
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm31 = [6,6,6,0,7,7,7,7,0,16,16,16,16,0,17,17]
-; AVX512-NEXT:    vpermi2d %zmm11, %zmm15, %zmm31
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3],xmm5[4],xmm9[4],xmm5[5],xmm9[5],xmm5[6],xmm9[6],xmm5[7],xmm9[7]
-; AVX512-NEXT:    vpshufb %xmm12, %xmm5, %xmm5
-; AVX512-NEXT:    vinserti32x4 $2, %xmm4, %zmm5, %zmm4
-; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3],xmm8[4],xmm3[4],xmm8[5],xmm3[5],xmm8[6],xmm3[6],xmm8[7],xmm3[7]
-; AVX512-NEXT:    vpshufb %xmm7, %xmm3, %xmm3
-; AVX512-NEXT:    vinserti32x4 $2, %xmm13, %zmm3, %zmm3
-; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm21[0,0,1,1]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm20, %zmm5, %zmm5
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm29 = [4,0,5,5,5,5,0,6,6,6,6,0,7,7,7,7]
+; AVX512-NEXT:    vpermd %zmm3, %zmm29, %zmm29
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm30 = [6,6,6,0,7,7,7,7,0,16,16,16,16,0,17,17]
+; AVX512-NEXT:    vpermi2d %zmm3, %zmm15, %zmm30
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3],xmm5[4],xmm9[4],xmm5[5],xmm9[5],xmm5[6],xmm9[6],xmm5[7],xmm9[7]
+; AVX512-NEXT:    vpshufb %xmm14, %xmm3, %xmm3
+; AVX512-NEXT:    vinserti32x4 $2, %xmm10, %zmm3, %zmm3
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
+; AVX512-NEXT:    vpshufb %xmm8, %xmm0, %xmm0
+; AVX512-NEXT:    vinserti32x4 $2, %xmm11, %zmm0, %zmm0
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm20[0,0,1,1]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm19, %zmm5, %zmm5
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm23[0,0,1,1]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm22, %zmm7, %zmm7
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm5 ^ (zmm8 & (zmm7 ^ zmm5))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 | (zmm7 & zmm28)
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm17[2,2,3,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm16, %zmm5
+; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,3,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm18, %zmm4
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm5 ^ (zmm8 & (zmm4 ^ zmm5))
 ; AVX512-NEXT:    vporq %zmm24, %zmm26, %zmm5
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm5 = zmm5[2,2,3,3,6,6,7,7]
 ; AVX512-NEXT:    vporq %zmm25, %zmm27, %zmm7
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm7 = zmm7[2,2,3,3,6,6,7,7]
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255]
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm5 ^ (zmm9 & (zmm7 ^ zmm5))
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm30 = zmm30 ^ (zmm29 & (zmm30 ^ zmm7))
-; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm17[2,2,3,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm16, %zmm5
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm19[2,2,3,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm18, %zmm7
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255]
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm5 ^ (zmm8 & (zmm7 ^ zmm5))
-; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,3,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm0
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,3,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm1, %zmm1
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm0 ^ (zmm9 & (zmm1 ^ zmm0))
-; AVX512-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 | (zmm7 & mem)
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm31 = zmm31 ^ (mem & (zmm31 ^ zmm1))
-; AVX512-NEXT:    vpermq {{.*#+}} zmm0 = zmm4[0,0,1,1,4,4,5,5]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm1, %zmm1
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm21[2,2,3,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm13, %zmm5, %zmm5
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm1 ^ (zmm8 & (zmm5 ^ zmm1))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm29 = zmm29 ^ (mem & (zmm29 ^ zmm7))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 | (zmm4 & mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm30 = zmm30 ^ (mem & (zmm30 ^ zmm5))
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm1 = zmm3[0,0,1,1,4,4,5,5]
+; AVX512-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[0,0,1,1,4,4,5,5]
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,1,1,1,1,0,2,2,2,2,0]
+; AVX512-NEXT:    vpermd %zmm15, %zmm1, %zmm1
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm0))
-; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm0 = [0,0,0,0,0,0,1,1,1,1,0,2,2,2,2,0]
-; AVX512-NEXT:    vpermd %zmm15, %zmm0, %zmm0
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
-; AVX512-NEXT:    vmovdqa64 %zmm14, 64(%r9)
-; AVX512-NEXT:    vmovdqa64 %zmm0, (%r9)
-; AVX512-NEXT:    vmovdqa64 %zmm31, 128(%r9)
-; AVX512-NEXT:    vmovdqa64 %zmm30, 256(%r9)
+; AVX512-NEXT:    vmovdqa64 %zmm12, 64(%r9)
+; AVX512-NEXT:    vmovdqa64 %zmm1, (%r9)
+; AVX512-NEXT:    vmovdqa64 %zmm30, 128(%r9)
+; AVX512-NEXT:    vmovdqa64 %zmm29, 256(%r9)
 ; AVX512-NEXT:    vmovdqa64 %zmm2, 192(%r9)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -4869,202 +4864,202 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %ymm0
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128,128,128]
 ; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm0, %ymm2
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm1, %ymm16
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm8
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm15 = [12,13,128,15,12,13,14,128,12,13,14,15,128,u,u,u,16,128,18,19,16,17,128,19,16,17,18,128,16,17,18,19]
-; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm8, %ymm3
-; AVX512-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm31
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm14
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm14[8],zero,xmm14[u,7],zero,xmm14[9],zero,xmm14[u],zero,xmm14[u,10],zero,xmm14[12],zero,xmm14[u,11]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm1, %ymm17
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [12,13,128,15,12,13,14,128,12,13,14,15,128,u,u,u,16,128,18,19,16,17,128,19,16,17,18,128,16,17,18,19]
+; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm1, %ymm3
+; AVX512-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm16
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm12
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm12[8],zero,xmm12[u,7],zero,xmm12[9],zero,xmm12[u],zero,xmm12[u,10],zero,xmm12[12],zero,xmm12[u,11]
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm9
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = zero,xmm9[8,u],zero,xmm9[7],zero,xmm9[9,u,11,u],zero,xmm9[10],zero,xmm9[12,u],zero
-; AVX512-FCP-NEXT:    vporq %xmm2, %xmm3, %xmm19
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [128,8,u,128,7,128,9,u,11,u,128,10,128,12,u,128]
+; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm9, %xmm3
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm4, %xmm28
+; AVX512-FCP-NEXT:    vporq %xmm2, %xmm3, %xmm18
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm10
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm12 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128]
-; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm10, %ymm2
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128]
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm10, %ymm2
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm11
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [u,u,12,13,128,u,u,u,14,128,u,u,14,15,128,u,u,u,16,128,u,u,16,17,128,u,u,u,18,128,u,u]
-; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm11, %ymm3
-; AVX512-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm20
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm15 = [u,u,12,13,128,u,u,u,14,128,u,u,14,15,128,u,u,u,16,128,u,u,16,17,128,u,u,u,18,128,u,u]
+; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm11, %ymm3
+; AVX512-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm19
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm6
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm6[6],zero,xmm6[8,u],zero,xmm6[7],zero,xmm6[9],zero,xmm6[11,u],zero,xmm6[10],zero,xmm6[12]
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm7
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128]
-; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm7, %xmm3
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm1, %xmm28
-; AVX512-FCP-NEXT:    vporq %xmm2, %xmm3, %xmm21
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128,128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128]
-; AVX512-FCP-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm8, %ymm3
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm1, %ymm30
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128]
+; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm7, %xmm3
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm4, %xmm29
+; AVX512-FCP-NEXT:    vporq %xmm2, %xmm3, %xmm20
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128,128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128]
+; AVX512-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm3
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm31
 ; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,128,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,128]
 ; AVX512-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm8, %ymm4
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm22
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30,27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30]
-; AVX512-FCP-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm0, %ymm3
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm1, %ymm29
+; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm21
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30,27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30]
+; AVX512-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm0, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm3, %ymm30
 ; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [19,128,21,0,128,20,128,22,128,24,0,128,23,128,25,0,19,128,21,0,128,20,128,22,128,24,0,128,23,128,25,0]
 ; AVX512-FCP-NEXT:    # ymm4 = mem[0,1,0,1]
 ; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm23
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128]
-; AVX512-FCP-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm11, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm1, %ymm18
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm22
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128]
+; AVX512-FCP-NEXT:    # ymm14 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm11, %ymm0
 ; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128,128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128]
 ; AVX512-FCP-NEXT:    # ymm1 = mem[0,1,0,1]
 ; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm10, %ymm5
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm5, %zmm24
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm5, %zmm23
 ; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [25,128,27,0,128,26,128,28,128,30,0,128,29,128,31,0,25,128,27,0,128,26,128,28,128,30,0,128,29,128,31,0]
 ; AVX512-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm10, %ymm5
 ; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25]
 ; AVX512-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
 ; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm11, %ymm10
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm10, %zmm25
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm10, %zmm24
 ; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm10
-; AVX512-FCP-NEXT:    vpshufb %ymm12, %ymm10, %ymm5
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %ymm12
-; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm11
-; AVX512-FCP-NEXT:    vporq %ymm5, %ymm11, %ymm26
+; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm10, %ymm5
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %ymm13
+; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm13, %ymm11
+; AVX512-FCP-NEXT:    vporq %ymm5, %ymm11, %ymm25
 ; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm10, %ymm1
-; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm12, %ymm0
-; AVX512-FCP-NEXT:    vporq %ymm1, %ymm0, %ymm27
+; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm13, %ymm0
+; AVX512-FCP-NEXT:    vporq %ymm1, %ymm0, %ymm26
 ; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm11
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm16, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm17, %ymm0
 ; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm11, %ymm0
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm5
-; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm5, %ymm1
-; AVX512-FCP-NEXT:    vporq %ymm0, %ymm1, %ymm16
-; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm5, %ymm0
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm11, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm15
+; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm15, %ymm1
 ; AVX512-FCP-NEXT:    vporq %ymm0, %ymm1, %ymm17
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm4
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm4[8],zero,xmm4[u,7],zero,xmm4[9],zero,xmm4[u],zero,xmm4[u,10],zero,xmm4[12],zero,xmm4[u,11]
-; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm2
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = zero,xmm2[8,u],zero,xmm2[7],zero,xmm2[9,u,11,u],zero,xmm2[10],zero,xmm2[12,u],zero
-; AVX512-FCP-NEXT:    vpor %xmm0, %xmm1, %xmm15
-; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm13
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = zero,xmm13[6],zero,xmm13[8,u],zero,xmm13[7],zero,xmm13[9],zero,xmm13[11,u],zero,xmm13[10],zero,xmm13[12]
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm1
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm28, %xmm8
-; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm1, %xmm8
-; AVX512-FCP-NEXT:    vporq %xmm0, %xmm8, %xmm28
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm18, %ymm0
-; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm12, %ymm0
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm10, %ymm3
-; AVX512-FCP-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm15, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm11, %ymm1
+; AVX512-FCP-NEXT:    vporq %ymm0, %ymm1, %ymm27
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm8
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm8[8],zero,xmm8[u,7],zero,xmm8[9],zero,xmm8[u],zero,xmm8[u,10],zero,xmm8[12],zero,xmm8[u,11]
+; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm5
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm28, %xmm1
+; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm5, %xmm1
+; AVX512-FCP-NEXT:    vporq %xmm0, %xmm1, %xmm28
+; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm4
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = zero,xmm4[6],zero,xmm4[8,u],zero,xmm4[7],zero,xmm4[9],zero,xmm4[11,u],zero,xmm4[10],zero,xmm4[12]
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm2
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm29, %xmm1
+; AVX512-FCP-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; AVX512-FCP-NEXT:    vporq %xmm0, %xmm1, %xmm29
+; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm13, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm10, %ymm1
+; AVX512-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm31, %ymm1
+; AVX512-FCP-NEXT:    vpshufb %ymm1, %ymm15, %ymm1
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm30, %ymm3
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm5, %ymm3
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm29, %ymm5
-; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm11, %ymm5
-; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm8
-; AVX512-FCP-NEXT:    vpor %ymm3, %ymm5, %ymm3
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [12,128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128]
-; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm8, %ymm5
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm11 = [1,1,2,2,2,2,2,2]
-; AVX512-FCP-NEXT:    vpermd %ymm8, %ymm11, %ymm11
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255]
-; AVX512-FCP-NEXT:    vpandn %ymm11, %ymm12, %ymm11
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm11, %zmm5
-; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm11
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm29 = [4,6,5,5,5,5,4,6,6,6,6,6,7,7,7,7]
-; AVX512-FCP-NEXT:    vpermd %ymm11, %ymm29, %ymm30
-; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm11, %ymm10
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0]
-; AVX512-FCP-NEXT:    vpandnq %ymm30, %ymm11, %ymm30
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm30, %zmm10, %zmm10
+; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm11, %ymm3
+; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm10
+; AVX512-FCP-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [12,128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128]
+; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm10, %ymm3
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm13 = [1,1,2,2,2,2,2,2]
+; AVX512-FCP-NEXT:    vpermd %ymm10, %ymm13, %ymm13
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm14 = [255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255]
+; AVX512-FCP-NEXT:    vpandn %ymm13, %ymm14, %ymm13
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm13, %zmm3
+; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm13
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm15 = [4,6,5,5,5,5,4,6,6,6,6,6,7,7,7,7]
+; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm13, %ymm11
+; AVX512-FCP-NEXT:    vpermd %ymm13, %ymm15, %ymm13
+; AVX512-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm13, %ymm13
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm11, %zmm11
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3],xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13]
+; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm5, %xmm5
+; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm28, %zmm5, %zmm5
 ; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8]
 ; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm15, %zmm2, %zmm2
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3],xmm13[4],xmm1[4],xmm13[5],xmm1[5],xmm13[6],xmm1[6],xmm13[7],xmm1[7]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8]
-; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm1, %xmm1
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm28, %zmm1, %zmm1
+; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm29, %zmm2, %zmm2
 ; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3],xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7]
-; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm6, %xmm6
-; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm7 = [2,2,3,3,8,8,9,9]
-; AVX512-FCP-NEXT:    vpermt2q %zmm6, %zmm7, %zmm0
-; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm14[0],xmm9[0],xmm14[1],xmm9[1],xmm14[2],xmm9[2],xmm14[3],xmm9[3],xmm14[4],xmm9[4],xmm14[5],xmm9[5],xmm14[6],xmm9[6],xmm14[7],xmm9[7]
 ; AVX512-FCP-NEXT:    vpshufb %xmm4, %xmm6, %xmm4
-; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm7, %zmm3
+; AVX512-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm6 = [2,2,3,3,8,8,9,9]
+; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm6, %zmm0
+; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3],xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]
+; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm4, %xmm4
+; AVX512-FCP-NEXT:    vpermt2q %zmm4, %zmm6, %zmm1
 ; AVX512-FCP-NEXT:    vmovdqa64 (%r8), %zmm4
-; AVX512-FCP-NEXT:    vpermd %zmm8, %zmm29, %zmm6
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm7 = [6,6,6,0,7,7,7,7,0,16,16,16,16,0,17,17]
-; AVX512-FCP-NEXT:    vpermi2d %zmm8, %zmm4, %zmm7
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm19[0,0,1,1]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm31, %zmm8, %zmm8
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm21[0,0,1,1]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm20, %zmm9, %zmm9
-; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm8 ^ (zmm13 & (zmm9 ^ zmm8))
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm9 & zmm12)
-; AVX512-FCP-NEXT:    vporq %zmm22, %zmm23, %zmm8
+; AVX512-FCP-NEXT:    vpermd %zmm10, %zmm15, %zmm6
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm7 = [6,6,6,0,7,7,7,7,16,16,16,16,16,16,17,17]
+; AVX512-FCP-NEXT:    vpermi2d %zmm10, %zmm4, %zmm7
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm18[0,0,1,1]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm16, %zmm8, %zmm8
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm20[0,0,1,1]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm9, %zmm9
+; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm8 ^ (zmm10 & (zmm9 ^ zmm8))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm9 & zmm14)
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm26[2,2,3,3]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm25, %zmm8
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm27[2,2,3,3]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm17, %zmm9
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm8 ^ (zmm10 & (zmm9 ^ zmm8))
+; AVX512-FCP-NEXT:    vporq %zmm21, %zmm22, %zmm8
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[2,2,3,3,6,6,7,7]
-; AVX512-FCP-NEXT:    vporq %zmm24, %zmm25, %zmm9
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[2,2,3,3,6,6,7,7]
+; AVX512-FCP-NEXT:    vporq %zmm23, %zmm24, %zmm10
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm10 = zmm10[2,2,3,3,6,6,7,7]
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm8 ^ (zmm12 & (zmm9 ^ zmm8))
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm11 & (zmm6 ^ zmm9))
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm27[2,2,3,3]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm26, %zmm8
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm17[2,2,3,3]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm16, %zmm9
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm8 ^ (zmm13 & (zmm9 ^ zmm8))
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm0 ^ (zmm12 & (zmm3 ^ zmm0))
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 | (zmm9 & mem)
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm2[0,0,1,1,4,4,5,5]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[0,0,1,1,4,4,5,5]
-; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm0))
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm0 = [0,0,0,0,0,0,1,1,1,1,0,2,2,2,2,0]
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm8 ^ (zmm12 & (zmm10 ^ zmm8))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm0 ^ (zmm12 & (zmm1 ^ zmm0))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm10))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 | (zmm9 & mem)
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm5[0,0,1,1,4,4,5,5]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm2 = zmm2[0,0,1,1,4,4,5,5]
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm0))
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm0 = [0,0,0,0,0,0,1,1,1,1,2,2,2,2,2,2]
 ; AVX512-FCP-NEXT:    vpermd %zmm4, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm3))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm1))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm7, 128(%r9)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm10, 64(%r9)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm11, 64(%r9)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, (%r9)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, 256(%r9)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, 192(%r9)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, 192(%r9)
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: store_i8_stride5_vf64:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm11
+; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm10
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128,128,128]
-; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm11, %ymm0
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm10, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm1, %ymm18
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm5
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [12,13,128,15,12,13,14,128,12,13,14,15,128,u,u,u,16,128,18,19,16,17,128,19,16,17,18,128,16,17,18,19]
 ; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm5, %ymm1
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm19
-; AVX512DQ-NEXT:    vporq %ymm0, %ymm1, %ymm20
-; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm12
+; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm21
+; AVX512DQ-NEXT:    vporq %ymm0, %ymm1, %ymm19
+; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm14
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11]
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm12, %xmm0
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm14, %xmm0
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm28
-; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm10
+; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm13
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm2 = [128,8,u,128,7,128,9,u,11,u,128,10,128,12,u,128]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm10, %xmm1
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm13, %xmm1
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm29
-; AVX512DQ-NEXT:    vporq %xmm0, %xmm1, %xmm21
+; AVX512DQ-NEXT:    vporq %xmm0, %xmm1, %xmm20
 ; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %ymm15
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm8 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128]
-; AVX512DQ-NEXT:    vpshufb %ymm8, %ymm15, %ymm0
-; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm13
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm7 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128]
+; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm15, %ymm0
+; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm12
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm3 = [u,u,12,13,128,u,u,u,14,128,u,u,14,15,128,u,u,u,16,128,u,u,16,17,128,u,u,u,18,128,u,u]
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm13, %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm12, %ymm1
 ; AVX512DQ-NEXT:    vporq %ymm0, %ymm1, %ymm22
 ; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm6
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,6,128,8,u,128,7,128,9,128,11,u,128,10,128,12]
 ; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm6, %xmm0
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm30
-; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm7
+; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm8
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm2 = [6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm7, %xmm1
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm8, %xmm1
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm31
 ; AVX512DQ-NEXT:    vporq %xmm0, %xmm1, %xmm23
 ; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128,128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128]
@@ -5076,133 +5071,132 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm24
 ; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30,27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30]
 ; AVX512DQ-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm11, %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm10, %ymm1
 ; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [19,128,21,0,128,20,128,22,128,24,0,128,23,128,25,0,19,128,21,0,128,20,128,22,128,24,0,128,23,128,25,0]
 ; AVX512DQ-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm11, %ymm11
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm11, %zmm26
-; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128]
-; AVX512DQ-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm13, %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm10, %ymm10
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm10, %zmm26
+; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128]
+; AVX512DQ-NEXT:    # ymm10 = mem[0,1,0,1]
+; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm12, %ymm1
 ; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128,128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128]
 ; AVX512DQ-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm15, %ymm14
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm14, %zmm25
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm15, %ymm11
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm11, %zmm25
 ; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [25,128,27,0,128,26,128,28,128,30,0,128,29,128,31,0,25,128,27,0,128,26,128,28,128,30,0,128,29,128,31,0]
 ; AVX512DQ-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm15, %ymm14
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm15, %ymm11
 ; AVX512DQ-NEXT:    vbroadcasti128 {{.*#+}} ymm15 = [18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25]
 ; AVX512DQ-NEXT:    # ymm15 = mem[0,1,0,1]
-; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm13, %ymm13
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm14, %zmm13, %zmm27
-; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm13
-; AVX512DQ-NEXT:    vpshufb %ymm8, %ymm13, %ymm8
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm14
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm14, %ymm3
-; AVX512DQ-NEXT:    vporq %ymm8, %ymm3, %ymm16
-; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm13, %ymm0
-; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm14, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm12, %ymm12
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm12, %zmm27
+; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm12
+; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm12, %ymm7
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm11
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm11, %ymm3
+; AVX512DQ-NEXT:    vporq %ymm7, %ymm3, %ymm16
+; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm12, %ymm0
+; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm11, %ymm3
 ; AVX512DQ-NEXT:    vporq %ymm0, %ymm3, %ymm17
 ; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm3
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm18, %ymm0
 ; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm8
-; AVX512DQ-NEXT:    vmovdqa64 %ymm19, %ymm15
-; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm8, %ymm15
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm7
+; AVX512DQ-NEXT:    vmovdqa64 %ymm21, %ymm15
+; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm7, %ymm15
 ; AVX512DQ-NEXT:    vporq %ymm0, %ymm15, %ymm18
-; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm8, %ymm0
+; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm7, %ymm0
 ; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
-; AVX512DQ-NEXT:    vporq %ymm0, %ymm2, %ymm19
-; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm14, %ymm0
-; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm13, %ymm1
-; AVX512DQ-NEXT:    vpor %ymm0, %ymm1, %ymm0
-; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm8, %ymm1
+; AVX512DQ-NEXT:    vpor %ymm0, %ymm2, %ymm4
+; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm11, %ymm0
+; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm12, %ymm1
+; AVX512DQ-NEXT:    vpor %ymm0, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm7, %ymm0
 ; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm3, %ymm2
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm5
-; AVX512DQ-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vporq %ymm0, %ymm2, %ymm21
 ; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm9
-; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm8
-; AVX512DQ-NEXT:    vmovdqa64 %xmm28, %xmm2
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm5, %xmm2
-; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm9, %xmm3
-; AVX512DQ-NEXT:    vpor %xmm2, %xmm3, %xmm4
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm3
-; AVX512DQ-NEXT:    vmovdqa 32(%r8), %ymm11
+; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm7
+; AVX512DQ-NEXT:    vmovdqa64 %xmm28, %xmm0
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm0
+; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm2
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm9, %xmm2
+; AVX512DQ-NEXT:    vpor %xmm0, %xmm2, %xmm10
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm0
+; AVX512DQ-NEXT:    vmovdqa 32(%r8), %ymm3
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm30, %xmm2
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm8, %xmm2
-; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm13
-; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm3, %xmm13
-; AVX512DQ-NEXT:    vpor %xmm2, %xmm13, %xmm13
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm14 = [12,128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128]
-; AVX512DQ-NEXT:    vpshufb %ymm14, %ymm11, %ymm2
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
+; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm11
+; AVX512DQ-NEXT:    vpshufb %xmm11, %xmm0, %xmm11
+; AVX512DQ-NEXT:    vpor %xmm2, %xmm11, %xmm11
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm12 = [12,128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128]
+; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm3, %ymm2
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm15 = mem[1,1,2,2]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[0,1,1,1]
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255]
 ; AVX512DQ-NEXT:    vpandnq %ymm15, %ymm28, %ymm15
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm15, %zmm2
 ; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm15
-; AVX512DQ-NEXT:    vpshufb %ymm14, %ymm15, %ymm14
+; AVX512DQ-NEXT:    vpshufb %ymm12, %ymm15, %ymm12
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm15 = ymm15[0,2,1,1,4,6,5,5]
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[2,3,3,2]
-; AVX512DQ-NEXT:    vpandnq %ymm15, %ymm29, %ymm15
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm15, %zmm14, %zmm14
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3],xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm7 = [2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8]
-; AVX512DQ-NEXT:    vpshufb %xmm7, %xmm6, %xmm6
+; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm15, %ymm15
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm15, %zmm12, %zmm12
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1],xmm6[2],xmm8[2],xmm6[3],xmm8[3],xmm6[4],xmm8[4],xmm6[5],xmm8[5],xmm6[6],xmm8[6],xmm6[7],xmm8[7]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm8 = [2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8]
+; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm6, %xmm6
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,1,1]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3],xmm12[4],xmm10[4],xmm12[5],xmm10[5],xmm12[6],xmm10[6],xmm12[7],xmm10[7]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm12 = [0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13]
-; AVX512DQ-NEXT:    vpshufb %xmm12, %xmm10, %xmm10
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,1,1]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3],xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13]
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm13, %xmm13
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,1,1]
 ; AVX512DQ-NEXT:    vmovdqa64 (%r8), %zmm15
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm30 = [4,0,5,5,5,5,0,6,6,6,6,0,7,7,7,7]
-; AVX512DQ-NEXT:    vpermd %zmm11, %zmm30, %zmm30
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm31 = [6,6,6,0,7,7,7,7,0,16,16,16,16,0,17,17]
-; AVX512DQ-NEXT:    vpermi2d %zmm11, %zmm15, %zmm31
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3],xmm5[4],xmm9[4],xmm5[5],xmm9[5],xmm5[6],xmm9[6],xmm5[7],xmm9[7]
-; AVX512DQ-NEXT:    vpshufb %xmm12, %xmm5, %xmm5
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm4, %zmm5, %zmm4
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3],xmm8[4],xmm3[4],xmm8[5],xmm3[5],xmm8[6],xmm3[6],xmm8[7],xmm3[7]
-; AVX512DQ-NEXT:    vpshufb %xmm7, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm13, %zmm3, %zmm3
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm21[0,0,1,1]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm20, %zmm5, %zmm5
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm29 = [4,0,5,5,5,5,0,6,6,6,6,0,7,7,7,7]
+; AVX512DQ-NEXT:    vpermd %zmm3, %zmm29, %zmm29
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm30 = [6,6,6,0,7,7,7,7,0,16,16,16,16,0,17,17]
+; AVX512DQ-NEXT:    vpermi2d %zmm3, %zmm15, %zmm30
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3],xmm5[4],xmm9[4],xmm5[5],xmm9[5],xmm5[6],xmm9[6],xmm5[7],xmm9[7]
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm3, %xmm3
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm10, %zmm3, %zmm3
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
+; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm11, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm20[0,0,1,1]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm19, %zmm5, %zmm5
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm23[0,0,1,1]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm22, %zmm7, %zmm7
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm5 ^ (zmm8 & (zmm7 ^ zmm5))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 | (zmm7 & zmm28)
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm17[2,2,3,3]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm16, %zmm5
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,3,3]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm18, %zmm4
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm5 ^ (zmm8 & (zmm4 ^ zmm5))
 ; AVX512DQ-NEXT:    vporq %zmm24, %zmm26, %zmm5
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm5 = zmm5[2,2,3,3,6,6,7,7]
 ; AVX512DQ-NEXT:    vporq %zmm25, %zmm27, %zmm7
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm7 = zmm7[2,2,3,3,6,6,7,7]
-; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm5 ^ (zmm9 & (zmm7 ^ zmm5))
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm30 = zmm30 ^ (zmm29 & (zmm30 ^ zmm7))
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm17[2,2,3,3]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm16, %zmm5
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm19[2,2,3,3]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm18, %zmm7
+; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255]
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm5 ^ (zmm8 & (zmm7 ^ zmm5))
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,2,3,3]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,2,3,3]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm1, %zmm1
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm0 ^ (zmm9 & (zmm1 ^ zmm0))
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 | (zmm7 & mem)
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm31 = zmm31 ^ (mem & (zmm31 ^ zmm1))
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm0 = zmm4[0,0,1,1,4,4,5,5]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm21[2,2,3,3]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm13, %zmm5, %zmm5
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm1 ^ (zmm8 & (zmm5 ^ zmm1))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm29 = zmm29 ^ (mem & (zmm29 ^ zmm7))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm12 = zmm12 | (zmm4 & mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm30 = zmm30 ^ (mem & (zmm30 ^ zmm5))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm1 = zmm3[0,0,1,1,4,4,5,5]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[0,0,1,1,4,4,5,5]
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
+; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm1 = [0,0,0,0,0,0,1,1,1,1,0,2,2,2,2,0]
+; AVX512DQ-NEXT:    vpermd %zmm15, %zmm1, %zmm1
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm0))
-; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm0 = [0,0,0,0,0,0,1,1,1,1,0,2,2,2,2,0]
-; AVX512DQ-NEXT:    vpermd %zmm15, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
-; AVX512DQ-NEXT:    vmovdqa64 %zmm14, 64(%r9)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm0, (%r9)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm31, 128(%r9)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm30, 256(%r9)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm12, 64(%r9)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm1, (%r9)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm30, 128(%r9)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm29, 256(%r9)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm2, 192(%r9)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
@@ -5212,165 +5206,165 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128,128,128]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm0, %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm1, %ymm16
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm8
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm15 = [12,13,128,15,12,13,14,128,12,13,14,15,128,u,u,u,16,128,18,19,16,17,128,19,16,17,18,128,16,17,18,19]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm8, %ymm3
-; AVX512DQ-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm31
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm14
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm14[8],zero,xmm14[u,7],zero,xmm14[9],zero,xmm14[u],zero,xmm14[u,10],zero,xmm14[12],zero,xmm14[u,11]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm1, %ymm17
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [12,13,128,15,12,13,14,128,12,13,14,15,128,u,u,u,16,128,18,19,16,17,128,19,16,17,18,128,16,17,18,19]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm1, %ymm3
+; AVX512DQ-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm16
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm12
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm12[8],zero,xmm12[u,7],zero,xmm12[9],zero,xmm12[u],zero,xmm12[u,10],zero,xmm12[12],zero,xmm12[u,11]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm9
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = zero,xmm9[8,u],zero,xmm9[7],zero,xmm9[9,u,11,u],zero,xmm9[10],zero,xmm9[12,u],zero
-; AVX512DQ-FCP-NEXT:    vporq %xmm2, %xmm3, %xmm19
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [128,8,u,128,7,128,9,u,11,u,128,10,128,12,u,128]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm4, %xmm9, %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm4, %xmm28
+; AVX512DQ-FCP-NEXT:    vporq %xmm2, %xmm3, %xmm18
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm10
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm12 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm10, %ymm2
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm10, %ymm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %ymm11
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [u,u,12,13,128,u,u,u,14,128,u,u,14,15,128,u,u,u,16,128,u,u,16,17,128,u,u,u,18,128,u,u]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm11, %ymm3
-; AVX512DQ-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm20
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm15 = [u,u,12,13,128,u,u,u,14,128,u,u,14,15,128,u,u,u,16,128,u,u,16,17,128,u,u,u,18,128,u,u]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm11, %ymm3
+; AVX512DQ-FCP-NEXT:    vporq %ymm2, %ymm3, %ymm19
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm6
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm6[6],zero,xmm6[8,u],zero,xmm6[7],zero,xmm6[9],zero,xmm6[11,u],zero,xmm6[10],zero,xmm6[12]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm7
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm1 = [6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm7, %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm1, %xmm28
-; AVX512DQ-FCP-NEXT:    vporq %xmm2, %xmm3, %xmm21
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128,128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128]
-; AVX512DQ-FCP-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm8, %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm1, %ymm30
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm4, %xmm7, %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm4, %xmm29
+; AVX512DQ-FCP-NEXT:    vporq %xmm2, %xmm3, %xmm20
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128,128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128]
+; AVX512DQ-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm31
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,128,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,128]
 ; AVX512DQ-FCP-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm8, %ymm4
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm22
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30,27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30]
-; AVX512DQ-FCP-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm0, %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm1, %ymm29
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm21
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30,27,0,128,26,128,28,0,128,0,128,29,128,31,0,128,30]
+; AVX512DQ-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm0, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm3, %ymm30
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [19,128,21,0,128,20,128,22,128,24,0,128,23,128,25,0,19,128,21,0,128,20,128,22,128,24,0,128,23,128,25,0]
 ; AVX512DQ-FCP-NEXT:    # ymm4 = mem[0,1,0,1]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm23
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128]
-; AVX512DQ-FCP-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm11, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm1, %ymm18
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm22
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128]
+; AVX512DQ-FCP-NEXT:    # ymm14 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm11, %ymm0
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128,128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128]
 ; AVX512DQ-FCP-NEXT:    # ymm1 = mem[0,1,0,1]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm10, %ymm5
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm5, %zmm24
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm5, %zmm23
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [25,128,27,0,128,26,128,28,128,30,0,128,29,128,31,0,25,128,27,0,128,26,128,28,128,30,0,128,29,128,31,0]
 ; AVX512DQ-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm10, %ymm5
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25]
 ; AVX512DQ-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm11, %ymm10
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm10, %zmm25
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm10, %zmm24
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm10
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm12, %ymm10, %ymm5
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %ymm12
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm11
-; AVX512DQ-FCP-NEXT:    vporq %ymm5, %ymm11, %ymm26
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm10, %ymm5
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %ymm13
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm13, %ymm11
+; AVX512DQ-FCP-NEXT:    vporq %ymm5, %ymm11, %ymm25
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm10, %ymm1
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm12, %ymm0
-; AVX512DQ-FCP-NEXT:    vporq %ymm1, %ymm0, %ymm27
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm13, %ymm0
+; AVX512DQ-FCP-NEXT:    vporq %ymm1, %ymm0, %ymm26
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm11
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm16, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm17, %ymm0
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm11, %ymm0
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm5
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm5, %ymm1
-; AVX512DQ-FCP-NEXT:    vporq %ymm0, %ymm1, %ymm16
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm5, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm11, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm15
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm15, %ymm1
 ; AVX512DQ-FCP-NEXT:    vporq %ymm0, %ymm1, %ymm17
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm4
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm4[8],zero,xmm4[u,7],zero,xmm4[9],zero,xmm4[u],zero,xmm4[u,10],zero,xmm4[12],zero,xmm4[u,11]
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm2
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = zero,xmm2[8,u],zero,xmm2[7],zero,xmm2[9,u,11,u],zero,xmm2[10],zero,xmm2[12,u],zero
-; AVX512DQ-FCP-NEXT:    vpor %xmm0, %xmm1, %xmm15
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm13
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = zero,xmm13[6],zero,xmm13[8,u],zero,xmm13[7],zero,xmm13[9],zero,xmm13[11,u],zero,xmm13[10],zero,xmm13[12]
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm28, %xmm8
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm1, %xmm8
-; AVX512DQ-FCP-NEXT:    vporq %xmm0, %xmm8, %xmm28
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm18, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm12, %ymm0
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm10, %ymm3
-; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm15, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm11, %ymm1
+; AVX512DQ-FCP-NEXT:    vporq %ymm0, %ymm1, %ymm27
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm8
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm8[8],zero,xmm8[u,7],zero,xmm8[9],zero,xmm8[u],zero,xmm8[u,10],zero,xmm8[12],zero,xmm8[u,11]
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm28, %xmm1
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm5, %xmm1
+; AVX512DQ-FCP-NEXT:    vporq %xmm0, %xmm1, %xmm28
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm4
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = zero,xmm4[6],zero,xmm4[8,u],zero,xmm4[7],zero,xmm4[9],zero,xmm4[11,u],zero,xmm4[10],zero,xmm4[12]
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm29, %xmm1
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; AVX512DQ-FCP-NEXT:    vporq %xmm0, %xmm1, %xmm29
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm13, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm10, %ymm1
+; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm31, %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm1, %ymm15, %ymm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm30, %ymm3
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm5, %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm29, %ymm5
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm11, %ymm5
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm8
-; AVX512DQ-FCP-NEXT:    vpor %ymm3, %ymm5, %ymm3
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [12,128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm8, %ymm5
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm11 = [1,1,2,2,2,2,2,2]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm8, %ymm11, %ymm11
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255]
-; AVX512DQ-FCP-NEXT:    vpandn %ymm11, %ymm12, %ymm11
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm11, %zmm5
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm11
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm29 = [4,6,5,5,5,5,4,6,6,6,6,6,7,7,7,7]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm11, %ymm29, %ymm30
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm11, %ymm10
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0]
-; AVX512DQ-FCP-NEXT:    vpandnq %ymm30, %ymm11, %ymm30
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm30, %zmm10, %zmm10
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm11, %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm10
+; AVX512DQ-FCP-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [12,128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm10, %ymm3
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm13 = [1,1,2,2,2,2,2,2]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm10, %ymm13, %ymm13
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm14 = [255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255]
+; AVX512DQ-FCP-NEXT:    vpandn %ymm13, %ymm14, %ymm13
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm13, %zmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm13
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm15 = [4,6,5,5,5,5,4,6,6,6,6,6,7,7,7,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm13, %ymm11
+; AVX512DQ-FCP-NEXT:    vpermd %ymm13, %ymm15, %ymm13
+; AVX512DQ-FCP-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm13, %ymm13
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm11, %zmm11
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3],xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm5, %xmm5
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm28, %zmm5, %zmm5
 ; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm4 = [2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm15, %zmm2, %zmm2
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3],xmm13[4],xmm1[4],xmm13[5],xmm1[5],xmm13[6],xmm1[6],xmm13[7],xmm1[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm1, %xmm1
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm28, %zmm1, %zmm1
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm29, %zmm2, %zmm2
 ; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3],xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm6, %xmm6
-; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm7 = [2,2,3,3,8,8,9,9]
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm6, %zmm7, %zmm0
-; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm14[0],xmm9[0],xmm14[1],xmm9[1],xmm14[2],xmm9[2],xmm14[3],xmm9[3],xmm14[4],xmm9[4],xmm14[5],xmm9[5],xmm14[6],xmm9[6],xmm14[7],xmm9[7]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm4, %xmm6, %xmm4
-; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm7, %zmm3
+; AVX512DQ-FCP-NEXT:    vpmovsxbq {{.*#+}} zmm6 = [2,2,3,3,8,8,9,9]
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm6, %zmm0
+; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3],xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm4, %xmm4
+; AVX512DQ-FCP-NEXT:    vpermt2q %zmm4, %zmm6, %zmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 (%r8), %zmm4
-; AVX512DQ-FCP-NEXT:    vpermd %zmm8, %zmm29, %zmm6
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm7 = [6,6,6,0,7,7,7,7,0,16,16,16,16,0,17,17]
-; AVX512DQ-FCP-NEXT:    vpermi2d %zmm8, %zmm4, %zmm7
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm19[0,0,1,1]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm31, %zmm8, %zmm8
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm21[0,0,1,1]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm20, %zmm9, %zmm9
-; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm8 ^ (zmm13 & (zmm9 ^ zmm8))
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm9 & zmm12)
-; AVX512DQ-FCP-NEXT:    vporq %zmm22, %zmm23, %zmm8
+; AVX512DQ-FCP-NEXT:    vpermd %zmm10, %zmm15, %zmm6
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm7 = [6,6,6,0,7,7,7,7,16,16,16,16,16,16,17,17]
+; AVX512DQ-FCP-NEXT:    vpermi2d %zmm10, %zmm4, %zmm7
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm18[0,0,1,1]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm16, %zmm8, %zmm8
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm20[0,0,1,1]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm19, %zmm9, %zmm9
+; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm8 ^ (zmm10 & (zmm9 ^ zmm8))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm9 & zmm14)
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm26[2,2,3,3]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm25, %zmm8
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm27[2,2,3,3]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm17, %zmm9
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm8 ^ (zmm10 & (zmm9 ^ zmm8))
+; AVX512DQ-FCP-NEXT:    vporq %zmm21, %zmm22, %zmm8
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[2,2,3,3,6,6,7,7]
-; AVX512DQ-FCP-NEXT:    vporq %zmm24, %zmm25, %zmm9
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[2,2,3,3,6,6,7,7]
+; AVX512DQ-FCP-NEXT:    vporq %zmm23, %zmm24, %zmm10
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm10 = zmm10[2,2,3,3,6,6,7,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm8 ^ (zmm12 & (zmm9 ^ zmm8))
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm11 & (zmm6 ^ zmm9))
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm27[2,2,3,3]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm26, %zmm8
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm17[2,2,3,3]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm16, %zmm9
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm8 ^ (zmm13 & (zmm9 ^ zmm8))
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm0 ^ (zmm12 & (zmm3 ^ zmm0))
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 | (zmm9 & mem)
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm2[0,0,1,1,4,4,5,5]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[0,0,1,1,4,4,5,5]
-; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm0))
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm0 = [0,0,0,0,0,0,1,1,1,1,0,2,2,2,2,0]
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm8 ^ (zmm12 & (zmm10 ^ zmm8))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm0 ^ (zmm12 & (zmm1 ^ zmm0))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm10))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 | (zmm9 & mem)
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm5[0,0,1,1,4,4,5,5]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm2 = zmm2[0,0,1,1,4,4,5,5]
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm0))
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm0 = [0,0,0,0,0,0,1,1,1,1,2,2,2,2,2,2]
 ; AVX512DQ-FCP-NEXT:    vpermd %zmm4, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1))
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm3))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, 128(%r9)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm10, 64(%r9)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm11, 64(%r9)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, (%r9)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, 256(%r9)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, 192(%r9)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, 192(%r9)
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
index 2e79e1bf5bae0..fab62f10434f9 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
@@ -2636,7 +2636,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,2,2,3]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm13
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm14 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm13 = zmm6 ^ (zmm14 & (zmm13 ^ zmm6))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm6 ^ (zmm14 & (zmm13 ^ zmm6))
 ; AVX512-NEXT:    vmovdqa (%r9), %xmm11
 ; AVX512-NEXT:    vmovdqa (%r8), %xmm12
 ; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm12[8],xmm11[8],xmm12[9],xmm11[9],xmm12[10],xmm11[10],xmm12[11],xmm11[11],xmm12[12],xmm11[12],xmm12[13],xmm11[13],xmm12[14],xmm11[14],xmm12[15],xmm11[15]
@@ -2647,7 +2647,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufhw {{.*#+}} ymm15 = ymm15[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[2,2,2,3]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm15, %zmm6, %zmm6
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm13))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm13))
 ; AVX512-NEXT:    vpbroadcastq {{.*#+}} xmm13 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
 ; AVX512-NEXT:    vpshufb %xmm13, %xmm9, %xmm15
 ; AVX512-NEXT:    vpshufb %xmm13, %xmm10, %xmm13
@@ -2665,7 +2665,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,6,5]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm8, %zmm7
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm8 = zmm7[0,0,0,1,4,4,4,5]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm13 ^ (mem & (zmm8 ^ zmm13))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm13 ^ (mem & (zmm8 ^ zmm13))
 ; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
 ; AVX512-NEXT:    vpshufb %xmm7, %xmm11, %xmm9
 ; AVX512-NEXT:    vpshufb %xmm7, %xmm12, %xmm10
@@ -2675,7 +2675,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,4,4,4]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm10, %zmm9
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[0,0,0,1,4,4,4,5]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm9 = zmm9 ^ (zmm14 & (zmm9 ^ zmm8))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 ^ (zmm14 & (zmm9 ^ zmm8))
 ; AVX512-NEXT:    vpshufb %ymm15, %ymm4, %ymm8
 ; AVX512-NEXT:    vpshufb %ymm15, %ymm2, %ymm10
 ; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm8 = ymm10[0],ymm8[0],ymm10[1],ymm8[1],ymm10[2],ymm8[2],ymm10[3],ymm8[3],ymm10[4],ymm8[4],ymm10[5],ymm8[5],ymm10[6],ymm8[6],ymm10[7],ymm8[7],ymm10[16],ymm8[16],ymm10[17],ymm8[17],ymm10[18],ymm8[18],ymm10[19],ymm8[19],ymm10[20],ymm8[20],ymm10[21],ymm8[21],ymm10[22],ymm8[22],ymm10[23],ymm8[23]
@@ -2691,7 +2691,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm2 = zmm2[2,2,2,3,6,6,6,7]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm8))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm8))
 ; AVX512-NEXT:    vpshufb %ymm7, %ymm1, %ymm3
 ; AVX512-NEXT:    vpshufb %ymm7, %ymm0, %ymm4
 ; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[16],ymm3[16],ymm4[17],ymm3[17],ymm4[18],ymm3[18],ymm4[19],ymm3[19],ymm4[20],ymm3[20],ymm4[21],ymm3[21],ymm4[22],ymm3[22],ymm4[23],ymm3[23]
@@ -2699,7 +2699,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,21,26,27,24,25,22,23,28,29,26,27,28,29,30,31]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm3, %zmm0
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,2,2,3,6,6,6,7]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
 ; AVX512-NEXT:    vmovdqa64 %zmm0, 128(%rax)
 ; AVX512-NEXT:    vmovdqa64 %zmm9, (%rax)
 ; AVX512-NEXT:    vmovdqa64 %zmm6, 64(%rax)
@@ -2731,7 +2731,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm8, %zmm8
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[2,2,2,3,6,6,6,7]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm6))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm6))
 ; AVX512-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm10 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
 ; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm1, %ymm6
 ; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm0, %ymm9
@@ -2740,7 +2740,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,21,26,27,24,25,22,23,28,29,26,27,28,29,30,31]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm6, %zmm6
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm6 = zmm6[2,2,2,3,6,6,6,7]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm8))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm8))
 ; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm9
 ; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm9, %xmm8
 ; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm11
@@ -2760,7 +2760,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vprold $16, %xmm14, %xmm14
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm14, %zmm13
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm14 = zmm13[0,0,0,1,4,4,4,5]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm14 = zmm14 ^ (mem & (zmm14 ^ zmm12))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (mem & (zmm14 ^ zmm12))
 ; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm12
 ; AVX512-FCP-NEXT:    vpshufb %xmm10, %xmm12, %xmm15
 ; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm13
@@ -2771,7 +2771,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm10, %zmm15, %zmm10
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm10 = zmm10[0,0,0,1,4,4,4,5]
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm10 = zmm10 ^ (zmm15 & (zmm10 ^ zmm14))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm15 & (zmm10 ^ zmm14))
 ; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[16],ymm5[16],ymm4[17],ymm5[17],ymm4[18],ymm5[18],ymm4[19],ymm5[19],ymm4[20],ymm5[20],ymm4[21],ymm5[21],ymm4[22],ymm5[22],ymm4[23],ymm5[23]
 ; AVX512-FCP-NEXT:    vprold $16, %ymm4, %ymm4
 ; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm11[8],xmm9[8],xmm11[9],xmm9[9],xmm11[10],xmm9[10],xmm11[11],xmm9[11],xmm11[12],xmm9[12],xmm11[13],xmm9[13],xmm11[14],xmm9[14],xmm11[15],xmm9[15]
@@ -2783,13 +2783,13 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[8,9,6,7,12,13,10,11,14,15,14,15,14,15,14,15]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm9, %zmm3
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm5 ^ (zmm15 & (zmm3 ^ zmm5))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm5 ^ (zmm15 & (zmm3 ^ zmm5))
 ; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,21,18,19,16,17,22,23,24,25,24,25,24,25,24,25]
 ; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,8,9,6,7,12,13,10,11,12,13,14,15]
 ; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm9, %zmm1
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm3))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm3))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 64(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm10, (%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, 128(%rax)
@@ -2825,7 +2825,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,2,2,3]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm13
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm14 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm13 = zmm6 ^ (zmm14 & (zmm13 ^ zmm6))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm6 ^ (zmm14 & (zmm13 ^ zmm6))
 ; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm11
 ; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm12
 ; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm12[8],xmm11[8],xmm12[9],xmm11[9],xmm12[10],xmm11[10],xmm12[11],xmm11[11],xmm12[12],xmm11[12],xmm12[13],xmm11[13],xmm12[14],xmm11[14],xmm12[15],xmm11[15]
@@ -2836,7 +2836,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm15 = ymm15[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[2,2,2,3]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm15, %zmm6, %zmm6
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm13))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm13))
 ; AVX512DQ-NEXT:    vpbroadcastq {{.*#+}} xmm13 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
 ; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm9, %xmm15
 ; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm10, %xmm13
@@ -2854,7 +2854,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,6,5]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm8, %zmm7
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm8 = zmm7[0,0,0,1,4,4,4,5]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm13 ^ (mem & (zmm8 ^ zmm13))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm13 ^ (mem & (zmm8 ^ zmm13))
 ; AVX512DQ-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
 ; AVX512DQ-NEXT:    vpshufb %xmm7, %xmm11, %xmm9
 ; AVX512DQ-NEXT:    vpshufb %xmm7, %xmm12, %xmm10
@@ -2864,7 +2864,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,4,4,4]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm10, %zmm9
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[0,0,0,1,4,4,4,5]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm9 = zmm9 ^ (zmm14 & (zmm9 ^ zmm8))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 ^ (zmm14 & (zmm9 ^ zmm8))
 ; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm4, %ymm8
 ; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm2, %ymm10
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm8 = ymm10[0],ymm8[0],ymm10[1],ymm8[1],ymm10[2],ymm8[2],ymm10[3],ymm8[3],ymm10[4],ymm8[4],ymm10[5],ymm8[5],ymm10[6],ymm8[6],ymm10[7],ymm8[7],ymm10[16],ymm8[16],ymm10[17],ymm8[17],ymm10[18],ymm8[18],ymm10[19],ymm8[19],ymm10[20],ymm8[20],ymm10[21],ymm8[21],ymm10[22],ymm8[22],ymm10[23],ymm8[23]
@@ -2880,7 +2880,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm2 = zmm2[2,2,2,3,6,6,6,7]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm8))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (mem & (zmm2 ^ zmm8))
 ; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm1, %ymm3
 ; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm0, %ymm4
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[16],ymm3[16],ymm4[17],ymm3[17],ymm4[18],ymm3[18],ymm4[19],ymm3[19],ymm4[20],ymm3[20],ymm4[21],ymm3[21],ymm4[22],ymm3[22],ymm4[23],ymm3[23]
@@ -2888,7 +2888,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,21,26,27,24,25,22,23,28,29,26,27,28,29,30,31]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm3, %zmm0
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,2,2,3,6,6,6,7]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 128(%rax)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm9, (%rax)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm6, 64(%rax)
@@ -2920,7 +2920,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm8, %zmm8
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[2,2,2,3,6,6,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm6))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm6))
 ; AVX512DQ-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm10 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm1, %ymm6
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm0, %ymm9
@@ -2929,7 +2929,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,21,26,27,24,25,22,23,28,29,26,27,28,29,30,31]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm6, %zmm6
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm6 = zmm6[2,2,2,3,6,6,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm8))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm8))
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm9
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm9, %xmm8
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm11
@@ -2949,7 +2949,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vprold $16, %xmm14, %xmm14
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm14, %zmm13
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm14 = zmm13[0,0,0,1,4,4,4,5]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm14 = zmm14 ^ (mem & (zmm14 ^ zmm12))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm14 = zmm14 ^ (mem & (zmm14 ^ zmm12))
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm12
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm10, %xmm12, %xmm15
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm13
@@ -2960,7 +2960,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm10, %zmm15, %zmm10
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm10 = zmm10[0,0,0,1,4,4,4,5]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm10 = zmm10 ^ (zmm15 & (zmm10 ^ zmm14))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm15 & (zmm10 ^ zmm14))
 ; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[16],ymm5[16],ymm4[17],ymm5[17],ymm4[18],ymm5[18],ymm4[19],ymm5[19],ymm4[20],ymm5[20],ymm4[21],ymm5[21],ymm4[22],ymm5[22],ymm4[23],ymm5[23]
 ; AVX512DQ-FCP-NEXT:    vprold $16, %ymm4, %ymm4
 ; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm11[8],xmm9[8],xmm11[9],xmm9[9],xmm11[10],xmm9[10],xmm11[11],xmm9[11],xmm11[12],xmm9[12],xmm11[13],xmm9[13],xmm11[14],xmm9[14],xmm11[15],xmm9[15]
@@ -2972,13 +2972,13 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[8,9,6,7,12,13,10,11,14,15,14,15,14,15,14,15]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm9, %zmm3
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm5 ^ (zmm15 & (zmm3 ^ zmm5))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm5 ^ (zmm15 & (zmm3 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,21,18,19,16,17,22,23,24,25,24,25,24,25,24,25]
 ; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,8,9,6,7,12,13,10,11,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm9, %zmm1
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm3))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 64(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm10, (%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, 128(%rax)
@@ -5564,18 +5564,18 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm2 ^ (zmm9 & (zmm8 ^ zmm2))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm7, %zmm2
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm5 & (zmm2 ^ zmm8))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm5 & (zmm2 ^ zmm8))
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm14[2,2,2,3]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm13, %zmm7
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm15, %zmm1
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm7 ^ (zmm9 & (zmm1 ^ zmm7))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm31, %zmm6
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm5 & (zmm6 ^ zmm1))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm5 & (zmm6 ^ zmm1))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm12, %zmm1
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm4 & (zmm1 ^ zmm2))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm4 & (zmm1 ^ zmm2))
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm11, %zmm2
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm4 & (zmm2 ^ zmm6))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm4 & (zmm2 ^ zmm6))
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[0,0,0,1,4,4,4,5]
 ; AVX512-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Folded Reload
 ; AVX512-NEXT:    # zmm3 = mem[0,0,0,1,4,4,4,5]
@@ -5587,7 +5587,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Folded Reload
 ; AVX512-NEXT:    # zmm3 = mem[0,0,0,1,4,4,4,5]
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (zmm5 & (zmm3 ^ zmm0))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm5 & (zmm3 ^ zmm0))
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm0 = zmm30[0,0,0,1,4,4,4,5]
 ; AVX512-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Folded Reload
 ; AVX512-NEXT:    # zmm6 = mem[0,0,0,1,4,4,4,5]
@@ -5597,7 +5597,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm9 & (zmm0 ^ zmm6))
 ; AVX512-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Folded Reload
 ; AVX512-NEXT:    # zmm6 = mem[0,0,0,1,4,4,4,5]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm5 & (zmm6 ^ zmm0))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm5 & (zmm6 ^ zmm0))
 ; AVX512-NEXT:    vpermq $234, (%rsp), %zmm0 # 64-byte Folded Reload
 ; AVX512-NEXT:    # zmm0 = mem[2,2,2,3,6,6,6,7]
 ; AVX512-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Folded Reload
@@ -5613,9 +5613,9 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm4 & (zmm5 ^ zmm7))
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm4 = zmm28[2,2,2,3,6,6,6,7]
 ; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm7 & (zmm4 ^ zmm0))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm7 & (zmm4 ^ zmm0))
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm0 = zmm25[2,2,2,3,6,6,6,7]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm7 & (zmm0 ^ zmm5))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm7 & (zmm0 ^ zmm5))
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-NEXT:    vmovdqa64 %zmm0, 128(%rax)
 ; AVX512-NEXT:    vmovdqa64 %zmm4, 320(%rax)
@@ -5793,7 +5793,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm1, %xmm1
 ; AVX512-FCP-NEXT:    vpermt2q %zmm0, %zmm4, %zmm1
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm10 & (zmm1 ^ zmm15))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm10 & (zmm1 ^ zmm15))
 ; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm15 = [0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0]
 ; AVX512-FCP-NEXT:    # ymm15 = mem[0,1,0,1]
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm20, %ymm0
@@ -5802,7 +5802,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm8, %xmm0
 ; AVX512-FCP-NEXT:    vpermt2q %zmm2, %zmm4, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm2 & (zmm0 ^ zmm1))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm2 & (zmm0 ^ zmm1))
 ; AVX512-FCP-NEXT:    vmovdqa64 %ymm22, %ymm8
 ; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
 ; AVX512-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm8[0],ymm1[1],ymm8[1],ymm1[2],ymm8[2],ymm1[3],ymm8[3],ymm1[4],ymm8[4],ymm1[5],ymm8[5],ymm1[6],ymm8[6],ymm1[7],ymm8[7],ymm1[16],ymm8[16],ymm1[17],ymm8[17],ymm1[18],ymm8[18],ymm1[19],ymm8[19],ymm1[20],ymm8[20],ymm1[21],ymm8[21],ymm1[22],ymm8[22],ymm1[23],ymm8[23]
@@ -5826,12 +5826,12 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
 ; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm13, %xmm3
 ; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm4, %zmm3
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (zmm10 & (zmm3 ^ zmm6))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm10 & (zmm3 ^ zmm6))
 ; AVX512-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
 ; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm1, %ymm1
 ; AVX512-FCP-NEXT:    vpshufb %xmm7, %xmm14, %xmm5
 ; AVX512-FCP-NEXT:    vpermt2q %zmm1, %zmm4, %zmm5
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (zmm2 & (zmm5 ^ zmm3))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm2 & (zmm5 ^ zmm3))
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, 256(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rax)
@@ -5847,7 +5847,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    # zmm1 = mem[2,2,2,3,6,6,6,7]
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm3 & (zmm1 ^ zmm0))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm3 & (zmm1 ^ zmm0))
 ; AVX512-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    # zmm0 = mem[2,2,2,3,6,6,6,7]
 ; AVX512-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Folded Reload
@@ -5856,7 +5856,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm23[2,2,2,3,6,6,6,7]
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm2 & (zmm0 ^ zmm4))
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm4 = zmm24[2,2,2,3,6,6,6,7]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm3 & (zmm4 ^ zmm0))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm3 & (zmm4 ^ zmm0))
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm30[0,0,0,1,4,4,4,5]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm26[0,0,0,1,4,4,4,5]
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm0 ^ (zmm2 & (zmm3 ^ zmm0))
@@ -5869,9 +5869,9 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm9 & (zmm2 ^ zmm5))
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm21[0,0,0,1,4,4,4,5]
 ; AVX512-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (zmm5 & (zmm3 ^ zmm0))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm5 & (zmm3 ^ zmm0))
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm28[0,0,0,1,4,4,4,5]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm5 & (zmm0 ^ zmm2))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm5 & (zmm0 ^ zmm2))
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, (%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, 192(%rax)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, 128(%rax)
@@ -6093,18 +6093,18 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm2 ^ (zmm9 & (zmm8 ^ zmm2))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm7, %zmm2
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm5 & (zmm2 ^ zmm8))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm5 & (zmm2 ^ zmm8))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm14[2,2,2,3]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm13, %zmm7
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm15, %zmm1
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm7 ^ (zmm9 & (zmm1 ^ zmm7))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm31, %zmm6
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm5 & (zmm6 ^ zmm1))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm5 & (zmm6 ^ zmm1))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm12, %zmm1
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm4 & (zmm1 ^ zmm2))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm4 & (zmm1 ^ zmm2))
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm11, %zmm2
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm4 & (zmm2 ^ zmm6))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm4 & (zmm2 ^ zmm6))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[0,0,0,1,4,4,4,5]
 ; AVX512DQ-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    # zmm3 = mem[0,0,0,1,4,4,4,5]
@@ -6116,7 +6116,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    # zmm3 = mem[0,0,0,1,4,4,4,5]
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (zmm5 & (zmm3 ^ zmm0))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm5 & (zmm3 ^ zmm0))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm0 = zmm30[0,0,0,1,4,4,4,5]
 ; AVX512DQ-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    # zmm6 = mem[0,0,0,1,4,4,4,5]
@@ -6126,7 +6126,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm9 & (zmm0 ^ zmm6))
 ; AVX512DQ-NEXT:    vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    # zmm6 = mem[0,0,0,1,4,4,4,5]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (zmm5 & (zmm6 ^ zmm0))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (zmm5 & (zmm6 ^ zmm0))
 ; AVX512DQ-NEXT:    vpermq $234, (%rsp), %zmm0 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    # zmm0 = mem[2,2,2,3,6,6,6,7]
 ; AVX512DQ-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Folded Reload
@@ -6142,9 +6142,9 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm4 & (zmm5 ^ zmm7))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm4 = zmm28[2,2,2,3,6,6,6,7]
 ; AVX512DQ-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm7 & (zmm4 ^ zmm0))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm7 & (zmm4 ^ zmm0))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm0 = zmm25[2,2,2,3,6,6,6,7]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm7 & (zmm0 ^ zmm5))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm7 & (zmm0 ^ zmm5))
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 128(%rax)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 320(%rax)
@@ -6322,7 +6322,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm1, %xmm1
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm0, %zmm4, %zmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm10 & (zmm1 ^ zmm15))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm10 & (zmm1 ^ zmm15))
 ; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm15 = [0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0,0,2,0,1,0,0,0,3,0,0,0,0,0,4,0,0]
 ; AVX512DQ-FCP-NEXT:    # ymm15 = mem[0,1,0,1]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm20, %ymm0
@@ -6331,7 +6331,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm8, %xmm0
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm2, %zmm4, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm2 & (zmm0 ^ zmm1))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm2 & (zmm0 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm22, %ymm8
 ; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm8[0],ymm1[1],ymm8[1],ymm1[2],ymm8[2],ymm1[3],ymm8[3],ymm1[4],ymm8[4],ymm1[5],ymm8[5],ymm1[6],ymm8[6],ymm1[7],ymm8[7],ymm1[16],ymm8[16],ymm1[17],ymm8[17],ymm1[18],ymm8[18],ymm1[19],ymm8[19],ymm1[20],ymm8[20],ymm1[21],ymm8[21],ymm1[22],ymm8[22],ymm1[23],ymm8[23]
@@ -6355,12 +6355,12 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm13, %xmm3
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm4, %zmm3
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (zmm10 & (zmm3 ^ zmm6))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm10 & (zmm3 ^ zmm6))
 ; AVX512DQ-FCP-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm1, %ymm1
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm7, %xmm14, %xmm5
 ; AVX512DQ-FCP-NEXT:    vpermt2q %zmm1, %zmm4, %zmm5
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (zmm2 & (zmm5 ^ zmm3))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm2 & (zmm5 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, 256(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rax)
@@ -6376,7 +6376,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # zmm1 = mem[2,2,2,3,6,6,6,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (zmm3 & (zmm1 ^ zmm0))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm3 & (zmm1 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # zmm0 = mem[2,2,2,3,6,6,6,7]
 ; AVX512DQ-FCP-NEXT:    vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Folded Reload
@@ -6385,7 +6385,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm23[2,2,2,3,6,6,6,7]
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm2 & (zmm0 ^ zmm4))
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm4 = zmm24[2,2,2,3,6,6,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm4 ^ (zmm3 & (zmm4 ^ zmm0))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (zmm3 & (zmm4 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm30[0,0,0,1,4,4,4,5]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm26[0,0,0,1,4,4,4,5]
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm0 ^ (zmm2 & (zmm3 ^ zmm0))
@@ -6398,9 +6398,9 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm2 = zmm2 ^ (zmm9 & (zmm2 ^ zmm5))
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm21[0,0,0,1,4,4,4,5]
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm3 = zmm3 ^ (zmm5 & (zmm3 ^ zmm0))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (zmm5 & (zmm3 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm28[0,0,0,1,4,4,4,5]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (zmm5 & (zmm0 ^ zmm2))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (zmm5 & (zmm0 ^ zmm2))
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, (%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, 192(%rax)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 128(%rax)
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
index 184cfa5c6ae3c..c5035aa6355af 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
@@ -1058,7 +1058,7 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[4,12],zero,ymm1[u,u,u,u,5,13],zero,ymm1[u,u,u,u,6,14,22,u,u,u,u],zero,zero,ymm1[23,u,u,u,u,u,u,u,u]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
 ; AVX512-NEXT:    vporq %zmm0, %zmm1, %zmm0
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
 ; AVX512-NEXT:    vextracti32x4 $2, %zmm0, 32(%rax)
 ; AVX512-NEXT:    vextracti32x4 $3, %zmm0, %xmm1
 ; AVX512-NEXT:    vmovq %xmm1, 48(%rax)
@@ -1140,7 +1140,7 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[4,12],zero,ymm1[u,u,u,u,5,13],zero,ymm1[u,u,u,u,6,14,22,u,u,u,u],zero,zero,ymm1[23,u,u,u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
 ; AVX512DQ-NEXT:    vporq %zmm0, %zmm1, %zmm0
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
 ; AVX512DQ-NEXT:    vextracti32x4 $2, %zmm0, 32(%rax)
 ; AVX512DQ-NEXT:    vextracti32x4 $3, %zmm0, %xmm1
 ; AVX512DQ-NEXT:    vmovq %xmm1, 48(%rax)
@@ -2065,32 +2065,28 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = ymm5[4],zero,ymm5[u,u,u,u,u,5],zero,ymm5[u,u,u,u,u,6],zero,ymm5[u,u,u,u,u],zero,ymm5[23,u,u,u,u,u],zero,ymm5[24,u,u]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm5[2,3,0,1]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm8 = zero,ymm8[4,u,u,u,u,u],zero,ymm8[5,u,u,u,u,u],zero,ymm8[6,u,u,u,u,u,23],zero,ymm8[u,u,u,u,u,24],zero,ymm8[u,u]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm9 = [255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm9 & (ymm8 | ymm7)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm8 = mem & (ymm8 | ymm7)
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm5[0,2,0,2]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,0,8],zero,ymm7[u,u,u,u,1,9],zero,ymm7[u,u,u,u,18,26],zero,ymm7[u,u,u,u,19,27],zero,ymm7[u,u,u,u]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm7, %zmm7
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm8 = xmm0[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,1,0]
-; AVX512-NEXT:    vpandn %ymm8, %ymm9, %ymm8
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm0[1,1,0,0,4,5,6,7]
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[0,1,2,0]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,1,0]
-; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm9, %ymm9
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm9, %zmm8
-; AVX512-NEXT:    vporq %zmm8, %zmm7, %zmm7
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm6))
+; AVX512-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[0,0,1,0,4,4,5,4]
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = (zmm8 & mem) | zmm7
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm6))
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[3,1,1,3]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[1],zero,zero,ymm6[u,u,u,10,2],zero,zero,ymm6[u,u,u,11,3],zero,zero,ymm6[u,u,u,20,28],zero,zero,ymm6[u,u,u,21,29],zero,zero,ymm6[u]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm4[1,3,3,1]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm8 = zero,ymm8[1,9,u,u,u],zero,zero,ymm8[2,10,u,u,u],zero,zero,ymm8[3,19,u,u,u],zero,zero,ymm8[28,20,u,u,u],zero,zero,ymm8[29,21,u]
-; AVX512-NEXT:    vpor %ymm6, %ymm8, %ymm6
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm0[0,1,2,3,4,5,5,6]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,1,0,1]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm4[1,3,3,1]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = zero,ymm7[1,9,u,u,u],zero,zero,ymm7[2,10,u,u,u],zero,zero,ymm7[3,19,u,u,u],zero,zero,ymm7[28,20,u,u,u],zero,zero,ymm7[29,21,u]
+; AVX512-NEXT:    vpor %ymm6, %ymm7, %ymm6
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm0[0,1,2,3,4,5,5,6]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,1,0,1]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[1,3,1,3]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,1,9],zero,ymm5[u,u,u,u,2,10],zero,ymm5[u,u,u,u,19,27],zero,ymm5[u,u,u,u,20,28],zero,ymm5[u,u,u,u,21]
-; AVX512-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm8 & ~mem)
+; AVX512-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm7 & ~mem)
 ; AVX512-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm5 ^ (mem & (ymm5 ^ ymm6))
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[1,3,2,3]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u],zero,zero,xmm4[6,14,u,u,u],zero,zero,xmm4[7,15,u,u,u]
@@ -2104,8 +2100,8 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpternlogq {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm3))
 ; AVX512-NEXT:    vinserti32x4 $2, %xmm0, %zmm5, %zmm1
 ; AVX512-NEXT:    vmovdqa %xmm0, 96(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm8, (%rax)
 ; AVX512-NEXT:    vmovdqa %ymm1, 64(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm7, (%rax)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
@@ -2146,7 +2142,7 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[0,4],zero,ymm7[u,u,u,u,1,5],zero,ymm7[u,u,u,u,2,6],zero,ymm7[u,u,u,u,19,23],zero,ymm7[u,u,u,u,24,28],zero,ymm7[u]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm9, %zmm7
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 | (zmm8 & mem)
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm6))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm6))
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[3,1,1,3]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[1],zero,zero,ymm6[u,u,u,10,2],zero,zero,ymm6[u,u,u,11,3],zero,zero,ymm6[u,u,u,20,28],zero,zero,ymm6[u,u,u,21,29],zero,zero,ymm6[u]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm4[1,3,3,1]
@@ -2208,32 +2204,28 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = ymm5[4],zero,ymm5[u,u,u,u,u,5],zero,ymm5[u,u,u,u,u,6],zero,ymm5[u,u,u,u,u],zero,ymm5[23,u,u,u,u,u],zero,ymm5[24,u,u]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm5[2,3,0,1]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm8 = zero,ymm8[4,u,u,u,u,u],zero,ymm8[5,u,u,u,u,u],zero,ymm8[6,u,u,u,u,u,23],zero,ymm8[u,u,u,u,u,24],zero,ymm8[u,u]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm9 = [255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm8 = ymm9 & (ymm8 | ymm7)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm8 = mem & (ymm8 | ymm7)
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm5[0,2,0,2]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,0,8],zero,ymm7[u,u,u,u,1,9],zero,ymm7[u,u,u,u,18,26],zero,ymm7[u,u,u,u,19,27],zero,ymm7[u,u,u,u]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm7, %zmm7
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm8 = xmm0[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,1,0]
-; AVX512DQ-NEXT:    vpandn %ymm8, %ymm9, %ymm8
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm0[1,1,0,0,4,5,6,7]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[0,1,2,0]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,1,0]
-; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm9, %ymm9
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm9, %zmm8
-; AVX512DQ-NEXT:    vporq %zmm8, %zmm7, %zmm7
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm6))
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[0,0,1,0,4,4,5,4]
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = (zmm8 & mem) | zmm7
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm6))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[3,1,1,3]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[1],zero,zero,ymm6[u,u,u,10,2],zero,zero,ymm6[u,u,u,11,3],zero,zero,ymm6[u,u,u,20,28],zero,zero,ymm6[u,u,u,21,29],zero,zero,ymm6[u]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm4[1,3,3,1]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm8 = zero,ymm8[1,9,u,u,u],zero,zero,ymm8[2,10,u,u,u],zero,zero,ymm8[3,19,u,u,u],zero,zero,ymm8[28,20,u,u,u],zero,zero,ymm8[29,21,u]
-; AVX512DQ-NEXT:    vpor %ymm6, %ymm8, %ymm6
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm0[0,1,2,3,4,5,5,6]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,1,0,1]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm4[1,3,3,1]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = zero,ymm7[1,9,u,u,u],zero,zero,ymm7[2,10,u,u,u],zero,zero,ymm7[3,19,u,u,u],zero,zero,ymm7[28,20,u,u,u],zero,zero,ymm7[29,21,u]
+; AVX512DQ-NEXT:    vpor %ymm6, %ymm7, %ymm6
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm0[0,1,2,3,4,5,5,6]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,1,0,1]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[1,3,1,3]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,1,9],zero,ymm5[u,u,u,u,2,10],zero,ymm5[u,u,u,u,19,27],zero,ymm5[u,u,u,u,20,28],zero,ymm5[u,u,u,u,21]
-; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm8 & ~mem)
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm5 | (ymm7 & ~mem)
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} ymm5 = ymm5 ^ (mem & (ymm5 ^ ymm6))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[1,3,2,3]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u],zero,zero,xmm4[6,14,u,u,u],zero,zero,xmm4[7,15,u,u,u]
@@ -2247,8 +2239,8 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} xmm0 = xmm0 ^ (mem & (xmm0 ^ xmm3))
 ; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm0, %zmm5, %zmm1
 ; AVX512DQ-NEXT:    vmovdqa %xmm0, 96(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm8, (%rax)
 ; AVX512DQ-NEXT:    vmovdqa %ymm1, 64(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm7, (%rax)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
@@ -2289,7 +2281,7 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[0,4],zero,ymm7[u,u,u,u,1,5],zero,ymm7[u,u,u,u,2,6],zero,ymm7[u,u,u,u,19,23],zero,ymm7[u,u,u,u,24,28],zero,ymm7[u]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm9, %zmm7
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 | (zmm8 & mem)
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm6))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm6))
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[3,1,1,3]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[1],zero,zero,ymm6[u,u,u,10,2],zero,zero,ymm6[u,u,u,11,3],zero,zero,ymm6[u,u,u,20,28],zero,zero,ymm6[u,u,u,21,29],zero,zero,ymm6[u]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm4[1,3,3,1]
@@ -4134,22 +4126,20 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm6[18],zero,zero,zero,zero,ymm6[21],zero,ymm6[19],zero,zero,zero,zero,ymm6[22],zero,ymm6[20]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm8 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm3[23],zero,zero,zero,zero,ymm3[26],zero,ymm3[24],zero,zero,zero,zero,ymm3[27],zero,ymm3[25]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm7, %zmm7
-; AVX512-NEXT:    vpermq {{.*#+}} zmm7 = zmm7[2,3,2,3,6,7,6,7]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm8 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm5[18,19,20,21],zero,ymm5[19],zero,ymm5[25,26,27,22],zero,ymm5[20],zero
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm9 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm4[23,u,u,u],zero,ymm4[26],zero,ymm4[24,u,u,u],zero,ymm4[27],zero
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm8, %zmm8
-; AVX512-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[2,3,2,3,6,7,6,7]
 ; AVX512-NEXT:    vporq %zmm7, %zmm8, %zmm7
+; AVX512-NEXT:    vpermq {{.*#+}} zmm7 = zmm7[2,3,2,3,6,7,6,7]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm8 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm4[21],zero,ymm4[19],zero,zero,zero,zero,ymm4[22],zero,ymm4[20],zero,zero
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm9 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm6[25],zero,ymm6[23],zero,zero,zero,zero,ymm6[26],zero,ymm6[24],zero,zero,zero,zero
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm8, %zmm8
-; AVX512-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[2,3,2,3,6,7,6,7]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm9 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21],zero,ymm3[19],zero,ymm3[21,20,21,22],zero,ymm3[20],zero,ymm3[22,23]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm10 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm5[23],zero,ymm5[21,22,23,26],zero,ymm5[24],zero,ymm5[28,29,26,27]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
-; AVX512-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[2,3,2,3,6,7,6,7]
 ; AVX512-NEXT:    vporq %zmm8, %zmm9, %zmm8
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm7))
+; AVX512-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[2,3,2,3,6,7,6,7]
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm7))
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,u,u,u,u,26,27,24,25]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} ymm9 = ymm0[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
 ; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm18
@@ -4158,14 +4148,13 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[20],zero,ymm2[18],zero,zero,zero,zero,ymm2[21],zero,ymm2[19],zero,zero,zero,zero,ymm2[22]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm9 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[25],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm7, %zmm7
-; AVX512-NEXT:    vpermq {{.*#+}} zmm7 = zmm7[2,3,2,3,6,7,6,7]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm9 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm1[18],zero,ymm1[20,21,20,21],zero,ymm1[19],zero,ymm1[19,20,21,22],zero
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,ymm1[23],zero,ymm1[23,24,25,26],zero,ymm1[24],zero,ymm1[30,31]
 ; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm19
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm9, %zmm9
-; AVX512-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[2,3,2,3,6,7,6,7]
 ; AVX512-NEXT:    vporq %zmm7, %zmm9, %zmm7
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm10 ^ (mem & (zmm7 ^ zmm10))
+; AVX512-NEXT:    vpermq {{.*#+}} zmm7 = zmm7[2,3,2,3,6,7,6,7]
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm10 ^ (mem & (zmm7 ^ zmm10))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm8))
 ; AVX512-NEXT:    vmovdqa (%rsi), %xmm9
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm8 = xmm9[u],zero,xmm9[7],zero,xmm9[5,u,u,u],zero,xmm9[8],zero,xmm9[6,u,u,u],zero
@@ -4185,7 +4174,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
 ; AVX512-NEXT:    vinserti32x4 $2, %xmm11, %zmm12, %zmm11
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm16 = zmm11[0,1,0,1,4,5,4,5]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm16 = zmm16 ^ (mem & (zmm16 ^ zmm8))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm16 ^ (mem & (zmm16 ^ zmm8))
 ; AVX512-NEXT:    vmovdqa (%r9), %xmm11
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm8 = zero,xmm11[4,u,u,u],zero,xmm11[7],zero,xmm11[5,u,u,u],zero,xmm11[8],zero,xmm11[6]
 ; AVX512-NEXT:    vmovdqa (%r8), %xmm12
@@ -4201,7 +4190,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,0]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm0
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm8 = zmm0[0,0,1,0,4,4,5,4]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm17))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm17))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm16))
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm4[14,u,u],zero,zero,zero,zero,ymm4[15,u,u],zero,zero,zero,zero,ymm4[16,u,u],zero,zero,zero,zero,ymm4[17,u,u],zero,zero,zero,zero,ymm4[18]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm1 = ymm3[0,1,14],zero,ymm3[u,u,0,1,14,15],zero,ymm3[u,u,13,2,3,16],zero,ymm3[u,u,28,29,16,17],zero,ymm3[u,u,19,28,29,18],zero
@@ -4217,7 +4206,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,1]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm9, %zmm1
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm0))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm0))
 ; AVX512-NEXT:    vmovdqa64 %ymm19, %ymm14
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = zero,ymm14[u],zero,zero,zero,zero,ymm14[14],zero,ymm14[u],zero,zero,zero,zero,ymm14[15],zero,ymm14[u],zero,zero,zero,zero,ymm14[16],zero,ymm14[u],zero,zero,zero,zero,ymm14[17],zero,ymm14[u],zero,zero
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm9 = ymm2[13,u,u,u,u,u],zero,ymm2[14,u,u,u,u,u],zero,ymm2[15,u,u,u,u,u],zero,ymm2[16,u,u,u,u,u],zero,ymm2[17,u,u,u]
@@ -4288,7 +4277,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
 ; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm10, %zmm13, %zmm10
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm15 = zmm10[0,1,0,1,4,5,4,5]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm15 = zmm15 ^ (mem & (zmm15 ^ zmm7))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = zmm15 ^ (mem & (zmm15 ^ zmm7))
 ; AVX512-FCP-NEXT:    vmovdqa (%r10), %xmm10
 ; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm10[1,1,0,0,4,5,6,7]
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm13 = [0,1,0,1,0,0,0,0]
@@ -4305,7 +4294,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
 ; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm7, %zmm0
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm7 = zmm0[0,1,0,1,4,5,4,5]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm16 ^ (mem & (zmm7 ^ zmm16))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm16 ^ (mem & (zmm7 ^ zmm16))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm15))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm4[14,u,u],zero,zero,zero,zero,ymm4[15,u,u],zero,zero,zero,zero,ymm4[16,u,u],zero,zero,zero,zero,ymm4[17,u,u],zero,zero,zero,zero,ymm4[18]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm3[0,1,14],zero,ymm3[u,u,0,1,14,15],zero,ymm3[u,u,13,2,3,16],zero,ymm3[u,u,28,29,16,17],zero,ymm3[u,u,19,28,29,18],zero
@@ -4321,7 +4310,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,1,0,1]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm8, %zmm9
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm0))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm0))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,ymm1[u],zero,zero,zero,zero,ymm1[14],zero,ymm1[u],zero,zero,zero,zero,ymm1[15],zero,ymm1[u],zero,zero,zero,zero,ymm1[16],zero,ymm1[u],zero,zero,zero,zero,ymm1[17],zero,ymm1[u],zero,zero
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = ymm2[13,u,u,u,u,u],zero,ymm2[14,u,u,u,u,u],zero,ymm2[15,u,u,u,u,u],zero,ymm2[16,u,u,u,u,u],zero,ymm2[17,u,u,u]
 ; AVX512-FCP-NEXT:    vpor %ymm0, %ymm8, %ymm0
@@ -4342,38 +4331,35 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm6[18],zero,zero,zero,zero,ymm6[21],zero,ymm6[19],zero,zero,zero,zero,ymm6[22],zero,ymm6[20]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm3[23],zero,zero,zero,zero,ymm3[26],zero,ymm3[24],zero,zero,zero,zero,ymm3[27],zero,ymm3[25]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm5[18,19,20,21],zero,ymm5[19],zero,ymm5[25,26,27,22],zero,ymm5[20],zero
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm4[23,u,u,u],zero,ymm4[26],zero,ymm4[24,u,u,u],zero,ymm4[27],zero
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[2,3,2,3,6,7,6,7]
 ; AVX512-FCP-NEXT:    vporq %zmm0, %zmm9, %zmm0
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm4[21],zero,ymm4[19],zero,zero,zero,zero,ymm4[22],zero,ymm4[20],zero,zero
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm6[25],zero,ymm6[23],zero,zero,zero,zero,ymm6[26],zero,ymm6[24],zero,zero,zero,zero
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[2,3,2,3,6,7,6,7]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21],zero,ymm3[19],zero,ymm3[21,20,21,22],zero,ymm3[20],zero,ymm3[22,23]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm5[23],zero,ymm5[21,22,23,26],zero,ymm5[24],zero,ymm5[28,29,26,27]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm10
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm10 = zmm10[2,3,2,3,6,7,6,7]
 ; AVX512-FCP-NEXT:    vporq %zmm9, %zmm10, %zmm9
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm0))
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[2,3,2,3,6,7,6,7]
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm0))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[20],zero,ymm2[18],zero,zero,zero,zero,ymm2[21],zero,ymm2[19],zero,zero,zero,zero,ymm2[22]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[25],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm1[18],zero,ymm1[20,21,20,21],zero,ymm1[19],zero,ymm1[19,20,21,22],zero
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,ymm1[23],zero,ymm1[23,24,25,26],zero,ymm1[24],zero,ymm1[30,31]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm10
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm10 = zmm10[2,3,2,3,6,7,6,7]
 ; AVX512-FCP-NEXT:    vporq %zmm0, %zmm10, %zmm0
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
 ; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} ymm10 = ymm12[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
 ; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm11 = [0,5,4,0,5,0,4,0]
 ; AVX512-FCP-NEXT:    vpermd %ymm10, %ymm11, %ymm10
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,3,2,3]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm10
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm0))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm0))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm9))
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm5[30],zero,ymm5[28],zero,zero,zero,zero,ymm5[31],zero,ymm5[29],zero,zero
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,u,u,u],zero,ymm6[30],zero,ymm6[28,u,u,u],zero,ymm6[31],zero,ymm6[29,u]
@@ -4413,22 +4399,20 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm6[18],zero,zero,zero,zero,ymm6[21],zero,ymm6[19],zero,zero,zero,zero,ymm6[22],zero,ymm6[20]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm8 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm3[23],zero,zero,zero,zero,ymm3[26],zero,ymm3[24],zero,zero,zero,zero,ymm3[27],zero,ymm3[25]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm7, %zmm7
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm7 = zmm7[2,3,2,3,6,7,6,7]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm8 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm5[18,19,20,21],zero,ymm5[19],zero,ymm5[25,26,27,22],zero,ymm5[20],zero
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm9 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm4[23,u,u,u],zero,ymm4[26],zero,ymm4[24,u,u,u],zero,ymm4[27],zero
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm8, %zmm8
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[2,3,2,3,6,7,6,7]
 ; AVX512DQ-NEXT:    vporq %zmm7, %zmm8, %zmm7
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm7 = zmm7[2,3,2,3,6,7,6,7]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm8 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm4[21],zero,ymm4[19],zero,zero,zero,zero,ymm4[22],zero,ymm4[20],zero,zero
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm9 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm6[25],zero,ymm6[23],zero,zero,zero,zero,ymm6[26],zero,ymm6[24],zero,zero,zero,zero
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm8, %zmm8
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[2,3,2,3,6,7,6,7]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm9 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21],zero,ymm3[19],zero,ymm3[21,20,21,22],zero,ymm3[20],zero,ymm3[22,23]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm10 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm5[23],zero,ymm5[21,22,23,26],zero,ymm5[24],zero,ymm5[28,29,26,27]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[2,3,2,3,6,7,6,7]
 ; AVX512DQ-NEXT:    vporq %zmm8, %zmm9, %zmm8
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm7))
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm8 = zmm8[2,3,2,3,6,7,6,7]
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm7))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,u,u,u,u,26,27,24,25]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm9 = ymm0[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm18
@@ -4437,14 +4421,13 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[20],zero,ymm2[18],zero,zero,zero,zero,ymm2[21],zero,ymm2[19],zero,zero,zero,zero,ymm2[22]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm9 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[25],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm7, %zmm7
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm7 = zmm7[2,3,2,3,6,7,6,7]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm9 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm1[18],zero,ymm1[20,21,20,21],zero,ymm1[19],zero,ymm1[19,20,21,22],zero
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,ymm1[23],zero,ymm1[23,24,25,26],zero,ymm1[24],zero,ymm1[30,31]
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm1, %ymm19
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm9, %zmm9
-; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[2,3,2,3,6,7,6,7]
 ; AVX512DQ-NEXT:    vporq %zmm7, %zmm9, %zmm7
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm10 ^ (mem & (zmm7 ^ zmm10))
+; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm7 = zmm7[2,3,2,3,6,7,6,7]
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm10 ^ (mem & (zmm7 ^ zmm10))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm8))
 ; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm9
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm8 = xmm9[u],zero,xmm9[7],zero,xmm9[5,u,u,u],zero,xmm9[8],zero,xmm9[6,u,u,u],zero
@@ -4464,7 +4447,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
 ; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm11, %zmm12, %zmm11
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm16 = zmm11[0,1,0,1,4,5,4,5]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm16 = zmm16 ^ (mem & (zmm16 ^ zmm8))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm16 ^ (mem & (zmm16 ^ zmm8))
 ; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm11
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm8 = zero,xmm11[4,u,u,u],zero,xmm11[7],zero,xmm11[5,u,u,u],zero,xmm11[8],zero,xmm11[6]
 ; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm12
@@ -4480,7 +4463,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,0]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm8 = zmm0[0,0,1,0,4,4,5,4]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm17))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm17))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm16))
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm4[14,u,u],zero,zero,zero,zero,ymm4[15,u,u],zero,zero,zero,zero,ymm4[16,u,u],zero,zero,zero,zero,ymm4[17,u,u],zero,zero,zero,zero,ymm4[18]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm1 = ymm3[0,1,14],zero,ymm3[u,u,0,1,14,15],zero,ymm3[u,u,13,2,3,16],zero,ymm3[u,u,28,29,16,17],zero,ymm3[u,u,19,28,29,18],zero
@@ -4496,7 +4479,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,1]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm9, %zmm1
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm0))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm1 = zmm1 ^ (mem & (zmm1 ^ zmm0))
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm19, %ymm14
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = zero,ymm14[u],zero,zero,zero,zero,ymm14[14],zero,ymm14[u],zero,zero,zero,zero,ymm14[15],zero,ymm14[u],zero,zero,zero,zero,ymm14[16],zero,ymm14[u],zero,zero,zero,zero,ymm14[17],zero,ymm14[u],zero,zero
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm9 = ymm2[13,u,u,u,u,u],zero,ymm2[14,u,u,u,u,u],zero,ymm2[15,u,u,u,u,u],zero,ymm2[16,u,u,u,u,u],zero,ymm2[17,u,u,u]
@@ -4567,7 +4550,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
 ; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm10, %zmm13, %zmm10
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm15 = zmm10[0,1,0,1,4,5,4,5]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm15 = zmm15 ^ (mem & (zmm15 ^ zmm7))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm15 = zmm15 ^ (mem & (zmm15 ^ zmm7))
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %xmm10
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm10[1,1,0,0,4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm13 = [0,1,0,1,0,0,0,0]
@@ -4584,7 +4567,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm7, %zmm0
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm7 = zmm0[0,1,0,1,4,5,4,5]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm7 = zmm16 ^ (mem & (zmm7 ^ zmm16))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm16 ^ (mem & (zmm7 ^ zmm16))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm7 = zmm7 ^ (mem & (zmm7 ^ zmm15))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm4[14,u,u],zero,zero,zero,zero,ymm4[15,u,u],zero,zero,zero,zero,ymm4[16,u,u],zero,zero,zero,zero,ymm4[17,u,u],zero,zero,zero,zero,ymm4[18]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm15 = ymm3[0,1,14],zero,ymm3[u,u,0,1,14,15],zero,ymm3[u,u,13,2,3,16],zero,ymm3[u,u,28,29,16,17],zero,ymm3[u,u,19,28,29,18],zero
@@ -4600,7 +4583,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,1,0,1]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm8, %zmm9
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm0))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,ymm1[u],zero,zero,zero,zero,ymm1[14],zero,ymm1[u],zero,zero,zero,zero,ymm1[15],zero,ymm1[u],zero,zero,zero,zero,ymm1[16],zero,ymm1[u],zero,zero,zero,zero,ymm1[17],zero,ymm1[u],zero,zero
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = ymm2[13,u,u,u,u,u],zero,ymm2[14,u,u,u,u,u],zero,ymm2[15,u,u,u,u,u],zero,ymm2[16,u,u,u,u,u],zero,ymm2[17,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpor %ymm0, %ymm8, %ymm0
@@ -4621,38 +4604,35 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm6[18],zero,zero,zero,zero,ymm6[21],zero,ymm6[19],zero,zero,zero,zero,ymm6[22],zero,ymm6[20]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm3[23],zero,zero,zero,zero,ymm3[26],zero,ymm3[24],zero,zero,zero,zero,ymm3[27],zero,ymm3[25]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,ymm5[18,19,20,21],zero,ymm5[19],zero,ymm5[25,26,27,22],zero,ymm5[20],zero
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm4[23,u,u,u],zero,ymm4[26],zero,ymm4[24,u,u,u],zero,ymm4[27],zero
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[2,3,2,3,6,7,6,7]
 ; AVX512DQ-FCP-NEXT:    vporq %zmm0, %zmm9, %zmm0
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm4[21],zero,ymm4[19],zero,zero,zero,zero,ymm4[22],zero,ymm4[20],zero,zero
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm6[25],zero,ymm6[23],zero,zero,zero,zero,ymm6[26],zero,ymm6[24],zero,zero,zero,zero
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[2,3,2,3,6,7,6,7]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21],zero,ymm3[19],zero,ymm3[21,20,21,22],zero,ymm3[20],zero,ymm3[22,23]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,ymm5[23],zero,ymm5[21,22,23,26],zero,ymm5[24],zero,ymm5[28,29,26,27]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm10
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm10 = zmm10[2,3,2,3,6,7,6,7]
 ; AVX512DQ-FCP-NEXT:    vporq %zmm9, %zmm10, %zmm9
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm0))
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm9 = zmm9[2,3,2,3,6,7,6,7]
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm9 = zmm9 ^ (mem & (zmm9 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[20],zero,ymm2[18],zero,zero,zero,zero,ymm2[21],zero,ymm2[19],zero,zero,zero,zero,ymm2[22]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm2[25],zero,ymm2[23],zero,zero,zero,zero,ymm2[26],zero,ymm2[24],zero,zero
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,ymm1[18],zero,ymm1[20,21,20,21],zero,ymm1[19],zero,ymm1[19,20,21,22],zero
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,ymm1[23],zero,ymm1[23,24,25,26],zero,ymm1[24],zero,ymm1[30,31]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm10
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm10 = zmm10[2,3,2,3,6,7,6,7]
 ; AVX512DQ-FCP-NEXT:    vporq %zmm0, %zmm10, %zmm0
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
 ; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} ymm10 = ymm12[2,1,1,2,4,5,6,7,10,9,9,10,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm11 = [0,5,4,0,5,0,4,0]
 ; AVX512DQ-FCP-NEXT:    vpermd %ymm10, %ymm11, %ymm10
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,3,2,3]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm10
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm0))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm0))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm10 = zmm10 ^ (mem & (zmm10 ^ zmm9))
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm5[30],zero,ymm5[28],zero,zero,zero,zero,ymm5[31],zero,ymm5[29],zero,zero
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,u,u,u],zero,ymm6[30],zero,ymm6[28,u,u,u],zero,ymm6[31],zero,ymm6[29,u]
@@ -8835,13 +8815,13 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm19 = zmm19 ^ (mem & (zmm19 ^ zmm12))
 ; AVX512-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Folded Reload
 ; AVX512-NEXT:    # zmm8 = mem[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm11))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm11))
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm15[0,0,1,0]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm14, %zmm11
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm11 = zmm11 ^ (mem & (zmm11 ^ zmm30))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (mem & (zmm11 ^ zmm30))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (mem & (zmm11 ^ zmm22))
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm0 = zmm31[0,1,0,1,4,5,4,5]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm28 ^ (mem & (zmm0 ^ zmm28))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm28 ^ (mem & (zmm0 ^ zmm28))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm7))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512-NEXT:    vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 64-byte Folded Reload
@@ -8853,12 +8833,12 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm1))
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm1 = zmm10[0,1,0,1,4,5,4,5]
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm5 = zmm5[0,0,1,0,4,4,5,4]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm1))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm1))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm3))
 ; AVX512-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
 ; AVX512-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7]
-; AVX512-NEXT:    vpternlogd {{.*#+}} zmm16 = zmm16 ^ (mem & (zmm16 ^ zmm1))
+; AVX512-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm16 ^ (mem & (zmm16 ^ zmm1))
 ; AVX512-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm16 ^ (mem & (zmm16 ^ zmm17))
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-NEXT:    vmovdqa64 %zmm16, 128(%rax)
@@ -9218,7 +9198,7 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm8, %zmm8
 ; AVX512-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    # zmm13 = mem[2,3,2,3,6,7,6,7]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm13 = zmm13 ^ (mem & (zmm13 ^ zmm30))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (mem & (zmm13 ^ zmm30))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm17 # 32-byte Folded Reload
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm17 = zmm17 ^ (mem & (zmm17 ^ zmm6))
@@ -9227,23 +9207,23 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm18 = zmm18 ^ (mem & (zmm18 ^ zmm17))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm31, %zmm5, %zmm3
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm5 = zmm28[0,1,0,1,4,5,4,5]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm3 ^ (mem & (zmm5 ^ zmm3))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm3 ^ (mem & (zmm5 ^ zmm3))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm20))
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm26[0,1,0,1,4,5,4,5]
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm6 = zmm9[0,1,0,1,4,5,4,5]
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm3))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm16, %zmm3
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm4 = zmm7[0,1,0,1,4,5,4,5]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm3 ^ (mem & (zmm4 ^ zmm3))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm3 ^ (mem & (zmm4 ^ zmm3))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm6))
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 64-byte Folded Reload
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm15, %zmm6
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm3[2,3,2,3,6,7,6,7]
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm3))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm3))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm1))
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm21))
+; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm21))
 ; AVX512-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm10))
 ; AVX512-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512-FCP-NEXT:    vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
@@ -9622,13 +9602,13 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm19 = zmm19 ^ (mem & (zmm19 ^ zmm12))
 ; AVX512DQ-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    # zmm8 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm11))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm8 = zmm8 ^ (mem & (zmm8 ^ zmm11))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm15[0,0,1,0]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm14, %zmm11
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm11 = zmm11 ^ (mem & (zmm11 ^ zmm30))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (mem & (zmm11 ^ zmm30))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm11 = zmm11 ^ (mem & (zmm11 ^ zmm22))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm0 = zmm31[0,1,0,1,4,5,4,5]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm28 ^ (mem & (zmm0 ^ zmm28))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm28 ^ (mem & (zmm0 ^ zmm28))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm7))
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512DQ-NEXT:    vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 64-byte Folded Reload
@@ -9640,12 +9620,12 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm3 = zmm3 ^ (mem & (zmm3 ^ zmm1))
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm1 = zmm10[0,1,0,1,4,5,4,5]
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm5 = zmm5[0,0,1,0,4,4,5,4]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm1))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm1))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm3))
 ; AVX512DQ-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512DQ-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7]
-; AVX512DQ-NEXT:    vpternlogd {{.*#+}} zmm16 = zmm16 ^ (mem & (zmm16 ^ zmm1))
+; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm16 ^ (mem & (zmm16 ^ zmm1))
 ; AVX512DQ-NEXT:    vpternlogq {{.*#+}} zmm16 = zmm16 ^ (mem & (zmm16 ^ zmm17))
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm16, 128(%rax)
@@ -10005,7 +9985,7 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm8, %zmm8
 ; AVX512DQ-FCP-NEXT:    vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    # zmm13 = mem[2,3,2,3,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm13 = zmm13 ^ (mem & (zmm13 ^ zmm30))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm13 = zmm13 ^ (mem & (zmm13 ^ zmm30))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm17 # 32-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm17 = zmm17 ^ (mem & (zmm17 ^ zmm6))
@@ -10014,23 +9994,23 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm18 = zmm18 ^ (mem & (zmm18 ^ zmm17))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm31, %zmm5, %zmm3
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm5 = zmm28[0,1,0,1,4,5,4,5]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm5 = zmm3 ^ (mem & (zmm5 ^ zmm3))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm3 ^ (mem & (zmm5 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm5 = zmm5 ^ (mem & (zmm5 ^ zmm20))
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm26[0,1,0,1,4,5,4,5]
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm6 = zmm9[0,1,0,1,4,5,4,5]
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm16, %zmm3
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm4 = zmm7[0,1,0,1,4,5,4,5]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm4 = zmm3 ^ (mem & (zmm4 ^ zmm3))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm3 ^ (mem & (zmm4 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm4 = zmm4 ^ (mem & (zmm4 ^ zmm6))
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vporq {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 64-byte Folded Reload
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm15, %zmm6
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} zmm3 = zmm3[2,3,2,3,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm3))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm3))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm6 = zmm6 ^ (mem & (zmm6 ^ zmm1))
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm21))
+; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm21))
 ; AVX512DQ-FCP-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm10))
 ; AVX512DQ-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512DQ-FCP-NEXT:    vshufi64x2 $84, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
diff --git a/llvm/test/CodeGen/X86/vector-pack-512.ll b/llvm/test/CodeGen/X86/vector-pack-512.ll
index dc60bfdca53b2..30e61a68bb22f 100644
--- a/llvm/test/CodeGen/X86/vector-pack-512.ll
+++ b/llvm/test/CodeGen/X86/vector-pack-512.ll
@@ -245,21 +245,12 @@ define <64 x i8> @concat_trunc_packuswb_512(<32 x i16> %a0, <32 x i16> %a1) noun
 }
 
 define <32 x i16> @concat_packsswd_int_2x256(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> %a2, <8 x i32> %a3) {
-; AVX512F-LABEL: concat_packsswd_int_2x256:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpackssdw %ymm3, %ymm2, %ymm1
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    retq
-;
-; AVX512BW-LABEL: concat_packsswd_int_2x256:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
-; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpackssdw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    retq
+; AVX512-LABEL: concat_packsswd_int_2x256:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpackssdw %ymm3, %ymm2, %ymm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512-NEXT:    retq
   %lo = tail call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> %a0, <8 x i32> %a1)
   %hi = tail call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> %a2, <8 x i32> %a3)
   %res = shufflevector <16 x i16> %lo, <16 x i16> %hi, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
@@ -268,21 +259,12 @@ define <32 x i16> @concat_packsswd_int_2x256(<8 x i32> %a0, <8 x i32> %a1, <8 x
 declare <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32>, <8 x i32>)
 
 define <32 x i16> @concat_packuswd_int_2x256(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> %a2, <8 x i32> %a3) {
-; AVX512F-LABEL: concat_packuswd_int_2x256:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpackusdw %ymm3, %ymm2, %ymm1
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    retq
-;
-; AVX512BW-LABEL: concat_packuswd_int_2x256:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
-; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpackusdw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    retq
+; AVX512-LABEL: concat_packuswd_int_2x256:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpackusdw %ymm3, %ymm2, %ymm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512-NEXT:    retq
   %lo = tail call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> %a0, <8 x i32> %a1)
   %hi = tail call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> %a2, <8 x i32> %a3)
   %res = shufflevector <16 x i16> %lo, <16 x i16> %hi, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
diff --git a/llvm/test/CodeGen/X86/vector-rotate-256.ll b/llvm/test/CodeGen/X86/vector-rotate-256.ll
index ec45f64bdb0e6..5879d56e1a08c 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-256.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-256.ll
@@ -584,15 +584,15 @@ define <8 x i32> @splatvar_rotate_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[2,2,3,3]
 ; AVX1-NEXT:    vpsllq %xmm1, %xmm3, %xmm3
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,2,3,3]
-; AVX1-NEXT:    vpsllq %xmm1, %xmm4, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,1,1]
 ; AVX1-NEXT:    vpsllq %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[1,3],xmm3[1,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,2,3,3]
+; AVX1-NEXT:    vpsllq %xmm1, %xmm3, %xmm3
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
 ; AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm3[1,3]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm3[1,3],ymm0[5,7],ymm3[5,7]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_rotate_v8i32:
@@ -1374,19 +1374,17 @@ define <16 x i16> @splatconstant_rotate_v16i16(<16 x i16> %a) nounwind {
 define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind {
 ; AVX1-LABEL: splatconstant_rotate_v32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm2
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm2
-; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    vbroadcastss {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX1-NEXT:    vandnps %ymm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %ymm3, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatconstant_rotate_v32i8:
@@ -1635,19 +1633,17 @@ define <16 x i16> @splatconstant_rotate_mask_v16i16(<16 x i16> %a) nounwind {
 define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind {
 ; AVX1-LABEL: splatconstant_rotate_mask_v32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm2
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm2
-; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    vbroadcastss {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX1-NEXT:    vandnps %ymm1, %ymm3, %ymm1
 ; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %ymm3, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
index ca303b4c7ebf6..d8852956c66f3 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
@@ -1424,11 +1424,9 @@ define <32 x i8> @constant_shift_v32i8_pairs(<32 x i8> %a) nounwind {
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [512,16384,4096,1024,32768,16384,8192,4096]
 ; AVX1-NEXT:    vpmulhuw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1,63,63,15,15,3,3,127,127,63,63,31,31,15,15]
-; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmulhuw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_shift_v32i8_pairs:
@@ -1487,11 +1485,9 @@ define <32 x i8> @constant_shift_v32i8_pairs(<32 x i8> %a) nounwind {
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [512,16384,4096,1024,32768,16384,8192,4096]
 ; X86-AVX1-NEXT:    vpmulhuw %xmm2, %xmm1, %xmm1
-; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1,63,63,15,15,3,3,127,127,63,63,31,31,15,15]
-; X86-AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vpmulhuw %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX1-NEXT:    retl
 ;
 ; X86-AVX2-LABEL: constant_shift_v32i8_pairs:
@@ -1508,11 +1504,10 @@ define <32 x i8> @constant_shift_v32i8_quads(<32 x i8> %a) nounwind {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmulhuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 # [8192,8192,16384,16384,32768,32768,u,u]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm0[6,7]
-; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vpmulhuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [512,512,1024,1024,2048,2048,4096,4096]
-; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_shift_v32i8_quads:
@@ -1553,11 +1548,10 @@ define <32 x i8> @constant_shift_v32i8_quads(<32 x i8> %a) nounwind {
 ; X86-AVX1:       # %bb.0:
 ; X86-AVX1-NEXT:    vpmulhuw {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm1 # [8192,8192,16384,16384,32768,32768,u,u]
 ; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm0[6,7]
-; X86-AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; X86-AVX1-NEXT:    vpmulhuw {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 # [512,512,1024,1024,2048,2048,4096,4096]
-; X86-AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX1-NEXT:    retl
 ;
 ; X86-AVX2-LABEL: constant_shift_v32i8_quads:
@@ -1865,13 +1859,11 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) nounwind {
 define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
 ; AVX1-LABEL: splatconstant_shift_v32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpsrlw $3, %xmm1, %xmm1
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatconstant_shift_v32i8:
@@ -1909,13 +1901,11 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
 ;
 ; X86-AVX1-LABEL: splatconstant_shift_v32i8:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X86-AVX1-NEXT:    vpsrlw $3, %xmm1, %xmm1
-; X86-AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
-; X86-AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; X86-AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX1-NEXT:    retl
 ;
 ; X86-AVX2-LABEL: splatconstant_shift_v32i8:
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
index c80f24ad57773..4f55f7af20f47 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
@@ -1306,21 +1306,19 @@ define <16 x i16> @constant_shift_v16i16_pairs(<16 x i16> %a) nounwind {
 define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) nounwind {
 ; AVX1-LABEL: constant_shift_v32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = [1,4,16,64,128,32,8,2]
-; AVX1-NEXT:    vpmaddubsw %xmm2, %xmm1, %xmm3
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1]
-; AVX1-NEXT:    vpmaddubsw %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1]
+; AVX1-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vpsllw $8, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpmaddubsw %xmm1, %xmm3, %xmm1
 ; AVX1-NEXT:    vpsllw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpor %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpmaddubsw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpmaddubsw %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpsllw $8, %xmm0, %xmm0
-; AVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = [1,4,16,64,128,32,8,2]
+; AVX1-NEXT:    vpmaddubsw %xmm2, %xmm3, %xmm3
+; AVX1-NEXT:    vpmaddubsw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_shift_v32i8:
@@ -1383,21 +1381,19 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) nounwind {
 ;
 ; X86-AVX1-LABEL: constant_shift_v32i8:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X86-AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = [1,4,16,64,128,32,8,2]
-; X86-AVX1-NEXT:    vpmaddubsw %xmm2, %xmm1, %xmm3
-; X86-AVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; X86-AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
-; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1]
-; X86-AVX1-NEXT:    vpmaddubsw %xmm5, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1]
+; X86-AVX1-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm2
+; X86-AVX1-NEXT:    vpsllw $8, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X86-AVX1-NEXT:    vpmaddubsw %xmm1, %xmm3, %xmm1
 ; X86-AVX1-NEXT:    vpsllw $8, %xmm1, %xmm1
-; X86-AVX1-NEXT:    vpor %xmm1, %xmm3, %xmm1
-; X86-AVX1-NEXT:    vpmaddubsw %xmm2, %xmm0, %xmm2
-; X86-AVX1-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; X86-AVX1-NEXT:    vpmaddubsw %xmm5, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpsllw $8, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
-; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; X86-AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = [1,4,16,64,128,32,8,2]
+; X86-AVX1-NEXT:    vpmaddubsw %xmm2, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpmaddubsw %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; X86-AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
+; X86-AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; X86-AVX1-NEXT:    retl
 ;
 ; X86-AVX2-LABEL: constant_shift_v32i8:
@@ -1416,11 +1412,10 @@ define <32 x i8> @constant_shift_v32i8_pairs(<32 x i8> %a) nounwind {
 ; AVX1-LABEL: constant_shift_v32i8_pairs:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 # [8,128,64,4,128,1,128,2]
-; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [32,1,16,128,64,2,16,1]
-; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_shift_v32i8_pairs:
@@ -1474,11 +1469,10 @@ define <32 x i8> @constant_shift_v32i8_pairs(<32 x i8> %a) nounwind {
 ; X86-AVX1-LABEL: constant_shift_v32i8_pairs:
 ; X86-AVX1:       # %bb.0:
 ; X86-AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm1 # [8,128,64,4,128,1,128,2]
-; X86-AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; X86-AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 # [32,1,16,128,64,2,16,1]
-; X86-AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX1-NEXT:    retl
 ;
 ; X86-AVX2-LABEL: constant_shift_v32i8_pairs:
@@ -1494,11 +1488,10 @@ define <32 x i8> @constant_shift_v32i8_quads(<32 x i8> %a) nounwind {
 ; AVX1-LABEL: constant_shift_v32i8_quads:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 # [4,4,8,8,1,1,2,2]
-; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [64,64,128,128,16,16,32,32]
-; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_shift_v32i8_quads:
@@ -1538,11 +1531,10 @@ define <32 x i8> @constant_shift_v32i8_quads(<32 x i8> %a) nounwind {
 ; X86-AVX1-LABEL: constant_shift_v32i8_quads:
 ; X86-AVX1:       # %bb.0:
 ; X86-AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm1 # [4,4,8,8,1,1,2,2]
-; X86-AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; X86-AVX1-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 # [64,64,128,128,16,16,32,32]
-; X86-AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX1-NEXT:    retl
 ;
 ; X86-AVX2-LABEL: constant_shift_v32i8_quads:
@@ -1720,13 +1712,11 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) nounwind {
 define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
 ; AVX1-LABEL: splatconstant_shift_v32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpsllw $3, %xmm1, %xmm1
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248]
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsllw $3, %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vpsllw $3, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatconstant_shift_v32i8:
@@ -1764,13 +1754,11 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
 ;
 ; X86-AVX1-LABEL: splatconstant_shift_v32i8:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X86-AVX1-NEXT:    vpsllw $3, %xmm1, %xmm1
-; X86-AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248]
-; X86-AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpsllw $3, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; X86-AVX1-NEXT:    vpsllw $3, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
 ; X86-AVX1-NEXT:    retl
 ;
 ; X86-AVX2-LABEL: splatconstant_shift_v32i8:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
index d08bfc6e2d7ea..28d637a50109d 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
@@ -5059,18 +5059,39 @@ define void @shuffle_v32i8_store_00_08_16_24_01_09_17_25_02_10_18_26_03_11_19_27
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2OR512VL-LABEL: shuffle_v32i8_store_00_08_16_24_01_09_17_25_02_10_18_26_03_11_19_27_04_12_20_28_05_13_21_29_06_14_22_30_07_15_23_31:
-; AVX2OR512VL:       # %bb.0:
-; AVX2OR512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2OR512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,8,1,9,2,10,3,11,4,12,5,13,6,14,7,15]
-; AVX2OR512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX2OR512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX2OR512VL-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX2OR512VL-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX2OR512VL-NEXT:    vmovdqa %xmm0, 16(%rdi)
-; AVX2OR512VL-NEXT:    vmovdqa %xmm2, (%rdi)
-; AVX2OR512VL-NEXT:    vzeroupper
-; AVX2OR512VL-NEXT:    retq
+; AVX2-LABEL: shuffle_v32i8_store_00_08_16_24_01_09_17_25_02_10_18_26_03_11_19_27_04_12_20_28_05_13_21_29_06_14_22_30_07_15_23_31:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,8,1,9,2,10,3,11,4,12,5,13,6,14,7,15]
+; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX2-NEXT:    vmovdqa %xmm0, 16(%rdi)
+; AVX2-NEXT:    vmovdqa %xmm2, (%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: shuffle_v32i8_store_00_08_16_24_01_09_17_25_02_10_18_26_03_11_19_27_04_12_20_28_05_13_21_29_06_14_22_30_07_15_23_31:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,8,1,9,2,10,3,11,4,12,5,13,6,14,7,15]
+; AVX512VLBW-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512VLBW-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX512VLBW-NEXT:    vmovdqa %xmm0, 16(%rdi)
+; AVX512VLBW-NEXT:    vmovdqa %xmm2, (%rdi)
+; AVX512VLBW-NEXT:    vzeroupper
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI-LABEL: shuffle_v32i8_store_00_08_16_24_01_09_17_25_02_10_18_26_03_11_19_27_04_12_20_28_05_13_21_29_06_14_22_30_07_15_23_31:
+; AVX512VLVBMI:       # %bb.0:
+; AVX512VLVBMI-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,8,16,24,1,9,17,25,2,10,18,26,3,11,19,27,4,12,20,28,5,13,21,29,6,14,22,30,7,15,23,31]
+; AVX512VLVBMI-NEXT:    vpermb %ymm0, %ymm1, %ymm0
+; AVX512VLVBMI-NEXT:    vmovdqa %ymm0, (%rdi)
+; AVX512VLVBMI-NEXT:    vzeroupper
+; AVX512VLVBMI-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: shuffle_v32i8_store_00_08_16_24_01_09_17_25_02_10_18_26_03_11_19_27_04_12_20_28_05_13_21_29_06_14_22_30_07_15_23_31:
 ; XOPAVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
index d6208aca3b2b7..fb8618be17f06 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
@@ -1682,12 +1682,19 @@ define <4 x i64> @shuffle_v4i64_0044_v2i64(<2 x i64> %a, <2 x i64> %b) {
 }
 
 define <4 x i64> @shuffle_v4i64_1032_v2i64(<2 x i64> %a, <2 x i64> %b) {
-; AVX1OR2-LABEL: shuffle_v4i64_1032_v2i64:
-; AVX1OR2:       # %bb.0:
-; AVX1OR2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX1OR2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1OR2-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[2,3,0,1,6,7,4,5]
-; AVX1OR2-NEXT:    retq
+; AVX1-LABEL: shuffle_v4i64_1032_v2i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: shuffle_v4i64_1032_v2i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[2,3,0,1,6,7,4,5]
+; AVX2-NEXT:    retq
 ;
 ; AVX512VL-SLOW-LABEL: shuffle_v4i64_1032_v2i64:
 ; AVX512VL-SLOW:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
index ec54b75513582..f09821bb189aa 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
@@ -775,6 +775,77 @@ define <32 x i8> @combine_pshufb_pshufb_or_pshufb(<32 x i8> %a0) {
   ret <32 x i8> %4
 }
 
+; Not beneficial to concatenate both inputs just to create a 256-bit vpaddb
+define <32 x i8> @concat_add_unnecessary(<16 x i8> %a0, <16 x i8> noundef %a1, <16 x i8> %a2) nounwind {
+; CHECK-LABEL: concat_add_unnecessary:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpaddb %xmm1, %xmm0, %xmm1
+; CHECK-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
+; CHECK-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %lo = add <16 x i8> %a0, %a1
+  %hi = add <16 x i8> %a0, %a2
+  %res = shufflevector <16 x i8> %lo, <16 x i8> %hi, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  ret <32 x i8> %res
+}
+
+; Not beneficial to concatenate both inputs just to create a 256-bit vpmullw
+define <16 x i16> @concat_mul_unnecessary(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> %a2) nounwind {
+; CHECK-LABEL: concat_mul_unnecessary:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmullw %xmm1, %xmm0, %xmm1
+; CHECK-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
+; CHECK-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %lo = mul <8 x i16> %a0, %a1
+  %hi = mul <8 x i16> %a0, %a2
+  %res = shufflevector <8 x i16> %lo, <8 x i16> %hi, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i16> %res
+}
+
+; Not beneficial to concatenate both inputs just to create a 256-bit palignr
+define <32 x i8> @concat_alignr_unnecessary(<16 x i8> %a0, <16 x i8> noundef %a1, <16 x i8> %a2) nounwind {
+; CHECK-LABEL: concat_alignr_unnecessary:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpalignr {{.*#+}} xmm1 = xmm1[3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2]
+; CHECK-NEXT:    vpalignr {{.*#+}} xmm0 = xmm2[3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2]
+; CHECK-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %lo = shufflevector <16 x i8> %a1, <16 x i8> %a0, <16 x i32> <i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18>
+  %hi = shufflevector <16 x i8> %a2, <16 x i8> %a0, <16 x i32> <i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18>
+  %res = shufflevector <16 x i8> %lo, <16 x i8> %hi, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  ret <32 x i8> %res
+}
+
+; Not beneficial to concatenate both inputs just to create a 256-bit packss
+define <32 x i8> @concat_packss_unnecessary(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> %a2) nounwind {
+; CHECK-LABEL: concat_packss_unnecessary:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
+; CHECK-NEXT:    vpacksswb %xmm2, %xmm0, %xmm0
+; CHECK-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %lo = tail call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> %a0, <8 x i16> %a1)
+  %hi = tail call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> %a0, <8 x i16> %a2)
+  %res = shufflevector <16 x i8> %lo, <16 x i8> %hi, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  ret <32 x i8> %res
+}
+declare <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16>, <8 x i16>)
+
+; Not beneficial to concatenate both inputs just to create a 256-bit pshufb
+define <32 x i8> @concat_pshufb_unnecessary(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2) nounwind {
+; CHECK-LABEL: concat_pshufb_unnecessary:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshufb %xmm1, %xmm0, %xmm1
+; CHECK-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; CHECK-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %lo = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %a0, <16 x i8> %a1)
+  %hi = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %a0, <16 x i8> %a2)
+  %res = shufflevector <16 x i8> %lo, <16 x i8> %hi, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  ret <32 x i8> %res
+}
+
 define <8 x i32> @constant_fold_permd() {
 ; AVX2-LABEL: constant_fold_permd:
 ; AVX2:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-v192.ll b/llvm/test/CodeGen/X86/vector-shuffle-v192.ll
index fea59d9657612..497f71aea2227 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-v192.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-v192.ll
@@ -228,7 +228,7 @@ define <64 x i8> @f2(ptr %p0) {
 ; AVX512F-NEXT:    vpor %xmm1, %xmm3, %xmm1
 ; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512F-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: f2:
@@ -511,7 +511,7 @@ define <64 x i8> @f4(ptr %p0) {
 ; AVX512F-NEXT:    vpor %xmm1, %xmm3, %xmm1
 ; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
+; AVX512F-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm2))
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: f4:
diff --git a/llvm/test/CodeGen/X86/vector-trunc-packus.ll b/llvm/test/CodeGen/X86/vector-trunc-packus.ll
index 0af5e9aeccd92..cdb1026452691 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-packus.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-packus.ll
@@ -676,26 +676,26 @@ define <8 x i32> @trunc_packus_v8i64_v8i32(ptr %p0) "min-legal-vector-width"="25
 ; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm2
 ; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm3
 ; AVX1-NEXT:    vpmovsxbd {{.*#+}} xmm4 = [4294967295,0,4294967295,0]
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm2, %xmm4, %xmm2
 ; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm5
 ; AVX1-NEXT:    vblendvpd %xmm5, %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm4, %xmm5
 ; AVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm4, %xmm1
 ; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm2, %xmm5
-; AVX1-NEXT:    vpand %xmm2, %xmm5, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm5
+; AVX1-NEXT:    vpand %xmm1, %xmm5, %xmm1
 ; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm5
 ; AVX1-NEXT:    vpand %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm3, %xmm5
-; AVX1-NEXT:    vpand %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm4
-; AVX1-NEXT:    vpand %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm3, %xmm1
+; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm2, %xmm3
+; AVX1-NEXT:    vpand %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm2[0,2],xmm1[0,2]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: trunc_packus_v8i64_v8i32:
diff --git a/llvm/test/CodeGen/X86/vector-trunc-ssat.ll b/llvm/test/CodeGen/X86/vector-trunc-ssat.ll
index 3c03c521c2722..b6557e929a5fd 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-ssat.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-ssat.ll
@@ -697,27 +697,27 @@ define <8 x i32> @trunc_ssat_v8i64_v8i32(ptr %p0) "min-legal-vector-width"="256"
 ; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm3
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [2147483647,2147483647]
 ; AVX1-NEXT:    # xmm4 = mem[0,0]
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm2, %xmm4, %xmm2
 ; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm5
 ; AVX1-NEXT:    vblendvpd %xmm5, %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm4, %xmm5
 ; AVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm4, %xmm1
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [18446744071562067968,18446744071562067968]
 ; AVX1-NEXT:    # xmm4 = mem[0,0]
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm2, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm3, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm1, %xmm5
 ; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm5
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm3, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm1, %xmm3, %xmm4, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm2, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm3, %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm2[0,2],xmm1[0,2]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: trunc_ssat_v8i64_v8i32:
diff --git a/llvm/test/CodeGen/X86/vector-trunc-usat.ll b/llvm/test/CodeGen/X86/vector-trunc-usat.ll
index c1d22dc7daf21..23af4283dc79a 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-usat.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-usat.ll
@@ -456,25 +456,25 @@ define <8 x i32> @trunc_usat_v8i64_v8i32(ptr %p0) {
 ; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm3
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm4 = [9223372036854775808,9223372036854775808]
 ; AVX1-NEXT:    # xmm4 = mem[0,0]
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm5
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm6 = [9223372041149743103,9223372041149743103]
 ; AVX1-NEXT:    # xmm6 = mem[0,0]
 ; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm5
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm7 = [4294967295,4294967295]
 ; AVX1-NEXT:    # xmm7 = mem[0,0]
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm2, %xmm7, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm7, %xmm1
 ; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm5
 ; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm5
 ; AVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm7, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm5
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm5
-; AVX1-NEXT:    vblendvpd %xmm5, %xmm3, %xmm7, %xmm3
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vblendvpd %xmm4, %xmm1, %xmm7, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm6, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm1, %xmm3, %xmm7, %xmm1
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm6, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm3, %xmm2, %xmm7, %xmm2
+; AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm2[0,2],xmm1[0,2]
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: trunc_usat_v8i64_v8i32:
diff --git a/llvm/test/CodeGen/X86/widen_bitcnt.ll b/llvm/test/CodeGen/X86/widen_bitcnt.ll
index cca9d4aa2a9f0..56001468898e4 100644
--- a/llvm/test/CodeGen/X86/widen_bitcnt.ll
+++ b/llvm/test/CodeGen/X86/widen_bitcnt.ll
@@ -241,77 +241,59 @@ define <8 x i32> @widen_ctpop_v2i32_v8i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32
 ;
 ; AVX2-LABEL: widen_ctpop_v2i32_v8i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpsrlw $4, %xmm0, %xmm4
-; AVX2-NEXT:    vpbroadcastb {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX2-NEXT:    vpand %xmm5, %xmm4, %xmm4
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; AVX2-NEXT:    vpshufb %xmm4, %xmm6, %xmm4
-; AVX2-NEXT:    vpand %xmm5, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
-; AVX2-NEXT:    vpsrlw $4, %xmm1, %xmm7
-; AVX2-NEXT:    vpand %xmm5, %xmm7, %xmm7
-; AVX2-NEXT:    vpshufb %xmm7, %xmm6, %xmm7
-; AVX2-NEXT:    vpand %xmm5, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %xmm1, %xmm6, %xmm1
-; AVX2-NEXT:    vpsrlw $4, %xmm2, %xmm8
-; AVX2-NEXT:    vpand %xmm5, %xmm8, %xmm8
-; AVX2-NEXT:    vpshufb %xmm8, %xmm6, %xmm8
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm8[0]
-; AVX2-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; AVX2-NEXT:    vpshufb %xmm2, %xmm6, %xmm2
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; AVX2-NEXT:    vpaddb %xmm0, %xmm4, %xmm0
-; AVX2-NEXT:    vpsrlw $4, %xmm3, %xmm2
-; AVX2-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; AVX2-NEXT:    vpshufb %xmm2, %xmm6, %xmm2
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm7[0],xmm2[0]
-; AVX2-NEXT:    vpand %xmm5, %xmm3, %xmm3
-; AVX2-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; AVX2-NEXT:    vpaddb %xmm1, %xmm2, %xmm1
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpsadbw %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX2-NEXT:    vpsadbw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    # kill: def $xmm1 killed $xmm1 def $ymm1
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX2-NEXT:    vpand %ymm3, %ymm1, %ymm4
+; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX2-NEXT:    # ymm5 = mem[0,1,0,1]
+; AVX2-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
+; AVX2-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
+; AVX2-NEXT:    vpaddb %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[1],ymm4[1],ymm1[4],ymm4[4],ymm1[5],ymm4[5]
+; AVX2-NEXT:    vpsadbw %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm3, %ymm0, %ymm2
+; AVX2-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
+; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm4[0],ymm0[1],ymm4[1],ymm0[4],ymm4[4],ymm0[5],ymm4[5]
+; AVX2-NEXT:    vpsadbw %ymm4, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512VL-LABEL: widen_ctpop_v2i32_v8i32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %xmm0, %xmm4
-; AVX512VL-NEXT:    vpbroadcastb {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %xmm5, %xmm4, %xmm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm6, %xmm4
-; AVX512VL-NEXT:    vpand %xmm5, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
-; AVX512VL-NEXT:    vpsrlw $4, %xmm1, %xmm7
-; AVX512VL-NEXT:    vpand %xmm5, %xmm7, %xmm7
-; AVX512VL-NEXT:    vpshufb %xmm7, %xmm6, %xmm7
-; AVX512VL-NEXT:    vpand %xmm5, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm1, %xmm6, %xmm1
-; AVX512VL-NEXT:    vpsrlw $4, %xmm2, %xmm8
-; AVX512VL-NEXT:    vpand %xmm5, %xmm8, %xmm8
-; AVX512VL-NEXT:    vpshufb %xmm8, %xmm6, %xmm8
-; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm8[0]
-; AVX512VL-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm6, %xmm2
-; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; AVX512VL-NEXT:    vpaddb %xmm0, %xmm4, %xmm0
-; AVX512VL-NEXT:    vpsrlw $4, %xmm3, %xmm2
-; AVX512VL-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm6, %xmm2
-; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm7[0],xmm2[0]
-; AVX512VL-NEXT:    vpand %xmm5, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
-; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; AVX512VL-NEXT:    vpaddb %xmm1, %xmm2, %xmm1
-; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpsadbw %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX512VL-NEXT:    vpsadbw %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    # kill: def $xmm1 killed $xmm1 def $ymm1
+; AVX512VL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512VL-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm4
+; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX512VL-NEXT:    # ymm5 = mem[0,1,0,1]
+; AVX512VL-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
+; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpshufb %ymm1, %ymm5, %ymm1
+; AVX512VL-NEXT:    vpaddb %ymm4, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VL-NEXT:    vpunpckldq {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[1],ymm4[1],ymm1[4],ymm4[4],ymm1[5],ymm4[5]
+; AVX512VL-NEXT:    vpsadbw %ymm4, %ymm1, %ymm1
+; AVX512VL-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpshufb %ymm0, %ymm5, %ymm0
+; AVX512VL-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm4[0],ymm0[1],ymm4[1],ymm0[4],ymm4[4],ymm0[5],ymm4[5]
+; AVX512VL-NEXT:    vpsadbw %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
@@ -1297,48 +1279,35 @@ define <8 x i32> @widen_cttz_v2i32_v8i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>
 ;
 ; AVX2-LABEL: widen_cttz_v2i32_v8i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpaddd %xmm4, %xmm0, %xmm5
-; AVX2-NEXT:    vpandn %xmm5, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $4, %xmm0, %xmm5
-; AVX2-NEXT:    vpbroadcastb {{.*#+}} xmm6 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX2-NEXT:    vpand %xmm6, %xmm5, %xmm5
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm7 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; AVX2-NEXT:    vpshufb %xmm5, %xmm7, %xmm5
-; AVX2-NEXT:    vpand %xmm6, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb %xmm0, %xmm7, %xmm0
-; AVX2-NEXT:    vpaddd %xmm4, %xmm1, %xmm8
-; AVX2-NEXT:    vpandn %xmm8, %xmm1, %xmm1
-; AVX2-NEXT:    vpsrlw $4, %xmm1, %xmm8
-; AVX2-NEXT:    vpand %xmm6, %xmm8, %xmm8
-; AVX2-NEXT:    vpshufb %xmm8, %xmm7, %xmm8
-; AVX2-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %xmm1, %xmm7, %xmm1
-; AVX2-NEXT:    vpaddd %xmm4, %xmm2, %xmm9
-; AVX2-NEXT:    vpandn %xmm9, %xmm2, %xmm2
-; AVX2-NEXT:    vpsrlw $4, %xmm2, %xmm9
-; AVX2-NEXT:    vpand %xmm6, %xmm9, %xmm9
-; AVX2-NEXT:    vpshufb %xmm9, %xmm7, %xmm9
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm9[0]
-; AVX2-NEXT:    vpand %xmm6, %xmm2, %xmm2
-; AVX2-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; AVX2-NEXT:    vpaddb %xmm0, %xmm5, %xmm0
-; AVX2-NEXT:    vpaddd %xmm4, %xmm3, %xmm2
-; AVX2-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; AVX2-NEXT:    vpsrlw $4, %xmm2, %xmm3
-; AVX2-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; AVX2-NEXT:    vpshufb %xmm3, %xmm7, %xmm3
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm8[0],xmm3[0]
-; AVX2-NEXT:    vpand %xmm6, %xmm2, %xmm2
-; AVX2-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX2-NEXT:    vpaddb %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpsadbw %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX2-NEXT:    vpsadbw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    # kill: def $xmm1 killed $xmm1 def $ymm1
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
+; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm4
+; AVX2-NEXT:    vpandn %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpbroadcastb {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm5
+; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX2-NEXT:    # ymm6 = mem[0,1,0,1]
+; AVX2-NEXT:    vpshufb %ymm5, %ymm6, %ymm5
+; AVX2-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpshufb %ymm1, %ymm6, %ymm1
+; AVX2-NEXT:    vpaddb %ymm5, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm1 = ymm1[0],ymm5[0],ymm1[1],ymm5[1],ymm1[4],ymm5[4],ymm1[5],ymm5[5]
+; AVX2-NEXT:    vpsadbw %ymm5, %ymm1, %ymm1
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %ymm3, %ymm0, %ymm2
+; AVX2-NEXT:    vpandn %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm2
+; AVX2-NEXT:    vpshufb %ymm2, %ymm6, %ymm2
+; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb %ymm0, %ymm6, %ymm0
+; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[1],ymm5[1],ymm0[4],ymm5[4],ymm0[5],ymm5[5]
+; AVX2-NEXT:    vpsadbw %ymm5, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
@@ -1640,48 +1609,35 @@ define <8 x i32> @widen_cttz_undef_v2i32_v8i32(<2 x i32> %a0, <2 x i32> %a1, <2
 ;
 ; AVX2-LABEL: widen_cttz_undef_v2i32_v8i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpaddd %xmm4, %xmm0, %xmm5
-; AVX2-NEXT:    vpandn %xmm5, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $4, %xmm0, %xmm5
-; AVX2-NEXT:    vpbroadcastb {{.*#+}} xmm6 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX2-NEXT:    vpand %xmm6, %xmm5, %xmm5
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm7 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; AVX2-NEXT:    vpshufb %xmm5, %xmm7, %xmm5
-; AVX2-NEXT:    vpand %xmm6, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb %xmm0, %xmm7, %xmm0
-; AVX2-NEXT:    vpaddd %xmm4, %xmm1, %xmm8
-; AVX2-NEXT:    vpandn %xmm8, %xmm1, %xmm1
-; AVX2-NEXT:    vpsrlw $4, %xmm1, %xmm8
-; AVX2-NEXT:    vpand %xmm6, %xmm8, %xmm8
-; AVX2-NEXT:    vpshufb %xmm8, %xmm7, %xmm8
-; AVX2-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %xmm1, %xmm7, %xmm1
-; AVX2-NEXT:    vpaddd %xmm4, %xmm2, %xmm9
-; AVX2-NEXT:    vpandn %xmm9, %xmm2, %xmm2
-; AVX2-NEXT:    vpsrlw $4, %xmm2, %xmm9
-; AVX2-NEXT:    vpand %xmm6, %xmm9, %xmm9
-; AVX2-NEXT:    vpshufb %xmm9, %xmm7, %xmm9
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm9[0]
-; AVX2-NEXT:    vpand %xmm6, %xmm2, %xmm2
-; AVX2-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; AVX2-NEXT:    vpaddb %xmm0, %xmm5, %xmm0
-; AVX2-NEXT:    vpaddd %xmm4, %xmm3, %xmm2
-; AVX2-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; AVX2-NEXT:    vpsrlw $4, %xmm2, %xmm3
-; AVX2-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; AVX2-NEXT:    vpshufb %xmm3, %xmm7, %xmm3
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm8[0],xmm3[0]
-; AVX2-NEXT:    vpand %xmm6, %xmm2, %xmm2
-; AVX2-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; AVX2-NEXT:    vpaddb %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpsadbw %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX2-NEXT:    vpsadbw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    # kill: def $xmm1 killed $xmm1 def $ymm1
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
+; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm4
+; AVX2-NEXT:    vpandn %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpbroadcastb {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm5
+; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX2-NEXT:    # ymm6 = mem[0,1,0,1]
+; AVX2-NEXT:    vpshufb %ymm5, %ymm6, %ymm5
+; AVX2-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpshufb %ymm1, %ymm6, %ymm1
+; AVX2-NEXT:    vpaddb %ymm5, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm1 = ymm1[0],ymm5[0],ymm1[1],ymm5[1],ymm1[4],ymm5[4],ymm1[5],ymm5[5]
+; AVX2-NEXT:    vpsadbw %ymm5, %ymm1, %ymm1
+; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %ymm3, %ymm0, %ymm2
+; AVX2-NEXT:    vpandn %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm2
+; AVX2-NEXT:    vpshufb %ymm2, %ymm6, %ymm2
+; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb %ymm0, %ymm6, %ymm0
+; AVX2-NEXT:    vpaddb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[1],ymm5[1],ymm0[4],ymm5[4],ymm0[5],ymm5[5]
+; AVX2-NEXT:    vpsadbw %ymm5, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/win32-seh-catchpad.ll b/llvm/test/CodeGen/X86/win32-seh-catchpad.ll
index fa7335cb7bf64..832ddcab5227b 100644
--- a/llvm/test/CodeGen/X86/win32-seh-catchpad.ll
+++ b/llvm/test/CodeGen/X86/win32-seh-catchpad.ll
@@ -196,6 +196,7 @@ __except:
 
 ; CHECK-LABEL: _code_in_catchpad:
 ; CHECK: # %__except.ret
+; CHECK-NEXT:         $ehgcr_4_1:
 ; CHECK-NEXT:         movl    -24(%ebp), %esp
 ; CHECK-NEXT:         addl    $12, %ebp
 ; CHECK-NEXT:         movl    $-1, -16(%ebp)
diff --git a/llvm/test/CodeGen/X86/x86-interleaved-access.ll b/llvm/test/CodeGen/X86/x86-interleaved-access.ll
index 3e76bffb77a66..edadcdadad42e 100644
--- a/llvm/test/CodeGen/X86/x86-interleaved-access.ll
+++ b/llvm/test/CodeGen/X86/x86-interleaved-access.ll
@@ -915,24 +915,44 @@ ret void
 }
 
 define void @interleaved_store_vf16_i8_stride3(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, ptr %p) nounwind {
-; AVX1OR2-LABEL: interleaved_store_vf16_i8_stride3:
-; AVX1OR2:       # %bb.0:
-; AVX1OR2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
-; AVX1OR2-NEXT:    vpalignr {{.*#+}} xmm3 = xmm1[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
-; AVX1OR2-NEXT:    vpalignr {{.*#+}} xmm4 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
-; AVX1OR2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
-; AVX1OR2-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
-; AVX1OR2-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
-; AVX1OR2-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
-; AVX1OR2-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX1OR2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
-; AVX1OR2-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX1OR2-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
-; AVX1OR2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vmovdqu %xmm0, 16(%rdi)
-; AVX1OR2-NEXT:    vmovdqu %xmm1, (%rdi)
-; AVX1OR2-NEXT:    vmovdqu %xmm2, 32(%rdi)
-; AVX1OR2-NEXT:    retq
+; AVX1-LABEL: interleaved_store_vf16_i8_stride3:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
+; AVX1-NEXT:    vpalignr {{.*#+}} xmm3 = xmm1[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
+; AVX1-NEXT:    vpalignr {{.*#+}} xmm4 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
+; AVX1-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
+; AVX1-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
+; AVX1-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
+; AVX1-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
+; AVX1-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
+; AVX1-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vmovdqu %xmm0, 16(%rdi)
+; AVX1-NEXT:    vmovdqu %xmm1, (%rdi)
+; AVX1-NEXT:    vmovdqu %xmm2, 32(%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: interleaved_store_vf16_i8_stride3:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15,0,1,2,3,4,5]
+; AVX2-NEXT:    vpalignr {{.*#+}} xmm3 = xmm1[11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,10]
+; AVX2-NEXT:    vpalignr {{.*#+}} xmm4 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
+; AVX2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm3[5,6,7,8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4]
+; AVX2-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm3[0,1,2,3,4]
+; AVX2-NEXT:    vpalignr {{.*#+}} xmm1 = xmm4[5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4]
+; AVX2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[5,6,7,8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4]
+; AVX2-NEXT:    vpalignr {{.*#+}} xmm2 = xmm2[5,6,7,8,9,10,11,12,13,14,15],xmm4[0,1,2,3,4]
+; AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5]
+; AVX2-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX2-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vmovdqu %xmm2, 32(%rdi)
+; AVX2-NEXT:    vmovdqu %ymm0, (%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: interleaved_store_vf16_i8_stride3:
 ; AVX512:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
index 409d889a9cbc3..35f25d36cb2e9 100644
--- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
+++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
@@ -3652,13 +3652,11 @@ define void @vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12(ptr %i
 ; AVX512BW-FAST-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
 ; AVX512BW-FAST:       # %bb.0:
 ; AVX512BW-FAST-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512BW-FAST-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,25,0,27,0,29,0,31,0,41,0,43,0,45,0,47]
 ; AVX512BW-FAST-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-FAST-NEXT:    vpermi2w %zmm2, %zmm0, %zmm1
-; AVX512BW-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1],zero,zero,xmm0[0,1],zero,zero,xmm0[0,1],zero,zero,xmm0[0,1],zero,zero
-; AVX512BW-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm0
-; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
+; AVX512BW-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-FAST-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,25,0,27,0,29,0,31,0,41,0,43,0,45,0,47,0,49,0,51,0,53,0,55,0,0,0,0,0,0,0,0]
+; AVX512BW-FAST-NEXT:    vpermi2w %zmm1, %zmm0, %zmm2
+; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm2, %zmm0
 ; AVX512BW-FAST-NEXT:    vmovdqa64 %zmm0, (%rcx)
 ; AVX512BW-FAST-NEXT:    vzeroupper
 ; AVX512BW-FAST-NEXT:    retq
@@ -3857,13 +3855,11 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
 ; AVX512BW-FAST-LABEL: vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8:
 ; AVX512BW-FAST:       # %bb.0:
 ; AVX512BW-FAST-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512BW-FAST-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,25,26,0,28,29,0,31,40,0,42,43,0,45,46,0]
 ; AVX512BW-FAST-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-FAST-NEXT:    vpermi2w %zmm2, %zmm0, %zmm1
-; AVX512BW-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1],zero,zero,zero,zero,xmm0[0,1],zero,zero,zero,zero
-; AVX512BW-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm0
-; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
+; AVX512BW-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-FAST-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,25,26,0,28,29,0,31,40,0,42,43,0,45,46,0,48,49,0,51,52,0,54,55,0,0,0,0,0,0,0,0]
+; AVX512BW-FAST-NEXT:    vpermi2w %zmm1, %zmm0, %zmm2
+; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm2, %zmm0
 ; AVX512BW-FAST-NEXT:    vmovdqa64 %zmm0, (%rcx)
 ; AVX512BW-FAST-NEXT:    vzeroupper
 ; AVX512BW-FAST-NEXT:    retq
@@ -4085,13 +4081,11 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ; AVX512BW-FAST-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
 ; AVX512BW-FAST:       # %bb.0:
 ; AVX512BW-FAST-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512BW-FAST-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,25,26,27,0,29,30,31,0,41,42,43,0,45,46,47]
 ; AVX512BW-FAST-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-FAST-NEXT:    vpermi2w %zmm2, %zmm0, %zmm1
-; AVX512BW-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1],zero,zero,zero,zero,zero,zero,xmm0[0,1],zero,zero,zero,zero,zero,zero
-; AVX512BW-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm0
-; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
+; AVX512BW-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-FAST-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,25,26,27,0,29,30,31,0,41,42,43,0,45,46,47,0,49,50,51,0,53,54,55,0,0,0,0,0,0,0,0]
+; AVX512BW-FAST-NEXT:    vpermi2w %zmm1, %zmm0, %zmm2
+; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm2, %zmm0
 ; AVX512BW-FAST-NEXT:    vmovdqa64 %zmm0, (%rcx)
 ; AVX512BW-FAST-NEXT:    vzeroupper
 ; AVX512BW-FAST-NEXT:    retq
@@ -4292,13 +4286,11 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
 ; AVX512BW-FAST-LABEL: vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4:
 ; AVX512BW-FAST:       # %bb.0:
 ; AVX512BW-FAST-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512BW-FAST-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [0,25,26,27,28,29,0,31,40,41,42,43,0,45,46,47]
 ; AVX512BW-FAST-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-FAST-NEXT:    vpermi2w %zmm2, %zmm0, %zmm1
-; AVX512BW-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm0
-; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
+; AVX512BW-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-FAST-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,25,26,27,28,29,0,31,40,41,42,43,0,45,46,47,48,49,0,51,52,53,54,55,0,0,0,0,0,0,0,0]
+; AVX512BW-FAST-NEXT:    vpermi2w %zmm1, %zmm0, %zmm2
+; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm2, %zmm0
 ; AVX512BW-FAST-NEXT:    vmovdqa64 %zmm0, (%rcx)
 ; AVX512BW-FAST-NEXT:    vzeroupper
 ; AVX512BW-FAST-NEXT:    retq
@@ -5101,32 +5093,17 @@ define void @vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3(ptr %i
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
-; AVX512BW-SLOW-LABEL: vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3:
-; AVX512BW-SLOW:       # %bb.0:
-; AVX512BW-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512BW-SLOW-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [0,13,14,15,0,21,22,23]
-; AVX512BW-SLOW-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-SLOW-NEXT:    vpermi2d %zmm2, %zmm0, %zmm1
-; AVX512BW-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7]
-; AVX512BW-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512BW-SLOW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
-; AVX512BW-SLOW-NEXT:    vmovdqa64 %zmm0, (%rcx)
-; AVX512BW-SLOW-NEXT:    vzeroupper
-; AVX512BW-SLOW-NEXT:    retq
-;
-; AVX512BW-FAST-LABEL: vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3:
-; AVX512BW-FAST:       # %bb.0:
-; AVX512BW-FAST-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512BW-FAST-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-FAST-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [0,13,14,15,0,21,22,23,0,25,26,27,28,29,30,31]
-; AVX512BW-FAST-NEXT:    vpermi2d %zmm1, %zmm0, %zmm2
-; AVX512BW-FAST-NEXT:    vpaddb (%rdx), %zmm2, %zmm0
-; AVX512BW-FAST-NEXT:    vmovdqa64 %zmm0, (%rcx)
-; AVX512BW-FAST-NEXT:    vzeroupper
-; AVX512BW-FAST-NEXT:    retq
+; AVX512BW-LABEL: vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512BW-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovsxbd {{.*#+}} zmm2 = [0,13,14,15,0,21,22,23,0,25,26,27,28,29,30,31]
+; AVX512BW-NEXT:    vpermi2d %zmm1, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpaddb (%rdx), %zmm2, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rcx)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %in.vec.base = load <64 x i8>, ptr %in.vec.base.ptr, align 64
   %in.vec.bias = load <64 x i8>, ptr %in.vec.bias.ptr, align 64
   %in.vec = add <64 x i8> %in.vec.base, %in.vec.bias
@@ -5381,13 +5358,11 @@ define void @vec384_i64_widen_to_i128_factor2_broadcast_to_v3i128_factor3(ptr %i
 ; AVX512BW-SLOW-LABEL: vec384_i64_widen_to_i128_factor2_broadcast_to_v3i128_factor3:
 ; AVX512BW-SLOW:       # %bb.0:
 ; AVX512BW-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512BW-SLOW-NEXT:    vpmovsxbq {{.*#+}} ymm1 = [0,7,0,11]
 ; AVX512BW-SLOW-NEXT:    vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
-; AVX512BW-SLOW-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
-; AVX512BW-SLOW-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm0
-; AVX512BW-SLOW-NEXT:    vpaddb (%rdx), %zmm0, %zmm0
+; AVX512BW-SLOW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-SLOW-NEXT:    vpmovsxbq {{.*#+}} zmm2 = [0,7,0,11,0,13,0,0]
+; AVX512BW-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
+; AVX512BW-SLOW-NEXT:    vpaddb (%rdx), %zmm2, %zmm0
 ; AVX512BW-SLOW-NEXT:    vmovdqa64 %zmm0, (%rcx)
 ; AVX512BW-SLOW-NEXT:    vzeroupper
 ; AVX512BW-SLOW-NEXT:    retq
diff --git a/llvm/test/CodeGen/XCore/dwarf_debug.ll b/llvm/test/CodeGen/XCore/dwarf_debug.ll
index 43851130c08f4..084d0fca86320 100644
--- a/llvm/test/CodeGen/XCore/dwarf_debug.ll
+++ b/llvm/test/CodeGen/XCore/dwarf_debug.ll
@@ -6,7 +6,7 @@
 ; CHECK-LABEL: f
 ; CHECK: entsp [[S:[0-9]+]]
 ; ...the prologue...
-; CHECK: .loc 1 2 0 prologue_end      # test.c:2:0
+; CHECK: .loc 1 2 0 prologue_end      # test.c:2
 ; CHECK: add r0, r0, 1
 ; CHECK: retsp [[S]]
 define i32 @f(i32 %a) !dbg !4 {
diff --git a/llvm/test/DebugInfo/Generic/subrange_type.ll b/llvm/test/DebugInfo/Generic/subrange_type.ll
new file mode 100644
index 0000000000000..28fd70eab5d72
--- /dev/null
+++ b/llvm/test/DebugInfo/Generic/subrange_type.ll
@@ -0,0 +1,39 @@
+;; Check output of DISubrangeType.
+
+; RUN: %llc_dwarf -filetype=obj -O0 < %s | llvm-dwarfdump -debug-info - | FileCheck %s
+
+; CHECK: DW_TAG_array_type
+; CHECK-TYPE: DW_AT_type{{.*}}"unsigned int"
+; CHECK-NOT: NULL
+; CHECK: DW_TAG_subrange_type
+; CHECK-NOT: NULL
+; CHECK: DW_AT_lower_bound (-7)
+; CHECK-NEXT: DW_AT_upper_bound (0)
+; CHECK-NEXT: DW_AT_bit_stride (6)
+
+; ModuleID = 'subrange_type.ll'
+source_filename = "/dir/subrange_type.adb"
+
+!llvm.module.flags = !{!0, !1}
+!llvm.dbg.cu = !{!2}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = !{i32 2, !"Dwarf Version", i32 4}
+!2 = distinct !DICompileUnit(language: DW_LANG_Ada95, file: !3, producer: "GNAT/LLVM", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !5, globals: !4, imports: !4)
+!3 = !DIFile(filename: "subrange_type.adb", directory: "/dir")
+!4 = !{}
+!5 = !{!29}
+!6 = distinct !DISubprogram(name: "sr", scope: !3, file: !3, line: 1, type: !7, scopeLine: 1, spFlags: DISPFlagDefinition, unit: !2, retainedNodes: !9)
+!7 = !DISubroutineType(types: !8)
+!8 = !{null}
+!9 = !{!10}
+!10 = !DILocalVariable(name: "x", scope: !6, file: !3, line: 3, type: !31, align: 32)
+!11 = !DISubrangeType(name: "sr__int_range", file: !3, line: 2, size: 32, align: 32, baseType: !12, lowerBound: i64 -7, upperBound: i64 0, stride: 6)
+!12 = !DIBasicType(name: "sr__Tint_rangeB", size: 32, encoding: DW_ATE_signed)
+!13 = !DILocation(line: 3, column: 4, scope: !6)
+!14 = !DILocation(line: 6, column: 5, scope: !6)
+
+
+!29 = !DICompositeType(tag: DW_TAG_array_type, size: 64, align: 32, file: !3, scope: !6, baseType: !31, elements: !32)
+!31 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!32 = !{!11}
diff --git a/llvm/test/DebugInfo/X86/inline-seldag-test.ll b/llvm/test/DebugInfo/X86/inline-seldag-test.ll
index 6417612a54580..6a62d2bab884b 100644
--- a/llvm/test/DebugInfo/X86/inline-seldag-test.ll
+++ b/llvm/test/DebugInfo/X86/inline-seldag-test.ll
@@ -18,8 +18,8 @@
 ; Make sure the condition test is attributed to the inline function, not the
 ; location of the test's operands within the caller.
 
-; ASM: # inline-seldag-test.c:2:0
-; ASM-NOT: .loc
+; ASM: # inline-seldag-test.c:4:0
+; ASM: .loc 1 2 0 # inline-seldag-test.c:2 @[ inline-seldag-test.c:6:7 ]
 ; ASM: testl
 
 ; Function Attrs: nounwind uwtable
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-smaxv.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-smaxv.ll
index cbbd55d3e3497..632268e08022c 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-smaxv.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-smaxv.ll
@@ -2,9 +2,6 @@
 ; RUN: opt < %s -passes=msan -S | FileCheck %s
 ;
 ; Forked from llvm/test/CodeGen/AArch64/arm64-uminv.ll
-;
-; Handled suboptimally (visitInstruction):
-; - llvm.aarch64.neon.smaxv
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android9001"
@@ -15,16 +12,12 @@ define signext i8 @test_vmaxv_s8(<8 x i8> %a1) #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i8> [[TMP0]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1:![0-9]+]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3:[0-9]+]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP1]] to i32
 ; CHECK-NEXT:    [[VMAXV_I:%.*]] = tail call i32 @llvm.aarch64.neon.smaxv.i32.v8i8(<8 x i8> [[A1]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc i32 [[TMP2]] to i8
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[VMAXV_I]] to i8
-; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store i8 [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i8 [[TMP4]]
 ;
 entry:
@@ -39,16 +32,12 @@ define signext i16 @test_vmaxv_s16(<4 x i16> %a1) #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i32
 ; CHECK-NEXT:    [[VMAXV_I:%.*]] = tail call i32 @llvm.aarch64.neon.smaxv.i32.v4i16(<4 x i16> [[A1]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc i32 [[TMP2]] to i16
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[VMAXV_I]] to i16
-; CHECK-NEXT:    store i16 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store i16 [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i16 [[TMP4]]
 ;
 entry:
@@ -63,15 +52,9 @@ define i32 @test_vmaxv_s32(<2 x i32> %a1) #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP0]])
 ; CHECK-NEXT:    [[VMAXV_I:%.*]] = tail call i32 @llvm.aarch64.neon.smaxv.i32.v2i32(<2 x i32> [[A1]])
-; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store i32 [[TMP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 [[VMAXV_I]]
 ;
 entry:
@@ -85,16 +68,12 @@ define signext i8 @test_vmaxvq_s8(<16 x i8> %a1) #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP1]] to i32
 ; CHECK-NEXT:    [[VMAXV_I:%.*]] = tail call i32 @llvm.aarch64.neon.smaxv.i32.v16i8(<16 x i8> [[A1]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc i32 [[TMP2]] to i8
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[VMAXV_I]] to i8
-; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store i8 [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i8 [[TMP4]]
 ;
 entry:
@@ -109,16 +88,12 @@ define signext i16 @test_vmaxvq_s16(<8 x i16> %a1) #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i32
 ; CHECK-NEXT:    [[VMAXV_I:%.*]] = tail call i32 @llvm.aarch64.neon.smaxv.i32.v8i16(<8 x i16> [[A1]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc i32 [[TMP2]] to i16
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[VMAXV_I]] to i16
-; CHECK-NEXT:    store i16 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store i16 [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i16 [[TMP4]]
 ;
 entry:
@@ -133,15 +108,9 @@ define i32 @test_vmaxvq_s32(<4 x i32> %a1) #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP0]])
 ; CHECK-NEXT:    [[VMAXV_I:%.*]] = tail call i32 @llvm.aarch64.neon.smaxv.i32.v4i32(<4 x i32> [[A1]])
-; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store i32 [[TMP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 [[VMAXV_I]]
 ;
 entry:
@@ -156,16 +125,12 @@ define <8 x i8> @test_vmaxv_s8_used_by_laneop(<8 x i8> %a1, <8 x i8> %a2) #0 {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[TMP0]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.smaxv.i32.v8i8(<8 x i8> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i8
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i8> [[TMP1]], i8 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i8> [[TMP1]], i8 [[_MSPROP1]], i32 3
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x i8> [[A1]], i8 [[TMP6]], i32 3
 ; CHECK-NEXT:    store <8 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i8> [[TMP7]]
@@ -184,16 +149,12 @@ define <4 x i16> @test_vmaxv_s16_used_by_laneop(<4 x i16> %a1, <4 x i16> %a2) #0
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i16> [[TMP0]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.smaxv.i32.v4i16(<4 x i16> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i16
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i16> [[TMP1]], i16 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i16> [[TMP1]], i16 [[_MSPROP1]], i32 3
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i16> [[A1]], i16 [[TMP6]], i32 3
 ; CHECK-NEXT:    store <4 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i16> [[TMP7]]
@@ -212,15 +173,9 @@ define <2 x i32> @test_vmaxv_s32_used_by_laneop(<2 x i32> %a1, <2 x i32> %a2) #0
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP0]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP0]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.smaxv.i32.v2i32(<2 x i32> [[A2]])
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i32> [[TMP1]], i32 0, i32 1
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[TMP2]], i32 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[A1]], i32 [[TMP5]], i32 1
 ; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i32> [[TMP6]]
@@ -238,16 +193,12 @@ define <16 x i8> @test_vmaxvq_s8_used_by_laneop(<16 x i8> %a1, <16 x i8> %a2) #0
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.smaxv.i32.v16i8(<16 x i8> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i8
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i8> [[TMP1]], i8 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i8> [[TMP1]], i8 [[_MSPROP1]], i32 3
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <16 x i8> [[A1]], i8 [[TMP6]], i32 3
 ; CHECK-NEXT:    store <16 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i8> [[TMP7]]
@@ -266,16 +217,12 @@ define <8 x i16> @test_vmaxvq_s16_used_by_laneop(<8 x i16> %a1, <8 x i16> %a2) #
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.smaxv.i32.v8i16(<8 x i16> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i16
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> [[TMP1]], i16 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> [[TMP1]], i16 [[_MSPROP1]], i32 3
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x i16> [[A1]], i16 [[TMP6]], i32 3
 ; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i16> [[TMP7]]
@@ -294,15 +241,9 @@ define <4 x i32> @test_vmaxvq_s32_used_by_laneop(<4 x i32> %a1, <4 x i32> %a2) #
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP0]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.smaxv.i32.v4i32(<4 x i32> [[A2]])
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP1]], i32 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[TMP2]], i32 3
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[A1]], i32 [[TMP5]], i32 3
 ; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i32> [[TMP6]]
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-sminv.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-sminv.ll
index b4f3cce44c306..267061027cd52 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-sminv.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-sminv.ll
@@ -2,9 +2,6 @@
 ; RUN: opt < %s -passes=msan -S | FileCheck %s
 ;
 ; Forked from llvm/test/CodeGen/AArch64/arm64-uminv.ll
-;
-; Handled suboptimally (visitInstruction):
-; - llvm.aarch64.neon.sminv
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android9001"
@@ -15,16 +12,12 @@ define signext i8 @test_vminv_s8(<8 x i8> %a1) #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i8> [[TMP0]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1:![0-9]+]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3:[0-9]+]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP1]] to i32
 ; CHECK-NEXT:    [[VMINV_I:%.*]] = tail call i32 @llvm.aarch64.neon.sminv.i32.v8i8(<8 x i8> [[A1]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc i32 [[TMP2]] to i8
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[VMINV_I]] to i8
-; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store i8 [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i8 [[TMP4]]
 ;
 entry:
@@ -39,16 +32,12 @@ define signext i16 @test_vminv_s16(<4 x i16> %a1) #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i32
 ; CHECK-NEXT:    [[VMINV_I:%.*]] = tail call i32 @llvm.aarch64.neon.sminv.i32.v4i16(<4 x i16> [[A1]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc i32 [[TMP2]] to i16
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[VMINV_I]] to i16
-; CHECK-NEXT:    store i16 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store i16 [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i16 [[TMP4]]
 ;
 entry:
@@ -63,15 +52,9 @@ define i32 @test_vminv_s32(<2 x i32> %a1) #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP0]])
 ; CHECK-NEXT:    [[VMINV_I:%.*]] = tail call i32 @llvm.aarch64.neon.sminv.i32.v2i32(<2 x i32> [[A1]])
-; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store i32 [[TMP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 [[VMINV_I]]
 ;
 entry:
@@ -85,16 +68,12 @@ define signext i8 @test_vminvq_s8(<16 x i8> %a1) #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP1]] to i32
 ; CHECK-NEXT:    [[VMINV_I:%.*]] = tail call i32 @llvm.aarch64.neon.sminv.i32.v16i8(<16 x i8> [[A1]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc i32 [[TMP2]] to i8
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[VMINV_I]] to i8
-; CHECK-NEXT:    store i8 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store i8 [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i8 [[TMP4]]
 ;
 entry:
@@ -109,16 +88,12 @@ define signext i16 @test_vminvq_s16(<8 x i16> %a1) #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i32
 ; CHECK-NEXT:    [[VMINV_I:%.*]] = tail call i32 @llvm.aarch64.neon.sminv.i32.v8i16(<8 x i16> [[A1]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = trunc i32 [[TMP2]] to i16
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[VMINV_I]] to i16
-; CHECK-NEXT:    store i16 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store i16 [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i16 [[TMP4]]
 ;
 entry:
@@ -133,15 +108,9 @@ define i32 @test_vminvq_s32(<4 x i32> %a1) #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP0]])
 ; CHECK-NEXT:    [[VMINV_I:%.*]] = tail call i32 @llvm.aarch64.neon.sminv.i32.v4i32(<4 x i32> [[A1]])
-; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store i32 [[TMP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 [[VMINV_I]]
 ;
 entry:
@@ -156,16 +125,12 @@ define <8 x i8> @test_vminv_s8_used_by_laneop(<8 x i8> %a1, <8 x i8> %a2) #0 {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[TMP0]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.sminv.i32.v8i8(<8 x i8> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i8
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i8> [[TMP1]], i8 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i8> [[TMP1]], i8 [[_MSPROP1]], i32 3
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x i8> [[A1]], i8 [[TMP6]], i32 3
 ; CHECK-NEXT:    store <8 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i8> [[TMP7]]
@@ -184,16 +149,12 @@ define <4 x i16> @test_vminv_s16_used_by_laneop(<4 x i16> %a1, <4 x i16> %a2) #0
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i16> [[TMP0]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.sminv.i32.v4i16(<4 x i16> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i16
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i16> [[TMP1]], i16 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i16> [[TMP1]], i16 [[_MSPROP1]], i32 3
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i16> [[A1]], i16 [[TMP6]], i32 3
 ; CHECK-NEXT:    store <4 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i16> [[TMP7]]
@@ -212,15 +173,9 @@ define <2 x i32> @test_vminv_s32_used_by_laneop(<2 x i32> %a1, <2 x i32> %a2) #0
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP0]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP0]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.sminv.i32.v2i32(<2 x i32> [[A2]])
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i32> [[TMP1]], i32 0, i32 1
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[TMP2]], i32 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[A1]], i32 [[TMP5]], i32 1
 ; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i32> [[TMP6]]
@@ -238,16 +193,12 @@ define <16 x i8> @test_vminvq_s8_used_by_laneop(<16 x i8> %a1, <16 x i8> %a2) #0
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.sminv.i32.v16i8(<16 x i8> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i8
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i8> [[TMP1]], i8 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i8> [[TMP1]], i8 [[_MSPROP1]], i32 3
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <16 x i8> [[A1]], i8 [[TMP6]], i32 3
 ; CHECK-NEXT:    store <16 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i8> [[TMP7]]
@@ -266,16 +217,12 @@ define <8 x i16> @test_vminvq_s16_used_by_laneop(<8 x i16> %a1, <8 x i16> %a2) #
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.sminv.i32.v8i16(<8 x i16> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i16
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> [[TMP1]], i16 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> [[TMP1]], i16 [[_MSPROP1]], i32 3
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x i16> [[A1]], i16 [[TMP6]], i32 3
 ; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i16> [[TMP7]]
@@ -294,15 +241,9 @@ define <4 x i32> @test_vminvq_s32_used_by_laneop(<4 x i32> %a1, <4 x i32> %a2) #
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR3]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP0]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.sminv.i32.v4i32(<4 x i32> [[A2]])
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP1]], i32 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[TMP2]], i32 3
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[A1]], i32 [[TMP5]], i32 3
 ; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i32> [[TMP6]]
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-umaxv.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-umaxv.ll
index a2af44e1862d1..95f11a05d9d2d 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-umaxv.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-umaxv.ll
@@ -2,14 +2,6 @@
 ; RUN: opt < %s -passes=msan -S | FileCheck %s
 ;
 ; Forked from llvm/test/CodeGen/AArch64/arm64-umaxv.ll
-;
-; Handled suboptimally (visitInstruction):
-; - llvm.aarch64.neon.umaxv.i32.v16i8
-; - llvm.aarch64.neon.umaxv.i32.v2i32
-; - llvm.aarch64.neon.umaxv.i32.v4i16
-; - llvm.aarch64.neon.umaxv.i32.v4i32
-; - llvm.aarch64.neon.umaxv.i32.v8i16
-; - llvm.aarch64.neon.umaxv.i32.v8i8
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android9001"
@@ -20,25 +12,24 @@ define i32 @vmax_u8x8(<8 x i8> %a) nounwind ssp #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP3]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1:![0-9]+]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    [[VMAXV_I:%.*]] = tail call i32 @llvm.aarch64.neon.umaxv.i32.v8i8(<8 x i8> [[A]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[TMP3]])
+; CHECK-NEXT:    [[VMAXV_I:%.*]] = zext i8 [[TMP7]] to i32
+; CHECK-NEXT:    [[VMAXV_I1:%.*]] = tail call i32 @llvm.aarch64.neon.umaxv.i32.v8i8(<8 x i8> [[A]]) #[[ATTR3:[0-9]+]]
 ; CHECK-NEXT:    [[TMP:%.*]] = trunc i32 [[VMAXV_I]] to i8
-; CHECK-NEXT:    [[TMP0:%.*]] = xor i8 [[TMP]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i8 -1, [[TMP0]]
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i32 [[VMAXV_I1]] to i8
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i8 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = or i8 [[TMP]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i8 [[TMP4]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[TMP6]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 0
-; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 false, [[TMP2]]
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i8 [[TMP]], 0
-; CHECK-NEXT:    br i1 [[_MSPROP_ICMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
-; CHECK:       [[BB7]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i8 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSPROP_ICMP]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5:[0-9]+]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       [[BB8]]:
+; CHECK:       [[BB10]]:
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label %[[RETURN:.*]], label %[[IF_THEN:.*]]
 ; CHECK:       [[IF_THEN]]:
 ; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
@@ -46,8 +37,8 @@ define i32 @vmax_u8x8(<8 x i8> %a) nounwind ssp #0 {
 ; CHECK-NEXT:    [[_MSRET:%.*]] = load i32, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    br label %[[RETURN]]
 ; CHECK:       [[RETURN]]:
-; CHECK-NEXT:    [[_MSPHI_S:%.*]] = phi i32 [ [[_MSRET]], %[[IF_THEN]] ], [ 0, %[[BB8]] ]
-; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[CALL1]], %[[IF_THEN]] ], [ 0, %[[BB8]] ]
+; CHECK-NEXT:    [[_MSPHI_S:%.*]] = phi i32 [ [[_MSRET]], %[[IF_THEN]] ], [ 0, %[[BB10]] ]
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[CALL1]], %[[IF_THEN]] ], [ 0, %[[BB10]] ]
 ; CHECK-NEXT:    store i32 [[_MSPHI_S]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 [[RETVAL_0]]
 ;
@@ -74,25 +65,24 @@ define i32 @vmax_u4x16(<4 x i16> %a) nounwind ssp #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP3]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    [[VMAXV_I:%.*]] = tail call i32 @llvm.aarch64.neon.umaxv.i32.v4i16(<4 x i16> [[A]]) #[[ATTR3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP3]])
+; CHECK-NEXT:    [[VMAXV_I:%.*]] = zext i16 [[TMP7]] to i32
+; CHECK-NEXT:    [[VMAXV_I1:%.*]] = tail call i32 @llvm.aarch64.neon.umaxv.i32.v4i16(<4 x i16> [[A]]) #[[ATTR3]]
 ; CHECK-NEXT:    [[TMP:%.*]] = trunc i32 [[VMAXV_I]] to i16
-; CHECK-NEXT:    [[TMP0:%.*]] = xor i16 [[TMP]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i16 -1, [[TMP0]]
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i32 [[VMAXV_I1]] to i16
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i16 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = or i16 [[TMP]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i16 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i16 [[TMP4]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[TMP6]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i16 [[TMP1]], 0
-; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 false, [[TMP2]]
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 [[TMP]], 0
-; CHECK-NEXT:    br i1 [[_MSPROP_ICMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
-; CHECK:       [[BB7]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSPROP_ICMP]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       [[BB8]]:
+; CHECK:       [[BB10]]:
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label %[[RETURN:.*]], label %[[IF_THEN:.*]]
 ; CHECK:       [[IF_THEN]]:
 ; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
@@ -100,8 +90,8 @@ define i32 @vmax_u4x16(<4 x i16> %a) nounwind ssp #0 {
 ; CHECK-NEXT:    [[_MSRET:%.*]] = load i32, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    br label %[[RETURN]]
 ; CHECK:       [[RETURN]]:
-; CHECK-NEXT:    [[_MSPHI_S:%.*]] = phi i32 [ [[_MSRET]], %[[IF_THEN]] ], [ 0, %[[BB8]] ]
-; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[CALL1]], %[[IF_THEN]] ], [ 0, %[[BB8]] ]
+; CHECK-NEXT:    [[_MSPHI_S:%.*]] = phi i32 [ [[_MSRET]], %[[IF_THEN]] ], [ 0, %[[BB10]] ]
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[CALL1]], %[[IF_THEN]] ], [ 0, %[[BB10]] ]
 ; CHECK-NEXT:    store i32 [[_MSPHI_S]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 [[RETVAL_0]]
 ;
@@ -126,25 +116,24 @@ define i32 @vmax_u8x16(<8 x i16> %a) nounwind ssp #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    [[VMAXV_I:%.*]] = tail call i32 @llvm.aarch64.neon.umaxv.i32.v8i16(<8 x i16> [[A]]) #[[ATTR3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[TMP3]])
+; CHECK-NEXT:    [[VMAXV_I:%.*]] = zext i16 [[TMP7]] to i32
+; CHECK-NEXT:    [[VMAXV_I1:%.*]] = tail call i32 @llvm.aarch64.neon.umaxv.i32.v8i16(<8 x i16> [[A]]) #[[ATTR3]]
 ; CHECK-NEXT:    [[TMP:%.*]] = trunc i32 [[VMAXV_I]] to i16
-; CHECK-NEXT:    [[TMP0:%.*]] = xor i16 [[TMP]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i16 -1, [[TMP0]]
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i32 [[VMAXV_I1]] to i16
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i16 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = or i16 [[TMP]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i16 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i16 [[TMP4]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[TMP6]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i16 [[TMP1]], 0
-; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 false, [[TMP2]]
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 [[TMP]], 0
-; CHECK-NEXT:    br i1 [[_MSPROP_ICMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
-; CHECK:       [[BB7]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSPROP_ICMP]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       [[BB8]]:
+; CHECK:       [[BB10]]:
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label %[[RETURN:.*]], label %[[IF_THEN:.*]]
 ; CHECK:       [[IF_THEN]]:
 ; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
@@ -152,8 +141,8 @@ define i32 @vmax_u8x16(<8 x i16> %a) nounwind ssp #0 {
 ; CHECK-NEXT:    [[_MSRET:%.*]] = load i32, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    br label %[[RETURN]]
 ; CHECK:       [[RETURN]]:
-; CHECK-NEXT:    [[_MSPHI_S:%.*]] = phi i32 [ [[_MSRET]], %[[IF_THEN]] ], [ 0, %[[BB8]] ]
-; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[CALL1]], %[[IF_THEN]] ], [ 0, %[[BB8]] ]
+; CHECK-NEXT:    [[_MSPHI_S:%.*]] = phi i32 [ [[_MSRET]], %[[IF_THEN]] ], [ 0, %[[BB10]] ]
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[CALL1]], %[[IF_THEN]] ], [ 0, %[[BB10]] ]
 ; CHECK-NEXT:    store i32 [[_MSPHI_S]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 [[RETVAL_0]]
 ;
@@ -178,25 +167,24 @@ define i32 @vmax_u16x8(<16 x i8> %a) nounwind ssp #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i8> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    [[VMAXV_I:%.*]] = tail call i32 @llvm.aarch64.neon.umaxv.i32.v16i8(<16 x i8> [[A]]) #[[ATTR3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[TMP3]])
+; CHECK-NEXT:    [[VMAXV_I:%.*]] = zext i8 [[TMP7]] to i32
+; CHECK-NEXT:    [[VMAXV_I1:%.*]] = tail call i32 @llvm.aarch64.neon.umaxv.i32.v16i8(<16 x i8> [[A]]) #[[ATTR3]]
 ; CHECK-NEXT:    [[TMP:%.*]] = trunc i32 [[VMAXV_I]] to i8
-; CHECK-NEXT:    [[TMP0:%.*]] = xor i8 [[TMP]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i8 -1, [[TMP0]]
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i32 [[VMAXV_I1]] to i8
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i8 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = or i8 [[TMP]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i8 [[TMP4]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[TMP6]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 0
-; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 false, [[TMP2]]
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i8 [[TMP]], 0
-; CHECK-NEXT:    br i1 [[_MSPROP_ICMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
-; CHECK:       [[BB7]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i8 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSPROP_ICMP]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       [[BB8]]:
+; CHECK:       [[BB10]]:
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label %[[RETURN:.*]], label %[[IF_THEN:.*]]
 ; CHECK:       [[IF_THEN]]:
 ; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
@@ -204,8 +192,8 @@ define i32 @vmax_u16x8(<16 x i8> %a) nounwind ssp #0 {
 ; CHECK-NEXT:    [[_MSRET:%.*]] = load i32, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    br label %[[RETURN]]
 ; CHECK:       [[RETURN]]:
-; CHECK-NEXT:    [[_MSPHI_S:%.*]] = phi i32 [ [[_MSRET]], %[[IF_THEN]] ], [ 0, %[[BB8]] ]
-; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[CALL1]], %[[IF_THEN]] ], [ 0, %[[BB8]] ]
+; CHECK-NEXT:    [[_MSPHI_S:%.*]] = phi i32 [ [[_MSRET]], %[[IF_THEN]] ], [ 0, %[[BB10]] ]
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[CALL1]], %[[IF_THEN]] ], [ 0, %[[BB10]] ]
 ; CHECK-NEXT:    store i32 [[_MSPHI_S]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 [[RETVAL_0]]
 ;
@@ -231,16 +219,12 @@ define <8 x i8> @test_vmaxv_u8_used_by_laneop(<8 x i8> %a1, <8 x i8> %a2) #0 {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[TMP0]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.umaxv.i32.v8i8(<8 x i8> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i8
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i8> [[TMP1]], i8 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i8> [[TMP1]], i8 [[_MSPROP1]], i32 3
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x i8> [[A1]], i8 [[TMP6]], i32 3
 ; CHECK-NEXT:    store <8 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i8> [[TMP7]]
@@ -259,16 +243,12 @@ define <4 x i16> @test_vmaxv_u16_used_by_laneop(<4 x i16> %a1, <4 x i16> %a2) #0
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i16> [[TMP0]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.umaxv.i32.v4i16(<4 x i16> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i16
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i16> [[TMP1]], i16 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i16> [[TMP1]], i16 [[_MSPROP1]], i32 3
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i16> [[A1]], i16 [[TMP6]], i32 3
 ; CHECK-NEXT:    store <4 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i16> [[TMP7]]
@@ -287,15 +267,9 @@ define <2 x i32> @test_vmaxv_u32_used_by_laneop(<2 x i32> %a1, <2 x i32> %a2) #0
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP0]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP0]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.umaxv.i32.v2i32(<2 x i32> [[A2]])
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i32> [[TMP1]], i32 0, i32 1
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[TMP2]], i32 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[A1]], i32 [[TMP5]], i32 1
 ; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i32> [[TMP6]]
@@ -313,16 +287,12 @@ define <16 x i8> @test_vmaxvq_u8_used_by_laneop(<16 x i8> %a1, <16 x i8> %a2) #0
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.umaxv.i32.v16i8(<16 x i8> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i8
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i8> [[TMP1]], i8 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i8> [[TMP1]], i8 [[_MSPROP1]], i32 3
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <16 x i8> [[A1]], i8 [[TMP6]], i32 3
 ; CHECK-NEXT:    store <16 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i8> [[TMP7]]
@@ -341,16 +311,12 @@ define <8 x i16> @test_vmaxvq_u16_used_by_laneop(<8 x i16> %a1, <8 x i16> %a2) #
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.umaxv.i32.v8i16(<8 x i16> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i16
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> [[TMP1]], i16 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> [[TMP1]], i16 [[_MSPROP1]], i32 3
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x i16> [[A1]], i16 [[TMP6]], i32 3
 ; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i16> [[TMP7]]
@@ -369,15 +335,9 @@ define <4 x i32> @test_vmaxvq_u32_used_by_laneop(<4 x i32> %a1, <4 x i32> %a2) #
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP0]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.umaxv.i32.v4i32(<4 x i32> [[A2]])
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP1]], i32 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[TMP2]], i32 3
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[A1]], i32 [[TMP5]], i32 3
 ; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i32> [[TMP6]]
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-uminv.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-uminv.ll
index 67038f495be7b..ad51395691050 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-uminv.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-uminv.ll
@@ -2,14 +2,6 @@
 ; RUN: opt < %s -passes=msan -S | FileCheck %s
 ;
 ; Forked from llvm/test/CodeGen/AArch64/arm64-uminv.ll
-;
-; Handled suboptimally (visitInstruction):
-; - llvm.aarch64.neon.uminv.i32.v16i8
-; - llvm.aarch64.neon.uminv.i32.v2i32
-; - llvm.aarch64.neon.uminv.i32.v4i16
-; - llvm.aarch64.neon.uminv.i32.v4i32
-; - llvm.aarch64.neon.uminv.i32.v8i16
-; - llvm.aarch64.neon.uminv.i32.v8i8
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android9001"
@@ -20,25 +12,24 @@ define i32 @vmin_u8x8(<8 x i8> %a) nounwind ssp #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i8> [[TMP3]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1:![0-9]+]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    [[VMINV_I:%.*]] = tail call i32 @llvm.aarch64.neon.uminv.i32.v8i8(<8 x i8> [[A]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[TMP3]])
+; CHECK-NEXT:    [[VMINV_I:%.*]] = zext i8 [[TMP7]] to i32
+; CHECK-NEXT:    [[VMINV_I1:%.*]] = tail call i32 @llvm.aarch64.neon.uminv.i32.v8i8(<8 x i8> [[A]]) #[[ATTR3:[0-9]+]]
 ; CHECK-NEXT:    [[TMP:%.*]] = trunc i32 [[VMINV_I]] to i8
-; CHECK-NEXT:    [[TMP0:%.*]] = xor i8 [[TMP]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i8 -1, [[TMP0]]
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i32 [[VMINV_I1]] to i8
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i8 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = or i8 [[TMP]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i8 [[TMP4]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[TMP6]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 0
-; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 false, [[TMP2]]
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i8 [[TMP]], 0
-; CHECK-NEXT:    br i1 [[_MSPROP_ICMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
-; CHECK:       [[BB7]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i8 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSPROP_ICMP]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5:[0-9]+]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       [[BB8]]:
+; CHECK:       [[BB10]]:
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label %[[RETURN:.*]], label %[[IF_THEN:.*]]
 ; CHECK:       [[IF_THEN]]:
 ; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
@@ -46,8 +37,8 @@ define i32 @vmin_u8x8(<8 x i8> %a) nounwind ssp #0 {
 ; CHECK-NEXT:    [[_MSRET:%.*]] = load i32, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    br label %[[RETURN]]
 ; CHECK:       [[RETURN]]:
-; CHECK-NEXT:    [[_MSPHI_S:%.*]] = phi i32 [ [[_MSRET]], %[[IF_THEN]] ], [ 0, %[[BB8]] ]
-; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[CALL1]], %[[IF_THEN]] ], [ 0, %[[BB8]] ]
+; CHECK-NEXT:    [[_MSPHI_S:%.*]] = phi i32 [ [[_MSRET]], %[[IF_THEN]] ], [ 0, %[[BB10]] ]
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[CALL1]], %[[IF_THEN]] ], [ 0, %[[BB10]] ]
 ; CHECK-NEXT:    store i32 [[_MSPHI_S]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 [[RETVAL_0]]
 ;
@@ -74,25 +65,24 @@ define i32 @vmin_u4x16(<4 x i16> %a) nounwind ssp #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP3]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP4]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    [[VMINV_I:%.*]] = tail call i32 @llvm.aarch64.neon.uminv.i32.v4i16(<4 x i16> [[A]]) #[[ATTR3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP3]])
+; CHECK-NEXT:    [[VMINV_I:%.*]] = zext i16 [[TMP7]] to i32
+; CHECK-NEXT:    [[VMINV_I1:%.*]] = tail call i32 @llvm.aarch64.neon.uminv.i32.v4i16(<4 x i16> [[A]]) #[[ATTR3]]
 ; CHECK-NEXT:    [[TMP:%.*]] = trunc i32 [[VMINV_I]] to i16
-; CHECK-NEXT:    [[TMP0:%.*]] = xor i16 [[TMP]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i16 -1, [[TMP0]]
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i32 [[VMINV_I1]] to i16
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i16 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = or i16 [[TMP]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i16 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i16 [[TMP4]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[TMP6]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i16 [[TMP1]], 0
-; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 false, [[TMP2]]
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 [[TMP]], 0
-; CHECK-NEXT:    br i1 [[_MSPROP_ICMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
-; CHECK:       [[BB7]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSPROP_ICMP]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       [[BB8]]:
+; CHECK:       [[BB10]]:
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label %[[RETURN:.*]], label %[[IF_THEN:.*]]
 ; CHECK:       [[IF_THEN]]:
 ; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
@@ -100,8 +90,8 @@ define i32 @vmin_u4x16(<4 x i16> %a) nounwind ssp #0 {
 ; CHECK-NEXT:    [[_MSRET:%.*]] = load i32, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    br label %[[RETURN]]
 ; CHECK:       [[RETURN]]:
-; CHECK-NEXT:    [[_MSPHI_S:%.*]] = phi i32 [ [[_MSRET]], %[[IF_THEN]] ], [ 0, %[[BB8]] ]
-; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[CALL1]], %[[IF_THEN]] ], [ 0, %[[BB8]] ]
+; CHECK-NEXT:    [[_MSPHI_S:%.*]] = phi i32 [ [[_MSRET]], %[[IF_THEN]] ], [ 0, %[[BB10]] ]
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[CALL1]], %[[IF_THEN]] ], [ 0, %[[BB10]] ]
 ; CHECK-NEXT:    store i32 [[_MSPHI_S]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 [[RETVAL_0]]
 ;
@@ -126,25 +116,24 @@ define i32 @vmin_u8x16(<8 x i16> %a) nounwind ssp #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    [[VMINV_I:%.*]] = tail call i32 @llvm.aarch64.neon.uminv.i32.v8i16(<8 x i16> [[A]]) #[[ATTR3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[TMP3]])
+; CHECK-NEXT:    [[VMINV_I:%.*]] = zext i16 [[TMP7]] to i32
+; CHECK-NEXT:    [[VMINV_I1:%.*]] = tail call i32 @llvm.aarch64.neon.uminv.i32.v8i16(<8 x i16> [[A]]) #[[ATTR3]]
 ; CHECK-NEXT:    [[TMP:%.*]] = trunc i32 [[VMINV_I]] to i16
-; CHECK-NEXT:    [[TMP0:%.*]] = xor i16 [[TMP]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i16 -1, [[TMP0]]
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i32 [[VMINV_I1]] to i16
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i16 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = or i16 [[TMP]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i16 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i16 [[TMP4]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[TMP6]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i16 [[TMP1]], 0
-; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 false, [[TMP2]]
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 [[TMP]], 0
-; CHECK-NEXT:    br i1 [[_MSPROP_ICMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
-; CHECK:       [[BB7]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSPROP_ICMP]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       [[BB8]]:
+; CHECK:       [[BB10]]:
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label %[[RETURN:.*]], label %[[IF_THEN:.*]]
 ; CHECK:       [[IF_THEN]]:
 ; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
@@ -152,8 +141,8 @@ define i32 @vmin_u8x16(<8 x i16> %a) nounwind ssp #0 {
 ; CHECK-NEXT:    [[_MSRET:%.*]] = load i32, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    br label %[[RETURN]]
 ; CHECK:       [[RETURN]]:
-; CHECK-NEXT:    [[_MSPHI_S:%.*]] = phi i32 [ [[_MSRET]], %[[IF_THEN]] ], [ 0, %[[BB8]] ]
-; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[CALL1]], %[[IF_THEN]] ], [ 0, %[[BB8]] ]
+; CHECK-NEXT:    [[_MSPHI_S:%.*]] = phi i32 [ [[_MSRET]], %[[IF_THEN]] ], [ 0, %[[BB10]] ]
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[CALL1]], %[[IF_THEN]] ], [ 0, %[[BB10]] ]
 ; CHECK-NEXT:    store i32 [[_MSPHI_S]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 [[RETVAL_0]]
 ;
@@ -178,25 +167,24 @@ define i32 @vmin_u16x8(<16 x i8> %a) nounwind ssp #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i8> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    [[VMINV_I:%.*]] = tail call i32 @llvm.aarch64.neon.uminv.i32.v16i8(<16 x i8> [[A]]) #[[ATTR3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[TMP3]])
+; CHECK-NEXT:    [[VMINV_I:%.*]] = zext i8 [[TMP7]] to i32
+; CHECK-NEXT:    [[VMINV_I1:%.*]] = tail call i32 @llvm.aarch64.neon.uminv.i32.v16i8(<16 x i8> [[A]]) #[[ATTR3]]
 ; CHECK-NEXT:    [[TMP:%.*]] = trunc i32 [[VMINV_I]] to i8
-; CHECK-NEXT:    [[TMP0:%.*]] = xor i8 [[TMP]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = and i8 -1, [[TMP0]]
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i32 [[VMINV_I1]] to i8
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i8 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = or i8 [[TMP]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i8 [[TMP4]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[TMP6]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 0
-; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 false, [[TMP2]]
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i8 [[TMP]], 0
-; CHECK-NEXT:    br i1 [[_MSPROP_ICMP]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
-; CHECK:       [[BB7]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    [[_MSPROP_ICMP:%.*]] = and i1 [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i8 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[_MSPROP_ICMP]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       [[BB8]]:
+; CHECK:       [[BB10]]:
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label %[[RETURN:.*]], label %[[IF_THEN:.*]]
 ; CHECK:       [[IF_THEN]]:
 ; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
@@ -204,8 +192,8 @@ define i32 @vmin_u16x8(<16 x i8> %a) nounwind ssp #0 {
 ; CHECK-NEXT:    [[_MSRET:%.*]] = load i32, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    br label %[[RETURN]]
 ; CHECK:       [[RETURN]]:
-; CHECK-NEXT:    [[_MSPHI_S:%.*]] = phi i32 [ [[_MSRET]], %[[IF_THEN]] ], [ 0, %[[BB8]] ]
-; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[CALL1]], %[[IF_THEN]] ], [ 0, %[[BB8]] ]
+; CHECK-NEXT:    [[_MSPHI_S:%.*]] = phi i32 [ [[_MSRET]], %[[IF_THEN]] ], [ 0, %[[BB10]] ]
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[CALL1]], %[[IF_THEN]] ], [ 0, %[[BB10]] ]
 ; CHECK-NEXT:    store i32 [[_MSPHI_S]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 [[RETVAL_0]]
 ;
@@ -231,16 +219,12 @@ define <8 x i8> @test_vminv_u8_used_by_laneop(<8 x i8> %a1, <8 x i8> %a2) #0 {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[TMP0]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.uminv.i32.v8i8(<8 x i8> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i8
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i8> [[TMP1]], i8 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i8> [[TMP1]], i8 [[_MSPROP1]], i32 3
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x i8> [[A1]], i8 [[TMP6]], i32 3
 ; CHECK-NEXT:    store <8 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i8> [[TMP7]]
@@ -259,16 +243,12 @@ define <4 x i16> @test_vminv_u16_used_by_laneop(<4 x i16> %a1, <4 x i16> %a2) #0
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i16> [[TMP0]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.uminv.i32.v4i16(<4 x i16> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i16
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i16> [[TMP1]], i16 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i16> [[TMP1]], i16 [[_MSPROP1]], i32 3
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i16> [[A1]], i16 [[TMP6]], i32 3
 ; CHECK-NEXT:    store <4 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i16> [[TMP7]]
@@ -287,15 +267,9 @@ define <2 x i32> @test_vminv_u32_used_by_laneop(<2 x i32> %a1, <2 x i32> %a2) #0
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[TMP0]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP0]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.uminv.i32.v2i32(<2 x i32> [[A2]])
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i32> [[TMP1]], i32 0, i32 1
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[TMP2]], i32 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[A1]], i32 [[TMP5]], i32 1
 ; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i32> [[TMP6]]
@@ -313,16 +287,12 @@ define <16 x i8> @test_vminvq_u8_used_by_laneop(<16 x i8> %a1, <16 x i8> %a2) #0
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.uminv.i32.v16i8(<16 x i8> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i8
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i8> [[TMP1]], i8 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <16 x i8> [[TMP1]], i8 [[_MSPROP1]], i32 3
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <16 x i8> [[A1]], i8 [[TMP6]], i32 3
 ; CHECK-NEXT:    store <16 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i8> [[TMP7]]
@@ -341,16 +311,12 @@ define <8 x i16> @test_vminvq_u16_used_by_laneop(<8 x i16> %a1, <8 x i16> %a2) #
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[TMP0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.uminv.i32.v8i16(<8 x i16> [[A2]])
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = trunc i32 [[TMP3]] to i16
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc i32 [[TMP5]] to i16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> [[TMP1]], i16 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <8 x i16> [[TMP1]], i16 [[_MSPROP1]], i32 3
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x i16> [[A1]], i16 [[TMP6]], i32 3
 ; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i16> [[TMP7]]
@@ -369,15 +335,9 @@ define <4 x i32> @test_vminvq_u32_used_by_laneop(<4 x i32> %a1, <4 x i32> %a2) #
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP2]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB3:.*]], label %[[BB4:.*]], !prof [[PROF1]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP0]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.aarch64.neon.uminv.i32.v4i32(<4 x i32> [[A2]])
-; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP1]], i32 0, i32 3
+; CHECK-NEXT:    [[_MSPROP:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[TMP2]], i32 3
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[A1]], i32 [[TMP5]], i32 3
 ; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i32> [[TMP6]]
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vmax.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vmax.ll
index a46a5b1c747db..e2457c0a51d46 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vmax.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-vmax.ll
@@ -3,7 +3,8 @@
 ;
 ; Forked from llvm/test/CodeGen/AArch64/arm64-vmax.ll
 ;
-; Pairwise instructions which are handled incorrectly by heuristics:
+; Pairwise instructions which are explicitly handled by
+; handlePairwiseShadowOrIntrinsic:
 ; - llvm.aarch64.neon.fmaxp (floating-point maximum pairwise)
 ; - llvm.aarch64.neon.fminp
 ; - llvm.aarch64.neon.fmaxnmp (floating-point maximum number pairwise)
@@ -1017,7 +1018,9 @@ define <8 x i8> @smaxp_8b(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i8>, ptr [[TMP12]], align 8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i8> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <8 x i8> [[_MSLD]], <8 x i8> [[_MSLD1]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <8 x i8> [[_MSLD]], <8 x i8> [[_MSLD1]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i8> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <8 x i8> @llvm.aarch64.neon.smaxp.v8i8(<8 x i8> [[TMPVAR1]], <8 x i8> [[TMPVAR2]])
 ; CHECK-NEXT:    store <8 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i8> [[TMPVAR3]]
@@ -1056,7 +1059,9 @@ define <16 x i8> @smaxp_16b(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <16 x i8>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i8> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <16 x i8> [[_MSLD]], <16 x i8> [[_MSLD1]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <16 x i8> [[_MSLD]], <16 x i8> [[_MSLD1]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i8> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <16 x i8> @llvm.aarch64.neon.smaxp.v16i8(<16 x i8> [[TMPVAR1]], <16 x i8> [[TMPVAR2]])
 ; CHECK-NEXT:    store <16 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i8> [[TMPVAR3]]
@@ -1095,7 +1100,9 @@ define <4 x i16> @smaxp_4h(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP12]], align 8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i16> [[_MSLD]], <4 x i16> [[_MSLD1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i16> [[_MSLD]], <4 x i16> [[_MSLD1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i16> @llvm.aarch64.neon.smaxp.v4i16(<4 x i16> [[TMPVAR1]], <4 x i16> [[TMPVAR2]])
 ; CHECK-NEXT:    store <4 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i16> [[TMPVAR3]]
@@ -1134,7 +1141,9 @@ define <8 x i16> @smaxp_8h(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <8 x i16> [[_MSLD]], <8 x i16> [[_MSLD1]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <8 x i16> [[_MSLD]], <8 x i16> [[_MSLD1]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <8 x i16> @llvm.aarch64.neon.smaxp.v8i16(<8 x i16> [[TMPVAR1]], <8 x i16> [[TMPVAR2]])
 ; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i16> [[TMPVAR3]]
@@ -1173,7 +1182,9 @@ define <2 x i32> @smaxp_2s(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i32> [[_MSLD]], <2 x i32> [[_MSLD1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x i32> [[_MSLD]], <2 x i32> [[_MSLD1]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i32> @llvm.aarch64.neon.smaxp.v2i32(<2 x i32> [[TMPVAR1]], <2 x i32> [[TMPVAR2]])
 ; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i32> [[TMPVAR3]]
@@ -1212,7 +1223,9 @@ define <4 x i32> @smaxp_4s(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> [[_MSLD1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> [[_MSLD1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i32> @llvm.aarch64.neon.smaxp.v4i32(<4 x i32> [[TMPVAR1]], <4 x i32> [[TMPVAR2]])
 ; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i32> [[TMPVAR3]]
@@ -1258,7 +1271,9 @@ define <8 x i8> @umaxp_8b(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i8>, ptr [[TMP12]], align 8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i8> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <8 x i8> [[_MSLD]], <8 x i8> [[_MSLD1]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <8 x i8> [[_MSLD]], <8 x i8> [[_MSLD1]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i8> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <8 x i8> @llvm.aarch64.neon.umaxp.v8i8(<8 x i8> [[TMPVAR1]], <8 x i8> [[TMPVAR2]])
 ; CHECK-NEXT:    store <8 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i8> [[TMPVAR3]]
@@ -1297,7 +1312,9 @@ define <16 x i8> @umaxp_16b(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <16 x i8>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i8> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <16 x i8> [[_MSLD]], <16 x i8> [[_MSLD1]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <16 x i8> [[_MSLD]], <16 x i8> [[_MSLD1]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i8> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <16 x i8> @llvm.aarch64.neon.umaxp.v16i8(<16 x i8> [[TMPVAR1]], <16 x i8> [[TMPVAR2]])
 ; CHECK-NEXT:    store <16 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i8> [[TMPVAR3]]
@@ -1336,7 +1353,9 @@ define <4 x i16> @umaxp_4h(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP12]], align 8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i16> [[_MSLD]], <4 x i16> [[_MSLD1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i16> [[_MSLD]], <4 x i16> [[_MSLD1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i16> @llvm.aarch64.neon.umaxp.v4i16(<4 x i16> [[TMPVAR1]], <4 x i16> [[TMPVAR2]])
 ; CHECK-NEXT:    store <4 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i16> [[TMPVAR3]]
@@ -1375,7 +1394,9 @@ define <8 x i16> @umaxp_8h(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <8 x i16> [[_MSLD]], <8 x i16> [[_MSLD1]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <8 x i16> [[_MSLD]], <8 x i16> [[_MSLD1]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <8 x i16> @llvm.aarch64.neon.umaxp.v8i16(<8 x i16> [[TMPVAR1]], <8 x i16> [[TMPVAR2]])
 ; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i16> [[TMPVAR3]]
@@ -1414,7 +1435,9 @@ define <2 x i32> @umaxp_2s(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i32> [[_MSLD]], <2 x i32> [[_MSLD1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x i32> [[_MSLD]], <2 x i32> [[_MSLD1]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i32> @llvm.aarch64.neon.umaxp.v2i32(<2 x i32> [[TMPVAR1]], <2 x i32> [[TMPVAR2]])
 ; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i32> [[TMPVAR3]]
@@ -1453,7 +1476,9 @@ define <4 x i32> @umaxp_4s(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> [[_MSLD1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> [[_MSLD1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i32> @llvm.aarch64.neon.umaxp.v4i32(<4 x i32> [[TMPVAR1]], <4 x i32> [[TMPVAR2]])
 ; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i32> [[TMPVAR3]]
@@ -1500,7 +1525,9 @@ define <8 x i8> @sminp_8b(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i8>, ptr [[TMP12]], align 8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i8> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <8 x i8> [[_MSLD]], <8 x i8> [[_MSLD1]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <8 x i8> [[_MSLD]], <8 x i8> [[_MSLD1]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i8> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <8 x i8> @llvm.aarch64.neon.sminp.v8i8(<8 x i8> [[TMPVAR1]], <8 x i8> [[TMPVAR2]])
 ; CHECK-NEXT:    store <8 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i8> [[TMPVAR3]]
@@ -1539,7 +1566,9 @@ define <16 x i8> @sminp_16b(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <16 x i8>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i8> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <16 x i8> [[_MSLD]], <16 x i8> [[_MSLD1]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <16 x i8> [[_MSLD]], <16 x i8> [[_MSLD1]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i8> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <16 x i8> @llvm.aarch64.neon.sminp.v16i8(<16 x i8> [[TMPVAR1]], <16 x i8> [[TMPVAR2]])
 ; CHECK-NEXT:    store <16 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i8> [[TMPVAR3]]
@@ -1578,7 +1607,9 @@ define <4 x i16> @sminp_4h(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP12]], align 8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i16> [[_MSLD]], <4 x i16> [[_MSLD1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i16> [[_MSLD]], <4 x i16> [[_MSLD1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i16> @llvm.aarch64.neon.sminp.v4i16(<4 x i16> [[TMPVAR1]], <4 x i16> [[TMPVAR2]])
 ; CHECK-NEXT:    store <4 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i16> [[TMPVAR3]]
@@ -1617,7 +1648,9 @@ define <8 x i16> @sminp_8h(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <8 x i16> [[_MSLD]], <8 x i16> [[_MSLD1]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <8 x i16> [[_MSLD]], <8 x i16> [[_MSLD1]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <8 x i16> @llvm.aarch64.neon.sminp.v8i16(<8 x i16> [[TMPVAR1]], <8 x i16> [[TMPVAR2]])
 ; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i16> [[TMPVAR3]]
@@ -1656,7 +1689,9 @@ define <2 x i32> @sminp_2s(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i32> [[_MSLD]], <2 x i32> [[_MSLD1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x i32> [[_MSLD]], <2 x i32> [[_MSLD1]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i32> @llvm.aarch64.neon.sminp.v2i32(<2 x i32> [[TMPVAR1]], <2 x i32> [[TMPVAR2]])
 ; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i32> [[TMPVAR3]]
@@ -1695,7 +1730,9 @@ define <4 x i32> @sminp_4s(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> [[_MSLD1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> [[_MSLD1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i32> @llvm.aarch64.neon.sminp.v4i32(<4 x i32> [[TMPVAR1]], <4 x i32> [[TMPVAR2]])
 ; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i32> [[TMPVAR3]]
@@ -1741,7 +1778,9 @@ define <8 x i8> @uminp_8b(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i8>, ptr [[TMP12]], align 8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i8> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <8 x i8> [[_MSLD]], <8 x i8> [[_MSLD1]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <8 x i8> [[_MSLD]], <8 x i8> [[_MSLD1]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i8> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <8 x i8> @llvm.aarch64.neon.uminp.v8i8(<8 x i8> [[TMPVAR1]], <8 x i8> [[TMPVAR2]])
 ; CHECK-NEXT:    store <8 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i8> [[TMPVAR3]]
@@ -1780,7 +1819,9 @@ define <16 x i8> @uminp_16b(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <16 x i8>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i8> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <16 x i8> [[_MSLD]], <16 x i8> [[_MSLD1]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <16 x i8> [[_MSLD]], <16 x i8> [[_MSLD1]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i8> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <16 x i8> @llvm.aarch64.neon.uminp.v16i8(<16 x i8> [[TMPVAR1]], <16 x i8> [[TMPVAR2]])
 ; CHECK-NEXT:    store <16 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i8> [[TMPVAR3]]
@@ -1819,7 +1860,9 @@ define <4 x i16> @uminp_4h(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i16>, ptr [[TMP12]], align 8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i16> [[_MSLD]], <4 x i16> [[_MSLD1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i16> [[_MSLD]], <4 x i16> [[_MSLD1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i16> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i16> @llvm.aarch64.neon.uminp.v4i16(<4 x i16> [[TMPVAR1]], <4 x i16> [[TMPVAR2]])
 ; CHECK-NEXT:    store <4 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i16> [[TMPVAR3]]
@@ -1858,7 +1901,9 @@ define <8 x i16> @uminp_8h(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <8 x i16>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <8 x i16> [[_MSLD]], <8 x i16> [[_MSLD1]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <8 x i16> [[_MSLD]], <8 x i16> [[_MSLD1]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <8 x i16> @llvm.aarch64.neon.uminp.v8i16(<8 x i16> [[TMPVAR1]], <8 x i16> [[TMPVAR2]])
 ; CHECK-NEXT:    store <8 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i16> [[TMPVAR3]]
@@ -1897,7 +1942,9 @@ define <2 x i32> @uminp_2s(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i32> [[_MSLD]], <2 x i32> [[_MSLD1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x i32> [[_MSLD]], <2 x i32> [[_MSLD1]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x i32> @llvm.aarch64.neon.uminp.v2i32(<2 x i32> [[TMPVAR1]], <2 x i32> [[TMPVAR2]])
 ; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i32> [[TMPVAR3]]
@@ -1936,7 +1983,9 @@ define <4 x i32> @uminp_4s(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> [[_MSLD1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> [[_MSLD1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x i32> @llvm.aarch64.neon.uminp.v4i32(<4 x i32> [[TMPVAR1]], <4 x i32> [[TMPVAR2]])
 ; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i32> [[TMPVAR3]]
@@ -2103,7 +2152,9 @@ define <2 x float> @fmaxp_2s(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i32> [[_MSLD]], <2 x i32> [[_MSLD1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x i32> [[_MSLD]], <2 x i32> [[_MSLD1]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x float> @llvm.aarch64.neon.fmaxp.v2f32(<2 x float> [[TMPVAR1]], <2 x float> [[TMPVAR2]])
 ; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x float> [[TMPVAR3]]
@@ -2142,7 +2193,9 @@ define <4 x float> @fmaxp_4s(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> [[_MSLD1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> [[_MSLD1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x float> @llvm.aarch64.neon.fmaxp.v4f32(<4 x float> [[TMPVAR1]], <4 x float> [[TMPVAR2]])
 ; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x float> [[TMPVAR3]]
@@ -2181,7 +2234,9 @@ define <2 x double> @fmaxp_2d(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i64> [[_MSLD]], <2 x i64> [[_MSLD1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x i64> [[_MSLD]], <2 x i64> [[_MSLD1]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x double> @llvm.aarch64.neon.fmaxp.v2f64(<2 x double> [[TMPVAR1]], <2 x double> [[TMPVAR2]])
 ; CHECK-NEXT:    store <2 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x double> [[TMPVAR3]]
@@ -2345,7 +2400,9 @@ define <2 x float> @fminp_2s(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i32> [[_MSLD]], <2 x i32> [[_MSLD1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x i32> [[_MSLD]], <2 x i32> [[_MSLD1]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x float> @llvm.aarch64.neon.fminp.v2f32(<2 x float> [[TMPVAR1]], <2 x float> [[TMPVAR2]])
 ; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x float> [[TMPVAR3]]
@@ -2384,7 +2441,9 @@ define <4 x float> @fminp_4s(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> [[_MSLD1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> [[_MSLD1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x float> @llvm.aarch64.neon.fminp.v4f32(<4 x float> [[TMPVAR1]], <4 x float> [[TMPVAR2]])
 ; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x float> [[TMPVAR3]]
@@ -2423,7 +2482,9 @@ define <2 x double> @fminp_2d(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i64> [[_MSLD]], <2 x i64> [[_MSLD1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x i64> [[_MSLD]], <2 x i64> [[_MSLD1]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x double> @llvm.aarch64.neon.fminp.v2f64(<2 x double> [[TMPVAR1]], <2 x double> [[TMPVAR2]])
 ; CHECK-NEXT:    store <2 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x double> [[TMPVAR3]]
@@ -2466,7 +2527,9 @@ define <2 x float> @fminnmp_2s(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i32> [[_MSLD]], <2 x i32> [[_MSLD1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x i32> [[_MSLD]], <2 x i32> [[_MSLD1]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x float> @llvm.aarch64.neon.fminnmp.v2f32(<2 x float> [[TMPVAR1]], <2 x float> [[TMPVAR2]])
 ; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x float> [[TMPVAR3]]
@@ -2505,7 +2568,9 @@ define <4 x float> @fminnmp_4s(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> [[_MSLD1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> [[_MSLD1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x float> @llvm.aarch64.neon.fminnmp.v4f32(<4 x float> [[TMPVAR1]], <4 x float> [[TMPVAR2]])
 ; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x float> [[TMPVAR3]]
@@ -2544,7 +2609,9 @@ define <2 x double> @fminnmp_2d(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i64> [[_MSLD]], <2 x i64> [[_MSLD1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x i64> [[_MSLD]], <2 x i64> [[_MSLD1]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x double> @llvm.aarch64.neon.fminnmp.v2f64(<2 x double> [[TMPVAR1]], <2 x double> [[TMPVAR2]])
 ; CHECK-NEXT:    store <2 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x double> [[TMPVAR3]]
@@ -2587,7 +2654,9 @@ define <2 x float> @fmaxnmp_2s(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i32>, ptr [[TMP12]], align 8
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i32> [[_MSLD]], <2 x i32> [[_MSLD1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x i32> [[_MSLD]], <2 x i32> [[_MSLD1]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x float> @llvm.aarch64.neon.fmaxnmp.v2f32(<2 x float> [[TMPVAR1]], <2 x float> [[TMPVAR2]])
 ; CHECK-NEXT:    store <2 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x float> [[TMPVAR3]]
@@ -2626,7 +2695,9 @@ define <4 x float> @fmaxnmp_4s(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> [[_MSLD1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[_MSLD]], <4 x i32> [[_MSLD1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <4 x float> @llvm.aarch64.neon.fmaxnmp.v4f32(<4 x float> [[TMPVAR1]], <4 x float> [[TMPVAR2]])
 ; CHECK-NEXT:    store <4 x i32> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x float> [[TMPVAR3]]
@@ -2665,7 +2736,9 @@ define <2 x double> @fmaxnmp_2d(ptr %A, ptr %B) nounwind #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = xor i64 [[TMP10]], 193514046488576
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; CHECK-NEXT:    [[_MSLD1:%.*]] = load <2 x i64>, ptr [[TMP12]], align 16
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[_MSLD]], [[_MSLD1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i64> [[_MSLD]], <2 x i64> [[_MSLD1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x i64> [[_MSLD]], <2 x i64> [[_MSLD1]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP13]], [[TMP14]]
 ; CHECK-NEXT:    [[TMPVAR3:%.*]] = call <2 x double> @llvm.aarch64.neon.fmaxnmp.v2f64(<2 x double> [[TMPVAR1]], <2 x double> [[TMPVAR2]])
 ; CHECK-NEXT:    store <2 x i64> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x double> [[TMPVAR3]]
@@ -2681,6 +2754,3 @@ declare <4 x float> @llvm.aarch64.neon.fmaxnmp.v4f32(<4 x float>, <4 x float>) n
 declare <2 x double> @llvm.aarch64.neon.fmaxnmp.v2f64(<2 x double>, <2 x double>) nounwind readnone
 
 attributes #0 = { sanitize_memory }
-;.
-; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
-;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/expand-experimental-reductions.ll b/llvm/test/Instrumentation/MemorySanitizer/expand-experimental-reductions.ll
index 7a4b433fc8be6..0696ac92e59b8 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/expand-experimental-reductions.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/expand-experimental-reductions.ll
@@ -2,14 +2,6 @@
 ; RUN: opt < %s -passes=msan -S | FileCheck %s
 ;
 ; Forked from llvm/test/CodeGen/Generic/expand-experimental-reductions.ll
-;
-; Handled suboptimally by visitInstruction:
-; - llvm.vector.reduce.smax
-; - llvm.vector.reduce.smin
-; - llvm.vector.reduce.umax
-; - llvm.vector.reduce.umin
-; - llvm.vector.reduce.fmax
-; - llvm.vector.reduce.fmin
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -268,15 +260,9 @@ define i64 @smax_i64(<2 x i64> %vec) #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1:![0-9]+]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5:[0-9]+]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP0]])
 ; CHECK-NEXT:    [[R:%.*]] = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> [[VEC]])
-; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store i64 [[TMP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i64 [[R]]
 ;
 entry:
@@ -290,15 +276,9 @@ define i64 @smin_i64(<2 x i64> %vec) #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP0]])
 ; CHECK-NEXT:    [[R:%.*]] = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> [[VEC]])
-; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store i64 [[TMP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i64 [[R]]
 ;
 entry:
@@ -312,15 +292,9 @@ define i64 @umax_i64(<2 x i64> %vec) #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP0]])
 ; CHECK-NEXT:    [[R:%.*]] = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> [[VEC]])
-; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store i64 [[TMP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i64 [[R]]
 ;
 entry:
@@ -334,15 +308,9 @@ define i64 @umin_i64(<2 x i64> %vec) #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP0]])
 ; CHECK-NEXT:    [[R:%.*]] = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> [[VEC]])
-; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store i64 [[TMP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i64 [[R]]
 ;
 entry:
@@ -357,15 +325,9 @@ define double @fmax_f64(<2 x double> %vec) #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP0]])
 ; CHECK-NEXT:    [[R:%.*]] = call double @llvm.vector.reduce.fmax.v2f64(<2 x double> [[VEC]])
-; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store i64 [[TMP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret double [[R]]
 ;
 entry:
@@ -380,15 +342,9 @@ define double @fmin_f64(<2 x double> %vec) #0 {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[TMP0]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB2:.*]], label %[[BB3:.*]], !prof [[PROF1]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[TMP0]])
 ; CHECK-NEXT:    [[R:%.*]] = call double @llvm.vector.reduce.fmin.v2f64(<2 x double> [[VEC]])
-; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store i64 [[TMP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret double [[R]]
 ;
 entry:
@@ -417,6 +373,3 @@ entry:
 }
 
 attributes #0 = { sanitize_memory }
-;.
-; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
-;.
diff --git a/llvm/test/LTO/RISCV/lit.local.cfg b/llvm/test/LTO/RISCV/lit.local.cfg
new file mode 100644
index 0000000000000..a3d2298159063
--- /dev/null
+++ b/llvm/test/LTO/RISCV/lit.local.cfg
@@ -0,0 +1,2 @@
+if "RISCV" not in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/LTO/RISCV/riscv-ilp32e.ll b/llvm/test/LTO/RISCV/riscv-ilp32e.ll
new file mode 100644
index 0000000000000..bbca58e65556d
--- /dev/null
+++ b/llvm/test/LTO/RISCV/riscv-ilp32e.ll
@@ -0,0 +1,21 @@
+; Check that we don't crash on DataLayout incompatibility issue.
+; RUN: llvm-as %s -o %t.o
+; RUN: llvm-lto2 run -r %t.o,_start %t.o -o %t.elf
+; RUN: llvm-readobj -h %t.elf.0 | FileCheck %s --check-prefixes=CHECK
+; CHECK:  Machine: EM_RISCV (0xF3)
+; CHECK:  EF_RISCV_RVE (0x8)
+
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32-S32"
+target triple = "riscv32-unknown-unknown-elf"
+
+define dso_local i32 @_start() #0 {
+entry:
+  ret i32 0
+}
+
+attributes #0 = { "target-cpu"="generic-rv32" "target-features"="+32bit,+e" }
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 1, !"target-abi", !"ilp32e"}
diff --git a/llvm/test/MC/AMDGPU/gfx11_unsupported.s b/llvm/test/MC/AMDGPU/gfx11_unsupported.s
index d5d10ce548c4d..cd30647fe6378 100644
--- a/llvm/test/MC/AMDGPU/gfx11_unsupported.s
+++ b/llvm/test/MC/AMDGPU/gfx11_unsupported.s
@@ -322,6 +322,9 @@ image_sample_cd_o v252, v[1:4], s[8:15], s[12:15] dmask:0x1
 image_sample_cd_o_g16 v[5:6], v[1:4], s[8:15], s[12:15] dmask:0x3
 // CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
+s_alloc_vgpr s0
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
 s_atomic_add flat_scratch_hi, s[2:3], s0
 // CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s b/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s
index 1e12e5bb48828..55322878de85b 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s
@@ -1,4255 +1,4271 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 // RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s
 
+s_alloc_vgpr 0x1235
+// GFX12: s_alloc_vgpr 0x1235                     ; encoding: [0xff,0x53,0x80,0xbe,0x35,0x12,0x00,0x00]
+
+s_alloc_vgpr 18
+// GFX12: s_alloc_vgpr 18                         ; encoding: [0x92,0x53,0x80,0xbe]
+
+s_alloc_vgpr s35
+// GFX12: s_alloc_vgpr s35                        ; encoding: [0x23,0x53,0x80,0xbe]
+
+s_alloc_vgpr m0
+// GFX12: s_alloc_vgpr m0                         ; encoding: [0x7d,0x53,0x80,0xbe]
+
+s_alloc_vgpr scc
+// GFX12: s_alloc_vgpr src_scc                    ; encoding: [0xfd,0x53,0x80,0xbe]
+
 s_sleep_var 0x1234
-// GFX12: encoding: [0xff,0x58,0x80,0xbe,0x34,0x12,0x00,0x00]
+// GFX12: s_sleep_var 0x1234                      ; encoding: [0xff,0x58,0x80,0xbe,0x34,0x12,0x00,0x00]
 
 s_sleep_var s1
-// GFX12: encoding: [0x01,0x58,0x80,0xbe]
+// GFX12: s_sleep_var s1                          ; encoding: [0x01,0x58,0x80,0xbe]
 
 s_cvt_f32_i32 s5, s1
-// GFX12: encoding: [0x01,0x64,0x85,0xbe]
+// GFX12: s_cvt_f32_i32 s5, s1                    ; encoding: [0x01,0x64,0x85,0xbe]
 
 s_cvt_f32_i32 s105, s1
-// GFX12: encoding: [0x01,0x64,0xe9,0xbe]
+// GFX12: s_cvt_f32_i32 s105, s1                  ; encoding: [0x01,0x64,0xe9,0xbe]
 
 s_cvt_f32_i32 s5, s105
-// GFX12: encoding: [0x69,0x64,0x85,0xbe]
+// GFX12: s_cvt_f32_i32 s5, s105                  ; encoding: [0x69,0x64,0x85,0xbe]
 
 s_cvt_f32_i32 s5, s103
-// GFX12: encoding: [0x67,0x64,0x85,0xbe]
+// GFX12: s_cvt_f32_i32 s5, s103                  ; encoding: [0x67,0x64,0x85,0xbe]
 
 s_cvt_f32_i32 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x64,0x85,0xbe]
+// GFX12: s_cvt_f32_i32 s5, vcc_lo                ; encoding: [0x6a,0x64,0x85,0xbe]
 
 s_cvt_f32_i32 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x64,0x85,0xbe]
+// GFX12: s_cvt_f32_i32 s5, vcc_hi                ; encoding: [0x6b,0x64,0x85,0xbe]
 
 s_cvt_f32_i32 s5, ttmp11
-// GFX12: encoding: [0x77,0x64,0x85,0xbe]
+// GFX12: s_cvt_f32_i32 s5, ttmp11                ; encoding: [0x77,0x64,0x85,0xbe]
 
 s_cvt_f32_i32 s5, m0
-// GFX12: encoding: [0x7d,0x64,0x85,0xbe]
+// GFX12: s_cvt_f32_i32 s5, m0                    ; encoding: [0x7d,0x64,0x85,0xbe]
 
 s_cvt_f32_i32 s5, exec_lo
-// GFX12: encoding: [0x7e,0x64,0x85,0xbe]
+// GFX12: s_cvt_f32_i32 s5, exec_lo               ; encoding: [0x7e,0x64,0x85,0xbe]
 
 s_cvt_f32_i32 s5, exec_hi
-// GFX12: encoding: [0x7f,0x64,0x85,0xbe]
+// GFX12: s_cvt_f32_i32 s5, exec_hi               ; encoding: [0x7f,0x64,0x85,0xbe]
 
 s_cvt_f32_i32 s5, 0
-// GFX12: encoding: [0x80,0x64,0x85,0xbe]
+// GFX12: s_cvt_f32_i32 s5, 0                     ; encoding: [0x80,0x64,0x85,0xbe]
 
 s_cvt_f32_i32 s5, -1
-// GFX12: encoding: [0xc1,0x64,0x85,0xbe]
+// GFX12: s_cvt_f32_i32 s5, -1                    ; encoding: [0xc1,0x64,0x85,0xbe]
 
 s_cvt_f32_i32 s5, 0.5
-// GFX12: encoding: [0xf0,0x64,0x85,0xbe]
+// GFX12: s_cvt_f32_i32 s5, 0.5                   ; encoding: [0xf0,0x64,0x85,0xbe]
 
 s_cvt_f32_i32 s5, -4.0
-// GFX12: encoding: [0xf7,0x64,0x85,0xbe]
+// GFX12: s_cvt_f32_i32 s5, -4.0                  ; encoding: [0xf7,0x64,0x85,0xbe]
 
 s_cvt_f32_i32 s5, 0xaf123456
-// GFX12: encoding: [0xff,0x64,0x85,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_cvt_f32_i32 s5, 0xaf123456            ; encoding: [0xff,0x64,0x85,0xbe,0x56,0x34,0x12,0xaf]
 
 s_cvt_f32_i32 s5, 0x3f717273
-// GFX12: encoding: [0xff,0x64,0x85,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_cvt_f32_i32 s5, 0x3f717273            ; encoding: [0xff,0x64,0x85,0xbe,0x73,0x72,0x71,0x3f]
 
 s_cvt_f32_u32 s5, s1
-// GFX12: encoding: [0x01,0x65,0x85,0xbe]
+// GFX12: s_cvt_f32_u32 s5, s1                    ; encoding: [0x01,0x65,0x85,0xbe]
 
 s_cvt_f32_u32 s105, s1
-// GFX12: encoding: [0x01,0x65,0xe9,0xbe]
+// GFX12: s_cvt_f32_u32 s105, s1                  ; encoding: [0x01,0x65,0xe9,0xbe]
 
 s_cvt_f32_u32 s5, s105
-// GFX12: encoding: [0x69,0x65,0x85,0xbe]
+// GFX12: s_cvt_f32_u32 s5, s105                  ; encoding: [0x69,0x65,0x85,0xbe]
 
 s_cvt_f32_u32 s5, s103
-// GFX12: encoding: [0x67,0x65,0x85,0xbe]
+// GFX12: s_cvt_f32_u32 s5, s103                  ; encoding: [0x67,0x65,0x85,0xbe]
 
 s_cvt_f32_u32 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x65,0x85,0xbe]
+// GFX12: s_cvt_f32_u32 s5, vcc_lo                ; encoding: [0x6a,0x65,0x85,0xbe]
 
 s_cvt_f32_u32 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x65,0x85,0xbe]
+// GFX12: s_cvt_f32_u32 s5, vcc_hi                ; encoding: [0x6b,0x65,0x85,0xbe]
 
 s_cvt_f32_u32 s5, ttmp11
-// GFX12: encoding: [0x77,0x65,0x85,0xbe]
+// GFX12: s_cvt_f32_u32 s5, ttmp11                ; encoding: [0x77,0x65,0x85,0xbe]
 
 s_cvt_f32_u32 s5, m0
-// GFX12: encoding: [0x7d,0x65,0x85,0xbe]
+// GFX12: s_cvt_f32_u32 s5, m0                    ; encoding: [0x7d,0x65,0x85,0xbe]
 
 s_cvt_f32_u32 s5, exec_lo
-// GFX12: encoding: [0x7e,0x65,0x85,0xbe]
+// GFX12: s_cvt_f32_u32 s5, exec_lo               ; encoding: [0x7e,0x65,0x85,0xbe]
 
 s_cvt_f32_u32 s5, exec_hi
-// GFX12: encoding: [0x7f,0x65,0x85,0xbe]
+// GFX12: s_cvt_f32_u32 s5, exec_hi               ; encoding: [0x7f,0x65,0x85,0xbe]
 
 s_cvt_f32_u32 s5, 0
-// GFX12: encoding: [0x80,0x65,0x85,0xbe]
+// GFX12: s_cvt_f32_u32 s5, 0                     ; encoding: [0x80,0x65,0x85,0xbe]
 
 s_cvt_f32_u32 s5, -1
-// GFX12: encoding: [0xc1,0x65,0x85,0xbe]
+// GFX12: s_cvt_f32_u32 s5, -1                    ; encoding: [0xc1,0x65,0x85,0xbe]
 
 s_cvt_f32_u32 s5, 0.5
-// GFX12: encoding: [0xf0,0x65,0x85,0xbe]
+// GFX12: s_cvt_f32_u32 s5, 0.5                   ; encoding: [0xf0,0x65,0x85,0xbe]
 
 s_cvt_f32_u32 s5, -4.0
-// GFX12: encoding: [0xf7,0x65,0x85,0xbe]
+// GFX12: s_cvt_f32_u32 s5, -4.0                  ; encoding: [0xf7,0x65,0x85,0xbe]
 
 s_cvt_f32_u32 s5, 0xaf123456
-// GFX12: encoding: [0xff,0x65,0x85,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_cvt_f32_u32 s5, 0xaf123456            ; encoding: [0xff,0x65,0x85,0xbe,0x56,0x34,0x12,0xaf]
 
 s_cvt_f32_u32 s5, 0x3f717273
-// GFX12: encoding: [0xff,0x65,0x85,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_cvt_f32_u32 s5, 0x3f717273            ; encoding: [0xff,0x65,0x85,0xbe,0x73,0x72,0x71,0x3f]
 
 s_cvt_u32_f32 s5, s1
-// GFX12: encoding: [0x01,0x67,0x85,0xbe]
+// GFX12: s_cvt_u32_f32 s5, s1                    ; encoding: [0x01,0x67,0x85,0xbe]
 
 s_cvt_u32_f32 s105, s1
-// GFX12: encoding: [0x01,0x67,0xe9,0xbe]
+// GFX12: s_cvt_u32_f32 s105, s1                  ; encoding: [0x01,0x67,0xe9,0xbe]
 
 s_cvt_u32_f32 s5, s105
-// GFX12: encoding: [0x69,0x67,0x85,0xbe]
+// GFX12: s_cvt_u32_f32 s5, s105                  ; encoding: [0x69,0x67,0x85,0xbe]
 
 s_cvt_u32_f32 s5, s103
-// GFX12: encoding: [0x67,0x67,0x85,0xbe]
+// GFX12: s_cvt_u32_f32 s5, s103                  ; encoding: [0x67,0x67,0x85,0xbe]
 
 s_cvt_u32_f32 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x67,0x85,0xbe]
+// GFX12: s_cvt_u32_f32 s5, vcc_lo                ; encoding: [0x6a,0x67,0x85,0xbe]
 
 s_cvt_u32_f32 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x67,0x85,0xbe]
+// GFX12: s_cvt_u32_f32 s5, vcc_hi                ; encoding: [0x6b,0x67,0x85,0xbe]
 
 s_cvt_u32_f32 s5, ttmp11
-// GFX12: encoding: [0x77,0x67,0x85,0xbe]
+// GFX12: s_cvt_u32_f32 s5, ttmp11                ; encoding: [0x77,0x67,0x85,0xbe]
 
 s_cvt_u32_f32 s5, m0
-// GFX12: encoding: [0x7d,0x67,0x85,0xbe]
+// GFX12: s_cvt_u32_f32 s5, m0                    ; encoding: [0x7d,0x67,0x85,0xbe]
 
 s_cvt_u32_f32 s5, exec_lo
-// GFX12: encoding: [0x7e,0x67,0x85,0xbe]
+// GFX12: s_cvt_u32_f32 s5, exec_lo               ; encoding: [0x7e,0x67,0x85,0xbe]
 
 s_cvt_u32_f32 s5, exec_hi
-// GFX12: encoding: [0x7f,0x67,0x85,0xbe]
+// GFX12: s_cvt_u32_f32 s5, exec_hi               ; encoding: [0x7f,0x67,0x85,0xbe]
 
 s_cvt_u32_f32 s5, 0
-// GFX12: encoding: [0x80,0x67,0x85,0xbe]
+// GFX12: s_cvt_u32_f32 s5, 0                     ; encoding: [0x80,0x67,0x85,0xbe]
 
 s_cvt_u32_f32 s5, -1
-// GFX12: encoding: [0xc1,0x67,0x85,0xbe]
+// GFX12: s_cvt_u32_f32 s5, -1                    ; encoding: [0xc1,0x67,0x85,0xbe]
 
 s_cvt_u32_f32 s5, 0.5
-// GFX12: encoding: [0xf0,0x67,0x85,0xbe]
+// GFX12: s_cvt_u32_f32 s5, 0.5                   ; encoding: [0xf0,0x67,0x85,0xbe]
 
 s_cvt_u32_f32 s5, -4.0
-// GFX12: encoding: [0xf7,0x67,0x85,0xbe]
+// GFX12: s_cvt_u32_f32 s5, -4.0                  ; encoding: [0xf7,0x67,0x85,0xbe]
 
 s_cvt_u32_f32 s5, 0xaf123456
-// GFX12: encoding: [0xff,0x67,0x85,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_cvt_u32_f32 s5, 0xaf123456            ; encoding: [0xff,0x67,0x85,0xbe,0x56,0x34,0x12,0xaf]
 
 s_cvt_u32_f32 s5, 0x3f717273
-// GFX12: encoding: [0xff,0x67,0x85,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_cvt_u32_f32 s5, 0x3f717273            ; encoding: [0xff,0x67,0x85,0xbe,0x73,0x72,0x71,0x3f]
 
 s_cvt_i32_f32 s5, s1
-// GFX12: encoding: [0x01,0x66,0x85,0xbe]
+// GFX12: s_cvt_i32_f32 s5, s1                    ; encoding: [0x01,0x66,0x85,0xbe]
 
 s_cvt_i32_f32 s105, s1
-// GFX12: encoding: [0x01,0x66,0xe9,0xbe]
+// GFX12: s_cvt_i32_f32 s105, s1                  ; encoding: [0x01,0x66,0xe9,0xbe]
 
 s_cvt_i32_f32 s5, s105
-// GFX12: encoding: [0x69,0x66,0x85,0xbe]
+// GFX12: s_cvt_i32_f32 s5, s105                  ; encoding: [0x69,0x66,0x85,0xbe]
 
 s_cvt_i32_f32 s5, s103
-// GFX12: encoding: [0x67,0x66,0x85,0xbe]
+// GFX12: s_cvt_i32_f32 s5, s103                  ; encoding: [0x67,0x66,0x85,0xbe]
 
 s_cvt_i32_f32 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x66,0x85,0xbe]
+// GFX12: s_cvt_i32_f32 s5, vcc_lo                ; encoding: [0x6a,0x66,0x85,0xbe]
 
 s_cvt_i32_f32 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x66,0x85,0xbe]
+// GFX12: s_cvt_i32_f32 s5, vcc_hi                ; encoding: [0x6b,0x66,0x85,0xbe]
 
 s_cvt_i32_f32 s5, ttmp11
-// GFX12: encoding: [0x77,0x66,0x85,0xbe]
+// GFX12: s_cvt_i32_f32 s5, ttmp11                ; encoding: [0x77,0x66,0x85,0xbe]
 
 s_cvt_i32_f32 s5, m0
-// GFX12: encoding: [0x7d,0x66,0x85,0xbe]
+// GFX12: s_cvt_i32_f32 s5, m0                    ; encoding: [0x7d,0x66,0x85,0xbe]
 
 s_cvt_i32_f32 s5, exec_lo
-// GFX12: encoding: [0x7e,0x66,0x85,0xbe]
+// GFX12: s_cvt_i32_f32 s5, exec_lo               ; encoding: [0x7e,0x66,0x85,0xbe]
 
 s_cvt_i32_f32 s5, exec_hi
-// GFX12: encoding: [0x7f,0x66,0x85,0xbe]
+// GFX12: s_cvt_i32_f32 s5, exec_hi               ; encoding: [0x7f,0x66,0x85,0xbe]
 
 s_cvt_i32_f32 s5, 0
-// GFX12: encoding: [0x80,0x66,0x85,0xbe]
+// GFX12: s_cvt_i32_f32 s5, 0                     ; encoding: [0x80,0x66,0x85,0xbe]
 
 s_cvt_i32_f32 s5, -1
-// GFX12: encoding: [0xc1,0x66,0x85,0xbe]
+// GFX12: s_cvt_i32_f32 s5, -1                    ; encoding: [0xc1,0x66,0x85,0xbe]
 
 s_cvt_i32_f32 s5, 0.5
-// GFX12: encoding: [0xf0,0x66,0x85,0xbe]
+// GFX12: s_cvt_i32_f32 s5, 0.5                   ; encoding: [0xf0,0x66,0x85,0xbe]
 
 s_cvt_i32_f32 s5, -4.0
-// GFX12: encoding: [0xf7,0x66,0x85,0xbe]
+// GFX12: s_cvt_i32_f32 s5, -4.0                  ; encoding: [0xf7,0x66,0x85,0xbe]
 
 s_cvt_i32_f32 s5, 0xaf123456
-// GFX12: encoding: [0xff,0x66,0x85,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_cvt_i32_f32 s5, 0xaf123456            ; encoding: [0xff,0x66,0x85,0xbe,0x56,0x34,0x12,0xaf]
 
 s_cvt_i32_f32 s5, 0x3f717273
-// GFX12: encoding: [0xff,0x66,0x85,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_cvt_i32_f32 s5, 0x3f717273            ; encoding: [0xff,0x66,0x85,0xbe,0x73,0x72,0x71,0x3f]
 
 s_cvt_f16_f32 s5, s1
-// GFX12: encoding: [0x01,0x68,0x85,0xbe]
+// GFX12: s_cvt_f16_f32 s5, s1                    ; encoding: [0x01,0x68,0x85,0xbe]
 
 s_cvt_f16_f32 s105, s1
-// GFX12: encoding: [0x01,0x68,0xe9,0xbe]
+// GFX12: s_cvt_f16_f32 s105, s1                  ; encoding: [0x01,0x68,0xe9,0xbe]
 
 s_cvt_f16_f32 s5, s105
-// GFX12: encoding: [0x69,0x68,0x85,0xbe]
+// GFX12: s_cvt_f16_f32 s5, s105                  ; encoding: [0x69,0x68,0x85,0xbe]
 
 s_cvt_f16_f32 s5, s103
-// GFX12: encoding: [0x67,0x68,0x85,0xbe]
+// GFX12: s_cvt_f16_f32 s5, s103                  ; encoding: [0x67,0x68,0x85,0xbe]
 
 s_cvt_f16_f32 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x68,0x85,0xbe]
+// GFX12: s_cvt_f16_f32 s5, vcc_lo                ; encoding: [0x6a,0x68,0x85,0xbe]
 
 s_cvt_f16_f32 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x68,0x85,0xbe]
+// GFX12: s_cvt_f16_f32 s5, vcc_hi                ; encoding: [0x6b,0x68,0x85,0xbe]
 
 s_cvt_f16_f32 s5, ttmp11
-// GFX12: encoding: [0x77,0x68,0x85,0xbe]
+// GFX12: s_cvt_f16_f32 s5, ttmp11                ; encoding: [0x77,0x68,0x85,0xbe]
 
 s_cvt_f16_f32 s5, m0
-// GFX12: encoding: [0x7d,0x68,0x85,0xbe]
+// GFX12: s_cvt_f16_f32 s5, m0                    ; encoding: [0x7d,0x68,0x85,0xbe]
 
 s_cvt_f16_f32 s5, exec_lo
-// GFX12: encoding: [0x7e,0x68,0x85,0xbe]
+// GFX12: s_cvt_f16_f32 s5, exec_lo               ; encoding: [0x7e,0x68,0x85,0xbe]
 
 s_cvt_f16_f32 s5, exec_hi
-// GFX12: encoding: [0x7f,0x68,0x85,0xbe]
+// GFX12: s_cvt_f16_f32 s5, exec_hi               ; encoding: [0x7f,0x68,0x85,0xbe]
 
 s_cvt_f16_f32 s5, 0
-// GFX12: encoding: [0x80,0x68,0x85,0xbe]
+// GFX12: s_cvt_f16_f32 s5, 0                     ; encoding: [0x80,0x68,0x85,0xbe]
 
 s_cvt_f16_f32 s5, -1
-// GFX12: encoding: [0xc1,0x68,0x85,0xbe]
+// GFX12: s_cvt_f16_f32 s5, -1                    ; encoding: [0xc1,0x68,0x85,0xbe]
 
 s_cvt_f16_f32 s5, 0.5
-// GFX12: encoding: [0xf0,0x68,0x85,0xbe]
+// GFX12: s_cvt_f16_f32 s5, 0.5                   ; encoding: [0xf0,0x68,0x85,0xbe]
 
 s_cvt_f16_f32 s5, -4.0
-// GFX12: encoding: [0xf7,0x68,0x85,0xbe]
+// GFX12: s_cvt_f16_f32 s5, -4.0                  ; encoding: [0xf7,0x68,0x85,0xbe]
 
 s_cvt_f16_f32 s5, 0xaf123456
-// GFX12: encoding: [0xff,0x68,0x85,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_cvt_f16_f32 s5, 0xaf123456            ; encoding: [0xff,0x68,0x85,0xbe,0x56,0x34,0x12,0xaf]
 
 s_cvt_f16_f32 s5, 0x3f717273
-// GFX12: encoding: [0xff,0x68,0x85,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_cvt_f16_f32 s5, 0x3f717273            ; encoding: [0xff,0x68,0x85,0xbe,0x73,0x72,0x71,0x3f]
 
 s_cvt_f32_f16 s5, s1
-// GFX12: encoding: [0x01,0x69,0x85,0xbe]
+// GFX12: s_cvt_f32_f16 s5, s1                    ; encoding: [0x01,0x69,0x85,0xbe]
 
 s_cvt_f32_f16 s105, s1
-// GFX12: encoding: [0x01,0x69,0xe9,0xbe]
+// GFX12: s_cvt_f32_f16 s105, s1                  ; encoding: [0x01,0x69,0xe9,0xbe]
 
 s_cvt_f32_f16 s5, s105
-// GFX12: encoding: [0x69,0x69,0x85,0xbe]
+// GFX12: s_cvt_f32_f16 s5, s105                  ; encoding: [0x69,0x69,0x85,0xbe]
 
 s_cvt_f32_f16 s5, s103
-// GFX12: encoding: [0x67,0x69,0x85,0xbe]
+// GFX12: s_cvt_f32_f16 s5, s103                  ; encoding: [0x67,0x69,0x85,0xbe]
 
 s_cvt_f32_f16 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x69,0x85,0xbe]
+// GFX12: s_cvt_f32_f16 s5, vcc_lo                ; encoding: [0x6a,0x69,0x85,0xbe]
 
 s_cvt_f32_f16 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x69,0x85,0xbe]
+// GFX12: s_cvt_f32_f16 s5, vcc_hi                ; encoding: [0x6b,0x69,0x85,0xbe]
 
 s_cvt_f32_f16 s5, ttmp11
-// GFX12: encoding: [0x77,0x69,0x85,0xbe]
+// GFX12: s_cvt_f32_f16 s5, ttmp11                ; encoding: [0x77,0x69,0x85,0xbe]
 
 s_cvt_f32_f16 s5, m0
-// GFX12: encoding: [0x7d,0x69,0x85,0xbe]
+// GFX12: s_cvt_f32_f16 s5, m0                    ; encoding: [0x7d,0x69,0x85,0xbe]
 
 s_cvt_f32_f16 s5, exec_lo
-// GFX12: encoding: [0x7e,0x69,0x85,0xbe]
+// GFX12: s_cvt_f32_f16 s5, exec_lo               ; encoding: [0x7e,0x69,0x85,0xbe]
 
 s_cvt_f32_f16 s5, exec_hi
-// GFX12: encoding: [0x7f,0x69,0x85,0xbe]
+// GFX12: s_cvt_f32_f16 s5, exec_hi               ; encoding: [0x7f,0x69,0x85,0xbe]
 
 s_cvt_f32_f16 s5, 0
-// GFX12: encoding: [0x80,0x69,0x85,0xbe]
+// GFX12: s_cvt_f32_f16 s5, 0                     ; encoding: [0x80,0x69,0x85,0xbe]
 
 s_cvt_f32_f16 s5, -1
-// GFX12: encoding: [0xc1,0x69,0x85,0xbe]
+// GFX12: s_cvt_f32_f16 s5, -1                    ; encoding: [0xc1,0x69,0x85,0xbe]
 
 s_cvt_hi_f32_f16 s5, s1
-// GFX12: encoding: [0x01,0x6a,0x85,0xbe]
+// GFX12: s_cvt_hi_f32_f16 s5, s1                 ; encoding: [0x01,0x6a,0x85,0xbe]
 
 s_cvt_hi_f32_f16 s105, s1
-// GFX12: encoding: [0x01,0x6a,0xe9,0xbe]
+// GFX12: s_cvt_hi_f32_f16 s105, s1               ; encoding: [0x01,0x6a,0xe9,0xbe]
 
 s_cvt_hi_f32_f16 s5, s105
-// GFX12: encoding: [0x69,0x6a,0x85,0xbe]
+// GFX12: s_cvt_hi_f32_f16 s5, s105               ; encoding: [0x69,0x6a,0x85,0xbe]
 
 s_cvt_hi_f32_f16 s5, s103
-// GFX12: encoding: [0x67,0x6a,0x85,0xbe]
+// GFX12: s_cvt_hi_f32_f16 s5, s103               ; encoding: [0x67,0x6a,0x85,0xbe]
 
 s_cvt_hi_f32_f16 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x6a,0x85,0xbe]
+// GFX12: s_cvt_hi_f32_f16 s5, vcc_lo             ; encoding: [0x6a,0x6a,0x85,0xbe]
 
 s_cvt_hi_f32_f16 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x6a,0x85,0xbe]
+// GFX12: s_cvt_hi_f32_f16 s5, vcc_hi             ; encoding: [0x6b,0x6a,0x85,0xbe]
 
 s_cvt_hi_f32_f16 s5, ttmp11
-// GFX12: encoding: [0x77,0x6a,0x85,0xbe]
+// GFX12: s_cvt_hi_f32_f16 s5, ttmp11             ; encoding: [0x77,0x6a,0x85,0xbe]
 
 s_cvt_hi_f32_f16 s5, m0
-// GFX12: encoding: [0x7d,0x6a,0x85,0xbe]
+// GFX12: s_cvt_hi_f32_f16 s5, m0                 ; encoding: [0x7d,0x6a,0x85,0xbe]
 
 s_cvt_hi_f32_f16 s5, exec_lo
-// GFX12: encoding: [0x7e,0x6a,0x85,0xbe]
+// GFX12: s_cvt_hi_f32_f16 s5, exec_lo            ; encoding: [0x7e,0x6a,0x85,0xbe]
 
 s_cvt_hi_f32_f16 s5, exec_hi
-// GFX12: encoding: [0x7f,0x6a,0x85,0xbe]
+// GFX12: s_cvt_hi_f32_f16 s5, exec_hi            ; encoding: [0x7f,0x6a,0x85,0xbe]
 
 s_cvt_hi_f32_f16 s5, 0
-// GFX12: encoding: [0x80,0x6a,0x85,0xbe]
+// GFX12: s_cvt_hi_f32_f16 s5, 0                  ; encoding: [0x80,0x6a,0x85,0xbe]
 
 s_cvt_hi_f32_f16 s5, -1
-// GFX12: encoding: [0xc1,0x6a,0x85,0xbe]
+// GFX12: s_cvt_hi_f32_f16 s5, -1                 ; encoding: [0xc1,0x6a,0x85,0xbe]
 
 s_trunc_f32 s5, s1
-// GFX12: encoding: [0x01,0x62,0x85,0xbe]
+// GFX12: s_trunc_f32 s5, s1                      ; encoding: [0x01,0x62,0x85,0xbe]
 
 s_trunc_f32 s105, s1
-// GFX12: encoding: [0x01,0x62,0xe9,0xbe]
+// GFX12: s_trunc_f32 s105, s1                    ; encoding: [0x01,0x62,0xe9,0xbe]
 
 s_trunc_f32 s5, s105
-// GFX12: encoding: [0x69,0x62,0x85,0xbe]
+// GFX12: s_trunc_f32 s5, s105                    ; encoding: [0x69,0x62,0x85,0xbe]
 
 s_trunc_f32 s5, s103
-// GFX12: encoding: [0x67,0x62,0x85,0xbe]
+// GFX12: s_trunc_f32 s5, s103                    ; encoding: [0x67,0x62,0x85,0xbe]
 
 s_trunc_f32 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x62,0x85,0xbe]
+// GFX12: s_trunc_f32 s5, vcc_lo                  ; encoding: [0x6a,0x62,0x85,0xbe]
 
 s_trunc_f32 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x62,0x85,0xbe]
+// GFX12: s_trunc_f32 s5, vcc_hi                  ; encoding: [0x6b,0x62,0x85,0xbe]
 
 s_trunc_f32 s5, ttmp11
-// GFX12: encoding: [0x77,0x62,0x85,0xbe]
+// GFX12: s_trunc_f32 s5, ttmp11                  ; encoding: [0x77,0x62,0x85,0xbe]
 
 s_trunc_f32 s5, m0
-// GFX12: encoding: [0x7d,0x62,0x85,0xbe]
+// GFX12: s_trunc_f32 s5, m0                      ; encoding: [0x7d,0x62,0x85,0xbe]
 
 s_trunc_f32 s5, exec_lo
-// GFX12: encoding: [0x7e,0x62,0x85,0xbe]
+// GFX12: s_trunc_f32 s5, exec_lo                 ; encoding: [0x7e,0x62,0x85,0xbe]
 
 s_trunc_f32 s5, exec_hi
-// GFX12: encoding: [0x7f,0x62,0x85,0xbe]
+// GFX12: s_trunc_f32 s5, exec_hi                 ; encoding: [0x7f,0x62,0x85,0xbe]
 
 s_trunc_f32 s5, 0
-// GFX12: encoding: [0x80,0x62,0x85,0xbe]
+// GFX12: s_trunc_f32 s5, 0                       ; encoding: [0x80,0x62,0x85,0xbe]
 
 s_trunc_f32 s5, -1
-// GFX12: encoding: [0xc1,0x62,0x85,0xbe]
+// GFX12: s_trunc_f32 s5, -1                      ; encoding: [0xc1,0x62,0x85,0xbe]
 
 s_trunc_f32 s5, 0.5
-// GFX12: encoding: [0xf0,0x62,0x85,0xbe]
+// GFX12: s_trunc_f32 s5, 0.5                     ; encoding: [0xf0,0x62,0x85,0xbe]
 
 s_trunc_f32 s5, -4.0
-// GFX12: encoding: [0xf7,0x62,0x85,0xbe]
+// GFX12: s_trunc_f32 s5, -4.0                    ; encoding: [0xf7,0x62,0x85,0xbe]
 
 s_trunc_f32 s5, 0xaf123456
-// GFX12: encoding: [0xff,0x62,0x85,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_trunc_f32 s5, 0xaf123456              ; encoding: [0xff,0x62,0x85,0xbe,0x56,0x34,0x12,0xaf]
 
 s_trunc_f32 s5, 0x3f717273
-// GFX12: encoding: [0xff,0x62,0x85,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_trunc_f32 s5, 0x3f717273              ; encoding: [0xff,0x62,0x85,0xbe,0x73,0x72,0x71,0x3f]
 
 s_ceil_f32 s5, s1
-// GFX12: encoding: [0x01,0x60,0x85,0xbe]
+// GFX12: s_ceil_f32 s5, s1                       ; encoding: [0x01,0x60,0x85,0xbe]
 
 s_ceil_f32 s105, s1
-// GFX12: encoding: [0x01,0x60,0xe9,0xbe]
+// GFX12: s_ceil_f32 s105, s1                     ; encoding: [0x01,0x60,0xe9,0xbe]
 
 s_ceil_f32 s5, s105
-// GFX12: encoding: [0x69,0x60,0x85,0xbe]
+// GFX12: s_ceil_f32 s5, s105                     ; encoding: [0x69,0x60,0x85,0xbe]
 
 s_ceil_f32 s5, s103
-// GFX12: encoding: [0x67,0x60,0x85,0xbe]
+// GFX12: s_ceil_f32 s5, s103                     ; encoding: [0x67,0x60,0x85,0xbe]
 
 s_ceil_f32 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x60,0x85,0xbe]
+// GFX12: s_ceil_f32 s5, vcc_lo                   ; encoding: [0x6a,0x60,0x85,0xbe]
 
 s_ceil_f32 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x60,0x85,0xbe]
+// GFX12: s_ceil_f32 s5, vcc_hi                   ; encoding: [0x6b,0x60,0x85,0xbe]
 
 s_ceil_f32 s5, ttmp11
-// GFX12: encoding: [0x77,0x60,0x85,0xbe]
+// GFX12: s_ceil_f32 s5, ttmp11                   ; encoding: [0x77,0x60,0x85,0xbe]
 
 s_ceil_f32 s5, m0
-// GFX12: encoding: [0x7d,0x60,0x85,0xbe]
+// GFX12: s_ceil_f32 s5, m0                       ; encoding: [0x7d,0x60,0x85,0xbe]
 
 s_ceil_f32 s5, exec_lo
-// GFX12: encoding: [0x7e,0x60,0x85,0xbe]
+// GFX12: s_ceil_f32 s5, exec_lo                  ; encoding: [0x7e,0x60,0x85,0xbe]
 
 s_ceil_f32 s5, exec_hi
-// GFX12: encoding: [0x7f,0x60,0x85,0xbe]
+// GFX12: s_ceil_f32 s5, exec_hi                  ; encoding: [0x7f,0x60,0x85,0xbe]
 
 s_ceil_f32 s5, 0
-// GFX12: encoding: [0x80,0x60,0x85,0xbe]
+// GFX12: s_ceil_f32 s5, 0                        ; encoding: [0x80,0x60,0x85,0xbe]
 
 s_ceil_f32 s5, -1
-// GFX12: encoding: [0xc1,0x60,0x85,0xbe]
+// GFX12: s_ceil_f32 s5, -1                       ; encoding: [0xc1,0x60,0x85,0xbe]
 
 s_ceil_f32 s5, 0.5
-// GFX12: encoding: [0xf0,0x60,0x85,0xbe]
+// GFX12: s_ceil_f32 s5, 0.5                      ; encoding: [0xf0,0x60,0x85,0xbe]
 
 s_ceil_f32 s5, -4.0
-// GFX12: encoding: [0xf7,0x60,0x85,0xbe]
+// GFX12: s_ceil_f32 s5, -4.0                     ; encoding: [0xf7,0x60,0x85,0xbe]
 
 s_ceil_f32 s5, 0xaf123456
-// GFX12: encoding: [0xff,0x60,0x85,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_ceil_f32 s5, 0xaf123456               ; encoding: [0xff,0x60,0x85,0xbe,0x56,0x34,0x12,0xaf]
 
 s_ceil_f32 s5, 0x3f717273
-// GFX12: encoding: [0xff,0x60,0x85,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_ceil_f32 s5, 0x3f717273               ; encoding: [0xff,0x60,0x85,0xbe,0x73,0x72,0x71,0x3f]
 
 s_rndne_f32 s5, s1
-// GFX12: encoding: [0x01,0x63,0x85,0xbe]
+// GFX12: s_rndne_f32 s5, s1                      ; encoding: [0x01,0x63,0x85,0xbe]
 
 s_rndne_f32 s105, s1
-// GFX12: encoding: [0x01,0x63,0xe9,0xbe]
+// GFX12: s_rndne_f32 s105, s1                    ; encoding: [0x01,0x63,0xe9,0xbe]
 
 s_rndne_f32 s5, s105
-// GFX12: encoding: [0x69,0x63,0x85,0xbe]
+// GFX12: s_rndne_f32 s5, s105                    ; encoding: [0x69,0x63,0x85,0xbe]
 
 s_rndne_f32 s5, s103
-// GFX12: encoding: [0x67,0x63,0x85,0xbe]
+// GFX12: s_rndne_f32 s5, s103                    ; encoding: [0x67,0x63,0x85,0xbe]
 
 s_rndne_f32 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x63,0x85,0xbe]
+// GFX12: s_rndne_f32 s5, vcc_lo                  ; encoding: [0x6a,0x63,0x85,0xbe]
 
 s_rndne_f32 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x63,0x85,0xbe]
+// GFX12: s_rndne_f32 s5, vcc_hi                  ; encoding: [0x6b,0x63,0x85,0xbe]
 
 s_rndne_f32 s5, ttmp11
-// GFX12: encoding: [0x77,0x63,0x85,0xbe]
+// GFX12: s_rndne_f32 s5, ttmp11                  ; encoding: [0x77,0x63,0x85,0xbe]
 
 s_rndne_f32 s5, m0
-// GFX12: encoding: [0x7d,0x63,0x85,0xbe]
+// GFX12: s_rndne_f32 s5, m0                      ; encoding: [0x7d,0x63,0x85,0xbe]
 
 s_rndne_f32 s5, exec_lo
-// GFX12: encoding: [0x7e,0x63,0x85,0xbe]
+// GFX12: s_rndne_f32 s5, exec_lo                 ; encoding: [0x7e,0x63,0x85,0xbe]
 
 s_rndne_f32 s5, exec_hi
-// GFX12: encoding: [0x7f,0x63,0x85,0xbe]
+// GFX12: s_rndne_f32 s5, exec_hi                 ; encoding: [0x7f,0x63,0x85,0xbe]
 
 s_rndne_f32 s5, 0
-// GFX12: encoding: [0x80,0x63,0x85,0xbe]
+// GFX12: s_rndne_f32 s5, 0                       ; encoding: [0x80,0x63,0x85,0xbe]
 
 s_rndne_f32 s5, -1
-// GFX12: encoding: [0xc1,0x63,0x85,0xbe]
+// GFX12: s_rndne_f32 s5, -1                      ; encoding: [0xc1,0x63,0x85,0xbe]
 
 s_rndne_f32 s5, 0.5
-// GFX12: encoding: [0xf0,0x63,0x85,0xbe]
+// GFX12: s_rndne_f32 s5, 0.5                     ; encoding: [0xf0,0x63,0x85,0xbe]
 
 s_rndne_f32 s5, -4.0
-// GFX12: encoding: [0xf7,0x63,0x85,0xbe]
+// GFX12: s_rndne_f32 s5, -4.0                    ; encoding: [0xf7,0x63,0x85,0xbe]
 
 s_rndne_f32 s5, 0xaf123456
-// GFX12: encoding: [0xff,0x63,0x85,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_rndne_f32 s5, 0xaf123456              ; encoding: [0xff,0x63,0x85,0xbe,0x56,0x34,0x12,0xaf]
 
 s_rndne_f32 s5, 0x3f717273
-// GFX12: encoding: [0xff,0x63,0x85,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_rndne_f32 s5, 0x3f717273              ; encoding: [0xff,0x63,0x85,0xbe,0x73,0x72,0x71,0x3f]
 
 s_floor_f32 s5, s1
-// GFX12: encoding: [0x01,0x61,0x85,0xbe]
+// GFX12: s_floor_f32 s5, s1                      ; encoding: [0x01,0x61,0x85,0xbe]
 
 s_floor_f32 s105, s1
-// GFX12: encoding: [0x01,0x61,0xe9,0xbe]
+// GFX12: s_floor_f32 s105, s1                    ; encoding: [0x01,0x61,0xe9,0xbe]
 
 s_floor_f32 s5, s105
-// GFX12: encoding: [0x69,0x61,0x85,0xbe]
+// GFX12: s_floor_f32 s5, s105                    ; encoding: [0x69,0x61,0x85,0xbe]
 
 s_floor_f32 s5, s103
-// GFX12: encoding: [0x67,0x61,0x85,0xbe]
+// GFX12: s_floor_f32 s5, s103                    ; encoding: [0x67,0x61,0x85,0xbe]
 
 s_floor_f32 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x61,0x85,0xbe]
+// GFX12: s_floor_f32 s5, vcc_lo                  ; encoding: [0x6a,0x61,0x85,0xbe]
 
 s_floor_f32 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x61,0x85,0xbe]
+// GFX12: s_floor_f32 s5, vcc_hi                  ; encoding: [0x6b,0x61,0x85,0xbe]
 
 s_floor_f32 s5, ttmp11
-// GFX12: encoding: [0x77,0x61,0x85,0xbe]
+// GFX12: s_floor_f32 s5, ttmp11                  ; encoding: [0x77,0x61,0x85,0xbe]
 
 s_floor_f32 s5, m0
-// GFX12: encoding: [0x7d,0x61,0x85,0xbe]
+// GFX12: s_floor_f32 s5, m0                      ; encoding: [0x7d,0x61,0x85,0xbe]
 
 s_floor_f32 s5, exec_lo
-// GFX12: encoding: [0x7e,0x61,0x85,0xbe]
+// GFX12: s_floor_f32 s5, exec_lo                 ; encoding: [0x7e,0x61,0x85,0xbe]
 
 s_floor_f32 s5, exec_hi
-// GFX12: encoding: [0x7f,0x61,0x85,0xbe]
+// GFX12: s_floor_f32 s5, exec_hi                 ; encoding: [0x7f,0x61,0x85,0xbe]
 
 s_floor_f32 s5, 0
-// GFX12: encoding: [0x80,0x61,0x85,0xbe]
+// GFX12: s_floor_f32 s5, 0                       ; encoding: [0x80,0x61,0x85,0xbe]
 
 s_floor_f32 s5, -1
-// GFX12: encoding: [0xc1,0x61,0x85,0xbe]
+// GFX12: s_floor_f32 s5, -1                      ; encoding: [0xc1,0x61,0x85,0xbe]
 
 s_floor_f32 s5, 0.5
-// GFX12: encoding: [0xf0,0x61,0x85,0xbe]
+// GFX12: s_floor_f32 s5, 0.5                     ; encoding: [0xf0,0x61,0x85,0xbe]
 
 s_floor_f32 s5, -4.0
-// GFX12: encoding: [0xf7,0x61,0x85,0xbe]
+// GFX12: s_floor_f32 s5, -4.0                    ; encoding: [0xf7,0x61,0x85,0xbe]
 
 s_floor_f32 s5, 0xaf123456
-// GFX12: encoding: [0xff,0x61,0x85,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_floor_f32 s5, 0xaf123456              ; encoding: [0xff,0x61,0x85,0xbe,0x56,0x34,0x12,0xaf]
 
 s_floor_f32 s5, 0x3f717273
-// GFX12: encoding: [0xff,0x61,0x85,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_floor_f32 s5, 0x3f717273              ; encoding: [0xff,0x61,0x85,0xbe,0x73,0x72,0x71,0x3f]
 
 s_floor_f16 s5, s1
-// GFX12: encoding: [0x01,0x6c,0x85,0xbe]
+// GFX12: s_floor_f16 s5, s1                      ; encoding: [0x01,0x6c,0x85,0xbe]
 
 s_floor_f16 s105, s1
-// GFX12: encoding: [0x01,0x6c,0xe9,0xbe]
+// GFX12: s_floor_f16 s105, s1                    ; encoding: [0x01,0x6c,0xe9,0xbe]
 
 s_floor_f16 s5, s105
-// GFX12: encoding: [0x69,0x6c,0x85,0xbe]
+// GFX12: s_floor_f16 s5, s105                    ; encoding: [0x69,0x6c,0x85,0xbe]
 
 s_floor_f16 s5, s101
-// GFX12: encoding: [0x65,0x6c,0x85,0xbe]
+// GFX12: s_floor_f16 s5, s101                    ; encoding: [0x65,0x6c,0x85,0xbe]
 
 s_floor_f16 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x6c,0x85,0xbe]
+// GFX12: s_floor_f16 s5, vcc_lo                  ; encoding: [0x6a,0x6c,0x85,0xbe]
 
 s_floor_f16 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x6c,0x85,0xbe]
+// GFX12: s_floor_f16 s5, vcc_hi                  ; encoding: [0x6b,0x6c,0x85,0xbe]
 
 s_floor_f16 s5, m0
-// GFX12: encoding: [0x7d,0x6c,0x85,0xbe]
+// GFX12: s_floor_f16 s5, m0                      ; encoding: [0x7d,0x6c,0x85,0xbe]
 
 s_floor_f16 s5, exec_lo
-// GFX12: encoding: [0x7e,0x6c,0x85,0xbe]
+// GFX12: s_floor_f16 s5, exec_lo                 ; encoding: [0x7e,0x6c,0x85,0xbe]
 
 s_floor_f16 s5, exec_hi
-// GFX12: encoding: [0x7f,0x6c,0x85,0xbe]
+// GFX12: s_floor_f16 s5, exec_hi                 ; encoding: [0x7f,0x6c,0x85,0xbe]
 
 s_floor_f16 s5, 0
-// GFX12: encoding: [0x80,0x6c,0x85,0xbe]
+// GFX12: s_floor_f16 s5, 0                       ; encoding: [0x80,0x6c,0x85,0xbe]
 
 s_floor_f16 s5, -1
-// GFX12: encoding: [0xc1,0x6c,0x85,0xbe]
+// GFX12: s_floor_f16 s5, -1                      ; encoding: [0xc1,0x6c,0x85,0xbe]
 
 s_floor_f16 s5, 0.5
-// GFX12: encoding: [0xf0,0x6c,0x85,0xbe]
+// GFX12: s_floor_f16 s5, 0.5                     ; encoding: [0xf0,0x6c,0x85,0xbe]
 
 s_floor_f16 s5, -4.0
-// GFX12: encoding: [0xf7,0x6c,0x85,0xbe]
+// GFX12: s_floor_f16 s5, -4.0                    ; encoding: [0xf7,0x6c,0x85,0xbe]
 
 s_floor_f16 s5, 0xfe0b
-// GFX12: encoding: [0xff,0x6c,0x85,0xbe,0x0b,0xfe,0x00,0x00]
+// GFX12: s_floor_f16 s5, 0xfe0b                  ; encoding: [0xff,0x6c,0x85,0xbe,0x0b,0xfe,0x00,0x00]
 
 s_floor_f16 s5, 0x3456
-// GFX12: encoding: [0xff,0x6c,0x85,0xbe,0x56,0x34,0x00,0x00]
+// GFX12: s_floor_f16 s5, 0x3456                  ; encoding: [0xff,0x6c,0x85,0xbe,0x56,0x34,0x00,0x00]
 
 s_ceil_f16 s5, s1
-// GFX12: encoding: [0x01,0x6b,0x85,0xbe]
+// GFX12: s_ceil_f16 s5, s1                       ; encoding: [0x01,0x6b,0x85,0xbe]
 
 s_ceil_f16 s105, s1
-// GFX12: encoding: [0x01,0x6b,0xe9,0xbe]
+// GFX12: s_ceil_f16 s105, s1                     ; encoding: [0x01,0x6b,0xe9,0xbe]
 
 s_ceil_f16 s5, s105
-// GFX12: encoding: [0x69,0x6b,0x85,0xbe]
+// GFX12: s_ceil_f16 s5, s105                     ; encoding: [0x69,0x6b,0x85,0xbe]
 
 s_ceil_f16 s5, s101
-// GFX12: encoding: [0x65,0x6b,0x85,0xbe]
+// GFX12: s_ceil_f16 s5, s101                     ; encoding: [0x65,0x6b,0x85,0xbe]
 
 s_ceil_f16 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x6b,0x85,0xbe]
+// GFX12: s_ceil_f16 s5, vcc_lo                   ; encoding: [0x6a,0x6b,0x85,0xbe]
 
 s_ceil_f16 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x6b,0x85,0xbe]
+// GFX12: s_ceil_f16 s5, vcc_hi                   ; encoding: [0x6b,0x6b,0x85,0xbe]
 
 s_ceil_f16 s5, m0
-// GFX12: encoding: [0x7d,0x6b,0x85,0xbe]
+// GFX12: s_ceil_f16 s5, m0                       ; encoding: [0x7d,0x6b,0x85,0xbe]
 
 s_ceil_f16 s5, exec_lo
-// GFX12: encoding: [0x7e,0x6b,0x85,0xbe]
+// GFX12: s_ceil_f16 s5, exec_lo                  ; encoding: [0x7e,0x6b,0x85,0xbe]
 
 s_ceil_f16 s5, exec_hi
-// GFX12: encoding: [0x7f,0x6b,0x85,0xbe]
+// GFX12: s_ceil_f16 s5, exec_hi                  ; encoding: [0x7f,0x6b,0x85,0xbe]
 
 s_ceil_f16 s5, 0
-// GFX12: encoding: [0x80,0x6b,0x85,0xbe]
+// GFX12: s_ceil_f16 s5, 0                        ; encoding: [0x80,0x6b,0x85,0xbe]
 
 s_ceil_f16 s5, -1
-// GFX12: encoding: [0xc1,0x6b,0x85,0xbe]
+// GFX12: s_ceil_f16 s5, -1                       ; encoding: [0xc1,0x6b,0x85,0xbe]
 
 s_ceil_f16 s5, 0.5
-// GFX12: encoding: [0xf0,0x6b,0x85,0xbe]
+// GFX12: s_ceil_f16 s5, 0.5                      ; encoding: [0xf0,0x6b,0x85,0xbe]
 
 s_ceil_f16 s5, -4.0
-// GFX12: encoding: [0xf7,0x6b,0x85,0xbe]
+// GFX12: s_ceil_f16 s5, -4.0                     ; encoding: [0xf7,0x6b,0x85,0xbe]
 
 s_ceil_f16 s5, 0xfe0b
-// GFX12: encoding: [0xff,0x6b,0x85,0xbe,0x0b,0xfe,0x00,0x00]
+// GFX12: s_ceil_f16 s5, 0xfe0b                   ; encoding: [0xff,0x6b,0x85,0xbe,0x0b,0xfe,0x00,0x00]
 
 s_ceil_f16 s5, 0x3456
-// GFX12: encoding: [0xff,0x6b,0x85,0xbe,0x56,0x34,0x00,0x00]
+// GFX12: s_ceil_f16 s5, 0x3456                   ; encoding: [0xff,0x6b,0x85,0xbe,0x56,0x34,0x00,0x00]
 
 s_trunc_f16 s5, s1
-// GFX12: encoding: [0x01,0x6d,0x85,0xbe]
+// GFX12: s_trunc_f16 s5, s1                      ; encoding: [0x01,0x6d,0x85,0xbe]
 
 s_trunc_f16 s105, s1
-// GFX12: encoding: [0x01,0x6d,0xe9,0xbe]
+// GFX12: s_trunc_f16 s105, s1                    ; encoding: [0x01,0x6d,0xe9,0xbe]
 
 s_trunc_f16 s5, s105
-// GFX12: encoding: [0x69,0x6d,0x85,0xbe]
+// GFX12: s_trunc_f16 s5, s105                    ; encoding: [0x69,0x6d,0x85,0xbe]
 
 s_trunc_f16 s5, s101
-// GFX12: encoding: [0x65,0x6d,0x85,0xbe]
+// GFX12: s_trunc_f16 s5, s101                    ; encoding: [0x65,0x6d,0x85,0xbe]
 
 s_trunc_f16 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x6d,0x85,0xbe]
+// GFX12: s_trunc_f16 s5, vcc_lo                  ; encoding: [0x6a,0x6d,0x85,0xbe]
 
 s_trunc_f16 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x6d,0x85,0xbe]
+// GFX12: s_trunc_f16 s5, vcc_hi                  ; encoding: [0x6b,0x6d,0x85,0xbe]
 
 s_trunc_f16 s5, m0
-// GFX12: encoding: [0x7d,0x6d,0x85,0xbe]
+// GFX12: s_trunc_f16 s5, m0                      ; encoding: [0x7d,0x6d,0x85,0xbe]
 
 s_trunc_f16 s5, exec_lo
-// GFX12: encoding: [0x7e,0x6d,0x85,0xbe]
+// GFX12: s_trunc_f16 s5, exec_lo                 ; encoding: [0x7e,0x6d,0x85,0xbe]
 
 s_trunc_f16 s5, exec_hi
-// GFX12: encoding: [0x7f,0x6d,0x85,0xbe]
+// GFX12: s_trunc_f16 s5, exec_hi                 ; encoding: [0x7f,0x6d,0x85,0xbe]
 
 s_trunc_f16 s5, 0
-// GFX12: encoding: [0x80,0x6d,0x85,0xbe]
+// GFX12: s_trunc_f16 s5, 0                       ; encoding: [0x80,0x6d,0x85,0xbe]
 
 s_trunc_f16 s5, -1
-// GFX12: encoding: [0xc1,0x6d,0x85,0xbe]
+// GFX12: s_trunc_f16 s5, -1                      ; encoding: [0xc1,0x6d,0x85,0xbe]
 
 s_trunc_f16 s5, 0.5
-// GFX12: encoding: [0xf0,0x6d,0x85,0xbe]
+// GFX12: s_trunc_f16 s5, 0.5                     ; encoding: [0xf0,0x6d,0x85,0xbe]
 
 s_trunc_f16 s5, -4.0
-// GFX12: encoding: [0xf7,0x6d,0x85,0xbe]
+// GFX12: s_trunc_f16 s5, -4.0                    ; encoding: [0xf7,0x6d,0x85,0xbe]
 
 s_trunc_f16 s5, 0xfe0b
-// GFX12: encoding: [0xff,0x6d,0x85,0xbe,0x0b,0xfe,0x00,0x00]
+// GFX12: s_trunc_f16 s5, 0xfe0b                  ; encoding: [0xff,0x6d,0x85,0xbe,0x0b,0xfe,0x00,0x00]
 
 s_trunc_f16 s5, 0x3456
-// GFX12: encoding: [0xff,0x6d,0x85,0xbe,0x56,0x34,0x00,0x00]
+// GFX12: s_trunc_f16 s5, 0x3456                  ; encoding: [0xff,0x6d,0x85,0xbe,0x56,0x34,0x00,0x00]
 
 s_rndne_f16 s5, s1
-// GFX12: encoding: [0x01,0x6e,0x85,0xbe]
+// GFX12: s_rndne_f16 s5, s1                      ; encoding: [0x01,0x6e,0x85,0xbe]
 
 s_rndne_f16 s105, s1
-// GFX12: encoding: [0x01,0x6e,0xe9,0xbe]
+// GFX12: s_rndne_f16 s105, s1                    ; encoding: [0x01,0x6e,0xe9,0xbe]
 
 s_rndne_f16 s5, s105
-// GFX12: encoding: [0x69,0x6e,0x85,0xbe]
+// GFX12: s_rndne_f16 s5, s105                    ; encoding: [0x69,0x6e,0x85,0xbe]
 
 s_rndne_f16 s5, s101
-// GFX12: encoding: [0x65,0x6e,0x85,0xbe]
+// GFX12: s_rndne_f16 s5, s101                    ; encoding: [0x65,0x6e,0x85,0xbe]
 
 s_rndne_f16 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x6e,0x85,0xbe]
+// GFX12: s_rndne_f16 s5, vcc_lo                  ; encoding: [0x6a,0x6e,0x85,0xbe]
 
 s_rndne_f16 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x6e,0x85,0xbe]
+// GFX12: s_rndne_f16 s5, vcc_hi                  ; encoding: [0x6b,0x6e,0x85,0xbe]
 
 s_rndne_f16 s5, m0
-// GFX12: encoding: [0x7d,0x6e,0x85,0xbe]
+// GFX12: s_rndne_f16 s5, m0                      ; encoding: [0x7d,0x6e,0x85,0xbe]
 
 s_rndne_f16 s5, exec_lo
-// GFX12: encoding: [0x7e,0x6e,0x85,0xbe]
+// GFX12: s_rndne_f16 s5, exec_lo                 ; encoding: [0x7e,0x6e,0x85,0xbe]
 
 s_rndne_f16 s5, exec_hi
-// GFX12: encoding: [0x7f,0x6e,0x85,0xbe]
+// GFX12: s_rndne_f16 s5, exec_hi                 ; encoding: [0x7f,0x6e,0x85,0xbe]
 
 s_rndne_f16 s5, 0
-// GFX12: encoding: [0x80,0x6e,0x85,0xbe]
+// GFX12: s_rndne_f16 s5, 0                       ; encoding: [0x80,0x6e,0x85,0xbe]
 
 s_rndne_f16 s5, -1
-// GFX12: encoding: [0xc1,0x6e,0x85,0xbe]
+// GFX12: s_rndne_f16 s5, -1                      ; encoding: [0xc1,0x6e,0x85,0xbe]
 
 s_rndne_f16 s5, 0.5
-// GFX12: encoding: [0xf0,0x6e,0x85,0xbe]
+// GFX12: s_rndne_f16 s5, 0.5                     ; encoding: [0xf0,0x6e,0x85,0xbe]
 
 s_rndne_f16 s5, -4.0
-// GFX12: encoding: [0xf7,0x6e,0x85,0xbe]
+// GFX12: s_rndne_f16 s5, -4.0                    ; encoding: [0xf7,0x6e,0x85,0xbe]
 
 s_rndne_f16 s5, 0xfe0b
-// GFX12: encoding: [0xff,0x6e,0x85,0xbe,0x0b,0xfe,0x00,0x00]
+// GFX12: s_rndne_f16 s5, 0xfe0b                  ; encoding: [0xff,0x6e,0x85,0xbe,0x0b,0xfe,0x00,0x00]
 
 s_rndne_f16 s5, 0x3456
-// GFX12: encoding: [0xff,0x6e,0x85,0xbe,0x56,0x34,0x00,0x00]
+// GFX12: s_rndne_f16 s5, 0x3456                  ; encoding: [0xff,0x6e,0x85,0xbe,0x56,0x34,0x00,0x00]
 
 s_barrier_signal -2
-// GFX12: encoding: [0xc2,0x4e,0x80,0xbe]
+// GFX12: s_barrier_signal -2                     ; encoding: [0xc2,0x4e,0x80,0xbe]
 
 s_barrier_signal -1
-// GFX12: encoding: [0xc1,0x4e,0x80,0xbe]
+// GFX12: s_barrier_signal -1                     ; encoding: [0xc1,0x4e,0x80,0xbe]
 
 s_barrier_signal m0
-// GFX12: encoding: [0x7d,0x4e,0x80,0xbe]
+// GFX12: s_barrier_signal m0                     ; encoding: [0x7d,0x4e,0x80,0xbe]
 
 s_barrier_signal_isfirst -2
-// GFX12: encoding: [0xc2,0x4f,0x80,0xbe]
+// GFX12: s_barrier_signal_isfirst -2             ; encoding: [0xc2,0x4f,0x80,0xbe]
 
 s_barrier_signal_isfirst -1
-// GFX12: encoding: [0xc1,0x4f,0x80,0xbe]
+// GFX12: s_barrier_signal_isfirst -1             ; encoding: [0xc1,0x4f,0x80,0xbe]
 
 s_barrier_signal_isfirst m0
-// GFX12: encoding: [0x7d,0x4f,0x80,0xbe]
+// GFX12: s_barrier_signal_isfirst m0             ; encoding: [0x7d,0x4f,0x80,0xbe]
 
 s_get_barrier_state s3, -1
-// GFX12: encoding: [0xc1,0x50,0x83,0xbe]
+// GFX12: s_get_barrier_state s3, -1              ; encoding: [0xc1,0x50,0x83,0xbe]
 
 s_get_barrier_state s3, -2
-// GFX12: encoding: [0xc2,0x50,0x83,0xbe]
+// GFX12: s_get_barrier_state s3, -2              ; encoding: [0xc2,0x50,0x83,0xbe]
 
 s_get_barrier_state s3, m0
-// GFX12: encoding: [0x7d,0x50,0x83,0xbe]
+// GFX12: s_get_barrier_state s3, m0              ; encoding: [0x7d,0x50,0x83,0xbe]
 
 s_mov_b32 s0, s1
-// GFX12: encoding: [0x01,0x00,0x80,0xbe]
+// GFX12: s_mov_b32 s0, s1                        ; encoding: [0x01,0x00,0x80,0xbe]
 
 s_mov_b32 s105, s104
-// GFX12: encoding: [0x68,0x00,0xe9,0xbe]
+// GFX12: s_mov_b32 s105, s104                    ; encoding: [0x68,0x00,0xe9,0xbe]
 
 s_mov_b32 s0, s104
-// GFX12: encoding: [0x68,0x00,0x80,0xbe]
+// GFX12: s_mov_b32 s0, s104                      ; encoding: [0x68,0x00,0x80,0xbe]
 
 s_mov_b32 s105, s1
-// GFX12: encoding: [0x01,0x00,0xe9,0xbe]
+// GFX12: s_mov_b32 s105, s1                      ; encoding: [0x01,0x00,0xe9,0xbe]
 
 s_mov_b32 exec_lo, s1
-// GFX12: encoding: [0x01,0x00,0xfe,0xbe]
+// GFX12: s_mov_b32 exec_lo, s1                   ; encoding: [0x01,0x00,0xfe,0xbe]
 
 s_mov_b32 exec_hi, s1
-// GFX12: encoding: [0x01,0x00,0xff,0xbe]
+// GFX12: s_mov_b32 exec_hi, s1                   ; encoding: [0x01,0x00,0xff,0xbe]
 
 s_mov_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x00,0xea,0xbe]
+// GFX12: s_mov_b32 vcc_lo, s1                    ; encoding: [0x01,0x00,0xea,0xbe]
 
 s_mov_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x00,0xeb,0xbe]
+// GFX12: s_mov_b32 vcc_hi, s1                    ; encoding: [0x01,0x00,0xeb,0xbe]
 
 s_mov_b32 m0, s1
-// GFX12: encoding: [0x01,0x00,0xfd,0xbe]
+// GFX12: s_mov_b32 m0, s1                        ; encoding: [0x01,0x00,0xfd,0xbe]
 
 s_mov_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x00,0x80,0xbe]
+// GFX12: s_mov_b32 s0, exec_lo                   ; encoding: [0x7e,0x00,0x80,0xbe]
 
 s_mov_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x00,0x80,0xbe]
+// GFX12: s_mov_b32 s0, exec_hi                   ; encoding: [0x7f,0x00,0x80,0xbe]
 
 s_mov_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x00,0x80,0xbe]
+// GFX12: s_mov_b32 s0, vcc_lo                    ; encoding: [0x6a,0x00,0x80,0xbe]
 
 s_mov_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x00,0x80,0xbe]
+// GFX12: s_mov_b32 s0, vcc_hi                    ; encoding: [0x6b,0x00,0x80,0xbe]
 
 s_mov_b32 s0, m0
-// GFX12: encoding: [0x7d,0x00,0x80,0xbe]
+// GFX12: s_mov_b32 s0, m0                        ; encoding: [0x7d,0x00,0x80,0xbe]
 
 s_mov_b32 s0, 0
-// GFX12: encoding: [0x80,0x00,0x80,0xbe]
+// GFX12: s_mov_b32 s0, 0                         ; encoding: [0x80,0x00,0x80,0xbe]
 
 s_mov_b32 s0, -1
-// GFX12: encoding: [0xc1,0x00,0x80,0xbe]
+// GFX12: s_mov_b32 s0, -1                        ; encoding: [0xc1,0x00,0x80,0xbe]
 
 s_mov_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x00,0x80,0xbe]
+// GFX12: s_mov_b32 s0, 0.5                       ; encoding: [0xf0,0x00,0x80,0xbe]
 
 s_mov_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x00,0x80,0xbe]
+// GFX12: s_mov_b32 s0, -4.0                      ; encoding: [0xf7,0x00,0x80,0xbe]
 
 s_mov_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x00,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_mov_b32 s0, 0x3f717273                ; encoding: [0xff,0x00,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_mov_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x00,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_mov_b32 s0, 0xaf123456                ; encoding: [0xff,0x00,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_mov_b32 s0, null
-// GFX12: encoding: [0x7c,0x00,0x80,0xbe]
+// GFX12: s_mov_b32 s0, null                      ; encoding: [0x7c,0x00,0x80,0xbe]
 
 s_mov_b32 null, s1
-// GFX12: encoding: [0x01,0x00,0xfc,0xbe]
+// GFX12: s_mov_b32 null, s1                      ; encoding: [0x01,0x00,0xfc,0xbe]
 
 s_mov_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x01,0x80,0xbe]
+// GFX12: s_mov_b64 s[0:1], s[2:3]                ; encoding: [0x02,0x01,0x80,0xbe]
 
 s_mov_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x01,0xe8,0xbe]
+// GFX12: s_mov_b64 s[104:105], s[102:103]        ; encoding: [0x66,0x01,0xe8,0xbe]
 
 s_mov_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x01,0x80,0xbe]
+// GFX12: s_mov_b64 s[0:1], s[102:103]            ; encoding: [0x66,0x01,0x80,0xbe]
 
 s_mov_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x01,0xe8,0xbe]
+// GFX12: s_mov_b64 s[104:105], s[2:3]            ; encoding: [0x02,0x01,0xe8,0xbe]
 
 s_mov_b64 exec, s[2:3]
-// GFX12: encoding: [0x02,0x01,0xfe,0xbe]
+// GFX12: s_mov_b64 exec, s[2:3]                  ; encoding: [0x02,0x01,0xfe,0xbe]
 
 s_mov_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x01,0xea,0xbe]
+// GFX12: s_mov_b64 vcc, s[2:3]                   ; encoding: [0x02,0x01,0xea,0xbe]
 
 s_mov_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x01,0x80,0xbe]
+// GFX12: s_mov_b64 s[0:1], exec                  ; encoding: [0x7e,0x01,0x80,0xbe]
 
 s_mov_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x01,0x80,0xbe]
+// GFX12: s_mov_b64 s[0:1], vcc                   ; encoding: [0x6a,0x01,0x80,0xbe]
 
 s_mov_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x01,0x80,0xbe]
+// GFX12: s_mov_b64 s[0:1], 0                     ; encoding: [0x80,0x01,0x80,0xbe]
 
 s_mov_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x01,0x80,0xbe]
+// GFX12: s_mov_b64 s[0:1], -1                    ; encoding: [0xc1,0x01,0x80,0xbe]
 
 s_mov_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x01,0x80,0xbe]
+// GFX12: s_mov_b64 s[0:1], 0.5                   ; encoding: [0xf0,0x01,0x80,0xbe]
 
 s_mov_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x01,0x80,0xbe]
+// GFX12: s_mov_b64 s[0:1], -4.0                  ; encoding: [0xf7,0x01,0x80,0xbe]
 
 s_mov_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x01,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_mov_b64 s[0:1], 0x3f717273            ; encoding: [0xff,0x01,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_mov_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x01,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_mov_b64 s[0:1], 0xaf123456            ; encoding: [0xff,0x01,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_mov_b64 s[0:1], null
-// GFX12: encoding: [0x7c,0x01,0x80,0xbe]
+// GFX12: s_mov_b64 s[0:1], null                  ; encoding: [0x7c,0x01,0x80,0xbe]
 
 s_mov_b64 null, s[2:3]
-// GFX12: encoding: [0x02,0x01,0xfc,0xbe]
+// GFX12: s_mov_b64 null, s[2:3]                  ; encoding: [0x02,0x01,0xfc,0xbe]
 
 s_cmov_b32 s0, s1
-// GFX12: encoding: [0x01,0x02,0x80,0xbe]
+// GFX12: s_cmov_b32 s0, s1                       ; encoding: [0x01,0x02,0x80,0xbe]
 
 s_cmov_b32 s105, s104
-// GFX12: encoding: [0x68,0x02,0xe9,0xbe]
+// GFX12: s_cmov_b32 s105, s104                   ; encoding: [0x68,0x02,0xe9,0xbe]
 
 s_cmov_b32 s0, s104
-// GFX12: encoding: [0x68,0x02,0x80,0xbe]
+// GFX12: s_cmov_b32 s0, s104                     ; encoding: [0x68,0x02,0x80,0xbe]
 
 s_cmov_b32 s105, s1
-// GFX12: encoding: [0x01,0x02,0xe9,0xbe]
+// GFX12: s_cmov_b32 s105, s1                     ; encoding: [0x01,0x02,0xe9,0xbe]
 
 s_cmov_b32 exec_lo, s1
-// GFX12: encoding: [0x01,0x02,0xfe,0xbe]
+// GFX12: s_cmov_b32 exec_lo, s1                  ; encoding: [0x01,0x02,0xfe,0xbe]
 
 s_cmov_b32 exec_hi, s1
-// GFX12: encoding: [0x01,0x02,0xff,0xbe]
+// GFX12: s_cmov_b32 exec_hi, s1                  ; encoding: [0x01,0x02,0xff,0xbe]
 
 s_cmov_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x02,0xea,0xbe]
+// GFX12: s_cmov_b32 vcc_lo, s1                   ; encoding: [0x01,0x02,0xea,0xbe]
 
 s_cmov_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x02,0xeb,0xbe]
+// GFX12: s_cmov_b32 vcc_hi, s1                   ; encoding: [0x01,0x02,0xeb,0xbe]
 
 s_cmov_b32 m0, s1
-// GFX12: encoding: [0x01,0x02,0xfd,0xbe]
+// GFX12: s_cmov_b32 m0, s1                       ; encoding: [0x01,0x02,0xfd,0xbe]
 
 s_cmov_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x02,0x80,0xbe]
+// GFX12: s_cmov_b32 s0, exec_lo                  ; encoding: [0x7e,0x02,0x80,0xbe]
 
 s_cmov_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x02,0x80,0xbe]
+// GFX12: s_cmov_b32 s0, exec_hi                  ; encoding: [0x7f,0x02,0x80,0xbe]
 
 s_cmov_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x02,0x80,0xbe]
+// GFX12: s_cmov_b32 s0, vcc_lo                   ; encoding: [0x6a,0x02,0x80,0xbe]
 
 s_cmov_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x02,0x80,0xbe]
+// GFX12: s_cmov_b32 s0, vcc_hi                   ; encoding: [0x6b,0x02,0x80,0xbe]
 
 s_cmov_b32 s0, m0
-// GFX12: encoding: [0x7d,0x02,0x80,0xbe]
+// GFX12: s_cmov_b32 s0, m0                       ; encoding: [0x7d,0x02,0x80,0xbe]
 
 s_cmov_b32 s0, 0
-// GFX12: encoding: [0x80,0x02,0x80,0xbe]
+// GFX12: s_cmov_b32 s0, 0                        ; encoding: [0x80,0x02,0x80,0xbe]
 
 s_cmov_b32 s0, -1
-// GFX12: encoding: [0xc1,0x02,0x80,0xbe]
+// GFX12: s_cmov_b32 s0, -1                       ; encoding: [0xc1,0x02,0x80,0xbe]
 
 s_cmov_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x02,0x80,0xbe]
+// GFX12: s_cmov_b32 s0, 0.5                      ; encoding: [0xf0,0x02,0x80,0xbe]
 
 s_cmov_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x02,0x80,0xbe]
+// GFX12: s_cmov_b32 s0, -4.0                     ; encoding: [0xf7,0x02,0x80,0xbe]
 
 s_cmov_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x02,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmov_b32 s0, 0x3f717273               ; encoding: [0xff,0x02,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_cmov_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x02,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmov_b32 s0, 0xaf123456               ; encoding: [0xff,0x02,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_cmov_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x03,0x80,0xbe]
+// GFX12: s_cmov_b64 s[0:1], s[2:3]               ; encoding: [0x02,0x03,0x80,0xbe]
 
 s_cmov_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x03,0xe8,0xbe]
+// GFX12: s_cmov_b64 s[104:105], s[102:103]       ; encoding: [0x66,0x03,0xe8,0xbe]
 
 s_cmov_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x03,0x80,0xbe]
+// GFX12: s_cmov_b64 s[0:1], s[102:103]           ; encoding: [0x66,0x03,0x80,0xbe]
 
 s_cmov_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x03,0xe8,0xbe]
+// GFX12: s_cmov_b64 s[104:105], s[2:3]           ; encoding: [0x02,0x03,0xe8,0xbe]
 
 s_cmov_b64 exec, s[2:3]
-// GFX12: encoding: [0x02,0x03,0xfe,0xbe]
+// GFX12: s_cmov_b64 exec, s[2:3]                 ; encoding: [0x02,0x03,0xfe,0xbe]
 
 s_cmov_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x03,0xea,0xbe]
+// GFX12: s_cmov_b64 vcc, s[2:3]                  ; encoding: [0x02,0x03,0xea,0xbe]
 
 s_cmov_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x03,0x80,0xbe]
+// GFX12: s_cmov_b64 s[0:1], exec                 ; encoding: [0x7e,0x03,0x80,0xbe]
 
 s_cmov_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x03,0x80,0xbe]
+// GFX12: s_cmov_b64 s[0:1], vcc                  ; encoding: [0x6a,0x03,0x80,0xbe]
 
 s_cmov_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x03,0x80,0xbe]
+// GFX12: s_cmov_b64 s[0:1], 0                    ; encoding: [0x80,0x03,0x80,0xbe]
 
 s_cmov_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x03,0x80,0xbe]
+// GFX12: s_cmov_b64 s[0:1], -1                   ; encoding: [0xc1,0x03,0x80,0xbe]
 
 s_cmov_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x03,0x80,0xbe]
+// GFX12: s_cmov_b64 s[0:1], 0.5                  ; encoding: [0xf0,0x03,0x80,0xbe]
 
 s_cmov_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x03,0x80,0xbe]
+// GFX12: s_cmov_b64 s[0:1], -4.0                 ; encoding: [0xf7,0x03,0x80,0xbe]
 
 s_cmov_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x03,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmov_b64 s[0:1], 0x3f717273           ; encoding: [0xff,0x03,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_cmov_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x03,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmov_b64 s[0:1], 0xaf123456           ; encoding: [0xff,0x03,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_not_b32 s0, s1
-// GFX12: encoding: [0x01,0x1e,0x80,0xbe]
+// GFX12: s_not_b32 s0, s1                        ; encoding: [0x01,0x1e,0x80,0xbe]
 
 s_not_b32 s105, s104
-// GFX12: encoding: [0x68,0x1e,0xe9,0xbe]
+// GFX12: s_not_b32 s105, s104                    ; encoding: [0x68,0x1e,0xe9,0xbe]
 
 s_not_b32 s0, s104
-// GFX12: encoding: [0x68,0x1e,0x80,0xbe]
+// GFX12: s_not_b32 s0, s104                      ; encoding: [0x68,0x1e,0x80,0xbe]
 
 s_not_b32 s105, s1
-// GFX12: encoding: [0x01,0x1e,0xe9,0xbe]
+// GFX12: s_not_b32 s105, s1                      ; encoding: [0x01,0x1e,0xe9,0xbe]
 
 s_not_b32 exec_lo, s1
-// GFX12: encoding: [0x01,0x1e,0xfe,0xbe]
+// GFX12: s_not_b32 exec_lo, s1                   ; encoding: [0x01,0x1e,0xfe,0xbe]
 
 s_not_b32 exec_hi, s1
-// GFX12: encoding: [0x01,0x1e,0xff,0xbe]
+// GFX12: s_not_b32 exec_hi, s1                   ; encoding: [0x01,0x1e,0xff,0xbe]
 
 s_not_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x1e,0xea,0xbe]
+// GFX12: s_not_b32 vcc_lo, s1                    ; encoding: [0x01,0x1e,0xea,0xbe]
 
 s_not_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x1e,0xeb,0xbe]
+// GFX12: s_not_b32 vcc_hi, s1                    ; encoding: [0x01,0x1e,0xeb,0xbe]
 
 s_not_b32 m0, s1
-// GFX12: encoding: [0x01,0x1e,0xfd,0xbe]
+// GFX12: s_not_b32 m0, s1                        ; encoding: [0x01,0x1e,0xfd,0xbe]
 
 s_not_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x1e,0x80,0xbe]
+// GFX12: s_not_b32 s0, exec_lo                   ; encoding: [0x7e,0x1e,0x80,0xbe]
 
 s_not_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x1e,0x80,0xbe]
+// GFX12: s_not_b32 s0, exec_hi                   ; encoding: [0x7f,0x1e,0x80,0xbe]
 
 s_not_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x1e,0x80,0xbe]
+// GFX12: s_not_b32 s0, vcc_lo                    ; encoding: [0x6a,0x1e,0x80,0xbe]
 
 s_not_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x1e,0x80,0xbe]
+// GFX12: s_not_b32 s0, vcc_hi                    ; encoding: [0x6b,0x1e,0x80,0xbe]
 
 s_not_b32 s0, m0
-// GFX12: encoding: [0x7d,0x1e,0x80,0xbe]
+// GFX12: s_not_b32 s0, m0                        ; encoding: [0x7d,0x1e,0x80,0xbe]
 
 s_not_b32 s0, 0
-// GFX12: encoding: [0x80,0x1e,0x80,0xbe]
+// GFX12: s_not_b32 s0, 0                         ; encoding: [0x80,0x1e,0x80,0xbe]
 
 s_not_b32 s0, -1
-// GFX12: encoding: [0xc1,0x1e,0x80,0xbe]
+// GFX12: s_not_b32 s0, -1                        ; encoding: [0xc1,0x1e,0x80,0xbe]
 
 s_not_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x1e,0x80,0xbe]
+// GFX12: s_not_b32 s0, 0.5                       ; encoding: [0xf0,0x1e,0x80,0xbe]
 
 s_not_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x1e,0x80,0xbe]
+// GFX12: s_not_b32 s0, -4.0                      ; encoding: [0xf7,0x1e,0x80,0xbe]
 
 s_not_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x1e,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_not_b32 s0, 0x3f717273                ; encoding: [0xff,0x1e,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_not_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x1e,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_not_b32 s0, 0xaf123456                ; encoding: [0xff,0x1e,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_not_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x1f,0x80,0xbe]
+// GFX12: s_not_b64 s[0:1], s[2:3]                ; encoding: [0x02,0x1f,0x80,0xbe]
 
 s_not_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x1f,0xe8,0xbe]
+// GFX12: s_not_b64 s[104:105], s[102:103]        ; encoding: [0x66,0x1f,0xe8,0xbe]
 
 s_not_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x1f,0x80,0xbe]
+// GFX12: s_not_b64 s[0:1], s[102:103]            ; encoding: [0x66,0x1f,0x80,0xbe]
 
 s_not_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x1f,0xe8,0xbe]
+// GFX12: s_not_b64 s[104:105], s[2:3]            ; encoding: [0x02,0x1f,0xe8,0xbe]
 
 s_not_b64 exec, s[2:3]
-// GFX12: encoding: [0x02,0x1f,0xfe,0xbe]
+// GFX12: s_not_b64 exec, s[2:3]                  ; encoding: [0x02,0x1f,0xfe,0xbe]
 
 s_not_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x1f,0xea,0xbe]
+// GFX12: s_not_b64 vcc, s[2:3]                   ; encoding: [0x02,0x1f,0xea,0xbe]
 
 s_not_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x1f,0x80,0xbe]
+// GFX12: s_not_b64 s[0:1], exec                  ; encoding: [0x7e,0x1f,0x80,0xbe]
 
 s_not_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x1f,0x80,0xbe]
+// GFX12: s_not_b64 s[0:1], vcc                   ; encoding: [0x6a,0x1f,0x80,0xbe]
 
 s_not_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x1f,0x80,0xbe]
+// GFX12: s_not_b64 s[0:1], 0                     ; encoding: [0x80,0x1f,0x80,0xbe]
 
 s_not_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x1f,0x80,0xbe]
+// GFX12: s_not_b64 s[0:1], -1                    ; encoding: [0xc1,0x1f,0x80,0xbe]
 
 s_not_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x1f,0x80,0xbe]
+// GFX12: s_not_b64 s[0:1], 0.5                   ; encoding: [0xf0,0x1f,0x80,0xbe]
 
 s_not_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x1f,0x80,0xbe]
+// GFX12: s_not_b64 s[0:1], -4.0                  ; encoding: [0xf7,0x1f,0x80,0xbe]
 
 s_not_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x1f,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_not_b64 s[0:1], 0x3f717273            ; encoding: [0xff,0x1f,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_not_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x1f,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_not_b64 s[0:1], 0xaf123456            ; encoding: [0xff,0x1f,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_wqm_b32 s0, s1
-// GFX12: encoding: [0x01,0x1c,0x80,0xbe]
+// GFX12: s_wqm_b32 s0, s1                        ; encoding: [0x01,0x1c,0x80,0xbe]
 
 s_wqm_b32 s105, s104
-// GFX12: encoding: [0x68,0x1c,0xe9,0xbe]
+// GFX12: s_wqm_b32 s105, s104                    ; encoding: [0x68,0x1c,0xe9,0xbe]
 
 s_wqm_b32 s0, s104
-// GFX12: encoding: [0x68,0x1c,0x80,0xbe]
+// GFX12: s_wqm_b32 s0, s104                      ; encoding: [0x68,0x1c,0x80,0xbe]
 
 s_wqm_b32 s105, s1
-// GFX12: encoding: [0x01,0x1c,0xe9,0xbe]
+// GFX12: s_wqm_b32 s105, s1                      ; encoding: [0x01,0x1c,0xe9,0xbe]
 
 s_wqm_b32 exec_lo, s1
-// GFX12: encoding: [0x01,0x1c,0xfe,0xbe]
+// GFX12: s_wqm_b32 exec_lo, s1                   ; encoding: [0x01,0x1c,0xfe,0xbe]
 
 s_wqm_b32 exec_hi, s1
-// GFX12: encoding: [0x01,0x1c,0xff,0xbe]
+// GFX12: s_wqm_b32 exec_hi, s1                   ; encoding: [0x01,0x1c,0xff,0xbe]
 
 s_wqm_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x1c,0xea,0xbe]
+// GFX12: s_wqm_b32 vcc_lo, s1                    ; encoding: [0x01,0x1c,0xea,0xbe]
 
 s_wqm_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x1c,0xeb,0xbe]
+// GFX12: s_wqm_b32 vcc_hi, s1                    ; encoding: [0x01,0x1c,0xeb,0xbe]
 
 s_wqm_b32 m0, s1
-// GFX12: encoding: [0x01,0x1c,0xfd,0xbe]
+// GFX12: s_wqm_b32 m0, s1                        ; encoding: [0x01,0x1c,0xfd,0xbe]
 
 s_wqm_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x1c,0x80,0xbe]
+// GFX12: s_wqm_b32 s0, exec_lo                   ; encoding: [0x7e,0x1c,0x80,0xbe]
 
 s_wqm_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x1c,0x80,0xbe]
+// GFX12: s_wqm_b32 s0, exec_hi                   ; encoding: [0x7f,0x1c,0x80,0xbe]
 
 s_wqm_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x1c,0x80,0xbe]
+// GFX12: s_wqm_b32 s0, vcc_lo                    ; encoding: [0x6a,0x1c,0x80,0xbe]
 
 s_wqm_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x1c,0x80,0xbe]
+// GFX12: s_wqm_b32 s0, vcc_hi                    ; encoding: [0x6b,0x1c,0x80,0xbe]
 
 s_wqm_b32 s0, m0
-// GFX12: encoding: [0x7d,0x1c,0x80,0xbe]
+// GFX12: s_wqm_b32 s0, m0                        ; encoding: [0x7d,0x1c,0x80,0xbe]
 
 s_wqm_b32 s0, 0
-// GFX12: encoding: [0x80,0x1c,0x80,0xbe]
+// GFX12: s_wqm_b32 s0, 0                         ; encoding: [0x80,0x1c,0x80,0xbe]
 
 s_wqm_b32 s0, -1
-// GFX12: encoding: [0xc1,0x1c,0x80,0xbe]
+// GFX12: s_wqm_b32 s0, -1                        ; encoding: [0xc1,0x1c,0x80,0xbe]
 
 s_wqm_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x1c,0x80,0xbe]
+// GFX12: s_wqm_b32 s0, 0.5                       ; encoding: [0xf0,0x1c,0x80,0xbe]
 
 s_wqm_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x1c,0x80,0xbe]
+// GFX12: s_wqm_b32 s0, -4.0                      ; encoding: [0xf7,0x1c,0x80,0xbe]
 
 s_wqm_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x1c,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_wqm_b32 s0, 0x3f717273                ; encoding: [0xff,0x1c,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_wqm_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x1c,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_wqm_b32 s0, 0xaf123456                ; encoding: [0xff,0x1c,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_wqm_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x1d,0x80,0xbe]
+// GFX12: s_wqm_b64 s[0:1], s[2:3]                ; encoding: [0x02,0x1d,0x80,0xbe]
 
 s_wqm_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x1d,0xe8,0xbe]
+// GFX12: s_wqm_b64 s[104:105], s[102:103]        ; encoding: [0x66,0x1d,0xe8,0xbe]
 
 s_wqm_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x1d,0x80,0xbe]
+// GFX12: s_wqm_b64 s[0:1], s[102:103]            ; encoding: [0x66,0x1d,0x80,0xbe]
 
 s_wqm_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x1d,0xe8,0xbe]
+// GFX12: s_wqm_b64 s[104:105], s[2:3]            ; encoding: [0x02,0x1d,0xe8,0xbe]
 
 s_wqm_b64 exec, s[2:3]
-// GFX12: encoding: [0x02,0x1d,0xfe,0xbe]
+// GFX12: s_wqm_b64 exec, s[2:3]                  ; encoding: [0x02,0x1d,0xfe,0xbe]
 
 s_wqm_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x1d,0xea,0xbe]
+// GFX12: s_wqm_b64 vcc, s[2:3]                   ; encoding: [0x02,0x1d,0xea,0xbe]
 
 s_wqm_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x1d,0x80,0xbe]
+// GFX12: s_wqm_b64 s[0:1], exec                  ; encoding: [0x7e,0x1d,0x80,0xbe]
 
 s_wqm_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x1d,0x80,0xbe]
+// GFX12: s_wqm_b64 s[0:1], vcc                   ; encoding: [0x6a,0x1d,0x80,0xbe]
 
 s_wqm_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x1d,0x80,0xbe]
+// GFX12: s_wqm_b64 s[0:1], 0                     ; encoding: [0x80,0x1d,0x80,0xbe]
 
 s_wqm_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x1d,0x80,0xbe]
+// GFX12: s_wqm_b64 s[0:1], -1                    ; encoding: [0xc1,0x1d,0x80,0xbe]
 
 s_wqm_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x1d,0x80,0xbe]
+// GFX12: s_wqm_b64 s[0:1], 0.5                   ; encoding: [0xf0,0x1d,0x80,0xbe]
 
 s_wqm_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x1d,0x80,0xbe]
+// GFX12: s_wqm_b64 s[0:1], -4.0                  ; encoding: [0xf7,0x1d,0x80,0xbe]
 
 s_wqm_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x1d,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_wqm_b64 s[0:1], 0x3f717273            ; encoding: [0xff,0x1d,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_wqm_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x1d,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_wqm_b64 s[0:1], 0xaf123456            ; encoding: [0xff,0x1d,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_brev_b32 s0, s1
-// GFX12: encoding: [0x01,0x04,0x80,0xbe]
+// GFX12: s_brev_b32 s0, s1                       ; encoding: [0x01,0x04,0x80,0xbe]
 
 s_brev_b32 s105, s104
-// GFX12: encoding: [0x68,0x04,0xe9,0xbe]
+// GFX12: s_brev_b32 s105, s104                   ; encoding: [0x68,0x04,0xe9,0xbe]
 
 s_brev_b32 s0, s104
-// GFX12: encoding: [0x68,0x04,0x80,0xbe]
+// GFX12: s_brev_b32 s0, s104                     ; encoding: [0x68,0x04,0x80,0xbe]
 
 s_brev_b32 s105, s1
-// GFX12: encoding: [0x01,0x04,0xe9,0xbe]
+// GFX12: s_brev_b32 s105, s1                     ; encoding: [0x01,0x04,0xe9,0xbe]
 
 s_brev_b32 exec_lo, s1
-// GFX12: encoding: [0x01,0x04,0xfe,0xbe]
+// GFX12: s_brev_b32 exec_lo, s1                  ; encoding: [0x01,0x04,0xfe,0xbe]
 
 s_brev_b32 exec_hi, s1
-// GFX12: encoding: [0x01,0x04,0xff,0xbe]
+// GFX12: s_brev_b32 exec_hi, s1                  ; encoding: [0x01,0x04,0xff,0xbe]
 
 s_brev_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x04,0xea,0xbe]
+// GFX12: s_brev_b32 vcc_lo, s1                   ; encoding: [0x01,0x04,0xea,0xbe]
 
 s_brev_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x04,0xeb,0xbe]
+// GFX12: s_brev_b32 vcc_hi, s1                   ; encoding: [0x01,0x04,0xeb,0xbe]
 
 s_brev_b32 m0, s1
-// GFX12: encoding: [0x01,0x04,0xfd,0xbe]
+// GFX12: s_brev_b32 m0, s1                       ; encoding: [0x01,0x04,0xfd,0xbe]
 
 s_brev_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x04,0x80,0xbe]
+// GFX12: s_brev_b32 s0, exec_lo                  ; encoding: [0x7e,0x04,0x80,0xbe]
 
 s_brev_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x04,0x80,0xbe]
+// GFX12: s_brev_b32 s0, exec_hi                  ; encoding: [0x7f,0x04,0x80,0xbe]
 
 s_brev_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x04,0x80,0xbe]
+// GFX12: s_brev_b32 s0, vcc_lo                   ; encoding: [0x6a,0x04,0x80,0xbe]
 
 s_brev_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x04,0x80,0xbe]
+// GFX12: s_brev_b32 s0, vcc_hi                   ; encoding: [0x6b,0x04,0x80,0xbe]
 
 s_brev_b32 s0, m0
-// GFX12: encoding: [0x7d,0x04,0x80,0xbe]
+// GFX12: s_brev_b32 s0, m0                       ; encoding: [0x7d,0x04,0x80,0xbe]
 
 s_brev_b32 s0, 0
-// GFX12: encoding: [0x80,0x04,0x80,0xbe]
+// GFX12: s_brev_b32 s0, 0                        ; encoding: [0x80,0x04,0x80,0xbe]
 
 s_brev_b32 s0, -1
-// GFX12: encoding: [0xc1,0x04,0x80,0xbe]
+// GFX12: s_brev_b32 s0, -1                       ; encoding: [0xc1,0x04,0x80,0xbe]
 
 s_brev_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x04,0x80,0xbe]
+// GFX12: s_brev_b32 s0, 0.5                      ; encoding: [0xf0,0x04,0x80,0xbe]
 
 s_brev_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x04,0x80,0xbe]
+// GFX12: s_brev_b32 s0, -4.0                     ; encoding: [0xf7,0x04,0x80,0xbe]
 
 s_brev_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x04,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_brev_b32 s0, 0x3f717273               ; encoding: [0xff,0x04,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_brev_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x04,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_brev_b32 s0, 0xaf123456               ; encoding: [0xff,0x04,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_brev_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x05,0x80,0xbe]
+// GFX12: s_brev_b64 s[0:1], s[2:3]               ; encoding: [0x02,0x05,0x80,0xbe]
 
 s_brev_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x05,0xe8,0xbe]
+// GFX12: s_brev_b64 s[104:105], s[102:103]       ; encoding: [0x66,0x05,0xe8,0xbe]
 
 s_brev_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x05,0x80,0xbe]
+// GFX12: s_brev_b64 s[0:1], s[102:103]           ; encoding: [0x66,0x05,0x80,0xbe]
 
 s_brev_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x05,0xe8,0xbe]
+// GFX12: s_brev_b64 s[104:105], s[2:3]           ; encoding: [0x02,0x05,0xe8,0xbe]
 
 s_brev_b64 exec, s[2:3]
-// GFX12: encoding: [0x02,0x05,0xfe,0xbe]
+// GFX12: s_brev_b64 exec, s[2:3]                 ; encoding: [0x02,0x05,0xfe,0xbe]
 
 s_brev_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x05,0xea,0xbe]
+// GFX12: s_brev_b64 vcc, s[2:3]                  ; encoding: [0x02,0x05,0xea,0xbe]
 
 s_brev_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x05,0x80,0xbe]
+// GFX12: s_brev_b64 s[0:1], exec                 ; encoding: [0x7e,0x05,0x80,0xbe]
 
 s_brev_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x05,0x80,0xbe]
+// GFX12: s_brev_b64 s[0:1], vcc                  ; encoding: [0x6a,0x05,0x80,0xbe]
 
 s_brev_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x05,0x80,0xbe]
+// GFX12: s_brev_b64 s[0:1], 0                    ; encoding: [0x80,0x05,0x80,0xbe]
 
 s_brev_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x05,0x80,0xbe]
+// GFX12: s_brev_b64 s[0:1], -1                   ; encoding: [0xc1,0x05,0x80,0xbe]
 
 s_brev_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x05,0x80,0xbe]
+// GFX12: s_brev_b64 s[0:1], 0.5                  ; encoding: [0xf0,0x05,0x80,0xbe]
 
 s_brev_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x05,0x80,0xbe]
+// GFX12: s_brev_b64 s[0:1], -4.0                 ; encoding: [0xf7,0x05,0x80,0xbe]
 
 s_brev_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x05,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_brev_b64 s[0:1], 0x3f717273           ; encoding: [0xff,0x05,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_brev_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x05,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_brev_b64 s[0:1], 0xaf123456           ; encoding: [0xff,0x05,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_bcnt0_i32_b32 s0, s1
-// GFX12: encoding: [0x01,0x16,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b32 s0, s1                  ; encoding: [0x01,0x16,0x80,0xbe]
 
 s_bcnt0_i32_b32 s105, s104
-// GFX12: encoding: [0x68,0x16,0xe9,0xbe]
+// GFX12: s_bcnt0_i32_b32 s105, s104              ; encoding: [0x68,0x16,0xe9,0xbe]
 
 s_bcnt0_i32_b32 s0, s104
-// GFX12: encoding: [0x68,0x16,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b32 s0, s104                ; encoding: [0x68,0x16,0x80,0xbe]
 
 s_bcnt0_i32_b32 s105, s1
-// GFX12: encoding: [0x01,0x16,0xe9,0xbe]
+// GFX12: s_bcnt0_i32_b32 s105, s1                ; encoding: [0x01,0x16,0xe9,0xbe]
 
 s_bcnt0_i32_b32 exec_lo, s1
-// GFX12: encoding: [0x01,0x16,0xfe,0xbe]
+// GFX12: s_bcnt0_i32_b32 exec_lo, s1             ; encoding: [0x01,0x16,0xfe,0xbe]
 
 s_bcnt0_i32_b32 exec_hi, s1
-// GFX12: encoding: [0x01,0x16,0xff,0xbe]
+// GFX12: s_bcnt0_i32_b32 exec_hi, s1             ; encoding: [0x01,0x16,0xff,0xbe]
 
 s_bcnt0_i32_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x16,0xea,0xbe]
+// GFX12: s_bcnt0_i32_b32 vcc_lo, s1              ; encoding: [0x01,0x16,0xea,0xbe]
 
 s_bcnt0_i32_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x16,0xeb,0xbe]
+// GFX12: s_bcnt0_i32_b32 vcc_hi, s1              ; encoding: [0x01,0x16,0xeb,0xbe]
 
 s_bcnt0_i32_b32 m0, s1
-// GFX12: encoding: [0x01,0x16,0xfd,0xbe]
+// GFX12: s_bcnt0_i32_b32 m0, s1                  ; encoding: [0x01,0x16,0xfd,0xbe]
 
 s_bcnt0_i32_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x16,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b32 s0, exec_lo             ; encoding: [0x7e,0x16,0x80,0xbe]
 
 s_bcnt0_i32_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x16,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b32 s0, exec_hi             ; encoding: [0x7f,0x16,0x80,0xbe]
 
 s_bcnt0_i32_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x16,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b32 s0, vcc_lo              ; encoding: [0x6a,0x16,0x80,0xbe]
 
 s_bcnt0_i32_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x16,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b32 s0, vcc_hi              ; encoding: [0x6b,0x16,0x80,0xbe]
 
 s_bcnt0_i32_b32 s0, m0
-// GFX12: encoding: [0x7d,0x16,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b32 s0, m0                  ; encoding: [0x7d,0x16,0x80,0xbe]
 
 s_bcnt0_i32_b32 s0, 0
-// GFX12: encoding: [0x80,0x16,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b32 s0, 0                   ; encoding: [0x80,0x16,0x80,0xbe]
 
 s_bcnt0_i32_b32 s0, -1
-// GFX12: encoding: [0xc1,0x16,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b32 s0, -1                  ; encoding: [0xc1,0x16,0x80,0xbe]
 
 s_bcnt0_i32_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x16,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b32 s0, 0.5                 ; encoding: [0xf0,0x16,0x80,0xbe]
 
 s_bcnt0_i32_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x16,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b32 s0, -4.0                ; encoding: [0xf7,0x16,0x80,0xbe]
 
 s_bcnt0_i32_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x16,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_bcnt0_i32_b32 s0, 0x3f717273          ; encoding: [0xff,0x16,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_bcnt0_i32_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x16,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_bcnt0_i32_b32 s0, 0xaf123456          ; encoding: [0xff,0x16,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_bcnt0_i32_b64 s0, s[2:3]
-// GFX12: encoding: [0x02,0x17,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b64 s0, s[2:3]              ; encoding: [0x02,0x17,0x80,0xbe]
 
 s_bcnt0_i32_b64 s105, s[102:103]
-// GFX12: encoding: [0x66,0x17,0xe9,0xbe]
+// GFX12: s_bcnt0_i32_b64 s105, s[102:103]        ; encoding: [0x66,0x17,0xe9,0xbe]
 
 s_bcnt0_i32_b64 s0, s[102:103]
-// GFX12: encoding: [0x66,0x17,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b64 s0, s[102:103]          ; encoding: [0x66,0x17,0x80,0xbe]
 
 s_bcnt0_i32_b64 s105, s[2:3]
-// GFX12: encoding: [0x02,0x17,0xe9,0xbe]
+// GFX12: s_bcnt0_i32_b64 s105, s[2:3]            ; encoding: [0x02,0x17,0xe9,0xbe]
 
 s_bcnt0_i32_b64 exec_lo, s[2:3]
-// GFX12: encoding: [0x02,0x17,0xfe,0xbe]
+// GFX12: s_bcnt0_i32_b64 exec_lo, s[2:3]         ; encoding: [0x02,0x17,0xfe,0xbe]
 
 s_bcnt0_i32_b64 exec_hi, s[2:3]
-// GFX12: encoding: [0x02,0x17,0xff,0xbe]
+// GFX12: s_bcnt0_i32_b64 exec_hi, s[2:3]         ; encoding: [0x02,0x17,0xff,0xbe]
 
 s_bcnt0_i32_b64 vcc_lo, s[2:3]
-// GFX12: encoding: [0x02,0x17,0xea,0xbe]
+// GFX12: s_bcnt0_i32_b64 vcc_lo, s[2:3]          ; encoding: [0x02,0x17,0xea,0xbe]
 
 s_bcnt0_i32_b64 vcc_hi, s[2:3]
-// GFX12: encoding: [0x02,0x17,0xeb,0xbe]
+// GFX12: s_bcnt0_i32_b64 vcc_hi, s[2:3]          ; encoding: [0x02,0x17,0xeb,0xbe]
 
 s_bcnt0_i32_b64 m0, s[2:3]
-// GFX12: encoding: [0x02,0x17,0xfd,0xbe]
+// GFX12: s_bcnt0_i32_b64 m0, s[2:3]              ; encoding: [0x02,0x17,0xfd,0xbe]
 
 s_bcnt0_i32_b64 s0, exec
-// GFX12: encoding: [0x7e,0x17,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b64 s0, exec                ; encoding: [0x7e,0x17,0x80,0xbe]
 
 s_bcnt0_i32_b64 s0, vcc
-// GFX12: encoding: [0x6a,0x17,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b64 s0, vcc                 ; encoding: [0x6a,0x17,0x80,0xbe]
 
 s_bcnt0_i32_b64 s0, 0
-// GFX12: encoding: [0x80,0x17,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b64 s0, 0                   ; encoding: [0x80,0x17,0x80,0xbe]
 
 s_bcnt0_i32_b64 s0, -1
-// GFX12: encoding: [0xc1,0x17,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b64 s0, -1                  ; encoding: [0xc1,0x17,0x80,0xbe]
 
 s_bcnt0_i32_b64 s0, 0.5
-// GFX12: encoding: [0xf0,0x17,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b64 s0, 0.5                 ; encoding: [0xf0,0x17,0x80,0xbe]
 
 s_bcnt0_i32_b64 s0, -4.0
-// GFX12: encoding: [0xf7,0x17,0x80,0xbe]
+// GFX12: s_bcnt0_i32_b64 s0, -4.0                ; encoding: [0xf7,0x17,0x80,0xbe]
 
 s_bcnt0_i32_b64 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x17,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_bcnt0_i32_b64 s0, 0x3f717273          ; encoding: [0xff,0x17,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_bcnt0_i32_b64 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x17,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_bcnt0_i32_b64 s0, 0xaf123456          ; encoding: [0xff,0x17,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_bcnt1_i32_b32 s0, s1
-// GFX12: encoding: [0x01,0x18,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b32 s0, s1                  ; encoding: [0x01,0x18,0x80,0xbe]
 
 s_bcnt1_i32_b32 s105, s104
-// GFX12: encoding: [0x68,0x18,0xe9,0xbe]
+// GFX12: s_bcnt1_i32_b32 s105, s104              ; encoding: [0x68,0x18,0xe9,0xbe]
 
 s_bcnt1_i32_b32 s0, s104
-// GFX12: encoding: [0x68,0x18,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b32 s0, s104                ; encoding: [0x68,0x18,0x80,0xbe]
 
 s_bcnt1_i32_b32 s105, s1
-// GFX12: encoding: [0x01,0x18,0xe9,0xbe]
+// GFX12: s_bcnt1_i32_b32 s105, s1                ; encoding: [0x01,0x18,0xe9,0xbe]
 
 s_bcnt1_i32_b32 exec_lo, s1
-// GFX12: encoding: [0x01,0x18,0xfe,0xbe]
+// GFX12: s_bcnt1_i32_b32 exec_lo, s1             ; encoding: [0x01,0x18,0xfe,0xbe]
 
 s_bcnt1_i32_b32 exec_hi, s1
-// GFX12: encoding: [0x01,0x18,0xff,0xbe]
+// GFX12: s_bcnt1_i32_b32 exec_hi, s1             ; encoding: [0x01,0x18,0xff,0xbe]
 
 s_bcnt1_i32_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x18,0xea,0xbe]
+// GFX12: s_bcnt1_i32_b32 vcc_lo, s1              ; encoding: [0x01,0x18,0xea,0xbe]
 
 s_bcnt1_i32_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x18,0xeb,0xbe]
+// GFX12: s_bcnt1_i32_b32 vcc_hi, s1              ; encoding: [0x01,0x18,0xeb,0xbe]
 
 s_bcnt1_i32_b32 m0, s1
-// GFX12: encoding: [0x01,0x18,0xfd,0xbe]
+// GFX12: s_bcnt1_i32_b32 m0, s1                  ; encoding: [0x01,0x18,0xfd,0xbe]
 
 s_bcnt1_i32_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x18,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b32 s0, exec_lo             ; encoding: [0x7e,0x18,0x80,0xbe]
 
 s_bcnt1_i32_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x18,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b32 s0, exec_hi             ; encoding: [0x7f,0x18,0x80,0xbe]
 
 s_bcnt1_i32_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x18,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b32 s0, vcc_lo              ; encoding: [0x6a,0x18,0x80,0xbe]
 
 s_bcnt1_i32_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x18,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b32 s0, vcc_hi              ; encoding: [0x6b,0x18,0x80,0xbe]
 
 s_bcnt1_i32_b32 s0, m0
-// GFX12: encoding: [0x7d,0x18,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b32 s0, m0                  ; encoding: [0x7d,0x18,0x80,0xbe]
 
 s_bcnt1_i32_b32 s0, 0
-// GFX12: encoding: [0x80,0x18,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b32 s0, 0                   ; encoding: [0x80,0x18,0x80,0xbe]
 
 s_bcnt1_i32_b32 s0, -1
-// GFX12: encoding: [0xc1,0x18,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b32 s0, -1                  ; encoding: [0xc1,0x18,0x80,0xbe]
 
 s_bcnt1_i32_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x18,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b32 s0, 0.5                 ; encoding: [0xf0,0x18,0x80,0xbe]
 
 s_bcnt1_i32_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x18,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b32 s0, -4.0                ; encoding: [0xf7,0x18,0x80,0xbe]
 
 s_bcnt1_i32_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x18,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_bcnt1_i32_b32 s0, 0x3f717273          ; encoding: [0xff,0x18,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_bcnt1_i32_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x18,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_bcnt1_i32_b32 s0, 0xaf123456          ; encoding: [0xff,0x18,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_bcnt1_i32_b64 s0, s[2:3]
-// GFX12: encoding: [0x02,0x19,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b64 s0, s[2:3]              ; encoding: [0x02,0x19,0x80,0xbe]
 
 s_bcnt1_i32_b64 s105, s[102:103]
-// GFX12: encoding: [0x66,0x19,0xe9,0xbe]
+// GFX12: s_bcnt1_i32_b64 s105, s[102:103]        ; encoding: [0x66,0x19,0xe9,0xbe]
 
 s_bcnt1_i32_b64 s0, s[102:103]
-// GFX12: encoding: [0x66,0x19,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b64 s0, s[102:103]          ; encoding: [0x66,0x19,0x80,0xbe]
 
 s_bcnt1_i32_b64 s105, s[2:3]
-// GFX12: encoding: [0x02,0x19,0xe9,0xbe]
+// GFX12: s_bcnt1_i32_b64 s105, s[2:3]            ; encoding: [0x02,0x19,0xe9,0xbe]
 
 s_bcnt1_i32_b64 exec_lo, s[2:3]
-// GFX12: encoding: [0x02,0x19,0xfe,0xbe]
+// GFX12: s_bcnt1_i32_b64 exec_lo, s[2:3]         ; encoding: [0x02,0x19,0xfe,0xbe]
 
 s_bcnt1_i32_b64 exec_hi, s[2:3]
-// GFX12: encoding: [0x02,0x19,0xff,0xbe]
+// GFX12: s_bcnt1_i32_b64 exec_hi, s[2:3]         ; encoding: [0x02,0x19,0xff,0xbe]
 
 s_bcnt1_i32_b64 vcc_lo, s[2:3]
-// GFX12: encoding: [0x02,0x19,0xea,0xbe]
+// GFX12: s_bcnt1_i32_b64 vcc_lo, s[2:3]          ; encoding: [0x02,0x19,0xea,0xbe]
 
 s_bcnt1_i32_b64 vcc_hi, s[2:3]
-// GFX12: encoding: [0x02,0x19,0xeb,0xbe]
+// GFX12: s_bcnt1_i32_b64 vcc_hi, s[2:3]          ; encoding: [0x02,0x19,0xeb,0xbe]
 
 s_bcnt1_i32_b64 m0, s[2:3]
-// GFX12: encoding: [0x02,0x19,0xfd,0xbe]
+// GFX12: s_bcnt1_i32_b64 m0, s[2:3]              ; encoding: [0x02,0x19,0xfd,0xbe]
 
 s_bcnt1_i32_b64 s0, exec
-// GFX12: encoding: [0x7e,0x19,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b64 s0, exec                ; encoding: [0x7e,0x19,0x80,0xbe]
 
 s_bcnt1_i32_b64 s0, vcc
-// GFX12: encoding: [0x6a,0x19,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b64 s0, vcc                 ; encoding: [0x6a,0x19,0x80,0xbe]
 
 s_bcnt1_i32_b64 s0, 0
-// GFX12: encoding: [0x80,0x19,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b64 s0, 0                   ; encoding: [0x80,0x19,0x80,0xbe]
 
 s_bcnt1_i32_b64 s0, -1
-// GFX12: encoding: [0xc1,0x19,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b64 s0, -1                  ; encoding: [0xc1,0x19,0x80,0xbe]
 
 s_bcnt1_i32_b64 s0, 0.5
-// GFX12: encoding: [0xf0,0x19,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b64 s0, 0.5                 ; encoding: [0xf0,0x19,0x80,0xbe]
 
 s_bcnt1_i32_b64 s0, -4.0
-// GFX12: encoding: [0xf7,0x19,0x80,0xbe]
+// GFX12: s_bcnt1_i32_b64 s0, -4.0                ; encoding: [0xf7,0x19,0x80,0xbe]
 
 s_bcnt1_i32_b64 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x19,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_bcnt1_i32_b64 s0, 0x3f717273          ; encoding: [0xff,0x19,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_bcnt1_i32_b64 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x19,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_bcnt1_i32_b64 s0, 0xaf123456          ; encoding: [0xff,0x19,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_ff1_i32_b32 s0, s1
-// GFX12: encoding: [0x01,0x08,0x80,0xbe]
+// GFX12: s_ctz_i32_b32 s0, s1                    ; encoding: [0x01,0x08,0x80,0xbe]
 
 s_ff1_i32_b32 s105, s104
-// GFX12: encoding: [0x68,0x08,0xe9,0xbe]
+// GFX12: s_ctz_i32_b32 s105, s104                ; encoding: [0x68,0x08,0xe9,0xbe]
 
 s_ff1_i32_b32 s0, s104
-// GFX12: encoding: [0x68,0x08,0x80,0xbe]
+// GFX12: s_ctz_i32_b32 s0, s104                  ; encoding: [0x68,0x08,0x80,0xbe]
 
 s_ff1_i32_b32 s105, s1
-// GFX12: encoding: [0x01,0x08,0xe9,0xbe]
+// GFX12: s_ctz_i32_b32 s105, s1                  ; encoding: [0x01,0x08,0xe9,0xbe]
 
 s_ff1_i32_b32 exec_lo, s1
-// GFX12: encoding: [0x01,0x08,0xfe,0xbe]
+// GFX12: s_ctz_i32_b32 exec_lo, s1               ; encoding: [0x01,0x08,0xfe,0xbe]
 
 s_ff1_i32_b32 exec_hi, s1
-// GFX12: encoding: [0x01,0x08,0xff,0xbe]
+// GFX12: s_ctz_i32_b32 exec_hi, s1               ; encoding: [0x01,0x08,0xff,0xbe]
 
 s_ff1_i32_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x08,0xea,0xbe]
+// GFX12: s_ctz_i32_b32 vcc_lo, s1                ; encoding: [0x01,0x08,0xea,0xbe]
 
 s_ff1_i32_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x08,0xeb,0xbe]
+// GFX12: s_ctz_i32_b32 vcc_hi, s1                ; encoding: [0x01,0x08,0xeb,0xbe]
 
 s_ff1_i32_b32 m0, s1
-// GFX12: encoding: [0x01,0x08,0xfd,0xbe]
+// GFX12: s_ctz_i32_b32 m0, s1                    ; encoding: [0x01,0x08,0xfd,0xbe]
 
 s_ff1_i32_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x08,0x80,0xbe]
+// GFX12: s_ctz_i32_b32 s0, exec_lo               ; encoding: [0x7e,0x08,0x80,0xbe]
 
 s_ff1_i32_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x08,0x80,0xbe]
+// GFX12: s_ctz_i32_b32 s0, exec_hi               ; encoding: [0x7f,0x08,0x80,0xbe]
 
 s_ff1_i32_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x08,0x80,0xbe]
+// GFX12: s_ctz_i32_b32 s0, vcc_lo                ; encoding: [0x6a,0x08,0x80,0xbe]
 
 s_ff1_i32_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x08,0x80,0xbe]
+// GFX12: s_ctz_i32_b32 s0, vcc_hi                ; encoding: [0x6b,0x08,0x80,0xbe]
 
 s_ff1_i32_b32 s0, m0
-// GFX12: encoding: [0x7d,0x08,0x80,0xbe]
+// GFX12: s_ctz_i32_b32 s0, m0                    ; encoding: [0x7d,0x08,0x80,0xbe]
 
 s_ff1_i32_b32 s0, 0
-// GFX12: encoding: [0x80,0x08,0x80,0xbe]
+// GFX12: s_ctz_i32_b32 s0, 0                     ; encoding: [0x80,0x08,0x80,0xbe]
 
 s_ff1_i32_b32 s0, -1
-// GFX12: encoding: [0xc1,0x08,0x80,0xbe]
+// GFX12: s_ctz_i32_b32 s0, -1                    ; encoding: [0xc1,0x08,0x80,0xbe]
 
 s_ff1_i32_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x08,0x80,0xbe]
+// GFX12: s_ctz_i32_b32 s0, 0.5                   ; encoding: [0xf0,0x08,0x80,0xbe]
 
 s_ff1_i32_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x08,0x80,0xbe]
+// GFX12: s_ctz_i32_b32 s0, -4.0                  ; encoding: [0xf7,0x08,0x80,0xbe]
 
 s_ff1_i32_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x08,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_ctz_i32_b32 s0, 0x3f717273            ; encoding: [0xff,0x08,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_ff1_i32_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x08,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_ctz_i32_b32 s0, 0xaf123456            ; encoding: [0xff,0x08,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_ff1_i32_b64 s0, s[2:3]
-// GFX12: encoding: [0x02,0x09,0x80,0xbe]
+// GFX12: s_ctz_i32_b64 s0, s[2:3]                ; encoding: [0x02,0x09,0x80,0xbe]
 
 s_ff1_i32_b64 s105, s[102:103]
-// GFX12: encoding: [0x66,0x09,0xe9,0xbe]
+// GFX12: s_ctz_i32_b64 s105, s[102:103]          ; encoding: [0x66,0x09,0xe9,0xbe]
 
 s_ff1_i32_b64 s0, s[102:103]
-// GFX12: encoding: [0x66,0x09,0x80,0xbe]
+// GFX12: s_ctz_i32_b64 s0, s[102:103]            ; encoding: [0x66,0x09,0x80,0xbe]
 
 s_ff1_i32_b64 s105, s[2:3]
-// GFX12: encoding: [0x02,0x09,0xe9,0xbe]
+// GFX12: s_ctz_i32_b64 s105, s[2:3]              ; encoding: [0x02,0x09,0xe9,0xbe]
 
 s_ff1_i32_b64 exec_lo, s[2:3]
-// GFX12: encoding: [0x02,0x09,0xfe,0xbe]
+// GFX12: s_ctz_i32_b64 exec_lo, s[2:3]           ; encoding: [0x02,0x09,0xfe,0xbe]
 
 s_ff1_i32_b64 exec_hi, s[2:3]
-// GFX12: encoding: [0x02,0x09,0xff,0xbe]
+// GFX12: s_ctz_i32_b64 exec_hi, s[2:3]           ; encoding: [0x02,0x09,0xff,0xbe]
 
 s_ff1_i32_b64 vcc_lo, s[2:3]
-// GFX12: encoding: [0x02,0x09,0xea,0xbe]
+// GFX12: s_ctz_i32_b64 vcc_lo, s[2:3]            ; encoding: [0x02,0x09,0xea,0xbe]
 
 s_ff1_i32_b64 vcc_hi, s[2:3]
-// GFX12: encoding: [0x02,0x09,0xeb,0xbe]
+// GFX12: s_ctz_i32_b64 vcc_hi, s[2:3]            ; encoding: [0x02,0x09,0xeb,0xbe]
 
 s_ff1_i32_b64 m0, s[2:3]
-// GFX12: encoding: [0x02,0x09,0xfd,0xbe]
+// GFX12: s_ctz_i32_b64 m0, s[2:3]                ; encoding: [0x02,0x09,0xfd,0xbe]
 
 s_ff1_i32_b64 s0, exec
-// GFX12: encoding: [0x7e,0x09,0x80,0xbe]
+// GFX12: s_ctz_i32_b64 s0, exec                  ; encoding: [0x7e,0x09,0x80,0xbe]
 
 s_ff1_i32_b64 s0, vcc
-// GFX12: encoding: [0x6a,0x09,0x80,0xbe]
+// GFX12: s_ctz_i32_b64 s0, vcc                   ; encoding: [0x6a,0x09,0x80,0xbe]
 
 s_ff1_i32_b64 s0, 0
-// GFX12: encoding: [0x80,0x09,0x80,0xbe]
+// GFX12: s_ctz_i32_b64 s0, 0                     ; encoding: [0x80,0x09,0x80,0xbe]
 
 s_ff1_i32_b64 s0, -1
-// GFX12: encoding: [0xc1,0x09,0x80,0xbe]
+// GFX12: s_ctz_i32_b64 s0, -1                    ; encoding: [0xc1,0x09,0x80,0xbe]
 
 s_ff1_i32_b64 s0, 0.5
-// GFX12: encoding: [0xf0,0x09,0x80,0xbe]
+// GFX12: s_ctz_i32_b64 s0, 0.5                   ; encoding: [0xf0,0x09,0x80,0xbe]
 
 s_ff1_i32_b64 s0, -4.0
-// GFX12: encoding: [0xf7,0x09,0x80,0xbe]
+// GFX12: s_ctz_i32_b64 s0, -4.0                  ; encoding: [0xf7,0x09,0x80,0xbe]
 
 s_ff1_i32_b64 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x09,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_ctz_i32_b64 s0, 0x3f717273            ; encoding: [0xff,0x09,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_ff1_i32_b64 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x09,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_ctz_i32_b64 s0, 0xaf123456            ; encoding: [0xff,0x09,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_flbit_i32_b32 s0, s1
-// GFX12: encoding: [0x01,0x0a,0x80,0xbe]
+// GFX12: s_clz_i32_u32 s0, s1                    ; encoding: [0x01,0x0a,0x80,0xbe]
 
 s_flbit_i32_b32 s105, s104
-// GFX12: encoding: [0x68,0x0a,0xe9,0xbe]
+// GFX12: s_clz_i32_u32 s105, s104                ; encoding: [0x68,0x0a,0xe9,0xbe]
 
 s_flbit_i32_b32 s0, s104
-// GFX12: encoding: [0x68,0x0a,0x80,0xbe]
+// GFX12: s_clz_i32_u32 s0, s104                  ; encoding: [0x68,0x0a,0x80,0xbe]
 
 s_flbit_i32_b32 s105, s1
-// GFX12: encoding: [0x01,0x0a,0xe9,0xbe]
+// GFX12: s_clz_i32_u32 s105, s1                  ; encoding: [0x01,0x0a,0xe9,0xbe]
 
 s_flbit_i32_b32 exec_lo, s1
-// GFX12: encoding: [0x01,0x0a,0xfe,0xbe]
+// GFX12: s_clz_i32_u32 exec_lo, s1               ; encoding: [0x01,0x0a,0xfe,0xbe]
 
 s_flbit_i32_b32 exec_hi, s1
-// GFX12: encoding: [0x01,0x0a,0xff,0xbe]
+// GFX12: s_clz_i32_u32 exec_hi, s1               ; encoding: [0x01,0x0a,0xff,0xbe]
 
 s_flbit_i32_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x0a,0xea,0xbe]
+// GFX12: s_clz_i32_u32 vcc_lo, s1                ; encoding: [0x01,0x0a,0xea,0xbe]
 
 s_flbit_i32_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x0a,0xeb,0xbe]
+// GFX12: s_clz_i32_u32 vcc_hi, s1                ; encoding: [0x01,0x0a,0xeb,0xbe]
 
 s_flbit_i32_b32 m0, s1
-// GFX12: encoding: [0x01,0x0a,0xfd,0xbe]
+// GFX12: s_clz_i32_u32 m0, s1                    ; encoding: [0x01,0x0a,0xfd,0xbe]
 
 s_flbit_i32_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x0a,0x80,0xbe]
+// GFX12: s_clz_i32_u32 s0, exec_lo               ; encoding: [0x7e,0x0a,0x80,0xbe]
 
 s_flbit_i32_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x0a,0x80,0xbe]
+// GFX12: s_clz_i32_u32 s0, exec_hi               ; encoding: [0x7f,0x0a,0x80,0xbe]
 
 s_flbit_i32_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x0a,0x80,0xbe]
+// GFX12: s_clz_i32_u32 s0, vcc_lo                ; encoding: [0x6a,0x0a,0x80,0xbe]
 
 s_flbit_i32_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x0a,0x80,0xbe]
+// GFX12: s_clz_i32_u32 s0, vcc_hi                ; encoding: [0x6b,0x0a,0x80,0xbe]
 
 s_flbit_i32_b32 s0, m0
-// GFX12: encoding: [0x7d,0x0a,0x80,0xbe]
+// GFX12: s_clz_i32_u32 s0, m0                    ; encoding: [0x7d,0x0a,0x80,0xbe]
 
 s_flbit_i32_b32 s0, 0
-// GFX12: encoding: [0x80,0x0a,0x80,0xbe]
+// GFX12: s_clz_i32_u32 s0, 0                     ; encoding: [0x80,0x0a,0x80,0xbe]
 
 s_flbit_i32_b32 s0, -1
-// GFX12: encoding: [0xc1,0x0a,0x80,0xbe]
+// GFX12: s_clz_i32_u32 s0, -1                    ; encoding: [0xc1,0x0a,0x80,0xbe]
 
 s_flbit_i32_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x0a,0x80,0xbe]
+// GFX12: s_clz_i32_u32 s0, 0.5                   ; encoding: [0xf0,0x0a,0x80,0xbe]
 
 s_flbit_i32_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x0a,0x80,0xbe]
+// GFX12: s_clz_i32_u32 s0, -4.0                  ; encoding: [0xf7,0x0a,0x80,0xbe]
 
 s_flbit_i32_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x0a,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_clz_i32_u32 s0, 0x3f717273            ; encoding: [0xff,0x0a,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_flbit_i32_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x0a,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_clz_i32_u32 s0, 0xaf123456            ; encoding: [0xff,0x0a,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_flbit_i32_b64 s0, s[2:3]
-// GFX12: encoding: [0x02,0x0b,0x80,0xbe]
+// GFX12: s_clz_i32_u64 s0, s[2:3]                ; encoding: [0x02,0x0b,0x80,0xbe]
 
 s_flbit_i32_b64 s105, s[102:103]
-// GFX12: encoding: [0x66,0x0b,0xe9,0xbe]
+// GFX12: s_clz_i32_u64 s105, s[102:103]          ; encoding: [0x66,0x0b,0xe9,0xbe]
 
 s_flbit_i32_b64 s0, s[102:103]
-// GFX12: encoding: [0x66,0x0b,0x80,0xbe]
+// GFX12: s_clz_i32_u64 s0, s[102:103]            ; encoding: [0x66,0x0b,0x80,0xbe]
 
 s_flbit_i32_b64 s105, s[2:3]
-// GFX12: encoding: [0x02,0x0b,0xe9,0xbe]
+// GFX12: s_clz_i32_u64 s105, s[2:3]              ; encoding: [0x02,0x0b,0xe9,0xbe]
 
 s_flbit_i32_b64 exec_lo, s[2:3]
-// GFX12: encoding: [0x02,0x0b,0xfe,0xbe]
+// GFX12: s_clz_i32_u64 exec_lo, s[2:3]           ; encoding: [0x02,0x0b,0xfe,0xbe]
 
 s_flbit_i32_b64 exec_hi, s[2:3]
-// GFX12: encoding: [0x02,0x0b,0xff,0xbe]
+// GFX12: s_clz_i32_u64 exec_hi, s[2:3]           ; encoding: [0x02,0x0b,0xff,0xbe]
 
 s_flbit_i32_b64 vcc_lo, s[2:3]
-// GFX12: encoding: [0x02,0x0b,0xea,0xbe]
+// GFX12: s_clz_i32_u64 vcc_lo, s[2:3]            ; encoding: [0x02,0x0b,0xea,0xbe]
 
 s_flbit_i32_b64 vcc_hi, s[2:3]
-// GFX12: encoding: [0x02,0x0b,0xeb,0xbe]
+// GFX12: s_clz_i32_u64 vcc_hi, s[2:3]            ; encoding: [0x02,0x0b,0xeb,0xbe]
 
 s_flbit_i32_b64 m0, s[2:3]
-// GFX12: encoding: [0x02,0x0b,0xfd,0xbe]
+// GFX12: s_clz_i32_u64 m0, s[2:3]                ; encoding: [0x02,0x0b,0xfd,0xbe]
 
 s_flbit_i32_b64 s0, exec
-// GFX12: encoding: [0x7e,0x0b,0x80,0xbe]
+// GFX12: s_clz_i32_u64 s0, exec                  ; encoding: [0x7e,0x0b,0x80,0xbe]
 
 s_flbit_i32_b64 s0, vcc
-// GFX12: encoding: [0x6a,0x0b,0x80,0xbe]
+// GFX12: s_clz_i32_u64 s0, vcc                   ; encoding: [0x6a,0x0b,0x80,0xbe]
 
 s_flbit_i32_b64 s0, 0
-// GFX12: encoding: [0x80,0x0b,0x80,0xbe]
+// GFX12: s_clz_i32_u64 s0, 0                     ; encoding: [0x80,0x0b,0x80,0xbe]
 
 s_flbit_i32_b64 s0, -1
-// GFX12: encoding: [0xc1,0x0b,0x80,0xbe]
+// GFX12: s_clz_i32_u64 s0, -1                    ; encoding: [0xc1,0x0b,0x80,0xbe]
 
 s_flbit_i32_b64 s0, 0.5
-// GFX12: encoding: [0xf0,0x0b,0x80,0xbe]
+// GFX12: s_clz_i32_u64 s0, 0.5                   ; encoding: [0xf0,0x0b,0x80,0xbe]
 
 s_flbit_i32_b64 s0, -4.0
-// GFX12: encoding: [0xf7,0x0b,0x80,0xbe]
+// GFX12: s_clz_i32_u64 s0, -4.0                  ; encoding: [0xf7,0x0b,0x80,0xbe]
 
 s_flbit_i32_b64 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x0b,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_clz_i32_u64 s0, 0x3f717273            ; encoding: [0xff,0x0b,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_flbit_i32_b64 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x0b,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_clz_i32_u64 s0, 0xaf123456            ; encoding: [0xff,0x0b,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_flbit_i32 s0, s1
-// GFX12: encoding: [0x01,0x0c,0x80,0xbe]
+// GFX12: s_cls_i32 s0, s1                        ; encoding: [0x01,0x0c,0x80,0xbe]
 
 s_flbit_i32 s105, s104
-// GFX12: encoding: [0x68,0x0c,0xe9,0xbe]
+// GFX12: s_cls_i32 s105, s104                    ; encoding: [0x68,0x0c,0xe9,0xbe]
 
 s_flbit_i32 s0, s104
-// GFX12: encoding: [0x68,0x0c,0x80,0xbe]
+// GFX12: s_cls_i32 s0, s104                      ; encoding: [0x68,0x0c,0x80,0xbe]
 
 s_flbit_i32 s105, s1
-// GFX12: encoding: [0x01,0x0c,0xe9,0xbe]
+// GFX12: s_cls_i32 s105, s1                      ; encoding: [0x01,0x0c,0xe9,0xbe]
 
 s_flbit_i32 exec_lo, s1
-// GFX12: encoding: [0x01,0x0c,0xfe,0xbe]
+// GFX12: s_cls_i32 exec_lo, s1                   ; encoding: [0x01,0x0c,0xfe,0xbe]
 
 s_flbit_i32 exec_hi, s1
-// GFX12: encoding: [0x01,0x0c,0xff,0xbe]
+// GFX12: s_cls_i32 exec_hi, s1                   ; encoding: [0x01,0x0c,0xff,0xbe]
 
 s_flbit_i32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x0c,0xea,0xbe]
+// GFX12: s_cls_i32 vcc_lo, s1                    ; encoding: [0x01,0x0c,0xea,0xbe]
 
 s_flbit_i32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x0c,0xeb,0xbe]
+// GFX12: s_cls_i32 vcc_hi, s1                    ; encoding: [0x01,0x0c,0xeb,0xbe]
 
 s_flbit_i32 m0, s1
-// GFX12: encoding: [0x01,0x0c,0xfd,0xbe]
+// GFX12: s_cls_i32 m0, s1                        ; encoding: [0x01,0x0c,0xfd,0xbe]
 
 s_flbit_i32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x0c,0x80,0xbe]
+// GFX12: s_cls_i32 s0, exec_lo                   ; encoding: [0x7e,0x0c,0x80,0xbe]
 
 s_flbit_i32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x0c,0x80,0xbe]
+// GFX12: s_cls_i32 s0, exec_hi                   ; encoding: [0x7f,0x0c,0x80,0xbe]
 
 s_flbit_i32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x0c,0x80,0xbe]
+// GFX12: s_cls_i32 s0, vcc_lo                    ; encoding: [0x6a,0x0c,0x80,0xbe]
 
 s_flbit_i32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x0c,0x80,0xbe]
+// GFX12: s_cls_i32 s0, vcc_hi                    ; encoding: [0x6b,0x0c,0x80,0xbe]
 
 s_flbit_i32 s0, m0
-// GFX12: encoding: [0x7d,0x0c,0x80,0xbe]
+// GFX12: s_cls_i32 s0, m0                        ; encoding: [0x7d,0x0c,0x80,0xbe]
 
 s_flbit_i32 s0, 0
-// GFX12: encoding: [0x80,0x0c,0x80,0xbe]
+// GFX12: s_cls_i32 s0, 0                         ; encoding: [0x80,0x0c,0x80,0xbe]
 
 s_flbit_i32 s0, -1
-// GFX12: encoding: [0xc1,0x0c,0x80,0xbe]
+// GFX12: s_cls_i32 s0, -1                        ; encoding: [0xc1,0x0c,0x80,0xbe]
 
 s_flbit_i32 s0, 0.5
-// GFX12: encoding: [0xf0,0x0c,0x80,0xbe]
+// GFX12: s_cls_i32 s0, 0.5                       ; encoding: [0xf0,0x0c,0x80,0xbe]
 
 s_flbit_i32 s0, -4.0
-// GFX12: encoding: [0xf7,0x0c,0x80,0xbe]
+// GFX12: s_cls_i32 s0, -4.0                      ; encoding: [0xf7,0x0c,0x80,0xbe]
 
 s_flbit_i32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x0c,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_cls_i32 s0, 0x3f717273                ; encoding: [0xff,0x0c,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_flbit_i32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x0c,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_cls_i32 s0, 0xaf123456                ; encoding: [0xff,0x0c,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_flbit_i32_i64 s0, s[2:3]
-// GFX12: encoding: [0x02,0x0d,0x80,0xbe]
+// GFX12: s_cls_i32_i64 s0, s[2:3]                ; encoding: [0x02,0x0d,0x80,0xbe]
 
 s_flbit_i32_i64 s105, s[102:103]
-// GFX12: encoding: [0x66,0x0d,0xe9,0xbe]
+// GFX12: s_cls_i32_i64 s105, s[102:103]          ; encoding: [0x66,0x0d,0xe9,0xbe]
 
 s_flbit_i32_i64 s0, s[102:103]
-// GFX12: encoding: [0x66,0x0d,0x80,0xbe]
+// GFX12: s_cls_i32_i64 s0, s[102:103]            ; encoding: [0x66,0x0d,0x80,0xbe]
 
 s_flbit_i32_i64 s105, s[2:3]
-// GFX12: encoding: [0x02,0x0d,0xe9,0xbe]
+// GFX12: s_cls_i32_i64 s105, s[2:3]              ; encoding: [0x02,0x0d,0xe9,0xbe]
 
 s_flbit_i32_i64 exec_lo, s[2:3]
-// GFX12: encoding: [0x02,0x0d,0xfe,0xbe]
+// GFX12: s_cls_i32_i64 exec_lo, s[2:3]           ; encoding: [0x02,0x0d,0xfe,0xbe]
 
 s_flbit_i32_i64 exec_hi, s[2:3]
-// GFX12: encoding: [0x02,0x0d,0xff,0xbe]
+// GFX12: s_cls_i32_i64 exec_hi, s[2:3]           ; encoding: [0x02,0x0d,0xff,0xbe]
 
 s_flbit_i32_i64 vcc_lo, s[2:3]
-// GFX12: encoding: [0x02,0x0d,0xea,0xbe]
+// GFX12: s_cls_i32_i64 vcc_lo, s[2:3]            ; encoding: [0x02,0x0d,0xea,0xbe]
 
 s_flbit_i32_i64 vcc_hi, s[2:3]
-// GFX12: encoding: [0x02,0x0d,0xeb,0xbe]
+// GFX12: s_cls_i32_i64 vcc_hi, s[2:3]            ; encoding: [0x02,0x0d,0xeb,0xbe]
 
 s_flbit_i32_i64 m0, s[2:3]
-// GFX12: encoding: [0x02,0x0d,0xfd,0xbe]
+// GFX12: s_cls_i32_i64 m0, s[2:3]                ; encoding: [0x02,0x0d,0xfd,0xbe]
 
 s_flbit_i32_i64 s0, exec
-// GFX12: encoding: [0x7e,0x0d,0x80,0xbe]
+// GFX12: s_cls_i32_i64 s0, exec                  ; encoding: [0x7e,0x0d,0x80,0xbe]
 
 s_flbit_i32_i64 s0, vcc
-// GFX12: encoding: [0x6a,0x0d,0x80,0xbe]
+// GFX12: s_cls_i32_i64 s0, vcc                   ; encoding: [0x6a,0x0d,0x80,0xbe]
 
 s_flbit_i32_i64 s0, 0
-// GFX12: encoding: [0x80,0x0d,0x80,0xbe]
+// GFX12: s_cls_i32_i64 s0, 0                     ; encoding: [0x80,0x0d,0x80,0xbe]
 
 s_flbit_i32_i64 s0, -1
-// GFX12: encoding: [0xc1,0x0d,0x80,0xbe]
+// GFX12: s_cls_i32_i64 s0, -1                    ; encoding: [0xc1,0x0d,0x80,0xbe]
 
 s_flbit_i32_i64 s0, 0.5
-// GFX12: encoding: [0xf0,0x0d,0x80,0xbe]
+// GFX12: s_cls_i32_i64 s0, 0.5                   ; encoding: [0xf0,0x0d,0x80,0xbe]
 
 s_flbit_i32_i64 s0, -4.0
-// GFX12: encoding: [0xf7,0x0d,0x80,0xbe]
+// GFX12: s_cls_i32_i64 s0, -4.0                  ; encoding: [0xf7,0x0d,0x80,0xbe]
 
 s_flbit_i32_i64 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x0d,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_cls_i32_i64 s0, 0x3f717273            ; encoding: [0xff,0x0d,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_flbit_i32_i64 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x0d,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_cls_i32_i64 s0, 0xaf123456            ; encoding: [0xff,0x0d,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_sext_i32_i8 s0, s1
-// GFX12: encoding: [0x01,0x0e,0x80,0xbe]
+// GFX12: s_sext_i32_i8 s0, s1                    ; encoding: [0x01,0x0e,0x80,0xbe]
 
 s_sext_i32_i8 s105, s104
-// GFX12: encoding: [0x68,0x0e,0xe9,0xbe]
+// GFX12: s_sext_i32_i8 s105, s104                ; encoding: [0x68,0x0e,0xe9,0xbe]
 
 s_sext_i32_i8 s0, s104
-// GFX12: encoding: [0x68,0x0e,0x80,0xbe]
+// GFX12: s_sext_i32_i8 s0, s104                  ; encoding: [0x68,0x0e,0x80,0xbe]
 
 s_sext_i32_i8 s105, s1
-// GFX12: encoding: [0x01,0x0e,0xe9,0xbe]
+// GFX12: s_sext_i32_i8 s105, s1                  ; encoding: [0x01,0x0e,0xe9,0xbe]
 
 s_sext_i32_i8 exec_lo, s1
-// GFX12: encoding: [0x01,0x0e,0xfe,0xbe]
+// GFX12: s_sext_i32_i8 exec_lo, s1               ; encoding: [0x01,0x0e,0xfe,0xbe]
 
 s_sext_i32_i8 exec_hi, s1
-// GFX12: encoding: [0x01,0x0e,0xff,0xbe]
+// GFX12: s_sext_i32_i8 exec_hi, s1               ; encoding: [0x01,0x0e,0xff,0xbe]
 
 s_sext_i32_i8 vcc_lo, s1
-// GFX12: encoding: [0x01,0x0e,0xea,0xbe]
+// GFX12: s_sext_i32_i8 vcc_lo, s1                ; encoding: [0x01,0x0e,0xea,0xbe]
 
 s_sext_i32_i8 vcc_hi, s1
-// GFX12: encoding: [0x01,0x0e,0xeb,0xbe]
+// GFX12: s_sext_i32_i8 vcc_hi, s1                ; encoding: [0x01,0x0e,0xeb,0xbe]
 
 s_sext_i32_i8 m0, s1
-// GFX12: encoding: [0x01,0x0e,0xfd,0xbe]
+// GFX12: s_sext_i32_i8 m0, s1                    ; encoding: [0x01,0x0e,0xfd,0xbe]
 
 s_sext_i32_i8 s0, exec_lo
-// GFX12: encoding: [0x7e,0x0e,0x80,0xbe]
+// GFX12: s_sext_i32_i8 s0, exec_lo               ; encoding: [0x7e,0x0e,0x80,0xbe]
 
 s_sext_i32_i8 s0, exec_hi
-// GFX12: encoding: [0x7f,0x0e,0x80,0xbe]
+// GFX12: s_sext_i32_i8 s0, exec_hi               ; encoding: [0x7f,0x0e,0x80,0xbe]
 
 s_sext_i32_i8 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x0e,0x80,0xbe]
+// GFX12: s_sext_i32_i8 s0, vcc_lo                ; encoding: [0x6a,0x0e,0x80,0xbe]
 
 s_sext_i32_i8 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x0e,0x80,0xbe]
+// GFX12: s_sext_i32_i8 s0, vcc_hi                ; encoding: [0x6b,0x0e,0x80,0xbe]
 
 s_sext_i32_i8 s0, m0
-// GFX12: encoding: [0x7d,0x0e,0x80,0xbe]
+// GFX12: s_sext_i32_i8 s0, m0                    ; encoding: [0x7d,0x0e,0x80,0xbe]
 
 s_sext_i32_i8 s0, 0
-// GFX12: encoding: [0x80,0x0e,0x80,0xbe]
+// GFX12: s_sext_i32_i8 s0, 0                     ; encoding: [0x80,0x0e,0x80,0xbe]
 
 s_sext_i32_i8 s0, -1
-// GFX12: encoding: [0xc1,0x0e,0x80,0xbe]
+// GFX12: s_sext_i32_i8 s0, -1                    ; encoding: [0xc1,0x0e,0x80,0xbe]
 
 s_sext_i32_i8 s0, 0.5
-// GFX12: encoding: [0xf0,0x0e,0x80,0xbe]
+// GFX12: s_sext_i32_i8 s0, 0.5                   ; encoding: [0xf0,0x0e,0x80,0xbe]
 
 s_sext_i32_i8 s0, -4.0
-// GFX12: encoding: [0xf7,0x0e,0x80,0xbe]
+// GFX12: s_sext_i32_i8 s0, -4.0                  ; encoding: [0xf7,0x0e,0x80,0xbe]
 
 s_sext_i32_i8 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x0e,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_sext_i32_i8 s0, 0x3f717273            ; encoding: [0xff,0x0e,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_sext_i32_i8 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x0e,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_sext_i32_i8 s0, 0xaf123456            ; encoding: [0xff,0x0e,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_sext_i32_i16 s0, s1
-// GFX12: encoding: [0x01,0x0f,0x80,0xbe]
+// GFX12: s_sext_i32_i16 s0, s1                   ; encoding: [0x01,0x0f,0x80,0xbe]
 
 s_sext_i32_i16 s105, s104
-// GFX12: encoding: [0x68,0x0f,0xe9,0xbe]
+// GFX12: s_sext_i32_i16 s105, s104               ; encoding: [0x68,0x0f,0xe9,0xbe]
 
 s_sext_i32_i16 s0, s104
-// GFX12: encoding: [0x68,0x0f,0x80,0xbe]
+// GFX12: s_sext_i32_i16 s0, s104                 ; encoding: [0x68,0x0f,0x80,0xbe]
 
 s_sext_i32_i16 s105, s1
-// GFX12: encoding: [0x01,0x0f,0xe9,0xbe]
+// GFX12: s_sext_i32_i16 s105, s1                 ; encoding: [0x01,0x0f,0xe9,0xbe]
 
 s_sext_i32_i16 exec_lo, s1
-// GFX12: encoding: [0x01,0x0f,0xfe,0xbe]
+// GFX12: s_sext_i32_i16 exec_lo, s1              ; encoding: [0x01,0x0f,0xfe,0xbe]
 
 s_sext_i32_i16 exec_hi, s1
-// GFX12: encoding: [0x01,0x0f,0xff,0xbe]
+// GFX12: s_sext_i32_i16 exec_hi, s1              ; encoding: [0x01,0x0f,0xff,0xbe]
 
 s_sext_i32_i16 vcc_lo, s1
-// GFX12: encoding: [0x01,0x0f,0xea,0xbe]
+// GFX12: s_sext_i32_i16 vcc_lo, s1               ; encoding: [0x01,0x0f,0xea,0xbe]
 
 s_sext_i32_i16 vcc_hi, s1
-// GFX12: encoding: [0x01,0x0f,0xeb,0xbe]
+// GFX12: s_sext_i32_i16 vcc_hi, s1               ; encoding: [0x01,0x0f,0xeb,0xbe]
 
 s_sext_i32_i16 m0, s1
-// GFX12: encoding: [0x01,0x0f,0xfd,0xbe]
+// GFX12: s_sext_i32_i16 m0, s1                   ; encoding: [0x01,0x0f,0xfd,0xbe]
 
 s_sext_i32_i16 s0, exec_lo
-// GFX12: encoding: [0x7e,0x0f,0x80,0xbe]
+// GFX12: s_sext_i32_i16 s0, exec_lo              ; encoding: [0x7e,0x0f,0x80,0xbe]
 
 s_sext_i32_i16 s0, exec_hi
-// GFX12: encoding: [0x7f,0x0f,0x80,0xbe]
+// GFX12: s_sext_i32_i16 s0, exec_hi              ; encoding: [0x7f,0x0f,0x80,0xbe]
 
 s_sext_i32_i16 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x0f,0x80,0xbe]
+// GFX12: s_sext_i32_i16 s0, vcc_lo               ; encoding: [0x6a,0x0f,0x80,0xbe]
 
 s_sext_i32_i16 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x0f,0x80,0xbe]
+// GFX12: s_sext_i32_i16 s0, vcc_hi               ; encoding: [0x6b,0x0f,0x80,0xbe]
 
 s_sext_i32_i16 s0, m0
-// GFX12: encoding: [0x7d,0x0f,0x80,0xbe]
+// GFX12: s_sext_i32_i16 s0, m0                   ; encoding: [0x7d,0x0f,0x80,0xbe]
 
 s_sext_i32_i16 s0, 0
-// GFX12: encoding: [0x80,0x0f,0x80,0xbe]
+// GFX12: s_sext_i32_i16 s0, 0                    ; encoding: [0x80,0x0f,0x80,0xbe]
 
 s_sext_i32_i16 s0, -1
-// GFX12: encoding: [0xc1,0x0f,0x80,0xbe]
+// GFX12: s_sext_i32_i16 s0, -1                   ; encoding: [0xc1,0x0f,0x80,0xbe]
 
 s_sext_i32_i16 s0, 0.5
-// GFX12: encoding: [0xf0,0x0f,0x80,0xbe]
+// GFX12: s_sext_i32_i16 s0, 0.5                  ; encoding: [0xf0,0x0f,0x80,0xbe]
 
 s_sext_i32_i16 s0, -4.0
-// GFX12: encoding: [0xf7,0x0f,0x80,0xbe]
+// GFX12: s_sext_i32_i16 s0, -4.0                 ; encoding: [0xf7,0x0f,0x80,0xbe]
 
 s_sext_i32_i16 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x0f,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_sext_i32_i16 s0, 0x3f717273           ; encoding: [0xff,0x0f,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_sext_i32_i16 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x0f,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_sext_i32_i16 s0, 0xaf123456           ; encoding: [0xff,0x0f,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_bitset0_b32 s0, s1
-// GFX12: encoding: [0x01,0x10,0x80,0xbe]
+// GFX12: s_bitset0_b32 s0, s1                    ; encoding: [0x01,0x10,0x80,0xbe]
 
 s_bitset0_b32 s105, s104
-// GFX12: encoding: [0x68,0x10,0xe9,0xbe]
+// GFX12: s_bitset0_b32 s105, s104                ; encoding: [0x68,0x10,0xe9,0xbe]
 
 s_bitset0_b32 s0, s104
-// GFX12: encoding: [0x68,0x10,0x80,0xbe]
+// GFX12: s_bitset0_b32 s0, s104                  ; encoding: [0x68,0x10,0x80,0xbe]
 
 s_bitset0_b32 s105, s1
-// GFX12: encoding: [0x01,0x10,0xe9,0xbe]
+// GFX12: s_bitset0_b32 s105, s1                  ; encoding: [0x01,0x10,0xe9,0xbe]
 
 s_bitset0_b32 exec_lo, s1
-// GFX12: encoding: [0x01,0x10,0xfe,0xbe]
+// GFX12: s_bitset0_b32 exec_lo, s1               ; encoding: [0x01,0x10,0xfe,0xbe]
 
 s_bitset0_b32 exec_hi, s1
-// GFX12: encoding: [0x01,0x10,0xff,0xbe]
+// GFX12: s_bitset0_b32 exec_hi, s1               ; encoding: [0x01,0x10,0xff,0xbe]
 
 s_bitset0_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x10,0xea,0xbe]
+// GFX12: s_bitset0_b32 vcc_lo, s1                ; encoding: [0x01,0x10,0xea,0xbe]
 
 s_bitset0_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x10,0xeb,0xbe]
+// GFX12: s_bitset0_b32 vcc_hi, s1                ; encoding: [0x01,0x10,0xeb,0xbe]
 
 s_bitset0_b32 m0, s1
-// GFX12: encoding: [0x01,0x10,0xfd,0xbe]
+// GFX12: s_bitset0_b32 m0, s1                    ; encoding: [0x01,0x10,0xfd,0xbe]
 
 s_bitset0_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x10,0x80,0xbe]
+// GFX12: s_bitset0_b32 s0, exec_lo               ; encoding: [0x7e,0x10,0x80,0xbe]
 
 s_bitset0_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x10,0x80,0xbe]
+// GFX12: s_bitset0_b32 s0, exec_hi               ; encoding: [0x7f,0x10,0x80,0xbe]
 
 s_bitset0_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x10,0x80,0xbe]
+// GFX12: s_bitset0_b32 s0, vcc_lo                ; encoding: [0x6a,0x10,0x80,0xbe]
 
 s_bitset0_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x10,0x80,0xbe]
+// GFX12: s_bitset0_b32 s0, vcc_hi                ; encoding: [0x6b,0x10,0x80,0xbe]
 
 s_bitset0_b32 s0, m0
-// GFX12: encoding: [0x7d,0x10,0x80,0xbe]
+// GFX12: s_bitset0_b32 s0, m0                    ; encoding: [0x7d,0x10,0x80,0xbe]
 
 s_bitset0_b32 s0, 0
-// GFX12: encoding: [0x80,0x10,0x80,0xbe]
+// GFX12: s_bitset0_b32 s0, 0                     ; encoding: [0x80,0x10,0x80,0xbe]
 
 s_bitset0_b32 s0, -1
-// GFX12: encoding: [0xc1,0x10,0x80,0xbe]
+// GFX12: s_bitset0_b32 s0, -1                    ; encoding: [0xc1,0x10,0x80,0xbe]
 
 s_bitset0_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x10,0x80,0xbe]
+// GFX12: s_bitset0_b32 s0, 0.5                   ; encoding: [0xf0,0x10,0x80,0xbe]
 
 s_bitset0_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x10,0x80,0xbe]
+// GFX12: s_bitset0_b32 s0, -4.0                  ; encoding: [0xf7,0x10,0x80,0xbe]
 
 s_bitset0_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x10,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_bitset0_b32 s0, 0x3f717273            ; encoding: [0xff,0x10,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_bitset0_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x10,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_bitset0_b32 s0, 0xaf123456            ; encoding: [0xff,0x10,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_bitset0_b64 s[0:1], s2
-// GFX12: encoding: [0x02,0x11,0x80,0xbe]
+// GFX12: s_bitset0_b64 s[0:1], s2                ; encoding: [0x02,0x11,0x80,0xbe]
 
 s_bitset0_b64 s[104:105], s102
-// GFX12: encoding: [0x66,0x11,0xe8,0xbe]
+// GFX12: s_bitset0_b64 s[104:105], s102          ; encoding: [0x66,0x11,0xe8,0xbe]
 
 s_bitset0_b64 s[0:1], s102
-// GFX12: encoding: [0x66,0x11,0x80,0xbe]
+// GFX12: s_bitset0_b64 s[0:1], s102              ; encoding: [0x66,0x11,0x80,0xbe]
 
 s_bitset0_b64 s[104:105], s2
-// GFX12: encoding: [0x02,0x11,0xe8,0xbe]
+// GFX12: s_bitset0_b64 s[104:105], s2            ; encoding: [0x02,0x11,0xe8,0xbe]
 
 s_bitset0_b64 exec, s2
-// GFX12: encoding: [0x02,0x11,0xfe,0xbe]
+// GFX12: s_bitset0_b64 exec, s2                  ; encoding: [0x02,0x11,0xfe,0xbe]
 
 s_bitset0_b64 vcc, s2
-// GFX12: encoding: [0x02,0x11,0xea,0xbe]
+// GFX12: s_bitset0_b64 vcc, s2                   ; encoding: [0x02,0x11,0xea,0xbe]
 
 s_bitset0_b64 s[0:1], exec_lo
-// GFX12: encoding: [0x7e,0x11,0x80,0xbe]
+// GFX12: s_bitset0_b64 s[0:1], exec_lo           ; encoding: [0x7e,0x11,0x80,0xbe]
 
 s_bitset0_b64 s[0:1], exec_hi
-// GFX12: encoding: [0x7f,0x11,0x80,0xbe]
+// GFX12: s_bitset0_b64 s[0:1], exec_hi           ; encoding: [0x7f,0x11,0x80,0xbe]
 
 s_bitset0_b64 s[0:1], vcc_lo
-// GFX12: encoding: [0x6a,0x11,0x80,0xbe]
+// GFX12: s_bitset0_b64 s[0:1], vcc_lo            ; encoding: [0x6a,0x11,0x80,0xbe]
 
 s_bitset0_b64 s[0:1], vcc_hi
-// GFX12: encoding: [0x6b,0x11,0x80,0xbe]
+// GFX12: s_bitset0_b64 s[0:1], vcc_hi            ; encoding: [0x6b,0x11,0x80,0xbe]
 
 s_bitset0_b64 s[0:1], m0
-// GFX12: encoding: [0x7d,0x11,0x80,0xbe]
+// GFX12: s_bitset0_b64 s[0:1], m0                ; encoding: [0x7d,0x11,0x80,0xbe]
 
 s_bitset0_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x11,0x80,0xbe]
+// GFX12: s_bitset0_b64 s[0:1], 0                 ; encoding: [0x80,0x11,0x80,0xbe]
 
 s_bitset0_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x11,0x80,0xbe]
+// GFX12: s_bitset0_b64 s[0:1], -1                ; encoding: [0xc1,0x11,0x80,0xbe]
 
 s_bitset0_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x11,0x80,0xbe]
+// GFX12: s_bitset0_b64 s[0:1], 0.5               ; encoding: [0xf0,0x11,0x80,0xbe]
 
 s_bitset0_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x11,0x80,0xbe]
+// GFX12: s_bitset0_b64 s[0:1], -4.0              ; encoding: [0xf7,0x11,0x80,0xbe]
 
 s_bitset0_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x11,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_bitset0_b64 s[0:1], 0x3f717273        ; encoding: [0xff,0x11,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_bitset0_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x11,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_bitset0_b64 s[0:1], 0xaf123456        ; encoding: [0xff,0x11,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_bitset1_b32 s0, s1
-// GFX12: encoding: [0x01,0x12,0x80,0xbe]
+// GFX12: s_bitset1_b32 s0, s1                    ; encoding: [0x01,0x12,0x80,0xbe]
 
 s_bitset1_b32 s105, s104
-// GFX12: encoding: [0x68,0x12,0xe9,0xbe]
+// GFX12: s_bitset1_b32 s105, s104                ; encoding: [0x68,0x12,0xe9,0xbe]
 
 s_bitset1_b32 s0, s104
-// GFX12: encoding: [0x68,0x12,0x80,0xbe]
+// GFX12: s_bitset1_b32 s0, s104                  ; encoding: [0x68,0x12,0x80,0xbe]
 
 s_bitset1_b32 s105, s1
-// GFX12: encoding: [0x01,0x12,0xe9,0xbe]
+// GFX12: s_bitset1_b32 s105, s1                  ; encoding: [0x01,0x12,0xe9,0xbe]
 
 s_bitset1_b32 exec_lo, s1
-// GFX12: encoding: [0x01,0x12,0xfe,0xbe]
+// GFX12: s_bitset1_b32 exec_lo, s1               ; encoding: [0x01,0x12,0xfe,0xbe]
 
 s_bitset1_b32 exec_hi, s1
-// GFX12: encoding: [0x01,0x12,0xff,0xbe]
+// GFX12: s_bitset1_b32 exec_hi, s1               ; encoding: [0x01,0x12,0xff,0xbe]
 
 s_bitset1_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x12,0xea,0xbe]
+// GFX12: s_bitset1_b32 vcc_lo, s1                ; encoding: [0x01,0x12,0xea,0xbe]
 
 s_bitset1_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x12,0xeb,0xbe]
+// GFX12: s_bitset1_b32 vcc_hi, s1                ; encoding: [0x01,0x12,0xeb,0xbe]
 
 s_bitset1_b32 m0, s1
-// GFX12: encoding: [0x01,0x12,0xfd,0xbe]
+// GFX12: s_bitset1_b32 m0, s1                    ; encoding: [0x01,0x12,0xfd,0xbe]
 
 s_bitset1_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x12,0x80,0xbe]
+// GFX12: s_bitset1_b32 s0, exec_lo               ; encoding: [0x7e,0x12,0x80,0xbe]
 
 s_bitset1_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x12,0x80,0xbe]
+// GFX12: s_bitset1_b32 s0, exec_hi               ; encoding: [0x7f,0x12,0x80,0xbe]
 
 s_bitset1_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x12,0x80,0xbe]
+// GFX12: s_bitset1_b32 s0, vcc_lo                ; encoding: [0x6a,0x12,0x80,0xbe]
 
 s_bitset1_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x12,0x80,0xbe]
+// GFX12: s_bitset1_b32 s0, vcc_hi                ; encoding: [0x6b,0x12,0x80,0xbe]
 
 s_bitset1_b32 s0, m0
-// GFX12: encoding: [0x7d,0x12,0x80,0xbe]
+// GFX12: s_bitset1_b32 s0, m0                    ; encoding: [0x7d,0x12,0x80,0xbe]
 
 s_bitset1_b32 s0, 0
-// GFX12: encoding: [0x80,0x12,0x80,0xbe]
+// GFX12: s_bitset1_b32 s0, 0                     ; encoding: [0x80,0x12,0x80,0xbe]
 
 s_bitset1_b32 s0, -1
-// GFX12: encoding: [0xc1,0x12,0x80,0xbe]
+// GFX12: s_bitset1_b32 s0, -1                    ; encoding: [0xc1,0x12,0x80,0xbe]
 
 s_bitset1_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x12,0x80,0xbe]
+// GFX12: s_bitset1_b32 s0, 0.5                   ; encoding: [0xf0,0x12,0x80,0xbe]
 
 s_bitset1_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x12,0x80,0xbe]
+// GFX12: s_bitset1_b32 s0, -4.0                  ; encoding: [0xf7,0x12,0x80,0xbe]
 
 s_bitset1_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x12,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_bitset1_b32 s0, 0x3f717273            ; encoding: [0xff,0x12,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_bitset1_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x12,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_bitset1_b32 s0, 0xaf123456            ; encoding: [0xff,0x12,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_bitset1_b64 s[0:1], s2
-// GFX12: encoding: [0x02,0x13,0x80,0xbe]
+// GFX12: s_bitset1_b64 s[0:1], s2                ; encoding: [0x02,0x13,0x80,0xbe]
 
 s_bitset1_b64 s[104:105], s102
-// GFX12: encoding: [0x66,0x13,0xe8,0xbe]
+// GFX12: s_bitset1_b64 s[104:105], s102          ; encoding: [0x66,0x13,0xe8,0xbe]
 
 s_bitset1_b64 s[0:1], s102
-// GFX12: encoding: [0x66,0x13,0x80,0xbe]
+// GFX12: s_bitset1_b64 s[0:1], s102              ; encoding: [0x66,0x13,0x80,0xbe]
 
 s_bitset1_b64 s[104:105], s2
-// GFX12: encoding: [0x02,0x13,0xe8,0xbe]
+// GFX12: s_bitset1_b64 s[104:105], s2            ; encoding: [0x02,0x13,0xe8,0xbe]
 
 s_bitset1_b64 exec, s2
-// GFX12: encoding: [0x02,0x13,0xfe,0xbe]
+// GFX12: s_bitset1_b64 exec, s2                  ; encoding: [0x02,0x13,0xfe,0xbe]
 
 s_bitset1_b64 vcc, s2
-// GFX12: encoding: [0x02,0x13,0xea,0xbe]
+// GFX12: s_bitset1_b64 vcc, s2                   ; encoding: [0x02,0x13,0xea,0xbe]
 
 s_bitset1_b64 s[0:1], exec_lo
-// GFX12: encoding: [0x7e,0x13,0x80,0xbe]
+// GFX12: s_bitset1_b64 s[0:1], exec_lo           ; encoding: [0x7e,0x13,0x80,0xbe]
 
 s_bitset1_b64 s[0:1], exec_hi
-// GFX12: encoding: [0x7f,0x13,0x80,0xbe]
+// GFX12: s_bitset1_b64 s[0:1], exec_hi           ; encoding: [0x7f,0x13,0x80,0xbe]
 
 s_bitset1_b64 s[0:1], vcc_lo
-// GFX12: encoding: [0x6a,0x13,0x80,0xbe]
+// GFX12: s_bitset1_b64 s[0:1], vcc_lo            ; encoding: [0x6a,0x13,0x80,0xbe]
 
 s_bitset1_b64 s[0:1], vcc_hi
-// GFX12: encoding: [0x6b,0x13,0x80,0xbe]
+// GFX12: s_bitset1_b64 s[0:1], vcc_hi            ; encoding: [0x6b,0x13,0x80,0xbe]
 
 s_bitset1_b64 s[0:1], m0
-// GFX12: encoding: [0x7d,0x13,0x80,0xbe]
+// GFX12: s_bitset1_b64 s[0:1], m0                ; encoding: [0x7d,0x13,0x80,0xbe]
 
 s_bitset1_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x13,0x80,0xbe]
+// GFX12: s_bitset1_b64 s[0:1], 0                 ; encoding: [0x80,0x13,0x80,0xbe]
 
 s_bitset1_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x13,0x80,0xbe]
+// GFX12: s_bitset1_b64 s[0:1], -1                ; encoding: [0xc1,0x13,0x80,0xbe]
 
 s_bitset1_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x13,0x80,0xbe]
+// GFX12: s_bitset1_b64 s[0:1], 0.5               ; encoding: [0xf0,0x13,0x80,0xbe]
 
 s_bitset1_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x13,0x80,0xbe]
+// GFX12: s_bitset1_b64 s[0:1], -4.0              ; encoding: [0xf7,0x13,0x80,0xbe]
 
 s_bitset1_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x13,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_bitset1_b64 s[0:1], 0x3f717273        ; encoding: [0xff,0x13,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_bitset1_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x13,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_bitset1_b64 s[0:1], 0xaf123456        ; encoding: [0xff,0x13,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_getpc_b64 s[0:1]
-// GFX12: encoding: [0x00,0x47,0x80,0xbe]
+// GFX12: s_getpc_b64 s[0:1]                      ; encoding: [0x00,0x47,0x80,0xbe]
 
 s_getpc_b64 s[104:105]
-// GFX12: encoding: [0x00,0x47,0xe8,0xbe]
+// GFX12: s_getpc_b64 s[104:105]                  ; encoding: [0x00,0x47,0xe8,0xbe]
 
 s_getpc_b64 exec
-// GFX12: encoding: [0x00,0x47,0xfe,0xbe]
+// GFX12: s_getpc_b64 exec                        ; encoding: [0x00,0x47,0xfe,0xbe]
 
 s_getpc_b64 vcc
-// GFX12: encoding: [0x00,0x47,0xea,0xbe]
+// GFX12: s_getpc_b64 vcc                         ; encoding: [0x00,0x47,0xea,0xbe]
 
 s_setpc_b64 s[0:1]
-// GFX12: encoding: [0x00,0x48,0x80,0xbe]
+// GFX12: s_setpc_b64 s[0:1]                      ; encoding: [0x00,0x48,0x80,0xbe]
 
 s_setpc_b64 s[104:105]
-// GFX12: encoding: [0x68,0x48,0x80,0xbe]
+// GFX12: s_setpc_b64 s[104:105]                  ; encoding: [0x68,0x48,0x80,0xbe]
 
 s_setpc_b64 vcc
-// GFX12: encoding: [0x6a,0x48,0x80,0xbe]
+// GFX12: s_setpc_b64 vcc                         ; encoding: [0x6a,0x48,0x80,0xbe]
 
 s_swappc_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x49,0x80,0xbe]
+// GFX12: s_swappc_b64 s[0:1], s[2:3]             ; encoding: [0x02,0x49,0x80,0xbe]
 
 s_swappc_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x49,0xe8,0xbe]
+// GFX12: s_swappc_b64 s[104:105], s[102:103]     ; encoding: [0x66,0x49,0xe8,0xbe]
 
 s_swappc_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x49,0x80,0xbe]
+// GFX12: s_swappc_b64 s[0:1], s[102:103]         ; encoding: [0x66,0x49,0x80,0xbe]
 
 s_swappc_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x49,0xe8,0xbe]
+// GFX12: s_swappc_b64 s[104:105], s[2:3]         ; encoding: [0x02,0x49,0xe8,0xbe]
 
 s_swappc_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x49,0xea,0xbe]
+// GFX12: s_swappc_b64 vcc, s[2:3]                ; encoding: [0x02,0x49,0xea,0xbe]
 
 s_swappc_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x49,0x80,0xbe]
+// GFX12: s_swappc_b64 s[0:1], vcc                ; encoding: [0x6a,0x49,0x80,0xbe]
 
 s_rfe_b64 s[0:1]
-// GFX12: encoding: [0x00,0x4a,0x80,0xbe]
+// GFX12: s_rfe_b64 s[0:1]                        ; encoding: [0x00,0x4a,0x80,0xbe]
 
 s_rfe_b64 s[104:105]
-// GFX12: encoding: [0x68,0x4a,0x80,0xbe]
+// GFX12: s_rfe_b64 s[104:105]                    ; encoding: [0x68,0x4a,0x80,0xbe]
 
 s_rfe_b64 vcc
-// GFX12: encoding: [0x6a,0x4a,0x80,0xbe]
+// GFX12: s_rfe_b64 vcc                           ; encoding: [0x6a,0x4a,0x80,0xbe]
 
 s_and_saveexec_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x21,0x80,0xbe]
+// GFX12: s_and_saveexec_b64 s[0:1], s[2:3]       ; encoding: [0x02,0x21,0x80,0xbe]
 
 s_and_saveexec_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x21,0xe8,0xbe]
+// GFX12: s_and_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x21,0xe8,0xbe]
 
 s_and_saveexec_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x21,0x80,0xbe]
+// GFX12: s_and_saveexec_b64 s[0:1], s[102:103]   ; encoding: [0x66,0x21,0x80,0xbe]
 
 s_and_saveexec_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x21,0xe8,0xbe]
+// GFX12: s_and_saveexec_b64 s[104:105], s[2:3]   ; encoding: [0x02,0x21,0xe8,0xbe]
 
 s_and_saveexec_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x21,0xea,0xbe]
+// GFX12: s_and_saveexec_b64 vcc, s[2:3]          ; encoding: [0x02,0x21,0xea,0xbe]
 
 s_and_saveexec_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x21,0x80,0xbe]
+// GFX12: s_and_saveexec_b64 s[0:1], exec         ; encoding: [0x7e,0x21,0x80,0xbe]
 
 s_and_saveexec_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x21,0x80,0xbe]
+// GFX12: s_and_saveexec_b64 s[0:1], vcc          ; encoding: [0x6a,0x21,0x80,0xbe]
 
 s_and_saveexec_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x21,0x80,0xbe]
+// GFX12: s_and_saveexec_b64 s[0:1], 0            ; encoding: [0x80,0x21,0x80,0xbe]
 
 s_and_saveexec_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x21,0x80,0xbe]
+// GFX12: s_and_saveexec_b64 s[0:1], -1           ; encoding: [0xc1,0x21,0x80,0xbe]
 
 s_and_saveexec_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x21,0x80,0xbe]
+// GFX12: s_and_saveexec_b64 s[0:1], 0.5          ; encoding: [0xf0,0x21,0x80,0xbe]
 
 s_and_saveexec_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x21,0x80,0xbe]
+// GFX12: s_and_saveexec_b64 s[0:1], -4.0         ; encoding: [0xf7,0x21,0x80,0xbe]
 
 s_and_saveexec_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x21,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_and_saveexec_b64 s[0:1], 0x3f717273   ; encoding: [0xff,0x21,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_and_saveexec_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x21,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_saveexec_b64 s[0:1], 0xaf123456   ; encoding: [0xff,0x21,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_or_saveexec_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x23,0x80,0xbe]
+// GFX12: s_or_saveexec_b64 s[0:1], s[2:3]        ; encoding: [0x02,0x23,0x80,0xbe]
 
 s_or_saveexec_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x23,0xe8,0xbe]
+// GFX12: s_or_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x23,0xe8,0xbe]
 
 s_or_saveexec_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x23,0x80,0xbe]
+// GFX12: s_or_saveexec_b64 s[0:1], s[102:103]    ; encoding: [0x66,0x23,0x80,0xbe]
 
 s_or_saveexec_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x23,0xe8,0xbe]
+// GFX12: s_or_saveexec_b64 s[104:105], s[2:3]    ; encoding: [0x02,0x23,0xe8,0xbe]
 
 s_or_saveexec_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x23,0xea,0xbe]
+// GFX12: s_or_saveexec_b64 vcc, s[2:3]           ; encoding: [0x02,0x23,0xea,0xbe]
 
 s_or_saveexec_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x23,0x80,0xbe]
+// GFX12: s_or_saveexec_b64 s[0:1], exec          ; encoding: [0x7e,0x23,0x80,0xbe]
 
 s_or_saveexec_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x23,0x80,0xbe]
+// GFX12: s_or_saveexec_b64 s[0:1], vcc           ; encoding: [0x6a,0x23,0x80,0xbe]
 
 s_or_saveexec_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x23,0x80,0xbe]
+// GFX12: s_or_saveexec_b64 s[0:1], 0             ; encoding: [0x80,0x23,0x80,0xbe]
 
 s_or_saveexec_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x23,0x80,0xbe]
+// GFX12: s_or_saveexec_b64 s[0:1], -1            ; encoding: [0xc1,0x23,0x80,0xbe]
 
 s_or_saveexec_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x23,0x80,0xbe]
+// GFX12: s_or_saveexec_b64 s[0:1], 0.5           ; encoding: [0xf0,0x23,0x80,0xbe]
 
 s_or_saveexec_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x23,0x80,0xbe]
+// GFX12: s_or_saveexec_b64 s[0:1], -4.0          ; encoding: [0xf7,0x23,0x80,0xbe]
 
 s_or_saveexec_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x23,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_or_saveexec_b64 s[0:1], 0x3f717273    ; encoding: [0xff,0x23,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_or_saveexec_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x23,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_saveexec_b64 s[0:1], 0xaf123456    ; encoding: [0xff,0x23,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_xor_saveexec_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x25,0x80,0xbe]
+// GFX12: s_xor_saveexec_b64 s[0:1], s[2:3]       ; encoding: [0x02,0x25,0x80,0xbe]
 
 s_xor_saveexec_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x25,0xe8,0xbe]
+// GFX12: s_xor_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x25,0xe8,0xbe]
 
 s_xor_saveexec_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x25,0x80,0xbe]
+// GFX12: s_xor_saveexec_b64 s[0:1], s[102:103]   ; encoding: [0x66,0x25,0x80,0xbe]
 
 s_xor_saveexec_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x25,0xe8,0xbe]
+// GFX12: s_xor_saveexec_b64 s[104:105], s[2:3]   ; encoding: [0x02,0x25,0xe8,0xbe]
 
 s_xor_saveexec_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x25,0xea,0xbe]
+// GFX12: s_xor_saveexec_b64 vcc, s[2:3]          ; encoding: [0x02,0x25,0xea,0xbe]
 
 s_xor_saveexec_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x25,0x80,0xbe]
+// GFX12: s_xor_saveexec_b64 s[0:1], exec         ; encoding: [0x7e,0x25,0x80,0xbe]
 
 s_xor_saveexec_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x25,0x80,0xbe]
+// GFX12: s_xor_saveexec_b64 s[0:1], vcc          ; encoding: [0x6a,0x25,0x80,0xbe]
 
 s_xor_saveexec_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x25,0x80,0xbe]
+// GFX12: s_xor_saveexec_b64 s[0:1], 0            ; encoding: [0x80,0x25,0x80,0xbe]
 
 s_xor_saveexec_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x25,0x80,0xbe]
+// GFX12: s_xor_saveexec_b64 s[0:1], -1           ; encoding: [0xc1,0x25,0x80,0xbe]
 
 s_xor_saveexec_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x25,0x80,0xbe]
+// GFX12: s_xor_saveexec_b64 s[0:1], 0.5          ; encoding: [0xf0,0x25,0x80,0xbe]
 
 s_xor_saveexec_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x25,0x80,0xbe]
+// GFX12: s_xor_saveexec_b64 s[0:1], -4.0         ; encoding: [0xf7,0x25,0x80,0xbe]
 
 s_xor_saveexec_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x25,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_xor_saveexec_b64 s[0:1], 0x3f717273   ; encoding: [0xff,0x25,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_xor_saveexec_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x25,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_xor_saveexec_b64 s[0:1], 0xaf123456   ; encoding: [0xff,0x25,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_andn2_saveexec_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x31,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b64 s[0:1], s[2:3]  ; encoding: [0x02,0x31,0x80,0xbe]
 
 s_andn2_saveexec_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x31,0xe8,0xbe]
+// GFX12: s_and_not1_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x31,0xe8,0xbe]
 
 s_andn2_saveexec_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x31,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b64 s[0:1], s[102:103] ; encoding: [0x66,0x31,0x80,0xbe]
 
 s_andn2_saveexec_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x31,0xe8,0xbe]
+// GFX12: s_and_not1_saveexec_b64 s[104:105], s[2:3] ; encoding: [0x02,0x31,0xe8,0xbe]
 
 s_andn2_saveexec_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x31,0xea,0xbe]
+// GFX12: s_and_not1_saveexec_b64 vcc, s[2:3]     ; encoding: [0x02,0x31,0xea,0xbe]
 
 s_andn2_saveexec_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x31,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b64 s[0:1], exec    ; encoding: [0x7e,0x31,0x80,0xbe]
 
 s_andn2_saveexec_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x31,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b64 s[0:1], vcc     ; encoding: [0x6a,0x31,0x80,0xbe]
 
 s_andn2_saveexec_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x31,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b64 s[0:1], 0       ; encoding: [0x80,0x31,0x80,0xbe]
 
 s_andn2_saveexec_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x31,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b64 s[0:1], -1      ; encoding: [0xc1,0x31,0x80,0xbe]
 
 s_andn2_saveexec_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x31,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b64 s[0:1], 0.5     ; encoding: [0xf0,0x31,0x80,0xbe]
 
 s_andn2_saveexec_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x31,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b64 s[0:1], -4.0    ; encoding: [0xf7,0x31,0x80,0xbe]
 
 s_andn2_saveexec_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x31,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_and_not1_saveexec_b64 s[0:1], 0x3f717273 ; encoding: [0xff,0x31,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_andn2_saveexec_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x31,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not1_saveexec_b64 s[0:1], 0xaf123456 ; encoding: [0xff,0x31,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_orn2_saveexec_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x33,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b64 s[0:1], s[2:3]   ; encoding: [0x02,0x33,0x80,0xbe]
 
 s_orn2_saveexec_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x33,0xe8,0xbe]
+// GFX12: s_or_not1_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x33,0xe8,0xbe]
 
 s_orn2_saveexec_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x33,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b64 s[0:1], s[102:103] ; encoding: [0x66,0x33,0x80,0xbe]
 
 s_orn2_saveexec_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x33,0xe8,0xbe]
+// GFX12: s_or_not1_saveexec_b64 s[104:105], s[2:3] ; encoding: [0x02,0x33,0xe8,0xbe]
 
 s_orn2_saveexec_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x33,0xea,0xbe]
+// GFX12: s_or_not1_saveexec_b64 vcc, s[2:3]      ; encoding: [0x02,0x33,0xea,0xbe]
 
 s_orn2_saveexec_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x33,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b64 s[0:1], exec     ; encoding: [0x7e,0x33,0x80,0xbe]
 
 s_orn2_saveexec_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x33,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b64 s[0:1], vcc      ; encoding: [0x6a,0x33,0x80,0xbe]
 
 s_orn2_saveexec_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x33,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b64 s[0:1], 0        ; encoding: [0x80,0x33,0x80,0xbe]
 
 s_orn2_saveexec_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x33,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b64 s[0:1], -1       ; encoding: [0xc1,0x33,0x80,0xbe]
 
 s_orn2_saveexec_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x33,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b64 s[0:1], 0.5      ; encoding: [0xf0,0x33,0x80,0xbe]
 
 s_orn2_saveexec_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x33,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b64 s[0:1], -4.0     ; encoding: [0xf7,0x33,0x80,0xbe]
 
 s_orn2_saveexec_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x33,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_or_not1_saveexec_b64 s[0:1], 0x3f717273 ; encoding: [0xff,0x33,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_orn2_saveexec_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x33,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_not1_saveexec_b64 s[0:1], 0xaf123456 ; encoding: [0xff,0x33,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_nand_saveexec_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x27,0x80,0xbe]
+// GFX12: s_nand_saveexec_b64 s[0:1], s[2:3]      ; encoding: [0x02,0x27,0x80,0xbe]
 
 s_nand_saveexec_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x27,0xe8,0xbe]
+// GFX12: s_nand_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x27,0xe8,0xbe]
 
 s_nand_saveexec_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x27,0x80,0xbe]
+// GFX12: s_nand_saveexec_b64 s[0:1], s[102:103]  ; encoding: [0x66,0x27,0x80,0xbe]
 
 s_nand_saveexec_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x27,0xe8,0xbe]
+// GFX12: s_nand_saveexec_b64 s[104:105], s[2:3]  ; encoding: [0x02,0x27,0xe8,0xbe]
 
 s_nand_saveexec_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x27,0xea,0xbe]
+// GFX12: s_nand_saveexec_b64 vcc, s[2:3]         ; encoding: [0x02,0x27,0xea,0xbe]
 
 s_nand_saveexec_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x27,0x80,0xbe]
+// GFX12: s_nand_saveexec_b64 s[0:1], exec        ; encoding: [0x7e,0x27,0x80,0xbe]
 
 s_nand_saveexec_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x27,0x80,0xbe]
+// GFX12: s_nand_saveexec_b64 s[0:1], vcc         ; encoding: [0x6a,0x27,0x80,0xbe]
 
 s_nand_saveexec_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x27,0x80,0xbe]
+// GFX12: s_nand_saveexec_b64 s[0:1], 0           ; encoding: [0x80,0x27,0x80,0xbe]
 
 s_nand_saveexec_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x27,0x80,0xbe]
+// GFX12: s_nand_saveexec_b64 s[0:1], -1          ; encoding: [0xc1,0x27,0x80,0xbe]
 
 s_nand_saveexec_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x27,0x80,0xbe]
+// GFX12: s_nand_saveexec_b64 s[0:1], 0.5         ; encoding: [0xf0,0x27,0x80,0xbe]
 
 s_nand_saveexec_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x27,0x80,0xbe]
+// GFX12: s_nand_saveexec_b64 s[0:1], -4.0        ; encoding: [0xf7,0x27,0x80,0xbe]
 
 s_nand_saveexec_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x27,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_nand_saveexec_b64 s[0:1], 0x3f717273  ; encoding: [0xff,0x27,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_nand_saveexec_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x27,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_nand_saveexec_b64 s[0:1], 0xaf123456  ; encoding: [0xff,0x27,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_nor_saveexec_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x29,0x80,0xbe]
+// GFX12: s_nor_saveexec_b64 s[0:1], s[2:3]       ; encoding: [0x02,0x29,0x80,0xbe]
 
 s_nor_saveexec_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x29,0xe8,0xbe]
+// GFX12: s_nor_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x29,0xe8,0xbe]
 
 s_nor_saveexec_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x29,0x80,0xbe]
+// GFX12: s_nor_saveexec_b64 s[0:1], s[102:103]   ; encoding: [0x66,0x29,0x80,0xbe]
 
 s_nor_saveexec_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x29,0xe8,0xbe]
+// GFX12: s_nor_saveexec_b64 s[104:105], s[2:3]   ; encoding: [0x02,0x29,0xe8,0xbe]
 
 s_nor_saveexec_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x29,0xea,0xbe]
+// GFX12: s_nor_saveexec_b64 vcc, s[2:3]          ; encoding: [0x02,0x29,0xea,0xbe]
 
 s_nor_saveexec_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x29,0x80,0xbe]
+// GFX12: s_nor_saveexec_b64 s[0:1], exec         ; encoding: [0x7e,0x29,0x80,0xbe]
 
 s_nor_saveexec_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x29,0x80,0xbe]
+// GFX12: s_nor_saveexec_b64 s[0:1], vcc          ; encoding: [0x6a,0x29,0x80,0xbe]
 
 s_nor_saveexec_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x29,0x80,0xbe]
+// GFX12: s_nor_saveexec_b64 s[0:1], 0            ; encoding: [0x80,0x29,0x80,0xbe]
 
 s_nor_saveexec_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x29,0x80,0xbe]
+// GFX12: s_nor_saveexec_b64 s[0:1], -1           ; encoding: [0xc1,0x29,0x80,0xbe]
 
 s_nor_saveexec_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x29,0x80,0xbe]
+// GFX12: s_nor_saveexec_b64 s[0:1], 0.5          ; encoding: [0xf0,0x29,0x80,0xbe]
 
 s_nor_saveexec_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x29,0x80,0xbe]
+// GFX12: s_nor_saveexec_b64 s[0:1], -4.0         ; encoding: [0xf7,0x29,0x80,0xbe]
 
 s_nor_saveexec_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x29,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_nor_saveexec_b64 s[0:1], 0x3f717273   ; encoding: [0xff,0x29,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_nor_saveexec_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x29,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_nor_saveexec_b64 s[0:1], 0xaf123456   ; encoding: [0xff,0x29,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_xnor_saveexec_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x2b,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b64 s[0:1], s[2:3]      ; encoding: [0x02,0x2b,0x80,0xbe]
 
 s_xnor_saveexec_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x2b,0xe8,0xbe]
+// GFX12: s_xnor_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x2b,0xe8,0xbe]
 
 s_xnor_saveexec_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x2b,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b64 s[0:1], s[102:103]  ; encoding: [0x66,0x2b,0x80,0xbe]
 
 s_xnor_saveexec_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x2b,0xe8,0xbe]
+// GFX12: s_xnor_saveexec_b64 s[104:105], s[2:3]  ; encoding: [0x02,0x2b,0xe8,0xbe]
 
 s_xnor_saveexec_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x2b,0xea,0xbe]
+// GFX12: s_xnor_saveexec_b64 vcc, s[2:3]         ; encoding: [0x02,0x2b,0xea,0xbe]
 
 s_xnor_saveexec_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x2b,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b64 s[0:1], exec        ; encoding: [0x7e,0x2b,0x80,0xbe]
 
 s_xnor_saveexec_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x2b,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b64 s[0:1], vcc         ; encoding: [0x6a,0x2b,0x80,0xbe]
 
 s_xnor_saveexec_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x2b,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b64 s[0:1], 0           ; encoding: [0x80,0x2b,0x80,0xbe]
 
 s_xnor_saveexec_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x2b,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b64 s[0:1], -1          ; encoding: [0xc1,0x2b,0x80,0xbe]
 
 s_xnor_saveexec_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x2b,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b64 s[0:1], 0.5         ; encoding: [0xf0,0x2b,0x80,0xbe]
 
 s_xnor_saveexec_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x2b,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b64 s[0:1], -4.0        ; encoding: [0xf7,0x2b,0x80,0xbe]
 
 s_xnor_saveexec_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x2b,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_xnor_saveexec_b64 s[0:1], 0x3f717273  ; encoding: [0xff,0x2b,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_xnor_saveexec_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x2b,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_xnor_saveexec_b64 s[0:1], 0xaf123456  ; encoding: [0xff,0x2b,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_quadmask_b32 s0, s1
-// GFX12: encoding: [0x01,0x1a,0x80,0xbe]
+// GFX12: s_quadmask_b32 s0, s1                   ; encoding: [0x01,0x1a,0x80,0xbe]
 
 s_quadmask_b32 s105, s104
-// GFX12: encoding: [0x68,0x1a,0xe9,0xbe]
+// GFX12: s_quadmask_b32 s105, s104               ; encoding: [0x68,0x1a,0xe9,0xbe]
 
 s_quadmask_b32 s0, s104
-// GFX12: encoding: [0x68,0x1a,0x80,0xbe]
+// GFX12: s_quadmask_b32 s0, s104                 ; encoding: [0x68,0x1a,0x80,0xbe]
 
 s_quadmask_b32 s105, s1
-// GFX12: encoding: [0x01,0x1a,0xe9,0xbe]
+// GFX12: s_quadmask_b32 s105, s1                 ; encoding: [0x01,0x1a,0xe9,0xbe]
 
 s_quadmask_b32 exec_lo, s1
-// GFX12: encoding: [0x01,0x1a,0xfe,0xbe]
+// GFX12: s_quadmask_b32 exec_lo, s1              ; encoding: [0x01,0x1a,0xfe,0xbe]
 
 s_quadmask_b32 exec_hi, s1
-// GFX12: encoding: [0x01,0x1a,0xff,0xbe]
+// GFX12: s_quadmask_b32 exec_hi, s1              ; encoding: [0x01,0x1a,0xff,0xbe]
 
 s_quadmask_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x1a,0xea,0xbe]
+// GFX12: s_quadmask_b32 vcc_lo, s1               ; encoding: [0x01,0x1a,0xea,0xbe]
 
 s_quadmask_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x1a,0xeb,0xbe]
+// GFX12: s_quadmask_b32 vcc_hi, s1               ; encoding: [0x01,0x1a,0xeb,0xbe]
 
 s_quadmask_b32 m0, s1
-// GFX12: encoding: [0x01,0x1a,0xfd,0xbe]
+// GFX12: s_quadmask_b32 m0, s1                   ; encoding: [0x01,0x1a,0xfd,0xbe]
 
 s_quadmask_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x1a,0x80,0xbe]
+// GFX12: s_quadmask_b32 s0, exec_lo              ; encoding: [0x7e,0x1a,0x80,0xbe]
 
 s_quadmask_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x1a,0x80,0xbe]
+// GFX12: s_quadmask_b32 s0, exec_hi              ; encoding: [0x7f,0x1a,0x80,0xbe]
 
 s_quadmask_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x1a,0x80,0xbe]
+// GFX12: s_quadmask_b32 s0, vcc_lo               ; encoding: [0x6a,0x1a,0x80,0xbe]
 
 s_quadmask_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x1a,0x80,0xbe]
+// GFX12: s_quadmask_b32 s0, vcc_hi               ; encoding: [0x6b,0x1a,0x80,0xbe]
 
 s_quadmask_b32 s0, m0
-// GFX12: encoding: [0x7d,0x1a,0x80,0xbe]
+// GFX12: s_quadmask_b32 s0, m0                   ; encoding: [0x7d,0x1a,0x80,0xbe]
 
 s_quadmask_b32 s0, 0
-// GFX12: encoding: [0x80,0x1a,0x80,0xbe]
+// GFX12: s_quadmask_b32 s0, 0                    ; encoding: [0x80,0x1a,0x80,0xbe]
 
 s_quadmask_b32 s0, -1
-// GFX12: encoding: [0xc1,0x1a,0x80,0xbe]
+// GFX12: s_quadmask_b32 s0, -1                   ; encoding: [0xc1,0x1a,0x80,0xbe]
 
 s_quadmask_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x1a,0x80,0xbe]
+// GFX12: s_quadmask_b32 s0, 0.5                  ; encoding: [0xf0,0x1a,0x80,0xbe]
 
 s_quadmask_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x1a,0x80,0xbe]
+// GFX12: s_quadmask_b32 s0, -4.0                 ; encoding: [0xf7,0x1a,0x80,0xbe]
 
 s_quadmask_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x1a,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_quadmask_b32 s0, 0x3f717273           ; encoding: [0xff,0x1a,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_quadmask_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x1a,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_quadmask_b32 s0, 0xaf123456           ; encoding: [0xff,0x1a,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_quadmask_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x1b,0x80,0xbe]
+// GFX12: s_quadmask_b64 s[0:1], s[2:3]           ; encoding: [0x02,0x1b,0x80,0xbe]
 
 s_quadmask_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x1b,0xe8,0xbe]
+// GFX12: s_quadmask_b64 s[104:105], s[102:103]   ; encoding: [0x66,0x1b,0xe8,0xbe]
 
 s_quadmask_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x1b,0x80,0xbe]
+// GFX12: s_quadmask_b64 s[0:1], s[102:103]       ; encoding: [0x66,0x1b,0x80,0xbe]
 
 s_quadmask_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x1b,0xe8,0xbe]
+// GFX12: s_quadmask_b64 s[104:105], s[2:3]       ; encoding: [0x02,0x1b,0xe8,0xbe]
 
 s_quadmask_b64 exec, s[2:3]
-// GFX12: encoding: [0x02,0x1b,0xfe,0xbe]
+// GFX12: s_quadmask_b64 exec, s[2:3]             ; encoding: [0x02,0x1b,0xfe,0xbe]
 
 s_quadmask_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x1b,0xea,0xbe]
+// GFX12: s_quadmask_b64 vcc, s[2:3]              ; encoding: [0x02,0x1b,0xea,0xbe]
 
 s_quadmask_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x1b,0x80,0xbe]
+// GFX12: s_quadmask_b64 s[0:1], exec             ; encoding: [0x7e,0x1b,0x80,0xbe]
 
 s_quadmask_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x1b,0x80,0xbe]
+// GFX12: s_quadmask_b64 s[0:1], vcc              ; encoding: [0x6a,0x1b,0x80,0xbe]
 
 s_quadmask_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x1b,0x80,0xbe]
+// GFX12: s_quadmask_b64 s[0:1], 0                ; encoding: [0x80,0x1b,0x80,0xbe]
 
 s_quadmask_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x1b,0x80,0xbe]
+// GFX12: s_quadmask_b64 s[0:1], -1               ; encoding: [0xc1,0x1b,0x80,0xbe]
 
 s_quadmask_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x1b,0x80,0xbe]
+// GFX12: s_quadmask_b64 s[0:1], 0.5              ; encoding: [0xf0,0x1b,0x80,0xbe]
 
 s_quadmask_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x1b,0x80,0xbe]
+// GFX12: s_quadmask_b64 s[0:1], -4.0             ; encoding: [0xf7,0x1b,0x80,0xbe]
 
 s_quadmask_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x1b,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_quadmask_b64 s[0:1], 0x3f717273       ; encoding: [0xff,0x1b,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_quadmask_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x1b,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_quadmask_b64 s[0:1], 0xaf123456       ; encoding: [0xff,0x1b,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_movrels_b32 s0, s1
-// GFX12: encoding: [0x01,0x40,0x80,0xbe]
+// GFX12: s_movrels_b32 s0, s1                    ; encoding: [0x01,0x40,0x80,0xbe]
 
 s_movrels_b32 s105, s104
-// GFX12: encoding: [0x68,0x40,0xe9,0xbe]
+// GFX12: s_movrels_b32 s105, s104                ; encoding: [0x68,0x40,0xe9,0xbe]
 
 s_movrels_b32 s0, s104
-// GFX12: encoding: [0x68,0x40,0x80,0xbe]
+// GFX12: s_movrels_b32 s0, s104                  ; encoding: [0x68,0x40,0x80,0xbe]
 
 s_movrels_b32 s105, s1
-// GFX12: encoding: [0x01,0x40,0xe9,0xbe]
+// GFX12: s_movrels_b32 s105, s1                  ; encoding: [0x01,0x40,0xe9,0xbe]
 
 s_movrels_b32 exec_lo, s1
-// GFX12: encoding: [0x01,0x40,0xfe,0xbe]
+// GFX12: s_movrels_b32 exec_lo, s1               ; encoding: [0x01,0x40,0xfe,0xbe]
 
 s_movrels_b32 exec_hi, s1
-// GFX12: encoding: [0x01,0x40,0xff,0xbe]
+// GFX12: s_movrels_b32 exec_hi, s1               ; encoding: [0x01,0x40,0xff,0xbe]
 
 s_movrels_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x40,0xea,0xbe]
+// GFX12: s_movrels_b32 vcc_lo, s1                ; encoding: [0x01,0x40,0xea,0xbe]
 
 s_movrels_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x40,0xeb,0xbe]
+// GFX12: s_movrels_b32 vcc_hi, s1                ; encoding: [0x01,0x40,0xeb,0xbe]
 
 s_movrels_b32 m0, s1
-// GFX12: encoding: [0x01,0x40,0xfd,0xbe]
+// GFX12: s_movrels_b32 m0, s1                    ; encoding: [0x01,0x40,0xfd,0xbe]
 
 s_movrels_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x40,0x80,0xbe]
+// GFX12: s_movrels_b32 s0, vcc_lo                ; encoding: [0x6a,0x40,0x80,0xbe]
 
 s_movrels_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x40,0x80,0xbe]
+// GFX12: s_movrels_b32 s0, vcc_hi                ; encoding: [0x6b,0x40,0x80,0xbe]
 
 s_movrels_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x41,0x80,0xbe]
+// GFX12: s_movrels_b64 s[0:1], s[2:3]            ; encoding: [0x02,0x41,0x80,0xbe]
 
 s_movrels_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x41,0xe8,0xbe]
+// GFX12: s_movrels_b64 s[104:105], s[102:103]    ; encoding: [0x66,0x41,0xe8,0xbe]
 
 s_movrels_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x41,0x80,0xbe]
+// GFX12: s_movrels_b64 s[0:1], s[102:103]        ; encoding: [0x66,0x41,0x80,0xbe]
 
 s_movrels_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x41,0xe8,0xbe]
+// GFX12: s_movrels_b64 s[104:105], s[2:3]        ; encoding: [0x02,0x41,0xe8,0xbe]
 
 s_movrels_b64 exec, s[2:3]
-// GFX12: encoding: [0x02,0x41,0xfe,0xbe]
+// GFX12: s_movrels_b64 exec, s[2:3]              ; encoding: [0x02,0x41,0xfe,0xbe]
 
 s_movrels_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x41,0xea,0xbe]
+// GFX12: s_movrels_b64 vcc, s[2:3]               ; encoding: [0x02,0x41,0xea,0xbe]
 
 s_movrels_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x41,0x80,0xbe]
+// GFX12: s_movrels_b64 s[0:1], vcc               ; encoding: [0x6a,0x41,0x80,0xbe]
 
 s_movreld_b32 s0, s1
-// GFX12: encoding: [0x01,0x42,0x80,0xbe]
+// GFX12: s_movreld_b32 s0, s1                    ; encoding: [0x01,0x42,0x80,0xbe]
 
 s_movreld_b32 s105, s104
-// GFX12: encoding: [0x68,0x42,0xe9,0xbe]
+// GFX12: s_movreld_b32 s105, s104                ; encoding: [0x68,0x42,0xe9,0xbe]
 
 s_movreld_b32 s0, s104
-// GFX12: encoding: [0x68,0x42,0x80,0xbe]
+// GFX12: s_movreld_b32 s0, s104                  ; encoding: [0x68,0x42,0x80,0xbe]
 
 s_movreld_b32 s105, s1
-// GFX12: encoding: [0x01,0x42,0xe9,0xbe]
+// GFX12: s_movreld_b32 s105, s1                  ; encoding: [0x01,0x42,0xe9,0xbe]
 
 s_movreld_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x42,0xea,0xbe]
+// GFX12: s_movreld_b32 vcc_lo, s1                ; encoding: [0x01,0x42,0xea,0xbe]
 
 s_movreld_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x42,0xeb,0xbe]
+// GFX12: s_movreld_b32 vcc_hi, s1                ; encoding: [0x01,0x42,0xeb,0xbe]
 
 s_movreld_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x42,0x80,0xbe]
+// GFX12: s_movreld_b32 s0, exec_lo               ; encoding: [0x7e,0x42,0x80,0xbe]
 
 s_movreld_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x42,0x80,0xbe]
+// GFX12: s_movreld_b32 s0, exec_hi               ; encoding: [0x7f,0x42,0x80,0xbe]
 
 s_movreld_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x42,0x80,0xbe]
+// GFX12: s_movreld_b32 s0, vcc_lo                ; encoding: [0x6a,0x42,0x80,0xbe]
 
 s_movreld_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x42,0x80,0xbe]
+// GFX12: s_movreld_b32 s0, vcc_hi                ; encoding: [0x6b,0x42,0x80,0xbe]
 
 s_movreld_b32 s0, m0
-// GFX12: encoding: [0x7d,0x42,0x80,0xbe]
+// GFX12: s_movreld_b32 s0, m0                    ; encoding: [0x7d,0x42,0x80,0xbe]
 
 s_movreld_b32 s0, 0
-// GFX12: encoding: [0x80,0x42,0x80,0xbe]
+// GFX12: s_movreld_b32 s0, 0                     ; encoding: [0x80,0x42,0x80,0xbe]
 
 s_movreld_b32 s0, -1
-// GFX12: encoding: [0xc1,0x42,0x80,0xbe]
+// GFX12: s_movreld_b32 s0, -1                    ; encoding: [0xc1,0x42,0x80,0xbe]
 
 s_movreld_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x42,0x80,0xbe]
+// GFX12: s_movreld_b32 s0, 0.5                   ; encoding: [0xf0,0x42,0x80,0xbe]
 
 s_movreld_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x42,0x80,0xbe]
+// GFX12: s_movreld_b32 s0, -4.0                  ; encoding: [0xf7,0x42,0x80,0xbe]
 
 s_movreld_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x42,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_movreld_b32 s0, 0x3f717273            ; encoding: [0xff,0x42,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_movreld_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x42,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_movreld_b32 s0, 0xaf123456            ; encoding: [0xff,0x42,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_movreld_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x43,0x80,0xbe]
+// GFX12: s_movreld_b64 s[0:1], s[2:3]            ; encoding: [0x02,0x43,0x80,0xbe]
 
 s_movreld_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x43,0xe8,0xbe]
+// GFX12: s_movreld_b64 s[104:105], s[102:103]    ; encoding: [0x66,0x43,0xe8,0xbe]
 
 s_movreld_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x43,0x80,0xbe]
+// GFX12: s_movreld_b64 s[0:1], s[102:103]        ; encoding: [0x66,0x43,0x80,0xbe]
 
 s_movreld_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x43,0xe8,0xbe]
+// GFX12: s_movreld_b64 s[104:105], s[2:3]        ; encoding: [0x02,0x43,0xe8,0xbe]
 
 s_movreld_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x43,0xea,0xbe]
+// GFX12: s_movreld_b64 vcc, s[2:3]               ; encoding: [0x02,0x43,0xea,0xbe]
 
 s_movreld_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x43,0x80,0xbe]
+// GFX12: s_movreld_b64 s[0:1], exec              ; encoding: [0x7e,0x43,0x80,0xbe]
 
 s_movreld_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x43,0x80,0xbe]
+// GFX12: s_movreld_b64 s[0:1], vcc               ; encoding: [0x6a,0x43,0x80,0xbe]
 
 s_movreld_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x43,0x80,0xbe]
+// GFX12: s_movreld_b64 s[0:1], 0                 ; encoding: [0x80,0x43,0x80,0xbe]
 
 s_movreld_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x43,0x80,0xbe]
+// GFX12: s_movreld_b64 s[0:1], -1                ; encoding: [0xc1,0x43,0x80,0xbe]
 
 s_movreld_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x43,0x80,0xbe]
+// GFX12: s_movreld_b64 s[0:1], 0.5               ; encoding: [0xf0,0x43,0x80,0xbe]
 
 s_movreld_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x43,0x80,0xbe]
+// GFX12: s_movreld_b64 s[0:1], -4.0              ; encoding: [0xf7,0x43,0x80,0xbe]
 
 s_movreld_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x43,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_movreld_b64 s[0:1], 0x3f717273        ; encoding: [0xff,0x43,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_movreld_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x43,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_movreld_b64 s[0:1], 0xaf123456        ; encoding: [0xff,0x43,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_abs_i32 s0, s1
-// GFX12: encoding: [0x01,0x15,0x80,0xbe]
+// GFX12: s_abs_i32 s0, s1                        ; encoding: [0x01,0x15,0x80,0xbe]
 
 s_abs_i32 s105, s104
-// GFX12: encoding: [0x68,0x15,0xe9,0xbe]
+// GFX12: s_abs_i32 s105, s104                    ; encoding: [0x68,0x15,0xe9,0xbe]
 
 s_abs_i32 s0, s104
-// GFX12: encoding: [0x68,0x15,0x80,0xbe]
+// GFX12: s_abs_i32 s0, s104                      ; encoding: [0x68,0x15,0x80,0xbe]
 
 s_abs_i32 s105, s1
-// GFX12: encoding: [0x01,0x15,0xe9,0xbe]
+// GFX12: s_abs_i32 s105, s1                      ; encoding: [0x01,0x15,0xe9,0xbe]
 
 s_abs_i32 exec_lo, s1
-// GFX12: encoding: [0x01,0x15,0xfe,0xbe]
+// GFX12: s_abs_i32 exec_lo, s1                   ; encoding: [0x01,0x15,0xfe,0xbe]
 
 s_abs_i32 exec_hi, s1
-// GFX12: encoding: [0x01,0x15,0xff,0xbe]
+// GFX12: s_abs_i32 exec_hi, s1                   ; encoding: [0x01,0x15,0xff,0xbe]
 
 s_abs_i32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x15,0xea,0xbe]
+// GFX12: s_abs_i32 vcc_lo, s1                    ; encoding: [0x01,0x15,0xea,0xbe]
 
 s_abs_i32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x15,0xeb,0xbe]
+// GFX12: s_abs_i32 vcc_hi, s1                    ; encoding: [0x01,0x15,0xeb,0xbe]
 
 s_abs_i32 m0, s1
-// GFX12: encoding: [0x01,0x15,0xfd,0xbe]
+// GFX12: s_abs_i32 m0, s1                        ; encoding: [0x01,0x15,0xfd,0xbe]
 
 s_abs_i32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x15,0x80,0xbe]
+// GFX12: s_abs_i32 s0, exec_lo                   ; encoding: [0x7e,0x15,0x80,0xbe]
 
 s_abs_i32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x15,0x80,0xbe]
+// GFX12: s_abs_i32 s0, exec_hi                   ; encoding: [0x7f,0x15,0x80,0xbe]
 
 s_abs_i32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x15,0x80,0xbe]
+// GFX12: s_abs_i32 s0, vcc_lo                    ; encoding: [0x6a,0x15,0x80,0xbe]
 
 s_abs_i32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x15,0x80,0xbe]
+// GFX12: s_abs_i32 s0, vcc_hi                    ; encoding: [0x6b,0x15,0x80,0xbe]
 
 s_abs_i32 s0, m0
-// GFX12: encoding: [0x7d,0x15,0x80,0xbe]
+// GFX12: s_abs_i32 s0, m0                        ; encoding: [0x7d,0x15,0x80,0xbe]
 
 s_abs_i32 s0, 0
-// GFX12: encoding: [0x80,0x15,0x80,0xbe]
+// GFX12: s_abs_i32 s0, 0                         ; encoding: [0x80,0x15,0x80,0xbe]
 
 s_abs_i32 s0, -1
-// GFX12: encoding: [0xc1,0x15,0x80,0xbe]
+// GFX12: s_abs_i32 s0, -1                        ; encoding: [0xc1,0x15,0x80,0xbe]
 
 s_abs_i32 s0, 0.5
-// GFX12: encoding: [0xf0,0x15,0x80,0xbe]
+// GFX12: s_abs_i32 s0, 0.5                       ; encoding: [0xf0,0x15,0x80,0xbe]
 
 s_abs_i32 s0, -4.0
-// GFX12: encoding: [0xf7,0x15,0x80,0xbe]
+// GFX12: s_abs_i32 s0, -4.0                      ; encoding: [0xf7,0x15,0x80,0xbe]
 
 s_abs_i32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x15,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_abs_i32 s0, 0x3f717273                ; encoding: [0xff,0x15,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_abs_i32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x15,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_abs_i32 s0, 0xaf123456                ; encoding: [0xff,0x15,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_andn1_saveexec_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x2d,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b64 s[0:1], s[2:3]  ; encoding: [0x02,0x2d,0x80,0xbe]
 
 s_andn1_saveexec_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x2d,0xe8,0xbe]
+// GFX12: s_and_not0_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x2d,0xe8,0xbe]
 
 s_andn1_saveexec_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x2d,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b64 s[0:1], s[102:103] ; encoding: [0x66,0x2d,0x80,0xbe]
 
 s_andn1_saveexec_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x2d,0xe8,0xbe]
+// GFX12: s_and_not0_saveexec_b64 s[104:105], s[2:3] ; encoding: [0x02,0x2d,0xe8,0xbe]
 
 s_andn1_saveexec_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x2d,0xea,0xbe]
+// GFX12: s_and_not0_saveexec_b64 vcc, s[2:3]     ; encoding: [0x02,0x2d,0xea,0xbe]
 
 s_andn1_saveexec_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x2d,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b64 s[0:1], exec    ; encoding: [0x7e,0x2d,0x80,0xbe]
 
 s_andn1_saveexec_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x2d,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b64 s[0:1], vcc     ; encoding: [0x6a,0x2d,0x80,0xbe]
 
 s_andn1_saveexec_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x2d,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b64 s[0:1], 0       ; encoding: [0x80,0x2d,0x80,0xbe]
 
 s_andn1_saveexec_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x2d,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b64 s[0:1], -1      ; encoding: [0xc1,0x2d,0x80,0xbe]
 
 s_andn1_saveexec_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x2d,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b64 s[0:1], 0.5     ; encoding: [0xf0,0x2d,0x80,0xbe]
 
 s_andn1_saveexec_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x2d,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b64 s[0:1], -4.0    ; encoding: [0xf7,0x2d,0x80,0xbe]
 
 s_andn1_saveexec_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x2d,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_and_not0_saveexec_b64 s[0:1], 0x3f717273 ; encoding: [0xff,0x2d,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_andn1_saveexec_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x2d,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not0_saveexec_b64 s[0:1], 0xaf123456 ; encoding: [0xff,0x2d,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_orn1_saveexec_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x2f,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b64 s[0:1], s[2:3]   ; encoding: [0x02,0x2f,0x80,0xbe]
 
 s_orn1_saveexec_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x2f,0xe8,0xbe]
+// GFX12: s_or_not0_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x2f,0xe8,0xbe]
 
 s_orn1_saveexec_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x2f,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b64 s[0:1], s[102:103] ; encoding: [0x66,0x2f,0x80,0xbe]
 
 s_orn1_saveexec_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x2f,0xe8,0xbe]
+// GFX12: s_or_not0_saveexec_b64 s[104:105], s[2:3] ; encoding: [0x02,0x2f,0xe8,0xbe]
 
 s_orn1_saveexec_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x2f,0xea,0xbe]
+// GFX12: s_or_not0_saveexec_b64 vcc, s[2:3]      ; encoding: [0x02,0x2f,0xea,0xbe]
 
 s_orn1_saveexec_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x2f,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b64 s[0:1], exec     ; encoding: [0x7e,0x2f,0x80,0xbe]
 
 s_orn1_saveexec_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x2f,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b64 s[0:1], vcc      ; encoding: [0x6a,0x2f,0x80,0xbe]
 
 s_orn1_saveexec_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x2f,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b64 s[0:1], 0        ; encoding: [0x80,0x2f,0x80,0xbe]
 
 s_orn1_saveexec_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x2f,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b64 s[0:1], -1       ; encoding: [0xc1,0x2f,0x80,0xbe]
 
 s_orn1_saveexec_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x2f,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b64 s[0:1], 0.5      ; encoding: [0xf0,0x2f,0x80,0xbe]
 
 s_orn1_saveexec_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x2f,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b64 s[0:1], -4.0     ; encoding: [0xf7,0x2f,0x80,0xbe]
 
 s_orn1_saveexec_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x2f,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_or_not0_saveexec_b64 s[0:1], 0x3f717273 ; encoding: [0xff,0x2f,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_orn1_saveexec_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x2f,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_not0_saveexec_b64 s[0:1], 0xaf123456 ; encoding: [0xff,0x2f,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_andn1_wrexec_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x35,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b64 s[0:1], s[2:3]    ; encoding: [0x02,0x35,0x80,0xbe]
 
 s_andn1_wrexec_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x35,0xe8,0xbe]
+// GFX12: s_and_not0_wrexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x35,0xe8,0xbe]
 
 s_andn1_wrexec_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x35,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b64 s[0:1], s[102:103] ; encoding: [0x66,0x35,0x80,0xbe]
 
 s_andn1_wrexec_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x35,0xe8,0xbe]
+// GFX12: s_and_not0_wrexec_b64 s[104:105], s[2:3] ; encoding: [0x02,0x35,0xe8,0xbe]
 
 s_andn1_wrexec_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x35,0xea,0xbe]
+// GFX12: s_and_not0_wrexec_b64 vcc, s[2:3]       ; encoding: [0x02,0x35,0xea,0xbe]
 
 s_andn1_wrexec_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x35,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b64 s[0:1], exec      ; encoding: [0x7e,0x35,0x80,0xbe]
 
 s_andn1_wrexec_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x35,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b64 s[0:1], vcc       ; encoding: [0x6a,0x35,0x80,0xbe]
 
 s_andn1_wrexec_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x35,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b64 s[0:1], 0         ; encoding: [0x80,0x35,0x80,0xbe]
 
 s_andn1_wrexec_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x35,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b64 s[0:1], -1        ; encoding: [0xc1,0x35,0x80,0xbe]
 
 s_andn1_wrexec_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x35,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b64 s[0:1], 0.5       ; encoding: [0xf0,0x35,0x80,0xbe]
 
 s_andn1_wrexec_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x35,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b64 s[0:1], -4.0      ; encoding: [0xf7,0x35,0x80,0xbe]
 
 s_andn1_wrexec_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x35,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_and_not0_wrexec_b64 s[0:1], 0x3f717273 ; encoding: [0xff,0x35,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_andn1_wrexec_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x35,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not0_wrexec_b64 s[0:1], 0xaf123456 ; encoding: [0xff,0x35,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_andn2_wrexec_b64 s[0:1], s[2:3]
-// GFX12: encoding: [0x02,0x37,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b64 s[0:1], s[2:3]    ; encoding: [0x02,0x37,0x80,0xbe]
 
 s_andn2_wrexec_b64 s[104:105], s[102:103]
-// GFX12: encoding: [0x66,0x37,0xe8,0xbe]
+// GFX12: s_and_not1_wrexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x37,0xe8,0xbe]
 
 s_andn2_wrexec_b64 s[0:1], s[102:103]
-// GFX12: encoding: [0x66,0x37,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b64 s[0:1], s[102:103] ; encoding: [0x66,0x37,0x80,0xbe]
 
 s_andn2_wrexec_b64 s[104:105], s[2:3]
-// GFX12: encoding: [0x02,0x37,0xe8,0xbe]
+// GFX12: s_and_not1_wrexec_b64 s[104:105], s[2:3] ; encoding: [0x02,0x37,0xe8,0xbe]
 
 s_andn2_wrexec_b64 vcc, s[2:3]
-// GFX12: encoding: [0x02,0x37,0xea,0xbe]
+// GFX12: s_and_not1_wrexec_b64 vcc, s[2:3]       ; encoding: [0x02,0x37,0xea,0xbe]
 
 s_andn2_wrexec_b64 s[0:1], exec
-// GFX12: encoding: [0x7e,0x37,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b64 s[0:1], exec      ; encoding: [0x7e,0x37,0x80,0xbe]
 
 s_andn2_wrexec_b64 s[0:1], vcc
-// GFX12: encoding: [0x6a,0x37,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b64 s[0:1], vcc       ; encoding: [0x6a,0x37,0x80,0xbe]
 
 s_andn2_wrexec_b64 s[0:1], 0
-// GFX12: encoding: [0x80,0x37,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b64 s[0:1], 0         ; encoding: [0x80,0x37,0x80,0xbe]
 
 s_andn2_wrexec_b64 s[0:1], -1
-// GFX12: encoding: [0xc1,0x37,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b64 s[0:1], -1        ; encoding: [0xc1,0x37,0x80,0xbe]
 
 s_andn2_wrexec_b64 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x37,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b64 s[0:1], 0.5       ; encoding: [0xf0,0x37,0x80,0xbe]
 
 s_andn2_wrexec_b64 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x37,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b64 s[0:1], -4.0      ; encoding: [0xf7,0x37,0x80,0xbe]
 
 s_andn2_wrexec_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x37,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_and_not1_wrexec_b64 s[0:1], 0x3f717273 ; encoding: [0xff,0x37,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_andn2_wrexec_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x37,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not1_wrexec_b64 s[0:1], 0xaf123456 ; encoding: [0xff,0x37,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_bitreplicate_b64_b32 s[0:1], s2
-// GFX12: encoding: [0x02,0x14,0x80,0xbe]
+// GFX12: s_bitreplicate_b64_b32 s[0:1], s2       ; encoding: [0x02,0x14,0x80,0xbe]
 
 s_bitreplicate_b64_b32 s[104:105], s102
-// GFX12: encoding: [0x66,0x14,0xe8,0xbe]
+// GFX12: s_bitreplicate_b64_b32 s[104:105], s102 ; encoding: [0x66,0x14,0xe8,0xbe]
 
 s_bitreplicate_b64_b32 s[0:1], s102
-// GFX12: encoding: [0x66,0x14,0x80,0xbe]
+// GFX12: s_bitreplicate_b64_b32 s[0:1], s102     ; encoding: [0x66,0x14,0x80,0xbe]
 
 s_bitreplicate_b64_b32 s[104:105], s2
-// GFX12: encoding: [0x02,0x14,0xe8,0xbe]
+// GFX12: s_bitreplicate_b64_b32 s[104:105], s2   ; encoding: [0x02,0x14,0xe8,0xbe]
 
 s_bitreplicate_b64_b32 exec, s2
-// GFX12: encoding: [0x02,0x14,0xfe,0xbe]
+// GFX12: s_bitreplicate_b64_b32 exec, s2         ; encoding: [0x02,0x14,0xfe,0xbe]
 
 s_bitreplicate_b64_b32 vcc, s2
-// GFX12: encoding: [0x02,0x14,0xea,0xbe]
+// GFX12: s_bitreplicate_b64_b32 vcc, s2          ; encoding: [0x02,0x14,0xea,0xbe]
 
 s_bitreplicate_b64_b32 s[0:1], exec_lo
-// GFX12: encoding: [0x7e,0x14,0x80,0xbe]
+// GFX12: s_bitreplicate_b64_b32 s[0:1], exec_lo  ; encoding: [0x7e,0x14,0x80,0xbe]
 
 s_bitreplicate_b64_b32 s[0:1], exec_hi
-// GFX12: encoding: [0x7f,0x14,0x80,0xbe]
+// GFX12: s_bitreplicate_b64_b32 s[0:1], exec_hi  ; encoding: [0x7f,0x14,0x80,0xbe]
 
 s_bitreplicate_b64_b32 s[0:1], vcc_lo
-// GFX12: encoding: [0x6a,0x14,0x80,0xbe]
+// GFX12: s_bitreplicate_b64_b32 s[0:1], vcc_lo   ; encoding: [0x6a,0x14,0x80,0xbe]
 
 s_bitreplicate_b64_b32 s[0:1], vcc_hi
-// GFX12: encoding: [0x6b,0x14,0x80,0xbe]
+// GFX12: s_bitreplicate_b64_b32 s[0:1], vcc_hi   ; encoding: [0x6b,0x14,0x80,0xbe]
 
 s_bitreplicate_b64_b32 s[0:1], m0
-// GFX12: encoding: [0x7d,0x14,0x80,0xbe]
+// GFX12: s_bitreplicate_b64_b32 s[0:1], m0       ; encoding: [0x7d,0x14,0x80,0xbe]
 
 s_bitreplicate_b64_b32 s[0:1], 0
-// GFX12: encoding: [0x80,0x14,0x80,0xbe]
+// GFX12: s_bitreplicate_b64_b32 s[0:1], 0        ; encoding: [0x80,0x14,0x80,0xbe]
 
 s_bitreplicate_b64_b32 s[0:1], -1
-// GFX12: encoding: [0xc1,0x14,0x80,0xbe]
+// GFX12: s_bitreplicate_b64_b32 s[0:1], -1       ; encoding: [0xc1,0x14,0x80,0xbe]
 
 s_bitreplicate_b64_b32 s[0:1], 0.5
-// GFX12: encoding: [0xf0,0x14,0x80,0xbe]
+// GFX12: s_bitreplicate_b64_b32 s[0:1], 0.5      ; encoding: [0xf0,0x14,0x80,0xbe]
 
 s_bitreplicate_b64_b32 s[0:1], -4.0
-// GFX12: encoding: [0xf7,0x14,0x80,0xbe]
+// GFX12: s_bitreplicate_b64_b32 s[0:1], -4.0     ; encoding: [0xf7,0x14,0x80,0xbe]
 
 s_bitreplicate_b64_b32 s[0:1], 0x3f717273
-// GFX12: encoding: [0xff,0x14,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_bitreplicate_b64_b32 s[0:1], 0x3f717273 ; encoding: [0xff,0x14,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_bitreplicate_b64_b32 s[0:1], 0xaf123456
-// GFX12: encoding: [0xff,0x14,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_bitreplicate_b64_b32 s[0:1], 0xaf123456 ; encoding: [0xff,0x14,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_and_saveexec_b32 s0, s1
-// GFX12: encoding: [0x01,0x20,0x80,0xbe]
+// GFX12: s_and_saveexec_b32 s0, s1               ; encoding: [0x01,0x20,0x80,0xbe]
 
 s_and_saveexec_b32 s105, s104
-// GFX12: encoding: [0x68,0x20,0xe9,0xbe]
+// GFX12: s_and_saveexec_b32 s105, s104           ; encoding: [0x68,0x20,0xe9,0xbe]
 
 s_and_saveexec_b32 s0, s104
-// GFX12: encoding: [0x68,0x20,0x80,0xbe]
+// GFX12: s_and_saveexec_b32 s0, s104             ; encoding: [0x68,0x20,0x80,0xbe]
 
 s_and_saveexec_b32 s105, s1
-// GFX12: encoding: [0x01,0x20,0xe9,0xbe]
+// GFX12: s_and_saveexec_b32 s105, s1             ; encoding: [0x01,0x20,0xe9,0xbe]
 
 s_and_saveexec_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x20,0xea,0xbe]
+// GFX12: s_and_saveexec_b32 vcc_lo, s1           ; encoding: [0x01,0x20,0xea,0xbe]
 
 s_and_saveexec_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x20,0xeb,0xbe]
+// GFX12: s_and_saveexec_b32 vcc_hi, s1           ; encoding: [0x01,0x20,0xeb,0xbe]
 
 s_and_saveexec_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x20,0x80,0xbe]
+// GFX12: s_and_saveexec_b32 s0, exec_lo          ; encoding: [0x7e,0x20,0x80,0xbe]
 
 s_and_saveexec_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x20,0x80,0xbe]
+// GFX12: s_and_saveexec_b32 s0, exec_hi          ; encoding: [0x7f,0x20,0x80,0xbe]
 
 s_and_saveexec_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x20,0x80,0xbe]
+// GFX12: s_and_saveexec_b32 s0, vcc_lo           ; encoding: [0x6a,0x20,0x80,0xbe]
 
 s_and_saveexec_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x20,0x80,0xbe]
+// GFX12: s_and_saveexec_b32 s0, vcc_hi           ; encoding: [0x6b,0x20,0x80,0xbe]
 
 s_and_saveexec_b32 s0, m0
-// GFX12: encoding: [0x7d,0x20,0x80,0xbe]
+// GFX12: s_and_saveexec_b32 s0, m0               ; encoding: [0x7d,0x20,0x80,0xbe]
 
 s_and_saveexec_b32 s0, 0
-// GFX12: encoding: [0x80,0x20,0x80,0xbe]
+// GFX12: s_and_saveexec_b32 s0, 0                ; encoding: [0x80,0x20,0x80,0xbe]
 
 s_and_saveexec_b32 s0, -1
-// GFX12: encoding: [0xc1,0x20,0x80,0xbe]
+// GFX12: s_and_saveexec_b32 s0, -1               ; encoding: [0xc1,0x20,0x80,0xbe]
 
 s_and_saveexec_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x20,0x80,0xbe]
+// GFX12: s_and_saveexec_b32 s0, 0.5              ; encoding: [0xf0,0x20,0x80,0xbe]
 
 s_and_saveexec_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x20,0x80,0xbe]
+// GFX12: s_and_saveexec_b32 s0, -4.0             ; encoding: [0xf7,0x20,0x80,0xbe]
 
 s_and_saveexec_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x20,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_and_saveexec_b32 s0, 0x3f717273       ; encoding: [0xff,0x20,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_and_saveexec_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x20,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_saveexec_b32 s0, 0xaf123456       ; encoding: [0xff,0x20,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_or_saveexec_b32 s0, s1
-// GFX12: encoding: [0x01,0x22,0x80,0xbe]
+// GFX12: s_or_saveexec_b32 s0, s1                ; encoding: [0x01,0x22,0x80,0xbe]
 
 s_or_saveexec_b32 s105, s104
-// GFX12: encoding: [0x68,0x22,0xe9,0xbe]
+// GFX12: s_or_saveexec_b32 s105, s104            ; encoding: [0x68,0x22,0xe9,0xbe]
 
 s_or_saveexec_b32 s0, s104
-// GFX12: encoding: [0x68,0x22,0x80,0xbe]
+// GFX12: s_or_saveexec_b32 s0, s104              ; encoding: [0x68,0x22,0x80,0xbe]
 
 s_or_saveexec_b32 s105, s1
-// GFX12: encoding: [0x01,0x22,0xe9,0xbe]
+// GFX12: s_or_saveexec_b32 s105, s1              ; encoding: [0x01,0x22,0xe9,0xbe]
 
 s_or_saveexec_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x22,0xea,0xbe]
+// GFX12: s_or_saveexec_b32 vcc_lo, s1            ; encoding: [0x01,0x22,0xea,0xbe]
 
 s_or_saveexec_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x22,0xeb,0xbe]
+// GFX12: s_or_saveexec_b32 vcc_hi, s1            ; encoding: [0x01,0x22,0xeb,0xbe]
 
 s_or_saveexec_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x22,0x80,0xbe]
+// GFX12: s_or_saveexec_b32 s0, exec_lo           ; encoding: [0x7e,0x22,0x80,0xbe]
 
 s_or_saveexec_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x22,0x80,0xbe]
+// GFX12: s_or_saveexec_b32 s0, exec_hi           ; encoding: [0x7f,0x22,0x80,0xbe]
 
 s_or_saveexec_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x22,0x80,0xbe]
+// GFX12: s_or_saveexec_b32 s0, vcc_lo            ; encoding: [0x6a,0x22,0x80,0xbe]
 
 s_or_saveexec_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x22,0x80,0xbe]
+// GFX12: s_or_saveexec_b32 s0, vcc_hi            ; encoding: [0x6b,0x22,0x80,0xbe]
 
 s_or_saveexec_b32 s0, m0
-// GFX12: encoding: [0x7d,0x22,0x80,0xbe]
+// GFX12: s_or_saveexec_b32 s0, m0                ; encoding: [0x7d,0x22,0x80,0xbe]
 
 s_or_saveexec_b32 s0, 0
-// GFX12: encoding: [0x80,0x22,0x80,0xbe]
+// GFX12: s_or_saveexec_b32 s0, 0                 ; encoding: [0x80,0x22,0x80,0xbe]
 
 s_or_saveexec_b32 s0, -1
-// GFX12: encoding: [0xc1,0x22,0x80,0xbe]
+// GFX12: s_or_saveexec_b32 s0, -1                ; encoding: [0xc1,0x22,0x80,0xbe]
 
 s_or_saveexec_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x22,0x80,0xbe]
+// GFX12: s_or_saveexec_b32 s0, 0.5               ; encoding: [0xf0,0x22,0x80,0xbe]
 
 s_or_saveexec_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x22,0x80,0xbe]
+// GFX12: s_or_saveexec_b32 s0, -4.0              ; encoding: [0xf7,0x22,0x80,0xbe]
 
 s_or_saveexec_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x22,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_or_saveexec_b32 s0, 0x3f717273        ; encoding: [0xff,0x22,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_or_saveexec_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x22,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_saveexec_b32 s0, 0xaf123456        ; encoding: [0xff,0x22,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_xor_saveexec_b32 s0, s1
-// GFX12: encoding: [0x01,0x24,0x80,0xbe]
+// GFX12: s_xor_saveexec_b32 s0, s1               ; encoding: [0x01,0x24,0x80,0xbe]
 
 s_xor_saveexec_b32 s105, s104
-// GFX12: encoding: [0x68,0x24,0xe9,0xbe]
+// GFX12: s_xor_saveexec_b32 s105, s104           ; encoding: [0x68,0x24,0xe9,0xbe]
 
 s_xor_saveexec_b32 s0, s104
-// GFX12: encoding: [0x68,0x24,0x80,0xbe]
+// GFX12: s_xor_saveexec_b32 s0, s104             ; encoding: [0x68,0x24,0x80,0xbe]
 
 s_xor_saveexec_b32 s105, s1
-// GFX12: encoding: [0x01,0x24,0xe9,0xbe]
+// GFX12: s_xor_saveexec_b32 s105, s1             ; encoding: [0x01,0x24,0xe9,0xbe]
 
 s_xor_saveexec_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x24,0xea,0xbe]
+// GFX12: s_xor_saveexec_b32 vcc_lo, s1           ; encoding: [0x01,0x24,0xea,0xbe]
 
 s_xor_saveexec_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x24,0xeb,0xbe]
+// GFX12: s_xor_saveexec_b32 vcc_hi, s1           ; encoding: [0x01,0x24,0xeb,0xbe]
 
 s_xor_saveexec_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x24,0x80,0xbe]
+// GFX12: s_xor_saveexec_b32 s0, exec_lo          ; encoding: [0x7e,0x24,0x80,0xbe]
 
 s_xor_saveexec_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x24,0x80,0xbe]
+// GFX12: s_xor_saveexec_b32 s0, exec_hi          ; encoding: [0x7f,0x24,0x80,0xbe]
 
 s_xor_saveexec_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x24,0x80,0xbe]
+// GFX12: s_xor_saveexec_b32 s0, vcc_lo           ; encoding: [0x6a,0x24,0x80,0xbe]
 
 s_xor_saveexec_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x24,0x80,0xbe]
+// GFX12: s_xor_saveexec_b32 s0, vcc_hi           ; encoding: [0x6b,0x24,0x80,0xbe]
 
 s_xor_saveexec_b32 s0, m0
-// GFX12: encoding: [0x7d,0x24,0x80,0xbe]
+// GFX12: s_xor_saveexec_b32 s0, m0               ; encoding: [0x7d,0x24,0x80,0xbe]
 
 s_xor_saveexec_b32 s0, 0
-// GFX12: encoding: [0x80,0x24,0x80,0xbe]
+// GFX12: s_xor_saveexec_b32 s0, 0                ; encoding: [0x80,0x24,0x80,0xbe]
 
 s_xor_saveexec_b32 s0, -1
-// GFX12: encoding: [0xc1,0x24,0x80,0xbe]
+// GFX12: s_xor_saveexec_b32 s0, -1               ; encoding: [0xc1,0x24,0x80,0xbe]
 
 s_xor_saveexec_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x24,0x80,0xbe]
+// GFX12: s_xor_saveexec_b32 s0, 0.5              ; encoding: [0xf0,0x24,0x80,0xbe]
 
 s_xor_saveexec_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x24,0x80,0xbe]
+// GFX12: s_xor_saveexec_b32 s0, -4.0             ; encoding: [0xf7,0x24,0x80,0xbe]
 
 s_xor_saveexec_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x24,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_xor_saveexec_b32 s0, 0x3f717273       ; encoding: [0xff,0x24,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_xor_saveexec_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x24,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_xor_saveexec_b32 s0, 0xaf123456       ; encoding: [0xff,0x24,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_andn2_saveexec_b32 s0, s1
-// GFX12: encoding: [0x01,0x30,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s0, s1          ; encoding: [0x01,0x30,0x80,0xbe]
 
 s_andn2_saveexec_b32 s105, s104
-// GFX12: encoding: [0x68,0x30,0xe9,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s105, s104      ; encoding: [0x68,0x30,0xe9,0xbe]
 
 s_andn2_saveexec_b32 s0, s104
-// GFX12: encoding: [0x68,0x30,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s0, s104        ; encoding: [0x68,0x30,0x80,0xbe]
 
 s_andn2_saveexec_b32 s105, s1
-// GFX12: encoding: [0x01,0x30,0xe9,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s105, s1        ; encoding: [0x01,0x30,0xe9,0xbe]
 
 s_andn2_saveexec_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x30,0xea,0xbe]
+// GFX12: s_and_not1_saveexec_b32 vcc_lo, s1      ; encoding: [0x01,0x30,0xea,0xbe]
 
 s_andn2_saveexec_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x30,0xeb,0xbe]
+// GFX12: s_and_not1_saveexec_b32 vcc_hi, s1      ; encoding: [0x01,0x30,0xeb,0xbe]
 
 s_andn2_saveexec_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x30,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s0, exec_lo     ; encoding: [0x7e,0x30,0x80,0xbe]
 
 s_andn2_saveexec_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x30,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s0, exec_hi     ; encoding: [0x7f,0x30,0x80,0xbe]
 
 s_andn2_saveexec_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x30,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s0, vcc_lo      ; encoding: [0x6a,0x30,0x80,0xbe]
 
 s_andn2_saveexec_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x30,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s0, vcc_hi      ; encoding: [0x6b,0x30,0x80,0xbe]
 
 s_andn2_saveexec_b32 s0, m0
-// GFX12: encoding: [0x7d,0x30,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s0, m0          ; encoding: [0x7d,0x30,0x80,0xbe]
 
 s_andn2_saveexec_b32 s0, 0
-// GFX12: encoding: [0x80,0x30,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s0, 0           ; encoding: [0x80,0x30,0x80,0xbe]
 
 s_andn2_saveexec_b32 s0, -1
-// GFX12: encoding: [0xc1,0x30,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s0, -1          ; encoding: [0xc1,0x30,0x80,0xbe]
 
 s_andn2_saveexec_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x30,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s0, 0.5         ; encoding: [0xf0,0x30,0x80,0xbe]
 
 s_andn2_saveexec_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x30,0x80,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s0, -4.0        ; encoding: [0xf7,0x30,0x80,0xbe]
 
 s_andn2_saveexec_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x30,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_and_not1_saveexec_b32 s0, 0x3f717273  ; encoding: [0xff,0x30,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_andn2_saveexec_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x30,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not1_saveexec_b32 s0, 0xaf123456  ; encoding: [0xff,0x30,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_orn2_saveexec_b32 s0, s1
-// GFX12: encoding: [0x01,0x32,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s0, s1           ; encoding: [0x01,0x32,0x80,0xbe]
 
 s_orn2_saveexec_b32 s105, s104
-// GFX12: encoding: [0x68,0x32,0xe9,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s105, s104       ; encoding: [0x68,0x32,0xe9,0xbe]
 
 s_orn2_saveexec_b32 s0, s104
-// GFX12: encoding: [0x68,0x32,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s0, s104         ; encoding: [0x68,0x32,0x80,0xbe]
 
 s_orn2_saveexec_b32 s105, s1
-// GFX12: encoding: [0x01,0x32,0xe9,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s105, s1         ; encoding: [0x01,0x32,0xe9,0xbe]
 
 s_orn2_saveexec_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x32,0xea,0xbe]
+// GFX12: s_or_not1_saveexec_b32 vcc_lo, s1       ; encoding: [0x01,0x32,0xea,0xbe]
 
 s_orn2_saveexec_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x32,0xeb,0xbe]
+// GFX12: s_or_not1_saveexec_b32 vcc_hi, s1       ; encoding: [0x01,0x32,0xeb,0xbe]
 
 s_orn2_saveexec_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x32,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s0, exec_lo      ; encoding: [0x7e,0x32,0x80,0xbe]
 
 s_orn2_saveexec_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x32,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s0, exec_hi      ; encoding: [0x7f,0x32,0x80,0xbe]
 
 s_orn2_saveexec_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x32,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s0, vcc_lo       ; encoding: [0x6a,0x32,0x80,0xbe]
 
 s_orn2_saveexec_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x32,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s0, vcc_hi       ; encoding: [0x6b,0x32,0x80,0xbe]
 
 s_orn2_saveexec_b32 s0, m0
-// GFX12: encoding: [0x7d,0x32,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s0, m0           ; encoding: [0x7d,0x32,0x80,0xbe]
 
 s_orn2_saveexec_b32 s0, 0
-// GFX12: encoding: [0x80,0x32,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s0, 0            ; encoding: [0x80,0x32,0x80,0xbe]
 
 s_orn2_saveexec_b32 s0, -1
-// GFX12: encoding: [0xc1,0x32,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s0, -1           ; encoding: [0xc1,0x32,0x80,0xbe]
 
 s_orn2_saveexec_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x32,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s0, 0.5          ; encoding: [0xf0,0x32,0x80,0xbe]
 
 s_orn2_saveexec_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x32,0x80,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s0, -4.0         ; encoding: [0xf7,0x32,0x80,0xbe]
 
 s_orn2_saveexec_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x32,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_or_not1_saveexec_b32 s0, 0x3f717273   ; encoding: [0xff,0x32,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_orn2_saveexec_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x32,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_not1_saveexec_b32 s0, 0xaf123456   ; encoding: [0xff,0x32,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_nand_saveexec_b32 s0, s1
-// GFX12: encoding: [0x01,0x26,0x80,0xbe]
+// GFX12: s_nand_saveexec_b32 s0, s1              ; encoding: [0x01,0x26,0x80,0xbe]
 
 s_nand_saveexec_b32 s105, s104
-// GFX12: encoding: [0x68,0x26,0xe9,0xbe]
+// GFX12: s_nand_saveexec_b32 s105, s104          ; encoding: [0x68,0x26,0xe9,0xbe]
 
 s_nand_saveexec_b32 s0, s104
-// GFX12: encoding: [0x68,0x26,0x80,0xbe]
+// GFX12: s_nand_saveexec_b32 s0, s104            ; encoding: [0x68,0x26,0x80,0xbe]
 
 s_nand_saveexec_b32 s105, s1
-// GFX12: encoding: [0x01,0x26,0xe9,0xbe]
+// GFX12: s_nand_saveexec_b32 s105, s1            ; encoding: [0x01,0x26,0xe9,0xbe]
 
 s_nand_saveexec_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x26,0xea,0xbe]
+// GFX12: s_nand_saveexec_b32 vcc_lo, s1          ; encoding: [0x01,0x26,0xea,0xbe]
 
 s_nand_saveexec_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x26,0xeb,0xbe]
+// GFX12: s_nand_saveexec_b32 vcc_hi, s1          ; encoding: [0x01,0x26,0xeb,0xbe]
 
 s_nand_saveexec_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x26,0x80,0xbe]
+// GFX12: s_nand_saveexec_b32 s0, exec_lo         ; encoding: [0x7e,0x26,0x80,0xbe]
 
 s_nand_saveexec_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x26,0x80,0xbe]
+// GFX12: s_nand_saveexec_b32 s0, exec_hi         ; encoding: [0x7f,0x26,0x80,0xbe]
 
 s_nand_saveexec_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x26,0x80,0xbe]
+// GFX12: s_nand_saveexec_b32 s0, vcc_lo          ; encoding: [0x6a,0x26,0x80,0xbe]
 
 s_nand_saveexec_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x26,0x80,0xbe]
+// GFX12: s_nand_saveexec_b32 s0, vcc_hi          ; encoding: [0x6b,0x26,0x80,0xbe]
 
 s_nand_saveexec_b32 s0, m0
-// GFX12: encoding: [0x7d,0x26,0x80,0xbe]
+// GFX12: s_nand_saveexec_b32 s0, m0              ; encoding: [0x7d,0x26,0x80,0xbe]
 
 s_nand_saveexec_b32 s0, 0
-// GFX12: encoding: [0x80,0x26,0x80,0xbe]
+// GFX12: s_nand_saveexec_b32 s0, 0               ; encoding: [0x80,0x26,0x80,0xbe]
 
 s_nand_saveexec_b32 s0, -1
-// GFX12: encoding: [0xc1,0x26,0x80,0xbe]
+// GFX12: s_nand_saveexec_b32 s0, -1              ; encoding: [0xc1,0x26,0x80,0xbe]
 
 s_nand_saveexec_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x26,0x80,0xbe]
+// GFX12: s_nand_saveexec_b32 s0, 0.5             ; encoding: [0xf0,0x26,0x80,0xbe]
 
 s_nand_saveexec_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x26,0x80,0xbe]
+// GFX12: s_nand_saveexec_b32 s0, -4.0            ; encoding: [0xf7,0x26,0x80,0xbe]
 
 s_nand_saveexec_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x26,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_nand_saveexec_b32 s0, 0x3f717273      ; encoding: [0xff,0x26,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_nand_saveexec_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x26,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_nand_saveexec_b32 s0, 0xaf123456      ; encoding: [0xff,0x26,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_nor_saveexec_b32 s0, s1
-// GFX12: encoding: [0x01,0x28,0x80,0xbe]
+// GFX12: s_nor_saveexec_b32 s0, s1               ; encoding: [0x01,0x28,0x80,0xbe]
 
 s_nor_saveexec_b32 s105, s104
-// GFX12: encoding: [0x68,0x28,0xe9,0xbe]
+// GFX12: s_nor_saveexec_b32 s105, s104           ; encoding: [0x68,0x28,0xe9,0xbe]
 
 s_nor_saveexec_b32 s0, s104
-// GFX12: encoding: [0x68,0x28,0x80,0xbe]
+// GFX12: s_nor_saveexec_b32 s0, s104             ; encoding: [0x68,0x28,0x80,0xbe]
 
 s_nor_saveexec_b32 s105, s1
-// GFX12: encoding: [0x01,0x28,0xe9,0xbe]
+// GFX12: s_nor_saveexec_b32 s105, s1             ; encoding: [0x01,0x28,0xe9,0xbe]
 
 s_nor_saveexec_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x28,0xea,0xbe]
+// GFX12: s_nor_saveexec_b32 vcc_lo, s1           ; encoding: [0x01,0x28,0xea,0xbe]
 
 s_nor_saveexec_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x28,0xeb,0xbe]
+// GFX12: s_nor_saveexec_b32 vcc_hi, s1           ; encoding: [0x01,0x28,0xeb,0xbe]
 
 s_nor_saveexec_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x28,0x80,0xbe]
+// GFX12: s_nor_saveexec_b32 s0, exec_lo          ; encoding: [0x7e,0x28,0x80,0xbe]
 
 s_nor_saveexec_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x28,0x80,0xbe]
+// GFX12: s_nor_saveexec_b32 s0, exec_hi          ; encoding: [0x7f,0x28,0x80,0xbe]
 
 s_nor_saveexec_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x28,0x80,0xbe]
+// GFX12: s_nor_saveexec_b32 s0, vcc_lo           ; encoding: [0x6a,0x28,0x80,0xbe]
 
 s_nor_saveexec_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x28,0x80,0xbe]
+// GFX12: s_nor_saveexec_b32 s0, vcc_hi           ; encoding: [0x6b,0x28,0x80,0xbe]
 
 s_nor_saveexec_b32 s0, m0
-// GFX12: encoding: [0x7d,0x28,0x80,0xbe]
+// GFX12: s_nor_saveexec_b32 s0, m0               ; encoding: [0x7d,0x28,0x80,0xbe]
 
 s_nor_saveexec_b32 s0, 0
-// GFX12: encoding: [0x80,0x28,0x80,0xbe]
+// GFX12: s_nor_saveexec_b32 s0, 0                ; encoding: [0x80,0x28,0x80,0xbe]
 
 s_nor_saveexec_b32 s0, -1
-// GFX12: encoding: [0xc1,0x28,0x80,0xbe]
+// GFX12: s_nor_saveexec_b32 s0, -1               ; encoding: [0xc1,0x28,0x80,0xbe]
 
 s_nor_saveexec_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x28,0x80,0xbe]
+// GFX12: s_nor_saveexec_b32 s0, 0.5              ; encoding: [0xf0,0x28,0x80,0xbe]
 
 s_nor_saveexec_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x28,0x80,0xbe]
+// GFX12: s_nor_saveexec_b32 s0, -4.0             ; encoding: [0xf7,0x28,0x80,0xbe]
 
 s_nor_saveexec_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x28,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_nor_saveexec_b32 s0, 0x3f717273       ; encoding: [0xff,0x28,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_nor_saveexec_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x28,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_nor_saveexec_b32 s0, 0xaf123456       ; encoding: [0xff,0x28,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_xnor_saveexec_b32 s0, s1
-// GFX12: encoding: [0x01,0x2a,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b32 s0, s1              ; encoding: [0x01,0x2a,0x80,0xbe]
 
 s_xnor_saveexec_b32 s105, s104
-// GFX12: encoding: [0x68,0x2a,0xe9,0xbe]
+// GFX12: s_xnor_saveexec_b32 s105, s104          ; encoding: [0x68,0x2a,0xe9,0xbe]
 
 s_xnor_saveexec_b32 s0, s104
-// GFX12: encoding: [0x68,0x2a,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b32 s0, s104            ; encoding: [0x68,0x2a,0x80,0xbe]
 
 s_xnor_saveexec_b32 s105, s1
-// GFX12: encoding: [0x01,0x2a,0xe9,0xbe]
+// GFX12: s_xnor_saveexec_b32 s105, s1            ; encoding: [0x01,0x2a,0xe9,0xbe]
 
 s_xnor_saveexec_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x2a,0xea,0xbe]
+// GFX12: s_xnor_saveexec_b32 vcc_lo, s1          ; encoding: [0x01,0x2a,0xea,0xbe]
 
 s_xnor_saveexec_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x2a,0xeb,0xbe]
+// GFX12: s_xnor_saveexec_b32 vcc_hi, s1          ; encoding: [0x01,0x2a,0xeb,0xbe]
 
 s_xnor_saveexec_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x2a,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b32 s0, exec_lo         ; encoding: [0x7e,0x2a,0x80,0xbe]
 
 s_xnor_saveexec_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x2a,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b32 s0, exec_hi         ; encoding: [0x7f,0x2a,0x80,0xbe]
 
 s_xnor_saveexec_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x2a,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b32 s0, vcc_lo          ; encoding: [0x6a,0x2a,0x80,0xbe]
 
 s_xnor_saveexec_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x2a,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b32 s0, vcc_hi          ; encoding: [0x6b,0x2a,0x80,0xbe]
 
 s_xnor_saveexec_b32 s0, m0
-// GFX12: encoding: [0x7d,0x2a,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b32 s0, m0              ; encoding: [0x7d,0x2a,0x80,0xbe]
 
 s_xnor_saveexec_b32 s0, 0
-// GFX12: encoding: [0x80,0x2a,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b32 s0, 0               ; encoding: [0x80,0x2a,0x80,0xbe]
 
 s_xnor_saveexec_b32 s0, -1
-// GFX12: encoding: [0xc1,0x2a,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b32 s0, -1              ; encoding: [0xc1,0x2a,0x80,0xbe]
 
 s_xnor_saveexec_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x2a,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b32 s0, 0.5             ; encoding: [0xf0,0x2a,0x80,0xbe]
 
 s_xnor_saveexec_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x2a,0x80,0xbe]
+// GFX12: s_xnor_saveexec_b32 s0, -4.0            ; encoding: [0xf7,0x2a,0x80,0xbe]
 
 s_xnor_saveexec_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x2a,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_xnor_saveexec_b32 s0, 0x3f717273      ; encoding: [0xff,0x2a,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_xnor_saveexec_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x2a,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_xnor_saveexec_b32 s0, 0xaf123456      ; encoding: [0xff,0x2a,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_andn1_saveexec_b32 s0, s1
-// GFX12: encoding: [0x01,0x2c,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s0, s1          ; encoding: [0x01,0x2c,0x80,0xbe]
 
 s_andn1_saveexec_b32 s105, s104
-// GFX12: encoding: [0x68,0x2c,0xe9,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s105, s104      ; encoding: [0x68,0x2c,0xe9,0xbe]
 
 s_andn1_saveexec_b32 s0, s104
-// GFX12: encoding: [0x68,0x2c,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s0, s104        ; encoding: [0x68,0x2c,0x80,0xbe]
 
 s_andn1_saveexec_b32 s105, s1
-// GFX12: encoding: [0x01,0x2c,0xe9,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s105, s1        ; encoding: [0x01,0x2c,0xe9,0xbe]
 
 s_andn1_saveexec_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x2c,0xea,0xbe]
+// GFX12: s_and_not0_saveexec_b32 vcc_lo, s1      ; encoding: [0x01,0x2c,0xea,0xbe]
 
 s_andn1_saveexec_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x2c,0xeb,0xbe]
+// GFX12: s_and_not0_saveexec_b32 vcc_hi, s1      ; encoding: [0x01,0x2c,0xeb,0xbe]
 
 s_andn1_saveexec_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x2c,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s0, exec_lo     ; encoding: [0x7e,0x2c,0x80,0xbe]
 
 s_andn1_saveexec_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x2c,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s0, exec_hi     ; encoding: [0x7f,0x2c,0x80,0xbe]
 
 s_andn1_saveexec_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x2c,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s0, vcc_lo      ; encoding: [0x6a,0x2c,0x80,0xbe]
 
 s_andn1_saveexec_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x2c,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s0, vcc_hi      ; encoding: [0x6b,0x2c,0x80,0xbe]
 
 s_andn1_saveexec_b32 s0, m0
-// GFX12: encoding: [0x7d,0x2c,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s0, m0          ; encoding: [0x7d,0x2c,0x80,0xbe]
 
 s_andn1_saveexec_b32 s0, 0
-// GFX12: encoding: [0x80,0x2c,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s0, 0           ; encoding: [0x80,0x2c,0x80,0xbe]
 
 s_andn1_saveexec_b32 s0, -1
-// GFX12: encoding: [0xc1,0x2c,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s0, -1          ; encoding: [0xc1,0x2c,0x80,0xbe]
 
 s_andn1_saveexec_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x2c,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s0, 0.5         ; encoding: [0xf0,0x2c,0x80,0xbe]
 
 s_andn1_saveexec_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x2c,0x80,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s0, -4.0        ; encoding: [0xf7,0x2c,0x80,0xbe]
 
 s_andn1_saveexec_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x2c,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_and_not0_saveexec_b32 s0, 0x3f717273  ; encoding: [0xff,0x2c,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_andn1_saveexec_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x2c,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not0_saveexec_b32 s0, 0xaf123456  ; encoding: [0xff,0x2c,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_orn1_saveexec_b32 s0, s1
-// GFX12: encoding: [0x01,0x2e,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s0, s1           ; encoding: [0x01,0x2e,0x80,0xbe]
 
 s_orn1_saveexec_b32 s105, s104
-// GFX12: encoding: [0x68,0x2e,0xe9,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s105, s104       ; encoding: [0x68,0x2e,0xe9,0xbe]
 
 s_orn1_saveexec_b32 s0, s104
-// GFX12: encoding: [0x68,0x2e,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s0, s104         ; encoding: [0x68,0x2e,0x80,0xbe]
 
 s_orn1_saveexec_b32 s105, s1
-// GFX12: encoding: [0x01,0x2e,0xe9,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s105, s1         ; encoding: [0x01,0x2e,0xe9,0xbe]
 
 s_orn1_saveexec_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x2e,0xea,0xbe]
+// GFX12: s_or_not0_saveexec_b32 vcc_lo, s1       ; encoding: [0x01,0x2e,0xea,0xbe]
 
 s_orn1_saveexec_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x2e,0xeb,0xbe]
+// GFX12: s_or_not0_saveexec_b32 vcc_hi, s1       ; encoding: [0x01,0x2e,0xeb,0xbe]
 
 s_orn1_saveexec_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x2e,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s0, exec_lo      ; encoding: [0x7e,0x2e,0x80,0xbe]
 
 s_orn1_saveexec_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x2e,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s0, exec_hi      ; encoding: [0x7f,0x2e,0x80,0xbe]
 
 s_orn1_saveexec_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x2e,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s0, vcc_lo       ; encoding: [0x6a,0x2e,0x80,0xbe]
 
 s_orn1_saveexec_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x2e,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s0, vcc_hi       ; encoding: [0x6b,0x2e,0x80,0xbe]
 
 s_orn1_saveexec_b32 s0, m0
-// GFX12: encoding: [0x7d,0x2e,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s0, m0           ; encoding: [0x7d,0x2e,0x80,0xbe]
 
 s_orn1_saveexec_b32 s0, 0
-// GFX12: encoding: [0x80,0x2e,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s0, 0            ; encoding: [0x80,0x2e,0x80,0xbe]
 
 s_orn1_saveexec_b32 s0, -1
-// GFX12: encoding: [0xc1,0x2e,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s0, -1           ; encoding: [0xc1,0x2e,0x80,0xbe]
 
 s_orn1_saveexec_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x2e,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s0, 0.5          ; encoding: [0xf0,0x2e,0x80,0xbe]
 
 s_orn1_saveexec_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x2e,0x80,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s0, -4.0         ; encoding: [0xf7,0x2e,0x80,0xbe]
 
 s_orn1_saveexec_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x2e,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_or_not0_saveexec_b32 s0, 0x3f717273   ; encoding: [0xff,0x2e,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_orn1_saveexec_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x2e,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_not0_saveexec_b32 s0, 0xaf123456   ; encoding: [0xff,0x2e,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_andn1_wrexec_b32 s0, s1
-// GFX12: encoding: [0x01,0x34,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s0, s1            ; encoding: [0x01,0x34,0x80,0xbe]
 
 s_andn1_wrexec_b32 s105, s104
-// GFX12: encoding: [0x68,0x34,0xe9,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s105, s104        ; encoding: [0x68,0x34,0xe9,0xbe]
 
 s_andn1_wrexec_b32 s0, s104
-// GFX12: encoding: [0x68,0x34,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s0, s104          ; encoding: [0x68,0x34,0x80,0xbe]
 
 s_andn1_wrexec_b32 s105, s1
-// GFX12: encoding: [0x01,0x34,0xe9,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s105, s1          ; encoding: [0x01,0x34,0xe9,0xbe]
 
 s_andn1_wrexec_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x34,0xea,0xbe]
+// GFX12: s_and_not0_wrexec_b32 vcc_lo, s1        ; encoding: [0x01,0x34,0xea,0xbe]
 
 s_andn1_wrexec_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x34,0xeb,0xbe]
+// GFX12: s_and_not0_wrexec_b32 vcc_hi, s1        ; encoding: [0x01,0x34,0xeb,0xbe]
 
 s_andn1_wrexec_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x34,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s0, exec_lo       ; encoding: [0x7e,0x34,0x80,0xbe]
 
 s_andn1_wrexec_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x34,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s0, exec_hi       ; encoding: [0x7f,0x34,0x80,0xbe]
 
 s_andn1_wrexec_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x34,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s0, vcc_lo        ; encoding: [0x6a,0x34,0x80,0xbe]
 
 s_andn1_wrexec_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x34,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s0, vcc_hi        ; encoding: [0x6b,0x34,0x80,0xbe]
 
 s_andn1_wrexec_b32 s0, m0
-// GFX12: encoding: [0x7d,0x34,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s0, m0            ; encoding: [0x7d,0x34,0x80,0xbe]
 
 s_andn1_wrexec_b32 s0, 0
-// GFX12: encoding: [0x80,0x34,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s0, 0             ; encoding: [0x80,0x34,0x80,0xbe]
 
 s_andn1_wrexec_b32 s0, -1
-// GFX12: encoding: [0xc1,0x34,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s0, -1            ; encoding: [0xc1,0x34,0x80,0xbe]
 
 s_andn1_wrexec_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x34,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s0, 0.5           ; encoding: [0xf0,0x34,0x80,0xbe]
 
 s_andn1_wrexec_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x34,0x80,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s0, -4.0          ; encoding: [0xf7,0x34,0x80,0xbe]
 
 s_andn1_wrexec_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x34,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_and_not0_wrexec_b32 s0, 0x3f717273    ; encoding: [0xff,0x34,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_andn1_wrexec_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x34,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not0_wrexec_b32 s0, 0xaf123456    ; encoding: [0xff,0x34,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_andn2_wrexec_b32 s0, s1
-// GFX12: encoding: [0x01,0x36,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s0, s1            ; encoding: [0x01,0x36,0x80,0xbe]
 
 s_andn2_wrexec_b32 s105, s104
-// GFX12: encoding: [0x68,0x36,0xe9,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s105, s104        ; encoding: [0x68,0x36,0xe9,0xbe]
 
 s_andn2_wrexec_b32 s0, s104
-// GFX12: encoding: [0x68,0x36,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s0, s104          ; encoding: [0x68,0x36,0x80,0xbe]
 
 s_andn2_wrexec_b32 s105, s1
-// GFX12: encoding: [0x01,0x36,0xe9,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s105, s1          ; encoding: [0x01,0x36,0xe9,0xbe]
 
 s_andn2_wrexec_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x36,0xea,0xbe]
+// GFX12: s_and_not1_wrexec_b32 vcc_lo, s1        ; encoding: [0x01,0x36,0xea,0xbe]
 
 s_andn2_wrexec_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x36,0xeb,0xbe]
+// GFX12: s_and_not1_wrexec_b32 vcc_hi, s1        ; encoding: [0x01,0x36,0xeb,0xbe]
 
 s_andn2_wrexec_b32 s0, exec_lo
-// GFX12: encoding: [0x7e,0x36,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s0, exec_lo       ; encoding: [0x7e,0x36,0x80,0xbe]
 
 s_andn2_wrexec_b32 s0, exec_hi
-// GFX12: encoding: [0x7f,0x36,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s0, exec_hi       ; encoding: [0x7f,0x36,0x80,0xbe]
 
 s_andn2_wrexec_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x36,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s0, vcc_lo        ; encoding: [0x6a,0x36,0x80,0xbe]
 
 s_andn2_wrexec_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x36,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s0, vcc_hi        ; encoding: [0x6b,0x36,0x80,0xbe]
 
 s_andn2_wrexec_b32 s0, m0
-// GFX12: encoding: [0x7d,0x36,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s0, m0            ; encoding: [0x7d,0x36,0x80,0xbe]
 
 s_andn2_wrexec_b32 s0, 0
-// GFX12: encoding: [0x80,0x36,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s0, 0             ; encoding: [0x80,0x36,0x80,0xbe]
 
 s_andn2_wrexec_b32 s0, -1
-// GFX12: encoding: [0xc1,0x36,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s0, -1            ; encoding: [0xc1,0x36,0x80,0xbe]
 
 s_andn2_wrexec_b32 s0, 0.5
-// GFX12: encoding: [0xf0,0x36,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s0, 0.5           ; encoding: [0xf0,0x36,0x80,0xbe]
 
 s_andn2_wrexec_b32 s0, -4.0
-// GFX12: encoding: [0xf7,0x36,0x80,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s0, -4.0          ; encoding: [0xf7,0x36,0x80,0xbe]
 
 s_andn2_wrexec_b32 s0, 0x3f717273
-// GFX12: encoding: [0xff,0x36,0x80,0xbe,0x73,0x72,0x71,0x3f]
+// GFX12: s_and_not1_wrexec_b32 s0, 0x3f717273    ; encoding: [0xff,0x36,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
 s_andn2_wrexec_b32 s0, 0xaf123456
-// GFX12: encoding: [0xff,0x36,0x80,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not1_wrexec_b32 s0, 0xaf123456    ; encoding: [0xff,0x36,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
 s_movrelsd_2_b32 s0, s1
-// GFX12: encoding: [0x01,0x44,0x80,0xbe]
+// GFX12: s_movrelsd_2_b32 s0, s1                 ; encoding: [0x01,0x44,0x80,0xbe]
 
 s_movrelsd_2_b32 s105, s104
-// GFX12: encoding: [0x68,0x44,0xe9,0xbe]
+// GFX12: s_movrelsd_2_b32 s105, s104             ; encoding: [0x68,0x44,0xe9,0xbe]
 
 s_movrelsd_2_b32 s0, s104
-// GFX12: encoding: [0x68,0x44,0x80,0xbe]
+// GFX12: s_movrelsd_2_b32 s0, s104               ; encoding: [0x68,0x44,0x80,0xbe]
 
 s_movrelsd_2_b32 s105, s1
-// GFX12: encoding: [0x01,0x44,0xe9,0xbe]
+// GFX12: s_movrelsd_2_b32 s105, s1               ; encoding: [0x01,0x44,0xe9,0xbe]
 
 s_movrelsd_2_b32 vcc_lo, s1
-// GFX12: encoding: [0x01,0x44,0xea,0xbe]
+// GFX12: s_movrelsd_2_b32 vcc_lo, s1             ; encoding: [0x01,0x44,0xea,0xbe]
 
 s_movrelsd_2_b32 vcc_hi, s1
-// GFX12: encoding: [0x01,0x44,0xeb,0xbe]
+// GFX12: s_movrelsd_2_b32 vcc_hi, s1             ; encoding: [0x01,0x44,0xeb,0xbe]
 
 s_movrelsd_2_b32 s0, vcc_lo
-// GFX12: encoding: [0x6a,0x44,0x80,0xbe]
+// GFX12: s_movrelsd_2_b32 s0, vcc_lo             ; encoding: [0x6a,0x44,0x80,0xbe]
 
 s_movrelsd_2_b32 s0, vcc_hi
-// GFX12: encoding: [0x6b,0x44,0x80,0xbe]
+// GFX12: s_movrelsd_2_b32 s0, vcc_hi             ; encoding: [0x6b,0x44,0x80,0xbe]
 
 s_sendmsg_rtn_b32 s1, 0x0
-// GFX12: encoding: [0x00,0x4c,0x81,0xbe]
+// GFX12: s_sendmsg_rtn_b32 s1, sendmsg(0, 0, 0)  ; encoding: [0x00,0x4c,0x81,0xbe]
 
 s_sendmsg_rtn_b32 s2, 0x12
-// GFX12: encoding: [0x12,0x4c,0x82,0xbe]
+// GFX12: s_sendmsg_rtn_b32 s2, sendmsg(18, 0, 0) ; encoding: [0x12,0x4c,0x82,0xbe]
 
 s_sendmsg_rtn_b32 s3, 0xff
-// GFX12: encoding: [0xff,0x4c,0x83,0xbe]
+// GFX12: s_sendmsg_rtn_b32 s3, sendmsg(255, 0, 0) ; encoding: [0xff,0x4c,0x83,0xbe]
 
 s_sendmsg_rtn_b64 s[0:1], 0x0
-// GFX12: encoding: [0x00,0x4d,0x80,0xbe]
+// GFX12: s_sendmsg_rtn_b64 s[0:1], sendmsg(0, 0, 0) ; encoding: [0x00,0x4d,0x80,0xbe]
 
 s_sendmsg_rtn_b64 s[2:3], 0x12
-// GFX12: encoding: [0x12,0x4d,0x82,0xbe]
+// GFX12: s_sendmsg_rtn_b64 s[2:3], sendmsg(18, 0, 0) ; encoding: [0x12,0x4d,0x82,0xbe]
 
 s_sendmsg_rtn_b64 s[4:5], 0xff
-// GFX12: encoding: [0xff,0x4d,0x84,0xbe]
+// GFX12: s_sendmsg_rtn_b64 s[4:5], sendmsg(255, 0, 0) ; encoding: [0xff,0x4d,0x84,0xbe]
 
 s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_DOORBELL)
-// GFX12: encoding: [0x80,0x4c,0x80,0xbe]
+// GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_DOORBELL) ; encoding: [0x80,0x4c,0x80,0xbe]
 
 s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_DDID)
-// GFX12: encoding: [0x81,0x4c,0x80,0xbe]
+// GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_DDID) ; encoding: [0x81,0x4c,0x80,0xbe]
 
 s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_TMA)
-// GFX12: encoding: [0x82,0x4c,0x80,0xbe]
+// GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_TMA) ; encoding: [0x82,0x4c,0x80,0xbe]
 
 s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_REALTIME)
-// GFX12: encoding: [0x83,0x4c,0x80,0xbe]
+// GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_REALTIME) ; encoding: [0x83,0x4c,0x80,0xbe]
 
 s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_SAVE_WAVE)
-// GFX12: encoding: [0x84,0x4c,0x80,0xbe]
+// GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_SAVE_WAVE) ; encoding: [0x84,0x4c,0x80,0xbe]
 
 s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_TBA)
-// GFX12: encoding: [0x85,0x4c,0x80,0xbe]
+// GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_TBA) ; encoding: [0x85,0x4c,0x80,0xbe]
 
 s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_TBA_TO_PC)
-// GFX12: encoding: [0x86,0x4c,0x80,0xbe]
+// GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_TBA_TO_PC) ; encoding: [0x86,0x4c,0x80,0xbe]
 
 s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_SE_AID_ID)
-// GFX12: encoding: [0x87,0x4c,0x80,0xbe]
+// GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_SE_AID_ID) ; encoding: [0x87,0x4c,0x80,0xbe]
 
 s_ctz_i32_b32 s5, s1
-// GFX12: encoding: [0x01,0x08,0x85,0xbe]
+// GFX12: s_ctz_i32_b32 s5, s1                    ; encoding: [0x01,0x08,0x85,0xbe]
 
 s_ctz_i32_b32 s5, s105
-// GFX12: encoding: [0x69,0x08,0x85,0xbe]
+// GFX12: s_ctz_i32_b32 s5, s105                  ; encoding: [0x69,0x08,0x85,0xbe]
 
 s_ctz_i32_b32 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x08,0x85,0xbe]
+// GFX12: s_ctz_i32_b32 s5, vcc_lo                ; encoding: [0x6a,0x08,0x85,0xbe]
 
 s_ctz_i32_b32 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x08,0x85,0xbe]
+// GFX12: s_ctz_i32_b32 s5, vcc_hi                ; encoding: [0x6b,0x08,0x85,0xbe]
 
 s_ctz_i32_b32 s5, ttmp15
-// GFX12: encoding: [0x7b,0x08,0x85,0xbe]
+// GFX12: s_ctz_i32_b32 s5, ttmp15                ; encoding: [0x7b,0x08,0x85,0xbe]
 
 s_ctz_i32_b32 s105, m0
-// GFX12: encoding: [0x7d,0x08,0xe9,0xbe]
+// GFX12: s_ctz_i32_b32 s105, m0                  ; encoding: [0x7d,0x08,0xe9,0xbe]
 
 s_ctz_i32_b32 vcc_lo, exec_lo
-// GFX12: encoding: [0x7e,0x08,0xea,0xbe]
+// GFX12: s_ctz_i32_b32 vcc_lo, exec_lo           ; encoding: [0x7e,0x08,0xea,0xbe]
 
 s_ctz_i32_b32 vcc_hi, exec_hi
-// GFX12: encoding: [0x7f,0x08,0xeb,0xbe]
+// GFX12: s_ctz_i32_b32 vcc_hi, exec_hi           ; encoding: [0x7f,0x08,0xeb,0xbe]
 
 s_ctz_i32_b32 ttmp15, null
-// GFX12: encoding: [0x7c,0x08,0xfb,0xbe]
+// GFX12: s_ctz_i32_b32 ttmp15, null              ; encoding: [0x7c,0x08,0xfb,0xbe]
 
 s_ctz_i32_b32 m0, -1
-// GFX12: encoding: [0xc1,0x08,0xfd,0xbe]
+// GFX12: s_ctz_i32_b32 m0, -1                    ; encoding: [0xc1,0x08,0xfd,0xbe]
 
 s_ctz_i32_b32 exec_lo, 0.5
-// GFX12: encoding: [0xf0,0x08,0xfe,0xbe]
+// GFX12: s_ctz_i32_b32 exec_lo, 0.5              ; encoding: [0xf0,0x08,0xfe,0xbe]
 
 s_ctz_i32_b32 exec_hi, src_scc
-// GFX12: encoding: [0xfd,0x08,0xff,0xbe]
+// GFX12: s_ctz_i32_b32 exec_hi, src_scc          ; encoding: [0xfd,0x08,0xff,0xbe]
 
 s_ctz_i32_b32 null, 0xaf123456
-// GFX12: encoding: [0xff,0x08,0xfc,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_ctz_i32_b32 null, 0xaf123456          ; encoding: [0xff,0x08,0xfc,0xbe,0x56,0x34,0x12,0xaf]
 
 s_ctz_i32_b64 s5, s[2:3]
-// GFX12: encoding: [0x02,0x09,0x85,0xbe]
+// GFX12: s_ctz_i32_b64 s5, s[2:3]                ; encoding: [0x02,0x09,0x85,0xbe]
 
 s_ctz_i32_b64 s5, s[104:105]
-// GFX12: encoding: [0x68,0x09,0x85,0xbe]
+// GFX12: s_ctz_i32_b64 s5, s[104:105]            ; encoding: [0x68,0x09,0x85,0xbe]
 
 s_ctz_i32_b64 s105, vcc
-// GFX12: encoding: [0x6a,0x09,0xe9,0xbe]
+// GFX12: s_ctz_i32_b64 s105, vcc                 ; encoding: [0x6a,0x09,0xe9,0xbe]
 
 s_ctz_i32_b64 vcc_lo, ttmp[14:15]
-// GFX12: encoding: [0x7a,0x09,0xea,0xbe]
+// GFX12: s_ctz_i32_b64 vcc_lo, ttmp[14:15]       ; encoding: [0x7a,0x09,0xea,0xbe]
 
 s_ctz_i32_b64 vcc_hi, exec
-// GFX12: encoding: [0x7e,0x09,0xeb,0xbe]
+// GFX12: s_ctz_i32_b64 vcc_hi, exec              ; encoding: [0x7e,0x09,0xeb,0xbe]
 
 s_ctz_i32_b64 ttmp15, null
-// GFX12: encoding: [0x7c,0x09,0xfb,0xbe]
+// GFX12: s_ctz_i32_b64 ttmp15, null              ; encoding: [0x7c,0x09,0xfb,0xbe]
 
 s_ctz_i32_b64 m0, -1
-// GFX12: encoding: [0xc1,0x09,0xfd,0xbe]
+// GFX12: s_ctz_i32_b64 m0, -1                    ; encoding: [0xc1,0x09,0xfd,0xbe]
 
 s_ctz_i32_b64 exec_lo, 0.5
-// GFX12: encoding: [0xf0,0x09,0xfe,0xbe]
+// GFX12: s_ctz_i32_b64 exec_lo, 0.5              ; encoding: [0xf0,0x09,0xfe,0xbe]
 
 s_ctz_i32_b64 exec_hi, src_scc
-// GFX12: encoding: [0xfd,0x09,0xff,0xbe]
+// GFX12: s_ctz_i32_b64 exec_hi, src_scc          ; encoding: [0xfd,0x09,0xff,0xbe]
 
 s_ctz_i32_b64 null, 0xaf123456
-// GFX12: encoding: [0xff,0x09,0xfc,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_ctz_i32_b64 null, 0xaf123456          ; encoding: [0xff,0x09,0xfc,0xbe,0x56,0x34,0x12,0xaf]
 
 s_and_not1_saveexec_b64 s[10:11], s[2:3]
-// GFX12: encoding: [0x02,0x31,0x8a,0xbe]
+// GFX12: s_and_not1_saveexec_b64 s[10:11], s[2:3] ; encoding: [0x02,0x31,0x8a,0xbe]
 
 s_and_not1_saveexec_b64 s[10:11], s[104:105]
-// GFX12: encoding: [0x68,0x31,0x8a,0xbe]
+// GFX12: s_and_not1_saveexec_b64 s[10:11], s[104:105] ; encoding: [0x68,0x31,0x8a,0xbe]
 
 s_and_not1_saveexec_b64 s[10:11], vcc
-// GFX12: encoding: [0x6a,0x31,0x8a,0xbe]
+// GFX12: s_and_not1_saveexec_b64 s[10:11], vcc   ; encoding: [0x6a,0x31,0x8a,0xbe]
 
 s_and_not1_saveexec_b64 s[10:11], ttmp[14:15]
-// GFX12: encoding: [0x7a,0x31,0x8a,0xbe]
+// GFX12: s_and_not1_saveexec_b64 s[10:11], ttmp[14:15] ; encoding: [0x7a,0x31,0x8a,0xbe]
 
 s_and_not1_saveexec_b64 s[10:11], null
-// GFX12: encoding: [0x7c,0x31,0x8a,0xbe]
+// GFX12: s_and_not1_saveexec_b64 s[10:11], null  ; encoding: [0x7c,0x31,0x8a,0xbe]
 
 s_and_not1_saveexec_b64 s[104:105], -1
-// GFX12: encoding: [0xc1,0x31,0xe8,0xbe]
+// GFX12: s_and_not1_saveexec_b64 s[104:105], -1  ; encoding: [0xc1,0x31,0xe8,0xbe]
 
 s_and_not1_saveexec_b64 vcc, 0.5
-// GFX12: encoding: [0xf0,0x31,0xea,0xbe]
+// GFX12: s_and_not1_saveexec_b64 vcc, 0.5        ; encoding: [0xf0,0x31,0xea,0xbe]
 
 s_and_not1_saveexec_b64 ttmp[14:15], src_scc
-// GFX12: encoding: [0xfd,0x31,0xfa,0xbe]
+// GFX12: s_and_not1_saveexec_b64 ttmp[14:15], src_scc ; encoding: [0xfd,0x31,0xfa,0xbe]
 
 s_and_not1_saveexec_b64 null, 0xaf123456
-// GFX12: encoding: [0xff,0x31,0xfc,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not1_saveexec_b64 null, 0xaf123456 ; encoding: [0xff,0x31,0xfc,0xbe,0x56,0x34,0x12,0xaf]
 
 s_and_not0_saveexec_b32 s5, s1
-// GFX12: encoding: [0x01,0x2c,0x85,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s5, s1          ; encoding: [0x01,0x2c,0x85,0xbe]
 
 s_and_not0_saveexec_b32 s5, s105
-// GFX12: encoding: [0x69,0x2c,0x85,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s5, s105        ; encoding: [0x69,0x2c,0x85,0xbe]
 
 s_and_not0_saveexec_b32 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x2c,0x85,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s5, vcc_lo      ; encoding: [0x6a,0x2c,0x85,0xbe]
 
 s_and_not0_saveexec_b32 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x2c,0x85,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s5, vcc_hi      ; encoding: [0x6b,0x2c,0x85,0xbe]
 
 s_and_not0_saveexec_b32 s5, ttmp15
-// GFX12: encoding: [0x7b,0x2c,0x85,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s5, ttmp15      ; encoding: [0x7b,0x2c,0x85,0xbe]
 
 s_and_not0_saveexec_b32 s5, exec_lo
-// GFX12: encoding: [0x7e,0x2c,0x85,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s5, exec_lo     ; encoding: [0x7e,0x2c,0x85,0xbe]
 
 s_and_not0_saveexec_b32 s105, null
-// GFX12: encoding: [0x7c,0x2c,0xe9,0xbe]
+// GFX12: s_and_not0_saveexec_b32 s105, null      ; encoding: [0x7c,0x2c,0xe9,0xbe]
 
 s_and_not0_saveexec_b32 vcc_lo, -1
-// GFX12: encoding: [0xc1,0x2c,0xea,0xbe]
+// GFX12: s_and_not0_saveexec_b32 vcc_lo, -1      ; encoding: [0xc1,0x2c,0xea,0xbe]
 
 s_and_not0_saveexec_b32 vcc_hi, 0.5
-// GFX12: encoding: [0xf0,0x2c,0xeb,0xbe]
+// GFX12: s_and_not0_saveexec_b32 vcc_hi, 0.5     ; encoding: [0xf0,0x2c,0xeb,0xbe]
 
 s_and_not0_saveexec_b32 ttmp15, src_scc
-// GFX12: encoding: [0xfd,0x2c,0xfb,0xbe]
+// GFX12: s_and_not0_saveexec_b32 ttmp15, src_scc ; encoding: [0xfd,0x2c,0xfb,0xbe]
 
 s_and_not0_saveexec_b32 null, 0xaf123456
-// GFX12: encoding: [0xff,0x2c,0xfc,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not0_saveexec_b32 null, 0xaf123456 ; encoding: [0xff,0x2c,0xfc,0xbe,0x56,0x34,0x12,0xaf]
 
 s_and_not0_saveexec_b64 s[10:11], s[2:3]
-// GFX12: encoding: [0x02,0x2d,0x8a,0xbe]
+// GFX12: s_and_not0_saveexec_b64 s[10:11], s[2:3] ; encoding: [0x02,0x2d,0x8a,0xbe]
 
 s_and_not0_saveexec_b64 s[10:11], s[104:105]
-// GFX12: encoding: [0x68,0x2d,0x8a,0xbe]
+// GFX12: s_and_not0_saveexec_b64 s[10:11], s[104:105] ; encoding: [0x68,0x2d,0x8a,0xbe]
 
 s_and_not0_saveexec_b64 s[10:11], vcc
-// GFX12: encoding: [0x6a,0x2d,0x8a,0xbe]
+// GFX12: s_and_not0_saveexec_b64 s[10:11], vcc   ; encoding: [0x6a,0x2d,0x8a,0xbe]
 
 s_and_not0_saveexec_b64 s[10:11], ttmp[14:15]
-// GFX12: encoding: [0x7a,0x2d,0x8a,0xbe]
+// GFX12: s_and_not0_saveexec_b64 s[10:11], ttmp[14:15] ; encoding: [0x7a,0x2d,0x8a,0xbe]
 
 s_and_not0_saveexec_b64 s[10:11], null
-// GFX12: encoding: [0x7c,0x2d,0x8a,0xbe]
+// GFX12: s_and_not0_saveexec_b64 s[10:11], null  ; encoding: [0x7c,0x2d,0x8a,0xbe]
 
 s_and_not0_saveexec_b64 s[104:105], -1
-// GFX12: encoding: [0xc1,0x2d,0xe8,0xbe]
+// GFX12: s_and_not0_saveexec_b64 s[104:105], -1  ; encoding: [0xc1,0x2d,0xe8,0xbe]
 
 s_and_not0_saveexec_b64 vcc, 0.5
-// GFX12: encoding: [0xf0,0x2d,0xea,0xbe]
+// GFX12: s_and_not0_saveexec_b64 vcc, 0.5        ; encoding: [0xf0,0x2d,0xea,0xbe]
 
 s_and_not0_saveexec_b64 ttmp[14:15], src_scc
-// GFX12: encoding: [0xfd,0x2d,0xfa,0xbe]
+// GFX12: s_and_not0_saveexec_b64 ttmp[14:15], src_scc ; encoding: [0xfd,0x2d,0xfa,0xbe]
 
 s_and_not0_saveexec_b64 null, 0xaf123456
-// GFX12: encoding: [0xff,0x2d,0xfc,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not0_saveexec_b64 null, 0xaf123456 ; encoding: [0xff,0x2d,0xfc,0xbe,0x56,0x34,0x12,0xaf]
 
 s_and_not0_wrexec_b32 s5, s1
-// GFX12: encoding: [0x01,0x34,0x85,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s5, s1            ; encoding: [0x01,0x34,0x85,0xbe]
 
 s_and_not0_wrexec_b32 s5, s105
-// GFX12: encoding: [0x69,0x34,0x85,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s5, s105          ; encoding: [0x69,0x34,0x85,0xbe]
 
 s_and_not0_wrexec_b32 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x34,0x85,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s5, vcc_lo        ; encoding: [0x6a,0x34,0x85,0xbe]
 
 s_and_not0_wrexec_b32 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x34,0x85,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s5, vcc_hi        ; encoding: [0x6b,0x34,0x85,0xbe]
 
 s_and_not0_wrexec_b32 s5, ttmp15
-// GFX12: encoding: [0x7b,0x34,0x85,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s5, ttmp15        ; encoding: [0x7b,0x34,0x85,0xbe]
 
 s_and_not0_wrexec_b32 s5, exec_lo
-// GFX12: encoding: [0x7e,0x34,0x85,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s5, exec_lo       ; encoding: [0x7e,0x34,0x85,0xbe]
 
 s_and_not0_wrexec_b32 s105, null
-// GFX12: encoding: [0x7c,0x34,0xe9,0xbe]
+// GFX12: s_and_not0_wrexec_b32 s105, null        ; encoding: [0x7c,0x34,0xe9,0xbe]
 
 s_and_not0_wrexec_b32 vcc_lo, -1
-// GFX12: encoding: [0xc1,0x34,0xea,0xbe]
+// GFX12: s_and_not0_wrexec_b32 vcc_lo, -1        ; encoding: [0xc1,0x34,0xea,0xbe]
 
 s_and_not0_wrexec_b32 vcc_hi, 0.5
-// GFX12: encoding: [0xf0,0x34,0xeb,0xbe]
+// GFX12: s_and_not0_wrexec_b32 vcc_hi, 0.5       ; encoding: [0xf0,0x34,0xeb,0xbe]
 
 s_and_not0_wrexec_b32 ttmp15, src_scc
-// GFX12: encoding: [0xfd,0x34,0xfb,0xbe]
+// GFX12: s_and_not0_wrexec_b32 ttmp15, src_scc   ; encoding: [0xfd,0x34,0xfb,0xbe]
 
 s_and_not0_wrexec_b32 null, 0xaf123456
-// GFX12: encoding: [0xff,0x34,0xfc,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not0_wrexec_b32 null, 0xaf123456  ; encoding: [0xff,0x34,0xfc,0xbe,0x56,0x34,0x12,0xaf]
 
 s_and_not0_wrexec_b64 s[10:11], s[2:3]
-// GFX12: encoding: [0x02,0x35,0x8a,0xbe]
+// GFX12: s_and_not0_wrexec_b64 s[10:11], s[2:3]  ; encoding: [0x02,0x35,0x8a,0xbe]
 
 s_and_not0_wrexec_b64 s[10:11], s[104:105]
-// GFX12: encoding: [0x68,0x35,0x8a,0xbe]
+// GFX12: s_and_not0_wrexec_b64 s[10:11], s[104:105] ; encoding: [0x68,0x35,0x8a,0xbe]
 
 s_and_not0_wrexec_b64 s[10:11], vcc
-// GFX12: encoding: [0x6a,0x35,0x8a,0xbe]
+// GFX12: s_and_not0_wrexec_b64 s[10:11], vcc     ; encoding: [0x6a,0x35,0x8a,0xbe]
 
 s_and_not0_wrexec_b64 s[10:11], ttmp[14:15]
-// GFX12: encoding: [0x7a,0x35,0x8a,0xbe]
+// GFX12: s_and_not0_wrexec_b64 s[10:11], ttmp[14:15] ; encoding: [0x7a,0x35,0x8a,0xbe]
 
 s_and_not0_wrexec_b64 s[10:11], null
-// GFX12: encoding: [0x7c,0x35,0x8a,0xbe]
+// GFX12: s_and_not0_wrexec_b64 s[10:11], null    ; encoding: [0x7c,0x35,0x8a,0xbe]
 
 s_and_not0_wrexec_b64 s[104:105], -1
-// GFX12: encoding: [0xc1,0x35,0xe8,0xbe]
+// GFX12: s_and_not0_wrexec_b64 s[104:105], -1    ; encoding: [0xc1,0x35,0xe8,0xbe]
 
 s_and_not0_wrexec_b64 vcc, 0.5
-// GFX12: encoding: [0xf0,0x35,0xea,0xbe]
+// GFX12: s_and_not0_wrexec_b64 vcc, 0.5          ; encoding: [0xf0,0x35,0xea,0xbe]
 
 s_and_not0_wrexec_b64 ttmp[14:15], src_scc
-// GFX12: encoding: [0xfd,0x35,0xfa,0xbe]
+// GFX12: s_and_not0_wrexec_b64 ttmp[14:15], src_scc ; encoding: [0xfd,0x35,0xfa,0xbe]
 
 s_and_not0_wrexec_b64 null, 0xaf123456
-// GFX12: encoding: [0xff,0x35,0xfc,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not0_wrexec_b64 null, 0xaf123456  ; encoding: [0xff,0x35,0xfc,0xbe,0x56,0x34,0x12,0xaf]
 
 s_and_not1_saveexec_b32 s5, s1
-// GFX12: encoding: [0x01,0x30,0x85,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s5, s1          ; encoding: [0x01,0x30,0x85,0xbe]
 
 s_and_not1_saveexec_b32 s5, s105
-// GFX12: encoding: [0x69,0x30,0x85,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s5, s105        ; encoding: [0x69,0x30,0x85,0xbe]
 
 s_and_not1_saveexec_b32 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x30,0x85,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s5, vcc_lo      ; encoding: [0x6a,0x30,0x85,0xbe]
 
 s_and_not1_saveexec_b32 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x30,0x85,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s5, vcc_hi      ; encoding: [0x6b,0x30,0x85,0xbe]
 
 s_and_not1_saveexec_b32 s5, ttmp15
-// GFX12: encoding: [0x7b,0x30,0x85,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s5, ttmp15      ; encoding: [0x7b,0x30,0x85,0xbe]
 
 s_and_not1_saveexec_b32 s5, exec_lo
-// GFX12: encoding: [0x7e,0x30,0x85,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s5, exec_lo     ; encoding: [0x7e,0x30,0x85,0xbe]
 
 s_and_not1_saveexec_b32 s105, null
-// GFX12: encoding: [0x7c,0x30,0xe9,0xbe]
+// GFX12: s_and_not1_saveexec_b32 s105, null      ; encoding: [0x7c,0x30,0xe9,0xbe]
 
 s_and_not1_saveexec_b32 vcc_lo, -1
-// GFX12: encoding: [0xc1,0x30,0xea,0xbe]
+// GFX12: s_and_not1_saveexec_b32 vcc_lo, -1      ; encoding: [0xc1,0x30,0xea,0xbe]
 
 s_and_not1_saveexec_b32 vcc_hi, 0.5
-// GFX12: encoding: [0xf0,0x30,0xeb,0xbe]
+// GFX12: s_and_not1_saveexec_b32 vcc_hi, 0.5     ; encoding: [0xf0,0x30,0xeb,0xbe]
 
 s_and_not1_saveexec_b32 ttmp15, src_scc
-// GFX12: encoding: [0xfd,0x30,0xfb,0xbe]
+// GFX12: s_and_not1_saveexec_b32 ttmp15, src_scc ; encoding: [0xfd,0x30,0xfb,0xbe]
 
 s_and_not1_saveexec_b32 null, 0xaf123456
-// GFX12: encoding: [0xff,0x30,0xfc,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not1_saveexec_b32 null, 0xaf123456 ; encoding: [0xff,0x30,0xfc,0xbe,0x56,0x34,0x12,0xaf]
 
 s_and_not1_wrexec_b32 s5, s1
-// GFX12: encoding: [0x01,0x36,0x85,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s5, s1            ; encoding: [0x01,0x36,0x85,0xbe]
 
 s_and_not1_wrexec_b32 s5, s105
-// GFX12: encoding: [0x69,0x36,0x85,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s5, s105          ; encoding: [0x69,0x36,0x85,0xbe]
 
 s_and_not1_wrexec_b32 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x36,0x85,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s5, vcc_lo        ; encoding: [0x6a,0x36,0x85,0xbe]
 
 s_and_not1_wrexec_b32 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x36,0x85,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s5, vcc_hi        ; encoding: [0x6b,0x36,0x85,0xbe]
 
 s_and_not1_wrexec_b32 s5, ttmp15
-// GFX12: encoding: [0x7b,0x36,0x85,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s5, ttmp15        ; encoding: [0x7b,0x36,0x85,0xbe]
 
 s_and_not1_wrexec_b32 s5, exec_lo
-// GFX12: encoding: [0x7e,0x36,0x85,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s5, exec_lo       ; encoding: [0x7e,0x36,0x85,0xbe]
 
 s_and_not1_wrexec_b32 s105, null
-// GFX12: encoding: [0x7c,0x36,0xe9,0xbe]
+// GFX12: s_and_not1_wrexec_b32 s105, null        ; encoding: [0x7c,0x36,0xe9,0xbe]
 
 s_and_not1_wrexec_b32 vcc_lo, -1
-// GFX12: encoding: [0xc1,0x36,0xea,0xbe]
+// GFX12: s_and_not1_wrexec_b32 vcc_lo, -1        ; encoding: [0xc1,0x36,0xea,0xbe]
 
 s_and_not1_wrexec_b32 vcc_hi, 0.5
-// GFX12: encoding: [0xf0,0x36,0xeb,0xbe]
+// GFX12: s_and_not1_wrexec_b32 vcc_hi, 0.5       ; encoding: [0xf0,0x36,0xeb,0xbe]
 
 s_and_not1_wrexec_b32 ttmp15, src_scc
-// GFX12: encoding: [0xfd,0x36,0xfb,0xbe]
+// GFX12: s_and_not1_wrexec_b32 ttmp15, src_scc   ; encoding: [0xfd,0x36,0xfb,0xbe]
 
 s_and_not1_wrexec_b32 null, 0xaf123456
-// GFX12: encoding: [0xff,0x36,0xfc,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not1_wrexec_b32 null, 0xaf123456  ; encoding: [0xff,0x36,0xfc,0xbe,0x56,0x34,0x12,0xaf]
 
 s_and_not1_wrexec_b64 s[10:11], s[2:3]
-// GFX12: encoding: [0x02,0x37,0x8a,0xbe]
+// GFX12: s_and_not1_wrexec_b64 s[10:11], s[2:3]  ; encoding: [0x02,0x37,0x8a,0xbe]
 
 s_and_not1_wrexec_b64 s[10:11], s[104:105]
-// GFX12: encoding: [0x68,0x37,0x8a,0xbe]
+// GFX12: s_and_not1_wrexec_b64 s[10:11], s[104:105] ; encoding: [0x68,0x37,0x8a,0xbe]
 
 s_and_not1_wrexec_b64 s[10:11], vcc
-// GFX12: encoding: [0x6a,0x37,0x8a,0xbe]
+// GFX12: s_and_not1_wrexec_b64 s[10:11], vcc     ; encoding: [0x6a,0x37,0x8a,0xbe]
 
 s_and_not1_wrexec_b64 s[10:11], ttmp[14:15]
-// GFX12: encoding: [0x7a,0x37,0x8a,0xbe]
+// GFX12: s_and_not1_wrexec_b64 s[10:11], ttmp[14:15] ; encoding: [0x7a,0x37,0x8a,0xbe]
 
 s_and_not1_wrexec_b64 s[10:11], null
-// GFX12: encoding: [0x7c,0x37,0x8a,0xbe]
+// GFX12: s_and_not1_wrexec_b64 s[10:11], null    ; encoding: [0x7c,0x37,0x8a,0xbe]
 
 s_and_not1_wrexec_b64 s[104:105], -1
-// GFX12: encoding: [0xc1,0x37,0xe8,0xbe]
+// GFX12: s_and_not1_wrexec_b64 s[104:105], -1    ; encoding: [0xc1,0x37,0xe8,0xbe]
 
 s_and_not1_wrexec_b64 vcc, 0.5
-// GFX12: encoding: [0xf0,0x37,0xea,0xbe]
+// GFX12: s_and_not1_wrexec_b64 vcc, 0.5          ; encoding: [0xf0,0x37,0xea,0xbe]
 
 s_and_not1_wrexec_b64 ttmp[14:15], src_scc
-// GFX12: encoding: [0xfd,0x37,0xfa,0xbe]
+// GFX12: s_and_not1_wrexec_b64 ttmp[14:15], src_scc ; encoding: [0xfd,0x37,0xfa,0xbe]
 
 s_and_not1_wrexec_b64 null, 0xaf123456
-// GFX12: encoding: [0xff,0x37,0xfc,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not1_wrexec_b64 null, 0xaf123456  ; encoding: [0xff,0x37,0xfc,0xbe,0x56,0x34,0x12,0xaf]
 
 s_cls_i32 s5, s1
-// GFX12: encoding: [0x01,0x0c,0x85,0xbe]
+// GFX12: s_cls_i32 s5, s1                        ; encoding: [0x01,0x0c,0x85,0xbe]
 
 s_cls_i32 s5, s105
-// GFX12: encoding: [0x69,0x0c,0x85,0xbe]
+// GFX12: s_cls_i32 s5, s105                      ; encoding: [0x69,0x0c,0x85,0xbe]
 
 s_cls_i32 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x0c,0x85,0xbe]
+// GFX12: s_cls_i32 s5, vcc_lo                    ; encoding: [0x6a,0x0c,0x85,0xbe]
 
 s_cls_i32 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x0c,0x85,0xbe]
+// GFX12: s_cls_i32 s5, vcc_hi                    ; encoding: [0x6b,0x0c,0x85,0xbe]
 
 s_cls_i32 s5, ttmp15
-// GFX12: encoding: [0x7b,0x0c,0x85,0xbe]
+// GFX12: s_cls_i32 s5, ttmp15                    ; encoding: [0x7b,0x0c,0x85,0xbe]
 
 s_cls_i32 s105, m0
-// GFX12: encoding: [0x7d,0x0c,0xe9,0xbe]
+// GFX12: s_cls_i32 s105, m0                      ; encoding: [0x7d,0x0c,0xe9,0xbe]
 
 s_cls_i32 vcc_lo, exec_lo
-// GFX12: encoding: [0x7e,0x0c,0xea,0xbe]
+// GFX12: s_cls_i32 vcc_lo, exec_lo               ; encoding: [0x7e,0x0c,0xea,0xbe]
 
 s_cls_i32 vcc_hi, exec_hi
-// GFX12: encoding: [0x7f,0x0c,0xeb,0xbe]
+// GFX12: s_cls_i32 vcc_hi, exec_hi               ; encoding: [0x7f,0x0c,0xeb,0xbe]
 
 s_cls_i32 ttmp15, null
-// GFX12: encoding: [0x7c,0x0c,0xfb,0xbe]
+// GFX12: s_cls_i32 ttmp15, null                  ; encoding: [0x7c,0x0c,0xfb,0xbe]
 
 s_cls_i32 m0, -1
-// GFX12: encoding: [0xc1,0x0c,0xfd,0xbe]
+// GFX12: s_cls_i32 m0, -1                        ; encoding: [0xc1,0x0c,0xfd,0xbe]
 
 s_cls_i32 exec_lo, 0.5
-// GFX12: encoding: [0xf0,0x0c,0xfe,0xbe]
+// GFX12: s_cls_i32 exec_lo, 0.5                  ; encoding: [0xf0,0x0c,0xfe,0xbe]
 
 s_cls_i32 exec_hi, src_scc
-// GFX12: encoding: [0xfd,0x0c,0xff,0xbe]
+// GFX12: s_cls_i32 exec_hi, src_scc              ; encoding: [0xfd,0x0c,0xff,0xbe]
 
 s_cls_i32 null, 0xaf123456
-// GFX12: encoding: [0xff,0x0c,0xfc,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_cls_i32 null, 0xaf123456              ; encoding: [0xff,0x0c,0xfc,0xbe,0x56,0x34,0x12,0xaf]
 
 s_cls_i32_i64 s5, s[2:3]
-// GFX12: encoding: [0x02,0x0d,0x85,0xbe]
+// GFX12: s_cls_i32_i64 s5, s[2:3]                ; encoding: [0x02,0x0d,0x85,0xbe]
 
 s_cls_i32_i64 s5, s[104:105]
-// GFX12: encoding: [0x68,0x0d,0x85,0xbe]
+// GFX12: s_cls_i32_i64 s5, s[104:105]            ; encoding: [0x68,0x0d,0x85,0xbe]
 
 s_cls_i32_i64 s105, vcc
-// GFX12: encoding: [0x6a,0x0d,0xe9,0xbe]
+// GFX12: s_cls_i32_i64 s105, vcc                 ; encoding: [0x6a,0x0d,0xe9,0xbe]
 
 s_cls_i32_i64 vcc_lo, ttmp[14:15]
-// GFX12: encoding: [0x7a,0x0d,0xea,0xbe]
+// GFX12: s_cls_i32_i64 vcc_lo, ttmp[14:15]       ; encoding: [0x7a,0x0d,0xea,0xbe]
 
 s_cls_i32_i64 vcc_hi, exec
-// GFX12: encoding: [0x7e,0x0d,0xeb,0xbe]
+// GFX12: s_cls_i32_i64 vcc_hi, exec              ; encoding: [0x7e,0x0d,0xeb,0xbe]
 
 s_cls_i32_i64 ttmp15, null
-// GFX12: encoding: [0x7c,0x0d,0xfb,0xbe]
+// GFX12: s_cls_i32_i64 ttmp15, null              ; encoding: [0x7c,0x0d,0xfb,0xbe]
 
 s_cls_i32_i64 m0, -1
-// GFX12: encoding: [0xc1,0x0d,0xfd,0xbe]
+// GFX12: s_cls_i32_i64 m0, -1                    ; encoding: [0xc1,0x0d,0xfd,0xbe]
 
 s_cls_i32_i64 exec_lo, 0.5
-// GFX12: encoding: [0xf0,0x0d,0xfe,0xbe]
+// GFX12: s_cls_i32_i64 exec_lo, 0.5              ; encoding: [0xf0,0x0d,0xfe,0xbe]
 
 s_cls_i32_i64 exec_hi, src_scc
-// GFX12: encoding: [0xfd,0x0d,0xff,0xbe]
+// GFX12: s_cls_i32_i64 exec_hi, src_scc          ; encoding: [0xfd,0x0d,0xff,0xbe]
 
 s_cls_i32_i64 null, 0xaf123456
-// GFX12: encoding: [0xff,0x0d,0xfc,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_cls_i32_i64 null, 0xaf123456          ; encoding: [0xff,0x0d,0xfc,0xbe,0x56,0x34,0x12,0xaf]
 
 s_clz_i32_u32 s5, s1
-// GFX12: encoding: [0x01,0x0a,0x85,0xbe]
+// GFX12: s_clz_i32_u32 s5, s1                    ; encoding: [0x01,0x0a,0x85,0xbe]
 
 s_clz_i32_u32 s5, s105
-// GFX12: encoding: [0x69,0x0a,0x85,0xbe]
+// GFX12: s_clz_i32_u32 s5, s105                  ; encoding: [0x69,0x0a,0x85,0xbe]
 
 s_clz_i32_u32 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x0a,0x85,0xbe]
+// GFX12: s_clz_i32_u32 s5, vcc_lo                ; encoding: [0x6a,0x0a,0x85,0xbe]
 
 s_clz_i32_u32 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x0a,0x85,0xbe]
+// GFX12: s_clz_i32_u32 s5, vcc_hi                ; encoding: [0x6b,0x0a,0x85,0xbe]
 
 s_clz_i32_u32 s5, ttmp15
-// GFX12: encoding: [0x7b,0x0a,0x85,0xbe]
+// GFX12: s_clz_i32_u32 s5, ttmp15                ; encoding: [0x7b,0x0a,0x85,0xbe]
 
 s_clz_i32_u32 s105, m0
-// GFX12: encoding: [0x7d,0x0a,0xe9,0xbe]
+// GFX12: s_clz_i32_u32 s105, m0                  ; encoding: [0x7d,0x0a,0xe9,0xbe]
 
 s_clz_i32_u32 vcc_lo, exec_lo
-// GFX12: encoding: [0x7e,0x0a,0xea,0xbe]
+// GFX12: s_clz_i32_u32 vcc_lo, exec_lo           ; encoding: [0x7e,0x0a,0xea,0xbe]
 
 s_clz_i32_u32 vcc_hi, exec_hi
-// GFX12: encoding: [0x7f,0x0a,0xeb,0xbe]
+// GFX12: s_clz_i32_u32 vcc_hi, exec_hi           ; encoding: [0x7f,0x0a,0xeb,0xbe]
 
 s_clz_i32_u32 ttmp15, null
-// GFX12: encoding: [0x7c,0x0a,0xfb,0xbe]
+// GFX12: s_clz_i32_u32 ttmp15, null              ; encoding: [0x7c,0x0a,0xfb,0xbe]
 
 s_clz_i32_u32 m0, -1
-// GFX12: encoding: [0xc1,0x0a,0xfd,0xbe]
+// GFX12: s_clz_i32_u32 m0, -1                    ; encoding: [0xc1,0x0a,0xfd,0xbe]
 
 s_clz_i32_u32 exec_lo, 0.5
-// GFX12: encoding: [0xf0,0x0a,0xfe,0xbe]
+// GFX12: s_clz_i32_u32 exec_lo, 0.5              ; encoding: [0xf0,0x0a,0xfe,0xbe]
 
 s_clz_i32_u32 exec_hi, src_scc
-// GFX12: encoding: [0xfd,0x0a,0xff,0xbe]
+// GFX12: s_clz_i32_u32 exec_hi, src_scc          ; encoding: [0xfd,0x0a,0xff,0xbe]
 
 s_clz_i32_u32 null, 0xaf123456
-// GFX12: encoding: [0xff,0x0a,0xfc,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_clz_i32_u32 null, 0xaf123456          ; encoding: [0xff,0x0a,0xfc,0xbe,0x56,0x34,0x12,0xaf]
 
 s_clz_i32_u64 s5, s[2:3]
-// GFX12: encoding: [0x02,0x0b,0x85,0xbe]
+// GFX12: s_clz_i32_u64 s5, s[2:3]                ; encoding: [0x02,0x0b,0x85,0xbe]
 
 s_clz_i32_u64 s5, s[104:105]
-// GFX12: encoding: [0x68,0x0b,0x85,0xbe]
+// GFX12: s_clz_i32_u64 s5, s[104:105]            ; encoding: [0x68,0x0b,0x85,0xbe]
 
 s_clz_i32_u64 s105, vcc
-// GFX12: encoding: [0x6a,0x0b,0xe9,0xbe]
+// GFX12: s_clz_i32_u64 s105, vcc                 ; encoding: [0x6a,0x0b,0xe9,0xbe]
 
 s_clz_i32_u64 vcc_lo, ttmp[14:15]
-// GFX12: encoding: [0x7a,0x0b,0xea,0xbe]
+// GFX12: s_clz_i32_u64 vcc_lo, ttmp[14:15]       ; encoding: [0x7a,0x0b,0xea,0xbe]
 
 s_clz_i32_u64 vcc_hi, exec
-// GFX12: encoding: [0x7e,0x0b,0xeb,0xbe]
+// GFX12: s_clz_i32_u64 vcc_hi, exec              ; encoding: [0x7e,0x0b,0xeb,0xbe]
 
 s_clz_i32_u64 ttmp15, null
-// GFX12: encoding: [0x7c,0x0b,0xfb,0xbe]
+// GFX12: s_clz_i32_u64 ttmp15, null              ; encoding: [0x7c,0x0b,0xfb,0xbe]
 
 s_clz_i32_u64 m0, -1
-// GFX12: encoding: [0xc1,0x0b,0xfd,0xbe]
+// GFX12: s_clz_i32_u64 m0, -1                    ; encoding: [0xc1,0x0b,0xfd,0xbe]
 
 s_clz_i32_u64 exec_lo, 0.5
-// GFX12: encoding: [0xf0,0x0b,0xfe,0xbe]
+// GFX12: s_clz_i32_u64 exec_lo, 0.5              ; encoding: [0xf0,0x0b,0xfe,0xbe]
 
 s_clz_i32_u64 exec_hi, src_scc
-// GFX12: encoding: [0xfd,0x0b,0xff,0xbe]
+// GFX12: s_clz_i32_u64 exec_hi, src_scc          ; encoding: [0xfd,0x0b,0xff,0xbe]
 
 s_clz_i32_u64 null, 0xaf123456
-// GFX12: encoding: [0xff,0x0b,0xfc,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_clz_i32_u64 null, 0xaf123456          ; encoding: [0xff,0x0b,0xfc,0xbe,0x56,0x34,0x12,0xaf]
 
 s_or_not0_saveexec_b32 s5, s1
-// GFX12: encoding: [0x01,0x2e,0x85,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s5, s1           ; encoding: [0x01,0x2e,0x85,0xbe]
 
 s_or_not0_saveexec_b32 s5, s105
-// GFX12: encoding: [0x69,0x2e,0x85,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s5, s105         ; encoding: [0x69,0x2e,0x85,0xbe]
 
 s_or_not0_saveexec_b32 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x2e,0x85,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s5, vcc_lo       ; encoding: [0x6a,0x2e,0x85,0xbe]
 
 s_or_not0_saveexec_b32 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x2e,0x85,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s5, vcc_hi       ; encoding: [0x6b,0x2e,0x85,0xbe]
 
 s_or_not0_saveexec_b32 s5, ttmp15
-// GFX12: encoding: [0x7b,0x2e,0x85,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s5, ttmp15       ; encoding: [0x7b,0x2e,0x85,0xbe]
 
 s_or_not0_saveexec_b32 s5, exec_lo
-// GFX12: encoding: [0x7e,0x2e,0x85,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s5, exec_lo      ; encoding: [0x7e,0x2e,0x85,0xbe]
 
 s_or_not0_saveexec_b32 s105, null
-// GFX12: encoding: [0x7c,0x2e,0xe9,0xbe]
+// GFX12: s_or_not0_saveexec_b32 s105, null       ; encoding: [0x7c,0x2e,0xe9,0xbe]
 
 s_or_not0_saveexec_b32 vcc_lo, -1
-// GFX12: encoding: [0xc1,0x2e,0xea,0xbe]
+// GFX12: s_or_not0_saveexec_b32 vcc_lo, -1       ; encoding: [0xc1,0x2e,0xea,0xbe]
 
 s_or_not0_saveexec_b32 vcc_hi, 0.5
-// GFX12: encoding: [0xf0,0x2e,0xeb,0xbe]
+// GFX12: s_or_not0_saveexec_b32 vcc_hi, 0.5      ; encoding: [0xf0,0x2e,0xeb,0xbe]
 
 s_or_not0_saveexec_b32 ttmp15, src_scc
-// GFX12: encoding: [0xfd,0x2e,0xfb,0xbe]
+// GFX12: s_or_not0_saveexec_b32 ttmp15, src_scc  ; encoding: [0xfd,0x2e,0xfb,0xbe]
 
 s_or_not0_saveexec_b32 null, 0xaf123456
-// GFX12: encoding: [0xff,0x2e,0xfc,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_not0_saveexec_b32 null, 0xaf123456 ; encoding: [0xff,0x2e,0xfc,0xbe,0x56,0x34,0x12,0xaf]
 
 s_or_not0_saveexec_b64 s[10:11], s[2:3]
-// GFX12: encoding: [0x02,0x2f,0x8a,0xbe]
+// GFX12: s_or_not0_saveexec_b64 s[10:11], s[2:3] ; encoding: [0x02,0x2f,0x8a,0xbe]
 
 s_or_not0_saveexec_b64 s[10:11], s[104:105]
-// GFX12: encoding: [0x68,0x2f,0x8a,0xbe]
+// GFX12: s_or_not0_saveexec_b64 s[10:11], s[104:105] ; encoding: [0x68,0x2f,0x8a,0xbe]
 
 s_or_not0_saveexec_b64 s[10:11], vcc
-// GFX12: encoding: [0x6a,0x2f,0x8a,0xbe]
+// GFX12: s_or_not0_saveexec_b64 s[10:11], vcc    ; encoding: [0x6a,0x2f,0x8a,0xbe]
 
 s_or_not0_saveexec_b64 s[10:11], ttmp[14:15]
-// GFX12: encoding: [0x7a,0x2f,0x8a,0xbe]
+// GFX12: s_or_not0_saveexec_b64 s[10:11], ttmp[14:15] ; encoding: [0x7a,0x2f,0x8a,0xbe]
 
 s_or_not0_saveexec_b64 s[10:11], null
-// GFX12: encoding: [0x7c,0x2f,0x8a,0xbe]
+// GFX12: s_or_not0_saveexec_b64 s[10:11], null   ; encoding: [0x7c,0x2f,0x8a,0xbe]
 
 s_or_not0_saveexec_b64 s[104:105], -1
-// GFX12: encoding: [0xc1,0x2f,0xe8,0xbe]
+// GFX12: s_or_not0_saveexec_b64 s[104:105], -1   ; encoding: [0xc1,0x2f,0xe8,0xbe]
 
 s_or_not0_saveexec_b64 vcc, 0.5
-// GFX12: encoding: [0xf0,0x2f,0xea,0xbe]
+// GFX12: s_or_not0_saveexec_b64 vcc, 0.5         ; encoding: [0xf0,0x2f,0xea,0xbe]
 
 s_or_not0_saveexec_b64 ttmp[14:15], src_scc
-// GFX12: encoding: [0xfd,0x2f,0xfa,0xbe]
+// GFX12: s_or_not0_saveexec_b64 ttmp[14:15], src_scc ; encoding: [0xfd,0x2f,0xfa,0xbe]
 
 s_or_not0_saveexec_b64 null, 0xaf123456
-// GFX12: encoding: [0xff,0x2f,0xfc,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_not0_saveexec_b64 null, 0xaf123456 ; encoding: [0xff,0x2f,0xfc,0xbe,0x56,0x34,0x12,0xaf]
 
 s_or_not1_saveexec_b32 s5, s1
-// GFX12: encoding: [0x01,0x32,0x85,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s5, s1           ; encoding: [0x01,0x32,0x85,0xbe]
 
 s_or_not1_saveexec_b32 s5, s105
-// GFX12: encoding: [0x69,0x32,0x85,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s5, s105         ; encoding: [0x69,0x32,0x85,0xbe]
 
 s_or_not1_saveexec_b32 s5, vcc_lo
-// GFX12: encoding: [0x6a,0x32,0x85,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s5, vcc_lo       ; encoding: [0x6a,0x32,0x85,0xbe]
 
 s_or_not1_saveexec_b32 s5, vcc_hi
-// GFX12: encoding: [0x6b,0x32,0x85,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s5, vcc_hi       ; encoding: [0x6b,0x32,0x85,0xbe]
 
 s_or_not1_saveexec_b32 s5, ttmp15
-// GFX12: encoding: [0x7b,0x32,0x85,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s5, ttmp15       ; encoding: [0x7b,0x32,0x85,0xbe]
 
 s_or_not1_saveexec_b32 s5, exec_lo
-// GFX12: encoding: [0x7e,0x32,0x85,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s5, exec_lo      ; encoding: [0x7e,0x32,0x85,0xbe]
 
 s_or_not1_saveexec_b32 s105, null
-// GFX12: encoding: [0x7c,0x32,0xe9,0xbe]
+// GFX12: s_or_not1_saveexec_b32 s105, null       ; encoding: [0x7c,0x32,0xe9,0xbe]
 
 s_or_not1_saveexec_b32 vcc_lo, -1
-// GFX12: encoding: [0xc1,0x32,0xea,0xbe]
+// GFX12: s_or_not1_saveexec_b32 vcc_lo, -1       ; encoding: [0xc1,0x32,0xea,0xbe]
 
 s_or_not1_saveexec_b32 vcc_hi, 0.5
-// GFX12: encoding: [0xf0,0x32,0xeb,0xbe]
+// GFX12: s_or_not1_saveexec_b32 vcc_hi, 0.5      ; encoding: [0xf0,0x32,0xeb,0xbe]
 
 s_or_not1_saveexec_b32 ttmp15, src_scc
-// GFX12: encoding: [0xfd,0x32,0xfb,0xbe]
+// GFX12: s_or_not1_saveexec_b32 ttmp15, src_scc  ; encoding: [0xfd,0x32,0xfb,0xbe]
 
 s_or_not1_saveexec_b32 null, 0xaf123456
-// GFX12: encoding: [0xff,0x32,0xfc,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_not1_saveexec_b32 null, 0xaf123456 ; encoding: [0xff,0x32,0xfc,0xbe,0x56,0x34,0x12,0xaf]
 
 s_or_not1_saveexec_b64 s[10:11], s[2:3]
-// GFX12: encoding: [0x02,0x33,0x8a,0xbe]
+// GFX12: s_or_not1_saveexec_b64 s[10:11], s[2:3] ; encoding: [0x02,0x33,0x8a,0xbe]
 
 s_or_not1_saveexec_b64 s[10:11], s[104:105]
-// GFX12: encoding: [0x68,0x33,0x8a,0xbe]
+// GFX12: s_or_not1_saveexec_b64 s[10:11], s[104:105] ; encoding: [0x68,0x33,0x8a,0xbe]
 
 s_or_not1_saveexec_b64 s[10:11], vcc
-// GFX12: encoding: [0x6a,0x33,0x8a,0xbe]
+// GFX12: s_or_not1_saveexec_b64 s[10:11], vcc    ; encoding: [0x6a,0x33,0x8a,0xbe]
 
 s_or_not1_saveexec_b64 s[10:11], ttmp[14:15]
-// GFX12: encoding: [0x7a,0x33,0x8a,0xbe]
+// GFX12: s_or_not1_saveexec_b64 s[10:11], ttmp[14:15] ; encoding: [0x7a,0x33,0x8a,0xbe]
 
 s_or_not1_saveexec_b64 s[10:11], null
-// GFX12: encoding: [0x7c,0x33,0x8a,0xbe]
+// GFX12: s_or_not1_saveexec_b64 s[10:11], null   ; encoding: [0x7c,0x33,0x8a,0xbe]
 
 s_or_not1_saveexec_b64 s[104:105], -1
-// GFX12: encoding: [0xc1,0x33,0xe8,0xbe]
+// GFX12: s_or_not1_saveexec_b64 s[104:105], -1   ; encoding: [0xc1,0x33,0xe8,0xbe]
 
 s_or_not1_saveexec_b64 vcc, 0.5
-// GFX12: encoding: [0xf0,0x33,0xea,0xbe]
+// GFX12: s_or_not1_saveexec_b64 vcc, 0.5         ; encoding: [0xf0,0x33,0xea,0xbe]
 
 s_or_not1_saveexec_b64 ttmp[14:15], src_scc
-// GFX12: encoding: [0xfd,0x33,0xfa,0xbe]
+// GFX12: s_or_not1_saveexec_b64 ttmp[14:15], src_scc ; encoding: [0xfd,0x33,0xfa,0xbe]
 
 s_or_not1_saveexec_b64 null, 0xaf123456
-// GFX12: encoding: [0xff,0x33,0xfc,0xbe,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_not1_saveexec_b64 null, 0xaf123456 ; encoding: [0xff,0x33,0xfc,0xbe,0x56,0x34,0x12,0xaf]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sop2.s b/llvm/test/MC/AMDGPU/gfx12_asm_sop2.s
index c9b58d892fea2..bc762ed28f17f 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_sop2.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_sop2.s
@@ -1,6295 +1,6298 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 // RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck -check-prefix=GFX12 %s
 
 s_add_nc_u64 s[0:1], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0x80,0xa9]
+// GFX12: s_add_nc_u64 s[0:1], s[2:3], s[4:5]     ; encoding: [0x02,0x04,0x80,0xa9]
 
 s_add_nc_u64 s[100:101], s[102:103], s[104:105]
-// GFX12: encoding: [0x66,0x68,0xe4,0xa9]
+// GFX12: s_add_nc_u64 s[100:101], s[102:103], s[104:105] ; encoding: [0x66,0x68,0xe4,0xa9]
 
 s_add_nc_u64 s[0:1], s[104:105], s[102:103]
-// GFX12: encoding: [0x68,0x66,0x80,0xa9]
+// GFX12: s_add_nc_u64 s[0:1], s[104:105], s[102:103] ; encoding: [0x68,0x66,0x80,0xa9]
 
 s_add_nc_u64 s[104:105], s[0:1], s[102:103]
-// GFX12: encoding: [0x00,0x66,0xe8,0xa9]
+// GFX12: s_add_nc_u64 s[104:105], s[0:1], s[102:103] ; encoding: [0x00,0x66,0xe8,0xa9]
 
 s_add_nc_u64 s[104:105], s[102:103], s[2:3]
-// GFX12: encoding: [0x66,0x02,0xe8,0xa9]
+// GFX12: s_add_nc_u64 s[104:105], s[102:103], s[2:3] ; encoding: [0x66,0x02,0xe8,0xa9]
 
 s_add_nc_u64 s[104:105], s[0:1], s[2:3]
-// GFX12: encoding: [0x00,0x02,0xe8,0xa9]
+// GFX12: s_add_nc_u64 s[104:105], s[0:1], s[2:3] ; encoding: [0x00,0x02,0xe8,0xa9]
 
 s_add_nc_u64 s[0:1], s[102:103], s[2:3]
-// GFX12: encoding: [0x66,0x02,0x80,0xa9]
+// GFX12: s_add_nc_u64 s[0:1], s[102:103], s[2:3] ; encoding: [0x66,0x02,0x80,0xa9]
 
 s_add_nc_u64 s[0:1], s[2:3], s[102:103]
-// GFX12: encoding: [0x02,0x66,0x80,0xa9]
+// GFX12: s_add_nc_u64 s[0:1], s[2:3], s[102:103] ; encoding: [0x02,0x66,0x80,0xa9]
 
 s_add_nc_u64 exec, s[0:1], s[2:3]
-// GFX12: encoding: [0x00,0x02,0xfe,0xa9]
+// GFX12: s_add_nc_u64 exec, s[0:1], s[2:3]       ; encoding: [0x00,0x02,0xfe,0xa9]
 
 s_add_nc_u64 vcc, s[0:1], s[2:3]
-// GFX12: encoding: [0x00,0x02,0xea,0xa9]
+// GFX12: s_add_nc_u64 vcc, s[0:1], s[2:3]        ; encoding: [0x00,0x02,0xea,0xa9]
 
 s_add_nc_u64 s[0:1], exec, s[2:3]
-// GFX12: encoding: [0x7e,0x02,0x80,0xa9]
+// GFX12: s_add_nc_u64 s[0:1], exec, s[2:3]       ; encoding: [0x7e,0x02,0x80,0xa9]
 
 s_add_nc_u64 s[0:1], vcc, s[2:3]
-// GFX12: encoding: [0x6a,0x02,0x80,0xa9]
+// GFX12: s_add_nc_u64 s[0:1], vcc, s[2:3]        ; encoding: [0x6a,0x02,0x80,0xa9]
 
 s_add_nc_u64 s[0:1], 0, s[2:3]
-// GFX12: encoding: [0x80,0x02,0x80,0xa9]
+// GFX12: s_add_nc_u64 s[0:1], 0, s[2:3]          ; encoding: [0x80,0x02,0x80,0xa9]
 
 s_add_nc_u64 s[0:1], -1, s[2:3]
-// GFX12: encoding: [0xc1,0x02,0x80,0xa9]
+// GFX12: s_add_nc_u64 s[0:1], -1, s[2:3]         ; encoding: [0xc1,0x02,0x80,0xa9]
 
 s_add_nc_u64 s[0:1], 0.5, s[2:3]
-// GFX12: encoding: [0xf0,0x02,0x80,0xa9]
+// GFX12: s_add_nc_u64 s[0:1], 0.5, s[2:3]        ; encoding: [0xf0,0x02,0x80,0xa9]
 
 s_add_nc_u64 s[0:1], -4.0, s[2:3]
-// GFX12: encoding: [0xf7,0x02,0x80,0xa9]
+// GFX12: s_add_nc_u64 s[0:1], -4.0, s[2:3]       ; encoding: [0xf7,0x02,0x80,0xa9]
 
 s_add_nc_u64 s[0:1], 0x3f717273, s[2:3]
-// GFX12: encoding: [0xff,0x02,0x80,0xa9,0x73,0x72,0x71,0x3f]
+// GFX12: s_add_nc_u64 s[0:1], 0x3f717273, s[2:3] ; encoding: [0xff,0x02,0x80,0xa9,0x73,0x72,0x71,0x3f]
 
 s_add_nc_u64 s[0:1], 0xaf123456, s[2:3]
-// GFX12: encoding: [0xff,0x02,0x80,0xa9,0x56,0x34,0x12,0xaf]
+// GFX12: s_add_nc_u64 s[0:1], 0xaf123456, s[2:3] ; encoding: [0xff,0x02,0x80,0xa9,0x56,0x34,0x12,0xaf]
 
 s_add_nc_u64 s[0:1], s[2:3], exec
-// GFX12: encoding: [0x02,0x7e,0x80,0xa9]
+// GFX12: s_add_nc_u64 s[0:1], s[2:3], exec       ; encoding: [0x02,0x7e,0x80,0xa9]
 
 s_add_nc_u64 s[0:1], s[2:3], vcc
-// GFX12: encoding: [0x02,0x6a,0x80,0xa9]
+// GFX12: s_add_nc_u64 s[0:1], s[2:3], vcc        ; encoding: [0x02,0x6a,0x80,0xa9]
 
 s_add_nc_u64 s[0:1], s[2:3], 0
-// GFX12: encoding: [0x02,0x80,0x80,0xa9]
+// GFX12: s_add_nc_u64 s[0:1], s[2:3], 0          ; encoding: [0x02,0x80,0x80,0xa9]
 
 s_add_nc_u64 s[0:1], s[2:3], -1
-// GFX12: encoding: [0x02,0xc1,0x80,0xa9]
+// GFX12: s_add_nc_u64 s[0:1], s[2:3], -1         ; encoding: [0x02,0xc1,0x80,0xa9]
 
 s_add_nc_u64 s[0:1], s[2:3], 0.5
-// GFX12: encoding: [0x02,0xf0,0x80,0xa9]
+// GFX12: s_add_nc_u64 s[0:1], s[2:3], 0.5        ; encoding: [0x02,0xf0,0x80,0xa9]
 
 s_add_nc_u64 s[0:1], s[2:3], -4.0
-// GFX12: encoding: [0x02,0xf7,0x80,0xa9]
+// GFX12: s_add_nc_u64 s[0:1], s[2:3], -4.0       ; encoding: [0x02,0xf7,0x80,0xa9]
 
 s_add_nc_u64 s[0:1], s[2:3], 0x3f717273
-// GFX12: encoding: [0x02,0xff,0x80,0xa9,0x73,0x72,0x71,0x3f]
+// GFX12: s_add_nc_u64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0xa9,0x73,0x72,0x71,0x3f]
 
 s_add_nc_u64 s[0:1], s[2:3], 0xaf123456
-// GFX12: encoding: [0x02,0xff,0x80,0xa9,0x56,0x34,0x12,0xaf]
+// GFX12: s_add_nc_u64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0xa9,0x56,0x34,0x12,0xaf]
 
 s_sub_nc_u64 s[0:1], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0x00,0xaa]
+// GFX12: s_sub_nc_u64 s[0:1], s[2:3], s[4:5]     ; encoding: [0x02,0x04,0x00,0xaa]
 
 s_sub_nc_u64 s[100:101], s[102:103], s[104:105]
-// GFX12: encoding: [0x66,0x68,0x64,0xaa]
+// GFX12: s_sub_nc_u64 s[100:101], s[102:103], s[104:105] ; encoding: [0x66,0x68,0x64,0xaa]
 
 s_sub_nc_u64 s[0:1], s[104:105], s[102:103]
-// GFX12: encoding: [0x68,0x66,0x00,0xaa]
+// GFX12: s_sub_nc_u64 s[0:1], s[104:105], s[102:103] ; encoding: [0x68,0x66,0x00,0xaa]
 
 s_sub_nc_u64 s[104:105], s[0:1], s[102:103]
-// GFX12: encoding: [0x00,0x66,0x68,0xaa]
+// GFX12: s_sub_nc_u64 s[104:105], s[0:1], s[102:103] ; encoding: [0x00,0x66,0x68,0xaa]
 
 s_sub_nc_u64 s[104:105], s[102:103], s[2:3]
-// GFX12: encoding: [0x66,0x02,0x68,0xaa]
+// GFX12: s_sub_nc_u64 s[104:105], s[102:103], s[2:3] ; encoding: [0x66,0x02,0x68,0xaa]
 
 s_sub_nc_u64 s[104:105], s[0:1], s[2:3]
-// GFX12: encoding: [0x00,0x02,0x68,0xaa]
+// GFX12: s_sub_nc_u64 s[104:105], s[0:1], s[2:3] ; encoding: [0x00,0x02,0x68,0xaa]
 
 s_sub_nc_u64 s[0:1], s[102:103], s[2:3]
-// GFX12: encoding: [0x66,0x02,0x00,0xaa]
+// GFX12: s_sub_nc_u64 s[0:1], s[102:103], s[2:3] ; encoding: [0x66,0x02,0x00,0xaa]
 
 s_sub_nc_u64 s[0:1], s[2:3], s[102:103]
-// GFX12: encoding: [0x02,0x66,0x00,0xaa]
+// GFX12: s_sub_nc_u64 s[0:1], s[2:3], s[102:103] ; encoding: [0x02,0x66,0x00,0xaa]
 
 s_sub_nc_u64 exec, s[0:1], s[2:3]
-// GFX12: encoding: [0x00,0x02,0x7e,0xaa]
+// GFX12: s_sub_nc_u64 exec, s[0:1], s[2:3]       ; encoding: [0x00,0x02,0x7e,0xaa]
 
 s_sub_nc_u64 vcc, s[0:1], s[2:3]
-// GFX12: encoding: [0x00,0x02,0x6a,0xaa]
+// GFX12: s_sub_nc_u64 vcc, s[0:1], s[2:3]        ; encoding: [0x00,0x02,0x6a,0xaa]
 
 s_sub_nc_u64 s[0:1], exec, s[2:3]
-// GFX12: encoding: [0x7e,0x02,0x00,0xaa]
+// GFX12: s_sub_nc_u64 s[0:1], exec, s[2:3]       ; encoding: [0x7e,0x02,0x00,0xaa]
 
 s_sub_nc_u64 s[0:1], vcc, s[2:3]
-// GFX12: encoding: [0x6a,0x02,0x00,0xaa]
+// GFX12: s_sub_nc_u64 s[0:1], vcc, s[2:3]        ; encoding: [0x6a,0x02,0x00,0xaa]
 
 s_sub_nc_u64 s[0:1], 0, s[2:3]
-// GFX12: encoding: [0x80,0x02,0x00,0xaa]
+// GFX12: s_sub_nc_u64 s[0:1], 0, s[2:3]          ; encoding: [0x80,0x02,0x00,0xaa]
 
 s_sub_nc_u64 s[0:1], -1, s[2:3]
-// GFX12: encoding: [0xc1,0x02,0x00,0xaa]
+// GFX12: s_sub_nc_u64 s[0:1], -1, s[2:3]         ; encoding: [0xc1,0x02,0x00,0xaa]
 
 s_sub_nc_u64 s[0:1], 0.5, s[2:3]
-// GFX12: encoding: [0xf0,0x02,0x00,0xaa]
+// GFX12: s_sub_nc_u64 s[0:1], 0.5, s[2:3]        ; encoding: [0xf0,0x02,0x00,0xaa]
 
 s_sub_nc_u64 s[0:1], -4.0, s[2:3]
-// GFX12: encoding: [0xf7,0x02,0x00,0xaa]
+// GFX12: s_sub_nc_u64 s[0:1], -4.0, s[2:3]       ; encoding: [0xf7,0x02,0x00,0xaa]
 
 s_sub_nc_u64 s[0:1], 0x3f717273, s[2:3]
-// GFX12: encoding: [0xff,0x02,0x00,0xaa,0x73,0x72,0x71,0x3f]
+// GFX12: s_sub_nc_u64 s[0:1], 0x3f717273, s[2:3] ; encoding: [0xff,0x02,0x00,0xaa,0x73,0x72,0x71,0x3f]
 
 s_sub_nc_u64 s[0:1], 0xaf123456, s[2:3]
-// GFX12: encoding: [0xff,0x02,0x00,0xaa,0x56,0x34,0x12,0xaf]
+// GFX12: s_sub_nc_u64 s[0:1], 0xaf123456, s[2:3] ; encoding: [0xff,0x02,0x00,0xaa,0x56,0x34,0x12,0xaf]
 
 s_sub_nc_u64 s[0:1], s[2:3], exec
-// GFX12: encoding: [0x02,0x7e,0x00,0xaa]
+// GFX12: s_sub_nc_u64 s[0:1], s[2:3], exec       ; encoding: [0x02,0x7e,0x00,0xaa]
 
 s_sub_nc_u64 s[0:1], s[2:3], vcc
-// GFX12: encoding: [0x02,0x6a,0x00,0xaa]
+// GFX12: s_sub_nc_u64 s[0:1], s[2:3], vcc        ; encoding: [0x02,0x6a,0x00,0xaa]
 
 s_sub_nc_u64 s[0:1], s[2:3], 0
-// GFX12: encoding: [0x02,0x80,0x00,0xaa]
+// GFX12: s_sub_nc_u64 s[0:1], s[2:3], 0          ; encoding: [0x02,0x80,0x00,0xaa]
 
 s_sub_nc_u64 s[0:1], s[2:3], -1
-// GFX12: encoding: [0x02,0xc1,0x00,0xaa]
+// GFX12: s_sub_nc_u64 s[0:1], s[2:3], -1         ; encoding: [0x02,0xc1,0x00,0xaa]
 
 s_sub_nc_u64 s[0:1], s[2:3], 0.5
-// GFX12: encoding: [0x02,0xf0,0x00,0xaa]
+// GFX12: s_sub_nc_u64 s[0:1], s[2:3], 0.5        ; encoding: [0x02,0xf0,0x00,0xaa]
 
 s_sub_nc_u64 s[0:1], s[2:3], -4.0
-// GFX12: encoding: [0x02,0xf7,0x00,0xaa]
+// GFX12: s_sub_nc_u64 s[0:1], s[2:3], -4.0       ; encoding: [0x02,0xf7,0x00,0xaa]
 
 s_sub_nc_u64 s[0:1], s[2:3], 0x3f717273
-// GFX12: encoding: [0x02,0xff,0x00,0xaa,0x73,0x72,0x71,0x3f]
+// GFX12: s_sub_nc_u64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x00,0xaa,0x73,0x72,0x71,0x3f]
 
 s_sub_nc_u64 s[0:1], s[2:3], 0xaf123456
-// GFX12: encoding: [0x02,0xff,0x00,0xaa,0x56,0x34,0x12,0xaf]
+// GFX12: s_sub_nc_u64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x00,0xaa,0x56,0x34,0x12,0xaf]
 
 s_mul_u64 s[0:1], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0x80,0xaa]
+// GFX12: s_mul_u64 s[0:1], s[2:3], s[4:5]        ; encoding: [0x02,0x04,0x80,0xaa]
 
 s_mul_u64 s[100:101], s[102:103], s[104:105]
-// GFX12: encoding: [0x66,0x68,0xe4,0xaa]
+// GFX12: s_mul_u64 s[100:101], s[102:103], s[104:105] ; encoding: [0x66,0x68,0xe4,0xaa]
 
 s_mul_u64 s[0:1], s[104:105], s[102:103]
-// GFX12: encoding: [0x68,0x66,0x80,0xaa]
+// GFX12: s_mul_u64 s[0:1], s[104:105], s[102:103] ; encoding: [0x68,0x66,0x80,0xaa]
 
 s_mul_u64 s[104:105], s[0:1], s[102:103]
-// GFX12: encoding: [0x00,0x66,0xe8,0xaa]
+// GFX12: s_mul_u64 s[104:105], s[0:1], s[102:103] ; encoding: [0x00,0x66,0xe8,0xaa]
 
 s_mul_u64 s[104:105], s[102:103], s[2:3]
-// GFX12: encoding: [0x66,0x02,0xe8,0xaa]
+// GFX12: s_mul_u64 s[104:105], s[102:103], s[2:3] ; encoding: [0x66,0x02,0xe8,0xaa]
 
 s_mul_u64 s[104:105], s[0:1], s[2:3]
-// GFX12: encoding: [0x00,0x02,0xe8,0xaa]
+// GFX12: s_mul_u64 s[104:105], s[0:1], s[2:3]    ; encoding: [0x00,0x02,0xe8,0xaa]
 
 s_mul_u64 s[0:1], s[102:103], s[2:3]
-// GFX12: encoding: [0x66,0x02,0x80,0xaa]
+// GFX12: s_mul_u64 s[0:1], s[102:103], s[2:3]    ; encoding: [0x66,0x02,0x80,0xaa]
 
 s_mul_u64 s[0:1], s[2:3], s[102:103]
-// GFX12: encoding: [0x02,0x66,0x80,0xaa]
+// GFX12: s_mul_u64 s[0:1], s[2:3], s[102:103]    ; encoding: [0x02,0x66,0x80,0xaa]
 
 s_mul_u64 exec, s[0:1], s[2:3]
-// GFX12: encoding: [0x00,0x02,0xfe,0xaa]
+// GFX12: s_mul_u64 exec, s[0:1], s[2:3]          ; encoding: [0x00,0x02,0xfe,0xaa]
 
 s_mul_u64 vcc, s[0:1], s[2:3]
-// GFX12: encoding: [0x00,0x02,0xea,0xaa]
+// GFX12: s_mul_u64 vcc, s[0:1], s[2:3]           ; encoding: [0x00,0x02,0xea,0xaa]
 
 s_mul_u64 s[0:1], exec, s[2:3]
-// GFX12: encoding: [0x7e,0x02,0x80,0xaa]
+// GFX12: s_mul_u64 s[0:1], exec, s[2:3]          ; encoding: [0x7e,0x02,0x80,0xaa]
 
 s_mul_u64 s[0:1], vcc, s[2:3]
-// GFX12: encoding: [0x6a,0x02,0x80,0xaa]
+// GFX12: s_mul_u64 s[0:1], vcc, s[2:3]           ; encoding: [0x6a,0x02,0x80,0xaa]
 
 s_mul_u64 s[0:1], 0, s[2:3]
-// GFX12: encoding: [0x80,0x02,0x80,0xaa]
+// GFX12: s_mul_u64 s[0:1], 0, s[2:3]             ; encoding: [0x80,0x02,0x80,0xaa]
 
 s_mul_u64 s[0:1], -1, s[2:3]
-// GFX12: encoding: [0xc1,0x02,0x80,0xaa]
+// GFX12: s_mul_u64 s[0:1], -1, s[2:3]            ; encoding: [0xc1,0x02,0x80,0xaa]
 
 s_mul_u64 s[0:1], 0.5, s[2:3]
-// GFX12: encoding: [0xf0,0x02,0x80,0xaa]
+// GFX12: s_mul_u64 s[0:1], 0.5, s[2:3]           ; encoding: [0xf0,0x02,0x80,0xaa]
 
 s_mul_u64 s[0:1], -4.0, s[2:3]
-// GFX12: encoding: [0xf7,0x02,0x80,0xaa]
+// GFX12: s_mul_u64 s[0:1], -4.0, s[2:3]          ; encoding: [0xf7,0x02,0x80,0xaa]
 
 s_mul_u64 s[0:1], 0x3f717273, s[2:3]
-// GFX12: encoding: [0xff,0x02,0x80,0xaa,0x73,0x72,0x71,0x3f]
+// GFX12: s_mul_u64 s[0:1], 0x3f717273, s[2:3]    ; encoding: [0xff,0x02,0x80,0xaa,0x73,0x72,0x71,0x3f]
 
 s_mul_u64 s[0:1], 0xaf123456, s[2:3]
-// GFX12: encoding: [0xff,0x02,0x80,0xaa,0x56,0x34,0x12,0xaf]
+// GFX12: s_mul_u64 s[0:1], 0xaf123456, s[2:3]    ; encoding: [0xff,0x02,0x80,0xaa,0x56,0x34,0x12,0xaf]
 
 s_mul_u64 s[0:1], s[2:3], exec
-// GFX12: encoding: [0x02,0x7e,0x80,0xaa]
+// GFX12: s_mul_u64 s[0:1], s[2:3], exec          ; encoding: [0x02,0x7e,0x80,0xaa]
 
 s_mul_u64 s[0:1], s[2:3], vcc
-// GFX12: encoding: [0x02,0x6a,0x80,0xaa]
+// GFX12: s_mul_u64 s[0:1], s[2:3], vcc           ; encoding: [0x02,0x6a,0x80,0xaa]
 
 s_mul_u64 s[0:1], s[2:3], 0
-// GFX12: encoding: [0x02,0x80,0x80,0xaa]
+// GFX12: s_mul_u64 s[0:1], s[2:3], 0             ; encoding: [0x02,0x80,0x80,0xaa]
 
 s_mul_u64 s[0:1], s[2:3], -1
-// GFX12: encoding: [0x02,0xc1,0x80,0xaa]
+// GFX12: s_mul_u64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x80,0xaa]
 
 s_mul_u64 s[0:1], s[2:3], 0.5
-// GFX12: encoding: [0x02,0xf0,0x80,0xaa]
+// GFX12: s_mul_u64 s[0:1], s[2:3], 0.5           ; encoding: [0x02,0xf0,0x80,0xaa]
 
 s_mul_u64 s[0:1], s[2:3], -4.0
-// GFX12: encoding: [0x02,0xf7,0x80,0xaa]
+// GFX12: s_mul_u64 s[0:1], s[2:3], -4.0          ; encoding: [0x02,0xf7,0x80,0xaa]
 
 s_mul_u64 s[0:1], s[2:3], 0x3f717273
-// GFX12: encoding: [0x02,0xff,0x80,0xaa,0x73,0x72,0x71,0x3f]
+// GFX12: s_mul_u64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0xaa,0x73,0x72,0x71,0x3f]
 
 s_mul_u64 s[0:1], s[2:3], 0xaf123456
-// GFX12: encoding: [0x02,0xff,0x80,0xaa,0x56,0x34,0x12,0xaf]
+// GFX12: s_mul_u64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0xaa,0x56,0x34,0x12,0xaf]
 
 s_add_f32 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x05,0xa0]
+// GFX12: s_add_f32 s5, s1, s2                    ; encoding: [0x01,0x02,0x05,0xa0]
 
 s_add_f32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0xa0]
+// GFX12: s_add_f32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0xa0]
 
 s_add_f32 s5, s105, s2
-// GFX12: encoding: [0x69,0x02,0x05,0xa0]
+// GFX12: s_add_f32 s5, s105, s2                  ; encoding: [0x69,0x02,0x05,0xa0]
 
 s_add_f32 s5, s101, s2
-// GFX12: encoding: [0x65,0x02,0x05,0xa0]
+// GFX12: s_add_f32 s5, s101, s2                  ; encoding: [0x65,0x02,0x05,0xa0]
 
 s_add_f32 s5, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x05,0xa0]
+// GFX12: s_add_f32 s5, vcc_lo, s2                ; encoding: [0x6a,0x02,0x05,0xa0]
 
 s_add_f32 s5, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x05,0xa0]
+// GFX12: s_add_f32 s5, vcc_hi, s2                ; encoding: [0x6b,0x02,0x05,0xa0]
 
 s_add_f32 s5, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x05,0xa0]
+// GFX12: s_add_f32 s5, m0, s2                    ; encoding: [0x7d,0x02,0x05,0xa0]
 
 s_add_f32 s5, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x05,0xa0]
+// GFX12: s_add_f32 s5, exec_lo, s2               ; encoding: [0x7e,0x02,0x05,0xa0]
 
 s_add_f32 s5, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x05,0xa0]
+// GFX12: s_add_f32 s5, exec_hi, s2               ; encoding: [0x7f,0x02,0x05,0xa0]
 
 s_add_f32 s5, 0, s2
-// GFX12: encoding: [0x80,0x02,0x05,0xa0]
+// GFX12: s_add_f32 s5, 0, s2                     ; encoding: [0x80,0x02,0x05,0xa0]
 
 s_add_f32 s5, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x05,0xa0]
+// GFX12: s_add_f32 s5, -1, s2                    ; encoding: [0xc1,0x02,0x05,0xa0]
 
 s_add_f32 s5, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x05,0xa0]
+// GFX12: s_add_f32 s5, 0.5, s2                   ; encoding: [0xf0,0x02,0x05,0xa0]
 
 s_add_f32 s5, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x05,0xa0]
+// GFX12: s_add_f32 s5, -4.0, s2                  ; encoding: [0xf7,0x02,0x05,0xa0]
 
 s_add_f32 s5, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x05,0xa0,0x56,0x34,0x12,0xaf]
+// GFX12: s_add_f32 s5, 0xaf123456, s2            ; encoding: [0xff,0x02,0x05,0xa0,0x56,0x34,0x12,0xaf]
 
 s_add_f32 s5, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x05,0xa0,0x73,0x72,0x71,0x3f]
+// GFX12: s_add_f32 s5, 0x3f717273, s2            ; encoding: [0xff,0x02,0x05,0xa0,0x73,0x72,0x71,0x3f]
 
 s_add_f32 s5, s1, s105
-// GFX12: encoding: [0x01,0x69,0x05,0xa0]
+// GFX12: s_add_f32 s5, s1, s105                  ; encoding: [0x01,0x69,0x05,0xa0]
 
 s_sub_f32 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x85,0xa0]
+// GFX12: s_sub_f32 s5, s1, s2                    ; encoding: [0x01,0x02,0x85,0xa0]
 
 s_sub_f32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0xe9,0xa0]
+// GFX12: s_sub_f32 s105, s1, s2                  ; encoding: [0x01,0x02,0xe9,0xa0]
 
 s_sub_f32 s5, s105, s2
-// GFX12: encoding: [0x69,0x02,0x85,0xa0]
+// GFX12: s_sub_f32 s5, s105, s2                  ; encoding: [0x69,0x02,0x85,0xa0]
 
 s_sub_f32 s5, s101, s2
-// GFX12: encoding: [0x65,0x02,0x85,0xa0]
+// GFX12: s_sub_f32 s5, s101, s2                  ; encoding: [0x65,0x02,0x85,0xa0]
 
 s_sub_f32 s5, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x85,0xa0]
+// GFX12: s_sub_f32 s5, vcc_lo, s2                ; encoding: [0x6a,0x02,0x85,0xa0]
 
 s_sub_f32 s5, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x85,0xa0]
+// GFX12: s_sub_f32 s5, vcc_hi, s2                ; encoding: [0x6b,0x02,0x85,0xa0]
 
 s_sub_f32 s5, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x85,0xa0]
+// GFX12: s_sub_f32 s5, m0, s2                    ; encoding: [0x7d,0x02,0x85,0xa0]
 
 s_sub_f32 s5, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x85,0xa0]
+// GFX12: s_sub_f32 s5, exec_lo, s2               ; encoding: [0x7e,0x02,0x85,0xa0]
 
 s_sub_f32 s5, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x85,0xa0]
+// GFX12: s_sub_f32 s5, exec_hi, s2               ; encoding: [0x7f,0x02,0x85,0xa0]
 
 s_sub_f32 s5, 0, s2
-// GFX12: encoding: [0x80,0x02,0x85,0xa0]
+// GFX12: s_sub_f32 s5, 0, s2                     ; encoding: [0x80,0x02,0x85,0xa0]
 
 s_sub_f32 s5, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x85,0xa0]
+// GFX12: s_sub_f32 s5, -1, s2                    ; encoding: [0xc1,0x02,0x85,0xa0]
 
 s_sub_f32 s5, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x85,0xa0]
+// GFX12: s_sub_f32 s5, 0.5, s2                   ; encoding: [0xf0,0x02,0x85,0xa0]
 
 s_sub_f32 s5, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x85,0xa0]
+// GFX12: s_sub_f32 s5, -4.0, s2                  ; encoding: [0xf7,0x02,0x85,0xa0]
 
 s_sub_f32 s5, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x85,0xa0,0x56,0x34,0x12,0xaf]
+// GFX12: s_sub_f32 s5, 0xaf123456, s2            ; encoding: [0xff,0x02,0x85,0xa0,0x56,0x34,0x12,0xaf]
 
 s_sub_f32 s5, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x85,0xa0,0x73,0x72,0x71,0x3f]
+// GFX12: s_sub_f32 s5, 0x3f717273, s2            ; encoding: [0xff,0x02,0x85,0xa0,0x73,0x72,0x71,0x3f]
 
 s_sub_f32 s5, s1, s105
-// GFX12: encoding: [0x01,0x69,0x85,0xa0]
+// GFX12: s_sub_f32 s5, s1, s105                  ; encoding: [0x01,0x69,0x85,0xa0]
 
 s_mul_f32 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x05,0xa2]
+// GFX12: s_mul_f32 s5, s1, s2                    ; encoding: [0x01,0x02,0x05,0xa2]
 
 s_mul_f32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0xa2]
+// GFX12: s_mul_f32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0xa2]
 
 s_mul_f32 s5, s105, s2
-// GFX12: encoding: [0x69,0x02,0x05,0xa2]
+// GFX12: s_mul_f32 s5, s105, s2                  ; encoding: [0x69,0x02,0x05,0xa2]
 
 s_mul_f32 s5, s103, s2
-// GFX12: encoding: [0x67,0x02,0x05,0xa2]
+// GFX12: s_mul_f32 s5, s103, s2                  ; encoding: [0x67,0x02,0x05,0xa2]
 
 s_mul_f32 s5, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x05,0xa2]
+// GFX12: s_mul_f32 s5, vcc_lo, s2                ; encoding: [0x6a,0x02,0x05,0xa2]
 
 s_mul_f32 s5, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x05,0xa2]
+// GFX12: s_mul_f32 s5, vcc_hi, s2                ; encoding: [0x6b,0x02,0x05,0xa2]
 
 s_mul_f32 s5, ttmp11, s2
-// GFX12: encoding: [0x77,0x02,0x05,0xa2]
+// GFX12: s_mul_f32 s5, ttmp11, s2                ; encoding: [0x77,0x02,0x05,0xa2]
 
 s_mul_f32 s5, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x05,0xa2]
+// GFX12: s_mul_f32 s5, m0, s2                    ; encoding: [0x7d,0x02,0x05,0xa2]
 
 s_mul_f32 s5, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x05,0xa2]
+// GFX12: s_mul_f32 s5, exec_lo, s2               ; encoding: [0x7e,0x02,0x05,0xa2]
 
 s_mul_f32 s5, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x05,0xa2]
+// GFX12: s_mul_f32 s5, exec_hi, s2               ; encoding: [0x7f,0x02,0x05,0xa2]
 
 s_mul_f32 s5, 0, s2
-// GFX12: encoding: [0x80,0x02,0x05,0xa2]
+// GFX12: s_mul_f32 s5, 0, s2                     ; encoding: [0x80,0x02,0x05,0xa2]
 
 s_mul_f32 s5, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x05,0xa2]
+// GFX12: s_mul_f32 s5, -1, s2                    ; encoding: [0xc1,0x02,0x05,0xa2]
 
 s_mul_f32 s5, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x05,0xa2]
+// GFX12: s_mul_f32 s5, 0.5, s2                   ; encoding: [0xf0,0x02,0x05,0xa2]
 
 s_mul_f32 s5, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x05,0xa2]
+// GFX12: s_mul_f32 s5, -4.0, s2                  ; encoding: [0xf7,0x02,0x05,0xa2]
 
 s_mul_f32 s5, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x05,0xa2,0x56,0x34,0x12,0xaf]
+// GFX12: s_mul_f32 s5, 0xaf123456, s2            ; encoding: [0xff,0x02,0x05,0xa2,0x56,0x34,0x12,0xaf]
 
 s_mul_f32 s5, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x05,0xa2,0x73,0x72,0x71,0x3f]
+// GFX12: s_mul_f32 s5, 0x3f717273, s2            ; encoding: [0xff,0x02,0x05,0xa2,0x73,0x72,0x71,0x3f]
 
 s_mul_f32 s5, s1, s105
-// GFX12: encoding: [0x01,0x69,0x05,0xa2]
+// GFX12: s_mul_f32 s5, s1, s105                  ; encoding: [0x01,0x69,0x05,0xa2]
 
 s_min_num_f32 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x05,0xa1]
+// GFX12: s_min_num_f32 s5, s1, s2                ; encoding: [0x01,0x02,0x05,0xa1]
 
 s_min_num_f32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0xa1]
+// GFX12: s_min_num_f32 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0xa1]
 
 s_min_num_f32 s5, s105, s2
-// GFX12: encoding: [0x69,0x02,0x05,0xa1]
+// GFX12: s_min_num_f32 s5, s105, s2              ; encoding: [0x69,0x02,0x05,0xa1]
 
 s_min_num_f32 s5, s103, s2
-// GFX12: encoding: [0x67,0x02,0x05,0xa1]
+// GFX12: s_min_num_f32 s5, s103, s2              ; encoding: [0x67,0x02,0x05,0xa1]
 
 s_min_num_f32 s5, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x05,0xa1]
+// GFX12: s_min_num_f32 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x05,0xa1]
 
 s_min_num_f32 s5, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x05,0xa1]
+// GFX12: s_min_num_f32 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x05,0xa1]
 
 s_min_num_f32 s5, ttmp11, s2
-// GFX12: encoding: [0x77,0x02,0x05,0xa1]
+// GFX12: s_min_num_f32 s5, ttmp11, s2            ; encoding: [0x77,0x02,0x05,0xa1]
 
 s_min_num_f32 s5, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x05,0xa1]
+// GFX12: s_min_num_f32 s5, m0, s2                ; encoding: [0x7d,0x02,0x05,0xa1]
 
 s_min_num_f32 s5, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x05,0xa1]
+// GFX12: s_min_num_f32 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x05,0xa1]
 
 s_min_num_f32 s5, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x05,0xa1]
+// GFX12: s_min_num_f32 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x05,0xa1]
 
 s_min_num_f32 s5, 0, s2
-// GFX12: encoding: [0x80,0x02,0x05,0xa1]
+// GFX12: s_min_num_f32 s5, 0, s2                 ; encoding: [0x80,0x02,0x05,0xa1]
 
 s_min_num_f32 s5, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x05,0xa1]
+// GFX12: s_min_num_f32 s5, -1, s2                ; encoding: [0xc1,0x02,0x05,0xa1]
 
 s_min_num_f32 s5, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x05,0xa1]
+// GFX12: s_min_num_f32 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x05,0xa1]
 
 s_min_num_f32 s5, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x05,0xa1]
+// GFX12: s_min_num_f32 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x05,0xa1]
 
 s_min_num_f32 s5, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x05,0xa1,0x56,0x34,0x12,0xaf]
+// GFX12: s_min_num_f32 s5, 0xaf123456, s2        ; encoding: [0xff,0x02,0x05,0xa1,0x56,0x34,0x12,0xaf]
 
 s_min_num_f32 s5, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x05,0xa1,0x73,0x72,0x71,0x3f]
+// GFX12: s_min_num_f32 s5, 0x3f717273, s2        ; encoding: [0xff,0x02,0x05,0xa1,0x73,0x72,0x71,0x3f]
 
 s_min_num_f32 s5, s1, s105
-// GFX12: encoding: [0x01,0x69,0x05,0xa1]
+// GFX12: s_min_num_f32 s5, s1, s105              ; encoding: [0x01,0x69,0x05,0xa1]
 
 s_max_num_f32 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x85,0xa1]
+// GFX12: s_max_num_f32 s5, s1, s2                ; encoding: [0x01,0x02,0x85,0xa1]
 
 s_max_num_f32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0xe9,0xa1]
+// GFX12: s_max_num_f32 s105, s1, s2              ; encoding: [0x01,0x02,0xe9,0xa1]
 
 s_max_num_f32 s5, s105, s2
-// GFX12: encoding: [0x69,0x02,0x85,0xa1]
+// GFX12: s_max_num_f32 s5, s105, s2              ; encoding: [0x69,0x02,0x85,0xa1]
 
 s_max_num_f32 s5, s103, s2
-// GFX12: encoding: [0x67,0x02,0x85,0xa1]
+// GFX12: s_max_num_f32 s5, s103, s2              ; encoding: [0x67,0x02,0x85,0xa1]
 
 s_max_num_f32 s5, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x85,0xa1]
+// GFX12: s_max_num_f32 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x85,0xa1]
 
 s_max_num_f32 s5, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x85,0xa1]
+// GFX12: s_max_num_f32 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x85,0xa1]
 
 s_max_num_f32 s5, ttmp11, s2
-// GFX12: encoding: [0x77,0x02,0x85,0xa1]
+// GFX12: s_max_num_f32 s5, ttmp11, s2            ; encoding: [0x77,0x02,0x85,0xa1]
 
 s_max_num_f32 s5, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x85,0xa1]
+// GFX12: s_max_num_f32 s5, m0, s2                ; encoding: [0x7d,0x02,0x85,0xa1]
 
 s_max_num_f32 s5, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x85,0xa1]
+// GFX12: s_max_num_f32 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x85,0xa1]
 
 s_max_num_f32 s5, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x85,0xa1]
+// GFX12: s_max_num_f32 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x85,0xa1]
 
 s_max_num_f32 s5, 0, s2
-// GFX12: encoding: [0x80,0x02,0x85,0xa1]
+// GFX12: s_max_num_f32 s5, 0, s2                 ; encoding: [0x80,0x02,0x85,0xa1]
 
 s_max_num_f32 s5, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x85,0xa1]
+// GFX12: s_max_num_f32 s5, -1, s2                ; encoding: [0xc1,0x02,0x85,0xa1]
 
 s_max_num_f32 s5, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x85,0xa1]
+// GFX12: s_max_num_f32 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x85,0xa1]
 
 s_max_num_f32 s5, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x85,0xa1]
+// GFX12: s_max_num_f32 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x85,0xa1]
 
 s_max_num_f32 s5, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x85,0xa1,0x56,0x34,0x12,0xaf]
+// GFX12: s_max_num_f32 s5, 0xaf123456, s2        ; encoding: [0xff,0x02,0x85,0xa1,0x56,0x34,0x12,0xaf]
 
 s_max_num_f32 s5, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x85,0xa1,0x73,0x72,0x71,0x3f]
+// GFX12: s_max_num_f32 s5, 0x3f717273, s2        ; encoding: [0xff,0x02,0x85,0xa1,0x73,0x72,0x71,0x3f]
 
 s_max_num_f32 s5, s1, s105
-// GFX12: encoding: [0x01,0x69,0x85,0xa1]
+// GFX12: s_max_num_f32 s5, s1, s105              ; encoding: [0x01,0x69,0x85,0xa1]
 
 s_minimum_f32 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x85,0xa7]
+// GFX12: s_minimum_f32 s5, s1, s2                ; encoding: [0x01,0x02,0x85,0xa7]
 
 s_minimum_f32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0xe9,0xa7]
+// GFX12: s_minimum_f32 s105, s1, s2              ; encoding: [0x01,0x02,0xe9,0xa7]
 
 s_minimum_f32 s5, s105, s2
-// GFX12: encoding: [0x69,0x02,0x85,0xa7]
+// GFX12: s_minimum_f32 s5, s105, s2              ; encoding: [0x69,0x02,0x85,0xa7]
 
 s_minimum_f32 s5, s103, s2
-// GFX12: encoding: [0x67,0x02,0x85,0xa7]
+// GFX12: s_minimum_f32 s5, s103, s2              ; encoding: [0x67,0x02,0x85,0xa7]
 
 s_minimum_f32 s5, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x85,0xa7]
+// GFX12: s_minimum_f32 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x85,0xa7]
 
 s_minimum_f32 s5, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x85,0xa7]
+// GFX12: s_minimum_f32 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x85,0xa7]
 
 s_minimum_f32 s5, ttmp11, s2
-// GFX12: encoding: [0x77,0x02,0x85,0xa7]
+// GFX12: s_minimum_f32 s5, ttmp11, s2            ; encoding: [0x77,0x02,0x85,0xa7]
 
 s_minimum_f32 s5, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x85,0xa7]
+// GFX12: s_minimum_f32 s5, m0, s2                ; encoding: [0x7d,0x02,0x85,0xa7]
 
 s_minimum_f32 s5, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x85,0xa7]
+// GFX12: s_minimum_f32 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x85,0xa7]
 
 s_minimum_f32 s5, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x85,0xa7]
+// GFX12: s_minimum_f32 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x85,0xa7]
 
 s_minimum_f32 s5, 0, s2
-// GFX12: encoding: [0x80,0x02,0x85,0xa7]
+// GFX12: s_minimum_f32 s5, 0, s2                 ; encoding: [0x80,0x02,0x85,0xa7]
 
 s_minimum_f32 s5, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x85,0xa7]
+// GFX12: s_minimum_f32 s5, -1, s2                ; encoding: [0xc1,0x02,0x85,0xa7]
 
 s_minimum_f32 s5, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x85,0xa7]
+// GFX12: s_minimum_f32 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x85,0xa7]
 
 s_minimum_f32 s5, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x85,0xa7]
+// GFX12: s_minimum_f32 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x85,0xa7]
 
 s_minimum_f32 s5, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x85,0xa7,0x56,0x34,0x12,0xaf]
+// GFX12: s_minimum_f32 s5, 0xaf123456, s2        ; encoding: [0xff,0x02,0x85,0xa7,0x56,0x34,0x12,0xaf]
 
 s_minimum_f32 s5, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x85,0xa7,0x73,0x72,0x71,0x3f]
+// GFX12: s_minimum_f32 s5, 0x3f717273, s2        ; encoding: [0xff,0x02,0x85,0xa7,0x73,0x72,0x71,0x3f]
 
 s_minimum_f32 s5, s1, s105
-// GFX12: encoding: [0x01,0x69,0x85,0xa7]
+// GFX12: s_minimum_f32 s5, s1, s105              ; encoding: [0x01,0x69,0x85,0xa7]
 
 s_maximum_f32 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x05,0xa8]
+// GFX12: s_maximum_f32 s5, s1, s2                ; encoding: [0x01,0x02,0x05,0xa8]
 
 s_maximum_f32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0xa8]
+// GFX12: s_maximum_f32 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0xa8]
 
 s_maximum_f32 s5, s105, s2
-// GFX12: encoding: [0x69,0x02,0x05,0xa8]
+// GFX12: s_maximum_f32 s5, s105, s2              ; encoding: [0x69,0x02,0x05,0xa8]
 
 s_maximum_f32 s5, s103, s2
-// GFX12: encoding: [0x67,0x02,0x05,0xa8]
+// GFX12: s_maximum_f32 s5, s103, s2              ; encoding: [0x67,0x02,0x05,0xa8]
 
 s_maximum_f32 s5, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x05,0xa8]
+// GFX12: s_maximum_f32 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x05,0xa8]
 
 s_maximum_f32 s5, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x05,0xa8]
+// GFX12: s_maximum_f32 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x05,0xa8]
 
 s_maximum_f32 s5, ttmp11, s2
-// GFX12: encoding: [0x77,0x02,0x05,0xa8]
+// GFX12: s_maximum_f32 s5, ttmp11, s2            ; encoding: [0x77,0x02,0x05,0xa8]
 
 s_maximum_f32 s5, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x05,0xa8]
+// GFX12: s_maximum_f32 s5, m0, s2                ; encoding: [0x7d,0x02,0x05,0xa8]
 
 s_maximum_f32 s5, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x05,0xa8]
+// GFX12: s_maximum_f32 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x05,0xa8]
 
 s_maximum_f32 s5, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x05,0xa8]
+// GFX12: s_maximum_f32 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x05,0xa8]
 
 s_maximum_f32 s5, 0, s2
-// GFX12: encoding: [0x80,0x02,0x05,0xa8]
+// GFX12: s_maximum_f32 s5, 0, s2                 ; encoding: [0x80,0x02,0x05,0xa8]
 
 s_maximum_f32 s5, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x05,0xa8]
+// GFX12: s_maximum_f32 s5, -1, s2                ; encoding: [0xc1,0x02,0x05,0xa8]
 
 s_maximum_f32 s5, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x05,0xa8]
+// GFX12: s_maximum_f32 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x05,0xa8]
 
 s_maximum_f32 s5, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x05,0xa8]
+// GFX12: s_maximum_f32 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x05,0xa8]
 
 s_maximum_f32 s5, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x05,0xa8,0x56,0x34,0x12,0xaf]
+// GFX12: s_maximum_f32 s5, 0xaf123456, s2        ; encoding: [0xff,0x02,0x05,0xa8,0x56,0x34,0x12,0xaf]
 
 s_maximum_f32 s5, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x05,0xa8,0x73,0x72,0x71,0x3f]
+// GFX12: s_maximum_f32 s5, 0x3f717273, s2        ; encoding: [0xff,0x02,0x05,0xa8,0x73,0x72,0x71,0x3f]
 
 s_maximum_f32 s5, s1, s105
-// GFX12: encoding: [0x01,0x69,0x05,0xa8]
+// GFX12: s_maximum_f32 s5, s1, s105              ; encoding: [0x01,0x69,0x05,0xa8]
 
 s_fmac_f32 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x85,0xa3]
+// GFX12: s_fmac_f32 s5, s1, s2                   ; encoding: [0x01,0x02,0x85,0xa3]
 
 s_fmac_f32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0xe9,0xa3]
+// GFX12: s_fmac_f32 s105, s1, s2                 ; encoding: [0x01,0x02,0xe9,0xa3]
 
 s_fmac_f32 s5, s105, s2
-// GFX12: encoding: [0x69,0x02,0x85,0xa3]
+// GFX12: s_fmac_f32 s5, s105, s2                 ; encoding: [0x69,0x02,0x85,0xa3]
 
 s_fmac_f32 s5, s103, s2
-// GFX12: encoding: [0x67,0x02,0x85,0xa3]
+// GFX12: s_fmac_f32 s5, s103, s2                 ; encoding: [0x67,0x02,0x85,0xa3]
 
 s_fmac_f32 s5, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x85,0xa3]
+// GFX12: s_fmac_f32 s5, vcc_lo, s2               ; encoding: [0x6a,0x02,0x85,0xa3]
 
 s_fmac_f32 s5, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x85,0xa3]
+// GFX12: s_fmac_f32 s5, vcc_hi, s2               ; encoding: [0x6b,0x02,0x85,0xa3]
 
 s_fmac_f32 s5, ttmp11, s2
-// GFX12: encoding: [0x77,0x02,0x85,0xa3]
+// GFX12: s_fmac_f32 s5, ttmp11, s2               ; encoding: [0x77,0x02,0x85,0xa3]
 
 s_fmac_f32 s5, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x85,0xa3]
+// GFX12: s_fmac_f32 s5, m0, s2                   ; encoding: [0x7d,0x02,0x85,0xa3]
 
 s_fmac_f32 s5, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x85,0xa3]
+// GFX12: s_fmac_f32 s5, exec_lo, s2              ; encoding: [0x7e,0x02,0x85,0xa3]
 
 s_fmac_f32 s5, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x85,0xa3]
+// GFX12: s_fmac_f32 s5, exec_hi, s2              ; encoding: [0x7f,0x02,0x85,0xa3]
 
 s_fmac_f32 s5, 0, s2
-// GFX12: encoding: [0x80,0x02,0x85,0xa3]
+// GFX12: s_fmac_f32 s5, 0, s2                    ; encoding: [0x80,0x02,0x85,0xa3]
 
 s_fmac_f32 s5, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x85,0xa3]
+// GFX12: s_fmac_f32 s5, -1, s2                   ; encoding: [0xc1,0x02,0x85,0xa3]
 
 s_fmac_f32 s5, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x85,0xa3]
+// GFX12: s_fmac_f32 s5, 0.5, s2                  ; encoding: [0xf0,0x02,0x85,0xa3]
 
 s_fmac_f32 s5, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x85,0xa3]
+// GFX12: s_fmac_f32 s5, -4.0, s2                 ; encoding: [0xf7,0x02,0x85,0xa3]
 
 s_fmac_f32 s5, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x85,0xa3,0x56,0x34,0x12,0xaf]
+// GFX12: s_fmac_f32 s5, 0xaf123456, s2           ; encoding: [0xff,0x02,0x85,0xa3,0x56,0x34,0x12,0xaf]
 
 s_fmac_f32 s5, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x85,0xa3,0x73,0x72,0x71,0x3f]
+// GFX12: s_fmac_f32 s5, 0x3f717273, s2           ; encoding: [0xff,0x02,0x85,0xa3,0x73,0x72,0x71,0x3f]
 
 s_fmac_f32 s5, s1, s105
-// GFX12: encoding: [0x01,0x69,0x85,0xa3]
+// GFX12: s_fmac_f32 s5, s1, s105                 ; encoding: [0x01,0x69,0x85,0xa3]
 
 s_fmamk_f32 s5, s1, 0x11213141, s3
-// GFX12: encoding: [0x01,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
+// GFX12: s_fmamk_f32 s5, s1, 0x11213141, s3      ; encoding: [0x01,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
 
 s_fmamk_f32 s105, s1, 0x11213141, s3
-// GFX12: encoding: [0x01,0x03,0x69,0xa3,0x41,0x31,0x21,0x11]
+// GFX12: s_fmamk_f32 s105, s1, 0x11213141, s3    ; encoding: [0x01,0x03,0x69,0xa3,0x41,0x31,0x21,0x11]
 
 s_fmamk_f32 s5, s105, 0x11213141, s3
-// GFX12: encoding: [0x69,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
+// GFX12: s_fmamk_f32 s5, s105, 0x11213141, s3    ; encoding: [0x69,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
 
 s_fmamk_f32 s5, 0, 0x11213141, s3
-// GFX12: encoding: [0x80,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
+// GFX12: s_fmamk_f32 s5, 0, 0x11213141, s3       ; encoding: [0x80,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
 
 s_fmamk_f32 s5, -1, 0x11213141, s3
-// GFX12: encoding: [0xc1,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
+// GFX12: s_fmamk_f32 s5, -1, 0x11213141, s3      ; encoding: [0xc1,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
 
 s_fmamk_f32 s5, 0.5, 0x11213141, s3
-// GFX12: encoding: [0xf0,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
+// GFX12: s_fmamk_f32 s5, 0.5, 0x11213141, s3     ; encoding: [0xf0,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
 
 s_fmamk_f32 s5, -4.0, 0x11213141, s3
-// GFX12: encoding: [0xf7,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
+// GFX12: s_fmamk_f32 s5, -4.0, 0x11213141, s3    ; encoding: [0xf7,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
 
 s_fmamk_f32 s5, s1, 0xa1b1c1d1, s3
-// GFX12: encoding: [0x01,0x03,0x05,0xa3,0xd1,0xc1,0xb1,0xa1]
+// GFX12: s_fmamk_f32 s5, s1, 0xa1b1c1d1, s3      ; encoding: [0x01,0x03,0x05,0xa3,0xd1,0xc1,0xb1,0xa1]
 
 s_fmamk_f32 s5, s1, 0x11213141, s105
-// GFX12: encoding: [0x01,0x69,0x05,0xa3,0x41,0x31,0x21,0x11]
+// GFX12: s_fmamk_f32 s5, s1, 0x11213141, s105    ; encoding: [0x01,0x69,0x05,0xa3,0x41,0x31,0x21,0x11]
 
 s_fmamk_f32 s5, 0x11213141, 0x11213141, s105
+// GFX12: s_fmamk_f32 s5, 0x11213141, 0x11213141, s105 ; encoding: [0xff,0x69,0x05,0xa3,0x41,0x31,0x21,0x11]
 // GFX12 encoding: [0xff,0x69,0x05,0xa3,0x41,0x31,0x21,0x11]
 
 s_fmamk_f32 s5, s105, 0x11213141, 0x11213141
+// GFX12: s_fmamk_f32 s5, s105, 0x11213141, 0x11213141 ; encoding: [0x69,0xff,0x05,0xa3,0x41,0x31,0x21,0x11]
 // GFX12 encoding: [0x69,0xff,0x05,0xa3,0x41,0x31,0x21,0x11]
 
 s_fmaak_f32 s5, s1, s2, 0x11213141
-// GFX12: encoding: [0x01,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
+// GFX12: s_fmaak_f32 s5, s1, s2, 0x11213141      ; encoding: [0x01,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 
 s_fmaak_f32 s105, s1, s2, 0x11213141
-// GFX12: encoding: [0x01,0x02,0xe9,0xa2,0x41,0x31,0x21,0x11]
+// GFX12: s_fmaak_f32 s105, s1, s2, 0x11213141    ; encoding: [0x01,0x02,0xe9,0xa2,0x41,0x31,0x21,0x11]
 
 s_fmaak_f32 s5, s105, s2, 0x11213141
-// GFX12: encoding: [0x69,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
+// GFX12: s_fmaak_f32 s5, s105, s2, 0x11213141    ; encoding: [0x69,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 
 s_fmaak_f32 s5, 0, s2, 0x11213141
-// GFX12: encoding: [0x80,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
+// GFX12: s_fmaak_f32 s5, 0, s2, 0x11213141       ; encoding: [0x80,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 
 s_fmaak_f32 s5, -1, s2, 0x11213141
-// GFX12: encoding: [0xc1,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
+// GFX12: s_fmaak_f32 s5, -1, s2, 0x11213141      ; encoding: [0xc1,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 
 s_fmaak_f32 s5, 0.5, s2, 0x11213141
-// GFX12: encoding: [0xf0,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
+// GFX12: s_fmaak_f32 s5, 0.5, s2, 0x11213141     ; encoding: [0xf0,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 
 s_fmaak_f32 s5, -4.0, s2, 0x11213141
-// GFX12: encoding: [0xf7,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
+// GFX12: s_fmaak_f32 s5, -4.0, s2, 0x11213141    ; encoding: [0xf7,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 
 s_fmaak_f32 s5, s1, s105, 0x11213141
-// GFX12: encoding: [0x01,0x69,0x85,0xa2,0x41,0x31,0x21,0x11]
+// GFX12: s_fmaak_f32 s5, s1, s105, 0x11213141    ; encoding: [0x01,0x69,0x85,0xa2,0x41,0x31,0x21,0x11]
 
 s_fmaak_f32 s5, s1, s2, 0xa1b1c1d1
-// GFX12: encoding: [0x01,0x02,0x85,0xa2,0xd1,0xc1,0xb1,0xa1]
+// GFX12: s_fmaak_f32 s5, s1, s2, 0xa1b1c1d1      ; encoding: [0x01,0x02,0x85,0xa2,0xd1,0xc1,0xb1,0xa1]
 
 s_fmaak_f32 s5, 0x11213141, s2, 0x11213141
-// GFX12: encoding: [0xff,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
+// GFX12: s_fmaak_f32 s5, 0x11213141, s2, 0x11213141 ; encoding: [0xff,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 
 s_fmaak_f32 s5, s105, 0x11213141, 0x11213141
-// GFX12: encoding: [0x69,0xff,0x85,0xa2,0x41,0x31,0x21,0x11]
+// GFX12: s_fmaak_f32 s5, s105, 0x11213141, 0x11213141 ; encoding: [0x69,0xff,0x85,0xa2,0x41,0x31,0x21,0x11]
 
 s_fmaak_f32 s5, 0x11213141, 0x11213141, 0x11213141
-// GFX12: encoding: [0xff,0xff,0x85,0xa2,0x41,0x31,0x21,0x11]
+// GFX12: s_fmaak_f32 s5, 0x11213141, 0x11213141, 0x11213141 ; encoding: [0xff,0xff,0x85,0xa2,0x41,0x31,0x21,0x11]
 
 s_cvt_pk_rtz_f16_f32 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x05,0xa4]
+// GFX12: s_cvt_pk_rtz_f16_f32 s5, s1, s2         ; encoding: [0x01,0x02,0x05,0xa4]
 
 s_cvt_pk_rtz_f16_f32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0xa4]
+// GFX12: s_cvt_pk_rtz_f16_f32 s105, s1, s2       ; encoding: [0x01,0x02,0x69,0xa4]
 
 s_cvt_pk_rtz_f16_f32 s5, s105, s2
-// GFX12: encoding: [0x69,0x02,0x05,0xa4]
+// GFX12: s_cvt_pk_rtz_f16_f32 s5, s105, s2       ; encoding: [0x69,0x02,0x05,0xa4]
 
 s_cvt_pk_rtz_f16_f32 s5, s103, s2
-// GFX12: encoding: [0x67,0x02,0x05,0xa4]
+// GFX12: s_cvt_pk_rtz_f16_f32 s5, s103, s2       ; encoding: [0x67,0x02,0x05,0xa4]
 
 s_cvt_pk_rtz_f16_f32 s5, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x05,0xa4]
+// GFX12: s_cvt_pk_rtz_f16_f32 s5, vcc_lo, s2     ; encoding: [0x6a,0x02,0x05,0xa4]
 
 s_cvt_pk_rtz_f16_f32 s5, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x05,0xa4]
+// GFX12: s_cvt_pk_rtz_f16_f32 s5, vcc_hi, s2     ; encoding: [0x6b,0x02,0x05,0xa4]
 
 s_cvt_pk_rtz_f16_f32 s5, ttmp11, s2
-// GFX12: encoding: [0x77,0x02,0x05,0xa4]
+// GFX12: s_cvt_pk_rtz_f16_f32 s5, ttmp11, s2     ; encoding: [0x77,0x02,0x05,0xa4]
 
 s_cvt_pk_rtz_f16_f32 s5, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x05,0xa4]
+// GFX12: s_cvt_pk_rtz_f16_f32 s5, m0, s2         ; encoding: [0x7d,0x02,0x05,0xa4]
 
 s_cvt_pk_rtz_f16_f32 s5, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x05,0xa4]
+// GFX12: s_cvt_pk_rtz_f16_f32 s5, exec_lo, s2    ; encoding: [0x7e,0x02,0x05,0xa4]
 
 s_cvt_pk_rtz_f16_f32 s5, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x05,0xa4]
+// GFX12: s_cvt_pk_rtz_f16_f32 s5, exec_hi, s2    ; encoding: [0x7f,0x02,0x05,0xa4]
 
 s_cvt_pk_rtz_f16_f32 s5, 0, s2
-// GFX12: encoding: [0x80,0x02,0x05,0xa4]
+// GFX12: s_cvt_pk_rtz_f16_f32 s5, 0, s2          ; encoding: [0x80,0x02,0x05,0xa4]
 
 s_cvt_pk_rtz_f16_f32 s5, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x05,0xa4]
+// GFX12: s_cvt_pk_rtz_f16_f32 s5, -1, s2         ; encoding: [0xc1,0x02,0x05,0xa4]
 
 s_cvt_pk_rtz_f16_f32 s5, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x05,0xa4]
+// GFX12: s_cvt_pk_rtz_f16_f32 s5, 0.5, s2        ; encoding: [0xf0,0x02,0x05,0xa4]
 
 s_cvt_pk_rtz_f16_f32 s5, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x05,0xa4]
+// GFX12: s_cvt_pk_rtz_f16_f32 s5, -4.0, s2       ; encoding: [0xf7,0x02,0x05,0xa4]
 
 s_cvt_pk_rtz_f16_f32 s5, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x05,0xa4,0x56,0x34,0x12,0xaf]
+// GFX12: s_cvt_pk_rtz_f16_f32 s5, 0xaf123456, s2 ; encoding: [0xff,0x02,0x05,0xa4,0x56,0x34,0x12,0xaf]
 
 s_cvt_pk_rtz_f16_f32 s5, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x05,0xa4,0x73,0x72,0x71,0x3f]
+// GFX12: s_cvt_pk_rtz_f16_f32 s5, 0x3f717273, s2 ; encoding: [0xff,0x02,0x05,0xa4,0x73,0x72,0x71,0x3f]
 
 s_cvt_pk_rtz_f16_f32 s5, s1, s105
-// GFX12: encoding: [0x01,0x69,0x05,0xa4]
+// GFX12: s_cvt_pk_rtz_f16_f32 s5, s1, s105       ; encoding: [0x01,0x69,0x05,0xa4]
 
 s_add_f16 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x85,0xa4]
+// GFX12: s_add_f16 s5, s1, s2                    ; encoding: [0x01,0x02,0x85,0xa4]
 
 s_add_f16 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0xe9,0xa4]
+// GFX12: s_add_f16 s105, s1, s2                  ; encoding: [0x01,0x02,0xe9,0xa4]
 
 s_add_f16 s5, s105, s2
-// GFX12: encoding: [0x69,0x02,0x85,0xa4]
+// GFX12: s_add_f16 s5, s105, s2                  ; encoding: [0x69,0x02,0x85,0xa4]
 
 s_add_f16 s5, s101, s2
-// GFX12: encoding: [0x65,0x02,0x85,0xa4]
+// GFX12: s_add_f16 s5, s101, s2                  ; encoding: [0x65,0x02,0x85,0xa4]
 
 s_add_f16 s5, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x85,0xa4]
+// GFX12: s_add_f16 s5, vcc_lo, s2                ; encoding: [0x6a,0x02,0x85,0xa4]
 
 s_add_f16 s5, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x85,0xa4]
+// GFX12: s_add_f16 s5, vcc_hi, s2                ; encoding: [0x6b,0x02,0x85,0xa4]
 
 s_add_f16 s5, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x85,0xa4]
+// GFX12: s_add_f16 s5, m0, s2                    ; encoding: [0x7d,0x02,0x85,0xa4]
 
 s_add_f16 s5, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x85,0xa4]
+// GFX12: s_add_f16 s5, exec_lo, s2               ; encoding: [0x7e,0x02,0x85,0xa4]
 
 s_add_f16 s5, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x85,0xa4]
+// GFX12: s_add_f16 s5, exec_hi, s2               ; encoding: [0x7f,0x02,0x85,0xa4]
 
 s_add_f16 s5, 0, s2
-// GFX12: encoding: [0x80,0x02,0x85,0xa4]
+// GFX12: s_add_f16 s5, 0, s2                     ; encoding: [0x80,0x02,0x85,0xa4]
 
 s_add_f16 s5, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x85,0xa4]
+// GFX12: s_add_f16 s5, -1, s2                    ; encoding: [0xc1,0x02,0x85,0xa4]
 
 s_add_f16 s5, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x85,0xa4]
+// GFX12: s_add_f16 s5, 0.5, s2                   ; encoding: [0xf0,0x02,0x85,0xa4]
 
 s_add_f16 s5, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x85,0xa4]
+// GFX12: s_add_f16 s5, -4.0, s2                  ; encoding: [0xf7,0x02,0x85,0xa4]
 
 s_add_f16 s5, 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x85,0xa4,0x0b,0xfe,0x00,0x00]
+// GFX12: s_add_f16 s5, 0xfe0b, s2                ; encoding: [0xff,0x02,0x85,0xa4,0x0b,0xfe,0x00,0x00]
 
 s_add_f16 s5, 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x85,0xa4,0x56,0x34,0x00,0x00]
+// GFX12: s_add_f16 s5, 0x3456, s2                ; encoding: [0xff,0x02,0x85,0xa4,0x56,0x34,0x00,0x00]
 
 s_add_f16 s5, s1, s105
-// GFX12: encoding: [0x01,0x69,0x85,0xa4]
+// GFX12: s_add_f16 s5, s1, s105                  ; encoding: [0x01,0x69,0x85,0xa4]
 
 s_sub_f16 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x05,0xa5]
+// GFX12: s_sub_f16 s5, s1, s2                    ; encoding: [0x01,0x02,0x05,0xa5]
 
 s_sub_f16 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0xa5]
+// GFX12: s_sub_f16 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0xa5]
 
 s_sub_f16 s5, s105, s2
-// GFX12: encoding: [0x69,0x02,0x05,0xa5]
+// GFX12: s_sub_f16 s5, s105, s2                  ; encoding: [0x69,0x02,0x05,0xa5]
 
 s_sub_f16 s5, s101, s2
-// GFX12: encoding: [0x65,0x02,0x05,0xa5]
+// GFX12: s_sub_f16 s5, s101, s2                  ; encoding: [0x65,0x02,0x05,0xa5]
 
 s_sub_f16 s5, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x05,0xa5]
+// GFX12: s_sub_f16 s5, vcc_lo, s2                ; encoding: [0x6a,0x02,0x05,0xa5]
 
 s_sub_f16 s5, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x05,0xa5]
+// GFX12: s_sub_f16 s5, vcc_hi, s2                ; encoding: [0x6b,0x02,0x05,0xa5]
 
 s_sub_f16 s5, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x05,0xa5]
+// GFX12: s_sub_f16 s5, m0, s2                    ; encoding: [0x7d,0x02,0x05,0xa5]
 
 s_sub_f16 s5, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x05,0xa5]
+// GFX12: s_sub_f16 s5, exec_lo, s2               ; encoding: [0x7e,0x02,0x05,0xa5]
 
 s_sub_f16 s5, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x05,0xa5]
+// GFX12: s_sub_f16 s5, exec_hi, s2               ; encoding: [0x7f,0x02,0x05,0xa5]
 
 s_sub_f16 s5, 0, s2
-// GFX12: encoding: [0x80,0x02,0x05,0xa5]
+// GFX12: s_sub_f16 s5, 0, s2                     ; encoding: [0x80,0x02,0x05,0xa5]
 
 s_sub_f16 s5, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x05,0xa5]
+// GFX12: s_sub_f16 s5, -1, s2                    ; encoding: [0xc1,0x02,0x05,0xa5]
 
 s_sub_f16 s5, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x05,0xa5]
+// GFX12: s_sub_f16 s5, 0.5, s2                   ; encoding: [0xf0,0x02,0x05,0xa5]
 
 s_sub_f16 s5, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x05,0xa5]
+// GFX12: s_sub_f16 s5, -4.0, s2                  ; encoding: [0xf7,0x02,0x05,0xa5]
 
 s_sub_f16 s5, 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x05,0xa5,0x0b,0xfe,0x00,0x00]
+// GFX12: s_sub_f16 s5, 0xfe0b, s2                ; encoding: [0xff,0x02,0x05,0xa5,0x0b,0xfe,0x00,0x00]
 
 s_sub_f16 s5, 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x05,0xa5,0x56,0x34,0x00,0x00]
+// GFX12: s_sub_f16 s5, 0x3456, s2                ; encoding: [0xff,0x02,0x05,0xa5,0x56,0x34,0x00,0x00]
 
 s_sub_f16 s5, s1, s105
-// GFX12: encoding: [0x01,0x69,0x05,0xa5]
+// GFX12: s_sub_f16 s5, s1, s105                  ; encoding: [0x01,0x69,0x05,0xa5]
 
 s_mul_f16 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x85,0xa6]
+// GFX12: s_mul_f16 s5, s1, s2                    ; encoding: [0x01,0x02,0x85,0xa6]
 
 s_mul_f16 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0xe9,0xa6]
+// GFX12: s_mul_f16 s105, s1, s2                  ; encoding: [0x01,0x02,0xe9,0xa6]
 
 s_mul_f16 s5, s105, s2
-// GFX12: encoding: [0x69,0x02,0x85,0xa6]
+// GFX12: s_mul_f16 s5, s105, s2                  ; encoding: [0x69,0x02,0x85,0xa6]
 
 s_mul_f16 s5, s101, s2
-// GFX12: encoding: [0x65,0x02,0x85,0xa6]
+// GFX12: s_mul_f16 s5, s101, s2                  ; encoding: [0x65,0x02,0x85,0xa6]
 
 s_mul_f16 s5, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x85,0xa6]
+// GFX12: s_mul_f16 s5, vcc_lo, s2                ; encoding: [0x6a,0x02,0x85,0xa6]
 
 s_mul_f16 s5, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x85,0xa6]
+// GFX12: s_mul_f16 s5, vcc_hi, s2                ; encoding: [0x6b,0x02,0x85,0xa6]
 
 s_mul_f16 s5, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x85,0xa6]
+// GFX12: s_mul_f16 s5, m0, s2                    ; encoding: [0x7d,0x02,0x85,0xa6]
 
 s_mul_f16 s5, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x85,0xa6]
+// GFX12: s_mul_f16 s5, exec_lo, s2               ; encoding: [0x7e,0x02,0x85,0xa6]
 
 s_mul_f16 s5, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x85,0xa6]
+// GFX12: s_mul_f16 s5, exec_hi, s2               ; encoding: [0x7f,0x02,0x85,0xa6]
 
 s_mul_f16 s5, 0, s2
-// GFX12: encoding: [0x80,0x02,0x85,0xa6]
+// GFX12: s_mul_f16 s5, 0, s2                     ; encoding: [0x80,0x02,0x85,0xa6]
 
 s_mul_f16 s5, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x85,0xa6]
+// GFX12: s_mul_f16 s5, -1, s2                    ; encoding: [0xc1,0x02,0x85,0xa6]
 
 s_mul_f16 s5, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x85,0xa6]
+// GFX12: s_mul_f16 s5, 0.5, s2                   ; encoding: [0xf0,0x02,0x85,0xa6]
 
 s_mul_f16 s5, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x85,0xa6]
+// GFX12: s_mul_f16 s5, -4.0, s2                  ; encoding: [0xf7,0x02,0x85,0xa6]
 
 s_mul_f16 s5, 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x85,0xa6,0x0b,0xfe,0x00,0x00]
+// GFX12: s_mul_f16 s5, 0xfe0b, s2                ; encoding: [0xff,0x02,0x85,0xa6,0x0b,0xfe,0x00,0x00]
 
 s_mul_f16 s5, 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x85,0xa6,0x56,0x34,0x00,0x00]
+// GFX12: s_mul_f16 s5, 0x3456, s2                ; encoding: [0xff,0x02,0x85,0xa6,0x56,0x34,0x00,0x00]
 
 s_mul_f16 s5, s1, s105
-// GFX12: encoding: [0x01,0x69,0x85,0xa6]
+// GFX12: s_mul_f16 s5, s1, s105                  ; encoding: [0x01,0x69,0x85,0xa6]
 
 s_fmac_f16 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x05,0xa7]
+// GFX12: s_fmac_f16 s5, s1, s2                   ; encoding: [0x01,0x02,0x05,0xa7]
 
 s_fmac_f16 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0xa7]
+// GFX12: s_fmac_f16 s105, s1, s2                 ; encoding: [0x01,0x02,0x69,0xa7]
 
 s_fmac_f16 s5, s105, s2
-// GFX12: encoding: [0x69,0x02,0x05,0xa7]
+// GFX12: s_fmac_f16 s5, s105, s2                 ; encoding: [0x69,0x02,0x05,0xa7]
 
 s_fmac_f16 s5, s103, s2
-// GFX12: encoding: [0x67,0x02,0x05,0xa7]
+// GFX12: s_fmac_f16 s5, s103, s2                 ; encoding: [0x67,0x02,0x05,0xa7]
 
 s_fmac_f16 s5, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x05,0xa7]
+// GFX12: s_fmac_f16 s5, vcc_lo, s2               ; encoding: [0x6a,0x02,0x05,0xa7]
 
 s_fmac_f16 s5, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x05,0xa7]
+// GFX12: s_fmac_f16 s5, vcc_hi, s2               ; encoding: [0x6b,0x02,0x05,0xa7]
 
 s_fmac_f16 s5, ttmp11, s2
-// GFX12: encoding: [0x77,0x02,0x05,0xa7]
+// GFX12: s_fmac_f16 s5, ttmp11, s2               ; encoding: [0x77,0x02,0x05,0xa7]
 
 s_fmac_f16 s5, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x05,0xa7]
+// GFX12: s_fmac_f16 s5, m0, s2                   ; encoding: [0x7d,0x02,0x05,0xa7]
 
 s_fmac_f16 s5, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x05,0xa7]
+// GFX12: s_fmac_f16 s5, exec_lo, s2              ; encoding: [0x7e,0x02,0x05,0xa7]
 
 s_fmac_f16 s5, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x05,0xa7]
+// GFX12: s_fmac_f16 s5, exec_hi, s2              ; encoding: [0x7f,0x02,0x05,0xa7]
 
 s_fmac_f16 s5, 0, s2
-// GFX12: encoding: [0x80,0x02,0x05,0xa7]
+// GFX12: s_fmac_f16 s5, 0, s2                    ; encoding: [0x80,0x02,0x05,0xa7]
 
 s_fmac_f16 s5, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x05,0xa7]
+// GFX12: s_fmac_f16 s5, -1, s2                   ; encoding: [0xc1,0x02,0x05,0xa7]
 
 s_fmac_f16 s5, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x05,0xa7]
+// GFX12: s_fmac_f16 s5, 0.5, s2                  ; encoding: [0xf0,0x02,0x05,0xa7]
 
 s_fmac_f16 s5, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x05,0xa7]
+// GFX12: s_fmac_f16 s5, -4.0, s2                 ; encoding: [0xf7,0x02,0x05,0xa7]
 
 s_fmac_f16 s5, 0x1234, s2
-// GFX12: encoding: [0xff,0x02,0x05,0xa7,0x34,0x12,0x00,0x00]
+// GFX12: s_fmac_f16 s5, 0x1234, s2               ; encoding: [0xff,0x02,0x05,0xa7,0x34,0x12,0x00,0x00]
 
 s_fmac_f16 s5, s1, s105
-// GFX12: encoding: [0x01,0x69,0x05,0xa7]
+// GFX12: s_fmac_f16 s5, s1, s105                 ; encoding: [0x01,0x69,0x05,0xa7]
 
 s_max_num_f16 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x05,0xa6]
+// GFX12: s_max_num_f16 s5, s1, s2                ; encoding: [0x01,0x02,0x05,0xa6]
 
 s_max_num_f16 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0xa6]
+// GFX12: s_max_num_f16 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0xa6]
 
 s_max_num_f16 s5, s105, s2
-// GFX12: encoding: [0x69,0x02,0x05,0xa6]
+// GFX12: s_max_num_f16 s5, s105, s2              ; encoding: [0x69,0x02,0x05,0xa6]
 
 s_max_num_f16 s5, s101, s2
-// GFX12: encoding: [0x65,0x02,0x05,0xa6]
+// GFX12: s_max_num_f16 s5, s101, s2              ; encoding: [0x65,0x02,0x05,0xa6]
 
 s_max_num_f16 s5, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x05,0xa6]
+// GFX12: s_max_num_f16 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x05,0xa6]
 
 s_max_num_f16 s5, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x05,0xa6]
+// GFX12: s_max_num_f16 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x05,0xa6]
 
 s_max_num_f16 s5, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x05,0xa6]
+// GFX12: s_max_num_f16 s5, m0, s2                ; encoding: [0x7d,0x02,0x05,0xa6]
 
 s_max_num_f16 s5, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x05,0xa6]
+// GFX12: s_max_num_f16 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x05,0xa6]
 
 s_max_num_f16 s5, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x05,0xa6]
+// GFX12: s_max_num_f16 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x05,0xa6]
 
 s_max_num_f16 s5, 0, s2
-// GFX12: encoding: [0x80,0x02,0x05,0xa6]
+// GFX12: s_max_num_f16 s5, 0, s2                 ; encoding: [0x80,0x02,0x05,0xa6]
 
 s_max_num_f16 s5, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x05,0xa6]
+// GFX12: s_max_num_f16 s5, -1, s2                ; encoding: [0xc1,0x02,0x05,0xa6]
 
 s_max_num_f16 s5, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x05,0xa6]
+// GFX12: s_max_num_f16 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x05,0xa6]
 
 s_max_num_f16 s5, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x05,0xa6]
+// GFX12: s_max_num_f16 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x05,0xa6]
 
 s_max_num_f16 s5, 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x05,0xa6,0x0b,0xfe,0x00,0x00]
+// GFX12: s_max_num_f16 s5, 0xfe0b, s2            ; encoding: [0xff,0x02,0x05,0xa6,0x0b,0xfe,0x00,0x00]
 
 s_max_num_f16 s5, 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x05,0xa6,0x56,0x34,0x00,0x00]
+// GFX12: s_max_num_f16 s5, 0x3456, s2            ; encoding: [0xff,0x02,0x05,0xa6,0x56,0x34,0x00,0x00]
 
 s_max_num_f16 s5, s1, s105
-// GFX12: encoding: [0x01,0x69,0x05,0xa6]
+// GFX12: s_max_num_f16 s5, s1, s105              ; encoding: [0x01,0x69,0x05,0xa6]
 
 s_min_num_f16 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x85,0xa5]
+// GFX12: s_min_num_f16 s5, s1, s2                ; encoding: [0x01,0x02,0x85,0xa5]
 
 s_min_num_f16 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0xe9,0xa5]
+// GFX12: s_min_num_f16 s105, s1, s2              ; encoding: [0x01,0x02,0xe9,0xa5]
 
 s_min_num_f16 s5, s105, s2
-// GFX12: encoding: [0x69,0x02,0x85,0xa5]
+// GFX12: s_min_num_f16 s5, s105, s2              ; encoding: [0x69,0x02,0x85,0xa5]
 
 s_min_num_f16 s5, s101, s2
-// GFX12: encoding: [0x65,0x02,0x85,0xa5]
+// GFX12: s_min_num_f16 s5, s101, s2              ; encoding: [0x65,0x02,0x85,0xa5]
 
 s_min_num_f16 s5, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x85,0xa5]
+// GFX12: s_min_num_f16 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x85,0xa5]
 
 s_min_num_f16 s5, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x85,0xa5]
+// GFX12: s_min_num_f16 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x85,0xa5]
 
 s_min_num_f16 s5, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x85,0xa5]
+// GFX12: s_min_num_f16 s5, m0, s2                ; encoding: [0x7d,0x02,0x85,0xa5]
 
 s_min_num_f16 s5, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x85,0xa5]
+// GFX12: s_min_num_f16 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x85,0xa5]
 
 s_min_num_f16 s5, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x85,0xa5]
+// GFX12: s_min_num_f16 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x85,0xa5]
 
 s_min_num_f16 s5, 0, s2
-// GFX12: encoding: [0x80,0x02,0x85,0xa5]
+// GFX12: s_min_num_f16 s5, 0, s2                 ; encoding: [0x80,0x02,0x85,0xa5]
 
 s_min_num_f16 s5, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x85,0xa5]
+// GFX12: s_min_num_f16 s5, -1, s2                ; encoding: [0xc1,0x02,0x85,0xa5]
 
 s_min_num_f16 s5, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x85,0xa5]
+// GFX12: s_min_num_f16 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x85,0xa5]
 
 s_min_num_f16 s5, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x85,0xa5]
+// GFX12: s_min_num_f16 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x85,0xa5]
 
 s_min_num_f16 s5, 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x85,0xa5,0x0b,0xfe,0x00,0x00]
+// GFX12: s_min_num_f16 s5, 0xfe0b, s2            ; encoding: [0xff,0x02,0x85,0xa5,0x0b,0xfe,0x00,0x00]
 
 s_min_num_f16 s5, 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x85,0xa5,0x56,0x34,0x00,0x00]
+// GFX12: s_min_num_f16 s5, 0x3456, s2            ; encoding: [0xff,0x02,0x85,0xa5,0x56,0x34,0x00,0x00]
 
 s_min_num_f16 s5, s1, s105
-// GFX12: encoding: [0x01,0x69,0x85,0xa5]
+// GFX12: s_min_num_f16 s5, s1, s105              ; encoding: [0x01,0x69,0x85,0xa5]
 
 s_maximum_f16 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x05,0xa9]
+// GFX12: s_maximum_f16 s5, s1, s2                ; encoding: [0x01,0x02,0x05,0xa9]
 
 s_maximum_f16 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0xa9]
+// GFX12: s_maximum_f16 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0xa9]
 
 s_maximum_f16 s5, s105, s2
-// GFX12: encoding: [0x69,0x02,0x05,0xa9]
+// GFX12: s_maximum_f16 s5, s105, s2              ; encoding: [0x69,0x02,0x05,0xa9]
 
 s_maximum_f16 s5, s101, s2
-// GFX12: encoding: [0x65,0x02,0x05,0xa9]
+// GFX12: s_maximum_f16 s5, s101, s2              ; encoding: [0x65,0x02,0x05,0xa9]
 
 s_maximum_f16 s5, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x05,0xa9]
+// GFX12: s_maximum_f16 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x05,0xa9]
 
 s_maximum_f16 s5, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x05,0xa9]
+// GFX12: s_maximum_f16 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x05,0xa9]
 
 s_maximum_f16 s5, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x05,0xa9]
+// GFX12: s_maximum_f16 s5, m0, s2                ; encoding: [0x7d,0x02,0x05,0xa9]
 
 s_maximum_f16 s5, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x05,0xa9]
+// GFX12: s_maximum_f16 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x05,0xa9]
 
 s_maximum_f16 s5, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x05,0xa9]
+// GFX12: s_maximum_f16 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x05,0xa9]
 
 s_maximum_f16 s5, 0, s2
-// GFX12: encoding: [0x80,0x02,0x05,0xa9]
+// GFX12: s_maximum_f16 s5, 0, s2                 ; encoding: [0x80,0x02,0x05,0xa9]
 
 s_maximum_f16 s5, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x05,0xa9]
+// GFX12: s_maximum_f16 s5, -1, s2                ; encoding: [0xc1,0x02,0x05,0xa9]
 
 s_maximum_f16 s5, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x05,0xa9]
+// GFX12: s_maximum_f16 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x05,0xa9]
 
 s_maximum_f16 s5, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x05,0xa9]
+// GFX12: s_maximum_f16 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x05,0xa9]
 
 s_maximum_f16 s5, 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x05,0xa9,0x0b,0xfe,0x00,0x00]
+// GFX12: s_maximum_f16 s5, 0xfe0b, s2            ; encoding: [0xff,0x02,0x05,0xa9,0x0b,0xfe,0x00,0x00]
 
 s_maximum_f16 s5, 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x05,0xa9,0x56,0x34,0x00,0x00]
+// GFX12: s_maximum_f16 s5, 0x3456, s2            ; encoding: [0xff,0x02,0x05,0xa9,0x56,0x34,0x00,0x00]
 
 s_maximum_f16 s5, s1, s105
-// GFX12: encoding: [0x01,0x69,0x05,0xa9]
+// GFX12: s_maximum_f16 s5, s1, s105              ; encoding: [0x01,0x69,0x05,0xa9]
 
 s_minimum_f16 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x85,0xa8]
+// GFX12: s_minimum_f16 s5, s1, s2                ; encoding: [0x01,0x02,0x85,0xa8]
 
 s_minimum_f16 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0xe9,0xa8]
+// GFX12: s_minimum_f16 s105, s1, s2              ; encoding: [0x01,0x02,0xe9,0xa8]
 
 s_minimum_f16 s5, s105, s2
-// GFX12: encoding: [0x69,0x02,0x85,0xa8]
+// GFX12: s_minimum_f16 s5, s105, s2              ; encoding: [0x69,0x02,0x85,0xa8]
 
 s_minimum_f16 s5, s101, s2
-// GFX12: encoding: [0x65,0x02,0x85,0xa8]
+// GFX12: s_minimum_f16 s5, s101, s2              ; encoding: [0x65,0x02,0x85,0xa8]
 
 s_minimum_f16 s5, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x85,0xa8]
+// GFX12: s_minimum_f16 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x85,0xa8]
 
 s_minimum_f16 s5, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x85,0xa8]
+// GFX12: s_minimum_f16 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x85,0xa8]
 
 s_minimum_f16 s5, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x85,0xa8]
+// GFX12: s_minimum_f16 s5, m0, s2                ; encoding: [0x7d,0x02,0x85,0xa8]
 
 s_minimum_f16 s5, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x85,0xa8]
+// GFX12: s_minimum_f16 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x85,0xa8]
 
 s_minimum_f16 s5, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x85,0xa8]
+// GFX12: s_minimum_f16 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x85,0xa8]
 
 s_minimum_f16 s5, 0, s2
-// GFX12: encoding: [0x80,0x02,0x85,0xa8]
+// GFX12: s_minimum_f16 s5, 0, s2                 ; encoding: [0x80,0x02,0x85,0xa8]
 
 s_minimum_f16 s5, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x85,0xa8]
+// GFX12: s_minimum_f16 s5, -1, s2                ; encoding: [0xc1,0x02,0x85,0xa8]
 
 s_minimum_f16 s5, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x85,0xa8]
+// GFX12: s_minimum_f16 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x85,0xa8]
 
 s_minimum_f16 s5, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x85,0xa8]
+// GFX12: s_minimum_f16 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x85,0xa8]
 
 s_minimum_f16 s5, 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x85,0xa8,0x0b,0xfe,0x00,0x00]
+// GFX12: s_minimum_f16 s5, 0xfe0b, s2            ; encoding: [0xff,0x02,0x85,0xa8,0x0b,0xfe,0x00,0x00]
 
 s_minimum_f16 s5, 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x85,0xa8,0x56,0x34,0x00,0x00]
+// GFX12: s_minimum_f16 s5, 0x3456, s2            ; encoding: [0xff,0x02,0x85,0xa8,0x56,0x34,0x00,0x00]
 
 s_minimum_f16 s5, s1, s105
-// GFX12: encoding: [0x01,0x69,0x85,0xa8]
+// GFX12: s_minimum_f16 s5, s1, s105              ; encoding: [0x01,0x69,0x85,0xa8]
 
 s_add_co_u32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x80]
+// GFX12: s_add_co_u32 s0, s1, s2                 ; encoding: [0x01,0x02,0x00,0x80]
 
 s_add_co_u32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x80]
+// GFX12: s_add_co_u32 s105, s104, s103           ; encoding: [0x68,0x67,0x69,0x80]
 
 s_add_co_u32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x80]
+// GFX12: s_add_co_u32 s0, s104, s103             ; encoding: [0x68,0x67,0x00,0x80]
 
 s_add_co_u32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x80]
+// GFX12: s_add_co_u32 s105, s1, s103             ; encoding: [0x01,0x67,0x69,0x80]
 
 s_add_co_u32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x80]
+// GFX12: s_add_co_u32 s105, s104, s2             ; encoding: [0x68,0x02,0x69,0x80]
 
 s_add_co_u32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x80]
+// GFX12: s_add_co_u32 s105, s1, s2               ; encoding: [0x01,0x02,0x69,0x80]
 
 s_add_co_u32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x80]
+// GFX12: s_add_co_u32 s0, s104, s2               ; encoding: [0x68,0x02,0x00,0x80]
 
 s_add_co_u32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x80]
+// GFX12: s_add_co_u32 s0, s1, s103               ; encoding: [0x01,0x67,0x00,0x80]
 
 s_add_co_u32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x80]
+// GFX12: s_add_co_u32 exec_lo, s1, s2            ; encoding: [0x01,0x02,0x7e,0x80]
 
 s_add_co_u32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x80]
+// GFX12: s_add_co_u32 exec_hi, s1, s2            ; encoding: [0x01,0x02,0x7f,0x80]
 
 s_add_co_u32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x80]
+// GFX12: s_add_co_u32 vcc_lo, s1, s2             ; encoding: [0x01,0x02,0x6a,0x80]
 
 s_add_co_u32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x80]
+// GFX12: s_add_co_u32 vcc_hi, s1, s2             ; encoding: [0x01,0x02,0x6b,0x80]
 
 s_add_co_u32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x80]
+// GFX12: s_add_co_u32 m0, s1, s2                 ; encoding: [0x01,0x02,0x7d,0x80]
 
 s_add_co_u32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x80]
+// GFX12: s_add_co_u32 s0, exec_lo, s2            ; encoding: [0x7e,0x02,0x00,0x80]
 
 s_add_co_u32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x80]
+// GFX12: s_add_co_u32 s0, exec_hi, s2            ; encoding: [0x7f,0x02,0x00,0x80]
 
 s_add_co_u32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x80]
+// GFX12: s_add_co_u32 s0, vcc_lo, s2             ; encoding: [0x6a,0x02,0x00,0x80]
 
 s_add_co_u32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x80]
+// GFX12: s_add_co_u32 s0, vcc_hi, s2             ; encoding: [0x6b,0x02,0x00,0x80]
 
 s_add_co_u32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x80]
+// GFX12: s_add_co_u32 s0, m0, s2                 ; encoding: [0x7d,0x02,0x00,0x80]
 
 s_add_co_u32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x80]
+// GFX12: s_add_co_u32 s0, 0, s2                  ; encoding: [0x80,0x02,0x00,0x80]
 
 s_add_co_u32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x80]
+// GFX12: s_add_co_u32 s0, -1, s2                 ; encoding: [0xc1,0x02,0x00,0x80]
 
 s_add_co_u32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x80]
+// GFX12: s_add_co_u32 s0, 0.5, s2                ; encoding: [0xf0,0x02,0x00,0x80]
 
 s_add_co_u32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x80]
+// GFX12: s_add_co_u32 s0, -4.0, s2               ; encoding: [0xf7,0x02,0x00,0x80]
 
 s_add_co_u32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x80,0x73,0x72,0x71,0x3f]
+// GFX12: s_add_co_u32 s0, 0x3f717273, s2         ; encoding: [0xff,0x02,0x00,0x80,0x73,0x72,0x71,0x3f]
 
 s_add_co_u32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x80,0x56,0x34,0x12,0xaf]
+// GFX12: s_add_co_u32 s0, 0xaf123456, s2         ; encoding: [0xff,0x02,0x00,0x80,0x56,0x34,0x12,0xaf]
 
 s_add_co_u32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x80]
+// GFX12: s_add_co_u32 s0, s1, exec_lo            ; encoding: [0x01,0x7e,0x00,0x80]
 
 s_add_co_u32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x80]
+// GFX12: s_add_co_u32 s0, s1, exec_hi            ; encoding: [0x01,0x7f,0x00,0x80]
 
 s_add_co_u32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x80]
+// GFX12: s_add_co_u32 s0, s1, vcc_lo             ; encoding: [0x01,0x6a,0x00,0x80]
 
 s_add_co_u32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x80]
+// GFX12: s_add_co_u32 s0, s1, vcc_hi             ; encoding: [0x01,0x6b,0x00,0x80]
 
 s_add_co_u32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x80]
+// GFX12: s_add_co_u32 s0, s1, m0                 ; encoding: [0x01,0x7d,0x00,0x80]
 
 s_add_co_u32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x80]
+// GFX12: s_add_co_u32 s0, s1, 0                  ; encoding: [0x01,0x80,0x00,0x80]
 
 s_add_co_u32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x80]
+// GFX12: s_add_co_u32 s0, s1, -1                 ; encoding: [0x01,0xc1,0x00,0x80]
 
 s_add_co_u32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x80]
+// GFX12: s_add_co_u32 s0, s1, 0.5                ; encoding: [0x01,0xf0,0x00,0x80]
 
 s_add_co_u32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x80]
+// GFX12: s_add_co_u32 s0, s1, -4.0               ; encoding: [0x01,0xf7,0x00,0x80]
 
 s_add_co_u32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x80,0x73,0x72,0x71,0x3f]
+// GFX12: s_add_co_u32 s0, s1, 0x3f717273         ; encoding: [0x01,0xff,0x00,0x80,0x73,0x72,0x71,0x3f]
 
 s_add_co_u32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x80,0x56,0x34,0x12,0xaf]
+// GFX12: s_add_co_u32 s0, s1, 0xaf123456         ; encoding: [0x01,0xff,0x00,0x80,0x56,0x34,0x12,0xaf]
 
 s_sub_co_u32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, s1, s2                 ; encoding: [0x01,0x02,0x80,0x80]
 
 s_sub_co_u32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0xe9,0x80]
+// GFX12: s_sub_co_u32 s105, s104, s103           ; encoding: [0x68,0x67,0xe9,0x80]
 
 s_sub_co_u32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, s104, s103             ; encoding: [0x68,0x67,0x80,0x80]
 
 s_sub_co_u32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0xe9,0x80]
+// GFX12: s_sub_co_u32 s105, s1, s103             ; encoding: [0x01,0x67,0xe9,0x80]
 
 s_sub_co_u32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0xe9,0x80]
+// GFX12: s_sub_co_u32 s105, s104, s2             ; encoding: [0x68,0x02,0xe9,0x80]
 
 s_sub_co_u32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0xe9,0x80]
+// GFX12: s_sub_co_u32 s105, s1, s2               ; encoding: [0x01,0x02,0xe9,0x80]
 
 s_sub_co_u32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, s104, s2               ; encoding: [0x68,0x02,0x80,0x80]
 
 s_sub_co_u32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, s1, s103               ; encoding: [0x01,0x67,0x80,0x80]
 
 s_sub_co_u32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfe,0x80]
+// GFX12: s_sub_co_u32 exec_lo, s1, s2            ; encoding: [0x01,0x02,0xfe,0x80]
 
 s_sub_co_u32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xff,0x80]
+// GFX12: s_sub_co_u32 exec_hi, s1, s2            ; encoding: [0x01,0x02,0xff,0x80]
 
 s_sub_co_u32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xea,0x80]
+// GFX12: s_sub_co_u32 vcc_lo, s1, s2             ; encoding: [0x01,0x02,0xea,0x80]
 
 s_sub_co_u32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xeb,0x80]
+// GFX12: s_sub_co_u32 vcc_hi, s1, s2             ; encoding: [0x01,0x02,0xeb,0x80]
 
 s_sub_co_u32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfd,0x80]
+// GFX12: s_sub_co_u32 m0, s1, s2                 ; encoding: [0x01,0x02,0xfd,0x80]
 
 s_sub_co_u32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, exec_lo, s2            ; encoding: [0x7e,0x02,0x80,0x80]
 
 s_sub_co_u32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, exec_hi, s2            ; encoding: [0x7f,0x02,0x80,0x80]
 
 s_sub_co_u32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, vcc_lo, s2             ; encoding: [0x6a,0x02,0x80,0x80]
 
 s_sub_co_u32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, vcc_hi, s2             ; encoding: [0x6b,0x02,0x80,0x80]
 
 s_sub_co_u32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, m0, s2                 ; encoding: [0x7d,0x02,0x80,0x80]
 
 s_sub_co_u32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, 0, s2                  ; encoding: [0x80,0x02,0x80,0x80]
 
 s_sub_co_u32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, -1, s2                 ; encoding: [0xc1,0x02,0x80,0x80]
 
 s_sub_co_u32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, 0.5, s2                ; encoding: [0xf0,0x02,0x80,0x80]
 
 s_sub_co_u32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, -4.0, s2               ; encoding: [0xf7,0x02,0x80,0x80]
 
 s_sub_co_u32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x80,0x73,0x72,0x71,0x3f]
+// GFX12: s_sub_co_u32 s0, 0x3f717273, s2         ; encoding: [0xff,0x02,0x80,0x80,0x73,0x72,0x71,0x3f]
 
 s_sub_co_u32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x80,0x56,0x34,0x12,0xaf]
+// GFX12: s_sub_co_u32 s0, 0xaf123456, s2         ; encoding: [0xff,0x02,0x80,0x80,0x56,0x34,0x12,0xaf]
 
 s_sub_co_u32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, s1, exec_lo            ; encoding: [0x01,0x7e,0x80,0x80]
 
 s_sub_co_u32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, s1, exec_hi            ; encoding: [0x01,0x7f,0x80,0x80]
 
 s_sub_co_u32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, s1, vcc_lo             ; encoding: [0x01,0x6a,0x80,0x80]
 
 s_sub_co_u32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, s1, vcc_hi             ; encoding: [0x01,0x6b,0x80,0x80]
 
 s_sub_co_u32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, s1, m0                 ; encoding: [0x01,0x7d,0x80,0x80]
 
 s_sub_co_u32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, s1, 0                  ; encoding: [0x01,0x80,0x80,0x80]
 
 s_sub_co_u32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, s1, -1                 ; encoding: [0x01,0xc1,0x80,0x80]
 
 s_sub_co_u32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, s1, 0.5                ; encoding: [0x01,0xf0,0x80,0x80]
 
 s_sub_co_u32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x80,0x80]
+// GFX12: s_sub_co_u32 s0, s1, -4.0               ; encoding: [0x01,0xf7,0x80,0x80]
 
 s_sub_co_u32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x80,0x80,0x73,0x72,0x71,0x3f]
+// GFX12: s_sub_co_u32 s0, s1, 0x3f717273         ; encoding: [0x01,0xff,0x80,0x80,0x73,0x72,0x71,0x3f]
 
 s_sub_co_u32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x80,0x80,0x56,0x34,0x12,0xaf]
+// GFX12: s_sub_co_u32 s0, s1, 0xaf123456         ; encoding: [0x01,0xff,0x80,0x80,0x56,0x34,0x12,0xaf]
 
 s_add_co_i32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x81]
+// GFX12: s_add_co_i32 s0, s1, s2                 ; encoding: [0x01,0x02,0x00,0x81]
 
 s_add_co_i32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x81]
+// GFX12: s_add_co_i32 s105, s104, s103           ; encoding: [0x68,0x67,0x69,0x81]
 
 s_add_co_i32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x81]
+// GFX12: s_add_co_i32 s0, s104, s103             ; encoding: [0x68,0x67,0x00,0x81]
 
 s_add_co_i32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x81]
+// GFX12: s_add_co_i32 s105, s1, s103             ; encoding: [0x01,0x67,0x69,0x81]
 
 s_add_co_i32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x81]
+// GFX12: s_add_co_i32 s105, s104, s2             ; encoding: [0x68,0x02,0x69,0x81]
 
 s_add_co_i32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x81]
+// GFX12: s_add_co_i32 s105, s1, s2               ; encoding: [0x01,0x02,0x69,0x81]
 
 s_add_co_i32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x81]
+// GFX12: s_add_co_i32 s0, s104, s2               ; encoding: [0x68,0x02,0x00,0x81]
 
 s_add_co_i32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x81]
+// GFX12: s_add_co_i32 s0, s1, s103               ; encoding: [0x01,0x67,0x00,0x81]
 
 s_add_co_i32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x81]
+// GFX12: s_add_co_i32 exec_lo, s1, s2            ; encoding: [0x01,0x02,0x7e,0x81]
 
 s_add_co_i32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x81]
+// GFX12: s_add_co_i32 exec_hi, s1, s2            ; encoding: [0x01,0x02,0x7f,0x81]
 
 s_add_co_i32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x81]
+// GFX12: s_add_co_i32 vcc_lo, s1, s2             ; encoding: [0x01,0x02,0x6a,0x81]
 
 s_add_co_i32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x81]
+// GFX12: s_add_co_i32 vcc_hi, s1, s2             ; encoding: [0x01,0x02,0x6b,0x81]
 
 s_add_co_i32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x81]
+// GFX12: s_add_co_i32 m0, s1, s2                 ; encoding: [0x01,0x02,0x7d,0x81]
 
 s_add_co_i32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x81]
+// GFX12: s_add_co_i32 s0, exec_lo, s2            ; encoding: [0x7e,0x02,0x00,0x81]
 
 s_add_co_i32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x81]
+// GFX12: s_add_co_i32 s0, exec_hi, s2            ; encoding: [0x7f,0x02,0x00,0x81]
 
 s_add_co_i32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x81]
+// GFX12: s_add_co_i32 s0, vcc_lo, s2             ; encoding: [0x6a,0x02,0x00,0x81]
 
 s_add_co_i32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x81]
+// GFX12: s_add_co_i32 s0, vcc_hi, s2             ; encoding: [0x6b,0x02,0x00,0x81]
 
 s_add_co_i32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x81]
+// GFX12: s_add_co_i32 s0, m0, s2                 ; encoding: [0x7d,0x02,0x00,0x81]
 
 s_add_co_i32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x81]
+// GFX12: s_add_co_i32 s0, 0, s2                  ; encoding: [0x80,0x02,0x00,0x81]
 
 s_add_co_i32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x81]
+// GFX12: s_add_co_i32 s0, -1, s2                 ; encoding: [0xc1,0x02,0x00,0x81]
 
 s_add_co_i32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x81]
+// GFX12: s_add_co_i32 s0, 0.5, s2                ; encoding: [0xf0,0x02,0x00,0x81]
 
 s_add_co_i32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x81]
+// GFX12: s_add_co_i32 s0, -4.0, s2               ; encoding: [0xf7,0x02,0x00,0x81]
 
 s_add_co_i32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x81,0x73,0x72,0x71,0x3f]
+// GFX12: s_add_co_i32 s0, 0x3f717273, s2         ; encoding: [0xff,0x02,0x00,0x81,0x73,0x72,0x71,0x3f]
 
 s_add_co_i32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x81,0x56,0x34,0x12,0xaf]
+// GFX12: s_add_co_i32 s0, 0xaf123456, s2         ; encoding: [0xff,0x02,0x00,0x81,0x56,0x34,0x12,0xaf]
 
 s_add_co_i32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x81]
+// GFX12: s_add_co_i32 s0, s1, exec_lo            ; encoding: [0x01,0x7e,0x00,0x81]
 
 s_add_co_i32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x81]
+// GFX12: s_add_co_i32 s0, s1, exec_hi            ; encoding: [0x01,0x7f,0x00,0x81]
 
 s_add_co_i32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x81]
+// GFX12: s_add_co_i32 s0, s1, vcc_lo             ; encoding: [0x01,0x6a,0x00,0x81]
 
 s_add_co_i32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x81]
+// GFX12: s_add_co_i32 s0, s1, vcc_hi             ; encoding: [0x01,0x6b,0x00,0x81]
 
 s_add_co_i32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x81]
+// GFX12: s_add_co_i32 s0, s1, m0                 ; encoding: [0x01,0x7d,0x00,0x81]
 
 s_add_co_i32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x81]
+// GFX12: s_add_co_i32 s0, s1, 0                  ; encoding: [0x01,0x80,0x00,0x81]
 
 s_add_co_i32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x81]
+// GFX12: s_add_co_i32 s0, s1, -1                 ; encoding: [0x01,0xc1,0x00,0x81]
 
 s_add_co_i32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x81]
+// GFX12: s_add_co_i32 s0, s1, 0.5                ; encoding: [0x01,0xf0,0x00,0x81]
 
 s_add_co_i32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x81]
+// GFX12: s_add_co_i32 s0, s1, -4.0               ; encoding: [0x01,0xf7,0x00,0x81]
 
 s_add_co_i32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x81,0x73,0x72,0x71,0x3f]
+// GFX12: s_add_co_i32 s0, s1, 0x3f717273         ; encoding: [0x01,0xff,0x00,0x81,0x73,0x72,0x71,0x3f]
 
 s_add_co_i32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x81,0x56,0x34,0x12,0xaf]
+// GFX12: s_add_co_i32 s0, s1, 0xaf123456         ; encoding: [0x01,0xff,0x00,0x81,0x56,0x34,0x12,0xaf]
 
 s_sub_co_i32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, s1, s2                 ; encoding: [0x01,0x02,0x80,0x81]
 
 s_sub_co_i32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0xe9,0x81]
+// GFX12: s_sub_co_i32 s105, s104, s103           ; encoding: [0x68,0x67,0xe9,0x81]
 
 s_sub_co_i32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, s104, s103             ; encoding: [0x68,0x67,0x80,0x81]
 
 s_sub_co_i32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0xe9,0x81]
+// GFX12: s_sub_co_i32 s105, s1, s103             ; encoding: [0x01,0x67,0xe9,0x81]
 
 s_sub_co_i32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0xe9,0x81]
+// GFX12: s_sub_co_i32 s105, s104, s2             ; encoding: [0x68,0x02,0xe9,0x81]
 
 s_sub_co_i32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0xe9,0x81]
+// GFX12: s_sub_co_i32 s105, s1, s2               ; encoding: [0x01,0x02,0xe9,0x81]
 
 s_sub_co_i32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, s104, s2               ; encoding: [0x68,0x02,0x80,0x81]
 
 s_sub_co_i32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, s1, s103               ; encoding: [0x01,0x67,0x80,0x81]
 
 s_sub_co_i32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfe,0x81]
+// GFX12: s_sub_co_i32 exec_lo, s1, s2            ; encoding: [0x01,0x02,0xfe,0x81]
 
 s_sub_co_i32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xff,0x81]
+// GFX12: s_sub_co_i32 exec_hi, s1, s2            ; encoding: [0x01,0x02,0xff,0x81]
 
 s_sub_co_i32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xea,0x81]
+// GFX12: s_sub_co_i32 vcc_lo, s1, s2             ; encoding: [0x01,0x02,0xea,0x81]
 
 s_sub_co_i32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xeb,0x81]
+// GFX12: s_sub_co_i32 vcc_hi, s1, s2             ; encoding: [0x01,0x02,0xeb,0x81]
 
 s_sub_co_i32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfd,0x81]
+// GFX12: s_sub_co_i32 m0, s1, s2                 ; encoding: [0x01,0x02,0xfd,0x81]
 
 s_sub_co_i32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, exec_lo, s2            ; encoding: [0x7e,0x02,0x80,0x81]
 
 s_sub_co_i32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, exec_hi, s2            ; encoding: [0x7f,0x02,0x80,0x81]
 
 s_sub_co_i32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, vcc_lo, s2             ; encoding: [0x6a,0x02,0x80,0x81]
 
 s_sub_co_i32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, vcc_hi, s2             ; encoding: [0x6b,0x02,0x80,0x81]
 
 s_sub_co_i32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, m0, s2                 ; encoding: [0x7d,0x02,0x80,0x81]
 
 s_sub_co_i32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, 0, s2                  ; encoding: [0x80,0x02,0x80,0x81]
 
 s_sub_co_i32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, -1, s2                 ; encoding: [0xc1,0x02,0x80,0x81]
 
 s_sub_co_i32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, 0.5, s2                ; encoding: [0xf0,0x02,0x80,0x81]
 
 s_sub_co_i32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, -4.0, s2               ; encoding: [0xf7,0x02,0x80,0x81]
 
 s_sub_co_i32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x81,0x73,0x72,0x71,0x3f]
+// GFX12: s_sub_co_i32 s0, 0x3f717273, s2         ; encoding: [0xff,0x02,0x80,0x81,0x73,0x72,0x71,0x3f]
 
 s_sub_co_i32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x81,0x56,0x34,0x12,0xaf]
+// GFX12: s_sub_co_i32 s0, 0xaf123456, s2         ; encoding: [0xff,0x02,0x80,0x81,0x56,0x34,0x12,0xaf]
 
 s_sub_co_i32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, s1, exec_lo            ; encoding: [0x01,0x7e,0x80,0x81]
 
 s_sub_co_i32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, s1, exec_hi            ; encoding: [0x01,0x7f,0x80,0x81]
 
 s_sub_co_i32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, s1, vcc_lo             ; encoding: [0x01,0x6a,0x80,0x81]
 
 s_sub_co_i32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, s1, vcc_hi             ; encoding: [0x01,0x6b,0x80,0x81]
 
 s_sub_co_i32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, s1, m0                 ; encoding: [0x01,0x7d,0x80,0x81]
 
 s_sub_co_i32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, s1, 0                  ; encoding: [0x01,0x80,0x80,0x81]
 
 s_sub_co_i32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, s1, -1                 ; encoding: [0x01,0xc1,0x80,0x81]
 
 s_sub_co_i32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, s1, 0.5                ; encoding: [0x01,0xf0,0x80,0x81]
 
 s_sub_co_i32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x80,0x81]
+// GFX12: s_sub_co_i32 s0, s1, -4.0               ; encoding: [0x01,0xf7,0x80,0x81]
 
 s_sub_co_i32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x80,0x81,0x73,0x72,0x71,0x3f]
+// GFX12: s_sub_co_i32 s0, s1, 0x3f717273         ; encoding: [0x01,0xff,0x80,0x81,0x73,0x72,0x71,0x3f]
 
 s_sub_co_i32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x80,0x81,0x56,0x34,0x12,0xaf]
+// GFX12: s_sub_co_i32 s0, s1, 0xaf123456         ; encoding: [0x01,0xff,0x80,0x81,0x56,0x34,0x12,0xaf]
 
 s_add_co_ci_u32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, s1, s2              ; encoding: [0x01,0x02,0x00,0x82]
 
 s_add_co_ci_u32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x82]
+// GFX12: s_add_co_ci_u32 s105, s104, s103        ; encoding: [0x68,0x67,0x69,0x82]
 
 s_add_co_ci_u32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, s104, s103          ; encoding: [0x68,0x67,0x00,0x82]
 
 s_add_co_ci_u32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x82]
+// GFX12: s_add_co_ci_u32 s105, s1, s103          ; encoding: [0x01,0x67,0x69,0x82]
 
 s_add_co_ci_u32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x82]
+// GFX12: s_add_co_ci_u32 s105, s104, s2          ; encoding: [0x68,0x02,0x69,0x82]
 
 s_add_co_ci_u32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x82]
+// GFX12: s_add_co_ci_u32 s105, s1, s2            ; encoding: [0x01,0x02,0x69,0x82]
 
 s_add_co_ci_u32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, s104, s2            ; encoding: [0x68,0x02,0x00,0x82]
 
 s_add_co_ci_u32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, s1, s103            ; encoding: [0x01,0x67,0x00,0x82]
 
 s_add_co_ci_u32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x82]
+// GFX12: s_add_co_ci_u32 exec_lo, s1, s2         ; encoding: [0x01,0x02,0x7e,0x82]
 
 s_add_co_ci_u32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x82]
+// GFX12: s_add_co_ci_u32 exec_hi, s1, s2         ; encoding: [0x01,0x02,0x7f,0x82]
 
 s_add_co_ci_u32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x82]
+// GFX12: s_add_co_ci_u32 vcc_lo, s1, s2          ; encoding: [0x01,0x02,0x6a,0x82]
 
 s_add_co_ci_u32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x82]
+// GFX12: s_add_co_ci_u32 vcc_hi, s1, s2          ; encoding: [0x01,0x02,0x6b,0x82]
 
 s_add_co_ci_u32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x82]
+// GFX12: s_add_co_ci_u32 m0, s1, s2              ; encoding: [0x01,0x02,0x7d,0x82]
 
 s_add_co_ci_u32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, exec_lo, s2         ; encoding: [0x7e,0x02,0x00,0x82]
 
 s_add_co_ci_u32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, exec_hi, s2         ; encoding: [0x7f,0x02,0x00,0x82]
 
 s_add_co_ci_u32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, vcc_lo, s2          ; encoding: [0x6a,0x02,0x00,0x82]
 
 s_add_co_ci_u32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, vcc_hi, s2          ; encoding: [0x6b,0x02,0x00,0x82]
 
 s_add_co_ci_u32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, m0, s2              ; encoding: [0x7d,0x02,0x00,0x82]
 
 s_add_co_ci_u32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, 0, s2               ; encoding: [0x80,0x02,0x00,0x82]
 
 s_add_co_ci_u32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, -1, s2              ; encoding: [0xc1,0x02,0x00,0x82]
 
 s_add_co_ci_u32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, 0.5, s2             ; encoding: [0xf0,0x02,0x00,0x82]
 
 s_add_co_ci_u32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, -4.0, s2            ; encoding: [0xf7,0x02,0x00,0x82]
 
 s_add_co_ci_u32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x82,0x73,0x72,0x71,0x3f]
+// GFX12: s_add_co_ci_u32 s0, 0x3f717273, s2      ; encoding: [0xff,0x02,0x00,0x82,0x73,0x72,0x71,0x3f]
 
 s_add_co_ci_u32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x82,0x56,0x34,0x12,0xaf]
+// GFX12: s_add_co_ci_u32 s0, 0xaf123456, s2      ; encoding: [0xff,0x02,0x00,0x82,0x56,0x34,0x12,0xaf]
 
 s_add_co_ci_u32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, s1, exec_lo         ; encoding: [0x01,0x7e,0x00,0x82]
 
 s_add_co_ci_u32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, s1, exec_hi         ; encoding: [0x01,0x7f,0x00,0x82]
 
 s_add_co_ci_u32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, s1, vcc_lo          ; encoding: [0x01,0x6a,0x00,0x82]
 
 s_add_co_ci_u32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, s1, vcc_hi          ; encoding: [0x01,0x6b,0x00,0x82]
 
 s_add_co_ci_u32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, s1, m0              ; encoding: [0x01,0x7d,0x00,0x82]
 
 s_add_co_ci_u32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, s1, 0               ; encoding: [0x01,0x80,0x00,0x82]
 
 s_add_co_ci_u32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, s1, -1              ; encoding: [0x01,0xc1,0x00,0x82]
 
 s_add_co_ci_u32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, s1, 0.5             ; encoding: [0x01,0xf0,0x00,0x82]
 
 s_add_co_ci_u32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x82]
+// GFX12: s_add_co_ci_u32 s0, s1, -4.0            ; encoding: [0x01,0xf7,0x00,0x82]
 
 s_add_co_ci_u32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x82,0x73,0x72,0x71,0x3f]
+// GFX12: s_add_co_ci_u32 s0, s1, 0x3f717273      ; encoding: [0x01,0xff,0x00,0x82,0x73,0x72,0x71,0x3f]
 
 s_add_co_ci_u32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x82,0x56,0x34,0x12,0xaf]
+// GFX12: s_add_co_ci_u32 s0, s1, 0xaf123456      ; encoding: [0x01,0xff,0x00,0x82,0x56,0x34,0x12,0xaf]
 
 s_sub_co_ci_u32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, s1, s2              ; encoding: [0x01,0x02,0x80,0x82]
 
 s_sub_co_ci_u32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0xe9,0x82]
+// GFX12: s_sub_co_ci_u32 s105, s104, s103        ; encoding: [0x68,0x67,0xe9,0x82]
 
 s_sub_co_ci_u32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, s104, s103          ; encoding: [0x68,0x67,0x80,0x82]
 
 s_sub_co_ci_u32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0xe9,0x82]
+// GFX12: s_sub_co_ci_u32 s105, s1, s103          ; encoding: [0x01,0x67,0xe9,0x82]
 
 s_sub_co_ci_u32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0xe9,0x82]
+// GFX12: s_sub_co_ci_u32 s105, s104, s2          ; encoding: [0x68,0x02,0xe9,0x82]
 
 s_sub_co_ci_u32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0xe9,0x82]
+// GFX12: s_sub_co_ci_u32 s105, s1, s2            ; encoding: [0x01,0x02,0xe9,0x82]
 
 s_sub_co_ci_u32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, s104, s2            ; encoding: [0x68,0x02,0x80,0x82]
 
 s_sub_co_ci_u32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, s1, s103            ; encoding: [0x01,0x67,0x80,0x82]
 
 s_sub_co_ci_u32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfe,0x82]
+// GFX12: s_sub_co_ci_u32 exec_lo, s1, s2         ; encoding: [0x01,0x02,0xfe,0x82]
 
 s_sub_co_ci_u32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xff,0x82]
+// GFX12: s_sub_co_ci_u32 exec_hi, s1, s2         ; encoding: [0x01,0x02,0xff,0x82]
 
 s_sub_co_ci_u32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xea,0x82]
+// GFX12: s_sub_co_ci_u32 vcc_lo, s1, s2          ; encoding: [0x01,0x02,0xea,0x82]
 
 s_sub_co_ci_u32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xeb,0x82]
+// GFX12: s_sub_co_ci_u32 vcc_hi, s1, s2          ; encoding: [0x01,0x02,0xeb,0x82]
 
 s_sub_co_ci_u32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfd,0x82]
+// GFX12: s_sub_co_ci_u32 m0, s1, s2              ; encoding: [0x01,0x02,0xfd,0x82]
 
 s_sub_co_ci_u32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, exec_lo, s2         ; encoding: [0x7e,0x02,0x80,0x82]
 
 s_sub_co_ci_u32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, exec_hi, s2         ; encoding: [0x7f,0x02,0x80,0x82]
 
 s_sub_co_ci_u32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, vcc_lo, s2          ; encoding: [0x6a,0x02,0x80,0x82]
 
 s_sub_co_ci_u32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, vcc_hi, s2          ; encoding: [0x6b,0x02,0x80,0x82]
 
 s_sub_co_ci_u32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, m0, s2              ; encoding: [0x7d,0x02,0x80,0x82]
 
 s_sub_co_ci_u32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, 0, s2               ; encoding: [0x80,0x02,0x80,0x82]
 
 s_sub_co_ci_u32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, -1, s2              ; encoding: [0xc1,0x02,0x80,0x82]
 
 s_sub_co_ci_u32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, 0.5, s2             ; encoding: [0xf0,0x02,0x80,0x82]
 
 s_sub_co_ci_u32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, -4.0, s2            ; encoding: [0xf7,0x02,0x80,0x82]
 
 s_sub_co_ci_u32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x82,0x73,0x72,0x71,0x3f]
+// GFX12: s_sub_co_ci_u32 s0, 0x3f717273, s2      ; encoding: [0xff,0x02,0x80,0x82,0x73,0x72,0x71,0x3f]
 
 s_sub_co_ci_u32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x82,0x56,0x34,0x12,0xaf]
+// GFX12: s_sub_co_ci_u32 s0, 0xaf123456, s2      ; encoding: [0xff,0x02,0x80,0x82,0x56,0x34,0x12,0xaf]
 
 s_sub_co_ci_u32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, s1, exec_lo         ; encoding: [0x01,0x7e,0x80,0x82]
 
 s_sub_co_ci_u32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, s1, exec_hi         ; encoding: [0x01,0x7f,0x80,0x82]
 
 s_sub_co_ci_u32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, s1, vcc_lo          ; encoding: [0x01,0x6a,0x80,0x82]
 
 s_sub_co_ci_u32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, s1, vcc_hi          ; encoding: [0x01,0x6b,0x80,0x82]
 
 s_sub_co_ci_u32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, s1, m0              ; encoding: [0x01,0x7d,0x80,0x82]
 
 s_sub_co_ci_u32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, s1, 0               ; encoding: [0x01,0x80,0x80,0x82]
 
 s_sub_co_ci_u32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, s1, -1              ; encoding: [0x01,0xc1,0x80,0x82]
 
 s_sub_co_ci_u32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, s1, 0.5             ; encoding: [0x01,0xf0,0x80,0x82]
 
 s_sub_co_ci_u32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x80,0x82]
+// GFX12: s_sub_co_ci_u32 s0, s1, -4.0            ; encoding: [0x01,0xf7,0x80,0x82]
 
 s_sub_co_ci_u32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x80,0x82,0x73,0x72,0x71,0x3f]
+// GFX12: s_sub_co_ci_u32 s0, s1, 0x3f717273      ; encoding: [0x01,0xff,0x80,0x82,0x73,0x72,0x71,0x3f]
 
 s_sub_co_ci_u32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x80,0x82,0x56,0x34,0x12,0xaf]
+// GFX12: s_sub_co_ci_u32 s0, s1, 0xaf123456      ; encoding: [0x01,0xff,0x80,0x82,0x56,0x34,0x12,0xaf]
 
 s_min_i32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x89]
+// GFX12: s_min_i32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x89]
 
 s_min_i32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x89]
+// GFX12: s_min_i32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x89]
 
 s_min_i32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x89]
+// GFX12: s_min_i32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x89]
 
 s_min_i32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x89]
+// GFX12: s_min_i32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x89]
 
 s_min_i32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x89]
+// GFX12: s_min_i32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x89]
 
 s_min_i32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x89]
+// GFX12: s_min_i32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x89]
 
 s_min_i32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x89]
+// GFX12: s_min_i32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x89]
 
 s_min_i32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x89]
+// GFX12: s_min_i32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x89]
 
 s_min_i32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x89]
+// GFX12: s_min_i32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x89]
 
 s_min_i32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x89]
+// GFX12: s_min_i32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x89]
 
 s_min_i32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x89]
+// GFX12: s_min_i32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x89]
 
 s_min_i32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x89]
+// GFX12: s_min_i32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x89]
 
 s_min_i32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x89]
+// GFX12: s_min_i32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x89]
 
 s_min_i32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x89]
+// GFX12: s_min_i32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x89]
 
 s_min_i32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x89]
+// GFX12: s_min_i32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x89]
 
 s_min_i32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x89]
+// GFX12: s_min_i32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x89]
 
 s_min_i32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x89]
+// GFX12: s_min_i32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x89]
 
 s_min_i32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x89]
+// GFX12: s_min_i32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x89]
 
 s_min_i32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x89]
+// GFX12: s_min_i32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x89]
 
 s_min_i32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x89]
+// GFX12: s_min_i32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x89]
 
 s_min_i32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x89]
+// GFX12: s_min_i32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x89]
 
 s_min_i32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x89]
+// GFX12: s_min_i32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x89]
 
 s_min_i32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x89,0x73,0x72,0x71,0x3f]
+// GFX12: s_min_i32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x89,0x73,0x72,0x71,0x3f]
 
 s_min_i32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x89,0x56,0x34,0x12,0xaf]
+// GFX12: s_min_i32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x89,0x56,0x34,0x12,0xaf]
 
 s_min_i32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x89]
+// GFX12: s_min_i32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x89]
 
 s_min_i32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x89]
+// GFX12: s_min_i32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x89]
 
 s_min_i32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x89]
+// GFX12: s_min_i32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x89]
 
 s_min_i32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x89]
+// GFX12: s_min_i32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x89]
 
 s_min_i32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x89]
+// GFX12: s_min_i32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x89]
 
 s_min_i32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x89]
+// GFX12: s_min_i32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x89]
 
 s_min_i32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x89]
+// GFX12: s_min_i32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x89]
 
 s_min_i32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x89]
+// GFX12: s_min_i32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x89]
 
 s_min_i32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x89]
+// GFX12: s_min_i32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x89]
 
 s_min_i32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x89,0x73,0x72,0x71,0x3f]
+// GFX12: s_min_i32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x89,0x73,0x72,0x71,0x3f]
 
 s_min_i32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x89,0x56,0x34,0x12,0xaf]
+// GFX12: s_min_i32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x89,0x56,0x34,0x12,0xaf]
 
 s_min_u32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x80,0x89]
+// GFX12: s_min_u32 s0, s1, s2                    ; encoding: [0x01,0x02,0x80,0x89]
 
 s_min_u32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0xe9,0x89]
+// GFX12: s_min_u32 s105, s104, s103              ; encoding: [0x68,0x67,0xe9,0x89]
 
 s_min_u32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x80,0x89]
+// GFX12: s_min_u32 s0, s104, s103                ; encoding: [0x68,0x67,0x80,0x89]
 
 s_min_u32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0xe9,0x89]
+// GFX12: s_min_u32 s105, s1, s103                ; encoding: [0x01,0x67,0xe9,0x89]
 
 s_min_u32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0xe9,0x89]
+// GFX12: s_min_u32 s105, s104, s2                ; encoding: [0x68,0x02,0xe9,0x89]
 
 s_min_u32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0xe9,0x89]
+// GFX12: s_min_u32 s105, s1, s2                  ; encoding: [0x01,0x02,0xe9,0x89]
 
 s_min_u32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x80,0x89]
+// GFX12: s_min_u32 s0, s104, s2                  ; encoding: [0x68,0x02,0x80,0x89]
 
 s_min_u32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x80,0x89]
+// GFX12: s_min_u32 s0, s1, s103                  ; encoding: [0x01,0x67,0x80,0x89]
 
 s_min_u32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfe,0x89]
+// GFX12: s_min_u32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0xfe,0x89]
 
 s_min_u32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xff,0x89]
+// GFX12: s_min_u32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0xff,0x89]
 
 s_min_u32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xea,0x89]
+// GFX12: s_min_u32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0xea,0x89]
 
 s_min_u32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xeb,0x89]
+// GFX12: s_min_u32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0xeb,0x89]
 
 s_min_u32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfd,0x89]
+// GFX12: s_min_u32 m0, s1, s2                    ; encoding: [0x01,0x02,0xfd,0x89]
 
 s_min_u32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x80,0x89]
+// GFX12: s_min_u32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x80,0x89]
 
 s_min_u32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x80,0x89]
+// GFX12: s_min_u32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x80,0x89]
 
 s_min_u32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x80,0x89]
+// GFX12: s_min_u32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x80,0x89]
 
 s_min_u32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x80,0x89]
+// GFX12: s_min_u32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x80,0x89]
 
 s_min_u32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x80,0x89]
+// GFX12: s_min_u32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x80,0x89]
 
 s_min_u32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x80,0x89]
+// GFX12: s_min_u32 s0, 0, s2                     ; encoding: [0x80,0x02,0x80,0x89]
 
 s_min_u32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x80,0x89]
+// GFX12: s_min_u32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x80,0x89]
 
 s_min_u32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x80,0x89]
+// GFX12: s_min_u32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x80,0x89]
 
 s_min_u32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x80,0x89]
+// GFX12: s_min_u32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x80,0x89]
 
 s_min_u32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x89,0x73,0x72,0x71,0x3f]
+// GFX12: s_min_u32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x80,0x89,0x73,0x72,0x71,0x3f]
 
 s_min_u32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x89,0x56,0x34,0x12,0xaf]
+// GFX12: s_min_u32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x80,0x89,0x56,0x34,0x12,0xaf]
 
 s_min_u32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x80,0x89]
+// GFX12: s_min_u32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x80,0x89]
 
 s_min_u32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x80,0x89]
+// GFX12: s_min_u32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x80,0x89]
 
 s_min_u32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x80,0x89]
+// GFX12: s_min_u32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x80,0x89]
 
 s_min_u32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x80,0x89]
+// GFX12: s_min_u32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x80,0x89]
 
 s_min_u32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x80,0x89]
+// GFX12: s_min_u32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x80,0x89]
 
 s_min_u32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x80,0x89]
+// GFX12: s_min_u32 s0, s1, 0                     ; encoding: [0x01,0x80,0x80,0x89]
 
 s_min_u32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x80,0x89]
+// GFX12: s_min_u32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x80,0x89]
 
 s_min_u32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x80,0x89]
+// GFX12: s_min_u32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x80,0x89]
 
 s_min_u32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x80,0x89]
+// GFX12: s_min_u32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x80,0x89]
 
 s_min_u32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x80,0x89,0x73,0x72,0x71,0x3f]
+// GFX12: s_min_u32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x80,0x89,0x73,0x72,0x71,0x3f]
 
 s_min_u32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x80,0x89,0x56,0x34,0x12,0xaf]
+// GFX12: s_min_u32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x80,0x89,0x56,0x34,0x12,0xaf]
 
 s_max_i32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x8a]
+// GFX12: s_max_i32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x8a]
 
 s_max_i32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x8a]
+// GFX12: s_max_i32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x8a]
 
 s_max_i32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x8a]
+// GFX12: s_max_i32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x8a]
 
 s_max_i32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x8a]
+// GFX12: s_max_i32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x8a]
 
 s_max_i32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x8a]
+// GFX12: s_max_i32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x8a]
 
 s_max_i32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x8a]
+// GFX12: s_max_i32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x8a]
 
 s_max_i32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x8a]
+// GFX12: s_max_i32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x8a]
 
 s_max_i32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x8a]
+// GFX12: s_max_i32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x8a]
 
 s_max_i32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x8a]
+// GFX12: s_max_i32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x8a]
 
 s_max_i32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x8a]
+// GFX12: s_max_i32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x8a]
 
 s_max_i32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x8a]
+// GFX12: s_max_i32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x8a]
 
 s_max_i32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x8a]
+// GFX12: s_max_i32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x8a]
 
 s_max_i32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x8a]
+// GFX12: s_max_i32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x8a]
 
 s_max_i32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x8a]
+// GFX12: s_max_i32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x8a]
 
 s_max_i32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x8a]
+// GFX12: s_max_i32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x8a]
 
 s_max_i32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x8a]
+// GFX12: s_max_i32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x8a]
 
 s_max_i32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x8a]
+// GFX12: s_max_i32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x8a]
 
 s_max_i32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x8a]
+// GFX12: s_max_i32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x8a]
 
 s_max_i32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x8a]
+// GFX12: s_max_i32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x8a]
 
 s_max_i32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x8a]
+// GFX12: s_max_i32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x8a]
 
 s_max_i32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x8a]
+// GFX12: s_max_i32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x8a]
 
 s_max_i32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x8a]
+// GFX12: s_max_i32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x8a]
 
 s_max_i32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x8a,0x73,0x72,0x71,0x3f]
+// GFX12: s_max_i32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x8a,0x73,0x72,0x71,0x3f]
 
 s_max_i32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x8a,0x56,0x34,0x12,0xaf]
+// GFX12: s_max_i32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x8a,0x56,0x34,0x12,0xaf]
 
 s_max_i32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x8a]
+// GFX12: s_max_i32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x8a]
 
 s_max_i32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x8a]
+// GFX12: s_max_i32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x8a]
 
 s_max_i32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x8a]
+// GFX12: s_max_i32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x8a]
 
 s_max_i32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x8a]
+// GFX12: s_max_i32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x8a]
 
 s_max_i32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x8a]
+// GFX12: s_max_i32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x8a]
 
 s_max_i32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x8a]
+// GFX12: s_max_i32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x8a]
 
 s_max_i32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x8a]
+// GFX12: s_max_i32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x8a]
 
 s_max_i32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x8a]
+// GFX12: s_max_i32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x8a]
 
 s_max_i32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x8a]
+// GFX12: s_max_i32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x8a]
 
 s_max_i32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x8a,0x73,0x72,0x71,0x3f]
+// GFX12: s_max_i32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x8a,0x73,0x72,0x71,0x3f]
 
 s_max_i32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x8a,0x56,0x34,0x12,0xaf]
+// GFX12: s_max_i32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x8a,0x56,0x34,0x12,0xaf]
 
 s_max_u32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x80,0x8a]
+// GFX12: s_max_u32 s0, s1, s2                    ; encoding: [0x01,0x02,0x80,0x8a]
 
 s_max_u32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0xe9,0x8a]
+// GFX12: s_max_u32 s105, s104, s103              ; encoding: [0x68,0x67,0xe9,0x8a]
 
 s_max_u32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x80,0x8a]
+// GFX12: s_max_u32 s0, s104, s103                ; encoding: [0x68,0x67,0x80,0x8a]
 
 s_max_u32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0xe9,0x8a]
+// GFX12: s_max_u32 s105, s1, s103                ; encoding: [0x01,0x67,0xe9,0x8a]
 
 s_max_u32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0xe9,0x8a]
+// GFX12: s_max_u32 s105, s104, s2                ; encoding: [0x68,0x02,0xe9,0x8a]
 
 s_max_u32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0xe9,0x8a]
+// GFX12: s_max_u32 s105, s1, s2                  ; encoding: [0x01,0x02,0xe9,0x8a]
 
 s_max_u32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x80,0x8a]
+// GFX12: s_max_u32 s0, s104, s2                  ; encoding: [0x68,0x02,0x80,0x8a]
 
 s_max_u32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x80,0x8a]
+// GFX12: s_max_u32 s0, s1, s103                  ; encoding: [0x01,0x67,0x80,0x8a]
 
 s_max_u32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfe,0x8a]
+// GFX12: s_max_u32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0xfe,0x8a]
 
 s_max_u32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xff,0x8a]
+// GFX12: s_max_u32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0xff,0x8a]
 
 s_max_u32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xea,0x8a]
+// GFX12: s_max_u32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0xea,0x8a]
 
 s_max_u32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xeb,0x8a]
+// GFX12: s_max_u32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0xeb,0x8a]
 
 s_max_u32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfd,0x8a]
+// GFX12: s_max_u32 m0, s1, s2                    ; encoding: [0x01,0x02,0xfd,0x8a]
 
 s_max_u32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x80,0x8a]
+// GFX12: s_max_u32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x80,0x8a]
 
 s_max_u32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x80,0x8a]
+// GFX12: s_max_u32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x80,0x8a]
 
 s_max_u32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x80,0x8a]
+// GFX12: s_max_u32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x80,0x8a]
 
 s_max_u32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x80,0x8a]
+// GFX12: s_max_u32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x80,0x8a]
 
 s_max_u32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x80,0x8a]
+// GFX12: s_max_u32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x80,0x8a]
 
 s_max_u32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x80,0x8a]
+// GFX12: s_max_u32 s0, 0, s2                     ; encoding: [0x80,0x02,0x80,0x8a]
 
 s_max_u32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x80,0x8a]
+// GFX12: s_max_u32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x80,0x8a]
 
 s_max_u32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x80,0x8a]
+// GFX12: s_max_u32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x80,0x8a]
 
 s_max_u32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x80,0x8a]
+// GFX12: s_max_u32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x80,0x8a]
 
 s_max_u32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x8a,0x73,0x72,0x71,0x3f]
+// GFX12: s_max_u32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x80,0x8a,0x73,0x72,0x71,0x3f]
 
 s_max_u32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x8a,0x56,0x34,0x12,0xaf]
+// GFX12: s_max_u32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x80,0x8a,0x56,0x34,0x12,0xaf]
 
 s_max_u32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x80,0x8a]
+// GFX12: s_max_u32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x80,0x8a]
 
 s_max_u32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x80,0x8a]
+// GFX12: s_max_u32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x80,0x8a]
 
 s_max_u32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x80,0x8a]
+// GFX12: s_max_u32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x80,0x8a]
 
 s_max_u32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x80,0x8a]
+// GFX12: s_max_u32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x80,0x8a]
 
 s_max_u32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x80,0x8a]
+// GFX12: s_max_u32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x80,0x8a]
 
 s_max_u32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x80,0x8a]
+// GFX12: s_max_u32 s0, s1, 0                     ; encoding: [0x01,0x80,0x80,0x8a]
 
 s_max_u32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x80,0x8a]
+// GFX12: s_max_u32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x80,0x8a]
 
 s_max_u32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x80,0x8a]
+// GFX12: s_max_u32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x80,0x8a]
 
 s_max_u32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x80,0x8a]
+// GFX12: s_max_u32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x80,0x8a]
 
 s_max_u32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x80,0x8a,0x73,0x72,0x71,0x3f]
+// GFX12: s_max_u32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x80,0x8a,0x73,0x72,0x71,0x3f]
 
 s_max_u32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x80,0x8a,0x56,0x34,0x12,0xaf]
+// GFX12: s_max_u32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x80,0x8a,0x56,0x34,0x12,0xaf]
 
 s_cselect_b32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x98]
+// GFX12: s_cselect_b32 s0, s1, s2                ; encoding: [0x01,0x02,0x00,0x98]
 
 s_cselect_b32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x98]
+// GFX12: s_cselect_b32 s105, s104, s103          ; encoding: [0x68,0x67,0x69,0x98]
 
 s_cselect_b32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x98]
+// GFX12: s_cselect_b32 s0, s104, s103            ; encoding: [0x68,0x67,0x00,0x98]
 
 s_cselect_b32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x98]
+// GFX12: s_cselect_b32 s105, s1, s103            ; encoding: [0x01,0x67,0x69,0x98]
 
 s_cselect_b32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x98]
+// GFX12: s_cselect_b32 s105, s104, s2            ; encoding: [0x68,0x02,0x69,0x98]
 
 s_cselect_b32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x98]
+// GFX12: s_cselect_b32 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0x98]
 
 s_cselect_b32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x98]
+// GFX12: s_cselect_b32 s0, s104, s2              ; encoding: [0x68,0x02,0x00,0x98]
 
 s_cselect_b32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x98]
+// GFX12: s_cselect_b32 s0, s1, s103              ; encoding: [0x01,0x67,0x00,0x98]
 
 s_cselect_b32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x98]
+// GFX12: s_cselect_b32 exec_lo, s1, s2           ; encoding: [0x01,0x02,0x7e,0x98]
 
 s_cselect_b32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x98]
+// GFX12: s_cselect_b32 exec_hi, s1, s2           ; encoding: [0x01,0x02,0x7f,0x98]
 
 s_cselect_b32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x98]
+// GFX12: s_cselect_b32 vcc_lo, s1, s2            ; encoding: [0x01,0x02,0x6a,0x98]
 
 s_cselect_b32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x98]
+// GFX12: s_cselect_b32 vcc_hi, s1, s2            ; encoding: [0x01,0x02,0x6b,0x98]
 
 s_cselect_b32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x98]
+// GFX12: s_cselect_b32 m0, s1, s2                ; encoding: [0x01,0x02,0x7d,0x98]
 
 s_cselect_b32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x98]
+// GFX12: s_cselect_b32 s0, exec_lo, s2           ; encoding: [0x7e,0x02,0x00,0x98]
 
 s_cselect_b32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x98]
+// GFX12: s_cselect_b32 s0, exec_hi, s2           ; encoding: [0x7f,0x02,0x00,0x98]
 
 s_cselect_b32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x98]
+// GFX12: s_cselect_b32 s0, vcc_lo, s2            ; encoding: [0x6a,0x02,0x00,0x98]
 
 s_cselect_b32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x98]
+// GFX12: s_cselect_b32 s0, vcc_hi, s2            ; encoding: [0x6b,0x02,0x00,0x98]
 
 s_cselect_b32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x98]
+// GFX12: s_cselect_b32 s0, m0, s2                ; encoding: [0x7d,0x02,0x00,0x98]
 
 s_cselect_b32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x98]
+// GFX12: s_cselect_b32 s0, 0, s2                 ; encoding: [0x80,0x02,0x00,0x98]
 
 s_cselect_b32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x98]
+// GFX12: s_cselect_b32 s0, -1, s2                ; encoding: [0xc1,0x02,0x00,0x98]
 
 s_cselect_b32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x98]
+// GFX12: s_cselect_b32 s0, 0.5, s2               ; encoding: [0xf0,0x02,0x00,0x98]
 
 s_cselect_b32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x98]
+// GFX12: s_cselect_b32 s0, -4.0, s2              ; encoding: [0xf7,0x02,0x00,0x98]
 
 s_cselect_b32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x98,0x73,0x72,0x71,0x3f]
+// GFX12: s_cselect_b32 s0, 0x3f717273, s2        ; encoding: [0xff,0x02,0x00,0x98,0x73,0x72,0x71,0x3f]
 
 s_cselect_b32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x98,0x56,0x34,0x12,0xaf]
+// GFX12: s_cselect_b32 s0, 0xaf123456, s2        ; encoding: [0xff,0x02,0x00,0x98,0x56,0x34,0x12,0xaf]
 
 s_cselect_b32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x98]
+// GFX12: s_cselect_b32 s0, s1, exec_lo           ; encoding: [0x01,0x7e,0x00,0x98]
 
 s_cselect_b32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x98]
+// GFX12: s_cselect_b32 s0, s1, exec_hi           ; encoding: [0x01,0x7f,0x00,0x98]
 
 s_cselect_b32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x98]
+// GFX12: s_cselect_b32 s0, s1, vcc_lo            ; encoding: [0x01,0x6a,0x00,0x98]
 
 s_cselect_b32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x98]
+// GFX12: s_cselect_b32 s0, s1, vcc_hi            ; encoding: [0x01,0x6b,0x00,0x98]
 
 s_cselect_b32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x98]
+// GFX12: s_cselect_b32 s0, s1, m0                ; encoding: [0x01,0x7d,0x00,0x98]
 
 s_cselect_b32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x98]
+// GFX12: s_cselect_b32 s0, s1, 0                 ; encoding: [0x01,0x80,0x00,0x98]
 
 s_cselect_b32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x98]
+// GFX12: s_cselect_b32 s0, s1, -1                ; encoding: [0x01,0xc1,0x00,0x98]
 
 s_cselect_b32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x98]
+// GFX12: s_cselect_b32 s0, s1, 0.5               ; encoding: [0x01,0xf0,0x00,0x98]
 
 s_cselect_b32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x98]
+// GFX12: s_cselect_b32 s0, s1, -4.0              ; encoding: [0x01,0xf7,0x00,0x98]
 
 s_cselect_b32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x98,0x73,0x72,0x71,0x3f]
+// GFX12: s_cselect_b32 s0, s1, 0x3f717273        ; encoding: [0x01,0xff,0x00,0x98,0x73,0x72,0x71,0x3f]
 
 s_cselect_b32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x98,0x56,0x34,0x12,0xaf]
+// GFX12: s_cselect_b32 s0, s1, 0xaf123456        ; encoding: [0x01,0xff,0x00,0x98,0x56,0x34,0x12,0xaf]
 
 s_cselect_b64 s[0:1], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0x80,0x98]
+// GFX12: s_cselect_b64 s[0:1], s[2:3], s[4:5]    ; encoding: [0x02,0x04,0x80,0x98]
 
 s_cselect_b64 s[104:105], s[102:103], s[100:101]
-// GFX12: encoding: [0x66,0x64,0xe8,0x98]
+// GFX12: s_cselect_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x98]
 
 s_cselect_b64 s[0:1], s[102:103], s[100:101]
-// GFX12: encoding: [0x66,0x64,0x80,0x98]
+// GFX12: s_cselect_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x98]
 
 s_cselect_b64 s[104:105], s[2:3], s[100:101]
-// GFX12: encoding: [0x02,0x64,0xe8,0x98]
+// GFX12: s_cselect_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x98]
 
 s_cselect_b64 s[104:105], s[102:103], s[4:5]
-// GFX12: encoding: [0x66,0x04,0xe8,0x98]
+// GFX12: s_cselect_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x98]
 
 s_cselect_b64 s[104:105], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xe8,0x98]
+// GFX12: s_cselect_b64 s[104:105], s[2:3], s[4:5] ; encoding: [0x02,0x04,0xe8,0x98]
 
 s_cselect_b64 s[0:1], s[102:103], s[4:5]
-// GFX12: encoding: [0x66,0x04,0x80,0x98]
+// GFX12: s_cselect_b64 s[0:1], s[102:103], s[4:5] ; encoding: [0x66,0x04,0x80,0x98]
 
 s_cselect_b64 s[0:1], s[2:3], s[100:101]
-// GFX12: encoding: [0x02,0x64,0x80,0x98]
+// GFX12: s_cselect_b64 s[0:1], s[2:3], s[100:101] ; encoding: [0x02,0x64,0x80,0x98]
 
 s_cselect_b64 exec, s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xfe,0x98]
+// GFX12: s_cselect_b64 exec, s[2:3], s[4:5]      ; encoding: [0x02,0x04,0xfe,0x98]
 
 s_cselect_b64 vcc, s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xea,0x98]
+// GFX12: s_cselect_b64 vcc, s[2:3], s[4:5]       ; encoding: [0x02,0x04,0xea,0x98]
 
 s_cselect_b64 s[0:1], exec, s[4:5]
-// GFX12: encoding: [0x7e,0x04,0x80,0x98]
+// GFX12: s_cselect_b64 s[0:1], exec, s[4:5]      ; encoding: [0x7e,0x04,0x80,0x98]
 
 s_cselect_b64 s[0:1], vcc, s[4:5]
-// GFX12: encoding: [0x6a,0x04,0x80,0x98]
+// GFX12: s_cselect_b64 s[0:1], vcc, s[4:5]       ; encoding: [0x6a,0x04,0x80,0x98]
 
 s_cselect_b64 s[0:1], 0, s[4:5]
-// GFX12: encoding: [0x80,0x04,0x80,0x98]
+// GFX12: s_cselect_b64 s[0:1], 0, s[4:5]         ; encoding: [0x80,0x04,0x80,0x98]
 
 s_cselect_b64 s[0:1], -1, s[4:5]
-// GFX12: encoding: [0xc1,0x04,0x80,0x98]
+// GFX12: s_cselect_b64 s[0:1], -1, s[4:5]        ; encoding: [0xc1,0x04,0x80,0x98]
 
 s_cselect_b64 s[0:1], 0.5, s[4:5]
-// GFX12: encoding: [0xf0,0x04,0x80,0x98]
+// GFX12: s_cselect_b64 s[0:1], 0.5, s[4:5]       ; encoding: [0xf0,0x04,0x80,0x98]
 
 s_cselect_b64 s[0:1], -4.0, s[4:5]
-// GFX12: encoding: [0xf7,0x04,0x80,0x98]
+// GFX12: s_cselect_b64 s[0:1], -4.0, s[4:5]      ; encoding: [0xf7,0x04,0x80,0x98]
 
 s_cselect_b64 s[0:1], 0x3f717273, s[4:5]
-// GFX12: encoding: [0xff,0x04,0x80,0x98,0x73,0x72,0x71,0x3f]
+// GFX12: s_cselect_b64 s[0:1], 0x3f717273, s[4:5] ; encoding: [0xff,0x04,0x80,0x98,0x73,0x72,0x71,0x3f]
 
 s_cselect_b64 s[0:1], 0xaf123456, s[4:5]
-// GFX12: encoding: [0xff,0x04,0x80,0x98,0x56,0x34,0x12,0xaf]
+// GFX12: s_cselect_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x98,0x56,0x34,0x12,0xaf]
 
 s_cselect_b64 s[0:1], s[2:3], exec
-// GFX12: encoding: [0x02,0x7e,0x80,0x98]
+// GFX12: s_cselect_b64 s[0:1], s[2:3], exec      ; encoding: [0x02,0x7e,0x80,0x98]
 
 s_cselect_b64 s[0:1], s[2:3], vcc
-// GFX12: encoding: [0x02,0x6a,0x80,0x98]
+// GFX12: s_cselect_b64 s[0:1], s[2:3], vcc       ; encoding: [0x02,0x6a,0x80,0x98]
 
 s_cselect_b64 s[0:1], s[2:3], 0
-// GFX12: encoding: [0x02,0x80,0x80,0x98]
+// GFX12: s_cselect_b64 s[0:1], s[2:3], 0         ; encoding: [0x02,0x80,0x80,0x98]
 
 s_cselect_b64 s[0:1], s[2:3], -1
-// GFX12: encoding: [0x02,0xc1,0x80,0x98]
+// GFX12: s_cselect_b64 s[0:1], s[2:3], -1        ; encoding: [0x02,0xc1,0x80,0x98]
 
 s_cselect_b64 s[0:1], s[2:3], 0.5
-// GFX12: encoding: [0x02,0xf0,0x80,0x98]
+// GFX12: s_cselect_b64 s[0:1], s[2:3], 0.5       ; encoding: [0x02,0xf0,0x80,0x98]
 
 s_cselect_b64 s[0:1], s[2:3], -4.0
-// GFX12: encoding: [0x02,0xf7,0x80,0x98]
+// GFX12: s_cselect_b64 s[0:1], s[2:3], -4.0      ; encoding: [0x02,0xf7,0x80,0x98]
 
 s_cselect_b64 s[0:1], s[2:3], 0x3f717273
-// GFX12: encoding: [0x02,0xff,0x80,0x98,0x73,0x72,0x71,0x3f]
+// GFX12: s_cselect_b64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0x98,0x73,0x72,0x71,0x3f]
 
 s_cselect_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: encoding: [0x02,0xff,0x80,0x98,0x56,0x34,0x12,0xaf]
+// GFX12: s_cselect_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x98,0x56,0x34,0x12,0xaf]
 
 s_and_b32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x8b]
+// GFX12: s_and_b32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x8b]
 
 s_and_b32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x8b]
+// GFX12: s_and_b32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x8b]
 
 s_and_b32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x8b]
+// GFX12: s_and_b32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x8b]
 
 s_and_b32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x8b]
+// GFX12: s_and_b32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x8b]
 
 s_and_b32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x8b]
+// GFX12: s_and_b32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x8b]
 
 s_and_b32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x8b]
+// GFX12: s_and_b32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x8b]
 
 s_and_b32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x8b]
+// GFX12: s_and_b32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x8b]
 
 s_and_b32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x8b]
+// GFX12: s_and_b32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x8b]
 
 s_and_b32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x8b]
+// GFX12: s_and_b32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x8b]
 
 s_and_b32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x8b]
+// GFX12: s_and_b32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x8b]
 
 s_and_b32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x8b]
+// GFX12: s_and_b32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x8b]
 
 s_and_b32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x8b]
+// GFX12: s_and_b32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x8b]
 
 s_and_b32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x8b]
+// GFX12: s_and_b32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x8b]
 
 s_and_b32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x8b]
+// GFX12: s_and_b32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x8b]
 
 s_and_b32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x8b]
+// GFX12: s_and_b32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x8b]
 
 s_and_b32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x8b]
+// GFX12: s_and_b32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x8b]
 
 s_and_b32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x8b]
+// GFX12: s_and_b32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x8b]
 
 s_and_b32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x8b]
+// GFX12: s_and_b32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x8b]
 
 s_and_b32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x8b]
+// GFX12: s_and_b32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x8b]
 
 s_and_b32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x8b]
+// GFX12: s_and_b32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x8b]
 
 s_and_b32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x8b]
+// GFX12: s_and_b32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x8b]
 
 s_and_b32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x8b]
+// GFX12: s_and_b32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x8b]
 
 s_and_b32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x8b,0x73,0x72,0x71,0x3f]
+// GFX12: s_and_b32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x8b,0x73,0x72,0x71,0x3f]
 
 s_and_b32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x8b,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_b32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x8b,0x56,0x34,0x12,0xaf]
 
 s_and_b32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x8b]
+// GFX12: s_and_b32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x8b]
 
 s_and_b32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x8b]
+// GFX12: s_and_b32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x8b]
 
 s_and_b32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x8b]
+// GFX12: s_and_b32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x8b]
 
 s_and_b32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x8b]
+// GFX12: s_and_b32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x8b]
 
 s_and_b32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x8b]
+// GFX12: s_and_b32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x8b]
 
 s_and_b32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x8b]
+// GFX12: s_and_b32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x8b]
 
 s_and_b32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x8b]
+// GFX12: s_and_b32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x8b]
 
 s_and_b32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x8b]
+// GFX12: s_and_b32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x8b]
 
 s_and_b32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x8b]
+// GFX12: s_and_b32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x8b]
 
 s_and_b32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x8b,0x73,0x72,0x71,0x3f]
+// GFX12: s_and_b32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x8b,0x73,0x72,0x71,0x3f]
 
 s_and_b32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x8b,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_b32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x8b,0x56,0x34,0x12,0xaf]
 
 s_and_b32 s0, s1, null
-// GFX12: encoding: [0x01,0x7c,0x00,0x8b]
+// GFX12: s_and_b32 s0, s1, null                  ; encoding: [0x01,0x7c,0x00,0x8b]
 
 s_and_b32 s0, null, s2
-// GFX12: encoding: [0x7c,0x02,0x00,0x8b]
+// GFX12: s_and_b32 s0, null, s2                  ; encoding: [0x7c,0x02,0x00,0x8b]
 
 s_and_b32 null, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7c,0x8b]
+// GFX12: s_and_b32 null, s1, s2                  ; encoding: [0x01,0x02,0x7c,0x8b]
 
 s_and_b64 s[0:1], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0x80,0x8b]
+// GFX12: s_and_b64 s[0:1], s[2:3], s[4:5]        ; encoding: [0x02,0x04,0x80,0x8b]
 
 s_and_b64 s[104:105], s[102:103], s[100:101]
-// GFX12: encoding: [0x66,0x64,0xe8,0x8b]
+// GFX12: s_and_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x8b]
 
 s_and_b64 s[0:1], s[102:103], s[100:101]
-// GFX12: encoding: [0x66,0x64,0x80,0x8b]
+// GFX12: s_and_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x8b]
 
 s_and_b64 s[104:105], s[2:3], s[100:101]
-// GFX12: encoding: [0x02,0x64,0xe8,0x8b]
+// GFX12: s_and_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x8b]
 
 s_and_b64 s[104:105], s[102:103], s[4:5]
-// GFX12: encoding: [0x66,0x04,0xe8,0x8b]
+// GFX12: s_and_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x8b]
 
 s_and_b64 s[104:105], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xe8,0x8b]
+// GFX12: s_and_b64 s[104:105], s[2:3], s[4:5]    ; encoding: [0x02,0x04,0xe8,0x8b]
 
 s_and_b64 s[0:1], s[102:103], s[4:5]
-// GFX12: encoding: [0x66,0x04,0x80,0x8b]
+// GFX12: s_and_b64 s[0:1], s[102:103], s[4:5]    ; encoding: [0x66,0x04,0x80,0x8b]
 
 s_and_b64 s[0:1], s[2:3], s[100:101]
-// GFX12: encoding: [0x02,0x64,0x80,0x8b]
+// GFX12: s_and_b64 s[0:1], s[2:3], s[100:101]    ; encoding: [0x02,0x64,0x80,0x8b]
 
 s_and_b64 exec, s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xfe,0x8b]
+// GFX12: s_and_b64 exec, s[2:3], s[4:5]          ; encoding: [0x02,0x04,0xfe,0x8b]
 
 s_and_b64 vcc, s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xea,0x8b]
+// GFX12: s_and_b64 vcc, s[2:3], s[4:5]           ; encoding: [0x02,0x04,0xea,0x8b]
 
 s_and_b64 s[0:1], exec, s[4:5]
-// GFX12: encoding: [0x7e,0x04,0x80,0x8b]
+// GFX12: s_and_b64 s[0:1], exec, s[4:5]          ; encoding: [0x7e,0x04,0x80,0x8b]
 
 s_and_b64 s[0:1], vcc, s[4:5]
-// GFX12: encoding: [0x6a,0x04,0x80,0x8b]
+// GFX12: s_and_b64 s[0:1], vcc, s[4:5]           ; encoding: [0x6a,0x04,0x80,0x8b]
 
 s_and_b64 s[0:1], 0, s[4:5]
-// GFX12: encoding: [0x80,0x04,0x80,0x8b]
+// GFX12: s_and_b64 s[0:1], 0, s[4:5]             ; encoding: [0x80,0x04,0x80,0x8b]
 
 s_and_b64 s[0:1], -1, s[4:5]
-// GFX12: encoding: [0xc1,0x04,0x80,0x8b]
+// GFX12: s_and_b64 s[0:1], -1, s[4:5]            ; encoding: [0xc1,0x04,0x80,0x8b]
 
 s_and_b64 s[0:1], 0.5, s[4:5]
-// GFX12: encoding: [0xf0,0x04,0x80,0x8b]
+// GFX12: s_and_b64 s[0:1], 0.5, s[4:5]           ; encoding: [0xf0,0x04,0x80,0x8b]
 
 s_and_b64 s[0:1], -4.0, s[4:5]
-// GFX12: encoding: [0xf7,0x04,0x80,0x8b]
+// GFX12: s_and_b64 s[0:1], -4.0, s[4:5]          ; encoding: [0xf7,0x04,0x80,0x8b]
 
 s_and_b64 s[0:1], 0x3f717273, s[4:5]
-// GFX12: encoding: [0xff,0x04,0x80,0x8b,0x73,0x72,0x71,0x3f]
+// GFX12: s_and_b64 s[0:1], 0x3f717273, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8b,0x73,0x72,0x71,0x3f]
 
 s_and_b64 s[0:1], 0xaf123456, s[4:5]
-// GFX12: encoding: [0xff,0x04,0x80,0x8b,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8b,0x56,0x34,0x12,0xaf]
 
 s_and_b64 s[0:1], s[2:3], exec
-// GFX12: encoding: [0x02,0x7e,0x80,0x8b]
+// GFX12: s_and_b64 s[0:1], s[2:3], exec          ; encoding: [0x02,0x7e,0x80,0x8b]
 
 s_and_b64 s[0:1], s[2:3], vcc
-// GFX12: encoding: [0x02,0x6a,0x80,0x8b]
+// GFX12: s_and_b64 s[0:1], s[2:3], vcc           ; encoding: [0x02,0x6a,0x80,0x8b]
 
 s_and_b64 s[0:1], s[2:3], 0
-// GFX12: encoding: [0x02,0x80,0x80,0x8b]
+// GFX12: s_and_b64 s[0:1], s[2:3], 0             ; encoding: [0x02,0x80,0x80,0x8b]
 
 s_and_b64 s[0:1], s[2:3], -1
-// GFX12: encoding: [0x02,0xc1,0x80,0x8b]
+// GFX12: s_and_b64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x80,0x8b]
 
 s_and_b64 s[0:1], s[2:3], 0.5
-// GFX12: encoding: [0x02,0xf0,0x80,0x8b]
+// GFX12: s_and_b64 s[0:1], s[2:3], 0.5           ; encoding: [0x02,0xf0,0x80,0x8b]
 
 s_and_b64 s[0:1], s[2:3], -4.0
-// GFX12: encoding: [0x02,0xf7,0x80,0x8b]
+// GFX12: s_and_b64 s[0:1], s[2:3], -4.0          ; encoding: [0x02,0xf7,0x80,0x8b]
 
 s_and_b64 s[0:1], s[2:3], 0x3f717273
-// GFX12: encoding: [0x02,0xff,0x80,0x8b,0x73,0x72,0x71,0x3f]
+// GFX12: s_and_b64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0x8b,0x73,0x72,0x71,0x3f]
 
 s_and_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: encoding: [0x02,0xff,0x80,0x8b,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8b,0x56,0x34,0x12,0xaf]
 
 s_or_b32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x8c]
+// GFX12: s_or_b32 s0, s1, s2                     ; encoding: [0x01,0x02,0x00,0x8c]
 
 s_or_b32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x8c]
+// GFX12: s_or_b32 s105, s104, s103               ; encoding: [0x68,0x67,0x69,0x8c]
 
 s_or_b32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x8c]
+// GFX12: s_or_b32 s0, s104, s103                 ; encoding: [0x68,0x67,0x00,0x8c]
 
 s_or_b32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x8c]
+// GFX12: s_or_b32 s105, s1, s103                 ; encoding: [0x01,0x67,0x69,0x8c]
 
 s_or_b32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x8c]
+// GFX12: s_or_b32 s105, s104, s2                 ; encoding: [0x68,0x02,0x69,0x8c]
 
 s_or_b32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x8c]
+// GFX12: s_or_b32 s105, s1, s2                   ; encoding: [0x01,0x02,0x69,0x8c]
 
 s_or_b32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x8c]
+// GFX12: s_or_b32 s0, s104, s2                   ; encoding: [0x68,0x02,0x00,0x8c]
 
 s_or_b32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x8c]
+// GFX12: s_or_b32 s0, s1, s103                   ; encoding: [0x01,0x67,0x00,0x8c]
 
 s_or_b32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x8c]
+// GFX12: s_or_b32 exec_lo, s1, s2                ; encoding: [0x01,0x02,0x7e,0x8c]
 
 s_or_b32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x8c]
+// GFX12: s_or_b32 exec_hi, s1, s2                ; encoding: [0x01,0x02,0x7f,0x8c]
 
 s_or_b32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x8c]
+// GFX12: s_or_b32 vcc_lo, s1, s2                 ; encoding: [0x01,0x02,0x6a,0x8c]
 
 s_or_b32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x8c]
+// GFX12: s_or_b32 vcc_hi, s1, s2                 ; encoding: [0x01,0x02,0x6b,0x8c]
 
 s_or_b32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x8c]
+// GFX12: s_or_b32 m0, s1, s2                     ; encoding: [0x01,0x02,0x7d,0x8c]
 
 s_or_b32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x8c]
+// GFX12: s_or_b32 s0, exec_lo, s2                ; encoding: [0x7e,0x02,0x00,0x8c]
 
 s_or_b32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x8c]
+// GFX12: s_or_b32 s0, exec_hi, s2                ; encoding: [0x7f,0x02,0x00,0x8c]
 
 s_or_b32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x8c]
+// GFX12: s_or_b32 s0, vcc_lo, s2                 ; encoding: [0x6a,0x02,0x00,0x8c]
 
 s_or_b32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x8c]
+// GFX12: s_or_b32 s0, vcc_hi, s2                 ; encoding: [0x6b,0x02,0x00,0x8c]
 
 s_or_b32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x8c]
+// GFX12: s_or_b32 s0, m0, s2                     ; encoding: [0x7d,0x02,0x00,0x8c]
 
 s_or_b32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x8c]
+// GFX12: s_or_b32 s0, 0, s2                      ; encoding: [0x80,0x02,0x00,0x8c]
 
 s_or_b32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x8c]
+// GFX12: s_or_b32 s0, -1, s2                     ; encoding: [0xc1,0x02,0x00,0x8c]
 
 s_or_b32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x8c]
+// GFX12: s_or_b32 s0, 0.5, s2                    ; encoding: [0xf0,0x02,0x00,0x8c]
 
 s_or_b32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x8c]
+// GFX12: s_or_b32 s0, -4.0, s2                   ; encoding: [0xf7,0x02,0x00,0x8c]
 
 s_or_b32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x8c,0x73,0x72,0x71,0x3f]
+// GFX12: s_or_b32 s0, 0x3f717273, s2             ; encoding: [0xff,0x02,0x00,0x8c,0x73,0x72,0x71,0x3f]
 
 s_or_b32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x8c,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_b32 s0, 0xaf123456, s2             ; encoding: [0xff,0x02,0x00,0x8c,0x56,0x34,0x12,0xaf]
 
 s_or_b32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x8c]
+// GFX12: s_or_b32 s0, s1, exec_lo                ; encoding: [0x01,0x7e,0x00,0x8c]
 
 s_or_b32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x8c]
+// GFX12: s_or_b32 s0, s1, exec_hi                ; encoding: [0x01,0x7f,0x00,0x8c]
 
 s_or_b32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x8c]
+// GFX12: s_or_b32 s0, s1, vcc_lo                 ; encoding: [0x01,0x6a,0x00,0x8c]
 
 s_or_b32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x8c]
+// GFX12: s_or_b32 s0, s1, vcc_hi                 ; encoding: [0x01,0x6b,0x00,0x8c]
 
 s_or_b32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x8c]
+// GFX12: s_or_b32 s0, s1, m0                     ; encoding: [0x01,0x7d,0x00,0x8c]
 
 s_or_b32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x8c]
+// GFX12: s_or_b32 s0, s1, 0                      ; encoding: [0x01,0x80,0x00,0x8c]
 
 s_or_b32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x8c]
+// GFX12: s_or_b32 s0, s1, -1                     ; encoding: [0x01,0xc1,0x00,0x8c]
 
 s_or_b32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x8c]
+// GFX12: s_or_b32 s0, s1, 0.5                    ; encoding: [0x01,0xf0,0x00,0x8c]
 
 s_or_b32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x8c]
+// GFX12: s_or_b32 s0, s1, -4.0                   ; encoding: [0x01,0xf7,0x00,0x8c]
 
 s_or_b32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x8c,0x73,0x72,0x71,0x3f]
+// GFX12: s_or_b32 s0, s1, 0x3f717273             ; encoding: [0x01,0xff,0x00,0x8c,0x73,0x72,0x71,0x3f]
 
 s_or_b32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x8c,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_b32 s0, s1, 0xaf123456             ; encoding: [0x01,0xff,0x00,0x8c,0x56,0x34,0x12,0xaf]
 
 s_or_b64 s[0:1], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0x80,0x8c]
+// GFX12: s_or_b64 s[0:1], s[2:3], s[4:5]         ; encoding: [0x02,0x04,0x80,0x8c]
 
 s_or_b64 s[104:105], s[102:103], s[100:101]
-// GFX12: encoding: [0x66,0x64,0xe8,0x8c]
+// GFX12: s_or_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x8c]
 
 s_or_b64 s[0:1], s[102:103], s[100:101]
-// GFX12: encoding: [0x66,0x64,0x80,0x8c]
+// GFX12: s_or_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x8c]
 
 s_or_b64 s[104:105], s[2:3], s[100:101]
-// GFX12: encoding: [0x02,0x64,0xe8,0x8c]
+// GFX12: s_or_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x8c]
 
 s_or_b64 s[104:105], s[102:103], s[4:5]
-// GFX12: encoding: [0x66,0x04,0xe8,0x8c]
+// GFX12: s_or_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x8c]
 
 s_or_b64 s[104:105], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xe8,0x8c]
+// GFX12: s_or_b64 s[104:105], s[2:3], s[4:5]     ; encoding: [0x02,0x04,0xe8,0x8c]
 
 s_or_b64 s[0:1], s[102:103], s[4:5]
-// GFX12: encoding: [0x66,0x04,0x80,0x8c]
+// GFX12: s_or_b64 s[0:1], s[102:103], s[4:5]     ; encoding: [0x66,0x04,0x80,0x8c]
 
 s_or_b64 s[0:1], s[2:3], s[100:101]
-// GFX12: encoding: [0x02,0x64,0x80,0x8c]
+// GFX12: s_or_b64 s[0:1], s[2:3], s[100:101]     ; encoding: [0x02,0x64,0x80,0x8c]
 
 s_or_b64 exec, s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xfe,0x8c]
+// GFX12: s_or_b64 exec, s[2:3], s[4:5]           ; encoding: [0x02,0x04,0xfe,0x8c]
 
 s_or_b64 vcc, s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xea,0x8c]
+// GFX12: s_or_b64 vcc, s[2:3], s[4:5]            ; encoding: [0x02,0x04,0xea,0x8c]
 
 s_or_b64 s[0:1], exec, s[4:5]
-// GFX12: encoding: [0x7e,0x04,0x80,0x8c]
+// GFX12: s_or_b64 s[0:1], exec, s[4:5]           ; encoding: [0x7e,0x04,0x80,0x8c]
 
 s_or_b64 s[0:1], vcc, s[4:5]
-// GFX12: encoding: [0x6a,0x04,0x80,0x8c]
+// GFX12: s_or_b64 s[0:1], vcc, s[4:5]            ; encoding: [0x6a,0x04,0x80,0x8c]
 
 s_or_b64 s[0:1], 0, s[4:5]
-// GFX12: encoding: [0x80,0x04,0x80,0x8c]
+// GFX12: s_or_b64 s[0:1], 0, s[4:5]              ; encoding: [0x80,0x04,0x80,0x8c]
 
 s_or_b64 s[0:1], -1, s[4:5]
-// GFX12: encoding: [0xc1,0x04,0x80,0x8c]
+// GFX12: s_or_b64 s[0:1], -1, s[4:5]             ; encoding: [0xc1,0x04,0x80,0x8c]
 
 s_or_b64 s[0:1], 0.5, s[4:5]
-// GFX12: encoding: [0xf0,0x04,0x80,0x8c]
+// GFX12: s_or_b64 s[0:1], 0.5, s[4:5]            ; encoding: [0xf0,0x04,0x80,0x8c]
 
 s_or_b64 s[0:1], -4.0, s[4:5]
-// GFX12: encoding: [0xf7,0x04,0x80,0x8c]
+// GFX12: s_or_b64 s[0:1], -4.0, s[4:5]           ; encoding: [0xf7,0x04,0x80,0x8c]
 
 s_or_b64 s[0:1], 0x3f717273, s[4:5]
-// GFX12: encoding: [0xff,0x04,0x80,0x8c,0x73,0x72,0x71,0x3f]
+// GFX12: s_or_b64 s[0:1], 0x3f717273, s[4:5]     ; encoding: [0xff,0x04,0x80,0x8c,0x73,0x72,0x71,0x3f]
 
 s_or_b64 s[0:1], 0xaf123456, s[4:5]
-// GFX12: encoding: [0xff,0x04,0x80,0x8c,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_b64 s[0:1], 0xaf123456, s[4:5]     ; encoding: [0xff,0x04,0x80,0x8c,0x56,0x34,0x12,0xaf]
 
 s_or_b64 s[0:1], s[2:3], exec
-// GFX12: encoding: [0x02,0x7e,0x80,0x8c]
+// GFX12: s_or_b64 s[0:1], s[2:3], exec           ; encoding: [0x02,0x7e,0x80,0x8c]
 
 s_or_b64 s[0:1], s[2:3], vcc
-// GFX12: encoding: [0x02,0x6a,0x80,0x8c]
+// GFX12: s_or_b64 s[0:1], s[2:3], vcc            ; encoding: [0x02,0x6a,0x80,0x8c]
 
 s_or_b64 s[0:1], s[2:3], 0
-// GFX12: encoding: [0x02,0x80,0x80,0x8c]
+// GFX12: s_or_b64 s[0:1], s[2:3], 0              ; encoding: [0x02,0x80,0x80,0x8c]
 
 s_or_b64 s[0:1], s[2:3], -1
-// GFX12: encoding: [0x02,0xc1,0x80,0x8c]
+// GFX12: s_or_b64 s[0:1], s[2:3], -1             ; encoding: [0x02,0xc1,0x80,0x8c]
 
 s_or_b64 s[0:1], s[2:3], 0.5
-// GFX12: encoding: [0x02,0xf0,0x80,0x8c]
+// GFX12: s_or_b64 s[0:1], s[2:3], 0.5            ; encoding: [0x02,0xf0,0x80,0x8c]
 
 s_or_b64 s[0:1], s[2:3], -4.0
-// GFX12: encoding: [0x02,0xf7,0x80,0x8c]
+// GFX12: s_or_b64 s[0:1], s[2:3], -4.0           ; encoding: [0x02,0xf7,0x80,0x8c]
 
 s_or_b64 s[0:1], s[2:3], 0x3f717273
-// GFX12: encoding: [0x02,0xff,0x80,0x8c,0x73,0x72,0x71,0x3f]
+// GFX12: s_or_b64 s[0:1], s[2:3], 0x3f717273     ; encoding: [0x02,0xff,0x80,0x8c,0x73,0x72,0x71,0x3f]
 
 s_or_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: encoding: [0x02,0xff,0x80,0x8c,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_b64 s[0:1], s[2:3], 0xaf123456     ; encoding: [0x02,0xff,0x80,0x8c,0x56,0x34,0x12,0xaf]
 
 s_xor_b32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x8d]
+// GFX12: s_xor_b32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x8d]
 
 s_xor_b32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x8d]
+// GFX12: s_xor_b32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x8d]
 
 s_xor_b32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x8d]
+// GFX12: s_xor_b32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x8d]
 
 s_xor_b32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x8d]
+// GFX12: s_xor_b32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x8d]
 
 s_xor_b32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x8d]
+// GFX12: s_xor_b32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x8d]
 
 s_xor_b32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x8d]
+// GFX12: s_xor_b32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x8d]
 
 s_xor_b32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x8d]
+// GFX12: s_xor_b32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x8d]
 
 s_xor_b32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x8d]
+// GFX12: s_xor_b32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x8d]
 
 s_xor_b32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x8d]
+// GFX12: s_xor_b32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x8d]
 
 s_xor_b32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x8d]
+// GFX12: s_xor_b32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x8d]
 
 s_xor_b32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x8d]
+// GFX12: s_xor_b32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x8d]
 
 s_xor_b32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x8d]
+// GFX12: s_xor_b32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x8d]
 
 s_xor_b32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x8d]
+// GFX12: s_xor_b32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x8d]
 
 s_xor_b32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x8d]
+// GFX12: s_xor_b32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x8d]
 
 s_xor_b32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x8d]
+// GFX12: s_xor_b32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x8d]
 
 s_xor_b32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x8d]
+// GFX12: s_xor_b32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x8d]
 
 s_xor_b32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x8d]
+// GFX12: s_xor_b32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x8d]
 
 s_xor_b32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x8d]
+// GFX12: s_xor_b32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x8d]
 
 s_xor_b32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x8d]
+// GFX12: s_xor_b32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x8d]
 
 s_xor_b32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x8d]
+// GFX12: s_xor_b32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x8d]
 
 s_xor_b32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x8d]
+// GFX12: s_xor_b32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x8d]
 
 s_xor_b32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x8d]
+// GFX12: s_xor_b32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x8d]
 
 s_xor_b32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x8d,0x73,0x72,0x71,0x3f]
+// GFX12: s_xor_b32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x8d,0x73,0x72,0x71,0x3f]
 
 s_xor_b32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x8d,0x56,0x34,0x12,0xaf]
+// GFX12: s_xor_b32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x8d,0x56,0x34,0x12,0xaf]
 
 s_xor_b32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x8d]
+// GFX12: s_xor_b32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x8d]
 
 s_xor_b32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x8d]
+// GFX12: s_xor_b32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x8d]
 
 s_xor_b32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x8d]
+// GFX12: s_xor_b32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x8d]
 
 s_xor_b32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x8d]
+// GFX12: s_xor_b32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x8d]
 
 s_xor_b32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x8d]
+// GFX12: s_xor_b32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x8d]
 
 s_xor_b32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x8d]
+// GFX12: s_xor_b32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x8d]
 
 s_xor_b32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x8d]
+// GFX12: s_xor_b32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x8d]
 
 s_xor_b32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x8d]
+// GFX12: s_xor_b32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x8d]
 
 s_xor_b32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x8d]
+// GFX12: s_xor_b32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x8d]
 
 s_xor_b32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x8d,0x73,0x72,0x71,0x3f]
+// GFX12: s_xor_b32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x8d,0x73,0x72,0x71,0x3f]
 
 s_xor_b32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x8d,0x56,0x34,0x12,0xaf]
+// GFX12: s_xor_b32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x8d,0x56,0x34,0x12,0xaf]
 
 s_xor_b64 s[0:1], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0x80,0x8d]
+// GFX12: s_xor_b64 s[0:1], s[2:3], s[4:5]        ; encoding: [0x02,0x04,0x80,0x8d]
 
 s_xor_b64 s[104:105], s[102:103], s[100:101]
-// GFX12: encoding: [0x66,0x64,0xe8,0x8d]
+// GFX12: s_xor_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x8d]
 
 s_xor_b64 s[0:1], s[102:103], s[100:101]
-// GFX12: encoding: [0x66,0x64,0x80,0x8d]
+// GFX12: s_xor_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x8d]
 
 s_xor_b64 s[104:105], s[2:3], s[100:101]
-// GFX12: encoding: [0x02,0x64,0xe8,0x8d]
+// GFX12: s_xor_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x8d]
 
 s_xor_b64 s[104:105], s[102:103], s[4:5]
-// GFX12: encoding: [0x66,0x04,0xe8,0x8d]
+// GFX12: s_xor_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x8d]
 
 s_xor_b64 s[104:105], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xe8,0x8d]
+// GFX12: s_xor_b64 s[104:105], s[2:3], s[4:5]    ; encoding: [0x02,0x04,0xe8,0x8d]
 
 s_xor_b64 s[0:1], s[102:103], s[4:5]
-// GFX12: encoding: [0x66,0x04,0x80,0x8d]
+// GFX12: s_xor_b64 s[0:1], s[102:103], s[4:5]    ; encoding: [0x66,0x04,0x80,0x8d]
 
 s_xor_b64 s[0:1], s[2:3], s[100:101]
-// GFX12: encoding: [0x02,0x64,0x80,0x8d]
+// GFX12: s_xor_b64 s[0:1], s[2:3], s[100:101]    ; encoding: [0x02,0x64,0x80,0x8d]
 
 s_xor_b64 exec, s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xfe,0x8d]
+// GFX12: s_xor_b64 exec, s[2:3], s[4:5]          ; encoding: [0x02,0x04,0xfe,0x8d]
 
 s_xor_b64 vcc, s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xea,0x8d]
+// GFX12: s_xor_b64 vcc, s[2:3], s[4:5]           ; encoding: [0x02,0x04,0xea,0x8d]
 
 s_xor_b64 s[0:1], exec, s[4:5]
-// GFX12: encoding: [0x7e,0x04,0x80,0x8d]
+// GFX12: s_xor_b64 s[0:1], exec, s[4:5]          ; encoding: [0x7e,0x04,0x80,0x8d]
 
 s_xor_b64 s[0:1], vcc, s[4:5]
-// GFX12: encoding: [0x6a,0x04,0x80,0x8d]
+// GFX12: s_xor_b64 s[0:1], vcc, s[4:5]           ; encoding: [0x6a,0x04,0x80,0x8d]
 
 s_xor_b64 s[0:1], 0, s[4:5]
-// GFX12: encoding: [0x80,0x04,0x80,0x8d]
+// GFX12: s_xor_b64 s[0:1], 0, s[4:5]             ; encoding: [0x80,0x04,0x80,0x8d]
 
 s_xor_b64 s[0:1], -1, s[4:5]
-// GFX12: encoding: [0xc1,0x04,0x80,0x8d]
+// GFX12: s_xor_b64 s[0:1], -1, s[4:5]            ; encoding: [0xc1,0x04,0x80,0x8d]
 
 s_xor_b64 s[0:1], 0.5, s[4:5]
-// GFX12: encoding: [0xf0,0x04,0x80,0x8d]
+// GFX12: s_xor_b64 s[0:1], 0.5, s[4:5]           ; encoding: [0xf0,0x04,0x80,0x8d]
 
 s_xor_b64 s[0:1], -4.0, s[4:5]
-// GFX12: encoding: [0xf7,0x04,0x80,0x8d]
+// GFX12: s_xor_b64 s[0:1], -4.0, s[4:5]          ; encoding: [0xf7,0x04,0x80,0x8d]
 
 s_xor_b64 s[0:1], 0x3f717273, s[4:5]
-// GFX12: encoding: [0xff,0x04,0x80,0x8d,0x73,0x72,0x71,0x3f]
+// GFX12: s_xor_b64 s[0:1], 0x3f717273, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8d,0x73,0x72,0x71,0x3f]
 
 s_xor_b64 s[0:1], 0xaf123456, s[4:5]
-// GFX12: encoding: [0xff,0x04,0x80,0x8d,0x56,0x34,0x12,0xaf]
+// GFX12: s_xor_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8d,0x56,0x34,0x12,0xaf]
 
 s_xor_b64 s[0:1], s[2:3], exec
-// GFX12: encoding: [0x02,0x7e,0x80,0x8d]
+// GFX12: s_xor_b64 s[0:1], s[2:3], exec          ; encoding: [0x02,0x7e,0x80,0x8d]
 
 s_xor_b64 s[0:1], s[2:3], vcc
-// GFX12: encoding: [0x02,0x6a,0x80,0x8d]
+// GFX12: s_xor_b64 s[0:1], s[2:3], vcc           ; encoding: [0x02,0x6a,0x80,0x8d]
 
 s_xor_b64 s[0:1], s[2:3], 0
-// GFX12: encoding: [0x02,0x80,0x80,0x8d]
+// GFX12: s_xor_b64 s[0:1], s[2:3], 0             ; encoding: [0x02,0x80,0x80,0x8d]
 
 s_xor_b64 s[0:1], s[2:3], -1
-// GFX12: encoding: [0x02,0xc1,0x80,0x8d]
+// GFX12: s_xor_b64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x80,0x8d]
 
 s_xor_b64 s[0:1], s[2:3], 0.5
-// GFX12: encoding: [0x02,0xf0,0x80,0x8d]
+// GFX12: s_xor_b64 s[0:1], s[2:3], 0.5           ; encoding: [0x02,0xf0,0x80,0x8d]
 
 s_xor_b64 s[0:1], s[2:3], -4.0
-// GFX12: encoding: [0x02,0xf7,0x80,0x8d]
+// GFX12: s_xor_b64 s[0:1], s[2:3], -4.0          ; encoding: [0x02,0xf7,0x80,0x8d]
 
 s_xor_b64 s[0:1], s[2:3], 0x3f717273
-// GFX12: encoding: [0x02,0xff,0x80,0x8d,0x73,0x72,0x71,0x3f]
+// GFX12: s_xor_b64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0x8d,0x73,0x72,0x71,0x3f]
 
 s_xor_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: encoding: [0x02,0xff,0x80,0x8d,0x56,0x34,0x12,0xaf]
+// GFX12: s_xor_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8d,0x56,0x34,0x12,0xaf]
 
 s_andn2_b32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, s1, s2               ; encoding: [0x01,0x02,0x00,0x91]
 
 s_andn2_b32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x91]
+// GFX12: s_and_not1_b32 s105, s104, s103         ; encoding: [0x68,0x67,0x69,0x91]
 
 s_andn2_b32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, s104, s103           ; encoding: [0x68,0x67,0x00,0x91]
 
 s_andn2_b32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x91]
+// GFX12: s_and_not1_b32 s105, s1, s103           ; encoding: [0x01,0x67,0x69,0x91]
 
 s_andn2_b32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x91]
+// GFX12: s_and_not1_b32 s105, s104, s2           ; encoding: [0x68,0x02,0x69,0x91]
 
 s_andn2_b32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x91]
+// GFX12: s_and_not1_b32 s105, s1, s2             ; encoding: [0x01,0x02,0x69,0x91]
 
 s_andn2_b32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, s104, s2             ; encoding: [0x68,0x02,0x00,0x91]
 
 s_andn2_b32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, s1, s103             ; encoding: [0x01,0x67,0x00,0x91]
 
 s_andn2_b32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x91]
+// GFX12: s_and_not1_b32 exec_lo, s1, s2          ; encoding: [0x01,0x02,0x7e,0x91]
 
 s_andn2_b32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x91]
+// GFX12: s_and_not1_b32 exec_hi, s1, s2          ; encoding: [0x01,0x02,0x7f,0x91]
 
 s_andn2_b32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x91]
+// GFX12: s_and_not1_b32 vcc_lo, s1, s2           ; encoding: [0x01,0x02,0x6a,0x91]
 
 s_andn2_b32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x91]
+// GFX12: s_and_not1_b32 vcc_hi, s1, s2           ; encoding: [0x01,0x02,0x6b,0x91]
 
 s_andn2_b32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x91]
+// GFX12: s_and_not1_b32 m0, s1, s2               ; encoding: [0x01,0x02,0x7d,0x91]
 
 s_andn2_b32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, exec_lo, s2          ; encoding: [0x7e,0x02,0x00,0x91]
 
 s_andn2_b32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, exec_hi, s2          ; encoding: [0x7f,0x02,0x00,0x91]
 
 s_andn2_b32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, vcc_lo, s2           ; encoding: [0x6a,0x02,0x00,0x91]
 
 s_andn2_b32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, vcc_hi, s2           ; encoding: [0x6b,0x02,0x00,0x91]
 
 s_andn2_b32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, m0, s2               ; encoding: [0x7d,0x02,0x00,0x91]
 
 s_andn2_b32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, 0, s2                ; encoding: [0x80,0x02,0x00,0x91]
 
 s_andn2_b32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, -1, s2               ; encoding: [0xc1,0x02,0x00,0x91]
 
 s_andn2_b32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, 0.5, s2              ; encoding: [0xf0,0x02,0x00,0x91]
 
 s_andn2_b32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, -4.0, s2             ; encoding: [0xf7,0x02,0x00,0x91]
 
 s_andn2_b32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x91,0x73,0x72,0x71,0x3f]
+// GFX12: s_and_not1_b32 s0, 0x3f717273, s2       ; encoding: [0xff,0x02,0x00,0x91,0x73,0x72,0x71,0x3f]
 
 s_andn2_b32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x91,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not1_b32 s0, 0xaf123456, s2       ; encoding: [0xff,0x02,0x00,0x91,0x56,0x34,0x12,0xaf]
 
 s_andn2_b32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, s1, exec_lo          ; encoding: [0x01,0x7e,0x00,0x91]
 
 s_andn2_b32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, s1, exec_hi          ; encoding: [0x01,0x7f,0x00,0x91]
 
 s_andn2_b32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, s1, vcc_lo           ; encoding: [0x01,0x6a,0x00,0x91]
 
 s_andn2_b32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, s1, vcc_hi           ; encoding: [0x01,0x6b,0x00,0x91]
 
 s_andn2_b32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, s1, m0               ; encoding: [0x01,0x7d,0x00,0x91]
 
 s_andn2_b32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, s1, 0                ; encoding: [0x01,0x80,0x00,0x91]
 
 s_andn2_b32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, s1, -1               ; encoding: [0x01,0xc1,0x00,0x91]
 
 s_andn2_b32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, s1, 0.5              ; encoding: [0x01,0xf0,0x00,0x91]
 
 s_andn2_b32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x91]
+// GFX12: s_and_not1_b32 s0, s1, -4.0             ; encoding: [0x01,0xf7,0x00,0x91]
 
 s_andn2_b32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x91,0x73,0x72,0x71,0x3f]
+// GFX12: s_and_not1_b32 s0, s1, 0x3f717273       ; encoding: [0x01,0xff,0x00,0x91,0x73,0x72,0x71,0x3f]
 
 s_andn2_b32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x91,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not1_b32 s0, s1, 0xaf123456       ; encoding: [0x01,0xff,0x00,0x91,0x56,0x34,0x12,0xaf]
 
 s_andn2_b64 s[0:1], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0x80,0x91]
+// GFX12: s_and_not1_b64 s[0:1], s[2:3], s[4:5]   ; encoding: [0x02,0x04,0x80,0x91]
 
 s_andn2_b64 s[104:105], s[102:103], s[100:101]
-// GFX12: encoding: [0x66,0x64,0xe8,0x91]
+// GFX12: s_and_not1_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x91]
 
 s_andn2_b64 s[0:1], s[102:103], s[100:101]
-// GFX12: encoding: [0x66,0x64,0x80,0x91]
+// GFX12: s_and_not1_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x91]
 
 s_andn2_b64 s[104:105], s[2:3], s[100:101]
-// GFX12: encoding: [0x02,0x64,0xe8,0x91]
+// GFX12: s_and_not1_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x91]
 
 s_andn2_b64 s[104:105], s[102:103], s[4:5]
-// GFX12: encoding: [0x66,0x04,0xe8,0x91]
+// GFX12: s_and_not1_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x91]
 
 s_andn2_b64 s[104:105], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xe8,0x91]
+// GFX12: s_and_not1_b64 s[104:105], s[2:3], s[4:5] ; encoding: [0x02,0x04,0xe8,0x91]
 
 s_andn2_b64 s[0:1], s[102:103], s[4:5]
-// GFX12: encoding: [0x66,0x04,0x80,0x91]
+// GFX12: s_and_not1_b64 s[0:1], s[102:103], s[4:5] ; encoding: [0x66,0x04,0x80,0x91]
 
 s_andn2_b64 s[0:1], s[2:3], s[100:101]
-// GFX12: encoding: [0x02,0x64,0x80,0x91]
+// GFX12: s_and_not1_b64 s[0:1], s[2:3], s[100:101] ; encoding: [0x02,0x64,0x80,0x91]
 
 s_andn2_b64 exec, s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xfe,0x91]
+// GFX12: s_and_not1_b64 exec, s[2:3], s[4:5]     ; encoding: [0x02,0x04,0xfe,0x91]
 
 s_andn2_b64 vcc, s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xea,0x91]
+// GFX12: s_and_not1_b64 vcc, s[2:3], s[4:5]      ; encoding: [0x02,0x04,0xea,0x91]
 
 s_andn2_b64 s[0:1], exec, s[4:5]
-// GFX12: encoding: [0x7e,0x04,0x80,0x91]
+// GFX12: s_and_not1_b64 s[0:1], exec, s[4:5]     ; encoding: [0x7e,0x04,0x80,0x91]
 
 s_andn2_b64 s[0:1], vcc, s[4:5]
-// GFX12: encoding: [0x6a,0x04,0x80,0x91]
+// GFX12: s_and_not1_b64 s[0:1], vcc, s[4:5]      ; encoding: [0x6a,0x04,0x80,0x91]
 
 s_andn2_b64 s[0:1], 0, s[4:5]
-// GFX12: encoding: [0x80,0x04,0x80,0x91]
+// GFX12: s_and_not1_b64 s[0:1], 0, s[4:5]        ; encoding: [0x80,0x04,0x80,0x91]
 
 s_andn2_b64 s[0:1], -1, s[4:5]
-// GFX12: encoding: [0xc1,0x04,0x80,0x91]
+// GFX12: s_and_not1_b64 s[0:1], -1, s[4:5]       ; encoding: [0xc1,0x04,0x80,0x91]
 
 s_andn2_b64 s[0:1], 0.5, s[4:5]
-// GFX12: encoding: [0xf0,0x04,0x80,0x91]
+// GFX12: s_and_not1_b64 s[0:1], 0.5, s[4:5]      ; encoding: [0xf0,0x04,0x80,0x91]
 
 s_andn2_b64 s[0:1], -4.0, s[4:5]
-// GFX12: encoding: [0xf7,0x04,0x80,0x91]
+// GFX12: s_and_not1_b64 s[0:1], -4.0, s[4:5]     ; encoding: [0xf7,0x04,0x80,0x91]
 
 s_andn2_b64 s[0:1], 0x3f717273, s[4:5]
-// GFX12: encoding: [0xff,0x04,0x80,0x91,0x73,0x72,0x71,0x3f]
+// GFX12: s_and_not1_b64 s[0:1], 0x3f717273, s[4:5] ; encoding: [0xff,0x04,0x80,0x91,0x73,0x72,0x71,0x3f]
 
 s_andn2_b64 s[0:1], 0xaf123456, s[4:5]
-// GFX12: encoding: [0xff,0x04,0x80,0x91,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not1_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x91,0x56,0x34,0x12,0xaf]
 
 s_andn2_b64 s[0:1], s[2:3], exec
-// GFX12: encoding: [0x02,0x7e,0x80,0x91]
+// GFX12: s_and_not1_b64 s[0:1], s[2:3], exec     ; encoding: [0x02,0x7e,0x80,0x91]
 
 s_andn2_b64 s[0:1], s[2:3], vcc
-// GFX12: encoding: [0x02,0x6a,0x80,0x91]
+// GFX12: s_and_not1_b64 s[0:1], s[2:3], vcc      ; encoding: [0x02,0x6a,0x80,0x91]
 
 s_andn2_b64 s[0:1], s[2:3], 0
-// GFX12: encoding: [0x02,0x80,0x80,0x91]
+// GFX12: s_and_not1_b64 s[0:1], s[2:3], 0        ; encoding: [0x02,0x80,0x80,0x91]
 
 s_andn2_b64 s[0:1], s[2:3], -1
-// GFX12: encoding: [0x02,0xc1,0x80,0x91]
+// GFX12: s_and_not1_b64 s[0:1], s[2:3], -1       ; encoding: [0x02,0xc1,0x80,0x91]
 
 s_andn2_b64 s[0:1], s[2:3], 0.5
-// GFX12: encoding: [0x02,0xf0,0x80,0x91]
+// GFX12: s_and_not1_b64 s[0:1], s[2:3], 0.5      ; encoding: [0x02,0xf0,0x80,0x91]
 
 s_andn2_b64 s[0:1], s[2:3], -4.0
-// GFX12: encoding: [0x02,0xf7,0x80,0x91]
+// GFX12: s_and_not1_b64 s[0:1], s[2:3], -4.0     ; encoding: [0x02,0xf7,0x80,0x91]
 
 s_andn2_b64 s[0:1], s[2:3], 0x3f717273
-// GFX12: encoding: [0x02,0xff,0x80,0x91,0x73,0x72,0x71,0x3f]
+// GFX12: s_and_not1_b64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0x91,0x73,0x72,0x71,0x3f]
 
 s_andn2_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: encoding: [0x02,0xff,0x80,0x91,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not1_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x91,0x56,0x34,0x12,0xaf]
 
 s_orn2_b32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, s1, s2                ; encoding: [0x01,0x02,0x00,0x92]
 
 s_orn2_b32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x92]
+// GFX12: s_or_not1_b32 s105, s104, s103          ; encoding: [0x68,0x67,0x69,0x92]
 
 s_orn2_b32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, s104, s103            ; encoding: [0x68,0x67,0x00,0x92]
 
 s_orn2_b32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x92]
+// GFX12: s_or_not1_b32 s105, s1, s103            ; encoding: [0x01,0x67,0x69,0x92]
 
 s_orn2_b32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x92]
+// GFX12: s_or_not1_b32 s105, s104, s2            ; encoding: [0x68,0x02,0x69,0x92]
 
 s_orn2_b32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x92]
+// GFX12: s_or_not1_b32 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0x92]
 
 s_orn2_b32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, s104, s2              ; encoding: [0x68,0x02,0x00,0x92]
 
 s_orn2_b32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, s1, s103              ; encoding: [0x01,0x67,0x00,0x92]
 
 s_orn2_b32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x92]
+// GFX12: s_or_not1_b32 exec_lo, s1, s2           ; encoding: [0x01,0x02,0x7e,0x92]
 
 s_orn2_b32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x92]
+// GFX12: s_or_not1_b32 exec_hi, s1, s2           ; encoding: [0x01,0x02,0x7f,0x92]
 
 s_orn2_b32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x92]
+// GFX12: s_or_not1_b32 vcc_lo, s1, s2            ; encoding: [0x01,0x02,0x6a,0x92]
 
 s_orn2_b32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x92]
+// GFX12: s_or_not1_b32 vcc_hi, s1, s2            ; encoding: [0x01,0x02,0x6b,0x92]
 
 s_orn2_b32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x92]
+// GFX12: s_or_not1_b32 m0, s1, s2                ; encoding: [0x01,0x02,0x7d,0x92]
 
 s_orn2_b32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, exec_lo, s2           ; encoding: [0x7e,0x02,0x00,0x92]
 
 s_orn2_b32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, exec_hi, s2           ; encoding: [0x7f,0x02,0x00,0x92]
 
 s_orn2_b32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, vcc_lo, s2            ; encoding: [0x6a,0x02,0x00,0x92]
 
 s_orn2_b32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, vcc_hi, s2            ; encoding: [0x6b,0x02,0x00,0x92]
 
 s_orn2_b32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, m0, s2                ; encoding: [0x7d,0x02,0x00,0x92]
 
 s_orn2_b32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, 0, s2                 ; encoding: [0x80,0x02,0x00,0x92]
 
 s_orn2_b32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, -1, s2                ; encoding: [0xc1,0x02,0x00,0x92]
 
 s_orn2_b32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, 0.5, s2               ; encoding: [0xf0,0x02,0x00,0x92]
 
 s_orn2_b32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, -4.0, s2              ; encoding: [0xf7,0x02,0x00,0x92]
 
 s_orn2_b32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x92,0x73,0x72,0x71,0x3f]
+// GFX12: s_or_not1_b32 s0, 0x3f717273, s2        ; encoding: [0xff,0x02,0x00,0x92,0x73,0x72,0x71,0x3f]
 
 s_orn2_b32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x92,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_not1_b32 s0, 0xaf123456, s2        ; encoding: [0xff,0x02,0x00,0x92,0x56,0x34,0x12,0xaf]
 
 s_orn2_b32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, s1, exec_lo           ; encoding: [0x01,0x7e,0x00,0x92]
 
 s_orn2_b32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, s1, exec_hi           ; encoding: [0x01,0x7f,0x00,0x92]
 
 s_orn2_b32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, s1, vcc_lo            ; encoding: [0x01,0x6a,0x00,0x92]
 
 s_orn2_b32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, s1, vcc_hi            ; encoding: [0x01,0x6b,0x00,0x92]
 
 s_orn2_b32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, s1, m0                ; encoding: [0x01,0x7d,0x00,0x92]
 
 s_orn2_b32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, s1, 0                 ; encoding: [0x01,0x80,0x00,0x92]
 
 s_orn2_b32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, s1, -1                ; encoding: [0x01,0xc1,0x00,0x92]
 
 s_orn2_b32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, s1, 0.5               ; encoding: [0x01,0xf0,0x00,0x92]
 
 s_orn2_b32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x92]
+// GFX12: s_or_not1_b32 s0, s1, -4.0              ; encoding: [0x01,0xf7,0x00,0x92]
 
 s_orn2_b32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x92,0x73,0x72,0x71,0x3f]
+// GFX12: s_or_not1_b32 s0, s1, 0x3f717273        ; encoding: [0x01,0xff,0x00,0x92,0x73,0x72,0x71,0x3f]
 
 s_orn2_b32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x92,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_not1_b32 s0, s1, 0xaf123456        ; encoding: [0x01,0xff,0x00,0x92,0x56,0x34,0x12,0xaf]
 
 s_orn2_b64 s[0:1], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0x80,0x92]
+// GFX12: s_or_not1_b64 s[0:1], s[2:3], s[4:5]    ; encoding: [0x02,0x04,0x80,0x92]
 
 s_orn2_b64 s[104:105], s[102:103], s[100:101]
-// GFX12: encoding: [0x66,0x64,0xe8,0x92]
+// GFX12: s_or_not1_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x92]
 
 s_orn2_b64 s[0:1], s[102:103], s[100:101]
-// GFX12: encoding: [0x66,0x64,0x80,0x92]
+// GFX12: s_or_not1_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x92]
 
 s_orn2_b64 s[104:105], s[2:3], s[100:101]
-// GFX12: encoding: [0x02,0x64,0xe8,0x92]
+// GFX12: s_or_not1_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x92]
 
 s_orn2_b64 s[104:105], s[102:103], s[4:5]
-// GFX12: encoding: [0x66,0x04,0xe8,0x92]
+// GFX12: s_or_not1_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x92]
 
 s_orn2_b64 s[104:105], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xe8,0x92]
+// GFX12: s_or_not1_b64 s[104:105], s[2:3], s[4:5] ; encoding: [0x02,0x04,0xe8,0x92]
 
 s_orn2_b64 s[0:1], s[102:103], s[4:5]
-// GFX12: encoding: [0x66,0x04,0x80,0x92]
+// GFX12: s_or_not1_b64 s[0:1], s[102:103], s[4:5] ; encoding: [0x66,0x04,0x80,0x92]
 
 s_orn2_b64 s[0:1], s[2:3], s[100:101]
-// GFX12: encoding: [0x02,0x64,0x80,0x92]
+// GFX12: s_or_not1_b64 s[0:1], s[2:3], s[100:101] ; encoding: [0x02,0x64,0x80,0x92]
 
 s_orn2_b64 exec, s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xfe,0x92]
+// GFX12: s_or_not1_b64 exec, s[2:3], s[4:5]      ; encoding: [0x02,0x04,0xfe,0x92]
 
 s_orn2_b64 vcc, s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xea,0x92]
+// GFX12: s_or_not1_b64 vcc, s[2:3], s[4:5]       ; encoding: [0x02,0x04,0xea,0x92]
 
 s_orn2_b64 s[0:1], exec, s[4:5]
-// GFX12: encoding: [0x7e,0x04,0x80,0x92]
+// GFX12: s_or_not1_b64 s[0:1], exec, s[4:5]      ; encoding: [0x7e,0x04,0x80,0x92]
 
 s_orn2_b64 s[0:1], vcc, s[4:5]
-// GFX12: encoding: [0x6a,0x04,0x80,0x92]
+// GFX12: s_or_not1_b64 s[0:1], vcc, s[4:5]       ; encoding: [0x6a,0x04,0x80,0x92]
 
 s_orn2_b64 s[0:1], 0, s[4:5]
-// GFX12: encoding: [0x80,0x04,0x80,0x92]
+// GFX12: s_or_not1_b64 s[0:1], 0, s[4:5]         ; encoding: [0x80,0x04,0x80,0x92]
 
 s_orn2_b64 s[0:1], -1, s[4:5]
-// GFX12: encoding: [0xc1,0x04,0x80,0x92]
+// GFX12: s_or_not1_b64 s[0:1], -1, s[4:5]        ; encoding: [0xc1,0x04,0x80,0x92]
 
 s_orn2_b64 s[0:1], 0.5, s[4:5]
-// GFX12: encoding: [0xf0,0x04,0x80,0x92]
+// GFX12: s_or_not1_b64 s[0:1], 0.5, s[4:5]       ; encoding: [0xf0,0x04,0x80,0x92]
 
 s_orn2_b64 s[0:1], -4.0, s[4:5]
-// GFX12: encoding: [0xf7,0x04,0x80,0x92]
+// GFX12: s_or_not1_b64 s[0:1], -4.0, s[4:5]      ; encoding: [0xf7,0x04,0x80,0x92]
 
 s_orn2_b64 s[0:1], 0x3f717273, s[4:5]
-// GFX12: encoding: [0xff,0x04,0x80,0x92,0x73,0x72,0x71,0x3f]
+// GFX12: s_or_not1_b64 s[0:1], 0x3f717273, s[4:5] ; encoding: [0xff,0x04,0x80,0x92,0x73,0x72,0x71,0x3f]
 
 s_orn2_b64 s[0:1], 0xaf123456, s[4:5]
-// GFX12: encoding: [0xff,0x04,0x80,0x92,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_not1_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x92,0x56,0x34,0x12,0xaf]
 
 s_orn2_b64 s[0:1], s[2:3], exec
-// GFX12: encoding: [0x02,0x7e,0x80,0x92]
+// GFX12: s_or_not1_b64 s[0:1], s[2:3], exec      ; encoding: [0x02,0x7e,0x80,0x92]
 
 s_orn2_b64 s[0:1], s[2:3], vcc
-// GFX12: encoding: [0x02,0x6a,0x80,0x92]
+// GFX12: s_or_not1_b64 s[0:1], s[2:3], vcc       ; encoding: [0x02,0x6a,0x80,0x92]
 
 s_orn2_b64 s[0:1], s[2:3], 0
-// GFX12: encoding: [0x02,0x80,0x80,0x92]
+// GFX12: s_or_not1_b64 s[0:1], s[2:3], 0         ; encoding: [0x02,0x80,0x80,0x92]
 
 s_orn2_b64 s[0:1], s[2:3], -1
-// GFX12: encoding: [0x02,0xc1,0x80,0x92]
+// GFX12: s_or_not1_b64 s[0:1], s[2:3], -1        ; encoding: [0x02,0xc1,0x80,0x92]
 
 s_orn2_b64 s[0:1], s[2:3], 0.5
-// GFX12: encoding: [0x02,0xf0,0x80,0x92]
+// GFX12: s_or_not1_b64 s[0:1], s[2:3], 0.5       ; encoding: [0x02,0xf0,0x80,0x92]
 
 s_orn2_b64 s[0:1], s[2:3], -4.0
-// GFX12: encoding: [0x02,0xf7,0x80,0x92]
+// GFX12: s_or_not1_b64 s[0:1], s[2:3], -4.0      ; encoding: [0x02,0xf7,0x80,0x92]
 
 s_orn2_b64 s[0:1], s[2:3], 0x3f717273
-// GFX12: encoding: [0x02,0xff,0x80,0x92,0x73,0x72,0x71,0x3f]
+// GFX12: s_or_not1_b64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0x92,0x73,0x72,0x71,0x3f]
 
 s_orn2_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: encoding: [0x02,0xff,0x80,0x92,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_not1_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x92,0x56,0x34,0x12,0xaf]
 
 s_nand_b32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x8e]
+// GFX12: s_nand_b32 s0, s1, s2                   ; encoding: [0x01,0x02,0x00,0x8e]
 
 s_nand_b32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x8e]
+// GFX12: s_nand_b32 s105, s104, s103             ; encoding: [0x68,0x67,0x69,0x8e]
 
 s_nand_b32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x8e]
+// GFX12: s_nand_b32 s0, s104, s103               ; encoding: [0x68,0x67,0x00,0x8e]
 
 s_nand_b32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x8e]
+// GFX12: s_nand_b32 s105, s1, s103               ; encoding: [0x01,0x67,0x69,0x8e]
 
 s_nand_b32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x8e]
+// GFX12: s_nand_b32 s105, s104, s2               ; encoding: [0x68,0x02,0x69,0x8e]
 
 s_nand_b32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x8e]
+// GFX12: s_nand_b32 s105, s1, s2                 ; encoding: [0x01,0x02,0x69,0x8e]
 
 s_nand_b32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x8e]
+// GFX12: s_nand_b32 s0, s104, s2                 ; encoding: [0x68,0x02,0x00,0x8e]
 
 s_nand_b32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x8e]
+// GFX12: s_nand_b32 s0, s1, s103                 ; encoding: [0x01,0x67,0x00,0x8e]
 
 s_nand_b32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x8e]
+// GFX12: s_nand_b32 exec_lo, s1, s2              ; encoding: [0x01,0x02,0x7e,0x8e]
 
 s_nand_b32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x8e]
+// GFX12: s_nand_b32 exec_hi, s1, s2              ; encoding: [0x01,0x02,0x7f,0x8e]
 
 s_nand_b32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x8e]
+// GFX12: s_nand_b32 vcc_lo, s1, s2               ; encoding: [0x01,0x02,0x6a,0x8e]
 
 s_nand_b32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x8e]
+// GFX12: s_nand_b32 vcc_hi, s1, s2               ; encoding: [0x01,0x02,0x6b,0x8e]
 
 s_nand_b32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x8e]
+// GFX12: s_nand_b32 m0, s1, s2                   ; encoding: [0x01,0x02,0x7d,0x8e]
 
 s_nand_b32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x8e]
+// GFX12: s_nand_b32 s0, exec_lo, s2              ; encoding: [0x7e,0x02,0x00,0x8e]
 
 s_nand_b32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x8e]
+// GFX12: s_nand_b32 s0, exec_hi, s2              ; encoding: [0x7f,0x02,0x00,0x8e]
 
 s_nand_b32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x8e]
+// GFX12: s_nand_b32 s0, vcc_lo, s2               ; encoding: [0x6a,0x02,0x00,0x8e]
 
 s_nand_b32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x8e]
+// GFX12: s_nand_b32 s0, vcc_hi, s2               ; encoding: [0x6b,0x02,0x00,0x8e]
 
 s_nand_b32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x8e]
+// GFX12: s_nand_b32 s0, m0, s2                   ; encoding: [0x7d,0x02,0x00,0x8e]
 
 s_nand_b32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x8e]
+// GFX12: s_nand_b32 s0, 0, s2                    ; encoding: [0x80,0x02,0x00,0x8e]
 
 s_nand_b32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x8e]
+// GFX12: s_nand_b32 s0, -1, s2                   ; encoding: [0xc1,0x02,0x00,0x8e]
 
 s_nand_b32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x8e]
+// GFX12: s_nand_b32 s0, 0.5, s2                  ; encoding: [0xf0,0x02,0x00,0x8e]
 
 s_nand_b32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x8e]
+// GFX12: s_nand_b32 s0, -4.0, s2                 ; encoding: [0xf7,0x02,0x00,0x8e]
 
 s_nand_b32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x8e,0x73,0x72,0x71,0x3f]
+// GFX12: s_nand_b32 s0, 0x3f717273, s2           ; encoding: [0xff,0x02,0x00,0x8e,0x73,0x72,0x71,0x3f]
 
 s_nand_b32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x8e,0x56,0x34,0x12,0xaf]
+// GFX12: s_nand_b32 s0, 0xaf123456, s2           ; encoding: [0xff,0x02,0x00,0x8e,0x56,0x34,0x12,0xaf]
 
 s_nand_b32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x8e]
+// GFX12: s_nand_b32 s0, s1, exec_lo              ; encoding: [0x01,0x7e,0x00,0x8e]
 
 s_nand_b32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x8e]
+// GFX12: s_nand_b32 s0, s1, exec_hi              ; encoding: [0x01,0x7f,0x00,0x8e]
 
 s_nand_b32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x8e]
+// GFX12: s_nand_b32 s0, s1, vcc_lo               ; encoding: [0x01,0x6a,0x00,0x8e]
 
 s_nand_b32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x8e]
+// GFX12: s_nand_b32 s0, s1, vcc_hi               ; encoding: [0x01,0x6b,0x00,0x8e]
 
 s_nand_b32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x8e]
+// GFX12: s_nand_b32 s0, s1, m0                   ; encoding: [0x01,0x7d,0x00,0x8e]
 
 s_nand_b32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x8e]
+// GFX12: s_nand_b32 s0, s1, 0                    ; encoding: [0x01,0x80,0x00,0x8e]
 
 s_nand_b32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x8e]
+// GFX12: s_nand_b32 s0, s1, -1                   ; encoding: [0x01,0xc1,0x00,0x8e]
 
 s_nand_b32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x8e]
+// GFX12: s_nand_b32 s0, s1, 0.5                  ; encoding: [0x01,0xf0,0x00,0x8e]
 
 s_nand_b32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x8e]
+// GFX12: s_nand_b32 s0, s1, -4.0                 ; encoding: [0x01,0xf7,0x00,0x8e]
 
 s_nand_b32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x8e,0x73,0x72,0x71,0x3f]
+// GFX12: s_nand_b32 s0, s1, 0x3f717273           ; encoding: [0x01,0xff,0x00,0x8e,0x73,0x72,0x71,0x3f]
 
 s_nand_b32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x8e,0x56,0x34,0x12,0xaf]
+// GFX12: s_nand_b32 s0, s1, 0xaf123456           ; encoding: [0x01,0xff,0x00,0x8e,0x56,0x34,0x12,0xaf]
 
 s_nand_b64 s[0:1], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0x80,0x8e]
+// GFX12: s_nand_b64 s[0:1], s[2:3], s[4:5]       ; encoding: [0x02,0x04,0x80,0x8e]
 
 s_nand_b64 s[104:105], s[102:103], s[100:101]
-// GFX12: encoding: [0x66,0x64,0xe8,0x8e]
+// GFX12: s_nand_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x8e]
 
 s_nand_b64 s[0:1], s[102:103], s[100:101]
-// GFX12: encoding: [0x66,0x64,0x80,0x8e]
+// GFX12: s_nand_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x8e]
 
 s_nand_b64 s[104:105], s[2:3], s[100:101]
-// GFX12: encoding: [0x02,0x64,0xe8,0x8e]
+// GFX12: s_nand_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x8e]
 
 s_nand_b64 s[104:105], s[102:103], s[4:5]
-// GFX12: encoding: [0x66,0x04,0xe8,0x8e]
+// GFX12: s_nand_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x8e]
 
 s_nand_b64 s[104:105], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xe8,0x8e]
+// GFX12: s_nand_b64 s[104:105], s[2:3], s[4:5]   ; encoding: [0x02,0x04,0xe8,0x8e]
 
 s_nand_b64 s[0:1], s[102:103], s[4:5]
-// GFX12: encoding: [0x66,0x04,0x80,0x8e]
+// GFX12: s_nand_b64 s[0:1], s[102:103], s[4:5]   ; encoding: [0x66,0x04,0x80,0x8e]
 
 s_nand_b64 s[0:1], s[2:3], s[100:101]
-// GFX12: encoding: [0x02,0x64,0x80,0x8e]
+// GFX12: s_nand_b64 s[0:1], s[2:3], s[100:101]   ; encoding: [0x02,0x64,0x80,0x8e]
 
 s_nand_b64 exec, s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xfe,0x8e]
+// GFX12: s_nand_b64 exec, s[2:3], s[4:5]         ; encoding: [0x02,0x04,0xfe,0x8e]
 
 s_nand_b64 vcc, s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xea,0x8e]
+// GFX12: s_nand_b64 vcc, s[2:3], s[4:5]          ; encoding: [0x02,0x04,0xea,0x8e]
 
 s_nand_b64 s[0:1], exec, s[4:5]
-// GFX12: encoding: [0x7e,0x04,0x80,0x8e]
+// GFX12: s_nand_b64 s[0:1], exec, s[4:5]         ; encoding: [0x7e,0x04,0x80,0x8e]
 
 s_nand_b64 s[0:1], vcc, s[4:5]
-// GFX12: encoding: [0x6a,0x04,0x80,0x8e]
+// GFX12: s_nand_b64 s[0:1], vcc, s[4:5]          ; encoding: [0x6a,0x04,0x80,0x8e]
 
 s_nand_b64 s[0:1], 0, s[4:5]
-// GFX12: encoding: [0x80,0x04,0x80,0x8e]
+// GFX12: s_nand_b64 s[0:1], 0, s[4:5]            ; encoding: [0x80,0x04,0x80,0x8e]
 
 s_nand_b64 s[0:1], -1, s[4:5]
-// GFX12: encoding: [0xc1,0x04,0x80,0x8e]
+// GFX12: s_nand_b64 s[0:1], -1, s[4:5]           ; encoding: [0xc1,0x04,0x80,0x8e]
 
 s_nand_b64 s[0:1], 0.5, s[4:5]
-// GFX12: encoding: [0xf0,0x04,0x80,0x8e]
+// GFX12: s_nand_b64 s[0:1], 0.5, s[4:5]          ; encoding: [0xf0,0x04,0x80,0x8e]
 
 s_nand_b64 s[0:1], -4.0, s[4:5]
-// GFX12: encoding: [0xf7,0x04,0x80,0x8e]
+// GFX12: s_nand_b64 s[0:1], -4.0, s[4:5]         ; encoding: [0xf7,0x04,0x80,0x8e]
 
 s_nand_b64 s[0:1], 0x3f717273, s[4:5]
-// GFX12: encoding: [0xff,0x04,0x80,0x8e,0x73,0x72,0x71,0x3f]
+// GFX12: s_nand_b64 s[0:1], 0x3f717273, s[4:5]   ; encoding: [0xff,0x04,0x80,0x8e,0x73,0x72,0x71,0x3f]
 
 s_nand_b64 s[0:1], 0xaf123456, s[4:5]
-// GFX12: encoding: [0xff,0x04,0x80,0x8e,0x56,0x34,0x12,0xaf]
+// GFX12: s_nand_b64 s[0:1], 0xaf123456, s[4:5]   ; encoding: [0xff,0x04,0x80,0x8e,0x56,0x34,0x12,0xaf]
 
 s_nand_b64 s[0:1], s[2:3], exec
-// GFX12: encoding: [0x02,0x7e,0x80,0x8e]
+// GFX12: s_nand_b64 s[0:1], s[2:3], exec         ; encoding: [0x02,0x7e,0x80,0x8e]
 
 s_nand_b64 s[0:1], s[2:3], vcc
-// GFX12: encoding: [0x02,0x6a,0x80,0x8e]
+// GFX12: s_nand_b64 s[0:1], s[2:3], vcc          ; encoding: [0x02,0x6a,0x80,0x8e]
 
 s_nand_b64 s[0:1], s[2:3], 0
-// GFX12: encoding: [0x02,0x80,0x80,0x8e]
+// GFX12: s_nand_b64 s[0:1], s[2:3], 0            ; encoding: [0x02,0x80,0x80,0x8e]
 
 s_nand_b64 s[0:1], s[2:3], -1
-// GFX12: encoding: [0x02,0xc1,0x80,0x8e]
+// GFX12: s_nand_b64 s[0:1], s[2:3], -1           ; encoding: [0x02,0xc1,0x80,0x8e]
 
 s_nand_b64 s[0:1], s[2:3], 0.5
-// GFX12: encoding: [0x02,0xf0,0x80,0x8e]
+// GFX12: s_nand_b64 s[0:1], s[2:3], 0.5          ; encoding: [0x02,0xf0,0x80,0x8e]
 
 s_nand_b64 s[0:1], s[2:3], -4.0
-// GFX12: encoding: [0x02,0xf7,0x80,0x8e]
+// GFX12: s_nand_b64 s[0:1], s[2:3], -4.0         ; encoding: [0x02,0xf7,0x80,0x8e]
 
 s_nand_b64 s[0:1], s[2:3], 0x3f717273
-// GFX12: encoding: [0x02,0xff,0x80,0x8e,0x73,0x72,0x71,0x3f]
+// GFX12: s_nand_b64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x8e,0x73,0x72,0x71,0x3f]
 
 s_nand_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: encoding: [0x02,0xff,0x80,0x8e,0x56,0x34,0x12,0xaf]
+// GFX12: s_nand_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x8e,0x56,0x34,0x12,0xaf]
 
 s_nor_b32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x8f]
+// GFX12: s_nor_b32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x8f]
 
 s_nor_b32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x8f]
+// GFX12: s_nor_b32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x8f]
 
 s_nor_b32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x8f]
+// GFX12: s_nor_b32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x8f]
 
 s_nor_b32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x8f]
+// GFX12: s_nor_b32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x8f]
 
 s_nor_b32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x8f]
+// GFX12: s_nor_b32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x8f]
 
 s_nor_b32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x8f]
+// GFX12: s_nor_b32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x8f]
 
 s_nor_b32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x8f]
+// GFX12: s_nor_b32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x8f]
 
 s_nor_b32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x8f]
+// GFX12: s_nor_b32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x8f]
 
 s_nor_b32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x8f]
+// GFX12: s_nor_b32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x8f]
 
 s_nor_b32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x8f]
+// GFX12: s_nor_b32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x8f]
 
 s_nor_b32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x8f]
+// GFX12: s_nor_b32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x8f]
 
 s_nor_b32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x8f]
+// GFX12: s_nor_b32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x8f]
 
 s_nor_b32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x8f]
+// GFX12: s_nor_b32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x8f]
 
 s_nor_b32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x8f]
+// GFX12: s_nor_b32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x8f]
 
 s_nor_b32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x8f]
+// GFX12: s_nor_b32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x8f]
 
 s_nor_b32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x8f]
+// GFX12: s_nor_b32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x8f]
 
 s_nor_b32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x8f]
+// GFX12: s_nor_b32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x8f]
 
 s_nor_b32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x8f]
+// GFX12: s_nor_b32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x8f]
 
 s_nor_b32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x8f]
+// GFX12: s_nor_b32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x8f]
 
 s_nor_b32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x8f]
+// GFX12: s_nor_b32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x8f]
 
 s_nor_b32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x8f]
+// GFX12: s_nor_b32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x8f]
 
 s_nor_b32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x8f]
+// GFX12: s_nor_b32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x8f]
 
 s_nor_b32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x8f,0x73,0x72,0x71,0x3f]
+// GFX12: s_nor_b32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x8f,0x73,0x72,0x71,0x3f]
 
 s_nor_b32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x8f,0x56,0x34,0x12,0xaf]
+// GFX12: s_nor_b32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x8f,0x56,0x34,0x12,0xaf]
 
 s_nor_b32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x8f]
+// GFX12: s_nor_b32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x8f]
 
 s_nor_b32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x8f]
+// GFX12: s_nor_b32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x8f]
 
 s_nor_b32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x8f]
+// GFX12: s_nor_b32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x8f]
 
 s_nor_b32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x8f]
+// GFX12: s_nor_b32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x8f]
 
 s_nor_b32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x8f]
+// GFX12: s_nor_b32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x8f]
 
 s_nor_b32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x8f]
+// GFX12: s_nor_b32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x8f]
 
 s_nor_b32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x8f]
+// GFX12: s_nor_b32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x8f]
 
 s_nor_b32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x8f]
+// GFX12: s_nor_b32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x8f]
 
 s_nor_b32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x8f]
+// GFX12: s_nor_b32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x8f]
 
 s_nor_b32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x8f,0x73,0x72,0x71,0x3f]
+// GFX12: s_nor_b32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x8f,0x73,0x72,0x71,0x3f]
 
 s_nor_b32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x8f,0x56,0x34,0x12,0xaf]
+// GFX12: s_nor_b32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x8f,0x56,0x34,0x12,0xaf]
 
 s_nor_b64 s[0:1], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0x80,0x8f]
+// GFX12: s_nor_b64 s[0:1], s[2:3], s[4:5]        ; encoding: [0x02,0x04,0x80,0x8f]
 
 s_nor_b64 s[104:105], s[102:103], s[100:101]
-// GFX12: encoding: [0x66,0x64,0xe8,0x8f]
+// GFX12: s_nor_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x8f]
 
 s_nor_b64 s[0:1], s[102:103], s[100:101]
-// GFX12: encoding: [0x66,0x64,0x80,0x8f]
+// GFX12: s_nor_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x8f]
 
 s_nor_b64 s[104:105], s[2:3], s[100:101]
-// GFX12: encoding: [0x02,0x64,0xe8,0x8f]
+// GFX12: s_nor_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x8f]
 
 s_nor_b64 s[104:105], s[102:103], s[4:5]
-// GFX12: encoding: [0x66,0x04,0xe8,0x8f]
+// GFX12: s_nor_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x8f]
 
 s_nor_b64 s[104:105], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xe8,0x8f]
+// GFX12: s_nor_b64 s[104:105], s[2:3], s[4:5]    ; encoding: [0x02,0x04,0xe8,0x8f]
 
 s_nor_b64 s[0:1], s[102:103], s[4:5]
-// GFX12: encoding: [0x66,0x04,0x80,0x8f]
+// GFX12: s_nor_b64 s[0:1], s[102:103], s[4:5]    ; encoding: [0x66,0x04,0x80,0x8f]
 
 s_nor_b64 s[0:1], s[2:3], s[100:101]
-// GFX12: encoding: [0x02,0x64,0x80,0x8f]
+// GFX12: s_nor_b64 s[0:1], s[2:3], s[100:101]    ; encoding: [0x02,0x64,0x80,0x8f]
 
 s_nor_b64 exec, s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xfe,0x8f]
+// GFX12: s_nor_b64 exec, s[2:3], s[4:5]          ; encoding: [0x02,0x04,0xfe,0x8f]
 
 s_nor_b64 vcc, s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xea,0x8f]
+// GFX12: s_nor_b64 vcc, s[2:3], s[4:5]           ; encoding: [0x02,0x04,0xea,0x8f]
 
 s_nor_b64 s[0:1], exec, s[4:5]
-// GFX12: encoding: [0x7e,0x04,0x80,0x8f]
+// GFX12: s_nor_b64 s[0:1], exec, s[4:5]          ; encoding: [0x7e,0x04,0x80,0x8f]
 
 s_nor_b64 s[0:1], vcc, s[4:5]
-// GFX12: encoding: [0x6a,0x04,0x80,0x8f]
+// GFX12: s_nor_b64 s[0:1], vcc, s[4:5]           ; encoding: [0x6a,0x04,0x80,0x8f]
 
 s_nor_b64 s[0:1], 0, s[4:5]
-// GFX12: encoding: [0x80,0x04,0x80,0x8f]
+// GFX12: s_nor_b64 s[0:1], 0, s[4:5]             ; encoding: [0x80,0x04,0x80,0x8f]
 
 s_nor_b64 s[0:1], -1, s[4:5]
-// GFX12: encoding: [0xc1,0x04,0x80,0x8f]
+// GFX12: s_nor_b64 s[0:1], -1, s[4:5]            ; encoding: [0xc1,0x04,0x80,0x8f]
 
 s_nor_b64 s[0:1], 0.5, s[4:5]
-// GFX12: encoding: [0xf0,0x04,0x80,0x8f]
+// GFX12: s_nor_b64 s[0:1], 0.5, s[4:5]           ; encoding: [0xf0,0x04,0x80,0x8f]
 
 s_nor_b64 s[0:1], -4.0, s[4:5]
-// GFX12: encoding: [0xf7,0x04,0x80,0x8f]
+// GFX12: s_nor_b64 s[0:1], -4.0, s[4:5]          ; encoding: [0xf7,0x04,0x80,0x8f]
 
 s_nor_b64 s[0:1], 0x3f717273, s[4:5]
-// GFX12: encoding: [0xff,0x04,0x80,0x8f,0x73,0x72,0x71,0x3f]
+// GFX12: s_nor_b64 s[0:1], 0x3f717273, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8f,0x73,0x72,0x71,0x3f]
 
 s_nor_b64 s[0:1], 0xaf123456, s[4:5]
-// GFX12: encoding: [0xff,0x04,0x80,0x8f,0x56,0x34,0x12,0xaf]
+// GFX12: s_nor_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8f,0x56,0x34,0x12,0xaf]
 
 s_nor_b64 s[0:1], s[2:3], exec
-// GFX12: encoding: [0x02,0x7e,0x80,0x8f]
+// GFX12: s_nor_b64 s[0:1], s[2:3], exec          ; encoding: [0x02,0x7e,0x80,0x8f]
 
 s_nor_b64 s[0:1], s[2:3], vcc
-// GFX12: encoding: [0x02,0x6a,0x80,0x8f]
+// GFX12: s_nor_b64 s[0:1], s[2:3], vcc           ; encoding: [0x02,0x6a,0x80,0x8f]
 
 s_nor_b64 s[0:1], s[2:3], 0
-// GFX12: encoding: [0x02,0x80,0x80,0x8f]
+// GFX12: s_nor_b64 s[0:1], s[2:3], 0             ; encoding: [0x02,0x80,0x80,0x8f]
 
 s_nor_b64 s[0:1], s[2:3], -1
-// GFX12: encoding: [0x02,0xc1,0x80,0x8f]
+// GFX12: s_nor_b64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x80,0x8f]
 
 s_nor_b64 s[0:1], s[2:3], 0.5
-// GFX12: encoding: [0x02,0xf0,0x80,0x8f]
+// GFX12: s_nor_b64 s[0:1], s[2:3], 0.5           ; encoding: [0x02,0xf0,0x80,0x8f]
 
 s_nor_b64 s[0:1], s[2:3], -4.0
-// GFX12: encoding: [0x02,0xf7,0x80,0x8f]
+// GFX12: s_nor_b64 s[0:1], s[2:3], -4.0          ; encoding: [0x02,0xf7,0x80,0x8f]
 
 s_nor_b64 s[0:1], s[2:3], 0x3f717273
-// GFX12: encoding: [0x02,0xff,0x80,0x8f,0x73,0x72,0x71,0x3f]
+// GFX12: s_nor_b64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0x8f,0x73,0x72,0x71,0x3f]
 
 s_nor_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: encoding: [0x02,0xff,0x80,0x8f,0x56,0x34,0x12,0xaf]
+// GFX12: s_nor_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8f,0x56,0x34,0x12,0xaf]
 
 s_xnor_b32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x90]
+// GFX12: s_xnor_b32 s0, s1, s2                   ; encoding: [0x01,0x02,0x00,0x90]
 
 s_xnor_b32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x90]
+// GFX12: s_xnor_b32 s105, s104, s103             ; encoding: [0x68,0x67,0x69,0x90]
 
 s_xnor_b32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x90]
+// GFX12: s_xnor_b32 s0, s104, s103               ; encoding: [0x68,0x67,0x00,0x90]
 
 s_xnor_b32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x90]
+// GFX12: s_xnor_b32 s105, s1, s103               ; encoding: [0x01,0x67,0x69,0x90]
 
 s_xnor_b32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x90]
+// GFX12: s_xnor_b32 s105, s104, s2               ; encoding: [0x68,0x02,0x69,0x90]
 
 s_xnor_b32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x90]
+// GFX12: s_xnor_b32 s105, s1, s2                 ; encoding: [0x01,0x02,0x69,0x90]
 
 s_xnor_b32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x90]
+// GFX12: s_xnor_b32 s0, s104, s2                 ; encoding: [0x68,0x02,0x00,0x90]
 
 s_xnor_b32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x90]
+// GFX12: s_xnor_b32 s0, s1, s103                 ; encoding: [0x01,0x67,0x00,0x90]
 
 s_xnor_b32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x90]
+// GFX12: s_xnor_b32 exec_lo, s1, s2              ; encoding: [0x01,0x02,0x7e,0x90]
 
 s_xnor_b32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x90]
+// GFX12: s_xnor_b32 exec_hi, s1, s2              ; encoding: [0x01,0x02,0x7f,0x90]
 
 s_xnor_b32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x90]
+// GFX12: s_xnor_b32 vcc_lo, s1, s2               ; encoding: [0x01,0x02,0x6a,0x90]
 
 s_xnor_b32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x90]
+// GFX12: s_xnor_b32 vcc_hi, s1, s2               ; encoding: [0x01,0x02,0x6b,0x90]
 
 s_xnor_b32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x90]
+// GFX12: s_xnor_b32 m0, s1, s2                   ; encoding: [0x01,0x02,0x7d,0x90]
 
 s_xnor_b32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x90]
+// GFX12: s_xnor_b32 s0, exec_lo, s2              ; encoding: [0x7e,0x02,0x00,0x90]
 
 s_xnor_b32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x90]
+// GFX12: s_xnor_b32 s0, exec_hi, s2              ; encoding: [0x7f,0x02,0x00,0x90]
 
 s_xnor_b32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x90]
+// GFX12: s_xnor_b32 s0, vcc_lo, s2               ; encoding: [0x6a,0x02,0x00,0x90]
 
 s_xnor_b32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x90]
+// GFX12: s_xnor_b32 s0, vcc_hi, s2               ; encoding: [0x6b,0x02,0x00,0x90]
 
 s_xnor_b32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x90]
+// GFX12: s_xnor_b32 s0, m0, s2                   ; encoding: [0x7d,0x02,0x00,0x90]
 
 s_xnor_b32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x90]
+// GFX12: s_xnor_b32 s0, 0, s2                    ; encoding: [0x80,0x02,0x00,0x90]
 
 s_xnor_b32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x90]
+// GFX12: s_xnor_b32 s0, -1, s2                   ; encoding: [0xc1,0x02,0x00,0x90]
 
 s_xnor_b32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x90]
+// GFX12: s_xnor_b32 s0, 0.5, s2                  ; encoding: [0xf0,0x02,0x00,0x90]
 
 s_xnor_b32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x90]
+// GFX12: s_xnor_b32 s0, -4.0, s2                 ; encoding: [0xf7,0x02,0x00,0x90]
 
 s_xnor_b32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x90,0x73,0x72,0x71,0x3f]
+// GFX12: s_xnor_b32 s0, 0x3f717273, s2           ; encoding: [0xff,0x02,0x00,0x90,0x73,0x72,0x71,0x3f]
 
 s_xnor_b32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x90,0x56,0x34,0x12,0xaf]
+// GFX12: s_xnor_b32 s0, 0xaf123456, s2           ; encoding: [0xff,0x02,0x00,0x90,0x56,0x34,0x12,0xaf]
 
 s_xnor_b32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x90]
+// GFX12: s_xnor_b32 s0, s1, exec_lo              ; encoding: [0x01,0x7e,0x00,0x90]
 
 s_xnor_b32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x90]
+// GFX12: s_xnor_b32 s0, s1, exec_hi              ; encoding: [0x01,0x7f,0x00,0x90]
 
 s_xnor_b32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x90]
+// GFX12: s_xnor_b32 s0, s1, vcc_lo               ; encoding: [0x01,0x6a,0x00,0x90]
 
 s_xnor_b32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x90]
+// GFX12: s_xnor_b32 s0, s1, vcc_hi               ; encoding: [0x01,0x6b,0x00,0x90]
 
 s_xnor_b32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x90]
+// GFX12: s_xnor_b32 s0, s1, m0                   ; encoding: [0x01,0x7d,0x00,0x90]
 
 s_xnor_b32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x90]
+// GFX12: s_xnor_b32 s0, s1, 0                    ; encoding: [0x01,0x80,0x00,0x90]
 
 s_xnor_b32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x90]
+// GFX12: s_xnor_b32 s0, s1, -1                   ; encoding: [0x01,0xc1,0x00,0x90]
 
 s_xnor_b32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x90]
+// GFX12: s_xnor_b32 s0, s1, 0.5                  ; encoding: [0x01,0xf0,0x00,0x90]
 
 s_xnor_b32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x90]
+// GFX12: s_xnor_b32 s0, s1, -4.0                 ; encoding: [0x01,0xf7,0x00,0x90]
 
 s_xnor_b32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x90,0x73,0x72,0x71,0x3f]
+// GFX12: s_xnor_b32 s0, s1, 0x3f717273           ; encoding: [0x01,0xff,0x00,0x90,0x73,0x72,0x71,0x3f]
 
 s_xnor_b32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x90,0x56,0x34,0x12,0xaf]
+// GFX12: s_xnor_b32 s0, s1, 0xaf123456           ; encoding: [0x01,0xff,0x00,0x90,0x56,0x34,0x12,0xaf]
 
 s_xnor_b64 s[0:1], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0x80,0x90]
+// GFX12: s_xnor_b64 s[0:1], s[2:3], s[4:5]       ; encoding: [0x02,0x04,0x80,0x90]
 
 s_xnor_b64 s[104:105], s[102:103], s[100:101]
-// GFX12: encoding: [0x66,0x64,0xe8,0x90]
+// GFX12: s_xnor_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x90]
 
 s_xnor_b64 s[0:1], s[102:103], s[100:101]
-// GFX12: encoding: [0x66,0x64,0x80,0x90]
+// GFX12: s_xnor_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x90]
 
 s_xnor_b64 s[104:105], s[2:3], s[100:101]
-// GFX12: encoding: [0x02,0x64,0xe8,0x90]
+// GFX12: s_xnor_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x90]
 
 s_xnor_b64 s[104:105], s[102:103], s[4:5]
-// GFX12: encoding: [0x66,0x04,0xe8,0x90]
+// GFX12: s_xnor_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x90]
 
 s_xnor_b64 s[104:105], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xe8,0x90]
+// GFX12: s_xnor_b64 s[104:105], s[2:3], s[4:5]   ; encoding: [0x02,0x04,0xe8,0x90]
 
 s_xnor_b64 s[0:1], s[102:103], s[4:5]
-// GFX12: encoding: [0x66,0x04,0x80,0x90]
+// GFX12: s_xnor_b64 s[0:1], s[102:103], s[4:5]   ; encoding: [0x66,0x04,0x80,0x90]
 
 s_xnor_b64 s[0:1], s[2:3], s[100:101]
-// GFX12: encoding: [0x02,0x64,0x80,0x90]
+// GFX12: s_xnor_b64 s[0:1], s[2:3], s[100:101]   ; encoding: [0x02,0x64,0x80,0x90]
 
 s_xnor_b64 exec, s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xfe,0x90]
+// GFX12: s_xnor_b64 exec, s[2:3], s[4:5]         ; encoding: [0x02,0x04,0xfe,0x90]
 
 s_xnor_b64 vcc, s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0xea,0x90]
+// GFX12: s_xnor_b64 vcc, s[2:3], s[4:5]          ; encoding: [0x02,0x04,0xea,0x90]
 
 s_xnor_b64 s[0:1], exec, s[4:5]
-// GFX12: encoding: [0x7e,0x04,0x80,0x90]
+// GFX12: s_xnor_b64 s[0:1], exec, s[4:5]         ; encoding: [0x7e,0x04,0x80,0x90]
 
 s_xnor_b64 s[0:1], vcc, s[4:5]
-// GFX12: encoding: [0x6a,0x04,0x80,0x90]
+// GFX12: s_xnor_b64 s[0:1], vcc, s[4:5]          ; encoding: [0x6a,0x04,0x80,0x90]
 
 s_xnor_b64 s[0:1], 0, s[4:5]
-// GFX12: encoding: [0x80,0x04,0x80,0x90]
+// GFX12: s_xnor_b64 s[0:1], 0, s[4:5]            ; encoding: [0x80,0x04,0x80,0x90]
 
 s_xnor_b64 s[0:1], -1, s[4:5]
-// GFX12: encoding: [0xc1,0x04,0x80,0x90]
+// GFX12: s_xnor_b64 s[0:1], -1, s[4:5]           ; encoding: [0xc1,0x04,0x80,0x90]
 
 s_xnor_b64 s[0:1], 0.5, s[4:5]
-// GFX12: encoding: [0xf0,0x04,0x80,0x90]
+// GFX12: s_xnor_b64 s[0:1], 0.5, s[4:5]          ; encoding: [0xf0,0x04,0x80,0x90]
 
 s_xnor_b64 s[0:1], -4.0, s[4:5]
-// GFX12: encoding: [0xf7,0x04,0x80,0x90]
+// GFX12: s_xnor_b64 s[0:1], -4.0, s[4:5]         ; encoding: [0xf7,0x04,0x80,0x90]
 
 s_xnor_b64 s[0:1], 0x3f717273, s[4:5]
-// GFX12: encoding: [0xff,0x04,0x80,0x90,0x73,0x72,0x71,0x3f]
+// GFX12: s_xnor_b64 s[0:1], 0x3f717273, s[4:5]   ; encoding: [0xff,0x04,0x80,0x90,0x73,0x72,0x71,0x3f]
 
 s_xnor_b64 s[0:1], 0xaf123456, s[4:5]
-// GFX12: encoding: [0xff,0x04,0x80,0x90,0x56,0x34,0x12,0xaf]
+// GFX12: s_xnor_b64 s[0:1], 0xaf123456, s[4:5]   ; encoding: [0xff,0x04,0x80,0x90,0x56,0x34,0x12,0xaf]
 
 s_xnor_b64 s[0:1], s[2:3], exec
-// GFX12: encoding: [0x02,0x7e,0x80,0x90]
+// GFX12: s_xnor_b64 s[0:1], s[2:3], exec         ; encoding: [0x02,0x7e,0x80,0x90]
 
 s_xnor_b64 s[0:1], s[2:3], vcc
-// GFX12: encoding: [0x02,0x6a,0x80,0x90]
+// GFX12: s_xnor_b64 s[0:1], s[2:3], vcc          ; encoding: [0x02,0x6a,0x80,0x90]
 
 s_xnor_b64 s[0:1], s[2:3], 0
-// GFX12: encoding: [0x02,0x80,0x80,0x90]
+// GFX12: s_xnor_b64 s[0:1], s[2:3], 0            ; encoding: [0x02,0x80,0x80,0x90]
 
 s_xnor_b64 s[0:1], s[2:3], -1
-// GFX12: encoding: [0x02,0xc1,0x80,0x90]
+// GFX12: s_xnor_b64 s[0:1], s[2:3], -1           ; encoding: [0x02,0xc1,0x80,0x90]
 
 s_xnor_b64 s[0:1], s[2:3], 0.5
-// GFX12: encoding: [0x02,0xf0,0x80,0x90]
+// GFX12: s_xnor_b64 s[0:1], s[2:3], 0.5          ; encoding: [0x02,0xf0,0x80,0x90]
 
 s_xnor_b64 s[0:1], s[2:3], -4.0
-// GFX12: encoding: [0x02,0xf7,0x80,0x90]
+// GFX12: s_xnor_b64 s[0:1], s[2:3], -4.0         ; encoding: [0x02,0xf7,0x80,0x90]
 
 s_xnor_b64 s[0:1], s[2:3], 0x3f717273
-// GFX12: encoding: [0x02,0xff,0x80,0x90,0x73,0x72,0x71,0x3f]
+// GFX12: s_xnor_b64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x90,0x73,0x72,0x71,0x3f]
 
 s_xnor_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: encoding: [0x02,0xff,0x80,0x90,0x56,0x34,0x12,0xaf]
+// GFX12: s_xnor_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x90,0x56,0x34,0x12,0xaf]
 
 s_lshl_b32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x84]
+// GFX12: s_lshl_b32 s0, s1, s2                   ; encoding: [0x01,0x02,0x00,0x84]
 
 s_lshl_b32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x84]
+// GFX12: s_lshl_b32 s105, s104, s103             ; encoding: [0x68,0x67,0x69,0x84]
 
 s_lshl_b32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x84]
+// GFX12: s_lshl_b32 s0, s104, s103               ; encoding: [0x68,0x67,0x00,0x84]
 
 s_lshl_b32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x84]
+// GFX12: s_lshl_b32 s105, s1, s103               ; encoding: [0x01,0x67,0x69,0x84]
 
 s_lshl_b32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x84]
+// GFX12: s_lshl_b32 s105, s104, s2               ; encoding: [0x68,0x02,0x69,0x84]
 
 s_lshl_b32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x84]
+// GFX12: s_lshl_b32 s105, s1, s2                 ; encoding: [0x01,0x02,0x69,0x84]
 
 s_lshl_b32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x84]
+// GFX12: s_lshl_b32 s0, s104, s2                 ; encoding: [0x68,0x02,0x00,0x84]
 
 s_lshl_b32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x84]
+// GFX12: s_lshl_b32 s0, s1, s103                 ; encoding: [0x01,0x67,0x00,0x84]
 
 s_lshl_b32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x84]
+// GFX12: s_lshl_b32 exec_lo, s1, s2              ; encoding: [0x01,0x02,0x7e,0x84]
 
 s_lshl_b32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x84]
+// GFX12: s_lshl_b32 exec_hi, s1, s2              ; encoding: [0x01,0x02,0x7f,0x84]
 
 s_lshl_b32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x84]
+// GFX12: s_lshl_b32 vcc_lo, s1, s2               ; encoding: [0x01,0x02,0x6a,0x84]
 
 s_lshl_b32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x84]
+// GFX12: s_lshl_b32 vcc_hi, s1, s2               ; encoding: [0x01,0x02,0x6b,0x84]
 
 s_lshl_b32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x84]
+// GFX12: s_lshl_b32 m0, s1, s2                   ; encoding: [0x01,0x02,0x7d,0x84]
 
 s_lshl_b32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x84]
+// GFX12: s_lshl_b32 s0, exec_lo, s2              ; encoding: [0x7e,0x02,0x00,0x84]
 
 s_lshl_b32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x84]
+// GFX12: s_lshl_b32 s0, exec_hi, s2              ; encoding: [0x7f,0x02,0x00,0x84]
 
 s_lshl_b32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x84]
+// GFX12: s_lshl_b32 s0, vcc_lo, s2               ; encoding: [0x6a,0x02,0x00,0x84]
 
 s_lshl_b32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x84]
+// GFX12: s_lshl_b32 s0, vcc_hi, s2               ; encoding: [0x6b,0x02,0x00,0x84]
 
 s_lshl_b32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x84]
+// GFX12: s_lshl_b32 s0, m0, s2                   ; encoding: [0x7d,0x02,0x00,0x84]
 
 s_lshl_b32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x84]
+// GFX12: s_lshl_b32 s0, 0, s2                    ; encoding: [0x80,0x02,0x00,0x84]
 
 s_lshl_b32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x84]
+// GFX12: s_lshl_b32 s0, -1, s2                   ; encoding: [0xc1,0x02,0x00,0x84]
 
 s_lshl_b32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x84]
+// GFX12: s_lshl_b32 s0, 0.5, s2                  ; encoding: [0xf0,0x02,0x00,0x84]
 
 s_lshl_b32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x84]
+// GFX12: s_lshl_b32 s0, -4.0, s2                 ; encoding: [0xf7,0x02,0x00,0x84]
 
 s_lshl_b32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x84,0x73,0x72,0x71,0x3f]
+// GFX12: s_lshl_b32 s0, 0x3f717273, s2           ; encoding: [0xff,0x02,0x00,0x84,0x73,0x72,0x71,0x3f]
 
 s_lshl_b32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x84,0x56,0x34,0x12,0xaf]
+// GFX12: s_lshl_b32 s0, 0xaf123456, s2           ; encoding: [0xff,0x02,0x00,0x84,0x56,0x34,0x12,0xaf]
 
 s_lshl_b32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x84]
+// GFX12: s_lshl_b32 s0, s1, exec_lo              ; encoding: [0x01,0x7e,0x00,0x84]
 
 s_lshl_b32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x84]
+// GFX12: s_lshl_b32 s0, s1, exec_hi              ; encoding: [0x01,0x7f,0x00,0x84]
 
 s_lshl_b32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x84]
+// GFX12: s_lshl_b32 s0, s1, vcc_lo               ; encoding: [0x01,0x6a,0x00,0x84]
 
 s_lshl_b32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x84]
+// GFX12: s_lshl_b32 s0, s1, vcc_hi               ; encoding: [0x01,0x6b,0x00,0x84]
 
 s_lshl_b32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x84]
+// GFX12: s_lshl_b32 s0, s1, m0                   ; encoding: [0x01,0x7d,0x00,0x84]
 
 s_lshl_b32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x84]
+// GFX12: s_lshl_b32 s0, s1, 0                    ; encoding: [0x01,0x80,0x00,0x84]
 
 s_lshl_b32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x84]
+// GFX12: s_lshl_b32 s0, s1, -1                   ; encoding: [0x01,0xc1,0x00,0x84]
 
 s_lshl_b32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x84]
+// GFX12: s_lshl_b32 s0, s1, 0.5                  ; encoding: [0x01,0xf0,0x00,0x84]
 
 s_lshl_b32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x84]
+// GFX12: s_lshl_b32 s0, s1, -4.0                 ; encoding: [0x01,0xf7,0x00,0x84]
 
 s_lshl_b32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x84,0x73,0x72,0x71,0x3f]
+// GFX12: s_lshl_b32 s0, s1, 0x3f717273           ; encoding: [0x01,0xff,0x00,0x84,0x73,0x72,0x71,0x3f]
 
 s_lshl_b32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x84,0x56,0x34,0x12,0xaf]
+// GFX12: s_lshl_b32 s0, s1, 0xaf123456           ; encoding: [0x01,0xff,0x00,0x84,0x56,0x34,0x12,0xaf]
 
 s_lshl_b64 s[0:1], s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0x80,0x84]
+// GFX12: s_lshl_b64 s[0:1], s[2:3], s4           ; encoding: [0x02,0x04,0x80,0x84]
 
 s_lshl_b64 s[104:105], s[102:103], s100
-// GFX12: encoding: [0x66,0x64,0xe8,0x84]
+// GFX12: s_lshl_b64 s[104:105], s[102:103], s100 ; encoding: [0x66,0x64,0xe8,0x84]
 
 s_lshl_b64 s[0:1], s[102:103], s100
-// GFX12: encoding: [0x66,0x64,0x80,0x84]
+// GFX12: s_lshl_b64 s[0:1], s[102:103], s100     ; encoding: [0x66,0x64,0x80,0x84]
 
 s_lshl_b64 s[104:105], s[2:3], s100
-// GFX12: encoding: [0x02,0x64,0xe8,0x84]
+// GFX12: s_lshl_b64 s[104:105], s[2:3], s100     ; encoding: [0x02,0x64,0xe8,0x84]
 
 s_lshl_b64 s[104:105], s[102:103], s4
-// GFX12: encoding: [0x66,0x04,0xe8,0x84]
+// GFX12: s_lshl_b64 s[104:105], s[102:103], s4   ; encoding: [0x66,0x04,0xe8,0x84]
 
 s_lshl_b64 s[104:105], s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0xe8,0x84]
+// GFX12: s_lshl_b64 s[104:105], s[2:3], s4       ; encoding: [0x02,0x04,0xe8,0x84]
 
 s_lshl_b64 s[0:1], s[102:103], s4
-// GFX12: encoding: [0x66,0x04,0x80,0x84]
+// GFX12: s_lshl_b64 s[0:1], s[102:103], s4       ; encoding: [0x66,0x04,0x80,0x84]
 
 s_lshl_b64 s[0:1], s[2:3], s100
-// GFX12: encoding: [0x02,0x64,0x80,0x84]
+// GFX12: s_lshl_b64 s[0:1], s[2:3], s100         ; encoding: [0x02,0x64,0x80,0x84]
 
 s_lshl_b64 exec, s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0xfe,0x84]
+// GFX12: s_lshl_b64 exec, s[2:3], s4             ; encoding: [0x02,0x04,0xfe,0x84]
 
 s_lshl_b64 vcc, s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0xea,0x84]
+// GFX12: s_lshl_b64 vcc, s[2:3], s4              ; encoding: [0x02,0x04,0xea,0x84]
 
 s_lshl_b64 s[0:1], exec, s4
-// GFX12: encoding: [0x7e,0x04,0x80,0x84]
+// GFX12: s_lshl_b64 s[0:1], exec, s4             ; encoding: [0x7e,0x04,0x80,0x84]
 
 s_lshl_b64 s[0:1], vcc, s4
-// GFX12: encoding: [0x6a,0x04,0x80,0x84]
+// GFX12: s_lshl_b64 s[0:1], vcc, s4              ; encoding: [0x6a,0x04,0x80,0x84]
 
 s_lshl_b64 s[0:1], 0, s4
-// GFX12: encoding: [0x80,0x04,0x80,0x84]
+// GFX12: s_lshl_b64 s[0:1], 0, s4                ; encoding: [0x80,0x04,0x80,0x84]
 
 s_lshl_b64 s[0:1], -1, s4
-// GFX12: encoding: [0xc1,0x04,0x80,0x84]
+// GFX12: s_lshl_b64 s[0:1], -1, s4               ; encoding: [0xc1,0x04,0x80,0x84]
 
 s_lshl_b64 s[0:1], 0.5, s4
-// GFX12: encoding: [0xf0,0x04,0x80,0x84]
+// GFX12: s_lshl_b64 s[0:1], 0.5, s4              ; encoding: [0xf0,0x04,0x80,0x84]
 
 s_lshl_b64 s[0:1], -4.0, s4
-// GFX12: encoding: [0xf7,0x04,0x80,0x84]
+// GFX12: s_lshl_b64 s[0:1], -4.0, s4             ; encoding: [0xf7,0x04,0x80,0x84]
 
 s_lshl_b64 s[0:1], 0x3f717273, s4
-// GFX12: encoding: [0xff,0x04,0x80,0x84,0x73,0x72,0x71,0x3f]
+// GFX12: s_lshl_b64 s[0:1], 0x3f717273, s4       ; encoding: [0xff,0x04,0x80,0x84,0x73,0x72,0x71,0x3f]
 
 s_lshl_b64 s[0:1], 0xaf123456, s4
-// GFX12: encoding: [0xff,0x04,0x80,0x84,0x56,0x34,0x12,0xaf]
+// GFX12: s_lshl_b64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x84,0x56,0x34,0x12,0xaf]
 
 s_lshl_b64 s[0:1], s[2:3], exec_lo
-// GFX12: encoding: [0x02,0x7e,0x80,0x84]
+// GFX12: s_lshl_b64 s[0:1], s[2:3], exec_lo      ; encoding: [0x02,0x7e,0x80,0x84]
 
 s_lshl_b64 s[0:1], s[2:3], vcc_lo
-// GFX12: encoding: [0x02,0x6a,0x80,0x84]
+// GFX12: s_lshl_b64 s[0:1], s[2:3], vcc_lo       ; encoding: [0x02,0x6a,0x80,0x84]
 
 s_lshl_b64 s[0:1], s[2:3], 0
-// GFX12: encoding: [0x02,0x80,0x80,0x84]
+// GFX12: s_lshl_b64 s[0:1], s[2:3], 0            ; encoding: [0x02,0x80,0x80,0x84]
 
 s_lshl_b64 s[0:1], s[2:3], -1
-// GFX12: encoding: [0x02,0xc1,0x80,0x84]
+// GFX12: s_lshl_b64 s[0:1], s[2:3], -1           ; encoding: [0x02,0xc1,0x80,0x84]
 
 s_lshl_b64 s[0:1], s[2:3], 0.5
-// GFX12: encoding: [0x02,0xf0,0x80,0x84]
+// GFX12: s_lshl_b64 s[0:1], s[2:3], 0.5          ; encoding: [0x02,0xf0,0x80,0x84]
 
 s_lshl_b64 s[0:1], s[2:3], -4.0
-// GFX12: encoding: [0x02,0xf7,0x80,0x84]
+// GFX12: s_lshl_b64 s[0:1], s[2:3], -4.0         ; encoding: [0x02,0xf7,0x80,0x84]
 
 s_lshl_b64 s[0:1], s[2:3], 0x3f717273
-// GFX12: encoding: [0x02,0xff,0x80,0x84,0x73,0x72,0x71,0x3f]
+// GFX12: s_lshl_b64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x84,0x73,0x72,0x71,0x3f]
 
 s_lshl_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: encoding: [0x02,0xff,0x80,0x84,0x56,0x34,0x12,0xaf]
+// GFX12: s_lshl_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x84,0x56,0x34,0x12,0xaf]
 
 s_lshr_b32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x85]
+// GFX12: s_lshr_b32 s0, s1, s2                   ; encoding: [0x01,0x02,0x00,0x85]
 
 s_lshr_b32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x85]
+// GFX12: s_lshr_b32 s105, s104, s103             ; encoding: [0x68,0x67,0x69,0x85]
 
 s_lshr_b32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x85]
+// GFX12: s_lshr_b32 s0, s104, s103               ; encoding: [0x68,0x67,0x00,0x85]
 
 s_lshr_b32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x85]
+// GFX12: s_lshr_b32 s105, s1, s103               ; encoding: [0x01,0x67,0x69,0x85]
 
 s_lshr_b32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x85]
+// GFX12: s_lshr_b32 s105, s104, s2               ; encoding: [0x68,0x02,0x69,0x85]
 
 s_lshr_b32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x85]
+// GFX12: s_lshr_b32 s105, s1, s2                 ; encoding: [0x01,0x02,0x69,0x85]
 
 s_lshr_b32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x85]
+// GFX12: s_lshr_b32 s0, s104, s2                 ; encoding: [0x68,0x02,0x00,0x85]
 
 s_lshr_b32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x85]
+// GFX12: s_lshr_b32 s0, s1, s103                 ; encoding: [0x01,0x67,0x00,0x85]
 
 s_lshr_b32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x85]
+// GFX12: s_lshr_b32 exec_lo, s1, s2              ; encoding: [0x01,0x02,0x7e,0x85]
 
 s_lshr_b32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x85]
+// GFX12: s_lshr_b32 exec_hi, s1, s2              ; encoding: [0x01,0x02,0x7f,0x85]
 
 s_lshr_b32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x85]
+// GFX12: s_lshr_b32 vcc_lo, s1, s2               ; encoding: [0x01,0x02,0x6a,0x85]
 
 s_lshr_b32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x85]
+// GFX12: s_lshr_b32 vcc_hi, s1, s2               ; encoding: [0x01,0x02,0x6b,0x85]
 
 s_lshr_b32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x85]
+// GFX12: s_lshr_b32 m0, s1, s2                   ; encoding: [0x01,0x02,0x7d,0x85]
 
 s_lshr_b32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x85]
+// GFX12: s_lshr_b32 s0, exec_lo, s2              ; encoding: [0x7e,0x02,0x00,0x85]
 
 s_lshr_b32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x85]
+// GFX12: s_lshr_b32 s0, exec_hi, s2              ; encoding: [0x7f,0x02,0x00,0x85]
 
 s_lshr_b32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x85]
+// GFX12: s_lshr_b32 s0, vcc_lo, s2               ; encoding: [0x6a,0x02,0x00,0x85]
 
 s_lshr_b32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x85]
+// GFX12: s_lshr_b32 s0, vcc_hi, s2               ; encoding: [0x6b,0x02,0x00,0x85]
 
 s_lshr_b32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x85]
+// GFX12: s_lshr_b32 s0, m0, s2                   ; encoding: [0x7d,0x02,0x00,0x85]
 
 s_lshr_b32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x85]
+// GFX12: s_lshr_b32 s0, 0, s2                    ; encoding: [0x80,0x02,0x00,0x85]
 
 s_lshr_b32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x85]
+// GFX12: s_lshr_b32 s0, -1, s2                   ; encoding: [0xc1,0x02,0x00,0x85]
 
 s_lshr_b32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x85]
+// GFX12: s_lshr_b32 s0, 0.5, s2                  ; encoding: [0xf0,0x02,0x00,0x85]
 
 s_lshr_b32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x85]
+// GFX12: s_lshr_b32 s0, -4.0, s2                 ; encoding: [0xf7,0x02,0x00,0x85]
 
 s_lshr_b32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x85,0x73,0x72,0x71,0x3f]
+// GFX12: s_lshr_b32 s0, 0x3f717273, s2           ; encoding: [0xff,0x02,0x00,0x85,0x73,0x72,0x71,0x3f]
 
 s_lshr_b32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x85,0x56,0x34,0x12,0xaf]
+// GFX12: s_lshr_b32 s0, 0xaf123456, s2           ; encoding: [0xff,0x02,0x00,0x85,0x56,0x34,0x12,0xaf]
 
 s_lshr_b32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x85]
+// GFX12: s_lshr_b32 s0, s1, exec_lo              ; encoding: [0x01,0x7e,0x00,0x85]
 
 s_lshr_b32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x85]
+// GFX12: s_lshr_b32 s0, s1, exec_hi              ; encoding: [0x01,0x7f,0x00,0x85]
 
 s_lshr_b32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x85]
+// GFX12: s_lshr_b32 s0, s1, vcc_lo               ; encoding: [0x01,0x6a,0x00,0x85]
 
 s_lshr_b32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x85]
+// GFX12: s_lshr_b32 s0, s1, vcc_hi               ; encoding: [0x01,0x6b,0x00,0x85]
 
 s_lshr_b32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x85]
+// GFX12: s_lshr_b32 s0, s1, m0                   ; encoding: [0x01,0x7d,0x00,0x85]
 
 s_lshr_b32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x85]
+// GFX12: s_lshr_b32 s0, s1, 0                    ; encoding: [0x01,0x80,0x00,0x85]
 
 s_lshr_b32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x85]
+// GFX12: s_lshr_b32 s0, s1, -1                   ; encoding: [0x01,0xc1,0x00,0x85]
 
 s_lshr_b32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x85]
+// GFX12: s_lshr_b32 s0, s1, 0.5                  ; encoding: [0x01,0xf0,0x00,0x85]
 
 s_lshr_b32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x85]
+// GFX12: s_lshr_b32 s0, s1, -4.0                 ; encoding: [0x01,0xf7,0x00,0x85]
 
 s_lshr_b32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x85,0x73,0x72,0x71,0x3f]
+// GFX12: s_lshr_b32 s0, s1, 0x3f717273           ; encoding: [0x01,0xff,0x00,0x85,0x73,0x72,0x71,0x3f]
 
 s_lshr_b32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x85,0x56,0x34,0x12,0xaf]
+// GFX12: s_lshr_b32 s0, s1, 0xaf123456           ; encoding: [0x01,0xff,0x00,0x85,0x56,0x34,0x12,0xaf]
 
 s_lshr_b64 s[0:1], s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0x80,0x85]
+// GFX12: s_lshr_b64 s[0:1], s[2:3], s4           ; encoding: [0x02,0x04,0x80,0x85]
 
 s_lshr_b64 s[104:105], s[102:103], s100
-// GFX12: encoding: [0x66,0x64,0xe8,0x85]
+// GFX12: s_lshr_b64 s[104:105], s[102:103], s100 ; encoding: [0x66,0x64,0xe8,0x85]
 
 s_lshr_b64 s[0:1], s[102:103], s100
-// GFX12: encoding: [0x66,0x64,0x80,0x85]
+// GFX12: s_lshr_b64 s[0:1], s[102:103], s100     ; encoding: [0x66,0x64,0x80,0x85]
 
 s_lshr_b64 s[104:105], s[2:3], s100
-// GFX12: encoding: [0x02,0x64,0xe8,0x85]
+// GFX12: s_lshr_b64 s[104:105], s[2:3], s100     ; encoding: [0x02,0x64,0xe8,0x85]
 
 s_lshr_b64 s[104:105], s[102:103], s4
-// GFX12: encoding: [0x66,0x04,0xe8,0x85]
+// GFX12: s_lshr_b64 s[104:105], s[102:103], s4   ; encoding: [0x66,0x04,0xe8,0x85]
 
 s_lshr_b64 s[104:105], s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0xe8,0x85]
+// GFX12: s_lshr_b64 s[104:105], s[2:3], s4       ; encoding: [0x02,0x04,0xe8,0x85]
 
 s_lshr_b64 s[0:1], s[102:103], s4
-// GFX12: encoding: [0x66,0x04,0x80,0x85]
+// GFX12: s_lshr_b64 s[0:1], s[102:103], s4       ; encoding: [0x66,0x04,0x80,0x85]
 
 s_lshr_b64 s[0:1], s[2:3], s100
-// GFX12: encoding: [0x02,0x64,0x80,0x85]
+// GFX12: s_lshr_b64 s[0:1], s[2:3], s100         ; encoding: [0x02,0x64,0x80,0x85]
 
 s_lshr_b64 exec, s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0xfe,0x85]
+// GFX12: s_lshr_b64 exec, s[2:3], s4             ; encoding: [0x02,0x04,0xfe,0x85]
 
 s_lshr_b64 vcc, s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0xea,0x85]
+// GFX12: s_lshr_b64 vcc, s[2:3], s4              ; encoding: [0x02,0x04,0xea,0x85]
 
 s_lshr_b64 s[0:1], exec, s4
-// GFX12: encoding: [0x7e,0x04,0x80,0x85]
+// GFX12: s_lshr_b64 s[0:1], exec, s4             ; encoding: [0x7e,0x04,0x80,0x85]
 
 s_lshr_b64 s[0:1], vcc, s4
-// GFX12: encoding: [0x6a,0x04,0x80,0x85]
+// GFX12: s_lshr_b64 s[0:1], vcc, s4              ; encoding: [0x6a,0x04,0x80,0x85]
 
 s_lshr_b64 s[0:1], 0, s4
-// GFX12: encoding: [0x80,0x04,0x80,0x85]
+// GFX12: s_lshr_b64 s[0:1], 0, s4                ; encoding: [0x80,0x04,0x80,0x85]
 
 s_lshr_b64 s[0:1], -1, s4
-// GFX12: encoding: [0xc1,0x04,0x80,0x85]
+// GFX12: s_lshr_b64 s[0:1], -1, s4               ; encoding: [0xc1,0x04,0x80,0x85]
 
 s_lshr_b64 s[0:1], 0.5, s4
-// GFX12: encoding: [0xf0,0x04,0x80,0x85]
+// GFX12: s_lshr_b64 s[0:1], 0.5, s4              ; encoding: [0xf0,0x04,0x80,0x85]
 
 s_lshr_b64 s[0:1], -4.0, s4
-// GFX12: encoding: [0xf7,0x04,0x80,0x85]
+// GFX12: s_lshr_b64 s[0:1], -4.0, s4             ; encoding: [0xf7,0x04,0x80,0x85]
 
 s_lshr_b64 s[0:1], 0x3f717273, s4
-// GFX12: encoding: [0xff,0x04,0x80,0x85,0x73,0x72,0x71,0x3f]
+// GFX12: s_lshr_b64 s[0:1], 0x3f717273, s4       ; encoding: [0xff,0x04,0x80,0x85,0x73,0x72,0x71,0x3f]
 
 s_lshr_b64 s[0:1], 0xaf123456, s4
-// GFX12: encoding: [0xff,0x04,0x80,0x85,0x56,0x34,0x12,0xaf]
+// GFX12: s_lshr_b64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x85,0x56,0x34,0x12,0xaf]
 
 s_lshr_b64 s[0:1], s[2:3], exec_lo
-// GFX12: encoding: [0x02,0x7e,0x80,0x85]
+// GFX12: s_lshr_b64 s[0:1], s[2:3], exec_lo      ; encoding: [0x02,0x7e,0x80,0x85]
 
 s_lshr_b64 s[0:1], s[2:3], vcc_lo
-// GFX12: encoding: [0x02,0x6a,0x80,0x85]
+// GFX12: s_lshr_b64 s[0:1], s[2:3], vcc_lo       ; encoding: [0x02,0x6a,0x80,0x85]
 
 s_lshr_b64 s[0:1], s[2:3], 0
-// GFX12: encoding: [0x02,0x80,0x80,0x85]
+// GFX12: s_lshr_b64 s[0:1], s[2:3], 0            ; encoding: [0x02,0x80,0x80,0x85]
 
 s_lshr_b64 s[0:1], s[2:3], -1
-// GFX12: encoding: [0x02,0xc1,0x80,0x85]
+// GFX12: s_lshr_b64 s[0:1], s[2:3], -1           ; encoding: [0x02,0xc1,0x80,0x85]
 
 s_lshr_b64 s[0:1], s[2:3], 0.5
-// GFX12: encoding: [0x02,0xf0,0x80,0x85]
+// GFX12: s_lshr_b64 s[0:1], s[2:3], 0.5          ; encoding: [0x02,0xf0,0x80,0x85]
 
 s_lshr_b64 s[0:1], s[2:3], -4.0
-// GFX12: encoding: [0x02,0xf7,0x80,0x85]
+// GFX12: s_lshr_b64 s[0:1], s[2:3], -4.0         ; encoding: [0x02,0xf7,0x80,0x85]
 
 s_lshr_b64 s[0:1], s[2:3], 0x3f717273
-// GFX12: encoding: [0x02,0xff,0x80,0x85,0x73,0x72,0x71,0x3f]
+// GFX12: s_lshr_b64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x85,0x73,0x72,0x71,0x3f]
 
 s_lshr_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: encoding: [0x02,0xff,0x80,0x85,0x56,0x34,0x12,0xaf]
+// GFX12: s_lshr_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x85,0x56,0x34,0x12,0xaf]
 
 s_ashr_i32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x86]
+// GFX12: s_ashr_i32 s0, s1, s2                   ; encoding: [0x01,0x02,0x00,0x86]
 
 s_ashr_i32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x86]
+// GFX12: s_ashr_i32 s105, s104, s103             ; encoding: [0x68,0x67,0x69,0x86]
 
 s_ashr_i32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x86]
+// GFX12: s_ashr_i32 s0, s104, s103               ; encoding: [0x68,0x67,0x00,0x86]
 
 s_ashr_i32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x86]
+// GFX12: s_ashr_i32 s105, s1, s103               ; encoding: [0x01,0x67,0x69,0x86]
 
 s_ashr_i32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x86]
+// GFX12: s_ashr_i32 s105, s104, s2               ; encoding: [0x68,0x02,0x69,0x86]
 
 s_ashr_i32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x86]
+// GFX12: s_ashr_i32 s105, s1, s2                 ; encoding: [0x01,0x02,0x69,0x86]
 
 s_ashr_i32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x86]
+// GFX12: s_ashr_i32 s0, s104, s2                 ; encoding: [0x68,0x02,0x00,0x86]
 
 s_ashr_i32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x86]
+// GFX12: s_ashr_i32 s0, s1, s103                 ; encoding: [0x01,0x67,0x00,0x86]
 
 s_ashr_i32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x86]
+// GFX12: s_ashr_i32 exec_lo, s1, s2              ; encoding: [0x01,0x02,0x7e,0x86]
 
 s_ashr_i32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x86]
+// GFX12: s_ashr_i32 exec_hi, s1, s2              ; encoding: [0x01,0x02,0x7f,0x86]
 
 s_ashr_i32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x86]
+// GFX12: s_ashr_i32 vcc_lo, s1, s2               ; encoding: [0x01,0x02,0x6a,0x86]
 
 s_ashr_i32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x86]
+// GFX12: s_ashr_i32 vcc_hi, s1, s2               ; encoding: [0x01,0x02,0x6b,0x86]
 
 s_ashr_i32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x86]
+// GFX12: s_ashr_i32 m0, s1, s2                   ; encoding: [0x01,0x02,0x7d,0x86]
 
 s_ashr_i32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x86]
+// GFX12: s_ashr_i32 s0, exec_lo, s2              ; encoding: [0x7e,0x02,0x00,0x86]
 
 s_ashr_i32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x86]
+// GFX12: s_ashr_i32 s0, exec_hi, s2              ; encoding: [0x7f,0x02,0x00,0x86]
 
 s_ashr_i32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x86]
+// GFX12: s_ashr_i32 s0, vcc_lo, s2               ; encoding: [0x6a,0x02,0x00,0x86]
 
 s_ashr_i32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x86]
+// GFX12: s_ashr_i32 s0, vcc_hi, s2               ; encoding: [0x6b,0x02,0x00,0x86]
 
 s_ashr_i32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x86]
+// GFX12: s_ashr_i32 s0, m0, s2                   ; encoding: [0x7d,0x02,0x00,0x86]
 
 s_ashr_i32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x86]
+// GFX12: s_ashr_i32 s0, 0, s2                    ; encoding: [0x80,0x02,0x00,0x86]
 
 s_ashr_i32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x86]
+// GFX12: s_ashr_i32 s0, -1, s2                   ; encoding: [0xc1,0x02,0x00,0x86]
 
 s_ashr_i32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x86]
+// GFX12: s_ashr_i32 s0, 0.5, s2                  ; encoding: [0xf0,0x02,0x00,0x86]
 
 s_ashr_i32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x86]
+// GFX12: s_ashr_i32 s0, -4.0, s2                 ; encoding: [0xf7,0x02,0x00,0x86]
 
 s_ashr_i32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x86,0x73,0x72,0x71,0x3f]
+// GFX12: s_ashr_i32 s0, 0x3f717273, s2           ; encoding: [0xff,0x02,0x00,0x86,0x73,0x72,0x71,0x3f]
 
 s_ashr_i32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x86,0x56,0x34,0x12,0xaf]
+// GFX12: s_ashr_i32 s0, 0xaf123456, s2           ; encoding: [0xff,0x02,0x00,0x86,0x56,0x34,0x12,0xaf]
 
 s_ashr_i32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x86]
+// GFX12: s_ashr_i32 s0, s1, exec_lo              ; encoding: [0x01,0x7e,0x00,0x86]
 
 s_ashr_i32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x86]
+// GFX12: s_ashr_i32 s0, s1, exec_hi              ; encoding: [0x01,0x7f,0x00,0x86]
 
 s_ashr_i32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x86]
+// GFX12: s_ashr_i32 s0, s1, vcc_lo               ; encoding: [0x01,0x6a,0x00,0x86]
 
 s_ashr_i32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x86]
+// GFX12: s_ashr_i32 s0, s1, vcc_hi               ; encoding: [0x01,0x6b,0x00,0x86]
 
 s_ashr_i32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x86]
+// GFX12: s_ashr_i32 s0, s1, m0                   ; encoding: [0x01,0x7d,0x00,0x86]
 
 s_ashr_i32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x86]
+// GFX12: s_ashr_i32 s0, s1, 0                    ; encoding: [0x01,0x80,0x00,0x86]
 
 s_ashr_i32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x86]
+// GFX12: s_ashr_i32 s0, s1, -1                   ; encoding: [0x01,0xc1,0x00,0x86]
 
 s_ashr_i32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x86]
+// GFX12: s_ashr_i32 s0, s1, 0.5                  ; encoding: [0x01,0xf0,0x00,0x86]
 
 s_ashr_i32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x86]
+// GFX12: s_ashr_i32 s0, s1, -4.0                 ; encoding: [0x01,0xf7,0x00,0x86]
 
 s_ashr_i32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x86,0x73,0x72,0x71,0x3f]
+// GFX12: s_ashr_i32 s0, s1, 0x3f717273           ; encoding: [0x01,0xff,0x00,0x86,0x73,0x72,0x71,0x3f]
 
 s_ashr_i32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x86,0x56,0x34,0x12,0xaf]
+// GFX12: s_ashr_i32 s0, s1, 0xaf123456           ; encoding: [0x01,0xff,0x00,0x86,0x56,0x34,0x12,0xaf]
 
 s_ashr_i64 s[0:1], s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0x80,0x86]
+// GFX12: s_ashr_i64 s[0:1], s[2:3], s4           ; encoding: [0x02,0x04,0x80,0x86]
 
 s_ashr_i64 s[104:105], s[102:103], s100
-// GFX12: encoding: [0x66,0x64,0xe8,0x86]
+// GFX12: s_ashr_i64 s[104:105], s[102:103], s100 ; encoding: [0x66,0x64,0xe8,0x86]
 
 s_ashr_i64 s[0:1], s[102:103], s100
-// GFX12: encoding: [0x66,0x64,0x80,0x86]
+// GFX12: s_ashr_i64 s[0:1], s[102:103], s100     ; encoding: [0x66,0x64,0x80,0x86]
 
 s_ashr_i64 s[104:105], s[2:3], s100
-// GFX12: encoding: [0x02,0x64,0xe8,0x86]
+// GFX12: s_ashr_i64 s[104:105], s[2:3], s100     ; encoding: [0x02,0x64,0xe8,0x86]
 
 s_ashr_i64 s[104:105], s[102:103], s4
-// GFX12: encoding: [0x66,0x04,0xe8,0x86]
+// GFX12: s_ashr_i64 s[104:105], s[102:103], s4   ; encoding: [0x66,0x04,0xe8,0x86]
 
 s_ashr_i64 s[104:105], s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0xe8,0x86]
+// GFX12: s_ashr_i64 s[104:105], s[2:3], s4       ; encoding: [0x02,0x04,0xe8,0x86]
 
 s_ashr_i64 s[0:1], s[102:103], s4
-// GFX12: encoding: [0x66,0x04,0x80,0x86]
+// GFX12: s_ashr_i64 s[0:1], s[102:103], s4       ; encoding: [0x66,0x04,0x80,0x86]
 
 s_ashr_i64 s[0:1], s[2:3], s100
-// GFX12: encoding: [0x02,0x64,0x80,0x86]
+// GFX12: s_ashr_i64 s[0:1], s[2:3], s100         ; encoding: [0x02,0x64,0x80,0x86]
 
 s_ashr_i64 exec, s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0xfe,0x86]
+// GFX12: s_ashr_i64 exec, s[2:3], s4             ; encoding: [0x02,0x04,0xfe,0x86]
 
 s_ashr_i64 vcc, s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0xea,0x86]
+// GFX12: s_ashr_i64 vcc, s[2:3], s4              ; encoding: [0x02,0x04,0xea,0x86]
 
 s_ashr_i64 s[0:1], exec, s4
-// GFX12: encoding: [0x7e,0x04,0x80,0x86]
+// GFX12: s_ashr_i64 s[0:1], exec, s4             ; encoding: [0x7e,0x04,0x80,0x86]
 
 s_ashr_i64 s[0:1], vcc, s4
-// GFX12: encoding: [0x6a,0x04,0x80,0x86]
+// GFX12: s_ashr_i64 s[0:1], vcc, s4              ; encoding: [0x6a,0x04,0x80,0x86]
 
 s_ashr_i64 s[0:1], 0, s4
-// GFX12: encoding: [0x80,0x04,0x80,0x86]
+// GFX12: s_ashr_i64 s[0:1], 0, s4                ; encoding: [0x80,0x04,0x80,0x86]
 
 s_ashr_i64 s[0:1], -1, s4
-// GFX12: encoding: [0xc1,0x04,0x80,0x86]
+// GFX12: s_ashr_i64 s[0:1], -1, s4               ; encoding: [0xc1,0x04,0x80,0x86]
 
 s_ashr_i64 s[0:1], 0.5, s4
-// GFX12: encoding: [0xf0,0x04,0x80,0x86]
+// GFX12: s_ashr_i64 s[0:1], 0.5, s4              ; encoding: [0xf0,0x04,0x80,0x86]
 
 s_ashr_i64 s[0:1], -4.0, s4
-// GFX12: encoding: [0xf7,0x04,0x80,0x86]
+// GFX12: s_ashr_i64 s[0:1], -4.0, s4             ; encoding: [0xf7,0x04,0x80,0x86]
 
 s_ashr_i64 s[0:1], 0x3f717273, s4
-// GFX12: encoding: [0xff,0x04,0x80,0x86,0x73,0x72,0x71,0x3f]
+// GFX12: s_ashr_i64 s[0:1], 0x3f717273, s4       ; encoding: [0xff,0x04,0x80,0x86,0x73,0x72,0x71,0x3f]
 
 s_ashr_i64 s[0:1], 0xaf123456, s4
-// GFX12: encoding: [0xff,0x04,0x80,0x86,0x56,0x34,0x12,0xaf]
+// GFX12: s_ashr_i64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x86,0x56,0x34,0x12,0xaf]
 
 s_ashr_i64 s[0:1], s[2:3], exec_lo
-// GFX12: encoding: [0x02,0x7e,0x80,0x86]
+// GFX12: s_ashr_i64 s[0:1], s[2:3], exec_lo      ; encoding: [0x02,0x7e,0x80,0x86]
 
 s_ashr_i64 s[0:1], s[2:3], vcc_lo
-// GFX12: encoding: [0x02,0x6a,0x80,0x86]
+// GFX12: s_ashr_i64 s[0:1], s[2:3], vcc_lo       ; encoding: [0x02,0x6a,0x80,0x86]
 
 s_ashr_i64 s[0:1], s[2:3], 0
-// GFX12: encoding: [0x02,0x80,0x80,0x86]
+// GFX12: s_ashr_i64 s[0:1], s[2:3], 0            ; encoding: [0x02,0x80,0x80,0x86]
 
 s_ashr_i64 s[0:1], s[2:3], -1
-// GFX12: encoding: [0x02,0xc1,0x80,0x86]
+// GFX12: s_ashr_i64 s[0:1], s[2:3], -1           ; encoding: [0x02,0xc1,0x80,0x86]
 
 s_ashr_i64 s[0:1], s[2:3], 0.5
-// GFX12: encoding: [0x02,0xf0,0x80,0x86]
+// GFX12: s_ashr_i64 s[0:1], s[2:3], 0.5          ; encoding: [0x02,0xf0,0x80,0x86]
 
 s_ashr_i64 s[0:1], s[2:3], -4.0
-// GFX12: encoding: [0x02,0xf7,0x80,0x86]
+// GFX12: s_ashr_i64 s[0:1], s[2:3], -4.0         ; encoding: [0x02,0xf7,0x80,0x86]
 
 s_ashr_i64 s[0:1], s[2:3], 0x3f717273
-// GFX12: encoding: [0x02,0xff,0x80,0x86,0x73,0x72,0x71,0x3f]
+// GFX12: s_ashr_i64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x86,0x73,0x72,0x71,0x3f]
 
 s_ashr_i64 s[0:1], s[2:3], 0xaf123456
-// GFX12: encoding: [0x02,0xff,0x80,0x86,0x56,0x34,0x12,0xaf]
+// GFX12: s_ashr_i64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x86,0x56,0x34,0x12,0xaf]
 
 s_bfm_b32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x95]
+// GFX12: s_bfm_b32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x95]
 
 s_bfm_b32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x95]
+// GFX12: s_bfm_b32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x95]
 
 s_bfm_b32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x95]
+// GFX12: s_bfm_b32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x95]
 
 s_bfm_b32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x95]
+// GFX12: s_bfm_b32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x95]
 
 s_bfm_b32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x95]
+// GFX12: s_bfm_b32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x95]
 
 s_bfm_b32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x95]
+// GFX12: s_bfm_b32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x95]
 
 s_bfm_b32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x95]
+// GFX12: s_bfm_b32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x95]
 
 s_bfm_b32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x95]
+// GFX12: s_bfm_b32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x95]
 
 s_bfm_b32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x95]
+// GFX12: s_bfm_b32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x95]
 
 s_bfm_b32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x95]
+// GFX12: s_bfm_b32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x95]
 
 s_bfm_b32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x95]
+// GFX12: s_bfm_b32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x95]
 
 s_bfm_b32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x95]
+// GFX12: s_bfm_b32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x95]
 
 s_bfm_b32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x95]
+// GFX12: s_bfm_b32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x95]
 
 s_bfm_b32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x95]
+// GFX12: s_bfm_b32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x95]
 
 s_bfm_b32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x95]
+// GFX12: s_bfm_b32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x95]
 
 s_bfm_b32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x95]
+// GFX12: s_bfm_b32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x95]
 
 s_bfm_b32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x95]
+// GFX12: s_bfm_b32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x95]
 
 s_bfm_b32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x95]
+// GFX12: s_bfm_b32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x95]
 
 s_bfm_b32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x95]
+// GFX12: s_bfm_b32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x95]
 
 s_bfm_b32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x95]
+// GFX12: s_bfm_b32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x95]
 
 s_bfm_b32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x95]
+// GFX12: s_bfm_b32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x95]
 
 s_bfm_b32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x95]
+// GFX12: s_bfm_b32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x95]
 
 s_bfm_b32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x95,0x73,0x72,0x71,0x3f]
+// GFX12: s_bfm_b32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x95,0x73,0x72,0x71,0x3f]
 
 s_bfm_b32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x95,0x56,0x34,0x12,0xaf]
+// GFX12: s_bfm_b32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x95,0x56,0x34,0x12,0xaf]
 
 s_bfm_b32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x95]
+// GFX12: s_bfm_b32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x95]
 
 s_bfm_b32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x95]
+// GFX12: s_bfm_b32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x95]
 
 s_bfm_b32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x95]
+// GFX12: s_bfm_b32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x95]
 
 s_bfm_b32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x95]
+// GFX12: s_bfm_b32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x95]
 
 s_bfm_b32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x95]
+// GFX12: s_bfm_b32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x95]
 
 s_bfm_b32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x95]
+// GFX12: s_bfm_b32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x95]
 
 s_bfm_b32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x95]
+// GFX12: s_bfm_b32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x95]
 
 s_bfm_b32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x95]
+// GFX12: s_bfm_b32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x95]
 
 s_bfm_b32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x95]
+// GFX12: s_bfm_b32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x95]
 
 s_bfm_b32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x95,0x73,0x72,0x71,0x3f]
+// GFX12: s_bfm_b32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x95,0x73,0x72,0x71,0x3f]
 
 s_bfm_b32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x95,0x56,0x34,0x12,0xaf]
+// GFX12: s_bfm_b32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x95,0x56,0x34,0x12,0xaf]
 
 s_bfm_b64 s[0:1], s2, s3
-// GFX12: encoding: [0x02,0x03,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], s2, s3                ; encoding: [0x02,0x03,0x80,0x95]
 
 s_bfm_b64 s[104:105], s103, s102
-// GFX12: encoding: [0x67,0x66,0xe8,0x95]
+// GFX12: s_bfm_b64 s[104:105], s103, s102        ; encoding: [0x67,0x66,0xe8,0x95]
 
 s_bfm_b64 s[0:1], s104, s103
-// GFX12: encoding: [0x68,0x67,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], s104, s103            ; encoding: [0x68,0x67,0x80,0x95]
 
 s_bfm_b64 s[104:105], s2, s103
-// GFX12: encoding: [0x02,0x67,0xe8,0x95]
+// GFX12: s_bfm_b64 s[104:105], s2, s103          ; encoding: [0x02,0x67,0xe8,0x95]
 
 s_bfm_b64 s[104:105], s104, s3
-// GFX12: encoding: [0x68,0x03,0xe8,0x95]
+// GFX12: s_bfm_b64 s[104:105], s104, s3          ; encoding: [0x68,0x03,0xe8,0x95]
 
 s_bfm_b64 s[104:105], s2, s3
-// GFX12: encoding: [0x02,0x03,0xe8,0x95]
+// GFX12: s_bfm_b64 s[104:105], s2, s3            ; encoding: [0x02,0x03,0xe8,0x95]
 
 s_bfm_b64 s[0:1], s104, s3
-// GFX12: encoding: [0x68,0x03,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], s104, s3              ; encoding: [0x68,0x03,0x80,0x95]
 
 s_bfm_b64 s[0:1], s2, s103
-// GFX12: encoding: [0x02,0x67,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], s2, s103              ; encoding: [0x02,0x67,0x80,0x95]
 
 s_bfm_b64 exec, s2, s3
-// GFX12: encoding: [0x02,0x03,0xfe,0x95]
+// GFX12: s_bfm_b64 exec, s2, s3                  ; encoding: [0x02,0x03,0xfe,0x95]
 
 s_bfm_b64 vcc, s2, s3
-// GFX12: encoding: [0x02,0x03,0xea,0x95]
+// GFX12: s_bfm_b64 vcc, s2, s3                   ; encoding: [0x02,0x03,0xea,0x95]
 
 s_bfm_b64 s[0:1], exec_lo, s3
-// GFX12: encoding: [0x7e,0x03,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], exec_lo, s3           ; encoding: [0x7e,0x03,0x80,0x95]
 
 s_bfm_b64 s[0:1], exec_hi, s3
-// GFX12: encoding: [0x7f,0x03,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], exec_hi, s3           ; encoding: [0x7f,0x03,0x80,0x95]
 
 s_bfm_b64 s[0:1], vcc_lo, s3
-// GFX12: encoding: [0x6a,0x03,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], vcc_lo, s3            ; encoding: [0x6a,0x03,0x80,0x95]
 
 s_bfm_b64 s[0:1], vcc_hi, s3
-// GFX12: encoding: [0x6b,0x03,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], vcc_hi, s3            ; encoding: [0x6b,0x03,0x80,0x95]
 
 s_bfm_b64 s[0:1], m0, s3
-// GFX12: encoding: [0x7d,0x03,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], m0, s3                ; encoding: [0x7d,0x03,0x80,0x95]
 
 s_bfm_b64 s[0:1], 0, s3
-// GFX12: encoding: [0x80,0x03,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], 0, s3                 ; encoding: [0x80,0x03,0x80,0x95]
 
 s_bfm_b64 s[0:1], -1, s3
-// GFX12: encoding: [0xc1,0x03,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], -1, s3                ; encoding: [0xc1,0x03,0x80,0x95]
 
 s_bfm_b64 s[0:1], 0.5, s3
-// GFX12: encoding: [0xf0,0x03,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], 0.5, s3               ; encoding: [0xf0,0x03,0x80,0x95]
 
 s_bfm_b64 s[0:1], -4.0, s3
-// GFX12: encoding: [0xf7,0x03,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], -4.0, s3              ; encoding: [0xf7,0x03,0x80,0x95]
 
 s_bfm_b64 s[0:1], 0x3f717273, s3
-// GFX12: encoding: [0xff,0x03,0x80,0x95,0x73,0x72,0x71,0x3f]
+// GFX12: s_bfm_b64 s[0:1], 0x3f717273, s3        ; encoding: [0xff,0x03,0x80,0x95,0x73,0x72,0x71,0x3f]
 
 s_bfm_b64 s[0:1], 0xaf123456, s3
-// GFX12: encoding: [0xff,0x03,0x80,0x95,0x56,0x34,0x12,0xaf]
+// GFX12: s_bfm_b64 s[0:1], 0xaf123456, s3        ; encoding: [0xff,0x03,0x80,0x95,0x56,0x34,0x12,0xaf]
 
 s_bfm_b64 s[0:1], s2, exec_lo
-// GFX12: encoding: [0x02,0x7e,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], s2, exec_lo           ; encoding: [0x02,0x7e,0x80,0x95]
 
 s_bfm_b64 s[0:1], s2, exec_hi
-// GFX12: encoding: [0x02,0x7f,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], s2, exec_hi           ; encoding: [0x02,0x7f,0x80,0x95]
 
 s_bfm_b64 s[0:1], s2, vcc_lo
-// GFX12: encoding: [0x02,0x6a,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], s2, vcc_lo            ; encoding: [0x02,0x6a,0x80,0x95]
 
 s_bfm_b64 s[0:1], s2, vcc_hi
-// GFX12: encoding: [0x02,0x6b,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], s2, vcc_hi            ; encoding: [0x02,0x6b,0x80,0x95]
 
 s_bfm_b64 s[0:1], s2, m0
-// GFX12: encoding: [0x02,0x7d,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], s2, m0                ; encoding: [0x02,0x7d,0x80,0x95]
 
 s_bfm_b64 s[0:1], s2, 0
-// GFX12: encoding: [0x02,0x80,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], s2, 0                 ; encoding: [0x02,0x80,0x80,0x95]
 
 s_bfm_b64 s[0:1], s2, -1
-// GFX12: encoding: [0x02,0xc1,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], s2, -1                ; encoding: [0x02,0xc1,0x80,0x95]
 
 s_bfm_b64 s[0:1], s2, 0.5
-// GFX12: encoding: [0x02,0xf0,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], s2, 0.5               ; encoding: [0x02,0xf0,0x80,0x95]
 
 s_bfm_b64 s[0:1], s2, -4.0
-// GFX12: encoding: [0x02,0xf7,0x80,0x95]
+// GFX12: s_bfm_b64 s[0:1], s2, -4.0              ; encoding: [0x02,0xf7,0x80,0x95]
 
 s_bfm_b64 s[0:1], s2, 0x3f717273
-// GFX12: encoding: [0x02,0xff,0x80,0x95,0x73,0x72,0x71,0x3f]
+// GFX12: s_bfm_b64 s[0:1], s2, 0x3f717273        ; encoding: [0x02,0xff,0x80,0x95,0x73,0x72,0x71,0x3f]
 
 s_bfm_b64 s[0:1], s2, 0xaf123456
-// GFX12: encoding: [0x02,0xff,0x80,0x95,0x56,0x34,0x12,0xaf]
+// GFX12: s_bfm_b64 s[0:1], s2, 0xaf123456        ; encoding: [0x02,0xff,0x80,0x95,0x56,0x34,0x12,0xaf]
 
 s_mul_i32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x96]
+// GFX12: s_mul_i32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x96]
 
 s_mul_i32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x96]
+// GFX12: s_mul_i32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x96]
 
 s_mul_i32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x96]
+// GFX12: s_mul_i32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x96]
 
 s_mul_i32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x96]
+// GFX12: s_mul_i32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x96]
 
 s_mul_i32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x96]
+// GFX12: s_mul_i32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x96]
 
 s_mul_i32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x96]
+// GFX12: s_mul_i32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x96]
 
 s_mul_i32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x96]
+// GFX12: s_mul_i32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x96]
 
 s_mul_i32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x96]
+// GFX12: s_mul_i32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x96]
 
 s_mul_i32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x96]
+// GFX12: s_mul_i32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x96]
 
 s_mul_i32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x96]
+// GFX12: s_mul_i32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x96]
 
 s_mul_i32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x96]
+// GFX12: s_mul_i32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x96]
 
 s_mul_i32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x96]
+// GFX12: s_mul_i32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x96]
 
 s_mul_i32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x96]
+// GFX12: s_mul_i32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x96]
 
 s_mul_i32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x96]
+// GFX12: s_mul_i32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x96]
 
 s_mul_i32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x96]
+// GFX12: s_mul_i32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x96]
 
 s_mul_i32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x96]
+// GFX12: s_mul_i32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x96]
 
 s_mul_i32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x96]
+// GFX12: s_mul_i32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x96]
 
 s_mul_i32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x96]
+// GFX12: s_mul_i32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x96]
 
 s_mul_i32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x96]
+// GFX12: s_mul_i32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x96]
 
 s_mul_i32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x96]
+// GFX12: s_mul_i32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x96]
 
 s_mul_i32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x96]
+// GFX12: s_mul_i32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x96]
 
 s_mul_i32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x96]
+// GFX12: s_mul_i32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x96]
 
 s_mul_i32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x96,0x73,0x72,0x71,0x3f]
+// GFX12: s_mul_i32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x96,0x73,0x72,0x71,0x3f]
 
 s_mul_i32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x96,0x56,0x34,0x12,0xaf]
+// GFX12: s_mul_i32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x96,0x56,0x34,0x12,0xaf]
 
 s_mul_i32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x96]
+// GFX12: s_mul_i32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x96]
 
 s_mul_i32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x96]
+// GFX12: s_mul_i32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x96]
 
 s_mul_i32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x96]
+// GFX12: s_mul_i32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x96]
 
 s_mul_i32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x96]
+// GFX12: s_mul_i32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x96]
 
 s_mul_i32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x96]
+// GFX12: s_mul_i32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x96]
 
 s_mul_i32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x96]
+// GFX12: s_mul_i32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x96]
 
 s_mul_i32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x96]
+// GFX12: s_mul_i32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x96]
 
 s_mul_i32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x96]
+// GFX12: s_mul_i32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x96]
 
 s_mul_i32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x96]
+// GFX12: s_mul_i32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x96]
 
 s_mul_i32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x96,0x73,0x72,0x71,0x3f]
+// GFX12: s_mul_i32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x96,0x73,0x72,0x71,0x3f]
 
 s_mul_i32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x96,0x56,0x34,0x12,0xaf]
+// GFX12: s_mul_i32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x96,0x56,0x34,0x12,0xaf]
 
 s_bfe_u32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x93]
+// GFX12: s_bfe_u32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x93]
 
 s_bfe_u32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x93]
+// GFX12: s_bfe_u32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x93]
 
 s_bfe_u32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x93]
+// GFX12: s_bfe_u32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x93]
 
 s_bfe_u32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x93]
+// GFX12: s_bfe_u32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x93]
 
 s_bfe_u32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x93]
+// GFX12: s_bfe_u32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x93]
 
 s_bfe_u32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x93]
+// GFX12: s_bfe_u32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x93]
 
 s_bfe_u32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x93]
+// GFX12: s_bfe_u32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x93]
 
 s_bfe_u32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x93]
+// GFX12: s_bfe_u32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x93]
 
 s_bfe_u32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x93]
+// GFX12: s_bfe_u32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x93]
 
 s_bfe_u32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x93]
+// GFX12: s_bfe_u32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x93]
 
 s_bfe_u32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x93]
+// GFX12: s_bfe_u32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x93]
 
 s_bfe_u32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x93]
+// GFX12: s_bfe_u32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x93]
 
 s_bfe_u32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x93]
+// GFX12: s_bfe_u32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x93]
 
 s_bfe_u32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x93]
+// GFX12: s_bfe_u32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x93]
 
 s_bfe_u32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x93]
+// GFX12: s_bfe_u32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x93]
 
 s_bfe_u32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x93]
+// GFX12: s_bfe_u32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x93]
 
 s_bfe_u32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x93]
+// GFX12: s_bfe_u32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x93]
 
 s_bfe_u32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x93]
+// GFX12: s_bfe_u32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x93]
 
 s_bfe_u32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x93]
+// GFX12: s_bfe_u32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x93]
 
 s_bfe_u32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x93]
+// GFX12: s_bfe_u32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x93]
 
 s_bfe_u32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x93]
+// GFX12: s_bfe_u32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x93]
 
 s_bfe_u32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x93]
+// GFX12: s_bfe_u32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x93]
 
 s_bfe_u32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x93,0x73,0x72,0x71,0x3f]
+// GFX12: s_bfe_u32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x93,0x73,0x72,0x71,0x3f]
 
 s_bfe_u32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x93,0x56,0x34,0x12,0xaf]
+// GFX12: s_bfe_u32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x93,0x56,0x34,0x12,0xaf]
 
 s_bfe_u32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x93]
+// GFX12: s_bfe_u32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x93]
 
 s_bfe_u32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x93]
+// GFX12: s_bfe_u32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x93]
 
 s_bfe_u32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x93]
+// GFX12: s_bfe_u32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x93]
 
 s_bfe_u32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x93]
+// GFX12: s_bfe_u32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x93]
 
 s_bfe_u32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x93]
+// GFX12: s_bfe_u32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x93]
 
 s_bfe_u32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x93]
+// GFX12: s_bfe_u32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x93]
 
 s_bfe_u32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x93]
+// GFX12: s_bfe_u32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x93]
 
 s_bfe_u32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x93]
+// GFX12: s_bfe_u32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x93]
 
 s_bfe_u32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x93]
+// GFX12: s_bfe_u32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x93]
 
 s_bfe_u32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x93,0x73,0x72,0x71,0x3f]
+// GFX12: s_bfe_u32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x93,0x73,0x72,0x71,0x3f]
 
 s_bfe_u32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x93,0x56,0x34,0x12,0xaf]
+// GFX12: s_bfe_u32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x93,0x56,0x34,0x12,0xaf]
 
 s_bfe_i32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x80,0x93]
+// GFX12: s_bfe_i32 s0, s1, s2                    ; encoding: [0x01,0x02,0x80,0x93]
 
 s_bfe_i32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0xe9,0x93]
+// GFX12: s_bfe_i32 s105, s104, s103              ; encoding: [0x68,0x67,0xe9,0x93]
 
 s_bfe_i32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x80,0x93]
+// GFX12: s_bfe_i32 s0, s104, s103                ; encoding: [0x68,0x67,0x80,0x93]
 
 s_bfe_i32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0xe9,0x93]
+// GFX12: s_bfe_i32 s105, s1, s103                ; encoding: [0x01,0x67,0xe9,0x93]
 
 s_bfe_i32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0xe9,0x93]
+// GFX12: s_bfe_i32 s105, s104, s2                ; encoding: [0x68,0x02,0xe9,0x93]
 
 s_bfe_i32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0xe9,0x93]
+// GFX12: s_bfe_i32 s105, s1, s2                  ; encoding: [0x01,0x02,0xe9,0x93]
 
 s_bfe_i32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x80,0x93]
+// GFX12: s_bfe_i32 s0, s104, s2                  ; encoding: [0x68,0x02,0x80,0x93]
 
 s_bfe_i32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x80,0x93]
+// GFX12: s_bfe_i32 s0, s1, s103                  ; encoding: [0x01,0x67,0x80,0x93]
 
 s_bfe_i32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfe,0x93]
+// GFX12: s_bfe_i32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0xfe,0x93]
 
 s_bfe_i32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xff,0x93]
+// GFX12: s_bfe_i32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0xff,0x93]
 
 s_bfe_i32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xea,0x93]
+// GFX12: s_bfe_i32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0xea,0x93]
 
 s_bfe_i32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xeb,0x93]
+// GFX12: s_bfe_i32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0xeb,0x93]
 
 s_bfe_i32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfd,0x93]
+// GFX12: s_bfe_i32 m0, s1, s2                    ; encoding: [0x01,0x02,0xfd,0x93]
 
 s_bfe_i32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x80,0x93]
+// GFX12: s_bfe_i32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x80,0x93]
 
 s_bfe_i32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x80,0x93]
+// GFX12: s_bfe_i32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x80,0x93]
 
 s_bfe_i32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x80,0x93]
+// GFX12: s_bfe_i32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x80,0x93]
 
 s_bfe_i32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x80,0x93]
+// GFX12: s_bfe_i32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x80,0x93]
 
 s_bfe_i32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x80,0x93]
+// GFX12: s_bfe_i32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x80,0x93]
 
 s_bfe_i32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x80,0x93]
+// GFX12: s_bfe_i32 s0, 0, s2                     ; encoding: [0x80,0x02,0x80,0x93]
 
 s_bfe_i32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x80,0x93]
+// GFX12: s_bfe_i32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x80,0x93]
 
 s_bfe_i32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x80,0x93]
+// GFX12: s_bfe_i32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x80,0x93]
 
 s_bfe_i32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x80,0x93]
+// GFX12: s_bfe_i32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x80,0x93]
 
 s_bfe_i32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x93,0x73,0x72,0x71,0x3f]
+// GFX12: s_bfe_i32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x80,0x93,0x73,0x72,0x71,0x3f]
 
 s_bfe_i32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x93,0x56,0x34,0x12,0xaf]
+// GFX12: s_bfe_i32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x80,0x93,0x56,0x34,0x12,0xaf]
 
 s_bfe_i32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x80,0x93]
+// GFX12: s_bfe_i32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x80,0x93]
 
 s_bfe_i32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x80,0x93]
+// GFX12: s_bfe_i32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x80,0x93]
 
 s_bfe_i32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x80,0x93]
+// GFX12: s_bfe_i32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x80,0x93]
 
 s_bfe_i32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x80,0x93]
+// GFX12: s_bfe_i32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x80,0x93]
 
 s_bfe_i32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x80,0x93]
+// GFX12: s_bfe_i32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x80,0x93]
 
 s_bfe_i32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x80,0x93]
+// GFX12: s_bfe_i32 s0, s1, 0                     ; encoding: [0x01,0x80,0x80,0x93]
 
 s_bfe_i32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x80,0x93]
+// GFX12: s_bfe_i32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x80,0x93]
 
 s_bfe_i32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x80,0x93]
+// GFX12: s_bfe_i32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x80,0x93]
 
 s_bfe_i32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x80,0x93]
+// GFX12: s_bfe_i32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x80,0x93]
 
 s_bfe_i32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x80,0x93,0x73,0x72,0x71,0x3f]
+// GFX12: s_bfe_i32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x80,0x93,0x73,0x72,0x71,0x3f]
 
 s_bfe_i32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x80,0x93,0x56,0x34,0x12,0xaf]
+// GFX12: s_bfe_i32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x80,0x93,0x56,0x34,0x12,0xaf]
 
 s_bfe_u64 s[0:1], s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0x00,0x94]
+// GFX12: s_bfe_u64 s[0:1], s[2:3], s4            ; encoding: [0x02,0x04,0x00,0x94]
 
 s_bfe_u64 s[104:105], s[102:103], s100
-// GFX12: encoding: [0x66,0x64,0x68,0x94]
+// GFX12: s_bfe_u64 s[104:105], s[102:103], s100  ; encoding: [0x66,0x64,0x68,0x94]
 
 s_bfe_u64 s[0:1], s[102:103], s100
-// GFX12: encoding: [0x66,0x64,0x00,0x94]
+// GFX12: s_bfe_u64 s[0:1], s[102:103], s100      ; encoding: [0x66,0x64,0x00,0x94]
 
 s_bfe_u64 s[104:105], s[2:3], s100
-// GFX12: encoding: [0x02,0x64,0x68,0x94]
+// GFX12: s_bfe_u64 s[104:105], s[2:3], s100      ; encoding: [0x02,0x64,0x68,0x94]
 
 s_bfe_u64 s[104:105], s[102:103], s4
-// GFX12: encoding: [0x66,0x04,0x68,0x94]
+// GFX12: s_bfe_u64 s[104:105], s[102:103], s4    ; encoding: [0x66,0x04,0x68,0x94]
 
 s_bfe_u64 s[104:105], s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0x68,0x94]
+// GFX12: s_bfe_u64 s[104:105], s[2:3], s4        ; encoding: [0x02,0x04,0x68,0x94]
 
 s_bfe_u64 s[0:1], s[102:103], s4
-// GFX12: encoding: [0x66,0x04,0x00,0x94]
+// GFX12: s_bfe_u64 s[0:1], s[102:103], s4        ; encoding: [0x66,0x04,0x00,0x94]
 
 s_bfe_u64 s[0:1], s[2:3], s100
-// GFX12: encoding: [0x02,0x64,0x00,0x94]
+// GFX12: s_bfe_u64 s[0:1], s[2:3], s100          ; encoding: [0x02,0x64,0x00,0x94]
 
 s_bfe_u64 exec, s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0x7e,0x94]
+// GFX12: s_bfe_u64 exec, s[2:3], s4              ; encoding: [0x02,0x04,0x7e,0x94]
 
 s_bfe_u64 vcc, s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0x6a,0x94]
+// GFX12: s_bfe_u64 vcc, s[2:3], s4               ; encoding: [0x02,0x04,0x6a,0x94]
 
 s_bfe_u64 s[0:1], exec, s4
-// GFX12: encoding: [0x7e,0x04,0x00,0x94]
+// GFX12: s_bfe_u64 s[0:1], exec, s4              ; encoding: [0x7e,0x04,0x00,0x94]
 
 s_bfe_u64 s[0:1], vcc, s4
-// GFX12: encoding: [0x6a,0x04,0x00,0x94]
+// GFX12: s_bfe_u64 s[0:1], vcc, s4               ; encoding: [0x6a,0x04,0x00,0x94]
 
 s_bfe_u64 s[0:1], 0, s4
-// GFX12: encoding: [0x80,0x04,0x00,0x94]
+// GFX12: s_bfe_u64 s[0:1], 0, s4                 ; encoding: [0x80,0x04,0x00,0x94]
 
 s_bfe_u64 s[0:1], -1, s4
-// GFX12: encoding: [0xc1,0x04,0x00,0x94]
+// GFX12: s_bfe_u64 s[0:1], -1, s4                ; encoding: [0xc1,0x04,0x00,0x94]
 
 s_bfe_u64 s[0:1], 0.5, s4
-// GFX12: encoding: [0xf0,0x04,0x00,0x94]
+// GFX12: s_bfe_u64 s[0:1], 0.5, s4               ; encoding: [0xf0,0x04,0x00,0x94]
 
 s_bfe_u64 s[0:1], -4.0, s4
-// GFX12: encoding: [0xf7,0x04,0x00,0x94]
+// GFX12: s_bfe_u64 s[0:1], -4.0, s4              ; encoding: [0xf7,0x04,0x00,0x94]
 
 s_bfe_u64 s[0:1], 0x3f717273, s4
-// GFX12: encoding: [0xff,0x04,0x00,0x94,0x73,0x72,0x71,0x3f]
+// GFX12: s_bfe_u64 s[0:1], 0x3f717273, s4        ; encoding: [0xff,0x04,0x00,0x94,0x73,0x72,0x71,0x3f]
 
 s_bfe_u64 s[0:1], 0xaf123456, s4
-// GFX12: encoding: [0xff,0x04,0x00,0x94,0x56,0x34,0x12,0xaf]
+// GFX12: s_bfe_u64 s[0:1], 0xaf123456, s4        ; encoding: [0xff,0x04,0x00,0x94,0x56,0x34,0x12,0xaf]
 
 s_bfe_u64 s[0:1], s[2:3], exec_lo
-// GFX12: encoding: [0x02,0x7e,0x00,0x94]
+// GFX12: s_bfe_u64 s[0:1], s[2:3], exec_lo       ; encoding: [0x02,0x7e,0x00,0x94]
 
 s_bfe_u64 s[0:1], s[2:3], vcc_lo
-// GFX12: encoding: [0x02,0x6a,0x00,0x94]
+// GFX12: s_bfe_u64 s[0:1], s[2:3], vcc_lo        ; encoding: [0x02,0x6a,0x00,0x94]
 
 s_bfe_u64 s[0:1], s[2:3], 0
-// GFX12: encoding: [0x02,0x80,0x00,0x94]
+// GFX12: s_bfe_u64 s[0:1], s[2:3], 0             ; encoding: [0x02,0x80,0x00,0x94]
 
 s_bfe_u64 s[0:1], s[2:3], -1
-// GFX12: encoding: [0x02,0xc1,0x00,0x94]
+// GFX12: s_bfe_u64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x00,0x94]
 
 s_bfe_u64 s[0:1], s[2:3], 0.5
-// GFX12: encoding: [0x02,0xf0,0x00,0x94]
+// GFX12: s_bfe_u64 s[0:1], s[2:3], 0.5           ; encoding: [0x02,0xf0,0x00,0x94]
 
 s_bfe_u64 s[0:1], s[2:3], -4.0
-// GFX12: encoding: [0x02,0xf7,0x00,0x94]
+// GFX12: s_bfe_u64 s[0:1], s[2:3], -4.0          ; encoding: [0x02,0xf7,0x00,0x94]
 
 s_bfe_u64 s[0:1], s[2:3], 0x3f717273
-// GFX12: encoding: [0x02,0xff,0x00,0x94,0x73,0x72,0x71,0x3f]
+// GFX12: s_bfe_u64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x00,0x94,0x73,0x72,0x71,0x3f]
 
 s_bfe_u64 s[0:1], s[2:3], 0xaf123456
-// GFX12: encoding: [0x02,0xff,0x00,0x94,0x56,0x34,0x12,0xaf]
+// GFX12: s_bfe_u64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x00,0x94,0x56,0x34,0x12,0xaf]
 
 s_bfe_i64 s[0:1], s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0x80,0x94]
+// GFX12: s_bfe_i64 s[0:1], s[2:3], s4            ; encoding: [0x02,0x04,0x80,0x94]
 
 s_bfe_i64 s[104:105], s[102:103], s100
-// GFX12: encoding: [0x66,0x64,0xe8,0x94]
+// GFX12: s_bfe_i64 s[104:105], s[102:103], s100  ; encoding: [0x66,0x64,0xe8,0x94]
 
 s_bfe_i64 s[0:1], s[102:103], s100
-// GFX12: encoding: [0x66,0x64,0x80,0x94]
+// GFX12: s_bfe_i64 s[0:1], s[102:103], s100      ; encoding: [0x66,0x64,0x80,0x94]
 
 s_bfe_i64 s[104:105], s[2:3], s100
-// GFX12: encoding: [0x02,0x64,0xe8,0x94]
+// GFX12: s_bfe_i64 s[104:105], s[2:3], s100      ; encoding: [0x02,0x64,0xe8,0x94]
 
 s_bfe_i64 s[104:105], s[102:103], s4
-// GFX12: encoding: [0x66,0x04,0xe8,0x94]
+// GFX12: s_bfe_i64 s[104:105], s[102:103], s4    ; encoding: [0x66,0x04,0xe8,0x94]
 
 s_bfe_i64 s[104:105], s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0xe8,0x94]
+// GFX12: s_bfe_i64 s[104:105], s[2:3], s4        ; encoding: [0x02,0x04,0xe8,0x94]
 
 s_bfe_i64 s[0:1], s[102:103], s4
-// GFX12: encoding: [0x66,0x04,0x80,0x94]
+// GFX12: s_bfe_i64 s[0:1], s[102:103], s4        ; encoding: [0x66,0x04,0x80,0x94]
 
 s_bfe_i64 s[0:1], s[2:3], s100
-// GFX12: encoding: [0x02,0x64,0x80,0x94]
+// GFX12: s_bfe_i64 s[0:1], s[2:3], s100          ; encoding: [0x02,0x64,0x80,0x94]
 
 s_bfe_i64 exec, s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0xfe,0x94]
+// GFX12: s_bfe_i64 exec, s[2:3], s4              ; encoding: [0x02,0x04,0xfe,0x94]
 
 s_bfe_i64 vcc, s[2:3], s4
-// GFX12: encoding: [0x02,0x04,0xea,0x94]
+// GFX12: s_bfe_i64 vcc, s[2:3], s4               ; encoding: [0x02,0x04,0xea,0x94]
 
 s_bfe_i64 s[0:1], exec, s4
-// GFX12: encoding: [0x7e,0x04,0x80,0x94]
+// GFX12: s_bfe_i64 s[0:1], exec, s4              ; encoding: [0x7e,0x04,0x80,0x94]
 
 s_bfe_i64 s[0:1], vcc, s4
-// GFX12: encoding: [0x6a,0x04,0x80,0x94]
+// GFX12: s_bfe_i64 s[0:1], vcc, s4               ; encoding: [0x6a,0x04,0x80,0x94]
 
 s_bfe_i64 s[0:1], 0, s4
-// GFX12: encoding: [0x80,0x04,0x80,0x94]
+// GFX12: s_bfe_i64 s[0:1], 0, s4                 ; encoding: [0x80,0x04,0x80,0x94]
 
 s_bfe_i64 s[0:1], -1, s4
-// GFX12: encoding: [0xc1,0x04,0x80,0x94]
+// GFX12: s_bfe_i64 s[0:1], -1, s4                ; encoding: [0xc1,0x04,0x80,0x94]
 
 s_bfe_i64 s[0:1], 0.5, s4
-// GFX12: encoding: [0xf0,0x04,0x80,0x94]
+// GFX12: s_bfe_i64 s[0:1], 0.5, s4               ; encoding: [0xf0,0x04,0x80,0x94]
 
 s_bfe_i64 s[0:1], -4.0, s4
-// GFX12: encoding: [0xf7,0x04,0x80,0x94]
+// GFX12: s_bfe_i64 s[0:1], -4.0, s4              ; encoding: [0xf7,0x04,0x80,0x94]
 
 s_bfe_i64 s[0:1], 0x3f717273, s4
-// GFX12: encoding: [0xff,0x04,0x80,0x94,0x73,0x72,0x71,0x3f]
+// GFX12: s_bfe_i64 s[0:1], 0x3f717273, s4        ; encoding: [0xff,0x04,0x80,0x94,0x73,0x72,0x71,0x3f]
 
 s_bfe_i64 s[0:1], 0xaf123456, s4
-// GFX12: encoding: [0xff,0x04,0x80,0x94,0x56,0x34,0x12,0xaf]
+// GFX12: s_bfe_i64 s[0:1], 0xaf123456, s4        ; encoding: [0xff,0x04,0x80,0x94,0x56,0x34,0x12,0xaf]
 
 s_bfe_i64 s[0:1], s[2:3], exec_lo
-// GFX12: encoding: [0x02,0x7e,0x80,0x94]
+// GFX12: s_bfe_i64 s[0:1], s[2:3], exec_lo       ; encoding: [0x02,0x7e,0x80,0x94]
 
 s_bfe_i64 s[0:1], s[2:3], vcc_lo
-// GFX12: encoding: [0x02,0x6a,0x80,0x94]
+// GFX12: s_bfe_i64 s[0:1], s[2:3], vcc_lo        ; encoding: [0x02,0x6a,0x80,0x94]
 
 s_bfe_i64 s[0:1], s[2:3], 0
-// GFX12: encoding: [0x02,0x80,0x80,0x94]
+// GFX12: s_bfe_i64 s[0:1], s[2:3], 0             ; encoding: [0x02,0x80,0x80,0x94]
 
 s_bfe_i64 s[0:1], s[2:3], -1
-// GFX12: encoding: [0x02,0xc1,0x80,0x94]
+// GFX12: s_bfe_i64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x80,0x94]
 
 s_bfe_i64 s[0:1], s[2:3], 0.5
-// GFX12: encoding: [0x02,0xf0,0x80,0x94]
+// GFX12: s_bfe_i64 s[0:1], s[2:3], 0.5           ; encoding: [0x02,0xf0,0x80,0x94]
 
 s_bfe_i64 s[0:1], s[2:3], -4.0
-// GFX12: encoding: [0x02,0xf7,0x80,0x94]
+// GFX12: s_bfe_i64 s[0:1], s[2:3], -4.0          ; encoding: [0x02,0xf7,0x80,0x94]
 
 s_bfe_i64 s[0:1], s[2:3], 0x3f717273
-// GFX12: encoding: [0x02,0xff,0x80,0x94,0x73,0x72,0x71,0x3f]
+// GFX12: s_bfe_i64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0x94,0x73,0x72,0x71,0x3f]
 
 s_bfe_i64 s[0:1], s[2:3], 0xaf123456
-// GFX12: encoding: [0x02,0xff,0x80,0x94,0x56,0x34,0x12,0xaf]
+// GFX12: s_bfe_i64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x94,0x56,0x34,0x12,0xaf]
 
 s_absdiff_i32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, s1, s2                ; encoding: [0x01,0x02,0x00,0x83]
 
 s_absdiff_i32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x83]
+// GFX12: s_absdiff_i32 s105, s104, s103          ; encoding: [0x68,0x67,0x69,0x83]
 
 s_absdiff_i32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, s104, s103            ; encoding: [0x68,0x67,0x00,0x83]
 
 s_absdiff_i32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x83]
+// GFX12: s_absdiff_i32 s105, s1, s103            ; encoding: [0x01,0x67,0x69,0x83]
 
 s_absdiff_i32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x83]
+// GFX12: s_absdiff_i32 s105, s104, s2            ; encoding: [0x68,0x02,0x69,0x83]
 
 s_absdiff_i32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x83]
+// GFX12: s_absdiff_i32 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0x83]
 
 s_absdiff_i32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, s104, s2              ; encoding: [0x68,0x02,0x00,0x83]
 
 s_absdiff_i32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, s1, s103              ; encoding: [0x01,0x67,0x00,0x83]
 
 s_absdiff_i32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x83]
+// GFX12: s_absdiff_i32 exec_lo, s1, s2           ; encoding: [0x01,0x02,0x7e,0x83]
 
 s_absdiff_i32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x83]
+// GFX12: s_absdiff_i32 exec_hi, s1, s2           ; encoding: [0x01,0x02,0x7f,0x83]
 
 s_absdiff_i32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x83]
+// GFX12: s_absdiff_i32 vcc_lo, s1, s2            ; encoding: [0x01,0x02,0x6a,0x83]
 
 s_absdiff_i32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x83]
+// GFX12: s_absdiff_i32 vcc_hi, s1, s2            ; encoding: [0x01,0x02,0x6b,0x83]
 
 s_absdiff_i32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x83]
+// GFX12: s_absdiff_i32 m0, s1, s2                ; encoding: [0x01,0x02,0x7d,0x83]
 
 s_absdiff_i32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, exec_lo, s2           ; encoding: [0x7e,0x02,0x00,0x83]
 
 s_absdiff_i32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, exec_hi, s2           ; encoding: [0x7f,0x02,0x00,0x83]
 
 s_absdiff_i32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, vcc_lo, s2            ; encoding: [0x6a,0x02,0x00,0x83]
 
 s_absdiff_i32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, vcc_hi, s2            ; encoding: [0x6b,0x02,0x00,0x83]
 
 s_absdiff_i32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, m0, s2                ; encoding: [0x7d,0x02,0x00,0x83]
 
 s_absdiff_i32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, 0, s2                 ; encoding: [0x80,0x02,0x00,0x83]
 
 s_absdiff_i32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, -1, s2                ; encoding: [0xc1,0x02,0x00,0x83]
 
 s_absdiff_i32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, 0.5, s2               ; encoding: [0xf0,0x02,0x00,0x83]
 
 s_absdiff_i32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, -4.0, s2              ; encoding: [0xf7,0x02,0x00,0x83]
 
 s_absdiff_i32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x83,0x73,0x72,0x71,0x3f]
+// GFX12: s_absdiff_i32 s0, 0x3f717273, s2        ; encoding: [0xff,0x02,0x00,0x83,0x73,0x72,0x71,0x3f]
 
 s_absdiff_i32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x83,0x56,0x34,0x12,0xaf]
+// GFX12: s_absdiff_i32 s0, 0xaf123456, s2        ; encoding: [0xff,0x02,0x00,0x83,0x56,0x34,0x12,0xaf]
 
 s_absdiff_i32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, s1, exec_lo           ; encoding: [0x01,0x7e,0x00,0x83]
 
 s_absdiff_i32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, s1, exec_hi           ; encoding: [0x01,0x7f,0x00,0x83]
 
 s_absdiff_i32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, s1, vcc_lo            ; encoding: [0x01,0x6a,0x00,0x83]
 
 s_absdiff_i32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, s1, vcc_hi            ; encoding: [0x01,0x6b,0x00,0x83]
 
 s_absdiff_i32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, s1, m0                ; encoding: [0x01,0x7d,0x00,0x83]
 
 s_absdiff_i32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, s1, 0                 ; encoding: [0x01,0x80,0x00,0x83]
 
 s_absdiff_i32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, s1, -1                ; encoding: [0x01,0xc1,0x00,0x83]
 
 s_absdiff_i32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, s1, 0.5               ; encoding: [0x01,0xf0,0x00,0x83]
 
 s_absdiff_i32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x83]
+// GFX12: s_absdiff_i32 s0, s1, -4.0              ; encoding: [0x01,0xf7,0x00,0x83]
 
 s_absdiff_i32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x83,0x73,0x72,0x71,0x3f]
+// GFX12: s_absdiff_i32 s0, s1, 0x3f717273        ; encoding: [0x01,0xff,0x00,0x83,0x73,0x72,0x71,0x3f]
 
 s_absdiff_i32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x83,0x56,0x34,0x12,0xaf]
+// GFX12: s_absdiff_i32 s0, s1, 0xaf123456        ; encoding: [0x01,0xff,0x00,0x83,0x56,0x34,0x12,0xaf]
 
 s_lshl1_add_u32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, s1, s2              ; encoding: [0x01,0x02,0x00,0x87]
 
 s_lshl1_add_u32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x87]
+// GFX12: s_lshl1_add_u32 s105, s104, s103        ; encoding: [0x68,0x67,0x69,0x87]
 
 s_lshl1_add_u32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, s104, s103          ; encoding: [0x68,0x67,0x00,0x87]
 
 s_lshl1_add_u32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x87]
+// GFX12: s_lshl1_add_u32 s105, s1, s103          ; encoding: [0x01,0x67,0x69,0x87]
 
 s_lshl1_add_u32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x87]
+// GFX12: s_lshl1_add_u32 s105, s104, s2          ; encoding: [0x68,0x02,0x69,0x87]
 
 s_lshl1_add_u32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x87]
+// GFX12: s_lshl1_add_u32 s105, s1, s2            ; encoding: [0x01,0x02,0x69,0x87]
 
 s_lshl1_add_u32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, s104, s2            ; encoding: [0x68,0x02,0x00,0x87]
 
 s_lshl1_add_u32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, s1, s103            ; encoding: [0x01,0x67,0x00,0x87]
 
 s_lshl1_add_u32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x87]
+// GFX12: s_lshl1_add_u32 exec_lo, s1, s2         ; encoding: [0x01,0x02,0x7e,0x87]
 
 s_lshl1_add_u32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x87]
+// GFX12: s_lshl1_add_u32 exec_hi, s1, s2         ; encoding: [0x01,0x02,0x7f,0x87]
 
 s_lshl1_add_u32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x87]
+// GFX12: s_lshl1_add_u32 vcc_lo, s1, s2          ; encoding: [0x01,0x02,0x6a,0x87]
 
 s_lshl1_add_u32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x87]
+// GFX12: s_lshl1_add_u32 vcc_hi, s1, s2          ; encoding: [0x01,0x02,0x6b,0x87]
 
 s_lshl1_add_u32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x87]
+// GFX12: s_lshl1_add_u32 m0, s1, s2              ; encoding: [0x01,0x02,0x7d,0x87]
 
 s_lshl1_add_u32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, exec_lo, s2         ; encoding: [0x7e,0x02,0x00,0x87]
 
 s_lshl1_add_u32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, exec_hi, s2         ; encoding: [0x7f,0x02,0x00,0x87]
 
 s_lshl1_add_u32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, vcc_lo, s2          ; encoding: [0x6a,0x02,0x00,0x87]
 
 s_lshl1_add_u32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, vcc_hi, s2          ; encoding: [0x6b,0x02,0x00,0x87]
 
 s_lshl1_add_u32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, m0, s2              ; encoding: [0x7d,0x02,0x00,0x87]
 
 s_lshl1_add_u32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, 0, s2               ; encoding: [0x80,0x02,0x00,0x87]
 
 s_lshl1_add_u32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, -1, s2              ; encoding: [0xc1,0x02,0x00,0x87]
 
 s_lshl1_add_u32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, 0.5, s2             ; encoding: [0xf0,0x02,0x00,0x87]
 
 s_lshl1_add_u32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, -4.0, s2            ; encoding: [0xf7,0x02,0x00,0x87]
 
 s_lshl1_add_u32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x87,0x73,0x72,0x71,0x3f]
+// GFX12: s_lshl1_add_u32 s0, 0x3f717273, s2      ; encoding: [0xff,0x02,0x00,0x87,0x73,0x72,0x71,0x3f]
 
 s_lshl1_add_u32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x87,0x56,0x34,0x12,0xaf]
+// GFX12: s_lshl1_add_u32 s0, 0xaf123456, s2      ; encoding: [0xff,0x02,0x00,0x87,0x56,0x34,0x12,0xaf]
 
 s_lshl1_add_u32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, s1, exec_lo         ; encoding: [0x01,0x7e,0x00,0x87]
 
 s_lshl1_add_u32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, s1, exec_hi         ; encoding: [0x01,0x7f,0x00,0x87]
 
 s_lshl1_add_u32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, s1, vcc_lo          ; encoding: [0x01,0x6a,0x00,0x87]
 
 s_lshl1_add_u32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, s1, vcc_hi          ; encoding: [0x01,0x6b,0x00,0x87]
 
 s_lshl1_add_u32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, s1, m0              ; encoding: [0x01,0x7d,0x00,0x87]
 
 s_lshl1_add_u32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, s1, 0               ; encoding: [0x01,0x80,0x00,0x87]
 
 s_lshl1_add_u32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, s1, -1              ; encoding: [0x01,0xc1,0x00,0x87]
 
 s_lshl1_add_u32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, s1, 0.5             ; encoding: [0x01,0xf0,0x00,0x87]
 
 s_lshl1_add_u32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x87]
+// GFX12: s_lshl1_add_u32 s0, s1, -4.0            ; encoding: [0x01,0xf7,0x00,0x87]
 
 s_lshl1_add_u32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x87,0x73,0x72,0x71,0x3f]
+// GFX12: s_lshl1_add_u32 s0, s1, 0x3f717273      ; encoding: [0x01,0xff,0x00,0x87,0x73,0x72,0x71,0x3f]
 
 s_lshl1_add_u32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x87,0x56,0x34,0x12,0xaf]
+// GFX12: s_lshl1_add_u32 s0, s1, 0xaf123456      ; encoding: [0x01,0xff,0x00,0x87,0x56,0x34,0x12,0xaf]
 
 s_lshl2_add_u32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, s1, s2              ; encoding: [0x01,0x02,0x80,0x87]
 
 s_lshl2_add_u32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0xe9,0x87]
+// GFX12: s_lshl2_add_u32 s105, s104, s103        ; encoding: [0x68,0x67,0xe9,0x87]
 
 s_lshl2_add_u32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, s104, s103          ; encoding: [0x68,0x67,0x80,0x87]
 
 s_lshl2_add_u32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0xe9,0x87]
+// GFX12: s_lshl2_add_u32 s105, s1, s103          ; encoding: [0x01,0x67,0xe9,0x87]
 
 s_lshl2_add_u32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0xe9,0x87]
+// GFX12: s_lshl2_add_u32 s105, s104, s2          ; encoding: [0x68,0x02,0xe9,0x87]
 
 s_lshl2_add_u32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0xe9,0x87]
+// GFX12: s_lshl2_add_u32 s105, s1, s2            ; encoding: [0x01,0x02,0xe9,0x87]
 
 s_lshl2_add_u32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, s104, s2            ; encoding: [0x68,0x02,0x80,0x87]
 
 s_lshl2_add_u32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, s1, s103            ; encoding: [0x01,0x67,0x80,0x87]
 
 s_lshl2_add_u32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfe,0x87]
+// GFX12: s_lshl2_add_u32 exec_lo, s1, s2         ; encoding: [0x01,0x02,0xfe,0x87]
 
 s_lshl2_add_u32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xff,0x87]
+// GFX12: s_lshl2_add_u32 exec_hi, s1, s2         ; encoding: [0x01,0x02,0xff,0x87]
 
 s_lshl2_add_u32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xea,0x87]
+// GFX12: s_lshl2_add_u32 vcc_lo, s1, s2          ; encoding: [0x01,0x02,0xea,0x87]
 
 s_lshl2_add_u32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xeb,0x87]
+// GFX12: s_lshl2_add_u32 vcc_hi, s1, s2          ; encoding: [0x01,0x02,0xeb,0x87]
 
 s_lshl2_add_u32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfd,0x87]
+// GFX12: s_lshl2_add_u32 m0, s1, s2              ; encoding: [0x01,0x02,0xfd,0x87]
 
 s_lshl2_add_u32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, exec_lo, s2         ; encoding: [0x7e,0x02,0x80,0x87]
 
 s_lshl2_add_u32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, exec_hi, s2         ; encoding: [0x7f,0x02,0x80,0x87]
 
 s_lshl2_add_u32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, vcc_lo, s2          ; encoding: [0x6a,0x02,0x80,0x87]
 
 s_lshl2_add_u32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, vcc_hi, s2          ; encoding: [0x6b,0x02,0x80,0x87]
 
 s_lshl2_add_u32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, m0, s2              ; encoding: [0x7d,0x02,0x80,0x87]
 
 s_lshl2_add_u32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, 0, s2               ; encoding: [0x80,0x02,0x80,0x87]
 
 s_lshl2_add_u32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, -1, s2              ; encoding: [0xc1,0x02,0x80,0x87]
 
 s_lshl2_add_u32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, 0.5, s2             ; encoding: [0xf0,0x02,0x80,0x87]
 
 s_lshl2_add_u32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, -4.0, s2            ; encoding: [0xf7,0x02,0x80,0x87]
 
 s_lshl2_add_u32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x87,0x73,0x72,0x71,0x3f]
+// GFX12: s_lshl2_add_u32 s0, 0x3f717273, s2      ; encoding: [0xff,0x02,0x80,0x87,0x73,0x72,0x71,0x3f]
 
 s_lshl2_add_u32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x87,0x56,0x34,0x12,0xaf]
+// GFX12: s_lshl2_add_u32 s0, 0xaf123456, s2      ; encoding: [0xff,0x02,0x80,0x87,0x56,0x34,0x12,0xaf]
 
 s_lshl2_add_u32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, s1, exec_lo         ; encoding: [0x01,0x7e,0x80,0x87]
 
 s_lshl2_add_u32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, s1, exec_hi         ; encoding: [0x01,0x7f,0x80,0x87]
 
 s_lshl2_add_u32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, s1, vcc_lo          ; encoding: [0x01,0x6a,0x80,0x87]
 
 s_lshl2_add_u32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, s1, vcc_hi          ; encoding: [0x01,0x6b,0x80,0x87]
 
 s_lshl2_add_u32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, s1, m0              ; encoding: [0x01,0x7d,0x80,0x87]
 
 s_lshl2_add_u32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, s1, 0               ; encoding: [0x01,0x80,0x80,0x87]
 
 s_lshl2_add_u32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, s1, -1              ; encoding: [0x01,0xc1,0x80,0x87]
 
 s_lshl2_add_u32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, s1, 0.5             ; encoding: [0x01,0xf0,0x80,0x87]
 
 s_lshl2_add_u32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x80,0x87]
+// GFX12: s_lshl2_add_u32 s0, s1, -4.0            ; encoding: [0x01,0xf7,0x80,0x87]
 
 s_lshl2_add_u32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x80,0x87,0x73,0x72,0x71,0x3f]
+// GFX12: s_lshl2_add_u32 s0, s1, 0x3f717273      ; encoding: [0x01,0xff,0x80,0x87,0x73,0x72,0x71,0x3f]
 
 s_lshl2_add_u32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x80,0x87,0x56,0x34,0x12,0xaf]
+// GFX12: s_lshl2_add_u32 s0, s1, 0xaf123456      ; encoding: [0x01,0xff,0x80,0x87,0x56,0x34,0x12,0xaf]
 
 s_lshl3_add_u32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, s1, s2              ; encoding: [0x01,0x02,0x00,0x88]
 
 s_lshl3_add_u32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x88]
+// GFX12: s_lshl3_add_u32 s105, s104, s103        ; encoding: [0x68,0x67,0x69,0x88]
 
 s_lshl3_add_u32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, s104, s103          ; encoding: [0x68,0x67,0x00,0x88]
 
 s_lshl3_add_u32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x88]
+// GFX12: s_lshl3_add_u32 s105, s1, s103          ; encoding: [0x01,0x67,0x69,0x88]
 
 s_lshl3_add_u32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x88]
+// GFX12: s_lshl3_add_u32 s105, s104, s2          ; encoding: [0x68,0x02,0x69,0x88]
 
 s_lshl3_add_u32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x88]
+// GFX12: s_lshl3_add_u32 s105, s1, s2            ; encoding: [0x01,0x02,0x69,0x88]
 
 s_lshl3_add_u32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, s104, s2            ; encoding: [0x68,0x02,0x00,0x88]
 
 s_lshl3_add_u32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, s1, s103            ; encoding: [0x01,0x67,0x00,0x88]
 
 s_lshl3_add_u32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x88]
+// GFX12: s_lshl3_add_u32 exec_lo, s1, s2         ; encoding: [0x01,0x02,0x7e,0x88]
 
 s_lshl3_add_u32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x88]
+// GFX12: s_lshl3_add_u32 exec_hi, s1, s2         ; encoding: [0x01,0x02,0x7f,0x88]
 
 s_lshl3_add_u32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x88]
+// GFX12: s_lshl3_add_u32 vcc_lo, s1, s2          ; encoding: [0x01,0x02,0x6a,0x88]
 
 s_lshl3_add_u32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x88]
+// GFX12: s_lshl3_add_u32 vcc_hi, s1, s2          ; encoding: [0x01,0x02,0x6b,0x88]
 
 s_lshl3_add_u32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x88]
+// GFX12: s_lshl3_add_u32 m0, s1, s2              ; encoding: [0x01,0x02,0x7d,0x88]
 
 s_lshl3_add_u32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, exec_lo, s2         ; encoding: [0x7e,0x02,0x00,0x88]
 
 s_lshl3_add_u32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, exec_hi, s2         ; encoding: [0x7f,0x02,0x00,0x88]
 
 s_lshl3_add_u32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, vcc_lo, s2          ; encoding: [0x6a,0x02,0x00,0x88]
 
 s_lshl3_add_u32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, vcc_hi, s2          ; encoding: [0x6b,0x02,0x00,0x88]
 
 s_lshl3_add_u32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, m0, s2              ; encoding: [0x7d,0x02,0x00,0x88]
 
 s_lshl3_add_u32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, 0, s2               ; encoding: [0x80,0x02,0x00,0x88]
 
 s_lshl3_add_u32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, -1, s2              ; encoding: [0xc1,0x02,0x00,0x88]
 
 s_lshl3_add_u32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, 0.5, s2             ; encoding: [0xf0,0x02,0x00,0x88]
 
 s_lshl3_add_u32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, -4.0, s2            ; encoding: [0xf7,0x02,0x00,0x88]
 
 s_lshl3_add_u32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x88,0x73,0x72,0x71,0x3f]
+// GFX12: s_lshl3_add_u32 s0, 0x3f717273, s2      ; encoding: [0xff,0x02,0x00,0x88,0x73,0x72,0x71,0x3f]
 
 s_lshl3_add_u32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x88,0x56,0x34,0x12,0xaf]
+// GFX12: s_lshl3_add_u32 s0, 0xaf123456, s2      ; encoding: [0xff,0x02,0x00,0x88,0x56,0x34,0x12,0xaf]
 
 s_lshl3_add_u32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, s1, exec_lo         ; encoding: [0x01,0x7e,0x00,0x88]
 
 s_lshl3_add_u32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, s1, exec_hi         ; encoding: [0x01,0x7f,0x00,0x88]
 
 s_lshl3_add_u32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, s1, vcc_lo          ; encoding: [0x01,0x6a,0x00,0x88]
 
 s_lshl3_add_u32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, s1, vcc_hi          ; encoding: [0x01,0x6b,0x00,0x88]
 
 s_lshl3_add_u32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, s1, m0              ; encoding: [0x01,0x7d,0x00,0x88]
 
 s_lshl3_add_u32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, s1, 0               ; encoding: [0x01,0x80,0x00,0x88]
 
 s_lshl3_add_u32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, s1, -1              ; encoding: [0x01,0xc1,0x00,0x88]
 
 s_lshl3_add_u32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, s1, 0.5             ; encoding: [0x01,0xf0,0x00,0x88]
 
 s_lshl3_add_u32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x88]
+// GFX12: s_lshl3_add_u32 s0, s1, -4.0            ; encoding: [0x01,0xf7,0x00,0x88]
 
 s_lshl3_add_u32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x88,0x73,0x72,0x71,0x3f]
+// GFX12: s_lshl3_add_u32 s0, s1, 0x3f717273      ; encoding: [0x01,0xff,0x00,0x88,0x73,0x72,0x71,0x3f]
 
 s_lshl3_add_u32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x88,0x56,0x34,0x12,0xaf]
+// GFX12: s_lshl3_add_u32 s0, s1, 0xaf123456      ; encoding: [0x01,0xff,0x00,0x88,0x56,0x34,0x12,0xaf]
 
 s_lshl4_add_u32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, s1, s2              ; encoding: [0x01,0x02,0x80,0x88]
 
 s_lshl4_add_u32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0xe9,0x88]
+// GFX12: s_lshl4_add_u32 s105, s104, s103        ; encoding: [0x68,0x67,0xe9,0x88]
 
 s_lshl4_add_u32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, s104, s103          ; encoding: [0x68,0x67,0x80,0x88]
 
 s_lshl4_add_u32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0xe9,0x88]
+// GFX12: s_lshl4_add_u32 s105, s1, s103          ; encoding: [0x01,0x67,0xe9,0x88]
 
 s_lshl4_add_u32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0xe9,0x88]
+// GFX12: s_lshl4_add_u32 s105, s104, s2          ; encoding: [0x68,0x02,0xe9,0x88]
 
 s_lshl4_add_u32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0xe9,0x88]
+// GFX12: s_lshl4_add_u32 s105, s1, s2            ; encoding: [0x01,0x02,0xe9,0x88]
 
 s_lshl4_add_u32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, s104, s2            ; encoding: [0x68,0x02,0x80,0x88]
 
 s_lshl4_add_u32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, s1, s103            ; encoding: [0x01,0x67,0x80,0x88]
 
 s_lshl4_add_u32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfe,0x88]
+// GFX12: s_lshl4_add_u32 exec_lo, s1, s2         ; encoding: [0x01,0x02,0xfe,0x88]
 
 s_lshl4_add_u32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xff,0x88]
+// GFX12: s_lshl4_add_u32 exec_hi, s1, s2         ; encoding: [0x01,0x02,0xff,0x88]
 
 s_lshl4_add_u32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xea,0x88]
+// GFX12: s_lshl4_add_u32 vcc_lo, s1, s2          ; encoding: [0x01,0x02,0xea,0x88]
 
 s_lshl4_add_u32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xeb,0x88]
+// GFX12: s_lshl4_add_u32 vcc_hi, s1, s2          ; encoding: [0x01,0x02,0xeb,0x88]
 
 s_lshl4_add_u32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfd,0x88]
+// GFX12: s_lshl4_add_u32 m0, s1, s2              ; encoding: [0x01,0x02,0xfd,0x88]
 
 s_lshl4_add_u32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, exec_lo, s2         ; encoding: [0x7e,0x02,0x80,0x88]
 
 s_lshl4_add_u32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, exec_hi, s2         ; encoding: [0x7f,0x02,0x80,0x88]
 
 s_lshl4_add_u32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, vcc_lo, s2          ; encoding: [0x6a,0x02,0x80,0x88]
 
 s_lshl4_add_u32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, vcc_hi, s2          ; encoding: [0x6b,0x02,0x80,0x88]
 
 s_lshl4_add_u32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, m0, s2              ; encoding: [0x7d,0x02,0x80,0x88]
 
 s_lshl4_add_u32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, 0, s2               ; encoding: [0x80,0x02,0x80,0x88]
 
 s_lshl4_add_u32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, -1, s2              ; encoding: [0xc1,0x02,0x80,0x88]
 
 s_lshl4_add_u32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, 0.5, s2             ; encoding: [0xf0,0x02,0x80,0x88]
 
 s_lshl4_add_u32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, -4.0, s2            ; encoding: [0xf7,0x02,0x80,0x88]
 
 s_lshl4_add_u32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x88,0x73,0x72,0x71,0x3f]
+// GFX12: s_lshl4_add_u32 s0, 0x3f717273, s2      ; encoding: [0xff,0x02,0x80,0x88,0x73,0x72,0x71,0x3f]
 
 s_lshl4_add_u32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x88,0x56,0x34,0x12,0xaf]
+// GFX12: s_lshl4_add_u32 s0, 0xaf123456, s2      ; encoding: [0xff,0x02,0x80,0x88,0x56,0x34,0x12,0xaf]
 
 s_lshl4_add_u32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, s1, exec_lo         ; encoding: [0x01,0x7e,0x80,0x88]
 
 s_lshl4_add_u32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, s1, exec_hi         ; encoding: [0x01,0x7f,0x80,0x88]
 
 s_lshl4_add_u32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, s1, vcc_lo          ; encoding: [0x01,0x6a,0x80,0x88]
 
 s_lshl4_add_u32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, s1, vcc_hi          ; encoding: [0x01,0x6b,0x80,0x88]
 
 s_lshl4_add_u32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, s1, m0              ; encoding: [0x01,0x7d,0x80,0x88]
 
 s_lshl4_add_u32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, s1, 0               ; encoding: [0x01,0x80,0x80,0x88]
 
 s_lshl4_add_u32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, s1, -1              ; encoding: [0x01,0xc1,0x80,0x88]
 
 s_lshl4_add_u32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, s1, 0.5             ; encoding: [0x01,0xf0,0x80,0x88]
 
 s_lshl4_add_u32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x80,0x88]
+// GFX12: s_lshl4_add_u32 s0, s1, -4.0            ; encoding: [0x01,0xf7,0x80,0x88]
 
 s_lshl4_add_u32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x80,0x88,0x73,0x72,0x71,0x3f]
+// GFX12: s_lshl4_add_u32 s0, s1, 0x3f717273      ; encoding: [0x01,0xff,0x80,0x88,0x73,0x72,0x71,0x3f]
 
 s_lshl4_add_u32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x80,0x88,0x56,0x34,0x12,0xaf]
+// GFX12: s_lshl4_add_u32 s0, s1, 0xaf123456      ; encoding: [0x01,0xff,0x80,0x88,0x56,0x34,0x12,0xaf]
 
 s_pack_ll_b32_b16 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, s1, s2            ; encoding: [0x01,0x02,0x00,0x99]
 
 s_pack_ll_b32_b16 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x99]
+// GFX12: s_pack_ll_b32_b16 s105, s104, s103      ; encoding: [0x68,0x67,0x69,0x99]
 
 s_pack_ll_b32_b16 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, s104, s103        ; encoding: [0x68,0x67,0x00,0x99]
 
 s_pack_ll_b32_b16 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x99]
+// GFX12: s_pack_ll_b32_b16 s105, s1, s103        ; encoding: [0x01,0x67,0x69,0x99]
 
 s_pack_ll_b32_b16 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x99]
+// GFX12: s_pack_ll_b32_b16 s105, s104, s2        ; encoding: [0x68,0x02,0x69,0x99]
 
 s_pack_ll_b32_b16 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x99]
+// GFX12: s_pack_ll_b32_b16 s105, s1, s2          ; encoding: [0x01,0x02,0x69,0x99]
 
 s_pack_ll_b32_b16 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, s104, s2          ; encoding: [0x68,0x02,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, s1, s103          ; encoding: [0x01,0x67,0x00,0x99]
 
 s_pack_ll_b32_b16 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x99]
+// GFX12: s_pack_ll_b32_b16 exec_lo, s1, s2       ; encoding: [0x01,0x02,0x7e,0x99]
 
 s_pack_ll_b32_b16 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x99]
+// GFX12: s_pack_ll_b32_b16 exec_hi, s1, s2       ; encoding: [0x01,0x02,0x7f,0x99]
 
 s_pack_ll_b32_b16 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x99]
+// GFX12: s_pack_ll_b32_b16 vcc_lo, s1, s2        ; encoding: [0x01,0x02,0x6a,0x99]
 
 s_pack_ll_b32_b16 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x99]
+// GFX12: s_pack_ll_b32_b16 vcc_hi, s1, s2        ; encoding: [0x01,0x02,0x6b,0x99]
 
 s_pack_ll_b32_b16 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x99]
+// GFX12: s_pack_ll_b32_b16 m0, s1, s2            ; encoding: [0x01,0x02,0x7d,0x99]
 
 s_pack_ll_b32_b16 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, exec_lo, s2       ; encoding: [0x7e,0x02,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, exec_hi, s2       ; encoding: [0x7f,0x02,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, vcc_lo, s2        ; encoding: [0x6a,0x02,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, vcc_hi, s2        ; encoding: [0x6b,0x02,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, m0, s2            ; encoding: [0x7d,0x02,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, 0, s2             ; encoding: [0x80,0x02,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, -1, s2            ; encoding: [0xc1,0x02,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, 0.5, s2           ; encoding: [0xf0,0x02,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, -4.0, s2          ; encoding: [0xf7,0x02,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x99,0x73,0x72,0x71,0x3f]
+// GFX12: s_pack_ll_b32_b16 s0, 0x3f717273, s2    ; encoding: [0xff,0x02,0x00,0x99,0x73,0x72,0x71,0x3f]
 
 s_pack_ll_b32_b16 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x99,0x56,0x34,0x12,0xaf]
+// GFX12: s_pack_ll_b32_b16 s0, 0xaf123456, s2    ; encoding: [0xff,0x02,0x00,0x99,0x56,0x34,0x12,0xaf]
 
 s_pack_ll_b32_b16 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, s1, exec_lo       ; encoding: [0x01,0x7e,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, s1, exec_hi       ; encoding: [0x01,0x7f,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, s1, vcc_lo        ; encoding: [0x01,0x6a,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, s1, vcc_hi        ; encoding: [0x01,0x6b,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, s1, m0            ; encoding: [0x01,0x7d,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, s1, 0             ; encoding: [0x01,0x80,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, s1, -1            ; encoding: [0x01,0xc1,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, s1, 0.5           ; encoding: [0x01,0xf0,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x99]
+// GFX12: s_pack_ll_b32_b16 s0, s1, -4.0          ; encoding: [0x01,0xf7,0x00,0x99]
 
 s_pack_ll_b32_b16 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x99,0x73,0x72,0x71,0x3f]
+// GFX12: s_pack_ll_b32_b16 s0, s1, 0x3f717273    ; encoding: [0x01,0xff,0x00,0x99,0x73,0x72,0x71,0x3f]
 
 s_pack_ll_b32_b16 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x99,0x56,0x34,0x12,0xaf]
+// GFX12: s_pack_ll_b32_b16 s0, s1, 0xaf123456    ; encoding: [0x01,0xff,0x00,0x99,0x56,0x34,0x12,0xaf]
 
 s_pack_lh_b32_b16 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, s1, s2            ; encoding: [0x01,0x02,0x80,0x99]
 
 s_pack_lh_b32_b16 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0xe9,0x99]
+// GFX12: s_pack_lh_b32_b16 s105, s104, s103      ; encoding: [0x68,0x67,0xe9,0x99]
 
 s_pack_lh_b32_b16 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, s104, s103        ; encoding: [0x68,0x67,0x80,0x99]
 
 s_pack_lh_b32_b16 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0xe9,0x99]
+// GFX12: s_pack_lh_b32_b16 s105, s1, s103        ; encoding: [0x01,0x67,0xe9,0x99]
 
 s_pack_lh_b32_b16 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0xe9,0x99]
+// GFX12: s_pack_lh_b32_b16 s105, s104, s2        ; encoding: [0x68,0x02,0xe9,0x99]
 
 s_pack_lh_b32_b16 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0xe9,0x99]
+// GFX12: s_pack_lh_b32_b16 s105, s1, s2          ; encoding: [0x01,0x02,0xe9,0x99]
 
 s_pack_lh_b32_b16 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, s104, s2          ; encoding: [0x68,0x02,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, s1, s103          ; encoding: [0x01,0x67,0x80,0x99]
 
 s_pack_lh_b32_b16 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfe,0x99]
+// GFX12: s_pack_lh_b32_b16 exec_lo, s1, s2       ; encoding: [0x01,0x02,0xfe,0x99]
 
 s_pack_lh_b32_b16 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xff,0x99]
+// GFX12: s_pack_lh_b32_b16 exec_hi, s1, s2       ; encoding: [0x01,0x02,0xff,0x99]
 
 s_pack_lh_b32_b16 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xea,0x99]
+// GFX12: s_pack_lh_b32_b16 vcc_lo, s1, s2        ; encoding: [0x01,0x02,0xea,0x99]
 
 s_pack_lh_b32_b16 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xeb,0x99]
+// GFX12: s_pack_lh_b32_b16 vcc_hi, s1, s2        ; encoding: [0x01,0x02,0xeb,0x99]
 
 s_pack_lh_b32_b16 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfd,0x99]
+// GFX12: s_pack_lh_b32_b16 m0, s1, s2            ; encoding: [0x01,0x02,0xfd,0x99]
 
 s_pack_lh_b32_b16 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, exec_lo, s2       ; encoding: [0x7e,0x02,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, exec_hi, s2       ; encoding: [0x7f,0x02,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, vcc_lo, s2        ; encoding: [0x6a,0x02,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, vcc_hi, s2        ; encoding: [0x6b,0x02,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, m0, s2            ; encoding: [0x7d,0x02,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, 0, s2             ; encoding: [0x80,0x02,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, -1, s2            ; encoding: [0xc1,0x02,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, 0.5, s2           ; encoding: [0xf0,0x02,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, -4.0, s2          ; encoding: [0xf7,0x02,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x99,0x73,0x72,0x71,0x3f]
+// GFX12: s_pack_lh_b32_b16 s0, 0x3f717273, s2    ; encoding: [0xff,0x02,0x80,0x99,0x73,0x72,0x71,0x3f]
 
 s_pack_lh_b32_b16 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x99,0x56,0x34,0x12,0xaf]
+// GFX12: s_pack_lh_b32_b16 s0, 0xaf123456, s2    ; encoding: [0xff,0x02,0x80,0x99,0x56,0x34,0x12,0xaf]
 
 s_pack_lh_b32_b16 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, s1, exec_lo       ; encoding: [0x01,0x7e,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, s1, exec_hi       ; encoding: [0x01,0x7f,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, s1, vcc_lo        ; encoding: [0x01,0x6a,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, s1, vcc_hi        ; encoding: [0x01,0x6b,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, s1, m0            ; encoding: [0x01,0x7d,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, s1, 0             ; encoding: [0x01,0x80,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, s1, -1            ; encoding: [0x01,0xc1,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, s1, 0.5           ; encoding: [0x01,0xf0,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x80,0x99]
+// GFX12: s_pack_lh_b32_b16 s0, s1, -4.0          ; encoding: [0x01,0xf7,0x80,0x99]
 
 s_pack_lh_b32_b16 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x80,0x99,0x73,0x72,0x71,0x3f]
+// GFX12: s_pack_lh_b32_b16 s0, s1, 0x3f717273    ; encoding: [0x01,0xff,0x80,0x99,0x73,0x72,0x71,0x3f]
 
 s_pack_lh_b32_b16 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x80,0x99,0x56,0x34,0x12,0xaf]
+// GFX12: s_pack_lh_b32_b16 s0, s1, 0xaf123456    ; encoding: [0x01,0xff,0x80,0x99,0x56,0x34,0x12,0xaf]
 
 s_pack_hh_b32_b16 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, s1, s2            ; encoding: [0x01,0x02,0x00,0x9a]
 
 s_pack_hh_b32_b16 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x9a]
+// GFX12: s_pack_hh_b32_b16 s105, s104, s103      ; encoding: [0x68,0x67,0x69,0x9a]
 
 s_pack_hh_b32_b16 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, s104, s103        ; encoding: [0x68,0x67,0x00,0x9a]
 
 s_pack_hh_b32_b16 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x9a]
+// GFX12: s_pack_hh_b32_b16 s105, s1, s103        ; encoding: [0x01,0x67,0x69,0x9a]
 
 s_pack_hh_b32_b16 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x9a]
+// GFX12: s_pack_hh_b32_b16 s105, s104, s2        ; encoding: [0x68,0x02,0x69,0x9a]
 
 s_pack_hh_b32_b16 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x9a]
+// GFX12: s_pack_hh_b32_b16 s105, s1, s2          ; encoding: [0x01,0x02,0x69,0x9a]
 
 s_pack_hh_b32_b16 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, s104, s2          ; encoding: [0x68,0x02,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, s1, s103          ; encoding: [0x01,0x67,0x00,0x9a]
 
 s_pack_hh_b32_b16 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x9a]
+// GFX12: s_pack_hh_b32_b16 exec_lo, s1, s2       ; encoding: [0x01,0x02,0x7e,0x9a]
 
 s_pack_hh_b32_b16 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x9a]
+// GFX12: s_pack_hh_b32_b16 exec_hi, s1, s2       ; encoding: [0x01,0x02,0x7f,0x9a]
 
 s_pack_hh_b32_b16 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x9a]
+// GFX12: s_pack_hh_b32_b16 vcc_lo, s1, s2        ; encoding: [0x01,0x02,0x6a,0x9a]
 
 s_pack_hh_b32_b16 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x9a]
+// GFX12: s_pack_hh_b32_b16 vcc_hi, s1, s2        ; encoding: [0x01,0x02,0x6b,0x9a]
 
 s_pack_hh_b32_b16 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x9a]
+// GFX12: s_pack_hh_b32_b16 m0, s1, s2            ; encoding: [0x01,0x02,0x7d,0x9a]
 
 s_pack_hh_b32_b16 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, exec_lo, s2       ; encoding: [0x7e,0x02,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, exec_hi, s2       ; encoding: [0x7f,0x02,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, vcc_lo, s2        ; encoding: [0x6a,0x02,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, vcc_hi, s2        ; encoding: [0x6b,0x02,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, m0, s2            ; encoding: [0x7d,0x02,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, 0, s2             ; encoding: [0x80,0x02,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, -1, s2            ; encoding: [0xc1,0x02,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, 0.5, s2           ; encoding: [0xf0,0x02,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, -4.0, s2          ; encoding: [0xf7,0x02,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x9a,0x73,0x72,0x71,0x3f]
+// GFX12: s_pack_hh_b32_b16 s0, 0x3f717273, s2    ; encoding: [0xff,0x02,0x00,0x9a,0x73,0x72,0x71,0x3f]
 
 s_pack_hh_b32_b16 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x9a,0x56,0x34,0x12,0xaf]
+// GFX12: s_pack_hh_b32_b16 s0, 0xaf123456, s2    ; encoding: [0xff,0x02,0x00,0x9a,0x56,0x34,0x12,0xaf]
 
 s_pack_hh_b32_b16 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, s1, exec_lo       ; encoding: [0x01,0x7e,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, s1, exec_hi       ; encoding: [0x01,0x7f,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, s1, vcc_lo        ; encoding: [0x01,0x6a,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, s1, vcc_hi        ; encoding: [0x01,0x6b,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, s1, m0            ; encoding: [0x01,0x7d,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, s1, 0             ; encoding: [0x01,0x80,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, s1, -1            ; encoding: [0x01,0xc1,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, s1, 0.5           ; encoding: [0x01,0xf0,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x9a]
+// GFX12: s_pack_hh_b32_b16 s0, s1, -4.0          ; encoding: [0x01,0xf7,0x00,0x9a]
 
 s_pack_hh_b32_b16 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x9a,0x73,0x72,0x71,0x3f]
+// GFX12: s_pack_hh_b32_b16 s0, s1, 0x3f717273    ; encoding: [0x01,0xff,0x00,0x9a,0x73,0x72,0x71,0x3f]
 
 s_pack_hh_b32_b16 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x9a,0x56,0x34,0x12,0xaf]
+// GFX12: s_pack_hh_b32_b16 s0, s1, 0xaf123456    ; encoding: [0x01,0xff,0x00,0x9a,0x56,0x34,0x12,0xaf]
 
 s_mul_hi_u32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, s1, s2                 ; encoding: [0x01,0x02,0x80,0x96]
 
 s_mul_hi_u32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0xe9,0x96]
+// GFX12: s_mul_hi_u32 s105, s104, s103           ; encoding: [0x68,0x67,0xe9,0x96]
 
 s_mul_hi_u32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, s104, s103             ; encoding: [0x68,0x67,0x80,0x96]
 
 s_mul_hi_u32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0xe9,0x96]
+// GFX12: s_mul_hi_u32 s105, s1, s103             ; encoding: [0x01,0x67,0xe9,0x96]
 
 s_mul_hi_u32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0xe9,0x96]
+// GFX12: s_mul_hi_u32 s105, s104, s2             ; encoding: [0x68,0x02,0xe9,0x96]
 
 s_mul_hi_u32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0xe9,0x96]
+// GFX12: s_mul_hi_u32 s105, s1, s2               ; encoding: [0x01,0x02,0xe9,0x96]
 
 s_mul_hi_u32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, s104, s2               ; encoding: [0x68,0x02,0x80,0x96]
 
 s_mul_hi_u32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, s1, s103               ; encoding: [0x01,0x67,0x80,0x96]
 
 s_mul_hi_u32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfe,0x96]
+// GFX12: s_mul_hi_u32 exec_lo, s1, s2            ; encoding: [0x01,0x02,0xfe,0x96]
 
 s_mul_hi_u32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xff,0x96]
+// GFX12: s_mul_hi_u32 exec_hi, s1, s2            ; encoding: [0x01,0x02,0xff,0x96]
 
 s_mul_hi_u32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0xea,0x96]
+// GFX12: s_mul_hi_u32 vcc_lo, s1, s2             ; encoding: [0x01,0x02,0xea,0x96]
 
 s_mul_hi_u32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0xeb,0x96]
+// GFX12: s_mul_hi_u32 vcc_hi, s1, s2             ; encoding: [0x01,0x02,0xeb,0x96]
 
 s_mul_hi_u32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0xfd,0x96]
+// GFX12: s_mul_hi_u32 m0, s1, s2                 ; encoding: [0x01,0x02,0xfd,0x96]
 
 s_mul_hi_u32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, exec_lo, s2            ; encoding: [0x7e,0x02,0x80,0x96]
 
 s_mul_hi_u32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, exec_hi, s2            ; encoding: [0x7f,0x02,0x80,0x96]
 
 s_mul_hi_u32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, vcc_lo, s2             ; encoding: [0x6a,0x02,0x80,0x96]
 
 s_mul_hi_u32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, vcc_hi, s2             ; encoding: [0x6b,0x02,0x80,0x96]
 
 s_mul_hi_u32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, m0, s2                 ; encoding: [0x7d,0x02,0x80,0x96]
 
 s_mul_hi_u32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, 0, s2                  ; encoding: [0x80,0x02,0x80,0x96]
 
 s_mul_hi_u32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, -1, s2                 ; encoding: [0xc1,0x02,0x80,0x96]
 
 s_mul_hi_u32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, 0.5, s2                ; encoding: [0xf0,0x02,0x80,0x96]
 
 s_mul_hi_u32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, -4.0, s2               ; encoding: [0xf7,0x02,0x80,0x96]
 
 s_mul_hi_u32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x96,0x73,0x72,0x71,0x3f]
+// GFX12: s_mul_hi_u32 s0, 0x3f717273, s2         ; encoding: [0xff,0x02,0x80,0x96,0x73,0x72,0x71,0x3f]
 
 s_mul_hi_u32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x80,0x96,0x56,0x34,0x12,0xaf]
+// GFX12: s_mul_hi_u32 s0, 0xaf123456, s2         ; encoding: [0xff,0x02,0x80,0x96,0x56,0x34,0x12,0xaf]
 
 s_mul_hi_u32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, s1, exec_lo            ; encoding: [0x01,0x7e,0x80,0x96]
 
 s_mul_hi_u32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, s1, exec_hi            ; encoding: [0x01,0x7f,0x80,0x96]
 
 s_mul_hi_u32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, s1, vcc_lo             ; encoding: [0x01,0x6a,0x80,0x96]
 
 s_mul_hi_u32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, s1, vcc_hi             ; encoding: [0x01,0x6b,0x80,0x96]
 
 s_mul_hi_u32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, s1, m0                 ; encoding: [0x01,0x7d,0x80,0x96]
 
 s_mul_hi_u32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, s1, 0                  ; encoding: [0x01,0x80,0x80,0x96]
 
 s_mul_hi_u32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, s1, -1                 ; encoding: [0x01,0xc1,0x80,0x96]
 
 s_mul_hi_u32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, s1, 0.5                ; encoding: [0x01,0xf0,0x80,0x96]
 
 s_mul_hi_u32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x80,0x96]
+// GFX12: s_mul_hi_u32 s0, s1, -4.0               ; encoding: [0x01,0xf7,0x80,0x96]
 
 s_mul_hi_u32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x80,0x96,0x73,0x72,0x71,0x3f]
+// GFX12: s_mul_hi_u32 s0, s1, 0x3f717273         ; encoding: [0x01,0xff,0x80,0x96,0x73,0x72,0x71,0x3f]
 
 s_mul_hi_u32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x80,0x96,0x56,0x34,0x12,0xaf]
+// GFX12: s_mul_hi_u32 s0, s1, 0xaf123456         ; encoding: [0x01,0xff,0x80,0x96,0x56,0x34,0x12,0xaf]
 
 s_mul_hi_i32 s0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, s1, s2                 ; encoding: [0x01,0x02,0x00,0x97]
 
 s_mul_hi_i32 s105, s104, s103
-// GFX12: encoding: [0x68,0x67,0x69,0x97]
+// GFX12: s_mul_hi_i32 s105, s104, s103           ; encoding: [0x68,0x67,0x69,0x97]
 
 s_mul_hi_i32 s0, s104, s103
-// GFX12: encoding: [0x68,0x67,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, s104, s103             ; encoding: [0x68,0x67,0x00,0x97]
 
 s_mul_hi_i32 s105, s1, s103
-// GFX12: encoding: [0x01,0x67,0x69,0x97]
+// GFX12: s_mul_hi_i32 s105, s1, s103             ; encoding: [0x01,0x67,0x69,0x97]
 
 s_mul_hi_i32 s105, s104, s2
-// GFX12: encoding: [0x68,0x02,0x69,0x97]
+// GFX12: s_mul_hi_i32 s105, s104, s2             ; encoding: [0x68,0x02,0x69,0x97]
 
 s_mul_hi_i32 s105, s1, s2
-// GFX12: encoding: [0x01,0x02,0x69,0x97]
+// GFX12: s_mul_hi_i32 s105, s1, s2               ; encoding: [0x01,0x02,0x69,0x97]
 
 s_mul_hi_i32 s0, s104, s2
-// GFX12: encoding: [0x68,0x02,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, s104, s2               ; encoding: [0x68,0x02,0x00,0x97]
 
 s_mul_hi_i32 s0, s1, s103
-// GFX12: encoding: [0x01,0x67,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, s1, s103               ; encoding: [0x01,0x67,0x00,0x97]
 
 s_mul_hi_i32 exec_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7e,0x97]
+// GFX12: s_mul_hi_i32 exec_lo, s1, s2            ; encoding: [0x01,0x02,0x7e,0x97]
 
 s_mul_hi_i32 exec_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7f,0x97]
+// GFX12: s_mul_hi_i32 exec_hi, s1, s2            ; encoding: [0x01,0x02,0x7f,0x97]
 
 s_mul_hi_i32 vcc_lo, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6a,0x97]
+// GFX12: s_mul_hi_i32 vcc_lo, s1, s2             ; encoding: [0x01,0x02,0x6a,0x97]
 
 s_mul_hi_i32 vcc_hi, s1, s2
-// GFX12: encoding: [0x01,0x02,0x6b,0x97]
+// GFX12: s_mul_hi_i32 vcc_hi, s1, s2             ; encoding: [0x01,0x02,0x6b,0x97]
 
 s_mul_hi_i32 m0, s1, s2
-// GFX12: encoding: [0x01,0x02,0x7d,0x97]
+// GFX12: s_mul_hi_i32 m0, s1, s2                 ; encoding: [0x01,0x02,0x7d,0x97]
 
 s_mul_hi_i32 s0, exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, exec_lo, s2            ; encoding: [0x7e,0x02,0x00,0x97]
 
 s_mul_hi_i32 s0, exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, exec_hi, s2            ; encoding: [0x7f,0x02,0x00,0x97]
 
 s_mul_hi_i32 s0, vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, vcc_lo, s2             ; encoding: [0x6a,0x02,0x00,0x97]
 
 s_mul_hi_i32 s0, vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, vcc_hi, s2             ; encoding: [0x6b,0x02,0x00,0x97]
 
 s_mul_hi_i32 s0, m0, s2
-// GFX12: encoding: [0x7d,0x02,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, m0, s2                 ; encoding: [0x7d,0x02,0x00,0x97]
 
 s_mul_hi_i32 s0, 0, s2
-// GFX12: encoding: [0x80,0x02,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, 0, s2                  ; encoding: [0x80,0x02,0x00,0x97]
 
 s_mul_hi_i32 s0, -1, s2
-// GFX12: encoding: [0xc1,0x02,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, -1, s2                 ; encoding: [0xc1,0x02,0x00,0x97]
 
 s_mul_hi_i32 s0, 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, 0.5, s2                ; encoding: [0xf0,0x02,0x00,0x97]
 
 s_mul_hi_i32 s0, -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, -4.0, s2               ; encoding: [0xf7,0x02,0x00,0x97]
 
 s_mul_hi_i32 s0, 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x97,0x73,0x72,0x71,0x3f]
+// GFX12: s_mul_hi_i32 s0, 0x3f717273, s2         ; encoding: [0xff,0x02,0x00,0x97,0x73,0x72,0x71,0x3f]
 
 s_mul_hi_i32 s0, 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x00,0x97,0x56,0x34,0x12,0xaf]
+// GFX12: s_mul_hi_i32 s0, 0xaf123456, s2         ; encoding: [0xff,0x02,0x00,0x97,0x56,0x34,0x12,0xaf]
 
 s_mul_hi_i32 s0, s1, exec_lo
-// GFX12: encoding: [0x01,0x7e,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, s1, exec_lo            ; encoding: [0x01,0x7e,0x00,0x97]
 
 s_mul_hi_i32 s0, s1, exec_hi
-// GFX12: encoding: [0x01,0x7f,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, s1, exec_hi            ; encoding: [0x01,0x7f,0x00,0x97]
 
 s_mul_hi_i32 s0, s1, vcc_lo
-// GFX12: encoding: [0x01,0x6a,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, s1, vcc_lo             ; encoding: [0x01,0x6a,0x00,0x97]
 
 s_mul_hi_i32 s0, s1, vcc_hi
-// GFX12: encoding: [0x01,0x6b,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, s1, vcc_hi             ; encoding: [0x01,0x6b,0x00,0x97]
 
 s_mul_hi_i32 s0, s1, m0
-// GFX12: encoding: [0x01,0x7d,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, s1, m0                 ; encoding: [0x01,0x7d,0x00,0x97]
 
 s_mul_hi_i32 s0, s1, 0
-// GFX12: encoding: [0x01,0x80,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, s1, 0                  ; encoding: [0x01,0x80,0x00,0x97]
 
 s_mul_hi_i32 s0, s1, -1
-// GFX12: encoding: [0x01,0xc1,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, s1, -1                 ; encoding: [0x01,0xc1,0x00,0x97]
 
 s_mul_hi_i32 s0, s1, 0.5
-// GFX12: encoding: [0x01,0xf0,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, s1, 0.5                ; encoding: [0x01,0xf0,0x00,0x97]
 
 s_mul_hi_i32 s0, s1, -4.0
-// GFX12: encoding: [0x01,0xf7,0x00,0x97]
+// GFX12: s_mul_hi_i32 s0, s1, -4.0               ; encoding: [0x01,0xf7,0x00,0x97]
 
 s_mul_hi_i32 s0, s1, 0x3f717273
-// GFX12: encoding: [0x01,0xff,0x00,0x97,0x73,0x72,0x71,0x3f]
+// GFX12: s_mul_hi_i32 s0, s1, 0x3f717273         ; encoding: [0x01,0xff,0x00,0x97,0x73,0x72,0x71,0x3f]
 
 s_mul_hi_i32 s0, s1, 0xaf123456
-// GFX12: encoding: [0x01,0xff,0x00,0x97,0x56,0x34,0x12,0xaf]
+// GFX12: s_mul_hi_i32 s0, s1, 0xaf123456         ; encoding: [0x01,0xff,0x00,0x97,0x56,0x34,0x12,0xaf]
 
 s_or_not1_b32 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x05,0x92]
+// GFX12: s_or_not1_b32 s5, s1, s2                ; encoding: [0x01,0x02,0x05,0x92]
 
 s_or_not1_b32 s5, s105, s105
-// GFX12: encoding: [0x69,0x69,0x05,0x92]
+// GFX12: s_or_not1_b32 s5, s105, s105            ; encoding: [0x69,0x69,0x05,0x92]
 
 s_or_not1_b32 s5, vcc_lo, ttmp15
-// GFX12: encoding: [0x6a,0x7b,0x05,0x92]
+// GFX12: s_or_not1_b32 s5, vcc_lo, ttmp15        ; encoding: [0x6a,0x7b,0x05,0x92]
 
 s_or_not1_b32 s5, vcc_hi, 0xaf123456
-// GFX12: encoding: [0x6b,0xff,0x05,0x92,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_not1_b32 s5, vcc_hi, 0xaf123456    ; encoding: [0x6b,0xff,0x05,0x92,0x56,0x34,0x12,0xaf]
 
 s_or_not1_b32 s5, ttmp15, src_scc
-// GFX12: encoding: [0x7b,0xfd,0x05,0x92]
+// GFX12: s_or_not1_b32 s5, ttmp15, src_scc       ; encoding: [0x7b,0xfd,0x05,0x92]
 
 s_or_not1_b32 s105, m0, 0.5
-// GFX12: encoding: [0x7d,0xf0,0x69,0x92]
+// GFX12: s_or_not1_b32 s105, m0, 0.5             ; encoding: [0x7d,0xf0,0x69,0x92]
 
 s_or_not1_b32 vcc_lo, exec_lo, -1
-// GFX12: encoding: [0x7e,0xc1,0x6a,0x92]
+// GFX12: s_or_not1_b32 vcc_lo, exec_lo, -1       ; encoding: [0x7e,0xc1,0x6a,0x92]
 
 s_or_not1_b32 vcc_hi, exec_hi, null
-// GFX12: encoding: [0x7f,0x7c,0x6b,0x92]
+// GFX12: s_or_not1_b32 vcc_hi, exec_hi, null     ; encoding: [0x7f,0x7c,0x6b,0x92]
 
 s_or_not1_b32 ttmp15, null, exec_lo
-// GFX12: encoding: [0x7c,0x7e,0x7b,0x92]
+// GFX12: s_or_not1_b32 ttmp15, null, exec_lo     ; encoding: [0x7c,0x7e,0x7b,0x92]
 
 s_or_not1_b32 m0, -1, exec_hi
-// GFX12: encoding: [0xc1,0x7f,0x7d,0x92]
+// GFX12: s_or_not1_b32 m0, -1, exec_hi           ; encoding: [0xc1,0x7f,0x7d,0x92]
 
 s_or_not1_b32 exec_lo, 0.5, m0
-// GFX12: encoding: [0xf0,0x7d,0x7e,0x92]
+// GFX12: s_or_not1_b32 exec_lo, 0.5, m0          ; encoding: [0xf0,0x7d,0x7e,0x92]
 
 s_or_not1_b32 exec_hi, src_scc, vcc_lo
-// GFX12: encoding: [0xfd,0x6a,0x7f,0x92]
+// GFX12: s_or_not1_b32 exec_hi, src_scc, vcc_lo  ; encoding: [0xfd,0x6a,0x7f,0x92]
 
 s_or_not1_b32 null, 0xaf123456, vcc_hi
-// GFX12: encoding: [0xff,0x6b,0x7c,0x92,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_not1_b32 null, 0xaf123456, vcc_hi  ; encoding: [0xff,0x6b,0x7c,0x92,0x56,0x34,0x12,0xaf]
 
 s_pack_hl_b32_b16 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x85,0x9a]
+// GFX12: s_pack_hl_b32_b16 s5, s1, s2            ; encoding: [0x01,0x02,0x85,0x9a]
 
 s_pack_hl_b32_b16 s5, s105, s105
-// GFX12: encoding: [0x69,0x69,0x85,0x9a]
+// GFX12: s_pack_hl_b32_b16 s5, s105, s105        ; encoding: [0x69,0x69,0x85,0x9a]
 
 s_pack_hl_b32_b16 s5, vcc_lo, ttmp15
-// GFX12: encoding: [0x6a,0x7b,0x85,0x9a]
+// GFX12: s_pack_hl_b32_b16 s5, vcc_lo, ttmp15    ; encoding: [0x6a,0x7b,0x85,0x9a]
 
 s_pack_hl_b32_b16 s5, vcc_hi, 0xfe0b
-// GFX12: encoding: [0x6b,0xff,0x85,0x9a,0x0b,0xfe,0x00,0x00]
+// GFX12: s_pack_hl_b32_b16 s5, vcc_hi, 0xfe0b    ; encoding: [0x6b,0xff,0x85,0x9a,0x0b,0xfe,0x00,0x00]
 
 s_pack_hl_b32_b16 s5, ttmp15, src_scc
-// GFX12: encoding: [0x7b,0xfd,0x85,0x9a]
+// GFX12: s_pack_hl_b32_b16 s5, ttmp15, src_scc   ; encoding: [0x7b,0xfd,0x85,0x9a]
 
 s_pack_hl_b32_b16 s105, m0, 0.5
-// GFX12: encoding: [0x7d,0xf0,0xe9,0x9a]
+// GFX12: s_pack_hl_b32_b16 s105, m0, 0.5         ; encoding: [0x7d,0xf0,0xe9,0x9a]
 
 s_pack_hl_b32_b16 vcc_lo, exec_lo, -1
-// GFX12: encoding: [0x7e,0xc1,0xea,0x9a]
+// GFX12: s_pack_hl_b32_b16 vcc_lo, exec_lo, -1   ; encoding: [0x7e,0xc1,0xea,0x9a]
 
 s_pack_hl_b32_b16 vcc_hi, exec_hi, null
-// GFX12: encoding: [0x7f,0x7c,0xeb,0x9a]
+// GFX12: s_pack_hl_b32_b16 vcc_hi, exec_hi, null ; encoding: [0x7f,0x7c,0xeb,0x9a]
 
 s_pack_hl_b32_b16 ttmp15, null, exec_lo
-// GFX12: encoding: [0x7c,0x7e,0xfb,0x9a]
+// GFX12: s_pack_hl_b32_b16 ttmp15, null, exec_lo ; encoding: [0x7c,0x7e,0xfb,0x9a]
 
 s_pack_hl_b32_b16 m0, -1, exec_hi
-// GFX12: encoding: [0xc1,0x7f,0xfd,0x9a]
+// GFX12: s_pack_hl_b32_b16 m0, -1, exec_hi       ; encoding: [0xc1,0x7f,0xfd,0x9a]
 
 s_pack_hl_b32_b16 exec_lo, 0.5, m0
-// GFX12: encoding: [0xf0,0x7d,0xfe,0x9a]
+// GFX12: s_pack_hl_b32_b16 exec_lo, 0.5, m0      ; encoding: [0xf0,0x7d,0xfe,0x9a]
 
 s_pack_hl_b32_b16 exec_hi, src_scc, vcc_lo
-// GFX12: encoding: [0xfd,0x6a,0xff,0x9a]
+// GFX12: s_pack_hl_b32_b16 exec_hi, src_scc, vcc_lo ; encoding: [0xfd,0x6a,0xff,0x9a]
 
 s_pack_hl_b32_b16 null, 0xaf123456, vcc_hi
-// GFX12: encoding: [0xff,0x6b,0xfc,0x9a,0x56,0x34,0x12,0xaf]
+// GFX12: s_pack_hl_b32_b16 null, 0xaf123456, vcc_hi ; encoding: [0xff,0x6b,0xfc,0x9a,0x56,0x34,0x12,0xaf]
 
 s_and_not1_b32 s5, s1, s2
-// GFX12: encoding: [0x01,0x02,0x05,0x91]
+// GFX12: s_and_not1_b32 s5, s1, s2               ; encoding: [0x01,0x02,0x05,0x91]
 
 s_and_not1_b32 s5, s105, s105
-// GFX12: encoding: [0x69,0x69,0x05,0x91]
+// GFX12: s_and_not1_b32 s5, s105, s105           ; encoding: [0x69,0x69,0x05,0x91]
 
 s_and_not1_b32 s5, vcc_lo, ttmp15
-// GFX12: encoding: [0x6a,0x7b,0x05,0x91]
+// GFX12: s_and_not1_b32 s5, vcc_lo, ttmp15       ; encoding: [0x6a,0x7b,0x05,0x91]
 
 s_and_not1_b32 s5, vcc_hi, 0xaf123456
-// GFX12: encoding: [0x6b,0xff,0x05,0x91,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not1_b32 s5, vcc_hi, 0xaf123456   ; encoding: [0x6b,0xff,0x05,0x91,0x56,0x34,0x12,0xaf]
 
 s_and_not1_b32 s5, ttmp15, src_scc
-// GFX12: encoding: [0x7b,0xfd,0x05,0x91]
+// GFX12: s_and_not1_b32 s5, ttmp15, src_scc      ; encoding: [0x7b,0xfd,0x05,0x91]
 
 s_and_not1_b32 s105, m0, 0.5
-// GFX12: encoding: [0x7d,0xf0,0x69,0x91]
+// GFX12: s_and_not1_b32 s105, m0, 0.5            ; encoding: [0x7d,0xf0,0x69,0x91]
 
 s_and_not1_b32 vcc_lo, exec_lo, -1
-// GFX12: encoding: [0x7e,0xc1,0x6a,0x91]
+// GFX12: s_and_not1_b32 vcc_lo, exec_lo, -1      ; encoding: [0x7e,0xc1,0x6a,0x91]
 
 s_and_not1_b32 vcc_hi, exec_hi, null
-// GFX12: encoding: [0x7f,0x7c,0x6b,0x91]
+// GFX12: s_and_not1_b32 vcc_hi, exec_hi, null    ; encoding: [0x7f,0x7c,0x6b,0x91]
 
 s_and_not1_b32 ttmp15, null, exec_lo
-// GFX12: encoding: [0x7c,0x7e,0x7b,0x91]
+// GFX12: s_and_not1_b32 ttmp15, null, exec_lo    ; encoding: [0x7c,0x7e,0x7b,0x91]
 
 s_and_not1_b32 m0, -1, exec_hi
-// GFX12: encoding: [0xc1,0x7f,0x7d,0x91]
+// GFX12: s_and_not1_b32 m0, -1, exec_hi          ; encoding: [0xc1,0x7f,0x7d,0x91]
 
 s_and_not1_b32 exec_lo, 0.5, m0
-// GFX12: encoding: [0xf0,0x7d,0x7e,0x91]
+// GFX12: s_and_not1_b32 exec_lo, 0.5, m0         ; encoding: [0xf0,0x7d,0x7e,0x91]
 
 s_and_not1_b32 exec_hi, src_scc, vcc_lo
-// GFX12: encoding: [0xfd,0x6a,0x7f,0x91]
+// GFX12: s_and_not1_b32 exec_hi, src_scc, vcc_lo ; encoding: [0xfd,0x6a,0x7f,0x91]
 
 s_and_not1_b32 null, 0xaf123456, vcc_hi
-// GFX12: encoding: [0xff,0x6b,0x7c,0x91,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not1_b32 null, 0xaf123456, vcc_hi ; encoding: [0xff,0x6b,0x7c,0x91,0x56,0x34,0x12,0xaf]
 
 s_and_not1_b64 s[10:11], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0x8a,0x91]
+// GFX12: s_and_not1_b64 s[10:11], s[2:3], s[4:5] ; encoding: [0x02,0x04,0x8a,0x91]
 
 s_and_not1_b64 s[10:11], s[104:105], s[104:105]
-// GFX12: encoding: [0x68,0x68,0x8a,0x91]
+// GFX12: s_and_not1_b64 s[10:11], s[104:105], s[104:105] ; encoding: [0x68,0x68,0x8a,0x91]
 
 s_and_not1_b64 s[10:11], vcc, ttmp[14:15]
-// GFX12: encoding: [0x6a,0x7a,0x8a,0x91]
+// GFX12: s_and_not1_b64 s[10:11], vcc, ttmp[14:15] ; encoding: [0x6a,0x7a,0x8a,0x91]
 
 s_and_not1_b64 s[10:11], ttmp[14:15], 0xaf123456
-// GFX12: encoding: [0x7a,0xff,0x8a,0x91,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not1_b64 s[10:11], ttmp[14:15], 0xaf123456 ; encoding: [0x7a,0xff,0x8a,0x91,0x56,0x34,0x12,0xaf]
 
 s_and_not1_b64 s[10:11], exec, src_scc
-// GFX12: encoding: [0x7e,0xfd,0x8a,0x91]
+// GFX12: s_and_not1_b64 s[10:11], exec, src_scc  ; encoding: [0x7e,0xfd,0x8a,0x91]
 
 s_and_not1_b64 s[104:105], null, 0.5
-// GFX12: encoding: [0x7c,0xf0,0xe8,0x91]
+// GFX12: s_and_not1_b64 s[104:105], null, 0.5    ; encoding: [0x7c,0xf0,0xe8,0x91]
 
 s_and_not1_b64 vcc, -1, -1
-// GFX12: encoding: [0xc1,0xc1,0xea,0x91]
+// GFX12: s_and_not1_b64 vcc, -1, -1              ; encoding: [0xc1,0xc1,0xea,0x91]
 
 s_and_not1_b64 ttmp[14:15], 0.5, null
-// GFX12: encoding: [0xf0,0x7c,0xfa,0x91]
+// GFX12: s_and_not1_b64 ttmp[14:15], 0.5, null   ; encoding: [0xf0,0x7c,0xfa,0x91]
 
 s_and_not1_b64 exec, src_scc, exec
-// GFX12: encoding: [0xfd,0x7e,0xfe,0x91]
+// GFX12: s_and_not1_b64 exec, src_scc, exec      ; encoding: [0xfd,0x7e,0xfe,0x91]
 
 s_and_not1_b64 null, 0xaf123456, vcc
-// GFX12: encoding: [0xff,0x6a,0xfc,0x91,0x56,0x34,0x12,0xaf]
+// GFX12: s_and_not1_b64 null, 0xaf123456, vcc    ; encoding: [0xff,0x6a,0xfc,0x91,0x56,0x34,0x12,0xaf]
 
 s_or_not1_b64 s[10:11], s[2:3], s[4:5]
-// GFX12: encoding: [0x02,0x04,0x8a,0x92]
+// GFX12: s_or_not1_b64 s[10:11], s[2:3], s[4:5]  ; encoding: [0x02,0x04,0x8a,0x92]
 
 s_or_not1_b64 s[10:11], s[104:105], s[104:105]
-// GFX12: encoding: [0x68,0x68,0x8a,0x92]
+// GFX12: s_or_not1_b64 s[10:11], s[104:105], s[104:105] ; encoding: [0x68,0x68,0x8a,0x92]
 
 s_or_not1_b64 s[10:11], vcc, ttmp[14:15]
-// GFX12: encoding: [0x6a,0x7a,0x8a,0x92]
+// GFX12: s_or_not1_b64 s[10:11], vcc, ttmp[14:15] ; encoding: [0x6a,0x7a,0x8a,0x92]
 
 s_or_not1_b64 s[10:11], ttmp[14:15], 0xaf123456
-// GFX12: encoding: [0x7a,0xff,0x8a,0x92,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_not1_b64 s[10:11], ttmp[14:15], 0xaf123456 ; encoding: [0x7a,0xff,0x8a,0x92,0x56,0x34,0x12,0xaf]
 
 s_or_not1_b64 s[10:11], exec, src_scc
-// GFX12: encoding: [0x7e,0xfd,0x8a,0x92]
+// GFX12: s_or_not1_b64 s[10:11], exec, src_scc   ; encoding: [0x7e,0xfd,0x8a,0x92]
 
 s_or_not1_b64 s[104:105], null, 0.5
-// GFX12: encoding: [0x7c,0xf0,0xe8,0x92]
+// GFX12: s_or_not1_b64 s[104:105], null, 0.5     ; encoding: [0x7c,0xf0,0xe8,0x92]
 
 s_or_not1_b64 vcc, -1, -1
-// GFX12: encoding: [0xc1,0xc1,0xea,0x92]
+// GFX12: s_or_not1_b64 vcc, -1, -1               ; encoding: [0xc1,0xc1,0xea,0x92]
 
 s_or_not1_b64 ttmp[14:15], 0.5, null
-// GFX12: encoding: [0xf0,0x7c,0xfa,0x92]
+// GFX12: s_or_not1_b64 ttmp[14:15], 0.5, null    ; encoding: [0xf0,0x7c,0xfa,0x92]
 
 s_or_not1_b64 exec, src_scc, exec
-// GFX12: encoding: [0xfd,0x7e,0xfe,0x92]
+// GFX12: s_or_not1_b64 exec, src_scc, exec       ; encoding: [0xfd,0x7e,0xfe,0x92]
 
 s_or_not1_b64 null, 0xaf123456, vcc
-// GFX12: encoding: [0xff,0x6a,0xfc,0x92,0x56,0x34,0x12,0xaf]
+// GFX12: s_or_not1_b64 null, 0xaf123456, vcc     ; encoding: [0xff,0x6a,0xfc,0x92,0x56,0x34,0x12,0xaf]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sopc.s b/llvm/test/MC/AMDGPU/gfx12_asm_sopc.s
index 56a95d504b93d..6d8d6dc16fd09 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_sopc.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_sopc.s
@@ -1,2161 +1,2162 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
 
 s_cmp_lt_f32 s1, s2
-// GFX12: encoding: [0x01,0x02,0x41,0xbf]
+// GFX12: s_cmp_lt_f32 s1, s2                     ; encoding: [0x01,0x02,0x41,0xbf]
 
 s_cmp_lt_f32 s105, s2
-// GFX12: encoding: [0x69,0x02,0x41,0xbf]
+// GFX12: s_cmp_lt_f32 s105, s2                   ; encoding: [0x69,0x02,0x41,0xbf]
 
 s_cmp_lt_f32 s101, s2
-// GFX12: encoding: [0x65,0x02,0x41,0xbf]
+// GFX12: s_cmp_lt_f32 s101, s2                   ; encoding: [0x65,0x02,0x41,0xbf]
 
 s_cmp_lt_f32 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x41,0xbf]
+// GFX12: s_cmp_lt_f32 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x41,0xbf]
 
 s_cmp_lt_f32 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x41,0xbf]
+// GFX12: s_cmp_lt_f32 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x41,0xbf]
 
 s_cmp_lt_f32 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x41,0xbf]
+// GFX12: s_cmp_lt_f32 m0, s2                     ; encoding: [0x7d,0x02,0x41,0xbf]
 
 s_cmp_lt_f32 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x41,0xbf]
+// GFX12: s_cmp_lt_f32 exec_lo, s2                ; encoding: [0x7e,0x02,0x41,0xbf]
 
 s_cmp_lt_f32 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x41,0xbf]
+// GFX12: s_cmp_lt_f32 exec_hi, s2                ; encoding: [0x7f,0x02,0x41,0xbf]
 
 s_cmp_lt_f32 0, s2
-// GFX12: encoding: [0x80,0x02,0x41,0xbf]
+// GFX12: s_cmp_lt_f32 0, s2                      ; encoding: [0x80,0x02,0x41,0xbf]
 
 s_cmp_lt_f32 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x41,0xbf]
+// GFX12: s_cmp_lt_f32 -1, s2                     ; encoding: [0xc1,0x02,0x41,0xbf]
 
 s_cmp_lt_f32 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x41,0xbf]
+// GFX12: s_cmp_lt_f32 0.5, s2                    ; encoding: [0xf0,0x02,0x41,0xbf]
 
 s_cmp_lt_f32 -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x41,0xbf]
+// GFX12: s_cmp_lt_f32 -4.0, s2                   ; encoding: [0xf7,0x02,0x41,0xbf]
 
 s_cmp_lt_f32 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x41,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_lt_f32 0xaf123456, s2             ; encoding: [0xff,0x02,0x41,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_lt_f32 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x41,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_lt_f32 0x3f717273, s2             ; encoding: [0xff,0x02,0x41,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_lt_f32 s1, s105
-// GFX12: encoding: [0x01,0x69,0x41,0xbf]
+// GFX12: s_cmp_lt_f32 s1, s105                   ; encoding: [0x01,0x69,0x41,0xbf]
 
 s_cmp_eq_f32 s1, s2
-// GFX12: encoding: [0x01,0x02,0x42,0xbf]
+// GFX12: s_cmp_eq_f32 s1, s2                     ; encoding: [0x01,0x02,0x42,0xbf]
 
 s_cmp_eq_f32 s105, s2
-// GFX12: encoding: [0x69,0x02,0x42,0xbf]
+// GFX12: s_cmp_eq_f32 s105, s2                   ; encoding: [0x69,0x02,0x42,0xbf]
 
 s_cmp_eq_f32 s101, s2
-// GFX12: encoding: [0x65,0x02,0x42,0xbf]
+// GFX12: s_cmp_eq_f32 s101, s2                   ; encoding: [0x65,0x02,0x42,0xbf]
 
 s_cmp_eq_f32 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x42,0xbf]
+// GFX12: s_cmp_eq_f32 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x42,0xbf]
 
 s_cmp_eq_f32 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x42,0xbf]
+// GFX12: s_cmp_eq_f32 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x42,0xbf]
 
 s_cmp_eq_f32 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x42,0xbf]
+// GFX12: s_cmp_eq_f32 m0, s2                     ; encoding: [0x7d,0x02,0x42,0xbf]
 
 s_cmp_eq_f32 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x42,0xbf]
+// GFX12: s_cmp_eq_f32 exec_lo, s2                ; encoding: [0x7e,0x02,0x42,0xbf]
 
 s_cmp_eq_f32 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x42,0xbf]
+// GFX12: s_cmp_eq_f32 exec_hi, s2                ; encoding: [0x7f,0x02,0x42,0xbf]
 
 s_cmp_eq_f32 0, s2
-// GFX12: encoding: [0x80,0x02,0x42,0xbf]
+// GFX12: s_cmp_eq_f32 0, s2                      ; encoding: [0x80,0x02,0x42,0xbf]
 
 s_cmp_eq_f32 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x42,0xbf]
+// GFX12: s_cmp_eq_f32 -1, s2                     ; encoding: [0xc1,0x02,0x42,0xbf]
 
 s_cmp_eq_f32 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x42,0xbf]
+// GFX12: s_cmp_eq_f32 0.5, s2                    ; encoding: [0xf0,0x02,0x42,0xbf]
 
 s_cmp_eq_f32 -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x42,0xbf]
+// GFX12: s_cmp_eq_f32 -4.0, s2                   ; encoding: [0xf7,0x02,0x42,0xbf]
 
 s_cmp_eq_f32 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x42,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_eq_f32 0xaf123456, s2             ; encoding: [0xff,0x02,0x42,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_eq_f32 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x42,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_eq_f32 0x3f717273, s2             ; encoding: [0xff,0x02,0x42,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_eq_f32 s1, s105
-// GFX12: encoding: [0x01,0x69,0x42,0xbf]
+// GFX12: s_cmp_eq_f32 s1, s105                   ; encoding: [0x01,0x69,0x42,0xbf]
 
 s_cmp_le_f32 s1, s2
-// GFX12: encoding: [0x01,0x02,0x43,0xbf]
+// GFX12: s_cmp_le_f32 s1, s2                     ; encoding: [0x01,0x02,0x43,0xbf]
 
 s_cmp_le_f32 s105, s2
-// GFX12: encoding: [0x69,0x02,0x43,0xbf]
+// GFX12: s_cmp_le_f32 s105, s2                   ; encoding: [0x69,0x02,0x43,0xbf]
 
 s_cmp_le_f32 s101, s2
-// GFX12: encoding: [0x65,0x02,0x43,0xbf]
+// GFX12: s_cmp_le_f32 s101, s2                   ; encoding: [0x65,0x02,0x43,0xbf]
 
 s_cmp_le_f32 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x43,0xbf]
+// GFX12: s_cmp_le_f32 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x43,0xbf]
 
 s_cmp_le_f32 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x43,0xbf]
+// GFX12: s_cmp_le_f32 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x43,0xbf]
 
 s_cmp_le_f32 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x43,0xbf]
+// GFX12: s_cmp_le_f32 m0, s2                     ; encoding: [0x7d,0x02,0x43,0xbf]
 
 s_cmp_le_f32 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x43,0xbf]
+// GFX12: s_cmp_le_f32 exec_lo, s2                ; encoding: [0x7e,0x02,0x43,0xbf]
 
 s_cmp_le_f32 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x43,0xbf]
+// GFX12: s_cmp_le_f32 exec_hi, s2                ; encoding: [0x7f,0x02,0x43,0xbf]
 
 s_cmp_le_f32 0, s2
-// GFX12: encoding: [0x80,0x02,0x43,0xbf]
+// GFX12: s_cmp_le_f32 0, s2                      ; encoding: [0x80,0x02,0x43,0xbf]
 
 s_cmp_le_f32 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x43,0xbf]
+// GFX12: s_cmp_le_f32 -1, s2                     ; encoding: [0xc1,0x02,0x43,0xbf]
 
 s_cmp_le_f32 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x43,0xbf]
+// GFX12: s_cmp_le_f32 0.5, s2                    ; encoding: [0xf0,0x02,0x43,0xbf]
 
 s_cmp_le_f32 -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x43,0xbf]
+// GFX12: s_cmp_le_f32 -4.0, s2                   ; encoding: [0xf7,0x02,0x43,0xbf]
 
 s_cmp_le_f32 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x43,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_le_f32 0xaf123456, s2             ; encoding: [0xff,0x02,0x43,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_le_f32 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x43,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_le_f32 0x3f717273, s2             ; encoding: [0xff,0x02,0x43,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_le_f32 s1, s105
-// GFX12: encoding: [0x01,0x69,0x43,0xbf]
+// GFX12: s_cmp_le_f32 s1, s105                   ; encoding: [0x01,0x69,0x43,0xbf]
 
 s_cmp_gt_f32 s1, s2
-// GFX12: encoding: [0x01,0x02,0x44,0xbf]
+// GFX12: s_cmp_gt_f32 s1, s2                     ; encoding: [0x01,0x02,0x44,0xbf]
 
 s_cmp_gt_f32 s105, s2
-// GFX12: encoding: [0x69,0x02,0x44,0xbf]
+// GFX12: s_cmp_gt_f32 s105, s2                   ; encoding: [0x69,0x02,0x44,0xbf]
 
 s_cmp_gt_f32 s101, s2
-// GFX12: encoding: [0x65,0x02,0x44,0xbf]
+// GFX12: s_cmp_gt_f32 s101, s2                   ; encoding: [0x65,0x02,0x44,0xbf]
 
 s_cmp_gt_f32 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x44,0xbf]
+// GFX12: s_cmp_gt_f32 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x44,0xbf]
 
 s_cmp_gt_f32 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x44,0xbf]
+// GFX12: s_cmp_gt_f32 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x44,0xbf]
 
 s_cmp_gt_f32 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x44,0xbf]
+// GFX12: s_cmp_gt_f32 m0, s2                     ; encoding: [0x7d,0x02,0x44,0xbf]
 
 s_cmp_gt_f32 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x44,0xbf]
+// GFX12: s_cmp_gt_f32 exec_lo, s2                ; encoding: [0x7e,0x02,0x44,0xbf]
 
 s_cmp_gt_f32 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x44,0xbf]
+// GFX12: s_cmp_gt_f32 exec_hi, s2                ; encoding: [0x7f,0x02,0x44,0xbf]
 
 s_cmp_gt_f32 0, s2
-// GFX12: encoding: [0x80,0x02,0x44,0xbf]
+// GFX12: s_cmp_gt_f32 0, s2                      ; encoding: [0x80,0x02,0x44,0xbf]
 
 s_cmp_gt_f32 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x44,0xbf]
+// GFX12: s_cmp_gt_f32 -1, s2                     ; encoding: [0xc1,0x02,0x44,0xbf]
 
 s_cmp_gt_f32 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x44,0xbf]
+// GFX12: s_cmp_gt_f32 0.5, s2                    ; encoding: [0xf0,0x02,0x44,0xbf]
 
 s_cmp_gt_f32 -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x44,0xbf]
+// GFX12: s_cmp_gt_f32 -4.0, s2                   ; encoding: [0xf7,0x02,0x44,0xbf]
 
 s_cmp_gt_f32 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x44,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_gt_f32 0xaf123456, s2             ; encoding: [0xff,0x02,0x44,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_gt_f32 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x44,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_gt_f32 0x3f717273, s2             ; encoding: [0xff,0x02,0x44,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_gt_f32 s1, s105
-// GFX12: encoding: [0x01,0x69,0x44,0xbf]
+// GFX12: s_cmp_gt_f32 s1, s105                   ; encoding: [0x01,0x69,0x44,0xbf]
 
 s_cmp_lg_f32 s1, s2
-// GFX12: encoding: [0x01,0x02,0x45,0xbf]
+// GFX12: s_cmp_lg_f32 s1, s2                     ; encoding: [0x01,0x02,0x45,0xbf]
 
 s_cmp_lg_f32 s105, s2
-// GFX12: encoding: [0x69,0x02,0x45,0xbf]
+// GFX12: s_cmp_lg_f32 s105, s2                   ; encoding: [0x69,0x02,0x45,0xbf]
 
 s_cmp_lg_f32 s101, s2
-// GFX12: encoding: [0x65,0x02,0x45,0xbf]
+// GFX12: s_cmp_lg_f32 s101, s2                   ; encoding: [0x65,0x02,0x45,0xbf]
 
 s_cmp_lg_f32 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x45,0xbf]
+// GFX12: s_cmp_lg_f32 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x45,0xbf]
 
 s_cmp_lg_f32 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x45,0xbf]
+// GFX12: s_cmp_lg_f32 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x45,0xbf]
 
 s_cmp_lg_f32 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x45,0xbf]
+// GFX12: s_cmp_lg_f32 m0, s2                     ; encoding: [0x7d,0x02,0x45,0xbf]
 
 s_cmp_lg_f32 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x45,0xbf]
+// GFX12: s_cmp_lg_f32 exec_lo, s2                ; encoding: [0x7e,0x02,0x45,0xbf]
 
 s_cmp_lg_f32 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x45,0xbf]
+// GFX12: s_cmp_lg_f32 exec_hi, s2                ; encoding: [0x7f,0x02,0x45,0xbf]
 
 s_cmp_lg_f32 0, s2
-// GFX12: encoding: [0x80,0x02,0x45,0xbf]
+// GFX12: s_cmp_lg_f32 0, s2                      ; encoding: [0x80,0x02,0x45,0xbf]
 
 s_cmp_lg_f32 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x45,0xbf]
+// GFX12: s_cmp_lg_f32 -1, s2                     ; encoding: [0xc1,0x02,0x45,0xbf]
 
 s_cmp_lg_f32 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x45,0xbf]
+// GFX12: s_cmp_lg_f32 0.5, s2                    ; encoding: [0xf0,0x02,0x45,0xbf]
 
 s_cmp_lg_f32 -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x45,0xbf]
+// GFX12: s_cmp_lg_f32 -4.0, s2                   ; encoding: [0xf7,0x02,0x45,0xbf]
 
 s_cmp_lg_f32 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x45,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_lg_f32 0xaf123456, s2             ; encoding: [0xff,0x02,0x45,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_lg_f32 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x45,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_lg_f32 0x3f717273, s2             ; encoding: [0xff,0x02,0x45,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_lg_f32 s1, s105
-// GFX12: encoding: [0x01,0x69,0x45,0xbf]
+// GFX12: s_cmp_lg_f32 s1, s105                   ; encoding: [0x01,0x69,0x45,0xbf]
 
 s_cmp_ge_f32 s1, s2
-// GFX12: encoding: [0x01,0x02,0x46,0xbf]
+// GFX12: s_cmp_ge_f32 s1, s2                     ; encoding: [0x01,0x02,0x46,0xbf]
 
 s_cmp_ge_f32 s105, s2
-// GFX12: encoding: [0x69,0x02,0x46,0xbf]
+// GFX12: s_cmp_ge_f32 s105, s2                   ; encoding: [0x69,0x02,0x46,0xbf]
 
 s_cmp_ge_f32 s101, s2
-// GFX12: encoding: [0x65,0x02,0x46,0xbf]
+// GFX12: s_cmp_ge_f32 s101, s2                   ; encoding: [0x65,0x02,0x46,0xbf]
 
 s_cmp_ge_f32 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x46,0xbf]
+// GFX12: s_cmp_ge_f32 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x46,0xbf]
 
 s_cmp_ge_f32 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x46,0xbf]
+// GFX12: s_cmp_ge_f32 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x46,0xbf]
 
 s_cmp_ge_f32 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x46,0xbf]
+// GFX12: s_cmp_ge_f32 m0, s2                     ; encoding: [0x7d,0x02,0x46,0xbf]
 
 s_cmp_ge_f32 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x46,0xbf]
+// GFX12: s_cmp_ge_f32 exec_lo, s2                ; encoding: [0x7e,0x02,0x46,0xbf]
 
 s_cmp_ge_f32 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x46,0xbf]
+// GFX12: s_cmp_ge_f32 exec_hi, s2                ; encoding: [0x7f,0x02,0x46,0xbf]
 
 s_cmp_ge_f32 0, s2
-// GFX12: encoding: [0x80,0x02,0x46,0xbf]
+// GFX12: s_cmp_ge_f32 0, s2                      ; encoding: [0x80,0x02,0x46,0xbf]
 
 s_cmp_ge_f32 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x46,0xbf]
+// GFX12: s_cmp_ge_f32 -1, s2                     ; encoding: [0xc1,0x02,0x46,0xbf]
 
 s_cmp_ge_f32 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x46,0xbf]
+// GFX12: s_cmp_ge_f32 0.5, s2                    ; encoding: [0xf0,0x02,0x46,0xbf]
 
 s_cmp_ge_f32 -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x46,0xbf]
+// GFX12: s_cmp_ge_f32 -4.0, s2                   ; encoding: [0xf7,0x02,0x46,0xbf]
 
 s_cmp_ge_f32 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x46,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_ge_f32 0xaf123456, s2             ; encoding: [0xff,0x02,0x46,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_ge_f32 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x46,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_ge_f32 0x3f717273, s2             ; encoding: [0xff,0x02,0x46,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_ge_f32 s1, s105
-// GFX12: encoding: [0x01,0x69,0x46,0xbf]
+// GFX12: s_cmp_ge_f32 s1, s105                   ; encoding: [0x01,0x69,0x46,0xbf]
 
 s_cmp_o_f32 s1, s2
-// GFX12: encoding: [0x01,0x02,0x47,0xbf]
+// GFX12: s_cmp_o_f32 s1, s2                      ; encoding: [0x01,0x02,0x47,0xbf]
 
 s_cmp_o_f32 s105, s2
-// GFX12: encoding: [0x69,0x02,0x47,0xbf]
+// GFX12: s_cmp_o_f32 s105, s2                    ; encoding: [0x69,0x02,0x47,0xbf]
 
 s_cmp_o_f32 s101, s2
-// GFX12: encoding: [0x65,0x02,0x47,0xbf]
+// GFX12: s_cmp_o_f32 s101, s2                    ; encoding: [0x65,0x02,0x47,0xbf]
 
 s_cmp_o_f32 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x47,0xbf]
+// GFX12: s_cmp_o_f32 vcc_lo, s2                  ; encoding: [0x6a,0x02,0x47,0xbf]
 
 s_cmp_o_f32 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x47,0xbf]
+// GFX12: s_cmp_o_f32 vcc_hi, s2                  ; encoding: [0x6b,0x02,0x47,0xbf]
 
 s_cmp_o_f32 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x47,0xbf]
+// GFX12: s_cmp_o_f32 m0, s2                      ; encoding: [0x7d,0x02,0x47,0xbf]
 
 s_cmp_o_f32 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x47,0xbf]
+// GFX12: s_cmp_o_f32 exec_lo, s2                 ; encoding: [0x7e,0x02,0x47,0xbf]
 
 s_cmp_o_f32 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x47,0xbf]
+// GFX12: s_cmp_o_f32 exec_hi, s2                 ; encoding: [0x7f,0x02,0x47,0xbf]
 
 s_cmp_o_f32 0, s2
-// GFX12: encoding: [0x80,0x02,0x47,0xbf]
+// GFX12: s_cmp_o_f32 0, s2                       ; encoding: [0x80,0x02,0x47,0xbf]
 
 s_cmp_o_f32 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x47,0xbf]
+// GFX12: s_cmp_o_f32 -1, s2                      ; encoding: [0xc1,0x02,0x47,0xbf]
 
 s_cmp_o_f32 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x47,0xbf]
+// GFX12: s_cmp_o_f32 0.5, s2                     ; encoding: [0xf0,0x02,0x47,0xbf]
 
 s_cmp_o_f32 -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x47,0xbf]
+// GFX12: s_cmp_o_f32 -4.0, s2                    ; encoding: [0xf7,0x02,0x47,0xbf]
 
 s_cmp_o_f32 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x47,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_o_f32 0xaf123456, s2              ; encoding: [0xff,0x02,0x47,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_o_f32 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x47,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_o_f32 0x3f717273, s2              ; encoding: [0xff,0x02,0x47,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_o_f32 s1, s105
-// GFX12: encoding: [0x01,0x69,0x47,0xbf]
+// GFX12: s_cmp_o_f32 s1, s105                    ; encoding: [0x01,0x69,0x47,0xbf]
 
 s_cmp_u_f32 s1, s2
-// GFX12: encoding: [0x01,0x02,0x48,0xbf]
+// GFX12: s_cmp_u_f32 s1, s2                      ; encoding: [0x01,0x02,0x48,0xbf]
 
 s_cmp_u_f32 s105, s2
-// GFX12: encoding: [0x69,0x02,0x48,0xbf]
+// GFX12: s_cmp_u_f32 s105, s2                    ; encoding: [0x69,0x02,0x48,0xbf]
 
 s_cmp_u_f32 s101, s2
-// GFX12: encoding: [0x65,0x02,0x48,0xbf]
+// GFX12: s_cmp_u_f32 s101, s2                    ; encoding: [0x65,0x02,0x48,0xbf]
 
 s_cmp_u_f32 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x48,0xbf]
+// GFX12: s_cmp_u_f32 vcc_lo, s2                  ; encoding: [0x6a,0x02,0x48,0xbf]
 
 s_cmp_u_f32 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x48,0xbf]
+// GFX12: s_cmp_u_f32 vcc_hi, s2                  ; encoding: [0x6b,0x02,0x48,0xbf]
 
 s_cmp_u_f32 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x48,0xbf]
+// GFX12: s_cmp_u_f32 m0, s2                      ; encoding: [0x7d,0x02,0x48,0xbf]
 
 s_cmp_u_f32 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x48,0xbf]
+// GFX12: s_cmp_u_f32 exec_lo, s2                 ; encoding: [0x7e,0x02,0x48,0xbf]
 
 s_cmp_u_f32 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x48,0xbf]
+// GFX12: s_cmp_u_f32 exec_hi, s2                 ; encoding: [0x7f,0x02,0x48,0xbf]
 
 s_cmp_u_f32 0, s2
-// GFX12: encoding: [0x80,0x02,0x48,0xbf]
+// GFX12: s_cmp_u_f32 0, s2                       ; encoding: [0x80,0x02,0x48,0xbf]
 
 s_cmp_u_f32 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x48,0xbf]
+// GFX12: s_cmp_u_f32 -1, s2                      ; encoding: [0xc1,0x02,0x48,0xbf]
 
 s_cmp_u_f32 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x48,0xbf]
+// GFX12: s_cmp_u_f32 0.5, s2                     ; encoding: [0xf0,0x02,0x48,0xbf]
 
 s_cmp_u_f32 -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x48,0xbf]
+// GFX12: s_cmp_u_f32 -4.0, s2                    ; encoding: [0xf7,0x02,0x48,0xbf]
 
 s_cmp_u_f32 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x48,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_u_f32 0xaf123456, s2              ; encoding: [0xff,0x02,0x48,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_u_f32 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x48,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_u_f32 0x3f717273, s2              ; encoding: [0xff,0x02,0x48,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_u_f32 s1, s105
-// GFX12: encoding: [0x01,0x69,0x48,0xbf]
+// GFX12: s_cmp_u_f32 s1, s105                    ; encoding: [0x01,0x69,0x48,0xbf]
 
 s_cmp_nge_f32 s1, s2
-// GFX12: encoding: [0x01,0x02,0x49,0xbf]
+// GFX12: s_cmp_nge_f32 s1, s2                    ; encoding: [0x01,0x02,0x49,0xbf]
 
 s_cmp_nge_f32 s105, s2
-// GFX12: encoding: [0x69,0x02,0x49,0xbf]
+// GFX12: s_cmp_nge_f32 s105, s2                  ; encoding: [0x69,0x02,0x49,0xbf]
 
 s_cmp_nge_f32 s101, s2
-// GFX12: encoding: [0x65,0x02,0x49,0xbf]
+// GFX12: s_cmp_nge_f32 s101, s2                  ; encoding: [0x65,0x02,0x49,0xbf]
 
 s_cmp_nge_f32 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x49,0xbf]
+// GFX12: s_cmp_nge_f32 vcc_lo, s2                ; encoding: [0x6a,0x02,0x49,0xbf]
 
 s_cmp_nge_f32 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x49,0xbf]
+// GFX12: s_cmp_nge_f32 vcc_hi, s2                ; encoding: [0x6b,0x02,0x49,0xbf]
 
 s_cmp_nge_f32 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x49,0xbf]
+// GFX12: s_cmp_nge_f32 m0, s2                    ; encoding: [0x7d,0x02,0x49,0xbf]
 
 s_cmp_nge_f32 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x49,0xbf]
+// GFX12: s_cmp_nge_f32 exec_lo, s2               ; encoding: [0x7e,0x02,0x49,0xbf]
 
 s_cmp_nge_f32 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x49,0xbf]
+// GFX12: s_cmp_nge_f32 exec_hi, s2               ; encoding: [0x7f,0x02,0x49,0xbf]
 
 s_cmp_nge_f32 0, s2
-// GFX12: encoding: [0x80,0x02,0x49,0xbf]
+// GFX12: s_cmp_nge_f32 0, s2                     ; encoding: [0x80,0x02,0x49,0xbf]
 
 s_cmp_nge_f32 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x49,0xbf]
+// GFX12: s_cmp_nge_f32 -1, s2                    ; encoding: [0xc1,0x02,0x49,0xbf]
 
 s_cmp_nge_f32 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x49,0xbf]
+// GFX12: s_cmp_nge_f32 0.5, s2                   ; encoding: [0xf0,0x02,0x49,0xbf]
 
 s_cmp_nge_f32 -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x49,0xbf]
+// GFX12: s_cmp_nge_f32 -4.0, s2                  ; encoding: [0xf7,0x02,0x49,0xbf]
 
 s_cmp_nge_f32 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x49,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_nge_f32 0xaf123456, s2            ; encoding: [0xff,0x02,0x49,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_nge_f32 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x49,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_nge_f32 0x3f717273, s2            ; encoding: [0xff,0x02,0x49,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_nge_f32 s1, s105
-// GFX12: encoding: [0x01,0x69,0x49,0xbf]
+// GFX12: s_cmp_nge_f32 s1, s105                  ; encoding: [0x01,0x69,0x49,0xbf]
 
 s_cmp_nlg_f32 s1, s2
-// GFX12: encoding: [0x01,0x02,0x4a,0xbf]
+// GFX12: s_cmp_nlg_f32 s1, s2                    ; encoding: [0x01,0x02,0x4a,0xbf]
 
 s_cmp_nlg_f32 s105, s2
-// GFX12: encoding: [0x69,0x02,0x4a,0xbf]
+// GFX12: s_cmp_nlg_f32 s105, s2                  ; encoding: [0x69,0x02,0x4a,0xbf]
 
 s_cmp_nlg_f32 s101, s2
-// GFX12: encoding: [0x65,0x02,0x4a,0xbf]
+// GFX12: s_cmp_nlg_f32 s101, s2                  ; encoding: [0x65,0x02,0x4a,0xbf]
 
 s_cmp_nlg_f32 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x4a,0xbf]
+// GFX12: s_cmp_nlg_f32 vcc_lo, s2                ; encoding: [0x6a,0x02,0x4a,0xbf]
 
 s_cmp_nlg_f32 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x4a,0xbf]
+// GFX12: s_cmp_nlg_f32 vcc_hi, s2                ; encoding: [0x6b,0x02,0x4a,0xbf]
 
 s_cmp_nlg_f32 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x4a,0xbf]
+// GFX12: s_cmp_nlg_f32 m0, s2                    ; encoding: [0x7d,0x02,0x4a,0xbf]
 
 s_cmp_nlg_f32 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x4a,0xbf]
+// GFX12: s_cmp_nlg_f32 exec_lo, s2               ; encoding: [0x7e,0x02,0x4a,0xbf]
 
 s_cmp_nlg_f32 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x4a,0xbf]
+// GFX12: s_cmp_nlg_f32 exec_hi, s2               ; encoding: [0x7f,0x02,0x4a,0xbf]
 
 s_cmp_nlg_f32 0, s2
-// GFX12: encoding: [0x80,0x02,0x4a,0xbf]
+// GFX12: s_cmp_nlg_f32 0, s2                     ; encoding: [0x80,0x02,0x4a,0xbf]
 
 s_cmp_nlg_f32 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x4a,0xbf]
+// GFX12: s_cmp_nlg_f32 -1, s2                    ; encoding: [0xc1,0x02,0x4a,0xbf]
 
 s_cmp_nlg_f32 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x4a,0xbf]
+// GFX12: s_cmp_nlg_f32 0.5, s2                   ; encoding: [0xf0,0x02,0x4a,0xbf]
 
 s_cmp_nlg_f32 -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x4a,0xbf]
+// GFX12: s_cmp_nlg_f32 -4.0, s2                  ; encoding: [0xf7,0x02,0x4a,0xbf]
 
 s_cmp_nlg_f32 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x4a,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_nlg_f32 0xaf123456, s2            ; encoding: [0xff,0x02,0x4a,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_nlg_f32 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x4a,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_nlg_f32 0x3f717273, s2            ; encoding: [0xff,0x02,0x4a,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_nlg_f32 s1, s105
-// GFX12: encoding: [0x01,0x69,0x4a,0xbf]
+// GFX12: s_cmp_nlg_f32 s1, s105                  ; encoding: [0x01,0x69,0x4a,0xbf]
 
 s_cmp_ngt_f32 s1, s2
-// GFX12: encoding: [0x01,0x02,0x4b,0xbf]
+// GFX12: s_cmp_ngt_f32 s1, s2                    ; encoding: [0x01,0x02,0x4b,0xbf]
 
 s_cmp_ngt_f32 s105, s2
-// GFX12: encoding: [0x69,0x02,0x4b,0xbf]
+// GFX12: s_cmp_ngt_f32 s105, s2                  ; encoding: [0x69,0x02,0x4b,0xbf]
 
 s_cmp_ngt_f32 s101, s2
-// GFX12: encoding: [0x65,0x02,0x4b,0xbf]
+// GFX12: s_cmp_ngt_f32 s101, s2                  ; encoding: [0x65,0x02,0x4b,0xbf]
 
 s_cmp_ngt_f32 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x4b,0xbf]
+// GFX12: s_cmp_ngt_f32 vcc_lo, s2                ; encoding: [0x6a,0x02,0x4b,0xbf]
 
 s_cmp_ngt_f32 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x4b,0xbf]
+// GFX12: s_cmp_ngt_f32 vcc_hi, s2                ; encoding: [0x6b,0x02,0x4b,0xbf]
 
 s_cmp_ngt_f32 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x4b,0xbf]
+// GFX12: s_cmp_ngt_f32 m0, s2                    ; encoding: [0x7d,0x02,0x4b,0xbf]
 
 s_cmp_ngt_f32 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x4b,0xbf]
+// GFX12: s_cmp_ngt_f32 exec_lo, s2               ; encoding: [0x7e,0x02,0x4b,0xbf]
 
 s_cmp_ngt_f32 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x4b,0xbf]
+// GFX12: s_cmp_ngt_f32 exec_hi, s2               ; encoding: [0x7f,0x02,0x4b,0xbf]
 
 s_cmp_ngt_f32 0, s2
-// GFX12: encoding: [0x80,0x02,0x4b,0xbf]
+// GFX12: s_cmp_ngt_f32 0, s2                     ; encoding: [0x80,0x02,0x4b,0xbf]
 
 s_cmp_ngt_f32 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x4b,0xbf]
+// GFX12: s_cmp_ngt_f32 -1, s2                    ; encoding: [0xc1,0x02,0x4b,0xbf]
 
 s_cmp_ngt_f32 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x4b,0xbf]
+// GFX12: s_cmp_ngt_f32 0.5, s2                   ; encoding: [0xf0,0x02,0x4b,0xbf]
 
 s_cmp_ngt_f32 -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x4b,0xbf]
+// GFX12: s_cmp_ngt_f32 -4.0, s2                  ; encoding: [0xf7,0x02,0x4b,0xbf]
 
 s_cmp_ngt_f32 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x4b,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_ngt_f32 0xaf123456, s2            ; encoding: [0xff,0x02,0x4b,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_ngt_f32 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x4b,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_ngt_f32 0x3f717273, s2            ; encoding: [0xff,0x02,0x4b,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_ngt_f32 s1, s105
-// GFX12: encoding: [0x01,0x69,0x4b,0xbf]
+// GFX12: s_cmp_ngt_f32 s1, s105                  ; encoding: [0x01,0x69,0x4b,0xbf]
 
 s_cmp_nle_f32 s1, s2
-// GFX12: encoding: [0x01,0x02,0x4c,0xbf]
+// GFX12: s_cmp_nle_f32 s1, s2                    ; encoding: [0x01,0x02,0x4c,0xbf]
 
 s_cmp_nle_f32 s105, s2
-// GFX12: encoding: [0x69,0x02,0x4c,0xbf]
+// GFX12: s_cmp_nle_f32 s105, s2                  ; encoding: [0x69,0x02,0x4c,0xbf]
 
 s_cmp_nle_f32 s101, s2
-// GFX12: encoding: [0x65,0x02,0x4c,0xbf]
+// GFX12: s_cmp_nle_f32 s101, s2                  ; encoding: [0x65,0x02,0x4c,0xbf]
 
 s_cmp_nle_f32 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x4c,0xbf]
+// GFX12: s_cmp_nle_f32 vcc_lo, s2                ; encoding: [0x6a,0x02,0x4c,0xbf]
 
 s_cmp_nle_f32 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x4c,0xbf]
+// GFX12: s_cmp_nle_f32 vcc_hi, s2                ; encoding: [0x6b,0x02,0x4c,0xbf]
 
 s_cmp_nle_f32 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x4c,0xbf]
+// GFX12: s_cmp_nle_f32 m0, s2                    ; encoding: [0x7d,0x02,0x4c,0xbf]
 
 s_cmp_nle_f32 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x4c,0xbf]
+// GFX12: s_cmp_nle_f32 exec_lo, s2               ; encoding: [0x7e,0x02,0x4c,0xbf]
 
 s_cmp_nle_f32 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x4c,0xbf]
+// GFX12: s_cmp_nle_f32 exec_hi, s2               ; encoding: [0x7f,0x02,0x4c,0xbf]
 
 s_cmp_nle_f32 0, s2
-// GFX12: encoding: [0x80,0x02,0x4c,0xbf]
+// GFX12: s_cmp_nle_f32 0, s2                     ; encoding: [0x80,0x02,0x4c,0xbf]
 
 s_cmp_nle_f32 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x4c,0xbf]
+// GFX12: s_cmp_nle_f32 -1, s2                    ; encoding: [0xc1,0x02,0x4c,0xbf]
 
 s_cmp_nle_f32 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x4c,0xbf]
+// GFX12: s_cmp_nle_f32 0.5, s2                   ; encoding: [0xf0,0x02,0x4c,0xbf]
 
 s_cmp_nle_f32 -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x4c,0xbf]
+// GFX12: s_cmp_nle_f32 -4.0, s2                  ; encoding: [0xf7,0x02,0x4c,0xbf]
 
 s_cmp_nle_f32 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x4c,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_nle_f32 0xaf123456, s2            ; encoding: [0xff,0x02,0x4c,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_nle_f32 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x4c,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_nle_f32 0x3f717273, s2            ; encoding: [0xff,0x02,0x4c,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_nle_f32 s1, s105
-// GFX12: encoding: [0x01,0x69,0x4c,0xbf]
+// GFX12: s_cmp_nle_f32 s1, s105                  ; encoding: [0x01,0x69,0x4c,0xbf]
 
 s_cmp_neq_f32 s1, s2
-// GFX12: encoding: [0x01,0x02,0x4d,0xbf]
+// GFX12: s_cmp_neq_f32 s1, s2                    ; encoding: [0x01,0x02,0x4d,0xbf]
 
 s_cmp_neq_f32 s105, s2
-// GFX12: encoding: [0x69,0x02,0x4d,0xbf]
+// GFX12: s_cmp_neq_f32 s105, s2                  ; encoding: [0x69,0x02,0x4d,0xbf]
 
 s_cmp_neq_f32 s101, s2
-// GFX12: encoding: [0x65,0x02,0x4d,0xbf]
+// GFX12: s_cmp_neq_f32 s101, s2                  ; encoding: [0x65,0x02,0x4d,0xbf]
 
 s_cmp_neq_f32 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x4d,0xbf]
+// GFX12: s_cmp_neq_f32 vcc_lo, s2                ; encoding: [0x6a,0x02,0x4d,0xbf]
 
 s_cmp_neq_f32 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x4d,0xbf]
+// GFX12: s_cmp_neq_f32 vcc_hi, s2                ; encoding: [0x6b,0x02,0x4d,0xbf]
 
 s_cmp_neq_f32 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x4d,0xbf]
+// GFX12: s_cmp_neq_f32 m0, s2                    ; encoding: [0x7d,0x02,0x4d,0xbf]
 
 s_cmp_neq_f32 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x4d,0xbf]
+// GFX12: s_cmp_neq_f32 exec_lo, s2               ; encoding: [0x7e,0x02,0x4d,0xbf]
 
 s_cmp_neq_f32 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x4d,0xbf]
+// GFX12: s_cmp_neq_f32 exec_hi, s2               ; encoding: [0x7f,0x02,0x4d,0xbf]
 
 s_cmp_neq_f32 0, s2
-// GFX12: encoding: [0x80,0x02,0x4d,0xbf]
+// GFX12: s_cmp_neq_f32 0, s2                     ; encoding: [0x80,0x02,0x4d,0xbf]
 
 s_cmp_neq_f32 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x4d,0xbf]
+// GFX12: s_cmp_neq_f32 -1, s2                    ; encoding: [0xc1,0x02,0x4d,0xbf]
 
 s_cmp_neq_f32 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x4d,0xbf]
+// GFX12: s_cmp_neq_f32 0.5, s2                   ; encoding: [0xf0,0x02,0x4d,0xbf]
 
 s_cmp_neq_f32 -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x4d,0xbf]
+// GFX12: s_cmp_neq_f32 -4.0, s2                  ; encoding: [0xf7,0x02,0x4d,0xbf]
 
 s_cmp_neq_f32 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x4d,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_neq_f32 0xaf123456, s2            ; encoding: [0xff,0x02,0x4d,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_neq_f32 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x4d,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_neq_f32 0x3f717273, s2            ; encoding: [0xff,0x02,0x4d,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_neq_f32 s1, s105
-// GFX12: encoding: [0x01,0x69,0x4d,0xbf]
+// GFX12: s_cmp_neq_f32 s1, s105                  ; encoding: [0x01,0x69,0x4d,0xbf]
 
 s_cmp_nlt_f32 s1, s2
-// GFX12: encoding: [0x01,0x02,0x4e,0xbf]
+// GFX12: s_cmp_nlt_f32 s1, s2                    ; encoding: [0x01,0x02,0x4e,0xbf]
 
 s_cmp_nlt_f32 s105, s2
-// GFX12: encoding: [0x69,0x02,0x4e,0xbf]
+// GFX12: s_cmp_nlt_f32 s105, s2                  ; encoding: [0x69,0x02,0x4e,0xbf]
 
 s_cmp_nlt_f32 s101, s2
-// GFX12: encoding: [0x65,0x02,0x4e,0xbf]
+// GFX12: s_cmp_nlt_f32 s101, s2                  ; encoding: [0x65,0x02,0x4e,0xbf]
 
 s_cmp_nlt_f32 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x4e,0xbf]
+// GFX12: s_cmp_nlt_f32 vcc_lo, s2                ; encoding: [0x6a,0x02,0x4e,0xbf]
 
 s_cmp_nlt_f32 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x4e,0xbf]
+// GFX12: s_cmp_nlt_f32 vcc_hi, s2                ; encoding: [0x6b,0x02,0x4e,0xbf]
 
 s_cmp_nlt_f32 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x4e,0xbf]
+// GFX12: s_cmp_nlt_f32 m0, s2                    ; encoding: [0x7d,0x02,0x4e,0xbf]
 
 s_cmp_nlt_f32 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x4e,0xbf]
+// GFX12: s_cmp_nlt_f32 exec_lo, s2               ; encoding: [0x7e,0x02,0x4e,0xbf]
 
 s_cmp_nlt_f32 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x4e,0xbf]
+// GFX12: s_cmp_nlt_f32 exec_hi, s2               ; encoding: [0x7f,0x02,0x4e,0xbf]
 
 s_cmp_nlt_f32 0, s2
-// GFX12: encoding: [0x80,0x02,0x4e,0xbf]
+// GFX12: s_cmp_nlt_f32 0, s2                     ; encoding: [0x80,0x02,0x4e,0xbf]
 
 s_cmp_nlt_f32 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x4e,0xbf]
+// GFX12: s_cmp_nlt_f32 -1, s2                    ; encoding: [0xc1,0x02,0x4e,0xbf]
 
 s_cmp_nlt_f32 0.5, s2
-// GFX12: encoding: [0xf0,0x02,0x4e,0xbf]
+// GFX12: s_cmp_nlt_f32 0.5, s2                   ; encoding: [0xf0,0x02,0x4e,0xbf]
 
 s_cmp_nlt_f32 -4.0, s2
-// GFX12: encoding: [0xf7,0x02,0x4e,0xbf]
+// GFX12: s_cmp_nlt_f32 -4.0, s2                  ; encoding: [0xf7,0x02,0x4e,0xbf]
 
 s_cmp_nlt_f32 0xaf123456, s2
-// GFX12: encoding: [0xff,0x02,0x4e,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_nlt_f32 0xaf123456, s2            ; encoding: [0xff,0x02,0x4e,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_nlt_f32 0x3f717273, s2
-// GFX12: encoding: [0xff,0x02,0x4e,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_nlt_f32 0x3f717273, s2            ; encoding: [0xff,0x02,0x4e,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_nlt_f32 s1, s105
-// GFX12: encoding: [0x01,0x69,0x4e,0xbf]
+// GFX12: s_cmp_nlt_f32 s1, s105                  ; encoding: [0x01,0x69,0x4e,0xbf]
 
 s_cmp_lt_f16 s1, s2
-// GFX12: encoding: [0x01,0x02,0x51,0xbf]
+// GFX12: s_cmp_lt_f16 s1, s2                     ; encoding: [0x01,0x02,0x51,0xbf]
 
 s_cmp_lt_f16 s101, s2
-// GFX12: encoding: [0x65,0x02,0x51,0xbf]
+// GFX12: s_cmp_lt_f16 s101, s2                   ; encoding: [0x65,0x02,0x51,0xbf]
 
 s_cmp_lt_f16 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x51,0xbf]
+// GFX12: s_cmp_lt_f16 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x51,0xbf]
 
 s_cmp_lt_f16 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x51,0xbf]
+// GFX12: s_cmp_lt_f16 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x51,0xbf]
 
 s_cmp_lt_f16 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x51,0xbf]
+// GFX12: s_cmp_lt_f16 m0, s2                     ; encoding: [0x7d,0x02,0x51,0xbf]
 
 s_cmp_lt_f16 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x51,0xbf]
+// GFX12: s_cmp_lt_f16 exec_lo, s2                ; encoding: [0x7e,0x02,0x51,0xbf]
 
 s_cmp_lt_f16 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x51,0xbf]
+// GFX12: s_cmp_lt_f16 exec_hi, s2                ; encoding: [0x7f,0x02,0x51,0xbf]
 
 s_cmp_lt_f16 0, s2
-// GFX12: encoding: [0x80,0x02,0x51,0xbf]
+// GFX12: s_cmp_lt_f16 0, s2                      ; encoding: [0x80,0x02,0x51,0xbf]
 
 s_cmp_lt_f16 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x51,0xbf]
+// GFX12: s_cmp_lt_f16 -1, s2                     ; encoding: [0xc1,0x02,0x51,0xbf]
 
 s_cmp_lt_f16 0x3800, s2
-// GFX12: encoding: [0xff,0x02,0x51,0xbf,0x00,0x38,0x00,0x00]
+// GFX12: s_cmp_lt_f16 0x3800, s2                 ; encoding: [0xff,0x02,0x51,0xbf,0x00,0x38,0x00,0x00]
 
 s_cmp_lt_f16 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x51,0xbf,0x0b,0xfe,0x00,0x00]
+// GFX12: s_cmp_lt_f16 0xfe0b, s2                 ; encoding: [0xff,0x02,0x51,0xbf,0x0b,0xfe,0x00,0x00]
 
 s_cmp_lt_f16 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x51,0xbf,0x56,0x34,0x00,0x00]
+// GFX12: s_cmp_lt_f16 0x3456, s2                 ; encoding: [0xff,0x02,0x51,0xbf,0x56,0x34,0x00,0x00]
 
 s_cmp_eq_f16 s1, s2
-// GFX12: encoding: [0x01,0x02,0x52,0xbf]
+// GFX12: s_cmp_eq_f16 s1, s2                     ; encoding: [0x01,0x02,0x52,0xbf]
 
 s_cmp_eq_f16 s101, s2
-// GFX12: encoding: [0x65,0x02,0x52,0xbf]
+// GFX12: s_cmp_eq_f16 s101, s2                   ; encoding: [0x65,0x02,0x52,0xbf]
 
 s_cmp_eq_f16 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x52,0xbf]
+// GFX12: s_cmp_eq_f16 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x52,0xbf]
 
 s_cmp_eq_f16 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x52,0xbf]
+// GFX12: s_cmp_eq_f16 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x52,0xbf]
 
 s_cmp_eq_f16 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x52,0xbf]
+// GFX12: s_cmp_eq_f16 m0, s2                     ; encoding: [0x7d,0x02,0x52,0xbf]
 
 s_cmp_eq_f16 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x52,0xbf]
+// GFX12: s_cmp_eq_f16 exec_lo, s2                ; encoding: [0x7e,0x02,0x52,0xbf]
 
 s_cmp_eq_f16 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x52,0xbf]
+// GFX12: s_cmp_eq_f16 exec_hi, s2                ; encoding: [0x7f,0x02,0x52,0xbf]
 
 s_cmp_eq_f16 0, s2
-// GFX12: encoding: [0x80,0x02,0x52,0xbf]
+// GFX12: s_cmp_eq_f16 0, s2                      ; encoding: [0x80,0x02,0x52,0xbf]
 
 s_cmp_eq_f16 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x52,0xbf]
+// GFX12: s_cmp_eq_f16 -1, s2                     ; encoding: [0xc1,0x02,0x52,0xbf]
 
 s_cmp_eq_f16 0x3800, s2
-// GFX12: encoding: [0xff,0x02,0x52,0xbf,0x00,0x38,0x00,0x00]
+// GFX12: s_cmp_eq_f16 0x3800, s2                 ; encoding: [0xff,0x02,0x52,0xbf,0x00,0x38,0x00,0x00]
 
 s_cmp_eq_f16 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x52,0xbf,0x0b,0xfe,0x00,0x00]
+// GFX12: s_cmp_eq_f16 0xfe0b, s2                 ; encoding: [0xff,0x02,0x52,0xbf,0x0b,0xfe,0x00,0x00]
 
 s_cmp_eq_f16 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x52,0xbf,0x56,0x34,0x00,0x00]
+// GFX12: s_cmp_eq_f16 0x3456, s2                 ; encoding: [0xff,0x02,0x52,0xbf,0x56,0x34,0x00,0x00]
 
 s_cmp_le_f16 s1, s2
-// GFX12: encoding: [0x01,0x02,0x53,0xbf]
+// GFX12: s_cmp_le_f16 s1, s2                     ; encoding: [0x01,0x02,0x53,0xbf]
 
 s_cmp_le_f16 s101, s2
-// GFX12: encoding: [0x65,0x02,0x53,0xbf]
+// GFX12: s_cmp_le_f16 s101, s2                   ; encoding: [0x65,0x02,0x53,0xbf]
 
 s_cmp_le_f16 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x53,0xbf]
+// GFX12: s_cmp_le_f16 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x53,0xbf]
 
 s_cmp_le_f16 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x53,0xbf]
+// GFX12: s_cmp_le_f16 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x53,0xbf]
 
 s_cmp_le_f16 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x53,0xbf]
+// GFX12: s_cmp_le_f16 m0, s2                     ; encoding: [0x7d,0x02,0x53,0xbf]
 
 s_cmp_le_f16 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x53,0xbf]
+// GFX12: s_cmp_le_f16 exec_lo, s2                ; encoding: [0x7e,0x02,0x53,0xbf]
 
 s_cmp_le_f16 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x53,0xbf]
+// GFX12: s_cmp_le_f16 exec_hi, s2                ; encoding: [0x7f,0x02,0x53,0xbf]
 
 s_cmp_le_f16 0, s2
-// GFX12: encoding: [0x80,0x02,0x53,0xbf]
+// GFX12: s_cmp_le_f16 0, s2                      ; encoding: [0x80,0x02,0x53,0xbf]
 
 s_cmp_le_f16 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x53,0xbf]
+// GFX12: s_cmp_le_f16 -1, s2                     ; encoding: [0xc1,0x02,0x53,0xbf]
 
 s_cmp_le_f16 0x3800, s2
-// GFX12: encoding: [0xff,0x02,0x53,0xbf,0x00,0x38,0x00,0x00]
+// GFX12: s_cmp_le_f16 0x3800, s2                 ; encoding: [0xff,0x02,0x53,0xbf,0x00,0x38,0x00,0x00]
 
 s_cmp_le_f16 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x53,0xbf,0x0b,0xfe,0x00,0x00]
+// GFX12: s_cmp_le_f16 0xfe0b, s2                 ; encoding: [0xff,0x02,0x53,0xbf,0x0b,0xfe,0x00,0x00]
 
 s_cmp_le_f16 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x53,0xbf,0x56,0x34,0x00,0x00]
+// GFX12: s_cmp_le_f16 0x3456, s2                 ; encoding: [0xff,0x02,0x53,0xbf,0x56,0x34,0x00,0x00]
 
 s_cmp_gt_f16 s1, s2
-// GFX12: encoding: [0x01,0x02,0x54,0xbf]
+// GFX12: s_cmp_gt_f16 s1, s2                     ; encoding: [0x01,0x02,0x54,0xbf]
 
 s_cmp_gt_f16 s101, s2
-// GFX12: encoding: [0x65,0x02,0x54,0xbf]
+// GFX12: s_cmp_gt_f16 s101, s2                   ; encoding: [0x65,0x02,0x54,0xbf]
 
 s_cmp_gt_f16 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x54,0xbf]
+// GFX12: s_cmp_gt_f16 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x54,0xbf]
 
 s_cmp_gt_f16 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x54,0xbf]
+// GFX12: s_cmp_gt_f16 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x54,0xbf]
 
 s_cmp_gt_f16 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x54,0xbf]
+// GFX12: s_cmp_gt_f16 m0, s2                     ; encoding: [0x7d,0x02,0x54,0xbf]
 
 s_cmp_gt_f16 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x54,0xbf]
+// GFX12: s_cmp_gt_f16 exec_lo, s2                ; encoding: [0x7e,0x02,0x54,0xbf]
 
 s_cmp_gt_f16 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x54,0xbf]
+// GFX12: s_cmp_gt_f16 exec_hi, s2                ; encoding: [0x7f,0x02,0x54,0xbf]
 
 s_cmp_gt_f16 0, s2
-// GFX12: encoding: [0x80,0x02,0x54,0xbf]
+// GFX12: s_cmp_gt_f16 0, s2                      ; encoding: [0x80,0x02,0x54,0xbf]
 
 s_cmp_gt_f16 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x54,0xbf]
+// GFX12: s_cmp_gt_f16 -1, s2                     ; encoding: [0xc1,0x02,0x54,0xbf]
 
 s_cmp_gt_f16 0x3800, s2
-// GFX12: encoding: [0xff,0x02,0x54,0xbf,0x00,0x38,0x00,0x00]
+// GFX12: s_cmp_gt_f16 0x3800, s2                 ; encoding: [0xff,0x02,0x54,0xbf,0x00,0x38,0x00,0x00]
 
 s_cmp_gt_f16 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x54,0xbf,0x0b,0xfe,0x00,0x00]
+// GFX12: s_cmp_gt_f16 0xfe0b, s2                 ; encoding: [0xff,0x02,0x54,0xbf,0x0b,0xfe,0x00,0x00]
 
 s_cmp_gt_f16 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x54,0xbf,0x56,0x34,0x00,0x00]
+// GFX12: s_cmp_gt_f16 0x3456, s2                 ; encoding: [0xff,0x02,0x54,0xbf,0x56,0x34,0x00,0x00]
 
 s_cmp_lg_f16 s1, s2
-// GFX12: encoding: [0x01,0x02,0x55,0xbf]
+// GFX12: s_cmp_lg_f16 s1, s2                     ; encoding: [0x01,0x02,0x55,0xbf]
 
 s_cmp_lg_f16 s101, s2
-// GFX12: encoding: [0x65,0x02,0x55,0xbf]
+// GFX12: s_cmp_lg_f16 s101, s2                   ; encoding: [0x65,0x02,0x55,0xbf]
 
 s_cmp_lg_f16 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x55,0xbf]
+// GFX12: s_cmp_lg_f16 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x55,0xbf]
 
 s_cmp_lg_f16 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x55,0xbf]
+// GFX12: s_cmp_lg_f16 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x55,0xbf]
 
 s_cmp_lg_f16 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x55,0xbf]
+// GFX12: s_cmp_lg_f16 m0, s2                     ; encoding: [0x7d,0x02,0x55,0xbf]
 
 s_cmp_lg_f16 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x55,0xbf]
+// GFX12: s_cmp_lg_f16 exec_lo, s2                ; encoding: [0x7e,0x02,0x55,0xbf]
 
 s_cmp_lg_f16 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x55,0xbf]
+// GFX12: s_cmp_lg_f16 exec_hi, s2                ; encoding: [0x7f,0x02,0x55,0xbf]
 
 s_cmp_lg_f16 0, s2
-// GFX12: encoding: [0x80,0x02,0x55,0xbf]
+// GFX12: s_cmp_lg_f16 0, s2                      ; encoding: [0x80,0x02,0x55,0xbf]
 
 s_cmp_lg_f16 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x55,0xbf]
+// GFX12: s_cmp_lg_f16 -1, s2                     ; encoding: [0xc1,0x02,0x55,0xbf]
 
 s_cmp_lg_f16 0x3800, s2
-// GFX12: encoding: [0xff,0x02,0x55,0xbf,0x00,0x38,0x00,0x00]
+// GFX12: s_cmp_lg_f16 0x3800, s2                 ; encoding: [0xff,0x02,0x55,0xbf,0x00,0x38,0x00,0x00]
 
 s_cmp_lg_f16 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x55,0xbf,0x0b,0xfe,0x00,0x00]
+// GFX12: s_cmp_lg_f16 0xfe0b, s2                 ; encoding: [0xff,0x02,0x55,0xbf,0x0b,0xfe,0x00,0x00]
 
 s_cmp_lg_f16 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x55,0xbf,0x56,0x34,0x00,0x00]
+// GFX12: s_cmp_lg_f16 0x3456, s2                 ; encoding: [0xff,0x02,0x55,0xbf,0x56,0x34,0x00,0x00]
 
 s_cmp_ge_f16 s1, s2
-// GFX12: encoding: [0x01,0x02,0x56,0xbf]
+// GFX12: s_cmp_ge_f16 s1, s2                     ; encoding: [0x01,0x02,0x56,0xbf]
 
 s_cmp_ge_f16 s101, s2
-// GFX12: encoding: [0x65,0x02,0x56,0xbf]
+// GFX12: s_cmp_ge_f16 s101, s2                   ; encoding: [0x65,0x02,0x56,0xbf]
 
 s_cmp_ge_f16 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x56,0xbf]
+// GFX12: s_cmp_ge_f16 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x56,0xbf]
 
 s_cmp_ge_f16 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x56,0xbf]
+// GFX12: s_cmp_ge_f16 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x56,0xbf]
 
 s_cmp_ge_f16 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x56,0xbf]
+// GFX12: s_cmp_ge_f16 m0, s2                     ; encoding: [0x7d,0x02,0x56,0xbf]
 
 s_cmp_ge_f16 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x56,0xbf]
+// GFX12: s_cmp_ge_f16 exec_lo, s2                ; encoding: [0x7e,0x02,0x56,0xbf]
 
 s_cmp_ge_f16 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x56,0xbf]
+// GFX12: s_cmp_ge_f16 exec_hi, s2                ; encoding: [0x7f,0x02,0x56,0xbf]
 
 s_cmp_ge_f16 0, s2
-// GFX12: encoding: [0x80,0x02,0x56,0xbf]
+// GFX12: s_cmp_ge_f16 0, s2                      ; encoding: [0x80,0x02,0x56,0xbf]
 
 s_cmp_ge_f16 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x56,0xbf]
+// GFX12: s_cmp_ge_f16 -1, s2                     ; encoding: [0xc1,0x02,0x56,0xbf]
 
 s_cmp_ge_f16 0x3800, s2
-// GFX12: encoding: [0xff,0x02,0x56,0xbf,0x00,0x38,0x00,0x00]
+// GFX12: s_cmp_ge_f16 0x3800, s2                 ; encoding: [0xff,0x02,0x56,0xbf,0x00,0x38,0x00,0x00]
 
 s_cmp_ge_f16 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x56,0xbf,0x0b,0xfe,0x00,0x00]
+// GFX12: s_cmp_ge_f16 0xfe0b, s2                 ; encoding: [0xff,0x02,0x56,0xbf,0x0b,0xfe,0x00,0x00]
 
 s_cmp_ge_f16 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x56,0xbf,0x56,0x34,0x00,0x00]
+// GFX12: s_cmp_ge_f16 0x3456, s2                 ; encoding: [0xff,0x02,0x56,0xbf,0x56,0x34,0x00,0x00]
 
 s_cmp_o_f16 s1, s2
-// GFX12: encoding: [0x01,0x02,0x57,0xbf]
+// GFX12: s_cmp_o_f16 s1, s2                      ; encoding: [0x01,0x02,0x57,0xbf]
 
 s_cmp_o_f16 s101, s2
-// GFX12: encoding: [0x65,0x02,0x57,0xbf]
+// GFX12: s_cmp_o_f16 s101, s2                    ; encoding: [0x65,0x02,0x57,0xbf]
 
 s_cmp_o_f16 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x57,0xbf]
+// GFX12: s_cmp_o_f16 vcc_lo, s2                  ; encoding: [0x6a,0x02,0x57,0xbf]
 
 s_cmp_o_f16 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x57,0xbf]
+// GFX12: s_cmp_o_f16 vcc_hi, s2                  ; encoding: [0x6b,0x02,0x57,0xbf]
 
 s_cmp_o_f16 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x57,0xbf]
+// GFX12: s_cmp_o_f16 m0, s2                      ; encoding: [0x7d,0x02,0x57,0xbf]
 
 s_cmp_o_f16 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x57,0xbf]
+// GFX12: s_cmp_o_f16 exec_lo, s2                 ; encoding: [0x7e,0x02,0x57,0xbf]
 
 s_cmp_o_f16 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x57,0xbf]
+// GFX12: s_cmp_o_f16 exec_hi, s2                 ; encoding: [0x7f,0x02,0x57,0xbf]
 
 s_cmp_o_f16 0, s2
-// GFX12: encoding: [0x80,0x02,0x57,0xbf]
+// GFX12: s_cmp_o_f16 0, s2                       ; encoding: [0x80,0x02,0x57,0xbf]
 
 s_cmp_o_f16 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x57,0xbf]
+// GFX12: s_cmp_o_f16 -1, s2                      ; encoding: [0xc1,0x02,0x57,0xbf]
 
 s_cmp_o_f16 0x3800, s2
-// GFX12: encoding: [0xff,0x02,0x57,0xbf,0x00,0x38,0x00,0x00]
+// GFX12: s_cmp_o_f16 0x3800, s2                  ; encoding: [0xff,0x02,0x57,0xbf,0x00,0x38,0x00,0x00]
 
 s_cmp_o_f16 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x57,0xbf,0x0b,0xfe,0x00,0x00]
+// GFX12: s_cmp_o_f16 0xfe0b, s2                  ; encoding: [0xff,0x02,0x57,0xbf,0x0b,0xfe,0x00,0x00]
 
 s_cmp_o_f16 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x57,0xbf,0x56,0x34,0x00,0x00]
+// GFX12: s_cmp_o_f16 0x3456, s2                  ; encoding: [0xff,0x02,0x57,0xbf,0x56,0x34,0x00,0x00]
 
 s_cmp_u_f16 s1, s2
-// GFX12: encoding: [0x01,0x02,0x58,0xbf]
+// GFX12: s_cmp_u_f16 s1, s2                      ; encoding: [0x01,0x02,0x58,0xbf]
 
 s_cmp_u_f16 s101, s2
-// GFX12: encoding: [0x65,0x02,0x58,0xbf]
+// GFX12: s_cmp_u_f16 s101, s2                    ; encoding: [0x65,0x02,0x58,0xbf]
 
 s_cmp_u_f16 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x58,0xbf]
+// GFX12: s_cmp_u_f16 vcc_lo, s2                  ; encoding: [0x6a,0x02,0x58,0xbf]
 
 s_cmp_u_f16 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x58,0xbf]
+// GFX12: s_cmp_u_f16 vcc_hi, s2                  ; encoding: [0x6b,0x02,0x58,0xbf]
 
 s_cmp_u_f16 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x58,0xbf]
+// GFX12: s_cmp_u_f16 m0, s2                      ; encoding: [0x7d,0x02,0x58,0xbf]
 
 s_cmp_u_f16 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x58,0xbf]
+// GFX12: s_cmp_u_f16 exec_lo, s2                 ; encoding: [0x7e,0x02,0x58,0xbf]
 
 s_cmp_u_f16 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x58,0xbf]
+// GFX12: s_cmp_u_f16 exec_hi, s2                 ; encoding: [0x7f,0x02,0x58,0xbf]
 
 s_cmp_u_f16 0, s2
-// GFX12: encoding: [0x80,0x02,0x58,0xbf]
+// GFX12: s_cmp_u_f16 0, s2                       ; encoding: [0x80,0x02,0x58,0xbf]
 
 s_cmp_u_f16 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x58,0xbf]
+// GFX12: s_cmp_u_f16 -1, s2                      ; encoding: [0xc1,0x02,0x58,0xbf]
 
 s_cmp_u_f16 0x3800, s2
-// GFX12: encoding: [0xff,0x02,0x58,0xbf,0x00,0x38,0x00,0x00]
+// GFX12: s_cmp_u_f16 0x3800, s2                  ; encoding: [0xff,0x02,0x58,0xbf,0x00,0x38,0x00,0x00]
 
 s_cmp_u_f16 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x58,0xbf,0x0b,0xfe,0x00,0x00]
+// GFX12: s_cmp_u_f16 0xfe0b, s2                  ; encoding: [0xff,0x02,0x58,0xbf,0x0b,0xfe,0x00,0x00]
 
 s_cmp_u_f16 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x58,0xbf,0x56,0x34,0x00,0x00]
+// GFX12: s_cmp_u_f16 0x3456, s2                  ; encoding: [0xff,0x02,0x58,0xbf,0x56,0x34,0x00,0x00]
 
 s_cmp_nge_f16 s1, s2
-// GFX12: encoding: [0x01,0x02,0x59,0xbf]
+// GFX12: s_cmp_nge_f16 s1, s2                    ; encoding: [0x01,0x02,0x59,0xbf]
 
 s_cmp_nge_f16 s101, s2
-// GFX12: encoding: [0x65,0x02,0x59,0xbf]
+// GFX12: s_cmp_nge_f16 s101, s2                  ; encoding: [0x65,0x02,0x59,0xbf]
 
 s_cmp_nge_f16 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x59,0xbf]
+// GFX12: s_cmp_nge_f16 vcc_lo, s2                ; encoding: [0x6a,0x02,0x59,0xbf]
 
 s_cmp_nge_f16 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x59,0xbf]
+// GFX12: s_cmp_nge_f16 vcc_hi, s2                ; encoding: [0x6b,0x02,0x59,0xbf]
 
 s_cmp_nge_f16 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x59,0xbf]
+// GFX12: s_cmp_nge_f16 m0, s2                    ; encoding: [0x7d,0x02,0x59,0xbf]
 
 s_cmp_nge_f16 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x59,0xbf]
+// GFX12: s_cmp_nge_f16 exec_lo, s2               ; encoding: [0x7e,0x02,0x59,0xbf]
 
 s_cmp_nge_f16 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x59,0xbf]
+// GFX12: s_cmp_nge_f16 exec_hi, s2               ; encoding: [0x7f,0x02,0x59,0xbf]
 
 s_cmp_nge_f16 0, s2
-// GFX12: encoding: [0x80,0x02,0x59,0xbf]
+// GFX12: s_cmp_nge_f16 0, s2                     ; encoding: [0x80,0x02,0x59,0xbf]
 
 s_cmp_nge_f16 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x59,0xbf]
+// GFX12: s_cmp_nge_f16 -1, s2                    ; encoding: [0xc1,0x02,0x59,0xbf]
 
 s_cmp_nge_f16 0x3800, s2
-// GFX12: encoding: [0xff,0x02,0x59,0xbf,0x00,0x38,0x00,0x00]
+// GFX12: s_cmp_nge_f16 0x3800, s2                ; encoding: [0xff,0x02,0x59,0xbf,0x00,0x38,0x00,0x00]
 
 s_cmp_nge_f16 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x59,0xbf,0x0b,0xfe,0x00,0x00]
+// GFX12: s_cmp_nge_f16 0xfe0b, s2                ; encoding: [0xff,0x02,0x59,0xbf,0x0b,0xfe,0x00,0x00]
 
 s_cmp_nge_f16 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x59,0xbf,0x56,0x34,0x00,0x00]
+// GFX12: s_cmp_nge_f16 0x3456, s2                ; encoding: [0xff,0x02,0x59,0xbf,0x56,0x34,0x00,0x00]
 
 s_cmp_nlg_f16 s1, s2
-// GFX12: encoding: [0x01,0x02,0x5a,0xbf]
+// GFX12: s_cmp_nlg_f16 s1, s2                    ; encoding: [0x01,0x02,0x5a,0xbf]
 
 s_cmp_nlg_f16 s101, s2
-// GFX12: encoding: [0x65,0x02,0x5a,0xbf]
+// GFX12: s_cmp_nlg_f16 s101, s2                  ; encoding: [0x65,0x02,0x5a,0xbf]
 
 s_cmp_nlg_f16 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x5a,0xbf]
+// GFX12: s_cmp_nlg_f16 vcc_lo, s2                ; encoding: [0x6a,0x02,0x5a,0xbf]
 
 s_cmp_nlg_f16 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x5a,0xbf]
+// GFX12: s_cmp_nlg_f16 vcc_hi, s2                ; encoding: [0x6b,0x02,0x5a,0xbf]
 
 s_cmp_nlg_f16 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x5a,0xbf]
+// GFX12: s_cmp_nlg_f16 m0, s2                    ; encoding: [0x7d,0x02,0x5a,0xbf]
 
 s_cmp_nlg_f16 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x5a,0xbf]
+// GFX12: s_cmp_nlg_f16 exec_lo, s2               ; encoding: [0x7e,0x02,0x5a,0xbf]
 
 s_cmp_nlg_f16 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x5a,0xbf]
+// GFX12: s_cmp_nlg_f16 exec_hi, s2               ; encoding: [0x7f,0x02,0x5a,0xbf]
 
 s_cmp_nlg_f16 0, s2
-// GFX12: encoding: [0x80,0x02,0x5a,0xbf]
+// GFX12: s_cmp_nlg_f16 0, s2                     ; encoding: [0x80,0x02,0x5a,0xbf]
 
 s_cmp_nlg_f16 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x5a,0xbf]
+// GFX12: s_cmp_nlg_f16 -1, s2                    ; encoding: [0xc1,0x02,0x5a,0xbf]
 
 s_cmp_nlg_f16 0x3800, s2
-// GFX12: encoding: [0xff,0x02,0x5a,0xbf,0x00,0x38,0x00,0x00]
+// GFX12: s_cmp_nlg_f16 0x3800, s2                ; encoding: [0xff,0x02,0x5a,0xbf,0x00,0x38,0x00,0x00]
 
 s_cmp_nlg_f16 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x5a,0xbf,0x0b,0xfe,0x00,0x00]
+// GFX12: s_cmp_nlg_f16 0xfe0b, s2                ; encoding: [0xff,0x02,0x5a,0xbf,0x0b,0xfe,0x00,0x00]
 
 s_cmp_nlg_f16 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x5a,0xbf,0x56,0x34,0x00,0x00]
+// GFX12: s_cmp_nlg_f16 0x3456, s2                ; encoding: [0xff,0x02,0x5a,0xbf,0x56,0x34,0x00,0x00]
 
 s_cmp_ngt_f16 s1, s2
-// GFX12: encoding: [0x01,0x02,0x5b,0xbf]
+// GFX12: s_cmp_ngt_f16 s1, s2                    ; encoding: [0x01,0x02,0x5b,0xbf]
 
 s_cmp_ngt_f16 s101, s2
-// GFX12: encoding: [0x65,0x02,0x5b,0xbf]
+// GFX12: s_cmp_ngt_f16 s101, s2                  ; encoding: [0x65,0x02,0x5b,0xbf]
 
 s_cmp_ngt_f16 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x5b,0xbf]
+// GFX12: s_cmp_ngt_f16 vcc_lo, s2                ; encoding: [0x6a,0x02,0x5b,0xbf]
 
 s_cmp_ngt_f16 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x5b,0xbf]
+// GFX12: s_cmp_ngt_f16 vcc_hi, s2                ; encoding: [0x6b,0x02,0x5b,0xbf]
 
 s_cmp_ngt_f16 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x5b,0xbf]
+// GFX12: s_cmp_ngt_f16 m0, s2                    ; encoding: [0x7d,0x02,0x5b,0xbf]
 
 s_cmp_ngt_f16 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x5b,0xbf]
+// GFX12: s_cmp_ngt_f16 exec_lo, s2               ; encoding: [0x7e,0x02,0x5b,0xbf]
 
 s_cmp_ngt_f16 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x5b,0xbf]
+// GFX12: s_cmp_ngt_f16 exec_hi, s2               ; encoding: [0x7f,0x02,0x5b,0xbf]
 
 s_cmp_ngt_f16 0, s2
-// GFX12: encoding: [0x80,0x02,0x5b,0xbf]
+// GFX12: s_cmp_ngt_f16 0, s2                     ; encoding: [0x80,0x02,0x5b,0xbf]
 
 s_cmp_ngt_f16 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x5b,0xbf]
+// GFX12: s_cmp_ngt_f16 -1, s2                    ; encoding: [0xc1,0x02,0x5b,0xbf]
 
 s_cmp_ngt_f16 0x3800, s2
-// GFX12: encoding: [0xff,0x02,0x5b,0xbf,0x00,0x38,0x00,0x00]
+// GFX12: s_cmp_ngt_f16 0x3800, s2                ; encoding: [0xff,0x02,0x5b,0xbf,0x00,0x38,0x00,0x00]
 
 s_cmp_ngt_f16 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x5b,0xbf,0x0b,0xfe,0x00,0x00]
+// GFX12: s_cmp_ngt_f16 0xfe0b, s2                ; encoding: [0xff,0x02,0x5b,0xbf,0x0b,0xfe,0x00,0x00]
 
 s_cmp_ngt_f16 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x5b,0xbf,0x56,0x34,0x00,0x00]
+// GFX12: s_cmp_ngt_f16 0x3456, s2                ; encoding: [0xff,0x02,0x5b,0xbf,0x56,0x34,0x00,0x00]
 
 s_cmp_nle_f16 s1, s2
-// GFX12: encoding: [0x01,0x02,0x5c,0xbf]
+// GFX12: s_cmp_nle_f16 s1, s2                    ; encoding: [0x01,0x02,0x5c,0xbf]
 
 s_cmp_nle_f16 s101, s2
-// GFX12: encoding: [0x65,0x02,0x5c,0xbf]
+// GFX12: s_cmp_nle_f16 s101, s2                  ; encoding: [0x65,0x02,0x5c,0xbf]
 
 s_cmp_nle_f16 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x5c,0xbf]
+// GFX12: s_cmp_nle_f16 vcc_lo, s2                ; encoding: [0x6a,0x02,0x5c,0xbf]
 
 s_cmp_nle_f16 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x5c,0xbf]
+// GFX12: s_cmp_nle_f16 vcc_hi, s2                ; encoding: [0x6b,0x02,0x5c,0xbf]
 
 s_cmp_nle_f16 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x5c,0xbf]
+// GFX12: s_cmp_nle_f16 m0, s2                    ; encoding: [0x7d,0x02,0x5c,0xbf]
 
 s_cmp_nle_f16 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x5c,0xbf]
+// GFX12: s_cmp_nle_f16 exec_lo, s2               ; encoding: [0x7e,0x02,0x5c,0xbf]
 
 s_cmp_nle_f16 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x5c,0xbf]
+// GFX12: s_cmp_nle_f16 exec_hi, s2               ; encoding: [0x7f,0x02,0x5c,0xbf]
 
 s_cmp_nle_f16 0, s2
-// GFX12: encoding: [0x80,0x02,0x5c,0xbf]
+// GFX12: s_cmp_nle_f16 0, s2                     ; encoding: [0x80,0x02,0x5c,0xbf]
 
 s_cmp_nle_f16 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x5c,0xbf]
+// GFX12: s_cmp_nle_f16 -1, s2                    ; encoding: [0xc1,0x02,0x5c,0xbf]
 
 s_cmp_nle_f16 0x3800, s2
-// GFX12: encoding: [0xff,0x02,0x5c,0xbf,0x00,0x38,0x00,0x00]
+// GFX12: s_cmp_nle_f16 0x3800, s2                ; encoding: [0xff,0x02,0x5c,0xbf,0x00,0x38,0x00,0x00]
 
 s_cmp_nle_f16 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x5c,0xbf,0x0b,0xfe,0x00,0x00]
+// GFX12: s_cmp_nle_f16 0xfe0b, s2                ; encoding: [0xff,0x02,0x5c,0xbf,0x0b,0xfe,0x00,0x00]
 
 s_cmp_nle_f16 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x5c,0xbf,0x56,0x34,0x00,0x00]
+// GFX12: s_cmp_nle_f16 0x3456, s2                ; encoding: [0xff,0x02,0x5c,0xbf,0x56,0x34,0x00,0x00]
 
 s_cmp_neq_f16 s1, s2
-// GFX12: encoding: [0x01,0x02,0x5d,0xbf]
+// GFX12: s_cmp_neq_f16 s1, s2                    ; encoding: [0x01,0x02,0x5d,0xbf]
 
 s_cmp_neq_f16 s101, s2
-// GFX12: encoding: [0x65,0x02,0x5d,0xbf]
+// GFX12: s_cmp_neq_f16 s101, s2                  ; encoding: [0x65,0x02,0x5d,0xbf]
 
 s_cmp_neq_f16 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x5d,0xbf]
+// GFX12: s_cmp_neq_f16 vcc_lo, s2                ; encoding: [0x6a,0x02,0x5d,0xbf]
 
 s_cmp_neq_f16 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x5d,0xbf]
+// GFX12: s_cmp_neq_f16 vcc_hi, s2                ; encoding: [0x6b,0x02,0x5d,0xbf]
 
 s_cmp_neq_f16 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x5d,0xbf]
+// GFX12: s_cmp_neq_f16 m0, s2                    ; encoding: [0x7d,0x02,0x5d,0xbf]
 
 s_cmp_neq_f16 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x5d,0xbf]
+// GFX12: s_cmp_neq_f16 exec_lo, s2               ; encoding: [0x7e,0x02,0x5d,0xbf]
 
 s_cmp_neq_f16 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x5d,0xbf]
+// GFX12: s_cmp_neq_f16 exec_hi, s2               ; encoding: [0x7f,0x02,0x5d,0xbf]
 
 s_cmp_neq_f16 0, s2
-// GFX12: encoding: [0x80,0x02,0x5d,0xbf]
+// GFX12: s_cmp_neq_f16 0, s2                     ; encoding: [0x80,0x02,0x5d,0xbf]
 
 s_cmp_neq_f16 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x5d,0xbf]
+// GFX12: s_cmp_neq_f16 -1, s2                    ; encoding: [0xc1,0x02,0x5d,0xbf]
 
 s_cmp_neq_f16 0x3800, s2
-// GFX12: encoding: [0xff,0x02,0x5d,0xbf,0x00,0x38,0x00,0x00]
+// GFX12: s_cmp_neq_f16 0x3800, s2                ; encoding: [0xff,0x02,0x5d,0xbf,0x00,0x38,0x00,0x00]
 
 s_cmp_neq_f16 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x5d,0xbf,0x0b,0xfe,0x00,0x00]
+// GFX12: s_cmp_neq_f16 0xfe0b, s2                ; encoding: [0xff,0x02,0x5d,0xbf,0x0b,0xfe,0x00,0x00]
 
 s_cmp_neq_f16 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x5d,0xbf,0x56,0x34,0x00,0x00]
+// GFX12: s_cmp_neq_f16 0x3456, s2                ; encoding: [0xff,0x02,0x5d,0xbf,0x56,0x34,0x00,0x00]
 
 s_cmp_nlt_f16 s1, s2
-// GFX12: encoding: [0x01,0x02,0x5e,0xbf]
+// GFX12: s_cmp_nlt_f16 s1, s2                    ; encoding: [0x01,0x02,0x5e,0xbf]
 
 s_cmp_nlt_f16 s101, s2
-// GFX12: encoding: [0x65,0x02,0x5e,0xbf]
+// GFX12: s_cmp_nlt_f16 s101, s2                  ; encoding: [0x65,0x02,0x5e,0xbf]
 
 s_cmp_nlt_f16 vcc_lo, s2
-// GFX12: encoding: [0x6a,0x02,0x5e,0xbf]
+// GFX12: s_cmp_nlt_f16 vcc_lo, s2                ; encoding: [0x6a,0x02,0x5e,0xbf]
 
 s_cmp_nlt_f16 vcc_hi, s2
-// GFX12: encoding: [0x6b,0x02,0x5e,0xbf]
+// GFX12: s_cmp_nlt_f16 vcc_hi, s2                ; encoding: [0x6b,0x02,0x5e,0xbf]
 
 s_cmp_nlt_f16 m0, s2
-// GFX12: encoding: [0x7d,0x02,0x5e,0xbf]
+// GFX12: s_cmp_nlt_f16 m0, s2                    ; encoding: [0x7d,0x02,0x5e,0xbf]
 
 s_cmp_nlt_f16 exec_lo, s2
-// GFX12: encoding: [0x7e,0x02,0x5e,0xbf]
+// GFX12: s_cmp_nlt_f16 exec_lo, s2               ; encoding: [0x7e,0x02,0x5e,0xbf]
 
 s_cmp_nlt_f16 exec_hi, s2
-// GFX12: encoding: [0x7f,0x02,0x5e,0xbf]
+// GFX12: s_cmp_nlt_f16 exec_hi, s2               ; encoding: [0x7f,0x02,0x5e,0xbf]
 
 s_cmp_nlt_f16 0, s2
-// GFX12: encoding: [0x80,0x02,0x5e,0xbf]
+// GFX12: s_cmp_nlt_f16 0, s2                     ; encoding: [0x80,0x02,0x5e,0xbf]
 
 s_cmp_nlt_f16 -1, s2
-// GFX12: encoding: [0xc1,0x02,0x5e,0xbf]
+// GFX12: s_cmp_nlt_f16 -1, s2                    ; encoding: [0xc1,0x02,0x5e,0xbf]
 
 s_cmp_nlt_f16 0x3800, s2
-// GFX12: encoding: [0xff,0x02,0x5e,0xbf,0x00,0x38,0x00,0x00]
+// GFX12: s_cmp_nlt_f16 0x3800, s2                ; encoding: [0xff,0x02,0x5e,0xbf,0x00,0x38,0x00,0x00]
 
 s_cmp_nlt_f16 0xfe0b, s2
-// GFX12: encoding: [0xff,0x02,0x5e,0xbf,0x0b,0xfe,0x00,0x00]
+// GFX12: s_cmp_nlt_f16 0xfe0b, s2                ; encoding: [0xff,0x02,0x5e,0xbf,0x0b,0xfe,0x00,0x00]
 
 s_cmp_nlt_f16 0x3456, s2
-// GFX12: encoding: [0xff,0x02,0x5e,0xbf,0x56,0x34,0x00,0x00]
+// GFX12: s_cmp_nlt_f16 0x3456, s2                ; encoding: [0xff,0x02,0x5e,0xbf,0x56,0x34,0x00,0x00]
 
 s_cmp_eq_i32 s0, s1
-// GFX12: encoding: [0x00,0x01,0x00,0xbf]
+// GFX12: s_cmp_eq_i32 s0, s1                     ; encoding: [0x00,0x01,0x00,0xbf]
 
 s_cmp_eq_i32 s105, s104
-// GFX12: encoding: [0x69,0x68,0x00,0xbf]
+// GFX12: s_cmp_eq_i32 s105, s104                 ; encoding: [0x69,0x68,0x00,0xbf]
 
 s_cmp_eq_i32 s0, s104
-// GFX12: encoding: [0x00,0x68,0x00,0xbf]
+// GFX12: s_cmp_eq_i32 s0, s104                   ; encoding: [0x00,0x68,0x00,0xbf]
 
 s_cmp_eq_i32 s105, s1
-// GFX12: encoding: [0x69,0x01,0x00,0xbf]
+// GFX12: s_cmp_eq_i32 s105, s1                   ; encoding: [0x69,0x01,0x00,0xbf]
 
 s_cmp_eq_i32 exec_lo, s1
-// GFX12: encoding: [0x7e,0x01,0x00,0xbf]
+// GFX12: s_cmp_eq_i32 exec_lo, s1                ; encoding: [0x7e,0x01,0x00,0xbf]
 
 s_cmp_eq_i32 exec_hi, s1
-// GFX12: encoding: [0x7f,0x01,0x00,0xbf]
+// GFX12: s_cmp_eq_i32 exec_hi, s1                ; encoding: [0x7f,0x01,0x00,0xbf]
 
 s_cmp_eq_i32 vcc_lo, s1
-// GFX12: encoding: [0x6a,0x01,0x00,0xbf]
+// GFX12: s_cmp_eq_i32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x00,0xbf]
 
 s_cmp_eq_i32 vcc_hi, s1
-// GFX12: encoding: [0x6b,0x01,0x00,0xbf]
+// GFX12: s_cmp_eq_i32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x00,0xbf]
 
 s_cmp_eq_i32 m0, s1
-// GFX12: encoding: [0x7d,0x01,0x00,0xbf]
+// GFX12: s_cmp_eq_i32 m0, s1                     ; encoding: [0x7d,0x01,0x00,0xbf]
 
 s_cmp_eq_i32 s0, exec_lo
-// GFX12: encoding: [0x00,0x7e,0x00,0xbf]
+// GFX12: s_cmp_eq_i32 s0, exec_lo                ; encoding: [0x00,0x7e,0x00,0xbf]
 
 s_cmp_eq_i32 s0, exec_hi
-// GFX12: encoding: [0x00,0x7f,0x00,0xbf]
+// GFX12: s_cmp_eq_i32 s0, exec_hi                ; encoding: [0x00,0x7f,0x00,0xbf]
 
 s_cmp_eq_i32 s0, vcc_lo
-// GFX12: encoding: [0x00,0x6a,0x00,0xbf]
+// GFX12: s_cmp_eq_i32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x00,0xbf]
 
 s_cmp_eq_i32 s0, vcc_hi
-// GFX12: encoding: [0x00,0x6b,0x00,0xbf]
+// GFX12: s_cmp_eq_i32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x00,0xbf]
 
 s_cmp_eq_i32 s0, m0
-// GFX12: encoding: [0x00,0x7d,0x00,0xbf]
+// GFX12: s_cmp_eq_i32 s0, m0                     ; encoding: [0x00,0x7d,0x00,0xbf]
 
 s_cmp_eq_i32 s0, 0
-// GFX12: encoding: [0x00,0x80,0x00,0xbf]
+// GFX12: s_cmp_eq_i32 s0, 0                      ; encoding: [0x00,0x80,0x00,0xbf]
 
 s_cmp_eq_i32 s0, -1
-// GFX12: encoding: [0x00,0xc1,0x00,0xbf]
+// GFX12: s_cmp_eq_i32 s0, -1                     ; encoding: [0x00,0xc1,0x00,0xbf]
 
 s_cmp_eq_i32 s0, 0.5
-// GFX12: encoding: [0x00,0xf0,0x00,0xbf]
+// GFX12: s_cmp_eq_i32 s0, 0.5                    ; encoding: [0x00,0xf0,0x00,0xbf]
 
 s_cmp_eq_i32 s0, -4.0
-// GFX12: encoding: [0x00,0xf7,0x00,0xbf]
+// GFX12: s_cmp_eq_i32 s0, -4.0                   ; encoding: [0x00,0xf7,0x00,0xbf]
 
 s_cmp_eq_i32 s0, 0x3f717273
-// GFX12: encoding: [0x00,0xff,0x00,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_eq_i32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x00,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_eq_i32 s0, 0xaf123456
-// GFX12: encoding: [0x00,0xff,0x00,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_eq_i32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x00,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_lg_i32 s0, s1
-// GFX12: encoding: [0x00,0x01,0x01,0xbf]
+// GFX12: s_cmp_lg_i32 s0, s1                     ; encoding: [0x00,0x01,0x01,0xbf]
 
 s_cmp_lg_i32 s105, s104
-// GFX12: encoding: [0x69,0x68,0x01,0xbf]
+// GFX12: s_cmp_lg_i32 s105, s104                 ; encoding: [0x69,0x68,0x01,0xbf]
 
 s_cmp_lg_i32 s0, s104
-// GFX12: encoding: [0x00,0x68,0x01,0xbf]
+// GFX12: s_cmp_lg_i32 s0, s104                   ; encoding: [0x00,0x68,0x01,0xbf]
 
 s_cmp_lg_i32 s105, s1
-// GFX12: encoding: [0x69,0x01,0x01,0xbf]
+// GFX12: s_cmp_lg_i32 s105, s1                   ; encoding: [0x69,0x01,0x01,0xbf]
 
 s_cmp_lg_i32 exec_lo, s1
-// GFX12: encoding: [0x7e,0x01,0x01,0xbf]
+// GFX12: s_cmp_lg_i32 exec_lo, s1                ; encoding: [0x7e,0x01,0x01,0xbf]
 
 s_cmp_lg_i32 exec_hi, s1
-// GFX12: encoding: [0x7f,0x01,0x01,0xbf]
+// GFX12: s_cmp_lg_i32 exec_hi, s1                ; encoding: [0x7f,0x01,0x01,0xbf]
 
 s_cmp_lg_i32 vcc_lo, s1
-// GFX12: encoding: [0x6a,0x01,0x01,0xbf]
+// GFX12: s_cmp_lg_i32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x01,0xbf]
 
 s_cmp_lg_i32 vcc_hi, s1
-// GFX12: encoding: [0x6b,0x01,0x01,0xbf]
+// GFX12: s_cmp_lg_i32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x01,0xbf]
 
 s_cmp_lg_i32 m0, s1
-// GFX12: encoding: [0x7d,0x01,0x01,0xbf]
+// GFX12: s_cmp_lg_i32 m0, s1                     ; encoding: [0x7d,0x01,0x01,0xbf]
 
 s_cmp_lg_i32 s0, exec_lo
-// GFX12: encoding: [0x00,0x7e,0x01,0xbf]
+// GFX12: s_cmp_lg_i32 s0, exec_lo                ; encoding: [0x00,0x7e,0x01,0xbf]
 
 s_cmp_lg_i32 s0, exec_hi
-// GFX12: encoding: [0x00,0x7f,0x01,0xbf]
+// GFX12: s_cmp_lg_i32 s0, exec_hi                ; encoding: [0x00,0x7f,0x01,0xbf]
 
 s_cmp_lg_i32 s0, vcc_lo
-// GFX12: encoding: [0x00,0x6a,0x01,0xbf]
+// GFX12: s_cmp_lg_i32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x01,0xbf]
 
 s_cmp_lg_i32 s0, vcc_hi
-// GFX12: encoding: [0x00,0x6b,0x01,0xbf]
+// GFX12: s_cmp_lg_i32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x01,0xbf]
 
 s_cmp_lg_i32 s0, m0
-// GFX12: encoding: [0x00,0x7d,0x01,0xbf]
+// GFX12: s_cmp_lg_i32 s0, m0                     ; encoding: [0x00,0x7d,0x01,0xbf]
 
 s_cmp_lg_i32 s0, 0
-// GFX12: encoding: [0x00,0x80,0x01,0xbf]
+// GFX12: s_cmp_lg_i32 s0, 0                      ; encoding: [0x00,0x80,0x01,0xbf]
 
 s_cmp_lg_i32 s0, -1
-// GFX12: encoding: [0x00,0xc1,0x01,0xbf]
+// GFX12: s_cmp_lg_i32 s0, -1                     ; encoding: [0x00,0xc1,0x01,0xbf]
 
 s_cmp_lg_i32 s0, 0.5
-// GFX12: encoding: [0x00,0xf0,0x01,0xbf]
+// GFX12: s_cmp_lg_i32 s0, 0.5                    ; encoding: [0x00,0xf0,0x01,0xbf]
 
 s_cmp_lg_i32 s0, -4.0
-// GFX12: encoding: [0x00,0xf7,0x01,0xbf]
+// GFX12: s_cmp_lg_i32 s0, -4.0                   ; encoding: [0x00,0xf7,0x01,0xbf]
 
 s_cmp_lg_i32 s0, 0x3f717273
-// GFX12: encoding: [0x00,0xff,0x01,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_lg_i32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x01,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_lg_i32 s0, 0xaf123456
-// GFX12: encoding: [0x00,0xff,0x01,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_lg_i32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x01,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_gt_i32 s0, s1
-// GFX12: encoding: [0x00,0x01,0x02,0xbf]
+// GFX12: s_cmp_gt_i32 s0, s1                     ; encoding: [0x00,0x01,0x02,0xbf]
 
 s_cmp_gt_i32 s105, s104
-// GFX12: encoding: [0x69,0x68,0x02,0xbf]
+// GFX12: s_cmp_gt_i32 s105, s104                 ; encoding: [0x69,0x68,0x02,0xbf]
 
 s_cmp_gt_i32 s0, s104
-// GFX12: encoding: [0x00,0x68,0x02,0xbf]
+// GFX12: s_cmp_gt_i32 s0, s104                   ; encoding: [0x00,0x68,0x02,0xbf]
 
 s_cmp_gt_i32 s105, s1
-// GFX12: encoding: [0x69,0x01,0x02,0xbf]
+// GFX12: s_cmp_gt_i32 s105, s1                   ; encoding: [0x69,0x01,0x02,0xbf]
 
 s_cmp_gt_i32 exec_lo, s1
-// GFX12: encoding: [0x7e,0x01,0x02,0xbf]
+// GFX12: s_cmp_gt_i32 exec_lo, s1                ; encoding: [0x7e,0x01,0x02,0xbf]
 
 s_cmp_gt_i32 exec_hi, s1
-// GFX12: encoding: [0x7f,0x01,0x02,0xbf]
+// GFX12: s_cmp_gt_i32 exec_hi, s1                ; encoding: [0x7f,0x01,0x02,0xbf]
 
 s_cmp_gt_i32 vcc_lo, s1
-// GFX12: encoding: [0x6a,0x01,0x02,0xbf]
+// GFX12: s_cmp_gt_i32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x02,0xbf]
 
 s_cmp_gt_i32 vcc_hi, s1
-// GFX12: encoding: [0x6b,0x01,0x02,0xbf]
+// GFX12: s_cmp_gt_i32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x02,0xbf]
 
 s_cmp_gt_i32 m0, s1
-// GFX12: encoding: [0x7d,0x01,0x02,0xbf]
+// GFX12: s_cmp_gt_i32 m0, s1                     ; encoding: [0x7d,0x01,0x02,0xbf]
 
 s_cmp_gt_i32 s0, exec_lo
-// GFX12: encoding: [0x00,0x7e,0x02,0xbf]
+// GFX12: s_cmp_gt_i32 s0, exec_lo                ; encoding: [0x00,0x7e,0x02,0xbf]
 
 s_cmp_gt_i32 s0, exec_hi
-// GFX12: encoding: [0x00,0x7f,0x02,0xbf]
+// GFX12: s_cmp_gt_i32 s0, exec_hi                ; encoding: [0x00,0x7f,0x02,0xbf]
 
 s_cmp_gt_i32 s0, vcc_lo
-// GFX12: encoding: [0x00,0x6a,0x02,0xbf]
+// GFX12: s_cmp_gt_i32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x02,0xbf]
 
 s_cmp_gt_i32 s0, vcc_hi
-// GFX12: encoding: [0x00,0x6b,0x02,0xbf]
+// GFX12: s_cmp_gt_i32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x02,0xbf]
 
 s_cmp_gt_i32 s0, m0
-// GFX12: encoding: [0x00,0x7d,0x02,0xbf]
+// GFX12: s_cmp_gt_i32 s0, m0                     ; encoding: [0x00,0x7d,0x02,0xbf]
 
 s_cmp_gt_i32 s0, 0
-// GFX12: encoding: [0x00,0x80,0x02,0xbf]
+// GFX12: s_cmp_gt_i32 s0, 0                      ; encoding: [0x00,0x80,0x02,0xbf]
 
 s_cmp_gt_i32 s0, -1
-// GFX12: encoding: [0x00,0xc1,0x02,0xbf]
+// GFX12: s_cmp_gt_i32 s0, -1                     ; encoding: [0x00,0xc1,0x02,0xbf]
 
 s_cmp_gt_i32 s0, 0.5
-// GFX12: encoding: [0x00,0xf0,0x02,0xbf]
+// GFX12: s_cmp_gt_i32 s0, 0.5                    ; encoding: [0x00,0xf0,0x02,0xbf]
 
 s_cmp_gt_i32 s0, -4.0
-// GFX12: encoding: [0x00,0xf7,0x02,0xbf]
+// GFX12: s_cmp_gt_i32 s0, -4.0                   ; encoding: [0x00,0xf7,0x02,0xbf]
 
 s_cmp_gt_i32 s0, 0x3f717273
-// GFX12: encoding: [0x00,0xff,0x02,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_gt_i32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x02,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_gt_i32 s0, 0xaf123456
-// GFX12: encoding: [0x00,0xff,0x02,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_gt_i32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x02,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_ge_i32 s0, s1
-// GFX12: encoding: [0x00,0x01,0x03,0xbf]
+// GFX12: s_cmp_ge_i32 s0, s1                     ; encoding: [0x00,0x01,0x03,0xbf]
 
 s_cmp_ge_i32 s105, s104
-// GFX12: encoding: [0x69,0x68,0x03,0xbf]
+// GFX12: s_cmp_ge_i32 s105, s104                 ; encoding: [0x69,0x68,0x03,0xbf]
 
 s_cmp_ge_i32 s0, s104
-// GFX12: encoding: [0x00,0x68,0x03,0xbf]
+// GFX12: s_cmp_ge_i32 s0, s104                   ; encoding: [0x00,0x68,0x03,0xbf]
 
 s_cmp_ge_i32 s105, s1
-// GFX12: encoding: [0x69,0x01,0x03,0xbf]
+// GFX12: s_cmp_ge_i32 s105, s1                   ; encoding: [0x69,0x01,0x03,0xbf]
 
 s_cmp_ge_i32 exec_lo, s1
-// GFX12: encoding: [0x7e,0x01,0x03,0xbf]
+// GFX12: s_cmp_ge_i32 exec_lo, s1                ; encoding: [0x7e,0x01,0x03,0xbf]
 
 s_cmp_ge_i32 exec_hi, s1
-// GFX12: encoding: [0x7f,0x01,0x03,0xbf]
+// GFX12: s_cmp_ge_i32 exec_hi, s1                ; encoding: [0x7f,0x01,0x03,0xbf]
 
 s_cmp_ge_i32 vcc_lo, s1
-// GFX12: encoding: [0x6a,0x01,0x03,0xbf]
+// GFX12: s_cmp_ge_i32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x03,0xbf]
 
 s_cmp_ge_i32 vcc_hi, s1
-// GFX12: encoding: [0x6b,0x01,0x03,0xbf]
+// GFX12: s_cmp_ge_i32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x03,0xbf]
 
 s_cmp_ge_i32 m0, s1
-// GFX12: encoding: [0x7d,0x01,0x03,0xbf]
+// GFX12: s_cmp_ge_i32 m0, s1                     ; encoding: [0x7d,0x01,0x03,0xbf]
 
 s_cmp_ge_i32 s0, exec_lo
-// GFX12: encoding: [0x00,0x7e,0x03,0xbf]
+// GFX12: s_cmp_ge_i32 s0, exec_lo                ; encoding: [0x00,0x7e,0x03,0xbf]
 
 s_cmp_ge_i32 s0, exec_hi
-// GFX12: encoding: [0x00,0x7f,0x03,0xbf]
+// GFX12: s_cmp_ge_i32 s0, exec_hi                ; encoding: [0x00,0x7f,0x03,0xbf]
 
 s_cmp_ge_i32 s0, vcc_lo
-// GFX12: encoding: [0x00,0x6a,0x03,0xbf]
+// GFX12: s_cmp_ge_i32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x03,0xbf]
 
 s_cmp_ge_i32 s0, vcc_hi
-// GFX12: encoding: [0x00,0x6b,0x03,0xbf]
+// GFX12: s_cmp_ge_i32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x03,0xbf]
 
 s_cmp_ge_i32 s0, m0
-// GFX12: encoding: [0x00,0x7d,0x03,0xbf]
+// GFX12: s_cmp_ge_i32 s0, m0                     ; encoding: [0x00,0x7d,0x03,0xbf]
 
 s_cmp_ge_i32 s0, 0
-// GFX12: encoding: [0x00,0x80,0x03,0xbf]
+// GFX12: s_cmp_ge_i32 s0, 0                      ; encoding: [0x00,0x80,0x03,0xbf]
 
 s_cmp_ge_i32 s0, -1
-// GFX12: encoding: [0x00,0xc1,0x03,0xbf]
+// GFX12: s_cmp_ge_i32 s0, -1                     ; encoding: [0x00,0xc1,0x03,0xbf]
 
 s_cmp_ge_i32 s0, 0.5
-// GFX12: encoding: [0x00,0xf0,0x03,0xbf]
+// GFX12: s_cmp_ge_i32 s0, 0.5                    ; encoding: [0x00,0xf0,0x03,0xbf]
 
 s_cmp_ge_i32 s0, -4.0
-// GFX12: encoding: [0x00,0xf7,0x03,0xbf]
+// GFX12: s_cmp_ge_i32 s0, -4.0                   ; encoding: [0x00,0xf7,0x03,0xbf]
 
 s_cmp_ge_i32 s0, 0x3f717273
-// GFX12: encoding: [0x00,0xff,0x03,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_ge_i32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x03,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_ge_i32 s0, 0xaf123456
-// GFX12: encoding: [0x00,0xff,0x03,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_ge_i32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x03,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_lt_i32 s0, s1
-// GFX12: encoding: [0x00,0x01,0x04,0xbf]
+// GFX12: s_cmp_lt_i32 s0, s1                     ; encoding: [0x00,0x01,0x04,0xbf]
 
 s_cmp_lt_i32 s105, s104
-// GFX12: encoding: [0x69,0x68,0x04,0xbf]
+// GFX12: s_cmp_lt_i32 s105, s104                 ; encoding: [0x69,0x68,0x04,0xbf]
 
 s_cmp_lt_i32 s0, s104
-// GFX12: encoding: [0x00,0x68,0x04,0xbf]
+// GFX12: s_cmp_lt_i32 s0, s104                   ; encoding: [0x00,0x68,0x04,0xbf]
 
 s_cmp_lt_i32 s105, s1
-// GFX12: encoding: [0x69,0x01,0x04,0xbf]
+// GFX12: s_cmp_lt_i32 s105, s1                   ; encoding: [0x69,0x01,0x04,0xbf]
 
 s_cmp_lt_i32 exec_lo, s1
-// GFX12: encoding: [0x7e,0x01,0x04,0xbf]
+// GFX12: s_cmp_lt_i32 exec_lo, s1                ; encoding: [0x7e,0x01,0x04,0xbf]
 
 s_cmp_lt_i32 exec_hi, s1
-// GFX12: encoding: [0x7f,0x01,0x04,0xbf]
+// GFX12: s_cmp_lt_i32 exec_hi, s1                ; encoding: [0x7f,0x01,0x04,0xbf]
 
 s_cmp_lt_i32 vcc_lo, s1
-// GFX12: encoding: [0x6a,0x01,0x04,0xbf]
+// GFX12: s_cmp_lt_i32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x04,0xbf]
 
 s_cmp_lt_i32 vcc_hi, s1
-// GFX12: encoding: [0x6b,0x01,0x04,0xbf]
+// GFX12: s_cmp_lt_i32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x04,0xbf]
 
 s_cmp_lt_i32 m0, s1
-// GFX12: encoding: [0x7d,0x01,0x04,0xbf]
+// GFX12: s_cmp_lt_i32 m0, s1                     ; encoding: [0x7d,0x01,0x04,0xbf]
 
 s_cmp_lt_i32 s0, exec_lo
-// GFX12: encoding: [0x00,0x7e,0x04,0xbf]
+// GFX12: s_cmp_lt_i32 s0, exec_lo                ; encoding: [0x00,0x7e,0x04,0xbf]
 
 s_cmp_lt_i32 s0, exec_hi
-// GFX12: encoding: [0x00,0x7f,0x04,0xbf]
+// GFX12: s_cmp_lt_i32 s0, exec_hi                ; encoding: [0x00,0x7f,0x04,0xbf]
 
 s_cmp_lt_i32 s0, vcc_lo
-// GFX12: encoding: [0x00,0x6a,0x04,0xbf]
+// GFX12: s_cmp_lt_i32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x04,0xbf]
 
 s_cmp_lt_i32 s0, vcc_hi
-// GFX12: encoding: [0x00,0x6b,0x04,0xbf]
+// GFX12: s_cmp_lt_i32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x04,0xbf]
 
 s_cmp_lt_i32 s0, m0
-// GFX12: encoding: [0x00,0x7d,0x04,0xbf]
+// GFX12: s_cmp_lt_i32 s0, m0                     ; encoding: [0x00,0x7d,0x04,0xbf]
 
 s_cmp_lt_i32 s0, 0
-// GFX12: encoding: [0x00,0x80,0x04,0xbf]
+// GFX12: s_cmp_lt_i32 s0, 0                      ; encoding: [0x00,0x80,0x04,0xbf]
 
 s_cmp_lt_i32 s0, -1
-// GFX12: encoding: [0x00,0xc1,0x04,0xbf]
+// GFX12: s_cmp_lt_i32 s0, -1                     ; encoding: [0x00,0xc1,0x04,0xbf]
 
 s_cmp_lt_i32 s0, 0.5
-// GFX12: encoding: [0x00,0xf0,0x04,0xbf]
+// GFX12: s_cmp_lt_i32 s0, 0.5                    ; encoding: [0x00,0xf0,0x04,0xbf]
 
 s_cmp_lt_i32 s0, -4.0
-// GFX12: encoding: [0x00,0xf7,0x04,0xbf]
+// GFX12: s_cmp_lt_i32 s0, -4.0                   ; encoding: [0x00,0xf7,0x04,0xbf]
 
 s_cmp_lt_i32 s0, 0x3f717273
-// GFX12: encoding: [0x00,0xff,0x04,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_lt_i32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x04,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_lt_i32 s0, 0xaf123456
-// GFX12: encoding: [0x00,0xff,0x04,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_lt_i32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x04,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_le_i32 s0, s1
-// GFX12: encoding: [0x00,0x01,0x05,0xbf]
+// GFX12: s_cmp_le_i32 s0, s1                     ; encoding: [0x00,0x01,0x05,0xbf]
 
 s_cmp_le_i32 s105, s104
-// GFX12: encoding: [0x69,0x68,0x05,0xbf]
+// GFX12: s_cmp_le_i32 s105, s104                 ; encoding: [0x69,0x68,0x05,0xbf]
 
 s_cmp_le_i32 s0, s104
-// GFX12: encoding: [0x00,0x68,0x05,0xbf]
+// GFX12: s_cmp_le_i32 s0, s104                   ; encoding: [0x00,0x68,0x05,0xbf]
 
 s_cmp_le_i32 s105, s1
-// GFX12: encoding: [0x69,0x01,0x05,0xbf]
+// GFX12: s_cmp_le_i32 s105, s1                   ; encoding: [0x69,0x01,0x05,0xbf]
 
 s_cmp_le_i32 exec_lo, s1
-// GFX12: encoding: [0x7e,0x01,0x05,0xbf]
+// GFX12: s_cmp_le_i32 exec_lo, s1                ; encoding: [0x7e,0x01,0x05,0xbf]
 
 s_cmp_le_i32 exec_hi, s1
-// GFX12: encoding: [0x7f,0x01,0x05,0xbf]
+// GFX12: s_cmp_le_i32 exec_hi, s1                ; encoding: [0x7f,0x01,0x05,0xbf]
 
 s_cmp_le_i32 vcc_lo, s1
-// GFX12: encoding: [0x6a,0x01,0x05,0xbf]
+// GFX12: s_cmp_le_i32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x05,0xbf]
 
 s_cmp_le_i32 vcc_hi, s1
-// GFX12: encoding: [0x6b,0x01,0x05,0xbf]
+// GFX12: s_cmp_le_i32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x05,0xbf]
 
 s_cmp_le_i32 m0, s1
-// GFX12: encoding: [0x7d,0x01,0x05,0xbf]
+// GFX12: s_cmp_le_i32 m0, s1                     ; encoding: [0x7d,0x01,0x05,0xbf]
 
 s_cmp_le_i32 s0, exec_lo
-// GFX12: encoding: [0x00,0x7e,0x05,0xbf]
+// GFX12: s_cmp_le_i32 s0, exec_lo                ; encoding: [0x00,0x7e,0x05,0xbf]
 
 s_cmp_le_i32 s0, exec_hi
-// GFX12: encoding: [0x00,0x7f,0x05,0xbf]
+// GFX12: s_cmp_le_i32 s0, exec_hi                ; encoding: [0x00,0x7f,0x05,0xbf]
 
 s_cmp_le_i32 s0, vcc_lo
-// GFX12: encoding: [0x00,0x6a,0x05,0xbf]
+// GFX12: s_cmp_le_i32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x05,0xbf]
 
 s_cmp_le_i32 s0, vcc_hi
-// GFX12: encoding: [0x00,0x6b,0x05,0xbf]
+// GFX12: s_cmp_le_i32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x05,0xbf]
 
 s_cmp_le_i32 s0, m0
-// GFX12: encoding: [0x00,0x7d,0x05,0xbf]
+// GFX12: s_cmp_le_i32 s0, m0                     ; encoding: [0x00,0x7d,0x05,0xbf]
 
 s_cmp_le_i32 s0, 0
-// GFX12: encoding: [0x00,0x80,0x05,0xbf]
+// GFX12: s_cmp_le_i32 s0, 0                      ; encoding: [0x00,0x80,0x05,0xbf]
 
 s_cmp_le_i32 s0, -1
-// GFX12: encoding: [0x00,0xc1,0x05,0xbf]
+// GFX12: s_cmp_le_i32 s0, -1                     ; encoding: [0x00,0xc1,0x05,0xbf]
 
 s_cmp_le_i32 s0, 0.5
-// GFX12: encoding: [0x00,0xf0,0x05,0xbf]
+// GFX12: s_cmp_le_i32 s0, 0.5                    ; encoding: [0x00,0xf0,0x05,0xbf]
 
 s_cmp_le_i32 s0, -4.0
-// GFX12: encoding: [0x00,0xf7,0x05,0xbf]
+// GFX12: s_cmp_le_i32 s0, -4.0                   ; encoding: [0x00,0xf7,0x05,0xbf]
 
 s_cmp_le_i32 s0, 0x3f717273
-// GFX12: encoding: [0x00,0xff,0x05,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_le_i32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x05,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_le_i32 s0, 0xaf123456
-// GFX12: encoding: [0x00,0xff,0x05,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_le_i32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x05,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_eq_u32 s0, s1
-// GFX12: encoding: [0x00,0x01,0x06,0xbf]
+// GFX12: s_cmp_eq_u32 s0, s1                     ; encoding: [0x00,0x01,0x06,0xbf]
 
 s_cmp_eq_u32 s105, s104
-// GFX12: encoding: [0x69,0x68,0x06,0xbf]
+// GFX12: s_cmp_eq_u32 s105, s104                 ; encoding: [0x69,0x68,0x06,0xbf]
 
 s_cmp_eq_u32 s0, s104
-// GFX12: encoding: [0x00,0x68,0x06,0xbf]
+// GFX12: s_cmp_eq_u32 s0, s104                   ; encoding: [0x00,0x68,0x06,0xbf]
 
 s_cmp_eq_u32 s105, s1
-// GFX12: encoding: [0x69,0x01,0x06,0xbf]
+// GFX12: s_cmp_eq_u32 s105, s1                   ; encoding: [0x69,0x01,0x06,0xbf]
 
 s_cmp_eq_u32 exec_lo, s1
-// GFX12: encoding: [0x7e,0x01,0x06,0xbf]
+// GFX12: s_cmp_eq_u32 exec_lo, s1                ; encoding: [0x7e,0x01,0x06,0xbf]
 
 s_cmp_eq_u32 exec_hi, s1
-// GFX12: encoding: [0x7f,0x01,0x06,0xbf]
+// GFX12: s_cmp_eq_u32 exec_hi, s1                ; encoding: [0x7f,0x01,0x06,0xbf]
 
 s_cmp_eq_u32 vcc_lo, s1
-// GFX12: encoding: [0x6a,0x01,0x06,0xbf]
+// GFX12: s_cmp_eq_u32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x06,0xbf]
 
 s_cmp_eq_u32 vcc_hi, s1
-// GFX12: encoding: [0x6b,0x01,0x06,0xbf]
+// GFX12: s_cmp_eq_u32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x06,0xbf]
 
 s_cmp_eq_u32 m0, s1
-// GFX12: encoding: [0x7d,0x01,0x06,0xbf]
+// GFX12: s_cmp_eq_u32 m0, s1                     ; encoding: [0x7d,0x01,0x06,0xbf]
 
 s_cmp_eq_u32 s0, exec_lo
-// GFX12: encoding: [0x00,0x7e,0x06,0xbf]
+// GFX12: s_cmp_eq_u32 s0, exec_lo                ; encoding: [0x00,0x7e,0x06,0xbf]
 
 s_cmp_eq_u32 s0, exec_hi
-// GFX12: encoding: [0x00,0x7f,0x06,0xbf]
+// GFX12: s_cmp_eq_u32 s0, exec_hi                ; encoding: [0x00,0x7f,0x06,0xbf]
 
 s_cmp_eq_u32 s0, vcc_lo
-// GFX12: encoding: [0x00,0x6a,0x06,0xbf]
+// GFX12: s_cmp_eq_u32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x06,0xbf]
 
 s_cmp_eq_u32 s0, vcc_hi
-// GFX12: encoding: [0x00,0x6b,0x06,0xbf]
+// GFX12: s_cmp_eq_u32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x06,0xbf]
 
 s_cmp_eq_u32 s0, m0
-// GFX12: encoding: [0x00,0x7d,0x06,0xbf]
+// GFX12: s_cmp_eq_u32 s0, m0                     ; encoding: [0x00,0x7d,0x06,0xbf]
 
 s_cmp_eq_u32 s0, 0
-// GFX12: encoding: [0x00,0x80,0x06,0xbf]
+// GFX12: s_cmp_eq_u32 s0, 0                      ; encoding: [0x00,0x80,0x06,0xbf]
 
 s_cmp_eq_u32 s0, -1
-// GFX12: encoding: [0x00,0xc1,0x06,0xbf]
+// GFX12: s_cmp_eq_u32 s0, -1                     ; encoding: [0x00,0xc1,0x06,0xbf]
 
 s_cmp_eq_u32 s0, 0.5
-// GFX12: encoding: [0x00,0xf0,0x06,0xbf]
+// GFX12: s_cmp_eq_u32 s0, 0.5                    ; encoding: [0x00,0xf0,0x06,0xbf]
 
 s_cmp_eq_u32 s0, -4.0
-// GFX12: encoding: [0x00,0xf7,0x06,0xbf]
+// GFX12: s_cmp_eq_u32 s0, -4.0                   ; encoding: [0x00,0xf7,0x06,0xbf]
 
 s_cmp_eq_u32 s0, 0x3f717273
-// GFX12: encoding: [0x00,0xff,0x06,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_eq_u32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x06,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_eq_u32 s0, 0xaf123456
-// GFX12: encoding: [0x00,0xff,0x06,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_eq_u32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x06,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_lg_u32 s0, s1
-// GFX12: encoding: [0x00,0x01,0x07,0xbf]
+// GFX12: s_cmp_lg_u32 s0, s1                     ; encoding: [0x00,0x01,0x07,0xbf]
 
 s_cmp_lg_u32 s105, s104
-// GFX12: encoding: [0x69,0x68,0x07,0xbf]
+// GFX12: s_cmp_lg_u32 s105, s104                 ; encoding: [0x69,0x68,0x07,0xbf]
 
 s_cmp_lg_u32 s0, s104
-// GFX12: encoding: [0x00,0x68,0x07,0xbf]
+// GFX12: s_cmp_lg_u32 s0, s104                   ; encoding: [0x00,0x68,0x07,0xbf]
 
 s_cmp_lg_u32 s105, s1
-// GFX12: encoding: [0x69,0x01,0x07,0xbf]
+// GFX12: s_cmp_lg_u32 s105, s1                   ; encoding: [0x69,0x01,0x07,0xbf]
 
 s_cmp_lg_u32 exec_lo, s1
-// GFX12: encoding: [0x7e,0x01,0x07,0xbf]
+// GFX12: s_cmp_lg_u32 exec_lo, s1                ; encoding: [0x7e,0x01,0x07,0xbf]
 
 s_cmp_lg_u32 exec_hi, s1
-// GFX12: encoding: [0x7f,0x01,0x07,0xbf]
+// GFX12: s_cmp_lg_u32 exec_hi, s1                ; encoding: [0x7f,0x01,0x07,0xbf]
 
 s_cmp_lg_u32 vcc_lo, s1
-// GFX12: encoding: [0x6a,0x01,0x07,0xbf]
+// GFX12: s_cmp_lg_u32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x07,0xbf]
 
 s_cmp_lg_u32 vcc_hi, s1
-// GFX12: encoding: [0x6b,0x01,0x07,0xbf]
+// GFX12: s_cmp_lg_u32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x07,0xbf]
 
 s_cmp_lg_u32 m0, s1
-// GFX12: encoding: [0x7d,0x01,0x07,0xbf]
+// GFX12: s_cmp_lg_u32 m0, s1                     ; encoding: [0x7d,0x01,0x07,0xbf]
 
 s_cmp_lg_u32 s0, exec_lo
-// GFX12: encoding: [0x00,0x7e,0x07,0xbf]
+// GFX12: s_cmp_lg_u32 s0, exec_lo                ; encoding: [0x00,0x7e,0x07,0xbf]
 
 s_cmp_lg_u32 s0, exec_hi
-// GFX12: encoding: [0x00,0x7f,0x07,0xbf]
+// GFX12: s_cmp_lg_u32 s0, exec_hi                ; encoding: [0x00,0x7f,0x07,0xbf]
 
 s_cmp_lg_u32 s0, vcc_lo
-// GFX12: encoding: [0x00,0x6a,0x07,0xbf]
+// GFX12: s_cmp_lg_u32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x07,0xbf]
 
 s_cmp_lg_u32 s0, vcc_hi
-// GFX12: encoding: [0x00,0x6b,0x07,0xbf]
+// GFX12: s_cmp_lg_u32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x07,0xbf]
 
 s_cmp_lg_u32 s0, m0
-// GFX12: encoding: [0x00,0x7d,0x07,0xbf]
+// GFX12: s_cmp_lg_u32 s0, m0                     ; encoding: [0x00,0x7d,0x07,0xbf]
 
 s_cmp_lg_u32 s0, 0
-// GFX12: encoding: [0x00,0x80,0x07,0xbf]
+// GFX12: s_cmp_lg_u32 s0, 0                      ; encoding: [0x00,0x80,0x07,0xbf]
 
 s_cmp_lg_u32 s0, -1
-// GFX12: encoding: [0x00,0xc1,0x07,0xbf]
+// GFX12: s_cmp_lg_u32 s0, -1                     ; encoding: [0x00,0xc1,0x07,0xbf]
 
 s_cmp_lg_u32 s0, 0.5
-// GFX12: encoding: [0x00,0xf0,0x07,0xbf]
+// GFX12: s_cmp_lg_u32 s0, 0.5                    ; encoding: [0x00,0xf0,0x07,0xbf]
 
 s_cmp_lg_u32 s0, -4.0
-// GFX12: encoding: [0x00,0xf7,0x07,0xbf]
+// GFX12: s_cmp_lg_u32 s0, -4.0                   ; encoding: [0x00,0xf7,0x07,0xbf]
 
 s_cmp_lg_u32 s0, 0x3f717273
-// GFX12: encoding: [0x00,0xff,0x07,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_lg_u32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x07,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_lg_u32 s0, 0xaf123456
-// GFX12: encoding: [0x00,0xff,0x07,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_lg_u32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x07,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_gt_u32 s0, s1
-// GFX12: encoding: [0x00,0x01,0x08,0xbf]
+// GFX12: s_cmp_gt_u32 s0, s1                     ; encoding: [0x00,0x01,0x08,0xbf]
 
 s_cmp_gt_u32 s105, s104
-// GFX12: encoding: [0x69,0x68,0x08,0xbf]
+// GFX12: s_cmp_gt_u32 s105, s104                 ; encoding: [0x69,0x68,0x08,0xbf]
 
 s_cmp_gt_u32 s0, s104
-// GFX12: encoding: [0x00,0x68,0x08,0xbf]
+// GFX12: s_cmp_gt_u32 s0, s104                   ; encoding: [0x00,0x68,0x08,0xbf]
 
 s_cmp_gt_u32 s105, s1
-// GFX12: encoding: [0x69,0x01,0x08,0xbf]
+// GFX12: s_cmp_gt_u32 s105, s1                   ; encoding: [0x69,0x01,0x08,0xbf]
 
 s_cmp_gt_u32 exec_lo, s1
-// GFX12: encoding: [0x7e,0x01,0x08,0xbf]
+// GFX12: s_cmp_gt_u32 exec_lo, s1                ; encoding: [0x7e,0x01,0x08,0xbf]
 
 s_cmp_gt_u32 exec_hi, s1
-// GFX12: encoding: [0x7f,0x01,0x08,0xbf]
+// GFX12: s_cmp_gt_u32 exec_hi, s1                ; encoding: [0x7f,0x01,0x08,0xbf]
 
 s_cmp_gt_u32 vcc_lo, s1
-// GFX12: encoding: [0x6a,0x01,0x08,0xbf]
+// GFX12: s_cmp_gt_u32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x08,0xbf]
 
 s_cmp_gt_u32 vcc_hi, s1
-// GFX12: encoding: [0x6b,0x01,0x08,0xbf]
+// GFX12: s_cmp_gt_u32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x08,0xbf]
 
 s_cmp_gt_u32 m0, s1
-// GFX12: encoding: [0x7d,0x01,0x08,0xbf]
+// GFX12: s_cmp_gt_u32 m0, s1                     ; encoding: [0x7d,0x01,0x08,0xbf]
 
 s_cmp_gt_u32 s0, exec_lo
-// GFX12: encoding: [0x00,0x7e,0x08,0xbf]
+// GFX12: s_cmp_gt_u32 s0, exec_lo                ; encoding: [0x00,0x7e,0x08,0xbf]
 
 s_cmp_gt_u32 s0, exec_hi
-// GFX12: encoding: [0x00,0x7f,0x08,0xbf]
+// GFX12: s_cmp_gt_u32 s0, exec_hi                ; encoding: [0x00,0x7f,0x08,0xbf]
 
 s_cmp_gt_u32 s0, vcc_lo
-// GFX12: encoding: [0x00,0x6a,0x08,0xbf]
+// GFX12: s_cmp_gt_u32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x08,0xbf]
 
 s_cmp_gt_u32 s0, vcc_hi
-// GFX12: encoding: [0x00,0x6b,0x08,0xbf]
+// GFX12: s_cmp_gt_u32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x08,0xbf]
 
 s_cmp_gt_u32 s0, m0
-// GFX12: encoding: [0x00,0x7d,0x08,0xbf]
+// GFX12: s_cmp_gt_u32 s0, m0                     ; encoding: [0x00,0x7d,0x08,0xbf]
 
 s_cmp_gt_u32 s0, 0
-// GFX12: encoding: [0x00,0x80,0x08,0xbf]
+// GFX12: s_cmp_gt_u32 s0, 0                      ; encoding: [0x00,0x80,0x08,0xbf]
 
 s_cmp_gt_u32 s0, -1
-// GFX12: encoding: [0x00,0xc1,0x08,0xbf]
+// GFX12: s_cmp_gt_u32 s0, -1                     ; encoding: [0x00,0xc1,0x08,0xbf]
 
 s_cmp_gt_u32 s0, 0.5
-// GFX12: encoding: [0x00,0xf0,0x08,0xbf]
+// GFX12: s_cmp_gt_u32 s0, 0.5                    ; encoding: [0x00,0xf0,0x08,0xbf]
 
 s_cmp_gt_u32 s0, -4.0
-// GFX12: encoding: [0x00,0xf7,0x08,0xbf]
+// GFX12: s_cmp_gt_u32 s0, -4.0                   ; encoding: [0x00,0xf7,0x08,0xbf]
 
 s_cmp_gt_u32 s0, 0x3f717273
-// GFX12: encoding: [0x00,0xff,0x08,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_gt_u32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x08,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_gt_u32 s0, 0xaf123456
-// GFX12: encoding: [0x00,0xff,0x08,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_gt_u32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x08,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_ge_u32 s0, s1
-// GFX12: encoding: [0x00,0x01,0x09,0xbf]
+// GFX12: s_cmp_ge_u32 s0, s1                     ; encoding: [0x00,0x01,0x09,0xbf]
 
 s_cmp_ge_u32 s105, s104
-// GFX12: encoding: [0x69,0x68,0x09,0xbf]
+// GFX12: s_cmp_ge_u32 s105, s104                 ; encoding: [0x69,0x68,0x09,0xbf]
 
 s_cmp_ge_u32 s0, s104
-// GFX12: encoding: [0x00,0x68,0x09,0xbf]
+// GFX12: s_cmp_ge_u32 s0, s104                   ; encoding: [0x00,0x68,0x09,0xbf]
 
 s_cmp_ge_u32 s105, s1
-// GFX12: encoding: [0x69,0x01,0x09,0xbf]
+// GFX12: s_cmp_ge_u32 s105, s1                   ; encoding: [0x69,0x01,0x09,0xbf]
 
 s_cmp_ge_u32 exec_lo, s1
-// GFX12: encoding: [0x7e,0x01,0x09,0xbf]
+// GFX12: s_cmp_ge_u32 exec_lo, s1                ; encoding: [0x7e,0x01,0x09,0xbf]
 
 s_cmp_ge_u32 exec_hi, s1
-// GFX12: encoding: [0x7f,0x01,0x09,0xbf]
+// GFX12: s_cmp_ge_u32 exec_hi, s1                ; encoding: [0x7f,0x01,0x09,0xbf]
 
 s_cmp_ge_u32 vcc_lo, s1
-// GFX12: encoding: [0x6a,0x01,0x09,0xbf]
+// GFX12: s_cmp_ge_u32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x09,0xbf]
 
 s_cmp_ge_u32 vcc_hi, s1
-// GFX12: encoding: [0x6b,0x01,0x09,0xbf]
+// GFX12: s_cmp_ge_u32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x09,0xbf]
 
 s_cmp_ge_u32 m0, s1
-// GFX12: encoding: [0x7d,0x01,0x09,0xbf]
+// GFX12: s_cmp_ge_u32 m0, s1                     ; encoding: [0x7d,0x01,0x09,0xbf]
 
 s_cmp_ge_u32 s0, exec_lo
-// GFX12: encoding: [0x00,0x7e,0x09,0xbf]
+// GFX12: s_cmp_ge_u32 s0, exec_lo                ; encoding: [0x00,0x7e,0x09,0xbf]
 
 s_cmp_ge_u32 s0, exec_hi
-// GFX12: encoding: [0x00,0x7f,0x09,0xbf]
+// GFX12: s_cmp_ge_u32 s0, exec_hi                ; encoding: [0x00,0x7f,0x09,0xbf]
 
 s_cmp_ge_u32 s0, vcc_lo
-// GFX12: encoding: [0x00,0x6a,0x09,0xbf]
+// GFX12: s_cmp_ge_u32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x09,0xbf]
 
 s_cmp_ge_u32 s0, vcc_hi
-// GFX12: encoding: [0x00,0x6b,0x09,0xbf]
+// GFX12: s_cmp_ge_u32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x09,0xbf]
 
 s_cmp_ge_u32 s0, m0
-// GFX12: encoding: [0x00,0x7d,0x09,0xbf]
+// GFX12: s_cmp_ge_u32 s0, m0                     ; encoding: [0x00,0x7d,0x09,0xbf]
 
 s_cmp_ge_u32 s0, 0
-// GFX12: encoding: [0x00,0x80,0x09,0xbf]
+// GFX12: s_cmp_ge_u32 s0, 0                      ; encoding: [0x00,0x80,0x09,0xbf]
 
 s_cmp_ge_u32 s0, -1
-// GFX12: encoding: [0x00,0xc1,0x09,0xbf]
+// GFX12: s_cmp_ge_u32 s0, -1                     ; encoding: [0x00,0xc1,0x09,0xbf]
 
 s_cmp_ge_u32 s0, 0.5
-// GFX12: encoding: [0x00,0xf0,0x09,0xbf]
+// GFX12: s_cmp_ge_u32 s0, 0.5                    ; encoding: [0x00,0xf0,0x09,0xbf]
 
 s_cmp_ge_u32 s0, -4.0
-// GFX12: encoding: [0x00,0xf7,0x09,0xbf]
+// GFX12: s_cmp_ge_u32 s0, -4.0                   ; encoding: [0x00,0xf7,0x09,0xbf]
 
 s_cmp_ge_u32 s0, 0x3f717273
-// GFX12: encoding: [0x00,0xff,0x09,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_ge_u32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x09,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_ge_u32 s0, 0xaf123456
-// GFX12: encoding: [0x00,0xff,0x09,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_ge_u32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x09,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_lt_u32 s0, s1
-// GFX12: encoding: [0x00,0x01,0x0a,0xbf]
+// GFX12: s_cmp_lt_u32 s0, s1                     ; encoding: [0x00,0x01,0x0a,0xbf]
 
 s_cmp_lt_u32 s105, s104
-// GFX12: encoding: [0x69,0x68,0x0a,0xbf]
+// GFX12: s_cmp_lt_u32 s105, s104                 ; encoding: [0x69,0x68,0x0a,0xbf]
 
 s_cmp_lt_u32 s0, s104
-// GFX12: encoding: [0x00,0x68,0x0a,0xbf]
+// GFX12: s_cmp_lt_u32 s0, s104                   ; encoding: [0x00,0x68,0x0a,0xbf]
 
 s_cmp_lt_u32 s105, s1
-// GFX12: encoding: [0x69,0x01,0x0a,0xbf]
+// GFX12: s_cmp_lt_u32 s105, s1                   ; encoding: [0x69,0x01,0x0a,0xbf]
 
 s_cmp_lt_u32 exec_lo, s1
-// GFX12: encoding: [0x7e,0x01,0x0a,0xbf]
+// GFX12: s_cmp_lt_u32 exec_lo, s1                ; encoding: [0x7e,0x01,0x0a,0xbf]
 
 s_cmp_lt_u32 exec_hi, s1
-// GFX12: encoding: [0x7f,0x01,0x0a,0xbf]
+// GFX12: s_cmp_lt_u32 exec_hi, s1                ; encoding: [0x7f,0x01,0x0a,0xbf]
 
 s_cmp_lt_u32 vcc_lo, s1
-// GFX12: encoding: [0x6a,0x01,0x0a,0xbf]
+// GFX12: s_cmp_lt_u32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x0a,0xbf]
 
 s_cmp_lt_u32 vcc_hi, s1
-// GFX12: encoding: [0x6b,0x01,0x0a,0xbf]
+// GFX12: s_cmp_lt_u32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x0a,0xbf]
 
 s_cmp_lt_u32 m0, s1
-// GFX12: encoding: [0x7d,0x01,0x0a,0xbf]
+// GFX12: s_cmp_lt_u32 m0, s1                     ; encoding: [0x7d,0x01,0x0a,0xbf]
 
 s_cmp_lt_u32 s0, exec_lo
-// GFX12: encoding: [0x00,0x7e,0x0a,0xbf]
+// GFX12: s_cmp_lt_u32 s0, exec_lo                ; encoding: [0x00,0x7e,0x0a,0xbf]
 
 s_cmp_lt_u32 s0, exec_hi
-// GFX12: encoding: [0x00,0x7f,0x0a,0xbf]
+// GFX12: s_cmp_lt_u32 s0, exec_hi                ; encoding: [0x00,0x7f,0x0a,0xbf]
 
 s_cmp_lt_u32 s0, vcc_lo
-// GFX12: encoding: [0x00,0x6a,0x0a,0xbf]
+// GFX12: s_cmp_lt_u32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x0a,0xbf]
 
 s_cmp_lt_u32 s0, vcc_hi
-// GFX12: encoding: [0x00,0x6b,0x0a,0xbf]
+// GFX12: s_cmp_lt_u32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x0a,0xbf]
 
 s_cmp_lt_u32 s0, m0
-// GFX12: encoding: [0x00,0x7d,0x0a,0xbf]
+// GFX12: s_cmp_lt_u32 s0, m0                     ; encoding: [0x00,0x7d,0x0a,0xbf]
 
 s_cmp_lt_u32 s0, 0
-// GFX12: encoding: [0x00,0x80,0x0a,0xbf]
+// GFX12: s_cmp_lt_u32 s0, 0                      ; encoding: [0x00,0x80,0x0a,0xbf]
 
 s_cmp_lt_u32 s0, -1
-// GFX12: encoding: [0x00,0xc1,0x0a,0xbf]
+// GFX12: s_cmp_lt_u32 s0, -1                     ; encoding: [0x00,0xc1,0x0a,0xbf]
 
 s_cmp_lt_u32 s0, 0.5
-// GFX12: encoding: [0x00,0xf0,0x0a,0xbf]
+// GFX12: s_cmp_lt_u32 s0, 0.5                    ; encoding: [0x00,0xf0,0x0a,0xbf]
 
 s_cmp_lt_u32 s0, -4.0
-// GFX12: encoding: [0x00,0xf7,0x0a,0xbf]
+// GFX12: s_cmp_lt_u32 s0, -4.0                   ; encoding: [0x00,0xf7,0x0a,0xbf]
 
 s_cmp_lt_u32 s0, 0x3f717273
-// GFX12: encoding: [0x00,0xff,0x0a,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_lt_u32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x0a,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_lt_u32 s0, 0xaf123456
-// GFX12: encoding: [0x00,0xff,0x0a,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_lt_u32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x0a,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_le_u32 s0, s1
-// GFX12: encoding: [0x00,0x01,0x0b,0xbf]
+// GFX12: s_cmp_le_u32 s0, s1                     ; encoding: [0x00,0x01,0x0b,0xbf]
 
 s_cmp_le_u32 s105, s104
-// GFX12: encoding: [0x69,0x68,0x0b,0xbf]
+// GFX12: s_cmp_le_u32 s105, s104                 ; encoding: [0x69,0x68,0x0b,0xbf]
 
 s_cmp_le_u32 s0, s104
-// GFX12: encoding: [0x00,0x68,0x0b,0xbf]
+// GFX12: s_cmp_le_u32 s0, s104                   ; encoding: [0x00,0x68,0x0b,0xbf]
 
 s_cmp_le_u32 s105, s1
-// GFX12: encoding: [0x69,0x01,0x0b,0xbf]
+// GFX12: s_cmp_le_u32 s105, s1                   ; encoding: [0x69,0x01,0x0b,0xbf]
 
 s_cmp_le_u32 exec_lo, s1
-// GFX12: encoding: [0x7e,0x01,0x0b,0xbf]
+// GFX12: s_cmp_le_u32 exec_lo, s1                ; encoding: [0x7e,0x01,0x0b,0xbf]
 
 s_cmp_le_u32 exec_hi, s1
-// GFX12: encoding: [0x7f,0x01,0x0b,0xbf]
+// GFX12: s_cmp_le_u32 exec_hi, s1                ; encoding: [0x7f,0x01,0x0b,0xbf]
 
 s_cmp_le_u32 vcc_lo, s1
-// GFX12: encoding: [0x6a,0x01,0x0b,0xbf]
+// GFX12: s_cmp_le_u32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x0b,0xbf]
 
 s_cmp_le_u32 vcc_hi, s1
-// GFX12: encoding: [0x6b,0x01,0x0b,0xbf]
+// GFX12: s_cmp_le_u32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x0b,0xbf]
 
 s_cmp_le_u32 m0, s1
-// GFX12: encoding: [0x7d,0x01,0x0b,0xbf]
+// GFX12: s_cmp_le_u32 m0, s1                     ; encoding: [0x7d,0x01,0x0b,0xbf]
 
 s_cmp_le_u32 s0, exec_lo
-// GFX12: encoding: [0x00,0x7e,0x0b,0xbf]
+// GFX12: s_cmp_le_u32 s0, exec_lo                ; encoding: [0x00,0x7e,0x0b,0xbf]
 
 s_cmp_le_u32 s0, exec_hi
-// GFX12: encoding: [0x00,0x7f,0x0b,0xbf]
+// GFX12: s_cmp_le_u32 s0, exec_hi                ; encoding: [0x00,0x7f,0x0b,0xbf]
 
 s_cmp_le_u32 s0, vcc_lo
-// GFX12: encoding: [0x00,0x6a,0x0b,0xbf]
+// GFX12: s_cmp_le_u32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x0b,0xbf]
 
 s_cmp_le_u32 s0, vcc_hi
-// GFX12: encoding: [0x00,0x6b,0x0b,0xbf]
+// GFX12: s_cmp_le_u32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x0b,0xbf]
 
 s_cmp_le_u32 s0, m0
-// GFX12: encoding: [0x00,0x7d,0x0b,0xbf]
+// GFX12: s_cmp_le_u32 s0, m0                     ; encoding: [0x00,0x7d,0x0b,0xbf]
 
 s_cmp_le_u32 s0, 0
-// GFX12: encoding: [0x00,0x80,0x0b,0xbf]
+// GFX12: s_cmp_le_u32 s0, 0                      ; encoding: [0x00,0x80,0x0b,0xbf]
 
 s_cmp_le_u32 s0, -1
-// GFX12: encoding: [0x00,0xc1,0x0b,0xbf]
+// GFX12: s_cmp_le_u32 s0, -1                     ; encoding: [0x00,0xc1,0x0b,0xbf]
 
 s_cmp_le_u32 s0, 0.5
-// GFX12: encoding: [0x00,0xf0,0x0b,0xbf]
+// GFX12: s_cmp_le_u32 s0, 0.5                    ; encoding: [0x00,0xf0,0x0b,0xbf]
 
 s_cmp_le_u32 s0, -4.0
-// GFX12: encoding: [0x00,0xf7,0x0b,0xbf]
+// GFX12: s_cmp_le_u32 s0, -4.0                   ; encoding: [0x00,0xf7,0x0b,0xbf]
 
 s_cmp_le_u32 s0, 0x3f717273
-// GFX12: encoding: [0x00,0xff,0x0b,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_le_u32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x0b,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_le_u32 s0, 0xaf123456
-// GFX12: encoding: [0x00,0xff,0x0b,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_le_u32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x0b,0xbf,0x56,0x34,0x12,0xaf]
 
 s_bitcmp0_b32 s0, s1
-// GFX12: encoding: [0x00,0x01,0x0c,0xbf]
+// GFX12: s_bitcmp0_b32 s0, s1                    ; encoding: [0x00,0x01,0x0c,0xbf]
 
 s_bitcmp0_b32 s105, s104
-// GFX12: encoding: [0x69,0x68,0x0c,0xbf]
+// GFX12: s_bitcmp0_b32 s105, s104                ; encoding: [0x69,0x68,0x0c,0xbf]
 
 s_bitcmp0_b32 s0, s104
-// GFX12: encoding: [0x00,0x68,0x0c,0xbf]
+// GFX12: s_bitcmp0_b32 s0, s104                  ; encoding: [0x00,0x68,0x0c,0xbf]
 
 s_bitcmp0_b32 s105, s1
-// GFX12: encoding: [0x69,0x01,0x0c,0xbf]
+// GFX12: s_bitcmp0_b32 s105, s1                  ; encoding: [0x69,0x01,0x0c,0xbf]
 
 s_bitcmp0_b32 exec_lo, s1
-// GFX12: encoding: [0x7e,0x01,0x0c,0xbf]
+// GFX12: s_bitcmp0_b32 exec_lo, s1               ; encoding: [0x7e,0x01,0x0c,0xbf]
 
 s_bitcmp0_b32 exec_hi, s1
-// GFX12: encoding: [0x7f,0x01,0x0c,0xbf]
+// GFX12: s_bitcmp0_b32 exec_hi, s1               ; encoding: [0x7f,0x01,0x0c,0xbf]
 
 s_bitcmp0_b32 vcc_lo, s1
-// GFX12: encoding: [0x6a,0x01,0x0c,0xbf]
+// GFX12: s_bitcmp0_b32 vcc_lo, s1                ; encoding: [0x6a,0x01,0x0c,0xbf]
 
 s_bitcmp0_b32 vcc_hi, s1
-// GFX12: encoding: [0x6b,0x01,0x0c,0xbf]
+// GFX12: s_bitcmp0_b32 vcc_hi, s1                ; encoding: [0x6b,0x01,0x0c,0xbf]
 
 s_bitcmp0_b32 m0, s1
-// GFX12: encoding: [0x7d,0x01,0x0c,0xbf]
+// GFX12: s_bitcmp0_b32 m0, s1                    ; encoding: [0x7d,0x01,0x0c,0xbf]
 
 s_bitcmp0_b32 s0, exec_lo
-// GFX12: encoding: [0x00,0x7e,0x0c,0xbf]
+// GFX12: s_bitcmp0_b32 s0, exec_lo               ; encoding: [0x00,0x7e,0x0c,0xbf]
 
 s_bitcmp0_b32 s0, exec_hi
-// GFX12: encoding: [0x00,0x7f,0x0c,0xbf]
+// GFX12: s_bitcmp0_b32 s0, exec_hi               ; encoding: [0x00,0x7f,0x0c,0xbf]
 
 s_bitcmp0_b32 s0, vcc_lo
-// GFX12: encoding: [0x00,0x6a,0x0c,0xbf]
+// GFX12: s_bitcmp0_b32 s0, vcc_lo                ; encoding: [0x00,0x6a,0x0c,0xbf]
 
 s_bitcmp0_b32 s0, vcc_hi
-// GFX12: encoding: [0x00,0x6b,0x0c,0xbf]
+// GFX12: s_bitcmp0_b32 s0, vcc_hi                ; encoding: [0x00,0x6b,0x0c,0xbf]
 
 s_bitcmp0_b32 s0, m0
-// GFX12: encoding: [0x00,0x7d,0x0c,0xbf]
+// GFX12: s_bitcmp0_b32 s0, m0                    ; encoding: [0x00,0x7d,0x0c,0xbf]
 
 s_bitcmp0_b32 s0, 0
-// GFX12: encoding: [0x00,0x80,0x0c,0xbf]
+// GFX12: s_bitcmp0_b32 s0, 0                     ; encoding: [0x00,0x80,0x0c,0xbf]
 
 s_bitcmp0_b32 s0, -1
-// GFX12: encoding: [0x00,0xc1,0x0c,0xbf]
+// GFX12: s_bitcmp0_b32 s0, -1                    ; encoding: [0x00,0xc1,0x0c,0xbf]
 
 s_bitcmp0_b32 s0, 0.5
-// GFX12: encoding: [0x00,0xf0,0x0c,0xbf]
+// GFX12: s_bitcmp0_b32 s0, 0.5                   ; encoding: [0x00,0xf0,0x0c,0xbf]
 
 s_bitcmp0_b32 s0, -4.0
-// GFX12: encoding: [0x00,0xf7,0x0c,0xbf]
+// GFX12: s_bitcmp0_b32 s0, -4.0                  ; encoding: [0x00,0xf7,0x0c,0xbf]
 
 s_bitcmp0_b32 s0, 0x3f717273
-// GFX12: encoding: [0x00,0xff,0x0c,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_bitcmp0_b32 s0, 0x3f717273            ; encoding: [0x00,0xff,0x0c,0xbf,0x73,0x72,0x71,0x3f]
 
 s_bitcmp0_b32 s0, 0xaf123456
-// GFX12: encoding: [0x00,0xff,0x0c,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_bitcmp0_b32 s0, 0xaf123456            ; encoding: [0x00,0xff,0x0c,0xbf,0x56,0x34,0x12,0xaf]
 
 s_bitcmp1_b32 s0, s1
-// GFX12: encoding: [0x00,0x01,0x0d,0xbf]
+// GFX12: s_bitcmp1_b32 s0, s1                    ; encoding: [0x00,0x01,0x0d,0xbf]
 
 s_bitcmp1_b32 s105, s104
-// GFX12: encoding: [0x69,0x68,0x0d,0xbf]
+// GFX12: s_bitcmp1_b32 s105, s104                ; encoding: [0x69,0x68,0x0d,0xbf]
 
 s_bitcmp1_b32 s0, s104
-// GFX12: encoding: [0x00,0x68,0x0d,0xbf]
+// GFX12: s_bitcmp1_b32 s0, s104                  ; encoding: [0x00,0x68,0x0d,0xbf]
 
 s_bitcmp1_b32 s105, s1
-// GFX12: encoding: [0x69,0x01,0x0d,0xbf]
+// GFX12: s_bitcmp1_b32 s105, s1                  ; encoding: [0x69,0x01,0x0d,0xbf]
 
 s_bitcmp1_b32 exec_lo, s1
-// GFX12: encoding: [0x7e,0x01,0x0d,0xbf]
+// GFX12: s_bitcmp1_b32 exec_lo, s1               ; encoding: [0x7e,0x01,0x0d,0xbf]
 
 s_bitcmp1_b32 exec_hi, s1
-// GFX12: encoding: [0x7f,0x01,0x0d,0xbf]
+// GFX12: s_bitcmp1_b32 exec_hi, s1               ; encoding: [0x7f,0x01,0x0d,0xbf]
 
 s_bitcmp1_b32 vcc_lo, s1
-// GFX12: encoding: [0x6a,0x01,0x0d,0xbf]
+// GFX12: s_bitcmp1_b32 vcc_lo, s1                ; encoding: [0x6a,0x01,0x0d,0xbf]
 
 s_bitcmp1_b32 vcc_hi, s1
-// GFX12: encoding: [0x6b,0x01,0x0d,0xbf]
+// GFX12: s_bitcmp1_b32 vcc_hi, s1                ; encoding: [0x6b,0x01,0x0d,0xbf]
 
 s_bitcmp1_b32 m0, s1
-// GFX12: encoding: [0x7d,0x01,0x0d,0xbf]
+// GFX12: s_bitcmp1_b32 m0, s1                    ; encoding: [0x7d,0x01,0x0d,0xbf]
 
 s_bitcmp1_b32 s0, exec_lo
-// GFX12: encoding: [0x00,0x7e,0x0d,0xbf]
+// GFX12: s_bitcmp1_b32 s0, exec_lo               ; encoding: [0x00,0x7e,0x0d,0xbf]
 
 s_bitcmp1_b32 s0, exec_hi
-// GFX12: encoding: [0x00,0x7f,0x0d,0xbf]
+// GFX12: s_bitcmp1_b32 s0, exec_hi               ; encoding: [0x00,0x7f,0x0d,0xbf]
 
 s_bitcmp1_b32 s0, vcc_lo
-// GFX12: encoding: [0x00,0x6a,0x0d,0xbf]
+// GFX12: s_bitcmp1_b32 s0, vcc_lo                ; encoding: [0x00,0x6a,0x0d,0xbf]
 
 s_bitcmp1_b32 s0, vcc_hi
-// GFX12: encoding: [0x00,0x6b,0x0d,0xbf]
+// GFX12: s_bitcmp1_b32 s0, vcc_hi                ; encoding: [0x00,0x6b,0x0d,0xbf]
 
 s_bitcmp1_b32 s0, m0
-// GFX12: encoding: [0x00,0x7d,0x0d,0xbf]
+// GFX12: s_bitcmp1_b32 s0, m0                    ; encoding: [0x00,0x7d,0x0d,0xbf]
 
 s_bitcmp1_b32 s0, 0
-// GFX12: encoding: [0x00,0x80,0x0d,0xbf]
+// GFX12: s_bitcmp1_b32 s0, 0                     ; encoding: [0x00,0x80,0x0d,0xbf]
 
 s_bitcmp1_b32 s0, -1
-// GFX12: encoding: [0x00,0xc1,0x0d,0xbf]
+// GFX12: s_bitcmp1_b32 s0, -1                    ; encoding: [0x00,0xc1,0x0d,0xbf]
 
 s_bitcmp1_b32 s0, 0.5
-// GFX12: encoding: [0x00,0xf0,0x0d,0xbf]
+// GFX12: s_bitcmp1_b32 s0, 0.5                   ; encoding: [0x00,0xf0,0x0d,0xbf]
 
 s_bitcmp1_b32 s0, -4.0
-// GFX12: encoding: [0x00,0xf7,0x0d,0xbf]
+// GFX12: s_bitcmp1_b32 s0, -4.0                  ; encoding: [0x00,0xf7,0x0d,0xbf]
 
 s_bitcmp1_b32 s0, 0x3f717273
-// GFX12: encoding: [0x00,0xff,0x0d,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_bitcmp1_b32 s0, 0x3f717273            ; encoding: [0x00,0xff,0x0d,0xbf,0x73,0x72,0x71,0x3f]
 
 s_bitcmp1_b32 s0, 0xaf123456
-// GFX12: encoding: [0x00,0xff,0x0d,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_bitcmp1_b32 s0, 0xaf123456            ; encoding: [0x00,0xff,0x0d,0xbf,0x56,0x34,0x12,0xaf]
 
 s_bitcmp0_b64 s[0:1], s2
-// GFX12: encoding: [0x00,0x02,0x0e,0xbf]
+// GFX12: s_bitcmp0_b64 s[0:1], s2                ; encoding: [0x00,0x02,0x0e,0xbf]
 
 s_bitcmp0_b64 s[104:105], s102
-// GFX12: encoding: [0x68,0x66,0x0e,0xbf]
+// GFX12: s_bitcmp0_b64 s[104:105], s102          ; encoding: [0x68,0x66,0x0e,0xbf]
 
 s_bitcmp0_b64 s[0:1], s102
-// GFX12: encoding: [0x00,0x66,0x0e,0xbf]
+// GFX12: s_bitcmp0_b64 s[0:1], s102              ; encoding: [0x00,0x66,0x0e,0xbf]
 
 s_bitcmp0_b64 s[104:105], s2
-// GFX12: encoding: [0x68,0x02,0x0e,0xbf]
+// GFX12: s_bitcmp0_b64 s[104:105], s2            ; encoding: [0x68,0x02,0x0e,0xbf]
 
 s_bitcmp0_b64 exec, s2
-// GFX12: encoding: [0x7e,0x02,0x0e,0xbf]
+// GFX12: s_bitcmp0_b64 exec, s2                  ; encoding: [0x7e,0x02,0x0e,0xbf]
 
 s_bitcmp0_b64 vcc, s2
-// GFX12: encoding: [0x6a,0x02,0x0e,0xbf]
+// GFX12: s_bitcmp0_b64 vcc, s2                   ; encoding: [0x6a,0x02,0x0e,0xbf]
 
 s_bitcmp0_b64 s[0:1], exec_lo
-// GFX12: encoding: [0x00,0x7e,0x0e,0xbf]
+// GFX12: s_bitcmp0_b64 s[0:1], exec_lo           ; encoding: [0x00,0x7e,0x0e,0xbf]
 
 s_bitcmp0_b64 s[0:1], exec_hi
-// GFX12: encoding: [0x00,0x7f,0x0e,0xbf]
+// GFX12: s_bitcmp0_b64 s[0:1], exec_hi           ; encoding: [0x00,0x7f,0x0e,0xbf]
 
 s_bitcmp0_b64 s[0:1], vcc_lo
-// GFX12: encoding: [0x00,0x6a,0x0e,0xbf]
+// GFX12: s_bitcmp0_b64 s[0:1], vcc_lo            ; encoding: [0x00,0x6a,0x0e,0xbf]
 
 s_bitcmp0_b64 s[0:1], vcc_hi
-// GFX12: encoding: [0x00,0x6b,0x0e,0xbf]
+// GFX12: s_bitcmp0_b64 s[0:1], vcc_hi            ; encoding: [0x00,0x6b,0x0e,0xbf]
 
 s_bitcmp0_b64 s[0:1], m0
-// GFX12: encoding: [0x00,0x7d,0x0e,0xbf]
+// GFX12: s_bitcmp0_b64 s[0:1], m0                ; encoding: [0x00,0x7d,0x0e,0xbf]
 
 s_bitcmp0_b64 s[0:1], 0
-// GFX12: encoding: [0x00,0x80,0x0e,0xbf]
+// GFX12: s_bitcmp0_b64 s[0:1], 0                 ; encoding: [0x00,0x80,0x0e,0xbf]
 
 s_bitcmp0_b64 s[0:1], -1
-// GFX12: encoding: [0x00,0xc1,0x0e,0xbf]
+// GFX12: s_bitcmp0_b64 s[0:1], -1                ; encoding: [0x00,0xc1,0x0e,0xbf]
 
 s_bitcmp0_b64 s[0:1], 0.5
-// GFX12: encoding: [0x00,0xf0,0x0e,0xbf]
+// GFX12: s_bitcmp0_b64 s[0:1], 0.5               ; encoding: [0x00,0xf0,0x0e,0xbf]
 
 s_bitcmp0_b64 s[0:1], -4.0
-// GFX12: encoding: [0x00,0xf7,0x0e,0xbf]
+// GFX12: s_bitcmp0_b64 s[0:1], -4.0              ; encoding: [0x00,0xf7,0x0e,0xbf]
 
 s_bitcmp0_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0x00,0xff,0x0e,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_bitcmp0_b64 s[0:1], 0x3f717273        ; encoding: [0x00,0xff,0x0e,0xbf,0x73,0x72,0x71,0x3f]
 
 s_bitcmp0_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0x00,0xff,0x0e,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_bitcmp0_b64 s[0:1], 0xaf123456        ; encoding: [0x00,0xff,0x0e,0xbf,0x56,0x34,0x12,0xaf]
 
 s_bitcmp1_b64 s[0:1], s2
-// GFX12: encoding: [0x00,0x02,0x0f,0xbf]
+// GFX12: s_bitcmp1_b64 s[0:1], s2                ; encoding: [0x00,0x02,0x0f,0xbf]
 
 s_bitcmp1_b64 s[104:105], s102
-// GFX12: encoding: [0x68,0x66,0x0f,0xbf]
+// GFX12: s_bitcmp1_b64 s[104:105], s102          ; encoding: [0x68,0x66,0x0f,0xbf]
 
 s_bitcmp1_b64 s[0:1], s102
-// GFX12: encoding: [0x00,0x66,0x0f,0xbf]
+// GFX12: s_bitcmp1_b64 s[0:1], s102              ; encoding: [0x00,0x66,0x0f,0xbf]
 
 s_bitcmp1_b64 s[104:105], s2
-// GFX12: encoding: [0x68,0x02,0x0f,0xbf]
+// GFX12: s_bitcmp1_b64 s[104:105], s2            ; encoding: [0x68,0x02,0x0f,0xbf]
 
 s_bitcmp1_b64 exec, s2
-// GFX12: encoding: [0x7e,0x02,0x0f,0xbf]
+// GFX12: s_bitcmp1_b64 exec, s2                  ; encoding: [0x7e,0x02,0x0f,0xbf]
 
 s_bitcmp1_b64 vcc, s2
-// GFX12: encoding: [0x6a,0x02,0x0f,0xbf]
+// GFX12: s_bitcmp1_b64 vcc, s2                   ; encoding: [0x6a,0x02,0x0f,0xbf]
 
 s_bitcmp1_b64 s[0:1], exec_lo
-// GFX12: encoding: [0x00,0x7e,0x0f,0xbf]
+// GFX12: s_bitcmp1_b64 s[0:1], exec_lo           ; encoding: [0x00,0x7e,0x0f,0xbf]
 
 s_bitcmp1_b64 s[0:1], exec_hi
-// GFX12: encoding: [0x00,0x7f,0x0f,0xbf]
+// GFX12: s_bitcmp1_b64 s[0:1], exec_hi           ; encoding: [0x00,0x7f,0x0f,0xbf]
 
 s_bitcmp1_b64 s[0:1], vcc_lo
-// GFX12: encoding: [0x00,0x6a,0x0f,0xbf]
+// GFX12: s_bitcmp1_b64 s[0:1], vcc_lo            ; encoding: [0x00,0x6a,0x0f,0xbf]
 
 s_bitcmp1_b64 s[0:1], vcc_hi
-// GFX12: encoding: [0x00,0x6b,0x0f,0xbf]
+// GFX12: s_bitcmp1_b64 s[0:1], vcc_hi            ; encoding: [0x00,0x6b,0x0f,0xbf]
 
 s_bitcmp1_b64 s[0:1], m0
-// GFX12: encoding: [0x00,0x7d,0x0f,0xbf]
+// GFX12: s_bitcmp1_b64 s[0:1], m0                ; encoding: [0x00,0x7d,0x0f,0xbf]
 
 s_bitcmp1_b64 s[0:1], 0
-// GFX12: encoding: [0x00,0x80,0x0f,0xbf]
+// GFX12: s_bitcmp1_b64 s[0:1], 0                 ; encoding: [0x00,0x80,0x0f,0xbf]
 
 s_bitcmp1_b64 s[0:1], -1
-// GFX12: encoding: [0x00,0xc1,0x0f,0xbf]
+// GFX12: s_bitcmp1_b64 s[0:1], -1                ; encoding: [0x00,0xc1,0x0f,0xbf]
 
 s_bitcmp1_b64 s[0:1], 0.5
-// GFX12: encoding: [0x00,0xf0,0x0f,0xbf]
+// GFX12: s_bitcmp1_b64 s[0:1], 0.5               ; encoding: [0x00,0xf0,0x0f,0xbf]
 
 s_bitcmp1_b64 s[0:1], -4.0
-// GFX12: encoding: [0x00,0xf7,0x0f,0xbf]
+// GFX12: s_bitcmp1_b64 s[0:1], -4.0              ; encoding: [0x00,0xf7,0x0f,0xbf]
 
 s_bitcmp1_b64 s[0:1], 0x3f717273
-// GFX12: encoding: [0x00,0xff,0x0f,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_bitcmp1_b64 s[0:1], 0x3f717273        ; encoding: [0x00,0xff,0x0f,0xbf,0x73,0x72,0x71,0x3f]
 
 s_bitcmp1_b64 s[0:1], 0xaf123456
-// GFX12: encoding: [0x00,0xff,0x0f,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_bitcmp1_b64 s[0:1], 0xaf123456        ; encoding: [0x00,0xff,0x0f,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_eq_u64 s[0:1], s[2:3]
-// GFX12: encoding: [0x00,0x02,0x10,0xbf]
+// GFX12: s_cmp_eq_u64 s[0:1], s[2:3]             ; encoding: [0x00,0x02,0x10,0xbf]
 
 s_cmp_eq_u64 s[104:105], s[102:103]
-// GFX12: encoding: [0x68,0x66,0x10,0xbf]
+// GFX12: s_cmp_eq_u64 s[104:105], s[102:103]     ; encoding: [0x68,0x66,0x10,0xbf]
 
 s_cmp_eq_u64 s[0:1], s[102:103]
-// GFX12: encoding: [0x00,0x66,0x10,0xbf]
+// GFX12: s_cmp_eq_u64 s[0:1], s[102:103]         ; encoding: [0x00,0x66,0x10,0xbf]
 
 s_cmp_eq_u64 s[104:105], s[2:3]
-// GFX12: encoding: [0x68,0x02,0x10,0xbf]
+// GFX12: s_cmp_eq_u64 s[104:105], s[2:3]         ; encoding: [0x68,0x02,0x10,0xbf]
 
 s_cmp_eq_u64 exec, s[2:3]
-// GFX12: encoding: [0x7e,0x02,0x10,0xbf]
+// GFX12: s_cmp_eq_u64 exec, s[2:3]               ; encoding: [0x7e,0x02,0x10,0xbf]
 
 s_cmp_eq_u64 vcc, s[2:3]
-// GFX12: encoding: [0x6a,0x02,0x10,0xbf]
+// GFX12: s_cmp_eq_u64 vcc, s[2:3]                ; encoding: [0x6a,0x02,0x10,0xbf]
 
 s_cmp_eq_u64 s[0:1], exec
-// GFX12: encoding: [0x00,0x7e,0x10,0xbf]
+// GFX12: s_cmp_eq_u64 s[0:1], exec               ; encoding: [0x00,0x7e,0x10,0xbf]
 
 s_cmp_eq_u64 s[0:1], vcc
-// GFX12: encoding: [0x00,0x6a,0x10,0xbf]
+// GFX12: s_cmp_eq_u64 s[0:1], vcc                ; encoding: [0x00,0x6a,0x10,0xbf]
 
 s_cmp_eq_u64 s[0:1], 0
-// GFX12: encoding: [0x00,0x80,0x10,0xbf]
+// GFX12: s_cmp_eq_u64 s[0:1], 0                  ; encoding: [0x00,0x80,0x10,0xbf]
 
 s_cmp_eq_u64 s[0:1], -1
-// GFX12: encoding: [0x00,0xc1,0x10,0xbf]
+// GFX12: s_cmp_eq_u64 s[0:1], -1                 ; encoding: [0x00,0xc1,0x10,0xbf]
 
 s_cmp_eq_u64 s[0:1], 0.5
-// GFX12: encoding: [0x00,0xf0,0x10,0xbf]
+// GFX12: s_cmp_eq_u64 s[0:1], 0.5                ; encoding: [0x00,0xf0,0x10,0xbf]
 
 s_cmp_eq_u64 s[0:1], -4.0
-// GFX12: encoding: [0x00,0xf7,0x10,0xbf]
+// GFX12: s_cmp_eq_u64 s[0:1], -4.0               ; encoding: [0x00,0xf7,0x10,0xbf]
 
 s_cmp_eq_u64 s[0:1], 0x3f717273
-// GFX12: encoding: [0x00,0xff,0x10,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_eq_u64 s[0:1], 0x3f717273         ; encoding: [0x00,0xff,0x10,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_eq_u64 s[0:1], 0xaf123456
-// GFX12: encoding: [0x00,0xff,0x10,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_eq_u64 s[0:1], 0xaf123456         ; encoding: [0x00,0xff,0x10,0xbf,0x56,0x34,0x12,0xaf]
 
 s_cmp_lg_u64 s[0:1], s[2:3]
-// GFX12: encoding: [0x00,0x02,0x11,0xbf]
+// GFX12: s_cmp_lg_u64 s[0:1], s[2:3]             ; encoding: [0x00,0x02,0x11,0xbf]
 
 s_cmp_lg_u64 s[104:105], s[102:103]
-// GFX12: encoding: [0x68,0x66,0x11,0xbf]
+// GFX12: s_cmp_lg_u64 s[104:105], s[102:103]     ; encoding: [0x68,0x66,0x11,0xbf]
 
 s_cmp_lg_u64 s[0:1], s[102:103]
-// GFX12: encoding: [0x00,0x66,0x11,0xbf]
+// GFX12: s_cmp_lg_u64 s[0:1], s[102:103]         ; encoding: [0x00,0x66,0x11,0xbf]
 
 s_cmp_lg_u64 s[104:105], s[2:3]
-// GFX12: encoding: [0x68,0x02,0x11,0xbf]
+// GFX12: s_cmp_lg_u64 s[104:105], s[2:3]         ; encoding: [0x68,0x02,0x11,0xbf]
 
 s_cmp_lg_u64 exec, s[2:3]
-// GFX12: encoding: [0x7e,0x02,0x11,0xbf]
+// GFX12: s_cmp_lg_u64 exec, s[2:3]               ; encoding: [0x7e,0x02,0x11,0xbf]
 
 s_cmp_lg_u64 vcc, s[2:3]
-// GFX12: encoding: [0x6a,0x02,0x11,0xbf]
+// GFX12: s_cmp_lg_u64 vcc, s[2:3]                ; encoding: [0x6a,0x02,0x11,0xbf]
 
 s_cmp_lg_u64 s[0:1], exec
-// GFX12: encoding: [0x00,0x7e,0x11,0xbf]
+// GFX12: s_cmp_lg_u64 s[0:1], exec               ; encoding: [0x00,0x7e,0x11,0xbf]
 
 s_cmp_lg_u64 s[0:1], vcc
-// GFX12: encoding: [0x00,0x6a,0x11,0xbf]
+// GFX12: s_cmp_lg_u64 s[0:1], vcc                ; encoding: [0x00,0x6a,0x11,0xbf]
 
 s_cmp_lg_u64 s[0:1], 0
-// GFX12: encoding: [0x00,0x80,0x11,0xbf]
+// GFX12: s_cmp_lg_u64 s[0:1], 0                  ; encoding: [0x00,0x80,0x11,0xbf]
 
 s_cmp_lg_u64 s[0:1], -1
-// GFX12: encoding: [0x00,0xc1,0x11,0xbf]
+// GFX12: s_cmp_lg_u64 s[0:1], -1                 ; encoding: [0x00,0xc1,0x11,0xbf]
 
 s_cmp_lg_u64 s[0:1], 0.5
-// GFX12: encoding: [0x00,0xf0,0x11,0xbf]
+// GFX12: s_cmp_lg_u64 s[0:1], 0.5                ; encoding: [0x00,0xf0,0x11,0xbf]
 
 s_cmp_lg_u64 s[0:1], -4.0
-// GFX12: encoding: [0x00,0xf7,0x11,0xbf]
+// GFX12: s_cmp_lg_u64 s[0:1], -4.0               ; encoding: [0x00,0xf7,0x11,0xbf]
 
 s_cmp_lg_u64 s[0:1], 0x3f717273
-// GFX12: encoding: [0x00,0xff,0x11,0xbf,0x73,0x72,0x71,0x3f]
+// GFX12: s_cmp_lg_u64 s[0:1], 0x3f717273         ; encoding: [0x00,0xff,0x11,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_lg_u64 s[0:1], 0xaf123456
-// GFX12: encoding: [0x00,0xff,0x11,0xbf,0x56,0x34,0x12,0xaf]
+// GFX12: s_cmp_lg_u64 s[0:1], 0xaf123456         ; encoding: [0x00,0xff,0x11,0xbf,0x56,0x34,0x12,0xaf]
diff --git a/llvm/test/MC/AMDGPU/gfx12_err.s b/llvm/test/MC/AMDGPU/gfx12_err.s
index b07e160454ef7..5269288f9d1e8 100644
--- a/llvm/test/MC/AMDGPU/gfx12_err.s
+++ b/llvm/test/MC/AMDGPU/gfx12_err.s
@@ -116,3 +116,12 @@ s_prefetch_inst s[14:15], 0xffffff, m0, 7
 
 s_endpgm_ordered_ps_done
 // GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+s_alloc_vgpr v0
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+s_alloc_vgpr exec
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+s_alloc_vgpr vcc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_sop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_sop1.txt
index 291f348a35cc6..99a210b4ed088 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_sop1.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_sop1.txt
@@ -2548,6 +2548,9 @@
 # GFX11: s_setpc_b64 vcc                         ; encoding: [0x6a,0x48,0x80,0xbe]
 0x6a,0x48,0x80,0xbe
 
+# GFX11: s_setpc_b64 -11/*Invalid immediate*/    ; encoding: [0xf5,0x48,0x80,0xbe]
+0xcb,0x48,0xf5,0xbe
+
 # GFX11: s_sext_i32_i16 exec_hi, s1              ; encoding: [0x01,0x0f,0xff,0xbe]
 0x01,0x0f,0xff,0xbe
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt
index fa7d020bdd726..31f61e48126c1 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt
@@ -1,3688 +1,3701 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s
 
-# GFX12: s_sleep_var 0x1234                   ; encoding: [0xff,0x58,0x80,0xbe,0x34,0x12,0x00,0x00]
+0xff,0x53,0x80,0xbe,0x35,0x12,0x00,0x00
+# GFX12: s_alloc_vgpr 0x1235                     ; encoding: [0xff,0x53,0x80,0xbe,0x35,0x12,0x00,0x00]
+
+0x92,0x53,0x80,0xbe
+# GFX12: s_alloc_vgpr 18                         ; encoding: [0x92,0x53,0x80,0xbe]
+
+0x23,0x53,0x80,0xbe
+# GFX12: s_alloc_vgpr s35                        ; encoding: [0x23,0x53,0x80,0xbe]
+
 0xff,0x58,0x80,0xbe,0x34,0x12,0x00,0x00
+# GFX12: s_sleep_var 0x1234                      ; encoding: [0xff,0x58,0x80,0xbe,0x34,0x12,0x00,0x00]
 
-# GFX12: s_sleep_var s1                       ; encoding: [0x01,0x58,0x80,0xbe]
 0x01,0x58,0x80,0xbe
+# GFX12: s_sleep_var s1                          ; encoding: [0x01,0x58,0x80,0xbe]
 
-# GFX12: s_cvt_f32_i32 s5, s1                    ; encoding: [0x01,0x64,0x85,0xbe]
 0x01,0x64,0x85,0xbe
+# GFX12: s_cvt_f32_i32 s5, s1                    ; encoding: [0x01,0x64,0x85,0xbe]
 
-# GFX12: s_cvt_f32_i32 s105, s1                  ; encoding: [0x01,0x64,0xe9,0xbe]
 0x01,0x64,0xe9,0xbe
+# GFX12: s_cvt_f32_i32 s105, s1                  ; encoding: [0x01,0x64,0xe9,0xbe]
 
-# GFX12: s_cvt_f32_i32 s5, s105                  ; encoding: [0x69,0x64,0x85,0xbe]
 0x69,0x64,0x85,0xbe
+# GFX12: s_cvt_f32_i32 s5, s105                  ; encoding: [0x69,0x64,0x85,0xbe]
 
-# GFX12: s_cvt_f32_i32 s5, s103                  ; encoding: [0x67,0x64,0x85,0xbe]
 0x67,0x64,0x85,0xbe
+# GFX12: s_cvt_f32_i32 s5, s103                  ; encoding: [0x67,0x64,0x85,0xbe]
 
-# GFX12: s_cvt_f32_i32 s5, vcc_lo                ; encoding: [0x6a,0x64,0x85,0xbe]
 0x6a,0x64,0x85,0xbe
+# GFX12: s_cvt_f32_i32 s5, vcc_lo                ; encoding: [0x6a,0x64,0x85,0xbe]
 
-# GFX12: s_cvt_f32_i32 s5, vcc_hi                ; encoding: [0x6b,0x64,0x85,0xbe]
 0x6b,0x64,0x85,0xbe
+# GFX12: s_cvt_f32_i32 s5, vcc_hi                ; encoding: [0x6b,0x64,0x85,0xbe]
 
-# GFX12: s_cvt_f32_i32 s5, ttmp11                ; encoding: [0x77,0x64,0x85,0xbe]
 0x77,0x64,0x85,0xbe
+# GFX12: s_cvt_f32_i32 s5, ttmp11                ; encoding: [0x77,0x64,0x85,0xbe]
 
-# GFX12: s_cvt_f32_i32 s5, m0                    ; encoding: [0x7d,0x64,0x85,0xbe]
 0x7d,0x64,0x85,0xbe
+# GFX12: s_cvt_f32_i32 s5, m0                    ; encoding: [0x7d,0x64,0x85,0xbe]
 
-# GFX12: s_cvt_f32_i32 s5, exec_lo               ; encoding: [0x7e,0x64,0x85,0xbe]
 0x7e,0x64,0x85,0xbe
+# GFX12: s_cvt_f32_i32 s5, exec_lo               ; encoding: [0x7e,0x64,0x85,0xbe]
 
-# GFX12: s_cvt_f32_i32 s5, exec_hi               ; encoding: [0x7f,0x64,0x85,0xbe]
 0x7f,0x64,0x85,0xbe
+# GFX12: s_cvt_f32_i32 s5, exec_hi               ; encoding: [0x7f,0x64,0x85,0xbe]
 
-# GFX12: s_cvt_f32_i32 s5, 0                     ; encoding: [0x80,0x64,0x85,0xbe]
 0x80,0x64,0x85,0xbe
+# GFX12: s_cvt_f32_i32 s5, 0                     ; encoding: [0x80,0x64,0x85,0xbe]
 
-# GFX12: s_cvt_f32_i32 s5, -1                    ; encoding: [0xc1,0x64,0x85,0xbe]
 0xc1,0x64,0x85,0xbe
+# GFX12: s_cvt_f32_i32 s5, -1                    ; encoding: [0xc1,0x64,0x85,0xbe]
 
-# GFX12: s_cvt_f32_i32 s5, 0.5                   ; encoding: [0xf0,0x64,0x85,0xbe]
 0xf0,0x64,0x85,0xbe
+# GFX12: s_cvt_f32_i32 s5, 0.5                   ; encoding: [0xf0,0x64,0x85,0xbe]
 
-# GFX12: s_cvt_f32_i32 s5, -4.0                  ; encoding: [0xf7,0x64,0x85,0xbe]
 0xf7,0x64,0x85,0xbe
+# GFX12: s_cvt_f32_i32 s5, -4.0                  ; encoding: [0xf7,0x64,0x85,0xbe]
 
-# GFX12: s_cvt_f32_i32 s5, 0xaf123456            ; encoding: [0xff,0x64,0x85,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x64,0x85,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_cvt_f32_i32 s5, 0xaf123456            ; encoding: [0xff,0x64,0x85,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cvt_f32_i32 s5, 0x3f717273            ; encoding: [0xff,0x64,0x85,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x64,0x85,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_cvt_f32_i32 s5, 0x3f717273            ; encoding: [0xff,0x64,0x85,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cvt_f32_u32 s5, s1                    ; encoding: [0x01,0x65,0x85,0xbe]
 0x01,0x65,0x85,0xbe
+# GFX12: s_cvt_f32_u32 s5, s1                    ; encoding: [0x01,0x65,0x85,0xbe]
 
-# GFX12: s_cvt_f32_u32 s105, s1                  ; encoding: [0x01,0x65,0xe9,0xbe]
 0x01,0x65,0xe9,0xbe
+# GFX12: s_cvt_f32_u32 s105, s1                  ; encoding: [0x01,0x65,0xe9,0xbe]
 
-# GFX12: s_cvt_f32_u32 s5, s105                  ; encoding: [0x69,0x65,0x85,0xbe]
 0x69,0x65,0x85,0xbe
+# GFX12: s_cvt_f32_u32 s5, s105                  ; encoding: [0x69,0x65,0x85,0xbe]
 
-# GFX12: s_cvt_f32_u32 s5, s103                  ; encoding: [0x67,0x65,0x85,0xbe]
 0x67,0x65,0x85,0xbe
+# GFX12: s_cvt_f32_u32 s5, s103                  ; encoding: [0x67,0x65,0x85,0xbe]
 
-# GFX12: s_cvt_f32_u32 s5, vcc_lo                ; encoding: [0x6a,0x65,0x85,0xbe]
 0x6a,0x65,0x85,0xbe
+# GFX12: s_cvt_f32_u32 s5, vcc_lo                ; encoding: [0x6a,0x65,0x85,0xbe]
 
-# GFX12: s_cvt_f32_u32 s5, vcc_hi                ; encoding: [0x6b,0x65,0x85,0xbe]
 0x6b,0x65,0x85,0xbe
+# GFX12: s_cvt_f32_u32 s5, vcc_hi                ; encoding: [0x6b,0x65,0x85,0xbe]
 
-# GFX12: s_cvt_f32_u32 s5, ttmp11                ; encoding: [0x77,0x65,0x85,0xbe]
 0x77,0x65,0x85,0xbe
+# GFX12: s_cvt_f32_u32 s5, ttmp11                ; encoding: [0x77,0x65,0x85,0xbe]
 
-# GFX12: s_cvt_f32_u32 s5, m0                    ; encoding: [0x7d,0x65,0x85,0xbe]
 0x7d,0x65,0x85,0xbe
+# GFX12: s_cvt_f32_u32 s5, m0                    ; encoding: [0x7d,0x65,0x85,0xbe]
 
-# GFX12: s_cvt_f32_u32 s5, exec_lo               ; encoding: [0x7e,0x65,0x85,0xbe]
 0x7e,0x65,0x85,0xbe
+# GFX12: s_cvt_f32_u32 s5, exec_lo               ; encoding: [0x7e,0x65,0x85,0xbe]
 
-# GFX12: s_cvt_f32_u32 s5, exec_hi               ; encoding: [0x7f,0x65,0x85,0xbe]
 0x7f,0x65,0x85,0xbe
+# GFX12: s_cvt_f32_u32 s5, exec_hi               ; encoding: [0x7f,0x65,0x85,0xbe]
 
-# GFX12: s_cvt_f32_u32 s5, 0                     ; encoding: [0x80,0x65,0x85,0xbe]
 0x80,0x65,0x85,0xbe
+# GFX12: s_cvt_f32_u32 s5, 0                     ; encoding: [0x80,0x65,0x85,0xbe]
 
-# GFX12: s_cvt_f32_u32 s5, -1                    ; encoding: [0xc1,0x65,0x85,0xbe]
 0xc1,0x65,0x85,0xbe
+# GFX12: s_cvt_f32_u32 s5, -1                    ; encoding: [0xc1,0x65,0x85,0xbe]
 
-# GFX12: s_cvt_f32_u32 s5, 0.5                   ; encoding: [0xf0,0x65,0x85,0xbe]
 0xf0,0x65,0x85,0xbe
+# GFX12: s_cvt_f32_u32 s5, 0.5                   ; encoding: [0xf0,0x65,0x85,0xbe]
 
-# GFX12: s_cvt_f32_u32 s5, -4.0                  ; encoding: [0xf7,0x65,0x85,0xbe]
 0xf7,0x65,0x85,0xbe
+# GFX12: s_cvt_f32_u32 s5, -4.0                  ; encoding: [0xf7,0x65,0x85,0xbe]
 
-# GFX12: s_cvt_f32_u32 s5, 0xaf123456            ; encoding: [0xff,0x65,0x85,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x65,0x85,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_cvt_f32_u32 s5, 0xaf123456            ; encoding: [0xff,0x65,0x85,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cvt_f32_u32 s5, 0x3f717273            ; encoding: [0xff,0x65,0x85,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x65,0x85,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_cvt_f32_u32 s5, 0x3f717273            ; encoding: [0xff,0x65,0x85,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cvt_u32_f32 s5, s1                    ; encoding: [0x01,0x67,0x85,0xbe]
 0x01,0x67,0x85,0xbe
+# GFX12: s_cvt_u32_f32 s5, s1                    ; encoding: [0x01,0x67,0x85,0xbe]
 
-# GFX12: s_cvt_u32_f32 s105, s1                  ; encoding: [0x01,0x67,0xe9,0xbe]
 0x01,0x67,0xe9,0xbe
+# GFX12: s_cvt_u32_f32 s105, s1                  ; encoding: [0x01,0x67,0xe9,0xbe]
 
-# GFX12: s_cvt_u32_f32 s5, s105                  ; encoding: [0x69,0x67,0x85,0xbe]
 0x69,0x67,0x85,0xbe
+# GFX12: s_cvt_u32_f32 s5, s105                  ; encoding: [0x69,0x67,0x85,0xbe]
 
-# GFX12: s_cvt_u32_f32 s5, s103                  ; encoding: [0x67,0x67,0x85,0xbe]
 0x67,0x67,0x85,0xbe
+# GFX12: s_cvt_u32_f32 s5, s103                  ; encoding: [0x67,0x67,0x85,0xbe]
 
-# GFX12: s_cvt_u32_f32 s5, vcc_lo                ; encoding: [0x6a,0x67,0x85,0xbe]
 0x6a,0x67,0x85,0xbe
+# GFX12: s_cvt_u32_f32 s5, vcc_lo                ; encoding: [0x6a,0x67,0x85,0xbe]
 
-# GFX12: s_cvt_u32_f32 s5, vcc_hi                ; encoding: [0x6b,0x67,0x85,0xbe]
 0x6b,0x67,0x85,0xbe
+# GFX12: s_cvt_u32_f32 s5, vcc_hi                ; encoding: [0x6b,0x67,0x85,0xbe]
 
-# GFX12: s_cvt_u32_f32 s5, ttmp11                ; encoding: [0x77,0x67,0x85,0xbe]
 0x77,0x67,0x85,0xbe
+# GFX12: s_cvt_u32_f32 s5, ttmp11                ; encoding: [0x77,0x67,0x85,0xbe]
 
-# GFX12: s_cvt_u32_f32 s5, m0                    ; encoding: [0x7d,0x67,0x85,0xbe]
 0x7d,0x67,0x85,0xbe
+# GFX12: s_cvt_u32_f32 s5, m0                    ; encoding: [0x7d,0x67,0x85,0xbe]
 
-# GFX12: s_cvt_u32_f32 s5, exec_lo               ; encoding: [0x7e,0x67,0x85,0xbe]
 0x7e,0x67,0x85,0xbe
+# GFX12: s_cvt_u32_f32 s5, exec_lo               ; encoding: [0x7e,0x67,0x85,0xbe]
 
-# GFX12: s_cvt_u32_f32 s5, exec_hi               ; encoding: [0x7f,0x67,0x85,0xbe]
 0x7f,0x67,0x85,0xbe
+# GFX12: s_cvt_u32_f32 s5, exec_hi               ; encoding: [0x7f,0x67,0x85,0xbe]
 
-# GFX12: s_cvt_u32_f32 s5, 0                     ; encoding: [0x80,0x67,0x85,0xbe]
 0x80,0x67,0x85,0xbe
+# GFX12: s_cvt_u32_f32 s5, 0                     ; encoding: [0x80,0x67,0x85,0xbe]
 
-# GFX12: s_cvt_u32_f32 s5, -1                    ; encoding: [0xc1,0x67,0x85,0xbe]
 0xc1,0x67,0x85,0xbe
+# GFX12: s_cvt_u32_f32 s5, -1                    ; encoding: [0xc1,0x67,0x85,0xbe]
 
-# GFX12: s_cvt_u32_f32 s5, 0.5                   ; encoding: [0xf0,0x67,0x85,0xbe]
 0xf0,0x67,0x85,0xbe
+# GFX12: s_cvt_u32_f32 s5, 0.5                   ; encoding: [0xf0,0x67,0x85,0xbe]
 
-# GFX12: s_cvt_u32_f32 s5, -4.0                  ; encoding: [0xf7,0x67,0x85,0xbe]
 0xf7,0x67,0x85,0xbe
+# GFX12: s_cvt_u32_f32 s5, -4.0                  ; encoding: [0xf7,0x67,0x85,0xbe]
 
-# GFX12: s_cvt_u32_f32 s5, 0xaf123456            ; encoding: [0xff,0x67,0x85,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x67,0x85,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_cvt_u32_f32 s5, 0xaf123456            ; encoding: [0xff,0x67,0x85,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cvt_u32_f32 s5, 0x3f717273            ; encoding: [0xff,0x67,0x85,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x67,0x85,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_cvt_u32_f32 s5, 0x3f717273            ; encoding: [0xff,0x67,0x85,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cvt_i32_f32 s5, s1                    ; encoding: [0x01,0x66,0x85,0xbe]
 0x01,0x66,0x85,0xbe
+# GFX12: s_cvt_i32_f32 s5, s1                    ; encoding: [0x01,0x66,0x85,0xbe]
 
-# GFX12: s_cvt_i32_f32 s105, s1                  ; encoding: [0x01,0x66,0xe9,0xbe]
 0x01,0x66,0xe9,0xbe
+# GFX12: s_cvt_i32_f32 s105, s1                  ; encoding: [0x01,0x66,0xe9,0xbe]
 
-# GFX12: s_cvt_i32_f32 s5, s105                  ; encoding: [0x69,0x66,0x85,0xbe]
 0x69,0x66,0x85,0xbe
+# GFX12: s_cvt_i32_f32 s5, s105                  ; encoding: [0x69,0x66,0x85,0xbe]
 
-# GFX12: s_cvt_i32_f32 s5, s103                  ; encoding: [0x67,0x66,0x85,0xbe]
 0x67,0x66,0x85,0xbe
+# GFX12: s_cvt_i32_f32 s5, s103                  ; encoding: [0x67,0x66,0x85,0xbe]
 
-# GFX12: s_cvt_i32_f32 s5, vcc_lo                ; encoding: [0x6a,0x66,0x85,0xbe]
 0x6a,0x66,0x85,0xbe
+# GFX12: s_cvt_i32_f32 s5, vcc_lo                ; encoding: [0x6a,0x66,0x85,0xbe]
 
-# GFX12: s_cvt_i32_f32 s5, vcc_hi                ; encoding: [0x6b,0x66,0x85,0xbe]
 0x6b,0x66,0x85,0xbe
+# GFX12: s_cvt_i32_f32 s5, vcc_hi                ; encoding: [0x6b,0x66,0x85,0xbe]
 
-# GFX12: s_cvt_i32_f32 s5, ttmp11                ; encoding: [0x77,0x66,0x85,0xbe]
 0x77,0x66,0x85,0xbe
+# GFX12: s_cvt_i32_f32 s5, ttmp11                ; encoding: [0x77,0x66,0x85,0xbe]
 
-# GFX12: s_cvt_i32_f32 s5, m0                    ; encoding: [0x7d,0x66,0x85,0xbe]
 0x7d,0x66,0x85,0xbe
+# GFX12: s_cvt_i32_f32 s5, m0                    ; encoding: [0x7d,0x66,0x85,0xbe]
 
-# GFX12: s_cvt_i32_f32 s5, exec_lo               ; encoding: [0x7e,0x66,0x85,0xbe]
 0x7e,0x66,0x85,0xbe
+# GFX12: s_cvt_i32_f32 s5, exec_lo               ; encoding: [0x7e,0x66,0x85,0xbe]
 
-# GFX12: s_cvt_i32_f32 s5, exec_hi               ; encoding: [0x7f,0x66,0x85,0xbe]
 0x7f,0x66,0x85,0xbe
+# GFX12: s_cvt_i32_f32 s5, exec_hi               ; encoding: [0x7f,0x66,0x85,0xbe]
 
-# GFX12: s_cvt_i32_f32 s5, 0                     ; encoding: [0x80,0x66,0x85,0xbe]
 0x80,0x66,0x85,0xbe
+# GFX12: s_cvt_i32_f32 s5, 0                     ; encoding: [0x80,0x66,0x85,0xbe]
 
-# GFX12: s_cvt_i32_f32 s5, -1                    ; encoding: [0xc1,0x66,0x85,0xbe]
 0xc1,0x66,0x85,0xbe
+# GFX12: s_cvt_i32_f32 s5, -1                    ; encoding: [0xc1,0x66,0x85,0xbe]
 
-# GFX12: s_cvt_i32_f32 s5, 0.5                   ; encoding: [0xf0,0x66,0x85,0xbe]
 0xf0,0x66,0x85,0xbe
+# GFX12: s_cvt_i32_f32 s5, 0.5                   ; encoding: [0xf0,0x66,0x85,0xbe]
 
-# GFX12: s_cvt_i32_f32 s5, -4.0                  ; encoding: [0xf7,0x66,0x85,0xbe]
 0xf7,0x66,0x85,0xbe
+# GFX12: s_cvt_i32_f32 s5, -4.0                  ; encoding: [0xf7,0x66,0x85,0xbe]
 
-# GFX12: s_cvt_i32_f32 s5, 0xaf123456            ; encoding: [0xff,0x66,0x85,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x66,0x85,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_cvt_i32_f32 s5, 0xaf123456            ; encoding: [0xff,0x66,0x85,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cvt_i32_f32 s5, 0x3f717273            ; encoding: [0xff,0x66,0x85,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x66,0x85,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_cvt_i32_f32 s5, 0x3f717273            ; encoding: [0xff,0x66,0x85,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cvt_f16_f32 s5, s1                    ; encoding: [0x01,0x68,0x85,0xbe]
 0x01,0x68,0x85,0xbe
+# GFX12: s_cvt_f16_f32 s5, s1                    ; encoding: [0x01,0x68,0x85,0xbe]
 
-# GFX12: s_cvt_f16_f32 s105, s1                  ; encoding: [0x01,0x68,0xe9,0xbe]
 0x01,0x68,0xe9,0xbe
+# GFX12: s_cvt_f16_f32 s105, s1                  ; encoding: [0x01,0x68,0xe9,0xbe]
 
-# GFX12: s_cvt_f16_f32 s5, s105                  ; encoding: [0x69,0x68,0x85,0xbe]
 0x69,0x68,0x85,0xbe
+# GFX12: s_cvt_f16_f32 s5, s105                  ; encoding: [0x69,0x68,0x85,0xbe]
 
-# GFX12: s_cvt_f16_f32 s5, s103                  ; encoding: [0x67,0x68,0x85,0xbe]
 0x67,0x68,0x85,0xbe
+# GFX12: s_cvt_f16_f32 s5, s103                  ; encoding: [0x67,0x68,0x85,0xbe]
 
-# GFX12: s_cvt_f16_f32 s5, vcc_lo                ; encoding: [0x6a,0x68,0x85,0xbe]
 0x6a,0x68,0x85,0xbe
+# GFX12: s_cvt_f16_f32 s5, vcc_lo                ; encoding: [0x6a,0x68,0x85,0xbe]
 
-# GFX12: s_cvt_f16_f32 s5, vcc_hi                ; encoding: [0x6b,0x68,0x85,0xbe]
 0x6b,0x68,0x85,0xbe
+# GFX12: s_cvt_f16_f32 s5, vcc_hi                ; encoding: [0x6b,0x68,0x85,0xbe]
 
-# GFX12: s_cvt_f16_f32 s5, ttmp11                ; encoding: [0x77,0x68,0x85,0xbe]
 0x77,0x68,0x85,0xbe
+# GFX12: s_cvt_f16_f32 s5, ttmp11                ; encoding: [0x77,0x68,0x85,0xbe]
 
-# GFX12: s_cvt_f16_f32 s5, m0                    ; encoding: [0x7d,0x68,0x85,0xbe]
 0x7d,0x68,0x85,0xbe
+# GFX12: s_cvt_f16_f32 s5, m0                    ; encoding: [0x7d,0x68,0x85,0xbe]
 
-# GFX12: s_cvt_f16_f32 s5, exec_lo               ; encoding: [0x7e,0x68,0x85,0xbe]
 0x7e,0x68,0x85,0xbe
+# GFX12: s_cvt_f16_f32 s5, exec_lo               ; encoding: [0x7e,0x68,0x85,0xbe]
 
-# GFX12: s_cvt_f16_f32 s5, exec_hi               ; encoding: [0x7f,0x68,0x85,0xbe]
 0x7f,0x68,0x85,0xbe
+# GFX12: s_cvt_f16_f32 s5, exec_hi               ; encoding: [0x7f,0x68,0x85,0xbe]
 
-# GFX12: s_cvt_f16_f32 s5, 0                     ; encoding: [0x80,0x68,0x85,0xbe]
 0x80,0x68,0x85,0xbe
+# GFX12: s_cvt_f16_f32 s5, 0                     ; encoding: [0x80,0x68,0x85,0xbe]
 
-# GFX12: s_cvt_f16_f32 s5, -1                    ; encoding: [0xc1,0x68,0x85,0xbe]
 0xc1,0x68,0x85,0xbe
+# GFX12: s_cvt_f16_f32 s5, -1                    ; encoding: [0xc1,0x68,0x85,0xbe]
 
-# GFX12: s_cvt_f16_f32 s5, 0.5                   ; encoding: [0xf0,0x68,0x85,0xbe]
 0xf0,0x68,0x85,0xbe
+# GFX12: s_cvt_f16_f32 s5, 0.5                   ; encoding: [0xf0,0x68,0x85,0xbe]
 
-# GFX12: s_cvt_f16_f32 s5, -4.0                  ; encoding: [0xf7,0x68,0x85,0xbe]
 0xf7,0x68,0x85,0xbe
+# GFX12: s_cvt_f16_f32 s5, -4.0                  ; encoding: [0xf7,0x68,0x85,0xbe]
 
-# GFX12: s_cvt_f16_f32 s5, 0xaf123456            ; encoding: [0xff,0x68,0x85,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x68,0x85,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_cvt_f16_f32 s5, 0xaf123456            ; encoding: [0xff,0x68,0x85,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cvt_f16_f32 s5, 0x3f717273            ; encoding: [0xff,0x68,0x85,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x68,0x85,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_cvt_f16_f32 s5, 0x3f717273            ; encoding: [0xff,0x68,0x85,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cvt_f32_f16 s5, s1                    ; encoding: [0x01,0x69,0x85,0xbe]
 0x01,0x69,0x85,0xbe
+# GFX12: s_cvt_f32_f16 s5, s1                    ; encoding: [0x01,0x69,0x85,0xbe]
 
-# GFX12: s_cvt_f32_f16 s105, s1                  ; encoding: [0x01,0x69,0xe9,0xbe]
 0x01,0x69,0xe9,0xbe
+# GFX12: s_cvt_f32_f16 s105, s1                  ; encoding: [0x01,0x69,0xe9,0xbe]
 
-# GFX12: s_cvt_f32_f16 s5, s105                  ; encoding: [0x69,0x69,0x85,0xbe]
 0x69,0x69,0x85,0xbe
+# GFX12: s_cvt_f32_f16 s5, s105                  ; encoding: [0x69,0x69,0x85,0xbe]
 
-# GFX12: s_cvt_f32_f16 s5, s103                  ; encoding: [0x67,0x69,0x85,0xbe]
 0x67,0x69,0x85,0xbe
+# GFX12: s_cvt_f32_f16 s5, s103                  ; encoding: [0x67,0x69,0x85,0xbe]
 
-# GFX12: s_cvt_f32_f16 s5, vcc_lo                ; encoding: [0x6a,0x69,0x85,0xbe]
 0x6a,0x69,0x85,0xbe
+# GFX12: s_cvt_f32_f16 s5, vcc_lo                ; encoding: [0x6a,0x69,0x85,0xbe]
 
-# GFX12: s_cvt_f32_f16 s5, vcc_hi                ; encoding: [0x6b,0x69,0x85,0xbe]
 0x6b,0x69,0x85,0xbe
+# GFX12: s_cvt_f32_f16 s5, vcc_hi                ; encoding: [0x6b,0x69,0x85,0xbe]
 
-# GFX12: s_cvt_f32_f16 s5, ttmp11                ; encoding: [0x77,0x69,0x85,0xbe]
 0x77,0x69,0x85,0xbe
+# GFX12: s_cvt_f32_f16 s5, ttmp11                ; encoding: [0x77,0x69,0x85,0xbe]
 
-# GFX12: s_cvt_f32_f16 s5, m0                    ; encoding: [0x7d,0x69,0x85,0xbe]
 0x7d,0x69,0x85,0xbe
+# GFX12: s_cvt_f32_f16 s5, m0                    ; encoding: [0x7d,0x69,0x85,0xbe]
 
-# GFX12: s_cvt_f32_f16 s5, exec_lo               ; encoding: [0x7e,0x69,0x85,0xbe]
 0x7e,0x69,0x85,0xbe
+# GFX12: s_cvt_f32_f16 s5, exec_lo               ; encoding: [0x7e,0x69,0x85,0xbe]
 
-# GFX12: s_cvt_f32_f16 s5, exec_hi               ; encoding: [0x7f,0x69,0x85,0xbe]
 0x7f,0x69,0x85,0xbe
+# GFX12: s_cvt_f32_f16 s5, exec_hi               ; encoding: [0x7f,0x69,0x85,0xbe]
 
-# GFX12: s_cvt_f32_f16 s5, 0                     ; encoding: [0x80,0x69,0x85,0xbe]
 0x80,0x69,0x85,0xbe
+# GFX12: s_cvt_f32_f16 s5, 0                     ; encoding: [0x80,0x69,0x85,0xbe]
 
-# GFX12: s_cvt_f32_f16 s5, -1                    ; encoding: [0xc1,0x69,0x85,0xbe]
 0xc1,0x69,0x85,0xbe
+# GFX12: s_cvt_f32_f16 s5, -1                    ; encoding: [0xc1,0x69,0x85,0xbe]
 
-# GFX12: s_cvt_hi_f32_f16 s5, s1                 ; encoding: [0x01,0x6a,0x85,0xbe]
 0x01,0x6a,0x85,0xbe
+# GFX12: s_cvt_hi_f32_f16 s5, s1                 ; encoding: [0x01,0x6a,0x85,0xbe]
 
-# GFX12: s_cvt_hi_f32_f16 s105, s1               ; encoding: [0x01,0x6a,0xe9,0xbe]
 0x01,0x6a,0xe9,0xbe
+# GFX12: s_cvt_hi_f32_f16 s105, s1               ; encoding: [0x01,0x6a,0xe9,0xbe]
 
-# GFX12: s_cvt_hi_f32_f16 s5, s105               ; encoding: [0x69,0x6a,0x85,0xbe]
 0x69,0x6a,0x85,0xbe
+# GFX12: s_cvt_hi_f32_f16 s5, s105               ; encoding: [0x69,0x6a,0x85,0xbe]
 
-# GFX12: s_cvt_hi_f32_f16 s5, s103               ; encoding: [0x67,0x6a,0x85,0xbe]
 0x67,0x6a,0x85,0xbe
+# GFX12: s_cvt_hi_f32_f16 s5, s103               ; encoding: [0x67,0x6a,0x85,0xbe]
 
-# GFX12: s_cvt_hi_f32_f16 s5, vcc_lo             ; encoding: [0x6a,0x6a,0x85,0xbe]
 0x6a,0x6a,0x85,0xbe
+# GFX12: s_cvt_hi_f32_f16 s5, vcc_lo             ; encoding: [0x6a,0x6a,0x85,0xbe]
 
-# GFX12: s_cvt_hi_f32_f16 s5, vcc_hi             ; encoding: [0x6b,0x6a,0x85,0xbe]
 0x6b,0x6a,0x85,0xbe
+# GFX12: s_cvt_hi_f32_f16 s5, vcc_hi             ; encoding: [0x6b,0x6a,0x85,0xbe]
 
-# GFX12: s_cvt_hi_f32_f16 s5, ttmp11             ; encoding: [0x77,0x6a,0x85,0xbe]
 0x77,0x6a,0x85,0xbe
+# GFX12: s_cvt_hi_f32_f16 s5, ttmp11             ; encoding: [0x77,0x6a,0x85,0xbe]
 
-# GFX12: s_cvt_hi_f32_f16 s5, m0                 ; encoding: [0x7d,0x6a,0x85,0xbe]
 0x7d,0x6a,0x85,0xbe
+# GFX12: s_cvt_hi_f32_f16 s5, m0                 ; encoding: [0x7d,0x6a,0x85,0xbe]
 
-# GFX12: s_cvt_hi_f32_f16 s5, exec_lo            ; encoding: [0x7e,0x6a,0x85,0xbe]
 0x7e,0x6a,0x85,0xbe
+# GFX12: s_cvt_hi_f32_f16 s5, exec_lo            ; encoding: [0x7e,0x6a,0x85,0xbe]
 
-# GFX12: s_cvt_hi_f32_f16 s5, exec_hi            ; encoding: [0x7f,0x6a,0x85,0xbe]
 0x7f,0x6a,0x85,0xbe
+# GFX12: s_cvt_hi_f32_f16 s5, exec_hi            ; encoding: [0x7f,0x6a,0x85,0xbe]
 
-# GFX12: s_cvt_hi_f32_f16 s5, 0                  ; encoding: [0x80,0x6a,0x85,0xbe]
 0x80,0x6a,0x85,0xbe
+# GFX12: s_cvt_hi_f32_f16 s5, 0                  ; encoding: [0x80,0x6a,0x85,0xbe]
 
-# GFX12: s_cvt_hi_f32_f16 s5, -1                 ; encoding: [0xc1,0x6a,0x85,0xbe]
 0xc1,0x6a,0x85,0xbe
+# GFX12: s_cvt_hi_f32_f16 s5, -1                 ; encoding: [0xc1,0x6a,0x85,0xbe]
 
-# GFX12: s_trunc_f32 s5, s1                      ; encoding: [0x01,0x62,0x85,0xbe]
 0x01,0x62,0x85,0xbe
+# GFX12: s_trunc_f32 s5, s1                      ; encoding: [0x01,0x62,0x85,0xbe]
 
-# GFX12: s_trunc_f32 s105, s1                    ; encoding: [0x01,0x62,0xe9,0xbe]
 0x01,0x62,0xe9,0xbe
+# GFX12: s_trunc_f32 s105, s1                    ; encoding: [0x01,0x62,0xe9,0xbe]
 
-# GFX12: s_trunc_f32 s5, s105                    ; encoding: [0x69,0x62,0x85,0xbe]
 0x69,0x62,0x85,0xbe
+# GFX12: s_trunc_f32 s5, s105                    ; encoding: [0x69,0x62,0x85,0xbe]
 
-# GFX12: s_trunc_f32 s5, s103                    ; encoding: [0x67,0x62,0x85,0xbe]
 0x67,0x62,0x85,0xbe
+# GFX12: s_trunc_f32 s5, s103                    ; encoding: [0x67,0x62,0x85,0xbe]
 
-# GFX12: s_trunc_f32 s5, vcc_lo                  ; encoding: [0x6a,0x62,0x85,0xbe]
 0x6a,0x62,0x85,0xbe
+# GFX12: s_trunc_f32 s5, vcc_lo                  ; encoding: [0x6a,0x62,0x85,0xbe]
 
-# GFX12: s_trunc_f32 s5, vcc_hi                  ; encoding: [0x6b,0x62,0x85,0xbe]
 0x6b,0x62,0x85,0xbe
+# GFX12: s_trunc_f32 s5, vcc_hi                  ; encoding: [0x6b,0x62,0x85,0xbe]
 
-# GFX12: s_trunc_f32 s5, ttmp11                  ; encoding: [0x77,0x62,0x85,0xbe]
 0x77,0x62,0x85,0xbe
+# GFX12: s_trunc_f32 s5, ttmp11                  ; encoding: [0x77,0x62,0x85,0xbe]
 
-# GFX12: s_trunc_f32 s5, m0                      ; encoding: [0x7d,0x62,0x85,0xbe]
 0x7d,0x62,0x85,0xbe
+# GFX12: s_trunc_f32 s5, m0                      ; encoding: [0x7d,0x62,0x85,0xbe]
 
-# GFX12: s_trunc_f32 s5, exec_lo                 ; encoding: [0x7e,0x62,0x85,0xbe]
 0x7e,0x62,0x85,0xbe
+# GFX12: s_trunc_f32 s5, exec_lo                 ; encoding: [0x7e,0x62,0x85,0xbe]
 
-# GFX12: s_trunc_f32 s5, exec_hi                 ; encoding: [0x7f,0x62,0x85,0xbe]
 0x7f,0x62,0x85,0xbe
+# GFX12: s_trunc_f32 s5, exec_hi                 ; encoding: [0x7f,0x62,0x85,0xbe]
 
-# GFX12: s_trunc_f32 s5, 0                       ; encoding: [0x80,0x62,0x85,0xbe]
 0x80,0x62,0x85,0xbe
+# GFX12: s_trunc_f32 s5, 0                       ; encoding: [0x80,0x62,0x85,0xbe]
 
-# GFX12: s_trunc_f32 s5, -1                      ; encoding: [0xc1,0x62,0x85,0xbe]
 0xc1,0x62,0x85,0xbe
+# GFX12: s_trunc_f32 s5, -1                      ; encoding: [0xc1,0x62,0x85,0xbe]
 
-# GFX12: s_trunc_f32 s5, 0.5                     ; encoding: [0xf0,0x62,0x85,0xbe]
 0xf0,0x62,0x85,0xbe
+# GFX12: s_trunc_f32 s5, 0.5                     ; encoding: [0xf0,0x62,0x85,0xbe]
 
-# GFX12: s_trunc_f32 s5, -4.0                    ; encoding: [0xf7,0x62,0x85,0xbe]
 0xf7,0x62,0x85,0xbe
+# GFX12: s_trunc_f32 s5, -4.0                    ; encoding: [0xf7,0x62,0x85,0xbe]
 
-# GFX12: s_trunc_f32 s5, 0xaf123456              ; encoding: [0xff,0x62,0x85,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x62,0x85,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_trunc_f32 s5, 0xaf123456              ; encoding: [0xff,0x62,0x85,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_trunc_f32 s5, 0x3f717273              ; encoding: [0xff,0x62,0x85,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x62,0x85,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_trunc_f32 s5, 0x3f717273              ; encoding: [0xff,0x62,0x85,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_ceil_f32 s5, s1                       ; encoding: [0x01,0x60,0x85,0xbe]
 0x01,0x60,0x85,0xbe
+# GFX12: s_ceil_f32 s5, s1                       ; encoding: [0x01,0x60,0x85,0xbe]
 
-# GFX12: s_ceil_f32 s105, s1                     ; encoding: [0x01,0x60,0xe9,0xbe]
 0x01,0x60,0xe9,0xbe
+# GFX12: s_ceil_f32 s105, s1                     ; encoding: [0x01,0x60,0xe9,0xbe]
 
-# GFX12: s_ceil_f32 s5, s105                     ; encoding: [0x69,0x60,0x85,0xbe]
 0x69,0x60,0x85,0xbe
+# GFX12: s_ceil_f32 s5, s105                     ; encoding: [0x69,0x60,0x85,0xbe]
 
-# GFX12: s_ceil_f32 s5, s103                     ; encoding: [0x67,0x60,0x85,0xbe]
 0x67,0x60,0x85,0xbe
+# GFX12: s_ceil_f32 s5, s103                     ; encoding: [0x67,0x60,0x85,0xbe]
 
-# GFX12: s_ceil_f32 s5, vcc_lo                   ; encoding: [0x6a,0x60,0x85,0xbe]
 0x6a,0x60,0x85,0xbe
+# GFX12: s_ceil_f32 s5, vcc_lo                   ; encoding: [0x6a,0x60,0x85,0xbe]
 
-# GFX12: s_ceil_f32 s5, vcc_hi                   ; encoding: [0x6b,0x60,0x85,0xbe]
 0x6b,0x60,0x85,0xbe
+# GFX12: s_ceil_f32 s5, vcc_hi                   ; encoding: [0x6b,0x60,0x85,0xbe]
 
-# GFX12: s_ceil_f32 s5, ttmp11                   ; encoding: [0x77,0x60,0x85,0xbe]
 0x77,0x60,0x85,0xbe
+# GFX12: s_ceil_f32 s5, ttmp11                   ; encoding: [0x77,0x60,0x85,0xbe]
 
-# GFX12: s_ceil_f32 s5, m0                       ; encoding: [0x7d,0x60,0x85,0xbe]
 0x7d,0x60,0x85,0xbe
+# GFX12: s_ceil_f32 s5, m0                       ; encoding: [0x7d,0x60,0x85,0xbe]
 
-# GFX12: s_ceil_f32 s5, exec_lo                  ; encoding: [0x7e,0x60,0x85,0xbe]
 0x7e,0x60,0x85,0xbe
+# GFX12: s_ceil_f32 s5, exec_lo                  ; encoding: [0x7e,0x60,0x85,0xbe]
 
-# GFX12: s_ceil_f32 s5, exec_hi                  ; encoding: [0x7f,0x60,0x85,0xbe]
 0x7f,0x60,0x85,0xbe
+# GFX12: s_ceil_f32 s5, exec_hi                  ; encoding: [0x7f,0x60,0x85,0xbe]
 
-# GFX12: s_ceil_f32 s5, 0                        ; encoding: [0x80,0x60,0x85,0xbe]
 0x80,0x60,0x85,0xbe
+# GFX12: s_ceil_f32 s5, 0                        ; encoding: [0x80,0x60,0x85,0xbe]
 
-# GFX12: s_ceil_f32 s5, -1                       ; encoding: [0xc1,0x60,0x85,0xbe]
 0xc1,0x60,0x85,0xbe
+# GFX12: s_ceil_f32 s5, -1                       ; encoding: [0xc1,0x60,0x85,0xbe]
 
-# GFX12: s_ceil_f32 s5, 0.5                      ; encoding: [0xf0,0x60,0x85,0xbe]
 0xf0,0x60,0x85,0xbe
+# GFX12: s_ceil_f32 s5, 0.5                      ; encoding: [0xf0,0x60,0x85,0xbe]
 
-# GFX12: s_ceil_f32 s5, -4.0                     ; encoding: [0xf7,0x60,0x85,0xbe]
 0xf7,0x60,0x85,0xbe
+# GFX12: s_ceil_f32 s5, -4.0                     ; encoding: [0xf7,0x60,0x85,0xbe]
 
-# GFX12: s_ceil_f32 s5, 0xaf123456               ; encoding: [0xff,0x60,0x85,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x60,0x85,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_ceil_f32 s5, 0xaf123456               ; encoding: [0xff,0x60,0x85,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_ceil_f32 s5, 0x3f717273               ; encoding: [0xff,0x60,0x85,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x60,0x85,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_ceil_f32 s5, 0x3f717273               ; encoding: [0xff,0x60,0x85,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_rndne_f32 s5, s1                      ; encoding: [0x01,0x63,0x85,0xbe]
 0x01,0x63,0x85,0xbe
+# GFX12: s_rndne_f32 s5, s1                      ; encoding: [0x01,0x63,0x85,0xbe]
 
-# GFX12: s_rndne_f32 s105, s1                    ; encoding: [0x01,0x63,0xe9,0xbe]
 0x01,0x63,0xe9,0xbe
+# GFX12: s_rndne_f32 s105, s1                    ; encoding: [0x01,0x63,0xe9,0xbe]
 
-# GFX12: s_rndne_f32 s5, s105                    ; encoding: [0x69,0x63,0x85,0xbe]
 0x69,0x63,0x85,0xbe
+# GFX12: s_rndne_f32 s5, s105                    ; encoding: [0x69,0x63,0x85,0xbe]
 
-# GFX12: s_rndne_f32 s5, s103                    ; encoding: [0x67,0x63,0x85,0xbe]
 0x67,0x63,0x85,0xbe
+# GFX12: s_rndne_f32 s5, s103                    ; encoding: [0x67,0x63,0x85,0xbe]
 
-# GFX12: s_rndne_f32 s5, vcc_lo                  ; encoding: [0x6a,0x63,0x85,0xbe]
 0x6a,0x63,0x85,0xbe
+# GFX12: s_rndne_f32 s5, vcc_lo                  ; encoding: [0x6a,0x63,0x85,0xbe]
 
-# GFX12: s_rndne_f32 s5, vcc_hi                  ; encoding: [0x6b,0x63,0x85,0xbe]
 0x6b,0x63,0x85,0xbe
+# GFX12: s_rndne_f32 s5, vcc_hi                  ; encoding: [0x6b,0x63,0x85,0xbe]
 
-# GFX12: s_rndne_f32 s5, ttmp11                  ; encoding: [0x77,0x63,0x85,0xbe]
 0x77,0x63,0x85,0xbe
+# GFX12: s_rndne_f32 s5, ttmp11                  ; encoding: [0x77,0x63,0x85,0xbe]
 
-# GFX12: s_rndne_f32 s5, m0                      ; encoding: [0x7d,0x63,0x85,0xbe]
 0x7d,0x63,0x85,0xbe
+# GFX12: s_rndne_f32 s5, m0                      ; encoding: [0x7d,0x63,0x85,0xbe]
 
-# GFX12: s_rndne_f32 s5, exec_lo                 ; encoding: [0x7e,0x63,0x85,0xbe]
 0x7e,0x63,0x85,0xbe
+# GFX12: s_rndne_f32 s5, exec_lo                 ; encoding: [0x7e,0x63,0x85,0xbe]
 
-# GFX12: s_rndne_f32 s5, exec_hi                 ; encoding: [0x7f,0x63,0x85,0xbe]
 0x7f,0x63,0x85,0xbe
+# GFX12: s_rndne_f32 s5, exec_hi                 ; encoding: [0x7f,0x63,0x85,0xbe]
 
-# GFX12: s_rndne_f32 s5, 0                       ; encoding: [0x80,0x63,0x85,0xbe]
 0x80,0x63,0x85,0xbe
+# GFX12: s_rndne_f32 s5, 0                       ; encoding: [0x80,0x63,0x85,0xbe]
 
-# GFX12: s_rndne_f32 s5, -1                      ; encoding: [0xc1,0x63,0x85,0xbe]
 0xc1,0x63,0x85,0xbe
+# GFX12: s_rndne_f32 s5, -1                      ; encoding: [0xc1,0x63,0x85,0xbe]
 
-# GFX12: s_rndne_f32 s5, 0.5                     ; encoding: [0xf0,0x63,0x85,0xbe]
 0xf0,0x63,0x85,0xbe
+# GFX12: s_rndne_f32 s5, 0.5                     ; encoding: [0xf0,0x63,0x85,0xbe]
 
-# GFX12: s_rndne_f32 s5, -4.0                    ; encoding: [0xf7,0x63,0x85,0xbe]
 0xf7,0x63,0x85,0xbe
+# GFX12: s_rndne_f32 s5, -4.0                    ; encoding: [0xf7,0x63,0x85,0xbe]
 
-# GFX12: s_rndne_f32 s5, 0xaf123456              ; encoding: [0xff,0x63,0x85,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x63,0x85,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_rndne_f32 s5, 0xaf123456              ; encoding: [0xff,0x63,0x85,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_rndne_f32 s5, 0x3f717273              ; encoding: [0xff,0x63,0x85,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x63,0x85,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_rndne_f32 s5, 0x3f717273              ; encoding: [0xff,0x63,0x85,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_floor_f32 s5, s1                      ; encoding: [0x01,0x61,0x85,0xbe]
 0x01,0x61,0x85,0xbe
+# GFX12: s_floor_f32 s5, s1                      ; encoding: [0x01,0x61,0x85,0xbe]
 
-# GFX12: s_floor_f32 s105, s1                    ; encoding: [0x01,0x61,0xe9,0xbe]
 0x01,0x61,0xe9,0xbe
+# GFX12: s_floor_f32 s105, s1                    ; encoding: [0x01,0x61,0xe9,0xbe]
 
-# GFX12: s_floor_f32 s5, s105                    ; encoding: [0x69,0x61,0x85,0xbe]
 0x69,0x61,0x85,0xbe
+# GFX12: s_floor_f32 s5, s105                    ; encoding: [0x69,0x61,0x85,0xbe]
 
-# GFX12: s_floor_f32 s5, s103                    ; encoding: [0x67,0x61,0x85,0xbe]
 0x67,0x61,0x85,0xbe
+# GFX12: s_floor_f32 s5, s103                    ; encoding: [0x67,0x61,0x85,0xbe]
 
-# GFX12: s_floor_f32 s5, vcc_lo                  ; encoding: [0x6a,0x61,0x85,0xbe]
 0x6a,0x61,0x85,0xbe
+# GFX12: s_floor_f32 s5, vcc_lo                  ; encoding: [0x6a,0x61,0x85,0xbe]
 
-# GFX12: s_floor_f32 s5, vcc_hi                  ; encoding: [0x6b,0x61,0x85,0xbe]
 0x6b,0x61,0x85,0xbe
+# GFX12: s_floor_f32 s5, vcc_hi                  ; encoding: [0x6b,0x61,0x85,0xbe]
 
-# GFX12: s_floor_f32 s5, ttmp11                  ; encoding: [0x77,0x61,0x85,0xbe]
 0x77,0x61,0x85,0xbe
+# GFX12: s_floor_f32 s5, ttmp11                  ; encoding: [0x77,0x61,0x85,0xbe]
 
-# GFX12: s_floor_f32 s5, m0                      ; encoding: [0x7d,0x61,0x85,0xbe]
 0x7d,0x61,0x85,0xbe
+# GFX12: s_floor_f32 s5, m0                      ; encoding: [0x7d,0x61,0x85,0xbe]
 
-# GFX12: s_floor_f32 s5, exec_lo                 ; encoding: [0x7e,0x61,0x85,0xbe]
 0x7e,0x61,0x85,0xbe
+# GFX12: s_floor_f32 s5, exec_lo                 ; encoding: [0x7e,0x61,0x85,0xbe]
 
-# GFX12: s_floor_f32 s5, exec_hi                 ; encoding: [0x7f,0x61,0x85,0xbe]
 0x7f,0x61,0x85,0xbe
+# GFX12: s_floor_f32 s5, exec_hi                 ; encoding: [0x7f,0x61,0x85,0xbe]
 
-# GFX12: s_floor_f32 s5, 0                       ; encoding: [0x80,0x61,0x85,0xbe]
 0x80,0x61,0x85,0xbe
+# GFX12: s_floor_f32 s5, 0                       ; encoding: [0x80,0x61,0x85,0xbe]
 
-# GFX12: s_floor_f32 s5, -1                      ; encoding: [0xc1,0x61,0x85,0xbe]
 0xc1,0x61,0x85,0xbe
+# GFX12: s_floor_f32 s5, -1                      ; encoding: [0xc1,0x61,0x85,0xbe]
 
-# GFX12: s_floor_f32 s5, 0.5                     ; encoding: [0xf0,0x61,0x85,0xbe]
 0xf0,0x61,0x85,0xbe
+# GFX12: s_floor_f32 s5, 0.5                     ; encoding: [0xf0,0x61,0x85,0xbe]
 
-# GFX12: s_floor_f32 s5, -4.0                    ; encoding: [0xf7,0x61,0x85,0xbe]
 0xf7,0x61,0x85,0xbe
+# GFX12: s_floor_f32 s5, -4.0                    ; encoding: [0xf7,0x61,0x85,0xbe]
 
-# GFX12: s_floor_f32 s5, 0xaf123456              ; encoding: [0xff,0x61,0x85,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x61,0x85,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_floor_f32 s5, 0xaf123456              ; encoding: [0xff,0x61,0x85,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_floor_f32 s5, 0x3f717273              ; encoding: [0xff,0x61,0x85,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x61,0x85,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_floor_f32 s5, 0x3f717273              ; encoding: [0xff,0x61,0x85,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_floor_f16 s5, s1                      ; encoding: [0x01,0x6c,0x85,0xbe]
 0x01,0x6c,0x85,0xbe
+# GFX12: s_floor_f16 s5, s1                      ; encoding: [0x01,0x6c,0x85,0xbe]
 
-# GFX12: s_floor_f16 s105, s1                    ; encoding: [0x01,0x6c,0xe9,0xbe]
 0x01,0x6c,0xe9,0xbe
+# GFX12: s_floor_f16 s105, s1                    ; encoding: [0x01,0x6c,0xe9,0xbe]
 
-# GFX12: s_floor_f16 s5, s105                    ; encoding: [0x69,0x6c,0x85,0xbe]
 0x69,0x6c,0x85,0xbe
+# GFX12: s_floor_f16 s5, s105                    ; encoding: [0x69,0x6c,0x85,0xbe]
 
-# GFX12: s_floor_f16 s5, s101                    ; encoding: [0x65,0x6c,0x85,0xbe]
 0x65,0x6c,0x85,0xbe
+# GFX12: s_floor_f16 s5, s101                    ; encoding: [0x65,0x6c,0x85,0xbe]
 
-# GFX12: s_floor_f16 s5, vcc_lo                  ; encoding: [0x6a,0x6c,0x85,0xbe]
 0x6a,0x6c,0x85,0xbe
+# GFX12: s_floor_f16 s5, vcc_lo                  ; encoding: [0x6a,0x6c,0x85,0xbe]
 
-# GFX12: s_floor_f16 s5, vcc_hi                  ; encoding: [0x6b,0x6c,0x85,0xbe]
 0x6b,0x6c,0x85,0xbe
+# GFX12: s_floor_f16 s5, vcc_hi                  ; encoding: [0x6b,0x6c,0x85,0xbe]
 
-# GFX12: s_floor_f16 s5, m0                      ; encoding: [0x7d,0x6c,0x85,0xbe]
 0x7d,0x6c,0x85,0xbe
+# GFX12: s_floor_f16 s5, m0                      ; encoding: [0x7d,0x6c,0x85,0xbe]
 
-# GFX12: s_floor_f16 s5, exec_lo                 ; encoding: [0x7e,0x6c,0x85,0xbe]
 0x7e,0x6c,0x85,0xbe
+# GFX12: s_floor_f16 s5, exec_lo                 ; encoding: [0x7e,0x6c,0x85,0xbe]
 
-# GFX12: s_floor_f16 s5, exec_hi                 ; encoding: [0x7f,0x6c,0x85,0xbe]
 0x7f,0x6c,0x85,0xbe
+# GFX12: s_floor_f16 s5, exec_hi                 ; encoding: [0x7f,0x6c,0x85,0xbe]
 
-# GFX12: s_floor_f16 s5, 0                       ; encoding: [0x80,0x6c,0x85,0xbe]
 0x80,0x6c,0x85,0xbe
+# GFX12: s_floor_f16 s5, 0                       ; encoding: [0x80,0x6c,0x85,0xbe]
 
-# GFX12: s_floor_f16 s5, -1                      ; encoding: [0xc1,0x6c,0x85,0xbe]
 0xc1,0x6c,0x85,0xbe
+# GFX12: s_floor_f16 s5, -1                      ; encoding: [0xc1,0x6c,0x85,0xbe]
 
-# GFX12: s_floor_f16 s5, 0.5                     ; encoding: [0xf0,0x6c,0x85,0xbe]
 0xf0,0x6c,0x85,0xbe
+# GFX12: s_floor_f16 s5, 0.5                     ; encoding: [0xf0,0x6c,0x85,0xbe]
 
-# GFX12: s_floor_f16 s5, -4.0                    ; encoding: [0xf7,0x6c,0x85,0xbe]
 0xf7,0x6c,0x85,0xbe
+# GFX12: s_floor_f16 s5, -4.0                    ; encoding: [0xf7,0x6c,0x85,0xbe]
 
-# GFX12: s_floor_f16 s5, 0xfe0b                  ; encoding: [0xff,0x6c,0x85,0xbe,0x0b,0xfe,0x00,0x00]
 0xff,0x6c,0x85,0xbe,0x0b,0xfe,0x00,0x00
+# GFX12: s_floor_f16 s5, 0xfe0b                  ; encoding: [0xff,0x6c,0x85,0xbe,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_floor_f16 s5, 0x3456                  ; encoding: [0xff,0x6c,0x85,0xbe,0x56,0x34,0x00,0x00]
 0xff,0x6c,0x85,0xbe,0x56,0x34,0x00,0x00
+# GFX12: s_floor_f16 s5, 0x3456                  ; encoding: [0xff,0x6c,0x85,0xbe,0x56,0x34,0x00,0x00]
 
-# GFX12: s_ceil_f16 s5, s1                       ; encoding: [0x01,0x6b,0x85,0xbe]
 0x01,0x6b,0x85,0xbe
+# GFX12: s_ceil_f16 s5, s1                       ; encoding: [0x01,0x6b,0x85,0xbe]
 
-# GFX12: s_ceil_f16 s105, s1                     ; encoding: [0x01,0x6b,0xe9,0xbe]
 0x01,0x6b,0xe9,0xbe
+# GFX12: s_ceil_f16 s105, s1                     ; encoding: [0x01,0x6b,0xe9,0xbe]
 
-# GFX12: s_ceil_f16 s5, s105                     ; encoding: [0x69,0x6b,0x85,0xbe]
 0x69,0x6b,0x85,0xbe
+# GFX12: s_ceil_f16 s5, s105                     ; encoding: [0x69,0x6b,0x85,0xbe]
 
-# GFX12: s_ceil_f16 s5, s101                     ; encoding: [0x65,0x6b,0x85,0xbe]
 0x65,0x6b,0x85,0xbe
+# GFX12: s_ceil_f16 s5, s101                     ; encoding: [0x65,0x6b,0x85,0xbe]
 
-# GFX12: s_ceil_f16 s5, vcc_lo                   ; encoding: [0x6a,0x6b,0x85,0xbe]
 0x6a,0x6b,0x85,0xbe
+# GFX12: s_ceil_f16 s5, vcc_lo                   ; encoding: [0x6a,0x6b,0x85,0xbe]
 
-# GFX12: s_ceil_f16 s5, vcc_hi                   ; encoding: [0x6b,0x6b,0x85,0xbe]
 0x6b,0x6b,0x85,0xbe
+# GFX12: s_ceil_f16 s5, vcc_hi                   ; encoding: [0x6b,0x6b,0x85,0xbe]
 
-# GFX12: s_ceil_f16 s5, m0                       ; encoding: [0x7d,0x6b,0x85,0xbe]
 0x7d,0x6b,0x85,0xbe
+# GFX12: s_ceil_f16 s5, m0                       ; encoding: [0x7d,0x6b,0x85,0xbe]
 
-# GFX12: s_ceil_f16 s5, exec_lo                  ; encoding: [0x7e,0x6b,0x85,0xbe]
 0x7e,0x6b,0x85,0xbe
+# GFX12: s_ceil_f16 s5, exec_lo                  ; encoding: [0x7e,0x6b,0x85,0xbe]
 
-# GFX12: s_ceil_f16 s5, exec_hi                  ; encoding: [0x7f,0x6b,0x85,0xbe]
 0x7f,0x6b,0x85,0xbe
+# GFX12: s_ceil_f16 s5, exec_hi                  ; encoding: [0x7f,0x6b,0x85,0xbe]
 
-# GFX12: s_ceil_f16 s5, 0                        ; encoding: [0x80,0x6b,0x85,0xbe]
 0x80,0x6b,0x85,0xbe
+# GFX12: s_ceil_f16 s5, 0                        ; encoding: [0x80,0x6b,0x85,0xbe]
 
-# GFX12: s_ceil_f16 s5, -1                       ; encoding: [0xc1,0x6b,0x85,0xbe]
 0xc1,0x6b,0x85,0xbe
+# GFX12: s_ceil_f16 s5, -1                       ; encoding: [0xc1,0x6b,0x85,0xbe]
 
-# GFX12: s_ceil_f16 s5, 0.5                      ; encoding: [0xf0,0x6b,0x85,0xbe]
 0xf0,0x6b,0x85,0xbe
+# GFX12: s_ceil_f16 s5, 0.5                      ; encoding: [0xf0,0x6b,0x85,0xbe]
 
-# GFX12: s_ceil_f16 s5, -4.0                     ; encoding: [0xf7,0x6b,0x85,0xbe]
 0xf7,0x6b,0x85,0xbe
+# GFX12: s_ceil_f16 s5, -4.0                     ; encoding: [0xf7,0x6b,0x85,0xbe]
 
-# GFX12: s_ceil_f16 s5, 0xfe0b                   ; encoding: [0xff,0x6b,0x85,0xbe,0x0b,0xfe,0x00,0x00]
 0xff,0x6b,0x85,0xbe,0x0b,0xfe,0x00,0x00
+# GFX12: s_ceil_f16 s5, 0xfe0b                   ; encoding: [0xff,0x6b,0x85,0xbe,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_ceil_f16 s5, 0x3456                   ; encoding: [0xff,0x6b,0x85,0xbe,0x56,0x34,0x00,0x00]
 0xff,0x6b,0x85,0xbe,0x56,0x34,0x00,0x00
+# GFX12: s_ceil_f16 s5, 0x3456                   ; encoding: [0xff,0x6b,0x85,0xbe,0x56,0x34,0x00,0x00]
 
-# GFX12: s_trunc_f16 s5, s1                      ; encoding: [0x01,0x6d,0x85,0xbe]
 0x01,0x6d,0x85,0xbe
+# GFX12: s_trunc_f16 s5, s1                      ; encoding: [0x01,0x6d,0x85,0xbe]
 
-# GFX12: s_trunc_f16 s105, s1                    ; encoding: [0x01,0x6d,0xe9,0xbe]
 0x01,0x6d,0xe9,0xbe
+# GFX12: s_trunc_f16 s105, s1                    ; encoding: [0x01,0x6d,0xe9,0xbe]
 
-# GFX12: s_trunc_f16 s5, s105                    ; encoding: [0x69,0x6d,0x85,0xbe]
 0x69,0x6d,0x85,0xbe
+# GFX12: s_trunc_f16 s5, s105                    ; encoding: [0x69,0x6d,0x85,0xbe]
 
-# GFX12: s_trunc_f16 s5, s101                    ; encoding: [0x65,0x6d,0x85,0xbe]
 0x65,0x6d,0x85,0xbe
+# GFX12: s_trunc_f16 s5, s101                    ; encoding: [0x65,0x6d,0x85,0xbe]
 
-# GFX12: s_trunc_f16 s5, vcc_lo                  ; encoding: [0x6a,0x6d,0x85,0xbe]
 0x6a,0x6d,0x85,0xbe
+# GFX12: s_trunc_f16 s5, vcc_lo                  ; encoding: [0x6a,0x6d,0x85,0xbe]
 
-# GFX12: s_trunc_f16 s5, vcc_hi                  ; encoding: [0x6b,0x6d,0x85,0xbe]
 0x6b,0x6d,0x85,0xbe
+# GFX12: s_trunc_f16 s5, vcc_hi                  ; encoding: [0x6b,0x6d,0x85,0xbe]
 
-# GFX12: s_trunc_f16 s5, m0                      ; encoding: [0x7d,0x6d,0x85,0xbe]
 0x7d,0x6d,0x85,0xbe
+# GFX12: s_trunc_f16 s5, m0                      ; encoding: [0x7d,0x6d,0x85,0xbe]
 
-# GFX12: s_trunc_f16 s5, exec_lo                 ; encoding: [0x7e,0x6d,0x85,0xbe]
 0x7e,0x6d,0x85,0xbe
+# GFX12: s_trunc_f16 s5, exec_lo                 ; encoding: [0x7e,0x6d,0x85,0xbe]
 
-# GFX12: s_trunc_f16 s5, exec_hi                 ; encoding: [0x7f,0x6d,0x85,0xbe]
 0x7f,0x6d,0x85,0xbe
+# GFX12: s_trunc_f16 s5, exec_hi                 ; encoding: [0x7f,0x6d,0x85,0xbe]
 
-# GFX12: s_trunc_f16 s5, 0                       ; encoding: [0x80,0x6d,0x85,0xbe]
 0x80,0x6d,0x85,0xbe
+# GFX12: s_trunc_f16 s5, 0                       ; encoding: [0x80,0x6d,0x85,0xbe]
 
-# GFX12: s_trunc_f16 s5, -1                      ; encoding: [0xc1,0x6d,0x85,0xbe]
 0xc1,0x6d,0x85,0xbe
+# GFX12: s_trunc_f16 s5, -1                      ; encoding: [0xc1,0x6d,0x85,0xbe]
 
-# GFX12: s_trunc_f16 s5, 0.5                     ; encoding: [0xf0,0x6d,0x85,0xbe]
 0xf0,0x6d,0x85,0xbe
+# GFX12: s_trunc_f16 s5, 0.5                     ; encoding: [0xf0,0x6d,0x85,0xbe]
 
-# GFX12: s_trunc_f16 s5, -4.0                    ; encoding: [0xf7,0x6d,0x85,0xbe]
 0xf7,0x6d,0x85,0xbe
+# GFX12: s_trunc_f16 s5, -4.0                    ; encoding: [0xf7,0x6d,0x85,0xbe]
 
-# GFX12: s_trunc_f16 s5, 0xfe0b                  ; encoding: [0xff,0x6d,0x85,0xbe,0x0b,0xfe,0x00,0x00]
 0xff,0x6d,0x85,0xbe,0x0b,0xfe,0x00,0x00
+# GFX12: s_trunc_f16 s5, 0xfe0b                  ; encoding: [0xff,0x6d,0x85,0xbe,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_trunc_f16 s5, 0x3456                  ; encoding: [0xff,0x6d,0x85,0xbe,0x56,0x34,0x00,0x00]
 0xff,0x6d,0x85,0xbe,0x56,0x34,0x00,0x00
+# GFX12: s_trunc_f16 s5, 0x3456                  ; encoding: [0xff,0x6d,0x85,0xbe,0x56,0x34,0x00,0x00]
 
-# GFX12: s_rndne_f16 s5, s1                      ; encoding: [0x01,0x6e,0x85,0xbe]
 0x01,0x6e,0x85,0xbe
+# GFX12: s_rndne_f16 s5, s1                      ; encoding: [0x01,0x6e,0x85,0xbe]
 
-# GFX12: s_rndne_f16 s105, s1                    ; encoding: [0x01,0x6e,0xe9,0xbe]
 0x01,0x6e,0xe9,0xbe
+# GFX12: s_rndne_f16 s105, s1                    ; encoding: [0x01,0x6e,0xe9,0xbe]
 
-# GFX12: s_rndne_f16 s5, s105                    ; encoding: [0x69,0x6e,0x85,0xbe]
 0x69,0x6e,0x85,0xbe
+# GFX12: s_rndne_f16 s5, s105                    ; encoding: [0x69,0x6e,0x85,0xbe]
 
-# GFX12: s_rndne_f16 s5, s101                    ; encoding: [0x65,0x6e,0x85,0xbe]
 0x65,0x6e,0x85,0xbe
+# GFX12: s_rndne_f16 s5, s101                    ; encoding: [0x65,0x6e,0x85,0xbe]
 
-# GFX12: s_rndne_f16 s5, vcc_lo                  ; encoding: [0x6a,0x6e,0x85,0xbe]
 0x6a,0x6e,0x85,0xbe
+# GFX12: s_rndne_f16 s5, vcc_lo                  ; encoding: [0x6a,0x6e,0x85,0xbe]
 
-# GFX12: s_rndne_f16 s5, vcc_hi                  ; encoding: [0x6b,0x6e,0x85,0xbe]
 0x6b,0x6e,0x85,0xbe
+# GFX12: s_rndne_f16 s5, vcc_hi                  ; encoding: [0x6b,0x6e,0x85,0xbe]
 
-# GFX12: s_rndne_f16 s5, m0                      ; encoding: [0x7d,0x6e,0x85,0xbe]
 0x7d,0x6e,0x85,0xbe
+# GFX12: s_rndne_f16 s5, m0                      ; encoding: [0x7d,0x6e,0x85,0xbe]
 
-# GFX12: s_rndne_f16 s5, exec_lo                 ; encoding: [0x7e,0x6e,0x85,0xbe]
 0x7e,0x6e,0x85,0xbe
+# GFX12: s_rndne_f16 s5, exec_lo                 ; encoding: [0x7e,0x6e,0x85,0xbe]
 
-# GFX12: s_rndne_f16 s5, exec_hi                 ; encoding: [0x7f,0x6e,0x85,0xbe]
 0x7f,0x6e,0x85,0xbe
+# GFX12: s_rndne_f16 s5, exec_hi                 ; encoding: [0x7f,0x6e,0x85,0xbe]
 
-# GFX12: s_rndne_f16 s5, 0                       ; encoding: [0x80,0x6e,0x85,0xbe]
 0x80,0x6e,0x85,0xbe
+# GFX12: s_rndne_f16 s5, 0                       ; encoding: [0x80,0x6e,0x85,0xbe]
 
-# GFX12: s_rndne_f16 s5, -1                      ; encoding: [0xc1,0x6e,0x85,0xbe]
 0xc1,0x6e,0x85,0xbe
+# GFX12: s_rndne_f16 s5, -1                      ; encoding: [0xc1,0x6e,0x85,0xbe]
 
-# GFX12: s_rndne_f16 s5, 0.5                     ; encoding: [0xf0,0x6e,0x85,0xbe]
 0xf0,0x6e,0x85,0xbe
+# GFX12: s_rndne_f16 s5, 0.5                     ; encoding: [0xf0,0x6e,0x85,0xbe]
 
-# GFX12: s_rndne_f16 s5, -4.0                    ; encoding: [0xf7,0x6e,0x85,0xbe]
 0xf7,0x6e,0x85,0xbe
+# GFX12: s_rndne_f16 s5, -4.0                    ; encoding: [0xf7,0x6e,0x85,0xbe]
 
-# GFX12: s_rndne_f16 s5, 0xfe0b                  ; encoding: [0xff,0x6e,0x85,0xbe,0x0b,0xfe,0x00,0x00]
 0xff,0x6e,0x85,0xbe,0x0b,0xfe,0x00,0x00
+# GFX12: s_rndne_f16 s5, 0xfe0b                  ; encoding: [0xff,0x6e,0x85,0xbe,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_rndne_f16 s5, 0x3456                  ; encoding: [0xff,0x6e,0x85,0xbe,0x56,0x34,0x00,0x00]
 0xff,0x6e,0x85,0xbe,0x56,0x34,0x00,0x00
+# GFX12: s_rndne_f16 s5, 0x3456                  ; encoding: [0xff,0x6e,0x85,0xbe,0x56,0x34,0x00,0x00]
 
-# GFX12: s_barrier_signal -2                      ; encoding: [0xc2,0x4e,0x80,0xbe]
 0xc2,0x4e,0x80,0xbe
+# GFX12: s_barrier_signal -2                     ; encoding: [0xc2,0x4e,0x80,0xbe]
 
-# GFX12: s_barrier_signal -1                      ; encoding: [0xc1,0x4e,0x80,0xbe]
 0xc1,0x4e,0x80,0xbe
+# GFX12: s_barrier_signal -1                     ; encoding: [0xc1,0x4e,0x80,0xbe]
 
-# GFX12: s_barrier_signal m0                      ; encoding: [0x7d,0x4e,0x80,0xbe]
 0x7d,0x4e,0x80,0xbe
+# GFX12: s_barrier_signal m0                     ; encoding: [0x7d,0x4e,0x80,0xbe]
 
-# GFX12: s_barrier_signal_isfirst -2              ; encoding: [0xc2,0x4f,0x80,0xbe]
 0xc2,0x4f,0x80,0xbe
+# GFX12: s_barrier_signal_isfirst -2             ; encoding: [0xc2,0x4f,0x80,0xbe]
 
-# GFX12: s_barrier_signal_isfirst -1              ; encoding: [0xc1,0x4f,0x80,0xbe]
 0xc1,0x4f,0x80,0xbe
+# GFX12: s_barrier_signal_isfirst -1             ; encoding: [0xc1,0x4f,0x80,0xbe]
 
-# GFX12: s_barrier_signal_isfirst m0              ; encoding: [0x7d,0x4f,0x80,0xbe]
 0x7d,0x4f,0x80,0xbe
+# GFX12: s_barrier_signal_isfirst m0             ; encoding: [0x7d,0x4f,0x80,0xbe]
 
-# GFX12: s_get_barrier_state s3, -1               ; encoding: [0xc1,0x50,0x83,0xbe]
 0xc1,0x50,0x83,0xbe
+# GFX12: s_get_barrier_state s3, -1              ; encoding: [0xc1,0x50,0x83,0xbe]
 
-# GFX12: s_get_barrier_state s3, -2               ; encoding: [0xc2,0x50,0x83,0xbe]
 0xc2,0x50,0x83,0xbe
+# GFX12: s_get_barrier_state s3, -2              ; encoding: [0xc2,0x50,0x83,0xbe]
 
-# GFX12: s_get_barrier_state s3, m0               ; encoding: [0x7d,0x50,0x83,0xbe]
 0x7d,0x50,0x83,0xbe
+# GFX12: s_get_barrier_state s3, m0              ; encoding: [0x7d,0x50,0x83,0xbe]
 
-# GFX12: s_abs_i32 exec_hi, s1                   ; encoding: [0x01,0x15,0xff,0xbe]
 0x01,0x15,0xff,0xbe
+# GFX12: s_abs_i32 exec_hi, s1                   ; encoding: [0x01,0x15,0xff,0xbe]
 
-# GFX12: s_abs_i32 exec_lo, s1                   ; encoding: [0x01,0x15,0xfe,0xbe]
 0x01,0x15,0xfe,0xbe
+# GFX12: s_abs_i32 exec_lo, s1                   ; encoding: [0x01,0x15,0xfe,0xbe]
 
-# GFX12: s_abs_i32 m0, s1                        ; encoding: [0x01,0x15,0xfd,0xbe]
 0x01,0x15,0xfd,0xbe
+# GFX12: s_abs_i32 m0, s1                        ; encoding: [0x01,0x15,0xfd,0xbe]
 
-# GFX12: s_abs_i32 s0, 0.5                       ; encoding: [0xf0,0x15,0x80,0xbe]
 0xf0,0x15,0x80,0xbe
+# GFX12: s_abs_i32 s0, 0.5                       ; encoding: [0xf0,0x15,0x80,0xbe]
 
-# GFX12: s_abs_i32 s0, 0                         ; encoding: [0x80,0x15,0x80,0xbe]
 0x80,0x15,0x80,0xbe
+# GFX12: s_abs_i32 s0, 0                         ; encoding: [0x80,0x15,0x80,0xbe]
 
-# GFX12: s_abs_i32 s0, 0x3f717273                ; encoding: [0xff,0x15,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x15,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_abs_i32 s0, 0x3f717273                ; encoding: [0xff,0x15,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_abs_i32 s0, 0xaf123456                ; encoding: [0xff,0x15,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x15,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_abs_i32 s0, 0xaf123456                ; encoding: [0xff,0x15,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_abs_i32 s0, -1                        ; encoding: [0xc1,0x15,0x80,0xbe]
 0xc1,0x15,0x80,0xbe
+# GFX12: s_abs_i32 s0, -1                        ; encoding: [0xc1,0x15,0x80,0xbe]
 
-# GFX12: s_abs_i32 s0, -4.0                      ; encoding: [0xf7,0x15,0x80,0xbe]
 0xf7,0x15,0x80,0xbe
+# GFX12: s_abs_i32 s0, -4.0                      ; encoding: [0xf7,0x15,0x80,0xbe]
 
-# GFX12: s_abs_i32 s0, exec_hi                   ; encoding: [0x7f,0x15,0x80,0xbe]
 0x7f,0x15,0x80,0xbe
+# GFX12: s_abs_i32 s0, exec_hi                   ; encoding: [0x7f,0x15,0x80,0xbe]
 
-# GFX12: s_abs_i32 s0, exec_lo                   ; encoding: [0x7e,0x15,0x80,0xbe]
 0x7e,0x15,0x80,0xbe
+# GFX12: s_abs_i32 s0, exec_lo                   ; encoding: [0x7e,0x15,0x80,0xbe]
 
-# GFX12: s_abs_i32 s0, m0                        ; encoding: [0x7d,0x15,0x80,0xbe]
 0x7d,0x15,0x80,0xbe
+# GFX12: s_abs_i32 s0, m0                        ; encoding: [0x7d,0x15,0x80,0xbe]
 
-# GFX12: s_abs_i32 s0, s104                      ; encoding: [0x68,0x15,0x80,0xbe]
 0x68,0x15,0x80,0xbe
+# GFX12: s_abs_i32 s0, s104                      ; encoding: [0x68,0x15,0x80,0xbe]
 
-# GFX12: s_abs_i32 s0, s1                        ; encoding: [0x01,0x15,0x80,0xbe]
 0x01,0x15,0x80,0xbe
+# GFX12: s_abs_i32 s0, s1                        ; encoding: [0x01,0x15,0x80,0xbe]
 
-# GFX12: s_abs_i32 s0, vcc_hi                    ; encoding: [0x6b,0x15,0x80,0xbe]
 0x6b,0x15,0x80,0xbe
+# GFX12: s_abs_i32 s0, vcc_hi                    ; encoding: [0x6b,0x15,0x80,0xbe]
 
-# GFX12: s_abs_i32 s0, vcc_lo                    ; encoding: [0x6a,0x15,0x80,0xbe]
 0x6a,0x15,0x80,0xbe
+# GFX12: s_abs_i32 s0, vcc_lo                    ; encoding: [0x6a,0x15,0x80,0xbe]
 
-# GFX12: s_abs_i32 s105, s104                    ; encoding: [0x68,0x15,0xe9,0xbe]
 0x68,0x15,0xe9,0xbe
+# GFX12: s_abs_i32 s105, s104                    ; encoding: [0x68,0x15,0xe9,0xbe]
 
-# GFX12: s_abs_i32 s105, s1                      ; encoding: [0x01,0x15,0xe9,0xbe]
 0x01,0x15,0xe9,0xbe
+# GFX12: s_abs_i32 s105, s1                      ; encoding: [0x01,0x15,0xe9,0xbe]
 
-# GFX12: s_abs_i32 vcc_hi, s1                    ; encoding: [0x01,0x15,0xeb,0xbe]
 0x01,0x15,0xeb,0xbe
+# GFX12: s_abs_i32 vcc_hi, s1                    ; encoding: [0x01,0x15,0xeb,0xbe]
 
-# GFX12: s_abs_i32 vcc_lo, s1                    ; encoding: [0x01,0x15,0xea,0xbe]
 0x01,0x15,0xea,0xbe
+# GFX12: s_abs_i32 vcc_lo, s1                    ; encoding: [0x01,0x15,0xea,0xbe]
 
-# GFX12: s_and_not0_saveexec_b32 s0, 0.5         ; encoding: [0xf0,0x2c,0x80,0xbe]
 0xf0,0x2c,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b32 s0, 0.5         ; encoding: [0xf0,0x2c,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b32 s0, 0           ; encoding: [0x80,0x2c,0x80,0xbe]
 0x80,0x2c,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b32 s0, 0           ; encoding: [0x80,0x2c,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b32 s0, 0x3f717273  ; encoding: [0xff,0x2c,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x2c,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_and_not0_saveexec_b32 s0, 0x3f717273  ; encoding: [0xff,0x2c,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_and_not0_saveexec_b32 s0, 0xaf123456  ; encoding: [0xff,0x2c,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x2c,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_and_not0_saveexec_b32 s0, 0xaf123456  ; encoding: [0xff,0x2c,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_and_not0_saveexec_b32 s0, -1          ; encoding: [0xc1,0x2c,0x80,0xbe]
 0xc1,0x2c,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b32 s0, -1          ; encoding: [0xc1,0x2c,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b32 s0, -4.0        ; encoding: [0xf7,0x2c,0x80,0xbe]
 0xf7,0x2c,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b32 s0, -4.0        ; encoding: [0xf7,0x2c,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b32 s0, exec_hi     ; encoding: [0x7f,0x2c,0x80,0xbe]
 0x7f,0x2c,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b32 s0, exec_hi     ; encoding: [0x7f,0x2c,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b32 s0, exec_lo     ; encoding: [0x7e,0x2c,0x80,0xbe]
 0x7e,0x2c,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b32 s0, exec_lo     ; encoding: [0x7e,0x2c,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b32 s0, m0          ; encoding: [0x7d,0x2c,0x80,0xbe]
 0x7d,0x2c,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b32 s0, m0          ; encoding: [0x7d,0x2c,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b32 s0, s104        ; encoding: [0x68,0x2c,0x80,0xbe]
 0x68,0x2c,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b32 s0, s104        ; encoding: [0x68,0x2c,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b32 s0, s1          ; encoding: [0x01,0x2c,0x80,0xbe]
 0x01,0x2c,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b32 s0, s1          ; encoding: [0x01,0x2c,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b32 s0, vcc_hi      ; encoding: [0x6b,0x2c,0x80,0xbe]
 0x6b,0x2c,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b32 s0, vcc_hi      ; encoding: [0x6b,0x2c,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b32 s0, vcc_lo      ; encoding: [0x6a,0x2c,0x80,0xbe]
 0x6a,0x2c,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b32 s0, vcc_lo      ; encoding: [0x6a,0x2c,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b32 s105, s104      ; encoding: [0x68,0x2c,0xe9,0xbe]
 0x68,0x2c,0xe9,0xbe
+# GFX12: s_and_not0_saveexec_b32 s105, s104      ; encoding: [0x68,0x2c,0xe9,0xbe]
 
-# GFX12: s_and_not0_saveexec_b32 s105, s1        ; encoding: [0x01,0x2c,0xe9,0xbe]
 0x01,0x2c,0xe9,0xbe
+# GFX12: s_and_not0_saveexec_b32 s105, s1        ; encoding: [0x01,0x2c,0xe9,0xbe]
 
-# GFX12: s_and_not0_saveexec_b32 vcc_hi, s1      ; encoding: [0x01,0x2c,0xeb,0xbe]
 0x01,0x2c,0xeb,0xbe
+# GFX12: s_and_not0_saveexec_b32 vcc_hi, s1      ; encoding: [0x01,0x2c,0xeb,0xbe]
 
-# GFX12: s_and_not0_saveexec_b32 vcc_lo, s1      ; encoding: [0x01,0x2c,0xea,0xbe]
 0x01,0x2c,0xea,0xbe
+# GFX12: s_and_not0_saveexec_b32 vcc_lo, s1      ; encoding: [0x01,0x2c,0xea,0xbe]
 
-# GFX12: s_and_not0_saveexec_b64 s[0:1], 0.5     ; encoding: [0xf0,0x2d,0x80,0xbe]
 0xf0,0x2d,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b64 s[0:1], 0.5     ; encoding: [0xf0,0x2d,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b64 s[0:1], 0       ; encoding: [0x80,0x2d,0x80,0xbe]
 0x80,0x2d,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b64 s[0:1], 0       ; encoding: [0x80,0x2d,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b64 s[0:1], 0x3f717273 ; encoding: [0xff,0x2d,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x2d,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_and_not0_saveexec_b64 s[0:1], 0x3f717273 ; encoding: [0xff,0x2d,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_and_not0_saveexec_b64 s[0:1], 0xaf123456 ; encoding: [0xff,0x2d,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x2d,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_and_not0_saveexec_b64 s[0:1], 0xaf123456 ; encoding: [0xff,0x2d,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_and_not0_saveexec_b64 s[0:1], -1      ; encoding: [0xc1,0x2d,0x80,0xbe]
 0xc1,0x2d,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b64 s[0:1], -1      ; encoding: [0xc1,0x2d,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b64 s[0:1], -4.0    ; encoding: [0xf7,0x2d,0x80,0xbe]
 0xf7,0x2d,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b64 s[0:1], -4.0    ; encoding: [0xf7,0x2d,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b64 s[0:1], exec    ; encoding: [0x7e,0x2d,0x80,0xbe]
 0x7e,0x2d,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b64 s[0:1], exec    ; encoding: [0x7e,0x2d,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b64 s[0:1], s[102:103] ; encoding: [0x66,0x2d,0x80,0xbe]
 0x66,0x2d,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b64 s[0:1], s[102:103] ; encoding: [0x66,0x2d,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b64 s[0:1], s[2:3]  ; encoding: [0x02,0x2d,0x80,0xbe]
 0x02,0x2d,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b64 s[0:1], s[2:3]  ; encoding: [0x02,0x2d,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b64 s[0:1], vcc     ; encoding: [0x6a,0x2d,0x80,0xbe]
 0x6a,0x2d,0x80,0xbe
+# GFX12: s_and_not0_saveexec_b64 s[0:1], vcc     ; encoding: [0x6a,0x2d,0x80,0xbe]
 
-# GFX12: s_and_not0_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x2d,0xe8,0xbe]
 0x66,0x2d,0xe8,0xbe
+# GFX12: s_and_not0_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x2d,0xe8,0xbe]
 
-# GFX12: s_and_not0_saveexec_b64 s[104:105], s[2:3] ; encoding: [0x02,0x2d,0xe8,0xbe]
 0x02,0x2d,0xe8,0xbe
+# GFX12: s_and_not0_saveexec_b64 s[104:105], s[2:3] ; encoding: [0x02,0x2d,0xe8,0xbe]
 
-# GFX12: s_and_not0_saveexec_b64 vcc, s[2:3]     ; encoding: [0x02,0x2d,0xea,0xbe]
 0x02,0x2d,0xea,0xbe
+# GFX12: s_and_not0_saveexec_b64 vcc, s[2:3]     ; encoding: [0x02,0x2d,0xea,0xbe]
 
-# GFX12: s_and_not0_wrexec_b32 s0, 0.5           ; encoding: [0xf0,0x34,0x80,0xbe]
 0xf0,0x34,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b32 s0, 0.5           ; encoding: [0xf0,0x34,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b32 s0, 0             ; encoding: [0x80,0x34,0x80,0xbe]
 0x80,0x34,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b32 s0, 0             ; encoding: [0x80,0x34,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b32 s0, 0x3f717273    ; encoding: [0xff,0x34,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x34,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_and_not0_wrexec_b32 s0, 0x3f717273    ; encoding: [0xff,0x34,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_and_not0_wrexec_b32 s0, 0xaf123456    ; encoding: [0xff,0x34,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x34,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_and_not0_wrexec_b32 s0, 0xaf123456    ; encoding: [0xff,0x34,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_and_not0_wrexec_b32 s0, -1            ; encoding: [0xc1,0x34,0x80,0xbe]
 0xc1,0x34,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b32 s0, -1            ; encoding: [0xc1,0x34,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b32 s0, -4.0          ; encoding: [0xf7,0x34,0x80,0xbe]
 0xf7,0x34,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b32 s0, -4.0          ; encoding: [0xf7,0x34,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b32 s0, exec_hi       ; encoding: [0x7f,0x34,0x80,0xbe]
 0x7f,0x34,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b32 s0, exec_hi       ; encoding: [0x7f,0x34,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b32 s0, exec_lo       ; encoding: [0x7e,0x34,0x80,0xbe]
 0x7e,0x34,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b32 s0, exec_lo       ; encoding: [0x7e,0x34,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b32 s0, m0            ; encoding: [0x7d,0x34,0x80,0xbe]
 0x7d,0x34,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b32 s0, m0            ; encoding: [0x7d,0x34,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b32 s0, s104          ; encoding: [0x68,0x34,0x80,0xbe]
 0x68,0x34,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b32 s0, s104          ; encoding: [0x68,0x34,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b32 s0, s1            ; encoding: [0x01,0x34,0x80,0xbe]
 0x01,0x34,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b32 s0, s1            ; encoding: [0x01,0x34,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b32 s0, vcc_hi        ; encoding: [0x6b,0x34,0x80,0xbe]
 0x6b,0x34,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b32 s0, vcc_hi        ; encoding: [0x6b,0x34,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b32 s0, vcc_lo        ; encoding: [0x6a,0x34,0x80,0xbe]
 0x6a,0x34,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b32 s0, vcc_lo        ; encoding: [0x6a,0x34,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b32 s105, s104        ; encoding: [0x68,0x34,0xe9,0xbe]
 0x68,0x34,0xe9,0xbe
+# GFX12: s_and_not0_wrexec_b32 s105, s104        ; encoding: [0x68,0x34,0xe9,0xbe]
 
-# GFX12: s_and_not0_wrexec_b32 s105, s1          ; encoding: [0x01,0x34,0xe9,0xbe]
 0x01,0x34,0xe9,0xbe
+# GFX12: s_and_not0_wrexec_b32 s105, s1          ; encoding: [0x01,0x34,0xe9,0xbe]
 
-# GFX12: s_and_not0_wrexec_b32 vcc_hi, s1        ; encoding: [0x01,0x34,0xeb,0xbe]
 0x01,0x34,0xeb,0xbe
+# GFX12: s_and_not0_wrexec_b32 vcc_hi, s1        ; encoding: [0x01,0x34,0xeb,0xbe]
 
-# GFX12: s_and_not0_wrexec_b32 vcc_lo, s1        ; encoding: [0x01,0x34,0xea,0xbe]
 0x01,0x34,0xea,0xbe
+# GFX12: s_and_not0_wrexec_b32 vcc_lo, s1        ; encoding: [0x01,0x34,0xea,0xbe]
 
-# GFX12: s_and_not0_wrexec_b64 s[0:1], 0.5       ; encoding: [0xf0,0x35,0x80,0xbe]
 0xf0,0x35,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b64 s[0:1], 0.5       ; encoding: [0xf0,0x35,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b64 s[0:1], 0         ; encoding: [0x80,0x35,0x80,0xbe]
 0x80,0x35,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b64 s[0:1], 0         ; encoding: [0x80,0x35,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b64 s[0:1], 0x3f717273 ; encoding: [0xff,0x35,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x35,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_and_not0_wrexec_b64 s[0:1], 0x3f717273 ; encoding: [0xff,0x35,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_and_not0_wrexec_b64 s[0:1], 0xaf123456 ; encoding: [0xff,0x35,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x35,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_and_not0_wrexec_b64 s[0:1], 0xaf123456 ; encoding: [0xff,0x35,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_and_not0_wrexec_b64 s[0:1], -1        ; encoding: [0xc1,0x35,0x80,0xbe]
 0xc1,0x35,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b64 s[0:1], -1        ; encoding: [0xc1,0x35,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b64 s[0:1], -4.0      ; encoding: [0xf7,0x35,0x80,0xbe]
 0xf7,0x35,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b64 s[0:1], -4.0      ; encoding: [0xf7,0x35,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b64 s[0:1], exec      ; encoding: [0x7e,0x35,0x80,0xbe]
 0x7e,0x35,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b64 s[0:1], exec      ; encoding: [0x7e,0x35,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b64 s[0:1], s[102:103] ; encoding: [0x66,0x35,0x80,0xbe]
 0x66,0x35,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b64 s[0:1], s[102:103] ; encoding: [0x66,0x35,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b64 s[0:1], s[2:3]    ; encoding: [0x02,0x35,0x80,0xbe]
 0x02,0x35,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b64 s[0:1], s[2:3]    ; encoding: [0x02,0x35,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b64 s[0:1], vcc       ; encoding: [0x6a,0x35,0x80,0xbe]
 0x6a,0x35,0x80,0xbe
+# GFX12: s_and_not0_wrexec_b64 s[0:1], vcc       ; encoding: [0x6a,0x35,0x80,0xbe]
 
-# GFX12: s_and_not0_wrexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x35,0xe8,0xbe]
 0x66,0x35,0xe8,0xbe
+# GFX12: s_and_not0_wrexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x35,0xe8,0xbe]
 
-# GFX12: s_and_not0_wrexec_b64 s[104:105], s[2:3] ; encoding: [0x02,0x35,0xe8,0xbe]
 0x02,0x35,0xe8,0xbe
+# GFX12: s_and_not0_wrexec_b64 s[104:105], s[2:3] ; encoding: [0x02,0x35,0xe8,0xbe]
 
-# GFX12: s_and_not0_wrexec_b64 vcc, s[2:3]       ; encoding: [0x02,0x35,0xea,0xbe]
 0x02,0x35,0xea,0xbe
+# GFX12: s_and_not0_wrexec_b64 vcc, s[2:3]       ; encoding: [0x02,0x35,0xea,0xbe]
 
-# GFX12: s_and_not1_saveexec_b32 s0, 0.5         ; encoding: [0xf0,0x30,0x80,0xbe]
 0xf0,0x30,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b32 s0, 0.5         ; encoding: [0xf0,0x30,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b32 s0, 0           ; encoding: [0x80,0x30,0x80,0xbe]
 0x80,0x30,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b32 s0, 0           ; encoding: [0x80,0x30,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b32 s0, 0x3f717273  ; encoding: [0xff,0x30,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x30,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_and_not1_saveexec_b32 s0, 0x3f717273  ; encoding: [0xff,0x30,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_and_not1_saveexec_b32 s0, 0xaf123456  ; encoding: [0xff,0x30,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x30,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_and_not1_saveexec_b32 s0, 0xaf123456  ; encoding: [0xff,0x30,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_and_not1_saveexec_b32 s0, -1          ; encoding: [0xc1,0x30,0x80,0xbe]
 0xc1,0x30,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b32 s0, -1          ; encoding: [0xc1,0x30,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b32 s0, -4.0        ; encoding: [0xf7,0x30,0x80,0xbe]
 0xf7,0x30,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b32 s0, -4.0        ; encoding: [0xf7,0x30,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b32 s0, exec_hi     ; encoding: [0x7f,0x30,0x80,0xbe]
 0x7f,0x30,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b32 s0, exec_hi     ; encoding: [0x7f,0x30,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b32 s0, exec_lo     ; encoding: [0x7e,0x30,0x80,0xbe]
 0x7e,0x30,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b32 s0, exec_lo     ; encoding: [0x7e,0x30,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b32 s0, m0          ; encoding: [0x7d,0x30,0x80,0xbe]
 0x7d,0x30,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b32 s0, m0          ; encoding: [0x7d,0x30,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b32 s0, s104        ; encoding: [0x68,0x30,0x80,0xbe]
 0x68,0x30,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b32 s0, s104        ; encoding: [0x68,0x30,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b32 s0, s1          ; encoding: [0x01,0x30,0x80,0xbe]
 0x01,0x30,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b32 s0, s1          ; encoding: [0x01,0x30,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b32 s0, vcc_hi      ; encoding: [0x6b,0x30,0x80,0xbe]
 0x6b,0x30,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b32 s0, vcc_hi      ; encoding: [0x6b,0x30,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b32 s0, vcc_lo      ; encoding: [0x6a,0x30,0x80,0xbe]
 0x6a,0x30,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b32 s0, vcc_lo      ; encoding: [0x6a,0x30,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b32 s105, s104      ; encoding: [0x68,0x30,0xe9,0xbe]
 0x68,0x30,0xe9,0xbe
+# GFX12: s_and_not1_saveexec_b32 s105, s104      ; encoding: [0x68,0x30,0xe9,0xbe]
 
-# GFX12: s_and_not1_saveexec_b32 s105, s1        ; encoding: [0x01,0x30,0xe9,0xbe]
 0x01,0x30,0xe9,0xbe
+# GFX12: s_and_not1_saveexec_b32 s105, s1        ; encoding: [0x01,0x30,0xe9,0xbe]
 
-# GFX12: s_and_not1_saveexec_b32 vcc_hi, s1      ; encoding: [0x01,0x30,0xeb,0xbe]
 0x01,0x30,0xeb,0xbe
+# GFX12: s_and_not1_saveexec_b32 vcc_hi, s1      ; encoding: [0x01,0x30,0xeb,0xbe]
 
-# GFX12: s_and_not1_saveexec_b32 vcc_lo, s1      ; encoding: [0x01,0x30,0xea,0xbe]
 0x01,0x30,0xea,0xbe
+# GFX12: s_and_not1_saveexec_b32 vcc_lo, s1      ; encoding: [0x01,0x30,0xea,0xbe]
 
-# GFX12: s_and_not1_saveexec_b64 s[0:1], 0.5     ; encoding: [0xf0,0x31,0x80,0xbe]
 0xf0,0x31,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b64 s[0:1], 0.5     ; encoding: [0xf0,0x31,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b64 s[0:1], 0       ; encoding: [0x80,0x31,0x80,0xbe]
 0x80,0x31,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b64 s[0:1], 0       ; encoding: [0x80,0x31,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b64 s[0:1], 0x3f717273 ; encoding: [0xff,0x31,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x31,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_and_not1_saveexec_b64 s[0:1], 0x3f717273 ; encoding: [0xff,0x31,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_and_not1_saveexec_b64 s[0:1], 0xaf123456 ; encoding: [0xff,0x31,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x31,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_and_not1_saveexec_b64 s[0:1], 0xaf123456 ; encoding: [0xff,0x31,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_and_not1_saveexec_b64 s[0:1], -1      ; encoding: [0xc1,0x31,0x80,0xbe]
 0xc1,0x31,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b64 s[0:1], -1      ; encoding: [0xc1,0x31,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b64 s[0:1], -4.0    ; encoding: [0xf7,0x31,0x80,0xbe]
 0xf7,0x31,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b64 s[0:1], -4.0    ; encoding: [0xf7,0x31,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b64 s[0:1], exec    ; encoding: [0x7e,0x31,0x80,0xbe]
 0x7e,0x31,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b64 s[0:1], exec    ; encoding: [0x7e,0x31,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b64 s[0:1], s[102:103] ; encoding: [0x66,0x31,0x80,0xbe]
 0x66,0x31,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b64 s[0:1], s[102:103] ; encoding: [0x66,0x31,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b64 s[0:1], s[2:3]  ; encoding: [0x02,0x31,0x80,0xbe]
 0x02,0x31,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b64 s[0:1], s[2:3]  ; encoding: [0x02,0x31,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b64 s[0:1], vcc     ; encoding: [0x6a,0x31,0x80,0xbe]
 0x6a,0x31,0x80,0xbe
+# GFX12: s_and_not1_saveexec_b64 s[0:1], vcc     ; encoding: [0x6a,0x31,0x80,0xbe]
 
-# GFX12: s_and_not1_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x31,0xe8,0xbe]
 0x66,0x31,0xe8,0xbe
+# GFX12: s_and_not1_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x31,0xe8,0xbe]
 
-# GFX12: s_and_not1_saveexec_b64 s[104:105], s[2:3] ; encoding: [0x02,0x31,0xe8,0xbe]
 0x02,0x31,0xe8,0xbe
+# GFX12: s_and_not1_saveexec_b64 s[104:105], s[2:3] ; encoding: [0x02,0x31,0xe8,0xbe]
 
-# GFX12: s_and_not1_saveexec_b64 vcc, s[2:3]     ; encoding: [0x02,0x31,0xea,0xbe]
 0x02,0x31,0xea,0xbe
+# GFX12: s_and_not1_saveexec_b64 vcc, s[2:3]     ; encoding: [0x02,0x31,0xea,0xbe]
 
-# GFX12: s_and_not1_wrexec_b32 s0, 0.5           ; encoding: [0xf0,0x36,0x80,0xbe]
 0xf0,0x36,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b32 s0, 0.5           ; encoding: [0xf0,0x36,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b32 s0, 0             ; encoding: [0x80,0x36,0x80,0xbe]
 0x80,0x36,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b32 s0, 0             ; encoding: [0x80,0x36,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b32 s0, 0x3f717273    ; encoding: [0xff,0x36,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x36,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_and_not1_wrexec_b32 s0, 0x3f717273    ; encoding: [0xff,0x36,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_and_not1_wrexec_b32 s0, 0xaf123456    ; encoding: [0xff,0x36,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x36,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_and_not1_wrexec_b32 s0, 0xaf123456    ; encoding: [0xff,0x36,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_and_not1_wrexec_b32 s0, -1            ; encoding: [0xc1,0x36,0x80,0xbe]
 0xc1,0x36,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b32 s0, -1            ; encoding: [0xc1,0x36,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b32 s0, -4.0          ; encoding: [0xf7,0x36,0x80,0xbe]
 0xf7,0x36,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b32 s0, -4.0          ; encoding: [0xf7,0x36,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b32 s0, exec_hi       ; encoding: [0x7f,0x36,0x80,0xbe]
 0x7f,0x36,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b32 s0, exec_hi       ; encoding: [0x7f,0x36,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b32 s0, exec_lo       ; encoding: [0x7e,0x36,0x80,0xbe]
 0x7e,0x36,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b32 s0, exec_lo       ; encoding: [0x7e,0x36,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b32 s0, m0            ; encoding: [0x7d,0x36,0x80,0xbe]
 0x7d,0x36,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b32 s0, m0            ; encoding: [0x7d,0x36,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b32 s0, s104          ; encoding: [0x68,0x36,0x80,0xbe]
 0x68,0x36,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b32 s0, s104          ; encoding: [0x68,0x36,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b32 s0, s1            ; encoding: [0x01,0x36,0x80,0xbe]
 0x01,0x36,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b32 s0, s1            ; encoding: [0x01,0x36,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b32 s0, vcc_hi        ; encoding: [0x6b,0x36,0x80,0xbe]
 0x6b,0x36,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b32 s0, vcc_hi        ; encoding: [0x6b,0x36,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b32 s0, vcc_lo        ; encoding: [0x6a,0x36,0x80,0xbe]
 0x6a,0x36,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b32 s0, vcc_lo        ; encoding: [0x6a,0x36,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b32 s105, s104        ; encoding: [0x68,0x36,0xe9,0xbe]
 0x68,0x36,0xe9,0xbe
+# GFX12: s_and_not1_wrexec_b32 s105, s104        ; encoding: [0x68,0x36,0xe9,0xbe]
 
-# GFX12: s_and_not1_wrexec_b32 s105, s1          ; encoding: [0x01,0x36,0xe9,0xbe]
 0x01,0x36,0xe9,0xbe
+# GFX12: s_and_not1_wrexec_b32 s105, s1          ; encoding: [0x01,0x36,0xe9,0xbe]
 
-# GFX12: s_and_not1_wrexec_b32 vcc_hi, s1        ; encoding: [0x01,0x36,0xeb,0xbe]
 0x01,0x36,0xeb,0xbe
+# GFX12: s_and_not1_wrexec_b32 vcc_hi, s1        ; encoding: [0x01,0x36,0xeb,0xbe]
 
-# GFX12: s_and_not1_wrexec_b32 vcc_lo, s1        ; encoding: [0x01,0x36,0xea,0xbe]
 0x01,0x36,0xea,0xbe
+# GFX12: s_and_not1_wrexec_b32 vcc_lo, s1        ; encoding: [0x01,0x36,0xea,0xbe]
 
-# GFX12: s_and_not1_wrexec_b64 s[0:1], 0.5       ; encoding: [0xf0,0x37,0x80,0xbe]
 0xf0,0x37,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b64 s[0:1], 0.5       ; encoding: [0xf0,0x37,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b64 s[0:1], 0         ; encoding: [0x80,0x37,0x80,0xbe]
 0x80,0x37,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b64 s[0:1], 0         ; encoding: [0x80,0x37,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b64 s[0:1], 0x3f717273 ; encoding: [0xff,0x37,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x37,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_and_not1_wrexec_b64 s[0:1], 0x3f717273 ; encoding: [0xff,0x37,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_and_not1_wrexec_b64 s[0:1], 0xaf123456 ; encoding: [0xff,0x37,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x37,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_and_not1_wrexec_b64 s[0:1], 0xaf123456 ; encoding: [0xff,0x37,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_and_not1_wrexec_b64 s[0:1], -1        ; encoding: [0xc1,0x37,0x80,0xbe]
 0xc1,0x37,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b64 s[0:1], -1        ; encoding: [0xc1,0x37,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b64 s[0:1], -4.0      ; encoding: [0xf7,0x37,0x80,0xbe]
 0xf7,0x37,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b64 s[0:1], -4.0      ; encoding: [0xf7,0x37,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b64 s[0:1], exec      ; encoding: [0x7e,0x37,0x80,0xbe]
 0x7e,0x37,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b64 s[0:1], exec      ; encoding: [0x7e,0x37,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b64 s[0:1], s[102:103] ; encoding: [0x66,0x37,0x80,0xbe]
 0x66,0x37,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b64 s[0:1], s[102:103] ; encoding: [0x66,0x37,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b64 s[0:1], s[2:3]    ; encoding: [0x02,0x37,0x80,0xbe]
 0x02,0x37,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b64 s[0:1], s[2:3]    ; encoding: [0x02,0x37,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b64 s[0:1], vcc       ; encoding: [0x6a,0x37,0x80,0xbe]
 0x6a,0x37,0x80,0xbe
+# GFX12: s_and_not1_wrexec_b64 s[0:1], vcc       ; encoding: [0x6a,0x37,0x80,0xbe]
 
-# GFX12: s_and_not1_wrexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x37,0xe8,0xbe]
 0x66,0x37,0xe8,0xbe
+# GFX12: s_and_not1_wrexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x37,0xe8,0xbe]
 
-# GFX12: s_and_not1_wrexec_b64 s[104:105], s[2:3] ; encoding: [0x02,0x37,0xe8,0xbe]
 0x02,0x37,0xe8,0xbe
+# GFX12: s_and_not1_wrexec_b64 s[104:105], s[2:3] ; encoding: [0x02,0x37,0xe8,0xbe]
 
-# GFX12: s_and_not1_wrexec_b64 vcc, s[2:3]       ; encoding: [0x02,0x37,0xea,0xbe]
 0x02,0x37,0xea,0xbe
+# GFX12: s_and_not1_wrexec_b64 vcc, s[2:3]       ; encoding: [0x02,0x37,0xea,0xbe]
 
-# GFX12: s_and_saveexec_b32 s0, 0.5              ; encoding: [0xf0,0x20,0x80,0xbe]
 0xf0,0x20,0x80,0xbe
+# GFX12: s_and_saveexec_b32 s0, 0.5              ; encoding: [0xf0,0x20,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b32 s0, 0                ; encoding: [0x80,0x20,0x80,0xbe]
 0x80,0x20,0x80,0xbe
+# GFX12: s_and_saveexec_b32 s0, 0                ; encoding: [0x80,0x20,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b32 s0, 0x3f717273       ; encoding: [0xff,0x20,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x20,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_and_saveexec_b32 s0, 0x3f717273       ; encoding: [0xff,0x20,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_and_saveexec_b32 s0, 0xaf123456       ; encoding: [0xff,0x20,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x20,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_and_saveexec_b32 s0, 0xaf123456       ; encoding: [0xff,0x20,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_and_saveexec_b32 s0, -1               ; encoding: [0xc1,0x20,0x80,0xbe]
 0xc1,0x20,0x80,0xbe
+# GFX12: s_and_saveexec_b32 s0, -1               ; encoding: [0xc1,0x20,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b32 s0, -4.0             ; encoding: [0xf7,0x20,0x80,0xbe]
 0xf7,0x20,0x80,0xbe
+# GFX12: s_and_saveexec_b32 s0, -4.0             ; encoding: [0xf7,0x20,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b32 s0, exec_hi          ; encoding: [0x7f,0x20,0x80,0xbe]
 0x7f,0x20,0x80,0xbe
+# GFX12: s_and_saveexec_b32 s0, exec_hi          ; encoding: [0x7f,0x20,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b32 s0, exec_lo          ; encoding: [0x7e,0x20,0x80,0xbe]
 0x7e,0x20,0x80,0xbe
+# GFX12: s_and_saveexec_b32 s0, exec_lo          ; encoding: [0x7e,0x20,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b32 s0, m0               ; encoding: [0x7d,0x20,0x80,0xbe]
 0x7d,0x20,0x80,0xbe
+# GFX12: s_and_saveexec_b32 s0, m0               ; encoding: [0x7d,0x20,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b32 s0, s104             ; encoding: [0x68,0x20,0x80,0xbe]
 0x68,0x20,0x80,0xbe
+# GFX12: s_and_saveexec_b32 s0, s104             ; encoding: [0x68,0x20,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b32 s0, s1               ; encoding: [0x01,0x20,0x80,0xbe]
 0x01,0x20,0x80,0xbe
+# GFX12: s_and_saveexec_b32 s0, s1               ; encoding: [0x01,0x20,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b32 s0, vcc_hi           ; encoding: [0x6b,0x20,0x80,0xbe]
 0x6b,0x20,0x80,0xbe
+# GFX12: s_and_saveexec_b32 s0, vcc_hi           ; encoding: [0x6b,0x20,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b32 s0, vcc_lo           ; encoding: [0x6a,0x20,0x80,0xbe]
 0x6a,0x20,0x80,0xbe
+# GFX12: s_and_saveexec_b32 s0, vcc_lo           ; encoding: [0x6a,0x20,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b32 s105, s104           ; encoding: [0x68,0x20,0xe9,0xbe]
 0x68,0x20,0xe9,0xbe
+# GFX12: s_and_saveexec_b32 s105, s104           ; encoding: [0x68,0x20,0xe9,0xbe]
 
-# GFX12: s_and_saveexec_b32 s105, s1             ; encoding: [0x01,0x20,0xe9,0xbe]
 0x01,0x20,0xe9,0xbe
+# GFX12: s_and_saveexec_b32 s105, s1             ; encoding: [0x01,0x20,0xe9,0xbe]
 
-# GFX12: s_and_saveexec_b32 vcc_hi, s1           ; encoding: [0x01,0x20,0xeb,0xbe]
 0x01,0x20,0xeb,0xbe
+# GFX12: s_and_saveexec_b32 vcc_hi, s1           ; encoding: [0x01,0x20,0xeb,0xbe]
 
-# GFX12: s_and_saveexec_b32 vcc_lo, s1           ; encoding: [0x01,0x20,0xea,0xbe]
 0x01,0x20,0xea,0xbe
+# GFX12: s_and_saveexec_b32 vcc_lo, s1           ; encoding: [0x01,0x20,0xea,0xbe]
 
-# GFX12: s_and_saveexec_b64 s[0:1], 0.5          ; encoding: [0xf0,0x21,0x80,0xbe]
 0xf0,0x21,0x80,0xbe
+# GFX12: s_and_saveexec_b64 s[0:1], 0.5          ; encoding: [0xf0,0x21,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b64 s[0:1], 0            ; encoding: [0x80,0x21,0x80,0xbe]
 0x80,0x21,0x80,0xbe
+# GFX12: s_and_saveexec_b64 s[0:1], 0            ; encoding: [0x80,0x21,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b64 s[0:1], 0x3f717273   ; encoding: [0xff,0x21,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x21,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_and_saveexec_b64 s[0:1], 0x3f717273   ; encoding: [0xff,0x21,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_and_saveexec_b64 s[0:1], 0xaf123456   ; encoding: [0xff,0x21,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x21,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_and_saveexec_b64 s[0:1], 0xaf123456   ; encoding: [0xff,0x21,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_and_saveexec_b64 s[0:1], -1           ; encoding: [0xc1,0x21,0x80,0xbe]
 0xc1,0x21,0x80,0xbe
+# GFX12: s_and_saveexec_b64 s[0:1], -1           ; encoding: [0xc1,0x21,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b64 s[0:1], -4.0         ; encoding: [0xf7,0x21,0x80,0xbe]
 0xf7,0x21,0x80,0xbe
+# GFX12: s_and_saveexec_b64 s[0:1], -4.0         ; encoding: [0xf7,0x21,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b64 s[0:1], exec         ; encoding: [0x7e,0x21,0x80,0xbe]
 0x7e,0x21,0x80,0xbe
+# GFX12: s_and_saveexec_b64 s[0:1], exec         ; encoding: [0x7e,0x21,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b64 s[0:1], s[102:103]   ; encoding: [0x66,0x21,0x80,0xbe]
 0x66,0x21,0x80,0xbe
+# GFX12: s_and_saveexec_b64 s[0:1], s[102:103]   ; encoding: [0x66,0x21,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b64 s[0:1], s[2:3]       ; encoding: [0x02,0x21,0x80,0xbe]
 0x02,0x21,0x80,0xbe
+# GFX12: s_and_saveexec_b64 s[0:1], s[2:3]       ; encoding: [0x02,0x21,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b64 s[0:1], vcc          ; encoding: [0x6a,0x21,0x80,0xbe]
 0x6a,0x21,0x80,0xbe
+# GFX12: s_and_saveexec_b64 s[0:1], vcc          ; encoding: [0x6a,0x21,0x80,0xbe]
 
-# GFX12: s_and_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x21,0xe8,0xbe]
 0x66,0x21,0xe8,0xbe
+# GFX12: s_and_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x21,0xe8,0xbe]
 
-# GFX12: s_and_saveexec_b64 s[104:105], s[2:3]   ; encoding: [0x02,0x21,0xe8,0xbe]
 0x02,0x21,0xe8,0xbe
+# GFX12: s_and_saveexec_b64 s[104:105], s[2:3]   ; encoding: [0x02,0x21,0xe8,0xbe]
 
-# GFX12: s_and_saveexec_b64 vcc, s[2:3]          ; encoding: [0x02,0x21,0xea,0xbe]
 0x02,0x21,0xea,0xbe
+# GFX12: s_and_saveexec_b64 vcc, s[2:3]          ; encoding: [0x02,0x21,0xea,0xbe]
 
-# GFX12: s_bcnt0_i32_b32 exec_hi, s1             ; encoding: [0x01,0x16,0xff,0xbe]
 0x01,0x16,0xff,0xbe
+# GFX12: s_bcnt0_i32_b32 exec_hi, s1             ; encoding: [0x01,0x16,0xff,0xbe]
 
-# GFX12: s_bcnt0_i32_b32 exec_lo, s1             ; encoding: [0x01,0x16,0xfe,0xbe]
 0x01,0x16,0xfe,0xbe
+# GFX12: s_bcnt0_i32_b32 exec_lo, s1             ; encoding: [0x01,0x16,0xfe,0xbe]
 
-# GFX12: s_bcnt0_i32_b32 m0, s1                  ; encoding: [0x01,0x16,0xfd,0xbe]
 0x01,0x16,0xfd,0xbe
+# GFX12: s_bcnt0_i32_b32 m0, s1                  ; encoding: [0x01,0x16,0xfd,0xbe]
 
-# GFX12: s_bcnt0_i32_b32 s0, 0.5                 ; encoding: [0xf0,0x16,0x80,0xbe]
 0xf0,0x16,0x80,0xbe
+# GFX12: s_bcnt0_i32_b32 s0, 0.5                 ; encoding: [0xf0,0x16,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b32 s0, 0                   ; encoding: [0x80,0x16,0x80,0xbe]
 0x80,0x16,0x80,0xbe
+# GFX12: s_bcnt0_i32_b32 s0, 0                   ; encoding: [0x80,0x16,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b32 s0, 0x3f717273          ; encoding: [0xff,0x16,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x16,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_bcnt0_i32_b32 s0, 0x3f717273          ; encoding: [0xff,0x16,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bcnt0_i32_b32 s0, 0xaf123456          ; encoding: [0xff,0x16,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x16,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_bcnt0_i32_b32 s0, 0xaf123456          ; encoding: [0xff,0x16,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bcnt0_i32_b32 s0, -1                  ; encoding: [0xc1,0x16,0x80,0xbe]
 0xc1,0x16,0x80,0xbe
+# GFX12: s_bcnt0_i32_b32 s0, -1                  ; encoding: [0xc1,0x16,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b32 s0, -4.0                ; encoding: [0xf7,0x16,0x80,0xbe]
 0xf7,0x16,0x80,0xbe
+# GFX12: s_bcnt0_i32_b32 s0, -4.0                ; encoding: [0xf7,0x16,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b32 s0, exec_hi             ; encoding: [0x7f,0x16,0x80,0xbe]
 0x7f,0x16,0x80,0xbe
+# GFX12: s_bcnt0_i32_b32 s0, exec_hi             ; encoding: [0x7f,0x16,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b32 s0, exec_lo             ; encoding: [0x7e,0x16,0x80,0xbe]
 0x7e,0x16,0x80,0xbe
+# GFX12: s_bcnt0_i32_b32 s0, exec_lo             ; encoding: [0x7e,0x16,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b32 s0, m0                  ; encoding: [0x7d,0x16,0x80,0xbe]
 0x7d,0x16,0x80,0xbe
+# GFX12: s_bcnt0_i32_b32 s0, m0                  ; encoding: [0x7d,0x16,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b32 s0, s104                ; encoding: [0x68,0x16,0x80,0xbe]
 0x68,0x16,0x80,0xbe
+# GFX12: s_bcnt0_i32_b32 s0, s104                ; encoding: [0x68,0x16,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b32 s0, s1                  ; encoding: [0x01,0x16,0x80,0xbe]
 0x01,0x16,0x80,0xbe
+# GFX12: s_bcnt0_i32_b32 s0, s1                  ; encoding: [0x01,0x16,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b32 s0, vcc_hi              ; encoding: [0x6b,0x16,0x80,0xbe]
 0x6b,0x16,0x80,0xbe
+# GFX12: s_bcnt0_i32_b32 s0, vcc_hi              ; encoding: [0x6b,0x16,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b32 s0, vcc_lo              ; encoding: [0x6a,0x16,0x80,0xbe]
 0x6a,0x16,0x80,0xbe
+# GFX12: s_bcnt0_i32_b32 s0, vcc_lo              ; encoding: [0x6a,0x16,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b32 s105, s104              ; encoding: [0x68,0x16,0xe9,0xbe]
 0x68,0x16,0xe9,0xbe
+# GFX12: s_bcnt0_i32_b32 s105, s104              ; encoding: [0x68,0x16,0xe9,0xbe]
 
-# GFX12: s_bcnt0_i32_b32 s105, s1                ; encoding: [0x01,0x16,0xe9,0xbe]
 0x01,0x16,0xe9,0xbe
+# GFX12: s_bcnt0_i32_b32 s105, s1                ; encoding: [0x01,0x16,0xe9,0xbe]
 
-# GFX12: s_bcnt0_i32_b32 vcc_hi, s1              ; encoding: [0x01,0x16,0xeb,0xbe]
 0x01,0x16,0xeb,0xbe
+# GFX12: s_bcnt0_i32_b32 vcc_hi, s1              ; encoding: [0x01,0x16,0xeb,0xbe]
 
-# GFX12: s_bcnt0_i32_b32 vcc_lo, s1              ; encoding: [0x01,0x16,0xea,0xbe]
 0x01,0x16,0xea,0xbe
+# GFX12: s_bcnt0_i32_b32 vcc_lo, s1              ; encoding: [0x01,0x16,0xea,0xbe]
 
-# GFX12: s_bcnt0_i32_b64 exec_hi, s[2:3]         ; encoding: [0x02,0x17,0xff,0xbe]
 0x02,0x17,0xff,0xbe
+# GFX12: s_bcnt0_i32_b64 exec_hi, s[2:3]         ; encoding: [0x02,0x17,0xff,0xbe]
 
-# GFX12: s_bcnt0_i32_b64 exec_lo, s[2:3]         ; encoding: [0x02,0x17,0xfe,0xbe]
 0x02,0x17,0xfe,0xbe
+# GFX12: s_bcnt0_i32_b64 exec_lo, s[2:3]         ; encoding: [0x02,0x17,0xfe,0xbe]
 
-# GFX12: s_bcnt0_i32_b64 m0, s[2:3]              ; encoding: [0x02,0x17,0xfd,0xbe]
 0x02,0x17,0xfd,0xbe
+# GFX12: s_bcnt0_i32_b64 m0, s[2:3]              ; encoding: [0x02,0x17,0xfd,0xbe]
 
-# GFX12: s_bcnt0_i32_b64 s0, 0.5                 ; encoding: [0xf0,0x17,0x80,0xbe]
 0xf0,0x17,0x80,0xbe
+# GFX12: s_bcnt0_i32_b64 s0, 0.5                 ; encoding: [0xf0,0x17,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b64 s0, 0                   ; encoding: [0x80,0x17,0x80,0xbe]
 0x80,0x17,0x80,0xbe
+# GFX12: s_bcnt0_i32_b64 s0, 0                   ; encoding: [0x80,0x17,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b64 s0, 0x3f717273          ; encoding: [0xff,0x17,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x17,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_bcnt0_i32_b64 s0, 0x3f717273          ; encoding: [0xff,0x17,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bcnt0_i32_b64 s0, 0xaf123456          ; encoding: [0xff,0x17,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x17,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_bcnt0_i32_b64 s0, 0xaf123456          ; encoding: [0xff,0x17,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bcnt0_i32_b64 s0, -1                  ; encoding: [0xc1,0x17,0x80,0xbe]
 0xc1,0x17,0x80,0xbe
+# GFX12: s_bcnt0_i32_b64 s0, -1                  ; encoding: [0xc1,0x17,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b64 s0, -4.0                ; encoding: [0xf7,0x17,0x80,0xbe]
 0xf7,0x17,0x80,0xbe
+# GFX12: s_bcnt0_i32_b64 s0, -4.0                ; encoding: [0xf7,0x17,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b64 s0, exec                ; encoding: [0x7e,0x17,0x80,0xbe]
 0x7e,0x17,0x80,0xbe
+# GFX12: s_bcnt0_i32_b64 s0, exec                ; encoding: [0x7e,0x17,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b64 s0, s[102:103]          ; encoding: [0x66,0x17,0x80,0xbe]
 0x66,0x17,0x80,0xbe
+# GFX12: s_bcnt0_i32_b64 s0, s[102:103]          ; encoding: [0x66,0x17,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b64 s0, s[2:3]              ; encoding: [0x02,0x17,0x80,0xbe]
 0x02,0x17,0x80,0xbe
+# GFX12: s_bcnt0_i32_b64 s0, s[2:3]              ; encoding: [0x02,0x17,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b64 s0, vcc                 ; encoding: [0x6a,0x17,0x80,0xbe]
 0x6a,0x17,0x80,0xbe
+# GFX12: s_bcnt0_i32_b64 s0, vcc                 ; encoding: [0x6a,0x17,0x80,0xbe]
 
-# GFX12: s_bcnt0_i32_b64 s105, s[102:103]        ; encoding: [0x66,0x17,0xe9,0xbe]
 0x66,0x17,0xe9,0xbe
+# GFX12: s_bcnt0_i32_b64 s105, s[102:103]        ; encoding: [0x66,0x17,0xe9,0xbe]
 
-# GFX12: s_bcnt0_i32_b64 s105, s[2:3]            ; encoding: [0x02,0x17,0xe9,0xbe]
 0x02,0x17,0xe9,0xbe
+# GFX12: s_bcnt0_i32_b64 s105, s[2:3]            ; encoding: [0x02,0x17,0xe9,0xbe]
 
-# GFX12: s_bcnt0_i32_b64 vcc_hi, s[2:3]          ; encoding: [0x02,0x17,0xeb,0xbe]
 0x02,0x17,0xeb,0xbe
+# GFX12: s_bcnt0_i32_b64 vcc_hi, s[2:3]          ; encoding: [0x02,0x17,0xeb,0xbe]
 
-# GFX12: s_bcnt0_i32_b64 vcc_lo, s[2:3]          ; encoding: [0x02,0x17,0xea,0xbe]
 0x02,0x17,0xea,0xbe
+# GFX12: s_bcnt0_i32_b64 vcc_lo, s[2:3]          ; encoding: [0x02,0x17,0xea,0xbe]
 
-# GFX12: s_bcnt1_i32_b32 exec_hi, s1             ; encoding: [0x01,0x18,0xff,0xbe]
 0x01,0x18,0xff,0xbe
+# GFX12: s_bcnt1_i32_b32 exec_hi, s1             ; encoding: [0x01,0x18,0xff,0xbe]
 
-# GFX12: s_bcnt1_i32_b32 exec_lo, s1             ; encoding: [0x01,0x18,0xfe,0xbe]
 0x01,0x18,0xfe,0xbe
+# GFX12: s_bcnt1_i32_b32 exec_lo, s1             ; encoding: [0x01,0x18,0xfe,0xbe]
 
-# GFX12: s_bcnt1_i32_b32 m0, s1                  ; encoding: [0x01,0x18,0xfd,0xbe]
 0x01,0x18,0xfd,0xbe
+# GFX12: s_bcnt1_i32_b32 m0, s1                  ; encoding: [0x01,0x18,0xfd,0xbe]
 
-# GFX12: s_bcnt1_i32_b32 s0, 0.5                 ; encoding: [0xf0,0x18,0x80,0xbe]
 0xf0,0x18,0x80,0xbe
+# GFX12: s_bcnt1_i32_b32 s0, 0.5                 ; encoding: [0xf0,0x18,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b32 s0, 0                   ; encoding: [0x80,0x18,0x80,0xbe]
 0x80,0x18,0x80,0xbe
+# GFX12: s_bcnt1_i32_b32 s0, 0                   ; encoding: [0x80,0x18,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b32 s0, 0x3f717273          ; encoding: [0xff,0x18,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x18,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_bcnt1_i32_b32 s0, 0x3f717273          ; encoding: [0xff,0x18,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bcnt1_i32_b32 s0, 0xaf123456          ; encoding: [0xff,0x18,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x18,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_bcnt1_i32_b32 s0, 0xaf123456          ; encoding: [0xff,0x18,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bcnt1_i32_b32 s0, -1                  ; encoding: [0xc1,0x18,0x80,0xbe]
 0xc1,0x18,0x80,0xbe
+# GFX12: s_bcnt1_i32_b32 s0, -1                  ; encoding: [0xc1,0x18,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b32 s0, -4.0                ; encoding: [0xf7,0x18,0x80,0xbe]
 0xf7,0x18,0x80,0xbe
+# GFX12: s_bcnt1_i32_b32 s0, -4.0                ; encoding: [0xf7,0x18,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b32 s0, exec_hi             ; encoding: [0x7f,0x18,0x80,0xbe]
 0x7f,0x18,0x80,0xbe
+# GFX12: s_bcnt1_i32_b32 s0, exec_hi             ; encoding: [0x7f,0x18,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b32 s0, exec_lo             ; encoding: [0x7e,0x18,0x80,0xbe]
 0x7e,0x18,0x80,0xbe
+# GFX12: s_bcnt1_i32_b32 s0, exec_lo             ; encoding: [0x7e,0x18,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b32 s0, m0                  ; encoding: [0x7d,0x18,0x80,0xbe]
 0x7d,0x18,0x80,0xbe
+# GFX12: s_bcnt1_i32_b32 s0, m0                  ; encoding: [0x7d,0x18,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b32 s0, s104                ; encoding: [0x68,0x18,0x80,0xbe]
 0x68,0x18,0x80,0xbe
+# GFX12: s_bcnt1_i32_b32 s0, s104                ; encoding: [0x68,0x18,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b32 s0, s1                  ; encoding: [0x01,0x18,0x80,0xbe]
 0x01,0x18,0x80,0xbe
+# GFX12: s_bcnt1_i32_b32 s0, s1                  ; encoding: [0x01,0x18,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b32 s0, vcc_hi              ; encoding: [0x6b,0x18,0x80,0xbe]
 0x6b,0x18,0x80,0xbe
+# GFX12: s_bcnt1_i32_b32 s0, vcc_hi              ; encoding: [0x6b,0x18,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b32 s0, vcc_lo              ; encoding: [0x6a,0x18,0x80,0xbe]
 0x6a,0x18,0x80,0xbe
+# GFX12: s_bcnt1_i32_b32 s0, vcc_lo              ; encoding: [0x6a,0x18,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b32 s105, s104              ; encoding: [0x68,0x18,0xe9,0xbe]
 0x68,0x18,0xe9,0xbe
+# GFX12: s_bcnt1_i32_b32 s105, s104              ; encoding: [0x68,0x18,0xe9,0xbe]
 
-# GFX12: s_bcnt1_i32_b32 s105, s1                ; encoding: [0x01,0x18,0xe9,0xbe]
 0x01,0x18,0xe9,0xbe
+# GFX12: s_bcnt1_i32_b32 s105, s1                ; encoding: [0x01,0x18,0xe9,0xbe]
 
-# GFX12: s_bcnt1_i32_b32 vcc_hi, s1              ; encoding: [0x01,0x18,0xeb,0xbe]
 0x01,0x18,0xeb,0xbe
+# GFX12: s_bcnt1_i32_b32 vcc_hi, s1              ; encoding: [0x01,0x18,0xeb,0xbe]
 
-# GFX12: s_bcnt1_i32_b32 vcc_lo, s1              ; encoding: [0x01,0x18,0xea,0xbe]
 0x01,0x18,0xea,0xbe
+# GFX12: s_bcnt1_i32_b32 vcc_lo, s1              ; encoding: [0x01,0x18,0xea,0xbe]
 
-# GFX12: s_bcnt1_i32_b64 exec_hi, s[2:3]         ; encoding: [0x02,0x19,0xff,0xbe]
 0x02,0x19,0xff,0xbe
+# GFX12: s_bcnt1_i32_b64 exec_hi, s[2:3]         ; encoding: [0x02,0x19,0xff,0xbe]
 
-# GFX12: s_bcnt1_i32_b64 exec_lo, s[2:3]         ; encoding: [0x02,0x19,0xfe,0xbe]
 0x02,0x19,0xfe,0xbe
+# GFX12: s_bcnt1_i32_b64 exec_lo, s[2:3]         ; encoding: [0x02,0x19,0xfe,0xbe]
 
-# GFX12: s_bcnt1_i32_b64 m0, s[2:3]              ; encoding: [0x02,0x19,0xfd,0xbe]
 0x02,0x19,0xfd,0xbe
+# GFX12: s_bcnt1_i32_b64 m0, s[2:3]              ; encoding: [0x02,0x19,0xfd,0xbe]
 
-# GFX12: s_bcnt1_i32_b64 s0, 0.5                 ; encoding: [0xf0,0x19,0x80,0xbe]
 0xf0,0x19,0x80,0xbe
+# GFX12: s_bcnt1_i32_b64 s0, 0.5                 ; encoding: [0xf0,0x19,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b64 s0, 0                   ; encoding: [0x80,0x19,0x80,0xbe]
 0x80,0x19,0x80,0xbe
+# GFX12: s_bcnt1_i32_b64 s0, 0                   ; encoding: [0x80,0x19,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b64 s0, 0x3f717273          ; encoding: [0xff,0x19,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x19,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_bcnt1_i32_b64 s0, 0x3f717273          ; encoding: [0xff,0x19,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bcnt1_i32_b64 s0, 0xaf123456          ; encoding: [0xff,0x19,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x19,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_bcnt1_i32_b64 s0, 0xaf123456          ; encoding: [0xff,0x19,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bcnt1_i32_b64 s0, -1                  ; encoding: [0xc1,0x19,0x80,0xbe]
 0xc1,0x19,0x80,0xbe
+# GFX12: s_bcnt1_i32_b64 s0, -1                  ; encoding: [0xc1,0x19,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b64 s0, -4.0                ; encoding: [0xf7,0x19,0x80,0xbe]
 0xf7,0x19,0x80,0xbe
+# GFX12: s_bcnt1_i32_b64 s0, -4.0                ; encoding: [0xf7,0x19,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b64 s0, exec                ; encoding: [0x7e,0x19,0x80,0xbe]
 0x7e,0x19,0x80,0xbe
+# GFX12: s_bcnt1_i32_b64 s0, exec                ; encoding: [0x7e,0x19,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b64 s0, s[102:103]          ; encoding: [0x66,0x19,0x80,0xbe]
 0x66,0x19,0x80,0xbe
+# GFX12: s_bcnt1_i32_b64 s0, s[102:103]          ; encoding: [0x66,0x19,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b64 s0, s[2:3]              ; encoding: [0x02,0x19,0x80,0xbe]
 0x02,0x19,0x80,0xbe
+# GFX12: s_bcnt1_i32_b64 s0, s[2:3]              ; encoding: [0x02,0x19,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b64 s0, vcc                 ; encoding: [0x6a,0x19,0x80,0xbe]
 0x6a,0x19,0x80,0xbe
+# GFX12: s_bcnt1_i32_b64 s0, vcc                 ; encoding: [0x6a,0x19,0x80,0xbe]
 
-# GFX12: s_bcnt1_i32_b64 s105, s[102:103]        ; encoding: [0x66,0x19,0xe9,0xbe]
 0x66,0x19,0xe9,0xbe
+# GFX12: s_bcnt1_i32_b64 s105, s[102:103]        ; encoding: [0x66,0x19,0xe9,0xbe]
 
-# GFX12: s_bcnt1_i32_b64 s105, s[2:3]            ; encoding: [0x02,0x19,0xe9,0xbe]
 0x02,0x19,0xe9,0xbe
+# GFX12: s_bcnt1_i32_b64 s105, s[2:3]            ; encoding: [0x02,0x19,0xe9,0xbe]
 
-# GFX12: s_bcnt1_i32_b64 vcc_hi, s[2:3]          ; encoding: [0x02,0x19,0xeb,0xbe]
 0x02,0x19,0xeb,0xbe
+# GFX12: s_bcnt1_i32_b64 vcc_hi, s[2:3]          ; encoding: [0x02,0x19,0xeb,0xbe]
 
-# GFX12: s_bcnt1_i32_b64 vcc_lo, s[2:3]          ; encoding: [0x02,0x19,0xea,0xbe]
 0x02,0x19,0xea,0xbe
+# GFX12: s_bcnt1_i32_b64 vcc_lo, s[2:3]          ; encoding: [0x02,0x19,0xea,0xbe]
 
-# GFX12: s_bitreplicate_b64_b32 exec, s2         ; encoding: [0x02,0x14,0xfe,0xbe]
 0x02,0x14,0xfe,0xbe
+# GFX12: s_bitreplicate_b64_b32 exec, s2         ; encoding: [0x02,0x14,0xfe,0xbe]
 
-# GFX12: s_bitreplicate_b64_b32 s[0:1], 0.5      ; encoding: [0xf0,0x14,0x80,0xbe]
 0xf0,0x14,0x80,0xbe
+# GFX12: s_bitreplicate_b64_b32 s[0:1], 0.5      ; encoding: [0xf0,0x14,0x80,0xbe]
 
-# GFX12: s_bitreplicate_b64_b32 s[0:1], 0        ; encoding: [0x80,0x14,0x80,0xbe]
 0x80,0x14,0x80,0xbe
+# GFX12: s_bitreplicate_b64_b32 s[0:1], 0        ; encoding: [0x80,0x14,0x80,0xbe]
 
-# GFX12: s_bitreplicate_b64_b32 s[0:1], 0x3f717273 ; encoding: [0xff,0x14,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x14,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_bitreplicate_b64_b32 s[0:1], 0x3f717273 ; encoding: [0xff,0x14,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bitreplicate_b64_b32 s[0:1], 0xaf123456 ; encoding: [0xff,0x14,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x14,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_bitreplicate_b64_b32 s[0:1], 0xaf123456 ; encoding: [0xff,0x14,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bitreplicate_b64_b32 s[0:1], -1       ; encoding: [0xc1,0x14,0x80,0xbe]
 0xc1,0x14,0x80,0xbe
+# GFX12: s_bitreplicate_b64_b32 s[0:1], -1       ; encoding: [0xc1,0x14,0x80,0xbe]
 
-# GFX12: s_bitreplicate_b64_b32 s[0:1], -4.0     ; encoding: [0xf7,0x14,0x80,0xbe]
 0xf7,0x14,0x80,0xbe
+# GFX12: s_bitreplicate_b64_b32 s[0:1], -4.0     ; encoding: [0xf7,0x14,0x80,0xbe]
 
-# GFX12: s_bitreplicate_b64_b32 s[0:1], exec_hi  ; encoding: [0x7f,0x14,0x80,0xbe]
 0x7f,0x14,0x80,0xbe
+# GFX12: s_bitreplicate_b64_b32 s[0:1], exec_hi  ; encoding: [0x7f,0x14,0x80,0xbe]
 
-# GFX12: s_bitreplicate_b64_b32 s[0:1], exec_lo  ; encoding: [0x7e,0x14,0x80,0xbe]
 0x7e,0x14,0x80,0xbe
+# GFX12: s_bitreplicate_b64_b32 s[0:1], exec_lo  ; encoding: [0x7e,0x14,0x80,0xbe]
 
-# GFX12: s_bitreplicate_b64_b32 s[0:1], m0       ; encoding: [0x7d,0x14,0x80,0xbe]
 0x7d,0x14,0x80,0xbe
+# GFX12: s_bitreplicate_b64_b32 s[0:1], m0       ; encoding: [0x7d,0x14,0x80,0xbe]
 
-# GFX12: s_bitreplicate_b64_b32 s[0:1], s102     ; encoding: [0x66,0x14,0x80,0xbe]
 0x66,0x14,0x80,0xbe
+# GFX12: s_bitreplicate_b64_b32 s[0:1], s102     ; encoding: [0x66,0x14,0x80,0xbe]
 
-# GFX12: s_bitreplicate_b64_b32 s[0:1], s2       ; encoding: [0x02,0x14,0x80,0xbe]
 0x02,0x14,0x80,0xbe
+# GFX12: s_bitreplicate_b64_b32 s[0:1], s2       ; encoding: [0x02,0x14,0x80,0xbe]
 
-# GFX12: s_bitreplicate_b64_b32 s[0:1], vcc_hi   ; encoding: [0x6b,0x14,0x80,0xbe]
 0x6b,0x14,0x80,0xbe
+# GFX12: s_bitreplicate_b64_b32 s[0:1], vcc_hi   ; encoding: [0x6b,0x14,0x80,0xbe]
 
-# GFX12: s_bitreplicate_b64_b32 s[0:1], vcc_lo   ; encoding: [0x6a,0x14,0x80,0xbe]
 0x6a,0x14,0x80,0xbe
+# GFX12: s_bitreplicate_b64_b32 s[0:1], vcc_lo   ; encoding: [0x6a,0x14,0x80,0xbe]
 
-# GFX12: s_bitreplicate_b64_b32 s[104:105], s102 ; encoding: [0x66,0x14,0xe8,0xbe]
 0x66,0x14,0xe8,0xbe
+# GFX12: s_bitreplicate_b64_b32 s[104:105], s102 ; encoding: [0x66,0x14,0xe8,0xbe]
 
-# GFX12: s_bitreplicate_b64_b32 s[104:105], s2   ; encoding: [0x02,0x14,0xe8,0xbe]
 0x02,0x14,0xe8,0xbe
+# GFX12: s_bitreplicate_b64_b32 s[104:105], s2   ; encoding: [0x02,0x14,0xe8,0xbe]
 
-# GFX12: s_bitreplicate_b64_b32 vcc, s2          ; encoding: [0x02,0x14,0xea,0xbe]
 0x02,0x14,0xea,0xbe
+# GFX12: s_bitreplicate_b64_b32 vcc, s2          ; encoding: [0x02,0x14,0xea,0xbe]
 
-# GFX12: s_bitset0_b32 exec_hi, s1               ; encoding: [0x01,0x10,0xff,0xbe]
 0x01,0x10,0xff,0xbe
+# GFX12: s_bitset0_b32 exec_hi, s1               ; encoding: [0x01,0x10,0xff,0xbe]
 
-# GFX12: s_bitset0_b32 exec_lo, s1               ; encoding: [0x01,0x10,0xfe,0xbe]
 0x01,0x10,0xfe,0xbe
+# GFX12: s_bitset0_b32 exec_lo, s1               ; encoding: [0x01,0x10,0xfe,0xbe]
 
-# GFX12: s_bitset0_b32 m0, s1                    ; encoding: [0x01,0x10,0xfd,0xbe]
 0x01,0x10,0xfd,0xbe
+# GFX12: s_bitset0_b32 m0, s1                    ; encoding: [0x01,0x10,0xfd,0xbe]
 
-# GFX12: s_bitset0_b32 s0, 0.5                   ; encoding: [0xf0,0x10,0x80,0xbe]
 0xf0,0x10,0x80,0xbe
+# GFX12: s_bitset0_b32 s0, 0.5                   ; encoding: [0xf0,0x10,0x80,0xbe]
 
-# GFX12: s_bitset0_b32 s0, 0                     ; encoding: [0x80,0x10,0x80,0xbe]
 0x80,0x10,0x80,0xbe
+# GFX12: s_bitset0_b32 s0, 0                     ; encoding: [0x80,0x10,0x80,0xbe]
 
-# GFX12: s_bitset0_b32 s0, 0x3f717273            ; encoding: [0xff,0x10,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x10,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_bitset0_b32 s0, 0x3f717273            ; encoding: [0xff,0x10,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bitset0_b32 s0, 0xaf123456            ; encoding: [0xff,0x10,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x10,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_bitset0_b32 s0, 0xaf123456            ; encoding: [0xff,0x10,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bitset0_b32 s0, -1                    ; encoding: [0xc1,0x10,0x80,0xbe]
 0xc1,0x10,0x80,0xbe
+# GFX12: s_bitset0_b32 s0, -1                    ; encoding: [0xc1,0x10,0x80,0xbe]
 
-# GFX12: s_bitset0_b32 s0, -4.0                  ; encoding: [0xf7,0x10,0x80,0xbe]
 0xf7,0x10,0x80,0xbe
+# GFX12: s_bitset0_b32 s0, -4.0                  ; encoding: [0xf7,0x10,0x80,0xbe]
 
-# GFX12: s_bitset0_b32 s0, exec_hi               ; encoding: [0x7f,0x10,0x80,0xbe]
 0x7f,0x10,0x80,0xbe
+# GFX12: s_bitset0_b32 s0, exec_hi               ; encoding: [0x7f,0x10,0x80,0xbe]
 
-# GFX12: s_bitset0_b32 s0, exec_lo               ; encoding: [0x7e,0x10,0x80,0xbe]
 0x7e,0x10,0x80,0xbe
+# GFX12: s_bitset0_b32 s0, exec_lo               ; encoding: [0x7e,0x10,0x80,0xbe]
 
-# GFX12: s_bitset0_b32 s0, m0                    ; encoding: [0x7d,0x10,0x80,0xbe]
 0x7d,0x10,0x80,0xbe
+# GFX12: s_bitset0_b32 s0, m0                    ; encoding: [0x7d,0x10,0x80,0xbe]
 
-# GFX12: s_bitset0_b32 s0, s104                  ; encoding: [0x68,0x10,0x80,0xbe]
 0x68,0x10,0x80,0xbe
+# GFX12: s_bitset0_b32 s0, s104                  ; encoding: [0x68,0x10,0x80,0xbe]
 
-# GFX12: s_bitset0_b32 s0, s1                    ; encoding: [0x01,0x10,0x80,0xbe]
 0x01,0x10,0x80,0xbe
+# GFX12: s_bitset0_b32 s0, s1                    ; encoding: [0x01,0x10,0x80,0xbe]
 
-# GFX12: s_bitset0_b32 s0, vcc_hi                ; encoding: [0x6b,0x10,0x80,0xbe]
 0x6b,0x10,0x80,0xbe
+# GFX12: s_bitset0_b32 s0, vcc_hi                ; encoding: [0x6b,0x10,0x80,0xbe]
 
-# GFX12: s_bitset0_b32 s0, vcc_lo                ; encoding: [0x6a,0x10,0x80,0xbe]
 0x6a,0x10,0x80,0xbe
+# GFX12: s_bitset0_b32 s0, vcc_lo                ; encoding: [0x6a,0x10,0x80,0xbe]
 
-# GFX12: s_bitset0_b32 s105, s104                ; encoding: [0x68,0x10,0xe9,0xbe]
 0x68,0x10,0xe9,0xbe
+# GFX12: s_bitset0_b32 s105, s104                ; encoding: [0x68,0x10,0xe9,0xbe]
 
-# GFX12: s_bitset0_b32 s105, s1                  ; encoding: [0x01,0x10,0xe9,0xbe]
 0x01,0x10,0xe9,0xbe
+# GFX12: s_bitset0_b32 s105, s1                  ; encoding: [0x01,0x10,0xe9,0xbe]
 
-# GFX12: s_bitset0_b32 vcc_hi, s1                ; encoding: [0x01,0x10,0xeb,0xbe]
 0x01,0x10,0xeb,0xbe
+# GFX12: s_bitset0_b32 vcc_hi, s1                ; encoding: [0x01,0x10,0xeb,0xbe]
 
-# GFX12: s_bitset0_b32 vcc_lo, s1                ; encoding: [0x01,0x10,0xea,0xbe]
 0x01,0x10,0xea,0xbe
+# GFX12: s_bitset0_b32 vcc_lo, s1                ; encoding: [0x01,0x10,0xea,0xbe]
 
-# GFX12: s_bitset0_b64 exec, s2                  ; encoding: [0x02,0x11,0xfe,0xbe]
 0x02,0x11,0xfe,0xbe
+# GFX12: s_bitset0_b64 exec, s2                  ; encoding: [0x02,0x11,0xfe,0xbe]
 
-# GFX12: s_bitset0_b64 s[0:1], 0.5               ; encoding: [0xf0,0x11,0x80,0xbe]
 0xf0,0x11,0x80,0xbe
+# GFX12: s_bitset0_b64 s[0:1], 0.5               ; encoding: [0xf0,0x11,0x80,0xbe]
 
-# GFX12: s_bitset0_b64 s[0:1], 0                 ; encoding: [0x80,0x11,0x80,0xbe]
 0x80,0x11,0x80,0xbe
+# GFX12: s_bitset0_b64 s[0:1], 0                 ; encoding: [0x80,0x11,0x80,0xbe]
 
-# GFX12: s_bitset0_b64 s[0:1], 0x3f717273        ; encoding: [0xff,0x11,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x11,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_bitset0_b64 s[0:1], 0x3f717273        ; encoding: [0xff,0x11,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bitset0_b64 s[0:1], 0xaf123456        ; encoding: [0xff,0x11,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x11,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_bitset0_b64 s[0:1], 0xaf123456        ; encoding: [0xff,0x11,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bitset0_b64 s[0:1], -1                ; encoding: [0xc1,0x11,0x80,0xbe]
 0xc1,0x11,0x80,0xbe
+# GFX12: s_bitset0_b64 s[0:1], -1                ; encoding: [0xc1,0x11,0x80,0xbe]
 
-# GFX12: s_bitset0_b64 s[0:1], -4.0              ; encoding: [0xf7,0x11,0x80,0xbe]
 0xf7,0x11,0x80,0xbe
+# GFX12: s_bitset0_b64 s[0:1], -4.0              ; encoding: [0xf7,0x11,0x80,0xbe]
 
-# GFX12: s_bitset0_b64 s[0:1], exec_hi           ; encoding: [0x7f,0x11,0x80,0xbe]
 0x7f,0x11,0x80,0xbe
+# GFX12: s_bitset0_b64 s[0:1], exec_hi           ; encoding: [0x7f,0x11,0x80,0xbe]
 
-# GFX12: s_bitset0_b64 s[0:1], exec_lo           ; encoding: [0x7e,0x11,0x80,0xbe]
 0x7e,0x11,0x80,0xbe
+# GFX12: s_bitset0_b64 s[0:1], exec_lo           ; encoding: [0x7e,0x11,0x80,0xbe]
 
-# GFX12: s_bitset0_b64 s[0:1], m0                ; encoding: [0x7d,0x11,0x80,0xbe]
 0x7d,0x11,0x80,0xbe
+# GFX12: s_bitset0_b64 s[0:1], m0                ; encoding: [0x7d,0x11,0x80,0xbe]
 
-# GFX12: s_bitset0_b64 s[0:1], s102              ; encoding: [0x66,0x11,0x80,0xbe]
 0x66,0x11,0x80,0xbe
+# GFX12: s_bitset0_b64 s[0:1], s102              ; encoding: [0x66,0x11,0x80,0xbe]
 
-# GFX12: s_bitset0_b64 s[0:1], s2                ; encoding: [0x02,0x11,0x80,0xbe]
 0x02,0x11,0x80,0xbe
+# GFX12: s_bitset0_b64 s[0:1], s2                ; encoding: [0x02,0x11,0x80,0xbe]
 
-# GFX12: s_bitset0_b64 s[0:1], vcc_hi            ; encoding: [0x6b,0x11,0x80,0xbe]
 0x6b,0x11,0x80,0xbe
+# GFX12: s_bitset0_b64 s[0:1], vcc_hi            ; encoding: [0x6b,0x11,0x80,0xbe]
 
-# GFX12: s_bitset0_b64 s[0:1], vcc_lo            ; encoding: [0x6a,0x11,0x80,0xbe]
 0x6a,0x11,0x80,0xbe
+# GFX12: s_bitset0_b64 s[0:1], vcc_lo            ; encoding: [0x6a,0x11,0x80,0xbe]
 
-# GFX12: s_bitset0_b64 s[104:105], s102          ; encoding: [0x66,0x11,0xe8,0xbe]
 0x66,0x11,0xe8,0xbe
+# GFX12: s_bitset0_b64 s[104:105], s102          ; encoding: [0x66,0x11,0xe8,0xbe]
 
-# GFX12: s_bitset0_b64 s[104:105], s2            ; encoding: [0x02,0x11,0xe8,0xbe]
 0x02,0x11,0xe8,0xbe
+# GFX12: s_bitset0_b64 s[104:105], s2            ; encoding: [0x02,0x11,0xe8,0xbe]
 
-# GFX12: s_bitset0_b64 vcc, s2                   ; encoding: [0x02,0x11,0xea,0xbe]
 0x02,0x11,0xea,0xbe
+# GFX12: s_bitset0_b64 vcc, s2                   ; encoding: [0x02,0x11,0xea,0xbe]
 
-# GFX12: s_bitset1_b32 exec_hi, s1               ; encoding: [0x01,0x12,0xff,0xbe]
 0x01,0x12,0xff,0xbe
+# GFX12: s_bitset1_b32 exec_hi, s1               ; encoding: [0x01,0x12,0xff,0xbe]
 
-# GFX12: s_bitset1_b32 exec_lo, s1               ; encoding: [0x01,0x12,0xfe,0xbe]
 0x01,0x12,0xfe,0xbe
+# GFX12: s_bitset1_b32 exec_lo, s1               ; encoding: [0x01,0x12,0xfe,0xbe]
 
-# GFX12: s_bitset1_b32 m0, s1                    ; encoding: [0x01,0x12,0xfd,0xbe]
 0x01,0x12,0xfd,0xbe
+# GFX12: s_bitset1_b32 m0, s1                    ; encoding: [0x01,0x12,0xfd,0xbe]
 
-# GFX12: s_bitset1_b32 s0, 0.5                   ; encoding: [0xf0,0x12,0x80,0xbe]
 0xf0,0x12,0x80,0xbe
+# GFX12: s_bitset1_b32 s0, 0.5                   ; encoding: [0xf0,0x12,0x80,0xbe]
 
-# GFX12: s_bitset1_b32 s0, 0                     ; encoding: [0x80,0x12,0x80,0xbe]
 0x80,0x12,0x80,0xbe
+# GFX12: s_bitset1_b32 s0, 0                     ; encoding: [0x80,0x12,0x80,0xbe]
 
-# GFX12: s_bitset1_b32 s0, 0x3f717273            ; encoding: [0xff,0x12,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x12,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_bitset1_b32 s0, 0x3f717273            ; encoding: [0xff,0x12,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bitset1_b32 s0, 0xaf123456            ; encoding: [0xff,0x12,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x12,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_bitset1_b32 s0, 0xaf123456            ; encoding: [0xff,0x12,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bitset1_b32 s0, -1                    ; encoding: [0xc1,0x12,0x80,0xbe]
 0xc1,0x12,0x80,0xbe
+# GFX12: s_bitset1_b32 s0, -1                    ; encoding: [0xc1,0x12,0x80,0xbe]
 
-# GFX12: s_bitset1_b32 s0, -4.0                  ; encoding: [0xf7,0x12,0x80,0xbe]
 0xf7,0x12,0x80,0xbe
+# GFX12: s_bitset1_b32 s0, -4.0                  ; encoding: [0xf7,0x12,0x80,0xbe]
 
-# GFX12: s_bitset1_b32 s0, exec_hi               ; encoding: [0x7f,0x12,0x80,0xbe]
 0x7f,0x12,0x80,0xbe
+# GFX12: s_bitset1_b32 s0, exec_hi               ; encoding: [0x7f,0x12,0x80,0xbe]
 
-# GFX12: s_bitset1_b32 s0, exec_lo               ; encoding: [0x7e,0x12,0x80,0xbe]
 0x7e,0x12,0x80,0xbe
+# GFX12: s_bitset1_b32 s0, exec_lo               ; encoding: [0x7e,0x12,0x80,0xbe]
 
-# GFX12: s_bitset1_b32 s0, m0                    ; encoding: [0x7d,0x12,0x80,0xbe]
 0x7d,0x12,0x80,0xbe
+# GFX12: s_bitset1_b32 s0, m0                    ; encoding: [0x7d,0x12,0x80,0xbe]
 
-# GFX12: s_bitset1_b32 s0, s104                  ; encoding: [0x68,0x12,0x80,0xbe]
 0x68,0x12,0x80,0xbe
+# GFX12: s_bitset1_b32 s0, s104                  ; encoding: [0x68,0x12,0x80,0xbe]
 
-# GFX12: s_bitset1_b32 s0, s1                    ; encoding: [0x01,0x12,0x80,0xbe]
 0x01,0x12,0x80,0xbe
+# GFX12: s_bitset1_b32 s0, s1                    ; encoding: [0x01,0x12,0x80,0xbe]
 
-# GFX12: s_bitset1_b32 s0, vcc_hi                ; encoding: [0x6b,0x12,0x80,0xbe]
 0x6b,0x12,0x80,0xbe
+# GFX12: s_bitset1_b32 s0, vcc_hi                ; encoding: [0x6b,0x12,0x80,0xbe]
 
-# GFX12: s_bitset1_b32 s0, vcc_lo                ; encoding: [0x6a,0x12,0x80,0xbe]
 0x6a,0x12,0x80,0xbe
+# GFX12: s_bitset1_b32 s0, vcc_lo                ; encoding: [0x6a,0x12,0x80,0xbe]
 
-# GFX12: s_bitset1_b32 s105, s104                ; encoding: [0x68,0x12,0xe9,0xbe]
 0x68,0x12,0xe9,0xbe
+# GFX12: s_bitset1_b32 s105, s104                ; encoding: [0x68,0x12,0xe9,0xbe]
 
-# GFX12: s_bitset1_b32 s105, s1                  ; encoding: [0x01,0x12,0xe9,0xbe]
 0x01,0x12,0xe9,0xbe
+# GFX12: s_bitset1_b32 s105, s1                  ; encoding: [0x01,0x12,0xe9,0xbe]
 
-# GFX12: s_bitset1_b32 vcc_hi, s1                ; encoding: [0x01,0x12,0xeb,0xbe]
 0x01,0x12,0xeb,0xbe
+# GFX12: s_bitset1_b32 vcc_hi, s1                ; encoding: [0x01,0x12,0xeb,0xbe]
 
-# GFX12: s_bitset1_b32 vcc_lo, s1                ; encoding: [0x01,0x12,0xea,0xbe]
 0x01,0x12,0xea,0xbe
+# GFX12: s_bitset1_b32 vcc_lo, s1                ; encoding: [0x01,0x12,0xea,0xbe]
 
-# GFX12: s_bitset1_b64 exec, s2                  ; encoding: [0x02,0x13,0xfe,0xbe]
 0x02,0x13,0xfe,0xbe
+# GFX12: s_bitset1_b64 exec, s2                  ; encoding: [0x02,0x13,0xfe,0xbe]
 
-# GFX12: s_bitset1_b64 s[0:1], 0.5               ; encoding: [0xf0,0x13,0x80,0xbe]
 0xf0,0x13,0x80,0xbe
+# GFX12: s_bitset1_b64 s[0:1], 0.5               ; encoding: [0xf0,0x13,0x80,0xbe]
 
-# GFX12: s_bitset1_b64 s[0:1], 0                 ; encoding: [0x80,0x13,0x80,0xbe]
 0x80,0x13,0x80,0xbe
+# GFX12: s_bitset1_b64 s[0:1], 0                 ; encoding: [0x80,0x13,0x80,0xbe]
 
-# GFX12: s_bitset1_b64 s[0:1], 0x3f717273        ; encoding: [0xff,0x13,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x13,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_bitset1_b64 s[0:1], 0x3f717273        ; encoding: [0xff,0x13,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bitset1_b64 s[0:1], 0xaf123456        ; encoding: [0xff,0x13,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x13,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_bitset1_b64 s[0:1], 0xaf123456        ; encoding: [0xff,0x13,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bitset1_b64 s[0:1], -1                ; encoding: [0xc1,0x13,0x80,0xbe]
 0xc1,0x13,0x80,0xbe
+# GFX12: s_bitset1_b64 s[0:1], -1                ; encoding: [0xc1,0x13,0x80,0xbe]
 
-# GFX12: s_bitset1_b64 s[0:1], -4.0              ; encoding: [0xf7,0x13,0x80,0xbe]
 0xf7,0x13,0x80,0xbe
+# GFX12: s_bitset1_b64 s[0:1], -4.0              ; encoding: [0xf7,0x13,0x80,0xbe]
 
-# GFX12: s_bitset1_b64 s[0:1], exec_hi           ; encoding: [0x7f,0x13,0x80,0xbe]
 0x7f,0x13,0x80,0xbe
+# GFX12: s_bitset1_b64 s[0:1], exec_hi           ; encoding: [0x7f,0x13,0x80,0xbe]
 
-# GFX12: s_bitset1_b64 s[0:1], exec_lo           ; encoding: [0x7e,0x13,0x80,0xbe]
 0x7e,0x13,0x80,0xbe
+# GFX12: s_bitset1_b64 s[0:1], exec_lo           ; encoding: [0x7e,0x13,0x80,0xbe]
 
-# GFX12: s_bitset1_b64 s[0:1], m0                ; encoding: [0x7d,0x13,0x80,0xbe]
 0x7d,0x13,0x80,0xbe
+# GFX12: s_bitset1_b64 s[0:1], m0                ; encoding: [0x7d,0x13,0x80,0xbe]
 
-# GFX12: s_bitset1_b64 s[0:1], s102              ; encoding: [0x66,0x13,0x80,0xbe]
 0x66,0x13,0x80,0xbe
+# GFX12: s_bitset1_b64 s[0:1], s102              ; encoding: [0x66,0x13,0x80,0xbe]
 
-# GFX12: s_bitset1_b64 s[0:1], s2                ; encoding: [0x02,0x13,0x80,0xbe]
 0x02,0x13,0x80,0xbe
+# GFX12: s_bitset1_b64 s[0:1], s2                ; encoding: [0x02,0x13,0x80,0xbe]
 
-# GFX12: s_bitset1_b64 s[0:1], vcc_hi            ; encoding: [0x6b,0x13,0x80,0xbe]
 0x6b,0x13,0x80,0xbe
+# GFX12: s_bitset1_b64 s[0:1], vcc_hi            ; encoding: [0x6b,0x13,0x80,0xbe]
 
-# GFX12: s_bitset1_b64 s[0:1], vcc_lo            ; encoding: [0x6a,0x13,0x80,0xbe]
 0x6a,0x13,0x80,0xbe
+# GFX12: s_bitset1_b64 s[0:1], vcc_lo            ; encoding: [0x6a,0x13,0x80,0xbe]
 
-# GFX12: s_bitset1_b64 s[104:105], s102          ; encoding: [0x66,0x13,0xe8,0xbe]
 0x66,0x13,0xe8,0xbe
+# GFX12: s_bitset1_b64 s[104:105], s102          ; encoding: [0x66,0x13,0xe8,0xbe]
 
-# GFX12: s_bitset1_b64 s[104:105], s2            ; encoding: [0x02,0x13,0xe8,0xbe]
 0x02,0x13,0xe8,0xbe
+# GFX12: s_bitset1_b64 s[104:105], s2            ; encoding: [0x02,0x13,0xe8,0xbe]
 
-# GFX12: s_bitset1_b64 vcc, s2                   ; encoding: [0x02,0x13,0xea,0xbe]
 0x02,0x13,0xea,0xbe
+# GFX12: s_bitset1_b64 vcc, s2                   ; encoding: [0x02,0x13,0xea,0xbe]
 
-# GFX12: s_brev_b32 exec_hi, s1                  ; encoding: [0x01,0x04,0xff,0xbe]
 0x01,0x04,0xff,0xbe
+# GFX12: s_brev_b32 exec_hi, s1                  ; encoding: [0x01,0x04,0xff,0xbe]
 
-# GFX12: s_brev_b32 exec_lo, s1                  ; encoding: [0x01,0x04,0xfe,0xbe]
 0x01,0x04,0xfe,0xbe
+# GFX12: s_brev_b32 exec_lo, s1                  ; encoding: [0x01,0x04,0xfe,0xbe]
 
-# GFX12: s_brev_b32 m0, s1                       ; encoding: [0x01,0x04,0xfd,0xbe]
 0x01,0x04,0xfd,0xbe
+# GFX12: s_brev_b32 m0, s1                       ; encoding: [0x01,0x04,0xfd,0xbe]
 
-# GFX12: s_brev_b32 s0, 0.5                      ; encoding: [0xf0,0x04,0x80,0xbe]
 0xf0,0x04,0x80,0xbe
+# GFX12: s_brev_b32 s0, 0.5                      ; encoding: [0xf0,0x04,0x80,0xbe]
 
-# GFX12: s_brev_b32 s0, 0                        ; encoding: [0x80,0x04,0x80,0xbe]
 0x80,0x04,0x80,0xbe
+# GFX12: s_brev_b32 s0, 0                        ; encoding: [0x80,0x04,0x80,0xbe]
 
-# GFX12: s_brev_b32 s0, 0x3f717273               ; encoding: [0xff,0x04,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x04,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_brev_b32 s0, 0x3f717273               ; encoding: [0xff,0x04,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_brev_b32 s0, 0xaf123456               ; encoding: [0xff,0x04,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x04,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_brev_b32 s0, 0xaf123456               ; encoding: [0xff,0x04,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_brev_b32 s0, -1                       ; encoding: [0xc1,0x04,0x80,0xbe]
 0xc1,0x04,0x80,0xbe
+# GFX12: s_brev_b32 s0, -1                       ; encoding: [0xc1,0x04,0x80,0xbe]
 
-# GFX12: s_brev_b32 s0, -4.0                     ; encoding: [0xf7,0x04,0x80,0xbe]
 0xf7,0x04,0x80,0xbe
+# GFX12: s_brev_b32 s0, -4.0                     ; encoding: [0xf7,0x04,0x80,0xbe]
 
-# GFX12: s_brev_b32 s0, exec_hi                  ; encoding: [0x7f,0x04,0x80,0xbe]
 0x7f,0x04,0x80,0xbe
+# GFX12: s_brev_b32 s0, exec_hi                  ; encoding: [0x7f,0x04,0x80,0xbe]
 
-# GFX12: s_brev_b32 s0, exec_lo                  ; encoding: [0x7e,0x04,0x80,0xbe]
 0x7e,0x04,0x80,0xbe
+# GFX12: s_brev_b32 s0, exec_lo                  ; encoding: [0x7e,0x04,0x80,0xbe]
 
-# GFX12: s_brev_b32 s0, m0                       ; encoding: [0x7d,0x04,0x80,0xbe]
 0x7d,0x04,0x80,0xbe
+# GFX12: s_brev_b32 s0, m0                       ; encoding: [0x7d,0x04,0x80,0xbe]
 
-# GFX12: s_brev_b32 s0, s104                     ; encoding: [0x68,0x04,0x80,0xbe]
 0x68,0x04,0x80,0xbe
+# GFX12: s_brev_b32 s0, s104                     ; encoding: [0x68,0x04,0x80,0xbe]
 
-# GFX12: s_brev_b32 s0, s1                       ; encoding: [0x01,0x04,0x80,0xbe]
 0x01,0x04,0x80,0xbe
+# GFX12: s_brev_b32 s0, s1                       ; encoding: [0x01,0x04,0x80,0xbe]
 
-# GFX12: s_brev_b32 s0, vcc_hi                   ; encoding: [0x6b,0x04,0x80,0xbe]
 0x6b,0x04,0x80,0xbe
+# GFX12: s_brev_b32 s0, vcc_hi                   ; encoding: [0x6b,0x04,0x80,0xbe]
 
-# GFX12: s_brev_b32 s0, vcc_lo                   ; encoding: [0x6a,0x04,0x80,0xbe]
 0x6a,0x04,0x80,0xbe
+# GFX12: s_brev_b32 s0, vcc_lo                   ; encoding: [0x6a,0x04,0x80,0xbe]
 
-# GFX12: s_brev_b32 s105, s104                   ; encoding: [0x68,0x04,0xe9,0xbe]
 0x68,0x04,0xe9,0xbe
+# GFX12: s_brev_b32 s105, s104                   ; encoding: [0x68,0x04,0xe9,0xbe]
 
-# GFX12: s_brev_b32 s105, s1                     ; encoding: [0x01,0x04,0xe9,0xbe]
 0x01,0x04,0xe9,0xbe
+# GFX12: s_brev_b32 s105, s1                     ; encoding: [0x01,0x04,0xe9,0xbe]
 
-# GFX12: s_brev_b32 vcc_hi, s1                   ; encoding: [0x01,0x04,0xeb,0xbe]
 0x01,0x04,0xeb,0xbe
+# GFX12: s_brev_b32 vcc_hi, s1                   ; encoding: [0x01,0x04,0xeb,0xbe]
 
-# GFX12: s_brev_b32 vcc_lo, s1                   ; encoding: [0x01,0x04,0xea,0xbe]
 0x01,0x04,0xea,0xbe
+# GFX12: s_brev_b32 vcc_lo, s1                   ; encoding: [0x01,0x04,0xea,0xbe]
 
-# GFX12: s_brev_b64 exec, s[2:3]                 ; encoding: [0x02,0x05,0xfe,0xbe]
 0x02,0x05,0xfe,0xbe
+# GFX12: s_brev_b64 exec, s[2:3]                 ; encoding: [0x02,0x05,0xfe,0xbe]
 
-# GFX12: s_brev_b64 s[0:1], 0.5                  ; encoding: [0xf0,0x05,0x80,0xbe]
 0xf0,0x05,0x80,0xbe
+# GFX12: s_brev_b64 s[0:1], 0.5                  ; encoding: [0xf0,0x05,0x80,0xbe]
 
-# GFX12: s_brev_b64 s[0:1], 0                    ; encoding: [0x80,0x05,0x80,0xbe]
 0x80,0x05,0x80,0xbe
+# GFX12: s_brev_b64 s[0:1], 0                    ; encoding: [0x80,0x05,0x80,0xbe]
 
-# GFX12: s_brev_b64 s[0:1], 0x3f717273           ; encoding: [0xff,0x05,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x05,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_brev_b64 s[0:1], 0x3f717273           ; encoding: [0xff,0x05,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_brev_b64 s[0:1], 0xaf123456           ; encoding: [0xff,0x05,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x05,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_brev_b64 s[0:1], 0xaf123456           ; encoding: [0xff,0x05,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_brev_b64 s[0:1], -1                   ; encoding: [0xc1,0x05,0x80,0xbe]
 0xc1,0x05,0x80,0xbe
+# GFX12: s_brev_b64 s[0:1], -1                   ; encoding: [0xc1,0x05,0x80,0xbe]
 
-# GFX12: s_brev_b64 s[0:1], -4.0                 ; encoding: [0xf7,0x05,0x80,0xbe]
 0xf7,0x05,0x80,0xbe
+# GFX12: s_brev_b64 s[0:1], -4.0                 ; encoding: [0xf7,0x05,0x80,0xbe]
 
-# GFX12: s_brev_b64 s[0:1], exec                 ; encoding: [0x7e,0x05,0x80,0xbe]
 0x7e,0x05,0x80,0xbe
+# GFX12: s_brev_b64 s[0:1], exec                 ; encoding: [0x7e,0x05,0x80,0xbe]
 
-# GFX12: s_brev_b64 s[0:1], s[102:103]           ; encoding: [0x66,0x05,0x80,0xbe]
 0x66,0x05,0x80,0xbe
+# GFX12: s_brev_b64 s[0:1], s[102:103]           ; encoding: [0x66,0x05,0x80,0xbe]
 
-# GFX12: s_brev_b64 s[0:1], s[2:3]               ; encoding: [0x02,0x05,0x80,0xbe]
 0x02,0x05,0x80,0xbe
+# GFX12: s_brev_b64 s[0:1], s[2:3]               ; encoding: [0x02,0x05,0x80,0xbe]
 
-# GFX12: s_brev_b64 s[0:1], vcc                  ; encoding: [0x6a,0x05,0x80,0xbe]
 0x6a,0x05,0x80,0xbe
+# GFX12: s_brev_b64 s[0:1], vcc                  ; encoding: [0x6a,0x05,0x80,0xbe]
 
-# GFX12: s_brev_b64 s[104:105], s[102:103]       ; encoding: [0x66,0x05,0xe8,0xbe]
 0x66,0x05,0xe8,0xbe
+# GFX12: s_brev_b64 s[104:105], s[102:103]       ; encoding: [0x66,0x05,0xe8,0xbe]
 
-# GFX12: s_brev_b64 s[104:105], s[2:3]           ; encoding: [0x02,0x05,0xe8,0xbe]
 0x02,0x05,0xe8,0xbe
+# GFX12: s_brev_b64 s[104:105], s[2:3]           ; encoding: [0x02,0x05,0xe8,0xbe]
 
-# GFX12: s_brev_b64 vcc, s[2:3]                  ; encoding: [0x02,0x05,0xea,0xbe]
 0x02,0x05,0xea,0xbe
+# GFX12: s_brev_b64 vcc, s[2:3]                  ; encoding: [0x02,0x05,0xea,0xbe]
 
-# GFX12: s_cls_i32 exec_hi, s1                   ; encoding: [0x01,0x0c,0xff,0xbe]
 0x01,0x0c,0xff,0xbe
+# GFX12: s_cls_i32 exec_hi, s1                   ; encoding: [0x01,0x0c,0xff,0xbe]
 
-# GFX12: s_cls_i32 exec_lo, s1                   ; encoding: [0x01,0x0c,0xfe,0xbe]
 0x01,0x0c,0xfe,0xbe
+# GFX12: s_cls_i32 exec_lo, s1                   ; encoding: [0x01,0x0c,0xfe,0xbe]
 
-# GFX12: s_cls_i32_i64 exec_hi, s[2:3]           ; encoding: [0x02,0x0d,0xff,0xbe]
 0x02,0x0d,0xff,0xbe
+# GFX12: s_cls_i32_i64 exec_hi, s[2:3]           ; encoding: [0x02,0x0d,0xff,0xbe]
 
-# GFX12: s_cls_i32_i64 exec_lo, s[2:3]           ; encoding: [0x02,0x0d,0xfe,0xbe]
 0x02,0x0d,0xfe,0xbe
+# GFX12: s_cls_i32_i64 exec_lo, s[2:3]           ; encoding: [0x02,0x0d,0xfe,0xbe]
 
-# GFX12: s_cls_i32_i64 m0, s[2:3]                ; encoding: [0x02,0x0d,0xfd,0xbe]
 0x02,0x0d,0xfd,0xbe
+# GFX12: s_cls_i32_i64 m0, s[2:3]                ; encoding: [0x02,0x0d,0xfd,0xbe]
 
-# GFX12: s_cls_i32_i64 s0, 0.5                   ; encoding: [0xf0,0x0d,0x80,0xbe]
 0xf0,0x0d,0x80,0xbe
+# GFX12: s_cls_i32_i64 s0, 0.5                   ; encoding: [0xf0,0x0d,0x80,0xbe]
 
-# GFX12: s_cls_i32_i64 s0, 0                     ; encoding: [0x80,0x0d,0x80,0xbe]
 0x80,0x0d,0x80,0xbe
+# GFX12: s_cls_i32_i64 s0, 0                     ; encoding: [0x80,0x0d,0x80,0xbe]
 
-# GFX12: s_cls_i32_i64 s0, 0x3f717273            ; encoding: [0xff,0x0d,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x0d,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_cls_i32_i64 s0, 0x3f717273            ; encoding: [0xff,0x0d,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cls_i32_i64 s0, 0xaf123456            ; encoding: [0xff,0x0d,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x0d,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_cls_i32_i64 s0, 0xaf123456            ; encoding: [0xff,0x0d,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cls_i32_i64 s0, -1                    ; encoding: [0xc1,0x0d,0x80,0xbe]
 0xc1,0x0d,0x80,0xbe
+# GFX12: s_cls_i32_i64 s0, -1                    ; encoding: [0xc1,0x0d,0x80,0xbe]
 
-# GFX12: s_cls_i32_i64 s0, -4.0                  ; encoding: [0xf7,0x0d,0x80,0xbe]
 0xf7,0x0d,0x80,0xbe
+# GFX12: s_cls_i32_i64 s0, -4.0                  ; encoding: [0xf7,0x0d,0x80,0xbe]
 
-# GFX12: s_cls_i32_i64 s0, exec                  ; encoding: [0x7e,0x0d,0x80,0xbe]
 0x7e,0x0d,0x80,0xbe
+# GFX12: s_cls_i32_i64 s0, exec                  ; encoding: [0x7e,0x0d,0x80,0xbe]
 
-# GFX12: s_cls_i32_i64 s0, s[102:103]            ; encoding: [0x66,0x0d,0x80,0xbe]
 0x66,0x0d,0x80,0xbe
+# GFX12: s_cls_i32_i64 s0, s[102:103]            ; encoding: [0x66,0x0d,0x80,0xbe]
 
-# GFX12: s_cls_i32_i64 s0, s[2:3]                ; encoding: [0x02,0x0d,0x80,0xbe]
 0x02,0x0d,0x80,0xbe
+# GFX12: s_cls_i32_i64 s0, s[2:3]                ; encoding: [0x02,0x0d,0x80,0xbe]
 
-# GFX12: s_cls_i32_i64 s0, vcc                   ; encoding: [0x6a,0x0d,0x80,0xbe]
 0x6a,0x0d,0x80,0xbe
+# GFX12: s_cls_i32_i64 s0, vcc                   ; encoding: [0x6a,0x0d,0x80,0xbe]
 
-# GFX12: s_cls_i32_i64 s105, s[102:103]          ; encoding: [0x66,0x0d,0xe9,0xbe]
 0x66,0x0d,0xe9,0xbe
+# GFX12: s_cls_i32_i64 s105, s[102:103]          ; encoding: [0x66,0x0d,0xe9,0xbe]
 
-# GFX12: s_cls_i32_i64 s105, s[2:3]              ; encoding: [0x02,0x0d,0xe9,0xbe]
 0x02,0x0d,0xe9,0xbe
+# GFX12: s_cls_i32_i64 s105, s[2:3]              ; encoding: [0x02,0x0d,0xe9,0xbe]
 
-# GFX12: s_cls_i32_i64 vcc_hi, s[2:3]            ; encoding: [0x02,0x0d,0xeb,0xbe]
 0x02,0x0d,0xeb,0xbe
+# GFX12: s_cls_i32_i64 vcc_hi, s[2:3]            ; encoding: [0x02,0x0d,0xeb,0xbe]
 
-# GFX12: s_cls_i32_i64 vcc_lo, s[2:3]            ; encoding: [0x02,0x0d,0xea,0xbe]
 0x02,0x0d,0xea,0xbe
+# GFX12: s_cls_i32_i64 vcc_lo, s[2:3]            ; encoding: [0x02,0x0d,0xea,0xbe]
 
-# GFX12: s_cls_i32 m0, s1                        ; encoding: [0x01,0x0c,0xfd,0xbe]
 0x01,0x0c,0xfd,0xbe
+# GFX12: s_cls_i32 m0, s1                        ; encoding: [0x01,0x0c,0xfd,0xbe]
 
-# GFX12: s_cls_i32 s0, 0.5                       ; encoding: [0xf0,0x0c,0x80,0xbe]
 0xf0,0x0c,0x80,0xbe
+# GFX12: s_cls_i32 s0, 0.5                       ; encoding: [0xf0,0x0c,0x80,0xbe]
 
-# GFX12: s_cls_i32 s0, 0                         ; encoding: [0x80,0x0c,0x80,0xbe]
 0x80,0x0c,0x80,0xbe
+# GFX12: s_cls_i32 s0, 0                         ; encoding: [0x80,0x0c,0x80,0xbe]
 
-# GFX12: s_cls_i32 s0, 0x3f717273                ; encoding: [0xff,0x0c,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x0c,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_cls_i32 s0, 0x3f717273                ; encoding: [0xff,0x0c,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cls_i32 s0, 0xaf123456                ; encoding: [0xff,0x0c,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x0c,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_cls_i32 s0, 0xaf123456                ; encoding: [0xff,0x0c,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cls_i32 s0, -1                        ; encoding: [0xc1,0x0c,0x80,0xbe]
 0xc1,0x0c,0x80,0xbe
+# GFX12: s_cls_i32 s0, -1                        ; encoding: [0xc1,0x0c,0x80,0xbe]
 
-# GFX12: s_cls_i32 s0, -4.0                      ; encoding: [0xf7,0x0c,0x80,0xbe]
 0xf7,0x0c,0x80,0xbe
+# GFX12: s_cls_i32 s0, -4.0                      ; encoding: [0xf7,0x0c,0x80,0xbe]
 
-# GFX12: s_cls_i32 s0, exec_hi                   ; encoding: [0x7f,0x0c,0x80,0xbe]
 0x7f,0x0c,0x80,0xbe
+# GFX12: s_cls_i32 s0, exec_hi                   ; encoding: [0x7f,0x0c,0x80,0xbe]
 
-# GFX12: s_cls_i32 s0, exec_lo                   ; encoding: [0x7e,0x0c,0x80,0xbe]
 0x7e,0x0c,0x80,0xbe
+# GFX12: s_cls_i32 s0, exec_lo                   ; encoding: [0x7e,0x0c,0x80,0xbe]
 
-# GFX12: s_cls_i32 s0, m0                        ; encoding: [0x7d,0x0c,0x80,0xbe]
 0x7d,0x0c,0x80,0xbe
+# GFX12: s_cls_i32 s0, m0                        ; encoding: [0x7d,0x0c,0x80,0xbe]
 
-# GFX12: s_cls_i32 s0, s104                      ; encoding: [0x68,0x0c,0x80,0xbe]
 0x68,0x0c,0x80,0xbe
+# GFX12: s_cls_i32 s0, s104                      ; encoding: [0x68,0x0c,0x80,0xbe]
 
-# GFX12: s_cls_i32 s0, s1                        ; encoding: [0x01,0x0c,0x80,0xbe]
 0x01,0x0c,0x80,0xbe
+# GFX12: s_cls_i32 s0, s1                        ; encoding: [0x01,0x0c,0x80,0xbe]
 
-# GFX12: s_cls_i32 s0, vcc_hi                    ; encoding: [0x6b,0x0c,0x80,0xbe]
 0x6b,0x0c,0x80,0xbe
+# GFX12: s_cls_i32 s0, vcc_hi                    ; encoding: [0x6b,0x0c,0x80,0xbe]
 
-# GFX12: s_cls_i32 s0, vcc_lo                    ; encoding: [0x6a,0x0c,0x80,0xbe]
 0x6a,0x0c,0x80,0xbe
+# GFX12: s_cls_i32 s0, vcc_lo                    ; encoding: [0x6a,0x0c,0x80,0xbe]
 
-# GFX12: s_cls_i32 s105, s104                    ; encoding: [0x68,0x0c,0xe9,0xbe]
 0x68,0x0c,0xe9,0xbe
+# GFX12: s_cls_i32 s105, s104                    ; encoding: [0x68,0x0c,0xe9,0xbe]
 
-# GFX12: s_cls_i32 s105, s1                      ; encoding: [0x01,0x0c,0xe9,0xbe]
 0x01,0x0c,0xe9,0xbe
+# GFX12: s_cls_i32 s105, s1                      ; encoding: [0x01,0x0c,0xe9,0xbe]
 
-# GFX12: s_cls_i32 vcc_hi, s1                    ; encoding: [0x01,0x0c,0xeb,0xbe]
 0x01,0x0c,0xeb,0xbe
+# GFX12: s_cls_i32 vcc_hi, s1                    ; encoding: [0x01,0x0c,0xeb,0xbe]
 
-# GFX12: s_cls_i32 vcc_lo, s1                    ; encoding: [0x01,0x0c,0xea,0xbe]
 0x01,0x0c,0xea,0xbe
+# GFX12: s_cls_i32 vcc_lo, s1                    ; encoding: [0x01,0x0c,0xea,0xbe]
 
-# GFX12: s_clz_i32_u32 exec_hi, s1               ; encoding: [0x01,0x0a,0xff,0xbe]
 0x01,0x0a,0xff,0xbe
+# GFX12: s_clz_i32_u32 exec_hi, s1               ; encoding: [0x01,0x0a,0xff,0xbe]
 
-# GFX12: s_clz_i32_u32 exec_lo, s1               ; encoding: [0x01,0x0a,0xfe,0xbe]
 0x01,0x0a,0xfe,0xbe
+# GFX12: s_clz_i32_u32 exec_lo, s1               ; encoding: [0x01,0x0a,0xfe,0xbe]
 
-# GFX12: s_clz_i32_u32 m0, s1                    ; encoding: [0x01,0x0a,0xfd,0xbe]
 0x01,0x0a,0xfd,0xbe
+# GFX12: s_clz_i32_u32 m0, s1                    ; encoding: [0x01,0x0a,0xfd,0xbe]
 
-# GFX12: s_clz_i32_u32 s0, 0.5                   ; encoding: [0xf0,0x0a,0x80,0xbe]
 0xf0,0x0a,0x80,0xbe
+# GFX12: s_clz_i32_u32 s0, 0.5                   ; encoding: [0xf0,0x0a,0x80,0xbe]
 
-# GFX12: s_clz_i32_u32 s0, 0                     ; encoding: [0x80,0x0a,0x80,0xbe]
 0x80,0x0a,0x80,0xbe
+# GFX12: s_clz_i32_u32 s0, 0                     ; encoding: [0x80,0x0a,0x80,0xbe]
 
-# GFX12: s_clz_i32_u32 s0, 0x3f717273            ; encoding: [0xff,0x0a,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x0a,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_clz_i32_u32 s0, 0x3f717273            ; encoding: [0xff,0x0a,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_clz_i32_u32 s0, 0xaf123456            ; encoding: [0xff,0x0a,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x0a,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_clz_i32_u32 s0, 0xaf123456            ; encoding: [0xff,0x0a,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_clz_i32_u32 s0, -1                    ; encoding: [0xc1,0x0a,0x80,0xbe]
 0xc1,0x0a,0x80,0xbe
+# GFX12: s_clz_i32_u32 s0, -1                    ; encoding: [0xc1,0x0a,0x80,0xbe]
 
-# GFX12: s_clz_i32_u32 s0, -4.0                  ; encoding: [0xf7,0x0a,0x80,0xbe]
 0xf7,0x0a,0x80,0xbe
+# GFX12: s_clz_i32_u32 s0, -4.0                  ; encoding: [0xf7,0x0a,0x80,0xbe]
 
-# GFX12: s_clz_i32_u32 s0, exec_hi               ; encoding: [0x7f,0x0a,0x80,0xbe]
 0x7f,0x0a,0x80,0xbe
+# GFX12: s_clz_i32_u32 s0, exec_hi               ; encoding: [0x7f,0x0a,0x80,0xbe]
 
-# GFX12: s_clz_i32_u32 s0, exec_lo               ; encoding: [0x7e,0x0a,0x80,0xbe]
 0x7e,0x0a,0x80,0xbe
+# GFX12: s_clz_i32_u32 s0, exec_lo               ; encoding: [0x7e,0x0a,0x80,0xbe]
 
-# GFX12: s_clz_i32_u32 s0, m0                    ; encoding: [0x7d,0x0a,0x80,0xbe]
 0x7d,0x0a,0x80,0xbe
+# GFX12: s_clz_i32_u32 s0, m0                    ; encoding: [0x7d,0x0a,0x80,0xbe]
 
-# GFX12: s_clz_i32_u32 s0, s104                  ; encoding: [0x68,0x0a,0x80,0xbe]
 0x68,0x0a,0x80,0xbe
+# GFX12: s_clz_i32_u32 s0, s104                  ; encoding: [0x68,0x0a,0x80,0xbe]
 
-# GFX12: s_clz_i32_u32 s0, s1                    ; encoding: [0x01,0x0a,0x80,0xbe]
 0x01,0x0a,0x80,0xbe
+# GFX12: s_clz_i32_u32 s0, s1                    ; encoding: [0x01,0x0a,0x80,0xbe]
 
-# GFX12: s_clz_i32_u32 s0, vcc_hi                ; encoding: [0x6b,0x0a,0x80,0xbe]
 0x6b,0x0a,0x80,0xbe
+# GFX12: s_clz_i32_u32 s0, vcc_hi                ; encoding: [0x6b,0x0a,0x80,0xbe]
 
-# GFX12: s_clz_i32_u32 s0, vcc_lo                ; encoding: [0x6a,0x0a,0x80,0xbe]
 0x6a,0x0a,0x80,0xbe
+# GFX12: s_clz_i32_u32 s0, vcc_lo                ; encoding: [0x6a,0x0a,0x80,0xbe]
 
-# GFX12: s_clz_i32_u32 s105, s104                ; encoding: [0x68,0x0a,0xe9,0xbe]
 0x68,0x0a,0xe9,0xbe
+# GFX12: s_clz_i32_u32 s105, s104                ; encoding: [0x68,0x0a,0xe9,0xbe]
 
-# GFX12: s_clz_i32_u32 s105, s1                  ; encoding: [0x01,0x0a,0xe9,0xbe]
 0x01,0x0a,0xe9,0xbe
+# GFX12: s_clz_i32_u32 s105, s1                  ; encoding: [0x01,0x0a,0xe9,0xbe]
 
-# GFX12: s_clz_i32_u32 vcc_hi, s1                ; encoding: [0x01,0x0a,0xeb,0xbe]
 0x01,0x0a,0xeb,0xbe
+# GFX12: s_clz_i32_u32 vcc_hi, s1                ; encoding: [0x01,0x0a,0xeb,0xbe]
 
-# GFX12: s_clz_i32_u32 vcc_lo, s1                ; encoding: [0x01,0x0a,0xea,0xbe]
 0x01,0x0a,0xea,0xbe
+# GFX12: s_clz_i32_u32 vcc_lo, s1                ; encoding: [0x01,0x0a,0xea,0xbe]
 
-# GFX12: s_clz_i32_u64 exec_hi, s[2:3]           ; encoding: [0x02,0x0b,0xff,0xbe]
 0x02,0x0b,0xff,0xbe
+# GFX12: s_clz_i32_u64 exec_hi, s[2:3]           ; encoding: [0x02,0x0b,0xff,0xbe]
 
-# GFX12: s_clz_i32_u64 exec_lo, s[2:3]           ; encoding: [0x02,0x0b,0xfe,0xbe]
 0x02,0x0b,0xfe,0xbe
+# GFX12: s_clz_i32_u64 exec_lo, s[2:3]           ; encoding: [0x02,0x0b,0xfe,0xbe]
 
-# GFX12: s_clz_i32_u64 m0, s[2:3]                ; encoding: [0x02,0x0b,0xfd,0xbe]
 0x02,0x0b,0xfd,0xbe
+# GFX12: s_clz_i32_u64 m0, s[2:3]                ; encoding: [0x02,0x0b,0xfd,0xbe]
 
-# GFX12: s_clz_i32_u64 s0, 0.5                   ; encoding: [0xf0,0x0b,0x80,0xbe]
 0xf0,0x0b,0x80,0xbe
+# GFX12: s_clz_i32_u64 s0, 0.5                   ; encoding: [0xf0,0x0b,0x80,0xbe]
 
-# GFX12: s_clz_i32_u64 s0, 0                     ; encoding: [0x80,0x0b,0x80,0xbe]
 0x80,0x0b,0x80,0xbe
+# GFX12: s_clz_i32_u64 s0, 0                     ; encoding: [0x80,0x0b,0x80,0xbe]
 
-# GFX12: s_clz_i32_u64 s0, 0x3f717273            ; encoding: [0xff,0x0b,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x0b,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_clz_i32_u64 s0, 0x3f717273            ; encoding: [0xff,0x0b,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_clz_i32_u64 s0, 0xaf123456            ; encoding: [0xff,0x0b,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x0b,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_clz_i32_u64 s0, 0xaf123456            ; encoding: [0xff,0x0b,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_clz_i32_u64 s0, -1                    ; encoding: [0xc1,0x0b,0x80,0xbe]
 0xc1,0x0b,0x80,0xbe
+# GFX12: s_clz_i32_u64 s0, -1                    ; encoding: [0xc1,0x0b,0x80,0xbe]
 
-# GFX12: s_clz_i32_u64 s0, -4.0                  ; encoding: [0xf7,0x0b,0x80,0xbe]
 0xf7,0x0b,0x80,0xbe
+# GFX12: s_clz_i32_u64 s0, -4.0                  ; encoding: [0xf7,0x0b,0x80,0xbe]
 
-# GFX12: s_clz_i32_u64 s0, exec                  ; encoding: [0x7e,0x0b,0x80,0xbe]
 0x7e,0x0b,0x80,0xbe
+# GFX12: s_clz_i32_u64 s0, exec                  ; encoding: [0x7e,0x0b,0x80,0xbe]
 
-# GFX12: s_clz_i32_u64 s0, s[102:103]            ; encoding: [0x66,0x0b,0x80,0xbe]
 0x66,0x0b,0x80,0xbe
+# GFX12: s_clz_i32_u64 s0, s[102:103]            ; encoding: [0x66,0x0b,0x80,0xbe]
 
-# GFX12: s_clz_i32_u64 s0, s[2:3]                ; encoding: [0x02,0x0b,0x80,0xbe]
 0x02,0x0b,0x80,0xbe
+# GFX12: s_clz_i32_u64 s0, s[2:3]                ; encoding: [0x02,0x0b,0x80,0xbe]
 
-# GFX12: s_clz_i32_u64 s0, vcc                   ; encoding: [0x6a,0x0b,0x80,0xbe]
 0x6a,0x0b,0x80,0xbe
+# GFX12: s_clz_i32_u64 s0, vcc                   ; encoding: [0x6a,0x0b,0x80,0xbe]
 
-# GFX12: s_clz_i32_u64 s105, s[102:103]          ; encoding: [0x66,0x0b,0xe9,0xbe]
 0x66,0x0b,0xe9,0xbe
+# GFX12: s_clz_i32_u64 s105, s[102:103]          ; encoding: [0x66,0x0b,0xe9,0xbe]
 
-# GFX12: s_clz_i32_u64 s105, s[2:3]              ; encoding: [0x02,0x0b,0xe9,0xbe]
 0x02,0x0b,0xe9,0xbe
+# GFX12: s_clz_i32_u64 s105, s[2:3]              ; encoding: [0x02,0x0b,0xe9,0xbe]
 
-# GFX12: s_clz_i32_u64 vcc_hi, s[2:3]            ; encoding: [0x02,0x0b,0xeb,0xbe]
 0x02,0x0b,0xeb,0xbe
+# GFX12: s_clz_i32_u64 vcc_hi, s[2:3]            ; encoding: [0x02,0x0b,0xeb,0xbe]
 
-# GFX12: s_clz_i32_u64 vcc_lo, s[2:3]            ; encoding: [0x02,0x0b,0xea,0xbe]
 0x02,0x0b,0xea,0xbe
+# GFX12: s_clz_i32_u64 vcc_lo, s[2:3]            ; encoding: [0x02,0x0b,0xea,0xbe]
 
-# GFX12: s_cmov_b32 exec_hi, s1                  ; encoding: [0x01,0x02,0xff,0xbe]
 0x01,0x02,0xff,0xbe
+# GFX12: s_cmov_b32 exec_hi, s1                  ; encoding: [0x01,0x02,0xff,0xbe]
 
-# GFX12: s_cmov_b32 exec_lo, s1                  ; encoding: [0x01,0x02,0xfe,0xbe]
 0x01,0x02,0xfe,0xbe
+# GFX12: s_cmov_b32 exec_lo, s1                  ; encoding: [0x01,0x02,0xfe,0xbe]
 
-# GFX12: s_cmov_b32 m0, s1                       ; encoding: [0x01,0x02,0xfd,0xbe]
 0x01,0x02,0xfd,0xbe
+# GFX12: s_cmov_b32 m0, s1                       ; encoding: [0x01,0x02,0xfd,0xbe]
 
-# GFX12: s_cmov_b32 s0, 0.5                      ; encoding: [0xf0,0x02,0x80,0xbe]
 0xf0,0x02,0x80,0xbe
+# GFX12: s_cmov_b32 s0, 0.5                      ; encoding: [0xf0,0x02,0x80,0xbe]
 
-# GFX12: s_cmov_b32 s0, 0                        ; encoding: [0x80,0x02,0x80,0xbe]
 0x80,0x02,0x80,0xbe
+# GFX12: s_cmov_b32 s0, 0                        ; encoding: [0x80,0x02,0x80,0xbe]
 
-# GFX12: s_cmov_b32 s0, 0x3f717273               ; encoding: [0xff,0x02,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_cmov_b32 s0, 0x3f717273               ; encoding: [0xff,0x02,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmov_b32 s0, 0xaf123456               ; encoding: [0xff,0x02,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_cmov_b32 s0, 0xaf123456               ; encoding: [0xff,0x02,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmov_b32 s0, -1                       ; encoding: [0xc1,0x02,0x80,0xbe]
 0xc1,0x02,0x80,0xbe
+# GFX12: s_cmov_b32 s0, -1                       ; encoding: [0xc1,0x02,0x80,0xbe]
 
-# GFX12: s_cmov_b32 s0, -4.0                     ; encoding: [0xf7,0x02,0x80,0xbe]
 0xf7,0x02,0x80,0xbe
+# GFX12: s_cmov_b32 s0, -4.0                     ; encoding: [0xf7,0x02,0x80,0xbe]
 
-# GFX12: s_cmov_b32 s0, exec_hi                  ; encoding: [0x7f,0x02,0x80,0xbe]
 0x7f,0x02,0x80,0xbe
+# GFX12: s_cmov_b32 s0, exec_hi                  ; encoding: [0x7f,0x02,0x80,0xbe]
 
-# GFX12: s_cmov_b32 s0, exec_lo                  ; encoding: [0x7e,0x02,0x80,0xbe]
 0x7e,0x02,0x80,0xbe
+# GFX12: s_cmov_b32 s0, exec_lo                  ; encoding: [0x7e,0x02,0x80,0xbe]
 
-# GFX12: s_cmov_b32 s0, m0                       ; encoding: [0x7d,0x02,0x80,0xbe]
 0x7d,0x02,0x80,0xbe
+# GFX12: s_cmov_b32 s0, m0                       ; encoding: [0x7d,0x02,0x80,0xbe]
 
-# GFX12: s_cmov_b32 s0, s104                     ; encoding: [0x68,0x02,0x80,0xbe]
 0x68,0x02,0x80,0xbe
+# GFX12: s_cmov_b32 s0, s104                     ; encoding: [0x68,0x02,0x80,0xbe]
 
-# GFX12: s_cmov_b32 s0, s1                       ; encoding: [0x01,0x02,0x80,0xbe]
 0x01,0x02,0x80,0xbe
+# GFX12: s_cmov_b32 s0, s1                       ; encoding: [0x01,0x02,0x80,0xbe]
 
-# GFX12: s_cmov_b32 s0, vcc_hi                   ; encoding: [0x6b,0x02,0x80,0xbe]
 0x6b,0x02,0x80,0xbe
+# GFX12: s_cmov_b32 s0, vcc_hi                   ; encoding: [0x6b,0x02,0x80,0xbe]
 
-# GFX12: s_cmov_b32 s0, vcc_lo                   ; encoding: [0x6a,0x02,0x80,0xbe]
 0x6a,0x02,0x80,0xbe
+# GFX12: s_cmov_b32 s0, vcc_lo                   ; encoding: [0x6a,0x02,0x80,0xbe]
 
-# GFX12: s_cmov_b32 s105, s104                   ; encoding: [0x68,0x02,0xe9,0xbe]
 0x68,0x02,0xe9,0xbe
+# GFX12: s_cmov_b32 s105, s104                   ; encoding: [0x68,0x02,0xe9,0xbe]
 
-# GFX12: s_cmov_b32 s105, s1                     ; encoding: [0x01,0x02,0xe9,0xbe]
 0x01,0x02,0xe9,0xbe
+# GFX12: s_cmov_b32 s105, s1                     ; encoding: [0x01,0x02,0xe9,0xbe]
 
-# GFX12: s_cmov_b32 vcc_hi, s1                   ; encoding: [0x01,0x02,0xeb,0xbe]
 0x01,0x02,0xeb,0xbe
+# GFX12: s_cmov_b32 vcc_hi, s1                   ; encoding: [0x01,0x02,0xeb,0xbe]
 
-# GFX12: s_cmov_b32 vcc_lo, s1                   ; encoding: [0x01,0x02,0xea,0xbe]
 0x01,0x02,0xea,0xbe
+# GFX12: s_cmov_b32 vcc_lo, s1                   ; encoding: [0x01,0x02,0xea,0xbe]
 
-# GFX12: s_cmov_b64 exec, s[2:3]                 ; encoding: [0x02,0x03,0xfe,0xbe]
 0x02,0x03,0xfe,0xbe
+# GFX12: s_cmov_b64 exec, s[2:3]                 ; encoding: [0x02,0x03,0xfe,0xbe]
 
-# GFX12: s_cmov_b64 s[0:1], 0.5                  ; encoding: [0xf0,0x03,0x80,0xbe]
 0xf0,0x03,0x80,0xbe
+# GFX12: s_cmov_b64 s[0:1], 0.5                  ; encoding: [0xf0,0x03,0x80,0xbe]
 
-# GFX12: s_cmov_b64 s[0:1], 0                    ; encoding: [0x80,0x03,0x80,0xbe]
 0x80,0x03,0x80,0xbe
+# GFX12: s_cmov_b64 s[0:1], 0                    ; encoding: [0x80,0x03,0x80,0xbe]
 
-# GFX12: s_cmov_b64 s[0:1], 0x3f717273           ; encoding: [0xff,0x03,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x03,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_cmov_b64 s[0:1], 0x3f717273           ; encoding: [0xff,0x03,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmov_b64 s[0:1], 0xaf123456           ; encoding: [0xff,0x03,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x03,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_cmov_b64 s[0:1], 0xaf123456           ; encoding: [0xff,0x03,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmov_b64 s[0:1], -1                   ; encoding: [0xc1,0x03,0x80,0xbe]
 0xc1,0x03,0x80,0xbe
+# GFX12: s_cmov_b64 s[0:1], -1                   ; encoding: [0xc1,0x03,0x80,0xbe]
 
-# GFX12: s_cmov_b64 s[0:1], -4.0                 ; encoding: [0xf7,0x03,0x80,0xbe]
 0xf7,0x03,0x80,0xbe
+# GFX12: s_cmov_b64 s[0:1], -4.0                 ; encoding: [0xf7,0x03,0x80,0xbe]
 
-# GFX12: s_cmov_b64 s[0:1], exec                 ; encoding: [0x7e,0x03,0x80,0xbe]
 0x7e,0x03,0x80,0xbe
+# GFX12: s_cmov_b64 s[0:1], exec                 ; encoding: [0x7e,0x03,0x80,0xbe]
 
-# GFX12: s_cmov_b64 s[0:1], s[102:103]           ; encoding: [0x66,0x03,0x80,0xbe]
 0x66,0x03,0x80,0xbe
+# GFX12: s_cmov_b64 s[0:1], s[102:103]           ; encoding: [0x66,0x03,0x80,0xbe]
 
-# GFX12: s_cmov_b64 s[0:1], s[2:3]               ; encoding: [0x02,0x03,0x80,0xbe]
 0x02,0x03,0x80,0xbe
+# GFX12: s_cmov_b64 s[0:1], s[2:3]               ; encoding: [0x02,0x03,0x80,0xbe]
 
-# GFX12: s_cmov_b64 s[0:1], vcc                  ; encoding: [0x6a,0x03,0x80,0xbe]
 0x6a,0x03,0x80,0xbe
+# GFX12: s_cmov_b64 s[0:1], vcc                  ; encoding: [0x6a,0x03,0x80,0xbe]
 
-# GFX12: s_cmov_b64 s[104:105], s[102:103]       ; encoding: [0x66,0x03,0xe8,0xbe]
 0x66,0x03,0xe8,0xbe
+# GFX12: s_cmov_b64 s[104:105], s[102:103]       ; encoding: [0x66,0x03,0xe8,0xbe]
 
-# GFX12: s_cmov_b64 s[104:105], s[2:3]           ; encoding: [0x02,0x03,0xe8,0xbe]
 0x02,0x03,0xe8,0xbe
+# GFX12: s_cmov_b64 s[104:105], s[2:3]           ; encoding: [0x02,0x03,0xe8,0xbe]
 
-# GFX12: s_cmov_b64 vcc, s[2:3]                  ; encoding: [0x02,0x03,0xea,0xbe]
 0x02,0x03,0xea,0xbe
+# GFX12: s_cmov_b64 vcc, s[2:3]                  ; encoding: [0x02,0x03,0xea,0xbe]
 
-# GFX12: s_ctz_i32_b32 exec_hi, s1               ; encoding: [0x01,0x08,0xff,0xbe]
 0x01,0x08,0xff,0xbe
+# GFX12: s_ctz_i32_b32 exec_hi, s1               ; encoding: [0x01,0x08,0xff,0xbe]
 
-# GFX12: s_ctz_i32_b32 exec_lo, s1               ; encoding: [0x01,0x08,0xfe,0xbe]
 0x01,0x08,0xfe,0xbe
+# GFX12: s_ctz_i32_b32 exec_lo, s1               ; encoding: [0x01,0x08,0xfe,0xbe]
 
-# GFX12: s_ctz_i32_b32 m0, s1                    ; encoding: [0x01,0x08,0xfd,0xbe]
 0x01,0x08,0xfd,0xbe
+# GFX12: s_ctz_i32_b32 m0, s1                    ; encoding: [0x01,0x08,0xfd,0xbe]
 
-# GFX12: s_ctz_i32_b32 s0, 0.5                   ; encoding: [0xf0,0x08,0x80,0xbe]
 0xf0,0x08,0x80,0xbe
+# GFX12: s_ctz_i32_b32 s0, 0.5                   ; encoding: [0xf0,0x08,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b32 s0, 0                     ; encoding: [0x80,0x08,0x80,0xbe]
 0x80,0x08,0x80,0xbe
+# GFX12: s_ctz_i32_b32 s0, 0                     ; encoding: [0x80,0x08,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b32 s0, 0x3f717273            ; encoding: [0xff,0x08,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x08,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_ctz_i32_b32 s0, 0x3f717273            ; encoding: [0xff,0x08,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_ctz_i32_b32 s0, 0xaf123456            ; encoding: [0xff,0x08,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x08,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_ctz_i32_b32 s0, 0xaf123456            ; encoding: [0xff,0x08,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_ctz_i32_b32 s0, -1                    ; encoding: [0xc1,0x08,0x80,0xbe]
 0xc1,0x08,0x80,0xbe
+# GFX12: s_ctz_i32_b32 s0, -1                    ; encoding: [0xc1,0x08,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b32 s0, -4.0                  ; encoding: [0xf7,0x08,0x80,0xbe]
 0xf7,0x08,0x80,0xbe
+# GFX12: s_ctz_i32_b32 s0, -4.0                  ; encoding: [0xf7,0x08,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b32 s0, exec_hi               ; encoding: [0x7f,0x08,0x80,0xbe]
 0x7f,0x08,0x80,0xbe
+# GFX12: s_ctz_i32_b32 s0, exec_hi               ; encoding: [0x7f,0x08,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b32 s0, exec_lo               ; encoding: [0x7e,0x08,0x80,0xbe]
 0x7e,0x08,0x80,0xbe
+# GFX12: s_ctz_i32_b32 s0, exec_lo               ; encoding: [0x7e,0x08,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b32 s0, m0                    ; encoding: [0x7d,0x08,0x80,0xbe]
 0x7d,0x08,0x80,0xbe
+# GFX12: s_ctz_i32_b32 s0, m0                    ; encoding: [0x7d,0x08,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b32 s0, s104                  ; encoding: [0x68,0x08,0x80,0xbe]
 0x68,0x08,0x80,0xbe
+# GFX12: s_ctz_i32_b32 s0, s104                  ; encoding: [0x68,0x08,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b32 s0, s1                    ; encoding: [0x01,0x08,0x80,0xbe]
 0x01,0x08,0x80,0xbe
+# GFX12: s_ctz_i32_b32 s0, s1                    ; encoding: [0x01,0x08,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b32 s0, vcc_hi                ; encoding: [0x6b,0x08,0x80,0xbe]
 0x6b,0x08,0x80,0xbe
+# GFX12: s_ctz_i32_b32 s0, vcc_hi                ; encoding: [0x6b,0x08,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b32 s0, vcc_lo                ; encoding: [0x6a,0x08,0x80,0xbe]
 0x6a,0x08,0x80,0xbe
+# GFX12: s_ctz_i32_b32 s0, vcc_lo                ; encoding: [0x6a,0x08,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b32 s105, s104                ; encoding: [0x68,0x08,0xe9,0xbe]
 0x68,0x08,0xe9,0xbe
+# GFX12: s_ctz_i32_b32 s105, s104                ; encoding: [0x68,0x08,0xe9,0xbe]
 
-# GFX12: s_ctz_i32_b32 s105, s1                  ; encoding: [0x01,0x08,0xe9,0xbe]
 0x01,0x08,0xe9,0xbe
+# GFX12: s_ctz_i32_b32 s105, s1                  ; encoding: [0x01,0x08,0xe9,0xbe]
 
-# GFX12: s_ctz_i32_b32 vcc_hi, s1                ; encoding: [0x01,0x08,0xeb,0xbe]
 0x01,0x08,0xeb,0xbe
+# GFX12: s_ctz_i32_b32 vcc_hi, s1                ; encoding: [0x01,0x08,0xeb,0xbe]
 
-# GFX12: s_ctz_i32_b32 vcc_lo, s1                ; encoding: [0x01,0x08,0xea,0xbe]
 0x01,0x08,0xea,0xbe
+# GFX12: s_ctz_i32_b32 vcc_lo, s1                ; encoding: [0x01,0x08,0xea,0xbe]
 
-# GFX12: s_ctz_i32_b64 exec_hi, s[2:3]           ; encoding: [0x02,0x09,0xff,0xbe]
 0x02,0x09,0xff,0xbe
+# GFX12: s_ctz_i32_b64 exec_hi, s[2:3]           ; encoding: [0x02,0x09,0xff,0xbe]
 
-# GFX12: s_ctz_i32_b64 exec_lo, s[2:3]           ; encoding: [0x02,0x09,0xfe,0xbe]
 0x02,0x09,0xfe,0xbe
+# GFX12: s_ctz_i32_b64 exec_lo, s[2:3]           ; encoding: [0x02,0x09,0xfe,0xbe]
 
-# GFX12: s_ctz_i32_b64 m0, s[2:3]                ; encoding: [0x02,0x09,0xfd,0xbe]
 0x02,0x09,0xfd,0xbe
+# GFX12: s_ctz_i32_b64 m0, s[2:3]                ; encoding: [0x02,0x09,0xfd,0xbe]
 
-# GFX12: s_ctz_i32_b64 s0, 0.5                   ; encoding: [0xf0,0x09,0x80,0xbe]
 0xf0,0x09,0x80,0xbe
+# GFX12: s_ctz_i32_b64 s0, 0.5                   ; encoding: [0xf0,0x09,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b64 s0, 0                     ; encoding: [0x80,0x09,0x80,0xbe]
 0x80,0x09,0x80,0xbe
+# GFX12: s_ctz_i32_b64 s0, 0                     ; encoding: [0x80,0x09,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b64 s0, 0x3f717273            ; encoding: [0xff,0x09,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x09,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_ctz_i32_b64 s0, 0x3f717273            ; encoding: [0xff,0x09,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_ctz_i32_b64 s0, 0xaf123456            ; encoding: [0xff,0x09,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x09,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_ctz_i32_b64 s0, 0xaf123456            ; encoding: [0xff,0x09,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_ctz_i32_b64 s0, -1                    ; encoding: [0xc1,0x09,0x80,0xbe]
 0xc1,0x09,0x80,0xbe
+# GFX12: s_ctz_i32_b64 s0, -1                    ; encoding: [0xc1,0x09,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b64 s0, -4.0                  ; encoding: [0xf7,0x09,0x80,0xbe]
 0xf7,0x09,0x80,0xbe
+# GFX12: s_ctz_i32_b64 s0, -4.0                  ; encoding: [0xf7,0x09,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b64 s0, exec                  ; encoding: [0x7e,0x09,0x80,0xbe]
 0x7e,0x09,0x80,0xbe
+# GFX12: s_ctz_i32_b64 s0, exec                  ; encoding: [0x7e,0x09,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b64 s0, s[102:103]            ; encoding: [0x66,0x09,0x80,0xbe]
 0x66,0x09,0x80,0xbe
+# GFX12: s_ctz_i32_b64 s0, s[102:103]            ; encoding: [0x66,0x09,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b64 s0, s[2:3]                ; encoding: [0x02,0x09,0x80,0xbe]
 0x02,0x09,0x80,0xbe
+# GFX12: s_ctz_i32_b64 s0, s[2:3]                ; encoding: [0x02,0x09,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b64 s0, vcc                   ; encoding: [0x6a,0x09,0x80,0xbe]
 0x6a,0x09,0x80,0xbe
+# GFX12: s_ctz_i32_b64 s0, vcc                   ; encoding: [0x6a,0x09,0x80,0xbe]
 
-# GFX12: s_ctz_i32_b64 s105, s[102:103]          ; encoding: [0x66,0x09,0xe9,0xbe]
 0x66,0x09,0xe9,0xbe
+# GFX12: s_ctz_i32_b64 s105, s[102:103]          ; encoding: [0x66,0x09,0xe9,0xbe]
 
-# GFX12: s_ctz_i32_b64 s105, s[2:3]              ; encoding: [0x02,0x09,0xe9,0xbe]
 0x02,0x09,0xe9,0xbe
+# GFX12: s_ctz_i32_b64 s105, s[2:3]              ; encoding: [0x02,0x09,0xe9,0xbe]
 
-# GFX12: s_ctz_i32_b64 vcc_hi, s[2:3]            ; encoding: [0x02,0x09,0xeb,0xbe]
 0x02,0x09,0xeb,0xbe
+# GFX12: s_ctz_i32_b64 vcc_hi, s[2:3]            ; encoding: [0x02,0x09,0xeb,0xbe]
 
-# GFX12: s_ctz_i32_b64 vcc_lo, s[2:3]            ; encoding: [0x02,0x09,0xea,0xbe]
 0x02,0x09,0xea,0xbe
+# GFX12: s_ctz_i32_b64 vcc_lo, s[2:3]            ; encoding: [0x02,0x09,0xea,0xbe]
 
-# GFX12: s_getpc_b64 exec                        ; encoding: [0x00,0x47,0xfe,0xbe]
 0x00,0x47,0xfe,0xbe
+# GFX12: s_getpc_b64 exec                        ; encoding: [0x00,0x47,0xfe,0xbe]
 
-# GFX12: s_getpc_b64 s[0:1]                      ; encoding: [0x00,0x47,0x80,0xbe]
 0x00,0x47,0x80,0xbe
+# GFX12: s_getpc_b64 s[0:1]                      ; encoding: [0x00,0x47,0x80,0xbe]
 
-# GFX12: s_getpc_b64 s[104:105]                  ; encoding: [0x00,0x47,0xe8,0xbe]
 0x00,0x47,0xe8,0xbe
+# GFX12: s_getpc_b64 s[104:105]                  ; encoding: [0x00,0x47,0xe8,0xbe]
 
-# GFX12: s_getpc_b64 vcc                         ; encoding: [0x00,0x47,0xea,0xbe]
 0x00,0x47,0xea,0xbe
+# GFX12: s_getpc_b64 vcc                         ; encoding: [0x00,0x47,0xea,0xbe]
 
-# GFX12: s_mov_b32 exec_hi, s1                   ; encoding: [0x01,0x00,0xff,0xbe]
 0x01,0x00,0xff,0xbe
+# GFX12: s_mov_b32 exec_hi, s1                   ; encoding: [0x01,0x00,0xff,0xbe]
 
-# GFX12: s_mov_b32 exec_lo, s1                   ; encoding: [0x01,0x00,0xfe,0xbe]
 0x01,0x00,0xfe,0xbe
+# GFX12: s_mov_b32 exec_lo, s1                   ; encoding: [0x01,0x00,0xfe,0xbe]
 
-# GFX12: s_mov_b32 m0, s1                        ; encoding: [0x01,0x00,0xfd,0xbe]
 0x01,0x00,0xfd,0xbe
+# GFX12: s_mov_b32 m0, s1                        ; encoding: [0x01,0x00,0xfd,0xbe]
 
-# GFX12: s_mov_b32 s0, 0.5                       ; encoding: [0xf0,0x00,0x80,0xbe]
 0xf0,0x00,0x80,0xbe
+# GFX12: s_mov_b32 s0, 0.5                       ; encoding: [0xf0,0x00,0x80,0xbe]
 
-# GFX12: s_mov_b32 s0, 0                         ; encoding: [0x80,0x00,0x80,0xbe]
 0x80,0x00,0x80,0xbe
+# GFX12: s_mov_b32 s0, 0                         ; encoding: [0x80,0x00,0x80,0xbe]
 
-# GFX12: s_mov_b32 s0, 0x3f717273                ; encoding: [0xff,0x00,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x00,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_mov_b32 s0, 0x3f717273                ; encoding: [0xff,0x00,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_mov_b32 s0, 0xaf123456                ; encoding: [0xff,0x00,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x00,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_mov_b32 s0, 0xaf123456                ; encoding: [0xff,0x00,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_mov_b32 s0, -1                        ; encoding: [0xc1,0x00,0x80,0xbe]
 0xc1,0x00,0x80,0xbe
+# GFX12: s_mov_b32 s0, -1                        ; encoding: [0xc1,0x00,0x80,0xbe]
 
-# GFX12: s_mov_b32 s0, -4.0                      ; encoding: [0xf7,0x00,0x80,0xbe]
 0xf7,0x00,0x80,0xbe
+# GFX12: s_mov_b32 s0, -4.0                      ; encoding: [0xf7,0x00,0x80,0xbe]
 
-# GFX12: s_mov_b32 s0, exec_hi                   ; encoding: [0x7f,0x00,0x80,0xbe]
 0x7f,0x00,0x80,0xbe
+# GFX12: s_mov_b32 s0, exec_hi                   ; encoding: [0x7f,0x00,0x80,0xbe]
 
-# GFX12: s_mov_b32 s0, exec_lo                   ; encoding: [0x7e,0x00,0x80,0xbe]
 0x7e,0x00,0x80,0xbe
+# GFX12: s_mov_b32 s0, exec_lo                   ; encoding: [0x7e,0x00,0x80,0xbe]
 
-# GFX12: s_mov_b32 s0, m0                        ; encoding: [0x7d,0x00,0x80,0xbe]
 0x7d,0x00,0x80,0xbe
+# GFX12: s_mov_b32 s0, m0                        ; encoding: [0x7d,0x00,0x80,0xbe]
 
-# GFX12: s_mov_b32 s0, s104                      ; encoding: [0x68,0x00,0x80,0xbe]
 0x68,0x00,0x80,0xbe
+# GFX12: s_mov_b32 s0, s104                      ; encoding: [0x68,0x00,0x80,0xbe]
 
-# GFX12: s_mov_b32 s0, s1                        ; encoding: [0x01,0x00,0x80,0xbe]
 0x01,0x00,0x80,0xbe
+# GFX12: s_mov_b32 s0, s1                        ; encoding: [0x01,0x00,0x80,0xbe]
 
-# GFX12: s_mov_b32 s0, vcc_hi                    ; encoding: [0x6b,0x00,0x80,0xbe]
 0x6b,0x00,0x80,0xbe
+# GFX12: s_mov_b32 s0, vcc_hi                    ; encoding: [0x6b,0x00,0x80,0xbe]
 
-# GFX12: s_mov_b32 s0, vcc_lo                    ; encoding: [0x6a,0x00,0x80,0xbe]
 0x6a,0x00,0x80,0xbe
+# GFX12: s_mov_b32 s0, vcc_lo                    ; encoding: [0x6a,0x00,0x80,0xbe]
 
-# GFX12: s_mov_b32 s105, s104                    ; encoding: [0x68,0x00,0xe9,0xbe]
 0x68,0x00,0xe9,0xbe
+# GFX12: s_mov_b32 s105, s104                    ; encoding: [0x68,0x00,0xe9,0xbe]
 
-# GFX12: s_mov_b32 s105, s1                      ; encoding: [0x01,0x00,0xe9,0xbe]
 0x01,0x00,0xe9,0xbe
+# GFX12: s_mov_b32 s105, s1                      ; encoding: [0x01,0x00,0xe9,0xbe]
 
-# GFX12: s_mov_b32 vcc_hi, s1                    ; encoding: [0x01,0x00,0xeb,0xbe]
 0x01,0x00,0xeb,0xbe
+# GFX12: s_mov_b32 vcc_hi, s1                    ; encoding: [0x01,0x00,0xeb,0xbe]
 
-# GFX12: s_mov_b32 vcc_lo, s1                    ; encoding: [0x01,0x00,0xea,0xbe]
 0x01,0x00,0xea,0xbe
+# GFX12: s_mov_b32 vcc_lo, s1                    ; encoding: [0x01,0x00,0xea,0xbe]
 
-# GFX12: s_mov_b32 s0, null                      ; encoding: [0x7c,0x00,0x80,0xbe]
 0x7c,0x00,0x80,0xbe
+# GFX12: s_mov_b32 s0, null                      ; encoding: [0x7c,0x00,0x80,0xbe]
 
-# GFX12: s_mov_b32 null, s1                      ; encoding: [0x01,0x00,0xfc,0xbe]
 0x01,0x00,0xfc,0xbe
+# GFX12: s_mov_b32 null, s1                      ; encoding: [0x01,0x00,0xfc,0xbe]
 
-# GFX12: s_mov_b64 exec, s[2:3]                  ; encoding: [0x02,0x01,0xfe,0xbe]
 0x02,0x01,0xfe,0xbe
+# GFX12: s_mov_b64 exec, s[2:3]                  ; encoding: [0x02,0x01,0xfe,0xbe]
 
-# GFX12: s_mov_b64 s[0:1], 0.5                   ; encoding: [0xf0,0x01,0x80,0xbe]
 0xf0,0x01,0x80,0xbe
+# GFX12: s_mov_b64 s[0:1], 0.5                   ; encoding: [0xf0,0x01,0x80,0xbe]
 
-# GFX12: s_mov_b64 s[0:1], 0                     ; encoding: [0x80,0x01,0x80,0xbe]
 0x80,0x01,0x80,0xbe
+# GFX12: s_mov_b64 s[0:1], 0                     ; encoding: [0x80,0x01,0x80,0xbe]
 
-# GFX12: s_mov_b64 s[0:1], 0x3f717273            ; encoding: [0xff,0x01,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x01,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_mov_b64 s[0:1], 0x3f717273            ; encoding: [0xff,0x01,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_mov_b64 s[0:1], 0xaf123456            ; encoding: [0xff,0x01,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x01,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_mov_b64 s[0:1], 0xaf123456            ; encoding: [0xff,0x01,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_mov_b64 s[0:1], -1                    ; encoding: [0xc1,0x01,0x80,0xbe]
 0xc1,0x01,0x80,0xbe
+# GFX12: s_mov_b64 s[0:1], -1                    ; encoding: [0xc1,0x01,0x80,0xbe]
 
-# GFX12: s_mov_b64 s[0:1], -4.0                  ; encoding: [0xf7,0x01,0x80,0xbe]
 0xf7,0x01,0x80,0xbe
+# GFX12: s_mov_b64 s[0:1], -4.0                  ; encoding: [0xf7,0x01,0x80,0xbe]
 
-# GFX12: s_mov_b64 s[0:1], exec                  ; encoding: [0x7e,0x01,0x80,0xbe]
 0x7e,0x01,0x80,0xbe
+# GFX12: s_mov_b64 s[0:1], exec                  ; encoding: [0x7e,0x01,0x80,0xbe]
 
-# GFX12: s_mov_b64 s[0:1], s[102:103]            ; encoding: [0x66,0x01,0x80,0xbe]
 0x66,0x01,0x80,0xbe
+# GFX12: s_mov_b64 s[0:1], s[102:103]            ; encoding: [0x66,0x01,0x80,0xbe]
 
-# GFX12: s_mov_b64 s[0:1], s[2:3]                ; encoding: [0x02,0x01,0x80,0xbe]
 0x02,0x01,0x80,0xbe
+# GFX12: s_mov_b64 s[0:1], s[2:3]                ; encoding: [0x02,0x01,0x80,0xbe]
 
-# GFX12: s_mov_b64 s[0:1], vcc                   ; encoding: [0x6a,0x01,0x80,0xbe]
 0x6a,0x01,0x80,0xbe
+# GFX12: s_mov_b64 s[0:1], vcc                   ; encoding: [0x6a,0x01,0x80,0xbe]
 
-# GFX12: s_mov_b64 s[104:105], s[102:103]        ; encoding: [0x66,0x01,0xe8,0xbe]
 0x66,0x01,0xe8,0xbe
+# GFX12: s_mov_b64 s[104:105], s[102:103]        ; encoding: [0x66,0x01,0xe8,0xbe]
 
-# GFX12: s_mov_b64 s[104:105], s[2:3]            ; encoding: [0x02,0x01,0xe8,0xbe]
 0x02,0x01,0xe8,0xbe
+# GFX12: s_mov_b64 s[104:105], s[2:3]            ; encoding: [0x02,0x01,0xe8,0xbe]
 
-# GFX12: s_mov_b64 vcc, s[2:3]                   ; encoding: [0x02,0x01,0xea,0xbe]
 0x02,0x01,0xea,0xbe
+# GFX12: s_mov_b64 vcc, s[2:3]                   ; encoding: [0x02,0x01,0xea,0xbe]
 
-# GFX12: s_mov_b64 s[0:1], null                  ; encoding: [0x7c,0x01,0x80,0xbe]
 0x7c,0x01,0x80,0xbe
+# GFX12: s_mov_b64 s[0:1], null                  ; encoding: [0x7c,0x01,0x80,0xbe]
 
-# GFX12: s_mov_b64 null, s[2:3]                  ; encoding: [0x02,0x01,0xfc,0xbe]
 0x02,0x01,0xfc,0xbe
+# GFX12: s_mov_b64 null, s[2:3]                  ; encoding: [0x02,0x01,0xfc,0xbe]
 
-# GFX12: s_movreld_b32 s0, 0.5                   ; encoding: [0xf0,0x42,0x80,0xbe]
 0xf0,0x42,0x80,0xbe
+# GFX12: s_movreld_b32 s0, 0.5                   ; encoding: [0xf0,0x42,0x80,0xbe]
 
-# GFX12: s_movreld_b32 s0, 0                     ; encoding: [0x80,0x42,0x80,0xbe]
 0x80,0x42,0x80,0xbe
+# GFX12: s_movreld_b32 s0, 0                     ; encoding: [0x80,0x42,0x80,0xbe]
 
-# GFX12: s_movreld_b32 s0, 0x3f717273            ; encoding: [0xff,0x42,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x42,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_movreld_b32 s0, 0x3f717273            ; encoding: [0xff,0x42,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_movreld_b32 s0, 0xaf123456            ; encoding: [0xff,0x42,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x42,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_movreld_b32 s0, 0xaf123456            ; encoding: [0xff,0x42,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_movreld_b32 s0, -1                    ; encoding: [0xc1,0x42,0x80,0xbe]
 0xc1,0x42,0x80,0xbe
+# GFX12: s_movreld_b32 s0, -1                    ; encoding: [0xc1,0x42,0x80,0xbe]
 
-# GFX12: s_movreld_b32 s0, -4.0                  ; encoding: [0xf7,0x42,0x80,0xbe]
 0xf7,0x42,0x80,0xbe
+# GFX12: s_movreld_b32 s0, -4.0                  ; encoding: [0xf7,0x42,0x80,0xbe]
 
-# GFX12: s_movreld_b32 s0, exec_hi               ; encoding: [0x7f,0x42,0x80,0xbe]
 0x7f,0x42,0x80,0xbe
+# GFX12: s_movreld_b32 s0, exec_hi               ; encoding: [0x7f,0x42,0x80,0xbe]
 
-# GFX12: s_movreld_b32 s0, exec_lo               ; encoding: [0x7e,0x42,0x80,0xbe]
 0x7e,0x42,0x80,0xbe
+# GFX12: s_movreld_b32 s0, exec_lo               ; encoding: [0x7e,0x42,0x80,0xbe]
 
-# GFX12: s_movreld_b32 s0, m0                    ; encoding: [0x7d,0x42,0x80,0xbe]
 0x7d,0x42,0x80,0xbe
+# GFX12: s_movreld_b32 s0, m0                    ; encoding: [0x7d,0x42,0x80,0xbe]
 
-# GFX12: s_movreld_b32 s0, s104                  ; encoding: [0x68,0x42,0x80,0xbe]
 0x68,0x42,0x80,0xbe
+# GFX12: s_movreld_b32 s0, s104                  ; encoding: [0x68,0x42,0x80,0xbe]
 
-# GFX12: s_movreld_b32 s0, s1                    ; encoding: [0x01,0x42,0x80,0xbe]
 0x01,0x42,0x80,0xbe
+# GFX12: s_movreld_b32 s0, s1                    ; encoding: [0x01,0x42,0x80,0xbe]
 
-# GFX12: s_movreld_b32 s0, vcc_hi                ; encoding: [0x6b,0x42,0x80,0xbe]
 0x6b,0x42,0x80,0xbe
+# GFX12: s_movreld_b32 s0, vcc_hi                ; encoding: [0x6b,0x42,0x80,0xbe]
 
-# GFX12: s_movreld_b32 s0, vcc_lo                ; encoding: [0x6a,0x42,0x80,0xbe]
 0x6a,0x42,0x80,0xbe
+# GFX12: s_movreld_b32 s0, vcc_lo                ; encoding: [0x6a,0x42,0x80,0xbe]
 
-# GFX12: s_movreld_b32 s105, s104                ; encoding: [0x68,0x42,0xe9,0xbe]
 0x68,0x42,0xe9,0xbe
+# GFX12: s_movreld_b32 s105, s104                ; encoding: [0x68,0x42,0xe9,0xbe]
 
-# GFX12: s_movreld_b32 s105, s1                  ; encoding: [0x01,0x42,0xe9,0xbe]
 0x01,0x42,0xe9,0xbe
+# GFX12: s_movreld_b32 s105, s1                  ; encoding: [0x01,0x42,0xe9,0xbe]
 
-# GFX12: s_movreld_b32 vcc_hi, s1                ; encoding: [0x01,0x42,0xeb,0xbe]
 0x01,0x42,0xeb,0xbe
+# GFX12: s_movreld_b32 vcc_hi, s1                ; encoding: [0x01,0x42,0xeb,0xbe]
 
-# GFX12: s_movreld_b32 vcc_lo, s1                ; encoding: [0x01,0x42,0xea,0xbe]
 0x01,0x42,0xea,0xbe
+# GFX12: s_movreld_b32 vcc_lo, s1                ; encoding: [0x01,0x42,0xea,0xbe]
 
-# GFX12: s_movreld_b64 s[0:1], 0.5               ; encoding: [0xf0,0x43,0x80,0xbe]
 0xf0,0x43,0x80,0xbe
+# GFX12: s_movreld_b64 s[0:1], 0.5               ; encoding: [0xf0,0x43,0x80,0xbe]
 
-# GFX12: s_movreld_b64 s[0:1], 0                 ; encoding: [0x80,0x43,0x80,0xbe]
 0x80,0x43,0x80,0xbe
+# GFX12: s_movreld_b64 s[0:1], 0                 ; encoding: [0x80,0x43,0x80,0xbe]
 
-# GFX12: s_movreld_b64 s[0:1], 0x3f717273        ; encoding: [0xff,0x43,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x43,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_movreld_b64 s[0:1], 0x3f717273        ; encoding: [0xff,0x43,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_movreld_b64 s[0:1], 0xaf123456        ; encoding: [0xff,0x43,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x43,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_movreld_b64 s[0:1], 0xaf123456        ; encoding: [0xff,0x43,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_movreld_b64 s[0:1], -1                ; encoding: [0xc1,0x43,0x80,0xbe]
 0xc1,0x43,0x80,0xbe
+# GFX12: s_movreld_b64 s[0:1], -1                ; encoding: [0xc1,0x43,0x80,0xbe]
 
-# GFX12: s_movreld_b64 s[0:1], -4.0              ; encoding: [0xf7,0x43,0x80,0xbe]
 0xf7,0x43,0x80,0xbe
+# GFX12: s_movreld_b64 s[0:1], -4.0              ; encoding: [0xf7,0x43,0x80,0xbe]
 
-# GFX12: s_movreld_b64 s[0:1], exec              ; encoding: [0x7e,0x43,0x80,0xbe]
 0x7e,0x43,0x80,0xbe
+# GFX12: s_movreld_b64 s[0:1], exec              ; encoding: [0x7e,0x43,0x80,0xbe]
 
-# GFX12: s_movreld_b64 s[0:1], s[102:103]        ; encoding: [0x66,0x43,0x80,0xbe]
 0x66,0x43,0x80,0xbe
+# GFX12: s_movreld_b64 s[0:1], s[102:103]        ; encoding: [0x66,0x43,0x80,0xbe]
 
-# GFX12: s_movreld_b64 s[0:1], s[2:3]            ; encoding: [0x02,0x43,0x80,0xbe]
 0x02,0x43,0x80,0xbe
+# GFX12: s_movreld_b64 s[0:1], s[2:3]            ; encoding: [0x02,0x43,0x80,0xbe]
 
-# GFX12: s_movreld_b64 s[0:1], vcc               ; encoding: [0x6a,0x43,0x80,0xbe]
 0x6a,0x43,0x80,0xbe
+# GFX12: s_movreld_b64 s[0:1], vcc               ; encoding: [0x6a,0x43,0x80,0xbe]
 
-# GFX12: s_movreld_b64 s[104:105], s[102:103]    ; encoding: [0x66,0x43,0xe8,0xbe]
 0x66,0x43,0xe8,0xbe
+# GFX12: s_movreld_b64 s[104:105], s[102:103]    ; encoding: [0x66,0x43,0xe8,0xbe]
 
-# GFX12: s_movreld_b64 s[104:105], s[2:3]        ; encoding: [0x02,0x43,0xe8,0xbe]
 0x02,0x43,0xe8,0xbe
+# GFX12: s_movreld_b64 s[104:105], s[2:3]        ; encoding: [0x02,0x43,0xe8,0xbe]
 
-# GFX12: s_movreld_b64 vcc, s[2:3]               ; encoding: [0x02,0x43,0xea,0xbe]
 0x02,0x43,0xea,0xbe
+# GFX12: s_movreld_b64 vcc, s[2:3]               ; encoding: [0x02,0x43,0xea,0xbe]
 
-# GFX12: s_movrels_b32 exec_hi, s1               ; encoding: [0x01,0x40,0xff,0xbe]
 0x01,0x40,0xff,0xbe
+# GFX12: s_movrels_b32 exec_hi, s1               ; encoding: [0x01,0x40,0xff,0xbe]
 
-# GFX12: s_movrels_b32 exec_lo, s1               ; encoding: [0x01,0x40,0xfe,0xbe]
 0x01,0x40,0xfe,0xbe
+# GFX12: s_movrels_b32 exec_lo, s1               ; encoding: [0x01,0x40,0xfe,0xbe]
 
-# GFX12: s_movrels_b32 m0, s1                    ; encoding: [0x01,0x40,0xfd,0xbe]
 0x01,0x40,0xfd,0xbe
+# GFX12: s_movrels_b32 m0, s1                    ; encoding: [0x01,0x40,0xfd,0xbe]
 
-# GFX12: s_movrels_b32 s0, s104                  ; encoding: [0x68,0x40,0x80,0xbe]
 0x68,0x40,0x80,0xbe
+# GFX12: s_movrels_b32 s0, s104                  ; encoding: [0x68,0x40,0x80,0xbe]
 
-# GFX12: s_movrels_b32 s0, s1                    ; encoding: [0x01,0x40,0x80,0xbe]
 0x01,0x40,0x80,0xbe
+# GFX12: s_movrels_b32 s0, s1                    ; encoding: [0x01,0x40,0x80,0xbe]
 
-# GFX12: s_movrels_b32 s0, vcc_hi                ; encoding: [0x6b,0x40,0x80,0xbe]
 0x6b,0x40,0x80,0xbe
+# GFX12: s_movrels_b32 s0, vcc_hi                ; encoding: [0x6b,0x40,0x80,0xbe]
 
-# GFX12: s_movrels_b32 s0, vcc_lo                ; encoding: [0x6a,0x40,0x80,0xbe]
 0x6a,0x40,0x80,0xbe
+# GFX12: s_movrels_b32 s0, vcc_lo                ; encoding: [0x6a,0x40,0x80,0xbe]
 
-# GFX12: s_movrels_b32 s105, s104                ; encoding: [0x68,0x40,0xe9,0xbe]
 0x68,0x40,0xe9,0xbe
+# GFX12: s_movrels_b32 s105, s104                ; encoding: [0x68,0x40,0xe9,0xbe]
 
-# GFX12: s_movrels_b32 s105, s1                  ; encoding: [0x01,0x40,0xe9,0xbe]
 0x01,0x40,0xe9,0xbe
+# GFX12: s_movrels_b32 s105, s1                  ; encoding: [0x01,0x40,0xe9,0xbe]
 
-# GFX12: s_movrels_b32 vcc_hi, s1                ; encoding: [0x01,0x40,0xeb,0xbe]
 0x01,0x40,0xeb,0xbe
+# GFX12: s_movrels_b32 vcc_hi, s1                ; encoding: [0x01,0x40,0xeb,0xbe]
 
-# GFX12: s_movrels_b32 vcc_lo, s1                ; encoding: [0x01,0x40,0xea,0xbe]
 0x01,0x40,0xea,0xbe
+# GFX12: s_movrels_b32 vcc_lo, s1                ; encoding: [0x01,0x40,0xea,0xbe]
 
-# GFX12: s_movrels_b64 exec, s[2:3]              ; encoding: [0x02,0x41,0xfe,0xbe]
 0x02,0x41,0xfe,0xbe
+# GFX12: s_movrels_b64 exec, s[2:3]              ; encoding: [0x02,0x41,0xfe,0xbe]
 
-# GFX12: s_movrels_b64 s[0:1], s[102:103]        ; encoding: [0x66,0x41,0x80,0xbe]
 0x66,0x41,0x80,0xbe
+# GFX12: s_movrels_b64 s[0:1], s[102:103]        ; encoding: [0x66,0x41,0x80,0xbe]
 
-# GFX12: s_movrels_b64 s[0:1], s[2:3]            ; encoding: [0x02,0x41,0x80,0xbe]
 0x02,0x41,0x80,0xbe
+# GFX12: s_movrels_b64 s[0:1], s[2:3]            ; encoding: [0x02,0x41,0x80,0xbe]
 
-# GFX12: s_movrels_b64 s[0:1], vcc               ; encoding: [0x6a,0x41,0x80,0xbe]
 0x6a,0x41,0x80,0xbe
+# GFX12: s_movrels_b64 s[0:1], vcc               ; encoding: [0x6a,0x41,0x80,0xbe]
 
-# GFX12: s_movrels_b64 s[104:105], s[102:103]    ; encoding: [0x66,0x41,0xe8,0xbe]
 0x66,0x41,0xe8,0xbe
+# GFX12: s_movrels_b64 s[104:105], s[102:103]    ; encoding: [0x66,0x41,0xe8,0xbe]
 
-# GFX12: s_movrels_b64 s[104:105], s[2:3]        ; encoding: [0x02,0x41,0xe8,0xbe]
 0x02,0x41,0xe8,0xbe
+# GFX12: s_movrels_b64 s[104:105], s[2:3]        ; encoding: [0x02,0x41,0xe8,0xbe]
 
-# GFX12: s_movrels_b64 vcc, s[2:3]               ; encoding: [0x02,0x41,0xea,0xbe]
 0x02,0x41,0xea,0xbe
+# GFX12: s_movrels_b64 vcc, s[2:3]               ; encoding: [0x02,0x41,0xea,0xbe]
 
-# GFX12: s_movrelsd_2_b32 s0, s104               ; encoding: [0x68,0x44,0x80,0xbe]
 0x68,0x44,0x80,0xbe
+# GFX12: s_movrelsd_2_b32 s0, s104               ; encoding: [0x68,0x44,0x80,0xbe]
 
-# GFX12: s_movrelsd_2_b32 s0, s1                 ; encoding: [0x01,0x44,0x80,0xbe]
 0x01,0x44,0x80,0xbe
+# GFX12: s_movrelsd_2_b32 s0, s1                 ; encoding: [0x01,0x44,0x80,0xbe]
 
-# GFX12: s_movrelsd_2_b32 s0, vcc_hi             ; encoding: [0x6b,0x44,0x80,0xbe]
 0x6b,0x44,0x80,0xbe
+# GFX12: s_movrelsd_2_b32 s0, vcc_hi             ; encoding: [0x6b,0x44,0x80,0xbe]
 
-# GFX12: s_movrelsd_2_b32 s0, vcc_lo             ; encoding: [0x6a,0x44,0x80,0xbe]
 0x6a,0x44,0x80,0xbe
+# GFX12: s_movrelsd_2_b32 s0, vcc_lo             ; encoding: [0x6a,0x44,0x80,0xbe]
 
-# GFX12: s_movrelsd_2_b32 s105, s104             ; encoding: [0x68,0x44,0xe9,0xbe]
 0x68,0x44,0xe9,0xbe
+# GFX12: s_movrelsd_2_b32 s105, s104             ; encoding: [0x68,0x44,0xe9,0xbe]
 
-# GFX12: s_movrelsd_2_b32 s105, s1               ; encoding: [0x01,0x44,0xe9,0xbe]
 0x01,0x44,0xe9,0xbe
+# GFX12: s_movrelsd_2_b32 s105, s1               ; encoding: [0x01,0x44,0xe9,0xbe]
 
-# GFX12: s_movrelsd_2_b32 vcc_hi, s1             ; encoding: [0x01,0x44,0xeb,0xbe]
 0x01,0x44,0xeb,0xbe
+# GFX12: s_movrelsd_2_b32 vcc_hi, s1             ; encoding: [0x01,0x44,0xeb,0xbe]
 
-# GFX12: s_movrelsd_2_b32 vcc_lo, s1             ; encoding: [0x01,0x44,0xea,0xbe]
 0x01,0x44,0xea,0xbe
+# GFX12: s_movrelsd_2_b32 vcc_lo, s1             ; encoding: [0x01,0x44,0xea,0xbe]
 
-# GFX12: s_nand_saveexec_b32 s0, 0.5             ; encoding: [0xf0,0x26,0x80,0xbe]
 0xf0,0x26,0x80,0xbe
+# GFX12: s_nand_saveexec_b32 s0, 0.5             ; encoding: [0xf0,0x26,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b32 s0, 0               ; encoding: [0x80,0x26,0x80,0xbe]
 0x80,0x26,0x80,0xbe
+# GFX12: s_nand_saveexec_b32 s0, 0               ; encoding: [0x80,0x26,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b32 s0, 0x3f717273      ; encoding: [0xff,0x26,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x26,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_nand_saveexec_b32 s0, 0x3f717273      ; encoding: [0xff,0x26,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_nand_saveexec_b32 s0, 0xaf123456      ; encoding: [0xff,0x26,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x26,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_nand_saveexec_b32 s0, 0xaf123456      ; encoding: [0xff,0x26,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_nand_saveexec_b32 s0, -1              ; encoding: [0xc1,0x26,0x80,0xbe]
 0xc1,0x26,0x80,0xbe
+# GFX12: s_nand_saveexec_b32 s0, -1              ; encoding: [0xc1,0x26,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b32 s0, -4.0            ; encoding: [0xf7,0x26,0x80,0xbe]
 0xf7,0x26,0x80,0xbe
+# GFX12: s_nand_saveexec_b32 s0, -4.0            ; encoding: [0xf7,0x26,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b32 s0, exec_hi         ; encoding: [0x7f,0x26,0x80,0xbe]
 0x7f,0x26,0x80,0xbe
+# GFX12: s_nand_saveexec_b32 s0, exec_hi         ; encoding: [0x7f,0x26,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b32 s0, exec_lo         ; encoding: [0x7e,0x26,0x80,0xbe]
 0x7e,0x26,0x80,0xbe
+# GFX12: s_nand_saveexec_b32 s0, exec_lo         ; encoding: [0x7e,0x26,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b32 s0, m0              ; encoding: [0x7d,0x26,0x80,0xbe]
 0x7d,0x26,0x80,0xbe
+# GFX12: s_nand_saveexec_b32 s0, m0              ; encoding: [0x7d,0x26,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b32 s0, s104            ; encoding: [0x68,0x26,0x80,0xbe]
 0x68,0x26,0x80,0xbe
+# GFX12: s_nand_saveexec_b32 s0, s104            ; encoding: [0x68,0x26,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b32 s0, s1              ; encoding: [0x01,0x26,0x80,0xbe]
 0x01,0x26,0x80,0xbe
+# GFX12: s_nand_saveexec_b32 s0, s1              ; encoding: [0x01,0x26,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b32 s0, vcc_hi          ; encoding: [0x6b,0x26,0x80,0xbe]
 0x6b,0x26,0x80,0xbe
+# GFX12: s_nand_saveexec_b32 s0, vcc_hi          ; encoding: [0x6b,0x26,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b32 s0, vcc_lo          ; encoding: [0x6a,0x26,0x80,0xbe]
 0x6a,0x26,0x80,0xbe
+# GFX12: s_nand_saveexec_b32 s0, vcc_lo          ; encoding: [0x6a,0x26,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b32 s105, s104          ; encoding: [0x68,0x26,0xe9,0xbe]
 0x68,0x26,0xe9,0xbe
+# GFX12: s_nand_saveexec_b32 s105, s104          ; encoding: [0x68,0x26,0xe9,0xbe]
 
-# GFX12: s_nand_saveexec_b32 s105, s1            ; encoding: [0x01,0x26,0xe9,0xbe]
 0x01,0x26,0xe9,0xbe
+# GFX12: s_nand_saveexec_b32 s105, s1            ; encoding: [0x01,0x26,0xe9,0xbe]
 
-# GFX12: s_nand_saveexec_b32 vcc_hi, s1          ; encoding: [0x01,0x26,0xeb,0xbe]
 0x01,0x26,0xeb,0xbe
+# GFX12: s_nand_saveexec_b32 vcc_hi, s1          ; encoding: [0x01,0x26,0xeb,0xbe]
 
-# GFX12: s_nand_saveexec_b32 vcc_lo, s1          ; encoding: [0x01,0x26,0xea,0xbe]
 0x01,0x26,0xea,0xbe
+# GFX12: s_nand_saveexec_b32 vcc_lo, s1          ; encoding: [0x01,0x26,0xea,0xbe]
 
-# GFX12: s_nand_saveexec_b64 s[0:1], 0.5         ; encoding: [0xf0,0x27,0x80,0xbe]
 0xf0,0x27,0x80,0xbe
+# GFX12: s_nand_saveexec_b64 s[0:1], 0.5         ; encoding: [0xf0,0x27,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b64 s[0:1], 0           ; encoding: [0x80,0x27,0x80,0xbe]
 0x80,0x27,0x80,0xbe
+# GFX12: s_nand_saveexec_b64 s[0:1], 0           ; encoding: [0x80,0x27,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b64 s[0:1], 0x3f717273  ; encoding: [0xff,0x27,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x27,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_nand_saveexec_b64 s[0:1], 0x3f717273  ; encoding: [0xff,0x27,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_nand_saveexec_b64 s[0:1], 0xaf123456  ; encoding: [0xff,0x27,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x27,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_nand_saveexec_b64 s[0:1], 0xaf123456  ; encoding: [0xff,0x27,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_nand_saveexec_b64 s[0:1], -1          ; encoding: [0xc1,0x27,0x80,0xbe]
 0xc1,0x27,0x80,0xbe
+# GFX12: s_nand_saveexec_b64 s[0:1], -1          ; encoding: [0xc1,0x27,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b64 s[0:1], -4.0        ; encoding: [0xf7,0x27,0x80,0xbe]
 0xf7,0x27,0x80,0xbe
+# GFX12: s_nand_saveexec_b64 s[0:1], -4.0        ; encoding: [0xf7,0x27,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b64 s[0:1], exec        ; encoding: [0x7e,0x27,0x80,0xbe]
 0x7e,0x27,0x80,0xbe
+# GFX12: s_nand_saveexec_b64 s[0:1], exec        ; encoding: [0x7e,0x27,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b64 s[0:1], s[102:103]  ; encoding: [0x66,0x27,0x80,0xbe]
 0x66,0x27,0x80,0xbe
+# GFX12: s_nand_saveexec_b64 s[0:1], s[102:103]  ; encoding: [0x66,0x27,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b64 s[0:1], s[2:3]      ; encoding: [0x02,0x27,0x80,0xbe]
 0x02,0x27,0x80,0xbe
+# GFX12: s_nand_saveexec_b64 s[0:1], s[2:3]      ; encoding: [0x02,0x27,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b64 s[0:1], vcc         ; encoding: [0x6a,0x27,0x80,0xbe]
 0x6a,0x27,0x80,0xbe
+# GFX12: s_nand_saveexec_b64 s[0:1], vcc         ; encoding: [0x6a,0x27,0x80,0xbe]
 
-# GFX12: s_nand_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x27,0xe8,0xbe]
 0x66,0x27,0xe8,0xbe
+# GFX12: s_nand_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x27,0xe8,0xbe]
 
-# GFX12: s_nand_saveexec_b64 s[104:105], s[2:3]  ; encoding: [0x02,0x27,0xe8,0xbe]
 0x02,0x27,0xe8,0xbe
+# GFX12: s_nand_saveexec_b64 s[104:105], s[2:3]  ; encoding: [0x02,0x27,0xe8,0xbe]
 
-# GFX12: s_nand_saveexec_b64 vcc, s[2:3]         ; encoding: [0x02,0x27,0xea,0xbe]
 0x02,0x27,0xea,0xbe
+# GFX12: s_nand_saveexec_b64 vcc, s[2:3]         ; encoding: [0x02,0x27,0xea,0xbe]
 
-# GFX12: s_nor_saveexec_b32 s0, 0.5              ; encoding: [0xf0,0x28,0x80,0xbe]
 0xf0,0x28,0x80,0xbe
+# GFX12: s_nor_saveexec_b32 s0, 0.5              ; encoding: [0xf0,0x28,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b32 s0, 0                ; encoding: [0x80,0x28,0x80,0xbe]
 0x80,0x28,0x80,0xbe
+# GFX12: s_nor_saveexec_b32 s0, 0                ; encoding: [0x80,0x28,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b32 s0, 0x3f717273       ; encoding: [0xff,0x28,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x28,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_nor_saveexec_b32 s0, 0x3f717273       ; encoding: [0xff,0x28,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_nor_saveexec_b32 s0, 0xaf123456       ; encoding: [0xff,0x28,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x28,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_nor_saveexec_b32 s0, 0xaf123456       ; encoding: [0xff,0x28,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_nor_saveexec_b32 s0, -1               ; encoding: [0xc1,0x28,0x80,0xbe]
 0xc1,0x28,0x80,0xbe
+# GFX12: s_nor_saveexec_b32 s0, -1               ; encoding: [0xc1,0x28,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b32 s0, -4.0             ; encoding: [0xf7,0x28,0x80,0xbe]
 0xf7,0x28,0x80,0xbe
+# GFX12: s_nor_saveexec_b32 s0, -4.0             ; encoding: [0xf7,0x28,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b32 s0, exec_hi          ; encoding: [0x7f,0x28,0x80,0xbe]
 0x7f,0x28,0x80,0xbe
+# GFX12: s_nor_saveexec_b32 s0, exec_hi          ; encoding: [0x7f,0x28,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b32 s0, exec_lo          ; encoding: [0x7e,0x28,0x80,0xbe]
 0x7e,0x28,0x80,0xbe
+# GFX12: s_nor_saveexec_b32 s0, exec_lo          ; encoding: [0x7e,0x28,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b32 s0, m0               ; encoding: [0x7d,0x28,0x80,0xbe]
 0x7d,0x28,0x80,0xbe
+# GFX12: s_nor_saveexec_b32 s0, m0               ; encoding: [0x7d,0x28,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b32 s0, s104             ; encoding: [0x68,0x28,0x80,0xbe]
 0x68,0x28,0x80,0xbe
+# GFX12: s_nor_saveexec_b32 s0, s104             ; encoding: [0x68,0x28,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b32 s0, s1               ; encoding: [0x01,0x28,0x80,0xbe]
 0x01,0x28,0x80,0xbe
+# GFX12: s_nor_saveexec_b32 s0, s1               ; encoding: [0x01,0x28,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b32 s0, vcc_hi           ; encoding: [0x6b,0x28,0x80,0xbe]
 0x6b,0x28,0x80,0xbe
+# GFX12: s_nor_saveexec_b32 s0, vcc_hi           ; encoding: [0x6b,0x28,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b32 s0, vcc_lo           ; encoding: [0x6a,0x28,0x80,0xbe]
 0x6a,0x28,0x80,0xbe
+# GFX12: s_nor_saveexec_b32 s0, vcc_lo           ; encoding: [0x6a,0x28,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b32 s105, s104           ; encoding: [0x68,0x28,0xe9,0xbe]
 0x68,0x28,0xe9,0xbe
+# GFX12: s_nor_saveexec_b32 s105, s104           ; encoding: [0x68,0x28,0xe9,0xbe]
 
-# GFX12: s_nor_saveexec_b32 s105, s1             ; encoding: [0x01,0x28,0xe9,0xbe]
 0x01,0x28,0xe9,0xbe
+# GFX12: s_nor_saveexec_b32 s105, s1             ; encoding: [0x01,0x28,0xe9,0xbe]
 
-# GFX12: s_nor_saveexec_b32 vcc_hi, s1           ; encoding: [0x01,0x28,0xeb,0xbe]
 0x01,0x28,0xeb,0xbe
+# GFX12: s_nor_saveexec_b32 vcc_hi, s1           ; encoding: [0x01,0x28,0xeb,0xbe]
 
-# GFX12: s_nor_saveexec_b32 vcc_lo, s1           ; encoding: [0x01,0x28,0xea,0xbe]
 0x01,0x28,0xea,0xbe
+# GFX12: s_nor_saveexec_b32 vcc_lo, s1           ; encoding: [0x01,0x28,0xea,0xbe]
 
-# GFX12: s_nor_saveexec_b64 s[0:1], 0.5          ; encoding: [0xf0,0x29,0x80,0xbe]
 0xf0,0x29,0x80,0xbe
+# GFX12: s_nor_saveexec_b64 s[0:1], 0.5          ; encoding: [0xf0,0x29,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b64 s[0:1], 0            ; encoding: [0x80,0x29,0x80,0xbe]
 0x80,0x29,0x80,0xbe
+# GFX12: s_nor_saveexec_b64 s[0:1], 0            ; encoding: [0x80,0x29,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b64 s[0:1], 0x3f717273   ; encoding: [0xff,0x29,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x29,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_nor_saveexec_b64 s[0:1], 0x3f717273   ; encoding: [0xff,0x29,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_nor_saveexec_b64 s[0:1], 0xaf123456   ; encoding: [0xff,0x29,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x29,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_nor_saveexec_b64 s[0:1], 0xaf123456   ; encoding: [0xff,0x29,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_nor_saveexec_b64 s[0:1], -1           ; encoding: [0xc1,0x29,0x80,0xbe]
 0xc1,0x29,0x80,0xbe
+# GFX12: s_nor_saveexec_b64 s[0:1], -1           ; encoding: [0xc1,0x29,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b64 s[0:1], -4.0         ; encoding: [0xf7,0x29,0x80,0xbe]
 0xf7,0x29,0x80,0xbe
+# GFX12: s_nor_saveexec_b64 s[0:1], -4.0         ; encoding: [0xf7,0x29,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b64 s[0:1], exec         ; encoding: [0x7e,0x29,0x80,0xbe]
 0x7e,0x29,0x80,0xbe
+# GFX12: s_nor_saveexec_b64 s[0:1], exec         ; encoding: [0x7e,0x29,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b64 s[0:1], s[102:103]   ; encoding: [0x66,0x29,0x80,0xbe]
 0x66,0x29,0x80,0xbe
+# GFX12: s_nor_saveexec_b64 s[0:1], s[102:103]   ; encoding: [0x66,0x29,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b64 s[0:1], s[2:3]       ; encoding: [0x02,0x29,0x80,0xbe]
 0x02,0x29,0x80,0xbe
+# GFX12: s_nor_saveexec_b64 s[0:1], s[2:3]       ; encoding: [0x02,0x29,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b64 s[0:1], vcc          ; encoding: [0x6a,0x29,0x80,0xbe]
 0x6a,0x29,0x80,0xbe
+# GFX12: s_nor_saveexec_b64 s[0:1], vcc          ; encoding: [0x6a,0x29,0x80,0xbe]
 
-# GFX12: s_nor_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x29,0xe8,0xbe]
 0x66,0x29,0xe8,0xbe
+# GFX12: s_nor_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x29,0xe8,0xbe]
 
-# GFX12: s_nor_saveexec_b64 s[104:105], s[2:3]   ; encoding: [0x02,0x29,0xe8,0xbe]
 0x02,0x29,0xe8,0xbe
+# GFX12: s_nor_saveexec_b64 s[104:105], s[2:3]   ; encoding: [0x02,0x29,0xe8,0xbe]
 
-# GFX12: s_nor_saveexec_b64 vcc, s[2:3]          ; encoding: [0x02,0x29,0xea,0xbe]
 0x02,0x29,0xea,0xbe
+# GFX12: s_nor_saveexec_b64 vcc, s[2:3]          ; encoding: [0x02,0x29,0xea,0xbe]
 
-# GFX12: s_not_b32 exec_hi, s1                   ; encoding: [0x01,0x1e,0xff,0xbe]
 0x01,0x1e,0xff,0xbe
+# GFX12: s_not_b32 exec_hi, s1                   ; encoding: [0x01,0x1e,0xff,0xbe]
 
-# GFX12: s_not_b32 exec_lo, s1                   ; encoding: [0x01,0x1e,0xfe,0xbe]
 0x01,0x1e,0xfe,0xbe
+# GFX12: s_not_b32 exec_lo, s1                   ; encoding: [0x01,0x1e,0xfe,0xbe]
 
-# GFX12: s_not_b32 m0, s1                        ; encoding: [0x01,0x1e,0xfd,0xbe]
 0x01,0x1e,0xfd,0xbe
+# GFX12: s_not_b32 m0, s1                        ; encoding: [0x01,0x1e,0xfd,0xbe]
 
-# GFX12: s_not_b32 s0, 0.5                       ; encoding: [0xf0,0x1e,0x80,0xbe]
 0xf0,0x1e,0x80,0xbe
+# GFX12: s_not_b32 s0, 0.5                       ; encoding: [0xf0,0x1e,0x80,0xbe]
 
-# GFX12: s_not_b32 s0, 0                         ; encoding: [0x80,0x1e,0x80,0xbe]
 0x80,0x1e,0x80,0xbe
+# GFX12: s_not_b32 s0, 0                         ; encoding: [0x80,0x1e,0x80,0xbe]
 
-# GFX12: s_not_b32 s0, 0x3f717273                ; encoding: [0xff,0x1e,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x1e,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_not_b32 s0, 0x3f717273                ; encoding: [0xff,0x1e,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_not_b32 s0, 0xaf123456                ; encoding: [0xff,0x1e,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x1e,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_not_b32 s0, 0xaf123456                ; encoding: [0xff,0x1e,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_not_b32 s0, -1                        ; encoding: [0xc1,0x1e,0x80,0xbe]
 0xc1,0x1e,0x80,0xbe
+# GFX12: s_not_b32 s0, -1                        ; encoding: [0xc1,0x1e,0x80,0xbe]
 
-# GFX12: s_not_b32 s0, -4.0                      ; encoding: [0xf7,0x1e,0x80,0xbe]
 0xf7,0x1e,0x80,0xbe
+# GFX12: s_not_b32 s0, -4.0                      ; encoding: [0xf7,0x1e,0x80,0xbe]
 
-# GFX12: s_not_b32 s0, exec_hi                   ; encoding: [0x7f,0x1e,0x80,0xbe]
 0x7f,0x1e,0x80,0xbe
+# GFX12: s_not_b32 s0, exec_hi                   ; encoding: [0x7f,0x1e,0x80,0xbe]
 
-# GFX12: s_not_b32 s0, exec_lo                   ; encoding: [0x7e,0x1e,0x80,0xbe]
 0x7e,0x1e,0x80,0xbe
+# GFX12: s_not_b32 s0, exec_lo                   ; encoding: [0x7e,0x1e,0x80,0xbe]
 
-# GFX12: s_not_b32 s0, m0                        ; encoding: [0x7d,0x1e,0x80,0xbe]
 0x7d,0x1e,0x80,0xbe
+# GFX12: s_not_b32 s0, m0                        ; encoding: [0x7d,0x1e,0x80,0xbe]
 
-# GFX12: s_not_b32 s0, s104                      ; encoding: [0x68,0x1e,0x80,0xbe]
 0x68,0x1e,0x80,0xbe
+# GFX12: s_not_b32 s0, s104                      ; encoding: [0x68,0x1e,0x80,0xbe]
 
-# GFX12: s_not_b32 s0, s1                        ; encoding: [0x01,0x1e,0x80,0xbe]
 0x01,0x1e,0x80,0xbe
+# GFX12: s_not_b32 s0, s1                        ; encoding: [0x01,0x1e,0x80,0xbe]
 
-# GFX12: s_not_b32 s0, vcc_hi                    ; encoding: [0x6b,0x1e,0x80,0xbe]
 0x6b,0x1e,0x80,0xbe
+# GFX12: s_not_b32 s0, vcc_hi                    ; encoding: [0x6b,0x1e,0x80,0xbe]
 
-# GFX12: s_not_b32 s0, vcc_lo                    ; encoding: [0x6a,0x1e,0x80,0xbe]
 0x6a,0x1e,0x80,0xbe
+# GFX12: s_not_b32 s0, vcc_lo                    ; encoding: [0x6a,0x1e,0x80,0xbe]
 
-# GFX12: s_not_b32 s105, s104                    ; encoding: [0x68,0x1e,0xe9,0xbe]
 0x68,0x1e,0xe9,0xbe
+# GFX12: s_not_b32 s105, s104                    ; encoding: [0x68,0x1e,0xe9,0xbe]
 
-# GFX12: s_not_b32 s105, s1                      ; encoding: [0x01,0x1e,0xe9,0xbe]
 0x01,0x1e,0xe9,0xbe
+# GFX12: s_not_b32 s105, s1                      ; encoding: [0x01,0x1e,0xe9,0xbe]
 
-# GFX12: s_not_b32 vcc_hi, s1                    ; encoding: [0x01,0x1e,0xeb,0xbe]
 0x01,0x1e,0xeb,0xbe
+# GFX12: s_not_b32 vcc_hi, s1                    ; encoding: [0x01,0x1e,0xeb,0xbe]
 
-# GFX12: s_not_b32 vcc_lo, s1                    ; encoding: [0x01,0x1e,0xea,0xbe]
 0x01,0x1e,0xea,0xbe
+# GFX12: s_not_b32 vcc_lo, s1                    ; encoding: [0x01,0x1e,0xea,0xbe]
 
-# GFX12: s_not_b64 exec, s[2:3]                  ; encoding: [0x02,0x1f,0xfe,0xbe]
 0x02,0x1f,0xfe,0xbe
+# GFX12: s_not_b64 exec, s[2:3]                  ; encoding: [0x02,0x1f,0xfe,0xbe]
 
-# GFX12: s_not_b64 s[0:1], 0.5                   ; encoding: [0xf0,0x1f,0x80,0xbe]
 0xf0,0x1f,0x80,0xbe
+# GFX12: s_not_b64 s[0:1], 0.5                   ; encoding: [0xf0,0x1f,0x80,0xbe]
 
-# GFX12: s_not_b64 s[0:1], 0                     ; encoding: [0x80,0x1f,0x80,0xbe]
 0x80,0x1f,0x80,0xbe
+# GFX12: s_not_b64 s[0:1], 0                     ; encoding: [0x80,0x1f,0x80,0xbe]
 
-# GFX12: s_not_b64 s[0:1], 0x3f717273            ; encoding: [0xff,0x1f,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x1f,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_not_b64 s[0:1], 0x3f717273            ; encoding: [0xff,0x1f,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_not_b64 s[0:1], 0xaf123456            ; encoding: [0xff,0x1f,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x1f,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_not_b64 s[0:1], 0xaf123456            ; encoding: [0xff,0x1f,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_not_b64 s[0:1], -1                    ; encoding: [0xc1,0x1f,0x80,0xbe]
 0xc1,0x1f,0x80,0xbe
+# GFX12: s_not_b64 s[0:1], -1                    ; encoding: [0xc1,0x1f,0x80,0xbe]
 
-# GFX12: s_not_b64 s[0:1], -4.0                  ; encoding: [0xf7,0x1f,0x80,0xbe]
 0xf7,0x1f,0x80,0xbe
+# GFX12: s_not_b64 s[0:1], -4.0                  ; encoding: [0xf7,0x1f,0x80,0xbe]
 
-# GFX12: s_not_b64 s[0:1], exec                  ; encoding: [0x7e,0x1f,0x80,0xbe]
 0x7e,0x1f,0x80,0xbe
+# GFX12: s_not_b64 s[0:1], exec                  ; encoding: [0x7e,0x1f,0x80,0xbe]
 
-# GFX12: s_not_b64 s[0:1], s[102:103]            ; encoding: [0x66,0x1f,0x80,0xbe]
 0x66,0x1f,0x80,0xbe
+# GFX12: s_not_b64 s[0:1], s[102:103]            ; encoding: [0x66,0x1f,0x80,0xbe]
 
-# GFX12: s_not_b64 s[0:1], s[2:3]                ; encoding: [0x02,0x1f,0x80,0xbe]
 0x02,0x1f,0x80,0xbe
+# GFX12: s_not_b64 s[0:1], s[2:3]                ; encoding: [0x02,0x1f,0x80,0xbe]
 
-# GFX12: s_not_b64 s[0:1], vcc                   ; encoding: [0x6a,0x1f,0x80,0xbe]
 0x6a,0x1f,0x80,0xbe
+# GFX12: s_not_b64 s[0:1], vcc                   ; encoding: [0x6a,0x1f,0x80,0xbe]
 
-# GFX12: s_not_b64 s[104:105], s[102:103]        ; encoding: [0x66,0x1f,0xe8,0xbe]
 0x66,0x1f,0xe8,0xbe
+# GFX12: s_not_b64 s[104:105], s[102:103]        ; encoding: [0x66,0x1f,0xe8,0xbe]
 
-# GFX12: s_not_b64 s[104:105], s[2:3]            ; encoding: [0x02,0x1f,0xe8,0xbe]
 0x02,0x1f,0xe8,0xbe
+# GFX12: s_not_b64 s[104:105], s[2:3]            ; encoding: [0x02,0x1f,0xe8,0xbe]
 
-# GFX12: s_not_b64 vcc, s[2:3]                   ; encoding: [0x02,0x1f,0xea,0xbe]
 0x02,0x1f,0xea,0xbe
+# GFX12: s_not_b64 vcc, s[2:3]                   ; encoding: [0x02,0x1f,0xea,0xbe]
 
-# GFX12: s_or_not0_saveexec_b32 s0, 0.5          ; encoding: [0xf0,0x2e,0x80,0xbe]
 0xf0,0x2e,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b32 s0, 0.5          ; encoding: [0xf0,0x2e,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b32 s0, 0            ; encoding: [0x80,0x2e,0x80,0xbe]
 0x80,0x2e,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b32 s0, 0            ; encoding: [0x80,0x2e,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b32 s0, 0x3f717273   ; encoding: [0xff,0x2e,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x2e,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_or_not0_saveexec_b32 s0, 0x3f717273   ; encoding: [0xff,0x2e,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_or_not0_saveexec_b32 s0, 0xaf123456   ; encoding: [0xff,0x2e,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x2e,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_or_not0_saveexec_b32 s0, 0xaf123456   ; encoding: [0xff,0x2e,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_or_not0_saveexec_b32 s0, -1           ; encoding: [0xc1,0x2e,0x80,0xbe]
 0xc1,0x2e,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b32 s0, -1           ; encoding: [0xc1,0x2e,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b32 s0, -4.0         ; encoding: [0xf7,0x2e,0x80,0xbe]
 0xf7,0x2e,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b32 s0, -4.0         ; encoding: [0xf7,0x2e,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b32 s0, exec_hi      ; encoding: [0x7f,0x2e,0x80,0xbe]
 0x7f,0x2e,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b32 s0, exec_hi      ; encoding: [0x7f,0x2e,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b32 s0, exec_lo      ; encoding: [0x7e,0x2e,0x80,0xbe]
 0x7e,0x2e,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b32 s0, exec_lo      ; encoding: [0x7e,0x2e,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b32 s0, m0           ; encoding: [0x7d,0x2e,0x80,0xbe]
 0x7d,0x2e,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b32 s0, m0           ; encoding: [0x7d,0x2e,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b32 s0, s104         ; encoding: [0x68,0x2e,0x80,0xbe]
 0x68,0x2e,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b32 s0, s104         ; encoding: [0x68,0x2e,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b32 s0, s1           ; encoding: [0x01,0x2e,0x80,0xbe]
 0x01,0x2e,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b32 s0, s1           ; encoding: [0x01,0x2e,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b32 s0, vcc_hi       ; encoding: [0x6b,0x2e,0x80,0xbe]
 0x6b,0x2e,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b32 s0, vcc_hi       ; encoding: [0x6b,0x2e,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b32 s0, vcc_lo       ; encoding: [0x6a,0x2e,0x80,0xbe]
 0x6a,0x2e,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b32 s0, vcc_lo       ; encoding: [0x6a,0x2e,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b32 s105, s104       ; encoding: [0x68,0x2e,0xe9,0xbe]
 0x68,0x2e,0xe9,0xbe
+# GFX12: s_or_not0_saveexec_b32 s105, s104       ; encoding: [0x68,0x2e,0xe9,0xbe]
 
-# GFX12: s_or_not0_saveexec_b32 s105, s1         ; encoding: [0x01,0x2e,0xe9,0xbe]
 0x01,0x2e,0xe9,0xbe
+# GFX12: s_or_not0_saveexec_b32 s105, s1         ; encoding: [0x01,0x2e,0xe9,0xbe]
 
-# GFX12: s_or_not0_saveexec_b32 vcc_hi, s1       ; encoding: [0x01,0x2e,0xeb,0xbe]
 0x01,0x2e,0xeb,0xbe
+# GFX12: s_or_not0_saveexec_b32 vcc_hi, s1       ; encoding: [0x01,0x2e,0xeb,0xbe]
 
-# GFX12: s_or_not0_saveexec_b32 vcc_lo, s1       ; encoding: [0x01,0x2e,0xea,0xbe]
 0x01,0x2e,0xea,0xbe
+# GFX12: s_or_not0_saveexec_b32 vcc_lo, s1       ; encoding: [0x01,0x2e,0xea,0xbe]
 
-# GFX12: s_or_not0_saveexec_b64 s[0:1], 0.5      ; encoding: [0xf0,0x2f,0x80,0xbe]
 0xf0,0x2f,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b64 s[0:1], 0.5      ; encoding: [0xf0,0x2f,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b64 s[0:1], 0        ; encoding: [0x80,0x2f,0x80,0xbe]
 0x80,0x2f,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b64 s[0:1], 0        ; encoding: [0x80,0x2f,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b64 s[0:1], 0x3f717273 ; encoding: [0xff,0x2f,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x2f,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_or_not0_saveexec_b64 s[0:1], 0x3f717273 ; encoding: [0xff,0x2f,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_or_not0_saveexec_b64 s[0:1], 0xaf123456 ; encoding: [0xff,0x2f,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x2f,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_or_not0_saveexec_b64 s[0:1], 0xaf123456 ; encoding: [0xff,0x2f,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_or_not0_saveexec_b64 s[0:1], -1       ; encoding: [0xc1,0x2f,0x80,0xbe]
 0xc1,0x2f,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b64 s[0:1], -1       ; encoding: [0xc1,0x2f,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b64 s[0:1], -4.0     ; encoding: [0xf7,0x2f,0x80,0xbe]
 0xf7,0x2f,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b64 s[0:1], -4.0     ; encoding: [0xf7,0x2f,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b64 s[0:1], exec     ; encoding: [0x7e,0x2f,0x80,0xbe]
 0x7e,0x2f,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b64 s[0:1], exec     ; encoding: [0x7e,0x2f,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b64 s[0:1], s[102:103] ; encoding: [0x66,0x2f,0x80,0xbe]
 0x66,0x2f,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b64 s[0:1], s[102:103] ; encoding: [0x66,0x2f,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b64 s[0:1], s[2:3]   ; encoding: [0x02,0x2f,0x80,0xbe]
 0x02,0x2f,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b64 s[0:1], s[2:3]   ; encoding: [0x02,0x2f,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b64 s[0:1], vcc      ; encoding: [0x6a,0x2f,0x80,0xbe]
 0x6a,0x2f,0x80,0xbe
+# GFX12: s_or_not0_saveexec_b64 s[0:1], vcc      ; encoding: [0x6a,0x2f,0x80,0xbe]
 
-# GFX12: s_or_not0_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x2f,0xe8,0xbe]
 0x66,0x2f,0xe8,0xbe
+# GFX12: s_or_not0_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x2f,0xe8,0xbe]
 
-# GFX12: s_or_not0_saveexec_b64 s[104:105], s[2:3] ; encoding: [0x02,0x2f,0xe8,0xbe]
 0x02,0x2f,0xe8,0xbe
+# GFX12: s_or_not0_saveexec_b64 s[104:105], s[2:3] ; encoding: [0x02,0x2f,0xe8,0xbe]
 
-# GFX12: s_or_not0_saveexec_b64 vcc, s[2:3]      ; encoding: [0x02,0x2f,0xea,0xbe]
 0x02,0x2f,0xea,0xbe
+# GFX12: s_or_not0_saveexec_b64 vcc, s[2:3]      ; encoding: [0x02,0x2f,0xea,0xbe]
 
-# GFX12: s_or_not1_saveexec_b32 s0, 0.5          ; encoding: [0xf0,0x32,0x80,0xbe]
 0xf0,0x32,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b32 s0, 0.5          ; encoding: [0xf0,0x32,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b32 s0, 0            ; encoding: [0x80,0x32,0x80,0xbe]
 0x80,0x32,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b32 s0, 0            ; encoding: [0x80,0x32,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b32 s0, 0x3f717273   ; encoding: [0xff,0x32,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x32,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_or_not1_saveexec_b32 s0, 0x3f717273   ; encoding: [0xff,0x32,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_or_not1_saveexec_b32 s0, 0xaf123456   ; encoding: [0xff,0x32,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x32,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_or_not1_saveexec_b32 s0, 0xaf123456   ; encoding: [0xff,0x32,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_or_not1_saveexec_b32 s0, -1           ; encoding: [0xc1,0x32,0x80,0xbe]
 0xc1,0x32,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b32 s0, -1           ; encoding: [0xc1,0x32,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b32 s0, -4.0         ; encoding: [0xf7,0x32,0x80,0xbe]
 0xf7,0x32,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b32 s0, -4.0         ; encoding: [0xf7,0x32,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b32 s0, exec_hi      ; encoding: [0x7f,0x32,0x80,0xbe]
 0x7f,0x32,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b32 s0, exec_hi      ; encoding: [0x7f,0x32,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b32 s0, exec_lo      ; encoding: [0x7e,0x32,0x80,0xbe]
 0x7e,0x32,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b32 s0, exec_lo      ; encoding: [0x7e,0x32,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b32 s0, m0           ; encoding: [0x7d,0x32,0x80,0xbe]
 0x7d,0x32,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b32 s0, m0           ; encoding: [0x7d,0x32,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b32 s0, s104         ; encoding: [0x68,0x32,0x80,0xbe]
 0x68,0x32,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b32 s0, s104         ; encoding: [0x68,0x32,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b32 s0, s1           ; encoding: [0x01,0x32,0x80,0xbe]
 0x01,0x32,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b32 s0, s1           ; encoding: [0x01,0x32,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b32 s0, vcc_hi       ; encoding: [0x6b,0x32,0x80,0xbe]
 0x6b,0x32,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b32 s0, vcc_hi       ; encoding: [0x6b,0x32,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b32 s0, vcc_lo       ; encoding: [0x6a,0x32,0x80,0xbe]
 0x6a,0x32,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b32 s0, vcc_lo       ; encoding: [0x6a,0x32,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b32 s105, s104       ; encoding: [0x68,0x32,0xe9,0xbe]
 0x68,0x32,0xe9,0xbe
+# GFX12: s_or_not1_saveexec_b32 s105, s104       ; encoding: [0x68,0x32,0xe9,0xbe]
 
-# GFX12: s_or_not1_saveexec_b32 s105, s1         ; encoding: [0x01,0x32,0xe9,0xbe]
 0x01,0x32,0xe9,0xbe
+# GFX12: s_or_not1_saveexec_b32 s105, s1         ; encoding: [0x01,0x32,0xe9,0xbe]
 
-# GFX12: s_or_not1_saveexec_b32 vcc_hi, s1       ; encoding: [0x01,0x32,0xeb,0xbe]
 0x01,0x32,0xeb,0xbe
+# GFX12: s_or_not1_saveexec_b32 vcc_hi, s1       ; encoding: [0x01,0x32,0xeb,0xbe]
 
-# GFX12: s_or_not1_saveexec_b32 vcc_lo, s1       ; encoding: [0x01,0x32,0xea,0xbe]
 0x01,0x32,0xea,0xbe
+# GFX12: s_or_not1_saveexec_b32 vcc_lo, s1       ; encoding: [0x01,0x32,0xea,0xbe]
 
-# GFX12: s_or_not1_saveexec_b64 s[0:1], 0.5      ; encoding: [0xf0,0x33,0x80,0xbe]
 0xf0,0x33,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b64 s[0:1], 0.5      ; encoding: [0xf0,0x33,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b64 s[0:1], 0        ; encoding: [0x80,0x33,0x80,0xbe]
 0x80,0x33,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b64 s[0:1], 0        ; encoding: [0x80,0x33,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b64 s[0:1], 0x3f717273 ; encoding: [0xff,0x33,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x33,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_or_not1_saveexec_b64 s[0:1], 0x3f717273 ; encoding: [0xff,0x33,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_or_not1_saveexec_b64 s[0:1], 0xaf123456 ; encoding: [0xff,0x33,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x33,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_or_not1_saveexec_b64 s[0:1], 0xaf123456 ; encoding: [0xff,0x33,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_or_not1_saveexec_b64 s[0:1], -1       ; encoding: [0xc1,0x33,0x80,0xbe]
 0xc1,0x33,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b64 s[0:1], -1       ; encoding: [0xc1,0x33,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b64 s[0:1], -4.0     ; encoding: [0xf7,0x33,0x80,0xbe]
 0xf7,0x33,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b64 s[0:1], -4.0     ; encoding: [0xf7,0x33,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b64 s[0:1], exec     ; encoding: [0x7e,0x33,0x80,0xbe]
 0x7e,0x33,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b64 s[0:1], exec     ; encoding: [0x7e,0x33,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b64 s[0:1], s[102:103] ; encoding: [0x66,0x33,0x80,0xbe]
 0x66,0x33,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b64 s[0:1], s[102:103] ; encoding: [0x66,0x33,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b64 s[0:1], s[2:3]   ; encoding: [0x02,0x33,0x80,0xbe]
 0x02,0x33,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b64 s[0:1], s[2:3]   ; encoding: [0x02,0x33,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b64 s[0:1], vcc      ; encoding: [0x6a,0x33,0x80,0xbe]
 0x6a,0x33,0x80,0xbe
+# GFX12: s_or_not1_saveexec_b64 s[0:1], vcc      ; encoding: [0x6a,0x33,0x80,0xbe]
 
-# GFX12: s_or_not1_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x33,0xe8,0xbe]
 0x66,0x33,0xe8,0xbe
+# GFX12: s_or_not1_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x33,0xe8,0xbe]
 
-# GFX12: s_or_not1_saveexec_b64 s[104:105], s[2:3] ; encoding: [0x02,0x33,0xe8,0xbe]
 0x02,0x33,0xe8,0xbe
+# GFX12: s_or_not1_saveexec_b64 s[104:105], s[2:3] ; encoding: [0x02,0x33,0xe8,0xbe]
 
-# GFX12: s_or_not1_saveexec_b64 vcc, s[2:3]      ; encoding: [0x02,0x33,0xea,0xbe]
 0x02,0x33,0xea,0xbe
+# GFX12: s_or_not1_saveexec_b64 vcc, s[2:3]      ; encoding: [0x02,0x33,0xea,0xbe]
 
-# GFX12: s_or_saveexec_b32 s0, 0.5               ; encoding: [0xf0,0x22,0x80,0xbe]
 0xf0,0x22,0x80,0xbe
+# GFX12: s_or_saveexec_b32 s0, 0.5               ; encoding: [0xf0,0x22,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b32 s0, 0                 ; encoding: [0x80,0x22,0x80,0xbe]
 0x80,0x22,0x80,0xbe
+# GFX12: s_or_saveexec_b32 s0, 0                 ; encoding: [0x80,0x22,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b32 s0, 0x3f717273        ; encoding: [0xff,0x22,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x22,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_or_saveexec_b32 s0, 0x3f717273        ; encoding: [0xff,0x22,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_or_saveexec_b32 s0, 0xaf123456        ; encoding: [0xff,0x22,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x22,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_or_saveexec_b32 s0, 0xaf123456        ; encoding: [0xff,0x22,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_or_saveexec_b32 s0, -1                ; encoding: [0xc1,0x22,0x80,0xbe]
 0xc1,0x22,0x80,0xbe
+# GFX12: s_or_saveexec_b32 s0, -1                ; encoding: [0xc1,0x22,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b32 s0, -4.0              ; encoding: [0xf7,0x22,0x80,0xbe]
 0xf7,0x22,0x80,0xbe
+# GFX12: s_or_saveexec_b32 s0, -4.0              ; encoding: [0xf7,0x22,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b32 s0, exec_hi           ; encoding: [0x7f,0x22,0x80,0xbe]
 0x7f,0x22,0x80,0xbe
+# GFX12: s_or_saveexec_b32 s0, exec_hi           ; encoding: [0x7f,0x22,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b32 s0, exec_lo           ; encoding: [0x7e,0x22,0x80,0xbe]
 0x7e,0x22,0x80,0xbe
+# GFX12: s_or_saveexec_b32 s0, exec_lo           ; encoding: [0x7e,0x22,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b32 s0, m0                ; encoding: [0x7d,0x22,0x80,0xbe]
 0x7d,0x22,0x80,0xbe
+# GFX12: s_or_saveexec_b32 s0, m0                ; encoding: [0x7d,0x22,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b32 s0, s104              ; encoding: [0x68,0x22,0x80,0xbe]
 0x68,0x22,0x80,0xbe
+# GFX12: s_or_saveexec_b32 s0, s104              ; encoding: [0x68,0x22,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b32 s0, s1                ; encoding: [0x01,0x22,0x80,0xbe]
 0x01,0x22,0x80,0xbe
+# GFX12: s_or_saveexec_b32 s0, s1                ; encoding: [0x01,0x22,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b32 s0, vcc_hi            ; encoding: [0x6b,0x22,0x80,0xbe]
 0x6b,0x22,0x80,0xbe
+# GFX12: s_or_saveexec_b32 s0, vcc_hi            ; encoding: [0x6b,0x22,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b32 s0, vcc_lo            ; encoding: [0x6a,0x22,0x80,0xbe]
 0x6a,0x22,0x80,0xbe
+# GFX12: s_or_saveexec_b32 s0, vcc_lo            ; encoding: [0x6a,0x22,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b32 s105, s104            ; encoding: [0x68,0x22,0xe9,0xbe]
 0x68,0x22,0xe9,0xbe
+# GFX12: s_or_saveexec_b32 s105, s104            ; encoding: [0x68,0x22,0xe9,0xbe]
 
-# GFX12: s_or_saveexec_b32 s105, s1              ; encoding: [0x01,0x22,0xe9,0xbe]
 0x01,0x22,0xe9,0xbe
+# GFX12: s_or_saveexec_b32 s105, s1              ; encoding: [0x01,0x22,0xe9,0xbe]
 
-# GFX12: s_or_saveexec_b32 vcc_hi, s1            ; encoding: [0x01,0x22,0xeb,0xbe]
 0x01,0x22,0xeb,0xbe
+# GFX12: s_or_saveexec_b32 vcc_hi, s1            ; encoding: [0x01,0x22,0xeb,0xbe]
 
-# GFX12: s_or_saveexec_b32 vcc_lo, s1            ; encoding: [0x01,0x22,0xea,0xbe]
 0x01,0x22,0xea,0xbe
+# GFX12: s_or_saveexec_b32 vcc_lo, s1            ; encoding: [0x01,0x22,0xea,0xbe]
 
-# GFX12: s_or_saveexec_b64 s[0:1], 0.5           ; encoding: [0xf0,0x23,0x80,0xbe]
 0xf0,0x23,0x80,0xbe
+# GFX12: s_or_saveexec_b64 s[0:1], 0.5           ; encoding: [0xf0,0x23,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b64 s[0:1], 0             ; encoding: [0x80,0x23,0x80,0xbe]
 0x80,0x23,0x80,0xbe
+# GFX12: s_or_saveexec_b64 s[0:1], 0             ; encoding: [0x80,0x23,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b64 s[0:1], 0x3f717273    ; encoding: [0xff,0x23,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x23,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_or_saveexec_b64 s[0:1], 0x3f717273    ; encoding: [0xff,0x23,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_or_saveexec_b64 s[0:1], 0xaf123456    ; encoding: [0xff,0x23,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x23,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_or_saveexec_b64 s[0:1], 0xaf123456    ; encoding: [0xff,0x23,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_or_saveexec_b64 s[0:1], -1            ; encoding: [0xc1,0x23,0x80,0xbe]
 0xc1,0x23,0x80,0xbe
+# GFX12: s_or_saveexec_b64 s[0:1], -1            ; encoding: [0xc1,0x23,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b64 s[0:1], -4.0          ; encoding: [0xf7,0x23,0x80,0xbe]
 0xf7,0x23,0x80,0xbe
+# GFX12: s_or_saveexec_b64 s[0:1], -4.0          ; encoding: [0xf7,0x23,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b64 s[0:1], exec          ; encoding: [0x7e,0x23,0x80,0xbe]
 0x7e,0x23,0x80,0xbe
+# GFX12: s_or_saveexec_b64 s[0:1], exec          ; encoding: [0x7e,0x23,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b64 s[0:1], s[102:103]    ; encoding: [0x66,0x23,0x80,0xbe]
 0x66,0x23,0x80,0xbe
+# GFX12: s_or_saveexec_b64 s[0:1], s[102:103]    ; encoding: [0x66,0x23,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b64 s[0:1], s[2:3]        ; encoding: [0x02,0x23,0x80,0xbe]
 0x02,0x23,0x80,0xbe
+# GFX12: s_or_saveexec_b64 s[0:1], s[2:3]        ; encoding: [0x02,0x23,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b64 s[0:1], vcc           ; encoding: [0x6a,0x23,0x80,0xbe]
 0x6a,0x23,0x80,0xbe
+# GFX12: s_or_saveexec_b64 s[0:1], vcc           ; encoding: [0x6a,0x23,0x80,0xbe]
 
-# GFX12: s_or_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x23,0xe8,0xbe]
 0x66,0x23,0xe8,0xbe
+# GFX12: s_or_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x23,0xe8,0xbe]
 
-# GFX12: s_or_saveexec_b64 s[104:105], s[2:3]    ; encoding: [0x02,0x23,0xe8,0xbe]
 0x02,0x23,0xe8,0xbe
+# GFX12: s_or_saveexec_b64 s[104:105], s[2:3]    ; encoding: [0x02,0x23,0xe8,0xbe]
 
-# GFX12: s_or_saveexec_b64 vcc, s[2:3]           ; encoding: [0x02,0x23,0xea,0xbe]
 0x02,0x23,0xea,0xbe
+# GFX12: s_or_saveexec_b64 vcc, s[2:3]           ; encoding: [0x02,0x23,0xea,0xbe]
 
-# GFX12: s_quadmask_b32 exec_hi, s1              ; encoding: [0x01,0x1a,0xff,0xbe]
 0x01,0x1a,0xff,0xbe
+# GFX12: s_quadmask_b32 exec_hi, s1              ; encoding: [0x01,0x1a,0xff,0xbe]
 
-# GFX12: s_quadmask_b32 exec_lo, s1              ; encoding: [0x01,0x1a,0xfe,0xbe]
 0x01,0x1a,0xfe,0xbe
+# GFX12: s_quadmask_b32 exec_lo, s1              ; encoding: [0x01,0x1a,0xfe,0xbe]
 
-# GFX12: s_quadmask_b32 m0, s1                   ; encoding: [0x01,0x1a,0xfd,0xbe]
 0x01,0x1a,0xfd,0xbe
+# GFX12: s_quadmask_b32 m0, s1                   ; encoding: [0x01,0x1a,0xfd,0xbe]
 
-# GFX12: s_quadmask_b32 s0, 0.5                  ; encoding: [0xf0,0x1a,0x80,0xbe]
 0xf0,0x1a,0x80,0xbe
+# GFX12: s_quadmask_b32 s0, 0.5                  ; encoding: [0xf0,0x1a,0x80,0xbe]
 
-# GFX12: s_quadmask_b32 s0, 0                    ; encoding: [0x80,0x1a,0x80,0xbe]
 0x80,0x1a,0x80,0xbe
+# GFX12: s_quadmask_b32 s0, 0                    ; encoding: [0x80,0x1a,0x80,0xbe]
 
-# GFX12: s_quadmask_b32 s0, 0x3f717273           ; encoding: [0xff,0x1a,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x1a,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_quadmask_b32 s0, 0x3f717273           ; encoding: [0xff,0x1a,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_quadmask_b32 s0, 0xaf123456           ; encoding: [0xff,0x1a,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x1a,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_quadmask_b32 s0, 0xaf123456           ; encoding: [0xff,0x1a,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_quadmask_b32 s0, -1                   ; encoding: [0xc1,0x1a,0x80,0xbe]
 0xc1,0x1a,0x80,0xbe
+# GFX12: s_quadmask_b32 s0, -1                   ; encoding: [0xc1,0x1a,0x80,0xbe]
 
-# GFX12: s_quadmask_b32 s0, -4.0                 ; encoding: [0xf7,0x1a,0x80,0xbe]
 0xf7,0x1a,0x80,0xbe
+# GFX12: s_quadmask_b32 s0, -4.0                 ; encoding: [0xf7,0x1a,0x80,0xbe]
 
-# GFX12: s_quadmask_b32 s0, exec_hi              ; encoding: [0x7f,0x1a,0x80,0xbe]
 0x7f,0x1a,0x80,0xbe
+# GFX12: s_quadmask_b32 s0, exec_hi              ; encoding: [0x7f,0x1a,0x80,0xbe]
 
-# GFX12: s_quadmask_b32 s0, exec_lo              ; encoding: [0x7e,0x1a,0x80,0xbe]
 0x7e,0x1a,0x80,0xbe
+# GFX12: s_quadmask_b32 s0, exec_lo              ; encoding: [0x7e,0x1a,0x80,0xbe]
 
-# GFX12: s_quadmask_b32 s0, m0                   ; encoding: [0x7d,0x1a,0x80,0xbe]
 0x7d,0x1a,0x80,0xbe
+# GFX12: s_quadmask_b32 s0, m0                   ; encoding: [0x7d,0x1a,0x80,0xbe]
 
-# GFX12: s_quadmask_b32 s0, s104                 ; encoding: [0x68,0x1a,0x80,0xbe]
 0x68,0x1a,0x80,0xbe
+# GFX12: s_quadmask_b32 s0, s104                 ; encoding: [0x68,0x1a,0x80,0xbe]
 
-# GFX12: s_quadmask_b32 s0, s1                   ; encoding: [0x01,0x1a,0x80,0xbe]
 0x01,0x1a,0x80,0xbe
+# GFX12: s_quadmask_b32 s0, s1                   ; encoding: [0x01,0x1a,0x80,0xbe]
 
-# GFX12: s_quadmask_b32 s0, vcc_hi               ; encoding: [0x6b,0x1a,0x80,0xbe]
 0x6b,0x1a,0x80,0xbe
+# GFX12: s_quadmask_b32 s0, vcc_hi               ; encoding: [0x6b,0x1a,0x80,0xbe]
 
-# GFX12: s_quadmask_b32 s0, vcc_lo               ; encoding: [0x6a,0x1a,0x80,0xbe]
 0x6a,0x1a,0x80,0xbe
+# GFX12: s_quadmask_b32 s0, vcc_lo               ; encoding: [0x6a,0x1a,0x80,0xbe]
 
-# GFX12: s_quadmask_b32 s105, s104               ; encoding: [0x68,0x1a,0xe9,0xbe]
 0x68,0x1a,0xe9,0xbe
+# GFX12: s_quadmask_b32 s105, s104               ; encoding: [0x68,0x1a,0xe9,0xbe]
 
-# GFX12: s_quadmask_b32 s105, s1                 ; encoding: [0x01,0x1a,0xe9,0xbe]
 0x01,0x1a,0xe9,0xbe
+# GFX12: s_quadmask_b32 s105, s1                 ; encoding: [0x01,0x1a,0xe9,0xbe]
 
-# GFX12: s_quadmask_b32 vcc_hi, s1               ; encoding: [0x01,0x1a,0xeb,0xbe]
 0x01,0x1a,0xeb,0xbe
+# GFX12: s_quadmask_b32 vcc_hi, s1               ; encoding: [0x01,0x1a,0xeb,0xbe]
 
-# GFX12: s_quadmask_b32 vcc_lo, s1               ; encoding: [0x01,0x1a,0xea,0xbe]
 0x01,0x1a,0xea,0xbe
+# GFX12: s_quadmask_b32 vcc_lo, s1               ; encoding: [0x01,0x1a,0xea,0xbe]
 
-# GFX12: s_quadmask_b64 exec, s[2:3]             ; encoding: [0x02,0x1b,0xfe,0xbe]
 0x02,0x1b,0xfe,0xbe
+# GFX12: s_quadmask_b64 exec, s[2:3]             ; encoding: [0x02,0x1b,0xfe,0xbe]
 
-# GFX12: s_quadmask_b64 s[0:1], 0.5              ; encoding: [0xf0,0x1b,0x80,0xbe]
 0xf0,0x1b,0x80,0xbe
+# GFX12: s_quadmask_b64 s[0:1], 0.5              ; encoding: [0xf0,0x1b,0x80,0xbe]
 
-# GFX12: s_quadmask_b64 s[0:1], 0                ; encoding: [0x80,0x1b,0x80,0xbe]
 0x80,0x1b,0x80,0xbe
+# GFX12: s_quadmask_b64 s[0:1], 0                ; encoding: [0x80,0x1b,0x80,0xbe]
 
-# GFX12: s_quadmask_b64 s[0:1], 0x3f717273       ; encoding: [0xff,0x1b,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x1b,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_quadmask_b64 s[0:1], 0x3f717273       ; encoding: [0xff,0x1b,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_quadmask_b64 s[0:1], 0xaf123456       ; encoding: [0xff,0x1b,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x1b,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_quadmask_b64 s[0:1], 0xaf123456       ; encoding: [0xff,0x1b,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_quadmask_b64 s[0:1], -1               ; encoding: [0xc1,0x1b,0x80,0xbe]
 0xc1,0x1b,0x80,0xbe
+# GFX12: s_quadmask_b64 s[0:1], -1               ; encoding: [0xc1,0x1b,0x80,0xbe]
 
-# GFX12: s_quadmask_b64 s[0:1], -4.0             ; encoding: [0xf7,0x1b,0x80,0xbe]
 0xf7,0x1b,0x80,0xbe
+# GFX12: s_quadmask_b64 s[0:1], -4.0             ; encoding: [0xf7,0x1b,0x80,0xbe]
 
-# GFX12: s_quadmask_b64 s[0:1], exec             ; encoding: [0x7e,0x1b,0x80,0xbe]
 0x7e,0x1b,0x80,0xbe
+# GFX12: s_quadmask_b64 s[0:1], exec             ; encoding: [0x7e,0x1b,0x80,0xbe]
 
-# GFX12: s_quadmask_b64 s[0:1], s[102:103]       ; encoding: [0x66,0x1b,0x80,0xbe]
 0x66,0x1b,0x80,0xbe
+# GFX12: s_quadmask_b64 s[0:1], s[102:103]       ; encoding: [0x66,0x1b,0x80,0xbe]
 
-# GFX12: s_quadmask_b64 s[0:1], s[2:3]           ; encoding: [0x02,0x1b,0x80,0xbe]
 0x02,0x1b,0x80,0xbe
+# GFX12: s_quadmask_b64 s[0:1], s[2:3]           ; encoding: [0x02,0x1b,0x80,0xbe]
 
-# GFX12: s_quadmask_b64 s[0:1], vcc              ; encoding: [0x6a,0x1b,0x80,0xbe]
 0x6a,0x1b,0x80,0xbe
+# GFX12: s_quadmask_b64 s[0:1], vcc              ; encoding: [0x6a,0x1b,0x80,0xbe]
 
-# GFX12: s_quadmask_b64 s[104:105], s[102:103]   ; encoding: [0x66,0x1b,0xe8,0xbe]
 0x66,0x1b,0xe8,0xbe
+# GFX12: s_quadmask_b64 s[104:105], s[102:103]   ; encoding: [0x66,0x1b,0xe8,0xbe]
 
-# GFX12: s_quadmask_b64 s[104:105], s[2:3]       ; encoding: [0x02,0x1b,0xe8,0xbe]
 0x02,0x1b,0xe8,0xbe
+# GFX12: s_quadmask_b64 s[104:105], s[2:3]       ; encoding: [0x02,0x1b,0xe8,0xbe]
 
-# GFX12: s_quadmask_b64 vcc, s[2:3]              ; encoding: [0x02,0x1b,0xea,0xbe]
 0x02,0x1b,0xea,0xbe
+# GFX12: s_quadmask_b64 vcc, s[2:3]              ; encoding: [0x02,0x1b,0xea,0xbe]
 
-# GFX12: s_rfe_b64 s[0:1]                        ; encoding: [0x00,0x4a,0x80,0xbe]
 0x00,0x4a,0x80,0xbe
+# GFX12: s_rfe_b64 s[0:1]                        ; encoding: [0x00,0x4a,0x80,0xbe]
 
-# GFX12: s_rfe_b64 s[104:105]                    ; encoding: [0x68,0x4a,0x80,0xbe]
 0x68,0x4a,0x80,0xbe
+# GFX12: s_rfe_b64 s[104:105]                    ; encoding: [0x68,0x4a,0x80,0xbe]
 
-# GFX12: s_rfe_b64 vcc                           ; encoding: [0x6a,0x4a,0x80,0xbe]
 0x6a,0x4a,0x80,0xbe
+# GFX12: s_rfe_b64 vcc                           ; encoding: [0x6a,0x4a,0x80,0xbe]
 
-# GFX12: s_sendmsg_rtn_b32 s1, sendmsg(0, 0, 0)  ; encoding: [0x00,0x4c,0x81,0xbe]
 0x00,0x4c,0x81,0xbe
+# GFX12: s_sendmsg_rtn_b32 s1, sendmsg(0, 0, 0)  ; encoding: [0x00,0x4c,0x81,0xbe]
 
-# GFX12: s_sendmsg_rtn_b32 s2, sendmsg(18, 0, 0) ; encoding: [0x12,0x4c,0x82,0xbe]
 0x12,0x4c,0x82,0xbe
+# GFX12: s_sendmsg_rtn_b32 s2, sendmsg(18, 0, 0) ; encoding: [0x12,0x4c,0x82,0xbe]
 
-# GFX12: s_sendmsg_rtn_b32 s3, sendmsg(255, 0, 0) ; encoding: [0xff,0x4c,0x83,0xbe]
 0xff,0x4c,0x83,0xbe
+# GFX12: s_sendmsg_rtn_b32 s3, sendmsg(255, 0, 0) ; encoding: [0xff,0x4c,0x83,0xbe]
 
-# GFX12: s_sendmsg_rtn_b64 s[0:1], sendmsg(0, 0, 0) ; encoding: [0x00,0x4d,0x80,0xbe]
 0x00,0x4d,0x80,0xbe
+# GFX12: s_sendmsg_rtn_b64 s[0:1], sendmsg(0, 0, 0) ; encoding: [0x00,0x4d,0x80,0xbe]
 
-# GFX12: s_sendmsg_rtn_b64 s[2:3], sendmsg(18, 0, 0) ; encoding: [0x12,0x4d,0x82,0xbe]
 0x12,0x4d,0x82,0xbe
+# GFX12: s_sendmsg_rtn_b64 s[2:3], sendmsg(18, 0, 0) ; encoding: [0x12,0x4d,0x82,0xbe]
 
-# GFX12: s_sendmsg_rtn_b64 s[4:5], sendmsg(255, 0, 0) ; encoding: [0xff,0x4d,0x84,0xbe]
 0xff,0x4d,0x84,0xbe
+# GFX12: s_sendmsg_rtn_b64 s[4:5], sendmsg(255, 0, 0) ; encoding: [0xff,0x4d,0x84,0xbe]
 
-# GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_DOORBELL) ; encoding: [0x80,0x4c,0x80,0xbe]
 0x80,0x4c,0x80,0xbe
+# GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_DOORBELL) ; encoding: [0x80,0x4c,0x80,0xbe]
 
-# GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_DDID) ; encoding: [0x81,0x4c,0x80,0xbe]
 0x81,0x4c,0x80,0xbe
+# GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_DDID) ; encoding: [0x81,0x4c,0x80,0xbe]
 
-# GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_TMA) ; encoding: [0x82,0x4c,0x80,0xbe]
 0x82,0x4c,0x80,0xbe
+# GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_TMA) ; encoding: [0x82,0x4c,0x80,0xbe]
 
-# GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_REALTIME) ; encoding: [0x83,0x4c,0x80,0xbe]
 0x83,0x4c,0x80,0xbe
+# GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_REALTIME) ; encoding: [0x83,0x4c,0x80,0xbe]
 
-# GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_SAVE_WAVE) ; encoding: [0x84,0x4c,0x80,0xbe]
 0x84,0x4c,0x80,0xbe
+# GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_SAVE_WAVE) ; encoding: [0x84,0x4c,0x80,0xbe]
 
-# GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_TBA) ; encoding: [0x85,0x4c,0x80,0xbe]
 0x85,0x4c,0x80,0xbe
+# GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_TBA) ; encoding: [0x85,0x4c,0x80,0xbe]
 
-# GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_TBA_TO_PC) ; encoding: [0x86,0x4c,0x80,0xbe]
 0x86,0x4c,0x80,0xbe
+# GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_TBA_TO_PC) ; encoding: [0x86,0x4c,0x80,0xbe]
 
-# GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_SE_AID_ID) ; encoding: [0x87,0x4c,0x80,0xbe]
 0x87,0x4c,0x80,0xbe
+# GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_SE_AID_ID) ; encoding: [0x87,0x4c,0x80,0xbe]
 
-# GFX12: s_setpc_b64 s[0:1]                      ; encoding: [0x00,0x48,0x80,0xbe]
 0x00,0x48,0x80,0xbe
+# GFX12: s_setpc_b64 s[0:1]                      ; encoding: [0x00,0x48,0x80,0xbe]
 
-# GFX12: s_setpc_b64 s[104:105]                  ; encoding: [0x68,0x48,0x80,0xbe]
 0x68,0x48,0x80,0xbe
+# GFX12: s_setpc_b64 s[104:105]                  ; encoding: [0x68,0x48,0x80,0xbe]
 
-# GFX12: s_setpc_b64 vcc                         ; encoding: [0x6a,0x48,0x80,0xbe]
 0x6a,0x48,0x80,0xbe
+# GFX12: s_setpc_b64 vcc                         ; encoding: [0x6a,0x48,0x80,0xbe]
+
+0xcb,0x48,0xf5,0xbe
+# GFX12: s_setpc_b64 -11/*Invalid immediate*/    ; encoding: [0xf5,0x48,0x80,0xbe]
 
-# GFX12: s_sext_i32_i16 exec_hi, s1              ; encoding: [0x01,0x0f,0xff,0xbe]
 0x01,0x0f,0xff,0xbe
+# GFX12: s_sext_i32_i16 exec_hi, s1              ; encoding: [0x01,0x0f,0xff,0xbe]
 
-# GFX12: s_sext_i32_i16 exec_lo, s1              ; encoding: [0x01,0x0f,0xfe,0xbe]
 0x01,0x0f,0xfe,0xbe
+# GFX12: s_sext_i32_i16 exec_lo, s1              ; encoding: [0x01,0x0f,0xfe,0xbe]
 
-# GFX12: s_sext_i32_i16 m0, s1                   ; encoding: [0x01,0x0f,0xfd,0xbe]
 0x01,0x0f,0xfd,0xbe
+# GFX12: s_sext_i32_i16 m0, s1                   ; encoding: [0x01,0x0f,0xfd,0xbe]
 
-# GFX12: s_sext_i32_i16 s0, 0.5                  ; encoding: [0xf0,0x0f,0x80,0xbe]
 0xf0,0x0f,0x80,0xbe
+# GFX12: s_sext_i32_i16 s0, 0.5                  ; encoding: [0xf0,0x0f,0x80,0xbe]
 
-# GFX12: s_sext_i32_i16 s0, 0                    ; encoding: [0x80,0x0f,0x80,0xbe]
 0x80,0x0f,0x80,0xbe
+# GFX12: s_sext_i32_i16 s0, 0                    ; encoding: [0x80,0x0f,0x80,0xbe]
 
-# GFX12: s_sext_i32_i16 s0, 0x3f717273           ; encoding: [0xff,0x0f,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x0f,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_sext_i32_i16 s0, 0x3f717273           ; encoding: [0xff,0x0f,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_sext_i32_i16 s0, 0xaf123456           ; encoding: [0xff,0x0f,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x0f,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_sext_i32_i16 s0, 0xaf123456           ; encoding: [0xff,0x0f,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_sext_i32_i16 s0, -1                   ; encoding: [0xc1,0x0f,0x80,0xbe]
 0xc1,0x0f,0x80,0xbe
+# GFX12: s_sext_i32_i16 s0, -1                   ; encoding: [0xc1,0x0f,0x80,0xbe]
 
-# GFX12: s_sext_i32_i16 s0, -4.0                 ; encoding: [0xf7,0x0f,0x80,0xbe]
 0xf7,0x0f,0x80,0xbe
+# GFX12: s_sext_i32_i16 s0, -4.0                 ; encoding: [0xf7,0x0f,0x80,0xbe]
 
-# GFX12: s_sext_i32_i16 s0, exec_hi              ; encoding: [0x7f,0x0f,0x80,0xbe]
 0x7f,0x0f,0x80,0xbe
+# GFX12: s_sext_i32_i16 s0, exec_hi              ; encoding: [0x7f,0x0f,0x80,0xbe]
 
-# GFX12: s_sext_i32_i16 s0, exec_lo              ; encoding: [0x7e,0x0f,0x80,0xbe]
 0x7e,0x0f,0x80,0xbe
+# GFX12: s_sext_i32_i16 s0, exec_lo              ; encoding: [0x7e,0x0f,0x80,0xbe]
 
-# GFX12: s_sext_i32_i16 s0, m0                   ; encoding: [0x7d,0x0f,0x80,0xbe]
 0x7d,0x0f,0x80,0xbe
+# GFX12: s_sext_i32_i16 s0, m0                   ; encoding: [0x7d,0x0f,0x80,0xbe]
 
-# GFX12: s_sext_i32_i16 s0, s104                 ; encoding: [0x68,0x0f,0x80,0xbe]
 0x68,0x0f,0x80,0xbe
+# GFX12: s_sext_i32_i16 s0, s104                 ; encoding: [0x68,0x0f,0x80,0xbe]
 
-# GFX12: s_sext_i32_i16 s0, s1                   ; encoding: [0x01,0x0f,0x80,0xbe]
 0x01,0x0f,0x80,0xbe
+# GFX12: s_sext_i32_i16 s0, s1                   ; encoding: [0x01,0x0f,0x80,0xbe]
 
-# GFX12: s_sext_i32_i16 s0, vcc_hi               ; encoding: [0x6b,0x0f,0x80,0xbe]
 0x6b,0x0f,0x80,0xbe
+# GFX12: s_sext_i32_i16 s0, vcc_hi               ; encoding: [0x6b,0x0f,0x80,0xbe]
 
-# GFX12: s_sext_i32_i16 s0, vcc_lo               ; encoding: [0x6a,0x0f,0x80,0xbe]
 0x6a,0x0f,0x80,0xbe
+# GFX12: s_sext_i32_i16 s0, vcc_lo               ; encoding: [0x6a,0x0f,0x80,0xbe]
 
-# GFX12: s_sext_i32_i16 s105, s104               ; encoding: [0x68,0x0f,0xe9,0xbe]
 0x68,0x0f,0xe9,0xbe
+# GFX12: s_sext_i32_i16 s105, s104               ; encoding: [0x68,0x0f,0xe9,0xbe]
 
-# GFX12: s_sext_i32_i16 s105, s1                 ; encoding: [0x01,0x0f,0xe9,0xbe]
 0x01,0x0f,0xe9,0xbe
+# GFX12: s_sext_i32_i16 s105, s1                 ; encoding: [0x01,0x0f,0xe9,0xbe]
 
-# GFX12: s_sext_i32_i16 vcc_hi, s1               ; encoding: [0x01,0x0f,0xeb,0xbe]
 0x01,0x0f,0xeb,0xbe
+# GFX12: s_sext_i32_i16 vcc_hi, s1               ; encoding: [0x01,0x0f,0xeb,0xbe]
 
-# GFX12: s_sext_i32_i16 vcc_lo, s1               ; encoding: [0x01,0x0f,0xea,0xbe]
 0x01,0x0f,0xea,0xbe
+# GFX12: s_sext_i32_i16 vcc_lo, s1               ; encoding: [0x01,0x0f,0xea,0xbe]
 
-# GFX12: s_sext_i32_i8 exec_hi, s1               ; encoding: [0x01,0x0e,0xff,0xbe]
 0x01,0x0e,0xff,0xbe
+# GFX12: s_sext_i32_i8 exec_hi, s1               ; encoding: [0x01,0x0e,0xff,0xbe]
 
-# GFX12: s_sext_i32_i8 exec_lo, s1               ; encoding: [0x01,0x0e,0xfe,0xbe]
 0x01,0x0e,0xfe,0xbe
+# GFX12: s_sext_i32_i8 exec_lo, s1               ; encoding: [0x01,0x0e,0xfe,0xbe]
 
-# GFX12: s_sext_i32_i8 m0, s1                    ; encoding: [0x01,0x0e,0xfd,0xbe]
 0x01,0x0e,0xfd,0xbe
+# GFX12: s_sext_i32_i8 m0, s1                    ; encoding: [0x01,0x0e,0xfd,0xbe]
 
-# GFX12: s_sext_i32_i8 s0, 0.5                   ; encoding: [0xf0,0x0e,0x80,0xbe]
 0xf0,0x0e,0x80,0xbe
+# GFX12: s_sext_i32_i8 s0, 0.5                   ; encoding: [0xf0,0x0e,0x80,0xbe]
 
-# GFX12: s_sext_i32_i8 s0, 0                     ; encoding: [0x80,0x0e,0x80,0xbe]
 0x80,0x0e,0x80,0xbe
+# GFX12: s_sext_i32_i8 s0, 0                     ; encoding: [0x80,0x0e,0x80,0xbe]
 
-# GFX12: s_sext_i32_i8 s0, 0x3f717273            ; encoding: [0xff,0x0e,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x0e,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_sext_i32_i8 s0, 0x3f717273            ; encoding: [0xff,0x0e,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_sext_i32_i8 s0, 0xaf123456            ; encoding: [0xff,0x0e,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x0e,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_sext_i32_i8 s0, 0xaf123456            ; encoding: [0xff,0x0e,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_sext_i32_i8 s0, -1                    ; encoding: [0xc1,0x0e,0x80,0xbe]
 0xc1,0x0e,0x80,0xbe
+# GFX12: s_sext_i32_i8 s0, -1                    ; encoding: [0xc1,0x0e,0x80,0xbe]
 
-# GFX12: s_sext_i32_i8 s0, -4.0                  ; encoding: [0xf7,0x0e,0x80,0xbe]
 0xf7,0x0e,0x80,0xbe
+# GFX12: s_sext_i32_i8 s0, -4.0                  ; encoding: [0xf7,0x0e,0x80,0xbe]
 
-# GFX12: s_sext_i32_i8 s0, exec_hi               ; encoding: [0x7f,0x0e,0x80,0xbe]
 0x7f,0x0e,0x80,0xbe
+# GFX12: s_sext_i32_i8 s0, exec_hi               ; encoding: [0x7f,0x0e,0x80,0xbe]
 
-# GFX12: s_sext_i32_i8 s0, exec_lo               ; encoding: [0x7e,0x0e,0x80,0xbe]
 0x7e,0x0e,0x80,0xbe
+# GFX12: s_sext_i32_i8 s0, exec_lo               ; encoding: [0x7e,0x0e,0x80,0xbe]
 
-# GFX12: s_sext_i32_i8 s0, m0                    ; encoding: [0x7d,0x0e,0x80,0xbe]
 0x7d,0x0e,0x80,0xbe
+# GFX12: s_sext_i32_i8 s0, m0                    ; encoding: [0x7d,0x0e,0x80,0xbe]
 
-# GFX12: s_sext_i32_i8 s0, s104                  ; encoding: [0x68,0x0e,0x80,0xbe]
 0x68,0x0e,0x80,0xbe
+# GFX12: s_sext_i32_i8 s0, s104                  ; encoding: [0x68,0x0e,0x80,0xbe]
 
-# GFX12: s_sext_i32_i8 s0, s1                    ; encoding: [0x01,0x0e,0x80,0xbe]
 0x01,0x0e,0x80,0xbe
+# GFX12: s_sext_i32_i8 s0, s1                    ; encoding: [0x01,0x0e,0x80,0xbe]
 
-# GFX12: s_sext_i32_i8 s0, vcc_hi                ; encoding: [0x6b,0x0e,0x80,0xbe]
 0x6b,0x0e,0x80,0xbe
+# GFX12: s_sext_i32_i8 s0, vcc_hi                ; encoding: [0x6b,0x0e,0x80,0xbe]
 
-# GFX12: s_sext_i32_i8 s0, vcc_lo                ; encoding: [0x6a,0x0e,0x80,0xbe]
 0x6a,0x0e,0x80,0xbe
+# GFX12: s_sext_i32_i8 s0, vcc_lo                ; encoding: [0x6a,0x0e,0x80,0xbe]
 
-# GFX12: s_sext_i32_i8 s105, s104                ; encoding: [0x68,0x0e,0xe9,0xbe]
 0x68,0x0e,0xe9,0xbe
+# GFX12: s_sext_i32_i8 s105, s104                ; encoding: [0x68,0x0e,0xe9,0xbe]
 
-# GFX12: s_sext_i32_i8 s105, s1                  ; encoding: [0x01,0x0e,0xe9,0xbe]
 0x01,0x0e,0xe9,0xbe
+# GFX12: s_sext_i32_i8 s105, s1                  ; encoding: [0x01,0x0e,0xe9,0xbe]
 
-# GFX12: s_sext_i32_i8 vcc_hi, s1                ; encoding: [0x01,0x0e,0xeb,0xbe]
 0x01,0x0e,0xeb,0xbe
+# GFX12: s_sext_i32_i8 vcc_hi, s1                ; encoding: [0x01,0x0e,0xeb,0xbe]
 
-# GFX12: s_sext_i32_i8 vcc_lo, s1                ; encoding: [0x01,0x0e,0xea,0xbe]
 0x01,0x0e,0xea,0xbe
+# GFX12: s_sext_i32_i8 vcc_lo, s1                ; encoding: [0x01,0x0e,0xea,0xbe]
 
-# GFX12: s_swappc_b64 s[0:1], s[102:103]         ; encoding: [0x66,0x49,0x80,0xbe]
 0x66,0x49,0x80,0xbe
+# GFX12: s_swappc_b64 s[0:1], s[102:103]         ; encoding: [0x66,0x49,0x80,0xbe]
 
-# GFX12: s_swappc_b64 s[0:1], s[2:3]             ; encoding: [0x02,0x49,0x80,0xbe]
 0x02,0x49,0x80,0xbe
+# GFX12: s_swappc_b64 s[0:1], s[2:3]             ; encoding: [0x02,0x49,0x80,0xbe]
 
-# GFX12: s_swappc_b64 s[0:1], vcc                ; encoding: [0x6a,0x49,0x80,0xbe]
 0x6a,0x49,0x80,0xbe
+# GFX12: s_swappc_b64 s[0:1], vcc                ; encoding: [0x6a,0x49,0x80,0xbe]
 
-# GFX12: s_swappc_b64 s[104:105], s[102:103]     ; encoding: [0x66,0x49,0xe8,0xbe]
 0x66,0x49,0xe8,0xbe
+# GFX12: s_swappc_b64 s[104:105], s[102:103]     ; encoding: [0x66,0x49,0xe8,0xbe]
 
-# GFX12: s_swappc_b64 s[104:105], s[2:3]         ; encoding: [0x02,0x49,0xe8,0xbe]
 0x02,0x49,0xe8,0xbe
+# GFX12: s_swappc_b64 s[104:105], s[2:3]         ; encoding: [0x02,0x49,0xe8,0xbe]
 
-# GFX12: s_swappc_b64 vcc, s[2:3]                ; encoding: [0x02,0x49,0xea,0xbe]
 0x02,0x49,0xea,0xbe
+# GFX12: s_swappc_b64 vcc, s[2:3]                ; encoding: [0x02,0x49,0xea,0xbe]
 
-# GFX12: s_wqm_b32 exec_hi, s1                   ; encoding: [0x01,0x1c,0xff,0xbe]
 0x01,0x1c,0xff,0xbe
+# GFX12: s_wqm_b32 exec_hi, s1                   ; encoding: [0x01,0x1c,0xff,0xbe]
 
-# GFX12: s_wqm_b32 exec_lo, s1                   ; encoding: [0x01,0x1c,0xfe,0xbe]
 0x01,0x1c,0xfe,0xbe
+# GFX12: s_wqm_b32 exec_lo, s1                   ; encoding: [0x01,0x1c,0xfe,0xbe]
 
-# GFX12: s_wqm_b32 m0, s1                        ; encoding: [0x01,0x1c,0xfd,0xbe]
 0x01,0x1c,0xfd,0xbe
+# GFX12: s_wqm_b32 m0, s1                        ; encoding: [0x01,0x1c,0xfd,0xbe]
 
-# GFX12: s_wqm_b32 s0, 0.5                       ; encoding: [0xf0,0x1c,0x80,0xbe]
 0xf0,0x1c,0x80,0xbe
+# GFX12: s_wqm_b32 s0, 0.5                       ; encoding: [0xf0,0x1c,0x80,0xbe]
 
-# GFX12: s_wqm_b32 s0, 0                         ; encoding: [0x80,0x1c,0x80,0xbe]
 0x80,0x1c,0x80,0xbe
+# GFX12: s_wqm_b32 s0, 0                         ; encoding: [0x80,0x1c,0x80,0xbe]
 
-# GFX12: s_wqm_b32 s0, 0x3f717273                ; encoding: [0xff,0x1c,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x1c,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_wqm_b32 s0, 0x3f717273                ; encoding: [0xff,0x1c,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_wqm_b32 s0, 0xaf123456                ; encoding: [0xff,0x1c,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x1c,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_wqm_b32 s0, 0xaf123456                ; encoding: [0xff,0x1c,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_wqm_b32 s0, -1                        ; encoding: [0xc1,0x1c,0x80,0xbe]
 0xc1,0x1c,0x80,0xbe
+# GFX12: s_wqm_b32 s0, -1                        ; encoding: [0xc1,0x1c,0x80,0xbe]
 
-# GFX12: s_wqm_b32 s0, -4.0                      ; encoding: [0xf7,0x1c,0x80,0xbe]
 0xf7,0x1c,0x80,0xbe
+# GFX12: s_wqm_b32 s0, -4.0                      ; encoding: [0xf7,0x1c,0x80,0xbe]
 
-# GFX12: s_wqm_b32 s0, exec_hi                   ; encoding: [0x7f,0x1c,0x80,0xbe]
 0x7f,0x1c,0x80,0xbe
+# GFX12: s_wqm_b32 s0, exec_hi                   ; encoding: [0x7f,0x1c,0x80,0xbe]
 
-# GFX12: s_wqm_b32 s0, exec_lo                   ; encoding: [0x7e,0x1c,0x80,0xbe]
 0x7e,0x1c,0x80,0xbe
+# GFX12: s_wqm_b32 s0, exec_lo                   ; encoding: [0x7e,0x1c,0x80,0xbe]
 
-# GFX12: s_wqm_b32 s0, m0                        ; encoding: [0x7d,0x1c,0x80,0xbe]
 0x7d,0x1c,0x80,0xbe
+# GFX12: s_wqm_b32 s0, m0                        ; encoding: [0x7d,0x1c,0x80,0xbe]
 
-# GFX12: s_wqm_b32 s0, s104                      ; encoding: [0x68,0x1c,0x80,0xbe]
 0x68,0x1c,0x80,0xbe
+# GFX12: s_wqm_b32 s0, s104                      ; encoding: [0x68,0x1c,0x80,0xbe]
 
-# GFX12: s_wqm_b32 s0, s1                        ; encoding: [0x01,0x1c,0x80,0xbe]
 0x01,0x1c,0x80,0xbe
+# GFX12: s_wqm_b32 s0, s1                        ; encoding: [0x01,0x1c,0x80,0xbe]
 
-# GFX12: s_wqm_b32 s0, vcc_hi                    ; encoding: [0x6b,0x1c,0x80,0xbe]
 0x6b,0x1c,0x80,0xbe
+# GFX12: s_wqm_b32 s0, vcc_hi                    ; encoding: [0x6b,0x1c,0x80,0xbe]
 
-# GFX12: s_wqm_b32 s0, vcc_lo                    ; encoding: [0x6a,0x1c,0x80,0xbe]
 0x6a,0x1c,0x80,0xbe
+# GFX12: s_wqm_b32 s0, vcc_lo                    ; encoding: [0x6a,0x1c,0x80,0xbe]
 
-# GFX12: s_wqm_b32 s105, s104                    ; encoding: [0x68,0x1c,0xe9,0xbe]
 0x68,0x1c,0xe9,0xbe
+# GFX12: s_wqm_b32 s105, s104                    ; encoding: [0x68,0x1c,0xe9,0xbe]
 
-# GFX12: s_wqm_b32 s105, s1                      ; encoding: [0x01,0x1c,0xe9,0xbe]
 0x01,0x1c,0xe9,0xbe
+# GFX12: s_wqm_b32 s105, s1                      ; encoding: [0x01,0x1c,0xe9,0xbe]
 
-# GFX12: s_wqm_b32 vcc_hi, s1                    ; encoding: [0x01,0x1c,0xeb,0xbe]
 0x01,0x1c,0xeb,0xbe
+# GFX12: s_wqm_b32 vcc_hi, s1                    ; encoding: [0x01,0x1c,0xeb,0xbe]
 
-# GFX12: s_wqm_b32 vcc_lo, s1                    ; encoding: [0x01,0x1c,0xea,0xbe]
 0x01,0x1c,0xea,0xbe
+# GFX12: s_wqm_b32 vcc_lo, s1                    ; encoding: [0x01,0x1c,0xea,0xbe]
 
-# GFX12: s_wqm_b64 exec, s[2:3]                  ; encoding: [0x02,0x1d,0xfe,0xbe]
 0x02,0x1d,0xfe,0xbe
+# GFX12: s_wqm_b64 exec, s[2:3]                  ; encoding: [0x02,0x1d,0xfe,0xbe]
 
-# GFX12: s_wqm_b64 s[0:1], 0.5                   ; encoding: [0xf0,0x1d,0x80,0xbe]
 0xf0,0x1d,0x80,0xbe
+# GFX12: s_wqm_b64 s[0:1], 0.5                   ; encoding: [0xf0,0x1d,0x80,0xbe]
 
-# GFX12: s_wqm_b64 s[0:1], 0                     ; encoding: [0x80,0x1d,0x80,0xbe]
 0x80,0x1d,0x80,0xbe
+# GFX12: s_wqm_b64 s[0:1], 0                     ; encoding: [0x80,0x1d,0x80,0xbe]
 
-# GFX12: s_wqm_b64 s[0:1], 0x3f717273            ; encoding: [0xff,0x1d,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x1d,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_wqm_b64 s[0:1], 0x3f717273            ; encoding: [0xff,0x1d,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_wqm_b64 s[0:1], 0xaf123456            ; encoding: [0xff,0x1d,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x1d,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_wqm_b64 s[0:1], 0xaf123456            ; encoding: [0xff,0x1d,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_wqm_b64 s[0:1], -1                    ; encoding: [0xc1,0x1d,0x80,0xbe]
 0xc1,0x1d,0x80,0xbe
+# GFX12: s_wqm_b64 s[0:1], -1                    ; encoding: [0xc1,0x1d,0x80,0xbe]
 
-# GFX12: s_wqm_b64 s[0:1], -4.0                  ; encoding: [0xf7,0x1d,0x80,0xbe]
 0xf7,0x1d,0x80,0xbe
+# GFX12: s_wqm_b64 s[0:1], -4.0                  ; encoding: [0xf7,0x1d,0x80,0xbe]
 
-# GFX12: s_wqm_b64 s[0:1], exec                  ; encoding: [0x7e,0x1d,0x80,0xbe]
 0x7e,0x1d,0x80,0xbe
+# GFX12: s_wqm_b64 s[0:1], exec                  ; encoding: [0x7e,0x1d,0x80,0xbe]
 
-# GFX12: s_wqm_b64 s[0:1], s[102:103]            ; encoding: [0x66,0x1d,0x80,0xbe]
 0x66,0x1d,0x80,0xbe
+# GFX12: s_wqm_b64 s[0:1], s[102:103]            ; encoding: [0x66,0x1d,0x80,0xbe]
 
-# GFX12: s_wqm_b64 s[0:1], s[2:3]                ; encoding: [0x02,0x1d,0x80,0xbe]
 0x02,0x1d,0x80,0xbe
+# GFX12: s_wqm_b64 s[0:1], s[2:3]                ; encoding: [0x02,0x1d,0x80,0xbe]
 
-# GFX12: s_wqm_b64 s[0:1], vcc                   ; encoding: [0x6a,0x1d,0x80,0xbe]
 0x6a,0x1d,0x80,0xbe
+# GFX12: s_wqm_b64 s[0:1], vcc                   ; encoding: [0x6a,0x1d,0x80,0xbe]
 
-# GFX12: s_wqm_b64 s[104:105], s[102:103]        ; encoding: [0x66,0x1d,0xe8,0xbe]
 0x66,0x1d,0xe8,0xbe
+# GFX12: s_wqm_b64 s[104:105], s[102:103]        ; encoding: [0x66,0x1d,0xe8,0xbe]
 
-# GFX12: s_wqm_b64 s[104:105], s[2:3]            ; encoding: [0x02,0x1d,0xe8,0xbe]
 0x02,0x1d,0xe8,0xbe
+# GFX12: s_wqm_b64 s[104:105], s[2:3]            ; encoding: [0x02,0x1d,0xe8,0xbe]
 
-# GFX12: s_wqm_b64 vcc, s[2:3]                   ; encoding: [0x02,0x1d,0xea,0xbe]
 0x02,0x1d,0xea,0xbe
+# GFX12: s_wqm_b64 vcc, s[2:3]                   ; encoding: [0x02,0x1d,0xea,0xbe]
 
-# GFX12: s_xnor_saveexec_b32 s0, 0.5             ; encoding: [0xf0,0x2a,0x80,0xbe]
 0xf0,0x2a,0x80,0xbe
+# GFX12: s_xnor_saveexec_b32 s0, 0.5             ; encoding: [0xf0,0x2a,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b32 s0, 0               ; encoding: [0x80,0x2a,0x80,0xbe]
 0x80,0x2a,0x80,0xbe
+# GFX12: s_xnor_saveexec_b32 s0, 0               ; encoding: [0x80,0x2a,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b32 s0, 0x3f717273      ; encoding: [0xff,0x2a,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x2a,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_xnor_saveexec_b32 s0, 0x3f717273      ; encoding: [0xff,0x2a,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_xnor_saveexec_b32 s0, 0xaf123456      ; encoding: [0xff,0x2a,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x2a,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_xnor_saveexec_b32 s0, 0xaf123456      ; encoding: [0xff,0x2a,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_xnor_saveexec_b32 s0, -1              ; encoding: [0xc1,0x2a,0x80,0xbe]
 0xc1,0x2a,0x80,0xbe
+# GFX12: s_xnor_saveexec_b32 s0, -1              ; encoding: [0xc1,0x2a,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b32 s0, -4.0            ; encoding: [0xf7,0x2a,0x80,0xbe]
 0xf7,0x2a,0x80,0xbe
+# GFX12: s_xnor_saveexec_b32 s0, -4.0            ; encoding: [0xf7,0x2a,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b32 s0, exec_hi         ; encoding: [0x7f,0x2a,0x80,0xbe]
 0x7f,0x2a,0x80,0xbe
+# GFX12: s_xnor_saveexec_b32 s0, exec_hi         ; encoding: [0x7f,0x2a,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b32 s0, exec_lo         ; encoding: [0x7e,0x2a,0x80,0xbe]
 0x7e,0x2a,0x80,0xbe
+# GFX12: s_xnor_saveexec_b32 s0, exec_lo         ; encoding: [0x7e,0x2a,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b32 s0, m0              ; encoding: [0x7d,0x2a,0x80,0xbe]
 0x7d,0x2a,0x80,0xbe
+# GFX12: s_xnor_saveexec_b32 s0, m0              ; encoding: [0x7d,0x2a,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b32 s0, s104            ; encoding: [0x68,0x2a,0x80,0xbe]
 0x68,0x2a,0x80,0xbe
+# GFX12: s_xnor_saveexec_b32 s0, s104            ; encoding: [0x68,0x2a,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b32 s0, s1              ; encoding: [0x01,0x2a,0x80,0xbe]
 0x01,0x2a,0x80,0xbe
+# GFX12: s_xnor_saveexec_b32 s0, s1              ; encoding: [0x01,0x2a,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b32 s0, vcc_hi          ; encoding: [0x6b,0x2a,0x80,0xbe]
 0x6b,0x2a,0x80,0xbe
+# GFX12: s_xnor_saveexec_b32 s0, vcc_hi          ; encoding: [0x6b,0x2a,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b32 s0, vcc_lo          ; encoding: [0x6a,0x2a,0x80,0xbe]
 0x6a,0x2a,0x80,0xbe
+# GFX12: s_xnor_saveexec_b32 s0, vcc_lo          ; encoding: [0x6a,0x2a,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b32 s105, s104          ; encoding: [0x68,0x2a,0xe9,0xbe]
 0x68,0x2a,0xe9,0xbe
+# GFX12: s_xnor_saveexec_b32 s105, s104          ; encoding: [0x68,0x2a,0xe9,0xbe]
 
-# GFX12: s_xnor_saveexec_b32 s105, s1            ; encoding: [0x01,0x2a,0xe9,0xbe]
 0x01,0x2a,0xe9,0xbe
+# GFX12: s_xnor_saveexec_b32 s105, s1            ; encoding: [0x01,0x2a,0xe9,0xbe]
 
-# GFX12: s_xnor_saveexec_b32 vcc_hi, s1          ; encoding: [0x01,0x2a,0xeb,0xbe]
 0x01,0x2a,0xeb,0xbe
+# GFX12: s_xnor_saveexec_b32 vcc_hi, s1          ; encoding: [0x01,0x2a,0xeb,0xbe]
 
-# GFX12: s_xnor_saveexec_b32 vcc_lo, s1          ; encoding: [0x01,0x2a,0xea,0xbe]
 0x01,0x2a,0xea,0xbe
+# GFX12: s_xnor_saveexec_b32 vcc_lo, s1          ; encoding: [0x01,0x2a,0xea,0xbe]
 
-# GFX12: s_xnor_saveexec_b64 s[0:1], 0.5         ; encoding: [0xf0,0x2b,0x80,0xbe]
 0xf0,0x2b,0x80,0xbe
+# GFX12: s_xnor_saveexec_b64 s[0:1], 0.5         ; encoding: [0xf0,0x2b,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b64 s[0:1], 0           ; encoding: [0x80,0x2b,0x80,0xbe]
 0x80,0x2b,0x80,0xbe
+# GFX12: s_xnor_saveexec_b64 s[0:1], 0           ; encoding: [0x80,0x2b,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b64 s[0:1], 0x3f717273  ; encoding: [0xff,0x2b,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x2b,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_xnor_saveexec_b64 s[0:1], 0x3f717273  ; encoding: [0xff,0x2b,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_xnor_saveexec_b64 s[0:1], 0xaf123456  ; encoding: [0xff,0x2b,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x2b,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_xnor_saveexec_b64 s[0:1], 0xaf123456  ; encoding: [0xff,0x2b,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_xnor_saveexec_b64 s[0:1], -1          ; encoding: [0xc1,0x2b,0x80,0xbe]
 0xc1,0x2b,0x80,0xbe
+# GFX12: s_xnor_saveexec_b64 s[0:1], -1          ; encoding: [0xc1,0x2b,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b64 s[0:1], -4.0        ; encoding: [0xf7,0x2b,0x80,0xbe]
 0xf7,0x2b,0x80,0xbe
+# GFX12: s_xnor_saveexec_b64 s[0:1], -4.0        ; encoding: [0xf7,0x2b,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b64 s[0:1], exec        ; encoding: [0x7e,0x2b,0x80,0xbe]
 0x7e,0x2b,0x80,0xbe
+# GFX12: s_xnor_saveexec_b64 s[0:1], exec        ; encoding: [0x7e,0x2b,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b64 s[0:1], s[102:103]  ; encoding: [0x66,0x2b,0x80,0xbe]
 0x66,0x2b,0x80,0xbe
+# GFX12: s_xnor_saveexec_b64 s[0:1], s[102:103]  ; encoding: [0x66,0x2b,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b64 s[0:1], s[2:3]      ; encoding: [0x02,0x2b,0x80,0xbe]
 0x02,0x2b,0x80,0xbe
+# GFX12: s_xnor_saveexec_b64 s[0:1], s[2:3]      ; encoding: [0x02,0x2b,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b64 s[0:1], vcc         ; encoding: [0x6a,0x2b,0x80,0xbe]
 0x6a,0x2b,0x80,0xbe
+# GFX12: s_xnor_saveexec_b64 s[0:1], vcc         ; encoding: [0x6a,0x2b,0x80,0xbe]
 
-# GFX12: s_xnor_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x2b,0xe8,0xbe]
 0x66,0x2b,0xe8,0xbe
+# GFX12: s_xnor_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x2b,0xe8,0xbe]
 
-# GFX12: s_xnor_saveexec_b64 s[104:105], s[2:3]  ; encoding: [0x02,0x2b,0xe8,0xbe]
 0x02,0x2b,0xe8,0xbe
+# GFX12: s_xnor_saveexec_b64 s[104:105], s[2:3]  ; encoding: [0x02,0x2b,0xe8,0xbe]
 
-# GFX12: s_xnor_saveexec_b64 vcc, s[2:3]         ; encoding: [0x02,0x2b,0xea,0xbe]
 0x02,0x2b,0xea,0xbe
+# GFX12: s_xnor_saveexec_b64 vcc, s[2:3]         ; encoding: [0x02,0x2b,0xea,0xbe]
 
-# GFX12: s_xor_saveexec_b32 s0, 0.5              ; encoding: [0xf0,0x24,0x80,0xbe]
 0xf0,0x24,0x80,0xbe
+# GFX12: s_xor_saveexec_b32 s0, 0.5              ; encoding: [0xf0,0x24,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b32 s0, 0                ; encoding: [0x80,0x24,0x80,0xbe]
 0x80,0x24,0x80,0xbe
+# GFX12: s_xor_saveexec_b32 s0, 0                ; encoding: [0x80,0x24,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b32 s0, 0x3f717273       ; encoding: [0xff,0x24,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x24,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_xor_saveexec_b32 s0, 0x3f717273       ; encoding: [0xff,0x24,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_xor_saveexec_b32 s0, 0xaf123456       ; encoding: [0xff,0x24,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x24,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_xor_saveexec_b32 s0, 0xaf123456       ; encoding: [0xff,0x24,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_xor_saveexec_b32 s0, -1               ; encoding: [0xc1,0x24,0x80,0xbe]
 0xc1,0x24,0x80,0xbe
+# GFX12: s_xor_saveexec_b32 s0, -1               ; encoding: [0xc1,0x24,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b32 s0, -4.0             ; encoding: [0xf7,0x24,0x80,0xbe]
 0xf7,0x24,0x80,0xbe
+# GFX12: s_xor_saveexec_b32 s0, -4.0             ; encoding: [0xf7,0x24,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b32 s0, exec_hi          ; encoding: [0x7f,0x24,0x80,0xbe]
 0x7f,0x24,0x80,0xbe
+# GFX12: s_xor_saveexec_b32 s0, exec_hi          ; encoding: [0x7f,0x24,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b32 s0, exec_lo          ; encoding: [0x7e,0x24,0x80,0xbe]
 0x7e,0x24,0x80,0xbe
+# GFX12: s_xor_saveexec_b32 s0, exec_lo          ; encoding: [0x7e,0x24,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b32 s0, m0               ; encoding: [0x7d,0x24,0x80,0xbe]
 0x7d,0x24,0x80,0xbe
+# GFX12: s_xor_saveexec_b32 s0, m0               ; encoding: [0x7d,0x24,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b32 s0, s104             ; encoding: [0x68,0x24,0x80,0xbe]
 0x68,0x24,0x80,0xbe
+# GFX12: s_xor_saveexec_b32 s0, s104             ; encoding: [0x68,0x24,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b32 s0, s1               ; encoding: [0x01,0x24,0x80,0xbe]
 0x01,0x24,0x80,0xbe
+# GFX12: s_xor_saveexec_b32 s0, s1               ; encoding: [0x01,0x24,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b32 s0, vcc_hi           ; encoding: [0x6b,0x24,0x80,0xbe]
 0x6b,0x24,0x80,0xbe
+# GFX12: s_xor_saveexec_b32 s0, vcc_hi           ; encoding: [0x6b,0x24,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b32 s0, vcc_lo           ; encoding: [0x6a,0x24,0x80,0xbe]
 0x6a,0x24,0x80,0xbe
+# GFX12: s_xor_saveexec_b32 s0, vcc_lo           ; encoding: [0x6a,0x24,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b32 s105, s104           ; encoding: [0x68,0x24,0xe9,0xbe]
 0x68,0x24,0xe9,0xbe
+# GFX12: s_xor_saveexec_b32 s105, s104           ; encoding: [0x68,0x24,0xe9,0xbe]
 
-# GFX12: s_xor_saveexec_b32 s105, s1             ; encoding: [0x01,0x24,0xe9,0xbe]
 0x01,0x24,0xe9,0xbe
+# GFX12: s_xor_saveexec_b32 s105, s1             ; encoding: [0x01,0x24,0xe9,0xbe]
 
-# GFX12: s_xor_saveexec_b32 vcc_hi, s1           ; encoding: [0x01,0x24,0xeb,0xbe]
 0x01,0x24,0xeb,0xbe
+# GFX12: s_xor_saveexec_b32 vcc_hi, s1           ; encoding: [0x01,0x24,0xeb,0xbe]
 
-# GFX12: s_xor_saveexec_b32 vcc_lo, s1           ; encoding: [0x01,0x24,0xea,0xbe]
 0x01,0x24,0xea,0xbe
+# GFX12: s_xor_saveexec_b32 vcc_lo, s1           ; encoding: [0x01,0x24,0xea,0xbe]
 
-# GFX12: s_xor_saveexec_b64 s[0:1], 0.5          ; encoding: [0xf0,0x25,0x80,0xbe]
 0xf0,0x25,0x80,0xbe
+# GFX12: s_xor_saveexec_b64 s[0:1], 0.5          ; encoding: [0xf0,0x25,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b64 s[0:1], 0            ; encoding: [0x80,0x25,0x80,0xbe]
 0x80,0x25,0x80,0xbe
+# GFX12: s_xor_saveexec_b64 s[0:1], 0            ; encoding: [0x80,0x25,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b64 s[0:1], 0x3f717273   ; encoding: [0xff,0x25,0x80,0xbe,0x73,0x72,0x71,0x3f]
 0xff,0x25,0x80,0xbe,0x73,0x72,0x71,0x3f
+# GFX12: s_xor_saveexec_b64 s[0:1], 0x3f717273   ; encoding: [0xff,0x25,0x80,0xbe,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_xor_saveexec_b64 s[0:1], 0xaf123456   ; encoding: [0xff,0x25,0x80,0xbe,0x56,0x34,0x12,0xaf]
 0xff,0x25,0x80,0xbe,0x56,0x34,0x12,0xaf
+# GFX12: s_xor_saveexec_b64 s[0:1], 0xaf123456   ; encoding: [0xff,0x25,0x80,0xbe,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_xor_saveexec_b64 s[0:1], -1           ; encoding: [0xc1,0x25,0x80,0xbe]
 0xc1,0x25,0x80,0xbe
+# GFX12: s_xor_saveexec_b64 s[0:1], -1           ; encoding: [0xc1,0x25,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b64 s[0:1], -4.0         ; encoding: [0xf7,0x25,0x80,0xbe]
 0xf7,0x25,0x80,0xbe
+# GFX12: s_xor_saveexec_b64 s[0:1], -4.0         ; encoding: [0xf7,0x25,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b64 s[0:1], exec         ; encoding: [0x7e,0x25,0x80,0xbe]
 0x7e,0x25,0x80,0xbe
+# GFX12: s_xor_saveexec_b64 s[0:1], exec         ; encoding: [0x7e,0x25,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b64 s[0:1], s[102:103]   ; encoding: [0x66,0x25,0x80,0xbe]
 0x66,0x25,0x80,0xbe
+# GFX12: s_xor_saveexec_b64 s[0:1], s[102:103]   ; encoding: [0x66,0x25,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b64 s[0:1], s[2:3]       ; encoding: [0x02,0x25,0x80,0xbe]
 0x02,0x25,0x80,0xbe
+# GFX12: s_xor_saveexec_b64 s[0:1], s[2:3]       ; encoding: [0x02,0x25,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b64 s[0:1], vcc          ; encoding: [0x6a,0x25,0x80,0xbe]
 0x6a,0x25,0x80,0xbe
+# GFX12: s_xor_saveexec_b64 s[0:1], vcc          ; encoding: [0x6a,0x25,0x80,0xbe]
 
-# GFX12: s_xor_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x25,0xe8,0xbe]
 0x66,0x25,0xe8,0xbe
+# GFX12: s_xor_saveexec_b64 s[104:105], s[102:103] ; encoding: [0x66,0x25,0xe8,0xbe]
 
-# GFX12: s_xor_saveexec_b64 s[104:105], s[2:3]   ; encoding: [0x02,0x25,0xe8,0xbe]
 0x02,0x25,0xe8,0xbe
+# GFX12: s_xor_saveexec_b64 s[104:105], s[2:3]   ; encoding: [0x02,0x25,0xe8,0xbe]
 
-# GFX12: s_xor_saveexec_b64 vcc, s[2:3]          ; encoding: [0x02,0x25,0xea,0xbe]
 0x02,0x25,0xea,0xbe
+# GFX12: s_xor_saveexec_b64 vcc, s[2:3]          ; encoding: [0x02,0x25,0xea,0xbe]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop2.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop2.txt
index 7c79bbb14b5f5..5be288300c2f2 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop2.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop2.txt
@@ -1,6158 +1,6159 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s
 
-# GFX12: s_add_nc_u64 s[0:1], s[2:3], s[4:5]     ; encoding: [0x02,0x04,0x80,0xa9]
 0x02,0x04,0x80,0xa9
+# GFX12: s_add_nc_u64 s[0:1], s[2:3], s[4:5]     ; encoding: [0x02,0x04,0x80,0xa9]
 
-# GFX12: s_add_nc_u64 s[100:101], s[102:103], s[104:105] ; encoding: [0x66,0x68,0xe4,0xa9]
 0x66,0x68,0xe4,0xa9
+# GFX12: s_add_nc_u64 s[100:101], s[102:103], s[104:105] ; encoding: [0x66,0x68,0xe4,0xa9]
 
-# GFX12: s_add_nc_u64 s[0:1], s[104:105], s[102:103] ; encoding: [0x68,0x66,0x80,0xa9]
 0x68,0x66,0x80,0xa9
+# GFX12: s_add_nc_u64 s[0:1], s[104:105], s[102:103] ; encoding: [0x68,0x66,0x80,0xa9]
 
-# GFX12: s_add_nc_u64 s[104:105], s[0:1], s[102:103] ; encoding: [0x00,0x66,0xe8,0xa9]
 0x00,0x66,0xe8,0xa9
+# GFX12: s_add_nc_u64 s[104:105], s[0:1], s[102:103] ; encoding: [0x00,0x66,0xe8,0xa9]
 
-# GFX12: s_add_nc_u64 s[104:105], s[102:103], s[2:3] ; encoding: [0x66,0x02,0xe8,0xa9]
 0x66,0x02,0xe8,0xa9
+# GFX12: s_add_nc_u64 s[104:105], s[102:103], s[2:3] ; encoding: [0x66,0x02,0xe8,0xa9]
 
-# GFX12: s_add_nc_u64 s[104:105], s[0:1], s[2:3] ; encoding: [0x00,0x02,0xe8,0xa9]
 0x00,0x02,0xe8,0xa9
+# GFX12: s_add_nc_u64 s[104:105], s[0:1], s[2:3] ; encoding: [0x00,0x02,0xe8,0xa9]
 
-# GFX12: s_add_nc_u64 s[0:1], s[102:103], s[2:3] ; encoding: [0x66,0x02,0x80,0xa9]
 0x66,0x02,0x80,0xa9
+# GFX12: s_add_nc_u64 s[0:1], s[102:103], s[2:3] ; encoding: [0x66,0x02,0x80,0xa9]
 
-# GFX12: s_add_nc_u64 s[0:1], s[2:3], s[102:103] ; encoding: [0x02,0x66,0x80,0xa9]
 0x02,0x66,0x80,0xa9
+# GFX12: s_add_nc_u64 s[0:1], s[2:3], s[102:103] ; encoding: [0x02,0x66,0x80,0xa9]
 
-# GFX12: s_add_nc_u64 exec, s[0:1], s[2:3]       ; encoding: [0x00,0x02,0xfe,0xa9]
 0x00,0x02,0xfe,0xa9
+# GFX12: s_add_nc_u64 exec, s[0:1], s[2:3]       ; encoding: [0x00,0x02,0xfe,0xa9]
 
-# GFX12: s_add_nc_u64 vcc, s[0:1], s[2:3]        ; encoding: [0x00,0x02,0xea,0xa9]
 0x00,0x02,0xea,0xa9
+# GFX12: s_add_nc_u64 vcc, s[0:1], s[2:3]        ; encoding: [0x00,0x02,0xea,0xa9]
 
-# GFX12: s_add_nc_u64 s[0:1], exec, s[2:3]       ; encoding: [0x7e,0x02,0x80,0xa9]
 0x7e,0x02,0x80,0xa9
+# GFX12: s_add_nc_u64 s[0:1], exec, s[2:3]       ; encoding: [0x7e,0x02,0x80,0xa9]
 
-# GFX12: s_add_nc_u64 s[0:1], vcc, s[2:3]        ; encoding: [0x6a,0x02,0x80,0xa9]
 0x6a,0x02,0x80,0xa9
+# GFX12: s_add_nc_u64 s[0:1], vcc, s[2:3]        ; encoding: [0x6a,0x02,0x80,0xa9]
 
-# GFX12: s_add_nc_u64 s[0:1], 0, s[2:3]          ; encoding: [0x80,0x02,0x80,0xa9]
 0x80,0x02,0x80,0xa9
+# GFX12: s_add_nc_u64 s[0:1], 0, s[2:3]          ; encoding: [0x80,0x02,0x80,0xa9]
 
-# GFX12: s_add_nc_u64 s[0:1], -1, s[2:3]         ; encoding: [0xc1,0x02,0x80,0xa9]
 0xc1,0x02,0x80,0xa9
+# GFX12: s_add_nc_u64 s[0:1], -1, s[2:3]         ; encoding: [0xc1,0x02,0x80,0xa9]
 
-# GFX12: s_add_nc_u64 s[0:1], 0.5, s[2:3]        ; encoding: [0xf0,0x02,0x80,0xa9]
 0xf0,0x02,0x80,0xa9
+# GFX12: s_add_nc_u64 s[0:1], 0.5, s[2:3]        ; encoding: [0xf0,0x02,0x80,0xa9]
 
-# GFX12: s_add_nc_u64 s[0:1], -4.0, s[2:3]       ; encoding: [0xf7,0x02,0x80,0xa9]
 0xf7,0x02,0x80,0xa9
+# GFX12: s_add_nc_u64 s[0:1], -4.0, s[2:3]       ; encoding: [0xf7,0x02,0x80,0xa9]
 
-# GFX12: s_add_nc_u64 s[0:1], 0x3f717273, s[2:3] ; encoding: [0xff,0x02,0x80,0xa9,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x80,0xa9,0x73,0x72,0x71,0x3f
+# GFX12: s_add_nc_u64 s[0:1], 0x3f717273, s[2:3] ; encoding: [0xff,0x02,0x80,0xa9,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_add_nc_u64 s[0:1], 0xaf123456, s[2:3] ; encoding: [0xff,0x02,0x80,0xa9,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x80,0xa9,0x56,0x34,0x12,0xaf
+# GFX12: s_add_nc_u64 s[0:1], 0xaf123456, s[2:3] ; encoding: [0xff,0x02,0x80,0xa9,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_add_nc_u64 s[0:1], s[2:3], exec       ; encoding: [0x02,0x7e,0x80,0xa9]
 0x02,0x7e,0x80,0xa9
+# GFX12: s_add_nc_u64 s[0:1], s[2:3], exec       ; encoding: [0x02,0x7e,0x80,0xa9]
 
-# GFX12: s_add_nc_u64 s[0:1], s[2:3], vcc        ; encoding: [0x02,0x6a,0x80,0xa9]
 0x02,0x6a,0x80,0xa9
+# GFX12: s_add_nc_u64 s[0:1], s[2:3], vcc        ; encoding: [0x02,0x6a,0x80,0xa9]
 
-# GFX12: s_add_nc_u64 s[0:1], s[2:3], 0          ; encoding: [0x02,0x80,0x80,0xa9]
 0x02,0x80,0x80,0xa9
+# GFX12: s_add_nc_u64 s[0:1], s[2:3], 0          ; encoding: [0x02,0x80,0x80,0xa9]
 
-# GFX12: s_add_nc_u64 s[0:1], s[2:3], -1         ; encoding: [0x02,0xc1,0x80,0xa9]
 0x02,0xc1,0x80,0xa9
+# GFX12: s_add_nc_u64 s[0:1], s[2:3], -1         ; encoding: [0x02,0xc1,0x80,0xa9]
 
-# GFX12: s_add_nc_u64 s[0:1], s[2:3], 0.5        ; encoding: [0x02,0xf0,0x80,0xa9]
 0x02,0xf0,0x80,0xa9
+# GFX12: s_add_nc_u64 s[0:1], s[2:3], 0.5        ; encoding: [0x02,0xf0,0x80,0xa9]
 
-# GFX12: s_add_nc_u64 s[0:1], s[2:3], -4.0       ; encoding: [0x02,0xf7,0x80,0xa9]
 0x02,0xf7,0x80,0xa9
+# GFX12: s_add_nc_u64 s[0:1], s[2:3], -4.0       ; encoding: [0x02,0xf7,0x80,0xa9]
 
-# GFX12: s_add_nc_u64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0xa9,0x73,0x72,0x71,0x3f]
 0x02,0xff,0x80,0xa9,0x73,0x72,0x71,0x3f
+# GFX12: s_add_nc_u64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0xa9,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_add_nc_u64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0xa9,0x56,0x34,0x12,0xaf]
 0x02,0xff,0x80,0xa9,0x56,0x34,0x12,0xaf
+# GFX12: s_add_nc_u64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0xa9,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_sub_nc_u64 s[0:1], s[2:3], s[4:5]     ; encoding: [0x02,0x04,0x00,0xaa]
 0x02,0x04,0x00,0xaa
+# GFX12: s_sub_nc_u64 s[0:1], s[2:3], s[4:5]     ; encoding: [0x02,0x04,0x00,0xaa]
 
-# GFX12: s_sub_nc_u64 s[100:101], s[102:103], s[104:105] ; encoding: [0x66,0x68,0x64,0xaa]
 0x66,0x68,0x64,0xaa
+# GFX12: s_sub_nc_u64 s[100:101], s[102:103], s[104:105] ; encoding: [0x66,0x68,0x64,0xaa]
 
-# GFX12: s_sub_nc_u64 s[0:1], s[104:105], s[102:103] ; encoding: [0x68,0x66,0x00,0xaa]
 0x68,0x66,0x00,0xaa
+# GFX12: s_sub_nc_u64 s[0:1], s[104:105], s[102:103] ; encoding: [0x68,0x66,0x00,0xaa]
 
-# GFX12: s_sub_nc_u64 s[104:105], s[0:1], s[102:103] ; encoding: [0x00,0x66,0x68,0xaa]
 0x00,0x66,0x68,0xaa
+# GFX12: s_sub_nc_u64 s[104:105], s[0:1], s[102:103] ; encoding: [0x00,0x66,0x68,0xaa]
 
-# GFX12: s_sub_nc_u64 s[104:105], s[102:103], s[2:3] ; encoding: [0x66,0x02,0x68,0xaa]
 0x66,0x02,0x68,0xaa
+# GFX12: s_sub_nc_u64 s[104:105], s[102:103], s[2:3] ; encoding: [0x66,0x02,0x68,0xaa]
 
-# GFX12: s_sub_nc_u64 s[104:105], s[0:1], s[2:3] ; encoding: [0x00,0x02,0x68,0xaa]
 0x00,0x02,0x68,0xaa
+# GFX12: s_sub_nc_u64 s[104:105], s[0:1], s[2:3] ; encoding: [0x00,0x02,0x68,0xaa]
 
-# GFX12: s_sub_nc_u64 s[0:1], s[102:103], s[2:3] ; encoding: [0x66,0x02,0x00,0xaa]
 0x66,0x02,0x00,0xaa
+# GFX12: s_sub_nc_u64 s[0:1], s[102:103], s[2:3] ; encoding: [0x66,0x02,0x00,0xaa]
 
-# GFX12: s_sub_nc_u64 s[0:1], s[2:3], s[102:103] ; encoding: [0x02,0x66,0x00,0xaa]
 0x02,0x66,0x00,0xaa
+# GFX12: s_sub_nc_u64 s[0:1], s[2:3], s[102:103] ; encoding: [0x02,0x66,0x00,0xaa]
 
-# GFX12: s_sub_nc_u64 exec, s[0:1], s[2:3]       ; encoding: [0x00,0x02,0x7e,0xaa]
 0x00,0x02,0x7e,0xaa
+# GFX12: s_sub_nc_u64 exec, s[0:1], s[2:3]       ; encoding: [0x00,0x02,0x7e,0xaa]
 
-# GFX12: s_sub_nc_u64 vcc, s[0:1], s[2:3]        ; encoding: [0x00,0x02,0x6a,0xaa]
 0x00,0x02,0x6a,0xaa
+# GFX12: s_sub_nc_u64 vcc, s[0:1], s[2:3]        ; encoding: [0x00,0x02,0x6a,0xaa]
 
-# GFX12: s_sub_nc_u64 s[0:1], exec, s[2:3]       ; encoding: [0x7e,0x02,0x00,0xaa]
 0x7e,0x02,0x00,0xaa
+# GFX12: s_sub_nc_u64 s[0:1], exec, s[2:3]       ; encoding: [0x7e,0x02,0x00,0xaa]
 
-# GFX12: s_sub_nc_u64 s[0:1], vcc, s[2:3]        ; encoding: [0x6a,0x02,0x00,0xaa]
 0x6a,0x02,0x00,0xaa
+# GFX12: s_sub_nc_u64 s[0:1], vcc, s[2:3]        ; encoding: [0x6a,0x02,0x00,0xaa]
 
-# GFX12: s_sub_nc_u64 s[0:1], 0, s[2:3]          ; encoding: [0x80,0x02,0x00,0xaa]
 0x80,0x02,0x00,0xaa
+# GFX12: s_sub_nc_u64 s[0:1], 0, s[2:3]          ; encoding: [0x80,0x02,0x00,0xaa]
 
-# GFX12: s_sub_nc_u64 s[0:1], -1, s[2:3]         ; encoding: [0xc1,0x02,0x00,0xaa]
 0xc1,0x02,0x00,0xaa
+# GFX12: s_sub_nc_u64 s[0:1], -1, s[2:3]         ; encoding: [0xc1,0x02,0x00,0xaa]
 
-# GFX12: s_sub_nc_u64 s[0:1], 0.5, s[2:3]        ; encoding: [0xf0,0x02,0x00,0xaa]
 0xf0,0x02,0x00,0xaa
+# GFX12: s_sub_nc_u64 s[0:1], 0.5, s[2:3]        ; encoding: [0xf0,0x02,0x00,0xaa]
 
-# GFX12: s_sub_nc_u64 s[0:1], -4.0, s[2:3]       ; encoding: [0xf7,0x02,0x00,0xaa]
 0xf7,0x02,0x00,0xaa
+# GFX12: s_sub_nc_u64 s[0:1], -4.0, s[2:3]       ; encoding: [0xf7,0x02,0x00,0xaa]
 
-# GFX12: s_sub_nc_u64 s[0:1], 0x3f717273, s[2:3] ; encoding: [0xff,0x02,0x00,0xaa,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0xaa,0x73,0x72,0x71,0x3f
+# GFX12: s_sub_nc_u64 s[0:1], 0x3f717273, s[2:3] ; encoding: [0xff,0x02,0x00,0xaa,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_sub_nc_u64 s[0:1], 0xaf123456, s[2:3] ; encoding: [0xff,0x02,0x00,0xaa,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0xaa,0x56,0x34,0x12,0xaf
+# GFX12: s_sub_nc_u64 s[0:1], 0xaf123456, s[2:3] ; encoding: [0xff,0x02,0x00,0xaa,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_sub_nc_u64 s[0:1], s[2:3], exec       ; encoding: [0x02,0x7e,0x00,0xaa]
 0x02,0x7e,0x00,0xaa
+# GFX12: s_sub_nc_u64 s[0:1], s[2:3], exec       ; encoding: [0x02,0x7e,0x00,0xaa]
 
-# GFX12: s_sub_nc_u64 s[0:1], s[2:3], vcc        ; encoding: [0x02,0x6a,0x00,0xaa
 0x02,0x6a,0x00,0xaa
+# GFX12: s_sub_nc_u64 s[0:1], s[2:3], vcc        ; encoding: [0x02,0x6a,0x00,0xaa]
 
-# GFX12: s_sub_nc_u64 s[0:1], s[2:3], 0          ; encoding: [0x02,0x80,0x00,0xaa]
 0x02,0x80,0x00,0xaa
+# GFX12: s_sub_nc_u64 s[0:1], s[2:3], 0          ; encoding: [0x02,0x80,0x00,0xaa]
 
-# GFX12: s_sub_nc_u64 s[0:1], s[2:3], -1         ; encoding: [0x02,0xc1,0x00,0xaa]
 0x02,0xc1,0x00,0xaa
+# GFX12: s_sub_nc_u64 s[0:1], s[2:3], -1         ; encoding: [0x02,0xc1,0x00,0xaa]
 
-# GFX12: s_sub_nc_u64 s[0:1], s[2:3], 0.5        ; encoding: [0x02,0xf0,0x00,0xaa]
 0x02,0xf0,0x00,0xaa
+# GFX12: s_sub_nc_u64 s[0:1], s[2:3], 0.5        ; encoding: [0x02,0xf0,0x00,0xaa]
 
-# GFX12: s_sub_nc_u64 s[0:1], s[2:3], -4.0       ; encoding: [0x02,0xf7,0x00,0xaa]
 0x02,0xf7,0x00,0xaa
+# GFX12: s_sub_nc_u64 s[0:1], s[2:3], -4.0       ; encoding: [0x02,0xf7,0x00,0xaa]
 
-# GFX12: s_sub_nc_u64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x00,0xaa,0x73,0x72,0x71,0x3f]
 0x02,0xff,0x00,0xaa,0x73,0x72,0x71,0x3f
+# GFX12: s_sub_nc_u64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x00,0xaa,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_sub_nc_u64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x00,0xaa,0x56,0x34,0x12,0xaf]
 0x02,0xff,0x00,0xaa,0x56,0x34,0x12,0xaf
+# GFX12: s_sub_nc_u64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x00,0xaa,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_mul_u64 s[0:1], s[2:3], s[4:5]        ; encoding: [0x02,0x04,0x80,0xaa]
 0x02,0x04,0x80,0xaa
+# GFX12: s_mul_u64 s[0:1], s[2:3], s[4:5]        ; encoding: [0x02,0x04,0x80,0xaa]
 
-# GFX12: s_mul_u64 s[100:101], s[102:103], s[104:105] ; encoding: [0x66,0x68,0xe4,0xaa]
 0x66,0x68,0xe4,0xaa
+# GFX12: s_mul_u64 s[100:101], s[102:103], s[104:105] ; encoding: [0x66,0x68,0xe4,0xaa]
 
-# GFX12: s_mul_u64 s[0:1], s[104:105], s[102:103] ; encoding: [0x68,0x66,0x80,0xaa]
 0x68,0x66,0x80,0xaa
+# GFX12: s_mul_u64 s[0:1], s[104:105], s[102:103] ; encoding: [0x68,0x66,0x80,0xaa]
 
-# GFX12: s_mul_u64 s[104:105], s[0:1], s[102:103] ; encoding: [0x00,0x66,0xe8,0xaa]
 0x00,0x66,0xe8,0xaa
+# GFX12: s_mul_u64 s[104:105], s[0:1], s[102:103] ; encoding: [0x00,0x66,0xe8,0xaa]
 
-# GFX12: s_mul_u64 s[104:105], s[102:103], s[2:3] ; encoding: [0x66,0x02,0xe8,0xaa]
 0x66,0x02,0xe8,0xaa
+# GFX12: s_mul_u64 s[104:105], s[102:103], s[2:3] ; encoding: [0x66,0x02,0xe8,0xaa]
 
-# GFX12: s_mul_u64 s[104:105], s[0:1], s[2:3]    ; encoding: [0x00,0x02,0xe8,0xaa]
 0x00,0x02,0xe8,0xaa
+# GFX12: s_mul_u64 s[104:105], s[0:1], s[2:3]    ; encoding: [0x00,0x02,0xe8,0xaa]
 
-# GFX12: s_mul_u64 s[0:1], s[102:103], s[2:3]    ; encoding: [0x66,0x02,0x80,0xaa]
 0x66,0x02,0x80,0xaa
+# GFX12: s_mul_u64 s[0:1], s[102:103], s[2:3]    ; encoding: [0x66,0x02,0x80,0xaa]
 
-# GFX12: s_mul_u64 s[0:1], s[2:3], s[102:103]    ; encoding: [0x02,0x66,0x80,0xaa]
 0x02,0x66,0x80,0xaa
+# GFX12: s_mul_u64 s[0:1], s[2:3], s[102:103]    ; encoding: [0x02,0x66,0x80,0xaa]
 
-# GFX12: s_mul_u64 exec, s[0:1], s[2:3]          ; encoding: [0x00,0x02,0xfe,0xaa]
 0x00,0x02,0xfe,0xaa
+# GFX12: s_mul_u64 exec, s[0:1], s[2:3]          ; encoding: [0x00,0x02,0xfe,0xaa]
 
-# GFX12: s_mul_u64 vcc, s[0:1], s[2:3]           ; encoding: [0x00,0x02,0xea,0xaa]
 0x00,0x02,0xea,0xaa
+# GFX12: s_mul_u64 vcc, s[0:1], s[2:3]           ; encoding: [0x00,0x02,0xea,0xaa]
 
-# GFX12: s_mul_u64 s[0:1], exec, s[2:3]          ; encoding: [0x7e,0x02,0x80,0xaa]
 0x7e,0x02,0x80,0xaa
+# GFX12: s_mul_u64 s[0:1], exec, s[2:3]          ; encoding: [0x7e,0x02,0x80,0xaa]
 
-# GFX12: s_mul_u64 s[0:1], vcc, s[2:3]           ; encoding: [0x6a,0x02,0x80,0xaa]
 0x6a,0x02,0x80,0xaa
+# GFX12: s_mul_u64 s[0:1], vcc, s[2:3]           ; encoding: [0x6a,0x02,0x80,0xaa]
 
-# GFX12: s_mul_u64 s[0:1], 0, s[2:3]             ; encoding: [0x80,0x02,0x80,0xaa]
 0x80,0x02,0x80,0xaa
+# GFX12: s_mul_u64 s[0:1], 0, s[2:3]             ; encoding: [0x80,0x02,0x80,0xaa]
 
-# GFX12: s_mul_u64 s[0:1], -1, s[2:3]            ; encoding: [0xc1,0x02,0x80,0xaa]
 0xc1,0x02,0x80,0xaa
+# GFX12: s_mul_u64 s[0:1], -1, s[2:3]            ; encoding: [0xc1,0x02,0x80,0xaa]
 
-# GFX12: s_mul_u64 s[0:1], 0.5, s[2:3]           ; encoding: [0xf0,0x02,0x80,0xaa]
 0xf0,0x02,0x80,0xaa
+# GFX12: s_mul_u64 s[0:1], 0.5, s[2:3]           ; encoding: [0xf0,0x02,0x80,0xaa]
 
-# GFX12: s_mul_u64 s[0:1], -4.0, s[2:3]          ; encoding: [0xf7,0x02,0x80,0xaa]
 0xf7,0x02,0x80,0xaa
+# GFX12: s_mul_u64 s[0:1], -4.0, s[2:3]          ; encoding: [0xf7,0x02,0x80,0xaa]
 
-# GFX12: s_mul_u64 s[0:1], 0x3f717273, s[2:3]    ; encoding: [0xff,0x02,0x80,0xaa,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x80,0xaa,0x73,0x72,0x71,0x3f
+# GFX12: s_mul_u64 s[0:1], 0x3f717273, s[2:3]    ; encoding: [0xff,0x02,0x80,0xaa,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_mul_u64 s[0:1], 0xaf123456, s[2:3]    ; encoding: [0xff,0x02,0x80,0xaa,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x80,0xaa,0x56,0x34,0x12,0xaf
+# GFX12: s_mul_u64 s[0:1], 0xaf123456, s[2:3]    ; encoding: [0xff,0x02,0x80,0xaa,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_mul_u64 s[0:1], s[2:3], exec          ; encoding: [0x02,0x7e,0x80,0xaa]
 0x02,0x7e,0x80,0xaa
+# GFX12: s_mul_u64 s[0:1], s[2:3], exec          ; encoding: [0x02,0x7e,0x80,0xaa]
 
-# GFX12: s_mul_u64 s[0:1], s[2:3], vcc           ; encoding: [0x02,0x6a,0x80,0xaa]
 0x02,0x6a,0x80,0xaa
+# GFX12: s_mul_u64 s[0:1], s[2:3], vcc           ; encoding: [0x02,0x6a,0x80,0xaa]
 
-# GFX12: s_mul_u64 s[0:1], s[2:3], 0             ; encoding: [0x02,0x80,0x80,0xaa]
 0x02,0x80,0x80,0xaa
+# GFX12: s_mul_u64 s[0:1], s[2:3], 0             ; encoding: [0x02,0x80,0x80,0xaa]
 
-# GFX12: s_mul_u64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x80,0xaa]
 0x02,0xc1,0x80,0xaa
+# GFX12: s_mul_u64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x80,0xaa]
 
-# GFX12: s_mul_u64 s[0:1], s[2:3], 0.5           ; encoding: [0x02,0xf0,0x80,0xaa]
 0x02,0xf0,0x80,0xaa
+# GFX12: s_mul_u64 s[0:1], s[2:3], 0.5           ; encoding: [0x02,0xf0,0x80,0xaa]
 
-# GFX12: s_mul_u64 s[0:1], s[2:3], -4.0          ; encoding: [0x02,0xf7,0x80,0xaa]
 0x02,0xf7,0x80,0xaa
+# GFX12: s_mul_u64 s[0:1], s[2:3], -4.0          ; encoding: [0x02,0xf7,0x80,0xaa]
 
-# GFX12: s_mul_u64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0xaa,0x73,0x72,0x71,0x3f]
 0x02,0xff,0x80,0xaa,0x73,0x72,0x71,0x3f
+# GFX12: s_mul_u64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0xaa,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_mul_u64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0xaa,0x56,0x34,0x12,0xaf]
 0x02,0xff,0x80,0xaa,0x56,0x34,0x12,0xaf
+# GFX12: s_mul_u64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0xaa,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_add_f32 s5, s1, s2                    ; encoding: [0x01,0x02,0x05,0xa0]
 0x01,0x02,0x05,0xa0
+# GFX12: s_add_f32 s5, s1, s2                    ; encoding: [0x01,0x02,0x05,0xa0]
 
-# GFX12: s_add_f32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0xa0]
 0x01,0x02,0x69,0xa0
+# GFX12: s_add_f32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0xa0]
 
-# GFX12: s_add_f32 s5, s105, s2                  ; encoding: [0x69,0x02,0x05,0xa0]
 0x69,0x02,0x05,0xa0
+# GFX12: s_add_f32 s5, s105, s2                  ; encoding: [0x69,0x02,0x05,0xa0]
 
-# GFX12: s_add_f32 s5, s101, s2                  ; encoding: [0x65,0x02,0x05,0xa0]
 0x65,0x02,0x05,0xa0
+# GFX12: s_add_f32 s5, s101, s2                  ; encoding: [0x65,0x02,0x05,0xa0]
 
-# GFX12: s_add_f32 s5, vcc_lo, s2                ; encoding: [0x6a,0x02,0x05,0xa0]
 0x6a,0x02,0x05,0xa0
+# GFX12: s_add_f32 s5, vcc_lo, s2                ; encoding: [0x6a,0x02,0x05,0xa0]
 
-# GFX12: s_add_f32 s5, vcc_hi, s2                ; encoding: [0x6b,0x02,0x05,0xa0]
 0x6b,0x02,0x05,0xa0
+# GFX12: s_add_f32 s5, vcc_hi, s2                ; encoding: [0x6b,0x02,0x05,0xa0]
 
-# GFX12: s_add_f32 s5, m0, s2                    ; encoding: [0x7d,0x02,0x05,0xa0]
 0x7d,0x02,0x05,0xa0
+# GFX12: s_add_f32 s5, m0, s2                    ; encoding: [0x7d,0x02,0x05,0xa0]
 
-# GFX12: s_add_f32 s5, exec_lo, s2               ; encoding: [0x7e,0x02,0x05,0xa0]
 0x7e,0x02,0x05,0xa0
+# GFX12: s_add_f32 s5, exec_lo, s2               ; encoding: [0x7e,0x02,0x05,0xa0]
 
-# GFX12: s_add_f32 s5, exec_hi, s2               ; encoding: [0x7f,0x02,0x05,0xa0]
 0x7f,0x02,0x05,0xa0
+# GFX12: s_add_f32 s5, exec_hi, s2               ; encoding: [0x7f,0x02,0x05,0xa0]
 
-# GFX12: s_add_f32 s5, 0, s2                     ; encoding: [0x80,0x02,0x05,0xa0]
 0x80,0x02,0x05,0xa0
+# GFX12: s_add_f32 s5, 0, s2                     ; encoding: [0x80,0x02,0x05,0xa0]
 
-# GFX12: s_add_f32 s5, -1, s2                    ; encoding: [0xc1,0x02,0x05,0xa0]
 0xc1,0x02,0x05,0xa0
+# GFX12: s_add_f32 s5, -1, s2                    ; encoding: [0xc1,0x02,0x05,0xa0]
 
-# GFX12: s_add_f32 s5, 0.5, s2                   ; encoding: [0xf0,0x02,0x05,0xa0]
 0xf0,0x02,0x05,0xa0
+# GFX12: s_add_f32 s5, 0.5, s2                   ; encoding: [0xf0,0x02,0x05,0xa0]
 
-# GFX12: s_add_f32 s5, -4.0, s2                  ; encoding: [0xf7,0x02,0x05,0xa0]
 0xf7,0x02,0x05,0xa0
+# GFX12: s_add_f32 s5, -4.0, s2                  ; encoding: [0xf7,0x02,0x05,0xa0]
 
-# GFX12: s_add_f32 s5, 0xaf123456, s2            ; encoding: [0xff,0x02,0x05,0xa0,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x05,0xa0,0x56,0x34,0x12,0xaf
+# GFX12: s_add_f32 s5, 0xaf123456, s2            ; encoding: [0xff,0x02,0x05,0xa0,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_add_f32 s5, 0x3f717273, s2            ; encoding: [0xff,0x02,0x05,0xa0,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x05,0xa0,0x73,0x72,0x71,0x3f
+# GFX12: s_add_f32 s5, 0x3f717273, s2            ; encoding: [0xff,0x02,0x05,0xa0,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_add_f32 s5, s1, s105                  ; encoding: [0x01,0x69,0x05,0xa0]
 0x01,0x69,0x05,0xa0
+# GFX12: s_add_f32 s5, s1, s105                  ; encoding: [0x01,0x69,0x05,0xa0]
 
-# GFX12: s_sub_f32 s5, s1, s2                    ; encoding: [0x01,0x02,0x85,0xa0]
 0x01,0x02,0x85,0xa0
+# GFX12: s_sub_f32 s5, s1, s2                    ; encoding: [0x01,0x02,0x85,0xa0]
 
-# GFX12: s_sub_f32 s105, s1, s2                  ; encoding: [0x01,0x02,0xe9,0xa0]
 0x01,0x02,0xe9,0xa0
+# GFX12: s_sub_f32 s105, s1, s2                  ; encoding: [0x01,0x02,0xe9,0xa0]
 
-# GFX12: s_sub_f32 s5, s105, s2                  ; encoding: [0x69,0x02,0x85,0xa0]
 0x69,0x02,0x85,0xa0
+# GFX12: s_sub_f32 s5, s105, s2                  ; encoding: [0x69,0x02,0x85,0xa0]
 
-# GFX12: s_sub_f32 s5, s101, s2                  ; encoding: [0x65,0x02,0x85,0xa0]
 0x65,0x02,0x85,0xa0
+# GFX12: s_sub_f32 s5, s101, s2                  ; encoding: [0x65,0x02,0x85,0xa0]
 
-# GFX12: s_sub_f32 s5, vcc_lo, s2                ; encoding: [0x6a,0x02,0x85,0xa0]
 0x6a,0x02,0x85,0xa0
+# GFX12: s_sub_f32 s5, vcc_lo, s2                ; encoding: [0x6a,0x02,0x85,0xa0]
 
-# GFX12: s_sub_f32 s5, vcc_hi, s2                ; encoding: [0x6b,0x02,0x85,0xa0]
 0x6b,0x02,0x85,0xa0
+# GFX12: s_sub_f32 s5, vcc_hi, s2                ; encoding: [0x6b,0x02,0x85,0xa0]
 
-# GFX12: s_sub_f32 s5, m0, s2                    ; encoding: [0x7d,0x02,0x85,0xa0]
 0x7d,0x02,0x85,0xa0
+# GFX12: s_sub_f32 s5, m0, s2                    ; encoding: [0x7d,0x02,0x85,0xa0]
 
-# GFX12: s_sub_f32 s5, exec_lo, s2               ; encoding: [0x7e,0x02,0x85,0xa0]
 0x7e,0x02,0x85,0xa0
+# GFX12: s_sub_f32 s5, exec_lo, s2               ; encoding: [0x7e,0x02,0x85,0xa0]
 
-# GFX12: s_sub_f32 s5, exec_hi, s2               ; encoding: [0x7f,0x02,0x85,0xa0]
 0x7f,0x02,0x85,0xa0
+# GFX12: s_sub_f32 s5, exec_hi, s2               ; encoding: [0x7f,0x02,0x85,0xa0]
 
-# GFX12: s_sub_f32 s5, 0, s2                     ; encoding: [0x80,0x02,0x85,0xa0]
 0x80,0x02,0x85,0xa0
+# GFX12: s_sub_f32 s5, 0, s2                     ; encoding: [0x80,0x02,0x85,0xa0]
 
-# GFX12: s_sub_f32 s5, -1, s2                    ; encoding: [0xc1,0x02,0x85,0xa0]
 0xc1,0x02,0x85,0xa0
+# GFX12: s_sub_f32 s5, -1, s2                    ; encoding: [0xc1,0x02,0x85,0xa0]
 
-# GFX12: s_sub_f32 s5, 0.5, s2                   ; encoding: [0xf0,0x02,0x85,0xa0]
 0xf0,0x02,0x85,0xa0
+# GFX12: s_sub_f32 s5, 0.5, s2                   ; encoding: [0xf0,0x02,0x85,0xa0]
 
-# GFX12: s_sub_f32 s5, -4.0, s2                  ; encoding: [0xf7,0x02,0x85,0xa0]
 0xf7,0x02,0x85,0xa0
+# GFX12: s_sub_f32 s5, -4.0, s2                  ; encoding: [0xf7,0x02,0x85,0xa0]
 
-# GFX12: s_sub_f32 s5, 0xaf123456, s2            ; encoding: [0xff,0x02,0x85,0xa0,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x85,0xa0,0x56,0x34,0x12,0xaf
+# GFX12: s_sub_f32 s5, 0xaf123456, s2            ; encoding: [0xff,0x02,0x85,0xa0,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_sub_f32 s5, 0x3f717273, s2            ; encoding: [0xff,0x02,0x85,0xa0,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x85,0xa0,0x73,0x72,0x71,0x3f
+# GFX12: s_sub_f32 s5, 0x3f717273, s2            ; encoding: [0xff,0x02,0x85,0xa0,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_sub_f32 s5, s1, s105                  ; encoding: [0x01,0x69,0x85,0xa0]
 0x01,0x69,0x85,0xa0
+# GFX12: s_sub_f32 s5, s1, s105                  ; encoding: [0x01,0x69,0x85,0xa0]
 
-# GFX12: s_mul_f32 s5, s1, s2                    ; encoding: [0x01,0x02,0x05,0xa2]
 0x01,0x02,0x05,0xa2
+# GFX12: s_mul_f32 s5, s1, s2                    ; encoding: [0x01,0x02,0x05,0xa2]
 
-# GFX12: s_mul_f32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0xa2]
 0x01,0x02,0x69,0xa2
+# GFX12: s_mul_f32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0xa2]
 
-# GFX12: s_mul_f32 s5, s105, s2                  ; encoding: [0x69,0x02,0x05,0xa2]
 0x69,0x02,0x05,0xa2
+# GFX12: s_mul_f32 s5, s105, s2                  ; encoding: [0x69,0x02,0x05,0xa2]
 
-# GFX12: s_mul_f32 s5, s103, s2                  ; encoding: [0x67,0x02,0x05,0xa2]
 0x67,0x02,0x05,0xa2
+# GFX12: s_mul_f32 s5, s103, s2                  ; encoding: [0x67,0x02,0x05,0xa2]
 
-# GFX12: s_mul_f32 s5, vcc_lo, s2                ; encoding: [0x6a,0x02,0x05,0xa2]
 0x6a,0x02,0x05,0xa2
+# GFX12: s_mul_f32 s5, vcc_lo, s2                ; encoding: [0x6a,0x02,0x05,0xa2]
 
-# GFX12: s_mul_f32 s5, vcc_hi, s2                ; encoding: [0x6b,0x02,0x05,0xa2]
 0x6b,0x02,0x05,0xa2
+# GFX12: s_mul_f32 s5, vcc_hi, s2                ; encoding: [0x6b,0x02,0x05,0xa2]
 
-# GFX12: s_mul_f32 s5, ttmp11, s2                ; encoding: [0x77,0x02,0x05,0xa2]
 0x77,0x02,0x05,0xa2
+# GFX12: s_mul_f32 s5, ttmp11, s2                ; encoding: [0x77,0x02,0x05,0xa2]
 
-# GFX12: s_mul_f32 s5, m0, s2                    ; encoding: [0x7d,0x02,0x05,0xa2]
 0x7d,0x02,0x05,0xa2
+# GFX12: s_mul_f32 s5, m0, s2                    ; encoding: [0x7d,0x02,0x05,0xa2]
 
-# GFX12: s_mul_f32 s5, exec_lo, s2               ; encoding: [0x7e,0x02,0x05,0xa2]
 0x7e,0x02,0x05,0xa2
+# GFX12: s_mul_f32 s5, exec_lo, s2               ; encoding: [0x7e,0x02,0x05,0xa2]
 
-# GFX12: s_mul_f32 s5, exec_hi, s2               ; encoding: [0x7f,0x02,0x05,0xa2]
 0x7f,0x02,0x05,0xa2
+# GFX12: s_mul_f32 s5, exec_hi, s2               ; encoding: [0x7f,0x02,0x05,0xa2]
 
-# GFX12: s_mul_f32 s5, 0, s2                     ; encoding: [0x80,0x02,0x05,0xa2]
 0x80,0x02,0x05,0xa2
+# GFX12: s_mul_f32 s5, 0, s2                     ; encoding: [0x80,0x02,0x05,0xa2]
 
-# GFX12: s_mul_f32 s5, -1, s2                    ; encoding: [0xc1,0x02,0x05,0xa2]
 0xc1,0x02,0x05,0xa2
+# GFX12: s_mul_f32 s5, -1, s2                    ; encoding: [0xc1,0x02,0x05,0xa2]
 
-# GFX12: s_mul_f32 s5, 0.5, s2                   ; encoding: [0xf0,0x02,0x05,0xa2]
 0xf0,0x02,0x05,0xa2
+# GFX12: s_mul_f32 s5, 0.5, s2                   ; encoding: [0xf0,0x02,0x05,0xa2]
 
-# GFX12: s_mul_f32 s5, -4.0, s2                  ; encoding: [0xf7,0x02,0x05,0xa2]
 0xf7,0x02,0x05,0xa2
+# GFX12: s_mul_f32 s5, -4.0, s2                  ; encoding: [0xf7,0x02,0x05,0xa2]
 
-# GFX12: s_mul_f32 s5, 0xaf123456, s2            ; encoding: [0xff,0x02,0x05,0xa2,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x05,0xa2,0x56,0x34,0x12,0xaf
+# GFX12: s_mul_f32 s5, 0xaf123456, s2            ; encoding: [0xff,0x02,0x05,0xa2,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_mul_f32 s5, 0x3f717273, s2            ; encoding: [0xff,0x02,0x05,0xa2,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x05,0xa2,0x73,0x72,0x71,0x3f
+# GFX12: s_mul_f32 s5, 0x3f717273, s2            ; encoding: [0xff,0x02,0x05,0xa2,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_mul_f32 s5, s1, s105                  ; encoding: [0x01,0x69,0x05,0xa2]
 0x01,0x69,0x05,0xa2
+# GFX12: s_mul_f32 s5, s1, s105                  ; encoding: [0x01,0x69,0x05,0xa2]
 
-# GFX12: s_min_num_f32 s5, s1, s2                ; encoding: [0x01,0x02,0x05,0xa1]
 0x01,0x02,0x05,0xa1
+# GFX12: s_min_num_f32 s5, s1, s2                ; encoding: [0x01,0x02,0x05,0xa1]
 
-# GFX12: s_min_num_f32 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0xa1]
 0x01,0x02,0x69,0xa1
+# GFX12: s_min_num_f32 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0xa1]
 
-# GFX12: s_min_num_f32 s5, s105, s2              ; encoding: [0x69,0x02,0x05,0xa1]
 0x69,0x02,0x05,0xa1
+# GFX12: s_min_num_f32 s5, s105, s2              ; encoding: [0x69,0x02,0x05,0xa1]
 
-# GFX12: s_min_num_f32 s5, s103, s2              ; encoding: [0x67,0x02,0x05,0xa1]
 0x67,0x02,0x05,0xa1
+# GFX12: s_min_num_f32 s5, s103, s2              ; encoding: [0x67,0x02,0x05,0xa1]
 
-# GFX12: s_min_num_f32 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x05,0xa1]
 0x6a,0x02,0x05,0xa1
+# GFX12: s_min_num_f32 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x05,0xa1]
 
-# GFX12: s_min_num_f32 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x05,0xa1]
 0x6b,0x02,0x05,0xa1
+# GFX12: s_min_num_f32 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x05,0xa1]
 
-# GFX12: s_min_num_f32 s5, ttmp11, s2            ; encoding: [0x77,0x02,0x05,0xa1]
 0x77,0x02,0x05,0xa1
+# GFX12: s_min_num_f32 s5, ttmp11, s2            ; encoding: [0x77,0x02,0x05,0xa1]
 
-# GFX12: s_min_num_f32 s5, m0, s2                ; encoding: [0x7d,0x02,0x05,0xa1]
 0x7d,0x02,0x05,0xa1
+# GFX12: s_min_num_f32 s5, m0, s2                ; encoding: [0x7d,0x02,0x05,0xa1]
 
-# GFX12: s_min_num_f32 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x05,0xa1]
 0x7e,0x02,0x05,0xa1
+# GFX12: s_min_num_f32 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x05,0xa1]
 
-# GFX12: s_min_num_f32 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x05,0xa1]
 0x7f,0x02,0x05,0xa1
+# GFX12: s_min_num_f32 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x05,0xa1]
 
-# GFX12: s_min_num_f32 s5, 0, s2                 ; encoding: [0x80,0x02,0x05,0xa1]
 0x80,0x02,0x05,0xa1
+# GFX12: s_min_num_f32 s5, 0, s2                 ; encoding: [0x80,0x02,0x05,0xa1]
 
-# GFX12: s_min_num_f32 s5, -1, s2                ; encoding: [0xc1,0x02,0x05,0xa1]
 0xc1,0x02,0x05,0xa1
+# GFX12: s_min_num_f32 s5, -1, s2                ; encoding: [0xc1,0x02,0x05,0xa1]
 
-# GFX12: s_min_num_f32 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x05,0xa1]
 0xf0,0x02,0x05,0xa1
+# GFX12: s_min_num_f32 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x05,0xa1]
 
-# GFX12: s_min_num_f32 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x05,0xa1]
 0xf7,0x02,0x05,0xa1
+# GFX12: s_min_num_f32 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x05,0xa1]
 
-# GFX12: s_min_num_f32 s5, 0xaf123456, s2        ; encoding: [0xff,0x02,0x05,0xa1,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x05,0xa1,0x56,0x34,0x12,0xaf
+# GFX12: s_min_num_f32 s5, 0xaf123456, s2        ; encoding: [0xff,0x02,0x05,0xa1,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_min_num_f32 s5, 0x3f717273, s2        ; encoding: [0xff,0x02,0x05,0xa1,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x05,0xa1,0x73,0x72,0x71,0x3f
+# GFX12: s_min_num_f32 s5, 0x3f717273, s2        ; encoding: [0xff,0x02,0x05,0xa1,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_min_num_f32 s5, s1, s105              ; encoding: [0x01,0x69,0x05,0xa1]
 0x01,0x69,0x05,0xa1
+# GFX12: s_min_num_f32 s5, s1, s105              ; encoding: [0x01,0x69,0x05,0xa1]
 
-# GFX12: s_max_num_f32 s5, s1, s2                ; encoding: [0x01,0x02,0x85,0xa1]
 0x01,0x02,0x85,0xa1
+# GFX12: s_max_num_f32 s5, s1, s2                ; encoding: [0x01,0x02,0x85,0xa1]
 
-# GFX12: s_max_num_f32 s105, s1, s2              ; encoding: [0x01,0x02,0xe9,0xa1]
 0x01,0x02,0xe9,0xa1
+# GFX12: s_max_num_f32 s105, s1, s2              ; encoding: [0x01,0x02,0xe9,0xa1]
 
-# GFX12: s_max_num_f32 s5, s105, s2              ; encoding: [0x69,0x02,0x85,0xa1]
 0x69,0x02,0x85,0xa1
+# GFX12: s_max_num_f32 s5, s105, s2              ; encoding: [0x69,0x02,0x85,0xa1]
 
-# GFX12: s_max_num_f32 s5, s103, s2              ; encoding: [0x67,0x02,0x85,0xa1]
 0x67,0x02,0x85,0xa1
+# GFX12: s_max_num_f32 s5, s103, s2              ; encoding: [0x67,0x02,0x85,0xa1]
 
-# GFX12: s_max_num_f32 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x85,0xa1]
 0x6a,0x02,0x85,0xa1
+# GFX12: s_max_num_f32 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x85,0xa1]
 
-# GFX12: s_max_num_f32 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x85,0xa1]
 0x6b,0x02,0x85,0xa1
+# GFX12: s_max_num_f32 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x85,0xa1]
 
-# GFX12: s_max_num_f32 s5, ttmp11, s2            ; encoding: [0x77,0x02,0x85,0xa1]
 0x77,0x02,0x85,0xa1
+# GFX12: s_max_num_f32 s5, ttmp11, s2            ; encoding: [0x77,0x02,0x85,0xa1]
 
-# GFX12: s_max_num_f32 s5, m0, s2                ; encoding: [0x7d,0x02,0x85,0xa1]
 0x7d,0x02,0x85,0xa1
+# GFX12: s_max_num_f32 s5, m0, s2                ; encoding: [0x7d,0x02,0x85,0xa1]
 
-# GFX12: s_max_num_f32 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x85,0xa1]
 0x7e,0x02,0x85,0xa1
+# GFX12: s_max_num_f32 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x85,0xa1]
 
-# GFX12: s_max_num_f32 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x85,0xa1]
 0x7f,0x02,0x85,0xa1
+# GFX12: s_max_num_f32 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x85,0xa1]
 
-# GFX12: s_max_num_f32 s5, 0, s2                 ; encoding: [0x80,0x02,0x85,0xa1]
 0x80,0x02,0x85,0xa1
+# GFX12: s_max_num_f32 s5, 0, s2                 ; encoding: [0x80,0x02,0x85,0xa1]
 
-# GFX12: s_max_num_f32 s5, -1, s2                ; encoding: [0xc1,0x02,0x85,0xa1]
 0xc1,0x02,0x85,0xa1
+# GFX12: s_max_num_f32 s5, -1, s2                ; encoding: [0xc1,0x02,0x85,0xa1]
 
-# GFX12: s_max_num_f32 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x85,0xa1]
 0xf0,0x02,0x85,0xa1
+# GFX12: s_max_num_f32 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x85,0xa1]
 
-# GFX12: s_max_num_f32 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x85,0xa1]
 0xf7,0x02,0x85,0xa1
+# GFX12: s_max_num_f32 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x85,0xa1]
 
-# GFX12: s_max_num_f32 s5, 0xaf123456, s2        ; encoding: [0xff,0x02,0x85,0xa1,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x85,0xa1,0x56,0x34,0x12,0xaf
+# GFX12: s_max_num_f32 s5, 0xaf123456, s2        ; encoding: [0xff,0x02,0x85,0xa1,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_max_num_f32 s5, 0x3f717273, s2        ; encoding: [0xff,0x02,0x85,0xa1,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x85,0xa1,0x73,0x72,0x71,0x3f
+# GFX12: s_max_num_f32 s5, 0x3f717273, s2        ; encoding: [0xff,0x02,0x85,0xa1,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_max_num_f32 s5, s1, s105              ; encoding: [0x01,0x69,0x85,0xa1]
 0x01,0x69,0x85,0xa1
+# GFX12: s_max_num_f32 s5, s1, s105              ; encoding: [0x01,0x69,0x85,0xa1]
 
-# GFX12: s_minimum_f32 s5, s1, s2                ; encoding: [0x01,0x02,0x85,0xa7]
 0x01,0x02,0x85,0xa7
+# GFX12: s_minimum_f32 s5, s1, s2                ; encoding: [0x01,0x02,0x85,0xa7]
 
-# GFX12: s_minimum_f32 s105, s1, s2              ; encoding: [0x01,0x02,0xe9,0xa7]
 0x01,0x02,0xe9,0xa7
+# GFX12: s_minimum_f32 s105, s1, s2              ; encoding: [0x01,0x02,0xe9,0xa7]
 
-# GFX12: s_minimum_f32 s5, s105, s2              ; encoding: [0x69,0x02,0x85,0xa7]
 0x69,0x02,0x85,0xa7
+# GFX12: s_minimum_f32 s5, s105, s2              ; encoding: [0x69,0x02,0x85,0xa7]
 
-# GFX12: s_minimum_f32 s5, s103, s2              ; encoding: [0x67,0x02,0x85,0xa7]
 0x67,0x02,0x85,0xa7
+# GFX12: s_minimum_f32 s5, s103, s2              ; encoding: [0x67,0x02,0x85,0xa7]
 
-# GFX12: s_minimum_f32 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x85,0xa7]
 0x6a,0x02,0x85,0xa7
+# GFX12: s_minimum_f32 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x85,0xa7]
 
-# GFX12: s_minimum_f32 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x85,0xa7]
 0x6b,0x02,0x85,0xa7
+# GFX12: s_minimum_f32 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x85,0xa7]
 
-# GFX12: s_minimum_f32 s5, ttmp11, s2            ; encoding: [0x77,0x02,0x85,0xa7]
 0x77,0x02,0x85,0xa7
+# GFX12: s_minimum_f32 s5, ttmp11, s2            ; encoding: [0x77,0x02,0x85,0xa7]
 
-# GFX12: s_minimum_f32 s5, m0, s2                ; encoding: [0x7d,0x02,0x85,0xa7]
 0x7d,0x02,0x85,0xa7
+# GFX12: s_minimum_f32 s5, m0, s2                ; encoding: [0x7d,0x02,0x85,0xa7]
 
-# GFX12: s_minimum_f32 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x85,0xa7]
 0x7e,0x02,0x85,0xa7
+# GFX12: s_minimum_f32 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x85,0xa7]
 
-# GFX12: s_minimum_f32 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x85,0xa7]
 0x7f,0x02,0x85,0xa7
+# GFX12: s_minimum_f32 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x85,0xa7]
 
-# GFX12: s_minimum_f32 s5, 0, s2                 ; encoding: [0x80,0x02,0x85,0xa7]
 0x80,0x02,0x85,0xa7
+# GFX12: s_minimum_f32 s5, 0, s2                 ; encoding: [0x80,0x02,0x85,0xa7]
 
-# GFX12: s_minimum_f32 s5, -1, s2                ; encoding: [0xc1,0x02,0x85,0xa7]
 0xc1,0x02,0x85,0xa7
+# GFX12: s_minimum_f32 s5, -1, s2                ; encoding: [0xc1,0x02,0x85,0xa7]
 
-# GFX12: s_minimum_f32 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x85,0xa7]
 0xf0,0x02,0x85,0xa7
+# GFX12: s_minimum_f32 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x85,0xa7]
 
-# GFX12: s_minimum_f32 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x85,0xa7]
 0xf7,0x02,0x85,0xa7
+# GFX12: s_minimum_f32 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x85,0xa7]
 
-# GFX12: s_minimum_f32 s5, 0xaf123456, s2        ; encoding: [0xff,0x02,0x85,0xa7,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x85,0xa7,0x56,0x34,0x12,0xaf
+# GFX12: s_minimum_f32 s5, 0xaf123456, s2        ; encoding: [0xff,0x02,0x85,0xa7,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_minimum_f32 s5, 0x3f717273, s2        ; encoding: [0xff,0x02,0x85,0xa7,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x85,0xa7,0x73,0x72,0x71,0x3f
+# GFX12: s_minimum_f32 s5, 0x3f717273, s2        ; encoding: [0xff,0x02,0x85,0xa7,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_minimum_f32 s5, s1, s105              ; encoding: [0x01,0x69,0x85,0xa7]
 0x01,0x69,0x85,0xa7
+# GFX12: s_minimum_f32 s5, s1, s105              ; encoding: [0x01,0x69,0x85,0xa7]
 
-# GFX12: s_maximum_f32 s5, s1, s2                ; encoding: [0x01,0x02,0x05,0xa8]
 0x01,0x02,0x05,0xa8
+# GFX12: s_maximum_f32 s5, s1, s2                ; encoding: [0x01,0x02,0x05,0xa8]
 
-# GFX12: s_maximum_f32 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0xa8]
 0x01,0x02,0x69,0xa8
+# GFX12: s_maximum_f32 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0xa8]
 
-# GFX12: s_maximum_f32 s5, s105, s2              ; encoding: [0x69,0x02,0x05,0xa8]
 0x69,0x02,0x05,0xa8
+# GFX12: s_maximum_f32 s5, s105, s2              ; encoding: [0x69,0x02,0x05,0xa8]
 
-# GFX12: s_maximum_f32 s5, s103, s2              ; encoding: [0x67,0x02,0x05,0xa8]
 0x67,0x02,0x05,0xa8
+# GFX12: s_maximum_f32 s5, s103, s2              ; encoding: [0x67,0x02,0x05,0xa8]
 
-# GFX12: s_maximum_f32 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x05,0xa8]
 0x6a,0x02,0x05,0xa8
+# GFX12: s_maximum_f32 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x05,0xa8]
 
-# GFX12: s_maximum_f32 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x05,0xa8]
 0x6b,0x02,0x05,0xa8
+# GFX12: s_maximum_f32 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x05,0xa8]
 
-# GFX12: s_maximum_f32 s5, ttmp11, s2            ; encoding: [0x77,0x02,0x05,0xa8]
 0x77,0x02,0x05,0xa8
+# GFX12: s_maximum_f32 s5, ttmp11, s2            ; encoding: [0x77,0x02,0x05,0xa8]
 
-# GFX12: s_maximum_f32 s5, m0, s2                ; encoding: [0x7d,0x02,0x05,0xa8]
 0x7d,0x02,0x05,0xa8
+# GFX12: s_maximum_f32 s5, m0, s2                ; encoding: [0x7d,0x02,0x05,0xa8]
 
-# GFX12: s_maximum_f32 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x05,0xa8]
 0x7e,0x02,0x05,0xa8
+# GFX12: s_maximum_f32 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x05,0xa8]
 
-# GFX12: s_maximum_f32 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x05,0xa8]
 0x7f,0x02,0x05,0xa8
+# GFX12: s_maximum_f32 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x05,0xa8]
 
-# GFX12: s_maximum_f32 s5, 0, s2                 ; encoding: [0x80,0x02,0x05,0xa8]
 0x80,0x02,0x05,0xa8
+# GFX12: s_maximum_f32 s5, 0, s2                 ; encoding: [0x80,0x02,0x05,0xa8]
 
-# GFX12: s_maximum_f32 s5, -1, s2                ; encoding: [0xc1,0x02,0x05,0xa8]
 0xc1,0x02,0x05,0xa8
+# GFX12: s_maximum_f32 s5, -1, s2                ; encoding: [0xc1,0x02,0x05,0xa8]
 
-# GFX12: s_maximum_f32 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x05,0xa8]
 0xf0,0x02,0x05,0xa8
+# GFX12: s_maximum_f32 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x05,0xa8]
 
-# GFX12: s_maximum_f32 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x05,0xa8]
 0xf7,0x02,0x05,0xa8
+# GFX12: s_maximum_f32 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x05,0xa8]
 
-# GFX12: s_maximum_f32 s5, 0xaf123456, s2        ; encoding: [0xff,0x02,0x05,0xa8,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x05,0xa8,0x56,0x34,0x12,0xaf
+# GFX12: s_maximum_f32 s5, 0xaf123456, s2        ; encoding: [0xff,0x02,0x05,0xa8,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_maximum_f32 s5, 0x3f717273, s2        ; encoding: [0xff,0x02,0x05,0xa8,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x05,0xa8,0x73,0x72,0x71,0x3f
+# GFX12: s_maximum_f32 s5, 0x3f717273, s2        ; encoding: [0xff,0x02,0x05,0xa8,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_maximum_f32 s5, s1, s105              ; encoding: [0x01,0x69,0x05,0xa8]
 0x01,0x69,0x05,0xa8
+# GFX12: s_maximum_f32 s5, s1, s105              ; encoding: [0x01,0x69,0x05,0xa8]
 
-# GFX12: s_fmac_f32 s5, s1, s2                   ; encoding: [0x01,0x02,0x85,0xa3]
 0x01,0x02,0x85,0xa3
+# GFX12: s_fmac_f32 s5, s1, s2                   ; encoding: [0x01,0x02,0x85,0xa3]
 
-# GFX12: s_fmac_f32 s105, s1, s2                 ; encoding: [0x01,0x02,0xe9,0xa3]
 0x01,0x02,0xe9,0xa3
+# GFX12: s_fmac_f32 s105, s1, s2                 ; encoding: [0x01,0x02,0xe9,0xa3]
 
-# GFX12: s_fmac_f32 s5, s105, s2                 ; encoding: [0x69,0x02,0x85,0xa3]
 0x69,0x02,0x85,0xa3
+# GFX12: s_fmac_f32 s5, s105, s2                 ; encoding: [0x69,0x02,0x85,0xa3]
 
-# GFX12: s_fmac_f32 s5, s103, s2                 ; encoding: [0x67,0x02,0x85,0xa3]
 0x67,0x02,0x85,0xa3
+# GFX12: s_fmac_f32 s5, s103, s2                 ; encoding: [0x67,0x02,0x85,0xa3]
 
-# GFX12: s_fmac_f32 s5, vcc_lo, s2               ; encoding: [0x6a,0x02,0x85,0xa3]
 0x6a,0x02,0x85,0xa3
+# GFX12: s_fmac_f32 s5, vcc_lo, s2               ; encoding: [0x6a,0x02,0x85,0xa3]
 
-# GFX12: s_fmac_f32 s5, vcc_hi, s2               ; encoding: [0x6b,0x02,0x85,0xa3]
 0x6b,0x02,0x85,0xa3
+# GFX12: s_fmac_f32 s5, vcc_hi, s2               ; encoding: [0x6b,0x02,0x85,0xa3]
 
-# GFX12: s_fmac_f32 s5, ttmp11, s2               ; encoding: [0x77,0x02,0x85,0xa3]
 0x77,0x02,0x85,0xa3
+# GFX12: s_fmac_f32 s5, ttmp11, s2               ; encoding: [0x77,0x02,0x85,0xa3]
 
-# GFX12: s_fmac_f32 s5, m0, s2                   ; encoding: [0x7d,0x02,0x85,0xa3]
 0x7d,0x02,0x85,0xa3
+# GFX12: s_fmac_f32 s5, m0, s2                   ; encoding: [0x7d,0x02,0x85,0xa3]
 
-# GFX12: s_fmac_f32 s5, exec_lo, s2              ; encoding: [0x7e,0x02,0x85,0xa3]
 0x7e,0x02,0x85,0xa3
+# GFX12: s_fmac_f32 s5, exec_lo, s2              ; encoding: [0x7e,0x02,0x85,0xa3]
 
-# GFX12: s_fmac_f32 s5, exec_hi, s2              ; encoding: [0x7f,0x02,0x85,0xa3]
 0x7f,0x02,0x85,0xa3
+# GFX12: s_fmac_f32 s5, exec_hi, s2              ; encoding: [0x7f,0x02,0x85,0xa3]
 
-# GFX12: s_fmac_f32 s5, 0, s2                    ; encoding: [0x80,0x02,0x85,0xa3]
 0x80,0x02,0x85,0xa3
+# GFX12: s_fmac_f32 s5, 0, s2                    ; encoding: [0x80,0x02,0x85,0xa3]
 
-# GFX12: s_fmac_f32 s5, -1, s2                   ; encoding: [0xc1,0x02,0x85,0xa3]
 0xc1,0x02,0x85,0xa3
+# GFX12: s_fmac_f32 s5, -1, s2                   ; encoding: [0xc1,0x02,0x85,0xa3]
 
-# GFX12: s_fmac_f32 s5, 0.5, s2                  ; encoding: [0xf0,0x02,0x85,0xa3]
 0xf0,0x02,0x85,0xa3
+# GFX12: s_fmac_f32 s5, 0.5, s2                  ; encoding: [0xf0,0x02,0x85,0xa3]
 
-# GFX12: s_fmac_f32 s5, -4.0, s2                 ; encoding: [0xf7,0x02,0x85,0xa3]
 0xf7,0x02,0x85,0xa3
+# GFX12: s_fmac_f32 s5, -4.0, s2                 ; encoding: [0xf7,0x02,0x85,0xa3]
 
-# GFX12: s_fmac_f32 s5, 0xaf123456, s2           ; encoding: [0xff,0x02,0x85,0xa3,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x85,0xa3,0x56,0x34,0x12,0xaf
+# GFX12: s_fmac_f32 s5, 0xaf123456, s2           ; encoding: [0xff,0x02,0x85,0xa3,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_fmac_f32 s5, 0x3f717273, s2           ; encoding: [0xff,0x02,0x85,0xa3,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x85,0xa3,0x73,0x72,0x71,0x3f
+# GFX12: s_fmac_f32 s5, 0x3f717273, s2           ; encoding: [0xff,0x02,0x85,0xa3,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_fmac_f32 s5, s1, s105                 ; encoding: [0x01,0x69,0x85,0xa3]
 0x01,0x69,0x85,0xa3
+# GFX12: s_fmac_f32 s5, s1, s105                 ; encoding: [0x01,0x69,0x85,0xa3]
 
-# GFX12: s_fmamk_f32 s5, s1, 0x11213141, s3      ; encoding: [0x01,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
 0x01,0x03,0x05,0xa3,0x41,0x31,0x21,0x11
+# GFX12: s_fmamk_f32 s5, s1, 0x11213141, s3      ; encoding: [0x01,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmamk_f32 s105, s1, 0x11213141, s3    ; encoding: [0x01,0x03,0x69,0xa3,0x41,0x31,0x21,0x11]
 0x01,0x03,0x69,0xa3,0x41,0x31,0x21,0x11
+# GFX12: s_fmamk_f32 s105, s1, 0x11213141, s3    ; encoding: [0x01,0x03,0x69,0xa3,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmamk_f32 s5, s105, 0x11213141, s3    ; encoding: [0x69,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
 0x69,0x03,0x05,0xa3,0x41,0x31,0x21,0x11
+# GFX12: s_fmamk_f32 s5, s105, 0x11213141, s3    ; encoding: [0x69,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmamk_f32 s5, 0, 0x11213141, s3       ; encoding: [0x80,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
 0x80,0x03,0x05,0xa3,0x41,0x31,0x21,0x11
+# GFX12: s_fmamk_f32 s5, 0, 0x11213141, s3       ; encoding: [0x80,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmamk_f32 s5, -1, 0x11213141, s3      ; encoding: [0xc1,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
 0xc1,0x03,0x05,0xa3,0x41,0x31,0x21,0x11
+# GFX12: s_fmamk_f32 s5, -1, 0x11213141, s3      ; encoding: [0xc1,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmamk_f32 s5, 0.5, 0x11213141, s3     ; encoding: [0xf0,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
 0xf0,0x03,0x05,0xa3,0x41,0x31,0x21,0x11
+# GFX12: s_fmamk_f32 s5, 0.5, 0x11213141, s3     ; encoding: [0xf0,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmamk_f32 s5, -4.0, 0x11213141, s3    ; encoding: [0xf7,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
 0xf7,0x03,0x05,0xa3,0x41,0x31,0x21,0x11
+# GFX12: s_fmamk_f32 s5, -4.0, 0x11213141, s3    ; encoding: [0xf7,0x03,0x05,0xa3,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmamk_f32 s5, s1, 0xa1b1c1d1, s3      ; encoding: [0x01,0x03,0x05,0xa3,0xd1,0xc1,0xb1,0xa1]
 0x01,0x03,0x05,0xa3,0xd1,0xc1,0xb1,0xa1
+# GFX12: s_fmamk_f32 s5, s1, 0xa1b1c1d1, s3      ; encoding: [0x01,0x03,0x05,0xa3,0xd1,0xc1,0xb1,0xa1]
 
-# GFX12: s_fmamk_f32 s5, s1, 0x11213141, s105    ; encoding: [0x01,0x69,0x05,0xa3,0x41,0x31,0x21,0x11]
 0x01,0x69,0x05,0xa3,0x41,0x31,0x21,0x11
+# GFX12: s_fmamk_f32 s5, s1, 0x11213141, s105    ; encoding: [0x01,0x69,0x05,0xa3,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmamk_f32 s5, 0x11213141, 0x11213141, s105 ; encoding: [0xff,0x69,0x05,0xa3,0x41,0x31,0x21,0x11]
 0xff,0x69,0x05,0xa3,0x41,0x31,0x21,0x11
+# GFX12: s_fmamk_f32 s5, 0x11213141, 0x11213141, s105 ; encoding: [0xff,0x69,0x05,0xa3,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmamk_f32 s5, s105, 0x11213141, 0x11213141 ; encoding: [0x69,0xff,0x05,0xa3,0x41,0x31,0x21,0x11]
 0x69,0xff,0x05,0xa3,0x41,0x31,0x21,0x11
+# GFX12: s_fmamk_f32 s5, s105, 0x11213141, 0x11213141 ; encoding: [0x69,0xff,0x05,0xa3,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmaak_f32 s5, 0x11213141, 0x11213141, 0x11213141 ; encoding: [0xff,0xff,0x85,0xa2,0x41,0x31,0x21,0x11]
 0xff,0xff,0x85,0xa2,0x41,0x31,0x21,0x11
+# GFX12: s_fmaak_f32 s5, 0x11213141, 0x11213141, 0x11213141 ; encoding: [0xff,0xff,0x85,0xa2,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmaak_f32 s5, s1, s2, 0x11213141      ; encoding: [0x01,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 0x01,0x02,0x85,0xa2,0x41,0x31,0x21,0x11
+# GFX12: s_fmaak_f32 s5, s1, s2, 0x11213141      ; encoding: [0x01,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmaak_f32 s105, s1, s2, 0x11213141    ; encoding: [0x01,0x02,0xe9,0xa2,0x41,0x31,0x21,0x11]
 0x01,0x02,0xe9,0xa2,0x41,0x31,0x21,0x11
+# GFX12: s_fmaak_f32 s105, s1, s2, 0x11213141    ; encoding: [0x01,0x02,0xe9,0xa2,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmaak_f32 s5, s105, s2, 0x11213141    ; encoding: [0x69,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 0x69,0x02,0x85,0xa2,0x41,0x31,0x21,0x11
+# GFX12: s_fmaak_f32 s5, s105, s2, 0x11213141    ; encoding: [0x69,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmaak_f32 s5, 0, s2, 0x11213141       ; encoding: [0x80,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 0x80,0x02,0x85,0xa2,0x41,0x31,0x21,0x11
+# GFX12: s_fmaak_f32 s5, 0, s2, 0x11213141       ; encoding: [0x80,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmaak_f32 s5, -1, s2, 0x11213141      ; encoding: [0xc1,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 0xc1,0x02,0x85,0xa2,0x41,0x31,0x21,0x11
+# GFX12: s_fmaak_f32 s5, -1, s2, 0x11213141      ; encoding: [0xc1,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmaak_f32 s5, 0.5, s2, 0x11213141     ; encoding: [0xf0,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 0xf0,0x02,0x85,0xa2,0x41,0x31,0x21,0x11
+# GFX12: s_fmaak_f32 s5, 0.5, s2, 0x11213141     ; encoding: [0xf0,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmaak_f32 s5, -4.0, s2, 0x11213141    ; encoding: [0xf7,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 0xf7,0x02,0x85,0xa2,0x41,0x31,0x21,0x11
+# GFX12: s_fmaak_f32 s5, -4.0, s2, 0x11213141    ; encoding: [0xf7,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmaak_f32 s5, s1, s105, 0x11213141    ; encoding: [0x01,0x69,0x85,0xa2,0x41,0x31,0x21,0x11]
 0x01,0x69,0x85,0xa2,0x41,0x31,0x21,0x11
+# GFX12: s_fmaak_f32 s5, s1, s105, 0x11213141    ; encoding: [0x01,0x69,0x85,0xa2,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmaak_f32 s5, s1, s2, 0xa1b1c1d1      ; encoding: [0x01,0x02,0x85,0xa2,0xd1,0xc1,0xb1,0xa1]
 0x01,0x02,0x85,0xa2,0xd1,0xc1,0xb1,0xa1
+# GFX12: s_fmaak_f32 s5, s1, s2, 0xa1b1c1d1      ; encoding: [0x01,0x02,0x85,0xa2,0xd1,0xc1,0xb1,0xa1]
 
-# GFX12: s_fmaak_f32 s5, 0x11213141, s2, 0x11213141 ; encoding: [0xff,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 0xff,0x02,0x85,0xa2,0x41,0x31,0x21,0x11
+# GFX12: s_fmaak_f32 s5, 0x11213141, s2, 0x11213141 ; encoding: [0xff,0x02,0x85,0xa2,0x41,0x31,0x21,0x11]
 
-# GFX12: s_fmaak_f32 s5, s105, 0x11213141, 0x11213141 ; encoding: [0x69,0xff,0x85,0xa2,0x41,0x31,0x21,0x11]
 0x69,0xff,0x85,0xa2,0x41,0x31,0x21,0x11
+# GFX12: s_fmaak_f32 s5, s105, 0x11213141, 0x11213141 ; encoding: [0x69,0xff,0x85,0xa2,0x41,0x31,0x21,0x11]
 
-# GFX12: s_cvt_pk_rtz_f16_f32 s5, s1, s2         ; encoding: [0x01,0x02,0x05,0xa4]
 0x01,0x02,0x05,0xa4
+# GFX12: s_cvt_pk_rtz_f16_f32 s5, s1, s2         ; encoding: [0x01,0x02,0x05,0xa4]
 
-# GFX12: s_cvt_pk_rtz_f16_f32 s105, s1, s2       ; encoding: [0x01,0x02,0x69,0xa4]
 0x01,0x02,0x69,0xa4
+# GFX12: s_cvt_pk_rtz_f16_f32 s105, s1, s2       ; encoding: [0x01,0x02,0x69,0xa4]
 
-# GFX12: s_cvt_pk_rtz_f16_f32 s5, s105, s2       ; encoding: [0x69,0x02,0x05,0xa4]
 0x69,0x02,0x05,0xa4
+# GFX12: s_cvt_pk_rtz_f16_f32 s5, s105, s2       ; encoding: [0x69,0x02,0x05,0xa4]
 
-# GFX12: s_cvt_pk_rtz_f16_f32 s5, s103, s2       ; encoding: [0x67,0x02,0x05,0xa4]
 0x67,0x02,0x05,0xa4
+# GFX12: s_cvt_pk_rtz_f16_f32 s5, s103, s2       ; encoding: [0x67,0x02,0x05,0xa4]
 
-# GFX12: s_cvt_pk_rtz_f16_f32 s5, vcc_lo, s2     ; encoding: [0x6a,0x02,0x05,0xa4]
 0x6a,0x02,0x05,0xa4
+# GFX12: s_cvt_pk_rtz_f16_f32 s5, vcc_lo, s2     ; encoding: [0x6a,0x02,0x05,0xa4]
 
-# GFX12: s_cvt_pk_rtz_f16_f32 s5, vcc_hi, s2     ; encoding: [0x6b,0x02,0x05,0xa4]
 0x6b,0x02,0x05,0xa4
+# GFX12: s_cvt_pk_rtz_f16_f32 s5, vcc_hi, s2     ; encoding: [0x6b,0x02,0x05,0xa4]
 
-# GFX12: s_cvt_pk_rtz_f16_f32 s5, ttmp11, s2     ; encoding: [0x77,0x02,0x05,0xa4]
 0x77,0x02,0x05,0xa4
+# GFX12: s_cvt_pk_rtz_f16_f32 s5, ttmp11, s2     ; encoding: [0x77,0x02,0x05,0xa4]
 
-# GFX12: s_cvt_pk_rtz_f16_f32 s5, m0, s2         ; encoding: [0x7d,0x02,0x05,0xa4]
 0x7d,0x02,0x05,0xa4
+# GFX12: s_cvt_pk_rtz_f16_f32 s5, m0, s2         ; encoding: [0x7d,0x02,0x05,0xa4]
 
-# GFX12: s_cvt_pk_rtz_f16_f32 s5, exec_lo, s2    ; encoding: [0x7e,0x02,0x05,0xa4]
 0x7e,0x02,0x05,0xa4
+# GFX12: s_cvt_pk_rtz_f16_f32 s5, exec_lo, s2    ; encoding: [0x7e,0x02,0x05,0xa4]
 
-# GFX12: s_cvt_pk_rtz_f16_f32 s5, exec_hi, s2    ; encoding: [0x7f,0x02,0x05,0xa4]
 0x7f,0x02,0x05,0xa4
+# GFX12: s_cvt_pk_rtz_f16_f32 s5, exec_hi, s2    ; encoding: [0x7f,0x02,0x05,0xa4]
 
-# GFX12: s_cvt_pk_rtz_f16_f32 s5, 0, s2          ; encoding: [0x80,0x02,0x05,0xa4]
 0x80,0x02,0x05,0xa4
+# GFX12: s_cvt_pk_rtz_f16_f32 s5, 0, s2          ; encoding: [0x80,0x02,0x05,0xa4]
 
-# GFX12: s_cvt_pk_rtz_f16_f32 s5, -1, s2         ; encoding: [0xc1,0x02,0x05,0xa4]
 0xc1,0x02,0x05,0xa4
+# GFX12: s_cvt_pk_rtz_f16_f32 s5, -1, s2         ; encoding: [0xc1,0x02,0x05,0xa4]
 
-# GFX12: s_cvt_pk_rtz_f16_f32 s5, 0.5, s2        ; encoding: [0xf0,0x02,0x05,0xa4]
 0xf0,0x02,0x05,0xa4
+# GFX12: s_cvt_pk_rtz_f16_f32 s5, 0.5, s2        ; encoding: [0xf0,0x02,0x05,0xa4]
 
-# GFX12: s_cvt_pk_rtz_f16_f32 s5, -4.0, s2       ; encoding: [0xf7,0x02,0x05,0xa4]
 0xf7,0x02,0x05,0xa4
+# GFX12: s_cvt_pk_rtz_f16_f32 s5, -4.0, s2       ; encoding: [0xf7,0x02,0x05,0xa4]
 
-# GFX12: s_cvt_pk_rtz_f16_f32 s5, 0xaf123456, s2 ; encoding: [0xff,0x02,0x05,0xa4,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x05,0xa4,0x56,0x34,0x12,0xaf
+# GFX12: s_cvt_pk_rtz_f16_f32 s5, 0xaf123456, s2 ; encoding: [0xff,0x02,0x05,0xa4,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cvt_pk_rtz_f16_f32 s5, 0x3f717273, s2 ; encoding: [0xff,0x02,0x05,0xa4,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x05,0xa4,0x73,0x72,0x71,0x3f
+# GFX12: s_cvt_pk_rtz_f16_f32 s5, 0x3f717273, s2 ; encoding: [0xff,0x02,0x05,0xa4,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cvt_pk_rtz_f16_f32 s5, s1, s105       ; encoding: [0x01,0x69,0x05,0xa4]
 0x01,0x69,0x05,0xa4
+# GFX12: s_cvt_pk_rtz_f16_f32 s5, s1, s105       ; encoding: [0x01,0x69,0x05,0xa4]
 
-# GFX12: s_add_f16 s5, s1, s2                    ; encoding: [0x01,0x02,0x85,0xa4]
 0x01,0x02,0x85,0xa4
+# GFX12: s_add_f16 s5, s1, s2                    ; encoding: [0x01,0x02,0x85,0xa4]
 
-# GFX12: s_add_f16 s105, s1, s2                  ; encoding: [0x01,0x02,0xe9,0xa4]
 0x01,0x02,0xe9,0xa4
+# GFX12: s_add_f16 s105, s1, s2                  ; encoding: [0x01,0x02,0xe9,0xa4]
 
-# GFX12: s_add_f16 s5, s105, s2                  ; encoding: [0x69,0x02,0x85,0xa4]
 0x69,0x02,0x85,0xa4
+# GFX12: s_add_f16 s5, s105, s2                  ; encoding: [0x69,0x02,0x85,0xa4]
 
-# GFX12: s_add_f16 s5, s101, s2                  ; encoding: [0x65,0x02,0x85,0xa4]
 0x65,0x02,0x85,0xa4
+# GFX12: s_add_f16 s5, s101, s2                  ; encoding: [0x65,0x02,0x85,0xa4]
 
-# GFX12: s_add_f16 s5, vcc_lo, s2                ; encoding: [0x6a,0x02,0x85,0xa4]
 0x6a,0x02,0x85,0xa4
+# GFX12: s_add_f16 s5, vcc_lo, s2                ; encoding: [0x6a,0x02,0x85,0xa4]
 
-# GFX12: s_add_f16 s5, vcc_hi, s2                ; encoding: [0x6b,0x02,0x85,0xa4]
 0x6b,0x02,0x85,0xa4
+# GFX12: s_add_f16 s5, vcc_hi, s2                ; encoding: [0x6b,0x02,0x85,0xa4]
 
-# GFX12: s_add_f16 s5, m0, s2                    ; encoding: [0x7d,0x02,0x85,0xa4]
 0x7d,0x02,0x85,0xa4
+# GFX12: s_add_f16 s5, m0, s2                    ; encoding: [0x7d,0x02,0x85,0xa4]
 
-# GFX12: s_add_f16 s5, exec_lo, s2               ; encoding: [0x7e,0x02,0x85,0xa4]
 0x7e,0x02,0x85,0xa4
+# GFX12: s_add_f16 s5, exec_lo, s2               ; encoding: [0x7e,0x02,0x85,0xa4]
 
-# GFX12: s_add_f16 s5, exec_hi, s2               ; encoding: [0x7f,0x02,0x85,0xa4]
 0x7f,0x02,0x85,0xa4
+# GFX12: s_add_f16 s5, exec_hi, s2               ; encoding: [0x7f,0x02,0x85,0xa4]
 
-# GFX12: s_add_f16 s5, 0, s2                     ; encoding: [0x80,0x02,0x85,0xa4]
 0x80,0x02,0x85,0xa4
+# GFX12: s_add_f16 s5, 0, s2                     ; encoding: [0x80,0x02,0x85,0xa4]
 
-# GFX12: s_add_f16 s5, -1, s2                    ; encoding: [0xc1,0x02,0x85,0xa4]
 0xc1,0x02,0x85,0xa4
+# GFX12: s_add_f16 s5, -1, s2                    ; encoding: [0xc1,0x02,0x85,0xa4]
 
-# GFX12: s_add_f16 s5, 0.5, s2                   ; encoding: [0xf0,0x02,0x85,0xa4]
 0xf0,0x02,0x85,0xa4
+# GFX12: s_add_f16 s5, 0.5, s2                   ; encoding: [0xf0,0x02,0x85,0xa4]
 
-# GFX12: s_add_f16 s5, -4.0, s2                  ; encoding: [0xf7,0x02,0x85,0xa4]
 0xf7,0x02,0x85,0xa4
+# GFX12: s_add_f16 s5, -4.0, s2                  ; encoding: [0xf7,0x02,0x85,0xa4]
 
-# GFX12: s_add_f16 s5, 0xfe0b, s2                ; encoding: [0xff,0x02,0x85,0xa4,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x85,0xa4,0x0b,0xfe,0x00,0x00
+# GFX12: s_add_f16 s5, 0xfe0b, s2                ; encoding: [0xff,0x02,0x85,0xa4,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_add_f16 s5, 0x3456, s2                ; encoding: [0xff,0x02,0x85,0xa4,0x56,0x34,0x00,0x00]
 0xff,0x02,0x85,0xa4,0x56,0x34,0x00,0x00
+# GFX12: s_add_f16 s5, 0x3456, s2                ; encoding: [0xff,0x02,0x85,0xa4,0x56,0x34,0x00,0x00]
 
-# GFX12: s_add_f16 s5, s1, s105                  ; encoding: [0x01,0x69,0x85,0xa4]
 0x01,0x69,0x85,0xa4
+# GFX12: s_add_f16 s5, s1, s105                  ; encoding: [0x01,0x69,0x85,0xa4]
 
-# GFX12: s_sub_f16 s5, s1, s2                    ; encoding: [0x01,0x02,0x05,0xa5]
 0x01,0x02,0x05,0xa5
+# GFX12: s_sub_f16 s5, s1, s2                    ; encoding: [0x01,0x02,0x05,0xa5]
 
-# GFX12: s_sub_f16 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0xa5]
 0x01,0x02,0x69,0xa5
+# GFX12: s_sub_f16 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0xa5]
 
-# GFX12: s_sub_f16 s5, s105, s2                  ; encoding: [0x69,0x02,0x05,0xa5]
 0x69,0x02,0x05,0xa5
+# GFX12: s_sub_f16 s5, s105, s2                  ; encoding: [0x69,0x02,0x05,0xa5]
 
-# GFX12: s_sub_f16 s5, s101, s2                  ; encoding: [0x65,0x02,0x05,0xa5]
 0x65,0x02,0x05,0xa5
+# GFX12: s_sub_f16 s5, s101, s2                  ; encoding: [0x65,0x02,0x05,0xa5]
 
-# GFX12: s_sub_f16 s5, vcc_lo, s2                ; encoding: [0x6a,0x02,0x05,0xa5]
 0x6a,0x02,0x05,0xa5
+# GFX12: s_sub_f16 s5, vcc_lo, s2                ; encoding: [0x6a,0x02,0x05,0xa5]
 
-# GFX12: s_sub_f16 s5, vcc_hi, s2                ; encoding: [0x6b,0x02,0x05,0xa5]
 0x6b,0x02,0x05,0xa5
+# GFX12: s_sub_f16 s5, vcc_hi, s2                ; encoding: [0x6b,0x02,0x05,0xa5]
 
-# GFX12: s_sub_f16 s5, m0, s2                    ; encoding: [0x7d,0x02,0x05,0xa5]
 0x7d,0x02,0x05,0xa5
+# GFX12: s_sub_f16 s5, m0, s2                    ; encoding: [0x7d,0x02,0x05,0xa5]
 
-# GFX12: s_sub_f16 s5, exec_lo, s2               ; encoding: [0x7e,0x02,0x05,0xa5]
 0x7e,0x02,0x05,0xa5
+# GFX12: s_sub_f16 s5, exec_lo, s2               ; encoding: [0x7e,0x02,0x05,0xa5]
 
-# GFX12: s_sub_f16 s5, exec_hi, s2               ; encoding: [0x7f,0x02,0x05,0xa5]
 0x7f,0x02,0x05,0xa5
+# GFX12: s_sub_f16 s5, exec_hi, s2               ; encoding: [0x7f,0x02,0x05,0xa5]
 
-# GFX12: s_sub_f16 s5, 0, s2                     ; encoding: [0x80,0x02,0x05,0xa5]
 0x80,0x02,0x05,0xa5
+# GFX12: s_sub_f16 s5, 0, s2                     ; encoding: [0x80,0x02,0x05,0xa5]
 
-# GFX12: s_sub_f16 s5, -1, s2                    ; encoding: [0xc1,0x02,0x05,0xa5]
 0xc1,0x02,0x05,0xa5
+# GFX12: s_sub_f16 s5, -1, s2                    ; encoding: [0xc1,0x02,0x05,0xa5]
 
-# GFX12: s_sub_f16 s5, 0.5, s2                   ; encoding: [0xf0,0x02,0x05,0xa5]
 0xf0,0x02,0x05,0xa5
+# GFX12: s_sub_f16 s5, 0.5, s2                   ; encoding: [0xf0,0x02,0x05,0xa5]
 
-# GFX12: s_sub_f16 s5, -4.0, s2                  ; encoding: [0xf7,0x02,0x05,0xa5]
 0xf7,0x02,0x05,0xa5
+# GFX12: s_sub_f16 s5, -4.0, s2                  ; encoding: [0xf7,0x02,0x05,0xa5]
 
-# GFX12: s_sub_f16 s5, 0xfe0b, s2                ; encoding: [0xff,0x02,0x05,0xa5,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x05,0xa5,0x0b,0xfe,0x00,0x00
+# GFX12: s_sub_f16 s5, 0xfe0b, s2                ; encoding: [0xff,0x02,0x05,0xa5,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_sub_f16 s5, 0x3456, s2                ; encoding: [0xff,0x02,0x05,0xa5,0x56,0x34,0x00,0x00]
 0xff,0x02,0x05,0xa5,0x56,0x34,0x00,0x00
+# GFX12: s_sub_f16 s5, 0x3456, s2                ; encoding: [0xff,0x02,0x05,0xa5,0x56,0x34,0x00,0x00]
 
-# GFX12: s_sub_f16 s5, s1, s105                  ; encoding: [0x01,0x69,0x05,0xa5]
 0x01,0x69,0x05,0xa5
+# GFX12: s_sub_f16 s5, s1, s105                  ; encoding: [0x01,0x69,0x05,0xa5]
 
-# GFX12: s_mul_f16 s5, s1, s2                    ; encoding: [0x01,0x02,0x85,0xa6]
 0x01,0x02,0x85,0xa6
+# GFX12: s_mul_f16 s5, s1, s2                    ; encoding: [0x01,0x02,0x85,0xa6]
 
-# GFX12: s_mul_f16 s105, s1, s2                  ; encoding: [0x01,0x02,0xe9,0xa6]
 0x01,0x02,0xe9,0xa6
+# GFX12: s_mul_f16 s105, s1, s2                  ; encoding: [0x01,0x02,0xe9,0xa6]
 
-# GFX12: s_mul_f16 s5, s105, s2                  ; encoding: [0x69,0x02,0x85,0xa6]
 0x69,0x02,0x85,0xa6
+# GFX12: s_mul_f16 s5, s105, s2                  ; encoding: [0x69,0x02,0x85,0xa6]
 
-# GFX12: s_mul_f16 s5, s101, s2                  ; encoding: [0x65,0x02,0x85,0xa6]
 0x65,0x02,0x85,0xa6
+# GFX12: s_mul_f16 s5, s101, s2                  ; encoding: [0x65,0x02,0x85,0xa6]
 
-# GFX12: s_mul_f16 s5, vcc_lo, s2                ; encoding: [0x6a,0x02,0x85,0xa6]
 0x6a,0x02,0x85,0xa6
+# GFX12: s_mul_f16 s5, vcc_lo, s2                ; encoding: [0x6a,0x02,0x85,0xa6]
 
-# GFX12: s_mul_f16 s5, vcc_hi, s2                ; encoding: [0x6b,0x02,0x85,0xa6]
 0x6b,0x02,0x85,0xa6
+# GFX12: s_mul_f16 s5, vcc_hi, s2                ; encoding: [0x6b,0x02,0x85,0xa6]
 
-# GFX12: s_mul_f16 s5, m0, s2                    ; encoding: [0x7d,0x02,0x85,0xa6]
 0x7d,0x02,0x85,0xa6
+# GFX12: s_mul_f16 s5, m0, s2                    ; encoding: [0x7d,0x02,0x85,0xa6]
 
-# GFX12: s_mul_f16 s5, exec_lo, s2               ; encoding: [0x7e,0x02,0x85,0xa6]
 0x7e,0x02,0x85,0xa6
+# GFX12: s_mul_f16 s5, exec_lo, s2               ; encoding: [0x7e,0x02,0x85,0xa6]
 
-# GFX12: s_mul_f16 s5, exec_hi, s2               ; encoding: [0x7f,0x02,0x85,0xa6]
 0x7f,0x02,0x85,0xa6
+# GFX12: s_mul_f16 s5, exec_hi, s2               ; encoding: [0x7f,0x02,0x85,0xa6]
 
-# GFX12: s_mul_f16 s5, 0, s2                     ; encoding: [0x80,0x02,0x85,0xa6]
 0x80,0x02,0x85,0xa6
+# GFX12: s_mul_f16 s5, 0, s2                     ; encoding: [0x80,0x02,0x85,0xa6]
 
-# GFX12: s_mul_f16 s5, -1, s2                    ; encoding: [0xc1,0x02,0x85,0xa6]
 0xc1,0x02,0x85,0xa6
+# GFX12: s_mul_f16 s5, -1, s2                    ; encoding: [0xc1,0x02,0x85,0xa6]
 
-# GFX12: s_mul_f16 s5, 0.5, s2                   ; encoding: [0xf0,0x02,0x85,0xa6]
 0xf0,0x02,0x85,0xa6
+# GFX12: s_mul_f16 s5, 0.5, s2                   ; encoding: [0xf0,0x02,0x85,0xa6]
 
-# GFX12: s_mul_f16 s5, -4.0, s2                  ; encoding: [0xf7,0x02,0x85,0xa6]
 0xf7,0x02,0x85,0xa6
+# GFX12: s_mul_f16 s5, -4.0, s2                  ; encoding: [0xf7,0x02,0x85,0xa6]
 
-# GFX12: s_mul_f16 s5, 0xfe0b, s2                ; encoding: [0xff,0x02,0x85,0xa6,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x85,0xa6,0x0b,0xfe,0x00,0x00
+# GFX12: s_mul_f16 s5, 0xfe0b, s2                ; encoding: [0xff,0x02,0x85,0xa6,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_mul_f16 s5, 0x3456, s2                ; encoding: [0xff,0x02,0x85,0xa6,0x56,0x34,0x00,0x00]
 0xff,0x02,0x85,0xa6,0x56,0x34,0x00,0x00
+# GFX12: s_mul_f16 s5, 0x3456, s2                ; encoding: [0xff,0x02,0x85,0xa6,0x56,0x34,0x00,0x00]
 
-# GFX12: s_mul_f16 s5, s1, s105                  ; encoding: [0x01,0x69,0x85,0xa6]
 0x01,0x69,0x85,0xa6
+# GFX12: s_mul_f16 s5, s1, s105                  ; encoding: [0x01,0x69,0x85,0xa6]
 
-# GFX12: s_fmac_f16 s5, s1, s2                   ; encoding: [0x01,0x02,0x05,0xa7]
 0x01,0x02,0x05,0xa7
+# GFX12: s_fmac_f16 s5, s1, s2                   ; encoding: [0x01,0x02,0x05,0xa7]
 
-# GFX12: s_fmac_f16 s105, s1, s2                 ; encoding: [0x01,0x02,0x69,0xa7]
 0x01,0x02,0x69,0xa7
+# GFX12: s_fmac_f16 s105, s1, s2                 ; encoding: [0x01,0x02,0x69,0xa7]
 
-# GFX12: s_fmac_f16 s5, s105, s2                 ; encoding: [0x69,0x02,0x05,0xa7]
 0x69,0x02,0x05,0xa7
+# GFX12: s_fmac_f16 s5, s105, s2                 ; encoding: [0x69,0x02,0x05,0xa7]
 
-# GFX12: s_fmac_f16 s5, s103, s2                 ; encoding: [0x67,0x02,0x05,0xa7]
 0x67,0x02,0x05,0xa7
+# GFX12: s_fmac_f16 s5, s103, s2                 ; encoding: [0x67,0x02,0x05,0xa7]
 
-# GFX12: s_fmac_f16 s5, vcc_lo, s2               ; encoding: [0x6a,0x02,0x05,0xa7]
 0x6a,0x02,0x05,0xa7
+# GFX12: s_fmac_f16 s5, vcc_lo, s2               ; encoding: [0x6a,0x02,0x05,0xa7]
 
-# GFX12: s_fmac_f16 s5, vcc_hi, s2               ; encoding: [0x6b,0x02,0x05,0xa7]
 0x6b,0x02,0x05,0xa7
+# GFX12: s_fmac_f16 s5, vcc_hi, s2               ; encoding: [0x6b,0x02,0x05,0xa7]
 
-# GFX12: s_fmac_f16 s5, ttmp11, s2               ; encoding: [0x77,0x02,0x05,0xa7]
 0x77,0x02,0x05,0xa7
+# GFX12: s_fmac_f16 s5, ttmp11, s2               ; encoding: [0x77,0x02,0x05,0xa7]
 
-# GFX12: s_fmac_f16 s5, m0, s2                   ; encoding: [0x7d,0x02,0x05,0xa7]
 0x7d,0x02,0x05,0xa7
+# GFX12: s_fmac_f16 s5, m0, s2                   ; encoding: [0x7d,0x02,0x05,0xa7]
 
-# GFX12: s_fmac_f16 s5, exec_lo, s2              ; encoding: [0x7e,0x02,0x05,0xa7]
 0x7e,0x02,0x05,0xa7
+# GFX12: s_fmac_f16 s5, exec_lo, s2              ; encoding: [0x7e,0x02,0x05,0xa7]
 
-# GFX12: s_fmac_f16 s5, exec_hi, s2              ; encoding: [0x7f,0x02,0x05,0xa7]
 0x7f,0x02,0x05,0xa7
+# GFX12: s_fmac_f16 s5, exec_hi, s2              ; encoding: [0x7f,0x02,0x05,0xa7]
 
-# GFX12: s_fmac_f16 s5, 0, s2                    ; encoding: [0x80,0x02,0x05,0xa7]
 0x80,0x02,0x05,0xa7
+# GFX12: s_fmac_f16 s5, 0, s2                    ; encoding: [0x80,0x02,0x05,0xa7]
 
-# GFX12: s_fmac_f16 s5, -1, s2                   ; encoding: [0xc1,0x02,0x05,0xa7]
 0xc1,0x02,0x05,0xa7
+# GFX12: s_fmac_f16 s5, -1, s2                   ; encoding: [0xc1,0x02,0x05,0xa7]
 
-# GFX12: s_fmac_f16 s5, 0.5, s2                  ; encoding: [0xf0,0x02,0x05,0xa7]
 0xf0,0x02,0x05,0xa7
+# GFX12: s_fmac_f16 s5, 0.5, s2                  ; encoding: [0xf0,0x02,0x05,0xa7]
 
-# GFX12: s_fmac_f16 s5, -4.0, s2                 ; encoding: [0xf7,0x02,0x05,0xa7]
 0xf7,0x02,0x05,0xa7
+# GFX12: s_fmac_f16 s5, -4.0, s2                 ; encoding: [0xf7,0x02,0x05,0xa7]
 
-# GFX12: s_fmac_f16 s5, 0x1234, s2               ; encoding: [0xff,0x02,0x05,0xa7,0x34,0x12,0x00,0x00]
 0xff,0x02,0x05,0xa7,0x34,0x12,0x00,0x00
+# GFX12: s_fmac_f16 s5, 0x1234, s2               ; encoding: [0xff,0x02,0x05,0xa7,0x34,0x12,0x00,0x00]
 
-# GFX12: s_fmac_f16 s5, s1, s105                 ; encoding: [0x01,0x69,0x05,0xa7]
 0x01,0x69,0x05,0xa7
+# GFX12: s_fmac_f16 s5, s1, s105                 ; encoding: [0x01,0x69,0x05,0xa7]
 
-# GFX12: s_max_num_f16 s5, s1, s2                ; encoding: [0x01,0x02,0x05,0xa6]
 0x01,0x02,0x05,0xa6
+# GFX12: s_max_num_f16 s5, s1, s2                ; encoding: [0x01,0x02,0x05,0xa6]
 
-# GFX12: s_max_num_f16 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0xa6]
 0x01,0x02,0x69,0xa6
+# GFX12: s_max_num_f16 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0xa6]
 
-# GFX12: s_max_num_f16 s5, s105, s2              ; encoding: [0x69,0x02,0x05,0xa6]
 0x69,0x02,0x05,0xa6
+# GFX12: s_max_num_f16 s5, s105, s2              ; encoding: [0x69,0x02,0x05,0xa6]
 
-# GFX12: s_max_num_f16 s5, s101, s2              ; encoding: [0x65,0x02,0x05,0xa6]
 0x65,0x02,0x05,0xa6
+# GFX12: s_max_num_f16 s5, s101, s2              ; encoding: [0x65,0x02,0x05,0xa6]
 
-# GFX12: s_max_num_f16 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x05,0xa6]
 0x6a,0x02,0x05,0xa6
+# GFX12: s_max_num_f16 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x05,0xa6]
 
-# GFX12: s_max_num_f16 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x05,0xa6]
 0x6b,0x02,0x05,0xa6
+# GFX12: s_max_num_f16 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x05,0xa6]
 
-# GFX12: s_max_num_f16 s5, m0, s2                ; encoding: [0x7d,0x02,0x05,0xa6]
 0x7d,0x02,0x05,0xa6
+# GFX12: s_max_num_f16 s5, m0, s2                ; encoding: [0x7d,0x02,0x05,0xa6]
 
-# GFX12: s_max_num_f16 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x05,0xa6]
 0x7e,0x02,0x05,0xa6
+# GFX12: s_max_num_f16 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x05,0xa6]
 
-# GFX12: s_max_num_f16 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x05,0xa6]
 0x7f,0x02,0x05,0xa6
+# GFX12: s_max_num_f16 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x05,0xa6]
 
-# GFX12: s_max_num_f16 s5, 0, s2                 ; encoding: [0x80,0x02,0x05,0xa6]
 0x80,0x02,0x05,0xa6
+# GFX12: s_max_num_f16 s5, 0, s2                 ; encoding: [0x80,0x02,0x05,0xa6]
 
-# GFX12: s_max_num_f16 s5, -1, s2                ; encoding: [0xc1,0x02,0x05,0xa6]
 0xc1,0x02,0x05,0xa6
+# GFX12: s_max_num_f16 s5, -1, s2                ; encoding: [0xc1,0x02,0x05,0xa6]
 
-# GFX12: s_max_num_f16 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x05,0xa6]
 0xf0,0x02,0x05,0xa6
+# GFX12: s_max_num_f16 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x05,0xa6]
 
-# GFX12: s_max_num_f16 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x05,0xa6]
 0xf7,0x02,0x05,0xa6
+# GFX12: s_max_num_f16 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x05,0xa6]
 
-# GFX12: s_max_num_f16 s5, 0xfe0b, s2            ; encoding: [0xff,0x02,0x05,0xa6,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x05,0xa6,0x0b,0xfe,0x00,0x00
+# GFX12: s_max_num_f16 s5, 0xfe0b, s2            ; encoding: [0xff,0x02,0x05,0xa6,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_max_num_f16 s5, 0x3456, s2            ; encoding: [0xff,0x02,0x05,0xa6,0x56,0x34,0x00,0x00]
 0xff,0x02,0x05,0xa6,0x56,0x34,0x00,0x00
+# GFX12: s_max_num_f16 s5, 0x3456, s2            ; encoding: [0xff,0x02,0x05,0xa6,0x56,0x34,0x00,0x00]
 
-# GFX12: s_max_num_f16 s5, s1, s105              ; encoding: [0x01,0x69,0x05,0xa6]
 0x01,0x69,0x05,0xa6
+# GFX12: s_max_num_f16 s5, s1, s105              ; encoding: [0x01,0x69,0x05,0xa6]
 
-# GFX12: s_min_num_f16 s5, s1, s2                ; encoding: [0x01,0x02,0x85,0xa5]
 0x01,0x02,0x85,0xa5
+# GFX12: s_min_num_f16 s5, s1, s2                ; encoding: [0x01,0x02,0x85,0xa5]
 
-# GFX12: s_min_num_f16 s105, s1, s2              ; encoding: [0x01,0x02,0xe9,0xa5]
 0x01,0x02,0xe9,0xa5
+# GFX12: s_min_num_f16 s105, s1, s2              ; encoding: [0x01,0x02,0xe9,0xa5]
 
-# GFX12: s_min_num_f16 s5, s105, s2              ; encoding: [0x69,0x02,0x85,0xa5]
 0x69,0x02,0x85,0xa5
+# GFX12: s_min_num_f16 s5, s105, s2              ; encoding: [0x69,0x02,0x85,0xa5]
 
-# GFX12: s_min_num_f16 s5, s101, s2              ; encoding: [0x65,0x02,0x85,0xa5]
 0x65,0x02,0x85,0xa5
+# GFX12: s_min_num_f16 s5, s101, s2              ; encoding: [0x65,0x02,0x85,0xa5]
 
-# GFX12: s_min_num_f16 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x85,0xa5]
 0x6a,0x02,0x85,0xa5
+# GFX12: s_min_num_f16 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x85,0xa5]
 
-# GFX12: s_min_num_f16 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x85,0xa5]
 0x6b,0x02,0x85,0xa5
+# GFX12: s_min_num_f16 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x85,0xa5]
 
-# GFX12: s_min_num_f16 s5, m0, s2                ; encoding: [0x7d,0x02,0x85,0xa5]
 0x7d,0x02,0x85,0xa5
+# GFX12: s_min_num_f16 s5, m0, s2                ; encoding: [0x7d,0x02,0x85,0xa5]
 
-# GFX12: s_min_num_f16 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x85,0xa5]
 0x7e,0x02,0x85,0xa5
+# GFX12: s_min_num_f16 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x85,0xa5]
 
-# GFX12: s_min_num_f16 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x85,0xa5]
 0x7f,0x02,0x85,0xa5
+# GFX12: s_min_num_f16 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x85,0xa5]
 
-# GFX12: s_min_num_f16 s5, 0, s2                 ; encoding: [0x80,0x02,0x85,0xa5]
 0x80,0x02,0x85,0xa5
+# GFX12: s_min_num_f16 s5, 0, s2                 ; encoding: [0x80,0x02,0x85,0xa5]
 
-# GFX12: s_min_num_f16 s5, -1, s2                ; encoding: [0xc1,0x02,0x85,0xa5]
 0xc1,0x02,0x85,0xa5
+# GFX12: s_min_num_f16 s5, -1, s2                ; encoding: [0xc1,0x02,0x85,0xa5]
 
-# GFX12: s_min_num_f16 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x85,0xa5]
 0xf0,0x02,0x85,0xa5
+# GFX12: s_min_num_f16 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x85,0xa5]
 
-# GFX12: s_min_num_f16 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x85,0xa5]
 0xf7,0x02,0x85,0xa5
+# GFX12: s_min_num_f16 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x85,0xa5]
 
-# GFX12: s_min_num_f16 s5, 0xfe0b, s2            ; encoding: [0xff,0x02,0x85,0xa5,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x85,0xa5,0x0b,0xfe,0x00,0x00
+# GFX12: s_min_num_f16 s5, 0xfe0b, s2            ; encoding: [0xff,0x02,0x85,0xa5,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_min_num_f16 s5, 0x3456, s2            ; encoding: [0xff,0x02,0x85,0xa5,0x56,0x34,0x00,0x00]
 0xff,0x02,0x85,0xa5,0x56,0x34,0x00,0x00
+# GFX12: s_min_num_f16 s5, 0x3456, s2            ; encoding: [0xff,0x02,0x85,0xa5,0x56,0x34,0x00,0x00]
 
-# GFX12: s_min_num_f16 s5, s1, s105              ; encoding: [0x01,0x69,0x85,0xa5]
 0x01,0x69,0x85,0xa5
+# GFX12: s_min_num_f16 s5, s1, s105              ; encoding: [0x01,0x69,0x85,0xa5]
 
-# GFX12: s_maximum_f16 s5, s1, s2                ; encoding: [0x01,0x02,0x05,0xa9]
 0x01,0x02,0x05,0xa9
+# GFX12: s_maximum_f16 s5, s1, s2                ; encoding: [0x01,0x02,0x05,0xa9]
 
-# GFX12: s_maximum_f16 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0xa9]
 0x01,0x02,0x69,0xa9
+# GFX12: s_maximum_f16 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0xa9]
 
-# GFX12: s_maximum_f16 s5, s105, s2              ; encoding: [0x69,0x02,0x05,0xa9]
 0x69,0x02,0x05,0xa9
+# GFX12: s_maximum_f16 s5, s105, s2              ; encoding: [0x69,0x02,0x05,0xa9]
 
-# GFX12: s_maximum_f16 s5, s101, s2              ; encoding: [0x65,0x02,0x05,0xa9]
 0x65,0x02,0x05,0xa9
+# GFX12: s_maximum_f16 s5, s101, s2              ; encoding: [0x65,0x02,0x05,0xa9]
 
-# GFX12: s_maximum_f16 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x05,0xa9]
 0x6a,0x02,0x05,0xa9
+# GFX12: s_maximum_f16 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x05,0xa9]
 
-# GFX12: s_maximum_f16 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x05,0xa9]
 0x6b,0x02,0x05,0xa9
+# GFX12: s_maximum_f16 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x05,0xa9]
 
-# GFX12: s_maximum_f16 s5, m0, s2                ; encoding: [0x7d,0x02,0x05,0xa9]
 0x7d,0x02,0x05,0xa9
+# GFX12: s_maximum_f16 s5, m0, s2                ; encoding: [0x7d,0x02,0x05,0xa9]
 
-# GFX12: s_maximum_f16 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x05,0xa9]
 0x7e,0x02,0x05,0xa9
+# GFX12: s_maximum_f16 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x05,0xa9]
 
-# GFX12: s_maximum_f16 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x05,0xa9]
 0x7f,0x02,0x05,0xa9
+# GFX12: s_maximum_f16 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x05,0xa9]
 
-# GFX12: s_maximum_f16 s5, 0, s2                 ; encoding: [0x80,0x02,0x05,0xa9]
 0x80,0x02,0x05,0xa9
+# GFX12: s_maximum_f16 s5, 0, s2                 ; encoding: [0x80,0x02,0x05,0xa9]
 
-# GFX12: s_maximum_f16 s5, -1, s2                ; encoding: [0xc1,0x02,0x05,0xa9]
 0xc1,0x02,0x05,0xa9
+# GFX12: s_maximum_f16 s5, -1, s2                ; encoding: [0xc1,0x02,0x05,0xa9]
 
-# GFX12: s_maximum_f16 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x05,0xa9]
 0xf0,0x02,0x05,0xa9
+# GFX12: s_maximum_f16 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x05,0xa9]
 
-# GFX12: s_maximum_f16 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x05,0xa9]
 0xf7,0x02,0x05,0xa9
+# GFX12: s_maximum_f16 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x05,0xa9]
 
-# GFX12: s_maximum_f16 s5, 0xfe0b, s2            ; encoding: [0xff,0x02,0x05,0xa9,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x05,0xa9,0x0b,0xfe,0x00,0x00
+# GFX12: s_maximum_f16 s5, 0xfe0b, s2            ; encoding: [0xff,0x02,0x05,0xa9,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_maximum_f16 s5, 0x3456, s2            ; encoding: [0xff,0x02,0x05,0xa9,0x56,0x34,0x00,0x00]
 0xff,0x02,0x05,0xa9,0x56,0x34,0x00,0x00
+# GFX12: s_maximum_f16 s5, 0x3456, s2            ; encoding: [0xff,0x02,0x05,0xa9,0x56,0x34,0x00,0x00]
 
-# GFX12: s_maximum_f16 s5, s1, s105              ; encoding: [0x01,0x69,0x05,0xa9]
 0x01,0x69,0x05,0xa9
+# GFX12: s_maximum_f16 s5, s1, s105              ; encoding: [0x01,0x69,0x05,0xa9]
 
-# GFX12: s_minimum_f16 s5, s1, s2                ; encoding: [0x01,0x02,0x85,0xa8]
 0x01,0x02,0x85,0xa8
+# GFX12: s_minimum_f16 s5, s1, s2                ; encoding: [0x01,0x02,0x85,0xa8]
 
-# GFX12: s_minimum_f16 s105, s1, s2              ; encoding: [0x01,0x02,0xe9,0xa8]
 0x01,0x02,0xe9,0xa8
+# GFX12: s_minimum_f16 s105, s1, s2              ; encoding: [0x01,0x02,0xe9,0xa8]
 
-# GFX12: s_minimum_f16 s5, s105, s2              ; encoding: [0x69,0x02,0x85,0xa8]
 0x69,0x02,0x85,0xa8
+# GFX12: s_minimum_f16 s5, s105, s2              ; encoding: [0x69,0x02,0x85,0xa8]
 
-# GFX12: s_minimum_f16 s5, s101, s2              ; encoding: [0x65,0x02,0x85,0xa8]
 0x65,0x02,0x85,0xa8
+# GFX12: s_minimum_f16 s5, s101, s2              ; encoding: [0x65,0x02,0x85,0xa8]
 
-# GFX12: s_minimum_f16 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x85,0xa8]
 0x6a,0x02,0x85,0xa8
+# GFX12: s_minimum_f16 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x85,0xa8]
 
-# GFX12: s_minimum_f16 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x85,0xa8]
 0x6b,0x02,0x85,0xa8
+# GFX12: s_minimum_f16 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x85,0xa8]
 
-# GFX12: s_minimum_f16 s5, m0, s2                ; encoding: [0x7d,0x02,0x85,0xa8]
 0x7d,0x02,0x85,0xa8
+# GFX12: s_minimum_f16 s5, m0, s2                ; encoding: [0x7d,0x02,0x85,0xa8]
 
-# GFX12: s_minimum_f16 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x85,0xa8]
 0x7e,0x02,0x85,0xa8
+# GFX12: s_minimum_f16 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x85,0xa8]
 
-# GFX12: s_minimum_f16 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x85,0xa8]
 0x7f,0x02,0x85,0xa8
+# GFX12: s_minimum_f16 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x85,0xa8]
 
-# GFX12: s_minimum_f16 s5, 0, s2                 ; encoding: [0x80,0x02,0x85,0xa8]
 0x80,0x02,0x85,0xa8
+# GFX12: s_minimum_f16 s5, 0, s2                 ; encoding: [0x80,0x02,0x85,0xa8]
 
-# GFX12: s_minimum_f16 s5, -1, s2                ; encoding: [0xc1,0x02,0x85,0xa8]
 0xc1,0x02,0x85,0xa8
+# GFX12: s_minimum_f16 s5, -1, s2                ; encoding: [0xc1,0x02,0x85,0xa8]
 
-# GFX12: s_minimum_f16 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x85,0xa8]
 0xf0,0x02,0x85,0xa8
+# GFX12: s_minimum_f16 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x85,0xa8]
 
-# GFX12: s_minimum_f16 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x85,0xa8]
 0xf7,0x02,0x85,0xa8
+# GFX12: s_minimum_f16 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x85,0xa8]
 
-# GFX12: s_minimum_f16 s5, 0xfe0b, s2            ; encoding: [0xff,0x02,0x85,0xa8,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x85,0xa8,0x0b,0xfe,0x00,0x00
+# GFX12: s_minimum_f16 s5, 0xfe0b, s2            ; encoding: [0xff,0x02,0x85,0xa8,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_minimum_f16 s5, 0x3456, s2            ; encoding: [0xff,0x02,0x85,0xa8,0x56,0x34,0x00,0x00]
 0xff,0x02,0x85,0xa8,0x56,0x34,0x00,0x00
+# GFX12: s_minimum_f16 s5, 0x3456, s2            ; encoding: [0xff,0x02,0x85,0xa8,0x56,0x34,0x00,0x00]
 
-# GFX12: s_minimum_f16 s5, s1, s105              ; encoding: [0x01,0x69,0x85,0xa8]
 0x01,0x69,0x85,0xa8
+# GFX12: s_minimum_f16 s5, s1, s105              ; encoding: [0x01,0x69,0x85,0xa8]
 
-# GFX12: s_absdiff_i32 exec_hi, s1, s2           ; encoding: [0x01,0x02,0x7f,0x83]
 0x01,0x02,0x7f,0x83
+# GFX12: s_absdiff_i32 exec_hi, s1, s2           ; encoding: [0x01,0x02,0x7f,0x83]
 
-# GFX12: s_absdiff_i32 exec_lo, s1, s2           ; encoding: [0x01,0x02,0x7e,0x83]
 0x01,0x02,0x7e,0x83
+# GFX12: s_absdiff_i32 exec_lo, s1, s2           ; encoding: [0x01,0x02,0x7e,0x83]
 
-# GFX12: s_absdiff_i32 m0, s1, s2                ; encoding: [0x01,0x02,0x7d,0x83]
 0x01,0x02,0x7d,0x83
+# GFX12: s_absdiff_i32 m0, s1, s2                ; encoding: [0x01,0x02,0x7d,0x83]
 
-# GFX12: s_absdiff_i32 s0, 0.5, s2               ; encoding: [0xf0,0x02,0x00,0x83]
 0xf0,0x02,0x00,0x83
+# GFX12: s_absdiff_i32 s0, 0.5, s2               ; encoding: [0xf0,0x02,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, 0, s2                 ; encoding: [0x80,0x02,0x00,0x83]
 0x80,0x02,0x00,0x83
+# GFX12: s_absdiff_i32 s0, 0, s2                 ; encoding: [0x80,0x02,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, 0x3f717273, s2        ; encoding: [0xff,0x02,0x00,0x83,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x83,0x73,0x72,0x71,0x3f
+# GFX12: s_absdiff_i32 s0, 0x3f717273, s2        ; encoding: [0xff,0x02,0x00,0x83,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_absdiff_i32 s0, 0xaf123456, s2        ; encoding: [0xff,0x02,0x00,0x83,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x83,0x56,0x34,0x12,0xaf
+# GFX12: s_absdiff_i32 s0, 0xaf123456, s2        ; encoding: [0xff,0x02,0x00,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_absdiff_i32 s0, -1, s2                ; encoding: [0xc1,0x02,0x00,0x83]
 0xc1,0x02,0x00,0x83
+# GFX12: s_absdiff_i32 s0, -1, s2                ; encoding: [0xc1,0x02,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, -4.0, s2              ; encoding: [0xf7,0x02,0x00,0x83]
 0xf7,0x02,0x00,0x83
+# GFX12: s_absdiff_i32 s0, -4.0, s2              ; encoding: [0xf7,0x02,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, exec_hi, s2           ; encoding: [0x7f,0x02,0x00,0x83]
 0x7f,0x02,0x00,0x83
+# GFX12: s_absdiff_i32 s0, exec_hi, s2           ; encoding: [0x7f,0x02,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, exec_lo, s2           ; encoding: [0x7e,0x02,0x00,0x83]
 0x7e,0x02,0x00,0x83
+# GFX12: s_absdiff_i32 s0, exec_lo, s2           ; encoding: [0x7e,0x02,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, m0, s2                ; encoding: [0x7d,0x02,0x00,0x83]
 0x7d,0x02,0x00,0x83
+# GFX12: s_absdiff_i32 s0, m0, s2                ; encoding: [0x7d,0x02,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, s104, s103            ; encoding: [0x68,0x67,0x00,0x83]
 0x68,0x67,0x00,0x83
+# GFX12: s_absdiff_i32 s0, s104, s103            ; encoding: [0x68,0x67,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, s104, s2              ; encoding: [0x68,0x02,0x00,0x83]
 0x68,0x02,0x00,0x83
+# GFX12: s_absdiff_i32 s0, s104, s2              ; encoding: [0x68,0x02,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, s1, 0.5               ; encoding: [0x01,0xf0,0x00,0x83]
 0x01,0xf0,0x00,0x83
+# GFX12: s_absdiff_i32 s0, s1, 0.5               ; encoding: [0x01,0xf0,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, s1, 0                 ; encoding: [0x01,0x80,0x00,0x83]
 0x01,0x80,0x00,0x83
+# GFX12: s_absdiff_i32 s0, s1, 0                 ; encoding: [0x01,0x80,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, s1, 0x3f717273        ; encoding: [0x01,0xff,0x00,0x83,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x83,0x73,0x72,0x71,0x3f
+# GFX12: s_absdiff_i32 s0, s1, 0x3f717273        ; encoding: [0x01,0xff,0x00,0x83,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_absdiff_i32 s0, s1, 0xaf123456        ; encoding: [0x01,0xff,0x00,0x83,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x83,0x56,0x34,0x12,0xaf
+# GFX12: s_absdiff_i32 s0, s1, 0xaf123456        ; encoding: [0x01,0xff,0x00,0x83,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_absdiff_i32 s0, s1, -1                ; encoding: [0x01,0xc1,0x00,0x83]
 0x01,0xc1,0x00,0x83
+# GFX12: s_absdiff_i32 s0, s1, -1                ; encoding: [0x01,0xc1,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, s1, -4.0              ; encoding: [0x01,0xf7,0x00,0x83]
 0x01,0xf7,0x00,0x83
+# GFX12: s_absdiff_i32 s0, s1, -4.0              ; encoding: [0x01,0xf7,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, s1, exec_hi           ; encoding: [0x01,0x7f,0x00,0x83]
 0x01,0x7f,0x00,0x83
+# GFX12: s_absdiff_i32 s0, s1, exec_hi           ; encoding: [0x01,0x7f,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, s1, exec_lo           ; encoding: [0x01,0x7e,0x00,0x83]
 0x01,0x7e,0x00,0x83
+# GFX12: s_absdiff_i32 s0, s1, exec_lo           ; encoding: [0x01,0x7e,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, s1, m0                ; encoding: [0x01,0x7d,0x00,0x83]
 0x01,0x7d,0x00,0x83
+# GFX12: s_absdiff_i32 s0, s1, m0                ; encoding: [0x01,0x7d,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, s1, s103              ; encoding: [0x01,0x67,0x00,0x83]
 0x01,0x67,0x00,0x83
+# GFX12: s_absdiff_i32 s0, s1, s103              ; encoding: [0x01,0x67,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, s1, s2                ; encoding: [0x01,0x02,0x00,0x83]
 0x01,0x02,0x00,0x83
+# GFX12: s_absdiff_i32 s0, s1, s2                ; encoding: [0x01,0x02,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, s1, vcc_hi            ; encoding: [0x01,0x6b,0x00,0x83]
 0x01,0x6b,0x00,0x83
+# GFX12: s_absdiff_i32 s0, s1, vcc_hi            ; encoding: [0x01,0x6b,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, s1, vcc_lo            ; encoding: [0x01,0x6a,0x00,0x83]
 0x01,0x6a,0x00,0x83
+# GFX12: s_absdiff_i32 s0, s1, vcc_lo            ; encoding: [0x01,0x6a,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, vcc_hi, s2            ; encoding: [0x6b,0x02,0x00,0x83]
 0x6b,0x02,0x00,0x83
+# GFX12: s_absdiff_i32 s0, vcc_hi, s2            ; encoding: [0x6b,0x02,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s0, vcc_lo, s2            ; encoding: [0x6a,0x02,0x00,0x83]
 0x6a,0x02,0x00,0x83
+# GFX12: s_absdiff_i32 s0, vcc_lo, s2            ; encoding: [0x6a,0x02,0x00,0x83]
 
-# GFX12: s_absdiff_i32 s105, s104, s103          ; encoding: [0x68,0x67,0x69,0x83]
 0x68,0x67,0x69,0x83
+# GFX12: s_absdiff_i32 s105, s104, s103          ; encoding: [0x68,0x67,0x69,0x83]
 
-# GFX12: s_absdiff_i32 s105, s104, s2            ; encoding: [0x68,0x02,0x69,0x83]
 0x68,0x02,0x69,0x83
+# GFX12: s_absdiff_i32 s105, s104, s2            ; encoding: [0x68,0x02,0x69,0x83]
 
-# GFX12: s_absdiff_i32 s105, s1, s103            ; encoding: [0x01,0x67,0x69,0x83]
 0x01,0x67,0x69,0x83
+# GFX12: s_absdiff_i32 s105, s1, s103            ; encoding: [0x01,0x67,0x69,0x83]
 
-# GFX12: s_absdiff_i32 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0x83]
 0x01,0x02,0x69,0x83
+# GFX12: s_absdiff_i32 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0x83]
 
-# GFX12: s_absdiff_i32 vcc_hi, s1, s2            ; encoding: [0x01,0x02,0x6b,0x83]
 0x01,0x02,0x6b,0x83
+# GFX12: s_absdiff_i32 vcc_hi, s1, s2            ; encoding: [0x01,0x02,0x6b,0x83]
 
-# GFX12: s_absdiff_i32 vcc_lo, s1, s2            ; encoding: [0x01,0x02,0x6a,0x83]
 0x01,0x02,0x6a,0x83
+# GFX12: s_absdiff_i32 vcc_lo, s1, s2            ; encoding: [0x01,0x02,0x6a,0x83]
 
-# GFX12: s_add_co_ci_u32 exec_hi, s1, s2         ; encoding: [0x01,0x02,0x7f,0x82]
 0x01,0x02,0x7f,0x82
+# GFX12: s_add_co_ci_u32 exec_hi, s1, s2         ; encoding: [0x01,0x02,0x7f,0x82]
 
-# GFX12: s_add_co_ci_u32 exec_lo, s1, s2         ; encoding: [0x01,0x02,0x7e,0x82]
 0x01,0x02,0x7e,0x82
+# GFX12: s_add_co_ci_u32 exec_lo, s1, s2         ; encoding: [0x01,0x02,0x7e,0x82]
 
-# GFX12: s_add_co_ci_u32 m0, s1, s2              ; encoding: [0x01,0x02,0x7d,0x82]
 0x01,0x02,0x7d,0x82
+# GFX12: s_add_co_ci_u32 m0, s1, s2              ; encoding: [0x01,0x02,0x7d,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, 0.5, s2             ; encoding: [0xf0,0x02,0x00,0x82]
 0xf0,0x02,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, 0.5, s2             ; encoding: [0xf0,0x02,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, 0, s2               ; encoding: [0x80,0x02,0x00,0x82]
 0x80,0x02,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, 0, s2               ; encoding: [0x80,0x02,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, 0x3f717273, s2      ; encoding: [0xff,0x02,0x00,0x82,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x82,0x73,0x72,0x71,0x3f
+# GFX12: s_add_co_ci_u32 s0, 0x3f717273, s2      ; encoding: [0xff,0x02,0x00,0x82,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_add_co_ci_u32 s0, 0xaf123456, s2      ; encoding: [0xff,0x02,0x00,0x82,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x82,0x56,0x34,0x12,0xaf
+# GFX12: s_add_co_ci_u32 s0, 0xaf123456, s2      ; encoding: [0xff,0x02,0x00,0x82,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_add_co_ci_u32 s0, -1, s2              ; encoding: [0xc1,0x02,0x00,0x82]
 0xc1,0x02,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, -1, s2              ; encoding: [0xc1,0x02,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, -4.0, s2            ; encoding: [0xf7,0x02,0x00,0x82]
 0xf7,0x02,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, -4.0, s2            ; encoding: [0xf7,0x02,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, exec_hi, s2         ; encoding: [0x7f,0x02,0x00,0x82]
 0x7f,0x02,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, exec_hi, s2         ; encoding: [0x7f,0x02,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, exec_lo, s2         ; encoding: [0x7e,0x02,0x00,0x82]
 0x7e,0x02,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, exec_lo, s2         ; encoding: [0x7e,0x02,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, m0, s2              ; encoding: [0x7d,0x02,0x00,0x82]
 0x7d,0x02,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, m0, s2              ; encoding: [0x7d,0x02,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, s104, s103          ; encoding: [0x68,0x67,0x00,0x82]
 0x68,0x67,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, s104, s103          ; encoding: [0x68,0x67,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, s104, s2            ; encoding: [0x68,0x02,0x00,0x82]
 0x68,0x02,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, s104, s2            ; encoding: [0x68,0x02,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, s1, 0.5             ; encoding: [0x01,0xf0,0x00,0x82]
 0x01,0xf0,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, s1, 0.5             ; encoding: [0x01,0xf0,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, s1, 0               ; encoding: [0x01,0x80,0x00,0x82]
 0x01,0x80,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, s1, 0               ; encoding: [0x01,0x80,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, s1, 0x3f717273      ; encoding: [0x01,0xff,0x00,0x82,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x82,0x73,0x72,0x71,0x3f
+# GFX12: s_add_co_ci_u32 s0, s1, 0x3f717273      ; encoding: [0x01,0xff,0x00,0x82,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_add_co_ci_u32 s0, s1, 0xaf123456      ; encoding: [0x01,0xff,0x00,0x82,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x82,0x56,0x34,0x12,0xaf
+# GFX12: s_add_co_ci_u32 s0, s1, 0xaf123456      ; encoding: [0x01,0xff,0x00,0x82,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_add_co_ci_u32 s0, s1, -1              ; encoding: [0x01,0xc1,0x00,0x82]
 0x01,0xc1,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, s1, -1              ; encoding: [0x01,0xc1,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, s1, -4.0            ; encoding: [0x01,0xf7,0x00,0x82]
 0x01,0xf7,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, s1, -4.0            ; encoding: [0x01,0xf7,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, s1, exec_hi         ; encoding: [0x01,0x7f,0x00,0x82]
 0x01,0x7f,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, s1, exec_hi         ; encoding: [0x01,0x7f,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, s1, exec_lo         ; encoding: [0x01,0x7e,0x00,0x82]
 0x01,0x7e,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, s1, exec_lo         ; encoding: [0x01,0x7e,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, s1, m0              ; encoding: [0x01,0x7d,0x00,0x82]
 0x01,0x7d,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, s1, m0              ; encoding: [0x01,0x7d,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, s1, s103            ; encoding: [0x01,0x67,0x00,0x82]
 0x01,0x67,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, s1, s103            ; encoding: [0x01,0x67,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, s1, s2              ; encoding: [0x01,0x02,0x00,0x82]
 0x01,0x02,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, s1, s2              ; encoding: [0x01,0x02,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, s1, vcc_hi          ; encoding: [0x01,0x6b,0x00,0x82]
 0x01,0x6b,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, s1, vcc_hi          ; encoding: [0x01,0x6b,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, s1, vcc_lo          ; encoding: [0x01,0x6a,0x00,0x82]
 0x01,0x6a,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, s1, vcc_lo          ; encoding: [0x01,0x6a,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, vcc_hi, s2          ; encoding: [0x6b,0x02,0x00,0x82]
 0x6b,0x02,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, vcc_hi, s2          ; encoding: [0x6b,0x02,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s0, vcc_lo, s2          ; encoding: [0x6a,0x02,0x00,0x82]
 0x6a,0x02,0x00,0x82
+# GFX12: s_add_co_ci_u32 s0, vcc_lo, s2          ; encoding: [0x6a,0x02,0x00,0x82]
 
-# GFX12: s_add_co_ci_u32 s105, s104, s103        ; encoding: [0x68,0x67,0x69,0x82]
 0x68,0x67,0x69,0x82
+# GFX12: s_add_co_ci_u32 s105, s104, s103        ; encoding: [0x68,0x67,0x69,0x82]
 
-# GFX12: s_add_co_ci_u32 s105, s104, s2          ; encoding: [0x68,0x02,0x69,0x82]
 0x68,0x02,0x69,0x82
+# GFX12: s_add_co_ci_u32 s105, s104, s2          ; encoding: [0x68,0x02,0x69,0x82]
 
-# GFX12: s_add_co_ci_u32 s105, s1, s103          ; encoding: [0x01,0x67,0x69,0x82]
 0x01,0x67,0x69,0x82
+# GFX12: s_add_co_ci_u32 s105, s1, s103          ; encoding: [0x01,0x67,0x69,0x82]
 
-# GFX12: s_add_co_ci_u32 s105, s1, s2            ; encoding: [0x01,0x02,0x69,0x82]
 0x01,0x02,0x69,0x82
+# GFX12: s_add_co_ci_u32 s105, s1, s2            ; encoding: [0x01,0x02,0x69,0x82]
 
-# GFX12: s_add_co_ci_u32 vcc_hi, s1, s2          ; encoding: [0x01,0x02,0x6b,0x82]
 0x01,0x02,0x6b,0x82
+# GFX12: s_add_co_ci_u32 vcc_hi, s1, s2          ; encoding: [0x01,0x02,0x6b,0x82]
 
-# GFX12: s_add_co_ci_u32 vcc_lo, s1, s2          ; encoding: [0x01,0x02,0x6a,0x82]
 0x01,0x02,0x6a,0x82
+# GFX12: s_add_co_ci_u32 vcc_lo, s1, s2          ; encoding: [0x01,0x02,0x6a,0x82]
 
-# GFX12: s_add_co_i32 exec_hi, s1, s2            ; encoding: [0x01,0x02,0x7f,0x81]
 0x01,0x02,0x7f,0x81
+# GFX12: s_add_co_i32 exec_hi, s1, s2            ; encoding: [0x01,0x02,0x7f,0x81]
 
-# GFX12: s_add_co_i32 exec_lo, s1, s2            ; encoding: [0x01,0x02,0x7e,0x81]
 0x01,0x02,0x7e,0x81
+# GFX12: s_add_co_i32 exec_lo, s1, s2            ; encoding: [0x01,0x02,0x7e,0x81]
 
-# GFX12: s_add_co_i32 m0, s1, s2                 ; encoding: [0x01,0x02,0x7d,0x81]
 0x01,0x02,0x7d,0x81
+# GFX12: s_add_co_i32 m0, s1, s2                 ; encoding: [0x01,0x02,0x7d,0x81]
 
-# GFX12: s_add_co_i32 s0, 0.5, s2                ; encoding: [0xf0,0x02,0x00,0x81]
 0xf0,0x02,0x00,0x81
+# GFX12: s_add_co_i32 s0, 0.5, s2                ; encoding: [0xf0,0x02,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, 0, s2                  ; encoding: [0x80,0x02,0x00,0x81]
 0x80,0x02,0x00,0x81
+# GFX12: s_add_co_i32 s0, 0, s2                  ; encoding: [0x80,0x02,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, 0x3f717273, s2         ; encoding: [0xff,0x02,0x00,0x81,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x81,0x73,0x72,0x71,0x3f
+# GFX12: s_add_co_i32 s0, 0x3f717273, s2         ; encoding: [0xff,0x02,0x00,0x81,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_add_co_i32 s0, 0xaf123456, s2         ; encoding: [0xff,0x02,0x00,0x81,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x81,0x56,0x34,0x12,0xaf
+# GFX12: s_add_co_i32 s0, 0xaf123456, s2         ; encoding: [0xff,0x02,0x00,0x81,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_add_co_i32 s0, -1, s2                 ; encoding: [0xc1,0x02,0x00,0x81]
 0xc1,0x02,0x00,0x81
+# GFX12: s_add_co_i32 s0, -1, s2                 ; encoding: [0xc1,0x02,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, -4.0, s2               ; encoding: [0xf7,0x02,0x00,0x81]
 0xf7,0x02,0x00,0x81
+# GFX12: s_add_co_i32 s0, -4.0, s2               ; encoding: [0xf7,0x02,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, exec_hi, s2            ; encoding: [0x7f,0x02,0x00,0x81]
 0x7f,0x02,0x00,0x81
+# GFX12: s_add_co_i32 s0, exec_hi, s2            ; encoding: [0x7f,0x02,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, exec_lo, s2            ; encoding: [0x7e,0x02,0x00,0x81]
 0x7e,0x02,0x00,0x81
+# GFX12: s_add_co_i32 s0, exec_lo, s2            ; encoding: [0x7e,0x02,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, m0, s2                 ; encoding: [0x7d,0x02,0x00,0x81]
 0x7d,0x02,0x00,0x81
+# GFX12: s_add_co_i32 s0, m0, s2                 ; encoding: [0x7d,0x02,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, s104, s103             ; encoding: [0x68,0x67,0x00,0x81]
 0x68,0x67,0x00,0x81
+# GFX12: s_add_co_i32 s0, s104, s103             ; encoding: [0x68,0x67,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, s104, s2               ; encoding: [0x68,0x02,0x00,0x81]
 0x68,0x02,0x00,0x81
+# GFX12: s_add_co_i32 s0, s104, s2               ; encoding: [0x68,0x02,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, s1, 0.5                ; encoding: [0x01,0xf0,0x00,0x81]
 0x01,0xf0,0x00,0x81
+# GFX12: s_add_co_i32 s0, s1, 0.5                ; encoding: [0x01,0xf0,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, s1, 0                  ; encoding: [0x01,0x80,0x00,0x81]
 0x01,0x80,0x00,0x81
+# GFX12: s_add_co_i32 s0, s1, 0                  ; encoding: [0x01,0x80,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, s1, 0x3f717273         ; encoding: [0x01,0xff,0x00,0x81,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x81,0x73,0x72,0x71,0x3f
+# GFX12: s_add_co_i32 s0, s1, 0x3f717273         ; encoding: [0x01,0xff,0x00,0x81,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_add_co_i32 s0, s1, 0xaf123456         ; encoding: [0x01,0xff,0x00,0x81,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x81,0x56,0x34,0x12,0xaf
+# GFX12: s_add_co_i32 s0, s1, 0xaf123456         ; encoding: [0x01,0xff,0x00,0x81,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_add_co_i32 s0, s1, -1                 ; encoding: [0x01,0xc1,0x00,0x81]
 0x01,0xc1,0x00,0x81
+# GFX12: s_add_co_i32 s0, s1, -1                 ; encoding: [0x01,0xc1,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, s1, -4.0               ; encoding: [0x01,0xf7,0x00,0x81]
 0x01,0xf7,0x00,0x81
+# GFX12: s_add_co_i32 s0, s1, -4.0               ; encoding: [0x01,0xf7,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, s1, exec_hi            ; encoding: [0x01,0x7f,0x00,0x81]
 0x01,0x7f,0x00,0x81
+# GFX12: s_add_co_i32 s0, s1, exec_hi            ; encoding: [0x01,0x7f,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, s1, exec_lo            ; encoding: [0x01,0x7e,0x00,0x81]
 0x01,0x7e,0x00,0x81
+# GFX12: s_add_co_i32 s0, s1, exec_lo            ; encoding: [0x01,0x7e,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, s1, m0                 ; encoding: [0x01,0x7d,0x00,0x81]
 0x01,0x7d,0x00,0x81
+# GFX12: s_add_co_i32 s0, s1, m0                 ; encoding: [0x01,0x7d,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, s1, s103               ; encoding: [0x01,0x67,0x00,0x81]
 0x01,0x67,0x00,0x81
+# GFX12: s_add_co_i32 s0, s1, s103               ; encoding: [0x01,0x67,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, s1, s2                 ; encoding: [0x01,0x02,0x00,0x81]
 0x01,0x02,0x00,0x81
+# GFX12: s_add_co_i32 s0, s1, s2                 ; encoding: [0x01,0x02,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, s1, vcc_hi             ; encoding: [0x01,0x6b,0x00,0x81]
 0x01,0x6b,0x00,0x81
+# GFX12: s_add_co_i32 s0, s1, vcc_hi             ; encoding: [0x01,0x6b,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, s1, vcc_lo             ; encoding: [0x01,0x6a,0x00,0x81]
 0x01,0x6a,0x00,0x81
+# GFX12: s_add_co_i32 s0, s1, vcc_lo             ; encoding: [0x01,0x6a,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, vcc_hi, s2             ; encoding: [0x6b,0x02,0x00,0x81]
 0x6b,0x02,0x00,0x81
+# GFX12: s_add_co_i32 s0, vcc_hi, s2             ; encoding: [0x6b,0x02,0x00,0x81]
 
-# GFX12: s_add_co_i32 s0, vcc_lo, s2             ; encoding: [0x6a,0x02,0x00,0x81]
 0x6a,0x02,0x00,0x81
+# GFX12: s_add_co_i32 s0, vcc_lo, s2             ; encoding: [0x6a,0x02,0x00,0x81]
 
-# GFX12: s_add_co_i32 s105, s104, s103           ; encoding: [0x68,0x67,0x69,0x81]
 0x68,0x67,0x69,0x81
+# GFX12: s_add_co_i32 s105, s104, s103           ; encoding: [0x68,0x67,0x69,0x81]
 
-# GFX12: s_add_co_i32 s105, s104, s2             ; encoding: [0x68,0x02,0x69,0x81]
 0x68,0x02,0x69,0x81
+# GFX12: s_add_co_i32 s105, s104, s2             ; encoding: [0x68,0x02,0x69,0x81]
 
-# GFX12: s_add_co_i32 s105, s1, s103             ; encoding: [0x01,0x67,0x69,0x81]
 0x01,0x67,0x69,0x81
+# GFX12: s_add_co_i32 s105, s1, s103             ; encoding: [0x01,0x67,0x69,0x81]
 
-# GFX12: s_add_co_i32 s105, s1, s2               ; encoding: [0x01,0x02,0x69,0x81]
 0x01,0x02,0x69,0x81
+# GFX12: s_add_co_i32 s105, s1, s2               ; encoding: [0x01,0x02,0x69,0x81]
 
-# GFX12: s_add_co_i32 vcc_hi, s1, s2             ; encoding: [0x01,0x02,0x6b,0x81]
 0x01,0x02,0x6b,0x81
+# GFX12: s_add_co_i32 vcc_hi, s1, s2             ; encoding: [0x01,0x02,0x6b,0x81]
 
-# GFX12: s_add_co_i32 vcc_lo, s1, s2             ; encoding: [0x01,0x02,0x6a,0x81]
 0x01,0x02,0x6a,0x81
+# GFX12: s_add_co_i32 vcc_lo, s1, s2             ; encoding: [0x01,0x02,0x6a,0x81]
 
-# GFX12: s_add_co_u32 exec_hi, s1, s2            ; encoding: [0x01,0x02,0x7f,0x80]
 0x01,0x02,0x7f,0x80
+# GFX12: s_add_co_u32 exec_hi, s1, s2            ; encoding: [0x01,0x02,0x7f,0x80]
 
-# GFX12: s_add_co_u32 exec_lo, s1, s2            ; encoding: [0x01,0x02,0x7e,0x80]
 0x01,0x02,0x7e,0x80
+# GFX12: s_add_co_u32 exec_lo, s1, s2            ; encoding: [0x01,0x02,0x7e,0x80]
 
-# GFX12: s_add_co_u32 m0, s1, s2                 ; encoding: [0x01,0x02,0x7d,0x80]
 0x01,0x02,0x7d,0x80
+# GFX12: s_add_co_u32 m0, s1, s2                 ; encoding: [0x01,0x02,0x7d,0x80]
 
-# GFX12: s_add_co_u32 s0, 0.5, s2                ; encoding: [0xf0,0x02,0x00,0x80]
 0xf0,0x02,0x00,0x80
+# GFX12: s_add_co_u32 s0, 0.5, s2                ; encoding: [0xf0,0x02,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, 0, s2                  ; encoding: [0x80,0x02,0x00,0x80]
 0x80,0x02,0x00,0x80
+# GFX12: s_add_co_u32 s0, 0, s2                  ; encoding: [0x80,0x02,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, 0x3f717273, s2         ; encoding: [0xff,0x02,0x00,0x80,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x80,0x73,0x72,0x71,0x3f
+# GFX12: s_add_co_u32 s0, 0x3f717273, s2         ; encoding: [0xff,0x02,0x00,0x80,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_add_co_u32 s0, 0xaf123456, s2         ; encoding: [0xff,0x02,0x00,0x80,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x80,0x56,0x34,0x12,0xaf
+# GFX12: s_add_co_u32 s0, 0xaf123456, s2         ; encoding: [0xff,0x02,0x00,0x80,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_add_co_u32 s0, -1, s2                 ; encoding: [0xc1,0x02,0x00,0x80]
 0xc1,0x02,0x00,0x80
+# GFX12: s_add_co_u32 s0, -1, s2                 ; encoding: [0xc1,0x02,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, -4.0, s2               ; encoding: [0xf7,0x02,0x00,0x80]
 0xf7,0x02,0x00,0x80
+# GFX12: s_add_co_u32 s0, -4.0, s2               ; encoding: [0xf7,0x02,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, exec_hi, s2            ; encoding: [0x7f,0x02,0x00,0x80]
 0x7f,0x02,0x00,0x80
+# GFX12: s_add_co_u32 s0, exec_hi, s2            ; encoding: [0x7f,0x02,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, exec_lo, s2            ; encoding: [0x7e,0x02,0x00,0x80]
 0x7e,0x02,0x00,0x80
+# GFX12: s_add_co_u32 s0, exec_lo, s2            ; encoding: [0x7e,0x02,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, m0, s2                 ; encoding: [0x7d,0x02,0x00,0x80]
 0x7d,0x02,0x00,0x80
+# GFX12: s_add_co_u32 s0, m0, s2                 ; encoding: [0x7d,0x02,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, s104, s103             ; encoding: [0x68,0x67,0x00,0x80]
 0x68,0x67,0x00,0x80
+# GFX12: s_add_co_u32 s0, s104, s103             ; encoding: [0x68,0x67,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, s104, s2               ; encoding: [0x68,0x02,0x00,0x80]
 0x68,0x02,0x00,0x80
+# GFX12: s_add_co_u32 s0, s104, s2               ; encoding: [0x68,0x02,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, s1, 0.5                ; encoding: [0x01,0xf0,0x00,0x80]
 0x01,0xf0,0x00,0x80
+# GFX12: s_add_co_u32 s0, s1, 0.5                ; encoding: [0x01,0xf0,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, s1, 0                  ; encoding: [0x01,0x80,0x00,0x80]
 0x01,0x80,0x00,0x80
+# GFX12: s_add_co_u32 s0, s1, 0                  ; encoding: [0x01,0x80,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, s1, 0x3f717273         ; encoding: [0x01,0xff,0x00,0x80,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x80,0x73,0x72,0x71,0x3f
+# GFX12: s_add_co_u32 s0, s1, 0x3f717273         ; encoding: [0x01,0xff,0x00,0x80,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_add_co_u32 s0, s1, 0xaf123456         ; encoding: [0x01,0xff,0x00,0x80,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x80,0x56,0x34,0x12,0xaf
+# GFX12: s_add_co_u32 s0, s1, 0xaf123456         ; encoding: [0x01,0xff,0x00,0x80,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_add_co_u32 s0, s1, -1                 ; encoding: [0x01,0xc1,0x00,0x80]
 0x01,0xc1,0x00,0x80
+# GFX12: s_add_co_u32 s0, s1, -1                 ; encoding: [0x01,0xc1,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, s1, -4.0               ; encoding: [0x01,0xf7,0x00,0x80]
 0x01,0xf7,0x00,0x80
+# GFX12: s_add_co_u32 s0, s1, -4.0               ; encoding: [0x01,0xf7,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, s1, exec_hi            ; encoding: [0x01,0x7f,0x00,0x80]
 0x01,0x7f,0x00,0x80
+# GFX12: s_add_co_u32 s0, s1, exec_hi            ; encoding: [0x01,0x7f,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, s1, exec_lo            ; encoding: [0x01,0x7e,0x00,0x80]
 0x01,0x7e,0x00,0x80
+# GFX12: s_add_co_u32 s0, s1, exec_lo            ; encoding: [0x01,0x7e,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, s1, m0                 ; encoding: [0x01,0x7d,0x00,0x80]
 0x01,0x7d,0x00,0x80
+# GFX12: s_add_co_u32 s0, s1, m0                 ; encoding: [0x01,0x7d,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, s1, s103               ; encoding: [0x01,0x67,0x00,0x80]
 0x01,0x67,0x00,0x80
+# GFX12: s_add_co_u32 s0, s1, s103               ; encoding: [0x01,0x67,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, s1, s2                 ; encoding: [0x01,0x02,0x00,0x80]
 0x01,0x02,0x00,0x80
+# GFX12: s_add_co_u32 s0, s1, s2                 ; encoding: [0x01,0x02,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, s1, vcc_hi             ; encoding: [0x01,0x6b,0x00,0x80]
 0x01,0x6b,0x00,0x80
+# GFX12: s_add_co_u32 s0, s1, vcc_hi             ; encoding: [0x01,0x6b,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, s1, vcc_lo             ; encoding: [0x01,0x6a,0x00,0x80]
 0x01,0x6a,0x00,0x80
+# GFX12: s_add_co_u32 s0, s1, vcc_lo             ; encoding: [0x01,0x6a,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, vcc_hi, s2             ; encoding: [0x6b,0x02,0x00,0x80]
 0x6b,0x02,0x00,0x80
+# GFX12: s_add_co_u32 s0, vcc_hi, s2             ; encoding: [0x6b,0x02,0x00,0x80]
 
-# GFX12: s_add_co_u32 s0, vcc_lo, s2             ; encoding: [0x6a,0x02,0x00,0x80]
 0x6a,0x02,0x00,0x80
+# GFX12: s_add_co_u32 s0, vcc_lo, s2             ; encoding: [0x6a,0x02,0x00,0x80]
 
-# GFX12: s_add_co_u32 s105, s104, s103           ; encoding: [0x68,0x67,0x69,0x80]
 0x68,0x67,0x69,0x80
+# GFX12: s_add_co_u32 s105, s104, s103           ; encoding: [0x68,0x67,0x69,0x80]
 
-# GFX12: s_add_co_u32 s105, s104, s2             ; encoding: [0x68,0x02,0x69,0x80]
 0x68,0x02,0x69,0x80
+# GFX12: s_add_co_u32 s105, s104, s2             ; encoding: [0x68,0x02,0x69,0x80]
 
-# GFX12: s_add_co_u32 s105, s1, s103             ; encoding: [0x01,0x67,0x69,0x80]
 0x01,0x67,0x69,0x80
+# GFX12: s_add_co_u32 s105, s1, s103             ; encoding: [0x01,0x67,0x69,0x80]
 
-# GFX12: s_add_co_u32 s105, s1, s2               ; encoding: [0x01,0x02,0x69,0x80]
 0x01,0x02,0x69,0x80
+# GFX12: s_add_co_u32 s105, s1, s2               ; encoding: [0x01,0x02,0x69,0x80]
 
-# GFX12: s_add_co_u32 vcc_hi, s1, s2             ; encoding: [0x01,0x02,0x6b,0x80]
 0x01,0x02,0x6b,0x80
+# GFX12: s_add_co_u32 vcc_hi, s1, s2             ; encoding: [0x01,0x02,0x6b,0x80]
 
-# GFX12: s_add_co_u32 vcc_lo, s1, s2             ; encoding: [0x01,0x02,0x6a,0x80]
 0x01,0x02,0x6a,0x80
+# GFX12: s_add_co_u32 vcc_lo, s1, s2             ; encoding: [0x01,0x02,0x6a,0x80]
 
-# GFX12: s_and_b32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x8b]
 0x01,0x02,0x7f,0x8b
+# GFX12: s_and_b32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x8b]
 
-# GFX12: s_and_b32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x8b]
 0x01,0x02,0x7e,0x8b
+# GFX12: s_and_b32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x8b]
 
-# GFX12: s_and_b32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x8b]
 0x01,0x02,0x7d,0x8b
+# GFX12: s_and_b32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x8b]
 
-# GFX12: s_and_b32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x8b]
 0xf0,0x02,0x00,0x8b
+# GFX12: s_and_b32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x8b]
 0x80,0x02,0x00,0x8b
+# GFX12: s_and_b32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x8b,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x8b,0x73,0x72,0x71,0x3f
+# GFX12: s_and_b32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x8b,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_and_b32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x8b,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x8b,0x56,0x34,0x12,0xaf
+# GFX12: s_and_b32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x8b,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_and_b32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x8b]
 0xc1,0x02,0x00,0x8b
+# GFX12: s_and_b32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x8b]
 0xf7,0x02,0x00,0x8b
+# GFX12: s_and_b32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x8b]
 0x7f,0x02,0x00,0x8b
+# GFX12: s_and_b32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x8b]
 0x7e,0x02,0x00,0x8b
+# GFX12: s_and_b32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x8b]
 0x7d,0x02,0x00,0x8b
+# GFX12: s_and_b32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x8b]
 0x68,0x67,0x00,0x8b
+# GFX12: s_and_b32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x8b]
 0x68,0x02,0x00,0x8b
+# GFX12: s_and_b32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x8b]
 0x01,0xf0,0x00,0x8b
+# GFX12: s_and_b32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x8b]
 0x01,0x80,0x00,0x8b
+# GFX12: s_and_b32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x8b,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x8b,0x73,0x72,0x71,0x3f
+# GFX12: s_and_b32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x8b,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_and_b32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x8b,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x8b,0x56,0x34,0x12,0xaf
+# GFX12: s_and_b32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x8b,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_and_b32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x8b]
 0x01,0xc1,0x00,0x8b
+# GFX12: s_and_b32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x8b]
 0x01,0xf7,0x00,0x8b
+# GFX12: s_and_b32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x8b]
 0x01,0x7f,0x00,0x8b
+# GFX12: s_and_b32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x8b]
 0x01,0x7e,0x00,0x8b
+# GFX12: s_and_b32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x8b]
 0x01,0x7d,0x00,0x8b
+# GFX12: s_and_b32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x8b]
 0x01,0x67,0x00,0x8b
+# GFX12: s_and_b32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x8b]
 0x01,0x02,0x00,0x8b
+# GFX12: s_and_b32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x8b]
 0x01,0x6b,0x00,0x8b
+# GFX12: s_and_b32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x8b]
 0x01,0x6a,0x00,0x8b
+# GFX12: s_and_b32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x8b]
 0x6b,0x02,0x00,0x8b
+# GFX12: s_and_b32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x8b]
 0x6a,0x02,0x00,0x8b
+# GFX12: s_and_b32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x8b]
 
-# GFX12: s_and_b32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x8b]
 0x68,0x67,0x69,0x8b
+# GFX12: s_and_b32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x8b]
 
-# GFX12: s_and_b32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x8b]
 0x68,0x02,0x69,0x8b
+# GFX12: s_and_b32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x8b]
 
-# GFX12: s_and_b32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x8b]
 0x01,0x67,0x69,0x8b
+# GFX12: s_and_b32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x8b]
 
-# GFX12: s_and_b32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x8b]
 0x01,0x02,0x69,0x8b
+# GFX12: s_and_b32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x8b]
 
-# GFX12: s_and_b32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x8b]
 0x01,0x02,0x6b,0x8b
+# GFX12: s_and_b32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x8b]
 
-# GFX12: s_and_b32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x8b]
 0x01,0x02,0x6a,0x8b
+# GFX12: s_and_b32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x8b]
 
-# GFX12: s_and_b32 s0, s1, null                  ; encoding: [0x01,0x7c,0x00,0x8b]
 0x01,0x7c,0x00,0x8b
+# GFX12: s_and_b32 s0, s1, null                  ; encoding: [0x01,0x7c,0x00,0x8b]
 
-# GFX12: s_and_b32 s0, null, s2                  ; encoding: [0x7c,0x02,0x00,0x8b]
 0x7c,0x02,0x00,0x8b
+# GFX12: s_and_b32 s0, null, s2                  ; encoding: [0x7c,0x02,0x00,0x8b]
 
-# GFX12: s_and_b32 null, s1, s2                  ; encoding: [0x01,0x02,0x7c,0x8b]
 0x01,0x02,0x7c,0x8b
+# GFX12: s_and_b32 null, s1, s2                  ; encoding: [0x01,0x02,0x7c,0x8b]
 
-# GFX12: s_and_b64 exec, s[2:3], s[4:5]          ; encoding: [0x02,0x04,0xfe,0x8b]
 0x02,0x04,0xfe,0x8b
+# GFX12: s_and_b64 exec, s[2:3], s[4:5]          ; encoding: [0x02,0x04,0xfe,0x8b]
 
-# GFX12: s_and_b64 s[0:1], 0.5, s[4:5]           ; encoding: [0xf0,0x04,0x80,0x8b]
 0xf0,0x04,0x80,0x8b
+# GFX12: s_and_b64 s[0:1], 0.5, s[4:5]           ; encoding: [0xf0,0x04,0x80,0x8b]
 
-# GFX12: s_and_b64 s[0:1], 0, s[4:5]             ; encoding: [0x80,0x04,0x80,0x8b]
 0x80,0x04,0x80,0x8b
+# GFX12: s_and_b64 s[0:1], 0, s[4:5]             ; encoding: [0x80,0x04,0x80,0x8b]
 
-# GFX12: s_and_b64 s[0:1], 0x3f717273, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8b,0x73,0x72,0x71,0x3f]
 0xff,0x04,0x80,0x8b,0x73,0x72,0x71,0x3f
+# GFX12: s_and_b64 s[0:1], 0x3f717273, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8b,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_and_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8b,0x56,0x34,0x12,0xaf]
 0xff,0x04,0x80,0x8b,0x56,0x34,0x12,0xaf
+# GFX12: s_and_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8b,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_and_b64 s[0:1], -1, s[4:5]            ; encoding: [0xc1,0x04,0x80,0x8b]
 0xc1,0x04,0x80,0x8b
+# GFX12: s_and_b64 s[0:1], -1, s[4:5]            ; encoding: [0xc1,0x04,0x80,0x8b]
 
-# GFX12: s_and_b64 s[0:1], -4.0, s[4:5]          ; encoding: [0xf7,0x04,0x80,0x8b]
 0xf7,0x04,0x80,0x8b
+# GFX12: s_and_b64 s[0:1], -4.0, s[4:5]          ; encoding: [0xf7,0x04,0x80,0x8b]
 
-# GFX12: s_and_b64 s[0:1], exec, s[4:5]          ; encoding: [0x7e,0x04,0x80,0x8b]
 0x7e,0x04,0x80,0x8b
+# GFX12: s_and_b64 s[0:1], exec, s[4:5]          ; encoding: [0x7e,0x04,0x80,0x8b]
 
-# GFX12: s_and_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x8b]
 0x66,0x64,0x80,0x8b
+# GFX12: s_and_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x8b]
 
-# GFX12: s_and_b64 s[0:1], s[102:103], s[4:5]    ; encoding: [0x66,0x04,0x80,0x8b]
 0x66,0x04,0x80,0x8b
+# GFX12: s_and_b64 s[0:1], s[102:103], s[4:5]    ; encoding: [0x66,0x04,0x80,0x8b]
 
-# GFX12: s_and_b64 s[0:1], s[2:3], 0.5           ; encoding: [0x02,0xf0,0x80,0x8b]
 0x02,0xf0,0x80,0x8b
+# GFX12: s_and_b64 s[0:1], s[2:3], 0.5           ; encoding: [0x02,0xf0,0x80,0x8b]
 
-# GFX12: s_and_b64 s[0:1], s[2:3], 0             ; encoding: [0x02,0x80,0x80,0x8b]
 0x02,0x80,0x80,0x8b
+# GFX12: s_and_b64 s[0:1], s[2:3], 0             ; encoding: [0x02,0x80,0x80,0x8b]
 
-# GFX12: s_and_b64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0x8b,0x73,0x72,0x71,0x3f]
 0x02,0xff,0x80,0x8b,0x73,0x72,0x71,0x3f
+# GFX12: s_and_b64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0x8b,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_and_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8b,0x56,0x34,0x12,0xaf]
 0x02,0xff,0x80,0x8b,0x56,0x34,0x12,0xaf
+# GFX12: s_and_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8b,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_and_b64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x80,0x8b]
 0x02,0xc1,0x80,0x8b
+# GFX12: s_and_b64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x80,0x8b]
 
-# GFX12: s_and_b64 s[0:1], s[2:3], -4.0          ; encoding: [0x02,0xf7,0x80,0x8b]
 0x02,0xf7,0x80,0x8b
+# GFX12: s_and_b64 s[0:1], s[2:3], -4.0          ; encoding: [0x02,0xf7,0x80,0x8b]
 
-# GFX12: s_and_b64 s[0:1], s[2:3], exec          ; encoding: [0x02,0x7e,0x80,0x8b]
 0x02,0x7e,0x80,0x8b
+# GFX12: s_and_b64 s[0:1], s[2:3], exec          ; encoding: [0x02,0x7e,0x80,0x8b]
 
-# GFX12: s_and_b64 s[0:1], s[2:3], s[100:101]    ; encoding: [0x02,0x64,0x80,0x8b]
 0x02,0x64,0x80,0x8b
+# GFX12: s_and_b64 s[0:1], s[2:3], s[100:101]    ; encoding: [0x02,0x64,0x80,0x8b]
 
-# GFX12: s_and_b64 s[0:1], s[2:3], s[4:5]        ; encoding: [0x02,0x04,0x80,0x8b]
 0x02,0x04,0x80,0x8b
+# GFX12: s_and_b64 s[0:1], s[2:3], s[4:5]        ; encoding: [0x02,0x04,0x80,0x8b]
 
-# GFX12: s_and_b64 s[0:1], s[2:3], vcc           ; encoding: [0x02,0x6a,0x80,0x8b]
 0x02,0x6a,0x80,0x8b
+# GFX12: s_and_b64 s[0:1], s[2:3], vcc           ; encoding: [0x02,0x6a,0x80,0x8b]
 
-# GFX12: s_and_b64 s[0:1], vcc, s[4:5]           ; encoding: [0x6a,0x04,0x80,0x8b]
 0x6a,0x04,0x80,0x8b
+# GFX12: s_and_b64 s[0:1], vcc, s[4:5]           ; encoding: [0x6a,0x04,0x80,0x8b]
 
-# GFX12: s_and_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x8b]
 0x66,0x64,0xe8,0x8b
+# GFX12: s_and_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x8b]
 
-# GFX12: s_and_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x8b]
 0x66,0x04,0xe8,0x8b
+# GFX12: s_and_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x8b]
 
-# GFX12: s_and_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x8b]
 0x02,0x64,0xe8,0x8b
+# GFX12: s_and_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x8b]
 
-# GFX12: s_and_b64 s[104:105], s[2:3], s[4:5]    ; encoding: [0x02,0x04,0xe8,0x8b]
 0x02,0x04,0xe8,0x8b
+# GFX12: s_and_b64 s[104:105], s[2:3], s[4:5]    ; encoding: [0x02,0x04,0xe8,0x8b]
 
-# GFX12: s_and_b64 vcc, s[2:3], s[4:5]           ; encoding: [0x02,0x04,0xea,0x8b]
 0x02,0x04,0xea,0x8b
+# GFX12: s_and_b64 vcc, s[2:3], s[4:5]           ; encoding: [0x02,0x04,0xea,0x8b]
 
-# GFX12: s_and_not1_b32 exec_hi, s1, s2          ; encoding: [0x01,0x02,0x7f,0x91]
 0x01,0x02,0x7f,0x91
+# GFX12: s_and_not1_b32 exec_hi, s1, s2          ; encoding: [0x01,0x02,0x7f,0x91]
 
-# GFX12: s_and_not1_b32 exec_lo, s1, s2          ; encoding: [0x01,0x02,0x7e,0x91]
 0x01,0x02,0x7e,0x91
+# GFX12: s_and_not1_b32 exec_lo, s1, s2          ; encoding: [0x01,0x02,0x7e,0x91]
 
-# GFX12: s_and_not1_b32 m0, s1, s2               ; encoding: [0x01,0x02,0x7d,0x91]
 0x01,0x02,0x7d,0x91
+# GFX12: s_and_not1_b32 m0, s1, s2               ; encoding: [0x01,0x02,0x7d,0x91]
 
-# GFX12: s_and_not1_b32 s0, 0.5, s2              ; encoding: [0xf0,0x02,0x00,0x91]
 0xf0,0x02,0x00,0x91
+# GFX12: s_and_not1_b32 s0, 0.5, s2              ; encoding: [0xf0,0x02,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, 0, s2                ; encoding: [0x80,0x02,0x00,0x91]
 0x80,0x02,0x00,0x91
+# GFX12: s_and_not1_b32 s0, 0, s2                ; encoding: [0x80,0x02,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, 0x3f717273, s2       ; encoding: [0xff,0x02,0x00,0x91,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x91,0x73,0x72,0x71,0x3f
+# GFX12: s_and_not1_b32 s0, 0x3f717273, s2       ; encoding: [0xff,0x02,0x00,0x91,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_and_not1_b32 s0, 0xaf123456, s2       ; encoding: [0xff,0x02,0x00,0x91,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x91,0x56,0x34,0x12,0xaf
+# GFX12: s_and_not1_b32 s0, 0xaf123456, s2       ; encoding: [0xff,0x02,0x00,0x91,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_and_not1_b32 s0, -1, s2               ; encoding: [0xc1,0x02,0x00,0x91]
 0xc1,0x02,0x00,0x91
+# GFX12: s_and_not1_b32 s0, -1, s2               ; encoding: [0xc1,0x02,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, -4.0, s2             ; encoding: [0xf7,0x02,0x00,0x91]
 0xf7,0x02,0x00,0x91
+# GFX12: s_and_not1_b32 s0, -4.0, s2             ; encoding: [0xf7,0x02,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, exec_hi, s2          ; encoding: [0x7f,0x02,0x00,0x91]
 0x7f,0x02,0x00,0x91
+# GFX12: s_and_not1_b32 s0, exec_hi, s2          ; encoding: [0x7f,0x02,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, exec_lo, s2          ; encoding: [0x7e,0x02,0x00,0x91]
 0x7e,0x02,0x00,0x91
+# GFX12: s_and_not1_b32 s0, exec_lo, s2          ; encoding: [0x7e,0x02,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, m0, s2               ; encoding: [0x7d,0x02,0x00,0x91]
 0x7d,0x02,0x00,0x91
+# GFX12: s_and_not1_b32 s0, m0, s2               ; encoding: [0x7d,0x02,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, s104, s103           ; encoding: [0x68,0x67,0x00,0x91]
 0x68,0x67,0x00,0x91
+# GFX12: s_and_not1_b32 s0, s104, s103           ; encoding: [0x68,0x67,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, s104, s2             ; encoding: [0x68,0x02,0x00,0x91]
 0x68,0x02,0x00,0x91
+# GFX12: s_and_not1_b32 s0, s104, s2             ; encoding: [0x68,0x02,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, s1, 0.5              ; encoding: [0x01,0xf0,0x00,0x91]
 0x01,0xf0,0x00,0x91
+# GFX12: s_and_not1_b32 s0, s1, 0.5              ; encoding: [0x01,0xf0,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, s1, 0                ; encoding: [0x01,0x80,0x00,0x91]
 0x01,0x80,0x00,0x91
+# GFX12: s_and_not1_b32 s0, s1, 0                ; encoding: [0x01,0x80,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, s1, 0x3f717273       ; encoding: [0x01,0xff,0x00,0x91,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x91,0x73,0x72,0x71,0x3f
+# GFX12: s_and_not1_b32 s0, s1, 0x3f717273       ; encoding: [0x01,0xff,0x00,0x91,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_and_not1_b32 s0, s1, 0xaf123456       ; encoding: [0x01,0xff,0x00,0x91,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x91,0x56,0x34,0x12,0xaf
+# GFX12: s_and_not1_b32 s0, s1, 0xaf123456       ; encoding: [0x01,0xff,0x00,0x91,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_and_not1_b32 s0, s1, -1               ; encoding: [0x01,0xc1,0x00,0x91]
 0x01,0xc1,0x00,0x91
+# GFX12: s_and_not1_b32 s0, s1, -1               ; encoding: [0x01,0xc1,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, s1, -4.0             ; encoding: [0x01,0xf7,0x00,0x91]
 0x01,0xf7,0x00,0x91
+# GFX12: s_and_not1_b32 s0, s1, -4.0             ; encoding: [0x01,0xf7,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, s1, exec_hi          ; encoding: [0x01,0x7f,0x00,0x91]
 0x01,0x7f,0x00,0x91
+# GFX12: s_and_not1_b32 s0, s1, exec_hi          ; encoding: [0x01,0x7f,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, s1, exec_lo          ; encoding: [0x01,0x7e,0x00,0x91]
 0x01,0x7e,0x00,0x91
+# GFX12: s_and_not1_b32 s0, s1, exec_lo          ; encoding: [0x01,0x7e,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, s1, m0               ; encoding: [0x01,0x7d,0x00,0x91]
 0x01,0x7d,0x00,0x91
+# GFX12: s_and_not1_b32 s0, s1, m0               ; encoding: [0x01,0x7d,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, s1, s103             ; encoding: [0x01,0x67,0x00,0x91]
 0x01,0x67,0x00,0x91
+# GFX12: s_and_not1_b32 s0, s1, s103             ; encoding: [0x01,0x67,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, s1, s2               ; encoding: [0x01,0x02,0x00,0x91]
 0x01,0x02,0x00,0x91
+# GFX12: s_and_not1_b32 s0, s1, s2               ; encoding: [0x01,0x02,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, s1, vcc_hi           ; encoding: [0x01,0x6b,0x00,0x91]
 0x01,0x6b,0x00,0x91
+# GFX12: s_and_not1_b32 s0, s1, vcc_hi           ; encoding: [0x01,0x6b,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, s1, vcc_lo           ; encoding: [0x01,0x6a,0x00,0x91]
 0x01,0x6a,0x00,0x91
+# GFX12: s_and_not1_b32 s0, s1, vcc_lo           ; encoding: [0x01,0x6a,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, vcc_hi, s2           ; encoding: [0x6b,0x02,0x00,0x91]
 0x6b,0x02,0x00,0x91
+# GFX12: s_and_not1_b32 s0, vcc_hi, s2           ; encoding: [0x6b,0x02,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s0, vcc_lo, s2           ; encoding: [0x6a,0x02,0x00,0x91]
 0x6a,0x02,0x00,0x91
+# GFX12: s_and_not1_b32 s0, vcc_lo, s2           ; encoding: [0x6a,0x02,0x00,0x91]
 
-# GFX12: s_and_not1_b32 s105, s104, s103         ; encoding: [0x68,0x67,0x69,0x91]
 0x68,0x67,0x69,0x91
+# GFX12: s_and_not1_b32 s105, s104, s103         ; encoding: [0x68,0x67,0x69,0x91]
 
-# GFX12: s_and_not1_b32 s105, s104, s2           ; encoding: [0x68,0x02,0x69,0x91]
 0x68,0x02,0x69,0x91
+# GFX12: s_and_not1_b32 s105, s104, s2           ; encoding: [0x68,0x02,0x69,0x91]
 
-# GFX12: s_and_not1_b32 s105, s1, s103           ; encoding: [0x01,0x67,0x69,0x91]
 0x01,0x67,0x69,0x91
+# GFX12: s_and_not1_b32 s105, s1, s103           ; encoding: [0x01,0x67,0x69,0x91]
 
-# GFX12: s_and_not1_b32 s105, s1, s2             ; encoding: [0x01,0x02,0x69,0x91]
 0x01,0x02,0x69,0x91
+# GFX12: s_and_not1_b32 s105, s1, s2             ; encoding: [0x01,0x02,0x69,0x91]
 
-# GFX12: s_and_not1_b32 vcc_hi, s1, s2           ; encoding: [0x01,0x02,0x6b,0x91]
 0x01,0x02,0x6b,0x91
+# GFX12: s_and_not1_b32 vcc_hi, s1, s2           ; encoding: [0x01,0x02,0x6b,0x91]
 
-# GFX12: s_and_not1_b32 vcc_lo, s1, s2           ; encoding: [0x01,0x02,0x6a,0x91]
 0x01,0x02,0x6a,0x91
+# GFX12: s_and_not1_b32 vcc_lo, s1, s2           ; encoding: [0x01,0x02,0x6a,0x91]
 
-# GFX12: s_and_not1_b64 exec, s[2:3], s[4:5]     ; encoding: [0x02,0x04,0xfe,0x91]
 0x02,0x04,0xfe,0x91
+# GFX12: s_and_not1_b64 exec, s[2:3], s[4:5]     ; encoding: [0x02,0x04,0xfe,0x91]
 
-# GFX12: s_and_not1_b64 s[0:1], 0.5, s[4:5]      ; encoding: [0xf0,0x04,0x80,0x91]
 0xf0,0x04,0x80,0x91
+# GFX12: s_and_not1_b64 s[0:1], 0.5, s[4:5]      ; encoding: [0xf0,0x04,0x80,0x91]
 
-# GFX12: s_and_not1_b64 s[0:1], 0, s[4:5]        ; encoding: [0x80,0x04,0x80,0x91]
 0x80,0x04,0x80,0x91
+# GFX12: s_and_not1_b64 s[0:1], 0, s[4:5]        ; encoding: [0x80,0x04,0x80,0x91]
 
-# GFX12: s_and_not1_b64 s[0:1], 0x3f717273, s[4:5] ; encoding: [0xff,0x04,0x80,0x91,0x73,0x72,0x71,0x3f]
 0xff,0x04,0x80,0x91,0x73,0x72,0x71,0x3f
+# GFX12: s_and_not1_b64 s[0:1], 0x3f717273, s[4:5] ; encoding: [0xff,0x04,0x80,0x91,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_and_not1_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x91,0x56,0x34,0x12,0xaf]
 0xff,0x04,0x80,0x91,0x56,0x34,0x12,0xaf
+# GFX12: s_and_not1_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x91,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_and_not1_b64 s[0:1], -1, s[4:5]       ; encoding: [0xc1,0x04,0x80,0x91]
 0xc1,0x04,0x80,0x91
+# GFX12: s_and_not1_b64 s[0:1], -1, s[4:5]       ; encoding: [0xc1,0x04,0x80,0x91]
 
-# GFX12: s_and_not1_b64 s[0:1], -4.0, s[4:5]     ; encoding: [0xf7,0x04,0x80,0x91]
 0xf7,0x04,0x80,0x91
+# GFX12: s_and_not1_b64 s[0:1], -4.0, s[4:5]     ; encoding: [0xf7,0x04,0x80,0x91]
 
-# GFX12: s_and_not1_b64 s[0:1], exec, s[4:5]     ; encoding: [0x7e,0x04,0x80,0x91]
 0x7e,0x04,0x80,0x91
+# GFX12: s_and_not1_b64 s[0:1], exec, s[4:5]     ; encoding: [0x7e,0x04,0x80,0x91]
 
-# GFX12: s_and_not1_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x91]
 0x66,0x64,0x80,0x91
+# GFX12: s_and_not1_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x91]
 
-# GFX12: s_and_not1_b64 s[0:1], s[102:103], s[4:5] ; encoding: [0x66,0x04,0x80,0x91]
 0x66,0x04,0x80,0x91
+# GFX12: s_and_not1_b64 s[0:1], s[102:103], s[4:5] ; encoding: [0x66,0x04,0x80,0x91]
 
-# GFX12: s_and_not1_b64 s[0:1], s[2:3], 0.5      ; encoding: [0x02,0xf0,0x80,0x91]
 0x02,0xf0,0x80,0x91
+# GFX12: s_and_not1_b64 s[0:1], s[2:3], 0.5      ; encoding: [0x02,0xf0,0x80,0x91]
 
-# GFX12: s_and_not1_b64 s[0:1], s[2:3], 0        ; encoding: [0x02,0x80,0x80,0x91]
 0x02,0x80,0x80,0x91
+# GFX12: s_and_not1_b64 s[0:1], s[2:3], 0        ; encoding: [0x02,0x80,0x80,0x91]
 
-# GFX12: s_and_not1_b64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0x91,0x73,0x72,0x71,0x3f]
 0x02,0xff,0x80,0x91,0x73,0x72,0x71,0x3f
+# GFX12: s_and_not1_b64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0x91,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_and_not1_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x91,0x56,0x34,0x12,0xaf]
 0x02,0xff,0x80,0x91,0x56,0x34,0x12,0xaf
+# GFX12: s_and_not1_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x91,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_and_not1_b64 s[0:1], s[2:3], -1       ; encoding: [0x02,0xc1,0x80,0x91]
 0x02,0xc1,0x80,0x91
+# GFX12: s_and_not1_b64 s[0:1], s[2:3], -1       ; encoding: [0x02,0xc1,0x80,0x91]
 
-# GFX12: s_and_not1_b64 s[0:1], s[2:3], -4.0     ; encoding: [0x02,0xf7,0x80,0x91]
 0x02,0xf7,0x80,0x91
+# GFX12: s_and_not1_b64 s[0:1], s[2:3], -4.0     ; encoding: [0x02,0xf7,0x80,0x91]
 
-# GFX12: s_and_not1_b64 s[0:1], s[2:3], exec     ; encoding: [0x02,0x7e,0x80,0x91]
 0x02,0x7e,0x80,0x91
+# GFX12: s_and_not1_b64 s[0:1], s[2:3], exec     ; encoding: [0x02,0x7e,0x80,0x91]
 
-# GFX12: s_and_not1_b64 s[0:1], s[2:3], s[100:101] ; encoding: [0x02,0x64,0x80,0x91]
 0x02,0x64,0x80,0x91
+# GFX12: s_and_not1_b64 s[0:1], s[2:3], s[100:101] ; encoding: [0x02,0x64,0x80,0x91]
 
-# GFX12: s_and_not1_b64 s[0:1], s[2:3], s[4:5]   ; encoding: [0x02,0x04,0x80,0x91]
 0x02,0x04,0x80,0x91
+# GFX12: s_and_not1_b64 s[0:1], s[2:3], s[4:5]   ; encoding: [0x02,0x04,0x80,0x91]
 
-# GFX12: s_and_not1_b64 s[0:1], s[2:3], vcc      ; encoding: [0x02,0x6a,0x80,0x91]
 0x02,0x6a,0x80,0x91
+# GFX12: s_and_not1_b64 s[0:1], s[2:3], vcc      ; encoding: [0x02,0x6a,0x80,0x91]
 
-# GFX12: s_and_not1_b64 s[0:1], vcc, s[4:5]      ; encoding: [0x6a,0x04,0x80,0x91]
 0x6a,0x04,0x80,0x91
+# GFX12: s_and_not1_b64 s[0:1], vcc, s[4:5]      ; encoding: [0x6a,0x04,0x80,0x91]
 
-# GFX12: s_and_not1_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x91]
 0x66,0x64,0xe8,0x91
+# GFX12: s_and_not1_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x91]
 
-# GFX12: s_and_not1_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x91]
 0x66,0x04,0xe8,0x91
+# GFX12: s_and_not1_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x91]
 
-# GFX12: s_and_not1_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x91]
 0x02,0x64,0xe8,0x91
+# GFX12: s_and_not1_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x91]
 
-# GFX12: s_and_not1_b64 s[104:105], s[2:3], s[4:5] ; encoding: [0x02,0x04,0xe8,0x91]
 0x02,0x04,0xe8,0x91
+# GFX12: s_and_not1_b64 s[104:105], s[2:3], s[4:5] ; encoding: [0x02,0x04,0xe8,0x91]
 
-# GFX12: s_and_not1_b64 vcc, s[2:3], s[4:5]      ; encoding: [0x02,0x04,0xea,0x91]
 0x02,0x04,0xea,0x91
+# GFX12: s_and_not1_b64 vcc, s[2:3], s[4:5]      ; encoding: [0x02,0x04,0xea,0x91]
 
-# GFX12: s_ashr_i32 exec_hi, s1, s2              ; encoding: [0x01,0x02,0x7f,0x86]
 0x01,0x02,0x7f,0x86
+# GFX12: s_ashr_i32 exec_hi, s1, s2              ; encoding: [0x01,0x02,0x7f,0x86]
 
-# GFX12: s_ashr_i32 exec_lo, s1, s2              ; encoding: [0x01,0x02,0x7e,0x86]
 0x01,0x02,0x7e,0x86
+# GFX12: s_ashr_i32 exec_lo, s1, s2              ; encoding: [0x01,0x02,0x7e,0x86]
 
-# GFX12: s_ashr_i32 m0, s1, s2                   ; encoding: [0x01,0x02,0x7d,0x86]
 0x01,0x02,0x7d,0x86
+# GFX12: s_ashr_i32 m0, s1, s2                   ; encoding: [0x01,0x02,0x7d,0x86]
 
-# GFX12: s_ashr_i32 s0, 0.5, s2                  ; encoding: [0xf0,0x02,0x00,0x86]
 0xf0,0x02,0x00,0x86
+# GFX12: s_ashr_i32 s0, 0.5, s2                  ; encoding: [0xf0,0x02,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, 0, s2                    ; encoding: [0x80,0x02,0x00,0x86]
 0x80,0x02,0x00,0x86
+# GFX12: s_ashr_i32 s0, 0, s2                    ; encoding: [0x80,0x02,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, 0x3f717273, s2           ; encoding: [0xff,0x02,0x00,0x86,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x86,0x73,0x72,0x71,0x3f
+# GFX12: s_ashr_i32 s0, 0x3f717273, s2           ; encoding: [0xff,0x02,0x00,0x86,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_ashr_i32 s0, 0xaf123456, s2           ; encoding: [0xff,0x02,0x00,0x86,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x86,0x56,0x34,0x12,0xaf
+# GFX12: s_ashr_i32 s0, 0xaf123456, s2           ; encoding: [0xff,0x02,0x00,0x86,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_ashr_i32 s0, -1, s2                   ; encoding: [0xc1,0x02,0x00,0x86]
 0xc1,0x02,0x00,0x86
+# GFX12: s_ashr_i32 s0, -1, s2                   ; encoding: [0xc1,0x02,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, -4.0, s2                 ; encoding: [0xf7,0x02,0x00,0x86]
 0xf7,0x02,0x00,0x86
+# GFX12: s_ashr_i32 s0, -4.0, s2                 ; encoding: [0xf7,0x02,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, exec_hi, s2              ; encoding: [0x7f,0x02,0x00,0x86]
 0x7f,0x02,0x00,0x86
+# GFX12: s_ashr_i32 s0, exec_hi, s2              ; encoding: [0x7f,0x02,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, exec_lo, s2              ; encoding: [0x7e,0x02,0x00,0x86]
 0x7e,0x02,0x00,0x86
+# GFX12: s_ashr_i32 s0, exec_lo, s2              ; encoding: [0x7e,0x02,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, m0, s2                   ; encoding: [0x7d,0x02,0x00,0x86]
 0x7d,0x02,0x00,0x86
+# GFX12: s_ashr_i32 s0, m0, s2                   ; encoding: [0x7d,0x02,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, s104, s103               ; encoding: [0x68,0x67,0x00,0x86]
 0x68,0x67,0x00,0x86
+# GFX12: s_ashr_i32 s0, s104, s103               ; encoding: [0x68,0x67,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, s104, s2                 ; encoding: [0x68,0x02,0x00,0x86]
 0x68,0x02,0x00,0x86
+# GFX12: s_ashr_i32 s0, s104, s2                 ; encoding: [0x68,0x02,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, s1, 0.5                  ; encoding: [0x01,0xf0,0x00,0x86]
 0x01,0xf0,0x00,0x86
+# GFX12: s_ashr_i32 s0, s1, 0.5                  ; encoding: [0x01,0xf0,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, s1, 0                    ; encoding: [0x01,0x80,0x00,0x86]
 0x01,0x80,0x00,0x86
+# GFX12: s_ashr_i32 s0, s1, 0                    ; encoding: [0x01,0x80,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, s1, 0x3f717273           ; encoding: [0x01,0xff,0x00,0x86,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x86,0x73,0x72,0x71,0x3f
+# GFX12: s_ashr_i32 s0, s1, 0x3f717273           ; encoding: [0x01,0xff,0x00,0x86,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_ashr_i32 s0, s1, 0xaf123456           ; encoding: [0x01,0xff,0x00,0x86,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x86,0x56,0x34,0x12,0xaf
+# GFX12: s_ashr_i32 s0, s1, 0xaf123456           ; encoding: [0x01,0xff,0x00,0x86,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_ashr_i32 s0, s1, -1                   ; encoding: [0x01,0xc1,0x00,0x86]
 0x01,0xc1,0x00,0x86
+# GFX12: s_ashr_i32 s0, s1, -1                   ; encoding: [0x01,0xc1,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, s1, -4.0                 ; encoding: [0x01,0xf7,0x00,0x86]
 0x01,0xf7,0x00,0x86
+# GFX12: s_ashr_i32 s0, s1, -4.0                 ; encoding: [0x01,0xf7,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, s1, exec_hi              ; encoding: [0x01,0x7f,0x00,0x86]
 0x01,0x7f,0x00,0x86
+# GFX12: s_ashr_i32 s0, s1, exec_hi              ; encoding: [0x01,0x7f,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, s1, exec_lo              ; encoding: [0x01,0x7e,0x00,0x86]
 0x01,0x7e,0x00,0x86
+# GFX12: s_ashr_i32 s0, s1, exec_lo              ; encoding: [0x01,0x7e,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, s1, m0                   ; encoding: [0x01,0x7d,0x00,0x86]
 0x01,0x7d,0x00,0x86
+# GFX12: s_ashr_i32 s0, s1, m0                   ; encoding: [0x01,0x7d,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, s1, s103                 ; encoding: [0x01,0x67,0x00,0x86]
 0x01,0x67,0x00,0x86
+# GFX12: s_ashr_i32 s0, s1, s103                 ; encoding: [0x01,0x67,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, s1, s2                   ; encoding: [0x01,0x02,0x00,0x86]
 0x01,0x02,0x00,0x86
+# GFX12: s_ashr_i32 s0, s1, s2                   ; encoding: [0x01,0x02,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, s1, vcc_hi               ; encoding: [0x01,0x6b,0x00,0x86]
 0x01,0x6b,0x00,0x86
+# GFX12: s_ashr_i32 s0, s1, vcc_hi               ; encoding: [0x01,0x6b,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, s1, vcc_lo               ; encoding: [0x01,0x6a,0x00,0x86]
 0x01,0x6a,0x00,0x86
+# GFX12: s_ashr_i32 s0, s1, vcc_lo               ; encoding: [0x01,0x6a,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, vcc_hi, s2               ; encoding: [0x6b,0x02,0x00,0x86]
 0x6b,0x02,0x00,0x86
+# GFX12: s_ashr_i32 s0, vcc_hi, s2               ; encoding: [0x6b,0x02,0x00,0x86]
 
-# GFX12: s_ashr_i32 s0, vcc_lo, s2               ; encoding: [0x6a,0x02,0x00,0x86]
 0x6a,0x02,0x00,0x86
+# GFX12: s_ashr_i32 s0, vcc_lo, s2               ; encoding: [0x6a,0x02,0x00,0x86]
 
-# GFX12: s_ashr_i32 s105, s104, s103             ; encoding: [0x68,0x67,0x69,0x86]
 0x68,0x67,0x69,0x86
+# GFX12: s_ashr_i32 s105, s104, s103             ; encoding: [0x68,0x67,0x69,0x86]
 
-# GFX12: s_ashr_i32 s105, s104, s2               ; encoding: [0x68,0x02,0x69,0x86]
 0x68,0x02,0x69,0x86
+# GFX12: s_ashr_i32 s105, s104, s2               ; encoding: [0x68,0x02,0x69,0x86]
 
-# GFX12: s_ashr_i32 s105, s1, s103               ; encoding: [0x01,0x67,0x69,0x86]
 0x01,0x67,0x69,0x86
+# GFX12: s_ashr_i32 s105, s1, s103               ; encoding: [0x01,0x67,0x69,0x86]
 
-# GFX12: s_ashr_i32 s105, s1, s2                 ; encoding: [0x01,0x02,0x69,0x86]
 0x01,0x02,0x69,0x86
+# GFX12: s_ashr_i32 s105, s1, s2                 ; encoding: [0x01,0x02,0x69,0x86]
 
-# GFX12: s_ashr_i32 vcc_hi, s1, s2               ; encoding: [0x01,0x02,0x6b,0x86]
 0x01,0x02,0x6b,0x86
+# GFX12: s_ashr_i32 vcc_hi, s1, s2               ; encoding: [0x01,0x02,0x6b,0x86]
 
-# GFX12: s_ashr_i32 vcc_lo, s1, s2               ; encoding: [0x01,0x02,0x6a,0x86]
 0x01,0x02,0x6a,0x86
+# GFX12: s_ashr_i32 vcc_lo, s1, s2               ; encoding: [0x01,0x02,0x6a,0x86]
 
-# GFX12: s_ashr_i64 exec, s[2:3], s4             ; encoding: [0x02,0x04,0xfe,0x86]
 0x02,0x04,0xfe,0x86
+# GFX12: s_ashr_i64 exec, s[2:3], s4             ; encoding: [0x02,0x04,0xfe,0x86]
 
-# GFX12: s_ashr_i64 s[0:1], 0.5, s4              ; encoding: [0xf0,0x04,0x80,0x86]
 0xf0,0x04,0x80,0x86
+# GFX12: s_ashr_i64 s[0:1], 0.5, s4              ; encoding: [0xf0,0x04,0x80,0x86]
 
-# GFX12: s_ashr_i64 s[0:1], 0, s4                ; encoding: [0x80,0x04,0x80,0x86]
 0x80,0x04,0x80,0x86
+# GFX12: s_ashr_i64 s[0:1], 0, s4                ; encoding: [0x80,0x04,0x80,0x86]
 
-# GFX12: s_ashr_i64 s[0:1], 0x3f717273, s4       ; encoding: [0xff,0x04,0x80,0x86,0x73,0x72,0x71,0x3f]
 0xff,0x04,0x80,0x86,0x73,0x72,0x71,0x3f
+# GFX12: s_ashr_i64 s[0:1], 0x3f717273, s4       ; encoding: [0xff,0x04,0x80,0x86,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_ashr_i64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x86,0x56,0x34,0x12,0xaf]
 0xff,0x04,0x80,0x86,0x56,0x34,0x12,0xaf
+# GFX12: s_ashr_i64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x86,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_ashr_i64 s[0:1], -1, s4               ; encoding: [0xc1,0x04,0x80,0x86]
 0xc1,0x04,0x80,0x86
+# GFX12: s_ashr_i64 s[0:1], -1, s4               ; encoding: [0xc1,0x04,0x80,0x86]
 
-# GFX12: s_ashr_i64 s[0:1], -4.0, s4             ; encoding: [0xf7,0x04,0x80,0x86]
 0xf7,0x04,0x80,0x86
+# GFX12: s_ashr_i64 s[0:1], -4.0, s4             ; encoding: [0xf7,0x04,0x80,0x86]
 
-# GFX12: s_ashr_i64 s[0:1], exec, s4             ; encoding: [0x7e,0x04,0x80,0x86]
 0x7e,0x04,0x80,0x86
+# GFX12: s_ashr_i64 s[0:1], exec, s4             ; encoding: [0x7e,0x04,0x80,0x86]
 
-# GFX12: s_ashr_i64 s[0:1], s[102:103], s100     ; encoding: [0x66,0x64,0x80,0x86]
 0x66,0x64,0x80,0x86
+# GFX12: s_ashr_i64 s[0:1], s[102:103], s100     ; encoding: [0x66,0x64,0x80,0x86]
 
-# GFX12: s_ashr_i64 s[0:1], s[102:103], s4       ; encoding: [0x66,0x04,0x80,0x86]
 0x66,0x04,0x80,0x86
+# GFX12: s_ashr_i64 s[0:1], s[102:103], s4       ; encoding: [0x66,0x04,0x80,0x86]
 
-# GFX12: s_ashr_i64 s[0:1], s[2:3], 0.5          ; encoding: [0x02,0xf0,0x80,0x86]
 0x02,0xf0,0x80,0x86
+# GFX12: s_ashr_i64 s[0:1], s[2:3], 0.5          ; encoding: [0x02,0xf0,0x80,0x86]
 
-# GFX12: s_ashr_i64 s[0:1], s[2:3], 0            ; encoding: [0x02,0x80,0x80,0x86]
 0x02,0x80,0x80,0x86
+# GFX12: s_ashr_i64 s[0:1], s[2:3], 0            ; encoding: [0x02,0x80,0x80,0x86]
 
-# GFX12: s_ashr_i64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x86,0x73,0x72,0x71,0x3f]
 0x02,0xff,0x80,0x86,0x73,0x72,0x71,0x3f
+# GFX12: s_ashr_i64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x86,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_ashr_i64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x86,0x56,0x34,0x12,0xaf]
 0x02,0xff,0x80,0x86,0x56,0x34,0x12,0xaf
+# GFX12: s_ashr_i64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x86,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_ashr_i64 s[0:1], s[2:3], -1           ; encoding: [0x02,0xc1,0x80,0x86]
 0x02,0xc1,0x80,0x86
+# GFX12: s_ashr_i64 s[0:1], s[2:3], -1           ; encoding: [0x02,0xc1,0x80,0x86]
 
-# GFX12: s_ashr_i64 s[0:1], s[2:3], -4.0         ; encoding: [0x02,0xf7,0x80,0x86]
 0x02,0xf7,0x80,0x86
+# GFX12: s_ashr_i64 s[0:1], s[2:3], -4.0         ; encoding: [0x02,0xf7,0x80,0x86]
 
-# GFX12: s_ashr_i64 s[0:1], s[2:3], exec_lo      ; encoding: [0x02,0x7e,0x80,0x86]
 0x02,0x7e,0x80,0x86
+# GFX12: s_ashr_i64 s[0:1], s[2:3], exec_lo      ; encoding: [0x02,0x7e,0x80,0x86]
 
-# GFX12: s_ashr_i64 s[0:1], s[2:3], s100         ; encoding: [0x02,0x64,0x80,0x86]
 0x02,0x64,0x80,0x86
+# GFX12: s_ashr_i64 s[0:1], s[2:3], s100         ; encoding: [0x02,0x64,0x80,0x86]
 
-# GFX12: s_ashr_i64 s[0:1], s[2:3], s4           ; encoding: [0x02,0x04,0x80,0x86]
 0x02,0x04,0x80,0x86
+# GFX12: s_ashr_i64 s[0:1], s[2:3], s4           ; encoding: [0x02,0x04,0x80,0x86]
 
-# GFX12: s_ashr_i64 s[0:1], s[2:3], vcc_lo       ; encoding: [0x02,0x6a,0x80,0x86]
 0x02,0x6a,0x80,0x86
+# GFX12: s_ashr_i64 s[0:1], s[2:3], vcc_lo       ; encoding: [0x02,0x6a,0x80,0x86]
 
-# GFX12: s_ashr_i64 s[0:1], vcc, s4              ; encoding: [0x6a,0x04,0x80,0x86]
 0x6a,0x04,0x80,0x86
+# GFX12: s_ashr_i64 s[0:1], vcc, s4              ; encoding: [0x6a,0x04,0x80,0x86]
 
-# GFX12: s_ashr_i64 s[104:105], s[102:103], s100 ; encoding: [0x66,0x64,0xe8,0x86]
 0x66,0x64,0xe8,0x86
+# GFX12: s_ashr_i64 s[104:105], s[102:103], s100 ; encoding: [0x66,0x64,0xe8,0x86]
 
-# GFX12: s_ashr_i64 s[104:105], s[102:103], s4   ; encoding: [0x66,0x04,0xe8,0x86]
 0x66,0x04,0xe8,0x86
+# GFX12: s_ashr_i64 s[104:105], s[102:103], s4   ; encoding: [0x66,0x04,0xe8,0x86]
 
-# GFX12: s_ashr_i64 s[104:105], s[2:3], s100     ; encoding: [0x02,0x64,0xe8,0x86]
 0x02,0x64,0xe8,0x86
+# GFX12: s_ashr_i64 s[104:105], s[2:3], s100     ; encoding: [0x02,0x64,0xe8,0x86]
 
-# GFX12: s_ashr_i64 s[104:105], s[2:3], s4       ; encoding: [0x02,0x04,0xe8,0x86]
 0x02,0x04,0xe8,0x86
+# GFX12: s_ashr_i64 s[104:105], s[2:3], s4       ; encoding: [0x02,0x04,0xe8,0x86]
 
-# GFX12: s_ashr_i64 vcc, s[2:3], s4              ; encoding: [0x02,0x04,0xea,0x86]
 0x02,0x04,0xea,0x86
+# GFX12: s_ashr_i64 vcc, s[2:3], s4              ; encoding: [0x02,0x04,0xea,0x86]
 
-# GFX12: s_bfe_i32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0xff,0x93]
 0x01,0x02,0xff,0x93
+# GFX12: s_bfe_i32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0xff,0x93]
 
-# GFX12: s_bfe_i32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0xfe,0x93]
 0x01,0x02,0xfe,0x93
+# GFX12: s_bfe_i32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0xfe,0x93]
 
-# GFX12: s_bfe_i32 m0, s1, s2                    ; encoding: [0x01,0x02,0xfd,0x93]
 0x01,0x02,0xfd,0x93
+# GFX12: s_bfe_i32 m0, s1, s2                    ; encoding: [0x01,0x02,0xfd,0x93]
 
-# GFX12: s_bfe_i32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x80,0x93]
 0xf0,0x02,0x80,0x93
+# GFX12: s_bfe_i32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, 0, s2                     ; encoding: [0x80,0x02,0x80,0x93]
 0x80,0x02,0x80,0x93
+# GFX12: s_bfe_i32 s0, 0, s2                     ; encoding: [0x80,0x02,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x80,0x93,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x80,0x93,0x73,0x72,0x71,0x3f
+# GFX12: s_bfe_i32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x80,0x93,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bfe_i32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x80,0x93,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x80,0x93,0x56,0x34,0x12,0xaf
+# GFX12: s_bfe_i32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x80,0x93,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bfe_i32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x80,0x93]
 0xc1,0x02,0x80,0x93
+# GFX12: s_bfe_i32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x80,0x93]
 0xf7,0x02,0x80,0x93
+# GFX12: s_bfe_i32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x80,0x93]
 0x7f,0x02,0x80,0x93
+# GFX12: s_bfe_i32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x80,0x93]
 0x7e,0x02,0x80,0x93
+# GFX12: s_bfe_i32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x80,0x93]
 0x7d,0x02,0x80,0x93
+# GFX12: s_bfe_i32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, s104, s103                ; encoding: [0x68,0x67,0x80,0x93]
 0x68,0x67,0x80,0x93
+# GFX12: s_bfe_i32 s0, s104, s103                ; encoding: [0x68,0x67,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, s104, s2                  ; encoding: [0x68,0x02,0x80,0x93]
 0x68,0x02,0x80,0x93
+# GFX12: s_bfe_i32 s0, s104, s2                  ; encoding: [0x68,0x02,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x80,0x93]
 0x01,0xf0,0x80,0x93
+# GFX12: s_bfe_i32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, s1, 0                     ; encoding: [0x01,0x80,0x80,0x93]
 0x01,0x80,0x80,0x93
+# GFX12: s_bfe_i32 s0, s1, 0                     ; encoding: [0x01,0x80,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x80,0x93,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x80,0x93,0x73,0x72,0x71,0x3f
+# GFX12: s_bfe_i32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x80,0x93,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bfe_i32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x80,0x93,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x80,0x93,0x56,0x34,0x12,0xaf
+# GFX12: s_bfe_i32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x80,0x93,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bfe_i32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x80,0x93]
 0x01,0xc1,0x80,0x93
+# GFX12: s_bfe_i32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x80,0x93]
 0x01,0xf7,0x80,0x93
+# GFX12: s_bfe_i32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x80,0x93]
 0x01,0x7f,0x80,0x93
+# GFX12: s_bfe_i32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x80,0x93]
 0x01,0x7e,0x80,0x93
+# GFX12: s_bfe_i32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x80,0x93]
 0x01,0x7d,0x80,0x93
+# GFX12: s_bfe_i32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, s1, s103                  ; encoding: [0x01,0x67,0x80,0x93]
 0x01,0x67,0x80,0x93
+# GFX12: s_bfe_i32 s0, s1, s103                  ; encoding: [0x01,0x67,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, s1, s2                    ; encoding: [0x01,0x02,0x80,0x93]
 0x01,0x02,0x80,0x93
+# GFX12: s_bfe_i32 s0, s1, s2                    ; encoding: [0x01,0x02,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x80,0x93]
 0x01,0x6b,0x80,0x93
+# GFX12: s_bfe_i32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x80,0x93]
 0x01,0x6a,0x80,0x93
+# GFX12: s_bfe_i32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x80,0x93]
 0x6b,0x02,0x80,0x93
+# GFX12: s_bfe_i32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x80,0x93]
 
-# GFX12: s_bfe_i32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x80,0x93]
 0x6a,0x02,0x80,0x93
+# GFX12: s_bfe_i32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x80,0x93]
 
-# GFX12: s_bfe_i32 s105, s104, s103              ; encoding: [0x68,0x67,0xe9,0x93]
 0x68,0x67,0xe9,0x93
+# GFX12: s_bfe_i32 s105, s104, s103              ; encoding: [0x68,0x67,0xe9,0x93]
 
-# GFX12: s_bfe_i32 s105, s104, s2                ; encoding: [0x68,0x02,0xe9,0x93]
 0x68,0x02,0xe9,0x93
+# GFX12: s_bfe_i32 s105, s104, s2                ; encoding: [0x68,0x02,0xe9,0x93]
 
-# GFX12: s_bfe_i32 s105, s1, s103                ; encoding: [0x01,0x67,0xe9,0x93]
 0x01,0x67,0xe9,0x93
+# GFX12: s_bfe_i32 s105, s1, s103                ; encoding: [0x01,0x67,0xe9,0x93]
 
-# GFX12: s_bfe_i32 s105, s1, s2                  ; encoding: [0x01,0x02,0xe9,0x93]
 0x01,0x02,0xe9,0x93
+# GFX12: s_bfe_i32 s105, s1, s2                  ; encoding: [0x01,0x02,0xe9,0x93]
 
-# GFX12: s_bfe_i32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0xeb,0x93]
 0x01,0x02,0xeb,0x93
+# GFX12: s_bfe_i32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0xeb,0x93]
 
-# GFX12: s_bfe_i32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0xea,0x93]
 0x01,0x02,0xea,0x93
+# GFX12: s_bfe_i32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0xea,0x93]
 
-# GFX12: s_bfe_i64 exec, s[2:3], s4              ; encoding: [0x02,0x04,0xfe,0x94]
 0x02,0x04,0xfe,0x94
+# GFX12: s_bfe_i64 exec, s[2:3], s4              ; encoding: [0x02,0x04,0xfe,0x94]
 
-# GFX12: s_bfe_i64 s[0:1], 0.5, s4               ; encoding: [0xf0,0x04,0x80,0x94]
 0xf0,0x04,0x80,0x94
+# GFX12: s_bfe_i64 s[0:1], 0.5, s4               ; encoding: [0xf0,0x04,0x80,0x94]
 
-# GFX12: s_bfe_i64 s[0:1], 0, s4                 ; encoding: [0x80,0x04,0x80,0x94]
 0x80,0x04,0x80,0x94
+# GFX12: s_bfe_i64 s[0:1], 0, s4                 ; encoding: [0x80,0x04,0x80,0x94]
 
-# GFX12: s_bfe_i64 s[0:1], 0x3f717273, s4        ; encoding: [0xff,0x04,0x80,0x94,0x73,0x72,0x71,0x3f]
 0xff,0x04,0x80,0x94,0x73,0x72,0x71,0x3f
+# GFX12: s_bfe_i64 s[0:1], 0x3f717273, s4        ; encoding: [0xff,0x04,0x80,0x94,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bfe_i64 s[0:1], 0xaf123456, s4        ; encoding: [0xff,0x04,0x80,0x94,0x56,0x34,0x12,0xaf]
 0xff,0x04,0x80,0x94,0x56,0x34,0x12,0xaf
+# GFX12: s_bfe_i64 s[0:1], 0xaf123456, s4        ; encoding: [0xff,0x04,0x80,0x94,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bfe_i64 s[0:1], -1, s4                ; encoding: [0xc1,0x04,0x80,0x94]
 0xc1,0x04,0x80,0x94
+# GFX12: s_bfe_i64 s[0:1], -1, s4                ; encoding: [0xc1,0x04,0x80,0x94]
 
-# GFX12: s_bfe_i64 s[0:1], -4.0, s4              ; encoding: [0xf7,0x04,0x80,0x94]
 0xf7,0x04,0x80,0x94
+# GFX12: s_bfe_i64 s[0:1], -4.0, s4              ; encoding: [0xf7,0x04,0x80,0x94]
 
-# GFX12: s_bfe_i64 s[0:1], exec, s4              ; encoding: [0x7e,0x04,0x80,0x94]
 0x7e,0x04,0x80,0x94
+# GFX12: s_bfe_i64 s[0:1], exec, s4              ; encoding: [0x7e,0x04,0x80,0x94]
 
-# GFX12: s_bfe_i64 s[0:1], s[102:103], s100      ; encoding: [0x66,0x64,0x80,0x94]
 0x66,0x64,0x80,0x94
+# GFX12: s_bfe_i64 s[0:1], s[102:103], s100      ; encoding: [0x66,0x64,0x80,0x94]
 
-# GFX12: s_bfe_i64 s[0:1], s[102:103], s4        ; encoding: [0x66,0x04,0x80,0x94]
 0x66,0x04,0x80,0x94
+# GFX12: s_bfe_i64 s[0:1], s[102:103], s4        ; encoding: [0x66,0x04,0x80,0x94]
 
-# GFX12: s_bfe_i64 s[0:1], s[2:3], 0.5           ; encoding: [0x02,0xf0,0x80,0x94]
 0x02,0xf0,0x80,0x94
+# GFX12: s_bfe_i64 s[0:1], s[2:3], 0.5           ; encoding: [0x02,0xf0,0x80,0x94]
 
-# GFX12: s_bfe_i64 s[0:1], s[2:3], 0             ; encoding: [0x02,0x80,0x80,0x94]
 0x02,0x80,0x80,0x94
+# GFX12: s_bfe_i64 s[0:1], s[2:3], 0             ; encoding: [0x02,0x80,0x80,0x94]
 
-# GFX12: s_bfe_i64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0x94,0x73,0x72,0x71,0x3f]
 0x02,0xff,0x80,0x94,0x73,0x72,0x71,0x3f
+# GFX12: s_bfe_i64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0x94,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bfe_i64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x94,0x56,0x34,0x12,0xaf]
 0x02,0xff,0x80,0x94,0x56,0x34,0x12,0xaf
+# GFX12: s_bfe_i64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x94,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bfe_i64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x80,0x94]
 0x02,0xc1,0x80,0x94
+# GFX12: s_bfe_i64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x80,0x94]
 
-# GFX12: s_bfe_i64 s[0:1], s[2:3], -4.0          ; encoding: [0x02,0xf7,0x80,0x94]
 0x02,0xf7,0x80,0x94
+# GFX12: s_bfe_i64 s[0:1], s[2:3], -4.0          ; encoding: [0x02,0xf7,0x80,0x94]
 
-# GFX12: s_bfe_i64 s[0:1], s[2:3], exec_lo       ; encoding: [0x02,0x7e,0x80,0x94]
 0x02,0x7e,0x80,0x94
+# GFX12: s_bfe_i64 s[0:1], s[2:3], exec_lo       ; encoding: [0x02,0x7e,0x80,0x94]
 
-# GFX12: s_bfe_i64 s[0:1], s[2:3], s100          ; encoding: [0x02,0x64,0x80,0x94]
 0x02,0x64,0x80,0x94
+# GFX12: s_bfe_i64 s[0:1], s[2:3], s100          ; encoding: [0x02,0x64,0x80,0x94]
 
-# GFX12: s_bfe_i64 s[0:1], s[2:3], s4            ; encoding: [0x02,0x04,0x80,0x94]
 0x02,0x04,0x80,0x94
+# GFX12: s_bfe_i64 s[0:1], s[2:3], s4            ; encoding: [0x02,0x04,0x80,0x94]
 
-# GFX12: s_bfe_i64 s[0:1], s[2:3], vcc_lo        ; encoding: [0x02,0x6a,0x80,0x94]
 0x02,0x6a,0x80,0x94
+# GFX12: s_bfe_i64 s[0:1], s[2:3], vcc_lo        ; encoding: [0x02,0x6a,0x80,0x94]
 
-# GFX12: s_bfe_i64 s[0:1], vcc, s4               ; encoding: [0x6a,0x04,0x80,0x94]
 0x6a,0x04,0x80,0x94
+# GFX12: s_bfe_i64 s[0:1], vcc, s4               ; encoding: [0x6a,0x04,0x80,0x94]
 
-# GFX12: s_bfe_i64 s[104:105], s[102:103], s100  ; encoding: [0x66,0x64,0xe8,0x94]
 0x66,0x64,0xe8,0x94
+# GFX12: s_bfe_i64 s[104:105], s[102:103], s100  ; encoding: [0x66,0x64,0xe8,0x94]
 
-# GFX12: s_bfe_i64 s[104:105], s[102:103], s4    ; encoding: [0x66,0x04,0xe8,0x94]
 0x66,0x04,0xe8,0x94
+# GFX12: s_bfe_i64 s[104:105], s[102:103], s4    ; encoding: [0x66,0x04,0xe8,0x94]
 
-# GFX12: s_bfe_i64 s[104:105], s[2:3], s100      ; encoding: [0x02,0x64,0xe8,0x94]
 0x02,0x64,0xe8,0x94
+# GFX12: s_bfe_i64 s[104:105], s[2:3], s100      ; encoding: [0x02,0x64,0xe8,0x94]
 
-# GFX12: s_bfe_i64 s[104:105], s[2:3], s4        ; encoding: [0x02,0x04,0xe8,0x94]
 0x02,0x04,0xe8,0x94
+# GFX12: s_bfe_i64 s[104:105], s[2:3], s4        ; encoding: [0x02,0x04,0xe8,0x94]
 
-# GFX12: s_bfe_i64 vcc, s[2:3], s4               ; encoding: [0x02,0x04,0xea,0x94]
 0x02,0x04,0xea,0x94
+# GFX12: s_bfe_i64 vcc, s[2:3], s4               ; encoding: [0x02,0x04,0xea,0x94]
 
-# GFX12: s_bfe_u32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x93]
 0x01,0x02,0x7f,0x93
+# GFX12: s_bfe_u32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x93]
 
-# GFX12: s_bfe_u32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x93]
 0x01,0x02,0x7e,0x93
+# GFX12: s_bfe_u32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x93]
 
-# GFX12: s_bfe_u32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x93]
 0x01,0x02,0x7d,0x93
+# GFX12: s_bfe_u32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x93]
 
-# GFX12: s_bfe_u32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x93]
 0xf0,0x02,0x00,0x93
+# GFX12: s_bfe_u32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x93]
 0x80,0x02,0x00,0x93
+# GFX12: s_bfe_u32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x93,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x93,0x73,0x72,0x71,0x3f
+# GFX12: s_bfe_u32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x93,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bfe_u32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x93,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x93,0x56,0x34,0x12,0xaf
+# GFX12: s_bfe_u32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x93,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bfe_u32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x93]
 0xc1,0x02,0x00,0x93
+# GFX12: s_bfe_u32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x93]
 0xf7,0x02,0x00,0x93
+# GFX12: s_bfe_u32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x93]
 0x7f,0x02,0x00,0x93
+# GFX12: s_bfe_u32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x93]
 0x7e,0x02,0x00,0x93
+# GFX12: s_bfe_u32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x93]
 0x7d,0x02,0x00,0x93
+# GFX12: s_bfe_u32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x93]
 0x68,0x67,0x00,0x93
+# GFX12: s_bfe_u32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x93]
 0x68,0x02,0x00,0x93
+# GFX12: s_bfe_u32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x93]
 0x01,0xf0,0x00,0x93
+# GFX12: s_bfe_u32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x93]
 0x01,0x80,0x00,0x93
+# GFX12: s_bfe_u32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x93,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x93,0x73,0x72,0x71,0x3f
+# GFX12: s_bfe_u32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x93,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bfe_u32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x93,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x93,0x56,0x34,0x12,0xaf
+# GFX12: s_bfe_u32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x93,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bfe_u32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x93]
 0x01,0xc1,0x00,0x93
+# GFX12: s_bfe_u32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x93]
 0x01,0xf7,0x00,0x93
+# GFX12: s_bfe_u32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x93]
 0x01,0x7f,0x00,0x93
+# GFX12: s_bfe_u32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x93]
 0x01,0x7e,0x00,0x93
+# GFX12: s_bfe_u32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x93]
 0x01,0x7d,0x00,0x93
+# GFX12: s_bfe_u32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x93]
 0x01,0x67,0x00,0x93
+# GFX12: s_bfe_u32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x93]
 0x01,0x02,0x00,0x93
+# GFX12: s_bfe_u32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x93]
 0x01,0x6b,0x00,0x93
+# GFX12: s_bfe_u32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x93]
 0x01,0x6a,0x00,0x93
+# GFX12: s_bfe_u32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x93]
 0x6b,0x02,0x00,0x93
+# GFX12: s_bfe_u32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x93]
 
-# GFX12: s_bfe_u32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x93]
 0x6a,0x02,0x00,0x93
+# GFX12: s_bfe_u32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x93]
 
-# GFX12: s_bfe_u32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x93]
 0x68,0x67,0x69,0x93
+# GFX12: s_bfe_u32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x93]
 
-# GFX12: s_bfe_u32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x93]
 0x68,0x02,0x69,0x93
+# GFX12: s_bfe_u32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x93]
 
-# GFX12: s_bfe_u32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x93]
 0x01,0x67,0x69,0x93
+# GFX12: s_bfe_u32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x93]
 
-# GFX12: s_bfe_u32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x93]
 0x01,0x02,0x69,0x93
+# GFX12: s_bfe_u32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x93]
 
-# GFX12: s_bfe_u32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x93]
 0x01,0x02,0x6b,0x93
+# GFX12: s_bfe_u32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x93]
 
-# GFX12: s_bfe_u32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x93]
 0x01,0x02,0x6a,0x93
+# GFX12: s_bfe_u32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x93]
 
-# GFX12: s_bfe_u64 exec, s[2:3], s4              ; encoding: [0x02,0x04,0x7e,0x94]
 0x02,0x04,0x7e,0x94
+# GFX12: s_bfe_u64 exec, s[2:3], s4              ; encoding: [0x02,0x04,0x7e,0x94]
 
-# GFX12: s_bfe_u64 s[0:1], 0.5, s4               ; encoding: [0xf0,0x04,0x00,0x94]
 0xf0,0x04,0x00,0x94
+# GFX12: s_bfe_u64 s[0:1], 0.5, s4               ; encoding: [0xf0,0x04,0x00,0x94]
 
-# GFX12: s_bfe_u64 s[0:1], 0, s4                 ; encoding: [0x80,0x04,0x00,0x94]
 0x80,0x04,0x00,0x94
+# GFX12: s_bfe_u64 s[0:1], 0, s4                 ; encoding: [0x80,0x04,0x00,0x94]
 
-# GFX12: s_bfe_u64 s[0:1], 0x3f717273, s4        ; encoding: [0xff,0x04,0x00,0x94,0x73,0x72,0x71,0x3f]
 0xff,0x04,0x00,0x94,0x73,0x72,0x71,0x3f
+# GFX12: s_bfe_u64 s[0:1], 0x3f717273, s4        ; encoding: [0xff,0x04,0x00,0x94,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bfe_u64 s[0:1], 0xaf123456, s4        ; encoding: [0xff,0x04,0x00,0x94,0x56,0x34,0x12,0xaf]
 0xff,0x04,0x00,0x94,0x56,0x34,0x12,0xaf
+# GFX12: s_bfe_u64 s[0:1], 0xaf123456, s4        ; encoding: [0xff,0x04,0x00,0x94,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bfe_u64 s[0:1], -1, s4                ; encoding: [0xc1,0x04,0x00,0x94]
 0xc1,0x04,0x00,0x94
+# GFX12: s_bfe_u64 s[0:1], -1, s4                ; encoding: [0xc1,0x04,0x00,0x94]
 
-# GFX12: s_bfe_u64 s[0:1], -4.0, s4              ; encoding: [0xf7,0x04,0x00,0x94]
 0xf7,0x04,0x00,0x94
+# GFX12: s_bfe_u64 s[0:1], -4.0, s4              ; encoding: [0xf7,0x04,0x00,0x94]
 
-# GFX12: s_bfe_u64 s[0:1], exec, s4              ; encoding: [0x7e,0x04,0x00,0x94]
 0x7e,0x04,0x00,0x94
+# GFX12: s_bfe_u64 s[0:1], exec, s4              ; encoding: [0x7e,0x04,0x00,0x94]
 
-# GFX12: s_bfe_u64 s[0:1], s[102:103], s100      ; encoding: [0x66,0x64,0x00,0x94]
 0x66,0x64,0x00,0x94
+# GFX12: s_bfe_u64 s[0:1], s[102:103], s100      ; encoding: [0x66,0x64,0x00,0x94]
 
-# GFX12: s_bfe_u64 s[0:1], s[102:103], s4        ; encoding: [0x66,0x04,0x00,0x94]
 0x66,0x04,0x00,0x94
+# GFX12: s_bfe_u64 s[0:1], s[102:103], s4        ; encoding: [0x66,0x04,0x00,0x94]
 
-# GFX12: s_bfe_u64 s[0:1], s[2:3], 0.5           ; encoding: [0x02,0xf0,0x00,0x94]
 0x02,0xf0,0x00,0x94
+# GFX12: s_bfe_u64 s[0:1], s[2:3], 0.5           ; encoding: [0x02,0xf0,0x00,0x94]
 
-# GFX12: s_bfe_u64 s[0:1], s[2:3], 0             ; encoding: [0x02,0x80,0x00,0x94]
 0x02,0x80,0x00,0x94
+# GFX12: s_bfe_u64 s[0:1], s[2:3], 0             ; encoding: [0x02,0x80,0x00,0x94]
 
-# GFX12: s_bfe_u64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x00,0x94,0x73,0x72,0x71,0x3f]
 0x02,0xff,0x00,0x94,0x73,0x72,0x71,0x3f
+# GFX12: s_bfe_u64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x00,0x94,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bfe_u64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x00,0x94,0x56,0x34,0x12,0xaf]
 0x02,0xff,0x00,0x94,0x56,0x34,0x12,0xaf
+# GFX12: s_bfe_u64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x00,0x94,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bfe_u64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x00,0x94]
 0x02,0xc1,0x00,0x94
+# GFX12: s_bfe_u64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x00,0x94]
 
-# GFX12: s_bfe_u64 s[0:1], s[2:3], -4.0          ; encoding: [0x02,0xf7,0x00,0x94]
 0x02,0xf7,0x00,0x94
+# GFX12: s_bfe_u64 s[0:1], s[2:3], -4.0          ; encoding: [0x02,0xf7,0x00,0x94]
 
-# GFX12: s_bfe_u64 s[0:1], s[2:3], exec_lo       ; encoding: [0x02,0x7e,0x00,0x94]
 0x02,0x7e,0x00,0x94
+# GFX12: s_bfe_u64 s[0:1], s[2:3], exec_lo       ; encoding: [0x02,0x7e,0x00,0x94]
 
-# GFX12: s_bfe_u64 s[0:1], s[2:3], s100          ; encoding: [0x02,0x64,0x00,0x94]
 0x02,0x64,0x00,0x94
+# GFX12: s_bfe_u64 s[0:1], s[2:3], s100          ; encoding: [0x02,0x64,0x00,0x94]
 
-# GFX12: s_bfe_u64 s[0:1], s[2:3], s4            ; encoding: [0x02,0x04,0x00,0x94]
 0x02,0x04,0x00,0x94
+# GFX12: s_bfe_u64 s[0:1], s[2:3], s4            ; encoding: [0x02,0x04,0x00,0x94]
 
-# GFX12: s_bfe_u64 s[0:1], s[2:3], vcc_lo        ; encoding: [0x02,0x6a,0x00,0x94]
 0x02,0x6a,0x00,0x94
+# GFX12: s_bfe_u64 s[0:1], s[2:3], vcc_lo        ; encoding: [0x02,0x6a,0x00,0x94]
 
-# GFX12: s_bfe_u64 s[0:1], vcc, s4               ; encoding: [0x6a,0x04,0x00,0x94]
 0x6a,0x04,0x00,0x94
+# GFX12: s_bfe_u64 s[0:1], vcc, s4               ; encoding: [0x6a,0x04,0x00,0x94]
 
-# GFX12: s_bfe_u64 s[104:105], s[102:103], s100  ; encoding: [0x66,0x64,0x68,0x94]
 0x66,0x64,0x68,0x94
+# GFX12: s_bfe_u64 s[104:105], s[102:103], s100  ; encoding: [0x66,0x64,0x68,0x94]
 
-# GFX12: s_bfe_u64 s[104:105], s[102:103], s4    ; encoding: [0x66,0x04,0x68,0x94]
 0x66,0x04,0x68,0x94
+# GFX12: s_bfe_u64 s[104:105], s[102:103], s4    ; encoding: [0x66,0x04,0x68,0x94]
 
-# GFX12: s_bfe_u64 s[104:105], s[2:3], s100      ; encoding: [0x02,0x64,0x68,0x94]
 0x02,0x64,0x68,0x94
+# GFX12: s_bfe_u64 s[104:105], s[2:3], s100      ; encoding: [0x02,0x64,0x68,0x94]
 
-# GFX12: s_bfe_u64 s[104:105], s[2:3], s4        ; encoding: [0x02,0x04,0x68,0x94]
 0x02,0x04,0x68,0x94
+# GFX12: s_bfe_u64 s[104:105], s[2:3], s4        ; encoding: [0x02,0x04,0x68,0x94]
 
-# GFX12: s_bfe_u64 vcc, s[2:3], s4               ; encoding: [0x02,0x04,0x6a,0x94]
 0x02,0x04,0x6a,0x94
+# GFX12: s_bfe_u64 vcc, s[2:3], s4               ; encoding: [0x02,0x04,0x6a,0x94]
 
-# GFX12: s_bfm_b32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x95]
 0x01,0x02,0x7f,0x95
+# GFX12: s_bfm_b32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x95]
 
-# GFX12: s_bfm_b32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x95]
 0x01,0x02,0x7e,0x95
+# GFX12: s_bfm_b32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x95]
 
-# GFX12: s_bfm_b32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x95]
 0x01,0x02,0x7d,0x95
+# GFX12: s_bfm_b32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x95]
 
-# GFX12: s_bfm_b32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x95]
 0xf0,0x02,0x00,0x95
+# GFX12: s_bfm_b32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x95]
 0x80,0x02,0x00,0x95
+# GFX12: s_bfm_b32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x95,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x95,0x73,0x72,0x71,0x3f
+# GFX12: s_bfm_b32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x95,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bfm_b32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x95,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x95,0x56,0x34,0x12,0xaf
+# GFX12: s_bfm_b32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x95,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bfm_b32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x95]
 0xc1,0x02,0x00,0x95
+# GFX12: s_bfm_b32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x95]
 0xf7,0x02,0x00,0x95
+# GFX12: s_bfm_b32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x95]
 0x7f,0x02,0x00,0x95
+# GFX12: s_bfm_b32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x95]
 0x7e,0x02,0x00,0x95
+# GFX12: s_bfm_b32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x95]
 0x7d,0x02,0x00,0x95
+# GFX12: s_bfm_b32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x95]
 0x68,0x67,0x00,0x95
+# GFX12: s_bfm_b32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x95]
 0x68,0x02,0x00,0x95
+# GFX12: s_bfm_b32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x95]
 0x01,0xf0,0x00,0x95
+# GFX12: s_bfm_b32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x95]
 0x01,0x80,0x00,0x95
+# GFX12: s_bfm_b32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x95,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x95,0x73,0x72,0x71,0x3f
+# GFX12: s_bfm_b32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x95,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bfm_b32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x95,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x95,0x56,0x34,0x12,0xaf
+# GFX12: s_bfm_b32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x95,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bfm_b32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x95]
 0x01,0xc1,0x00,0x95
+# GFX12: s_bfm_b32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x95]
 0x01,0xf7,0x00,0x95
+# GFX12: s_bfm_b32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x95]
 0x01,0x7f,0x00,0x95
+# GFX12: s_bfm_b32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x95]
 0x01,0x7e,0x00,0x95
+# GFX12: s_bfm_b32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x95]
 0x01,0x7d,0x00,0x95
+# GFX12: s_bfm_b32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x95]
 0x01,0x67,0x00,0x95
+# GFX12: s_bfm_b32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x95]
 0x01,0x02,0x00,0x95
+# GFX12: s_bfm_b32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x95]
 0x01,0x6b,0x00,0x95
+# GFX12: s_bfm_b32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x95]
 0x01,0x6a,0x00,0x95
+# GFX12: s_bfm_b32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x95]
 0x6b,0x02,0x00,0x95
+# GFX12: s_bfm_b32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x95]
 
-# GFX12: s_bfm_b32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x95]
 0x6a,0x02,0x00,0x95
+# GFX12: s_bfm_b32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x95]
 
-# GFX12: s_bfm_b32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x95]
 0x68,0x67,0x69,0x95
+# GFX12: s_bfm_b32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x95]
 
-# GFX12: s_bfm_b32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x95]
 0x68,0x02,0x69,0x95
+# GFX12: s_bfm_b32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x95]
 
-# GFX12: s_bfm_b32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x95]
 0x01,0x67,0x69,0x95
+# GFX12: s_bfm_b32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x95]
 
-# GFX12: s_bfm_b32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x95]
 0x01,0x02,0x69,0x95
+# GFX12: s_bfm_b32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x95]
 
-# GFX12: s_bfm_b32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x95]
 0x01,0x02,0x6b,0x95
+# GFX12: s_bfm_b32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x95]
 
-# GFX12: s_bfm_b32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x95]
 0x01,0x02,0x6a,0x95
+# GFX12: s_bfm_b32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x95]
 
-# GFX12: s_bfm_b64 exec, s2, s3                  ; encoding: [0x02,0x03,0xfe,0x95]
 0x02,0x03,0xfe,0x95
+# GFX12: s_bfm_b64 exec, s2, s3                  ; encoding: [0x02,0x03,0xfe,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], 0.5, s3               ; encoding: [0xf0,0x03,0x80,0x95]
 0xf0,0x03,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], 0.5, s3               ; encoding: [0xf0,0x03,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], 0, s3                 ; encoding: [0x80,0x03,0x80,0x95]
 0x80,0x03,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], 0, s3                 ; encoding: [0x80,0x03,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], 0x3f717273, s3        ; encoding: [0xff,0x03,0x80,0x95,0x73,0x72,0x71,0x3f]
 0xff,0x03,0x80,0x95,0x73,0x72,0x71,0x3f
+# GFX12: s_bfm_b64 s[0:1], 0x3f717273, s3        ; encoding: [0xff,0x03,0x80,0x95,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bfm_b64 s[0:1], 0xaf123456, s3        ; encoding: [0xff,0x03,0x80,0x95,0x56,0x34,0x12,0xaf]
 0xff,0x03,0x80,0x95,0x56,0x34,0x12,0xaf
+# GFX12: s_bfm_b64 s[0:1], 0xaf123456, s3        ; encoding: [0xff,0x03,0x80,0x95,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bfm_b64 s[0:1], -1, s3                ; encoding: [0xc1,0x03,0x80,0x95]
 0xc1,0x03,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], -1, s3                ; encoding: [0xc1,0x03,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], -4.0, s3              ; encoding: [0xf7,0x03,0x80,0x95]
 0xf7,0x03,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], -4.0, s3              ; encoding: [0xf7,0x03,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], exec_hi, s3           ; encoding: [0x7f,0x03,0x80,0x95]
 0x7f,0x03,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], exec_hi, s3           ; encoding: [0x7f,0x03,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], exec_lo, s3           ; encoding: [0x7e,0x03,0x80,0x95]
 0x7e,0x03,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], exec_lo, s3           ; encoding: [0x7e,0x03,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], m0, s3                ; encoding: [0x7d,0x03,0x80,0x95]
 0x7d,0x03,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], m0, s3                ; encoding: [0x7d,0x03,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], s104, s103            ; encoding: [0x68,0x67,0x80,0x95]
 0x68,0x67,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], s104, s103            ; encoding: [0x68,0x67,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], s104, s3              ; encoding: [0x68,0x03,0x80,0x95]
 0x68,0x03,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], s104, s3              ; encoding: [0x68,0x03,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], s2, 0.5               ; encoding: [0x02,0xf0,0x80,0x95]
 0x02,0xf0,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], s2, 0.5               ; encoding: [0x02,0xf0,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], s2, 0                 ; encoding: [0x02,0x80,0x80,0x95]
 0x02,0x80,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], s2, 0                 ; encoding: [0x02,0x80,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], s2, 0x3f717273        ; encoding: [0x02,0xff,0x80,0x95,0x73,0x72,0x71,0x3f]
 0x02,0xff,0x80,0x95,0x73,0x72,0x71,0x3f
+# GFX12: s_bfm_b64 s[0:1], s2, 0x3f717273        ; encoding: [0x02,0xff,0x80,0x95,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bfm_b64 s[0:1], s2, 0xaf123456        ; encoding: [0x02,0xff,0x80,0x95,0x56,0x34,0x12,0xaf]
 0x02,0xff,0x80,0x95,0x56,0x34,0x12,0xaf
+# GFX12: s_bfm_b64 s[0:1], s2, 0xaf123456        ; encoding: [0x02,0xff,0x80,0x95,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bfm_b64 s[0:1], s2, -1                ; encoding: [0x02,0xc1,0x80,0x95]
 0x02,0xc1,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], s2, -1                ; encoding: [0x02,0xc1,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], s2, -4.0              ; encoding: [0x02,0xf7,0x80,0x95]
 0x02,0xf7,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], s2, -4.0              ; encoding: [0x02,0xf7,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], s2, exec_hi           ; encoding: [0x02,0x7f,0x80,0x95]
 0x02,0x7f,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], s2, exec_hi           ; encoding: [0x02,0x7f,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], s2, exec_lo           ; encoding: [0x02,0x7e,0x80,0x95]
 0x02,0x7e,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], s2, exec_lo           ; encoding: [0x02,0x7e,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], s2, m0                ; encoding: [0x02,0x7d,0x80,0x95]
 0x02,0x7d,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], s2, m0                ; encoding: [0x02,0x7d,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], s2, s103              ; encoding: [0x02,0x67,0x80,0x95]
 0x02,0x67,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], s2, s103              ; encoding: [0x02,0x67,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], s2, s3                ; encoding: [0x02,0x03,0x80,0x95]
 0x02,0x03,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], s2, s3                ; encoding: [0x02,0x03,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], s2, vcc_hi            ; encoding: [0x02,0x6b,0x80,0x95]
 0x02,0x6b,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], s2, vcc_hi            ; encoding: [0x02,0x6b,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], s2, vcc_lo            ; encoding: [0x02,0x6a,0x80,0x95]
 0x02,0x6a,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], s2, vcc_lo            ; encoding: [0x02,0x6a,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], vcc_hi, s3            ; encoding: [0x6b,0x03,0x80,0x95]
 0x6b,0x03,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], vcc_hi, s3            ; encoding: [0x6b,0x03,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[0:1], vcc_lo, s3            ; encoding: [0x6a,0x03,0x80,0x95]
 0x6a,0x03,0x80,0x95
+# GFX12: s_bfm_b64 s[0:1], vcc_lo, s3            ; encoding: [0x6a,0x03,0x80,0x95]
 
-# GFX12: s_bfm_b64 s[104:105], s103, s102        ; encoding: [0x67,0x66,0xe8,0x95]
 0x67,0x66,0xe8,0x95
+# GFX12: s_bfm_b64 s[104:105], s103, s102        ; encoding: [0x67,0x66,0xe8,0x95]
 
-# GFX12: s_bfm_b64 s[104:105], s104, s3          ; encoding: [0x68,0x03,0xe8,0x95]
 0x68,0x03,0xe8,0x95
+# GFX12: s_bfm_b64 s[104:105], s104, s3          ; encoding: [0x68,0x03,0xe8,0x95]
 
-# GFX12: s_bfm_b64 s[104:105], s2, s103          ; encoding: [0x02,0x67,0xe8,0x95]
 0x02,0x67,0xe8,0x95
+# GFX12: s_bfm_b64 s[104:105], s2, s103          ; encoding: [0x02,0x67,0xe8,0x95]
 
-# GFX12: s_bfm_b64 s[104:105], s2, s3            ; encoding: [0x02,0x03,0xe8,0x95]
 0x02,0x03,0xe8,0x95
+# GFX12: s_bfm_b64 s[104:105], s2, s3            ; encoding: [0x02,0x03,0xe8,0x95]
 
-# GFX12: s_bfm_b64 vcc, s2, s3                   ; encoding: [0x02,0x03,0xea,0x95]
 0x02,0x03,0xea,0x95
+# GFX12: s_bfm_b64 vcc, s2, s3                   ; encoding: [0x02,0x03,0xea,0x95]
 
-# GFX12: s_cselect_b32 exec_hi, s1, s2           ; encoding: [0x01,0x02,0x7f,0x98]
 0x01,0x02,0x7f,0x98
+# GFX12: s_cselect_b32 exec_hi, s1, s2           ; encoding: [0x01,0x02,0x7f,0x98]
 
-# GFX12: s_cselect_b32 exec_lo, s1, s2           ; encoding: [0x01,0x02,0x7e,0x98]
 0x01,0x02,0x7e,0x98
+# GFX12: s_cselect_b32 exec_lo, s1, s2           ; encoding: [0x01,0x02,0x7e,0x98]
 
-# GFX12: s_cselect_b32 m0, s1, s2                ; encoding: [0x01,0x02,0x7d,0x98]
 0x01,0x02,0x7d,0x98
+# GFX12: s_cselect_b32 m0, s1, s2                ; encoding: [0x01,0x02,0x7d,0x98]
 
-# GFX12: s_cselect_b32 s0, 0.5, s2               ; encoding: [0xf0,0x02,0x00,0x98]
 0xf0,0x02,0x00,0x98
+# GFX12: s_cselect_b32 s0, 0.5, s2               ; encoding: [0xf0,0x02,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, 0, s2                 ; encoding: [0x80,0x02,0x00,0x98]
 0x80,0x02,0x00,0x98
+# GFX12: s_cselect_b32 s0, 0, s2                 ; encoding: [0x80,0x02,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, 0x3f717273, s2        ; encoding: [0xff,0x02,0x00,0x98,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x98,0x73,0x72,0x71,0x3f
+# GFX12: s_cselect_b32 s0, 0x3f717273, s2        ; encoding: [0xff,0x02,0x00,0x98,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cselect_b32 s0, 0xaf123456, s2        ; encoding: [0xff,0x02,0x00,0x98,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x98,0x56,0x34,0x12,0xaf
+# GFX12: s_cselect_b32 s0, 0xaf123456, s2        ; encoding: [0xff,0x02,0x00,0x98,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cselect_b32 s0, -1, s2                ; encoding: [0xc1,0x02,0x00,0x98]
 0xc1,0x02,0x00,0x98
+# GFX12: s_cselect_b32 s0, -1, s2                ; encoding: [0xc1,0x02,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, -4.0, s2              ; encoding: [0xf7,0x02,0x00,0x98]
 0xf7,0x02,0x00,0x98
+# GFX12: s_cselect_b32 s0, -4.0, s2              ; encoding: [0xf7,0x02,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, exec_hi, s2           ; encoding: [0x7f,0x02,0x00,0x98]
 0x7f,0x02,0x00,0x98
+# GFX12: s_cselect_b32 s0, exec_hi, s2           ; encoding: [0x7f,0x02,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, exec_lo, s2           ; encoding: [0x7e,0x02,0x00,0x98]
 0x7e,0x02,0x00,0x98
+# GFX12: s_cselect_b32 s0, exec_lo, s2           ; encoding: [0x7e,0x02,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, m0, s2                ; encoding: [0x7d,0x02,0x00,0x98]
 0x7d,0x02,0x00,0x98
+# GFX12: s_cselect_b32 s0, m0, s2                ; encoding: [0x7d,0x02,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, s104, s103            ; encoding: [0x68,0x67,0x00,0x98]
 0x68,0x67,0x00,0x98
+# GFX12: s_cselect_b32 s0, s104, s103            ; encoding: [0x68,0x67,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, s104, s2              ; encoding: [0x68,0x02,0x00,0x98]
 0x68,0x02,0x00,0x98
+# GFX12: s_cselect_b32 s0, s104, s2              ; encoding: [0x68,0x02,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, s1, 0.5               ; encoding: [0x01,0xf0,0x00,0x98]
 0x01,0xf0,0x00,0x98
+# GFX12: s_cselect_b32 s0, s1, 0.5               ; encoding: [0x01,0xf0,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, s1, 0                 ; encoding: [0x01,0x80,0x00,0x98]
 0x01,0x80,0x00,0x98
+# GFX12: s_cselect_b32 s0, s1, 0                 ; encoding: [0x01,0x80,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, s1, 0x3f717273        ; encoding: [0x01,0xff,0x00,0x98,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x98,0x73,0x72,0x71,0x3f
+# GFX12: s_cselect_b32 s0, s1, 0x3f717273        ; encoding: [0x01,0xff,0x00,0x98,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cselect_b32 s0, s1, 0xaf123456        ; encoding: [0x01,0xff,0x00,0x98,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x98,0x56,0x34,0x12,0xaf
+# GFX12: s_cselect_b32 s0, s1, 0xaf123456        ; encoding: [0x01,0xff,0x00,0x98,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cselect_b32 s0, s1, -1                ; encoding: [0x01,0xc1,0x00,0x98]
 0x01,0xc1,0x00,0x98
+# GFX12: s_cselect_b32 s0, s1, -1                ; encoding: [0x01,0xc1,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, s1, -4.0              ; encoding: [0x01,0xf7,0x00,0x98]
 0x01,0xf7,0x00,0x98
+# GFX12: s_cselect_b32 s0, s1, -4.0              ; encoding: [0x01,0xf7,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, s1, exec_hi           ; encoding: [0x01,0x7f,0x00,0x98]
 0x01,0x7f,0x00,0x98
+# GFX12: s_cselect_b32 s0, s1, exec_hi           ; encoding: [0x01,0x7f,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, s1, exec_lo           ; encoding: [0x01,0x7e,0x00,0x98]
 0x01,0x7e,0x00,0x98
+# GFX12: s_cselect_b32 s0, s1, exec_lo           ; encoding: [0x01,0x7e,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, s1, m0                ; encoding: [0x01,0x7d,0x00,0x98]
 0x01,0x7d,0x00,0x98
+# GFX12: s_cselect_b32 s0, s1, m0                ; encoding: [0x01,0x7d,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, s1, s103              ; encoding: [0x01,0x67,0x00,0x98]
 0x01,0x67,0x00,0x98
+# GFX12: s_cselect_b32 s0, s1, s103              ; encoding: [0x01,0x67,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, s1, s2                ; encoding: [0x01,0x02,0x00,0x98]
 0x01,0x02,0x00,0x98
+# GFX12: s_cselect_b32 s0, s1, s2                ; encoding: [0x01,0x02,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, s1, vcc_hi            ; encoding: [0x01,0x6b,0x00,0x98]
 0x01,0x6b,0x00,0x98
+# GFX12: s_cselect_b32 s0, s1, vcc_hi            ; encoding: [0x01,0x6b,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, s1, vcc_lo            ; encoding: [0x01,0x6a,0x00,0x98]
 0x01,0x6a,0x00,0x98
+# GFX12: s_cselect_b32 s0, s1, vcc_lo            ; encoding: [0x01,0x6a,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, vcc_hi, s2            ; encoding: [0x6b,0x02,0x00,0x98]
 0x6b,0x02,0x00,0x98
+# GFX12: s_cselect_b32 s0, vcc_hi, s2            ; encoding: [0x6b,0x02,0x00,0x98]
 
-# GFX12: s_cselect_b32 s0, vcc_lo, s2            ; encoding: [0x6a,0x02,0x00,0x98]
 0x6a,0x02,0x00,0x98
+# GFX12: s_cselect_b32 s0, vcc_lo, s2            ; encoding: [0x6a,0x02,0x00,0x98]
 
-# GFX12: s_cselect_b32 s105, s104, s103          ; encoding: [0x68,0x67,0x69,0x98]
 0x68,0x67,0x69,0x98
+# GFX12: s_cselect_b32 s105, s104, s103          ; encoding: [0x68,0x67,0x69,0x98]
 
-# GFX12: s_cselect_b32 s105, s104, s2            ; encoding: [0x68,0x02,0x69,0x98]
 0x68,0x02,0x69,0x98
+# GFX12: s_cselect_b32 s105, s104, s2            ; encoding: [0x68,0x02,0x69,0x98]
 
-# GFX12: s_cselect_b32 s105, s1, s103            ; encoding: [0x01,0x67,0x69,0x98]
 0x01,0x67,0x69,0x98
+# GFX12: s_cselect_b32 s105, s1, s103            ; encoding: [0x01,0x67,0x69,0x98]
 
-# GFX12: s_cselect_b32 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0x98]
 0x01,0x02,0x69,0x98
+# GFX12: s_cselect_b32 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0x98]
 
-# GFX12: s_cselect_b32 vcc_hi, s1, s2            ; encoding: [0x01,0x02,0x6b,0x98]
 0x01,0x02,0x6b,0x98
+# GFX12: s_cselect_b32 vcc_hi, s1, s2            ; encoding: [0x01,0x02,0x6b,0x98]
 
-# GFX12: s_cselect_b32 vcc_lo, s1, s2            ; encoding: [0x01,0x02,0x6a,0x98]
 0x01,0x02,0x6a,0x98
+# GFX12: s_cselect_b32 vcc_lo, s1, s2            ; encoding: [0x01,0x02,0x6a,0x98]
 
-# GFX12: s_cselect_b64 exec, s[2:3], s[4:5]      ; encoding: [0x02,0x04,0xfe,0x98]
 0x02,0x04,0xfe,0x98
+# GFX12: s_cselect_b64 exec, s[2:3], s[4:5]      ; encoding: [0x02,0x04,0xfe,0x98]
 
-# GFX12: s_cselect_b64 s[0:1], 0.5, s[4:5]       ; encoding: [0xf0,0x04,0x80,0x98]
 0xf0,0x04,0x80,0x98
+# GFX12: s_cselect_b64 s[0:1], 0.5, s[4:5]       ; encoding: [0xf0,0x04,0x80,0x98]
 
-# GFX12: s_cselect_b64 s[0:1], 0, s[4:5]         ; encoding: [0x80,0x04,0x80,0x98]
 0x80,0x04,0x80,0x98
+# GFX12: s_cselect_b64 s[0:1], 0, s[4:5]         ; encoding: [0x80,0x04,0x80,0x98]
 
-# GFX12: s_cselect_b64 s[0:1], 0x3f717273, s[4:5] ; encoding: [0xff,0x04,0x80,0x98,0x73,0x72,0x71,0x3f]
 0xff,0x04,0x80,0x98,0x73,0x72,0x71,0x3f
+# GFX12: s_cselect_b64 s[0:1], 0x3f717273, s[4:5] ; encoding: [0xff,0x04,0x80,0x98,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cselect_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x98,0x56,0x34,0x12,0xaf]
 0xff,0x04,0x80,0x98,0x56,0x34,0x12,0xaf
+# GFX12: s_cselect_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x98,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cselect_b64 s[0:1], -1, s[4:5]        ; encoding: [0xc1,0x04,0x80,0x98]
 0xc1,0x04,0x80,0x98
+# GFX12: s_cselect_b64 s[0:1], -1, s[4:5]        ; encoding: [0xc1,0x04,0x80,0x98]
 
-# GFX12: s_cselect_b64 s[0:1], -4.0, s[4:5]      ; encoding: [0xf7,0x04,0x80,0x98]
 0xf7,0x04,0x80,0x98
+# GFX12: s_cselect_b64 s[0:1], -4.0, s[4:5]      ; encoding: [0xf7,0x04,0x80,0x98]
 
-# GFX12: s_cselect_b64 s[0:1], exec, s[4:5]      ; encoding: [0x7e,0x04,0x80,0x98]
 0x7e,0x04,0x80,0x98
+# GFX12: s_cselect_b64 s[0:1], exec, s[4:5]      ; encoding: [0x7e,0x04,0x80,0x98]
 
-# GFX12: s_cselect_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x98]
 0x66,0x64,0x80,0x98
+# GFX12: s_cselect_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x98]
 
-# GFX12: s_cselect_b64 s[0:1], s[102:103], s[4:5] ; encoding: [0x66,0x04,0x80,0x98]
 0x66,0x04,0x80,0x98
+# GFX12: s_cselect_b64 s[0:1], s[102:103], s[4:5] ; encoding: [0x66,0x04,0x80,0x98]
 
-# GFX12: s_cselect_b64 s[0:1], s[2:3], 0.5       ; encoding: [0x02,0xf0,0x80,0x98]
 0x02,0xf0,0x80,0x98
+# GFX12: s_cselect_b64 s[0:1], s[2:3], 0.5       ; encoding: [0x02,0xf0,0x80,0x98]
 
-# GFX12: s_cselect_b64 s[0:1], s[2:3], 0         ; encoding: [0x02,0x80,0x80,0x98]
 0x02,0x80,0x80,0x98
+# GFX12: s_cselect_b64 s[0:1], s[2:3], 0         ; encoding: [0x02,0x80,0x80,0x98]
 
-# GFX12: s_cselect_b64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0x98,0x73,0x72,0x71,0x3f]
 0x02,0xff,0x80,0x98,0x73,0x72,0x71,0x3f
+# GFX12: s_cselect_b64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0x98,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cselect_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x98,0x56,0x34,0x12,0xaf]
 0x02,0xff,0x80,0x98,0x56,0x34,0x12,0xaf
+# GFX12: s_cselect_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x98,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cselect_b64 s[0:1], s[2:3], -1        ; encoding: [0x02,0xc1,0x80,0x98]
 0x02,0xc1,0x80,0x98
+# GFX12: s_cselect_b64 s[0:1], s[2:3], -1        ; encoding: [0x02,0xc1,0x80,0x98]
 
-# GFX12: s_cselect_b64 s[0:1], s[2:3], -4.0      ; encoding: [0x02,0xf7,0x80,0x98]
 0x02,0xf7,0x80,0x98
+# GFX12: s_cselect_b64 s[0:1], s[2:3], -4.0      ; encoding: [0x02,0xf7,0x80,0x98]
 
-# GFX12: s_cselect_b64 s[0:1], s[2:3], exec      ; encoding: [0x02,0x7e,0x80,0x98]
 0x02,0x7e,0x80,0x98
+# GFX12: s_cselect_b64 s[0:1], s[2:3], exec      ; encoding: [0x02,0x7e,0x80,0x98]
 
-# GFX12: s_cselect_b64 s[0:1], s[2:3], s[100:101] ; encoding: [0x02,0x64,0x80,0x98]
 0x02,0x64,0x80,0x98
+# GFX12: s_cselect_b64 s[0:1], s[2:3], s[100:101] ; encoding: [0x02,0x64,0x80,0x98]
 
-# GFX12: s_cselect_b64 s[0:1], s[2:3], s[4:5]    ; encoding: [0x02,0x04,0x80,0x98]
 0x02,0x04,0x80,0x98
+# GFX12: s_cselect_b64 s[0:1], s[2:3], s[4:5]    ; encoding: [0x02,0x04,0x80,0x98]
 
-# GFX12: s_cselect_b64 s[0:1], s[2:3], vcc       ; encoding: [0x02,0x6a,0x80,0x98]
 0x02,0x6a,0x80,0x98
+# GFX12: s_cselect_b64 s[0:1], s[2:3], vcc       ; encoding: [0x02,0x6a,0x80,0x98]
 
-# GFX12: s_cselect_b64 s[0:1], vcc, s[4:5]       ; encoding: [0x6a,0x04,0x80,0x98]
 0x6a,0x04,0x80,0x98
+# GFX12: s_cselect_b64 s[0:1], vcc, s[4:5]       ; encoding: [0x6a,0x04,0x80,0x98]
 
-# GFX12: s_cselect_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x98]
 0x66,0x64,0xe8,0x98
+# GFX12: s_cselect_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x98]
 
-# GFX12: s_cselect_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x98]
 0x66,0x04,0xe8,0x98
+# GFX12: s_cselect_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x98]
 
-# GFX12: s_cselect_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x98]
 0x02,0x64,0xe8,0x98
+# GFX12: s_cselect_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x98]
 
-# GFX12: s_cselect_b64 s[104:105], s[2:3], s[4:5] ; encoding: [0x02,0x04,0xe8,0x98]
 0x02,0x04,0xe8,0x98
+# GFX12: s_cselect_b64 s[104:105], s[2:3], s[4:5] ; encoding: [0x02,0x04,0xe8,0x98]
 
-# GFX12: s_cselect_b64 vcc, s[2:3], s[4:5]       ; encoding: [0x02,0x04,0xea,0x98]
 0x02,0x04,0xea,0x98
+# GFX12: s_cselect_b64 vcc, s[2:3], s[4:5]       ; encoding: [0x02,0x04,0xea,0x98]
 
-# GFX12: s_lshl1_add_u32 exec_hi, s1, s2         ; encoding: [0x01,0x02,0x7f,0x87]
 0x01,0x02,0x7f,0x87
+# GFX12: s_lshl1_add_u32 exec_hi, s1, s2         ; encoding: [0x01,0x02,0x7f,0x87]
 
-# GFX12: s_lshl1_add_u32 exec_lo, s1, s2         ; encoding: [0x01,0x02,0x7e,0x87]
 0x01,0x02,0x7e,0x87
+# GFX12: s_lshl1_add_u32 exec_lo, s1, s2         ; encoding: [0x01,0x02,0x7e,0x87]
 
-# GFX12: s_lshl1_add_u32 m0, s1, s2              ; encoding: [0x01,0x02,0x7d,0x87]
 0x01,0x02,0x7d,0x87
+# GFX12: s_lshl1_add_u32 m0, s1, s2              ; encoding: [0x01,0x02,0x7d,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, 0.5, s2             ; encoding: [0xf0,0x02,0x00,0x87]
 0xf0,0x02,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, 0.5, s2             ; encoding: [0xf0,0x02,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, 0, s2               ; encoding: [0x80,0x02,0x00,0x87]
 0x80,0x02,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, 0, s2               ; encoding: [0x80,0x02,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, 0x3f717273, s2      ; encoding: [0xff,0x02,0x00,0x87,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x87,0x73,0x72,0x71,0x3f
+# GFX12: s_lshl1_add_u32 s0, 0x3f717273, s2      ; encoding: [0xff,0x02,0x00,0x87,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_lshl1_add_u32 s0, 0xaf123456, s2      ; encoding: [0xff,0x02,0x00,0x87,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x87,0x56,0x34,0x12,0xaf
+# GFX12: s_lshl1_add_u32 s0, 0xaf123456, s2      ; encoding: [0xff,0x02,0x00,0x87,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_lshl1_add_u32 s0, -1, s2              ; encoding: [0xc1,0x02,0x00,0x87]
 0xc1,0x02,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, -1, s2              ; encoding: [0xc1,0x02,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, -4.0, s2            ; encoding: [0xf7,0x02,0x00,0x87]
 0xf7,0x02,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, -4.0, s2            ; encoding: [0xf7,0x02,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, exec_hi, s2         ; encoding: [0x7f,0x02,0x00,0x87]
 0x7f,0x02,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, exec_hi, s2         ; encoding: [0x7f,0x02,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, exec_lo, s2         ; encoding: [0x7e,0x02,0x00,0x87]
 0x7e,0x02,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, exec_lo, s2         ; encoding: [0x7e,0x02,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, m0, s2              ; encoding: [0x7d,0x02,0x00,0x87]
 0x7d,0x02,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, m0, s2              ; encoding: [0x7d,0x02,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, s104, s103          ; encoding: [0x68,0x67,0x00,0x87]
 0x68,0x67,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, s104, s103          ; encoding: [0x68,0x67,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, s104, s2            ; encoding: [0x68,0x02,0x00,0x87]
 0x68,0x02,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, s104, s2            ; encoding: [0x68,0x02,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, s1, 0.5             ; encoding: [0x01,0xf0,0x00,0x87]
 0x01,0xf0,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, s1, 0.5             ; encoding: [0x01,0xf0,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, s1, 0               ; encoding: [0x01,0x80,0x00,0x87]
 0x01,0x80,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, s1, 0               ; encoding: [0x01,0x80,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, s1, 0x3f717273      ; encoding: [0x01,0xff,0x00,0x87,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x87,0x73,0x72,0x71,0x3f
+# GFX12: s_lshl1_add_u32 s0, s1, 0x3f717273      ; encoding: [0x01,0xff,0x00,0x87,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_lshl1_add_u32 s0, s1, 0xaf123456      ; encoding: [0x01,0xff,0x00,0x87,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x87,0x56,0x34,0x12,0xaf
+# GFX12: s_lshl1_add_u32 s0, s1, 0xaf123456      ; encoding: [0x01,0xff,0x00,0x87,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_lshl1_add_u32 s0, s1, -1              ; encoding: [0x01,0xc1,0x00,0x87]
 0x01,0xc1,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, s1, -1              ; encoding: [0x01,0xc1,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, s1, -4.0            ; encoding: [0x01,0xf7,0x00,0x87]
 0x01,0xf7,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, s1, -4.0            ; encoding: [0x01,0xf7,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, s1, exec_hi         ; encoding: [0x01,0x7f,0x00,0x87]
 0x01,0x7f,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, s1, exec_hi         ; encoding: [0x01,0x7f,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, s1, exec_lo         ; encoding: [0x01,0x7e,0x00,0x87]
 0x01,0x7e,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, s1, exec_lo         ; encoding: [0x01,0x7e,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, s1, m0              ; encoding: [0x01,0x7d,0x00,0x87]
 0x01,0x7d,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, s1, m0              ; encoding: [0x01,0x7d,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, s1, s103            ; encoding: [0x01,0x67,0x00,0x87]
 0x01,0x67,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, s1, s103            ; encoding: [0x01,0x67,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, s1, s2              ; encoding: [0x01,0x02,0x00,0x87]
 0x01,0x02,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, s1, s2              ; encoding: [0x01,0x02,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, s1, vcc_hi          ; encoding: [0x01,0x6b,0x00,0x87]
 0x01,0x6b,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, s1, vcc_hi          ; encoding: [0x01,0x6b,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, s1, vcc_lo          ; encoding: [0x01,0x6a,0x00,0x87]
 0x01,0x6a,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, s1, vcc_lo          ; encoding: [0x01,0x6a,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, vcc_hi, s2          ; encoding: [0x6b,0x02,0x00,0x87]
 0x6b,0x02,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, vcc_hi, s2          ; encoding: [0x6b,0x02,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s0, vcc_lo, s2          ; encoding: [0x6a,0x02,0x00,0x87]
 0x6a,0x02,0x00,0x87
+# GFX12: s_lshl1_add_u32 s0, vcc_lo, s2          ; encoding: [0x6a,0x02,0x00,0x87]
 
-# GFX12: s_lshl1_add_u32 s105, s104, s103        ; encoding: [0x68,0x67,0x69,0x87]
 0x68,0x67,0x69,0x87
+# GFX12: s_lshl1_add_u32 s105, s104, s103        ; encoding: [0x68,0x67,0x69,0x87]
 
-# GFX12: s_lshl1_add_u32 s105, s104, s2          ; encoding: [0x68,0x02,0x69,0x87]
 0x68,0x02,0x69,0x87
+# GFX12: s_lshl1_add_u32 s105, s104, s2          ; encoding: [0x68,0x02,0x69,0x87]
 
-# GFX12: s_lshl1_add_u32 s105, s1, s103          ; encoding: [0x01,0x67,0x69,0x87]
 0x01,0x67,0x69,0x87
+# GFX12: s_lshl1_add_u32 s105, s1, s103          ; encoding: [0x01,0x67,0x69,0x87]
 
-# GFX12: s_lshl1_add_u32 s105, s1, s2            ; encoding: [0x01,0x02,0x69,0x87]
 0x01,0x02,0x69,0x87
+# GFX12: s_lshl1_add_u32 s105, s1, s2            ; encoding: [0x01,0x02,0x69,0x87]
 
-# GFX12: s_lshl1_add_u32 vcc_hi, s1, s2          ; encoding: [0x01,0x02,0x6b,0x87]
 0x01,0x02,0x6b,0x87
+# GFX12: s_lshl1_add_u32 vcc_hi, s1, s2          ; encoding: [0x01,0x02,0x6b,0x87]
 
-# GFX12: s_lshl1_add_u32 vcc_lo, s1, s2          ; encoding: [0x01,0x02,0x6a,0x87]
 0x01,0x02,0x6a,0x87
+# GFX12: s_lshl1_add_u32 vcc_lo, s1, s2          ; encoding: [0x01,0x02,0x6a,0x87]
 
-# GFX12: s_lshl2_add_u32 exec_hi, s1, s2         ; encoding: [0x01,0x02,0xff,0x87]
 0x01,0x02,0xff,0x87
+# GFX12: s_lshl2_add_u32 exec_hi, s1, s2         ; encoding: [0x01,0x02,0xff,0x87]
 
-# GFX12: s_lshl2_add_u32 exec_lo, s1, s2         ; encoding: [0x01,0x02,0xfe,0x87]
 0x01,0x02,0xfe,0x87
+# GFX12: s_lshl2_add_u32 exec_lo, s1, s2         ; encoding: [0x01,0x02,0xfe,0x87]
 
-# GFX12: s_lshl2_add_u32 m0, s1, s2              ; encoding: [0x01,0x02,0xfd,0x87]
 0x01,0x02,0xfd,0x87
+# GFX12: s_lshl2_add_u32 m0, s1, s2              ; encoding: [0x01,0x02,0xfd,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, 0.5, s2             ; encoding: [0xf0,0x02,0x80,0x87]
 0xf0,0x02,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, 0.5, s2             ; encoding: [0xf0,0x02,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, 0, s2               ; encoding: [0x80,0x02,0x80,0x87]
 0x80,0x02,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, 0, s2               ; encoding: [0x80,0x02,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, 0x3f717273, s2      ; encoding: [0xff,0x02,0x80,0x87,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x80,0x87,0x73,0x72,0x71,0x3f
+# GFX12: s_lshl2_add_u32 s0, 0x3f717273, s2      ; encoding: [0xff,0x02,0x80,0x87,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_lshl2_add_u32 s0, 0xaf123456, s2      ; encoding: [0xff,0x02,0x80,0x87,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x80,0x87,0x56,0x34,0x12,0xaf
+# GFX12: s_lshl2_add_u32 s0, 0xaf123456, s2      ; encoding: [0xff,0x02,0x80,0x87,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_lshl2_add_u32 s0, -1, s2              ; encoding: [0xc1,0x02,0x80,0x87]
 0xc1,0x02,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, -1, s2              ; encoding: [0xc1,0x02,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, -4.0, s2            ; encoding: [0xf7,0x02,0x80,0x87]
 0xf7,0x02,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, -4.0, s2            ; encoding: [0xf7,0x02,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, exec_hi, s2         ; encoding: [0x7f,0x02,0x80,0x87]
 0x7f,0x02,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, exec_hi, s2         ; encoding: [0x7f,0x02,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, exec_lo, s2         ; encoding: [0x7e,0x02,0x80,0x87]
 0x7e,0x02,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, exec_lo, s2         ; encoding: [0x7e,0x02,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, m0, s2              ; encoding: [0x7d,0x02,0x80,0x87]
 0x7d,0x02,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, m0, s2              ; encoding: [0x7d,0x02,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, s104, s103          ; encoding: [0x68,0x67,0x80,0x87]
 0x68,0x67,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, s104, s103          ; encoding: [0x68,0x67,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, s104, s2            ; encoding: [0x68,0x02,0x80,0x87]
 0x68,0x02,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, s104, s2            ; encoding: [0x68,0x02,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, s1, 0.5             ; encoding: [0x01,0xf0,0x80,0x87]
 0x01,0xf0,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, s1, 0.5             ; encoding: [0x01,0xf0,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, s1, 0               ; encoding: [0x01,0x80,0x80,0x87]
 0x01,0x80,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, s1, 0               ; encoding: [0x01,0x80,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, s1, 0x3f717273      ; encoding: [0x01,0xff,0x80,0x87,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x80,0x87,0x73,0x72,0x71,0x3f
+# GFX12: s_lshl2_add_u32 s0, s1, 0x3f717273      ; encoding: [0x01,0xff,0x80,0x87,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_lshl2_add_u32 s0, s1, 0xaf123456      ; encoding: [0x01,0xff,0x80,0x87,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x80,0x87,0x56,0x34,0x12,0xaf
+# GFX12: s_lshl2_add_u32 s0, s1, 0xaf123456      ; encoding: [0x01,0xff,0x80,0x87,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_lshl2_add_u32 s0, s1, -1              ; encoding: [0x01,0xc1,0x80,0x87]
 0x01,0xc1,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, s1, -1              ; encoding: [0x01,0xc1,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, s1, -4.0            ; encoding: [0x01,0xf7,0x80,0x87]
 0x01,0xf7,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, s1, -4.0            ; encoding: [0x01,0xf7,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, s1, exec_hi         ; encoding: [0x01,0x7f,0x80,0x87]
 0x01,0x7f,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, s1, exec_hi         ; encoding: [0x01,0x7f,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, s1, exec_lo         ; encoding: [0x01,0x7e,0x80,0x87]
 0x01,0x7e,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, s1, exec_lo         ; encoding: [0x01,0x7e,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, s1, m0              ; encoding: [0x01,0x7d,0x80,0x87]
 0x01,0x7d,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, s1, m0              ; encoding: [0x01,0x7d,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, s1, s103            ; encoding: [0x01,0x67,0x80,0x87]
 0x01,0x67,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, s1, s103            ; encoding: [0x01,0x67,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, s1, s2              ; encoding: [0x01,0x02,0x80,0x87]
 0x01,0x02,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, s1, s2              ; encoding: [0x01,0x02,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, s1, vcc_hi          ; encoding: [0x01,0x6b,0x80,0x87]
 0x01,0x6b,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, s1, vcc_hi          ; encoding: [0x01,0x6b,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, s1, vcc_lo          ; encoding: [0x01,0x6a,0x80,0x87]
 0x01,0x6a,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, s1, vcc_lo          ; encoding: [0x01,0x6a,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, vcc_hi, s2          ; encoding: [0x6b,0x02,0x80,0x87]
 0x6b,0x02,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, vcc_hi, s2          ; encoding: [0x6b,0x02,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s0, vcc_lo, s2          ; encoding: [0x6a,0x02,0x80,0x87]
 0x6a,0x02,0x80,0x87
+# GFX12: s_lshl2_add_u32 s0, vcc_lo, s2          ; encoding: [0x6a,0x02,0x80,0x87]
 
-# GFX12: s_lshl2_add_u32 s105, s104, s103        ; encoding: [0x68,0x67,0xe9,0x87]
 0x68,0x67,0xe9,0x87
+# GFX12: s_lshl2_add_u32 s105, s104, s103        ; encoding: [0x68,0x67,0xe9,0x87]
 
-# GFX12: s_lshl2_add_u32 s105, s104, s2          ; encoding: [0x68,0x02,0xe9,0x87]
 0x68,0x02,0xe9,0x87
+# GFX12: s_lshl2_add_u32 s105, s104, s2          ; encoding: [0x68,0x02,0xe9,0x87]
 
-# GFX12: s_lshl2_add_u32 s105, s1, s103          ; encoding: [0x01,0x67,0xe9,0x87]
 0x01,0x67,0xe9,0x87
+# GFX12: s_lshl2_add_u32 s105, s1, s103          ; encoding: [0x01,0x67,0xe9,0x87]
 
-# GFX12: s_lshl2_add_u32 s105, s1, s2            ; encoding: [0x01,0x02,0xe9,0x87]
 0x01,0x02,0xe9,0x87
+# GFX12: s_lshl2_add_u32 s105, s1, s2            ; encoding: [0x01,0x02,0xe9,0x87]
 
-# GFX12: s_lshl2_add_u32 vcc_hi, s1, s2          ; encoding: [0x01,0x02,0xeb,0x87]
 0x01,0x02,0xeb,0x87
+# GFX12: s_lshl2_add_u32 vcc_hi, s1, s2          ; encoding: [0x01,0x02,0xeb,0x87]
 
-# GFX12: s_lshl2_add_u32 vcc_lo, s1, s2          ; encoding: [0x01,0x02,0xea,0x87]
 0x01,0x02,0xea,0x87
+# GFX12: s_lshl2_add_u32 vcc_lo, s1, s2          ; encoding: [0x01,0x02,0xea,0x87]
 
-# GFX12: s_lshl3_add_u32 exec_hi, s1, s2         ; encoding: [0x01,0x02,0x7f,0x88]
 0x01,0x02,0x7f,0x88
+# GFX12: s_lshl3_add_u32 exec_hi, s1, s2         ; encoding: [0x01,0x02,0x7f,0x88]
 
-# GFX12: s_lshl3_add_u32 exec_lo, s1, s2         ; encoding: [0x01,0x02,0x7e,0x88]
 0x01,0x02,0x7e,0x88
+# GFX12: s_lshl3_add_u32 exec_lo, s1, s2         ; encoding: [0x01,0x02,0x7e,0x88]
 
-# GFX12: s_lshl3_add_u32 m0, s1, s2              ; encoding: [0x01,0x02,0x7d,0x88]
 0x01,0x02,0x7d,0x88
+# GFX12: s_lshl3_add_u32 m0, s1, s2              ; encoding: [0x01,0x02,0x7d,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, 0.5, s2             ; encoding: [0xf0,0x02,0x00,0x88]
 0xf0,0x02,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, 0.5, s2             ; encoding: [0xf0,0x02,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, 0, s2               ; encoding: [0x80,0x02,0x00,0x88]
 0x80,0x02,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, 0, s2               ; encoding: [0x80,0x02,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, 0x3f717273, s2      ; encoding: [0xff,0x02,0x00,0x88,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x88,0x73,0x72,0x71,0x3f
+# GFX12: s_lshl3_add_u32 s0, 0x3f717273, s2      ; encoding: [0xff,0x02,0x00,0x88,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_lshl3_add_u32 s0, 0xaf123456, s2      ; encoding: [0xff,0x02,0x00,0x88,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x88,0x56,0x34,0x12,0xaf
+# GFX12: s_lshl3_add_u32 s0, 0xaf123456, s2      ; encoding: [0xff,0x02,0x00,0x88,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_lshl3_add_u32 s0, -1, s2              ; encoding: [0xc1,0x02,0x00,0x88]
 0xc1,0x02,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, -1, s2              ; encoding: [0xc1,0x02,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, -4.0, s2            ; encoding: [0xf7,0x02,0x00,0x88]
 0xf7,0x02,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, -4.0, s2            ; encoding: [0xf7,0x02,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, exec_hi, s2         ; encoding: [0x7f,0x02,0x00,0x88]
 0x7f,0x02,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, exec_hi, s2         ; encoding: [0x7f,0x02,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, exec_lo, s2         ; encoding: [0x7e,0x02,0x00,0x88]
 0x7e,0x02,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, exec_lo, s2         ; encoding: [0x7e,0x02,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, m0, s2              ; encoding: [0x7d,0x02,0x00,0x88]
 0x7d,0x02,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, m0, s2              ; encoding: [0x7d,0x02,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, s104, s103          ; encoding: [0x68,0x67,0x00,0x88]
 0x68,0x67,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, s104, s103          ; encoding: [0x68,0x67,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, s104, s2            ; encoding: [0x68,0x02,0x00,0x88]
 0x68,0x02,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, s104, s2            ; encoding: [0x68,0x02,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, s1, 0.5             ; encoding: [0x01,0xf0,0x00,0x88]
 0x01,0xf0,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, s1, 0.5             ; encoding: [0x01,0xf0,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, s1, 0               ; encoding: [0x01,0x80,0x00,0x88]
 0x01,0x80,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, s1, 0               ; encoding: [0x01,0x80,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, s1, 0x3f717273      ; encoding: [0x01,0xff,0x00,0x88,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x88,0x73,0x72,0x71,0x3f
+# GFX12: s_lshl3_add_u32 s0, s1, 0x3f717273      ; encoding: [0x01,0xff,0x00,0x88,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_lshl3_add_u32 s0, s1, 0xaf123456      ; encoding: [0x01,0xff,0x00,0x88,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x88,0x56,0x34,0x12,0xaf
+# GFX12: s_lshl3_add_u32 s0, s1, 0xaf123456      ; encoding: [0x01,0xff,0x00,0x88,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_lshl3_add_u32 s0, s1, -1              ; encoding: [0x01,0xc1,0x00,0x88]
 0x01,0xc1,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, s1, -1              ; encoding: [0x01,0xc1,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, s1, -4.0            ; encoding: [0x01,0xf7,0x00,0x88]
 0x01,0xf7,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, s1, -4.0            ; encoding: [0x01,0xf7,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, s1, exec_hi         ; encoding: [0x01,0x7f,0x00,0x88]
 0x01,0x7f,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, s1, exec_hi         ; encoding: [0x01,0x7f,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, s1, exec_lo         ; encoding: [0x01,0x7e,0x00,0x88]
 0x01,0x7e,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, s1, exec_lo         ; encoding: [0x01,0x7e,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, s1, m0              ; encoding: [0x01,0x7d,0x00,0x88]
 0x01,0x7d,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, s1, m0              ; encoding: [0x01,0x7d,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, s1, s103            ; encoding: [0x01,0x67,0x00,0x88]
 0x01,0x67,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, s1, s103            ; encoding: [0x01,0x67,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, s1, s2              ; encoding: [0x01,0x02,0x00,0x88]
 0x01,0x02,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, s1, s2              ; encoding: [0x01,0x02,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, s1, vcc_hi          ; encoding: [0x01,0x6b,0x00,0x88]
 0x01,0x6b,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, s1, vcc_hi          ; encoding: [0x01,0x6b,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, s1, vcc_lo          ; encoding: [0x01,0x6a,0x00,0x88]
 0x01,0x6a,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, s1, vcc_lo          ; encoding: [0x01,0x6a,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, vcc_hi, s2          ; encoding: [0x6b,0x02,0x00,0x88]
 0x6b,0x02,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, vcc_hi, s2          ; encoding: [0x6b,0x02,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s0, vcc_lo, s2          ; encoding: [0x6a,0x02,0x00,0x88]
 0x6a,0x02,0x00,0x88
+# GFX12: s_lshl3_add_u32 s0, vcc_lo, s2          ; encoding: [0x6a,0x02,0x00,0x88]
 
-# GFX12: s_lshl3_add_u32 s105, s104, s103        ; encoding: [0x68,0x67,0x69,0x88]
 0x68,0x67,0x69,0x88
+# GFX12: s_lshl3_add_u32 s105, s104, s103        ; encoding: [0x68,0x67,0x69,0x88]
 
-# GFX12: s_lshl3_add_u32 s105, s104, s2          ; encoding: [0x68,0x02,0x69,0x88]
 0x68,0x02,0x69,0x88
+# GFX12: s_lshl3_add_u32 s105, s104, s2          ; encoding: [0x68,0x02,0x69,0x88]
 
-# GFX12: s_lshl3_add_u32 s105, s1, s103          ; encoding: [0x01,0x67,0x69,0x88]
 0x01,0x67,0x69,0x88
+# GFX12: s_lshl3_add_u32 s105, s1, s103          ; encoding: [0x01,0x67,0x69,0x88]
 
-# GFX12: s_lshl3_add_u32 s105, s1, s2            ; encoding: [0x01,0x02,0x69,0x88]
 0x01,0x02,0x69,0x88
+# GFX12: s_lshl3_add_u32 s105, s1, s2            ; encoding: [0x01,0x02,0x69,0x88]
 
-# GFX12: s_lshl3_add_u32 vcc_hi, s1, s2          ; encoding: [0x01,0x02,0x6b,0x88]
 0x01,0x02,0x6b,0x88
+# GFX12: s_lshl3_add_u32 vcc_hi, s1, s2          ; encoding: [0x01,0x02,0x6b,0x88]
 
-# GFX12: s_lshl3_add_u32 vcc_lo, s1, s2          ; encoding: [0x01,0x02,0x6a,0x88]
 0x01,0x02,0x6a,0x88
+# GFX12: s_lshl3_add_u32 vcc_lo, s1, s2          ; encoding: [0x01,0x02,0x6a,0x88]
 
-# GFX12: s_lshl4_add_u32 exec_hi, s1, s2         ; encoding: [0x01,0x02,0xff,0x88]
 0x01,0x02,0xff,0x88
+# GFX12: s_lshl4_add_u32 exec_hi, s1, s2         ; encoding: [0x01,0x02,0xff,0x88]
 
-# GFX12: s_lshl4_add_u32 exec_lo, s1, s2         ; encoding: [0x01,0x02,0xfe,0x88]
 0x01,0x02,0xfe,0x88
+# GFX12: s_lshl4_add_u32 exec_lo, s1, s2         ; encoding: [0x01,0x02,0xfe,0x88]
 
-# GFX12: s_lshl4_add_u32 m0, s1, s2              ; encoding: [0x01,0x02,0xfd,0x88]
 0x01,0x02,0xfd,0x88
+# GFX12: s_lshl4_add_u32 m0, s1, s2              ; encoding: [0x01,0x02,0xfd,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, 0.5, s2             ; encoding: [0xf0,0x02,0x80,0x88]
 0xf0,0x02,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, 0.5, s2             ; encoding: [0xf0,0x02,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, 0, s2               ; encoding: [0x80,0x02,0x80,0x88]
 0x80,0x02,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, 0, s2               ; encoding: [0x80,0x02,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, 0x3f717273, s2      ; encoding: [0xff,0x02,0x80,0x88,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x80,0x88,0x73,0x72,0x71,0x3f
+# GFX12: s_lshl4_add_u32 s0, 0x3f717273, s2      ; encoding: [0xff,0x02,0x80,0x88,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_lshl4_add_u32 s0, 0xaf123456, s2      ; encoding: [0xff,0x02,0x80,0x88,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x80,0x88,0x56,0x34,0x12,0xaf
+# GFX12: s_lshl4_add_u32 s0, 0xaf123456, s2      ; encoding: [0xff,0x02,0x80,0x88,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_lshl4_add_u32 s0, -1, s2              ; encoding: [0xc1,0x02,0x80,0x88]
 0xc1,0x02,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, -1, s2              ; encoding: [0xc1,0x02,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, -4.0, s2            ; encoding: [0xf7,0x02,0x80,0x88]
 0xf7,0x02,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, -4.0, s2            ; encoding: [0xf7,0x02,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, exec_hi, s2         ; encoding: [0x7f,0x02,0x80,0x88]
 0x7f,0x02,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, exec_hi, s2         ; encoding: [0x7f,0x02,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, exec_lo, s2         ; encoding: [0x7e,0x02,0x80,0x88]
 0x7e,0x02,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, exec_lo, s2         ; encoding: [0x7e,0x02,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, m0, s2              ; encoding: [0x7d,0x02,0x80,0x88]
 0x7d,0x02,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, m0, s2              ; encoding: [0x7d,0x02,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, s104, s103          ; encoding: [0x68,0x67,0x80,0x88]
 0x68,0x67,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, s104, s103          ; encoding: [0x68,0x67,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, s104, s2            ; encoding: [0x68,0x02,0x80,0x88]
 0x68,0x02,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, s104, s2            ; encoding: [0x68,0x02,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, s1, 0.5             ; encoding: [0x01,0xf0,0x80,0x88]
 0x01,0xf0,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, s1, 0.5             ; encoding: [0x01,0xf0,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, s1, 0               ; encoding: [0x01,0x80,0x80,0x88]
 0x01,0x80,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, s1, 0               ; encoding: [0x01,0x80,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, s1, 0x3f717273      ; encoding: [0x01,0xff,0x80,0x88,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x80,0x88,0x73,0x72,0x71,0x3f
+# GFX12: s_lshl4_add_u32 s0, s1, 0x3f717273      ; encoding: [0x01,0xff,0x80,0x88,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_lshl4_add_u32 s0, s1, 0xaf123456      ; encoding: [0x01,0xff,0x80,0x88,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x80,0x88,0x56,0x34,0x12,0xaf
+# GFX12: s_lshl4_add_u32 s0, s1, 0xaf123456      ; encoding: [0x01,0xff,0x80,0x88,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_lshl4_add_u32 s0, s1, -1              ; encoding: [0x01,0xc1,0x80,0x88]
 0x01,0xc1,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, s1, -1              ; encoding: [0x01,0xc1,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, s1, -4.0            ; encoding: [0x01,0xf7,0x80,0x88]
 0x01,0xf7,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, s1, -4.0            ; encoding: [0x01,0xf7,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, s1, exec_hi         ; encoding: [0x01,0x7f,0x80,0x88]
 0x01,0x7f,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, s1, exec_hi         ; encoding: [0x01,0x7f,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, s1, exec_lo         ; encoding: [0x01,0x7e,0x80,0x88]
 0x01,0x7e,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, s1, exec_lo         ; encoding: [0x01,0x7e,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, s1, m0              ; encoding: [0x01,0x7d,0x80,0x88]
 0x01,0x7d,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, s1, m0              ; encoding: [0x01,0x7d,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, s1, s103            ; encoding: [0x01,0x67,0x80,0x88]
 0x01,0x67,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, s1, s103            ; encoding: [0x01,0x67,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, s1, s2              ; encoding: [0x01,0x02,0x80,0x88]
 0x01,0x02,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, s1, s2              ; encoding: [0x01,0x02,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, s1, vcc_hi          ; encoding: [0x01,0x6b,0x80,0x88]
 0x01,0x6b,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, s1, vcc_hi          ; encoding: [0x01,0x6b,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, s1, vcc_lo          ; encoding: [0x01,0x6a,0x80,0x88]
 0x01,0x6a,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, s1, vcc_lo          ; encoding: [0x01,0x6a,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, vcc_hi, s2          ; encoding: [0x6b,0x02,0x80,0x88]
 0x6b,0x02,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, vcc_hi, s2          ; encoding: [0x6b,0x02,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s0, vcc_lo, s2          ; encoding: [0x6a,0x02,0x80,0x88]
 0x6a,0x02,0x80,0x88
+# GFX12: s_lshl4_add_u32 s0, vcc_lo, s2          ; encoding: [0x6a,0x02,0x80,0x88]
 
-# GFX12: s_lshl4_add_u32 s105, s104, s103        ; encoding: [0x68,0x67,0xe9,0x88]
 0x68,0x67,0xe9,0x88
+# GFX12: s_lshl4_add_u32 s105, s104, s103        ; encoding: [0x68,0x67,0xe9,0x88]
 
-# GFX12: s_lshl4_add_u32 s105, s104, s2          ; encoding: [0x68,0x02,0xe9,0x88]
 0x68,0x02,0xe9,0x88
+# GFX12: s_lshl4_add_u32 s105, s104, s2          ; encoding: [0x68,0x02,0xe9,0x88]
 
-# GFX12: s_lshl4_add_u32 s105, s1, s103          ; encoding: [0x01,0x67,0xe9,0x88]
 0x01,0x67,0xe9,0x88
+# GFX12: s_lshl4_add_u32 s105, s1, s103          ; encoding: [0x01,0x67,0xe9,0x88]
 
-# GFX12: s_lshl4_add_u32 s105, s1, s2            ; encoding: [0x01,0x02,0xe9,0x88]
 0x01,0x02,0xe9,0x88
+# GFX12: s_lshl4_add_u32 s105, s1, s2            ; encoding: [0x01,0x02,0xe9,0x88]
 
-# GFX12: s_lshl4_add_u32 vcc_hi, s1, s2          ; encoding: [0x01,0x02,0xeb,0x88]
 0x01,0x02,0xeb,0x88
+# GFX12: s_lshl4_add_u32 vcc_hi, s1, s2          ; encoding: [0x01,0x02,0xeb,0x88]
 
-# GFX12: s_lshl4_add_u32 vcc_lo, s1, s2          ; encoding: [0x01,0x02,0xea,0x88]
 0x01,0x02,0xea,0x88
+# GFX12: s_lshl4_add_u32 vcc_lo, s1, s2          ; encoding: [0x01,0x02,0xea,0x88]
 
-# GFX12: s_lshl_b32 exec_hi, s1, s2              ; encoding: [0x01,0x02,0x7f,0x84]
 0x01,0x02,0x7f,0x84
+# GFX12: s_lshl_b32 exec_hi, s1, s2              ; encoding: [0x01,0x02,0x7f,0x84]
 
-# GFX12: s_lshl_b32 exec_lo, s1, s2              ; encoding: [0x01,0x02,0x7e,0x84]
 0x01,0x02,0x7e,0x84
+# GFX12: s_lshl_b32 exec_lo, s1, s2              ; encoding: [0x01,0x02,0x7e,0x84]
 
-# GFX12: s_lshl_b32 m0, s1, s2                   ; encoding: [0x01,0x02,0x7d,0x84]
 0x01,0x02,0x7d,0x84
+# GFX12: s_lshl_b32 m0, s1, s2                   ; encoding: [0x01,0x02,0x7d,0x84]
 
-# GFX12: s_lshl_b32 s0, 0.5, s2                  ; encoding: [0xf0,0x02,0x00,0x84]
 0xf0,0x02,0x00,0x84
+# GFX12: s_lshl_b32 s0, 0.5, s2                  ; encoding: [0xf0,0x02,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, 0, s2                    ; encoding: [0x80,0x02,0x00,0x84]
 0x80,0x02,0x00,0x84
+# GFX12: s_lshl_b32 s0, 0, s2                    ; encoding: [0x80,0x02,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, 0x3f717273, s2           ; encoding: [0xff,0x02,0x00,0x84,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x84,0x73,0x72,0x71,0x3f
+# GFX12: s_lshl_b32 s0, 0x3f717273, s2           ; encoding: [0xff,0x02,0x00,0x84,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_lshl_b32 s0, 0xaf123456, s2           ; encoding: [0xff,0x02,0x00,0x84,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x84,0x56,0x34,0x12,0xaf
+# GFX12: s_lshl_b32 s0, 0xaf123456, s2           ; encoding: [0xff,0x02,0x00,0x84,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_lshl_b32 s0, -1, s2                   ; encoding: [0xc1,0x02,0x00,0x84]
 0xc1,0x02,0x00,0x84
+# GFX12: s_lshl_b32 s0, -1, s2                   ; encoding: [0xc1,0x02,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, -4.0, s2                 ; encoding: [0xf7,0x02,0x00,0x84]
 0xf7,0x02,0x00,0x84
+# GFX12: s_lshl_b32 s0, -4.0, s2                 ; encoding: [0xf7,0x02,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, exec_hi, s2              ; encoding: [0x7f,0x02,0x00,0x84]
 0x7f,0x02,0x00,0x84
+# GFX12: s_lshl_b32 s0, exec_hi, s2              ; encoding: [0x7f,0x02,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, exec_lo, s2              ; encoding: [0x7e,0x02,0x00,0x84]
 0x7e,0x02,0x00,0x84
+# GFX12: s_lshl_b32 s0, exec_lo, s2              ; encoding: [0x7e,0x02,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, m0, s2                   ; encoding: [0x7d,0x02,0x00,0x84]
 0x7d,0x02,0x00,0x84
+# GFX12: s_lshl_b32 s0, m0, s2                   ; encoding: [0x7d,0x02,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, s104, s103               ; encoding: [0x68,0x67,0x00,0x84]
 0x68,0x67,0x00,0x84
+# GFX12: s_lshl_b32 s0, s104, s103               ; encoding: [0x68,0x67,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, s104, s2                 ; encoding: [0x68,0x02,0x00,0x84]
 0x68,0x02,0x00,0x84
+# GFX12: s_lshl_b32 s0, s104, s2                 ; encoding: [0x68,0x02,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, s1, 0.5                  ; encoding: [0x01,0xf0,0x00,0x84]
 0x01,0xf0,0x00,0x84
+# GFX12: s_lshl_b32 s0, s1, 0.5                  ; encoding: [0x01,0xf0,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, s1, 0                    ; encoding: [0x01,0x80,0x00,0x84]
 0x01,0x80,0x00,0x84
+# GFX12: s_lshl_b32 s0, s1, 0                    ; encoding: [0x01,0x80,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, s1, 0x3f717273           ; encoding: [0x01,0xff,0x00,0x84,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x84,0x73,0x72,0x71,0x3f
+# GFX12: s_lshl_b32 s0, s1, 0x3f717273           ; encoding: [0x01,0xff,0x00,0x84,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_lshl_b32 s0, s1, 0xaf123456           ; encoding: [0x01,0xff,0x00,0x84,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x84,0x56,0x34,0x12,0xaf
+# GFX12: s_lshl_b32 s0, s1, 0xaf123456           ; encoding: [0x01,0xff,0x00,0x84,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_lshl_b32 s0, s1, -1                   ; encoding: [0x01,0xc1,0x00,0x84]
 0x01,0xc1,0x00,0x84
+# GFX12: s_lshl_b32 s0, s1, -1                   ; encoding: [0x01,0xc1,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, s1, -4.0                 ; encoding: [0x01,0xf7,0x00,0x84]
 0x01,0xf7,0x00,0x84
+# GFX12: s_lshl_b32 s0, s1, -4.0                 ; encoding: [0x01,0xf7,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, s1, exec_hi              ; encoding: [0x01,0x7f,0x00,0x84]
 0x01,0x7f,0x00,0x84
+# GFX12: s_lshl_b32 s0, s1, exec_hi              ; encoding: [0x01,0x7f,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, s1, exec_lo              ; encoding: [0x01,0x7e,0x00,0x84]
 0x01,0x7e,0x00,0x84
+# GFX12: s_lshl_b32 s0, s1, exec_lo              ; encoding: [0x01,0x7e,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, s1, m0                   ; encoding: [0x01,0x7d,0x00,0x84]
 0x01,0x7d,0x00,0x84
+# GFX12: s_lshl_b32 s0, s1, m0                   ; encoding: [0x01,0x7d,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, s1, s103                 ; encoding: [0x01,0x67,0x00,0x84]
 0x01,0x67,0x00,0x84
+# GFX12: s_lshl_b32 s0, s1, s103                 ; encoding: [0x01,0x67,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, s1, s2                   ; encoding: [0x01,0x02,0x00,0x84]
 0x01,0x02,0x00,0x84
+# GFX12: s_lshl_b32 s0, s1, s2                   ; encoding: [0x01,0x02,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, s1, vcc_hi               ; encoding: [0x01,0x6b,0x00,0x84]
 0x01,0x6b,0x00,0x84
+# GFX12: s_lshl_b32 s0, s1, vcc_hi               ; encoding: [0x01,0x6b,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, s1, vcc_lo               ; encoding: [0x01,0x6a,0x00,0x84]
 0x01,0x6a,0x00,0x84
+# GFX12: s_lshl_b32 s0, s1, vcc_lo               ; encoding: [0x01,0x6a,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, vcc_hi, s2               ; encoding: [0x6b,0x02,0x00,0x84]
 0x6b,0x02,0x00,0x84
+# GFX12: s_lshl_b32 s0, vcc_hi, s2               ; encoding: [0x6b,0x02,0x00,0x84]
 
-# GFX12: s_lshl_b32 s0, vcc_lo, s2               ; encoding: [0x6a,0x02,0x00,0x84]
 0x6a,0x02,0x00,0x84
+# GFX12: s_lshl_b32 s0, vcc_lo, s2               ; encoding: [0x6a,0x02,0x00,0x84]
 
-# GFX12: s_lshl_b32 s105, s104, s103             ; encoding: [0x68,0x67,0x69,0x84]
 0x68,0x67,0x69,0x84
+# GFX12: s_lshl_b32 s105, s104, s103             ; encoding: [0x68,0x67,0x69,0x84]
 
-# GFX12: s_lshl_b32 s105, s104, s2               ; encoding: [0x68,0x02,0x69,0x84]
 0x68,0x02,0x69,0x84
+# GFX12: s_lshl_b32 s105, s104, s2               ; encoding: [0x68,0x02,0x69,0x84]
 
-# GFX12: s_lshl_b32 s105, s1, s103               ; encoding: [0x01,0x67,0x69,0x84]
 0x01,0x67,0x69,0x84
+# GFX12: s_lshl_b32 s105, s1, s103               ; encoding: [0x01,0x67,0x69,0x84]
 
-# GFX12: s_lshl_b32 s105, s1, s2                 ; encoding: [0x01,0x02,0x69,0x84]
 0x01,0x02,0x69,0x84
+# GFX12: s_lshl_b32 s105, s1, s2                 ; encoding: [0x01,0x02,0x69,0x84]
 
-# GFX12: s_lshl_b32 vcc_hi, s1, s2               ; encoding: [0x01,0x02,0x6b,0x84]
 0x01,0x02,0x6b,0x84
+# GFX12: s_lshl_b32 vcc_hi, s1, s2               ; encoding: [0x01,0x02,0x6b,0x84]
 
-# GFX12: s_lshl_b32 vcc_lo, s1, s2               ; encoding: [0x01,0x02,0x6a,0x84]
 0x01,0x02,0x6a,0x84
+# GFX12: s_lshl_b32 vcc_lo, s1, s2               ; encoding: [0x01,0x02,0x6a,0x84]
 
-# GFX12: s_lshl_b64 exec, s[2:3], s4             ; encoding: [0x02,0x04,0xfe,0x84]
 0x02,0x04,0xfe,0x84
+# GFX12: s_lshl_b64 exec, s[2:3], s4             ; encoding: [0x02,0x04,0xfe,0x84]
 
-# GFX12: s_lshl_b64 s[0:1], 0.5, s4              ; encoding: [0xf0,0x04,0x80,0x84]
 0xf0,0x04,0x80,0x84
+# GFX12: s_lshl_b64 s[0:1], 0.5, s4              ; encoding: [0xf0,0x04,0x80,0x84]
 
-# GFX12: s_lshl_b64 s[0:1], 0, s4                ; encoding: [0x80,0x04,0x80,0x84]
 0x80,0x04,0x80,0x84
+# GFX12: s_lshl_b64 s[0:1], 0, s4                ; encoding: [0x80,0x04,0x80,0x84]
 
-# GFX12: s_lshl_b64 s[0:1], 0x3f717273, s4       ; encoding: [0xff,0x04,0x80,0x84,0x73,0x72,0x71,0x3f]
 0xff,0x04,0x80,0x84,0x73,0x72,0x71,0x3f
+# GFX12: s_lshl_b64 s[0:1], 0x3f717273, s4       ; encoding: [0xff,0x04,0x80,0x84,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_lshl_b64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x84,0x56,0x34,0x12,0xaf]
 0xff,0x04,0x80,0x84,0x56,0x34,0x12,0xaf
+# GFX12: s_lshl_b64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x84,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_lshl_b64 s[0:1], -1, s4               ; encoding: [0xc1,0x04,0x80,0x84]
 0xc1,0x04,0x80,0x84
+# GFX12: s_lshl_b64 s[0:1], -1, s4               ; encoding: [0xc1,0x04,0x80,0x84]
 
-# GFX12: s_lshl_b64 s[0:1], -4.0, s4             ; encoding: [0xf7,0x04,0x80,0x84]
 0xf7,0x04,0x80,0x84
+# GFX12: s_lshl_b64 s[0:1], -4.0, s4             ; encoding: [0xf7,0x04,0x80,0x84]
 
-# GFX12: s_lshl_b64 s[0:1], exec, s4             ; encoding: [0x7e,0x04,0x80,0x84]
 0x7e,0x04,0x80,0x84
+# GFX12: s_lshl_b64 s[0:1], exec, s4             ; encoding: [0x7e,0x04,0x80,0x84]
 
-# GFX12: s_lshl_b64 s[0:1], s[102:103], s100     ; encoding: [0x66,0x64,0x80,0x84]
 0x66,0x64,0x80,0x84
+# GFX12: s_lshl_b64 s[0:1], s[102:103], s100     ; encoding: [0x66,0x64,0x80,0x84]
 
-# GFX12: s_lshl_b64 s[0:1], s[102:103], s4       ; encoding: [0x66,0x04,0x80,0x84]
 0x66,0x04,0x80,0x84
+# GFX12: s_lshl_b64 s[0:1], s[102:103], s4       ; encoding: [0x66,0x04,0x80,0x84]
 
-# GFX12: s_lshl_b64 s[0:1], s[2:3], 0.5          ; encoding: [0x02,0xf0,0x80,0x84]
 0x02,0xf0,0x80,0x84
+# GFX12: s_lshl_b64 s[0:1], s[2:3], 0.5          ; encoding: [0x02,0xf0,0x80,0x84]
 
-# GFX12: s_lshl_b64 s[0:1], s[2:3], 0            ; encoding: [0x02,0x80,0x80,0x84]
 0x02,0x80,0x80,0x84
+# GFX12: s_lshl_b64 s[0:1], s[2:3], 0            ; encoding: [0x02,0x80,0x80,0x84]
 
-# GFX12: s_lshl_b64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x84,0x73,0x72,0x71,0x3f]
 0x02,0xff,0x80,0x84,0x73,0x72,0x71,0x3f
+# GFX12: s_lshl_b64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x84,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_lshl_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x84,0x56,0x34,0x12,0xaf]
 0x02,0xff,0x80,0x84,0x56,0x34,0x12,0xaf
+# GFX12: s_lshl_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x84,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_lshl_b64 s[0:1], s[2:3], -1           ; encoding: [0x02,0xc1,0x80,0x84]
 0x02,0xc1,0x80,0x84
+# GFX12: s_lshl_b64 s[0:1], s[2:3], -1           ; encoding: [0x02,0xc1,0x80,0x84]
 
-# GFX12: s_lshl_b64 s[0:1], s[2:3], -4.0         ; encoding: [0x02,0xf7,0x80,0x84]
 0x02,0xf7,0x80,0x84
+# GFX12: s_lshl_b64 s[0:1], s[2:3], -4.0         ; encoding: [0x02,0xf7,0x80,0x84]
 
-# GFX12: s_lshl_b64 s[0:1], s[2:3], exec_lo      ; encoding: [0x02,0x7e,0x80,0x84]
 0x02,0x7e,0x80,0x84
+# GFX12: s_lshl_b64 s[0:1], s[2:3], exec_lo      ; encoding: [0x02,0x7e,0x80,0x84]
 
-# GFX12: s_lshl_b64 s[0:1], s[2:3], s100         ; encoding: [0x02,0x64,0x80,0x84]
 0x02,0x64,0x80,0x84
+# GFX12: s_lshl_b64 s[0:1], s[2:3], s100         ; encoding: [0x02,0x64,0x80,0x84]
 
-# GFX12: s_lshl_b64 s[0:1], s[2:3], s4           ; encoding: [0x02,0x04,0x80,0x84]
 0x02,0x04,0x80,0x84
+# GFX12: s_lshl_b64 s[0:1], s[2:3], s4           ; encoding: [0x02,0x04,0x80,0x84]
 
-# GFX12: s_lshl_b64 s[0:1], s[2:3], vcc_lo       ; encoding: [0x02,0x6a,0x80,0x84]
 0x02,0x6a,0x80,0x84
+# GFX12: s_lshl_b64 s[0:1], s[2:3], vcc_lo       ; encoding: [0x02,0x6a,0x80,0x84]
 
-# GFX12: s_lshl_b64 s[0:1], vcc, s4              ; encoding: [0x6a,0x04,0x80,0x84]
 0x6a,0x04,0x80,0x84
+# GFX12: s_lshl_b64 s[0:1], vcc, s4              ; encoding: [0x6a,0x04,0x80,0x84]
 
-# GFX12: s_lshl_b64 s[104:105], s[102:103], s100 ; encoding: [0x66,0x64,0xe8,0x84]
 0x66,0x64,0xe8,0x84
+# GFX12: s_lshl_b64 s[104:105], s[102:103], s100 ; encoding: [0x66,0x64,0xe8,0x84]
 
-# GFX12: s_lshl_b64 s[104:105], s[102:103], s4   ; encoding: [0x66,0x04,0xe8,0x84]
 0x66,0x04,0xe8,0x84
+# GFX12: s_lshl_b64 s[104:105], s[102:103], s4   ; encoding: [0x66,0x04,0xe8,0x84]
 
-# GFX12: s_lshl_b64 s[104:105], s[2:3], s100     ; encoding: [0x02,0x64,0xe8,0x84]
 0x02,0x64,0xe8,0x84
+# GFX12: s_lshl_b64 s[104:105], s[2:3], s100     ; encoding: [0x02,0x64,0xe8,0x84]
 
-# GFX12: s_lshl_b64 s[104:105], s[2:3], s4       ; encoding: [0x02,0x04,0xe8,0x84]
 0x02,0x04,0xe8,0x84
+# GFX12: s_lshl_b64 s[104:105], s[2:3], s4       ; encoding: [0x02,0x04,0xe8,0x84]
 
-# GFX12: s_lshl_b64 vcc, s[2:3], s4              ; encoding: [0x02,0x04,0xea,0x84]
 0x02,0x04,0xea,0x84
+# GFX12: s_lshl_b64 vcc, s[2:3], s4              ; encoding: [0x02,0x04,0xea,0x84]
 
-# GFX12: s_lshr_b32 exec_hi, s1, s2              ; encoding: [0x01,0x02,0x7f,0x85]
 0x01,0x02,0x7f,0x85
+# GFX12: s_lshr_b32 exec_hi, s1, s2              ; encoding: [0x01,0x02,0x7f,0x85]
 
-# GFX12: s_lshr_b32 exec_lo, s1, s2              ; encoding: [0x01,0x02,0x7e,0x85]
 0x01,0x02,0x7e,0x85
+# GFX12: s_lshr_b32 exec_lo, s1, s2              ; encoding: [0x01,0x02,0x7e,0x85]
 
-# GFX12: s_lshr_b32 m0, s1, s2                   ; encoding: [0x01,0x02,0x7d,0x85]
 0x01,0x02,0x7d,0x85
+# GFX12: s_lshr_b32 m0, s1, s2                   ; encoding: [0x01,0x02,0x7d,0x85]
 
-# GFX12: s_lshr_b32 s0, 0.5, s2                  ; encoding: [0xf0,0x02,0x00,0x85]
 0xf0,0x02,0x00,0x85
+# GFX12: s_lshr_b32 s0, 0.5, s2                  ; encoding: [0xf0,0x02,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, 0, s2                    ; encoding: [0x80,0x02,0x00,0x85]
 0x80,0x02,0x00,0x85
+# GFX12: s_lshr_b32 s0, 0, s2                    ; encoding: [0x80,0x02,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, 0x3f717273, s2           ; encoding: [0xff,0x02,0x00,0x85,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x85,0x73,0x72,0x71,0x3f
+# GFX12: s_lshr_b32 s0, 0x3f717273, s2           ; encoding: [0xff,0x02,0x00,0x85,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_lshr_b32 s0, 0xaf123456, s2           ; encoding: [0xff,0x02,0x00,0x85,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x85,0x56,0x34,0x12,0xaf
+# GFX12: s_lshr_b32 s0, 0xaf123456, s2           ; encoding: [0xff,0x02,0x00,0x85,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_lshr_b32 s0, -1, s2                   ; encoding: [0xc1,0x02,0x00,0x85]
 0xc1,0x02,0x00,0x85
+# GFX12: s_lshr_b32 s0, -1, s2                   ; encoding: [0xc1,0x02,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, -4.0, s2                 ; encoding: [0xf7,0x02,0x00,0x85]
 0xf7,0x02,0x00,0x85
+# GFX12: s_lshr_b32 s0, -4.0, s2                 ; encoding: [0xf7,0x02,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, exec_hi, s2              ; encoding: [0x7f,0x02,0x00,0x85]
 0x7f,0x02,0x00,0x85
+# GFX12: s_lshr_b32 s0, exec_hi, s2              ; encoding: [0x7f,0x02,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, exec_lo, s2              ; encoding: [0x7e,0x02,0x00,0x85]
 0x7e,0x02,0x00,0x85
+# GFX12: s_lshr_b32 s0, exec_lo, s2              ; encoding: [0x7e,0x02,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, m0, s2                   ; encoding: [0x7d,0x02,0x00,0x85]
 0x7d,0x02,0x00,0x85
+# GFX12: s_lshr_b32 s0, m0, s2                   ; encoding: [0x7d,0x02,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, s104, s103               ; encoding: [0x68,0x67,0x00,0x85]
 0x68,0x67,0x00,0x85
+# GFX12: s_lshr_b32 s0, s104, s103               ; encoding: [0x68,0x67,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, s104, s2                 ; encoding: [0x68,0x02,0x00,0x85]
 0x68,0x02,0x00,0x85
+# GFX12: s_lshr_b32 s0, s104, s2                 ; encoding: [0x68,0x02,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, s1, 0.5                  ; encoding: [0x01,0xf0,0x00,0x85]
 0x01,0xf0,0x00,0x85
+# GFX12: s_lshr_b32 s0, s1, 0.5                  ; encoding: [0x01,0xf0,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, s1, 0                    ; encoding: [0x01,0x80,0x00,0x85]
 0x01,0x80,0x00,0x85
+# GFX12: s_lshr_b32 s0, s1, 0                    ; encoding: [0x01,0x80,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, s1, 0x3f717273           ; encoding: [0x01,0xff,0x00,0x85,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x85,0x73,0x72,0x71,0x3f
+# GFX12: s_lshr_b32 s0, s1, 0x3f717273           ; encoding: [0x01,0xff,0x00,0x85,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_lshr_b32 s0, s1, 0xaf123456           ; encoding: [0x01,0xff,0x00,0x85,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x85,0x56,0x34,0x12,0xaf
+# GFX12: s_lshr_b32 s0, s1, 0xaf123456           ; encoding: [0x01,0xff,0x00,0x85,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_lshr_b32 s0, s1, -1                   ; encoding: [0x01,0xc1,0x00,0x85]
 0x01,0xc1,0x00,0x85
+# GFX12: s_lshr_b32 s0, s1, -1                   ; encoding: [0x01,0xc1,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, s1, -4.0                 ; encoding: [0x01,0xf7,0x00,0x85]
 0x01,0xf7,0x00,0x85
+# GFX12: s_lshr_b32 s0, s1, -4.0                 ; encoding: [0x01,0xf7,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, s1, exec_hi              ; encoding: [0x01,0x7f,0x00,0x85]
 0x01,0x7f,0x00,0x85
+# GFX12: s_lshr_b32 s0, s1, exec_hi              ; encoding: [0x01,0x7f,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, s1, exec_lo              ; encoding: [0x01,0x7e,0x00,0x85]
 0x01,0x7e,0x00,0x85
+# GFX12: s_lshr_b32 s0, s1, exec_lo              ; encoding: [0x01,0x7e,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, s1, m0                   ; encoding: [0x01,0x7d,0x00,0x85]
 0x01,0x7d,0x00,0x85
+# GFX12: s_lshr_b32 s0, s1, m0                   ; encoding: [0x01,0x7d,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, s1, s103                 ; encoding: [0x01,0x67,0x00,0x85]
 0x01,0x67,0x00,0x85
+# GFX12: s_lshr_b32 s0, s1, s103                 ; encoding: [0x01,0x67,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, s1, s2                   ; encoding: [0x01,0x02,0x00,0x85]
 0x01,0x02,0x00,0x85
+# GFX12: s_lshr_b32 s0, s1, s2                   ; encoding: [0x01,0x02,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, s1, vcc_hi               ; encoding: [0x01,0x6b,0x00,0x85]
 0x01,0x6b,0x00,0x85
+# GFX12: s_lshr_b32 s0, s1, vcc_hi               ; encoding: [0x01,0x6b,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, s1, vcc_lo               ; encoding: [0x01,0x6a,0x00,0x85]
 0x01,0x6a,0x00,0x85
+# GFX12: s_lshr_b32 s0, s1, vcc_lo               ; encoding: [0x01,0x6a,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, vcc_hi, s2               ; encoding: [0x6b,0x02,0x00,0x85]
 0x6b,0x02,0x00,0x85
+# GFX12: s_lshr_b32 s0, vcc_hi, s2               ; encoding: [0x6b,0x02,0x00,0x85]
 
-# GFX12: s_lshr_b32 s0, vcc_lo, s2               ; encoding: [0x6a,0x02,0x00,0x85]
 0x6a,0x02,0x00,0x85
+# GFX12: s_lshr_b32 s0, vcc_lo, s2               ; encoding: [0x6a,0x02,0x00,0x85]
 
-# GFX12: s_lshr_b32 s105, s104, s103             ; encoding: [0x68,0x67,0x69,0x85]
 0x68,0x67,0x69,0x85
+# GFX12: s_lshr_b32 s105, s104, s103             ; encoding: [0x68,0x67,0x69,0x85]
 
-# GFX12: s_lshr_b32 s105, s104, s2               ; encoding: [0x68,0x02,0x69,0x85]
 0x68,0x02,0x69,0x85
+# GFX12: s_lshr_b32 s105, s104, s2               ; encoding: [0x68,0x02,0x69,0x85]
 
-# GFX12: s_lshr_b32 s105, s1, s103               ; encoding: [0x01,0x67,0x69,0x85]
 0x01,0x67,0x69,0x85
+# GFX12: s_lshr_b32 s105, s1, s103               ; encoding: [0x01,0x67,0x69,0x85]
 
-# GFX12: s_lshr_b32 s105, s1, s2                 ; encoding: [0x01,0x02,0x69,0x85]
 0x01,0x02,0x69,0x85
+# GFX12: s_lshr_b32 s105, s1, s2                 ; encoding: [0x01,0x02,0x69,0x85]
 
-# GFX12: s_lshr_b32 vcc_hi, s1, s2               ; encoding: [0x01,0x02,0x6b,0x85]
 0x01,0x02,0x6b,0x85
+# GFX12: s_lshr_b32 vcc_hi, s1, s2               ; encoding: [0x01,0x02,0x6b,0x85]
 
-# GFX12: s_lshr_b32 vcc_lo, s1, s2               ; encoding: [0x01,0x02,0x6a,0x85]
 0x01,0x02,0x6a,0x85
+# GFX12: s_lshr_b32 vcc_lo, s1, s2               ; encoding: [0x01,0x02,0x6a,0x85]
 
-# GFX12: s_lshr_b64 exec, s[2:3], s4             ; encoding: [0x02,0x04,0xfe,0x85]
 0x02,0x04,0xfe,0x85
+# GFX12: s_lshr_b64 exec, s[2:3], s4             ; encoding: [0x02,0x04,0xfe,0x85]
 
-# GFX12: s_lshr_b64 s[0:1], 0.5, s4              ; encoding: [0xf0,0x04,0x80,0x85]
 0xf0,0x04,0x80,0x85
+# GFX12: s_lshr_b64 s[0:1], 0.5, s4              ; encoding: [0xf0,0x04,0x80,0x85]
 
-# GFX12: s_lshr_b64 s[0:1], 0, s4                ; encoding: [0x80,0x04,0x80,0x85]
 0x80,0x04,0x80,0x85
+# GFX12: s_lshr_b64 s[0:1], 0, s4                ; encoding: [0x80,0x04,0x80,0x85]
 
-# GFX12: s_lshr_b64 s[0:1], 0x3f717273, s4       ; encoding: [0xff,0x04,0x80,0x85,0x73,0x72,0x71,0x3f]
 0xff,0x04,0x80,0x85,0x73,0x72,0x71,0x3f
+# GFX12: s_lshr_b64 s[0:1], 0x3f717273, s4       ; encoding: [0xff,0x04,0x80,0x85,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_lshr_b64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x85,0x56,0x34,0x12,0xaf]
 0xff,0x04,0x80,0x85,0x56,0x34,0x12,0xaf
+# GFX12: s_lshr_b64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x85,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_lshr_b64 s[0:1], -1, s4               ; encoding: [0xc1,0x04,0x80,0x85]
 0xc1,0x04,0x80,0x85
+# GFX12: s_lshr_b64 s[0:1], -1, s4               ; encoding: [0xc1,0x04,0x80,0x85]
 
-# GFX12: s_lshr_b64 s[0:1], -4.0, s4             ; encoding: [0xf7,0x04,0x80,0x85]
 0xf7,0x04,0x80,0x85
+# GFX12: s_lshr_b64 s[0:1], -4.0, s4             ; encoding: [0xf7,0x04,0x80,0x85]
 
-# GFX12: s_lshr_b64 s[0:1], exec, s4             ; encoding: [0x7e,0x04,0x80,0x85]
 0x7e,0x04,0x80,0x85
+# GFX12: s_lshr_b64 s[0:1], exec, s4             ; encoding: [0x7e,0x04,0x80,0x85]
 
-# GFX12: s_lshr_b64 s[0:1], s[102:103], s100     ; encoding: [0x66,0x64,0x80,0x85]
 0x66,0x64,0x80,0x85
+# GFX12: s_lshr_b64 s[0:1], s[102:103], s100     ; encoding: [0x66,0x64,0x80,0x85]
 
-# GFX12: s_lshr_b64 s[0:1], s[102:103], s4       ; encoding: [0x66,0x04,0x80,0x85]
 0x66,0x04,0x80,0x85
+# GFX12: s_lshr_b64 s[0:1], s[102:103], s4       ; encoding: [0x66,0x04,0x80,0x85]
 
-# GFX12: s_lshr_b64 s[0:1], s[2:3], 0.5          ; encoding: [0x02,0xf0,0x80,0x85]
 0x02,0xf0,0x80,0x85
+# GFX12: s_lshr_b64 s[0:1], s[2:3], 0.5          ; encoding: [0x02,0xf0,0x80,0x85]
 
-# GFX12: s_lshr_b64 s[0:1], s[2:3], 0            ; encoding: [0x02,0x80,0x80,0x85]
 0x02,0x80,0x80,0x85
+# GFX12: s_lshr_b64 s[0:1], s[2:3], 0            ; encoding: [0x02,0x80,0x80,0x85]
 
-# GFX12: s_lshr_b64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x85,0x73,0x72,0x71,0x3f]
 0x02,0xff,0x80,0x85,0x73,0x72,0x71,0x3f
+# GFX12: s_lshr_b64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x85,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_lshr_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x85,0x56,0x34,0x12,0xaf]
 0x02,0xff,0x80,0x85,0x56,0x34,0x12,0xaf
+# GFX12: s_lshr_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x85,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_lshr_b64 s[0:1], s[2:3], -1           ; encoding: [0x02,0xc1,0x80,0x85]
 0x02,0xc1,0x80,0x85
+# GFX12: s_lshr_b64 s[0:1], s[2:3], -1           ; encoding: [0x02,0xc1,0x80,0x85]
 
-# GFX12: s_lshr_b64 s[0:1], s[2:3], -4.0         ; encoding: [0x02,0xf7,0x80,0x85]
 0x02,0xf7,0x80,0x85
+# GFX12: s_lshr_b64 s[0:1], s[2:3], -4.0         ; encoding: [0x02,0xf7,0x80,0x85]
 
-# GFX12: s_lshr_b64 s[0:1], s[2:3], exec_lo      ; encoding: [0x02,0x7e,0x80,0x85]
 0x02,0x7e,0x80,0x85
+# GFX12: s_lshr_b64 s[0:1], s[2:3], exec_lo      ; encoding: [0x02,0x7e,0x80,0x85]
 
-# GFX12: s_lshr_b64 s[0:1], s[2:3], s100         ; encoding: [0x02,0x64,0x80,0x85]
 0x02,0x64,0x80,0x85
+# GFX12: s_lshr_b64 s[0:1], s[2:3], s100         ; encoding: [0x02,0x64,0x80,0x85]
 
-# GFX12: s_lshr_b64 s[0:1], s[2:3], s4           ; encoding: [0x02,0x04,0x80,0x85]
 0x02,0x04,0x80,0x85
+# GFX12: s_lshr_b64 s[0:1], s[2:3], s4           ; encoding: [0x02,0x04,0x80,0x85]
 
-# GFX12: s_lshr_b64 s[0:1], s[2:3], vcc_lo       ; encoding: [0x02,0x6a,0x80,0x85]
 0x02,0x6a,0x80,0x85
+# GFX12: s_lshr_b64 s[0:1], s[2:3], vcc_lo       ; encoding: [0x02,0x6a,0x80,0x85]
 
-# GFX12: s_lshr_b64 s[0:1], vcc, s4              ; encoding: [0x6a,0x04,0x80,0x85]
 0x6a,0x04,0x80,0x85
+# GFX12: s_lshr_b64 s[0:1], vcc, s4              ; encoding: [0x6a,0x04,0x80,0x85]
 
-# GFX12: s_lshr_b64 s[104:105], s[102:103], s100 ; encoding: [0x66,0x64,0xe8,0x85]
 0x66,0x64,0xe8,0x85
+# GFX12: s_lshr_b64 s[104:105], s[102:103], s100 ; encoding: [0x66,0x64,0xe8,0x85]
 
-# GFX12: s_lshr_b64 s[104:105], s[102:103], s4   ; encoding: [0x66,0x04,0xe8,0x85]
 0x66,0x04,0xe8,0x85
+# GFX12: s_lshr_b64 s[104:105], s[102:103], s4   ; encoding: [0x66,0x04,0xe8,0x85]
 
-# GFX12: s_lshr_b64 s[104:105], s[2:3], s100     ; encoding: [0x02,0x64,0xe8,0x85]
 0x02,0x64,0xe8,0x85
+# GFX12: s_lshr_b64 s[104:105], s[2:3], s100     ; encoding: [0x02,0x64,0xe8,0x85]
 
-# GFX12: s_lshr_b64 s[104:105], s[2:3], s4       ; encoding: [0x02,0x04,0xe8,0x85]
 0x02,0x04,0xe8,0x85
+# GFX12: s_lshr_b64 s[104:105], s[2:3], s4       ; encoding: [0x02,0x04,0xe8,0x85]
 
-# GFX12: s_lshr_b64 vcc, s[2:3], s4              ; encoding: [0x02,0x04,0xea,0x85]
 0x02,0x04,0xea,0x85
+# GFX12: s_lshr_b64 vcc, s[2:3], s4              ; encoding: [0x02,0x04,0xea,0x85]
 
-# GFX12: s_max_i32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x8a]
 0x01,0x02,0x7f,0x8a
+# GFX12: s_max_i32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x8a]
 
-# GFX12: s_max_i32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x8a]
 0x01,0x02,0x7e,0x8a
+# GFX12: s_max_i32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x8a]
 
-# GFX12: s_max_i32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x8a]
 0x01,0x02,0x7d,0x8a
+# GFX12: s_max_i32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x8a]
 
-# GFX12: s_max_i32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x8a]
 0xf0,0x02,0x00,0x8a
+# GFX12: s_max_i32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x8a]
 0x80,0x02,0x00,0x8a
+# GFX12: s_max_i32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x8a,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x8a,0x73,0x72,0x71,0x3f
+# GFX12: s_max_i32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x8a,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_max_i32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x8a,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x8a,0x56,0x34,0x12,0xaf
+# GFX12: s_max_i32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x8a,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_max_i32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x8a]
 0xc1,0x02,0x00,0x8a
+# GFX12: s_max_i32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x8a]
 0xf7,0x02,0x00,0x8a
+# GFX12: s_max_i32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x8a]
 0x7f,0x02,0x00,0x8a
+# GFX12: s_max_i32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x8a]
 0x7e,0x02,0x00,0x8a
+# GFX12: s_max_i32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x8a]
 0x7d,0x02,0x00,0x8a
+# GFX12: s_max_i32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x8a]
 0x68,0x67,0x00,0x8a
+# GFX12: s_max_i32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x8a]
 0x68,0x02,0x00,0x8a
+# GFX12: s_max_i32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x8a]
 0x01,0xf0,0x00,0x8a
+# GFX12: s_max_i32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x8a]
 0x01,0x80,0x00,0x8a
+# GFX12: s_max_i32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x8a,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x8a,0x73,0x72,0x71,0x3f
+# GFX12: s_max_i32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x8a,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_max_i32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x8a,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x8a,0x56,0x34,0x12,0xaf
+# GFX12: s_max_i32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x8a,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_max_i32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x8a]
 0x01,0xc1,0x00,0x8a
+# GFX12: s_max_i32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x8a]
 0x01,0xf7,0x00,0x8a
+# GFX12: s_max_i32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x8a]
 0x01,0x7f,0x00,0x8a
+# GFX12: s_max_i32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x8a]
 0x01,0x7e,0x00,0x8a
+# GFX12: s_max_i32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x8a]
 0x01,0x7d,0x00,0x8a
+# GFX12: s_max_i32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x8a]
 0x01,0x67,0x00,0x8a
+# GFX12: s_max_i32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x8a]
 0x01,0x02,0x00,0x8a
+# GFX12: s_max_i32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x8a]
 0x01,0x6b,0x00,0x8a
+# GFX12: s_max_i32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x8a]
 0x01,0x6a,0x00,0x8a
+# GFX12: s_max_i32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x8a]
 0x6b,0x02,0x00,0x8a
+# GFX12: s_max_i32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x8a]
 
-# GFX12: s_max_i32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x8a]
 0x6a,0x02,0x00,0x8a
+# GFX12: s_max_i32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x8a]
 
-# GFX12: s_max_i32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x8a]
 0x68,0x67,0x69,0x8a
+# GFX12: s_max_i32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x8a]
 
-# GFX12: s_max_i32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x8a]
 0x68,0x02,0x69,0x8a
+# GFX12: s_max_i32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x8a]
 
-# GFX12: s_max_i32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x8a]
 0x01,0x67,0x69,0x8a
+# GFX12: s_max_i32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x8a]
 
-# GFX12: s_max_i32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x8a]
 0x01,0x02,0x69,0x8a
+# GFX12: s_max_i32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x8a]
 
-# GFX12: s_max_i32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x8a]
 0x01,0x02,0x6b,0x8a
+# GFX12: s_max_i32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x8a]
 
-# GFX12: s_max_i32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x8a]
 0x01,0x02,0x6a,0x8a
+# GFX12: s_max_i32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x8a]
 
-# GFX12: s_max_u32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0xff,0x8a]
 0x01,0x02,0xff,0x8a
+# GFX12: s_max_u32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0xff,0x8a]
 
-# GFX12: s_max_u32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0xfe,0x8a]
 0x01,0x02,0xfe,0x8a
+# GFX12: s_max_u32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0xfe,0x8a]
 
-# GFX12: s_max_u32 m0, s1, s2                    ; encoding: [0x01,0x02,0xfd,0x8a]
 0x01,0x02,0xfd,0x8a
+# GFX12: s_max_u32 m0, s1, s2                    ; encoding: [0x01,0x02,0xfd,0x8a]
 
-# GFX12: s_max_u32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x80,0x8a]
 0xf0,0x02,0x80,0x8a
+# GFX12: s_max_u32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, 0, s2                     ; encoding: [0x80,0x02,0x80,0x8a]
 0x80,0x02,0x80,0x8a
+# GFX12: s_max_u32 s0, 0, s2                     ; encoding: [0x80,0x02,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x80,0x8a,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x80,0x8a,0x73,0x72,0x71,0x3f
+# GFX12: s_max_u32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x80,0x8a,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_max_u32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x80,0x8a,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x80,0x8a,0x56,0x34,0x12,0xaf
+# GFX12: s_max_u32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x80,0x8a,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_max_u32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x80,0x8a]
 0xc1,0x02,0x80,0x8a
+# GFX12: s_max_u32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x80,0x8a]
 0xf7,0x02,0x80,0x8a
+# GFX12: s_max_u32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x80,0x8a]
 0x7f,0x02,0x80,0x8a
+# GFX12: s_max_u32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x80,0x8a]
 0x7e,0x02,0x80,0x8a
+# GFX12: s_max_u32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x80,0x8a]
 0x7d,0x02,0x80,0x8a
+# GFX12: s_max_u32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, s104, s103                ; encoding: [0x68,0x67,0x80,0x8a]
 0x68,0x67,0x80,0x8a
+# GFX12: s_max_u32 s0, s104, s103                ; encoding: [0x68,0x67,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, s104, s2                  ; encoding: [0x68,0x02,0x80,0x8a]
 0x68,0x02,0x80,0x8a
+# GFX12: s_max_u32 s0, s104, s2                  ; encoding: [0x68,0x02,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x80,0x8a]
 0x01,0xf0,0x80,0x8a
+# GFX12: s_max_u32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, s1, 0                     ; encoding: [0x01,0x80,0x80,0x8a]
 0x01,0x80,0x80,0x8a
+# GFX12: s_max_u32 s0, s1, 0                     ; encoding: [0x01,0x80,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x80,0x8a,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x80,0x8a,0x73,0x72,0x71,0x3f
+# GFX12: s_max_u32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x80,0x8a,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_max_u32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x80,0x8a,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x80,0x8a,0x56,0x34,0x12,0xaf
+# GFX12: s_max_u32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x80,0x8a,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_max_u32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x80,0x8a]
 0x01,0xc1,0x80,0x8a
+# GFX12: s_max_u32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x80,0x8a]
 0x01,0xf7,0x80,0x8a
+# GFX12: s_max_u32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x80,0x8a]
 0x01,0x7f,0x80,0x8a
+# GFX12: s_max_u32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x80,0x8a]
 0x01,0x7e,0x80,0x8a
+# GFX12: s_max_u32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x80,0x8a]
 0x01,0x7d,0x80,0x8a
+# GFX12: s_max_u32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, s1, s103                  ; encoding: [0x01,0x67,0x80,0x8a]
 0x01,0x67,0x80,0x8a
+# GFX12: s_max_u32 s0, s1, s103                  ; encoding: [0x01,0x67,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, s1, s2                    ; encoding: [0x01,0x02,0x80,0x8a]
 0x01,0x02,0x80,0x8a
+# GFX12: s_max_u32 s0, s1, s2                    ; encoding: [0x01,0x02,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x80,0x8a]
 0x01,0x6b,0x80,0x8a
+# GFX12: s_max_u32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x80,0x8a]
 0x01,0x6a,0x80,0x8a
+# GFX12: s_max_u32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x80,0x8a]
 0x6b,0x02,0x80,0x8a
+# GFX12: s_max_u32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x80,0x8a]
 
-# GFX12: s_max_u32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x80,0x8a]
 0x6a,0x02,0x80,0x8a
+# GFX12: s_max_u32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x80,0x8a]
 
-# GFX12: s_max_u32 s105, s104, s103              ; encoding: [0x68,0x67,0xe9,0x8a]
 0x68,0x67,0xe9,0x8a
+# GFX12: s_max_u32 s105, s104, s103              ; encoding: [0x68,0x67,0xe9,0x8a]
 
-# GFX12: s_max_u32 s105, s104, s2                ; encoding: [0x68,0x02,0xe9,0x8a]
 0x68,0x02,0xe9,0x8a
+# GFX12: s_max_u32 s105, s104, s2                ; encoding: [0x68,0x02,0xe9,0x8a]
 
-# GFX12: s_max_u32 s105, s1, s103                ; encoding: [0x01,0x67,0xe9,0x8a]
 0x01,0x67,0xe9,0x8a
+# GFX12: s_max_u32 s105, s1, s103                ; encoding: [0x01,0x67,0xe9,0x8a]
 
-# GFX12: s_max_u32 s105, s1, s2                  ; encoding: [0x01,0x02,0xe9,0x8a]
 0x01,0x02,0xe9,0x8a
+# GFX12: s_max_u32 s105, s1, s2                  ; encoding: [0x01,0x02,0xe9,0x8a]
 
-# GFX12: s_max_u32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0xeb,0x8a]
 0x01,0x02,0xeb,0x8a
+# GFX12: s_max_u32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0xeb,0x8a]
 
-# GFX12: s_max_u32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0xea,0x8a]
 0x01,0x02,0xea,0x8a
+# GFX12: s_max_u32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0xea,0x8a]
 
-# GFX12: s_min_i32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x89]
 0x01,0x02,0x7f,0x89
+# GFX12: s_min_i32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x89]
 
-# GFX12: s_min_i32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x89]
 0x01,0x02,0x7e,0x89
+# GFX12: s_min_i32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x89]
 
-# GFX12: s_min_i32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x89]
 0x01,0x02,0x7d,0x89
+# GFX12: s_min_i32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x89]
 
-# GFX12: s_min_i32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x89]
 0xf0,0x02,0x00,0x89
+# GFX12: s_min_i32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x89]
 
-# GFX12: s_min_i32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x89]
 0x80,0x02,0x00,0x89
+# GFX12: s_min_i32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x89]
 
-# GFX12: s_min_i32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x89,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x89,0x73,0x72,0x71,0x3f
+# GFX12: s_min_i32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x89,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_min_i32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x89,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x89,0x56,0x34,0x12,0xaf
+# GFX12: s_min_i32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x89,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_min_i32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x89]
 0xc1,0x02,0x00,0x89
+# GFX12: s_min_i32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x89]
 
-# GFX12: s_min_i32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x89]
 0xf7,0x02,0x00,0x89
+# GFX12: s_min_i32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x89]
 
-# GFX12: s_min_i32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x89]
 0x7f,0x02,0x00,0x89
+# GFX12: s_min_i32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x89]
 
-# GFX12: s_min_i32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x89]
 0x7e,0x02,0x00,0x89
+# GFX12: s_min_i32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x89]
 
-# GFX12: s_min_i32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x89]
 0x7d,0x02,0x00,0x89
+# GFX12: s_min_i32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x89]
 
-# GFX12: s_min_i32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x89]
 0x68,0x67,0x00,0x89
+# GFX12: s_min_i32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x89]
 
-# GFX12: s_min_i32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x89]
 0x68,0x02,0x00,0x89
+# GFX12: s_min_i32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x89]
 
-# GFX12: s_min_i32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x89]
 0x01,0xf0,0x00,0x89
+# GFX12: s_min_i32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x89]
 
-# GFX12: s_min_i32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x89]
 0x01,0x80,0x00,0x89
+# GFX12: s_min_i32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x89]
 
-# GFX12: s_min_i32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x89,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x89,0x73,0x72,0x71,0x3f
+# GFX12: s_min_i32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x89,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_min_i32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x89,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x89,0x56,0x34,0x12,0xaf
+# GFX12: s_min_i32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x89,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_min_i32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x89]
 0x01,0xc1,0x00,0x89
+# GFX12: s_min_i32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x89]
 
-# GFX12: s_min_i32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x89]
 0x01,0xf7,0x00,0x89
+# GFX12: s_min_i32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x89]
 
-# GFX12: s_min_i32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x89]
 0x01,0x7f,0x00,0x89
+# GFX12: s_min_i32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x89]
 
-# GFX12: s_min_i32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x89]
 0x01,0x7e,0x00,0x89
+# GFX12: s_min_i32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x89]
 
-# GFX12: s_min_i32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x89]
 0x01,0x7d,0x00,0x89
+# GFX12: s_min_i32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x89]
 
-# GFX12: s_min_i32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x89]
 0x01,0x67,0x00,0x89
+# GFX12: s_min_i32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x89]
 
-# GFX12: s_min_i32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x89]
 0x01,0x02,0x00,0x89
+# GFX12: s_min_i32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x89]
 
-# GFX12: s_min_i32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x89]
 0x01,0x6b,0x00,0x89
+# GFX12: s_min_i32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x89]
 
-# GFX12: s_min_i32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x89]
 0x01,0x6a,0x00,0x89
+# GFX12: s_min_i32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x89]
 
-# GFX12: s_min_i32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x89]
 0x6b,0x02,0x00,0x89
+# GFX12: s_min_i32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x89]
 
-# GFX12: s_min_i32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x89]
 0x6a,0x02,0x00,0x89
+# GFX12: s_min_i32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x89]
 
-# GFX12: s_min_i32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x89]
 0x68,0x67,0x69,0x89
+# GFX12: s_min_i32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x89]
 
-# GFX12: s_min_i32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x89]
 0x68,0x02,0x69,0x89
+# GFX12: s_min_i32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x89]
 
-# GFX12: s_min_i32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x89]
 0x01,0x67,0x69,0x89
+# GFX12: s_min_i32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x89]
 
-# GFX12: s_min_i32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x89]
 0x01,0x02,0x69,0x89
+# GFX12: s_min_i32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x89]
 
-# GFX12: s_min_i32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x89]
 0x01,0x02,0x6b,0x89
+# GFX12: s_min_i32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x89]
 
-# GFX12: s_min_i32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x89]
 0x01,0x02,0x6a,0x89
+# GFX12: s_min_i32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x89]
 
-# GFX12: s_min_u32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0xff,0x89]
 0x01,0x02,0xff,0x89
+# GFX12: s_min_u32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0xff,0x89]
 
-# GFX12: s_min_u32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0xfe,0x89]
 0x01,0x02,0xfe,0x89
+# GFX12: s_min_u32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0xfe,0x89]
 
-# GFX12: s_min_u32 m0, s1, s2                    ; encoding: [0x01,0x02,0xfd,0x89]
 0x01,0x02,0xfd,0x89
+# GFX12: s_min_u32 m0, s1, s2                    ; encoding: [0x01,0x02,0xfd,0x89]
 
-# GFX12: s_min_u32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x80,0x89]
 0xf0,0x02,0x80,0x89
+# GFX12: s_min_u32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x80,0x89]
 
-# GFX12: s_min_u32 s0, 0, s2                     ; encoding: [0x80,0x02,0x80,0x89]
 0x80,0x02,0x80,0x89
+# GFX12: s_min_u32 s0, 0, s2                     ; encoding: [0x80,0x02,0x80,0x89]
 
-# GFX12: s_min_u32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x80,0x89,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x80,0x89,0x73,0x72,0x71,0x3f
+# GFX12: s_min_u32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x80,0x89,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_min_u32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x80,0x89,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x80,0x89,0x56,0x34,0x12,0xaf
+# GFX12: s_min_u32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x80,0x89,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_min_u32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x80,0x89]
 0xc1,0x02,0x80,0x89
+# GFX12: s_min_u32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x80,0x89]
 
-# GFX12: s_min_u32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x80,0x89]
 0xf7,0x02,0x80,0x89
+# GFX12: s_min_u32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x80,0x89]
 
-# GFX12: s_min_u32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x80,0x89]
 0x7f,0x02,0x80,0x89
+# GFX12: s_min_u32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x80,0x89]
 
-# GFX12: s_min_u32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x80,0x89]
 0x7e,0x02,0x80,0x89
+# GFX12: s_min_u32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x80,0x89]
 
-# GFX12: s_min_u32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x80,0x89]
 0x7d,0x02,0x80,0x89
+# GFX12: s_min_u32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x80,0x89]
 
-# GFX12: s_min_u32 s0, s104, s103                ; encoding: [0x68,0x67,0x80,0x89]
 0x68,0x67,0x80,0x89
+# GFX12: s_min_u32 s0, s104, s103                ; encoding: [0x68,0x67,0x80,0x89]
 
-# GFX12: s_min_u32 s0, s104, s2                  ; encoding: [0x68,0x02,0x80,0x89]
 0x68,0x02,0x80,0x89
+# GFX12: s_min_u32 s0, s104, s2                  ; encoding: [0x68,0x02,0x80,0x89]
 
-# GFX12: s_min_u32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x80,0x89]
 0x01,0xf0,0x80,0x89
+# GFX12: s_min_u32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x80,0x89]
 
-# GFX12: s_min_u32 s0, s1, 0                     ; encoding: [0x01,0x80,0x80,0x89]
 0x01,0x80,0x80,0x89
+# GFX12: s_min_u32 s0, s1, 0                     ; encoding: [0x01,0x80,0x80,0x89]
 
-# GFX12: s_min_u32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x80,0x89,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x80,0x89,0x73,0x72,0x71,0x3f
+# GFX12: s_min_u32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x80,0x89,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_min_u32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x80,0x89,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x80,0x89,0x56,0x34,0x12,0xaf
+# GFX12: s_min_u32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x80,0x89,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_min_u32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x80,0x89]
 0x01,0xc1,0x80,0x89
+# GFX12: s_min_u32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x80,0x89]
 
-# GFX12: s_min_u32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x80,0x89]
 0x01,0xf7,0x80,0x89
+# GFX12: s_min_u32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x80,0x89]
 
-# GFX12: s_min_u32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x80,0x89]
 0x01,0x7f,0x80,0x89
+# GFX12: s_min_u32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x80,0x89]
 
-# GFX12: s_min_u32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x80,0x89]
 0x01,0x7e,0x80,0x89
+# GFX12: s_min_u32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x80,0x89]
 
-# GFX12: s_min_u32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x80,0x89]
 0x01,0x7d,0x80,0x89
+# GFX12: s_min_u32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x80,0x89]
 
-# GFX12: s_min_u32 s0, s1, s103                  ; encoding: [0x01,0x67,0x80,0x89]
 0x01,0x67,0x80,0x89
+# GFX12: s_min_u32 s0, s1, s103                  ; encoding: [0x01,0x67,0x80,0x89]
 
-# GFX12: s_min_u32 s0, s1, s2                    ; encoding: [0x01,0x02,0x80,0x89]
 0x01,0x02,0x80,0x89
+# GFX12: s_min_u32 s0, s1, s2                    ; encoding: [0x01,0x02,0x80,0x89]
 
-# GFX12: s_min_u32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x80,0x89]
 0x01,0x6b,0x80,0x89
+# GFX12: s_min_u32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x80,0x89]
 
-# GFX12: s_min_u32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x80,0x89]
 0x01,0x6a,0x80,0x89
+# GFX12: s_min_u32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x80,0x89]
 
-# GFX12: s_min_u32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x80,0x89]
 0x6b,0x02,0x80,0x89
+# GFX12: s_min_u32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x80,0x89]
 
-# GFX12: s_min_u32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x80,0x89]
 0x6a,0x02,0x80,0x89
+# GFX12: s_min_u32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x80,0x89]
 
-# GFX12: s_min_u32 s105, s104, s103              ; encoding: [0x68,0x67,0xe9,0x89]
 0x68,0x67,0xe9,0x89
+# GFX12: s_min_u32 s105, s104, s103              ; encoding: [0x68,0x67,0xe9,0x89]
 
-# GFX12: s_min_u32 s105, s104, s2                ; encoding: [0x68,0x02,0xe9,0x89]
 0x68,0x02,0xe9,0x89
+# GFX12: s_min_u32 s105, s104, s2                ; encoding: [0x68,0x02,0xe9,0x89]
 
-# GFX12: s_min_u32 s105, s1, s103                ; encoding: [0x01,0x67,0xe9,0x89]
 0x01,0x67,0xe9,0x89
+# GFX12: s_min_u32 s105, s1, s103                ; encoding: [0x01,0x67,0xe9,0x89]
 
-# GFX12: s_min_u32 s105, s1, s2                  ; encoding: [0x01,0x02,0xe9,0x89]
 0x01,0x02,0xe9,0x89
+# GFX12: s_min_u32 s105, s1, s2                  ; encoding: [0x01,0x02,0xe9,0x89]
 
-# GFX12: s_min_u32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0xeb,0x89]
 0x01,0x02,0xeb,0x89
+# GFX12: s_min_u32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0xeb,0x89]
 
-# GFX12: s_min_u32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0xea,0x89]
 0x01,0x02,0xea,0x89
+# GFX12: s_min_u32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0xea,0x89]
 
-# GFX12: s_mul_hi_i32 exec_hi, s1, s2            ; encoding: [0x01,0x02,0x7f,0x97]
 0x01,0x02,0x7f,0x97
+# GFX12: s_mul_hi_i32 exec_hi, s1, s2            ; encoding: [0x01,0x02,0x7f,0x97]
 
-# GFX12: s_mul_hi_i32 exec_lo, s1, s2            ; encoding: [0x01,0x02,0x7e,0x97]
 0x01,0x02,0x7e,0x97
+# GFX12: s_mul_hi_i32 exec_lo, s1, s2            ; encoding: [0x01,0x02,0x7e,0x97]
 
-# GFX12: s_mul_hi_i32 m0, s1, s2                 ; encoding: [0x01,0x02,0x7d,0x97]
 0x01,0x02,0x7d,0x97
+# GFX12: s_mul_hi_i32 m0, s1, s2                 ; encoding: [0x01,0x02,0x7d,0x97]
 
-# GFX12: s_mul_hi_i32 s0, 0.5, s2                ; encoding: [0xf0,0x02,0x00,0x97]
 0xf0,0x02,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, 0.5, s2                ; encoding: [0xf0,0x02,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, 0, s2                  ; encoding: [0x80,0x02,0x00,0x97]
 0x80,0x02,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, 0, s2                  ; encoding: [0x80,0x02,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, 0x3f717273, s2         ; encoding: [0xff,0x02,0x00,0x97,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x97,0x73,0x72,0x71,0x3f
+# GFX12: s_mul_hi_i32 s0, 0x3f717273, s2         ; encoding: [0xff,0x02,0x00,0x97,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_mul_hi_i32 s0, 0xaf123456, s2         ; encoding: [0xff,0x02,0x00,0x97,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x97,0x56,0x34,0x12,0xaf
+# GFX12: s_mul_hi_i32 s0, 0xaf123456, s2         ; encoding: [0xff,0x02,0x00,0x97,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_mul_hi_i32 s0, -1, s2                 ; encoding: [0xc1,0x02,0x00,0x97]
 0xc1,0x02,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, -1, s2                 ; encoding: [0xc1,0x02,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, -4.0, s2               ; encoding: [0xf7,0x02,0x00,0x97]
 0xf7,0x02,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, -4.0, s2               ; encoding: [0xf7,0x02,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, exec_hi, s2            ; encoding: [0x7f,0x02,0x00,0x97]
 0x7f,0x02,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, exec_hi, s2            ; encoding: [0x7f,0x02,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, exec_lo, s2            ; encoding: [0x7e,0x02,0x00,0x97]
 0x7e,0x02,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, exec_lo, s2            ; encoding: [0x7e,0x02,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, m0, s2                 ; encoding: [0x7d,0x02,0x00,0x97]
 0x7d,0x02,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, m0, s2                 ; encoding: [0x7d,0x02,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, s104, s103             ; encoding: [0x68,0x67,0x00,0x97]
 0x68,0x67,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, s104, s103             ; encoding: [0x68,0x67,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, s104, s2               ; encoding: [0x68,0x02,0x00,0x97]
 0x68,0x02,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, s104, s2               ; encoding: [0x68,0x02,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, s1, 0.5                ; encoding: [0x01,0xf0,0x00,0x97]
 0x01,0xf0,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, s1, 0.5                ; encoding: [0x01,0xf0,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, s1, 0                  ; encoding: [0x01,0x80,0x00,0x97]
 0x01,0x80,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, s1, 0                  ; encoding: [0x01,0x80,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, s1, 0x3f717273         ; encoding: [0x01,0xff,0x00,0x97,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x97,0x73,0x72,0x71,0x3f
+# GFX12: s_mul_hi_i32 s0, s1, 0x3f717273         ; encoding: [0x01,0xff,0x00,0x97,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_mul_hi_i32 s0, s1, 0xaf123456         ; encoding: [0x01,0xff,0x00,0x97,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x97,0x56,0x34,0x12,0xaf
+# GFX12: s_mul_hi_i32 s0, s1, 0xaf123456         ; encoding: [0x01,0xff,0x00,0x97,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_mul_hi_i32 s0, s1, -1                 ; encoding: [0x01,0xc1,0x00,0x97]
 0x01,0xc1,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, s1, -1                 ; encoding: [0x01,0xc1,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, s1, -4.0               ; encoding: [0x01,0xf7,0x00,0x97]
 0x01,0xf7,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, s1, -4.0               ; encoding: [0x01,0xf7,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, s1, exec_hi            ; encoding: [0x01,0x7f,0x00,0x97]
 0x01,0x7f,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, s1, exec_hi            ; encoding: [0x01,0x7f,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, s1, exec_lo            ; encoding: [0x01,0x7e,0x00,0x97]
 0x01,0x7e,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, s1, exec_lo            ; encoding: [0x01,0x7e,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, s1, m0                 ; encoding: [0x01,0x7d,0x00,0x97]
 0x01,0x7d,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, s1, m0                 ; encoding: [0x01,0x7d,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, s1, s103               ; encoding: [0x01,0x67,0x00,0x97]
 0x01,0x67,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, s1, s103               ; encoding: [0x01,0x67,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, s1, s2                 ; encoding: [0x01,0x02,0x00,0x97]
 0x01,0x02,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, s1, s2                 ; encoding: [0x01,0x02,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, s1, vcc_hi             ; encoding: [0x01,0x6b,0x00,0x97]
 0x01,0x6b,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, s1, vcc_hi             ; encoding: [0x01,0x6b,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, s1, vcc_lo             ; encoding: [0x01,0x6a,0x00,0x97]
 0x01,0x6a,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, s1, vcc_lo             ; encoding: [0x01,0x6a,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, vcc_hi, s2             ; encoding: [0x6b,0x02,0x00,0x97]
 0x6b,0x02,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, vcc_hi, s2             ; encoding: [0x6b,0x02,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s0, vcc_lo, s2             ; encoding: [0x6a,0x02,0x00,0x97]
 0x6a,0x02,0x00,0x97
+# GFX12: s_mul_hi_i32 s0, vcc_lo, s2             ; encoding: [0x6a,0x02,0x00,0x97]
 
-# GFX12: s_mul_hi_i32 s105, s104, s103           ; encoding: [0x68,0x67,0x69,0x97]
 0x68,0x67,0x69,0x97
+# GFX12: s_mul_hi_i32 s105, s104, s103           ; encoding: [0x68,0x67,0x69,0x97]
 
-# GFX12: s_mul_hi_i32 s105, s104, s2             ; encoding: [0x68,0x02,0x69,0x97]
 0x68,0x02,0x69,0x97
+# GFX12: s_mul_hi_i32 s105, s104, s2             ; encoding: [0x68,0x02,0x69,0x97]
 
-# GFX12: s_mul_hi_i32 s105, s1, s103             ; encoding: [0x01,0x67,0x69,0x97]
 0x01,0x67,0x69,0x97
+# GFX12: s_mul_hi_i32 s105, s1, s103             ; encoding: [0x01,0x67,0x69,0x97]
 
-# GFX12: s_mul_hi_i32 s105, s1, s2               ; encoding: [0x01,0x02,0x69,0x97]
 0x01,0x02,0x69,0x97
+# GFX12: s_mul_hi_i32 s105, s1, s2               ; encoding: [0x01,0x02,0x69,0x97]
 
-# GFX12: s_mul_hi_i32 vcc_hi, s1, s2             ; encoding: [0x01,0x02,0x6b,0x97]
 0x01,0x02,0x6b,0x97
+# GFX12: s_mul_hi_i32 vcc_hi, s1, s2             ; encoding: [0x01,0x02,0x6b,0x97]
 
-# GFX12: s_mul_hi_i32 vcc_lo, s1, s2             ; encoding: [0x01,0x02,0x6a,0x97]
 0x01,0x02,0x6a,0x97
+# GFX12: s_mul_hi_i32 vcc_lo, s1, s2             ; encoding: [0x01,0x02,0x6a,0x97]
 
-# GFX12: s_mul_hi_u32 exec_hi, s1, s2            ; encoding: [0x01,0x02,0xff,0x96]
 0x01,0x02,0xff,0x96
+# GFX12: s_mul_hi_u32 exec_hi, s1, s2            ; encoding: [0x01,0x02,0xff,0x96]
 
-# GFX12: s_mul_hi_u32 exec_lo, s1, s2            ; encoding: [0x01,0x02,0xfe,0x96]
 0x01,0x02,0xfe,0x96
+# GFX12: s_mul_hi_u32 exec_lo, s1, s2            ; encoding: [0x01,0x02,0xfe,0x96]
 
-# GFX12: s_mul_hi_u32 m0, s1, s2                 ; encoding: [0x01,0x02,0xfd,0x96]
 0x01,0x02,0xfd,0x96
+# GFX12: s_mul_hi_u32 m0, s1, s2                 ; encoding: [0x01,0x02,0xfd,0x96]
 
-# GFX12: s_mul_hi_u32 s0, 0.5, s2                ; encoding: [0xf0,0x02,0x80,0x96]
 0xf0,0x02,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, 0.5, s2                ; encoding: [0xf0,0x02,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, 0, s2                  ; encoding: [0x80,0x02,0x80,0x96]
 0x80,0x02,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, 0, s2                  ; encoding: [0x80,0x02,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, 0x3f717273, s2         ; encoding: [0xff,0x02,0x80,0x96,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x80,0x96,0x73,0x72,0x71,0x3f
+# GFX12: s_mul_hi_u32 s0, 0x3f717273, s2         ; encoding: [0xff,0x02,0x80,0x96,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_mul_hi_u32 s0, 0xaf123456, s2         ; encoding: [0xff,0x02,0x80,0x96,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x80,0x96,0x56,0x34,0x12,0xaf
+# GFX12: s_mul_hi_u32 s0, 0xaf123456, s2         ; encoding: [0xff,0x02,0x80,0x96,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_mul_hi_u32 s0, -1, s2                 ; encoding: [0xc1,0x02,0x80,0x96]
 0xc1,0x02,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, -1, s2                 ; encoding: [0xc1,0x02,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, -4.0, s2               ; encoding: [0xf7,0x02,0x80,0x96]
 0xf7,0x02,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, -4.0, s2               ; encoding: [0xf7,0x02,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, exec_hi, s2            ; encoding: [0x7f,0x02,0x80,0x96]
 0x7f,0x02,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, exec_hi, s2            ; encoding: [0x7f,0x02,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, exec_lo, s2            ; encoding: [0x7e,0x02,0x80,0x96]
 0x7e,0x02,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, exec_lo, s2            ; encoding: [0x7e,0x02,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, m0, s2                 ; encoding: [0x7d,0x02,0x80,0x96]
 0x7d,0x02,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, m0, s2                 ; encoding: [0x7d,0x02,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, s104, s103             ; encoding: [0x68,0x67,0x80,0x96]
 0x68,0x67,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, s104, s103             ; encoding: [0x68,0x67,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, s104, s2               ; encoding: [0x68,0x02,0x80,0x96]
 0x68,0x02,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, s104, s2               ; encoding: [0x68,0x02,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, s1, 0.5                ; encoding: [0x01,0xf0,0x80,0x96]
 0x01,0xf0,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, s1, 0.5                ; encoding: [0x01,0xf0,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, s1, 0                  ; encoding: [0x01,0x80,0x80,0x96]
 0x01,0x80,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, s1, 0                  ; encoding: [0x01,0x80,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, s1, 0x3f717273         ; encoding: [0x01,0xff,0x80,0x96,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x80,0x96,0x73,0x72,0x71,0x3f
+# GFX12: s_mul_hi_u32 s0, s1, 0x3f717273         ; encoding: [0x01,0xff,0x80,0x96,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_mul_hi_u32 s0, s1, 0xaf123456         ; encoding: [0x01,0xff,0x80,0x96,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x80,0x96,0x56,0x34,0x12,0xaf
+# GFX12: s_mul_hi_u32 s0, s1, 0xaf123456         ; encoding: [0x01,0xff,0x80,0x96,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_mul_hi_u32 s0, s1, -1                 ; encoding: [0x01,0xc1,0x80,0x96]
 0x01,0xc1,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, s1, -1                 ; encoding: [0x01,0xc1,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, s1, -4.0               ; encoding: [0x01,0xf7,0x80,0x96]
 0x01,0xf7,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, s1, -4.0               ; encoding: [0x01,0xf7,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, s1, exec_hi            ; encoding: [0x01,0x7f,0x80,0x96]
 0x01,0x7f,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, s1, exec_hi            ; encoding: [0x01,0x7f,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, s1, exec_lo            ; encoding: [0x01,0x7e,0x80,0x96]
 0x01,0x7e,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, s1, exec_lo            ; encoding: [0x01,0x7e,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, s1, m0                 ; encoding: [0x01,0x7d,0x80,0x96]
 0x01,0x7d,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, s1, m0                 ; encoding: [0x01,0x7d,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, s1, s103               ; encoding: [0x01,0x67,0x80,0x96]
 0x01,0x67,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, s1, s103               ; encoding: [0x01,0x67,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, s1, s2                 ; encoding: [0x01,0x02,0x80,0x96]
 0x01,0x02,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, s1, s2                 ; encoding: [0x01,0x02,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, s1, vcc_hi             ; encoding: [0x01,0x6b,0x80,0x96]
 0x01,0x6b,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, s1, vcc_hi             ; encoding: [0x01,0x6b,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, s1, vcc_lo             ; encoding: [0x01,0x6a,0x80,0x96]
 0x01,0x6a,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, s1, vcc_lo             ; encoding: [0x01,0x6a,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, vcc_hi, s2             ; encoding: [0x6b,0x02,0x80,0x96]
 0x6b,0x02,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, vcc_hi, s2             ; encoding: [0x6b,0x02,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s0, vcc_lo, s2             ; encoding: [0x6a,0x02,0x80,0x96]
 0x6a,0x02,0x80,0x96
+# GFX12: s_mul_hi_u32 s0, vcc_lo, s2             ; encoding: [0x6a,0x02,0x80,0x96]
 
-# GFX12: s_mul_hi_u32 s105, s104, s103           ; encoding: [0x68,0x67,0xe9,0x96]
 0x68,0x67,0xe9,0x96
+# GFX12: s_mul_hi_u32 s105, s104, s103           ; encoding: [0x68,0x67,0xe9,0x96]
 
-# GFX12: s_mul_hi_u32 s105, s104, s2             ; encoding: [0x68,0x02,0xe9,0x96]
 0x68,0x02,0xe9,0x96
+# GFX12: s_mul_hi_u32 s105, s104, s2             ; encoding: [0x68,0x02,0xe9,0x96]
 
-# GFX12: s_mul_hi_u32 s105, s1, s103             ; encoding: [0x01,0x67,0xe9,0x96]
 0x01,0x67,0xe9,0x96
+# GFX12: s_mul_hi_u32 s105, s1, s103             ; encoding: [0x01,0x67,0xe9,0x96]
 
-# GFX12: s_mul_hi_u32 s105, s1, s2               ; encoding: [0x01,0x02,0xe9,0x96]
 0x01,0x02,0xe9,0x96
+# GFX12: s_mul_hi_u32 s105, s1, s2               ; encoding: [0x01,0x02,0xe9,0x96]
 
-# GFX12: s_mul_hi_u32 vcc_hi, s1, s2             ; encoding: [0x01,0x02,0xeb,0x96]
 0x01,0x02,0xeb,0x96
+# GFX12: s_mul_hi_u32 vcc_hi, s1, s2             ; encoding: [0x01,0x02,0xeb,0x96]
 
-# GFX12: s_mul_hi_u32 vcc_lo, s1, s2             ; encoding: [0x01,0x02,0xea,0x96]
 0x01,0x02,0xea,0x96
+# GFX12: s_mul_hi_u32 vcc_lo, s1, s2             ; encoding: [0x01,0x02,0xea,0x96]
 
-# GFX12: s_mul_i32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x96]
 0x01,0x02,0x7f,0x96
+# GFX12: s_mul_i32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x96]
 
-# GFX12: s_mul_i32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x96]
 0x01,0x02,0x7e,0x96
+# GFX12: s_mul_i32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x96]
 
-# GFX12: s_mul_i32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x96]
 0x01,0x02,0x7d,0x96
+# GFX12: s_mul_i32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x96]
 
-# GFX12: s_mul_i32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x96]
 0xf0,0x02,0x00,0x96
+# GFX12: s_mul_i32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x96]
 0x80,0x02,0x00,0x96
+# GFX12: s_mul_i32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x96,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x96,0x73,0x72,0x71,0x3f
+# GFX12: s_mul_i32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x96,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_mul_i32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x96,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x96,0x56,0x34,0x12,0xaf
+# GFX12: s_mul_i32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x96,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_mul_i32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x96]
 0xc1,0x02,0x00,0x96
+# GFX12: s_mul_i32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x96]
 0xf7,0x02,0x00,0x96
+# GFX12: s_mul_i32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x96]
 0x7f,0x02,0x00,0x96
+# GFX12: s_mul_i32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x96]
 0x7e,0x02,0x00,0x96
+# GFX12: s_mul_i32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x96]
 0x7d,0x02,0x00,0x96
+# GFX12: s_mul_i32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x96]
 0x68,0x67,0x00,0x96
+# GFX12: s_mul_i32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x96]
 0x68,0x02,0x00,0x96
+# GFX12: s_mul_i32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x96]
 0x01,0xf0,0x00,0x96
+# GFX12: s_mul_i32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x96]
 0x01,0x80,0x00,0x96
+# GFX12: s_mul_i32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x96,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x96,0x73,0x72,0x71,0x3f
+# GFX12: s_mul_i32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x96,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_mul_i32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x96,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x96,0x56,0x34,0x12,0xaf
+# GFX12: s_mul_i32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x96,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_mul_i32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x96]
 0x01,0xc1,0x00,0x96
+# GFX12: s_mul_i32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x96]
 0x01,0xf7,0x00,0x96
+# GFX12: s_mul_i32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x96]
 0x01,0x7f,0x00,0x96
+# GFX12: s_mul_i32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x96]
 0x01,0x7e,0x00,0x96
+# GFX12: s_mul_i32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x96]
 0x01,0x7d,0x00,0x96
+# GFX12: s_mul_i32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x96]
 0x01,0x67,0x00,0x96
+# GFX12: s_mul_i32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x96]
 0x01,0x02,0x00,0x96
+# GFX12: s_mul_i32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x96]
 0x01,0x6b,0x00,0x96
+# GFX12: s_mul_i32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x96]
 0x01,0x6a,0x00,0x96
+# GFX12: s_mul_i32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x96]
 0x6b,0x02,0x00,0x96
+# GFX12: s_mul_i32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x96]
 
-# GFX12: s_mul_i32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x96]
 0x6a,0x02,0x00,0x96
+# GFX12: s_mul_i32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x96]
 
-# GFX12: s_mul_i32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x96]
 0x68,0x67,0x69,0x96
+# GFX12: s_mul_i32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x96]
 
-# GFX12: s_mul_i32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x96]
 0x68,0x02,0x69,0x96
+# GFX12: s_mul_i32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x96]
 
-# GFX12: s_mul_i32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x96]
 0x01,0x67,0x69,0x96
+# GFX12: s_mul_i32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x96]
 
-# GFX12: s_mul_i32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x96]
 0x01,0x02,0x69,0x96
+# GFX12: s_mul_i32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x96]
 
-# GFX12: s_mul_i32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x96]
 0x01,0x02,0x6b,0x96
+# GFX12: s_mul_i32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x96]
 
-# GFX12: s_mul_i32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x96]
 0x01,0x02,0x6a,0x96
+# GFX12: s_mul_i32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x96]
 
-# GFX12: s_nand_b32 exec_hi, s1, s2              ; encoding: [0x01,0x02,0x7f,0x8e]
 0x01,0x02,0x7f,0x8e
+# GFX12: s_nand_b32 exec_hi, s1, s2              ; encoding: [0x01,0x02,0x7f,0x8e]
 
-# GFX12: s_nand_b32 exec_lo, s1, s2              ; encoding: [0x01,0x02,0x7e,0x8e]
 0x01,0x02,0x7e,0x8e
+# GFX12: s_nand_b32 exec_lo, s1, s2              ; encoding: [0x01,0x02,0x7e,0x8e]
 
-# GFX12: s_nand_b32 m0, s1, s2                   ; encoding: [0x01,0x02,0x7d,0x8e]
 0x01,0x02,0x7d,0x8e
+# GFX12: s_nand_b32 m0, s1, s2                   ; encoding: [0x01,0x02,0x7d,0x8e]
 
-# GFX12: s_nand_b32 s0, 0.5, s2                  ; encoding: [0xf0,0x02,0x00,0x8e]
 0xf0,0x02,0x00,0x8e
+# GFX12: s_nand_b32 s0, 0.5, s2                  ; encoding: [0xf0,0x02,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, 0, s2                    ; encoding: [0x80,0x02,0x00,0x8e]
 0x80,0x02,0x00,0x8e
+# GFX12: s_nand_b32 s0, 0, s2                    ; encoding: [0x80,0x02,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, 0x3f717273, s2           ; encoding: [0xff,0x02,0x00,0x8e,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x8e,0x73,0x72,0x71,0x3f
+# GFX12: s_nand_b32 s0, 0x3f717273, s2           ; encoding: [0xff,0x02,0x00,0x8e,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_nand_b32 s0, 0xaf123456, s2           ; encoding: [0xff,0x02,0x00,0x8e,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x8e,0x56,0x34,0x12,0xaf
+# GFX12: s_nand_b32 s0, 0xaf123456, s2           ; encoding: [0xff,0x02,0x00,0x8e,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_nand_b32 s0, -1, s2                   ; encoding: [0xc1,0x02,0x00,0x8e]
 0xc1,0x02,0x00,0x8e
+# GFX12: s_nand_b32 s0, -1, s2                   ; encoding: [0xc1,0x02,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, -4.0, s2                 ; encoding: [0xf7,0x02,0x00,0x8e]
 0xf7,0x02,0x00,0x8e
+# GFX12: s_nand_b32 s0, -4.0, s2                 ; encoding: [0xf7,0x02,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, exec_hi, s2              ; encoding: [0x7f,0x02,0x00,0x8e]
 0x7f,0x02,0x00,0x8e
+# GFX12: s_nand_b32 s0, exec_hi, s2              ; encoding: [0x7f,0x02,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, exec_lo, s2              ; encoding: [0x7e,0x02,0x00,0x8e]
 0x7e,0x02,0x00,0x8e
+# GFX12: s_nand_b32 s0, exec_lo, s2              ; encoding: [0x7e,0x02,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, m0, s2                   ; encoding: [0x7d,0x02,0x00,0x8e]
 0x7d,0x02,0x00,0x8e
+# GFX12: s_nand_b32 s0, m0, s2                   ; encoding: [0x7d,0x02,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, s104, s103               ; encoding: [0x68,0x67,0x00,0x8e]
 0x68,0x67,0x00,0x8e
+# GFX12: s_nand_b32 s0, s104, s103               ; encoding: [0x68,0x67,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, s104, s2                 ; encoding: [0x68,0x02,0x00,0x8e]
 0x68,0x02,0x00,0x8e
+# GFX12: s_nand_b32 s0, s104, s2                 ; encoding: [0x68,0x02,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, s1, 0.5                  ; encoding: [0x01,0xf0,0x00,0x8e]
 0x01,0xf0,0x00,0x8e
+# GFX12: s_nand_b32 s0, s1, 0.5                  ; encoding: [0x01,0xf0,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, s1, 0                    ; encoding: [0x01,0x80,0x00,0x8e]
 0x01,0x80,0x00,0x8e
+# GFX12: s_nand_b32 s0, s1, 0                    ; encoding: [0x01,0x80,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, s1, 0x3f717273           ; encoding: [0x01,0xff,0x00,0x8e,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x8e,0x73,0x72,0x71,0x3f
+# GFX12: s_nand_b32 s0, s1, 0x3f717273           ; encoding: [0x01,0xff,0x00,0x8e,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_nand_b32 s0, s1, 0xaf123456           ; encoding: [0x01,0xff,0x00,0x8e,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x8e,0x56,0x34,0x12,0xaf
+# GFX12: s_nand_b32 s0, s1, 0xaf123456           ; encoding: [0x01,0xff,0x00,0x8e,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_nand_b32 s0, s1, -1                   ; encoding: [0x01,0xc1,0x00,0x8e]
 0x01,0xc1,0x00,0x8e
+# GFX12: s_nand_b32 s0, s1, -1                   ; encoding: [0x01,0xc1,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, s1, -4.0                 ; encoding: [0x01,0xf7,0x00,0x8e]
 0x01,0xf7,0x00,0x8e
+# GFX12: s_nand_b32 s0, s1, -4.0                 ; encoding: [0x01,0xf7,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, s1, exec_hi              ; encoding: [0x01,0x7f,0x00,0x8e]
 0x01,0x7f,0x00,0x8e
+# GFX12: s_nand_b32 s0, s1, exec_hi              ; encoding: [0x01,0x7f,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, s1, exec_lo              ; encoding: [0x01,0x7e,0x00,0x8e]
 0x01,0x7e,0x00,0x8e
+# GFX12: s_nand_b32 s0, s1, exec_lo              ; encoding: [0x01,0x7e,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, s1, m0                   ; encoding: [0x01,0x7d,0x00,0x8e]
 0x01,0x7d,0x00,0x8e
+# GFX12: s_nand_b32 s0, s1, m0                   ; encoding: [0x01,0x7d,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, s1, s103                 ; encoding: [0x01,0x67,0x00,0x8e]
 0x01,0x67,0x00,0x8e
+# GFX12: s_nand_b32 s0, s1, s103                 ; encoding: [0x01,0x67,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, s1, s2                   ; encoding: [0x01,0x02,0x00,0x8e]
 0x01,0x02,0x00,0x8e
+# GFX12: s_nand_b32 s0, s1, s2                   ; encoding: [0x01,0x02,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, s1, vcc_hi               ; encoding: [0x01,0x6b,0x00,0x8e]
 0x01,0x6b,0x00,0x8e
+# GFX12: s_nand_b32 s0, s1, vcc_hi               ; encoding: [0x01,0x6b,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, s1, vcc_lo               ; encoding: [0x01,0x6a,0x00,0x8e]
 0x01,0x6a,0x00,0x8e
+# GFX12: s_nand_b32 s0, s1, vcc_lo               ; encoding: [0x01,0x6a,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, vcc_hi, s2               ; encoding: [0x6b,0x02,0x00,0x8e]
 0x6b,0x02,0x00,0x8e
+# GFX12: s_nand_b32 s0, vcc_hi, s2               ; encoding: [0x6b,0x02,0x00,0x8e]
 
-# GFX12: s_nand_b32 s0, vcc_lo, s2               ; encoding: [0x6a,0x02,0x00,0x8e]
 0x6a,0x02,0x00,0x8e
+# GFX12: s_nand_b32 s0, vcc_lo, s2               ; encoding: [0x6a,0x02,0x00,0x8e]
 
-# GFX12: s_nand_b32 s105, s104, s103             ; encoding: [0x68,0x67,0x69,0x8e]
 0x68,0x67,0x69,0x8e
+# GFX12: s_nand_b32 s105, s104, s103             ; encoding: [0x68,0x67,0x69,0x8e]
 
-# GFX12: s_nand_b32 s105, s104, s2               ; encoding: [0x68,0x02,0x69,0x8e]
 0x68,0x02,0x69,0x8e
+# GFX12: s_nand_b32 s105, s104, s2               ; encoding: [0x68,0x02,0x69,0x8e]
 
-# GFX12: s_nand_b32 s105, s1, s103               ; encoding: [0x01,0x67,0x69,0x8e]
 0x01,0x67,0x69,0x8e
+# GFX12: s_nand_b32 s105, s1, s103               ; encoding: [0x01,0x67,0x69,0x8e]
 
-# GFX12: s_nand_b32 s105, s1, s2                 ; encoding: [0x01,0x02,0x69,0x8e]
 0x01,0x02,0x69,0x8e
+# GFX12: s_nand_b32 s105, s1, s2                 ; encoding: [0x01,0x02,0x69,0x8e]
 
-# GFX12: s_nand_b32 vcc_hi, s1, s2               ; encoding: [0x01,0x02,0x6b,0x8e]
 0x01,0x02,0x6b,0x8e
+# GFX12: s_nand_b32 vcc_hi, s1, s2               ; encoding: [0x01,0x02,0x6b,0x8e]
 
-# GFX12: s_nand_b32 vcc_lo, s1, s2               ; encoding: [0x01,0x02,0x6a,0x8e]
 0x01,0x02,0x6a,0x8e
+# GFX12: s_nand_b32 vcc_lo, s1, s2               ; encoding: [0x01,0x02,0x6a,0x8e]
 
-# GFX12: s_nand_b64 exec, s[2:3], s[4:5]         ; encoding: [0x02,0x04,0xfe,0x8e]
 0x02,0x04,0xfe,0x8e
+# GFX12: s_nand_b64 exec, s[2:3], s[4:5]         ; encoding: [0x02,0x04,0xfe,0x8e]
 
-# GFX12: s_nand_b64 s[0:1], 0.5, s[4:5]          ; encoding: [0xf0,0x04,0x80,0x8e]
 0xf0,0x04,0x80,0x8e
+# GFX12: s_nand_b64 s[0:1], 0.5, s[4:5]          ; encoding: [0xf0,0x04,0x80,0x8e]
 
-# GFX12: s_nand_b64 s[0:1], 0, s[4:5]            ; encoding: [0x80,0x04,0x80,0x8e]
 0x80,0x04,0x80,0x8e
+# GFX12: s_nand_b64 s[0:1], 0, s[4:5]            ; encoding: [0x80,0x04,0x80,0x8e]
 
-# GFX12: s_nand_b64 s[0:1], 0x3f717273, s[4:5]   ; encoding: [0xff,0x04,0x80,0x8e,0x73,0x72,0x71,0x3f]
 0xff,0x04,0x80,0x8e,0x73,0x72,0x71,0x3f
+# GFX12: s_nand_b64 s[0:1], 0x3f717273, s[4:5]   ; encoding: [0xff,0x04,0x80,0x8e,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_nand_b64 s[0:1], 0xaf123456, s[4:5]   ; encoding: [0xff,0x04,0x80,0x8e,0x56,0x34,0x12,0xaf]
 0xff,0x04,0x80,0x8e,0x56,0x34,0x12,0xaf
+# GFX12: s_nand_b64 s[0:1], 0xaf123456, s[4:5]   ; encoding: [0xff,0x04,0x80,0x8e,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_nand_b64 s[0:1], -1, s[4:5]           ; encoding: [0xc1,0x04,0x80,0x8e]
 0xc1,0x04,0x80,0x8e
+# GFX12: s_nand_b64 s[0:1], -1, s[4:5]           ; encoding: [0xc1,0x04,0x80,0x8e]
 
-# GFX12: s_nand_b64 s[0:1], -4.0, s[4:5]         ; encoding: [0xf7,0x04,0x80,0x8e]
 0xf7,0x04,0x80,0x8e
+# GFX12: s_nand_b64 s[0:1], -4.0, s[4:5]         ; encoding: [0xf7,0x04,0x80,0x8e]
 
-# GFX12: s_nand_b64 s[0:1], exec, s[4:5]         ; encoding: [0x7e,0x04,0x80,0x8e]
 0x7e,0x04,0x80,0x8e
+# GFX12: s_nand_b64 s[0:1], exec, s[4:5]         ; encoding: [0x7e,0x04,0x80,0x8e]
 
-# GFX12: s_nand_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x8e]
 0x66,0x64,0x80,0x8e
+# GFX12: s_nand_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x8e]
 
-# GFX12: s_nand_b64 s[0:1], s[102:103], s[4:5]   ; encoding: [0x66,0x04,0x80,0x8e]
 0x66,0x04,0x80,0x8e
+# GFX12: s_nand_b64 s[0:1], s[102:103], s[4:5]   ; encoding: [0x66,0x04,0x80,0x8e]
 
-# GFX12: s_nand_b64 s[0:1], s[2:3], 0.5          ; encoding: [0x02,0xf0,0x80,0x8e]
 0x02,0xf0,0x80,0x8e
+# GFX12: s_nand_b64 s[0:1], s[2:3], 0.5          ; encoding: [0x02,0xf0,0x80,0x8e]
 
-# GFX12: s_nand_b64 s[0:1], s[2:3], 0            ; encoding: [0x02,0x80,0x80,0x8e]
 0x02,0x80,0x80,0x8e
+# GFX12: s_nand_b64 s[0:1], s[2:3], 0            ; encoding: [0x02,0x80,0x80,0x8e]
 
-# GFX12: s_nand_b64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x8e,0x73,0x72,0x71,0x3f]
 0x02,0xff,0x80,0x8e,0x73,0x72,0x71,0x3f
+# GFX12: s_nand_b64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x8e,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_nand_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x8e,0x56,0x34,0x12,0xaf]
 0x02,0xff,0x80,0x8e,0x56,0x34,0x12,0xaf
+# GFX12: s_nand_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x8e,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_nand_b64 s[0:1], s[2:3], -1           ; encoding: [0x02,0xc1,0x80,0x8e]
 0x02,0xc1,0x80,0x8e
+# GFX12: s_nand_b64 s[0:1], s[2:3], -1           ; encoding: [0x02,0xc1,0x80,0x8e]
 
-# GFX12: s_nand_b64 s[0:1], s[2:3], -4.0         ; encoding: [0x02,0xf7,0x80,0x8e]
 0x02,0xf7,0x80,0x8e
+# GFX12: s_nand_b64 s[0:1], s[2:3], -4.0         ; encoding: [0x02,0xf7,0x80,0x8e]
 
-# GFX12: s_nand_b64 s[0:1], s[2:3], exec         ; encoding: [0x02,0x7e,0x80,0x8e]
 0x02,0x7e,0x80,0x8e
+# GFX12: s_nand_b64 s[0:1], s[2:3], exec         ; encoding: [0x02,0x7e,0x80,0x8e]
 
-# GFX12: s_nand_b64 s[0:1], s[2:3], s[100:101]   ; encoding: [0x02,0x64,0x80,0x8e]
 0x02,0x64,0x80,0x8e
+# GFX12: s_nand_b64 s[0:1], s[2:3], s[100:101]   ; encoding: [0x02,0x64,0x80,0x8e]
 
-# GFX12: s_nand_b64 s[0:1], s[2:3], s[4:5]       ; encoding: [0x02,0x04,0x80,0x8e]
 0x02,0x04,0x80,0x8e
+# GFX12: s_nand_b64 s[0:1], s[2:3], s[4:5]       ; encoding: [0x02,0x04,0x80,0x8e]
 
-# GFX12: s_nand_b64 s[0:1], s[2:3], vcc          ; encoding: [0x02,0x6a,0x80,0x8e]
 0x02,0x6a,0x80,0x8e
+# GFX12: s_nand_b64 s[0:1], s[2:3], vcc          ; encoding: [0x02,0x6a,0x80,0x8e]
 
-# GFX12: s_nand_b64 s[0:1], vcc, s[4:5]          ; encoding: [0x6a,0x04,0x80,0x8e]
 0x6a,0x04,0x80,0x8e
+# GFX12: s_nand_b64 s[0:1], vcc, s[4:5]          ; encoding: [0x6a,0x04,0x80,0x8e]
 
-# GFX12: s_nand_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x8e]
 0x66,0x64,0xe8,0x8e
+# GFX12: s_nand_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x8e]
 
-# GFX12: s_nand_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x8e]
 0x66,0x04,0xe8,0x8e
+# GFX12: s_nand_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x8e]
 
-# GFX12: s_nand_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x8e]
 0x02,0x64,0xe8,0x8e
+# GFX12: s_nand_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x8e]
 
-# GFX12: s_nand_b64 s[104:105], s[2:3], s[4:5]   ; encoding: [0x02,0x04,0xe8,0x8e]
 0x02,0x04,0xe8,0x8e
+# GFX12: s_nand_b64 s[104:105], s[2:3], s[4:5]   ; encoding: [0x02,0x04,0xe8,0x8e]
 
-# GFX12: s_nand_b64 vcc, s[2:3], s[4:5]          ; encoding: [0x02,0x04,0xea,0x8e]
 0x02,0x04,0xea,0x8e
+# GFX12: s_nand_b64 vcc, s[2:3], s[4:5]          ; encoding: [0x02,0x04,0xea,0x8e]
 
-# GFX12: s_nor_b32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x8f]
 0x01,0x02,0x7f,0x8f
+# GFX12: s_nor_b32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x8f]
 
-# GFX12: s_nor_b32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x8f]
 0x01,0x02,0x7e,0x8f
+# GFX12: s_nor_b32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x8f]
 
-# GFX12: s_nor_b32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x8f]
 0x01,0x02,0x7d,0x8f
+# GFX12: s_nor_b32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x8f]
 
-# GFX12: s_nor_b32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x8f]
 0xf0,0x02,0x00,0x8f
+# GFX12: s_nor_b32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x8f]
 0x80,0x02,0x00,0x8f
+# GFX12: s_nor_b32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x8f,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x8f,0x73,0x72,0x71,0x3f
+# GFX12: s_nor_b32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x8f,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_nor_b32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x8f,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x8f,0x56,0x34,0x12,0xaf
+# GFX12: s_nor_b32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x8f,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_nor_b32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x8f]
 0xc1,0x02,0x00,0x8f
+# GFX12: s_nor_b32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x8f]
 0xf7,0x02,0x00,0x8f
+# GFX12: s_nor_b32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x8f]
 0x7f,0x02,0x00,0x8f
+# GFX12: s_nor_b32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x8f]
 0x7e,0x02,0x00,0x8f
+# GFX12: s_nor_b32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x8f]
 0x7d,0x02,0x00,0x8f
+# GFX12: s_nor_b32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x8f]
 0x68,0x67,0x00,0x8f
+# GFX12: s_nor_b32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x8f]
 0x68,0x02,0x00,0x8f
+# GFX12: s_nor_b32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x8f]
 0x01,0xf0,0x00,0x8f
+# GFX12: s_nor_b32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x8f]
 0x01,0x80,0x00,0x8f
+# GFX12: s_nor_b32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x8f,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x8f,0x73,0x72,0x71,0x3f
+# GFX12: s_nor_b32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x8f,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_nor_b32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x8f,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x8f,0x56,0x34,0x12,0xaf
+# GFX12: s_nor_b32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x8f,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_nor_b32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x8f]
 0x01,0xc1,0x00,0x8f
+# GFX12: s_nor_b32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x8f]
 0x01,0xf7,0x00,0x8f
+# GFX12: s_nor_b32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x8f]
 0x01,0x7f,0x00,0x8f
+# GFX12: s_nor_b32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x8f]
 0x01,0x7e,0x00,0x8f
+# GFX12: s_nor_b32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x8f]
 0x01,0x7d,0x00,0x8f
+# GFX12: s_nor_b32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x8f]
 0x01,0x67,0x00,0x8f
+# GFX12: s_nor_b32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x8f]
 0x01,0x02,0x00,0x8f
+# GFX12: s_nor_b32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x8f]
 0x01,0x6b,0x00,0x8f
+# GFX12: s_nor_b32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x8f]
 0x01,0x6a,0x00,0x8f
+# GFX12: s_nor_b32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x8f]
 0x6b,0x02,0x00,0x8f
+# GFX12: s_nor_b32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x8f]
 
-# GFX12: s_nor_b32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x8f]
 0x6a,0x02,0x00,0x8f
+# GFX12: s_nor_b32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x8f]
 
-# GFX12: s_nor_b32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x8f]
 0x68,0x67,0x69,0x8f
+# GFX12: s_nor_b32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x8f]
 
-# GFX12: s_nor_b32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x8f]
 0x68,0x02,0x69,0x8f
+# GFX12: s_nor_b32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x8f]
 
-# GFX12: s_nor_b32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x8f]
 0x01,0x67,0x69,0x8f
+# GFX12: s_nor_b32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x8f]
 
-# GFX12: s_nor_b32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x8f]
 0x01,0x02,0x69,0x8f
+# GFX12: s_nor_b32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x8f]
 
-# GFX12: s_nor_b32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x8f]
 0x01,0x02,0x6b,0x8f
+# GFX12: s_nor_b32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x8f]
 
-# GFX12: s_nor_b32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x8f]
 0x01,0x02,0x6a,0x8f
+# GFX12: s_nor_b32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x8f]
 
-# GFX12: s_nor_b64 exec, s[2:3], s[4:5]          ; encoding: [0x02,0x04,0xfe,0x8f]
 0x02,0x04,0xfe,0x8f
+# GFX12: s_nor_b64 exec, s[2:3], s[4:5]          ; encoding: [0x02,0x04,0xfe,0x8f]
 
-# GFX12: s_nor_b64 s[0:1], 0.5, s[4:5]           ; encoding: [0xf0,0x04,0x80,0x8f]
 0xf0,0x04,0x80,0x8f
+# GFX12: s_nor_b64 s[0:1], 0.5, s[4:5]           ; encoding: [0xf0,0x04,0x80,0x8f]
 
-# GFX12: s_nor_b64 s[0:1], 0, s[4:5]             ; encoding: [0x80,0x04,0x80,0x8f]
 0x80,0x04,0x80,0x8f
+# GFX12: s_nor_b64 s[0:1], 0, s[4:5]             ; encoding: [0x80,0x04,0x80,0x8f]
 
-# GFX12: s_nor_b64 s[0:1], 0x3f717273, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8f,0x73,0x72,0x71,0x3f]
 0xff,0x04,0x80,0x8f,0x73,0x72,0x71,0x3f
+# GFX12: s_nor_b64 s[0:1], 0x3f717273, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8f,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_nor_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8f,0x56,0x34,0x12,0xaf]
 0xff,0x04,0x80,0x8f,0x56,0x34,0x12,0xaf
+# GFX12: s_nor_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8f,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_nor_b64 s[0:1], -1, s[4:5]            ; encoding: [0xc1,0x04,0x80,0x8f]
 0xc1,0x04,0x80,0x8f
+# GFX12: s_nor_b64 s[0:1], -1, s[4:5]            ; encoding: [0xc1,0x04,0x80,0x8f]
 
-# GFX12: s_nor_b64 s[0:1], -4.0, s[4:5]          ; encoding: [0xf7,0x04,0x80,0x8f]
 0xf7,0x04,0x80,0x8f
+# GFX12: s_nor_b64 s[0:1], -4.0, s[4:5]          ; encoding: [0xf7,0x04,0x80,0x8f]
 
-# GFX12: s_nor_b64 s[0:1], exec, s[4:5]          ; encoding: [0x7e,0x04,0x80,0x8f]
 0x7e,0x04,0x80,0x8f
+# GFX12: s_nor_b64 s[0:1], exec, s[4:5]          ; encoding: [0x7e,0x04,0x80,0x8f]
 
-# GFX12: s_nor_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x8f]
 0x66,0x64,0x80,0x8f
+# GFX12: s_nor_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x8f]
 
-# GFX12: s_nor_b64 s[0:1], s[102:103], s[4:5]    ; encoding: [0x66,0x04,0x80,0x8f]
 0x66,0x04,0x80,0x8f
+# GFX12: s_nor_b64 s[0:1], s[102:103], s[4:5]    ; encoding: [0x66,0x04,0x80,0x8f]
 
-# GFX12: s_nor_b64 s[0:1], s[2:3], 0.5           ; encoding: [0x02,0xf0,0x80,0x8f]
 0x02,0xf0,0x80,0x8f
+# GFX12: s_nor_b64 s[0:1], s[2:3], 0.5           ; encoding: [0x02,0xf0,0x80,0x8f]
 
-# GFX12: s_nor_b64 s[0:1], s[2:3], 0             ; encoding: [0x02,0x80,0x80,0x8f]
 0x02,0x80,0x80,0x8f
+# GFX12: s_nor_b64 s[0:1], s[2:3], 0             ; encoding: [0x02,0x80,0x80,0x8f]
 
-# GFX12: s_nor_b64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0x8f,0x73,0x72,0x71,0x3f]
 0x02,0xff,0x80,0x8f,0x73,0x72,0x71,0x3f
+# GFX12: s_nor_b64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0x8f,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_nor_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8f,0x56,0x34,0x12,0xaf]
 0x02,0xff,0x80,0x8f,0x56,0x34,0x12,0xaf
+# GFX12: s_nor_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8f,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_nor_b64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x80,0x8f]
 0x02,0xc1,0x80,0x8f
+# GFX12: s_nor_b64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x80,0x8f]
 
-# GFX12: s_nor_b64 s[0:1], s[2:3], -4.0          ; encoding: [0x02,0xf7,0x80,0x8f]
 0x02,0xf7,0x80,0x8f
+# GFX12: s_nor_b64 s[0:1], s[2:3], -4.0          ; encoding: [0x02,0xf7,0x80,0x8f]
 
-# GFX12: s_nor_b64 s[0:1], s[2:3], exec          ; encoding: [0x02,0x7e,0x80,0x8f]
 0x02,0x7e,0x80,0x8f
+# GFX12: s_nor_b64 s[0:1], s[2:3], exec          ; encoding: [0x02,0x7e,0x80,0x8f]
 
-# GFX12: s_nor_b64 s[0:1], s[2:3], s[100:101]    ; encoding: [0x02,0x64,0x80,0x8f]
 0x02,0x64,0x80,0x8f
+# GFX12: s_nor_b64 s[0:1], s[2:3], s[100:101]    ; encoding: [0x02,0x64,0x80,0x8f]
 
-# GFX12: s_nor_b64 s[0:1], s[2:3], s[4:5]        ; encoding: [0x02,0x04,0x80,0x8f]
 0x02,0x04,0x80,0x8f
+# GFX12: s_nor_b64 s[0:1], s[2:3], s[4:5]        ; encoding: [0x02,0x04,0x80,0x8f]
 
-# GFX12: s_nor_b64 s[0:1], s[2:3], vcc           ; encoding: [0x02,0x6a,0x80,0x8f]
 0x02,0x6a,0x80,0x8f
+# GFX12: s_nor_b64 s[0:1], s[2:3], vcc           ; encoding: [0x02,0x6a,0x80,0x8f]
 
-# GFX12: s_nor_b64 s[0:1], vcc, s[4:5]           ; encoding: [0x6a,0x04,0x80,0x8f]
 0x6a,0x04,0x80,0x8f
+# GFX12: s_nor_b64 s[0:1], vcc, s[4:5]           ; encoding: [0x6a,0x04,0x80,0x8f]
 
-# GFX12: s_nor_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x8f]
 0x66,0x64,0xe8,0x8f
+# GFX12: s_nor_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x8f]
 
-# GFX12: s_nor_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x8f]
 0x66,0x04,0xe8,0x8f
+# GFX12: s_nor_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x8f]
 
-# GFX12: s_nor_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x8f]
 0x02,0x64,0xe8,0x8f
+# GFX12: s_nor_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x8f]
 
-# GFX12: s_nor_b64 s[104:105], s[2:3], s[4:5]    ; encoding: [0x02,0x04,0xe8,0x8f]
 0x02,0x04,0xe8,0x8f
+# GFX12: s_nor_b64 s[104:105], s[2:3], s[4:5]    ; encoding: [0x02,0x04,0xe8,0x8f]
 
-# GFX12: s_nor_b64 vcc, s[2:3], s[4:5]           ; encoding: [0x02,0x04,0xea,0x8f]
 0x02,0x04,0xea,0x8f
+# GFX12: s_nor_b64 vcc, s[2:3], s[4:5]           ; encoding: [0x02,0x04,0xea,0x8f]
 
-# GFX12: s_or_b32 exec_hi, s1, s2                ; encoding: [0x01,0x02,0x7f,0x8c]
 0x01,0x02,0x7f,0x8c
+# GFX12: s_or_b32 exec_hi, s1, s2                ; encoding: [0x01,0x02,0x7f,0x8c]
 
-# GFX12: s_or_b32 exec_lo, s1, s2                ; encoding: [0x01,0x02,0x7e,0x8c]
 0x01,0x02,0x7e,0x8c
+# GFX12: s_or_b32 exec_lo, s1, s2                ; encoding: [0x01,0x02,0x7e,0x8c]
 
-# GFX12: s_or_b32 m0, s1, s2                     ; encoding: [0x01,0x02,0x7d,0x8c]
 0x01,0x02,0x7d,0x8c
+# GFX12: s_or_b32 m0, s1, s2                     ; encoding: [0x01,0x02,0x7d,0x8c]
 
-# GFX12: s_or_b32 s0, 0.5, s2                    ; encoding: [0xf0,0x02,0x00,0x8c]
 0xf0,0x02,0x00,0x8c
+# GFX12: s_or_b32 s0, 0.5, s2                    ; encoding: [0xf0,0x02,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, 0, s2                      ; encoding: [0x80,0x02,0x00,0x8c]
 0x80,0x02,0x00,0x8c
+# GFX12: s_or_b32 s0, 0, s2                      ; encoding: [0x80,0x02,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, 0x3f717273, s2             ; encoding: [0xff,0x02,0x00,0x8c,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x8c,0x73,0x72,0x71,0x3f
+# GFX12: s_or_b32 s0, 0x3f717273, s2             ; encoding: [0xff,0x02,0x00,0x8c,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_or_b32 s0, 0xaf123456, s2             ; encoding: [0xff,0x02,0x00,0x8c,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x8c,0x56,0x34,0x12,0xaf
+# GFX12: s_or_b32 s0, 0xaf123456, s2             ; encoding: [0xff,0x02,0x00,0x8c,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_or_b32 s0, -1, s2                     ; encoding: [0xc1,0x02,0x00,0x8c]
 0xc1,0x02,0x00,0x8c
+# GFX12: s_or_b32 s0, -1, s2                     ; encoding: [0xc1,0x02,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, -4.0, s2                   ; encoding: [0xf7,0x02,0x00,0x8c]
 0xf7,0x02,0x00,0x8c
+# GFX12: s_or_b32 s0, -4.0, s2                   ; encoding: [0xf7,0x02,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, exec_hi, s2                ; encoding: [0x7f,0x02,0x00,0x8c]
 0x7f,0x02,0x00,0x8c
+# GFX12: s_or_b32 s0, exec_hi, s2                ; encoding: [0x7f,0x02,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, exec_lo, s2                ; encoding: [0x7e,0x02,0x00,0x8c]
 0x7e,0x02,0x00,0x8c
+# GFX12: s_or_b32 s0, exec_lo, s2                ; encoding: [0x7e,0x02,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, m0, s2                     ; encoding: [0x7d,0x02,0x00,0x8c]
 0x7d,0x02,0x00,0x8c
+# GFX12: s_or_b32 s0, m0, s2                     ; encoding: [0x7d,0x02,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, s104, s103                 ; encoding: [0x68,0x67,0x00,0x8c]
 0x68,0x67,0x00,0x8c
+# GFX12: s_or_b32 s0, s104, s103                 ; encoding: [0x68,0x67,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, s104, s2                   ; encoding: [0x68,0x02,0x00,0x8c]
 0x68,0x02,0x00,0x8c
+# GFX12: s_or_b32 s0, s104, s2                   ; encoding: [0x68,0x02,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, s1, 0.5                    ; encoding: [0x01,0xf0,0x00,0x8c]
 0x01,0xf0,0x00,0x8c
+# GFX12: s_or_b32 s0, s1, 0.5                    ; encoding: [0x01,0xf0,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, s1, 0                      ; encoding: [0x01,0x80,0x00,0x8c]
 0x01,0x80,0x00,0x8c
+# GFX12: s_or_b32 s0, s1, 0                      ; encoding: [0x01,0x80,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, s1, 0x3f717273             ; encoding: [0x01,0xff,0x00,0x8c,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x8c,0x73,0x72,0x71,0x3f
+# GFX12: s_or_b32 s0, s1, 0x3f717273             ; encoding: [0x01,0xff,0x00,0x8c,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_or_b32 s0, s1, 0xaf123456             ; encoding: [0x01,0xff,0x00,0x8c,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x8c,0x56,0x34,0x12,0xaf
+# GFX12: s_or_b32 s0, s1, 0xaf123456             ; encoding: [0x01,0xff,0x00,0x8c,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_or_b32 s0, s1, -1                     ; encoding: [0x01,0xc1,0x00,0x8c]
 0x01,0xc1,0x00,0x8c
+# GFX12: s_or_b32 s0, s1, -1                     ; encoding: [0x01,0xc1,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, s1, -4.0                   ; encoding: [0x01,0xf7,0x00,0x8c]
 0x01,0xf7,0x00,0x8c
+# GFX12: s_or_b32 s0, s1, -4.0                   ; encoding: [0x01,0xf7,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, s1, exec_hi                ; encoding: [0x01,0x7f,0x00,0x8c]
 0x01,0x7f,0x00,0x8c
+# GFX12: s_or_b32 s0, s1, exec_hi                ; encoding: [0x01,0x7f,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, s1, exec_lo                ; encoding: [0x01,0x7e,0x00,0x8c]
 0x01,0x7e,0x00,0x8c
+# GFX12: s_or_b32 s0, s1, exec_lo                ; encoding: [0x01,0x7e,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, s1, m0                     ; encoding: [0x01,0x7d,0x00,0x8c]
 0x01,0x7d,0x00,0x8c
+# GFX12: s_or_b32 s0, s1, m0                     ; encoding: [0x01,0x7d,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, s1, s103                   ; encoding: [0x01,0x67,0x00,0x8c]
 0x01,0x67,0x00,0x8c
+# GFX12: s_or_b32 s0, s1, s103                   ; encoding: [0x01,0x67,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, s1, s2                     ; encoding: [0x01,0x02,0x00,0x8c]
 0x01,0x02,0x00,0x8c
+# GFX12: s_or_b32 s0, s1, s2                     ; encoding: [0x01,0x02,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, s1, vcc_hi                 ; encoding: [0x01,0x6b,0x00,0x8c]
 0x01,0x6b,0x00,0x8c
+# GFX12: s_or_b32 s0, s1, vcc_hi                 ; encoding: [0x01,0x6b,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, s1, vcc_lo                 ; encoding: [0x01,0x6a,0x00,0x8c]
 0x01,0x6a,0x00,0x8c
+# GFX12: s_or_b32 s0, s1, vcc_lo                 ; encoding: [0x01,0x6a,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, vcc_hi, s2                 ; encoding: [0x6b,0x02,0x00,0x8c]
 0x6b,0x02,0x00,0x8c
+# GFX12: s_or_b32 s0, vcc_hi, s2                 ; encoding: [0x6b,0x02,0x00,0x8c]
 
-# GFX12: s_or_b32 s0, vcc_lo, s2                 ; encoding: [0x6a,0x02,0x00,0x8c]
 0x6a,0x02,0x00,0x8c
+# GFX12: s_or_b32 s0, vcc_lo, s2                 ; encoding: [0x6a,0x02,0x00,0x8c]
 
-# GFX12: s_or_b32 s105, s104, s103               ; encoding: [0x68,0x67,0x69,0x8c]
 0x68,0x67,0x69,0x8c
+# GFX12: s_or_b32 s105, s104, s103               ; encoding: [0x68,0x67,0x69,0x8c]
 
-# GFX12: s_or_b32 s105, s104, s2                 ; encoding: [0x68,0x02,0x69,0x8c]
 0x68,0x02,0x69,0x8c
+# GFX12: s_or_b32 s105, s104, s2                 ; encoding: [0x68,0x02,0x69,0x8c]
 
-# GFX12: s_or_b32 s105, s1, s103                 ; encoding: [0x01,0x67,0x69,0x8c]
 0x01,0x67,0x69,0x8c
+# GFX12: s_or_b32 s105, s1, s103                 ; encoding: [0x01,0x67,0x69,0x8c]
 
-# GFX12: s_or_b32 s105, s1, s2                   ; encoding: [0x01,0x02,0x69,0x8c]
 0x01,0x02,0x69,0x8c
+# GFX12: s_or_b32 s105, s1, s2                   ; encoding: [0x01,0x02,0x69,0x8c]
 
-# GFX12: s_or_b32 vcc_hi, s1, s2                 ; encoding: [0x01,0x02,0x6b,0x8c]
 0x01,0x02,0x6b,0x8c
+# GFX12: s_or_b32 vcc_hi, s1, s2                 ; encoding: [0x01,0x02,0x6b,0x8c]
 
-# GFX12: s_or_b32 vcc_lo, s1, s2                 ; encoding: [0x01,0x02,0x6a,0x8c]
 0x01,0x02,0x6a,0x8c
+# GFX12: s_or_b32 vcc_lo, s1, s2                 ; encoding: [0x01,0x02,0x6a,0x8c]
 
-# GFX12: s_or_b64 exec, s[2:3], s[4:5]           ; encoding: [0x02,0x04,0xfe,0x8c]
 0x02,0x04,0xfe,0x8c
+# GFX12: s_or_b64 exec, s[2:3], s[4:5]           ; encoding: [0x02,0x04,0xfe,0x8c]
 
-# GFX12: s_or_b64 s[0:1], 0.5, s[4:5]            ; encoding: [0xf0,0x04,0x80,0x8c]
 0xf0,0x04,0x80,0x8c
+# GFX12: s_or_b64 s[0:1], 0.5, s[4:5]            ; encoding: [0xf0,0x04,0x80,0x8c]
 
-# GFX12: s_or_b64 s[0:1], 0, s[4:5]              ; encoding: [0x80,0x04,0x80,0x8c]
 0x80,0x04,0x80,0x8c
+# GFX12: s_or_b64 s[0:1], 0, s[4:5]              ; encoding: [0x80,0x04,0x80,0x8c]
 
-# GFX12: s_or_b64 s[0:1], 0x3f717273, s[4:5]     ; encoding: [0xff,0x04,0x80,0x8c,0x73,0x72,0x71,0x3f]
 0xff,0x04,0x80,0x8c,0x73,0x72,0x71,0x3f
+# GFX12: s_or_b64 s[0:1], 0x3f717273, s[4:5]     ; encoding: [0xff,0x04,0x80,0x8c,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_or_b64 s[0:1], 0xaf123456, s[4:5]     ; encoding: [0xff,0x04,0x80,0x8c,0x56,0x34,0x12,0xaf]
 0xff,0x04,0x80,0x8c,0x56,0x34,0x12,0xaf
+# GFX12: s_or_b64 s[0:1], 0xaf123456, s[4:5]     ; encoding: [0xff,0x04,0x80,0x8c,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_or_b64 s[0:1], -1, s[4:5]             ; encoding: [0xc1,0x04,0x80,0x8c]
 0xc1,0x04,0x80,0x8c
+# GFX12: s_or_b64 s[0:1], -1, s[4:5]             ; encoding: [0xc1,0x04,0x80,0x8c]
 
-# GFX12: s_or_b64 s[0:1], -4.0, s[4:5]           ; encoding: [0xf7,0x04,0x80,0x8c]
 0xf7,0x04,0x80,0x8c
+# GFX12: s_or_b64 s[0:1], -4.0, s[4:5]           ; encoding: [0xf7,0x04,0x80,0x8c]
 
-# GFX12: s_or_b64 s[0:1], exec, s[4:5]           ; encoding: [0x7e,0x04,0x80,0x8c]
 0x7e,0x04,0x80,0x8c
+# GFX12: s_or_b64 s[0:1], exec, s[4:5]           ; encoding: [0x7e,0x04,0x80,0x8c]
 
-# GFX12: s_or_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x8c]
 0x66,0x64,0x80,0x8c
+# GFX12: s_or_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x8c]
 
-# GFX12: s_or_b64 s[0:1], s[102:103], s[4:5]     ; encoding: [0x66,0x04,0x80,0x8c]
 0x66,0x04,0x80,0x8c
+# GFX12: s_or_b64 s[0:1], s[102:103], s[4:5]     ; encoding: [0x66,0x04,0x80,0x8c]
 
-# GFX12: s_or_b64 s[0:1], s[2:3], 0.5            ; encoding: [0x02,0xf0,0x80,0x8c]
 0x02,0xf0,0x80,0x8c
+# GFX12: s_or_b64 s[0:1], s[2:3], 0.5            ; encoding: [0x02,0xf0,0x80,0x8c]
 
-# GFX12: s_or_b64 s[0:1], s[2:3], 0              ; encoding: [0x02,0x80,0x80,0x8c]
 0x02,0x80,0x80,0x8c
+# GFX12: s_or_b64 s[0:1], s[2:3], 0              ; encoding: [0x02,0x80,0x80,0x8c]
 
-# GFX12: s_or_b64 s[0:1], s[2:3], 0x3f717273     ; encoding: [0x02,0xff,0x80,0x8c,0x73,0x72,0x71,0x3f]
 0x02,0xff,0x80,0x8c,0x73,0x72,0x71,0x3f
+# GFX12: s_or_b64 s[0:1], s[2:3], 0x3f717273     ; encoding: [0x02,0xff,0x80,0x8c,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_or_b64 s[0:1], s[2:3], 0xaf123456     ; encoding: [0x02,0xff,0x80,0x8c,0x56,0x34,0x12,0xaf]
 0x02,0xff,0x80,0x8c,0x56,0x34,0x12,0xaf
+# GFX12: s_or_b64 s[0:1], s[2:3], 0xaf123456     ; encoding: [0x02,0xff,0x80,0x8c,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_or_b64 s[0:1], s[2:3], -1             ; encoding: [0x02,0xc1,0x80,0x8c]
 0x02,0xc1,0x80,0x8c
+# GFX12: s_or_b64 s[0:1], s[2:3], -1             ; encoding: [0x02,0xc1,0x80,0x8c]
 
-# GFX12: s_or_b64 s[0:1], s[2:3], -4.0           ; encoding: [0x02,0xf7,0x80,0x8c]
 0x02,0xf7,0x80,0x8c
+# GFX12: s_or_b64 s[0:1], s[2:3], -4.0           ; encoding: [0x02,0xf7,0x80,0x8c]
 
-# GFX12: s_or_b64 s[0:1], s[2:3], exec           ; encoding: [0x02,0x7e,0x80,0x8c]
 0x02,0x7e,0x80,0x8c
+# GFX12: s_or_b64 s[0:1], s[2:3], exec           ; encoding: [0x02,0x7e,0x80,0x8c]
 
-# GFX12: s_or_b64 s[0:1], s[2:3], s[100:101]     ; encoding: [0x02,0x64,0x80,0x8c]
 0x02,0x64,0x80,0x8c
+# GFX12: s_or_b64 s[0:1], s[2:3], s[100:101]     ; encoding: [0x02,0x64,0x80,0x8c]
 
-# GFX12: s_or_b64 s[0:1], s[2:3], s[4:5]         ; encoding: [0x02,0x04,0x80,0x8c]
 0x02,0x04,0x80,0x8c
+# GFX12: s_or_b64 s[0:1], s[2:3], s[4:5]         ; encoding: [0x02,0x04,0x80,0x8c]
 
-# GFX12: s_or_b64 s[0:1], s[2:3], vcc            ; encoding: [0x02,0x6a,0x80,0x8c]
 0x02,0x6a,0x80,0x8c
+# GFX12: s_or_b64 s[0:1], s[2:3], vcc            ; encoding: [0x02,0x6a,0x80,0x8c]
 
-# GFX12: s_or_b64 s[0:1], vcc, s[4:5]            ; encoding: [0x6a,0x04,0x80,0x8c]
 0x6a,0x04,0x80,0x8c
+# GFX12: s_or_b64 s[0:1], vcc, s[4:5]            ; encoding: [0x6a,0x04,0x80,0x8c]
 
-# GFX12: s_or_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x8c]
 0x66,0x64,0xe8,0x8c
+# GFX12: s_or_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x8c]
 
-# GFX12: s_or_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x8c]
 0x66,0x04,0xe8,0x8c
+# GFX12: s_or_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x8c]
 
-# GFX12: s_or_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x8c]
 0x02,0x64,0xe8,0x8c
+# GFX12: s_or_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x8c]
 
-# GFX12: s_or_b64 s[104:105], s[2:3], s[4:5]     ; encoding: [0x02,0x04,0xe8,0x8c]
 0x02,0x04,0xe8,0x8c
+# GFX12: s_or_b64 s[104:105], s[2:3], s[4:5]     ; encoding: [0x02,0x04,0xe8,0x8c]
 
-# GFX12: s_or_b64 vcc, s[2:3], s[4:5]            ; encoding: [0x02,0x04,0xea,0x8c]
 0x02,0x04,0xea,0x8c
+# GFX12: s_or_b64 vcc, s[2:3], s[4:5]            ; encoding: [0x02,0x04,0xea,0x8c]
 
-# GFX12: s_or_not1_b32 exec_hi, s1, s2           ; encoding: [0x01,0x02,0x7f,0x92]
 0x01,0x02,0x7f,0x92
+# GFX12: s_or_not1_b32 exec_hi, s1, s2           ; encoding: [0x01,0x02,0x7f,0x92]
 
-# GFX12: s_or_not1_b32 exec_lo, s1, s2           ; encoding: [0x01,0x02,0x7e,0x92]
 0x01,0x02,0x7e,0x92
+# GFX12: s_or_not1_b32 exec_lo, s1, s2           ; encoding: [0x01,0x02,0x7e,0x92]
 
-# GFX12: s_or_not1_b32 m0, s1, s2                ; encoding: [0x01,0x02,0x7d,0x92]
 0x01,0x02,0x7d,0x92
+# GFX12: s_or_not1_b32 m0, s1, s2                ; encoding: [0x01,0x02,0x7d,0x92]
 
-# GFX12: s_or_not1_b32 s0, 0.5, s2               ; encoding: [0xf0,0x02,0x00,0x92]
 0xf0,0x02,0x00,0x92
+# GFX12: s_or_not1_b32 s0, 0.5, s2               ; encoding: [0xf0,0x02,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, 0, s2                 ; encoding: [0x80,0x02,0x00,0x92]
 0x80,0x02,0x00,0x92
+# GFX12: s_or_not1_b32 s0, 0, s2                 ; encoding: [0x80,0x02,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, 0x3f717273, s2        ; encoding: [0xff,0x02,0x00,0x92,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x92,0x73,0x72,0x71,0x3f
+# GFX12: s_or_not1_b32 s0, 0x3f717273, s2        ; encoding: [0xff,0x02,0x00,0x92,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_or_not1_b32 s0, 0xaf123456, s2        ; encoding: [0xff,0x02,0x00,0x92,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x92,0x56,0x34,0x12,0xaf
+# GFX12: s_or_not1_b32 s0, 0xaf123456, s2        ; encoding: [0xff,0x02,0x00,0x92,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_or_not1_b32 s0, -1, s2                ; encoding: [0xc1,0x02,0x00,0x92]
 0xc1,0x02,0x00,0x92
+# GFX12: s_or_not1_b32 s0, -1, s2                ; encoding: [0xc1,0x02,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, -4.0, s2              ; encoding: [0xf7,0x02,0x00,0x92]
 0xf7,0x02,0x00,0x92
+# GFX12: s_or_not1_b32 s0, -4.0, s2              ; encoding: [0xf7,0x02,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, exec_hi, s2           ; encoding: [0x7f,0x02,0x00,0x92]
 0x7f,0x02,0x00,0x92
+# GFX12: s_or_not1_b32 s0, exec_hi, s2           ; encoding: [0x7f,0x02,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, exec_lo, s2           ; encoding: [0x7e,0x02,0x00,0x92]
 0x7e,0x02,0x00,0x92
+# GFX12: s_or_not1_b32 s0, exec_lo, s2           ; encoding: [0x7e,0x02,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, m0, s2                ; encoding: [0x7d,0x02,0x00,0x92]
 0x7d,0x02,0x00,0x92
+# GFX12: s_or_not1_b32 s0, m0, s2                ; encoding: [0x7d,0x02,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, s104, s103            ; encoding: [0x68,0x67,0x00,0x92]
 0x68,0x67,0x00,0x92
+# GFX12: s_or_not1_b32 s0, s104, s103            ; encoding: [0x68,0x67,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, s104, s2              ; encoding: [0x68,0x02,0x00,0x92]
 0x68,0x02,0x00,0x92
+# GFX12: s_or_not1_b32 s0, s104, s2              ; encoding: [0x68,0x02,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, s1, 0.5               ; encoding: [0x01,0xf0,0x00,0x92]
 0x01,0xf0,0x00,0x92
+# GFX12: s_or_not1_b32 s0, s1, 0.5               ; encoding: [0x01,0xf0,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, s1, 0                 ; encoding: [0x01,0x80,0x00,0x92]
 0x01,0x80,0x00,0x92
+# GFX12: s_or_not1_b32 s0, s1, 0                 ; encoding: [0x01,0x80,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, s1, 0x3f717273        ; encoding: [0x01,0xff,0x00,0x92,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x92,0x73,0x72,0x71,0x3f
+# GFX12: s_or_not1_b32 s0, s1, 0x3f717273        ; encoding: [0x01,0xff,0x00,0x92,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_or_not1_b32 s0, s1, 0xaf123456        ; encoding: [0x01,0xff,0x00,0x92,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x92,0x56,0x34,0x12,0xaf
+# GFX12: s_or_not1_b32 s0, s1, 0xaf123456        ; encoding: [0x01,0xff,0x00,0x92,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_or_not1_b32 s0, s1, -1                ; encoding: [0x01,0xc1,0x00,0x92]
 0x01,0xc1,0x00,0x92
+# GFX12: s_or_not1_b32 s0, s1, -1                ; encoding: [0x01,0xc1,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, s1, -4.0              ; encoding: [0x01,0xf7,0x00,0x92]
 0x01,0xf7,0x00,0x92
+# GFX12: s_or_not1_b32 s0, s1, -4.0              ; encoding: [0x01,0xf7,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, s1, exec_hi           ; encoding: [0x01,0x7f,0x00,0x92]
 0x01,0x7f,0x00,0x92
+# GFX12: s_or_not1_b32 s0, s1, exec_hi           ; encoding: [0x01,0x7f,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, s1, exec_lo           ; encoding: [0x01,0x7e,0x00,0x92]
 0x01,0x7e,0x00,0x92
+# GFX12: s_or_not1_b32 s0, s1, exec_lo           ; encoding: [0x01,0x7e,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, s1, m0                ; encoding: [0x01,0x7d,0x00,0x92]
 0x01,0x7d,0x00,0x92
+# GFX12: s_or_not1_b32 s0, s1, m0                ; encoding: [0x01,0x7d,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, s1, s103              ; encoding: [0x01,0x67,0x00,0x92]
 0x01,0x67,0x00,0x92
+# GFX12: s_or_not1_b32 s0, s1, s103              ; encoding: [0x01,0x67,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, s1, s2                ; encoding: [0x01,0x02,0x00,0x92]
 0x01,0x02,0x00,0x92
+# GFX12: s_or_not1_b32 s0, s1, s2                ; encoding: [0x01,0x02,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, s1, vcc_hi            ; encoding: [0x01,0x6b,0x00,0x92]
 0x01,0x6b,0x00,0x92
+# GFX12: s_or_not1_b32 s0, s1, vcc_hi            ; encoding: [0x01,0x6b,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, s1, vcc_lo            ; encoding: [0x01,0x6a,0x00,0x92]
 0x01,0x6a,0x00,0x92
+# GFX12: s_or_not1_b32 s0, s1, vcc_lo            ; encoding: [0x01,0x6a,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, vcc_hi, s2            ; encoding: [0x6b,0x02,0x00,0x92]
 0x6b,0x02,0x00,0x92
+# GFX12: s_or_not1_b32 s0, vcc_hi, s2            ; encoding: [0x6b,0x02,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s0, vcc_lo, s2            ; encoding: [0x6a,0x02,0x00,0x92]
 0x6a,0x02,0x00,0x92
+# GFX12: s_or_not1_b32 s0, vcc_lo, s2            ; encoding: [0x6a,0x02,0x00,0x92]
 
-# GFX12: s_or_not1_b32 s105, s104, s103          ; encoding: [0x68,0x67,0x69,0x92]
 0x68,0x67,0x69,0x92
+# GFX12: s_or_not1_b32 s105, s104, s103          ; encoding: [0x68,0x67,0x69,0x92]
 
-# GFX12: s_or_not1_b32 s105, s104, s2            ; encoding: [0x68,0x02,0x69,0x92]
 0x68,0x02,0x69,0x92
+# GFX12: s_or_not1_b32 s105, s104, s2            ; encoding: [0x68,0x02,0x69,0x92]
 
-# GFX12: s_or_not1_b32 s105, s1, s103            ; encoding: [0x01,0x67,0x69,0x92]
 0x01,0x67,0x69,0x92
+# GFX12: s_or_not1_b32 s105, s1, s103            ; encoding: [0x01,0x67,0x69,0x92]
 
-# GFX12: s_or_not1_b32 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0x92]
 0x01,0x02,0x69,0x92
+# GFX12: s_or_not1_b32 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0x92]
 
-# GFX12: s_or_not1_b32 vcc_hi, s1, s2            ; encoding: [0x01,0x02,0x6b,0x92]
 0x01,0x02,0x6b,0x92
+# GFX12: s_or_not1_b32 vcc_hi, s1, s2            ; encoding: [0x01,0x02,0x6b,0x92]
 
-# GFX12: s_or_not1_b32 vcc_lo, s1, s2            ; encoding: [0x01,0x02,0x6a,0x92]
 0x01,0x02,0x6a,0x92
+# GFX12: s_or_not1_b32 vcc_lo, s1, s2            ; encoding: [0x01,0x02,0x6a,0x92]
 
-# GFX12: s_or_not1_b64 exec, s[2:3], s[4:5]      ; encoding: [0x02,0x04,0xfe,0x92]
 0x02,0x04,0xfe,0x92
+# GFX12: s_or_not1_b64 exec, s[2:3], s[4:5]      ; encoding: [0x02,0x04,0xfe,0x92]
 
-# GFX12: s_or_not1_b64 s[0:1], 0.5, s[4:5]       ; encoding: [0xf0,0x04,0x80,0x92]
 0xf0,0x04,0x80,0x92
+# GFX12: s_or_not1_b64 s[0:1], 0.5, s[4:5]       ; encoding: [0xf0,0x04,0x80,0x92]
 
-# GFX12: s_or_not1_b64 s[0:1], 0, s[4:5]         ; encoding: [0x80,0x04,0x80,0x92]
 0x80,0x04,0x80,0x92
+# GFX12: s_or_not1_b64 s[0:1], 0, s[4:5]         ; encoding: [0x80,0x04,0x80,0x92]
 
-# GFX12: s_or_not1_b64 s[0:1], 0x3f717273, s[4:5] ; encoding: [0xff,0x04,0x80,0x92,0x73,0x72,0x71,0x3f]
 0xff,0x04,0x80,0x92,0x73,0x72,0x71,0x3f
+# GFX12: s_or_not1_b64 s[0:1], 0x3f717273, s[4:5] ; encoding: [0xff,0x04,0x80,0x92,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_or_not1_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x92,0x56,0x34,0x12,0xaf]
 0xff,0x04,0x80,0x92,0x56,0x34,0x12,0xaf
+# GFX12: s_or_not1_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x92,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_or_not1_b64 s[0:1], -1, s[4:5]        ; encoding: [0xc1,0x04,0x80,0x92]
 0xc1,0x04,0x80,0x92
+# GFX12: s_or_not1_b64 s[0:1], -1, s[4:5]        ; encoding: [0xc1,0x04,0x80,0x92]
 
-# GFX12: s_or_not1_b64 s[0:1], -4.0, s[4:5]      ; encoding: [0xf7,0x04,0x80,0x92]
 0xf7,0x04,0x80,0x92
+# GFX12: s_or_not1_b64 s[0:1], -4.0, s[4:5]      ; encoding: [0xf7,0x04,0x80,0x92]
 
-# GFX12: s_or_not1_b64 s[0:1], exec, s[4:5]      ; encoding: [0x7e,0x04,0x80,0x92]
 0x7e,0x04,0x80,0x92
+# GFX12: s_or_not1_b64 s[0:1], exec, s[4:5]      ; encoding: [0x7e,0x04,0x80,0x92]
 
-# GFX12: s_or_not1_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x92]
 0x66,0x64,0x80,0x92
+# GFX12: s_or_not1_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x92]
 
-# GFX12: s_or_not1_b64 s[0:1], s[102:103], s[4:5] ; encoding: [0x66,0x04,0x80,0x92]
 0x66,0x04,0x80,0x92
+# GFX12: s_or_not1_b64 s[0:1], s[102:103], s[4:5] ; encoding: [0x66,0x04,0x80,0x92]
 
-# GFX12: s_or_not1_b64 s[0:1], s[2:3], 0.5       ; encoding: [0x02,0xf0,0x80,0x92]
 0x02,0xf0,0x80,0x92
+# GFX12: s_or_not1_b64 s[0:1], s[2:3], 0.5       ; encoding: [0x02,0xf0,0x80,0x92]
 
-# GFX12: s_or_not1_b64 s[0:1], s[2:3], 0         ; encoding: [0x02,0x80,0x80,0x92]
 0x02,0x80,0x80,0x92
+# GFX12: s_or_not1_b64 s[0:1], s[2:3], 0         ; encoding: [0x02,0x80,0x80,0x92]
 
-# GFX12: s_or_not1_b64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0x92,0x73,0x72,0x71,0x3f]
 0x02,0xff,0x80,0x92,0x73,0x72,0x71,0x3f
+# GFX12: s_or_not1_b64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0x92,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_or_not1_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x92,0x56,0x34,0x12,0xaf]
 0x02,0xff,0x80,0x92,0x56,0x34,0x12,0xaf
+# GFX12: s_or_not1_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x92,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_or_not1_b64 s[0:1], s[2:3], -1        ; encoding: [0x02,0xc1,0x80,0x92]
 0x02,0xc1,0x80,0x92
+# GFX12: s_or_not1_b64 s[0:1], s[2:3], -1        ; encoding: [0x02,0xc1,0x80,0x92]
 
-# GFX12: s_or_not1_b64 s[0:1], s[2:3], -4.0      ; encoding: [0x02,0xf7,0x80,0x92]
 0x02,0xf7,0x80,0x92
+# GFX12: s_or_not1_b64 s[0:1], s[2:3], -4.0      ; encoding: [0x02,0xf7,0x80,0x92]
 
-# GFX12: s_or_not1_b64 s[0:1], s[2:3], exec      ; encoding: [0x02,0x7e,0x80,0x92]
 0x02,0x7e,0x80,0x92
+# GFX12: s_or_not1_b64 s[0:1], s[2:3], exec      ; encoding: [0x02,0x7e,0x80,0x92]
 
-# GFX12: s_or_not1_b64 s[0:1], s[2:3], s[100:101] ; encoding: [0x02,0x64,0x80,0x92]
 0x02,0x64,0x80,0x92
+# GFX12: s_or_not1_b64 s[0:1], s[2:3], s[100:101] ; encoding: [0x02,0x64,0x80,0x92]
 
-# GFX12: s_or_not1_b64 s[0:1], s[2:3], s[4:5]    ; encoding: [0x02,0x04,0x80,0x92]
 0x02,0x04,0x80,0x92
+# GFX12: s_or_not1_b64 s[0:1], s[2:3], s[4:5]    ; encoding: [0x02,0x04,0x80,0x92]
 
-# GFX12: s_or_not1_b64 s[0:1], s[2:3], vcc       ; encoding: [0x02,0x6a,0x80,0x92]
 0x02,0x6a,0x80,0x92
+# GFX12: s_or_not1_b64 s[0:1], s[2:3], vcc       ; encoding: [0x02,0x6a,0x80,0x92]
 
-# GFX12: s_or_not1_b64 s[0:1], vcc, s[4:5]       ; encoding: [0x6a,0x04,0x80,0x92]
 0x6a,0x04,0x80,0x92
+# GFX12: s_or_not1_b64 s[0:1], vcc, s[4:5]       ; encoding: [0x6a,0x04,0x80,0x92]
 
-# GFX12: s_or_not1_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x92]
 0x66,0x64,0xe8,0x92
+# GFX12: s_or_not1_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x92]
 
-# GFX12: s_or_not1_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x92]
 0x66,0x04,0xe8,0x92
+# GFX12: s_or_not1_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x92]
 
-# GFX12: s_or_not1_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x92]
 0x02,0x64,0xe8,0x92
+# GFX12: s_or_not1_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x92]
 
-# GFX12: s_or_not1_b64 s[104:105], s[2:3], s[4:5] ; encoding: [0x02,0x04,0xe8,0x92]
 0x02,0x04,0xe8,0x92
+# GFX12: s_or_not1_b64 s[104:105], s[2:3], s[4:5] ; encoding: [0x02,0x04,0xe8,0x92]
 
-# GFX12: s_or_not1_b64 vcc, s[2:3], s[4:5]       ; encoding: [0x02,0x04,0xea,0x92]
 0x02,0x04,0xea,0x92
+# GFX12: s_or_not1_b64 vcc, s[2:3], s[4:5]       ; encoding: [0x02,0x04,0xea,0x92]
 
-# GFX12: s_pack_hh_b32_b16 exec_hi, s1, s2       ; encoding: [0x01,0x02,0x7f,0x9a]
 0x01,0x02,0x7f,0x9a
+# GFX12: s_pack_hh_b32_b16 exec_hi, s1, s2       ; encoding: [0x01,0x02,0x7f,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 exec_lo, s1, s2       ; encoding: [0x01,0x02,0x7e,0x9a]
 0x01,0x02,0x7e,0x9a
+# GFX12: s_pack_hh_b32_b16 exec_lo, s1, s2       ; encoding: [0x01,0x02,0x7e,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 m0, s1, s2            ; encoding: [0x01,0x02,0x7d,0x9a]
 0x01,0x02,0x7d,0x9a
+# GFX12: s_pack_hh_b32_b16 m0, s1, s2            ; encoding: [0x01,0x02,0x7d,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, 0.5, s2           ; encoding: [0xf0,0x02,0x00,0x9a]
 0xf0,0x02,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, 0.5, s2           ; encoding: [0xf0,0x02,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, 0, s2             ; encoding: [0x80,0x02,0x00,0x9a]
 0x80,0x02,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, 0, s2             ; encoding: [0x80,0x02,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, 0x3f717273, s2    ; encoding: [0xff,0x02,0x00,0x9a,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x9a,0x73,0x72,0x71,0x3f
+# GFX12: s_pack_hh_b32_b16 s0, 0x3f717273, s2    ; encoding: [0xff,0x02,0x00,0x9a,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_pack_hh_b32_b16 s0, 0xaf123456, s2    ; encoding: [0xff,0x02,0x00,0x9a,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x9a,0x56,0x34,0x12,0xaf
+# GFX12: s_pack_hh_b32_b16 s0, 0xaf123456, s2    ; encoding: [0xff,0x02,0x00,0x9a,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_pack_hh_b32_b16 s0, -1, s2            ; encoding: [0xc1,0x02,0x00,0x9a]
 0xc1,0x02,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, -1, s2            ; encoding: [0xc1,0x02,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, -4.0, s2          ; encoding: [0xf7,0x02,0x00,0x9a]
 0xf7,0x02,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, -4.0, s2          ; encoding: [0xf7,0x02,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, exec_hi, s2       ; encoding: [0x7f,0x02,0x00,0x9a]
 0x7f,0x02,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, exec_hi, s2       ; encoding: [0x7f,0x02,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, exec_lo, s2       ; encoding: [0x7e,0x02,0x00,0x9a]
 0x7e,0x02,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, exec_lo, s2       ; encoding: [0x7e,0x02,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, m0, s2            ; encoding: [0x7d,0x02,0x00,0x9a]
 0x7d,0x02,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, m0, s2            ; encoding: [0x7d,0x02,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, s104, s103        ; encoding: [0x68,0x67,0x00,0x9a]
 0x68,0x67,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, s104, s103        ; encoding: [0x68,0x67,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, s104, s2          ; encoding: [0x68,0x02,0x00,0x9a]
 0x68,0x02,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, s104, s2          ; encoding: [0x68,0x02,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, s1, 0.5           ; encoding: [0x01,0xf0,0x00,0x9a]
 0x01,0xf0,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, s1, 0.5           ; encoding: [0x01,0xf0,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, s1, 0             ; encoding: [0x01,0x80,0x00,0x9a]
 0x01,0x80,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, s1, 0             ; encoding: [0x01,0x80,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, s1, 0x3f717273    ; encoding: [0x01,0xff,0x00,0x9a,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x9a,0x73,0x72,0x71,0x3f
+# GFX12: s_pack_hh_b32_b16 s0, s1, 0x3f717273    ; encoding: [0x01,0xff,0x00,0x9a,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_pack_hh_b32_b16 s0, s1, 0xaf123456    ; encoding: [0x01,0xff,0x00,0x9a,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x9a,0x56,0x34,0x12,0xaf
+# GFX12: s_pack_hh_b32_b16 s0, s1, 0xaf123456    ; encoding: [0x01,0xff,0x00,0x9a,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_pack_hh_b32_b16 s0, s1, -1            ; encoding: [0x01,0xc1,0x00,0x9a]
 0x01,0xc1,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, s1, -1            ; encoding: [0x01,0xc1,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, s1, -4.0          ; encoding: [0x01,0xf7,0x00,0x9a]
 0x01,0xf7,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, s1, -4.0          ; encoding: [0x01,0xf7,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, s1, exec_hi       ; encoding: [0x01,0x7f,0x00,0x9a]
 0x01,0x7f,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, s1, exec_hi       ; encoding: [0x01,0x7f,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, s1, exec_lo       ; encoding: [0x01,0x7e,0x00,0x9a]
 0x01,0x7e,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, s1, exec_lo       ; encoding: [0x01,0x7e,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, s1, m0            ; encoding: [0x01,0x7d,0x00,0x9a]
 0x01,0x7d,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, s1, m0            ; encoding: [0x01,0x7d,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, s1, s103          ; encoding: [0x01,0x67,0x00,0x9a]
 0x01,0x67,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, s1, s103          ; encoding: [0x01,0x67,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, s1, s2            ; encoding: [0x01,0x02,0x00,0x9a]
 0x01,0x02,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, s1, s2            ; encoding: [0x01,0x02,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, s1, vcc_hi        ; encoding: [0x01,0x6b,0x00,0x9a]
 0x01,0x6b,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, s1, vcc_hi        ; encoding: [0x01,0x6b,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, s1, vcc_lo        ; encoding: [0x01,0x6a,0x00,0x9a]
 0x01,0x6a,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, s1, vcc_lo        ; encoding: [0x01,0x6a,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, vcc_hi, s2        ; encoding: [0x6b,0x02,0x00,0x9a]
 0x6b,0x02,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, vcc_hi, s2        ; encoding: [0x6b,0x02,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s0, vcc_lo, s2        ; encoding: [0x6a,0x02,0x00,0x9a]
 0x6a,0x02,0x00,0x9a
+# GFX12: s_pack_hh_b32_b16 s0, vcc_lo, s2        ; encoding: [0x6a,0x02,0x00,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s105, s104, s103      ; encoding: [0x68,0x67,0x69,0x9a]
 0x68,0x67,0x69,0x9a
+# GFX12: s_pack_hh_b32_b16 s105, s104, s103      ; encoding: [0x68,0x67,0x69,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s105, s104, s2        ; encoding: [0x68,0x02,0x69,0x9a]
 0x68,0x02,0x69,0x9a
+# GFX12: s_pack_hh_b32_b16 s105, s104, s2        ; encoding: [0x68,0x02,0x69,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s105, s1, s103        ; encoding: [0x01,0x67,0x69,0x9a]
 0x01,0x67,0x69,0x9a
+# GFX12: s_pack_hh_b32_b16 s105, s1, s103        ; encoding: [0x01,0x67,0x69,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 s105, s1, s2          ; encoding: [0x01,0x02,0x69,0x9a]
 0x01,0x02,0x69,0x9a
+# GFX12: s_pack_hh_b32_b16 s105, s1, s2          ; encoding: [0x01,0x02,0x69,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 vcc_hi, s1, s2        ; encoding: [0x01,0x02,0x6b,0x9a]
 0x01,0x02,0x6b,0x9a
+# GFX12: s_pack_hh_b32_b16 vcc_hi, s1, s2        ; encoding: [0x01,0x02,0x6b,0x9a]
 
-# GFX12: s_pack_hh_b32_b16 vcc_lo, s1, s2        ; encoding: [0x01,0x02,0x6a,0x9a]
 0x01,0x02,0x6a,0x9a
+# GFX12: s_pack_hh_b32_b16 vcc_lo, s1, s2        ; encoding: [0x01,0x02,0x6a,0x9a]
 
-# GFX12: s_pack_lh_b32_b16 exec_hi, s1, s2       ; encoding: [0x01,0x02,0xff,0x99]
 0x01,0x02,0xff,0x99
+# GFX12: s_pack_lh_b32_b16 exec_hi, s1, s2       ; encoding: [0x01,0x02,0xff,0x99]
 
-# GFX12: s_pack_lh_b32_b16 exec_lo, s1, s2       ; encoding: [0x01,0x02,0xfe,0x99]
 0x01,0x02,0xfe,0x99
+# GFX12: s_pack_lh_b32_b16 exec_lo, s1, s2       ; encoding: [0x01,0x02,0xfe,0x99]
 
-# GFX12: s_pack_lh_b32_b16 m0, s1, s2            ; encoding: [0x01,0x02,0xfd,0x99]
 0x01,0x02,0xfd,0x99
+# GFX12: s_pack_lh_b32_b16 m0, s1, s2            ; encoding: [0x01,0x02,0xfd,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, 0.5, s2           ; encoding: [0xf0,0x02,0x80,0x99]
 0xf0,0x02,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, 0.5, s2           ; encoding: [0xf0,0x02,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, 0, s2             ; encoding: [0x80,0x02,0x80,0x99]
 0x80,0x02,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, 0, s2             ; encoding: [0x80,0x02,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, 0x3f717273, s2    ; encoding: [0xff,0x02,0x80,0x99,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x80,0x99,0x73,0x72,0x71,0x3f
+# GFX12: s_pack_lh_b32_b16 s0, 0x3f717273, s2    ; encoding: [0xff,0x02,0x80,0x99,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_pack_lh_b32_b16 s0, 0xaf123456, s2    ; encoding: [0xff,0x02,0x80,0x99,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x80,0x99,0x56,0x34,0x12,0xaf
+# GFX12: s_pack_lh_b32_b16 s0, 0xaf123456, s2    ; encoding: [0xff,0x02,0x80,0x99,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_pack_lh_b32_b16 s0, -1, s2            ; encoding: [0xc1,0x02,0x80,0x99]
 0xc1,0x02,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, -1, s2            ; encoding: [0xc1,0x02,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, -4.0, s2          ; encoding: [0xf7,0x02,0x80,0x99]
 0xf7,0x02,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, -4.0, s2          ; encoding: [0xf7,0x02,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, exec_hi, s2       ; encoding: [0x7f,0x02,0x80,0x99]
 0x7f,0x02,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, exec_hi, s2       ; encoding: [0x7f,0x02,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, exec_lo, s2       ; encoding: [0x7e,0x02,0x80,0x99]
 0x7e,0x02,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, exec_lo, s2       ; encoding: [0x7e,0x02,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, m0, s2            ; encoding: [0x7d,0x02,0x80,0x99]
 0x7d,0x02,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, m0, s2            ; encoding: [0x7d,0x02,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, s104, s103        ; encoding: [0x68,0x67,0x80,0x99]
 0x68,0x67,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, s104, s103        ; encoding: [0x68,0x67,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, s104, s2          ; encoding: [0x68,0x02,0x80,0x99]
 0x68,0x02,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, s104, s2          ; encoding: [0x68,0x02,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, s1, 0.5           ; encoding: [0x01,0xf0,0x80,0x99]
 0x01,0xf0,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, s1, 0.5           ; encoding: [0x01,0xf0,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, s1, 0             ; encoding: [0x01,0x80,0x80,0x99]
 0x01,0x80,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, s1, 0             ; encoding: [0x01,0x80,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, s1, 0x3f717273    ; encoding: [0x01,0xff,0x80,0x99,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x80,0x99,0x73,0x72,0x71,0x3f
+# GFX12: s_pack_lh_b32_b16 s0, s1, 0x3f717273    ; encoding: [0x01,0xff,0x80,0x99,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_pack_lh_b32_b16 s0, s1, 0xaf123456    ; encoding: [0x01,0xff,0x80,0x99,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x80,0x99,0x56,0x34,0x12,0xaf
+# GFX12: s_pack_lh_b32_b16 s0, s1, 0xaf123456    ; encoding: [0x01,0xff,0x80,0x99,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_pack_lh_b32_b16 s0, s1, -1            ; encoding: [0x01,0xc1,0x80,0x99]
 0x01,0xc1,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, s1, -1            ; encoding: [0x01,0xc1,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, s1, -4.0          ; encoding: [0x01,0xf7,0x80,0x99]
 0x01,0xf7,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, s1, -4.0          ; encoding: [0x01,0xf7,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, s1, exec_hi       ; encoding: [0x01,0x7f,0x80,0x99]
 0x01,0x7f,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, s1, exec_hi       ; encoding: [0x01,0x7f,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, s1, exec_lo       ; encoding: [0x01,0x7e,0x80,0x99]
 0x01,0x7e,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, s1, exec_lo       ; encoding: [0x01,0x7e,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, s1, m0            ; encoding: [0x01,0x7d,0x80,0x99]
 0x01,0x7d,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, s1, m0            ; encoding: [0x01,0x7d,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, s1, s103          ; encoding: [0x01,0x67,0x80,0x99]
 0x01,0x67,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, s1, s103          ; encoding: [0x01,0x67,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, s1, s2            ; encoding: [0x01,0x02,0x80,0x99]
 0x01,0x02,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, s1, s2            ; encoding: [0x01,0x02,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, s1, vcc_hi        ; encoding: [0x01,0x6b,0x80,0x99]
 0x01,0x6b,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, s1, vcc_hi        ; encoding: [0x01,0x6b,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, s1, vcc_lo        ; encoding: [0x01,0x6a,0x80,0x99]
 0x01,0x6a,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, s1, vcc_lo        ; encoding: [0x01,0x6a,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, vcc_hi, s2        ; encoding: [0x6b,0x02,0x80,0x99]
 0x6b,0x02,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, vcc_hi, s2        ; encoding: [0x6b,0x02,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s0, vcc_lo, s2        ; encoding: [0x6a,0x02,0x80,0x99]
 0x6a,0x02,0x80,0x99
+# GFX12: s_pack_lh_b32_b16 s0, vcc_lo, s2        ; encoding: [0x6a,0x02,0x80,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s105, s104, s103      ; encoding: [0x68,0x67,0xe9,0x99]
 0x68,0x67,0xe9,0x99
+# GFX12: s_pack_lh_b32_b16 s105, s104, s103      ; encoding: [0x68,0x67,0xe9,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s105, s104, s2        ; encoding: [0x68,0x02,0xe9,0x99]
 0x68,0x02,0xe9,0x99
+# GFX12: s_pack_lh_b32_b16 s105, s104, s2        ; encoding: [0x68,0x02,0xe9,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s105, s1, s103        ; encoding: [0x01,0x67,0xe9,0x99]
 0x01,0x67,0xe9,0x99
+# GFX12: s_pack_lh_b32_b16 s105, s1, s103        ; encoding: [0x01,0x67,0xe9,0x99]
 
-# GFX12: s_pack_lh_b32_b16 s105, s1, s2          ; encoding: [0x01,0x02,0xe9,0x99]
 0x01,0x02,0xe9,0x99
+# GFX12: s_pack_lh_b32_b16 s105, s1, s2          ; encoding: [0x01,0x02,0xe9,0x99]
 
-# GFX12: s_pack_lh_b32_b16 vcc_hi, s1, s2        ; encoding: [0x01,0x02,0xeb,0x99]
 0x01,0x02,0xeb,0x99
+# GFX12: s_pack_lh_b32_b16 vcc_hi, s1, s2        ; encoding: [0x01,0x02,0xeb,0x99]
 
-# GFX12: s_pack_lh_b32_b16 vcc_lo, s1, s2        ; encoding: [0x01,0x02,0xea,0x99]
 0x01,0x02,0xea,0x99
+# GFX12: s_pack_lh_b32_b16 vcc_lo, s1, s2        ; encoding: [0x01,0x02,0xea,0x99]
 
-# GFX12: s_pack_ll_b32_b16 exec_hi, s1, s2       ; encoding: [0x01,0x02,0x7f,0x99]
 0x01,0x02,0x7f,0x99
+# GFX12: s_pack_ll_b32_b16 exec_hi, s1, s2       ; encoding: [0x01,0x02,0x7f,0x99]
 
-# GFX12: s_pack_ll_b32_b16 exec_lo, s1, s2       ; encoding: [0x01,0x02,0x7e,0x99]
 0x01,0x02,0x7e,0x99
+# GFX12: s_pack_ll_b32_b16 exec_lo, s1, s2       ; encoding: [0x01,0x02,0x7e,0x99]
 
-# GFX12: s_pack_ll_b32_b16 m0, s1, s2            ; encoding: [0x01,0x02,0x7d,0x99]
 0x01,0x02,0x7d,0x99
+# GFX12: s_pack_ll_b32_b16 m0, s1, s2            ; encoding: [0x01,0x02,0x7d,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, 0.5, s2           ; encoding: [0xf0,0x02,0x00,0x99]
 0xf0,0x02,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, 0.5, s2           ; encoding: [0xf0,0x02,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, 0, s2             ; encoding: [0x80,0x02,0x00,0x99]
 0x80,0x02,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, 0, s2             ; encoding: [0x80,0x02,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, 0x3f717273, s2    ; encoding: [0xff,0x02,0x00,0x99,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x99,0x73,0x72,0x71,0x3f
+# GFX12: s_pack_ll_b32_b16 s0, 0x3f717273, s2    ; encoding: [0xff,0x02,0x00,0x99,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_pack_ll_b32_b16 s0, 0xaf123456, s2    ; encoding: [0xff,0x02,0x00,0x99,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x99,0x56,0x34,0x12,0xaf
+# GFX12: s_pack_ll_b32_b16 s0, 0xaf123456, s2    ; encoding: [0xff,0x02,0x00,0x99,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_pack_ll_b32_b16 s0, -1, s2            ; encoding: [0xc1,0x02,0x00,0x99]
 0xc1,0x02,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, -1, s2            ; encoding: [0xc1,0x02,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, -4.0, s2          ; encoding: [0xf7,0x02,0x00,0x99]
 0xf7,0x02,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, -4.0, s2          ; encoding: [0xf7,0x02,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, exec_hi, s2       ; encoding: [0x7f,0x02,0x00,0x99]
 0x7f,0x02,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, exec_hi, s2       ; encoding: [0x7f,0x02,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, exec_lo, s2       ; encoding: [0x7e,0x02,0x00,0x99]
 0x7e,0x02,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, exec_lo, s2       ; encoding: [0x7e,0x02,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, m0, s2            ; encoding: [0x7d,0x02,0x00,0x99]
 0x7d,0x02,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, m0, s2            ; encoding: [0x7d,0x02,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, s104, s103        ; encoding: [0x68,0x67,0x00,0x99]
 0x68,0x67,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, s104, s103        ; encoding: [0x68,0x67,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, s104, s2          ; encoding: [0x68,0x02,0x00,0x99]
 0x68,0x02,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, s104, s2          ; encoding: [0x68,0x02,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, s1, 0.5           ; encoding: [0x01,0xf0,0x00,0x99]
 0x01,0xf0,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, s1, 0.5           ; encoding: [0x01,0xf0,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, s1, 0             ; encoding: [0x01,0x80,0x00,0x99]
 0x01,0x80,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, s1, 0             ; encoding: [0x01,0x80,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, s1, 0x3f717273    ; encoding: [0x01,0xff,0x00,0x99,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x99,0x73,0x72,0x71,0x3f
+# GFX12: s_pack_ll_b32_b16 s0, s1, 0x3f717273    ; encoding: [0x01,0xff,0x00,0x99,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_pack_ll_b32_b16 s0, s1, 0xaf123456    ; encoding: [0x01,0xff,0x00,0x99,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x99,0x56,0x34,0x12,0xaf
+# GFX12: s_pack_ll_b32_b16 s0, s1, 0xaf123456    ; encoding: [0x01,0xff,0x00,0x99,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_pack_ll_b32_b16 s0, s1, -1            ; encoding: [0x01,0xc1,0x00,0x99]
 0x01,0xc1,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, s1, -1            ; encoding: [0x01,0xc1,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, s1, -4.0          ; encoding: [0x01,0xf7,0x00,0x99]
 0x01,0xf7,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, s1, -4.0          ; encoding: [0x01,0xf7,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, s1, exec_hi       ; encoding: [0x01,0x7f,0x00,0x99]
 0x01,0x7f,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, s1, exec_hi       ; encoding: [0x01,0x7f,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, s1, exec_lo       ; encoding: [0x01,0x7e,0x00,0x99]
 0x01,0x7e,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, s1, exec_lo       ; encoding: [0x01,0x7e,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, s1, m0            ; encoding: [0x01,0x7d,0x00,0x99]
 0x01,0x7d,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, s1, m0            ; encoding: [0x01,0x7d,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, s1, s103          ; encoding: [0x01,0x67,0x00,0x99]
 0x01,0x67,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, s1, s103          ; encoding: [0x01,0x67,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, s1, s2            ; encoding: [0x01,0x02,0x00,0x99]
 0x01,0x02,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, s1, s2            ; encoding: [0x01,0x02,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, s1, vcc_hi        ; encoding: [0x01,0x6b,0x00,0x99]
 0x01,0x6b,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, s1, vcc_hi        ; encoding: [0x01,0x6b,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, s1, vcc_lo        ; encoding: [0x01,0x6a,0x00,0x99]
 0x01,0x6a,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, s1, vcc_lo        ; encoding: [0x01,0x6a,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, vcc_hi, s2        ; encoding: [0x6b,0x02,0x00,0x99]
 0x6b,0x02,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, vcc_hi, s2        ; encoding: [0x6b,0x02,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s0, vcc_lo, s2        ; encoding: [0x6a,0x02,0x00,0x99]
 0x6a,0x02,0x00,0x99
+# GFX12: s_pack_ll_b32_b16 s0, vcc_lo, s2        ; encoding: [0x6a,0x02,0x00,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s105, s104, s103      ; encoding: [0x68,0x67,0x69,0x99]
 0x68,0x67,0x69,0x99
+# GFX12: s_pack_ll_b32_b16 s105, s104, s103      ; encoding: [0x68,0x67,0x69,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s105, s104, s2        ; encoding: [0x68,0x02,0x69,0x99]
 0x68,0x02,0x69,0x99
+# GFX12: s_pack_ll_b32_b16 s105, s104, s2        ; encoding: [0x68,0x02,0x69,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s105, s1, s103        ; encoding: [0x01,0x67,0x69,0x99]
 0x01,0x67,0x69,0x99
+# GFX12: s_pack_ll_b32_b16 s105, s1, s103        ; encoding: [0x01,0x67,0x69,0x99]
 
-# GFX12: s_pack_ll_b32_b16 s105, s1, s2          ; encoding: [0x01,0x02,0x69,0x99]
 0x01,0x02,0x69,0x99
+# GFX12: s_pack_ll_b32_b16 s105, s1, s2          ; encoding: [0x01,0x02,0x69,0x99]
 
-# GFX12: s_pack_ll_b32_b16 vcc_hi, s1, s2        ; encoding: [0x01,0x02,0x6b,0x99]
 0x01,0x02,0x6b,0x99
+# GFX12: s_pack_ll_b32_b16 vcc_hi, s1, s2        ; encoding: [0x01,0x02,0x6b,0x99]
 
-# GFX12: s_pack_ll_b32_b16 vcc_lo, s1, s2        ; encoding: [0x01,0x02,0x6a,0x99]
 0x01,0x02,0x6a,0x99
+# GFX12: s_pack_ll_b32_b16 vcc_lo, s1, s2        ; encoding: [0x01,0x02,0x6a,0x99]
 
-# GFX12: s_sub_co_ci_u32 exec_hi, s1, s2         ; encoding: [0x01,0x02,0xff,0x82]
 0x01,0x02,0xff,0x82
+# GFX12: s_sub_co_ci_u32 exec_hi, s1, s2         ; encoding: [0x01,0x02,0xff,0x82]
 
-# GFX12: s_sub_co_ci_u32 exec_lo, s1, s2         ; encoding: [0x01,0x02,0xfe,0x82]
 0x01,0x02,0xfe,0x82
+# GFX12: s_sub_co_ci_u32 exec_lo, s1, s2         ; encoding: [0x01,0x02,0xfe,0x82]
 
-# GFX12: s_sub_co_ci_u32 m0, s1, s2              ; encoding: [0x01,0x02,0xfd,0x82]
 0x01,0x02,0xfd,0x82
+# GFX12: s_sub_co_ci_u32 m0, s1, s2              ; encoding: [0x01,0x02,0xfd,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, 0.5, s2             ; encoding: [0xf0,0x02,0x80,0x82]
 0xf0,0x02,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, 0.5, s2             ; encoding: [0xf0,0x02,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, 0, s2               ; encoding: [0x80,0x02,0x80,0x82]
 0x80,0x02,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, 0, s2               ; encoding: [0x80,0x02,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, 0x3f717273, s2      ; encoding: [0xff,0x02,0x80,0x82,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x80,0x82,0x73,0x72,0x71,0x3f
+# GFX12: s_sub_co_ci_u32 s0, 0x3f717273, s2      ; encoding: [0xff,0x02,0x80,0x82,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_sub_co_ci_u32 s0, 0xaf123456, s2      ; encoding: [0xff,0x02,0x80,0x82,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x80,0x82,0x56,0x34,0x12,0xaf
+# GFX12: s_sub_co_ci_u32 s0, 0xaf123456, s2      ; encoding: [0xff,0x02,0x80,0x82,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_sub_co_ci_u32 s0, -1, s2              ; encoding: [0xc1,0x02,0x80,0x82]
 0xc1,0x02,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, -1, s2              ; encoding: [0xc1,0x02,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, -4.0, s2            ; encoding: [0xf7,0x02,0x80,0x82]
 0xf7,0x02,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, -4.0, s2            ; encoding: [0xf7,0x02,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, exec_hi, s2         ; encoding: [0x7f,0x02,0x80,0x82]
 0x7f,0x02,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, exec_hi, s2         ; encoding: [0x7f,0x02,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, exec_lo, s2         ; encoding: [0x7e,0x02,0x80,0x82]
 0x7e,0x02,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, exec_lo, s2         ; encoding: [0x7e,0x02,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, m0, s2              ; encoding: [0x7d,0x02,0x80,0x82]
 0x7d,0x02,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, m0, s2              ; encoding: [0x7d,0x02,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, s104, s103          ; encoding: [0x68,0x67,0x80,0x82]
 0x68,0x67,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, s104, s103          ; encoding: [0x68,0x67,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, s104, s2            ; encoding: [0x68,0x02,0x80,0x82]
 0x68,0x02,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, s104, s2            ; encoding: [0x68,0x02,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, s1, 0.5             ; encoding: [0x01,0xf0,0x80,0x82]
 0x01,0xf0,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, s1, 0.5             ; encoding: [0x01,0xf0,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, s1, 0               ; encoding: [0x01,0x80,0x80,0x82]
 0x01,0x80,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, s1, 0               ; encoding: [0x01,0x80,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, s1, 0x3f717273      ; encoding: [0x01,0xff,0x80,0x82,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x80,0x82,0x73,0x72,0x71,0x3f
+# GFX12: s_sub_co_ci_u32 s0, s1, 0x3f717273      ; encoding: [0x01,0xff,0x80,0x82,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_sub_co_ci_u32 s0, s1, 0xaf123456      ; encoding: [0x01,0xff,0x80,0x82,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x80,0x82,0x56,0x34,0x12,0xaf
+# GFX12: s_sub_co_ci_u32 s0, s1, 0xaf123456      ; encoding: [0x01,0xff,0x80,0x82,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_sub_co_ci_u32 s0, s1, -1              ; encoding: [0x01,0xc1,0x80,0x82]
 0x01,0xc1,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, s1, -1              ; encoding: [0x01,0xc1,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, s1, -4.0            ; encoding: [0x01,0xf7,0x80,0x82]
 0x01,0xf7,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, s1, -4.0            ; encoding: [0x01,0xf7,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, s1, exec_hi         ; encoding: [0x01,0x7f,0x80,0x82]
 0x01,0x7f,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, s1, exec_hi         ; encoding: [0x01,0x7f,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, s1, exec_lo         ; encoding: [0x01,0x7e,0x80,0x82]
 0x01,0x7e,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, s1, exec_lo         ; encoding: [0x01,0x7e,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, s1, m0              ; encoding: [0x01,0x7d,0x80,0x82]
 0x01,0x7d,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, s1, m0              ; encoding: [0x01,0x7d,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, s1, s103            ; encoding: [0x01,0x67,0x80,0x82]
 0x01,0x67,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, s1, s103            ; encoding: [0x01,0x67,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, s1, s2              ; encoding: [0x01,0x02,0x80,0x82]
 0x01,0x02,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, s1, s2              ; encoding: [0x01,0x02,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, s1, vcc_hi          ; encoding: [0x01,0x6b,0x80,0x82]
 0x01,0x6b,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, s1, vcc_hi          ; encoding: [0x01,0x6b,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, s1, vcc_lo          ; encoding: [0x01,0x6a,0x80,0x82]
 0x01,0x6a,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, s1, vcc_lo          ; encoding: [0x01,0x6a,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, vcc_hi, s2          ; encoding: [0x6b,0x02,0x80,0x82]
 0x6b,0x02,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, vcc_hi, s2          ; encoding: [0x6b,0x02,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s0, vcc_lo, s2          ; encoding: [0x6a,0x02,0x80,0x82]
 0x6a,0x02,0x80,0x82
+# GFX12: s_sub_co_ci_u32 s0, vcc_lo, s2          ; encoding: [0x6a,0x02,0x80,0x82]
 
-# GFX12: s_sub_co_ci_u32 s105, s104, s103        ; encoding: [0x68,0x67,0xe9,0x82]
 0x68,0x67,0xe9,0x82
+# GFX12: s_sub_co_ci_u32 s105, s104, s103        ; encoding: [0x68,0x67,0xe9,0x82]
 
-# GFX12: s_sub_co_ci_u32 s105, s104, s2          ; encoding: [0x68,0x02,0xe9,0x82]
 0x68,0x02,0xe9,0x82
+# GFX12: s_sub_co_ci_u32 s105, s104, s2          ; encoding: [0x68,0x02,0xe9,0x82]
 
-# GFX12: s_sub_co_ci_u32 s105, s1, s103          ; encoding: [0x01,0x67,0xe9,0x82]
 0x01,0x67,0xe9,0x82
+# GFX12: s_sub_co_ci_u32 s105, s1, s103          ; encoding: [0x01,0x67,0xe9,0x82]
 
-# GFX12: s_sub_co_ci_u32 s105, s1, s2            ; encoding: [0x01,0x02,0xe9,0x82]
 0x01,0x02,0xe9,0x82
+# GFX12: s_sub_co_ci_u32 s105, s1, s2            ; encoding: [0x01,0x02,0xe9,0x82]
 
-# GFX12: s_sub_co_ci_u32 vcc_hi, s1, s2          ; encoding: [0x01,0x02,0xeb,0x82]
 0x01,0x02,0xeb,0x82
+# GFX12: s_sub_co_ci_u32 vcc_hi, s1, s2          ; encoding: [0x01,0x02,0xeb,0x82]
 
-# GFX12: s_sub_co_ci_u32 vcc_lo, s1, s2          ; encoding: [0x01,0x02,0xea,0x82]
 0x01,0x02,0xea,0x82
+# GFX12: s_sub_co_ci_u32 vcc_lo, s1, s2          ; encoding: [0x01,0x02,0xea,0x82]
 
-# GFX12: s_sub_co_i32 exec_hi, s1, s2            ; encoding: [0x01,0x02,0xff,0x81]
 0x01,0x02,0xff,0x81
+# GFX12: s_sub_co_i32 exec_hi, s1, s2            ; encoding: [0x01,0x02,0xff,0x81]
 
-# GFX12: s_sub_co_i32 exec_lo, s1, s2            ; encoding: [0x01,0x02,0xfe,0x81]
 0x01,0x02,0xfe,0x81
+# GFX12: s_sub_co_i32 exec_lo, s1, s2            ; encoding: [0x01,0x02,0xfe,0x81]
 
-# GFX12: s_sub_co_i32 m0, s1, s2                 ; encoding: [0x01,0x02,0xfd,0x81]
 0x01,0x02,0xfd,0x81
+# GFX12: s_sub_co_i32 m0, s1, s2                 ; encoding: [0x01,0x02,0xfd,0x81]
 
-# GFX12: s_sub_co_i32 s0, 0.5, s2                ; encoding: [0xf0,0x02,0x80,0x81]
 0xf0,0x02,0x80,0x81
+# GFX12: s_sub_co_i32 s0, 0.5, s2                ; encoding: [0xf0,0x02,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, 0, s2                  ; encoding: [0x80,0x02,0x80,0x81]
 0x80,0x02,0x80,0x81
+# GFX12: s_sub_co_i32 s0, 0, s2                  ; encoding: [0x80,0x02,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, 0x3f717273, s2         ; encoding: [0xff,0x02,0x80,0x81,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x80,0x81,0x73,0x72,0x71,0x3f
+# GFX12: s_sub_co_i32 s0, 0x3f717273, s2         ; encoding: [0xff,0x02,0x80,0x81,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_sub_co_i32 s0, 0xaf123456, s2         ; encoding: [0xff,0x02,0x80,0x81,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x80,0x81,0x56,0x34,0x12,0xaf
+# GFX12: s_sub_co_i32 s0, 0xaf123456, s2         ; encoding: [0xff,0x02,0x80,0x81,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_sub_co_i32 s0, -1, s2                 ; encoding: [0xc1,0x02,0x80,0x81]
 0xc1,0x02,0x80,0x81
+# GFX12: s_sub_co_i32 s0, -1, s2                 ; encoding: [0xc1,0x02,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, -4.0, s2               ; encoding: [0xf7,0x02,0x80,0x81]
 0xf7,0x02,0x80,0x81
+# GFX12: s_sub_co_i32 s0, -4.0, s2               ; encoding: [0xf7,0x02,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, exec_hi, s2            ; encoding: [0x7f,0x02,0x80,0x81]
 0x7f,0x02,0x80,0x81
+# GFX12: s_sub_co_i32 s0, exec_hi, s2            ; encoding: [0x7f,0x02,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, exec_lo, s2            ; encoding: [0x7e,0x02,0x80,0x81]
 0x7e,0x02,0x80,0x81
+# GFX12: s_sub_co_i32 s0, exec_lo, s2            ; encoding: [0x7e,0x02,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, m0, s2                 ; encoding: [0x7d,0x02,0x80,0x81]
 0x7d,0x02,0x80,0x81
+# GFX12: s_sub_co_i32 s0, m0, s2                 ; encoding: [0x7d,0x02,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, s104, s103             ; encoding: [0x68,0x67,0x80,0x81]
 0x68,0x67,0x80,0x81
+# GFX12: s_sub_co_i32 s0, s104, s103             ; encoding: [0x68,0x67,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, s104, s2               ; encoding: [0x68,0x02,0x80,0x81]
 0x68,0x02,0x80,0x81
+# GFX12: s_sub_co_i32 s0, s104, s2               ; encoding: [0x68,0x02,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, s1, 0.5                ; encoding: [0x01,0xf0,0x80,0x81]
 0x01,0xf0,0x80,0x81
+# GFX12: s_sub_co_i32 s0, s1, 0.5                ; encoding: [0x01,0xf0,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, s1, 0                  ; encoding: [0x01,0x80,0x80,0x81]
 0x01,0x80,0x80,0x81
+# GFX12: s_sub_co_i32 s0, s1, 0                  ; encoding: [0x01,0x80,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, s1, 0x3f717273         ; encoding: [0x01,0xff,0x80,0x81,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x80,0x81,0x73,0x72,0x71,0x3f
+# GFX12: s_sub_co_i32 s0, s1, 0x3f717273         ; encoding: [0x01,0xff,0x80,0x81,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_sub_co_i32 s0, s1, 0xaf123456         ; encoding: [0x01,0xff,0x80,0x81,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x80,0x81,0x56,0x34,0x12,0xaf
+# GFX12: s_sub_co_i32 s0, s1, 0xaf123456         ; encoding: [0x01,0xff,0x80,0x81,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_sub_co_i32 s0, s1, -1                 ; encoding: [0x01,0xc1,0x80,0x81]
 0x01,0xc1,0x80,0x81
+# GFX12: s_sub_co_i32 s0, s1, -1                 ; encoding: [0x01,0xc1,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, s1, -4.0               ; encoding: [0x01,0xf7,0x80,0x81]
 0x01,0xf7,0x80,0x81
+# GFX12: s_sub_co_i32 s0, s1, -4.0               ; encoding: [0x01,0xf7,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, s1, exec_hi            ; encoding: [0x01,0x7f,0x80,0x81]
 0x01,0x7f,0x80,0x81
+# GFX12: s_sub_co_i32 s0, s1, exec_hi            ; encoding: [0x01,0x7f,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, s1, exec_lo            ; encoding: [0x01,0x7e,0x80,0x81]
 0x01,0x7e,0x80,0x81
+# GFX12: s_sub_co_i32 s0, s1, exec_lo            ; encoding: [0x01,0x7e,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, s1, m0                 ; encoding: [0x01,0x7d,0x80,0x81]
 0x01,0x7d,0x80,0x81
+# GFX12: s_sub_co_i32 s0, s1, m0                 ; encoding: [0x01,0x7d,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, s1, s103               ; encoding: [0x01,0x67,0x80,0x81]
 0x01,0x67,0x80,0x81
+# GFX12: s_sub_co_i32 s0, s1, s103               ; encoding: [0x01,0x67,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, s1, s2                 ; encoding: [0x01,0x02,0x80,0x81]
 0x01,0x02,0x80,0x81
+# GFX12: s_sub_co_i32 s0, s1, s2                 ; encoding: [0x01,0x02,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, s1, vcc_hi             ; encoding: [0x01,0x6b,0x80,0x81]
 0x01,0x6b,0x80,0x81
+# GFX12: s_sub_co_i32 s0, s1, vcc_hi             ; encoding: [0x01,0x6b,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, s1, vcc_lo             ; encoding: [0x01,0x6a,0x80,0x81]
 0x01,0x6a,0x80,0x81
+# GFX12: s_sub_co_i32 s0, s1, vcc_lo             ; encoding: [0x01,0x6a,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, vcc_hi, s2             ; encoding: [0x6b,0x02,0x80,0x81]
 0x6b,0x02,0x80,0x81
+# GFX12: s_sub_co_i32 s0, vcc_hi, s2             ; encoding: [0x6b,0x02,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s0, vcc_lo, s2             ; encoding: [0x6a,0x02,0x80,0x81]
 0x6a,0x02,0x80,0x81
+# GFX12: s_sub_co_i32 s0, vcc_lo, s2             ; encoding: [0x6a,0x02,0x80,0x81]
 
-# GFX12: s_sub_co_i32 s105, s104, s103           ; encoding: [0x68,0x67,0xe9,0x81]
 0x68,0x67,0xe9,0x81
+# GFX12: s_sub_co_i32 s105, s104, s103           ; encoding: [0x68,0x67,0xe9,0x81]
 
-# GFX12: s_sub_co_i32 s105, s104, s2             ; encoding: [0x68,0x02,0xe9,0x81]
 0x68,0x02,0xe9,0x81
+# GFX12: s_sub_co_i32 s105, s104, s2             ; encoding: [0x68,0x02,0xe9,0x81]
 
-# GFX12: s_sub_co_i32 s105, s1, s103             ; encoding: [0x01,0x67,0xe9,0x81]
 0x01,0x67,0xe9,0x81
+# GFX12: s_sub_co_i32 s105, s1, s103             ; encoding: [0x01,0x67,0xe9,0x81]
 
-# GFX12: s_sub_co_i32 s105, s1, s2               ; encoding: [0x01,0x02,0xe9,0x81]
 0x01,0x02,0xe9,0x81
+# GFX12: s_sub_co_i32 s105, s1, s2               ; encoding: [0x01,0x02,0xe9,0x81]
 
-# GFX12: s_sub_co_i32 vcc_hi, s1, s2             ; encoding: [0x01,0x02,0xeb,0x81]
 0x01,0x02,0xeb,0x81
+# GFX12: s_sub_co_i32 vcc_hi, s1, s2             ; encoding: [0x01,0x02,0xeb,0x81]
 
-# GFX12: s_sub_co_i32 vcc_lo, s1, s2             ; encoding: [0x01,0x02,0xea,0x81]
 0x01,0x02,0xea,0x81
+# GFX12: s_sub_co_i32 vcc_lo, s1, s2             ; encoding: [0x01,0x02,0xea,0x81]
 
-# GFX12: s_sub_co_u32 exec_hi, s1, s2            ; encoding: [0x01,0x02,0xff,0x80]
 0x01,0x02,0xff,0x80
+# GFX12: s_sub_co_u32 exec_hi, s1, s2            ; encoding: [0x01,0x02,0xff,0x80]
 
-# GFX12: s_sub_co_u32 exec_lo, s1, s2            ; encoding: [0x01,0x02,0xfe,0x80]
 0x01,0x02,0xfe,0x80
+# GFX12: s_sub_co_u32 exec_lo, s1, s2            ; encoding: [0x01,0x02,0xfe,0x80]
 
-# GFX12: s_sub_co_u32 m0, s1, s2                 ; encoding: [0x01,0x02,0xfd,0x80]
 0x01,0x02,0xfd,0x80
+# GFX12: s_sub_co_u32 m0, s1, s2                 ; encoding: [0x01,0x02,0xfd,0x80]
 
-# GFX12: s_sub_co_u32 s0, 0.5, s2                ; encoding: [0xf0,0x02,0x80,0x80]
 0xf0,0x02,0x80,0x80
+# GFX12: s_sub_co_u32 s0, 0.5, s2                ; encoding: [0xf0,0x02,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, 0, s2                  ; encoding: [0x80,0x02,0x80,0x80]
 0x80,0x02,0x80,0x80
+# GFX12: s_sub_co_u32 s0, 0, s2                  ; encoding: [0x80,0x02,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, 0x3f717273, s2         ; encoding: [0xff,0x02,0x80,0x80,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x80,0x80,0x73,0x72,0x71,0x3f
+# GFX12: s_sub_co_u32 s0, 0x3f717273, s2         ; encoding: [0xff,0x02,0x80,0x80,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_sub_co_u32 s0, 0xaf123456, s2         ; encoding: [0xff,0x02,0x80,0x80,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x80,0x80,0x56,0x34,0x12,0xaf
+# GFX12: s_sub_co_u32 s0, 0xaf123456, s2         ; encoding: [0xff,0x02,0x80,0x80,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_sub_co_u32 s0, -1, s2                 ; encoding: [0xc1,0x02,0x80,0x80]
 0xc1,0x02,0x80,0x80
+# GFX12: s_sub_co_u32 s0, -1, s2                 ; encoding: [0xc1,0x02,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, -4.0, s2               ; encoding: [0xf7,0x02,0x80,0x80]
 0xf7,0x02,0x80,0x80
+# GFX12: s_sub_co_u32 s0, -4.0, s2               ; encoding: [0xf7,0x02,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, exec_hi, s2            ; encoding: [0x7f,0x02,0x80,0x80]
 0x7f,0x02,0x80,0x80
+# GFX12: s_sub_co_u32 s0, exec_hi, s2            ; encoding: [0x7f,0x02,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, exec_lo, s2            ; encoding: [0x7e,0x02,0x80,0x80]
 0x7e,0x02,0x80,0x80
+# GFX12: s_sub_co_u32 s0, exec_lo, s2            ; encoding: [0x7e,0x02,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, m0, s2                 ; encoding: [0x7d,0x02,0x80,0x80]
 0x7d,0x02,0x80,0x80
+# GFX12: s_sub_co_u32 s0, m0, s2                 ; encoding: [0x7d,0x02,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, s104, s103             ; encoding: [0x68,0x67,0x80,0x80]
 0x68,0x67,0x80,0x80
+# GFX12: s_sub_co_u32 s0, s104, s103             ; encoding: [0x68,0x67,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, s104, s2               ; encoding: [0x68,0x02,0x80,0x80]
 0x68,0x02,0x80,0x80
+# GFX12: s_sub_co_u32 s0, s104, s2               ; encoding: [0x68,0x02,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, s1, 0.5                ; encoding: [0x01,0xf0,0x80,0x80]
 0x01,0xf0,0x80,0x80
+# GFX12: s_sub_co_u32 s0, s1, 0.5                ; encoding: [0x01,0xf0,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, s1, 0                  ; encoding: [0x01,0x80,0x80,0x80]
 0x01,0x80,0x80,0x80
+# GFX12: s_sub_co_u32 s0, s1, 0                  ; encoding: [0x01,0x80,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, s1, 0x3f717273         ; encoding: [0x01,0xff,0x80,0x80,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x80,0x80,0x73,0x72,0x71,0x3f
+# GFX12: s_sub_co_u32 s0, s1, 0x3f717273         ; encoding: [0x01,0xff,0x80,0x80,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_sub_co_u32 s0, s1, 0xaf123456         ; encoding: [0x01,0xff,0x80,0x80,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x80,0x80,0x56,0x34,0x12,0xaf
+# GFX12: s_sub_co_u32 s0, s1, 0xaf123456         ; encoding: [0x01,0xff,0x80,0x80,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_sub_co_u32 s0, s1, -1                 ; encoding: [0x01,0xc1,0x80,0x80]
 0x01,0xc1,0x80,0x80
+# GFX12: s_sub_co_u32 s0, s1, -1                 ; encoding: [0x01,0xc1,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, s1, -4.0               ; encoding: [0x01,0xf7,0x80,0x80]
 0x01,0xf7,0x80,0x80
+# GFX12: s_sub_co_u32 s0, s1, -4.0               ; encoding: [0x01,0xf7,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, s1, exec_hi            ; encoding: [0x01,0x7f,0x80,0x80]
 0x01,0x7f,0x80,0x80
+# GFX12: s_sub_co_u32 s0, s1, exec_hi            ; encoding: [0x01,0x7f,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, s1, exec_lo            ; encoding: [0x01,0x7e,0x80,0x80]
 0x01,0x7e,0x80,0x80
+# GFX12: s_sub_co_u32 s0, s1, exec_lo            ; encoding: [0x01,0x7e,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, s1, m0                 ; encoding: [0x01,0x7d,0x80,0x80]
 0x01,0x7d,0x80,0x80
+# GFX12: s_sub_co_u32 s0, s1, m0                 ; encoding: [0x01,0x7d,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, s1, s103               ; encoding: [0x01,0x67,0x80,0x80]
 0x01,0x67,0x80,0x80
+# GFX12: s_sub_co_u32 s0, s1, s103               ; encoding: [0x01,0x67,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, s1, s2                 ; encoding: [0x01,0x02,0x80,0x80]
 0x01,0x02,0x80,0x80
+# GFX12: s_sub_co_u32 s0, s1, s2                 ; encoding: [0x01,0x02,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, s1, vcc_hi             ; encoding: [0x01,0x6b,0x80,0x80]
 0x01,0x6b,0x80,0x80
+# GFX12: s_sub_co_u32 s0, s1, vcc_hi             ; encoding: [0x01,0x6b,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, s1, vcc_lo             ; encoding: [0x01,0x6a,0x80,0x80]
 0x01,0x6a,0x80,0x80
+# GFX12: s_sub_co_u32 s0, s1, vcc_lo             ; encoding: [0x01,0x6a,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, vcc_hi, s2             ; encoding: [0x6b,0x02,0x80,0x80]
 0x6b,0x02,0x80,0x80
+# GFX12: s_sub_co_u32 s0, vcc_hi, s2             ; encoding: [0x6b,0x02,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s0, vcc_lo, s2             ; encoding: [0x6a,0x02,0x80,0x80]
 0x6a,0x02,0x80,0x80
+# GFX12: s_sub_co_u32 s0, vcc_lo, s2             ; encoding: [0x6a,0x02,0x80,0x80]
 
-# GFX12: s_sub_co_u32 s105, s104, s103           ; encoding: [0x68,0x67,0xe9,0x80]
 0x68,0x67,0xe9,0x80
+# GFX12: s_sub_co_u32 s105, s104, s103           ; encoding: [0x68,0x67,0xe9,0x80]
 
-# GFX12: s_sub_co_u32 s105, s104, s2             ; encoding: [0x68,0x02,0xe9,0x80]
 0x68,0x02,0xe9,0x80
+# GFX12: s_sub_co_u32 s105, s104, s2             ; encoding: [0x68,0x02,0xe9,0x80]
 
-# GFX12: s_sub_co_u32 s105, s1, s103             ; encoding: [0x01,0x67,0xe9,0x80]
 0x01,0x67,0xe9,0x80
+# GFX12: s_sub_co_u32 s105, s1, s103             ; encoding: [0x01,0x67,0xe9,0x80]
 
-# GFX12: s_sub_co_u32 s105, s1, s2               ; encoding: [0x01,0x02,0xe9,0x80]
 0x01,0x02,0xe9,0x80
+# GFX12: s_sub_co_u32 s105, s1, s2               ; encoding: [0x01,0x02,0xe9,0x80]
 
-# GFX12: s_sub_co_u32 vcc_hi, s1, s2             ; encoding: [0x01,0x02,0xeb,0x80]
 0x01,0x02,0xeb,0x80
+# GFX12: s_sub_co_u32 vcc_hi, s1, s2             ; encoding: [0x01,0x02,0xeb,0x80]
 
-# GFX12: s_sub_co_u32 vcc_lo, s1, s2             ; encoding: [0x01,0x02,0xea,0x80]
 0x01,0x02,0xea,0x80
+# GFX12: s_sub_co_u32 vcc_lo, s1, s2             ; encoding: [0x01,0x02,0xea,0x80]
 
-# GFX12: s_xnor_b32 exec_hi, s1, s2              ; encoding: [0x01,0x02,0x7f,0x90]
 0x01,0x02,0x7f,0x90
+# GFX12: s_xnor_b32 exec_hi, s1, s2              ; encoding: [0x01,0x02,0x7f,0x90]
 
-# GFX12: s_xnor_b32 exec_lo, s1, s2              ; encoding: [0x01,0x02,0x7e,0x90]
 0x01,0x02,0x7e,0x90
+# GFX12: s_xnor_b32 exec_lo, s1, s2              ; encoding: [0x01,0x02,0x7e,0x90]
 
-# GFX12: s_xnor_b32 m0, s1, s2                   ; encoding: [0x01,0x02,0x7d,0x90]
 0x01,0x02,0x7d,0x90
+# GFX12: s_xnor_b32 m0, s1, s2                   ; encoding: [0x01,0x02,0x7d,0x90]
 
-# GFX12: s_xnor_b32 s0, 0.5, s2                  ; encoding: [0xf0,0x02,0x00,0x90]
 0xf0,0x02,0x00,0x90
+# GFX12: s_xnor_b32 s0, 0.5, s2                  ; encoding: [0xf0,0x02,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, 0, s2                    ; encoding: [0x80,0x02,0x00,0x90]
 0x80,0x02,0x00,0x90
+# GFX12: s_xnor_b32 s0, 0, s2                    ; encoding: [0x80,0x02,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, 0x3f717273, s2           ; encoding: [0xff,0x02,0x00,0x90,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x90,0x73,0x72,0x71,0x3f
+# GFX12: s_xnor_b32 s0, 0x3f717273, s2           ; encoding: [0xff,0x02,0x00,0x90,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_xnor_b32 s0, 0xaf123456, s2           ; encoding: [0xff,0x02,0x00,0x90,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x90,0x56,0x34,0x12,0xaf
+# GFX12: s_xnor_b32 s0, 0xaf123456, s2           ; encoding: [0xff,0x02,0x00,0x90,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_xnor_b32 s0, -1, s2                   ; encoding: [0xc1,0x02,0x00,0x90]
 0xc1,0x02,0x00,0x90
+# GFX12: s_xnor_b32 s0, -1, s2                   ; encoding: [0xc1,0x02,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, -4.0, s2                 ; encoding: [0xf7,0x02,0x00,0x90]
 0xf7,0x02,0x00,0x90
+# GFX12: s_xnor_b32 s0, -4.0, s2                 ; encoding: [0xf7,0x02,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, exec_hi, s2              ; encoding: [0x7f,0x02,0x00,0x90]
 0x7f,0x02,0x00,0x90
+# GFX12: s_xnor_b32 s0, exec_hi, s2              ; encoding: [0x7f,0x02,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, exec_lo, s2              ; encoding: [0x7e,0x02,0x00,0x90]
 0x7e,0x02,0x00,0x90
+# GFX12: s_xnor_b32 s0, exec_lo, s2              ; encoding: [0x7e,0x02,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, m0, s2                   ; encoding: [0x7d,0x02,0x00,0x90]
 0x7d,0x02,0x00,0x90
+# GFX12: s_xnor_b32 s0, m0, s2                   ; encoding: [0x7d,0x02,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, s104, s103               ; encoding: [0x68,0x67,0x00,0x90]
 0x68,0x67,0x00,0x90
+# GFX12: s_xnor_b32 s0, s104, s103               ; encoding: [0x68,0x67,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, s104, s2                 ; encoding: [0x68,0x02,0x00,0x90]
 0x68,0x02,0x00,0x90
+# GFX12: s_xnor_b32 s0, s104, s2                 ; encoding: [0x68,0x02,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, s1, 0.5                  ; encoding: [0x01,0xf0,0x00,0x90]
 0x01,0xf0,0x00,0x90
+# GFX12: s_xnor_b32 s0, s1, 0.5                  ; encoding: [0x01,0xf0,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, s1, 0                    ; encoding: [0x01,0x80,0x00,0x90]
 0x01,0x80,0x00,0x90
+# GFX12: s_xnor_b32 s0, s1, 0                    ; encoding: [0x01,0x80,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, s1, 0x3f717273           ; encoding: [0x01,0xff,0x00,0x90,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x90,0x73,0x72,0x71,0x3f
+# GFX12: s_xnor_b32 s0, s1, 0x3f717273           ; encoding: [0x01,0xff,0x00,0x90,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_xnor_b32 s0, s1, 0xaf123456           ; encoding: [0x01,0xff,0x00,0x90,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x90,0x56,0x34,0x12,0xaf
+# GFX12: s_xnor_b32 s0, s1, 0xaf123456           ; encoding: [0x01,0xff,0x00,0x90,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_xnor_b32 s0, s1, -1                   ; encoding: [0x01,0xc1,0x00,0x90]
 0x01,0xc1,0x00,0x90
+# GFX12: s_xnor_b32 s0, s1, -1                   ; encoding: [0x01,0xc1,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, s1, -4.0                 ; encoding: [0x01,0xf7,0x00,0x90]
 0x01,0xf7,0x00,0x90
+# GFX12: s_xnor_b32 s0, s1, -4.0                 ; encoding: [0x01,0xf7,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, s1, exec_hi              ; encoding: [0x01,0x7f,0x00,0x90]
 0x01,0x7f,0x00,0x90
+# GFX12: s_xnor_b32 s0, s1, exec_hi              ; encoding: [0x01,0x7f,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, s1, exec_lo              ; encoding: [0x01,0x7e,0x00,0x90]
 0x01,0x7e,0x00,0x90
+# GFX12: s_xnor_b32 s0, s1, exec_lo              ; encoding: [0x01,0x7e,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, s1, m0                   ; encoding: [0x01,0x7d,0x00,0x90]
 0x01,0x7d,0x00,0x90
+# GFX12: s_xnor_b32 s0, s1, m0                   ; encoding: [0x01,0x7d,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, s1, s103                 ; encoding: [0x01,0x67,0x00,0x90]
 0x01,0x67,0x00,0x90
+# GFX12: s_xnor_b32 s0, s1, s103                 ; encoding: [0x01,0x67,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, s1, s2                   ; encoding: [0x01,0x02,0x00,0x90]
 0x01,0x02,0x00,0x90
+# GFX12: s_xnor_b32 s0, s1, s2                   ; encoding: [0x01,0x02,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, s1, vcc_hi               ; encoding: [0x01,0x6b,0x00,0x90]
 0x01,0x6b,0x00,0x90
+# GFX12: s_xnor_b32 s0, s1, vcc_hi               ; encoding: [0x01,0x6b,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, s1, vcc_lo               ; encoding: [0x01,0x6a,0x00,0x90]
 0x01,0x6a,0x00,0x90
+# GFX12: s_xnor_b32 s0, s1, vcc_lo               ; encoding: [0x01,0x6a,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, vcc_hi, s2               ; encoding: [0x6b,0x02,0x00,0x90]
 0x6b,0x02,0x00,0x90
+# GFX12: s_xnor_b32 s0, vcc_hi, s2               ; encoding: [0x6b,0x02,0x00,0x90]
 
-# GFX12: s_xnor_b32 s0, vcc_lo, s2               ; encoding: [0x6a,0x02,0x00,0x90]
 0x6a,0x02,0x00,0x90
+# GFX12: s_xnor_b32 s0, vcc_lo, s2               ; encoding: [0x6a,0x02,0x00,0x90]
 
-# GFX12: s_xnor_b32 s105, s104, s103             ; encoding: [0x68,0x67,0x69,0x90]
 0x68,0x67,0x69,0x90
+# GFX12: s_xnor_b32 s105, s104, s103             ; encoding: [0x68,0x67,0x69,0x90]
 
-# GFX12: s_xnor_b32 s105, s104, s2               ; encoding: [0x68,0x02,0x69,0x90]
 0x68,0x02,0x69,0x90
+# GFX12: s_xnor_b32 s105, s104, s2               ; encoding: [0x68,0x02,0x69,0x90]
 
-# GFX12: s_xnor_b32 s105, s1, s103               ; encoding: [0x01,0x67,0x69,0x90]
 0x01,0x67,0x69,0x90
+# GFX12: s_xnor_b32 s105, s1, s103               ; encoding: [0x01,0x67,0x69,0x90]
 
-# GFX12: s_xnor_b32 s105, s1, s2                 ; encoding: [0x01,0x02,0x69,0x90]
 0x01,0x02,0x69,0x90
+# GFX12: s_xnor_b32 s105, s1, s2                 ; encoding: [0x01,0x02,0x69,0x90]
 
-# GFX12: s_xnor_b32 vcc_hi, s1, s2               ; encoding: [0x01,0x02,0x6b,0x90]
 0x01,0x02,0x6b,0x90
+# GFX12: s_xnor_b32 vcc_hi, s1, s2               ; encoding: [0x01,0x02,0x6b,0x90]
 
-# GFX12: s_xnor_b32 vcc_lo, s1, s2               ; encoding: [0x01,0x02,0x6a,0x90]
 0x01,0x02,0x6a,0x90
+# GFX12: s_xnor_b32 vcc_lo, s1, s2               ; encoding: [0x01,0x02,0x6a,0x90]
 
-# GFX12: s_xnor_b64 exec, s[2:3], s[4:5]         ; encoding: [0x02,0x04,0xfe,0x90]
 0x02,0x04,0xfe,0x90
+# GFX12: s_xnor_b64 exec, s[2:3], s[4:5]         ; encoding: [0x02,0x04,0xfe,0x90]
 
-# GFX12: s_xnor_b64 s[0:1], 0.5, s[4:5]          ; encoding: [0xf0,0x04,0x80,0x90]
 0xf0,0x04,0x80,0x90
+# GFX12: s_xnor_b64 s[0:1], 0.5, s[4:5]          ; encoding: [0xf0,0x04,0x80,0x90]
 
-# GFX12: s_xnor_b64 s[0:1], 0, s[4:5]            ; encoding: [0x80,0x04,0x80,0x90]
 0x80,0x04,0x80,0x90
+# GFX12: s_xnor_b64 s[0:1], 0, s[4:5]            ; encoding: [0x80,0x04,0x80,0x90]
 
-# GFX12: s_xnor_b64 s[0:1], 0x3f717273, s[4:5]   ; encoding: [0xff,0x04,0x80,0x90,0x73,0x72,0x71,0x3f]
 0xff,0x04,0x80,0x90,0x73,0x72,0x71,0x3f
+# GFX12: s_xnor_b64 s[0:1], 0x3f717273, s[4:5]   ; encoding: [0xff,0x04,0x80,0x90,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_xnor_b64 s[0:1], 0xaf123456, s[4:5]   ; encoding: [0xff,0x04,0x80,0x90,0x56,0x34,0x12,0xaf]
 0xff,0x04,0x80,0x90,0x56,0x34,0x12,0xaf
+# GFX12: s_xnor_b64 s[0:1], 0xaf123456, s[4:5]   ; encoding: [0xff,0x04,0x80,0x90,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_xnor_b64 s[0:1], -1, s[4:5]           ; encoding: [0xc1,0x04,0x80,0x90]
 0xc1,0x04,0x80,0x90
+# GFX12: s_xnor_b64 s[0:1], -1, s[4:5]           ; encoding: [0xc1,0x04,0x80,0x90]
 
-# GFX12: s_xnor_b64 s[0:1], -4.0, s[4:5]         ; encoding: [0xf7,0x04,0x80,0x90]
 0xf7,0x04,0x80,0x90
+# GFX12: s_xnor_b64 s[0:1], -4.0, s[4:5]         ; encoding: [0xf7,0x04,0x80,0x90]
 
-# GFX12: s_xnor_b64 s[0:1], exec, s[4:5]         ; encoding: [0x7e,0x04,0x80,0x90]
 0x7e,0x04,0x80,0x90
+# GFX12: s_xnor_b64 s[0:1], exec, s[4:5]         ; encoding: [0x7e,0x04,0x80,0x90]
 
-# GFX12: s_xnor_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x90]
 0x66,0x64,0x80,0x90
+# GFX12: s_xnor_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x90]
 
-# GFX12: s_xnor_b64 s[0:1], s[102:103], s[4:5]   ; encoding: [0x66,0x04,0x80,0x90]
 0x66,0x04,0x80,0x90
+# GFX12: s_xnor_b64 s[0:1], s[102:103], s[4:5]   ; encoding: [0x66,0x04,0x80,0x90]
 
-# GFX12: s_xnor_b64 s[0:1], s[2:3], 0.5          ; encoding: [0x02,0xf0,0x80,0x90]
 0x02,0xf0,0x80,0x90
+# GFX12: s_xnor_b64 s[0:1], s[2:3], 0.5          ; encoding: [0x02,0xf0,0x80,0x90]
 
-# GFX12: s_xnor_b64 s[0:1], s[2:3], 0            ; encoding: [0x02,0x80,0x80,0x90]
 0x02,0x80,0x80,0x90
+# GFX12: s_xnor_b64 s[0:1], s[2:3], 0            ; encoding: [0x02,0x80,0x80,0x90]
 
-# GFX12: s_xnor_b64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x90,0x73,0x72,0x71,0x3f]
 0x02,0xff,0x80,0x90,0x73,0x72,0x71,0x3f
+# GFX12: s_xnor_b64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x90,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_xnor_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x90,0x56,0x34,0x12,0xaf]
 0x02,0xff,0x80,0x90,0x56,0x34,0x12,0xaf
+# GFX12: s_xnor_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x90,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_xnor_b64 s[0:1], s[2:3], -1           ; encoding: [0x02,0xc1,0x80,0x90]
 0x02,0xc1,0x80,0x90
+# GFX12: s_xnor_b64 s[0:1], s[2:3], -1           ; encoding: [0x02,0xc1,0x80,0x90]
 
-# GFX12: s_xnor_b64 s[0:1], s[2:3], -4.0         ; encoding: [0x02,0xf7,0x80,0x90]
 0x02,0xf7,0x80,0x90
+# GFX12: s_xnor_b64 s[0:1], s[2:3], -4.0         ; encoding: [0x02,0xf7,0x80,0x90]
 
-# GFX12: s_xnor_b64 s[0:1], s[2:3], exec         ; encoding: [0x02,0x7e,0x80,0x90]
 0x02,0x7e,0x80,0x90
+# GFX12: s_xnor_b64 s[0:1], s[2:3], exec         ; encoding: [0x02,0x7e,0x80,0x90]
 
-# GFX12: s_xnor_b64 s[0:1], s[2:3], s[100:101]   ; encoding: [0x02,0x64,0x80,0x90]
 0x02,0x64,0x80,0x90
+# GFX12: s_xnor_b64 s[0:1], s[2:3], s[100:101]   ; encoding: [0x02,0x64,0x80,0x90]
 
-# GFX12: s_xnor_b64 s[0:1], s[2:3], s[4:5]       ; encoding: [0x02,0x04,0x80,0x90]
 0x02,0x04,0x80,0x90
+# GFX12: s_xnor_b64 s[0:1], s[2:3], s[4:5]       ; encoding: [0x02,0x04,0x80,0x90]
 
-# GFX12: s_xnor_b64 s[0:1], s[2:3], vcc          ; encoding: [0x02,0x6a,0x80,0x90]
 0x02,0x6a,0x80,0x90
+# GFX12: s_xnor_b64 s[0:1], s[2:3], vcc          ; encoding: [0x02,0x6a,0x80,0x90]
 
-# GFX12: s_xnor_b64 s[0:1], vcc, s[4:5]          ; encoding: [0x6a,0x04,0x80,0x90]
 0x6a,0x04,0x80,0x90
+# GFX12: s_xnor_b64 s[0:1], vcc, s[4:5]          ; encoding: [0x6a,0x04,0x80,0x90]
 
-# GFX12: s_xnor_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x90]
 0x66,0x64,0xe8,0x90
+# GFX12: s_xnor_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x90]
 
-# GFX12: s_xnor_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x90]
 0x66,0x04,0xe8,0x90
+# GFX12: s_xnor_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x90]
 
-# GFX12: s_xnor_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x90]
 0x02,0x64,0xe8,0x90
+# GFX12: s_xnor_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x90]
 
-# GFX12: s_xnor_b64 s[104:105], s[2:3], s[4:5]   ; encoding: [0x02,0x04,0xe8,0x90]
 0x02,0x04,0xe8,0x90
+# GFX12: s_xnor_b64 s[104:105], s[2:3], s[4:5]   ; encoding: [0x02,0x04,0xe8,0x90]
 
-# GFX12: s_xnor_b64 vcc, s[2:3], s[4:5]          ; encoding: [0x02,0x04,0xea,0x90]
 0x02,0x04,0xea,0x90
+# GFX12: s_xnor_b64 vcc, s[2:3], s[4:5]          ; encoding: [0x02,0x04,0xea,0x90]
 
-# GFX12: s_xor_b32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x8d]
 0x01,0x02,0x7f,0x8d
+# GFX12: s_xor_b32 exec_hi, s1, s2               ; encoding: [0x01,0x02,0x7f,0x8d]
 
-# GFX12: s_xor_b32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x8d]
 0x01,0x02,0x7e,0x8d
+# GFX12: s_xor_b32 exec_lo, s1, s2               ; encoding: [0x01,0x02,0x7e,0x8d]
 
-# GFX12: s_xor_b32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x8d]
 0x01,0x02,0x7d,0x8d
+# GFX12: s_xor_b32 m0, s1, s2                    ; encoding: [0x01,0x02,0x7d,0x8d]
 
-# GFX12: s_xor_b32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x8d]
 0xf0,0x02,0x00,0x8d
+# GFX12: s_xor_b32 s0, 0.5, s2                   ; encoding: [0xf0,0x02,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x8d]
 0x80,0x02,0x00,0x8d
+# GFX12: s_xor_b32 s0, 0, s2                     ; encoding: [0x80,0x02,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x8d,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x00,0x8d,0x73,0x72,0x71,0x3f
+# GFX12: s_xor_b32 s0, 0x3f717273, s2            ; encoding: [0xff,0x02,0x00,0x8d,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_xor_b32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x8d,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x00,0x8d,0x56,0x34,0x12,0xaf
+# GFX12: s_xor_b32 s0, 0xaf123456, s2            ; encoding: [0xff,0x02,0x00,0x8d,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_xor_b32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x8d]
 0xc1,0x02,0x00,0x8d
+# GFX12: s_xor_b32 s0, -1, s2                    ; encoding: [0xc1,0x02,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x8d]
 0xf7,0x02,0x00,0x8d
+# GFX12: s_xor_b32 s0, -4.0, s2                  ; encoding: [0xf7,0x02,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x8d]
 0x7f,0x02,0x00,0x8d
+# GFX12: s_xor_b32 s0, exec_hi, s2               ; encoding: [0x7f,0x02,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x8d]
 0x7e,0x02,0x00,0x8d
+# GFX12: s_xor_b32 s0, exec_lo, s2               ; encoding: [0x7e,0x02,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x8d]
 0x7d,0x02,0x00,0x8d
+# GFX12: s_xor_b32 s0, m0, s2                    ; encoding: [0x7d,0x02,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x8d]
 0x68,0x67,0x00,0x8d
+# GFX12: s_xor_b32 s0, s104, s103                ; encoding: [0x68,0x67,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x8d]
 0x68,0x02,0x00,0x8d
+# GFX12: s_xor_b32 s0, s104, s2                  ; encoding: [0x68,0x02,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x8d]
 0x01,0xf0,0x00,0x8d
+# GFX12: s_xor_b32 s0, s1, 0.5                   ; encoding: [0x01,0xf0,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x8d]
 0x01,0x80,0x00,0x8d
+# GFX12: s_xor_b32 s0, s1, 0                     ; encoding: [0x01,0x80,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x8d,0x73,0x72,0x71,0x3f]
 0x01,0xff,0x00,0x8d,0x73,0x72,0x71,0x3f
+# GFX12: s_xor_b32 s0, s1, 0x3f717273            ; encoding: [0x01,0xff,0x00,0x8d,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_xor_b32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x8d,0x56,0x34,0x12,0xaf]
 0x01,0xff,0x00,0x8d,0x56,0x34,0x12,0xaf
+# GFX12: s_xor_b32 s0, s1, 0xaf123456            ; encoding: [0x01,0xff,0x00,0x8d,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_xor_b32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x8d]
 0x01,0xc1,0x00,0x8d
+# GFX12: s_xor_b32 s0, s1, -1                    ; encoding: [0x01,0xc1,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x8d]
 0x01,0xf7,0x00,0x8d
+# GFX12: s_xor_b32 s0, s1, -4.0                  ; encoding: [0x01,0xf7,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x8d]
 0x01,0x7f,0x00,0x8d
+# GFX12: s_xor_b32 s0, s1, exec_hi               ; encoding: [0x01,0x7f,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x8d]
 0x01,0x7e,0x00,0x8d
+# GFX12: s_xor_b32 s0, s1, exec_lo               ; encoding: [0x01,0x7e,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x8d]
 0x01,0x7d,0x00,0x8d
+# GFX12: s_xor_b32 s0, s1, m0                    ; encoding: [0x01,0x7d,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x8d]
 0x01,0x67,0x00,0x8d
+# GFX12: s_xor_b32 s0, s1, s103                  ; encoding: [0x01,0x67,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x8d]
 0x01,0x02,0x00,0x8d
+# GFX12: s_xor_b32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x8d]
 0x01,0x6b,0x00,0x8d
+# GFX12: s_xor_b32 s0, s1, vcc_hi                ; encoding: [0x01,0x6b,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x8d]
 0x01,0x6a,0x00,0x8d
+# GFX12: s_xor_b32 s0, s1, vcc_lo                ; encoding: [0x01,0x6a,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x8d]
 0x6b,0x02,0x00,0x8d
+# GFX12: s_xor_b32 s0, vcc_hi, s2                ; encoding: [0x6b,0x02,0x00,0x8d]
 
-# GFX12: s_xor_b32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x8d]
 0x6a,0x02,0x00,0x8d
+# GFX12: s_xor_b32 s0, vcc_lo, s2                ; encoding: [0x6a,0x02,0x00,0x8d]
 
-# GFX12: s_xor_b32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x8d]
 0x68,0x67,0x69,0x8d
+# GFX12: s_xor_b32 s105, s104, s103              ; encoding: [0x68,0x67,0x69,0x8d]
 
-# GFX12: s_xor_b32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x8d]
 0x68,0x02,0x69,0x8d
+# GFX12: s_xor_b32 s105, s104, s2                ; encoding: [0x68,0x02,0x69,0x8d]
 
-# GFX12: s_xor_b32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x8d]
 0x01,0x67,0x69,0x8d
+# GFX12: s_xor_b32 s105, s1, s103                ; encoding: [0x01,0x67,0x69,0x8d]
 
-# GFX12: s_xor_b32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x8d]
 0x01,0x02,0x69,0x8d
+# GFX12: s_xor_b32 s105, s1, s2                  ; encoding: [0x01,0x02,0x69,0x8d]
 
-# GFX12: s_xor_b32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x8d]
 0x01,0x02,0x6b,0x8d
+# GFX12: s_xor_b32 vcc_hi, s1, s2                ; encoding: [0x01,0x02,0x6b,0x8d]
 
-# GFX12: s_xor_b32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x8d]
 0x01,0x02,0x6a,0x8d
+# GFX12: s_xor_b32 vcc_lo, s1, s2                ; encoding: [0x01,0x02,0x6a,0x8d]
 
-# GFX12: s_xor_b64 exec, s[2:3], s[4:5]          ; encoding: [0x02,0x04,0xfe,0x8d]
 0x02,0x04,0xfe,0x8d
+# GFX12: s_xor_b64 exec, s[2:3], s[4:5]          ; encoding: [0x02,0x04,0xfe,0x8d]
 
-# GFX12: s_xor_b64 s[0:1], 0.5, s[4:5]           ; encoding: [0xf0,0x04,0x80,0x8d]
 0xf0,0x04,0x80,0x8d
+# GFX12: s_xor_b64 s[0:1], 0.5, s[4:5]           ; encoding: [0xf0,0x04,0x80,0x8d]
 
-# GFX12: s_xor_b64 s[0:1], 0, s[4:5]             ; encoding: [0x80,0x04,0x80,0x8d]
 0x80,0x04,0x80,0x8d
+# GFX12: s_xor_b64 s[0:1], 0, s[4:5]             ; encoding: [0x80,0x04,0x80,0x8d]
 
-# GFX12: s_xor_b64 s[0:1], 0x3f717273, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8d,0x73,0x72,0x71,0x3f]
 0xff,0x04,0x80,0x8d,0x73,0x72,0x71,0x3f
+# GFX12: s_xor_b64 s[0:1], 0x3f717273, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8d,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_xor_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8d,0x56,0x34,0x12,0xaf]
 0xff,0x04,0x80,0x8d,0x56,0x34,0x12,0xaf
+# GFX12: s_xor_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8d,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_xor_b64 s[0:1], -1, s[4:5]            ; encoding: [0xc1,0x04,0x80,0x8d]
 0xc1,0x04,0x80,0x8d
+# GFX12: s_xor_b64 s[0:1], -1, s[4:5]            ; encoding: [0xc1,0x04,0x80,0x8d]
 
-# GFX12: s_xor_b64 s[0:1], -4.0, s[4:5]          ; encoding: [0xf7,0x04,0x80,0x8d]
 0xf7,0x04,0x80,0x8d
+# GFX12: s_xor_b64 s[0:1], -4.0, s[4:5]          ; encoding: [0xf7,0x04,0x80,0x8d]
 
-# GFX12: s_xor_b64 s[0:1], exec, s[4:5]          ; encoding: [0x7e,0x04,0x80,0x8d]
 0x7e,0x04,0x80,0x8d
+# GFX12: s_xor_b64 s[0:1], exec, s[4:5]          ; encoding: [0x7e,0x04,0x80,0x8d]
 
-# GFX12: s_xor_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x8d]
 0x66,0x64,0x80,0x8d
+# GFX12: s_xor_b64 s[0:1], s[102:103], s[100:101] ; encoding: [0x66,0x64,0x80,0x8d]
 
-# GFX12: s_xor_b64 s[0:1], s[102:103], s[4:5]    ; encoding: [0x66,0x04,0x80,0x8d]
 0x66,0x04,0x80,0x8d
+# GFX12: s_xor_b64 s[0:1], s[102:103], s[4:5]    ; encoding: [0x66,0x04,0x80,0x8d]
 
-# GFX12: s_xor_b64 s[0:1], s[2:3], 0.5           ; encoding: [0x02,0xf0,0x80,0x8d]
 0x02,0xf0,0x80,0x8d
+# GFX12: s_xor_b64 s[0:1], s[2:3], 0.5           ; encoding: [0x02,0xf0,0x80,0x8d]
 
-# GFX12: s_xor_b64 s[0:1], s[2:3], 0             ; encoding: [0x02,0x80,0x80,0x8d]
 0x02,0x80,0x80,0x8d
+# GFX12: s_xor_b64 s[0:1], s[2:3], 0             ; encoding: [0x02,0x80,0x80,0x8d]
 
-# GFX12: s_xor_b64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0x8d,0x73,0x72,0x71,0x3f]
 0x02,0xff,0x80,0x8d,0x73,0x72,0x71,0x3f
+# GFX12: s_xor_b64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0x8d,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_xor_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8d,0x56,0x34,0x12,0xaf]
 0x02,0xff,0x80,0x8d,0x56,0x34,0x12,0xaf
+# GFX12: s_xor_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8d,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_xor_b64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x80,0x8d]
 0x02,0xc1,0x80,0x8d
+# GFX12: s_xor_b64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x80,0x8d]
 
-# GFX12: s_xor_b64 s[0:1], s[2:3], -4.0          ; encoding: [0x02,0xf7,0x80,0x8d]
 0x02,0xf7,0x80,0x8d
+# GFX12: s_xor_b64 s[0:1], s[2:3], -4.0          ; encoding: [0x02,0xf7,0x80,0x8d]
 
-# GFX12: s_xor_b64 s[0:1], s[2:3], exec          ; encoding: [0x02,0x7e,0x80,0x8d]
 0x02,0x7e,0x80,0x8d
+# GFX12: s_xor_b64 s[0:1], s[2:3], exec          ; encoding: [0x02,0x7e,0x80,0x8d]
 
-# GFX12: s_xor_b64 s[0:1], s[2:3], s[100:101]    ; encoding: [0x02,0x64,0x80,0x8d]
 0x02,0x64,0x80,0x8d
+# GFX12: s_xor_b64 s[0:1], s[2:3], s[100:101]    ; encoding: [0x02,0x64,0x80,0x8d]
 
-# GFX12: s_xor_b64 s[0:1], s[2:3], s[4:5]        ; encoding: [0x02,0x04,0x80,0x8d]
 0x02,0x04,0x80,0x8d
+# GFX12: s_xor_b64 s[0:1], s[2:3], s[4:5]        ; encoding: [0x02,0x04,0x80,0x8d]
 
-# GFX12: s_xor_b64 s[0:1], s[2:3], vcc           ; encoding: [0x02,0x6a,0x80,0x8d]
 0x02,0x6a,0x80,0x8d
+# GFX12: s_xor_b64 s[0:1], s[2:3], vcc           ; encoding: [0x02,0x6a,0x80,0x8d]
 
-# GFX12: s_xor_b64 s[0:1], vcc, s[4:5]           ; encoding: [0x6a,0x04,0x80,0x8d]
 0x6a,0x04,0x80,0x8d
+# GFX12: s_xor_b64 s[0:1], vcc, s[4:5]           ; encoding: [0x6a,0x04,0x80,0x8d]
 
-# GFX12: s_xor_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x8d]
 0x66,0x64,0xe8,0x8d
+# GFX12: s_xor_b64 s[104:105], s[102:103], s[100:101] ; encoding: [0x66,0x64,0xe8,0x8d]
 
-# GFX12: s_xor_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x8d]
 0x66,0x04,0xe8,0x8d
+# GFX12: s_xor_b64 s[104:105], s[102:103], s[4:5] ; encoding: [0x66,0x04,0xe8,0x8d]
 
-# GFX12: s_xor_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x8d]
 0x02,0x64,0xe8,0x8d
+# GFX12: s_xor_b64 s[104:105], s[2:3], s[100:101] ; encoding: [0x02,0x64,0xe8,0x8d]
 
-# GFX12: s_xor_b64 s[104:105], s[2:3], s[4:5]    ; encoding: [0x02,0x04,0xe8,0x8d]
 0x02,0x04,0xe8,0x8d
+# GFX12: s_xor_b64 s[104:105], s[2:3], s[4:5]    ; encoding: [0x02,0x04,0xe8,0x8d]
 
-# GFX12: s_xor_b64 vcc, s[2:3], s[4:5]           ; encoding: [0x02,0x04,0xea,0x8d]
 0x02,0x04,0xea,0x8d
+# GFX12: s_xor_b64 vcc, s[2:3], s[4:5]           ; encoding: [0x02,0x04,0xea,0x8d]
 
-# GFX12: s_pack_hl_b32_b16 s5, s1, s2            ; encoding: [0x01,0x02,0x85,0x9a]
 0x01,0x02,0x85,0x9a
+# GFX12: s_pack_hl_b32_b16 s5, s1, s2            ; encoding: [0x01,0x02,0x85,0x9a]
 
-# GFX12: s_pack_hl_b32_b16 s5, s105, s105        ; encoding: [0x69,0x69,0x85,0x9a]
 0x69,0x69,0x85,0x9a
+# GFX12: s_pack_hl_b32_b16 s5, s105, s105        ; encoding: [0x69,0x69,0x85,0x9a]
 
-# GFX12: s_pack_hl_b32_b16 s5, vcc_lo, ttmp15    ; encoding: [0x6a,0x7b,0x85,0x9a]
 0x6a,0x7b,0x85,0x9a
+# GFX12: s_pack_hl_b32_b16 s5, vcc_lo, ttmp15    ; encoding: [0x6a,0x7b,0x85,0x9a]
 
-# GFX12: s_pack_hl_b32_b16 s5, vcc_hi, 0xfe0b    ; encoding: [0x6b,0xff,0x85,0x9a,0x0b,0xfe,0x00,0x00]
 0x6b,0xff,0x85,0x9a,0x0b,0xfe,0x00,0x00
+# GFX12: s_pack_hl_b32_b16 s5, vcc_hi, 0xfe0b    ; encoding: [0x6b,0xff,0x85,0x9a,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_pack_hl_b32_b16 s5, ttmp15, src_scc   ; encoding: [0x7b,0xfd,0x85,0x9a]
 0x7b,0xfd,0x85,0x9a
+# GFX12: s_pack_hl_b32_b16 s5, ttmp15, src_scc   ; encoding: [0x7b,0xfd,0x85,0x9a]
 
-# GFX12: s_pack_hl_b32_b16 s105, m0, 0.5         ; encoding: [0x7d,0xf0,0xe9,0x9a]
 0x7d,0xf0,0xe9,0x9a
+# GFX12: s_pack_hl_b32_b16 s105, m0, 0.5         ; encoding: [0x7d,0xf0,0xe9,0x9a]
 
-# GFX12: s_pack_hl_b32_b16 vcc_lo, exec_lo, -1   ; encoding: [0x7e,0xc1,0xea,0x9a]
 0x7e,0xc1,0xea,0x9a
+# GFX12: s_pack_hl_b32_b16 vcc_lo, exec_lo, -1   ; encoding: [0x7e,0xc1,0xea,0x9a]
 
-# GFX12: s_pack_hl_b32_b16 vcc_hi, exec_hi, null ; encoding: [0x7f,0x7c,0xeb,0x9a]
 0x7f,0x7c,0xeb,0x9a
+# GFX12: s_pack_hl_b32_b16 vcc_hi, exec_hi, null ; encoding: [0x7f,0x7c,0xeb,0x9a]
 
-# GFX12: s_pack_hl_b32_b16 ttmp15, null, exec_lo ; encoding: [0x7c,0x7e,0xfb,0x9a]
 0x7c,0x7e,0xfb,0x9a
+# GFX12: s_pack_hl_b32_b16 ttmp15, null, exec_lo ; encoding: [0x7c,0x7e,0xfb,0x9a]
 
-# GFX12: s_pack_hl_b32_b16 m0, -1, exec_hi       ; encoding: [0xc1,0x7f,0xfd,0x9a]
 0xc1,0x7f,0xfd,0x9a
+# GFX12: s_pack_hl_b32_b16 m0, -1, exec_hi       ; encoding: [0xc1,0x7f,0xfd,0x9a]
 
-# GFX12: s_pack_hl_b32_b16 exec_lo, 0.5, m0      ; encoding: [0xf0,0x7d,0xfe,0x9a]
 0xf0,0x7d,0xfe,0x9a
+# GFX12: s_pack_hl_b32_b16 exec_lo, 0.5, m0      ; encoding: [0xf0,0x7d,0xfe,0x9a]
 
-# GFX12: s_pack_hl_b32_b16 exec_hi, src_scc, vcc_lo ; encoding: [0xfd,0x6a,0xff,0x9a]
 0xfd,0x6a,0xff,0x9a
+# GFX12: s_pack_hl_b32_b16 exec_hi, src_scc, vcc_lo ; encoding: [0xfd,0x6a,0xff,0x9a]
 
-# GFX12: s_pack_hl_b32_b16 null, 0xaf123456, vcc_hi ; encoding: [0xff,0x6b,0xfc,0x9a,0x56,0x34,0x12,0xaf]
 0xff,0x6b,0xfc,0x9a,0x56,0x34,0x12,0xaf
+# GFX12: s_pack_hl_b32_b16 null, 0xaf123456, vcc_hi ; encoding: [0xff,0x6b,0xfc,0x9a,0x56,0x34,0x12,0xaf]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopc.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopc.txt
index ac023f10aed61..e59006c322b4e 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopc.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopc.txt
@@ -1,2161 +1,2162 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s
 
-# GFX12: s_cmp_lt_f32 s1, s2                     ; encoding: [0x01,0x02,0x41,0xbf]
 0x01,0x02,0x41,0xbf
+# GFX12: s_cmp_lt_f32 s1, s2                     ; encoding: [0x01,0x02,0x41,0xbf]
 
-# GFX12: s_cmp_lt_f32 s105, s2                   ; encoding: [0x69,0x02,0x41,0xbf]
 0x69,0x02,0x41,0xbf
+# GFX12: s_cmp_lt_f32 s105, s2                   ; encoding: [0x69,0x02,0x41,0xbf]
 
-# GFX12: s_cmp_lt_f32 s101, s2                   ; encoding: [0x65,0x02,0x41,0xbf]
 0x65,0x02,0x41,0xbf
+# GFX12: s_cmp_lt_f32 s101, s2                   ; encoding: [0x65,0x02,0x41,0xbf]
 
-# GFX12: s_cmp_lt_f32 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x41,0xbf]
 0x6a,0x02,0x41,0xbf
+# GFX12: s_cmp_lt_f32 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x41,0xbf]
 
-# GFX12: s_cmp_lt_f32 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x41,0xbf]
 0x6b,0x02,0x41,0xbf
+# GFX12: s_cmp_lt_f32 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x41,0xbf]
 
-# GFX12: s_cmp_lt_f32 m0, s2                     ; encoding: [0x7d,0x02,0x41,0xbf]
 0x7d,0x02,0x41,0xbf
+# GFX12: s_cmp_lt_f32 m0, s2                     ; encoding: [0x7d,0x02,0x41,0xbf]
 
-# GFX12: s_cmp_lt_f32 exec_lo, s2                ; encoding: [0x7e,0x02,0x41,0xbf]
 0x7e,0x02,0x41,0xbf
+# GFX12: s_cmp_lt_f32 exec_lo, s2                ; encoding: [0x7e,0x02,0x41,0xbf]
 
-# GFX12: s_cmp_lt_f32 exec_hi, s2                ; encoding: [0x7f,0x02,0x41,0xbf]
 0x7f,0x02,0x41,0xbf
+# GFX12: s_cmp_lt_f32 exec_hi, s2                ; encoding: [0x7f,0x02,0x41,0xbf]
 
-# GFX12: s_cmp_lt_f32 0, s2                      ; encoding: [0x80,0x02,0x41,0xbf]
 0x80,0x02,0x41,0xbf
+# GFX12: s_cmp_lt_f32 0, s2                      ; encoding: [0x80,0x02,0x41,0xbf]
 
-# GFX12: s_cmp_lt_f32 -1, s2                     ; encoding: [0xc1,0x02,0x41,0xbf]
 0xc1,0x02,0x41,0xbf
+# GFX12: s_cmp_lt_f32 -1, s2                     ; encoding: [0xc1,0x02,0x41,0xbf]
 
-# GFX12: s_cmp_lt_f32 0.5, s2                    ; encoding: [0xf0,0x02,0x41,0xbf]
 0xf0,0x02,0x41,0xbf
+# GFX12: s_cmp_lt_f32 0.5, s2                    ; encoding: [0xf0,0x02,0x41,0xbf]
 
-# GFX12: s_cmp_lt_f32 -4.0, s2                   ; encoding: [0xf7,0x02,0x41,0xbf]
 0xf7,0x02,0x41,0xbf
+# GFX12: s_cmp_lt_f32 -4.0, s2                   ; encoding: [0xf7,0x02,0x41,0xbf]
 
-# GFX12: s_cmp_lt_f32 0xaf123456, s2             ; encoding: [0xff,0x02,0x41,0xbf,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x41,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_lt_f32 0xaf123456, s2             ; encoding: [0xff,0x02,0x41,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_lt_f32 0x3f717273, s2             ; encoding: [0xff,0x02,0x41,0xbf,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x41,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_lt_f32 0x3f717273, s2             ; encoding: [0xff,0x02,0x41,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_lt_f32 s1, s105                   ; encoding: [0x01,0x69,0x41,0xbf]
 0x01,0x69,0x41,0xbf
+# GFX12: s_cmp_lt_f32 s1, s105                   ; encoding: [0x01,0x69,0x41,0xbf]
 
-# GFX12: s_cmp_eq_f32 s1, s2                     ; encoding: [0x01,0x02,0x42,0xbf]
 0x01,0x02,0x42,0xbf
+# GFX12: s_cmp_eq_f32 s1, s2                     ; encoding: [0x01,0x02,0x42,0xbf]
 
-# GFX12: s_cmp_eq_f32 s105, s2                   ; encoding: [0x69,0x02,0x42,0xbf]
 0x69,0x02,0x42,0xbf
+# GFX12: s_cmp_eq_f32 s105, s2                   ; encoding: [0x69,0x02,0x42,0xbf]
 
-# GFX12: s_cmp_eq_f32 s101, s2                   ; encoding: [0x65,0x02,0x42,0xbf]
 0x65,0x02,0x42,0xbf
+# GFX12: s_cmp_eq_f32 s101, s2                   ; encoding: [0x65,0x02,0x42,0xbf]
 
-# GFX12: s_cmp_eq_f32 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x42,0xbf]
 0x6a,0x02,0x42,0xbf
+# GFX12: s_cmp_eq_f32 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x42,0xbf]
 
-# GFX12: s_cmp_eq_f32 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x42,0xbf]
 0x6b,0x02,0x42,0xbf
+# GFX12: s_cmp_eq_f32 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x42,0xbf]
 
-# GFX12: s_cmp_eq_f32 m0, s2                     ; encoding: [0x7d,0x02,0x42,0xbf]
 0x7d,0x02,0x42,0xbf
+# GFX12: s_cmp_eq_f32 m0, s2                     ; encoding: [0x7d,0x02,0x42,0xbf]
 
-# GFX12: s_cmp_eq_f32 exec_lo, s2                ; encoding: [0x7e,0x02,0x42,0xbf]
 0x7e,0x02,0x42,0xbf
+# GFX12: s_cmp_eq_f32 exec_lo, s2                ; encoding: [0x7e,0x02,0x42,0xbf]
 
-# GFX12: s_cmp_eq_f32 exec_hi, s2                ; encoding: [0x7f,0x02,0x42,0xbf]
 0x7f,0x02,0x42,0xbf
+# GFX12: s_cmp_eq_f32 exec_hi, s2                ; encoding: [0x7f,0x02,0x42,0xbf]
 
-# GFX12: s_cmp_eq_f32 0, s2                      ; encoding: [0x80,0x02,0x42,0xbf]
 0x80,0x02,0x42,0xbf
+# GFX12: s_cmp_eq_f32 0, s2                      ; encoding: [0x80,0x02,0x42,0xbf]
 
-# GFX12: s_cmp_eq_f32 -1, s2                     ; encoding: [0xc1,0x02,0x42,0xbf]
 0xc1,0x02,0x42,0xbf
+# GFX12: s_cmp_eq_f32 -1, s2                     ; encoding: [0xc1,0x02,0x42,0xbf]
 
-# GFX12: s_cmp_eq_f32 0.5, s2                    ; encoding: [0xf0,0x02,0x42,0xbf]
 0xf0,0x02,0x42,0xbf
+# GFX12: s_cmp_eq_f32 0.5, s2                    ; encoding: [0xf0,0x02,0x42,0xbf]
 
-# GFX12: s_cmp_eq_f32 -4.0, s2                   ; encoding: [0xf7,0x02,0x42,0xbf]
 0xf7,0x02,0x42,0xbf
+# GFX12: s_cmp_eq_f32 -4.0, s2                   ; encoding: [0xf7,0x02,0x42,0xbf]
 
-# GFX12: s_cmp_eq_f32 0xaf123456, s2             ; encoding: [0xff,0x02,0x42,0xbf,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x42,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_eq_f32 0xaf123456, s2             ; encoding: [0xff,0x02,0x42,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_eq_f32 0x3f717273, s2             ; encoding: [0xff,0x02,0x42,0xbf,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x42,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_eq_f32 0x3f717273, s2             ; encoding: [0xff,0x02,0x42,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_eq_f32 s1, s105                   ; encoding: [0x01,0x69,0x42,0xbf]
 0x01,0x69,0x42,0xbf
+# GFX12: s_cmp_eq_f32 s1, s105                   ; encoding: [0x01,0x69,0x42,0xbf]
 
-# GFX12: s_cmp_le_f32 s1, s2                     ; encoding: [0x01,0x02,0x43,0xbf]
 0x01,0x02,0x43,0xbf
+# GFX12: s_cmp_le_f32 s1, s2                     ; encoding: [0x01,0x02,0x43,0xbf]
 
-# GFX12: s_cmp_le_f32 s105, s2                   ; encoding: [0x69,0x02,0x43,0xbf]
 0x69,0x02,0x43,0xbf
+# GFX12: s_cmp_le_f32 s105, s2                   ; encoding: [0x69,0x02,0x43,0xbf]
 
-# GFX12: s_cmp_le_f32 s101, s2                   ; encoding: [0x65,0x02,0x43,0xbf]
 0x65,0x02,0x43,0xbf
+# GFX12: s_cmp_le_f32 s101, s2                   ; encoding: [0x65,0x02,0x43,0xbf]
 
-# GFX12: s_cmp_le_f32 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x43,0xbf]
 0x6a,0x02,0x43,0xbf
+# GFX12: s_cmp_le_f32 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x43,0xbf]
 
-# GFX12: s_cmp_le_f32 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x43,0xbf]
 0x6b,0x02,0x43,0xbf
+# GFX12: s_cmp_le_f32 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x43,0xbf]
 
-# GFX12: s_cmp_le_f32 m0, s2                     ; encoding: [0x7d,0x02,0x43,0xbf]
 0x7d,0x02,0x43,0xbf
+# GFX12: s_cmp_le_f32 m0, s2                     ; encoding: [0x7d,0x02,0x43,0xbf]
 
-# GFX12: s_cmp_le_f32 exec_lo, s2                ; encoding: [0x7e,0x02,0x43,0xbf]
 0x7e,0x02,0x43,0xbf
+# GFX12: s_cmp_le_f32 exec_lo, s2                ; encoding: [0x7e,0x02,0x43,0xbf]
 
-# GFX12: s_cmp_le_f32 exec_hi, s2                ; encoding: [0x7f,0x02,0x43,0xbf]
 0x7f,0x02,0x43,0xbf
+# GFX12: s_cmp_le_f32 exec_hi, s2                ; encoding: [0x7f,0x02,0x43,0xbf]
 
-# GFX12: s_cmp_le_f32 0, s2                      ; encoding: [0x80,0x02,0x43,0xbf]
 0x80,0x02,0x43,0xbf
+# GFX12: s_cmp_le_f32 0, s2                      ; encoding: [0x80,0x02,0x43,0xbf]
 
-# GFX12: s_cmp_le_f32 -1, s2                     ; encoding: [0xc1,0x02,0x43,0xbf]
 0xc1,0x02,0x43,0xbf
+# GFX12: s_cmp_le_f32 -1, s2                     ; encoding: [0xc1,0x02,0x43,0xbf]
 
-# GFX12: s_cmp_le_f32 0.5, s2                    ; encoding: [0xf0,0x02,0x43,0xbf]
 0xf0,0x02,0x43,0xbf
+# GFX12: s_cmp_le_f32 0.5, s2                    ; encoding: [0xf0,0x02,0x43,0xbf]
 
-# GFX12: s_cmp_le_f32 -4.0, s2                   ; encoding: [0xf7,0x02,0x43,0xbf]
 0xf7,0x02,0x43,0xbf
+# GFX12: s_cmp_le_f32 -4.0, s2                   ; encoding: [0xf7,0x02,0x43,0xbf]
 
-# GFX12: s_cmp_le_f32 0xaf123456, s2             ; encoding: [0xff,0x02,0x43,0xbf,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x43,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_le_f32 0xaf123456, s2             ; encoding: [0xff,0x02,0x43,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_le_f32 0x3f717273, s2             ; encoding: [0xff,0x02,0x43,0xbf,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x43,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_le_f32 0x3f717273, s2             ; encoding: [0xff,0x02,0x43,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_le_f32 s1, s105                   ; encoding: [0x01,0x69,0x43,0xbf]
 0x01,0x69,0x43,0xbf
+# GFX12: s_cmp_le_f32 s1, s105                   ; encoding: [0x01,0x69,0x43,0xbf]
 
-# GFX12: s_cmp_gt_f32 s1, s2                     ; encoding: [0x01,0x02,0x44,0xbf]
 0x01,0x02,0x44,0xbf
+# GFX12: s_cmp_gt_f32 s1, s2                     ; encoding: [0x01,0x02,0x44,0xbf]
 
-# GFX12: s_cmp_gt_f32 s105, s2                   ; encoding: [0x69,0x02,0x44,0xbf]
 0x69,0x02,0x44,0xbf
+# GFX12: s_cmp_gt_f32 s105, s2                   ; encoding: [0x69,0x02,0x44,0xbf]
 
-# GFX12: s_cmp_gt_f32 s101, s2                   ; encoding: [0x65,0x02,0x44,0xbf]
 0x65,0x02,0x44,0xbf
+# GFX12: s_cmp_gt_f32 s101, s2                   ; encoding: [0x65,0x02,0x44,0xbf]
 
-# GFX12: s_cmp_gt_f32 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x44,0xbf]
 0x6a,0x02,0x44,0xbf
+# GFX12: s_cmp_gt_f32 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x44,0xbf]
 
-# GFX12: s_cmp_gt_f32 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x44,0xbf]
 0x6b,0x02,0x44,0xbf
+# GFX12: s_cmp_gt_f32 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x44,0xbf]
 
-# GFX12: s_cmp_gt_f32 m0, s2                     ; encoding: [0x7d,0x02,0x44,0xbf]
 0x7d,0x02,0x44,0xbf
+# GFX12: s_cmp_gt_f32 m0, s2                     ; encoding: [0x7d,0x02,0x44,0xbf]
 
-# GFX12: s_cmp_gt_f32 exec_lo, s2                ; encoding: [0x7e,0x02,0x44,0xbf]
 0x7e,0x02,0x44,0xbf
+# GFX12: s_cmp_gt_f32 exec_lo, s2                ; encoding: [0x7e,0x02,0x44,0xbf]
 
-# GFX12: s_cmp_gt_f32 exec_hi, s2                ; encoding: [0x7f,0x02,0x44,0xbf]
 0x7f,0x02,0x44,0xbf
+# GFX12: s_cmp_gt_f32 exec_hi, s2                ; encoding: [0x7f,0x02,0x44,0xbf]
 
-# GFX12: s_cmp_gt_f32 0, s2                      ; encoding: [0x80,0x02,0x44,0xbf]
 0x80,0x02,0x44,0xbf
+# GFX12: s_cmp_gt_f32 0, s2                      ; encoding: [0x80,0x02,0x44,0xbf]
 
-# GFX12: s_cmp_gt_f32 -1, s2                     ; encoding: [0xc1,0x02,0x44,0xbf]
 0xc1,0x02,0x44,0xbf
+# GFX12: s_cmp_gt_f32 -1, s2                     ; encoding: [0xc1,0x02,0x44,0xbf]
 
-# GFX12: s_cmp_gt_f32 0.5, s2                    ; encoding: [0xf0,0x02,0x44,0xbf]
 0xf0,0x02,0x44,0xbf
+# GFX12: s_cmp_gt_f32 0.5, s2                    ; encoding: [0xf0,0x02,0x44,0xbf]
 
-# GFX12: s_cmp_gt_f32 -4.0, s2                   ; encoding: [0xf7,0x02,0x44,0xbf]
 0xf7,0x02,0x44,0xbf
+# GFX12: s_cmp_gt_f32 -4.0, s2                   ; encoding: [0xf7,0x02,0x44,0xbf]
 
-# GFX12: s_cmp_gt_f32 0xaf123456, s2             ; encoding: [0xff,0x02,0x44,0xbf,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x44,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_gt_f32 0xaf123456, s2             ; encoding: [0xff,0x02,0x44,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_gt_f32 0x3f717273, s2             ; encoding: [0xff,0x02,0x44,0xbf,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x44,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_gt_f32 0x3f717273, s2             ; encoding: [0xff,0x02,0x44,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_gt_f32 s1, s105                   ; encoding: [0x01,0x69,0x44,0xbf]
 0x01,0x69,0x44,0xbf
+# GFX12: s_cmp_gt_f32 s1, s105                   ; encoding: [0x01,0x69,0x44,0xbf]
 
-# GFX12: s_cmp_lg_f32 s1, s2                     ; encoding: [0x01,0x02,0x45,0xbf]
 0x01,0x02,0x45,0xbf
+# GFX12: s_cmp_lg_f32 s1, s2                     ; encoding: [0x01,0x02,0x45,0xbf]
 
-# GFX12: s_cmp_lg_f32 s105, s2                   ; encoding: [0x69,0x02,0x45,0xbf]
 0x69,0x02,0x45,0xbf
+# GFX12: s_cmp_lg_f32 s105, s2                   ; encoding: [0x69,0x02,0x45,0xbf]
 
-# GFX12: s_cmp_lg_f32 s101, s2                   ; encoding: [0x65,0x02,0x45,0xbf]
 0x65,0x02,0x45,0xbf
+# GFX12: s_cmp_lg_f32 s101, s2                   ; encoding: [0x65,0x02,0x45,0xbf]
 
-# GFX12: s_cmp_lg_f32 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x45,0xbf]
 0x6a,0x02,0x45,0xbf
+# GFX12: s_cmp_lg_f32 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x45,0xbf]
 
-# GFX12: s_cmp_lg_f32 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x45,0xbf]
 0x6b,0x02,0x45,0xbf
+# GFX12: s_cmp_lg_f32 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x45,0xbf]
 
-# GFX12: s_cmp_lg_f32 m0, s2                     ; encoding: [0x7d,0x02,0x45,0xbf]
 0x7d,0x02,0x45,0xbf
+# GFX12: s_cmp_lg_f32 m0, s2                     ; encoding: [0x7d,0x02,0x45,0xbf]
 
-# GFX12: s_cmp_lg_f32 exec_lo, s2                ; encoding: [0x7e,0x02,0x45,0xbf]
 0x7e,0x02,0x45,0xbf
+# GFX12: s_cmp_lg_f32 exec_lo, s2                ; encoding: [0x7e,0x02,0x45,0xbf]
 
-# GFX12: s_cmp_lg_f32 exec_hi, s2                ; encoding: [0x7f,0x02,0x45,0xbf]
 0x7f,0x02,0x45,0xbf
+# GFX12: s_cmp_lg_f32 exec_hi, s2                ; encoding: [0x7f,0x02,0x45,0xbf]
 
-# GFX12: s_cmp_lg_f32 0, s2                      ; encoding: [0x80,0x02,0x45,0xbf]
 0x80,0x02,0x45,0xbf
+# GFX12: s_cmp_lg_f32 0, s2                      ; encoding: [0x80,0x02,0x45,0xbf]
 
-# GFX12: s_cmp_lg_f32 -1, s2                     ; encoding: [0xc1,0x02,0x45,0xbf]
 0xc1,0x02,0x45,0xbf
+# GFX12: s_cmp_lg_f32 -1, s2                     ; encoding: [0xc1,0x02,0x45,0xbf]
 
-# GFX12: s_cmp_lg_f32 0.5, s2                    ; encoding: [0xf0,0x02,0x45,0xbf]
 0xf0,0x02,0x45,0xbf
+# GFX12: s_cmp_lg_f32 0.5, s2                    ; encoding: [0xf0,0x02,0x45,0xbf]
 
-# GFX12: s_cmp_lg_f32 -4.0, s2                   ; encoding: [0xf7,0x02,0x45,0xbf]
 0xf7,0x02,0x45,0xbf
+# GFX12: s_cmp_lg_f32 -4.0, s2                   ; encoding: [0xf7,0x02,0x45,0xbf]
 
-# GFX12: s_cmp_lg_f32 0xaf123456, s2             ; encoding: [0xff,0x02,0x45,0xbf,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x45,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_lg_f32 0xaf123456, s2             ; encoding: [0xff,0x02,0x45,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_lg_f32 0x3f717273, s2             ; encoding: [0xff,0x02,0x45,0xbf,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x45,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_lg_f32 0x3f717273, s2             ; encoding: [0xff,0x02,0x45,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_lg_f32 s1, s105                   ; encoding: [0x01,0x69,0x45,0xbf]
 0x01,0x69,0x45,0xbf
+# GFX12: s_cmp_lg_f32 s1, s105                   ; encoding: [0x01,0x69,0x45,0xbf]
 
-# GFX12: s_cmp_ge_f32 s1, s2                     ; encoding: [0x01,0x02,0x46,0xbf]
 0x01,0x02,0x46,0xbf
+# GFX12: s_cmp_ge_f32 s1, s2                     ; encoding: [0x01,0x02,0x46,0xbf]
 
-# GFX12: s_cmp_ge_f32 s105, s2                   ; encoding: [0x69,0x02,0x46,0xbf]
 0x69,0x02,0x46,0xbf
+# GFX12: s_cmp_ge_f32 s105, s2                   ; encoding: [0x69,0x02,0x46,0xbf]
 
-# GFX12: s_cmp_ge_f32 s101, s2                   ; encoding: [0x65,0x02,0x46,0xbf]
 0x65,0x02,0x46,0xbf
+# GFX12: s_cmp_ge_f32 s101, s2                   ; encoding: [0x65,0x02,0x46,0xbf]
 
-# GFX12: s_cmp_ge_f32 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x46,0xbf]
 0x6a,0x02,0x46,0xbf
+# GFX12: s_cmp_ge_f32 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x46,0xbf]
 
-# GFX12: s_cmp_ge_f32 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x46,0xbf]
 0x6b,0x02,0x46,0xbf
+# GFX12: s_cmp_ge_f32 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x46,0xbf]
 
-# GFX12: s_cmp_ge_f32 m0, s2                     ; encoding: [0x7d,0x02,0x46,0xbf]
 0x7d,0x02,0x46,0xbf
+# GFX12: s_cmp_ge_f32 m0, s2                     ; encoding: [0x7d,0x02,0x46,0xbf]
 
-# GFX12: s_cmp_ge_f32 exec_lo, s2                ; encoding: [0x7e,0x02,0x46,0xbf]
 0x7e,0x02,0x46,0xbf
+# GFX12: s_cmp_ge_f32 exec_lo, s2                ; encoding: [0x7e,0x02,0x46,0xbf]
 
-# GFX12: s_cmp_ge_f32 exec_hi, s2                ; encoding: [0x7f,0x02,0x46,0xbf]
 0x7f,0x02,0x46,0xbf
+# GFX12: s_cmp_ge_f32 exec_hi, s2                ; encoding: [0x7f,0x02,0x46,0xbf]
 
-# GFX12: s_cmp_ge_f32 0, s2                      ; encoding: [0x80,0x02,0x46,0xbf]
 0x80,0x02,0x46,0xbf
+# GFX12: s_cmp_ge_f32 0, s2                      ; encoding: [0x80,0x02,0x46,0xbf]
 
-# GFX12: s_cmp_ge_f32 -1, s2                     ; encoding: [0xc1,0x02,0x46,0xbf]
 0xc1,0x02,0x46,0xbf
+# GFX12: s_cmp_ge_f32 -1, s2                     ; encoding: [0xc1,0x02,0x46,0xbf]
 
-# GFX12: s_cmp_ge_f32 0.5, s2                    ; encoding: [0xf0,0x02,0x46,0xbf]
 0xf0,0x02,0x46,0xbf
+# GFX12: s_cmp_ge_f32 0.5, s2                    ; encoding: [0xf0,0x02,0x46,0xbf]
 
-# GFX12: s_cmp_ge_f32 -4.0, s2                   ; encoding: [0xf7,0x02,0x46,0xbf]
 0xf7,0x02,0x46,0xbf
+# GFX12: s_cmp_ge_f32 -4.0, s2                   ; encoding: [0xf7,0x02,0x46,0xbf]
 
-# GFX12: s_cmp_ge_f32 0xaf123456, s2             ; encoding: [0xff,0x02,0x46,0xbf,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x46,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_ge_f32 0xaf123456, s2             ; encoding: [0xff,0x02,0x46,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_ge_f32 0x3f717273, s2             ; encoding: [0xff,0x02,0x46,0xbf,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x46,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_ge_f32 0x3f717273, s2             ; encoding: [0xff,0x02,0x46,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_ge_f32 s1, s105                   ; encoding: [0x01,0x69,0x46,0xbf]
 0x01,0x69,0x46,0xbf
+# GFX12: s_cmp_ge_f32 s1, s105                   ; encoding: [0x01,0x69,0x46,0xbf]
 
-# GFX12: s_cmp_o_f32 s1, s2                      ; encoding: [0x01,0x02,0x47,0xbf]
 0x01,0x02,0x47,0xbf
+# GFX12: s_cmp_o_f32 s1, s2                      ; encoding: [0x01,0x02,0x47,0xbf]
 
-# GFX12: s_cmp_o_f32 s105, s2                    ; encoding: [0x69,0x02,0x47,0xbf]
 0x69,0x02,0x47,0xbf
+# GFX12: s_cmp_o_f32 s105, s2                    ; encoding: [0x69,0x02,0x47,0xbf]
 
-# GFX12: s_cmp_o_f32 s101, s2                    ; encoding: [0x65,0x02,0x47,0xbf]
 0x65,0x02,0x47,0xbf
+# GFX12: s_cmp_o_f32 s101, s2                    ; encoding: [0x65,0x02,0x47,0xbf]
 
-# GFX12: s_cmp_o_f32 vcc_lo, s2                  ; encoding: [0x6a,0x02,0x47,0xbf]
 0x6a,0x02,0x47,0xbf
+# GFX12: s_cmp_o_f32 vcc_lo, s2                  ; encoding: [0x6a,0x02,0x47,0xbf]
 
-# GFX12: s_cmp_o_f32 vcc_hi, s2                  ; encoding: [0x6b,0x02,0x47,0xbf]
 0x6b,0x02,0x47,0xbf
+# GFX12: s_cmp_o_f32 vcc_hi, s2                  ; encoding: [0x6b,0x02,0x47,0xbf]
 
-# GFX12: s_cmp_o_f32 m0, s2                      ; encoding: [0x7d,0x02,0x47,0xbf]
 0x7d,0x02,0x47,0xbf
+# GFX12: s_cmp_o_f32 m0, s2                      ; encoding: [0x7d,0x02,0x47,0xbf]
 
-# GFX12: s_cmp_o_f32 exec_lo, s2                 ; encoding: [0x7e,0x02,0x47,0xbf]
 0x7e,0x02,0x47,0xbf
+# GFX12: s_cmp_o_f32 exec_lo, s2                 ; encoding: [0x7e,0x02,0x47,0xbf]
 
-# GFX12: s_cmp_o_f32 exec_hi, s2                 ; encoding: [0x7f,0x02,0x47,0xbf]
 0x7f,0x02,0x47,0xbf
+# GFX12: s_cmp_o_f32 exec_hi, s2                 ; encoding: [0x7f,0x02,0x47,0xbf]
 
-# GFX12: s_cmp_o_f32 0, s2                       ; encoding: [0x80,0x02,0x47,0xbf]
 0x80,0x02,0x47,0xbf
+# GFX12: s_cmp_o_f32 0, s2                       ; encoding: [0x80,0x02,0x47,0xbf]
 
-# GFX12: s_cmp_o_f32 -1, s2                      ; encoding: [0xc1,0x02,0x47,0xbf]
 0xc1,0x02,0x47,0xbf
+# GFX12: s_cmp_o_f32 -1, s2                      ; encoding: [0xc1,0x02,0x47,0xbf]
 
-# GFX12: s_cmp_o_f32 0.5, s2                     ; encoding: [0xf0,0x02,0x47,0xbf]
 0xf0,0x02,0x47,0xbf
+# GFX12: s_cmp_o_f32 0.5, s2                     ; encoding: [0xf0,0x02,0x47,0xbf]
 
-# GFX12: s_cmp_o_f32 -4.0, s2                    ; encoding: [0xf7,0x02,0x47,0xbf]
 0xf7,0x02,0x47,0xbf
+# GFX12: s_cmp_o_f32 -4.0, s2                    ; encoding: [0xf7,0x02,0x47,0xbf]
 
-# GFX12: s_cmp_o_f32 0xaf123456, s2              ; encoding: [0xff,0x02,0x47,0xbf,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x47,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_o_f32 0xaf123456, s2              ; encoding: [0xff,0x02,0x47,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_o_f32 0x3f717273, s2              ; encoding: [0xff,0x02,0x47,0xbf,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x47,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_o_f32 0x3f717273, s2              ; encoding: [0xff,0x02,0x47,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_o_f32 s1, s105                    ; encoding: [0x01,0x69,0x47,0xbf]
 0x01,0x69,0x47,0xbf
+# GFX12: s_cmp_o_f32 s1, s105                    ; encoding: [0x01,0x69,0x47,0xbf]
 
-# GFX12: s_cmp_u_f32 s1, s2                      ; encoding: [0x01,0x02,0x48,0xbf]
 0x01,0x02,0x48,0xbf
+# GFX12: s_cmp_u_f32 s1, s2                      ; encoding: [0x01,0x02,0x48,0xbf]
 
-# GFX12: s_cmp_u_f32 s105, s2                    ; encoding: [0x69,0x02,0x48,0xbf]
 0x69,0x02,0x48,0xbf
+# GFX12: s_cmp_u_f32 s105, s2                    ; encoding: [0x69,0x02,0x48,0xbf]
 
-# GFX12: s_cmp_u_f32 s101, s2                    ; encoding: [0x65,0x02,0x48,0xbf]
 0x65,0x02,0x48,0xbf
+# GFX12: s_cmp_u_f32 s101, s2                    ; encoding: [0x65,0x02,0x48,0xbf]
 
-# GFX12: s_cmp_u_f32 vcc_lo, s2                  ; encoding: [0x6a,0x02,0x48,0xbf]
 0x6a,0x02,0x48,0xbf
+# GFX12: s_cmp_u_f32 vcc_lo, s2                  ; encoding: [0x6a,0x02,0x48,0xbf]
 
-# GFX12: s_cmp_u_f32 vcc_hi, s2                  ; encoding: [0x6b,0x02,0x48,0xbf]
 0x6b,0x02,0x48,0xbf
+# GFX12: s_cmp_u_f32 vcc_hi, s2                  ; encoding: [0x6b,0x02,0x48,0xbf]
 
-# GFX12: s_cmp_u_f32 m0, s2                      ; encoding: [0x7d,0x02,0x48,0xbf]
 0x7d,0x02,0x48,0xbf
+# GFX12: s_cmp_u_f32 m0, s2                      ; encoding: [0x7d,0x02,0x48,0xbf]
 
-# GFX12: s_cmp_u_f32 exec_lo, s2                 ; encoding: [0x7e,0x02,0x48,0xbf]
 0x7e,0x02,0x48,0xbf
+# GFX12: s_cmp_u_f32 exec_lo, s2                 ; encoding: [0x7e,0x02,0x48,0xbf]
 
-# GFX12: s_cmp_u_f32 exec_hi, s2                 ; encoding: [0x7f,0x02,0x48,0xbf]
 0x7f,0x02,0x48,0xbf
+# GFX12: s_cmp_u_f32 exec_hi, s2                 ; encoding: [0x7f,0x02,0x48,0xbf]
 
-# GFX12: s_cmp_u_f32 0, s2                       ; encoding: [0x80,0x02,0x48,0xbf]
 0x80,0x02,0x48,0xbf
+# GFX12: s_cmp_u_f32 0, s2                       ; encoding: [0x80,0x02,0x48,0xbf]
 
-# GFX12: s_cmp_u_f32 -1, s2                      ; encoding: [0xc1,0x02,0x48,0xbf]
 0xc1,0x02,0x48,0xbf
+# GFX12: s_cmp_u_f32 -1, s2                      ; encoding: [0xc1,0x02,0x48,0xbf]
 
-# GFX12: s_cmp_u_f32 0.5, s2                     ; encoding: [0xf0,0x02,0x48,0xbf]
 0xf0,0x02,0x48,0xbf
+# GFX12: s_cmp_u_f32 0.5, s2                     ; encoding: [0xf0,0x02,0x48,0xbf]
 
-# GFX12: s_cmp_u_f32 -4.0, s2                    ; encoding: [0xf7,0x02,0x48,0xbf]
 0xf7,0x02,0x48,0xbf
+# GFX12: s_cmp_u_f32 -4.0, s2                    ; encoding: [0xf7,0x02,0x48,0xbf]
 
-# GFX12: s_cmp_u_f32 0xaf123456, s2              ; encoding: [0xff,0x02,0x48,0xbf,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x48,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_u_f32 0xaf123456, s2              ; encoding: [0xff,0x02,0x48,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_u_f32 0x3f717273, s2              ; encoding: [0xff,0x02,0x48,0xbf,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x48,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_u_f32 0x3f717273, s2              ; encoding: [0xff,0x02,0x48,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_u_f32 s1, s105                    ; encoding: [0x01,0x69,0x48,0xbf]
 0x01,0x69,0x48,0xbf
+# GFX12: s_cmp_u_f32 s1, s105                    ; encoding: [0x01,0x69,0x48,0xbf]
 
-# GFX12: s_cmp_nge_f32 s1, s2                    ; encoding: [0x01,0x02,0x49,0xbf]
 0x01,0x02,0x49,0xbf
+# GFX12: s_cmp_nge_f32 s1, s2                    ; encoding: [0x01,0x02,0x49,0xbf]
 
-# GFX12: s_cmp_nge_f32 s105, s2                  ; encoding: [0x69,0x02,0x49,0xbf]
 0x69,0x02,0x49,0xbf
+# GFX12: s_cmp_nge_f32 s105, s2                  ; encoding: [0x69,0x02,0x49,0xbf]
 
-# GFX12: s_cmp_nge_f32 s101, s2                  ; encoding: [0x65,0x02,0x49,0xbf]
 0x65,0x02,0x49,0xbf
+# GFX12: s_cmp_nge_f32 s101, s2                  ; encoding: [0x65,0x02,0x49,0xbf]
 
-# GFX12: s_cmp_nge_f32 vcc_lo, s2                ; encoding: [0x6a,0x02,0x49,0xbf]
 0x6a,0x02,0x49,0xbf
+# GFX12: s_cmp_nge_f32 vcc_lo, s2                ; encoding: [0x6a,0x02,0x49,0xbf]
 
-# GFX12: s_cmp_nge_f32 vcc_hi, s2                ; encoding: [0x6b,0x02,0x49,0xbf]
 0x6b,0x02,0x49,0xbf
+# GFX12: s_cmp_nge_f32 vcc_hi, s2                ; encoding: [0x6b,0x02,0x49,0xbf]
 
-# GFX12: s_cmp_nge_f32 m0, s2                    ; encoding: [0x7d,0x02,0x49,0xbf]
 0x7d,0x02,0x49,0xbf
+# GFX12: s_cmp_nge_f32 m0, s2                    ; encoding: [0x7d,0x02,0x49,0xbf]
 
-# GFX12: s_cmp_nge_f32 exec_lo, s2               ; encoding: [0x7e,0x02,0x49,0xbf]
 0x7e,0x02,0x49,0xbf
+# GFX12: s_cmp_nge_f32 exec_lo, s2               ; encoding: [0x7e,0x02,0x49,0xbf]
 
-# GFX12: s_cmp_nge_f32 exec_hi, s2               ; encoding: [0x7f,0x02,0x49,0xbf]
 0x7f,0x02,0x49,0xbf
+# GFX12: s_cmp_nge_f32 exec_hi, s2               ; encoding: [0x7f,0x02,0x49,0xbf]
 
-# GFX12: s_cmp_nge_f32 0, s2                     ; encoding: [0x80,0x02,0x49,0xbf]
 0x80,0x02,0x49,0xbf
+# GFX12: s_cmp_nge_f32 0, s2                     ; encoding: [0x80,0x02,0x49,0xbf]
 
-# GFX12: s_cmp_nge_f32 -1, s2                    ; encoding: [0xc1,0x02,0x49,0xbf]
 0xc1,0x02,0x49,0xbf
+# GFX12: s_cmp_nge_f32 -1, s2                    ; encoding: [0xc1,0x02,0x49,0xbf]
 
-# GFX12: s_cmp_nge_f32 0.5, s2                   ; encoding: [0xf0,0x02,0x49,0xbf]
 0xf0,0x02,0x49,0xbf
+# GFX12: s_cmp_nge_f32 0.5, s2                   ; encoding: [0xf0,0x02,0x49,0xbf]
 
-# GFX12: s_cmp_nge_f32 -4.0, s2                  ; encoding: [0xf7,0x02,0x49,0xbf]
 0xf7,0x02,0x49,0xbf
+# GFX12: s_cmp_nge_f32 -4.0, s2                  ; encoding: [0xf7,0x02,0x49,0xbf]
 
-# GFX12: s_cmp_nge_f32 0xaf123456, s2            ; encoding: [0xff,0x02,0x49,0xbf,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x49,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_nge_f32 0xaf123456, s2            ; encoding: [0xff,0x02,0x49,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_nge_f32 0x3f717273, s2            ; encoding: [0xff,0x02,0x49,0xbf,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x49,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_nge_f32 0x3f717273, s2            ; encoding: [0xff,0x02,0x49,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_nge_f32 s1, s105                  ; encoding: [0x01,0x69,0x49,0xbf]
 0x01,0x69,0x49,0xbf
+# GFX12: s_cmp_nge_f32 s1, s105                  ; encoding: [0x01,0x69,0x49,0xbf]
 
-# GFX12: s_cmp_nlg_f32 s1, s2                    ; encoding: [0x01,0x02,0x4a,0xbf]
 0x01,0x02,0x4a,0xbf
+# GFX12: s_cmp_nlg_f32 s1, s2                    ; encoding: [0x01,0x02,0x4a,0xbf]
 
-# GFX12: s_cmp_nlg_f32 s105, s2                  ; encoding: [0x69,0x02,0x4a,0xbf]
 0x69,0x02,0x4a,0xbf
+# GFX12: s_cmp_nlg_f32 s105, s2                  ; encoding: [0x69,0x02,0x4a,0xbf]
 
-# GFX12: s_cmp_nlg_f32 s101, s2                  ; encoding: [0x65,0x02,0x4a,0xbf]
 0x65,0x02,0x4a,0xbf
+# GFX12: s_cmp_nlg_f32 s101, s2                  ; encoding: [0x65,0x02,0x4a,0xbf]
 
-# GFX12: s_cmp_nlg_f32 vcc_lo, s2                ; encoding: [0x6a,0x02,0x4a,0xbf]
 0x6a,0x02,0x4a,0xbf
+# GFX12: s_cmp_nlg_f32 vcc_lo, s2                ; encoding: [0x6a,0x02,0x4a,0xbf]
 
-# GFX12: s_cmp_nlg_f32 vcc_hi, s2                ; encoding: [0x6b,0x02,0x4a,0xbf]
 0x6b,0x02,0x4a,0xbf
+# GFX12: s_cmp_nlg_f32 vcc_hi, s2                ; encoding: [0x6b,0x02,0x4a,0xbf]
 
-# GFX12: s_cmp_nlg_f32 m0, s2                    ; encoding: [0x7d,0x02,0x4a,0xbf]
 0x7d,0x02,0x4a,0xbf
+# GFX12: s_cmp_nlg_f32 m0, s2                    ; encoding: [0x7d,0x02,0x4a,0xbf]
 
-# GFX12: s_cmp_nlg_f32 exec_lo, s2               ; encoding: [0x7e,0x02,0x4a,0xbf]
 0x7e,0x02,0x4a,0xbf
+# GFX12: s_cmp_nlg_f32 exec_lo, s2               ; encoding: [0x7e,0x02,0x4a,0xbf]
 
-# GFX12: s_cmp_nlg_f32 exec_hi, s2               ; encoding: [0x7f,0x02,0x4a,0xbf]
 0x7f,0x02,0x4a,0xbf
+# GFX12: s_cmp_nlg_f32 exec_hi, s2               ; encoding: [0x7f,0x02,0x4a,0xbf]
 
-# GFX12: s_cmp_nlg_f32 0, s2                     ; encoding: [0x80,0x02,0x4a,0xbf]
 0x80,0x02,0x4a,0xbf
+# GFX12: s_cmp_nlg_f32 0, s2                     ; encoding: [0x80,0x02,0x4a,0xbf]
 
-# GFX12: s_cmp_nlg_f32 -1, s2                    ; encoding: [0xc1,0x02,0x4a,0xbf]
 0xc1,0x02,0x4a,0xbf
+# GFX12: s_cmp_nlg_f32 -1, s2                    ; encoding: [0xc1,0x02,0x4a,0xbf]
 
-# GFX12: s_cmp_nlg_f32 0.5, s2                   ; encoding: [0xf0,0x02,0x4a,0xbf]
 0xf0,0x02,0x4a,0xbf
+# GFX12: s_cmp_nlg_f32 0.5, s2                   ; encoding: [0xf0,0x02,0x4a,0xbf]
 
-# GFX12: s_cmp_nlg_f32 -4.0, s2                  ; encoding: [0xf7,0x02,0x4a,0xbf]
 0xf7,0x02,0x4a,0xbf
+# GFX12: s_cmp_nlg_f32 -4.0, s2                  ; encoding: [0xf7,0x02,0x4a,0xbf]
 
-# GFX12: s_cmp_nlg_f32 0xaf123456, s2            ; encoding: [0xff,0x02,0x4a,0xbf,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x4a,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_nlg_f32 0xaf123456, s2            ; encoding: [0xff,0x02,0x4a,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_nlg_f32 0x3f717273, s2            ; encoding: [0xff,0x02,0x4a,0xbf,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x4a,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_nlg_f32 0x3f717273, s2            ; encoding: [0xff,0x02,0x4a,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_nlg_f32 s1, s105                  ; encoding: [0x01,0x69,0x4a,0xbf]
 0x01,0x69,0x4a,0xbf
+# GFX12: s_cmp_nlg_f32 s1, s105                  ; encoding: [0x01,0x69,0x4a,0xbf]
 
-# GFX12: s_cmp_ngt_f32 s1, s2                    ; encoding: [0x01,0x02,0x4b,0xbf]
 0x01,0x02,0x4b,0xbf
+# GFX12: s_cmp_ngt_f32 s1, s2                    ; encoding: [0x01,0x02,0x4b,0xbf]
 
-# GFX12: s_cmp_ngt_f32 s105, s2                  ; encoding: [0x69,0x02,0x4b,0xbf]
 0x69,0x02,0x4b,0xbf
+# GFX12: s_cmp_ngt_f32 s105, s2                  ; encoding: [0x69,0x02,0x4b,0xbf]
 
-# GFX12: s_cmp_ngt_f32 s101, s2                  ; encoding: [0x65,0x02,0x4b,0xbf]
 0x65,0x02,0x4b,0xbf
+# GFX12: s_cmp_ngt_f32 s101, s2                  ; encoding: [0x65,0x02,0x4b,0xbf]
 
-# GFX12: s_cmp_ngt_f32 vcc_lo, s2                ; encoding: [0x6a,0x02,0x4b,0xbf]
 0x6a,0x02,0x4b,0xbf
+# GFX12: s_cmp_ngt_f32 vcc_lo, s2                ; encoding: [0x6a,0x02,0x4b,0xbf]
 
-# GFX12: s_cmp_ngt_f32 vcc_hi, s2                ; encoding: [0x6b,0x02,0x4b,0xbf]
 0x6b,0x02,0x4b,0xbf
+# GFX12: s_cmp_ngt_f32 vcc_hi, s2                ; encoding: [0x6b,0x02,0x4b,0xbf]
 
-# GFX12: s_cmp_ngt_f32 m0, s2                    ; encoding: [0x7d,0x02,0x4b,0xbf]
 0x7d,0x02,0x4b,0xbf
+# GFX12: s_cmp_ngt_f32 m0, s2                    ; encoding: [0x7d,0x02,0x4b,0xbf]
 
-# GFX12: s_cmp_ngt_f32 exec_lo, s2               ; encoding: [0x7e,0x02,0x4b,0xbf]
 0x7e,0x02,0x4b,0xbf
+# GFX12: s_cmp_ngt_f32 exec_lo, s2               ; encoding: [0x7e,0x02,0x4b,0xbf]
 
-# GFX12: s_cmp_ngt_f32 exec_hi, s2               ; encoding: [0x7f,0x02,0x4b,0xbf]
 0x7f,0x02,0x4b,0xbf
+# GFX12: s_cmp_ngt_f32 exec_hi, s2               ; encoding: [0x7f,0x02,0x4b,0xbf]
 
-# GFX12: s_cmp_ngt_f32 0, s2                     ; encoding: [0x80,0x02,0x4b,0xbf]
 0x80,0x02,0x4b,0xbf
+# GFX12: s_cmp_ngt_f32 0, s2                     ; encoding: [0x80,0x02,0x4b,0xbf]
 
-# GFX12: s_cmp_ngt_f32 -1, s2                    ; encoding: [0xc1,0x02,0x4b,0xbf]
 0xc1,0x02,0x4b,0xbf
+# GFX12: s_cmp_ngt_f32 -1, s2                    ; encoding: [0xc1,0x02,0x4b,0xbf]
 
-# GFX12: s_cmp_ngt_f32 0.5, s2                   ; encoding: [0xf0,0x02,0x4b,0xbf]
 0xf0,0x02,0x4b,0xbf
+# GFX12: s_cmp_ngt_f32 0.5, s2                   ; encoding: [0xf0,0x02,0x4b,0xbf]
 
-# GFX12: s_cmp_ngt_f32 -4.0, s2                  ; encoding: [0xf7,0x02,0x4b,0xbf]
 0xf7,0x02,0x4b,0xbf
+# GFX12: s_cmp_ngt_f32 -4.0, s2                  ; encoding: [0xf7,0x02,0x4b,0xbf]
 
-# GFX12: s_cmp_ngt_f32 0xaf123456, s2            ; encoding: [0xff,0x02,0x4b,0xbf,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x4b,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_ngt_f32 0xaf123456, s2            ; encoding: [0xff,0x02,0x4b,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_ngt_f32 0x3f717273, s2            ; encoding: [0xff,0x02,0x4b,0xbf,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x4b,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_ngt_f32 0x3f717273, s2            ; encoding: [0xff,0x02,0x4b,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_ngt_f32 s1, s105                  ; encoding: [0x01,0x69,0x4b,0xbf]
 0x01,0x69,0x4b,0xbf
+# GFX12: s_cmp_ngt_f32 s1, s105                  ; encoding: [0x01,0x69,0x4b,0xbf]
 
-# GFX12: s_cmp_nle_f32 s1, s2                    ; encoding: [0x01,0x02,0x4c,0xbf]
 0x01,0x02,0x4c,0xbf
+# GFX12: s_cmp_nle_f32 s1, s2                    ; encoding: [0x01,0x02,0x4c,0xbf]
 
-# GFX12: s_cmp_nle_f32 s105, s2                  ; encoding: [0x69,0x02,0x4c,0xbf]
 0x69,0x02,0x4c,0xbf
+# GFX12: s_cmp_nle_f32 s105, s2                  ; encoding: [0x69,0x02,0x4c,0xbf]
 
-# GFX12: s_cmp_nle_f32 s101, s2                  ; encoding: [0x65,0x02,0x4c,0xbf]
 0x65,0x02,0x4c,0xbf
+# GFX12: s_cmp_nle_f32 s101, s2                  ; encoding: [0x65,0x02,0x4c,0xbf]
 
-# GFX12: s_cmp_nle_f32 vcc_lo, s2                ; encoding: [0x6a,0x02,0x4c,0xbf]
 0x6a,0x02,0x4c,0xbf
+# GFX12: s_cmp_nle_f32 vcc_lo, s2                ; encoding: [0x6a,0x02,0x4c,0xbf]
 
-# GFX12: s_cmp_nle_f32 vcc_hi, s2                ; encoding: [0x6b,0x02,0x4c,0xbf]
 0x6b,0x02,0x4c,0xbf
+# GFX12: s_cmp_nle_f32 vcc_hi, s2                ; encoding: [0x6b,0x02,0x4c,0xbf]
 
-# GFX12: s_cmp_nle_f32 m0, s2                    ; encoding: [0x7d,0x02,0x4c,0xbf]
 0x7d,0x02,0x4c,0xbf
+# GFX12: s_cmp_nle_f32 m0, s2                    ; encoding: [0x7d,0x02,0x4c,0xbf]
 
-# GFX12: s_cmp_nle_f32 exec_lo, s2               ; encoding: [0x7e,0x02,0x4c,0xbf]
 0x7e,0x02,0x4c,0xbf
+# GFX12: s_cmp_nle_f32 exec_lo, s2               ; encoding: [0x7e,0x02,0x4c,0xbf]
 
-# GFX12: s_cmp_nle_f32 exec_hi, s2               ; encoding: [0x7f,0x02,0x4c,0xbf]
 0x7f,0x02,0x4c,0xbf
+# GFX12: s_cmp_nle_f32 exec_hi, s2               ; encoding: [0x7f,0x02,0x4c,0xbf]
 
-# GFX12: s_cmp_nle_f32 0, s2                     ; encoding: [0x80,0x02,0x4c,0xbf]
 0x80,0x02,0x4c,0xbf
+# GFX12: s_cmp_nle_f32 0, s2                     ; encoding: [0x80,0x02,0x4c,0xbf]
 
-# GFX12: s_cmp_nle_f32 -1, s2                    ; encoding: [0xc1,0x02,0x4c,0xbf]
 0xc1,0x02,0x4c,0xbf
+# GFX12: s_cmp_nle_f32 -1, s2                    ; encoding: [0xc1,0x02,0x4c,0xbf]
 
-# GFX12: s_cmp_nle_f32 0.5, s2                   ; encoding: [0xf0,0x02,0x4c,0xbf]
 0xf0,0x02,0x4c,0xbf
+# GFX12: s_cmp_nle_f32 0.5, s2                   ; encoding: [0xf0,0x02,0x4c,0xbf]
 
-# GFX12: s_cmp_nle_f32 -4.0, s2                  ; encoding: [0xf7,0x02,0x4c,0xbf]
 0xf7,0x02,0x4c,0xbf
+# GFX12: s_cmp_nle_f32 -4.0, s2                  ; encoding: [0xf7,0x02,0x4c,0xbf]
 
-# GFX12: s_cmp_nle_f32 0xaf123456, s2            ; encoding: [0xff,0x02,0x4c,0xbf,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x4c,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_nle_f32 0xaf123456, s2            ; encoding: [0xff,0x02,0x4c,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_nle_f32 0x3f717273, s2            ; encoding: [0xff,0x02,0x4c,0xbf,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x4c,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_nle_f32 0x3f717273, s2            ; encoding: [0xff,0x02,0x4c,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_nle_f32 s1, s105                  ; encoding: [0x01,0x69,0x4c,0xbf]
 0x01,0x69,0x4c,0xbf
+# GFX12: s_cmp_nle_f32 s1, s105                  ; encoding: [0x01,0x69,0x4c,0xbf]
 
-# GFX12: s_cmp_neq_f32 s1, s2                    ; encoding: [0x01,0x02,0x4d,0xbf]
 0x01,0x02,0x4d,0xbf
+# GFX12: s_cmp_neq_f32 s1, s2                    ; encoding: [0x01,0x02,0x4d,0xbf]
 
-# GFX12: s_cmp_neq_f32 s105, s2                  ; encoding: [0x69,0x02,0x4d,0xbf]
 0x69,0x02,0x4d,0xbf
+# GFX12: s_cmp_neq_f32 s105, s2                  ; encoding: [0x69,0x02,0x4d,0xbf]
 
-# GFX12: s_cmp_neq_f32 s101, s2                  ; encoding: [0x65,0x02,0x4d,0xbf]
 0x65,0x02,0x4d,0xbf
+# GFX12: s_cmp_neq_f32 s101, s2                  ; encoding: [0x65,0x02,0x4d,0xbf]
 
-# GFX12: s_cmp_neq_f32 vcc_lo, s2                ; encoding: [0x6a,0x02,0x4d,0xbf]
 0x6a,0x02,0x4d,0xbf
+# GFX12: s_cmp_neq_f32 vcc_lo, s2                ; encoding: [0x6a,0x02,0x4d,0xbf]
 
-# GFX12: s_cmp_neq_f32 vcc_hi, s2                ; encoding: [0x6b,0x02,0x4d,0xbf]
 0x6b,0x02,0x4d,0xbf
+# GFX12: s_cmp_neq_f32 vcc_hi, s2                ; encoding: [0x6b,0x02,0x4d,0xbf]
 
-# GFX12: s_cmp_neq_f32 m0, s2                    ; encoding: [0x7d,0x02,0x4d,0xbf]
 0x7d,0x02,0x4d,0xbf
+# GFX12: s_cmp_neq_f32 m0, s2                    ; encoding: [0x7d,0x02,0x4d,0xbf]
 
-# GFX12: s_cmp_neq_f32 exec_lo, s2               ; encoding: [0x7e,0x02,0x4d,0xbf]
 0x7e,0x02,0x4d,0xbf
+# GFX12: s_cmp_neq_f32 exec_lo, s2               ; encoding: [0x7e,0x02,0x4d,0xbf]
 
-# GFX12: s_cmp_neq_f32 exec_hi, s2               ; encoding: [0x7f,0x02,0x4d,0xbf]
 0x7f,0x02,0x4d,0xbf
+# GFX12: s_cmp_neq_f32 exec_hi, s2               ; encoding: [0x7f,0x02,0x4d,0xbf]
 
-# GFX12: s_cmp_neq_f32 0, s2                     ; encoding: [0x80,0x02,0x4d,0xbf]
 0x80,0x02,0x4d,0xbf
+# GFX12: s_cmp_neq_f32 0, s2                     ; encoding: [0x80,0x02,0x4d,0xbf]
 
-# GFX12: s_cmp_neq_f32 -1, s2                    ; encoding: [0xc1,0x02,0x4d,0xbf]
 0xc1,0x02,0x4d,0xbf
+# GFX12: s_cmp_neq_f32 -1, s2                    ; encoding: [0xc1,0x02,0x4d,0xbf]
 
-# GFX12: s_cmp_neq_f32 0.5, s2                   ; encoding: [0xf0,0x02,0x4d,0xbf]
 0xf0,0x02,0x4d,0xbf
+# GFX12: s_cmp_neq_f32 0.5, s2                   ; encoding: [0xf0,0x02,0x4d,0xbf]
 
-# GFX12: s_cmp_neq_f32 -4.0, s2                  ; encoding: [0xf7,0x02,0x4d,0xbf]
 0xf7,0x02,0x4d,0xbf
+# GFX12: s_cmp_neq_f32 -4.0, s2                  ; encoding: [0xf7,0x02,0x4d,0xbf]
 
-# GFX12: s_cmp_neq_f32 0xaf123456, s2            ; encoding: [0xff,0x02,0x4d,0xbf,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x4d,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_neq_f32 0xaf123456, s2            ; encoding: [0xff,0x02,0x4d,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_neq_f32 0x3f717273, s2            ; encoding: [0xff,0x02,0x4d,0xbf,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x4d,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_neq_f32 0x3f717273, s2            ; encoding: [0xff,0x02,0x4d,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_neq_f32 s1, s105                  ; encoding: [0x01,0x69,0x4d,0xbf]
 0x01,0x69,0x4d,0xbf
+# GFX12: s_cmp_neq_f32 s1, s105                  ; encoding: [0x01,0x69,0x4d,0xbf]
 
-# GFX12: s_cmp_nlt_f32 s1, s2                    ; encoding: [0x01,0x02,0x4e,0xbf]
 0x01,0x02,0x4e,0xbf
+# GFX12: s_cmp_nlt_f32 s1, s2                    ; encoding: [0x01,0x02,0x4e,0xbf]
 
-# GFX12: s_cmp_nlt_f32 s105, s2                  ; encoding: [0x69,0x02,0x4e,0xbf]
 0x69,0x02,0x4e,0xbf
+# GFX12: s_cmp_nlt_f32 s105, s2                  ; encoding: [0x69,0x02,0x4e,0xbf]
 
-# GFX12: s_cmp_nlt_f32 s101, s2                  ; encoding: [0x65,0x02,0x4e,0xbf]
 0x65,0x02,0x4e,0xbf
+# GFX12: s_cmp_nlt_f32 s101, s2                  ; encoding: [0x65,0x02,0x4e,0xbf]
 
-# GFX12: s_cmp_nlt_f32 vcc_lo, s2                ; encoding: [0x6a,0x02,0x4e,0xbf]
 0x6a,0x02,0x4e,0xbf
+# GFX12: s_cmp_nlt_f32 vcc_lo, s2                ; encoding: [0x6a,0x02,0x4e,0xbf]
 
-# GFX12: s_cmp_nlt_f32 vcc_hi, s2                ; encoding: [0x6b,0x02,0x4e,0xbf]
 0x6b,0x02,0x4e,0xbf
+# GFX12: s_cmp_nlt_f32 vcc_hi, s2                ; encoding: [0x6b,0x02,0x4e,0xbf]
 
-# GFX12: s_cmp_nlt_f32 m0, s2                    ; encoding: [0x7d,0x02,0x4e,0xbf]
 0x7d,0x02,0x4e,0xbf
+# GFX12: s_cmp_nlt_f32 m0, s2                    ; encoding: [0x7d,0x02,0x4e,0xbf]
 
-# GFX12: s_cmp_nlt_f32 exec_lo, s2               ; encoding: [0x7e,0x02,0x4e,0xbf]
 0x7e,0x02,0x4e,0xbf
+# GFX12: s_cmp_nlt_f32 exec_lo, s2               ; encoding: [0x7e,0x02,0x4e,0xbf]
 
-# GFX12: s_cmp_nlt_f32 exec_hi, s2               ; encoding: [0x7f,0x02,0x4e,0xbf]
 0x7f,0x02,0x4e,0xbf
+# GFX12: s_cmp_nlt_f32 exec_hi, s2               ; encoding: [0x7f,0x02,0x4e,0xbf]
 
-# GFX12: s_cmp_nlt_f32 0, s2                     ; encoding: [0x80,0x02,0x4e,0xbf]
 0x80,0x02,0x4e,0xbf
+# GFX12: s_cmp_nlt_f32 0, s2                     ; encoding: [0x80,0x02,0x4e,0xbf]
 
-# GFX12: s_cmp_nlt_f32 -1, s2                    ; encoding: [0xc1,0x02,0x4e,0xbf]
 0xc1,0x02,0x4e,0xbf
+# GFX12: s_cmp_nlt_f32 -1, s2                    ; encoding: [0xc1,0x02,0x4e,0xbf]
 
-# GFX12: s_cmp_nlt_f32 0.5, s2                   ; encoding: [0xf0,0x02,0x4e,0xbf]
 0xf0,0x02,0x4e,0xbf
+# GFX12: s_cmp_nlt_f32 0.5, s2                   ; encoding: [0xf0,0x02,0x4e,0xbf]
 
-# GFX12: s_cmp_nlt_f32 -4.0, s2                  ; encoding: [0xf7,0x02,0x4e,0xbf]
 0xf7,0x02,0x4e,0xbf
+# GFX12: s_cmp_nlt_f32 -4.0, s2                  ; encoding: [0xf7,0x02,0x4e,0xbf]
 
-# GFX12: s_cmp_nlt_f32 0xaf123456, s2            ; encoding: [0xff,0x02,0x4e,0xbf,0x56,0x34,0x12,0xaf]
 0xff,0x02,0x4e,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_nlt_f32 0xaf123456, s2            ; encoding: [0xff,0x02,0x4e,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_nlt_f32 0x3f717273, s2            ; encoding: [0xff,0x02,0x4e,0xbf,0x73,0x72,0x71,0x3f]
 0xff,0x02,0x4e,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_nlt_f32 0x3f717273, s2            ; encoding: [0xff,0x02,0x4e,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_nlt_f32 s1, s105                  ; encoding: [0x01,0x69,0x4e,0xbf]
 0x01,0x69,0x4e,0xbf
+# GFX12: s_cmp_nlt_f32 s1, s105                  ; encoding: [0x01,0x69,0x4e,0xbf]
 
-# GFX12: s_cmp_lt_f16 s1, s2                     ; encoding: [0x01,0x02,0x51,0xbf]
 0x01,0x02,0x51,0xbf
+# GFX12: s_cmp_lt_f16 s1, s2                     ; encoding: [0x01,0x02,0x51,0xbf]
 
-# GFX12: s_cmp_lt_f16 s101, s2                   ; encoding: [0x65,0x02,0x51,0xbf]
 0x65,0x02,0x51,0xbf
+# GFX12: s_cmp_lt_f16 s101, s2                   ; encoding: [0x65,0x02,0x51,0xbf]
 
-# GFX12: s_cmp_lt_f16 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x51,0xbf]
 0x6a,0x02,0x51,0xbf
+# GFX12: s_cmp_lt_f16 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x51,0xbf]
 
-# GFX12: s_cmp_lt_f16 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x51,0xbf]
 0x6b,0x02,0x51,0xbf
+# GFX12: s_cmp_lt_f16 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x51,0xbf]
 
-# GFX12: s_cmp_lt_f16 m0, s2                     ; encoding: [0x7d,0x02,0x51,0xbf]
 0x7d,0x02,0x51,0xbf
+# GFX12: s_cmp_lt_f16 m0, s2                     ; encoding: [0x7d,0x02,0x51,0xbf]
 
-# GFX12: s_cmp_lt_f16 exec_lo, s2                ; encoding: [0x7e,0x02,0x51,0xbf]
 0x7e,0x02,0x51,0xbf
+# GFX12: s_cmp_lt_f16 exec_lo, s2                ; encoding: [0x7e,0x02,0x51,0xbf]
 
-# GFX12: s_cmp_lt_f16 exec_hi, s2                ; encoding: [0x7f,0x02,0x51,0xbf]
 0x7f,0x02,0x51,0xbf
+# GFX12: s_cmp_lt_f16 exec_hi, s2                ; encoding: [0x7f,0x02,0x51,0xbf]
 
-# GFX12: s_cmp_lt_f16 0, s2                      ; encoding: [0x80,0x02,0x51,0xbf]
 0x80,0x02,0x51,0xbf
+# GFX12: s_cmp_lt_f16 0, s2                      ; encoding: [0x80,0x02,0x51,0xbf]
 
-# GFX12: s_cmp_lt_f16 -1, s2                     ; encoding: [0xc1,0x02,0x51,0xbf]
 0xc1,0x02,0x51,0xbf
+# GFX12: s_cmp_lt_f16 -1, s2                     ; encoding: [0xc1,0x02,0x51,0xbf]
 
-# GFX12: s_cmp_lt_f16 0x3800, s2                 ; encoding: [0xff,0x02,0x51,0xbf,0x00,0x38,0x00,0x00]
 0xff,0x02,0x51,0xbf,0x00,0x38,0x00,0x00
+# GFX12: s_cmp_lt_f16 0x3800, s2                 ; encoding: [0xff,0x02,0x51,0xbf,0x00,0x38,0x00,0x00]
 
-# GFX12: s_cmp_lt_f16 0xfe0b, s2                 ; encoding: [0xff,0x02,0x51,0xbf,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x51,0xbf,0x0b,0xfe,0x00,0x00
+# GFX12: s_cmp_lt_f16 0xfe0b, s2                 ; encoding: [0xff,0x02,0x51,0xbf,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_cmp_lt_f16 0x3456, s2                 ; encoding: [0xff,0x02,0x51,0xbf,0x56,0x34,0x00,0x00]
 0xff,0x02,0x51,0xbf,0x56,0x34,0x00,0x00
+# GFX12: s_cmp_lt_f16 0x3456, s2                 ; encoding: [0xff,0x02,0x51,0xbf,0x56,0x34,0x00,0x00]
 
-# GFX12: s_cmp_eq_f16 s1, s2                     ; encoding: [0x01,0x02,0x52,0xbf]
 0x01,0x02,0x52,0xbf
+# GFX12: s_cmp_eq_f16 s1, s2                     ; encoding: [0x01,0x02,0x52,0xbf]
 
-# GFX12: s_cmp_eq_f16 s101, s2                   ; encoding: [0x65,0x02,0x52,0xbf]
 0x65,0x02,0x52,0xbf
+# GFX12: s_cmp_eq_f16 s101, s2                   ; encoding: [0x65,0x02,0x52,0xbf]
 
-# GFX12: s_cmp_eq_f16 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x52,0xbf]
 0x6a,0x02,0x52,0xbf
+# GFX12: s_cmp_eq_f16 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x52,0xbf]
 
-# GFX12: s_cmp_eq_f16 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x52,0xbf]
 0x6b,0x02,0x52,0xbf
+# GFX12: s_cmp_eq_f16 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x52,0xbf]
 
-# GFX12: s_cmp_eq_f16 m0, s2                     ; encoding: [0x7d,0x02,0x52,0xbf]
 0x7d,0x02,0x52,0xbf
+# GFX12: s_cmp_eq_f16 m0, s2                     ; encoding: [0x7d,0x02,0x52,0xbf]
 
-# GFX12: s_cmp_eq_f16 exec_lo, s2                ; encoding: [0x7e,0x02,0x52,0xbf]
 0x7e,0x02,0x52,0xbf
+# GFX12: s_cmp_eq_f16 exec_lo, s2                ; encoding: [0x7e,0x02,0x52,0xbf]
 
-# GFX12: s_cmp_eq_f16 exec_hi, s2                ; encoding: [0x7f,0x02,0x52,0xbf]
 0x7f,0x02,0x52,0xbf
+# GFX12: s_cmp_eq_f16 exec_hi, s2                ; encoding: [0x7f,0x02,0x52,0xbf]
 
-# GFX12: s_cmp_eq_f16 0, s2                      ; encoding: [0x80,0x02,0x52,0xbf]
 0x80,0x02,0x52,0xbf
+# GFX12: s_cmp_eq_f16 0, s2                      ; encoding: [0x80,0x02,0x52,0xbf]
 
-# GFX12: s_cmp_eq_f16 -1, s2                     ; encoding: [0xc1,0x02,0x52,0xbf]
 0xc1,0x02,0x52,0xbf
+# GFX12: s_cmp_eq_f16 -1, s2                     ; encoding: [0xc1,0x02,0x52,0xbf]
 
-# GFX12: s_cmp_eq_f16 0x3800, s2                 ; encoding: [0xff,0x02,0x52,0xbf,0x00,0x38,0x00,0x00]
 0xff,0x02,0x52,0xbf,0x00,0x38,0x00,0x00
+# GFX12: s_cmp_eq_f16 0x3800, s2                 ; encoding: [0xff,0x02,0x52,0xbf,0x00,0x38,0x00,0x00]
 
-# GFX12: s_cmp_eq_f16 0xfe0b, s2                 ; encoding: [0xff,0x02,0x52,0xbf,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x52,0xbf,0x0b,0xfe,0x00,0x00
+# GFX12: s_cmp_eq_f16 0xfe0b, s2                 ; encoding: [0xff,0x02,0x52,0xbf,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_cmp_eq_f16 0x3456, s2                 ; encoding: [0xff,0x02,0x52,0xbf,0x56,0x34,0x00,0x00]
 0xff,0x02,0x52,0xbf,0x56,0x34,0x00,0x00
+# GFX12: s_cmp_eq_f16 0x3456, s2                 ; encoding: [0xff,0x02,0x52,0xbf,0x56,0x34,0x00,0x00]
 
-# GFX12: s_cmp_le_f16 s1, s2                     ; encoding: [0x01,0x02,0x53,0xbf]
 0x01,0x02,0x53,0xbf
+# GFX12: s_cmp_le_f16 s1, s2                     ; encoding: [0x01,0x02,0x53,0xbf]
 
-# GFX12: s_cmp_le_f16 s101, s2                   ; encoding: [0x65,0x02,0x53,0xbf]
 0x65,0x02,0x53,0xbf
+# GFX12: s_cmp_le_f16 s101, s2                   ; encoding: [0x65,0x02,0x53,0xbf]
 
-# GFX12: s_cmp_le_f16 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x53,0xbf]
 0x6a,0x02,0x53,0xbf
+# GFX12: s_cmp_le_f16 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x53,0xbf]
 
-# GFX12: s_cmp_le_f16 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x53,0xbf]
 0x6b,0x02,0x53,0xbf
+# GFX12: s_cmp_le_f16 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x53,0xbf]
 
-# GFX12: s_cmp_le_f16 m0, s2                     ; encoding: [0x7d,0x02,0x53,0xbf]
 0x7d,0x02,0x53,0xbf
+# GFX12: s_cmp_le_f16 m0, s2                     ; encoding: [0x7d,0x02,0x53,0xbf]
 
-# GFX12: s_cmp_le_f16 exec_lo, s2                ; encoding: [0x7e,0x02,0x53,0xbf]
 0x7e,0x02,0x53,0xbf
+# GFX12: s_cmp_le_f16 exec_lo, s2                ; encoding: [0x7e,0x02,0x53,0xbf]
 
-# GFX12: s_cmp_le_f16 exec_hi, s2                ; encoding: [0x7f,0x02,0x53,0xbf]
 0x7f,0x02,0x53,0xbf
+# GFX12: s_cmp_le_f16 exec_hi, s2                ; encoding: [0x7f,0x02,0x53,0xbf]
 
-# GFX12: s_cmp_le_f16 0, s2                      ; encoding: [0x80,0x02,0x53,0xbf]
 0x80,0x02,0x53,0xbf
+# GFX12: s_cmp_le_f16 0, s2                      ; encoding: [0x80,0x02,0x53,0xbf]
 
-# GFX12: s_cmp_le_f16 -1, s2                     ; encoding: [0xc1,0x02,0x53,0xbf]
 0xc1,0x02,0x53,0xbf
+# GFX12: s_cmp_le_f16 -1, s2                     ; encoding: [0xc1,0x02,0x53,0xbf]
 
-# GFX12: s_cmp_le_f16 0x3800, s2                 ; encoding: [0xff,0x02,0x53,0xbf,0x00,0x38,0x00,0x00]
 0xff,0x02,0x53,0xbf,0x00,0x38,0x00,0x00
+# GFX12: s_cmp_le_f16 0x3800, s2                 ; encoding: [0xff,0x02,0x53,0xbf,0x00,0x38,0x00,0x00]
 
-# GFX12: s_cmp_le_f16 0xfe0b, s2                 ; encoding: [0xff,0x02,0x53,0xbf,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x53,0xbf,0x0b,0xfe,0x00,0x00
+# GFX12: s_cmp_le_f16 0xfe0b, s2                 ; encoding: [0xff,0x02,0x53,0xbf,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_cmp_le_f16 0x3456, s2                 ; encoding: [0xff,0x02,0x53,0xbf,0x56,0x34,0x00,0x00]
 0xff,0x02,0x53,0xbf,0x56,0x34,0x00,0x00
+# GFX12: s_cmp_le_f16 0x3456, s2                 ; encoding: [0xff,0x02,0x53,0xbf,0x56,0x34,0x00,0x00]
 
-# GFX12: s_cmp_gt_f16 s1, s2                     ; encoding: [0x01,0x02,0x54,0xbf]
 0x01,0x02,0x54,0xbf
+# GFX12: s_cmp_gt_f16 s1, s2                     ; encoding: [0x01,0x02,0x54,0xbf]
 
-# GFX12: s_cmp_gt_f16 s101, s2                   ; encoding: [0x65,0x02,0x54,0xbf]
 0x65,0x02,0x54,0xbf
+# GFX12: s_cmp_gt_f16 s101, s2                   ; encoding: [0x65,0x02,0x54,0xbf]
 
-# GFX12: s_cmp_gt_f16 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x54,0xbf]
 0x6a,0x02,0x54,0xbf
+# GFX12: s_cmp_gt_f16 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x54,0xbf]
 
-# GFX12: s_cmp_gt_f16 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x54,0xbf]
 0x6b,0x02,0x54,0xbf
+# GFX12: s_cmp_gt_f16 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x54,0xbf]
 
-# GFX12: s_cmp_gt_f16 m0, s2                     ; encoding: [0x7d,0x02,0x54,0xbf]
 0x7d,0x02,0x54,0xbf
+# GFX12: s_cmp_gt_f16 m0, s2                     ; encoding: [0x7d,0x02,0x54,0xbf]
 
-# GFX12: s_cmp_gt_f16 exec_lo, s2                ; encoding: [0x7e,0x02,0x54,0xbf]
 0x7e,0x02,0x54,0xbf
+# GFX12: s_cmp_gt_f16 exec_lo, s2                ; encoding: [0x7e,0x02,0x54,0xbf]
 
-# GFX12: s_cmp_gt_f16 exec_hi, s2                ; encoding: [0x7f,0x02,0x54,0xbf]
 0x7f,0x02,0x54,0xbf
+# GFX12: s_cmp_gt_f16 exec_hi, s2                ; encoding: [0x7f,0x02,0x54,0xbf]
 
-# GFX12: s_cmp_gt_f16 0, s2                      ; encoding: [0x80,0x02,0x54,0xbf]
 0x80,0x02,0x54,0xbf
+# GFX12: s_cmp_gt_f16 0, s2                      ; encoding: [0x80,0x02,0x54,0xbf]
 
-# GFX12: s_cmp_gt_f16 -1, s2                     ; encoding: [0xc1,0x02,0x54,0xbf]
 0xc1,0x02,0x54,0xbf
+# GFX12: s_cmp_gt_f16 -1, s2                     ; encoding: [0xc1,0x02,0x54,0xbf]
 
-# GFX12: s_cmp_gt_f16 0x3800, s2                 ; encoding: [0xff,0x02,0x54,0xbf,0x00,0x38,0x00,0x00]
 0xff,0x02,0x54,0xbf,0x00,0x38,0x00,0x00
+# GFX12: s_cmp_gt_f16 0x3800, s2                 ; encoding: [0xff,0x02,0x54,0xbf,0x00,0x38,0x00,0x00]
 
-# GFX12: s_cmp_gt_f16 0xfe0b, s2                 ; encoding: [0xff,0x02,0x54,0xbf,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x54,0xbf,0x0b,0xfe,0x00,0x00
+# GFX12: s_cmp_gt_f16 0xfe0b, s2                 ; encoding: [0xff,0x02,0x54,0xbf,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_cmp_gt_f16 0x3456, s2                 ; encoding: [0xff,0x02,0x54,0xbf,0x56,0x34,0x00,0x00]
 0xff,0x02,0x54,0xbf,0x56,0x34,0x00,0x00
+# GFX12: s_cmp_gt_f16 0x3456, s2                 ; encoding: [0xff,0x02,0x54,0xbf,0x56,0x34,0x00,0x00]
 
-# GFX12: s_cmp_lg_f16 s1, s2                     ; encoding: [0x01,0x02,0x55,0xbf]
 0x01,0x02,0x55,0xbf
+# GFX12: s_cmp_lg_f16 s1, s2                     ; encoding: [0x01,0x02,0x55,0xbf]
 
-# GFX12: s_cmp_lg_f16 s101, s2                   ; encoding: [0x65,0x02,0x55,0xbf]
 0x65,0x02,0x55,0xbf
+# GFX12: s_cmp_lg_f16 s101, s2                   ; encoding: [0x65,0x02,0x55,0xbf]
 
-# GFX12: s_cmp_lg_f16 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x55,0xbf]
 0x6a,0x02,0x55,0xbf
+# GFX12: s_cmp_lg_f16 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x55,0xbf]
 
-# GFX12: s_cmp_lg_f16 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x55,0xbf]
 0x6b,0x02,0x55,0xbf
+# GFX12: s_cmp_lg_f16 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x55,0xbf]
 
-# GFX12: s_cmp_lg_f16 m0, s2                     ; encoding: [0x7d,0x02,0x55,0xbf]
 0x7d,0x02,0x55,0xbf
+# GFX12: s_cmp_lg_f16 m0, s2                     ; encoding: [0x7d,0x02,0x55,0xbf]
 
-# GFX12: s_cmp_lg_f16 exec_lo, s2                ; encoding: [0x7e,0x02,0x55,0xbf]
 0x7e,0x02,0x55,0xbf
+# GFX12: s_cmp_lg_f16 exec_lo, s2                ; encoding: [0x7e,0x02,0x55,0xbf]
 
-# GFX12: s_cmp_lg_f16 exec_hi, s2                ; encoding: [0x7f,0x02,0x55,0xbf]
 0x7f,0x02,0x55,0xbf
+# GFX12: s_cmp_lg_f16 exec_hi, s2                ; encoding: [0x7f,0x02,0x55,0xbf]
 
-# GFX12: s_cmp_lg_f16 0, s2                      ; encoding: [0x80,0x02,0x55,0xbf]
 0x80,0x02,0x55,0xbf
+# GFX12: s_cmp_lg_f16 0, s2                      ; encoding: [0x80,0x02,0x55,0xbf]
 
-# GFX12: s_cmp_lg_f16 -1, s2                     ; encoding: [0xc1,0x02,0x55,0xbf]
 0xc1,0x02,0x55,0xbf
+# GFX12: s_cmp_lg_f16 -1, s2                     ; encoding: [0xc1,0x02,0x55,0xbf]
 
-# GFX12: s_cmp_lg_f16 0x3800, s2                 ; encoding: [0xff,0x02,0x55,0xbf,0x00,0x38,0x00,0x00]
 0xff,0x02,0x55,0xbf,0x00,0x38,0x00,0x00
+# GFX12: s_cmp_lg_f16 0x3800, s2                 ; encoding: [0xff,0x02,0x55,0xbf,0x00,0x38,0x00,0x00]
 
-# GFX12: s_cmp_lg_f16 0xfe0b, s2                 ; encoding: [0xff,0x02,0x55,0xbf,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x55,0xbf,0x0b,0xfe,0x00,0x00
+# GFX12: s_cmp_lg_f16 0xfe0b, s2                 ; encoding: [0xff,0x02,0x55,0xbf,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_cmp_lg_f16 0x3456, s2                 ; encoding: [0xff,0x02,0x55,0xbf,0x56,0x34,0x00,0x00]
 0xff,0x02,0x55,0xbf,0x56,0x34,0x00,0x00
+# GFX12: s_cmp_lg_f16 0x3456, s2                 ; encoding: [0xff,0x02,0x55,0xbf,0x56,0x34,0x00,0x00]
 
-# GFX12: s_cmp_ge_f16 s1, s2                     ; encoding: [0x01,0x02,0x56,0xbf]
 0x01,0x02,0x56,0xbf
+# GFX12: s_cmp_ge_f16 s1, s2                     ; encoding: [0x01,0x02,0x56,0xbf]
 
-# GFX12: s_cmp_ge_f16 s101, s2                   ; encoding: [0x65,0x02,0x56,0xbf]
 0x65,0x02,0x56,0xbf
+# GFX12: s_cmp_ge_f16 s101, s2                   ; encoding: [0x65,0x02,0x56,0xbf]
 
-# GFX12: s_cmp_ge_f16 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x56,0xbf]
 0x6a,0x02,0x56,0xbf
+# GFX12: s_cmp_ge_f16 vcc_lo, s2                 ; encoding: [0x6a,0x02,0x56,0xbf]
 
-# GFX12: s_cmp_ge_f16 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x56,0xbf]
 0x6b,0x02,0x56,0xbf
+# GFX12: s_cmp_ge_f16 vcc_hi, s2                 ; encoding: [0x6b,0x02,0x56,0xbf]
 
-# GFX12: s_cmp_ge_f16 m0, s2                     ; encoding: [0x7d,0x02,0x56,0xbf]
 0x7d,0x02,0x56,0xbf
+# GFX12: s_cmp_ge_f16 m0, s2                     ; encoding: [0x7d,0x02,0x56,0xbf]
 
-# GFX12: s_cmp_ge_f16 exec_lo, s2                ; encoding: [0x7e,0x02,0x56,0xbf]
 0x7e,0x02,0x56,0xbf
+# GFX12: s_cmp_ge_f16 exec_lo, s2                ; encoding: [0x7e,0x02,0x56,0xbf]
 
-# GFX12: s_cmp_ge_f16 exec_hi, s2                ; encoding: [0x7f,0x02,0x56,0xbf]
 0x7f,0x02,0x56,0xbf
+# GFX12: s_cmp_ge_f16 exec_hi, s2                ; encoding: [0x7f,0x02,0x56,0xbf]
 
-# GFX12: s_cmp_ge_f16 0, s2                      ; encoding: [0x80,0x02,0x56,0xbf]
 0x80,0x02,0x56,0xbf
+# GFX12: s_cmp_ge_f16 0, s2                      ; encoding: [0x80,0x02,0x56,0xbf]
 
-# GFX12: s_cmp_ge_f16 -1, s2                     ; encoding: [0xc1,0x02,0x56,0xbf]
 0xc1,0x02,0x56,0xbf
+# GFX12: s_cmp_ge_f16 -1, s2                     ; encoding: [0xc1,0x02,0x56,0xbf]
 
-# GFX12: s_cmp_ge_f16 0x3800, s2                 ; encoding: [0xff,0x02,0x56,0xbf,0x00,0x38,0x00,0x00]
 0xff,0x02,0x56,0xbf,0x00,0x38,0x00,0x00
+# GFX12: s_cmp_ge_f16 0x3800, s2                 ; encoding: [0xff,0x02,0x56,0xbf,0x00,0x38,0x00,0x00]
 
-# GFX12: s_cmp_ge_f16 0xfe0b, s2                 ; encoding: [0xff,0x02,0x56,0xbf,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x56,0xbf,0x0b,0xfe,0x00,0x00
+# GFX12: s_cmp_ge_f16 0xfe0b, s2                 ; encoding: [0xff,0x02,0x56,0xbf,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_cmp_ge_f16 0x3456, s2                 ; encoding: [0xff,0x02,0x56,0xbf,0x56,0x34,0x00,0x00]
 0xff,0x02,0x56,0xbf,0x56,0x34,0x00,0x00
+# GFX12: s_cmp_ge_f16 0x3456, s2                 ; encoding: [0xff,0x02,0x56,0xbf,0x56,0x34,0x00,0x00]
 
-# GFX12: s_cmp_o_f16 s1, s2                      ; encoding: [0x01,0x02,0x57,0xbf]
 0x01,0x02,0x57,0xbf
+# GFX12: s_cmp_o_f16 s1, s2                      ; encoding: [0x01,0x02,0x57,0xbf]
 
-# GFX12: s_cmp_o_f16 s101, s2                    ; encoding: [0x65,0x02,0x57,0xbf]
 0x65,0x02,0x57,0xbf
+# GFX12: s_cmp_o_f16 s101, s2                    ; encoding: [0x65,0x02,0x57,0xbf]
 
-# GFX12: s_cmp_o_f16 vcc_lo, s2                  ; encoding: [0x6a,0x02,0x57,0xbf]
 0x6a,0x02,0x57,0xbf
+# GFX12: s_cmp_o_f16 vcc_lo, s2                  ; encoding: [0x6a,0x02,0x57,0xbf]
 
-# GFX12: s_cmp_o_f16 vcc_hi, s2                  ; encoding: [0x6b,0x02,0x57,0xbf]
 0x6b,0x02,0x57,0xbf
+# GFX12: s_cmp_o_f16 vcc_hi, s2                  ; encoding: [0x6b,0x02,0x57,0xbf]
 
-# GFX12: s_cmp_o_f16 m0, s2                      ; encoding: [0x7d,0x02,0x57,0xbf]
 0x7d,0x02,0x57,0xbf
+# GFX12: s_cmp_o_f16 m0, s2                      ; encoding: [0x7d,0x02,0x57,0xbf]
 
-# GFX12: s_cmp_o_f16 exec_lo, s2                 ; encoding: [0x7e,0x02,0x57,0xbf]
 0x7e,0x02,0x57,0xbf
+# GFX12: s_cmp_o_f16 exec_lo, s2                 ; encoding: [0x7e,0x02,0x57,0xbf]
 
-# GFX12: s_cmp_o_f16 exec_hi, s2                 ; encoding: [0x7f,0x02,0x57,0xbf]
 0x7f,0x02,0x57,0xbf
+# GFX12: s_cmp_o_f16 exec_hi, s2                 ; encoding: [0x7f,0x02,0x57,0xbf]
 
-# GFX12: s_cmp_o_f16 0, s2                       ; encoding: [0x80,0x02,0x57,0xbf]
 0x80,0x02,0x57,0xbf
+# GFX12: s_cmp_o_f16 0, s2                       ; encoding: [0x80,0x02,0x57,0xbf]
 
-# GFX12: s_cmp_o_f16 -1, s2                      ; encoding: [0xc1,0x02,0x57,0xbf]
 0xc1,0x02,0x57,0xbf
+# GFX12: s_cmp_o_f16 -1, s2                      ; encoding: [0xc1,0x02,0x57,0xbf]
 
-# GFX12: s_cmp_o_f16 0x3800, s2                  ; encoding: [0xff,0x02,0x57,0xbf,0x00,0x38,0x00,0x00]
 0xff,0x02,0x57,0xbf,0x00,0x38,0x00,0x00
+# GFX12: s_cmp_o_f16 0x3800, s2                  ; encoding: [0xff,0x02,0x57,0xbf,0x00,0x38,0x00,0x00]
 
-# GFX12: s_cmp_o_f16 0xfe0b, s2                  ; encoding: [0xff,0x02,0x57,0xbf,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x57,0xbf,0x0b,0xfe,0x00,0x00
+# GFX12: s_cmp_o_f16 0xfe0b, s2                  ; encoding: [0xff,0x02,0x57,0xbf,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_cmp_o_f16 0x3456, s2                  ; encoding: [0xff,0x02,0x57,0xbf,0x56,0x34,0x00,0x00]
 0xff,0x02,0x57,0xbf,0x56,0x34,0x00,0x00
+# GFX12: s_cmp_o_f16 0x3456, s2                  ; encoding: [0xff,0x02,0x57,0xbf,0x56,0x34,0x00,0x00]
 
-# GFX12: s_cmp_u_f16 s1, s2                      ; encoding: [0x01,0x02,0x58,0xbf]
 0x01,0x02,0x58,0xbf
+# GFX12: s_cmp_u_f16 s1, s2                      ; encoding: [0x01,0x02,0x58,0xbf]
 
-# GFX12: s_cmp_u_f16 s101, s2                    ; encoding: [0x65,0x02,0x58,0xbf]
 0x65,0x02,0x58,0xbf
+# GFX12: s_cmp_u_f16 s101, s2                    ; encoding: [0x65,0x02,0x58,0xbf]
 
-# GFX12: s_cmp_u_f16 vcc_lo, s2                  ; encoding: [0x6a,0x02,0x58,0xbf]
 0x6a,0x02,0x58,0xbf
+# GFX12: s_cmp_u_f16 vcc_lo, s2                  ; encoding: [0x6a,0x02,0x58,0xbf]
 
-# GFX12: s_cmp_u_f16 vcc_hi, s2                  ; encoding: [0x6b,0x02,0x58,0xbf]
 0x6b,0x02,0x58,0xbf
+# GFX12: s_cmp_u_f16 vcc_hi, s2                  ; encoding: [0x6b,0x02,0x58,0xbf]
 
-# GFX12: s_cmp_u_f16 m0, s2                      ; encoding: [0x7d,0x02,0x58,0xbf]
 0x7d,0x02,0x58,0xbf
+# GFX12: s_cmp_u_f16 m0, s2                      ; encoding: [0x7d,0x02,0x58,0xbf]
 
-# GFX12: s_cmp_u_f16 exec_lo, s2                 ; encoding: [0x7e,0x02,0x58,0xbf]
 0x7e,0x02,0x58,0xbf
+# GFX12: s_cmp_u_f16 exec_lo, s2                 ; encoding: [0x7e,0x02,0x58,0xbf]
 
-# GFX12: s_cmp_u_f16 exec_hi, s2                 ; encoding: [0x7f,0x02,0x58,0xbf]
 0x7f,0x02,0x58,0xbf
+# GFX12: s_cmp_u_f16 exec_hi, s2                 ; encoding: [0x7f,0x02,0x58,0xbf]
 
-# GFX12: s_cmp_u_f16 0, s2                       ; encoding: [0x80,0x02,0x58,0xbf]
 0x80,0x02,0x58,0xbf
+# GFX12: s_cmp_u_f16 0, s2                       ; encoding: [0x80,0x02,0x58,0xbf]
 
-# GFX12: s_cmp_u_f16 -1, s2                      ; encoding: [0xc1,0x02,0x58,0xbf]
 0xc1,0x02,0x58,0xbf
+# GFX12: s_cmp_u_f16 -1, s2                      ; encoding: [0xc1,0x02,0x58,0xbf]
 
-# GFX12: s_cmp_u_f16 0x3800, s2                  ; encoding: [0xff,0x02,0x58,0xbf,0x00,0x38,0x00,0x00]
 0xff,0x02,0x58,0xbf,0x00,0x38,0x00,0x00
+# GFX12: s_cmp_u_f16 0x3800, s2                  ; encoding: [0xff,0x02,0x58,0xbf,0x00,0x38,0x00,0x00]
 
-# GFX12: s_cmp_u_f16 0xfe0b, s2                  ; encoding: [0xff,0x02,0x58,0xbf,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x58,0xbf,0x0b,0xfe,0x00,0x00
+# GFX12: s_cmp_u_f16 0xfe0b, s2                  ; encoding: [0xff,0x02,0x58,0xbf,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_cmp_u_f16 0x3456, s2                  ; encoding: [0xff,0x02,0x58,0xbf,0x56,0x34,0x00,0x00]
 0xff,0x02,0x58,0xbf,0x56,0x34,0x00,0x00
+# GFX12: s_cmp_u_f16 0x3456, s2                  ; encoding: [0xff,0x02,0x58,0xbf,0x56,0x34,0x00,0x00]
 
-# GFX12: s_cmp_nge_f16 s1, s2                    ; encoding: [0x01,0x02,0x59,0xbf]
 0x01,0x02,0x59,0xbf
+# GFX12: s_cmp_nge_f16 s1, s2                    ; encoding: [0x01,0x02,0x59,0xbf]
 
-# GFX12: s_cmp_nge_f16 s101, s2                  ; encoding: [0x65,0x02,0x59,0xbf]
 0x65,0x02,0x59,0xbf
+# GFX12: s_cmp_nge_f16 s101, s2                  ; encoding: [0x65,0x02,0x59,0xbf]
 
-# GFX12: s_cmp_nge_f16 vcc_lo, s2                ; encoding: [0x6a,0x02,0x59,0xbf]
 0x6a,0x02,0x59,0xbf
+# GFX12: s_cmp_nge_f16 vcc_lo, s2                ; encoding: [0x6a,0x02,0x59,0xbf]
 
-# GFX12: s_cmp_nge_f16 vcc_hi, s2                ; encoding: [0x6b,0x02,0x59,0xbf]
 0x6b,0x02,0x59,0xbf
+# GFX12: s_cmp_nge_f16 vcc_hi, s2                ; encoding: [0x6b,0x02,0x59,0xbf]
 
-# GFX12: s_cmp_nge_f16 m0, s2                    ; encoding: [0x7d,0x02,0x59,0xbf]
 0x7d,0x02,0x59,0xbf
+# GFX12: s_cmp_nge_f16 m0, s2                    ; encoding: [0x7d,0x02,0x59,0xbf]
 
-# GFX12: s_cmp_nge_f16 exec_lo, s2               ; encoding: [0x7e,0x02,0x59,0xbf]
 0x7e,0x02,0x59,0xbf
+# GFX12: s_cmp_nge_f16 exec_lo, s2               ; encoding: [0x7e,0x02,0x59,0xbf]
 
-# GFX12: s_cmp_nge_f16 exec_hi, s2               ; encoding: [0x7f,0x02,0x59,0xbf]
 0x7f,0x02,0x59,0xbf
+# GFX12: s_cmp_nge_f16 exec_hi, s2               ; encoding: [0x7f,0x02,0x59,0xbf]
 
-# GFX12: s_cmp_nge_f16 0, s2                     ; encoding: [0x80,0x02,0x59,0xbf]
 0x80,0x02,0x59,0xbf
+# GFX12: s_cmp_nge_f16 0, s2                     ; encoding: [0x80,0x02,0x59,0xbf]
 
-# GFX12: s_cmp_nge_f16 -1, s2                    ; encoding: [0xc1,0x02,0x59,0xbf]
 0xc1,0x02,0x59,0xbf
+# GFX12: s_cmp_nge_f16 -1, s2                    ; encoding: [0xc1,0x02,0x59,0xbf]
 
-# GFX12: s_cmp_nge_f16 0x3800, s2                ; encoding: [0xff,0x02,0x59,0xbf,0x00,0x38,0x00,0x00]
 0xff,0x02,0x59,0xbf,0x00,0x38,0x00,0x00
+# GFX12: s_cmp_nge_f16 0x3800, s2                ; encoding: [0xff,0x02,0x59,0xbf,0x00,0x38,0x00,0x00]
 
-# GFX12: s_cmp_nge_f16 0xfe0b, s2                ; encoding: [0xff,0x02,0x59,0xbf,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x59,0xbf,0x0b,0xfe,0x00,0x00
+# GFX12: s_cmp_nge_f16 0xfe0b, s2                ; encoding: [0xff,0x02,0x59,0xbf,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_cmp_nge_f16 0x3456, s2                ; encoding: [0xff,0x02,0x59,0xbf,0x56,0x34,0x00,0x00]
 0xff,0x02,0x59,0xbf,0x56,0x34,0x00,0x00
+# GFX12: s_cmp_nge_f16 0x3456, s2                ; encoding: [0xff,0x02,0x59,0xbf,0x56,0x34,0x00,0x00]
 
-# GFX12: s_cmp_nlg_f16 s1, s2                    ; encoding: [0x01,0x02,0x5a,0xbf]
 0x01,0x02,0x5a,0xbf
+# GFX12: s_cmp_nlg_f16 s1, s2                    ; encoding: [0x01,0x02,0x5a,0xbf]
 
-# GFX12: s_cmp_nlg_f16 s101, s2                  ; encoding: [0x65,0x02,0x5a,0xbf]
 0x65,0x02,0x5a,0xbf
+# GFX12: s_cmp_nlg_f16 s101, s2                  ; encoding: [0x65,0x02,0x5a,0xbf]
 
-# GFX12: s_cmp_nlg_f16 vcc_lo, s2                ; encoding: [0x6a,0x02,0x5a,0xbf]
 0x6a,0x02,0x5a,0xbf
+# GFX12: s_cmp_nlg_f16 vcc_lo, s2                ; encoding: [0x6a,0x02,0x5a,0xbf]
 
-# GFX12: s_cmp_nlg_f16 vcc_hi, s2                ; encoding: [0x6b,0x02,0x5a,0xbf]
 0x6b,0x02,0x5a,0xbf
+# GFX12: s_cmp_nlg_f16 vcc_hi, s2                ; encoding: [0x6b,0x02,0x5a,0xbf]
 
-# GFX12: s_cmp_nlg_f16 m0, s2                    ; encoding: [0x7d,0x02,0x5a,0xbf]
 0x7d,0x02,0x5a,0xbf
+# GFX12: s_cmp_nlg_f16 m0, s2                    ; encoding: [0x7d,0x02,0x5a,0xbf]
 
-# GFX12: s_cmp_nlg_f16 exec_lo, s2               ; encoding: [0x7e,0x02,0x5a,0xbf]
 0x7e,0x02,0x5a,0xbf
+# GFX12: s_cmp_nlg_f16 exec_lo, s2               ; encoding: [0x7e,0x02,0x5a,0xbf]
 
-# GFX12: s_cmp_nlg_f16 exec_hi, s2               ; encoding: [0x7f,0x02,0x5a,0xbf]
 0x7f,0x02,0x5a,0xbf
+# GFX12: s_cmp_nlg_f16 exec_hi, s2               ; encoding: [0x7f,0x02,0x5a,0xbf]
 
-# GFX12: s_cmp_nlg_f16 0, s2                     ; encoding: [0x80,0x02,0x5a,0xbf]
 0x80,0x02,0x5a,0xbf
+# GFX12: s_cmp_nlg_f16 0, s2                     ; encoding: [0x80,0x02,0x5a,0xbf]
 
-# GFX12: s_cmp_nlg_f16 -1, s2                    ; encoding: [0xc1,0x02,0x5a,0xbf]
 0xc1,0x02,0x5a,0xbf
+# GFX12: s_cmp_nlg_f16 -1, s2                    ; encoding: [0xc1,0x02,0x5a,0xbf]
 
-# GFX12: s_cmp_nlg_f16 0x3800, s2                ; encoding: [0xff,0x02,0x5a,0xbf,0x00,0x38,0x00,0x00]
 0xff,0x02,0x5a,0xbf,0x00,0x38,0x00,0x00
+# GFX12: s_cmp_nlg_f16 0x3800, s2                ; encoding: [0xff,0x02,0x5a,0xbf,0x00,0x38,0x00,0x00]
 
-# GFX12: s_cmp_nlg_f16 0xfe0b, s2                ; encoding: [0xff,0x02,0x5a,0xbf,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x5a,0xbf,0x0b,0xfe,0x00,0x00
+# GFX12: s_cmp_nlg_f16 0xfe0b, s2                ; encoding: [0xff,0x02,0x5a,0xbf,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_cmp_nlg_f16 0x3456, s2                ; encoding: [0xff,0x02,0x5a,0xbf,0x56,0x34,0x00,0x00]
 0xff,0x02,0x5a,0xbf,0x56,0x34,0x00,0x00
+# GFX12: s_cmp_nlg_f16 0x3456, s2                ; encoding: [0xff,0x02,0x5a,0xbf,0x56,0x34,0x00,0x00]
 
-# GFX12: s_cmp_ngt_f16 s1, s2                    ; encoding: [0x01,0x02,0x5b,0xbf]
 0x01,0x02,0x5b,0xbf
+# GFX12: s_cmp_ngt_f16 s1, s2                    ; encoding: [0x01,0x02,0x5b,0xbf]
 
-# GFX12: s_cmp_ngt_f16 s101, s2                  ; encoding: [0x65,0x02,0x5b,0xbf]
 0x65,0x02,0x5b,0xbf
+# GFX12: s_cmp_ngt_f16 s101, s2                  ; encoding: [0x65,0x02,0x5b,0xbf]
 
-# GFX12: s_cmp_ngt_f16 vcc_lo, s2                ; encoding: [0x6a,0x02,0x5b,0xbf]
 0x6a,0x02,0x5b,0xbf
+# GFX12: s_cmp_ngt_f16 vcc_lo, s2                ; encoding: [0x6a,0x02,0x5b,0xbf]
 
-# GFX12: s_cmp_ngt_f16 vcc_hi, s2                ; encoding: [0x6b,0x02,0x5b,0xbf]
 0x6b,0x02,0x5b,0xbf
+# GFX12: s_cmp_ngt_f16 vcc_hi, s2                ; encoding: [0x6b,0x02,0x5b,0xbf]
 
-# GFX12: s_cmp_ngt_f16 m0, s2                    ; encoding: [0x7d,0x02,0x5b,0xbf]
 0x7d,0x02,0x5b,0xbf
+# GFX12: s_cmp_ngt_f16 m0, s2                    ; encoding: [0x7d,0x02,0x5b,0xbf]
 
-# GFX12: s_cmp_ngt_f16 exec_lo, s2               ; encoding: [0x7e,0x02,0x5b,0xbf]
 0x7e,0x02,0x5b,0xbf
+# GFX12: s_cmp_ngt_f16 exec_lo, s2               ; encoding: [0x7e,0x02,0x5b,0xbf]
 
-# GFX12: s_cmp_ngt_f16 exec_hi, s2               ; encoding: [0x7f,0x02,0x5b,0xbf]
 0x7f,0x02,0x5b,0xbf
+# GFX12: s_cmp_ngt_f16 exec_hi, s2               ; encoding: [0x7f,0x02,0x5b,0xbf]
 
-# GFX12: s_cmp_ngt_f16 0, s2                     ; encoding: [0x80,0x02,0x5b,0xbf]
 0x80,0x02,0x5b,0xbf
+# GFX12: s_cmp_ngt_f16 0, s2                     ; encoding: [0x80,0x02,0x5b,0xbf]
 
-# GFX12: s_cmp_ngt_f16 -1, s2                    ; encoding: [0xc1,0x02,0x5b,0xbf]
 0xc1,0x02,0x5b,0xbf
+# GFX12: s_cmp_ngt_f16 -1, s2                    ; encoding: [0xc1,0x02,0x5b,0xbf]
 
-# GFX12: s_cmp_ngt_f16 0x3800, s2                ; encoding: [0xff,0x02,0x5b,0xbf,0x00,0x38,0x00,0x00]
 0xff,0x02,0x5b,0xbf,0x00,0x38,0x00,0x00
+# GFX12: s_cmp_ngt_f16 0x3800, s2                ; encoding: [0xff,0x02,0x5b,0xbf,0x00,0x38,0x00,0x00]
 
-# GFX12: s_cmp_ngt_f16 0xfe0b, s2                ; encoding: [0xff,0x02,0x5b,0xbf,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x5b,0xbf,0x0b,0xfe,0x00,0x00
+# GFX12: s_cmp_ngt_f16 0xfe0b, s2                ; encoding: [0xff,0x02,0x5b,0xbf,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_cmp_ngt_f16 0x3456, s2                ; encoding: [0xff,0x02,0x5b,0xbf,0x56,0x34,0x00,0x00]
 0xff,0x02,0x5b,0xbf,0x56,0x34,0x00,0x00
+# GFX12: s_cmp_ngt_f16 0x3456, s2                ; encoding: [0xff,0x02,0x5b,0xbf,0x56,0x34,0x00,0x00]
 
-# GFX12: s_cmp_nle_f16 s1, s2                    ; encoding: [0x01,0x02,0x5c,0xbf]
 0x01,0x02,0x5c,0xbf
+# GFX12: s_cmp_nle_f16 s1, s2                    ; encoding: [0x01,0x02,0x5c,0xbf]
 
-# GFX12: s_cmp_nle_f16 s101, s2                  ; encoding: [0x65,0x02,0x5c,0xbf]
 0x65,0x02,0x5c,0xbf
+# GFX12: s_cmp_nle_f16 s101, s2                  ; encoding: [0x65,0x02,0x5c,0xbf]
 
-# GFX12: s_cmp_nle_f16 vcc_lo, s2                ; encoding: [0x6a,0x02,0x5c,0xbf]
 0x6a,0x02,0x5c,0xbf
+# GFX12: s_cmp_nle_f16 vcc_lo, s2                ; encoding: [0x6a,0x02,0x5c,0xbf]
 
-# GFX12: s_cmp_nle_f16 vcc_hi, s2                ; encoding: [0x6b,0x02,0x5c,0xbf]
 0x6b,0x02,0x5c,0xbf
+# GFX12: s_cmp_nle_f16 vcc_hi, s2                ; encoding: [0x6b,0x02,0x5c,0xbf]
 
-# GFX12: s_cmp_nle_f16 m0, s2                    ; encoding: [0x7d,0x02,0x5c,0xbf]
 0x7d,0x02,0x5c,0xbf
+# GFX12: s_cmp_nle_f16 m0, s2                    ; encoding: [0x7d,0x02,0x5c,0xbf]
 
-# GFX12: s_cmp_nle_f16 exec_lo, s2               ; encoding: [0x7e,0x02,0x5c,0xbf]
 0x7e,0x02,0x5c,0xbf
+# GFX12: s_cmp_nle_f16 exec_lo, s2               ; encoding: [0x7e,0x02,0x5c,0xbf]
 
-# GFX12: s_cmp_nle_f16 exec_hi, s2               ; encoding: [0x7f,0x02,0x5c,0xbf]
 0x7f,0x02,0x5c,0xbf
+# GFX12: s_cmp_nle_f16 exec_hi, s2               ; encoding: [0x7f,0x02,0x5c,0xbf]
 
-# GFX12: s_cmp_nle_f16 0, s2                     ; encoding: [0x80,0x02,0x5c,0xbf]
 0x80,0x02,0x5c,0xbf
+# GFX12: s_cmp_nle_f16 0, s2                     ; encoding: [0x80,0x02,0x5c,0xbf]
 
-# GFX12: s_cmp_nle_f16 -1, s2                    ; encoding: [0xc1,0x02,0x5c,0xbf]
 0xc1,0x02,0x5c,0xbf
+# GFX12: s_cmp_nle_f16 -1, s2                    ; encoding: [0xc1,0x02,0x5c,0xbf]
 
-# GFX12: s_cmp_nle_f16 0x3800, s2                ; encoding: [0xff,0x02,0x5c,0xbf,0x00,0x38,0x00,0x00]
 0xff,0x02,0x5c,0xbf,0x00,0x38,0x00,0x00
+# GFX12: s_cmp_nle_f16 0x3800, s2                ; encoding: [0xff,0x02,0x5c,0xbf,0x00,0x38,0x00,0x00]
 
-# GFX12: s_cmp_nle_f16 0xfe0b, s2                ; encoding: [0xff,0x02,0x5c,0xbf,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x5c,0xbf,0x0b,0xfe,0x00,0x00
+# GFX12: s_cmp_nle_f16 0xfe0b, s2                ; encoding: [0xff,0x02,0x5c,0xbf,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_cmp_nle_f16 0x3456, s2                ; encoding: [0xff,0x02,0x5c,0xbf,0x56,0x34,0x00,0x00]
 0xff,0x02,0x5c,0xbf,0x56,0x34,0x00,0x00
+# GFX12: s_cmp_nle_f16 0x3456, s2                ; encoding: [0xff,0x02,0x5c,0xbf,0x56,0x34,0x00,0x00]
 
-# GFX12: s_cmp_neq_f16 s1, s2                    ; encoding: [0x01,0x02,0x5d,0xbf]
 0x01,0x02,0x5d,0xbf
+# GFX12: s_cmp_neq_f16 s1, s2                    ; encoding: [0x01,0x02,0x5d,0xbf]
 
-# GFX12: s_cmp_neq_f16 s101, s2                  ; encoding: [0x65,0x02,0x5d,0xbf]
 0x65,0x02,0x5d,0xbf
+# GFX12: s_cmp_neq_f16 s101, s2                  ; encoding: [0x65,0x02,0x5d,0xbf]
 
-# GFX12: s_cmp_neq_f16 vcc_lo, s2                ; encoding: [0x6a,0x02,0x5d,0xbf]
 0x6a,0x02,0x5d,0xbf
+# GFX12: s_cmp_neq_f16 vcc_lo, s2                ; encoding: [0x6a,0x02,0x5d,0xbf]
 
-# GFX12: s_cmp_neq_f16 vcc_hi, s2                ; encoding: [0x6b,0x02,0x5d,0xbf]
 0x6b,0x02,0x5d,0xbf
+# GFX12: s_cmp_neq_f16 vcc_hi, s2                ; encoding: [0x6b,0x02,0x5d,0xbf]
 
-# GFX12: s_cmp_neq_f16 m0, s2                    ; encoding: [0x7d,0x02,0x5d,0xbf]
 0x7d,0x02,0x5d,0xbf
+# GFX12: s_cmp_neq_f16 m0, s2                    ; encoding: [0x7d,0x02,0x5d,0xbf]
 
-# GFX12: s_cmp_neq_f16 exec_lo, s2               ; encoding: [0x7e,0x02,0x5d,0xbf]
 0x7e,0x02,0x5d,0xbf
+# GFX12: s_cmp_neq_f16 exec_lo, s2               ; encoding: [0x7e,0x02,0x5d,0xbf]
 
-# GFX12: s_cmp_neq_f16 exec_hi, s2               ; encoding: [0x7f,0x02,0x5d,0xbf]
 0x7f,0x02,0x5d,0xbf
+# GFX12: s_cmp_neq_f16 exec_hi, s2               ; encoding: [0x7f,0x02,0x5d,0xbf]
 
-# GFX12: s_cmp_neq_f16 0, s2                     ; encoding: [0x80,0x02,0x5d,0xbf]
 0x80,0x02,0x5d,0xbf
+# GFX12: s_cmp_neq_f16 0, s2                     ; encoding: [0x80,0x02,0x5d,0xbf]
 
-# GFX12: s_cmp_neq_f16 -1, s2                    ; encoding: [0xc1,0x02,0x5d,0xbf]
 0xc1,0x02,0x5d,0xbf
+# GFX12: s_cmp_neq_f16 -1, s2                    ; encoding: [0xc1,0x02,0x5d,0xbf]
 
-# GFX12: s_cmp_neq_f16 0x3800, s2                ; encoding: [0xff,0x02,0x5d,0xbf,0x00,0x38,0x00,0x00]
 0xff,0x02,0x5d,0xbf,0x00,0x38,0x00,0x00
+# GFX12: s_cmp_neq_f16 0x3800, s2                ; encoding: [0xff,0x02,0x5d,0xbf,0x00,0x38,0x00,0x00]
 
-# GFX12: s_cmp_neq_f16 0xfe0b, s2                ; encoding: [0xff,0x02,0x5d,0xbf,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x5d,0xbf,0x0b,0xfe,0x00,0x00
+# GFX12: s_cmp_neq_f16 0xfe0b, s2                ; encoding: [0xff,0x02,0x5d,0xbf,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_cmp_neq_f16 0x3456, s2                ; encoding: [0xff,0x02,0x5d,0xbf,0x56,0x34,0x00,0x00]
 0xff,0x02,0x5d,0xbf,0x56,0x34,0x00,0x00
+# GFX12: s_cmp_neq_f16 0x3456, s2                ; encoding: [0xff,0x02,0x5d,0xbf,0x56,0x34,0x00,0x00]
 
-# GFX12: s_cmp_nlt_f16 s1, s2                    ; encoding: [0x01,0x02,0x5e,0xbf]
 0x01,0x02,0x5e,0xbf
+# GFX12: s_cmp_nlt_f16 s1, s2                    ; encoding: [0x01,0x02,0x5e,0xbf]
 
-# GFX12: s_cmp_nlt_f16 s101, s2                  ; encoding: [0x65,0x02,0x5e,0xbf]
 0x65,0x02,0x5e,0xbf
+# GFX12: s_cmp_nlt_f16 s101, s2                  ; encoding: [0x65,0x02,0x5e,0xbf]
 
-# GFX12: s_cmp_nlt_f16 vcc_lo, s2                ; encoding: [0x6a,0x02,0x5e,0xbf]
 0x6a,0x02,0x5e,0xbf
+# GFX12: s_cmp_nlt_f16 vcc_lo, s2                ; encoding: [0x6a,0x02,0x5e,0xbf]
 
-# GFX12: s_cmp_nlt_f16 vcc_hi, s2                ; encoding: [0x6b,0x02,0x5e,0xbf]
 0x6b,0x02,0x5e,0xbf
+# GFX12: s_cmp_nlt_f16 vcc_hi, s2                ; encoding: [0x6b,0x02,0x5e,0xbf]
 
-# GFX12: s_cmp_nlt_f16 m0, s2                    ; encoding: [0x7d,0x02,0x5e,0xbf]
 0x7d,0x02,0x5e,0xbf
+# GFX12: s_cmp_nlt_f16 m0, s2                    ; encoding: [0x7d,0x02,0x5e,0xbf]
 
-# GFX12: s_cmp_nlt_f16 exec_lo, s2               ; encoding: [0x7e,0x02,0x5e,0xbf]
 0x7e,0x02,0x5e,0xbf
+# GFX12: s_cmp_nlt_f16 exec_lo, s2               ; encoding: [0x7e,0x02,0x5e,0xbf]
 
-# GFX12: s_cmp_nlt_f16 exec_hi, s2               ; encoding: [0x7f,0x02,0x5e,0xbf]
 0x7f,0x02,0x5e,0xbf
+# GFX12: s_cmp_nlt_f16 exec_hi, s2               ; encoding: [0x7f,0x02,0x5e,0xbf]
 
-# GFX12: s_cmp_nlt_f16 0, s2                     ; encoding: [0x80,0x02,0x5e,0xbf]
 0x80,0x02,0x5e,0xbf
+# GFX12: s_cmp_nlt_f16 0, s2                     ; encoding: [0x80,0x02,0x5e,0xbf]
 
-# GFX12: s_cmp_nlt_f16 -1, s2                    ; encoding: [0xc1,0x02,0x5e,0xbf]
 0xc1,0x02,0x5e,0xbf
+# GFX12: s_cmp_nlt_f16 -1, s2                    ; encoding: [0xc1,0x02,0x5e,0xbf]
 
-# GFX12: s_cmp_nlt_f16 0x3800, s2                ; encoding: [0xff,0x02,0x5e,0xbf,0x00,0x38,0x00,0x00]
 0xff,0x02,0x5e,0xbf,0x00,0x38,0x00,0x00
+# GFX12: s_cmp_nlt_f16 0x3800, s2                ; encoding: [0xff,0x02,0x5e,0xbf,0x00,0x38,0x00,0x00]
 
-# GFX12: s_cmp_nlt_f16 0xfe0b, s2                ; encoding: [0xff,0x02,0x5e,0xbf,0x0b,0xfe,0x00,0x00]
 0xff,0x02,0x5e,0xbf,0x0b,0xfe,0x00,0x00
+# GFX12: s_cmp_nlt_f16 0xfe0b, s2                ; encoding: [0xff,0x02,0x5e,0xbf,0x0b,0xfe,0x00,0x00]
 
-# GFX12: s_cmp_nlt_f16 0x3456, s2                ; encoding: [0xff,0x02,0x5e,0xbf,0x56,0x34,0x00,0x00]
 0xff,0x02,0x5e,0xbf,0x56,0x34,0x00,0x00
+# GFX12: s_cmp_nlt_f16 0x3456, s2                ; encoding: [0xff,0x02,0x5e,0xbf,0x56,0x34,0x00,0x00]
 
-# GFX12: s_bitcmp0_b32 exec_hi, s1               ; encoding: [0x7f,0x01,0x0c,0xbf]
 0x7f,0x01,0x0c,0xbf
+# GFX12: s_bitcmp0_b32 exec_hi, s1               ; encoding: [0x7f,0x01,0x0c,0xbf]
 
-# GFX12: s_bitcmp0_b32 exec_lo, s1               ; encoding: [0x7e,0x01,0x0c,0xbf]
 0x7e,0x01,0x0c,0xbf
+# GFX12: s_bitcmp0_b32 exec_lo, s1               ; encoding: [0x7e,0x01,0x0c,0xbf]
 
-# GFX12: s_bitcmp0_b32 m0, s1                    ; encoding: [0x7d,0x01,0x0c,0xbf]
 0x7d,0x01,0x0c,0xbf
+# GFX12: s_bitcmp0_b32 m0, s1                    ; encoding: [0x7d,0x01,0x0c,0xbf]
 
-# GFX12: s_bitcmp0_b32 s0, 0.5                   ; encoding: [0x00,0xf0,0x0c,0xbf]
 0x00,0xf0,0x0c,0xbf
+# GFX12: s_bitcmp0_b32 s0, 0.5                   ; encoding: [0x00,0xf0,0x0c,0xbf]
 
-# GFX12: s_bitcmp0_b32 s0, 0                     ; encoding: [0x00,0x80,0x0c,0xbf]
 0x00,0x80,0x0c,0xbf
+# GFX12: s_bitcmp0_b32 s0, 0                     ; encoding: [0x00,0x80,0x0c,0xbf]
 
-# GFX12: s_bitcmp0_b32 s0, 0x3f717273            ; encoding: [0x00,0xff,0x0c,0xbf,0x73,0x72,0x71,0x3f]
 0x00,0xff,0x0c,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_bitcmp0_b32 s0, 0x3f717273            ; encoding: [0x00,0xff,0x0c,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bitcmp0_b32 s0, 0xaf123456            ; encoding: [0x00,0xff,0x0c,0xbf,0x56,0x34,0x12,0xaf]
 0x00,0xff,0x0c,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_bitcmp0_b32 s0, 0xaf123456            ; encoding: [0x00,0xff,0x0c,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bitcmp0_b32 s0, -1                    ; encoding: [0x00,0xc1,0x0c,0xbf]
 0x00,0xc1,0x0c,0xbf
+# GFX12: s_bitcmp0_b32 s0, -1                    ; encoding: [0x00,0xc1,0x0c,0xbf]
 
-# GFX12: s_bitcmp0_b32 s0, -4.0                  ; encoding: [0x00,0xf7,0x0c,0xbf]
 0x00,0xf7,0x0c,0xbf
+# GFX12: s_bitcmp0_b32 s0, -4.0                  ; encoding: [0x00,0xf7,0x0c,0xbf]
 
-# GFX12: s_bitcmp0_b32 s0, exec_hi               ; encoding: [0x00,0x7f,0x0c,0xbf]
 0x00,0x7f,0x0c,0xbf
+# GFX12: s_bitcmp0_b32 s0, exec_hi               ; encoding: [0x00,0x7f,0x0c,0xbf]
 
-# GFX12: s_bitcmp0_b32 s0, exec_lo               ; encoding: [0x00,0x7e,0x0c,0xbf]
 0x00,0x7e,0x0c,0xbf
+# GFX12: s_bitcmp0_b32 s0, exec_lo               ; encoding: [0x00,0x7e,0x0c,0xbf]
 
-# GFX12: s_bitcmp0_b32 s0, m0                    ; encoding: [0x00,0x7d,0x0c,0xbf]
 0x00,0x7d,0x0c,0xbf
+# GFX12: s_bitcmp0_b32 s0, m0                    ; encoding: [0x00,0x7d,0x0c,0xbf]
 
-# GFX12: s_bitcmp0_b32 s0, s104                  ; encoding: [0x00,0x68,0x0c,0xbf]
 0x00,0x68,0x0c,0xbf
+# GFX12: s_bitcmp0_b32 s0, s104                  ; encoding: [0x00,0x68,0x0c,0xbf]
 
-# GFX12: s_bitcmp0_b32 s0, s1                    ; encoding: [0x00,0x01,0x0c,0xbf]
 0x00,0x01,0x0c,0xbf
+# GFX12: s_bitcmp0_b32 s0, s1                    ; encoding: [0x00,0x01,0x0c,0xbf]
 
-# GFX12: s_bitcmp0_b32 s0, vcc_hi                ; encoding: [0x00,0x6b,0x0c,0xbf]
 0x00,0x6b,0x0c,0xbf
+# GFX12: s_bitcmp0_b32 s0, vcc_hi                ; encoding: [0x00,0x6b,0x0c,0xbf]
 
-# GFX12: s_bitcmp0_b32 s0, vcc_lo                ; encoding: [0x00,0x6a,0x0c,0xbf]
 0x00,0x6a,0x0c,0xbf
+# GFX12: s_bitcmp0_b32 s0, vcc_lo                ; encoding: [0x00,0x6a,0x0c,0xbf]
 
-# GFX12: s_bitcmp0_b32 s105, s104                ; encoding: [0x69,0x68,0x0c,0xbf]
 0x69,0x68,0x0c,0xbf
+# GFX12: s_bitcmp0_b32 s105, s104                ; encoding: [0x69,0x68,0x0c,0xbf]
 
-# GFX12: s_bitcmp0_b32 s105, s1                  ; encoding: [0x69,0x01,0x0c,0xbf]
 0x69,0x01,0x0c,0xbf
+# GFX12: s_bitcmp0_b32 s105, s1                  ; encoding: [0x69,0x01,0x0c,0xbf]
 
-# GFX12: s_bitcmp0_b32 vcc_hi, s1                ; encoding: [0x6b,0x01,0x0c,0xbf]
 0x6b,0x01,0x0c,0xbf
+# GFX12: s_bitcmp0_b32 vcc_hi, s1                ; encoding: [0x6b,0x01,0x0c,0xbf]
 
-# GFX12: s_bitcmp0_b32 vcc_lo, s1                ; encoding: [0x6a,0x01,0x0c,0xbf]
 0x6a,0x01,0x0c,0xbf
+# GFX12: s_bitcmp0_b32 vcc_lo, s1                ; encoding: [0x6a,0x01,0x0c,0xbf]
 
-# GFX12: s_bitcmp0_b64 exec, s2                  ; encoding: [0x7e,0x02,0x0e,0xbf]
 0x7e,0x02,0x0e,0xbf
+# GFX12: s_bitcmp0_b64 exec, s2                  ; encoding: [0x7e,0x02,0x0e,0xbf]
 
-# GFX12: s_bitcmp0_b64 s[0:1], 0.5               ; encoding: [0x00,0xf0,0x0e,0xbf]
 0x00,0xf0,0x0e,0xbf
+# GFX12: s_bitcmp0_b64 s[0:1], 0.5               ; encoding: [0x00,0xf0,0x0e,0xbf]
 
-# GFX12: s_bitcmp0_b64 s[0:1], 0                 ; encoding: [0x00,0x80,0x0e,0xbf]
 0x00,0x80,0x0e,0xbf
+# GFX12: s_bitcmp0_b64 s[0:1], 0                 ; encoding: [0x00,0x80,0x0e,0xbf]
 
-# GFX12: s_bitcmp0_b64 s[0:1], 0x3f717273        ; encoding: [0x00,0xff,0x0e,0xbf,0x73,0x72,0x71,0x3f]
 0x00,0xff,0x0e,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_bitcmp0_b64 s[0:1], 0x3f717273        ; encoding: [0x00,0xff,0x0e,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bitcmp0_b64 s[0:1], 0xaf123456        ; encoding: [0x00,0xff,0x0e,0xbf,0x56,0x34,0x12,0xaf]
 0x00,0xff,0x0e,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_bitcmp0_b64 s[0:1], 0xaf123456        ; encoding: [0x00,0xff,0x0e,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bitcmp0_b64 s[0:1], -1                ; encoding: [0x00,0xc1,0x0e,0xbf]
 0x00,0xc1,0x0e,0xbf
+# GFX12: s_bitcmp0_b64 s[0:1], -1                ; encoding: [0x00,0xc1,0x0e,0xbf]
 
-# GFX12: s_bitcmp0_b64 s[0:1], -4.0              ; encoding: [0x00,0xf7,0x0e,0xbf]
 0x00,0xf7,0x0e,0xbf
+# GFX12: s_bitcmp0_b64 s[0:1], -4.0              ; encoding: [0x00,0xf7,0x0e,0xbf]
 
-# GFX12: s_bitcmp0_b64 s[0:1], exec_hi           ; encoding: [0x00,0x7f,0x0e,0xbf]
 0x00,0x7f,0x0e,0xbf
+# GFX12: s_bitcmp0_b64 s[0:1], exec_hi           ; encoding: [0x00,0x7f,0x0e,0xbf]
 
-# GFX12: s_bitcmp0_b64 s[0:1], exec_lo           ; encoding: [0x00,0x7e,0x0e,0xbf]
 0x00,0x7e,0x0e,0xbf
+# GFX12: s_bitcmp0_b64 s[0:1], exec_lo           ; encoding: [0x00,0x7e,0x0e,0xbf]
 
-# GFX12: s_bitcmp0_b64 s[0:1], m0                ; encoding: [0x00,0x7d,0x0e,0xbf]
 0x00,0x7d,0x0e,0xbf
+# GFX12: s_bitcmp0_b64 s[0:1], m0                ; encoding: [0x00,0x7d,0x0e,0xbf]
 
-# GFX12: s_bitcmp0_b64 s[0:1], s102              ; encoding: [0x00,0x66,0x0e,0xbf]
 0x00,0x66,0x0e,0xbf
+# GFX12: s_bitcmp0_b64 s[0:1], s102              ; encoding: [0x00,0x66,0x0e,0xbf]
 
-# GFX12: s_bitcmp0_b64 s[0:1], s2                ; encoding: [0x00,0x02,0x0e,0xbf]
 0x00,0x02,0x0e,0xbf
+# GFX12: s_bitcmp0_b64 s[0:1], s2                ; encoding: [0x00,0x02,0x0e,0xbf]
 
-# GFX12: s_bitcmp0_b64 s[0:1], vcc_hi            ; encoding: [0x00,0x6b,0x0e,0xbf]
 0x00,0x6b,0x0e,0xbf
+# GFX12: s_bitcmp0_b64 s[0:1], vcc_hi            ; encoding: [0x00,0x6b,0x0e,0xbf]
 
-# GFX12: s_bitcmp0_b64 s[0:1], vcc_lo            ; encoding: [0x00,0x6a,0x0e,0xbf]
 0x00,0x6a,0x0e,0xbf
+# GFX12: s_bitcmp0_b64 s[0:1], vcc_lo            ; encoding: [0x00,0x6a,0x0e,0xbf]
 
-# GFX12: s_bitcmp0_b64 s[104:105], s102          ; encoding: [0x68,0x66,0x0e,0xbf]
 0x68,0x66,0x0e,0xbf
+# GFX12: s_bitcmp0_b64 s[104:105], s102          ; encoding: [0x68,0x66,0x0e,0xbf]
 
-# GFX12: s_bitcmp0_b64 s[104:105], s2            ; encoding: [0x68,0x02,0x0e,0xbf]
 0x68,0x02,0x0e,0xbf
+# GFX12: s_bitcmp0_b64 s[104:105], s2            ; encoding: [0x68,0x02,0x0e,0xbf]
 
-# GFX12: s_bitcmp0_b64 vcc, s2                   ; encoding: [0x6a,0x02,0x0e,0xbf]
 0x6a,0x02,0x0e,0xbf
+# GFX12: s_bitcmp0_b64 vcc, s2                   ; encoding: [0x6a,0x02,0x0e,0xbf]
 
-# GFX12: s_bitcmp1_b32 exec_hi, s1               ; encoding: [0x7f,0x01,0x0d,0xbf]
 0x7f,0x01,0x0d,0xbf
+# GFX12: s_bitcmp1_b32 exec_hi, s1               ; encoding: [0x7f,0x01,0x0d,0xbf]
 
-# GFX12: s_bitcmp1_b32 exec_lo, s1               ; encoding: [0x7e,0x01,0x0d,0xbf]
 0x7e,0x01,0x0d,0xbf
+# GFX12: s_bitcmp1_b32 exec_lo, s1               ; encoding: [0x7e,0x01,0x0d,0xbf]
 
-# GFX12: s_bitcmp1_b32 m0, s1                    ; encoding: [0x7d,0x01,0x0d,0xbf]
 0x7d,0x01,0x0d,0xbf
+# GFX12: s_bitcmp1_b32 m0, s1                    ; encoding: [0x7d,0x01,0x0d,0xbf]
 
-# GFX12: s_bitcmp1_b32 s0, 0.5                   ; encoding: [0x00,0xf0,0x0d,0xbf]
 0x00,0xf0,0x0d,0xbf
+# GFX12: s_bitcmp1_b32 s0, 0.5                   ; encoding: [0x00,0xf0,0x0d,0xbf]
 
-# GFX12: s_bitcmp1_b32 s0, 0                     ; encoding: [0x00,0x80,0x0d,0xbf]
 0x00,0x80,0x0d,0xbf
+# GFX12: s_bitcmp1_b32 s0, 0                     ; encoding: [0x00,0x80,0x0d,0xbf]
 
-# GFX12: s_bitcmp1_b32 s0, 0x3f717273            ; encoding: [0x00,0xff,0x0d,0xbf,0x73,0x72,0x71,0x3f]
 0x00,0xff,0x0d,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_bitcmp1_b32 s0, 0x3f717273            ; encoding: [0x00,0xff,0x0d,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bitcmp1_b32 s0, 0xaf123456            ; encoding: [0x00,0xff,0x0d,0xbf,0x56,0x34,0x12,0xaf]
 0x00,0xff,0x0d,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_bitcmp1_b32 s0, 0xaf123456            ; encoding: [0x00,0xff,0x0d,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bitcmp1_b32 s0, -1                    ; encoding: [0x00,0xc1,0x0d,0xbf]
 0x00,0xc1,0x0d,0xbf
+# GFX12: s_bitcmp1_b32 s0, -1                    ; encoding: [0x00,0xc1,0x0d,0xbf]
 
-# GFX12: s_bitcmp1_b32 s0, -4.0                  ; encoding: [0x00,0xf7,0x0d,0xbf]
 0x00,0xf7,0x0d,0xbf
+# GFX12: s_bitcmp1_b32 s0, -4.0                  ; encoding: [0x00,0xf7,0x0d,0xbf]
 
-# GFX12: s_bitcmp1_b32 s0, exec_hi               ; encoding: [0x00,0x7f,0x0d,0xbf]
 0x00,0x7f,0x0d,0xbf
+# GFX12: s_bitcmp1_b32 s0, exec_hi               ; encoding: [0x00,0x7f,0x0d,0xbf]
 
-# GFX12: s_bitcmp1_b32 s0, exec_lo               ; encoding: [0x00,0x7e,0x0d,0xbf]
 0x00,0x7e,0x0d,0xbf
+# GFX12: s_bitcmp1_b32 s0, exec_lo               ; encoding: [0x00,0x7e,0x0d,0xbf]
 
-# GFX12: s_bitcmp1_b32 s0, m0                    ; encoding: [0x00,0x7d,0x0d,0xbf]
 0x00,0x7d,0x0d,0xbf
+# GFX12: s_bitcmp1_b32 s0, m0                    ; encoding: [0x00,0x7d,0x0d,0xbf]
 
-# GFX12: s_bitcmp1_b32 s0, s104                  ; encoding: [0x00,0x68,0x0d,0xbf]
 0x00,0x68,0x0d,0xbf
+# GFX12: s_bitcmp1_b32 s0, s104                  ; encoding: [0x00,0x68,0x0d,0xbf]
 
-# GFX12: s_bitcmp1_b32 s0, s1                    ; encoding: [0x00,0x01,0x0d,0xbf]
 0x00,0x01,0x0d,0xbf
+# GFX12: s_bitcmp1_b32 s0, s1                    ; encoding: [0x00,0x01,0x0d,0xbf]
 
-# GFX12: s_bitcmp1_b32 s0, vcc_hi                ; encoding: [0x00,0x6b,0x0d,0xbf]
 0x00,0x6b,0x0d,0xbf
+# GFX12: s_bitcmp1_b32 s0, vcc_hi                ; encoding: [0x00,0x6b,0x0d,0xbf]
 
-# GFX12: s_bitcmp1_b32 s0, vcc_lo                ; encoding: [0x00,0x6a,0x0d,0xbf]
 0x00,0x6a,0x0d,0xbf
+# GFX12: s_bitcmp1_b32 s0, vcc_lo                ; encoding: [0x00,0x6a,0x0d,0xbf]
 
-# GFX12: s_bitcmp1_b32 s105, s104                ; encoding: [0x69,0x68,0x0d,0xbf]
 0x69,0x68,0x0d,0xbf
+# GFX12: s_bitcmp1_b32 s105, s104                ; encoding: [0x69,0x68,0x0d,0xbf]
 
-# GFX12: s_bitcmp1_b32 s105, s1                  ; encoding: [0x69,0x01,0x0d,0xbf]
 0x69,0x01,0x0d,0xbf
+# GFX12: s_bitcmp1_b32 s105, s1                  ; encoding: [0x69,0x01,0x0d,0xbf]
 
-# GFX12: s_bitcmp1_b32 vcc_hi, s1                ; encoding: [0x6b,0x01,0x0d,0xbf]
 0x6b,0x01,0x0d,0xbf
+# GFX12: s_bitcmp1_b32 vcc_hi, s1                ; encoding: [0x6b,0x01,0x0d,0xbf]
 
-# GFX12: s_bitcmp1_b32 vcc_lo, s1                ; encoding: [0x6a,0x01,0x0d,0xbf]
 0x6a,0x01,0x0d,0xbf
+# GFX12: s_bitcmp1_b32 vcc_lo, s1                ; encoding: [0x6a,0x01,0x0d,0xbf]
 
-# GFX12: s_bitcmp1_b64 exec, s2                  ; encoding: [0x7e,0x02,0x0f,0xbf]
 0x7e,0x02,0x0f,0xbf
+# GFX12: s_bitcmp1_b64 exec, s2                  ; encoding: [0x7e,0x02,0x0f,0xbf]
 
-# GFX12: s_bitcmp1_b64 s[0:1], 0.5               ; encoding: [0x00,0xf0,0x0f,0xbf]
 0x00,0xf0,0x0f,0xbf
+# GFX12: s_bitcmp1_b64 s[0:1], 0.5               ; encoding: [0x00,0xf0,0x0f,0xbf]
 
-# GFX12: s_bitcmp1_b64 s[0:1], 0                 ; encoding: [0x00,0x80,0x0f,0xbf]
 0x00,0x80,0x0f,0xbf
+# GFX12: s_bitcmp1_b64 s[0:1], 0                 ; encoding: [0x00,0x80,0x0f,0xbf]
 
-# GFX12: s_bitcmp1_b64 s[0:1], 0x3f717273        ; encoding: [0x00,0xff,0x0f,0xbf,0x73,0x72,0x71,0x3f]
 0x00,0xff,0x0f,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_bitcmp1_b64 s[0:1], 0x3f717273        ; encoding: [0x00,0xff,0x0f,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_bitcmp1_b64 s[0:1], 0xaf123456        ; encoding: [0x00,0xff,0x0f,0xbf,0x56,0x34,0x12,0xaf]
 0x00,0xff,0x0f,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_bitcmp1_b64 s[0:1], 0xaf123456        ; encoding: [0x00,0xff,0x0f,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_bitcmp1_b64 s[0:1], -1                ; encoding: [0x00,0xc1,0x0f,0xbf]
 0x00,0xc1,0x0f,0xbf
+# GFX12: s_bitcmp1_b64 s[0:1], -1                ; encoding: [0x00,0xc1,0x0f,0xbf]
 
-# GFX12: s_bitcmp1_b64 s[0:1], -4.0              ; encoding: [0x00,0xf7,0x0f,0xbf]
 0x00,0xf7,0x0f,0xbf
+# GFX12: s_bitcmp1_b64 s[0:1], -4.0              ; encoding: [0x00,0xf7,0x0f,0xbf]
 
-# GFX12: s_bitcmp1_b64 s[0:1], exec_hi           ; encoding: [0x00,0x7f,0x0f,0xbf]
 0x00,0x7f,0x0f,0xbf
+# GFX12: s_bitcmp1_b64 s[0:1], exec_hi           ; encoding: [0x00,0x7f,0x0f,0xbf]
 
-# GFX12: s_bitcmp1_b64 s[0:1], exec_lo           ; encoding: [0x00,0x7e,0x0f,0xbf]
 0x00,0x7e,0x0f,0xbf
+# GFX12: s_bitcmp1_b64 s[0:1], exec_lo           ; encoding: [0x00,0x7e,0x0f,0xbf]
 
-# GFX12: s_bitcmp1_b64 s[0:1], m0                ; encoding: [0x00,0x7d,0x0f,0xbf]
 0x00,0x7d,0x0f,0xbf
+# GFX12: s_bitcmp1_b64 s[0:1], m0                ; encoding: [0x00,0x7d,0x0f,0xbf]
 
-# GFX12: s_bitcmp1_b64 s[0:1], s102              ; encoding: [0x00,0x66,0x0f,0xbf]
 0x00,0x66,0x0f,0xbf
+# GFX12: s_bitcmp1_b64 s[0:1], s102              ; encoding: [0x00,0x66,0x0f,0xbf]
 
-# GFX12: s_bitcmp1_b64 s[0:1], s2                ; encoding: [0x00,0x02,0x0f,0xbf]
 0x00,0x02,0x0f,0xbf
+# GFX12: s_bitcmp1_b64 s[0:1], s2                ; encoding: [0x00,0x02,0x0f,0xbf]
 
-# GFX12: s_bitcmp1_b64 s[0:1], vcc_hi            ; encoding: [0x00,0x6b,0x0f,0xbf]
 0x00,0x6b,0x0f,0xbf
+# GFX12: s_bitcmp1_b64 s[0:1], vcc_hi            ; encoding: [0x00,0x6b,0x0f,0xbf]
 
-# GFX12: s_bitcmp1_b64 s[0:1], vcc_lo            ; encoding: [0x00,0x6a,0x0f,0xbf]
 0x00,0x6a,0x0f,0xbf
+# GFX12: s_bitcmp1_b64 s[0:1], vcc_lo            ; encoding: [0x00,0x6a,0x0f,0xbf]
 
-# GFX12: s_bitcmp1_b64 s[104:105], s102          ; encoding: [0x68,0x66,0x0f,0xbf]
 0x68,0x66,0x0f,0xbf
+# GFX12: s_bitcmp1_b64 s[104:105], s102          ; encoding: [0x68,0x66,0x0f,0xbf]
 
-# GFX12: s_bitcmp1_b64 s[104:105], s2            ; encoding: [0x68,0x02,0x0f,0xbf]
 0x68,0x02,0x0f,0xbf
+# GFX12: s_bitcmp1_b64 s[104:105], s2            ; encoding: [0x68,0x02,0x0f,0xbf]
 
-# GFX12: s_bitcmp1_b64 vcc, s2                   ; encoding: [0x6a,0x02,0x0f,0xbf]
 0x6a,0x02,0x0f,0xbf
+# GFX12: s_bitcmp1_b64 vcc, s2                   ; encoding: [0x6a,0x02,0x0f,0xbf]
 
-# GFX12: s_cmp_eq_i32 exec_hi, s1                ; encoding: [0x7f,0x01,0x00,0xbf]
 0x7f,0x01,0x00,0xbf
+# GFX12: s_cmp_eq_i32 exec_hi, s1                ; encoding: [0x7f,0x01,0x00,0xbf]
 
-# GFX12: s_cmp_eq_i32 exec_lo, s1                ; encoding: [0x7e,0x01,0x00,0xbf]
 0x7e,0x01,0x00,0xbf
+# GFX12: s_cmp_eq_i32 exec_lo, s1                ; encoding: [0x7e,0x01,0x00,0xbf]
 
-# GFX12: s_cmp_eq_i32 m0, s1                     ; encoding: [0x7d,0x01,0x00,0xbf]
 0x7d,0x01,0x00,0xbf
+# GFX12: s_cmp_eq_i32 m0, s1                     ; encoding: [0x7d,0x01,0x00,0xbf]
 
-# GFX12: s_cmp_eq_i32 s0, 0.5                    ; encoding: [0x00,0xf0,0x00,0xbf]
 0x00,0xf0,0x00,0xbf
+# GFX12: s_cmp_eq_i32 s0, 0.5                    ; encoding: [0x00,0xf0,0x00,0xbf]
 
-# GFX12: s_cmp_eq_i32 s0, 0                      ; encoding: [0x00,0x80,0x00,0xbf]
 0x00,0x80,0x00,0xbf
+# GFX12: s_cmp_eq_i32 s0, 0                      ; encoding: [0x00,0x80,0x00,0xbf]
 
-# GFX12: s_cmp_eq_i32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x00,0xbf,0x73,0x72,0x71,0x3f]
 0x00,0xff,0x00,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_eq_i32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x00,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_eq_i32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x00,0xbf,0x56,0x34,0x12,0xaf]
 0x00,0xff,0x00,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_eq_i32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x00,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_eq_i32 s0, -1                     ; encoding: [0x00,0xc1,0x00,0xbf]
 0x00,0xc1,0x00,0xbf
+# GFX12: s_cmp_eq_i32 s0, -1                     ; encoding: [0x00,0xc1,0x00,0xbf]
 
-# GFX12: s_cmp_eq_i32 s0, -4.0                   ; encoding: [0x00,0xf7,0x00,0xbf]
 0x00,0xf7,0x00,0xbf
+# GFX12: s_cmp_eq_i32 s0, -4.0                   ; encoding: [0x00,0xf7,0x00,0xbf]
 
-# GFX12: s_cmp_eq_i32 s0, exec_hi                ; encoding: [0x00,0x7f,0x00,0xbf]
 0x00,0x7f,0x00,0xbf
+# GFX12: s_cmp_eq_i32 s0, exec_hi                ; encoding: [0x00,0x7f,0x00,0xbf]
 
-# GFX12: s_cmp_eq_i32 s0, exec_lo                ; encoding: [0x00,0x7e,0x00,0xbf]
 0x00,0x7e,0x00,0xbf
+# GFX12: s_cmp_eq_i32 s0, exec_lo                ; encoding: [0x00,0x7e,0x00,0xbf]
 
-# GFX12: s_cmp_eq_i32 s0, m0                     ; encoding: [0x00,0x7d,0x00,0xbf]
 0x00,0x7d,0x00,0xbf
+# GFX12: s_cmp_eq_i32 s0, m0                     ; encoding: [0x00,0x7d,0x00,0xbf]
 
-# GFX12: s_cmp_eq_i32 s0, s104                   ; encoding: [0x00,0x68,0x00,0xbf]
 0x00,0x68,0x00,0xbf
+# GFX12: s_cmp_eq_i32 s0, s104                   ; encoding: [0x00,0x68,0x00,0xbf]
 
-# GFX12: s_cmp_eq_i32 s0, s1                     ; encoding: [0x00,0x01,0x00,0xbf]
 0x00,0x01,0x00,0xbf
+# GFX12: s_cmp_eq_i32 s0, s1                     ; encoding: [0x00,0x01,0x00,0xbf]
 
-# GFX12: s_cmp_eq_i32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x00,0xbf]
 0x00,0x6b,0x00,0xbf
+# GFX12: s_cmp_eq_i32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x00,0xbf]
 
-# GFX12: s_cmp_eq_i32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x00,0xbf]
 0x00,0x6a,0x00,0xbf
+# GFX12: s_cmp_eq_i32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x00,0xbf]
 
-# GFX12: s_cmp_eq_i32 s105, s104                 ; encoding: [0x69,0x68,0x00,0xbf]
 0x69,0x68,0x00,0xbf
+# GFX12: s_cmp_eq_i32 s105, s104                 ; encoding: [0x69,0x68,0x00,0xbf]
 
-# GFX12: s_cmp_eq_i32 s105, s1                   ; encoding: [0x69,0x01,0x00,0xbf]
 0x69,0x01,0x00,0xbf
+# GFX12: s_cmp_eq_i32 s105, s1                   ; encoding: [0x69,0x01,0x00,0xbf]
 
-# GFX12: s_cmp_eq_i32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x00,0xbf]
 0x6b,0x01,0x00,0xbf
+# GFX12: s_cmp_eq_i32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x00,0xbf]
 
-# GFX12: s_cmp_eq_i32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x00,0xbf]
 0x6a,0x01,0x00,0xbf
+# GFX12: s_cmp_eq_i32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x00,0xbf]
 
-# GFX12: s_cmp_eq_u32 exec_hi, s1                ; encoding: [0x7f,0x01,0x06,0xbf]
 0x7f,0x01,0x06,0xbf
+# GFX12: s_cmp_eq_u32 exec_hi, s1                ; encoding: [0x7f,0x01,0x06,0xbf]
 
-# GFX12: s_cmp_eq_u32 exec_lo, s1                ; encoding: [0x7e,0x01,0x06,0xbf]
 0x7e,0x01,0x06,0xbf
+# GFX12: s_cmp_eq_u32 exec_lo, s1                ; encoding: [0x7e,0x01,0x06,0xbf]
 
-# GFX12: s_cmp_eq_u32 m0, s1                     ; encoding: [0x7d,0x01,0x06,0xbf]
 0x7d,0x01,0x06,0xbf
+# GFX12: s_cmp_eq_u32 m0, s1                     ; encoding: [0x7d,0x01,0x06,0xbf]
 
-# GFX12: s_cmp_eq_u32 s0, 0.5                    ; encoding: [0x00,0xf0,0x06,0xbf]
 0x00,0xf0,0x06,0xbf
+# GFX12: s_cmp_eq_u32 s0, 0.5                    ; encoding: [0x00,0xf0,0x06,0xbf]
 
-# GFX12: s_cmp_eq_u32 s0, 0                      ; encoding: [0x00,0x80,0x06,0xbf]
 0x00,0x80,0x06,0xbf
+# GFX12: s_cmp_eq_u32 s0, 0                      ; encoding: [0x00,0x80,0x06,0xbf]
 
-# GFX12: s_cmp_eq_u32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x06,0xbf,0x73,0x72,0x71,0x3f]
 0x00,0xff,0x06,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_eq_u32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x06,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_eq_u32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x06,0xbf,0x56,0x34,0x12,0xaf]
 0x00,0xff,0x06,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_eq_u32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x06,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_eq_u32 s0, -1                     ; encoding: [0x00,0xc1,0x06,0xbf]
 0x00,0xc1,0x06,0xbf
+# GFX12: s_cmp_eq_u32 s0, -1                     ; encoding: [0x00,0xc1,0x06,0xbf]
 
-# GFX12: s_cmp_eq_u32 s0, -4.0                   ; encoding: [0x00,0xf7,0x06,0xbf]
 0x00,0xf7,0x06,0xbf
+# GFX12: s_cmp_eq_u32 s0, -4.0                   ; encoding: [0x00,0xf7,0x06,0xbf]
 
-# GFX12: s_cmp_eq_u32 s0, exec_hi                ; encoding: [0x00,0x7f,0x06,0xbf]
 0x00,0x7f,0x06,0xbf
+# GFX12: s_cmp_eq_u32 s0, exec_hi                ; encoding: [0x00,0x7f,0x06,0xbf]
 
-# GFX12: s_cmp_eq_u32 s0, exec_lo                ; encoding: [0x00,0x7e,0x06,0xbf]
 0x00,0x7e,0x06,0xbf
+# GFX12: s_cmp_eq_u32 s0, exec_lo                ; encoding: [0x00,0x7e,0x06,0xbf]
 
-# GFX12: s_cmp_eq_u32 s0, m0                     ; encoding: [0x00,0x7d,0x06,0xbf]
 0x00,0x7d,0x06,0xbf
+# GFX12: s_cmp_eq_u32 s0, m0                     ; encoding: [0x00,0x7d,0x06,0xbf]
 
-# GFX12: s_cmp_eq_u32 s0, s104                   ; encoding: [0x00,0x68,0x06,0xbf]
 0x00,0x68,0x06,0xbf
+# GFX12: s_cmp_eq_u32 s0, s104                   ; encoding: [0x00,0x68,0x06,0xbf]
 
-# GFX12: s_cmp_eq_u32 s0, s1                     ; encoding: [0x00,0x01,0x06,0xbf]
 0x00,0x01,0x06,0xbf
+# GFX12: s_cmp_eq_u32 s0, s1                     ; encoding: [0x00,0x01,0x06,0xbf]
 
-# GFX12: s_cmp_eq_u32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x06,0xbf]
 0x00,0x6b,0x06,0xbf
+# GFX12: s_cmp_eq_u32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x06,0xbf]
 
-# GFX12: s_cmp_eq_u32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x06,0xbf]
 0x00,0x6a,0x06,0xbf
+# GFX12: s_cmp_eq_u32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x06,0xbf]
 
-# GFX12: s_cmp_eq_u32 s105, s104                 ; encoding: [0x69,0x68,0x06,0xbf]
 0x69,0x68,0x06,0xbf
+# GFX12: s_cmp_eq_u32 s105, s104                 ; encoding: [0x69,0x68,0x06,0xbf]
 
-# GFX12: s_cmp_eq_u32 s105, s1                   ; encoding: [0x69,0x01,0x06,0xbf]
 0x69,0x01,0x06,0xbf
+# GFX12: s_cmp_eq_u32 s105, s1                   ; encoding: [0x69,0x01,0x06,0xbf]
 
-# GFX12: s_cmp_eq_u32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x06,0xbf]
 0x6b,0x01,0x06,0xbf
+# GFX12: s_cmp_eq_u32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x06,0xbf]
 
-# GFX12: s_cmp_eq_u32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x06,0xbf]
 0x6a,0x01,0x06,0xbf
+# GFX12: s_cmp_eq_u32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x06,0xbf]
 
-# GFX12: s_cmp_eq_u64 exec, s[2:3]               ; encoding: [0x7e,0x02,0x10,0xbf]
 0x7e,0x02,0x10,0xbf
+# GFX12: s_cmp_eq_u64 exec, s[2:3]               ; encoding: [0x7e,0x02,0x10,0xbf]
 
-# GFX12: s_cmp_eq_u64 s[0:1], 0.5                ; encoding: [0x00,0xf0,0x10,0xbf]
 0x00,0xf0,0x10,0xbf
+# GFX12: s_cmp_eq_u64 s[0:1], 0.5                ; encoding: [0x00,0xf0,0x10,0xbf]
 
-# GFX12: s_cmp_eq_u64 s[0:1], 0                  ; encoding: [0x00,0x80,0x10,0xbf]
 0x00,0x80,0x10,0xbf
+# GFX12: s_cmp_eq_u64 s[0:1], 0                  ; encoding: [0x00,0x80,0x10,0xbf]
 
-# GFX12: s_cmp_eq_u64 s[0:1], 0x3f717273         ; encoding: [0x00,0xff,0x10,0xbf,0x73,0x72,0x71,0x3f]
 0x00,0xff,0x10,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_eq_u64 s[0:1], 0x3f717273         ; encoding: [0x00,0xff,0x10,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_eq_u64 s[0:1], 0xaf123456         ; encoding: [0x00,0xff,0x10,0xbf,0x56,0x34,0x12,0xaf]
 0x00,0xff,0x10,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_eq_u64 s[0:1], 0xaf123456         ; encoding: [0x00,0xff,0x10,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_eq_u64 s[0:1], -1                 ; encoding: [0x00,0xc1,0x10,0xbf]
 0x00,0xc1,0x10,0xbf
+# GFX12: s_cmp_eq_u64 s[0:1], -1                 ; encoding: [0x00,0xc1,0x10,0xbf]
 
-# GFX12: s_cmp_eq_u64 s[0:1], -4.0               ; encoding: [0x00,0xf7,0x10,0xbf]
 0x00,0xf7,0x10,0xbf
+# GFX12: s_cmp_eq_u64 s[0:1], -4.0               ; encoding: [0x00,0xf7,0x10,0xbf]
 
-# GFX12: s_cmp_eq_u64 s[0:1], exec               ; encoding: [0x00,0x7e,0x10,0xbf]
 0x00,0x7e,0x10,0xbf
+# GFX12: s_cmp_eq_u64 s[0:1], exec               ; encoding: [0x00,0x7e,0x10,0xbf]
 
-# GFX12: s_cmp_eq_u64 s[0:1], s[102:103]         ; encoding: [0x00,0x66,0x10,0xbf]
 0x00,0x66,0x10,0xbf
+# GFX12: s_cmp_eq_u64 s[0:1], s[102:103]         ; encoding: [0x00,0x66,0x10,0xbf]
 
-# GFX12: s_cmp_eq_u64 s[0:1], s[2:3]             ; encoding: [0x00,0x02,0x10,0xbf]
 0x00,0x02,0x10,0xbf
+# GFX12: s_cmp_eq_u64 s[0:1], s[2:3]             ; encoding: [0x00,0x02,0x10,0xbf]
 
-# GFX12: s_cmp_eq_u64 s[0:1], vcc                ; encoding: [0x00,0x6a,0x10,0xbf]
 0x00,0x6a,0x10,0xbf
+# GFX12: s_cmp_eq_u64 s[0:1], vcc                ; encoding: [0x00,0x6a,0x10,0xbf]
 
-# GFX12: s_cmp_eq_u64 s[104:105], s[102:103]     ; encoding: [0x68,0x66,0x10,0xbf]
 0x68,0x66,0x10,0xbf
+# GFX12: s_cmp_eq_u64 s[104:105], s[102:103]     ; encoding: [0x68,0x66,0x10,0xbf]
 
-# GFX12: s_cmp_eq_u64 s[104:105], s[2:3]         ; encoding: [0x68,0x02,0x10,0xbf]
 0x68,0x02,0x10,0xbf
+# GFX12: s_cmp_eq_u64 s[104:105], s[2:3]         ; encoding: [0x68,0x02,0x10,0xbf]
 
-# GFX12: s_cmp_eq_u64 vcc, s[2:3]                ; encoding: [0x6a,0x02,0x10,0xbf]
 0x6a,0x02,0x10,0xbf
+# GFX12: s_cmp_eq_u64 vcc, s[2:3]                ; encoding: [0x6a,0x02,0x10,0xbf]
 
-# GFX12: s_cmp_ge_i32 exec_hi, s1                ; encoding: [0x7f,0x01,0x03,0xbf]
 0x7f,0x01,0x03,0xbf
+# GFX12: s_cmp_ge_i32 exec_hi, s1                ; encoding: [0x7f,0x01,0x03,0xbf]
 
-# GFX12: s_cmp_ge_i32 exec_lo, s1                ; encoding: [0x7e,0x01,0x03,0xbf]
 0x7e,0x01,0x03,0xbf
+# GFX12: s_cmp_ge_i32 exec_lo, s1                ; encoding: [0x7e,0x01,0x03,0xbf]
 
-# GFX12: s_cmp_ge_i32 m0, s1                     ; encoding: [0x7d,0x01,0x03,0xbf]
 0x7d,0x01,0x03,0xbf
+# GFX12: s_cmp_ge_i32 m0, s1                     ; encoding: [0x7d,0x01,0x03,0xbf]
 
-# GFX12: s_cmp_ge_i32 s0, 0.5                    ; encoding: [0x00,0xf0,0x03,0xbf]
 0x00,0xf0,0x03,0xbf
+# GFX12: s_cmp_ge_i32 s0, 0.5                    ; encoding: [0x00,0xf0,0x03,0xbf]
 
-# GFX12: s_cmp_ge_i32 s0, 0                      ; encoding: [0x00,0x80,0x03,0xbf]
 0x00,0x80,0x03,0xbf
+# GFX12: s_cmp_ge_i32 s0, 0                      ; encoding: [0x00,0x80,0x03,0xbf]
 
-# GFX12: s_cmp_ge_i32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x03,0xbf,0x73,0x72,0x71,0x3f]
 0x00,0xff,0x03,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_ge_i32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x03,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_ge_i32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x03,0xbf,0x56,0x34,0x12,0xaf]
 0x00,0xff,0x03,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_ge_i32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x03,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_ge_i32 s0, -1                     ; encoding: [0x00,0xc1,0x03,0xbf]
 0x00,0xc1,0x03,0xbf
+# GFX12: s_cmp_ge_i32 s0, -1                     ; encoding: [0x00,0xc1,0x03,0xbf]
 
-# GFX12: s_cmp_ge_i32 s0, -4.0                   ; encoding: [0x00,0xf7,0x03,0xbf]
 0x00,0xf7,0x03,0xbf
+# GFX12: s_cmp_ge_i32 s0, -4.0                   ; encoding: [0x00,0xf7,0x03,0xbf]
 
-# GFX12: s_cmp_ge_i32 s0, exec_hi                ; encoding: [0x00,0x7f,0x03,0xbf]
 0x00,0x7f,0x03,0xbf
+# GFX12: s_cmp_ge_i32 s0, exec_hi                ; encoding: [0x00,0x7f,0x03,0xbf]
 
-# GFX12: s_cmp_ge_i32 s0, exec_lo                ; encoding: [0x00,0x7e,0x03,0xbf]
 0x00,0x7e,0x03,0xbf
+# GFX12: s_cmp_ge_i32 s0, exec_lo                ; encoding: [0x00,0x7e,0x03,0xbf]
 
-# GFX12: s_cmp_ge_i32 s0, m0                     ; encoding: [0x00,0x7d,0x03,0xbf]
 0x00,0x7d,0x03,0xbf
+# GFX12: s_cmp_ge_i32 s0, m0                     ; encoding: [0x00,0x7d,0x03,0xbf]
 
-# GFX12: s_cmp_ge_i32 s0, s104                   ; encoding: [0x00,0x68,0x03,0xbf]
 0x00,0x68,0x03,0xbf
+# GFX12: s_cmp_ge_i32 s0, s104                   ; encoding: [0x00,0x68,0x03,0xbf]
 
-# GFX12: s_cmp_ge_i32 s0, s1                     ; encoding: [0x00,0x01,0x03,0xbf]
 0x00,0x01,0x03,0xbf
+# GFX12: s_cmp_ge_i32 s0, s1                     ; encoding: [0x00,0x01,0x03,0xbf]
 
-# GFX12: s_cmp_ge_i32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x03,0xbf]
 0x00,0x6b,0x03,0xbf
+# GFX12: s_cmp_ge_i32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x03,0xbf]
 
-# GFX12: s_cmp_ge_i32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x03,0xbf]
 0x00,0x6a,0x03,0xbf
+# GFX12: s_cmp_ge_i32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x03,0xbf]
 
-# GFX12: s_cmp_ge_i32 s105, s104                 ; encoding: [0x69,0x68,0x03,0xbf]
 0x69,0x68,0x03,0xbf
+# GFX12: s_cmp_ge_i32 s105, s104                 ; encoding: [0x69,0x68,0x03,0xbf]
 
-# GFX12: s_cmp_ge_i32 s105, s1                   ; encoding: [0x69,0x01,0x03,0xbf]
 0x69,0x01,0x03,0xbf
+# GFX12: s_cmp_ge_i32 s105, s1                   ; encoding: [0x69,0x01,0x03,0xbf]
 
-# GFX12: s_cmp_ge_i32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x03,0xbf]
 0x6b,0x01,0x03,0xbf
+# GFX12: s_cmp_ge_i32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x03,0xbf]
 
-# GFX12: s_cmp_ge_i32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x03,0xbf]
 0x6a,0x01,0x03,0xbf
+# GFX12: s_cmp_ge_i32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x03,0xbf]
 
-# GFX12: s_cmp_ge_u32 exec_hi, s1                ; encoding: [0x7f,0x01,0x09,0xbf]
 0x7f,0x01,0x09,0xbf
+# GFX12: s_cmp_ge_u32 exec_hi, s1                ; encoding: [0x7f,0x01,0x09,0xbf]
 
-# GFX12: s_cmp_ge_u32 exec_lo, s1                ; encoding: [0x7e,0x01,0x09,0xbf]
 0x7e,0x01,0x09,0xbf
+# GFX12: s_cmp_ge_u32 exec_lo, s1                ; encoding: [0x7e,0x01,0x09,0xbf]
 
-# GFX12: s_cmp_ge_u32 m0, s1                     ; encoding: [0x7d,0x01,0x09,0xbf]
 0x7d,0x01,0x09,0xbf
+# GFX12: s_cmp_ge_u32 m0, s1                     ; encoding: [0x7d,0x01,0x09,0xbf]
 
-# GFX12: s_cmp_ge_u32 s0, 0.5                    ; encoding: [0x00,0xf0,0x09,0xbf]
 0x00,0xf0,0x09,0xbf
+# GFX12: s_cmp_ge_u32 s0, 0.5                    ; encoding: [0x00,0xf0,0x09,0xbf]
 
-# GFX12: s_cmp_ge_u32 s0, 0                      ; encoding: [0x00,0x80,0x09,0xbf]
 0x00,0x80,0x09,0xbf
+# GFX12: s_cmp_ge_u32 s0, 0                      ; encoding: [0x00,0x80,0x09,0xbf]
 
-# GFX12: s_cmp_ge_u32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x09,0xbf,0x73,0x72,0x71,0x3f]
 0x00,0xff,0x09,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_ge_u32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x09,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_ge_u32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x09,0xbf,0x56,0x34,0x12,0xaf]
 0x00,0xff,0x09,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_ge_u32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x09,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_ge_u32 s0, -1                     ; encoding: [0x00,0xc1,0x09,0xbf]
 0x00,0xc1,0x09,0xbf
+# GFX12: s_cmp_ge_u32 s0, -1                     ; encoding: [0x00,0xc1,0x09,0xbf]
 
-# GFX12: s_cmp_ge_u32 s0, -4.0                   ; encoding: [0x00,0xf7,0x09,0xbf]
 0x00,0xf7,0x09,0xbf
+# GFX12: s_cmp_ge_u32 s0, -4.0                   ; encoding: [0x00,0xf7,0x09,0xbf]
 
-# GFX12: s_cmp_ge_u32 s0, exec_hi                ; encoding: [0x00,0x7f,0x09,0xbf]
 0x00,0x7f,0x09,0xbf
+# GFX12: s_cmp_ge_u32 s0, exec_hi                ; encoding: [0x00,0x7f,0x09,0xbf]
 
-# GFX12: s_cmp_ge_u32 s0, exec_lo                ; encoding: [0x00,0x7e,0x09,0xbf]
 0x00,0x7e,0x09,0xbf
+# GFX12: s_cmp_ge_u32 s0, exec_lo                ; encoding: [0x00,0x7e,0x09,0xbf]
 
-# GFX12: s_cmp_ge_u32 s0, m0                     ; encoding: [0x00,0x7d,0x09,0xbf]
 0x00,0x7d,0x09,0xbf
+# GFX12: s_cmp_ge_u32 s0, m0                     ; encoding: [0x00,0x7d,0x09,0xbf]
 
-# GFX12: s_cmp_ge_u32 s0, s104                   ; encoding: [0x00,0x68,0x09,0xbf]
 0x00,0x68,0x09,0xbf
+# GFX12: s_cmp_ge_u32 s0, s104                   ; encoding: [0x00,0x68,0x09,0xbf]
 
-# GFX12: s_cmp_ge_u32 s0, s1                     ; encoding: [0x00,0x01,0x09,0xbf]
 0x00,0x01,0x09,0xbf
+# GFX12: s_cmp_ge_u32 s0, s1                     ; encoding: [0x00,0x01,0x09,0xbf]
 
-# GFX12: s_cmp_ge_u32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x09,0xbf]
 0x00,0x6b,0x09,0xbf
+# GFX12: s_cmp_ge_u32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x09,0xbf]
 
-# GFX12: s_cmp_ge_u32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x09,0xbf]
 0x00,0x6a,0x09,0xbf
+# GFX12: s_cmp_ge_u32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x09,0xbf]
 
-# GFX12: s_cmp_ge_u32 s105, s104                 ; encoding: [0x69,0x68,0x09,0xbf]
 0x69,0x68,0x09,0xbf
+# GFX12: s_cmp_ge_u32 s105, s104                 ; encoding: [0x69,0x68,0x09,0xbf]
 
-# GFX12: s_cmp_ge_u32 s105, s1                   ; encoding: [0x69,0x01,0x09,0xbf]
 0x69,0x01,0x09,0xbf
+# GFX12: s_cmp_ge_u32 s105, s1                   ; encoding: [0x69,0x01,0x09,0xbf]
 
-# GFX12: s_cmp_ge_u32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x09,0xbf]
 0x6b,0x01,0x09,0xbf
+# GFX12: s_cmp_ge_u32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x09,0xbf]
 
-# GFX12: s_cmp_ge_u32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x09,0xbf]
 0x6a,0x01,0x09,0xbf
+# GFX12: s_cmp_ge_u32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x09,0xbf]
 
-# GFX12: s_cmp_gt_i32 exec_hi, s1                ; encoding: [0x7f,0x01,0x02,0xbf]
 0x7f,0x01,0x02,0xbf
+# GFX12: s_cmp_gt_i32 exec_hi, s1                ; encoding: [0x7f,0x01,0x02,0xbf]
 
-# GFX12: s_cmp_gt_i32 exec_lo, s1                ; encoding: [0x7e,0x01,0x02,0xbf]
 0x7e,0x01,0x02,0xbf
+# GFX12: s_cmp_gt_i32 exec_lo, s1                ; encoding: [0x7e,0x01,0x02,0xbf]
 
-# GFX12: s_cmp_gt_i32 m0, s1                     ; encoding: [0x7d,0x01,0x02,0xbf]
 0x7d,0x01,0x02,0xbf
+# GFX12: s_cmp_gt_i32 m0, s1                     ; encoding: [0x7d,0x01,0x02,0xbf]
 
-# GFX12: s_cmp_gt_i32 s0, 0.5                    ; encoding: [0x00,0xf0,0x02,0xbf]
 0x00,0xf0,0x02,0xbf
+# GFX12: s_cmp_gt_i32 s0, 0.5                    ; encoding: [0x00,0xf0,0x02,0xbf]
 
-# GFX12: s_cmp_gt_i32 s0, 0                      ; encoding: [0x00,0x80,0x02,0xbf]
 0x00,0x80,0x02,0xbf
+# GFX12: s_cmp_gt_i32 s0, 0                      ; encoding: [0x00,0x80,0x02,0xbf]
 
-# GFX12: s_cmp_gt_i32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x02,0xbf,0x73,0x72,0x71,0x3f]
 0x00,0xff,0x02,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_gt_i32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x02,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_gt_i32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x02,0xbf,0x56,0x34,0x12,0xaf]
 0x00,0xff,0x02,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_gt_i32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x02,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_gt_i32 s0, -1                     ; encoding: [0x00,0xc1,0x02,0xbf]
 0x00,0xc1,0x02,0xbf
+# GFX12: s_cmp_gt_i32 s0, -1                     ; encoding: [0x00,0xc1,0x02,0xbf]
 
-# GFX12: s_cmp_gt_i32 s0, -4.0                   ; encoding: [0x00,0xf7,0x02,0xbf]
 0x00,0xf7,0x02,0xbf
+# GFX12: s_cmp_gt_i32 s0, -4.0                   ; encoding: [0x00,0xf7,0x02,0xbf]
 
-# GFX12: s_cmp_gt_i32 s0, exec_hi                ; encoding: [0x00,0x7f,0x02,0xbf]
 0x00,0x7f,0x02,0xbf
+# GFX12: s_cmp_gt_i32 s0, exec_hi                ; encoding: [0x00,0x7f,0x02,0xbf]
 
-# GFX12: s_cmp_gt_i32 s0, exec_lo                ; encoding: [0x00,0x7e,0x02,0xbf]
 0x00,0x7e,0x02,0xbf
+# GFX12: s_cmp_gt_i32 s0, exec_lo                ; encoding: [0x00,0x7e,0x02,0xbf]
 
-# GFX12: s_cmp_gt_i32 s0, m0                     ; encoding: [0x00,0x7d,0x02,0xbf]
 0x00,0x7d,0x02,0xbf
+# GFX12: s_cmp_gt_i32 s0, m0                     ; encoding: [0x00,0x7d,0x02,0xbf]
 
-# GFX12: s_cmp_gt_i32 s0, s104                   ; encoding: [0x00,0x68,0x02,0xbf]
 0x00,0x68,0x02,0xbf
+# GFX12: s_cmp_gt_i32 s0, s104                   ; encoding: [0x00,0x68,0x02,0xbf]
 
-# GFX12: s_cmp_gt_i32 s0, s1                     ; encoding: [0x00,0x01,0x02,0xbf]
 0x00,0x01,0x02,0xbf
+# GFX12: s_cmp_gt_i32 s0, s1                     ; encoding: [0x00,0x01,0x02,0xbf]
 
-# GFX12: s_cmp_gt_i32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x02,0xbf]
 0x00,0x6b,0x02,0xbf
+# GFX12: s_cmp_gt_i32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x02,0xbf]
 
-# GFX12: s_cmp_gt_i32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x02,0xbf]
 0x00,0x6a,0x02,0xbf
+# GFX12: s_cmp_gt_i32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x02,0xbf]
 
-# GFX12: s_cmp_gt_i32 s105, s104                 ; encoding: [0x69,0x68,0x02,0xbf]
 0x69,0x68,0x02,0xbf
+# GFX12: s_cmp_gt_i32 s105, s104                 ; encoding: [0x69,0x68,0x02,0xbf]
 
-# GFX12: s_cmp_gt_i32 s105, s1                   ; encoding: [0x69,0x01,0x02,0xbf]
 0x69,0x01,0x02,0xbf
+# GFX12: s_cmp_gt_i32 s105, s1                   ; encoding: [0x69,0x01,0x02,0xbf]
 
-# GFX12: s_cmp_gt_i32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x02,0xbf]
 0x6b,0x01,0x02,0xbf
+# GFX12: s_cmp_gt_i32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x02,0xbf]
 
-# GFX12: s_cmp_gt_i32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x02,0xbf]
 0x6a,0x01,0x02,0xbf
+# GFX12: s_cmp_gt_i32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x02,0xbf]
 
-# GFX12: s_cmp_gt_u32 exec_hi, s1                ; encoding: [0x7f,0x01,0x08,0xbf]
 0x7f,0x01,0x08,0xbf
+# GFX12: s_cmp_gt_u32 exec_hi, s1                ; encoding: [0x7f,0x01,0x08,0xbf]
 
-# GFX12: s_cmp_gt_u32 exec_lo, s1                ; encoding: [0x7e,0x01,0x08,0xbf]
 0x7e,0x01,0x08,0xbf
+# GFX12: s_cmp_gt_u32 exec_lo, s1                ; encoding: [0x7e,0x01,0x08,0xbf]
 
-# GFX12: s_cmp_gt_u32 m0, s1                     ; encoding: [0x7d,0x01,0x08,0xbf]
 0x7d,0x01,0x08,0xbf
+# GFX12: s_cmp_gt_u32 m0, s1                     ; encoding: [0x7d,0x01,0x08,0xbf]
 
-# GFX12: s_cmp_gt_u32 s0, 0.5                    ; encoding: [0x00,0xf0,0x08,0xbf]
 0x00,0xf0,0x08,0xbf
+# GFX12: s_cmp_gt_u32 s0, 0.5                    ; encoding: [0x00,0xf0,0x08,0xbf]
 
-# GFX12: s_cmp_gt_u32 s0, 0                      ; encoding: [0x00,0x80,0x08,0xbf]
 0x00,0x80,0x08,0xbf
+# GFX12: s_cmp_gt_u32 s0, 0                      ; encoding: [0x00,0x80,0x08,0xbf]
 
-# GFX12: s_cmp_gt_u32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x08,0xbf,0x73,0x72,0x71,0x3f]
 0x00,0xff,0x08,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_gt_u32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x08,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_gt_u32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x08,0xbf,0x56,0x34,0x12,0xaf]
 0x00,0xff,0x08,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_gt_u32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x08,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_gt_u32 s0, -1                     ; encoding: [0x00,0xc1,0x08,0xbf]
 0x00,0xc1,0x08,0xbf
+# GFX12: s_cmp_gt_u32 s0, -1                     ; encoding: [0x00,0xc1,0x08,0xbf]
 
-# GFX12: s_cmp_gt_u32 s0, -4.0                   ; encoding: [0x00,0xf7,0x08,0xbf]
 0x00,0xf7,0x08,0xbf
+# GFX12: s_cmp_gt_u32 s0, -4.0                   ; encoding: [0x00,0xf7,0x08,0xbf]
 
-# GFX12: s_cmp_gt_u32 s0, exec_hi                ; encoding: [0x00,0x7f,0x08,0xbf]
 0x00,0x7f,0x08,0xbf
+# GFX12: s_cmp_gt_u32 s0, exec_hi                ; encoding: [0x00,0x7f,0x08,0xbf]
 
-# GFX12: s_cmp_gt_u32 s0, exec_lo                ; encoding: [0x00,0x7e,0x08,0xbf]
 0x00,0x7e,0x08,0xbf
+# GFX12: s_cmp_gt_u32 s0, exec_lo                ; encoding: [0x00,0x7e,0x08,0xbf]
 
-# GFX12: s_cmp_gt_u32 s0, m0                     ; encoding: [0x00,0x7d,0x08,0xbf]
 0x00,0x7d,0x08,0xbf
+# GFX12: s_cmp_gt_u32 s0, m0                     ; encoding: [0x00,0x7d,0x08,0xbf]
 
-# GFX12: s_cmp_gt_u32 s0, s104                   ; encoding: [0x00,0x68,0x08,0xbf]
 0x00,0x68,0x08,0xbf
+# GFX12: s_cmp_gt_u32 s0, s104                   ; encoding: [0x00,0x68,0x08,0xbf]
 
-# GFX12: s_cmp_gt_u32 s0, s1                     ; encoding: [0x00,0x01,0x08,0xbf]
 0x00,0x01,0x08,0xbf
+# GFX12: s_cmp_gt_u32 s0, s1                     ; encoding: [0x00,0x01,0x08,0xbf]
 
-# GFX12: s_cmp_gt_u32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x08,0xbf]
 0x00,0x6b,0x08,0xbf
+# GFX12: s_cmp_gt_u32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x08,0xbf]
 
-# GFX12: s_cmp_gt_u32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x08,0xbf]
 0x00,0x6a,0x08,0xbf
+# GFX12: s_cmp_gt_u32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x08,0xbf]
 
-# GFX12: s_cmp_gt_u32 s105, s104                 ; encoding: [0x69,0x68,0x08,0xbf]
 0x69,0x68,0x08,0xbf
+# GFX12: s_cmp_gt_u32 s105, s104                 ; encoding: [0x69,0x68,0x08,0xbf]
 
-# GFX12: s_cmp_gt_u32 s105, s1                   ; encoding: [0x69,0x01,0x08,0xbf]
 0x69,0x01,0x08,0xbf
+# GFX12: s_cmp_gt_u32 s105, s1                   ; encoding: [0x69,0x01,0x08,0xbf]
 
-# GFX12: s_cmp_gt_u32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x08,0xbf]
 0x6b,0x01,0x08,0xbf
+# GFX12: s_cmp_gt_u32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x08,0xbf]
 
-# GFX12: s_cmp_gt_u32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x08,0xbf]
 0x6a,0x01,0x08,0xbf
+# GFX12: s_cmp_gt_u32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x08,0xbf]
 
-# GFX12: s_cmp_le_i32 exec_hi, s1                ; encoding: [0x7f,0x01,0x05,0xbf]
 0x7f,0x01,0x05,0xbf
+# GFX12: s_cmp_le_i32 exec_hi, s1                ; encoding: [0x7f,0x01,0x05,0xbf]
 
-# GFX12: s_cmp_le_i32 exec_lo, s1                ; encoding: [0x7e,0x01,0x05,0xbf]
 0x7e,0x01,0x05,0xbf
+# GFX12: s_cmp_le_i32 exec_lo, s1                ; encoding: [0x7e,0x01,0x05,0xbf]
 
-# GFX12: s_cmp_le_i32 m0, s1                     ; encoding: [0x7d,0x01,0x05,0xbf]
 0x7d,0x01,0x05,0xbf
+# GFX12: s_cmp_le_i32 m0, s1                     ; encoding: [0x7d,0x01,0x05,0xbf]
 
-# GFX12: s_cmp_le_i32 s0, 0.5                    ; encoding: [0x00,0xf0,0x05,0xbf]
 0x00,0xf0,0x05,0xbf
+# GFX12: s_cmp_le_i32 s0, 0.5                    ; encoding: [0x00,0xf0,0x05,0xbf]
 
-# GFX12: s_cmp_le_i32 s0, 0                      ; encoding: [0x00,0x80,0x05,0xbf]
 0x00,0x80,0x05,0xbf
+# GFX12: s_cmp_le_i32 s0, 0                      ; encoding: [0x00,0x80,0x05,0xbf]
 
-# GFX12: s_cmp_le_i32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x05,0xbf,0x73,0x72,0x71,0x3f]
 0x00,0xff,0x05,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_le_i32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x05,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_le_i32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x05,0xbf,0x56,0x34,0x12,0xaf]
 0x00,0xff,0x05,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_le_i32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x05,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_le_i32 s0, -1                     ; encoding: [0x00,0xc1,0x05,0xbf]
 0x00,0xc1,0x05,0xbf
+# GFX12: s_cmp_le_i32 s0, -1                     ; encoding: [0x00,0xc1,0x05,0xbf]
 
-# GFX12: s_cmp_le_i32 s0, -4.0                   ; encoding: [0x00,0xf7,0x05,0xbf]
 0x00,0xf7,0x05,0xbf
+# GFX12: s_cmp_le_i32 s0, -4.0                   ; encoding: [0x00,0xf7,0x05,0xbf]
 
-# GFX12: s_cmp_le_i32 s0, exec_hi                ; encoding: [0x00,0x7f,0x05,0xbf]
 0x00,0x7f,0x05,0xbf
+# GFX12: s_cmp_le_i32 s0, exec_hi                ; encoding: [0x00,0x7f,0x05,0xbf]
 
-# GFX12: s_cmp_le_i32 s0, exec_lo                ; encoding: [0x00,0x7e,0x05,0xbf]
 0x00,0x7e,0x05,0xbf
+# GFX12: s_cmp_le_i32 s0, exec_lo                ; encoding: [0x00,0x7e,0x05,0xbf]
 
-# GFX12: s_cmp_le_i32 s0, m0                     ; encoding: [0x00,0x7d,0x05,0xbf]
 0x00,0x7d,0x05,0xbf
+# GFX12: s_cmp_le_i32 s0, m0                     ; encoding: [0x00,0x7d,0x05,0xbf]
 
-# GFX12: s_cmp_le_i32 s0, s104                   ; encoding: [0x00,0x68,0x05,0xbf]
 0x00,0x68,0x05,0xbf
+# GFX12: s_cmp_le_i32 s0, s104                   ; encoding: [0x00,0x68,0x05,0xbf]
 
-# GFX12: s_cmp_le_i32 s0, s1                     ; encoding: [0x00,0x01,0x05,0xbf]
 0x00,0x01,0x05,0xbf
+# GFX12: s_cmp_le_i32 s0, s1                     ; encoding: [0x00,0x01,0x05,0xbf]
 
-# GFX12: s_cmp_le_i32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x05,0xbf]
 0x00,0x6b,0x05,0xbf
+# GFX12: s_cmp_le_i32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x05,0xbf]
 
-# GFX12: s_cmp_le_i32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x05,0xbf]
 0x00,0x6a,0x05,0xbf
+# GFX12: s_cmp_le_i32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x05,0xbf]
 
-# GFX12: s_cmp_le_i32 s105, s104                 ; encoding: [0x69,0x68,0x05,0xbf]
 0x69,0x68,0x05,0xbf
+# GFX12: s_cmp_le_i32 s105, s104                 ; encoding: [0x69,0x68,0x05,0xbf]
 
-# GFX12: s_cmp_le_i32 s105, s1                   ; encoding: [0x69,0x01,0x05,0xbf]
 0x69,0x01,0x05,0xbf
+# GFX12: s_cmp_le_i32 s105, s1                   ; encoding: [0x69,0x01,0x05,0xbf]
 
-# GFX12: s_cmp_le_i32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x05,0xbf]
 0x6b,0x01,0x05,0xbf
+# GFX12: s_cmp_le_i32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x05,0xbf]
 
-# GFX12: s_cmp_le_i32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x05,0xbf]
 0x6a,0x01,0x05,0xbf
+# GFX12: s_cmp_le_i32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x05,0xbf]
 
-# GFX12: s_cmp_le_u32 exec_hi, s1                ; encoding: [0x7f,0x01,0x0b,0xbf]
 0x7f,0x01,0x0b,0xbf
+# GFX12: s_cmp_le_u32 exec_hi, s1                ; encoding: [0x7f,0x01,0x0b,0xbf]
 
-# GFX12: s_cmp_le_u32 exec_lo, s1                ; encoding: [0x7e,0x01,0x0b,0xbf]
 0x7e,0x01,0x0b,0xbf
+# GFX12: s_cmp_le_u32 exec_lo, s1                ; encoding: [0x7e,0x01,0x0b,0xbf]
 
-# GFX12: s_cmp_le_u32 m0, s1                     ; encoding: [0x7d,0x01,0x0b,0xbf]
 0x7d,0x01,0x0b,0xbf
+# GFX12: s_cmp_le_u32 m0, s1                     ; encoding: [0x7d,0x01,0x0b,0xbf]
 
-# GFX12: s_cmp_le_u32 s0, 0.5                    ; encoding: [0x00,0xf0,0x0b,0xbf]
 0x00,0xf0,0x0b,0xbf
+# GFX12: s_cmp_le_u32 s0, 0.5                    ; encoding: [0x00,0xf0,0x0b,0xbf]
 
-# GFX12: s_cmp_le_u32 s0, 0                      ; encoding: [0x00,0x80,0x0b,0xbf]
 0x00,0x80,0x0b,0xbf
+# GFX12: s_cmp_le_u32 s0, 0                      ; encoding: [0x00,0x80,0x0b,0xbf]
 
-# GFX12: s_cmp_le_u32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x0b,0xbf,0x73,0x72,0x71,0x3f]
 0x00,0xff,0x0b,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_le_u32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x0b,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_le_u32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x0b,0xbf,0x56,0x34,0x12,0xaf]
 0x00,0xff,0x0b,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_le_u32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x0b,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_le_u32 s0, -1                     ; encoding: [0x00,0xc1,0x0b,0xbf]
 0x00,0xc1,0x0b,0xbf
+# GFX12: s_cmp_le_u32 s0, -1                     ; encoding: [0x00,0xc1,0x0b,0xbf]
 
-# GFX12: s_cmp_le_u32 s0, -4.0                   ; encoding: [0x00,0xf7,0x0b,0xbf]
 0x00,0xf7,0x0b,0xbf
+# GFX12: s_cmp_le_u32 s0, -4.0                   ; encoding: [0x00,0xf7,0x0b,0xbf]
 
-# GFX12: s_cmp_le_u32 s0, exec_hi                ; encoding: [0x00,0x7f,0x0b,0xbf]
 0x00,0x7f,0x0b,0xbf
+# GFX12: s_cmp_le_u32 s0, exec_hi                ; encoding: [0x00,0x7f,0x0b,0xbf]
 
-# GFX12: s_cmp_le_u32 s0, exec_lo                ; encoding: [0x00,0x7e,0x0b,0xbf]
 0x00,0x7e,0x0b,0xbf
+# GFX12: s_cmp_le_u32 s0, exec_lo                ; encoding: [0x00,0x7e,0x0b,0xbf]
 
-# GFX12: s_cmp_le_u32 s0, m0                     ; encoding: [0x00,0x7d,0x0b,0xbf]
 0x00,0x7d,0x0b,0xbf
+# GFX12: s_cmp_le_u32 s0, m0                     ; encoding: [0x00,0x7d,0x0b,0xbf]
 
-# GFX12: s_cmp_le_u32 s0, s104                   ; encoding: [0x00,0x68,0x0b,0xbf]
 0x00,0x68,0x0b,0xbf
+# GFX12: s_cmp_le_u32 s0, s104                   ; encoding: [0x00,0x68,0x0b,0xbf]
 
-# GFX12: s_cmp_le_u32 s0, s1                     ; encoding: [0x00,0x01,0x0b,0xbf]
 0x00,0x01,0x0b,0xbf
+# GFX12: s_cmp_le_u32 s0, s1                     ; encoding: [0x00,0x01,0x0b,0xbf]
 
-# GFX12: s_cmp_le_u32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x0b,0xbf]
 0x00,0x6b,0x0b,0xbf
+# GFX12: s_cmp_le_u32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x0b,0xbf]
 
-# GFX12: s_cmp_le_u32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x0b,0xbf]
 0x00,0x6a,0x0b,0xbf
+# GFX12: s_cmp_le_u32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x0b,0xbf]
 
-# GFX12: s_cmp_le_u32 s105, s104                 ; encoding: [0x69,0x68,0x0b,0xbf]
 0x69,0x68,0x0b,0xbf
+# GFX12: s_cmp_le_u32 s105, s104                 ; encoding: [0x69,0x68,0x0b,0xbf]
 
-# GFX12: s_cmp_le_u32 s105, s1                   ; encoding: [0x69,0x01,0x0b,0xbf]
 0x69,0x01,0x0b,0xbf
+# GFX12: s_cmp_le_u32 s105, s1                   ; encoding: [0x69,0x01,0x0b,0xbf]
 
-# GFX12: s_cmp_le_u32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x0b,0xbf]
 0x6b,0x01,0x0b,0xbf
+# GFX12: s_cmp_le_u32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x0b,0xbf]
 
-# GFX12: s_cmp_le_u32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x0b,0xbf]
 0x6a,0x01,0x0b,0xbf
+# GFX12: s_cmp_le_u32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x0b,0xbf]
 
-# GFX12: s_cmp_lg_i32 exec_hi, s1                ; encoding: [0x7f,0x01,0x01,0xbf]
 0x7f,0x01,0x01,0xbf
+# GFX12: s_cmp_lg_i32 exec_hi, s1                ; encoding: [0x7f,0x01,0x01,0xbf]
 
-# GFX12: s_cmp_lg_i32 exec_lo, s1                ; encoding: [0x7e,0x01,0x01,0xbf]
 0x7e,0x01,0x01,0xbf
+# GFX12: s_cmp_lg_i32 exec_lo, s1                ; encoding: [0x7e,0x01,0x01,0xbf]
 
-# GFX12: s_cmp_lg_i32 m0, s1                     ; encoding: [0x7d,0x01,0x01,0xbf]
 0x7d,0x01,0x01,0xbf
+# GFX12: s_cmp_lg_i32 m0, s1                     ; encoding: [0x7d,0x01,0x01,0xbf]
 
-# GFX12: s_cmp_lg_i32 s0, 0.5                    ; encoding: [0x00,0xf0,0x01,0xbf]
 0x00,0xf0,0x01,0xbf
+# GFX12: s_cmp_lg_i32 s0, 0.5                    ; encoding: [0x00,0xf0,0x01,0xbf]
 
-# GFX12: s_cmp_lg_i32 s0, 0                      ; encoding: [0x00,0x80,0x01,0xbf]
 0x00,0x80,0x01,0xbf
+# GFX12: s_cmp_lg_i32 s0, 0                      ; encoding: [0x00,0x80,0x01,0xbf]
 
-# GFX12: s_cmp_lg_i32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x01,0xbf,0x73,0x72,0x71,0x3f]
 0x00,0xff,0x01,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_lg_i32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x01,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_lg_i32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x01,0xbf,0x56,0x34,0x12,0xaf]
 0x00,0xff,0x01,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_lg_i32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x01,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_lg_i32 s0, -1                     ; encoding: [0x00,0xc1,0x01,0xbf]
 0x00,0xc1,0x01,0xbf
+# GFX12: s_cmp_lg_i32 s0, -1                     ; encoding: [0x00,0xc1,0x01,0xbf]
 
-# GFX12: s_cmp_lg_i32 s0, -4.0                   ; encoding: [0x00,0xf7,0x01,0xbf]
 0x00,0xf7,0x01,0xbf
+# GFX12: s_cmp_lg_i32 s0, -4.0                   ; encoding: [0x00,0xf7,0x01,0xbf]
 
-# GFX12: s_cmp_lg_i32 s0, exec_hi                ; encoding: [0x00,0x7f,0x01,0xbf]
 0x00,0x7f,0x01,0xbf
+# GFX12: s_cmp_lg_i32 s0, exec_hi                ; encoding: [0x00,0x7f,0x01,0xbf]
 
-# GFX12: s_cmp_lg_i32 s0, exec_lo                ; encoding: [0x00,0x7e,0x01,0xbf]
 0x00,0x7e,0x01,0xbf
+# GFX12: s_cmp_lg_i32 s0, exec_lo                ; encoding: [0x00,0x7e,0x01,0xbf]
 
-# GFX12: s_cmp_lg_i32 s0, m0                     ; encoding: [0x00,0x7d,0x01,0xbf]
 0x00,0x7d,0x01,0xbf
+# GFX12: s_cmp_lg_i32 s0, m0                     ; encoding: [0x00,0x7d,0x01,0xbf]
 
-# GFX12: s_cmp_lg_i32 s0, s104                   ; encoding: [0x00,0x68,0x01,0xbf]
 0x00,0x68,0x01,0xbf
+# GFX12: s_cmp_lg_i32 s0, s104                   ; encoding: [0x00,0x68,0x01,0xbf]
 
-# GFX12: s_cmp_lg_i32 s0, s1                     ; encoding: [0x00,0x01,0x01,0xbf]
 0x00,0x01,0x01,0xbf
+# GFX12: s_cmp_lg_i32 s0, s1                     ; encoding: [0x00,0x01,0x01,0xbf]
 
-# GFX12: s_cmp_lg_i32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x01,0xbf]
 0x00,0x6b,0x01,0xbf
+# GFX12: s_cmp_lg_i32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x01,0xbf]
 
-# GFX12: s_cmp_lg_i32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x01,0xbf]
 0x00,0x6a,0x01,0xbf
+# GFX12: s_cmp_lg_i32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x01,0xbf]
 
-# GFX12: s_cmp_lg_i32 s105, s104                 ; encoding: [0x69,0x68,0x01,0xbf]
 0x69,0x68,0x01,0xbf
+# GFX12: s_cmp_lg_i32 s105, s104                 ; encoding: [0x69,0x68,0x01,0xbf]
 
-# GFX12: s_cmp_lg_i32 s105, s1                   ; encoding: [0x69,0x01,0x01,0xbf]
 0x69,0x01,0x01,0xbf
+# GFX12: s_cmp_lg_i32 s105, s1                   ; encoding: [0x69,0x01,0x01,0xbf]
 
-# GFX12: s_cmp_lg_i32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x01,0xbf]
 0x6b,0x01,0x01,0xbf
+# GFX12: s_cmp_lg_i32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x01,0xbf]
 
-# GFX12: s_cmp_lg_i32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x01,0xbf]
 0x6a,0x01,0x01,0xbf
+# GFX12: s_cmp_lg_i32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x01,0xbf]
 
-# GFX12: s_cmp_lg_u32 exec_hi, s1                ; encoding: [0x7f,0x01,0x07,0xbf]
 0x7f,0x01,0x07,0xbf
+# GFX12: s_cmp_lg_u32 exec_hi, s1                ; encoding: [0x7f,0x01,0x07,0xbf]
 
-# GFX12: s_cmp_lg_u32 exec_lo, s1                ; encoding: [0x7e,0x01,0x07,0xbf]
 0x7e,0x01,0x07,0xbf
+# GFX12: s_cmp_lg_u32 exec_lo, s1                ; encoding: [0x7e,0x01,0x07,0xbf]
 
-# GFX12: s_cmp_lg_u32 m0, s1                     ; encoding: [0x7d,0x01,0x07,0xbf]
 0x7d,0x01,0x07,0xbf
+# GFX12: s_cmp_lg_u32 m0, s1                     ; encoding: [0x7d,0x01,0x07,0xbf]
 
-# GFX12: s_cmp_lg_u32 s0, 0.5                    ; encoding: [0x00,0xf0,0x07,0xbf]
 0x00,0xf0,0x07,0xbf
+# GFX12: s_cmp_lg_u32 s0, 0.5                    ; encoding: [0x00,0xf0,0x07,0xbf]
 
-# GFX12: s_cmp_lg_u32 s0, 0                      ; encoding: [0x00,0x80,0x07,0xbf]
 0x00,0x80,0x07,0xbf
+# GFX12: s_cmp_lg_u32 s0, 0                      ; encoding: [0x00,0x80,0x07,0xbf]
 
-# GFX12: s_cmp_lg_u32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x07,0xbf,0x73,0x72,0x71,0x3f]
 0x00,0xff,0x07,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_lg_u32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x07,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_lg_u32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x07,0xbf,0x56,0x34,0x12,0xaf]
 0x00,0xff,0x07,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_lg_u32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x07,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_lg_u32 s0, -1                     ; encoding: [0x00,0xc1,0x07,0xbf]
 0x00,0xc1,0x07,0xbf
+# GFX12: s_cmp_lg_u32 s0, -1                     ; encoding: [0x00,0xc1,0x07,0xbf]
 
-# GFX12: s_cmp_lg_u32 s0, -4.0                   ; encoding: [0x00,0xf7,0x07,0xbf]
 0x00,0xf7,0x07,0xbf
+# GFX12: s_cmp_lg_u32 s0, -4.0                   ; encoding: [0x00,0xf7,0x07,0xbf]
 
-# GFX12: s_cmp_lg_u32 s0, exec_hi                ; encoding: [0x00,0x7f,0x07,0xbf]
 0x00,0x7f,0x07,0xbf
+# GFX12: s_cmp_lg_u32 s0, exec_hi                ; encoding: [0x00,0x7f,0x07,0xbf]
 
-# GFX12: s_cmp_lg_u32 s0, exec_lo                ; encoding: [0x00,0x7e,0x07,0xbf]
 0x00,0x7e,0x07,0xbf
+# GFX12: s_cmp_lg_u32 s0, exec_lo                ; encoding: [0x00,0x7e,0x07,0xbf]
 
-# GFX12: s_cmp_lg_u32 s0, m0                     ; encoding: [0x00,0x7d,0x07,0xbf]
 0x00,0x7d,0x07,0xbf
+# GFX12: s_cmp_lg_u32 s0, m0                     ; encoding: [0x00,0x7d,0x07,0xbf]
 
-# GFX12: s_cmp_lg_u32 s0, s104                   ; encoding: [0x00,0x68,0x07,0xbf]
 0x00,0x68,0x07,0xbf
+# GFX12: s_cmp_lg_u32 s0, s104                   ; encoding: [0x00,0x68,0x07,0xbf]
 
-# GFX12: s_cmp_lg_u32 s0, s1                     ; encoding: [0x00,0x01,0x07,0xbf]
 0x00,0x01,0x07,0xbf
+# GFX12: s_cmp_lg_u32 s0, s1                     ; encoding: [0x00,0x01,0x07,0xbf]
 
-# GFX12: s_cmp_lg_u32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x07,0xbf]
 0x00,0x6b,0x07,0xbf
+# GFX12: s_cmp_lg_u32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x07,0xbf]
 
-# GFX12: s_cmp_lg_u32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x07,0xbf]
 0x00,0x6a,0x07,0xbf
+# GFX12: s_cmp_lg_u32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x07,0xbf]
 
-# GFX12: s_cmp_lg_u32 s105, s104                 ; encoding: [0x69,0x68,0x07,0xbf]
 0x69,0x68,0x07,0xbf
+# GFX12: s_cmp_lg_u32 s105, s104                 ; encoding: [0x69,0x68,0x07,0xbf]
 
-# GFX12: s_cmp_lg_u32 s105, s1                   ; encoding: [0x69,0x01,0x07,0xbf]
 0x69,0x01,0x07,0xbf
+# GFX12: s_cmp_lg_u32 s105, s1                   ; encoding: [0x69,0x01,0x07,0xbf]
 
-# GFX12: s_cmp_lg_u32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x07,0xbf]
 0x6b,0x01,0x07,0xbf
+# GFX12: s_cmp_lg_u32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x07,0xbf]
 
-# GFX12: s_cmp_lg_u32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x07,0xbf]
 0x6a,0x01,0x07,0xbf
+# GFX12: s_cmp_lg_u32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x07,0xbf]
 
-# GFX12: s_cmp_lg_u64 exec, s[2:3]               ; encoding: [0x7e,0x02,0x11,0xbf]
 0x7e,0x02,0x11,0xbf
+# GFX12: s_cmp_lg_u64 exec, s[2:3]               ; encoding: [0x7e,0x02,0x11,0xbf]
 
-# GFX12: s_cmp_lg_u64 s[0:1], 0.5                ; encoding: [0x00,0xf0,0x11,0xbf]
 0x00,0xf0,0x11,0xbf
+# GFX12: s_cmp_lg_u64 s[0:1], 0.5                ; encoding: [0x00,0xf0,0x11,0xbf]
 
-# GFX12: s_cmp_lg_u64 s[0:1], 0                  ; encoding: [0x00,0x80,0x11,0xbf]
 0x00,0x80,0x11,0xbf
+# GFX12: s_cmp_lg_u64 s[0:1], 0                  ; encoding: [0x00,0x80,0x11,0xbf]
 
-# GFX12: s_cmp_lg_u64 s[0:1], 0x3f717273         ; encoding: [0x00,0xff,0x11,0xbf,0x73,0x72,0x71,0x3f]
 0x00,0xff,0x11,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_lg_u64 s[0:1], 0x3f717273         ; encoding: [0x00,0xff,0x11,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_lg_u64 s[0:1], 0xaf123456         ; encoding: [0x00,0xff,0x11,0xbf,0x56,0x34,0x12,0xaf]
 0x00,0xff,0x11,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_lg_u64 s[0:1], 0xaf123456         ; encoding: [0x00,0xff,0x11,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_lg_u64 s[0:1], -1                 ; encoding: [0x00,0xc1,0x11,0xbf]
 0x00,0xc1,0x11,0xbf
+# GFX12: s_cmp_lg_u64 s[0:1], -1                 ; encoding: [0x00,0xc1,0x11,0xbf]
 
-# GFX12: s_cmp_lg_u64 s[0:1], -4.0               ; encoding: [0x00,0xf7,0x11,0xbf]
 0x00,0xf7,0x11,0xbf
+# GFX12: s_cmp_lg_u64 s[0:1], -4.0               ; encoding: [0x00,0xf7,0x11,0xbf]
 
-# GFX12: s_cmp_lg_u64 s[0:1], exec               ; encoding: [0x00,0x7e,0x11,0xbf]
 0x00,0x7e,0x11,0xbf
+# GFX12: s_cmp_lg_u64 s[0:1], exec               ; encoding: [0x00,0x7e,0x11,0xbf]
 
-# GFX12: s_cmp_lg_u64 s[0:1], s[102:103]         ; encoding: [0x00,0x66,0x11,0xbf]
 0x00,0x66,0x11,0xbf
+# GFX12: s_cmp_lg_u64 s[0:1], s[102:103]         ; encoding: [0x00,0x66,0x11,0xbf]
 
-# GFX12: s_cmp_lg_u64 s[0:1], s[2:3]             ; encoding: [0x00,0x02,0x11,0xbf]
 0x00,0x02,0x11,0xbf
+# GFX12: s_cmp_lg_u64 s[0:1], s[2:3]             ; encoding: [0x00,0x02,0x11,0xbf]
 
-# GFX12: s_cmp_lg_u64 s[0:1], vcc                ; encoding: [0x00,0x6a,0x11,0xbf]
 0x00,0x6a,0x11,0xbf
+# GFX12: s_cmp_lg_u64 s[0:1], vcc                ; encoding: [0x00,0x6a,0x11,0xbf]
 
-# GFX12: s_cmp_lg_u64 s[104:105], s[102:103]     ; encoding: [0x68,0x66,0x11,0xbf]
 0x68,0x66,0x11,0xbf
+# GFX12: s_cmp_lg_u64 s[104:105], s[102:103]     ; encoding: [0x68,0x66,0x11,0xbf]
 
-# GFX12: s_cmp_lg_u64 s[104:105], s[2:3]         ; encoding: [0x68,0x02,0x11,0xbf]
 0x68,0x02,0x11,0xbf
+# GFX12: s_cmp_lg_u64 s[104:105], s[2:3]         ; encoding: [0x68,0x02,0x11,0xbf]
 
-# GFX12: s_cmp_lg_u64 vcc, s[2:3]                ; encoding: [0x6a,0x02,0x11,0xbf]
 0x6a,0x02,0x11,0xbf
+# GFX12: s_cmp_lg_u64 vcc, s[2:3]                ; encoding: [0x6a,0x02,0x11,0xbf]
 
-# GFX12: s_cmp_lt_i32 exec_hi, s1                ; encoding: [0x7f,0x01,0x04,0xbf]
 0x7f,0x01,0x04,0xbf
+# GFX12: s_cmp_lt_i32 exec_hi, s1                ; encoding: [0x7f,0x01,0x04,0xbf]
 
-# GFX12: s_cmp_lt_i32 exec_lo, s1                ; encoding: [0x7e,0x01,0x04,0xbf]
 0x7e,0x01,0x04,0xbf
+# GFX12: s_cmp_lt_i32 exec_lo, s1                ; encoding: [0x7e,0x01,0x04,0xbf]
 
-# GFX12: s_cmp_lt_i32 m0, s1                     ; encoding: [0x7d,0x01,0x04,0xbf]
 0x7d,0x01,0x04,0xbf
+# GFX12: s_cmp_lt_i32 m0, s1                     ; encoding: [0x7d,0x01,0x04,0xbf]
 
-# GFX12: s_cmp_lt_i32 s0, 0.5                    ; encoding: [0x00,0xf0,0x04,0xbf]
 0x00,0xf0,0x04,0xbf
+# GFX12: s_cmp_lt_i32 s0, 0.5                    ; encoding: [0x00,0xf0,0x04,0xbf]
 
-# GFX12: s_cmp_lt_i32 s0, 0                      ; encoding: [0x00,0x80,0x04,0xbf]
 0x00,0x80,0x04,0xbf
+# GFX12: s_cmp_lt_i32 s0, 0                      ; encoding: [0x00,0x80,0x04,0xbf]
 
-# GFX12: s_cmp_lt_i32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x04,0xbf,0x73,0x72,0x71,0x3f]
 0x00,0xff,0x04,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_lt_i32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x04,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_lt_i32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x04,0xbf,0x56,0x34,0x12,0xaf]
 0x00,0xff,0x04,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_lt_i32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x04,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_lt_i32 s0, -1                     ; encoding: [0x00,0xc1,0x04,0xbf]
 0x00,0xc1,0x04,0xbf
+# GFX12: s_cmp_lt_i32 s0, -1                     ; encoding: [0x00,0xc1,0x04,0xbf]
 
-# GFX12: s_cmp_lt_i32 s0, -4.0                   ; encoding: [0x00,0xf7,0x04,0xbf]
 0x00,0xf7,0x04,0xbf
+# GFX12: s_cmp_lt_i32 s0, -4.0                   ; encoding: [0x00,0xf7,0x04,0xbf]
 
-# GFX12: s_cmp_lt_i32 s0, exec_hi                ; encoding: [0x00,0x7f,0x04,0xbf]
 0x00,0x7f,0x04,0xbf
+# GFX12: s_cmp_lt_i32 s0, exec_hi                ; encoding: [0x00,0x7f,0x04,0xbf]
 
-# GFX12: s_cmp_lt_i32 s0, exec_lo                ; encoding: [0x00,0x7e,0x04,0xbf]
 0x00,0x7e,0x04,0xbf
+# GFX12: s_cmp_lt_i32 s0, exec_lo                ; encoding: [0x00,0x7e,0x04,0xbf]
 
-# GFX12: s_cmp_lt_i32 s0, m0                     ; encoding: [0x00,0x7d,0x04,0xbf]
 0x00,0x7d,0x04,0xbf
+# GFX12: s_cmp_lt_i32 s0, m0                     ; encoding: [0x00,0x7d,0x04,0xbf]
 
-# GFX12: s_cmp_lt_i32 s0, s104                   ; encoding: [0x00,0x68,0x04,0xbf]
 0x00,0x68,0x04,0xbf
+# GFX12: s_cmp_lt_i32 s0, s104                   ; encoding: [0x00,0x68,0x04,0xbf]
 
-# GFX12: s_cmp_lt_i32 s0, s1                     ; encoding: [0x00,0x01,0x04,0xbf]
 0x00,0x01,0x04,0xbf
+# GFX12: s_cmp_lt_i32 s0, s1                     ; encoding: [0x00,0x01,0x04,0xbf]
 
-# GFX12: s_cmp_lt_i32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x04,0xbf]
 0x00,0x6b,0x04,0xbf
+# GFX12: s_cmp_lt_i32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x04,0xbf]
 
-# GFX12: s_cmp_lt_i32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x04,0xbf]
 0x00,0x6a,0x04,0xbf
+# GFX12: s_cmp_lt_i32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x04,0xbf]
 
-# GFX12: s_cmp_lt_i32 s105, s104                 ; encoding: [0x69,0x68,0x04,0xbf]
 0x69,0x68,0x04,0xbf
+# GFX12: s_cmp_lt_i32 s105, s104                 ; encoding: [0x69,0x68,0x04,0xbf]
 
-# GFX12: s_cmp_lt_i32 s105, s1                   ; encoding: [0x69,0x01,0x04,0xbf]
 0x69,0x01,0x04,0xbf
+# GFX12: s_cmp_lt_i32 s105, s1                   ; encoding: [0x69,0x01,0x04,0xbf]
 
-# GFX12: s_cmp_lt_i32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x04,0xbf]
 0x6b,0x01,0x04,0xbf
+# GFX12: s_cmp_lt_i32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x04,0xbf]
 
-# GFX12: s_cmp_lt_i32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x04,0xbf]
 0x6a,0x01,0x04,0xbf
+# GFX12: s_cmp_lt_i32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x04,0xbf]
 
-# GFX12: s_cmp_lt_u32 exec_hi, s1                ; encoding: [0x7f,0x01,0x0a,0xbf]
 0x7f,0x01,0x0a,0xbf
+# GFX12: s_cmp_lt_u32 exec_hi, s1                ; encoding: [0x7f,0x01,0x0a,0xbf]
 
-# GFX12: s_cmp_lt_u32 exec_lo, s1                ; encoding: [0x7e,0x01,0x0a,0xbf]
 0x7e,0x01,0x0a,0xbf
+# GFX12: s_cmp_lt_u32 exec_lo, s1                ; encoding: [0x7e,0x01,0x0a,0xbf]
 
-# GFX12: s_cmp_lt_u32 m0, s1                     ; encoding: [0x7d,0x01,0x0a,0xbf]
 0x7d,0x01,0x0a,0xbf
+# GFX12: s_cmp_lt_u32 m0, s1                     ; encoding: [0x7d,0x01,0x0a,0xbf]
 
-# GFX12: s_cmp_lt_u32 s0, 0.5                    ; encoding: [0x00,0xf0,0x0a,0xbf]
 0x00,0xf0,0x0a,0xbf
+# GFX12: s_cmp_lt_u32 s0, 0.5                    ; encoding: [0x00,0xf0,0x0a,0xbf]
 
-# GFX12: s_cmp_lt_u32 s0, 0                      ; encoding: [0x00,0x80,0x0a,0xbf]
 0x00,0x80,0x0a,0xbf
+# GFX12: s_cmp_lt_u32 s0, 0                      ; encoding: [0x00,0x80,0x0a,0xbf]
 
-# GFX12: s_cmp_lt_u32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x0a,0xbf,0x73,0x72,0x71,0x3f]
 0x00,0xff,0x0a,0xbf,0x73,0x72,0x71,0x3f
+# GFX12: s_cmp_lt_u32 s0, 0x3f717273             ; encoding: [0x00,0xff,0x0a,0xbf,0x73,0x72,0x71,0x3f]
 
-# GFX12: s_cmp_lt_u32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x0a,0xbf,0x56,0x34,0x12,0xaf]
 0x00,0xff,0x0a,0xbf,0x56,0x34,0x12,0xaf
+# GFX12: s_cmp_lt_u32 s0, 0xaf123456             ; encoding: [0x00,0xff,0x0a,0xbf,0x56,0x34,0x12,0xaf]
 
-# GFX12: s_cmp_lt_u32 s0, -1                     ; encoding: [0x00,0xc1,0x0a,0xbf]
 0x00,0xc1,0x0a,0xbf
+# GFX12: s_cmp_lt_u32 s0, -1                     ; encoding: [0x00,0xc1,0x0a,0xbf]
 
-# GFX12: s_cmp_lt_u32 s0, -4.0                   ; encoding: [0x00,0xf7,0x0a,0xbf]
 0x00,0xf7,0x0a,0xbf
+# GFX12: s_cmp_lt_u32 s0, -4.0                   ; encoding: [0x00,0xf7,0x0a,0xbf]
 
-# GFX12: s_cmp_lt_u32 s0, exec_hi                ; encoding: [0x00,0x7f,0x0a,0xbf]
 0x00,0x7f,0x0a,0xbf
+# GFX12: s_cmp_lt_u32 s0, exec_hi                ; encoding: [0x00,0x7f,0x0a,0xbf]
 
-# GFX12: s_cmp_lt_u32 s0, exec_lo                ; encoding: [0x00,0x7e,0x0a,0xbf]
 0x00,0x7e,0x0a,0xbf
+# GFX12: s_cmp_lt_u32 s0, exec_lo                ; encoding: [0x00,0x7e,0x0a,0xbf]
 
-# GFX12: s_cmp_lt_u32 s0, m0                     ; encoding: [0x00,0x7d,0x0a,0xbf]
 0x00,0x7d,0x0a,0xbf
+# GFX12: s_cmp_lt_u32 s0, m0                     ; encoding: [0x00,0x7d,0x0a,0xbf]
 
-# GFX12: s_cmp_lt_u32 s0, s104                   ; encoding: [0x00,0x68,0x0a,0xbf]
 0x00,0x68,0x0a,0xbf
+# GFX12: s_cmp_lt_u32 s0, s104                   ; encoding: [0x00,0x68,0x0a,0xbf]
 
-# GFX12: s_cmp_lt_u32 s0, s1                     ; encoding: [0x00,0x01,0x0a,0xbf]
 0x00,0x01,0x0a,0xbf
+# GFX12: s_cmp_lt_u32 s0, s1                     ; encoding: [0x00,0x01,0x0a,0xbf]
 
-# GFX12: s_cmp_lt_u32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x0a,0xbf]
 0x00,0x6b,0x0a,0xbf
+# GFX12: s_cmp_lt_u32 s0, vcc_hi                 ; encoding: [0x00,0x6b,0x0a,0xbf]
 
-# GFX12: s_cmp_lt_u32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x0a,0xbf]
 0x00,0x6a,0x0a,0xbf
+# GFX12: s_cmp_lt_u32 s0, vcc_lo                 ; encoding: [0x00,0x6a,0x0a,0xbf]
 
-# GFX12: s_cmp_lt_u32 s105, s104                 ; encoding: [0x69,0x68,0x0a,0xbf]
 0x69,0x68,0x0a,0xbf
+# GFX12: s_cmp_lt_u32 s105, s104                 ; encoding: [0x69,0x68,0x0a,0xbf]
 
-# GFX12: s_cmp_lt_u32 s105, s1                   ; encoding: [0x69,0x01,0x0a,0xbf]
 0x69,0x01,0x0a,0xbf
+# GFX12: s_cmp_lt_u32 s105, s1                   ; encoding: [0x69,0x01,0x0a,0xbf]
 
-# GFX12: s_cmp_lt_u32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x0a,0xbf]
 0x6b,0x01,0x0a,0xbf
+# GFX12: s_cmp_lt_u32 vcc_hi, s1                 ; encoding: [0x6b,0x01,0x0a,0xbf]
 
-# GFX12: s_cmp_lt_u32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x0a,0xbf]
 0x6a,0x01,0x0a,0xbf
+# GFX12: s_cmp_lt_u32 vcc_lo, s1                 ; encoding: [0x6a,0x01,0x0a,0xbf]
diff --git a/llvm/test/MC/Hexagon/align-leb128.s b/llvm/test/MC/Hexagon/align-leb128.s
new file mode 100644
index 0000000000000..77018f0114311
--- /dev/null
+++ b/llvm/test/MC/Hexagon/align-leb128.s
@@ -0,0 +1,18 @@
+# RUN: llvm-mc -triple=hexagon -filetype=obj %s | llvm-readelf -x .data - \
+# RUN:   | FileCheck %s --match-full-lines
+
+# Illustrate the case when padding packets across labels also breaks leb128
+# relocations. This happens because .align padding is inserted once at the
+# very end of the section layout.
+L1:
+  nop
+L2:
+.size L1, L2-L1
+.align 16
+  nop
+.data
+.word L2-L1
+.uleb128 L2-L1
+
+# CHECK: Hex dump of section '.data':
+# CHECK-NEXT: 0x00000000 04000000 04 .....
diff --git a/llvm/test/MC/Hexagon/align.s b/llvm/test/MC/Hexagon/align.s
index 9c2978df71373..e17d09cfd8c96 100644
--- a/llvm/test/MC/Hexagon/align.s
+++ b/llvm/test/MC/Hexagon/align.s
@@ -58,3 +58,16 @@ r0 = vextract(v0, r0)
   r1 = sub (##1, r1) }
 .align 16
 { r0 = sub (#1, r0) }
+
+# Don't search backwards to pad packets beyond a label:
+{ r1 = add(r1, r0) }
+# CHECK-NEXT: { r1 = add(r1,r0)
+# CHECK-NOT:  nop
+
+post_label:
+.align 16
+# CHECK-LABEL: post_label
+# CHECK-NEXT: { nop
+# CHECK-NEXT:   nop }
+# CHECK-NEXT: { r1 = sub(#1,r1) }
+{ r1 = sub(#1, r1) }
diff --git a/llvm/test/MC/Xtensa/xtensa-mac16.s b/llvm/test/MC/Xtensa/xtensa-mac16.s
new file mode 100644
index 0000000000000..15200751ec8a0
--- /dev/null
+++ b/llvm/test/MC/Xtensa/xtensa-mac16.s
@@ -0,0 +1,233 @@
+# RUN: llvm-mc %s -triple=xtensa -mattr=+mac16 -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-INST %s
+
+.align	4
+LBL0:
+
+# CHECK-INST: umul.aa.ll	a2, a3
+# CHECK: encoding: [0x34,0x02,0x70]
+	umul.aa.ll	a2, a3
+# CHECK-INST: umul.aa.lh	a2, a3
+# CHECK: encoding: [0x34,0x02,0x72]
+	umul.aa.lh	a2, a3
+# CHECK-INST: umul.aa.hl	a2, a3
+# CHECK: encoding: [0x34,0x02,0x71]
+	umul.aa.hl	a2, a3
+# CHECK-INST: umul.aa.hh	a2, a3
+# CHECK: encoding: [0x34,0x02,0x73]
+	umul.aa.hh	a2, a3
+
+# CHECK-INST: mul.aa.ll	a2, a3
+# CHECK: encoding: [0x34,0x02,0x74]
+	mul.aa.ll	a2, a3
+# CHECK-INST: mul.aa.lh	a2, a3
+# CHECK: encoding: [0x34,0x02,0x76]
+	mul.aa.lh	a2, a3
+# CHECK-INST: mul.aa.hl	a2, a3
+# CHECK: encoding: [0x34,0x02,0x75]
+	mul.aa.hl	a2, a3
+# CHECK-INST: mul.aa.hh	a2, a3
+# CHECK: encoding: [0x34,0x02,0x77]
+	mul.aa.hh	a2, a3
+
+# CHECK-INST: mul.ad.ll	a2, m2
+# CHECK: encoding: [0x04,0x02,0x34]
+	mul.ad.ll	a2, m2
+# CHECK-INST: mul.ad.lh	a2, m2
+# CHECK: encoding: [0x04,0x02,0x36]
+	mul.ad.lh	a2, m2
+# CHECK-INST: mul.ad.hl	a2, m2
+# CHECK: encoding: [0x04,0x02,0x35]
+	mul.ad.hl	a2, m2
+# CHECK-INST: mul.ad.hh	a2, m2
+# CHECK: encoding: [0x04,0x02,0x37]
+	mul.ad.hh	a2, m2
+
+# CHECK-INST: mul.da.ll	m1, a3
+# CHECK: encoding: [0x34,0x40,0x64]
+	mul.da.ll	m1, a3
+# CHECK-INST: mul.da.lh	m1, a3
+# CHECK: encoding: [0x34,0x40,0x66]
+	mul.da.lh	m1, a3
+# CHECK-INST: mul.da.hl	m1, a3
+# CHECK: encoding: [0x34,0x40,0x65]
+	mul.da.hl	m1, a3
+# CHECK-INST: mul.da.hh	m1, a3
+# CHECK: encoding: [0x34,0x40,0x67]
+	mul.da.hh	m1, a3
+
+# CHECK-INST: mul.dd.ll	m1, m2
+# CHECK: encoding: [0x04,0x40,0x24]
+	mul.dd.ll	m1, m2
+# CHECK-INST: mul.dd.lh	m1, m2
+# CHECK: encoding: [0x04,0x40,0x26]
+	mul.dd.lh	m1, m2
+# CHECK-INST: mul.dd.hl	m1, m2
+# CHECK: encoding: [0x04,0x40,0x25]
+	mul.dd.hl	m1, m2
+# CHECK-INST: mul.dd.hh	m1, m2
+# CHECK: encoding: [0x04,0x40,0x27]
+	mul.dd.hh	m1, m2
+
+# CHECK-INST: mula.aa.ll	a2, a3
+# CHECK: encoding: [0x34,0x02,0x78]
+	mula.aa.ll	a2, a3
+# CHECK-INST: mula.aa.lh	a2, a3
+# CHECK: encoding: [0x34,0x02,0x7a]
+	mula.aa.lh	a2, a3
+# CHECK-INST: mula.aa.hl	a2, a3
+# CHECK: encoding: [0x34,0x02,0x79]
+	mula.aa.hl	a2, a3
+# CHECK-INST: mula.aa.hh	a2, a3
+# CHECK: encoding: [0x34,0x02,0x7b]
+	mula.aa.hh	a2, a3
+
+# CHECK-INST: mula.ad.ll	a2, m2
+# CHECK: encoding: [0x04,0x02,0x38]
+	mula.ad.ll	a2, m2
+# CHECK-INST: mula.ad.lh	a2, m2
+# CHECK: encoding: [0x04,0x02,0x3a]
+	mula.ad.lh	a2, m2
+# CHECK-INST: mula.ad.hl	a2, m2
+# CHECK: encoding: [0x04,0x02,0x39]
+	mula.ad.hl	a2, m2
+# CHECK-INST: mula.ad.hh	a2, m2
+# CHECK: encoding: [0x04,0x02,0x3b]
+	mula.ad.hh	a2, m2
+
+# CHECK-INST: mula.da.ll	m1, a3
+# CHECK: encoding: [0x34,0x40,0x68]
+	mula.da.ll	m1, a3
+# CHECK-INST: mula.da.lh	m1, a3
+# CHECK: encoding: [0x34,0x40,0x6a]
+	mula.da.lh	m1, a3
+# CHECK-INST: mula.da.hl	m1, a3
+# CHECK: encoding: [0x34,0x40,0x69]
+	mula.da.hl	m1, a3
+# CHECK-INST: mula.da.hh	m1, a3
+# CHECK: encoding: [0x34,0x40,0x6b]
+	mula.da.hh	m1, a3
+
+# CHECK-INST: mula.dd.ll	m1, m2
+# CHECK: encoding: [0x04,0x40,0x28]
+	mula.dd.ll	m1, m2
+# CHECK-INST: mula.dd.lh	m1, m2
+# CHECK: encoding: [0x04,0x40,0x2a]
+	mula.dd.lh	m1, m2
+# CHECK-INST: mula.dd.hl	m1, m2
+# CHECK: encoding: [0x04,0x40,0x29]
+	mula.dd.hl	m1, m2
+# CHECK-INST: mula.dd.hh	m1, m2
+# CHECK: encoding: [0x04,0x40,0x2b]
+	mula.dd.hh	m1, m2
+
+# CHECK-INST: muls.aa.ll	a2, a3
+# CHECK: encoding: [0x34,0x02,0x7c]
+	muls.aa.ll	a2, a3
+# CHECK-INST: muls.aa.lh	a2, a3
+# CHECK: encoding: [0x34,0x02,0x7e]
+	muls.aa.lh	a2, a3
+# CHECK-INST: muls.aa.hl	a2, a3
+# CHECK: encoding: [0x34,0x02,0x7d]
+	muls.aa.hl	a2, a3
+# CHECK-INST: muls.aa.hh	a2, a3
+# CHECK: encoding: [0x34,0x02,0x7f]
+	muls.aa.hh	a2, a3
+
+# CHECK-INST: muls.ad.ll	a2, m2
+# CHECK: encoding: [0x04,0x02,0x3c]
+	muls.ad.ll	a2, m2
+# CHECK-INST: muls.ad.lh	a2, m2
+# CHECK: encoding: [0x04,0x02,0x3e]
+	muls.ad.lh	a2, m2
+# CHECK-INST: muls.ad.hl	a2, m2
+# CHECK: encoding: [0x04,0x02,0x3d]
+	muls.ad.hl	a2, m2
+# CHECK-INST: muls.ad.hh	a2, m2
+# CHECK: encoding: [0x04,0x02,0x3f]
+	muls.ad.hh	a2, m2
+
+# CHECK-INST: muls.da.ll	m1, a3
+# CHECK: encoding: [0x34,0x40,0x6c]
+	muls.da.ll	m1, a3
+# CHECK-INST: muls.da.lh	m1, a3
+# CHECK: encoding: [0x34,0x40,0x6e]
+	muls.da.lh	m1, a3
+# CHECK-INST: muls.da.hl	m1, a3
+# CHECK: encoding: [0x34,0x40,0x6d]
+	muls.da.hl	m1, a3
+# CHECK-INST: muls.da.hh	m1, a3
+# CHECK: encoding: [0x34,0x40,0x6f]
+	muls.da.hh	m1, a3
+
+# CHECK-INST: muls.dd.ll	m1, m2
+# CHECK: encoding: [0x04,0x40,0x2c]
+	muls.dd.ll	m1, m2
+# CHECK-INST: muls.dd.lh	m1, m2
+# CHECK: encoding: [0x04,0x40,0x2e]
+	muls.dd.lh	m1, m2
+# CHECK-INST: muls.dd.hl	m1, m2
+# CHECK: encoding: [0x04,0x40,0x2d]
+	muls.dd.hl	m1, m2
+# CHECK-INST: muls.dd.hh	m1, m2
+# CHECK: encoding: [0x04,0x40,0x2f]
+	muls.dd.hh	m1, m2
+
+# CHECK-INST: mula.da.ll.lddec	 m1, a8, m0, a3
+# CHECK: encoding: [0x34,0x18,0x58]
+	mula.da.ll.lddec	 m1, a8, m0, a3
+# CHECK-INST: mula.da.hl.lddec	 m1, a8, m0, a3
+# CHECK: encoding: [0x34,0x18,0x59]
+	mula.da.hl.lddec	 m1, a8, m0, a3
+# CHECK-INST: mula.da.lh.lddec	 m1, a8, m0, a3
+# CHECK: encoding: [0x34,0x18,0x5a]
+	mula.da.lh.lddec	 m1, a8, m0, a3
+# CHECK-INST: mula.da.hh.lddec	 m1, a8, m0, a3
+# CHECK: encoding: [0x34,0x18,0x5b]
+	mula.da.hh.lddec	 m1, a8, m0, a3
+
+# CHECK-INST: mula.dd.ll.lddec	 m1, a8, m0, m2
+# CHECK: encoding: [0x04,0x18,0x18]
+	mula.dd.ll.lddec	 m1, a8, m0, m2
+# CHECK-INST: mula.dd.hl.lddec	 m1, a8, m0, m2
+# CHECK: encoding: [0x04,0x18,0x19]
+	mula.dd.hl.lddec	 m1, a8, m0, m2
+# CHECK-INST: mula.dd.lh.lddec	 m1, a8, m0, m2
+# CHECK: encoding: [0x04,0x18,0x1a]
+	mula.dd.lh.lddec	 m1, a8, m0, m2
+# CHECK-INST: mula.dd.hh.lddec	 m1, a8, m0, m2
+# CHECK: encoding: [0x04,0x18,0x1b]
+	mula.dd.hh.lddec	 m1, a8, m0, m2
+
+# CHECK-INST: mula.da.ll.ldinc	 m1, a8, m0, a3
+# CHECK: encoding: [0x34,0x18,0x48]
+	mula.da.ll.ldinc	 m1, a8, m0, a3
+# CHECK-INST: mula.da.hl.ldinc	 m1, a8, m0, a3
+# CHECK: encoding: [0x34,0x18,0x49]
+	mula.da.hl.ldinc	 m1, a8, m0, a3
+# CHECK-INST: mula.da.lh.ldinc	 m1, a8, m0, a3
+# CHECK: encoding: [0x34,0x18,0x4a]
+	mula.da.lh.ldinc	 m1, a8, m0, a3
+# CHECK-INST: mula.da.hh.ldinc	 m1, a8, m0, a3
+# CHECK: encoding: [0x34,0x18,0x4b]
+	mula.da.hh.ldinc	 m1, a8, m0, a3
+
+# CHECK-INST: mula.dd.ll.ldinc	 m1, a8, m0, m2
+# CHECK: encoding: [0x04,0x18,0x08]
+	mula.dd.ll.ldinc	 m1, a8, m0, m2
+# CHECK-INST: mula.dd.hl.ldinc	 m1, a8, m0, m2
+# CHECK: encoding: [0x04,0x18,0x09]
+	mula.dd.hl.ldinc	 m1, a8, m0, m2
+# CHECK-INST: mula.dd.lh.ldinc	 m1, a8, m0, m2
+# CHECK: encoding: [0x04,0x18,0x0a]
+	mula.dd.lh.ldinc	 m1, a8, m0, m2
+# CHECK-INST: mula.dd.hh.ldinc	 m1, a8, m0, m2
+# CHECK: encoding: [0x04,0x18,0x0b]
+	mula.dd.hh.ldinc	 m1, a8, m0, m2
+
+# CHECK-INST: lddec	 m0, a8
+# CHECK: encoding: [0x04,0x08,0x90]
+	lddec	 m0, a8
+# CHECK-INST: ldinc	 m0, a8
+# CHECK: encoding: [0x04,0x08,0x80]
+	ldinc	 m0, a8
diff --git a/llvm/test/MachineVerifier/AMDGPU/verify-av-mov-imm-pseudo.mir b/llvm/test/MachineVerifier/AMDGPU/verify-av-mov-imm-pseudo.mir
new file mode 100644
index 0000000000000..8973de09d9674
--- /dev/null
+++ b/llvm/test/MachineVerifier/AMDGPU/verify-av-mov-imm-pseudo.mir
@@ -0,0 +1,15 @@
+# RUN: not --crash llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -run-pass=none -o /dev/null %s 2>&1 | FileCheck %s
+---
+name: invalid_av_mov_b32_imm_pseudo
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %0:vgpr_32 = IMPLICIT_DEF
+
+    ; CHECK: *** Bad machine code: Illegal immediate value for operand. ***
+    $agpr0 = AV_MOV_B32_IMM_PSEUDO 65, implicit $exec
+
+    ; CHECK: *** Bad machine code: Illegal immediate value for operand. ***
+    $vgpr0 = AV_MOV_B32_IMM_PSEUDO 65, implicit $exec
+
+...
diff --git a/llvm/test/Other/print-prof-data.ll b/llvm/test/Other/print-prof-data.ll
new file mode 100644
index 0000000000000..a5a2d444222b5
--- /dev/null
+++ b/llvm/test/Other/print-prof-data.ll
@@ -0,0 +1,24 @@
+; RUN: opt %s -print-prof-data -S | FileCheck %s
+
+define void @foo(ptr %p) !prof !0 {
+  %isnull = icmp eq ptr %p, null
+  br i1 %isnull, label %yes, label %no, !prof !1
+yes:
+  %something = select i1 %isnull, i32 1, i32 2, !prof !2
+  br label %exit
+no:
+  call void %p(), !prof !3
+  br label %exit
+exit:
+  ret void
+}
+
+!0 = !{!"function_entry_count", i64 42}
+!1 = !{!"branch_weights", i64 20, i64 101}
+!2 = !{!"branch_weights", i64 5, i64 70}
+!3 = !{!"VP", i32 0, i64 4, i64 4445083295448962937, i64 2, i64 -2718743882639408571, i64 2}
+
+; CHECK: define void @foo(ptr %p) !0 = !{!"function_entry_count", i64 42} !prof !0 {
+; CHECK: br i1 %isnull, label %yes, label %no, !prof !1 ; !1 = !{!"branch_weights", i64 20, i64 101}
+; CHECK: %something = select i1 %isnull, i32 1, i32 2, !prof !2 ; !2 = !{!"branch_weights", i64 5, i64 70}
+; CHECK: call void %p(), !prof !3 ; !3 = !{!"VP", i32 0, i64 4, i64 4445083295448962937, i64 2, i64 -2718743882639408571, i64 2}
\ No newline at end of file
diff --git a/llvm/test/TableGen/x86-instr-mapping.inc b/llvm/test/TableGen/x86-instr-mapping.inc
index 4f64d4b8d93d0..1629e0eef93ec 100644
--- a/llvm/test/TableGen/x86-instr-mapping.inc
+++ b/llvm/test/TableGen/x86-instr-mapping.inc
@@ -1575,29 +1575,29 @@ static bool checkPredicate(unsigned Opc, const X86Subtarget *Subtarget) {
 
 #ifdef GET_X86_NF_TRANSFORM_TABLE
 static const X86TableEntry X86NFTransformTable[] = {
+  { X86::ADD16mi, X86::ADD16mi_NF },
   { X86::ADD16mi8, X86::ADD16mi8_NF },
   { X86::ADD16mi8_ND, X86::ADD16mi8_NF_ND },
-  { X86::ADD16mi, X86::ADD16mi_NF },
   { X86::ADD16mi_ND, X86::ADD16mi_NF_ND },
   { X86::ADD16mr, X86::ADD16mr_NF },
   { X86::ADD16mr_ND, X86::ADD16mr_NF_ND },
+  { X86::ADD16ri, X86::ADD16ri_NF },
   { X86::ADD16ri8, X86::ADD16ri8_NF },
   { X86::ADD16ri8_ND, X86::ADD16ri8_NF_ND },
-  { X86::ADD16ri, X86::ADD16ri_NF },
   { X86::ADD16ri_ND, X86::ADD16ri_NF_ND },
   { X86::ADD16rm, X86::ADD16rm_NF },
   { X86::ADD16rm_ND, X86::ADD16rm_NF_ND },
   { X86::ADD16rr, X86::ADD16rr_NF },
   { X86::ADD16rr_ND, X86::ADD16rr_NF_ND },
+  { X86::ADD32mi, X86::ADD32mi_NF },
   { X86::ADD32mi8, X86::ADD32mi8_NF },
   { X86::ADD32mi8_ND, X86::ADD32mi8_NF_ND },
-  { X86::ADD32mi, X86::ADD32mi_NF },
   { X86::ADD32mi_ND, X86::ADD32mi_NF_ND },
   { X86::ADD32mr, X86::ADD32mr_NF },
   { X86::ADD32mr_ND, X86::ADD32mr_NF_ND },
+  { X86::ADD32ri, X86::ADD32ri_NF },
   { X86::ADD32ri8, X86::ADD32ri8_NF },
   { X86::ADD32ri8_ND, X86::ADD32ri8_NF_ND },
-  { X86::ADD32ri, X86::ADD32ri_NF },
   { X86::ADD32ri_ND, X86::ADD32ri_NF_ND },
   { X86::ADD32rm, X86::ADD32rm_NF },
   { X86::ADD32rm_ND, X86::ADD32rm_NF_ND },
@@ -1627,29 +1627,29 @@ static const X86TableEntry X86NFTransformTable[] = {
   { X86::ADD8rm_ND, X86::ADD8rm_NF_ND },
   { X86::ADD8rr, X86::ADD8rr_NF },
   { X86::ADD8rr_ND, X86::ADD8rr_NF_ND },
+  { X86::AND16mi, X86::AND16mi_NF },
   { X86::AND16mi8, X86::AND16mi8_NF },
   { X86::AND16mi8_ND, X86::AND16mi8_NF_ND },
-  { X86::AND16mi, X86::AND16mi_NF },
   { X86::AND16mi_ND, X86::AND16mi_NF_ND },
   { X86::AND16mr, X86::AND16mr_NF },
   { X86::AND16mr_ND, X86::AND16mr_NF_ND },
+  { X86::AND16ri, X86::AND16ri_NF },
   { X86::AND16ri8, X86::AND16ri8_NF },
   { X86::AND16ri8_ND, X86::AND16ri8_NF_ND },
-  { X86::AND16ri, X86::AND16ri_NF },
   { X86::AND16ri_ND, X86::AND16ri_NF_ND },
   { X86::AND16rm, X86::AND16rm_NF },
   { X86::AND16rm_ND, X86::AND16rm_NF_ND },
   { X86::AND16rr, X86::AND16rr_NF },
   { X86::AND16rr_ND, X86::AND16rr_NF_ND },
+  { X86::AND32mi, X86::AND32mi_NF },
   { X86::AND32mi8, X86::AND32mi8_NF },
   { X86::AND32mi8_ND, X86::AND32mi8_NF_ND },
-  { X86::AND32mi, X86::AND32mi_NF },
   { X86::AND32mi_ND, X86::AND32mi_NF_ND },
   { X86::AND32mr, X86::AND32mr_NF },
   { X86::AND32mr_ND, X86::AND32mr_NF_ND },
+  { X86::AND32ri, X86::AND32ri_NF },
   { X86::AND32ri8, X86::AND32ri8_NF },
   { X86::AND32ri8_ND, X86::AND32ri8_NF_ND },
-  { X86::AND32ri, X86::AND32ri_NF },
   { X86::AND32ri_ND, X86::AND32ri_NF_ND },
   { X86::AND32rm, X86::AND32rm_NF },
   { X86::AND32rm_ND, X86::AND32rm_NF_ND },
@@ -1739,22 +1739,22 @@ static const X86TableEntry X86NFTransformTable[] = {
   { X86::IMUL16r, X86::IMUL16r_NF },
   { X86::IMUL16rm, X86::IMUL16rm_NF },
   { X86::IMUL16rm_ND, X86::IMUL16rm_NF_ND },
-  { X86::IMUL16rmi8, X86::IMUL16rmi8_NF },
   { X86::IMUL16rmi, X86::IMUL16rmi_NF },
+  { X86::IMUL16rmi8, X86::IMUL16rmi8_NF },
   { X86::IMUL16rr, X86::IMUL16rr_NF },
   { X86::IMUL16rr_ND, X86::IMUL16rr_NF_ND },
-  { X86::IMUL16rri8, X86::IMUL16rri8_NF },
   { X86::IMUL16rri, X86::IMUL16rri_NF },
+  { X86::IMUL16rri8, X86::IMUL16rri8_NF },
   { X86::IMUL32m, X86::IMUL32m_NF },
   { X86::IMUL32r, X86::IMUL32r_NF },
   { X86::IMUL32rm, X86::IMUL32rm_NF },
   { X86::IMUL32rm_ND, X86::IMUL32rm_NF_ND },
-  { X86::IMUL32rmi8, X86::IMUL32rmi8_NF },
   { X86::IMUL32rmi, X86::IMUL32rmi_NF },
+  { X86::IMUL32rmi8, X86::IMUL32rmi8_NF },
   { X86::IMUL32rr, X86::IMUL32rr_NF },
   { X86::IMUL32rr_ND, X86::IMUL32rr_NF_ND },
-  { X86::IMUL32rri8, X86::IMUL32rri8_NF },
   { X86::IMUL32rri, X86::IMUL32rri_NF },
+  { X86::IMUL32rri8, X86::IMUL32rri8_NF },
   { X86::IMUL64m, X86::IMUL64m_NF },
   { X86::IMUL64r, X86::IMUL64r_NF },
   { X86::IMUL64rm, X86::IMUL64rm_NF },
@@ -1813,29 +1813,29 @@ static const X86TableEntry X86NFTransformTable[] = {
   { X86::NEG8m_ND, X86::NEG8m_NF_ND },
   { X86::NEG8r, X86::NEG8r_NF },
   { X86::NEG8r_ND, X86::NEG8r_NF_ND },
+  { X86::OR16mi, X86::OR16mi_NF },
   { X86::OR16mi8, X86::OR16mi8_NF },
   { X86::OR16mi8_ND, X86::OR16mi8_NF_ND },
-  { X86::OR16mi, X86::OR16mi_NF },
   { X86::OR16mi_ND, X86::OR16mi_NF_ND },
   { X86::OR16mr, X86::OR16mr_NF },
   { X86::OR16mr_ND, X86::OR16mr_NF_ND },
+  { X86::OR16ri, X86::OR16ri_NF },
   { X86::OR16ri8, X86::OR16ri8_NF },
   { X86::OR16ri8_ND, X86::OR16ri8_NF_ND },
-  { X86::OR16ri, X86::OR16ri_NF },
   { X86::OR16ri_ND, X86::OR16ri_NF_ND },
   { X86::OR16rm, X86::OR16rm_NF },
   { X86::OR16rm_ND, X86::OR16rm_NF_ND },
   { X86::OR16rr, X86::OR16rr_NF },
   { X86::OR16rr_ND, X86::OR16rr_NF_ND },
+  { X86::OR32mi, X86::OR32mi_NF },
   { X86::OR32mi8, X86::OR32mi8_NF },
   { X86::OR32mi8_ND, X86::OR32mi8_NF_ND },
-  { X86::OR32mi, X86::OR32mi_NF },
   { X86::OR32mi_ND, X86::OR32mi_NF_ND },
   { X86::OR32mr, X86::OR32mr_NF },
   { X86::OR32mr_ND, X86::OR32mr_NF_ND },
+  { X86::OR32ri, X86::OR32ri_NF },
   { X86::OR32ri8, X86::OR32ri8_NF },
   { X86::OR32ri8_ND, X86::OR32ri8_NF_ND },
-  { X86::OR32ri, X86::OR32ri_NF },
   { X86::OR32ri_ND, X86::OR32ri_NF_ND },
   { X86::OR32rm, X86::OR32rm_NF },
   { X86::OR32rm_ND, X86::OR32rm_NF_ND },
@@ -2159,29 +2159,29 @@ static const X86TableEntry X86NFTransformTable[] = {
   { X86::SHRD64rrCL_ND, X86::SHRD64rrCL_NF_ND },
   { X86::SHRD64rri8, X86::SHRD64rri8_NF },
   { X86::SHRD64rri8_ND, X86::SHRD64rri8_NF_ND },
+  { X86::SUB16mi, X86::SUB16mi_NF },
   { X86::SUB16mi8, X86::SUB16mi8_NF },
   { X86::SUB16mi8_ND, X86::SUB16mi8_NF_ND },
-  { X86::SUB16mi, X86::SUB16mi_NF },
   { X86::SUB16mi_ND, X86::SUB16mi_NF_ND },
   { X86::SUB16mr, X86::SUB16mr_NF },
   { X86::SUB16mr_ND, X86::SUB16mr_NF_ND },
+  { X86::SUB16ri, X86::SUB16ri_NF },
   { X86::SUB16ri8, X86::SUB16ri8_NF },
   { X86::SUB16ri8_ND, X86::SUB16ri8_NF_ND },
-  { X86::SUB16ri, X86::SUB16ri_NF },
   { X86::SUB16ri_ND, X86::SUB16ri_NF_ND },
   { X86::SUB16rm, X86::SUB16rm_NF },
   { X86::SUB16rm_ND, X86::SUB16rm_NF_ND },
   { X86::SUB16rr, X86::SUB16rr_NF },
   { X86::SUB16rr_ND, X86::SUB16rr_NF_ND },
+  { X86::SUB32mi, X86::SUB32mi_NF },
   { X86::SUB32mi8, X86::SUB32mi8_NF },
   { X86::SUB32mi8_ND, X86::SUB32mi8_NF_ND },
-  { X86::SUB32mi, X86::SUB32mi_NF },
   { X86::SUB32mi_ND, X86::SUB32mi_NF_ND },
   { X86::SUB32mr, X86::SUB32mr_NF },
   { X86::SUB32mr_ND, X86::SUB32mr_NF_ND },
+  { X86::SUB32ri, X86::SUB32ri_NF },
   { X86::SUB32ri8, X86::SUB32ri8_NF },
   { X86::SUB32ri8_ND, X86::SUB32ri8_NF_ND },
-  { X86::SUB32ri, X86::SUB32ri_NF },
   { X86::SUB32ri_ND, X86::SUB32ri_NF_ND },
   { X86::SUB32rm, X86::SUB32rm_NF },
   { X86::SUB32rm_ND, X86::SUB32rm_NF_ND },
@@ -2217,29 +2217,29 @@ static const X86TableEntry X86NFTransformTable[] = {
   { X86::TZCNT32rr, X86::TZCNT32rr_NF },
   { X86::TZCNT64rm, X86::TZCNT64rm_NF },
   { X86::TZCNT64rr, X86::TZCNT64rr_NF },
+  { X86::XOR16mi, X86::XOR16mi_NF },
   { X86::XOR16mi8, X86::XOR16mi8_NF },
   { X86::XOR16mi8_ND, X86::XOR16mi8_NF_ND },
-  { X86::XOR16mi, X86::XOR16mi_NF },
   { X86::XOR16mi_ND, X86::XOR16mi_NF_ND },
   { X86::XOR16mr, X86::XOR16mr_NF },
   { X86::XOR16mr_ND, X86::XOR16mr_NF_ND },
+  { X86::XOR16ri, X86::XOR16ri_NF },
   { X86::XOR16ri8, X86::XOR16ri8_NF },
   { X86::XOR16ri8_ND, X86::XOR16ri8_NF_ND },
-  { X86::XOR16ri, X86::XOR16ri_NF },
   { X86::XOR16ri_ND, X86::XOR16ri_NF_ND },
   { X86::XOR16rm, X86::XOR16rm_NF },
   { X86::XOR16rm_ND, X86::XOR16rm_NF_ND },
   { X86::XOR16rr, X86::XOR16rr_NF },
   { X86::XOR16rr_ND, X86::XOR16rr_NF_ND },
+  { X86::XOR32mi, X86::XOR32mi_NF },
   { X86::XOR32mi8, X86::XOR32mi8_NF },
   { X86::XOR32mi8_ND, X86::XOR32mi8_NF_ND },
-  { X86::XOR32mi, X86::XOR32mi_NF },
   { X86::XOR32mi_ND, X86::XOR32mi_NF_ND },
   { X86::XOR32mr, X86::XOR32mr_NF },
   { X86::XOR32mr_ND, X86::XOR32mr_NF_ND },
+  { X86::XOR32ri, X86::XOR32ri_NF },
   { X86::XOR32ri8, X86::XOR32ri8_NF },
   { X86::XOR32ri8_ND, X86::XOR32ri8_NF_ND },
-  { X86::XOR32ri, X86::XOR32ri_NF },
   { X86::XOR32ri_ND, X86::XOR32ri_NF_ND },
   { X86::XOR32rm, X86::XOR32rm_NF },
   { X86::XOR32rm_ND, X86::XOR32rm_NF_ND },
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll b/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
index 5fa8f4e1b9f29..7fb1e0718795d 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
@@ -339,7 +339,7 @@ define i1 @test12(i32 %x) {
 ; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[X:%.*]] to i64
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i64 [[ZEXT]], 7
 ; CHECK-NEXT:    [[SHR:%.*]] = lshr i64 [[MUL]], 32
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i64 [[SHR]] to i32
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc nuw nsw i64 [[SHR]] to i32
 ; CHECK-NEXT:    ret i1 true
 ;
   %zext = zext i32 %x to i64
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/range.ll b/llvm/test/Transforms/CorrelatedValuePropagation/range.ll
index 03d71fa9b5277..caf1156d79229 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/range.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/range.ll
@@ -937,7 +937,7 @@ entry:
 define i1 @ctlz_with_range_metadata(i16 %x) {
 ; CHECK-LABEL: @ctlz_with_range_metadata(
 ; CHECK-NEXT:    [[CTLZ:%.*]] = call i16 @llvm.ctlz.i16(i16 [[X:%.*]], i1 false), !range [[RNG5:![0-9]+]]
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i16 [[CTLZ]] to i8
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc nuw nsw i16 [[CTLZ]] to i8
 ; CHECK-NEXT:    ret i1 true
 ;
   %ctlz = call i16 @llvm.ctlz.i16(i16 %x, i1 false), !range !{i16 0, i16 8}
@@ -949,7 +949,7 @@ define i1 @ctlz_with_range_metadata(i16 %x) {
 define i1 @abs_with_range_metadata(i16 %x) {
 ; CHECK-LABEL: @abs_with_range_metadata(
 ; CHECK-NEXT:    [[ABS:%.*]] = call i16 @llvm.abs.i16(i16 [[X:%.*]], i1 false), !range [[RNG5]]
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i16 [[ABS]] to i8
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc nuw nsw i16 [[ABS]] to i8
 ; CHECK-NEXT:    ret i1 true
 ;
   %abs = call i16 @llvm.abs.i16(i16 %x, i1 false), !range !{i16 0, i16 8}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/sext.ll b/llvm/test/Transforms/CorrelatedValuePropagation/sext.ll
index 0db520bfc68a2..648ef0864bf17 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/sext.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/sext.ll
@@ -20,7 +20,7 @@ define void @test1(i32 %n) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[EXT_WIDE1:%.*]] = zext nneg i32 [[A]] to i64
 ; CHECK-NEXT:    call void @use64(i64 [[EXT_WIDE1]])
-; CHECK-NEXT:    [[EXT]] = trunc i64 [[EXT_WIDE1]] to i32
+; CHECK-NEXT:    [[EXT]] = trunc nuw nsw i64 [[EXT_WIDE1]] to i32
 ; CHECK-NEXT:    br label [[FOR_COND]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
@@ -55,7 +55,7 @@ define void @test2(i32 %n) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[EXT_WIDE:%.*]] = sext i32 [[A]] to i64
 ; CHECK-NEXT:    call void @use64(i64 [[EXT_WIDE]])
-; CHECK-NEXT:    [[EXT]] = trunc i64 [[EXT_WIDE]] to i32
+; CHECK-NEXT:    [[EXT]] = trunc nsw i64 [[EXT_WIDE]] to i32
 ; CHECK-NEXT:    br label [[FOR_COND]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
@@ -87,7 +87,7 @@ define void @test3(i32 %n) {
 ; CHECK:       bb:
 ; CHECK-NEXT:    [[EXT_WIDE1:%.*]] = zext nneg i32 [[N]] to i64
 ; CHECK-NEXT:    call void @use64(i64 [[EXT_WIDE1]])
-; CHECK-NEXT:    [[EXT:%.*]] = trunc i64 [[EXT_WIDE1]] to i32
+; CHECK-NEXT:    [[EXT:%.*]] = trunc nuw nsw i64 [[EXT_WIDE1]] to i32
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
@@ -115,7 +115,7 @@ define void @test4(i32 %n) {
 ; CHECK:       bb:
 ; CHECK-NEXT:    [[EXT_WIDE:%.*]] = sext i32 [[N]] to i64
 ; CHECK-NEXT:    call void @use64(i64 [[EXT_WIDE]])
-; CHECK-NEXT:    [[EXT:%.*]] = trunc i64 [[EXT_WIDE]] to i32
+; CHECK-NEXT:    [[EXT:%.*]] = trunc nsw i64 [[EXT_WIDE]] to i32
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/trunc.ll b/llvm/test/Transforms/CorrelatedValuePropagation/trunc.ll
new file mode 100644
index 0000000000000..9b6604298840d
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/trunc.ll
@@ -0,0 +1,108 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=correlated-propagation -S | FileCheck %s
+
+
+define i1 @infer_nuw(i8 range(i8 0, 2) %A, i8 range(i8 0, 2) %B) {
+; CHECK-LABEL: define i1 @infer_nuw(
+; CHECK-SAME: i8 range(i8 0, 2) [[A:%.*]], i8 range(i8 0, 2) [[B:%.*]]) {
+; CHECK-NEXT:    [[XOR:%.*]] = xor i8 [[B]], [[A]]
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc nuw i8 [[XOR]] to i1
+; CHECK-NEXT:    ret i1 [[TRUNC]]
+;
+  %xor = xor i8 %B, %A
+  %trunc = trunc i8 %xor to i1
+  ret i1 %trunc
+}
+
+
+define i4 @infer_nsw(i8 %A) {
+; CHECK-LABEL: define range(i4 -4, 4) i4 @infer_nsw(
+; CHECK-SAME: i8 [[A:%.*]]) {
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr i8 [[A]], 5
+; CHECK-NEXT:    [[B:%.*]] = trunc nsw i8 [[ASHR]] to i4
+; CHECK-NEXT:    ret i4 [[B]]
+;
+  %ashr = ashr i8 %A, 5
+  %result = trunc i8 %ashr to i4
+  ret i4 %result
+}
+
+
+define i8 @infer_nuw_nsw(i16 range(i16 -5, -3) %A, i16 range(i16 -5, -3) %B) {
+; CHECK-LABEL: define range(i8 0, 8) i8 @infer_nuw_nsw(
+; CHECK-SAME: i16 range(i16 -5, -3) [[A:%.*]], i16 range(i16 -5, -3) [[B:%.*]]) {
+; CHECK-NEXT:    [[XOR:%.*]] = xor i16 [[B]], [[A]]
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc nuw nsw i16 [[XOR]] to i8
+; CHECK-NEXT:    ret i8 [[TRUNC]]
+;
+  %xor = xor i16 %B, %A
+  %trunc = trunc i16 %xor to i8
+  ret i8 %trunc
+}
+
+
+define i8 @infer_nsw_from_assume(i16 %x) {
+; CHECK-LABEL: define i8 @infer_nsw_from_assume(
+; CHECK-SAME: i16 [[X:%.*]]) {
+; CHECK-NEXT:    [[ADD:%.*]] = add i16 [[X]], 128
+; CHECK-NEXT:    [[OR_COND_I:%.*]] = icmp ult i16 [[ADD]], 256
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[OR_COND_I]])
+; CHECK-NEXT:    [[CONV1:%.*]] = trunc nsw i16 [[X]] to i8
+; CHECK-NEXT:    ret i8 [[CONV1]]
+;
+  %add = add i16 %x, 128
+  %or.cond.i = icmp ult i16 %add, 256
+  tail call void @llvm.assume(i1 %or.cond.i)
+  %conv1 = trunc i16 %x to i8
+  ret i8 %conv1
+}
+
+
+define i1 @rust_issue_122734(i8 range(i8 0, 3) %A, i8 range(i8 0, 3) %B) {
+; CHECK-LABEL: define i1 @rust_issue_122734(
+; CHECK-SAME: i8 range(i8 0, 3) [[A:%.*]], i8 range(i8 0, 3) [[B:%.*]]) {
+; CHECK-NEXT:  [[START:.*]]:
+; CHECK-NEXT:    [[LHS:%.*]] = icmp eq i8 [[A]], 2
+; CHECK-NEXT:    [[RHS:%.*]] = icmp eq i8 [[B]], 2
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[LHS]], [[RHS]]
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[LHS]], [[RHS]]
+; CHECK-NEXT:    br i1 [[OR]], label %[[IFTRUE:.*]], label %[[IFFALSE:.*]]
+; CHECK:       [[IFTRUE]]:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i1 [ [[XOR2:%.*]], %[[IFFALSE]] ], [ [[AND]], %[[START]] ]
+; CHECK-NEXT:    ret i1 [[PHI]]
+; CHECK:       [[IFFALSE]]:
+; CHECK-NEXT:    [[XOR:%.*]] = xor i8 [[A]], [[B]]
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc nuw i8 [[XOR]] to i1
+; CHECK-NEXT:    [[XOR2]] = xor i1 [[TRUNC]], true
+; CHECK-NEXT:    br label %[[IFTRUE]]
+;
+start:
+  %lhs = icmp eq i8 %A, 2
+  %rhs = icmp eq i8 %B, 2
+  %or = or i1 %lhs, %rhs
+  %and = and i1 %lhs, %rhs
+  br i1 %or, label %iftrue, label %iffalse
+
+iftrue:
+  %phi = phi i1 [ %xor2, %iffalse], [ %and, %start ]
+  ret i1 %phi
+
+iffalse:
+  %xor = xor i8 %A, %B
+  %trunc = trunc i8 %xor to i1
+  %xor2 = xor i1 %trunc, true
+  br label %iftrue
+}
+
+
+define i1 @overdefined_range_negative(i8 %A, i8 %B) {
+; CHECK-LABEL: define i1 @overdefined_range_negative(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[XOR:%.*]] = xor i8 [[A]], [[B]]
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[XOR]] to i1
+; CHECK-NEXT:    ret i1 [[TRUNC]]
+;
+  %xor = xor i8 %A, %B
+  %trunc = trunc i8 %xor to i1
+  ret i1 %trunc
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/zext.ll b/llvm/test/Transforms/CorrelatedValuePropagation/zext.ll
index 44434c696fe37..8e43244a8eb62 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/zext.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/zext.ll
@@ -14,7 +14,7 @@ define void @test1(i32 %n) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[EXT_WIDE:%.*]] = zext nneg i32 [[A]] to i64
 ; CHECK-NEXT:    call void @use64(i64 [[EXT_WIDE]])
-; CHECK-NEXT:    [[EXT]] = trunc i64 [[EXT_WIDE]] to i32
+; CHECK-NEXT:    [[EXT]] = trunc nuw nsw i64 [[EXT_WIDE]] to i32
 ; CHECK-NEXT:    br label [[FOR_COND]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
@@ -49,7 +49,7 @@ define void @test2(i32 %n) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[EXT_WIDE:%.*]] = zext i32 [[A]] to i64
 ; CHECK-NEXT:    call void @use64(i64 [[EXT_WIDE]])
-; CHECK-NEXT:    [[EXT]] = trunc i64 [[EXT_WIDE]] to i32
+; CHECK-NEXT:    [[EXT]] = trunc nuw i64 [[EXT_WIDE]] to i32
 ; CHECK-NEXT:    br label [[FOR_COND]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
@@ -81,7 +81,7 @@ define void @test3(i32 %n) {
 ; CHECK:       bb:
 ; CHECK-NEXT:    [[EXT_WIDE:%.*]] = zext nneg i32 [[N]] to i64
 ; CHECK-NEXT:    call void @use64(i64 [[EXT_WIDE]])
-; CHECK-NEXT:    [[EXT:%.*]] = trunc i64 [[EXT_WIDE]] to i32
+; CHECK-NEXT:    [[EXT:%.*]] = trunc nuw nsw i64 [[EXT_WIDE]] to i32
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
@@ -109,7 +109,7 @@ define void @test4(i32 %n) {
 ; CHECK:       bb:
 ; CHECK-NEXT:    [[EXT_WIDE:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    call void @use64(i64 [[EXT_WIDE]])
-; CHECK-NEXT:    [[EXT:%.*]] = trunc i64 [[EXT_WIDE]] to i32
+; CHECK-NEXT:    [[EXT:%.*]] = trunc nuw i64 [[EXT_WIDE]] to i32
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/phi-with-incoming-from-load.ll b/llvm/test/Transforms/InstCombine/AMDGPU/phi-with-incoming-from-load.ll
new file mode 100644
index 0000000000000..14fb45e43af86
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/phi-with-incoming-from-load.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=instcombine -S -o - %s | FileCheck %s
+
+target triple = "amdgcn-amd-amdhsa"
+
+%double_double = type { double, double }
+
+declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1)
+
+define void @_test(ptr addrspace(4) byref(%double_double) align 8 %in)  {
+; CHECK-LABEL: define void @_test(
+; CHECK-SAME: ptr addrspace(4) byref([[DOUBLE_DOUBLE:%.*]]) align 8 [[IN:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[ALPHA_UNION:%.*]] = addrspacecast ptr addrspace(4) [[IN]] to ptr
+; CHECK-NEXT:    [[LOAD:%.*]] = load i8, ptr addrspace(5) null, align 1
+; CHECK-NEXT:    [[LOADEDV:%.*]] = trunc i8 [[LOAD]] to i1
+; CHECK-NEXT:    br i1 [[LOADEDV]], label %[[COND_END:.*]], label %[[COND_FALSE:.*]]
+; CHECK:       [[COND_FALSE]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr addrspace(4) [[IN]], align 8
+; CHECK-NEXT:    br label %[[COND_END]]
+; CHECK:       [[COND_END]]:
+; CHECK-NEXT:    [[COND1:%.*]] = phi ptr [ [[TMP0]], %[[COND_FALSE]] ], [ [[ALPHA_UNION]], %[[ENTRY]] ]
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(16) poison, ptr noundef nonnull align 1 dereferenceable(16) [[COND1]], i64 16, i1 false)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %coerce = alloca %double_double, align 8, addrspace(5)
+  %alpha_union = addrspacecast ptr addrspace(5) %coerce to ptr
+  call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) align 8 %coerce, ptr addrspace(4) align 8 %in, i64 16, i1 false)
+  %load1 = load i8, ptr addrspace(5) null, align 1
+  %loadedv = trunc i8 %load1 to i1
+  br i1 %loadedv, label %cond.end, label %cond.false
+
+cond.false:
+  %load2 = load ptr, ptr addrspace(5) %coerce, align 8
+  br label %cond.end
+
+cond.end:
+  %cond = phi ptr [ %load2, %cond.false ], [ %alpha_union, %entry ]
+  call void @llvm.memcpy.p0.p0.i64(ptr poison, ptr %cond, i64 16, i1 false)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/simplify-demanded-vector-elts-lane-intrinsics.ll b/llvm/test/Transforms/InstCombine/AMDGPU/simplify-demanded-vector-elts-lane-intrinsics.ll
index e9d3b5e963b35..056caabb6d60a 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/simplify-demanded-vector-elts-lane-intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/simplify-demanded-vector-elts-lane-intrinsics.ll
@@ -64,8 +64,8 @@ define i16 @extract_elt2_v4i16_readfirstlane(<4 x i16> %src) {
 define <2 x i16> @extract_elt01_v4i16_readfirstlane(<4 x i16> %src) {
 ; CHECK-LABEL: define <2 x i16> @extract_elt01_v4i16_readfirstlane(
 ; CHECK-SAME: <4 x i16> [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[VEC:%.*]] = call <4 x i16> @llvm.amdgcn.readfirstlane.v4i16(<4 x i16> [[SRC]])
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[VEC]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[SRC]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = call <2 x i16> @llvm.amdgcn.readfirstlane.v2i16(<2 x i16> [[TMP1]])
 ; CHECK-NEXT:    ret <2 x i16> [[SHUFFLE]]
 ;
   %vec = call <4 x i16> @llvm.amdgcn.readfirstlane.v4i16(<4 x i16> %src)
@@ -76,8 +76,8 @@ define <2 x i16> @extract_elt01_v4i16_readfirstlane(<4 x i16> %src) {
 define <2 x i16> @extract_elt12_v4i16_readfirstlane(<4 x i16> %src) {
 ; CHECK-LABEL: define <2 x i16> @extract_elt12_v4i16_readfirstlane(
 ; CHECK-SAME: <4 x i16> [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[VEC:%.*]] = call <4 x i16> @llvm.amdgcn.readfirstlane.v4i16(<4 x i16> [[SRC]])
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[VEC]], <4 x i16> poison, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[SRC]], <4 x i16> poison, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = call <2 x i16> @llvm.amdgcn.readfirstlane.v2i16(<2 x i16> [[TMP1]])
 ; CHECK-NEXT:    ret <2 x i16> [[SHUFFLE]]
 ;
   %vec = call <4 x i16> @llvm.amdgcn.readfirstlane.v4i16(<4 x i16> %src)
@@ -88,8 +88,8 @@ define <2 x i16> @extract_elt12_v4i16_readfirstlane(<4 x i16> %src) {
 define <2 x i16> @extract_elt23_v4i16_readfirstlane(<4 x i16> %src) {
 ; CHECK-LABEL: define <2 x i16> @extract_elt23_v4i16_readfirstlane(
 ; CHECK-SAME: <4 x i16> [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[VEC:%.*]] = call <4 x i16> @llvm.amdgcn.readfirstlane.v4i16(<4 x i16> [[SRC]])
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[VEC]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[SRC]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = call <2 x i16> @llvm.amdgcn.readfirstlane.v2i16(<2 x i16> [[TMP1]])
 ; CHECK-NEXT:    ret <2 x i16> [[SHUFFLE]]
 ;
   %vec = call <4 x i16> @llvm.amdgcn.readfirstlane.v4i16(<4 x i16> %src)
@@ -100,8 +100,9 @@ define <2 x i16> @extract_elt23_v4i16_readfirstlane(<4 x i16> %src) {
 define <2 x i16> @extract_elt10_v4i16_readfirstlane(<4 x i16> %src) {
 ; CHECK-LABEL: define <2 x i16> @extract_elt10_v4i16_readfirstlane(
 ; CHECK-SAME: <4 x i16> [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[VEC:%.*]] = call <4 x i16> @llvm.amdgcn.readfirstlane.v4i16(<4 x i16> [[SRC]])
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[VEC]], <4 x i16> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[SRC]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.amdgcn.readfirstlane.v2i16(<2 x i16> [[TMP1]])
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <2 x i32> <i32 1, i32 0>
 ; CHECK-NEXT:    ret <2 x i16> [[SHUFFLE]]
 ;
   %vec = call <4 x i16> @llvm.amdgcn.readfirstlane.v4i16(<4 x i16> %src)
@@ -112,7 +113,9 @@ define <2 x i16> @extract_elt10_v4i16_readfirstlane(<4 x i16> %src) {
 define <2 x i16> @extract_elt32_v4i16_readfirstlane(<4 x i16> %src) {
 ; CHECK-LABEL: define <2 x i16> @extract_elt32_v4i16_readfirstlane(
 ; CHECK-SAME: <4 x i16> [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[VEC:%.*]] = call <4 x i16> @llvm.amdgcn.readfirstlane.v4i16(<4 x i16> [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[SRC]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.amdgcn.readfirstlane.v2i16(<2 x i16> [[TMP1]])
+; CHECK-NEXT:    [[VEC:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <4 x i32> <i32 poison, i32 poison, i32 0, i32 1>
 ; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[VEC]], <4 x i16> poison, <2 x i32> <i32 3, i32 2>
 ; CHECK-NEXT:    ret <2 x i16> [[SHUFFLE]]
 ;
@@ -258,8 +261,8 @@ define <3 x i16> @extract_elt123_v4i16_readfirstlane(<4 x i16> %src) {
 define <3 x i32> @extract_elt012_v4i32_readfirstlane(<4 x i32> %src) {
 ; CHECK-LABEL: define <3 x i32> @extract_elt012_v4i32_readfirstlane(
 ; CHECK-SAME: <4 x i32> [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[VEC:%.*]] = call <4 x i32> @llvm.amdgcn.readfirstlane.v4i32(<4 x i32> [[SRC]])
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[VEC]], <4 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[SRC]], <4 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = call <3 x i32> @llvm.amdgcn.readfirstlane.v3i32(<3 x i32> [[TMP1]])
 ; CHECK-NEXT:    ret <3 x i32> [[SHUFFLE]]
 ;
   %vec = call <4 x i32> @llvm.amdgcn.readfirstlane.v4i32(<4 x i32> %src)
@@ -270,8 +273,8 @@ define <3 x i32> @extract_elt012_v4i32_readfirstlane(<4 x i32> %src) {
 define <3 x i32> @extract_elt123_v4i32_readfirstlane(<4 x i32> %src) {
 ; CHECK-LABEL: define <3 x i32> @extract_elt123_v4i32_readfirstlane(
 ; CHECK-SAME: <4 x i32> [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[VEC:%.*]] = call <4 x i32> @llvm.amdgcn.readfirstlane.v4i32(<4 x i32> [[SRC]])
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[VEC]], <4 x i32> poison, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[SRC]], <4 x i32> poison, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = call <3 x i32> @llvm.amdgcn.readfirstlane.v3i32(<3 x i32> [[TMP1]])
 ; CHECK-NEXT:    ret <3 x i32> [[SHUFFLE]]
 ;
   %vec = call <4 x i32> @llvm.amdgcn.readfirstlane.v4i32(<4 x i32> %src)
@@ -282,7 +285,9 @@ define <3 x i32> @extract_elt123_v4i32_readfirstlane(<4 x i32> %src) {
 define <2 x i32> @extract_elt13_v4i32_readfirstlane(<4 x i32> %src) {
 ; CHECK-LABEL: define <2 x i32> @extract_elt13_v4i32_readfirstlane(
 ; CHECK-SAME: <4 x i32> [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[VEC:%.*]] = call <4 x i32> @llvm.amdgcn.readfirstlane.v4i32(<4 x i32> [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[SRC]], <4 x i32> poison, <3 x i32> <i32 1, i32 poison, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <3 x i32> @llvm.amdgcn.readfirstlane.v3i32(<3 x i32> [[TMP1]])
+; CHECK-NEXT:    [[VEC:%.*]] = shufflevector <3 x i32> [[TMP2]], <3 x i32> poison, <4 x i32> <i32 poison, i32 0, i32 poison, i32 2>
 ; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[VEC]], <4 x i32> poison, <2 x i32> <i32 1, i32 3>
 ; CHECK-NEXT:    ret <2 x i32> [[SHUFFLE]]
 ;
@@ -321,8 +326,9 @@ define < 2 x i32> @extract_elt13_v4i32_readfirstlane_source_simplify1(i32 %src0,
 ; CHECK-LABEL: define <2 x i32> @extract_elt13_v4i32_readfirstlane_source_simplify1(
 ; CHECK-SAME: i32 [[SRC0:%.*]], i32 [[SRC2:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[SRC0]], i64 0
-; CHECK-NEXT:    [[INS_1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 poison, i32 0, i32 poison, i32 0>
-; CHECK-NEXT:    [[VEC:%.*]] = call <4 x i32> @llvm.amdgcn.readfirstlane.v4i32(<4 x i32> [[INS_1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 0>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <3 x i32> @llvm.amdgcn.readfirstlane.v3i32(<3 x i32> [[TMP2]])
+; CHECK-NEXT:    [[VEC:%.*]] = shufflevector <3 x i32> [[TMP3]], <3 x i32> poison, <4 x i32> <i32 poison, i32 0, i32 poison, i32 2>
 ; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[VEC]], <4 x i32> poison, <2 x i32> <i32 1, i32 3>
 ; CHECK-NEXT:    ret <2 x i32> [[SHUFFLE]]
 ;
@@ -364,8 +370,9 @@ define < 2 x i32> @extract_elt13_v4i32_readfirstlane_source_simplify1_convergenc
 ; CHECK-SAME: i32 [[SRC0:%.*]], i32 [[SRC2:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[T:%.*]] = call token @llvm.experimental.convergence.entry()
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[SRC0]], i64 0
-; CHECK-NEXT:    [[INS_1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 poison, i32 0, i32 poison, i32 0>
-; CHECK-NEXT:    [[VEC:%.*]] = call <4 x i32> @llvm.amdgcn.readfirstlane.v4i32(<4 x i32> [[INS_1]]) [ "convergencectrl"(token [[T]]) ]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 0>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <3 x i32> @llvm.amdgcn.readfirstlane.v3i32(<3 x i32> [[TMP2]]) [ "convergencectrl"(token [[T]]) ]
+; CHECK-NEXT:    [[VEC:%.*]] = shufflevector <3 x i32> [[TMP3]], <3 x i32> poison, <4 x i32> <i32 poison, i32 0, i32 poison, i32 2>
 ; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[VEC]], <4 x i32> poison, <2 x i32> <i32 1, i32 3>
 ; CHECK-NEXT:    ret <2 x i32> [[SHUFFLE]]
 ;
@@ -404,7 +411,9 @@ define <2 x i1> @extract_elt01_v4i1_readfirstlane(<4 x i1> %src) {
 define <2 x i32> @extract_elt13_v8i32_readfirstlane(<8 x i32> %src) {
 ; CHECK-LABEL: define <2 x i32> @extract_elt13_v8i32_readfirstlane(
 ; CHECK-SAME: <8 x i32> [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[VEC:%.*]] = call <8 x i32> @llvm.amdgcn.readfirstlane.v8i32(<8 x i32> [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[SRC]], <8 x i32> poison, <3 x i32> <i32 1, i32 poison, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <3 x i32> @llvm.amdgcn.readfirstlane.v3i32(<3 x i32> [[TMP1]])
+; CHECK-NEXT:    [[VEC:%.*]] = shufflevector <3 x i32> [[TMP2]], <3 x i32> poison, <8 x i32> <i32 poison, i32 0, i32 poison, i32 2, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i32> [[VEC]], <8 x i32> poison, <2 x i32> <i32 1, i32 3>
 ; CHECK-NEXT:    ret <2 x i32> [[SHUFFLE]]
 ;
@@ -428,7 +437,9 @@ define <2 x i32> @extract_elt03_v4i32_readfirstlane(<4 x i32> %src) {
 define <3 x i32> @extract_elt124_v8i32_readfirstlane(<8 x i32> %src) {
 ; CHECK-LABEL: define <3 x i32> @extract_elt124_v8i32_readfirstlane(
 ; CHECK-SAME: <8 x i32> [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[VEC:%.*]] = call <8 x i32> @llvm.amdgcn.readfirstlane.v8i32(<8 x i32> [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[SRC]], <8 x i32> poison, <4 x i32> <i32 1, i32 2, i32 poison, i32 4>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.amdgcn.readfirstlane.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT:    [[VEC:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 poison, i32 0, i32 1, i32 poison, i32 3, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i32> [[VEC]], <8 x i32> poison, <3 x i32> <i32 1, i32 2, i32 4>
 ; CHECK-NEXT:    ret <3 x i32> [[SHUFFLE]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/iX-ext-split.ll b/llvm/test/Transforms/InstCombine/iX-ext-split.ll
index b88b62aa04375..fc804df0e4bec 100644
--- a/llvm/test/Transforms/InstCombine/iX-ext-split.ll
+++ b/llvm/test/Transforms/InstCombine/iX-ext-split.ll
@@ -89,6 +89,27 @@ entry:
   ret void
 }
 
+; (non)ext split from i64 to i128
+define void @i128_ext_split_store_i64(i64 %x, ptr %out)  {
+; CHECK-LABEL: define void @i128_ext_split_store_i64(
+; CHECK-SAME: i64 [[X:%.*]], ptr [[OUT:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[RES:%.*]] = sext i64 [[X]] to i128
+; CHECK-NEXT:    store i128 [[RES]], ptr [[OUT]], align 16
+; CHECK-NEXT:    ret void
+;
+entry:
+  %lo = zext i64 %x to i128
+
+  %sign = ashr i64 %x, 63
+  %widen = zext i64 %sign to i128
+  %hi = shl nuw i128 %widen, 64
+
+  %res = or disjoint i128 %hi, %lo
+  store i128 %res, ptr %out, align 16
+  ret void
+}
+
 ; negative test - wrong constant value
 define i128 @i128_ext_split_neg1(i32 %x) {
 ; CHECK-LABEL: define i128 @i128_ext_split_neg1(
diff --git a/llvm/test/Transforms/InstSimplify/ConstProp/inttoptr-gep-index-width.ll b/llvm/test/Transforms/InstSimplify/ConstProp/inttoptr-gep-index-width.ll
new file mode 100644
index 0000000000000..03056e8361e21
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/ConstProp/inttoptr-gep-index-width.ll
@@ -0,0 +1,14 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=instsimplify < %s | FileCheck %s
+
+target datalayout = "p:16:16:16:8"
+
+; The GEP should only modify the low 8 bits of the pointer.
+define ptr @test() {
+; CHECK-LABEL: define ptr @test() {
+; CHECK-NEXT:    ret ptr inttoptr (i16 -256 to ptr)
+;
+  %base = inttoptr i16 -1 to ptr
+  %gep = getelementptr i8, ptr %base, i8 1
+  ret ptr %gep
+}
diff --git a/llvm/test/Transforms/InstSimplify/fcmp.ll b/llvm/test/Transforms/InstSimplify/fcmp.ll
index 64132f5fb7db7..0c2be5210a741 100644
--- a/llvm/test/Transforms/InstSimplify/fcmp.ll
+++ b/llvm/test/Transforms/InstSimplify/fcmp.ll
@@ -16,3 +16,20 @@ define i1 @poison2(float %x) {
   %v = fcmp ueq float %x, poison
   ret i1 %v
 }
+
+define i1 @pr130408(x86_fp80 %x) {
+; CHECK-LABEL: @pr130408(
+; CHECK-NEXT:    [[BITS:%.*]] = bitcast x86_fp80 [[X:%.*]] to i80
+; CHECK-NEXT:    [[MASKED:%.*]] = and i80 [[BITS]], -604444463063240877801473
+; CHECK-NEXT:    [[OR:%.*]] = or i80 [[MASKED]], 302194561415509874573312
+; CHECK-NEXT:    [[FP:%.*]] = bitcast i80 [[OR]] to x86_fp80
+; CHECK-NEXT:    [[RES:%.*]] = fcmp uno x86_fp80 [[FP]], 0xK00000000000000000000
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %bits = bitcast x86_fp80 %x to i80
+  %masked = and i80 %bits, -604444463063240877801473
+  %or = or i80 %masked, 302194561415509874573312
+  %fp = bitcast i80 %or to x86_fp80
+  %res = fcmp uno x86_fp80 %fp, 0xK00000000000000000000
+  ret i1 %res
+}
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll
index 0ba812214c948..b109448bd5d7c 100644
--- a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll
@@ -71,9 +71,9 @@ define void @store_factor2(ptr %ptr, <16 x i16> %v0, <16 x i16> %v1) #0 {
 ; CHECK-SAME: ptr [[PTR:%.*]], <16 x i16> [[V0:%.*]], <16 x i16> [[V1:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[V0]], <16 x i16> [[V1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> poison, <16 x i16> [[TMP2]], i64 0)
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[V0]], <16 x i16> [[V1]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> poison, <16 x i16> [[TMP4]], i64 0)
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv8i16(<vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i1> [[TMP1]], ptr [[PTR]])
 ; CHECK-NEXT:    ret void
 ;
@@ -90,11 +90,11 @@ define void @store_factor3(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2
 ; CHECK-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v8i32(<vscale x 4 x i32> undef, <8 x i32> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v8i32(<vscale x 4 x i32> poison, <8 x i32> [[TMP2]], i64 0)
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v8i32(<vscale x 4 x i32> undef, <8 x i32> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v8i32(<vscale x 4 x i32> poison, <8 x i32> [[TMP4]], i64 0)
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v8i32(<vscale x 4 x i32> undef, <8 x i32> [[TMP6]], i64 0)
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v8i32(<vscale x 4 x i32> poison, <8 x i32> [[TMP6]], i64 0)
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st3.nxv4i32(<vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i1> [[TMP1]], ptr [[PTR]])
 ; CHECK-NEXT:    ret void
 ;
@@ -115,13 +115,13 @@ define void @store_factor4(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2
 ; CHECK-NEXT:    [[S1:%.*]] = shufflevector <4 x i64> [[V2]], <4 x i64> [[V3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP2]], i64 0)
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP4]], i64 0)
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP6]], i64 0)
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP6]], i64 0)
 ; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP8]], i64 0)
+; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP8]], i64 0)
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st4.nxv2i64(<vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
 ; CHECK-NEXT:    ret void
 ;
@@ -208,9 +208,9 @@ define void @store_ptrvec_factor2(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint <4 x ptr> [[V1]] to <4 x i64>
 ; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP4]], i64 0)
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP6]], i64 0)
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP6]], i64 0)
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP7]], <vscale x 2 x i1> [[TMP3]], ptr [[PTR]])
 ; CHECK-NEXT:    ret void
 ;
@@ -228,11 +228,11 @@ define void @store_ptrvec_factor3(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1, <4 x p
 ; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint <8 x ptr> [[S1]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP4]], i64 0)
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP6]], i64 0)
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP6]], i64 0)
 ; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP8]], i64 0)
+; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP8]], i64 0)
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st3.nxv2i64(<vscale x 2 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i1> [[TMP3]], ptr [[PTR]])
 ; CHECK-NEXT:    ret void
 ;
@@ -252,13 +252,13 @@ define void @store_ptrvec_factor4(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1, <4 x p
 ; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint <8 x ptr> [[S1]] to <8 x i64>
 ; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP4]], i64 0)
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP6]], i64 0)
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP6]], i64 0)
 ; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP8]], i64 0)
+; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP8]], i64 0)
 ; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP11:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP10]], i64 0)
+; CHECK-NEXT:    [[TMP11:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP10]], i64 0)
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st4.nxv2i64(<vscale x 2 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i1> [[TMP3]], ptr [[PTR]])
 ; CHECK-NEXT:    ret void
 ;
@@ -300,14 +300,14 @@ define void @store_factor2_wide(ptr %ptr, <8 x i64> %v0, <8 x i64> %v1) #0 {
 ; CHECK-SAME: ptr [[PTR:%.*]], <8 x i64> [[V0:%.*]], <8 x i64> [[V1:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i64> [[V0]], <8 x i64> [[V1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP2]], i64 0)
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i64> [[V0]], <8 x i64> [[V1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP4]], i64 0)
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP5]], <vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i64> [[V0]], <8 x i64> [[V1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP6]], i64 0)
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP6]], i64 0)
 ; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <8 x i64> [[V0]], <8 x i64> [[V1]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP8]], i64 0)
+; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP8]], i64 0)
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i64, ptr [[PTR]], i32 8
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i1> [[TMP1]], ptr [[TMP10]])
 ; CHECK-NEXT:    ret void
@@ -384,9 +384,9 @@ define void @store_min_not_max(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1) #1 {
 ; CHECK-SAME: ptr [[PTR:%.*]], <4 x i64> [[V0:%.*]], <4 x i64> [[V1:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 4)
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[V0]], <4 x i64> [[V1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP2]], i64 0)
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[V0]], <4 x i64> [[V1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP4]], i64 0)
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP5]], <vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
 ; CHECK-NEXT:    ret void
 ;
@@ -418,9 +418,9 @@ define void @store_min_ge_type(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1) #2 {
 ; CHECK-SAME: ptr [[PTR:%.*]], <4 x i64> [[V0:%.*]], <4 x i64> [[V1:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 4)
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[V0]], <4 x i64> [[V1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP2]], i64 0)
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[V0]], <4 x i64> [[V1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> poison, <4 x i64> [[TMP4]], i64 0)
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP5]], <vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
 ; CHECK-NEXT:    ret void
 ;
@@ -513,13 +513,13 @@ define void @store_double_factor4(ptr %ptr, <4 x double> %v0, <4 x double> %v1,
 ; CHECK-NEXT:    [[S1:%.*]] = shufflevector <4 x double> [[V2]], <4 x double> [[V3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> undef, <4 x double> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> poison, <4 x double> [[TMP2]], i64 0)
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> undef, <4 x double> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> poison, <4 x double> [[TMP4]], i64 0)
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> undef, <4 x double> [[TMP6]], i64 0)
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> poison, <4 x double> [[TMP6]], i64 0)
 ; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> undef, <4 x double> [[TMP8]], i64 0)
+; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> poison, <4 x double> [[TMP8]], i64 0)
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st4.nxv2f64(<vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[TMP5]], <vscale x 2 x double> [[TMP7]], <vscale x 2 x double> [[TMP9]], <vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
 ; CHECK-NEXT:    ret void
 ;
@@ -537,11 +537,11 @@ define void @store_float_factor3(ptr %ptr, <8 x float> %v0, <8 x float> %v1, <8
 ; CHECK-NEXT:    [[S1:%.*]] = shufflevector <8 x float> [[V2]], <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x float> [[S0]], <16 x float> [[S1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> undef, <8 x float> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> poison, <8 x float> [[TMP2]], i64 0)
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x float> [[S0]], <16 x float> [[S1]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> undef, <8 x float> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> poison, <8 x float> [[TMP4]], i64 0)
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <16 x float> [[S0]], <16 x float> [[S1]], <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> undef, <8 x float> [[TMP6]], i64 0)
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> poison, <8 x float> [[TMP6]], i64 0)
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st3.nxv4f32(<vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[TMP5]], <vscale x 4 x float> [[TMP7]], <vscale x 4 x i1> [[TMP1]], ptr [[PTR]])
 ; CHECK-NEXT:    ret void
 ;
@@ -560,9 +560,9 @@ define void @store_half_factor2(ptr %ptr, <16 x half> %v0, <16 x half> %v1) #0 {
 ; CHECK-SAME: ptr [[PTR:%.*]], <16 x half> [[V0:%.*]], <16 x half> [[V1:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x half> [[V0]], <16 x half> [[V1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v16f16(<vscale x 8 x half> undef, <16 x half> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v16f16(<vscale x 8 x half> poison, <16 x half> [[TMP2]], i64 0)
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x half> [[V0]], <16 x half> [[V1]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v16f16(<vscale x 8 x half> undef, <16 x half> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v16f16(<vscale x 8 x half> poison, <16 x half> [[TMP4]], i64 0)
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv8f16(<vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP5]], <vscale x 8 x i1> [[TMP1]], ptr [[PTR]])
 ; CHECK-NEXT:    ret void
 ;
@@ -578,9 +578,9 @@ define void @store_bfloat_factor2(ptr %ptr, <16 x bfloat> %v0, <16 x bfloat> %v1
 ; CHECK-SAME: ptr [[PTR:%.*]], <16 x bfloat> [[V0:%.*]], <16 x bfloat> [[V1:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x bfloat> [[V0]], <16 x bfloat> [[V1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v16bf16(<vscale x 8 x bfloat> undef, <16 x bfloat> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v16bf16(<vscale x 8 x bfloat> poison, <16 x bfloat> [[TMP2]], i64 0)
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x bfloat> [[V0]], <16 x bfloat> [[V1]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v16bf16(<vscale x 8 x bfloat> undef, <16 x bfloat> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v16bf16(<vscale x 8 x bfloat> poison, <16 x bfloat> [[TMP4]], i64 0)
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv8bf16(<vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x i1> [[TMP1]], ptr [[PTR]])
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll
index 07958f1c1a296..ede2e4066c263 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll
@@ -1,4 +1,5 @@
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -mattr=+relaxed-buffer-oob-mode -S -o - %s | FileCheck --check-prefixes=CHECK,CHECK-OOB-RELAXED %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck --check-prefixes=CHECK,CHECK-OOB-STRICT %s
 
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7"
 
@@ -72,9 +73,14 @@ entry:
   ret void
 }
 
-; CHECK-LABEL: @merge_fat_ptrs(
-; CHECK: load <4 x i16>
-; CHECK: store <4 x i16> zeroinitializer
+; CHECK-OOB-RELAXED-LABEL: @merge_fat_ptrs(
+; CHECK-OOB-RELAXED: load <4 x i16>
+; CHECK-OOB-RELAXED: store <4 x i16> zeroinitializer
+; CHECK-OOB-STRICT-LABEL: @merge_fat_ptrs(
+; CHECK-OOB-STRICT: load <2 x i16>
+; CHECK-OOB-STRICT: load <2 x i16>
+; CHECK-OOB-STRICT: store <2 x i16> zeroinitializer
+; CHECK-OOB-STRICT: store <2 x i16> zeroinitializer
 define amdgpu_kernel void @merge_fat_ptrs(ptr addrspace(7) nocapture %a, ptr addrspace(7) nocapture readonly %b) #0 {
 entry:
   %a.1 = getelementptr inbounds <2 x i16>, ptr addrspace(7) %a, i32 1
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/unaligned-buffer.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/unaligned-buffer.ll
new file mode 100644
index 0000000000000..d590a4a403fb7
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/unaligned-buffer.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=amdgcn--amdpal -passes=load-store-vectorizer -S -o - %s | FileCheck --check-prefix=OOB-STRICT %s
+; RUN: opt -mtriple=amdgcn--amdpal -passes=load-store-vectorizer -mattr=+relaxed-buffer-oob-mode -S -o - %s | FileCheck --check-prefixes=OOB-RELAXED %s
+
+; The test checks that relaxed-buffer-oob-mode allows merging loads even if the target load is not naturally aligned.
+
+define amdgpu_kernel void @merge_align_4(ptr addrspace(7) captures(none) %p) #0 {
+;
+; OOB-STRICT-LABEL: define amdgpu_kernel void @merge_align_4(
+; OOB-STRICT-SAME: ptr addrspace(7) captures(none) [[P:%.*]]) {
+; OOB-STRICT-NEXT:  [[ENTRY:.*:]]
+; OOB-STRICT-NEXT:    [[GEP_M8:%.*]] = getelementptr i8, ptr addrspace(7) [[P]], i32 -8
+; OOB-STRICT-NEXT:    [[LD_M8:%.*]] = load i32, ptr addrspace(7) [[GEP_M8]], align 4
+; OOB-STRICT-NEXT:    [[GEP_M4:%.*]] = getelementptr i8, ptr addrspace(7) [[P]], i32 -4
+; OOB-STRICT-NEXT:    [[LD_M4:%.*]] = load i32, ptr addrspace(7) [[GEP_M4]], align 4
+; OOB-STRICT-NEXT:    [[GEP_0:%.*]] = getelementptr i8, ptr addrspace(7) [[P]], i32 0
+; OOB-STRICT-NEXT:    [[LD_0:%.*]] = load i32, ptr addrspace(7) [[GEP_0]], align 4
+; OOB-STRICT-NEXT:    [[GEP_4:%.*]] = getelementptr i8, ptr addrspace(7) [[P]], i64 4
+; OOB-STRICT-NEXT:    [[LD_4:%.*]] = load i32, ptr addrspace(7) [[GEP_4]], align 4
+; OOB-STRICT-NEXT:    ret void
+;
+; OOB-RELAXED-LABEL: define amdgpu_kernel void @merge_align_4(
+; OOB-RELAXED-SAME: ptr addrspace(7) captures(none) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; OOB-RELAXED-NEXT:  [[ENTRY:.*:]]
+; OOB-RELAXED-NEXT:    [[GEP_M8:%.*]] = getelementptr i8, ptr addrspace(7) [[P]], i32 -8
+; OOB-RELAXED-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr addrspace(7) [[GEP_M8]], align 4
+; OOB-RELAXED-NEXT:    [[LD_M81:%.*]] = extractelement <4 x i32> [[TMP0]], i32 0
+; OOB-RELAXED-NEXT:    [[LD_M42:%.*]] = extractelement <4 x i32> [[TMP0]], i32 1
+; OOB-RELAXED-NEXT:    [[LD_03:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2
+; OOB-RELAXED-NEXT:    [[LD_44:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3
+; OOB-RELAXED-NEXT:    ret void
+;
+entry:
+  %gep_m8 = getelementptr i8, ptr addrspace(7) %p, i32 -8
+  %ld_m8 = load i32, ptr addrspace(7) %gep_m8, align 4
+  %gep_m4 = getelementptr i8, ptr addrspace(7) %p, i32 -4
+  %ld_m4 = load i32, ptr addrspace(7) %gep_m4, align 4
+  %gep_0 = getelementptr i8, ptr addrspace(7) %p, i32 0
+  %ld_0 = load i32, ptr addrspace(7) %gep_0, align 4
+  %gep_4 = getelementptr i8, ptr addrspace(7) %p, i64 4
+  %ld_4 = load i32, ptr addrspace(7) %gep_4, align 4
+  ret void
+}
+
+; The test checks that strict OOB mode (relaxed-buffer-oob-mode not set) allows merging loads if the target load is naturally aligned.
+
+define amdgpu_kernel void @merge_align_16(ptr addrspace(7) captures(none) %p) #0 {
+; OOB-STRICT-LABEL: define amdgpu_kernel void @merge_align_16(
+; OOB-STRICT-SAME: ptr addrspace(7) captures(none) [[P:%.*]]) {
+; OOB-STRICT-NEXT:  [[ENTRY:.*:]]
+; OOB-STRICT-NEXT:    [[GEP_M8:%.*]] = getelementptr i8, ptr addrspace(7) [[P]], i32 -8
+; OOB-STRICT-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr addrspace(7) [[GEP_M8]], align 16
+; OOB-STRICT-NEXT:    [[LD_M81:%.*]] = extractelement <4 x i32> [[TMP0]], i32 0
+; OOB-STRICT-NEXT:    [[LD_M42:%.*]] = extractelement <4 x i32> [[TMP0]], i32 1
+; OOB-STRICT-NEXT:    [[LD_03:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2
+; OOB-STRICT-NEXT:    [[LD_44:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3
+; OOB-STRICT-NEXT:    ret void
+;
+; OOB-RELAXED-LABEL: define amdgpu_kernel void @merge_align_16(
+; OOB-RELAXED-SAME: ptr addrspace(7) captures(none) [[P:%.*]]) #[[ATTR0]] {
+; OOB-RELAXED-NEXT:  [[ENTRY:.*:]]
+; OOB-RELAXED-NEXT:    [[GEP_M8:%.*]] = getelementptr i8, ptr addrspace(7) [[P]], i32 -8
+; OOB-RELAXED-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr addrspace(7) [[GEP_M8]], align 16
+; OOB-RELAXED-NEXT:    [[LD_M81:%.*]] = extractelement <4 x i32> [[TMP0]], i32 0
+; OOB-RELAXED-NEXT:    [[LD_M42:%.*]] = extractelement <4 x i32> [[TMP0]], i32 1
+; OOB-RELAXED-NEXT:    [[LD_03:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2
+; OOB-RELAXED-NEXT:    [[LD_44:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3
+; OOB-RELAXED-NEXT:    ret void
+;
+entry:
+  %gep_m8 = getelementptr i8, ptr addrspace(7) %p, i32 -8
+  %ld_m8 = load i32, ptr addrspace(7) %gep_m8, align 16
+  %gep_m4 = getelementptr i8, ptr addrspace(7) %p, i32 -4
+  %ld_m4 = load i32, ptr addrspace(7) %gep_m4, align 4
+  %gep_0 = getelementptr i8, ptr addrspace(7) %p, i32 0
+  %ld_0 = load i32, ptr addrspace(7) %gep_0, align 8
+  %gep_4 = getelementptr i8, ptr addrspace(7) %p, i64 4
+  %ld_4 = load i32, ptr addrspace(7) %gep_4, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopDataPrefetch/RISCV/basic.ll b/llvm/test/Transforms/LoopDataPrefetch/RISCV/basic.ll
index 71d9953a03da1..6edd0ff149b43 100644
--- a/llvm/test/Transforms/LoopDataPrefetch/RISCV/basic.ll
+++ b/llvm/test/Transforms/LoopDataPrefetch/RISCV/basic.ll
@@ -11,12 +11,16 @@ define void @foo(ptr nocapture %a, ptr nocapture readonly %b) {
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 3
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 64
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP4]], 64
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP3]]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    call void @llvm.prefetch.p0(ptr [[SCEVGEP]], i32 0, i32 3, i32 1)
 ; CHECK-NEXT:    [[TMP2:%.*]] = load double, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[TMP2]], 1.000000e+00
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    call void @llvm.prefetch.p0(ptr [[SCEVGEP1]], i32 1, i32 3, i32 1)
 ; CHECK-NEXT:    store double [[ADD]], ptr [[ARRAYIDX2]], align 8
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1600
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
index 254cdf2d14d9f..a07da0744fcf7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll
@@ -428,10 +428,10 @@ define void @test_blend_feeding_replicated_store_3(ptr noalias %src.1, ptr noali
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[L_1:%.*]] = load i8, ptr [[SRC_1]], align 1
-; CHECK-NEXT:    [[EXT:%.*]] = zext i8 [[L_1]] to i32
+; CHECK-NEXT:    [[L_3:%.*]] = load i8, ptr [[SRC_1]], align 1
+; CHECK-NEXT:    [[EXT:%.*]] = zext i8 [[L_3]] to i32
 ; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[X]], [[EXT]]
-; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[MUL]], 255
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[MUL]], [[EXT]]
 ; CHECK-NEXT:    [[L_2:%.*]] = load i8, ptr [[SRC_2]], align 1
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp eq i8 [[L_2]], 0
 ; CHECK-NEXT:    br i1 [[C_1]], label %[[THEN:.*]], label %[[ELSE_1:.*]]
@@ -459,7 +459,7 @@ loop.header:
   %l.1 = load i8, ptr %src.1, align 1
   %ext = zext i8 %l.1 to i32
   %mul = mul i32 %x, %ext
-  %div = sdiv i32 %mul, 255
+  %div = sdiv i32 %mul, %ext
   %l.2 = load i8, ptr %src.2, align 1
   %c.1 = icmp eq i8 %l.2, 0
   br i1 %c.1, label %then, label %else.1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
new file mode 100644
index 0000000000000..55c6c43b6306a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; REQUIRES: asserts
+; RUN: opt -S < %s -p loop-vectorize -enable-early-exit-vectorization -disable-output \
+; RUN:   -debug-only=loop-vectorize 2>&1 | FileCheck %s --check-prefixes=CHECK
+
+target triple = "aarch64-unknown-linux-gnu"
+
+declare void @init_mem(ptr, i64);
+
+define i64 @same_exit_block_pre_inc_use1_sve() #1 {
+; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1_sve'
+; CHECK: LV: Selecting VF: vscale x 16
+; CHECK: Calculating cost of work in exit block vector.early.exit
+; CHECK-NEXT: Cost of 6 for VF vscale x 16: EMIT vp<{{.*}}> = extract-first-active
+; CHECK-NEXT: Cost of 6 for VF vscale x 16: EMIT vp<{{.*}}> = extract-first-active
+; CHECK: LV: Minimum required TC for runtime checks to be profitable:32
+entry:
+  %p1 = alloca [1024 x i8]
+  %p2 = alloca [1024 x i8]
+  call void @init_mem(ptr %p1, i64 1024)
+  call void @init_mem(ptr %p2, i64 1024)
+  br label %loop
+
+loop:
+  %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
+  %index2 = phi i64 [ %index2.next, %loop.inc ], [ 15, %entry ]
+  %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+  %ld1 = load i8, ptr %arrayidx, align 1
+  %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+  %ld2 = load i8, ptr %arrayidx1, align 1
+  %cmp3 = icmp eq i8 %ld1, %ld2
+  br i1 %cmp3, label %loop.inc, label %loop.end
+
+loop.inc:
+  %index.next = add i64 %index, 1
+  %index2.next = add i64 %index2, 2
+  %exitcond = icmp ne i64 %index.next, 67
+  br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+  %val1 = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
+  %val2 = phi i64 [ %index2, %loop ], [ 98, %loop.inc ]
+  %retval = add i64 %val1, %val2
+  ret i64 %retval
+}
+
+define i64 @same_exit_block_pre_inc_use1_nosve() {
+; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1_nosve'
+; CHECK: LV: Selecting VF: 16
+; CHECK: Calculating cost of work in exit block vector.early.exit
+; CHECK-NEXT: Cost of 50 for VF 16: EMIT vp<{{.*}}> = extract-first-active
+; CHECK-NEXT: Cost of 50 for VF 16: EMIT vp<{{.*}}> = extract-first-active
+; CHECK: LV: Minimum required TC for runtime checks to be profitable:176
+; CHECK-NEXT: LV: Vectorization is not beneficial: expected trip count < minimum profitable VF (64 < 176)
+; CHECK-NEXT: LV: Too many memory checks needed.
+entry:
+  %p1 = alloca [1024 x i8]
+  %p2 = alloca [1024 x i8]
+  call void @init_mem(ptr %p1, i64 1024)
+  call void @init_mem(ptr %p2, i64 1024)
+  br label %loop
+
+loop:
+  %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
+  %index2 = phi i64 [ %index2.next, %loop.inc ], [ 15, %entry ]
+  %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+  %ld1 = load i8, ptr %arrayidx, align 1
+  %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+  %ld2 = load i8, ptr %arrayidx1, align 1
+  %cmp3 = icmp eq i8 %ld1, %ld2
+  br i1 %cmp3, label %loop.inc, label %loop.end
+
+loop.inc:
+  %index.next = add i64 %index, 1
+  %index2.next = add i64 %index2, 2
+  %exitcond = icmp ne i64 %index.next, 67
+  br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+  %val1 = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
+  %val2 = phi i64 [ %index2, %loop ], [ 98, %loop.inc ]
+  %retval = add i64 %val1, %val2
+  ret i64 %retval
+}
+
+attributes #1 = { "target-features"="+sve" vscale_range(1,16) }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll b/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll
index f969be80c3c00..ab21794baa924 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll
@@ -23,9 +23,9 @@
 ; Check that the extractvalue operands are actually free in vector code.
 
 ; FORCED:         [[E1:%.+]] = extractvalue { i64, i64 } %sv, 0
-; FORCED-NEXT:    [[E2:%.+]] = extractvalue { i64, i64 } %sv, 1
 ; FORCED-NEXT:    %broadcast.splatinsert = insertelement <2 x i64> poison, i64 [[E1]], i64 0
 ; FORCED-NEXT:    %broadcast.splat = shufflevector <2 x i64> %broadcast.splatinsert, <2 x i64> poison, <2 x i32> zeroinitializer
+; FORCED-NEXT:    [[E2:%.+]] = extractvalue { i64, i64 } %sv, 1
 ; FORCED-NEXT:    %broadcast.splatinsert1 = insertelement <2 x i64> poison, i64 [[E2]], i64 0
 ; FORCED-NEXT:    %broadcast.splat2 = shufflevector <2 x i64> %broadcast.splatinsert1, <2 x i64> poison, <2 x i32> zeroinitializer
 ; FORCED-NEXT:    [[ADD:%.+]] = add <2 x i64> %broadcast.splat, %broadcast.splat2
@@ -75,9 +75,9 @@ declare float @powf(float, float) readnone nounwind
 ; FORCED-LABEL: define void @test_getVectorCallCost
 
 ; FORCED:         [[E1:%.+]] = extractvalue { float, float } %sv, 0
-; FORCED-NEXT:    [[E2:%.+]] = extractvalue { float, float } %sv, 1
 ; FORCED-NEXT:    %broadcast.splatinsert = insertelement <2 x float> poison, float [[E1]], i64 0
 ; FORCED-NEXT:    %broadcast.splat = shufflevector <2 x float> %broadcast.splatinsert, <2 x float> poison, <2 x i32> zeroinitializer
+; FORCED-NEXT:    [[E2:%.+]] = extractvalue { float, float } %sv, 1
 ; FORCED-NEXT:    %broadcast.splatinsert1 = insertelement <2 x float> poison, float [[E2]], i64 0
 ; FORCED-NEXT:    %broadcast.splat2 = shufflevector <2 x float> %broadcast.splatinsert1, <2 x float> poison, <2 x i32> zeroinitializer
 
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 987d18c427aef..be8fb1857927c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -205,10 +205,10 @@ define void @test_exit_branch_cost(ptr %dst, ptr noalias %x.ptr, ptr noalias %y.
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT40:%.*]] = insertelement <2 x i1> poison, i1 [[C_3]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT41:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT40]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[C_4]], <2 x i1> [[BROADCAST_SPLAT41]], <2 x i1> zeroinitializer
-; CHECK-NEXT:    [[TMP11:%.*]] = xor <2 x i1> [[TMP2]], splat (i1 true)
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT56:%.*]] = insertelement <2 x i1> poison, i1 [[C_4]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT57:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT56]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[C_4]], <2 x i1> [[BROADCAST_SPLAT41]], <2 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <2 x i1> [[TMP1]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP33:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT57]], splat (i1 true)
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index 62186d5a20989..69794b9798f1c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -112,14 +112,14 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
 ; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; PRED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
 ; PRED-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; PRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[X]], i64 0
+; PRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
 ; PRED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
 ; PRED-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 8
 ; PRED-NEXT:    [[TMP13:%.*]] = sub i64 [[TMP0]], [[TMP12]]
 ; PRED-NEXT:    [[TMP14:%.*]] = icmp ugt i64 [[TMP0]], [[TMP12]]
 ; PRED-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i64 [[TMP13]], i64 0
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP0]])
-; PRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[X]], i64 0
-; PRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
 ; PRED-NEXT:    [[TMP16:%.*]] = trunc <vscale x 8 x i32> [[BROADCAST_SPLAT]] to <vscale x 8 x i16>
 ; PRED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; PRED:       vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll b/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
index 4c54796d9d16d..da628875c544e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
@@ -472,9 +472,9 @@ define void @add_e(ptr noalias nocapture readonly %p, ptr noalias nocapture %q,
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[CONV13]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc <16 x i32> [[BROADCAST_SPLAT]] to <16 x i8>
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <16 x i32> poison, i32 [[CONV11]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT2]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <16 x i32> [[BROADCAST_SPLAT]] to <16 x i8>
 ; CHECK-NEXT:    [[TMP2:%.*]] = trunc <16 x i32> [[BROADCAST_SPLAT3]] to <16 x i8>
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -510,9 +510,9 @@ define void @add_e(ptr noalias nocapture readonly %p, ptr noalias nocapture %q,
 ; CHECK-NEXT:    [[N_VEC5:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF4]]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i32> poison, i32 [[CONV13]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT6]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP17:%.*]] = trunc <4 x i32> [[BROADCAST_SPLAT7]] to <4 x i8>
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i32> poison, i32 [[CONV11]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT8]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = trunc <4 x i32> [[BROADCAST_SPLAT7]] to <4 x i8>
 ; CHECK-NEXT:    [[TMP18:%.*]] = trunc <4 x i32> [[BROADCAST_SPLAT9]] to <4 x i8>
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
@@ -620,9 +620,9 @@ define void @add_f(ptr noalias nocapture readonly %p, ptr noalias nocapture %q,
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[CONV13]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc <16 x i32> [[BROADCAST_SPLAT]] to <16 x i8>
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <16 x i32> poison, i32 [[CONV11]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT2]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <16 x i32> [[BROADCAST_SPLAT]] to <16 x i8>
 ; CHECK-NEXT:    [[TMP2:%.*]] = trunc <16 x i32> [[BROADCAST_SPLAT3]] to <16 x i8>
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -660,9 +660,9 @@ define void @add_f(ptr noalias nocapture readonly %p, ptr noalias nocapture %q,
 ; CHECK-NEXT:    [[N_VEC5:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF4]]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i32> poison, i32 [[CONV13]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT6]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP19:%.*]] = trunc <4 x i32> [[BROADCAST_SPLAT7]] to <4 x i8>
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i32> poison, i32 [[CONV11]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT8]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = trunc <4 x i32> [[BROADCAST_SPLAT7]] to <4 x i8>
 ; CHECK-NEXT:    [[TMP20:%.*]] = trunc <4 x i32> [[BROADCAST_SPLAT9]] to <4 x i8>
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
index ff0f5f3f711f4..85c9536dfbc43 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
@@ -160,9 +160,9 @@ define void @dotp_small_epilogue_vf(i64 %idx.neg, i8 %a) #1 {
 ; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP6]], [[WHILE_BODY]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    [[N_MOD_VF4:%.*]] = urem i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[N_VEC5:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF4]]
-; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[IDX_NEG]], [[N_VEC5]]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i8> poison, i8 [[A]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT6]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[IDX_NEG]], [[N_VEC5]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = sext <4 x i8> [[BROADCAST_SPLAT7]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
index 3d4f7e0e4924b..ce2a5af52d442 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
@@ -61,15 +61,15 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
 ; VSCALEFORTUNING2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; VSCALEFORTUNING2-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
 ; VSCALEFORTUNING2-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
-; VSCALEFORTUNING2-NEXT:    [[TMP7:%.*]] = add i64 [[Y]], 1
-; VSCALEFORTUNING2-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[SRC_1]], i64 [[TMP7]]
+; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z]], i64 0
+; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
 ; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; VSCALEFORTUNING2-NEXT:    [[TMP7:%.*]] = add i64 [[Y]], 1
+; VSCALEFORTUNING2-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[SRC_1]], i64 [[TMP7]]
 ; VSCALEFORTUNING2-NEXT:    [[TMP9:%.*]] = lshr <vscale x 4 x i32> [[BROADCAST_SPLAT]], splat (i32 1)
 ; VSCALEFORTUNING2-NEXT:    [[TMP10:%.*]] = shl <vscale x 4 x i32> [[BROADCAST_SPLAT]], splat (i32 1)
 ; VSCALEFORTUNING2-NEXT:    [[TMP11:%.*]] = or <vscale x 4 x i32> [[TMP9]], [[TMP10]]
-; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z]], i64 0
-; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; VSCALEFORTUNING2-NEXT:    [[TMP12:%.*]] = or <vscale x 4 x i32> [[BROADCAST_SPLAT2]], [[BROADCAST_SPLAT]]
 ; VSCALEFORTUNING2-NEXT:    [[TMP13:%.*]] = and <vscale x 4 x i32> [[TMP12]], splat (i32 1)
 ; VSCALEFORTUNING2-NEXT:    [[TMP14:%.*]] = xor <vscale x 4 x i32> [[TMP13]], splat (i32 1)
@@ -191,6 +191,10 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
 ; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; PRED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; PRED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; PRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z]], i64 0
+; PRED-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; PRED-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
+; PRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; PRED-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
 ; PRED-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
 ; PRED-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP0]], [[TMP7]]
@@ -199,13 +203,9 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
 ; PRED-NEXT:    [[TMP11:%.*]] = add i64 [[Y]], 1
 ; PRED-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[SRC_1]], i64 [[TMP11]]
-; PRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
-; PRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; PRED-NEXT:    [[TMP13:%.*]] = lshr <vscale x 4 x i32> [[BROADCAST_SPLAT]], splat (i32 1)
 ; PRED-NEXT:    [[TMP14:%.*]] = shl <vscale x 4 x i32> [[BROADCAST_SPLAT]], splat (i32 1)
 ; PRED-NEXT:    [[TMP15:%.*]] = or <vscale x 4 x i32> [[TMP13]], [[TMP14]]
-; PRED-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z]], i64 0
-; PRED-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; PRED-NEXT:    [[TMP16:%.*]] = or <vscale x 4 x i32> [[BROADCAST_SPLAT2]], [[BROADCAST_SPLAT]]
 ; PRED-NEXT:    [[TMP17:%.*]] = and <vscale x 4 x i32> [[TMP16]], splat (i32 1)
 ; PRED-NEXT:    [[TMP18:%.*]] = xor <vscale x 4 x i32> [[TMP17]], splat (i32 1)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
index 5f926db1131f6..ab4738bf2901b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
@@ -274,6 +274,7 @@ define i64 @loop_contains_safe_div() #1 {
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
 ; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 4
+; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP12]])
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
index cd71a1c2c04d2..5aad94a67aa58 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
@@ -243,9 +243,9 @@ define void @trunc_store(ptr %dst, ptr %src, i16 %x) #1 {
 ; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; PRED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; PRED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
-; PRED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1000)
 ; PRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i16> poison, i16 [[X]], i64 0
 ; PRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i16> poison, <vscale x 2 x i32> zeroinitializer
+; PRED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1000)
 ; PRED-NEXT:    [[TMP11:%.*]] = trunc <vscale x 2 x i16> [[BROADCAST_SPLAT]] to <vscale x 2 x i8>
 ; PRED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; PRED:       vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-extractvalue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-extractvalue.ll
index 91dd6e475ec47..473fabfc9fecc 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-extractvalue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-extractvalue.ll
@@ -5,9 +5,9 @@ target triple = "aarch64-unknown-linux-gnu"
 define void @widen_extractvalue(ptr %dst, {i64, i64} %sv) #0 {
 ; CHECK-LABEL: @widen_extractvalue(
 ; CHECK:        [[EXTRACT0:%.*]] = extractvalue { i64, i64 } [[SV:%.*]], 0
-; CHECK-NEXT:   [[EXTRACT1:%.*]] = extractvalue { i64, i64 } [[SV]], 1
 ; CHECK-NEXT:   [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[EXTRACT0]], i64 0
 ; CHECK-NEXT:   [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:   [[EXTRACT1:%.*]] = extractvalue { i64, i64 } [[SV]], 1
 ; CHECK-NEXT:   [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[EXTRACT1]], i64 0
 ; CHECK-NEXT:   [[DOTSPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK:        [[ADD:%.*]] = add <vscale x 2 x i64> [[DOTSPLAT]], [[DOTSPLAT2]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll
index 87338e55c16c8..362cf093f1807 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll
@@ -73,11 +73,11 @@ define i32 @any_of_reduction_used_in_blend_with_multiple_phis(ptr %src, i64 %N,
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C_1]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C_0]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = xor <vscale x 2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C_1]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP7:%.*]] = xor <vscale x 2 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> [[TMP7]], <vscale x 2 x i1> zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[SRC]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index 3839b367ae08c..780f44d16c41f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -328,9 +328,9 @@ define void @test_phi_in_latch_redundant(ptr %dst, i32 %a) {
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 37, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
-; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 9
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[A]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[N_VEC]], 9
 ; CHECK-NEXT:    [[TMP10:%.*]] = xor <vscale x 2 x i32> [[BROADCAST_SPLAT]], splat (i32 -1)
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; CHECK-NEXT:    [[TMP8:%.*]] = mul <vscale x 2 x i64> [[TMP6]], splat (i64 9)
@@ -352,7 +352,7 @@ define void @test_phi_in_latch_redundant(ptr %dst, i32 %a) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 37, [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/illegal-type.ll b/llvm/test/Transforms/LoopVectorize/RISCV/illegal-type.ll
index eeef8f199353b..d7afe41fd39a0 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/illegal-type.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/illegal-type.ll
@@ -107,10 +107,10 @@ define void @uniform_store_i1(ptr noalias %dst, ptr noalias %start, i64 %N) {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 32
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[N_VEC]], 8
-; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[TMP1]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <32 x ptr> poison, ptr [[START]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <32 x ptr> poison, ptr [[START:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <32 x ptr> [[BROADCAST_SPLATINSERT]], <32 x ptr> poison, <32 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[N_VEC]], 8
+; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP1]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[START]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
index 305d26d7f3bc1..da859963a7021 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
@@ -7,13 +7,13 @@ define void @test(ptr %p, i64 %a, i8 %b) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i8> poison, i8 [[B]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT1]], <4 x i8> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[A]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl <4 x i64> [[BROADCAST_SPLAT]], splat (i64 48)
 ; CHECK-NEXT:    [[TMP3:%.*]] = ashr <4 x i64> [[TMP2]], splat (i64 52)
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc <4 x i64> [[TMP3]] to <4 x i32>
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i8> poison, i8 [[B]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT1]], <4 x i8> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = zext <4 x i8> [[BROADCAST_SPLAT2]] to <4 x i32>
 ; CHECK-NEXT:    br label [[VECTOR_BODY1:%.*]]
 ; CHECK:       vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
index 000c1570ef92d..94c0036e4c8b6 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
@@ -270,16 +270,16 @@ define void @icmp_only_first_op_truncated(ptr noalias %dst, i32 %x, i64 %N, i64
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[N]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP7:%.*]] = trunc <vscale x 2 x i64> [[BROADCAST_SPLAT]] to <vscale x 2 x i32>
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[T]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = trunc <vscale x 2 x i64> [[BROADCAST_SPLAT]] to <vscale x 2 x i32>
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <vscale x 2 x i32> [[TMP7]], [[BROADCAST_SPLAT2]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[X]] to i64
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP9]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[DST]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT5]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[TMP10]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT6]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[DST]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index b11f72c886bdc..1f171414242bc 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -1101,12 +1101,12 @@ define i64 @cost_loop_invariant_recipes(i1 %x, i64 %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[Y:%.*]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[X:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext <2 x i1> [[TMP0]] to <2 x i64>
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[Y:%.*]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl <2 x i64> [[BROADCAST_SPLAT2]], [[TMP1]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll b/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
index 1a9e7ddb965f3..0c99ff6c3546c 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
@@ -14,9 +14,9 @@ define i32 @unused_blend_after_unrolling(ptr %p, i32 %a, i1 %c.1, i16 %x, i16 %y
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C_1]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT16:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT17:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT16]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP22:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT17]], splat (i1 true)
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll b/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll
index 6bed344d4d73b..9ce534bcf2179 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll
@@ -124,11 +124,11 @@ define void @_Z3fn1v() #0 {
 ; CHECK:       vector.ph25:
 ; CHECK-NEXT:    [[N_MOD_VF31:%.*]] = urem i64 [[TMP28]], 16
 ; CHECK-NEXT:    [[N_VEC32:%.*]] = sub i64 [[TMP28]], [[N_MOD_VF31]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i1> poison, i1 [[TOBOOL6]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i1> [[BROADCAST_SPLATINSERT]], <16 x i1> poison, <16 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP29:%.*]] = mul i64 [[N_VEC32]], 2
 ; CHECK-NEXT:    [[IND_END41:%.*]] = add i64 8, [[TMP29]]
 ; CHECK-NEXT:    [[IND_END43:%.*]] = mul i64 [[N_VEC32]], 2
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i1> poison, i1 [[TOBOOL6]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i1> [[BROADCAST_SPLATINSERT]], <16 x i1> poison, <16 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP34:%.*]] = xor <16 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
 ; CHECK-NEXT:    br label [[VECTOR_BODY29:%.*]]
 ; CHECK:       vector.body28:
@@ -170,11 +170,11 @@ define void @_Z3fn1v() #0 {
 ; CHECK-NEXT:    [[BC_RESUME_VAL44:%.*]] = phi i64 [ [[IND_END43]], [[VEC_EPILOG_ITER_CHECK43]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK24]] ]
 ; CHECK-NEXT:    [[N_MOD_VF52:%.*]] = urem i64 [[TMP28]], 8
 ; CHECK-NEXT:    [[N_VEC53:%.*]] = sub i64 [[TMP28]], [[N_MOD_VF52]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT49:%.*]] = insertelement <8 x i1> poison, i1 [[TOBOOL6]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT73:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT49]], <8 x i1> poison, <8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP43:%.*]] = mul i64 [[N_VEC53]], 2
 ; CHECK-NEXT:    [[IND_END54:%.*]] = add i64 8, [[TMP43]]
 ; CHECK-NEXT:    [[IND_END57:%.*]] = mul i64 [[N_VEC53]], 2
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT50:%.*]] = insertelement <8 x i1> poison, i1 [[TOBOOL6]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT73:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT50]], <8 x i1> poison, <8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP48:%.*]] = xor <8 x i1> [[BROADCAST_SPLAT73]], splat (i1 true)
 ; CHECK-NEXT:    [[DOTSPLATINSERT62:%.*]] = insertelement <8 x i64> poison, i64 [[BC_RESUME_VAL42]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT63:%.*]] = shufflevector <8 x i64> [[DOTSPLATINSERT62]], <8 x i64> poison, <8 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll
index 9c76b7efae072..0306567c5ce98 100644
--- a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll
@@ -135,9 +135,9 @@ define void @inv_val_store_to_inv_address_conditional_inv(ptr %a, i64 %n, ptr %b
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[SMAX2]], 9223372036854775804
 ; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = insertelement <4 x i32> poison, i32 [[K]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i1> poison, i1 [[CMP]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[NTRUNC]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i1> poison, i1 [[CMP]], i64 0
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT6]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[PREDPHI]], i64 0
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll b/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
new file mode 100644
index 0000000000000..388da8540646f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
@@ -0,0 +1,213 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -S %s | FileCheck %s
+
+@A = common global [1024 x i64] zeroinitializer, align 16
+@B = common global [1024 x i64] zeroinitializer, align 16
+
+; FIXME: The exit condition of the inner loop is incorrect when vectorizing.
+define void @inner_latch_header_first_successor(i64 %N, i32 %c, i64 %M) {
+; CHECK-LABEL: define void @inner_latch_header_first_successor(
+; CHECK-SAME: i64 [[N:%.*]], i32 [[C:%.*]], i64 [[M:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[M]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[M]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[M]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [1024 x i64], ptr @A, i64 0, <4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP0]], i32 4, <4 x i1> splat (i1 true), <4 x i64> poison)
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[BROADCAST_SPLAT]], <4 x ptr> [[TMP0]], i32 4, <4 x i1> splat (i1 true))
+; CHECK-NEXT:    br label %[[INNER3:.*]]
+; CHECK:       [[INNER3]]:
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER3]] ]
+; CHECK-NEXT:    [[VEC_PHI4:%.*]] = phi <4 x i64> [ [[WIDE_MASKED_GATHER]], %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER3]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [1024 x i64], ptr @B, i64 0, <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER5:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP1]], i32 4, <4 x i1> splat (i1 true), <4 x i64> poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i64> [[WIDE_MASKED_GATHER5]], [[VEC_PHI4]]
+; CHECK-NEXT:    [[TMP3]] = add nsw <4 x i64> [[TMP2]], [[VEC_PHI4]]
+; CHECK-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <4 x i64> [[TMP4]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[INNER3]]
+; CHECK:       [[VECTOR_LATCH]]:
+; CHECK-NEXT:    [[VEC_PHI6:%.*]] = phi <4 x i64> [ [[TMP3]], %[[INNER3]] ]
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[VEC_PHI6]], <4 x ptr> [[TMP0]], i32 4, <4 x i1> splat (i1 true))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[M]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[IV_OUTER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_OUTER_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 [[IV_OUTER]]
+; CHECK-NEXT:    [[RED_START:%.*]] = load i64, ptr [[GEP_A]], align 4
+; CHECK-NEXT:    store i32 [[C]], ptr [[GEP_A]], align 4
+; CHECK-NEXT:    br label %[[INNER:.*]]
+; CHECK:       [[INNER]]:
+; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[IV_INNER_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ [[RED_START]], %[[OUTER_HEADER]] ], [ [[RED_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds [1024 x i64], ptr @B, i64 0, i64 [[IV_INNER]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[GEP_B]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[TMP8]], [[RED]]
+; CHECK-NEXT:    [[RED_NEXT]] = add nsw i64 [[ADD]], [[RED]]
+; CHECK-NEXT:    [[IV_INNER_NEXT]] = add nuw nsw i64 [[IV_INNER]], 1
+; CHECK-NEXT:    [[INNER_EC:%.*]] = icmp ne i64 [[IV_INNER_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[INNER_EC]], label %[[INNER]], label %[[OUTER_LATCH]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[RED_NEXT_LCSSA:%.*]] = phi i64 [ [[RED_NEXT]], %[[INNER]] ]
+; CHECK-NEXT:    store i64 [[RED_NEXT_LCSSA]], ptr [[GEP_A]], align 4
+; CHECK-NEXT:    [[IV_OUTER_NEXT]] = add nuw nsw i64 [[IV_OUTER]], 1
+; CHECK-NEXT:    [[OUTER_EC:%.*]] = icmp eq i64 [[IV_OUTER_NEXT]], [[M]]
+; CHECK-NEXT:    br i1 [[OUTER_EC]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %iv.outer = phi i64 [ 0, %entry ], [ %iv.outer.next, %outer.latch ]
+  %gep.A = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.outer
+  %red.start = load i64 , ptr %gep.A, align 4
+  store i32 %c, ptr %gep.A, align 4
+  br label %inner
+
+inner:
+  %iv.inner = phi i64 [ 0, %outer.header ], [ %iv.inner.next, %inner ]
+  %red = phi i64 [ %red.start, %outer.header ], [ %red.next, %inner ]
+  %gep.B = getelementptr inbounds [1024 x i64], ptr @B, i64 0, i64 %iv.inner
+  %2 = load i64, ptr %gep.B, align 4
+  %add = add nsw i64 %2, %red
+  %red.next = add nsw i64 %add, %red
+  %iv.inner.next = add nuw nsw i64 %iv.inner, 1
+  %inner.ec = icmp ne i64 %iv.inner.next, %N
+  br i1 %inner.ec, label %inner, label %outer.latch
+
+outer.latch:
+  store i64 %red.next, ptr %gep.A, align 4
+  %iv.outer.next = add nuw nsw i64 %iv.outer, 1
+  %outer.ec = icmp eq i64 %iv.outer.next, %M
+  br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1
+
+exit:
+  ret void
+}
+
+
+define void @inner_latch_header_second_successor(i64 %N, i32 %c, i64 %M) {
+; CHECK-LABEL: define void @inner_latch_header_second_successor(
+; CHECK-SAME: i64 [[N:%.*]], i32 [[C:%.*]], i64 [[M:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[M]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[M]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[M]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [1024 x i64], ptr @A, i64 0, <4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP0]], i32 4, <4 x i1> splat (i1 true), <4 x i64> poison)
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[BROADCAST_SPLAT]], <4 x ptr> [[TMP0]], i32 4, <4 x i1> splat (i1 true))
+; CHECK-NEXT:    br label %[[INNER3:.*]]
+; CHECK:       [[INNER3]]:
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER3]] ]
+; CHECK-NEXT:    [[VEC_PHI4:%.*]] = phi <4 x i64> [ [[WIDE_MASKED_GATHER]], %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER3]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [1024 x i64], ptr @B, i64 0, <4 x i64> [[VEC_PHI]]
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER5:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP1]], i32 4, <4 x i1> splat (i1 true), <4 x i64> poison)
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i64> [[WIDE_MASKED_GATHER5]], [[VEC_PHI4]]
+; CHECK-NEXT:    [[TMP3]] = add nsw <4 x i64> [[TMP2]], [[VEC_PHI4]]
+; CHECK-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[INNER3]]
+; CHECK:       [[VECTOR_LATCH]]:
+; CHECK-NEXT:    [[VEC_PHI6:%.*]] = phi <4 x i64> [ [[TMP3]], %[[INNER3]] ]
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[VEC_PHI6]], <4 x ptr> [[TMP0]], i32 4, <4 x i1> splat (i1 true))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[M]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[IV_OUTER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_OUTER_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 [[IV_OUTER]]
+; CHECK-NEXT:    [[RED_START:%.*]] = load i64, ptr [[GEP_A]], align 4
+; CHECK-NEXT:    store i32 [[C]], ptr [[GEP_A]], align 4
+; CHECK-NEXT:    br label %[[INNER:.*]]
+; CHECK:       [[INNER]]:
+; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[IV_INNER_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ [[RED_START]], %[[OUTER_HEADER]] ], [ [[RED_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds [1024 x i64], ptr @B, i64 0, i64 [[IV_INNER]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[GEP_B]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[TMP8]], [[RED]]
+; CHECK-NEXT:    [[RED_NEXT]] = add nsw i64 [[ADD]], [[RED]]
+; CHECK-NEXT:    [[IV_INNER_NEXT]] = add nuw nsw i64 [[IV_INNER]], 1
+; CHECK-NEXT:    [[INNER_EC:%.*]] = icmp eq i64 [[IV_INNER_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[INNER_EC]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[RED_NEXT_LCSSA:%.*]] = phi i64 [ [[RED_NEXT]], %[[INNER]] ]
+; CHECK-NEXT:    store i64 [[RED_NEXT_LCSSA]], ptr [[GEP_A]], align 4
+; CHECK-NEXT:    [[IV_OUTER_NEXT]] = add nuw nsw i64 [[IV_OUTER]], 1
+; CHECK-NEXT:    [[OUTER_EC:%.*]] = icmp eq i64 [[IV_OUTER_NEXT]], [[M]]
+; CHECK-NEXT:    br i1 [[OUTER_EC]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %iv.outer = phi i64 [ 0, %entry ], [ %iv.outer.next, %outer.latch ]
+  %gep.A = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.outer
+  %red.start = load i64 , ptr %gep.A, align 4
+  store i32 %c, ptr %gep.A, align 4
+  br label %inner
+
+inner:
+  %iv.inner = phi i64 [ 0, %outer.header ], [ %iv.inner.next, %inner ]
+  %red = phi i64 [ %red.start, %outer.header ], [ %red.next, %inner ]
+  %gep.B = getelementptr inbounds [1024 x i64], ptr @B, i64 0, i64 %iv.inner
+  %2 = load i64, ptr %gep.B, align 4
+  %add = add nsw i64 %2, %red
+  %red.next = add nsw i64 %add, %red
+  %iv.inner.next = add nuw nsw i64 %iv.inner, 1
+  %inner.ec = icmp eq i64 %iv.inner.next, %N
+  br i1 %inner.ec, label %outer.latch, label %inner
+
+outer.latch:
+  store i64 %red.next, ptr %gep.A, align 4
+  %iv.outer.next = add nuw nsw i64 %iv.outer, 1
+  %outer.ec = icmp eq i64 %iv.outer.next, %M
+  br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1
+
+exit:
+  ret void
+}
+
+!1 = distinct !{!1, !2, !3}
+!2 = !{!"llvm.loop.vectorize.width", i32 4}
+!3 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_hcfg_construction.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_hcfg_construction.ll
index 4e5fc3126435d..7e90724744fb3 100644
--- a/llvm/test/Transforms/LoopVectorize/outer_loop_hcfg_construction.ll
+++ b/llvm/test/Transforms/LoopVectorize/outer_loop_hcfg_construction.ll
@@ -32,10 +32,10 @@ define void @non_outermost_loop_hcfg_construction(i64 %n, ptr %a) {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT2]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP2]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT2]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[MIDDLE_LOOP_LATCH4:%.*]] ]
@@ -173,10 +173,10 @@ define void @non_outermost_loop_hcfg_construction_other_loops_at_same_level(i64
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT2]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[ADD]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT2]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[MIDDLE_LOOP_J0_CLEANUP4:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/pr37248.ll b/llvm/test/Transforms/LoopVectorize/pr37248.ll
index 767fd2e188aae..d996c63777567 100644
--- a/llvm/test/Transforms/LoopVectorize/pr37248.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr37248.ll
@@ -38,53 +38,53 @@ define void @f1(ptr noalias %b, i1 %c, i32 %start) {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP1]], 2
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[IND_END:%.*]] = sub i32 [[START]], [[N_VEC]]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP10:%.*]] = sub i32 [[START]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i32 [[START]], [[INDEX]]
-; CHECK-NEXT:    [[TMP11:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
-; CHECK-NEXT:    [[TMP12:%.*]] = add i16 [[TMP11]], 0
-; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i1> [[TMP10]], i32 0
-; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; CHECK-NEXT:    [[TMP12:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
+; CHECK-NEXT:    [[TMP13:%.*]] = add i16 [[TMP12]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i1> [[TMP11]], i32 0
+; CHECK-NEXT:    br i1 [[TMP14]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
 ; CHECK-NEXT:    store i32 10, ptr [[B]], align 1
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK:       pred.store.continue:
-; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i1> [[TMP10]], i32 1
-; CHECK-NEXT:    br i1 [[TMP14]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i1> [[TMP11]], i32 1
+; CHECK-NEXT:    br i1 [[TMP15]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
 ; CHECK:       pred.store.if2:
 ; CHECK-NEXT:    store i32 10, ptr [[B]], align 1
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE3]]
 ; CHECK:       pred.store.continue3:
-; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [2 x i16], ptr @a, i16 0, i16 [[TMP12]]
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i16, ptr [[TMP15]], i32 0
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i16, ptr [[TMP16]], i32 -1
-; CHECK-NEXT:    store <2 x i16> zeroinitializer, ptr [[TMP17]], align 1
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [2 x i16], ptr @a, i16 0, i16 [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i16, ptr [[TMP16]], i32 0
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i16, ptr [[TMP17]], i32 -1
+; CHECK-NEXT:    store <2 x i16> zeroinitializer, ptr [[TMP18]], align 1
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY:%.*]] ], [ [[START]], [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY:%.*]] ], [ [[START]], [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[TMP19:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[DEC:%.*]], [[LAND_END:%.*]] ]
+; CHECK-NEXT:    [[TMP20:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[DEC:%.*]], [[LAND_END:%.*]] ]
 ; CHECK-NEXT:    br i1 [[C]], label [[LAND_END]], label [[LAND_RHS:%.*]]
 ; CHECK:       land.rhs:
 ; CHECK-NEXT:    store i32 10, ptr [[B]], align 1
 ; CHECK-NEXT:    br label [[LAND_END]]
 ; CHECK:       land.end:
-; CHECK-NEXT:    [[T:%.*]] = trunc i32 [[TMP19]] to i16
+; CHECK-NEXT:    [[T:%.*]] = trunc i32 [[TMP20]] to i16
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i16], ptr @a, i16 0, i16 [[T]]
 ; CHECK-NEXT:    store i16 0, ptr [[ARRAYIDX]], align 1
-; CHECK-NEXT:    [[DEC]] = add nsw i32 [[TMP19]], -1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP19]], 1
+; CHECK-NEXT:    [[DEC]] = add nsw i32 [[TMP20]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP20]], 1
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
@@ -138,38 +138,38 @@ define void @f2(ptr noalias %b, i1 %c, i32 %start) {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP1]], 2
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[IND_END:%.*]] = sub i32 [[START]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP10:%.*]] = sub i32 [[START]], [[N_VEC]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i32 [[START]], [[INDEX]]
-; CHECK-NEXT:    [[TMP10:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
-; CHECK-NEXT:    [[TMP11:%.*]] = add i16 [[TMP10]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [2 x i16], ptr @a, i16 0, i16 [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i16, ptr [[TMP12]], i32 0
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i16, ptr [[TMP13]], i32 -1
-; CHECK-NEXT:    store <2 x i16> zeroinitializer, ptr [[TMP14]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
+; CHECK-NEXT:    [[TMP12:%.*]] = add i16 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [2 x i16], ptr @a, i16 0, i16 [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i16, ptr [[TMP13]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i16, ptr [[TMP14]], i32 -1
+; CHECK-NEXT:    store <2 x i16> zeroinitializer, ptr [[TMP15]], align 1
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY:%.*]] ], [ [[START]], [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY:%.*]] ], [ [[START]], [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[TMP16:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[DEC:%.*]], [[LAND_END:%.*]] ]
+; CHECK-NEXT:    [[TMP17:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[DEC:%.*]], [[LAND_END:%.*]] ]
 ; CHECK-NEXT:    br i1 [[C]], label [[LAND_END]], label [[LAND_RHS:%.*]]
 ; CHECK:       land.rhs:
-; CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[B]], align 1
+; CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[B]], align 1
 ; CHECK-NEXT:    br label [[LAND_END]]
 ; CHECK:       land.end:
-; CHECK-NEXT:    [[TMP18:%.*]] = trunc i32 [[TMP16]] to i16
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i16], ptr @a, i16 0, i16 [[TMP18]]
+; CHECK-NEXT:    [[TMP19:%.*]] = trunc i32 [[TMP17]] to i16
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i16], ptr @a, i16 0, i16 [[TMP19]]
 ; CHECK-NEXT:    store i16 0, ptr [[ARRAYIDX]], align 1
-; CHECK-NEXT:    [[DEC]] = add nsw i32 [[TMP16]], -1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP16]], 1
+; CHECK-NEXT:    [[DEC]] = add nsw i32 [[TMP17]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP17]], 1
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll b/llvm/test/Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll
index a129a4b1928c6..67cf46e7d0f88 100644
--- a/llvm/test/Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll
@@ -6,17 +6,17 @@ define i32 @test(i32 %a, i1 %c.1, i1 %c.2 ) #0 {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[A:%.*]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT5:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i1> poison, i1 [[C_1:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT1]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x i1> poison, i1 [[C_2:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT3]], <2 x i1> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT4]], splat (i1 true)
-; CHECK-NEXT:    [[TMP7:%.*]] = select <2 x i1> [[TMP4]], <2 x i1> [[TMP6]], <2 x i1> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <2 x i32> poison, i32 [[A:%.*]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT5:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT4]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = select <2 x i1> [[TMP6]], <2 x i1> [[TMP4]], <2 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP2:%.*]] = xor <2 x i32> [[BROADCAST_SPLAT5]], splat (i32 1)
-; CHECK-NEXT:    [[TMP5:%.*]] = select <2 x i1> [[TMP4]], <2 x i1> [[BROADCAST_SPLAT4]], <2 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = select <2 x i1> [[TMP6]], <2 x i1> [[BROADCAST_SPLAT2]], <2 x i1> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -44,10 +44,10 @@ define i32 @test(i32 %a, i1 %c.1, i1 %c.2 ) #0 {
 ; CHECK:       loop.header:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[V_2:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[P_2:%.*]], [[LOOP_LATCH]] ]
-; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_LATCH]], label [[BODY_1:%.*]]
+; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_LATCH]], label [[BODY_1:%.*]]
 ; CHECK:       body.1:
 ; CHECK-NEXT:    [[V_2_ADD:%.*]] = add i32 [[V_2]], 10
-; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_LATCH]], label [[BODY_2:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_LATCH]], label [[BODY_2:%.*]]
 ; CHECK:       body.2:
 ; CHECK-NEXT:    [[ADD_1:%.*]] = add i32 [[V_2_ADD]], 20
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
index ca971f15e4878..796c1d116aa19 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
@@ -85,10 +85,10 @@ define i8 @PR34687_no_undef(i1 %c, i32 %x, i32 %n) {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C:%.*]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i1> poison, i1 [[C:%.*]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT2]], <4 x i1> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP0:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT2]], <4 x i32> splat (i32 1)
 ; CHECK-NEXT:    [[TMP1:%.*]] = sdiv <4 x i32> splat (i32 99), [[TMP0]]
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i32> [[TMP1]], <4 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll
index e24c6090b704b..9cb4b6f9d8680 100644
--- a/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll
+++ b/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll
@@ -351,7 +351,6 @@ define i64 @same_exit_block_pre_inc_use2() {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
@@ -365,11 +364,9 @@ define i64 @same_exit_block_pre_inc_use2() {
 ; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i1> [[TMP5]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       middle.split:
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[VEC_IND]], i32 3
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
@@ -391,7 +388,7 @@ define i64 @same_exit_block_pre_inc_use2() {
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ 67, [[LOOP]] ], [ [[INDEX]], [[LOOP_INC]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ 67, [[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ 67, [[LOOP]] ], [ [[INDEX]], [[LOOP_INC]] ], [ 66, [[MIDDLE_BLOCK]] ], [ 67, [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -451,7 +448,6 @@ define i64 @same_exit_block_pre_inc_use3() {
 ; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       middle.split:
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[VEC_IND]], i32 3
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
@@ -475,7 +471,7 @@ define i64 @same_exit_block_pre_inc_use3() {
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[INDEX_LCSSA:%.*]] = phi i64 [ [[INDEX]], [[LOOP_INC]] ], [ [[INDEX]], [[LOOP]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT:    [[INDEX_LCSSA:%.*]] = phi i64 [ [[INDEX]], [[LOOP_INC]] ], [ [[INDEX]], [[LOOP]] ], [ 66, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[INDEX_LCSSA]]
 ;
 entry:
@@ -597,9 +593,6 @@ define i64 @same_exit_block_post_inc_use() {
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP5]], align 1
@@ -607,10 +600,6 @@ define i64 @same_exit_block_post_inc_use() {
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP7]], align 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP3]], 1
 ; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]])
@@ -642,7 +631,7 @@ define i64 @same_exit_block_post_inc_use() {
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP17:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ [[INDEX_NEXT]], [[LOOP_INC]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ [[INDEX_NEXT]], [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -684,7 +673,6 @@ define i64 @same_exit_block_post_inc_use2() {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 1
@@ -709,11 +697,9 @@ define i64 @same_exit_block_post_inc_use2() {
 ; CHECK-NEXT:    [[TMP17:%.*]] = xor <4 x i1> [[TMP16]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP18:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP17]])
 ; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP20:%.*]] = or i1 [[TMP18]], [[TMP19]]
 ; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK:       middle.split:
-; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x i64> [[VEC_IND]], i32 3
 ; CHECK-NEXT:    br i1 [[TMP18]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
@@ -737,7 +723,7 @@ define i64 @same_exit_block_post_inc_use2() {
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX_NEXT]], [[LOOP]] ], [ [[INDEX]], [[LOOP_INC]] ], [ [[TMP21]], [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX_NEXT]], [[LOOP]] ], [ [[INDEX]], [[LOOP_INC]] ], [ 66, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
 entry:
@@ -869,7 +855,6 @@ define i64 @diff_exit_block_pre_inc_use2() {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
@@ -883,13 +868,11 @@ define i64 @diff_exit_block_pre_inc_use2() {
 ; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i1> [[TMP5]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[VEC_IND]], i32 3
 ; CHECK-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       vector.early.exit:
 ; CHECK-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
@@ -912,7 +895,7 @@ define i64 @diff_exit_block_pre_inc_use2() {
 ; CHECK-NEXT:    [[RETVAL1:%.*]] = phi i64 [ 67, [[LOOP]] ], [ 67, [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL1]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL2:%.*]] = phi i64 [ [[INDEX]], [[LOOP_INC]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RETVAL2:%.*]] = phi i64 [ [[INDEX]], [[LOOP_INC]] ], [ 66, [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL2]]
 ;
 entry:
@@ -978,7 +961,6 @@ define i64 @diff_exit_block_pre_inc_use3() {
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[VEC_IND]], i32 3
 ; CHECK-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       vector.early.exit:
 ; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true)
@@ -1003,7 +985,7 @@ define i64 @diff_exit_block_pre_inc_use3() {
 ; CHECK-NEXT:    [[INDEX_LCSSA:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[INDEX_LCSSA]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[INDEX_LCSSA1:%.*]] = phi i64 [ [[INDEX]], [[LOOP_INC]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[INDEX_LCSSA1:%.*]] = phi i64 [ [[INDEX]], [[LOOP_INC]] ], [ 66, [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i64 [[INDEX_LCSSA1]]
 ;
 entry:
@@ -1050,9 +1032,6 @@ define i64 @diff_exit_block_post_inc_use1() {
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP5]], align 1
@@ -1060,10 +1039,6 @@ define i64 @diff_exit_block_post_inc_use1() {
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP7]], align 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP3]], 1
 ; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]])
@@ -1098,7 +1073,7 @@ define i64 @diff_exit_block_post_inc_use1() {
 ; CHECK-NEXT:    [[RETVAL1:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL1]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL2:%.*]] = phi i64 [ [[INDEX_NEXT]], [[LOOP_INC]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RETVAL2:%.*]] = phi i64 [ [[INDEX_NEXT]], [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL2]]
 ;
 entry:
@@ -1144,7 +1119,6 @@ define i64 @diff_exit_block_post_inc_use2() {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 1
@@ -1169,13 +1143,11 @@ define i64 @diff_exit_block_post_inc_use2() {
 ; CHECK-NEXT:    [[TMP17:%.*]] = xor <4 x i1> [[TMP16]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP18:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP17]])
 ; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP20:%.*]] = or i1 [[TMP18]], [[TMP19]]
 ; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    br i1 [[TMP18]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x i64> [[VEC_IND]], i32 3
 ; CHECK-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       vector.early.exit:
 ; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP17]], i1 true)
@@ -1200,7 +1172,7 @@ define i64 @diff_exit_block_post_inc_use2() {
 ; CHECK-NEXT:    [[RETVAL1:%.*]] = phi i64 [ [[INDEX_NEXT]], [[LOOP]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL1]]
 ; CHECK:       loop.end:
-; CHECK-NEXT:    [[RETVAL2:%.*]] = phi i64 [ [[INDEX]], [[LOOP_INC]] ], [ [[TMP21]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RETVAL2:%.*]] = phi i64 [ [[INDEX]], [[LOOP_INC]] ], [ 66, [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL2]]
 ;
 entry:
@@ -1233,6 +1205,114 @@ loop.end:
   ret i64 %retval2
 }
 
+define i64 @diff_exit_block_post_inc_use3(i64 %start) {
+; CHECK-LABEL: define i64 @diff_exit_block_post_inc_use3(
+; CHECK-SAME: i64 [[START:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P1:%.*]] = alloca [1024 x i8], align 1
+; CHECK-NEXT:    [[P2:%.*]] = alloca [1024 x i8], align 1
+; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
+; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[START]], 64
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[OFFSET_IDX2:%.*]] = add i64 [[START]], [[INDEX1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX2]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX2]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX2]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[OFFSET_IDX2]], 3
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP3]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP4]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP5]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x i64> poison, i64 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x i64> [[TMP10]], i64 [[TMP7]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <4 x i64> [[TMP11]], i64 [[TMP8]], i32 2
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x i64> [[TMP12]], i64 [[TMP9]], i32 3
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP15]], align 1
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP17]], align 1
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD3]]
+; CHECK-NEXT:    [[INDEX_NEXT4]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT:    [[TMP19:%.*]] = xor <4 x i1> [[TMP18]], splat (i1 true)
+; CHECK-NEXT:    [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP19]])
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT4]], 64
+; CHECK-NEXT:    [[TMP22:%.*]] = or i1 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    br i1 [[TMP22]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; CHECK:       middle.split:
+; CHECK-NEXT:    br i1 [[TMP20]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[IND_ESCAPE:%.*]] = sub i64 [[TMP0]], 1
+; CHECK-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP19]], i1 true)
+; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i64> [[TMP13]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi i64 [ [[TMP0]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX2:%.*]] = phi i64 [ [[INDEX2_NEXT:%.*]], [[LOOP_INC]] ], [ [[BC_RESUME_VAL5]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[INDEX2_NEXT]] = add i64 [[INDEX2]], 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
+; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
+; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_EARLY_EXIT]]
+; CHECK:       loop.inc:
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK:       loop.early.exit:
+; CHECK-NEXT:    [[RETVAL1:%.*]] = phi i64 [ [[INDEX2_NEXT]], [[LOOP]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT:    ret i64 [[RETVAL1]]
+; CHECK:       loop.end:
+; CHECK-NEXT:    [[RETVAL2:%.*]] = phi i64 [ [[INDEX2]], [[LOOP_INC]] ], [ [[IND_ESCAPE]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i64 [[RETVAL2]]
+;
+entry:
+  %p1 = alloca [1024 x i8]
+  %p2 = alloca [1024 x i8]
+  call void @init_mem(ptr %p1, i64 1024)
+  call void @init_mem(ptr %p2, i64 1024)
+  br label %loop
+
+loop:
+  %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
+  %index2 = phi i64 [ %index2.next, %loop.inc ], [ %start, %entry ]
+  %index.next = add i64 %index, 1
+  %index2.next = add i64 %index2, 1
+  %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+  %ld1 = load i8, ptr %arrayidx, align 1
+  %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+  %ld2 = load i8, ptr %arrayidx1, align 1
+  %cmp3 = icmp eq i8 %ld1, %ld2
+  br i1 %cmp3, label %loop.inc, label %loop.early.exit
+
+loop.inc:
+  %exitcond = icmp ne i64 %index.next, 67
+  br i1 %exitcond, label %loop, label %loop.end
+
+loop.early.exit:
+  %retval1 = phi i64 [ %index2.next, %loop ]
+  ret i64 %retval1
+
+loop.end:
+  %retval2 = phi i64 [ %index2, %loop.inc ]
+  ret i64 %retval2
+}
+
 
 define i64 @loop_contains_safe_call() {
 ; CHECK-LABEL: define i64 @loop_contains_safe_call() {
@@ -1260,7 +1340,7 @@ define i64 @loop_contains_safe_call() {
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
@@ -1282,7 +1362,7 @@ define i64 @loop_contains_safe_call() {
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP33:![0-9]+]]
 ; CHECK:       loop.end:
 ; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
@@ -1339,7 +1419,7 @@ define i64 @loop_contains_safe_div() {
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
@@ -1361,7 +1441,7 @@ define i64 @loop_contains_safe_div() {
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP33:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP35:![0-9]+]]
 ; CHECK:       loop.end:
 ; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
@@ -1419,7 +1499,7 @@ define i64 @loop_contains_load_after_early_exit(ptr dereferenceable(1024) align(
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[WIDE_LOAD2]], i32 3
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
@@ -1443,7 +1523,7 @@ define i64 @loop_contains_load_after_early_exit(ptr dereferenceable(1024) align(
 ; CHECK-NEXT:    [[LD2:%.*]] = load i64, ptr [[ARRAYIDX2]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP35:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP37:![0-9]+]]
 ; CHECK:       loop.end:
 ; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ [[LD2]], [[LOOP_INC]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
@@ -1505,7 +1585,7 @@ define i64 @same_exit_block_pre_inc_use1_reverse() {
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT4]], 1020
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 -4)
 ; CHECK-NEXT:    [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP10]]
-; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
@@ -1528,7 +1608,7 @@ define i64 @same_exit_block_pre_inc_use1_reverse() {
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], -1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDEX_NEXT]], 0
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP_END]], label [[LOOP]], !llvm.loop [[LOOP37:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP_END]], label [[LOOP]], !llvm.loop [[LOOP39:![0-9]+]]
 ; CHECK:       loop.end:
 ; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 1024, [[LOOP_INC]] ], [ 1024, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
@@ -1585,7 +1665,7 @@ define i64 @same_exit_block_pre_inc_use1_deref_ptrs(ptr dereferenceable(1024) %p
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]]
 ; CHECK:       middle.split:
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
@@ -1608,7 +1688,7 @@ define i64 @same_exit_block_pre_inc_use1_deref_ptrs(ptr dereferenceable(1024) %p
 ; CHECK:       loop.inc:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP39:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP41:![0-9]+]]
 ; CHECK:       loop.end:
 ; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
@@ -1681,4 +1761,6 @@ attributes #0 = { "vector-function-abi-variant"="_ZGVsNxv_foo(foo_vec)" }
 ; CHECK: [[LOOP37]] = distinct !{[[LOOP37]], [[META2]], [[META1]]}
 ; CHECK: [[LOOP38]] = distinct !{[[LOOP38]], [[META1]], [[META2]]}
 ; CHECK: [[LOOP39]] = distinct !{[[LOOP39]], [[META2]], [[META1]]}
+; CHECK: [[LOOP40]] = distinct !{[[LOOP40]], [[META1]], [[META2]]}
+; CHECK: [[LOOP41]] = distinct !{[[LOOP41]], [[META2]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
new file mode 100644
index 0000000000000..4a0a5e3be5b19
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
@@ -0,0 +1,157 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-vectorize -enable-vplan-native-path -force-vector-width=4 -S %s | FileCheck %s
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32"
+
+define void @expand(ptr %src, ptr %dst, i64 %0) {
+; CHECK-LABEL: define void @expand(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i64 [[TMP0:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 1
+; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 1000)
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[SMAX]], -1
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 [[TMP2]], [[TMP0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = shl i64 [[TMP0]], 4
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 8
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]]
+; CHECK-NEXT:    [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 8
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP0]], 1
+; CHECK-NEXT:    [[SMAX6:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP6]], i64 1000)
+; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[SMAX6]], 4
+; CHECK-NEXT:    [[SCEVGEP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
+; CHECK-NEXT:    [[SMAX8:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP6]], i64 1000)
+; CHECK-NEXT:    [[TMP8:%.*]] = sub i64 [[SMAX8]], [[TMP0]]
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP8]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK:       [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT:    [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[TMP3]])
+; CHECK-NEXT:    [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = sub i64 0, [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ult ptr [[TMP10]], [[SCEVGEP]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW]]
+; CHECK-NEXT:    [[MUL2:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[TMP3]])
+; CHECK-NEXT:    [[MUL_RESULT3:%.*]] = extractvalue { i64, i1 } [[MUL2]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW4:%.*]] = extractvalue { i64, i1 } [[MUL2]], 1
+; CHECK-NEXT:    [[TMP13:%.*]] = sub i64 0, [[MUL_RESULT3]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 [[MUL_RESULT3]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ult ptr [[TMP14]], [[SCEVGEP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or i1 [[TMP15]], [[MUL_OVERFLOW4]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or i1 [[TMP12]], [[TMP16]]
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP7]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP5]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP8]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP8]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i64> [[DOTSPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[DOTSPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP19:%.*]] = load double, ptr [[SRC]], align 8, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
+; CHECK-NEXT:    [[TMP20:%.*]] = shl <4 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x i64> [[TMP20]], i32 0
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <4 x i64> [[TMP20]], i32 1
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <4 x i64> [[TMP20]], i32 2
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <4 x i64> [[TMP20]], i32 3
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP27]]
+; CHECK-NEXT:    store double [[TMP19]], ptr [[TMP22]], align 8, !alias.scope [[META3]]
+; CHECK-NEXT:    store double [[TMP19]], ptr [[TMP24]], align 8, !alias.scope [[META3]]
+; CHECK-NEXT:    store double [[TMP19]], ptr [[TMP26]], align 8, !alias.scope [[META3]]
+; CHECK-NEXT:    store double [[TMP19]], ptr [[TMP28]], align 8, !alias.scope [[META3]]
+; CHECK-NEXT:    [[TMP29:%.*]] = or disjoint <4 x i64> [[TMP20]], splat (i64 1)
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <4 x i64> [[TMP29]], i32 0
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <4 x i64> [[TMP29]], i32 1
+; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP32]]
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <4 x i64> [[TMP29]], i32 2
+; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP34]]
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <4 x i64> [[TMP29]], i32 3
+; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP36]]
+; CHECK-NEXT:    store double 0.000000e+00, ptr [[TMP31]], align 8, !alias.scope [[META3]]
+; CHECK-NEXT:    store double 0.000000e+00, ptr [[TMP33]], align 8, !alias.scope [[META3]]
+; CHECK-NEXT:    store double 0.000000e+00, ptr [[TMP35]], align 8, !alias.scope [[META3]]
+; CHECK-NEXT:    store double 0.000000e+00, ptr [[TMP37]], align 8, !alias.scope [[META3]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP8]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[OUTER_LATCH]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP18]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[OUTER_HEADER]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[INNER:.*]]
+; CHECK:       [[INNER]]:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT:    [[INNER_IV_NEXT]] = add i64 [[INNER_IV]], 1
+; CHECK-NEXT:    [[L:%.*]] = load double, ptr [[SRC]], align 8
+; CHECK-NEXT:    [[TMP39:%.*]] = shl i64 [[INNER_IV]], 1
+; CHECK-NEXT:    [[ARRAYIDX24:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP39]]
+; CHECK-NEXT:    store double [[L]], ptr [[ARRAYIDX24]], align 8
+; CHECK-NEXT:    [[TMP40:%.*]] = or disjoint i64 [[TMP39]], 1
+; CHECK-NEXT:    [[ARRAYIDX29:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP40]]
+; CHECK-NEXT:    store double 0.000000e+00, ptr [[ARRAYIDX29]], align 8
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i64 [[INNER_IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[CMP2]], label %[[INNER]], label %[[OUTER_LATCH]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add i64 [[OUTER_IV]], 1
+; CHECK-NEXT:    [[OUTER_EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 100
+; CHECK-NEXT:    br i1 [[OUTER_EC]], label %[[EXIT:.*]], label %[[OUTER_HEADER]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+  br label %inner
+
+inner:                                        ; preds = %inner, %outer.header
+  %inner.iv = phi i64 [ %0, %outer.header ], [ %inner.iv.next, %inner ]
+  %inner.iv.next = add i64 %inner.iv, 1
+  %l = load double, ptr %src, align 8
+  %2 = shl i64 %inner.iv, 1
+  %arrayidx24 = getelementptr double, ptr %dst, i64 %2
+  store double %l, ptr %arrayidx24, align 8
+  %3 = or disjoint i64 %2, 1
+  %arrayidx29 = getelementptr double, ptr %dst, i64 %3
+  store double 0.000000e+00, ptr %arrayidx29, align 8
+  %cmp2 = icmp slt i64 %inner.iv.next, 1000
+  br i1 %cmp2, label %inner, label %outer.latch
+
+outer.latch:
+  %outer.iv.next = add i64 %outer.iv, 1
+  %outer.ec = icmp eq i64 %outer.iv.next, 100
+  br i1 %outer.ec, label %exit, label %outer.header
+
+exit:
+  ret void
+}
+;.
+; CHECK: [[META0]] = !{[[META1:![0-9]+]]}
+; CHECK: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}
+; CHECK: [[META2]] = distinct !{[[META2]], !"LVerDomain"}
+; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
+; CHECK: [[META4]] = distinct !{[[META4]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META6:![0-9]+]], [[META7:![0-9]+]]}
+; CHECK: [[META6]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META7]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META6]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
index 52b2bcd9aac11..625a32c098f94 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
@@ -8,62 +8,33 @@
 define void @foo(i64 %n) {
 ; CHECK:      VPlan 'HCFGBuilder: Plain CFG
 ; CHECK-NEXT: {
-; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
-; CHECK-NEXT: Live-in ir<8> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-NEXT: Successor(s): vector.body
 ; CHECK-EMPTY:
-; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: Successor(s): vector loop
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT:   WIDEN-PHI ir<%outer.iv> = phi ir<0>, ir<%outer.iv.next>
+; CHECK-NEXT:   EMIT ir<%gep.1> = getelementptr ir<@arr2>, ir<0>, ir<%outer.iv>
+; CHECK-NEXT:   EMIT store ir<%outer.iv>, ir<%gep.1>
+; CHECK-NEXT:   EMIT ir<%add> = add ir<%outer.iv>, ir<%n>
+; CHECK-NEXT: Successor(s): inner
 ; CHECK-EMPTY:
-; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT:   vector.body:
-; CHECK-NEXT:     WIDEN-PHI ir<%outer.iv> = phi ir<0>, ir<%outer.iv.next>
-; CHECK-NEXT:     EMIT ir<%gep.1> = getelementptr ir<@arr2>, ir<0>, ir<%outer.iv>
-; CHECK-NEXT:     EMIT store ir<%outer.iv>, ir<%gep.1>
-; CHECK-NEXT:     EMIT ir<%add> = add ir<%outer.iv>, ir<%n>
-; CHECK-NEXT:   Successor(s): inner
-; CHECK-EMPTY:
-; CHECK-NEXT:   <x1> inner: {
-; CHECK-NEXT:     inner:
-; CHECK-NEXT:       WIDEN-PHI ir<%inner.iv> = phi ir<0>, ir<%inner.iv.next>
-; CHECK-NEXT:       EMIT ir<%gep.2> = getelementptr ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv>
-; CHECK-NEXT:       EMIT store ir<%add>, ir<%gep.2>
-; CHECK-NEXT:       EMIT ir<%inner.iv.next> = add ir<%inner.iv>, ir<1>
-; CHECK-NEXT:       EMIT ir<%inner.ec> = icmp ir<%inner.iv.next>, ir<8>
-; CHECK-NEXT:       EMIT branch-on-cond ir<%inner.ec>
-; CHECK-NEXT:   No successors
-; CHECK-NEXT:  }
-; CHECK-NEXT:  Successor(s): outer.latch
-; CHECK-EMPTY:
-; CHECK-NEXT:  outer.latch:
-; CHECK-NEXT:     EMIT ir<%outer.iv.next> = add ir<%outer.iv>, ir<1>
-; CHECK-NEXT:     EMIT ir<%outer.ec> = icmp ir<%outer.iv.next>, ir<8>
-; CHECK-NEXT:  Successor(s): vector.latch
-; CHECK-EMPTY:
-; CHECK-NEXT:   vector.latch:
+; CHECK-NEXT: <x1> inner: {
+; CHECK-NEXT:   inner:
+; CHECK-NEXT:     WIDEN-PHI ir<%inner.iv> = phi ir<0>, ir<%inner.iv.next>
+; CHECK-NEXT:     EMIT ir<%gep.2> = getelementptr ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv>
+; CHECK-NEXT:     EMIT store ir<%add>, ir<%gep.2>
+; CHECK-NEXT:     EMIT ir<%inner.iv.next> = add ir<%inner.iv>, ir<1>
+; CHECK-NEXT:     EMIT ir<%inner.ec> = icmp ir<%inner.iv.next>, ir<8>
+; CHECK-NEXT:     EMIT branch-on-cond ir<%inner.ec>
 ; CHECK-NEXT:   No successors
-; CHECK-NEXT:  }
-; CHECK-NEXT: Successor(s): middle.block
-; CHECK-EMPTY:
-; CHECK-NEXT: middle.block:
-; CHECK-NEXT:   EMIT vp<[[C:%.+]]> = icmp eq ir<8>, vp<[[VTC]]>
-; CHECK-NEXT:   EMIT branch-on-cond vp<[[C]]>
-; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<outer.header>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<outer.header>:
-; CHECK-NEXT:   IR   %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
-; CHECK-NEXT:   IR   %gep.1 = getelementptr inbounds [8 x i64], ptr @arr2, i64 0, i64 %outer.iv
-; CHECK-NEXT:   IR   store i64 %outer.iv, ptr %gep.1, align 4
-; CHECK-NEXT:   IR   %add = add nsw i64 %outer.iv, %n
-; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): outer.latch
 ; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: No successors
+; CHECK-NEXT: outer.latch:
+; CHECK-NEXT:   EMIT ir<%outer.iv.next> = add ir<%outer.iv>, ir<1>
+; CHECK-NEXT:   EMIT ir<%outer.ec> = icmp ir<%outer.iv.next>, ir<8>
+; CHECK-NEXT: Successor(s): vector.body
 ; CHECK-NEXT: }
 entry:
   br label %outer.header
diff --git a/llvm/test/Transforms/MemCpyOpt/fca2memcpy.ll b/llvm/test/Transforms/MemCpyOpt/fca2memcpy.ll
index 61e349e01ed91..7d4557aa331c4 100644
--- a/llvm/test/Transforms/MemCpyOpt/fca2memcpy.ll
+++ b/llvm/test/Transforms/MemCpyOpt/fca2memcpy.ll
@@ -51,8 +51,8 @@ define void @destroysrc(ptr %src, ptr %dst) {
 
 define void @destroynoaliassrc(ptr noalias %src, ptr %dst) {
 ; CHECK-LABEL: @destroynoaliassrc(
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DST:%.*]], ptr align 8 [[SRC]], i64 16, i1 false)
-; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[SRC:%.*]], i8 0, i64 16, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DST:%.*]], ptr align 8 [[SRC:%.*]], i64 16, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[SRC]], i8 0, i64 16, i1 false)
 ; CHECK-NEXT:    ret void
 ;
   %1 = load %S, ptr %src
@@ -79,9 +79,9 @@ define void @copyalias(ptr %src, ptr %dst) {
 ; sure we lift the computation as well if needed and possible.
 define void @addrproducer(ptr %src, ptr %dst) {
 ; CHECK-LABEL: @addrproducer(
-; CHECK-NEXT:    [[DST2:%.*]] = getelementptr [[S:%.*]], ptr [[DST]], i64 1
+; CHECK-NEXT:    [[DST2:%.*]] = getelementptr [[S:%.*]], ptr [[DST:%.*]], i64 1
 ; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr align 8 [[DST2]], ptr align 8 [[SRC:%.*]], i64 16, i1 false)
-; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[DST:%.*]], i8 undef, i64 16, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[DST]], i8 undef, i64 16, i1 false)
 ; CHECK-NEXT:    ret void
 ;
   %1 = load %S, ptr %src
@@ -113,8 +113,8 @@ define void @noaliasaddrproducer(ptr %src, ptr noalias %dst, ptr noalias %dstidp
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DSTIDPTR:%.*]], align 4
 ; CHECK-NEXT:    [[DSTINDEX:%.*]] = or i32 [[TMP2]], 1
 ; CHECK-NEXT:    [[DST2:%.*]] = getelementptr [[S:%.*]], ptr [[DST:%.*]], i32 [[DSTINDEX]]
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DST2]], ptr align 8 [[SRC]], i64 16, i1 false)
-; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[SRC:%.*]], i8 undef, i64 16, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DST2]], ptr align 8 [[SRC:%.*]], i64 16, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[SRC]], i8 undef, i64 16, i1 false)
 ; CHECK-NEXT:    ret void
 ;
   %1 = load %S, ptr %src
@@ -130,7 +130,7 @@ define void @throwing_call(ptr noalias %src, ptr %dst) {
 ; CHECK-LABEL: @throwing_call(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load [[S:%.*]], ptr [[SRC:%.*]], align 8
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[SRC]], i8 0, i64 16, i1 false)
-; CHECK-NEXT:    call void @call() [[ATTR2:#.*]]
+; CHECK-NEXT:    call void @call() #[[ATTR2:[0-9]+]]
 ; CHECK-NEXT:    store [[S]] [[TMP1]], ptr [[DST:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
@@ -156,4 +156,30 @@ loop:
   br label %loop
 }
 
+; There are multiple instructions that can clobber the source memory here.
+; We can move the dest write past the store to %ptr.24, but not the memcpy.
+; Make sure we don't perform fca2memcpy conversion in this case.
+define void @multiple_clobbering(ptr %ptr, ptr %ptr.copy) {
+; CHECK-LABEL: @multiple_clobbering(
+; CHECK-NEXT:    [[PTR_8:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR:%.*]], i64 8
+; CHECK-NEXT:    [[PTR_24:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 24
+; CHECK-NEXT:    [[PTR_32:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 32
+; CHECK-NEXT:    [[PTR_COPY_8:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR_COPY:%.*]], i64 8
+; CHECK-NEXT:    [[STRUCT:%.*]] = load { i32, i64 }, ptr [[PTR_COPY_8]], align 8
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[PTR_8]], ptr [[PTR_32]], i64 12, i1 false)
+; CHECK-NEXT:    store i64 1, ptr [[PTR_24]], align 8
+; CHECK-NEXT:    store { i32, i64 } [[STRUCT]], ptr [[PTR_32]], align 8
+; CHECK-NEXT:    ret void
+;
+  %ptr.8 = getelementptr inbounds nuw i8, ptr %ptr, i64 8
+  %ptr.24 = getelementptr inbounds nuw i8, ptr %ptr, i64 24
+  %ptr.32 = getelementptr inbounds nuw i8, ptr %ptr, i64 32
+  %ptr.copy.8 = getelementptr inbounds nuw i8, ptr %ptr.copy, i64 8
+  %struct = load { i32, i64 }, ptr %ptr.copy.8, align 8
+  call void @llvm.memcpy.p0.p0.i64(ptr %ptr.8, ptr %ptr.32, i64 12, i1 false)
+  store i64 1, ptr %ptr.24, align 8
+  store { i32, i64 } %struct, ptr %ptr.32, align 8
+  ret void
+}
+
 declare void @call()
diff --git a/llvm/test/Transforms/MergeFunc/comdat.ll b/llvm/test/Transforms/MergeFunc/comdat.ll
index 3770c772b3f88..ba887f86f7412 100644
--- a/llvm/test/Transforms/MergeFunc/comdat.ll
+++ b/llvm/test/Transforms/MergeFunc/comdat.ll
@@ -1,9 +1,11 @@
 ; RUN: opt -S -passes=mergefunc %s | FileCheck %s
 
+target triple = "x86_64-unknown-windows-msvc19.42.34436"
+
 @symbols = linkonce_odr global <{ ptr, ptr }> <{ ptr @f, ptr @g }>
 
-$f = comdat any
 $g = comdat any
+$f = comdat any
 
 define linkonce_odr hidden i32 @f(i32 %x, i32 %y) comdat {
   %sum = add i32 %x, %y
@@ -19,7 +21,28 @@ define linkonce_odr hidden i32 @g(i32 %x, i32 %y) comdat {
   ret i32 %sum3
 }
 
-; CHECK-DAG: define private i32 @0(i32 %x, i32 %y) comdat($f)
-; CHECK-DAG: define linkonce_odr hidden i32 @g(i32 %0, i32 %1) comdat {
-; CHECK-DAG: define linkonce_odr hidden i32 @f(i32 %0, i32 %1) {
+; CHECK: $g = comdat any
+; CHECK: $f = comdat any
 
+;.
+; CHECK: @symbols = linkonce_odr global <{ ptr, ptr }> <{ ptr @f, ptr @g }>
+;.
+; CHECK-LABEL: define private i32 @0(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[SUM2:%.*]] = add i32 [[X]], [[SUM]]
+; CHECK-NEXT:    [[SUM3:%.*]] = add i32 [[X]], [[SUM]]
+; CHECK-NEXT:    ret i32 [[SUM3]]
+;
+;
+; CHECK-LABEL: define linkonce_odr hidden i32 @g(
+; CHECK-SAME: i32 [[TMP0:%.*]], i32 [[TMP1:%.*]]) comdat {
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @[[GLOB0:[0-9]+]](i32 [[TMP0]], i32 [[TMP1]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+;
+; CHECK-LABEL: define linkonce_odr hidden i32 @f(
+; CHECK-SAME: i32 [[TMP0:%.*]], i32 [[TMP1:%.*]]) comdat {
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @[[GLOB0]](i32 [[TMP0]], i32 [[TMP1]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
diff --git a/llvm/test/Transforms/OpenMP/parallel_region_merging.ll b/llvm/test/Transforms/OpenMP/parallel_region_merging.ll
index bddab532b0d76..d587b9a9fdc54 100644
--- a/llvm/test/Transforms/OpenMP/parallel_region_merging.ll
+++ b/llvm/test/Transforms/OpenMP/parallel_region_merging.ll
@@ -4683,9 +4683,7 @@ entry:
 ; CHECK2-NEXT:    store ptr [[A_ADDR]], ptr [[GEP_A_ADDR]], align 8
 ; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge..omp_par, ptr [[STRUCTARG]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-; CHECK2:       omp.par.outlined.exit:
-; CHECK2-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-; CHECK2:       omp.par.exit.split:
+; CHECK2:       omp.par.exit:
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT_SPLIT:%.*]]
 ; CHECK2:       entry.split.split:
 ; CHECK2-NEXT:    ret void
@@ -4715,7 +4713,7 @@ entry:
 ; CHECK2-NEXT:    br label [[OMP_PAR_PRE_FINALIZE:%.*]]
 ; CHECK2:       omp.par.pre_finalize:
 ; CHECK2-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT_EXITSTUB:%.*]]
-; CHECK2:       omp.par.outlined.exit.exitStub:
+; CHECK2:       omp.par.exit.exitStub:
 ; CHECK2-NEXT:    ret void
 ;
 ;
@@ -4831,9 +4829,7 @@ entry:
 ; CHECK2-NEXT:    store ptr [[A_ADDR]], ptr [[GEP_A_ADDR]], align 8
 ; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_seq..omp_par, ptr [[STRUCTARG]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-; CHECK2:       omp.par.outlined.exit:
-; CHECK2-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-; CHECK2:       omp.par.exit.split:
+; CHECK2:       omp.par.exit:
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT_SPLIT:%.*]]
 ; CHECK2:       entry.split.split:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
@@ -4886,7 +4882,7 @@ entry:
 ; CHECK2:       omp_region.body.split:
 ; CHECK2-NEXT:    call void @__kmpc_end_master(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM]])
 ; CHECK2-NEXT:    br label [[OMP_REGION_END]]
-; CHECK2:       omp.par.outlined.exit.exitStub:
+; CHECK2:       omp.par.exit.exitStub:
 ; CHECK2-NEXT:    ret void
 ;
 ;
@@ -4925,9 +4921,7 @@ entry:
 ; CHECK2-NEXT:    store ptr [[P]], ptr [[GEP_P]], align 8
 ; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_seq_float..omp_par, ptr [[STRUCTARG]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-; CHECK2:       omp.par.outlined.exit:
-; CHECK2-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-; CHECK2:       omp.par.exit.split:
+; CHECK2:       omp.par.exit:
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT_SPLIT:%.*]]
 ; CHECK2:       entry.split.split:
 ; CHECK2-NEXT:    ret void
@@ -4982,7 +4976,7 @@ entry:
 ; CHECK2:       omp_region.body.split:
 ; CHECK2-NEXT:    call void @__kmpc_end_master(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM]])
 ; CHECK2-NEXT:    br label [[OMP_REGION_END]]
-; CHECK2:       omp.par.outlined.exit.exitStub:
+; CHECK2:       omp.par.exit.exitStub:
 ; CHECK2-NEXT:    ret void
 ;
 ;
@@ -5020,9 +5014,7 @@ entry:
 ; CHECK2-NEXT:    store ptr [[A_CASTED_SROA_0_0_INSERT_EXT_SEQ_OUTPUT_ALLOC]], ptr [[GEP_A_CASTED_SROA_0_0_INSERT_EXT_SEQ_OUTPUT_ALLOC]], align 8
 ; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_seq_firstprivate..omp_par, ptr [[STRUCTARG]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-; CHECK2:       omp.par.outlined.exit:
-; CHECK2-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-; CHECK2:       omp.par.exit.split:
+; CHECK2:       omp.par.exit:
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT_SPLIT:%.*]]
 ; CHECK2:       entry.split.split:
 ; CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
@@ -5080,7 +5072,7 @@ entry:
 ; CHECK2:       omp_region.body.split:
 ; CHECK2-NEXT:    call void @__kmpc_end_master(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM]])
 ; CHECK2-NEXT:    br label [[OMP_REGION_END]]
-; CHECK2:       omp.par.outlined.exit.exitStub:
+; CHECK2:       omp.par.exit.exitStub:
 ; CHECK2-NEXT:    ret void
 ;
 ;
@@ -5113,9 +5105,7 @@ entry:
 ; CHECK2-NEXT:    store ptr [[A_ADDR]], ptr [[GEP_A_ADDR]], align 8
 ; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_seq_sink_lt..omp_par, ptr [[STRUCTARG]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-; CHECK2:       omp.par.outlined.exit:
-; CHECK2-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-; CHECK2:       omp.par.exit.split:
+; CHECK2:       omp.par.exit:
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT_SPLIT:%.*]]
 ; CHECK2:       entry.split.split:
 ; CHECK2-NEXT:    ret void
@@ -5169,7 +5159,7 @@ entry:
 ; CHECK2:       omp_region.body.split:
 ; CHECK2-NEXT:    call void @__kmpc_end_master(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM]])
 ; CHECK2-NEXT:    br label [[OMP_REGION_END]]
-; CHECK2:       omp.par.outlined.exit.exitStub:
+; CHECK2:       omp.par.exit.exitStub:
 ; CHECK2-NEXT:    ret void
 ;
 ;
@@ -5210,9 +5200,7 @@ entry:
 ; CHECK2-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B]])
 ; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_seq_par_use..omp_par, ptr [[STRUCTARG]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-; CHECK2:       omp.par.outlined.exit:
-; CHECK2-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-; CHECK2:       omp.par.exit.split:
+; CHECK2:       omp.par.exit:
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT_SPLIT:%.*]]
 ; CHECK2:       entry.split.split:
 ; CHECK2-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[B]])
@@ -5268,7 +5256,7 @@ entry:
 ; CHECK2:       omp_region.body.split:
 ; CHECK2-NEXT:    call void @__kmpc_end_master(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM]])
 ; CHECK2-NEXT:    br label [[OMP_REGION_END]]
-; CHECK2:       omp.par.outlined.exit.exitStub:
+; CHECK2:       omp.par.exit.exitStub:
 ; CHECK2-NEXT:    ret void
 ;
 ;
@@ -5307,9 +5295,7 @@ entry:
 ; CHECK2-NEXT:    store ptr [[CANCEL2_ADDR]], ptr [[GEP_CANCEL2_ADDR]], align 8
 ; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_cancellable_regions..omp_par, ptr [[STRUCTARG]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-; CHECK2:       omp.par.outlined.exit:
-; CHECK2-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-; CHECK2:       omp.par.exit.split:
+; CHECK2:       omp.par.exit:
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT_SPLIT:%.*]]
 ; CHECK2:       entry.split.split:
 ; CHECK2-NEXT:    ret void
@@ -5341,7 +5327,7 @@ entry:
 ; CHECK2-NEXT:    br label [[OMP_PAR_PRE_FINALIZE:%.*]]
 ; CHECK2:       omp.par.pre_finalize:
 ; CHECK2-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT_EXITSTUB:%.*]]
-; CHECK2:       omp.par.outlined.exit.exitStub:
+; CHECK2:       omp.par.exit.exitStub:
 ; CHECK2-NEXT:    ret void
 ;
 ;
@@ -5394,9 +5380,7 @@ entry:
 ; CHECK2-NEXT:    store ptr [[CANCEL2_ADDR]], ptr [[GEP_CANCEL2_ADDR]], align 8
 ; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_cancellable_regions_seq..omp_par, ptr [[STRUCTARG]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-; CHECK2:       omp.par.outlined.exit:
-; CHECK2-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-; CHECK2:       omp.par.exit.split:
+; CHECK2:       omp.par.exit:
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT_SPLIT:%.*]]
 ; CHECK2:       entry.split.split:
 ; CHECK2-NEXT:    ret void
@@ -5452,7 +5436,7 @@ entry:
 ; CHECK2:       omp_region.body.split:
 ; CHECK2-NEXT:    call void @__kmpc_end_master(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM]])
 ; CHECK2-NEXT:    br label [[OMP_REGION_END]]
-; CHECK2:       omp.par.outlined.exit.exitStub:
+; CHECK2:       omp.par.exit.exitStub:
 ; CHECK2-NEXT:    ret void
 ;
 ;
@@ -5497,9 +5481,7 @@ entry:
 ; CHECK2-NEXT:    store ptr [[A_ADDR]], ptr [[GEP_A_ADDR]], align 8
 ; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_3..omp_par, ptr [[STRUCTARG]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-; CHECK2:       omp.par.outlined.exit:
-; CHECK2-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-; CHECK2:       omp.par.exit.split:
+; CHECK2:       omp.par.exit:
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT_SPLIT:%.*]]
 ; CHECK2:       entry.split.split:
 ; CHECK2-NEXT:    ret void
@@ -5532,7 +5514,7 @@ entry:
 ; CHECK2-NEXT:    br label [[OMP_PAR_PRE_FINALIZE:%.*]]
 ; CHECK2:       omp.par.pre_finalize:
 ; CHECK2-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT_EXITSTUB:%.*]]
-; CHECK2:       omp.par.outlined.exit.exitStub:
+; CHECK2:       omp.par.exit.exitStub:
 ; CHECK2-NEXT:    ret void
 ;
 ;
@@ -5583,9 +5565,7 @@ entry:
 ; CHECK2-NEXT:    store ptr [[ADD1_SEQ_OUTPUT_ALLOC]], ptr [[GEP_ADD1_SEQ_OUTPUT_ALLOC]], align 8
 ; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_3_seq..omp_par, ptr [[STRUCTARG]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-; CHECK2:       omp.par.outlined.exit:
-; CHECK2-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-; CHECK2:       omp.par.exit.split:
+; CHECK2:       omp.par.exit:
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT_SPLIT:%.*]]
 ; CHECK2:       entry.split.split:
 ; CHECK2-NEXT:    [[ADD1_SEQ_OUTPUT_LOAD:%.*]] = load i32, ptr [[ADD1_SEQ_OUTPUT_ALLOC]], align 4
@@ -5668,7 +5648,7 @@ entry:
 ; CHECK2:       omp_region.body.split:
 ; CHECK2-NEXT:    call void @__kmpc_end_master(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM]])
 ; CHECK2-NEXT:    br label [[OMP_REGION_END]]
-; CHECK2:       omp.par.outlined.exit.exitStub:
+; CHECK2:       omp.par.exit.exitStub:
 ; CHECK2-NEXT:    ret void
 ;
 ;
@@ -5820,9 +5800,7 @@ entry:
 ; CHECK2-NEXT:    store ptr [[A_ADDR]], ptr [[GEP_A_ADDR]], align 8
 ; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_2_unmergable_1..omp_par, ptr [[STRUCTARG]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
-; CHECK2:       omp.par.outlined.exit:
-; CHECK2-NEXT:    br label [[OMP_PAR_EXIT_SPLIT:%.*]]
-; CHECK2:       omp.par.exit.split:
+; CHECK2:       omp.par.exit:
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT_SPLIT:%.*]]
 ; CHECK2:       entry.split.split:
 ; CHECK2-NEXT:    call void (...) @foo()
@@ -5854,7 +5832,7 @@ entry:
 ; CHECK2-NEXT:    br label [[OMP_PAR_PRE_FINALIZE:%.*]]
 ; CHECK2:       omp.par.pre_finalize:
 ; CHECK2-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT_EXITSTUB:%.*]]
-; CHECK2:       omp.par.outlined.exit.exitStub:
+; CHECK2:       omp.par.exit.exitStub:
 ; CHECK2-NEXT:    ret void
 ;
 ;
diff --git a/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll b/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
index 1927060de868e..e4a5ebdc818e6 100644
--- a/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
+++ b/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
@@ -13,6 +13,11 @@ declare void @bar()
 ; LOWERING: @another_entrypoint_no_callees_ctx_root = global { ptr, ptr, ptr, i8 } zeroinitializer
 ; LOWERING: @__llvm_ctx_profile_callsite = external hidden thread_local global ptr
 ; LOWERING: @__llvm_ctx_profile_expected_callee = external hidden thread_local global ptr
+; LOWERING: @[[GLOB0:[0-9]+]] = internal global { ptr, i8 } zeroinitializer
+; LOWERING: @[[GLOB1:[0-9]+]] = internal global { ptr, i8 } zeroinitializer
+; LOWERING: @[[GLOB2:[0-9]+]] = internal global { ptr, i8 } zeroinitializer
+; LOWERING: @[[GLOB3:[0-9]+]] = internal global { ptr, i8 } zeroinitializer
+; LOWERING: @[[GLOB4:[0-9]+]] = internal global { ptr, i8 } zeroinitializer
 ;.
 define void @foo(i32 %a, ptr %fct) {
 ; INSTRUMENT-LABEL: define void @foo(
@@ -34,7 +39,7 @@ define void @foo(i32 %a, ptr %fct) {
 ;
 ; LOWERING-LABEL: define void @foo(
 ; LOWERING-SAME: i32 [[A:%.*]], ptr [[FCT:%.*]]) !guid [[META0:![0-9]+]] {
-; LOWERING-NEXT:    [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @foo, i64 6699318081062747564, i32 2, i32 2)
+; LOWERING-NEXT:    [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @[[GLOB0]], ptr @foo, i64 6699318081062747564, i32 2, i32 2)
 ; LOWERING-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
 ; LOWERING-NEXT:    [[TMP3:%.*]] = and i64 [[TMP2]], 1
 ; LOWERING-NEXT:    [[TMP4:%.*]] = call ptr @llvm.threadlocal.address.p0(ptr @__llvm_ctx_profile_expected_callee)
@@ -176,7 +181,7 @@ define void @simple(i32 %a) {
 ;
 ; LOWERING-LABEL: define void @simple(
 ; LOWERING-SAME: i32 [[A:%.*]]) !guid [[META3:![0-9]+]] {
-; LOWERING-NEXT:    [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @simple, i64 -3006003237940970099, i32 1, i32 0)
+; LOWERING-NEXT:    [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @[[GLOB1]], ptr @simple, i64 -3006003237940970099, i32 1, i32 0)
 ; LOWERING-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
 ; LOWERING-NEXT:    [[TMP3:%.*]] = and i64 [[TMP2]], -2
 ; LOWERING-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
@@ -200,7 +205,7 @@ define i32 @no_callsites(i32 %a) {
 ;
 ; LOWERING-LABEL: define i32 @no_callsites(
 ; LOWERING-SAME: i32 [[A:%.*]]) !guid [[META4:![0-9]+]] {
-; LOWERING-NEXT:    [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @no_callsites, i64 5679753335911435902, i32 2, i32 0)
+; LOWERING-NEXT:    [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @[[GLOB2]], ptr @no_callsites, i64 5679753335911435902, i32 2, i32 0)
 ; LOWERING-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
 ; LOWERING-NEXT:    [[TMP3:%.*]] = and i64 [[TMP2]], -2
 ; LOWERING-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
@@ -232,7 +237,7 @@ define void @no_counters() {
 ;
 ; LOWERING-LABEL: define void @no_counters(
 ; LOWERING-SAME: ) !guid [[META5:![0-9]+]] {
-; LOWERING-NEXT:    [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @no_counters, i64 5458232184388660970, i32 1, i32 1)
+; LOWERING-NEXT:    [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @[[GLOB3]], ptr @no_counters, i64 5458232184388660970, i32 1, i32 1)
 ; LOWERING-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
 ; LOWERING-NEXT:    [[TMP3:%.*]] = and i64 [[TMP2]], 1
 ; LOWERING-NEXT:    [[TMP4:%.*]] = call ptr @llvm.threadlocal.address.p0(ptr @__llvm_ctx_profile_expected_callee)
@@ -260,7 +265,7 @@ define void @inlineasm() {
 ;
 ; LOWERING-LABEL: define void @inlineasm(
 ; LOWERING-SAME: ) !guid [[META6:![0-9]+]] {
-; LOWERING-NEXT:    [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @inlineasm, i64 -3771893999295659109, i32 1, i32 0)
+; LOWERING-NEXT:    [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @[[GLOB4]], ptr @inlineasm, i64 -3771893999295659109, i32 1, i32 0)
 ; LOWERING-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
 ; LOWERING-NEXT:    [[TMP3:%.*]] = and i64 [[TMP2]], -2
 ; LOWERING-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/blendv-select.ll b/llvm/test/Transforms/PhaseOrdering/X86/blendv-select.ll
index c2ed7b9c84523..444e256f9854b 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/blendv-select.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/blendv-select.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s
-; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s
-; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX512
 
 ;
 ; PR58895 - replace shuffled _mm_blendv_epi8+icmp with select+icmp
@@ -12,10 +12,20 @@
 ;
 
 define <4 x double> @x86_pblendvb_v4f64_v2f64(<4 x double> %a, <4 x double> %b, <4 x double> %c, <4 x double> %d) {
-; CHECK-LABEL: @x86_pblendvb_v4f64_v2f64(
-; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <4 x double> [[C:%.*]], [[D:%.*]]
-; CHECK-NEXT:    [[DOTV:%.*]] = select <4 x i1> [[CMP]], <4 x double> [[B:%.*]], <4 x double> [[A:%.*]]
-; CHECK-NEXT:    ret <4 x double> [[DOTV]]
+; SSE-LABEL: @x86_pblendvb_v4f64_v2f64(
+; SSE-NEXT:    [[TMP1:%.*]] = fcmp olt <4 x double> [[C:%.*]], [[D:%.*]]
+; SSE-NEXT:    [[DOTV:%.*]] = select <4 x i1> [[TMP1]], <4 x double> [[B:%.*]], <4 x double> [[A:%.*]]
+; SSE-NEXT:    ret <4 x double> [[DOTV]]
+;
+; AVX2-LABEL: @x86_pblendvb_v4f64_v2f64(
+; AVX2-NEXT:    [[TMP1:%.*]] = fcmp olt <4 x double> [[C:%.*]], [[D:%.*]]
+; AVX2-NEXT:    [[DOTV:%.*]] = select <4 x i1> [[TMP1]], <4 x double> [[B:%.*]], <4 x double> [[A:%.*]]
+; AVX2-NEXT:    ret <4 x double> [[DOTV]]
+;
+; AVX512-LABEL: @x86_pblendvb_v4f64_v2f64(
+; AVX512-NEXT:    [[CMP:%.*]] = fcmp olt <4 x double> [[C:%.*]], [[D:%.*]]
+; AVX512-NEXT:    [[DOTV:%.*]] = select <4 x i1> [[CMP]], <4 x double> [[B:%.*]], <4 x double> [[A:%.*]]
+; AVX512-NEXT:    ret <4 x double> [[DOTV]]
 ;
   %a.bc = bitcast <4 x double> %a to <32 x i8>
   %b.bc = bitcast <4 x double> %b to <32 x i8>
@@ -36,10 +46,20 @@ define <4 x double> @x86_pblendvb_v4f64_v2f64(<4 x double> %a, <4 x double> %b,
 }
 
 define <8 x float> @x86_pblendvb_v8f32_v4f32(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float> %d) {
-; CHECK-LABEL: @x86_pblendvb_v8f32_v4f32(
-; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <8 x float> [[C:%.*]], [[D:%.*]]
-; CHECK-NEXT:    [[DOTV:%.*]] = select <8 x i1> [[CMP]], <8 x float> [[B:%.*]], <8 x float> [[A:%.*]]
-; CHECK-NEXT:    ret <8 x float> [[DOTV]]
+; SSE-LABEL: @x86_pblendvb_v8f32_v4f32(
+; SSE-NEXT:    [[TMP1:%.*]] = fcmp olt <8 x float> [[C:%.*]], [[D:%.*]]
+; SSE-NEXT:    [[DOTV:%.*]] = select <8 x i1> [[TMP1]], <8 x float> [[B:%.*]], <8 x float> [[A:%.*]]
+; SSE-NEXT:    ret <8 x float> [[DOTV]]
+;
+; AVX2-LABEL: @x86_pblendvb_v8f32_v4f32(
+; AVX2-NEXT:    [[TMP1:%.*]] = fcmp olt <8 x float> [[C:%.*]], [[D:%.*]]
+; AVX2-NEXT:    [[DOTV:%.*]] = select <8 x i1> [[TMP1]], <8 x float> [[B:%.*]], <8 x float> [[A:%.*]]
+; AVX2-NEXT:    ret <8 x float> [[DOTV]]
+;
+; AVX512-LABEL: @x86_pblendvb_v8f32_v4f32(
+; AVX512-NEXT:    [[CMP:%.*]] = fcmp olt <8 x float> [[C:%.*]], [[D:%.*]]
+; AVX512-NEXT:    [[DOTV:%.*]] = select <8 x i1> [[CMP]], <8 x float> [[B:%.*]], <8 x float> [[A:%.*]]
+; AVX512-NEXT:    ret <8 x float> [[DOTV]]
 ;
   %a.bc = bitcast <8 x float> %a to <32 x i8>
   %b.bc = bitcast <8 x float> %b to <32 x i8>
@@ -60,10 +80,20 @@ define <8 x float> @x86_pblendvb_v8f32_v4f32(<8 x float> %a, <8 x float> %b, <8
 }
 
 define <4 x i64> @x86_pblendvb_v4i64_v2i64(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c, <4 x i64> %d) {
-; CHECK-LABEL: @x86_pblendvb_v4i64_v2i64(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <4 x i64> [[C:%.*]], [[D:%.*]]
-; CHECK-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[CMP]], <4 x i64> [[B:%.*]], <4 x i64> [[A:%.*]]
-; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+; SSE-LABEL: @x86_pblendvb_v4i64_v2i64(
+; SSE-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i64> [[C:%.*]], [[D:%.*]]
+; SSE-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[B:%.*]], <4 x i64> [[A:%.*]]
+; SSE-NEXT:    ret <4 x i64> [[TMP2]]
+;
+; AVX2-LABEL: @x86_pblendvb_v4i64_v2i64(
+; AVX2-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i64> [[C:%.*]], [[D:%.*]]
+; AVX2-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[B:%.*]], <4 x i64> [[A:%.*]]
+; AVX2-NEXT:    ret <4 x i64> [[TMP2]]
+;
+; AVX512-LABEL: @x86_pblendvb_v4i64_v2i64(
+; AVX512-NEXT:    [[CMP:%.*]] = icmp slt <4 x i64> [[C:%.*]], [[D:%.*]]
+; AVX512-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[CMP]], <4 x i64> [[B:%.*]], <4 x i64> [[A:%.*]]
+; AVX512-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %a.bc = bitcast <4 x i64> %a to <32 x i8>
   %b.bc = bitcast <4 x i64> %b to <32 x i8>
@@ -84,15 +114,35 @@ define <4 x i64> @x86_pblendvb_v4i64_v2i64(<4 x i64> %a, <4 x i64> %b, <4 x i64>
 }
 
 define <4 x i64> @x86_pblendvb_v8i32_v4i32(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c, <4 x i64> %d) {
-; CHECK-LABEL: @x86_pblendvb_v8i32_v4i32(
-; CHECK-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <8 x i32>
-; CHECK-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <8 x i32>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <8 x i32> [[C_BC]], [[D_BC]]
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i64> [[A:%.*]] to <8 x i32>
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[B:%.*]] to <8 x i32>
-; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[CMP]], <8 x i32> [[TMP2]], <8 x i32> [[TMP1]]
-; CHECK-NEXT:    [[RES:%.*]] = bitcast <8 x i32> [[TMP3]] to <4 x i64>
-; CHECK-NEXT:    ret <4 x i64> [[RES]]
+; SSE-LABEL: @x86_pblendvb_v8i32_v4i32(
+; SSE-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <8 x i32>
+; SSE-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <8 x i32>
+; SSE-NEXT:    [[CMP:%.*]] = icmp slt <8 x i32> [[C_BC]], [[D_BC]]
+; SSE-NEXT:    [[TMP1:%.*]] = bitcast <4 x i64> [[A:%.*]] to <8 x i32>
+; SSE-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[B:%.*]] to <8 x i32>
+; SSE-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[CMP]], <8 x i32> [[TMP2]], <8 x i32> [[TMP1]]
+; SSE-NEXT:    [[RES:%.*]] = bitcast <8 x i32> [[TMP3]] to <4 x i64>
+; SSE-NEXT:    ret <4 x i64> [[RES]]
+;
+; AVX2-LABEL: @x86_pblendvb_v8i32_v4i32(
+; AVX2-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <8 x i32>
+; AVX2-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <8 x i32>
+; AVX2-NEXT:    [[TMP1:%.*]] = icmp slt <8 x i32> [[C_BC]], [[D_BC]]
+; AVX2-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[B:%.*]] to <8 x i32>
+; AVX2-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[A:%.*]] to <8 x i32>
+; AVX2-NEXT:    [[TMP4:%.*]] = select <8 x i1> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> [[TMP3]]
+; AVX2-NEXT:    [[RES:%.*]] = bitcast <8 x i32> [[TMP4]] to <4 x i64>
+; AVX2-NEXT:    ret <4 x i64> [[RES]]
+;
+; AVX512-LABEL: @x86_pblendvb_v8i32_v4i32(
+; AVX512-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <8 x i32>
+; AVX512-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <8 x i32>
+; AVX512-NEXT:    [[CMP:%.*]] = icmp slt <8 x i32> [[C_BC]], [[D_BC]]
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast <4 x i64> [[A:%.*]] to <8 x i32>
+; AVX512-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[B:%.*]] to <8 x i32>
+; AVX512-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[CMP]], <8 x i32> [[TMP2]], <8 x i32> [[TMP1]]
+; AVX512-NEXT:    [[RES:%.*]] = bitcast <8 x i32> [[TMP3]] to <4 x i64>
+; AVX512-NEXT:    ret <4 x i64> [[RES]]
 ;
   %a.bc = bitcast <4 x i64> %a to <32 x i8>
   %b.bc = bitcast <4 x i64> %b to <32 x i8>
@@ -115,15 +165,35 @@ define <4 x i64> @x86_pblendvb_v8i32_v4i32(<4 x i64> %a, <4 x i64> %b, <4 x i64>
 }
 
 define <4 x i64> @x86_pblendvb_v16i16_v8i16(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c, <4 x i64> %d) {
-; CHECK-LABEL: @x86_pblendvb_v16i16_v8i16(
-; CHECK-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <16 x i16>
-; CHECK-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <16 x i16>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <16 x i16> [[C_BC]], [[D_BC]]
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i64> [[A:%.*]] to <16 x i16>
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[B:%.*]] to <16 x i16>
-; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[CMP]], <16 x i16> [[TMP2]], <16 x i16> [[TMP1]]
-; CHECK-NEXT:    [[RES:%.*]] = bitcast <16 x i16> [[TMP3]] to <4 x i64>
-; CHECK-NEXT:    ret <4 x i64> [[RES]]
+; SSE-LABEL: @x86_pblendvb_v16i16_v8i16(
+; SSE-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <16 x i16>
+; SSE-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <16 x i16>
+; SSE-NEXT:    [[CMP:%.*]] = icmp slt <16 x i16> [[C_BC]], [[D_BC]]
+; SSE-NEXT:    [[TMP1:%.*]] = bitcast <4 x i64> [[A:%.*]] to <16 x i16>
+; SSE-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[B:%.*]] to <16 x i16>
+; SSE-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[CMP]], <16 x i16> [[TMP2]], <16 x i16> [[TMP1]]
+; SSE-NEXT:    [[RES:%.*]] = bitcast <16 x i16> [[TMP3]] to <4 x i64>
+; SSE-NEXT:    ret <4 x i64> [[RES]]
+;
+; AVX2-LABEL: @x86_pblendvb_v16i16_v8i16(
+; AVX2-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <16 x i16>
+; AVX2-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <16 x i16>
+; AVX2-NEXT:    [[TMP1:%.*]] = icmp slt <16 x i16> [[C_BC]], [[D_BC]]
+; AVX2-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[B:%.*]] to <16 x i16>
+; AVX2-NEXT:    [[TMP3:%.*]] = bitcast <4 x i64> [[A:%.*]] to <16 x i16>
+; AVX2-NEXT:    [[TMP4:%.*]] = select <16 x i1> [[TMP1]], <16 x i16> [[TMP2]], <16 x i16> [[TMP3]]
+; AVX2-NEXT:    [[RES:%.*]] = bitcast <16 x i16> [[TMP4]] to <4 x i64>
+; AVX2-NEXT:    ret <4 x i64> [[RES]]
+;
+; AVX512-LABEL: @x86_pblendvb_v16i16_v8i16(
+; AVX512-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <16 x i16>
+; AVX512-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <16 x i16>
+; AVX512-NEXT:    [[CMP:%.*]] = icmp slt <16 x i16> [[C_BC]], [[D_BC]]
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast <4 x i64> [[A:%.*]] to <16 x i16>
+; AVX512-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[B:%.*]] to <16 x i16>
+; AVX512-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[CMP]], <16 x i16> [[TMP2]], <16 x i16> [[TMP1]]
+; AVX512-NEXT:    [[RES:%.*]] = bitcast <16 x i16> [[TMP3]] to <4 x i64>
+; AVX512-NEXT:    ret <4 x i64> [[RES]]
 ;
   %a.bc = bitcast <4 x i64> %a to <32 x i8>
   %b.bc = bitcast <4 x i64> %b to <32 x i8>
@@ -146,15 +216,35 @@ define <4 x i64> @x86_pblendvb_v16i16_v8i16(<4 x i64> %a, <4 x i64> %b, <4 x i64
 }
 
 define <4 x i64> @x86_pblendvb_v32i8_v16i8(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c, <4 x i64> %d) {
-; CHECK-LABEL: @x86_pblendvb_v32i8_v16i8(
-; CHECK-NEXT:    [[A_BC:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8>
-; CHECK-NEXT:    [[B_BC:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8>
-; CHECK-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <32 x i8>
-; CHECK-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <32 x i8>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <32 x i8> [[C_BC]], [[D_BC]]
-; CHECK-NEXT:    [[CONCAT:%.*]] = select <32 x i1> [[CMP]], <32 x i8> [[B_BC]], <32 x i8> [[A_BC]]
-; CHECK-NEXT:    [[RES:%.*]] = bitcast <32 x i8> [[CONCAT]] to <4 x i64>
-; CHECK-NEXT:    ret <4 x i64> [[RES]]
+; SSE-LABEL: @x86_pblendvb_v32i8_v16i8(
+; SSE-NEXT:    [[TMP1:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8>
+; SSE-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8>
+; SSE-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <32 x i8>
+; SSE-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <32 x i8>
+; SSE-NEXT:    [[TMP3:%.*]] = icmp slt <32 x i8> [[C_BC]], [[D_BC]]
+; SSE-NEXT:    [[CONCAT:%.*]] = select <32 x i1> [[TMP3]], <32 x i8> [[TMP2]], <32 x i8> [[TMP1]]
+; SSE-NEXT:    [[RES:%.*]] = bitcast <32 x i8> [[CONCAT]] to <4 x i64>
+; SSE-NEXT:    ret <4 x i64> [[RES]]
+;
+; AVX2-LABEL: @x86_pblendvb_v32i8_v16i8(
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8>
+; AVX2-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8>
+; AVX2-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <32 x i8>
+; AVX2-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <32 x i8>
+; AVX2-NEXT:    [[TMP3:%.*]] = icmp slt <32 x i8> [[C_BC]], [[D_BC]]
+; AVX2-NEXT:    [[CONCAT:%.*]] = select <32 x i1> [[TMP3]], <32 x i8> [[TMP2]], <32 x i8> [[TMP1]]
+; AVX2-NEXT:    [[RES:%.*]] = bitcast <32 x i8> [[CONCAT]] to <4 x i64>
+; AVX2-NEXT:    ret <4 x i64> [[RES]]
+;
+; AVX512-LABEL: @x86_pblendvb_v32i8_v16i8(
+; AVX512-NEXT:    [[A_BC:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8>
+; AVX512-NEXT:    [[B_BC:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8>
+; AVX512-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <32 x i8>
+; AVX512-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <32 x i8>
+; AVX512-NEXT:    [[CMP:%.*]] = icmp slt <32 x i8> [[C_BC]], [[D_BC]]
+; AVX512-NEXT:    [[CONCAT:%.*]] = select <32 x i1> [[CMP]], <32 x i8> [[B_BC]], <32 x i8> [[A_BC]]
+; AVX512-NEXT:    [[RES:%.*]] = bitcast <32 x i8> [[CONCAT]] to <4 x i64>
+; AVX512-NEXT:    ret <4 x i64> [[RES]]
 ;
   %a.bc = bitcast <4 x i64> %a to <32 x i8>
   %b.bc = bitcast <4 x i64> %b to <32 x i8>
@@ -283,15 +373,35 @@ define <8 x i64> @x86_pblendvb_v16i32_v8i32(<8 x i64> %a, <8 x i64> %b, <8 x i64
 }
 
 define <8 x i64> @x86_pblendvb_v32i16_v16i16(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d) {
-; CHECK-LABEL: @x86_pblendvb_v32i16_v16i16(
-; CHECK-NEXT:    [[C_BC:%.*]] = bitcast <8 x i64> [[C:%.*]] to <32 x i16>
-; CHECK-NEXT:    [[D_BC:%.*]] = bitcast <8 x i64> [[D:%.*]] to <32 x i16>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <32 x i16> [[C_BC]], [[D_BC]]
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i64> [[A:%.*]] to <32 x i16>
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[B:%.*]] to <32 x i16>
-; CHECK-NEXT:    [[TMP3:%.*]] = select <32 x i1> [[CMP]], <32 x i16> [[TMP2]], <32 x i16> [[TMP1]]
-; CHECK-NEXT:    [[RES:%.*]] = bitcast <32 x i16> [[TMP3]] to <8 x i64>
-; CHECK-NEXT:    ret <8 x i64> [[RES]]
+; SSE-LABEL: @x86_pblendvb_v32i16_v16i16(
+; SSE-NEXT:    [[C_BC:%.*]] = bitcast <8 x i64> [[C:%.*]] to <32 x i16>
+; SSE-NEXT:    [[D_BC:%.*]] = bitcast <8 x i64> [[D:%.*]] to <32 x i16>
+; SSE-NEXT:    [[TMP1:%.*]] = icmp slt <32 x i16> [[C_BC]], [[D_BC]]
+; SSE-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[B:%.*]] to <32 x i16>
+; SSE-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64> [[A:%.*]] to <32 x i16>
+; SSE-NEXT:    [[TMP4:%.*]] = select <32 x i1> [[TMP1]], <32 x i16> [[TMP2]], <32 x i16> [[TMP3]]
+; SSE-NEXT:    [[RES:%.*]] = bitcast <32 x i16> [[TMP4]] to <8 x i64>
+; SSE-NEXT:    ret <8 x i64> [[RES]]
+;
+; AVX2-LABEL: @x86_pblendvb_v32i16_v16i16(
+; AVX2-NEXT:    [[C_BC:%.*]] = bitcast <8 x i64> [[C:%.*]] to <32 x i16>
+; AVX2-NEXT:    [[D_BC:%.*]] = bitcast <8 x i64> [[D:%.*]] to <32 x i16>
+; AVX2-NEXT:    [[CMP:%.*]] = icmp slt <32 x i16> [[C_BC]], [[D_BC]]
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast <8 x i64> [[A:%.*]] to <32 x i16>
+; AVX2-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[B:%.*]] to <32 x i16>
+; AVX2-NEXT:    [[TMP3:%.*]] = select <32 x i1> [[CMP]], <32 x i16> [[TMP2]], <32 x i16> [[TMP1]]
+; AVX2-NEXT:    [[RES:%.*]] = bitcast <32 x i16> [[TMP3]] to <8 x i64>
+; AVX2-NEXT:    ret <8 x i64> [[RES]]
+;
+; AVX512-LABEL: @x86_pblendvb_v32i16_v16i16(
+; AVX512-NEXT:    [[C_BC:%.*]] = bitcast <8 x i64> [[C:%.*]] to <32 x i16>
+; AVX512-NEXT:    [[D_BC:%.*]] = bitcast <8 x i64> [[D:%.*]] to <32 x i16>
+; AVX512-NEXT:    [[CMP:%.*]] = icmp slt <32 x i16> [[C_BC]], [[D_BC]]
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast <8 x i64> [[A:%.*]] to <32 x i16>
+; AVX512-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[B:%.*]] to <32 x i16>
+; AVX512-NEXT:    [[TMP3:%.*]] = select <32 x i1> [[CMP]], <32 x i16> [[TMP2]], <32 x i16> [[TMP1]]
+; AVX512-NEXT:    [[RES:%.*]] = bitcast <32 x i16> [[TMP3]] to <8 x i64>
+; AVX512-NEXT:    ret <8 x i64> [[RES]]
 ;
   %a.bc = bitcast <8 x i64> %a to <64 x i8>
   %b.bc = bitcast <8 x i64> %b to <64 x i8>
@@ -314,15 +424,35 @@ define <8 x i64> @x86_pblendvb_v32i16_v16i16(<8 x i64> %a, <8 x i64> %b, <8 x i6
 }
 
 define <8 x i64> @x86_pblendvb_v64i8_v32i8(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d) {
-; CHECK-LABEL: @x86_pblendvb_v64i8_v32i8(
-; CHECK-NEXT:    [[A_BC:%.*]] = bitcast <8 x i64> [[A:%.*]] to <64 x i8>
-; CHECK-NEXT:    [[B_BC:%.*]] = bitcast <8 x i64> [[B:%.*]] to <64 x i8>
-; CHECK-NEXT:    [[C_BC:%.*]] = bitcast <8 x i64> [[C:%.*]] to <64 x i8>
-; CHECK-NEXT:    [[D_BC:%.*]] = bitcast <8 x i64> [[D:%.*]] to <64 x i8>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <64 x i8> [[C_BC]], [[D_BC]]
-; CHECK-NEXT:    [[CONCAT:%.*]] = select <64 x i1> [[CMP]], <64 x i8> [[B_BC]], <64 x i8> [[A_BC]]
-; CHECK-NEXT:    [[RES:%.*]] = bitcast <64 x i8> [[CONCAT]] to <8 x i64>
-; CHECK-NEXT:    ret <8 x i64> [[RES]]
+; SSE-LABEL: @x86_pblendvb_v64i8_v32i8(
+; SSE-NEXT:    [[TMP1:%.*]] = bitcast <8 x i64> [[A:%.*]] to <64 x i8>
+; SSE-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[B:%.*]] to <64 x i8>
+; SSE-NEXT:    [[C_BC:%.*]] = bitcast <8 x i64> [[C:%.*]] to <64 x i8>
+; SSE-NEXT:    [[D_BC:%.*]] = bitcast <8 x i64> [[D:%.*]] to <64 x i8>
+; SSE-NEXT:    [[TMP3:%.*]] = icmp slt <64 x i8> [[C_BC]], [[D_BC]]
+; SSE-NEXT:    [[CONCAT:%.*]] = select <64 x i1> [[TMP3]], <64 x i8> [[TMP2]], <64 x i8> [[TMP1]]
+; SSE-NEXT:    [[RES:%.*]] = bitcast <64 x i8> [[CONCAT]] to <8 x i64>
+; SSE-NEXT:    ret <8 x i64> [[RES]]
+;
+; AVX2-LABEL: @x86_pblendvb_v64i8_v32i8(
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast <8 x i64> [[A:%.*]] to <64 x i8>
+; AVX2-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[B:%.*]] to <64 x i8>
+; AVX2-NEXT:    [[C_BC:%.*]] = bitcast <8 x i64> [[C:%.*]] to <64 x i8>
+; AVX2-NEXT:    [[D_BC:%.*]] = bitcast <8 x i64> [[D:%.*]] to <64 x i8>
+; AVX2-NEXT:    [[TMP3:%.*]] = icmp slt <64 x i8> [[C_BC]], [[D_BC]]
+; AVX2-NEXT:    [[CONCAT:%.*]] = select <64 x i1> [[TMP3]], <64 x i8> [[TMP2]], <64 x i8> [[TMP1]]
+; AVX2-NEXT:    [[RES:%.*]] = bitcast <64 x i8> [[CONCAT]] to <8 x i64>
+; AVX2-NEXT:    ret <8 x i64> [[RES]]
+;
+; AVX512-LABEL: @x86_pblendvb_v64i8_v32i8(
+; AVX512-NEXT:    [[A_BC:%.*]] = bitcast <8 x i64> [[A:%.*]] to <64 x i8>
+; AVX512-NEXT:    [[B_BC:%.*]] = bitcast <8 x i64> [[B:%.*]] to <64 x i8>
+; AVX512-NEXT:    [[C_BC:%.*]] = bitcast <8 x i64> [[C:%.*]] to <64 x i8>
+; AVX512-NEXT:    [[D_BC:%.*]] = bitcast <8 x i64> [[D:%.*]] to <64 x i8>
+; AVX512-NEXT:    [[CMP:%.*]] = icmp slt <64 x i8> [[C_BC]], [[D_BC]]
+; AVX512-NEXT:    [[CONCAT:%.*]] = select <64 x i1> [[CMP]], <64 x i8> [[B_BC]], <64 x i8> [[A_BC]]
+; AVX512-NEXT:    [[RES:%.*]] = bitcast <64 x i8> [[CONCAT]] to <8 x i64>
+; AVX512-NEXT:    ret <8 x i64> [[RES]]
 ;
   %a.bc = bitcast <8 x i64> %a to <64 x i8>
   %b.bc = bitcast <8 x i64> %b to <64 x i8>
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/div.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/div.ll
index bb88edff11634..e9cf1deac8eed 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/div.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/div.ll
@@ -433,26 +433,10 @@ define <8 x i16> @slp_v8i16_Op1_Op2_unknown(<8 x i16> %a, <8 x i16> %b)
 }
 
 define <4 x i32> @slp_v4i32_Op1_unknown_Op2_const(<4 x i32> %a)
-; NO-SVE-LABEL: define <4 x i32> @slp_v4i32_Op1_unknown_Op2_const(
-; NO-SVE-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
-; NO-SVE-NEXT:    [[A0:%.*]] = extractelement <4 x i32> [[A]], i32 0
-; NO-SVE-NEXT:    [[A1:%.*]] = extractelement <4 x i32> [[A]], i32 1
-; NO-SVE-NEXT:    [[A2:%.*]] = extractelement <4 x i32> [[A]], i32 2
-; NO-SVE-NEXT:    [[A3:%.*]] = extractelement <4 x i32> [[A]], i32 3
-; NO-SVE-NEXT:    [[TMP1:%.*]] = sdiv i32 [[A0]], 1
-; NO-SVE-NEXT:    [[TMP2:%.*]] = sdiv i32 [[A1]], 3
-; NO-SVE-NEXT:    [[TMP3:%.*]] = sdiv i32 [[A2]], 5
-; NO-SVE-NEXT:    [[TMP4:%.*]] = sdiv i32 [[A3]], 7
-; NO-SVE-NEXT:    [[R0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i32 0
-; NO-SVE-NEXT:    [[R1:%.*]] = insertelement <4 x i32> [[R0]], i32 [[TMP2]], i32 1
-; NO-SVE-NEXT:    [[R2:%.*]] = insertelement <4 x i32> [[R1]], i32 [[TMP3]], i32 2
-; NO-SVE-NEXT:    [[R3:%.*]] = insertelement <4 x i32> [[R2]], i32 [[TMP4]], i32 3
-; NO-SVE-NEXT:    ret <4 x i32> [[R3]]
-;
-; SVE-LABEL: define <4 x i32> @slp_v4i32_Op1_unknown_Op2_const(
-; SVE-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
-; SVE-NEXT:    [[TMP1:%.*]] = sdiv <4 x i32> [[A]], <i32 1, i32 3, i32 5, i32 7>
-; SVE-NEXT:    ret <4 x i32> [[TMP1]]
+; CHECK-LABEL: define <4 x i32> @slp_v4i32_Op1_unknown_Op2_const(
+; CHECK-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = sdiv <4 x i32> [[A]], <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
 {
   %a0 = extractelement <4 x i32> %a, i32 0
@@ -472,7 +456,7 @@ define <4 x i32> @slp_v4i32_Op1_unknown_Op2_const(<4 x i32> %a)
 
 define <4 x i32> @slp_v4i32_Op1_unknown_Op2_uniform_const(<4 x i32> %a)
 ; CHECK-LABEL: define <4 x i32> @slp_v4i32_Op1_unknown_Op2_uniform_const(
-; CHECK-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = sdiv <4 x i32> [[A]], splat (i32 5)
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
@@ -515,26 +499,10 @@ define <4 x i32> @slp_v4i32_Op1_unknown_Op2_uniform_const_pow2(<4 x i32> %a)
 }
 
 define <4 x i32> @slp_v4i32_Op1_unknown_Op2_const_pow2(<4 x i32> %a)
-; NO-SVE-LABEL: define <4 x i32> @slp_v4i32_Op1_unknown_Op2_const_pow2(
-; NO-SVE-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
-; NO-SVE-NEXT:    [[A0:%.*]] = extractelement <4 x i32> [[A]], i32 0
-; NO-SVE-NEXT:    [[A1:%.*]] = extractelement <4 x i32> [[A]], i32 1
-; NO-SVE-NEXT:    [[A2:%.*]] = extractelement <4 x i32> [[A]], i32 2
-; NO-SVE-NEXT:    [[A3:%.*]] = extractelement <4 x i32> [[A]], i32 3
-; NO-SVE-NEXT:    [[TMP1:%.*]] = sdiv i32 [[A0]], 1
-; NO-SVE-NEXT:    [[TMP2:%.*]] = sdiv i32 [[A1]], 2
-; NO-SVE-NEXT:    [[TMP3:%.*]] = sdiv i32 [[A2]], 4
-; NO-SVE-NEXT:    [[TMP4:%.*]] = sdiv i32 [[A3]], 8
-; NO-SVE-NEXT:    [[R0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i32 0
-; NO-SVE-NEXT:    [[R1:%.*]] = insertelement <4 x i32> [[R0]], i32 [[TMP2]], i32 1
-; NO-SVE-NEXT:    [[R2:%.*]] = insertelement <4 x i32> [[R1]], i32 [[TMP3]], i32 2
-; NO-SVE-NEXT:    [[R3:%.*]] = insertelement <4 x i32> [[R2]], i32 [[TMP4]], i32 3
-; NO-SVE-NEXT:    ret <4 x i32> [[R3]]
-;
-; SVE-LABEL: define <4 x i32> @slp_v4i32_Op1_unknown_Op2_const_pow2(
-; SVE-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
-; SVE-NEXT:    [[TMP1:%.*]] = sdiv <4 x i32> [[A]], <i32 1, i32 2, i32 4, i32 8>
-; SVE-NEXT:    ret <4 x i32> [[TMP1]]
+; CHECK-LABEL: define <4 x i32> @slp_v4i32_Op1_unknown_Op2_const_pow2(
+; CHECK-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = sdiv <4 x i32> [[A]], <i32 1, i32 2, i32 4, i32 8>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
 {
   %a0 = extractelement <4 x i32> %a, i32 0
diff --git a/llvm/test/Transforms/SandboxVectorizer/regions-from-bbs.ll b/llvm/test/Transforms/SandboxVectorizer/regions-from-bbs.ll
new file mode 100644
index 0000000000000..065baadb1478b
--- /dev/null
+++ b/llvm/test/Transforms/SandboxVectorizer/regions-from-bbs.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt --passes=sandbox-vectorizer -sbvec-passes='regions-from-bbs<null>' %s -S | FileCheck %s
+
+define void @foo(i8 %v) {
+; CHECK-LABEL: define void @foo(
+; CHECK-SAME: i8 [[V:%.*]]) {
+; CHECK-NEXT:  [[BBA:.*:]]
+; CHECK-NEXT:    [[ADD0:%.*]] = add i8 [[V]], 0, !sandboxvec [[META0:![0-9]+]]
+; CHECK-NEXT:    br label %[[BBB:.*]], !sandboxvec [[META0]]
+; CHECK:       [[BBB]]:
+; CHECK-NEXT:    [[ADD1:%.*]] = add i8 [[V]], 1, !sandboxvec [[META1:![0-9]+]]
+; CHECK-NEXT:    br label %[[BBC:.*]], !sandboxvec [[META1]]
+; CHECK:       [[BBC]]:
+; CHECK-NEXT:    ret void, !sandboxvec [[META2:![0-9]+]]
+;
+bbA:
+  %add0 = add i8 %v, 0
+  br label %bbB
+
+bbB:
+  %add1 = add i8 %v, 1
+  br label %bbC
+
+bbC:
+  ret void
+}
+;.
+; CHECK: [[META0]] = distinct !{!"sandboxregion"}
+; CHECK: [[META1]] = distinct !{!"sandboxregion"}
+; CHECK: [[META2]] = distinct !{!"sandboxregion"}
+;.
diff --git a/llvm/test/Transforms/StructurizeCFG/AMDGPU/loop-subregion-misordered.ll b/llvm/test/Transforms/StructurizeCFG/AMDGPU/loop-subregion-misordered.ll
index 385e37e2750d1..7c2d8b69bfbb3 100644
--- a/llvm/test/Transforms/StructurizeCFG/AMDGPU/loop-subregion-misordered.ll
+++ b/llvm/test/Transforms/StructurizeCFG/AMDGPU/loop-subregion-misordered.ll
@@ -38,7 +38,7 @@ define amdgpu_kernel void @loop_subregion_misordered(ptr addrspace(1) %arg0) #0
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne i32 [[TMP16]], 1
 ; CHECK-NEXT:    br i1 [[TMP17]], label [[BB62:%.*]], label [[FLOW:%.*]]
 ; CHECK:       Flow1:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[INC_I:%.*]], [[INCREMENT_I:%.*]] ], [ undef, [[BB62]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[INC_I:%.*]], [[INCREMENT_I:%.*]] ], [ poison, [[BB62]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ false, [[INCREMENT_I]] ], [ true, [[BB62]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = phi i1 [ true, [[INCREMENT_I]] ], [ false, [[BB62]] ]
 ; CHECK-NEXT:    br label [[FLOW]]
@@ -74,7 +74,7 @@ define amdgpu_kernel void @loop_subregion_misordered(ptr addrspace(1) %arg0) #0
 ; CHECK-NEXT:    call void asm sideeffect "s_nop 42", "~{memory}"() #[[ATTR0]]
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       Flow:
-; CHECK-NEXT:    [[TMP6]] = phi i32 [ [[TMP0]], [[FLOW1]] ], [ undef, [[LOOP_HEADER]] ]
+; CHECK-NEXT:    [[TMP6]] = phi i32 [ [[TMP0]], [[FLOW1]] ], [ poison, [[LOOP_HEADER]] ]
 ; CHECK-NEXT:    [[TMP7]] = phi i1 [ [[TMP1]], [[FLOW1]] ], [ false, [[LOOP_HEADER]] ]
 ; CHECK-NEXT:    [[TMP8]] = phi i1 [ [[TMP2]], [[FLOW1]] ], [ false, [[LOOP_HEADER]] ]
 ; CHECK-NEXT:    [[TMP9:%.*]] = phi i1 [ false, [[FLOW1]] ], [ true, [[LOOP_HEADER]] ]
diff --git a/llvm/test/Transforms/StructurizeCFG/AMDGPU/uniform-regions.ll b/llvm/test/Transforms/StructurizeCFG/AMDGPU/uniform-regions.ll
index 34c73ab8fd74f..7a2ba286eac1b 100644
--- a/llvm/test/Transforms/StructurizeCFG/AMDGPU/uniform-regions.ll
+++ b/llvm/test/Transforms/StructurizeCFG/AMDGPU/uniform-regions.ll
@@ -7,9 +7,9 @@ define amdgpu_cs void @uniform(i32 inreg %v) {
 ; CHECK-LABEL: @uniform(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CC:%.*]] = icmp eq i32 [[V:%.*]], 0
-; CHECK-NEXT:    br i1 [[CC]], label [[IF:%.*]], label [[END:%.*]], !structurizecfg.uniform !0
+; CHECK-NEXT:    br i1 [[CC]], label [[IF:%.*]], label [[END:%.*]], !structurizecfg.uniform [[META0:![0-9]+]]
 ; CHECK:       if:
-; CHECK-NEXT:    br label [[END]], !structurizecfg.uniform !0
+; CHECK-NEXT:    br label [[END]], !structurizecfg.uniform [[META0]]
 ; CHECK:       end:
 ; CHECK-NEXT:    ret void
 ;
@@ -37,14 +37,14 @@ define amdgpu_cs void @nonuniform(ptr addrspace(4) %ptr) {
 ; CHECK-NEXT:    [[CC2:%.*]] = icmp eq i32 [[V]], 0
 ; CHECK-NEXT:    br i1 [[CC2]], label [[END_LOOP:%.*]], label [[FLOW1:%.*]]
 ; CHECK:       Flow:
-; CHECK-NEXT:    [[TMP0]] = phi i32 [ [[TMP2:%.*]], [[FLOW1]] ], [ undef, [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0]] = phi i32 [ [[TMP2:%.*]], [[FLOW1]] ], [ poison, [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ [[TMP3:%.*]], [[FLOW1]] ], [ true, [[FOR_BODY]] ]
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[FOR_END:%.*]], label [[FOR_BODY]]
 ; CHECK:       end.loop:
 ; CHECK-NEXT:    [[I_INC:%.*]] = add i32 [[I]], 1
 ; CHECK-NEXT:    br label [[FLOW1]]
 ; CHECK:       Flow1:
-; CHECK-NEXT:    [[TMP2]] = phi i32 [ [[I_INC]], [[END_LOOP]] ], [ undef, [[MID_LOOP]] ]
+; CHECK-NEXT:    [[TMP2]] = phi i32 [ [[I_INC]], [[END_LOOP]] ], [ poison, [[MID_LOOP]] ]
 ; CHECK-NEXT:    [[TMP3]] = phi i1 [ false, [[END_LOOP]] ], [ true, [[MID_LOOP]] ]
 ; CHECK-NEXT:    br label [[FLOW]]
 ; CHECK:       for.end:
@@ -85,7 +85,7 @@ define amdgpu_cs void @uniform_branch_to_nonuniform_subregions(ptr addrspace(4)
 ; CHECK-LABEL: @uniform_branch_to_nonuniform_subregions(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[DATA:%.*]], 42
-; CHECK-NEXT:    br i1 [[C]], label [[UNIFORM_FOR_BODY:%.*]], label [[FOR_BODY:%.*]], !structurizecfg.uniform !0
+; CHECK-NEXT:    br i1 [[C]], label [[UNIFORM_FOR_BODY:%.*]], label [[FOR_BODY:%.*]], !structurizecfg.uniform [[META0]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP0:%.*]], [[FLOW1:%.*]] ]
 ; CHECK-NEXT:    [[CC:%.*]] = icmp ult i32 [[I]], 4
@@ -95,14 +95,14 @@ define amdgpu_cs void @uniform_branch_to_nonuniform_subregions(ptr addrspace(4)
 ; CHECK-NEXT:    [[CC2:%.*]] = icmp eq i32 [[V]], 0
 ; CHECK-NEXT:    br i1 [[CC2]], label [[END_LOOP:%.*]], label [[FLOW2:%.*]]
 ; CHECK:       Flow1:
-; CHECK-NEXT:    [[TMP0]] = phi i32 [ [[TMP2:%.*]], [[FLOW2]] ], [ undef, [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0]] = phi i32 [ [[TMP2:%.*]], [[FLOW2]] ], [ poison, [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ [[TMP3:%.*]], [[FLOW2]] ], [ true, [[FOR_BODY]] ]
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[FOR_END:%.*]], label [[FOR_BODY]]
 ; CHECK:       end.loop:
 ; CHECK-NEXT:    [[I_INC:%.*]] = add i32 [[I]], 1
 ; CHECK-NEXT:    br label [[FLOW2]]
 ; CHECK:       Flow2:
-; CHECK-NEXT:    [[TMP2]] = phi i32 [ [[I_INC]], [[END_LOOP]] ], [ undef, [[MID_LOOP]] ]
+; CHECK-NEXT:    [[TMP2]] = phi i32 [ [[I_INC]], [[END_LOOP]] ], [ poison, [[MID_LOOP]] ]
 ; CHECK-NEXT:    [[TMP3]] = phi i1 [ false, [[END_LOOP]] ], [ true, [[MID_LOOP]] ]
 ; CHECK-NEXT:    br label [[FLOW1]]
 ; CHECK:       for.end:
@@ -118,14 +118,14 @@ define amdgpu_cs void @uniform_branch_to_nonuniform_subregions(ptr addrspace(4)
 ; CHECK-NEXT:    [[UNIFORM_CC2:%.*]] = icmp eq i32 [[UNIFORM_V]], 0
 ; CHECK-NEXT:    br i1 [[UNIFORM_CC2]], label [[UNIFORM_END_LOOP:%.*]], label [[FLOW5:%.*]]
 ; CHECK:       Flow4:
-; CHECK-NEXT:    [[TMP4]] = phi i32 [ [[TMP6:%.*]], [[FLOW5]] ], [ undef, [[UNIFORM_FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP4]] = phi i32 [ [[TMP6:%.*]], [[FLOW5]] ], [ poison, [[UNIFORM_FOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP5:%.*]] = phi i1 [ [[TMP7:%.*]], [[FLOW5]] ], [ true, [[UNIFORM_FOR_BODY]] ]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[UNIFORM_FOR_END:%.*]], label [[UNIFORM_FOR_BODY]]
 ; CHECK:       uniform.end.loop:
 ; CHECK-NEXT:    [[UNIFORM_I_INC:%.*]] = add i32 [[UNIFORM_I]], 1
 ; CHECK-NEXT:    br label [[FLOW5]]
 ; CHECK:       Flow5:
-; CHECK-NEXT:    [[TMP6]] = phi i32 [ [[UNIFORM_I_INC]], [[UNIFORM_END_LOOP]] ], [ undef, [[UNIFORM_MID_LOOP]] ]
+; CHECK-NEXT:    [[TMP6]] = phi i32 [ [[UNIFORM_I_INC]], [[UNIFORM_END_LOOP]] ], [ poison, [[UNIFORM_MID_LOOP]] ]
 ; CHECK-NEXT:    [[TMP7]] = phi i1 [ false, [[UNIFORM_END_LOOP]] ], [ true, [[UNIFORM_MID_LOOP]] ]
 ; CHECK-NEXT:    br label [[FLOW4]]
 ; CHECK:       uniform.for.end:
diff --git a/llvm/test/Transforms/StructurizeCFG/interleaved-loop-order.ll b/llvm/test/Transforms/StructurizeCFG/interleaved-loop-order.ll
index 3ca70dab27193..91e88b9212e7a 100644
--- a/llvm/test/Transforms/StructurizeCFG/interleaved-loop-order.ll
+++ b/llvm/test/Transforms/StructurizeCFG/interleaved-loop-order.ll
@@ -18,7 +18,6 @@ define i1 @test_nested(i32 %x, i1 %b1, i1 %b2, i1 %b3) {
 ; CHECK:       exit.true:
 ; CHECK-NEXT:    br label [[FLOW13]]
 ; CHECK:       Flow13:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi i1 [ true, [[EXIT_TRUE]] ], [ undef, [[FLOW12:%.*]] ]
 ; CHECK-NEXT:    br i1 [[TMP2:%.*]], label [[EXIT_FALSE:%.*]], label [[EXIT:%.*]]
 ; CHECK:       exit.false:
 ; CHECK-NEXT:    br label [[EXIT]]
@@ -30,7 +29,7 @@ define i1 @test_nested(i32 %x, i1 %b1, i1 %b2, i1 %b3) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ [[TMP16:%.*]], [[FLOW11:%.*]] ], [ true, [[OUTER_LOOP_HEADER]] ]
 ; CHECK-NEXT:    [[TMP2]] = phi i1 [ [[TMP12:%.*]], [[FLOW11]] ], [ false, [[OUTER_LOOP_HEADER]] ]
 ; CHECK-NEXT:    [[TMP3]] = phi i1 [ false, [[FLOW11]] ], [ true, [[OUTER_LOOP_HEADER]] ]
-; CHECK-NEXT:    br i1 [[TMP1]], label [[FLOW12]], label [[OUTER_LOOP_HEADER]]
+; CHECK-NEXT:    br i1 [[TMP1]], label [[FLOW12:%.*]], label [[OUTER_LOOP_HEADER]]
 ; CHECK:       inner.loop.header:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi i1 [ [[TMP8:%.*]], [[FLOW4:%.*]] ], [ false, [[OUTER_LOOP_BODY]] ]
 ; CHECK-NEXT:    br i1 [[B2:%.*]], label [[INNER_LOOP_BODY:%.*]], label [[FLOW4]]
@@ -95,7 +94,7 @@ define i1 @test_nested(i32 %x, i1 %b1, i1 %b2, i1 %b3) {
 ; CHECK:       inner.loop.latch:
 ; CHECK-NEXT:    br label [[FLOW6]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[R:%.*]] = phi i1 [ [[TMP0]], [[FLOW13]] ], [ false, [[EXIT_FALSE]] ]
+; CHECK-NEXT:    [[R:%.*]] = phi i1 [ true, [[FLOW13]] ], [ false, [[EXIT_FALSE]] ]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
 entry:
diff --git a/llvm/test/Transforms/StructurizeCFG/loop-break-phi.ll b/llvm/test/Transforms/StructurizeCFG/loop-break-phi.ll
index 46881ec827286..799d6cc8655af 100644
--- a/llvm/test/Transforms/StructurizeCFG/loop-break-phi.ll
+++ b/llvm/test/Transforms/StructurizeCFG/loop-break-phi.ll
@@ -8,14 +8,13 @@ define float @while_break(i32 %z, float %v, i32 %x, i32 %y) #0 {
 ; CHECK-NEXT:    br label %[[HEADER:.*]]
 ; CHECK:       [[HEADER]]:
 ; CHECK-NEXT:    [[V_1:%.*]] = phi float [ [[V]], %[[ENTRY]] ], [ [[TMP8:%.*]], %[[FLOW2:.*]] ]
-; CHECK-NEXT:    [[IND:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP5:%.*]], %[[FLOW2]] ]
+; CHECK-NEXT:    [[IND:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP4:%.*]], %[[FLOW2]] ]
 ; CHECK-NEXT:    [[CC:%.*]] = icmp sge i32 [[IND]], [[X]]
 ; CHECK-NEXT:    br i1 [[CC]], label %[[ELSE:.*]], label %[[FLOW:.*]]
 ; CHECK:       [[FLOW]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi float [ [[V_1]], %[[ELSE]] ], [ undef, %[[HEADER]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ [[CC2:%.*]], %[[ELSE]] ], [ false, %[[HEADER]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = phi i1 [ false, %[[ELSE]] ], [ true, %[[HEADER]] ]
-; CHECK-NEXT:    br i1 [[TMP2]], label %[[IF:.*]], label %[[FLOW1:.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i1 [ [[CC2:%.*]], %[[ELSE]] ], [ false, %[[HEADER]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ false, %[[ELSE]] ], [ true, %[[HEADER]] ]
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[IF:.*]], label %[[FLOW1:.*]]
 ; CHECK:       [[IF]]:
 ; CHECK-NEXT:    [[V_IF:%.*]] = fadd float [[V_1]], 1.000000e+00
 ; CHECK-NEXT:    br label %[[FLOW1]]
@@ -23,17 +22,17 @@ define float @while_break(i32 %z, float %v, i32 %x, i32 %y) #0 {
 ; CHECK-NEXT:    [[CC2]] = icmp slt i32 [[IND]], [[Y]]
 ; CHECK-NEXT:    br label %[[FLOW]]
 ; CHECK:       [[FLOW1]]:
-; CHECK-NEXT:    [[TMP8]] = phi float [ [[V_IF]], %[[IF]] ], [ [[TMP0]], %[[FLOW]] ]
-; CHECK-NEXT:    [[TMP4:%.*]] = phi i1 [ true, %[[IF]] ], [ [[TMP1]], %[[FLOW]] ]
-; CHECK-NEXT:    br i1 [[TMP4]], label %[[LATCH:.*]], label %[[FLOW2]]
+; CHECK-NEXT:    [[TMP8]] = phi float [ [[V_IF]], %[[IF]] ], [ [[V_1]], %[[FLOW]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = phi i1 [ true, %[[IF]] ], [ [[TMP0]], %[[FLOW]] ]
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[LATCH:.*]], label %[[FLOW2]]
 ; CHECK:       [[LATCH]]:
 ; CHECK-NEXT:    [[IND_INC:%.*]] = add i32 [[IND]], 1
 ; CHECK-NEXT:    [[CC3:%.*]] = icmp slt i32 [[IND]], [[Z]]
 ; CHECK-NEXT:    br label %[[FLOW2]]
 ; CHECK:       [[FLOW2]]:
-; CHECK-NEXT:    [[TMP5]] = phi i32 [ [[IND_INC]], %[[LATCH]] ], [ undef, %[[FLOW1]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = phi i1 [ [[CC3]], %[[LATCH]] ], [ true, %[[FLOW1]] ]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[END:.*]], label %[[HEADER]]
+; CHECK-NEXT:    [[TMP4]] = phi i32 [ [[IND_INC]], %[[LATCH]] ], [ poison, %[[FLOW1]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = phi i1 [ [[CC3]], %[[LATCH]] ], [ true, %[[FLOW1]] ]
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[END:.*]], label %[[HEADER]]
 ; CHECK:       [[END]]:
 ; CHECK-NEXT:    ret float [[TMP8]]
 ;
@@ -80,7 +79,7 @@ define float @while_break2(i32 %z, float %v, i32 %x, i32 %y) #0 {
 ; CHECK-NEXT:    [[V_IF:%.*]] = fadd float [[V_1]], 1.000000e+00
 ; CHECK-NEXT:    br label %[[FLOW]]
 ; CHECK:       [[FLOW]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi float [ [[V_IF]], %[[IF]] ], [ undef, %[[HEADER]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi float [ [[V_IF]], %[[IF]] ], [ poison, %[[HEADER]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ true, %[[IF]] ], [ false, %[[HEADER]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = phi i1 [ false, %[[IF]] ], [ true, %[[HEADER]] ]
 ; CHECK-NEXT:    br i1 [[TMP2]], label %[[ELSE:.*]], label %[[FLOW1:.*]]
@@ -96,7 +95,7 @@ define float @while_break2(i32 %z, float %v, i32 %x, i32 %y) #0 {
 ; CHECK-NEXT:    [[CC3:%.*]] = icmp slt i32 [[IND]], [[Z]]
 ; CHECK-NEXT:    br label %[[FLOW2]]
 ; CHECK:       [[FLOW2]]:
-; CHECK-NEXT:    [[TMP5]] = phi i32 [ [[IND_INC]], %[[LATCH]] ], [ undef, %[[FLOW1]] ]
+; CHECK-NEXT:    [[TMP5]] = phi i32 [ [[IND_INC]], %[[LATCH]] ], [ poison, %[[FLOW1]] ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = phi i1 [ [[CC3]], %[[LATCH]] ], [ true, %[[FLOW1]] ]
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[END:.*]], label %[[HEADER]]
 ; CHECK:       [[END]]:
@@ -159,7 +158,7 @@ define < 2 x float> @while_break_two_chains_of_phi(float %v, i32 %x, i32 %y, i32
 ; CHECK-NEXT:    [[CC3:%.*]] = icmp slt i32 [[IND]], [[Z]]
 ; CHECK-NEXT:    br label %[[FLOW1]]
 ; CHECK:       [[FLOW1]]:
-; CHECK-NEXT:    [[TMP3]] = phi i32 [ [[IND_INC]], %[[LATCH]] ], [ undef, %[[FLOW]] ]
+; CHECK-NEXT:    [[TMP3]] = phi i32 [ [[IND_INC]], %[[LATCH]] ], [ poison, %[[FLOW]] ]
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi i1 [ [[CC3]], %[[LATCH]] ], [ true, %[[FLOW]] ]
 ; CHECK-NEXT:    br i1 [[TMP4]], label %[[END:.*]], label %[[HEADER]]
 ; CHECK:       [[END]]:
diff --git a/llvm/test/Transforms/StructurizeCFG/loop-continue-phi.ll b/llvm/test/Transforms/StructurizeCFG/loop-continue-phi.ll
index eec67e67b540d..0effbed6e311c 100644
--- a/llvm/test/Transforms/StructurizeCFG/loop-continue-phi.ll
+++ b/llvm/test/Transforms/StructurizeCFG/loop-continue-phi.ll
@@ -4,22 +4,21 @@
 define void @test1(i1 %arg) {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    %arg.inv = xor i1 %arg, true
-; CHECK-NEXT:    br label %loop
+; CHECK-NEXT:    [[ARG_INV:%.*]] = xor i1 [[ARG:%.*]], true
+; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       Flow:
-; CHECK-NEXT:    br label %Flow1
+; CHECK-NEXT:    br label [[FLOW1:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    %ctr = phi i32 [ 0, %entry ], [ %0, %Flow1 ]
-; CHECK-NEXT:    %ctr.next = add i32 %ctr, 1
-; CHECK-NEXT:    br i1 %arg.inv, label %loop.a, label %Flow1
+; CHECK-NEXT:    [[CTR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[CTR_NEXT:%.*]], [[FLOW1]] ]
+; CHECK-NEXT:    [[CTR_NEXT]] = add i32 [[CTR]], 1
+; CHECK-NEXT:    br i1 [[ARG_INV]], label [[LOOP_A:%.*]], label [[FLOW1]]
 ; CHECK:       loop.a:
-; CHECK-NEXT:    br i1 %arg.inv, label %loop.b, label %Flow
+; CHECK-NEXT:    br i1 [[ARG_INV]], label [[LOOP_B:%.*]], label [[FLOW:%.*]]
 ; CHECK:       loop.b:
-; CHECK-NEXT:    br label %Flow
+; CHECK-NEXT:    br label [[FLOW]]
 ; CHECK:       Flow1:
-; CHECK-NEXT:    %0 = phi i32 [ %ctr.next, %Flow ], [ undef, %loop ]
-; CHECK-NEXT:    %1 = phi i1 [ false, %Flow ], [ true, %loop ]
-; CHECK-NEXT:    br i1 %1, label %exit, label %loop
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i1 [ false, [[FLOW]] ], [ true, [[LOOP]] ]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[EXIT:%.*]], label [[LOOP]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/StructurizeCFG/one-loop-multiple-backedges.ll b/llvm/test/Transforms/StructurizeCFG/one-loop-multiple-backedges.ll
index 6f1c9a833804b..1389b12cdc53a 100644
--- a/llvm/test/Transforms/StructurizeCFG/one-loop-multiple-backedges.ll
+++ b/llvm/test/Transforms/StructurizeCFG/one-loop-multiple-backedges.ll
@@ -16,7 +16,6 @@ define void @blam(ptr addrspace(1) nocapture %arg, float %arg1, float %arg2) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = fcmp uge float 0.000000e+00, [[ARG2:%.*]]
 ; CHECK-NEXT:    br label [[FLOW]]
 ; CHECK:       Flow:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ 15, [[BB5]] ], [ undef, [[BB3]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ [[TMP6]], [[BB5]] ], [ [[TMP4]], [[BB3]] ]
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[BB7:%.*]], label [[FLOW1]]
 ; CHECK:       bb7:
@@ -24,10 +23,10 @@ define void @blam(ptr addrspace(1) nocapture %arg, float %arg1, float %arg2) {
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp sge i64 [[TMP8]], 5
 ; CHECK-NEXT:    br label [[FLOW1]]
 ; CHECK:       Flow1:
-; CHECK-NEXT:    [[TMP2]] = phi i64 [ [[TMP8]], [[BB7]] ], [ undef, [[FLOW]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = phi i32 [ 255, [[BB7]] ], [ [[TMP0]], [[FLOW]] ]
-; CHECK-NEXT:    [[TMP4:%.*]] = phi i1 [ [[TMP9]], [[BB7]] ], [ true, [[FLOW]] ]
-; CHECK-NEXT:    br i1 [[TMP4]], label [[BB10:%.*]], label [[BB3]]
+; CHECK-NEXT:    [[TMP2]] = phi i64 [ [[TMP8]], [[BB7]] ], [ poison, [[FLOW]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = phi i32 [ 255, [[BB7]] ], [ 15, [[FLOW]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = phi i1 [ [[TMP9]], [[BB7]] ], [ true, [[FLOW]] ]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[BB10:%.*]], label [[BB3]]
 ; CHECK:       bb10:
 ; CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(1) [[ARG:%.*]], align 4
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/StructurizeCFG/structurizer-keep-perf-md.ll b/llvm/test/Transforms/StructurizeCFG/structurizer-keep-perf-md.ll
index cdf5ca569701b..583c97852ff6d 100644
--- a/llvm/test/Transforms/StructurizeCFG/structurizer-keep-perf-md.ll
+++ b/llvm/test/Transforms/StructurizeCFG/structurizer-keep-perf-md.ll
@@ -7,15 +7,14 @@ define amdgpu_ps i32 @if_else(i32 %0) {
 ; OPT-NEXT:    [[C:%.*]] = icmp ne i32 [[TMP0]], 0
 ; OPT-NEXT:    br i1 [[C]], label %[[FALSE:.*]], label %[[FLOW:.*]], !prof [[PROF0:![0-9]+]]
 ; OPT:       [[FLOW]]:
-; OPT-NEXT:    [[TMP2:%.*]] = phi i32 [ 33, %[[FALSE]] ], [ undef, [[TMP1:%.*]] ]
-; OPT-NEXT:    [[TMP3:%.*]] = phi i1 [ false, %[[FALSE]] ], [ true, [[TMP1]] ]
-; OPT-NEXT:    br i1 [[TMP3]], label %[[TRUE:.*]], label %[[EXIT:.*]]
+; OPT-NEXT:    [[TMP2:%.*]] = phi i1 [ false, %[[FALSE]] ], [ true, [[TMP1:%.*]] ]
+; OPT-NEXT:    br i1 [[TMP2]], label %[[TRUE:.*]], label %[[EXIT:.*]]
 ; OPT:       [[TRUE]]:
 ; OPT-NEXT:    br label %[[EXIT]]
 ; OPT:       [[FALSE]]:
 ; OPT-NEXT:    br label %[[FLOW]]
 ; OPT:       [[EXIT]]:
-; OPT-NEXT:    [[RET:%.*]] = phi i32 [ [[TMP2]], %[[FLOW]] ], [ 42, %[[TRUE]] ]
+; OPT-NEXT:    [[RET:%.*]] = phi i32 [ 33, %[[FLOW]] ], [ 42, %[[TRUE]] ]
 ; OPT-NEXT:    ret i32 [[RET]]
 ;
   %c = icmp eq i32 %0, 0
@@ -45,7 +44,7 @@ define amdgpu_ps void @loop_if_break(i32 %n) {
 ; OPT-NEXT:    [[I_NEXT:%.*]] = sub i32 [[I]], 1
 ; OPT-NEXT:    br label %[[FLOW]]
 ; OPT:       [[FLOW]]:
-; OPT-NEXT:    [[TMP0]] = phi i32 [ [[I_NEXT]], %[[LOOP_BODY]] ], [ undef, %[[LOOP]] ]
+; OPT-NEXT:    [[TMP0]] = phi i32 [ [[I_NEXT]], %[[LOOP_BODY]] ], [ poison, %[[LOOP]] ]
 ; OPT-NEXT:    [[TMP1:%.*]] = phi i1 [ false, %[[LOOP_BODY]] ], [ true, %[[LOOP]] ]
 ; OPT-NEXT:    br i1 [[TMP1]], label %[[EXIT:.*]], label %[[LOOP]]
 ; OPT:       [[EXIT]]:
diff --git a/llvm/test/Transforms/StructurizeCFG/workarounds/needs-fix-reducible.ll b/llvm/test/Transforms/StructurizeCFG/workarounds/needs-fix-reducible.ll
index d09d7454793bb..8fbeb6e811805 100644
--- a/llvm/test/Transforms/StructurizeCFG/workarounds/needs-fix-reducible.ll
+++ b/llvm/test/Transforms/StructurizeCFG/workarounds/needs-fix-reducible.ll
@@ -12,22 +12,20 @@ define void @irreducible(i1 %PredEntry, i1 %PredB1, i1 %PredB2, i1 %PredB3, i1 %
 ; CHECK-NEXT:    [[PREDENTRY_INV:%.*]] = xor i1 [[PREDENTRY:%.*]], true
 ; CHECK-NEXT:    br label [[IRR_GUARD:%.*]]
 ; CHECK:       Flow4:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi i1 [ true, [[B3:%.*]] ], [ undef, [[B2:%.*]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ [[PREDB3:%.*]], [[B3]] ], [ true, [[B2]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ [[PREDB3:%.*]], [[B3:%.*]] ], [ true, [[B2:%.*]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = phi i1 [ [[PREDB3]], [[B3]] ], [ false, [[B2]] ]
 ; CHECK-NEXT:    br label [[FLOW3:%.*]]
 ; CHECK:       B1:
 ; CHECK-NEXT:    br label [[FLOW5:%.*]]
 ; CHECK:       Flow2:
-; CHECK-NEXT:    [[TMP3:%.*]] = phi i1 [ [[TMP9:%.*]], [[FLOW5]] ], [ [[TMP11:%.*]], [[FLOW:%.*]] ]
-; CHECK-NEXT:    [[TMP4:%.*]] = phi i1 [ [[TMP10:%.*]], [[FLOW5]] ], [ [[TMP12:%.*]], [[FLOW]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = phi i1 [ true, [[FLOW5]] ], [ [[TMP9:%.*]], [[FLOW:%.*]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i1 [ true, [[FLOW5]] ], [ [[TMP12:%.*]], [[FLOW]] ]
 ; CHECK-NEXT:    [[TMP5:%.*]] = phi i1 [ [[TMP7:%.*]], [[FLOW5]] ], [ true, [[FLOW]] ]
 ; CHECK-NEXT:    br i1 true, label [[FLOW6:%.*]], label [[FLOW]]
 ; CHECK:       B2:
 ; CHECK-NEXT:    br i1 [[PREDB2_INV]], label [[B3]], label [[FLOW4:%.*]]
 ; CHECK:       Flow3:
-; CHECK-NEXT:    [[TMP6:%.*]] = phi i1 [ [[TMP0]], [[FLOW4]] ], [ undef, [[IRR_GUARD1:%.*]] ]
-; CHECK-NEXT:    [[TMP7]] = phi i1 [ [[TMP1]], [[FLOW4]] ], [ true, [[IRR_GUARD1]] ]
+; CHECK-NEXT:    [[TMP7]] = phi i1 [ [[TMP1]], [[FLOW4]] ], [ true, [[IRR_GUARD1:%.*]] ]
 ; CHECK-NEXT:    [[TMP8:%.*]] = phi i1 [ [[TMP2]], [[FLOW4]] ], [ true, [[IRR_GUARD1]] ]
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[B1:%.*]], label [[FLOW5]]
 ; CHECK:       B3:
@@ -35,8 +33,6 @@ define void @irreducible(i1 %PredEntry, i1 %PredB1, i1 %PredB2, i1 %PredB3, i1 %
 ; CHECK:       B4:
 ; CHECK-NEXT:    br label [[FLOW]]
 ; CHECK:       Flow5:
-; CHECK-NEXT:    [[TMP9]] = phi i1 [ undef, [[B1]] ], [ [[TMP6]], [[FLOW3]] ]
-; CHECK-NEXT:    [[TMP10]] = phi i1 [ true, [[B1]] ], [ undef, [[FLOW3]] ]
 ; CHECK-NEXT:    br label [[FLOW2:%.*]]
 ; CHECK:       Flow6:
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[IRR_GUARD]]
@@ -46,7 +42,7 @@ define void @irreducible(i1 %PredEntry, i1 %PredB1, i1 %PredB2, i1 %PredB3, i1 %
 ; CHECK-NEXT:    [[GUARD_B4:%.*]] = phi i1 [ [[PREDENTRY_INV]], [[ENTRY:%.*]] ], [ [[TMP3]], [[FLOW6]] ]
 ; CHECK-NEXT:    br i1 [[GUARD_B4]], label [[B4:%.*]], label [[FLOW]]
 ; CHECK:       Flow:
-; CHECK-NEXT:    [[TMP11]] = phi i1 [ [[TMP3]], [[FLOW2]] ], [ undef, [[B4]] ], [ undef, [[IRR_GUARD]] ]
+; CHECK-NEXT:    [[TMP9]] = phi i1 [ [[TMP3]], [[FLOW2]] ], [ poison, [[B4]] ], [ poison, [[IRR_GUARD]] ]
 ; CHECK-NEXT:    [[TMP12]] = phi i1 [ [[TMP4]], [[FLOW2]] ], [ true, [[B4]] ], [ false, [[IRR_GUARD]] ]
 ; CHECK-NEXT:    [[TMP13:%.*]] = phi i1 [ false, [[FLOW2]] ], [ [[PREDB4:%.*]], [[B4]] ], [ true, [[IRR_GUARD]] ]
 ; CHECK-NEXT:    br i1 [[TMP13]], label [[IRR_GUARD1]], label [[FLOW2]]
diff --git a/llvm/test/Transforms/StructurizeCFG/workarounds/needs-fr-ule.ll b/llvm/test/Transforms/StructurizeCFG/workarounds/needs-fr-ule.ll
index 912beed6b2eed..17e24ea7f6d9a 100644
--- a/llvm/test/Transforms/StructurizeCFG/workarounds/needs-fr-ule.ll
+++ b/llvm/test/Transforms/StructurizeCFG/workarounds/needs-fr-ule.ll
@@ -47,10 +47,7 @@ define void @irreducible_mountain_bug(i1 %Pred0, i1 %Pred1, i1 %Pred2, i1 %Pred3
 ; CHECK:       while.body63:
 ; CHECK-NEXT:    br i1 [[PRED5_INV]], label [[WHILE_COND47]], label [[FLOW10:%.*]]
 ; CHECK:       Flow9:
-; CHECK-NEXT:    [[TMP5:%.*]] = phi i1 [ [[TMP3]], [[FLOW10]] ], [ undef, [[IRR_GUARD1:%.*]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = phi i1 [ false, [[FLOW10]] ], [ undef, [[IRR_GUARD1]] ]
-; CHECK-NEXT:    [[TMP7:%.*]] = phi i1 [ true, [[FLOW10]] ], [ undef, [[IRR_GUARD1]] ]
-; CHECK-NEXT:    [[TMP8:%.*]] = phi i1 [ true, [[FLOW10]] ], [ undef, [[IRR_GUARD1]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = phi i1 [ [[TMP3]], [[FLOW10]] ], [ poison, [[IRR_GUARD1:%.*]] ]
 ; CHECK-NEXT:    [[TMP9:%.*]] = phi i1 [ [[TMP4]], [[FLOW10]] ], [ true, [[IRR_GUARD1]] ]
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[COND_TRUE49:%.*]], label [[FLOW11]]
 ; CHECK:       while.cond47:
@@ -58,8 +55,7 @@ define void @irreducible_mountain_bug(i1 %Pred0, i1 %Pred1, i1 %Pred2, i1 %Pred3
 ; CHECK:       cond.end61:
 ; CHECK-NEXT:    br label [[FLOW12:%.*]]
 ; CHECK:       Flow17:
-; CHECK-NEXT:    [[TMP10:%.*]] = phi i1 [ [[TMP19:%.*]], [[FLOW18:%.*]] ], [ undef, [[LOOP_EXIT_GUARD2:%.*]] ]
-; CHECK-NEXT:    [[TMP11:%.*]] = phi i1 [ [[TMP20:%.*]], [[FLOW18]] ], [ [[DOTINV:%.*]], [[LOOP_EXIT_GUARD2]] ]
+; CHECK-NEXT:    [[TMP11:%.*]] = phi i1 [ [[TMP20:%.*]], [[FLOW18:%.*]] ], [ [[DOTINV:%.*]], [[LOOP_EXIT_GUARD2:%.*]] ]
 ; CHECK-NEXT:    br label [[FLOW16:%.*]]
 ; CHECK:       if.then69:
 ; CHECK-NEXT:    br label [[FLOW18]]
@@ -92,30 +88,26 @@ define void @irreducible_mountain_bug(i1 %Pred0, i1 %Pred1, i1 %Pred2, i1 %Pred3
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ; CHECK:       Flow15:
-; CHECK-NEXT:    [[TMP12:%.*]] = phi i1 [ true, [[LOR_RHS]] ], [ undef, [[WHILE_COND]] ]
 ; CHECK-NEXT:    [[TMP13:%.*]] = phi i1 [ true, [[LOR_RHS]] ], [ false, [[WHILE_COND]] ]
 ; CHECK-NEXT:    [[TMP14:%.*]] = phi i1 [ [[PRED9:%.*]], [[LOR_RHS]] ], [ true, [[WHILE_COND]] ]
 ; CHECK-NEXT:    br i1 [[TMP14]], label [[IRR_GUARD:%.*]], label [[FLOW16]]
 ; CHECK:       irr.guard:
-; CHECK-NEXT:    [[GUARD_COND_END61:%.*]] = phi i1 [ [[TMP28:%.*]], [[FLOW13:%.*]] ], [ [[TMP13]], [[FLOW15]] ]
+; CHECK-NEXT:    [[GUARD_COND_END61:%.*]] = phi i1 [ true, [[FLOW13:%.*]] ], [ [[TMP13]], [[FLOW15]] ]
 ; CHECK-NEXT:    br i1 [[GUARD_COND_END61]], label [[COND_END61:%.*]], label [[FLOW12]]
 ; CHECK:       Flow12:
-; CHECK-NEXT:    [[TMP15:%.*]] = phi i1 [ false, [[COND_END61]] ], [ undef, [[IRR_GUARD]] ]
-; CHECK-NEXT:    [[TMP16:%.*]] = phi i1 [ true, [[COND_END61]] ], [ undef, [[IRR_GUARD]] ]
 ; CHECK-NEXT:    [[TMP17:%.*]] = phi i1 [ true, [[COND_END61]] ], [ false, [[IRR_GUARD]] ]
 ; CHECK-NEXT:    [[TMP18:%.*]] = phi i1 [ [[PRED7:%.*]], [[COND_END61]] ], [ true, [[IRR_GUARD]] ]
 ; CHECK-NEXT:    br i1 [[TMP18]], label [[IRR_GUARD1]], label [[FLOW13]]
 ; CHECK:       irr.guard1:
-; CHECK-NEXT:    [[GUARD_WHILE_BODY63:%.*]] = phi i1 [ [[TMP23:%.*]], [[FLOW11]] ], [ [[TMP17]], [[FLOW12]] ]
+; CHECK-NEXT:    [[GUARD_WHILE_BODY63:%.*]] = phi i1 [ true, [[FLOW11]] ], [ [[TMP17]], [[FLOW12]] ]
 ; CHECK-NEXT:    br i1 [[GUARD_WHILE_BODY63]], label [[WHILE_BODY63]], label [[FLOW9]]
 ; CHECK:       Flow18:
-; CHECK-NEXT:    [[TMP19]] = phi i1 [ false, [[IF_THEN69:%.*]] ], [ [[TMP31:%.*]], [[LOOP_EXIT_GUARD3:%.*]] ]
-; CHECK-NEXT:    [[TMP20]] = phi i1 [ [[PRED8:%.*]], [[IF_THEN69]] ], [ [[DOTINV]], [[LOOP_EXIT_GUARD3]] ]
+; CHECK-NEXT:    [[TMP20]] = phi i1 [ [[PRED8:%.*]], [[IF_THEN69:%.*]] ], [ [[DOTINV]], [[LOOP_EXIT_GUARD3:%.*]] ]
 ; CHECK-NEXT:    br label [[FLOW17:%.*]]
 ; CHECK:       loop.exit.guard:
 ; CHECK-NEXT:    br i1 [[TMP21:%.*]], label [[WHILE_END76:%.*]], label [[FLOW8]]
 ; CHECK:       Flow16:
-; CHECK-NEXT:    [[TMP21]] = phi i1 [ [[TMP10]], [[FLOW17]] ], [ [[TMP12]], [[FLOW15]] ]
+; CHECK-NEXT:    [[TMP21]] = phi i1 [ false, [[FLOW17]] ], [ true, [[FLOW15]] ]
 ; CHECK-NEXT:    [[TMP22:%.*]] = phi i1 [ [[TMP11]], [[FLOW17]] ], [ true, [[FLOW15]] ]
 ; CHECK-NEXT:    br i1 [[TMP22]], label [[LOOP_EXIT_GUARD:%.*]], label [[WHILE_COND]]
 ; CHECK:       loop.exit.guard2:
@@ -123,17 +115,13 @@ define void @irreducible_mountain_bug(i1 %Pred0, i1 %Pred1, i1 %Pred2, i1 %Pred3
 ; CHECK:       loop.exit.guard3:
 ; CHECK-NEXT:    br i1 [[DOTINV14:%.*]], label [[IF_THEN69]], label [[FLOW18]]
 ; CHECK:       Flow11:
-; CHECK-NEXT:    [[TMP23]] = phi i1 [ true, [[COND_TRUE49]] ], [ undef, [[FLOW9]] ]
 ; CHECK-NEXT:    [[TMP24:%.*]] = phi i1 [ true, [[COND_TRUE49]] ], [ [[TMP5]], [[FLOW9]] ]
-; CHECK-NEXT:    [[TMP25:%.*]] = phi i1 [ false, [[COND_TRUE49]] ], [ [[TMP6]], [[FLOW9]] ]
-; CHECK-NEXT:    [[TMP26:%.*]] = phi i1 [ false, [[COND_TRUE49]] ], [ [[TMP7]], [[FLOW9]] ]
+; CHECK-NEXT:    [[TMP16:%.*]] = phi i1 [ false, [[COND_TRUE49]] ], [ true, [[FLOW9]] ]
 ; CHECK-NEXT:    [[TMP27:%.*]] = phi i1 [ [[PRED4:%.*]], [[COND_TRUE49]] ], [ true, [[FLOW9]] ]
 ; CHECK-NEXT:    br i1 [[TMP27]], label [[LOOP_EXIT_GUARD4:%.*]], label [[IRR_GUARD1]]
 ; CHECK:       Flow13:
-; CHECK-NEXT:    [[TMP28]] = phi i1 [ [[TMP8]], [[LOOP_EXIT_GUARD4]] ], [ undef, [[FLOW12]] ]
-; CHECK-NEXT:    [[TMP29:%.*]] = phi i1 [ [[TMP26]], [[LOOP_EXIT_GUARD4]] ], [ [[TMP15]], [[FLOW12]] ]
-; CHECK-NEXT:    [[TMP30:%.*]] = phi i1 [ [[TMP25]], [[LOOP_EXIT_GUARD4]] ], [ [[TMP16]], [[FLOW12]] ]
-; CHECK-NEXT:    [[TMP31]] = phi i1 [ [[TMP6]], [[LOOP_EXIT_GUARD4]] ], [ undef, [[FLOW12]] ]
+; CHECK-NEXT:    [[TMP29:%.*]] = phi i1 [ [[TMP16]], [[LOOP_EXIT_GUARD4]] ], [ false, [[FLOW12]] ]
+; CHECK-NEXT:    [[TMP30:%.*]] = phi i1 [ false, [[LOOP_EXIT_GUARD4]] ], [ true, [[FLOW12]] ]
 ; CHECK-NEXT:    [[TMP32:%.*]] = phi i1 [ [[TMP24]], [[LOOP_EXIT_GUARD4]] ], [ true, [[FLOW12]] ]
 ; CHECK-NEXT:    [[DOTINV14]] = xor i1 [[TMP29]], true
 ; CHECK-NEXT:    [[DOTINV]] = xor i1 [[TMP30]], true
diff --git a/llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll b/llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll
index 6f6fc4d0f4e64..ff989a655d54a 100644
--- a/llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll
+++ b/llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll
@@ -28,7 +28,6 @@ define void @exiting-block(i1 %PredH1, i1 %PredB2, i1 %PredB1, i1 %PredH2) {
 ; CHECK:       H1:
 ; CHECK-NEXT:    br i1 [[PREDH1_INV]], label [[B1:%.*]], label [[FLOW3:%.*]]
 ; CHECK:       Flow3:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi i1 [ true, [[B1]] ], [ undef, [[H1]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ [[PREDB1:%.*]], [[B1]] ], [ [[PREDH1]], [[H1]] ]
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[H2:%.*]], label [[FLOW4:%.*]]
 ; CHECK:       H2:
@@ -58,7 +57,7 @@ define void @exiting-block(i1 %PredH1, i1 %PredB2, i1 %PredB1, i1 %PredH2) {
 ; CHECK-NEXT:    [[TMP5]] = phi i1 [ false, [[L2]] ], [ true, [[B2]] ]
 ; CHECK-NEXT:    br label [[FLOW]]
 ; CHECK:       Flow4:
-; CHECK-NEXT:    [[TMP6]] = phi i1 [ false, [[FLOW5]] ], [ [[TMP0]], [[FLOW3]] ]
+; CHECK-NEXT:    [[TMP6]] = phi i1 [ false, [[FLOW5]] ], [ true, [[FLOW3]] ]
 ; CHECK-NEXT:    [[TMP7:%.*]] = phi i1 [ [[TMP4]], [[FLOW5]] ], [ true, [[FLOW3]] ]
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[LOOP_EXIT_GUARD:%.*]], label [[H1]]
 ; CHECK:       loop.exit.guard1:
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
index 09875c5e0af40..1c128c8f56a03 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
@@ -997,8 +997,10 @@ define <4 x i64> @bitcast_smax_v8i32_v4i32(<4 x i64> %a, <4 x i64> %b) {
 ; CHECK-NEXT:    [[A_BC0:%.*]] = bitcast <4 x i64> [[A:%.*]] to <8 x i32>
 ; CHECK-NEXT:    [[B_BC0:%.*]] = bitcast <4 x i64> [[B:%.*]] to <8 x i32>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <8 x i32> [[A_BC0]], [[B_BC0]]
-; CHECK-NEXT:    [[A_BC1:%.*]] = bitcast <4 x i64> [[A]] to <8 x i32>
-; CHECK-NEXT:    [[B_BC1:%.*]] = bitcast <4 x i64> [[B]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[B]], <4 x i64> [[B]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[B_BC1:%.*]] = bitcast <4 x i64> [[TMP2]] to <8 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[A]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[A_BC1:%.*]] = bitcast <4 x i64> [[TMP4]] to <8 x i32>
 ; CHECK-NEXT:    [[CONCAT:%.*]] = select <8 x i1> [[CMP]], <8 x i32> [[B_BC1]], <8 x i32> [[A_BC1]]
 ; CHECK-NEXT:    [[RES:%.*]] = bitcast <8 x i32> [[CONCAT]] to <4 x i64>
 ; CHECK-NEXT:    ret <4 x i64> [[RES]]
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-selects.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-selects.ll
index 6653bf3375423..2588f9116f322 100644
--- a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-selects.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-selects.ll
@@ -4,28 +4,12 @@
 ; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX512
 
 define <4 x i16> @src_v2tov4_i16(<2 x i1> %a, <2 x i1> %b, <2 x i16> %x, <2 x i16> %y, <2 x i16> %z) {
-; SSE-LABEL: define <4 x i16> @src_v2tov4_i16(
-; SSE-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]], <2 x i16> [[X:%.*]], <2 x i16> [[Y:%.*]], <2 x i16> [[Z:%.*]]) #[[ATTR0:[0-9]+]] {
-; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i1> [[A]], <2 x i1> [[B]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i16> [[X]], <2 x i16> [[Y]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[Z]], <2 x i16> [[X]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SSE-NEXT:    [[RES:%.*]] = select <4 x i1> [[TMP1]], <4 x i16> [[TMP2]], <4 x i16> [[TMP3]]
-; SSE-NEXT:    ret <4 x i16> [[RES]]
-;
-; AVX2-LABEL: define <4 x i16> @src_v2tov4_i16(
-; AVX2-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]], <2 x i16> [[X:%.*]], <2 x i16> [[Y:%.*]], <2 x i16> [[Z:%.*]]) #[[ATTR0:[0-9]+]] {
-; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i1> [[A]], <2 x i1> [[B]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i16> [[X]], <2 x i16> [[Y]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[Z]], <2 x i16> [[X]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; AVX2-NEXT:    [[RES:%.*]] = select <4 x i1> [[TMP1]], <4 x i16> [[TMP2]], <4 x i16> [[TMP3]]
-; AVX2-NEXT:    ret <4 x i16> [[RES]]
-;
-; AVX512-LABEL: define <4 x i16> @src_v2tov4_i16(
-; AVX512-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]], <2 x i16> [[X:%.*]], <2 x i16> [[Y:%.*]], <2 x i16> [[Z:%.*]]) #[[ATTR0:[0-9]+]] {
-; AVX512-NEXT:    [[SELECT_XZ:%.*]] = select <2 x i1> [[A]], <2 x i16> [[X]], <2 x i16> [[Z]]
-; AVX512-NEXT:    [[SELECT_YX:%.*]] = select <2 x i1> [[B]], <2 x i16> [[Y]], <2 x i16> [[X]]
-; AVX512-NEXT:    [[RES:%.*]] = shufflevector <2 x i16> [[SELECT_XZ]], <2 x i16> [[SELECT_YX]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; AVX512-NEXT:    ret <4 x i16> [[RES]]
+; CHECK-LABEL: define <4 x i16> @src_v2tov4_i16(
+; CHECK-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]], <2 x i16> [[X:%.*]], <2 x i16> [[Y:%.*]], <2 x i16> [[Z:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[SELECT_XZ:%.*]] = select <2 x i1> [[A]], <2 x i16> [[X]], <2 x i16> [[Z]]
+; CHECK-NEXT:    [[SELECT_YX:%.*]] = select <2 x i1> [[B]], <2 x i16> [[Y]], <2 x i16> [[X]]
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i16> [[SELECT_XZ]], <2 x i16> [[SELECT_YX]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i16> [[RES]]
 ;
   %select.xz = select <2 x i1> %a, <2 x i16> %x, <2 x i16> %z
   %select.yx = select <2 x i1> %b, <2 x i16> %y, <2 x i16> %x
@@ -34,28 +18,12 @@ define <4 x i16> @src_v2tov4_i16(<2 x i1> %a, <2 x i1> %b, <2 x i16> %x, <2 x i1
 }
 
 define <8 x i16> @src_v4tov8_i16(<4 x i1> %a, <4 x i1> %b, <4 x i16> %x, <4 x i16> %y, <4 x i16> %z) {
-; SSE-LABEL: define <8 x i16> @src_v4tov8_i16(
-; SSE-SAME: <4 x i1> [[A:%.*]], <4 x i1> [[B:%.*]], <4 x i16> [[X:%.*]], <4 x i16> [[Y:%.*]], <4 x i16> [[Z:%.*]]) #[[ATTR0]] {
-; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i1> [[A]], <4 x i1> [[B]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i16> [[X]], <4 x i16> [[Y]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i16> [[Z]], <4 x i16> [[X]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[RES:%.*]] = select <8 x i1> [[TMP1]], <8 x i16> [[TMP2]], <8 x i16> [[TMP3]]
-; SSE-NEXT:    ret <8 x i16> [[RES]]
-;
-; AVX2-LABEL: define <8 x i16> @src_v4tov8_i16(
-; AVX2-SAME: <4 x i1> [[A:%.*]], <4 x i1> [[B:%.*]], <4 x i16> [[X:%.*]], <4 x i16> [[Y:%.*]], <4 x i16> [[Z:%.*]]) #[[ATTR0]] {
-; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i1> [[A]], <4 x i1> [[B]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i16> [[X]], <4 x i16> [[Y]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i16> [[Z]], <4 x i16> [[X]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:    [[RES:%.*]] = select <8 x i1> [[TMP1]], <8 x i16> [[TMP2]], <8 x i16> [[TMP3]]
-; AVX2-NEXT:    ret <8 x i16> [[RES]]
-;
-; AVX512-LABEL: define <8 x i16> @src_v4tov8_i16(
-; AVX512-SAME: <4 x i1> [[A:%.*]], <4 x i1> [[B:%.*]], <4 x i16> [[X:%.*]], <4 x i16> [[Y:%.*]], <4 x i16> [[Z:%.*]]) #[[ATTR0]] {
-; AVX512-NEXT:    [[SELECT_XZ:%.*]] = select <4 x i1> [[A]], <4 x i16> [[X]], <4 x i16> [[Z]]
-; AVX512-NEXT:    [[SELECT_YX:%.*]] = select <4 x i1> [[B]], <4 x i16> [[Y]], <4 x i16> [[X]]
-; AVX512-NEXT:    [[RES:%.*]] = shufflevector <4 x i16> [[SELECT_XZ]], <4 x i16> [[SELECT_YX]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX512-NEXT:    ret <8 x i16> [[RES]]
+; CHECK-LABEL: define <8 x i16> @src_v4tov8_i16(
+; CHECK-SAME: <4 x i1> [[A:%.*]], <4 x i1> [[B:%.*]], <4 x i16> [[X:%.*]], <4 x i16> [[Y:%.*]], <4 x i16> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[SELECT_XZ:%.*]] = select <4 x i1> [[A]], <4 x i16> [[X]], <4 x i16> [[Z]]
+; CHECK-NEXT:    [[SELECT_YX:%.*]] = select <4 x i1> [[B]], <4 x i16> [[Y]], <4 x i16> [[X]]
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x i16> [[SELECT_XZ]], <4 x i16> [[SELECT_YX]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x i16> [[RES]]
 ;
   %select.xz = select <4 x i1> %a, <4 x i16> %x, <4 x i16> %z
   %select.yx = select <4 x i1> %b, <4 x i16> %y, <4 x i16> %x
@@ -94,12 +62,27 @@ define <8 x i16> @src_v8tov8_i16(<8 x i1> %a, <8 x i1> %b, <8 x i16> %x, <8 x i1
 }
 
 define <16 x i16> @src_v8tov16_i16(<8 x i1> %a, <8 x i1> %b, <8 x i16> %x, <8 x i16> %y, <8 x i16> %z) {
-; CHECK-LABEL: define <16 x i16> @src_v8tov16_i16(
-; CHECK-SAME: <8 x i1> [[A:%.*]], <8 x i1> [[B:%.*]], <8 x i16> [[X:%.*]], <8 x i16> [[Y:%.*]], <8 x i16> [[Z:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:    [[SELECT_XZ:%.*]] = select <8 x i1> [[A]], <8 x i16> [[X]], <8 x i16> [[Z]]
-; CHECK-NEXT:    [[SELECT_YX:%.*]] = select <8 x i1> [[B]], <8 x i16> [[Y]], <8 x i16> [[X]]
-; CHECK-NEXT:    [[RES:%.*]] = shufflevector <8 x i16> [[SELECT_XZ]], <8 x i16> [[SELECT_YX]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    ret <16 x i16> [[RES]]
+; SSE-LABEL: define <16 x i16> @src_v8tov16_i16(
+; SSE-SAME: <8 x i1> [[A:%.*]], <8 x i1> [[B:%.*]], <8 x i16> [[X:%.*]], <8 x i16> [[Y:%.*]], <8 x i16> [[Z:%.*]]) #[[ATTR0]] {
+; SSE-NEXT:    [[SELECT_XZ:%.*]] = select <8 x i1> [[A]], <8 x i16> [[X]], <8 x i16> [[Z]]
+; SSE-NEXT:    [[SELECT_YX:%.*]] = select <8 x i1> [[B]], <8 x i16> [[Y]], <8 x i16> [[X]]
+; SSE-NEXT:    [[RES:%.*]] = shufflevector <8 x i16> [[SELECT_XZ]], <8 x i16> [[SELECT_YX]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    ret <16 x i16> [[RES]]
+;
+; AVX2-LABEL: define <16 x i16> @src_v8tov16_i16(
+; AVX2-SAME: <8 x i1> [[A:%.*]], <8 x i1> [[B:%.*]], <8 x i16> [[X:%.*]], <8 x i16> [[Y:%.*]], <8 x i16> [[Z:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i1> [[A]], <8 x i1> [[B]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[X]], <8 x i16> [[Y]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[Z]], <8 x i16> [[X]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:    [[RES:%.*]] = select <16 x i1> [[TMP1]], <16 x i16> [[TMP2]], <16 x i16> [[TMP3]]
+; AVX2-NEXT:    ret <16 x i16> [[RES]]
+;
+; AVX512-LABEL: define <16 x i16> @src_v8tov16_i16(
+; AVX512-SAME: <8 x i1> [[A:%.*]], <8 x i1> [[B:%.*]], <8 x i16> [[X:%.*]], <8 x i16> [[Y:%.*]], <8 x i16> [[Z:%.*]]) #[[ATTR0]] {
+; AVX512-NEXT:    [[SELECT_XZ:%.*]] = select <8 x i1> [[A]], <8 x i16> [[X]], <8 x i16> [[Z]]
+; AVX512-NEXT:    [[SELECT_YX:%.*]] = select <8 x i1> [[B]], <8 x i16> [[Y]], <8 x i16> [[X]]
+; AVX512-NEXT:    [[RES:%.*]] = shufflevector <8 x i16> [[SELECT_XZ]], <8 x i16> [[SELECT_YX]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:    ret <16 x i16> [[RES]]
 ;
   %select.xz = select <8 x i1> %a, <8 x i16> %x, <8 x i16> %z
   %select.yx = select <8 x i1> %b, <8 x i16> %y, <8 x i16> %x
@@ -108,28 +91,12 @@ define <16 x i16> @src_v8tov16_i16(<8 x i1> %a, <8 x i1> %b, <8 x i16> %x, <8 x
 }
 
 define <4 x i32> @src_v2tov4_i32(<2 x i1> %a, <2 x i1> %b, <2 x i32> %x, <2 x i32> %y, <2 x i32> %z) {
-; SSE-LABEL: define <4 x i32> @src_v2tov4_i32(
-; SSE-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]], <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]], <2 x i32> [[Z:%.*]]) #[[ATTR0]] {
-; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i1> [[A]], <2 x i1> [[B]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[X]], <2 x i32> [[Y]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> [[Z]], <2 x i32> [[X]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SSE-NEXT:    [[RES:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]]
-; SSE-NEXT:    ret <4 x i32> [[RES]]
-;
-; AVX2-LABEL: define <4 x i32> @src_v2tov4_i32(
-; AVX2-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]], <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]], <2 x i32> [[Z:%.*]]) #[[ATTR0]] {
-; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i1> [[A]], <2 x i1> [[B]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[X]], <2 x i32> [[Y]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> [[Z]], <2 x i32> [[X]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; AVX2-NEXT:    [[RES:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]]
-; AVX2-NEXT:    ret <4 x i32> [[RES]]
-;
-; AVX512-LABEL: define <4 x i32> @src_v2tov4_i32(
-; AVX512-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]], <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]], <2 x i32> [[Z:%.*]]) #[[ATTR0]] {
-; AVX512-NEXT:    [[SELECT_XZ:%.*]] = select <2 x i1> [[A]], <2 x i32> [[X]], <2 x i32> [[Z]]
-; AVX512-NEXT:    [[SELECT_YX:%.*]] = select <2 x i1> [[B]], <2 x i32> [[Y]], <2 x i32> [[X]]
-; AVX512-NEXT:    [[RES:%.*]] = shufflevector <2 x i32> [[SELECT_XZ]], <2 x i32> [[SELECT_YX]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; AVX512-NEXT:    ret <4 x i32> [[RES]]
+; CHECK-LABEL: define <4 x i32> @src_v2tov4_i32(
+; CHECK-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]], <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]], <2 x i32> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[SELECT_XZ:%.*]] = select <2 x i1> [[A]], <2 x i32> [[X]], <2 x i32> [[Z]]
+; CHECK-NEXT:    [[SELECT_YX:%.*]] = select <2 x i1> [[B]], <2 x i32> [[Y]], <2 x i32> [[X]]
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i32> [[SELECT_XZ]], <2 x i32> [[SELECT_YX]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
 ;
   %select.xz = select <2 x i1> %a, <2 x i32> %x, <2 x i32> %z
   %select.yx = select <2 x i1> %b, <2 x i32> %y, <2 x i32> %x
@@ -138,12 +105,27 @@ define <4 x i32> @src_v2tov4_i32(<2 x i1> %a, <2 x i1> %b, <2 x i32> %x, <2 x i3
 }
 
 define <8 x i32> @src_v4tov8_i32(<4 x i1> %a, <4 x i1> %b, <4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
-; CHECK-LABEL: define <8 x i32> @src_v4tov8_i32(
-; CHECK-SAME: <4 x i1> [[A:%.*]], <4 x i1> [[B:%.*]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[SELECT_XZ:%.*]] = select <4 x i1> [[A]], <4 x i32> [[X]], <4 x i32> [[Z]]
-; CHECK-NEXT:    [[SELECT_YX:%.*]] = select <4 x i1> [[B]], <4 x i32> [[Y]], <4 x i32> [[X]]
-; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x i32> [[SELECT_XZ]], <4 x i32> [[SELECT_YX]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    ret <8 x i32> [[RES]]
+; SSE-LABEL: define <8 x i32> @src_v4tov8_i32(
+; SSE-SAME: <4 x i1> [[A:%.*]], <4 x i1> [[B:%.*]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
+; SSE-NEXT:    [[SELECT_XZ:%.*]] = select <4 x i1> [[A]], <4 x i32> [[X]], <4 x i32> [[Z]]
+; SSE-NEXT:    [[SELECT_YX:%.*]] = select <4 x i1> [[B]], <4 x i32> [[Y]], <4 x i32> [[X]]
+; SSE-NEXT:    [[RES:%.*]] = shufflevector <4 x i32> [[SELECT_XZ]], <4 x i32> [[SELECT_YX]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    ret <8 x i32> [[RES]]
+;
+; AVX2-LABEL: define <8 x i32> @src_v4tov8_i32(
+; AVX2-SAME: <4 x i1> [[A:%.*]], <4 x i1> [[B:%.*]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i1> [[A]], <4 x i1> [[B]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> [[Y]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> [[X]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX2-NEXT:    [[RES:%.*]] = select <8 x i1> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> [[TMP3]]
+; AVX2-NEXT:    ret <8 x i32> [[RES]]
+;
+; AVX512-LABEL: define <8 x i32> @src_v4tov8_i32(
+; AVX512-SAME: <4 x i1> [[A:%.*]], <4 x i1> [[B:%.*]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
+; AVX512-NEXT:    [[SELECT_XZ:%.*]] = select <4 x i1> [[A]], <4 x i32> [[X]], <4 x i32> [[Z]]
+; AVX512-NEXT:    [[SELECT_YX:%.*]] = select <4 x i1> [[B]], <4 x i32> [[Y]], <4 x i32> [[X]]
+; AVX512-NEXT:    [[RES:%.*]] = shufflevector <4 x i32> [[SELECT_XZ]], <4 x i32> [[SELECT_YX]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX512-NEXT:    ret <8 x i32> [[RES]]
 ;
   %select.xz = select <4 x i1> %a, <4 x i32> %x, <4 x i32> %z
   %select.yx = select <4 x i1> %b, <4 x i32> %y, <4 x i32> %x
@@ -182,12 +164,27 @@ define <2 x i64> @src_v2tov2_i64(<2 x i1> %a, <2 x i1> %b, <2 x i64> %x, <2 x i6
 }
 
 define <4 x i64> @src_v2tov4_i64(<2 x i1> %a, <2 x i1> %b, <2 x i64> %x, <2 x i64> %y, <2 x i64> %z) {
-; CHECK-LABEL: define <4 x i64> @src_v2tov4_i64(
-; CHECK-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]], <2 x i64> [[X:%.*]], <2 x i64> [[Y:%.*]], <2 x i64> [[Z:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[SELECT_XZ:%.*]] = select <2 x i1> [[A]], <2 x i64> [[X]], <2 x i64> [[Z]]
-; CHECK-NEXT:    [[SELECT_YX:%.*]] = select <2 x i1> [[B]], <2 x i64> [[Y]], <2 x i64> [[X]]
-; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i64> [[SELECT_XZ]], <2 x i64> [[SELECT_YX]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    ret <4 x i64> [[RES]]
+; SSE-LABEL: define <4 x i64> @src_v2tov4_i64(
+; SSE-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]], <2 x i64> [[X:%.*]], <2 x i64> [[Y:%.*]], <2 x i64> [[Z:%.*]]) #[[ATTR0]] {
+; SSE-NEXT:    [[SELECT_XZ:%.*]] = select <2 x i1> [[A]], <2 x i64> [[X]], <2 x i64> [[Z]]
+; SSE-NEXT:    [[SELECT_YX:%.*]] = select <2 x i1> [[B]], <2 x i64> [[Y]], <2 x i64> [[X]]
+; SSE-NEXT:    [[RES:%.*]] = shufflevector <2 x i64> [[SELECT_XZ]], <2 x i64> [[SELECT_YX]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    ret <4 x i64> [[RES]]
+;
+; AVX2-LABEL: define <4 x i64> @src_v2tov4_i64(
+; AVX2-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]], <2 x i64> [[X:%.*]], <2 x i64> [[Y:%.*]], <2 x i64> [[Z:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i1> [[A]], <2 x i1> [[B]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[X]], <2 x i64> [[Y]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i64> [[Z]], <2 x i64> [[X]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX2-NEXT:    [[RES:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[TMP2]], <4 x i64> [[TMP3]]
+; AVX2-NEXT:    ret <4 x i64> [[RES]]
+;
+; AVX512-LABEL: define <4 x i64> @src_v2tov4_i64(
+; AVX512-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]], <2 x i64> [[X:%.*]], <2 x i64> [[Y:%.*]], <2 x i64> [[Z:%.*]]) #[[ATTR0]] {
+; AVX512-NEXT:    [[SELECT_XZ:%.*]] = select <2 x i1> [[A]], <2 x i64> [[X]], <2 x i64> [[Z]]
+; AVX512-NEXT:    [[SELECT_YX:%.*]] = select <2 x i1> [[B]], <2 x i64> [[Y]], <2 x i64> [[X]]
+; AVX512-NEXT:    [[RES:%.*]] = shufflevector <2 x i64> [[SELECT_XZ]], <2 x i64> [[SELECT_YX]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512-NEXT:    ret <4 x i64> [[RES]]
 ;
   %select.xz = select <2 x i1> %a, <2 x i64> %x, <2 x i64> %z
   %select.yx = select <2 x i1> %b, <2 x i64> %y, <2 x i64> %x
@@ -226,12 +223,27 @@ define <4 x float> @src_v2tov4_float(<2 x i1> %a, <2 x i1> %b, <2 x float> %x, <
 }
 
 define <8 x float> @src_v4tov8_float(<4 x i1> %a, <4 x i1> %b, <4 x float> %x, <4 x float> %y, <4 x float> %z) {
-; CHECK-LABEL: define <8 x float> @src_v4tov8_float(
-; CHECK-SAME: <4 x i1> [[A:%.*]], <4 x i1> [[B:%.*]], <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[SELECT_XZ:%.*]] = select <4 x i1> [[A]], <4 x float> [[X]], <4 x float> [[Z]]
-; CHECK-NEXT:    [[SELECT_YX:%.*]] = select <4 x i1> [[B]], <4 x float> [[Y]], <4 x float> [[X]]
-; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x float> [[SELECT_XZ]], <4 x float> [[SELECT_YX]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    ret <8 x float> [[RES]]
+; SSE-LABEL: define <8 x float> @src_v4tov8_float(
+; SSE-SAME: <4 x i1> [[A:%.*]], <4 x i1> [[B:%.*]], <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]]) #[[ATTR0]] {
+; SSE-NEXT:    [[SELECT_XZ:%.*]] = select <4 x i1> [[A]], <4 x float> [[X]], <4 x float> [[Z]]
+; SSE-NEXT:    [[SELECT_YX:%.*]] = select <4 x i1> [[B]], <4 x float> [[Y]], <4 x float> [[X]]
+; SSE-NEXT:    [[RES:%.*]] = shufflevector <4 x float> [[SELECT_XZ]], <4 x float> [[SELECT_YX]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    ret <8 x float> [[RES]]
+;
+; AVX2-LABEL: define <8 x float> @src_v4tov8_float(
+; AVX2-SAME: <4 x i1> [[A:%.*]], <4 x i1> [[B:%.*]], <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i1> [[A]], <4 x i1> [[B]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[X]], <4 x float> [[Y]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[Z]], <4 x float> [[X]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX2-NEXT:    [[RES:%.*]] = select <8 x i1> [[TMP1]], <8 x float> [[TMP2]], <8 x float> [[TMP3]]
+; AVX2-NEXT:    ret <8 x float> [[RES]]
+;
+; AVX512-LABEL: define <8 x float> @src_v4tov8_float(
+; AVX512-SAME: <4 x i1> [[A:%.*]], <4 x i1> [[B:%.*]], <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]]) #[[ATTR0]] {
+; AVX512-NEXT:    [[SELECT_XZ:%.*]] = select <4 x i1> [[A]], <4 x float> [[X]], <4 x float> [[Z]]
+; AVX512-NEXT:    [[SELECT_YX:%.*]] = select <4 x i1> [[B]], <4 x float> [[Y]], <4 x float> [[X]]
+; AVX512-NEXT:    [[RES:%.*]] = shufflevector <4 x float> [[SELECT_XZ]], <4 x float> [[SELECT_YX]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX512-NEXT:    ret <8 x float> [[RES]]
 ;
   %select.xz = select <4 x i1> %a, <4 x float> %x, <4 x float> %z
   %select.yx = select <4 x i1> %b, <4 x float> %y, <4 x float> %x
@@ -270,12 +282,27 @@ define <2 x double> @src_v2tov2_double(<2 x i1> %a, <2 x i1> %b, <2 x double> %x
 }
 
 define <4 x double> @src_v2tov4_double(<2 x i1> %a, <2 x i1> %b, <2 x double> %x, <2 x double> %y, <2 x double> %z) {
-; CHECK-LABEL: define <4 x double> @src_v2tov4_double(
-; CHECK-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]], <2 x double> [[X:%.*]], <2 x double> [[Y:%.*]], <2 x double> [[Z:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[SELECT_XZ:%.*]] = select <2 x i1> [[A]], <2 x double> [[X]], <2 x double> [[Z]]
-; CHECK-NEXT:    [[SELECT_YX:%.*]] = select <2 x i1> [[B]], <2 x double> [[Y]], <2 x double> [[X]]
-; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x double> [[SELECT_XZ]], <2 x double> [[SELECT_YX]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    ret <4 x double> [[RES]]
+; SSE-LABEL: define <4 x double> @src_v2tov4_double(
+; SSE-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]], <2 x double> [[X:%.*]], <2 x double> [[Y:%.*]], <2 x double> [[Z:%.*]]) #[[ATTR0]] {
+; SSE-NEXT:    [[SELECT_XZ:%.*]] = select <2 x i1> [[A]], <2 x double> [[X]], <2 x double> [[Z]]
+; SSE-NEXT:    [[SELECT_YX:%.*]] = select <2 x i1> [[B]], <2 x double> [[Y]], <2 x double> [[X]]
+; SSE-NEXT:    [[RES:%.*]] = shufflevector <2 x double> [[SELECT_XZ]], <2 x double> [[SELECT_YX]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    ret <4 x double> [[RES]]
+;
+; AVX2-LABEL: define <4 x double> @src_v2tov4_double(
+; AVX2-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]], <2 x double> [[X:%.*]], <2 x double> [[Y:%.*]], <2 x double> [[Z:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i1> [[A]], <2 x i1> [[B]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[X]], <2 x double> [[Y]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[Z]], <2 x double> [[X]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX2-NEXT:    [[RES:%.*]] = select <4 x i1> [[TMP1]], <4 x double> [[TMP2]], <4 x double> [[TMP3]]
+; AVX2-NEXT:    ret <4 x double> [[RES]]
+;
+; AVX512-LABEL: define <4 x double> @src_v2tov4_double(
+; AVX512-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]], <2 x double> [[X:%.*]], <2 x double> [[Y:%.*]], <2 x double> [[Z:%.*]]) #[[ATTR0]] {
+; AVX512-NEXT:    [[SELECT_XZ:%.*]] = select <2 x i1> [[A]], <2 x double> [[X]], <2 x double> [[Z]]
+; AVX512-NEXT:    [[SELECT_YX:%.*]] = select <2 x i1> [[B]], <2 x double> [[Y]], <2 x double> [[X]]
+; AVX512-NEXT:    [[RES:%.*]] = shufflevector <2 x double> [[SELECT_XZ]], <2 x double> [[SELECT_YX]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512-NEXT:    ret <4 x double> [[RES]]
 ;
   %select.xz = select <2 x i1> %a, <2 x double> %x, <2 x double> %z
   %select.yx = select <2 x i1> %b, <2 x double> %y, <2 x double> %x
@@ -283,6 +310,37 @@ define <4 x double> @src_v2tov4_double(<2 x i1> %a, <2 x i1> %b, <2 x double> %x
   ret <4 x double> %res
 }
 
+; ISSUE - https://github.com/llvm/llvm-project/issues/130250
+; There should be no issues when the mask elements are in the following range
+; DestVectorSize * 2 < MaskEls < SrcVectorSize * 2
+define <2 x float> @test_mask0(<4 x i1> %c, <4 x float> %x, <4 x float> %y, <4 x float> %z) {
+; CHECK-LABEL: define <2 x float> @test_mask0(
+; CHECK-SAME: <4 x i1> [[C:%.*]], <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[SELECT_XY:%.*]] = select <4 x i1> [[C]], <4 x float> [[X]], <4 x float> [[Y]]
+; CHECK-NEXT:    [[SELECT_YZ:%.*]] = select <4 x i1> [[C]], <4 x float> [[Y]], <4 x float> [[Z]]
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x float> [[SELECT_XY]], <4 x float> [[SELECT_YZ]], <2 x i32> <i32 4, i32 7>
+; CHECK-NEXT:    ret <2 x float> [[RES]]
+;
+  %select.xy = select <4 x i1> %c, <4 x float> %x, <4 x float> %y
+  %select.yz = select <4 x i1> %c, <4 x float> %y, <4 x float> %z
+  %res = shufflevector <4 x float> %select.xy, <4 x float> %select.yz, <2 x i32> <i32 4, i32 7>
+  ret <2 x float> %res
+}
+
+define <2 x float> @test_mask1(<4 x i1> %c, <4 x float> %x, <4 x float> %y, <4 x float> %z) {
+; CHECK-LABEL: define <2 x float> @test_mask1(
+; CHECK-SAME: <4 x i1> [[C:%.*]], <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[SELECT_XY:%.*]] = select <4 x i1> [[C]], <4 x float> [[X]], <4 x float> [[Y]]
+; CHECK-NEXT:    [[SELECT_YZ:%.*]] = select <4 x i1> [[C]], <4 x float> [[Y]], <4 x float> [[Z]]
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <4 x float> [[SELECT_XY]], <4 x float> [[SELECT_YZ]], <2 x i32> <i32 7, i32 4>
+; CHECK-NEXT:    ret <2 x float> [[RES]]
+;
+  %select.xy = select <4 x i1> %c, <4 x float> %x, <4 x float> %y
+  %select.yz = select <4 x i1> %c, <4 x float> %y, <4 x float> %z
+  %res = shufflevector <4 x float> %select.xy, <4 x float> %select.yz, <2 x i32> <i32 7, i32 4>
+  ret <2 x float> %res
+}
+
 ; FMF Flags
 define <4 x float> @src_v2tov4_float_nnan(<2 x i1> %a, <2 x i1> %b, <2 x float> %x, <2 x float> %y, <2 x float> %z) {
 ; SSE-LABEL: define <4 x float> @src_v2tov4_float_nnan(
diff --git a/llvm/test/tools/llvm-ar/extract.test b/llvm/test/tools/llvm-ar/extract.test
index ccad703f3a149..bf46cc0743c7d 100644
--- a/llvm/test/tools/llvm-ar/extract.test
+++ b/llvm/test/tools/llvm-ar/extract.test
@@ -1,5 +1,5 @@
 ## Test extract operation.
-# XFAIL: system-darwin
+# XFAIL: target={{.*}}-darwin{{.*}}
 
 # RUN: rm -rf %t && mkdir -p %t/extracted/
 
diff --git a/llvm/test/tools/llvm-ar/print.test b/llvm/test/tools/llvm-ar/print.test
index a27cab4a227e1..997c05f225a8b 100644
--- a/llvm/test/tools/llvm-ar/print.test
+++ b/llvm/test/tools/llvm-ar/print.test
@@ -1,5 +1,5 @@
 ## Test Print output
-# XFAIL: system-darwin
+# XFAIL: target={{.*}}-darwin{{.*}}
 
 # RUN: rm -rf %t && mkdir -p %t
 # RUN: echo file1 > %t/1.txt
diff --git a/llvm/test/tools/llvm-ctxprof-util/Inputs/invalid-flat.yaml b/llvm/test/tools/llvm-ctxprof-util/Inputs/invalid-flat.yaml
new file mode 100644
index 0000000000000..c3bc89a9a3519
--- /dev/null
+++ b/llvm/test/tools/llvm-ctxprof-util/Inputs/invalid-flat.yaml
@@ -0,0 +1,2 @@
+FlatProfiles:
+  - Guid: 1
diff --git a/llvm/test/tools/llvm-ctxprof-util/Inputs/valid-ctx-only.yaml b/llvm/test/tools/llvm-ctxprof-util/Inputs/valid-ctx-only.yaml
new file mode 100644
index 0000000000000..0de489dd0b1eb
--- /dev/null
+++ b/llvm/test/tools/llvm-ctxprof-util/Inputs/valid-ctx-only.yaml
@@ -0,0 +1,14 @@
+
+Contexts:
+  - Guid:            1000
+    Counters:        [ 1, 2, 3 ]
+    Callsites:
+      - [  ]
+      - - Guid:            2000
+          Counters:        [ 4, 5 ]
+        - Guid:            18446744073709551613
+          Counters:        [ 6, 7, 8 ]
+      - - Guid:            3000
+          Counters:        [ 40, 50 ]
+  - Guid:            18446744073709551612
+    Counters:        [ 5, 9, 10 ]
diff --git a/llvm/test/tools/llvm-ctxprof-util/Inputs/valid-flat-first.yaml b/llvm/test/tools/llvm-ctxprof-util/Inputs/valid-flat-first.yaml
new file mode 100644
index 0000000000000..5567faaa9e0a4
--- /dev/null
+++ b/llvm/test/tools/llvm-ctxprof-util/Inputs/valid-flat-first.yaml
@@ -0,0 +1,19 @@
+
+FlatProfiles:
+  - Guid:            1234
+    Counters:        [ 5, 6, 7 ]
+  - Guid:            5555
+    Counters:        [ 1 ]
+Contexts:
+  - Guid:            1000
+    Counters:        [ 1, 2, 3 ]
+    Callsites:
+      - [  ]
+      - - Guid:            2000
+          Counters:        [ 4, 5 ]
+        - Guid:            18446744073709551613
+          Counters:        [ 6, 7, 8 ]
+      - - Guid:            3000
+          Counters:        [ 40, 50 ]
+  - Guid:            18446744073709551612
+    Counters:        [ 5, 9, 10 ]
diff --git a/llvm/test/tools/llvm-ctxprof-util/Inputs/valid-flat-only.yaml b/llvm/test/tools/llvm-ctxprof-util/Inputs/valid-flat-only.yaml
new file mode 100644
index 0000000000000..98231ed70d0ec
--- /dev/null
+++ b/llvm/test/tools/llvm-ctxprof-util/Inputs/valid-flat-only.yaml
@@ -0,0 +1,6 @@
+
+FlatProfiles:
+  - Guid:            1234
+    Counters:        [ 5, 6, 7 ]
+  - Guid:            5555
+    Counters:        [ 1 ]
diff --git a/llvm/test/tools/llvm-ctxprof-util/Inputs/valid.yaml b/llvm/test/tools/llvm-ctxprof-util/Inputs/valid.yaml
index 0de489dd0b1eb..1541b0d136514 100644
--- a/llvm/test/tools/llvm-ctxprof-util/Inputs/valid.yaml
+++ b/llvm/test/tools/llvm-ctxprof-util/Inputs/valid.yaml
@@ -12,3 +12,8 @@ Contexts:
           Counters:        [ 40, 50 ]
   - Guid:            18446744073709551612
     Counters:        [ 5, 9, 10 ]
+FlatProfiles:
+  - Guid:            1234
+    Counters:        [ 5, 6, 7 ]
+  - Guid:            5555
+    Counters:        [ 1 ]
diff --git a/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util-negative.test b/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util-negative.test
index 487d5ae1d17be..f312f50ffee8e 100644
--- a/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util-negative.test
+++ b/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util-negative.test
@@ -9,6 +9,7 @@
 ; RUN: not llvm-ctxprof-util fromYAML --input %S/Inputs/invalid-no-ctx.yaml 2>&1 | FileCheck %s --check-prefix=NO_CTX
 ; RUN: not llvm-ctxprof-util fromYAML --input %S/Inputs/invalid-no-counters.yaml 2>&1 | FileCheck %s --check-prefix=NO_COUNTERS
 ; RUN: not llvm-ctxprof-util fromYAML --input %S/Inputs/invalid-bad-subctx.yaml 2>&1 | FileCheck %s --check-prefix=BAD_SUBCTX
+; RUN: not llvm-ctxprof-util fromYAML --input %S/Inputs/invalid-flat.yaml 2>&1 | FileCheck %s --check-prefix=BAD_FLAT
 ; RUN: rm -rf %t
 ; RUN: not llvm-ctxprof-util fromYAML --input %S/Inputs/valid.yaml --output %t/output.bitstream 2>&1 | FileCheck %s --check-prefix=NO_DIR
 
@@ -21,4 +22,5 @@
 ; NO_CTX: YAML:1:1: error: not a mapping
 ; NO_COUNTERS: YAML:2:5: error: missing required key 'Counters'
 ; BAD_SUBCTX: YAML:4:18: error: not a sequence
+; BAD_FLAT: YAML:2:5: error: missing required key 'Counters'
 ; NO_DIR: failed to open output
diff --git a/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util.test b/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util.test
index 07cbdd97210fb..a9e350388577c 100644
--- a/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util.test
+++ b/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util.test
@@ -8,6 +8,21 @@
 ; RUN: llvm-ctxprof-util toYAML -input %t/valid.bitstream -output %t/valid2.yaml
 ; RUN: diff %t/valid2.yaml %S/Inputs/valid.yaml
 
+
+; RUN: llvm-ctxprof-util fromYAML -input %S/Inputs/valid-ctx-only.yaml -output %t/valid-ctx-only.bitstream
+; RUN: llvm-ctxprof-util toYAML -input %t/valid-ctx-only.bitstream -output %t/valid-ctx-only.yaml
+; RUN: diff %t/valid-ctx-only.yaml %S/Inputs/valid-ctx-only.yaml
+
+; RUN: llvm-ctxprof-util fromYAML -input %S/Inputs/valid-flat-only.yaml -output %t/valid-flat-only.bitstream
+; RUN: llvm-ctxprof-util toYAML -input %t/valid-flat-only.bitstream -output %t/valid-flat-only.yaml
+; RUN: diff %t/valid-flat-only.yaml %S/Inputs/valid-flat-only.yaml
+
+; This case is the "valid.yaml" case but with the flat profile first.
+; The output, though, should match valid.yaml
+; RUN: llvm-ctxprof-util fromYAML -input %S/Inputs/valid-flat-first.yaml -output %t/valid-flat-first.bitstream
+; RUN: llvm-ctxprof-util toYAML -input %t/valid-flat-first.bitstream -output %t/valid-flat-first.yaml
+; RUN: diff %t/valid-flat-first.yaml %S/Inputs/valid.yaml
+
 ; For the valid case, check against a reference output.
 ; Note that uint64_t are printed as signed values by llvm-bcanalyzer:
 ;  * 18446744073709551613 in yaml is -3 in the output
@@ -22,7 +37,7 @@
 ; EMPTY-NEXT: </Metadata>
 
 ; VALID:      <BLOCKINFO_BLOCK/>
-; VALID-NEXT: <Metadata NumWords=33 BlockCodeSize=2>
+; VALID-NEXT: <Metadata NumWords=45 BlockCodeSize=2>
 ; VALID-NEXT:   <Version op0=2/>
 ; VALID-NEXT:   <Contexts NumWords=29 BlockCodeSize=2>
 ; VALID-NEXT:     <Root NumWords=20 BlockCodeSize=2>
@@ -49,4 +64,14 @@
 ; VALID-NEXT:       <Counters op0=5 op1=9 op2=10/>
 ; VALID-NEXT:     </Root>
 ; VALID-NEXT:   </Contexts>
+; VALID-NEXT:   <FlatProfiles NumWords=10 BlockCodeSize=2>
+; VALID-NEXT:       <Flat NumWords=3 BlockCodeSize=2>
+; VALID-NEXT:         <GUID op0=1234/>
+; VALID-NEXT:         <Counters op0=5 op1=6 op2=7/>
+; VALID-NEXT:       </Flat>
+; VALID-NEXT:       <Flat NumWords=2 BlockCodeSize=2>
+; VALID-NEXT:         <GUID op0=5555/>
+; VALID-NEXT:         <Counters op0=1/>
+; VALID-NEXT:       </Flat>
+; VALID-NEXT:   </FlatProfiles>
 ; VALID-NEXT: </Metadata>
\ No newline at end of file
diff --git a/llvm/test/tools/llvm-exegesis/AArch64/setReg_init_check.s b/llvm/test/tools/llvm-exegesis/AArch64/setReg_init_check.s
index da28388c96094..a4350fc6dc2cb 100644
--- a/llvm/test/tools/llvm-exegesis/AArch64/setReg_init_check.s
+++ b/llvm/test/tools/llvm-exegesis/AArch64/setReg_init_check.s
@@ -37,3 +37,39 @@ RUN: FileCheck %s --check-prefix=FPR64-ASM < %t.s
 FPR64-ASM:          <foo>:
 FPR64-ASM:          movi d{{[0-9]+}}, #0000000000000000
 FPR64-ASM-NEXT:     addv h{{[0-9]+}}, v{{[0-9]+}}.4h
+
+## FPR32 Register Class Initialization Testcase
+RUN: llvm-exegesis -mcpu=neoverse-v2 -mode=latency --dump-object-to-disk=%d --opcode-name=FABSSr --benchmark-phase=assemble-measured-code 2>&1
+RUN: llvm-objdump -d %d > %t.s
+RUN: FileCheck %s --check-prefix=FPR32-ASM < %t.s
+FPR32-ASM:         <foo>:
+FPR32-ASM:         movi d{{[0-9]+}}, #0000000000000000
+FPR32-ASM-NEXT:    fabs s{{[0-9]+}}, s{{[0-9]+}}
+
+
+## FPR16 Register Class Initialization Testcase
+RUN: llvm-exegesis -mcpu=neoverse-v2 -mode=latency --dump-object-to-disk=%d --opcode-name=FABSHr --benchmark-phase=assemble-measured-code 2>&1
+RUN: llvm-objdump -d %d > %t.s
+RUN: FileCheck %s --check-prefix=FPR16-ASM < %t.s
+FPR16-ASM:         <foo>:
+FPR16-ASM:         movi d{{[0-9]+}}, #0000000000000000
+FPR16-ASM-NEXT:    fabs h{{[0-9]+}}, h{{[0-9]+}}
+
+## FPR8 Register Class Initialization Testcase
+RUN: llvm-exegesis -mcpu=neoverse-v2 -mode=latency --dump-object-to-disk=%d --opcode-name=SQABSv1i8 --benchmark-phase=assemble-measured-code 2>&1
+RUN: llvm-objdump -d %d > %t.s
+RUN: FileCheck %s --check-prefix=FPR8-ASM < %t.s
+FPR8-ASM:         <foo>:
+FPR8-ASM:         movi    d{{[0-9]+}}, #0000000000000000
+FPR8-ASM-NEXT:    sqabs   b{{[0-9]+}}, b{{[0-9]+}}
+
+
+## FPCR Register Class Initialization Testcase
+RUN: llvm-exegesis -mcpu=neoverse-v2 -mode=latency --dump-object-to-disk=%d --opcode-name=BFCVT --benchmark-phase=assemble-measured-code 2>&1
+RUN: llvm-objdump -d %d > %t.s
+RUN: FileCheck %s --check-prefix=FPCR-ASM < %t.s
+FPCR-ASM:         <foo>:
+FPCR-ASM:         movi    d{{[0-9]+}}, #0000000000000000
+FPCR-ASM-NEXT:    mov     x8, #0x0
+FPCR-ASM-NEXT:    msr     FPCR, x8
+FPCR-ASM-NEXT:    bfcvt   h{{[0-9]+}}, s{{[0-9]+}}
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-basic-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-basic-instructions.s
index a5330b9d6d2d6..e1c7bf56f45f2 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-basic-instructions.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-basic-instructions.s
@@ -24,21 +24,21 @@ adds     w13, w23, #291, lsl #12
 cmn      w2, #4095
 adds     w20, wsp, #0
 cmn      x3, #1, lsl #12
+cmp      wsp, #2342
 cmp      sp, #20, lsl #12
 cmp      x30, #4095
 subs     x4, sp, #3822
 cmn      w3, #291, lsl #12
 cmn      wsp, #1365
 cmn      sp, #1092, lsl #12
-mov      sp, x30
-mov      wsp, w20
-mov      x11, sp
-mov      w24, wsp
+mov      x10, #-63432
 
 #------------------------------------------------------------------------------
 # Add-subtract (shifted register)
 #------------------------------------------------------------------------------
 
+add      wsp, wsp, w10
+add      x25, x9, w25, uxtb
 add      w3, w5, w7
 add      wzr, w3, w5
 add      w20, wzr, w4
@@ -66,6 +66,8 @@ add      x2, x3, x4, asr #0
 add      x5, x6, x7, asr #21
 add      x8, x9, x10, asr #63
 adds     w3, w5, w7
+adds     w17, wsp, w25
+adds     x13, x23, w8, uxtb
 cmn      w3, w5
 adds     w20, wzr, w4
 adds     w4, w6, wzr
@@ -115,6 +117,9 @@ sub      x27, x28, x29, lsr #63
 sub      x2, x3, x4, asr #0
 sub      x5, x6, x7, asr #21
 sub      x8, x9, x10, asr #63
+sub      w13, wsp, w10
+sub      x16, x2, w19, uxtb
+subs     x13, x15, x14, sxtx #1
 subs     w3, w5, w7
 cmp      w3, w5
 subs     w4, w6, wzr
@@ -173,6 +178,8 @@ cmp      w14, w15, lsr #21
 cmp      w18, w19, asr #0
 cmp      w20, w21, asr #22
 cmp      w22, w23, asr #31
+cmp      wsp, w26
+cmp      x16, w27, uxtb
 cmp      x0, x3
 cmp      xzr, x4
 cmp      x5, xzr
@@ -187,6 +194,10 @@ cmp      x20, x21, asr #55
 cmp      x22, x23, asr #63
 cmp      wzr, w0
 cmp      xzr, x0
+mov      sp, x30
+mov      wsp, w20
+mov      x11, sp
+mov      w24, wsp
 
 #------------------------------------------------------------------------------
 # Add-subtract (shifted register)
@@ -269,15 +280,11 @@ lsr      w3, w2, #0
 lsr      w9, w10, #31
 lsr      x20, x21, #63
 lsr      wzr, wzr, #3
-lsr      w3, w2, #0
 lsl      w9, w10, #31
 lsl      x20, x21, #63
 lsl      w1, wzr, #3
-sbfx     w9, w10, #0, #1
 sbfiz    x2, x3, #63, #1
-asr      x19, x20, #0
 sbfiz    x9, x10, #5, #59
-asr      w9, w10, #0
 sbfiz    w11, w12, #31, #1
 sbfiz    w13, w14, #29, #3
 sbfiz    xzr, xzr, #10, #11
@@ -289,11 +296,8 @@ asr      w9, w10, #0
 asr      w11, w12, #31
 asr      w13, w14, #29
 sbfx     xzr, xzr, #10, #11
-bfxil    w9, w10, #0, #1
 bfi      x2, x3, #63, #1
-bfxil    x19, x20, #0, #64
 bfi      x9, x10, #5, #59
-bfxil    w9, w10, #0, #32
 bfi      w11, w12, #31, #1
 bfi      w13, w14, #29, #3
 bfi      xzr, xzr, #10, #11
@@ -305,11 +309,8 @@ bfxil    w9, w10, #0, #32
 bfxil    w11, w12, #31, #1
 bfxil    w13, w14, #29, #3
 bfxil    xzr, xzr, #10, #11
-ubfx     w9, w10, #0, #1
 lsl      x2, x3, #63
-lsr      x19, x20, #0
 lsl      x9, x10, #5
-lsr      w9, w10, #0
 lsl      w11, w12, #31
 lsl      w13, w14, #29
 ubfiz    xzr, xzr, #10, #11
@@ -332,6 +333,7 @@ cbnz     x2, #-4
 cbnz     x26, #1048572
 cbz      wzr, #0
 cbnz     xzr, #0
+cbnz     w21, test
 
 #------------------------------------------------------------------------------
 # Conditional branch (immediate)
@@ -440,7 +442,6 @@ cneg     x3, x5, gt
 cneg     xzr, x4, le
 cneg     x9, xzr, lt
 csneg    x4, x8, x8, al
-csinv    w9, w8, w8, nv
 
 #------------------------------------------------------------------------------
 # Data-processing (1 source)
@@ -473,7 +474,7 @@ lsr	x20, x21, x22
 asr	w23, w24, w25
 asr	x26, x27, x28
 ror	w0, w1, w2
-ror    x3, x4, x5
+ror     x3, x4, x5
 lsl	w6, w7, w8
 lsl	x9, x10, x11
 lsr	w12, w13, w14
@@ -487,6 +488,16 @@ ror	x27, x28, x29
 # Data-processing (3 sources)
 #------------------------------------------------------------------------------
 
+crc32cb  w30, w23, w15
+crc32cb  w31, w12, w14
+crc32cb  w28, w10, w11
+crc32b   w27, w12, w15
+crc32h   w3, w15, w21
+crc32w   w9, w18, w24
+crc32x   w19, w6, x25
+crc32ch  w25, w26, w16
+crc32cw  w27, w12, w23
+crc32cx  w21, w28, x5
 smulh    x30, x29, x28
 smulh    xzr, x27, x26
 umulh    x30, x29, x28
@@ -529,7 +540,6 @@ umull    x19, w20, w21
 umsubl   x3, w5, w2, x9
 umsubl   x16, w17, wzr, x18
 umnegl   x19, w20, w21
-smulh    x30, x29, x28
 smulh    x23, x22, xzr
 umulh    x23, x22, xzr
 mul      x19, x20, xzr
@@ -547,26 +557,26 @@ extr     w3, w5, w7, #0
 extr     w11, w13, w17, #31
 extr     x3, x5, x7, #15
 extr     x11, x13, x17, #63
-ror     x19, x23, #24
-ror     x29, xzr, #63
-ror     w9, w13, #31
+ror      x19, x23, #24
+ror      x29, xzr, #63
+ror      w9, w13, #31
 
 #------------------------------------------------------------------------------
 # Floating-point compare
 #------------------------------------------------------------------------------
 
+fcmp    h5, h21
+fcmp    h5, #0.0
+fcmpe   h22, h21
+fcmpe   h13, #0.0
 fcmp    s3, s5
 fcmp    s31, #0.0
-fcmp    s31, #0.0
 fcmpe   s29, s30
 fcmpe   s15, #0.0
-fcmpe   s15, #0.0
 fcmp    d4, d12
 fcmp    d23, #0.0
-fcmp    d23, #0.0
 fcmpe   d26, d22
 fcmpe   d29, #0.0
-fcmpe   d29, #0.0
 
 #------------------------------------------------------------------------------
 # Floating-point conditional compare
@@ -578,6 +588,8 @@ fccmp s31, s15, #13, hs
 fccmp d9, d31, #0, le
 fccmp d3, d0, #15, gt
 fccmp d31, d5, #7, ne
+fccmp h31, h3, #11, hs
+fccmpe h6, h1, #12, ne
 fccmpe s1, s31, #0, eq
 fccmpe s3, s0, #15, hs
 fccmpe s31, s15, #13, hs
@@ -591,27 +603,38 @@ fccmpe d31, d5, #7, ne
 
 fcsel s3, s20, s9, pl
 fcsel d9, d10, d11, mi
+fcsel h26, h2, h11, hs
 
 #------------------------------------------------------------------------------
 # Floating-point data-processing (1 source)
 #------------------------------------------------------------------------------
 
+fmov     h18, h28
 fmov     s0, s1
 fabs     s2, s3
+fneg     h2, h9
 fneg     s4, s5
 fsqrt    s6, s7
 fcvt     d8, s9
 fcvt     h10, s11
+frintn   h12, h3
 frintn   s12, s13
+frintp   h17, h31
 frintp   s14, s15
+frintm   h0, h21
 frintm   s16, s17
+frintz   h10, h29
 frintz   s18, s19
+frinta   h22, h10
 frinta   s20, s21
+frintx   h4, h5
 frintx   s22, s23
 frinti   s24, s25
+frinti   h31, h14
 fmov     d0, d1
 fabs     d2, d3
 fneg     d4, d5
+fsqrt    h13, h24
 fsqrt    d6, d7
 fcvt     s8, d9
 fcvt     h10, d11
@@ -630,13 +653,19 @@ fcvt     d28, h29
 #------------------------------------------------------------------------------
 
 fmul     s20, s19, s17
+fdiv     h1, h26, h23
 fdiv     s1, s2, s3
+fadd     h23, h27, h22
 fadd     s4, s5, s6
+fsub     h20, h11, h18
 fsub     s7, s8, s9
 fmax     s10, s11, s12
+fmax     h8, h7, h11
 fmin     s13, s14, s15
+fmaxnm   h29, h13, h14
 fmaxnm   s16, s17, s18
 fminnm   s19, s20, s21
+fnmul    h3, h15, h7
 fnmul    s22, s23, s2
 fmul     d20, d19, d17
 fdiv     d1, d2, d3
@@ -644,22 +673,28 @@ fadd     d4, d5, d6
 fsub     d7, d8, d9
 fmax     d10, d11, d12
 fmin     d13, d14, d15
+fmin     h4, h13, h17
 fmaxnm   d16, d17, d18
 fminnm   d19, d20, d21
+fminnm   h29, h23, h17
 fnmul    d22, d23, d24
 
 #------------------------------------------------------------------------------
 # Floating-point data-processing (1 source)
 #------------------------------------------------------------------------------
 
+fmadd h27, h0, h6, h28
 fmadd s3, s5, s6, s31
 fmadd d3, d13, d0, d23
+fmsub h25, h28, h12, h24
 fmsub s3, s5, s6, s31
 fmsub d3, d13, d0, d23
+fnmadd h3, h18, h31, h24
 fnmadd s3, s5, s6, s31
 fnmadd d3, d13, d0, d23
 fnmsub s3, s5, s6, s31
 fnmsub d3, d13, d0, d23
+fnmsub h3, h29, h24, h17
 
 #------------------------------------------------------------------------------
 # Floating-point <-> fixed-point conversion
@@ -814,8 +849,12 @@ fcvtas   w25, d26
 fcvtas   x27, d28
 fcvtau   w29, d30
 fcvtau   xzr, d0
+fmov     h6, w5
+fmov     h16, x27
+fmov     w15, h31
 fmov     w3, s9
 fmov     s9, w3
+fmov     x21, h14
 fmov     x20, d31
 fmov     d1, x15
 fmov     x3, v12.d[1]
@@ -825,6 +864,7 @@ fmov     v1.d[1], x19
 # Floating-point immediate
 #------------------------------------------------------------------------------
 
+fmov     h29, #0.50000000
 fmov     s2, #0.12500000
 fmov     s3, #1.00000000
 fmov     d30, #16.00000000
@@ -858,13 +898,10 @@ ldxrb      w30, [x0]
 ldxrh      w17, [x4]
 ldxr       w22, [sp]
 ldxr       x11, [x29]
-ldxr       x11, [x29]
-ldxr       x11, [x29]
 stxp       w12, w11, w10, [sp]
 stxp       wzr, x27, x9, [x12]
 ldxp       w0, wzr, [sp]
 ldxp       x17, x0, [x18]
-ldxp       x17, x0, [x18]
 stlxrb     w12, w22, [x0]
 stlxrh     w10, w1, [x1]
 stlxr      w9, w2, [x2]
@@ -873,25 +910,32 @@ ldaxrb     w8, [x4]
 ldaxrh     w7, [x5]
 ldaxr      w6, [sp]
 ldaxr      x5, [x6]
-ldaxr      x5, [x6]
-ldaxr      x5, [x6]
 stlxp      w4, w5, w6, [sp]
 stlxp      wzr, x6, x7, [x1]
 ldaxp      w5, w18, [sp]
 ldaxp      x6, x19, [x22]
-ldaxp      x6, x19, [x22]
+ldapur     w7, [x24]
+ldapur     x20, [x13]
+ldapurb    w13, [x17]
+ldapurh    w3, [x22]
+ldapursb   w7, [x8]
+ldapursb   x29, [x7]
+ldapursh   w17, [x19]
+ldapursh   x3, [x3]
+ldapursw   x3, [x18]
 stlrb      w24, [sp]
 stlrh      w25, [x30]
 stlr       w26, [x29]
 stlr       x27, [x28]
-stlr       x27, [x28]
-stlr       x27, [x28]
+stlur      w3, [x27]
+stlur      x23, [x25]
+stlurb     w30, [x17]
+stlurh     w9, [x29]
+ldarb      w16, [x21]
 ldarb      w23, [sp]
 ldarh      w22, [x30]
 ldar       wzr, [x29]
 ldar       x21, [x28]
-ldar       x21, [x28]
-ldar       x21, [x28]
 
 #------------------------------------------------------------------------------
 # Load/store (unscaled  immediate)
@@ -1102,16 +1146,23 @@ ldr      w17, [sp, #16380]
 ldr      w13, [x2, #4]
 ldrsw    x2, [x5, #4]
 ldrsw    x23, [sp, #16380]
+ldrsw    x21, [x25, x7]
 ldrh     w2, [x4]
 ldrsh    w23, [x6, #8190]
 ldrsh    wzr, [sp, #2]
 ldrsh    x29, [x2, #2]
+ldrsh    x25, [x8, w13, uxtw]
 ldrb     w26, [x3, #121]
 ldrb     w12, [x2]
 ldrsb    w27, [sp, #4095]
 ldrsb    xzr, [x15]
+ldrsb    x12, [x28, x27]
 str      x30, [sp]
 str      w20, [x4, #16380]
+str      b5, [x11]
+str      h23, [x15]
+str      s25, [x19]
+str      d15, [x2]
 strh     w17, [sp, #8190]
 strb     w23, [x3, #4095]
 strb     wzr, [x2]
@@ -1120,6 +1171,7 @@ ldr      h20, [x2, #8190]
 ldr      s10, [x19, #16380]
 ldr      d3, [x10, #32760]
 str      q12, [sp, #65520]
+ldr      q14, [x6, #4624]
 
 #------------------------------------------------------------------------------
 # Load/store (register offset)
@@ -1130,6 +1182,7 @@ ldrb     w9, [x27, x6]
 ldrsb    w10, [x30, x7]
 ldrb     w11, [x29, x3, sxtx]
 strb     w12, [x28, xzr, sxtx]
+strb     w5, [x26, w7, uxtw]
 ldrb     w14, [x26, w6, uxtw]
 ldrsb    w15, [x25, w7, uxtw]
 ldrb     w17, [x23, w9, sxtw]
@@ -1146,6 +1199,18 @@ ldrsh    w16, [x24, w8, uxtw #1]
 ldrh     w17, [x23, w9, sxtw]
 ldrh     w18, [x22, w10, sxtw]
 strh     w19, [x21, wzr, sxtw #1]
+ldr      b25, [x21, w8, uxtw]
+ldr      b8, [x30, x10]
+str      b14, [x13, x25]
+str      b30, [x16, w26, uxtw]
+ldr      h6, [x4, w4, uxtw]
+ldr      h11, [x13, x9]
+str      h16, [x5, x24]
+str      h15, [x15, w15, uxtw]
+ldr      s12, [x30, w5, uxtw]
+ldr      d24, [x26, w7, uxtw]
+str      s20, [x24, w10, uxtw]
+str      d5, [x26, x6]
 ldr      w3, [sp, x5]
 ldr      s9, [x27, x6]
 ldr      w10, [x30, x7, lsl #2]
@@ -1166,6 +1231,7 @@ ldr      x12, [x28, xzr, sxtx]
 ldr      x13, [x27, x5, sxtx #3]
 prfm     pldl1keep, [x26, w6, uxtw]
 ldr      x15, [x25, w7, uxtw]
+str      x27, [x26, w24, uxtw]
 ldr      x16, [x24, w8, uxtw #3]
 ldr      x17, [x23, w9, sxtw]
 ldr      x18, [x22, w10, sxtw]
@@ -1202,6 +1268,8 @@ stp      s27, s26, [sp, #-256]
 ldp      s1, s2, [x3, #44]
 stp      d3, d5, [x9, #504]
 stp      d7, d11, [x10, #-512]
+stnp     x20, x16, [x8]
+stp      x3, x6, [x16]
 ldp      d2, d3, [x30, #-8]
 stp      q3, q5, [sp]
 stp      q17, q19, [sp, #1008]
@@ -1280,6 +1348,12 @@ ldnp      q23, q29, [x1, #-1024]
 # Logical (immediate)
 #------------------------------------------------------------------------------
 
+and      wsp, w16, #0xe00
+and      x2, x22, #0x1e00
+ands     w14, w8, #0x70
+ands     x4, x10, #0x60
+eor      wsp, w4, #0xe00
+eor      x27, x25, #0x1e00
 mov      w3, #983055
 mov      x10, #-6148914691236517206
 
@@ -1301,6 +1375,10 @@ and      w3, wzr, w20, ror #2
 and      x7, x20, xzr, asr #63
 bic      x13, x20, x14, lsl #47
 bic      w2, w7, w9
+eon      w29, w4, w19
+eon      x19, x12, x2
+eor      w8, w27, w2
+eor      x22, x16, x6
 orr      w2, w7, w0, asr #31
 orr      x8, x9, x10, lsl #12
 orn      x3, x5, x7, asr #2
@@ -1332,6 +1410,12 @@ movk     x7, #0, lsl #32
 movz     x8, #0, lsl #48
 movk     x9, #0, lsl #48
 
+#------------------------------------------------------------------------------
+# Move immediate to Special Register
+#------------------------------------------------------------------------------
+
+msr     DAIFSet, #0
+
 #------------------------------------------------------------------------------
 # PC-relative addressing
 #------------------------------------------------------------------------------
@@ -1347,7 +1431,9 @@ adr      x0, #262144
 tbz     x12, #62, #0
 tbz     x12, #62, #4
 tbz     x12, #62, #-32768
+tbz     w17, #16, test
 tbnz    x12, #60, #32764
+tbnz	w3, #28, test
 
 #------------------------------------------------------------------------------
 # Unconditional branch (immediate)
@@ -1356,6 +1442,7 @@ tbnz    x12, #60, #32764
 b        #4
 b        #-4
 b        #134217724
+bl       test
 
 #------------------------------------------------------------------------------
 # Unconditional branch (register)
@@ -1396,16 +1483,16 @@ drps
 # CHECK-NEXT:  1      1     0.33                        cmn	w2, #4095
 # CHECK-NEXT:  1      1     0.33                        adds	w20, wsp, #0
 # CHECK-NEXT:  1      1     0.33                        cmn	x3, #1, lsl #12
+# CHECK-NEXT:  1      1     0.33                        cmp	wsp, #2342
 # CHECK-NEXT:  1      1     0.33                        cmp	sp, #20, lsl #12
 # CHECK-NEXT:  1      1     0.33                        cmp	x30, #4095
 # CHECK-NEXT:  1      1     0.33                        subs	x4, sp, #3822
 # CHECK-NEXT:  1      1     0.33                        cmn	w3, #291, lsl #12
 # CHECK-NEXT:  1      1     0.33                        cmn	wsp, #1365
 # CHECK-NEXT:  1      1     0.33                        cmn	sp, #1092, lsl #12
-# CHECK-NEXT:  1      1     0.25                        mov	sp, x30
-# CHECK-NEXT:  1      1     0.25                        mov	wsp, w20
-# CHECK-NEXT:  1      1     0.25                        mov	x11, sp
-# CHECK-NEXT:  1      1     0.25                        mov	w24, wsp
+# CHECK-NEXT:  1      1     0.25                        mov	x10, #-63432
+# CHECK-NEXT:  1      2     0.50                        add	wsp, wsp, w10
+# CHECK-NEXT:  1      2     0.50                        add	x25, x9, w25, uxtb
 # CHECK-NEXT:  1      1     0.25                        add	w3, w5, w7
 # CHECK-NEXT:  1      1     0.25                        add	wzr, w3, w5
 # CHECK-NEXT:  1      1     0.25                        add	w20, wzr, w4
@@ -1433,6 +1520,8 @@ drps
 # CHECK-NEXT:  1      2     0.50                        add	x5, x6, x7, asr #21
 # CHECK-NEXT:  1      2     0.50                        add	x8, x9, x10, asr #63
 # CHECK-NEXT:  1      1     0.33                        adds	w3, w5, w7
+# CHECK-NEXT:  1      2     0.50                        adds	w17, wsp, w25
+# CHECK-NEXT:  1      1     0.33                        adds	x13, x23, w8, uxtb
 # CHECK-NEXT:  1      1     0.33                        cmn	w3, w5
 # CHECK-NEXT:  1      1     0.33                        adds	w20, wzr, w4
 # CHECK-NEXT:  1      1     0.33                        adds	w4, w6, wzr
@@ -1482,6 +1571,9 @@ drps
 # CHECK-NEXT:  1      2     0.50                        sub	x2, x3, x4, asr #0
 # CHECK-NEXT:  1      2     0.50                        sub	x5, x6, x7, asr #21
 # CHECK-NEXT:  1      2     0.50                        sub	x8, x9, x10, asr #63
+# CHECK-NEXT:  1      2     0.50                        sub	w13, wsp, w10
+# CHECK-NEXT:  1      2     0.50                        sub	x16, x2, w19, uxtb
+# CHECK-NEXT:  1      2     0.50                        subs	x13, x15, x14, sxtx #1
 # CHECK-NEXT:  1      1     0.33                        subs	w3, w5, w7
 # CHECK-NEXT:  1      1     0.33                        cmp	w3, w5
 # CHECK-NEXT:  1      1     0.33                        subs	w4, w6, wzr
@@ -1540,6 +1632,8 @@ drps
 # CHECK-NEXT:  1      2     0.50                        cmp	w18, w19, asr #0
 # CHECK-NEXT:  1      2     0.50                        cmp	w20, w21, asr #22
 # CHECK-NEXT:  1      2     0.50                        cmp	w22, w23, asr #31
+# CHECK-NEXT:  1      2     0.50                        cmp	wsp, w26
+# CHECK-NEXT:  1      1     0.33                        cmp	x16, w27, uxtb
 # CHECK-NEXT:  1      1     0.33                        cmp	x0, x3
 # CHECK-NEXT:  1      1     0.33                        cmp	xzr, x4
 # CHECK-NEXT:  1      1     0.33                        cmp	x5, xzr
@@ -1554,6 +1648,10 @@ drps
 # CHECK-NEXT:  1      2     0.50                        cmp	x22, x23, asr #63
 # CHECK-NEXT:  1      1     0.33                        cmp	wzr, w0
 # CHECK-NEXT:  1      1     0.33                        cmp	xzr, x0
+# CHECK-NEXT:  1      1     0.25                        mov	sp, x30
+# CHECK-NEXT:  1      1     0.25                        mov	wsp, w20
+# CHECK-NEXT:  1      1     0.25                        mov	x11, sp
+# CHECK-NEXT:  1      1     0.25                        mov	w24, wsp
 # CHECK-NEXT:  1      1     0.25                        adc	w29, w27, w25
 # CHECK-NEXT:  1      1     0.25                        adc	wzr, w3, w4
 # CHECK-NEXT:  1      1     0.25                        adc	w9, wzr, w10
@@ -1626,15 +1724,11 @@ drps
 # CHECK-NEXT:  1      1     0.25                        lsr	w9, w10, #31
 # CHECK-NEXT:  1      1     0.25                        lsr	x20, x21, #63
 # CHECK-NEXT:  1      1     0.25                        lsr	wzr, wzr, #3
-# CHECK-NEXT:  1      1     0.25                        lsr	w3, w2, #0
 # CHECK-NEXT:  1      1     0.25                        lsl	w9, w10, #31
 # CHECK-NEXT:  1      1     0.25                        lsl	x20, x21, #63
 # CHECK-NEXT:  1      1     0.25                        lsl	w1, wzr, #3
-# CHECK-NEXT:  1      1     0.25                        sbfx	w9, w10, #0, #1
 # CHECK-NEXT:  1      1     0.25                        sbfiz	x2, x3, #63, #1
-# CHECK-NEXT:  1      1     0.25                        asr	x19, x20, #0
 # CHECK-NEXT:  1      1     0.25                        sbfiz	x9, x10, #5, #59
-# CHECK-NEXT:  1      1     0.25                        asr	w9, w10, #0
 # CHECK-NEXT:  1      1     0.25                        sbfiz	w11, w12, #31, #1
 # CHECK-NEXT:  1      1     0.25                        sbfiz	w13, w14, #29, #3
 # CHECK-NEXT:  1      1     0.25                        sbfiz	xzr, xzr, #10, #11
@@ -1646,11 +1740,8 @@ drps
 # CHECK-NEXT:  1      1     0.25                        asr	w11, w12, #31
 # CHECK-NEXT:  1      1     0.25                        asr	w13, w14, #29
 # CHECK-NEXT:  1      1     0.25                        sbfx	xzr, xzr, #10, #11
-# CHECK-NEXT:  1      2     0.50                        bfxil	w9, w10, #0, #1
 # CHECK-NEXT:  1      2     0.50                        bfi	x2, x3, #63, #1
-# CHECK-NEXT:  1      2     0.50                        bfxil	x19, x20, #0, #64
 # CHECK-NEXT:  1      2     0.50                        bfi	x9, x10, #5, #59
-# CHECK-NEXT:  1      2     0.50                        bfxil	w9, w10, #0, #32
 # CHECK-NEXT:  1      2     0.50                        bfi	w11, w12, #31, #1
 # CHECK-NEXT:  1      2     0.50                        bfi	w13, w14, #29, #3
 # CHECK-NEXT:  1      2     0.50                        bfc	xzr, #10, #11
@@ -1662,11 +1753,8 @@ drps
 # CHECK-NEXT:  1      2     0.50                        bfxil	w11, w12, #31, #1
 # CHECK-NEXT:  1      2     0.50                        bfxil	w13, w14, #29, #3
 # CHECK-NEXT:  1      2     0.50                        bfxil	xzr, xzr, #10, #11
-# CHECK-NEXT:  1      1     0.25                        ubfx	w9, w10, #0, #1
 # CHECK-NEXT:  1      1     0.25                        lsl	x2, x3, #63
-# CHECK-NEXT:  1      1     0.25                        lsr	x19, x20, #0
 # CHECK-NEXT:  1      1     0.25                        lsl	x9, x10, #5
-# CHECK-NEXT:  1      1     0.25                        lsr	w9, w10, #0
 # CHECK-NEXT:  1      1     0.25                        lsl	w11, w12, #31
 # CHECK-NEXT:  1      1     0.25                        lsl	w13, w14, #29
 # CHECK-NEXT:  1      1     0.25                        ubfiz	xzr, xzr, #10, #11
@@ -1684,6 +1772,7 @@ drps
 # CHECK-NEXT:  1      1     0.50                        cbnz	x26, #1048572
 # CHECK-NEXT:  1      1     0.50                        cbz	wzr, #0
 # CHECK-NEXT:  1      1     0.50                        cbnz	xzr, #0
+# CHECK-NEXT:  1      1     0.50                        cbnz	w21, test
 # CHECK-NEXT:  1      1     0.50                        b.ne	#4
 # CHECK-NEXT:  1      1     0.50                        b.ge	#1048572
 # CHECK-NEXT:  1      1     0.50                        b.ge	#-4
@@ -1772,7 +1861,6 @@ drps
 # CHECK-NEXT:  1      1     0.25                        cneg	xzr, x4, le
 # CHECK-NEXT:  1      1     0.25                        cneg	x9, xzr, lt
 # CHECK-NEXT:  1      1     0.25                        csneg	x4, x8, x8, al
-# CHECK-NEXT:  1      1     0.25                        csinv	w9, w8, w8, nv
 # CHECK-NEXT:  1      1     0.25                        rbit	w0, w7
 # CHECK-NEXT:  1      1     0.25                        rbit	x18, x3
 # CHECK-NEXT:  1      1     0.25                        rev16	w17, w1
@@ -1804,6 +1892,16 @@ drps
 # CHECK-NEXT:  1      1     0.25                        asr	x21, x22, x23
 # CHECK-NEXT:  1      1     0.25                        ror	w24, w25, w26
 # CHECK-NEXT:  1      1     0.25                        ror	x27, x28, x29
+# CHECK-NEXT:  1      2     1.00                        crc32cb	w30, w23, w15
+# CHECK-NEXT:  1      2     1.00                        crc32cb	wzr, w12, w14
+# CHECK-NEXT:  1      2     1.00                        crc32cb	w28, w10, w11
+# CHECK-NEXT:  1      2     1.00                        crc32b	w27, w12, w15
+# CHECK-NEXT:  1      2     1.00                        crc32h	w3, w15, w21
+# CHECK-NEXT:  1      2     1.00                        crc32w	w9, w18, w24
+# CHECK-NEXT:  1      2     1.00                        crc32x	w19, w6, x25
+# CHECK-NEXT:  1      2     1.00                        crc32ch	w25, w26, w16
+# CHECK-NEXT:  1      2     1.00                        crc32cw	w27, w12, w23
+# CHECK-NEXT:  1      2     1.00                        crc32cx	w21, w28, x5
 # CHECK-NEXT:  1      3     0.50                        smulh	x30, x29, x28
 # CHECK-NEXT:  1      3     0.50                        smulh	xzr, x27, x26
 # CHECK-NEXT:  1      3     0.50                        umulh	x30, x29, x28
@@ -1846,7 +1944,6 @@ drps
 # CHECK-NEXT:  1      2     1.00                        umsubl	x3, w5, w2, x9
 # CHECK-NEXT:  1      2     1.00                        umsubl	x16, w17, wzr, x18
 # CHECK-NEXT:  1      2     0.50                        umnegl	x19, w20, w21
-# CHECK-NEXT:  1      3     0.50                        smulh	x30, x29, x28
 # CHECK-NEXT:  1      3     0.50                        smulh	x23, x22, xzr
 # CHECK-NEXT:  1      3     0.50                        umulh	x23, x22, xzr
 # CHECK-NEXT:  1      2     0.50                        mul	x19, x20, xzr
@@ -1862,24 +1959,26 @@ drps
 # CHECK-NEXT:  1      1     0.25                        ror	x19, x23, #24
 # CHECK-NEXT:  1      1     0.25                        ror	x29, xzr, #63
 # CHECK-NEXT:  1      1     0.25                        ror	w9, w13, #31
+# CHECK-NEXT:  1      2     1.00                        fcmp	h5, h21
+# CHECK-NEXT:  1      2     1.00                        fcmp	h5, #0.0
+# CHECK-NEXT:  1      2     1.00                        fcmpe	h22, h21
+# CHECK-NEXT:  1      2     1.00                        fcmpe	h13, #0.0
 # CHECK-NEXT:  1      2     1.00                        fcmp	s3, s5
 # CHECK-NEXT:  1      2     1.00                        fcmp	s31, #0.0
-# CHECK-NEXT:  1      2     1.00                        fcmp	s31, #0.0
 # CHECK-NEXT:  1      2     1.00                        fcmpe	s29, s30
 # CHECK-NEXT:  1      2     1.00                        fcmpe	s15, #0.0
-# CHECK-NEXT:  1      2     1.00                        fcmpe	s15, #0.0
 # CHECK-NEXT:  1      2     1.00                        fcmp	d4, d12
 # CHECK-NEXT:  1      2     1.00                        fcmp	d23, #0.0
-# CHECK-NEXT:  1      2     1.00                        fcmp	d23, #0.0
 # CHECK-NEXT:  1      2     1.00                        fcmpe	d26, d22
 # CHECK-NEXT:  1      2     1.00                        fcmpe	d29, #0.0
-# CHECK-NEXT:  1      2     1.00                        fcmpe	d29, #0.0
 # CHECK-NEXT:  1      2     1.00                        fccmp	s1, s31, #0, eq
 # CHECK-NEXT:  1      2     1.00                        fccmp	s3, s0, #15, hs
 # CHECK-NEXT:  1      2     1.00                        fccmp	s31, s15, #13, hs
 # CHECK-NEXT:  1      2     1.00                        fccmp	d9, d31, #0, le
 # CHECK-NEXT:  1      2     1.00                        fccmp	d3, d0, #15, gt
 # CHECK-NEXT:  1      2     1.00                        fccmp	d31, d5, #7, ne
+# CHECK-NEXT:  1      2     1.00                        fccmp	h31, h3, #11, hs
+# CHECK-NEXT:  1      2     1.00                        fccmpe	h6, h1, #12, ne
 # CHECK-NEXT:  1      2     1.00                        fccmpe	s1, s31, #0, eq
 # CHECK-NEXT:  1      2     1.00                        fccmpe	s3, s0, #15, hs
 # CHECK-NEXT:  1      2     1.00                        fccmpe	s31, s15, #13, hs
@@ -1888,22 +1987,33 @@ drps
 # CHECK-NEXT:  1      2     1.00                        fccmpe	d31, d5, #7, ne
 # CHECK-NEXT:  1      2     0.50                        fcsel	s3, s20, s9, pl
 # CHECK-NEXT:  1      2     0.50                        fcsel	d9, d10, d11, mi
+# CHECK-NEXT:  1      2     0.50                        fcsel	h26, h2, h11, hs
+# CHECK-NEXT:  1      2     0.25                        fmov	h18, h28
 # CHECK-NEXT:  1      2     0.25                        fmov	s0, s1
 # CHECK-NEXT:  1      2     0.25                        fabs	s2, s3
+# CHECK-NEXT:  1      2     0.25                        fneg	h2, h9
 # CHECK-NEXT:  1      2     0.25                        fneg	s4, s5
 # CHECK-NEXT:  1      10    3.50                        fsqrt	s6, s7
 # CHECK-NEXT:  1      3     0.50                        fcvt	d8, s9
 # CHECK-NEXT:  1      3     0.50                        fcvt	h10, s11
+# CHECK-NEXT:  1      3     0.50                        frintn	h12, h3
 # CHECK-NEXT:  1      3     0.50                        frintn	s12, s13
+# CHECK-NEXT:  1      3     0.50                        frintp	h17, h31
 # CHECK-NEXT:  1      3     0.50                        frintp	s14, s15
+# CHECK-NEXT:  1      3     0.50                        frintm	h0, h21
 # CHECK-NEXT:  1      3     0.50                        frintm	s16, s17
+# CHECK-NEXT:  1      3     0.50                        frintz	h10, h29
 # CHECK-NEXT:  1      3     0.50                        frintz	s18, s19
+# CHECK-NEXT:  1      3     0.50                        frinta	h22, h10
 # CHECK-NEXT:  1      3     0.50                        frinta	s20, s21
+# CHECK-NEXT:  1      3     0.50                        frintx	h4, h5
 # CHECK-NEXT:  1      3     0.50                        frintx	s22, s23
 # CHECK-NEXT:  1      3     0.50                        frinti	s24, s25
+# CHECK-NEXT:  1      3     0.50                        frinti	h31, h14
 # CHECK-NEXT:  1      2     0.25                        fmov	d0, d1
 # CHECK-NEXT:  1      2     0.25                        fabs	d2, d3
 # CHECK-NEXT:  1      2     0.25                        fneg	d4, d5
+# CHECK-NEXT:  1      7     3.50                        fsqrt	h13, h24
 # CHECK-NEXT:  1      16    3.50                        fsqrt	d6, d7
 # CHECK-NEXT:  1      3     0.50                        fcvt	s8, d9
 # CHECK-NEXT:  1      3     0.50                        fcvt	h10, d11
@@ -1917,13 +2027,19 @@ drps
 # CHECK-NEXT:  1      3     0.50                        fcvt	s26, h27
 # CHECK-NEXT:  1      3     0.50                        fcvt	d28, h29
 # CHECK-NEXT:  1      3     0.25                        fmul	s20, s19, s17
+# CHECK-NEXT:  1      7     3.50                        fdiv	h1, h26, h23
 # CHECK-NEXT:  1      10    3.50                        fdiv	s1, s2, s3
+# CHECK-NEXT:  1      2     0.25                        fadd	h23, h27, h22
 # CHECK-NEXT:  1      2     0.25                        fadd	s4, s5, s6
+# CHECK-NEXT:  1      2     0.25                        fsub	h20, h11, h18
 # CHECK-NEXT:  1      2     0.25                        fsub	s7, s8, s9
 # CHECK-NEXT:  1      2     0.25                        fmax	s10, s11, s12
+# CHECK-NEXT:  1      2     0.25                        fmax	h8, h7, h11
 # CHECK-NEXT:  1      2     0.25                        fmin	s13, s14, s15
+# CHECK-NEXT:  1      2     0.25                        fmaxnm	h29, h13, h14
 # CHECK-NEXT:  1      2     0.25                        fmaxnm	s16, s17, s18
 # CHECK-NEXT:  1      2     0.25                        fminnm	s19, s20, s21
+# CHECK-NEXT:  1      3     0.25                        fnmul	h3, h15, h7
 # CHECK-NEXT:  1      3     0.25                        fnmul	s22, s23, s2
 # CHECK-NEXT:  1      3     0.25                        fmul	d20, d19, d17
 # CHECK-NEXT:  1      15    3.50                        fdiv	d1, d2, d3
@@ -1931,17 +2047,23 @@ drps
 # CHECK-NEXT:  1      2     0.25                        fsub	d7, d8, d9
 # CHECK-NEXT:  1      2     0.25                        fmax	d10, d11, d12
 # CHECK-NEXT:  1      2     0.25                        fmin	d13, d14, d15
+# CHECK-NEXT:  1      2     0.25                        fmin	h4, h13, h17
 # CHECK-NEXT:  1      2     0.25                        fmaxnm	d16, d17, d18
 # CHECK-NEXT:  1      2     0.25                        fminnm	d19, d20, d21
+# CHECK-NEXT:  1      2     0.25                        fminnm	h29, h23, h17
 # CHECK-NEXT:  1      3     0.25                        fnmul	d22, d23, d24
+# CHECK-NEXT:  1      4     0.25                        fmadd	h27, h0, h6, h28
 # CHECK-NEXT:  1      4     0.25                        fmadd	s3, s5, s6, s31
 # CHECK-NEXT:  1      4     0.25                        fmadd	d3, d13, d0, d23
+# CHECK-NEXT:  1      4     0.25                        fmsub	h25, h28, h12, h24
 # CHECK-NEXT:  1      4     0.25                        fmsub	s3, s5, s6, s31
 # CHECK-NEXT:  1      4     0.25                        fmsub	d3, d13, d0, d23
+# CHECK-NEXT:  1      4     0.25                        fnmadd	h3, h18, h31, h24
 # CHECK-NEXT:  1      4     0.25                        fnmadd	s3, s5, s6, s31
 # CHECK-NEXT:  1      4     0.25                        fnmadd	d3, d13, d0, d23
 # CHECK-NEXT:  1      4     0.25                        fnmsub	s3, s5, s6, s31
 # CHECK-NEXT:  1      4     0.25                        fnmsub	d3, d13, d0, d23
+# CHECK-NEXT:  1      4     0.25                        fnmsub	h3, h29, h24, h17
 # CHECK-NEXT:  1      3     0.50                        fcvtzs	w3, h5, #1
 # CHECK-NEXT:  1      3     0.50                        fcvtzs	wzr, h20, #13
 # CHECK-NEXT:  1      3     0.50                        fcvtzs	w19, h0, #32
@@ -2086,12 +2208,17 @@ drps
 # CHECK-NEXT:  1      3     1.00                        fcvtas	x27, d28
 # CHECK-NEXT:  1      3     1.00                        fcvtau	w29, d30
 # CHECK-NEXT:  1      3     1.00                        fcvtau	xzr, d0
+# CHECK-NEXT:  1      3     1.00                        fmov	h6, w5
+# CHECK-NEXT:  1      3     1.00                        fmov	h16, x27
+# CHECK-NEXT:  1      2     1.00                        fmov	w15, h31
 # CHECK-NEXT:  1      2     1.00                        fmov	w3, s9
 # CHECK-NEXT:  1      3     1.00                        fmov	s9, w3
+# CHECK-NEXT:  1      2     1.00                        fmov	x21, h14
 # CHECK-NEXT:  1      2     1.00                        fmov	x20, d31
 # CHECK-NEXT:  1      3     1.00                        fmov	d1, x15
 # CHECK-NEXT:  1      2     1.00                        fmov	x3, v12.d[1]
 # CHECK-NEXT:  2      5     1.00                        fmov	v1.d[1], x19
+# CHECK-NEXT:  1      2     0.25                        fmov	h29, #0.50000000
 # CHECK-NEXT:  1      2     0.25                        fmov	s2, #0.12500000
 # CHECK-NEXT:  1      2     0.25                        fmov	s3, #1.00000000
 # CHECK-NEXT:  1      2     0.25                        fmov	d30, #16.00000000
@@ -2115,13 +2242,10 @@ drps
 # CHECK-NEXT:  1      4     0.33    *      *      U     ldxrh	w17, [x4]
 # CHECK-NEXT:  1      4     0.33    *      *      U     ldxr	w22, [sp]
 # CHECK-NEXT:  1      4     0.33    *      *      U     ldxr	x11, [x29]
-# CHECK-NEXT:  1      4     0.33    *      *      U     ldxr	x11, [x29]
-# CHECK-NEXT:  1      4     0.33    *      *      U     ldxr	x11, [x29]
 # CHECK-NEXT:  3      5     0.67    *      *      U     stxp	w12, w11, w10, [sp]
 # CHECK-NEXT:  3      5     0.67    *      *      U     stxp	wzr, x27, x9, [x12]
 # CHECK-NEXT:  2      4     0.67    *      *      U     ldxp	w0, wzr, [sp]
 # CHECK-NEXT:  2      4     0.67    *      *      U     ldxp	x17, x0, [x18]
-# CHECK-NEXT:  2      4     0.67    *      *      U     ldxp	x17, x0, [x18]
 # CHECK-NEXT:  3      5     0.67    *      *      U     stlxrb	w12, w22, [x0]
 # CHECK-NEXT:  3      5     0.67    *      *      U     stlxrh	w10, w1, [x1]
 # CHECK-NEXT:  3      5     0.67    *      *      U     stlxr	w9, w2, [x2]
@@ -2130,25 +2254,32 @@ drps
 # CHECK-NEXT:  1      4     0.33    *      *      U     ldaxrh	w7, [x5]
 # CHECK-NEXT:  1      4     0.33    *      *      U     ldaxr	w6, [sp]
 # CHECK-NEXT:  1      4     0.33    *      *      U     ldaxr	x5, [x6]
-# CHECK-NEXT:  1      4     0.33    *      *      U     ldaxr	x5, [x6]
-# CHECK-NEXT:  1      4     0.33    *      *      U     ldaxr	x5, [x6]
 # CHECK-NEXT:  3      5     0.67    *      *      U     stlxp	w4, w5, w6, [sp]
 # CHECK-NEXT:  3      5     0.67    *      *      U     stlxp	wzr, x6, x7, [x1]
 # CHECK-NEXT:  2      4     0.67    *      *      U     ldaxp	w5, w18, [sp]
 # CHECK-NEXT:  2      4     0.67    *      *      U     ldaxp	x6, x19, [x22]
-# CHECK-NEXT:  2      4     0.67    *      *      U     ldaxp	x6, x19, [x22]
+# CHECK-NEXT:  2      1     0.50    *                   ldapur	w7, [x24]
+# CHECK-NEXT:  2      1     0.50    *                   ldapur	x20, [x13]
+# CHECK-NEXT:  2      1     0.50    *                   ldapurb	w13, [x17]
+# CHECK-NEXT:  2      1     0.50    *                   ldapurh	w3, [x22]
+# CHECK-NEXT:  2      1     0.50                  U     ldapursb	w7, [x8]
+# CHECK-NEXT:  2      1     0.50                  U     ldapursb	x29, [x7]
+# CHECK-NEXT:  2      1     0.50                  U     ldapursh	w17, [x19]
+# CHECK-NEXT:  2      1     0.50                  U     ldapursh	x3, [x3]
+# CHECK-NEXT:  2      1     0.50                  U     ldapursw	x3, [x18]
 # CHECK-NEXT:  2      1     0.50           *      U     stlrb	w24, [sp]
 # CHECK-NEXT:  2      1     0.50           *      U     stlrh	w25, [x30]
 # CHECK-NEXT:  2      1     0.50           *      U     stlr	w26, [x29]
 # CHECK-NEXT:  2      1     0.50           *      U     stlr	x27, [x28]
-# CHECK-NEXT:  2      1     0.50           *      U     stlr	x27, [x28]
-# CHECK-NEXT:  2      1     0.50           *      U     stlr	x27, [x28]
+# CHECK-NEXT:  2      1     0.50           *            stlur	w3, [x27]
+# CHECK-NEXT:  2      1     0.50           *            stlur	x23, [x25]
+# CHECK-NEXT:  2      1     0.50           *            stlurb	w30, [x17]
+# CHECK-NEXT:  2      1     0.50           *            stlurh	w9, [x29]
+# CHECK-NEXT:  1      4     0.33    *             U     ldarb	w16, [x21]
 # CHECK-NEXT:  1      4     0.33    *             U     ldarb	w23, [sp]
 # CHECK-NEXT:  1      4     0.33    *             U     ldarh	w22, [x30]
 # CHECK-NEXT:  1      4     0.33    *             U     ldar	wzr, [x29]
 # CHECK-NEXT:  1      4     0.33    *             U     ldar	x21, [x28]
-# CHECK-NEXT:  1      4     0.33    *             U     ldar	x21, [x28]
-# CHECK-NEXT:  1      4     0.33    *             U     ldar	x21, [x28]
 # CHECK-NEXT:  2      1     0.50           *            sturb	w9, [sp]
 # CHECK-NEXT:  2      1     0.50           *            sturh	wzr, [x12, #255]
 # CHECK-NEXT:  2      1     0.50           *            stur	w16, [x0, #-256]
@@ -2334,16 +2465,23 @@ drps
 # CHECK-NEXT:  1      4     0.33    *                   ldr	w13, [x2, #4]
 # CHECK-NEXT:  1      4     0.33    *                   ldrsw	x2, [x5, #4]
 # CHECK-NEXT:  1      4     0.33    *                   ldrsw	x23, [sp, #16380]
+# CHECK-NEXT:  1      4     0.33    *                   ldrsw	x21, [x25, x7]
 # CHECK-NEXT:  1      4     0.33    *                   ldrh	w2, [x4]
 # CHECK-NEXT:  1      4     0.33    *                   ldrsh	w23, [x6, #8190]
 # CHECK-NEXT:  1      4     0.33    *                   ldrsh	wzr, [sp, #2]
 # CHECK-NEXT:  1      4     0.33    *                   ldrsh	x29, [x2, #2]
+# CHECK-NEXT:  1      4     0.33    *                   ldrsh	x25, [x8, w13, uxtw]
 # CHECK-NEXT:  1      4     0.33    *                   ldrb	w26, [x3, #121]
 # CHECK-NEXT:  1      4     0.33    *                   ldrb	w12, [x2]
 # CHECK-NEXT:  1      4     0.33    *                   ldrsb	w27, [sp, #4095]
 # CHECK-NEXT:  1      4     0.33    *                   ldrsb	xzr, [x15]
+# CHECK-NEXT:  1      4     0.33    *                   ldrsb	x12, [x28, x27]
 # CHECK-NEXT:  2      1     0.50           *            str	x30, [sp]
 # CHECK-NEXT:  2      1     0.50           *            str	w20, [x4, #16380]
+# CHECK-NEXT:  2      2     0.50           *            str	b5, [x11]
+# CHECK-NEXT:  2      2     0.50           *            str	h23, [x15]
+# CHECK-NEXT:  2      2     0.50           *            str	s25, [x19]
+# CHECK-NEXT:  2      2     0.50           *            str	d15, [x2]
 # CHECK-NEXT:  2      1     0.50           *            strh	w17, [sp, #8190]
 # CHECK-NEXT:  2      1     0.50           *            strb	w23, [x3, #4095]
 # CHECK-NEXT:  2      1     0.50           *            strb	wzr, [x2]
@@ -2352,11 +2490,13 @@ drps
 # CHECK-NEXT:  1      6     0.33    *                   ldr	s10, [x19, #16380]
 # CHECK-NEXT:  1      6     0.33    *                   ldr	d3, [x10, #32760]
 # CHECK-NEXT:  2      2     0.50           *            str	q12, [sp, #65520]
+# CHECK-NEXT:  1      6     0.33    *                   ldr	q14, [x6, #4624]
 # CHECK-NEXT:  1      4     0.33    *                   ldrb	w3, [sp, x5]
 # CHECK-NEXT:  1      4     0.33    *                   ldrb	w9, [x27, x6]
 # CHECK-NEXT:  1      4     0.33    *                   ldrsb	w10, [x30, x7]
 # CHECK-NEXT:  1      4     0.33    *                   ldrb	w11, [x29, x3, sxtx]
 # CHECK-NEXT:  2      1     0.50           *            strb	w12, [x28, xzr, sxtx]
+# CHECK-NEXT:  2      1     0.50           *            strb	w5, [x26, w7, uxtw]
 # CHECK-NEXT:  1      4     0.33    *                   ldrb	w14, [x26, w6, uxtw]
 # CHECK-NEXT:  1      4     0.33    *                   ldrsb	w15, [x25, w7, uxtw]
 # CHECK-NEXT:  1      4     0.33    *                   ldrb	w17, [x23, w9, sxtw]
@@ -2373,6 +2513,18 @@ drps
 # CHECK-NEXT:  1      4     0.33    *                   ldrh	w17, [x23, w9, sxtw]
 # CHECK-NEXT:  1      4     0.33    *                   ldrh	w18, [x22, w10, sxtw]
 # CHECK-NEXT:  2      1     0.50           *            strh	w19, [x21, wzr, sxtw #1]
+# CHECK-NEXT:  1      6     0.33    *                   ldr	b25, [x21, w8, uxtw]
+# CHECK-NEXT:  1      6     0.33    *                   ldr	b8, [x30, x10]
+# CHECK-NEXT:  2      2     0.50           *            str	b14, [x13, x25]
+# CHECK-NEXT:  2      2     0.50           *            str	b30, [x16, w26, uxtw]
+# CHECK-NEXT:  2      7     0.33    *                   ldr	h6, [x4, w4, uxtw]
+# CHECK-NEXT:  2      7     0.33    *                   ldr	h11, [x13, x9]
+# CHECK-NEXT:  3      2     0.50           *            str	h16, [x5, x24]
+# CHECK-NEXT:  3      2     0.50           *            str	h15, [x15, w15, uxtw]
+# CHECK-NEXT:  1      6     0.33    *                   ldr	s12, [x30, w5, uxtw]
+# CHECK-NEXT:  1      6     0.33    *                   ldr	d24, [x26, w7, uxtw]
+# CHECK-NEXT:  2      2     0.50           *            str	s20, [x24, w10, uxtw]
+# CHECK-NEXT:  2      2     0.50           *            str	d5, [x26, x6]
 # CHECK-NEXT:  1      4     0.33    *                   ldr	w3, [sp, x5]
 # CHECK-NEXT:  1      6     0.33    *                   ldr	s9, [x27, x6]
 # CHECK-NEXT:  1      4     0.33    *                   ldr	w10, [x30, x7, lsl #2]
@@ -2393,6 +2545,7 @@ drps
 # CHECK-NEXT:  1      4     0.33    *                   ldr	x13, [x27, x5, sxtx #3]
 # CHECK-NEXT:  1      4     0.33                  U     prfm	pldl1keep, [x26, w6, uxtw]
 # CHECK-NEXT:  1      4     0.33    *                   ldr	x15, [x25, w7, uxtw]
+# CHECK-NEXT:  2      1     0.50           *            str	x27, [x26, w24, uxtw]
 # CHECK-NEXT:  1      4     0.33    *                   ldr	x16, [x24, w8, uxtw #3]
 # CHECK-NEXT:  1      4     0.33    *                   ldr	x17, [x23, w9, sxtw]
 # CHECK-NEXT:  1      4     0.33    *                   ldr	x18, [x22, w10, sxtw]
@@ -2424,6 +2577,8 @@ drps
 # CHECK-NEXT:  1      6     0.33    *                   ldp	s1, s2, [x3, #44]
 # CHECK-NEXT:  2      2     0.50           *            stp	d3, d5, [x9, #504]
 # CHECK-NEXT:  2      2     0.50           *            stp	d7, d11, [x10, #-512]
+# CHECK-NEXT:  2      1     0.50           *            stnp	x20, x16, [x8]
+# CHECK-NEXT:  2      1     0.50           *            stp	x3, x6, [x16]
 # CHECK-NEXT:  1      6     0.33    *                   ldp	d2, d3, [x30, #-8]
 # CHECK-NEXT:  2      2     0.50           *            stp	q3, q5, [sp]
 # CHECK-NEXT:  2      2     0.50           *            stp	q17, q19, [sp, #1008]
@@ -2482,6 +2637,12 @@ drps
 # CHECK-NEXT:  2      2     0.50           *            stnp	q3, q5, [sp]
 # CHECK-NEXT:  2      2     0.50           *            stnp	q17, q19, [sp, #1008]
 # CHECK-NEXT:  2      6     0.67    *                   ldnp	q23, q29, [x1, #-1024]
+# CHECK-NEXT:  1      1     0.25                        and	wsp, w16, #0xe00
+# CHECK-NEXT:  1      1     0.25                        and	x2, x22, #0x1e00
+# CHECK-NEXT:  1      1     0.33                        ands	w14, w8, #0x70
+# CHECK-NEXT:  1      1     0.33                        ands	x4, x10, #0x60
+# CHECK-NEXT:  1      1     0.25                        eor	wsp, w4, #0xe00
+# CHECK-NEXT:  1      1     0.25                        eor	x27, x25, #0x1e00
 # CHECK-NEXT:  1      1     0.25                        mov	w3, #983055
 # CHECK-NEXT:  1      1     0.25                        mov	x10, #-6148914691236517206
 # CHECK-NEXT:  1      1     0.25                        and	w12, w23, w21
@@ -2498,6 +2659,10 @@ drps
 # CHECK-NEXT:  1      1     0.25                        and	x7, x20, xzr, asr #63
 # CHECK-NEXT:  1      1     0.25                        bic	x13, x20, x14, lsl #47
 # CHECK-NEXT:  1      1     0.25                        bic	w2, w7, w9
+# CHECK-NEXT:  1      1     0.25                        eon	w29, w4, w19
+# CHECK-NEXT:  1      1     0.25                        eon	x19, x12, x2
+# CHECK-NEXT:  1      1     0.25                        eor	w8, w27, w2
+# CHECK-NEXT:  1      1     0.25                        eor	x22, x16, x6
 # CHECK-NEXT:  1      1     0.25                        orr	w2, w7, w0, asr #31
 # CHECK-NEXT:  1      1     0.25                        orr	x8, x9, x10, lsl #12
 # CHECK-NEXT:  1      1     0.25                        orn	x3, x5, x7, asr #2
@@ -2523,16 +2688,20 @@ drps
 # CHECK-NEXT:  1      1     0.25                        movk	x7, #0, lsl #32
 # CHECK-NEXT:  1      1     0.25                        movz	x8, #0, lsl #48
 # CHECK-NEXT:  1      1     0.25                        movk	x9, #0, lsl #48
+# CHECK-NEXT:  1      1     0.07                  U     msr	DAIFSet, #0
 # CHECK-NEXT:  1      1     0.25                        adr	x2, #1600
 # CHECK-NEXT:  1      1     0.25                        adrp	x21, #6553600
 # CHECK-NEXT:  1      1     0.25                        adr	x0, #262144
 # CHECK-NEXT:  1      1     0.50                        tbz	x12, #62, #0
 # CHECK-NEXT:  1      1     0.50                        tbz	x12, #62, #4
 # CHECK-NEXT:  1      1     0.50                        tbz	x12, #62, #-32768
+# CHECK-NEXT:  1      1     0.50                        tbz	w17, #16, test
 # CHECK-NEXT:  1      1     0.50                        tbnz	x12, #60, #32764
+# CHECK-NEXT:  1      1     0.50                        tbnz	w3, #28, test
 # CHECK-NEXT:  1      1     0.50                        b	#4
 # CHECK-NEXT:  1      1     0.50                        b	#-4
 # CHECK-NEXT:  1      1     0.50                        b	#134217724
+# CHECK-NEXT:  2      1     0.50                        bl	test
 # CHECK-NEXT:  1      1     0.50                        br	x20
 # CHECK-NEXT:  2      1     0.50                        blr	xzr
 # CHECK-NEXT:  1      1     0.50                  U     ret	x10
@@ -2562,7 +2731,7 @@ drps
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
-# CHECK-NEXT: 11.00  11.00  33.00  33.00  45.33  45.33  45.33  96.33  162.33 162.33 306.50 205.50 139.00 139.00 167.50 44.50  51.50  9.50
+# CHECK-NEXT: 13.00  13.00  40.50  40.50  48.00  48.00  48.00  96.67  175.17 175.17 322.50 209.50 142.00 142.00 189.00 55.50  65.50  13.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
@@ -2585,16 +2754,16 @@ drps
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	w2, #4095
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adds	w20, wsp, #0
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	x3, #1, lsl #12
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	wsp, #2342
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	sp, #20, lsl #12
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	x30, #4095
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     subs	x4, sp, #3822
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	w3, #291, lsl #12
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	wsp, #1365
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	sp, #1092, lsl #12
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	sp, x30
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	wsp, w20
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	x11, sp
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	w24, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	x10, #-63432
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	wsp, wsp, w10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	x25, x9, w25, uxtb
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w3, w5, w7
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	wzr, w3, w5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     add	w20, wzr, w4
@@ -2622,6 +2791,8 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	x5, x6, x7, asr #21
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     add	x8, x9, x10, asr #63
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adds	w3, w5, w7
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     adds	w17, wsp, w25
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adds	x13, x23, w8, uxtb
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmn	w3, w5
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adds	w20, wzr, w4
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adds	w4, w6, wzr
@@ -2671,6 +2842,9 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	x2, x3, x4, asr #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	x5, x6, x7, asr #21
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	x8, x9, x10, asr #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	w13, wsp, w10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     sub	x16, x2, w19, uxtb
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     subs	x13, x15, x14, sxtx #1
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     subs	w3, w5, w7
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	w3, w5
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     subs	w4, w6, wzr
@@ -2729,6 +2903,8 @@ drps
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	w18, w19, asr #0
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	w20, w21, asr #22
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	w22, w23, asr #31
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	wsp, w26
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	x16, w27, uxtb
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	x0, x3
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	xzr, x4
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	x5, xzr
@@ -2743,6 +2919,10 @@ drps
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50    -      -      -      -      -      -     cmp	x22, x23, asr #63
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	wzr, w0
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cmp	xzr, x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	sp, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	wsp, w20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	x11, sp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	w24, wsp
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	w29, w27, w25
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	wzr, w3, w4
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adc	w9, wzr, w10
@@ -2815,15 +2995,11 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w9, w10, #31
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	x20, x21, #63
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	wzr, wzr, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w3, w2, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	w9, w10, #31
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	x20, x21, #63
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	w1, wzr, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfx	w9, w10, #0, #1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfiz	x2, x3, #63, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	x19, x20, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfiz	x9, x10, #5, #59
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w9, w10, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfiz	w11, w12, #31, #1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfiz	w13, w14, #29, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfiz	xzr, xzr, #10, #11
@@ -2835,11 +3011,8 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w11, w12, #31
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	w13, w14, #29
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     sbfx	xzr, xzr, #10, #11
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	w9, w10, #0, #1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfi	x2, x3, #63, #1
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	x19, x20, #0, #64
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfi	x9, x10, #5, #59
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	w9, w10, #0, #32
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfi	w11, w12, #31, #1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfi	w13, w14, #29, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfc	xzr, #10, #11
@@ -2851,11 +3024,8 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	w11, w12, #31, #1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	w13, w14, #29, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     bfxil	xzr, xzr, #10, #11
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ubfx	w9, w10, #0, #1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	x2, x3, #63
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	x19, x20, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	x9, x10, #5
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsr	w9, w10, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	w11, w12, #31
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     lsl	w13, w14, #29
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ubfiz	xzr, xzr, #10, #11
@@ -2873,6 +3043,7 @@ drps
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbnz	x26, #1048572
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbz	wzr, #0
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbnz	xzr, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     cbnz	w21, test
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b.ne	#4
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b.ge	#1048572
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b.ge	#-4
@@ -2961,7 +3132,6 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cneg	xzr, x4, le
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     cneg	x9, xzr, lt
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csneg	x4, x8, x8, al
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     csinv	w9, w8, w8, nv
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     rbit	w0, w7
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     rbit	x18, x3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     rev16	w17, w1
@@ -2993,6 +3163,16 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     asr	x21, x22, x23
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	w24, w25, w26
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	x27, x28, x29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     crc32cb	w30, w23, w15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     crc32cb	wzr, w12, w14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     crc32cb	w28, w10, w11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     crc32b	w27, w12, w15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     crc32h	w3, w15, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     crc32w	w9, w18, w24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     crc32x	w19, w6, x25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     crc32ch	w25, w26, w16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     crc32cw	w27, w12, w23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     crc32cx	w21, w28, x5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smulh	x30, x29, x28
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smulh	xzr, x27, x26
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     umulh	x30, x29, x28
@@ -3035,7 +3215,6 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     umsubl	x3, w5, w2, x9
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     umsubl	x16, w17, wzr, x18
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     umnegl	x19, w20, w21
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smulh	x30, x29, x28
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     smulh	x23, x22, xzr
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     umulh	x23, x22, xzr
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -     mul	x19, x20, xzr
@@ -3051,24 +3230,26 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	x19, x23, #24
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	x29, xzr, #63
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ror	w9, w13, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	h5, h21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	h5, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	h22, h21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	h13, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	s3, s5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	s31, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	s31, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	s29, s30
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	s15, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	s15, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	d4, d12
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	d23, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmp	d23, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	d26, d22
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	d29, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcmpe	d29, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	s1, s31, #0, eq
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	s3, s0, #15, hs
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	s31, s15, #13, hs
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	d9, d31, #0, le
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	d3, d0, #15, gt
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	d31, d5, #7, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmp	h31, h3, #11, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	h6, h1, #12, ne
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	s1, s31, #0, eq
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	s3, s0, #15, hs
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	s31, s15, #13, hs
@@ -3077,22 +3258,33 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fccmpe	d31, d5, #7, ne
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcsel	s3, s20, s9, pl
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcsel	d9, d10, d11, mi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcsel	h26, h2, h11, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	h18, h28
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s0, s1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	s2, s3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	h2, h9
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	s4, s5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	s6, s7
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	d8, s9
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	h10, s11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	h12, h3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintn	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	h17, h31
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintp	s14, s15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	h0, h21
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintm	s16, s17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	h10, h29
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintz	s18, s19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	h22, h10
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	s20, s21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	h4, h5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frintx	s22, s23
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	s24, s25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinti	h31, h14
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	d0, d1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	d2, d3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	d4, d5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	h13, h24
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	d6, d7
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	s8, d9
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	h10, d11
@@ -3106,13 +3298,19 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	s26, h27
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvt	d28, h29
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	s20, s19, s17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	h1, h26, h23
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	s1, s2, s3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	h23, h27, h22
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	s4, s5, s6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	h20, h11, h18
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	s7, s8, s9
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	h8, h7, h11
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	s13, s14, s15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	h29, h13, h14
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	s16, s17, s18
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	s19, s20, s21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmul	h3, h15, h7
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmul	s22, s23, s2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	d20, d19, d17
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	d1, d2, d3
@@ -3120,17 +3318,23 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	d7, d8, d9
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmax	d10, d11, d12
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	d13, d14, d15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmin	h4, h13, h17
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	d16, d17, d18
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	d19, d20, d21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	h29, h23, h17
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmul	d22, d23, d24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmadd	h27, h0, h6, h28
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmadd	s3, s5, s6, s31
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmadd	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmsub	h25, h28, h12, h24
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmsub	s3, s5, s6, s31
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmsub	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmadd	h3, h18, h31, h24
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmadd	s3, s5, s6, s31
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmadd	d3, d13, d0, d23
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmsub	s3, s5, s6, s31
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmsub	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fnmsub	h3, h29, h24, h17
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	w3, h5, #1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	wzr, h20, #13
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	w19, h0, #32
@@ -3275,12 +3479,17 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtas	x27, d28
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtau	w29, d30
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     fcvtau	xzr, d0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     fmov	h6, w5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     fmov	h16, x27
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     fmov	w15, h31
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     fmov	w3, s9
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     fmov	s9, w3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     fmov	x21, h14
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     fmov	x20, d31
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     fmov	d1, x15
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     fmov	x3, v12.d[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.25   0.25   0.25   0.25   fmov	v1.d[1], x19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	h29, #0.50000000
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s2, #0.12500000
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	s3, #1.00000000
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	d30, #16.00000000
@@ -3304,13 +3513,10 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldxrh	w17, [x4]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldxr	w22, [sp]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldxr	x11, [x29]
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldxr	x11, [x29]
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldxr	x11, [x29]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stxp	w12, w11, w10, [sp]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stxp	wzr, x27, x9, [x12]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldxp	w0, wzr, [sp]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldxp	x17, x0, [x18]
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldxp	x17, x0, [x18]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stlxrb	w12, w22, [x0]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stlxrh	w10, w1, [x1]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stlxr	w9, w2, [x2]
@@ -3319,25 +3525,32 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldaxrh	w7, [x5]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldaxr	w6, [sp]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldaxr	x5, [x6]
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldaxr	x5, [x6]
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldaxr	x5, [x6]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stlxp	w4, w5, w6, [sp]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.33   0.83   0.83    -      -      -      -      -      -      -      -     stlxp	wzr, x6, x7, [x1]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldaxp	w5, w18, [sp]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldaxp	x6, x19, [x22]
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldaxp	x6, x19, [x22]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldapur	w7, [x24]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldapur	x20, [x13]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldapurb	w13, [x17]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldapurh	w3, [x22]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldapursb	w7, [x8]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldapursb	x29, [x7]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldapursh	w17, [x19]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldapursh	x3, [x3]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ldapursw	x3, [x18]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stlrb	w24, [sp]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stlrh	w25, [x30]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stlr	w26, [x29]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stlr	x27, [x28]
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stlr	x27, [x28]
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stlr	x27, [x28]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stlur	w3, [x27]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stlur	x23, [x25]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stlurb	w30, [x17]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stlurh	w9, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldarb	w16, [x21]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldarb	w23, [sp]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldarh	w22, [x30]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldar	wzr, [x29]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldar	x21, [x28]
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldar	x21, [x28]
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldar	x21, [x28]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sturb	w9, [sp]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     sturh	wzr, [x12, #255]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stur	w16, [x0, #-256]
@@ -3523,16 +3736,23 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w13, [x2, #4]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsw	x2, [x5, #4]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsw	x23, [sp, #16380]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsw	x21, [x25, x7]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrh	w2, [x4]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsh	w23, [x6, #8190]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsh	wzr, [sp, #2]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsh	x29, [x2, #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsh	x25, [x8, w13, uxtw]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w26, [x3, #121]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w12, [x2]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsb	w27, [sp, #4095]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsb	xzr, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsb	x12, [x28, x27]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     str	x30, [sp]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     str	w20, [x4, #16380]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     str	b5, [x11]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     str	h23, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     str	s25, [x19]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     str	d15, [x2]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     strh	w17, [sp, #8190]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     strb	w23, [x3, #4095]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     strb	wzr, [x2]
@@ -3541,11 +3761,13 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	s10, [x19, #16380]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	d3, [x10, #32760]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     str	q12, [sp, #65520]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	q14, [x6, #4624]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w3, [sp, x5]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w9, [x27, x6]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsb	w10, [x30, x7]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w11, [x29, x3, sxtx]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     strb	w12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     strb	w5, [x26, w7, uxtw]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w14, [x26, w6, uxtw]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrsb	w15, [x25, w7, uxtw]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrb	w17, [x23, w9, sxtw]
@@ -3562,6 +3784,18 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrh	w17, [x23, w9, sxtw]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldrh	w18, [x22, w10, sxtw]
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     strh	w19, [x21, wzr, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	b25, [x21, w8, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	b8, [x30, x10]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     str	b14, [x13, x25]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     str	b30, [x16, w26, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	h6, [x4, w4, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ldr	h11, [x13, x9]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	h16, [x5, x24]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     str	h15, [x15, w15, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	s12, [x30, w5, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	d24, [x26, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     str	s20, [x24, w10, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     str	d5, [x26, x6]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w3, [sp, x5]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	s9, [x27, x6]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	w10, [x30, x7, lsl #2]
@@ -3582,6 +3816,7 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x13, [x27, x5, sxtx #3]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     prfm	pldl1keep, [x26, w6, uxtw]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     str	x27, [x26, w24, uxtw]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x16, [x24, w8, uxtw #3]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x17, [x23, w9, sxtw]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldr	x18, [x22, w10, sxtw]
@@ -3613,6 +3848,8 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldp	s1, s2, [x3, #44]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stp	d3, d5, [x9, #504]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stp	d7, d11, [x10, #-512]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stnp	x20, x16, [x8]
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     stp	x3, x6, [x16]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ldp	d2, d3, [x30, #-8]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stp	q3, q5, [sp]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stp	q17, q19, [sp, #1008]
@@ -3671,6 +3908,12 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stnp	q3, q5, [sp]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     stnp	q17, q19, [sp, #1008]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ldnp	q23, q29, [x1, #-1024]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	wsp, w16, #0xe00
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	x2, x22, #0x1e00
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ands	w14, w8, #0x70
+# CHECK-NEXT:  -      -      -      -     0.33   0.33   0.33    -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     ands	x4, x10, #0x60
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     eor	wsp, w4, #0xe00
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     eor	x27, x25, #0x1e00
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	w3, #983055
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     mov	x10, #-6148914691236517206
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	w12, w23, w21
@@ -3687,6 +3930,10 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     and	x7, x20, xzr, asr #63
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     bic	x13, x20, x14, lsl #47
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     bic	w2, w7, w9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     eon	w29, w4, w19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     eon	x19, x12, x2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     eor	w8, w27, w2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     eor	x22, x16, x6
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     orr	w2, w7, w0, asr #31
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     orr	x8, x9, x10, lsl #12
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     orn	x3, x5, x7, asr #2
@@ -3712,16 +3959,20 @@ drps
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movk	x7, #0, lsl #32
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movz	x8, #0, lsl #48
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     movk	x9, #0, lsl #48
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	DAIFSet, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adr	x2, #1600
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adrp	x21, #6553600
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -      -      -      -     adr	x0, #262144
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tbz	x12, #62, #0
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tbz	x12, #62, #4
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tbz	x12, #62, #-32768
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tbz	w17, #16, test
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tbnz	x12, #60, #32764
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tbnz	w3, #28, test
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b	#4
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b	#-4
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     b	#134217724
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     bl	test
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     br	x20
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     blr	xzr
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     ret	x10
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-forwarding.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-forwarding.s
index 4de37f9600052..a720a6bde305b 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-forwarding.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-forwarding.s
@@ -119,11 +119,18 @@ bfmlalb v0.4s, v1.8h, v2.8h
 bfmlalb v0.4s, v0.8h, v1.8h
 # LLVM-MCA-END
 
-# LLVM-MCA-BEGIN crc32cb
+# LLVM-MCA-BEGIN crc32
 mul    w0, w0, w0
 crc32cb w0, w0, w1
 crc32cb w0, w0, w1
 crc32cb w0, w0, w0
+crc32b w0, w0, w15
+crc32h w0, w0, w21
+crc32w w0, w0, w24
+crc32x w0, w0, x25
+crc32ch w0, w0, w16
+crc32cw w0, w0, w23
+crc32cx w0, w0, x5
 # LLVM-MCA-END
 
 # LLVM-MCA-BEGIN Z sdot.s
@@ -850,30 +857,44 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: 3.     2     16.5   0.0    0.0       bfmlalb	v0.4s, v0.8h, v1.8h
 # CHECK-NEXT:        2     11.8   0.1    0.0       <total>
 
-# CHECK:      [16] Code Region - crc32cb
+# CHECK:      [16] Code Region - crc32
 
 # CHECK:      Iterations:        100
-# CHECK-NEXT: Instructions:      400
-# CHECK-NEXT: Total Cycles:      703
-# CHECK-NEXT: Total uOps:        400
+# CHECK-NEXT: Instructions:      1100
+# CHECK-NEXT: Total Cycles:      1403
+# CHECK-NEXT: Total uOps:        1100
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.57
-# CHECK-NEXT: IPC:               0.57
-# CHECK-NEXT: Block RThroughput: 3.0
+# CHECK-NEXT: uOps Per Cycle:    0.78
+# CHECK-NEXT: IPC:               0.78
+# CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT:                     0123456789          0
+# CHECK-NEXT: Index     0123456789          0123456789
 
-# CHECK:      [0,0]     DeeER.    .    ..   mul	w0, w0, w0
-# CHECK-NEXT: [0,1]     D==eeER   .    ..   crc32cb	w0, w0, w1
-# CHECK-NEXT: [0,2]     D===eeER  .    ..   crc32cb	w0, w0, w1
-# CHECK-NEXT: [0,3]     D=====eeER.    ..   crc32cb	w0, w0, w0
-# CHECK-NEXT: [1,0]     D=======eeER   ..   mul	w0, w0, w0
-# CHECK-NEXT: [1,1]     D=========eeER ..   crc32cb	w0, w0, w1
-# CHECK-NEXT: [1,2]     D==========eeER..   crc32cb	w0, w0, w1
-# CHECK-NEXT: [1,3]     D============eeER   crc32cb	w0, w0, w0
+# CHECK:      [0,0]     DeeER.    .    .    .    .    .   mul	w0, w0, w0
+# CHECK-NEXT: [0,1]     D==eeER   .    .    .    .    .   crc32cb	w0, w0, w1
+# CHECK-NEXT: [0,2]     D===eeER  .    .    .    .    .   crc32cb	w0, w0, w1
+# CHECK-NEXT: [0,3]     D=====eeER.    .    .    .    .   crc32cb	w0, w0, w0
+# CHECK-NEXT: [0,4]     D======eeER    .    .    .    .   crc32b	w0, w0, w15
+# CHECK-NEXT: [0,5]     D=======eeER   .    .    .    .   crc32h	w0, w0, w21
+# CHECK-NEXT: [0,6]     D========eeER  .    .    .    .   crc32w	w0, w0, w24
+# CHECK-NEXT: [0,7]     D=========eeER .    .    .    .   crc32x	w0, w0, x25
+# CHECK-NEXT: [0,8]     D==========eeER.    .    .    .   crc32ch	w0, w0, w16
+# CHECK-NEXT: [0,9]     D===========eeER    .    .    .   crc32cw	w0, w0, w23
+# CHECK-NEXT: [0,10]    D============eeER   .    .    .   crc32cx	w0, w0, x5
+# CHECK-NEXT: [1,0]     D==============eeER .    .    .   mul	w0, w0, w0
+# CHECK-NEXT: [1,1]     D================eeER    .    .   crc32cb	w0, w0, w1
+# CHECK-NEXT: [1,2]     D=================eeER   .    .   crc32cb	w0, w0, w1
+# CHECK-NEXT: [1,3]     D===================eeER .    .   crc32cb	w0, w0, w0
+# CHECK-NEXT: [1,4]     .D===================eeER.    .   crc32b	w0, w0, w15
+# CHECK-NEXT: [1,5]     .D====================eeER    .   crc32h	w0, w0, w21
+# CHECK-NEXT: [1,6]     .D=====================eeER   .   crc32w	w0, w0, w24
+# CHECK-NEXT: [1,7]     .D======================eeER  .   crc32x	w0, w0, x25
+# CHECK-NEXT: [1,8]     .D=======================eeER .   crc32ch	w0, w0, w16
+# CHECK-NEXT: [1,9]     .D========================eeER.   crc32cw	w0, w0, w23
+# CHECK-NEXT: [1,10]    .D=========================eeER   crc32cx	w0, w0, x5
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -882,11 +903,18 @@ bfmlalb z0.s, z0.h, z1.h
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     2     4.5    0.5    0.0       mul	w0, w0, w0
-# CHECK-NEXT: 1.     2     6.5    0.0    0.0       crc32cb	w0, w0, w1
-# CHECK-NEXT: 2.     2     7.5    0.0    0.0       crc32cb	w0, w0, w1
-# CHECK-NEXT: 3.     2     9.5    0.0    0.0       crc32cb	w0, w0, w0
-# CHECK-NEXT:        2     7.0    0.1    0.0       <total>
+# CHECK-NEXT: 0.     2     8.0    0.5    0.0       mul	w0, w0, w0
+# CHECK-NEXT: 1.     2     10.0   0.0    0.0       crc32cb	w0, w0, w1
+# CHECK-NEXT: 2.     2     11.0   0.0    0.0       crc32cb	w0, w0, w1
+# CHECK-NEXT: 3.     2     13.0   0.0    0.0       crc32cb	w0, w0, w0
+# CHECK-NEXT: 4.     2     13.5   0.0    0.0       crc32b	w0, w0, w15
+# CHECK-NEXT: 5.     2     14.5   0.0    0.0       crc32h	w0, w0, w21
+# CHECK-NEXT: 6.     2     15.5   0.0    0.0       crc32w	w0, w0, w24
+# CHECK-NEXT: 7.     2     16.5   0.0    0.0       crc32x	w0, w0, x25
+# CHECK-NEXT: 8.     2     17.5   0.0    0.0       crc32ch	w0, w0, w16
+# CHECK-NEXT: 9.     2     18.5   0.0    0.0       crc32cw	w0, w0, w23
+# CHECK-NEXT: 10.    2     19.5   0.0    0.0       crc32cx	w0, w0, x5
+# CHECK-NEXT:        2     14.3   0.0    0.0       <total>
 
 # CHECK:      [17] Code Region - Z sdot.s
 
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-misc-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-misc-instructions.s
new file mode 100644
index 0000000000000..4f48de8b42926
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-misc-instructions.s
@@ -0,0 +1,89 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=aarch64 -mcpu=neoverse-v1 -instruction-tables < %s | FileCheck %s
+
+#----------------------------------------------------------------------------
+# Basic instructions without scheduling information in Neoverse V1 SOG
+#----------------------------------------------------------------------------
+
+at s12e1r, x28
+brk      #0x8415
+clrex
+csdb
+dcps1
+dcps2
+dcps3
+dmb sy
+hlt #0x7a67
+hvc #0xecb9
+isb
+pssbb
+smc #0x7e57
+svc #0x89cb
+sysl x16, #5, c11, c8, #5
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.07                  U     at	s12e1r, x28
+# CHECK-NEXT:  1      1     0.07                  U     brk	#0x8415
+# CHECK-NEXT:  1      1     0.07    *      *      U     clrex
+# CHECK-NEXT:  1      1     0.07    *      *      U     csdb
+# CHECK-NEXT:  1      1     0.07                  U     dcps1
+# CHECK-NEXT:  1      1     0.07                  U     dcps2
+# CHECK-NEXT:  1      1     0.07                  U     dcps3
+# CHECK-NEXT:  1      1     0.07    *      *      U     dmb	sy
+# CHECK-NEXT:  1      1     0.07                  U     hlt	#0x7a67
+# CHECK-NEXT:  1      1     0.07                  U     hvc	#0xecb9
+# CHECK-NEXT:  1      1     0.07    *      *      U     isb
+# CHECK-NEXT:  1      1     0.07    *      *      U     pssbb
+# CHECK-NEXT:  1      1     0.07                  U     smc	#0x7e57
+# CHECK-NEXT:  1      1     0.07                  U     svc	#0x89cb
+# CHECK-NEXT:  1      1     0.07                  U     sysl	x16, #5, c11, c8, #5
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - V1UnitB
+# CHECK-NEXT: [0.1] - V1UnitB
+# CHECK-NEXT: [1.0] - V1UnitD
+# CHECK-NEXT: [1.1] - V1UnitD
+# CHECK-NEXT: [2.0] - V1UnitFlg
+# CHECK-NEXT: [2.1] - V1UnitFlg
+# CHECK-NEXT: [2.2] - V1UnitFlg
+# CHECK-NEXT: [3]   - V1UnitL2
+# CHECK-NEXT: [4.0] - V1UnitL01
+# CHECK-NEXT: [4.1] - V1UnitL01
+# CHECK-NEXT: [5]   - V1UnitM0
+# CHECK-NEXT: [6]   - V1UnitM1
+# CHECK-NEXT: [7.0] - V1UnitS
+# CHECK-NEXT: [7.1] - V1UnitS
+# CHECK-NEXT: [8]   - V1UnitV0
+# CHECK-NEXT: [9]   - V1UnitV1
+# CHECK-NEXT: [10]  - V1UnitV2
+# CHECK-NEXT: [11]  - V1UnitV3
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     at	s12e1r, x28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     brk	#0x8415
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     clrex
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     csdb
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     dcps1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     dcps2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     dcps3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     dmb	sy
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     hlt	#0x7a67
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     hvc	#0xecb9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     isb
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     pssbb
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     smc	#0x7e57
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     svc	#0x89cb
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     sysl	x16, #5, c11, c8, #5
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-neon-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-neon-instructions.s
index 20e76ef27c470..36a03fa8fe501 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-neon-instructions.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-neon-instructions.s
@@ -17,8 +17,10 @@ addhn v0.8b, v0.8h, v0.8h
 addhn2 v0.16b, v0.8h, v0.8h
 addhn2 v0.4s, v0.2d, v0.2d
 addhn2 v0.8h, v0.4s, v0.4s
+addp v7.2s, v1.2s, v2.2s
 addp v0.2d, v0.2d, v0.2d
 addp v0.8b, v0.8b, v0.8b
+addp d1, v14.2d
 addv s0, v0.4s
 addv h0, v0.4h
 addv h0, v0.8h
@@ -32,6 +34,7 @@ and v0.8b, v0.8b, v0.8b
 bfcvt h0, s0
 bfcvtn v0.4h, v0.4s
 bfcvtn2 v0.8h, v0.4s
+bfdot v0.2s, v24.4h, v14.2h[2]
 bfdot v0.2s, v0.4h, v0.4h
 bfdot v0.4s, v0.8h, v0.8h
 bfmlalb v0.4s, v0.8h, v0.8h
@@ -40,10 +43,16 @@ bfmlalt v0.4s, v0.8h, v0.8h
 bfmlalt v0.4s, v0.8h, v0.h[3]
 bfmmla v0.4s, v0.8h, v0.8h
 bic v0.4h, #15, lsl #8
+bic v23.8h, #101
 bic v0.8b, v0.8b, v0.8b
+bic v25.16b, v10.16b, v9.16b
+bic v24.2s, #70
+bit v5.8b, v12.8b, v22.8b
+bif v0.8b, v25.8b, v4.8b
 bif v0.16b, v0.16b, v0.16b
 bit v0.16b, v0.16b, v0.16b
 bsl v0.8b, v0.8b, v0.8b
+bsl v27.16b, v13.16b, v21.16b
 cls v0.16b, v0.16b
 cls v0.2s, v0.2s
 cls v0.4h, v0.4h
@@ -56,28 +65,37 @@ clz v0.4h, v0.4h
 clz v0.4s, v0.4s
 clz v0.8b, v0.8b
 clz v0.8h, v0.8h
+cmeq v9.8h, v16.8h, v24.8h
+cmeq v14.4h, v18.4h, #0
 cmeq d20, d21, 0
 cmeq d20, d21, d22
 cmeq v0.16b, v0.16b, 0
 cmeq v0.16b, v0.16b, v0.16b
+cmge v22.8h, v16.8h, v3.8h
+cmge v22.16b, v30.16b, #0
 cmge d20, d21, 0
 cmge d20, d21, d22
 cmge v0.4h, v0.4h, v0.4h
 cmge v0.8b, v0.8b, 0
+cmgt v3.2d, v29.2d, v11.2d
 cmgt d20, d21, 0
 cmgt d20, d21, d22
 cmgt v0.2s, v0.2s, 0
 cmgt v0.4s, v0.4s, v0.4s
+cmhi v28.4h, v25.4h, v21.4h
 cmhi d20, d21, d22
 cmhi v0.8h, v0.8h, v0.8h
 cmhs d20, d21, d22
 cmhs v0.8b, v0.8b, v0.8b
+cmle v21.2s, v19.2s, #0
 cmle d20, d21, 0
 cmle v0.2d, v0.2d, 0
+cmlt v26.4h, v12.4h, #0
 cmlt d20, d21, 0
 cmlt v0.8h, v0.8h, 0
 cmtst d20, d21, d22
 cmtst v0.2s, v0.2s, v0.2s
+cmtst v13.2d, v13.2d, v13.2d
 cnt v0.16b, v0.16b
 cnt v0.8b, v0.8b
 dup v0.16b,w28
@@ -103,7 +121,10 @@ ext v0.16b, v0.16b, v0.16b, #3
 ext v0.8b, v0.8b, v0.8b, #3
 fabd d29, d24, d20
 fabd s29, s24, s20
+fabd h27, h20, h17
+fabd v13.8h, v28.8h, v12.8h
 fabd v0.4s, v0.4s, v0.4s
+fabs h25, h7
 fabs v0.2d, v0.2d
 fabs v0.2s, v0.2s
 fabs v0.4h, v0.4h
@@ -111,41 +132,71 @@ fabs v0.4s, v0.4s
 fabs v0.8h, v0.8h
 facge d20, d21, d22
 facge s10, s11, s12
+facge h24, h26, h29
+facge v25.4h, v16.4h, v11.4h
+facge v19.2s, v24.2s, v5.2s
 facge v0.4s, v0.4s, v0.4s
 facgt d20, d21, d22
 facgt s10, s11, s12
+facgt h0, h4, h10
 facgt v0.2d, v0.2d, v0.2d
+facgt v22.8h, v14.8h, v31.8h
+facgt v22.4s, v8.4s, v2.4s
 fadd v0.4s, v0.4s, v0.4s
+faddp h10, v19.2h
+faddp d11, v28.2d
 faddp v0.2s, v0.2s, v0.2s
 faddp v0.4s, v0.4s, v0.4s
+faddp v16.2d, v11.2d, v5.2d
 fcadd v0.2s, v0.2s, v0.2s,  90
 fcadd v0.4s, v0.4s, v0.4s, 270
+fcmeq h30, h6, h1
+fcmeq h19, h23, #0.0
 fcmeq d20, d21, #0.0
 fcmeq d20, d21, d22
 fcmeq s10, s11, #0.0
 fcmeq s10, s11, s12
 fcmeq v0.2s, v0.2s, #0.0
 fcmeq v0.2s, v0.2s, v0.2s
+fcmeq v12.4s, v11.4s, v26.4s
+fcmeq v18.2d, v17.2d, #0.0
+fcmge h10, h23, #0.0
+fcmge h1, h16, h12
 fcmge d20, d21, #0.0
 fcmge d20, d21, d22
 fcmge s10, s11, #0.0
 fcmge s10, s11, s12
 fcmge v0.2d, v0.2d, #0.0
+fcmge v17.2d, v11.2d, v13.2d
 fcmge v0.4s, v0.4s, v0.4s
+fcmge v18.4h, v27.4h, #0.0
+fcmge v20.8h, v19.8h, v22.8h
+fcmge v17.2s, v11.2s, #0.0
+fcmgt h4, h5, h0
+fcmgt h0, h18, #0.0
 fcmgt d20, d21, #0.0
 fcmgt d20, d21, d22
 fcmgt s10, s11, #0.0
 fcmgt s10, s11, s12
 fcmgt v0.4s, v0.4s, #0.0
 fcmgt v0.4s, v0.4s, v0.4s
+fcmgt v24.8h, v24.8h, v28.8h
+fcmgt v0.8h, v11.8h, #0.0
+fcmgt v19.2d, v31.2d, #0.0
 fcmla v0.2s, v0.2s, v0.2s, #90
 fcmla v0.4s, v0.4s, v0.s[1], #0
+fcmle v16.8h, v11.8h, #0.0
+fcmle v22.4s, v30.4s, #0.0
 fcmle d20, d21, #0.0
 fcmle s10, s11, #0.0
 fcmle v0.2d, v0.2d, #0.0
+fcmle h18, h28, #0.0
+fcmlt h23, h7, #0.0
 fcmlt d20, d21, #0.0
 fcmlt s10, s11, #0.0
 fcmlt v0.4s, v0.4s, #0.0
+fcmlt v8.4h, v2.4h, #0.0
+fcmlt v7.2d, v16.2d, #0.0
 fcvtas d21, d14
 fcvtas s12, s13
 fcvtas h12, h13
@@ -232,9 +283,11 @@ fcvtzs v0.2d, v0.2d, #3
 fcvtzs v0.2s, v0.2s
 fcvtzs v0.2s, v0.2s, #3
 fcvtzs v0.4h, v0.4h
+fcvtzs v20.4h, v24.4h, #11
 fcvtzs v0.4s, v0.4s
 fcvtzs v0.4s, v0.4s, #3
 fcvtzs v0.8h, v0.8h
+fcvtzs v18.8h, v10.8h, #7
 fcvtzu d21, d12, #1
 fcvtzu d21, d14
 fcvtzu s12, s13
@@ -246,9 +299,11 @@ fcvtzu v0.2d, v0.2d, #3
 fcvtzu v0.2s, v0.2s
 fcvtzu v0.2s, v0.2s, #3
 fcvtzu v0.4h, v0.4h
+fcvtzu v19.4h, v26.4h, #9
 fcvtzu v0.4s, v0.4s
 fcvtzu v0.4s, v0.4s, #3
 fcvtzu v0.8h, v0.8h
+fcvtzu v27.8h, v6.8h, #11
 fdiv v0.2d, v0.2d, v0.2d
 fdiv v0.2s, v0.2s, v0.2s
 fdiv v0.4h, v0.4h, v0.4h
@@ -260,12 +315,19 @@ fmax v0.4s, v0.4s, v0.4s
 fmaxnm v0.2d, v0.2d, v0.2d
 fmaxnm v0.2s, v0.2s, v0.2s
 fmaxnm v0.4s, v0.4s, v0.4s
+fmaxnmp h25, v19.2h
+fmaxnmp d17, v29.2d
 fmaxnmp v0.2d, v0.2d, v0.2d
 fmaxnmp v0.2s, v0.2s, v0.2s
 fmaxnmp v0.4s, v0.4s, v0.4s
+fmaxnmv h0, v13.4h
+fmaxnmv h12, v11.8h
+fmaxnmv s28, v31.4s
 fmaxp v0.2d, v0.2d, v0.2d
 fmaxp v0.2s, v0.2s, v0.2s
 fmaxp v0.4s, v0.4s, v0.4s
+fmaxp h15, v25.2h
+fmaxp s6, v2.2s
 fmaxv h0, v0.4h
 fmaxv h0, v0.8h
 fmaxv s0, v0.4s
@@ -275,15 +337,31 @@ fmin v0.4s, v0.4s, v0.4s
 fminnm v0.2d, v0.2d, v0.2d
 fminnm v0.2s, v0.2s, v0.2s
 fminnm v0.4s, v0.4s, v0.4s
+fminnmp h20, v14.2h
+fminnmp d15, v8.2d
 fminnmp v0.2d, v0.2d, v0.2d
 fminnmp v0.2s, v0.2s, v0.2s
 fminnmp v0.4s, v0.4s, v0.4s
+fminnmv h19, v25.4h
+fminnmv h23, v17.8h
+fminnmv s29, v17.4s
 fminp v0.2d, v0.2d, v0.2d
 fminp v0.2s, v0.2s, v0.2s
 fminp v0.4s, v0.4s, v0.4s
+fminp h7, v10.2h
+fminp s17, v7.2s
+fminv h3, v30.4h
+fminv h29, v12.8h
+fminv s16, v19.4s
 fmla d0, d1, v0.d[1]
+fmla h23, h24, v15.h[4]
 fmla s0, s1, v0.s[3]
 fmla v0.2s, v0.2s, v0.2s
+fmla v29.8h, v15.8h, v10.h[4]
+fmla v2.2s, v16.2s, v28.s[0]
+fmla v14.4s, v14.4s, v5.s[3]
+fmla v1.4s, v24.4s, v12.4s
+fmla v10.2d, v14.2d, v21.d[1]
 fmlal v0.2s, v0.2h, v0.h[1]
 fmlal v0.4s, v0.4h, v0.h[3]
 fmlal v0.2s, v0.2h, v0.2h
@@ -293,8 +371,14 @@ fmlal2 v0.4s, v0.4h, v0.h[3]
 fmlal2 v0.2s, v0.2h, v0.2h
 fmlal2 v0.4s, v0.4h, v0.4h
 fmls d0, d4, v0.d[1]
+fmls h8, h14, v7.h[4]
 fmls s3, s5, v0.s[3]
 fmls v0.2s, v0.2s, v0.2s
+fmls v30.8h, v18.8h, v4.h[6]
+fmls v10.2s, v27.2s, v0.s[0]
+fmls v27.4s, v7.4s, v24.s[0]
+fmls v10.2d, v22.2d, v29.d[0]
+fmls v6.8h, v15.8h, v23.8h
 fmlsl v0.2s, v0.2h, v0.h[1]
 fmlsl v0.4s, v0.4h, v0.h[3]
 fmlsl v0.2s, v0.2h, v0.2h
@@ -306,21 +390,34 @@ fmlsl2 v0.4s, v0.4h, v0.4h
 fmov v0.2d, #-1.25
 fmov v0.2s, #13.0
 fmov v0.4s, #1.0
+fmul h18, h4, v7.h[3]
+fmul v10.4h, v2.4h, v7.h[5]
+fmul v5.2s, v12.2s, v9.s[0]
+fmul v15.4s, v30.4s, v2.s[3]
+fmul v11.2d, v31.2d, v24.d[1]
+fmul h28, h14, h3
 fmul d0, d1, v0.d[1]
 fmul s0, s1, v0.s[3]
 fmul v0.2s, v0.2s, v0.2s
 fmulx d0, d4, v0.d[1]
 fmulx d23, d11, d1
 fmulx s20, s22, s15
+fmulx h18, h17, v7.h[1]
+fmulx h20, h25, h0
 fmulx s3, s5, v0.s[3]
 fmulx v0.2d, v0.2d, v0.2d
+fmulx v28.4h, v25.4h, v15.h[1]
+fmulx v3.2s, v22.2s, v23.s[3]
 fmulx v0.2s, v0.2s, v0.2s
 fmulx v0.4s, v0.4s, v0.4s
+fmulx v5.4s, v28.4s, v15.s[3]
+fmulx v22.2d, v18.2d, v25.d[1]
 fneg v0.2d, v0.2d
 fneg v0.2s, v0.2s
 fneg v0.4h, v0.4h
 fneg v0.4s, v0.4s
 fneg v0.8h, v0.8h
+frecpe h20, h8
 frecpe d13, d13
 frecpe s19, s14
 frecpe v0.2d, v0.2d
@@ -328,9 +425,13 @@ frecpe v0.2s, v0.2s
 frecpe v0.4h, v0.4h
 frecpe v0.4s, v0.4s
 frecpe v0.8h, v0.8h
+frecps h29, h19, h8
+frecpx h18, h11
+frecps v12.8h, v25.8h, v4.8h
 frecps  v0.4s, v0.4s, v0.4s
 frecps d22, d30, d21
 frecps s21, s16, s13
+frecps v7.2d, v29.2d, v18.2d
 frecpx d16, d19
 frecpx s18, s10
 frinta v0.2d, v0.2d
@@ -368,13 +469,17 @@ frintz v0.2s, v0.2s
 frintz v0.4h, v0.4h
 frintz v0.4s, v0.4s
 frintz v0.8h, v0.8h
+frsqrte h23, h26
 frsqrte d21, d12
 frsqrte s22, s13
 frsqrte v0.2d, v0.2d
 frsqrte v0.2s, v0.2s
 frsqrte v0.4h, v0.4h
 frsqrte v0.4s, v0.4s
+frsqrts v20.4s, v26.4s, v27.4s
+frsqrts v8.4h, v9.4h, v30.4h
 frsqrte v0.8h, v0.8h
+frsqrts h28, h26, h1
 frsqrts d8, d22, d18
 frsqrts s21, s5, s12
 frsqrts v0.2d, v0.2d, v0.2d
@@ -383,53 +488,138 @@ fsqrt v0.2s, v0.2s
 fsqrt v0.4h, v0.4h
 fsqrt v0.4s, v0.4s
 fsqrt v0.8h, v0.8h
+fsub v13.8h, v15.8h, v17.8h
 fsub v0.2s, v0.2s, v0.2s
 ld1 { v0.16b }, [x0]
+ld1 { v0.16b, v1.16b }, [x14]
+ld1 { v19.16b, v20.16b, v21.16b }, [x10]
+ld1 { v13.16b, v14.16b, v15.16b, v16.16b }, [x9]
+ld1 { v24.8h }, [x27]
+ld1 { v1.8h, v2.8h }, [x27]
 ld1 { v0.8h, v1.8h }, [sp], #32
+ld1 { v21.8h, v22.8h, v23.8h }, [x22]
+ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x21]
+ld1 { v3.4s }, [x4]
+ld1 { v11.4s, v12.4s }, [x30]
+ld1 { v0.4s, v1.4s, v2.4s }, [x24]
+ld1 { v15.4s, v16.4s, v17.4s, v18.4s }, [x28]
 ld1 { v0.4s, v1.4s, v2.4s }, [x0], #48
+ld1 { v3.2d }, [x28]
+ld1 { v13.2d, v14.2d }, [x13]
+ld1 { v12.2d, v13.2d, v14.2d }, [x15]
 ld1 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
 ld1 { v0.1d }, [x15], x2
+ld1 { v27.1d, v28.1d }, [x7]
+ld1 { v14.1d, v15.1d, v16.1d }, [x3]
+ld1 { v22.1d, v23.1d, v24.1d, v25.1d }, [x4]
 ld1 { v0.2s, v1.2s }, [x15]
+ld1 { v16.2s, v17.2s, v18.2s }, [x27]
+ld1 { v21.2s, v22.2s, v23.2s, v24.2s }, [x21]
+ld1 { v25.4h, v26.4h }, [x3]
+ld1 { v20.4h, v21.4h, v22.4h, v23.4h }, [x15]
 ld1 { v0.4h, v1.4h, v2.4h }, [sp]
+ld1 { v24.8b, v25.8b }, [x6]
+ld1 { v7.8b, v8.8b, v9.8b }, [x12]
+ld1 { v4.8b, v5.8b, v6.8b, v7.8b }, [x13]
 ld1 { v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
 ld1 { v0.b }[7], [x0]
 ld1 { v0.h }[3], [x0], #2
+ld1 { v18.h }[3], [x1]
 ld1 { v0.s }[1], [x15]
 ld1 { v0.d }[0], [x15], #8
+ld1 { v11.d }[0], [x13]
 ld1r { v0.16b }, [x0]
 ld1r { v0.8h }, [x0], #2
 ld1r { v0.4s }, [x15]
+ld1r { v3.1d }, [x15]
 ld1r { v0.2d }, [x15], x16
+ld1r { v18.2d }, [x0]
+ld1r { v8.8b }, [x23]
+ld1r { v28.4h }, [x9]
+ld1r { v3.8h }, [x16]
+ld1r { v10.2s }, [x20]
+ld2 { v0.4h, v1.4h }, [x21]
+ld2 { v8.8h, v9.8h }, [x28]
+ld2 { v2.2s, v3.2s }, [x16]
+ld2 { v22.4s, v23.4s }, [x4]
+ld2 { v22.2d, v23.2d }, [x17]
+ld2 { v29.b, v30.b }[3], [x1]
+ld2 { v26.s, v27.s }[1], [x17]
+ld2 { v1.d, v2.d }[0], [x10]
 ld2 { v0.16b, v1.16b }, [x0]
+ld2 { v13.8b, v14.8b }, [x4]
 ld2 { v0.8b, v1.8b }, [x0], #16
 ld2 { v0.h, v1.h }[7], [x15]
 ld2 { v0.h, v1.h }[7], [x15], x8
 ld2r { v0.8b, v1.8b }, [x0]
+ld2r { v10.16b, v11.16b }, [x23]
 ld2r { v0.4h, v1.4h }, [x0], #4
+ld2r { v25.4h, v26.4h }, [x11]
+ld2r { v23.8h, v24.8h }, [x10]
 ld2r { v0.2s, v1.2s }, [sp]
+ld2r { v8.4s, v9.4s }, [x17]
 ld2r { v0.1d, v1.1d }, [sp], x8
+ld2r { v9.1d, v10.1d }, [x25]
+ld2r { v26.2d, v27.2d }, [x8]
+ld3 { v8.8b, v9.8b, v10.8b }, [x0]
+ld3 { v15.16b, v16.16b, v17.16b }, [x5]
 ld3 { v0.4h, v1.4h, v2.4h }, [x15]
 ld3 { v0.8h, v1.8h, v2.8h }, [x15], #48
+ld3 { v7.8h, v8.8h, v9.8h }, [x21]
+ld3 { v16.2s, v17.2s, v18.2s }, [x0]
+ld3 { v12.4s, v13.4s, v14.4s }, [x25]
+ld3 { v17.b, v18.b, v19.b }[2], [x27]
+ld3 { v18.h, v19.h, v20.h }[5], [x16]
+ld3 { v10.2d, v11.2d, v12.2d }, [x18]
 ld3 { v0.s, v1.s, v2.s }[3], [sp]
 ld3 { v0.s, v1.s, v2.s }[3], [sp], x3
+ld3 { v5.d, v6.d, v7.d }[1], [x14]
 ld3r { v0.8b, v1.8b, v2.8b }, [x15]
+ld3r { v17.16b, v18.16b, v19.16b }, [x3]
 ld3r { v0.4h, v1.4h, v2.4h }, [x15], #6
+ld3r { v3.4h, v4.4h, v5.4h }, [x1]
+ld3r { v6.8h, v7.8h, v8.8h }, [x28]
 ld3r { v0.2s, v1.2s, v2.2s }, [x0]
+ld3r { v28.4s, v29.4s, v30.4s }, [x2]
 ld3r { v0.1d, v1.1d, v2.1d }, [x0], x0
+ld3r { v1.1d, v2.1d, v3.1d }, [x28]
+ld3r { v8.2d, v9.2d, v10.2d }, [x3]
+ld4 { v6.8b, v7.8b, v8.8b, v9.8b }, [x27]
+ld4 { v11.16b, v12.16b, v13.16b, v14.16b }, [x5]
+ld4 { v21.4h, v22.4h, v23.4h, v24.4h }, [x14]
+ld4 { v9.8h, v10.8h, v11.8h, v12.8h }, [x1]
+ld4 { v17.4s, v18.4s, v19.4s, v20.4s }, [x4]
 ld4 { v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
 ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
 ld4 { v0.d, v1.d, v2.d, v3.d }[1], [x0]
+ld4 { v2.2d, v3.2d, v4.2d, v5.2d }, [x24]
+ld4 { v4.b, v5.b, v6.b, v7.b }[12], [x27]
+ld4 { v5.h, v6.h, v7.h, v8.h }[0], [x4]
 ld4 { v0.h, v1.h, v2.h, v3.h }[7], [x0], x0
+ld4 { v0.s, v1.s, v2.s, v3.s }[0], [x26]
+ld4r { v20.8b, v21.8b, v22.8b, v23.8b }, [x23]
+ld4r { v1.16b, v2.16b, v3.16b, v4.16b }, [x25]
+ld4r { v16.4h, v17.4h, v18.4h, v19.4h }, [x6]
 ld4r { v0.1d, v1.1d, v2.1d, v3.1d }, [sp]
 ld4r { v0.2d, v1.2d, v2.2d, v3.2d }, [sp]
+ld4r { v4.8h, v5.8h, v6.8h, v7.8h }, [x23]
+ld4r { v0.2s, v1.2s, v2.2s, v3.2s }, [x30]
 ld4r { v0.2s, v1.2s, v2.2s, v3.2s }, [sp], #16
+ld4r { v7.4s, v8.4s, v9.4s, v10.4s }, [x23]
 ld4r { v0.4s, v1.4s, v2.4s, v3.4s }, [sp], x8
 mla v0.8b, v0.8b, v0.8b
+mla v15.8h, v22.8h, v4.h[3]
+mla v28.2s, v10.2s, v2.s[0]
 mls v0.4h, v0.4h, v0.4h
+mls v25.8h, v29.8h, v0.h[4]
+mls v22.2s, v29.2s, v0.s[3]
+mls v26.4s, v5.4s, v28.4s
 mov b0, v0.b[15]
 mov d6, v0.d[1]
 mov h2, v0.h[5]
 mov s17, v0.s[2]
+mov w8, v8.s[0]
+mov x30, v18.d[0]
 mov v2.b[0], v0.b[0]
 mov v2.h[1], v0.h[1]
 mov v2.s[2], v0.s[2]
@@ -442,11 +632,24 @@ mov v0.16b, v0.16b
 mov v0.8b, v0.8b
 movi d15, #0xff00ff00ff00ff
 movi v0.16b, #31
+movi v14.8h, #174
+movi v13.4h, #74, lsl #8
 movi v0.2d, #0xff0000ff0000ffff
 movi v0.2s, #8, msl #8
+movi v19.2s, #226
+movi v1.4s, #122, msl #8
 movi v0.4s, #255, lsl #24
 movi v0.8b, #255
 mul v0.8b, v0.8b, v0.8b
+mul v26.4h, v20.4h, v14.h[5]
+mul v5.8h, v21.8h, v3.h[7]
+mul v29.2s, v10.2s, v3.s[1]
+mul v30.4s, v11.4s, v4.s[0]
+mul v30.4s, v11.4s, v4.4s
+mul v3.8h, v9.8h, v8.8h
+mvni v9.4h, #237
+mvni v8.8h, #171, lsl #8
+mvni v22.4s, #15, lsl #8
 mvni v0.2s, 0
 mvni v0.4s, #16, msl #16
 neg d29, d24
@@ -460,8 +663,11 @@ neg v0.8h, v0.8h
 not v0.16b, v0.16b
 not v0.8b, v0.8b
 orn v0.16b, v0.16b, v0.16b
+orn v29.8b, v19.8b, v16.8b
 orr v0.16b, v0.16b, v0.16b
+orr v9.4h, #18
 orr v0.8h, #31
+orr v4.4s, #0
 pmul v0.16b, v0.16b, v0.16b
 pmul v0.8b, v0.8b, v0.8b
 pmull v0.8h, v0.8b, v0.8b
@@ -506,6 +712,7 @@ sabal2 v0.2d, v0.4s, v0.4s
 sabal2 v0.4s, v0.8h, v0.8h
 sabal2 v0.8h, v0.16b, v0.16b
 sabd v0.4h, v0.4h, v0.4h
+sabd v12.2s, v11.2s, v27.2s
 sabdl v0.2d, v0.2s, v0.2s
 sabdl v0.4s, v0.4h, v0.4h
 sabdl v0.8h, v0.8b, v0.8b
@@ -541,6 +748,8 @@ saddw v0.8h, v0.8h, v0.8b
 saddw2 v0.2d, v0.2d, v0.4s
 saddw2 v0.4s, v0.4s, v0.8h
 saddw2 v0.8h, v0.8h, v0.16b
+scvtf h4, h8, #9
+scvtf h5, h14
 scvtf d21, d12
 scvtf d21, d12, #64
 scvtf s22, s13
@@ -552,26 +761,25 @@ scvtf v0.2s, v0.2s, #3
 scvtf v0.4h, v0.4h
 scvtf v0.4s, v0.4s
 scvtf v0.4s, v0.4s, #3
+scvtf v25.4h, v13.4h, #8
 scvtf v0.8h, v0.8h
+scvtf v4.8h, v8.8h, #10
 sdot v0.2s, v0.8b, v0.4b[2]
 sdot v0.2s, v0.8b, v0.8b
 sdot v0.4s, v0.16b, v0.16b
 sdot v0.4s, v0.16b, v0.4b[2]
 shadd v0.8b, v0.8b, v0.8b
+shadd v25.16b, v1.16b, v10.16b
 shl d7, d10, #12
+shl v23.8b, v18.8b, #6
 shl v0.16b, v0.16b, #3
 shl v0.2d, v0.2d, #3
 shl v0.4h, v0.4h, #3
+shl v0.8h, v23.8h, #10
 shl v0.4s, v0.4s, #3
-shll v0.2d, v0.2s, #32
 shll v0.4s, v0.4h, #16
 shll v0.8h, v0.8b, #8
 shll v0.2d, v0.2s, #32
-shll v0.4s, v0.4h, #16
-shll v0.8h, v0.8b, #8
-shll2 v0.2d, v0.4s, #32
-shll2 v0.4s, v0.8h, #16
-shll2 v0.8h, v0.16b, #8
 shll2 v0.2d, v0.4s, #32
 shll2 v0.4s, v0.8h, #16
 shll2 v0.8h, v0.16b, #8
@@ -583,6 +791,7 @@ shrn2 v0.4s, v0.2d, #3
 shrn2 v0.8h, v0.4s, #3
 shsub v0.2s, v0.2s, v0.2s
 shsub v0.4h, v0.4h, v0.4h
+shsub v15.8h, v5.8h, v27.8h
 sli d10, d14, #12
 sli v0.16b, v0.16b, #3
 sli v0.2d, v0.2d, #3
@@ -594,8 +803,10 @@ sli v0.8h, v0.8h, #3
 smax v0.2s, v0.2s, v0.2s
 smax v0.4h, v0.4h, v0.4h
 smax v0.8b, v0.8b, v0.8b
+smax v30.16b, v3.16b, v30.16b
 smaxp v0.2s, v0.2s, v0.2s
 smaxp v0.4h, v0.4h, v0.4h
+smaxp v21.8h, v16.8h, v7.8h
 smaxp v0.8b, v0.8b, v0.8b
 smaxv b0, v0.8b
 smaxv b0, v0.16b
@@ -614,23 +825,35 @@ sminv h0, v0.4h
 sminv h0, v0.8h
 sminv s0, v0.4s
 smlal v0.2d, v0.2s, v0.2s
+smlal v0.2d, v25.2s, v1.s[1]
 smlal v0.4s, v0.4h, v0.4h
+smlal v16.4s, v9.4h, v11.h[4]
 smlal v0.8h, v0.8b, v0.8b
 smlal2 v0.2d, v0.4s, v0.4s
+smlal2 v30.2d, v22.4s, v7.s[2]
 smlal2 v0.4s, v0.8h, v0.8h
 smlal2 v0.8h, v0.16b, v0.16b
 smlsl v0.2d, v0.2s, v0.2s
+smlsl v25.2d, v27.2s, v1.s[1]
 smlsl v0.4s, v0.4h, v0.4h
+smlsl v14.4s, v23.4h, v12.h[7]
 smlsl v0.8h, v0.8b, v0.8b
+smlal2 v1.4s, v9.8h, v0.h[6]
+smlsl2 v12.4s, v11.8h, v12.h[0]
 smlsl2 v0.2d, v0.4s, v0.4s
+smlsl2 v11.2d, v28.4s, v7.s[2]
 smlsl2 v0.4s, v0.8h, v0.8h
 smlsl2 v0.8h, v0.16b, v0.16b
 smmla v0.4s, v0.16b, v0.16b
 smull v0.2d, v0.2s, v0.2s
+smull v31.2d, v23.2s, v6.s[2]
 smull v0.4s, v0.4h, v0.4h
+smull v3.4s, v26.4h, v1.h[5]
 smull v0.8h, v0.8b, v0.8b
 smull2 v0.2d, v0.4s, v0.4s
+smull2 v11.2d, v1.4s, v7.s[0]
 smull2 v0.4s, v0.8h, v0.8h
+smull2 v13.4s, v18.8h, v0.h[3]
 smull2 v0.8h, v0.16b, v0.16b
 sqabs b19, b14
 sqabs d18, d12
@@ -644,6 +867,7 @@ sqabs v0.4s, v0.4s
 sqabs v0.8b, v0.8b
 sqabs v0.8h, v0.8h
 sqadd b20, b11, b15
+sqadd h12, h18, h10
 sqadd v0.16b, v0.16b, v0.16b
 sqadd v0.2s, v0.2s, v0.2s
 sqdmlal d19, s24, s12
@@ -651,17 +875,34 @@ sqdmlal d8, s9, v0.s[1]
 sqdmlal s0, h0, v0.h[3]
 sqdmlal s17, h27, h12
 sqdmlal v0.2d, v0.2s, v0.2s
+sqdmlal v11.2d, v24.2s, v0.s[3]
 sqdmlal v0.4s, v0.4h, v0.4h
+sqdmlal v20.4s, v30.4h, v12.h[3]
 sqdmlal2 v0.2d, v0.4s, v0.4s
+sqdmlal2 v23.2d, v30.4s, v6.s[0]
 sqdmlal2 v0.4s, v0.8h, v0.8h
+sqdmlal2 v2.4s, v17.8h, v5.h[6]
+sqdmulh v8.4h, v16.4h, v5.h[4]
+sqdmulh v16.2s, v24.2s, v7.s[2]
+sqdmull v8.4s, v19.4h, v1.h[2]
+sqdmull v20.2d, v10.2s, v6.s[2]
+sqdmull2 v10.4s, v25.8h, v0.h[7]
+sqdmull2 v4.2d, v29.4s, v2.s[3]
+sqrdmulh v0.8h, v15.8h, v0.h[5]
+sqrdmulh v6.2s, v29.2s, v4.s[2]
+sqrdmulh v31.2s, v17.2s, v4.2s
 sqdmlsl d12, s23, s13
 sqdmlsl d8, s9, v0.s[1]
 sqdmlsl s0, h0, v0.h[3]
 sqdmlsl s14, h12, h25
 sqdmlsl v0.2d, v0.2s, v0.2s
+sqdmlsl v26.2d, v7.2s, v3.s[0]
 sqdmlsl v0.4s, v0.4h, v0.4h
+sqdmlsl v4.4s, v22.4h, v13.h[2]
 sqdmlsl2 v0.2d, v0.4s, v0.4s
+sqdmlsl2 v4.2d, v3.4s, v3.s[2]
 sqdmlsl2 v0.4s, v0.8h, v0.8h
+sqdmlsl2 v2.4s, v28.8h, v4.h[6]
 sqdmulh h10, h11, h12
 sqdmulh h7, h15, v0.h[3]
 sqdmulh s15, s14, v0.s[1]
@@ -722,6 +963,9 @@ sqrshl h3, h4, h15
 sqrshl v0.2s, v0.2s, v0.2s
 sqrshl v0.4h, v0.4h, v0.4h
 sqrshl v0.8b, v0.8b, v0.8b
+sqshl s17, s4, s23
+sqsub b3, b13, b12
+sqsub v20.8h, v18.8h, v12.8h
 sqrshrn b10, h13, #2
 sqrshrn h15, s10, #6
 sqrshrn s15, d12, #9
@@ -747,6 +991,7 @@ sqshl h13, h18, #11
 sqshl h3, h4, h15
 sqshl s14, s17, #22
 sqshl v0.16b, v0.16b, #3
+sqshl v23.16b, v23.16b, v23.16b
 sqshl v0.2d, v0.2d, #3
 sqshl v0.2s, v0.2s, #3
 sqshl v0.2s, v0.2s, v0.2s
@@ -843,8 +1088,12 @@ sshl v0.2d, v0.2d, v0.2d
 sshl v0.2s, v0.2s, v0.2s
 sshl v0.4h, v0.4h, v0.4h
 sshl v0.8b, v0.8b, v0.8b
+sshll v9.8h, v2.8b, #0
+sshll v12.4s, v3.4h, #4
 sshll v0.2d, v0.2s, #3
+sshll2 v28.8h, v12.16b, #7
 sshll2 v0.4s, v0.8h, #3
+sshll2 v17.2d, v13.4s, #22
 sshr d15, d16, #12
 sshr v0.16b, v0.16b, #3
 sshr v0.2d, v0.2d, #3
@@ -873,30 +1122,91 @@ ssubw v0.8h, v0.8h, v0.8b
 ssubw2 v0.2d, v0.2d, v0.4s
 ssubw2 v0.4s, v0.4s, v0.8h
 ssubw2 v0.8h, v0.8h, v0.16b
+st1 { v18.8b }, [x15]
+st1 { v8.8b, v9.8b }, [x18]
+st1 { v15.8b, v16.8b, v17.8b }, [x0]
+st1 { v21.8b, v22.8b, v23.8b, v24.8b }, [x14]
 st1 { v0.16b }, [x0]
+st1 { v1.16b, v2.16b }, [x4]
+st1 { v27.16b, v28.16b, v29.16b }, [x18]
+st1 { v18.16b, v19.16b, v20.16b, v21.16b }, [x29]
+st1 { v19.4h }, [x7]
+st1 { v22.4h, v23.4h }, [x22]
+st1 { v13.4h, v14.4h, v15.4h }, [x7]
+st1 { v23.4h, v24.4h, v25.4h, v26.4h }, [x24]
+st1 { v27.8h }, [x17]
+st1 { v8.8h, v9.8h, v10.8h }, [x16]
+st1 { v7.8h, v8.8h, v9.8h, v10.8h }, [x19]
+st1 { v25.2s }, [x6]
+st1 { v13.2s, v14.2s }, [x9]
+st1 { v12.2s, v13.2s, v14.2s }, [x3]
+st1 { v6.2s, v7.2s, v8.2s, v9.2s }, [x13]
 st1 { v0.4s, v1.4s }, [sp], #32
+st1 { v22.4s }, [x19]
+st1 { v15.4s, v16.4s }, [x12]
+st1 { v26.4s, v27.4s, v28.4s, v29.4s }, [x12]
+st1 { v20.1d }, [x10]
+st1 { v21.1d, v22.1d }, [x29]
+st1 { v5.1d, v6.1d, v7.1d }, [x3]
+st1 { v0.1d, v1.1d, v2.1d, v3.1d }, [x10]
+st1 { v26.2d, v27.2d }, [x28]
 st1 { v0.2d, v1.2d, v2.2d }, [x0], #48
+st1 { v13.2d, v14.2d, v15.2d }, [x27]
 st1 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+st1 { v8.2d }, [x15]
 st1 { v0.8h }, [x15], x2
 st1 { v0.8h, v1.8h }, [x15]
 st1 { v0.4s, v1.4s, v2.4s }, [sp]
 st1 { v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+st1 { v1.b }[5], [x1]
+st1 { v0.h }[2], [x1]
+st1 { v31.s }[1], [x16]
 st1 { v0.d }[1], [x0]
 st1 { v0.d }[1], [x0], #8
 st2 { v0.16b, v1.16b }, [x0], x1
 st2 { v0.8b, v1.8b }, [x0]
+st2 { v6.16b, v7.16b }, [x23]
+st2 { v10.4h, v11.4h }, [x18]
+st2 { v10.8h, v11.8h }, [x18]
+st2 { v25.2s, v26.2s }, [x29]
+st2 { v26.4s, v27.4s }, [x14]
+st2 { v10.2d, v11.2d }, [x1]
+st2 { v21.b, v22.b }[15], [x15]
+st2 { v28.h, v29.h }[2], [x6]
 st2 { v0.s, v1.s }[3], [sp]
 st2 { v0.s, v1.s }[3], [sp], #8
+st2 { v17.d, v18.d }[1], [x1]
+st3 { v10.8b, v11.8b, v12.8b }, [x18]
+st3 { v26.16b, v27.16b, v28.16b }, [x4]
 st3 { v0.4h, v1.4h, v2.4h }, [x15]
 st3 { v0.8h, v1.8h, v2.8h }, [x15], x2
+st3 { v0.8h, v1.8h, v2.8h }, [x0]
+st3 { v19.2s, v20.2s, v21.2s }, [x30]
+st3 { v24.4s, v25.4s, v26.4s }, [x8]
+st3 { v24.2d, v25.2d, v26.2d }, [x25]
+st3 { v8.b, v9.b, v10.b }[4], [x18]
 st3 { v0.h, v1.h, v2.h }[7], [x15]
 st3 { v0.h, v1.h, v2.h }[7], [x15], #6
+st3 { v9.s, v10.s, v11.s }[2], [x20]
+st3 { v16.d, v17.d, v18.d }[0], [x13]
+st4 { v17.8b, v18.8b, v19.8b, v20.8b }, [x8]
+st4 { v7.16b, v8.16b, v9.16b, v10.16b }, [x15]
+st4 { v5.4h, v6.4h, v7.4h, v8.4h }, [x13]
+st4 { v11.8h, v12.8h, v13.8h, v14.8h }, [x1]
 st4 { v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
 st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+st4 { v21.4s, v22.4s, v23.4s, v24.4s }, [x6]
+st4 { v25.2d, v26.2d, v27.2d, v28.2d }, [x16]
 st4 { v0.b, v1.b, v2.b, v3.b }[15], [x0]
+st4 { v5.h, v6.h, v7.h, v8.h }[4], [x13]
+st4 { v22.s, v23.s, v24.s, v25.s }[0], [x7]
+st4 { v23.d, v24.d, v25.d, v26.d }[1], [x5]
 st4 { v0.d, v1.d, v2.d, v3.d }[1], [x0], x5
 sub d15, d5, d16
 sub v0.2d, v0.2d, v0.2d
+sub v15.2s, v14.2s, v11.2s
+subhn v7.4h, v10.4s, v13.4s
+subhn2 v24.4s, v24.2d, v8.2d
 sudot v0.2s, v0.8b, v0.4b[2]
 sudot v0.4s, v0.16b, v0.4b[2]
 suqadd b19, b14
@@ -941,6 +1251,7 @@ trn2 v0.4s, v0.4s, v0.4s
 trn2 v0.8b, v0.8b, v0.8b
 trn2 v0.8h, v0.8h, v0.8h
 uaba v0.8b, v0.8b, v0.8b
+uaba v13.16b, v14.16b, v19.16b
 uabal v0.2d, v0.2s, v0.2s
 uabal v0.4s, v0.4h, v0.4h
 uabal v0.8h, v0.8b, v0.8b
@@ -948,6 +1259,7 @@ uabal2 v0.2d, v0.4s, v0.4s
 uabal2 v0.4s, v0.8h, v0.8h
 uabal2 v0.8h, v0.16b, v0.16b
 uabd v0.4h, v0.4h, v0.4h
+uabd v23.4s, v4.4s, v30.4s
 uabdl v0.2d, v0.2s, v0.2s
 uabdl v0.4s, v0.4h, v0.4h
 uabdl v0.8h, v0.8b, v0.8b
@@ -983,8 +1295,12 @@ uaddw v0.8h, v0.8h, v0.8b
 uaddw2 v0.2d, v0.2d, v0.4s
 uaddw2 v0.4s, v0.4s, v0.8h
 uaddw2 v0.8h, v0.8h, v0.16b
+ucvtf h17, x12
+ucvtf h22, h16, #11
+ucvtf h7, h21
 ucvtf d21, d14
 ucvtf d21, d14, #64
+ucvtf s8, x0
 ucvtf s22, s13
 ucvtf s22, s13, #32
 ucvtf v0.2d, v0.2d
@@ -994,13 +1310,16 @@ ucvtf v0.2s, v0.2s, #3
 ucvtf v0.4h, v0.4h
 ucvtf v0.4s, v0.4s
 ucvtf v0.4s, v0.4s, #3
+ucvtf v18.4h, v11.4h, #7
 ucvtf v0.8h, v0.8h
+ucvtf v22.8h, v20.8h, #10
 udot v0.2s, v0.8b, v0.4b[2]
 udot v0.2s, v0.8b, v0.8b
 udot v0.4s, v0.16b, v0.16b
 udot v0.4s, v0.16b, v0.4b[2]
 uhadd v0.16b, v0.16b, v0.16b
 uhadd v0.8h, v0.8h, v0.8h
+uhsub v12.4h, v16.4h, v28.4h
 uhsub v0.4s, v0.4s, v0.4s
 umax v0.16b, v0.16b, v0.16b
 umax v0.4s, v0.4s, v0.4s
@@ -1016,7 +1335,9 @@ umaxv s0, v0.4s
 umin v0.2s, v0.2s, v0.2s
 umin v0.4h, v0.4h, v0.4h
 umin v0.8b, v0.8b, v0.8b
+umin v0.16b, v26.16b, v2.16b
 uminp v0.2s, v0.2s, v0.2s
+uminp v28.4s, v16.4s, v15.4s
 uminp v0.4h, v0.4h, v0.4h
 uminp v0.8b, v0.8b, v0.8b
 uminv b0, v0.8b
@@ -1025,35 +1346,51 @@ uminv h0, v0.4h
 uminv h0, v0.8h
 uminv s0, v0.4s
 umlal v0.2d, v0.2s, v0.2s
+umlal v28.2d, v31.2s, v0.s[1]
 umlal v0.4s, v0.4h, v0.4h
+umlal v22.4s, v14.4h, v0.h[6]
 umlal v0.8h, v0.8b, v0.8b
+umlal2 v10.2d, v4.4s, v3.s[2]
+umlal2 v31.4s, v7.8h, v15.h[5]
 umlal2 v0.2d, v0.4s, v0.4s
 umlal2 v0.4s, v0.8h, v0.8h
 umlal2 v0.8h, v0.16b, v0.16b
 umlsl v0.2d, v0.2s, v0.2s
+umlsl v20.2d, v20.2s, v2.s[0]
 umlsl v0.4s, v0.4h, v0.4h
+umlsl v21.4s, v12.4h, v7.h[5]
 umlsl v0.8h, v0.8b, v0.8b
 umlsl2 v0.2d, v0.4s, v0.4s
+umlsl2 v30.2d, v23.4s, v1.s[2]
 umlsl2 v0.4s, v0.8h, v0.8h
+umlsl2 v27.4s, v28.8h, v6.h[4]
 umlsl2 v0.8h, v0.16b, v0.16b
 ummla v0.4s, v0.16b, v0.16b
+umov w6, v22.b[0]
 umov w0, v0.b[1]
+umov w10, v25.h[0]
 umov w0, v0.h[1]
 umov w0, v0.s[1]
 umov x0, v0.d[1]
 umull v0.2d, v0.2s, v0.2s
+umull v22.2d, v28.2s, v6.s[1]
 umull v0.4s, v0.4h, v0.4h
+umull v27.4s, v1.4h, v8.h[6]
 umull v0.8h, v0.8b, v0.8b
 umull2 v0.2d, v0.4s, v0.4s
+umull2 v28.2d, v21.4s, v1.s[0]
 umull2 v0.4s, v0.8h, v0.8h
+umull2 v18.4s, v26.8h, v10.h[1]
 umull2 v0.8h, v0.16b, v0.16b
 uqadd h0, h1, h5
+uqadd s0, s24, s30
 uqadd v0.8h, v0.8h, v0.8h
+uqadd v14.2d, v22.2d, v20.2d
 uqrshl b11, b20, b30
 uqrshl s23, s20, s16
+uqrshl v25.8b, v13.8b, v23.8b
 uqrshl v0.16b, v0.16b, v0.16b
 uqrshl v0.4s, v0.4s, v0.4s
-uqrshl v0.4s, v0.4s, v0.4s
 uqrshl v0.8h, v0.8h, v0.8h
 uqrshrn b10, h12, #5
 uqrshrn h12, s10, #14
@@ -1076,6 +1413,7 @@ uqshl v0.2d, v0.2d, #3
 uqshl v0.2d, v0.2d, v0.2d
 uqshl v0.2s, v0.2s, #3
 uqshl v0.4h, v0.4h, #3
+uqshl v8.4h, v17.4h, v13.4h
 uqshl v0.4s, v0.4s, #3
 uqshl v0.4s, v0.4s, v0.4s
 uqshl v0.8b, v0.8b, #3
@@ -1090,8 +1428,10 @@ uqshrn v0.8b, v0.8h, #3
 uqshrn2 v0.16b, v0.8h, #3
 uqshrn2 v0.4s, v0.2d, #3
 uqshrn2 v0.8h, v0.4s, #3
+uqsub s16, s21, s6
 uqsub d16, d16, d16
 uqsub v0.4h, v0.4h, v0.4h
+uqsub v19.4s, v0.4s, v5.4s
 uqxtn b18, h18
 uqxtn h20, s17
 uqxtn s19, d14
@@ -1106,7 +1446,9 @@ urecpe v0.4s, v0.4s
 urhadd v0.16b, v0.16b, v0.16b
 urhadd v0.4s, v0.4s, v0.4s
 urhadd v0.8h, v0.8h, v0.8h
+urhadd v16.2s, v19.2s, v2.2s
 urshl d8, d7, d4
+urshl v31.8b, v5.8b, v3.8b
 urshl v0.16b, v0.16b, v0.16b
 urshl v0.2d, v0.2d, v0.2d
 urshl v0.4s, v0.4s, v0.4s
@@ -1134,10 +1476,15 @@ usdot v0.2s, v0.8b, v0.8b
 usdot v0.4s, v0.16b, v0.16b
 usdot v0.4s, v0.16b, v0.4b[2]
 ushl d0, d0, d0
+ushl v6.8b, v26.8b, v6.8b
 ushl v0.16b, v0.16b, v0.16b
 ushl v0.4s, v0.4s, v0.4s
 ushl v0.8h, v0.8h, v0.8h
 ushll v0.4s, v0.4h, #3
+ushll v18.8h, v24.8b, #4
+ushll v16.2d, v16.2s, #31
+ushll2 v31.2d, v12.4s, #11
+ushll2 v18.4s, v22.8h, #13
 ushll2 v0.8h, v0.16b, #3
 ushr d10, d17, #18
 ushr v0.16b, v0.16b, #3
@@ -1148,6 +1495,11 @@ ushr v0.4s, v0.4s, #3
 ushr v0.8b, v0.8b, #3
 ushr v0.8h, v0.8h, #3
 usmmla v0.4s, v0.16b, v0.16b
+smov w15, v22.b[0]
+smov w26, v27.h[0]
+smov x21, v0.b[0]
+smov x9, v27.h[0]
+smov x15, v3.s[0]
 smov w0, v0.b[1]
 smov w0, v0.h[1]
 smov x0, v0.b[1]
@@ -1244,8 +1596,10 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        addhn2	v0.16b, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        addhn2	v0.4s, v0.2d, v0.2d
 # CHECK-NEXT:  1      2     0.25                        addhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  1      2     0.25                        addp	v7.2s, v1.2s, v2.2s
 # CHECK-NEXT:  1      2     0.25                        addp	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  1      2     0.25                        addp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      2     0.25                        addp	d1, v14.2d
 # CHECK-NEXT:  1      2     0.50                        addv	s0, v0.4s
 # CHECK-NEXT:  1      2     0.50                        addv	h0, v0.4h
 # CHECK-NEXT:  2      4     0.50                        addv	h0, v0.8h
@@ -1259,6 +1613,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      3     0.50                        bfcvt	h0, s0
 # CHECK-NEXT:  1      4     0.50                        bfcvtn	v0.4h, v0.4s
 # CHECK-NEXT:  1      4     0.50                        bfcvtn2	v0.8h, v0.4s
+# CHECK-NEXT:  1      4     0.25                        bfdot	v0.2s, v24.4h, v14.2h[2]
 # CHECK-NEXT:  1      4     0.25                        bfdot	v0.2s, v0.4h, v0.4h
 # CHECK-NEXT:  1      4     0.25                        bfdot	v0.4s, v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.25                        bfmlalb	v0.4s, v0.8h, v0.8h
@@ -1267,10 +1622,16 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.25                        bfmlalt	v0.4s, v0.8h, v0.h[3]
 # CHECK-NEXT:  1      5     0.25                        bfmmla	v0.4s, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        bic	v0.4h, #15, lsl #8
+# CHECK-NEXT:  1      2     0.25                        bic	v23.8h, #101
 # CHECK-NEXT:  1      2     0.25                        bic	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      2     0.25                        bic	v25.16b, v10.16b, v9.16b
+# CHECK-NEXT:  1      2     0.25                        bic	v24.2s, #70
+# CHECK-NEXT:  1      2     0.25                        bit	v5.8b, v12.8b, v22.8b
+# CHECK-NEXT:  1      2     0.25                        bif	v0.8b, v25.8b, v4.8b
 # CHECK-NEXT:  1      2     0.25                        bif	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  1      2     0.25                        bit	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  1      2     0.25                        bsl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      2     0.25                        bsl	v27.16b, v13.16b, v21.16b
 # CHECK-NEXT:  1      2     0.25                        cls	v0.16b, v0.16b
 # CHECK-NEXT:  1      2     0.25                        cls	v0.2s, v0.2s
 # CHECK-NEXT:  1      2     0.25                        cls	v0.4h, v0.4h
@@ -1283,28 +1644,37 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        clz	v0.4s, v0.4s
 # CHECK-NEXT:  1      2     0.25                        clz	v0.8b, v0.8b
 # CHECK-NEXT:  1      2     0.25                        clz	v0.8h, v0.8h
+# CHECK-NEXT:  1      2     0.25                        cmeq	v9.8h, v16.8h, v24.8h
+# CHECK-NEXT:  1      2     0.25                        cmeq	v14.4h, v18.4h, #0
 # CHECK-NEXT:  1      2     0.25                        cmeq	d20, d21, #0
 # CHECK-NEXT:  1      2     0.25                        cmeq	d20, d21, d22
 # CHECK-NEXT:  1      2     0.25                        cmeq	v0.16b, v0.16b, #0
 # CHECK-NEXT:  1      2     0.25                        cmeq	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      2     0.25                        cmge	v22.8h, v16.8h, v3.8h
+# CHECK-NEXT:  1      2     0.25                        cmge	v22.16b, v30.16b, #0
 # CHECK-NEXT:  1      2     0.25                        cmge	d20, d21, #0
 # CHECK-NEXT:  1      2     0.25                        cmge	d20, d21, d22
 # CHECK-NEXT:  1      2     0.25                        cmge	v0.4h, v0.4h, v0.4h
 # CHECK-NEXT:  1      2     0.25                        cmge	v0.8b, v0.8b, #0
+# CHECK-NEXT:  1      2     0.25                        cmgt	v3.2d, v29.2d, v11.2d
 # CHECK-NEXT:  1      2     0.25                        cmgt	d20, d21, #0
 # CHECK-NEXT:  1      2     0.25                        cmgt	d20, d21, d22
 # CHECK-NEXT:  1      2     0.25                        cmgt	v0.2s, v0.2s, #0
 # CHECK-NEXT:  1      2     0.25                        cmgt	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      2     0.25                        cmhi	v28.4h, v25.4h, v21.4h
 # CHECK-NEXT:  1      2     0.25                        cmhi	d20, d21, d22
 # CHECK-NEXT:  1      2     0.25                        cmhi	v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        cmhs	d20, d21, d22
 # CHECK-NEXT:  1      2     0.25                        cmhs	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      2     0.25                        cmle	v21.2s, v19.2s, #0
 # CHECK-NEXT:  1      2     0.25                        cmle	d20, d21, #0
 # CHECK-NEXT:  1      2     0.25                        cmle	v0.2d, v0.2d, #0
+# CHECK-NEXT:  1      2     0.25                        cmlt	v26.4h, v12.4h, #0
 # CHECK-NEXT:  1      2     0.25                        cmlt	d20, d21, #0
 # CHECK-NEXT:  1      2     0.25                        cmlt	v0.8h, v0.8h, #0
 # CHECK-NEXT:  1      2     0.25                        cmtst	d20, d21, d22
 # CHECK-NEXT:  1      2     0.25                        cmtst	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      2     0.25                        cmtst	v13.2d, v13.2d, v13.2d
 # CHECK-NEXT:  1      2     0.25                        cnt	v0.16b, v0.16b
 # CHECK-NEXT:  1      2     0.25                        cnt	v0.8b, v0.8b
 # CHECK-NEXT:  1      3     1.00                        dup	v0.16b, w28
@@ -1330,7 +1700,10 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        ext	v0.8b, v0.8b, v0.8b, #3
 # CHECK-NEXT:  1      2     0.25                        fabd	d29, d24, d20
 # CHECK-NEXT:  1      2     0.25                        fabd	s29, s24, s20
+# CHECK-NEXT:  1      2     0.25                        fabd	h27, h20, h17
+# CHECK-NEXT:  1      2     0.25                        fabd	v13.8h, v28.8h, v12.8h
 # CHECK-NEXT:  1      2     0.25                        fabd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      2     0.25                        fabs	h25, h7
 # CHECK-NEXT:  1      2     0.25                        fabs	v0.2d, v0.2d
 # CHECK-NEXT:  1      2     0.25                        fabs	v0.2s, v0.2s
 # CHECK-NEXT:  1      2     0.25                        fabs	v0.4h, v0.4h
@@ -1338,41 +1711,71 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        fabs	v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        facge	d20, d21, d22
 # CHECK-NEXT:  1      2     0.25                        facge	s10, s11, s12
+# CHECK-NEXT:  1      2     0.25                        facge	h24, h26, h29
+# CHECK-NEXT:  1      2     0.25                        facge	v25.4h, v16.4h, v11.4h
+# CHECK-NEXT:  1      2     0.25                        facge	v19.2s, v24.2s, v5.2s
 # CHECK-NEXT:  1      2     0.25                        facge	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT:  1      2     0.25                        facgt	d20, d21, d22
 # CHECK-NEXT:  1      2     0.25                        facgt	s10, s11, s12
+# CHECK-NEXT:  1      2     0.25                        facgt	h0, h4, h10
 # CHECK-NEXT:  1      2     0.25                        facgt	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      2     0.25                        facgt	v22.8h, v14.8h, v31.8h
+# CHECK-NEXT:  1      2     0.25                        facgt	v22.4s, v8.4s, v2.4s
 # CHECK-NEXT:  1      2     0.25                        fadd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      2     0.25                        faddp	h10, v19.2h
+# CHECK-NEXT:  1      2     0.25                        faddp	d11, v28.2d
 # CHECK-NEXT:  1      2     0.25                        faddp	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      2     0.25                        faddp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      2     0.25                        faddp	v16.2d, v11.2d, v5.2d
 # CHECK-NEXT:  1      4     0.25                        fcadd	v0.2s, v0.2s, v0.2s, #90
 # CHECK-NEXT:  1      4     0.25                        fcadd	v0.4s, v0.4s, v0.4s, #270
+# CHECK-NEXT:  1      2     0.25                        fcmeq	h30, h6, h1
+# CHECK-NEXT:  1      2     0.25                        fcmeq	h19, h23, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmeq	d20, d21, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmeq	d20, d21, d22
 # CHECK-NEXT:  1      2     0.25                        fcmeq	s10, s11, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmeq	s10, s11, s12
 # CHECK-NEXT:  1      2     0.25                        fcmeq	v0.2s, v0.2s, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmeq	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      2     0.25                        fcmeq	v12.4s, v11.4s, v26.4s
+# CHECK-NEXT:  1      2     0.25                        fcmeq	v18.2d, v17.2d, #0.0
+# CHECK-NEXT:  1      2     0.25                        fcmge	h10, h23, #0.0
+# CHECK-NEXT:  1      2     0.25                        fcmge	h1, h16, h12
 # CHECK-NEXT:  1      2     0.25                        fcmge	d20, d21, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmge	d20, d21, d22
 # CHECK-NEXT:  1      2     0.25                        fcmge	s10, s11, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmge	s10, s11, s12
 # CHECK-NEXT:  1      2     0.25                        fcmge	v0.2d, v0.2d, #0.0
+# CHECK-NEXT:  1      2     0.25                        fcmge	v17.2d, v11.2d, v13.2d
 # CHECK-NEXT:  1      2     0.25                        fcmge	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      2     0.25                        fcmge	v18.4h, v27.4h, #0.0
+# CHECK-NEXT:  1      2     0.25                        fcmge	v20.8h, v19.8h, v22.8h
+# CHECK-NEXT:  1      2     0.25                        fcmge	v17.2s, v11.2s, #0.0
+# CHECK-NEXT:  1      2     0.25                        fcmgt	h4, h5, h0
+# CHECK-NEXT:  1      2     0.25                        fcmgt	h0, h18, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmgt	d20, d21, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmgt	d20, d21, d22
 # CHECK-NEXT:  1      2     0.25                        fcmgt	s10, s11, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmgt	s10, s11, s12
 # CHECK-NEXT:  1      2     0.25                        fcmgt	v0.4s, v0.4s, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmgt	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      2     0.25                        fcmgt	v24.8h, v24.8h, v28.8h
+# CHECK-NEXT:  1      2     0.25                        fcmgt	v0.8h, v11.8h, #0.0
+# CHECK-NEXT:  1      2     0.25                        fcmgt	v19.2d, v31.2d, #0.0
 # CHECK-NEXT:  1      4     0.25                        fcmla	v0.2s, v0.2s, v0.2s, #90
 # CHECK-NEXT:  1      4     0.25                        fcmla	v0.4s, v0.4s, v0.s[1], #0
+# CHECK-NEXT:  1      2     0.25                        fcmle	v16.8h, v11.8h, #0.0
+# CHECK-NEXT:  1      2     0.25                        fcmle	v22.4s, v30.4s, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmle	d20, d21, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmle	s10, s11, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmle	v0.2d, v0.2d, #0.0
+# CHECK-NEXT:  1      2     0.25                        fcmle	h18, h28, #0.0
+# CHECK-NEXT:  1      2     0.25                        fcmlt	h23, h7, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmlt	d20, d21, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmlt	s10, s11, #0.0
 # CHECK-NEXT:  1      2     0.25                        fcmlt	v0.4s, v0.4s, #0.0
+# CHECK-NEXT:  1      2     0.25                        fcmlt	v8.4h, v2.4h, #0.0
+# CHECK-NEXT:  1      2     0.25                        fcmlt	v7.2d, v16.2d, #0.0
 # CHECK-NEXT:  1      3     0.50                        fcvtas	d21, d14
 # CHECK-NEXT:  2      4     1.00                        fcvtas	s12, s13
 # CHECK-NEXT:  4      6     1.00                        fcvtas	h12, h13
@@ -1459,9 +1862,11 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      3     0.50                        fcvtzs	v0.2s, v0.2s
 # CHECK-NEXT:  1      3     0.50                        fcvtzs	v0.2s, v0.2s, #3
 # CHECK-NEXT:  2      4     1.00                        fcvtzs	v0.4h, v0.4h
+# CHECK-NEXT:  2      4     1.00                        fcvtzs	v20.4h, v24.4h, #11
 # CHECK-NEXT:  2      4     1.00                        fcvtzs	v0.4s, v0.4s
 # CHECK-NEXT:  2      4     1.00                        fcvtzs	v0.4s, v0.4s, #3
 # CHECK-NEXT:  4      6     1.00                        fcvtzs	v0.8h, v0.8h
+# CHECK-NEXT:  4      6     1.00                        fcvtzs	v18.8h, v10.8h, #7
 # CHECK-NEXT:  1      3     0.50                        fcvtzu	d21, d12, #1
 # CHECK-NEXT:  1      3     0.50                        fcvtzu	d21, d14
 # CHECK-NEXT:  2      4     1.00                        fcvtzu	s12, s13
@@ -1473,9 +1878,11 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      3     0.50                        fcvtzu	v0.2s, v0.2s
 # CHECK-NEXT:  1      3     0.50                        fcvtzu	v0.2s, v0.2s, #3
 # CHECK-NEXT:  2      4     1.00                        fcvtzu	v0.4h, v0.4h
+# CHECK-NEXT:  2      4     1.00                        fcvtzu	v19.4h, v26.4h, #9
 # CHECK-NEXT:  2      4     1.00                        fcvtzu	v0.4s, v0.4s
 # CHECK-NEXT:  2      4     1.00                        fcvtzu	v0.4s, v0.4s, #3
 # CHECK-NEXT:  4      6     1.00                        fcvtzu	v0.8h, v0.8h
+# CHECK-NEXT:  4      6     1.00                        fcvtzu	v27.8h, v6.8h, #11
 # CHECK-NEXT:  1      15    3.50                        fdiv	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  1      10    3.50                        fdiv	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      7     3.50                        fdiv	v0.4h, v0.4h, v0.4h
@@ -1487,12 +1894,19 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        fmaxnm	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  1      2     0.25                        fmaxnm	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      2     0.25                        fmaxnm	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      2     0.25                        fmaxnmp	h25, v19.2h
+# CHECK-NEXT:  1      2     0.25                        fmaxnmp	d17, v29.2d
 # CHECK-NEXT:  1      2     0.25                        fmaxnmp	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  1      2     0.25                        fmaxnmp	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      2     0.25                        fmaxnmp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  2      4     0.50                        fmaxnmv	h0, v13.4h
+# CHECK-NEXT:  3      6     0.75                        fmaxnmv	h12, v11.8h
+# CHECK-NEXT:  2      4     0.50                        fmaxnmv	s28, v31.4s
 # CHECK-NEXT:  1      2     0.25                        fmaxp	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  1      2     0.25                        fmaxp	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      2     0.25                        fmaxp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      2     0.25                        fmaxp	h15, v25.2h
+# CHECK-NEXT:  1      2     0.25                        fmaxp	s6, v2.2s
 # CHECK-NEXT:  2      4     0.50                        fmaxv	h0, v0.4h
 # CHECK-NEXT:  3      6     0.75                        fmaxv	h0, v0.8h
 # CHECK-NEXT:  2      4     0.50                        fmaxv	s0, v0.4s
@@ -1502,15 +1916,31 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        fminnm	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  1      2     0.25                        fminnm	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      2     0.25                        fminnm	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      2     0.25                        fminnmp	h20, v14.2h
+# CHECK-NEXT:  1      2     0.25                        fminnmp	d15, v8.2d
 # CHECK-NEXT:  1      2     0.25                        fminnmp	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  1      2     0.25                        fminnmp	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      2     0.25                        fminnmp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  2      4     0.50                        fminnmv	h19, v25.4h
+# CHECK-NEXT:  3      6     0.75                        fminnmv	h23, v17.8h
+# CHECK-NEXT:  2      4     0.50                        fminnmv	s29, v17.4s
 # CHECK-NEXT:  1      2     0.25                        fminp	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  1      2     0.25                        fminp	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      2     0.25                        fminp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      2     0.25                        fminp	h7, v10.2h
+# CHECK-NEXT:  1      2     0.25                        fminp	s17, v7.2s
+# CHECK-NEXT:  2      4     0.50                        fminv	h3, v30.4h
+# CHECK-NEXT:  3      6     0.75                        fminv	h29, v12.8h
+# CHECK-NEXT:  2      4     0.50                        fminv	s16, v19.4s
 # CHECK-NEXT:  1      4     0.25                        fmla	d0, d1, v0.d[1]
+# CHECK-NEXT:  1      4     0.25                        fmla	h23, h24, v15.h[4]
 # CHECK-NEXT:  1      4     0.25                        fmla	s0, s1, v0.s[3]
 # CHECK-NEXT:  1      4     0.25                        fmla	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.25                        fmla	v29.8h, v15.8h, v10.h[4]
+# CHECK-NEXT:  1      4     0.25                        fmla	v2.2s, v16.2s, v28.s[0]
+# CHECK-NEXT:  1      4     0.25                        fmla	v14.4s, v14.4s, v5.s[3]
+# CHECK-NEXT:  1      4     0.25                        fmla	v1.4s, v24.4s, v12.4s
+# CHECK-NEXT:  1      4     0.25                        fmla	v10.2d, v14.2d, v21.d[1]
 # CHECK-NEXT:  1      2     0.25                        fmlal	v0.2s, v0.2h, v0.h[1]
 # CHECK-NEXT:  1      2     0.25                        fmlal	v0.4s, v0.4h, v0.h[3]
 # CHECK-NEXT:  1      5     0.25                        fmlal	v0.2s, v0.2h, v0.2h
@@ -1520,8 +1950,14 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      5     0.25                        fmlal2	v0.2s, v0.2h, v0.2h
 # CHECK-NEXT:  1      5     0.25                        fmlal2	v0.4s, v0.4h, v0.4h
 # CHECK-NEXT:  1      4     0.25                        fmls	d0, d4, v0.d[1]
+# CHECK-NEXT:  1      4     0.25                        fmls	h8, h14, v7.h[4]
 # CHECK-NEXT:  1      4     0.25                        fmls	s3, s5, v0.s[3]
 # CHECK-NEXT:  1      4     0.25                        fmls	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.25                        fmls	v30.8h, v18.8h, v4.h[6]
+# CHECK-NEXT:  1      4     0.25                        fmls	v10.2s, v27.2s, v0.s[0]
+# CHECK-NEXT:  1      4     0.25                        fmls	v27.4s, v7.4s, v24.s[0]
+# CHECK-NEXT:  1      4     0.25                        fmls	v10.2d, v22.2d, v29.d[0]
+# CHECK-NEXT:  1      4     0.25                        fmls	v6.8h, v15.8h, v23.8h
 # CHECK-NEXT:  1      2     0.25                        fmlsl	v0.2s, v0.2h, v0.h[1]
 # CHECK-NEXT:  1      2     0.25                        fmlsl	v0.4s, v0.4h, v0.h[3]
 # CHECK-NEXT:  1      5     0.25                        fmlsl	v0.2s, v0.2h, v0.2h
@@ -1533,21 +1969,34 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        fmov	v0.2d, #-1.25000000
 # CHECK-NEXT:  1      2     0.25                        fmov	v0.2s, #13.00000000
 # CHECK-NEXT:  1      2     0.25                        fmov	v0.4s, #1.00000000
+# CHECK-NEXT:  1      3     0.25                        fmul	h18, h4, v7.h[3]
+# CHECK-NEXT:  1      3     0.25                        fmul	v10.4h, v2.4h, v7.h[5]
+# CHECK-NEXT:  1      3     0.25                        fmul	v5.2s, v12.2s, v9.s[0]
+# CHECK-NEXT:  1      3     0.25                        fmul	v15.4s, v30.4s, v2.s[3]
+# CHECK-NEXT:  1      3     0.25                        fmul	v11.2d, v31.2d, v24.d[1]
+# CHECK-NEXT:  1      3     0.25                        fmul	h28, h14, h3
 # CHECK-NEXT:  1      3     0.25                        fmul	d0, d1, v0.d[1]
 # CHECK-NEXT:  1      3     0.25                        fmul	s0, s1, v0.s[3]
 # CHECK-NEXT:  1      3     0.25                        fmul	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      3     0.25                        fmulx	d0, d4, v0.d[1]
 # CHECK-NEXT:  1      2     0.25                        fmulx	d23, d11, d1
 # CHECK-NEXT:  1      2     0.25                        fmulx	s20, s22, s15
+# CHECK-NEXT:  1      3     0.25                        fmulx	h18, h17, v7.h[1]
+# CHECK-NEXT:  1      2     0.25                        fmulx	h20, h25, h0
 # CHECK-NEXT:  1      3     0.25                        fmulx	s3, s5, v0.s[3]
 # CHECK-NEXT:  1      3     0.25                        fmulx	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      3     0.25                        fmulx	v28.4h, v25.4h, v15.h[1]
+# CHECK-NEXT:  1      3     0.25                        fmulx	v3.2s, v22.2s, v23.s[3]
 # CHECK-NEXT:  1      3     0.25                        fmulx	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      3     0.25                        fmulx	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     0.25                        fmulx	v5.4s, v28.4s, v15.s[3]
+# CHECK-NEXT:  1      3     0.25                        fmulx	v22.2d, v18.2d, v25.d[1]
 # CHECK-NEXT:  1      2     0.25                        fneg	v0.2d, v0.2d
 # CHECK-NEXT:  1      2     0.25                        fneg	v0.2s, v0.2s
 # CHECK-NEXT:  1      2     0.25                        fneg	v0.4h, v0.4h
 # CHECK-NEXT:  1      2     0.25                        fneg	v0.4s, v0.4s
 # CHECK-NEXT:  1      2     0.25                        fneg	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        frecpe	h20, h8
 # CHECK-NEXT:  1      3     0.50                        frecpe	d13, d13
 # CHECK-NEXT:  1      3     0.50                        frecpe	s19, s14
 # CHECK-NEXT:  1      4     0.50                        frecpe	v0.2d, v0.2d
@@ -1555,9 +2004,13 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.50                        frecpe	v0.4h, v0.4h
 # CHECK-NEXT:  1      4     0.50                        frecpe	v0.4s, v0.4s
 # CHECK-NEXT:  2      6     1.00                        frecpe	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.25                        frecps	h29, h19, h8
+# CHECK-NEXT:  1      3     0.50                        frecpx	h18, h11
+# CHECK-NEXT:  1      4     0.25                        frecps	v12.8h, v25.8h, v4.8h
 # CHECK-NEXT:  1      4     0.25                        frecps	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT:  1      4     0.25                        frecps	d22, d30, d21
 # CHECK-NEXT:  1      4     0.25                        frecps	s21, s16, s13
+# CHECK-NEXT:  1      4     0.25                        frecps	v7.2d, v29.2d, v18.2d
 # CHECK-NEXT:  1      3     0.50                        frecpx	d16, d19
 # CHECK-NEXT:  1      3     0.50                        frecpx	s18, s10
 # CHECK-NEXT:  1      3     0.50                        frinta	v0.2d, v0.2d
@@ -1595,13 +2048,17 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  2      4     1.00                        frintz	v0.4h, v0.4h
 # CHECK-NEXT:  2      4     1.00                        frintz	v0.4s, v0.4s
 # CHECK-NEXT:  4      6     1.00                        frintz	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        frsqrte	h23, h26
 # CHECK-NEXT:  1      3     0.50                        frsqrte	d21, d12
 # CHECK-NEXT:  1      3     0.50                        frsqrte	s22, s13
 # CHECK-NEXT:  1      4     0.50                        frsqrte	v0.2d, v0.2d
 # CHECK-NEXT:  1      3     0.50                        frsqrte	v0.2s, v0.2s
 # CHECK-NEXT:  1      4     0.50                        frsqrte	v0.4h, v0.4h
 # CHECK-NEXT:  1      4     0.50                        frsqrte	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.25                        frsqrts	v20.4s, v26.4s, v27.4s
+# CHECK-NEXT:  1      4     0.25                        frsqrts	v8.4h, v9.4h, v30.4h
 # CHECK-NEXT:  2      6     1.00                        frsqrte	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.25                        frsqrts	h28, h26, h1
 # CHECK-NEXT:  1      4     0.25                        frsqrts	d8, d22, d18
 # CHECK-NEXT:  1      4     0.25                        frsqrts	s21, s5, s12
 # CHECK-NEXT:  1      4     0.25                        frsqrts	v0.2d, v0.2d, v0.2d
@@ -1610,53 +2067,138 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      7     3.50                        fsqrt	v0.4h, v0.4h
 # CHECK-NEXT:  1      10    3.50                        fsqrt	v0.4s, v0.4s
 # CHECK-NEXT:  1      13    5.50                        fsqrt	v0.8h, v0.8h
+# CHECK-NEXT:  1      2     0.25                        fsub	v13.8h, v15.8h, v17.8h
 # CHECK-NEXT:  1      2     0.25                        fsub	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      6     0.33    *                   ld1	{ v0.16b }, [x0]
+# CHECK-NEXT:  2      6     0.67    *                   ld1	{ v0.16b, v1.16b }, [x14]
+# CHECK-NEXT:  3      6     1.00    *                   ld1	{ v19.16b, v20.16b, v21.16b }, [x10]
+# CHECK-NEXT:  4      7     1.33    *                   ld1	{ v13.16b, v14.16b, v15.16b, v16.16b }, [x9]
+# CHECK-NEXT:  1      6     0.33    *                   ld1	{ v24.8h }, [x27]
+# CHECK-NEXT:  2      6     0.67    *                   ld1	{ v1.8h, v2.8h }, [x27]
 # CHECK-NEXT:  3      6     0.67    *                   ld1	{ v0.8h, v1.8h }, [sp], #32
+# CHECK-NEXT:  3      6     1.00    *                   ld1	{ v21.8h, v22.8h, v23.8h }, [x22]
+# CHECK-NEXT:  4      7     1.33    *                   ld1	{ v0.8h, v1.8h, v2.8h, v3.8h }, [x21]
+# CHECK-NEXT:  1      6     0.33    *                   ld1	{ v3.4s }, [x4]
+# CHECK-NEXT:  2      6     0.67    *                   ld1	{ v11.4s, v12.4s }, [x30]
+# CHECK-NEXT:  3      6     1.00    *                   ld1	{ v0.4s, v1.4s, v2.4s }, [x24]
+# CHECK-NEXT:  4      7     1.33    *                   ld1	{ v15.4s, v16.4s, v17.4s, v18.4s }, [x28]
 # CHECK-NEXT:  4      6     1.00    *                   ld1	{ v0.4s, v1.4s, v2.4s }, [x0], #48
+# CHECK-NEXT:  1      6     0.33    *                   ld1	{ v3.2d }, [x28]
+# CHECK-NEXT:  2      6     0.67    *                   ld1	{ v13.2d, v14.2d }, [x13]
+# CHECK-NEXT:  3      6     1.00    *                   ld1	{ v12.2d, v13.2d, v14.2d }, [x15]
 # CHECK-NEXT:  4      7     1.33    *                   ld1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
 # CHECK-NEXT:  2      6     0.33    *                   ld1	{ v0.1d }, [x15], x2
+# CHECK-NEXT:  2      6     0.67    *                   ld1	{ v27.1d, v28.1d }, [x7]
+# CHECK-NEXT:  3      6     1.00    *                   ld1	{ v14.1d, v15.1d, v16.1d }, [x3]
+# CHECK-NEXT:  2      6     0.67    *                   ld1	{ v22.1d, v23.1d, v24.1d, v25.1d }, [x4]
 # CHECK-NEXT:  2      6     0.67    *                   ld1	{ v0.2s, v1.2s }, [x15]
+# CHECK-NEXT:  3      6     1.00    *                   ld1	{ v16.2s, v17.2s, v18.2s }, [x27]
+# CHECK-NEXT:  2      6     0.67    *                   ld1	{ v21.2s, v22.2s, v23.2s, v24.2s }, [x21]
+# CHECK-NEXT:  2      6     0.67    *                   ld1	{ v25.4h, v26.4h }, [x3]
+# CHECK-NEXT:  2      6     0.67    *                   ld1	{ v20.4h, v21.4h, v22.4h, v23.4h }, [x15]
 # CHECK-NEXT:  3      6     1.00    *                   ld1	{ v0.4h, v1.4h, v2.4h }, [sp]
+# CHECK-NEXT:  2      6     0.67    *                   ld1	{ v24.8b, v25.8b }, [x6]
+# CHECK-NEXT:  3      6     1.00    *                   ld1	{ v7.8b, v8.8b, v9.8b }, [x12]
+# CHECK-NEXT:  2      6     0.67    *                   ld1	{ v4.8b, v5.8b, v6.8b, v7.8b }, [x13]
 # CHECK-NEXT:  3      6     0.67    *                   ld1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
 # CHECK-NEXT:  2      8     0.33    *                   ld1	{ v0.b }[7], [x0]
 # CHECK-NEXT:  3      8     0.33    *                   ld1	{ v0.h }[3], [x0], #2
+# CHECK-NEXT:  2      8     0.33    *                   ld1	{ v18.h }[3], [x1]
 # CHECK-NEXT:  2      8     0.33    *                   ld1	{ v0.s }[1], [x15]
 # CHECK-NEXT:  3      8     0.33    *                   ld1	{ v0.d }[0], [x15], #8
+# CHECK-NEXT:  2      8     0.33    *                   ld1	{ v11.d }[0], [x13]
 # CHECK-NEXT:  2      8     0.33    *                   ld1r	{ v0.16b }, [x0]
 # CHECK-NEXT:  3      8     0.33    *                   ld1r	{ v0.8h }, [x0], #2
 # CHECK-NEXT:  2      8     0.33    *                   ld1r	{ v0.4s }, [x15]
+# CHECK-NEXT:  2      8     0.33    *                   ld1r	{ v3.1d }, [x15]
 # CHECK-NEXT:  3      8     0.33    *                   ld1r	{ v0.2d }, [x15], x16
+# CHECK-NEXT:  2      8     0.33    *                   ld1r	{ v18.2d }, [x0]
+# CHECK-NEXT:  2      8     0.33    *                   ld1r	{ v8.8b }, [x23]
+# CHECK-NEXT:  2      8     0.33    *                   ld1r	{ v28.4h }, [x9]
+# CHECK-NEXT:  2      8     0.33    *                   ld1r	{ v3.8h }, [x16]
+# CHECK-NEXT:  2      8     0.33    *                   ld1r	{ v10.2s }, [x20]
+# CHECK-NEXT:  3      8     0.50    *                   ld2	{ v0.4h, v1.4h }, [x21]
+# CHECK-NEXT:  4      8     0.67    *                   ld2	{ v8.8h, v9.8h }, [x28]
+# CHECK-NEXT:  3      8     0.50    *                   ld2	{ v2.2s, v3.2s }, [x16]
+# CHECK-NEXT:  4      8     0.67    *                   ld2	{ v22.4s, v23.4s }, [x4]
+# CHECK-NEXT:  4      8     0.67    *                   ld2	{ v22.2d, v23.2d }, [x17]
+# CHECK-NEXT:  3      8     0.50    *                   ld2	{ v29.b, v30.b }[3], [x1]
+# CHECK-NEXT:  3      8     0.50    *                   ld2	{ v26.s, v27.s }[1], [x17]
+# CHECK-NEXT:  3      8     0.50    *                   ld2	{ v1.d, v2.d }[0], [x10]
 # CHECK-NEXT:  4      8     0.67    *                   ld2	{ v0.16b, v1.16b }, [x0]
+# CHECK-NEXT:  3      8     0.50    *                   ld2	{ v13.8b, v14.8b }, [x4]
 # CHECK-NEXT:  4      8     0.50    *                   ld2	{ v0.8b, v1.8b }, [x0], #16
 # CHECK-NEXT:  3      8     0.50    *                   ld2	{ v0.h, v1.h }[7], [x15]
 # CHECK-NEXT:  4      8     0.50    *                   ld2	{ v0.h, v1.h }[7], [x15], x8
 # CHECK-NEXT:  3      8     0.50    *                   ld2r	{ v0.8b, v1.8b }, [x0]
+# CHECK-NEXT:  3      8     0.50    *                   ld2r	{ v10.16b, v11.16b }, [x23]
 # CHECK-NEXT:  4      8     0.50    *                   ld2r	{ v0.4h, v1.4h }, [x0], #4
+# CHECK-NEXT:  3      8     0.50    *                   ld2r	{ v25.4h, v26.4h }, [x11]
+# CHECK-NEXT:  3      8     0.50    *                   ld2r	{ v23.8h, v24.8h }, [x10]
 # CHECK-NEXT:  3      8     0.50    *                   ld2r	{ v0.2s, v1.2s }, [sp]
+# CHECK-NEXT:  3      8     0.50    *                   ld2r	{ v8.4s, v9.4s }, [x17]
 # CHECK-NEXT:  4      8     0.50    *                   ld2r	{ v0.1d, v1.1d }, [sp], x8
+# CHECK-NEXT:  3      8     0.50    *                   ld2r	{ v9.1d, v10.1d }, [x25]
+# CHECK-NEXT:  3      8     0.50    *                   ld2r	{ v26.2d, v27.2d }, [x8]
+# CHECK-NEXT:  5      8     0.75    *                   ld3	{ v8.8b, v9.8b, v10.8b }, [x0]
+# CHECK-NEXT:  6      8     1.00    *                   ld3	{ v15.16b, v16.16b, v17.16b }, [x5]
 # CHECK-NEXT:  5      8     0.75    *                   ld3	{ v0.4h, v1.4h, v2.4h }, [x15]
 # CHECK-NEXT:  7      8     1.00    *                   ld3	{ v0.8h, v1.8h, v2.8h }, [x15], #48
+# CHECK-NEXT:  6      8     1.00    *                   ld3	{ v7.8h, v8.8h, v9.8h }, [x21]
+# CHECK-NEXT:  5      8     0.75    *                   ld3	{ v16.2s, v17.2s, v18.2s }, [x0]
+# CHECK-NEXT:  6      8     1.00    *                   ld3	{ v12.4s, v13.4s, v14.4s }, [x25]
+# CHECK-NEXT:  5      8     0.75    *                   ld3	{ v17.b, v18.b, v19.b }[2], [x27]
+# CHECK-NEXT:  5      8     0.75    *                   ld3	{ v18.h, v19.h, v20.h }[5], [x16]
+# CHECK-NEXT:  6      8     1.00    *                   ld3	{ v10.2d, v11.2d, v12.2d }, [x18]
 # CHECK-NEXT:  5      8     0.75    *                   ld3	{ v0.s, v1.s, v2.s }[3], [sp]
 # CHECK-NEXT:  6      8     0.75    *                   ld3	{ v0.s, v1.s, v2.s }[3], [sp], x3
+# CHECK-NEXT:  5      8     0.75    *                   ld3	{ v5.d, v6.d, v7.d }[1], [x14]
 # CHECK-NEXT:  5      8     0.75    *                   ld3r	{ v0.8b, v1.8b, v2.8b }, [x15]
+# CHECK-NEXT:  5      8     0.75    *                   ld3r	{ v17.16b, v18.16b, v19.16b }, [x3]
 # CHECK-NEXT:  6      8     0.75    *                   ld3r	{ v0.4h, v1.4h, v2.4h }, [x15], #6
+# CHECK-NEXT:  5      8     0.75    *                   ld3r	{ v3.4h, v4.4h, v5.4h }, [x1]
+# CHECK-NEXT:  5      8     0.75    *                   ld3r	{ v6.8h, v7.8h, v8.8h }, [x28]
 # CHECK-NEXT:  5      8     0.75    *                   ld3r	{ v0.2s, v1.2s, v2.2s }, [x0]
+# CHECK-NEXT:  5      8     0.75    *                   ld3r	{ v28.4s, v29.4s, v30.4s }, [x2]
 # CHECK-NEXT:  6      8     0.75    *                   ld3r	{ v0.1d, v1.1d, v2.1d }, [x0], x0
+# CHECK-NEXT:  5      8     0.75    *                   ld3r	{ v1.1d, v2.1d, v3.1d }, [x28]
+# CHECK-NEXT:  5      8     0.75    *                   ld3r	{ v8.2d, v9.2d, v10.2d }, [x3]
+# CHECK-NEXT:  7      8     1.00    *                   ld4	{ v6.8b, v7.8b, v8.8b, v9.8b }, [x27]
+# CHECK-NEXT:  8      9     1.33    *                   ld4	{ v11.16b, v12.16b, v13.16b, v14.16b }, [x5]
+# CHECK-NEXT:  7      8     1.00    *                   ld4	{ v21.4h, v22.4h, v23.4h, v24.4h }, [x14]
+# CHECK-NEXT:  8      9     1.33    *                   ld4	{ v9.8h, v10.8h, v11.8h, v12.8h }, [x1]
+# CHECK-NEXT:  8      9     1.33    *                   ld4	{ v17.4s, v18.4s, v19.4s, v20.4s }, [x4]
 # CHECK-NEXT:  7      8     1.00    *                   ld4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
 # CHECK-NEXT:  9      9     1.33    *                   ld4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
 # CHECK-NEXT:  7      8     1.00    *                   ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0]
+# CHECK-NEXT:  8      9     1.33    *                   ld4	{ v2.2d, v3.2d, v4.2d, v5.2d }, [x24]
+# CHECK-NEXT:  7      8     1.00    *                   ld4	{ v4.b, v5.b, v6.b, v7.b }[12], [x27]
+# CHECK-NEXT:  7      8     1.00    *                   ld4	{ v5.h, v6.h, v7.h, v8.h }[0], [x4]
 # CHECK-NEXT:  8      8     1.00    *                   ld4	{ v0.h, v1.h, v2.h, v3.h }[7], [x0], x0
+# CHECK-NEXT:  7      8     1.00    *                   ld4	{ v0.s, v1.s, v2.s, v3.s }[0], [x26]
+# CHECK-NEXT:  7      8     1.00    *                   ld4r	{ v20.8b, v21.8b, v22.8b, v23.8b }, [x23]
+# CHECK-NEXT:  7      8     1.00    *                   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x25]
+# CHECK-NEXT:  7      8     1.00    *                   ld4r	{ v16.4h, v17.4h, v18.4h, v19.4h }, [x6]
 # CHECK-NEXT:  7      8     1.00    *                   ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp]
 # CHECK-NEXT:  7      8     1.00    *                   ld4r	{ v0.2d, v1.2d, v2.2d, v3.2d }, [sp]
+# CHECK-NEXT:  7      8     1.00    *                   ld4r	{ v4.8h, v5.8h, v6.8h, v7.8h }, [x23]
+# CHECK-NEXT:  7      8     1.00    *                   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [x30]
 # CHECK-NEXT:  8      8     1.00    *                   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp], #16
+# CHECK-NEXT:  7      8     1.00    *                   ld4r	{ v7.4s, v8.4s, v9.4s, v10.4s }, [x23]
 # CHECK-NEXT:  8      8     1.00    *                   ld4r	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], x8
 # CHECK-NEXT:  1      4     0.50                        mla	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        mla	v15.8h, v22.8h, v4.h[3]
+# CHECK-NEXT:  1      4     0.50                        mla	v28.2s, v10.2s, v2.s[0]
 # CHECK-NEXT:  1      4     0.50                        mls	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        mls	v25.8h, v29.8h, v0.h[4]
+# CHECK-NEXT:  1      4     0.50                        mls	v22.2s, v29.2s, v0.s[3]
+# CHECK-NEXT:  1      4     0.50                        mls	v26.4s, v5.4s, v28.4s
 # CHECK-NEXT:  1      2     0.25                        mov	b0, v0.b[15]
 # CHECK-NEXT:  1      2     0.25                        mov	d6, v0.d[1]
 # CHECK-NEXT:  1      2     0.25                        mov	h2, v0.h[5]
 # CHECK-NEXT:  1      2     0.25                        mov	s17, v0.s[2]
+# CHECK-NEXT:  1      2     0.25                        mov	w8, v8.s[0]
+# CHECK-NEXT:  1      2     0.25                        mov	x30, v18.d[0]
 # CHECK-NEXT:  1      2     0.25                        mov	v2.b[0], v0.b[0]
 # CHECK-NEXT:  1      2     0.25                        mov	v2.h[1], v0.h[1]
 # CHECK-NEXT:  1      2     0.25                        mov	v2.s[2], v0.s[2]
@@ -1669,11 +2211,24 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        mov	v0.8b, v0.8b
 # CHECK-NEXT:  1      2     0.25                        movi	d15, #0xff00ff00ff00ff
 # CHECK-NEXT:  1      2     0.25                        movi	v0.16b, #31
+# CHECK-NEXT:  1      2     0.25                        movi	v14.8h, #174
+# CHECK-NEXT:  1      2     0.25                        movi	v13.4h, #74, lsl #8
 # CHECK-NEXT:  1      2     0.25                        movi	v0.2d, #0xff0000ff0000ffff
 # CHECK-NEXT:  1      2     0.25                        movi	v0.2s, #8, msl #8
+# CHECK-NEXT:  1      2     0.25                        movi	v19.2s, #226
+# CHECK-NEXT:  1      2     0.25                        movi	v1.4s, #122, msl #8
 # CHECK-NEXT:  1      2     0.25                        movi	v0.4s, #255, lsl #24
 # CHECK-NEXT:  1      2     0.25                        movi	v0.8b, #255
 # CHECK-NEXT:  1      4     0.50                        mul	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        mul	v26.4h, v20.4h, v14.h[5]
+# CHECK-NEXT:  1      4     0.50                        mul	v5.8h, v21.8h, v3.h[7]
+# CHECK-NEXT:  1      4     0.50                        mul	v29.2s, v10.2s, v3.s[1]
+# CHECK-NEXT:  1      4     0.50                        mul	v30.4s, v11.4s, v4.s[0]
+# CHECK-NEXT:  1      4     0.50                        mul	v30.4s, v11.4s, v4.4s
+# CHECK-NEXT:  1      4     0.50                        mul	v3.8h, v9.8h, v8.8h
+# CHECK-NEXT:  1      2     0.25                        mvni	v9.4h, #237
+# CHECK-NEXT:  1      2     0.25                        mvni	v8.8h, #171, lsl #8
+# CHECK-NEXT:  1      2     0.25                        mvni	v22.4s, #15, lsl #8
 # CHECK-NEXT:  1      2     0.25                        mvni	v0.2s, #0
 # CHECK-NEXT:  1      2     0.25                        mvni	v0.4s, #16, msl #16
 # CHECK-NEXT:  1      2     0.25                        neg	d29, d24
@@ -1687,8 +2242,11 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        mvn	v0.16b, v0.16b
 # CHECK-NEXT:  1      2     0.25                        mvn	v0.8b, v0.8b
 # CHECK-NEXT:  1      2     0.25                        orn	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      2     0.25                        orn	v29.8b, v19.8b, v16.8b
 # CHECK-NEXT:  1      2     0.25                        mov	v0.16b, v0.16b
+# CHECK-NEXT:  1      2     0.25                        orr	v9.4h, #18
 # CHECK-NEXT:  1      2     0.25                        orr	v0.8h, #31
+# CHECK-NEXT:  1      2     0.25                        orr	v4.4s, #0
 # CHECK-NEXT:  1      3     0.50                        pmul	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  1      3     0.50                        pmul	v0.8b, v0.8b, v0.8b
 # CHECK-NEXT:  1      3     0.50                        pmull	v0.8h, v0.8b, v0.8b
@@ -1733,6 +2291,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.50                        sabal2	v0.4s, v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.50                        sabal2	v0.8h, v0.16b, v0.16b
 # CHECK-NEXT:  1      2     0.25                        sabd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      2     0.25                        sabd	v12.2s, v11.2s, v27.2s
 # CHECK-NEXT:  1      2     0.25                        sabdl	v0.2d, v0.2s, v0.2s
 # CHECK-NEXT:  1      2     0.25                        sabdl	v0.4s, v0.4h, v0.4h
 # CHECK-NEXT:  1      2     0.25                        sabdl	v0.8h, v0.8b, v0.8b
@@ -1768,6 +2327,8 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        saddw2	v0.2d, v0.2d, v0.4s
 # CHECK-NEXT:  1      2     0.25                        saddw2	v0.4s, v0.4s, v0.8h
 # CHECK-NEXT:  1      2     0.25                        saddw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  4      6     1.00                        scvtf	h4, h8, #9
+# CHECK-NEXT:  4      6     1.00                        scvtf	h5, h14
 # CHECK-NEXT:  1      3     0.50                        scvtf	d21, d12
 # CHECK-NEXT:  1      3     0.50                        scvtf	d21, d12, #64
 # CHECK-NEXT:  2      4     1.00                        scvtf	s22, s13
@@ -1779,26 +2340,25 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  2      4     1.00                        scvtf	v0.4h, v0.4h
 # CHECK-NEXT:  2      4     1.00                        scvtf	v0.4s, v0.4s
 # CHECK-NEXT:  2      4     1.00                        scvtf	v0.4s, v0.4s, #3
+# CHECK-NEXT:  2      4     1.00                        scvtf	v25.4h, v13.4h, #8
 # CHECK-NEXT:  4      6     1.00                        scvtf	v0.8h, v0.8h
+# CHECK-NEXT:  4      6     1.00                        scvtf	v4.8h, v8.8h, #10
 # CHECK-NEXT:  1      3     0.25                        sdot	v0.2s, v0.8b, v0.4b[2]
 # CHECK-NEXT:  1      3     0.25                        sdot	v0.2s, v0.8b, v0.8b
 # CHECK-NEXT:  1      3     0.25                        sdot	v0.4s, v0.16b, v0.16b
 # CHECK-NEXT:  1      3     0.25                        sdot	v0.4s, v0.16b, v0.4b[2]
 # CHECK-NEXT:  1      2     0.25                        shadd	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      2     0.25                        shadd	v25.16b, v1.16b, v10.16b
 # CHECK-NEXT:  1      2     0.25                        shl	d7, d10, #12
+# CHECK-NEXT:  1      2     0.50                        shl	v23.8b, v18.8b, #6
 # CHECK-NEXT:  1      2     0.50                        shl	v0.16b, v0.16b, #3
 # CHECK-NEXT:  1      2     0.50                        shl	v0.2d, v0.2d, #3
 # CHECK-NEXT:  1      2     0.50                        shl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      2     0.50                        shl	v0.8h, v23.8h, #10
 # CHECK-NEXT:  1      2     0.50                        shl	v0.4s, v0.4s, #3
-# CHECK-NEXT:  1      2     0.50                        shll	v0.2d, v0.2s, #32
 # CHECK-NEXT:  1      2     0.50                        shll	v0.4s, v0.4h, #16
 # CHECK-NEXT:  1      2     0.50                        shll	v0.8h, v0.8b, #8
 # CHECK-NEXT:  1      2     0.50                        shll	v0.2d, v0.2s, #32
-# CHECK-NEXT:  1      2     0.50                        shll	v0.4s, v0.4h, #16
-# CHECK-NEXT:  1      2     0.50                        shll	v0.8h, v0.8b, #8
-# CHECK-NEXT:  1      2     0.50                        shll2	v0.2d, v0.4s, #32
-# CHECK-NEXT:  1      2     0.50                        shll2	v0.4s, v0.8h, #16
-# CHECK-NEXT:  1      2     0.50                        shll2	v0.8h, v0.16b, #8
 # CHECK-NEXT:  1      2     0.50                        shll2	v0.2d, v0.4s, #32
 # CHECK-NEXT:  1      2     0.50                        shll2	v0.4s, v0.8h, #16
 # CHECK-NEXT:  1      2     0.50                        shll2	v0.8h, v0.16b, #8
@@ -1810,6 +2370,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.50                        shrn2	v0.8h, v0.4s, #3
 # CHECK-NEXT:  1      2     0.25                        shsub	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      2     0.25                        shsub	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      2     0.25                        shsub	v15.8h, v5.8h, v27.8h
 # CHECK-NEXT:  1      2     0.25                        sli	d10, d14, #12
 # CHECK-NEXT:  1      2     0.50                        sli	v0.16b, v0.16b, #3
 # CHECK-NEXT:  1      2     0.50                        sli	v0.2d, v0.2d, #3
@@ -1821,8 +2382,10 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        smax	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      2     0.25                        smax	v0.4h, v0.4h, v0.4h
 # CHECK-NEXT:  1      2     0.25                        smax	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      2     0.25                        smax	v30.16b, v3.16b, v30.16b
 # CHECK-NEXT:  1      2     0.25                        smaxp	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      2     0.25                        smaxp	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      2     0.25                        smaxp	v21.8h, v16.8h, v7.8h
 # CHECK-NEXT:  1      2     0.25                        smaxp	v0.8b, v0.8b, v0.8b
 # CHECK-NEXT:  2      4     0.50                        smaxv	b0, v0.8b
 # CHECK-NEXT:  2      4     1.00                        smaxv	b0, v0.16b
@@ -1841,23 +2404,35 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  2      4     0.50                        sminv	h0, v0.8h
 # CHECK-NEXT:  1      2     0.50                        sminv	s0, v0.4s
 # CHECK-NEXT:  1      4     0.50                        smlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        smlal	v0.2d, v25.2s, v1.s[1]
 # CHECK-NEXT:  1      4     0.50                        smlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        smlal	v16.4s, v9.4h, v11.h[4]
 # CHECK-NEXT:  1      4     0.50                        smlal	v0.8h, v0.8b, v0.8b
 # CHECK-NEXT:  1      4     0.50                        smlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        smlal2	v30.2d, v22.4s, v7.s[2]
 # CHECK-NEXT:  1      4     0.50                        smlal2	v0.4s, v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.50                        smlal2	v0.8h, v0.16b, v0.16b
 # CHECK-NEXT:  1      4     0.50                        smlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        smlsl	v25.2d, v27.2s, v1.s[1]
 # CHECK-NEXT:  1      4     0.50                        smlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        smlsl	v14.4s, v23.4h, v12.h[7]
 # CHECK-NEXT:  1      4     0.50                        smlsl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        smlal2	v1.4s, v9.8h, v0.h[6]
+# CHECK-NEXT:  1      4     0.50                        smlsl2	v12.4s, v11.8h, v12.h[0]
 # CHECK-NEXT:  1      4     0.50                        smlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        smlsl2	v11.2d, v28.4s, v7.s[2]
 # CHECK-NEXT:  1      4     0.50                        smlsl2	v0.4s, v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.50                        smlsl2	v0.8h, v0.16b, v0.16b
 # CHECK-NEXT:  1      3     0.25                        smmla	v0.4s, v0.16b, v0.16b
 # CHECK-NEXT:  1      3     0.50                        smull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     0.50                        smull	v31.2d, v23.2s, v6.s[2]
 # CHECK-NEXT:  1      3     0.50                        smull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     0.50                        smull	v3.4s, v26.4h, v1.h[5]
 # CHECK-NEXT:  1      3     0.50                        smull	v0.8h, v0.8b, v0.8b
 # CHECK-NEXT:  1      3     0.50                        smull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     0.50                        smull2	v11.2d, v1.4s, v7.s[0]
 # CHECK-NEXT:  1      3     0.50                        smull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     0.50                        smull2	v13.4s, v18.8h, v0.h[3]
 # CHECK-NEXT:  1      3     0.50                        smull2	v0.8h, v0.16b, v0.16b
 # CHECK-NEXT:  1      2     0.25                        sqabs	b19, b14
 # CHECK-NEXT:  1      2     0.25                        sqabs	d18, d12
@@ -1871,6 +2446,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        sqabs	v0.8b, v0.8b
 # CHECK-NEXT:  1      2     0.25                        sqabs	v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        sqadd	b20, b11, b15
+# CHECK-NEXT:  1      2     0.25                        sqadd	h12, h18, h10
 # CHECK-NEXT:  1      2     0.25                        sqadd	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  1      2     0.25                        sqadd	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      4     0.50                        sqdmlal	d19, s24, s12
@@ -1878,17 +2454,34 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.50                        sqdmlal	s0, h0, v0.h[3]
 # CHECK-NEXT:  1      4     0.50                        sqdmlal	s17, h27, h12
 # CHECK-NEXT:  1      4     0.50                        sqdmlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        sqdmlal	v11.2d, v24.2s, v0.s[3]
 # CHECK-NEXT:  1      4     0.50                        sqdmlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        sqdmlal	v20.4s, v30.4h, v12.h[3]
 # CHECK-NEXT:  1      4     0.50                        sqdmlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        sqdmlal2	v23.2d, v30.4s, v6.s[0]
 # CHECK-NEXT:  1      4     0.50                        sqdmlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        sqdmlal2	v2.4s, v17.8h, v5.h[6]
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	v8.4h, v16.4h, v5.h[4]
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	v16.2s, v24.2s, v7.s[2]
+# CHECK-NEXT:  1      3     0.50                        sqdmull	v8.4s, v19.4h, v1.h[2]
+# CHECK-NEXT:  1      3     0.50                        sqdmull	v20.2d, v10.2s, v6.s[2]
+# CHECK-NEXT:  1      3     0.50                        sqdmull2	v10.4s, v25.8h, v0.h[7]
+# CHECK-NEXT:  1      3     0.50                        sqdmull2	v4.2d, v29.4s, v2.s[3]
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	v0.8h, v15.8h, v0.h[5]
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	v6.2s, v29.2s, v4.s[2]
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	v31.2s, v17.2s, v4.2s
 # CHECK-NEXT:  1      4     0.50                        sqdmlsl	d12, s23, s13
 # CHECK-NEXT:  1      4     0.50                        sqdmlsl	d8, s9, v0.s[1]
 # CHECK-NEXT:  1      4     0.50                        sqdmlsl	s0, h0, v0.h[3]
 # CHECK-NEXT:  1      4     0.50                        sqdmlsl	s14, h12, h25
 # CHECK-NEXT:  1      4     0.50                        sqdmlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        sqdmlsl	v26.2d, v7.2s, v3.s[0]
 # CHECK-NEXT:  1      4     0.50                        sqdmlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        sqdmlsl	v4.4s, v22.4h, v13.h[2]
 # CHECK-NEXT:  1      4     0.50                        sqdmlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        sqdmlsl2	v4.2d, v3.4s, v3.s[2]
 # CHECK-NEXT:  1      4     0.50                        sqdmlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        sqdmlsl2	v2.4s, v28.8h, v4.h[6]
 # CHECK-NEXT:  1      4     0.50                        sqdmulh	h10, h11, h12
 # CHECK-NEXT:  1      4     0.50                        sqdmulh	h7, h15, v0.h[3]
 # CHECK-NEXT:  1      4     0.50                        sqdmulh	s15, s14, v0.s[1]
@@ -1949,6 +2542,9 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.50                        sqrshl	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      4     0.50                        sqrshl	v0.4h, v0.4h, v0.4h
 # CHECK-NEXT:  1      4     0.50                        sqrshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        sqshl	s17, s4, s23
+# CHECK-NEXT:  1      2     0.25                        sqsub	b3, b13, b12
+# CHECK-NEXT:  1      2     0.25                        sqsub	v20.8h, v18.8h, v12.8h
 # CHECK-NEXT:  1      2     0.25                        sqrshrn	b10, h13, #2
 # CHECK-NEXT:  1      2     0.25                        sqrshrn	h15, s10, #6
 # CHECK-NEXT:  1      2     0.25                        sqrshrn	s15, d12, #9
@@ -1974,6 +2570,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.50                        sqshl	h3, h4, h15
 # CHECK-NEXT:  1      4     0.50                        sqshl	s14, s17, #22
 # CHECK-NEXT:  1      4     0.50                        sqshl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      4     0.50                        sqshl	v23.16b, v23.16b, v23.16b
 # CHECK-NEXT:  1      4     0.50                        sqshl	v0.2d, v0.2d, #3
 # CHECK-NEXT:  1      4     0.50                        sqshl	v0.2s, v0.2s, #3
 # CHECK-NEXT:  1      4     0.50                        sqshl	v0.2s, v0.2s, v0.2s
@@ -2070,8 +2667,12 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.50                        sshl	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      2     0.50                        sshl	v0.4h, v0.4h, v0.4h
 # CHECK-NEXT:  1      2     0.50                        sshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      2     0.50                        sshll	v9.8h, v2.8b, #0
+# CHECK-NEXT:  1      2     0.50                        sshll	v12.4s, v3.4h, #4
 # CHECK-NEXT:  1      2     0.50                        sshll	v0.2d, v0.2s, #3
+# CHECK-NEXT:  1      2     0.50                        sshll2	v28.8h, v12.16b, #7
 # CHECK-NEXT:  1      2     0.50                        sshll2	v0.4s, v0.8h, #3
+# CHECK-NEXT:  1      2     0.50                        sshll2	v17.2d, v13.4s, #22
 # CHECK-NEXT:  1      2     0.25                        sshr	d15, d16, #12
 # CHECK-NEXT:  1      2     0.50                        sshr	v0.16b, v0.16b, #3
 # CHECK-NEXT:  1      2     0.50                        sshr	v0.2d, v0.2d, #3
@@ -2100,30 +2701,91 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        ssubw2	v0.2d, v0.2d, v0.4s
 # CHECK-NEXT:  1      2     0.25                        ssubw2	v0.4s, v0.4s, v0.8h
 # CHECK-NEXT:  1      2     0.25                        ssubw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  2      2     0.50           *            st1	{ v18.8b }, [x15]
+# CHECK-NEXT:  2      2     0.50           *            st1	{ v8.8b, v9.8b }, [x18]
+# CHECK-NEXT:  4      2     1.00           *            st1	{ v15.8b, v16.8b, v17.8b }, [x0]
+# CHECK-NEXT:  4      2     1.00           *            st1	{ v21.8b, v22.8b, v23.8b, v24.8b }, [x14]
 # CHECK-NEXT:  2      2     0.50           *            st1	{ v0.16b }, [x0]
+# CHECK-NEXT:  4      2     1.00           *            st1	{ v1.16b, v2.16b }, [x4]
+# CHECK-NEXT:  6      2     1.50           *            st1	{ v27.16b, v28.16b, v29.16b }, [x18]
+# CHECK-NEXT:  8      2     2.00           *            st1	{ v18.16b, v19.16b, v20.16b, v21.16b }, [x29]
+# CHECK-NEXT:  2      2     0.50           *            st1	{ v19.4h }, [x7]
+# CHECK-NEXT:  2      2     0.50           *            st1	{ v22.4h, v23.4h }, [x22]
+# CHECK-NEXT:  4      2     1.00           *            st1	{ v13.4h, v14.4h, v15.4h }, [x7]
+# CHECK-NEXT:  4      2     1.00           *            st1	{ v23.4h, v24.4h, v25.4h, v26.4h }, [x24]
+# CHECK-NEXT:  2      2     0.50           *            st1	{ v27.8h }, [x17]
+# CHECK-NEXT:  6      2     1.50           *            st1	{ v8.8h, v9.8h, v10.8h }, [x16]
+# CHECK-NEXT:  8      2     2.00           *            st1	{ v7.8h, v8.8h, v9.8h, v10.8h }, [x19]
+# CHECK-NEXT:  2      2     0.50           *            st1	{ v25.2s }, [x6]
+# CHECK-NEXT:  2      2     0.50           *            st1	{ v13.2s, v14.2s }, [x9]
+# CHECK-NEXT:  4      2     1.00           *            st1	{ v12.2s, v13.2s, v14.2s }, [x3]
+# CHECK-NEXT:  4      2     1.00           *            st1	{ v6.2s, v7.2s, v8.2s, v9.2s }, [x13]
 # CHECK-NEXT:  5      2     1.00           *            st1	{ v0.4s, v1.4s }, [sp], #32
+# CHECK-NEXT:  2      2     0.50           *            st1	{ v22.4s }, [x19]
+# CHECK-NEXT:  4      2     1.00           *            st1	{ v15.4s, v16.4s }, [x12]
+# CHECK-NEXT:  8      2     2.00           *            st1	{ v26.4s, v27.4s, v28.4s, v29.4s }, [x12]
+# CHECK-NEXT:  2      2     0.50           *            st1	{ v20.1d }, [x10]
+# CHECK-NEXT:  2      2     0.50           *            st1	{ v21.1d, v22.1d }, [x29]
+# CHECK-NEXT:  4      2     1.00           *            st1	{ v5.1d, v6.1d, v7.1d }, [x3]
+# CHECK-NEXT:  4      2     1.00           *            st1	{ v0.1d, v1.1d, v2.1d, v3.1d }, [x10]
+# CHECK-NEXT:  4      2     1.00           *            st1	{ v26.2d, v27.2d }, [x28]
 # CHECK-NEXT:  7      2     1.50           *            st1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
+# CHECK-NEXT:  6      2     1.50           *            st1	{ v13.2d, v14.2d, v15.2d }, [x27]
 # CHECK-NEXT:  8      2     2.00           *            st1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+# CHECK-NEXT:  2      2     0.50           *            st1	{ v8.2d }, [x15]
 # CHECK-NEXT:  3      2     0.50           *            st1	{ v0.8h }, [x15], x2
 # CHECK-NEXT:  4      2     1.00           *            st1	{ v0.8h, v1.8h }, [x15]
 # CHECK-NEXT:  6      2     1.50           *            st1	{ v0.4s, v1.4s, v2.4s }, [sp]
 # CHECK-NEXT:  5      2     1.00           *            st1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+# CHECK-NEXT:  2      4     0.50           *            st1	{ v1.b }[5], [x1]
+# CHECK-NEXT:  2      4     0.50           *            st1	{ v0.h }[2], [x1]
+# CHECK-NEXT:  2      4     0.50           *            st1	{ v31.s }[1], [x16]
 # CHECK-NEXT:  2      4     0.50           *            st1	{ v0.d }[1], [x0]
 # CHECK-NEXT:  3      4     0.50           *            st1	{ v0.d }[1], [x0], #8
 # CHECK-NEXT:  5      4     1.00           *            st2	{ v0.16b, v1.16b }, [x0], x1
 # CHECK-NEXT:  2      4     0.50           *            st2	{ v0.8b, v1.8b }, [x0]
+# CHECK-NEXT:  4      4     1.00           *            st2	{ v6.16b, v7.16b }, [x23]
+# CHECK-NEXT:  2      4     0.50           *            st2	{ v10.4h, v11.4h }, [x18]
+# CHECK-NEXT:  4      4     1.00           *            st2	{ v10.8h, v11.8h }, [x18]
+# CHECK-NEXT:  2      4     0.50           *            st2	{ v25.2s, v26.2s }, [x29]
+# CHECK-NEXT:  4      4     1.00           *            st2	{ v26.4s, v27.4s }, [x14]
+# CHECK-NEXT:  4      4     1.00           *            st2	{ v10.2d, v11.2d }, [x1]
+# CHECK-NEXT:  2      4     0.50           *            st2	{ v21.b, v22.b }[15], [x15]
+# CHECK-NEXT:  2      4     0.50           *            st2	{ v28.h, v29.h }[2], [x6]
 # CHECK-NEXT:  2      4     0.50           *            st2	{ v0.s, v1.s }[3], [sp]
 # CHECK-NEXT:  3      4     0.50           *            st2	{ v0.s, v1.s }[3], [sp], #8
+# CHECK-NEXT:  2      4     0.50           *            st2	{ v17.d, v18.d }[1], [x1]
+# CHECK-NEXT:  4      4     1.00           *            st3	{ v10.8b, v11.8b, v12.8b }, [x18]
+# CHECK-NEXT:  6      5     1.50           *            st3	{ v26.16b, v27.16b, v28.16b }, [x4]
 # CHECK-NEXT:  4      4     1.00           *            st3	{ v0.4h, v1.4h, v2.4h }, [x15]
 # CHECK-NEXT:  7      5     1.50           *            st3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
+# CHECK-NEXT:  6      5     1.50           *            st3	{ v0.8h, v1.8h, v2.8h }, [x0]
+# CHECK-NEXT:  4      4     1.00           *            st3	{ v19.2s, v20.2s, v21.2s }, [x30]
+# CHECK-NEXT:  6      5     1.50           *            st3	{ v24.4s, v25.4s, v26.4s }, [x8]
+# CHECK-NEXT:  6      5     1.50           *            st3	{ v24.2d, v25.2d, v26.2d }, [x25]
+# CHECK-NEXT:  4      4     1.00           *            st3	{ v8.b, v9.b, v10.b }[4], [x18]
 # CHECK-NEXT:  4      4     1.00           *            st3	{ v0.h, v1.h, v2.h }[7], [x15]
 # CHECK-NEXT:  5      4     1.00           *            st3	{ v0.h, v1.h, v2.h }[7], [x15], #6
+# CHECK-NEXT:  4      4     1.00           *            st3	{ v9.s, v10.s, v11.s }[2], [x20]
+# CHECK-NEXT:  4      4     1.00           *            st3	{ v16.d, v17.d, v18.d }[0], [x13]
+# CHECK-NEXT:  6      6     1.50           *            st4	{ v17.8b, v18.8b, v19.8b, v20.8b }, [x8]
+# CHECK-NEXT:  12     7     3.00           *            st4	{ v7.16b, v8.16b, v9.16b, v10.16b }, [x15]
+# CHECK-NEXT:  6      6     1.50           *            st4	{ v5.4h, v6.4h, v7.4h, v8.4h }, [x13]
+# CHECK-NEXT:  12     7     3.00           *            st4	{ v11.8h, v12.8h, v13.8h, v14.8h }, [x1]
 # CHECK-NEXT:  6      6     1.50           *            st4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
 # CHECK-NEXT:  13     7     3.00           *            st4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+# CHECK-NEXT:  12     7     3.00           *            st4	{ v21.4s, v22.4s, v23.4s, v24.4s }, [x6]
+# CHECK-NEXT:  8      4     2.00           *            st4	{ v25.2d, v26.2d, v27.2d, v28.2d }, [x16]
 # CHECK-NEXT:  6      6     1.00           *            st4	{ v0.b, v1.b, v2.b, v3.b }[15], [x0]
+# CHECK-NEXT:  6      6     1.00           *            st4	{ v5.h, v6.h, v7.h, v8.h }[4], [x13]
+# CHECK-NEXT:  6      6     1.00           *            st4	{ v22.s, v23.s, v24.s, v25.s }[0], [x7]
+# CHECK-NEXT:  4      4     1.00           *            st4	{ v23.d, v24.d, v25.d, v26.d }[1], [x5]
 # CHECK-NEXT:  5      4     1.00           *            st4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0], x5
 # CHECK-NEXT:  1      2     0.25                        sub	d15, d5, d16
 # CHECK-NEXT:  1      2     0.25                        sub	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      2     0.25                        sub	v15.2s, v14.2s, v11.2s
+# CHECK-NEXT:  1      2     0.25                        subhn	v7.4h, v10.4s, v13.4s
+# CHECK-NEXT:  1      2     0.25                        subhn2	v24.4s, v24.2d, v8.2d
 # CHECK-NEXT:  1      3     0.25                        sudot	v0.2s, v0.8b, v0.4b[2]
 # CHECK-NEXT:  1      3     0.25                        sudot	v0.4s, v0.16b, v0.4b[2]
 # CHECK-NEXT:  1      2     0.25                        suqadd	b19, b14
@@ -2168,6 +2830,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        trn2	v0.8b, v0.8b, v0.8b
 # CHECK-NEXT:  1      2     0.25                        trn2	v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.50                        uaba	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        uaba	v13.16b, v14.16b, v19.16b
 # CHECK-NEXT:  1      4     0.50                        uabal	v0.2d, v0.2s, v0.2s
 # CHECK-NEXT:  1      4     0.50                        uabal	v0.4s, v0.4h, v0.4h
 # CHECK-NEXT:  1      4     0.50                        uabal	v0.8h, v0.8b, v0.8b
@@ -2175,6 +2838,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.50                        uabal2	v0.4s, v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.50                        uabal2	v0.8h, v0.16b, v0.16b
 # CHECK-NEXT:  1      2     0.25                        uabd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      2     0.25                        uabd	v23.4s, v4.4s, v30.4s
 # CHECK-NEXT:  1      2     0.25                        uabdl	v0.2d, v0.2s, v0.2s
 # CHECK-NEXT:  1      2     0.25                        uabdl	v0.4s, v0.4h, v0.4h
 # CHECK-NEXT:  1      2     0.25                        uabdl	v0.8h, v0.8b, v0.8b
@@ -2210,8 +2874,12 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        uaddw2	v0.2d, v0.2d, v0.4s
 # CHECK-NEXT:  1      2     0.25                        uaddw2	v0.4s, v0.4s, v0.8h
 # CHECK-NEXT:  1      2     0.25                        uaddw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  1      3     1.00                        ucvtf	h17, x12
+# CHECK-NEXT:  4      6     1.00                        ucvtf	h22, h16, #11
+# CHECK-NEXT:  4      6     1.00                        ucvtf	h7, h21
 # CHECK-NEXT:  1      3     0.50                        ucvtf	d21, d14
 # CHECK-NEXT:  1      3     0.50                        ucvtf	d21, d14, #64
+# CHECK-NEXT:  1      3     1.00                        ucvtf	s8, x0
 # CHECK-NEXT:  2      4     1.00                        ucvtf	s22, s13
 # CHECK-NEXT:  2      4     1.00                        ucvtf	s22, s13, #32
 # CHECK-NEXT:  1      3     0.50                        ucvtf	v0.2d, v0.2d
@@ -2221,13 +2889,16 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  2      4     1.00                        ucvtf	v0.4h, v0.4h
 # CHECK-NEXT:  2      4     1.00                        ucvtf	v0.4s, v0.4s
 # CHECK-NEXT:  2      4     1.00                        ucvtf	v0.4s, v0.4s, #3
+# CHECK-NEXT:  2      4     1.00                        ucvtf	v18.4h, v11.4h, #7
 # CHECK-NEXT:  4      6     1.00                        ucvtf	v0.8h, v0.8h
+# CHECK-NEXT:  4      6     1.00                        ucvtf	v22.8h, v20.8h, #10
 # CHECK-NEXT:  1      3     0.25                        udot	v0.2s, v0.8b, v0.4b[2]
 # CHECK-NEXT:  1      3     0.25                        udot	v0.2s, v0.8b, v0.8b
 # CHECK-NEXT:  1      3     0.25                        udot	v0.4s, v0.16b, v0.16b
 # CHECK-NEXT:  1      3     0.25                        udot	v0.4s, v0.16b, v0.4b[2]
 # CHECK-NEXT:  1      2     0.25                        uhadd	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  1      2     0.25                        uhadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      2     0.25                        uhsub	v12.4h, v16.4h, v28.4h
 # CHECK-NEXT:  1      2     0.25                        uhsub	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT:  1      2     0.25                        umax	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  1      2     0.25                        umax	v0.4s, v0.4s, v0.4s
@@ -2243,7 +2914,9 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        umin	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  1      2     0.25                        umin	v0.4h, v0.4h, v0.4h
 # CHECK-NEXT:  1      2     0.25                        umin	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      2     0.25                        umin	v0.16b, v26.16b, v2.16b
 # CHECK-NEXT:  1      2     0.25                        uminp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      2     0.25                        uminp	v28.4s, v16.4s, v15.4s
 # CHECK-NEXT:  1      2     0.25                        uminp	v0.4h, v0.4h, v0.4h
 # CHECK-NEXT:  1      2     0.25                        uminp	v0.8b, v0.8b, v0.8b
 # CHECK-NEXT:  2      4     0.50                        uminv	b0, v0.8b
@@ -2252,35 +2925,51 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  2      4     0.50                        uminv	h0, v0.8h
 # CHECK-NEXT:  1      2     0.50                        uminv	s0, v0.4s
 # CHECK-NEXT:  1      4     0.50                        umlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        umlal	v28.2d, v31.2s, v0.s[1]
 # CHECK-NEXT:  1      4     0.50                        umlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        umlal	v22.4s, v14.4h, v0.h[6]
 # CHECK-NEXT:  1      4     0.50                        umlal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        umlal2	v10.2d, v4.4s, v3.s[2]
+# CHECK-NEXT:  1      4     0.50                        umlal2	v31.4s, v7.8h, v15.h[5]
 # CHECK-NEXT:  1      4     0.50                        umlal2	v0.2d, v0.4s, v0.4s
 # CHECK-NEXT:  1      4     0.50                        umlal2	v0.4s, v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.50                        umlal2	v0.8h, v0.16b, v0.16b
 # CHECK-NEXT:  1      4     0.50                        umlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        umlsl	v20.2d, v20.2s, v2.s[0]
 # CHECK-NEXT:  1      4     0.50                        umlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        umlsl	v21.4s, v12.4h, v7.h[5]
 # CHECK-NEXT:  1      4     0.50                        umlsl	v0.8h, v0.8b, v0.8b
 # CHECK-NEXT:  1      4     0.50                        umlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        umlsl2	v30.2d, v23.4s, v1.s[2]
 # CHECK-NEXT:  1      4     0.50                        umlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        umlsl2	v27.4s, v28.8h, v6.h[4]
 # CHECK-NEXT:  1      4     0.50                        umlsl2	v0.8h, v0.16b, v0.16b
 # CHECK-NEXT:  1      3     0.25                        ummla	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  1      2     0.25                        umov	w6, v22.b[0]
 # CHECK-NEXT:  1      2     0.25                        umov	w0, v0.b[1]
+# CHECK-NEXT:  1      2     0.25                        umov	w10, v25.h[0]
 # CHECK-NEXT:  1      2     0.25                        umov	w0, v0.h[1]
 # CHECK-NEXT:  1      2     0.25                        mov	w0, v0.s[1]
 # CHECK-NEXT:  1      2     0.25                        mov	x0, v0.d[1]
 # CHECK-NEXT:  1      3     0.50                        umull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     0.50                        umull	v22.2d, v28.2s, v6.s[1]
 # CHECK-NEXT:  1      3     0.50                        umull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     0.50                        umull	v27.4s, v1.4h, v8.h[6]
 # CHECK-NEXT:  1      3     0.50                        umull	v0.8h, v0.8b, v0.8b
 # CHECK-NEXT:  1      3     0.50                        umull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     0.50                        umull2	v28.2d, v21.4s, v1.s[0]
 # CHECK-NEXT:  1      3     0.50                        umull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     0.50                        umull2	v18.4s, v26.8h, v10.h[1]
 # CHECK-NEXT:  1      3     0.50                        umull2	v0.8h, v0.16b, v0.16b
 # CHECK-NEXT:  1      2     0.25                        uqadd	h0, h1, h5
+# CHECK-NEXT:  1      2     0.25                        uqadd	s0, s24, s30
 # CHECK-NEXT:  1      2     0.25                        uqadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      2     0.25                        uqadd	v14.2d, v22.2d, v20.2d
 # CHECK-NEXT:  1      4     0.50                        uqrshl	b11, b20, b30
 # CHECK-NEXT:  1      4     0.50                        uqrshl	s23, s20, s16
+# CHECK-NEXT:  1      4     0.50                        uqrshl	v25.8b, v13.8b, v23.8b
 # CHECK-NEXT:  1      4     0.50                        uqrshl	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  1      4     0.50                        uqrshl	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  1      4     0.50                        uqrshl	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT:  1      4     0.50                        uqrshl	v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        uqrshrn	b10, h12, #5
 # CHECK-NEXT:  1      2     0.25                        uqrshrn	h12, s10, #14
@@ -2303,6 +2992,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.50                        uqshl	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  1      4     0.50                        uqshl	v0.2s, v0.2s, #3
 # CHECK-NEXT:  1      4     0.50                        uqshl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      4     0.50                        uqshl	v8.4h, v17.4h, v13.4h
 # CHECK-NEXT:  1      4     0.50                        uqshl	v0.4s, v0.4s, #3
 # CHECK-NEXT:  1      4     0.50                        uqshl	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT:  1      4     0.50                        uqshl	v0.8b, v0.8b, #3
@@ -2317,8 +3007,10 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      4     0.50                        uqshrn2	v0.16b, v0.8h, #3
 # CHECK-NEXT:  1      4     0.50                        uqshrn2	v0.4s, v0.2d, #3
 # CHECK-NEXT:  1      4     0.50                        uqshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  1      2     0.25                        uqsub	s16, s21, s6
 # CHECK-NEXT:  1      2     0.25                        uqsub	d16, d16, d16
 # CHECK-NEXT:  1      2     0.25                        uqsub	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      2     0.25                        uqsub	v19.4s, v0.4s, v5.4s
 # CHECK-NEXT:  1      4     0.50                        uqxtn	b18, h18
 # CHECK-NEXT:  1      4     0.50                        uqxtn	h20, s17
 # CHECK-NEXT:  1      4     0.50                        uqxtn	s19, d14
@@ -2333,7 +3025,9 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.25                        urhadd	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  1      2     0.25                        urhadd	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT:  1      2     0.25                        urhadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      2     0.25                        urhadd	v16.2s, v19.2s, v2.2s
 # CHECK-NEXT:  1      4     0.50                        urshl	d8, d7, d4
+# CHECK-NEXT:  1      4     0.50                        urshl	v31.8b, v5.8b, v3.8b
 # CHECK-NEXT:  1      4     0.50                        urshl	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  1      4     0.50                        urshl	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  1      4     0.50                        urshl	v0.4s, v0.4s, v0.4s
@@ -2361,10 +3055,15 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      3     0.25                        usdot	v0.4s, v0.16b, v0.16b
 # CHECK-NEXT:  1      3     0.25                        usdot	v0.4s, v0.16b, v0.4b[2]
 # CHECK-NEXT:  1      2     0.50                        ushl	d0, d0, d0
+# CHECK-NEXT:  1      2     0.50                        ushl	v6.8b, v26.8b, v6.8b
 # CHECK-NEXT:  1      2     0.50                        ushl	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  1      2     0.50                        ushl	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT:  1      2     0.50                        ushl	v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.50                        ushll	v0.4s, v0.4h, #3
+# CHECK-NEXT:  1      2     0.50                        ushll	v18.8h, v24.8b, #4
+# CHECK-NEXT:  1      2     0.50                        ushll	v16.2d, v16.2s, #31
+# CHECK-NEXT:  1      2     0.50                        ushll2	v31.2d, v12.4s, #11
+# CHECK-NEXT:  1      2     0.50                        ushll2	v18.4s, v22.8h, #13
 # CHECK-NEXT:  1      2     0.50                        ushll2	v0.8h, v0.16b, #3
 # CHECK-NEXT:  1      2     0.25                        ushr	d10, d17, #18
 # CHECK-NEXT:  1      2     0.50                        ushr	v0.16b, v0.16b, #3
@@ -2375,6 +3074,11 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  1      2     0.50                        ushr	v0.8b, v0.8b, #3
 # CHECK-NEXT:  1      2     0.50                        ushr	v0.8h, v0.8h, #3
 # CHECK-NEXT:  1      3     0.25                        usmmla	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  1      2     0.25                  U     smov	w15, v22.b[0]
+# CHECK-NEXT:  1      2     0.25                  U     smov	w26, v27.h[0]
+# CHECK-NEXT:  1      2     0.25                  U     smov	x21, v0.b[0]
+# CHECK-NEXT:  1      2     0.25                  U     smov	x9, v27.h[0]
+# CHECK-NEXT:  1      2     0.25                  U     smov	x15, v3.s[0]
 # CHECK-NEXT:  1      2     0.25                        smov	w0, v0.b[1]
 # CHECK-NEXT:  1      2     0.25                        smov	w0, v0.h[1]
 # CHECK-NEXT:  1      2     0.25                        smov	x0, v0.b[1]
@@ -2468,7 +3172,7 @@ zip2 v0.8h, v0.8h, v0.8h
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -      -      -      -      -      -      -     26.67  49.17  49.17  18.75  7.75   7.75   7.75   431.00 356.50 385.50 311.00
+# CHECK-NEXT:  -      -      -      -      -      -      -     84.33  167.83 167.83 20.75  7.75   7.75   7.75   606.25 497.25 499.75 390.75
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
@@ -2488,8 +3192,10 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn2	v0.16b, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn2	v0.4s, v0.2d, v0.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	v7.2s, v1.2s, v2.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   addp	d1, v14.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   addv	s0, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   addv	h0, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   addv	h0, v0.8h
@@ -2503,6 +3209,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     bfcvt	h0, s0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     bfcvtn	v0.4h, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     bfcvtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfdot	v0.2s, v24.4h, v14.2h[2]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfdot	v0.2s, v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfdot	v0.4s, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalb	v0.4s, v0.8h, v0.8h
@@ -2511,10 +3218,16 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmlalt	v0.4s, v0.8h, v0.h[3]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bfmmla	v0.4s, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	v0.4h, #15, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	v23.8h, #101
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	v25.16b, v10.16b, v9.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bic	v24.2s, #70
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bit	v5.8b, v12.8b, v22.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bif	v0.8b, v25.8b, v4.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bif	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bit	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bsl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   bsl	v27.16b, v13.16b, v21.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cls	v0.4h, v0.4h
@@ -2527,28 +3240,37 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   clz	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	v9.8h, v16.8h, v24.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	v14.4h, v18.4h, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	d20, d21, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	d20, d21, d22
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	v0.16b, v0.16b, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmeq	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	v22.8h, v16.8h, v3.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	v22.16b, v30.16b, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	d20, d21, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	d20, d21, d22
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	v0.4h, v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmge	v0.8b, v0.8b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	v3.2d, v29.2d, v11.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	d20, d21, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	d20, d21, d22
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	v0.2s, v0.2s, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmgt	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhi	v28.4h, v25.4h, v21.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhi	d20, d21, d22
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhi	v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhs	d20, d21, d22
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmhs	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmle	v21.2s, v19.2s, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmle	d20, d21, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmle	v0.2d, v0.2d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmlt	v26.4h, v12.4h, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmlt	d20, d21, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmlt	v0.8h, v0.8h, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmtst	d20, d21, d22
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmtst	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cmtst	v13.2d, v13.2d, v13.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnt	v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   cnt	v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     dup	v0.16b, w28
@@ -2574,7 +3296,10 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ext	v0.8b, v0.8b, v0.8b, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	d29, d24, d20
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	s29, s24, s20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	h27, h20, h17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	v13.8h, v28.8h, v12.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	h25, h7
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.2d, v0.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.4h, v0.4h
@@ -2582,41 +3307,71 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fabs	v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facge	d20, d21, d22
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facge	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facge	h24, h26, h29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facge	v25.4h, v16.4h, v11.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facge	v19.2s, v24.2s, v5.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facge	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facgt	d20, d21, d22
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facgt	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facgt	h0, h4, h10
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facgt	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facgt	v22.8h, v14.8h, v31.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   facgt	v22.4s, v8.4s, v2.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fadd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	h10, v19.2h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	d11, v28.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   faddp	v16.2d, v11.2d, v5.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	v0.2s, v0.2s, v0.2s, #90
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcadd	v0.4s, v0.4s, v0.4s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	h30, h6, h1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	h19, h23, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	d20, d21, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	d20, d21, d22
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	s10, s11, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	s10, s11, s12
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	v0.2s, v0.2s, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	v12.4s, v11.4s, v26.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmeq	v18.2d, v17.2d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	h10, h23, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	h1, h16, h12
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	d20, d21, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	d20, d21, d22
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	s10, s11, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	s10, s11, s12
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	v0.2d, v0.2d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	v17.2d, v11.2d, v13.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	v18.4h, v27.4h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	v20.8h, v19.8h, v22.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmge	v17.2s, v11.2s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	h4, h5, h0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	h0, h18, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	d20, d21, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	d20, d21, d22
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	s10, s11, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	s10, s11, s12
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	v0.4s, v0.4s, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	v24.8h, v24.8h, v28.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	v0.8h, v11.8h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmgt	v19.2d, v31.2d, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	v0.2s, v0.2s, v0.2s, #90
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmla	v0.4s, v0.4s, v0.s[1], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmle	v16.8h, v11.8h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmle	v22.4s, v30.4s, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmle	d20, d21, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmle	s10, s11, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmle	v0.2d, v0.2d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmle	h18, h28, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmlt	h23, h7, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmlt	d20, d21, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmlt	s10, s11, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmlt	v0.4s, v0.4s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmlt	v8.4h, v2.4h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fcmlt	v7.2d, v16.2d, #0.0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtas	d21, d14
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtas	s12, s13
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtas	h12, h13
@@ -2703,9 +3458,11 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzs	v0.2s, v0.2s, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v20.4h, v24.4h, #11
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v0.4s, v0.4s, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzs	v18.8h, v10.8h, #7
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	d21, d12, #1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	d21, d14
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	s12, s13
@@ -2717,9 +3474,11 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     fcvtzu	v0.2s, v0.2s, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v19.4h, v26.4h, #9
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v0.4s, v0.4s, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     fcvtzu	v27.8h, v6.8h, #11
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fdiv	v0.4h, v0.4h, v0.4h
@@ -2731,12 +3490,19 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnm	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	h25, v19.2h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	d17, v29.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxnmp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fmaxnmv	h0, v13.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   fmaxnmv	h12, v11.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fmaxnmv	s28, v31.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	h15, v25.2h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmaxp	s6, v2.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fmaxv	h0, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   fmaxv	h0, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fmaxv	s0, v0.4s
@@ -2746,15 +3512,31 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnm	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	h20, v14.2h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	d15, v8.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminnmp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fminnmv	h19, v25.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   fminnmv	h23, v17.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fminnmv	s29, v17.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	h7, v10.2h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fminp	s17, v7.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fminv	h3, v30.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   fminv	h29, v12.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   fminv	s16, v19.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	d0, d1, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	h23, h24, v15.h[4]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	s0, s1, v0.s[3]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	v29.8h, v15.8h, v10.h[4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	v2.2s, v16.2s, v28.s[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	v14.4s, v14.4s, v5.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	v1.4s, v24.4s, v12.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmla	v10.2d, v14.2d, v21.d[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.2s, v0.2h, v0.h[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.4s, v0.4h, v0.h[3]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal	v0.2s, v0.2h, v0.2h
@@ -2764,8 +3546,14 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal2	v0.2s, v0.2h, v0.2h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlal2	v0.4s, v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	d0, d4, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	h8, h14, v7.h[4]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	s3, s5, v0.s[3]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	v30.8h, v18.8h, v4.h[6]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	v10.2s, v27.2s, v0.s[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	v27.4s, v7.4s, v24.s[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	v10.2d, v22.2d, v29.d[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmls	v6.8h, v15.8h, v23.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.2s, v0.2h, v0.h[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.4s, v0.4h, v0.h[3]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmlsl	v0.2s, v0.2h, v0.2h
@@ -2777,21 +3565,34 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	v0.2d, #-1.25000000
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	v0.2s, #13.00000000
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmov	v0.4s, #1.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	h18, h4, v7.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	v10.4h, v2.4h, v7.h[5]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	v5.2s, v12.2s, v9.s[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	v15.4s, v30.4s, v2.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	v11.2d, v31.2d, v24.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	h28, h14, h3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	d0, d1, v0.d[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	s0, s1, v0.s[3]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	d0, d4, v0.d[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	d23, d11, d1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	s20, s22, s15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	h18, h17, v7.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	h20, h25, h0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	s3, s5, v0.s[3]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v28.4h, v25.4h, v15.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v3.2s, v22.2s, v23.s[3]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v5.4s, v28.4s, v15.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulx	v22.2d, v18.2d, v25.d[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.2d, v0.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fneg	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	h20, h8
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	d13, d13
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	s19, s14
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	v0.2d, v0.2d
@@ -2799,9 +3600,13 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpe	v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frecpe	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	h29, h19, h8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpx	h18, h11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	v12.8h, v25.8h, v4.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	d22, d30, d21
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	s21, s16, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frecps	v7.2d, v29.2d, v18.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpx	d16, d19
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frecpx	s18, s10
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frinta	v0.2d, v0.2d
@@ -2839,13 +3644,17 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintz	v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintz	v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frintz	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	h23, h26
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	d21, d12
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	s22, s13
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	v0.2d, v0.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     frsqrte	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	v20.4s, v26.4s, v27.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	v8.4h, v9.4h, v30.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     frsqrte	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	h28, h26, h1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	d8, d22, d18
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	s21, s5, s12
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   frsqrts	v0.2d, v0.2d, v0.2d
@@ -2854,53 +3663,138 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50    -     3.50    -     fsqrt	v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     5.50    -     5.50    -     fsqrt	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	v13.8h, v15.8h, v17.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fsub	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ld1	{ v0.16b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ld1	{ v0.16b, v1.16b }, [x14]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -      -      -     ld1	{ v19.16b, v20.16b, v21.16b }, [x10]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     ld1	{ v13.16b, v14.16b, v15.16b, v16.16b }, [x9]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ld1	{ v24.8h }, [x27]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ld1	{ v1.8h, v2.8h }, [x27]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ld1	{ v0.8h, v1.8h }, [sp], #32
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -      -      -     ld1	{ v21.8h, v22.8h, v23.8h }, [x22]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     ld1	{ v0.8h, v1.8h, v2.8h, v3.8h }, [x21]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ld1	{ v3.4s }, [x4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ld1	{ v11.4s, v12.4s }, [x30]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -      -      -     ld1	{ v0.4s, v1.4s, v2.4s }, [x24]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     ld1	{ v15.4s, v16.4s, v17.4s, v18.4s }, [x28]
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.25   0.25   0.25   0.25    -      -      -      -     ld1	{ v0.4s, v1.4s, v2.4s }, [x0], #48
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ld1	{ v3.2d }, [x28]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ld1	{ v13.2d, v14.2d }, [x13]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -      -      -     ld1	{ v12.2d, v13.2d, v14.2d }, [x15]
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     ld1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25    -      -      -      -     ld1	{ v0.1d }, [x15], x2
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ld1	{ v27.1d, v28.1d }, [x7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -      -      -     ld1	{ v14.1d, v15.1d, v16.1d }, [x3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ld1	{ v22.1d, v23.1d, v24.1d, v25.1d }, [x4]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ld1	{ v0.2s, v1.2s }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -      -      -     ld1	{ v16.2s, v17.2s, v18.2s }, [x27]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ld1	{ v21.2s, v22.2s, v23.2s, v24.2s }, [x21]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ld1	{ v25.4h, v26.4h }, [x3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ld1	{ v20.4h, v21.4h, v22.4h, v23.4h }, [x15]
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -      -      -     ld1	{ v0.4h, v1.4h, v2.4h }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ld1	{ v24.8b, v25.8b }, [x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -      -      -     ld1	{ v7.8b, v8.8b, v9.8b }, [x12]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     ld1	{ v4.8b, v5.8b, v6.8b, v7.8b }, [x13]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25    -      -      -      -     ld1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1	{ v0.b }[7], [x0]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.25   0.25   0.25   0.25   ld1	{ v0.h }[3], [x0], #2
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1	{ v18.h }[3], [x1]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1	{ v0.s }[1], [x15]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.25   0.25   0.25   0.25   ld1	{ v0.d }[0], [x15], #8
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1	{ v11.d }[0], [x13]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1r	{ v0.16b }, [x0]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.25   0.25   0.25   0.25   ld1r	{ v0.8h }, [x0], #2
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1r	{ v0.4s }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1r	{ v3.1d }, [x15]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.25   0.25   0.25   0.25   ld1r	{ v0.2d }, [x15], x16
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1r	{ v18.2d }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1r	{ v8.8b }, [x23]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1r	{ v28.4h }, [x9]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1r	{ v3.8h }, [x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1r	{ v10.2s }, [x20]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v0.4h, v1.4h }, [x21]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v8.8h, v9.8h }, [x28]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v2.2s, v3.2s }, [x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v22.4s, v23.4s }, [x4]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v22.2d, v23.2d }, [x17]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v29.b, v30.b }[3], [x1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v26.s, v27.s }[1], [x17]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v1.d, v2.d }[0], [x10]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v0.16b, v1.16b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v13.8b, v14.8b }, [x4]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.50   0.50   0.50   0.50   ld2	{ v0.8b, v1.8b }, [x0], #16
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2	{ v0.h, v1.h }[7], [x15]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.50   0.50   0.50   0.50   ld2	{ v0.h, v1.h }[7], [x15], x8
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2r	{ v0.8b, v1.8b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2r	{ v10.16b, v11.16b }, [x23]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.50   0.50   0.50   0.50   ld2r	{ v0.4h, v1.4h }, [x0], #4
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2r	{ v25.4h, v26.4h }, [x11]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2r	{ v23.8h, v24.8h }, [x10]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2r	{ v0.2s, v1.2s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2r	{ v8.4s, v9.4s }, [x17]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33   0.25   0.25   0.25   0.25   0.50   0.50   0.50   0.50   ld2r	{ v0.1d, v1.1d }, [sp], x8
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2r	{ v9.1d, v10.1d }, [x25]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   ld2r	{ v26.2d, v27.2d }, [x8]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v8.8b, v9.8b, v10.8b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v15.16b, v16.16b, v17.16b }, [x5]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v0.4h, v1.4h, v2.4h }, [x15]
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.25   0.25   0.25   0.25   0.75   0.75   0.75   0.75   ld3	{ v0.8h, v1.8h, v2.8h }, [x15], #48
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v7.8h, v8.8h, v9.8h }, [x21]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v16.2s, v17.2s, v18.2s }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v12.4s, v13.4s, v14.4s }, [x25]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v17.b, v18.b, v19.b }[2], [x27]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v18.h, v19.h, v20.h }[5], [x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v10.2d, v11.2d, v12.2d }, [x18]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v0.s, v1.s, v2.s }[3], [sp]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25   0.75   0.75   0.75   0.75   ld3	{ v0.s, v1.s, v2.s }[3], [sp], x3
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3	{ v5.d, v6.d, v7.d }[1], [x14]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3r	{ v0.8b, v1.8b, v2.8b }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3r	{ v17.16b, v18.16b, v19.16b }, [x3]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25   0.75   0.75   0.75   0.75   ld3r	{ v0.4h, v1.4h, v2.4h }, [x15], #6
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3r	{ v3.4h, v4.4h, v5.4h }, [x1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3r	{ v6.8h, v7.8h, v8.8h }, [x28]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3r	{ v0.2s, v1.2s, v2.2s }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3r	{ v28.4s, v29.4s, v30.4s }, [x2]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67   0.25   0.25   0.25   0.25   0.75   0.75   0.75   0.75   ld3r	{ v0.1d, v1.1d, v2.1d }, [x0], x0
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3r	{ v1.1d, v2.1d, v3.1d }, [x28]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.75   0.75   0.75   0.75   ld3r	{ v8.2d, v9.2d, v10.2d }, [x3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v6.8b, v7.8b, v8.8b, v9.8b }, [x27]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v11.16b, v12.16b, v13.16b, v14.16b }, [x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v21.4h, v22.4h, v23.4h, v24.4h }, [x14]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v9.8h, v10.8h, v11.8h, v12.8h }, [x1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v17.4s, v18.4s, v19.4s, v20.4s }, [x4]
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33   0.25   0.25   0.25   0.25   1.00   1.00   1.00   1.00   ld4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v2.2d, v3.2d, v4.2d, v5.2d }, [x24]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v4.b, v5.b, v6.b, v7.b }[12], [x27]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v5.h, v6.h, v7.h, v8.h }[0], [x4]
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00   1.00   1.00   ld4	{ v0.h, v1.h, v2.h, v3.h }[7], [x0], x0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4	{ v0.s, v1.s, v2.s, v3.s }[0], [x26]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4r	{ v20.8b, v21.8b, v22.8b, v23.8b }, [x23]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x25]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4r	{ v16.4h, v17.4h, v18.4h, v19.4h }, [x6]
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp]
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4r	{ v0.2d, v1.2d, v2.2d, v3.2d }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4r	{ v4.8h, v5.8h, v6.8h, v7.8h }, [x23]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [x30]
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00   1.00   1.00   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp], #16
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     1.00   1.00   1.00   1.00   ld4r	{ v7.4s, v8.4s, v9.4s, v10.4s }, [x23]
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00   1.00   1.00   ld4r	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], x8
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mla	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mla	v15.8h, v22.8h, v4.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mla	v28.2s, v10.2s, v2.s[0]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mls	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mls	v25.8h, v29.8h, v0.h[4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mls	v22.2s, v29.2s, v0.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mls	v26.4s, v5.4s, v28.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	b0, v0.b[15]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	d6, v0.d[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	h2, v0.h[5]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	s17, v0.s[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	w8, v8.s[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	x30, v18.d[0]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.b[0], v0.b[0]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.h[1], v0.h[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v2.s[2], v0.s[2]
@@ -2913,11 +3807,24 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	d15, #0xff00ff00ff00ff
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.16b, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v14.8h, #174
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v13.4h, #74, lsl #8
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.2d, #0xff0000ff0000ffff
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.2s, #8, msl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v19.2s, #226
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v1.4s, #122, msl #8
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.4s, #255, lsl #24
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movi	v0.8b, #255
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	v26.4h, v20.4h, v14.h[5]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	v5.8h, v21.8h, v3.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	v29.2s, v10.2s, v3.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	v30.4s, v11.4s, v4.s[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	v30.4s, v11.4s, v4.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     mul	v3.8h, v9.8h, v8.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvni	v9.4h, #237
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvni	v8.8h, #171, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvni	v22.4s, #15, lsl #8
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvni	v0.2s, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvni	v0.4s, #16, msl #16
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   neg	d29, d24
@@ -2931,8 +3838,11 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvn	v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mvn	v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orn	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orn	v29.8b, v19.8b, v16.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	v9.4h, #18
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	v0.8h, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orr	v4.4s, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmul	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmul	v0.8b, v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmull	v0.8h, v0.8b, v0.8b
@@ -2977,6 +3887,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal2	v0.4s, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sabal2	v0.8h, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabd	v12.2s, v11.2s, v27.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl	v0.2d, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl	v0.4s, v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sabdl	v0.8h, v0.8b, v0.8b
@@ -3012,6 +3923,8 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw2	v0.2d, v0.2d, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw2	v0.4s, v0.4s, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   saddw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	h4, h8, #9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	h5, h14
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	d21, d12
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     scvtf	d21, d12, #64
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	s22, s13
@@ -3023,26 +3936,25 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v25.4h, v13.4h, #8
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     scvtf	v4.8h, v8.8h, #10
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.2s, v0.8b, v0.4b[2]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.2s, v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.4s, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sdot	v0.4s, v0.16b, v0.4b[2]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shadd	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shadd	v25.16b, v1.16b, v10.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shl	d7, d10, #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v23.8b, v18.8b, #6
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.16b, v0.16b, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.2d, v0.2d, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.8h, v23.8h, #10
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shl	v0.4s, v0.4s, #3
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.2d, v0.2s, #32
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.4s, v0.4h, #16
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.8h, v0.8b, #8
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.2d, v0.2s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.4s, v0.4h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll	v0.8h, v0.8b, #8
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.2d, v0.4s, #32
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.4s, v0.8h, #16
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.8h, v0.16b, #8
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.2d, v0.4s, #32
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.4s, v0.8h, #16
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shll2	v0.8h, v0.16b, #8
@@ -3054,6 +3966,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   shrn2	v0.8h, v0.4s, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   shsub	v15.8h, v5.8h, v27.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sli	d10, d14, #12
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.16b, v0.16b, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sli	v0.2d, v0.2d, #3
@@ -3065,8 +3978,10 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	v0.4h, v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smax	v30.16b, v3.16b, v30.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	v21.8h, v16.8h, v7.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smaxp	v0.8b, v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   smaxv	b0, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00   smaxv	b0, v0.16b
@@ -3085,23 +4000,35 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   sminv	h0, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sminv	s0, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal	v0.2d, v25.2s, v1.s[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal	v16.4s, v9.4h, v11.h[4]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal	v0.8h, v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal2	v30.2d, v22.4s, v7.s[2]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal2	v0.4s, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal2	v0.8h, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl	v25.2d, v27.2s, v1.s[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl	v14.4s, v23.4h, v12.h[7]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlal2	v1.4s, v9.8h, v0.h[6]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl2	v12.4s, v11.8h, v12.h[0]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl2	v11.2d, v28.4s, v7.s[2]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl2	v0.4s, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smlsl2	v0.8h, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smmla	v0.4s, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull	v31.2d, v23.2s, v6.s[2]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull	v3.4s, v26.4h, v1.h[5]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull	v0.8h, v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull2	v11.2d, v1.4s, v7.s[0]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull2	v13.4s, v18.8h, v0.h[3]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     smull2	v0.8h, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	b19, b14
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	d18, d12
@@ -3115,6 +4042,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqabs	v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	b20, b11, b15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	h12, h18, h10
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqadd	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	d19, s24, s12
@@ -3122,17 +4050,34 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	s0, h0, v0.h[3]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	s17, h27, h12
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	v11.2d, v24.2s, v0.s[3]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal	v20.4s, v30.4h, v12.h[3]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal2	v23.2d, v30.4s, v6.s[0]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlal2	v2.4s, v17.8h, v5.h[6]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	v8.4h, v16.4h, v5.h[4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	v16.2s, v24.2s, v7.s[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	v8.4s, v19.4h, v1.h[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull	v20.2d, v10.2s, v6.s[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull2	v10.4s, v25.8h, v0.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmull2	v4.2d, v29.4s, v2.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	v0.8h, v15.8h, v0.h[5]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	v6.2s, v29.2s, v4.s[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqrdmulh	v31.2s, v17.2s, v4.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	d12, s23, s13
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	d8, s9, v0.s[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	s0, h0, v0.h[3]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	s14, h12, h25
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	v26.2d, v7.2s, v3.s[0]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl	v4.4s, v22.4h, v13.h[2]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl2	v4.2d, v3.4s, v3.s[2]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmlsl2	v2.4s, v28.8h, v4.h[6]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	h10, h11, h12
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	h7, h15, v0.h[3]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     sqdmulh	s15, s14, v0.s[1]
@@ -3193,6 +4138,9 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	v0.4h, v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqrshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	s17, s4, s23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	b3, b13, b12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqsub	v20.8h, v18.8h, v12.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrshrn	b10, h13, #2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrshrn	h15, s10, #6
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sqrshrn	s15, d12, #9
@@ -3218,6 +4166,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	h3, h4, h15
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	s14, s17, #22
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v23.16b, v23.16b, v23.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.2d, v0.2d, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.2s, v0.2s, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sqshl	v0.2s, v0.2s, v0.2s
@@ -3314,8 +4263,12 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.4h, v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshll	v9.8h, v2.8b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshll	v12.4s, v3.4h, #4
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshll	v0.2d, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshll2	v28.8h, v12.16b, #7
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshll2	v0.4s, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshll2	v17.2d, v13.4s, #22
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sshr	d15, d16, #12
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.16b, v0.16b, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   sshr	v0.2d, v0.2d, #3
@@ -3344,30 +4297,91 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw2	v0.2d, v0.2d, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw2	v0.4s, v0.4s, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ssubw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v18.8b }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v8.8b, v9.8b }, [x18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st1	{ v15.8b, v16.8b, v17.8b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st1	{ v21.8b, v22.8b, v23.8b, v24.8b }, [x14]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v0.16b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st1	{ v1.16b, v2.16b }, [x4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     st1	{ v27.16b, v28.16b, v29.16b }, [x18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     st1	{ v18.16b, v19.16b, v20.16b, v21.16b }, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v19.4h }, [x7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v22.4h, v23.4h }, [x22]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st1	{ v13.4h, v14.4h, v15.4h }, [x7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st1	{ v23.4h, v24.4h, v25.4h, v26.4h }, [x24]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v27.8h }, [x17]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     st1	{ v8.8h, v9.8h, v10.8h }, [x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     st1	{ v7.8h, v8.8h, v9.8h, v10.8h }, [x19]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v25.2s }, [x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v13.2s, v14.2s }, [x9]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st1	{ v12.2s, v13.2s, v14.2s }, [x3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st1	{ v6.2s, v7.2s, v8.2s, v9.2s }, [x13]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00    -      -     st1	{ v0.4s, v1.4s }, [sp], #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v22.4s }, [x19]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st1	{ v15.4s, v16.4s }, [x12]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     st1	{ v26.4s, v27.4s, v28.4s, v29.4s }, [x12]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v20.1d }, [x10]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v21.1d, v22.1d }, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st1	{ v5.1d, v6.1d, v7.1d }, [x3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st1	{ v0.1d, v1.1d, v2.1d, v3.1d }, [x10]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st1	{ v26.2d, v27.2d }, [x28]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50   0.25   0.25   0.25   0.25   1.50   1.50    -      -     st1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     st1	{ v13.2d, v14.2d, v15.2d }, [x27]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     st1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v8.2d }, [x15]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     st1	{ v0.8h }, [x15], x2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st1	{ v0.8h, v1.8h }, [x15]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     st1	{ v0.4s, v1.4s, v2.4s }, [sp]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00    -      -     st1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v1.b }[5], [x1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v0.h }[2], [x1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v31.s }[1], [x16]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st1	{ v0.d }[1], [x0]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     st1	{ v0.d }[1], [x0], #8
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00    -      -     st2	{ v0.16b, v1.16b }, [x0], x1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st2	{ v0.8b, v1.8b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st2	{ v6.16b, v7.16b }, [x23]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st2	{ v10.4h, v11.4h }, [x18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st2	{ v10.8h, v11.8h }, [x18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st2	{ v25.2s, v26.2s }, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st2	{ v26.4s, v27.4s }, [x14]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st2	{ v10.2d, v11.2d }, [x1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st2	{ v21.b, v22.b }[15], [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st2	{ v28.h, v29.h }[2], [x6]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st2	{ v0.s, v1.s }[3], [sp]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50   0.25   0.25   0.25   0.25   0.50   0.50    -      -     st2	{ v0.s, v1.s }[3], [sp], #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -     st2	{ v17.d, v18.d }[1], [x1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st3	{ v10.8b, v11.8b, v12.8b }, [x18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     st3	{ v26.16b, v27.16b, v28.16b }, [x4]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st3	{ v0.4h, v1.4h, v2.4h }, [x15]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50   0.25   0.25   0.25   0.25   1.50   1.50    -      -     st3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     st3	{ v0.8h, v1.8h, v2.8h }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st3	{ v19.2s, v20.2s, v21.2s }, [x30]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     st3	{ v24.4s, v25.4s, v26.4s }, [x8]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     st3	{ v24.2d, v25.2d, v26.2d }, [x25]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st3	{ v8.b, v9.b, v10.b }[4], [x18]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st3	{ v0.h, v1.h, v2.h }[7], [x15]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00    -      -     st3	{ v0.h, v1.h, v2.h }[7], [x15], #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st3	{ v9.s, v10.s, v11.s }[2], [x20]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st3	{ v16.d, v17.d, v18.d }[0], [x13]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     st4	{ v17.8b, v18.8b, v19.8b, v20.8b }, [x8]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     3.00   3.00    -      -      -      -     3.00   3.00    -      -     st4	{ v7.16b, v8.16b, v9.16b, v10.16b }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     st4	{ v5.4h, v6.4h, v7.4h, v8.4h }, [x13]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     3.00   3.00    -      -      -      -     3.00   3.00    -      -     st4	{ v11.8h, v12.8h, v13.8h, v14.8h }, [x1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     1.50   1.50    -      -      -      -     1.50   1.50    -      -     st4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     3.00   3.00   0.25   0.25   0.25   0.25   3.00   3.00    -      -     st4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     3.00   3.00    -      -      -      -     3.00   3.00    -      -     st4	{ v21.4s, v22.4s, v23.4s, v24.4s }, [x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00    -      -      -      -     2.00   2.00    -      -     st4	{ v25.2d, v26.2d, v27.2d, v28.2d }, [x16]
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     0.75   0.75   0.75   0.75   st4	{ v0.b, v1.b, v2.b, v3.b }[15], [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     0.75   0.75   0.75   0.75   st4	{ v5.h, v6.h, v7.h, v8.h }[4], [x13]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -     0.75   0.75   0.75   0.75   st4	{ v22.s, v23.s, v24.s, v25.s }[0], [x7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -      -     st4	{ v23.d, v24.d, v25.d, v26.d }[1], [x5]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00   0.25   0.25   0.25   0.25   1.00   1.00    -      -     st4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0], x5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	d15, d5, d16
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sub	v15.2s, v14.2s, v11.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subhn	v7.4h, v10.4s, v13.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   subhn2	v24.4s, v24.2d, v8.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sudot	v0.2s, v0.8b, v0.4b[2]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   sudot	v0.4s, v0.16b, v0.4b[2]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   suqadd	b19, b14
@@ -3412,6 +4426,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.8b, v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   trn2	v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaba	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uaba	v13.16b, v14.16b, v19.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal	v0.2d, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal	v0.4s, v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal	v0.8h, v0.8b, v0.8b
@@ -3419,6 +4434,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal2	v0.4s, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uabal2	v0.8h, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabd	v23.4s, v4.4s, v30.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl	v0.2d, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl	v0.4s, v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uabdl	v0.8h, v0.8b, v0.8b
@@ -3454,8 +4470,12 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw2	v0.2d, v0.2d, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw2	v0.4s, v0.4s, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uaddw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	h17, x12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	h22, h16, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	h7, h21
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	d21, d14
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	d21, d14, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ucvtf	s8, x0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	s22, s13
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	s22, s13, #32
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     ucvtf	v0.2d, v0.2d
@@ -3465,13 +4485,16 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v18.4h, v11.4h, #7
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -     ucvtf	v22.8h, v20.8h, #10
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.2s, v0.8b, v0.4b[2]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.2s, v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.4s, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   udot	v0.4s, v0.16b, v0.4b[2]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhadd	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsub	v12.4h, v16.4h, v28.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uhsub	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umax	v0.4s, v0.4s, v0.4s
@@ -3487,7 +4510,9 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	v0.2s, v0.2s, v0.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	v0.4h, v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umin	v0.16b, v26.16b, v2.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	v28.4s, v16.4s, v15.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	v0.4h, v0.4h, v0.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uminp	v0.8b, v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   uminv	b0, v0.8b
@@ -3496,35 +4521,51 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.75   0.25   0.75   uminv	h0, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uminv	s0, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal	v28.2d, v31.2s, v0.s[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal	v22.4s, v14.4h, v0.h[6]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal2	v10.2d, v4.4s, v3.s[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal2	v31.4s, v7.8h, v15.h[5]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal2	v0.2d, v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal2	v0.4s, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlal2	v0.8h, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl	v20.2d, v20.2s, v2.s[0]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl	v21.4s, v12.4h, v7.h[5]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl	v0.8h, v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl2	v30.2d, v23.4s, v1.s[2]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl2	v27.4s, v28.8h, v6.h[4]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umlsl2	v0.8h, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ummla	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umov	w6, v22.b[0]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umov	w0, v0.b[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umov	w10, v25.h[0]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   umov	w0, v0.h[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	w0, v0.s[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   mov	x0, v0.d[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull	v22.2d, v28.2s, v6.s[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull	v27.4s, v1.4h, v8.h[6]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull	v0.8h, v0.8b, v0.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull2	v28.2d, v21.4s, v1.s[0]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull2	v18.4s, v26.8h, v10.h[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50    -     umull2	v0.8h, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	h0, h1, h5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	s0, s24, s30
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqadd	v14.2d, v22.2d, v20.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	b11, b20, b30
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	s23, s20, s16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v25.8b, v13.8b, v23.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.4s, v0.4s, v0.4s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqrshl	v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqrshrn	b10, h12, #5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqrshrn	h12, s10, #14
@@ -3547,6 +4588,7 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.2s, v0.2s, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v8.4h, v17.4h, v13.4h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.4s, v0.4s, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshl	v0.8b, v0.8b, #3
@@ -3561,8 +4603,10 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn2	v0.16b, v0.8h, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn2	v0.4s, v0.2d, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	s16, s21, s6
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	d16, d16, d16
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   uqsub	v19.4s, v0.4s, v5.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	b18, h18
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	h20, s17
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   uqxtn	s19, d14
@@ -3577,7 +4621,9 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   urhadd	v16.2s, v19.2s, v2.2s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	d8, d7, d4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v31.8b, v5.8b, v3.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.2d, v0.2d, v0.2d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   urshl	v0.4s, v0.4s, v0.4s
@@ -3605,10 +4651,15 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	v0.4s, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usdot	v0.4s, v0.16b, v0.4b[2]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	d0, d0, d0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	v6.8b, v26.8b, v6.8b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	v0.16b, v0.16b, v0.16b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	v0.4s, v0.4s, v0.4s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushl	v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushll	v0.4s, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushll	v18.8h, v24.8b, #4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushll	v16.2d, v16.2s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushll2	v31.2d, v12.4s, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushll2	v18.4s, v22.8h, #13
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushll2	v0.8h, v0.16b, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   ushr	d10, d17, #18
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.16b, v0.16b, #3
@@ -3619,6 +4670,11 @@ zip2 v0.8h, v0.8h, v0.8h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.8b, v0.8b, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50    -     0.50   ushr	v0.8h, v0.8h, #3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   usmmla	v0.4s, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smov	w15, v22.b[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smov	w26, v27.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smov	x21, v0.b[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smov	x9, v27.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smov	x15, v3.s[0]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smov	w0, v0.b[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smov	w0, v0.h[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   smov	x0, v0.b[1]
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-sve-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-sve-instructions.s
index 6b7616d68d4cd..bcbc5eecd924b 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-sve-instructions.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-sve-instructions.s
@@ -426,9 +426,12 @@ decp	xzr, p15.b
 decp	xzr, p15.d
 decp	xzr, p15.h
 decp	xzr, p15.s
+decd    z19.d
 decp	z31.d, p15.d
+dech    z23.h
 decp	z31.h, p15.h
 decp	z31.s, p15.s
+decw    z8.s
 decw	x0
 decw	x0, #14
 decw	x0, all, mul #16
@@ -1005,6 +1008,7 @@ ld1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
 ld1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
 ld1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
 ld1sb	{ z21.s }, p5/z, [x10, x21]
+ld1sb   { z23.s }, p5/z, [x17, z10.s, uxtw]
 ld1sb	{ z23.d }, p3/z, [x13, x8]
 ld1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
 ld1sb	{ z31.d }, p7/z, [sp, z31.d]
@@ -1368,8 +1372,17 @@ lsrr	z0.b, p0/m, z0.b, z0.b
 lsrr	z0.d, p0/m, z0.d, z0.d
 lsrr	z0.h, p0/m, z0.h, z0.h
 lsrr	z0.s, p0/m, z0.s, z0.s
+mad     z17.b, p7/m, z4.b, z5.b
+mad     z29.h, p4/m, z31.h, z18.h
+mad     z7.s, p4/m, z5.s, z29.s
 mad	z0.d, p0/m, z0.d, z0.d
+mla     z1.b, p0/m, z3.b, z3.b
+mla     z21.h, p2/m, z31.h, z30.h
+mla     z24.s, p3/m, z11.s, z9.s
 mla	z0.d, p0/m, z0.d, z0.d
+mls     z11.b, p1/m, z28.b, z6.b
+mls     z31.h, p0/m, z25.h, z24.h
+mls     z1.s, p5/m, z7.s, z13.s
 mls	z0.d, p0/m, z0.d, z0.d
 mov	p0.b, p0.b
 mov	p0.b, p0/m, p0.b
@@ -1444,7 +1457,8 @@ mov	z31.b, p15/m, z31.b
 mov	z31.b, p7/m, b31
 movprfx z31, z6
 mov	z31.b, p7/m, wsp
-mov	z31.b, wsp
+movprfx z31.b, p0/m, z4.b
+mov	z31.b, p0/m, wsp
 mov	z31.b, z31.b[63]
 mov	z31.d, p15/m, z31.d
 mov	z31.d, p7/m, d31
@@ -1485,6 +1499,9 @@ mrs	x3, ZCR_EL2
 mrs	x3, ZCR_EL3
 msr	ZCR_EL1, x3
 msb	z0.d, p0/m, z0.d, z0.d
+msb     z18.b, p1/m, z27.b, z0.b
+msb     z27.h, p5/m, z23.h, z1.h
+msb     z26.s, p2/m, z0.s, z2.s
 msr	ZCR_EL12, x3
 msr	ZCR_EL2, x3
 msr	ZCR_EL3, x3
@@ -1555,6 +1572,36 @@ pnext	p0.d, p15, p0.d
 pnext	p0.h, p15, p0.h
 pnext	p0.s, p15, p0.s
 pnext	p15.b, p15, p15.b
+prfb #14, p5, [x21]
+prfb pldl1keep, p7, [x4, x9]
+prfb pldl3strm, p4, [x3, z15.s, uxtw]
+prfb pldl1strm, p7, [x28, z4.d, uxtw]
+prfb pstl3keep, p2, [x18, z19.d]
+prfb pstl3keep, p1, [z28.s]
+prfb pstl2strm, p5, [z25.d]
+prfd pstl3strm, p3, [x21]
+prfd pstl2keep, p3, [x24, x24, lsl #3]
+prfd pstl1strm, p3, [x27, z27.s, sxtw #3]
+prfd pstl1keep, p0, [x21, z2.d, uxtw #3]
+prfd pldl1strm, p7, [x22, z22.d, lsl #3]
+prfd pldl2strm, p1, [z2.s]
+prfd #15, p1, [z17.d]
+prfh pldl2strm, p3, [x17]
+prfh pstl2keep, p1, [x28, x9, lsl #1]
+prfh pldl1strm, p6, [x0, z10.s, uxtw #1]
+prfh pldl3keep, p7, [x24, z21.d, uxtw #1]
+prfh pstl1strm, p5, [x10, z6.d, lsl #1]
+prfh pldl3strm, p6, [z0.s]
+prfh pstl2keep, p2, [z21.d]
+prfm pldl1strm, [x5]
+prfm pldl1keep, [x25, x16]
+prfw pldl2strm, p2, [x4]
+prfw pstl1keep, p4, [x18, x21, lsl #2]
+prfw pldl2strm, p0, [x15, z6.s, uxtw #2]
+prfw pstl2keep, p0, [x27, z18.d, sxtw #2]
+prfw pstl2keep, p3, [x19, z8.d, lsl #2]
+prfw #7, p7, [z27.s]
+prfw #7, p1, [z20.d]
 ptest	p15, p0.b
 ptest	p15, p15.b
 ptrue	p0.b, pow2
@@ -1653,6 +1700,7 @@ rdvl	x0, #0
 rdvl	x21, #-32
 rdvl	x23, #31
 rdvl	xzr, #-1
+rev     p1.h, p2.h
 rev	z0.b, z31.b
 rev	z0.d, z31.d
 rev	z0.h, z31.h
@@ -1671,8 +1719,10 @@ saddv	d0, p7, z31.b
 saddv	d0, p7, z31.h
 saddv	d0, p7, z31.s
 scvtf	z0.d, p0/m, z0.d
+scvtf   z18.d, p3/m, z16.s
 scvtf	z0.h, p0/m, z0.h
 scvtf	z0.h, p0/m, z0.s
+scvtf   z18.h, p1/m, z14.d
 scvtf	z0.s, p0/m, z0.d
 scvtf	z0.s, p0/m, z0.s
 sdiv	z0.d, p7/m, z0.d, z31.d
@@ -1703,6 +1753,7 @@ smax	z31.s, z31.s, #127
 smaxv	b0, p7, z31.b
 smaxv	h0, p7, z31.h
 smaxv	s0, p7, z31.s
+smaxv   d24, p5, z24.d
 smin	z0.b, z0.b, #-128
 smin	z0.d, z0.d, #-128
 smin	z0.h, z0.h, #-128
@@ -1718,6 +1769,7 @@ smin	z31.s, z31.s, #127
 sminv	b0, p7, z31.b
 sminv	h0, p7, z31.h
 sminv	s0, p7, z31.s
+sminv   d17, p2, z18.d
 smmla	z0.s, z1.b, z2.b
 smulh	z0.b, p7/m, z0.b, z31.b
 smulh	z0.d, p7/m, z0.d, z31.d
@@ -2146,9 +2198,11 @@ uabd	z31.s, p7/m, z31.s, z31.s
 uaddv	d0, p7, z31.b
 uaddv	d0, p7, z31.h
 uaddv	d0, p7, z31.s
+uaddv   d28, p6, z6.d
 ucvtf	z0.d, p0/m, z0.d
 ucvtf	z0.h, p0/m, z0.h
 ucvtf	z0.h, p0/m, z0.s
+ucvtf   z30.h, p2/m, z24.d
 ucvtf	z0.s, p0/m, z0.d
 ucvtf	z0.s, p0/m, z0.s
 udiv	z0.d, p7/m, z0.d, z31.d
@@ -2157,6 +2211,7 @@ udivr	z0.d, p7/m, z0.d, z31.d
 udivr	z0.s, p7/m, z0.s, z31.s
 udot	z0.d, z1.h, z15.h[1]
 udot	z0.d, z1.h, z31.h
+ucvtf   z24.d, p5/m, z9.s
 udot	z0.s, z1.b, z31.b
 udot	z0.s, z1.b, z7.b[3]
 umax	z0.b, z0.b, #0
@@ -2168,15 +2223,18 @@ umax	z31.s, p7/m, z31.s, z31.s
 umaxv	b0, p7, z31.b
 umaxv	h0, p7, z31.h
 umaxv	s0, p7, z31.s
+umaxv   d11, p4, z11.d
 umin	z0.b, z0.b, #0
 umin	z31.b, p7/m, z31.b, z31.b
 umin	z31.b, z31.b, #255
 umin	z31.d, p7/m, z31.d, z31.d
 umin	z31.h, p7/m, z31.h, z31.h
 umin	z31.s, p7/m, z31.s, z31.s
+umin    z21.s, z21.s, #139
 uminv	b0, p7, z31.b
 uminv	h0, p7, z31.h
 uminv	s0, p7, z31.s
+uminv   d24, p5, z29.d
 ummla	z0.s, z1.b, z2.b
 umulh	z0.b, p7/m, z0.b, z31.b
 umulh	z0.d, p7/m, z0.d, z31.d
@@ -2368,7 +2426,10 @@ uzp2	z31.d, z31.d, z31.d
 uzp2	z31.h, z31.h, z31.h
 uzp2	z31.s, z31.s, z31.s
 whilele	p0.b, w30, wzr
+whilele p6.h, x28, x30
 whilelo	p15.d, xzr, x30
+whilelo p3.b, x9, x7
+whilels p4.b, w4, w20
 whilels	p0.h, w30, wzr
 whilelt	p15.s, xzr, x30
 wrffr	p0.b
@@ -2840,9 +2901,12 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  1      2     1.00                        decp	xzr, p15.d
 # CHECK-NEXT:  1      2     1.00                        decp	xzr, p15.h
 # CHECK-NEXT:  1      2     1.00                        decp	xzr, p15.s
+# CHECK-NEXT:  1      2     1.00                        decd	z19.d
 # CHECK-NEXT:  3      7     2.00                        decp	z31.d, p15.d
+# CHECK-NEXT:  1      2     1.00                        dech	z23.h
 # CHECK-NEXT:  3      7     2.00                        decp	z31.h, p15.h
 # CHECK-NEXT:  3      7     2.00                        decp	z31.s, p15.s
+# CHECK-NEXT:  1      2     1.00                        decw	z8.s
 # CHECK-NEXT:  1      2     1.00                        decw	x0
 # CHECK-NEXT:  1      2     1.00                        decw	x0, #14
 # CHECK-NEXT:  1      2     1.00                        decw	x0, all, mul #16
@@ -3419,6 +3483,7 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  1      6     0.50    *                   ld1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
 # CHECK-NEXT:  1      6     0.50    *                   ld1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
 # CHECK-NEXT:  1      6     0.50    *                   ld1sb	{ z21.s }, p5/z, [x10, x21]
+# CHECK-NEXT:  2      9     0.33    *                   ld1sb	{ z23.s }, p5/z, [x17, z10.s, uxtw]
 # CHECK-NEXT:  1      6     0.50    *                   ld1sb	{ z23.d }, p3/z, [x13, x8]
 # CHECK-NEXT:  1      6     0.50    *                   ld1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
 # CHECK-NEXT:  4      9     0.67    *                   ld1sb	{ z31.d }, p7/z, [sp, z31.d]
@@ -3782,8 +3847,17 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  1      2     1.00                        lsrr	z0.d, p0/m, z0.d, z0.d
 # CHECK-NEXT:  1      2     1.00                        lsrr	z0.h, p0/m, z0.h, z0.h
 # CHECK-NEXT:  1      2     1.00                        lsrr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      4     1.00                        mad	z17.b, p7/m, z4.b, z5.b
+# CHECK-NEXT:  1      4     1.00                        mad	z29.h, p4/m, z31.h, z18.h
+# CHECK-NEXT:  1      4     1.00                        mad	z7.s, p4/m, z5.s, z29.s
 # CHECK-NEXT:  2      5     2.00                        mad	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      4     1.00                        mla	z1.b, p0/m, z3.b, z3.b
+# CHECK-NEXT:  1      4     1.00                        mla	z21.h, p2/m, z31.h, z30.h
+# CHECK-NEXT:  1      4     1.00                        mla	z24.s, p3/m, z11.s, z9.s
 # CHECK-NEXT:  2      5     2.00                        mla	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      4     1.00                        mls	z11.b, p1/m, z28.b, z6.b
+# CHECK-NEXT:  1      4     1.00                        mls	z31.h, p0/m, z25.h, z24.h
+# CHECK-NEXT:  1      4     1.00                        mls	z1.s, p5/m, z7.s, z13.s
 # CHECK-NEXT:  2      5     2.00                        mls	z0.d, p0/m, z0.d, z0.d
 # CHECK-NEXT:  1      1     1.00                        mov	p0.b, p0.b
 # CHECK-NEXT:  1      1     1.00                        mov	p0.b, p0/m, p0.b
@@ -3858,7 +3932,8 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  1      2     0.50                        mov	z31.b, p7/m, b31
 # CHECK-NEXT:  1      2     0.50                        movprfx	z31, z6
 # CHECK-NEXT:  2      5     1.00                        mov	z31.b, p7/m, wsp
-# CHECK-NEXT:  1      3     1.00                        mov	z31.b, wsp
+# CHECK-NEXT:  1      2     0.50                        movprfx	z31.b, p0/m, z4.b
+# CHECK-NEXT:  2      5     1.00                        mov	z31.b, p0/m, wsp
 # CHECK-NEXT:  1      2     0.50                        mov	z31.b, z31.b[63]
 # CHECK-NEXT:  1      2     0.50                        mov	z31.d, p15/m, z31.d
 # CHECK-NEXT:  1      2     0.50                        mov	z31.d, p7/m, d31
@@ -3899,6 +3974,9 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  1      1     0.07                  U     mrs	x3, ZCR_EL3
 # CHECK-NEXT:  1      1     0.07                  U     msr	ZCR_EL1, x3
 # CHECK-NEXT:  2      5     2.00                        msb	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      4     1.00                        msb	z18.b, p1/m, z27.b, z0.b
+# CHECK-NEXT:  1      4     1.00                        msb	z27.h, p5/m, z23.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        msb	z26.s, p2/m, z0.s, z2.s
 # CHECK-NEXT:  1      1     0.07                  U     msr	ZCR_EL12, x3
 # CHECK-NEXT:  1      1     0.07                  U     msr	ZCR_EL2, x3
 # CHECK-NEXT:  1      1     0.07                  U     msr	ZCR_EL3, x3
@@ -3969,6 +4047,36 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  1      2     1.00                        pnext	p0.h, p15, p0.h
 # CHECK-NEXT:  1      2     1.00                        pnext	p0.s, p15, p0.s
 # CHECK-NEXT:  1      2     1.00                        pnext	p15.b, p15, p15.b
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfb	#14, p5, [x21]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfb	pldl1keep, p7, [x4, x9]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfb	pldl3strm, p4, [x3, z15.s, uxtw]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfb	pldl1strm, p7, [x28, z4.d, uxtw]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfb	pstl3keep, p2, [x18, z19.d]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfb	pstl3keep, p1, [z28.s]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfb	pstl2strm, p5, [z25.d]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfd	pstl3strm, p3, [x21]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfd	pstl2keep, p3, [x24, x24, lsl #3]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfd	pstl1strm, p3, [x27, z27.s, sxtw #3]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfd	pstl1keep, p0, [x21, z2.d, uxtw #3]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfd	pldl1strm, p7, [x22, z22.d, lsl #3]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfd	pldl2strm, p1, [z2.s]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfd	#15, p1, [z17.d]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfh	pldl2strm, p3, [x17]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfh	pstl2keep, p1, [x28, x9, lsl #1]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfh	pldl1strm, p6, [x0, z10.s, uxtw #1]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfh	pldl3keep, p7, [x24, z21.d, uxtw #1]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfh	pstl1strm, p5, [x10, z6.d, lsl #1]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfh	pldl3strm, p6, [z0.s]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfh	pstl2keep, p2, [z21.d]
+# CHECK-NEXT:  1      4     0.33                  U     prfm	pldl1strm, [x5]
+# CHECK-NEXT:  1      4     0.33                  U     prfm	pldl1keep, [x25, x16]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfw	pldl2strm, p2, [x4]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfw	pstl1keep, p4, [x18, x21, lsl #2]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfw	pldl2strm, p0, [x15, z6.s, uxtw #2]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfw	pstl2keep, p0, [x27, z18.d, sxtw #2]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfw	pstl2keep, p3, [x19, z8.d, lsl #2]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfw	#7, p7, [z27.s]
+# CHECK-NEXT:  1      4     0.50    *      *      U     prfw	#7, p1, [z20.d]
 # CHECK-NEXT:  1      2     1.00                        ptest	p15, p0.b
 # CHECK-NEXT:  1      2     1.00                        ptest	p15, p15.b
 # CHECK-NEXT:  1      2     1.00                        ptrue	p0.b, pow2
@@ -4067,6 +4175,7 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  1      2     1.00                        rdvl	x21, #-32
 # CHECK-NEXT:  1      2     1.00                        rdvl	x23, #31
 # CHECK-NEXT:  1      2     1.00                        rdvl	xzr, #-1
+# CHECK-NEXT:  1      2     1.00                        rev	p1.h, p2.h
 # CHECK-NEXT:  1      2     0.50                        rev	z0.b, z31.b
 # CHECK-NEXT:  1      2     0.50                        rev	z0.d, z31.d
 # CHECK-NEXT:  1      2     0.50                        rev	z0.h, z31.h
@@ -4085,8 +4194,10 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  4      12    2.00                        saddv	d0, p7, z31.h
 # CHECK-NEXT:  4      10    2.00                        saddv	d0, p7, z31.s
 # CHECK-NEXT:  1      3     1.00                        scvtf	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      3     1.00                        scvtf	z18.d, p3/m, z16.s
 # CHECK-NEXT:  4      6     4.00                        scvtf	z0.h, p0/m, z0.h
 # CHECK-NEXT:  2      4     2.00                        scvtf	z0.h, p0/m, z0.s
+# CHECK-NEXT:  1      3     1.00                        scvtf	z18.h, p1/m, z14.d
 # CHECK-NEXT:  1      3     1.00                        scvtf	z0.s, p0/m, z0.d
 # CHECK-NEXT:  2      4     2.00                        scvtf	z0.s, p0/m, z0.s
 # CHECK-NEXT:  1      20    7.00                        sdiv	z0.d, p7/m, z0.d, z31.d
@@ -4117,6 +4228,7 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  5      14    2.00                        smaxv	b0, p7, z31.b
 # CHECK-NEXT:  4      12    2.00                        smaxv	h0, p7, z31.h
 # CHECK-NEXT:  4      10    2.00                        smaxv	s0, p7, z31.s
+# CHECK-NEXT:  2      8     0.50                        smaxv	d24, p5, z24.d
 # CHECK-NEXT:  1      2     0.50                        smin	z0.b, z0.b, #-128
 # CHECK-NEXT:  1      2     0.50                        smin	z0.d, z0.d, #-128
 # CHECK-NEXT:  1      2     0.50                        smin	z0.h, z0.h, #-128
@@ -4132,6 +4244,7 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  5      14    2.00                        sminv	b0, p7, z31.b
 # CHECK-NEXT:  4      12    2.00                        sminv	h0, p7, z31.h
 # CHECK-NEXT:  4      10    2.00                        sminv	s0, p7, z31.s
+# CHECK-NEXT:  2      8     0.50                        sminv	d17, p2, z18.d
 # CHECK-NEXT:  1      3     0.50                        smmla	z0.s, z1.b, z2.b
 # CHECK-NEXT:  1      4     1.00                        smulh	z0.b, p7/m, z0.b, z31.b
 # CHECK-NEXT:  2      5     2.00                        smulh	z0.d, p7/m, z0.d, z31.d
@@ -4560,9 +4673,11 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  5      14    2.00                        uaddv	d0, p7, z31.b
 # CHECK-NEXT:  4      12    2.00                        uaddv	d0, p7, z31.h
 # CHECK-NEXT:  4      10    2.00                        uaddv	d0, p7, z31.s
+# CHECK-NEXT:  2      8     0.50                        uaddv	d28, p6, z6.d
 # CHECK-NEXT:  1      3     1.00                        ucvtf	z0.d, p0/m, z0.d
 # CHECK-NEXT:  4      6     4.00                        ucvtf	z0.h, p0/m, z0.h
 # CHECK-NEXT:  2      4     2.00                        ucvtf	z0.h, p0/m, z0.s
+# CHECK-NEXT:  1      3     1.00                        ucvtf	z30.h, p2/m, z24.d
 # CHECK-NEXT:  1      3     1.00                        ucvtf	z0.s, p0/m, z0.d
 # CHECK-NEXT:  2      4     2.00                        ucvtf	z0.s, p0/m, z0.s
 # CHECK-NEXT:  1      20    7.00                        udiv	z0.d, p7/m, z0.d, z31.d
@@ -4571,6 +4686,7 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  1      12    7.00                        udivr	z0.s, p7/m, z0.s, z31.s
 # CHECK-NEXT:  1      4     1.00                        udot	z0.d, z1.h, z15.h[1]
 # CHECK-NEXT:  1      4     1.00                        udot	z0.d, z1.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        ucvtf	z24.d, p5/m, z9.s
 # CHECK-NEXT:  1      3     0.50                        udot	z0.s, z1.b, z31.b
 # CHECK-NEXT:  1      3     0.50                        udot	z0.s, z1.b, z7.b[3]
 # CHECK-NEXT:  1      2     0.50                        umax	z0.b, z0.b, #0
@@ -4582,15 +4698,18 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  5      14    2.00                        umaxv	b0, p7, z31.b
 # CHECK-NEXT:  4      12    2.00                        umaxv	h0, p7, z31.h
 # CHECK-NEXT:  4      10    2.00                        umaxv	s0, p7, z31.s
+# CHECK-NEXT:  2      8     0.50                        umaxv	d11, p4, z11.d
 # CHECK-NEXT:  1      2     0.50                        umin	z0.b, z0.b, #0
 # CHECK-NEXT:  1      2     0.50                        umin	z31.b, p7/m, z31.b, z31.b
 # CHECK-NEXT:  1      2     0.50                        umin	z31.b, z31.b, #255
 # CHECK-NEXT:  1      2     0.50                        umin	z31.d, p7/m, z31.d, z31.d
 # CHECK-NEXT:  1      2     0.50                        umin	z31.h, p7/m, z31.h, z31.h
 # CHECK-NEXT:  1      2     0.50                        umin	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      2     0.50                        umin	z21.s, z21.s, #139
 # CHECK-NEXT:  5      14    2.00                        uminv	b0, p7, z31.b
 # CHECK-NEXT:  4      12    2.00                        uminv	h0, p7, z31.h
 # CHECK-NEXT:  4      10    2.00                        uminv	s0, p7, z31.s
+# CHECK-NEXT:  2      8     0.50                        uminv	d24, p5, z29.d
 # CHECK-NEXT:  1      3     0.50                        ummla	z0.s, z1.b, z2.b
 # CHECK-NEXT:  1      4     1.00                        umulh	z0.b, p7/m, z0.b, z31.b
 # CHECK-NEXT:  2      5     2.00                        umulh	z0.d, p7/m, z0.d, z31.d
@@ -4782,7 +4901,10 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  1      2     0.50                        uzp2	z31.h, z31.h, z31.h
 # CHECK-NEXT:  1      2     0.50                        uzp2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  2      3     2.00                        whilele	p0.b, w30, wzr
+# CHECK-NEXT:  2      3     2.00                        whilele	p6.h, x28, x30
 # CHECK-NEXT:  2      3     2.00                        whilelo	p15.d, xzr, x30
+# CHECK-NEXT:  2      3     2.00                        whilelo	p3.b, x9, x7
+# CHECK-NEXT:  2      3     2.00                        whilels	p4.b, w4, w20
 # CHECK-NEXT:  2      3     2.00                        whilels	p0.h, w30, wzr
 # CHECK-NEXT:  2      3     2.00                        whilelt	p15.s, xzr, x30
 # CHECK-NEXT:  1      2     1.00           *      U     wrffr	p0.b
@@ -4842,7 +4964,7 @@ zip2	z31.s, z31.s, z31.s
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -      -      -      -      -      -      -     88.67  500.67 500.67 797.50 2.50   92.50  92.50  1252.00 921.00 178.50 181.50
+# CHECK-NEXT:  -      -      -      -      -      -      -     89.67  515.67 515.67 804.50 2.50   92.50  92.50  1276.50 926.50 180.00 183.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2.0]  [2.1]  [2.2]  [3]    [4.0]  [4.1]  [5]    [6]    [7.0]  [7.1]  [8]    [9]    [10]   [11]   Instructions:
@@ -5271,9 +5393,12 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	xzr, p15.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	xzr, p15.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decp	xzr, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     decd	z19.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     decp	z31.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     dech	z23.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     decp	z31.h, p15.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     0.50   0.50    -      -     decp	z31.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     decw	z8.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decw	x0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decw	x0, #14
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     decw	x0, all, mul #16
@@ -5850,6 +5975,7 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z21.s }, p5/z, [x10, x21]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   ld1sb	{ z23.s }, p5/z, [x17, z10.s, uxtw]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z23.d }, p3/z, [x13, x8]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     ld1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   ld1sb	{ z31.d }, p7/z, [sp, z31.d]
@@ -6213,8 +6339,17 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsrr	z0.d, p0/m, z0.d, z0.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsrr	z0.h, p0/m, z0.h, z0.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     lsrr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mad	z17.b, p7/m, z4.b, z5.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mad	z29.h, p4/m, z31.h, z18.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mad	z7.s, p4/m, z5.s, z29.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     mad	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mla	z1.b, p0/m, z3.b, z3.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mla	z21.h, p2/m, z31.h, z30.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mla	z24.s, p3/m, z11.s, z9.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     mla	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mls	z11.b, p1/m, z28.b, z6.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mls	z31.h, p0/m, z25.h, z24.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     mls	z1.s, p5/m, z7.s, z13.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     mls	z0.d, p0/m, z0.d, z0.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	p0.b, p0.b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	p0.b, p0/m, p0.b
@@ -6289,7 +6424,8 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.b, p7/m, b31
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     movprfx	z31, z6
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z31.b, p7/m, wsp
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     mov	z31.b, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     movprfx	z31.b, p0/m, z4.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     mov	z31.b, p0/m, wsp
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.b, z31.b[63]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.d, p15/m, z31.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     mov	z31.d, p7/m, d31
@@ -6330,6 +6466,9 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL1, x3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     msb	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     msb	z18.b, p1/m, z27.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     msb	z27.h, p5/m, z23.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     msb	z26.s, p2/m, z0.s, z2.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL12, x3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL2, x3
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL3, x3
@@ -6400,6 +6539,36 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pnext	p0.h, p15, p0.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pnext	p0.s, p15, p0.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     pnext	p15.b, p15, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfb	#14, p5, [x21]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfb	pldl1keep, p7, [x4, x9]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfb	pldl3strm, p4, [x3, z15.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfb	pldl1strm, p7, [x28, z4.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfb	pstl3keep, p2, [x18, z19.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfb	pstl3keep, p1, [z28.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfb	pstl2strm, p5, [z25.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfd	pstl3strm, p3, [x21]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfd	pstl2keep, p3, [x24, x24, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfd	pstl1strm, p3, [x27, z27.s, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfd	pstl1keep, p0, [x21, z2.d, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfd	pldl1strm, p7, [x22, z22.d, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfd	pldl2strm, p1, [z2.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfd	#15, p1, [z17.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfh	pldl2strm, p3, [x17]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfh	pstl2keep, p1, [x28, x9, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfh	pldl1strm, p6, [x0, z10.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfh	pldl3keep, p7, [x24, z21.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfh	pstl1strm, p5, [x10, z6.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfh	pldl3strm, p6, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfh	pstl2keep, p2, [z21.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     prfm	pldl1strm, [x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     prfm	pldl1keep, [x25, x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfw	pldl2strm, p2, [x4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfw	pstl1keep, p4, [x18, x21, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfw	pldl2strm, p0, [x15, z6.s, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfw	pstl2keep, p0, [x27, z18.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfw	pstl2keep, p3, [x19, z8.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfw	#7, p7, [z27.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -     prfw	#7, p1, [z20.d]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptest	p15, p0.b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptest	p15, p15.b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ptrue	p0.b, pow2
@@ -6498,6 +6667,7 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     rdvl	x21, #-32
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     rdvl	x23, #31
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     rdvl	xzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     rev	p1.h, p2.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rev	z0.b, z31.b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rev	z0.d, z31.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     rev	z0.h, z31.h
@@ -6516,8 +6686,10 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   saddv	d0, p7, z31.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   saddv	d0, p7, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     scvtf	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     scvtf	z18.d, p3/m, z16.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00    -      -      -     scvtf	z0.h, p0/m, z0.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     scvtf	z0.h, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     scvtf	z18.h, p1/m, z14.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     scvtf	z0.s, p0/m, z0.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     scvtf	z0.s, p0/m, z0.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     sdiv	z0.d, p7/m, z0.d, z31.d
@@ -6548,6 +6720,7 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.25   2.75   0.25   0.75   smaxv	b0, p7, z31.b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   smaxv	h0, p7, z31.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   smaxv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.25   0.25   smaxv	d24, p5, z24.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z0.b, z0.b, #-128
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z0.d, z0.d, #-128
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smin	z0.h, z0.h, #-128
@@ -6563,6 +6736,7 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.25   2.75   0.25   0.75   sminv	b0, p7, z31.b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   sminv	h0, p7, z31.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   sminv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.25   0.25   sminv	d17, p2, z18.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     smmla	z0.s, z1.b, z2.b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     smulh	z0.b, p7/m, z0.b, z31.b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     smulh	z0.d, p7/m, z0.d, z31.d
@@ -6991,9 +7165,11 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.25   2.75   0.25   0.75   uaddv	d0, p7, z31.b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   uaddv	d0, p7, z31.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   uaddv	d0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.25   0.25   uaddv	d28, p6, z6.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     ucvtf	z0.d, p0/m, z0.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00    -      -      -     ucvtf	z0.h, p0/m, z0.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     ucvtf	z0.h, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     ucvtf	z30.h, p2/m, z24.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     ucvtf	z0.s, p0/m, z0.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     ucvtf	z0.s, p0/m, z0.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     udiv	z0.d, p7/m, z0.d, z31.d
@@ -7002,6 +7178,7 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     udivr	z0.s, p7/m, z0.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     udot	z0.d, z1.h, z15.h[1]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     udot	z0.d, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     ucvtf	z24.d, p5/m, z9.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     udot	z0.s, z1.b, z31.b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     udot	z0.s, z1.b, z7.b[3]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umax	z0.b, z0.b, #0
@@ -7013,15 +7190,18 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.25   2.75   0.25   0.75   umaxv	b0, p7, z31.b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   umaxv	h0, p7, z31.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   umaxv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.25   0.25   umaxv	d11, p4, z11.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z0.b, z0.b, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z31.b, p7/m, z31.b, z31.b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z31.b, z31.b, #255
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z31.d, p7/m, z31.d, z31.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z31.h, p7/m, z31.h, z31.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umin	z21.s, z21.s, #139
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.25   2.75   0.25   0.75   uminv	b0, p7, z31.b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   uminv	h0, p7, z31.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   2.75   0.25   0.25   uminv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.25   0.25   uminv	d24, p5, z29.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     ummla	z0.s, z1.b, z2.b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     umulh	z0.b, p7/m, z0.b, z31.b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00    -      -      -     umulh	z0.d, p7/m, z0.d, z31.d
@@ -7213,7 +7393,10 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uzp2	z31.h, z31.h, z31.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     uzp2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     whilele	p0.b, w30, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     whilele	p6.h, x28, x30
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     whilelo	p15.d, xzr, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     whilelo	p3.b, x9, x7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     whilels	p4.b, w4, w20
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     whilels	p0.h, w30, wzr
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     whilelt	p15.s, xzr, x30
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     wrffr	p0.b
diff --git a/llvm/test/tools/llvm-ml/macro.asm b/llvm/test/tools/llvm-ml/macro.asm
index 3c81b89414fc7..3373ff4837f7c 100644
--- a/llvm/test/tools/llvm-ml/macro.asm
+++ b/llvm/test/tools/llvm-ml/macro.asm
@@ -100,6 +100,25 @@ substitution_test_uppercase PROC
   ret
 substitution_test_uppercase ENDP
 
+substitution_test_with_parentheses PROC
+; CHECK-LABEL: substitution_test_with_parentheses:
+
+  SubstitutionMacro(2, 8)
+; CHECK: mov eax, 2
+; CHECK-NEXT: mov eax, 2
+; CHECK-NEXT: mov eax, 2
+; CHECK-NEXT: mov eax, 2
+; CHECK: mov eax, dword ptr [rip + xa1]
+; CHECK-NEXT: mov eax, dword ptr [rip + x2]
+; CHECK-NEXT: mov eax, dword ptr [rip + x2]
+; CHECK: mov eax, 8
+; CHECK-NEXT: mov eax, 8
+; CHECK-NEXT: mov eax, 8
+; CHECK-NEXT: mov eax, 8
+
+  ret
+substitution_test_with_parentheses ENDP
+
 AmbiguousSubstitutionMacro MACRO x, y
   x&y BYTE 0
 ENDM
diff --git a/llvm/test/tools/llvm-objcopy/ELF/discard-all.test b/llvm/test/tools/llvm-objcopy/ELF/discard-all.test
index 50cad3ae9f533..ce65026715f71 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/discard-all.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/discard-all.test
@@ -32,6 +32,14 @@ Sections:
     Address:         0x1000
     AddressAlign:    0x0000000000000010
     Size:            64
+  - Name:            .rela.text
+    Type:            SHT_RELA
+    Link:            .symtab
+    Info:            .text
+    Relocations:
+      - Offset: 0
+        Symbol: Referenced
+        Type:   R_X86_64_PC32
 Symbols:
   - Name:     Local
     Type:     STT_FUNC
@@ -49,6 +57,8 @@ Symbols:
     Section:  .text
     Value:    0x1010
     Binding:  STB_GLOBAL
+  - Name:     Referenced
+    Section:  .text
   - Name:     Weak
     Type:     STT_FUNC
     Size:     8
@@ -85,6 +95,15 @@ Symbols:
 #CHECK-NEXT:    Section: Undefined
 #CHECK-NEXT:  }
 #CHECK-NEXT:  Symbol {
+#CHECK-NEXT:    Name: Referenced
+#CHECK-NEXT:    Value: 0x0
+#CHECK-NEXT:    Size: 0
+#CHECK-NEXT:    Binding: Local
+#CHECK-NEXT:    Type: None
+#CHECK-NEXT:    Other: 0
+#CHECK-NEXT:    Section: .text
+#CHECK-NEXT:  }
+#CHECK-NEXT:  Symbol {
 #CHECK-NEXT:    Name: Global
 #CHECK-NEXT:    Value: 0x1010
 #CHECK-NEXT:    Size: 8
diff --git a/llvm/test/tools/llvm-objcopy/ELF/discard-locals-rel.test b/llvm/test/tools/llvm-objcopy/ELF/discard-locals-rel.test
deleted file mode 100644
index 00bb8fcf18205..0000000000000
--- a/llvm/test/tools/llvm-objcopy/ELF/discard-locals-rel.test
+++ /dev/null
@@ -1,26 +0,0 @@
-# RUN: yaml2obj %s -o %t
-# RUN: not llvm-objcopy --discard-locals %t %t2 2>&1 | FileCheck %s -DFILE=%t
-
-!ELF
-FileHeader:
-  Class:           ELFCLASS64
-  Data:            ELFDATA2LSB
-  Type:            ET_REL
-  Machine:         EM_X86_64
-Sections:
-  - Name:            .text
-    Type:            SHT_PROGBITS
-  - Name:            .rel.text
-    Type:            SHT_REL
-    Link:            .symtab
-    Info:            .text
-    Relocations:
-      - Offset: 0x1000
-        Symbol: .L.rel
-        Type:   R_X86_64_PC32
-Symbols:
-  - Name:     .L.rel
-    Type:     STT_FUNC
-    Section:  .text
-
-# CHECK: error: '[[FILE]]': not stripping symbol '.L.rel' because it is named in a relocation
diff --git a/llvm/test/tools/llvm-objcopy/ELF/discard-locals.test b/llvm/test/tools/llvm-objcopy/ELF/discard-locals.test
index 4aba1cd1c782a..c1fe0f2b02b6e 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/discard-locals.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/discard-locals.test
@@ -29,6 +29,14 @@ FileHeader:
 Sections:
   - Name:            .text
     Type:            SHT_PROGBITS
+  - Name:            .rela.text
+    Type:            SHT_RELA
+    Link:            .symtab
+    Info:            .text
+    Relocations:
+      - Offset: 0
+        Symbol: .L.referenced
+        Type:   R_X86_64_PC32
   - Name:            .LLVM.Custom.Section
     Type:            SHT_PROGBITS
 Symbols:
@@ -48,6 +56,8 @@ Symbols:
   - Name:     .L.undefined
   - Name:     .L.abs
     Index:    SHN_ABS
+  - Name:     .L.referenced
+    Section:  .text
   - Name:     .L.Global
     Type:     STT_FUNC
     Section:  .text
@@ -109,6 +119,15 @@ Symbols:
 # CHECK-NEXT:     Section: Undefined
 # CHECK-NEXT:   }
 # CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: .L.referenced
+# CHECK-NEXT:     Value:
+# CHECK-NEXT:     Size:
+# CHECK-NEXT:     Binding: Local
+# CHECK-NEXT:     Type: None
+# CHECK-NEXT:     Other:
+# CHECK-NEXT:     Section: .text
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
 # CHECK-NEXT:     Name: .L.Global
 # CHECK-NEXT:     Value:
 # CHECK-NEXT:     Size:
diff --git a/llvm/test/tools/llvm-objcopy/strip-error-handling.test b/llvm/test/tools/llvm-objcopy/strip-error-handling.test
new file mode 100644
index 0000000000000..2db90336c3588
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/strip-error-handling.test
@@ -0,0 +1,25 @@
+## Checks that llvm-strip continues to strip objects after encountering a bad
+## one while emitting an error for each bad one.
+
+# RUN: echo "bad" > %t1
+# RUN: yaml2obj %s -o %t2
+# RUN: cp %t1 %t3
+# RUN: not llvm-strip --strip-sections %t1 %t2 %t3 2>&1 | FileCheck %s --check-prefix=ERROR -DFILE1=%t1 -DFILE3=%t3 --implicit-check-not=error:
+
+# ERROR: error: '[[FILE1]]': The file was not recognized as a valid object file
+# ERROR-NEXT: error: '[[FILE3]]': The file was not recognized as a valid object file
+
+# RUN: llvm-readobj --file-header %t2 | FileCheck %s --check-prefix=NUMSECTIONS
+
+# NUMSECTIONS: SectionHeaderCount: 0
+
+!ELF
+FileHeader:
+  Class: ELFCLASS64
+  Data:  ELFDATA2LSB
+  Type:  ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .debugGlobal
+    Type:    SHT_PROGBITS
+    Content: "00000000"
diff --git a/llvm/test/tools/llvm-objdump/ELF/dynamic-section.test b/llvm/test/tools/llvm-objdump/ELF/dynamic-section.test
index 5205c5a3876d5..b84a6e56881d4 100644
--- a/llvm/test/tools/llvm-objdump/ELF/dynamic-section.test
+++ b/llvm/test/tools/llvm-objdump/ELF/dynamic-section.test
@@ -470,3 +470,46 @@ Sections:
        Value: 0x1
      - Tag:   DT_NULL
        Value: 0x0
+
+# RUN: yaml2obj --docnum=5 %s -o %t5
+# RUN: llvm-objdump -p %t5 2>&1 | FileCheck %s --strict-whitespace -DFILE=%t5 --check-prefix=WARN
+
+# WARN: Dynamic Section:
+# WARN-NEXT: warning: '[[FILE]]': invalid string table offset
+# WARN-NEXT:  NEEDED 0x0000000000000010
+# WARN-NEXT:  NEEDED {{$}}
+# WARN-NEXT:  STRTAB 0x0000000000001000
+# WARN-NEXT:  STRSZ  0x0000000000000010
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .dynstr
+    Type:    SHT_STRTAB
+    Address: 0x1000
+    Size:    0x10
+  - Name: .dynamic
+    Type: SHT_DYNAMIC
+    Entries:
+     - Tag:   DT_NEEDED
+       Value: 0x10
+     - Tag:   DT_NEEDED
+       Value: 0x0F
+     - Tag:   DT_STRTAB
+       Value: 0x1000
+     - Tag:   DT_STRSZ
+       Value: 0x10
+     - Tag:   DT_NULL
+       Value: 0x0
+ProgramHeaders:
+  - Type:     PT_LOAD
+    VAddr:    0x1000
+    FirstSec: .dynstr
+    LastSec:  .dynamic
+  - Type:     PT_DYNAMIC
+    FirstSec: .dynamic
+    LastSec:  .dynamic
diff --git a/llvm/test/tools/llvm-readobj/ELF/AArch64/build-attributes-comprehensive.s b/llvm/test/tools/llvm-readobj/ELF/AArch64/build-attributes-comprehensive.s
new file mode 100644
index 0000000000000..0722d006c4f49
--- /dev/null
+++ b/llvm/test/tools/llvm-readobj/ELF/AArch64/build-attributes-comprehensive.s
@@ -0,0 +1,81 @@
+# RUN: llvm-mc -triple=aarch64-linux -filetype=obj < %s | llvm-readelf --arch-specific - | FileCheck %s --check-prefix=ATTR
+
+# ATTR: BuildAttributes {
+# ATTR-NEXT:   FormatVersion: 0x41
+# ATTR-NEXT:   Section 1 {
+# ATTR-NEXT:     SectionLength: 29
+# ATTR-NEXT:     VendorName: private_subsection_1 Optionality: optional Type: uleb128
+# ATTR-NEXT:     Attributes {
+# ATTR-NEXT:       1: 1
+# ATTR-NEXT:     }
+# ATTR-NEXT:   }
+# ATTR-NEXT:   Section 2 {
+# ATTR-NEXT:     SectionLength: 37
+# ATTR-NEXT:     VendorName: aeabi_feature_and_bits Optionality: optional Type: uleb128
+# ATTR-NEXT:     Attributes {
+# ATTR-NEXT:       Tag_Feature_BTI: 1
+# ATTR-NEXT:       Tag_Feature_PAC: 1
+# ATTR-NEXT:       Tag_Feature_GCS: 1
+# ATTR-NEXT:       3: 1
+# ATTR-NEXT:     }
+# ATTR-NEXT:   }
+# ATTR-NEXT:   Section 3 {
+# ATTR-NEXT:     SectionLength: 32
+# ATTR-NEXT:     VendorName: private_subsection_3 Optionality: optional Type: ntbs
+# ATTR-NEXT:     Attributes {
+# ATTR-NEXT:       1: "1"
+# ATTR-NEXT:     }
+# ATTR-NEXT:   }
+# ATTR-NEXT:   Section 4 {
+# ATTR-NEXT:     SectionLength: 35
+# ATTR-NEXT:     VendorName: aeabi_pauthabi Optionality: required Type: uleb128
+# ATTR-NEXT:     Attributes {
+# ATTR-NEXT:       Tag_PAuth_Schema: 1
+# ATTR-NEXT:       Tag_PAuth_Platform: 1
+# ATTR-NEXT:       5: 1
+# ATTR-NEXT:       6: 1
+# ATTR-NEXT:       7: 1
+# ATTR-NEXT:       8: 1
+# ATTR-NEXT:       9: 1
+# ATTR-NEXT:     }
+# ATTR-NEXT:   }
+# ATTR-NEXT:   Section 5 {
+# ATTR-NEXT:     SectionLength: 32
+# ATTR-NEXT:     VendorName: private_subsection_4 Optionality: required Type: ntbs
+# ATTR-NEXT:     Attributes {
+# ATTR-NEXT:       1: "1"
+# ATTR-NEXT:     }
+# ATTR-NEXT:   }
+# ATTR-NEXT:   Section 6 {
+# ATTR-NEXT:     SectionLength: 31
+# ATTR-NEXT:     VendorName: private_subsection_2 Optionality: required Type: uleb128
+# ATTR-NEXT:     Attributes {
+# ATTR-NEXT:       1: 1
+# ATTR-NEXT:       2: 1
+# ATTR-NEXT:     }
+# ATTR-NEXT:   }
+# ATTR-NEXT: }
+
+
+.aeabi_subsection private_subsection_1, optional, uleb128
+.aeabi_attribute 1, 1
+.aeabi_subsection aeabi_feature_and_bits, optional, uleb128
+.aeabi_attribute Tag_Feature_BTI, 1
+.aeabi_attribute 1, 1
+.aeabi_attribute 2, 1
+.aeabi_attribute 3, 1
+.aeabi_subsection private_subsection_3, optional, ntbs
+.aeabi_attribute 1, "1"
+.aeabi_subsection aeabi_pauthabi, required, uleb128
+.aeabi_attribute Tag_PAuth_Schema, 1
+.aeabi_attribute Tag_PAuth_Platform, 1
+.aeabi_attribute 5, 1
+.aeabi_attribute 6, 1
+.aeabi_attribute 7, 1
+.aeabi_attribute 8, 1
+.aeabi_attribute 9, 1
+.aeabi_subsection private_subsection_4, required, ntbs
+.aeabi_attribute 1, "1"
+.aeabi_subsection private_subsection_2, required, uleb128
+.aeabi_attribute 1, 1
+.aeabi_attribute 2, 1
diff --git a/llvm/tools/lli/lli.cpp b/llvm/tools/lli/lli.cpp
index 1417a58ab4487..ed1a80d3c1bc9 100644
--- a/llvm/tools/lli/lli.cpp
+++ b/llvm/tools/lli/lli.cpp
@@ -285,8 +285,8 @@ namespace {
 }
 
 LLVM_ATTRIBUTE_USED void linkComponents() {
-  errs() << (void *)&llvm_orc_registerEHFrameSectionWrapper
-         << (void *)&llvm_orc_deregisterEHFrameSectionWrapper
+  errs() << (void *)&llvm_orc_registerEHFrameSectionAllocAction
+         << (void *)&llvm_orc_deregisterEHFrameSectionAllocAction
          << (void *)&llvm_orc_registerJITLoaderGDBWrapper
          << (void *)&llvm_orc_registerJITLoaderGDBAllocAction;
 }
diff --git a/llvm/tools/llvm-exegesis/lib/AArch64/Target.cpp b/llvm/tools/llvm-exegesis/lib/AArch64/Target.cpp
index ed36cb2f75d5b..bf2b053003ce3 100644
--- a/llvm/tools/llvm-exegesis/lib/AArch64/Target.cpp
+++ b/llvm/tools/llvm-exegesis/lib/AArch64/Target.cpp
@@ -28,8 +28,8 @@ static unsigned getLoadImmediateOpcode(unsigned RegBitWidth) {
 // Generates instruction to load an immediate value into a register.
 static MCInst loadImmediate(MCRegister Reg, unsigned RegBitWidth,
                             const APInt &Value) {
-  assert (Value.getBitWidth() <= RegBitWidth &&
-          "Value must fit in the Register");
+  assert(Value.getBitWidth() <= RegBitWidth &&
+         "Value must fit in the Register");
   return MCInstBuilder(getLoadImmediateOpcode(RegBitWidth))
       .addReg(Reg)
       .addImm(Value.getZExtValue());
@@ -53,11 +53,25 @@ static MCInst loadPPRImmediate(MCRegister Reg, unsigned RegBitWidth,
       .addImm(31); // All lanes true for 16 bits
 }
 
+// Generates instructions to load an immediate value into an FPCR register.
+static std::vector<MCInst>
+loadFPCRImmediate(MCRegister Reg, unsigned RegBitWidth, const APInt &Value) {
+  MCRegister TempReg = AArch64::X8;
+  MCInst LoadImm = MCInstBuilder(AArch64::MOVi64imm).addReg(TempReg).addImm(0);
+  MCInst MoveToFPCR =
+      MCInstBuilder(AArch64::MSR).addImm(AArch64SysReg::FPCR).addReg(TempReg);
+  return {LoadImm, MoveToFPCR};
+}
+
 // Fetch base-instruction to load an FP immediate value into a register.
 static unsigned getLoadFPImmediateOpcode(unsigned RegBitWidth) {
   switch (RegBitWidth) {
+  case 16:
+    return AArch64::FMOVH0; // FMOVHi;
+  case 32:
+    return AArch64::FMOVS0; // FMOVSi;
   case 64:
-    return AArch64::MOVID; //FMOVDi;
+    return AArch64::MOVID; // FMOVDi;
   case 128:
     return AArch64::MOVIv2d_ns;
   }
@@ -67,11 +81,12 @@ static unsigned getLoadFPImmediateOpcode(unsigned RegBitWidth) {
 // Generates instruction to load an FP immediate value into a register.
 static MCInst loadFPImmediate(MCRegister Reg, unsigned RegBitWidth,
                               const APInt &Value) {
-  assert(Value.getZExtValue() == 0 &&
-         "Expected initialisation value 0");
-  return MCInstBuilder(getLoadFPImmediateOpcode(RegBitWidth))
-      .addReg(Reg)
-      .addImm(Value.getZExtValue());
+  assert(Value.getZExtValue() == 0 && "Expected initialisation value 0");
+  MCInst Instructions =
+      MCInstBuilder(getLoadFPImmediateOpcode(RegBitWidth)).addReg(Reg);
+  if (RegBitWidth >= 64)
+    Instructions.addOperand(MCOperand::createImm(Value.getZExtValue()));
+  return Instructions;
 }
 
 #include "AArch64GenExegesis.inc"
@@ -92,12 +107,20 @@ class ExegesisAArch64Target : public ExegesisTarget {
       return {loadImmediate(Reg, 64, Value)};
     if (AArch64::PPRRegClass.contains(Reg))
       return {loadPPRImmediate(Reg, 16, Value)};
+    if (AArch64::FPR8RegClass.contains(Reg))
+      return {loadFPImmediate(Reg - AArch64::B0 + AArch64::D0, 64, Value)};
+    if (AArch64::FPR16RegClass.contains(Reg))
+      return {loadFPImmediate(Reg, 16, Value)};
+    if (AArch64::FPR32RegClass.contains(Reg))
+      return {loadFPImmediate(Reg, 32, Value)};
     if (AArch64::FPR64RegClass.contains(Reg))
       return {loadFPImmediate(Reg, 64, Value)};
     if (AArch64::FPR128RegClass.contains(Reg))
       return {loadFPImmediate(Reg, 128, Value)};
     if (AArch64::ZPRRegClass.contains(Reg))
       return {loadZPRImmediate(Reg, 128, Value)};
+    if (Reg == AArch64::FPCR)
+      return {loadFPCRImmediate(Reg, 32, Value)};
 
     errs() << "setRegTo is not implemented, results will be unreliable\n";
     return {};
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp
index 8553eb70ebe49..8d33ae1edcaaa 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp
@@ -42,8 +42,8 @@ using namespace llvm::orc;
 ExitOnError ExitOnErr;
 
 LLVM_ATTRIBUTE_USED void linkComponents() {
-  errs() << (void *)&llvm_orc_registerEHFrameSectionWrapper
-         << (void *)&llvm_orc_deregisterEHFrameSectionWrapper
+  errs() << (void *)&llvm_orc_registerEHFrameSectionAllocAction
+         << (void *)&llvm_orc_deregisterEHFrameSectionAllocAction
          << (void *)&llvm_orc_registerJITLoaderGDBWrapper
          << (void *)&llvm_orc_registerJITLoaderGDBAllocAction;
 }
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink-macho.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink-macho.cpp
index 2fc56c9fcc72a..07dd3bd38ee61 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink-macho.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink-macho.cpp
@@ -72,14 +72,15 @@ Error registerMachOGraphInfo(Session &S, LinkGraph &G) {
   std::lock_guard<std::mutex> Lock(S.M);
 
   auto FileName = sys::path::filename(G.getName());
-  if (S.FileInfos.count(FileName)) {
+  auto [It, Inserted] = S.FileInfos.try_emplace(FileName);
+  if (!Inserted) {
     return make_error<StringError>("When -check is passed, file names must be "
                                    "distinct (duplicate: \"" +
                                        FileName + "\")",
                                    inconvertibleErrorCode());
   }
 
-  auto &FileInfo = S.FileInfos[FileName];
+  auto &FileInfo = It->second;
   LLVM_DEBUG({
     dbgs() << "Registering MachO file info for \"" << FileName << "\"\n";
   });
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
index e4ed5b2aefb9a..8a17abd58e98a 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
@@ -27,7 +27,6 @@
 #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
 #include "llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h"
 #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
-#include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h"
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
 #include "llvm/ExecutionEngine/Orc/GetDylibInterface.h"
 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
@@ -337,8 +336,8 @@ static ExitOnError ExitOnErr;
 
 static LLVM_ATTRIBUTE_USED void linkComponents() {
   errs() << "Linking in runtime functions\n"
-         << (void *)&llvm_orc_registerEHFrameSectionWrapper << '\n'
-         << (void *)&llvm_orc_deregisterEHFrameSectionWrapper << '\n'
+         << (void *)&llvm_orc_registerEHFrameSectionAllocAction << '\n'
+         << (void *)&llvm_orc_deregisterEHFrameSectionAllocAction << '\n'
          << (void *)&llvm_orc_registerJITLoaderGDBWrapper << '\n'
          << (void *)&llvm_orc_registerJITLoaderGDBAllocAction << '\n'
          << (void *)&llvm_orc_registerJITLoaderPerfStart << '\n'
@@ -1248,13 +1247,11 @@ Session::Session(std::unique_ptr<ExecutorProcessControl> EPC, Error &Err)
       if (!UseEHFrames)
         ObjLayer.addPlugin(ExitOnErr(UnwindInfoRegistrationPlugin::Create(ES)));
       else
-        ObjLayer.addPlugin(std::make_unique<EHFrameRegistrationPlugin>(
-            ES, ExitOnErr(EPCEHFrameRegistrar::Create(ES))));
+        ObjLayer.addPlugin(ExitOnErr(EHFrameRegistrationPlugin::Create(ES)));
     }
   } else if (TT.isOSBinFormatELF()) {
     if (!NoExec)
-      ObjLayer.addPlugin(std::make_unique<EHFrameRegistrationPlugin>(
-          ES, ExitOnErr(EPCEHFrameRegistrar::Create(this->ES))));
+      ObjLayer.addPlugin(ExitOnErr(EHFrameRegistrationPlugin::Create(ES)));
     if (DebuggerSupport)
       ObjLayer.addPlugin(std::make_unique<DebugObjectManagerPlugin>(
           ES, ExitOnErr(createJITLoaderGDBRegistrar(this->ES)), true, true));
diff --git a/llvm/tools/llvm-jitlistener/llvm-jitlistener.cpp b/llvm/tools/llvm-jitlistener/llvm-jitlistener.cpp
index 59949435f9ca4..367ea136234a6 100644
--- a/llvm/tools/llvm-jitlistener/llvm-jitlistener.cpp
+++ b/llvm/tools/llvm-jitlistener/llvm-jitlistener.cpp
@@ -168,7 +168,7 @@ class JitEventListenerTest {
 
     if (Tuple.isOSWindows() && !Tuple.isOSBinFormatELF()) {
       Tuple.setObjectFormat(Triple::ELF);
-      TheModule->setTargetTriple(Tuple.getTriple());
+      TheModule->setTargetTriple(Tuple);
     }
 
     // Compile the IR
diff --git a/llvm/tools/llvm-objcopy/llvm-objcopy.cpp b/llvm/tools/llvm-objcopy/llvm-objcopy.cpp
index 7e708e309f207..3b1108859faf0 100644
--- a/llvm/tools/llvm-objcopy/llvm-objcopy.cpp
+++ b/llvm/tools/llvm-objcopy/llvm-objcopy.cpp
@@ -247,14 +247,16 @@ int llvm_objcopy_main(int argc, char **argv, const llvm::ToolContext &) {
                           WithColor::error(errs(), ToolName));
     return 1;
   }
+
+  int ret = 0;
   for (ConfigManager &ConfigMgr : DriverConfig->CopyConfigs) {
     assert(!ConfigMgr.Common.ErrorCallback);
     ConfigMgr.Common.ErrorCallback = reportWarning;
     if (Error E = executeObjcopy(ConfigMgr)) {
       logAllUnhandledErrors(std::move(E), WithColor::error(errs(), ToolName));
-      return 1;
+      ret = 1;
     }
   }
 
-  return 0;
+  return ret;
 }
diff --git a/llvm/tools/llvm-objdump/ELFDump.cpp b/llvm/tools/llvm-objdump/ELFDump.cpp
index b7899bc3bcbee..3b288ea73398b 100644
--- a/llvm/tools/llvm-objdump/ELFDump.cpp
+++ b/llvm/tools/llvm-objdump/ELFDump.cpp
@@ -63,16 +63,24 @@ static Expected<StringRef> getDynamicStrTab(const ELFFile<ELFT> &Elf) {
   if (!DynamicEntriesOrError)
     return DynamicEntriesOrError.takeError();
 
+  typename ELFT::Xword StringTableSize{0};
+  const uint8_t *MappedAddr = nullptr;
   for (const typename ELFT::Dyn &Dyn : *DynamicEntriesOrError) {
     if (Dyn.d_tag == ELF::DT_STRTAB) {
       auto MappedAddrOrError = Elf.toMappedAddr(Dyn.getPtr());
       if (!MappedAddrOrError)
         return MappedAddrOrError.takeError();
-      return StringRef(reinterpret_cast<const char *>(*MappedAddrOrError));
+      MappedAddr = *MappedAddrOrError;
     }
+    if (Dyn.d_tag == ELF::DT_STRSZ)
+      StringTableSize = Dyn.getVal();
   }
+  if (MappedAddr && StringTableSize)
+    return StringRef(reinterpret_cast<const char *>(MappedAddr),
+                     StringTableSize);
 
-  // If the dynamic segment is not present, we fall back on the sections.
+  // If the dynamic segment is not present, or is missing the important tags, we
+  // fall back on the sections.
   auto SectionsOrError = Elf.sections();
   if (!SectionsOrError)
     return SectionsOrError.takeError();
@@ -221,6 +229,7 @@ template <class ELFT> void ELFDumper<ELFT>::printDynamicSection() {
   std::string TagFmt = "  %-" + std::to_string(MaxLen) + "s ";
 
   outs() << "\nDynamic Section:\n";
+
   for (const typename ELFT::Dyn &Dyn : DynamicEntries) {
     if (Dyn.d_tag == ELF::DT_NULL)
       continue;
@@ -235,6 +244,14 @@ template <class ELFT> void ELFDumper<ELFT>::printDynamicSection() {
       Expected<StringRef> StrTabOrErr = getDynamicStrTab(Elf);
       if (StrTabOrErr) {
         const char *Data = StrTabOrErr->data();
+        if (Dyn.getVal() >= StrTabOrErr->size()) {
+          reportWarning("invalid string table offset, string table size: 0x" +
+                            Twine::utohexstr(StrTabOrErr->size()),
+                        Obj.getFileName());
+          outs() << format(TagFmt.c_str(), Str.c_str())
+                 << format(Fmt, (uint64_t)Dyn.getVal());
+          continue;
+        }
         outs() << format(TagFmt.c_str(), Str.c_str()) << Data + Dyn.getVal()
                << "\n";
         continue;
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index f0c9a417f3935..9a5d3f91d6256 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -130,7 +130,7 @@ static cl::opt<std::string> DebugInfoFilename(
     cl::sub(ShowSubcommand), cl::sub(MergeSubcommand));
 static cl::opt<std::string>
     BinaryFilename("binary-file", cl::init(""),
-                   cl::desc("For merge, use the provided unstripped bianry to "
+                   cl::desc("For merge, use the provided unstripped binary to "
                             "correlate the raw profile."),
                    cl::sub(MergeSubcommand));
 static cl::list<std::string> DebugFileDirectory(
diff --git a/llvm/tools/llvm-profgen/MissingFrameInferrer.cpp b/llvm/tools/llvm-profgen/MissingFrameInferrer.cpp
index ac88fced9159c..edfe8979c7121 100644
--- a/llvm/tools/llvm-profgen/MissingFrameInferrer.cpp
+++ b/llvm/tools/llvm-profgen/MissingFrameInferrer.cpp
@@ -165,14 +165,14 @@ uint64_t MissingFrameInferrer::computeUniqueTailCallPath(
   if (CurSearchingDepth == MaximumSearchDepth)
     return 0;
 
-
-  if (!FuncToTailCallMap.count(From))
+  auto It = FuncToTailCallMap.find(From);
+  if (It == FuncToTailCallMap.end())
     return 0;
 
   CurSearchingDepth++;
   Visiting.insert(From);
   uint64_t NumPaths = 0;
-  for (auto TailCall : FuncToTailCallMap[From]) {
+  for (auto TailCall : It->second) {
     NumPaths += computeUniqueTailCallPath(TailCall, To, Path);
     // Stop analyzing the remaining if we are already seeing more than one
     // reachable paths.
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index e7825419ef9ec..d01cab211afd7 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -39,6 +39,7 @@
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Object/RelocationResolver.h"
 #include "llvm/Object/StackMapParser.h"
+#include "llvm/Support/AArch64AttributeParser.h"
 #include "llvm/Support/AMDGPUMetadata.h"
 #include "llvm/Support/ARMAttributeParser.h"
 #include "llvm/Support/ARMBuildAttributes.h"
@@ -2878,6 +2879,12 @@ template <class ELFT> void ELFDumper<ELFT>::printArchSpecificInfo() {
         ELF::SHT_ARM_ATTRIBUTES, std::make_unique<ARMAttributeParser>(&W),
         Obj.isLE() ? llvm::endianness::little : llvm::endianness::big);
     break;
+  case EM_AARCH64:
+    printAttributes(ELF::SHT_AARCH64_ATTRIBUTES,
+                    std::make_unique<AArch64AttributeParser>(&W),
+                    Obj.isLE() ? llvm::endianness::little
+                               : llvm::endianness::big);
+    break;
   case EM_RISCV:
     if (Obj.isLE())
       printAttributes(ELF::SHT_RISCV_ATTRIBUTES,
diff --git a/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp b/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp
index 06ac98b0c5e13..40a7b86078d87 100644
--- a/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp
+++ b/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp
@@ -929,15 +929,17 @@ static int linkAndVerify() {
                                        StringRef SymbolName,
                                        StringRef KindNameFilter)
       -> Expected<RuntimeDyldChecker::MemoryRegionInfo> {
-    if (!StubMap.count(StubContainer))
+    auto SMIt = StubMap.find(StubContainer);
+    if (SMIt == StubMap.end())
       return make_error<StringError>("Stub container not found: " +
                                          StubContainer,
                                      inconvertibleErrorCode());
-    if (!StubMap[StubContainer].count(SymbolName))
+    auto It = SMIt->second.find(SymbolName);
+    if (It == SMIt->second.end())
       return make_error<StringError>("Symbol name " + SymbolName +
                                          " in stub container " + StubContainer,
                                      inconvertibleErrorCode());
-    auto &SI = StubMap[StubContainer][SymbolName];
+    auto &SI = It->second;
     RuntimeDyldChecker::MemoryRegionInfo StubMemInfo;
     StubMemInfo.setTargetAddress(Dyld.getSectionLoadAddress(SI.SectionID) +
                                  SI.Offset);
diff --git a/llvm/unittests/ADT/APFloatTest.cpp b/llvm/unittests/ADT/APFloatTest.cpp
index c2d99f3312b88..09356191345f5 100644
--- a/llvm/unittests/ADT/APFloatTest.cpp
+++ b/llvm/unittests/ADT/APFloatTest.cpp
@@ -47,6 +47,37 @@ static std::string convertToString(double d, unsigned Prec, unsigned Pad,
   return std::string(Buffer.data(), Buffer.size());
 }
 
+namespace llvm {
+namespace detail {
+class IEEEFloatUnitTestHelper {
+public:
+  static void runTest(bool subtract, bool lhsSign,
+                      APFloat::ExponentType lhsExponent,
+                      APFloat::integerPart lhsSignificand, bool rhsSign,
+                      APFloat::ExponentType rhsExponent,
+                      APFloat::integerPart rhsSignificand, bool expectedSign,
+                      APFloat::ExponentType expectedExponent,
+                      APFloat::integerPart expectedSignificand,
+                      lostFraction expectedLoss) {
+    // `addOrSubtractSignificand` only uses the sign, exponent and significand
+    IEEEFloat lhs(1.0);
+    lhs.sign = lhsSign;
+    lhs.exponent = lhsExponent;
+    lhs.significand.part = lhsSignificand;
+    IEEEFloat rhs(1.0);
+    rhs.sign = rhsSign;
+    rhs.exponent = rhsExponent;
+    rhs.significand.part = rhsSignificand;
+    lostFraction resultLoss = lhs.addOrSubtractSignificand(rhs, subtract);
+    EXPECT_EQ(resultLoss, expectedLoss);
+    EXPECT_EQ(lhs.sign, expectedSign);
+    EXPECT_EQ(lhs.exponent, expectedExponent);
+    EXPECT_EQ(lhs.significand.part, expectedSignificand);
+  }
+};
+} // namespace detail
+} // namespace llvm
+
 namespace {
 
 TEST(APFloatTest, isSignaling) {
@@ -560,6 +591,104 @@ TEST(APFloatTest, FMA) {
     EXPECT_EQ(-8.85242279E-41f, f1.convertToFloat());
   }
 
+  // The `addOrSubtractSignificand` can be considered to have 9 possible cases
+  // when subtracting: all combinations of {cmpLessThan, cmpGreaterThan,
+  // cmpEqual} and {no loss, loss from lhs, loss from rhs}. Test each reachable
+  // case here.
+
+  // Regression test for failing the `assert(!carry)` in
+  // `addOrSubtractSignificand` and normalizing the exponent even when the
+  // significand is zero if there is a lost fraction.
+  // This tests cmpEqual, loss from lhs
+  {
+    APFloat f1(-1.4728589E-38f);
+    APFloat f2(3.7105144E-6f);
+    APFloat f3(5.5E-44f);
+    f1.fusedMultiplyAdd(f2, f3, APFloat::rmNearestTiesToEven);
+    EXPECT_EQ(-0.0f, f1.convertToFloat());
+  }
+
+  // Test cmpGreaterThan, no loss
+  {
+    APFloat f1(2.0f);
+    APFloat f2(2.0f);
+    APFloat f3(-3.5f);
+    f1.fusedMultiplyAdd(f2, f3, APFloat::rmNearestTiesToEven);
+    EXPECT_EQ(0.5f, f1.convertToFloat());
+  }
+
+  // Test cmpLessThan, no loss
+  {
+    APFloat f1(2.0f);
+    APFloat f2(2.0f);
+    APFloat f3(-4.5f);
+    f1.fusedMultiplyAdd(f2, f3, APFloat::rmNearestTiesToEven);
+    EXPECT_EQ(-0.5f, f1.convertToFloat());
+  }
+
+  // Test cmpEqual, no loss
+  {
+    APFloat f1(2.0f);
+    APFloat f2(2.0f);
+    APFloat f3(-4.0f);
+    f1.fusedMultiplyAdd(f2, f3, APFloat::rmNearestTiesToEven);
+    EXPECT_EQ(0.0f, f1.convertToFloat());
+  }
+
+  // Test cmpLessThan, loss from lhs
+  {
+    APFloat f1(2.0000002f);
+    APFloat f2(2.0000002f);
+    APFloat f3(-32.0f);
+    f1.fusedMultiplyAdd(f2, f3, APFloat::rmNearestTiesToEven);
+    EXPECT_EQ(-27.999998f, f1.convertToFloat());
+  }
+
+  // Test cmpGreaterThan, loss from rhs
+  {
+    APFloat f1(1e10f);
+    APFloat f2(1e10f);
+    APFloat f3(-2.0000002f);
+    f1.fusedMultiplyAdd(f2, f3, APFloat::rmNearestTiesToEven);
+    EXPECT_EQ(1e20f, f1.convertToFloat());
+  }
+
+  // Test cmpGreaterThan, loss from lhs
+  {
+    APFloat f1(1e-36f);
+    APFloat f2(0.0019531252f);
+    APFloat f3(-1e-45f);
+    f1.fusedMultiplyAdd(f2, f3, APFloat::rmNearestTiesToEven);
+    EXPECT_EQ(1.953124e-39f, f1.convertToFloat());
+  }
+
+  // {cmpEqual, cmpLessThan} with loss from rhs can't occur for the usage in
+  // `fusedMultiplyAdd` as `multiplySignificand` normalises the MSB of lhs to
+  // one bit below the top.
+
+  // Test cases from #104984
+  {
+    APFloat f1(0.24999998f);
+    APFloat f2(2.3509885e-38f);
+    APFloat f3(-1e-45f);
+    f1.fusedMultiplyAdd(f2, f3, APFloat::rmNearestTiesToEven);
+    EXPECT_EQ(5.87747e-39f, f1.convertToFloat());
+  }
+  {
+    APFloat f1(4.4501477170144023e-308);
+    APFloat f2(0.24999999999999997);
+    APFloat f3(-8.475904604373977e-309);
+    f1.fusedMultiplyAdd(f2, f3, APFloat::rmNearestTiesToEven);
+    EXPECT_EQ(2.64946468816203e-309, f1.convertToDouble());
+  }
+  {
+    APFloat f1(APFloat::IEEEhalf(), APInt(16, 0x8fffu));
+    APFloat f2(APFloat::IEEEhalf(), APInt(16, 0x2bffu));
+    APFloat f3(APFloat::IEEEhalf(), APInt(16, 0x0172u));
+    f1.fusedMultiplyAdd(f2, f3, APFloat::rmNearestTiesToEven);
+    EXPECT_EQ(0x808eu, f1.bitcastToAPInt().getZExtValue());
+  }
+
   // Test using only a single instance of APFloat.
   {
     APFloat F(1.5);
@@ -8168,4 +8297,54 @@ TEST(APFloatTest, Float4E2M1FNToFloat) {
   EXPECT_TRUE(SmallestDenorm.isDenormal());
   EXPECT_EQ(0x0.8p0, SmallestDenorm.convertToFloat());
 }
+
+TEST(APFloatTest, AddOrSubtractSignificand) {
+  typedef detail::IEEEFloatUnitTestHelper Helper;
+  // Test cases are all combinations of:
+  // {equal exponents, LHS larger exponent, RHS larger exponent}
+  // {equal significands, LHS larger significand, RHS larger significand}
+  // {no loss, loss}
+
+  // Equal exponents (loss cannot occur as their is no shifting)
+  Helper::runTest(true, false, 1, 0x10, false, 1, 0x5, false, 1, 0xb,
+                  lfExactlyZero);
+  Helper::runTest(false, false, -2, 0x20, true, -2, 0x20, false, -2, 0,
+                  lfExactlyZero);
+  Helper::runTest(false, true, 3, 0x20, false, 3, 0x30, false, 3, 0x10,
+                  lfExactlyZero);
+
+  // LHS larger exponent
+  // LHS significand greater after shitfing
+  Helper::runTest(true, false, 7, 0x100, false, 3, 0x100, false, 6, 0x1e0,
+                  lfExactlyZero);
+  Helper::runTest(true, false, 7, 0x100, false, 3, 0x101, false, 6, 0x1df,
+                  lfMoreThanHalf);
+  // Significands equal after shitfing
+  Helper::runTest(true, false, 7, 0x100, false, 3, 0x1000, false, 6, 0,
+                  lfExactlyZero);
+  Helper::runTest(true, false, 7, 0x100, false, 3, 0x1001, true, 6, 0,
+                  lfLessThanHalf);
+  // RHS significand greater after shitfing
+  Helper::runTest(true, false, 7, 0x100, false, 3, 0x10000, true, 6, 0x1e00,
+                  lfExactlyZero);
+  Helper::runTest(true, false, 7, 0x100, false, 3, 0x10001, true, 6, 0x1e00,
+                  lfLessThanHalf);
+
+  // RHS larger exponent
+  // RHS significand greater after shitfing
+  Helper::runTest(true, false, 3, 0x100, false, 7, 0x100, true, 6, 0x1e0,
+                  lfExactlyZero);
+  Helper::runTest(true, false, 3, 0x101, false, 7, 0x100, true, 6, 0x1df,
+                  lfMoreThanHalf);
+  // Significands equal after shitfing
+  Helper::runTest(true, false, 3, 0x1000, false, 7, 0x100, false, 6, 0,
+                  lfExactlyZero);
+  Helper::runTest(true, false, 3, 0x1001, false, 7, 0x100, false, 6, 0,
+                  lfLessThanHalf);
+  // LHS significand greater after shitfing
+  Helper::runTest(true, false, 3, 0x10000, false, 7, 0x100, false, 6, 0x1e00,
+                  lfExactlyZero);
+  Helper::runTest(true, false, 3, 0x10001, false, 7, 0x100, false, 6, 0x1e00,
+                  lfLessThanHalf);
+}
 } // namespace
diff --git a/llvm/unittests/ADT/EquivalenceClassesTest.cpp b/llvm/unittests/ADT/EquivalenceClassesTest.cpp
index 70e161a03d988..bfb7c8d185fc8 100644
--- a/llvm/unittests/ADT/EquivalenceClassesTest.cpp
+++ b/llvm/unittests/ADT/EquivalenceClassesTest.cpp
@@ -13,6 +13,15 @@ using namespace llvm;
 
 namespace llvm {
 
+TEST(EquivalenceClassesTest, CopyAssignemnt) {
+  EquivalenceClasses<int> EC, Copy;
+  EC.insert(1);
+  EC.insert(4);
+  EquivalenceClasses<int> &Ref = Copy = EC;
+  EXPECT_EQ(Copy.getNumClasses(), 2u);
+  EXPECT_EQ(&Ref, &Copy);
+}
+
 TEST(EquivalenceClassesTest, NoMerges) {
   EquivalenceClasses<int> EqClasses;
   // Until we merged any sets, check that every element is only equivalent to
diff --git a/llvm/unittests/ADT/IteratorTest.cpp b/llvm/unittests/ADT/IteratorTest.cpp
index a0d3c9b564d85..691fbce5080ff 100644
--- a/llvm/unittests/ADT/IteratorTest.cpp
+++ b/llvm/unittests/ADT/IteratorTest.cpp
@@ -22,6 +22,22 @@ using testing::ElementsAre;
 
 namespace {
 
+namespace adl_test {
+struct WithFreeBeginEnd {
+  int data[3] = {21, 22, 23};
+};
+
+auto begin(const WithFreeBeginEnd &X) { return std::begin(X.data); }
+auto end(const WithFreeBeginEnd &X) { return std::end(X.data); }
+
+struct WithFreeRBeginREnd {
+  int data[3] = {42, 43, 44};
+};
+
+auto rbegin(const WithFreeRBeginREnd &X) { return std::rbegin(X.data); }
+auto rend(const WithFreeRBeginREnd &X) { return std::rend(X.data); }
+} // namespace adl_test
+
 template <int> struct Shadow;
 
 struct WeirdIter
@@ -364,6 +380,14 @@ TEST(FilterIteratorTest, ReverseFilterRange) {
   EXPECT_EQ((SmallVector<int, 4>{6, 4, 2, 0}), Actual4);
 }
 
+TEST(FilterIteratorTest, ADL) {
+  // Make sure that we use the `begin`/`end` functions
+  // from `adl_test`, using ADL.
+  adl_test::WithFreeBeginEnd R;
+  auto IsOdd = [](int N) { return N % 2 != 0; };
+  EXPECT_THAT(make_filter_range(R, IsOdd), ElementsAre(21, 23));
+}
+
 TEST(PointerIterator, Basic) {
   int A[] = {1, 2, 3, 4};
   pointer_iterator<int *> Begin(std::begin(A)), End(std::end(A));
@@ -395,18 +419,9 @@ TEST(PointerIterator, Range) {
     EXPECT_EQ(A + I++, P);
 }
 
-namespace rbegin_detail {
-struct WithFreeRBegin {
-  int data[3] = {42, 43, 44};
-};
-
-auto rbegin(const WithFreeRBegin &X) { return std::rbegin(X.data); }
-auto rend(const WithFreeRBegin &X) { return std::rend(X.data); }
-} // namespace rbegin_detail
-
 TEST(ReverseTest, ADL) {
   // Check that we can find the rbegin/rend functions via ADL.
-  rbegin_detail::WithFreeRBegin Foo;
+  adl_test::WithFreeRBeginREnd Foo;
   EXPECT_THAT(reverse(Foo), ElementsAre(44, 43, 42));
 }
 
diff --git a/llvm/unittests/ADT/STLExtrasTest.cpp b/llvm/unittests/ADT/STLExtrasTest.cpp
index 406ff2bc16073..dbb094b0a3088 100644
--- a/llvm/unittests/ADT/STLExtrasTest.cpp
+++ b/llvm/unittests/ADT/STLExtrasTest.cpp
@@ -421,6 +421,24 @@ void swap(some_struct &lhs, some_struct &rhs) {
   rhs.swap_val = "rhs";
 }
 
+struct List {
+  std::list<int> data;
+};
+
+std::list<int>::const_iterator begin(const List &list) {
+  return list.data.begin();
+}
+std::list<int>::const_iterator end(const List &list) { return list.data.end(); }
+
+struct Pairs {
+  std::vector<std::pair<std::string, int>> data;
+  using const_iterator =
+      std::vector<std::pair<std::string, int>>::const_iterator;
+};
+
+Pairs::const_iterator begin(const Pairs &p) { return p.data.begin(); }
+Pairs::const_iterator end(const Pairs &p) { return p.data.end(); }
+
 struct requires_move {};
 int *begin(requires_move &&) { return nullptr; }
 int *end(requires_move &&) { return nullptr; }
@@ -504,6 +522,25 @@ TEST(STLExtrasTest, ConcatRange) {
   EXPECT_EQ(Expected, Test);
 }
 
+TEST(STLExtrasTest, ConcatRangeADL) {
+  // Make sure that we use the `begin`/`end` functions from `some_namespace`,
+  // using ADL.
+  some_namespace::some_struct S0;
+  S0.data = {1, 2};
+  some_namespace::some_struct S1;
+  S1.data = {3, 4};
+  EXPECT_THAT(concat<const int>(S0, S1), ElementsAre(1, 2, 3, 4));
+}
+
+TEST(STLExtrasTest, MakeFirstSecondRangeADL) {
+  // Make sure that we use the `begin`/`end` functions from `some_namespace`,
+  // using ADL.
+  some_namespace::Pairs Pairs;
+  Pairs.data = {{"foo", 1}, {"bar", 2}};
+  EXPECT_THAT(make_first_range(Pairs), ElementsAre("foo", "bar"));
+  EXPECT_THAT(make_second_range(Pairs), ElementsAre(1, 2));
+}
+
 template <typename T> struct Iterator {
   int i = 0;
   T operator*() const { return i; }
@@ -730,6 +767,18 @@ TEST(STLExtrasTest, DropEndDefaultTest) {
   EXPECT_EQ(i, 4);
 }
 
+TEST(STLExtrasTest, MapRangeTest) {
+  SmallVector<int, 5> Vec{0, 1, 2};
+  EXPECT_THAT(map_range(Vec, [](int V) { return V + 1; }),
+              ElementsAre(1, 2, 3));
+
+  // Make sure that we use the `begin`/`end` functions
+  // from `some_namespace`, using ADL.
+  some_namespace::some_struct S;
+  S.data = {3, 4, 5};
+  EXPECT_THAT(map_range(S, [](int V) { return V * 2; }), ElementsAre(6, 8, 10));
+}
+
 TEST(STLExtrasTest, EarlyIncrementTest) {
   std::list<int> L = {1, 2, 3, 4};
 
@@ -767,7 +816,7 @@ TEST(STLExtrasTest, EarlyIncrementTest) {
 #endif
 
   // Inserting shouldn't break anything. We should be able to keep dereferencing
-  // the currrent iterator and increment. The increment to go to the "next"
+  // the current iterator and increment. The increment to go to the "next"
   // iterator from before we inserted.
   L.insert(std::next(L.begin(), 2), -1);
   ++I;
@@ -781,6 +830,14 @@ TEST(STLExtrasTest, EarlyIncrementTest) {
   EXPECT_EQ(EIR.end(), I);
 }
 
+TEST(STLExtrasTest, EarlyIncADLTest) {
+  // Make sure that we use the `begin`/`end` functions from `some_namespace`,
+  // using ADL.
+  some_namespace::some_struct S;
+  S.data = {1, 2, 3};
+  EXPECT_THAT(make_early_inc_range(S), ElementsAre(1, 2, 3));
+}
+
 // A custom iterator that returns a pointer when dereferenced. This is used to
 // test make_early_inc_range with iterators that do not return a reference on
 // dereferencing.
@@ -854,6 +911,16 @@ TEST(STLExtrasTest, EarlyIncrementTestCustomPointerIterator) {
   EXPECT_EQ(EIR.end(), I);
 }
 
+TEST(STLExtrasTest, ReplaceADL) {
+  // Make sure that we use the `begin`/`end` functions from `some_namespace`,
+  // using ADL.
+  std::vector<int> Cont = {0, 1, 2, 3, 4, 5};
+  some_namespace::some_struct S;
+  S.data = {42, 43, 44};
+  replace(Cont, Cont.begin() + 2, Cont.begin() + 5, S);
+  EXPECT_THAT(Cont, ElementsAre(0, 1, 42, 43, 44, 5));
+}
+
 TEST(STLExtrasTest, AllEqual) {
   std::vector<int> V;
   EXPECT_TRUE(all_equal(V));
@@ -934,10 +1001,19 @@ TEST(STLExtrasTest, hasSingleElement) {
   const std::vector<int> V0 = {}, V1 = {1}, V2 = {1, 2};
   const std::vector<int> V10(10);
 
-  EXPECT_EQ(hasSingleElement(V0), false);
-  EXPECT_EQ(hasSingleElement(V1), true);
-  EXPECT_EQ(hasSingleElement(V2), false);
-  EXPECT_EQ(hasSingleElement(V10), false);
+  EXPECT_FALSE(hasSingleElement(V0));
+  EXPECT_TRUE(hasSingleElement(V1));
+  EXPECT_FALSE(hasSingleElement(V2));
+  EXPECT_FALSE(hasSingleElement(V10));
+
+  // Make sure that we use the `begin`/`end` functions
+  // from `some_namespace`, using ADL.
+  some_namespace::some_struct S;
+  EXPECT_FALSE(hasSingleElement(S));
+  S.data = V1;
+  EXPECT_TRUE(hasSingleElement(S));
+  S.data = V2;
+  EXPECT_FALSE(hasSingleElement(S));
 }
 
 TEST(STLExtrasTest, hasNItems) {
@@ -952,6 +1028,13 @@ TEST(STLExtrasTest, hasNItems) {
   EXPECT_TRUE(hasNItems(V3.begin(), V3.end(), 3, [](int x) { return x < 10; }));
   EXPECT_TRUE(hasNItems(V3.begin(), V3.end(), 0, [](int x) { return x > 10; }));
   EXPECT_TRUE(hasNItems(V3.begin(), V3.end(), 2, [](int x) { return x < 5; }));
+
+  // Make sure that we use the `begin`/`end` functions from `some_namespace`,
+  // using ADL.
+  some_namespace::List L;
+  L.data = {0, 1, 2};
+  EXPECT_FALSE(hasNItems(L, 2));
+  EXPECT_TRUE(hasNItems(L, 3));
 }
 
 TEST(STLExtras, hasNItemsOrMore) {
@@ -974,6 +1057,13 @@ TEST(STLExtras, hasNItemsOrMore) {
       hasNItemsOrMore(V3.begin(), V3.end(), 3, [](int x) { return x > 10; }));
   EXPECT_TRUE(
       hasNItemsOrMore(V3.begin(), V3.end(), 2, [](int x) { return x < 5; }));
+
+  // Make sure that we use the `begin`/`end` functions from `some_namespace`,
+  // using ADL.
+  some_namespace::List L;
+  L.data = {0, 1, 2};
+  EXPECT_TRUE(hasNItemsOrMore(L, 1));
+  EXPECT_FALSE(hasNItems(L, 4));
 }
 
 TEST(STLExtras, hasNItemsOrLess) {
@@ -1007,6 +1097,13 @@ TEST(STLExtras, hasNItemsOrLess) {
       hasNItemsOrLess(V3.begin(), V3.end(), 5, [](int x) { return x < 5; }));
   EXPECT_FALSE(
       hasNItemsOrLess(V3.begin(), V3.end(), 2, [](int x) { return x < 10; }));
+
+  // Make sure that we use the `begin`/`end` functions from `some_namespace`,
+  // using ADL.
+  some_namespace::List L;
+  L.data = {0, 1, 2};
+  EXPECT_FALSE(hasNItemsOrLess(L, 1));
+  EXPECT_TRUE(hasNItemsOrLess(L, 4));
 }
 
 TEST(STLExtras, MoveRange) {
diff --git a/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp b/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp
index 40cd055c1c3f8..25eb67e981588 100644
--- a/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp
@@ -950,6 +950,29 @@ TEST_F(AArch64GISelMITest, DeferredMatching) {
                        m_GAdd(m_Reg(X), m_GSub(m_Reg(), m_DeferredReg(X)))));
 }
 
+TEST_F(AArch64GISelMITest, AddLike) {
+  setUp();
+  if (!TM)
+    GTEST_SKIP();
+  auto s64 = LLT::scalar(64);
+
+  auto Cst1 = B.buildConstant(s64, 42);
+  auto Cst2 = B.buildConstant(s64, 314);
+
+  auto Or1 = B.buildOr(s64, Cst1, Cst2, MachineInstr::Disjoint);
+  auto Or2 = B.buildOr(s64, Cst1, Cst2);
+  auto Add = B.buildAdd(s64, Cst1, Cst2);
+  auto Sub = B.buildSub(s64, Cst1, Cst2);
+
+  EXPECT_TRUE(mi_match(Or1.getReg(0), *MRI, m_GDisjointOr(m_Reg(), m_Reg())));
+  EXPECT_FALSE(mi_match(Or2.getReg(0), *MRI, m_GDisjointOr(m_Reg(), m_Reg())));
+
+  EXPECT_TRUE(mi_match(Add.getReg(0), *MRI, m_GAddLike(m_Reg(), m_Reg())));
+  EXPECT_FALSE(mi_match(Sub.getReg(0), *MRI, m_GAddLike(m_Reg(), m_Reg())));
+  EXPECT_TRUE(mi_match(Or1.getReg(0), *MRI, m_GAddLike(m_Reg(), m_Reg())));
+  EXPECT_FALSE(mi_match(Or2.getReg(0), *MRI, m_GAddLike(m_Reg(), m_Reg())));
+}
+
 } // namespace
 
 int main(int argc, char **argv) {
diff --git a/llvm/unittests/ExecutionEngine/Orc/WrapperFunctionUtilsTest.cpp b/llvm/unittests/ExecutionEngine/Orc/WrapperFunctionUtilsTest.cpp
index 1ed690ae9b986..8de2412fed4d0 100644
--- a/llvm/unittests/ExecutionEngine/Orc/WrapperFunctionUtilsTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/WrapperFunctionUtilsTest.cpp
@@ -118,8 +118,9 @@ static WrapperFunctionResult voidNoopAsyncWrapper(const char *ArgData,
   auto RF = RP.get_future();
 
   WrapperFunction<void()>::handleAsync(
-      ArgData, ArgSize, voidNoopAsync,
-      [&](WrapperFunctionResult R) { RP.set_value(std::move(R)); });
+      ArgData, ArgSize,
+      [&](WrapperFunctionResult R) { RP.set_value(std::move(R)); },
+      voidNoopAsync);
 
   return RF.get();
 }
@@ -131,10 +132,10 @@ static WrapperFunctionResult addAsyncWrapper(const char *ArgData,
 
   WrapperFunction<int32_t(int32_t, int32_t)>::handleAsync(
       ArgData, ArgSize,
+      [&](WrapperFunctionResult R) { RP.set_value(std::move(R)); },
       [](unique_function<void(int32_t)> SendResult, int32_t X, int32_t Y) {
         SendResult(X + Y);
-      },
-      [&](WrapperFunctionResult R) { RP.set_value(std::move(R)); });
+      });
   return RF.get();
 }
 
diff --git a/llvm/unittests/Support/ARMAttributeParser.cpp b/llvm/unittests/Support/ARMAttributeParser.cpp
index a0568262b4da9..e06f7fab85fa7 100644
--- a/llvm/unittests/Support/ARMAttributeParser.cpp
+++ b/llvm/unittests/Support/ARMAttributeParser.cpp
@@ -37,7 +37,7 @@ bool testBuildAttr(unsigned Tag, unsigned Value,
   ARMAttributeParser Parser;
   cantFail(Parser.parse(Bytes, llvm::endianness::little));
 
-  std::optional<unsigned> Attr = Parser.getAttributeValue(ExpectedTag);
+  std::optional<unsigned> Attr = Parser.getAttributeValue("", ExpectedTag);
   return Attr && *Attr == ExpectedValue;
 }
 
diff --git a/llvm/unittests/Support/CMakeLists.txt b/llvm/unittests/Support/CMakeLists.txt
index 6de8165826442..6c4e7cb689b20 100644
--- a/llvm/unittests/Support/CMakeLists.txt
+++ b/llvm/unittests/Support/CMakeLists.txt
@@ -61,6 +61,7 @@ add_llvm_unittest(SupportTests
   MemoryBufferRefTest.cpp
   MemoryBufferTest.cpp
   MemoryTest.cpp
+  MustacheTest.cpp      
   ModRefTest.cpp
   NativeFormatTests.cpp
   OptimizedStructLayoutTest.cpp
diff --git a/llvm/unittests/Support/CSKYAttributeParserTest.cpp b/llvm/unittests/Support/CSKYAttributeParserTest.cpp
index 2c5d93b0b7565..25b151fe5c1c1 100644
--- a/llvm/unittests/Support/CSKYAttributeParserTest.cpp
+++ b/llvm/unittests/Support/CSKYAttributeParserTest.cpp
@@ -83,7 +83,7 @@ static bool testAttributeInt(unsigned Tag, unsigned Value, unsigned ExpectedTag,
   CSKYAttributeParser Parser;
   cantFail(Parser.parse(Bytes, llvm::endianness::little));
 
-  std::optional<unsigned> Attr = Parser.getAttributeValue(ExpectedTag);
+  std::optional<unsigned> Attr = Parser.getAttributeValue("", ExpectedTag);
   return Attr && *Attr == ExpectedValue;
 }
 
@@ -100,7 +100,7 @@ static bool testAttributeString(unsigned Tag, const char *Value,
   CSKYAttributeParser Parser;
   cantFail(Parser.parse(Bytes, llvm::endianness::little));
 
-  std::optional<StringRef> Attr = Parser.getAttributeString(ExpectedTag);
+  std::optional<StringRef> Attr = Parser.getAttributeString("", ExpectedTag);
   return Attr && *Attr == ExpectedValue;
 }
 
diff --git a/llvm/unittests/Support/ELFAttributeParserTest.cpp b/llvm/unittests/Support/ELFAttributeParserTest.cpp
index 38e7b09cc3c7d..31cc594188e33 100644
--- a/llvm/unittests/Support/ELFAttributeParserTest.cpp
+++ b/llvm/unittests/Support/ELFAttributeParserTest.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Support/ELFAttributeParser.h"
+#include "llvm/Support/ELFAttrParserCompact.h"
 #include "llvm/Support/ELFAttributes.h"
 #include "gtest/gtest.h"
 #include <string>
@@ -16,7 +16,7 @@ using namespace llvm;
 static const TagNameMap emptyTagNameMap;
 
 // This class is used to test the common part of the ELF attribute section.
-class AttributeHeaderParser : public ELFAttributeParser {
+class AttributeHeaderParser : public ELFCompactAttrParser {
   Error handler(uint64_t tag, bool &handled) override {
     // Treat all attributes as handled.
     handled = true;
@@ -25,8 +25,8 @@ class AttributeHeaderParser : public ELFAttributeParser {
 
 public:
   AttributeHeaderParser(ScopedPrinter *printer)
-      : ELFAttributeParser(printer, emptyTagNameMap, "test") {}
-  AttributeHeaderParser() : ELFAttributeParser(emptyTagNameMap, "test") {}
+      : ELFCompactAttrParser(printer, emptyTagNameMap, "test") {}
+  AttributeHeaderParser() : ELFCompactAttrParser(emptyTagNameMap, "test") {}
 };
 
 static void testParseError(ArrayRef<uint8_t> bytes, const char *msg) {
diff --git a/llvm/unittests/Support/LockFileManagerTest.cpp b/llvm/unittests/Support/LockFileManagerTest.cpp
index 552053d46e843..627b2daef650c 100644
--- a/llvm/unittests/Support/LockFileManagerTest.cpp
+++ b/llvm/unittests/Support/LockFileManagerTest.cpp
@@ -9,6 +9,7 @@
 #include "llvm/Support/LockFileManager.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Testing/Support/Error.h"
 #include "llvm/Testing/Support/SupportHelpers.h"
 #include "gtest/gtest.h"
 #include <memory>
@@ -27,12 +28,12 @@ TEST(LockFileManagerTest, Basic) {
   {
     // The lock file should not exist, so we should successfully acquire it.
     LockFileManager Locked1(LockedFile);
-    EXPECT_EQ(LockFileManager::LFS_Owned, Locked1.getState());
+    EXPECT_THAT_EXPECTED(Locked1.tryLock(), HasValue(true));
 
     // Attempting to reacquire the lock should fail.  Waiting on it would cause
     // deadlock, so don't try that.
     LockFileManager Locked2(LockedFile);
-    EXPECT_NE(LockFileManager::LFS_Owned, Locked2.getState());
+    EXPECT_THAT_EXPECTED(Locked2.tryLock(), HasValue(false));
   }
 
   // Now that the lock is out of scope, the file should be gone.
@@ -68,7 +69,7 @@ TEST(LockFileManagerTest, LinkLockExists) {
     // The lock file doesn't point to a real file, so we should successfully
     // acquire it.
     LockFileManager Locked(LockedFile);
-    EXPECT_EQ(LockFileManager::LFS_Owned, Locked.getState());
+    EXPECT_THAT_EXPECTED(Locked.tryLock(), HasValue(true));
   }
 
   // Now that the lock is out of scope, the file should be gone.
@@ -93,7 +94,7 @@ TEST(LockFileManagerTest, RelativePath) {
   {
     // The lock file should not exist, so we should successfully acquire it.
     LockFileManager Locked(LockedFile);
-    EXPECT_EQ(LockFileManager::LFS_Owned, Locked.getState());
+    EXPECT_THAT_EXPECTED(Locked.tryLock(), HasValue(true));
     EXPECT_TRUE(sys::fs::exists(FileLock.str()));
   }
 
diff --git a/llvm/unittests/Support/MustacheTest.cpp b/llvm/unittests/Support/MustacheTest.cpp
new file mode 100644
index 0000000000000..6ab3d4b01bc1b
--- /dev/null
+++ b/llvm/unittests/Support/MustacheTest.cpp
@@ -0,0 +1,1226 @@
+//===- llvm/unittest/Support/MustacheTest.cpp ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Test conforming to Mustache 1.4.2 spec found here:
+// https://github.com/mustache/spec
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/Mustache.h"
+#include "llvm/Support/raw_ostream.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::mustache;
+using namespace llvm::json;
+
+TEST(MustacheInterpolation, NoInterpolation) {
+  // Mustache-free templates should render as-is.
+  Value D = {};
+  auto T = Template("Hello from {Mustache}!\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Hello from {Mustache}!\n", Out);
+}
+
+TEST(MustacheInterpolation, BasicInterpolation) {
+  // Unadorned tags should interpolate content into the template.
+  Value D = Object{{"subject", "World"}};
+  auto T = Template("Hello, {{subject}}!");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Hello, World!", Out);
+}
+
+TEST(MustacheInterpolation, NoReinterpolation) {
+  // Interpolated tag output should not be re-interpolated.
+  Value D = Object{{"template", "{{planet}}"}, {"planet", "Earth"}};
+  auto T = Template("{{template}}: {{planet}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("{{planet}}: Earth", Out);
+}
+
+TEST(MustacheInterpolation, HTMLEscaping) {
+  // Interpolated tag output should not be re-interpolated.
+  Value D = Object{
+      {"forbidden", "& \" < >"},
+  };
+  auto T = Template("These characters should be HTML escaped: {{forbidden}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("These characters should be HTML escaped: &amp; &quot; &lt; &gt;\n",
+            Out);
+}
+
+TEST(MustacheInterpolation, Ampersand) {
+  // Interpolated tag output should not be re-interpolated.
+  Value D = Object{
+      {"forbidden", "& \" < >"},
+  };
+  auto T =
+      Template("These characters should not be HTML escaped: {{&forbidden}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("These characters should not be HTML escaped: & \" < >\n", Out);
+}
+
+TEST(MustacheInterpolation, BasicIntegerInterpolation) {
+  // Integers should interpolate seamlessly.
+  Value D = Object{{"mph", 85}};
+  auto T = Template("{{mph}} miles an hour!");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("85 miles an hour!", Out);
+}
+
+TEST(MustacheInterpolation, AmpersandIntegerInterpolation) {
+  // Integers should interpolate seamlessly.
+  Value D = Object{{"mph", 85}};
+  auto T = Template("{{&mph}} miles an hour!");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("85 miles an hour!", Out);
+}
+
+TEST(MustacheInterpolation, BasicDecimalInterpolation) {
+  // Decimals should interpolate seamlessly with proper significance.
+  Value D = Object{{"power", 1.21}};
+  auto T = Template("{{power}} jiggawatts!");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("1.21 jiggawatts!", Out);
+}
+
+TEST(MustacheInterpolation, BasicNullInterpolation) {
+  // Nulls should interpolate as the empty string.
+  Value D = Object{{"cannot", nullptr}};
+  auto T = Template("I ({{cannot}}) be seen!");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("I () be seen!", Out);
+}
+
+TEST(MustacheInterpolation, AmpersandNullInterpolation) {
+  // Nulls should interpolate as the empty string.
+  Value D = Object{{"cannot", nullptr}};
+  auto T = Template("I ({{&cannot}}) be seen!");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("I () be seen!", Out);
+}
+
+TEST(MustacheInterpolation, BasicContextMissInterpolation) {
+  // Failed context lookups should default to empty strings.
+  Value D = Object{};
+  auto T = Template("I ({{cannot}}) be seen!");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("I () be seen!", Out);
+}
+
+TEST(MustacheInterpolation, DottedNamesBasicInterpolation) {
+  // Dotted names should be considered a form of shorthand for sections.
+  Value D = Object{{"person", Object{{"name", "Joe"}}}};
+  auto T = Template("{{person.name}} == {{#person}}{{name}}{{/person}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Joe == Joe", Out);
+}
+
+TEST(MustacheInterpolation, DottedNamesAmpersandInterpolation) {
+  // Dotted names should be considered a form of shorthand for sections.
+  Value D = Object{{"person", Object{{"name", "Joe"}}}};
+  auto T = Template("{{&person.name}} == {{#person}}{{&name}}{{/person}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Joe == Joe", Out);
+}
+
+TEST(MustacheInterpolation, DottedNamesArbitraryDepth) {
+  // Dotted names should be functional to any level of nesting.
+  Value D = Object{
+      {"a",
+       Object{{"b",
+               Object{{"c",
+                       Object{{"d",
+                               Object{{"e", Object{{"name", "Phil"}}}}}}}}}}}};
+  auto T = Template("{{a.b.c.d.e.name}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Phil", Out);
+}
+
+TEST(MustacheInterpolation, DottedNamesBrokenChains) {
+  // Any falsey value prior to the last part of the name should yield ''.
+  Value D = Object{{"a", Object{}}};
+  auto T = Template("{{a.b.c}} == ");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" == ", Out);
+}
+
+TEST(MustacheInterpolation, DottedNamesBrokenChainResolution) {
+  // Each part of a dotted name should resolve only against its parent.
+  Value D =
+      Object{{"a", Object{{"b", Object{}}}}, {"c", Object{{"name", "Jim"}}}};
+  auto T = Template("{{a.b.c.name}} == ");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" == ", Out);
+}
+
+TEST(MustacheInterpolation, DottedNamesInitialResolution) {
+  // The first part of a dotted name should resolve as any other name.
+  Value D = Object{
+      {"a",
+       Object{
+           {"b",
+            Object{{"c",
+                    Object{{"d", Object{{"e", Object{{"name", "Phil"}}}}}}}}}}},
+      {"b",
+       Object{{"c", Object{{"d", Object{{"e", Object{{"name", "Wrong"}}}}}}}}}};
+  auto T = Template("{{#a}}{{b.c.d.e.name}}{{/a}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Phil", Out);
+}
+
+TEST(MustacheInterpolation, DottedNamesContextPrecedence) {
+  // Dotted names should be resolved against former resolutions.
+  Value D =
+      Object{{"a", Object{{"b", Object{}}}}, {"b", Object{{"c", "ERROR"}}}};
+  auto T = Template("{{#a}}{{b.c}}{{/a}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("", Out);
+}
+
+TEST(MustacheInterpolation, DottedNamesAreNotSingleKeys) {
+  // Dotted names shall not be parsed as single, atomic keys
+  Value D = Object{{"a.b", "c"}};
+  auto T = Template("{{a.b}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("", Out);
+}
+
+TEST(MustacheInterpolation, DottedNamesNoMasking) {
+  // Dotted Names in a given context are unavailable due to dot splitting
+  Value D = Object{{"a.b", "c"}, {"a", Object{{"b", "d"}}}};
+  auto T = Template("{{a.b}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("d", Out);
+}
+
+TEST(MustacheInterpolation, ImplicitIteratorsBasicInterpolation) {
+  // Unadorned tags should interpolate content into the template.
+  Value D = "world";
+  auto T = Template("Hello, {{.}}!\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Hello, world!\n", Out);
+}
+
+TEST(MustacheInterpolation, ImplicitIteratorsAmersand) {
+  // Basic interpolation should be HTML escaped.
+  Value D = "& \" < >";
+  auto T = Template("These characters should not be HTML escaped: {{&.}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("These characters should not be HTML escaped: & \" < >\n", Out);
+}
+
+TEST(MustacheInterpolation, ImplicitIteratorsInteger) {
+  // Integers should interpolate seamlessly.
+  Value D = 85;
+  auto T = Template("{{.}} miles an hour!\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("85 miles an hour!\n", Out);
+}
+
+TEST(MustacheInterpolation, InterpolationSurroundingWhitespace) {
+  // Interpolation should not alter surrounding whitespace.
+  Value D = Object{{"string", "---"}};
+  auto T = Template("| {{string}} |");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| --- |", Out);
+}
+
+TEST(MustacheInterpolation, AmersandSurroundingWhitespace) {
+  // Interpolation should not alter surrounding whitespace.
+  Value D = Object{{"string", "---"}};
+  auto T = Template("| {{&string}} |");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| --- |", Out);
+}
+
+TEST(MustacheInterpolation, StandaloneInterpolationWithWhitespace) {
+  // Standalone interpolation should not alter surrounding whitespace.
+  Value D = Object{{"string", "---"}};
+  auto T = Template("  {{string}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("  ---\n", Out);
+}
+
+TEST(MustacheInterpolation, StandaloneAmpersandWithWhitespace) {
+  // Standalone interpolation should not alter surrounding whitespace.
+  Value D = Object{{"string", "---"}};
+  auto T = Template("  {{&string}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("  ---\n", Out);
+}
+
+TEST(MustacheInterpolation, InterpolationWithPadding) {
+  // Superfluous in-tag whitespace should be ignored.
+  Value D = Object{{"string", "---"}};
+  auto T = Template("|{{ string }}|");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("|---|", Out);
+}
+
+TEST(MustacheInterpolation, AmpersandWithPadding) {
+  // Superfluous in-tag whitespace should be ignored.
+  Value D = Object{{"string", "---"}};
+  auto T = Template("|{{& string }}|");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("|---|", Out);
+}
+
+TEST(MustacheInterpolation, InterpolationWithPaddingAndNewlines) {
+  // Superfluous in-tag whitespace should be ignored.
+  Value D = Object{{"string", "---"}};
+  auto T = Template("|{{ string \n\n\n }}|");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("|---|", Out);
+}
+
+TEST(MustacheSections, Truthy) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template("{{#boolean}}This should be rendered.{{/boolean}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("This should be rendered.", Out);
+}
+
+TEST(MustacheSections, Falsey) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template("{{#boolean}}This should not be rendered.{{/boolean}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("", Out);
+}
+
+TEST(MustacheInterpolation, IsFalseyNull) {
+  // Mustache-free templates should render as-is.
+  Value D = Object{{"boolean", nullptr}};
+  auto T = Template("Hello, {{#boolean}}World{{/boolean}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Hello, ", Out);
+}
+
+TEST(MustacheInterpolation, IsFalseyArray) {
+  // Mustache-free templates should render as-is.
+  Value D = Object{{"boolean", Array()}};
+  auto T = Template("Hello, {{#boolean}}World{{/boolean}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Hello, ", Out);
+}
+
+TEST(MustacheInterpolation, IsFalseyObject) {
+  // Mustache-free templates should render as-is.
+  Value D = Object{{"boolean", Object{}}};
+  auto T = Template("Hello, {{#boolean}}World{{/boolean}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Hello, World", Out);
+}
+
+TEST(MustacheInterpolation, DoubleRendering) {
+  // Mustache-free templates should render as-is.
+  Value D1 = Object{{"subject", "World"}};
+  auto T = Template("Hello, {{subject}}!");
+  std::string Out1;
+  raw_string_ostream OS1(Out1);
+  T.render(D1, OS1);
+  EXPECT_EQ("Hello, World!", Out1);
+  std::string Out2;
+  raw_string_ostream OS2(Out2);
+  Value D2 = Object{{"subject", "New World"}};
+  T.render(D2, OS2);
+  EXPECT_EQ("Hello, New World!", Out2);
+}
+
+TEST(MustacheSections, NullIsFalsey) {
+  Value D = Object{{"null", nullptr}};
+  auto T = Template("{{#null}}This should not be rendered.{{/null}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("", Out);
+}
+
+TEST(MustacheSections, Context) {
+  Value D = Object{{"context", Object{{"name", "Joe"}}}};
+  auto T = Template("{{#context}}Hi {{name}}.{{/context}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Hi Joe.", Out);
+}
+
+TEST(MustacheSections, ParentContexts) {
+  Value D = Object{{"a", "foo"},
+                   {"b", "wrong"},
+                   {"sec", Object{{"b", "bar"}}},
+                   {"c", Object{{"d", "baz"}}}};
+  auto T = Template("{{#sec}}{{a}}, {{b}}, {{c.d}}{{/sec}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("foo, bar, baz", Out);
+}
+
+TEST(MustacheSections, VariableTest) {
+  Value D = Object{{"foo", "bar"}};
+  auto T = Template("{{#foo}}{{.}} is {{foo}}{{/foo}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("bar is bar", Out);
+}
+
+TEST(MustacheSections, ListContexts) {
+  Value D = Object{
+      {"tops",
+       Array{Object{
+           {"tname", Object{{"upper", "A"}, {"lower", "a"}}},
+           {"middles",
+            Array{Object{{"mname", "1"},
+                         {"bottoms", Array{Object{{"bname", "x"}},
+                                           Object{{"bname", "y"}}}}}}}}}}};
+  auto T = Template("{{#tops}}"
+                    "{{#middles}}"
+                    "{{tname.lower}}{{mname}}."
+                    "{{#bottoms}}"
+                    "{{tname.upper}}{{mname}}{{bname}}."
+                    "{{/bottoms}}"
+                    "{{/middles}}"
+                    "{{/tops}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("a1.A1x.A1y.", Out);
+}
+
+TEST(MustacheSections, DeeplyNestedContexts) {
+  Value D = Object{
+      {"a", Object{{"one", 1}}},
+      {"b", Object{{"two", 2}}},
+      {"c", Object{{"three", 3}, {"d", Object{{"four", 4}, {"five", 5}}}}}};
+  auto T = Template(
+      "{{#a}}\n{{one}}\n{{#b}}\n{{one}}{{two}}{{one}}\n{{#c}}\n{{one}}{{two}}{{"
+      "three}}{{two}}{{one}}\n{{#d}}\n{{one}}{{two}}{{three}}{{four}}{{three}}{"
+      "{two}}{{one}}\n{{#five}}\n{{one}}{{two}}{{three}}{{four}}{{five}}{{four}"
+      "}{{three}}{{two}}{{one}}\n{{one}}{{two}}{{three}}{{four}}{{.}}6{{.}}{{"
+      "four}}{{three}}{{two}}{{one}}\n{{one}}{{two}}{{three}}{{four}}{{five}}{{"
+      "four}}{{three}}{{two}}{{one}}\n{{/"
+      "five}}\n{{one}}{{two}}{{three}}{{four}}{{three}}{{two}}{{one}}\n{{/"
+      "d}}\n{{one}}{{two}}{{three}}{{two}}{{one}}\n{{/"
+      "c}}\n{{one}}{{two}}{{one}}\n{{/b}}\n{{one}}\n{{/a}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("1\n121\n12321\n1234321\n123454321\n12345654321\n123454321\n1234321"
+            "\n12321\n121\n1\n",
+            Out);
+}
+
+TEST(MustacheSections, List) {
+  Value D = Object{{"list", Array{Object{{"item", 1}}, Object{{"item", 2}},
+                                  Object{{"item", 3}}}}};
+  auto T = Template("{{#list}}{{item}}{{/list}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("123", Out);
+}
+
+TEST(MustacheSections, EmptyList) {
+  Value D = Object{{"list", Array{}}};
+  auto T = Template("{{#list}}Yay lists!{{/list}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("", Out);
+}
+
+TEST(MustacheSections, Doubled) {
+  Value D = Object{{"bool", true}, {"two", "second"}};
+  auto T = Template("{{#bool}}\n* first\n{{/bool}}\n* "
+                    "{{two}}\n{{#bool}}\n* third\n{{/bool}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("* first\n* second\n* third\n", Out);
+}
+
+TEST(MustacheSections, NestedTruthy) {
+  Value D = Object{{"bool", true}};
+  auto T = Template("| A {{#bool}}B {{#bool}}C{{/bool}} D{{/bool}} E |");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| A B C D E |", Out);
+}
+
+TEST(MustacheSections, NestedFalsey) {
+  Value D = Object{{"bool", false}};
+  auto T = Template("| A {{#bool}}B {{#bool}}C{{/bool}} D{{/bool}} E |");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| A  E |", Out);
+}
+
+TEST(MustacheSections, ContextMisses) {
+  Value D = Object{};
+  auto T = Template("[{{#missing}}Found key 'missing'!{{/missing}}]");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("[]", Out);
+}
+
+TEST(MustacheSections, ImplicitIteratorString) {
+  Value D = Object{{"list", Array{"a", "b", "c", "d", "e"}}};
+  auto T = Template("{{#list}}({{.}}){{/list}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("(a)(b)(c)(d)(e)", Out);
+}
+
+TEST(MustacheSections, ImplicitIteratorInteger) {
+  Value D = Object{{"list", Array{1, 2, 3, 4, 5}}};
+  auto T = Template("{{#list}}({{.}}){{/list}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("(1)(2)(3)(4)(5)", Out);
+}
+
+TEST(MustacheSections, ImplicitIteratorArray) {
+  Value D = Object{{"list", Array{Array{1, 2, 3}, Array{"a", "b", "c"}}}};
+  auto T = Template("{{#list}}({{#.}}{{.}}{{/.}}){{/list}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("(123)(abc)", Out);
+}
+
+TEST(MustacheSections, ImplicitIteratorHTMLEscaping) {
+  Value D = Object{{"list", Array{"&", "\"", "<", ">"}}};
+  auto T = Template("{{#list}}({{.}}){{/list}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("(&amp;)(&quot;)(&lt;)(&gt;)", Out);
+}
+
+TEST(MustacheSections, ImplicitIteratorAmpersand) {
+  Value D = Object{{"list", Array{"&", "\"", "<", ">"}}};
+  auto T = Template("{{#list}}({{&.}}){{/list}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("(&)(\")(<)(>)", Out);
+}
+
+TEST(MustacheSections, ImplicitIteratorRootLevel) {
+  Value D = Array{Object{{"value", "a"}}, Object{{"value", "b"}}};
+  auto T = Template("{{#.}}({{value}}){{/.}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("(a)(b)", Out);
+}
+
+TEST(MustacheSections, DottedNamesTruthy) {
+  Value D = Object{{"a", Object{{"b", Object{{"c", true}}}}}};
+  auto T = Template("{{#a.b.c}}Here{{/a.b.c}} == Here");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Here == Here", Out);
+}
+
+TEST(MustacheSections, DottedNamesFalsey) {
+  Value D = Object{{"a", Object{{"b", Object{{"c", false}}}}}};
+  auto T = Template("{{#a.b.c}}Here{{/a.b.c}} == ");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" == ", Out);
+}
+
+TEST(MustacheSections, DottedNamesBrokenChains) {
+  Value D = Object{{"a", Object{}}};
+  auto T = Template("{{#a.b.c}}Here{{/a.b.c}} == ");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" == ", Out);
+}
+
+TEST(MustacheSections, SurroundingWhitespace) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template(" | {{#boolean}}\t|\t{{/boolean}} | \n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" | \t|\t | \n", Out);
+}
+
+TEST(MustacheSections, InternalWhitespace) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template(
+      " | {{#boolean}} {{! Important Whitespace }}\n {{/boolean}} | \n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" |  \n  | \n", Out);
+}
+
+TEST(MustacheSections, IndentedInlineSections) {
+  Value D = Object{{"boolean", true}};
+  auto T =
+      Template(" {{#boolean}}YES{{/boolean}}\n {{#boolean}}GOOD{{/boolean}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" YES\n GOOD\n", Out);
+}
+
+TEST(MustacheSections, StandaloneLines) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template("| This Is\n{{#boolean}}\n|\n{{/boolean}}\n| A Line\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| This Is\n|\n| A Line\n", Out);
+}
+
+TEST(MustacheSections, IndentedStandaloneLines) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template("| This Is\n  {{#boolean}}\n|\n  {{/boolean}}\n| A Line\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| This Is\n|\n| A Line\n", Out);
+}
+
+TEST(MustacheSections, StandaloneLineEndings) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template("|\r\n{{#boolean}}\r\n{{/boolean}}\r\n|");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("|\r\n|", Out);
+}
+
+TEST(MustacheSections, StandaloneWithoutPreviousLine) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template("  {{#boolean}}\n#{{/boolean}}\n/");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("#\n/", Out);
+}
+
+TEST(MustacheSections, StandaloneWithoutNewline) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template("#{{#boolean}}\n/\n  {{/boolean}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("#\n/\n", Out);
+}
+
+TEST(MustacheSections, Padding) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template("|{{# boolean }}={{/ boolean }}|");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("|=|", Out);
+}
+
+TEST(MustacheInvertedSections, Falsey) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template("{{^boolean}}This should be rendered.{{/boolean}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("This should be rendered.", Out);
+}
+
+TEST(MustacheInvertedSections, Truthy) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template("{{^boolean}}This should not be rendered.{{/boolean}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("", Out);
+}
+
+TEST(MustacheInvertedSections, NullIsFalsey) {
+  Value D = Object{{"null", nullptr}};
+  auto T = Template("{{^null}}This should be rendered.{{/null}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("This should be rendered.", Out);
+}
+
+TEST(MustacheInvertedSections, Context) {
+  Value D = Object{{"context", Object{{"name", "Joe"}}}};
+  auto T = Template("{{^context}}Hi {{name}}.{{/context}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("", Out);
+}
+
+TEST(MustacheInvertedSections, List) {
+  Value D = Object{
+      {"list", Array{Object{{"n", 1}}, Object{{"n", 2}}, Object{{"n", 3}}}}};
+  auto T = Template("{{^list}}{{n}}{{/list}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("", Out);
+}
+
+TEST(MustacheInvertedSections, EmptyList) {
+  Value D = Object{{"list", Array{}}};
+  auto T = Template("{{^list}}Yay lists!{{/list}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Yay lists!", Out);
+}
+
+TEST(MustacheInvertedSections, Doubled) {
+  Value D = Object{{"bool", false}, {"two", "second"}};
+  auto T = Template("{{^bool}}\n* first\n{{/bool}}\n* "
+                    "{{two}}\n{{^bool}}\n* third\n{{/bool}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("* first\n* second\n* third\n", Out);
+}
+
+TEST(MustacheInvertedSections, NestedFalsey) {
+  Value D = Object{{"bool", false}};
+  auto T = Template("| A {{^bool}}B {{^bool}}C{{/bool}} D{{/bool}} E |");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| A B C D E |", Out);
+}
+
+TEST(MustacheInvertedSections, NestedTruthy) {
+  Value D = Object{{"bool", true}};
+  auto T = Template("| A {{^bool}}B {{^bool}}C{{/bool}} D{{/bool}} E |");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| A  E |", Out);
+}
+
+TEST(MustacheInvertedSections, ContextMisses) {
+  Value D = Object{};
+  auto T = Template("[{{^missing}}Cannot find key 'missing'!{{/missing}}]");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("[Cannot find key 'missing'!]", Out);
+}
+
+TEST(MustacheInvertedSections, DottedNamesTruthy) {
+  Value D = Object{{"a", Object{{"b", Object{{"c", true}}}}}};
+  auto T = Template("{{^a.b.c}}Not Here{{/a.b.c}} == ");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" == ", Out);
+}
+
+TEST(MustacheInvertedSections, DottedNamesFalsey) {
+  Value D = Object{{"a", Object{{"b", Object{{"c", false}}}}}};
+  auto T = Template("{{^a.b.c}}Not Here{{/a.b.c}} == Not Here");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Not Here == Not Here", Out);
+}
+
+TEST(MustacheInvertedSections, DottedNamesBrokenChains) {
+  Value D = Object{{"a", Object{}}};
+  auto T = Template("{{^a.b.c}}Not Here{{/a.b.c}} == Not Here");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Not Here == Not Here", Out);
+}
+
+TEST(MustacheInvertedSections, SurroundingWhitespace) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template(" | {{^boolean}}\t|\t{{/boolean}} | \n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" | \t|\t | \n", Out);
+}
+
+TEST(MustacheInvertedSections, InternalWhitespace) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template(
+      " | {{^boolean}} {{! Important Whitespace }}\n {{/boolean}} | \n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" |  \n  | \n", Out);
+}
+
+TEST(MustacheInvertedSections, IndentedInlineSections) {
+  Value D = Object{{"boolean", false}};
+  auto T =
+      Template(" {{^boolean}}NO{{/boolean}}\n {{^boolean}}WAY{{/boolean}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" NO\n WAY\n", Out);
+}
+
+TEST(MustacheInvertedSections, StandaloneLines) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template("| This Is\n{{^boolean}}\n|\n{{/boolean}}\n| A Line\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| This Is\n|\n| A Line\n", Out);
+}
+
+TEST(MustacheInvertedSections, StandaloneIndentedLines) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template("| This Is\n  {{^boolean}}\n|\n  {{/boolean}}\n| A Line\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| This Is\n|\n| A Line\n", Out);
+}
+
+TEST(MustacheInvertedSections, StandaloneLineEndings) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template("|\r\n{{^boolean}}\r\n{{/boolean}}\r\n|");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("|\r\n|", Out);
+}
+
+TEST(MustacheInvertedSections, StandaloneWithoutPreviousLine) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template("  {{^boolean}}\n^{{/boolean}}\n/");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("^\n/", Out);
+}
+
+TEST(MustacheInvertedSections, StandaloneWithoutNewline) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template("^{{^boolean}}\n/\n  {{/boolean}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("^\n/\n", Out);
+}
+
+TEST(MustacheInvertedSections, Padding) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template("|{{^ boolean }}={{/ boolean }}|");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("|=|", Out);
+}
+
+TEST(MustachePartials, BasicBehavior) {
+  Value D = Object{};
+  auto T = Template("{{>text}}");
+  T.registerPartial("text", "from partial");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("from partial", Out);
+}
+
+TEST(MustachePartials, FailedLookup) {
+  Value D = Object{};
+  auto T = Template("{{>text}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("", Out);
+}
+
+TEST(MustachePartials, Context) {
+  Value D = Object{{"text", "content"}};
+  auto T = Template("{{>partial}}");
+  T.registerPartial("partial", "*{{text}}*");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("*content*", Out);
+}
+
+TEST(MustachePartials, Recursion) {
+  Value D =
+      Object{{"content", "X"},
+             {"nodes", Array{Object{{"content", "Y"}, {"nodes", Array{}}}}}};
+  auto T = Template("{{>node}}");
+  T.registerPartial("node", "{{content}}({{#nodes}}{{>node}}{{/nodes}})");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("X(Y())", Out);
+}
+
+TEST(MustachePartials, Nested) {
+  Value D = Object{{"a", "hello"}, {"b", "world"}};
+  auto T = Template("{{>outer}}");
+  T.registerPartial("outer", "*{{a}} {{>inner}}*");
+  T.registerPartial("inner", "{{b}}!");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("*hello world!*", Out);
+}
+
+TEST(MustachePartials, SurroundingWhitespace) {
+  Value D = Object{};
+  auto T = Template("| {{>partial}} |");
+  T.registerPartial("partial", "\t|\t");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| \t|\t |", Out);
+}
+
+TEST(MustachePartials, InlineIndentation) {
+  Value D = Object{{"data", "|"}};
+  auto T = Template("  {{data}}  {{> partial}}\n");
+  T.registerPartial("partial", "<\n<");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("  |  <\n<\n", Out);
+}
+
+TEST(MustachePartials, PaddingWhitespace) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template("|{{> partial }}|");
+  T.registerPartial("partial", "[]");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("|[]|", Out);
+}
+
+TEST(MustacheLambdas, BasicInterpolation) {
+  Value D = Object{};
+  auto T = Template("Hello, {{lambda}}!");
+  Lambda L = []() -> llvm::json::Value { return "World"; };
+  T.registerLambda("lambda", L);
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Hello, World!", Out);
+}
+
+TEST(MustacheLambdas, InterpolationExpansion) {
+  Value D = Object{{"planet", "World"}};
+  auto T = Template("Hello, {{lambda}}!");
+  Lambda L = []() -> llvm::json::Value { return "{{planet}}"; };
+  T.registerLambda("lambda", L);
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Hello, World!", Out);
+}
+
+TEST(MustacheLambdas, BasicMultipleCalls) {
+  Value D = Object{};
+  auto T = Template("{{lambda}} == {{lambda}} == {{lambda}}");
+  int I = 0;
+  Lambda L = [&I]() -> llvm::json::Value {
+    I += 1;
+    return I;
+  };
+  T.registerLambda("lambda", L);
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("1 == 2 == 3", Out);
+}
+
+TEST(MustacheLambdas, Escaping) {
+  Value D = Object{};
+  auto T = Template("<{{lambda}}{{&lambda}}");
+  Lambda L = []() -> llvm::json::Value { return ">"; };
+  T.registerLambda("lambda", L);
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("<&gt;>", Out);
+}
+
+TEST(MustacheLambdas, Sections) {
+  Value D = Object{};
+  auto T = Template("<{{#lambda}}{{x}}{{/lambda}}>");
+  SectionLambda L = [](StringRef Text) -> llvm::json::Value {
+    if (Text == "{{x}}") {
+      return "yes";
+    }
+    return "no";
+  };
+  T.registerLambda("lambda", L);
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("<yes>", Out);
+}
+
+TEST(MustacheLambdas, SectionExpansion) {
+  Value D = Object{
+      {"planet", "Earth"},
+  };
+  auto T = Template("<{{#lambda}}-{{/lambda}}>");
+  SectionLambda L = [](StringRef Text) -> llvm::json::Value {
+    SmallString<128> Result;
+    Result += Text;
+    Result += "{{planet}}";
+    Result += Text;
+    return Result;
+  };
+  T.registerLambda("lambda", L);
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("<-Earth->", Out);
+}
+
+TEST(MustacheLambdas, SectionsMultipleCalls) {
+  Value D = Object{};
+  auto T = Template("{{#lambda}}FILE{{/lambda}} != {{#lambda}}LINE{{/lambda}}");
+  SectionLambda L = [](StringRef Text) -> llvm::json::Value {
+    SmallString<128> Result;
+    Result += "__";
+    Result += Text;
+    Result += "__";
+    return Result;
+  };
+  T.registerLambda("lambda", L);
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("__FILE__ != __LINE__", Out);
+}
+
+TEST(MustacheLambdas, InvertedSections) {
+  Value D = Object{{"static", "static"}};
+  auto T = Template("<{{^lambda}}{{static}}{{/lambda}}>");
+  SectionLambda L = [](StringRef Text) -> llvm::json::Value { return false; };
+  T.registerLambda("lambda", L);
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("<>", Out);
+}
+
+TEST(MustacheComments, Inline) {
+  // Comment blocks should be removed from the template.
+  Value D = {};
+  auto T = Template("12345{{! Comment Block! }}67890");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("1234567890", Out);
+}
+
+TEST(MustacheComments, Multiline) {
+  // Multiline comments should be permitted.
+  Value D = {};
+  auto T =
+      Template("12345{{!\n  This is a\n  multi-line comment...\n}}67890\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("1234567890\n", Out);
+}
+
+TEST(MustacheComments, Standalone) {
+  // All standalone comment lines should be removed.
+  Value D = {};
+  auto T = Template("Begin.\n{{! Comment Block! }}\nEnd.\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Begin.\nEnd.\n", Out);
+}
+
+TEST(MustacheComments, IndentedStandalone) {
+  // All standalone comment lines should be removed.
+  Value D = {};
+  auto T = Template("Begin.\n  {{! Indented Comment Block! }}\nEnd.\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Begin.\nEnd.\n", Out);
+}
+
+TEST(MustacheComments, StandaloneLineEndings) {
+  // "\r\n" should be considered a newline for standalone tags.
+  Value D = {};
+  auto T = Template("|\r\n{{! Standalone Comment }}\r\n|");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("|\r\n|", Out);
+}
+
+TEST(MustacheComments, StandaloneWithoutPreviousLine) {
+  // Standalone tags should not require a newline to precede them.
+  Value D = {};
+  auto T = Template("  {{! I'm Still Standalone }}\n!");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("!", Out);
+}
+
+TEST(MustacheComments, StandaloneWithoutNewline) {
+  // Standalone tags should not require a newline to follow them.
+  Value D = {};
+  auto T = Template("!\n  {{! I'm Still Standalone }}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("!\n", Out);
+}
+
+TEST(MustacheComments, MultilineStandalone) {
+  // All standalone comment lines should be removed.
+  Value D = {};
+  auto T = Template("Begin.\n{{!\nSomething's going on here...\n}}\nEnd.\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Begin.\nEnd.\n", Out);
+}
+
+TEST(MustacheComments, IndentedMultilineStandalone) {
+  // All standalone comment lines should be removed.
+  Value D = {};
+  auto T =
+      Template("Begin.\n  {{!\n    Something's going on here...\n  }}\nEnd.\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Begin.\nEnd.\n", Out);
+}
+
+TEST(MustacheComments, IndentedInline) {
+  // Inline comments should not strip whitespace.
+  Value D = {};
+  auto T = Template("  12 {{! 34 }}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("  12 \n", Out);
+}
+
+TEST(MustacheComments, SurroundingWhitespace) {
+  // Comment removal should preserve surrounding whitespace.
+  Value D = {};
+  auto T = Template("12345 {{! Comment Block! }} 67890");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("12345  67890", Out);
+}
+
+TEST(MustacheComments, VariableNameCollision) {
+  // Comments must never render, even if a variable with the same name exists.
+  Value D = Object{
+      {"! comment", 1}, {"! comment ", 2}, {"!comment", 3}, {"comment", 4}};
+  auto T = Template("comments never show: >{{! comment }}<");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("comments never show: ><", Out);
+}
diff --git a/llvm/unittests/Support/RISCVAttributeParserTest.cpp b/llvm/unittests/Support/RISCVAttributeParserTest.cpp
index aa73bb92d6e3e..777dc4d0f4d42 100644
--- a/llvm/unittests/Support/RISCVAttributeParserTest.cpp
+++ b/llvm/unittests/Support/RISCVAttributeParserTest.cpp
@@ -44,7 +44,7 @@ static bool testAttribute(unsigned Tag, unsigned Value, unsigned ExpectedTag,
   RISCVAttributeParser Parser;
   cantFail(Parser.parse(Bytes, llvm::endianness::little));
 
-  std::optional<unsigned> Attr = Parser.getAttributeValue(ExpectedTag);
+  std::optional<unsigned> Attr = Parser.getAttributeValue("", ExpectedTag);
   return Attr && *Attr == ExpectedValue;
 }
 
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index 00dc160c39a36..e24b2d47f6bc4 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -656,7 +656,7 @@ TEST(ParseArchString, RejectsConflictingExtensions) {
   for (StringRef Input :
        {"rv64i_xqcisls0p2", "rv64i_xqcia0p4", "rv64i_xqciac0p3",
         "rv64i_xqcicsr0p2", "rv64i_xqcilsm0p2", "rv64i_xqcicm0p2",
-        "rv64i_xqcics0p2", "rv64i_xqcicli0p2", "rv64i_xqciint0p2",
+        "rv64i_xqcics0p2", "rv64i_xqcicli0p2", "rv64i_xqciint0p4",
         "rv64i_xqcilo0p2", "rv64i_xqcilia0p2", "rv64i_xqcibm0p4"}) {
     EXPECT_THAT(
         toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
@@ -1132,7 +1132,7 @@ Experimental extensions
     xqcicm               0.2
     xqcics               0.2
     xqcicsr              0.2
-    xqciint              0.2
+    xqciint              0.4
     xqcilia              0.2
     xqcilo               0.2
     xqcilsm              0.2
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index 8d03e91fb26c3..caf5d2357411d 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -14,6 +14,7 @@
 
 #include "../lib/Transforms/Vectorize/VPlan.h"
 #include "../lib/Transforms/Vectorize/VPlanHCFGBuilder.h"
+#include "../lib/Transforms/Vectorize/VPlanTransforms.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/BasicAliasAnalysis.h"
 #include "llvm/Analysis/LoopInfo.h"
@@ -70,10 +71,11 @@ class VPlanTestIRBase : public testing::Test {
 
     Loop *L = LI->getLoopFor(LoopHeader);
     PredicatedScalarEvolution PSE(*SE, *L);
-    auto Plan = VPlan::createInitialVPlan(IntegerType::get(*Ctx, 64), PSE, true,
-                                          false, L);
+    auto Plan = std::make_unique<VPlan>(L);
     VPlanHCFGBuilder HCFGBuilder(L, LI.get(), *Plan);
     HCFGBuilder.buildHierarchicalCFG();
+    VPlanTransforms::introduceTopLevelVectorLoopRegion(
+        *Plan, IntegerType::get(*Ctx, 64), PSE, true, false, L);
     return Plan;
   }
 };
diff --git a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
index 59148a96d8e92..ed3ec924afd47 100644
--- a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
@@ -4139,23 +4139,19 @@ void CodeGenDAGPatterns::InferInstructionFlags() {
 
   // If requested by the target, guess any undefined properties.
   if (Target.guessInstructionProperties()) {
-    for (unsigned i = 0, e = Instructions.size(); i != e; ++i) {
-      CodeGenInstruction *InstInfo =
-          const_cast<CodeGenInstruction *>(Instructions[i]);
+    for (const CodeGenInstruction *InstInfo : Instructions) {
       if (InstInfo->InferredFrom)
         continue;
       // The mayLoad and mayStore flags default to false.
       // Conservatively assume hasSideEffects if it wasn't explicit.
       if (InstInfo->hasSideEffects_Unset)
-        InstInfo->hasSideEffects = true;
+        const_cast<CodeGenInstruction *>(InstInfo)->hasSideEffects = true;
     }
     return;
   }
 
   // Complain about any flags that are still undefined.
-  for (unsigned i = 0, e = Instructions.size(); i != e; ++i) {
-    CodeGenInstruction *InstInfo =
-        const_cast<CodeGenInstruction *>(Instructions[i]);
+  for (const CodeGenInstruction *InstInfo : Instructions) {
     if (InstInfo->InferredFrom)
       continue;
     if (InstInfo->hasSideEffects_Unset)
diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp
index 8947e887eaa3e..e1344ae54b20e 100644
--- a/llvm/utils/TableGen/DecoderEmitter.cpp
+++ b/llvm/utils/TableGen/DecoderEmitter.cpp
@@ -190,9 +190,9 @@ class DecoderEmitter {
 //
 // BIT_UNFILTERED is used as the init value for a filter position.  It is used
 // only for filter processings.
-typedef enum {
-  BIT_TRUE,      // '1'
+typedef enum : uint8_t {
   BIT_FALSE,     // '0'
+  BIT_TRUE,      // '1'
   BIT_UNSET,     // '?'
   BIT_UNFILTERED // unfiltered
 } bit_value_t;
@@ -249,7 +249,7 @@ static const BitsInit &getBitsField(const Record &def, StringRef str) {
         Bits.push_back(BI->getBit(Idx));
       }
     } else if (const BitInit *BI = dyn_cast<BitInit>(SI.Value)) {
-      Bits.push_back(const_cast<BitInit *>(BI));
+      Bits.push_back(BI);
     } else {
       for (unsigned Idx = 0U; Idx < SI.BitWidth; ++Idx)
         Bits.push_back(UnsetInit::get(def.getRecords()));
@@ -306,8 +306,7 @@ class FilterChooser;
 /// version and return the Opcode since the two have the same Asm format string.
 class Filter {
 protected:
-  const FilterChooser
-      *Owner;        // points to the FilterChooser who owns this filter
+  const FilterChooser &Owner; // FilterChooser who owns this filter
   unsigned StartBit; // the starting bit position
   unsigned NumBits;  // number of bits to filter
   bool Mixed;        // a mixed region contains both set and unset bits
@@ -329,7 +328,8 @@ class Filter {
 
 public:
   Filter(Filter &&f);
-  Filter(FilterChooser &owner, unsigned startBit, unsigned numBits, bool mixed);
+  Filter(const FilterChooser &owner, unsigned startBit, unsigned numBits,
+         bool mixed);
 
   ~Filter() = default;
 
@@ -428,6 +428,12 @@ class FilterChooser {
   // Parent emitter
   const DecoderEmitter *Emitter;
 
+  struct Island {
+    unsigned StartBit;
+    unsigned NumBits;
+    uint64_t FieldVal;
+  };
+
 public:
   FilterChooser(ArrayRef<EncodingAndInst> Insts,
                 const std::vector<EncodingIDAndOpcode> &IDs,
@@ -513,13 +519,10 @@ class FilterChooser {
   }
 
   // Calculates the island(s) needed to decode the instruction.
-  // This returns a lit of undecoded bits of an instructions, for example,
+  // This returns a list of undecoded bits of an instructions, for example,
   // Inst{20} = 1 && Inst{3-0} == 0b1111 represents two islands of yet-to-be
   // decoded bits in order to verify that the instruction matches the Opcode.
-  unsigned getIslands(std::vector<unsigned> &StartBits,
-                      std::vector<unsigned> &EndBits,
-                      std::vector<uint64_t> &FieldVals,
-                      const insn_t &Insn) const;
+  unsigned getIslands(std::vector<Island> &Islands, const insn_t &Insn) const;
 
   // Emits code to check the Predicates member of an instruction are true.
   // Returns true if predicate matches were emitted, false otherwise.
@@ -589,22 +592,22 @@ Filter::Filter(Filter &&f)
       FilterChooserMap(std::move(f.FilterChooserMap)),
       NumFiltered(f.NumFiltered), LastOpcFiltered(f.LastOpcFiltered) {}
 
-Filter::Filter(FilterChooser &owner, unsigned startBit, unsigned numBits,
+Filter::Filter(const FilterChooser &owner, unsigned startBit, unsigned numBits,
                bool mixed)
-    : Owner(&owner), StartBit(startBit), NumBits(numBits), Mixed(mixed) {
-  assert(StartBit + NumBits - 1 < Owner->BitWidth);
+    : Owner(owner), StartBit(startBit), NumBits(numBits), Mixed(mixed) {
+  assert(StartBit + NumBits - 1 < Owner.BitWidth);
 
   NumFiltered = 0;
   LastOpcFiltered = {0, 0};
 
-  for (const auto &OpcPair : Owner->Opcodes) {
+  for (const auto &OpcPair : Owner.Opcodes) {
     insn_t Insn;
 
     // Populates the insn given the uid.
-    Owner->insnWithID(Insn, OpcPair.EncodingID);
+    Owner.insnWithID(Insn, OpcPair.EncodingID);
 
     // Scans the segment for possibly well-specified encoding bits.
-    auto [Ok, Field] = Owner->fieldFromInsn(Insn, StartBit, NumBits);
+    auto [Ok, Field] = Owner.fieldFromInsn(Insn, StartBit, NumBits);
 
     if (Ok) {
       // The encoding bits are well-known.  Lets add the uid of the
@@ -631,7 +634,7 @@ Filter::Filter(FilterChooser &owner, unsigned startBit, unsigned numBits,
 // match the remaining undecoded encoding bits against the singleton.
 void Filter::recurse() {
   // Starts by inheriting our parent filter chooser's filter bit values.
-  std::vector<bit_value_t> BitValueArray(Owner->FilterBitValues);
+  std::vector<bit_value_t> BitValueArray(Owner.FilterBitValues);
 
   if (!VariableInstructions.empty()) {
     // Conservatively marks each segment position as BIT_UNSET.
@@ -642,9 +645,9 @@ void Filter::recurse() {
     // group of instructions whose segment values are variable.
     FilterChooserMap.try_emplace(
         NO_FIXED_SEGMENTS_SENTINEL,
-        std::make_unique<FilterChooser>(Owner->AllInstructions,
-                                        VariableInstructions, Owner->Operands,
-                                        BitValueArray, *Owner));
+        std::make_unique<FilterChooser>(Owner.AllInstructions,
+                                        VariableInstructions, Owner.Operands,
+                                        BitValueArray, Owner));
   }
 
   // No need to recurse for a singleton filtered instruction.
@@ -667,10 +670,10 @@ void Filter::recurse() {
 
     // Delegates to an inferior filter chooser for further processing on this
     // category of instructions.
-    FilterChooserMap.try_emplace(Inst.first,
-                                 std::make_unique<FilterChooser>(
-                                     Owner->AllInstructions, Inst.second,
-                                     Owner->Operands, BitValueArray, *Owner));
+    FilterChooserMap.try_emplace(
+        Inst.first,
+        std::make_unique<FilterChooser>(Owner.AllInstructions, Inst.second,
+                                        Owner.Operands, BitValueArray, Owner));
   }
 }
 
@@ -688,7 +691,7 @@ static void resolveTableFixups(DecoderTable &Table, const FixupList &Fixups,
     uint32_t Delta = DestIdx - FixupIdx - 3;
     // Our NumToSkip entries are 24-bits. Make sure our table isn't too
     // big.
-    assert(Delta < (1u << 24));
+    assert(isUInt<24>(Delta));
     Table[FixupIdx] = (uint8_t)Delta;
     Table[FixupIdx + 1] = (uint8_t)(Delta >> 8);
     Table[FixupIdx + 2] = (uint8_t)(Delta >> 16);
@@ -698,7 +701,7 @@ static void resolveTableFixups(DecoderTable &Table, const FixupList &Fixups,
 // Emit table entries to decode instructions given a segment or segments
 // of bits.
 void Filter::emitTableEntry(DecoderTableInfo &TableInfo) const {
-  assert((NumBits < (1u << 8)) && "NumBits overflowed uint8 table entry!");
+  assert(isUInt<8>(NumBits) && "NumBits overflowed uint8 table entry!");
   TableInfo.Table.push_back(MCD::OPC_ExtractField);
 
   SmallString<16> SBytes;
@@ -753,8 +756,7 @@ void Filter::emitTableEntry(DecoderTableInfo &TableInfo) const {
     // two as to account for the width of the NumToSkip field itself.
     if (PrevFilter) {
       uint32_t NumToSkip = Table.size() - PrevFilter - 3;
-      assert(NumToSkip < (1u << 24) &&
-             "disassembler decoding table too large!");
+      assert(isUInt<24>(NumToSkip) && "disassembler decoding table too large!");
       Table[PrevFilter] = (uint8_t)NumToSkip;
       Table[PrevFilter + 1] = (uint8_t)(NumToSkip >> 8);
       Table[PrevFilter + 2] = (uint8_t)(NumToSkip >> 16);
@@ -808,19 +810,15 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
   Indent += 2;
 
   // Emit ULEB128 encoded value to OS, returning the number of bytes emitted.
-  auto emitULEB128 = [](DecoderTable::const_iterator I,
+  auto emitULEB128 = [](DecoderTable::const_iterator &I,
                         formatted_raw_ostream &OS) {
-    unsigned Len = 0;
-    while (*I >= 128) {
+    while (*I >= 128)
       OS << (unsigned)*I++ << ", ";
-      Len++;
-    }
     OS << (unsigned)*I++ << ", ";
-    return Len + 1;
   };
 
   // Emit 24-bit numtoskip value to OS, returning the NumToSkip value.
-  auto emitNumToSkip = [](DecoderTable::const_iterator I,
+  auto emitNumToSkip = [](DecoderTable::const_iterator &I,
                           formatted_raw_ostream &OS) {
     uint8_t Byte = *I++;
     uint32_t NumToSkip = Byte;
@@ -857,7 +855,7 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       unsigned Start = decodeULEB128(Table.data() + Pos + 1, nullptr,
                                      Table.data() + Table.size(), &ErrMsg);
       assert(ErrMsg == nullptr && "ULEB128 value too large!");
-      I += emitULEB128(I, OS);
+      emitULEB128(I, OS);
 
       unsigned Len = *I++;
       OS << Len << ",  // Inst{";
@@ -870,11 +868,10 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       ++I;
       OS << Indent << "MCD::OPC_FilterValue, ";
       // The filter value is ULEB128 encoded.
-      I += emitULEB128(I, OS);
+      emitULEB128(I, OS);
 
       // 24-bit numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
-      I += 3;
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
     }
@@ -882,27 +879,25 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       ++I;
       OS << Indent << "MCD::OPC_CheckField, ";
       // ULEB128 encoded start value.
-      I += emitULEB128(I, OS);
+      emitULEB128(I, OS);
       // 8-bit length.
       unsigned Len = *I++;
       OS << Len << ", ";
       // ULEB128 encoded field value.
-      I += emitULEB128(I, OS);
+      emitULEB128(I, OS);
 
       // 24-bit numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
-      I += 3;
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
     }
     case MCD::OPC_CheckPredicate: {
       ++I;
       OS << Indent << "MCD::OPC_CheckPredicate, ";
-      I += emitULEB128(I, OS);
+      emitULEB128(I, OS);
 
       // 24-bit numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
-      I += 3;
       OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
       break;
     }
@@ -917,10 +912,10 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
       assert(ErrMsg == nullptr && "ULEB128 value too large!");
 
       OS << Indent << "MCD::OPC_" << (IsTry ? "Try" : "") << "Decode, ";
-      I += emitULEB128(I, OS);
+      emitULEB128(I, OS);
 
       // Decoder index.
-      I += emitULEB128(I, OS);
+      emitULEB128(I, OS);
 
       auto EncI = OpcodeToEncodingID.find(Opc);
       assert(EncI != OpcodeToEncodingID.end() && "no encoding entry");
@@ -935,7 +930,6 @@ void DecoderEmitter::emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
 
       // 24-bit numtoskip value.
       uint32_t NumToSkip = emitNumToSkip(I, OS);
-      I += 3;
 
       OS << "// Opcode: " << NumberedEncodings[EncodingID]
          << ", skip to: " << ((I - Table.begin()) + NumToSkip) << "\n";
@@ -1115,19 +1109,15 @@ void FilterChooser::dumpStack(raw_ostream &OS, const char *prefix) const {
 // This returns a list of undecoded bits of an instructions, for example,
 // Inst{20} = 1 && Inst{3-0} == 0b1111 represents two islands of yet-to-be
 // decoded bits in order to verify that the instruction matches the Opcode.
-unsigned FilterChooser::getIslands(std::vector<unsigned> &StartBits,
-                                   std::vector<unsigned> &EndBits,
-                                   std::vector<uint64_t> &FieldVals,
+unsigned FilterChooser::getIslands(std::vector<Island> &Islands,
                                    const insn_t &Insn) const {
-  unsigned Num, BitNo;
-  Num = BitNo = 0;
-
-  uint64_t FieldVal = 0;
+  uint64_t FieldVal;
+  unsigned StartBit;
 
   // 0: Init
   // 1: Water (the bit value does not affect decoding)
   // 2: Island (well-known bit value needed for decoding)
-  int State = 0;
+  unsigned State = 0;
 
   for (unsigned i = 0; i < BitWidth; ++i) {
     int64_t Val = Value(Insn[i]);
@@ -1141,35 +1131,26 @@ unsigned FilterChooser::getIslands(std::vector<unsigned> &StartBits,
         State = 1; // Still in Water
       else {
         State = 2; // Into the Island
-        BitNo = 0;
-        StartBits.push_back(i);
+        StartBit = i;
         FieldVal = Val;
       }
       break;
     case 2:
       if (Filtered || Val == -1) {
         State = 1; // Into the Water
-        EndBits.push_back(i - 1);
-        FieldVals.push_back(FieldVal);
-        ++Num;
+        Islands.push_back({StartBit, i - StartBit, FieldVal});
       } else {
         State = 2; // Still in Island
-        ++BitNo;
-        FieldVal = FieldVal | Val << BitNo;
+        FieldVal |= Val << (i - StartBit);
       }
       break;
     }
   }
   // If we are still in Island after the loop, do some housekeeping.
-  if (State == 2) {
-    EndBits.push_back(BitWidth - 1);
-    FieldVals.push_back(FieldVal);
-    ++Num;
-  }
+  if (State == 2)
+    Islands.push_back({StartBit, BitWidth - StartBit, FieldVal});
 
-  assert(StartBits.size() == Num && EndBits.size() == Num &&
-         FieldVals.size() == Num);
-  return Num;
+  return Islands.size();
 }
 
 void FilterChooser::emitBinaryParser(raw_ostream &OS, indent Indent,
@@ -1437,32 +1418,28 @@ void FilterChooser::emitSoftFailTableEntry(DecoderTableInfo &TableInfo,
 // Emits table entries to decode the singleton.
 void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo,
                                             EncodingIDAndOpcode Opc) const {
-  std::vector<unsigned> StartBits;
-  std::vector<unsigned> EndBits;
-  std::vector<uint64_t> FieldVals;
+  std::vector<Island> Islands;
   insn_t Insn;
   insnWithID(Insn, Opc.EncodingID);
 
   // Look for islands of undecoded bits of the singleton.
-  getIslands(StartBits, EndBits, FieldVals, Insn);
-
-  unsigned Size = StartBits.size();
+  unsigned Size = getIslands(Islands, Insn);
 
   // Emit the predicate table entry if one is needed.
   emitPredicateTableEntry(TableInfo, Opc.EncodingID);
 
   // Check any additional encoding fields needed.
   for (unsigned I = Size; I != 0; --I) {
-    unsigned NumBits = EndBits[I - 1] - StartBits[I - 1] + 1;
-    assert((NumBits < (1u << 8)) && "NumBits overflowed uint8 table entry!");
+    unsigned NumBits = Islands[I - 1].NumBits;
+    assert(isUInt<8>(NumBits) && "NumBits overflowed uint8 table entry!");
     TableInfo.Table.push_back(MCD::OPC_CheckField);
     uint8_t Buffer[16], *P;
-    encodeULEB128(StartBits[I - 1], Buffer);
+    encodeULEB128(Islands[I - 1].StartBit, Buffer);
     for (P = Buffer; *P >= 128; ++P)
       TableInfo.Table.push_back(*P);
     TableInfo.Table.push_back(*P);
     TableInfo.Table.push_back(NumBits);
-    encodeULEB128(FieldVals[I - 1], Buffer);
+    encodeULEB128(Islands[I - 1].FieldVal, Buffer);
     for (P = Buffer; *P >= 128; ++P)
       TableInfo.Table.push_back(*P);
     TableInfo.Table.push_back(*P);
@@ -1577,17 +1554,15 @@ bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) {
     assert(numInstructions == 3);
 
     for (const auto &Opcode : Opcodes) {
-      std::vector<unsigned> StartBits;
-      std::vector<unsigned> EndBits;
-      std::vector<uint64_t> FieldVals;
+      std::vector<Island> Islands;
       insn_t Insn;
 
       insnWithID(Insn, Opcode.EncodingID);
 
       // Look for islands of undecoded bits of any instruction.
-      if (getIslands(StartBits, EndBits, FieldVals, Insn) > 0) {
+      if (getIslands(Islands, Insn) > 0) {
         // Found an instruction with island(s).  Now just assign a filter.
-        runSingleFilter(StartBits[0], EndBits[0] - StartBits[0] + 1, true);
+        runSingleFilter(Islands[0].StartBit, Islands[0].NumBits, true);
         return true;
       }
     }
@@ -1612,16 +1587,14 @@ bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) {
   // (MIXED) ------ . ----> (MIXED)
   // (FILTERED)---- . ----> (FILTERED)
 
-  std::vector<bitAttr_t> bitAttrs;
+  std::vector<bitAttr_t> bitAttrs(BitWidth, ATTR_NONE);
 
   // FILTERED bit positions provide no entropy and are not worthy of pursuing.
   // Filter::recurse() set either BIT_TRUE or BIT_FALSE for each position.
   for (BitIndex = 0; BitIndex < BitWidth; ++BitIndex)
     if (FilterBitValues[BitIndex] == BIT_TRUE ||
         FilterBitValues[BitIndex] == BIT_FALSE)
-      bitAttrs.push_back(ATTR_FILTERED);
-    else
-      bitAttrs.push_back(ATTR_NONE);
+      bitAttrs[BitIndex] = ATTR_FILTERED;
 
   for (const auto &OpcPair : Opcodes) {
     insn_t insn;
diff --git a/llvm/utils/TableGen/FastISelEmitter.cpp b/llvm/utils/TableGen/FastISelEmitter.cpp
index 5d6194ddf1777..26625da393851 100644
--- a/llvm/utils/TableGen/FastISelEmitter.cpp
+++ b/llvm/utils/TableGen/FastISelEmitter.cpp
@@ -828,7 +828,7 @@ void FastISelMap::printFunctionDefinitions(raw_ostream &OS) {
       for (unsigned i = 0, e = MI->second.size(); i != e; ++i) {
         OS << "  if (";
         MI->second[i].emitImmediatePredicate(OS, ImmediatePredicates);
-        OS << ")\n    if (unsigned Reg = fastEmit_";
+        OS << ")\n    if (Register Reg = fastEmit_";
         MI->second[i].PrintManglingSuffix(OS, ImmediatePredicates);
         OS << "(VT, RetVT, Opcode";
         if (!MI->second[i].empty())
diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index d4510f2757349..c398b6f9bf7cd 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -1029,9 +1029,9 @@ void SubtargetEmitter::expandProcResources(
   for (unsigned I = 0, E = PRVec.size(); I != E; ++I) {
     const Record *PRDef = PRVec[I];
     ConstRecVec SubResources;
-    if (PRDef->isSubClassOf("ProcResGroup"))
+    if (PRDef->isSubClassOf("ProcResGroup")) {
       SubResources = PRDef->getValueAsListOfDefs("Resources");
-    else {
+    } else {
       SubResources.push_back(PRDef);
       PRDef = SchedModels.findProcResUnits(PRDef, PM, PRDef->getLoc());
       for (const Record *SubDef = PRDef;
@@ -1053,13 +1053,11 @@ void SubtargetEmitter::expandProcResources(
       if (PR == PRDef || !PR->isSubClassOf("ProcResGroup"))
         continue;
       ConstRecVec SuperResources = PR->getValueAsListOfDefs("Resources");
-      ConstRecIter SubI = SubResources.begin(), SubE = SubResources.end();
-      for (; SubI != SubE; ++SubI) {
-        if (!is_contained(SuperResources, *SubI)) {
-          break;
-        }
-      }
-      if (SubI == SubE) {
+      bool AllContained =
+          all_of(SubResources, [SuperResources](const Record *SubResource) {
+            return is_contained(SuperResources, SubResource);
+          });
+      if (AllContained) {
         PRVec.push_back(PR);
         ReleaseAtCycles.push_back(ReleaseAtCycles[I]);
         AcquireAtCycles.push_back(AcquireAtCycles[I]);
diff --git a/llvm/utils/TableGen/X86InstrMappingEmitter.cpp b/llvm/utils/TableGen/X86InstrMappingEmitter.cpp
index df43f39e0e9be..5a10aba8f357d 100644
--- a/llvm/utils/TableGen/X86InstrMappingEmitter.cpp
+++ b/llvm/utils/TableGen/X86InstrMappingEmitter.cpp
@@ -275,26 +275,26 @@ void X86InstrMappingEmitter::emitNFTransformTable(
     const Record *Rec = Inst->TheDef;
     if (!isInteresting(Rec))
       continue;
-    std::string Name = Rec->getName().str();
-    auto Pos = Name.find("_NF");
-    if (Pos == std::string::npos)
+    StringRef Name = Rec->getName();
+    if (Name.contains("_NF"))
       continue;
 
-    if (auto *NewRec = Records.getDef(Name.erase(Pos, 3))) {
+    if (auto *NewRec = Name.consume_back("_ND")
+                           ? Records.getDef(Name.str() + "_NF_ND")
+                           : Records.getDef(Name.str() + "_NF")) {
 #ifndef NDEBUG
       auto ClobberEFLAGS = [](const Record *R) {
         return llvm::any_of(
             R->getValueAsListOfDefs("Defs"),
             [](const Record *Def) { return Def->getName() == "EFLAGS"; });
       };
-      if (ClobberEFLAGS(Rec))
+      if (ClobberEFLAGS(NewRec))
         report_fatal_error("EFLAGS should not be clobbered by " +
-                           Rec->getName());
-      if (!ClobberEFLAGS(NewRec))
-        report_fatal_error("EFLAGS should be clobbered by " +
                            NewRec->getName());
+      if (!ClobberEFLAGS(Rec))
+        report_fatal_error("EFLAGS should be clobbered by " + Rec->getName());
 #endif
-      Table.emplace_back(&Target.getInstruction(NewRec), Inst);
+      Table.emplace_back(Inst, &Target.getInstruction(NewRec));
     }
   }
   printTable(Table, "X86NFTransformTable", "GET_X86_NF_TRANSFORM_TABLE", OS);
diff --git a/llvm/utils/gn/build/sync_source_lists_from_cmake.py b/llvm/utils/gn/build/sync_source_lists_from_cmake.py
index aada38bebd421..a8d922acbd1dd 100755
--- a/llvm/utils/gn/build/sync_source_lists_from_cmake.py
+++ b/llvm/utils/gn/build/sync_source_lists_from_cmake.py
@@ -61,13 +61,13 @@ def git_out(args):
     gn_files = git_out(["ls-files", "*BUILD.gn"]).splitlines()
 
     # Matches e.g. |   "foo.cpp",|, captures |foo| in group 1.
-    gn_cpp_re = re.compile(r'^\s*"([^$"]+\.(?:cpp|c|h|S))",$', re.MULTILINE)
+    gn_cpp_re = re.compile(r'^\s*"([^$"]+\.(?:cpp|c|h|mm|S))",$', re.MULTILINE)
     # Matches e.g. |   bar_sources = [ "foo.cpp" ]|, captures |foo| in group 1.
     gn_cpp_re2 = re.compile(
-        r'^\s*(?:.*_)?sources \+?= \[ "([^$"]+\.(?:cpp|c|h|S))" ]$', re.MULTILINE
+        r'^\s*(?:.*_)?sources \+?= \[ "([^$"]+\.(?:cpp|c|h|mm|S))" ]$', re.MULTILINE
     )
     # Matches e.g. |   foo.cpp|, captures |foo| in group 1.
-    cmake_cpp_re = re.compile(r"^\s*([A-Za-z_0-9./-]+\.(?:cpp|c|h|S))$", re.MULTILINE)
+    cmake_cpp_re = re.compile(r"^\s*([A-Za-z_0-9./-]+\.(?:cpp|c|h|mm|S))$", re.MULTILINE)
 
     changes_by_rev = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
 
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn
index 815c5a93c72f7..cad2c4912b2cc 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn
@@ -12,6 +12,7 @@ static_library("readability") {
     "//llvm/lib/Support",
   ]
   sources = [
+    "AmbiguousSmartptrResetCallCheck.cpp",
     "AvoidConstParamsInDecls.cpp",
     "AvoidNestedConditionalOperatorCheck.cpp",
     "AvoidReturnWithVoidValueCheck.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
index 2419b51d42748..66c01690af4c1 100644
--- a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
@@ -224,6 +224,7 @@ copy("Headers") {
     "hlsl/hlsl_alias_intrinsics.h",
     "hlsl/hlsl_basic_types.h",
     "hlsl/hlsl_detail.h",
+    "hlsl/hlsl_intrinsic_helpers.h",
     "hlsl/hlsl_intrinsics.h",
     "hresetintrin.h",
     "htmintrin.h",
diff --git a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
index d9c3257536639..53cbd2715df81 100644
--- a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
@@ -17,6 +17,7 @@ static_library("Checkers") {
     "AnalysisOrderChecker.cpp",
     "AnalyzerStatsChecker.cpp",
     "ArrayBoundChecker.cpp",
+    "AssumeModeling.cpp",
     "BasicObjCFoundationChecks.cpp",
     "BitwiseShiftChecker.cpp",
     "BlockInCriticalSectionChecker.cpp",
@@ -140,10 +141,10 @@ static_library("Checkers") {
     "WebKit/MemoryUnsafeCastChecker.cpp",
     "WebKit/PtrTypesSemantics.cpp",
     "WebKit/RawPtrRefCallArgsChecker.cpp",
+    "WebKit/RawPtrRefLambdaCapturesChecker.cpp",
     "WebKit/RawPtrRefLocalVarsChecker.cpp",
     "WebKit/RawPtrRefMemberChecker.cpp",
     "WebKit/RefCntblBaseVirtualDtorChecker.cpp",
-    "WebKit/UncountedLambdaCapturesChecker.cpp",
     "cert/InvalidPtrChecker.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index af3257d142d6a..47f504de2a105 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -1830,6 +1830,8 @@ if (current_toolchain == default_toolchain) {
       "__type_traits/negation.h",
       "__type_traits/promote.h",
       "__type_traits/rank.h",
+      "__type_traits/reference_constructs_from_temporary.h",
+      "__type_traits/reference_converts_from_temporary.h",
       "__type_traits/remove_all_extents.h",
       "__type_traits/remove_const.h",
       "__type_traits/remove_const_ref.h",
diff --git a/llvm/utils/gn/secondary/lldb/source/Host/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Host/BUILD.gn
index ddee2e2783b96..021386f5f55e1 100644
--- a/llvm/utils/gn/secondary/lldb/source/Host/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Host/BUILD.gn
@@ -16,8 +16,6 @@ static_library("Host") {
   ]
   public_deps = [ "//llvm/utils/gn/build/libs/xml" ]
   sources = [
-    "aix/HostInfoAIX.cpp",
-    "common/Alarm.cpp",
     "common/File.cpp",
     "common/FileAction.cpp",
     "common/FileCache.cpp",
@@ -91,6 +89,13 @@ static_library("Host") {
     ]
   }
 
+  if (current_os == "aix") {
+    sources += [
+      "aix/Host.cpp",
+      "aix/HostInfoAIX.cpp",
+    ]
+  }
+
   if (current_os == "mac") {
     sources += [
       "macosx/cfcpp/CFCBundle.cpp",
diff --git a/llvm/utils/gn/secondary/lldb/source/Host/macosx/objcxx/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Host/macosx/objcxx/BUILD.gn
index d26474457a8fb..7df6e24071027 100644
--- a/llvm/utils/gn/secondary/lldb/source/Host/macosx/objcxx/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Host/macosx/objcxx/BUILD.gn
@@ -12,6 +12,7 @@ static_library("objcxx") {
     "Host.mm",
     "HostInfoMacOSX.mm",
     "HostThreadMacOSX.mm",
+    "MemoryMonitorMacOSX.mm",
   ]
   frameworks = [
     "Foundation.framework",
diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
index 8c8b9ff99e117..a97d2f5bee723 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
@@ -27,7 +27,6 @@ static_library("Orc") {
     "ELFNixPlatform.cpp",
     "EPCDebugObjectRegistrar.cpp",
     "EPCDynamicLibrarySearchGenerator.cpp",
-    "EPCEHFrameRegistrar.cpp",
     "EPCGenericDylibManager.cpp",
     "EPCGenericJITLinkMemoryManager.cpp",
     "EPCGenericRTDyldMemoryManager.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
index ba13bb16731af..fe7ff6ef68f99 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -33,6 +33,7 @@ static_library("Support") {
     "Windows",
   ]
   sources = [
+    "AArch64AttributeParser.cpp",
     "AArch64BuildAttributes.cpp",
     "ABIBreak.cpp",
     "AMDGPUMetadata.cpp",
@@ -77,7 +78,8 @@ static_library("Support") {
     "DeltaTree.cpp",
     "DivisionByConstantInfo.cpp",
     "DynamicAPInt.cpp",
-    "ELFAttributeParser.cpp",
+    "ELFAttrParserCompact.cpp",
+    "ELFAttrParserExtended.cpp",
     "ELFAttributes.cpp",
     "Error.cpp",
     "ErrorHandling.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index b152fc844a493..cdde7682582ac 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -143,6 +143,7 @@ static_library("LLVMAMDGPUCodeGen") {
     "AMDGPUCombinerHelper.cpp",
     "AMDGPUCtorDtorLowering.cpp",
     "AMDGPUExportClustering.cpp",
+    "AMDGPUExportKernelRuntimeHandles.cpp",
     "AMDGPUFrameLowering.cpp",
     "AMDGPUGlobalISelDivergenceLowering.cpp",
     "AMDGPUGlobalISelUtils.cpp",
@@ -171,7 +172,6 @@ static_library("LLVMAMDGPUCodeGen") {
     "AMDGPUMacroFusion.cpp",
     "AMDGPUMarkLastScratchLoad.cpp",
     "AMDGPUMemoryUtils.cpp",
-    "AMDGPUOpenCLEnqueuedBlockLowering.cpp",
     "AMDGPUPerfHintAnalysis.cpp",
     "AMDGPUPostLegalizerCombiner.cpp",
     "AMDGPUPreLegalizerCombiner.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
index 559101e830961..f127c6ec3c131 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
@@ -134,6 +134,7 @@ static_library("LLVMRISCVCodeGen") {
     "RISCVInsertWriteVXRM.cpp",
     "RISCVInstrInfo.cpp",
     "RISCVLandingPadSetup.cpp",
+    "RISCVLoadStoreOptimizer.cpp",
     "RISCVMachineFunctionInfo.cpp",
     "RISCVMakeCompressible.cpp",
     "RISCVMergeBaseOffset.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
index 71493498ef03f..829bb9e3de617 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
@@ -18,6 +18,7 @@ static_library("Vectorize") {
     "SandboxVectorizer/Interval.cpp",
     "SandboxVectorizer/Legality.cpp",
     "SandboxVectorizer/Passes/BottomUpVec.cpp",
+    "SandboxVectorizer/Passes/RegionsFromBBs.cpp",
     "SandboxVectorizer/Passes/RegionsFromMetadata.cpp",
     "SandboxVectorizer/Passes/SeedCollection.cpp",
     "SandboxVectorizer/Passes/TransactionAcceptOrRevert.cpp",
@@ -29,6 +30,7 @@ static_library("Vectorize") {
     "SandboxVectorizer/VecUtils.cpp",
     "VPlan.cpp",
     "VPlanAnalysis.cpp",
+    "VPlanConstruction.cpp",
     "VPlanHCFGBuilder.cpp",
     "VPlanRecipes.cpp",
     "VPlanSLP.cpp",
diff --git a/mlir/docs/DeclarativeRewrites.md b/mlir/docs/DeclarativeRewrites.md
index 888ce57fa3b53..fd566a2393b63 100644
--- a/mlir/docs/DeclarativeRewrites.md
+++ b/mlir/docs/DeclarativeRewrites.md
@@ -237,9 +237,9 @@ In the above, we are using `BOp`'s result for building `COp`.
 
 Given that `COp` was specified with table-driven op definition, there will be
 several `build()` methods generated for it. One of them has aggregated
-parameters for result types, operands, and attributes in the signature: `void
+parameters for result types, operands, and properties in the signature: `void
 COp::build(..., ArrayRef<Type> resultTypes, Array<Value> operands,
-ArrayRef<NamedAttribute> attr)`. The pattern in the above calls this `build()`
+const COp::Properties& properties)`. The pattern in the above calls this `build()`
 method for constructing the `COp`.
 
 In general, arguments in the result pattern will be passed directly to the
diff --git a/mlir/docs/DefiningDialects/Operations.md b/mlir/docs/DefiningDialects/Operations.md
index 8ff60ac21424c..528070cd3ebff 100644
--- a/mlir/docs/DefiningDialects/Operations.md
+++ b/mlir/docs/DefiningDialects/Operations.md
@@ -465,7 +465,18 @@ def MyOp : ... {
 The following builders are generated:
 
 ```c++
+// All result-types/operands/properties/discardable attributes have one
+// aggregate parameter. `Properties` is the properties structure of
+// `MyOp`.
+static void build(OpBuilder &odsBuilder, OperationState &odsState,
+                  TypeRange resultTypes,
+                  ValueRange operands,
+                  Properties properties,
+                  ArrayRef<NamedAttribute> discardableAttributes = {});
+
 // All result-types/operands/attributes have one aggregate parameter.
+// Inherent properties and discardable attributes are mixed together in the
+//  `attributes` dictionary.
 static void build(OpBuilder &odsBuilder, OperationState &odsState,
                   TypeRange resultTypes,
                   ValueRange operands,
@@ -498,20 +509,28 @@ static void build(OpBuilder &odsBuilder, OperationState &odsState,
 
 // All operands/attributes have aggregate parameters.
 // Generated if return type can be inferred.
+static void build(OpBuilder &odsBuilder, OperationState &odsState,
+                  ValueRange operands,
+                  Properties properties,
+                  ArrayRef<NamedAttribute> discardableAttributes);
+
+// All operands/attributes have aggregate parameters.
+// Generated if return type can be inferred. Uses the legacy merged attribute
+// dictionary.
 static void build(OpBuilder &odsBuilder, OperationState &odsState,
                   ValueRange operands, ArrayRef<NamedAttribute> attributes);
 
 // (And manually specified builders depending on the specific op.)
 ```
 
-The first form provides basic uniformity so that we can create ops using the
-same form regardless of the exact op. This is particularly useful for
+The first two forms provide basic uniformity so that we can create ops using
+the same form regardless of the exact op. This is particularly useful for
 implementing declarative pattern rewrites.
 
-The second and third forms are good for use in manually written code, given that
+The third and fourth forms are good for use in manually written code, given that
 they provide better guarantee via signatures.
 
-The third form will be generated if any of the op's attribute has different
+The fourth form will be generated if any of the op's attribute has different
 `Attr.returnType` from `Attr.storageType` and we know how to build an attribute
 from an unwrapped value (i.e., `Attr.constBuilderCall` is defined.)
 Additionally, for the third form, if an attribute appearing later in the
diff --git a/mlir/docs/DialectConversion.md b/mlir/docs/DialectConversion.md
index abacd5a82c61e..f67d1411b3065 100644
--- a/mlir/docs/DialectConversion.md
+++ b/mlir/docs/DialectConversion.md
@@ -179,13 +179,13 @@ updated/remapped operands of an operation, such as when the types of results
 defined by an operation have changed. The general Rewrite Patterns can no longer
 be used in these situations, as the types of the operands of the operation being
 matched will not correspond with those expected by the user. This pattern
-provides, as an additional argument to the `matchAndRewrite` and `rewrite`
-methods, the list of operands that the operation should use after conversion. If
-an operand was the result of a non-converted operation, for example if it was
-already legal, the original operand is used. This means that the operands
-provided always have a 1-1 non-null correspondence with the operands on the
-operation. The original operands of the operation are still intact and may be
-inspected as normal. These patterns also utilize a special `PatternRewriter`,
+provides, as an additional argument to the `matchAndRewrite` method, the list
+of operands that the operation should use after conversion. If an operand was
+the result of a non-converted operation, for example if it was already legal,
+the original operand is used. This means that the operands provided always have
+a 1-1 non-null correspondence with the operands on the operation. The original
+operands of the operation are still intact and may be inspected as normal.
+These patterns also utilize a special `PatternRewriter`,
 `ConversionPatternRewriter`, that provides special hooks for use with the
 conversion infrastructure.
 
diff --git a/mlir/docs/Dialects/OpenMPDialect/_index.md b/mlir/docs/Dialects/OpenMPDialect/_index.md
index b651b3c06485c..adde176750437 100644
--- a/mlir/docs/Dialects/OpenMPDialect/_index.md
+++ b/mlir/docs/Dialects/OpenMPDialect/_index.md
@@ -352,12 +352,29 @@ let assemblyFormat = clausesAssemblyFormat # [{
 ```
 
 The `BlockArgOpenMPOpInterface` has been introduced to simplify the addition and
-handling of these kinds of clauses. It holds `num<ClauseName>BlockArgs()`
-functions that by default return 0, to be overriden by each clause through the
-`extraClassDeclaration` property. Based on these functions and the expected
-alphabetical sorting between entry block argument-defining clauses, it
-implements `get<ClauseName>BlockArgs()` functions that are the intended method
-of accessing clause-defined block arguments.
+handling of these kinds of clauses. Adding it to an operation directly, or
+indirectly through a clause, results in the addition of overridable
+`get<ClauseName>Vars()` and `num<ClauseName>BlockArgs()` public functions for
+all entry block argument-generating clauses. By default, the reported number of
+block arguments defined by a clause will correspond to the number of operands
+taken by the operation for that clause. This list of operands will be empty by
+default, and will automatically be overriden by getters of the corresponding
+`Variadic<...> $<clause_name>_vars` argument of the same clause's definition.
+
+In addition to these methods added to the actual operations, the
+`BlockArgOpenMPOpInterface` itself defines a set of methods based on the
+previous ones and on the convention that entry block arguments for multiple
+clauses are sorted alphabetically by clause name. These are listed below, and
+they represent the main way in which clause-defined block arguments should be
+accessed:
+  - `get<ClauseName>BlockArgsStart()`: Returns the index within the list of
+  entry block arguments where the first element defined by the given clause
+  should be located.
+  - `get<ClauseName>BlockArgs()`: Returns the list of entry block arguments
+  defined by the given clause.
+  - `getBlockArgsPairs()`: Returns a list of pairs where the first element is
+  the outside value, or operand, and the second element is the corresponding
+  entry block argument.
 
 ## Loop-Associated Directives
 
diff --git a/mlir/docs/PatternRewriter.md b/mlir/docs/PatternRewriter.md
index af0f56466e0cb..105a554b95851 100644
--- a/mlir/docs/PatternRewriter.md
+++ b/mlir/docs/PatternRewriter.md
@@ -48,13 +48,9 @@ operation type, a special tag must be provided to make the intent explicit:
 ### `matchAndRewrite` implementation
 
 This is the chunk of code that matches a given root `Operation` and performs a
-rewrite of the IR. A `RewritePattern` can specify this implementation either via
-the `matchAndRewrite` method or via separate `match` and `rewrite` methods when
-deriving from `RewritePattern::SplitMatchAndRewrite`. When using the combined
-`matchAndRewrite` method, no IR mutation should take place before the match is
-deemed successful. The combined `matchAndRewrite` is useful when non-trivially
-recomputable information is required by the matching and rewriting phase. See
-below for examples:
+rewrite of the IR. A `RewritePattern` can specify this implementation via the
+`matchAndRewrite` method. No IR mutation should take place before the match is
+deemed successful. See below for examples:
 
 ```c++
 class MyPattern : public RewritePattern {
@@ -67,21 +63,6 @@ public:
   MyPattern(PatternBenefit benefit)
       : RewritePattern(benefit, MatchAnyOpTypeTag()) {}
 
-  /// In this section, the `match` and `rewrite` implementation is specified
-  /// using the separate hooks.
-  LogicalResult match(Operation *op) const override {
-    // The `match` method returns `success()` if the pattern is a match, failure
-    // otherwise.
-    // ...
-  }
-  void rewrite(Operation *op, PatternRewriter &rewriter) const override {
-    // The `rewrite` method performs mutations on the IR rooted at `op` using
-    // the provided rewriter. All mutations must go through the provided
-    // rewriter.
-  }
-
-  /// In this section, the `match` and `rewrite` implementation is specified
-  /// using a single hook.
   LogicalResult matchAndRewrite(Operation *op, PatternRewriter &rewriter) const override {
     // The `matchAndRewrite` method performs both the matching and the mutation.
     // Note that the match must reach a successful point before IR mutation may
@@ -92,12 +73,6 @@ public:
 
 #### Restrictions
 
-Within the `match` section of a pattern, the following constraints apply:
-
-*   No mutation of the IR is allowed.
-
-Within the `rewrite` section of a pattern, the following constraints apply:
-
 *   All IR mutations, including creation, *must* be performed by the given
     `PatternRewriter`. This class provides hooks for performing all of the
     possible mutations that may take place within a pattern. For example, this
@@ -107,8 +82,6 @@ Within the `rewrite` section of a pattern, the following constraints apply:
 *   The root operation is required to either be: updated in-place, replaced, or
     erased.
 *   `matchAndRewrite` must return "success" if and only if the IR was modified.
-    `match` must return "success" if and only if the IR is going to be modified
-    during `rewrite`.
 
 
 ### Application Recursion
diff --git a/mlir/docs/Tutorials/QuickstartRewrites.md b/mlir/docs/Tutorials/QuickstartRewrites.md
index 604148bd9c600..493f9d5687374 100644
--- a/mlir/docs/Tutorials/QuickstartRewrites.md
+++ b/mlir/docs/Tutorials/QuickstartRewrites.md
@@ -216,25 +216,6 @@ In case ODS patterns and `matchAndRewrite`-style functions are not sufficient
 you can also specify rewrites as a general set of `RewritePattern`s:
 
 ```c++
-/// Multi-step rewrite using "match" and "rewrite". This allows for separating
-/// the concerns of matching and rewriting.
-struct ConvertTFLeakyRelu : public RewritePattern {
-  ConvertTFLeakyRelu(MLIRContext *context)
-      : RewritePattern("tf.LeakyRelu", 1, context) {}
-
-  LogicalResult match(Operation *op) const override {
-    return success();
-  }
-
-  void rewrite(Operation *op, PatternRewriter &rewriter) const override {
-    rewriter.replaceOpWithNewOp<TFL::LeakyReluOp>(
-        op, op->getResult(0).getType(), op->getOperand(0),
-        /*alpha=*/op->getAttrOfType<FloatAttr>("alpha"));
-  }
-};
-
-/// Single-step rewrite with "matchAndRewrite". This allows for performing the
-/// rewrite immediately upon a successful match.
 struct ConvertTFLeakyRelu : public RewritePattern {
   ConvertTFLeakyRelu(MLIRContext *context)
       : RewritePattern("tf.LeakyRelu", 1, context) {}
diff --git a/mlir/include/mlir-c/IR.h b/mlir/include/mlir-c/IR.h
index d562da1f90757..1a8e8737f7fed 100644
--- a/mlir/include/mlir-c/IR.h
+++ b/mlir/include/mlir-c/IR.h
@@ -162,6 +162,15 @@ MLIR_CAPI_EXPORTED bool mlirContextIsRegisteredOperation(MlirContext context,
 MLIR_CAPI_EXPORTED void mlirContextSetThreadPool(MlirContext context,
                                                  MlirLlvmThreadPool threadPool);
 
+/// Gets the number of threads of the thread pool of the context when
+/// multithreading is enabled. Returns 1 if no multithreading.
+MLIR_CAPI_EXPORTED unsigned mlirContextGetNumThreads(MlirContext context);
+
+/// Gets the thread pool of the context when enabled multithreading, otherwise
+/// an assertion is raised.
+MLIR_CAPI_EXPORTED MlirLlvmThreadPool
+mlirContextGetThreadPool(MlirContext context);
+
 //===----------------------------------------------------------------------===//
 // Dialect API.
 //===----------------------------------------------------------------------===//
@@ -261,15 +270,75 @@ MLIR_CAPI_EXPORTED MlirLocation mlirLocationFileLineColRangeGet(
     MlirContext context, MlirStringRef filename, unsigned start_line,
     unsigned start_col, unsigned end_line, unsigned end_col);
 
+/// Getter for filename of FileLineColRange.
+MLIR_CAPI_EXPORTED MlirIdentifier
+mlirLocationFileLineColRangeGetFilename(MlirLocation location);
+
+/// Getter for start_line of FileLineColRange.
+MLIR_CAPI_EXPORTED int
+mlirLocationFileLineColRangeGetStartLine(MlirLocation location);
+
+/// Getter for start_column of FileLineColRange.
+MLIR_CAPI_EXPORTED int
+mlirLocationFileLineColRangeGetStartColumn(MlirLocation location);
+
+/// Getter for end_line of FileLineColRange.
+MLIR_CAPI_EXPORTED int
+mlirLocationFileLineColRangeGetEndLine(MlirLocation location);
+
+/// Getter for end_column of FileLineColRange.
+MLIR_CAPI_EXPORTED int
+mlirLocationFileLineColRangeGetEndColumn(MlirLocation location);
+
+/// TypeID Getter for FileLineColRange.
+MLIR_CAPI_EXPORTED MlirTypeID mlirLocationFileLineColRangeGetTypeID(void);
+
+/// Checks whether the given location is an FileLineColRange.
+MLIR_CAPI_EXPORTED bool mlirLocationIsAFileLineColRange(MlirLocation location);
+
 /// Creates a call site location with a callee and a caller.
 MLIR_CAPI_EXPORTED MlirLocation mlirLocationCallSiteGet(MlirLocation callee,
                                                         MlirLocation caller);
 
+/// Getter for callee of CallSite.
+MLIR_CAPI_EXPORTED MlirLocation
+mlirLocationCallSiteGetCallee(MlirLocation location);
+
+/// Getter for caller of CallSite.
+MLIR_CAPI_EXPORTED MlirLocation
+mlirLocationCallSiteGetCaller(MlirLocation location);
+
+/// TypeID Getter for CallSite.
+MLIR_CAPI_EXPORTED MlirTypeID mlirLocationCallSiteGetTypeID(void);
+
+/// Checks whether the given location is an CallSite.
+MLIR_CAPI_EXPORTED bool mlirLocationIsACallSite(MlirLocation location);
+
 /// Creates a fused location with an array of locations and metadata.
 MLIR_CAPI_EXPORTED MlirLocation
 mlirLocationFusedGet(MlirContext ctx, intptr_t nLocations,
                      MlirLocation const *locations, MlirAttribute metadata);
 
+/// Getter for number of locations fused together.
+MLIR_CAPI_EXPORTED unsigned
+mlirLocationFusedGetNumLocations(MlirLocation location);
+
+/// Getter for locations of Fused. Requires pre-allocated memory of
+/// #fusedLocations X sizeof(MlirLocation).
+MLIR_CAPI_EXPORTED void
+mlirLocationFusedGetLocations(MlirLocation location,
+                              MlirLocation *locationsCPtr);
+
+/// Getter for metadata of Fused.
+MLIR_CAPI_EXPORTED MlirAttribute
+mlirLocationFusedGetMetadata(MlirLocation location);
+
+/// TypeID Getter for Fused.
+MLIR_CAPI_EXPORTED MlirTypeID mlirLocationFusedGetTypeID(void);
+
+/// Checks whether the given location is an Fused.
+MLIR_CAPI_EXPORTED bool mlirLocationIsAFused(MlirLocation location);
+
 /// Creates a name location owned by the given context. Providing null location
 /// for childLoc is allowed and if childLoc is null location, then the behavior
 /// is the same as having unknown child location.
@@ -277,6 +346,20 @@ MLIR_CAPI_EXPORTED MlirLocation mlirLocationNameGet(MlirContext context,
                                                     MlirStringRef name,
                                                     MlirLocation childLoc);
 
+/// Getter for name of Name.
+MLIR_CAPI_EXPORTED MlirIdentifier
+mlirLocationNameGetName(MlirLocation location);
+
+/// Getter for childLoc of Name.
+MLIR_CAPI_EXPORTED MlirLocation
+mlirLocationNameGetChildLoc(MlirLocation location);
+
+/// TypeID Getter for Name.
+MLIR_CAPI_EXPORTED MlirTypeID mlirLocationNameGetTypeID(void);
+
+/// Checks whether the given location is an Name.
+MLIR_CAPI_EXPORTED bool mlirLocationIsAName(MlirLocation location);
+
 /// Creates a location with unknown position owned by the given context.
 MLIR_CAPI_EXPORTED MlirLocation mlirLocationUnknownGet(MlirContext context);
 
@@ -978,6 +1061,12 @@ mlirValueReplaceAllUsesExcept(MlirValue of, MlirValue with,
                               intptr_t numExceptions,
                               MlirOperation *exceptions);
 
+/// Gets the location of the value.
+MLIR_CAPI_EXPORTED MlirLocation mlirValueGetLocation(MlirValue v);
+
+/// Gets the context that a value was created with.
+MLIR_CAPI_EXPORTED MlirContext mlirValueGetContext(MlirValue v);
+
 //===----------------------------------------------------------------------===//
 // OpOperand API.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Bindings/Python/Diagnostics.h b/mlir/include/mlir/Bindings/Python/Diagnostics.h
index ea80e14dde0f3..167002d561931 100644
--- a/mlir/include/mlir/Bindings/Python/Diagnostics.h
+++ b/mlir/include/mlir/Bindings/Python/Diagnostics.h
@@ -9,12 +9,13 @@
 #ifndef MLIR_BINDINGS_PYTHON_DIAGNOSTICS_H
 #define MLIR_BINDINGS_PYTHON_DIAGNOSTICS_H
 
-#include <cassert>
-#include <string>
-
 #include "mlir-c/Diagnostics.h"
 #include "mlir-c/IR.h"
-#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <cassert>
+#include <cstdint>
+#include <string>
 
 namespace mlir {
 namespace python {
@@ -24,33 +25,45 @@ namespace python {
 class CollectDiagnosticsToStringScope {
 public:
   explicit CollectDiagnosticsToStringScope(MlirContext ctx) : context(ctx) {
-    handlerID = mlirContextAttachDiagnosticHandler(ctx, &handler, &errorMessage,
-                                                   /*deleteUserData=*/nullptr);
+    handlerID =
+        mlirContextAttachDiagnosticHandler(ctx, &handler, &messageStream,
+                                           /*deleteUserData=*/nullptr);
   }
   ~CollectDiagnosticsToStringScope() {
-    assert(errorMessage.empty() && "unchecked error message");
+    assert(message.empty() && "unchecked error message");
     mlirContextDetachDiagnosticHandler(context, handlerID);
   }
 
-  [[nodiscard]] std::string takeMessage() { return std::move(errorMessage); }
+  [[nodiscard]] std::string takeMessage() {
+    std::string newMessage;
+    std::swap(message, newMessage);
+    return newMessage;
+  }
 
 private:
   static MlirLogicalResult handler(MlirDiagnostic diag, void *data) {
     auto printer = +[](MlirStringRef message, void *data) {
-      *static_cast<std::string *>(data) +=
-          llvm::StringRef(message.data, message.length);
+      *static_cast<llvm::raw_string_ostream *>(data)
+          << std::string_view(message.data, message.length);
     };
     MlirLocation loc = mlirDiagnosticGetLocation(diag);
-    *static_cast<std::string *>(data) += "at ";
+    *static_cast<llvm::raw_string_ostream *>(data) << "at ";
     mlirLocationPrint(loc, printer, data);
-    *static_cast<std::string *>(data) += ": ";
+    *static_cast<llvm::raw_string_ostream *>(data) << ": ";
     mlirDiagnosticPrint(diag, printer, data);
+    for (intptr_t i = 0; i < mlirDiagnosticGetNumNotes(diag); i++) {
+      *static_cast<llvm::raw_string_ostream *>(data) << "\n";
+      MlirDiagnostic note = mlirDiagnosticGetNote(diag, i);
+      handler(note, data);
+    }
     return mlirLogicalResultSuccess();
   }
 
   MlirContext context;
   MlirDiagnosticHandlerID handlerID;
-  std::string errorMessage = "";
+
+  std::string message;
+  llvm::raw_string_ostream messageStream{message};
 };
 
 } // namespace python
diff --git a/mlir/include/mlir/Bindings/Python/NanobindAdaptors.h b/mlir/include/mlir/Bindings/Python/NanobindAdaptors.h
index 0608182f00b7e..3646bf42e415f 100644
--- a/mlir/include/mlir/Bindings/Python/NanobindAdaptors.h
+++ b/mlir/include/mlir/Bindings/Python/NanobindAdaptors.h
@@ -321,6 +321,16 @@ struct type_caster<MlirType> {
   }
 };
 
+/// Casts MlirStringRef -> object.
+template <>
+struct type_caster<MlirStringRef> {
+  NB_TYPE_CASTER(MlirStringRef, const_name("MlirStringRef"))
+  static handle from_cpp(MlirStringRef s, rv_policy,
+                         cleanup_list *cleanup) noexcept {
+    return nanobind::str(s.data, s.length).release();
+  }
+};
+
 } // namespace detail
 } // namespace nanobind
 
diff --git a/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h b/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
index 8f82176f3b75f..e78f174ff8586 100644
--- a/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
+++ b/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
@@ -40,6 +40,8 @@ LogicalResult oneToOneRewrite(
 /// during the entire pattern lifetime.
 class ConvertToLLVMPattern : public ConversionPattern {
 public:
+  /// `SplitMatchAndRewrite` is deprecated. Use `matchAndRewrite` instead of
+  /// separate `match` and `rewrite`.
   using SplitMatchAndRewrite =
       detail::ConversionSplitMatchAndRewriteImpl<ConvertToLLVMPattern>;
 
@@ -149,6 +151,9 @@ class ConvertOpToLLVMPattern : public ConvertToLLVMPattern {
   using OpAdaptor = typename SourceOp::Adaptor;
   using OneToNOpAdaptor =
       typename SourceOp::template GenericAdaptor<ArrayRef<ValueRange>>;
+
+  /// `SplitMatchAndRewrite` is deprecated. Use `matchAndRewrite` instead of
+  /// separate `match` and `rewrite`.
   using SplitMatchAndRewrite = detail::ConversionSplitMatchAndRewriteImpl<
       ConvertOpToLLVMPattern<SourceOp>>;
 
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index 6074e0e8d822c..bbba495e613b2 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -10,7 +10,7 @@
 #define MLIR_CONVERSION_PASSES
 
 include "mlir/Pass/PassBase.td"
-
+include "mlir/Dialect/Vector/Transforms/VectorTransformsBase.td"
 
 //===----------------------------------------------------------------------===//
 // ToLLVM
@@ -1410,10 +1410,32 @@ def ConvertVectorToLLVMPass : Pass<"convert-vector-to-llvm"> {
            "bool", /*default=*/"false",
            "Enables the use of X86Vector dialect while lowering the vector "
 	   "dialect.">,
-    Option<"vectorTransformsOptions", "vector-transform-options",
-           "vector::VectorTransformsOptions",
-           /*default=*/"vector::VectorTransformsOptions()",
-           "Options to lower some operations like contractions and transposes.">,
+    Option<"vectorContractLowering", "vector-contract-lowering",
+           "vector::VectorContractLowering",
+           /*default=*/"vector::VectorContractLowering::Dot",
+           VectorContractLoweringAttr.summary, [{::llvm::cl::values(
+           clEnumValN(::mlir::vector::VectorContractLowering::Dot, "dot",
+            "Progressively lower to finer grained `vector.contract` and dot-products. (default)"),
+           clEnumValN(::mlir::vector::VectorContractLowering::Matmul, "matmul",
+            "Lower to `vector.matrix_multiply`, maps 1-1 to LLVM matrix intrinsics."),
+           clEnumValN(::mlir::vector::VectorContractLowering::OuterProduct, "outerproduct",
+            "Lower to `vector.outerproduct`."),
+           clEnumValN(::mlir::vector::VectorContractLowering::ParallelArith, "parallelarith",
+            "Lower contract with all reduction dimensions unrolled to 1 to a vector elementwise operations.")
+	        )}]>,
+    Option<"vectorTransposeLowering", "vector-transpose-lowering",
+           "vector::VectorTransposeLowering",
+           /*default=*/"vector::VectorTransposeLowering::EltWise",
+           VectorTransposeLoweringAttr.summary, [{::llvm::cl::values(
+           clEnumValN(::mlir::vector::VectorTransposeLowering::EltWise, "eltwise",
+            "Lower transpose into element-wise extract and inserts (default)"),
+           clEnumValN(::mlir::vector::VectorTransposeLowering::Flat, "flat",
+            "Lower 2-D transpose to `vector.flat_transpose`, maps 1-1 to LLVM matrix intrinsics"),
+           clEnumValN(::mlir::vector::VectorTransposeLowering::Shuffle1D, "shuffle1d",
+            "Lower 2-D transpose to `vector.shuffle` on 1-D vector."),
+           clEnumValN(::mlir::vector::VectorTransposeLowering::Shuffle16x16, "shuffle16x16",
+            "Lower 2-D transpose to `vector.shuffle` on 16x16 vector.")
+          )}]>,
   ];
 }
 
diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
index 7c950623f77f4..bbf38f2293448 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
@@ -46,6 +46,13 @@ bool isTopLevelValue(Value value, Region *region);
 /// trait `AffineScope`; `nullptr` if there is no such region.
 Region *getAffineScope(Operation *op);
 
+/// Returns the closest region enclosing `op` that is held by a non-affine
+/// operation; `nullptr` if there is no such region. This method is meant to
+/// be used by affine analysis methods (e.g. dependence analysis) which are
+/// only meaningful when performed among/between operations from the same
+/// analysis scope.
+Region *getAffineAnalysisScope(Operation *op);
+
 /// AffineDmaStartOp starts a non-blocking DMA operation that transfers data
 /// from a source memref to a destination memref. The source and destination
 /// memref need not be of the same dimensionality, but need to have the same
diff --git a/mlir/include/mlir/Dialect/DLTI/DLTIAttrs.td b/mlir/include/mlir/Dialect/DLTI/DLTIAttrs.td
index 007a417653e4c..cf3a72633587d 100644
--- a/mlir/include/mlir/Dialect/DLTI/DLTIAttrs.td
+++ b/mlir/include/mlir/Dialect/DLTI/DLTIAttrs.td
@@ -74,6 +74,9 @@ def DLTI_DataLayoutSpecAttr :
     /// Returns the endiannes identifier.
     StringAttr getEndiannessIdentifier(MLIRContext *context) const;
 
+    /// Returns the default memory space identifier.
+    StringAttr getDefaultMemorySpaceIdentifier(MLIRContext *context) const;
+
     /// Returns the alloca memory space identifier.
     StringAttr getAllocaMemorySpaceIdentifier(MLIRContext *context) const;
 
diff --git a/mlir/include/mlir/Dialect/DLTI/DLTIBase.td b/mlir/include/mlir/Dialect/DLTI/DLTIBase.td
index 136fe8781e717..1a08bafda54ee 100644
--- a/mlir/include/mlir/Dialect/DLTI/DLTIBase.td
+++ b/mlir/include/mlir/Dialect/DLTI/DLTIBase.td
@@ -53,6 +53,9 @@ def DLTI_Dialect : Dialect {
     constexpr const static ::llvm::StringLiteral
     kDataLayoutManglingModeKey = "dlti.mangling_mode";
 
+    constexpr const static ::llvm::StringLiteral
+    kDataLayoutDefaultMemorySpaceKey = "dlti.default_memory_space";
+
     constexpr const static ::llvm::StringLiteral
     kDataLayoutAllocaMemorySpaceKey = "dlti.alloca_memory_space";
     
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
index 18fec95f700c4..f194e70ee275b 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
@@ -652,6 +652,20 @@ def ROCDL_DPPUpdateOp : ROCDL_IntrOp<"update.dpp", [], [0],
   }];
 }
 
+def ROCDL_V2I16Type : FixedVectorOfLengthAndType<[2], [I16]>,
+                        BuildableType<"::mlir::VectorType::get("
+                          "{2},$_builder.getI16Type())">;
+
+def ROCDL_V2F16Type : FixedVectorOfLengthAndType<[2], [F16]>,
+                        BuildableType<"::mlir::VectorType::get("
+                          "{2},$_builder.getF16Type())">;
+
+def ROCDL_V2BF16Type : FixedVectorOfLengthAndType<[2], [BF16]>,
+                        BuildableType<"::mlir::VectorType::get("
+                          "{2},$_builder.getBF16Type())">;
+
+// TODO: The word and byte selectors are immarg in LLVM 
+// update to be attributes in MLIR
 //===---------------------------------------------------------------------===//
 // 16-bit float intrinsics
 //===---------------------------------------------------------------------===//
@@ -667,10 +681,168 @@ def ROCDL_CvtPkRtz:
   }];
 }
 
+def ROCDL_CvtScaleF32PkFp8F16 :
+    ROCDL_IntrOp<"cvt.scalef32.pk.fp8.f16", [], [], [Pure], 1>,
+    Arguments<(ins ROCDL_V2I16Type: $old, ROCDL_V2F16Type: $src, F32: $scale, I1:$wordSel)> {
+    let summary = "Scale and convert f16 to packed fp8";
+    let description = [{
+    Scale `src` by the exponent in `scale` then convert to packed fp8.
+    Store the result in low/high word based on $wordSel, preserving the other word.
+  }];
+  let assemblyFormat = [{
+    attr-dict $src `,` $scale `->` $old `[` $wordSel `]` `:` type($res)
+  }];
+}
+
+def ROCDL_CvtScaleF32PkFp8Bf16 :
+    ROCDL_IntrOp<"cvt.scalef32.pk.fp8.bf16", [], [], [Pure], 1>,
+    Arguments<(ins ROCDL_V2I16Type: $old, ROCDL_V2BF16Type: $src, F32: $scale, I1:$wordSel)> {
+    let summary = "Scale and convert packed bf16 to packed fp8";
+    let description = [{
+    Scale `src` by the exponent in `scale` then convert to packed fp8.
+    Store the result in low/high word based on $wordSel, preserving the other word.
+  }];
+  let assemblyFormat = [{
+    attr-dict $src `,` $scale `->` $old `[` $wordSel `]` `:` type($res)
+  }];
+}
+
+
+def ROCDL_CvtScaleF32PkBf8F16 :
+    ROCDL_IntrOp<"cvt.scalef32.pk.bf8.f16", [], [], [Pure], 1>,
+    Arguments<(ins ROCDL_V2I16Type: $old, ROCDL_V2F16Type: $src, F32: $scale, I1:$wordSel)> {
+    let summary = "Scale and convert f16 to packed bf8";
+    let description = [{
+    Scale `src` by the exponent in `scale` then convert to packed bf8.
+    Store the result in low/high word based on $wordSel, preserving the other word.
+  }];
+  let assemblyFormat = [{
+    attr-dict $src `,` $scale `->` $old `[` $wordSel `]` `:` type($res)
+  }];
+}
+
+
+def ROCDL_CvtScaleF32PkBf8Bf16 :
+    ROCDL_IntrOp<"cvt.scalef32.pk.bf8.bf16", [], [], [Pure], 1>,
+    Arguments<(ins ROCDL_V2I16Type: $old, ROCDL_V2BF16Type: $src, F32: $scale, I1:$wordSel)> {
+    let summary = "Scale and convert bf16 to packed bf8";
+    let description = [{
+    Scale `src` by the exponent in `scale` then convert to packed bf8.
+    Store the result in low/high word based on $wordSel, preserving the other word.
+  }];
+  let assemblyFormat = [{
+    attr-dict $src `,` $scale `->` $old `[` $wordSel `]` `:` type($res)
+  }];
+}
+
+def ROCDL_CvtScaleF32SrFp8F16 :
+    ROCDL_IntrOp<"cvt.scalef32.sr.fp8.f16", [], [], [Pure], 1>,
+    Arguments<(ins I32:$old, F16:$src, I32:$seed, F32: $scale, I32:$byteSel)> {
+    let summary = "Scale and convert f16 to packed fp8 using stochastic rounding";
+    let description = [{
+    Scale `src` by the exponent in `scale` then convert to packed p8 with stochastic rounding
+    using seed data in `seed`. store into the `byteSel`th byte of `old`, preserving the others.
+
+  }];
+  let assemblyFormat = [{
+    attr-dict $src `,` $seed `,` $scale `->` $old `[` $byteSel `]` `:` type($res)
+  }];
+}
+
+def ROCDL_CvtScaleF32SrBf8F16 :
+    ROCDL_IntrOp<"cvt.scalef32.sr.bf8.f16", [], [], [Pure], 1>,
+    Arguments<(ins I32:$old, F16:$src, I32:$seed, F32: $scale, I32:$byteSel)> {
+    let summary = "Scale and convert f16 to packed bf8 using stochastic rounding";
+    let description = [{
+    Scale `src` by the exponent in `scale` then convert to packed bf8 with stochastic rounding
+    using seed data in `seed`. store into the `byteSel`th byte of `old`, preserving the others.
+
+  }];
+  let assemblyFormat = [{
+    attr-dict $src `,` $seed `,` $scale `->` $old `[` $byteSel `]` `:` type($res)
+  }];
+}
+
+def ROCDL_CvtScaleF32SrFp8Bf16 :
+    ROCDL_IntrOp<"cvt.scalef32.sr.fp8.bf16", [], [], [Pure], 1>,
+    Arguments<(ins I32:$old, BF16:$src, I32:$seed, F32: $scale, I32:$byteSel)> {
+    let summary = "Scale and convert packed bf16 to packed fp8 using stochastic rounding";
+    let description = [{
+    Scale `src` by the exponent in `scale` then convert to packed fp8 with stochastic rounding
+    using seed data in `seed`. store into the `byteSel`th byte of `old`, preserving the others.
+
+  }];
+  let assemblyFormat = [{
+    attr-dict $src `,` $seed `,` $scale `->` $old `[` $byteSel `]` `:` type($res)
+  }];
+}
+
+def ROCDL_CvtScaleF32SrBf8Bf16:
+    ROCDL_IntrOp<"cvt.scalef32.sr.bf8.bf16", [], [], [Pure], 1>,
+    Arguments<(ins I32:$old, BF16:$src, I32:$seed, F32: $scale, I32:$byteSel)> {
+    let summary = "Scale and convert bf16 to packed fp8 using stochastic rounding";
+    let description = [{
+    Scale `src` by the exponent in `scale` then convert to packed p8 with stochastic rounding
+    using seed data in `seed`. store into the `byteSel`th byte of `old`, preserving the others.
+
+  }];
+  let assemblyFormat = [{
+    attr-dict $src `,` $seed `,` $scale `->` $old `[` $byteSel `]` `:` type($res)
+  }];
+}
+
+def ROCDL_CvtScaleF32PkF16Fp8 :
+    ROCDL_IntrOp<"cvt.scalef32.pk.f16.fp8", [], [], [Pure], 1>,
+    Arguments<(ins I32:$src, F32: $scale, I1:$wordSel)> {
+    let summary = "Scale and convert fp8 to packed f16";
+    let description = [{ Scale `src` based on $wordSel by the exponent in `scale` 
+    then convert to packed f16.
+  }];
+  let assemblyFormat = [{
+    attr-dict $src `[` $wordSel `]` `,` $scale `:` type($res)
+  }];
+}
+
+def ROCDL_CvtScaleF32PkF16Bf8 :
+    ROCDL_IntrOp<"cvt.scalef32.pk.f16.bf8", [], [], [Pure], 1>,
+    Arguments<(ins I32:$src, F32: $scale, I1:$wordSel)> {
+    let summary = "Scale and convert bf8 to packed f16";
+    let description = [{ Scale `src` based on $wordSel by the exponent in `scale`
+    then convert to packed f16.
+  }];
+  let assemblyFormat = [{
+    attr-dict $src `[` $wordSel `]` `,` $scale `:` type($res)
+  }];
+}
+
+def ROCDL_CvtScaleF16Fp8 :
+    ROCDL_IntrOp<"cvt.scalef32.f16.fp8", [], [], [Pure], 1>,
+    Arguments<(ins ROCDL_V2F16Type:$old, I32:$src, F32: $scale, I32:$byteSel, I1:$wordSel)> {
+    let summary = "Scale and convert fp8 to f16";
+    let description = [{ Scale `src` based on $wordSel by the exponent in `scale`
+    then convert to f16 store into the `byteSel`th byte of `old`, preserving the others.
+  }];
+  let assemblyFormat = [{
+    attr-dict $src `[` $wordSel `]` `,` $scale `->` $old `[` $byteSel `]` `:` type($res)
+  }];
+}
+
+def ROCDL_CvtScaleF16Bf8 :
+    ROCDL_IntrOp<"cvt.scalef32.f16.bf8", [], [], [Pure], 1>,
+    Arguments<(ins ROCDL_V2F16Type:$old, I32:$src, F32: $scale, I32:$byteSel, I1:$wordSel)> {
+    let summary = "Scale and convert fp8 to f16";
+    let description = [{ Scale `src` based on $wordSel by the exponent in `scale`
+    then convert to f16 store into the `byteSel`th byte of `old`, preserving the others.
+  }];
+  let assemblyFormat = [{
+    attr-dict $src `[` $wordSel `]` `,` $scale `->` $old `[` $byteSel `]` `:` type($res)
+  }];
+}
+
 //===---------------------------------------------------------------------===//
 // 32-bit float intrinsics
 //===---------------------------------------------------------------------===//
-def ROCDL_CvtScalePkF32Fp8 :
+def ROCDL_CvtScale32PkF32Fp8 :
     ROCDL_IntrOp<"cvt.scalef32.pk.f32.fp8", [], [], [Pure], 1>,
     Arguments<(ins I32:$src, F32: $scale, I1:$wordSel)> {
   let summary = "Scale and convert packed fp8 to packed f32";
@@ -682,7 +854,7 @@ def ROCDL_CvtScalePkF32Fp8 :
     attr-dict $src `[` $wordSel `]` `,` $scale `:` type($res)
   }];
 }
-def ROCDL_CvtScalePkF32Bf8 :
+def ROCDL_CvtScale32PkF32Bf8 :
     ROCDL_IntrOp<"cvt.scalef32.pk.f32.bf8", [], [], [Pure], 1>,
     Arguments<(ins I32:$src, F32: $scale, I1:$wordSel)> {
   let summary = "Scale and convert packed bf8 to packed f32";
@@ -697,10 +869,6 @@ def ROCDL_CvtScalePkF32Bf8 :
 //===---------------------------------------------------------------------===//
 // 8-bit float scale intrinsics
 //===---------------------------------------------------------------------===//
-def ROCDL_V2I16Type : FixedVectorOfLengthAndType<[2], [I16]>,
-                        BuildableType<"::mlir::VectorType::get("
-                          "{2},$_builder.getI16Type())">;
-
 def ROCDL_CvtScaleF32PkFp8F32:
     ROCDL_IntrOp<"cvt.scalef32.pk.fp8.f32", [], [], [Pure], 1>,
     Arguments<(ins ROCDL_V2I16Type:$old, F32:$srcA, F32:$srcB, F32:$scale, I1:$wordSel)> {
diff --git a/mlir/include/mlir/Dialect/Math/Transforms/Passes.h b/mlir/include/mlir/Dialect/Math/Transforms/Passes.h
index 9adc1c6940a15..c0fe5d3be448a 100644
--- a/mlir/include/mlir/Dialect/Math/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/Math/Transforms/Passes.h
@@ -9,6 +9,7 @@
 #ifndef MLIR_DIALECT_MATH_TRANSFORMS_PASSES_H_
 #define MLIR_DIALECT_MATH_TRANSFORMS_PASSES_H_
 
+#include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/Pass.h"
 
 namespace mlir {
@@ -52,12 +53,14 @@ void populatePolynomialApproximateErfcPattern(RewritePatternSet &patterns);
 // Adds patterns to convert to f32 around math functions for which `predicate`
 // returns true.
 void populateMathF32ExpansionPatterns(
-    RewritePatternSet &patterns, llvm::function_ref<bool(StringRef)> predicate);
+    RewritePatternSet &patterns, llvm::function_ref<bool(StringRef)> predicate,
+    PatternBenefit = 1);
 
 // Adds patterns to enable polynomial approximations for math functions for
 // which `predicate` returns true.
 void populateMathPolynomialApproximationPatterns(
-    RewritePatternSet &patterns, llvm::function_ref<bool(StringRef)> predicate);
+    RewritePatternSet &patterns, llvm::function_ref<bool(StringRef)> predicate,
+    PatternBenefit = 1);
 
 // Legacy. Calls both populateMathF32ExpansionPatterns and
 // populateMathPolynomialApproximationPatterns with predicates enabling a
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/NVGPU/IR/CMakeLists.txt
index 13d754ca06316..ecdaae7f24d93 100644
--- a/mlir/include/mlir/Dialect/NVGPU/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/CMakeLists.txt
@@ -1,5 +1,10 @@
 add_mlir_dialect(NVGPU nvgpu)
-add_mlir_doc(NVGPU NVGPU Dialects/ -gen-dialect-doc)
+add_mlir_doc(NVGPUOps NVGPU Dialects/ -gen-dialect-doc)
+
+set(LLVM_TARGET_DEFINITIONS NVGPUOps.td)
+mlir_tablegen(NVGPUOps.h.inc -gen-op-decls)
+mlir_tablegen(NVGPUOps.cpp.inc -gen-op-defs)
+add_public_tablegen_target(MLIRNVGPUOpsIncGen)
 
 set(LLVM_TARGET_DEFINITIONS NVGPU.td)
 mlir_tablegen(NVGPUEnums.h.inc -gen-enum-decls)
@@ -11,7 +16,7 @@ mlir_tablegen(NVGPUAttrDefs.h.inc -gen-attrdef-decls)
 mlir_tablegen(NVGPUAttrDefs.cpp.inc -gen-attrdef-defs)
 add_public_tablegen_target(MLIRNVGPUAttributesIncGen)
 
-set(LLVM_TARGET_DEFINITIONS NVGPU.td)
-mlir_tablegen(NVGPUAttrTypes.h.inc -gen-typedef-decls)
-mlir_tablegen(NVGPUAttrTypes.cpp.inc -gen-typedef-decls)
+set(LLVM_TARGET_DEFINITIONS NVGPUTypes.td)
+mlir_tablegen(NVGPUTypeDefs.h.inc -gen-typedef-decls)
+mlir_tablegen(NVGPUTypeDefs.cpp.inc -gen-typedef-defs)
 add_public_tablegen_target(MLIRNVGPUTypesIncGen)
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td
index f48fa9976da12..7f7a54cb0c57e 100644
--- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td
@@ -1,24 +1,13 @@
-//===-- NVGPU.td - NVGPU dialect operation definitions *- tablegen -*------===//
+//===-- NVGPU.td - Attribute defs for NVGPU dialect *- tablegen -*---------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
-// This file defines the basic operations for the NVGPU dialect.
-//
-// This NVGPU provides a bridge between the target agnostic GPU and Vector
-// dialects and lower level NVVM dialect. This allow representing PTX specific
-// operations while using MLIR high level concepts like memref and 2-D vector.
-//
-// Ops semantic are going to be based on vendor specific PTX defintion:
-// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html
-//
-//===----------------------------------------------------------------------===//
 
-#ifndef NVGPU
-#define NVGPU
+#ifndef MLIR_DIALECT_NVGPU_IR_NVGPU_TD
+#define MLIR_DIALECT_NVGPU_IR_NVGPU_TD
 
 include "mlir/Interfaces/InferTypeOpInterface.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
@@ -127,712 +116,4 @@ def TensorMapOOBAttr : EnumAttr<NVGPU_Dialect, TensorMapOOBKind, "oob">;
 def TensorMapInterleaveAttr : EnumAttr<NVGPU_Dialect, TensorMapInterleaveKind, "interleave">;
 def RcpRoundingModeAttr : EnumAttr<NVGPU_Dialect, RcpRoundingMode, "rcp_rounding_mode">;
 
-//===----------------------------------------------------------------------===//
-// NVGPU Type Definitions
-//===----------------------------------------------------------------------===//
-
-class NVGPU_Type<string name, string typeMnemonic,
-        list<Trait> traits = []> : TypeDef<NVGPU_Dialect, name, traits> {
-  let mnemonic = typeMnemonic;
-}
-
-def NVGPU_DeviceAsyncToken : NVGPU_Type<"DeviceAsyncToken",
-                                        "device.async.token", []> {
-  let summary = "device async token type";
-  let description = [{
-    `nvgpu.device.async.token` is a type returned by an asynchronous operation
-    that runs on the GPU (device). It is used to establish an SSA-based link
-    between the async operation (e.g. DeviceAsyncCopy) and operations that
-    group or synchronize the async operations (e.g. DeviceAsyncCreateGroupOp,
-    DeviceAsyncWaitOp).
-  }];
-}
-
-def NVGPU_MBarrierGroup : NVGPU_Type<"MBarrierGroup", "mbarrier.group", []> {
-  let summary = "mbarrier barrier type";
-  let description = [{
-    This is the type for one or more mbarrier object in shared memory that is 
-    used to synchronize a variable number of threads.
-
-    If `num_barriers` is not set, the number of mbarrier objects is 1.
-
-    A mbarrier object is 64 bit with 8 byte alignment. The mbarrier object 
-    can be initiated and invalidated.
-
-    [See for more details in PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#size-and-alignment-of-mbarrier-object)
-  }];    
-  let parameters = (ins "Attribute":$memorySpace, DefaultValuedParameter<"unsigned", "1">:$num_barriers);
-  let assemblyFormat = "`<` struct(params) `>`";
-  let builders = [
-    TypeBuilder<(ins "Attribute":$memorySpace), [{
-      return $_get($_ctxt, memorySpace, 1);
-    }]>
-  ];
-}
-
-def NVGPU_MBarrierToken : NVGPU_Type<"MBarrierToken", "mbarrier.token", []> { }
-
-// https://docs.nvidia.com/cuda/parallel-thread-execution/#tensor-map
-def NVGPU_TensorMapDescriptor : NVGPU_Type<"TensorMapDescriptor", "tensormap.descriptor", []> {
-  let summary = "TensorMap descriptor";
-  let parameters = (ins "MemRefType":$tensor,
-                        EnumParameter<TensorMapSwizzleKind>:$swizzle,
-                        EnumParameter<TensorMapL2PromoKind>:$l2promo,
-                        EnumParameter<TensorMapOOBKind>:$oob,
-                        EnumParameter<TensorMapInterleaveKind>:$interleave);
-  let description = [{
-    `nvgpu.tma.descriptor` is a type that represents a TMA descriptor. It is 
-    128-byte object either in constant space or kernel paramater.    
-  }];
-  let assemblyFormat = "`<` struct(params) `>`";
-}
-
-def NVGPU_WarpgroupMatrixDescriptor : NVGPU_Type<"WarpgroupMatrixDescriptor", "warpgroup.descriptor", []> {
-  let summary = "Warpgroup matrix descriptor type";
-  let description = [{
-  The descriptor specifies the properties of the matrix in shared memory that 
-  is a multiplicand in the matrix multiply and accumulate operation. 
-  
-  The descriptor is a 64-bit value contained in a register with the following:
-  ```
-  +---------+-----+-----------+-----+-----------+-----+-----+-----------+-----+
-  |   0-13  |14-15|   16-29   |30-31|   32-45   |46-48|49-51|   52-61   |62-63|
-  +---------+-----+-----------+-----+-----------+-----+-----+-----------+-----+
-  |  14bits |2bits|   14bits  |2bits|   14bits  |2bits|3bits|   10bits  |2bits|
-  +---------+-----+-----------+-----+-----------+-----+-----+-----------+-----+
-  | BaseAddr|  0  | LeadingDim|  0  |   Stride  |  0  |Offst|     0     |Swzle|
-  +---------+-----+-----------+-----+-----------+-----+-----+-----------+-----+
-  ```
-   
-  [See for more details in PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-shared-memory-layout-matrix-descriptor) 
-  
-  }];  
-  let parameters = (ins "MemRefType":$tensor);
-  let assemblyFormat = "`<` struct(params) `>`";
-}
-
-def NVGPU_WarpgroupAccumulator : NVGPU_Type<"WarpgroupAccumulator", "warpgroup.accumulator", []> {
-  let parameters = (ins "VectorType":$fragmented);
-  let assemblyFormat = "`<` struct(params) `>`";
-  let description = [{
-    This type represents the result matrix obtained from `nvgpu.warpgroup.mma`. 
-    The `$fragmented` type signifies the distributed or fragmented result 
-    vector that is collectively owned by all the threads in the warp-group 
-    that executed `nvgpu.warpgroup.mma`.
-    [See the details of register fragment layout for accumulator matrix D]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#wgmma-64n16-d) 
-  }];
-}
-
-//===----------------------------------------------------------------------===//
-// NVGPU Op Definitions
-//===----------------------------------------------------------------------===//
-
-class NVGPU_Op<string mnemonic, list<Trait> traits = []> :
-  Op<NVGPU_Dialect, mnemonic, traits> {}
-
-def NVGPU_LdMatrixOp : NVGPU_Op<"ldmatrix", [
-                                MemoryEffects<[MemRead]>,
-                                PredOpTrait<"srcMemref and res have same element type",
-                                            TCresVTEtIsSameAsOp<0, 0>>]> {
-  let description = [{
-    The `nvgpu.ldmatrix` op represents loading a matrix fragment from
-    memory to registers. The source and result type must be compatible
-    with lowering to the `nvvm.ldmatrix` instruction. This op represents
-    the distributed version of a `vector.transfer_read` as an intermediate
-    step between lowering from `vector.transfer_read` to `nvvm.ldmatrix`.
-
-    This operation is meant to follow the semantic of described here:
-    https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#warp-level-matrix-instructions-ldmatrix
-
-    Example:
-    ```mlir
-    %0 = nvgpu.ldmatrix %sm[%c0, %c0] {numTiles = 4 : i32, transpose = false} :
-      memref<?x?xf16, 3> -> vector<4x2xf16>
-    ```
-  }];
-
-  let arguments = (ins Arg<AnyMemRef, "", [MemReadAt<0, FullEffect>]>:$srcMemref,
-                           Variadic<Index>:$indices, BoolAttr:$transpose,
-                           I32Attr:$numTiles);
-  let results = (outs AnyVectorOfNonZeroRank:$res);
-  let assemblyFormat = [{
-    $srcMemref`[` $indices `]` attr-dict `:` type($srcMemref) `->` type($res)
-  }];
-
-  let hasVerifier = 1;
-}
-
-class NVGPU_MmaSyncOp<string mnemonic> :
-        NVGPU_Op<mnemonic,  [Pure,
-                             PredOpTrait<"matrixA and matrixB have same element type",
-                                         TCopVTEtIsSameAs<0, 1>>]> {
-  code extraBaseClassDeclaration = [{
-    std::array<int64_t, 3> getMmaShapeAsArray() {
-      ArrayAttr mmaShape = this->getMmaShape();
-      assert(mmaShape.size() == 3 && "mmaShape should be three integers");
-      return {::llvm::cast<IntegerAttr>(mmaShape[0]).getInt(),
-              ::llvm::cast<IntegerAttr>(mmaShape[1]).getInt(),
-              ::llvm::cast<IntegerAttr>(mmaShape[2]).getInt()};
-    }
-  }];
-
-  let hasVerifier = 1;
-}
-
-def NVGPU_MmaSyncOp : NVGPU_MmaSyncOp<"mma.sync"> {
-  let description = [{
-    The `nvgpu.mma.sync` op represents the warp-level matrix-multiply-and-
-    accumulate (mma) operation that is compatible with `nvvm.mma.sync`.
-    The operands and results vector sizes are thread-level onwership to
-    the warp-level mma operation shape. `mmaShape` attribute holds the
-    warp-level matrix-multiply shape.
-
-    The `nvgpu.mma.sync` op serves as an intermediate point between lowering from
-    `vector.contract` to `nvvm.mma.sync`.
-
-    This operation is meant to follow the semantic of described here:
-      https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#warp-level-matrix-instructions-mma
-
-    Example:
-
-    ```mlir
-    %res = nvgpu.mma.sync (%matrixA, %matrixB, %matrixC) {mmaShape = [16, 8, 16]} :
-        (vector<4x2xf16>, vector<2x2xf16>, vector<2x2xf32>) -> vector<2x2xf32>
-    ```
-  }];
-  let arguments = (ins AnyVectorOfNonZeroRank:$matrixA,
-                       AnyVectorOfNonZeroRank:$matrixB,
-                       AnyVectorOfNonZeroRank:$matrixC,
-                       I64ArrayAttr:$mmaShape,
-                       OptionalAttr<UnitAttr>:$tf32Enabled);
-
-  let results = (outs AnyVectorOfNonZeroRank:$res);
-
-  let builders = [
-    OpBuilder<(ins "Value":$matrixA,
-                   "Value":$matrixB,
-                   "Value":$matrixC,
-                   "ArrayAttr":$mmaShape)>,
-    OpBuilder<(ins "Value":$matrixA,
-                   "Value":$matrixB,
-                   "Value":$matrixC,
-                   "ArrayRef<int64_t>":$mmaShape,
-                   CArg<"bool", "false">:$tf32Enabled)>
-  ];
-
-  let assemblyFormat = [{
-    `(` $matrixA`,` $matrixB`,` $matrixC `)` attr-dict
-    `:` `(` type($matrixA) `,` type($matrixB) `,` type($matrixC) `)` `->` type($res)
-  }];
-
-  let extraClassDeclaration = extraBaseClassDeclaration;
-}
-
-def NVGPU_MmaSparseSyncMetadataType : FixedVectorOfLengthAndType<[2], [I16]>,
-                        BuildableType<"::mlir::VectorType::get("
-                          "{2},$_builder.getI16Type())">;
-
-def NVGPU_MmaSparseSyncOp : NVGPU_MmaSyncOp<"mma.sp.sync"> {
-  let description = [{
-  The `nvgu.mma.sp.sync` operation performs a warp-distributed MMA operation
-  where operand A is "structured sparse". In this case, the `matrixA` operand
-  represents the (warp-distributed) non-zero values of operand A, and the
-  `sparse_metadata` operand provides the indices.
-
-  The full description of the sparsity storage format and distribution scheme is
-  described in the PTX docs. This operation is meant to follow the semantic
-  described in the PTX documentation here:
-  https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#warp-level-matrix-instructions-for-sparse-mma
-
-  The way the indices are distributed among the threads in a warp is controlled
-  by the optional `sparsity_selector` operand, which is `0` by default. For
-  more information, please consult the PTX documentation linked above.
-
-  Example (targetingthe f16 16x8x32 `mma.sp` PTX instruction):
-
-  ```mlir
-  nvgpu.mma.sp.sync (%a, %b, %c) metadata (%meta) {mmaShape = [16, 8, 32]} :
-    (vector<4x2xf16>, vector<2x2xf16>, vector<2x2xf16>) -> vector<2x2xf16>
-  ```
-  }];
-
-  let arguments = (ins AnyVectorOfNonZeroRank:$matrixA,
-                       AnyVectorOfNonZeroRank:$matrixB,
-                       AnyVectorOfNonZeroRank:$matrixC,
-                       NVGPU_MmaSparseSyncMetadataType:$sparseMetadata,
-                       I64ArrayAttr:$mmaShape,
-                       DefaultValuedAttr<I32Attr, "0">:$sparsitySelector,
-                       OptionalAttr<UnitAttr>:$tf32Enabled
-                       );
-
-  let results = (outs AnyVectorOfNonZeroRank:$res);
-
-  let builders = [
-    OpBuilder<(ins "Value":$matrixA,
-                   "Value":$matrixB,
-                   "Value":$matrixC,
-                   "Value":$sparseMetadata,
-                   "ArrayRef<int64_t>":$mmaShape)>
-  ];
-
-  let assemblyFormat = [{
-    `(` $matrixA`,` $matrixB`,` $matrixC `)` `metadata` `(` $sparseMetadata `)` attr-dict
-    `:` `(` type($matrixA) `,` type($matrixB) `,` type($matrixC) `)` `->` type($res)
-  }];
-
-  let extraClassDeclaration = extraBaseClassDeclaration;
-}
-
-def NVGPU_DeviceAsyncCopyOp : NVGPU_Op<"device_async_copy", [
-                                       AttrSizedOperandSegments]> {
-  let summary = "device-side asynchronous copy";
-  let description = [{
-    The `nvgpu.device_async_copy` op initiates an asynchronous copy operation of
-    elements from source (global memory) to the destination (shared memory)
-    without blocking the thread. The async copy is added to a group.
-
-    This op is meant to be used with `nvgpu.device_async_create_group` and
-    `nvgpu.device_async_wait` to synchronize copies as explained in those ops
-    descriptions.
-
-    `bypassL1` attribute is hint to the hardware to bypass the L1 cache during
-    async copy, this hint may be ignored by the hardware.
-
-    `dstElements` attribute is the total number of elements written to
-    destination (shared memory).
-
-    `srcElements` argument is the total number of elements read from
-    source (global memory).
-
-    `srcElements` is an optional argument and when present the op only reads
-    `srcElements` number of elements from the source (global memory) and zero fills
-    the rest of the elements in the destination (shared memory).
-
-    In order to do a copy and wait for the result we need the following
-    combination:
-    ```
-    // copy 1.
-    %cp1 = nvgpu.device_async_copy %A[%c0], %B[%c0], 4 :memref<16xf32> to memref<16xf32, 3>
-    // copy 2.
-    %cp2 = nvgpu.device_async_copy %C[%c0], %D[%c0], 4 : memref<16xf32> to memref<16xf32, 3>
-    // group 1 contains copy 1 and copy 2.
-    %token1 = nvgpu.device_async_create_group %cp1, %cp2
-    // copy 3.
-    %cp3 = nvgpu.device_async_copy %E[%c0], %F[%c0], 4 : memref<16xf32> to memref<16xf32, 3>
-    // group 2 contains copy 3.
-    %token2 = nvgpu.device_async_create_group %cp3
-    // after the wait copy 1 and copy 2 are complete.
-    nvgpu.device_async_wait %token1
-    // after the wait copy 3 is complete.
-    nvgpu.device_async_wait %token2
-    ```
-
-    Example:
-
-    ```mlir
-    %0 = nvgpu.device_async_copy %src[%c0, %c0], %dst[%c0, %c0, %c0], 4 :
-      memref<4x5xf32> to memref<2x7x5xf32, 3>
-    ```
-  }];
-  let results = (outs NVGPU_DeviceAsyncToken:$asyncToken);
-  let arguments = (ins Arg<AnyMemRef, "", [MemWriteAt<0, FullEffect>]>:$dst,
-                       Variadic<Index>:$dstIndices,
-                       Arg<AnyMemRef, "", [MemReadAt<0, FullEffect>]>:$src,
-                       Variadic<Index>:$srcIndices,
-                       IndexAttr:$dstElements,
-                       Optional<Index>:$srcElements,
-                       OptionalAttr<UnitAttr>:$bypassL1);
-  let assemblyFormat = [{
-    $src `[` $srcIndices `]` `,` $dst `[` $dstIndices `]` `,` $dstElements (`,` $srcElements^)?
-      attr-dict `:` type($src) `to` type($dst)
-  }];
-  let hasVerifier = 1;
-}
-
-def NVGPU_DeviceAsyncCreateGroupOp : NVGPU_Op<"device_async_create_group", []> {
-  let summary = "device side asynchronous create group operation";
-  let description = [{
-    The `nvgpu.device_async_create_group` op creates a group of memory accesses
-    containing all the pending `device_async_copy` operations associated with
-    argument tokens. Each token can only be part of one group.
-
-    It returns a token that can be use to wait until the group fully completes.
-
-    This is meant to be used with `nvgpu.device_async_wait` to synchronize copies
-    as explained in those ops descriptions.
-
-    Groups are executed in the order they are created.
-
-    Example:
-
-    ```mlir
-    %0 = nvgpu.device_async_create_group
-  ```
-  }];
-  let results = (outs NVGPU_DeviceAsyncToken:$asyncToken);
-  let arguments = (ins Variadic<NVGPU_DeviceAsyncToken>:$inputTokens);
-  let assemblyFormat = [{
-    $inputTokens attr-dict
-  }];
-}
-
-def NVGPU_DeviceAsyncWaitOp : NVGPU_Op<"device_async_wait", []> {
-  let summary = "Wait for async gpu ops to complete.";
-  let description = [{
-    The `nvgpu.device_async_wait` op will block the execution thread until the group
-    associated with the source token is fully completed.
-
-    The optional `$numGroups` attribute gives an upper bound of the number of
-    groups uncompleted when the wait can unblock the thread. For example,  if
-    16 async groups are pushe and `$numGroups` is set to 12, then the thread
-    will unblock when 12 groups or fewer are in flight (4 groups have
-    completed).
-
-    Example:
-
-    ```mlir
-    nvgpu.device_async_wait %0
-    ```
-  }];
-  let arguments = (ins NVGPU_DeviceAsyncToken:$asyncDependencies,
-                       OptionalAttr<I32Attr>:$numGroups);
-  let assemblyFormat = [{
-    $asyncDependencies attr-dict
-  }];
-}
-
-def NVGPU_MBarrierCreateOp : NVGPU_Op<"mbarrier.create", []> {
-  let summary = "Creates a `nvgpu.mbarrier` object.";
-  let description = [{
-    The Op generates one or more `mbarrier` object, which is a barrier created in 
-    shared memory and supports various synchronization behaviors for threads.
-
-    The `mbarrier` object has the following type and alignment requirements:
-      Type: .b64, Alignment: 8, Memory space: .shared
-    
-    Example:
-    ```mlir
-      %barrier = nvgpu.mbarrier.create -> !nvgpu.mbarrier.barrier<memorySpace = #gpu.address_space<workgroup>>
-    ```
-    }];
-  let arguments = (ins);
-  let results = (outs NVGPU_MBarrierGroup:$barriers);
-  let assemblyFormat = [{
-     attr-dict `->` type($barriers)
-  }];
-}
-
-def NVGPU_MBarrierInitOp : NVGPU_Op<"mbarrier.init", []> {
-  let summary = "Initialize the `nvgpu.mbarrier`.";
-  let description = [{
-    The Op initializes the `mbarrier` object with the given number of threads.
-
-    Example:
-    ```mlir
-      %num_threads = gpu.block_dim x
-      %barrier = nvgpu.mbarrier.create -> !nvgpu.mbarrier.barrier<memorySpace = #gpu.address_space<workgroup>>
-      nvgpu.mbarrier.init %barrier, %num_threads : !nvgpu.mbarrier.barrier<memorySpace = #gpu.address_space<workgroup>>
-    ```
-  }];
-  let arguments = (ins NVGPU_MBarrierGroup:$barriers, Index:$count, Index:$mbarId, Optional<I1>:$predicate);
-  let assemblyFormat = "$barriers `[` $mbarId `]` `,` $count (`,` `predicate` `=` $predicate^)? attr-dict `:` type($barriers)";
-}
-
-def NVGPU_MBarrierTestWaitOp : NVGPU_Op<"mbarrier.test.wait", []> {
-  let summary = "Checks if the `nvgpu.mbarrier` has completed its current phase.";
-  let description = [{
-    Checks whether the mbarrier object has completed the phase. It is is a 
-    non-blocking instruction which tests for the completion of the phase.
-
-    Example:
-    ```mlir
-      %isComplete = nvgpu.mbarrier.test.wait %barrier, %token : !nvgpu.mbarrier.barrier<memorySpace = #gpu.address_space<workgroup>>, !nvgpu.mbarrier.token
-    ```
-  }];
-  let arguments = (ins NVGPU_MBarrierGroup:$barriers, NVGPU_MBarrierToken:$token, Index:$mbarId);
-  let results = (outs I1:$waitComplete);
-  let assemblyFormat = "$barriers `[` $mbarId `]` `,` $token attr-dict `:` type($barriers) `,` type($token)";
-}
-
-def NVGPU_MBarrierArriveOp : NVGPU_Op<"mbarrier.arrive", []> {
-  let summary = "Performs arrive operation on the `nvgpu.mbarrier.arrive`.";
-  let description = [{
-    The Op performs arrive-on operation on the `mbarrier` object and returns a 
-    `nvgpu.mbarrier.token`.
-
-    For more information, see
-    https://docs.nvidia.com/cuda/parallel-thread-execution/#arrive-on-operation-on-mbarrier-object
-
-    Example:
-    ```mlir
-      %token = nvgpu.mbarrier.arrive %barrier : !nvgpu.mbarrier.barrier<memorySpace = #gpu.address_space<workgroup>> -> !nvgpu.mbarrier.token
-    ```
-  }];
-  let arguments = (ins NVGPU_MBarrierGroup:$barriers, Index:$mbarId);
-  let results = (outs NVGPU_MBarrierToken:$token);
-let assemblyFormat = "$barriers `[` $mbarId `]` attr-dict `:` type($barriers) `->` type($token)";
-}
-
-def NVGPU_MBarrierArriveNoCompleteOp : NVGPU_Op<"mbarrier.arrive.nocomplete", []> {
-  let summary = "Performs arrive operation on the `nvgpu.mbarrier.arrive.nocomplete` as non-blocking.";
-  let description = [{
-    The Op performs arrive-on operation on the `mbarrier` object and returns a 
-    `nvgpu.mbarrier.token`.
-
-    The Op does not cause the `nvgpu.mbarrier` to complete its current phase.
-
-    Example:
-    ```mlir
-      %token = nvgpu.mbarrier.arrive.noComplete %barrier, %count : !nvgpu.mbarrier.barrier<memorySpace = #gpu.address_space<workgroup>> -> !nvgpu.mbarrier.token
-    ```
-  }];
-  let arguments = (ins NVGPU_MBarrierGroup:$barriers, Index:$mbarId,
-                       Index:$count);
-  let results = (outs NVGPU_MBarrierToken:$token);
-  let assemblyFormat = "$barriers `[` $mbarId `]` `,` $count attr-dict `:` type($barriers) `->` type($token)";
-}
-
-def NVGPU_MBarrierArriveExpectTxOp : NVGPU_Op<"mbarrier.arrive.expect_tx", []> {
-  let summary = "Performs expect_tx operation on the `nvgpu.mbarrier.arrive`";
-  let description = [{
-    A thread executing the Op performs an expect-tx operation on the mbarrier 
-    object at the location specified by the address operand $barrier. The 
-    expect-tx operation, with an $txcount argument, increases the tx-count of 
-    an mbarrier object by the value specified by $txcount. This makes the 
-    current phase of the mbarrier object to expect and track the completion of 
-    additional asynchronous transactions.
-    
-    The `$txCount` specifies the number of element to the expect-tx operation.
-
-    Example:
-    ```mlir
-      nvgpu.mbarrier.arrive.expect_tx %barrier, %ic0 : !nvgpu.mbarrier.barrier<memorySpace = #gpu.address_space<workgroup>>
-    ```
-  }];
-  let arguments = (ins NVGPU_MBarrierGroup:$barriers, Index:$txcount, Index:$mbarId, Optional<I1>:$predicate);
-  let assemblyFormat = "$barriers `[` $mbarId `]` `,` $txcount  (`,` `predicate` `=` $predicate^)? attr-dict `:` type($barriers)";
-}
-
-def NVGPU_MBarrierTryWaitParityOp : NVGPU_Op<"mbarrier.try_wait.parity", []> {
-  let summary = "Waits for the `nvgpu.mbarrier` to complete its current phase.";
-  let description = [{
-    Checks whether the mbarrier object has completed the phase. It is is a 
-    potentially blocking instruction which tests for the completion of the 
-    phase. Suspended thread resumes execution when the specified phase completes 
-    OR before the phase completes following a system-dependent time limit. 
-
-    The `$phaseParity` specifies either even phase (0) or odd phase (1) to 
-    wait.
-
-    Example:
-    ```mlir
-      nvgpu.mbarrier.try_wait.parity %barrier, %phaseParity, %ticks : !nvgpu.mbarrier.barrier<memorySpace = #gpu.address_space<workgroup>>
-    ```
-  }];
-  let arguments = (ins NVGPU_MBarrierGroup:$barriers, I1:$phaseParity, Index:$ticks, Index:$mbarId);
-  let assemblyFormat = "$barriers `[` $mbarId `]` `,` $phaseParity `,` $ticks attr-dict `:` type($barriers)";  
-}
-
-def NVGPU_TmaPrefetchOp : NVGPU_Op<"tma.prefetch.descriptor", []> {
-  let summary = "Prefetch given `nvgpu.tensormap.descriptor` ";
-  let description = [{
-    The Op brings the cache line containing the given `$tmaDescriptor` for 
-    subsequent use by the `tma.async.load` instruction.
-  }];
-  let arguments = (ins NVGPU_TensorMapDescriptor:$tensorMapDescriptor, Optional<I1>:$predicate);
-  let assemblyFormat = [{
-    $tensorMapDescriptor (`,` `predicate` `=` $predicate^)? attr-dict `:` type($tensorMapDescriptor)
-  }];
-}
-
-def NVGPU_TmaAsyncLoadOp : NVGPU_Op<"tma.async.load", [AttrSizedOperandSegments]> {
-  let summary = "TMA asynchronous load";
-  let description = [{
-    The Op loads a tile memory region from global memory to shared memory by 
-    Tensor Memory Access (TMA).
-    
-    `$tensorMapDescriptor` is tensor map descriptor which has information about
-    tile shape. The descriptor is created by `nvgpu.tma.create.descriptor`
-
-    The Op uses `$barrier` mbarrier based completion mechanism. 
-  }];  
-  let arguments = (ins  Arg<AnyMemRef, "", [MemWriteAt<0, FullEffect>]>:$dst,
-                        NVGPU_MBarrierGroup:$barriers,
-                        NVGPU_TensorMapDescriptor:$tensorMapDescriptor,
-                        Variadic<Index>:$coordinates, 
-                        Index:$mbarId,
-                        Optional<I16>:$multicastMask,
-                        Optional<I1>:$predicate);
-  let assemblyFormat = [{
-    $tensorMapDescriptor `[` $coordinates `]` `,` $barriers `[` $mbarId `]` 
-      `to` $dst
-      (`multicast_mask` `=` $multicastMask^ )?
-      (`,` `predicate` `=` $predicate^)?
-      attr-dict `:` type($tensorMapDescriptor) `,` type($barriers) 
-      `->` type($dst)
-  }];
-  let hasVerifier = 1;
-
-}
-
-def NVGPU_TmaAsyncStoreOp : NVGPU_Op<"tma.async.store", [AttrSizedOperandSegments]> {
-  let summary = "TMA asynchronous store";
-  let description = [{
-    The Op store a tile memory region from global memory to shared memory by 
-    Tensor Memory Access (TMA).
-    
-    `$tensorMapDescriptor` is tensor map descriptor which has information about
-    tile shape. The descriptor is created by `nvgpu.tma.create.descriptor`
-  }];  
-  let arguments = (ins  Arg<AnyMemRef, "", [MemReadAt<0, FullEffect>]>:$src,
-                        Arg<NVGPU_TensorMapDescriptor, "", [MemWriteAt<0, FullEffect>]>:$tensorMapDescriptor,
-                        Variadic<Index>:$coordinates, 
-                        Optional<I1>:$predicate);
-  let assemblyFormat = [{
-      $src `to` $tensorMapDescriptor `[` $coordinates `]`
-      (`,` `predicate` `=` $predicate^)?
-      attr-dict `:` type($src)
-      `->` type($tensorMapDescriptor)
-  }];
-  let hasVerifier = 1;
-}
-
-def NVGPU_TmaCreateDescriptorOp : NVGPU_Op<"tma.create.descriptor", []> {
-  let summary = "TMA create descriptor";
-  let description = [{
-    The Op creates a tensor map descriptor object representing tiled memory 
-    region. To do that it calls CUDA Driver's `cuTensorMapEncodeTiled`. The 
-    descriptor is used by Tensor Memory Access (TMA).
-
-    The `tensor` is the source tensor to be tiled. 
-
-    The `boxDimensions` is the size of the tiled memory region in each dimension.
-
-    For more information see below:
-    https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html
-  }];
-
-  let arguments = (ins AnyUnrankedMemRef:$tensor,
-                       Variadic<Index>:$boxDimensions);
-  let results = (outs NVGPU_TensorMapDescriptor:$tensorMap);
-  let assemblyFormat = [{
-         $tensor `box` `[` $boxDimensions `]` attr-dict `:` type($tensor) `->` type($tensorMap)
-  }];
-  let hasVerifier = 1;
-}
-
-def NVGPU_WarpgroupGenerateDescriptorOp : NVGPU_Op<"warpgroup.generate.descriptor", []> {
-  let summary = "Generate a warpgroup matrix descriptor";
-  let description = [{
-  This Op builds a `nvgpu.warpgroup.descriptor` that is used by 
-  `nvgpu.warpgroup.mma` to perform warpgroup-level matrix multiply and 
-  accumulate.
-
-  The descriptor specifies the properties of the matrix in shared memory that 
-  is a multiplicand in the matrix multiply and accumulate operation. 
-  }];  
-  let results = (outs NVGPU_WarpgroupMatrixDescriptor:$descriptor);
-  let arguments = (ins Arg<AnyMemRef, "", [MemRead]>:$tensor, 
-                       NVGPU_TensorMapDescriptor:$tensorMap);
-  let assemblyFormat = [{$tensor `,` $tensorMap attr-dict `:` type($tensor) `,` type($tensorMap) `->` type($descriptor)}];
-  let hasVerifier = 1;
-}
-
-def NVGPU_WarpgroupMmaOp : NVGPU_Op<"warpgroup.mma"> {
-  let description = [{
-    The `nvgpu.warpgroup.mma` op performs the warpgroup-level (4 warps) 
-    matrix-multiply-and-accumulate (mma) operation that results in 
-    `nvvm.wgmma.mma_async`. 
-    
-    The operands are `descriptorA` and `descriptorB` that are wgmma matrix 
-    descriptors that shows the properties of the matrix in shared memory. The 
-    results are thread-level ownership to the warpgroup-level mma operation 
-    shape. The shape is deduced from the descriptor types and output vector.
-
-    The Op encapsulates multiple `nvvm.wgmma.mma_async` operations to complete 
-    the given shape. As `nvvm.wgmma.async` Op, or its corresponding PTX 
-    instruction, is asynchronous, this Op groups the `nvvm.wgmma.async` and 
-    surrounds them between `wgmma.fence.aligned` and 
-    `wgmma.commit.group.sync.aligned`, `wgmma.wait.group.sync.aligned` Ops.
-
-    Example:
-    ```mlir
-      %r1,%r2 = nvgpu.warpgroup.mma %descA, %descB, %acc1, %acc2: 
-                 !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>, 
-                 !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>, 
-                 !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>>,
-                 !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>>
-                 -> 
-                 !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>>,
-                 !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>>
-    ```
-  }];
-
-  let arguments = (ins NVGPU_WarpgroupMatrixDescriptor:$descriptorA, 
-                       NVGPU_WarpgroupMatrixDescriptor:$descriptorB,                                               
-                       DefaultValuedOptionalAttr<I64Attr, "1">:$waitGroup,
-                       OptionalAttr<UnitAttr>:$transposeA,
-                       OptionalAttr<UnitAttr>:$transposeB,
-                       NVGPU_WarpgroupAccumulator:$matrixC);
-  let results = (outs NVGPU_WarpgroupAccumulator:$matrixD);
-  let assemblyFormat = [{    
-    $descriptorA`,` $descriptorB`,` $matrixC attr-dict
-    `:` type($descriptorA) `,` type($descriptorB) `,` type($matrixC) `->` type($matrixD)
-  }];
-  let hasVerifier = 1;
-}
-
-def NVGPU_WarpgroupMmaStoreOp : NVGPU_Op<"warpgroup.mma.store"> {
-  let description = [{
-    The `nvgpu.warpgroup.mma.store` op performs the store of fragmented result 
-    in $matrixD to given memref. 
-
-    [See the details of register fragment layout for accumulator matrix D]
-    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#wgmma-64n16-d) 
-
-    Note that, the op must be run with warp group.
-  }];
-
-  let arguments = (ins NVGPU_WarpgroupAccumulator:$matrixD,
-                       Arg<AnyMemRef, "", [MemWrite]>:$dstMemref);
-  
-  let assemblyFormat = [{
-    $matrixD `,` $dstMemref attr-dict `:` type($matrixD) `to` type($dstMemref)
-  }];
-  let hasVerifier = 1;
-}
-
-def NVGPU_WarpgroupMmaInitAccumulatorOp : NVGPU_Op<"warpgroup.mma.init.accumulator"> {  
-  let summary = "Initializes the accumulator matrix";
-
-  let description = [{
-    This Op generates and initializes the accumulator matrix for 
-    `nvgpu.warpgroup.mma` op to perform matrix-multiply-and-accumulate.
-  }];
-  let results = (outs NVGPU_WarpgroupAccumulator:$matrixC);
-  let assemblyFormat = "attr-dict `->` type($matrixC)";
-  let hasVerifier = 1;
-}
-
-def NVGPU_RcpOp : NVGPU_Op<"rcp", [Pure,
-                                   SameOperandsAndResultType]> {
-  let summary = "The reciprocal calculation for vector types";
-  let description = [{
-    Reciprocal calculation for `vector` types using `nvvm.rcp` OPs.
-
-    Currently, only the `approx` rounding mode and `ftz` are supported, and only for the `f32` type.
-
-    The input and output must be of the same vector type and shape.
-  }];
-  let arguments = (ins VectorOfNonZeroRankOf<[F32]>:$in,
-                       DefaultValuedAttr<RcpRoundingModeAttr, "RcpRoundingMode::APPROX">:$rounding,
-                       UnitAttr:$ftz);
-  let results = (outs VectorOfNonZeroRankOf<[F32]>:$out);
-  let assemblyFormat = [{
-    $in `{` `rounding` `=` $rounding (`,` `ftz` $ftz^)? `}` 
-    attr-dict `:` type($out)
-  }];
-  let hasVerifier = 1;
-}
-#endif // NVGPU
+#endif // MLIR_DIALECT_NVGPU_IR_NVGPU_TD
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUDialect.h b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUDialect.h
index db4c63b3390eb..61a57fb60bda4 100644
--- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUDialect.h
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUDialect.h
@@ -53,11 +53,11 @@ constexpr unsigned kMaxTMALastdimByte = 128;
 #include "mlir/Dialect/NVGPU/IR/NVGPUAttrDefs.h.inc"
 
 #define GET_TYPEDEF_CLASSES
-#include "mlir/Dialect/NVGPU/IR/NVGPUTypes.h.inc"
+#include "mlir/Dialect/NVGPU/IR/NVGPUTypeDefs.h.inc"
 
 #include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h.inc"
 
 #define GET_OP_CLASSES
-#include "mlir/Dialect/NVGPU/IR/NVGPU.h.inc"
+#include "mlir/Dialect/NVGPU/IR/NVGPUOps.h.inc"
 
 #endif // MLIR_DIALECT_NVGPU_NVGPUDIALECT_H_
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
new file mode 100644
index 0000000000000..eb0fb90d271ed
--- /dev/null
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUOps.td
@@ -0,0 +1,638 @@
+//===-- NVGPUOps.td - NVGPU dialect operation definitions *- tablegen -*---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the basic operations for the NVGPU dialect.
+//
+// This NVGPU provides a bridge between the target agnostic GPU and Vector
+// dialects and lower level NVVM dialect. This allow representing PTX specific
+// operations while using MLIR high level concepts like memref and 2-D vector.
+//
+// Ops semantic are going to be based on vendor specific PTX defintion:
+// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_NVGPU_IR_NVGPUOPS_TD
+#define MLIR_DIALECT_NVGPU_IR_NVGPUOPS_TD
+
+include "mlir/Dialect/NVGPU/IR/NVGPU.td"
+include "mlir/Dialect/NVGPU/IR/NVGPUTypes.td"
+
+//===----------------------------------------------------------------------===//
+// NVGPU Op Definitions
+//===----------------------------------------------------------------------===//
+
+class NVGPU_Op<string mnemonic, list<Trait> traits = []> :
+  Op<NVGPU_Dialect, mnemonic, traits> {}
+
+def NVGPU_LdMatrixOp : NVGPU_Op<"ldmatrix", [
+                                MemoryEffects<[MemRead]>,
+                                PredOpTrait<"srcMemref and res have same element type",
+                                            TCresVTEtIsSameAsOp<0, 0>>]> {
+  let description = [{
+    The `nvgpu.ldmatrix` op represents loading a matrix fragment from
+    memory to registers. The source and result type must be compatible
+    with lowering to the `nvvm.ldmatrix` instruction. This op represents
+    the distributed version of a `vector.transfer_read` as an intermediate
+    step between lowering from `vector.transfer_read` to `nvvm.ldmatrix`.
+
+    This operation is meant to follow the semantic of described here:
+    https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#warp-level-matrix-instructions-ldmatrix
+
+    Example:
+    ```mlir
+    %0 = nvgpu.ldmatrix %sm[%c0, %c0] {numTiles = 4 : i32, transpose = false} :
+      memref<?x?xf16, 3> -> vector<4x2xf16>
+    ```
+  }];
+
+  let arguments = (ins Arg<AnyMemRef, "", [MemReadAt<0, FullEffect>]>:$srcMemref,
+                           Variadic<Index>:$indices, BoolAttr:$transpose,
+                           I32Attr:$numTiles);
+  let results = (outs AnyVectorOfNonZeroRank:$res);
+  let assemblyFormat = [{
+    $srcMemref`[` $indices `]` attr-dict `:` type($srcMemref) `->` type($res)
+  }];
+
+  let hasVerifier = 1;
+}
+
+class NVGPU_MmaSyncOp<string mnemonic> :
+        NVGPU_Op<mnemonic,  [Pure,
+                             PredOpTrait<"matrixA and matrixB have same element type",
+                                         TCopVTEtIsSameAs<0, 1>>]> {
+  code extraBaseClassDeclaration = [{
+    std::array<int64_t, 3> getMmaShapeAsArray() {
+      ArrayAttr mmaShape = this->getMmaShape();
+      assert(mmaShape.size() == 3 && "mmaShape should be three integers");
+      return {::llvm::cast<IntegerAttr>(mmaShape[0]).getInt(),
+              ::llvm::cast<IntegerAttr>(mmaShape[1]).getInt(),
+              ::llvm::cast<IntegerAttr>(mmaShape[2]).getInt()};
+    }
+  }];
+
+  let hasVerifier = 1;
+}
+
+def NVGPU_MmaSyncOp : NVGPU_MmaSyncOp<"mma.sync"> {
+  let description = [{
+    The `nvgpu.mma.sync` op represents the warp-level matrix-multiply-and-
+    accumulate (mma) operation that is compatible with `nvvm.mma.sync`.
+    The operands and results vector sizes are thread-level onwership to
+    the warp-level mma operation shape. `mmaShape` attribute holds the
+    warp-level matrix-multiply shape.
+
+    The `nvgpu.mma.sync` op serves as an intermediate point between lowering from
+    `vector.contract` to `nvvm.mma.sync`.
+
+    This operation is meant to follow the semantic of described here:
+      https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#warp-level-matrix-instructions-mma
+
+    Example:
+
+    ```mlir
+    %res = nvgpu.mma.sync (%matrixA, %matrixB, %matrixC) {mmaShape = [16, 8, 16]} :
+        (vector<4x2xf16>, vector<2x2xf16>, vector<2x2xf32>) -> vector<2x2xf32>
+    ```
+  }];
+  let arguments = (ins AnyVectorOfNonZeroRank:$matrixA,
+                       AnyVectorOfNonZeroRank:$matrixB,
+                       AnyVectorOfNonZeroRank:$matrixC,
+                       I64ArrayAttr:$mmaShape,
+                       OptionalAttr<UnitAttr>:$tf32Enabled);
+
+  let results = (outs AnyVectorOfNonZeroRank:$res);
+
+  let builders = [
+    OpBuilder<(ins "Value":$matrixA,
+                   "Value":$matrixB,
+                   "Value":$matrixC,
+                   "ArrayAttr":$mmaShape)>,
+    OpBuilder<(ins "Value":$matrixA,
+                   "Value":$matrixB,
+                   "Value":$matrixC,
+                   "ArrayRef<int64_t>":$mmaShape,
+                   CArg<"bool", "false">:$tf32Enabled)>
+  ];
+
+  let assemblyFormat = [{
+    `(` $matrixA`,` $matrixB`,` $matrixC `)` attr-dict
+    `:` `(` type($matrixA) `,` type($matrixB) `,` type($matrixC) `)` `->` type($res)
+  }];
+
+  let extraClassDeclaration = extraBaseClassDeclaration;
+}
+
+def NVGPU_MmaSparseSyncMetadataType : FixedVectorOfLengthAndType<[2], [I16]>,
+                        BuildableType<"::mlir::VectorType::get("
+                          "{2},$_builder.getI16Type())">;
+
+def NVGPU_MmaSparseSyncOp : NVGPU_MmaSyncOp<"mma.sp.sync"> {
+  let description = [{
+  The `nvgu.mma.sp.sync` operation performs a warp-distributed MMA operation
+  where operand A is "structured sparse". In this case, the `matrixA` operand
+  represents the (warp-distributed) non-zero values of operand A, and the
+  `sparse_metadata` operand provides the indices.
+
+  The full description of the sparsity storage format and distribution scheme is
+  described in the PTX docs. This operation is meant to follow the semantic
+  described in the PTX documentation here:
+  https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#warp-level-matrix-instructions-for-sparse-mma
+
+  The way the indices are distributed among the threads in a warp is controlled
+  by the optional `sparsity_selector` operand, which is `0` by default. For
+  more information, please consult the PTX documentation linked above.
+
+  Example (targetingthe f16 16x8x32 `mma.sp` PTX instruction):
+
+  ```mlir
+  nvgpu.mma.sp.sync (%a, %b, %c) metadata (%meta) {mmaShape = [16, 8, 32]} :
+    (vector<4x2xf16>, vector<2x2xf16>, vector<2x2xf16>) -> vector<2x2xf16>
+  ```
+  }];
+
+  let arguments = (ins AnyVectorOfNonZeroRank:$matrixA,
+                       AnyVectorOfNonZeroRank:$matrixB,
+                       AnyVectorOfNonZeroRank:$matrixC,
+                       NVGPU_MmaSparseSyncMetadataType:$sparseMetadata,
+                       I64ArrayAttr:$mmaShape,
+                       DefaultValuedAttr<I32Attr, "0">:$sparsitySelector,
+                       OptionalAttr<UnitAttr>:$tf32Enabled
+                       );
+
+  let results = (outs AnyVectorOfNonZeroRank:$res);
+
+  let builders = [
+    OpBuilder<(ins "Value":$matrixA,
+                   "Value":$matrixB,
+                   "Value":$matrixC,
+                   "Value":$sparseMetadata,
+                   "ArrayRef<int64_t>":$mmaShape)>
+  ];
+
+  let assemblyFormat = [{
+    `(` $matrixA`,` $matrixB`,` $matrixC `)` `metadata` `(` $sparseMetadata `)` attr-dict
+    `:` `(` type($matrixA) `,` type($matrixB) `,` type($matrixC) `)` `->` type($res)
+  }];
+
+  let extraClassDeclaration = extraBaseClassDeclaration;
+}
+
+def NVGPU_DeviceAsyncCopyOp : NVGPU_Op<"device_async_copy", [
+                                       AttrSizedOperandSegments]> {
+  let summary = "device-side asynchronous copy";
+  let description = [{
+    The `nvgpu.device_async_copy` op initiates an asynchronous copy operation of
+    elements from source (global memory) to the destination (shared memory)
+    without blocking the thread. The async copy is added to a group.
+
+    This op is meant to be used with `nvgpu.device_async_create_group` and
+    `nvgpu.device_async_wait` to synchronize copies as explained in those ops
+    descriptions.
+
+    `bypassL1` attribute is hint to the hardware to bypass the L1 cache during
+    async copy, this hint may be ignored by the hardware.
+
+    `dstElements` attribute is the total number of elements written to
+    destination (shared memory).
+
+    `srcElements` argument is the total number of elements read from
+    source (global memory).
+
+    `srcElements` is an optional argument and when present the op only reads
+    `srcElements` number of elements from the source (global memory) and zero fills
+    the rest of the elements in the destination (shared memory).
+
+    In order to do a copy and wait for the result we need the following
+    combination:
+    ```
+    // copy 1.
+    %cp1 = nvgpu.device_async_copy %A[%c0], %B[%c0], 4 :memref<16xf32> to memref<16xf32, 3>
+    // copy 2.
+    %cp2 = nvgpu.device_async_copy %C[%c0], %D[%c0], 4 : memref<16xf32> to memref<16xf32, 3>
+    // group 1 contains copy 1 and copy 2.
+    %token1 = nvgpu.device_async_create_group %cp1, %cp2
+    // copy 3.
+    %cp3 = nvgpu.device_async_copy %E[%c0], %F[%c0], 4 : memref<16xf32> to memref<16xf32, 3>
+    // group 2 contains copy 3.
+    %token2 = nvgpu.device_async_create_group %cp3
+    // after the wait copy 1 and copy 2 are complete.
+    nvgpu.device_async_wait %token1
+    // after the wait copy 3 is complete.
+    nvgpu.device_async_wait %token2
+    ```
+
+    Example:
+
+    ```mlir
+    %0 = nvgpu.device_async_copy %src[%c0, %c0], %dst[%c0, %c0, %c0], 4 :
+      memref<4x5xf32> to memref<2x7x5xf32, 3>
+    ```
+  }];
+  let results = (outs NVGPU_DeviceAsyncToken:$asyncToken);
+  let arguments = (ins Arg<AnyMemRef, "", [MemWriteAt<0, FullEffect>]>:$dst,
+                       Variadic<Index>:$dstIndices,
+                       Arg<AnyMemRef, "", [MemReadAt<0, FullEffect>]>:$src,
+                       Variadic<Index>:$srcIndices,
+                       IndexAttr:$dstElements,
+                       Optional<Index>:$srcElements,
+                       OptionalAttr<UnitAttr>:$bypassL1);
+  let assemblyFormat = [{
+    $src `[` $srcIndices `]` `,` $dst `[` $dstIndices `]` `,` $dstElements (`,` $srcElements^)?
+      attr-dict `:` type($src) `to` type($dst)
+  }];
+  let hasVerifier = 1;
+}
+
+def NVGPU_DeviceAsyncCreateGroupOp : NVGPU_Op<"device_async_create_group", []> {
+  let summary = "device side asynchronous create group operation";
+  let description = [{
+    The `nvgpu.device_async_create_group` op creates a group of memory accesses
+    containing all the pending `device_async_copy` operations associated with
+    argument tokens. Each token can only be part of one group.
+
+    It returns a token that can be use to wait until the group fully completes.
+
+    This is meant to be used with `nvgpu.device_async_wait` to synchronize copies
+    as explained in those ops descriptions.
+
+    Groups are executed in the order they are created.
+
+    Example:
+
+    ```mlir
+    %0 = nvgpu.device_async_create_group
+  ```
+  }];
+  let results = (outs NVGPU_DeviceAsyncToken:$asyncToken);
+  let arguments = (ins Variadic<NVGPU_DeviceAsyncToken>:$inputTokens);
+  let assemblyFormat = [{
+    $inputTokens attr-dict
+  }];
+}
+
+def NVGPU_DeviceAsyncWaitOp : NVGPU_Op<"device_async_wait", []> {
+  let summary = "Wait for async gpu ops to complete.";
+  let description = [{
+    The `nvgpu.device_async_wait` op will block the execution thread until the group
+    associated with the source token is fully completed.
+
+    The optional `$numGroups` attribute gives an upper bound of the number of
+    groups uncompleted when the wait can unblock the thread. For example,  if
+    16 async groups are pushe and `$numGroups` is set to 12, then the thread
+    will unblock when 12 groups or fewer are in flight (4 groups have
+    completed).
+
+    Example:
+
+    ```mlir
+    nvgpu.device_async_wait %0
+    ```
+  }];
+  let arguments = (ins NVGPU_DeviceAsyncToken:$asyncDependencies,
+                       OptionalAttr<I32Attr>:$numGroups);
+  let assemblyFormat = [{
+    $asyncDependencies attr-dict
+  }];
+}
+
+def NVGPU_MBarrierCreateOp : NVGPU_Op<"mbarrier.create", []> {
+  let summary = "Creates a `nvgpu.mbarrier` object.";
+  let description = [{
+    The Op generates one or more `mbarrier` object, which is a barrier created in 
+    shared memory and supports various synchronization behaviors for threads.
+
+    The `mbarrier` object has the following type and alignment requirements:
+      Type: .b64, Alignment: 8, Memory space: .shared
+    
+    Example:
+    ```mlir
+      %barrier = nvgpu.mbarrier.create -> !nvgpu.mbarrier.barrier<memorySpace = #gpu.address_space<workgroup>>
+    ```
+    }];
+  let arguments = (ins);
+  let results = (outs NVGPU_MBarrierGroup:$barriers);
+  let assemblyFormat = [{
+     attr-dict `->` type($barriers)
+  }];
+}
+
+def NVGPU_MBarrierInitOp : NVGPU_Op<"mbarrier.init", []> {
+  let summary = "Initialize the `nvgpu.mbarrier`.";
+  let description = [{
+    The Op initializes the `mbarrier` object with the given number of threads.
+
+    Example:
+    ```mlir
+      %num_threads = gpu.block_dim x
+      %barrier = nvgpu.mbarrier.create -> !nvgpu.mbarrier.barrier<memorySpace = #gpu.address_space<workgroup>>
+      nvgpu.mbarrier.init %barrier, %num_threads : !nvgpu.mbarrier.barrier<memorySpace = #gpu.address_space<workgroup>>
+    ```
+  }];
+  let arguments = (ins NVGPU_MBarrierGroup:$barriers, Index:$count, Index:$mbarId, Optional<I1>:$predicate);
+  let assemblyFormat = "$barriers `[` $mbarId `]` `,` $count (`,` `predicate` `=` $predicate^)? attr-dict `:` type($barriers)";
+}
+
+def NVGPU_MBarrierTestWaitOp : NVGPU_Op<"mbarrier.test.wait", []> {
+  let summary = "Checks if the `nvgpu.mbarrier` has completed its current phase.";
+  let description = [{
+    Checks whether the mbarrier object has completed the phase. It is is a 
+    non-blocking instruction which tests for the completion of the phase.
+
+    Example:
+    ```mlir
+      %isComplete = nvgpu.mbarrier.test.wait %barrier, %token : !nvgpu.mbarrier.barrier<memorySpace = #gpu.address_space<workgroup>>, !nvgpu.mbarrier.token
+    ```
+  }];
+  let arguments = (ins NVGPU_MBarrierGroup:$barriers, NVGPU_MBarrierToken:$token, Index:$mbarId);
+  let results = (outs I1:$waitComplete);
+  let assemblyFormat = "$barriers `[` $mbarId `]` `,` $token attr-dict `:` type($barriers) `,` type($token)";
+}
+
+def NVGPU_MBarrierArriveOp : NVGPU_Op<"mbarrier.arrive", []> {
+  let summary = "Performs arrive operation on the `nvgpu.mbarrier.arrive`.";
+  let description = [{
+    The Op performs arrive-on operation on the `mbarrier` object and returns a 
+    `nvgpu.mbarrier.token`.
+
+    For more information, see
+    https://docs.nvidia.com/cuda/parallel-thread-execution/#arrive-on-operation-on-mbarrier-object
+
+    Example:
+    ```mlir
+      %token = nvgpu.mbarrier.arrive %barrier : !nvgpu.mbarrier.barrier<memorySpace = #gpu.address_space<workgroup>> -> !nvgpu.mbarrier.token
+    ```
+  }];
+  let arguments = (ins NVGPU_MBarrierGroup:$barriers, Index:$mbarId);
+  let results = (outs NVGPU_MBarrierToken:$token);
+let assemblyFormat = "$barriers `[` $mbarId `]` attr-dict `:` type($barriers) `->` type($token)";
+}
+
+def NVGPU_MBarrierArriveNoCompleteOp : NVGPU_Op<"mbarrier.arrive.nocomplete", []> {
+  let summary = "Performs arrive operation on the `nvgpu.mbarrier.arrive.nocomplete` as non-blocking.";
+  let description = [{
+    The Op performs arrive-on operation on the `mbarrier` object and returns a 
+    `nvgpu.mbarrier.token`.
+
+    The Op does not cause the `nvgpu.mbarrier` to complete its current phase.
+
+    Example:
+    ```mlir
+      %token = nvgpu.mbarrier.arrive.noComplete %barrier, %count : !nvgpu.mbarrier.barrier<memorySpace = #gpu.address_space<workgroup>> -> !nvgpu.mbarrier.token
+    ```
+  }];
+  let arguments = (ins NVGPU_MBarrierGroup:$barriers, Index:$mbarId,
+                       Index:$count);
+  let results = (outs NVGPU_MBarrierToken:$token);
+  let assemblyFormat = "$barriers `[` $mbarId `]` `,` $count attr-dict `:` type($barriers) `->` type($token)";
+}
+
+def NVGPU_MBarrierArriveExpectTxOp : NVGPU_Op<"mbarrier.arrive.expect_tx", []> {
+  let summary = "Performs expect_tx operation on the `nvgpu.mbarrier.arrive`";
+  let description = [{
+    A thread executing the Op performs an expect-tx operation on the mbarrier 
+    object at the location specified by the address operand $barrier. The 
+    expect-tx operation, with an $txcount argument, increases the tx-count of 
+    an mbarrier object by the value specified by $txcount. This makes the 
+    current phase of the mbarrier object to expect and track the completion of 
+    additional asynchronous transactions.
+    
+    The `$txCount` specifies the number of element to the expect-tx operation.
+
+    Example:
+    ```mlir
+      nvgpu.mbarrier.arrive.expect_tx %barrier, %ic0 : !nvgpu.mbarrier.barrier<memorySpace = #gpu.address_space<workgroup>>
+    ```
+  }];
+  let arguments = (ins NVGPU_MBarrierGroup:$barriers, Index:$txcount, Index:$mbarId, Optional<I1>:$predicate);
+  let assemblyFormat = "$barriers `[` $mbarId `]` `,` $txcount  (`,` `predicate` `=` $predicate^)? attr-dict `:` type($barriers)";
+}
+
+def NVGPU_MBarrierTryWaitParityOp : NVGPU_Op<"mbarrier.try_wait.parity", []> {
+  let summary = "Waits for the `nvgpu.mbarrier` to complete its current phase.";
+  let description = [{
+    Checks whether the mbarrier object has completed the phase. It is is a 
+    potentially blocking instruction which tests for the completion of the 
+    phase. Suspended thread resumes execution when the specified phase completes 
+    OR before the phase completes following a system-dependent time limit. 
+
+    The `$phaseParity` specifies either even phase (0) or odd phase (1) to 
+    wait.
+
+    Example:
+    ```mlir
+      nvgpu.mbarrier.try_wait.parity %barrier, %phaseParity, %ticks : !nvgpu.mbarrier.barrier<memorySpace = #gpu.address_space<workgroup>>
+    ```
+  }];
+  let arguments = (ins NVGPU_MBarrierGroup:$barriers, I1:$phaseParity, Index:$ticks, Index:$mbarId);
+  let assemblyFormat = "$barriers `[` $mbarId `]` `,` $phaseParity `,` $ticks attr-dict `:` type($barriers)";  
+}
+
+def NVGPU_TmaPrefetchOp : NVGPU_Op<"tma.prefetch.descriptor", []> {
+  let summary = "Prefetch given `nvgpu.tensormap.descriptor` ";
+  let description = [{
+    The Op brings the cache line containing the given `$tmaDescriptor` for 
+    subsequent use by the `tma.async.load` instruction.
+  }];
+  let arguments = (ins NVGPU_TensorMapDescriptor:$tensorMapDescriptor, Optional<I1>:$predicate);
+  let assemblyFormat = [{
+    $tensorMapDescriptor (`,` `predicate` `=` $predicate^)? attr-dict `:` type($tensorMapDescriptor)
+  }];
+}
+
+def NVGPU_TmaAsyncLoadOp : NVGPU_Op<"tma.async.load", [AttrSizedOperandSegments]> {
+  let summary = "TMA asynchronous load";
+  let description = [{
+    The Op loads a tile memory region from global memory to shared memory by 
+    Tensor Memory Access (TMA).
+    
+    `$tensorMapDescriptor` is tensor map descriptor which has information about
+    tile shape. The descriptor is created by `nvgpu.tma.create.descriptor`
+
+    The Op uses `$barrier` mbarrier based completion mechanism. 
+  }];  
+  let arguments = (ins  Arg<AnyMemRef, "", [MemWriteAt<0, FullEffect>]>:$dst,
+                        NVGPU_MBarrierGroup:$barriers,
+                        NVGPU_TensorMapDescriptor:$tensorMapDescriptor,
+                        Variadic<Index>:$coordinates, 
+                        Index:$mbarId,
+                        Optional<I16>:$multicastMask,
+                        Optional<I1>:$predicate);
+  let assemblyFormat = [{
+    $tensorMapDescriptor `[` $coordinates `]` `,` $barriers `[` $mbarId `]` 
+      `to` $dst
+      (`multicast_mask` `=` $multicastMask^ )?
+      (`,` `predicate` `=` $predicate^)?
+      attr-dict `:` type($tensorMapDescriptor) `,` type($barriers) 
+      `->` type($dst)
+  }];
+  let hasVerifier = 1;
+
+}
+
+def NVGPU_TmaAsyncStoreOp : NVGPU_Op<"tma.async.store", [AttrSizedOperandSegments]> {
+  let summary = "TMA asynchronous store";
+  let description = [{
+    The Op store a tile memory region from global memory to shared memory by 
+    Tensor Memory Access (TMA).
+    
+    `$tensorMapDescriptor` is tensor map descriptor which has information about
+    tile shape. The descriptor is created by `nvgpu.tma.create.descriptor`
+  }];  
+  let arguments = (ins  Arg<AnyMemRef, "", [MemReadAt<0, FullEffect>]>:$src,
+                        Arg<NVGPU_TensorMapDescriptor, "", [MemWriteAt<0, FullEffect>]>:$tensorMapDescriptor,
+                        Variadic<Index>:$coordinates, 
+                        Optional<I1>:$predicate);
+  let assemblyFormat = [{
+      $src `to` $tensorMapDescriptor `[` $coordinates `]`
+      (`,` `predicate` `=` $predicate^)?
+      attr-dict `:` type($src)
+      `->` type($tensorMapDescriptor)
+  }];
+  let hasVerifier = 1;
+}
+
+def NVGPU_TmaCreateDescriptorOp : NVGPU_Op<"tma.create.descriptor", []> {
+  let summary = "TMA create descriptor";
+  let description = [{
+    The Op creates a tensor map descriptor object representing tiled memory 
+    region. To do that it calls CUDA Driver's `cuTensorMapEncodeTiled`. The 
+    descriptor is used by Tensor Memory Access (TMA).
+
+    The `tensor` is the source tensor to be tiled. 
+
+    The `boxDimensions` is the size of the tiled memory region in each dimension.
+
+    For more information see below:
+    https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html
+  }];
+
+  let arguments = (ins AnyUnrankedMemRef:$tensor,
+                       Variadic<Index>:$boxDimensions);
+  let results = (outs NVGPU_TensorMapDescriptor:$tensorMap);
+  let assemblyFormat = [{
+         $tensor `box` `[` $boxDimensions `]` attr-dict `:` type($tensor) `->` type($tensorMap)
+  }];
+  let hasVerifier = 1;
+}
+
+def NVGPU_WarpgroupGenerateDescriptorOp : NVGPU_Op<"warpgroup.generate.descriptor", []> {
+  let summary = "Generate a warpgroup matrix descriptor";
+  let description = [{
+  This Op builds a `nvgpu.warpgroup.descriptor` that is used by 
+  `nvgpu.warpgroup.mma` to perform warpgroup-level matrix multiply and 
+  accumulate.
+
+  The descriptor specifies the properties of the matrix in shared memory that 
+  is a multiplicand in the matrix multiply and accumulate operation. 
+  }];  
+  let results = (outs NVGPU_WarpgroupMatrixDescriptor:$descriptor);
+  let arguments = (ins Arg<AnyMemRef, "", [MemRead]>:$tensor, 
+                       NVGPU_TensorMapDescriptor:$tensorMap);
+  let assemblyFormat = [{$tensor `,` $tensorMap attr-dict `:` type($tensor) `,` type($tensorMap) `->` type($descriptor)}];
+  let hasVerifier = 1;
+}
+
+def NVGPU_WarpgroupMmaOp : NVGPU_Op<"warpgroup.mma"> {
+  let description = [{
+    The `nvgpu.warpgroup.mma` op performs the warpgroup-level (4 warps) 
+    matrix-multiply-and-accumulate (mma) operation that results in 
+    `nvvm.wgmma.mma_async`. 
+    
+    The operands are `descriptorA` and `descriptorB` that are wgmma matrix 
+    descriptors that shows the properties of the matrix in shared memory. The 
+    results are thread-level ownership to the warpgroup-level mma operation 
+    shape. The shape is deduced from the descriptor types and output vector.
+
+    The Op encapsulates multiple `nvvm.wgmma.mma_async` operations to complete 
+    the given shape. As `nvvm.wgmma.async` Op, or its corresponding PTX 
+    instruction, is asynchronous, this Op groups the `nvvm.wgmma.async` and 
+    surrounds them between `wgmma.fence.aligned` and 
+    `wgmma.commit.group.sync.aligned`, `wgmma.wait.group.sync.aligned` Ops.
+
+    Example:
+    ```mlir
+      %r1,%r2 = nvgpu.warpgroup.mma %descA, %descB, %acc1, %acc2: 
+                 !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>, 
+                 !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>, 
+                 !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>>,
+                 !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>>
+                 -> 
+                 !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>>,
+                 !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>>
+    ```
+  }];
+
+  let arguments = (ins NVGPU_WarpgroupMatrixDescriptor:$descriptorA, 
+                       NVGPU_WarpgroupMatrixDescriptor:$descriptorB,                                               
+                       DefaultValuedOptionalAttr<I64Attr, "1">:$waitGroup,
+                       OptionalAttr<UnitAttr>:$transposeA,
+                       OptionalAttr<UnitAttr>:$transposeB,
+                       NVGPU_WarpgroupAccumulator:$matrixC);
+  let results = (outs NVGPU_WarpgroupAccumulator:$matrixD);
+  let assemblyFormat = [{    
+    $descriptorA`,` $descriptorB`,` $matrixC attr-dict
+    `:` type($descriptorA) `,` type($descriptorB) `,` type($matrixC) `->` type($matrixD)
+  }];
+  let hasVerifier = 1;
+}
+
+def NVGPU_WarpgroupMmaStoreOp : NVGPU_Op<"warpgroup.mma.store"> {
+  let description = [{
+    The `nvgpu.warpgroup.mma.store` op performs the store of fragmented result 
+    in $matrixD to given memref. 
+
+    [See the details of register fragment layout for accumulator matrix D]
+    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#wgmma-64n16-d) 
+
+    Note that, the op must be run with warp group.
+  }];
+
+  let arguments = (ins NVGPU_WarpgroupAccumulator:$matrixD,
+                       Arg<AnyMemRef, "", [MemWrite]>:$dstMemref);
+  
+  let assemblyFormat = [{
+    $matrixD `,` $dstMemref attr-dict `:` type($matrixD) `to` type($dstMemref)
+  }];
+  let hasVerifier = 1;
+}
+
+def NVGPU_WarpgroupMmaInitAccumulatorOp : NVGPU_Op<"warpgroup.mma.init.accumulator"> {  
+  let summary = "Initializes the accumulator matrix";
+
+  let description = [{
+    This Op generates and initializes the accumulator matrix for 
+    `nvgpu.warpgroup.mma` op to perform matrix-multiply-and-accumulate.
+  }];
+  let results = (outs NVGPU_WarpgroupAccumulator:$matrixC);
+  let assemblyFormat = "attr-dict `->` type($matrixC)";
+  let hasVerifier = 1;
+}
+
+def NVGPU_RcpOp : NVGPU_Op<"rcp", [Pure,
+                                   SameOperandsAndResultType]> {
+  let summary = "The reciprocal calculation for vector types";
+  let description = [{
+    Reciprocal calculation for `vector` types using `nvvm.rcp` OPs.
+
+    Currently, only the `approx` rounding mode and `ftz` are supported, and only for the `f32` type.
+
+    The input and output must be of the same vector type and shape.
+  }];
+  let arguments = (ins VectorOfNonZeroRankOf<[F32]>:$in,
+                       DefaultValuedAttr<RcpRoundingModeAttr, "RcpRoundingMode::APPROX">:$rounding,
+                       UnitAttr:$ftz);
+  let results = (outs VectorOfNonZeroRankOf<[F32]>:$out);
+  let assemblyFormat = [{
+    $in `{` `rounding` `=` $rounding (`,` `ftz` $ftz^)? `}` 
+    attr-dict `:` type($out)
+  }];
+  let hasVerifier = 1;
+}
+
+#endif // MLIR_DIALECT_NVGPU_IR_NVGPUOPS_TD
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUTypes.td b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUTypes.td
new file mode 100644
index 0000000000000..8836a1a9dfcd8
--- /dev/null
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUTypes.td
@@ -0,0 +1,117 @@
+//===- NVGPUTypes.td - NVGPU types -------------------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the NVGPU dialect types.
+//
+//===----------------------------------------------------------------------===//
+
+
+#ifndef MLIR_DIALECT_NVGPU_IR_NVGPUTYPES_TD
+#define MLIR_DIALECT_NVGPU_IR_NVGPUTYPES_TD
+
+include "mlir/IR/AttrTypeBase.td"
+include "mlir/Dialect/NVGPU/IR/NVGPU.td"
+
+//===----------------------------------------------------------------------===//
+// NVGPU Type Definitions
+//===----------------------------------------------------------------------===//
+
+class NVGPU_Type<string name, string typeMnemonic,
+        list<Trait> traits = []> : TypeDef<NVGPU_Dialect, name, traits> {
+  let mnemonic = typeMnemonic;
+}
+
+def NVGPU_DeviceAsyncToken : NVGPU_Type<"DeviceAsyncToken",
+                                        "device.async.token", []> {
+  let summary = "device async token type";
+  let description = [{
+    `nvgpu.device.async.token` is a type returned by an asynchronous operation
+    that runs on the GPU (device). It is used to establish an SSA-based link
+    between the async operation (e.g. DeviceAsyncCopy) and operations that
+    group or synchronize the async operations (e.g. DeviceAsyncCreateGroupOp,
+    DeviceAsyncWaitOp).
+  }];
+}
+
+def NVGPU_MBarrierGroup : NVGPU_Type<"MBarrierGroup", "mbarrier.group", []> {
+  let summary = "mbarrier barrier type";
+  let description = [{
+    This is the type for one or more mbarrier object in shared memory that is 
+    used to synchronize a variable number of threads.
+
+    If `num_barriers` is not set, the number of mbarrier objects is 1.
+
+    A mbarrier object is 64 bit with 8 byte alignment. The mbarrier object 
+    can be initiated and invalidated.
+
+    [See for more details in PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#size-and-alignment-of-mbarrier-object)
+  }];    
+  let parameters = (ins "Attribute":$memorySpace, DefaultValuedParameter<"unsigned", "1">:$num_barriers);
+  let assemblyFormat = "`<` struct(params) `>`";
+  let builders = [
+    TypeBuilder<(ins "Attribute":$memorySpace), [{
+      return $_get($_ctxt, memorySpace, 1);
+    }]>
+  ];
+}
+
+def NVGPU_MBarrierToken : NVGPU_Type<"MBarrierToken", "mbarrier.token", []> { }
+
+// https://docs.nvidia.com/cuda/parallel-thread-execution/#tensor-map
+def NVGPU_TensorMapDescriptor : NVGPU_Type<"TensorMapDescriptor", "tensormap.descriptor", []> {
+  let summary = "TensorMap descriptor";
+  let parameters = (ins "MemRefType":$tensor,
+                        EnumParameter<TensorMapSwizzleKind>:$swizzle,
+                        EnumParameter<TensorMapL2PromoKind>:$l2promo,
+                        EnumParameter<TensorMapOOBKind>:$oob,
+                        EnumParameter<TensorMapInterleaveKind>:$interleave);
+  let description = [{
+    `nvgpu.tma.descriptor` is a type that represents a TMA descriptor. It is 
+    128-byte object either in constant space or kernel paramater.    
+  }];
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
+def NVGPU_WarpgroupMatrixDescriptor : NVGPU_Type<"WarpgroupMatrixDescriptor", "warpgroup.descriptor", []> {
+  let summary = "Warpgroup matrix descriptor type";
+  let description = [{
+  The descriptor specifies the properties of the matrix in shared memory that 
+  is a multiplicand in the matrix multiply and accumulate operation. 
+  
+  The descriptor is a 64-bit value contained in a register with the following:
+  ```
+  +---------+-----+-----------+-----+-----------+-----+-----+-----------+-----+
+  |   0-13  |14-15|   16-29   |30-31|   32-45   |46-48|49-51|   52-61   |62-63|
+  +---------+-----+-----------+-----+-----------+-----+-----+-----------+-----+
+  |  14bits |2bits|   14bits  |2bits|   14bits  |2bits|3bits|   10bits  |2bits|
+  +---------+-----+-----------+-----+-----------+-----+-----+-----------+-----+
+  | BaseAddr|  0  | LeadingDim|  0  |   Stride  |  0  |Offst|     0     |Swzle|
+  +---------+-----+-----------+-----+-----------+-----+-----+-----------+-----+
+  ```
+   
+  [See for more details in PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-shared-memory-layout-matrix-descriptor) 
+  
+  }];  
+  let parameters = (ins "MemRefType":$tensor);
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
+def NVGPU_WarpgroupAccumulator : NVGPU_Type<"WarpgroupAccumulator", "warpgroup.accumulator", []> {
+  let parameters = (ins "VectorType":$fragmented);
+  let assemblyFormat = "`<` struct(params) `>`";
+  let description = [{
+    This type represents the result matrix obtained from `nvgpu.warpgroup.mma`. 
+    The `$fragmented` type signifies the distributed or fragmented result 
+    vector that is collectively owned by all the threads in the warp-group 
+    that executed `nvgpu.warpgroup.mma`.
+    [See the details of register fragment layout for accumulator matrix D]
+    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#wgmma-64n16-d) 
+  }];
+}
+
+#endif //MLIR_DIALECT_NVGPU_IR_NVGPUTYPES_TD
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td
index 32c28f72ec8e5..12da584926af8 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td
@@ -462,15 +462,14 @@ class OpenMP_HasDeviceAddrClauseSkip<
     bit description = false, bit extraClassDeclaration = false
   > : OpenMP_Clause<traits, arguments, assemblyFormat, description,
                     extraClassDeclaration> {
+  let traits = [
+    BlockArgOpenMPOpInterface
+  ];
+
   let arguments = (ins
     Variadic<OpenMP_PointerLikeType>:$has_device_addr_vars
   );
 
-  let optAssemblyFormat = [{
-    `has_device_addr` `(` $has_device_addr_vars `:` type($has_device_addr_vars)
-                      `)`
-  }];
-
   let description = [{
     The optional `has_device_addr_vars` indicates that list items already have
     device addresses, so they may be directly accessed from the target device.
@@ -560,12 +559,6 @@ class OpenMP_HostEvalClauseSkip<
     Variadic<AnyType>:$host_eval_vars
   );
 
-  let extraClassDeclaration = [{
-    unsigned numHostEvalBlockArgs() {
-      return getHostEvalVars().size();
-    }
-  }];
-
   let description = [{
     The optional `host_eval_vars` holds values defined outside of the region of
     the `IsolatedFromAbove` operation for which a corresponding entry block
@@ -624,12 +617,10 @@ class OpenMP_InReductionClauseSkip<
 
   let extraClassDeclaration = [{
     /// Returns the reduction variables.
-    SmallVector<Value> getReductionVars() {
+    SmallVector<Value> getAllReductionVars() {
       return SmallVector<Value>(getInReductionVars().begin(),
                                 getInReductionVars().end());
     }
-
-    unsigned numInReductionBlockArgs() { return getInReductionVars().size(); }
   }];
 
   // Description varies depending on the operation. Assembly format not defined
@@ -744,6 +735,9 @@ class OpenMP_MapClauseSkip<
     Variadic<OpenMP_PointerLikeType>:$map_vars
   );
 
+  // This assembly format should only be used by operations where `map` does not
+  // define entry block arguments. Otherwise, it must be printed and parsed
+  // together with the corresponding region.
   let optAssemblyFormat = [{
     `map_entries` `(` $map_vars `:` type($map_vars) `)`
   }];
@@ -1055,8 +1049,6 @@ class OpenMP_DetachClauseSkip<
     : OpenMP_Clause<traits, arguments, assemblyFormat, description,
                     extraClassDeclaration> {
 
-  let traits = [BlockArgOpenMPOpInterface];
-
   let arguments = (ins Optional<OpenMP_PointerLikeType>:$event_handle);
 
   let optAssemblyFormat = [{
@@ -1121,10 +1113,6 @@ class OpenMP_PrivateClauseSkip<
     OptionalAttr<SymbolRefArrayAttr>:$private_syms
   );
 
-  let extraClassDeclaration = [{
-    unsigned numPrivateBlockArgs() { return getPrivateVars().size(); }
-  }];
-
   // TODO: Add description.
   // Assembly format not defined because this clause must be processed together
   // with the first region of the operation, as it defines entry block
@@ -1181,7 +1169,6 @@ class OpenMP_ReductionClauseSkip<
   let extraClassDeclaration = [{
     /// Returns the number of reduction variables.
     unsigned getNumReductionVars() { return getReductionVars().size(); }
-    unsigned numReductionBlockArgs() { return getReductionVars().size(); }
   }];
 
   // Description varies depending on the operation.
@@ -1311,14 +1298,10 @@ class OpenMP_TaskReductionClauseSkip<
 
   let extraClassDeclaration = [{
     /// Returns the reduction variables.
-    SmallVector<Value> getReductionVars() {
+    SmallVector<Value> getAllReductionVars() {
       return SmallVector<Value>(getTaskReductionVars().begin(),
                                 getTaskReductionVars().end());
     }
-
-    unsigned numTaskReductionBlockArgs() {
-      return getTaskReductionVars().size();
-    }
   }];
 
   let description = [{
@@ -1408,12 +1391,6 @@ class OpenMP_UseDeviceAddrClauseSkip<
     Variadic<OpenMP_PointerLikeType>:$use_device_addr_vars
   );
 
-  let extraClassDeclaration = [{
-    unsigned numUseDeviceAddrBlockArgs() {
-      return getUseDeviceAddrVars().size();
-    }
-  }];
-
   let description = [{
     The optional `use_device_addr_vars` specifies the address of the objects in
     the device data environment.
@@ -1443,12 +1420,6 @@ class OpenMP_UseDevicePtrClauseSkip<
     Variadic<OpenMP_PointerLikeType>:$use_device_ptr_vars
   );
 
-  let extraClassDeclaration = [{
-    unsigned numUseDevicePtrBlockArgs() {
-      return getUseDevicePtrVars().size();
-    }
-  }];
-
   let description = [{
     The optional `use_device_ptr_vars` specifies the device pointers to the
     corresponding list items in the device data environment.
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index f5a8a7ba04375..401c4c11d8986 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -284,8 +284,8 @@ def SectionOp : OpenMP_Op<"section", traits = [
     // Override BlockArgOpenMPOpInterface methods based on the parent
     // omp.sections operation. Only forward-declare here because SectionsOp is
     // not completely defined at this point.
-    unsigned numPrivateBlockArgs();
-    unsigned numReductionBlockArgs();
+    OperandRange getPrivateVars();
+    OperandRange getReductionVars();
   }] # clausesExtraClassDeclaration;
   let assemblyFormat = "$region attr-dict";
 }
@@ -824,11 +824,6 @@ def TaskloopOp : OpenMP_Op<"taskloop", traits = [
     /// Returns the reduction variables
     SmallVector<Value> getAllReductionVars();
 
-    // Define BlockArgOpenMPOpInterface methods here because they are not
-    // inherited from the respective clauses.
-    unsigned numInReductionBlockArgs() { return getInReductionVars().size(); }
-    unsigned numReductionBlockArgs() { return getReductionVars().size(); }
-
     void getEffects(SmallVectorImpl<MemoryEffects::EffectInstance> &effects);
   }] # clausesExtraClassDeclaration;
 
@@ -1151,6 +1146,14 @@ def TargetDataOp: OpenMP_Op<"target_data", traits = [
     OpBuilder<(ins CArg<"const TargetDataOperands &">:$clauses)>
   ];
 
+  let extraClassDeclaration = [{
+    // Override BlockArgOpenMPOpInterface method because `map` clauses have no
+    // associated entry block arguments in this operation.
+    unsigned numMapBlockArgs() {
+      return 0;
+    }
+  }] # clausesExtraClassDeclaration;
+
   let assemblyFormat = clausesAssemblyFormat # [{
     custom<UseDeviceAddrUseDevicePtrRegion>(
         $region, $use_device_addr_vars, type($use_device_addr_vars),
@@ -1185,6 +1188,14 @@ def TargetEnterDataOp: OpenMP_Op<"target_enter_data", traits = [
     OpBuilder<(ins CArg<"const TargetEnterExitUpdateDataOperands &">:$clauses)>
   ];
 
+  let extraClassDeclaration = [{
+    // Override BlockArgOpenMPOpInterface method because `map` clauses have no
+    // associated entry block arguments in this operation.
+    unsigned numMapBlockArgs() {
+      return 0;
+    }
+  }] # clausesExtraClassDeclaration;
+
   let hasVerifier = 1;
 }
 
@@ -1213,6 +1224,14 @@ def TargetExitDataOp: OpenMP_Op<"target_exit_data", traits = [
     OpBuilder<(ins CArg<"const TargetEnterExitUpdateDataOperands &">:$clauses)>
   ];
 
+  let extraClassDeclaration = [{
+    // Override BlockArgOpenMPOpInterface method because `map` clauses have no
+    // associated entry block arguments in this operation.
+    unsigned numMapBlockArgs() {
+      return 0;
+    }
+  }] # clausesExtraClassDeclaration;
+
   let hasVerifier = 1;
 }
 
@@ -1249,6 +1268,14 @@ def TargetUpdateOp: OpenMP_Op<"target_update", traits = [
     OpBuilder<(ins CArg<"const TargetEnterExitUpdateDataOperands &">:$clauses)>
   ];
 
+  let extraClassDeclaration = [{
+    // Override BlockArgOpenMPOpInterface method because `map` clauses have no
+    // associated entry block arguments in this operation.
+    unsigned numMapBlockArgs() {
+      return 0;
+    }
+  }] # clausesExtraClassDeclaration;
+
   let hasVerifier = 1;
 }
 
@@ -1292,8 +1319,6 @@ def TargetOp : OpenMP_Op<"target", traits = [
   ];
 
   let extraClassDeclaration = [{
-    unsigned numMapBlockArgs() { return getMapVars().size(); }
-
     mlir::Value getMappedValueForPrivateVar(unsigned privVarIdx) {
       std::optional<DenseI64ArrayAttr> privateMapIdices = getPrivateMapsAttr();
 
@@ -1326,8 +1351,9 @@ def TargetOp : OpenMP_Op<"target", traits = [
   }] # clausesExtraClassDeclaration;
 
   let assemblyFormat = clausesAssemblyFormat # [{
-    custom<HostEvalInReductionMapPrivateRegion>(
-        $region, $host_eval_vars, type($host_eval_vars), $in_reduction_vars,
+    custom<TargetOpRegion>(
+        $region, $has_device_addr_vars, type($has_device_addr_vars),
+        $host_eval_vars, type($host_eval_vars), $in_reduction_vars,
         type($in_reduction_vars), $in_reduction_byref, $in_reduction_syms,
         $map_vars, type($map_vars), $private_vars, type($private_vars),
         $private_syms, $private_maps) attr-dict
@@ -1817,6 +1843,14 @@ def DeclareMapperInfoOp : OpenMP_Op<"declare_mapper.info", [
     OpBuilder<(ins CArg<"const DeclareMapperInfoOperands &">:$clauses)>
   ];
 
+  let extraClassDeclaration = [{
+    // Override BlockArgOpenMPOpInterface method because `map` clauses have no
+    // associated entry block arguments in this operation.
+    unsigned numMapBlockArgs() {
+      return 0;
+    }
+  }] # clausesExtraClassDeclaration;
+
   let hasVerifier = 1;
 }
 
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
index ad5580a161568..0766b4e8d1472 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
@@ -23,19 +23,41 @@ include "mlir/IR/OpBase.td"
 // arguments corresponding to each of these clauses.
 class BlockArgOpenMPClause<string clauseNameSnake, string clauseNameCamel,
     BlockArgOpenMPClause previousClause> {
-  // Default-implemented method to be overriden by the corresponding clause.
+  // Default-implemented method, overriden by the corresponding clause. It
+  // returns the range of operands passed to the operation associated to the
+  // clause.
+  //
+  // For the override to work, the clause tablegen definition must contain a
+  // `Variadic<...> $clause_name_vars` argument.
   //
   // Usage example:
   //
   // ```c++
-  // auto iface = cast<BlockArgOpenMPOpInterface>(op);
-  // unsigned numInReductionArgs = iface.numInReductionBlockArgs();
+  // OperandRange reductionVars = op.getReductionVars();
+  // ```
+  InterfaceMethod varsMethod = InterfaceMethod<
+    "Get operation operands associated to `" # clauseNameSnake # "`.",
+    "::mlir::OperandRange", "get" # clauseNameCamel # "Vars", (ins), [{}], [{
+      return {0, 0};
+    }]
+  >;
+
+  // It returns the number of entry block arguments introduced by the given
+  // clause.
+  //
+  // By default, it will be the number of operands corresponding to that clause,
+  // but it can be overriden by operations where this might not be the case
+  // (e.g. `map` clause in `omp.target_update`).
+  //
+  // Usage example:
+  //
+  // ```c++
+  // unsigned numInReductionArgs = op.numInReductionBlockArgs();
   // ```
   InterfaceMethod numArgsMethod = InterfaceMethod<
     "Get number of block arguments defined by `" # clauseNameSnake # "`.",
-    "unsigned", "num" # clauseNameCamel # "BlockArgs", (ins), [{}], [{
-      return 0;
-    }]
+    "unsigned", "num" # clauseNameCamel # "BlockArgs", (ins), [{}],
+    "return $_op." # varsMethod.name # "().size();"
   >;
 
   // Unified access method for the start index of clause-associated entry block
@@ -52,7 +74,7 @@ class BlockArgOpenMPClause<string clauseNameSnake, string clauseNameCamel,
     "unsigned", "get" # clauseNameCamel # "BlockArgsStart", (ins),
     !if(!initialized(previousClause), [{
         auto iface = ::llvm::cast<BlockArgOpenMPOpInterface>(*$_op);
-      }] # "return iface." # previousClause.startMethod.name # "() + $_op."
+      }] # "return iface." # previousClause.startMethod.name # "() + iface."
         # previousClause.numArgsMethod.name # "();",
         "return 0;"
     )
@@ -72,11 +94,14 @@ class BlockArgOpenMPClause<string clauseNameSnake, string clauseNameCamel,
     "get" # clauseNameCamel # "BlockArgs", (ins), [{
       auto iface = ::llvm::cast<BlockArgOpenMPOpInterface>(*$_op);
       return $_op->getRegion(0).getArguments().slice(
-    }] # "iface." # startMethod.name # "(), $_op." # numArgsMethod.name # "());"
+    }] # "iface." # startMethod.name # "(), iface." # numArgsMethod.name # "());"
   >;
 }
 
-def BlockArgHostEvalClause : BlockArgOpenMPClause<"host_eval", "HostEval", ?>;
+def BlockArgHasDeviceAddrClause : BlockArgOpenMPClause<
+    "has_device_addr", "HasDeviceAddr", ?>;
+def BlockArgHostEvalClause : BlockArgOpenMPClause<
+    "host_eval", "HostEval", BlockArgHasDeviceAddrClause>;
 def BlockArgInReductionClause : BlockArgOpenMPClause<
     "in_reduction", "InReduction", BlockArgHostEvalClause>;
 def BlockArgMapClause : BlockArgOpenMPClause<
@@ -100,15 +125,32 @@ def BlockArgOpenMPOpInterface : OpInterface<"BlockArgOpenMPOpInterface"> {
 
   let cppNamespace = "::mlir::omp";
 
-  defvar clauses = [ BlockArgHostEvalClause, BlockArgInReductionClause,
-    BlockArgMapClause, BlockArgPrivateClause, BlockArgReductionClause,
-    BlockArgTaskReductionClause, BlockArgUseDeviceAddrClause,
-    BlockArgUseDevicePtrClause ];
+  defvar clauses = [ BlockArgHasDeviceAddrClause, BlockArgHostEvalClause,
+    BlockArgInReductionClause, BlockArgMapClause, BlockArgPrivateClause,
+    BlockArgReductionClause, BlockArgTaskReductionClause,
+    BlockArgUseDeviceAddrClause, BlockArgUseDevicePtrClause ];
 
   let methods = !listconcat(
+    !foreach(clause, clauses, clause.varsMethod),
     !foreach(clause, clauses, clause.numArgsMethod),
     !foreach(clause, clauses, clause.startMethod),
-    !foreach(clause, clauses, clause.blockArgsMethod)
+    !foreach(clause, clauses, clause.blockArgsMethod),
+    [
+      InterfaceMethod<
+        "Populate a vector of pairs representing the matching between operands "
+        "and entry block arguments.", "void", "getBlockArgsPairs",
+        (ins "::llvm::SmallVectorImpl<std::pair<::mlir::Value, ::mlir::BlockArgument>> &" : $pairs),
+        [{
+          auto iface = ::llvm::cast<BlockArgOpenMPOpInterface>(*$_op);
+        }] # !interleave(!foreach(clause, clauses, [{
+        }] # "if (iface." # clause.numArgsMethod.name # "() > 0) {" # [{
+        }] # "  for (auto [var, arg] : ::llvm::zip_equal(" #
+                "iface." # clause.varsMethod.name # "()," #
+                "iface." # clause.blockArgsMethod.name # "()))" # [{
+            pairs.emplace_back(var, arg);
+        } }]), "\n")
+      >
+    ]
   );
 
   let verify = [{
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td
index adcf6fac752fe..46fdff08466c1 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td
@@ -11,6 +11,7 @@
 
 include "mlir/IR/AttrTypeBase.td"
 include "mlir/IR/EnumAttr.td"
+include "mlir/IR/OpAsmInterface.td"
 include "mlir/Dialect/SparseTensor/IR/SparseTensorBase.td"
 include "mlir/IR/TensorEncoding.td"
 
@@ -112,7 +113,7 @@ def SparseTensorDimSliceAttr : SparseTensor_Attr<"SparseTensorDimSlice", []> {
 
 // Sparse tensor encoding attribute.
 def SparseTensorEncodingAttr : SparseTensor_Attr<"SparseTensorEncoding",
-         [ DeclareAttrInterfaceMethods<VerifiableTensorEncoding> ] > {
+         [ DeclareAttrInterfaceMethods<VerifiableTensorEncoding>, OpAsmAttrInterface ] > {
   let mnemonic = "encoding";
 
   let description = [{
@@ -543,6 +544,15 @@ def SparseTensorEncodingAttr : SparseTensor_Attr<"SparseTensorEncoding",
     void printSymbols(AffineMap &map, AsmPrinter &printer) const;
     void printDimensions(AffineMap &map, AsmPrinter &printer, ArrayRef<::mlir::sparse_tensor::SparseTensorDimSliceAttr> dimSlices) const;
     void printLevels(AffineMap &map, AsmPrinter &printer, ArrayRef<::mlir::sparse_tensor::LevelType> lvlTypes) const;
+
+    //
+    // OpAsmAttrInterface methods.
+    //
+
+    ::mlir::OpAsmAliasResult getAlias(::llvm::raw_ostream &os) const {
+      os << "sparse";
+      return ::mlir::OpAsmAliasResult::OverridableAlias;
+    }
   }];
 
   let genVerifyDecl = 1;
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaComplianceData.h.inc b/mlir/include/mlir/Dialect/Tosa/IR/TosaComplianceData.h.inc
index a9b458acd87f2..f06b156c1e41a 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaComplianceData.h.inc
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaComplianceData.h.inc
@@ -35,9 +35,11 @@ profileComplianceMap = {
         {fp16T, fp16T, fp32T, fp32T},
         {fp32T, fp32T, fp32T, fp32T}}}}},
     {"tosa.matmul",
-     {{{Profile::pro_int}, {{i8T, i8T, i32T}}},
+     {{{Profile::pro_int}, {{i8T, i8T, i8T, i8T, i32T}}},
       {{Profile::pro_fp},
-       {{fp16T, fp16T, fp16T}, {fp16T, fp16T, fp32T}, {fp32T, fp32T, fp32T}}}}},
+       {{fp16T, fp16T, fp16T, fp16T, fp16T},
+        {fp16T, fp16T, fp16T, fp16T, fp32T},
+        {fp32T, fp32T, fp32T, fp32T, fp32T}}}}},
     {"tosa.max_pool2d",
      {{{Profile::pro_int}, {{i8T, i8T}}},
       {{Profile::pro_fp}, {{fp16T, fp16T}, {fp32T, fp32T}}}}},
@@ -112,8 +114,12 @@ profileComplianceMap = {
     {"tosa.logical_not",
      {{{Profile::pro_int, Profile::pro_fp}, {{boolT, boolT}}}}},
     {"tosa.negate",
-     {{{Profile::pro_int}, {{i8T, i8T}, {i16T, i16T}, {i32T, i32T}}},
-      {{Profile::pro_fp}, {{fp16T, fp16T}, {fp32T, fp32T}}}}},
+     {{{Profile::pro_int},
+       {{i8T, i8T, i8T, i8T},
+        {i16T, i16T, i16T, i16T},
+        {i32T, i32T, i32T, i32T}}},
+      {{Profile::pro_fp},
+       {{fp16T, fp16T, fp16T, fp16T}, {fp32T, fp32T, fp32T, fp32T}}}}},
     {"tosa.reciprocal",
      {{{Profile::pro_fp}, {{fp16T, fp16T}, {fp32T, fp32T}}}}},
     {"tosa.rsqrt", {{{Profile::pro_fp}, {{fp16T, fp16T}, {fp32T, fp32T}}}}},
@@ -216,15 +222,15 @@ profileComplianceMap = {
         {fp32T, fp16T}}}}},
     {"tosa.rescale",
      {{{Profile::pro_int},
-       {{i8T, i8T},
-        {i8T, i16T},
-        {i8T, i32T},
-        {i16T, i8T},
-        {i16T, i16T},
-        {i16T, i32T},
-        {i32T, i8T},
-        {i32T, i16T},
-        {i32T, i32T}}}}},
+       {{i8T, i8T, i8T, i8T},
+        {i8T, i8T, i16T, i16T},
+        {i8T, i8T, i32T, i32T},
+        {i16T, i16T, i8T, i8T},
+        {i16T, i16T, i16T, i16T},
+        {i16T, i16T, i32T, i32T},
+        {i32T, i32T, i8T, i8T},
+        {i32T, i32T, i16T, i16T},
+        {i32T, i32T, i32T, i32T}}}}},
     {"tosa.const",
      {{{Profile::pro_int}, {{boolT}, {i8T}, {i16T}, {i32T}}},
       {{Profile::pro_fp}, {{fp16T}, {fp32T}}}}},
@@ -273,10 +279,10 @@ extensionComplianceMap = {
       {{Extension::int16}, {{i16T, i8T, i48T, i48T}}},
       {{Extension::bf16}, {{bf16T, bf16T, fp32T, fp32T}}}}},
     {"tosa.matmul",
-     {{{Extension::int16}, {{i16T, i16T, i48T}}},
-      {{Extension::fp8e4m3}, {{fp8e4m3T, fp8e4m3T, fp16T}}},
-      {{Extension::fp8e5m2}, {{fp8e5m2T, fp8e5m2T, fp16T}}},
-      {{Extension::bf16}, {{bf16T, bf16T, fp32T}}}}},
+     {{{Extension::int16}, {{i16T, i16T, i16T, i16T, i48T}}},
+      {{Extension::fp8e4m3}, {{fp8e4m3T, fp8e4m3T, fp8e4m3T, fp8e4m3T, fp16T}}},
+      {{Extension::fp8e5m2}, {{fp8e5m2T, fp8e5m2T, fp8e5m2T, fp8e5m2T, fp16T}}},
+      {{Extension::bf16}, {{bf16T, bf16T, bf16T, bf16T, fp32T}}}}},
     {"tosa.max_pool2d",
      {{{Extension::int16}, {{i16T, i16T}}},
       {{Extension::fp8e4m3}, {{fp8e4m3T, fp8e4m3T}}},
@@ -308,7 +314,7 @@ extensionComplianceMap = {
     {"tosa.exp", {{{Extension::bf16}, {{bf16T, bf16T}}}}},
     {"tosa.floor", {{{Extension::bf16}, {{bf16T, bf16T}}}}},
     {"tosa.log", {{{Extension::bf16}, {{bf16T, bf16T}}}}},
-    {"tosa.negate", {{{Extension::bf16}, {{bf16T, bf16T}}}}},
+    {"tosa.negate", {{{Extension::bf16}, {{bf16T, bf16T, bf16T, bf16T}}}}},
     {"tosa.reciprocal", {{{Extension::bf16}, {{bf16T, bf16T}}}}},
     {"tosa.rsqrt", {{{Extension::bf16}, {{bf16T, bf16T}}}}},
     {"tosa.select", {{{Extension::bf16}, {{bf16T, bf16T, bf16T}}}}},
@@ -384,7 +390,10 @@ extensionComplianceMap = {
         {fp16T, fp8e5m2T},
         {fp32T, fp8e5m2T}}}}},
     {"tosa.rescale",
-     {{{Extension::int16}, {{i48T, i8T}, {i48T, i16T}, {i48T, i32T}}}}},
+     {{{Extension::int16},
+       {{i48T, i48T, i8T, i8T},
+        {i48T, i48T, i16T, i16T},
+        {i48T, i48T, i32T, i32T}}}}},
     {"tosa.const",
      {{{Extension::int4}, {{i4T}}},
       {{Extension::int16}, {{i48T}}},
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td
index f2328003e49c5..52130d31c248d 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td
@@ -178,13 +178,13 @@ def Tosa_AvgPool2dOpQuantInfoBuilder : OpBuilder<
                                   input, kernel, stride, pad, acc_type);
   }]>;
 
-// This builder is called on single-parameter unary operators that have a scale
+// This builder is called on single-parameter negate operators that have a scale
 // relationship between their input and output, expressed by the
 // UnaryOpQuantizationAttr.
-def Tosa_UnaryOpQuantInfoBuilder : OpBuilder<
+def Tosa_NegateOpQuantInfoBuilder : OpBuilder<
   (ins "Type":$outputType, "Value":$input),
   [{
-    buildUnaryOpWithQuantInfo($_builder, $_state, outputType, input);
+    buildNegateOpWithQuantInfo($_builder, $_state, outputType, input);
   }]>;
 
 // These builders are called on the TOSA pad operator that needs to create its
@@ -226,6 +226,8 @@ class Tosa_I32EnumAttr<string name, string description, string mnemonic,
 // FFT          : Fast Fourier Transform operations.
 // VARIABLE     : Stateful variable operations.
 // CONTROLFLOW  : Control Flow operations.
+// DOUBLEROUND  : Adds double rounding support to the RESCALE operator.
+// INEXACTROUND : Adds inexact rounding support to the RESCALE operator.
 //===----------------------------------------------------------------------===//
 
 def Tosa_NONE : I32EnumAttrCase<"none", 0>;
@@ -241,11 +243,14 @@ def Tosa_EXT_FP8E5M2      : I32EnumAttrCase<"fp8e5m2", 5>;
 def Tosa_EXT_FFT          : I32EnumAttrCase<"fft", 6>;
 def Tosa_EXT_VARIABLE     : I32EnumAttrCase<"variable", 7>;
 def Tosa_EXT_CONTROLFLOW  : I32EnumAttrCase<"controlflow", 8>;
+def Tosa_EXT_DOUBLEROUND  : I32EnumAttrCase<"doubleround", 9>;
+def Tosa_EXT_INEXACTROUND : I32EnumAttrCase<"inexactround", 10>;
 
 def Tosa_ExtensionAttr
     : Tosa_I32EnumAttr<"Extension", "supported TOSA extensions", "ext", [
       Tosa_EXT_INT16, Tosa_EXT_INT4, Tosa_EXT_BF16, Tosa_EXT_FP8E4M3,
-      Tosa_EXT_FP8E5M2, Tosa_EXT_FFT, Tosa_EXT_VARIABLE, Tosa_EXT_CONTROLFLOW, Tosa_EXT_NONE
+      Tosa_EXT_FP8E5M2, Tosa_EXT_FFT, Tosa_EXT_VARIABLE, Tosa_EXT_CONTROLFLOW,
+      Tosa_EXT_DOUBLEROUND, Tosa_EXT_INEXACTROUND, Tosa_EXT_NONE
     ]>;
 
 def Tosa_ExtensionArrayAttr
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
index 8782d0bb55878..b79993f48b379 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
@@ -300,19 +300,17 @@ def Tosa_FFT2dOp : Tosa_InferShapedTypeOp<"fft2d", [
 // Operator: matmul
 //===----------------------------------------------------------------------===//
 def Tosa_MatMulOp : Tosa_InferShapedTypeOp<"matmul"> {
-  let summary = "Matrix multiplication with bias";
+  let summary = "Matrix multiplication";
 
   let description = [{
-    Performs a two dimensional matrix multiplication. This allows both inputs to
-    be activations, rather than reserving weights as an attribute in the
-    FULLY_CONNECTED operator.
+    Performs two dimensional matrix multiplications.
   }];
 
   let arguments = (ins
     Tosa_Tensor3D:$a,
     Tosa_Tensor3D:$b,
-    OptionalAttr<I32Attr>:$a_zp,
-    OptionalAttr<I32Attr>:$b_zp
+    Tosa_ScalarIntOrFloatTensor:$a_zp,
+    Tosa_ScalarIntOrFloatTensor:$b_zp
   );
 
   let results = (outs
@@ -324,6 +322,13 @@ def Tosa_MatMulOp : Tosa_InferShapedTypeOp<"matmul"> {
     Extension<[Tosa_EXT_INT16, Tosa_EXT_FP8E4M3, Tosa_EXT_FP8E5M2, Tosa_EXT_BF16]>,
   ];
 
+  let extraClassDeclaration = [{
+    FailureOr<int64_t> getAZeroPoint();
+    FailureOr<int64_t> getBZeroPoint();
+    LogicalResult verifyAZeroPoint(int64_t zp);
+    LogicalResult verifyBZeroPoint(int64_t zp);
+  }];
+
   let builders = [Tosa_MatMulOpQuantInfoBuilder];
   let hasVerifier = 1;
 }
@@ -389,7 +394,7 @@ def Tosa_RFFT2dOp : Tosa_InferShapedTypeOp<"rfft2d", [
   }];
 
   let arguments = (ins
-    Tosa_Tensor3D:$input,
+    Tosa_Tensor3D:$input_real,
     DefaultValuedOptionalAttr<BoolAttr, "false">:$local_bound
   );
 
@@ -404,7 +409,7 @@ def Tosa_RFFT2dOp : Tosa_InferShapedTypeOp<"rfft2d", [
   ];
 
   let assemblyFormat = [{
-    $input attr-dict `:` `(` type($input) `)` `->` `(` type($output_real) `,` type($output_imag) `)`
+    $input_real attr-dict `:` `(` type($input_real) `)` `->` `(` type($output_real) `,` type($output_imag) `)`
   }];
 
   let hasVerifier = 1;
@@ -1349,7 +1354,9 @@ def Tosa_LogicalNotOp : Tosa_ElementwiseUnaryOp<"logical_not"> {
 //===----------------------------------------------------------------------===//
 // Operator: negate
 //===----------------------------------------------------------------------===//
-def Tosa_NegateOp : Tosa_ElementwiseUnaryOp<"negate"> {
+def Tosa_NegateOp : Tosa_InferShapedTypeOp<"negate", [
+    TosaElementwiseOperator,
+    Pure]> {
   let summary = "Elementwise negate op";
 
   let description = [{
@@ -1358,8 +1365,8 @@ def Tosa_NegateOp : Tosa_ElementwiseUnaryOp<"negate"> {
 
   let arguments = (ins
       Tosa_Tensor:$input1,
-      OptionalAttr<I32Attr>:$input1_zp,
-      OptionalAttr<I32Attr>:$output_zp
+      Tosa_ScalarIntOrFloatTensor:$input1_zp,
+      Tosa_ScalarIntOrFloatTensor:$output_zp
   );
 
   let results = (outs
@@ -1371,9 +1378,20 @@ def Tosa_NegateOp : Tosa_ElementwiseUnaryOp<"negate"> {
     Extension<[Tosa_EXT_BF16]>,
   ];
 
-  let builders = [Tosa_UnaryOpQuantInfoBuilder];
+  let builders = [Tosa_NegateOpQuantInfoBuilder];
+
+  let extraClassDeclaration = [{
+    FailureOr<int64_t> getInput1ZeroPoint();
+    FailureOr<int64_t> getOutputZeroPoint();
+    LogicalResult verifyInput1ZeroPoint(int64_t zp);
+    LogicalResult verifyOutputZeroPoint(int64_t zp);
+  }];
 
   let hasFolder = 1;
+  let hasVerifier = 1;
+
+  let assemblyFormat =
+      "operands attr-dict `:` functional-type(operands, results)";
 }
 
 //===----------------------------------------------------------------------===//
@@ -2234,28 +2252,58 @@ def Tosa_CastOp: Tosa_Op<"cast", [Pure,
   let summary = "Cast operation";
 
   let description = [{
-    Performs a set of permissible cast operations
+    Casts a tensor from one data type to another.
+    * This table is showing the supported conversions from the TOSA Specification.
+    * The MLIR dialect here can be used to represent other conversions. 
 
     | Mode                     | Input   | Output  |
     |--------------------------|---------|---------|
-    | signed 8 to bool         | int8    | Boolean |
-    | signed 16 to bool        | int16   | Boolean |
-    | signed 32 to bool        | int32   | Boolean |
-    | bool to 8                | Boolean | int8    |
-    | bool to 16               | Boolean | int16   |
-    | bool to 32               | Boolean | int32   |
-    | signed 8 to signed 16    | int8    | int16   |
-    | signed 8 to signed 32    | int8    | int32   |
-    | signed 16 to signed 8    | int16   | int8    |
-    | signed 16 to signed 32   | int16   | int32   |
-    | signed 32 to signed 8    | int32   | int8    |
-    | signed 32 to signed 16   | int32   | int16   |
-    | float to signed 8        | float   | int8    |
-    | float to signed 16       | float   | int16   |
-    | signed 8 to float        | int8    | float   |
-    | signed 16 to float       | int16   | float   |
-    | float 32 to float 64     | float32 | float64 |
-    | float 64 to float 32     | float64 | float32 |
+    | fp16 to fp32             | float16 | float32 |
+    | fp16 to int 16           | float16 | int16   |
+    | fp16 to int 32           | float16 | int32   |
+    | fp16 to int 8            | float16 | int8    |
+    | fp32 to fp16             | float32 | float16 |
+    | fp32 to int 16           | float32 | int16   |
+    | fp32 to int 32           | float32 | int32   |
+    | fp32 to int 8            | float32 | int8    |
+    | int 16 to fp16           | int16   | float16 |
+    | int 16 to fp32           | int16   | float32 |
+    | int 32 to fp16           | int32   | float16 |
+    | int 32 to fp32           | int32   | float32 |
+    | int 8 to fp16            | int8    | float16 |
+    | int 8 to fp32            | int8    | float32 |
+    | bool to int 16           | Boolean | int16   |
+    | bool to int 32           | Boolean | int32   |
+    | bool to int 8            | Boolean | int8    |
+    | int 16 to bool           | int16   | Boolean |
+    | int 16 to int 32         | int16   | int32   |
+    | int 16 to int 8          | int16   | int8    |
+    | int 32 to bool           | int32   | Boolean |
+    | int 32 to int 16         | int32   | int16   |
+    | int 32 to int 8          | int32   | int8    |
+    | int 8 to bool            | int8    | Boolean |
+    | int 8 to int 16          | int8    | int16   |
+    | int 8 to int 32          | int8    | int32   |
+    | bf16 to fp32             | bf16    | float32 |
+    | bf16 to int 16           | bf16    | int16   |
+    | bf16 to int 32           | bf16    | int32   |
+    | bf16 to int 8            | bf16    | int8    |
+    | fp32 to bf16             | float32 | bf16    |
+    | int 16 to bf16           | int16   | bf16    |
+    | int 32 to bf16           | int32   | bf16    |
+    | int 8 to bf16            | int8    | bf16    |
+    | bf16 to fp8e4m3          | bf16    | fp8e4m3 |
+    | fp8e4m3 to bf16          | fp8e4m3 | bf16    |
+    | bf16 to fp8e5m2          | bf16    | fp8e5m2 |
+    | fp8e5m2 to bf16          | fp8e5m2 | bf16    |
+    | fp16 to fp8e4m3          | float16 | fp8e4m3 |
+    | fp32 to fp8e4m3          | float32 | fp8e4m3 |
+    | fp8e4m3 to fp16          | fp8e4m3 | float16 |
+    | fp8e4m3 to fp32          | fp8e4m3 | float32 |
+    | fp16 to fp8e5m2          | float16 | fp8e5m2 |
+    | fp32 to fp8e5m2          | float32 | fp8e5m2 |
+    | fp8e5m2 to fp16          | fp8e5m2 | float16 |
+    | fp8e5m2 to fp32          | fp8e5m2 | float32 |
   }];
 
   let arguments = (ins
@@ -2279,9 +2327,7 @@ def Tosa_CastOp: Tosa_Op<"cast", [Pure,
 //===----------------------------------------------------------------------===//
 // Operator: rescale
 //===----------------------------------------------------------------------===//
-def Tosa_RescaleOp: Tosa_Op<"rescale", [Pure,
-      DeclareOpInterfaceMethods<InferShapedTypeOpInterface,
-                              ["inferReturnTypeComponents"]>]> {
+def Tosa_RescaleOp : Tosa_InferShapedTypeOp<"rescale"> {
   let summary = "Tosa rescale operator";
 
   let description = [{
@@ -2307,12 +2353,12 @@ def Tosa_RescaleOp: Tosa_Op<"rescale", [Pure,
 
   let arguments = (ins
     Tosa_Tensor:$input,
-    I32Attr:$input_zp,
-    I32Attr:$output_zp,
-    DenseI32ArrayAttr:$multiplier,
-    DenseI8ArrayAttr:$shift,
+    Tosa_1DInt16Or32Tensor:$multiplier,
+    Tosa_1DInt8Tensor:$shift,
+    Tosa_ScalarIntOrFloatTensor:$input_zp,
+    Tosa_ScalarIntOrFloatTensor:$output_zp,
     BoolAttr:$scale32,
-    BoolAttr:$double_round,
+    Tosa_RoundingTypeAttr:$rounding_mode,
     BoolAttr:$per_channel,
     BoolAttr: $input_unsigned,
     BoolAttr: $output_unsigned
@@ -2327,6 +2373,15 @@ def Tosa_RescaleOp: Tosa_Op<"rescale", [Pure,
     Extension<[Tosa_EXT_INT16]>,
   ];
 
+  let extraClassDeclaration = [{
+    FailureOr<int64_t> getInputZeroPoint();
+    FailureOr<int64_t> getOutputZeroPoint();
+    LogicalResult verifyInputZeroPoint(int64_t zp);
+    LogicalResult verifyOutputZeroPoint(int64_t zp);
+  }];
+
+  let hasVerifier = 1;
+
   let assemblyFormat = "operands attr-dict `:` functional-type(operands, results)";
 }
 
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaProfileCompliance.h b/mlir/include/mlir/Dialect/Tosa/IR/TosaProfileCompliance.h
index 969d06afc70d6..88f454f63e6f9 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaProfileCompliance.h
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaProfileCompliance.h
@@ -136,6 +136,8 @@ class TosaProfileCompliance {
     switch (ext) {
     case Extension::int16:
     case Extension::int4:
+    case Extension::doubleround:
+    case Extension::inexactround:
       return {Profile::pro_int};
     case Extension::bf16:
     case Extension::fp8e4m3:
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
index 7a8357ecfa430..0038d8c386ca7 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
@@ -43,6 +43,7 @@ class Tosa_QuantizedType<string n, list<int> params, bit signed>
 
 def Tosa_Int4 : I<4>;
 def Tosa_Int8 : I<8>;
+def Tosa_Int16 : I<16>;
 def Tosa_Int32 : I<32>;
 def Tosa_Int64 : I<64>;
 
@@ -54,7 +55,10 @@ def Tosa_Int : AnyTypeOf<[AnyUnsignedInteger,
                           AnySignlessInteger]>;
 
 def Tosa_Int32Or64 : AnyTypeOf<[Tosa_Int32,
-                   	        Tosa_Int64]>;
+                                Tosa_Int64]>;
+
+def Tosa_Int16Or32 : AnyTypeOf<[Tosa_Int16,
+                                Tosa_Int32]>;
 
 //===----------------------------------------------------------------------===//
 // Quantized Integer Types.
@@ -163,6 +167,10 @@ def Tosa_Tensor3D : AnyTypeOf<[Tosa_UnrankedTensor, TosaTensorRankOf<[Tosa_AnyNu
 def Tosa_Tensor4D : AnyTypeOf<[Tosa_UnrankedTensor, TosaTensorRankOf<[Tosa_AnyNumber], [4]>], "4-d tosa-conformant tensor", "::mlir::TensorType">;
 def Tosa_Tensor5D : AnyTypeOf<[Tosa_UnrankedTensor, TosaTensorRankOf<[Tosa_AnyNumber], [5]>], "5-d tosa-conformant tensor", "::mlir::TensorType">;
 
+// 1D tensor of specific types
+def Tosa_1DInt8Tensor : 1DTensorOf<[Tosa_Int8]>;
+def Tosa_1DInt16Or32Tensor : 1DTensorOf<[Tosa_Int16Or32]>;
+
 // Ranked tensors up to given rank.
 def Tosa_Tensor1Dto4D : AnyTypeOf<[
   Tosa_UnrankedTensor, TosaTensorRankOf<[Tosa_AnyNumber], [1,2,3,4]>]>;
@@ -239,6 +247,13 @@ def Tosa_NanPropagationAttr : StringBasedAttr<
           "::llvm::cast<StringAttr>($_self).getValue() == \"IGNORE\"">,
     "Supported NaN propagation strategies">;
 
+// Rounding mode for tosa.rescale
+def Tosa_RoundingTypeAttr : StringBasedAttr<
+    CPred<"::llvm::cast<StringAttr>($_self).getValue() == \"SINGLE_ROUND\"  || " #
+          "::llvm::cast<StringAttr>($_self).getValue() == \"INEXACT_ROUND\" || " #
+          "::llvm::cast<StringAttr>($_self).getValue() == \"DOUBLE_ROUND\"">,
+    "Supported rounding modes">;
+
 def Tosa_TensorTypeAttr : TypeAttrBase<"TensorType", "Tensor type attribute">;
 
 // Tensor to buffer types.
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaUtilOps.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaUtilOps.td
index 8756cb9e5de3a..8a27e5ba39331 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaUtilOps.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaUtilOps.td
@@ -44,7 +44,7 @@ def Tosa_ApplyScaleOp :
     Tosa_IntLike:$value,
     Tosa_IntLike:$multiplier,
     Tosa_Int8Like:$shift,
-    BoolAttr:$double_round
+    Tosa_RoundingTypeAttr:$rounding_mode
   );
 
   let results = (outs
diff --git a/mlir/include/mlir/Dialect/Tosa/Utils/QuantUtils.h b/mlir/include/mlir/Dialect/Tosa/Utils/QuantUtils.h
index cb0df7aff35f7..bdd8713037eea 100644
--- a/mlir/include/mlir/Dialect/Tosa/Utils/QuantUtils.h
+++ b/mlir/include/mlir/Dialect/Tosa/Utils/QuantUtils.h
@@ -31,6 +31,26 @@ namespace tosa {
 bool computeMultiplierAndShift(double scale, int32_t &multiplier,
                                int32_t &shift, int32_t scaleWidth);
 
+// Return a const value for array of IntType vec
+template <typename IntType>
+Value getConstTensorInt(OpBuilder &builder, Location loc,
+                        ArrayRef<IntType> vec) {
+  static_assert(
+      std::is_same<IntType, int8_t>::value ||
+          std::is_same<IntType, int16_t>::value ||
+          std::is_same<IntType, int32_t>::value,
+      "getConstTensorInt only supports int8_t, int16_t, and int32_t types.");
+
+  int64_t count = vec.size();
+  assert(count > 0 && "Vector must not be empty");
+  auto element_type = builder.getIntegerType(sizeof(IntType) * 8);
+  mlir::RankedTensorType const_type =
+      RankedTensorType::get({count}, element_type);
+  mlir::DenseElementsAttr const_attr = DenseElementsAttr::get(const_type, vec);
+  auto const_op = builder.create<tosa::ConstOp>(loc, const_type, const_attr);
+  return const_op.getResult();
+}
+
 //// Builds ConvOpQuantizationAttr from input and weight.
 ConvOpQuantizationAttr buildConvOpQuantizationAttr(OpBuilder &builder,
                                                    Value input, Value weight);
diff --git a/mlir/include/mlir/Dialect/Transform/Interfaces/TransformInterfaces.h b/mlir/include/mlir/Dialect/Transform/Interfaces/TransformInterfaces.h
index e51aac02936b5..b9f2af22e9483 100644
--- a/mlir/include/mlir/Dialect/Transform/Interfaces/TransformInterfaces.h
+++ b/mlir/include/mlir/Dialect/Transform/Interfaces/TransformInterfaces.h
@@ -1074,10 +1074,18 @@ class ErrorCheckingTrackingListener : public TrackingListener {
   /// resets the error state to "success".
   DiagnosedSilenceableFailure checkAndResetError();
 
+  /// Return the latest match notification message. Returns an empty string
+  /// when no error message was captured.
+  std::string getLatestMatchFailureMessage();
+
   /// Return "true" if this tracking listener had a failure.
   bool failed() const;
 
 protected:
+  void
+  notifyMatchFailure(Location loc,
+                     function_ref<void(Diagnostic &)> reasonCallback) override;
+
   void
   notifyPayloadReplacementNotFound(Operation *op, ValueRange values,
                                    DiagnosedSilenceableFailure &&diag) override;
@@ -1089,6 +1097,9 @@ class ErrorCheckingTrackingListener : public TrackingListener {
 
   /// The number of errors that have been encountered.
   int64_t errorCounter = 0;
+
+  /// Latest message from match failure notification.
+  std::optional<Diagnostic> matchFailure;
 };
 
 /// This is a special rewriter to be used in transform op implementations,
diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h b/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h
index 6aeae30a0a6c0..601a65333d026 100644
--- a/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h
+++ b/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h
@@ -9,6 +9,7 @@
 #ifndef MLIR_DIALECT_VECTOR_TRANSFORMS_LOWERINGPATTERNS_H
 #define MLIR_DIALECT_VECTOR_TRANSFORMS_LOWERINGPATTERNS_H
 
+#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
 #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
 
 namespace mlir {
@@ -47,7 +48,8 @@ namespace vector {
 /// Progressively lower a `vector.contract` with row-major matmul semantics to
 /// linearized `vector.extract` + `vector.outerproduct` + `vector.insert`.
 void populateVectorContractLoweringPatterns(
-    RewritePatternSet &patterns, VectorTransformsOptions options,
+    RewritePatternSet &patterns,
+    VectorContractLowering vectorContractLoweringOption,
     PatternBenefit benefit = 1, bool disableOuterProductLowering = false);
 
 /// Populate the pattern set with the following patterns:
@@ -142,9 +144,10 @@ void populateVectorShapeCastLoweringPatterns(RewritePatternSet &patterns,
 ///
 /// [TransposeOp2DToShuffleLowering]
 ///
-void populateVectorTransposeLoweringPatterns(RewritePatternSet &patterns,
-                                             VectorTransformsOptions options,
-                                             PatternBenefit benefit = 1);
+void populateVectorTransposeLoweringPatterns(
+    RewritePatternSet &patterns,
+    VectorTransposeLowering vectorTransposeLowering,
+    PatternBenefit benefit = 1);
 
 /// Populate the pattern set with the following patterns:
 ///
diff --git a/mlir/include/mlir/IR/OpDefinition.h b/mlir/include/mlir/IR/OpDefinition.h
index d91c573c03efe..4fad61580b31a 100644
--- a/mlir/include/mlir/IR/OpDefinition.h
+++ b/mlir/include/mlir/IR/OpDefinition.h
@@ -74,7 +74,10 @@ void ensureRegionTerminator(
 
 /// Structure used by default as a "marker" when no "Properties" are set on an
 /// Operation.
-struct EmptyProperties {};
+struct EmptyProperties {
+  bool operator==(const EmptyProperties &) const { return true; }
+  bool operator!=(const EmptyProperties &) const { return false; }
+};
 
 /// Traits to detect whether an Operation defined a `Properties` type, otherwise
 /// it'll default to `EmptyProperties`.
diff --git a/mlir/include/mlir/IR/OperationSupport.h b/mlir/include/mlir/IR/OperationSupport.h
index 2b0e50437afcc..2d9fb2bc5859e 100644
--- a/mlir/include/mlir/IR/OperationSupport.h
+++ b/mlir/include/mlir/IR/OperationSupport.h
@@ -1029,6 +1029,36 @@ struct OperationState {
   setProperties(Operation *op,
                 function_ref<InFlightDiagnostic()> emitError) const;
 
+  // Make `newProperties` the source of the properties that will be copied into
+  // the operation. The memory referenced by `newProperties` must remain live
+  // until after the `Operation` is created, at which time it may be
+  // deallocated. Calls to `getOrAddProperties<>() will return references to
+  // this memory.
+  template <typename T>
+  void useProperties(T &newProperties) {
+    assert(!properties &&
+           "Can't provide a properties struct when one has been allocated");
+    properties = &newProperties;
+#if defined(__clang__)
+#if __has_warning("-Wdangling-assignment-gsl")
+#pragma clang diagnostic push
+// https://github.com/llvm/llvm-project/issues/126600
+#pragma clang diagnostic ignored "-Wdangling-assignment-gsl"
+#endif
+#endif
+    propertiesDeleter = [](OpaqueProperties) {};
+    propertiesSetter = [](OpaqueProperties newProp,
+                          const OpaqueProperties prop) {
+      *newProp.as<T *>() = *prop.as<const T *>();
+    };
+#if defined(__clang__)
+#if __has_warning("-Wdangling-assignment-gsl")
+#pragma clang diagnostic pop
+#endif
+#endif
+    propertiesId = TypeID::get<T>();
+  }
+
   void addOperands(ValueRange newOperands);
 
   void addTypes(ArrayRef<Type> newTypes) {
diff --git a/mlir/include/mlir/IR/PatternMatch.h b/mlir/include/mlir/IR/PatternMatch.h
index 792b50d38817e..d1f00c34f87b4 100644
--- a/mlir/include/mlir/IR/PatternMatch.h
+++ b/mlir/include/mlir/IR/PatternMatch.h
@@ -237,6 +237,9 @@ class Pattern {
 namespace detail {
 /// Helper class that derives from a RewritePattern class and provides separate
 /// `match` and `rewrite` entry points instead of a combined `matchAndRewrite`.
+///
+/// This class is deprecated. Use `matchAndRewrite` instead of separate `match`
+/// and `rewrite`.
 template <typename PatternT>
 class SplitMatchAndRewriteImpl : public PatternT {
   using PatternT::PatternT;
@@ -268,6 +271,9 @@ class SplitMatchAndRewriteImpl : public PatternT {
 class RewritePattern : public Pattern {
 public:
   using OperationT = Operation *;
+
+  /// `SplitMatchAndRewrite` is deprecated. Use `matchAndRewrite` instead of
+  /// separate `match` and `rewrite`.
   using SplitMatchAndRewrite = detail::SplitMatchAndRewriteImpl<RewritePattern>;
 
   virtual ~RewritePattern() = default;
@@ -350,6 +356,9 @@ struct OpOrInterfaceRewritePatternBase : public RewritePattern {
 template <typename SourceOp>
 struct OpRewritePattern
     : public detail::OpOrInterfaceRewritePatternBase<SourceOp> {
+
+  /// `SplitMatchAndRewrite` is deprecated. Use `matchAndRewrite` instead of
+  /// separate `match` and `rewrite`.
   using SplitMatchAndRewrite =
       detail::SplitMatchAndRewriteImpl<OpRewritePattern<SourceOp>>;
 
@@ -368,6 +377,9 @@ struct OpRewritePattern
 template <typename SourceOp>
 struct OpInterfaceRewritePattern
     : public detail::OpOrInterfaceRewritePatternBase<SourceOp> {
+
+  /// `SplitMatchAndRewrite` is deprecated. Use `matchAndRewrite` instead of
+  /// separate `match` and `rewrite`.
   using SplitMatchAndRewrite =
       detail::SplitMatchAndRewriteImpl<OpInterfaceRewritePattern<SourceOp>>;
 
diff --git a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h
index 0603bc075ab5e..50d85a8ccb1be 100644
--- a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h
+++ b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h
@@ -34,6 +34,8 @@ using DataLayoutEntryList = llvm::SmallVector<DataLayoutEntryInterface, 4>;
 using DataLayoutEntryListRef = llvm::ArrayRef<DataLayoutEntryInterface>;
 using TargetDeviceSpecListRef = llvm::ArrayRef<TargetDeviceSpecInterface>;
 using TargetDeviceSpecEntry = std::pair<StringAttr, TargetDeviceSpecInterface>;
+using DataLayoutIdentifiedEntryMap =
+    ::llvm::DenseMap<::mlir::StringAttr, ::mlir::DataLayoutEntryInterface>;
 class DataLayoutOpInterface;
 class DataLayoutSpecInterface;
 class ModuleOp;
@@ -74,6 +76,10 @@ getDefaultIndexBitwidth(Type type, const DataLayout &dataLayout,
 /// DataLayoutInterface if specified, otherwise returns the default.
 Attribute getDefaultEndianness(DataLayoutEntryInterface entry);
 
+/// Default handler for the default memory space request. Dispatches to the
+/// DataLayoutInterface if specified, otherwise returns the default.
+Attribute getDefaultMemorySpace(DataLayoutEntryInterface entry);
+
 /// Default handler for alloca memory space request. Dispatches to the
 /// DataLayoutInterface if specified, otherwise returns the default.
 Attribute getDefaultAllocaMemorySpace(DataLayoutEntryInterface entry);
@@ -231,6 +237,9 @@ class DataLayout {
   /// Returns the specified endianness.
   Attribute getEndianness() const;
 
+  /// Returns the default memory space used for memory operations.
+  Attribute getDefaultMemorySpace() const;
+
   /// Returns the memory space used for AllocaOps.
   Attribute getAllocaMemorySpace() const;
 
@@ -285,7 +294,8 @@ class DataLayout {
   mutable std::optional<Attribute> endianness;
   /// Cache for the mangling mode.
   mutable std::optional<Attribute> manglingMode;
-  /// Cache for alloca, global, and program memory spaces.
+  /// Cache for default, alloca, global, and program memory spaces.
+  mutable std::optional<Attribute> defaultMemorySpace;
   mutable std::optional<Attribute> allocaMemorySpace;
   mutable std::optional<Attribute> programMemorySpace;
   mutable std::optional<Attribute> globalMemorySpace;
diff --git a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td
index 70551c4c5b144..818b441b9a770 100644
--- a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td
+++ b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td
@@ -135,6 +135,12 @@ def DataLayoutSpecInterface : AttrInterface<"DataLayoutSpecInterface", [DLTIQuer
       /*methodName=*/"getEndiannessIdentifier",
       /*args=*/(ins "::mlir::MLIRContext *":$context)
     >,
+    InterfaceMethod<
+      /*description=*/"Returns the default memory space identifier.",
+      /*retTy=*/"::mlir::StringAttr",
+      /*methodName=*/"getDefaultMemorySpaceIdentifier",
+      /*args=*/(ins "::mlir::MLIRContext *":$context)
+    >,
     InterfaceMethod<
       /*description=*/"Returns the alloca memory space identifier.",
       /*retTy=*/"::mlir::StringAttr",
@@ -475,6 +481,18 @@ def DataLayoutOpInterface : OpInterface<"DataLayoutOpInterface"> {
         return ::mlir::detail::getDefaultEndianness(entry);
       }]
     >,
+    StaticInterfaceMethod<
+      /*description=*/"Returns the memory space used by the ABI computed "
+                      "using the relevant entries. The data layout object "
+                      "can be used for recursive queries.",
+      /*retTy=*/"::mlir::Attribute",
+      /*methodName=*/"getDefaultMemorySpace",
+      /*args=*/(ins "::mlir::DataLayoutEntryInterface":$entry),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        return ::mlir::detail::getDefaultMemorySpace(entry);
+      }]
+    >,
     StaticInterfaceMethod<
       /*description=*/"Returns the memory space used by the ABI computed "
                       "using the relevant entries. The data layout object "
@@ -637,11 +655,16 @@ def DataLayoutTypeInterface : TypeInterface<"DataLayoutTypeInterface"> {
     InterfaceMethod<
       /*desc=*/"Returns true if the two lists of entries are compatible, that "
                "is, that `newLayout` spec entries can be nested in an op with "
-               "`oldLayout` spec entries.",
+               "`oldLayout` spec entries. `newSpec` and `identified` are"
+               "provided to further query data from the combined spec, e.g.,"
+               "the default address space. TODO: Revisit this method once"
+               "https://github.com/llvm/llvm-project/issues/130321 gets solved",
       /*retTy=*/"bool",
       /*methodName=*/"areCompatible",
       /*args=*/(ins "::mlir::DataLayoutEntryListRef":$oldLayout,
-                    "::mlir::DataLayoutEntryListRef":$newLayout),
+                    "::mlir::DataLayoutEntryListRef":$newLayout,
+                    "::mlir::DataLayoutSpecInterface":$newSpec,
+                    "const ::mlir::DataLayoutIdentifiedEntryMap&":$identified),
       /*methodBody=*/"",
       /*defaultImplementation=*/[{ return true; }]
     >,
diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h
index 120709bbe5b67..f54397e942ae0 100644
--- a/mlir/include/mlir/Transforms/DialectConversion.h
+++ b/mlir/include/mlir/Transforms/DialectConversion.h
@@ -598,6 +598,9 @@ class ConversionPattern : public RewritePattern {
   using OperationT = Operation *;
   using OpAdaptor = ArrayRef<Value>;
   using OneToNOpAdaptor = ArrayRef<ValueRange>;
+
+  /// `SplitMatchAndRewrite` is deprecated. Use `matchAndRewrite` instead of
+  /// separate `match` and `rewrite`.
   using SplitMatchAndRewrite =
       detail::ConversionSplitMatchAndRewriteImpl<ConversionPattern>;
 
@@ -669,6 +672,9 @@ class OpConversionPattern : public ConversionPattern {
   using OpAdaptor = typename SourceOp::Adaptor;
   using OneToNOpAdaptor =
       typename SourceOp::template GenericAdaptor<ArrayRef<ValueRange>>;
+
+  /// `SplitMatchAndRewrite` is deprecated. Use `matchAndRewrite` instead of
+  /// separate `match` and `rewrite`.
   using SplitMatchAndRewrite =
       detail::ConversionSplitMatchAndRewriteImpl<OpConversionPattern<SourceOp>>;
 
diff --git a/mlir/include/mlir/Transforms/RegionUtils.h b/mlir/include/mlir/Transforms/RegionUtils.h
index 5c57dd5b7532a..e6b928d8ebecc 100644
--- a/mlir/include/mlir/Transforms/RegionUtils.h
+++ b/mlir/include/mlir/Transforms/RegionUtils.h
@@ -15,6 +15,7 @@
 #include "llvm/ADT/SetVector.h"
 
 namespace mlir {
+class DominanceInfo;
 class RewriterBase;
 
 /// Check if all values in the provided range are defined above the `limit`
@@ -69,6 +70,16 @@ SmallVector<Value> makeRegionIsolatedFromAbove(
     llvm::function_ref<bool(Operation *)> cloneOperationIntoRegion =
         [](Operation *) { return false; });
 
+/// Move SSA values used within an operation before an insertion point,
+/// so that the operation itself (or its replacement) can be moved to
+/// the insertion point. Current support is only for movement of
+/// dependencies of `op` before `insertionPoint` in the same basic block.
+LogicalResult moveOperationDependencies(RewriterBase &rewriter, Operation *op,
+                                        Operation *insertionPoint,
+                                        DominanceInfo &dominance);
+LogicalResult moveOperationDependencies(RewriterBase &rewriter, Operation *op,
+                                        Operation *insertionPoint);
+
 /// Run a set of structural simplifications over the given regions. This
 /// includes transformations like unreachable block elimination, dead argument
 /// elimination, as well as some other DCE. This function returns success if any
diff --git a/mlir/lib/Bindings/Python/IRCore.cpp b/mlir/lib/Bindings/Python/IRCore.cpp
index 12793f7dd15be..78ba144acf1e9 100644
--- a/mlir/lib/Bindings/Python/IRCore.cpp
+++ b/mlir/lib/Bindings/Python/IRCore.cpp
@@ -2743,6 +2743,13 @@ void mlir::python::populateIRCore(nb::module_ &m) {
   // __init__.py will subclass it with site-specific functionality and set a
   // "Context" attribute on this module.
   //----------------------------------------------------------------------------
+
+  // Expose DefaultThreadPool to python
+  nb::class_<PyThreadPool>(m, "ThreadPool")
+      .def("__init__", [](PyThreadPool &self) { new (&self) PyThreadPool(); })
+      .def("get_max_concurrency", &PyThreadPool::getMaxConcurrency)
+      .def("_mlir_thread_pool_ptr", &PyThreadPool::_mlir_thread_pool_ptr);
+
   nb::class_<PyMlirContext>(m, "_BaseContext")
       .def("__init__",
            [](PyMlirContext &self) {
@@ -2814,6 +2821,25 @@ void mlir::python::populateIRCore(nb::module_ &m) {
             mlirContextEnableMultithreading(self.get(), enable);
           },
           nb::arg("enable"))
+      .def("set_thread_pool",
+           [](PyMlirContext &self, PyThreadPool &pool) {
+             // we should disable multi-threading first before setting
+             // new thread pool otherwise the assert in
+             // MLIRContext::setThreadPool will be raised.
+             mlirContextEnableMultithreading(self.get(), false);
+             mlirContextSetThreadPool(self.get(), pool.get());
+           })
+      .def("get_num_threads",
+           [](PyMlirContext &self) {
+             return mlirContextGetNumThreads(self.get());
+           })
+      .def("_mlir_thread_pool_ptr",
+           [](PyMlirContext &self) {
+             MlirLlvmThreadPool pool = mlirContextGetThreadPool(self.get());
+             std::stringstream ss;
+             ss << pool.ptr;
+             return ss.str();
+           })
       .def(
           "is_registered_operation",
           [](PyMlirContext &self, std::string &name) {
@@ -2943,6 +2969,9 @@ void mlir::python::populateIRCore(nb::module_ &m) {
           nb::arg("callee"), nb::arg("frames"),
           nb::arg("context").none() = nb::none(),
           kContextGetCallSiteLocationDocstring)
+      .def("is_a_callsite", mlirLocationIsACallSite)
+      .def_prop_ro("callee", mlirLocationCallSiteGetCallee)
+      .def_prop_ro("caller", mlirLocationCallSiteGetCaller)
       .def_static(
           "file",
           [](std::string filename, int line, int col,
@@ -2967,6 +2996,16 @@ void mlir::python::populateIRCore(nb::module_ &m) {
           nb::arg("filename"), nb::arg("start_line"), nb::arg("start_col"),
           nb::arg("end_line"), nb::arg("end_col"),
           nb::arg("context").none() = nb::none(), kContextGetFileRangeDocstring)
+      .def("is_a_file", mlirLocationIsAFileLineColRange)
+      .def_prop_ro("filename",
+                   [](MlirLocation loc) {
+                     return mlirIdentifierStr(
+                         mlirLocationFileLineColRangeGetFilename(loc));
+                   })
+      .def_prop_ro("start_line", mlirLocationFileLineColRangeGetStartLine)
+      .def_prop_ro("start_col", mlirLocationFileLineColRangeGetStartColumn)
+      .def_prop_ro("end_line", mlirLocationFileLineColRangeGetEndLine)
+      .def_prop_ro("end_col", mlirLocationFileLineColRangeGetEndColumn)
       .def_static(
           "fused",
           [](const std::vector<PyLocation> &pyLocations,
@@ -2984,6 +3023,16 @@ void mlir::python::populateIRCore(nb::module_ &m) {
           nb::arg("locations"), nb::arg("metadata").none() = nb::none(),
           nb::arg("context").none() = nb::none(),
           kContextGetFusedLocationDocstring)
+      .def("is_a_fused", mlirLocationIsAFused)
+      .def_prop_ro("locations",
+                   [](MlirLocation loc) {
+                     unsigned numLocations =
+                         mlirLocationFusedGetNumLocations(loc);
+                     std::vector<MlirLocation> locations(numLocations);
+                     if (numLocations)
+                       mlirLocationFusedGetLocations(loc, locations.data());
+                     return locations;
+                   })
       .def_static(
           "name",
           [](std::string name, std::optional<PyLocation> childLoc,
@@ -2998,6 +3047,12 @@ void mlir::python::populateIRCore(nb::module_ &m) {
           nb::arg("name"), nb::arg("childLoc").none() = nb::none(),
           nb::arg("context").none() = nb::none(),
           kContextGetNameLocationDocString)
+      .def("is_a_name", mlirLocationIsAName)
+      .def_prop_ro("name_str",
+                   [](MlirLocation loc) {
+                     return mlirIdentifierStr(mlirLocationNameGetName(loc));
+                   })
+      .def_prop_ro("child_loc", mlirLocationNameGetChildLoc)
       .def_static(
           "from_attr",
           [](PyAttribute &attribute, DefaultingPyMlirContext context) {
@@ -3148,9 +3203,7 @@ void mlir::python::populateIRCore(nb::module_ &m) {
                      auto &concreteOperation = self.getOperation();
                      concreteOperation.checkValid();
                      MlirOperation operation = concreteOperation.get();
-                     MlirStringRef name =
-                         mlirIdentifierStr(mlirOperationGetName(operation));
-                     return nb::str(name.data, name.length);
+                     return mlirIdentifierStr(mlirOperationGetName(operation));
                    })
       .def_prop_ro("operands",
                    [](PyOperationBase &self) {
@@ -3738,8 +3791,7 @@ void mlir::python::populateIRCore(nb::module_ &m) {
       .def_prop_ro(
           "name",
           [](PyNamedAttribute &self) {
-            return nb::str(mlirIdentifierStr(self.namedAttr.name).data,
-                           mlirIdentifierStr(self.namedAttr.name).length);
+            return mlirIdentifierStr(self.namedAttr.name);
           },
           "The name of the NamedAttribute binding")
       .def_prop_ro(
@@ -3972,7 +4024,16 @@ void mlir::python::populateIRCore(nb::module_ &m) {
           nb::arg("with"), nb::arg("exceptions"),
           kValueReplaceAllUsesExceptDocstring)
       .def(MLIR_PYTHON_MAYBE_DOWNCAST_ATTR,
-           [](PyValue &self) { return self.maybeDownCast(); });
+           [](PyValue &self) { return self.maybeDownCast(); })
+      .def_prop_ro(
+          "location",
+          [](MlirValue self) {
+            return PyLocation(
+                PyMlirContext::forContext(mlirValueGetContext(self)),
+                mlirValueGetLocation(self));
+          },
+          "Returns the source location the value");
+
   PyBlockArgument::bind(m);
   PyOpResult::bind(m);
   PyOpOperand::bind(m);
diff --git a/mlir/lib/Bindings/Python/IRModule.h b/mlir/lib/Bindings/Python/IRModule.h
index 1ed6240a6ca69..9befcce725bb7 100644
--- a/mlir/lib/Bindings/Python/IRModule.h
+++ b/mlir/lib/Bindings/Python/IRModule.h
@@ -11,6 +11,7 @@
 #define MLIR_BINDINGS_PYTHON_IRMODULES_H
 
 #include <optional>
+#include <sstream>
 #include <utility>
 #include <vector>
 
@@ -22,9 +23,10 @@
 #include "mlir-c/IR.h"
 #include "mlir-c/IntegerSet.h"
 #include "mlir-c/Transforms.h"
-#include "mlir/Bindings/Python/NanobindAdaptors.h"
 #include "mlir/Bindings/Python/Nanobind.h"
+#include "mlir/Bindings/Python/NanobindAdaptors.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/Support/ThreadPool.h"
 
 namespace mlir {
 namespace python {
@@ -158,6 +160,29 @@ class PyThreadContextEntry {
   FrameKind frameKind;
 };
 
+/// Wrapper around MlirLlvmThreadPool
+/// Python object owns the C++ thread pool
+class PyThreadPool {
+public:
+  PyThreadPool() {
+    ownedThreadPool = std::make_unique<llvm::DefaultThreadPool>();
+  }
+  PyThreadPool(const PyThreadPool &) = delete;
+  PyThreadPool(PyThreadPool &&) = delete;
+
+  int getMaxConcurrency() const { return ownedThreadPool->getMaxConcurrency(); }
+  MlirLlvmThreadPool get() { return wrap(ownedThreadPool.get()); }
+
+  std::string _mlir_thread_pool_ptr() const {
+    std::stringstream ss;
+    ss << ownedThreadPool.get();
+    return ss.str();
+  }
+
+private:
+  std::unique_ptr<llvm::ThreadPoolInterface> ownedThreadPool;
+};
+
 /// Wrapper around MlirContext.
 using PyMlirContextRef = PyObjectRef<PyMlirContext>;
 class PyMlirContext {
diff --git a/mlir/lib/CAPI/IR/IR.cpp b/mlir/lib/CAPI/IR/IR.cpp
index 6cd9ba2aef233..e0e386d55ede1 100644
--- a/mlir/lib/CAPI/IR/IR.cpp
+++ b/mlir/lib/CAPI/IR/IR.cpp
@@ -114,6 +114,14 @@ void mlirContextSetThreadPool(MlirContext context,
   unwrap(context)->setThreadPool(*unwrap(threadPool));
 }
 
+unsigned mlirContextGetNumThreads(MlirContext context) {
+  return unwrap(context)->getNumThreads();
+}
+
+MlirLlvmThreadPool mlirContextGetThreadPool(MlirContext context) {
+  return wrap(&unwrap(context)->getThreadPool());
+}
+
 //===----------------------------------------------------------------------===//
 // Dialect API.
 //===----------------------------------------------------------------------===//
@@ -259,7 +267,7 @@ MlirAttribute mlirLocationGetAttribute(MlirLocation location) {
 }
 
 MlirLocation mlirLocationFromAttribute(MlirAttribute attribute) {
-  return wrap(Location(llvm::cast<LocationAttr>(unwrap(attribute))));
+  return wrap(Location(llvm::dyn_cast<LocationAttr>(unwrap(attribute))));
 }
 
 MlirLocation mlirLocationFileLineColGet(MlirContext context,
@@ -278,10 +286,64 @@ mlirLocationFileLineColRangeGet(MlirContext context, MlirStringRef filename,
                                      startLine, startCol, endLine, endCol)));
 }
 
+MlirIdentifier mlirLocationFileLineColRangeGetFilename(MlirLocation location) {
+  return wrap(llvm::dyn_cast<FileLineColRange>(unwrap(location)).getFilename());
+}
+
+int mlirLocationFileLineColRangeGetStartLine(MlirLocation location) {
+  if (auto loc = llvm::dyn_cast<FileLineColRange>(unwrap(location)))
+    return loc.getStartLine();
+  return -1;
+}
+
+int mlirLocationFileLineColRangeGetStartColumn(MlirLocation location) {
+  if (auto loc = llvm::dyn_cast<FileLineColRange>(unwrap(location)))
+    return loc.getStartColumn();
+  return -1;
+}
+
+int mlirLocationFileLineColRangeGetEndLine(MlirLocation location) {
+  if (auto loc = llvm::dyn_cast<FileLineColRange>(unwrap(location)))
+    return loc.getEndLine();
+  return -1;
+}
+
+int mlirLocationFileLineColRangeGetEndColumn(MlirLocation location) {
+  if (auto loc = llvm::dyn_cast<FileLineColRange>(unwrap(location)))
+    return loc.getEndColumn();
+  return -1;
+}
+
+MlirTypeID mlirLocationFileLineColRangeGetTypeID() {
+  return wrap(FileLineColRange::getTypeID());
+}
+
+bool mlirLocationIsAFileLineColRange(MlirLocation location) {
+  return isa<FileLineColRange>(unwrap(location));
+}
+
 MlirLocation mlirLocationCallSiteGet(MlirLocation callee, MlirLocation caller) {
   return wrap(Location(CallSiteLoc::get(unwrap(callee), unwrap(caller))));
 }
 
+MlirLocation mlirLocationCallSiteGetCallee(MlirLocation location) {
+  return wrap(
+      Location(llvm::dyn_cast<CallSiteLoc>(unwrap(location)).getCallee()));
+}
+
+MlirLocation mlirLocationCallSiteGetCaller(MlirLocation location) {
+  return wrap(
+      Location(llvm::dyn_cast<CallSiteLoc>(unwrap(location)).getCaller()));
+}
+
+MlirTypeID mlirLocationCallSiteGetTypeID() {
+  return wrap(CallSiteLoc::getTypeID());
+}
+
+bool mlirLocationIsACallSite(MlirLocation location) {
+  return isa<CallSiteLoc>(unwrap(location));
+}
+
 MlirLocation mlirLocationFusedGet(MlirContext ctx, intptr_t nLocations,
                                   MlirLocation const *locations,
                                   MlirAttribute metadata) {
@@ -290,6 +352,30 @@ MlirLocation mlirLocationFusedGet(MlirContext ctx, intptr_t nLocations,
   return wrap(FusedLoc::get(unwrappedLocs, unwrap(metadata), unwrap(ctx)));
 }
 
+unsigned mlirLocationFusedGetNumLocations(MlirLocation location) {
+  if (auto locationsArrRef = llvm::dyn_cast<FusedLoc>(unwrap(location)))
+    return locationsArrRef.getLocations().size();
+  return 0;
+}
+
+void mlirLocationFusedGetLocations(MlirLocation location,
+                                   MlirLocation *locationsCPtr) {
+  if (auto locationsArrRef = llvm::dyn_cast<FusedLoc>(unwrap(location))) {
+    for (auto [i, location] : llvm::enumerate(locationsArrRef.getLocations()))
+      locationsCPtr[i] = wrap(location);
+  }
+}
+
+MlirAttribute mlirLocationFusedGetMetadata(MlirLocation location) {
+  return wrap(llvm::dyn_cast<FusedLoc>(unwrap(location)).getMetadata());
+}
+
+MlirTypeID mlirLocationFusedGetTypeID() { return wrap(FusedLoc::getTypeID()); }
+
+bool mlirLocationIsAFused(MlirLocation location) {
+  return isa<FusedLoc>(unwrap(location));
+}
+
 MlirLocation mlirLocationNameGet(MlirContext context, MlirStringRef name,
                                  MlirLocation childLoc) {
   if (mlirLocationIsNull(childLoc))
@@ -299,6 +385,21 @@ MlirLocation mlirLocationNameGet(MlirContext context, MlirStringRef name,
       StringAttr::get(unwrap(context), unwrap(name)), unwrap(childLoc))));
 }
 
+MlirIdentifier mlirLocationNameGetName(MlirLocation location) {
+  return wrap((llvm::dyn_cast<NameLoc>(unwrap(location)).getName()));
+}
+
+MlirLocation mlirLocationNameGetChildLoc(MlirLocation location) {
+  return wrap(
+      Location(llvm::dyn_cast<NameLoc>(unwrap(location)).getChildLoc()));
+}
+
+MlirTypeID mlirLocationNameGetTypeID() { return wrap(NameLoc::getTypeID()); }
+
+bool mlirLocationIsAName(MlirLocation location) {
+  return isa<NameLoc>(unwrap(location));
+}
+
 MlirLocation mlirLocationUnknownGet(MlirContext context) {
   return wrap(Location(UnknownLoc::get(unwrap(context))));
 }
@@ -975,25 +1076,26 @@ bool mlirValueIsAOpResult(MlirValue value) {
 }
 
 MlirBlock mlirBlockArgumentGetOwner(MlirValue value) {
-  return wrap(llvm::cast<BlockArgument>(unwrap(value)).getOwner());
+  return wrap(llvm::dyn_cast<BlockArgument>(unwrap(value)).getOwner());
 }
 
 intptr_t mlirBlockArgumentGetArgNumber(MlirValue value) {
   return static_cast<intptr_t>(
-      llvm::cast<BlockArgument>(unwrap(value)).getArgNumber());
+      llvm::dyn_cast<BlockArgument>(unwrap(value)).getArgNumber());
 }
 
 void mlirBlockArgumentSetType(MlirValue value, MlirType type) {
-  llvm::cast<BlockArgument>(unwrap(value)).setType(unwrap(type));
+  if (auto blockArg = llvm::dyn_cast<BlockArgument>(unwrap(value)))
+    blockArg.setType(unwrap(type));
 }
 
 MlirOperation mlirOpResultGetOwner(MlirValue value) {
-  return wrap(llvm::cast<OpResult>(unwrap(value)).getOwner());
+  return wrap(llvm::dyn_cast<OpResult>(unwrap(value)).getOwner());
 }
 
 intptr_t mlirOpResultGetResultNumber(MlirValue value) {
   return static_cast<intptr_t>(
-      llvm::cast<OpResult>(unwrap(value)).getResultNumber());
+      llvm::dyn_cast<OpResult>(unwrap(value)).getResultNumber());
 }
 
 MlirType mlirValueGetType(MlirValue value) {
@@ -1047,6 +1149,14 @@ void mlirValueReplaceAllUsesExcept(MlirValue oldValue, MlirValue newValue,
   oldValueCpp.replaceAllUsesExcept(newValueCpp, exceptionSet);
 }
 
+MlirLocation mlirValueGetLocation(MlirValue v) {
+  return wrap(unwrap(v).getLoc());
+}
+
+MlirContext mlirValueGetContext(MlirValue v) {
+  return wrap(unwrap(v).getContext());
+}
+
 //===----------------------------------------------------------------------===//
 // OpOperand API.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
index 734c4839f9a10..27be54728c1a1 100644
--- a/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
+++ b/mlir/lib/Conversion/ArithToAMDGPU/ArithToAMDGPU.cpp
@@ -41,48 +41,46 @@ struct ArithToAMDGPUConversionPass final
   void runOnOperation() override;
 };
 
-struct ExtFOnFloat8RewritePattern final
-    : OpRewritePattern<arith::ExtFOp>::SplitMatchAndRewrite {
-  using SplitMatchAndRewrite::SplitMatchAndRewrite;
+struct ExtFOnFloat8RewritePattern final : OpRewritePattern<arith::ExtFOp> {
+  using OpRewritePattern::OpRewritePattern;
 
   Chipset chipset;
   ExtFOnFloat8RewritePattern(MLIRContext *ctx, Chipset chipset)
-      : SplitMatchAndRewrite::SplitMatchAndRewrite(ctx), chipset(chipset) {}
+      : OpRewritePattern::OpRewritePattern(ctx), chipset(chipset) {}
 
-  LogicalResult match(arith::ExtFOp op) const override;
-  void rewrite(arith::ExtFOp op, PatternRewriter &rewriter) const override;
+  LogicalResult matchAndRewrite(arith::ExtFOp op,
+                                PatternRewriter &rewriter) const override;
 };
 
-struct TruncFToFloat8RewritePattern final
-    : OpRewritePattern<arith::TruncFOp>::SplitMatchAndRewrite {
+struct TruncFToFloat8RewritePattern final : OpRewritePattern<arith::TruncFOp> {
   bool saturateFP8 = false;
   TruncFToFloat8RewritePattern(MLIRContext *ctx, bool saturateFP8,
                                Chipset chipset)
-      : SplitMatchAndRewrite::SplitMatchAndRewrite(ctx),
-        saturateFP8(saturateFP8), chipset(chipset) {}
+      : OpRewritePattern::OpRewritePattern(ctx), saturateFP8(saturateFP8),
+        chipset(chipset) {}
   Chipset chipset;
 
-  LogicalResult match(arith::TruncFOp op) const override;
-  void rewrite(arith::TruncFOp op, PatternRewriter &rewriter) const override;
+  LogicalResult matchAndRewrite(arith::TruncFOp op,
+                                PatternRewriter &rewriter) const override;
 };
 
 struct TruncfToFloat16RewritePattern final
-    : public OpRewritePattern<arith::TruncFOp>::SplitMatchAndRewrite {
+    : public OpRewritePattern<arith::TruncFOp> {
 
-  using SplitMatchAndRewrite::SplitMatchAndRewrite;
+  using OpRewritePattern::OpRewritePattern;
 
-  LogicalResult match(arith::TruncFOp op) const override;
-  void rewrite(arith::TruncFOp op, PatternRewriter &rewriter) const override;
+  LogicalResult matchAndRewrite(arith::TruncFOp op,
+                                PatternRewriter &rewriter) const override;
 };
 
 } // end namespace
 
-static LogicalResult isSupportedF8(Type elementType, Chipset chipset) {
+static bool isSupportedF8(Type elementType, Chipset chipset) {
   if (chipset == kGfx942)
-    return success(isa<Float8E4M3FNUZType, Float8E5M2FNUZType>(elementType));
+    return isa<Float8E4M3FNUZType, Float8E5M2FNUZType>(elementType);
   if (hasOcpFp8(chipset))
-    return success(isa<Float8E4M3FNType, Float8E5M2Type>(elementType));
-  return failure();
+    return isa<Float8E4M3FNType, Float8E5M2Type>(elementType);
+  return false;
 }
 
 static Value castF32To(Type elementType, Value f32, Location loc,
@@ -96,35 +94,36 @@ static Value castF32To(Type elementType, Value f32, Location loc,
   llvm_unreachable("The only 32-bit float type is f32");
 }
 
-LogicalResult ExtFOnFloat8RewritePattern::match(arith::ExtFOp op) const {
+LogicalResult
+ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
+                                            PatternRewriter &rewriter) const {
   Type inType = op.getIn().getType();
-  if (auto inVecType = dyn_cast<VectorType>(inType)) {
+  auto inVecType = dyn_cast<VectorType>(inType);
+  if (inVecType) {
     if (inVecType.isScalable())
       return failure();
     inType = inVecType.getElementType();
   }
-  return isSupportedF8(inType, chipset);
-}
+  if (!isSupportedF8(inType, chipset))
+    return failure();
 
-void ExtFOnFloat8RewritePattern::rewrite(arith::ExtFOp op,
-                                         PatternRewriter &rewriter) const {
   Location loc = op.getLoc();
   Value in = op.getIn();
   Type outElemType = getElementTypeOrSelf(op.getOut().getType());
-  auto inType = dyn_cast<VectorType>(in.getType());
-  if (!inType) {
+  if (!inVecType) {
     Value asFloat = rewriter.create<amdgpu::ExtPackedFp8Op>(
         loc, rewriter.getF32Type(), in, 0);
     Value result = castF32To(outElemType, asFloat, loc, rewriter);
-    return rewriter.replaceOp(op, result);
+    rewriter.replaceOp(op, result);
+    return success();
   }
-  int64_t numElements = inType.getNumElements();
+  int64_t numElements = inVecType.getNumElements();
 
   Value zero = rewriter.create<arith::ConstantOp>(
       loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
   VectorType outType = cast<VectorType>(op.getOut().getType());
 
-  if (inType.getShape().empty()) {
+  if (inVecType.getShape().empty()) {
     Value zerodSplat =
         rewriter.createOrFold<vector::SplatOp>(loc, outType, zero);
     Value scalarIn =
@@ -133,17 +132,18 @@ void ExtFOnFloat8RewritePattern::rewrite(arith::ExtFOp op,
         rewriter.create<arith::ExtFOp>(loc, outElemType, scalarIn);
     Value result = rewriter.create<vector::InsertOp>(loc, scalarExt, zerodSplat,
                                                      ArrayRef<int64_t>{});
-    return rewriter.replaceOp(op, result);
+    rewriter.replaceOp(op, result);
+    return success();
   }
 
   VectorType flatTy = VectorType::get(SmallVector<int64_t>{numElements},
                                       outType.getElementType());
   Value result = rewriter.createOrFold<vector::SplatOp>(loc, flatTy, zero);
 
-  if (inType.getRank() > 1) {
-    inType = VectorType::get(SmallVector<int64_t>{numElements},
-                             inType.getElementType());
-    in = rewriter.create<vector::ShapeCastOp>(loc, inType, in);
+  if (inVecType.getRank() > 1) {
+    inVecType = VectorType::get(SmallVector<int64_t>{numElements},
+                                inVecType.getElementType());
+    in = rewriter.create<vector::ShapeCastOp>(loc, inVecType, in);
   }
 
   for (int64_t i = 0; i < numElements; i += 4) {
@@ -158,11 +158,12 @@ void ExtFOnFloat8RewritePattern::rewrite(arith::ExtFOp op,
     }
   }
 
-  if (inType.getRank() != outType.getRank()) {
+  if (inVecType.getRank() != outType.getRank()) {
     result = rewriter.create<vector::ShapeCastOp>(loc, outType, result);
   }
 
   rewriter.replaceOp(op, result);
+  return success();
 }
 
 static Value castToF32(Value value, Location loc, PatternRewriter &rewriter) {
@@ -222,12 +223,15 @@ static Value clampInput(PatternRewriter &rewriter, Location loc,
   return res;
 }
 
-LogicalResult TruncFToFloat8RewritePattern::match(arith::TruncFOp op) const {
+LogicalResult
+TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
+                                              PatternRewriter &rewriter) const {
   // Only supporting default rounding mode as of now.
   if (op.getRoundingmodeAttr())
     return failure();
   Type outType = op.getOut().getType();
-  if (auto outVecType = dyn_cast<VectorType>(outType)) {
+  auto outVecType = dyn_cast<VectorType>(outType);
+  if (outVecType) {
     if (outVecType.isScalable())
       return failure();
     outType = outVecType.getElementType();
@@ -237,11 +241,9 @@ LogicalResult TruncFToFloat8RewritePattern::match(arith::TruncFOp op) const {
     // Conversion between 8-bit floats is not supported with truncation enabled.
     return failure();
 
-  return isSupportedF8(outType, chipset);
-}
+  if (!isSupportedF8(outType, chipset))
+    return failure();
 
-void TruncFToFloat8RewritePattern::rewrite(arith::TruncFOp op,
-                                           PatternRewriter &rewriter) const {
   Location loc = op.getLoc();
   Value in = op.getIn();
   Type outElemType = getElementTypeOrSelf(op.getOut().getType());
@@ -255,13 +257,14 @@ void TruncFToFloat8RewritePattern::rewrite(arith::TruncFOp op,
         loc, truncResType, asFloat, /*sourceB=*/nullptr, 0,
         /*existing=*/nullptr);
     Value result = rewriter.create<vector::ExtractOp>(loc, asF8s, 0);
-    return rewriter.replaceOp(op, result);
+    rewriter.replaceOp(op, result);
+    return success();
   }
-  VectorType outType = cast<VectorType>(op.getOut().getType());
-  int64_t numElements = outType.getNumElements();
+
+  int64_t numElements = outVecType.getNumElements();
   Value zero = rewriter.create<arith::ConstantOp>(
       loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
-  if (outType.getShape().empty()) {
+  if (outVecType.getShape().empty()) {
     Value scalarIn =
         rewriter.create<vector::ExtractOp>(loc, in, ArrayRef<int64_t>{});
     // Recurse to send the 0-D vector case to the 1-D vector case
@@ -269,11 +272,12 @@ void TruncFToFloat8RewritePattern::rewrite(arith::TruncFOp op,
         rewriter.create<arith::TruncFOp>(loc, outElemType, scalarIn);
     Value result = rewriter.create<vector::InsertOp>(loc, scalarTrunc, zero,
                                                      ArrayRef<int64_t>{});
-    return rewriter.replaceOp(op, result);
+    rewriter.replaceOp(op, result);
+    return success();
   }
 
   VectorType flatTy = VectorType::get(SmallVector<int64_t>{numElements},
-                                      outType.getElementType());
+                                      outVecType.getElementType());
   Value result = rewriter.createOrFold<vector::SplatOp>(loc, flatTy, zero);
 
   if (inVectorTy.getRank() > 1) {
@@ -303,26 +307,27 @@ void TruncFToFloat8RewritePattern::rewrite(arith::TruncFOp op,
                                                            result, i, 1);
   }
 
-  if (inVectorTy.getRank() != outType.getRank()) {
-    result = rewriter.create<vector::ShapeCastOp>(loc, outType, result);
+  if (inVectorTy.getRank() != outVecType.getRank()) {
+    result = rewriter.create<vector::ShapeCastOp>(loc, outVecType, result);
   }
 
   rewriter.replaceOp(op, result);
+  return success();
 }
 
-LogicalResult TruncfToFloat16RewritePattern::match(arith::TruncFOp op) const {
+LogicalResult TruncfToFloat16RewritePattern::matchAndRewrite(
+    arith::TruncFOp op, PatternRewriter &rewriter) const {
   Type outType = op.getOut().getType();
   Type inputType = getElementTypeOrSelf(op.getIn());
-  if (auto outVecType = dyn_cast<VectorType>(outType)) {
+  auto outVecType = dyn_cast<VectorType>(outType);
+  if (outVecType) {
     if (outVecType.isScalable())
       return failure();
     outType = outVecType.getElementType();
   }
-  return success(outType.isF16() && inputType.isF32());
-}
+  if (!(outType.isF16() && inputType.isF32()))
+    return failure();
 
-void TruncfToFloat16RewritePattern::rewrite(arith::TruncFOp op,
-                                            PatternRewriter &rewriter) const {
   Location loc = op.getLoc();
   Value in = op.getIn();
   Type outElemType = getElementTypeOrSelf(op.getOut().getType());
@@ -335,13 +340,13 @@ void TruncfToFloat16RewritePattern::rewrite(arith::TruncFOp op,
     Value asF16s =
         rewriter.create<ROCDL::CvtPkRtz>(loc, truncResType, in, sourceB);
     Value result = rewriter.create<vector::ExtractOp>(loc, asF16s, 0);
-    return rewriter.replaceOp(op, result);
+    rewriter.replaceOp(op, result);
+    return success();
   }
-  VectorType outType = cast<VectorType>(op.getOut().getType());
-  int64_t numElements = outType.getNumElements();
+  int64_t numElements = outVecType.getNumElements();
   Value zero = rewriter.createOrFold<arith::ConstantOp>(
       loc, outElemType, rewriter.getFloatAttr(outElemType, 0.0));
-  Value result = rewriter.createOrFold<vector::SplatOp>(loc, outType, zero);
+  Value result = rewriter.createOrFold<vector::SplatOp>(loc, outVecType, zero);
 
   if (inVectorTy.getRank() > 1) {
     inVectorTy = VectorType::get(SmallVector<int64_t>{numElements},
@@ -371,11 +376,12 @@ void TruncfToFloat16RewritePattern::rewrite(arith::TruncFOp op,
                                                            result, i, 1);
   }
 
-  if (inVectorTy.getRank() != outType.getRank()) {
-    result = rewriter.create<vector::ShapeCastOp>(loc, outType, result);
+  if (inVectorTy.getRank() != outVecType.getRank()) {
+    result = rewriter.create<vector::ShapeCastOp>(loc, outVecType, result);
   }
 
   rewriter.replaceOp(op, result);
+  return success();
 }
 
 void mlir::arith::populateArithToAMDGPUConversionPatterns(
diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
index 80310ce56a51b..fe0ee11d84adb 100644
--- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
+++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
@@ -657,11 +657,12 @@ struct RankOpLowering : public ConvertOpToLLVMPattern<memref::RankOp> {
   }
 };
 
-struct MemRefCastOpLowering
-    : public ConvertOpToLLVMPattern<memref::CastOp>::SplitMatchAndRewrite {
-  using SplitMatchAndRewrite::SplitMatchAndRewrite;
+struct MemRefCastOpLowering : public ConvertOpToLLVMPattern<memref::CastOp> {
+  using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
 
-  LogicalResult match(memref::CastOp memRefCastOp) const override {
+  LogicalResult
+  matchAndRewrite(memref::CastOp memRefCastOp, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
     Type srcType = memRefCastOp.getOperand().getType();
     Type dstType = memRefCastOp.getType();
 
@@ -671,30 +672,22 @@ struct MemRefCastOpLowering
     // perform a sanity check that the underlying structs are the same. Once op
     // semantics are relaxed we can revisit.
     if (isa<MemRefType>(srcType) && isa<MemRefType>(dstType))
-      return success(typeConverter->convertType(srcType) ==
-                     typeConverter->convertType(dstType));
-
-    // At least one of the operands is unranked type
-    assert(isa<UnrankedMemRefType>(srcType) ||
-           isa<UnrankedMemRefType>(dstType));
+      if (typeConverter->convertType(srcType) !=
+          typeConverter->convertType(dstType))
+        return failure();
 
     // Unranked to unranked cast is disallowed
-    return !(isa<UnrankedMemRefType>(srcType) &&
-             isa<UnrankedMemRefType>(dstType))
-               ? success()
-               : failure();
-  }
+    if (isa<UnrankedMemRefType>(srcType) && isa<UnrankedMemRefType>(dstType))
+      return failure();
 
-  void rewrite(memref::CastOp memRefCastOp, OpAdaptor adaptor,
-               ConversionPatternRewriter &rewriter) const override {
-    auto srcType = memRefCastOp.getOperand().getType();
-    auto dstType = memRefCastOp.getType();
     auto targetStructType = typeConverter->convertType(memRefCastOp.getType());
     auto loc = memRefCastOp.getLoc();
 
     // For ranked/ranked case, just keep the original descriptor.
-    if (isa<MemRefType>(srcType) && isa<MemRefType>(dstType))
-      return rewriter.replaceOp(memRefCastOp, {adaptor.getSource()});
+    if (isa<MemRefType>(srcType) && isa<MemRefType>(dstType)) {
+      rewriter.replaceOp(memRefCastOp, {adaptor.getSource()});
+      return success();
+    }
 
     if (isa<MemRefType>(srcType) && isa<UnrankedMemRefType>(dstType)) {
       // Casting ranked to unranked memref type
@@ -733,6 +726,8 @@ struct MemRefCastOpLowering
     } else {
       llvm_unreachable("Unsupported unranked memref to unranked memref cast");
     }
+
+    return success();
   }
 };
 
diff --git a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
index 5c84b4063da2e..9dea12355a519 100644
--- a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
+++ b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
@@ -65,6 +65,11 @@ class ApplyScaleGenericOpConverter
 
   LogicalResult matchAndRewrite(tosa::ApplyScaleOp op,
                                 PatternRewriter &rewriter) const final {
+    StringRef roundingMode = op.getRoundingMode();
+    if (roundingMode != "DOUBLE_ROUND" && roundingMode != "SINGLE_ROUND") {
+      return failure();
+    }
+
     Location loc = op.getLoc();
     Value value = op.getValue();
     Value multiplier32 = op.getMultiplier();
@@ -96,7 +101,7 @@ class ApplyScaleGenericOpConverter
     multiply64 = rewriter.create<arith::AddIOp>(loc, multiply64, round);
 
     // Apply double rounding if necessary.
-    if (op.getDoubleRound()) {
+    if (op.getRoundingMode() == "DOUBLE_ROUND") {
       int64_t roundInt = 1 << 30;
       Value roundUp = getConstantValue(loc, i64Ty, roundInt, rewriter);
       Value roundDown = getConstantValue(loc, i64Ty, -roundInt, rewriter);
@@ -125,6 +130,11 @@ class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
 
   LogicalResult matchAndRewrite(tosa::ApplyScaleOp op,
                                 PatternRewriter &rewriter) const final {
+    StringRef roundingMode = op.getRoundingMode();
+    if (roundingMode != "DOUBLE_ROUND" && roundingMode != "SINGLE_ROUND") {
+      return failure();
+    }
+
     Location loc = op.getLoc();
 
     Type resultTy = op.getType();
@@ -170,7 +180,7 @@ class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
         rewriter.create<arith::SelectOp>(loc, shiftOver32, shiftHighR, zero32);
 
     // Conditionally perform our double round.
-    if (op.getDoubleRound()) {
+    if (op.getRoundingMode() == "DOUBLE_ROUND") {
       Value negOne32 = getConstantValue(loc, i32Ty, -1, rewriter);
       Value valuePositive = rewriter.create<arith::CmpIOp>(
           loc, arith::CmpIPredicate::sge, value32, zero32);
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
index a99cf293b9eac..b5e0efa71b3cc 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
@@ -84,10 +84,9 @@ materializeBinaryNanCheckIfRequired(OpTy op, PatternRewriter &rewriter,
 
 template <typename T>
 static arith::ConstantOp
-createConstFromIntAttribute(Operation *op, const std::string &attrName,
-                            Type requiredAttrType, OpBuilder &rewriter) {
-  auto castedN = static_cast<T>(
-      cast<IntegerAttr>(op->getAttr(attrName)).getValue().getSExtValue());
+createConstOpFromZpVal(Operation *op, const int64_t &zp, Type requiredAttrType,
+                       OpBuilder &rewriter) {
+  auto castedN = static_cast<T>(zp);
   return rewriter.create<arith::ConstantOp>(
       op->getLoc(), IntegerAttr::get(requiredAttrType, castedN));
 }
@@ -138,7 +137,7 @@ static Value createLinalgBodyCalculationForElementwiseOp(
   // tosa::MulOp
   if (isa<tosa::MulOp>(op)) {
     auto shift_val = cast<tosa::MulOp>(op).getShift();
-    ElementsAttr shift_elem;
+    DenseElementsAttr shift_elem;
     if (!shift_val.getImpl() ||
         !matchPattern(shift_val, m_Constant(&shift_elem))) {
       (void)rewriter.notifyMatchFailure(op, "shift value of mul not found");
@@ -170,7 +169,7 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
         auto result = rewriter.create<tosa::ApplyScaleOp>(
             loc, rewriter.getI32Type(), a, b, shiftConst,
-            rewriter.getBoolAttr(false));
+            rewriter.getStringAttr("SINGLE_ROUND"));
 
         if (elementTy.isInteger(32))
           return result;
@@ -193,18 +192,29 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
   // tosa::NegateOp
   if (isa<tosa::NegateOp>(op)) {
-    if (isa<FloatType>(elementTy))
-      return rewriter.create<arith::NegFOp>(loc, resultTypes, args);
+    auto negate = cast<tosa::NegateOp>(op);
 
-    if (isa<IntegerType>(elementTy)) {
-      auto inputZpAttr = cast<tosa::NegateOp>(op).getInput1ZpAttr();
-      auto outputZpAttr = cast<tosa::NegateOp>(op).getOutputZpAttr();
+    FailureOr<int64_t> maybeInZp = negate.getInput1ZeroPoint();
+    if (failed(maybeInZp)) {
+      (void)rewriter.notifyMatchFailure(
+          op, "input1 zero point cannot be statically determined");
+      return nullptr;
+    }
+
+    FailureOr<int64_t> maybeOutZp = negate.getOutputZeroPoint();
+    if (failed(maybeOutZp)) {
+      (void)rewriter.notifyMatchFailure(
+          op, "output zero point cannot be statically determined");
+      return nullptr;
+    }
+
+    int64_t inZp = *maybeInZp;
+    int64_t outZp = *maybeOutZp;
 
-      const int64_t inZp =
-          inputZpAttr ? inputZpAttr.getValue().getSExtValue() : 0;
-      const int64_t outZp =
-          outputZpAttr ? outputZpAttr.getValue().getSExtValue() : 0;
+    if (isa<FloatType>(elementTy))
+      return rewriter.create<arith::NegFOp>(loc, resultTypes, args[0]);
 
+    if (isa<IntegerType>(elementTy)) {
       if (!inZp && !outZp) {
         auto constant = rewriter.create<arith::ConstantOp>(
             loc, IntegerAttr::get(elementTy, 0));
@@ -1374,7 +1384,11 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     unsigned rank = inputTy.getRank();
 
     // This is an illegal configuration. terminate and log an error
-    if (op.getDoubleRound() && !op.getScale32())
+    if (op.getRoundingMode() == "INEXACT_ROUND")
+      return rewriter.notifyMatchFailure(
+          op, "tosa.rescale with rounding mode = 'INEXACT_ROUND' is not "
+              "currently supported");
+    if (op.getRoundingMode() == "DOUBLE_ROUND" && !op.getScale32())
       return rewriter.notifyMatchFailure(
           op, "tosa.rescale requires scale32 for double_round to be true");
 
@@ -1389,8 +1403,24 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
     }
 
     // The shift and multiplier values.
-    SmallVector<int32_t> multiplierValues(op.getMultiplier());
-    SmallVector<int8_t> shiftValues(op.getShift());
+    DenseElementsAttr shiftElems;
+    if (!matchPattern(op.getShift(), m_Constant(&shiftElems)))
+      return rewriter.notifyMatchFailure(
+          op, "tosa.rescale requires constant shift input values");
+
+    DenseElementsAttr multiplierElems;
+    if (!matchPattern(op.getMultiplier(), m_Constant(&multiplierElems)))
+      return rewriter.notifyMatchFailure(
+          op, "tosa.rescale requires constant multiplier input values");
+
+    llvm::SmallVector<int8_t> shiftValues =
+        llvm::to_vector(shiftElems.getValues<int8_t>());
+    // explicit cast is required here
+    llvm::SmallVector<int32_t> multiplierValues = llvm::to_vector(
+        llvm::map_range(multiplierElems.getValues<IntegerAttr>(),
+                        [](IntegerAttr attr) -> int32_t {
+                          return static_cast<int32_t>(attr.getInt());
+                        }));
 
     // If we shift by more than the bitwidth, this just sets to 0.
     for (int i = 0, s = multiplierValues.size(); i < s; i++) {
@@ -1402,9 +1432,13 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
 
     // Double round only occurs if shift is greater than 31, check that this
     // is ever true.
+
     bool doubleRound =
-        op.getDoubleRound() &&
+        op.getRoundingMode() == "DOUBLE_ROUND" &&
         llvm::any_of(shiftValues, [](int32_t v) { return v > 31; });
+    StringAttr roundingMode = doubleRound
+                                  ? rewriter.getStringAttr("DOUBLE_ROUND")
+                                  : rewriter.getStringAttr("SINGLE_ROUND");
 
     SmallVector<AffineMap> indexingMaps = {
         rewriter.getMultiDimIdentityMap(rank)};
@@ -1475,11 +1509,26 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
           // later.
           int32_t inBitwidth = valueTy.getIntOrFloatBitWidth() > 32 ? 48 : 32;
 
-          auto inputZp = createConstFromIntAttribute<int32_t>(
-              op, "input_zp", nestedBuilder.getIntegerType(inBitwidth),
+          FailureOr<int64_t> maybeIZp = op.getInputZeroPoint();
+          if (failed(maybeIZp)) {
+            (void)rewriter.notifyMatchFailure(
+                op, "input zero point cannot be statically determined");
+            return;
+          }
+
+          auto inputZp = createConstOpFromZpVal<int32_t>(
+              op, *maybeIZp, nestedBuilder.getIntegerType(inBitwidth),
               nestedBuilder);
-          auto outputZp = createConstFromIntAttribute<int32_t>(
-              op, "output_zp", nestedBuilder.getI32Type(), nestedBuilder);
+
+          FailureOr<int64_t> maybeOZp = op.getOutputZeroPoint();
+          if (failed(maybeOZp)) {
+            (void)rewriter.notifyMatchFailure(
+                op, "output zero point cannot be statically determined");
+            return;
+          };
+
+          auto outputZp = createConstOpFromZpVal<int32_t>(
+              op, *maybeOZp, nestedBuilder.getI32Type(), nestedBuilder);
 
           Value multiplier = multiplierConstant ? multiplierConstant
                                                 : blockArgs[multiplierArg];
@@ -1500,7 +1549,7 @@ class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
 
           value = nestedBuilder.create<tosa::ApplyScaleOp>(
               loc, nestedBuilder.getI32Type(), value, multiplier, shift,
-              nestedBuilder.getBoolAttr(doubleRound));
+              roundingMode);
 
           // Move to the new zero-point.
           value =
@@ -2598,7 +2647,7 @@ struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
     }
 
     auto loc = rfft2d.getLoc();
-    auto input = rfft2d.getInput();
+    auto input = rfft2d.getInputReal();
     auto elementType =
         dyn_cast<FloatType>(cast<ShapedType>(input.getType()).getElementType());
     if (!elementType)
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
index 2a2589e19d0ac..fc1cad2423450 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
@@ -270,8 +270,8 @@ class ConvConverter : public OpConversionPattern<TosaConvOp> {
       return rewriter.notifyMatchFailure(
           op, "weight zero point cannot be statically determined");
 
-    int64_t inputZpVal = *maybeIZp;
-    int64_t weightZpVal = *maybeWZp;
+    const int64_t inputZpVal = *maybeIZp;
+    const int64_t weightZpVal = *maybeWZp;
 
     if (op.verifyInputZeroPoint(inputZpVal).failed())
       return rewriter.notifyMatchFailure(
@@ -466,8 +466,8 @@ class DepthwiseConvConverter
       return rewriter.notifyMatchFailure(
           op, "weight zero point cannot be statically determined");
 
-    int64_t inputZpVal = *maybeIZp;
-    int64_t weightZpVal = *maybeWZp;
+    const int64_t inputZpVal = *maybeIZp;
+    const int64_t weightZpVal = *maybeWZp;
 
     if (op.verifyInputZeroPoint(inputZpVal).failed())
       return rewriter.notifyMatchFailure(
@@ -477,13 +477,13 @@ class DepthwiseConvConverter
       return rewriter.notifyMatchFailure(
           op, "weight zero point must be zero for non-int8 integer types");
 
-    bool hasZp = (inputZpVal != 0) || (weightZpVal != 0);
+    bool hasNullZps = (inputZpVal == 0) && (weightZpVal == 0);
     auto weightShape = weightTy.getShape();
     auto resultShape = resultTy.getShape();
 
     // Apply padding as necessary.
     TypedAttr zeroAttr = rewriter.getZeroAttr(inputETy);
-    if (hasZp) {
+    if (!hasNullZps) {
       int64_t intMin =
           APInt::getSignedMinValue(inputETy.getIntOrFloatBitWidth())
               .getSExtValue();
@@ -536,7 +536,7 @@ class DepthwiseConvConverter
     indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultRank));
     indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultRank));
 
-    if (!hasZp) {
+    if (hasNullZps) {
       Value conv = rewriter
                        .create<linalg::DepthwiseConv2DNhwcHwcmOp>(
                            loc, linalgConvTy, ValueRange{input, weight},
@@ -556,8 +556,13 @@ class DepthwiseConvConverter
                   getNParallelLoopsAttrs(resultRank),
                   [&](OpBuilder &nestedBuilder, Location nestedLoc,
                       ValueRange args) {
-                    Value added = nestedBuilder.create<arith::AddFOp>(
-                        loc, args[0], args[1]);
+                    Value added;
+                    if (llvm::isa<FloatType>(inputETy))
+                      added = nestedBuilder.create<arith::AddFOp>(loc, args[0],
+                                                                  args[1]);
+                    else
+                      added = nestedBuilder.create<arith::AddIOp>(loc, args[0],
+                                                                  args[1]);
                     nestedBuilder.create<linalg::YieldOp>(nestedLoc, added);
                   })
               .getResult(0);
@@ -621,15 +626,38 @@ class MatMulConverter : public OpConversionPattern<tosa::MatMulOp> {
                            .create<linalg::FillOp>(loc, ValueRange{zero},
                                                    ValueRange{emptyTensor})
                            .result();
-    if (!op.getAZp() && !op.getBZp()) {
+
+    FailureOr<int64_t> maybeAZp = op.getAZeroPoint();
+    FailureOr<int64_t> maybeBZp = op.getBZeroPoint();
+    if (failed(maybeAZp))
+      return rewriter.notifyMatchFailure(
+          op, "input a zero point cannot be statically determined");
+    if (failed(maybeBZp))
+      return rewriter.notifyMatchFailure(
+          op, "input b zero point cannot be statically determined");
+
+    const int64_t aZpVal = *maybeAZp;
+    const int64_t bZpVal = *maybeBZp;
+
+    if (op.verifyAZeroPoint(aZpVal).failed())
+      return rewriter.notifyMatchFailure(
+          op, "input a zero point must be zero for non-int8 integer types");
+
+    if (op.verifyBZeroPoint(bZpVal).failed())
+      return rewriter.notifyMatchFailure(
+          op, "input b zero point must be zero for non-int8 integer types");
+
+    if (aZpVal == 0 && bZpVal == 0) {
       rewriter.replaceOpWithNewOp<linalg::BatchMatmulOp>(
           op, TypeRange{op.getType()},
           ValueRange{adaptor.getA(), adaptor.getB()}, ValueRange{zeroTensor});
       return success();
     }
 
-    auto aZp = rewriter.create<arith::ConstantOp>(loc, op.getAZpAttr());
-    auto bZp = rewriter.create<arith::ConstantOp>(loc, op.getBZpAttr());
+    auto aZp = rewriter.create<arith::ConstantOp>(
+        loc, rewriter.getI32IntegerAttr(aZpVal));
+    auto bZp = rewriter.create<arith::ConstantOp>(
+        loc, rewriter.getI32IntegerAttr(bZpVal));
     rewriter.replaceOpWithNewOp<linalg::QuantizedBatchMatmulOp>(
         op, TypeRange{op.getType()},
         ValueRange{adaptor.getA(), adaptor.getB(), aZp, bZp}, zeroTensor);
@@ -834,8 +862,8 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
       return rewriter.notifyMatchFailure(
           op, "output zero point could not be statically determined");
 
-    int64_t inputZpVal = *maybeIZp;
-    int64_t outputZpVal = *maybeOZp;
+    const int64_t inputZpVal = *maybeIZp;
+    const int64_t outputZpVal = *maybeOZp;
 
     // Apply padding as necessary.
     llvm::SmallVector<int64_t> pad;
@@ -1003,9 +1031,9 @@ class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
 
             auto scaled =
                 rewriter
-                    .create<tosa::ApplyScaleOp>(loc, rewriter.getI32Type(),
-                                                poolVal, multiplier, shift,
-                                                rewriter.getBoolAttr(false))
+                    .create<tosa::ApplyScaleOp>(
+                        loc, rewriter.getI32Type(), poolVal, multiplier, shift,
+                        rewriter.getStringAttr("SINGLE_ROUND"))
                     .getResult();
 
             // If we have quantization information we need to apply output
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
index e3a81bd20212d..eb1555df5d574 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
@@ -69,11 +69,11 @@ void ConvertVectorToLLVMPass::runOnOperation() {
     populateVectorToVectorCanonicalizationPatterns(patterns);
     populateVectorBitCastLoweringPatterns(patterns);
     populateVectorBroadcastLoweringPatterns(patterns);
-    populateVectorContractLoweringPatterns(patterns, vectorTransformsOptions);
+    populateVectorContractLoweringPatterns(patterns, vectorContractLowering);
     populateVectorMaskOpLoweringPatterns(patterns);
     populateVectorShapeCastLoweringPatterns(patterns);
     populateVectorInterleaveLoweringPatterns(patterns);
-    populateVectorTransposeLoweringPatterns(patterns, vectorTransformsOptions);
+    populateVectorTransposeLoweringPatterns(patterns, vectorTransposeLowering);
     // Vector transfer ops with rank > 1 should be lowered with VectorToSCF.
     populateVectorTransferLoweringPatterns(patterns, /*maxTransferRank=*/1);
     populateVectorMaskMaterializationPatterns(patterns,
diff --git a/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp b/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp
index 9b776900c379a..84b76d33c3e67 100644
--- a/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp
+++ b/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp
@@ -626,7 +626,8 @@ DependenceResult mlir::affine::checkMemrefAccessDependence(
 
   // We can't analyze further if the ops lie in different affine scopes or have
   // no common block in an affine scope.
-  if (getAffineScope(srcAccess.opInst) != getAffineScope(dstAccess.opInst))
+  if (getAffineAnalysisScope(srcAccess.opInst) !=
+      getAffineAnalysisScope(dstAccess.opInst))
     return DependenceResult::Failure;
   if (!getCommonBlockInAffineScope(srcAccess.opInst, dstAccess.opInst))
     return DependenceResult::Failure;
diff --git a/mlir/lib/Dialect/Affine/Analysis/Utils.cpp b/mlir/lib/Dialect/Affine/Analysis/Utils.cpp
index 56a7d8de1b7f8..86aba7b187535 100644
--- a/mlir/lib/Dialect/Affine/Analysis/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Analysis/Utils.cpp
@@ -1988,6 +1988,8 @@ unsigned mlir::affine::getNestingDepth(Operation *op) {
   while ((currOp = currOp->getParentOp())) {
     if (isa<AffineForOp>(currOp))
       depth++;
+    if (auto parOp = dyn_cast<AffineParallelOp>(currOp))
+      depth += parOp.getNumDims();
   }
   return depth;
 }
@@ -2307,8 +2309,8 @@ FailureOr<AffineValueMap> mlir::affine::simplifyConstrainedMinMaxOp(
 
 Block *mlir::affine::findInnermostCommonBlockInScope(Operation *a,
                                                      Operation *b) {
-  Region *aScope = mlir::affine::getAffineScope(a);
-  Region *bScope = mlir::affine::getAffineScope(b);
+  Region *aScope = getAffineAnalysisScope(a);
+  Region *bScope = getAffineAnalysisScope(b);
   if (aScope != bScope)
     return nullptr;
 
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index 06493de2b09d4..8acb21d5074b4 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -270,6 +270,16 @@ Region *mlir::affine::getAffineScope(Operation *op) {
   return nullptr;
 }
 
+Region *mlir::affine::getAffineAnalysisScope(Operation *op) {
+  Operation *curOp = op;
+  while (auto *parentOp = curOp->getParentOp()) {
+    if (!isa<AffineForOp, AffineIfOp, AffineParallelOp>(parentOp))
+      return curOp->getParentRegion();
+    curOp = parentOp;
+  }
+  return nullptr;
+}
+
 // A Value can be used as a dimension id iff it meets one of the following
 // conditions:
 // *) It is valid as a symbol.
@@ -2304,6 +2314,10 @@ struct AffineForEmptyLoopFolder : public OpRewritePattern<AffineForOp> {
     for (unsigned i = 0, e = yieldOp->getNumOperands(); i < e; ++i) {
       Value val = yieldOp.getOperand(i);
       auto *iterArgIt = llvm::find(iterArgs, val);
+      // TODO: It should be possible to perform a replacement by computing the
+      // last value of the IV based on the bounds and the step.
+      if (val == forOp.getInductionVar())
+        return failure();
       if (iterArgIt == iterArgs.end()) {
         // `val` is defined outside of the loop.
         assert(forOp.isDefinedOutsideOfLoop(val) &&
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
index 21cd0e2e49c2a..bcba17bb21544 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
@@ -328,7 +328,9 @@ static std::optional<double> getAdditionalComputeFraction(
 // Creates and returns a private (single-user) memref for fused loop rooted at
 // 'forOp', with (potentially reduced) memref size based on the memref region
 // written to by `storeOps` at depth 'dstLoopDepth'. 'sliceInsertionBlock'
-// specifies the block in which the slice was/will be inserted.
+// specifies the block in which the slice was/will be inserted. The method
+// expects that all stores ops to the memref have the same access function.
+// Returns nullptr if the creation failed.
 static Value createPrivateMemRef(AffineForOp forOp,
                                  ArrayRef<Operation *> storeOps,
                                  unsigned dstLoopDepth,
@@ -336,6 +338,24 @@ static Value createPrivateMemRef(AffineForOp forOp,
                                  Block *sliceInsertionBlock,
                                  uint64_t localBufSizeThreshold) {
   assert(!storeOps.empty() && "no source stores supplied");
+
+  // Check if all stores have the same access function; we only support this
+  // case.
+  // TODO: Use union of memref write regions to compute private memref footprint
+  // for store ops with different access functions.
+  if (storeOps.size() > 1 &&
+      !std::equal(std::next(storeOps.begin()), storeOps.end(), storeOps.begin(),
+                  [](Operation *a, Operation *b) {
+                    MemRefAccess aM(cast<AffineWriteOpInterface>(a));
+                    MemRefAccess bM(cast<AffineWriteOpInterface>(b));
+                    return aM == bM;
+                  })) {
+    LLVM_DEBUG(llvm::dbgs()
+               << "Private memref creation unsupported for multiple producer "
+                  "stores with different access functions.\n");
+    return nullptr;
+  }
+
   Operation *srcStoreOp = storeOps[0];
 
   // Create builder to insert alloc op just before 'forOp'.
@@ -432,6 +452,8 @@ static Value createPrivateMemRef(AffineForOp forOp,
   assert(succeeded(res) &&
          "replaceAllMemrefUsesWith should always succeed here");
   (void)res;
+  LLVM_DEBUG(llvm::dbgs() << "Created private memref of type: " << newMemRefType
+                          << '\n');
   return newMemRef;
 }
 
@@ -1123,13 +1145,12 @@ struct GreedyFusion {
           // loads and stores. Any reference to the original ones becomes
           // invalid after this point.
           for (auto &memrefToStoresPair : privateMemRefToStores) {
-            // TODO: Use union of memref write regions to compute
-            // private memref footprint.
-            SmallVector<Operation *, 4> &storesForMemref =
-                memrefToStoresPair.second;
+            ArrayRef<Operation *> storesForMemref = memrefToStoresPair.second;
             Value newMemRef = createPrivateMemRef(
                 dstAffineForOp, storesForMemref, bestDstLoopDepth,
                 fastMemorySpace, sliceInsertionBlock, localBufSizeThreshold);
+            if (!newMemRef)
+              continue;
             // Create new node in dependence graph for 'newMemRef' alloc op.
             unsigned newMemRefNodeId = mdg->addNode(newMemRef.getDefiningOp());
             // Add edge from 'newMemRef' node to dstNode.
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
index c8400dfe8cd5c..45bda68bb8639 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
@@ -110,7 +110,7 @@ void LoopTiling::getTileSizes(ArrayRef<AffineForOp> band,
     return;
   }
 
-  // Use tileSizes and fill them with default tile size if it's short.
+  // Use supplied tile sizes and fill them with default tile size if it's short.
   if (!this->tileSizes.empty()) {
     tileSizes->assign(this->tileSizes.begin(), this->tileSizes.end());
     tileSizes->resize(band.size(), kDefaultTileSize);
@@ -118,6 +118,13 @@ void LoopTiling::getTileSizes(ArrayRef<AffineForOp> band,
   }
   tileSizes->resize(band.size());
 
+  // If the cache size is zero, set the minimum valid tile size. No good reason
+  // to pick another specific size over this.
+  if (cacheSizeInKiB == 0) {
+    std::fill(tileSizes->begin(), tileSizes->end(), 1);
+    return;
+  }
+
   // The first loop in the band.
   AffineForOp rootForOp = band[0];
   (void)rootForOp;
diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
index 7ef016f88be37..2723cff6900d0 100644
--- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
@@ -263,16 +263,17 @@ static void promoteIfBlock(AffineIfOp ifOp, bool elseBlock) {
 static Operation *getOutermostInvariantForOp(AffineIfOp ifOp) {
   // Walk up the parents past all for op that this conditional is invariant on.
   auto ifOperands = ifOp.getOperands();
-  auto *res = ifOp.getOperation();
-  while (!isa<func::FuncOp>(res->getParentOp())) {
+  Operation *res = ifOp;
+  while (!res->getParentOp()->hasTrait<OpTrait::IsIsolatedFromAbove>()) {
     auto *parentOp = res->getParentOp();
     if (auto forOp = dyn_cast<AffineForOp>(parentOp)) {
       if (llvm::is_contained(ifOperands, forOp.getInductionVar()))
         break;
     } else if (auto parallelOp = dyn_cast<AffineParallelOp>(parentOp)) {
-      for (auto iv : parallelOp.getIVs())
-        if (llvm::is_contained(ifOperands, iv))
-          break;
+      if (llvm::any_of(parallelOp.getIVs(), [&](Value iv) {
+            return llvm::is_contained(ifOperands, iv);
+          }))
+        break;
     } else if (!isa<AffineIfOp>(parentOp)) {
       // Won't walk up past anything other than affine.for/if ops.
       break;
@@ -438,11 +439,10 @@ LogicalResult mlir::affine::hoistAffineIfOp(AffineIfOp ifOp, bool *folded) {
   if (folded)
     *folded = false;
 
-  // The folding above should have ensured this, but the affine.if's
-  // canonicalization is missing composition of affine.applys into it.
+  // The folding above should have ensured this.
   assert(llvm::all_of(ifOp.getOperands(),
                       [](Value v) {
-                        return isTopLevelValue(v) || isAffineForInductionVar(v);
+                        return isTopLevelValue(v) || isAffineInductionVar(v);
                       }) &&
          "operands not composed");
 
@@ -636,7 +636,8 @@ static bool mustReachAtInnermost(const MemRefAccess &srcAccess,
                                  const MemRefAccess &destAccess) {
   // Affine dependence analysis is possible only if both ops in the same
   // AffineScope.
-  if (getAffineScope(srcAccess.opInst) != getAffineScope(destAccess.opInst))
+  if (getAffineAnalysisScope(srcAccess.opInst) !=
+      getAffineAnalysisScope(destAccess.opInst))
     return false;
 
   unsigned nsLoops =
@@ -659,9 +660,9 @@ static bool mayHaveEffect(Operation *srcMemOp, Operation *destMemOp,
   // AffineScope. Also, we can only check if our affine scope is isolated from
   // above; otherwise, values can from outside of the affine scope that the
   // check below cannot analyze.
-  Region *srcScope = getAffineScope(srcMemOp);
+  Region *srcScope = getAffineAnalysisScope(srcMemOp);
   if (srcAccess.memref == destAccess.memref &&
-      srcScope == getAffineScope(destMemOp)) {
+      srcScope == getAffineAnalysisScope(destMemOp)) {
     unsigned nsLoops = getNumCommonSurroundingLoops(*srcMemOp, *destMemOp);
     FlatAffineValueConstraints dependenceConstraints;
     for (unsigned d = nsLoops + 1; d > minSurroundingLoops; d--) {
diff --git a/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp b/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp
index f105534626082..62022bfb7df1e 100644
--- a/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp
@@ -40,36 +40,33 @@ struct EmulateUnsupportedFloatsPass
   void runOnOperation() override;
 };
 
-struct EmulateFloatPattern final : ConversionPattern::SplitMatchAndRewrite {
+struct EmulateFloatPattern final : ConversionPattern {
   EmulateFloatPattern(const TypeConverter &converter, MLIRContext *ctx)
-      : ConversionPattern::SplitMatchAndRewrite(
+      : ConversionPattern::ConversionPattern(
             converter, Pattern::MatchAnyOpTypeTag(), 1, ctx) {}
 
-  LogicalResult match(Operation *op) const override;
-  void rewrite(Operation *op, ArrayRef<Value> operands,
-               ConversionPatternRewriter &rewriter) const override;
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override;
 };
 } // end namespace
 
-LogicalResult EmulateFloatPattern::match(Operation *op) const {
+LogicalResult EmulateFloatPattern::matchAndRewrite(
+    Operation *op, ArrayRef<Value> operands,
+    ConversionPatternRewriter &rewriter) const {
   if (getTypeConverter()->isLegal(op))
     return failure();
   // The rewrite doesn't handle cloning regions.
   if (op->getNumRegions() != 0)
     return failure();
-  return success();
-}
 
-void EmulateFloatPattern::rewrite(Operation *op, ArrayRef<Value> operands,
-                                  ConversionPatternRewriter &rewriter) const {
   Location loc = op->getLoc();
   const TypeConverter *converter = getTypeConverter();
   SmallVector<Type> resultTypes;
   if (failed(converter->convertTypes(op->getResultTypes(), resultTypes))) {
     // Note to anyone looking for this error message: this is a "can't happen".
     // If you're seeing it, there's a bug.
-    op->emitOpError("type conversion failed in float emulation");
-    return;
+    return op->emitOpError("type conversion failed in float emulation");
   }
   Operation *expandedOp =
       rewriter.create(loc, op->getName().getIdentifier(), operands, resultTypes,
@@ -84,6 +81,7 @@ void EmulateFloatPattern::rewrite(Operation *op, ArrayRef<Value> operands,
     }
   }
   rewriter.replaceOp(op, newResults);
+  return success();
 }
 
 void mlir::arith::populateEmulateUnsupportedFloatsConversions(
diff --git a/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp b/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
index 6da28ddeede3c..f866c91ef6e39 100644
--- a/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/IntRangeOptimizations.cpp
@@ -115,14 +115,14 @@ class DataFlowListener : public RewriterBase::Listener {
 /// and replace their uses with that constant. Return success() if all results
 /// where thus replaced and the operation is erased. Also replace any block
 /// arguments with their constant values.
-struct MaterializeKnownConstantValues
-    : public RewritePattern::SplitMatchAndRewrite {
+struct MaterializeKnownConstantValues : public RewritePattern {
   MaterializeKnownConstantValues(MLIRContext *context, DataFlowSolver &s)
-      : RewritePattern::SplitMatchAndRewrite(Pattern::MatchAnyOpTypeTag(),
-                                             /*benefit=*/1, context),
+      : RewritePattern::RewritePattern(Pattern::MatchAnyOpTypeTag(),
+                                       /*benefit=*/1, context),
         solver(s) {}
 
-  LogicalResult match(Operation *op) const override {
+  LogicalResult matchAndRewrite(Operation *op,
+                                PatternRewriter &rewriter) const override {
     if (matchPattern(op, m_Constant()))
       return failure();
 
@@ -131,7 +131,8 @@ struct MaterializeKnownConstantValues
     };
     bool hasConstantResults = llvm::any_of(op->getResults(), needsReplacing);
     if (op->getNumRegions() == 0)
-      return success(hasConstantResults);
+      if (!hasConstantResults)
+        return failure();
     bool hasConstantRegionArgs = false;
     for (Region &region : op->getRegions()) {
       for (Block &block : region.getBlocks()) {
@@ -139,10 +140,9 @@ struct MaterializeKnownConstantValues
             llvm::any_of(block.getArguments(), needsReplacing);
       }
     }
-    return success(hasConstantResults || hasConstantRegionArgs);
-  }
+    if (!hasConstantResults && !hasConstantRegionArgs)
+      return failure();
 
-  void rewrite(Operation *op, PatternRewriter &rewriter) const override {
     bool replacedAll = (op->getNumResults() != 0);
     for (Value v : op->getResults())
       replacedAll &=
@@ -150,7 +150,7 @@ struct MaterializeKnownConstantValues
            v.use_empty());
     if (replacedAll && isOpTriviallyDead(op)) {
       rewriter.eraseOp(op);
-      return;
+      return success();
     }
 
     PatternRewriter::InsertionGuard guard(rewriter);
@@ -162,6 +162,8 @@ struct MaterializeKnownConstantValues
         }
       }
     }
+
+    return success();
   }
 
 private:
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
index 97d4aab9f3dd5..c0e3fca428376 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
@@ -142,7 +142,7 @@ struct OneShotBufferizePass
       // Configure op filter.
       OpFilter::Entry::FilterFn filterFn = [&](Operation *op) {
         // Filter may be specified via options.
-        if (this->dialectFilter.hasValue())
+        if (this->dialectFilter.hasValue() && !(*this->dialectFilter).empty())
           return llvm::is_contained(this->dialectFilter,
                                     op->getDialect()->getNamespace());
         // No filter specified: All other ops are allowed.
diff --git a/mlir/lib/Dialect/DLTI/DLTI.cpp b/mlir/lib/Dialect/DLTI/DLTI.cpp
index 92efecc62acd5..206d9f43a44a4 100644
--- a/mlir/lib/Dialect/DLTI/DLTI.cpp
+++ b/mlir/lib/Dialect/DLTI/DLTI.cpp
@@ -305,25 +305,8 @@ combineOneSpec(DataLayoutSpecInterface spec,
   DenseMap<StringAttr, DataLayoutEntryInterface> newEntriesForID;
   spec.bucketEntriesByType(newEntriesForType, newEntriesForID);
 
-  // Try overwriting the old entries with the new ones.
-  for (auto &kvp : newEntriesForType) {
-    if (!entriesForType.count(kvp.first)) {
-      entriesForType[kvp.first] = std::move(kvp.second);
-      continue;
-    }
-
-    Type typeSample = cast<Type>(kvp.second.front().getKey());
-    assert(&typeSample.getDialect() !=
-               typeSample.getContext()->getLoadedDialect<BuiltinDialect>() &&
-           "unexpected data layout entry for built-in type");
-
-    auto interface = cast<DataLayoutTypeInterface>(typeSample);
-    if (!interface.areCompatible(entriesForType.lookup(kvp.first), kvp.second))
-      return failure();
-
-    overwriteDuplicateEntries(entriesForType[kvp.first], kvp.second);
-  }
-
+  // Combine non-Type DL entries first so they are visible to the
+  // `type.areCompatible` method, allowing to query global properties.
   for (const auto &kvp : newEntriesForID) {
     StringAttr id = cast<StringAttr>(kvp.second.getKey());
     Dialect *dialect = id.getReferencedDialect();
@@ -332,7 +315,7 @@ combineOneSpec(DataLayoutSpecInterface spec,
       continue;
     }
 
-    // Attempt to combine the enties using the dialect interface. If the
+    // Attempt to combine the entries using the dialect interface. If the
     // dialect is not loaded for some reason, use the default combinator
     // that conservatively accepts identical entries only.
     entriesForID[id] =
@@ -344,6 +327,28 @@ combineOneSpec(DataLayoutSpecInterface spec,
       return failure();
   }
 
+  // Try overwriting the old entries with the new ones.
+  for (auto &kvp : newEntriesForType) {
+    if (!entriesForType.count(kvp.first)) {
+      entriesForType[kvp.first] = std::move(kvp.second);
+      continue;
+    }
+
+    Type typeSample = cast<Type>(kvp.second.front().getKey());
+    assert(&typeSample.getDialect() !=
+               typeSample.getContext()->getLoadedDialect<BuiltinDialect>() &&
+           "unexpected data layout entry for built-in type");
+
+    auto interface = cast<DataLayoutTypeInterface>(typeSample);
+    // TODO: Revisit this method and call once
+    // https://github.com/llvm/llvm-project/issues/130321 gets resolved.
+    if (!interface.areCompatible(entriesForType.lookup(kvp.first), kvp.second,
+                                 spec, entriesForID))
+      return failure();
+
+    overwriteDuplicateEntries(entriesForType[kvp.first], kvp.second);
+  }
+
   return success();
 }
 
@@ -379,6 +384,12 @@ DataLayoutSpecAttr::getEndiannessIdentifier(MLIRContext *context) const {
   return Builder(context).getStringAttr(DLTIDialect::kDataLayoutEndiannessKey);
 }
 
+StringAttr DataLayoutSpecAttr::getDefaultMemorySpaceIdentifier(
+    MLIRContext *context) const {
+  return Builder(context).getStringAttr(
+      DLTIDialect::kDataLayoutDefaultMemorySpaceKey);
+}
+
 StringAttr
 DataLayoutSpecAttr::getAllocaMemorySpaceIdentifier(MLIRContext *context) const {
   return Builder(context).getStringAttr(
@@ -609,7 +620,8 @@ class TargetDataLayoutInterface : public DataLayoutDialectInterface {
                             << DLTIDialect::kDataLayoutEndiannessBig << "' or '"
                             << DLTIDialect::kDataLayoutEndiannessLittle << "'";
     }
-    if (entryName == DLTIDialect::kDataLayoutAllocaMemorySpaceKey ||
+    if (entryName == DLTIDialect::kDataLayoutDefaultMemorySpaceKey ||
+        entryName == DLTIDialect::kDataLayoutAllocaMemorySpaceKey ||
         entryName == DLTIDialect::kDataLayoutProgramMemorySpaceKey ||
         entryName == DLTIDialect::kDataLayoutGlobalMemorySpaceKey ||
         entryName == DLTIDialect::kDataLayoutStackAlignmentKey ||
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
index 453b206de294e..8f39ede721c92 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp
@@ -349,8 +349,10 @@ LLVMPointerType::getIndexBitwidth(const DataLayout &dataLayout,
   return dataLayout.getTypeIndexBitwidth(get(getContext()));
 }
 
-bool LLVMPointerType::areCompatible(DataLayoutEntryListRef oldLayout,
-                                    DataLayoutEntryListRef newLayout) const {
+bool LLVMPointerType::areCompatible(
+    DataLayoutEntryListRef oldLayout, DataLayoutEntryListRef newLayout,
+    DataLayoutSpecInterface newSpec,
+    const DataLayoutIdentifiedEntryMap &map) const {
   for (DataLayoutEntryInterface newEntry : newLayout) {
     if (!newEntry.isTypeEntry())
       continue;
@@ -597,8 +599,10 @@ static uint64_t extractStructSpecValue(Attribute attr, StructDLEntryPos pos) {
       .getValues<uint64_t>()[static_cast<size_t>(pos)];
 }
 
-bool LLVMStructType::areCompatible(DataLayoutEntryListRef oldLayout,
-                                   DataLayoutEntryListRef newLayout) const {
+bool LLVMStructType::areCompatible(
+    DataLayoutEntryListRef oldLayout, DataLayoutEntryListRef newLayout,
+    DataLayoutSpecInterface newSpec,
+    const DataLayoutIdentifiedEntryMap &map) const {
   for (DataLayoutEntryInterface newEntry : newLayout) {
     if (!newEntry.isTypeEntry())
       continue;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
index a45b5c43f5d33..33667e7ab0c5c 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
@@ -566,7 +566,6 @@ static bool isFusableWithReshapeByDimExpansion(LinalgOp linalgOp,
   // - All the indexing maps for operands and results are projected
   //   permutations.
   // - The fused tensor is not a scalar.
-  // - All the loops for the reshaped operand are parallel loops.
   SmallVector<utils::IteratorType> iteratorTypes =
       linalgOp.getIteratorTypesArray();
   AffineMap operandMap = linalgOp.getMatchingIndexingMap(fusableOpOperand);
@@ -577,11 +576,7 @@ static bool isFusableWithReshapeByDimExpansion(LinalgOp linalgOp,
                             .getValue()
                             .isProjectedPermutation();
                       }) &&
-         operandMap.getNumResults() > 0 &&
-         llvm::all_of(operandMap.getResults(), [&](AffineExpr expr) {
-           return isParallelIterator(
-               iteratorTypes[cast<AffineDimExpr>(expr).getPosition()]);
-         });
+         operandMap.getNumResults() > 0;
 }
 
 namespace {
@@ -816,19 +811,35 @@ validateDynamicDimExpansion(LinalgOp linalgOp,
 }
 
 // Create an expanded transpose op.
-static Operation *
-createExpandedTransposeOp(PatternRewriter &rewriter, TransposeOp transposeOp,
-                          SmallVector<ReassociationIndices> reassociation,
-                          Value expandedInput, Value output) {
-  applyPermutationToVector(reassociation, transposeOp.getPermutation());
+// the reassociation map is already permuted hence we inverse permute and then
+// flatten it. Then we inverse permute it again to get the final expanded
+// transpose permutation. For example,
+//
+// permutation = [2, 0, 1]
+// reassociation_map for expansion = [[0, 1], [2], [3, 4, 5]]
+//
+// inverse permutation = [1, 2, 0]
+// applied to reassocation_map and then flattened becomes
+// flatened permutation = [2, 3, 4, 5, 0, 1]
+// final permuation is the inverse of the flattened permutation.
+//
+// Becomes
+//
+// permutation=[4, 5, 0, 1, 2, 3]
+
+static Operation *createExpandedTransposeOp(PatternRewriter &rewriter,
+                                            TransposeOp transposeOp,
+                                            Value expandedInput, Value output,
+                                            ExpansionInfo &expansionInfo) {
   SmallVector<int64_t> newPerm;
-  for (auto reassoc : reassociation) {
-    for (auto dim : reassoc) {
+  for (int64_t perm : invertPermutationVector(transposeOp.getPermutation())) {
+    auto reassoc = expansionInfo.getExpandedDims(perm);
+    for (int64_t dim : reassoc) {
       newPerm.push_back(dim);
     }
   }
   return rewriter.create<TransposeOp>(transposeOp.getLoc(), expandedInput,
-                                      output, newPerm);
+                                      output, invertPermutationVector(newPerm));
 }
 
 // Create an expanded generic op.
@@ -862,16 +873,18 @@ static Operation *createExpandedGenericOp(
 // Create an expanded fused op that retains the name for certain ops
 // such as fill, copy and transpose and produce a generic op for
 // rest of linalg ops.
-static Operation *createExpandedOp(
-    PatternRewriter &rewriter, LinalgOp linalgOp, TypeRange resultTypes,
-    ArrayRef<Value> expandedOpOperands, ArrayRef<Value> outputs,
-    ArrayRef<AffineMap> expandedOpIndexingMaps, ExpansionInfo &expansionInfo,
-    SmallVector<ReassociationIndices> reassociation) {
+static Operation *createExpandedOp(PatternRewriter &rewriter, LinalgOp linalgOp,
+                                   TypeRange resultTypes,
+                                   ArrayRef<Value> expandedOpOperands,
+                                   ArrayRef<Value> outputs,
+                                   ArrayRef<AffineMap> expandedOpIndexingMaps,
+                                   ExpansionInfo &expansionInfo) {
 
   return TypeSwitch<Operation *, Operation *>(linalgOp.getOperation())
       .Case<TransposeOp>([&](TransposeOp transposeOp) {
-        return createExpandedTransposeOp(rewriter, transposeOp, reassociation,
-                                         expandedOpOperands[0], outputs[0]);
+        return createExpandedTransposeOp(rewriter, transposeOp,
+                                         expandedOpOperands[0], outputs[0],
+                                         expansionInfo);
       })
       .Case<FillOp, CopyOp>([&](Operation *op) {
         return clone(rewriter, linalgOp, resultTypes,
@@ -991,12 +1004,9 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, Operation *reshapeOp,
   }
 
   TypeRange resultTypes = ValueRange(outputs).getTypes();
-  SmallVector<ReassociationIndices> reassociationBeforeExpansion =
-      isExpanding ? expandingReshapeOp.getReassociationIndices()
-                  : collapsingReshapeOp.getReassociationIndices();
-  Operation *fusedOp = createExpandedOp(
-      rewriter, linalgOp, resultTypes, expandedOpOperands, outputs,
-      expandedOpIndexingMaps, expansionInfo, reassociationBeforeExpansion);
+  Operation *fusedOp =
+      createExpandedOp(rewriter, linalgOp, resultTypes, expandedOpOperands,
+                       outputs, expandedOpIndexingMaps, expansionInfo);
   // Reshape the result values to their original shape if this is a collapsing
   // reshape folded into its consumer.
   SmallVector<Value> resultVals;
diff --git a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
index 167eebd786dba..a26e380232a91 100644
--- a/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
+++ b/mlir/lib/Dialect/Math/Transforms/PolynomialApproximation.cpp
@@ -1776,90 +1776,93 @@ void mlir::populatePolynomialApproximateErfcPattern(
 template <typename OpType>
 static void
 populateMathF32ExpansionPattern(RewritePatternSet &patterns,
-                                llvm::function_ref<bool(StringRef)> predicate) {
+                                llvm::function_ref<bool(StringRef)> predicate,
+                                PatternBenefit benefit) {
   if (predicate(OpType::getOperationName())) {
-    patterns.add<ReuseF32Expansion<OpType>>(patterns.getContext());
+    patterns.add<ReuseF32Expansion<OpType>>(patterns.getContext(), benefit);
   }
 }
 
 void mlir::populateMathF32ExpansionPatterns(
-    RewritePatternSet &patterns,
-    llvm::function_ref<bool(StringRef)> predicate) {
-  populateMathF32ExpansionPattern<math::AcosOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::AcoshOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::AsinOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::AsinhOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::AtanOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::Atan2Op>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::AtanhOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::CbrtOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::CosOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::CoshOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::ErfOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::ErfcOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::ExpOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::Exp2Op>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::ExpM1Op>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::LogOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::Log10Op>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::Log1pOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::Log2Op>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::PowFOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::RsqrtOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::SinOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::SinhOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::SqrtOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::TanOp>(patterns, predicate);
-  populateMathF32ExpansionPattern<math::TanhOp>(patterns, predicate);
+    RewritePatternSet &patterns, llvm::function_ref<bool(StringRef)> predicate,
+    PatternBenefit benefit) {
+  populateMathF32ExpansionPattern<math::AcosOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::AcoshOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::AsinOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::AsinhOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::AtanOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::Atan2Op>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::AtanhOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::CbrtOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::CosOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::CoshOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::ErfOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::ErfcOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::ExpOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::Exp2Op>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::ExpM1Op>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::LogOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::Log10Op>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::Log1pOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::Log2Op>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::PowFOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::RsqrtOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::SinOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::SinhOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::SqrtOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::TanOp>(patterns, predicate, benefit);
+  populateMathF32ExpansionPattern<math::TanhOp>(patterns, predicate, benefit);
 }
 
 template <typename OpType, typename PatternType>
 static void populateMathPolynomialApproximationPattern(
-    RewritePatternSet &patterns,
-    llvm::function_ref<bool(StringRef)> predicate) {
+    RewritePatternSet &patterns, llvm::function_ref<bool(StringRef)> predicate,
+    PatternBenefit benefit) {
   if (predicate(OpType::getOperationName())) {
-    patterns.add<PatternType>(patterns.getContext());
+    patterns.add<PatternType>(patterns.getContext(), benefit);
   }
 }
 
 void mlir::populateMathPolynomialApproximationPatterns(
-    RewritePatternSet &patterns,
-    llvm::function_ref<bool(StringRef)> predicate) {
+    RewritePatternSet &patterns, llvm::function_ref<bool(StringRef)> predicate,
+    PatternBenefit benefit) {
   populateMathPolynomialApproximationPattern<AcosOp,
                                              AcosPolynomialApproximation>(
-      patterns, predicate);
+      patterns, predicate, benefit);
   populateMathPolynomialApproximationPattern<AsinOp,
                                              AsinPolynomialApproximation>(
-      patterns, predicate);
+      patterns, predicate, benefit);
   populateMathPolynomialApproximationPattern<AtanOp, AtanApproximation>(
-      patterns, predicate);
+      patterns, predicate, benefit);
   populateMathPolynomialApproximationPattern<Atan2Op, Atan2Approximation>(
-      patterns, predicate);
+      patterns, predicate, benefit);
   populateMathPolynomialApproximationPattern<CbrtOp, CbrtApproximation>(
-      patterns, predicate);
+      patterns, predicate, benefit);
   populateMathPolynomialApproximationPattern<
-      CosOp, SinAndCosApproximation<false, math::CosOp>>(patterns, predicate);
+      CosOp, SinAndCosApproximation<false, math::CosOp>>(patterns, predicate,
+                                                         benefit);
   populateMathPolynomialApproximationPattern<ErfOp, ErfPolynomialApproximation>(
-      patterns, predicate);
+      patterns, predicate, benefit);
   populateMathPolynomialApproximationPattern<ErfcOp,
                                              ErfcPolynomialApproximation>(
-      patterns, predicate);
+      patterns, predicate, benefit);
   populateMathPolynomialApproximationPattern<ExpOp, ExpApproximation>(
-      patterns, predicate);
+      patterns, predicate, benefit);
   populateMathPolynomialApproximationPattern<ExpM1Op, ExpM1Approximation>(
-      patterns, predicate);
+      patterns, predicate, benefit);
   populateMathPolynomialApproximationPattern<LogOp, LogApproximation>(
-      patterns, predicate);
+      patterns, predicate, benefit);
   populateMathPolynomialApproximationPattern<Log2Op, Log2Approximation>(
-      patterns, predicate);
+      patterns, predicate, benefit);
   populateMathPolynomialApproximationPattern<Log1pOp, Log1pApproximation>(
-      patterns, predicate);
+      patterns, predicate, benefit);
   populateMathPolynomialApproximationPattern<RsqrtOp, RsqrtApproximation>(
-      patterns, predicate);
+      patterns, predicate, benefit);
   populateMathPolynomialApproximationPattern<
-      SinOp, SinAndCosApproximation<true, math::SinOp>>(patterns, predicate);
+      SinOp, SinAndCosApproximation<true, math::SinOp>>(patterns, predicate,
+                                                        benefit);
   populateMathPolynomialApproximationPattern<TanhOp, TanhApproximation>(
-      patterns, predicate);
+      patterns, predicate, benefit);
 }
 
 void mlir::populateMathPolynomialApproximationPatterns(
diff --git a/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp b/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
index f93ae0a7a298f..53a618d787333 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/RuntimeOpVerification.cpp
@@ -128,6 +128,50 @@ struct CastOpInterface
   }
 };
 
+struct CopyOpInterface
+    : public RuntimeVerifiableOpInterface::ExternalModel<CopyOpInterface,
+                                                         CopyOp> {
+  void generateRuntimeVerification(Operation *op, OpBuilder &builder,
+                                   Location loc) const {
+    auto copyOp = cast<CopyOp>(op);
+    BaseMemRefType sourceType = copyOp.getSource().getType();
+    BaseMemRefType targetType = copyOp.getTarget().getType();
+    auto rankedSourceType = dyn_cast<MemRefType>(sourceType);
+    auto rankedTargetType = dyn_cast<MemRefType>(targetType);
+
+    // TODO: Verification for unranked memrefs is not supported yet.
+    if (!rankedSourceType || !rankedTargetType)
+      return;
+
+    assert(sourceType.getRank() == targetType.getRank() && "rank mismatch");
+    for (int64_t i = 0, e = sourceType.getRank(); i < e; ++i) {
+      // Fully static dimensions in both source and target operand are already
+      // verified by the op verifier.
+      if (!rankedSourceType.isDynamicDim(i) &&
+          !rankedTargetType.isDynamicDim(i))
+        continue;
+      auto getDimSize = [&](Value memRef, MemRefType type,
+                            int64_t dim) -> Value {
+        return type.isDynamicDim(dim)
+                   ? builder.create<DimOp>(loc, memRef, dim).getResult()
+                   : builder
+                         .create<arith::ConstantIndexOp>(loc,
+                                                         type.getDimSize(dim))
+                         .getResult();
+      };
+      Value sourceDim = getDimSize(copyOp.getSource(), rankedSourceType, i);
+      Value targetDim = getDimSize(copyOp.getTarget(), rankedTargetType, i);
+      Value sameDimSize = builder.create<arith::CmpIOp>(
+          loc, arith::CmpIPredicate::eq, sourceDim, targetDim);
+      builder.create<cf::AssertOp>(
+          loc, sameDimSize,
+          RuntimeVerifiableOpInterface::generateErrorMessage(
+              op, "size of " + std::to_string(i) +
+                      "-th source/target dim does not match"));
+    }
+  }
+};
+
 /// Verifies that the indices on load/store ops are in-bounds of the memref's
 /// index space: 0 <= index#i < dim#i
 template <typename LoadStoreOp>
@@ -335,6 +379,7 @@ void mlir::memref::registerRuntimeVerifiableOpInterfaceExternalModels(
     DialectRegistry &registry) {
   registry.addExtension(+[](MLIRContext *ctx, memref::MemRefDialect *dialect) {
     CastOp::attachInterface<CastOpInterface>(*ctx);
+    CopyOp::attachInterface<CopyOpInterface>(*ctx);
     ExpandShapeOp::attachInterface<ExpandShapeOpInterface>(*ctx);
     LoadOp::attachInterface<LoadStoreOpInterface<LoadOp>>(*ctx);
     ReinterpretCastOp::attachInterface<ReinterpretCastOpInterface>(*ctx);
diff --git a/mlir/lib/Dialect/NVGPU/IR/CMakeLists.txt b/mlir/lib/Dialect/NVGPU/IR/CMakeLists.txt
index 4d47ce4746dbb..10aa502ee67f8 100644
--- a/mlir/lib/Dialect/NVGPU/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/NVGPU/IR/CMakeLists.txt
@@ -5,7 +5,7 @@ add_mlir_dialect_library(MLIRNVGPUDialect
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/NVGPU
 
   DEPENDS
-  MLIRNVGPUIncGen
+  MLIRNVGPUOpsIncGen
   MLIRNVGPUEnumsIncGen
   MLIRNVGPUAttributesIncGen
   MLIRNVGPUTypesIncGen
diff --git a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
index ba86e8d6ceaf9..abbdb6a0f53ec 100644
--- a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
+++ b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
@@ -35,7 +35,7 @@ using namespace mlir::nvgpu;
 void nvgpu::NVGPUDialect::initialize() {
   addTypes<
 #define GET_TYPEDEF_LIST
-#include "mlir/Dialect/NVGPU/IR/NVGPUTypes.cpp.inc"
+#include "mlir/Dialect/NVGPU/IR/NVGPUTypeDefs.cpp.inc"
       >();
   addAttributes<
 #define GET_ATTRDEF_LIST
@@ -43,7 +43,7 @@ void nvgpu::NVGPUDialect::initialize() {
       >();
   addOperations<
 #define GET_OP_LIST
-#include "mlir/Dialect/NVGPU/IR/NVGPU.cpp.inc"
+#include "mlir/Dialect/NVGPU/IR/NVGPUOps.cpp.inc"
       >();
 }
 
@@ -681,7 +681,7 @@ LogicalResult RcpOp::verify() {
 #include "mlir/Dialect/NVGPU/IR/NVGPUEnums.cpp.inc"
 
 #define GET_OP_CLASSES
-#include "mlir/Dialect/NVGPU/IR/NVGPU.cpp.inc"
+#include "mlir/Dialect/NVGPU/IR/NVGPUOps.cpp.inc"
 
 #define GET_TYPEDEF_CLASSES
-#include "mlir/Dialect/NVGPU/IR/NVGPUTypes.cpp.inc"
+#include "mlir/Dialect/NVGPU/IR/NVGPUTypeDefs.cpp.inc"
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 23ddc2fd53347..84b4d30076646 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -601,6 +601,7 @@ struct ReductionParseArgs {
 };
 
 struct AllRegionParseArgs {
+  std::optional<MapParseArgs> hasDeviceAddrArgs;
   std::optional<MapParseArgs> hostEvalArgs;
   std::optional<ReductionParseArgs> inReductionArgs;
   std::optional<MapParseArgs> mapArgs;
@@ -759,6 +760,11 @@ static ParseResult parseBlockArgRegion(OpAsmParser &parser, Region &region,
                                        AllRegionParseArgs args) {
   llvm::SmallVector<OpAsmParser::Argument> entryBlockArgs;
 
+  if (failed(parseBlockArgClause(parser, entryBlockArgs, "has_device_addr",
+                                 args.hasDeviceAddrArgs)))
+    return parser.emitError(parser.getCurrentLocation())
+           << "invalid `has_device_addr` format";
+
   if (failed(parseBlockArgClause(parser, entryBlockArgs, "host_eval",
                                  args.hostEvalArgs)))
     return parser.emitError(parser.getCurrentLocation())
@@ -802,8 +808,12 @@ static ParseResult parseBlockArgRegion(OpAsmParser &parser, Region &region,
   return parser.parseRegion(region, entryBlockArgs);
 }
 
-static ParseResult parseHostEvalInReductionMapPrivateRegion(
+// These parseXyz functions correspond to the custom<Xyz> definitions
+// in the .td file(s).
+static ParseResult parseTargetOpRegion(
     OpAsmParser &parser, Region &region,
+    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &hasDeviceAddrVars,
+    SmallVectorImpl<Type> &hasDeviceAddrTypes,
     SmallVectorImpl<OpAsmParser::UnresolvedOperand> &hostEvalVars,
     SmallVectorImpl<Type> &hostEvalTypes,
     SmallVectorImpl<OpAsmParser::UnresolvedOperand> &inReductionVars,
@@ -815,6 +825,7 @@ static ParseResult parseHostEvalInReductionMapPrivateRegion(
     llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
     DenseI64ArrayAttr &privateMaps) {
   AllRegionParseArgs args;
+  args.hasDeviceAddrArgs.emplace(hasDeviceAddrVars, hasDeviceAddrTypes);
   args.hostEvalArgs.emplace(hostEvalVars, hostEvalTypes);
   args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
                                inReductionByref, inReductionSyms);
@@ -935,6 +946,7 @@ struct ReductionPrintArgs {
       : vars(vars), types(types), byref(byref), syms(syms), modifier(mod) {}
 };
 struct AllRegionPrintArgs {
+  std::optional<MapPrintArgs> hasDeviceAddrArgs;
   std::optional<MapPrintArgs> hostEvalArgs;
   std::optional<ReductionPrintArgs> inReductionArgs;
   std::optional<MapPrintArgs> mapArgs;
@@ -1028,6 +1040,9 @@ static void printBlockArgRegion(OpAsmPrinter &p, Operation *op, Region &region,
   auto iface = llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(op);
   MLIRContext *ctx = op->getContext();
 
+  printBlockArgClause(p, ctx, "has_device_addr",
+                      iface.getHasDeviceAddrBlockArgs(),
+                      args.hasDeviceAddrArgs);
   printBlockArgClause(p, ctx, "host_eval", iface.getHostEvalBlockArgs(),
                       args.hostEvalArgs);
   printBlockArgClause(p, ctx, "in_reduction", iface.getInReductionBlockArgs(),
@@ -1050,14 +1065,20 @@ static void printBlockArgRegion(OpAsmPrinter &p, Operation *op, Region &region,
   p.printRegion(region, /*printEntryBlockArgs=*/false);
 }
 
-static void printHostEvalInReductionMapPrivateRegion(
-    OpAsmPrinter &p, Operation *op, Region &region, ValueRange hostEvalVars,
-    TypeRange hostEvalTypes, ValueRange inReductionVars,
-    TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref,
-    ArrayAttr inReductionSyms, ValueRange mapVars, TypeRange mapTypes,
-    ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms,
-    DenseI64ArrayAttr privateMaps) {
+// These parseXyz functions correspond to the custom<Xyz> definitions
+// in the .td file(s).
+static void
+printTargetOpRegion(OpAsmPrinter &p, Operation *op, Region &region,
+                    ValueRange hasDeviceAddrVars, TypeRange hasDeviceAddrTypes,
+                    ValueRange hostEvalVars, TypeRange hostEvalTypes,
+                    ValueRange inReductionVars, TypeRange inReductionTypes,
+                    DenseBoolArrayAttr inReductionByref,
+                    ArrayAttr inReductionSyms, ValueRange mapVars,
+                    TypeRange mapTypes, ValueRange privateVars,
+                    TypeRange privateTypes, ArrayAttr privateSyms,
+                    DenseI64ArrayAttr privateMaps) {
   AllRegionPrintArgs args;
+  args.hasDeviceAddrArgs.emplace(hasDeviceAddrVars, hasDeviceAddrTypes);
   args.hostEvalArgs.emplace(hostEvalVars, hostEvalTypes);
   args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
                                inReductionByref, inReductionSyms);
@@ -1458,8 +1479,8 @@ uint64_t mapTypeToBitFlag(uint64_t value,
 /// Parses a map_entries map type from a string format back into its numeric
 /// value.
 ///
-/// map-clause = `map_clauses (  ( `(` `always, `? `close, `? `present, `? (
-/// `to` | `from` | `delete` `)` )+ `)` )
+/// map-clause = `map_clauses (  ( `(` `always, `? `implicit, `? `ompx_hold, `?
+/// `close, `? `present, `? ( `to` | `from` | `delete` `)` )+ `)` )
 static ParseResult parseMapClause(OpAsmParser &parser, IntegerAttr &mapType) {
   llvm::omp::OpenMPOffloadMappingFlags mapTypeBits =
       llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE;
@@ -1477,6 +1498,9 @@ static ParseResult parseMapClause(OpAsmParser &parser, IntegerAttr &mapType) {
     if (mapTypeMod == "implicit")
       mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
 
+    if (mapTypeMod == "ompx_hold")
+      mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
+
     if (mapTypeMod == "close")
       mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
 
@@ -1526,6 +1550,9 @@ static void printMapClause(OpAsmPrinter &p, Operation *op,
   if (mapTypeToBitFlag(mapTypeBits,
                        llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT))
     mapTypeStrs.push_back("implicit");
+  if (mapTypeToBitFlag(mapTypeBits,
+                       llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD))
+    mapTypeStrs.push_back("ompx_hold");
   if (mapTypeToBitFlag(mapTypeBits,
                        llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_CLOSE))
     mapTypeStrs.push_back("close");
@@ -2221,12 +2248,12 @@ LogicalResult TeamsOp::verify() {
 // SectionOp
 //===----------------------------------------------------------------------===//
 
-unsigned SectionOp::numPrivateBlockArgs() {
-  return getParentOp().numPrivateBlockArgs();
+OperandRange SectionOp::getPrivateVars() {
+  return getParentOp().getPrivateVars();
 }
 
-unsigned SectionOp::numReductionBlockArgs() {
-  return getParentOp().numReductionBlockArgs();
+OperandRange SectionOp::getReductionVars() {
+  return getParentOp().getReductionVars();
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Ptr/IR/PtrTypes.cpp b/mlir/lib/Dialect/Ptr/IR/PtrTypes.cpp
index 49c8ed977d50c..1fc424c05f6c7 100644
--- a/mlir/lib/Dialect/Ptr/IR/PtrTypes.cpp
+++ b/mlir/lib/Dialect/Ptr/IR/PtrTypes.cpp
@@ -23,13 +23,12 @@ using namespace mlir::ptr;
 
 constexpr const static unsigned kDefaultPointerSizeBits = 64;
 constexpr const static unsigned kBitsInByte = 8;
-constexpr const static unsigned kDefaultPointerAlignment = 8;
-
-static Attribute getDefaultMemorySpace(PtrType ptr) { return nullptr; }
+constexpr const static unsigned kDefaultPointerAlignmentBits = 8;
 
 /// Searches the data layout for the pointer spec, returns nullptr if it is not
 /// found.
-static SpecAttr getPointerSpec(DataLayoutEntryListRef params, PtrType type) {
+static SpecAttr getPointerSpec(DataLayoutEntryListRef params, PtrType type,
+                               Attribute defaultMemorySpace) {
   for (DataLayoutEntryInterface entry : params) {
     if (!entry.isTypeEntry())
       continue;
@@ -41,20 +40,22 @@ static SpecAttr getPointerSpec(DataLayoutEntryListRef params, PtrType type) {
   }
   // If not found, and this is the pointer to the default memory space, assume
   // 64-bit pointers.
-  if (type.getMemorySpace() == getDefaultMemorySpace(type))
+  if (type.getMemorySpace() == defaultMemorySpace)
     return SpecAttr::get(type.getContext(), kDefaultPointerSizeBits,
-                         kDefaultPointerAlignment, kDefaultPointerAlignment,
-                         kDefaultPointerSizeBits);
+                         kDefaultPointerAlignmentBits,
+                         kDefaultPointerAlignmentBits, kDefaultPointerSizeBits);
   return nullptr;
 }
 
 bool PtrType::areCompatible(DataLayoutEntryListRef oldLayout,
-                            DataLayoutEntryListRef newLayout) const {
+                            DataLayoutEntryListRef newLayout,
+                            DataLayoutSpecInterface newSpec,
+                            const DataLayoutIdentifiedEntryMap &map) const {
   for (DataLayoutEntryInterface newEntry : newLayout) {
     if (!newEntry.isTypeEntry())
       continue;
     uint32_t size = kDefaultPointerSizeBits;
-    uint32_t abi = kDefaultPointerAlignment;
+    uint32_t abi = kDefaultPointerAlignmentBits;
     auto newType = llvm::cast<PtrType>(llvm::cast<Type>(newEntry.getKey()));
     const auto *it =
         llvm::find_if(oldLayout, [&](DataLayoutEntryInterface entry) {
@@ -65,10 +66,12 @@ bool PtrType::areCompatible(DataLayoutEntryListRef oldLayout,
           return false;
         });
     if (it == oldLayout.end()) {
+      Attribute defaultMemorySpace = mlir::detail::getDefaultMemorySpace(
+          map.lookup(newSpec.getDefaultMemorySpaceIdentifier(getContext())));
       it = llvm::find_if(oldLayout, [&](DataLayoutEntryInterface entry) {
         if (auto type = llvm::dyn_cast_if_present<Type>(entry.getKey())) {
           auto ptrTy = llvm::cast<PtrType>(type);
-          return ptrTy.getMemorySpace() == getDefaultMemorySpace(ptrTy);
+          return ptrTy.getMemorySpace() == defaultMemorySpace;
         }
         return false;
       });
@@ -90,43 +93,44 @@ bool PtrType::areCompatible(DataLayoutEntryListRef oldLayout,
 
 uint64_t PtrType::getABIAlignment(const DataLayout &dataLayout,
                                   DataLayoutEntryListRef params) const {
-  if (SpecAttr spec = getPointerSpec(params, *this))
+  Attribute defaultMemorySpace = dataLayout.getDefaultMemorySpace();
+  if (SpecAttr spec = getPointerSpec(params, *this, defaultMemorySpace))
     return spec.getAbi() / kBitsInByte;
 
-  return dataLayout.getTypeABIAlignment(
-      get(getContext(), getDefaultMemorySpace(*this)));
+  return dataLayout.getTypeABIAlignment(get(getContext(), defaultMemorySpace));
 }
 
 std::optional<uint64_t>
 PtrType::getIndexBitwidth(const DataLayout &dataLayout,
                           DataLayoutEntryListRef params) const {
-  if (SpecAttr spec = getPointerSpec(params, *this)) {
+  Attribute defaultMemorySpace = dataLayout.getDefaultMemorySpace();
+  if (SpecAttr spec = getPointerSpec(params, *this, defaultMemorySpace)) {
     return spec.getIndex() == SpecAttr::kOptionalSpecValue ? spec.getSize()
                                                            : spec.getIndex();
   }
 
-  return dataLayout.getTypeIndexBitwidth(
-      get(getContext(), getDefaultMemorySpace(*this)));
+  return dataLayout.getTypeIndexBitwidth(get(getContext(), defaultMemorySpace));
 }
 
 llvm::TypeSize PtrType::getTypeSizeInBits(const DataLayout &dataLayout,
                                           DataLayoutEntryListRef params) const {
-  if (SpecAttr spec = getPointerSpec(params, *this))
+  Attribute defaultMemorySpace = dataLayout.getDefaultMemorySpace();
+  if (SpecAttr spec = getPointerSpec(params, *this, defaultMemorySpace))
     return llvm::TypeSize::getFixed(spec.getSize());
 
   // For other memory spaces, use the size of the pointer to the default memory
   // space.
-  return dataLayout.getTypeSizeInBits(
-      get(getContext(), getDefaultMemorySpace(*this)));
+  return dataLayout.getTypeSizeInBits(get(getContext(), defaultMemorySpace));
 }
 
 uint64_t PtrType::getPreferredAlignment(const DataLayout &dataLayout,
                                         DataLayoutEntryListRef params) const {
-  if (SpecAttr spec = getPointerSpec(params, *this))
+  Attribute defaultMemorySpace = dataLayout.getDefaultMemorySpace();
+  if (SpecAttr spec = getPointerSpec(params, *this, defaultMemorySpace))
     return spec.getPreferred() / kBitsInByte;
 
   return dataLayout.getTypePreferredAlignment(
-      get(getContext(), getDefaultMemorySpace(*this)));
+      get(getContext(), defaultMemorySpace));
 }
 
 LogicalResult PtrType::verifyEntries(DataLayoutEntryListRef entries,
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
index c56dbcca2175d..a60410d01ac57 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp
@@ -1374,9 +1374,8 @@ LogicalResult mlir::spirv::unrollVectorsInFuncBodies(Operation *op) {
   // further transformations to canonicalize/cancel.
   {
     RewritePatternSet patterns(context);
-    auto options = vector::VectorTransformsOptions().setVectorTransposeLowering(
-        vector::VectorTransposeLowering::EltWise);
-    vector::populateVectorTransposeLoweringPatterns(patterns, options);
+    vector::populateVectorTransposeLoweringPatterns(
+        patterns, vector::VectorTransposeLowering::EltWise);
     vector::populateVectorShapeCastLoweringPatterns(patterns);
     if (failed(applyPatternsGreedily(op, std::move(patterns))))
       return failure();
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index 9854cfcc279b5..fcbef0c14739f 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -2778,22 +2778,7 @@ Operation *SparseTensorDialect::materializeConstant(OpBuilder &builder,
   return nullptr;
 }
 
-namespace {
-struct SparseTensorAsmDialectInterface : public OpAsmDialectInterface {
-  using OpAsmDialectInterface::OpAsmDialectInterface;
-
-  AliasResult getAlias(Attribute attr, raw_ostream &os) const override {
-    if (isa<SparseTensorEncodingAttr>(attr)) {
-      os << "sparse";
-      return AliasResult::OverridableAlias;
-    }
-    return AliasResult::NoAlias;
-  }
-};
-} // namespace
-
 void SparseTensorDialect::initialize() {
-  addInterface<SparseTensorAsmDialectInterface>();
   addAttributes<
 #define GET_ATTRDEF_LIST
 #include "mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.cpp.inc"
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index e3c8899cd44d9..e5a32c0bbd4e0 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -2352,13 +2352,52 @@ static LogicalResult produceSliceErrorMsg(SliceVerificationResult result,
   }
 }
 
+/// Verify that the offsets/sizes/strides-style access into the given tensor
+/// is in-bounds. Only static information is verified.
+static LogicalResult verifyInBoundsSlice(Operation *op,
+                                         RankedTensorType tensorType,
+                                         ArrayRef<int64_t> staticOffsets,
+                                         ArrayRef<int64_t> staticSizes,
+                                         ArrayRef<int64_t> staticStrides) {
+  for (int64_t i = 0, e = tensorType.getRank(); i < e; ++i) {
+    // Nothing to verify for dynamic source dims.
+    if (tensorType.isDynamicDim(i))
+      continue;
+    // Nothing to verify if the offset is dynamic.
+    if (ShapedType::isDynamic(staticOffsets[i]))
+      continue;
+    if (staticOffsets[i] >= tensorType.getDimSize(i))
+      return op->emitOpError("offset ")
+             << i << " is out-of-bounds: " << staticOffsets[i]
+             << " >= " << tensorType.getDimSize(i);
+    if (ShapedType::isDynamic(staticSizes[i]) ||
+        ShapedType::isDynamic(staticStrides[i]))
+      continue;
+    int64_t lastPos =
+        staticOffsets[i] + (staticSizes[i] - 1) * staticStrides[i];
+    if (lastPos >= tensorType.getDimSize(i))
+      return op->emitOpError("slice along dimension ")
+             << i << " runs out-of-bounds: " << lastPos
+             << " >= " << tensorType.getDimSize(i);
+  }
+  return success();
+}
+
 /// Verifier for ExtractSliceOp.
 LogicalResult ExtractSliceOp::verify() {
+  RankedTensorType sourceType = getSourceType();
+
   // Verify result type against inferred type.
   RankedTensorType expectedType = ExtractSliceOp::inferResultType(
-      getSourceType(), getMixedOffsets(), getMixedSizes(), getMixedStrides());
+      sourceType, getMixedOffsets(), getMixedSizes(), getMixedStrides());
   SliceVerificationResult result = isRankReducedType(expectedType, getType());
-  return produceSliceErrorMsg(result, *this, expectedType);
+  if (result != SliceVerificationResult::Success)
+    return produceSliceErrorMsg(result, *this, expectedType);
+
+  // Verify that offsets, sizes, strides do not run out-of-bounds with respect
+  // to the source tensor.
+  return verifyInBoundsSlice(getOperation(), sourceType, getStaticOffsets(),
+                             getStaticSizes(), getStaticStrides());
 }
 
 llvm::SmallBitVector ExtractSliceOp::getDroppedDims() {
@@ -2729,11 +2768,18 @@ static SliceVerificationResult verifyInsertSliceOp(
 
 /// Verifier for InsertSliceOp.
 LogicalResult InsertSliceOp::verify() {
+  // Verify result type against inferred type.
   RankedTensorType expectedType;
   SliceVerificationResult result =
       verifyInsertSliceOp(getSourceType(), getType(), getStaticOffsets(),
                           getStaticSizes(), getStaticStrides(), &expectedType);
-  return produceSliceErrorMsg(result, *this, expectedType);
+  if (result != SliceVerificationResult::Success)
+    return produceSliceErrorMsg(result, *this, expectedType);
+
+  // Verify that offsets, sizes, strides do not run out-of-bounds with respect
+  // to the source tensor.
+  return verifyInBoundsSlice(getOperation(), getDestType(), getStaticOffsets(),
+                             getStaticSizes(), getStaticStrides());
 }
 
 /// If we have two consecutive InsertSliceOp writing to the same slice, we
@@ -3747,11 +3793,18 @@ LogicalResult ParallelInsertSliceOp::verify() {
     return this->emitError("expected ParallelCombiningOpInterface parent, got:")
            << *(getOperation()->getParentOp());
 
+  // Verify result type against inferred type.
   RankedTensorType expectedType;
   SliceVerificationResult result =
       verifyInsertSliceOp(getSourceType(), getDestType(), getStaticOffsets(),
                           getStaticSizes(), getStaticStrides(), &expectedType);
-  return produceSliceErrorMsg(result, *this, expectedType);
+  if (result != SliceVerificationResult::Success)
+    return produceSliceErrorMsg(result, *this, expectedType);
+
+  // Verify that offsets, sizes, strides do not run out-of-bounds with respect
+  // to the source tensor.
+  return verifyInBoundsSlice(getOperation(), getDestType(), getStaticOffsets(),
+                             getStaticSizes(), getStaticStrides());
 }
 
 void ParallelInsertSliceOp::getCanonicalizationPatterns(
diff --git a/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
index ae8e3528b02e0..acedf51d0e240 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
@@ -160,6 +160,12 @@ struct BubbleUpExpandThroughParallelCollapse
     auto expandReInds = expandOp.getReassociationIndices();
     auto collapseReInds = collapseOp.getReassociationIndices();
 
+    // Special case where the collapsed tensor to expand is a 0-D tensor,
+    // then the reassociation maps will be empty and not produce valid results.
+    if (expandReInds.size() == 0) {
+      return failure();
+    }
+
     // Reshapes are parallel to each other if none of the reassociation indices
     // have greater than 1 index for both reshapes.
     for (auto [expandReassociation, collapseReassociation] :
diff --git a/mlir/lib/Dialect/Tosa/IR/ShardingInterfaceImpl.cpp b/mlir/lib/Dialect/Tosa/IR/ShardingInterfaceImpl.cpp
index ffbb707344b8c..be29298a35aeb 100644
--- a/mlir/lib/Dialect/Tosa/IR/ShardingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/ShardingInterfaceImpl.cpp
@@ -55,11 +55,52 @@ struct MatMulOpSharding
     SmallVector<AffineMap> maps;
     maps.push_back(AffineMap::getMultiDimMapWithTargets(4, {0, 1, 3}, ctx));
     maps.push_back(AffineMap::getMultiDimMapWithTargets(4, {0, 3, 2}, ctx));
+    maps.push_back(AffineMap::get(0, 0, {}, ctx));
+    maps.push_back(AffineMap::get(0, 0, {}, ctx));
     maps.push_back(AffineMap::getMultiDimMapWithTargets(4, {0, 1, 2}, ctx));
     return maps;
   }
 };
 
+struct NegateOpSharding
+    : public ShardingInterface::ExternalModel<NegateOpSharding, NegateOp> {
+  SmallVector<utils::IteratorType> getLoopIteratorTypes(Operation *op) const {
+    Value val = op->getOperand(0);
+    auto type = dyn_cast<RankedTensorType>(val.getType());
+    if (!type)
+      return {};
+    SmallVector<utils::IteratorType> types(type.getRank(),
+                                           utils::IteratorType::parallel);
+    return types;
+  }
+
+  SmallVector<AffineMap> getIndexingMaps(Operation *op) const {
+    MLIRContext *ctx = op->getContext();
+    Value val = op->getOperand(0);
+    auto type = dyn_cast<RankedTensorType>(val.getType());
+    if (!type)
+      return {};
+    int64_t rank = type.getRank();
+    SmallVector<AffineMap> maps = {
+        AffineMap::getMultiDimIdentityMap(rank, ctx),
+        AffineMap::get(0, 0, {}, ctx), AffineMap::get(0, 0, {}, ctx),
+        AffineMap::getMultiDimIdentityMap(rank, ctx)};
+    return maps;
+  }
+
+  LogicalResult spmdize(Operation *op, ArrayRef<Value> spmdizedOperands,
+                        ArrayRef<MeshSharding> operandShardings,
+                        ArrayRef<MeshSharding> resultShardings,
+                        IRMapping &spmdizationMap,
+                        SymbolTableCollection &symbolTable,
+                        OpBuilder &builder) const {
+    spmdizeTriviallyShardableOperation(*op, spmdizedOperands, operandShardings,
+                                       resultShardings, spmdizationMap,
+                                       symbolTable, builder);
+    return success();
+  }
+};
+
 template <typename OpType>
 static void registerElemwiseOne(MLIRContext *ctx) {
   OpType::template attachInterface<ElementwiseShardingInterface<OpType>>(*ctx);
@@ -82,9 +123,10 @@ void mlir::tosa::registerShardingInterfaceExternalModels(
         BitwiseOrOp, BitwiseXorOp, IntDivOp, LogicalAndOp, LogicalLeftShiftOp,
         LogicalRightShiftOp, LogicalOrOp, LogicalXorOp, MaximumOp, MinimumOp,
         MulOp, PowOp, SubOp, AbsOp, BitwiseNotOp, CeilOp, ClzOp, ExpOp, FloorOp,
-        LogOp, LogicalNotOp, NegateOp, ReciprocalOp, RsqrtOp, SelectOp, EqualOp,
+        LogOp, LogicalNotOp, ReciprocalOp, RsqrtOp, SelectOp, EqualOp,
         GreaterOp, GreaterEqualOp>(ctx);
 
     MatMulOp::attachInterface<MatMulOpSharding>(*ctx);
+    NegateOp::attachInterface<NegateOpSharding>(*ctx);
   });
 }
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
index 3e99c1f717d09..09d2c5d35263c 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
@@ -1143,13 +1143,36 @@ OpFoldResult tosa::ExpOp::fold(FoldAdaptor adaptor) {
 }
 
 OpFoldResult tosa::NegateOp::fold(FoldAdaptor adaptor) {
-  auto input = getInput1();
   // Element-wise negate(negate(x)) = x
-  if (auto op = input.getDefiningOp<tosa::NegateOp>()) {
-    return op.getInput1();
+  // iff all zero points are constant 0
+  auto definingOp = getInput1().getDefiningOp<tosa::NegateOp>();
+  if (!definingOp) {
+    // defining op of input1 is not a negate, cannot fold
+    return {};
   }
 
-  return {};
+  if (FailureOr<int64_t> maybeIZp = getInput1ZeroPoint();
+      failed(maybeIZp) || *maybeIZp != 0) {
+    // input1 zero point is not constant 0, cannot fold
+    return {};
+  }
+  if (FailureOr<int64_t> maybeOZp = getOutputZeroPoint();
+      failed(maybeOZp) || *maybeOZp != 0) {
+    // output zero point is not constant 0, cannot fold
+    return {};
+  }
+  if (FailureOr<int64_t> maybeIZp = definingOp.getInput1ZeroPoint();
+      failed(maybeIZp) || *maybeIZp != 0) {
+    // definingOp's input1 zero point is not constant 0, cannot fold
+    return {};
+  }
+  if (FailureOr<int64_t> maybeOZp = definingOp.getOutputZeroPoint();
+      failed(maybeOZp) || *maybeOZp != 0) {
+    // definingOp's output zero point is not constant 0, cannot fold
+    return {};
+  }
+
+  return definingOp.getInput1();
 }
 
 OpFoldResult tosa::AbsOp::fold(FoldAdaptor adaptor) {
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index 086ffb753c333..d3b7b811fc8b0 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -254,6 +254,25 @@ static Type getStorageElementTypeOrSelf(Type type) {
   return elementType;
 }
 
+static LogicalResult verifyRescaleValueAndZpTypes(Operation *op, Value val,
+                                                  Value valZp, StringRef name) {
+  Type eType = getStorageElementTypeOrSelf(val.getType());
+  Type eZpType = getStorageElementTypeOrSelf(valZp.getType());
+
+  bool bothInts =
+      mlir::isa<IntegerType>(eType) && mlir::isa<IntegerType>(eZpType);
+  bool sameBitWidth =
+      (eType.getIntOrFloatBitWidth() == eZpType.getIntOrFloatBitWidth());
+
+  if (!bothInts || !sameBitWidth) {
+    return op->emitOpError()
+           << "expected " << name << " and " << name
+           << "_zp to both be integer of the same bitwidth, but got " << eType
+           << " vs. " << eZpType;
+  }
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // TOSA Operator Verifiers.
 //===----------------------------------------------------------------------===//
@@ -522,14 +541,7 @@ LogicalResult tosa::AvgPool2dOp::verify() {
   if (succeeded(maybeOZp) && verifyOutputZeroPoint(*maybeOZp).failed())
     return failure();
 
-  if ((inputETy.isF32() && resultETy.isF32()) ||
-      (inputETy.isF16() && resultETy.isF16()) ||
-      (inputETy.isBF16() && resultETy.isBF16()) ||
-      (inputETy.isInteger(8) && resultETy.isInteger(8)) ||
-      (inputETy.isInteger(16) && resultETy.isInteger(16)))
-    return success();
-
-  return emitOpError("input/output element types are incompatible.");
+  return success();
 }
 
 LogicalResult tosa::ClampOp::verify() {
@@ -636,23 +648,13 @@ buildTransConvOpWithQuantInfo(OpBuilder &builder, OperationState &result,
 static void buildMatMulOpWithQuantInfo(OpBuilder &builder,
                                        OperationState &result, Type outputType,
                                        Value a, Value b) {
-  result.addOperands({a, b});
-  auto quantAttr = ::buildMatMulOpQuantizationAttr(builder, a, b);
-
-  if (quantAttr) {
-    result.addAttribute("a_zp", builder.getI32IntegerAttr(
-                                    static_cast<int32_t>(quantAttr.getAZp())));
-    result.addAttribute("b_zp", builder.getI32IntegerAttr(
-                                    static_cast<int32_t>(quantAttr.getBZp())));
+  auto zps = createZPsAsConst(builder, a, b);
+  result.addOperands({a, b, zps.first, zps.second});
 
-    auto inputType = llvm::dyn_cast<ShapedType>(a.getType());
-    assert(inputType && "Input must be a shaped tensor type!");
-
-    auto inputQType = llvm::dyn_cast<mlir::quant::UniformQuantizedType>(
-        inputType.getElementType());
-    assert(inputQType && "Tensor must have quantized datatype!");
-
-    unsigned inputBits = inputQType.getStorageTypeIntegralWidth();
+  Type finalOutputType{outputType};
+  if (auto quantAttr = buildMatMulOpQuantizationAttr(builder, a, b)) {
+    auto eType = getStorageElementTypeOrSelf(a.getType());
+    auto inputBits = eType.getIntOrFloatBitWidth();
 
     auto outputShapedType = llvm::dyn_cast<ShapedType>(outputType);
     assert(outputShapedType && "Output must be a shaped type");
@@ -662,11 +664,10 @@ static void buildMatMulOpWithQuantInfo(OpBuilder &builder,
       accElementType = builder.getIntegerType(48);
     else
       accElementType = builder.getI32Type();
-    auto accType = outputShapedType.clone(accElementType);
-    result.addTypes(accType);
-  } else {
-    result.addTypes(outputType);
+
+    finalOutputType = outputShapedType.clone(accElementType);
   }
+  result.addTypes(finalOutputType);
 }
 
 /// Both the tosa.avg_pool2d and unary ops use the same
@@ -715,23 +716,43 @@ buildAvgPool2dOpWithQuantInfo(OpBuilder &builder, OperationState &result,
   result.types.push_back(outputType);
 }
 
-/// This builder is called on single-parameter unary operators that have scale
-/// relationship between their input and output, expressed by the
-/// UnaryOpQuantizationAttr.
-static void buildUnaryOpWithQuantInfo(OpBuilder &builder,
-                                      OperationState &result, Type outputType,
-                                      Value input) {
-  result.addOperands(input);
+/// This builder is called on single-parameter negate operator
+/// to construct input and output zero points based on their
+/// types.
+static void buildNegateOpWithQuantInfo(OpBuilder &builder,
+                                       OperationState &result, Type outputType,
+                                       Value input) {
+  const Location loc{result.location};
+  int64_t input1Zp{0};
+  int64_t outputZp{0};
   auto quantAttr = buildUnaryOpQuantizationAttr(builder, input, outputType);
   if (quantAttr) {
-    // note: negateOp has attributes input1_zp and output_zp
-    result.addAttribute("input1_zp",
-                        builder.getI32IntegerAttr(
-                            static_cast<int32_t>(quantAttr.getInputZp())));
-    result.addAttribute("output_zp",
-                        builder.getI32IntegerAttr(
-                            static_cast<int32_t>(quantAttr.getOutputZp())));
+    input1Zp = quantAttr.getInputZp();
+    outputZp = quantAttr.getOutputZp();
   }
+  const std::optional<Value> input1ZpOp =
+      createZeroPointTensor(builder, loc, input.getType(), input1Zp);
+  if (!input1ZpOp) {
+    (void)emitError(
+        loc, "Failed to create input1 zero point for quantized NEGATE op");
+  }
+
+  const std::optional<Value> outputZpOp =
+      createZeroPointTensor(builder, loc, input.getType(), outputZp);
+  if (!outputZpOp) {
+    (void)emitError(
+        loc, "Failed to create output zero point for quantized NEGATE op");
+  }
+
+  if (input1ZpOp && outputZpOp) {
+    result.addOperands({input, input1ZpOp.value(), outputZpOp.value()});
+  } else {
+    // failed to create one or more zero points above: just add input as
+    // operands. This will trigger error in building the op because of
+    // missing zero points
+    result.addOperands({input});
+  }
+
   result.types.push_back(outputType);
 }
 
@@ -823,7 +844,7 @@ LogicalResult tosa::RFFT2dOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
     RFFT2dOp::Adaptor adaptor,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
-  ShapeAdaptor inputShape(adaptor.getInput().getType());
+  ShapeAdaptor inputShape(adaptor.getInputReal().getType());
 
   if (!inputShape.hasRank())
     return failure();
@@ -860,7 +881,8 @@ LogicalResult tosa::RFFT2dOp::verify() {
   if (failed(verifyCompatibleShapes(outputTypes)))
     return emitOpError("expected output shapes to match, got ") << outputTypes;
 
-  const auto inputType = llvm::dyn_cast<RankedTensorType>(getInput().getType());
+  const auto inputType =
+      llvm::dyn_cast<RankedTensorType>(getInputReal().getType());
   if (!inputType)
     return success();
 
@@ -888,7 +910,7 @@ LogicalResult tosa::RFFT2dOp::verify() {
   // Output width dimension expected to be input_width / 2 + 1
   const int64_t outputWidth = outputType.getDimSize(2);
   if (!ShapedType::isDynamic(width) && !ShapedType::isDynamic(outputWidth) &&
-      (outputWidth - 1) * 2 != width)
+      (outputWidth != (width / 2) + 1))
     return emitOpError(
                "expected output width to be equal to input_width / 2 + 1, got ")
            << outputWidth;
@@ -1147,16 +1169,39 @@ LogicalResult MatMulOp::verify() {
       return emitOpError("expect quantized operands to have same widths, got ")
              << aQuantWidth << " and " << bQuantWidth;
     }
+  } else {
+    // non-quantized element types
+    if (aElementType != bElementType) {
+      return emitOpError("expect same element type for inputs a and b, got ")
+             << aElementType << " and " << bElementType;
+    }
+  }
 
-    return success();
+  // check a_zp and b_zp
+  auto aEType = getStorageElementTypeOrSelf(aType);
+  auto aZpEType = getStorageElementTypeOrSelf(getAZp().getType());
+  if (aEType != aZpEType) {
+    return emitOpError("expect input a and a_zp have the same "
+                       "element type, got ")
+           << aEType << " and " << aZpEType;
   }
 
-  // non-quantized element types
-  if (aElementType != bElementType) {
-    return emitOpError("expect same element type for inputs a and b, got ")
-           << aElementType << " and " << bElementType;
+  auto bEType = getStorageElementTypeOrSelf(bType);
+  auto bZpEType = getStorageElementTypeOrSelf(getBZp().getType());
+  if (bEType != bZpEType) {
+    return emitOpError("expect input b and b_zp have the same "
+                       "element type, got ")
+           << bEType << " and " << bZpEType;
   }
 
+  FailureOr<int64_t> maybeAZp = getAZeroPoint();
+  if (succeeded(maybeAZp) && verifyAZeroPoint(*maybeAZp).failed())
+    return failure();
+
+  FailureOr<int64_t> maybeBZp = getBZeroPoint();
+  if (succeeded(maybeBZp) && verifyBZeroPoint(*maybeBZp).failed())
+    return failure();
+
   return success();
 }
 
@@ -1703,6 +1748,33 @@ static LogicalResult verifyZeroPoint(T op, Value val, const int64_t &zp,
   return success();
 }
 
+static LogicalResult verifyZeroPoint(tosa::RescaleOp op, Value zpVal,
+                                     const int64_t &zp,
+                                     const std::string &operand) {
+  bool isInputZp = (operand == "Input");
+
+  bool tensorUnsigned =
+      isInputZp ? op.getInputUnsigned() : op.getOutputUnsigned();
+  StringRef tensorName = isInputZp ? "input" : "output";
+
+  Type zpElemType = getElementTypeOrSelf(zpVal);
+
+  if (zp != 0) {
+    if (!zpElemType.isInteger(8) &&
+        !(zpElemType.isInteger(16) && tensorUnsigned)) {
+      return op.emitOpError()
+             << "expect " << tensorName << "_zp of 0, got " << zp;
+    }
+    if (zpElemType.isInteger(16) && tensorUnsigned && zp != 32768) {
+      return op.emitOpError() << "expect " << tensorName
+                              << "_zp of 0 or 32768 for unsigned int16 "
+                              << tensorName << ", got " << zp;
+    }
+  }
+
+  return success();
+}
+
 #define ZERO_POINT_HELPER(OP, OPERAND_NAME)                                    \
   FailureOr<int64_t> tosa::OP::get##OPERAND_NAME##ZeroPoint() {                \
     return getZeroPoint(*this, get##OPERAND_NAME##Zp());                       \
@@ -1721,6 +1793,12 @@ ZERO_POINT_HELPER(TransposeConv2DOp, Input)
 ZERO_POINT_HELPER(TransposeConv2DOp, Weight)
 ZERO_POINT_HELPER(AvgPool2dOp, Input)
 ZERO_POINT_HELPER(AvgPool2dOp, Output)
+ZERO_POINT_HELPER(MatMulOp, A)
+ZERO_POINT_HELPER(MatMulOp, B)
+ZERO_POINT_HELPER(NegateOp, Input1)
+ZERO_POINT_HELPER(NegateOp, Output)
+ZERO_POINT_HELPER(RescaleOp, Input)
+ZERO_POINT_HELPER(RescaleOp, Output)
 #undef ZERO_POINT_HELPER
 
 LogicalResult tosa::TransposeOp::inferReturnTypeComponents(
@@ -2223,10 +2301,8 @@ NARY_SHAPE_INFER(tosa::LogicalRightShiftOp)
 NARY_SHAPE_INFER(tosa::LogicalXorOp)
 NARY_SHAPE_INFER(tosa::MaximumOp)
 NARY_SHAPE_INFER(tosa::MinimumOp)
-NARY_SHAPE_INFER(tosa::NegateOp)
 NARY_SHAPE_INFER(tosa::PowOp)
 NARY_SHAPE_INFER(tosa::ReciprocalOp)
-NARY_SHAPE_INFER(tosa::RescaleOp)
 NARY_SHAPE_INFER(tosa::ReverseOp)
 NARY_SHAPE_INFER(tosa::RsqrtOp)
 NARY_SHAPE_INFER(tosa::SinOp)
@@ -2237,6 +2313,55 @@ NARY_SHAPE_INFER(tosa::ErfOp)
 NARY_SHAPE_INFER(tosa::SigmoidOp)
 #undef PRED_SHAPE_INFER
 
+LogicalResult tosa::NegateOp::inferReturnTypeComponents(
+    MLIRContext *context, ::std::optional<Location> location,
+    NegateOp::Adaptor adaptor,
+    SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
+  ShapeAdaptor inputShape(adaptor.getInput1().getType());
+  inferredReturnShapes.push_back(ShapedTypeComponents(inputShape));
+  return success();
+}
+
+LogicalResult tosa::NegateOp::verify() {
+  // Verify same element type
+  const Type input1Type = getInput1().getType();
+  const Type outputType = getOutput().getType();
+  if (verifySameElementTypes(*this, input1Type, outputType).failed())
+    return failure();
+
+  // Verify same shape
+  const SmallVector<Type, 2> types = {input1Type, outputType};
+  if (failed(verifyCompatibleShapes(types)))
+    return emitOpError() << "requires the same shape for input1 and output";
+
+  const Type input1EType = getStorageElementTypeOrSelf(getInput1().getType());
+  const Type input1ZpEType =
+      getStorageElementTypeOrSelf(getInput1Zp().getType());
+  if (input1EType != input1ZpEType) {
+    return emitOpError("expect both input1 and its zero point are the same "
+                       "element type, got ")
+           << input1EType << " and " << input1ZpEType;
+  }
+  const Type outputEType = getStorageElementTypeOrSelf(getOutput().getType());
+  const Type outputZpEType =
+      getStorageElementTypeOrSelf(getOutputZp().getType());
+  if (outputEType != outputZpEType) {
+    return emitOpError("expect both output and its zero point are the same "
+                       "element type, got ")
+           << outputEType << " and " << outputZpEType;
+  }
+
+  FailureOr<int64_t> maybeIZp = getInput1ZeroPoint();
+  if (succeeded(maybeIZp) && verifyInput1ZeroPoint(*maybeIZp).failed())
+    return failure();
+
+  FailureOr<int64_t> maybeOZp = getOutputZeroPoint();
+  if (succeeded(maybeOZp) && verifyOutputZeroPoint(*maybeOZp).failed())
+    return failure();
+
+  return success();
+}
+
 static LogicalResult poolingInferReturnTypes(
     ShapeAdaptor inputShape, ArrayRef<int64_t> kernel, ArrayRef<int64_t> stride,
     ArrayRef<int64_t> pad,
@@ -2676,6 +2801,127 @@ LogicalResult TransposeConv2DOp::verify() {
   return success();
 }
 
+LogicalResult RescaleOp::verify() {
+  auto inputType = llvm::dyn_cast<ShapedType>(getInput().getType());
+  if (!inputType) {
+    emitOpError("expect shaped tensor for input, got ") << getInput().getType();
+    return failure();
+  }
+
+  auto inputElementType =
+      getStorageElementTypeOrSelf(inputType.getElementType());
+  if (!mlir::isa<IntegerType>(inputElementType)) {
+    emitOpError("expect input to have integer element type, got ")
+        << inputElementType;
+    return failure();
+  }
+
+  auto outputType = llvm::dyn_cast<ShapedType>(getOutput().getType());
+  if (!outputType) {
+    emitOpError("expect shaped tensor for output, got ")
+        << getOutput().getType();
+    return failure();
+  }
+
+  auto outputElementType =
+      getStorageElementTypeOrSelf(outputType.getElementType());
+  if (!mlir::isa<IntegerType>(outputElementType)) {
+    emitOpError("expect output to have integer element type, got ")
+        << outputElementType;
+    return failure();
+  }
+
+  if (verifyRescaleValueAndZpTypes(*this, getInput(), getInputZp(), "input")
+          .failed())
+    return failure();
+
+  if (verifyRescaleValueAndZpTypes(*this, getOutput(), getOutputZp(), "output")
+          .failed())
+    return failure();
+
+  FailureOr<int64_t> maybeIZp = getInputZeroPoint();
+  if (succeeded(maybeIZp) && verifyInputZeroPoint(*maybeIZp).failed())
+    return failure();
+
+  FailureOr<int64_t> maybeOZp = getOutputZeroPoint();
+  if (succeeded(maybeOZp) && verifyOutputZeroPoint(*maybeOZp).failed())
+    return failure();
+
+  auto multiplierType = llvm::dyn_cast<ShapedType>(getMultiplier().getType());
+  if (!multiplierType) {
+    emitOpError("expect shaped tensor for multiplier, got ")
+        << getMultiplier().getType();
+    return failure();
+  }
+
+  auto shiftType = llvm::dyn_cast<ShapedType>(getShift().getType());
+  if (!shiftType) {
+    emitOpError("expect shaped tensor for shift, got ") << getShift().getType();
+    return failure();
+  }
+
+  // multiplier element type must be i32 for scale32 = true
+  if (getScale32() && !multiplierType.getElementType().isInteger(32)) {
+    emitOpError("expect i32 element type for multiplier for scale32=true, got ")
+        << multiplierType.getElementType();
+    return failure();
+  }
+
+  // multiplier element type must be i16 for scale32 = false
+  if (!getScale32() && !multiplierType.getElementType().isInteger(16)) {
+    emitOpError(
+        "expect i16 element type for multiplier for scale32=false, got ")
+        << multiplierType.getElementType();
+    return failure();
+  }
+
+  if (!inputType.hasRank())
+    return success();
+
+  // multiplier/shift must have shape = {numChannels},
+  // where numChannel is 1 if per_channel = false
+  // otherwise numChannel is dimension in input shape's last axis
+  int64_t numChannels = 1;
+  if (getPerChannel()) {
+    numChannels = inputType.getDimSize(inputType.getRank() - 1);
+  }
+
+  if (!multiplierType.hasRank())
+    return success();
+
+  ArrayRef<int64_t> multiplierShape = multiplierType.getShape();
+  // multiplier input has rank 1 by dialect definition
+  if (multiplierShape[0] != ShapedType::kDynamic &&
+      multiplierShape[0] != numChannels) {
+    emitOpError("expect shape of { ")
+        << numChannels << " } for multiplier input, got { "
+        << multiplierShape[0] << " }";
+    return failure();
+  }
+
+  if (!shiftType.hasRank())
+    return success();
+
+  ArrayRef<int64_t> shiftShape = shiftType.getShape();
+  // shift input has rank 1 by dialect definition
+  if (shiftShape[0] != ShapedType::kDynamic && shiftShape[0] != numChannels) {
+    emitOpError("expect shape of { ")
+        << numChannels << " } for shift input, got { " << shiftShape[0] << " }";
+    return failure();
+  }
+
+  return success();
+}
+
+LogicalResult RescaleOp::inferReturnTypeComponents(
+    MLIRContext *context, ::std::optional<Location> location,
+    RescaleOp::Adaptor adaptor,
+    SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
+  ShapeAdaptor inputShape(adaptor.getInput().getType());
+  inferredReturnShapes.push_back(ShapedTypeComponents(inputShape));
+  return success();
+}
+
 LogicalResult IfOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
     IfOp::Adaptor adaptor,
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaProfileCompliance.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaProfileCompliance.cpp
index 345616c9563b5..ed2c40598458c 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaProfileCompliance.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaProfileCompliance.cpp
@@ -160,7 +160,7 @@ void ProfileInfoDepot::populateProfileInfo(tosa::FFT2dOp op) {
 
 template <>
 void ProfileInfoDepot::populateProfileInfo(tosa::RFFT2dOp op) {
-  addValue(op.getInput());
+  addValue(op.getInputReal());
   addValue(op.getOutputReal());
   addValue(op.getOutputImag());
 }
@@ -175,6 +175,17 @@ void ProfileInfoDepot::populateProfileInfo(tosa::SelectOp op) {
 template <>
 void ProfileInfoDepot::populateProfileInfo(tosa::RescaleOp op) {
   addValue(op.getInput());
+  addValue(op.getInputZp());
+  addValue(op.getOutputZp());
+  addValue(op.getOutput());
+}
+
+template <>
+void ProfileInfoDepot::populateProfileInfo(tosa::MatMulOp op) {
+  addValue(op.getA());
+  addValue(op.getB());
+  addValue(op.getAZp());
+  addValue(op.getBZp());
   addValue(op.getOutput());
 }
 
@@ -218,6 +229,7 @@ LogicalResult ProfileInfoDepot::populatationDispatch(Operation *op) {
   POPULATE_PROFILE_INFO_CUSTOM(Resize)
   POPULATE_PROFILE_INFO_CUSTOM(Select)
   POPULATE_PROFILE_INFO_CUSTOM(Rescale)
+  POPULATE_PROFILE_INFO_CUSTOM(MatMul)
 
   // Type Invariant Extension, a capability extension that is independent
   // of the data type, meaning any compatible type can be used. No type
@@ -235,7 +247,6 @@ LogicalResult ProfileInfoDepot::populatationDispatch(Operation *op) {
   POPULATE_PROFILE_INFO_COMMON(Cast)
   POPULATE_PROFILE_INFO_COMMON(Const)
   POPULATE_PROFILE_INFO_COMMON(ArgMax)
-  POPULATE_PROFILE_INFO_COMMON(MatMul)
   POPULATE_PROFILE_INFO_COMMON(Sub)
   POPULATE_PROFILE_INFO_COMMON(Maximum)
   POPULATE_PROFILE_INFO_COMMON(Minimum)
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
index b8604ef40cc93..6b1b651a90cfb 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
@@ -104,6 +104,7 @@ struct TosaValidation : public tosa::impl::TosaValidationBase<TosaValidation> {
   }
 
   LogicalResult applyLevelCheck(Operation *op);
+  LogicalResult applyAttributeCheck(Operation *op);
 
   // check variable read/write data types against variable declarations
   LogicalResult applyVariableCheck(Operation *op);
@@ -386,6 +387,25 @@ struct TosaValidation : public tosa::impl::TosaValidationBase<TosaValidation> {
     return true;
   }
 
+  bool attributeCheckRescale(Operation *op) {
+    if (auto rescale = dyn_cast<tosa::RescaleOp>(op)) {
+      if (rescale.getRoundingMode() == "DOUBLE_ROUND" &&
+          !targetEnv.allows(Extension::doubleround)) {
+        op->emitOpError()
+            << "failed attribute check: rounding_mode = DOUBLE_ROUND "
+            << "requires extension [doubleround]";
+        return false;
+      } else if (rescale.getRoundingMode() == "INEXACT_ROUND" &&
+                 !targetEnv.allows(Extension::inexactround)) {
+        op->emitOpError()
+            << "failed attribute check: rounding_mode = INEXACT_ROUND "
+            << "requires extension [inexactround]";
+        return false;
+      }
+    }
+    return true;
+  }
+
   // configure profile and level values from pass options profileName and
   // levelName
   void configLevelAndProfile() {
@@ -415,7 +435,8 @@ struct TosaValidation : public tosa::impl::TosaValidationBase<TosaValidation> {
         } else {
           llvm::errs() << "unknown TOSA extension name passed in: " << ext
                        << ", supported extension are int16, int4, bf16, "
-                       << "fp8e4m3, fp8e5m2, fft, variable and controlflow\n";
+                       << "fp8e4m3, fp8e5m2, fft, variable, controlflow, "
+                       << "doubleround and inexactround\n";
           return signalPassFailure();
         }
       }
@@ -642,6 +663,12 @@ LogicalResult TosaValidation::applyLevelCheck(Operation *op) {
   return success();
 }
 
+LogicalResult TosaValidation::applyAttributeCheck(Operation *op) {
+  if (!attributeCheckRescale(op))
+    return failure();
+  return success();
+}
+
 inline bool CompatibleTypes(const mlir::Type &type,
                             const mlir::Type &declaredType) {
   // for now, simply use type equality comparison
@@ -936,6 +963,10 @@ void TosaValidation::runOnOperation() {
     if (failed(applyLevelCheck(op)))
       signalPassFailure();
 
+    // check additional attribute restrictions
+    if (failed(applyAttributeCheck(op)))
+      signalPassFailure();
+
     // do variable type checks
     if (failed(applyVariableCheck(op)))
       signalPassFailure();
diff --git a/mlir/lib/Dialect/Transform/Interfaces/TransformInterfaces.cpp b/mlir/lib/Dialect/Transform/Interfaces/TransformInterfaces.cpp
index 1e0ef5add358e..e0a5df0c758b3 100644
--- a/mlir/lib/Dialect/Transform/Interfaces/TransformInterfaces.cpp
+++ b/mlir/lib/Dialect/Transform/Interfaces/TransformInterfaces.cpp
@@ -1390,6 +1390,21 @@ void transform::ErrorCheckingTrackingListener::notifyPayloadReplacementNotFound(
   ++errorCounter;
 }
 
+std::string
+transform::ErrorCheckingTrackingListener::getLatestMatchFailureMessage() {
+  if (!matchFailure) {
+    return "";
+  }
+  return matchFailure->str();
+}
+
+void transform::ErrorCheckingTrackingListener::notifyMatchFailure(
+    Location loc, function_ref<void(Diagnostic &)> reasonCallback) {
+  Diagnostic diag(loc, DiagnosticSeverity::Remark);
+  reasonCallback(diag);
+  matchFailure = std::move(diag);
+}
+
 //===----------------------------------------------------------------------===//
 // TransformRewriter
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
index 241e83e234d62..20c577273d786 100644
--- a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
+++ b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
@@ -102,9 +102,7 @@ void transform::ApplyLowerBroadcastPatternsOp::populatePatterns(
 
 void transform::ApplyLowerContractionPatternsOp::populatePatterns(
     RewritePatternSet &patterns) {
-  vector::VectorTransformsOptions vectorTransformOptions;
-  vectorTransformOptions.setVectorTransformsOptions(getLoweringStrategy());
-  populateVectorContractLoweringPatterns(patterns, vectorTransformOptions,
+  populateVectorContractLoweringPatterns(patterns, getLoweringStrategy(),
                                          /*benefit=*/1,
                                          /*disableOuterProductLowering=*/true);
 }
@@ -161,9 +159,8 @@ void transform::ApplyLowerTransferPatternsOp::populatePatterns(
 
 void transform::ApplyLowerTransposePatternsOp::populatePatterns(
     RewritePatternSet &patterns) {
-  vector::populateVectorTransposeLoweringPatterns(
-      patterns, vector::VectorTransformsOptions().setVectorTransposeLowering(
-                    getLoweringStrategy()));
+  vector::populateVectorTransposeLoweringPatterns(patterns,
+                                                  getLoweringStrategy());
   if (getAvx2LoweringStrategy()) {
     auto avx2LoweringOptions =
         x86vector::avx2::LoweringOptions().setTransposeOptions(
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
index 21261478f0648..c74d0622b3828 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp
@@ -215,13 +215,13 @@ namespace {
 /// ```
 ///    %flattened_a = vector.shape_cast %a
 ///    %flattened_b = vector.shape_cast %b
-///    %flattened_d = vector.matmul %flattened_a, %flattened_b
+///    %flattened_d = vector.matrix_multiply %flattened_a, %flattened_b
 ///    %d = vector.shape_cast %%flattened_d
 ///    %e = add %c, %d
 /// ```
-/// `vector.matmul` later lowers to `llvm.matrix.multiply`.
+/// `vector.matrix_multiply` later lowers to `llvm.matrix.multiply`.
 //
-/// This only kicks in when VectorTransformsOptions is set to OuterProduct and
+/// This only kicks in when vectorContractLowering is set to Matmul and
 /// the vector.contract op is a row-major matrix multiply.
 class ContractionOpToMatmulOpLowering
     : public vector::MaskableOpRewritePattern<vector::ContractionOp> {
@@ -236,11 +236,11 @@ class ContractionOpToMatmulOpLowering
   }
 
   ContractionOpToMatmulOpLowering(
-      vector::VectorTransformsOptions vectorTransformOptions,
+      vector::VectorContractLowering vectorContractLowering,
       MLIRContext *context, PatternBenefit benefit = 1,
       FilterConstraintType constraint = defaultFilter)
       : MaskableOpRewritePattern<vector::ContractionOp>(context, benefit),
-        vectorTransformOptions(vectorTransformOptions),
+        vectorContractLowering(vectorContractLowering),
         filter(std::move(constraint)) {}
 
   FailureOr<Value>
@@ -249,7 +249,7 @@ class ContractionOpToMatmulOpLowering
 
 private:
   /// Options to control the vector patterns.
-  vector::VectorTransformsOptions vectorTransformOptions;
+  vector::VectorContractLowering vectorContractLowering;
   FilterConstraintType filter;
 };
 
@@ -266,7 +266,7 @@ class ContractionOpToMatmulOpLowering
 ///    %cK = vector.outerproduct %atRowK, %bRowK, %cK-1
 /// ```
 ///
-/// This only kicks in when VectorTransformsOptions is set to OuterProduct and
+/// This only kicks in when vectorContractLowering is set to OuterProduct and
 /// the vector.contract op is a row-major matrix multiply.
 class ContractionOpToOuterProductOpLowering
     : public MaskableOpRewritePattern<vector::ContractionOp> {
@@ -281,11 +281,11 @@ class ContractionOpToOuterProductOpLowering
   }
 
   ContractionOpToOuterProductOpLowering(
-      vector::VectorTransformsOptions vectorTransformOptions,
+      vector::VectorContractLowering vectorContractLowering,
       MLIRContext *context, PatternBenefit benefit = 1,
       FilterConstraintType constraint = defaultFilter)
       : MaskableOpRewritePattern<vector::ContractionOp>(context, benefit),
-        vectorTransformOptions(vectorTransformOptions),
+        vectorContractLowering(vectorContractLowering),
         filter(std::move(constraint)) {}
 
   FailureOr<Value>
@@ -294,7 +294,7 @@ class ContractionOpToOuterProductOpLowering
 
 private:
   /// Options to control the vector patterns.
-  vector::VectorTransformsOptions vectorTransformOptions;
+  vector::VectorContractLowering vectorContractLowering;
   FilterConstraintType filter;
 };
 
@@ -329,11 +329,11 @@ class ContractionOpToDotLowering
   }
 
   ContractionOpToDotLowering(
-      vector::VectorTransformsOptions vectorTransformOptions,
+      vector::VectorContractLowering vectorContractLowering,
       MLIRContext *context, PatternBenefit benefit = 1,
       const FilterConstraintType &constraint = defaultFilter)
       : MaskableOpRewritePattern<vector::ContractionOp>(context, benefit),
-        vectorTransformOptions(vectorTransformOptions), filter(defaultFilter) {}
+        vectorContractLowering(vectorContractLowering), filter(defaultFilter) {}
 
   FailureOr<Value>
   matchAndRewriteMaskableOp(vector::ContractionOp op, MaskingOpInterface maskOp,
@@ -341,7 +341,7 @@ class ContractionOpToDotLowering
 
 private:
   /// Options to control the vector patterns.
-  vector::VectorTransformsOptions vectorTransformOptions;
+  vector::VectorContractLowering vectorContractLowering;
   FilterConstraintType filter;
 };
 
@@ -370,11 +370,12 @@ class ContractionOpLowering
     return success();
   }
 
-  ContractionOpLowering(vector::VectorTransformsOptions vectorTransformOptions,
-                        MLIRContext *context, PatternBenefit benefit = 1,
-                        FilterConstraintType constraint = defaultFilter)
+  ContractionOpLowering(
+      vector::VectorContractLowering vectorContractLoweringOption,
+      MLIRContext *context, PatternBenefit benefit = 1,
+      FilterConstraintType constraint = defaultFilter)
       : MaskableOpRewritePattern<vector::ContractionOp>(context, benefit),
-        vectorTransformOptions(vectorTransformOptions),
+        vectorContractLoweringOption(vectorContractLoweringOption),
         filter(std::move(constraint)) {}
 
   FailureOr<Value>
@@ -383,7 +384,7 @@ class ContractionOpLowering
 
 private:
   /// Options to control the vector patterns.
-  vector::VectorTransformsOptions vectorTransformOptions;
+  vector::VectorContractLowering vectorContractLoweringOption;
   FilterConstraintType filter;
   // Lower one parallel dimension.
   FailureOr<Value> lowerParallel(PatternRewriter &rewriter,
@@ -635,14 +636,13 @@ struct UnrolledOuterProductGenerator
 ///    %cK = vector.outerproduct %atRowK, %bRowK, %cK-1
 /// ```
 ///
-/// This only kicks in when VectorTransformsOptions is set to OuterProduct but
+/// This only kicks in when vectorContractLowering is set to OuterProduct but
 /// otherwise supports any layout permutation of the matrix-multiply.
 FailureOr<Value>
 ContractionOpToOuterProductOpLowering::matchAndRewriteMaskableOp(
     vector::ContractionOp op, MaskingOpInterface maskOp,
     PatternRewriter &rewriter) const {
-  if (vectorTransformOptions.vectorContractLowering !=
-      vector::VectorContractLowering::OuterProduct)
+  if (vectorContractLowering != vector::VectorContractLowering::OuterProduct)
     return failure();
 
   if (failed(filter(op)))
@@ -672,8 +672,7 @@ FailureOr<Value> ContractionOpToDotLowering::matchAndRewriteMaskableOp(
   if (failed(filter(op)))
     return failure();
 
-  if (vectorTransformOptions.vectorContractLowering !=
-      vector::VectorContractLowering::Dot)
+  if (vectorContractLowering != vector::VectorContractLowering::Dot)
     return failure();
 
   auto iteratorTypes = op.getIteratorTypes().getValue();
@@ -789,11 +788,11 @@ struct ContractOpToElementwise
     return success();
   }
   ContractOpToElementwise(
-      vector::VectorTransformsOptions vectorTransformOptions,
+      vector::VectorContractLowering vectorContractLowering,
       MLIRContext *context, PatternBenefit benefit = 1,
       const FilterConstraintType &constraint = defaultFilter)
       : MaskableOpRewritePattern<vector::ContractionOp>(context, benefit),
-        vectorTransformOptions(vectorTransformOptions), filter(defaultFilter) {}
+        vectorContractLowering(vectorContractLowering), filter(defaultFilter) {}
 
   FailureOr<Value>
   matchAndRewriteMaskableOp(vector::ContractionOp contractOp,
@@ -806,8 +805,7 @@ struct ContractOpToElementwise
     if (failed(filter(contractOp)))
       return failure();
 
-    if (vectorTransformOptions.vectorContractLowering !=
-        vector::VectorContractLowering::ParallelArith)
+    if (vectorContractLowering != vector::VectorContractLowering::ParallelArith)
       return failure();
 
     ArrayRef<int64_t> lhsShape = contractOp.getLhsType().getShape();
@@ -898,7 +896,7 @@ struct ContractOpToElementwise
 
 private:
   /// Options to control the vector patterns.
-  vector::VectorTransformsOptions vectorTransformOptions;
+  vector::VectorContractLowering vectorContractLowering;
   FilterConstraintType filter;
 };
 
@@ -913,7 +911,7 @@ struct ContractOpToElementwise
 /// until a pure contraction is reached (no free/batch dimensions),
 /// which is replaced by a dot-product.
 ///
-/// This only kicks in when either VectorTransformsOptions is set
+/// This only kicks in when either vectorContractLoweringOption is set
 /// to DOT or when other contraction patterns fail.
 //
 // TODO: break down into transpose/reshape/cast ops
@@ -941,25 +939,25 @@ FailureOr<Value> ContractionOpLowering::matchAndRewriteMaskableOp(
   // TODO: implement benefits, cost models.
   MLIRContext *ctx = op.getContext();
 
-  ContractionOpToMatmulOpLowering pat1(vectorTransformOptions, ctx);
+  ContractionOpToMatmulOpLowering pat1(vectorContractLoweringOption, ctx);
   FailureOr<Value> newVal1 =
       pat1.matchAndRewriteMaskableOp(op, maskOp, rewriter);
   if (!failed(newVal1))
     return newVal1;
 
-  ContractionOpToOuterProductOpLowering pat2(vectorTransformOptions, ctx);
+  ContractionOpToOuterProductOpLowering pat2(vectorContractLoweringOption, ctx);
   FailureOr<Value> newVal2 =
       pat2.matchAndRewriteMaskableOp(op, maskOp, rewriter);
   if (!failed(newVal2))
     return newVal2;
 
-  ContractionOpToDotLowering pat3(vectorTransformOptions, ctx);
+  ContractionOpToDotLowering pat3(vectorContractLoweringOption, ctx);
   FailureOr<Value> newVal3 =
       pat3.matchAndRewriteMaskableOp(op, maskOp, rewriter);
   if (!failed(newVal3))
     return newVal3;
 
-  ContractOpToElementwise pat4(vectorTransformOptions, ctx);
+  ContractOpToElementwise pat4(vectorContractLoweringOption, ctx);
   FailureOr<Value> newVal4 =
       pat4.matchAndRewriteMaskableOp(op, maskOp, rewriter);
   if (!failed(newVal4))
@@ -1273,14 +1271,14 @@ class OuterProductOpLowering : public OpRewritePattern<vector::OuterProductOp> {
 ///    %mtb = maybe_transpose
 ///    %flattened_a = vector.shape_cast %mta
 ///    %flattened_b = vector.shape_cast %mtb
-///    %flattened_d = vector.matmul %flattened_a, %flattened_b
+///    %flattened_d = vector.matrix_multiply %flattened_a, %flattened_b
 ///    %mtd = vector.shape_cast %flattened_d
 ///    %d = maybe_untranspose %mtd
 ///    %e = add %c, %d
 /// ```
-/// `vector.matmul` later lowers to `llvm.matrix.multiply`.
+/// `vector.matrix_multiply` later lowers to `llvm.matrix.multiply`.
 //
-/// This only kicks in when VectorTransformsOptions is set to `Matmul`.
+/// This only kicks in when vectorContractLowering is set to `Matmul`.
 /// vector.transpose operations are inserted if the vector.contract op is not a
 /// row-major matrix multiply.
 ///
@@ -1292,8 +1290,7 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
   if (maskOp)
     return failure();
 
-  if (vectorTransformOptions.vectorContractLowering !=
-      vector::VectorContractLowering::Matmul)
+  if (vectorContractLowering != vector::VectorContractLowering::Matmul)
     return failure();
   if (failed(filter(op)))
     return failure();
@@ -1382,13 +1379,14 @@ FailureOr<Value> ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp(
 } // namespace
 
 void mlir::vector::populateVectorContractLoweringPatterns(
-    RewritePatternSet &patterns, VectorTransformsOptions options,
-    PatternBenefit benefit, bool disableOuterProductLowering) {
+    RewritePatternSet &patterns,
+    VectorContractLowering vectorContractLoweringOption, PatternBenefit benefit,
+    bool disableOuterProductLowering) {
   if (!disableOuterProductLowering)
     patterns.add<OuterProductOpLowering>(patterns.getContext(), benefit);
   patterns.add<ContractionOpLowering, ContractionOpToMatmulOpLowering,
                ContractionOpToOuterProductOpLowering>(
-      options, patterns.getContext(), benefit);
+      vectorContractLoweringOption, patterns.getContext(), benefit);
 }
 
 void mlir::vector::populateVectorOuterProductLoweringPatterns(
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
index fb4dee33bc5f5..732e316c93381 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp
@@ -304,10 +304,10 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
 public:
   using OpRewritePattern::OpRewritePattern;
 
-  TransposeOpLowering(vector::VectorTransformsOptions vectorTransformOptions,
+  TransposeOpLowering(vector::VectorTransposeLowering vectorTransposeLowering,
                       MLIRContext *context, PatternBenefit benefit = 1)
       : OpRewritePattern<vector::TransposeOp>(context, benefit),
-        vectorTransformOptions(vectorTransformOptions) {}
+        vectorTransposeLowering(vectorTransposeLowering) {}
 
   LogicalResult matchAndRewrite(vector::TransposeOp op,
                                 PatternRewriter &rewriter) const override {
@@ -324,14 +324,13 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
     // Set up convenience transposition table.
     ArrayRef<int64_t> transp = op.getPermutation();
 
-    if (isShuffleLike(vectorTransformOptions.vectorTransposeLowering) &&
+    if (isShuffleLike(vectorTransposeLowering) &&
         succeeded(isTranspose2DSlice(op)))
       return rewriter.notifyMatchFailure(
           op, "Options specifies lowering to shuffle");
 
     // Handle a true 2-D matrix transpose differently when requested.
-    if (vectorTransformOptions.vectorTransposeLowering ==
-            vector::VectorTransposeLowering::Flat &&
+    if (vectorTransposeLowering == vector::VectorTransposeLowering::Flat &&
         resType.getRank() == 2 && transp[0] == 1 && transp[1] == 0) {
       Type flattenedType =
           VectorType::get(resType.getNumElements(), resType.getElementType());
@@ -380,7 +379,7 @@ class TransposeOpLowering : public OpRewritePattern<vector::TransposeOp> {
 
 private:
   /// Options to control the vector patterns.
-  vector::VectorTransformsOptions vectorTransformOptions;
+  vector::VectorTransposeLowering vectorTransposeLowering;
 };
 
 /// Rewrites vector.transpose as vector.shape_cast. This pattern is only applied
@@ -454,14 +453,14 @@ class TransposeOp2DToShuffleLowering
   using OpRewritePattern::OpRewritePattern;
 
   TransposeOp2DToShuffleLowering(
-      vector::VectorTransformsOptions vectorTransformOptions,
+      vector::VectorTransposeLowering vectorTransposeLowering,
       MLIRContext *context, PatternBenefit benefit = 1)
       : OpRewritePattern<vector::TransposeOp>(context, benefit),
-        vectorTransformOptions(vectorTransformOptions) {}
+        vectorTransposeLowering(vectorTransposeLowering) {}
 
   LogicalResult matchAndRewrite(vector::TransposeOp op,
                                 PatternRewriter &rewriter) const override {
-    if (!isShuffleLike(vectorTransformOptions.vectorTransposeLowering))
+    if (!isShuffleLike(vectorTransposeLowering))
       return rewriter.notifyMatchFailure(
           op, "not using vector shuffle based lowering");
 
@@ -487,8 +486,7 @@ class TransposeOp2DToShuffleLowering
                                                           op.getVector());
 
     Value res;
-    if (vectorTransformOptions.vectorTransposeLowering ==
-            VectorTransposeLowering::Shuffle16x16 &&
+    if (vectorTransposeLowering == VectorTransposeLowering::Shuffle16x16 &&
         m == 16 && n == 16) {
       reshInput =
           rewriter.create<vector::ShapeCastOp>(loc, reshInputType, reshInput);
@@ -506,15 +504,15 @@ class TransposeOp2DToShuffleLowering
 
 private:
   /// Options to control the vector patterns.
-  vector::VectorTransformsOptions vectorTransformOptions;
+  vector::VectorTransposeLowering vectorTransposeLowering;
 };
 } // namespace
 
 void mlir::vector::populateVectorTransposeLoweringPatterns(
-    RewritePatternSet &patterns, VectorTransformsOptions options,
-    PatternBenefit benefit) {
+    RewritePatternSet &patterns,
+    VectorTransposeLowering vectorTransposeLowering, PatternBenefit benefit) {
   patterns.add<Transpose2DWithUnitDimToShapeCast>(patterns.getContext(),
                                                   benefit);
   patterns.add<TransposeOpLowering, TransposeOp2DToShuffleLowering>(
-      options, patterns.getContext(), benefit);
+      vectorTransposeLowering, patterns.getContext(), benefit);
 }
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
index 2413a4126f3f7..8c9e2d889808a 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
@@ -772,17 +772,16 @@ class FlattenContiguousRowMajorTransferWritePattern
 /// `vector.extract` and `vector.extract_element`.
 template <class VectorExtractOp>
 class RewriteScalarExtractOfTransferReadBase
-    : public OpRewritePattern<VectorExtractOp>::SplitMatchAndRewrite {
-  using Base = typename OpRewritePattern<VectorExtractOp>::SplitMatchAndRewrite;
+    : public OpRewritePattern<VectorExtractOp> {
+  using Base = OpRewritePattern<VectorExtractOp>;
 
 public:
   RewriteScalarExtractOfTransferReadBase(MLIRContext *context,
                                          PatternBenefit benefit,
                                          bool allowMultipleUses)
-      : Base::SplitMatchAndRewrite(context, benefit),
-        allowMultipleUses(allowMultipleUses) {}
+      : Base(context, benefit), allowMultipleUses(allowMultipleUses) {}
 
-  LogicalResult match(VectorExtractOp extractOp) const override {
+  LogicalResult match(VectorExtractOp extractOp) const {
     auto xferOp =
         extractOp.getVector().template getDefiningOp<vector::TransferReadOp>();
     if (!xferOp)
@@ -828,8 +827,11 @@ class RewriteScalarExtractElementOfTransferRead
   using RewriteScalarExtractOfTransferReadBase::
       RewriteScalarExtractOfTransferReadBase;
 
-  void rewrite(vector::ExtractElementOp extractOp,
-               PatternRewriter &rewriter) const override {
+  LogicalResult matchAndRewrite(vector::ExtractElementOp extractOp,
+                                PatternRewriter &rewriter) const override {
+    if (failed(match(extractOp)))
+      return failure();
+
     // Construct scalar load.
     auto loc = extractOp.getLoc();
     auto xferOp = extractOp.getVector().getDefiningOp<vector::TransferReadOp>();
@@ -856,6 +858,8 @@ class RewriteScalarExtractElementOfTransferRead
       rewriter.replaceOpWithNewOp<tensor::ExtractOp>(
           extractOp, xferOp.getSource(), newIndices);
     }
+
+    return success();
   }
 };
 
@@ -872,8 +876,11 @@ class RewriteScalarExtractOfTransferRead
   using RewriteScalarExtractOfTransferReadBase::
       RewriteScalarExtractOfTransferReadBase;
 
-  void rewrite(vector::ExtractOp extractOp,
-               PatternRewriter &rewriter) const override {
+  LogicalResult matchAndRewrite(vector::ExtractOp extractOp,
+                                PatternRewriter &rewriter) const override {
+    if (failed(match(extractOp)))
+      return failure();
+
     // Construct scalar load.
     auto xferOp = extractOp.getVector().getDefiningOp<vector::TransferReadOp>();
     SmallVector<Value> newIndices(xferOp.getIndices().begin(),
@@ -899,6 +906,8 @@ class RewriteScalarExtractOfTransferRead
       rewriter.replaceOpWithNewOp<tensor::ExtractOp>(
           extractOp, xferOp.getSource(), newIndices);
     }
+
+    return success();
   }
 };
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
index 08ba972b12ce6..04c38f9f7b2e3 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
@@ -355,6 +355,11 @@ struct UnrollMultiReductionPattern
 
   LogicalResult matchAndRewrite(vector::MultiDimReductionOp reductionOp,
                                 PatternRewriter &rewriter) const override {
+    auto resultType = reductionOp->getResult(0).getType();
+    if (resultType.isIntOrFloat()) {
+      return rewriter.notifyMatchFailure(reductionOp,
+                                         "Unrolling scalars is not supported");
+    }
     std::optional<SmallVector<int64_t>> targetShape =
         getTargetShape(options, reductionOp);
     if (!targetShape)
diff --git a/mlir/lib/Interfaces/DataLayoutInterfaces.cpp b/mlir/lib/Interfaces/DataLayoutInterfaces.cpp
index 2d098e2db6a96..7afa6ac3a00e4 100644
--- a/mlir/lib/Interfaces/DataLayoutInterfaces.cpp
+++ b/mlir/lib/Interfaces/DataLayoutInterfaces.cpp
@@ -246,6 +246,16 @@ Attribute mlir::detail::getDefaultEndianness(DataLayoutEntryInterface entry) {
   return entry.getValue();
 }
 
+// Returns the default memory space if specified in the given entry. If the
+// entry is empty the default memory space represented by an empty attribute is
+// returned.
+Attribute mlir::detail::getDefaultMemorySpace(DataLayoutEntryInterface entry) {
+  if (!entry)
+    return Attribute();
+
+  return entry.getValue();
+}
+
 // Returns the memory space used for alloca operations if specified in the
 // given entry. If the entry is empty the default memory space represented by
 // an empty attribute is returned.
@@ -605,6 +615,22 @@ mlir::Attribute mlir::DataLayout::getEndianness() const {
   return *endianness;
 }
 
+mlir::Attribute mlir::DataLayout::getDefaultMemorySpace() const {
+  checkValid();
+  if (defaultMemorySpace)
+    return *defaultMemorySpace;
+  DataLayoutEntryInterface entry;
+  if (originalLayout)
+    entry = originalLayout.getSpecForIdentifier(
+        originalLayout.getDefaultMemorySpaceIdentifier(
+            originalLayout.getContext()));
+  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
+    defaultMemorySpace = iface.getDefaultMemorySpace(entry);
+  else
+    defaultMemorySpace = detail::getDefaultMemorySpace(entry);
+  return *defaultMemorySpace;
+}
+
 mlir::Attribute mlir::DataLayout::getAllocaMemorySpace() const {
   checkValid();
   if (allocaMemorySpace)
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 32c7c501d03c3..3373f19a006ba 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -168,10 +168,6 @@ static LogicalResult checkImplementationStatus(Operation &op) {
     if (op.getDistScheduleChunkSize())
       result = todo("dist_schedule with chunk_size");
   };
-  auto checkHasDeviceAddr = [&todo](auto op, LogicalResult &result) {
-    if (!op.getHasDeviceAddrVars().empty())
-      result = todo("has_device_addr");
-  };
   auto checkHint = [](auto op, LogicalResult &) {
     if (op.getHint())
       op.emitWarning("hint clause discarded");
@@ -311,7 +307,6 @@ static LogicalResult checkImplementationStatus(Operation &op) {
         checkAllocate(op, result);
         checkBare(op, result);
         checkDevice(op, result);
-        checkHasDeviceAddr(op, result);
         checkInReduction(op, result);
         checkIsDevicePtr(op, result);
         checkPrivate(op, result);
@@ -1452,6 +1447,12 @@ allocatePrivateVars(llvm::IRBuilderBase &builder,
 
   llvm::BasicBlock *afterAllocas = allocaTerminator->getSuccessor(0);
 
+  unsigned int allocaAS =
+      moduleTranslation.getLLVMModule()->getDataLayout().getAllocaAddrSpace();
+  unsigned int defaultAS = moduleTranslation.getLLVMModule()
+                               ->getDataLayout()
+                               .getProgramAddressSpace();
+
   for (auto [privDecl, mlirPrivVar, blockArg] :
        llvm::zip_equal(privateDecls, mlirPrivateVars, privateBlockArgs)) {
     llvm::Type *llvmAllocType =
@@ -1459,6 +1460,10 @@ allocatePrivateVars(llvm::IRBuilderBase &builder,
     builder.SetInsertPoint(allocaIP.getBlock()->getTerminator());
     llvm::Value *llvmPrivateVar = builder.CreateAlloca(
         llvmAllocType, /*ArraySize=*/nullptr, "omp.private.alloc");
+    if (allocaAS != defaultAS)
+      llvmPrivateVar = builder.CreateAddrSpaceCast(llvmPrivateVar,
+                                                   builder.getPtrTy(defaultAS));
+
     llvmPrivateVars.push_back(llvmPrivateVar);
   }
 
@@ -3173,8 +3178,9 @@ llvm::Value *getSizeInBytes(DataLayout &dl, const mlir::Type &type,
 static void collectMapDataFromMapOperands(
     MapInfoData &mapData, SmallVectorImpl<Value> &mapVars,
     LLVM::ModuleTranslation &moduleTranslation, DataLayout &dl,
-    llvm::IRBuilderBase &builder, const ArrayRef<Value> &useDevPtrOperands = {},
-    const ArrayRef<Value> &useDevAddrOperands = {}) {
+    llvm::IRBuilderBase &builder, ArrayRef<Value> useDevPtrOperands = {},
+    ArrayRef<Value> useDevAddrOperands = {},
+    ArrayRef<Value> hasDevAddrOperands = {}) {
   auto checkIsAMember = [](const auto &mapVars, auto mapOp) {
     // Check if this is a member mapping and correctly assign that it is, if
     // it is a member of a larger object.
@@ -3280,6 +3286,52 @@ static void collectMapDataFromMapOperands(
 
   addDevInfos(useDevAddrOperands, llvm::OpenMPIRBuilder::DeviceInfoTy::Address);
   addDevInfos(useDevPtrOperands, llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer);
+
+  for (Value mapValue : hasDevAddrOperands) {
+    auto mapOp = cast<omp::MapInfoOp>(mapValue.getDefiningOp());
+    Value offloadPtr =
+        mapOp.getVarPtrPtr() ? mapOp.getVarPtrPtr() : mapOp.getVarPtr();
+    llvm::Value *origValue = moduleTranslation.lookupValue(offloadPtr);
+    auto mapType = static_cast<llvm::omp::OpenMPOffloadMappingFlags>(
+        mapOp.getMapType().value());
+    auto mapTypeAlways = llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
+
+    mapData.OriginalValue.push_back(origValue);
+    mapData.BasePointers.push_back(origValue);
+    mapData.Pointers.push_back(origValue);
+    mapData.IsDeclareTarget.push_back(false);
+    mapData.BaseType.push_back(
+        moduleTranslation.convertType(mapOp.getVarType()));
+    mapData.Sizes.push_back(
+        builder.getInt64(dl.getTypeSize(mapOp.getVarType())));
+    mapData.MapClause.push_back(mapOp.getOperation());
+    if (llvm::to_underlying(mapType & mapTypeAlways)) {
+      // Descriptors are mapped with the ALWAYS flag, since they can get
+      // rematerialized, so the address of the decriptor for a given object
+      // may change from one place to another.
+      mapData.Types.push_back(mapType);
+      // Technically it's possible for a non-descriptor mapping to have
+      // both has-device-addr and ALWAYS, so lookup the mapper in case it
+      // exists.
+      if (mapOp.getMapperId()) {
+        mapData.Mappers.push_back(
+            SymbolTable::lookupNearestSymbolFrom<omp::DeclareMapperOp>(
+                mapOp, mapOp.getMapperIdAttr()));
+      } else {
+        mapData.Mappers.push_back(nullptr);
+      }
+    } else {
+      mapData.Types.push_back(
+          llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
+      mapData.Mappers.push_back(nullptr);
+    }
+    mapData.Names.push_back(LLVM::createMappingInformation(
+        mapOp.getLoc(), *moduleTranslation.getOpenMPBuilder()));
+    mapData.DevicePointers.push_back(
+        llvm::OpenMPIRBuilder::DeviceInfoTy::Address);
+    mapData.IsAMapping.push_back(false);
+    mapData.IsAMember.push_back(checkIsAMember(hasDevAddrOperands, mapOp));
+  }
 }
 
 static int getMapDataMemberIdx(MapInfoData &mapData, omp::MapInfoOp memberOp) {
@@ -3982,7 +4034,6 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
     return failure();
 
   using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
-
   MapInfoData mapData;
   collectMapDataFromMapOperands(mapData, mapVars, moduleTranslation, DL,
                                 builder, useDevicePtrVars, useDeviceAddrVars);
@@ -4695,7 +4746,9 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
   llvm::DenseMap<Value, Value> mappedPrivateVars;
   DataLayout dl = DataLayout(opInst.getParentOfType<ModuleOp>());
   SmallVector<Value> mapVars = targetOp.getMapVars();
+  SmallVector<Value> hdaVars = targetOp.getHasDeviceAddrVars();
   ArrayRef<BlockArgument> mapBlockArgs = argIface.getMapBlockArgs();
+  ArrayRef<BlockArgument> hdaBlockArgs = argIface.getHasDeviceAddrBlockArgs();
   llvm::Function *llvmOutlinedFn = nullptr;
 
   // TODO: It can also be false if a compile-time constant `false` IF clause is
@@ -4780,6 +4833,12 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
           moduleTranslation.lookupValue(mapInfoOp.getVarPtr());
       moduleTranslation.mapValue(arg, mapOpValue);
     }
+    for (auto [arg, mapOp] : llvm::zip_equal(hdaBlockArgs, hdaVars)) {
+      auto mapInfoOp = cast<omp::MapInfoOp>(mapOp.getDefiningOp());
+      llvm::Value *mapOpValue =
+          moduleTranslation.lookupValue(mapInfoOp.getVarPtr());
+      moduleTranslation.mapValue(arg, mapOpValue);
+    }
 
     // Do privatization after moduleTranslation has already recorded
     // mapped values.
@@ -4847,7 +4906,8 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
 
   MapInfoData mapData;
   collectMapDataFromMapOperands(mapData, mapVars, moduleTranslation, dl,
-                                builder);
+                                builder, /*useDevPtrOperands=*/{},
+                                /*useDevAddrOperands=*/{}, hdaVars);
 
   MapInfosTy combinedInfos;
   auto genMapInfoCB =
@@ -4901,7 +4961,7 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
       kernelInput.push_back(value);
   }
 
-  for (size_t i = 0; i < mapVars.size(); ++i) {
+  for (size_t i = 0, e = mapData.OriginalValue.size(); i != e; ++i) {
     // declare target arguments are not passed to kernels as arguments
     // TODO: We currently do not handle cases where a member is explicitly
     // passed in as an argument, this will likley need to be handled in
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index cff5a1940e77e..f24c2777cbbb8 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -2263,10 +2263,15 @@ LogicalResult ModuleImport::convertInvokeAttributes(llvm::InvokeInst *inst,
 LogicalResult ModuleImport::convertCallAttributes(llvm::CallInst *inst,
                                                   CallOp op) {
   setFastmathFlagsAttr(inst, op.getOperation());
+  // Query the attributes directly instead of using `inst->getFnAttr(Kind)`, the
+  // latter does additional lookup to the parent and inherits, changing the
+  // semantics too early.
+  llvm::AttributeList callAttrs = inst->getAttributes();
+
   op.setTailCallKind(convertTailCallKindFromLLVM(inst->getTailCallKind()));
-  op.setConvergent(inst->isConvergent());
-  op.setNoUnwind(inst->doesNotThrow());
-  op.setWillReturn(inst->hasFnAttr(llvm::Attribute::WillReturn));
+  op.setConvergent(callAttrs.getFnAttr(llvm::Attribute::Convergent).isValid());
+  op.setNoUnwind(callAttrs.getFnAttr(llvm::Attribute::NoUnwind).isValid());
+  op.setWillReturn(callAttrs.getFnAttr(llvm::Attribute::WillReturn).isValid());
 
   llvm::MemoryEffects memEffects = inst->getMemoryEffects();
   ModRefInfo othermem = convertModRefInfoFromLLVM(
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index b1bc26a545f62..8503a7f6b4fb5 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -1494,9 +1494,6 @@ LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) {
   if (auto tuneCpu = func.getTuneCpu())
     llvmFunc->addFnAttr("tune-cpu", *tuneCpu);
 
-  if (auto targetFeatures = func.getTargetFeatures())
-    llvmFunc->addFnAttr("target-features", targetFeatures->getFeaturesString());
-
   if (auto attr = func.getVscaleRange())
     llvmFunc->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs(
         getLLVMContext(), attr->getMinRange().getInt(),
@@ -1528,12 +1525,6 @@ LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) {
   if (auto fpContract = func.getFpContract())
     llvmFunc->addFnAttr("fp-contract", *fpContract);
 
-  // Add function attribute frame-pointer, if found.
-  if (FramePointerKindAttr attr = func.getFramePointerAttr())
-    llvmFunc->addFnAttr("frame-pointer",
-                        LLVM::framePointerKind::stringifyFramePointerKind(
-                            (attr.getFramePointerKind())));
-
   // First, create all blocks so we can jump to them.
   llvm::LLVMContext &llvmContext = llvmFunc->getContext();
   for (auto &bb : func) {
@@ -1606,6 +1597,11 @@ static void convertFunctionAttributes(LLVMFuncOp func,
     llvmFunc->addFnAttr(llvm::Attribute::NoUnwind);
   if (func.getWillReturnAttr())
     llvmFunc->addFnAttr(llvm::Attribute::WillReturn);
+  if (TargetFeaturesAttr targetFeatAttr = func.getTargetFeaturesAttr())
+    llvmFunc->addFnAttr("target-features", targetFeatAttr.getFeaturesString());
+  if (FramePointerKindAttr fpAttr = func.getFramePointerAttr())
+    llvmFunc->addFnAttr("frame-pointer", stringifyFramePointerKind(
+                                             fpAttr.getFramePointerKind()));
   convertFunctionMemoryAttributes(func, llvmFunc);
 }
 
diff --git a/mlir/lib/Transforms/Utils/RegionUtils.cpp b/mlir/lib/Transforms/Utils/RegionUtils.cpp
index e55ef6eb66b9c..da0d486f0fdcb 100644
--- a/mlir/lib/Transforms/Utils/RegionUtils.cpp
+++ b/mlir/lib/Transforms/Utils/RegionUtils.cpp
@@ -7,9 +7,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Transforms/RegionUtils.h"
+
+#include "mlir/Analysis/SliceAnalysis.h"
 #include "mlir/Analysis/TopologicalSortUtils.h"
 #include "mlir/IR/Block.h"
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/Dominance.h"
 #include "mlir/IR/IRMapping.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/PatternMatch.h"
@@ -1054,3 +1057,61 @@ LogicalResult mlir::simplifyRegions(RewriterBase &rewriter,
   return success(eliminatedBlocks || eliminatedOpsOrArgs ||
                  mergedIdenticalBlocks || droppedRedundantArguments);
 }
+
+//===---------------------------------------------------------------------===//
+// Move operation dependencies
+//===---------------------------------------------------------------------===//
+
+LogicalResult mlir::moveOperationDependencies(RewriterBase &rewriter,
+                                              Operation *op,
+                                              Operation *insertionPoint,
+                                              DominanceInfo &dominance) {
+  // Currently unsupported case where the op and insertion point are
+  // in different basic blocks.
+  if (op->getBlock() != insertionPoint->getBlock()) {
+    return rewriter.notifyMatchFailure(
+        op, "unsupported caes where operation and insertion point are not in "
+            "the same basic block");
+  }
+  // If `insertionPoint` does not dominate `op`, do nothing
+  if (!dominance.properlyDominates(insertionPoint, op)) {
+    return rewriter.notifyMatchFailure(op,
+                                       "insertion point does not dominate op");
+  }
+
+  // Find the backward slice of operation for each `Value` the operation
+  // depends on. Prune the slice to only include operations not already
+  // dominated by the `insertionPoint`
+  BackwardSliceOptions options;
+  options.inclusive = false;
+  options.omitUsesFromAbove = false;
+  // Since current support is to only move within a same basic block,
+  // the slices dont need to look past block arguments.
+  options.omitBlockArguments = true;
+  options.filter = [&](Operation *sliceBoundaryOp) {
+    return !dominance.properlyDominates(sliceBoundaryOp, insertionPoint);
+  };
+  llvm::SetVector<Operation *> slice;
+  getBackwardSlice(op, &slice, options);
+
+  // If the slice contains `insertionPoint` cannot move the dependencies.
+  if (slice.contains(insertionPoint)) {
+    return rewriter.notifyMatchFailure(
+        op,
+        "cannot move dependencies before operation in backward slice of op");
+  }
+
+  // We should move the slice in topological order, but `getBackwardSlice`
+  // already does that. So no need to sort again.
+  for (Operation *op : slice) {
+    rewriter.moveOpBefore(op, insertionPoint);
+  }
+  return success();
+}
+
+LogicalResult mlir::moveOperationDependencies(RewriterBase &rewriter,
+                                              Operation *op,
+                                              Operation *insertionPoint) {
+  DominanceInfo dominance(op);
+  return moveOperationDependencies(rewriter, op, insertionPoint, dominance);
+}
diff --git a/mlir/python/mlir/_mlir_libs/__init__.py b/mlir/python/mlir/_mlir_libs/__init__.py
index d021dde05dd87..083a9075fe4c5 100644
--- a/mlir/python/mlir/_mlir_libs/__init__.py
+++ b/mlir/python/mlir/_mlir_libs/__init__.py
@@ -148,13 +148,25 @@ def process_initializer_module(module_name):
             break
 
     class Context(ir._BaseContext):
-        def __init__(self, load_on_create_dialects=None, *args, **kwargs):
+        def __init__(
+            self, load_on_create_dialects=None, thread_pool=None, *args, **kwargs
+        ):
             super().__init__(*args, **kwargs)
             self.append_dialect_registry(get_dialect_registry())
             for hook in post_init_hooks:
                 hook(self)
+            if disable_multithreading and thread_pool is not None:
+                raise ValueError(
+                    "Context constructor has given thread_pool argument, "
+                    "but disable_multithreading flag is True. "
+                    "Please, set thread_pool argument to None or "
+                    "set disable_multithreading flag to False."
+                )
             if not disable_multithreading:
-                self.enable_multithreading(True)
+                if thread_pool is None:
+                    self.enable_multithreading(True)
+                else:
+                    self.set_thread_pool(thread_pool)
             if load_on_create_dialects is not None:
                 logger.debug(
                     "Loading all dialects from load_on_create_dialects arg %r",
diff --git a/mlir/python/mlir/dialects/NVGPUOps.td b/mlir/python/mlir/dialects/NVGPUOps.td
index ae54822cd9070..cdf651901e074 100644
--- a/mlir/python/mlir/dialects/NVGPUOps.td
+++ b/mlir/python/mlir/dialects/NVGPUOps.td
@@ -9,6 +9,6 @@
 #ifndef PYTHON_BINDINGS_NVGPU_OPS
 #define PYTHON_BINDINGS_NVGPU_OPS
 
-include "mlir/Dialect/NVGPU/IR/NVGPU.td"
+include "mlir/Dialect/NVGPU/IR/NVGPUOps.td"
 
 #endif
diff --git a/mlir/test/Conversion/TosaToArith/tosa-to-arith-invalid.mlir b/mlir/test/Conversion/TosaToArith/tosa-to-arith-invalid.mlir
new file mode 100644
index 0000000000000..4b324955439aa
--- /dev/null
+++ b/mlir/test/Conversion/TosaToArith/tosa-to-arith-invalid.mlir
@@ -0,0 +1,8 @@
+// RUN: mlir-opt --split-input-file --tosa-to-arith="include-apply-rescale=true use-32-bit=true" %s -verify-diagnostics
+
+// CHECK-LABEL: @apply_scale_unsupported_inexact_round
+func.func @apply_scale_unsupported_inexact_round(%arg0 : i64, %arg1 : i32, %arg2 : i8) -> (i32) {
+  // expected-error@+1 {{failed to legalize operation 'tosa.apply_scale'}}
+  %res = tosa.apply_scale %arg0, %arg1, %arg2 {rounding_mode = "INEXACT_ROUND"} : (i64, i32, i8) -> i32
+  return %res : i32
+}
diff --git a/mlir/test/Conversion/TosaToArith/tosa-to-arith.mlir b/mlir/test/Conversion/TosaToArith/tosa-to-arith.mlir
index 14f811727c456..db68ca40879f4 100644
--- a/mlir/test/Conversion/TosaToArith/tosa-to-arith.mlir
+++ b/mlir/test/Conversion/TosaToArith/tosa-to-arith.mlir
@@ -67,7 +67,7 @@ func.func @apply_scale_test_i32(%arg0 : i32, %arg1 : i32, %arg2 : i8) -> (i32) {
   // CHECK-DAG: %[[LOWALIGN:.+]] = arith.select %[[OVER31]], %[[C0]], %[[LOR]]
   // CHECK-DAG: %[[RESULT:.+]] = arith.addi %[[LOWALIGN]], %[[HIALIGN]]
   // CHECK: return %[[RESULT]]
-  %res = tosa.apply_scale %arg0, %arg1, %arg2 {double_round = true} : (i32, i32, i8) -> i32
+  %res = tosa.apply_scale %arg0, %arg1, %arg2 {rounding_mode = "DOUBLE_ROUND"} : (i32, i32, i8) -> i32
   return %res : i32
 }
 
@@ -77,7 +77,7 @@ func.func @apply_scale_test_i32(%arg0 : i32, %arg1 : i32, %arg2 : i8) -> (i32) {
 // SCALE: tosa.apply_scale
 func.func @apply_scale_test_vector(%arg0 : vector<4xi32>, %arg1 : vector<4xi32>, %arg2 : vector<4xi8>) -> (vector<4xi32>) {
   // CHECK-NOT: "tosa.apply_scale"
-  %res = tosa.apply_scale %arg0, %arg1, %arg2 {double_round = true} : (vector<4xi32>, vector<4xi32>, vector<4xi8>) -> vector<4xi32>
+  %res = tosa.apply_scale %arg0, %arg1, %arg2 {rounding_mode = "DOUBLE_ROUND"} : (vector<4xi32>, vector<4xi32>, vector<4xi8>) -> vector<4xi32>
   return %res : vector<4xi32>
 }
 
@@ -115,7 +115,7 @@ func.func @apply_scale_test_i48(%arg0 : i48, %arg1 : i32, %arg2 : i8) -> (i32) {
   // CHECK-DAG: %[[SHR:.+]] = arith.shrsi %[[RES64]], %[[S64]]
   // CHECK-DAG: %[[TRUNC:.+]] = arith.trunci %[[SHR]] : i64 to i32
   // CHECK: return %[[TRUNC]]
-  %res = tosa.apply_scale %arg0, %arg1, %arg2 {double_round = true} : (i48, i32, i8) -> i32
+  %res = tosa.apply_scale %arg0, %arg1, %arg2 {rounding_mode = "DOUBLE_ROUND"} : (i48, i32, i8) -> i32
   return %res : i32
 }
 
@@ -152,6 +152,6 @@ func.func @apply_scale_test_i64(%arg0 : i64, %arg1 : i32, %arg2 : i8) -> (i32) {
   // CHECK-DAG: %[[SHR:.+]] = arith.shrsi %[[RES64]], %[[S64]]
   // CHECK-DAG: %[[TRUNC:.+]] = arith.trunci %[[SHR]] : i64 to i32
   // CHECK: return %[[TRUNC]]
-  %res = tosa.apply_scale %arg0, %arg1, %arg2 {double_round = true} : (i64, i32, i8) -> i32
+  %res = tosa.apply_scale %arg0, %arg1, %arg2 {rounding_mode = "DOUBLE_ROUND"} : (i64, i32, i8) -> i32
   return %res : i32
 }
diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir
index cc737e6c94b48..d1a9671873de0 100644
--- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir
+++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir
@@ -33,8 +33,12 @@ func.func @unranked_add(%arg0 : tensor<10x10xf32> , %arg1 : tensor<10x10xf32>, %
 
 // CHECK-LABEL: @rescale_unsupported_type
 func.func @rescale_unsupported_type(%arg0: tensor<13x21x3x!quant.uniform<u8:f32, 0.015655439347028732:127>>) -> tensor<13x21x3x!quant.uniform<i8:f32, 0.015655439347028732:-1>> {
+  %multiplier = "tosa.const"() {values = dense<1073741824> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<127> : tensor<1xi8>} : () -> tensor<1xi8>
+  %output_zp = "tosa.const"() {values = dense<-1> : tensor<1xi8>} : () -> tensor<1xi8>
   // expected-error@+1 {{failed to legalize operation 'tosa.rescale'}}
-  %0 = tosa.rescale %arg0 {double_round = false, input_zp = 127 : i32, multiplier = array<i32: 1073741824>, output_zp = -1 : i32, per_channel = false, scale32 = true, shift = array<i8: 30>, input_unsigned = true, output_unsigned = false} : (tensor<13x21x3x!quant.uniform<u8:f32, 0.015655439347028732:127>>) -> tensor<13x21x3x!quant.uniform<i8:f32, 0.015655439347028732:-1>>
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", input_zp = 127 : i32, output_zp = -1 : i32, per_channel = false, scale32 = true, input_unsigned = true, output_unsigned = false} : (tensor<13x21x3x!quant.uniform<u8:f32, 0.015655439347028732:127>>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<13x21x3x!quant.uniform<i8:f32, 0.015655439347028732:-1>>
   return %0 : tensor<13x21x3x!quant.uniform<i8:f32, 0.015655439347028732:-1>>
 }
 
diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir
index 5bb4a3bddb51b..19c12ba3edbd4 100644
--- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir
+++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir
@@ -8,7 +8,9 @@ func.func @matmul(%arg0: tensor<1x5x3xf32>, %arg1: tensor<1x3x6xf32>) -> (tensor
   // CHECK: [[INIT:%.+]] = tensor.empty()
   // CHECK: [[FILLED:%.+]] = linalg.fill ins([[C0]] : f32) outs([[INIT]] : tensor<1x5x6xf32>) -> tensor<1x5x6xf32>
   // CHECK: linalg.batch_matmul ins(%arg0, %arg1 : tensor<1x5x3xf32>, tensor<1x3x6xf32>) outs([[FILLED]] : tensor<1x5x6xf32>) -> tensor<1x5x6xf32>
-  %0 = tosa.matmul %arg0, %arg1 : (tensor<1x5x3xf32>, tensor<1x3x6xf32>)  -> tensor<1x5x6xf32>
+  %a_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %b_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %0 = tosa.matmul %arg0, %arg1, %a_zp, %b_zp : (tensor<1x5x3xf32>, tensor<1x3x6xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<1x5x6xf32>
   return %0 : tensor<1x5x6xf32>
 }
 
@@ -23,7 +25,9 @@ func.func @matmul_quantized(%arg0: tensor<1x5x3xi8>, %arg1: tensor<1x3x6xi8>) ->
   // CHECK: [[ONE:%.+]] = arith.constant 1
   // CHECK: [[TWO:%.+]] = arith.constant 2
   // CHECK: linalg.quantized_batch_matmul ins(%arg0, %arg1, [[ONE]], [[TWO]] : tensor<1x5x3xi8>, tensor<1x3x6xi8>, i32, i32) outs([[FILLED]] : tensor<1x5x6xi32>) -> tensor<1x5x6xi32>
-  %0 = tosa.matmul %arg0, %arg1 {a_zp = 1 : i32, b_zp = 2 : i32} : (tensor<1x5x3xi8>, tensor<1x3x6xi8>) -> tensor<1x5x6xi32>
+  %a_zp = "tosa.const"() <{values = dense<1> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %b_zp = "tosa.const"() <{values = dense<2> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %0 = tosa.matmul %arg0, %arg1, %a_zp, %b_zp : (tensor<1x5x3xi8>, tensor<1x3x6xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x5x6xi32>
   return %0 : tensor<1x5x6xi32>
 }
 
@@ -37,7 +41,9 @@ func.func @matmul_dyn_batch(%arg0: tensor<?x5x3xf32>, %arg1: tensor<?x3x6xf32>)
   // CHECK: %[[INIT:.+]] = tensor.empty(%[[DIM]])
   // CHECK: %[[FILLED:.+]] = linalg.fill ins(%[[C0_0]] : f32) outs(%[[INIT]] : tensor<?x5x6xf32>) -> tensor<?x5x6xf32>
   // CHECK: linalg.batch_matmul ins(%arg0, %arg1 : tensor<?x5x3xf32>, tensor<?x3x6xf32>) outs(%[[FILLED]] : tensor<?x5x6xf32>) -> tensor<?x5x6xf32>
-  %0 = tosa.matmul %arg0, %arg1 : (tensor<?x5x3xf32>, tensor<?x3x6xf32>) -> tensor<?x5x6xf32>
+  %a_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %b_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %0 = tosa.matmul %arg0, %arg1, %a_zp, %b_zp : (tensor<?x5x3xf32>, tensor<?x3x6xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<?x5x6xf32>
   return %0 : tensor<?x5x6xf32>
 }
 
@@ -51,7 +57,9 @@ func.func @matmul_dyn_independent_dim(%arg0: tensor<1x5x3xf32>, %arg1: tensor<1x
   // CHECK: %[[INIT:.+]] = tensor.empty(%[[DIM]])
   // CHECK: %[[FILLED:.+]] = linalg.fill ins(%[[C0]] : f32) outs(%[[INIT]] : tensor<1x5x?xf32>) -> tensor<1x5x?xf32>
   // CHECK: linalg.batch_matmul ins(%arg0, %arg1 : tensor<1x5x3xf32>, tensor<1x3x?xf32>) outs(%[[FILLED]] : tensor<1x5x?xf32>) -> tensor<1x5x?xf32>
-  %0 = tosa.matmul %arg0, %arg1 : (tensor<1x5x3xf32>, tensor<1x3x?xf32>) -> tensor<1x5x?xf32>
+  %a_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %b_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %0 = tosa.matmul %arg0, %arg1, %a_zp, %b_zp : (tensor<1x5x3xf32>, tensor<1x3x?xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<1x5x?xf32>
   return %0 : tensor<1x5x?xf32>
 }
 
@@ -63,7 +71,9 @@ func.func @matmul_dyn_independent_dim(%arg0: tensor<1x5x?xf32>, %arg1: tensor<1x
   // CHECK: %[[INIT:.+]] = tensor.empty()
   // CHECK: %[[FILLED:.+]] = linalg.fill ins(%[[C0]] : f32) outs(%[[INIT]] : tensor<1x5x6xf32>) -> tensor<1x5x6xf32>
   // CHECK: linalg.batch_matmul ins(%arg0, %arg1 : tensor<1x5x?xf32>, tensor<1x?x6xf32>) outs(%[[FILLED]] : tensor<1x5x6xf32>) -> tensor<1x5x6xf32>
-  %0 = tosa.matmul %arg0, %arg1 : (tensor<1x5x?xf32>, tensor<1x?x6xf32>) -> tensor<1x5x6xf32>
+  %a_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %b_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %0 = tosa.matmul %arg0, %arg1, %a_zp, %b_zp : (tensor<1x5x?xf32>, tensor<1x?x6xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<1x5x6xf32>
   return %0 : tensor<1x5x6xf32>
 }
 
@@ -77,7 +87,9 @@ func.func @matmul_dyn_output(%arg0: tensor<1x1x8xf32>, %arg1: tensor<1x8x1xf32>)
   // CHECK: %[[INIT:.+]] = tensor.empty(%[[DIM0]]) : tensor<?x1x1xf32>
   // CHECK: %[[FILLED:.+]] = linalg.fill ins(%[[CST]] : f32) outs(%[[INIT]] : tensor<?x1x1xf32>) -> tensor<?x1x1xf32>
   // CHECK: linalg.batch_matmul ins(%arg0, %arg1 : tensor<1x1x8xf32>, tensor<1x8x1xf32>) outs(%[[FILLED]] : tensor<?x1x1xf32>) -> tensor<?x1x1xf32>
-  %0 = tosa.matmul %arg0, %arg1 : (tensor<1x1x8xf32>, tensor<1x8x1xf32>) -> tensor<?x1x1xf32>
+  %a_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %b_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %0 = tosa.matmul %arg0, %arg1, %a_zp, %b_zp : (tensor<1x1x8xf32>, tensor<1x8x1xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<?x1x1xf32>
   return %0 : tensor<?x1x1xf32>
 }
 
@@ -411,7 +423,7 @@ func.func @avg_pool_i8(%arg0: tensor<1x6x34x62xi8>) -> (tensor<1x5x33x62xi8>) {
   // CHECK: %[[TRUNC_SHIFT:.+]] = arith.trunci %[[SUB]]
   // CHECK: %[[C30:.+]] = arith.constant 30
   // CHECK: %[[SHIFT:.+]] = arith.addi %[[TRUNC_SHIFT]], %[[C30]] : i8
-  // CHECK: %[[SCALED:.+]] = tosa.apply_scale %[[IN]], %[[TRUNC_MUL]], %[[SHIFT]] {double_round = false}
+  // CHECK: %[[SCALED:.+]] = tosa.apply_scale %[[IN]], %[[TRUNC_MUL]], %[[SHIFT]] {rounding_mode = "SINGLE_ROUND"}
 
   // Perform the normalization.
   // CHECK: %[[CMIN:.+]] = arith.constant -128
@@ -812,6 +824,30 @@ func.func @depthwise_conv2d_dyn_w_h(%arg0: tensor<2x?x?x3xf32>, %arg1: tensor<3x
 
 // -----
 
+// CHECK: #[[$MAP0:.*]] = affine_map<(d0, d1, d2, d3) -> (d3)>
+// CHECK: #[[$MAP1:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
+
+// CHECK-LABEL: @depthwise_int_conv_zero_zp
+func.func @depthwise_int_conv_zero_zp(%arg0 : tensor<1x7x5x3xi8>, %arg1 : tensor<3x1x3x11xi8>, %arg2 : tensor<33xi32>) -> () {
+  // CHECK: [[INIT:%.+]] = tensor.empty()
+  // CHECK: [[CST0:%.+]] = arith.constant 0
+  // CHECK: [[FILL:%.+]] = linalg.fill ins([[CST0]]{{.*}}outs([[INIT]]
+  // CHECK: [[OUT:%.+]] = tensor.empty()
+  // CHECK: [[DEPTH:%.+]] = linalg.depthwise_conv_2d_nhwc_hwcm {dilations = dense<1> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} ins(%arg0, %arg1 : tensor<1x7x5x3xi8>, tensor<3x1x3x11xi8>) outs([[FILL]] : tensor<1x5x5x3x11xi32>)
+  // CHECK: [[COLLAPSED:%.+]] = tensor.collapse_shape [[DEPTH]] {{\[}}[0], [1], [2], [3, 4]]
+  // CHECK: [[BIAS:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP1]], #[[$MAP1]]], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%arg2, [[COLLAPSED]] : tensor<33xi32>, tensor<1x5x5x33xi32>) outs([[OUT]] : tensor<1x5x5x33xi32>) {
+  // CHECK: ^bb0(%[[ARG3:[0-9a-zA-Z_]+]]: i32, %[[ARG4:[0-9a-zA-Z_]+]]: i32, %[[ARG5:[0-9a-zA-Z_]+]]: i32):
+  // CHECK:   [[ADD:%.+]] = arith.addi %[[ARG3]], %[[ARG4]] : i32
+  // CHECK:   linalg.yield [[ADD]] : i32
+  // CHECK: } -> tensor<1x5x5x33xi32>
+  %input_zp = "tosa.const"() <{values = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %weight_zp = "tosa.const"() <{values = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %2 = tosa.depthwise_conv2d %arg0, %arg1, %arg2, %input_zp, %weight_zp {acc_type = i32, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, dilation = array<i64: 1, 1> } : (tensor<1x7x5x3xi8>, tensor<3x1x3x11xi8>, tensor<33xi32>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x5x5x33xi32>
+  return
+}
+
+// -----
+
 // CHECK: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d4)>
 // CHECK: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3, d4)>
 
diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
index 9ba9965315fd3..18ce8571eeea0 100644
--- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
+++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
@@ -477,7 +477,9 @@ func.func @test_simple_f32(%arg0: tensor<1xf32>) -> () {
 
   // CHECK: linalg.generic
   // CHECK: arith.negf
-  %5 = tosa.negate %0 : (tensor<1xf32>) -> tensor<1xf32>
+  %in_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %out_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %5 = tosa.negate %0, %in_zp, %out_zp : (tensor<1xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<1xf32>
 
   // CHECK: linalg.generic
   // CHECK: pow
@@ -662,10 +664,12 @@ func.func @test_simple_i32(%arg0: tensor<1xi32>, %unsigned: tensor<1xui32>, %uns
   %40 = tosa.int_div %arg0, %arg0 : (tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
 
   // CHECK: linalg.generic
-  // CHECK: ^bb0(%[[ARG1:.*]]: i32, %[[ARG2:.*]]: i32):
+  // CHECK: ^bb0(%[[ARG1:.*]]: i32, %[[ARG2:.*]]: i32, %[[ARG3:.*]]: i32, %[[ARG4:.*]]: i32):
   // CHECK: [[ZERO:%.+]] = arith.constant 0
   // CHECK: arith.subi [[ZERO]], %[[ARG1]]
-  %5 = tosa.negate %arg0 : (tensor<1xi32>) -> tensor<1xi32>
+  %in_zp = "tosa.const"() <{values = dense<0> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %out_zp = "tosa.const"() <{values = dense<0> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %5 = tosa.negate %arg0, %in_zp, %out_zp : (tensor<1xi32>, tensor<1xi32>, tensor<1xi32>) -> tensor<1xi32>
 
   // CHECK: linalg.generic
   // CHECK: and
@@ -852,40 +856,22 @@ func.func @test_bool(%arg0: tensor<1xi1>, %arg1: tensor<1xi1>) -> () {
 // CHECK-LABEL: @test_negate_quantized
 func.func @test_negate_quantized(%arg0: tensor<1xi8>) -> () {
   // CHECK: linalg.generic
-  // CHECK: ^bb0(%[[BBARG0:.+]]: i8,
-  // CHECK: [[ZERO:%.+]] = arith.constant 0
-  // CHECK: [[SUB:%.+]] = arith.subi [[ZERO]], %[[BBARG0]]
-  // CHECK: linalg.yield [[SUB]]
-  %0 = tosa.negate %arg0 {input_zp1 = 0 : i32, output_zp = 0 : i32} : (tensor<1xi8>) -> tensor<1xi8>
-
-  // CHECK: linalg.generic
-  // CHECK: ^bb0(%[[BBARG0:.+]]: i8,
-  // CHECK: [[C32639:%.+]] = arith.constant 32639
+  // CHECK: ^bb0(%[[BBARG0:.+]]: i8, %[[BBARG1:.+]]: i8, %[[BBARG2:.+]]: i8, %[[BBARG3:.+]]: i8
+  // CHECK: [[CNST:%.+]] = arith.constant 7
   // CHECK: [[EXT:%.+]] = arith.extsi %[[BBARG0]] : i8 to i16
-  // CHECK: [[SUB:%.+]] = arith.subi [[C32639]], [[EXT]]
+  // CHECK: [[SUB:%.+]] = arith.subi [[CNST]], [[EXT]]
   // CHECK: [[MIN:%.+]] = arith.constant -128
   // CHECK: [[MAX:%.+]] = arith.constant 127
   // CHECK: [[LBOUND:%.+]] = arith.maxsi [[MIN]], [[SUB]]
   // CHECK: [[UBOUND:%.+]] = arith.minsi [[MAX]], [[LBOUND]]
   // CHECK: [[TRUNC:%.+]] = arith.trunci [[UBOUND]]
   // CHECK: linalg.yield [[TRUNC]]
-  %1 = tosa.negate %arg0 {input1_zp = 32639 : i32, output_zp = 0 : i32} : (tensor<1xi8>) -> tensor<1xi8>
+  %in_zp0 = "tosa.const"() <{values = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %out_zp0 = "tosa.const"() <{values = dense<7> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %0 = tosa.negate %arg0, %in_zp0, %out_zp0 : (tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1xi8>
 
   // CHECK: linalg.generic
-  // CHECK: ^bb0(%[[BBARG0:.+]]: i8,
-  // CHECK: [[C32640:%.+]] = arith.constant 32640
-  // CHECK: [[EXT:%.+]] = arith.extsi %[[BBARG0]] : i8 to i32
-  // CHECK: [[SUB:%.+]] = arith.subi [[C32640]], [[EXT]]
-  // CHECK: [[MIN:%.+]] = arith.constant -128
-  // CHECK: [[MAX:%.+]] = arith.constant 127
-  // CHECK: [[LBOUND:%.+]] = arith.maxsi [[MIN]], [[SUB]]
-  // CHECK: [[UBOUND:%.+]] = arith.minsi [[MAX]], [[LBOUND]]
-  // CHECK: [[TRUNC:%.+]] = arith.trunci [[UBOUND]]
-  // CHECK: linalg.yield [[TRUNC]]
-  %2 = tosa.negate %arg0 {input1_zp = 32640 : i32, output_zp = 0 : i32} : (tensor<1xi8>) -> tensor<1xi8>
-
-  // CHECK: linalg.generic
-  // CHECK: ^bb0(%[[BBARG0:.+]]: i8,
+  // CHECK: ^bb0(%[[BBARG0:.+]]: i8, %[[BBARG1:.+]]: i8, %[[BBARG2:.+]]: i8, %[[BBARG3:.+]]: i8
   // CHECK: [[C_128:%.+]] = arith.constant -128
   // CHECK: [[EXT:%.+]] = arith.extsi %[[BBARG0]] : i8 to i16
   // CHECK: [[SUB:%.+]] = arith.subi [[C_128]], [[EXT]]
@@ -895,14 +881,18 @@ func.func @test_negate_quantized(%arg0: tensor<1xi8>) -> () {
   // CHECK: [[UBOUND:%.+]] = arith.minsi [[MAX]], [[LBOUND]]
   // CHECK: [[TRUNC:%.+]] = arith.trunci [[UBOUND]]
   // CHECK: linalg.yield [[TRUNC]]
-  %3 = tosa.negate %arg0 {input1_zp = -128 : i32, output_zp = 0 : i32} : (tensor<1xi8>) -> tensor<1xi8>
+  %in_zp3 = "tosa.const"() <{values = dense<-128> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %out_zp3 = "tosa.const"() <{values = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %3 = tosa.negate %arg0, %in_zp3, %out_zp3 : (tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1xi8>
 
   // CHECK: linalg.generic
   // CHECK: ^bb0(%[[BBARG0:.+]]: i8,
   // CHECK: [[ZERO:%.+]] = arith.constant 0
   // CHECK: [[SUB:%.+]] = arith.subi [[ZERO]],
   // CHECK: linalg.yield [[SUB]]
-  %4 = tosa.negate %arg0 {quantization_info = #tosa.unary_quant<input_zp = 0, output_zp = 0>} : (tensor<1xi8>) -> tensor<1xi8>
+  %in_zp4 = "tosa.const"() <{values = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %out_zp4 = "tosa.const"() <{values = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %4 = tosa.negate %arg0, %in_zp4, %out_zp4 : (tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1xi8>
 
   return
 }
@@ -1141,7 +1131,7 @@ func.func @rescale_i8(%arg0 : tensor<2xi8>) -> () {
   // CHECK: [[C22:%.+]] = arith.constant 22
   // CHECK-DAG: [[IN32:%.+]] = arith.extsi [[IN]]
   // CHECK-DAG: [[IN_ZEROED:%.+]] = arith.subi [[IN32]], [[C17]]
-  // CHECK-DAG: [[SCALED:%.+]] = tosa.apply_scale [[IN_ZEROED]], [[C0]], [[C1]] {double_round = false}
+  // CHECK-DAG: [[SCALED:%.+]] = tosa.apply_scale [[IN_ZEROED]], [[C0]], [[C1]] {rounding_mode = "SINGLE_ROUND"}
   // CHECK-DAG: [[SCALED_ZEROED:%.+]] = arith.addi [[SCALED]], [[C22]]
   // CHECK-DAG: [[CMIN:%.+]] = arith.constant -128
   // CHECK-DAG: [[CMAX:%.+]] = arith.constant 127
@@ -1149,7 +1139,11 @@ func.func @rescale_i8(%arg0 : tensor<2xi8>) -> () {
   // CHECK-DAG: [[BOUNDED:%.+]] = arith.minsi [[CMAX]], [[LOWER]]
   // CHECK-DAG: [[TRUNC:%.+]] = arith.trunci [[BOUNDED]]
   // CHECK-DAG: linalg.yield [[TRUNC]]
-  %0 = tosa.rescale %arg0 {input_zp = 17 : i32, output_zp = 22 : i32, multiplier = array<i32: 19689>, shift = array<i8: 15>, scale32 = false, double_round = false, per_channel = false, input_unsigned = false, output_unsigned = false} : (tensor<2xi8>) -> tensor<2xi8>
+  %multiplier = "tosa.const"() {values = dense<19689> : tensor<1xi16>} : () -> tensor<1xi16>
+  %shift = "tosa.const"() {values = dense<15> : tensor<1xi8>} : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<17> : tensor<1xi8>} : () -> tensor<1xi8>
+  %output_zp = "tosa.const"() {values = dense<22> : tensor<1xi8>} : () -> tensor<1xi8>
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {scale32 = false, rounding_mode = "SINGLE_ROUND", per_channel = false, input_unsigned = false, output_unsigned = false} : (tensor<2xi8>, tensor<1xi16>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<2xi8>
 
   // CHECK: return
   return
@@ -1170,7 +1164,7 @@ func.func @rescale_i8_unsigned_output(%arg0 : tensor<2xi8>) -> () {
   // CHECK: [[C22:%.+]] = arith.constant 22
   // CHECK-DAG: [[IN32:%.+]] = arith.extsi [[IN]]
   // CHECK-DAG: [[IN_ZEROED:%.+]] = arith.subi [[IN32]], [[C17]]
-  // CHECK-DAG: [[SCALED:%.+]] = tosa.apply_scale [[IN_ZEROED]], [[C0]], [[C1]] {double_round = false}
+  // CHECK-DAG: [[SCALED:%.+]] = tosa.apply_scale [[IN_ZEROED]], [[C0]], [[C1]] {rounding_mode = "SINGLE_ROUND"}
   // CHECK-DAG: [[SCALED_ZEROED:%.+]] = arith.addi [[SCALED]], [[C22]]
   // CHECK-DAG: [[CMIN:%.+]] = arith.constant 0
   // CHECK-DAG: [[CMAX:%.+]] = arith.constant 255
@@ -1178,7 +1172,11 @@ func.func @rescale_i8_unsigned_output(%arg0 : tensor<2xi8>) -> () {
   // CHECK-DAG: [[BOUNDED:%.+]] = arith.minsi [[CMAX]], [[LOWER]]
   // CHECK-DAG: [[TRUNC:%.+]] = arith.trunci [[BOUNDED]]
   // CHECK: linalg.yield [[TRUNC]]
-  %1 = tosa.rescale %arg0 {input_zp = 17 : i32, output_zp = 22 : i32, multiplier = array<i32: 19689>, shift = array<i8: 15>, scale32 = false, double_round = false, per_channel = false, input_unsigned = false, output_unsigned = true} : (tensor<2xi8>) -> tensor<2xi8>
+  %multiplier = "tosa.const"() {values = dense<19689> : tensor<1xi16> } : () -> tensor<1xi16>
+  %shift = "tosa.const"() {values = dense<15> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<17> : tensor<1xi8>} : () -> tensor<1xi8>
+  %output_zp = "tosa.const"() {values = dense<22> : tensor<1xi8>} : () -> tensor<1xi8>
+  %1 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {scale32 = false, rounding_mode = "SINGLE_ROUND", per_channel = false, input_unsigned = false, output_unsigned = true} : (tensor<2xi8>, tensor<1xi16>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<2xi8>
 
   // CHECK: return
   return
@@ -1191,17 +1189,21 @@ func.func @rescale_i8_unsigned_output(%arg0 : tensor<2xi8>) -> () {
 // CHECK-LABEL: @rescale_i8_dyn_batch
 // CHECK-SAME: (%[[ARG0:[0-9a-zA-Z_]*]]:
 func.func @rescale_i8_dyn_batch(%arg0 : tensor<?x2xi8>) -> () {
+  %multiplier = "tosa.const"() {values = dense<19689> : tensor<1xi16>} : () -> tensor<1xi16>
+  %shift = "tosa.const"() {values = dense<15> : tensor<1xi8>} : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<17> : tensor<1xi8>} : () -> tensor<1xi8>
+  %output_zp = "tosa.const"() {values = dense<22> : tensor<1xi8>} : () -> tensor<1xi8>
   // CHECK: %[[C0:.+]] = arith.constant 0
   // CHECK: %[[BATCH:.+]] = tensor.dim %[[ARG0]], %[[C0]]
   // CHECK: %[[INIT:.+]] = tensor.empty(%[[BATCH]]) : tensor<?x2xi8>
   // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP0]]], iterator_types = ["parallel", "parallel"]} ins(%[[ARG0]] : tensor<?x2xi8>) outs(%[[INIT]] : tensor<?x2xi8>)
-  %0 = tosa.rescale %arg0 {input_zp = 17 : i32, output_zp = 22 : i32, multiplier = array<i32: 19689>, shift = array<i8: 15>, scale32 = false, double_round = false, per_channel = false, input_unsigned = false, output_unsigned = false} : (tensor<?x2xi8>) -> tensor<?x2xi8>
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {scale32 = false, rounding_mode = "SINGLE_ROUND", per_channel = false, input_unsigned = false, output_unsigned = false} : (tensor<?x2xi8>, tensor<1xi16>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<?x2xi8>
 
   // CHECK: %[[C0:.+]] = arith.constant 0
   // CHECK: %[[BATCH:.+]] = tensor.dim %[[ARG0]], %[[C0]]
   // CHECK: %[[INIT:.+]] = tensor.empty(%[[BATCH]]) : tensor<?x2xi8>
   // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP0]]], iterator_types = ["parallel", "parallel"]} ins(%[[ARG0]] : tensor<?x2xi8>) outs(%[[INIT]] : tensor<?x2xi8>)
-  %1 = tosa.rescale %arg0 {input_zp = 17 : i32, output_zp = 22 : i32, multiplier = array<i32: 19689>, shift = array<i8: 15>, scale32 = false, double_round = false, per_channel = false, input_unsigned = false, output_unsigned = true} : (tensor<?x2xi8>) -> tensor<?x2xi8>
+  %1 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {scale32 = false, rounding_mode = "SINGLE_ROUND", per_channel = false, input_unsigned = false, output_unsigned = true} : (tensor<?x2xi8>, tensor<1xi16>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<?x2xi8>
 
   return
 }
@@ -1213,13 +1215,17 @@ func.func @rescale_i8_dyn_batch(%arg0 : tensor<?x2xi8>) -> () {
 // CHECK-LABEL: @rescale_dyn
 // CHECK-SAME: (%[[ARG0:[0-9a-zA-Z_]*]]:
 func.func @rescale_dyn(%arg0 : tensor<1x?x?x32xi32>) -> () {
+  %input_zp = "tosa.const"() {values = dense<0> : tensor<1xi32>} : () -> tensor<1xi32>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
   // CHECK: %[[C1:.+]] = arith.constant 1
   // CHECK: %[[DIM1:.+]] = tensor.dim %[[ARG0]], %[[C1]]
   // CHECK: %[[C2:.+]] = arith.constant 2
   // CHECK: %[[DIM2:.+]] = tensor.dim %[[ARG0]], %[[C2]]
   // CHECK: %[[INIT:.+]] = tensor.empty(%[[DIM1]], %[[DIM2]])
   // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP1]], #[[$MAP1]]], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%[[ARG0]] : tensor<1x?x?x32xi32>) outs(%[[INIT]] : tensor<1x?x?x32xi8>)
-  %0 = tosa.rescale %arg0 {double_round = true, input_zp = 0 : i32, multiplier = array<i32: 1376784203>, output_zp = 0 : i32, per_channel = false, scale32 = true, shift = array<i8: 38>, input_unsigned = false, output_unsigned = false} : (tensor<1x?x?x32xi32>) -> tensor<1x?x?x32xi8>
+  %multiplier = "tosa.const"() {values = dense<1376784203> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<38> : tensor<1xi8> } : () -> tensor<1xi8>
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "DOUBLE_ROUND", input_zp = 0 : i32, output_zp = 0 : i32, per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = false} : (tensor<1x?x?x32xi32>, tensor<1xi32>, tensor<1xi8>, tensor<1xi32>, tensor<1xi8>) -> tensor<1x?x?x32xi8>
   return
 }
 
@@ -1239,7 +1245,7 @@ func.func @rescale_i8_unsigned_input(%arg0 : tensor<2xi8>) -> () {
   // CHECK: [[C22:%.+]] = arith.constant 22
   // CHECK-DAG: [[IN32:%.+]] = arith.extui [[IN]]
   // CHECK-DAG: [[IN_ZEROED:%.+]] = arith.subi [[IN32]], [[C17]]
-  // CHECK-DAG: [[SCALED:%.+]] = tosa.apply_scale [[IN_ZEROED]], [[C0]], [[C1]] {double_round = false}
+  // CHECK-DAG: [[SCALED:%.+]] = tosa.apply_scale [[IN_ZEROED]], [[C0]], [[C1]] {rounding_mode = "SINGLE_ROUND"}
   // CHECK-DAG: [[SCALED_ZEROED:%.+]] = arith.addi [[SCALED]], [[C22]]
   // CHECK-DAG: [[CMIN:%.+]] = arith.constant -128
   // CHECK-DAG: [[CMAX:%.+]] = arith.constant 127
@@ -1247,7 +1253,11 @@ func.func @rescale_i8_unsigned_input(%arg0 : tensor<2xi8>) -> () {
   // CHECK-DAG: [[BOUNDED:%.+]] = arith.minsi [[CMAX]], [[LOWER]]
   // CHECK-DAG: [[TRUNC:%.+]] = arith.trunci [[BOUNDED]]
   // CHECK: linalg.yield [[TRUNC]]
-  %0 = tosa.rescale %arg0 {input_zp = 17 : i32, output_zp = 22 : i32, multiplier = array<i32: 19689>, shift = array<i8: 15>, scale32 = false, double_round = false, per_channel = false, input_unsigned = true, output_unsigned = false} : (tensor<2xi8>) -> tensor<2xi8>
+  %multiplier = "tosa.const"() {values = dense<19689> : tensor<1xi16> } : () -> tensor<1xi16>
+  %shift = "tosa.const"() {values = dense<15> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<17> : tensor<1xi8>} : () -> tensor<1xi8>
+  %output_zp = "tosa.const"() {values = dense<22> : tensor<1xi8>} : () -> tensor<1xi8>
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {scale32 = false, rounding_mode = "SINGLE_ROUND", per_channel = false, input_unsigned = true, output_unsigned = false} : (tensor<2xi8>, tensor<1xi16>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<2xi8>
 
   return
 }
@@ -1264,12 +1274,12 @@ func.func @rescale_per_channel(%arg0 : tensor<3xi8>) -> (tensor<3xi8>) {
   // CHECK: [[INIT:%.+]] = tensor.empty()
   // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP0]], #[[$MAP0]], #[[$MAP0]]], iterator_types = ["parallel"]} ins(%[[ARG0]], [[MULTIPLIERS]], [[SHIFTS]] : tensor<3xi8>, tensor<3xi32>, tensor<3xi8>) outs([[INIT]] : tensor<3xi8>)
   // CHECK: ^bb0([[IN:%.+]]: i8, [[MULTIPLIER:%.+]]: i32, [[SHIFT:%.+]]: i8, [[UNUSED:%.+]]: i8):
-  // CHECK: [[C243:%.+]] = arith.constant 243
-  // CHECK: [[C252:%.+]] = arith.constant 252
+  // CHECK: [[C243:%.+]] = arith.constant 43
+  // CHECK: [[C252:%.+]] = arith.constant 52
 
   // CHECK-DAG: [[IN32:%.+]] = arith.extsi [[IN]]
   // CHECK-DAG: [[IN_ZEROED:%.+]] = arith.subi [[IN32]], [[C243]]
-  // CHECK-DAG: [[SCALED:%.+]] = tosa.apply_scale [[IN_ZEROED]], [[MULTIPLIER]], [[SHIFT]] {double_round = false}
+  // CHECK-DAG: [[SCALED:%.+]] = tosa.apply_scale [[IN_ZEROED]], [[MULTIPLIER]], [[SHIFT]] {rounding_mode = "SINGLE_ROUND"}
   // CHECK-DAG: [[SCALED_ZEROED:%.+]] = arith.addi [[SCALED]], [[C252]]
   // CHECK-DAG: [[CMIN:%.+]] = arith.constant -128
   // CHECK-DAG: [[CMAX:%.+]] = arith.constant 127
@@ -1277,7 +1287,11 @@ func.func @rescale_per_channel(%arg0 : tensor<3xi8>) -> (tensor<3xi8>) {
   // CHECK-DAG: [[BOUNDED:%.+]] = arith.minsi [[CMAX]], [[LOWER]]
   // CHECK-DAG: [[TRUNC:%.+]] = arith.trunci [[BOUNDED]]
   // CHECK-DAG: linalg.yield [[TRUNC]]
-  %0 = tosa.rescale %arg0 {input_zp = 243 : i32, output_zp = 252 : i32, multiplier = array<i32: 42, 43, 44>, shift = array<i8: 14, 15, 64>, scale32 = false, double_round = false, per_channel = false, input_unsigned = false, output_unsigned = false} : (tensor<3xi8>) -> tensor<3xi8>
+  %multiplier = "tosa.const"() {values = dense<[42, 43, 44]> : tensor<3xi16>} : () -> tensor<3xi16>
+  %shift = "tosa.const"() {values = dense<[14, 15, 64]> : tensor<3xi8>} : () -> tensor<3xi8>
+  %input_zp = "tosa.const"() {values = dense<43> : tensor<1xi8>} : () -> tensor<1xi8>
+  %output_zp = "tosa.const"() {values = dense<52> : tensor<1xi8>} : () -> tensor<1xi8>
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {scale32 = false, rounding_mode = "SINGLE_ROUND", per_channel = true, input_unsigned = false, output_unsigned = false} : (tensor<3xi8>, tensor<3xi16>, tensor<3xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<3xi8>
 
   // CHECK: return [[GENERIC]]
   return %0 : tensor<3xi8>
@@ -1287,19 +1301,43 @@ func.func @rescale_per_channel(%arg0 : tensor<3xi8>) -> (tensor<3xi8>) {
 
 // CHECK-LABEL: @rescaleDoubleRound
 func.func @rescaleDoubleRound(%arg0 : tensor<2xi8>) -> (tensor<2xi8>) {
+  %multiplier = "tosa.const"() {values = dense<19689> : tensor<1xi32>} : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<33> : tensor<1xi8>} : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<43> : tensor<1xi8>} : () -> tensor<1xi8>
+  %output_zp = "tosa.const"() {values = dense<52> : tensor<1xi8>} : () -> tensor<1xi8>
+
   // CHECK: linalg.generic
   // CHECK: tosa.apply_scale
-  // CHECK-SAME:  {double_round = true}
-  %0 = tosa.rescale %arg0 {input_zp = 243 : i32, output_zp = 252 : i32, multiplier = array<i32: 19689>, shift = array<i8: 33>, scale32 = true, double_round = true, per_channel = false, input_unsigned = false, output_unsigned = false} : (tensor<2xi8>) -> tensor<2xi8>
+  // CHECK-SAME: {rounding_mode = "DOUBLE_ROUND"}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {scale32 = true, rounding_mode = "DOUBLE_ROUND", per_channel = false, input_unsigned = false, output_unsigned = false} : (tensor<2xi8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<2xi8>
   return %0 : tensor<2xi8>
 }
 
+// -----
+
 // CHECK-LABEL: @rescaleUnnecessaryDoubleRound
 func.func @rescaleUnnecessaryDoubleRound(%arg0 : tensor<2xi8>) -> (tensor<2xi8>) {
+  %multiplier = "tosa.const"() {values = dense<19689> : tensor<1xi32>} : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<15> : tensor<1xi8>} : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<43> : tensor<1xi8>} : () -> tensor<1xi8>
+  %output_zp = "tosa.const"() {values = dense<52> : tensor<1xi8>} : () -> tensor<1xi8>
+
   // CHECK: linalg.generic
   // CHECK: tosa.apply_scale
-  // CHECK-SAME:  {double_round = false}
-  %0 = tosa.rescale %arg0 {input_zp = 243 : i32, output_zp = 252 : i32, multiplier = array<i32: 19689>, shift = array<i8: 15>, scale32 = true, double_round = true, per_channel = false, input_unsigned = false, output_unsigned = false} : (tensor<2xi8>) -> tensor<2xi8>
+  // CHECK-SAME:  {rounding_mode = "SINGLE_ROUND"}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {scale32 = true, rounding_mode = "DOUBLE_ROUND", per_channel = false, input_unsigned = false, output_unsigned = false} : (tensor<2xi8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<2xi8>
+  return %0 : tensor<2xi8>
+}
+
+// -----
+
+func.func @unsupportedRescaleInexactRound(%arg0 : tensor<2xi8>) -> (tensor<2xi8>) {
+  %multiplier = "tosa.const"() {values = dense<19689> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<33> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
+  // expected-error@+1 {{failed to legalize operation 'tosa.rescale'}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {input_zp = 243 : i32, output_zp = 252 : i32, scale32 = true, rounding_mode = "INEXACT_ROUND", per_channel = false, input_unsigned = false, output_unsigned = false} : (tensor<2xi8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<2xi8>
   return %0 : tensor<2xi8>
 }
 
diff --git a/mlir/test/Conversion/VectorToLLVM/pass-option-serialization.mlir b/mlir/test/Conversion/VectorToLLVM/pass-option-serialization.mlir
new file mode 100644
index 0000000000000..ebf06c57a1b3b
--- /dev/null
+++ b/mlir/test/Conversion/VectorToLLVM/pass-option-serialization.mlir
@@ -0,0 +1,30 @@
+// Ensure that ConvertVectorToLLVMPass options remain serialisable.
+
+// This test also allows us to exercise these options (to some extent) even if we
+// don't use them in other Vector to LLVM conversion tests. This is quite relevant
+// for the `Vector` Dialect (and `--convert-vector-to-llvm` pass) as in many cases
+// we use the Transform Dialect (TD) rather than `--convert-vector-to-llvm` for
+// testing. So here we don't check the correctness of the passes, as they're
+// covered by other tests that use TD, but we still provide some test coverage of
+// these pass options.
+
+// We don't need to actually parse any IR to print the pass options. We just need
+// to provide --dump-pass-pipeline
+
+// RUN: mlir-opt --convert-vector-to-llvm --dump-pass-pipeline %s 2>&1 | FileCheck %s --check-prefix=DEFAULT
+
+// RUN: mlir-opt --convert-vector-to-llvm='vector-contract-lowering=matmul vector-transpose-lowering=flat' \
+// RUN:          --dump-pass-pipeline 2>&1 | FileCheck %s --check-prefix=NON-DEFAULT
+
+// CHECK: builtin.module(
+// CHECK-SAME: convert-vector-to-llvm{
+// CHECK-SAME: enable-amx={{[aA-zZ0-9]+}}
+// CHECK-SAME: enable-arm-neon={{[aA-zZ0-9]+}}
+// CHECK-SAME: enable-arm-sve={{[aA-zZ0-9]+}}
+// CHECK-SAME: enable-x86vector={{[aA-zZ0-9]+}}
+// CHECK-SAME: force-32bit-vector-indices={{[aA-zZ0-9]+}}
+// CHECK-SAME: reassociate-fp-reductions={{[aA-zZ0-9]+}}
+// DEFAULT: vector-contract-lowering=dot
+// DEFAULT: vector-transpose-lowering=eltwise
+// NON-DEFAULT: vector-contract-lowering=matmul
+// NON-DEFAULT: vector-transpose-lowering=flat
diff --git a/mlir/test/Dialect/Affine/canonicalize.mlir b/mlir/test/Dialect/Affine/canonicalize.mlir
index a9ac13ad71624..d39c0c6e41df2 100644
--- a/mlir/test/Dialect/Affine/canonicalize.mlir
+++ b/mlir/test/Dialect/Affine/canonicalize.mlir
@@ -2262,3 +2262,12 @@ func.func @cst_value_to_cst_attr_basis_linearize_index(%arg0 : index, %arg1 : in
   %0 = affine.linearize_index disjoint [%arg0, %arg1, %arg2] by  (%c2, 3, %c4) : index
   return %0 : index
 }
+
+// CHECK-LABEL: func @for_empty_body_folder_iv_yield
+func.func @for_empty_body_folder_iv_yield() -> index {
+  %c18 = arith.constant 18 : index
+  %10 = affine.for %arg3 = 0 to 114 iter_args(%arg4 = %c18) -> (index) {
+    affine.yield %arg3 : index
+  }
+  return %10 : index
+}
diff --git a/mlir/test/Dialect/Affine/loop-fusion-2.mlir b/mlir/test/Dialect/Affine/loop-fusion-2.mlir
index 9648504b7f960..b26a539b2f7d5 100644
--- a/mlir/test/Dialect/Affine/loop-fusion-2.mlir
+++ b/mlir/test/Dialect/Affine/loop-fusion-2.mlir
@@ -39,16 +39,20 @@ func.func @should_fuse_at_depth_above_loop_carried_dependence(%arg0: memref<64x4
 
   // We can fuse source loop nest '%i0' into dst loop nest '%i2', but the
   // depth at which we can insert the src loop nest slice into the dst loop
-  // lest must be decreased because of a loop carried dependence on loop '%i3'.
+  // nest must be decreased because of a loop carried dependence on loop '%i3'.
   // As a result, the source loop nest is inserted at dst loop nest depth 1,
   // just above the loop with the carried dependence. In addition, the source
   // loop nest iteration bounds on its loop '%i1' are reduced to 1, so the
-  // memref size can be reduced to 128x1xf32.
+  // memref size can be reduced to 64x1xf32.
 
-  // CHECK:       memref.alloc() : memref<64x1xf32>
+  // In this case, since we have multiple producer stores (for %out) with
+  // different access functions and we don't yet support private memref
+  // computation in such cases, a 64x1 private memref isn't created.
+
+  // CHECK:       memref.alloc() : memref<64x4xf32>
   // CHECK:       affine.for %{{.*}} = 0 to 4 {
   // CHECK-NEXT:    affine.for %{{.*}} = 0 to 64 {
-  // CHECK-NEXT:      affine.store %{{.*}}, %{{.*}}[%{{.*}}, 0] : memref<64x1xf32>
+  // CHECK-NEXT:      affine.store %{{.*}}, %{{.*}}[%{{.*}}, %{{.*}}] : memref<64x4xf32>
   // CHECK-NEXT:    }
   // CHECK-NEXT:    affine.for %{{.*}} = 0 to 4 {
   // CHECK-NEXT:      affine.for %{{.*}} = 0 to 16 {
@@ -62,9 +66,9 @@ func.func @should_fuse_at_depth_above_loop_carried_dependence(%arg0: memref<64x4
   // CHECK-NEXT:        }
   // CHECK-NEXT:        affine.for %{{.*}} = 0 to 16 {
   // CHECK-NEXT:          %{{.*}} = "op2"() : () -> f32
-  // CHECK:               affine.load %{{.*}}[%{{.*}} * 16 + %{{.*}}, 0] : memref<64x1xf32>
+  // CHECK:               affine.load %{{.*}}[%{{.*}} * 16 + %{{.*}}, %{{.*}}] : memref<64x4xf32>
   // CHECK-NEXT:          arith.addf %{{.*}}, %{{.*}} : f32
-  // CHECK:               affine.store %{{.*}}, %{{.*}}[%{{.*}} * 16 + %{{.*}}, 0] : memref<64x1xf32>
+  // CHECK:               affine.store %{{.*}}, %{{.*}}[%{{.*}} * 16 + %{{.*}}, %{{.*}}] : memref<64x4xf32>
   // CHECK-NEXT:        }
   // CHECK-NEXT:      }
   // CHECK-NEXT:    }
diff --git a/mlir/test/Dialect/Affine/loop-fusion-4.mlir b/mlir/test/Dialect/Affine/loop-fusion-4.mlir
index b5b951ad5eb0e..4b9eca45492fb 100644
--- a/mlir/test/Dialect/Affine/loop-fusion-4.mlir
+++ b/mlir/test/Dialect/Affine/loop-fusion-4.mlir
@@ -622,3 +622,47 @@ func.func @zero_tolerance(%arg0: memref<65536xcomplex<f64>>, %arg1: memref<30x13
 }
 func.func private @__external_levelwise_forward_ntt(memref<30x131072xi64>)
 func.func private @__external_reduce_barrett(i64, i64, i64, i64, i128) -> i64
+
+// An unrolled loop nest. Fusion here should correctly fuse while preserving
+// dependences between store-load pairs of the same memref. A private memref
+// of size 1x1x1 can't be created.
+
+// PRODUCER-CONSUMER-MAXIMAL-LABEL: func @unrolled
+func.func @unrolled(%arg0: memref<2x4xf32>, %arg1: memref<1x2x4xf32>) {
+  %alloc = memref.alloc() : memref<1x2x4xf32>
+  affine.for %i = 0 to 1 {
+    %0 = affine.load %arg0[0, 0] : memref<2x4xf32>
+    %1 = affine.load %arg0[0, 1] : memref<2x4xf32>
+    %2 = affine.load %arg0[0, 2] : memref<2x4xf32>
+    %3 = affine.load %arg0[0, 3] : memref<2x4xf32>
+    %4 = affine.load %arg0[1, 0] : memref<2x4xf32>
+    %5 = affine.load %arg0[1, 1] : memref<2x4xf32>
+    %6 = affine.load %arg0[1, 2] : memref<2x4xf32>
+    %7 = affine.load %arg0[1, 3] : memref<2x4xf32>
+
+    affine.store %0, %alloc[0, 0, 0] : memref<1x2x4xf32>
+    affine.store %1, %alloc[0, 0, 1] : memref<1x2x4xf32>
+    affine.store %2, %alloc[0, 0, 2] : memref<1x2x4xf32>
+    affine.store %3, %alloc[0, 0, 3] : memref<1x2x4xf32>
+    affine.store %4, %alloc[0, 1, 0] : memref<1x2x4xf32>
+    affine.store %5, %alloc[0, 1, 1] : memref<1x2x4xf32>
+    affine.store %6, %alloc[0, 1, 2] : memref<1x2x4xf32>
+    affine.store %7, %alloc[0, 1, 3] : memref<1x2x4xf32>
+  }
+
+  affine.for %i = 0 to 2 {
+    affine.for %j = 0 to 4 {
+      %8 = affine.load %alloc[0, %i, %j] : memref<1x2x4xf32>
+      %9 = arith.negf %8 : f32
+      affine.store %9, %arg1[0, %i, %j] : memref<1x2x4xf32>
+    }
+  }
+  // PRODUCER-CONSUMER-MAXIMAL:      affine.for %{{.*}} = 0 to 2 {
+  // PRODUCER-CONSUMER-MAXIMAL-NEXT:   affine.for %{{.*}} = 0 to 4 {
+  // PRODUCER-CONSUMER-MAXIMAL-NEXT:     affine.load %{{.*}}[0, 0]
+  // PRODUCER-CONSUMER-MAXIMAL:          affine.load %{{.*}}[1, 3]
+  // PRODUCER-CONSUMER-MAXIMAL:          affine.store %{{.*}}[0, 0, 0]
+  // PRODUCER-CONSUMER-MAXIMAL:          affine.store %{{.*}}[0, 1, 3]
+  // PRODUCER-CONSUMER-MAXIMAL:          affine.load %{{.*}}[0, %{{.*}}, %{{.*}}]
+  return
+}
diff --git a/mlir/test/Dialect/Affine/loop-tiling.mlir b/mlir/test/Dialect/Affine/loop-tiling.mlir
index c01e3e910d6a0..d2aca48e615ae 100644
--- a/mlir/test/Dialect/Affine/loop-tiling.mlir
+++ b/mlir/test/Dialect/Affine/loop-tiling.mlir
@@ -1,13 +1,14 @@
 // RUN: mlir-opt %s -split-input-file -affine-loop-tile="tile-size=32" | FileCheck %s
 // RUN: mlir-opt %s -split-input-file -affine-loop-tile="cache-size=512" | FileCheck %s --check-prefix=MODEL
+// RUN: mlir-opt %s -split-input-file -affine-loop-tile="cache-size=0" | FileCheck %s --check-prefix=ZERO-CACHE
 // RUN: mlir-opt %s -split-input-file -affine-loop-tile="tile-size=32 separate" | FileCheck %s --check-prefix=SEPARATE
 
-// -----
-
 // CHECK-DAG: [[$UB:#map[0-9]*]] = affine_map<(d0) -> (d0 + 32)>
 // CHECK-DAG: [[$UB_MIN:#map[0-9]*]] = affine_map<(d0) -> (d0 + 32, 50)>
 // CHECK-DAG: [[$ID:#map[0-9]*]] = affine_map<(d0) -> (d0)>
 // CHECK-DAG: [[$ID_PLUS_21:#map[0-9]*]] = affine_map<(d0) -> (d0 + 21)>
+// ZERO-CACHE-DAG: affine_map<(d0) -> (d0)>
+// ZERO-CACHE-DAG: affine_map<(d0) -> (d0 + 1)>
 
 // CHECK-LABEL: func @loop_tiling()
 // CHECK-NEXT:   affine.for %{{.*}} = 0 to 256 step 32 {
diff --git a/mlir/test/Dialect/Affine/loop-unswitch.mlir b/mlir/test/Dialect/Affine/loop-unswitch.mlir
index 5a58941937bf5..c94f5515d49ea 100644
--- a/mlir/test/Dialect/Affine/loop-unswitch.mlir
+++ b/mlir/test/Dialect/Affine/loop-unswitch.mlir
@@ -254,3 +254,31 @@ func.func @multiple_if(%N : index) {
 // CHECK-NEXT: return
 
 func.func private @external()
+
+// Check to ensure affine.parallel ops are handled as well.
+
+#set = affine_set<(d0) : (-d0 + 3 >= 0)>
+// CHECK-LABEL: affine_parallel
+func.func @affine_parallel(%arg0: memref<35xf32>) -> memref<35xf32> {
+  %0 = llvm.mlir.constant(1.000000e+00 : f32) : f32
+  %alloc = memref.alloc() {alignment = 64 : i64} : memref<35xf32>
+  // CHECK: affine.parallel
+  affine.parallel (%arg1) = (0) to (35) step (32) {
+    // This can't be hoisted further.
+    // CHECK-NEXT: affine.if
+    affine.if #set(%arg1) {
+      affine.parallel (%arg2) = (%arg1) to (%arg1 + 32) {
+        %1 = affine.load %arg0[%arg2] : memref<35xf32>
+        %2 = llvm.fdiv %0, %1 : f32
+        affine.store %2, %alloc[%arg2] : memref<35xf32>
+      }
+    } else {
+      affine.parallel (%arg2) = (%arg1) to (min(%arg1 + 32, 35)) {
+        %1 = affine.load %arg0[%arg2] : memref<35xf32>
+        %2 = llvm.fdiv %0, %1 : f32
+        affine.store %2, %alloc[%arg2] : memref<35xf32>
+      }
+    }
+  }
+  return %alloc : memref<35xf32>
+}
diff --git a/mlir/test/Dialect/Affine/parallelize.mlir b/mlir/test/Dialect/Affine/parallelize.mlir
index b3bb20929c334..bfd1720959861 100644
--- a/mlir/test/Dialect/Affine/parallelize.mlir
+++ b/mlir/test/Dialect/Affine/parallelize.mlir
@@ -341,3 +341,23 @@ func.func @test_add_inv_or_terminal_symbol(%arg0: memref<9x9xi32>, %arg1: i1) {
   }
   return
 }
+
+// Ensure that outer parallel loops are taken into account when computing the
+// loop depth in dependency analysis during parallelization. With correct
+// depth, the analysis should see the inner loop as sequential due to reads and
+// writes to the same address indexed by the outer (parallel) loop.
+//
+// CHECK-LABEL: @explicit_parallel
+func.func @explicit_parallel(%arg0: memref<1x123x194xf64>, %arg5: memref<34x99x194xf64>) {
+  // CHECK: affine.parallel
+  affine.parallel (%arg7, %arg8) = (0, 0) to (85, 180) {
+    // CHECK: affine.for
+    affine.for %arg9 = 0 to 18 {
+      %0 = affine.load %arg0[0, %arg7 + 19, %arg8 + 7] : memref<1x123x194xf64>
+      %1 = affine.load %arg5[%arg9 + 8, %arg7 + 7, %arg8 + 7] : memref<34x99x194xf64>
+      %2 = arith.addf %0, %1 {fastmathFlags = #llvm.fastmath<none>} : f64
+      affine.store %1, %arg0[0, %arg7 + 19, %arg8 + 7] : memref<1x123x194xf64>
+    }
+  }
+  return
+}
diff --git a/mlir/test/Dialect/Affine/scalrep.mlir b/mlir/test/Dialect/Affine/scalrep.mlir
index fdfe3bfb62f95..092597860c8d9 100644
--- a/mlir/test/Dialect/Affine/scalrep.mlir
+++ b/mlir/test/Dialect/Affine/scalrep.mlir
@@ -738,12 +738,11 @@ func.func @with_inner_ops(%arg0: memref<?xf64>, %arg1: memref<?xf64>, %arg2: i1)
   return
 }
 
-// CHECK:  %[[pi:.+]] = arith.constant 3.140000e+00 : f64
-// CHECK:  %{{.*}} = scf.if %arg2 -> (f64) {
-// CHECK:        scf.yield %{{.*}} : f64
+// Semantics of non-affine region ops would be unknown.
+
 // CHECK:      } else {
-// CHECK:        scf.yield %[[pi]] : f64
-// CHECK:      }
+// CHECK-NEXT:   %[[Y:.*]] = affine.load
+// CHECK-NEXT:   scf.yield %[[Y]] : f64
 
 // Check if scalar replacement works correctly when affine memory ops are in the
 // body of an scf.for.
@@ -952,3 +951,35 @@ func.func @consecutive_store() {
   }
   return
 }
+
+// CHECK-LABEL: func @scf_for_if
+func.func @scf_for_if(%arg0: memref<?xi32>, %arg1: i32) -> i32 attributes {llvm.linkage = #llvm.linkage<external>} {
+  %c1 = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %c0_i32 = arith.constant 0 : i32
+  %c5_i32 = arith.constant 5 : i32
+  %c10_i32 = arith.constant 10 : i32
+  %0 = memref.alloca() : memref<1xi32>
+  %1 = llvm.mlir.undef : i32
+  affine.store %1, %0[0] : memref<1xi32>
+  affine.store %c0_i32, %0[0] : memref<1xi32>
+  %2 = arith.index_cast %arg1 : i32 to index
+  scf.for %arg2 = %c0 to %2 step %c1 {
+    %4 = memref.load %arg0[%arg2] : memref<?xi32>
+    %5 = arith.muli %4, %c5_i32 : i32
+    %6 = arith.cmpi sgt, %5, %c10_i32 : i32
+    // CHECK: scf.if
+    scf.if %6 {
+      // No forwarding should happen here since we have an scf.for around and we
+      // can't analyze the flow of values.
+      // CHECK: affine.load
+      %7 = affine.load %0[0] : memref<1xi32>
+      %8 = arith.addi %5, %7 : i32
+      // CHECK: affine.store
+      affine.store %8, %0[0] : memref<1xi32>
+    }
+  }
+  // CHECK: affine.load
+  %3 = affine.load %0[0] : memref<1xi32>
+  return %3 : i32
+}
diff --git a/mlir/test/Dialect/LLVMIR/layout.mlir b/mlir/test/Dialect/LLVMIR/layout.mlir
index 2dfc289c93065..4e60f991f1bf5 100644
--- a/mlir/test/Dialect/LLVMIR/layout.mlir
+++ b/mlir/test/Dialect/LLVMIR/layout.mlir
@@ -6,6 +6,7 @@ module {
     // CHECK: alignment = 8
     // CHECK: alloca_memory_space = 0
     // CHECK: bitsize = 64
+    // CHECK: default_memory_space = 0
     // CHECK: endianness = ""
     // CHECK: global_memory_space = 0
     // CHECK: index = 64
@@ -18,6 +19,7 @@ module {
     // CHECK: alignment = 8
     // CHECK: alloca_memory_space = 0
     // CHECK: bitsize = 64
+    // CHECK: default_memory_space = 0
     // CHECK: endianness = ""
     // CHECK: global_memory_space = 0
     // CHECK: index = 64
@@ -30,6 +32,7 @@ module {
     // CHECK: alignment = 8
     // CHECK: alloca_memory_space = 0
     // CHECK: bitsize = 64
+    // CHECK: default_memory_space = 0
     // CHECK: endianness = ""
     // CHECK: global_memory_space = 0
     // CHECK: index = 64
@@ -50,6 +53,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec<
   #dlti.dl_entry<!llvm.ptr<5>, dense<[64, 64, 64]> : vector<3xi64>>,
   #dlti.dl_entry<!llvm.ptr<4>, dense<[32, 64, 64, 24]> : vector<4xi64>>,
   #dlti.dl_entry<"dlti.endianness", "little">,
+  #dlti.dl_entry<"dlti.default_memory_space", 7 : ui64>,
   #dlti.dl_entry<"dlti.alloca_memory_space", 5 : ui64>,
   #dlti.dl_entry<"dlti.global_memory_space", 2 : ui64>,
   #dlti.dl_entry<"dlti.program_memory_space", 3 : ui64>,
@@ -61,6 +65,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec<
     // CHECK: alignment = 4
     // CHECK: alloca_memory_space = 5
     // CHECK: bitsize = 32
+    // CHECK: default_memory_space = 7
     // CHECK: endianness = "little"
     // CHECK: global_memory_space = 2
     // CHECK: index = 32
@@ -73,6 +78,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec<
     // CHECK: alignment = 4
     // CHECK: alloca_memory_space = 5
     // CHECK: bitsize = 32
+    // CHECK: default_memory_space = 7
     // CHECK: endianness = "little"
     // CHECK: global_memory_space = 2
     // CHECK: index = 32
@@ -84,6 +90,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec<
     // CHECK: alignment = 8
     // CHECK: alloca_memory_space = 5
     // CHECK: bitsize = 64
+    // CHECK: default_memory_space = 7
     // CHECK: endianness = "little"
     // CHECK: global_memory_space = 2
     // CHECK: index = 64
@@ -96,6 +103,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec<
     // CHECK: alignment = 8
     // CHECK: alloca_memory_space = 5
     // CHECK: bitsize = 32
+    // CHECK: default_memory_space = 7
     // CHECK: endianness = "little"
     // CHECK: global_memory_space = 2
     // CHECK: index = 24
diff --git a/mlir/test/Dialect/LLVMIR/rocdl.mlir b/mlir/test/Dialect/LLVMIR/rocdl.mlir
index 05469b64a8083..bc917041998d8 100644
--- a/mlir/test/Dialect/LLVMIR/rocdl.mlir
+++ b/mlir/test/Dialect/LLVMIR/rocdl.mlir
@@ -759,19 +759,27 @@ llvm.func @rocdl.raw.buffer.i32(%rsrc : vector<4xi32>,
   llvm.return
 }
 
-llvm.func @rocdl_8bit_floats(%source: i32, %stoch: i32) -> i32 {
+llvm.func @rocdl_8bit_floats(%source: i32, %source_half: f16, %source_bfloat: bf16, %stoch: i32) -> i32 {
 // CHECK-LABEL: @rocdl_8bit_floats
 // CHECK: rocdl.cvt.f32.bf8
 // CHECK: rocdl.cvt.f32.fp8
 // CHECK: rocdl.cvt.scalef32.f32.bf8
 // CHECK: rocdl.cvt.scalef32.f32.fp8
+// CHECK: rocdl.cvt.scalef32.pk.f16.bf8 
+// CHECK: rocdl.cvt.scalef32.pk.f16.fp8 
+// CHECK: rocdl.cvt.scalef32.f16.fp8
+// CHECK: rocdl.cvt.scalef32.f16.bf8
 // CHECK: rocdl.cvt.pk.bf8.f32
 // CHECK: rocdl.cvt.pk.fp8.f32
 // CHECK: rocdl.cvt.sr.bf8.f32
 // CHECK: rocdl.cvt.sr.fp8.f32
 // CHECK: rocdl.cvt.scalef32.sr.fp8.f32
+// CHECK: rocdl.cvt.scalef32.sr.fp8.f16
+// CHECK: rocdl.cvt.scalef32.sr.fp8.bf16
 // CHECK: rocdl.cvt.sr.bf8.f32
 // CHECK: rocdl.cvt.scalef32.sr.bf8.f32
+// CHECK: rocdl.cvt.scalef32.sr.bf8.f16
+// CHECK: rocdl.cvt.scalef32.sr.bf8.bf16
 // CHECK: rocdl.cvt.scalef32.pk.f32.fp8
 // CHECK: rocdl.cvt.scalef32.pk.f32.bf8
   %c0 = llvm.mlir.constant(0 : i32) : i32
@@ -783,13 +791,21 @@ llvm.func @rocdl_8bit_floats(%source: i32, %stoch: i32) -> i32 {
   %v2 = rocdl.cvt.f32.fp8 %source[%c0] : f32
   %v1_scaled = rocdl.cvt.scalef32.f32.bf8 %source[%c0], %c4 : f32
   %v2_scaled = rocdl.cvt.scalef32.f32.fp8 %source[%c0], %c4 : f32
+  %v3_scaled = rocdl.cvt.scalef32.pk.f16.bf8 %source[%false], %c4 : vector<2xf16>
+  %v4_scaled = rocdl.cvt.scalef32.pk.f16.fp8 %source[%false], %c4 : vector<2xf16>
+  %v5 = rocdl.cvt.scalef32.f16.fp8 %source[%false], %c4 -> %v3_scaled[%c0] : f16
+  %v6  = rocdl.cvt.scalef32.f16.bf8 %source[%false], %c4 -> %v3_scaled[%c0] : f16
   %source2 = rocdl.cvt.pk.bf8.f32 %v1, %v2 -> %source[%false] : i32
   %source3 = rocdl.cvt.pk.fp8.f32 %v1, %v2 -> %source2[%false] : i32
   %source4 = rocdl.cvt.sr.bf8.f32 %v1, %stoch -> %source3[%c2] : i32
   %source5 = rocdl.cvt.sr.fp8.f32 %v2, %stoch -> %source4[%c3] : i32
   %source5_scaled = rocdl.cvt.scalef32.sr.fp8.f32 %v2, %stoch, %c4 -> %source4[%c3] : i32
+  %source5_scaled_half = rocdl.cvt.scalef32.sr.fp8.f16 %source_half, %stoch, %c4 -> %source4[%c3] : i32
+  %source5_scaled_bfloat = rocdl.cvt.scalef32.sr.fp8.bf16 %source_bfloat, %stoch, %c4 -> %source4[%c3] : i32
   %source6 = rocdl.cvt.sr.bf8.f32 %v1, %stoch -> %source3[%c3] : i32
   %source6_scaled  = rocdl.cvt.scalef32.sr.bf8.f32 %v2, %stoch, %c4 -> %source3[%c3] : i32
+  %source6_scaled_half = rocdl.cvt.scalef32.sr.bf8.f16 %source_half, %stoch, %c4 -> %source3[%c3] : i32
+  %source6_scaled_bfloat =  rocdl.cvt.scalef32.sr.bf8.bf16 %source_bfloat, %stoch, %c4 -> %source3[%c3] : i32
   %source7_scaled = rocdl.cvt.scalef32.pk.f32.fp8 %source[%false], %c4 : f32
   %source8_scaled = rocdl.cvt.scalef32.pk.f32.bf8 %source[%false], %c4 : f32
   llvm.return %source5 : i32
@@ -805,6 +821,18 @@ llvm.func @rocdl_8bit_packed_v2i16(%sourceA: f32, %sourceB: f32, %old: vector<2x
   llvm.return %source_scaled : vector<2xi16>
 }
 
+llvm.func @rocdl_v2f16_v2i16(%source: vector<2xf16>, %source2: vector<2xbf16>, %old: vector<2xi16>) -> vector<2xi16> {
+// CHECK-LABEL: @rocdl_v2f16_v2i16
+// CHECK: rocdl.cvt.scalef32.pk.fp8.f16
+  %c0 = llvm.mlir.constant(1.0 : f32) : f32
+  %false = llvm.mlir.constant(false) : i1
+  %source_scaled = rocdl.cvt.scalef32.pk.fp8.f16 %source, %c0 -> %old[%false] : vector<2xi16>
+  %source2_scaled = rocdl.cvt.scalef32.pk.fp8.bf16 %source2, %c0 -> %old[%false] : vector<2xi16>
+  %source3_scaled = rocdl.cvt.scalef32.pk.bf8.f16 %source, %c0 -> %old[%false] : vector<2xi16>
+  %source4_scaled = rocdl.cvt.scalef32.pk.bf8.bf16 %source2, %c0 -> %old[%false] : vector<2xi16>
+  llvm.return %source_scaled : vector<2xi16>
+}
+
 llvm.func @rocdl.s.waitcnt() {
   // CHECK-LABEL: rocdl.s.waitcnt
   // CHECK: rocdl.s.waitcnt 0
diff --git a/mlir/test/Dialect/Linalg/reshape_fusion.mlir b/mlir/test/Dialect/Linalg/reshape_fusion.mlir
index 4da9c0851ac70..3244418d445b7 100644
--- a/mlir/test/Dialect/Linalg/reshape_fusion.mlir
+++ b/mlir/test/Dialect/Linalg/reshape_fusion.mlir
@@ -195,7 +195,7 @@ func.func @generic_op_reshape_consumer_static(%arg0: tensor<264x4xf32>)
 // CHECK-SAME:     : tensor<8x33x4xf32>
 //  CHECK-DAG:   %[[INIT:.+]] = tensor.empty()
 //      CHECK:   %[[T0:.+]] = tensor.expand_shape %[[ARG0]] {{\[\[}}0, 1], [2]] output_shape [8, 33, 4] : tensor<264x4xf32> into tensor<8x33x4xf32>
-//      CHECK:   %[[T1:.+]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1], [2]] output_shape [8, 33, 4] : tensor<264x4xf32> into tensor<8x33x4xf32>
+//      CHECK:   %[[T1:.+]] = tensor.expand_shape %[[INIT]] {{\[\[}}0, 1], [2]] output_shape [8, 33, 4] : tensor<264x4xf32> into tensor<8x33x4xf32>
 //      CHECK:   %[[T2:.+]] = linalg.generic
 // CHECK-SAME:     indexing_maps = [#[[MAP2]], #[[MAP2]], #[[MAP2]]]
 // CHECK-SAME:     ["parallel", "parallel", "parallel"]
@@ -203,6 +203,29 @@ func.func @generic_op_reshape_consumer_static(%arg0: tensor<264x4xf32>)
 // CHECK-SAME:     outs(%[[T1]] : tensor<8x33x4xf32>)
 //      CHECK:   return %[[T2]] : tensor<8x33x4xf32>
 
+// -----
+
+func.func @reshape_as_consumer_transpose
+  (%a :  tensor<4x210x6xf32>)
+    -> tensor<2x3x4x5x6x7xf32> {
+  %b = tensor.empty() : tensor<6x4x210xf32>
+  %c = linalg.transpose
+          ins(%a : tensor<4x210x6xf32>)
+         outs(%b : tensor<6x4x210xf32>) permutation = [2, 0, 1]
+  %d = tensor.expand_shape %c [[0, 1], [2], [3, 4, 5]] output_shape [2, 3, 4, 5, 6, 7] : tensor<6x4x210xf32> into tensor<2x3x4x5x6x7xf32>
+  return %d : tensor<2x3x4x5x6x7xf32>
+}
+//      CHECK: func @reshape_as_consumer_transpose
+// CHECK-SAME:   %[[ARG0:[a-zA-Z0-9_]+]]: tensor<4x210x6xf32>
+//  CHECK-DAG:   %[[INIT:.+]] = tensor.empty()
+//  CHECK-DAG:   %[[T0:.+]] = tensor.expand_shape %[[ARG0]] {{\[\[}}0], [1, 2, 3], [4, 5]] output_shape [4, 5, 6, 7, 2, 3] : tensor<4x210x6xf32> into tensor<4x5x6x7x2x3xf32>
+//  CHECK-DAG:   %[[T1:.+]] = tensor.expand_shape %[[INIT]] {{\[\[}}0, 1], [2], [3, 4, 5]] output_shape [2, 3, 4, 5, 6, 7] : tensor<6x4x210xf32> into tensor<2x3x4x5x6x7xf32
+//      CHECK:   %[[T2:.+]] = linalg.transpose ins(%[[T0]] : tensor<4x5x6x7x2x3xf32>)
+// CHECK-SAME:     outs(%[[T1]] : tensor<2x3x4x5x6x7xf32>)
+// CHECK-SAME:     permutation = [4, 5, 0, 1, 2, 3]
+//      CHECK:   return %[[T2]] : tensor<2x3x4x5x6x7xf32>
+
+
 // -----
 
 #map0 = affine_map<(d0, d1, d2) -> (d2, d0, d1)>
@@ -482,6 +505,31 @@ func.func @generic_op_reshape_consumer_fusion_projected(%arg0 : tensor<?x?xf32>,
 
 // -----
 
+func.func @fuse_collapse_reduction(%arg0: tensor<10x10x20xf32>) -> tensor<100xf32> {
+  %c0 = arith.constant 0 : index
+  %c_0 = arith.constant 0.0 : f32
+  %0 = tensor.collapse_shape %arg0 [[0, 1], [2]] : tensor<10x10x20xf32> into tensor<100x20xf32>
+  %2 = tensor.empty() : tensor<100xf32>
+  %3 = linalg.fill ins(%c_0 : f32) outs(%2 : tensor<100xf32>) -> tensor<100xf32>
+  %4 = linalg.generic {
+    indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0)>],
+    iterator_types = ["parallel", "reduction"]}
+    ins(%0 : tensor<100x20xf32>) outs(%3 : tensor<100xf32>) {
+      ^bb0(%arg1 : f32, %arg2: f32):
+        %4 = arith.addf %arg1, %arg2 : f32
+        linalg.yield %4 : f32
+    } -> tensor<100xf32>
+  return %4 : tensor<100xf32>
+}
+
+//      CHECK: func @fuse_collapse_reduction
+// CHECK-SAME:     %[[ARG0:.+]]: tensor<10x10x20xf32>
+//      CHECK:   %[[GENERIC:.+]] = linalg.generic
+// CHECK-SAME:       ins(%[[ARG0]] : tensor<10x10x20xf32>)
+//      CHECK:   %[[COLLAPSE:.+]] = tensor.collapse_shape %[[GENERIC]]
+//      CHECK:   return %[[COLLAPSE]]
+// -----
+
 func.func @no_fuse_dynamic_dims(%arg0: tensor<?x?xf32>) -> tensor<?xf32> {
   %c0 = arith.constant 0 : index
   %0 = tensor.collapse_shape %arg0 [[0, 1]] : tensor<?x?xf32> into tensor<?xf32>
@@ -859,37 +907,29 @@ func.func @linalg_copy_reshape_producer_fusion(%arg0 : tensor<?x7x?x8xf32>,
 
 // -----
 
-func.func @linalg_transpose_reshape_producer_fusion(%arg0 : tensor<?x7x?x8xf32>,
-                                              %arg1 : tensor<?x?xf32>) ->
-                                              tensor<?x?xf32>
-{
-  %0 = tensor.collapse_shape %arg0 [[0, 1], [2, 3]] :
-    tensor<?x7x?x8xf32> into tensor<?x?xf32>
-  %1 = linalg.transpose ins(%0 : tensor<?x?xf32>)
-       outs(%arg1 : tensor<?x?xf32>) permutation = [1, 0]
-  return %1 : tensor<?x?xf32>
+
+func.func @reshape_as_producer_transpose
+  (%a :  tensor<4x5x6x7x2x3xf32>)
+    -> tensor<6x4x210xf32> {
+  %b = tensor.empty() : tensor<6x4x210xf32>
+  %c = tensor.collapse_shape %a [[0], [1, 2, 3], [4, 5]] :
+    tensor<4x5x6x7x2x3xf32> into tensor<4x210x6xf32>
+  %d = linalg.transpose
+          ins(%c : tensor<4x210x6xf32>)
+         outs(%b : tensor<6x4x210xf32>) permutation = [2, 0, 1]
+  return %d : tensor<6x4x210xf32>
 }
 
-//      CHECK: func @linalg_transpose_reshape_producer_fusion
-// CHECK-SAME:   %[[ARG0:[a-zA-Z0-9_]+]]: tensor<?x7x?x8xf32>
-// CHECK-SAME:   %[[ARG1:[a-zA-Z0-9_]+]]: tensor<?x?xf32>
-//  CHECK-DAG:   %[[C8:.+]] = arith.constant 8 : index
-//  CHECK-DAG:   %[[C7:.+]] = arith.constant 7 : index
-//  CHECK-DAG:   %[[C1:.+]] = arith.constant 1 : index
-//  CHECK-DAG:   %[[C0:.+]] = arith.constant 0 : index
-//  CHECK-DAG:   %[[DIM:.+]] = tensor.dim %[[ARG1]], %[[C0]] : tensor<?x?xf32>
-//  CHECK-DAG:   %[[DIM_0:.+]] = tensor.dim %[[ARG1]], %[[C1]] : tensor<?x?xf32>
-//  CHECK-DAG:   %[[VAL_0:.+]] = arith.divsi %[[DIM_0]], %[[C7]] : index
-//  CHECK-DAG:   %[[VAL_1:.+]] = arith.divsi %[[DIM]], %[[C8]] : index
-//      CHECK:   %[[T1:.+]] = tensor.expand_shape %[[ARG1]] {{\[\[}}0, 1], [2, 3]] output_shape [%[[VAL_1]], 8, %[[VAL_0]], 7] : tensor<?x?xf32> into tensor<?x8x?x7xf32>
-//      CHECK:   %[[T2:.+]] = linalg.transpose
-// CHECK-SAME:     ins(%[[ARG0]] : tensor<?x7x?x8xf32>)
-// CHECK-SAME:     outs(%[[T1]] : tensor<?x8x?x7xf32>)
-// CHECK-SAME:   permutation = [2, 3, 0, 1]
-//      CHECK:   %[[T3:.+]] = tensor.collapse_shape %[[T2]]
-// CHECK-SAME:     [0, 1], [2, 3]
-// CHECK-SAME:     tensor<?x8x?x7xf32> into tensor<?x?xf32>
-//      CHECK:   return %[[T3]]
+//      CHECK: func @reshape_as_producer_transpose
+// CHECK-SAME:   %[[ARG0:[a-zA-Z0-9_]+]]: tensor<4x5x6x7x2x3xf32>
+//  CHECK-DAG:   %[[INIT:.+]] = tensor.empty()
+//  CHECK-DAG:   %[[T0:.+]] = tensor.expand_shape %[[INIT]] {{\[\[}}0, 1], [2], [3, 4, 5]] output_shape [2, 3, 4, 5, 6, 7] : tensor<6x4x210xf32> into tensor<2x3x4x5x6x7xf32>
+//      CHECK:   %[[T1:.+]] = linalg.transpose ins(%[[ARG0]] : tensor<4x5x6x7x2x3xf32>)
+// CHECK-SAME:     outs(%[[T0]] : tensor<2x3x4x5x6x7xf32>)
+// CHECK-SAME:     permutation = [4, 5, 0, 1, 2, 3]
+//      CHECK:   %[[T2:.+]] = tensor.collapse_shape %[[T1]] {{\[\[}}0, 1], [2], [3, 4, 5]] : tensor<2x3x4x5x6x7xf32> into tensor<6x4x210xf32>
+//      CHECK:   return %[[T2]] : tensor<6x4x210xf32>
+
 
 // -----
 
diff --git a/mlir/test/Dialect/Mesh/sharding-propagation.mlir b/mlir/test/Dialect/Mesh/sharding-propagation.mlir
index 83136f613b020..aa5fa00488f08 100644
--- a/mlir/test/Dialect/Mesh/sharding-propagation.mlir
+++ b/mlir/test/Dialect/Mesh/sharding-propagation.mlir
@@ -77,7 +77,7 @@ func.func @element_wise_on_graph_input(%arg0: tensor<8x16xf32>) -> tensor<8x16xf
 
 // CHECK-LABEL: func.func @arrow_structure
 // CHECK-SAME:    %[[ARG:.*]]: tensor<8x16xf32>
-func.func @arrow_structure(%arg0: tensor<8x16xf32>) -> (tensor<8x16xf32>, tensor<8x16xf32>) {
+func.func @arrow_structure(%arg0: tensor<8x16xf32>, %arg1: tensor<1xf32>, %arg2: tensor<1xf32>) -> (tensor<8x16xf32>, tensor<8x16xf32>) {
   // CHECK-NEXT:  %[[S1:.*]] = mesh.sharding @mesh_2d split_axes = {{\[\[}}0], [1]] : !mesh.sharding
   // CHECK-NEXT:  %[[V1:.*]] = mesh.shard %[[ARG]] to %[[S1]] annotate_for_users  : tensor<8x16xf32>
   // CHECK-NEXT:  %[[V2:.*]] = tosa.tanh %[[V1]]
@@ -85,12 +85,15 @@ func.func @arrow_structure(%arg0: tensor<8x16xf32>) -> (tensor<8x16xf32>, tensor
   %0 = tosa.tanh %arg0 : (tensor<8x16xf32>) -> tensor<8x16xf32>
   // CHECK-NEXT:  %[[V4:.*]] = mesh.shard %[[V3]] to %[[S1]] annotate_for_users  : tensor<8x16xf32>
   // CHECK-NEXT:  %[[V5:.*]] = tosa.abs %[[V4]]
- // CHECK-NEXT:   %[[V6:.*]] = mesh.shard %[[V5]] to %[[S1]]  : tensor<8x16xf32>
-  %1 = tosa.abs %0 : (tensor<8x16xf32>) -> tensor<8x16xf32>
-  // CHECK-NEXT:  %[[V7:.*]] = tosa.negate %[[V4]]
+  // CHECK-NEXT:  %[[V6:.*]] = mesh.shard %[[V5]] to %[[S1]]  : tensor<8x16xf32>
+  %1 = tosa.abs %0: (tensor<8x16xf32>) -> tensor<8x16xf32>
+  // CHECK-NEXT:  %[[S3:.*]] = mesh.sharding @mesh_2d split_axes = {{\[\[}}]] : !mesh.sharding
+  // CHECK-NEXT:  %[[ZP1:.*]] = mesh.shard %arg1 to %[[S3]] annotate_for_users : tensor<1xf32>
+  // CHECK-NEXT:  %[[ZP2:.*]] = mesh.shard %arg2 to %[[S3]] annotate_for_users : tensor<1xf32>
+  // CHECK-NEXT:  %[[V7:.*]] = tosa.negate %[[V4]], %[[ZP1]], %[[ZP2]]
   // CHECK-NEXT:  %[[S8:.*]] = mesh.sharding @mesh_2d split_axes = {{\[\[}}0], [1]] : !mesh.sharding
   // CHECK-NEXT:  %[[V8:.*]] = mesh.shard %[[V7]] to %[[S8]]  : tensor<8x16xf32>
-  %2 = tosa.negate %0 : (tensor<8x16xf32>) -> tensor<8x16xf32>
+  %2 = tosa.negate %0, %arg1, %arg2 : (tensor<8x16xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<8x16xf32>
   %s3 = mesh.sharding @mesh_2d split_axes = [[0], [1]] : !mesh.sharding
   %3 = mesh.shard %2 to %s3  : tensor<8x16xf32>
   // CHECK-NEXT: return %[[V6]], %[[V8]]
@@ -98,14 +101,16 @@ func.func @arrow_structure(%arg0: tensor<8x16xf32>) -> (tensor<8x16xf32>, tensor
 }
 
 // CHECK-LABEL: func.func @matmul_on_def_shard_batch_and_m
-// CHECK-SAME:     %[[ARG0:.*]]: tensor<2x16x8xf32>, %[[ARG1:.*]]: tensor<2x8x32xf32>
-func.func @matmul_on_def_shard_batch_and_m(%arg0: tensor<2x16x8xf32>, %arg1: tensor<2x8x32xf32>) -> tensor<2x16x32xf32> {
+// CHECK-SAME:     %[[ARG0:.*]]: tensor<2x16x8xf32>, %[[ARG1:.*]]: tensor<2x8x32xf32>, %[[ARG2:.*]]: tensor<1xf32>
+func.func @matmul_on_def_shard_batch_and_m(%arg0: tensor<2x16x8xf32>, %arg1: tensor<2x8x32xf32>, %arg2: tensor<1xf32>) -> tensor<2x16x32xf32> {
   // CHECK-NEXT:  %[[S0:.*]] = mesh.sharding @mesh_2d split_axes = {{\[\[}}0], [1]] : !mesh.sharding
   // CHECK-NEXT:  %[[V0:.*]] = mesh.shard %[[ARG0]] to %[[S0]] annotate_for_users  : tensor<2x16x8xf32>
   // CHECK-NEXT:  %[[S1:.*]] = mesh.sharding @mesh_2d split_axes = {{\[\[}}0]] : !mesh.sharding
   // CHECK-NEXT:  %[[V1:.*]] = mesh.shard %[[ARG1]] to %[[S1]] annotate_for_users  : tensor<2x8x32xf32>
-  // CHECK-NEXT:  %[[V2:.*]] = tosa.matmul %[[V0]], %[[V1]]
-  %0 = tosa.matmul %arg0, %arg1 : (tensor<2x16x8xf32>, tensor<2x8x32xf32>) -> tensor<2x16x32xf32>
+  // CHECK-NEXT:  %[[S2:.*]] = mesh.sharding @mesh_2d split_axes = {{\[\[}}]] : !mesh.sharding
+  // CHECK-NEXT:  %[[ZP:.*]] = mesh.shard %[[ARG2]] to %[[S2]] annotate_for_users  : tensor<1xf32>
+  // CHECK-NEXT:  %[[V2:.*]] = tosa.matmul %[[V0]], %[[V1]], %[[ZP]], %[[ZP]]
+  %0 = tosa.matmul %arg0, %arg1, %arg2, %arg2 : (tensor<2x16x8xf32>, tensor<2x8x32xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<2x16x32xf32>
   // CHECK-NEXT:  %[[S3:.*]] = mesh.sharding @mesh_2d split_axes = {{\[\[}}0], [1]] : !mesh.sharding
   // CHECK-NEXT:  %[[V3:.*]] = mesh.shard %[[V2]] to %[[S3]]  : tensor<2x16x32xf32>
   %s1 = mesh.sharding @mesh_2d split_axes = [[0], [1]] : !mesh.sharding
@@ -115,14 +120,16 @@ func.func @matmul_on_def_shard_batch_and_m(%arg0: tensor<2x16x8xf32>, %arg1: ten
 }
 
 // CHECK-LABEL: func.func @matmul_on_def_shard_m_and_k
-// CHECK-SAME:     %[[ARG0:.*]]: tensor<2x16x8xf32>, %[[ARG1:.*]]: tensor<2x8x32xf32>
-func.func @matmul_on_def_shard_m_and_k(%arg0: tensor<2x16x8xf32>, %arg1: tensor<2x8x32xf32>) -> tensor<2x16x32xf32> {
+// CHECK-SAME:     %[[ARG0:.*]]: tensor<2x16x8xf32>, %[[ARG1:.*]]: tensor<2x8x32xf32>, %[[ARG2:.*]]: tensor<1xf32>
+func.func @matmul_on_def_shard_m_and_k(%arg0: tensor<2x16x8xf32>, %arg1: tensor<2x8x32xf32>, %arg2: tensor<1xf32>) -> tensor<2x16x32xf32> {
   // CHECK-NEXT:  %[[S0:.*]] = mesh.sharding @mesh_2d split_axes = {{\[\[}}], [1], [0]] : !mesh.sharding
   // CHECK-NEXT:  %[[V0:.*]] = mesh.shard %[[ARG0]] to %[[S0]] annotate_for_users  : tensor<2x16x8xf32>
   // CHECK-NEXT:  %[[S1:.*]] = mesh.sharding @mesh_2d split_axes = {{\[\[}}], [0]] : !mesh.sharding
   // CHECK-NEXT:  %[[V1:.*]] = mesh.shard %[[ARG1]] to %[[S1]] annotate_for_users  : tensor<2x8x32xf32>
-  // CHECK-NEXT:  %[[V2:.*]] = tosa.matmul %[[V0]], %[[V1]]
-  %0 = tosa.matmul %arg0, %arg1 : (tensor<2x16x8xf32>, tensor<2x8x32xf32>) -> tensor<2x16x32xf32>
+  // CHECK-NEXT:  %[[S2:.*]] = mesh.sharding @mesh_2d split_axes = {{\[\[}}]] : !mesh.sharding
+  // CHECK-NEXT:  %[[ZP:.*]] = mesh.shard %[[ARG2]] to %[[S2]] annotate_for_users  : tensor<1xf32>
+  // CHECK-NEXT:  %[[V2:.*]] = tosa.matmul %[[V0]], %[[V1]], %[[ZP]], %[[ZP]]
+  %0 = tosa.matmul %arg0, %arg1, %arg2, %arg2 : (tensor<2x16x8xf32>, tensor<2x8x32xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<2x16x32xf32>
   // CHECK-NEXT:  %[[S3:.*]] = mesh.sharding @mesh_2d split_axes = {{\[\[}}], [1]] partial = sum [0] : !mesh.sharding
   // CHECK-NEXT:  %[[V3:.*]] = mesh.shard %[[V2]] to %[[S3]]  : tensor<2x16x32xf32>
   %s1 = mesh.sharding @mesh_2d split_axes = [[], [1]] partial = sum [0] : !mesh.sharding
@@ -132,16 +139,18 @@ func.func @matmul_on_def_shard_m_and_k(%arg0: tensor<2x16x8xf32>, %arg1: tensor<
 }
 
 // CHECK-LABEL: func.func @matmul_on_use_shard_m_and_k
-// CHECK-SAME:     %[[ARG0:.*]]: tensor<2x16x8xf32>, %[[ARG1:.*]]: tensor<2x8x32xf32>
-func.func @matmul_on_use_shard_m_and_k(%arg0: tensor<2x16x8xf32>, %arg1: tensor<2x8x32xf32>) -> tensor<2x16x32xf32> {
+// CHECK-SAME:     %[[ARG0:.*]]: tensor<2x16x8xf32>, %[[ARG1:.*]]: tensor<2x8x32xf32>, %[[ARG2:.*]]: tensor<1xf32>
+func.func @matmul_on_use_shard_m_and_k(%arg0: tensor<2x16x8xf32>, %arg1: tensor<2x8x32xf32>, %arg2: tensor<1xf32>) -> tensor<2x16x32xf32> {
   // CHECK-NEXT:  %[[S0:.*]] = mesh.sharding @mesh_2d split_axes = {{\[\[}}], [1], [0]] : !mesh.sharding
   // CHECK-NEXT:  %[[V0:.*]] = mesh.shard %[[ARG0]] to %[[S0]] annotate_for_users  : tensor<2x16x8xf32>
   %s0 = mesh.sharding @mesh_2d split_axes = [[], [1], [0]] : !mesh.sharding
   %0 = mesh.shard %arg0 to %s0 annotate_for_users  : tensor<2x16x8xf32>
   // CHECK-NEXT:  %[[S1:.*]] = mesh.sharding @mesh_2d split_axes = {{\[\[}}], [0]] : !mesh.sharding
   // CHECK-NEXT:  %[[V1:.*]] = mesh.shard %[[ARG1]] to %[[S1]] annotate_for_users  : tensor<2x8x32xf32>
-  // CHECK-NEXT:  %[[V2:.*]] = tosa.matmul %[[V0]], %[[V1]]
-  %1 = tosa.matmul %0, %arg1 : (tensor<2x16x8xf32>, tensor<2x8x32xf32>) -> tensor<2x16x32xf32>
+  // CHECK-NEXT:  %[[S2:.*]] = mesh.sharding @mesh_2d split_axes = {{\[\[}}]] : !mesh.sharding
+  // CHECK-NEXT:  %[[ZP:.*]] = mesh.shard %[[ARG2]] to %[[S2]] annotate_for_users  : tensor<1xf32>
+  // CHECK-NEXT:  %[[V2:.*]] = tosa.matmul %[[V0]], %[[V1]], %[[ZP]], %[[ZP]]
+  %1 = tosa.matmul %0, %arg1, %arg2, %arg2 : (tensor<2x16x8xf32>, tensor<2x8x32xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<2x16x32xf32>
   // CHECK-NEXT:  %[[S3:.*]] = mesh.sharding @mesh_2d split_axes = {{\[\[}}], [1]] partial = sum [0] : !mesh.sharding
   // CHECK-NEXT:  %[[V3:.*]] = mesh.shard %[[V2]] to %[[S3]]  : tensor<2x16x32xf32>
   // CHECK-NEXT:  return %[[V3]]
@@ -149,8 +158,8 @@ func.func @matmul_on_use_shard_m_and_k(%arg0: tensor<2x16x8xf32>, %arg1: tensor<
 }
 
 // CHECK-LABEL: func.func @matmul_on_use_shard_m_and_duplicted_k
-// CHECK-SAME:     %[[ARG0:.*]]: tensor<2x16x8xf32>, %[[ARG1:.*]]: tensor<2x8x32xf32>
-func.func @matmul_on_use_shard_m_and_duplicted_k(%arg0: tensor<2x16x8xf32>, %arg1: tensor<2x8x32xf32>) -> tensor<2x16x32xf32> {
+// CHECK-SAME:     %[[ARG0:.*]]: tensor<2x16x8xf32>, %[[ARG1:.*]]: tensor<2x8x32xf32>, %[[ARG2:.*]]: tensor<1xf32>
+func.func @matmul_on_use_shard_m_and_duplicted_k(%arg0: tensor<2x16x8xf32>, %arg1: tensor<2x8x32xf32>, %arg2: tensor<1xf32>) -> tensor<2x16x32xf32> {
   // CHECK-NEXT:  %[[S0:.*]] = mesh.sharding @mesh_2d split_axes = {{\[\[}}], [1], [0]] : !mesh.sharding
   // CHECK-NEXT:  %[[V0:.*]] = mesh.shard %[[ARG0]] to %[[S0]] annotate_for_users  : tensor<2x16x8xf32>
   %s0 = mesh.sharding @mesh_2d split_axes = [[], [1], [0]] : !mesh.sharding
@@ -159,8 +168,10 @@ func.func @matmul_on_use_shard_m_and_duplicted_k(%arg0: tensor<2x16x8xf32>, %arg
   // CHECK-NEXT:  %[[V1:.*]] = mesh.shard %[[ARG1]] to %[[S1]] annotate_for_users  : tensor<2x8x32xf32>
   %s1 = mesh.sharding @mesh_2d split_axes = [[], [0]] : !mesh.sharding
   %1 = mesh.shard %arg1 to %s1 annotate_for_users  : tensor<2x8x32xf32>
-  // CHECK-NEXT:  %[[V2:.*]] = tosa.matmul %[[V0]], %[[V1]]
-  %2 = tosa.matmul %0, %1 : (tensor<2x16x8xf32>, tensor<2x8x32xf32>) -> tensor<2x16x32xf32>
+  // CHECK-NEXT:  %[[S2:.*]] = mesh.sharding @mesh_2d split_axes = {{\[\[}}]] : !mesh.sharding
+  // CHECK-NEXT:  %[[ZP:.*]] = mesh.shard %[[ARG2]] to %[[S2]] annotate_for_users  : tensor<1xf32>
+  // CHECK-NEXT:  %[[V2:.*]] = tosa.matmul %[[V0]], %[[V1]], %[[ZP]], %[[ZP]]
+  %2 = tosa.matmul %0, %1, %arg2, %arg2 : (tensor<2x16x8xf32>, tensor<2x8x32xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<2x16x32xf32>
   // CHECK-NEXT:  %[[S3:.*]] = mesh.sharding @mesh_2d split_axes = {{\[\[}}], [1]] partial = sum [0] : !mesh.sharding
   // CHECK-NEXT:  %[[V3:.*]] = mesh.shard %[[V2]] to %[[S3]]  : tensor<2x16x32xf32>
   // CHECK-NEXT:  return %[[V3]]
@@ -199,14 +210,16 @@ func.func @resolve_conflicting_annotations(
 
 // https://arxiv.org/abs/2211.05102 Figure 2(a)
 // CHECK-LABEL: func.func @mlp_1d_weight_stationary
-// CHECK-SAME:     %[[ARG0:.*]]: tensor<2x4x8xf32>, %[[ARG1:.*]]: tensor<2x8x32xf32>, %[[ARG2:.*]]: tensor<2x32x8xf32>
-func.func @mlp_1d_weight_stationary(%arg0: tensor<2x4x8xf32>, %arg1: tensor<2x8x32xf32>, %arg2: tensor<2x32x8xf32>) -> tensor<2x4x8xf32> {
+// CHECK-SAME:     %[[ARG0:.*]]: tensor<2x4x8xf32>, %[[ARG1:.*]]: tensor<2x8x32xf32>, %[[ARG2:.*]]: tensor<2x32x8xf32>, %[[ARG3:.*]]: tensor<1xf32>
+func.func @mlp_1d_weight_stationary(%arg0: tensor<2x4x8xf32>, %arg1: tensor<2x8x32xf32>, %arg2: tensor<2x32x8xf32>, %arg3: tensor<1xf32>) -> tensor<2x4x8xf32> {
   %s0 = mesh.sharding @mesh_1d split_axes = [[], [], [0]] : !mesh.sharding
   %0 = mesh.shard %arg0 to %s0  : tensor<2x4x8xf32>
   // CHECK-DAG: %[[S1:.*]] = mesh.sharding @mesh_1d split_axes = {{\[\[}}], [], [0]] : !mesh.sharding
   // CHECK-DAG: %[[S2:.*]] = mesh.sharding @mesh_1d split_axes = {{\[\[}}], [], [0]] : !mesh.sharding
+  // CHECK-DAG: %[[S3:.*]] = mesh.sharding @mesh_1d split_axes = {{\[\[}}]] : !mesh.sharding
+  // CHECK-DAG:  %[[ZP:.*]] = mesh.shard %[[ARG3]] to %[[S3]] annotate_for_users  : tensor<1xf32>
   // CHECK: %[[V0:.*]] = tosa.matmul
-  %1 = tosa.matmul %0, %arg1 : (tensor<2x4x8xf32>, tensor<2x8x32xf32>) -> tensor<2x4x32xf32>
+  %1 = tosa.matmul %0, %arg1, %arg3, %arg3 : (tensor<2x4x8xf32>, tensor<2x8x32xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<2x4x32xf32>
   // CHECK-NEXT:  %[[V1:.*]] = mesh.shard %[[V0]] to %[[S2]]  : tensor<2x4x32xf32>
   // CHECK-NEXT:  %[[V2:.*]] = mesh.shard %[[V1]] to %[[S2]] annotate_for_users  : tensor<2x4x32xf32>
   // CHECK-DAG: %[[V3:.*]] = tosa.sigmoid %[[V2]]
@@ -215,8 +228,8 @@ func.func @mlp_1d_weight_stationary(%arg0: tensor<2x4x8xf32>, %arg1: tensor<2x8x
   // CHECK-NEXT:  %[[V5:.*]] = mesh.shard %[[V4]] to %[[S2]] annotate_for_users  : tensor<2x4x32xf32>
   // CHECK-DAG: %[[S6:.*]] = mesh.sharding @mesh_1d split_axes = {{\[\[}}], [0]] : !mesh.sharding
   // CHECK-NEXT:  %[[V6:.*]] = mesh.shard %[[ARG2]] to %[[S6]] annotate_for_users  : tensor<2x32x8xf32>
-  // CHECK-DAG: %[[V7:.*]] = tosa.matmul %[[V5]], %[[V6]]
-  %3 = tosa.matmul %2, %arg2 : (tensor<2x4x32xf32>, tensor<2x32x8xf32>) -> tensor<2x4x8xf32>
+  // CHECK-DAG: %[[V7:.*]] = tosa.matmul %[[V5]], %[[V6]], %[[ZP]], %[[ZP]]
+  %3 = tosa.matmul %2, %arg2, %arg3, %arg3 : (tensor<2x4x32xf32>, tensor<2x32x8xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<2x4x8xf32>
   %s4 = mesh.sharding @mesh_1d split_axes = [[], [], []] partial = sum [0] : !mesh.sharding
   %4 = mesh.shard %3 to %s4  : tensor<2x4x8xf32>
   // CHECK: %[[S8:.*]] = mesh.sharding @mesh_1d split_axes = {{\[\[}}], [], []] partial =  sum [0] : !mesh.sharding
@@ -230,8 +243,8 @@ func.func @mlp_1d_weight_stationary(%arg0: tensor<2x4x8xf32>, %arg1: tensor<2x8x
 
 // https://arxiv.org/abs/2211.05102 Figure 2(b)
 // CHECK-LABEL: func.func @mlp_2d_weight_stationary
-// CHECK-SAME:     %[[ARG0:.*]]: tensor<2x4x8xf32>, %[[ARG1:.*]]: tensor<2x8x32xf32>, %[[ARG2:.*]]: tensor<2x32x8xf32>
-func.func @mlp_2d_weight_stationary(%arg0: tensor<2x4x8xf32>, %arg1: tensor<2x8x32xf32>, %arg2: tensor<2x32x8xf32>) -> tensor<2x4x8xf32> {
+// CHECK-SAME:     %[[ARG0:.*]]: tensor<2x4x8xf32>, %[[ARG1:.*]]: tensor<2x8x32xf32>, %[[ARG2:.*]]: tensor<2x32x8xf32>, %[[ARG3:.*]]: tensor<1xf32>
+func.func @mlp_2d_weight_stationary(%arg0: tensor<2x4x8xf32>, %arg1: tensor<2x8x32xf32>, %arg2: tensor<2x32x8xf32>, %arg3: tensor<1xf32>) -> tensor<2x4x8xf32> {
   // CHECK-DAG: %[[S0:.*]] = mesh.sharding @mesh_3d split_axes = {{\[\[}}], [], [0, 1, 2]] : !mesh.sharding
   // CHECK-NEXT:  %[[V0:.*]] = mesh.shard %[[ARG0]] to %[[S0]]  : tensor<2x4x8xf32>
   %s0 = mesh.sharding @mesh_3d split_axes = [[], [], [0, 1, 2]] : !mesh.sharding
@@ -240,8 +253,10 @@ func.func @mlp_2d_weight_stationary(%arg0: tensor<2x4x8xf32>, %arg1: tensor<2x8x
   // CHECK-NEXT:  %[[V1:.*]] = mesh.shard %[[V0]] to %[[S1]] annotate_for_users  : tensor<2x4x8xf32>
   // CHECK-DAG: %[[S2:.*]] = mesh.sharding @mesh_3d split_axes = {{\[\[}}], [0], [1, 2]] : !mesh.sharding
   // CHECK-NEXT:  %[[V2:.*]] = mesh.shard %[[ARG1]] to %[[S2]] annotate_for_users  : tensor<2x8x32xf32>
-  // CHECK-DAG: %[[V3:.*]] = tosa.matmul %[[V1]], %[[V2]]
-  %1 = tosa.matmul %0, %arg1 : (tensor<2x4x8xf32>, tensor<2x8x32xf32>) -> tensor<2x4x32xf32>
+  // CHECK-DAG: %[[S3:.*]] = mesh.sharding @mesh_3d split_axes = {{\[\[}}]] : !mesh.sharding
+  // CHECK-DAG:  %[[ZP:.*]] = mesh.shard %[[ARG3]] to %[[S3]] annotate_for_users  : tensor<1xf32>
+  // CHECK-DAG: %[[V3:.*]] = tosa.matmul %[[V1]], %[[V2]], %[[ZP]], %[[ZP]]
+  %1 = tosa.matmul %0, %arg1, %arg3, %arg3 : (tensor<2x4x8xf32>, tensor<2x8x32xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<2x4x32xf32>
   // CHECK-DAG: %[[S4:.*]] = mesh.sharding @mesh_3d split_axes = {{\[\[}}], [], [1, 2]] partial = sum [0] : !mesh.sharding
   // CHECK-NEXT:  %[[V4:.*]] = mesh.shard %[[V3]] to %[[S4]]  : tensor<2x4x32xf32>
   %s2 = mesh.sharding @mesh_3d split_axes = [[], [], [1, 2]] partial = sum [0] : !mesh.sharding
@@ -254,8 +269,8 @@ func.func @mlp_2d_weight_stationary(%arg0: tensor<2x4x8xf32>, %arg1: tensor<2x8x
   // CHECK-NEXT:  %[[V8:.*]] = mesh.shard %[[V7]] to %[[S5]] annotate_for_users  : tensor<2x4x32xf32>
   // CHECK-DAG: %[[S9:.*]] = mesh.sharding @mesh_3d split_axes = {{\[\[}}], [1, 2], [0]] : !mesh.sharding
   // CHECK-NEXT:  %[[V9:.*]] = mesh.shard %[[ARG2]] to %[[S9]] annotate_for_users  : tensor<2x32x8xf32>
-  // CHECK-DAG: %[[V10:.*]] = tosa.matmul %[[V8]], %[[V9]]
-  %4 = tosa.matmul %3, %arg2 : (tensor<2x4x32xf32>, tensor<2x32x8xf32>) -> tensor<2x4x8xf32>
+  // CHECK-DAG: %[[V10:.*]] = tosa.matmul %[[V8]], %[[V9]], %[[ZP]], %[[ZP]]
+  %4 = tosa.matmul %3, %arg2, %arg3, %arg3 : (tensor<2x4x32xf32>, tensor<2x32x8xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<2x4x8xf32>
   // CHECK-DAG: %[[S11:.*]] = mesh.sharding @mesh_3d split_axes = {{\[\[}}], [], [0]] partial = sum [1, 2] : !mesh.sharding
   // CHECK-NEXT:  %[[V11:.*]] = mesh.shard %[[V10]] to %[[S11]]  : tensor<2x4x8xf32>
   %s5 = mesh.sharding @mesh_3d split_axes = [[], [], [0]] partial = sum[1, 2] : !mesh.sharding
diff --git a/mlir/test/Dialect/Mesh/spmdization.mlir b/mlir/test/Dialect/Mesh/spmdization.mlir
index 59f7162e21013..5c9fd29444f04 100644
--- a/mlir/test/Dialect/Mesh/spmdization.mlir
+++ b/mlir/test/Dialect/Mesh/spmdization.mlir
@@ -176,7 +176,7 @@ func.func @multiple_chained_ops(
   %4 = mesh.shard %3 to %s4  annotate_for_users : tensor<2xi8>
   // CHECK: %[[ABS2:.*]] = tosa.abs %[[RESHARD2]] : (tensor<2xi8>) -> tensor<2xi8>
   %5 = tosa.abs %4 : (tensor<2xi8>) -> tensor<2xi8>
-  // CHECK: %[[RESHARD3:.*]] = mesh.all_slice %[[ABS2]] on @mesh_1d mesh_axes = [0] slice_axis = 0 : 
+  // CHECK: %[[RESHARD3:.*]] = mesh.all_slice %[[ABS2]] on @mesh_1d mesh_axes = [0] slice_axis = 0 :
   // CHECK-SAME: tensor<2xi8> -> tensor<1xi8>
   %s6 = mesh.sharding @mesh_1d split_axes = [[]] : !mesh.sharding
   %6 = mesh.shard %5 to %s6  : tensor<2xi8>
@@ -207,7 +207,11 @@ mesh.mesh @mesh_1d_4(shape = 4)
 // CHECK-LABEL: func @ew_chain_with_halo
 func.func @ew_chain_with_halo(
   // CHECK-SAME: %[[IN1:[A-Za-z0-9_]+]]: tensor<5x16xf32>
-  %arg0: tensor<8x16xf32>)
+  %arg0: tensor<8x16xf32>,
+  // CHECK-SAME: %[[IN2:[A-Za-z0-9_]+]]: tensor<1xf32>
+  %arg1: tensor<1xf32>,
+  // CHECK-SAME: %[[IN3:[A-Za-z0-9_]+]]: tensor<1xf32>
+  %arg2: tensor<1xf32>)
   // CHECK-SAME: -> tensor<5x16xf32>
    -> tensor<8x16xf32> {
   %ssharding_annotated = mesh.sharding @mesh_1d_4 split_axes = [[0]] halo_sizes = [2, 1] : !mesh.sharding
@@ -224,8 +228,11 @@ func.func @ew_chain_with_halo(
   %sharding_annotated_2 = mesh.shard %1 to %ssharding_annotated_2  : tensor<8x16xf32>
   %ssharding_annotated_4 = mesh.sharding @mesh_1d_4 split_axes = [[0]] halo_sizes = [2, 1] : !mesh.sharding
   %sharding_annotated_4 = mesh.shard %sharding_annotated_2 to %ssharding_annotated_4  annotate_for_users : tensor<8x16xf32>
-  // CHECK-NEXT: %[[TMP3:.*]] = tosa.negate %[[TMP2]] : (tensor<5x16xf32>) -> tensor<5x16xf32>
-  %2 = tosa.negate %sharding_annotated_4 : (tensor<8x16xf32>) -> tensor<8x16xf32>
+  // CHECK-NEXT: %[[TMP3:.*]] = tosa.negate %[[TMP2]], %[[IN2]], %[[IN3]] : (tensor<5x16xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<5x16xf32>
+  %sharding_1 = mesh.sharding @mesh_1d_4 split_axes = [[]] : !mesh.sharding
+  %zero_point_1 = mesh.shard %arg1 to %sharding_1 annotate_for_users : tensor<1xf32>
+  %zero_point_2 = mesh.shard %arg2 to %sharding_1 annotate_for_users : tensor<1xf32>
+  %2 = tosa.negate %sharding_annotated_4, %zero_point_1, %zero_point_2 : (tensor<8x16xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<8x16xf32>
   %ssharding_annotated_5 = mesh.sharding @mesh_1d_4 split_axes = [[0]] halo_sizes = [2, 1] : !mesh.sharding
   %sharding_annotated_5 = mesh.shard %2 to %ssharding_annotated_5  : tensor<8x16xf32>
   %ssharding_annotated_6 = mesh.sharding @mesh_1d_4 split_axes = [[0]] halo_sizes = [2, 1] : !mesh.sharding
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 72bb1db72377b..654db471e05b6 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -761,7 +761,6 @@ func.func @omp_distribute(%chunk_size : i32, %data_var : memref<i32>, %arg0 : i3
   return
 }
 
-
 // CHECK-LABEL: omp_target
 func.func @omp_target(%if_cond : i1, %device : si32,  %num_threads : i32, %device_ptr: memref<i32>, %device_addr: memref<?xi32>, %map1: memref<?xi32>, %map2: memref<?xi32>) -> () {
 
@@ -773,17 +772,19 @@ func.func @omp_target(%if_cond : i1, %device : si32,  %num_threads : i32, %devic
     }) {nowait, operandSegmentSizes = array<i32: 0,0,0,1,0,0,1,0,0,0,0,1>} : ( si32, i1, i32 ) -> ()
 
     // Test with optional map clause.
-    // CHECK: %[[MAP_A:.*]] = omp.map.info var_ptr(%[[VAL_1:.*]] : memref<?xi32>, tensor<?xi32>)   map_clauses(tofrom) capture(ByRef) -> memref<?xi32> {name = ""}
-    // CHECK: %[[MAP_B:.*]] = omp.map.info var_ptr(%[[VAL_2:.*]] : memref<?xi32>, tensor<?xi32>)   map_clauses(exit_release_or_enter_alloc) capture(ByRef) -> memref<?xi32> {name = ""}
-    // CHECK: omp.target has_device_addr(%[[VAL_5:.*]] : memref<?xi32>) is_device_ptr(%[[VAL_4:.*]] : memref<i32>) map_entries(%[[MAP_A]] -> {{.*}}, %[[MAP_B]] -> {{.*}} : memref<?xi32>, memref<?xi32>) {
+    // CHECK: %[[MAP_A:.*]] = omp.map.info var_ptr(%[[VAL_1:.*]] : memref<?xi32>, tensor<?xi32>)   map_clauses(always, to) capture(ByRef) -> memref<?xi32> {name = ""}
+    // CHECK: %[[MAP_B:.*]] = omp.map.info var_ptr(%[[VAL_2:.*]] : memref<?xi32>, tensor<?xi32>)   map_clauses(tofrom) capture(ByRef) -> memref<?xi32> {name = ""}
+    // CHECK: %[[MAP_C:.*]] = omp.map.info var_ptr(%[[VAL_3:.*]] : memref<?xi32>, tensor<?xi32>)   map_clauses(exit_release_or_enter_alloc) capture(ByRef) -> memref<?xi32> {name = ""}
+    // CHECK: omp.target is_device_ptr(%[[VAL_4:.*]] : memref<i32>) has_device_addr(%[[MAP_A]] -> {{.*}} : memref<?xi32>) map_entries(%[[MAP_B]] -> {{.*}}, %[[MAP_C]] -> {{.*}} : memref<?xi32>, memref<?xi32>) {
+    %mapv0 = omp.map.info var_ptr(%device_addr : memref<?xi32>, tensor<?xi32>)   map_clauses(always, to) capture(ByRef) -> memref<?xi32> {name = ""}
     %mapv1 = omp.map.info var_ptr(%map1 : memref<?xi32>, tensor<?xi32>)   map_clauses(tofrom) capture(ByRef) -> memref<?xi32> {name = ""}
     %mapv2 = omp.map.info var_ptr(%map2 : memref<?xi32>, tensor<?xi32>)   map_clauses(exit_release_or_enter_alloc) capture(ByRef) -> memref<?xi32> {name = ""}
-    omp.target is_device_ptr(%device_ptr : memref<i32>) has_device_addr(%device_addr : memref<?xi32>) map_entries(%mapv1 -> %arg0, %mapv2 -> %arg1 : memref<?xi32>, memref<?xi32>) {
+    omp.target is_device_ptr(%device_ptr : memref<i32>) has_device_addr(%mapv0 -> %arg0 : memref<?xi32>) map_entries(%mapv1 -> %arg1, %mapv2 -> %arg2 : memref<?xi32>, memref<?xi32>) {
       omp.terminator
     }
-    // CHECK: %[[MAP_C:.*]] = omp.map.info var_ptr(%[[VAL_1:.*]] : memref<?xi32>, tensor<?xi32>)   map_clauses(to) capture(ByRef) -> memref<?xi32> {name = ""}
-    // CHECK: %[[MAP_D:.*]] = omp.map.info var_ptr(%[[VAL_2:.*]] : memref<?xi32>, tensor<?xi32>)   map_clauses(always, from) capture(ByRef) -> memref<?xi32> {name = ""}
-    // CHECK: omp.target map_entries(%[[MAP_C]] -> {{.*}}, %[[MAP_D]] -> {{.*}} : memref<?xi32>, memref<?xi32>) {
+    // CHECK: %[[MAP_D:.*]] = omp.map.info var_ptr(%[[VAL_1:.*]] : memref<?xi32>, tensor<?xi32>)   map_clauses(to) capture(ByRef) -> memref<?xi32> {name = ""}
+    // CHECK: %[[MAP_E:.*]] = omp.map.info var_ptr(%[[VAL_2:.*]] : memref<?xi32>, tensor<?xi32>)   map_clauses(always, from) capture(ByRef) -> memref<?xi32> {name = ""}
+    // CHECK: omp.target map_entries(%[[MAP_D]] -> {{.*}}, %[[MAP_E]] -> {{.*}} : memref<?xi32>, memref<?xi32>) {
     %mapv3 = omp.map.info var_ptr(%map1 : memref<?xi32>, tensor<?xi32>)   map_clauses(to) capture(ByRef) -> memref<?xi32> {name = ""}
     %mapv4 = omp.map.info var_ptr(%map2 : memref<?xi32>, tensor<?xi32>)   map_clauses(always, from) capture(ByRef) -> memref<?xi32> {name = ""}
     omp.target map_entries(%mapv3 -> %arg0, %mapv4 -> %arg1 : memref<?xi32>, memref<?xi32>) {
@@ -825,6 +826,12 @@ func.func @omp_target_data (%if_cond : i1, %device : si32, %device_ptr: memref<i
     %mapv6 = omp.map.info var_ptr(%map2 : memref<?xi32>, tensor<?xi32>)   map_clauses(exit_release_or_enter_alloc) capture(ByRef) -> memref<?xi32> {name = ""}
     omp.target_exit_data if(%if_cond) device(%device : si32) nowait map_entries(%mapv6 : memref<?xi32>)
 
+    // CHECK: %[[MAP_A:.*]] = omp.map.info var_ptr(%[[VAL_2:.*]] : memref<?xi32>, tensor<?xi32>)   map_clauses(ompx_hold, to) capture(ByRef) -> memref<?xi32> {name = ""}
+    // CHECK: omp.target_data map_entries(%[[MAP_A]] : memref<?xi32>)
+    %mapv7 = omp.map.info var_ptr(%map1 : memref<?xi32>, tensor<?xi32>)   map_clauses(ompx_hold, to) capture(ByRef) -> memref<?xi32> {name = ""}
+    omp.target_data map_entries(%mapv7 : memref<?xi32>) {
+      omp.terminator
+    }
     return
 }
 
diff --git a/mlir/test/Dialect/Ptr/layout.mlir b/mlir/test/Dialect/Ptr/layout.mlir
index 73189a388942a..356c57402651f 100644
--- a/mlir/test/Dialect/Ptr/layout.mlir
+++ b/mlir/test/Dialect/Ptr/layout.mlir
@@ -4,16 +4,18 @@ module attributes { dlti.dl_spec = #dlti.dl_spec<
   #dlti.dl_entry<!ptr.ptr, #ptr.spec<size = 32, abi = 32, preferred = 64>>,
   #dlti.dl_entry<!ptr.ptr<5>,#ptr.spec<size = 64, abi = 64, preferred = 64>>,
   #dlti.dl_entry<!ptr.ptr<4>, #ptr.spec<size = 32, abi = 64, preferred = 64, index = 24>>,
+  #dlti.dl_entry<"dlti.default_memory_space", 7 : ui64>,
   #dlti.dl_entry<"dlti.alloca_memory_space", 5 : ui64>,
   #dlti.dl_entry<"dlti.global_memory_space", 2 : ui64>,
   #dlti.dl_entry<"dlti.program_memory_space", 3 : ui64>,
   #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>
 >} {
-  // CHECK: @spec
+  // CHECK-LABEL: @spec
   func.func @spec() {
     // CHECK: alignment = 4
     // CHECK: alloca_memory_space = 5
     // CHECK: bitsize = 32
+    // CHECK: default_memory_space = 7
     // CHECK: global_memory_space = 2
     // CHECK: index = 32
     // CHECK: preferred = 8
@@ -21,19 +23,21 @@ module attributes { dlti.dl_spec = #dlti.dl_spec<
     // CHECK: size = 4
     // CHECK: stack_alignment = 128
     "test.data_layout_query"() : () -> !ptr.ptr
-    // CHECK: alignment = 4
+    // CHECK: alignment = 1
     // CHECK: alloca_memory_space = 5
-    // CHECK: bitsize = 32
+    // CHECK: bitsize = 64
+    // CHECK: default_memory_space = 7
     // CHECK: global_memory_space = 2
-    // CHECK: index = 32
-    // CHECK: preferred = 8
+    // CHECK: index = 64
+    // CHECK: preferred = 1
     // CHECK: program_memory_space = 3
-    // CHECK: size = 4
+    // CHECK: size = 8
     // CHECK: stack_alignment = 128
     "test.data_layout_query"() : () -> !ptr.ptr<3>
     // CHECK: alignment = 8
     // CHECK: alloca_memory_space = 5
     // CHECK: bitsize = 64
+    // CHECK: default_memory_space = 7
     // CHECK: global_memory_space = 2
     // CHECK: index = 64
     // CHECK: preferred = 8
@@ -44,6 +48,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec<
     // CHECK: alignment = 8
     // CHECK: alloca_memory_space = 5
     // CHECK: bitsize = 32
+    // CHECK: default_memory_space = 7
     // CHECK: global_memory_space = 2
     // CHECK: index = 24
     // CHECK: preferred = 8
@@ -57,6 +62,36 @@ module attributes { dlti.dl_spec = #dlti.dl_spec<
 
 // -----
 
+module attributes { dlti.dl_spec = #dlti.dl_spec<
+  #dlti.dl_entry<!ptr.ptr<1 : ui64>, #ptr.spec<size = 32, abi = 32, preferred = 32>>,
+  #dlti.dl_entry<"dlti.default_memory_space", 1 : ui64>
+>} {
+  // CHECK-LABEL: @default_memory_space
+  func.func @default_memory_space() {
+    // CHECK: alignment = 4
+    // CHECK: bitsize = 32
+    // CHECK: index = 32
+    // CHECK: preferred = 4
+    // CHECK: size = 4
+    "test.data_layout_query"() : () -> !ptr.ptr
+    // CHECK: alignment = 4
+    // CHECK: bitsize = 32
+    // CHECK: index = 32
+    // CHECK: preferred = 4
+    // CHECK: size = 4
+    "test.data_layout_query"() : () -> !ptr.ptr<1>
+    // CHECK: alignment = 4
+    // CHECK: bitsize = 32
+    // CHECK: index = 32
+    // CHECK: preferred = 4
+    // CHECK: size = 4
+    "test.data_layout_query"() : () -> !ptr.ptr<2>
+    return
+  }
+}
+
+// -----
+
 // expected-error@+2 {{preferred alignment is expected to be at least as large as ABI alignment}}
 module attributes { dlti.dl_spec = #dlti.dl_spec<
   #dlti.dl_entry<!ptr.ptr, #ptr.spec<size = 64, abi = 64, preferred = 32>>
diff --git a/mlir/test/Dialect/Tensor/bubble-reshapes.mlir b/mlir/test/Dialect/Tensor/bubble-reshapes.mlir
index cf6b12852bcd3..eeed794884942 100644
--- a/mlir/test/Dialect/Tensor/bubble-reshapes.mlir
+++ b/mlir/test/Dialect/Tensor/bubble-reshapes.mlir
@@ -45,3 +45,17 @@ func.func @no_bubble_partial_intersecting_reshapes(%arg0: tensor<?x?x?x?xf32>, %
 //      CHECK:   %[[COLLAPSE:.+]] = tensor.collapse_shape %[[ARG0]] {{\[}}[0, 1, 2], [3]]
 //      CHECK:   %[[EXPAND:.+]] = tensor.expand_shape %[[COLLAPSE]] {{\[}}[0, 1], [2, 3]]
 //      CHECK:   return %[[EXPAND]]
+
+// -----
+
+func.func @no_bubble_0d_tensor_reshapes(%arg0: tensor<?xf32>, %s0: index, %s1: index, %s2: index, %s3: index) -> tensor<?x?x?x?xf32> {
+  %collapse = tensor.collapse_shape %arg0 [] : tensor<?xf32> into tensor<f32>
+  %expand = tensor.expand_shape %collapse []
+              output_shape [%s0, %s1, %s2, %s3] : tensor<f32> into tensor<?x?x?x?xf32>
+  return %expand : tensor<?x?x?x?xf32>
+}
+//      CHECK: func @no_bubble_0d_tensor_reshapes
+// CHECK-SAME:   %[[ARG0:.+]]: tensor<?xf32>
+//      CHECK:   %[[COLLAPSE:.+]] = tensor.collapse_shape %[[ARG0]] {{\[}}]
+//      CHECK:   %[[EXPAND:.+]] = tensor.expand_shape %[[COLLAPSE]] {{\[}}]
+//      CHECK:   return %[[EXPAND]]
diff --git a/mlir/test/Dialect/Tensor/bubble-up-extract-slice-op.mlir b/mlir/test/Dialect/Tensor/bubble-up-extract-slice-op.mlir
index 252e7494bff79..3900bc56f433d 100644
--- a/mlir/test/Dialect/Tensor/bubble-up-extract-slice-op.mlir
+++ b/mlir/test/Dialect/Tensor/bubble-up-extract-slice-op.mlir
@@ -31,7 +31,7 @@ func.func @no_bubble_up_extract_slice_on_non_contiguous(%src: tensor<60xf32>) ->
 
 func.func @no_bubble_up_extract_slice_on_stride(%src: tensor<60xf32>) -> tensor<1x1x5xf32> {
   %expand = tensor.expand_shape %src [[0, 1, 2]] output_shape [2, 3, 10] : tensor<60xf32> into tensor<2x3x10xf32>
-  %extract = tensor.extract_slice %expand[0, 0, 5][1, 1, 5][1, 1, 2] : tensor<2x3x10xf32> to tensor<1x1x5xf32>
+  %extract = tensor.extract_slice %expand[0, 0, 0][1, 1, 5][1, 1, 2] : tensor<2x3x10xf32> to tensor<1x1x5xf32>
   return %extract : tensor<1x1x5xf32>
 }
 
diff --git a/mlir/test/Dialect/Tensor/drop-redundant-insert-slice-rank-expansion.mlir b/mlir/test/Dialect/Tensor/drop-redundant-insert-slice-rank-expansion.mlir
index 88e55062f4770..0496b93257a9e 100644
--- a/mlir/test/Dialect/Tensor/drop-redundant-insert-slice-rank-expansion.mlir
+++ b/mlir/test/Dialect/Tensor/drop-redundant-insert-slice-rank-expansion.mlir
@@ -51,16 +51,16 @@ func.func @no_fold_more_unit_dims_insert_slice_of_extract_slice(%in : tensor<?x8
 
 // -----
 
-func.func @no_fold_strided_insert_slice_of_extract_slice(%in : tensor<?x8x2x8xf32>, %dest : tensor<1x4x4xf32>) -> tensor<1x4x4xf32> {
+func.func @no_fold_strided_insert_slice_of_extract_slice(%in : tensor<?x8x2x8xf32>, %dest : tensor<1x15x15xf32>) -> tensor<1x15x15xf32> {
   %extracted_slice = tensor.extract_slice %in[0, 0, 0, 0] [1, 8, 1, 8] [1, 1, 1, 1] : tensor<?x8x2x8xf32> to tensor<8x8xf32>
-  %inserted_slice = tensor.insert_slice %extracted_slice into %dest[0, 0, 0] [1, 8, 8] [1, 2, 2] : tensor<8x8xf32> into tensor<1x4x4xf32>
-  return %inserted_slice : tensor<1x4x4xf32>
+  %inserted_slice = tensor.insert_slice %extracted_slice into %dest[0, 0, 0] [1, 8, 8] [1, 2, 2] : tensor<8x8xf32> into tensor<1x15x15xf32>
+  return %inserted_slice : tensor<1x15x15xf32>
 }
 // CHECK-LABEL: func.func @no_fold_strided_insert_slice_of_extract_slice(
 // CHECK-SAME:      %[[ARG0:.*]]: tensor<?x8x2x8xf32>
 // CHECK:         %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]]
 // CHECK:         %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[EXTRACTED_SLICE]]
-// CHECK:         return %[[INSERTED_SLICE]] : tensor<1x4x4xf32>
+// CHECK:         return %[[INSERTED_SLICE]] : tensor<1x15x15xf32>
 
 // -----
 
diff --git a/mlir/test/Dialect/Tensor/fold-tensor-subset-ops.mlir b/mlir/test/Dialect/Tensor/fold-tensor-subset-ops.mlir
index 988b5d835c16e..cf8711eb64ab9 100644
--- a/mlir/test/Dialect/Tensor/fold-tensor-subset-ops.mlir
+++ b/mlir/test/Dialect/Tensor/fold-tensor-subset-ops.mlir
@@ -271,38 +271,34 @@ func.func @insert_slice_of_transfer_write_rank_extending(%t1 : tensor<?x?x12xf32
 
 // -----
 
-//       CHECK: #[[$map:.*]] = affine_map<()[s0] -> (s0 + 2)>
 // CHECK-LABEL: func @insert_slice_of_insert_slice(
 //  CHECK-SAME:     %[[t:[0-9a-z]*]]: tensor<f32>
 //  CHECK-SAME:     %[[r1:[0-9a-z]*]]: tensor<1x14xf32>
 //  CHECK-SAME:     %[[pos:[0-9a-z]*]]: index
-//       CHECK:   %[[add:.*]] = affine.apply #[[$map]]()[%[[pos]]]
-//       CHECK:   tensor.insert_slice %[[t]] into %[[r1]][4, %[[add]]] [1, 1] [1, 1] : tensor<f32> into tensor<1x14xf32>
+//       CHECK:   tensor.insert_slice %[[t]] into %[[r1]][0, %[[pos]]] [1, 1] [1, 1] : tensor<f32> into tensor<1x14xf32>
 func.func @insert_slice_of_insert_slice(%t: tensor<f32>, %r0: tensor<1x1xf32>, %r1: tensor<1x14xf32>, %pos: index)
     -> tensor<1x14xf32> 
 {
-  %0 = tensor.insert_slice %t into %r0[1, 2] [1, 1] [1, 1] 
+  %0 = tensor.insert_slice %t into %r0[0, 0] [1, 1] [1, 1] 
     : tensor<f32> into tensor<1x1xf32>
-  %1 = tensor.insert_slice %0 into %r1[3, %pos] [1, 1] [1, 1] 
+  %1 = tensor.insert_slice %0 into %r1[0, %pos] [1, 1] [1, 1] 
     : tensor<1x1xf32> into tensor<1x14xf32>
   return %1 : tensor<1x14xf32>
 }
 
 // -----
 
-//   CHECK-DAG: #[[$map:.*]] = affine_map<()[s0] -> (s0 + 2)>
 // CHECK-LABEL: func @insert_slice_of_insert_slice(
 //  CHECK-SAME:     %[[t:[0-9a-z]*]]: tensor<f32>
 //  CHECK-SAME:     %[[r1:[0-9a-z]*]]: tensor<1x14xf32>
 //  CHECK-SAME:     %[[pos:[0-9a-z]*]]: index
-//       CHECK:   %[[composed_pos:.+]] = affine.apply #[[$map]]()[%[[pos]]]
-//       CHECK:   tensor.insert_slice %[[t]] into %[[r1]][3, %[[composed_pos]]] [1, 1] [1, 1] : tensor<f32> into tensor<1x14xf32>
+//       CHECK:   tensor.insert_slice %[[t]] into %[[r1]][0, %[[pos]]] [1, 1] [1, 1] : tensor<f32> into tensor<1x14xf32>
 func.func @insert_slice_of_insert_slice(%t: tensor<f32>, %r0: tensor<1xf32>, %r1: tensor<1x14xf32>, %pos: index)
     -> tensor<1x14xf32> 
 {
-  %0 = tensor.insert_slice %t into %r0[2] [1] [1] 
+  %0 = tensor.insert_slice %t into %r0[0] [1] [1] 
     : tensor<f32> into tensor<1xf32>
-  %1 = tensor.insert_slice %0 into %r1[3, %pos] [1, 1] [1, 1] 
+  %1 = tensor.insert_slice %0 into %r1[0, %pos] [1, 1] [1, 1] 
     : tensor<1xf32> into tensor<1x14xf32>
   return %1 : tensor<1x14xf32>
 }
diff --git a/mlir/test/Dialect/Tensor/invalid.mlir b/mlir/test/Dialect/Tensor/invalid.mlir
index 654169841c1c1..eeb3967337806 100644
--- a/mlir/test/Dialect/Tensor/invalid.mlir
+++ b/mlir/test/Dialect/Tensor/invalid.mlir
@@ -258,6 +258,22 @@ func.func @illegal_num_offsets(%arg0 : tensor<?x?x?xf32>, %arg1 : index, %arg2 :
 
 // -----
 
+func.func @extract_slice_offset_out_of_bounds(%arg0: tensor<10xf32>) {
+  // expected-error@+1 {{offset 0 is out-of-bounds: 10 >= 10}}
+  %0 = tensor.extract_slice %arg0 [10][1][1] : tensor<10xf32> to tensor<1xf32>
+  return
+}
+
+// -----
+
+func.func @extract_slice_runs_out_of_bounds(%arg0: tensor<9xf32>) {
+  // expected-error@+1 {{slice along dimension 0 runs out-of-bounds: 9 >= 9}}
+  %0 = tensor.extract_slice %arg0 [3][4][2] : tensor<9xf32> to tensor<4xf32>
+  return
+}
+
+// -----
+
 func.func @insert_slice_wrong_result_rank(%t1: tensor<?xf32>, %t2: tensor<?x?xf32>, %idx : index) {
   // expected-error @+1 {{expected rank to be smaller or equal to the other rank.}}
   %0 = tensor.insert_slice %t2 into %t1[0][4][1] : tensor<?x?xf32> into tensor<?xf32>
@@ -296,6 +312,22 @@ func.func @insert_slice_wrong_dynamic_type(%t1: tensor<?x4x4xf32>, %t2: tensor<8
 
 // -----
 
+func.func @insert_slice_offset_out_of_bounds(%arg0: tensor<1xf32>, %arg1: tensor<10xf32>) {
+  // expected-error@+1 {{offset 0 is out-of-bounds: 10 >= 10}}
+  %0 = tensor.insert_slice %arg0 into %arg1 [10][1][1] : tensor<1xf32> into tensor<10xf32>
+  return
+}
+
+// -----
+
+func.func @insert_slice_runs_out_of_bounds(%arg0: tensor<4xf32>, %arg1: tensor<9xf32>) {
+  // expected-error@+1 {{slice along dimension 0 runs out-of-bounds: 9 >= 9}}
+  %0 = tensor.insert_slice %arg0 into %arg1 [3][4][2] : tensor<4xf32> into tensor<9xf32>
+  return
+}
+
+// -----
+
 func.func @illegal_expanding_reshape_static_tensor
     (%arg0: tensor<2x3x20xf32>) -> tensor<2x3x2x4x5xf32> {
   // expected-error @+1 {{expected dimension 2 of collapsed type to be static value of 40}}
diff --git a/mlir/test/Dialect/Tosa/availability.mlir b/mlir/test/Dialect/Tosa/availability.mlir
index e90f0af1f9c7d..7867df2e3917f 100644
--- a/mlir/test/Dialect/Tosa/availability.mlir
+++ b/mlir/test/Dialect/Tosa/availability.mlir
@@ -69,10 +69,10 @@ func.func @test_fft2d(%arg0: tensor<1x4x8xf32>, %arg1: tensor<1x4x8xf32>) -> (te
 
 // -----
 // CHECK-LABEL: matmul
-func.func @test_matmul(%arg0: tensor<1x14x19xf32>, %arg1: tensor<1x19x28xf32>) -> tensor<1x14x28xf32> {
+func.func @test_matmul(%arg0: tensor<1x14x19xf32>, %arg1: tensor<1x19x28xf32>, %a_zp: tensor<1xf32>, %b_zp: tensor<1xf32>) -> tensor<1x14x28xf32> {
   // CHECK: profiles: [ [pro_int, pro_fp] ]
   // CHECK: extensions: [ [int16, fp8e4m3, fp8e5m2, bf16] ]
-  %0 = tosa.matmul %arg0, %arg1 : (tensor<1x14x19xf32>, tensor<1x19x28xf32>) -> tensor<1x14x28xf32>
+  %0 = tosa.matmul %arg0, %arg1, %a_zp, %b_zp : (tensor<1x14x19xf32>, tensor<1x19x28xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<1x14x28xf32>
   return %0 : tensor<1x14x28xf32>
 }
 
@@ -380,7 +380,9 @@ func.func @test_logical_not(%arg0: tensor<1x21x3xi1>) -> tensor<1x21x3xi1> {
 func.func @test_negate(%arg0: tensor<13x21x3xf32>) -> tensor<13x21x3xf32> {
   // CHECK: profiles: [ [pro_int, pro_fp] ]
   // CHECK: extensions: [ [bf16] ]
-  %0 = tosa.negate %arg0 : (tensor<13x21x3xf32>) -> tensor<13x21x3xf32>
+  %input_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %output_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %0 = tosa.negate %arg0, %input_zp, %output_zp : (tensor<13x21x3xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<13x21x3xf32>
   return %0 : tensor<13x21x3xf32>
 }
 
@@ -610,16 +612,18 @@ func.func @test_cast1(%arg0: tensor<13x21x3xi32>) -> tensor<13x21x3xf32> {
 }
 
 // -----
-// CHECK-LABEL: rescale
+// CHECK-LABEL: test_rescale
 func.func @test_rescale(%arg0: tensor<13x21x3x!quant.uniform<u8:f32, 0.015655439347028732:127>>, %multiplier : tensor<1xi32>, %shift : tensor<1xi8>) -> tensor<13x21x3x!quant.uniform<i8:f32, 0.015655439347028732:-1>> {
-  // CHECK: profiles: [ [pro_int] ]
-  // CHECK: extensions: [ [int16] ]
-  %0 = tosa.rescale %arg0 {double_round = false, input_zp = 127 : i32, multiplier = array<i32: 1073741824>, output_zp = -1 : i32, per_channel = false, scale32 = true, shift = array<i8: 30>, input_unsigned = true, output_unsigned = false} : (tensor<13x21x3x!quant.uniform<u8:f32, 0.015655439347028732:127>>) -> tensor<13x21x3x!quant.uniform<i8:f32, 0.015655439347028732:-1>>
+  %input_zp = "tosa.const"() {values = dense<127> : tensor<1xi8>} : () -> tensor<1xi8>
+  %output_zp = "tosa.const"() {values = dense<-1> : tensor<1xi8>} : () -> tensor<1xi8>
+  // CHECK: tosa.rescale profiles: [ [pro_int] ]
+  // CHECK: tosa.rescale extensions: [ [int16] ]
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", scale32 = true, per_channel = false, input_unsigned = false, output_unsigned = false} : (tensor<13x21x3x!quant.uniform<u8:f32, 0.015655439347028732:127>>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<13x21x3x!quant.uniform<i8:f32, 0.015655439347028732:-1>>
   return %0 : tensor<13x21x3x!quant.uniform<i8:f32, 0.015655439347028732:-1>>
 }
 
 // -----
-// CHECK-LABEL: const
+// CHECK-LABEL: test_const
 func.func @test_const(%arg0 : index) -> tensor<4xi32> {
   // CHECK: profiles: [ [pro_int, pro_fp] ]
   // CHECK: extensions: [ [int4, int16, fp8e4m3, fp8e5m2, bf16] ]
diff --git a/mlir/test/Dialect/Tosa/canonicalize.mlir b/mlir/test/Dialect/Tosa/canonicalize.mlir
index a78479cc0bead..3bc438e465e1d 100644
--- a/mlir/test/Dialect/Tosa/canonicalize.mlir
+++ b/mlir/test/Dialect/Tosa/canonicalize.mlir
@@ -856,13 +856,54 @@ func.func @fold_exp_log(%arg0: tensor<?x1xf32>) -> tensor<?x1xf32> {
 // CHECK-LABEL: @fold_negate_negate
 func.func @fold_negate_negate(%arg0: tensor<?x1xf32>) -> tensor<?x1xf32> {
   // CHECK: return %arg{{.*}} : tensor<?x1xf32>
-  %0 = tosa.negate %arg0 : (tensor<?x1xf32>) -> tensor<?x1xf32>
-  %1 = tosa.negate %0 : (tensor<?x1xf32>) -> tensor<?x1xf32>
+  %in_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %out_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %0 = tosa.negate %arg0, %in_zp, %out_zp : (tensor<?x1xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<?x1xf32>
+  %1 = tosa.negate %0, %in_zp, %out_zp : (tensor<?x1xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<?x1xf32>
   return %1 : tensor<?x1xf32>
 }
 
 // -----
 
+// CHECK-LABEL: @no_fold_negate_negate_non_const_zp
+func.func @no_fold_negate_negate_non_const_zp(%arg0: tensor<?x1xf32>, %in_zp: tensor<1xf32>) -> tensor<?x1xf32> {
+  // cannot fold if any zp is not constant
+  // CHECK: tosa.negate
+  // CHECK: tosa.negate
+  // CHECK: tosa.negate
+  // CHECK: tosa.negate
+  // CHECK: tosa.negate
+  %zero = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %0 = tosa.negate %arg0, %in_zp, %zero : (tensor<?x1xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<?x1xf32>
+  %1 = tosa.negate %0, %zero, %zero : (tensor<?x1xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<?x1xf32>
+  %2 = tosa.negate %1, %zero, %in_zp : (tensor<?x1xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<?x1xf32>
+  %3 = tosa.negate %2, %zero, %zero : (tensor<?x1xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<?x1xf32>
+  %4 = tosa.negate %3, %in_zp, %zero : (tensor<?x1xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<?x1xf32>
+  return %4 : tensor<?x1xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @no_fold_negate_negate_non_zero_zp
+func.func @no_fold_negate_negate_non_zero_zp(%arg0: tensor<?x1xi8>) -> tensor<?x1xi8> {
+  // cannot fold if any zp is not constant 0
+  // CHECK: tosa.negate
+  // CHECK: tosa.negate
+  // CHECK: tosa.negate
+  // CHECK: tosa.negate
+  // CHECK: tosa.negate
+  %zero = "tosa.const"() <{values = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %one = "tosa.const"() <{values = dense<1> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %0 = tosa.negate %arg0, %zero, %one : (tensor<?x1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<?x1xi8>
+  %1 = tosa.negate %0, %zero, %zero : (tensor<?x1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<?x1xi8>
+  %2 = tosa.negate %1, %one, %zero : (tensor<?x1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<?x1xi8>
+  %3 = tosa.negate %2, %zero, %zero : (tensor<?x1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<?x1xi8>
+  %4 = tosa.negate %3, %zero, %one : (tensor<?x1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<?x1xi8>
+  return %4 : tensor<?x1xi8>
+}
+
+// -----
+
 // CHECK-LABEL: @fold_abs_abs
 func.func @fold_abs_abs(%arg0: tensor<?x1xf32>) -> tensor<?x1xf32> {
   // CHECK: %[[ABS:.*]] = tosa.abs %arg{{.*}} : (tensor<?x1xf32>) -> tensor<?x1xf32>
@@ -903,7 +944,11 @@ func.func @reshape_quant_nofold() -> tensor<1x1x1x1xi32> {
    %0 = "tosa.const"() {values = dense<127> : tensor<i8>} : () -> tensor<!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>
    %cst0 = "tosa.const_shape"() {values = dense<[1, 1, 1, 1]> : tensor<4xindex>} : () -> !tosa.shape<4>
    %1 = tosa.reshape %0, %cst0 : (tensor<!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>, !tosa.shape<4>) -> tensor<1x1x1x1x!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>
-   %2 = tosa.rescale %1 {double_round = true, input_zp = -128 : i32, multiplier = array<i32: 1073741824>, output_zp = 0 : i32, per_channel = false, scale32 = true, shift = array<i8: 30>, input_unsigned = false, output_unsigned = false} : (tensor<1x1x1x1x!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>) -> tensor<1x1x1x1xi32>
+   %multiplier = "tosa.const"() {values = dense<1073741824> : tensor<1xi32> } : () -> tensor<1xi32>
+   %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+   %input_zp = "tosa.const"() {values = dense<-128> : tensor<1xi8>} : () -> tensor<1xi8>
+   %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi32>} : () -> tensor<1xi32>
+   %2 = tosa.rescale %1, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "DOUBLE_ROUND", scale32 = true, per_channel = false, input_unsigned = false, output_unsigned = false} : (tensor<1x1x1x1x!quant.uniform<i8:f32, 3.0757404601899907E-5:-128>>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi32>) -> tensor<1x1x1x1xi32>
    return %2 : tensor<1x1x1x1xi32>
 }
 
diff --git a/mlir/test/Dialect/Tosa/invalid.mlir b/mlir/test/Dialect/Tosa/invalid.mlir
index f2956d84c9650..3ea0b61cac064 100644
--- a/mlir/test/Dialect/Tosa/invalid.mlir
+++ b/mlir/test/Dialect/Tosa/invalid.mlir
@@ -4,7 +4,7 @@
 // validation flow.
 //--------------------------------------------------------------------------------------------------
 
-// RUN: mlir-opt %s -split-input-file -verify-diagnostics --tosa-validate="profile=pro_int,pro_fp extension=int16,int4,bf16,fp8e4m3,fp8e5m2,fft,variable,controlflow strict-op-spec-alignment"
+// RUN: mlir-opt %s -split-input-file -verify-diagnostics --tosa-validate="profile=pro_int,pro_fp extension=int16,int4,bf16,fp8e4m3,fp8e5m2,fft,variable,controlflow,doubleround,inexactround strict-op-spec-alignment"
 
 func.func @test_const() -> tensor<1xf32> {
   // expected-error@+1{{'tosa.const' op expected same attr/result element types}}
@@ -287,7 +287,7 @@ func.func @test_pad_invalid_padConst_rank(%arg0: tensor<13x21xf32>, %arg1: tenso
 // -----
 
 func.func @test_pad_padding_shape_mismatch(%arg0: tensor<13x21x3xf32>) -> tensor<13x21x3xf32> {
-  %0 = tosa.const_shape {values = dense<1> : tensor<4xindex>} : () -> !tosa.shape<4>  
+  %0 = tosa.const_shape {values = dense<1> : tensor<4xindex>} : () -> !tosa.shape<4>
   %pad_const = "tosa.const"() {values = dense<3.14> : tensor<1xf32>} : () -> tensor<1xf32>
   // expected-error@+1 {{'tosa.pad' op expected padding tensor dim 0 to have size 6 (2*rank(shape1)) but got size 4}}
   %1 = tosa.pad %arg0, %0, %pad_const : (tensor<13x21x3xf32>, !tosa.shape<4>, tensor<1xf32>) -> tensor<13x21x3xf32>
@@ -989,6 +989,7 @@ func.func @test_mismatch_in_out_shape_clz(%arg0: tensor<13x21x3xi32>) -> tensor<
 }
 
 // -----
+
 // CHECK-LABEL: test_mismatch_in_out_data_type_cos
 func.func @test_mismatch_in_out_data_type_cos(%arg0: tensor<13x21x3xf32>) -> tensor<13x21x3xf16> {
   // expected-error@+1 {{'tosa.cos' op requires the same element type for all operands and results}}
@@ -1431,3 +1432,324 @@ func.func @test_argmax_invalid_output_shape(%arg0: tensor<1x2x3xf32>) -> tensor<
   %0 = tosa.argmax %arg0 {axis = 0 : i32}: (tensor<1x2x3xf32>) -> tensor<1x2x3xi32>
   return %0 : tensor<1x2x3xi32>
 }
+
+// -----
+
+func.func @test_rescale_invalid_input_type(%arg0: tensor<13x21x3xf32>) -> tensor<13x21x3xi32> {
+  %multiplier = "tosa.const"() {values = dense<1073741824> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<0.0> : tensor<1xf32>} : () -> tensor<1xf32>
+  %output_zp = "tosa.const"() {values = dense<0.0> : tensor<1xf32>} : () -> tensor<1xf32>
+  // expected-error@+1 {{'tosa.rescale' op expect input to have integer element type, got 'f32'}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = false} : (tensor<13x21x3xf32>, tensor<1xi32>, tensor<1xi8>, tensor<1xf32>, tensor<1xf32>) -> tensor<13x21x3xi32>
+  return %0 : tensor<13x21x3xi32>
+}
+
+// -----
+
+func.func @test_rescale_invalid_output_type(%arg0: tensor<13x21x3xi32>) -> tensor<13x21x3xf32> {
+  %multiplier = "tosa.const"() {values = dense<1073741824> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<0> : tensor<1xi32>} : () -> tensor<1xi32>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi32>} : () -> tensor<1xi32>
+  // expected-error@+1 {{'tosa.rescale' op expect output to have integer element type, got 'f32'}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = false} : (tensor<13x21x3xi32>, tensor<1xi32>, tensor<1xi8>, tensor<1xi32>, tensor<1xi32>) -> tensor<13x21x3xf32>
+  return %0 : tensor<13x21x3xf32>
+}
+
+// -----
+
+func.func @test_rescale_invalid_multiplier_type(%arg0: tensor<13x21x3xi32>) -> tensor<13x21x3xf32> {
+  %multiplier = "tosa.const"() {values = dense<1073741824> : tensor<1xi48> } : () -> tensor<1xi48>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi16> } : () -> tensor<1xi16>
+  %input_zp = "tosa.const"() {values = dense<0> : tensor<1xi32>} : () -> tensor<1xi32>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi32>} : () -> tensor<1xi32>
+  // expected-error@+1 {{'tosa.rescale' op operand #1 must be 1D tensor of 16-bit signless integer or 32-bit signless integer values, but got 'tensor<1xi48>'}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = false} : (tensor<13x21x3xi32>, tensor<1xi48>, tensor<1xi16>, tensor<1xi32>, tensor<1xi32>) -> tensor<13x21x3xf32>
+  return %0 : tensor<13x21x3xf32>
+}
+
+// -----
+
+func.func @test_rescale_invalid_shift_type(%arg0: tensor<13x21x3xi32>) -> tensor<13x21x3xf32> {
+  %multiplier = "tosa.const"() {values = dense<1073741824> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi16> } : () -> tensor<1xi16>
+  %input_zp = "tosa.const"() {values = dense<1> : tensor<1xi32>} : () -> tensor<1xi32>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi32>} : () -> tensor<1xi32>
+  // expected-error@+1 {{'tosa.rescale' op operand #2 must be 1D tensor of 8-bit signless integer values, but got 'tensor<1xi16>'}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = false} : (tensor<13x21x3xi32>, tensor<1xi32>, tensor<1xi16>, tensor<1xi32>, tensor<1xi32>) -> tensor<13x21x3xf32>
+  return %0 : tensor<13x21x3xf32>
+}
+
+// -----
+
+func.func @test_rescale_invalid_input_zp_i32(%arg0: tensor<13x21x3xi32>) -> tensor<13x21x3xi32> {
+  %multiplier = "tosa.const"() {values = dense<1073741824> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<1> : tensor<1xi32>} : () -> tensor<1xi32>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi32>} : () -> tensor<1xi32>
+  // expected-error@+1 {{'tosa.rescale' op expect input_zp of 0, got 1}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = false} : (tensor<13x21x3xi32>, tensor<1xi32>, tensor<1xi8>, tensor<1xi32>, tensor<1xi32>) -> tensor<13x21x3xi32>
+  return %0 : tensor<13x21x3xi32>
+}
+
+// -----
+
+func.func @test_rescale_invalid_input_zp_s16(%arg0: tensor<13x21x3xi16>) -> tensor<13x21x3xi16> {
+  %multiplier = "tosa.const"() {values = dense<1073741824> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<1> : tensor<1xi16>} : () -> tensor<1xi16>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  // expected-error@+1 {{'tosa.rescale' op expect input_zp of 0, got 1}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = false} : (tensor<13x21x3xi16>, tensor<1xi32>, tensor<1xi8>, tensor<1xi16>, tensor<1xi16>) -> tensor<13x21x3xi16>
+  return %0 : tensor<13x21x3xi16>
+}
+
+// -----
+
+func.func @test_rescale_invalid_input_zp_u16(%arg0: tensor<13x21x3xi16>) -> tensor<13x21x3xi16> {
+  %multiplier = "tosa.const"() {values = dense<1073741824> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<1> : tensor<1xi16>} : () -> tensor<1xi16>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  // expected-error@+1 {{'tosa.rescale' op expect input_zp of 0 or 32768 for unsigned int16 input, got 1}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", per_channel = false, scale32 = true, input_unsigned = true, output_unsigned = false} : (tensor<13x21x3xi16>, tensor<1xi32>, tensor<1xi8>, tensor<1xi16>, tensor<1xi16>) -> tensor<13x21x3xi16>
+  return %0 : tensor<13x21x3xi16>
+}
+
+
+// -----
+
+func.func @test_rescale_invalid_output_zp_i32(%arg0: tensor<13x21x3xi32>) -> tensor<13x21x3xi32> {
+  %multiplier = "tosa.const"() {values = dense<1073741824> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<0> : tensor<1xi32>} : () -> tensor<1xi32>
+  %output_zp = "tosa.const"() {values = dense<-1> : tensor<1xi32>} : () -> tensor<1xi32>
+  // expected-error@+1 {{'tosa.rescale' op expect output_zp of 0, got -1}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = false} : (tensor<13x21x3xi32>, tensor<1xi32>, tensor<1xi8>, tensor<1xi32>, tensor<1xi32>) -> tensor<13x21x3xi32>
+  return %0 : tensor<13x21x3xi32>
+}
+
+// -----
+
+func.func @test_rescale_invalid_output_zp_s16(%arg0: tensor<13x21x3xi16>) -> tensor<13x21x3xi16> {
+  %multiplier = "tosa.const"() {values = dense<1073741824> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  %output_zp = "tosa.const"() {values = dense<-1> : tensor<1xi16>} : () -> tensor<1xi16>
+  // expected-error@+1 {{'tosa.rescale' op expect output_zp of 0, got -1}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", per_channel = false, scale32 = true, input_unsigned = true, output_unsigned = false} : (tensor<13x21x3xi16>, tensor<1xi32>, tensor<1xi8>, tensor<1xi16>, tensor<1xi16>) -> tensor<13x21x3xi16>
+  return %0 : tensor<13x21x3xi16>
+}
+
+// -----
+
+func.func @test_rescale_invalid_output_zp_u16(%arg0: tensor<13x21x3xi16>) -> tensor<13x21x3xi16> {
+  %multiplier = "tosa.const"() {values = dense<1073741824> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  %output_zp = "tosa.const"() {values = dense<-1> : tensor<1xi16>} : () -> tensor<1xi16>
+  // expected-error@+1 {{'tosa.rescale' op expect output_zp of 0 or 32768 for unsigned int16 output, got -1}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = true} : (tensor<13x21x3xi16>, tensor<1xi32>, tensor<1xi8>, tensor<1xi16>, tensor<1xi16>) -> tensor<13x21x3xi16>
+  return %0 : tensor<13x21x3xi16>
+}
+
+// -----
+
+func.func @test_rescale_invalid_multiplier_i16(%arg0: tensor<13x21x3xi16>) -> tensor<13x21x3xi16> {
+  %multiplier = "tosa.const"() {values = dense<19689> : tensor<1xi16> } : () -> tensor<1xi16>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  // expected-error@+1 {{'tosa.rescale' op expect i32 element type for multiplier for scale32=true, got 'i16'}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = true} : (tensor<13x21x3xi16>, tensor<1xi16>, tensor<1xi8>, tensor<1xi16>, tensor<1xi16>) -> tensor<13x21x3xi16>
+  return %0 : tensor<13x21x3xi16>
+}
+
+// -----
+
+func.func @test_rescale_invalid_multiplier_i32(%arg0: tensor<13x21x3xi16>) -> tensor<13x21x3xi16> {
+  %multiplier = "tosa.const"() {values = dense<19689> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  // expected-error@+1 {{'tosa.rescale' op expect i16 element type for multiplier for scale32=false, got 'i32'}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", per_channel = false, scale32 = false, input_unsigned = false, output_unsigned = true} : (tensor<13x21x3xi16>, tensor<1xi32>, tensor<1xi8>, tensor<1xi16>, tensor<1xi16>) -> tensor<13x21x3xi16>
+  return %0 : tensor<13x21x3xi16>
+}
+
+// -----
+
+func.func @test_rescale_invalid_multiplier_rank(%arg0: tensor<13x21x3xi16>) -> tensor<13x21x3xi16> {
+  %multiplier = "tosa.const"() {values = dense<19689> : tensor<1x1xi32> } : () -> tensor<1x1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  // expected-error@+1 {{'tosa.rescale' op operand #1 must be 1D tensor of 16-bit signless integer or 32-bit signless integer values, but got 'tensor<1x1xi32>'}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = true} : (tensor<13x21x3xi16>, tensor<1x1xi32>, tensor<1xi8>, tensor<1xi16>, tensor<1xi16>) -> tensor<13x21x3xi16>
+  return %0 : tensor<13x21x3xi16>
+}
+
+// -----
+
+func.func @test_rescale_invalid_shift_rank(%arg0: tensor<13x21x3xi16>) -> tensor<13x21x3xi16> {
+  %multiplier = "tosa.const"() {values = dense<19689> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1x1xi8> } : () -> tensor<1x1xi8>
+  %input_zp = "tosa.const"() {values = dense<1> : tensor<1xi16>} : () -> tensor<1xi16>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  // expected-error@+1 {{'tosa.rescale' op operand #2 must be 1D tensor of 8-bit signless integer values, but got 'tensor<1x1xi8>'}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = true} : (tensor<13x21x3xi16>, tensor<1xi32>, tensor<1x1xi8>, tensor<1xi16>, tensor<1xi16>) -> tensor<13x21x3xi16>
+  return %0 : tensor<13x21x3xi16>
+}
+
+// -----
+
+func.func @test_rescale_invalid_perchannel_multiplier_shape(%arg0: tensor<13x21x3xi16>) -> tensor<13x21x3xi16> {
+  %multiplier = "tosa.const"() {values = dense<19689> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<3xi8> } : () -> tensor<3xi8>
+  %input_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  // expected-error@+1 {{'tosa.rescale' op expect shape of { 3 } for multiplier input, got { 1 }}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", per_channel = true, scale32 = true, input_unsigned = false, output_unsigned = true} : (tensor<13x21x3xi16>, tensor<1xi32>, tensor<3xi8>, tensor<1xi16>, tensor<1xi16>) -> tensor<13x21x3xi16>
+  return %0 : tensor<13x21x3xi16>
+}
+
+// -----
+
+func.func @test_rescale_invalid_non_perchannel_multiplier_shape(%arg0: tensor<13x21x3xi16>) -> tensor<13x21x3xi16> {
+  %multiplier = "tosa.const"() {values = dense<19689> : tensor<3xi32> } : () -> tensor<3xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  // expected-error@+1 {{'tosa.rescale' op expect shape of { 1 } for multiplier input, got { 3 }}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = true} : (tensor<13x21x3xi16>, tensor<3xi32>, tensor<1xi8>, tensor<1xi16>, tensor<1xi16>) -> tensor<13x21x3xi16>
+  return %0 : tensor<13x21x3xi16>
+}
+
+// -----
+
+func.func @test_rescale_invalid_perchannel_shift_shape(%arg0: tensor<13x21x3xi16>) -> tensor<13x21x3xi16> {
+  %multiplier = "tosa.const"() {values = dense<19689> : tensor<3xi32> } : () -> tensor<3xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  // expected-error@+1 {{'tosa.rescale' op expect shape of { 3 } for shift input, got { 1 }}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", per_channel = true, scale32 = true, input_unsigned = false, output_unsigned = true} : (tensor<13x21x3xi16>, tensor<3xi32>, tensor<1xi8>, tensor<1xi16>, tensor<1xi16>) -> tensor<13x21x3xi16>
+  return %0 : tensor<13x21x3xi16>
+}
+
+// -----
+
+func.func @test_rescale_invalid_non_perchannel_shift_shape(%arg0: tensor<13x21x3xi16>) -> tensor<13x21x3xi16> {
+  %multiplier = "tosa.const"() {values = dense<19689> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<3xi8> } : () -> tensor<3xi8>
+  %input_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi16>} : () -> tensor<1xi16>
+  // expected-error@+1 {{'tosa.rescale' op expect shape of { 1 } for shift input, got { 3 }}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = true} : (tensor<13x21x3xi16>, tensor<1xi32>, tensor<3xi8>, tensor<1xi16>, tensor<1xi16>) -> tensor<13x21x3xi16>
+  return %0 : tensor<13x21x3xi16>
+}
+
+// -----
+// CHECK-LABEL: test_matmul_a_zp_same_element_type
+func.func @test_matmul_a_zp_same_element_type(%arg0: tensor<1x14x19xf32>, %arg1: tensor<1x19x28xf32>) -> tensor<1x14x28xf32> {
+%azp0 = "tosa.const"() <{values = dense<0.0> : tensor<1xf16>}> : () -> tensor<1xf16>
+%bzp0 = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+// expected-error@+1 {{'tosa.matmul' op expect input a and a_zp have the same element type, got 'f32' and 'f16'}}
+%0 = tosa.matmul %arg0, %arg1, %azp0, %bzp0 : (tensor<1x14x19xf32>, tensor<1x19x28xf32>, tensor<1xf16>, tensor<1xf32>)  -> tensor<1x14x28xf32>
+  return %0 : tensor<1x14x28xf32>
+}
+
+// -----
+// CHECK-LABEL: test_matmul_b_zp_same_element_type
+func.func @test_matmul_b_zp_same_element_type(%arg0: tensor<1x14x19xf32>, %arg1: tensor<1x19x28xf32>) -> tensor<1x14x28xf32> {
+%azp0 = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+%bzp0 = "tosa.const"() <{values = dense<0.0> : tensor<1xf16>}> : () -> tensor<1xf16>
+// expected-error@+1 {{'tosa.matmul' op expect input b and b_zp have the same element type, got 'f32' and 'f16'}}
+%0 = tosa.matmul %arg0, %arg1, %azp0, %bzp0 : (tensor<1x14x19xf32>, tensor<1x19x28xf32>, tensor<1xf32>, tensor<1xf16>)  -> tensor<1x14x28xf32>
+  return %0 : tensor<1x14x28xf32>
+}
+
+// -----
+// CHECK-LABEL: test_matmul_a_zp_non_zero
+func.func @test_matmul_a_zp_non_zero(%arg0: tensor<1x14x19xf32>, %arg1: tensor<1x19x28xf32>) -> tensor<1x14x28xf32> {
+%azp0 = "tosa.const"() <{values = dense<1.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+%bzp0 = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+// expected-error@+1 {{'tosa.matmul' op a zero point must be zero for non-int8 integer types}}
+%0 = tosa.matmul %arg0, %arg1, %azp0, %bzp0 : (tensor<1x14x19xf32>, tensor<1x19x28xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<1x14x28xf32>
+  return %0 : tensor<1x14x28xf32>
+}
+
+// -----
+// CHECK-LABEL: test_matmul_b_zp_non_zero
+func.func @test_matmul_b_zp_non_zero(%arg0: tensor<1x14x19xf32>, %arg1: tensor<1x19x28xf32>) -> tensor<1x14x28xf32> {
+%azp0 = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+%bzp0 = "tosa.const"() <{values = dense<-1.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+// expected-error@+1 {{'tosa.matmul' op b zero point must be zero for non-int8 integer types}}
+%0 = tosa.matmul %arg0, %arg1, %azp0, %bzp0 : (tensor<1x14x19xf32>, tensor<1x19x28xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<1x14x28xf32>
+  return %0 : tensor<1x14x28xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_negate_same_element_type
+func.func @test_negate_same_element_type(%arg0: tensor<1x16x16x8xf16>, %arg1: tensor<1xf16>, %arg2: tensor<1xf16>) -> tensor<1x16x16x8xf32> {
+  // expected-error@+1 {{'tosa.negate' op expect input and output to have same element type, got 'f16' and 'f32'}}
+  %0 = tosa.negate %arg0, %arg1, %arg2
+      : (tensor<1x16x16x8xf16>, tensor<1xf16>, tensor<1xf16>) -> tensor<1x16x16x8xf32>
+  return %0 : tensor<1x16x16x8xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_negate_same_shape
+func.func @test_negate_same_shape(%arg0: tensor<1x16x16x16xf16>, %arg1: tensor<1xf16>, %arg2: tensor<1xf16>) -> tensor<1x16x16x8xf16> {
+  // expected-error@+1 {{'tosa.negate' op requires the same shape for input1 and output}}
+  %0 = tosa.negate %arg0, %arg1, %arg2
+      : (tensor<1x16x16x16xf16>, tensor<1xf16>, tensor<1xf16>) -> tensor<1x16x16x8xf16>
+  return %0 : tensor<1x16x16x8xf16>
+}
+
+// -----
+
+// CHECK-LABEL: test_negate_input_zp_same_element_type
+func.func @test_negate_input_zp_same_element_type(%arg0: tensor<1x16x16x8xf16>, %arg1: tensor<1xi8>, %arg2: tensor<1xf16>) -> tensor<1x16x16x8xf16> {
+  // expected-error@+1 {{'tosa.negate' op expect both input1 and its zero point are the same element type, got 'f16' and 'i8'}}
+  %0 = tosa.negate %arg0, %arg1, %arg2
+      : (tensor<1x16x16x8xf16>, tensor<1xi8>, tensor<1xf16>) -> tensor<1x16x16x8xf16>
+  return %0 : tensor<1x16x16x8xf16>
+}
+
+// -----
+
+// CHECK-LABEL: test_negate_output_zp_same_element_type
+func.func @test_negate_output_zp_same_element_type(%arg0: tensor<1x16x16x8xi8>, %arg1: tensor<1xi8>, %arg2: tensor<1xf16>) -> tensor<1x16x16x8xi8> {
+  // expected-error@+1 {{'tosa.negate' op expect both output and its zero point are the same element type, got 'i8' and 'f16'}}
+  %0 = tosa.negate %arg0, %arg1, %arg2
+      : (tensor<1x16x16x8xi8>, tensor<1xi8>, tensor<1xf16>) -> tensor<1x16x16x8xi8>
+  return %0 : tensor<1x16x16x8xi8>
+}
+
+// -----
+
+// CHECK-LABEL: test_negate_input_zp_non_zero
+func.func @test_negate_input_zp_non_zero(%arg0: tensor<1x16x16x8xf32>) -> tensor<1x16x16x8xf32> {
+  %input_zp = "tosa.const"() {values = dense<-1.0> : tensor<1xf32>} : () -> tensor<1xf32>
+  %output_zp = "tosa.const"() {values = dense<0.0> : tensor<1xf32>} : () -> tensor<1xf32>
+  // expected-error@+1 {{'tosa.negate' op input1 zero point must be zero for non-int8 integer types}}
+  %0 = tosa.negate %arg0, %input_zp, %output_zp
+      : (tensor<1x16x16x8xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<1x16x16x8xf32>
+  return %0 : tensor<1x16x16x8xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_negate_output_zp_non_zero
+func.func @test_negate_output_zp_non_zero(%arg0: tensor<1x16x16x8xf32>) -> tensor<1x16x16x8xf32> {
+  %input_zp = "tosa.const"() {values = dense<0.0> : tensor<1xf32>} : () -> tensor<1xf32>
+  %output_zp = "tosa.const"() {values = dense<-1.0> : tensor<1xf32>} : () -> tensor<1xf32>
+  // expected-error@+1 {{'tosa.negate' op output zero point must be zero for non-int8 integer types}}
+  %0 = tosa.negate %arg0, %input_zp, %output_zp
+      : (tensor<1x16x16x8xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<1x16x16x8xf32>
+  return %0 : tensor<1x16x16x8xf32>
+}
diff --git a/mlir/test/Dialect/Tosa/invalid_extension.mlir b/mlir/test/Dialect/Tosa/invalid_extension.mlir
index 8352abc0c406e..1a8366528b35c 100644
--- a/mlir/test/Dialect/Tosa/invalid_extension.mlir
+++ b/mlir/test/Dialect/Tosa/invalid_extension.mlir
@@ -70,3 +70,39 @@ func.func @test_while_loop(%arg0: tensor<10xi32>, %arg1: tensor<i32>) {
   return
 }
 
+// -----
+
+// CHECK-LABEL: test_single_round_rescale
+func.func @test_single_round_rescale(%arg0: tensor<13x21x3xi8>) -> tensor<13x21x3xi8> {
+  %multiplier = "tosa.const"() {values = dense<1073741824> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
+  // CHECK tosa.rescale
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", input_zp = 127 : i32, output_zp = -1 : i32, per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = false} : (tensor<13x21x3xi8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<13x21x3xi8>
+  return %0 : tensor<13x21x3xi8>
+}
+
+// -----
+
+func.func @test_double_round_rescale(%arg0: tensor<13x21x3xi8>) -> tensor<13x21x3xi8> {
+  %multiplier = "tosa.const"() {values = dense<1073741824> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
+  // expected-error@+1 {{'tosa.rescale' op failed attribute check: rounding_mode = DOUBLE_ROUND requires extension [doubleround]}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "DOUBLE_ROUND", input_zp = 127 : i32, output_zp = -1 : i32, per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = false} : (tensor<13x21x3xi8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<13x21x3xi8>
+  return %0 : tensor<13x21x3xi8>
+}
+
+// -----
+
+func.func @test_inexact_round_rescale(%arg0: tensor<13x21x3xi8>) -> tensor<13x21x3xi8> {
+  %multiplier = "tosa.const"() {values = dense<1073741824> : tensor<1xi32> } : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
+  // expected-error@+1 {{'tosa.rescale' op failed attribute check: rounding_mode = INEXACT_ROUND requires extension [inexactround]}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "INEXACT_ROUND", input_zp = 127 : i32, output_zp = -1 : i32, per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = false} : (tensor<13x21x3xi8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<13x21x3xi8>
+  return %0 : tensor<13x21x3xi8>
+}
diff --git a/mlir/test/Dialect/Tosa/level_check.mlir b/mlir/test/Dialect/Tosa/level_check.mlir
index 8f64f217514de..71b9e68433caa 100644
--- a/mlir/test/Dialect/Tosa/level_check.mlir
+++ b/mlir/test/Dialect/Tosa/level_check.mlir
@@ -249,9 +249,9 @@ func.func @test_logical_not_rank_invalid(%arg0: tensor<1x1x1x1x1x21x3xi1>) -> te
 
 // -----
 
-func.func @test_negate_rank_invalid(%arg0: tensor<1x1x1x1x13x21x3xf32>) -> tensor<1x1x1x1x13x21x3xf32> {
+func.func @test_negate_rank_invalid(%arg0: tensor<1x1x1x1x13x21x3xf32>, %arg1: tensor<1xf32>) -> tensor<1x1x1x1x13x21x3xf32> {
   // expected-error@+1 {{'tosa.negate' op failed level check: operand rank(shape) <= MAX_RANK}}
-  %0 = tosa.negate %arg0 : (tensor<1x1x1x1x13x21x3xf32>) -> tensor<1x1x1x1x13x21x3xf32>
+  %0 = tosa.negate %arg0, %arg1, %arg1 : (tensor<1x1x1x1x13x21x3xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<1x1x1x1x13x21x3xf32>
   return %0 : tensor<1x1x1x1x13x21x3xf32>
 }
 
@@ -432,8 +432,12 @@ func.func @test_cast_rank_invalid(%arg0: tensor<1x1x1x1x13x21x3xi32>) -> tensor<
 // -----
 
 func.func @test_rescale_rank_invalid(%arg0: tensor<1x1x1x1x13x21x3xi8>) -> tensor<1x1x1x1x13x21x3xi8> {
+  %multiplier = "tosa.const"() {values = dense<1073741824> : tensor<1xi32>} : () -> tensor<1xi32>
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8>} : () -> tensor<1xi8>
+  %input_zp = "tosa.const"() {values = dense<127> : tensor<1xi8>} : () -> tensor<1xi8>
+  %output_zp = "tosa.const"() {values = dense<-1> : tensor<1xi8>} : () -> tensor<1xi8>
   // expected-error@+1 {{'tosa.rescale' op failed level check: operand rank(shape) <= MAX_RANK}}
-  %0 = tosa.rescale %arg0 {double_round = false, input_zp = 127 : i32, output_zp = -1 : i32, multiplier = array<i32: 1073741824>, shift = array<i8: 30>, per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = false} : (tensor<1x1x1x1x13x21x3xi8>) -> tensor<1x1x1x1x13x21x3xi8>
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", input_zp = 127 : i32, output_zp = -1 : i32, per_channel = false, scale32 = true, input_unsigned = false, output_unsigned = false} : (tensor<1x1x1x1x13x21x3xi8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x1x1x1x13x21x3xi8>
   return %0 : tensor<1x1x1x1x13x21x3xi8>
 }
 
@@ -1108,8 +1112,9 @@ func.func @test_rfft2d_tensor_size_invalid(%arg0: tensor<536870912x8x16xf32>) ->
 // -----
 
 func.func @test_matmul_tensor_size_invalid(%arg0: tensor<23178x20000x19xf32>, %arg1: tensor<23178x19x28xf32>) -> tensor<23178x20000x28xf32> {
+  %zero = "tosa.const"() {values = dense<0.0> : tensor<1xf32>} : () -> tensor<1xf32>
   // expected-error@+1 {{'tosa.matmul' op failed level check: operand tensor size (in bytes) <= (1 << MAX_LOG2_SIZE - 1)}}
-  %0 = tosa.matmul %arg0, %arg1 : (tensor<23178x20000x19xf32>, tensor<23178x19x28xf32>) -> tensor<23178x20000x28xf32>
+  %0 = tosa.matmul %arg0, %arg1, %zero, %zero : (tensor<23178x20000x19xf32>, tensor<23178x19x28xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<23178x20000x28xf32>
   return %0 : tensor<23178x20000x28xf32>
 }
 
diff --git a/mlir/test/Dialect/Tosa/ops.mlir b/mlir/test/Dialect/Tosa/ops.mlir
index 2e60a3449f1c6..017fc9615345d 100644
--- a/mlir/test/Dialect/Tosa/ops.mlir
+++ b/mlir/test/Dialect/Tosa/ops.mlir
@@ -94,7 +94,11 @@ func.func @test_conv2d_q8xi4(%arg0: tensor<1x11x11x3xi8>) -> tensor<1x1x1x3xi8>
   %izp = "tosa.const"() {values = dense<0> : tensor<1xi8>} : () -> tensor<1xi8>
   %wzp = "tosa.const"() {values = dense<0> : tensor<1xi4>} : () -> tensor<1xi4>
   %2 = "tosa.conv2d"(%arg0, %0, %1, %izp, %wzp) {acc_type = i32, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>} : (tensor<1x11x11x3xi8>, tensor<3x11x11x3xi4>, tensor<3xi32>, tensor<1xi8>, tensor<1xi4>) -> tensor<1x1x1x3xi32>
-  %3 = "tosa.rescale"(%2) {double_round = true, input_zp = 0 : i32, multiplier = array<i32: 2026291432, 1079222024, 1693132724>, output_zp = 27 : i32, per_channel = true, scale32 = true, shift = array<i8: 37, 36, 37>, input_unsigned = false, output_unsigned = false} : (tensor<1x1x1x3xi32>) -> tensor<1x1x1x3xi8>
+  %multiplier = "tosa.const"() {values = dense<[2026291432, 1079222024, 1693132724]> : tensor<3xi32>} : () -> tensor<3xi32>
+  %shift = "tosa.const"() {values = dense<[37, 36, 37]> : tensor<3xi8>} : () -> tensor<3xi8>
+  %rescale_input_zp = "tosa.const"() <{values = dense<0> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %rescale_output_zp = "tosa.const"() <{values = dense<27> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %3 = tosa.rescale %2, %multiplier, %shift, %rescale_input_zp, %rescale_output_zp {rounding_mode = "DOUBLE_ROUND", scale32 = true, per_channel = true, input_unsigned = false, output_unsigned = false} : (tensor<1x1x1x3xi32>, tensor<3xi32>, tensor<3xi8>, tensor<1xi32>, tensor<1xi8>) -> tensor<1x1x1x3xi8>
   return %3 : tensor<1x1x1x3xi8>
 }
 
@@ -143,7 +147,9 @@ func.func @test_fft2d_with_local_bound(%arg0: tensor<1x4x8xf32>, %arg1: tensor<1
 // -----
 // CHECK-LABEL: test_matmul
 func.func @test_matmul(%arg0: tensor<1x14x19xf32>, %arg1: tensor<1x19x28xf32>) -> tensor<1x14x28xf32> {
-  %0 = tosa.matmul %arg0, %arg1 : (tensor<1x14x19xf32>, tensor<1x19x28xf32>) -> tensor<1x14x28xf32>
+%azp0 = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+%bzp0 = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+%0 = tosa.matmul %arg0, %arg1, %azp0, %bzp0 : (tensor<1x14x19xf32>, tensor<1x19x28xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<1x14x28xf32>
   return %0 : tensor<1x14x28xf32>
 }
 
@@ -175,6 +181,13 @@ func.func @test_rfft2d(%arg0: tensor<13x8x16xf32>) -> (tensor<13x8x9xf32>, tenso
   return %0, %1 : tensor<13x8x9xf32>, tensor<13x8x9xf32>
 }
 
+// -----
+// CHECK-LABEL: rfft2d_width1
+func.func @test_rfft2d_width1(%arg0: tensor<1x1x1xf32>) -> (tensor<1x1x1xf32>, tensor<1x1x1xf32>) {
+  %0, %1 = tosa.rfft2d %arg0 : (tensor<1x1x1xf32>) -> (tensor<1x1x1xf32>, tensor<1x1x1xf32>)
+  return %0, %1 : tensor<1x1x1xf32>, tensor<1x1x1xf32>
+}
+
 // -----
 // CHECK-LABEL: rfft2d_with_local_bound
 func.func @test_rfft2d_with_local_bound(%arg0: tensor<13x8x16xf32>) -> (tensor<13x8x9xf32>, tensor<13x8x9xf32>) {
@@ -476,8 +489,8 @@ func.func @test_logical_not(%arg0: tensor<1x21x3xi1>) -> tensor<1x21x3xi1> {
 
 // -----
 // CHECK-LABEL: negate
-func.func @test_negate(%arg0: tensor<13x21x3xf32>) -> tensor<13x21x3xf32> {
-  %0 = tosa.negate %arg0 : (tensor<13x21x3xf32>) -> tensor<13x21x3xf32>
+func.func @test_negate(%arg0: tensor<13x21x3xf32>, %arg1: tensor<1xf32>, %arg2: tensor<1xf32>) -> tensor<13x21x3xf32> {
+  %0 = tosa.negate %arg0, %arg1, %arg2 : (tensor<13x21x3xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<13x21x3xf32>
   return %0 : tensor<13x21x3xf32>
 }
 
@@ -711,7 +724,11 @@ func.func @test_cast3(%arg0: tensor<13x21x3xi32>) -> tensor<13x21x3x!quant.unifo
 // -----
 // CHECK-LABEL: rescale
 func.func @test_rescale(%arg0: tensor<13x21x3x!quant.uniform<u8:f32, 0.015655439347028732:127>>) -> tensor<13x21x3x!quant.uniform<i8:f32, 0.015655439347028732:-1>> {
-    %0 = tosa.rescale %arg0 {double_round = false, input_zp = 127 : i32, multiplier = array<i32: 1073741824>, output_zp = -1 : i32, per_channel = false, scale32 = true, shift = array<i8: 30>, input_unsigned = false, output_unsigned = false} : (tensor<13x21x3x!quant.uniform<u8:f32, 0.015655439347028732:127>>) -> tensor<13x21x3x!quant.uniform<i8:f32, 0.015655439347028732:-1>>
+   %multiplier = "tosa.const"() {values = dense<1073741824> : tensor<1xi32> } : () -> tensor<1xi32>
+   %shift = "tosa.const"() {values = dense<30> : tensor<1xi8> } : () -> tensor<1xi8>
+   %input_zp = "tosa.const"() <{values = dense<127> : tensor<1xi8>}> : () -> tensor<1xi8>
+   %output_zp = "tosa.const"() <{values = dense<-1> : tensor<1xi8>}> : () -> tensor<1xi8>
+   %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", scale32 = true, per_channel = false, input_unsigned = false, output_unsigned = false} : (tensor<13x21x3x!quant.uniform<u8:f32, 0.015655439347028732:127>>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<13x21x3x!quant.uniform<i8:f32, 0.015655439347028732:-1>>
     return %0 : tensor<13x21x3x!quant.uniform<i8:f32, 0.015655439347028732:-1>>
 }
 
@@ -776,3 +793,295 @@ func.func @test_const_shape() -> !tosa.shape<4> {
   %cst = tosa.const_shape {values = dense<1> : tensor<4xindex>} : () -> !tosa.shape<4>
   return %cst : !tosa.shape<4>
 }
+
+// F8 support tests
+
+// -----
+// CHECK-LABEL: argmax_f8E5M2
+func.func @test_argmax_f8E5M2(%arg0: tensor<12x8x16xf8E5M2>) -> tensor<12x16xi32> {
+  %0 = tosa.argmax %arg0 { axis = 1 : i32 } : (tensor<12x8x16xf8E5M2>) -> tensor<12x16xi32>
+  return %0 : tensor<12x16xi32>
+}
+
+// -----
+// CHECK-LABEL: avg_pool2d_f8E5M2
+func.func @test_avg_pool2d_f8E5M2(%arg0: tensor<1x7x7x9xf8E5M2>) -> tensor<1x7x7x9xf8E5M2> {
+  %input_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf8E5M2>}> : () -> tensor<1xf8E5M2>
+  %output_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf8E5M2>}> : () -> tensor<1xf8E5M2>
+  %0 = tosa.avg_pool2d %arg0, %input_zp, %output_zp {acc_type = f16, kernel = array<i64: 2, 2>, pad = array<i64: 0, 1, 0, 1>, stride = array<i64: 1, 1>} : (tensor<1x7x7x9xf8E5M2>, tensor<1xf8E5M2>, tensor<1xf8E5M2>) -> tensor<1x7x7x9xf8E5M2>
+  return %0 : tensor<1x7x7x9xf8E5M2>
+}
+
+// -----
+// CHECK-LABEL: conv2d_f8E5M2
+func.func @test_conv2d_f8E5M2(%arg0: tensor<1x4x4x4xf8E5M2>, %arg1: tensor<8x1x1x4xf8E5M2>, %arg2: tensor<8xf16>) -> tensor<1x4x4x8xf16> {
+  %input_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf8E5M2>}> : () -> tensor<1xf8E5M2>
+  %weight_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf8E5M2>}> : () -> tensor<1xf8E5M2>
+  %0 = tosa.conv2d %arg0, %arg1, %arg2, %input_zp, %weight_zp {acc_type = f16, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, local_bound = true} : (tensor<1x4x4x4xf8E5M2>, tensor<8x1x1x4xf8E5M2>, tensor<8xf16>, tensor<1xf8E5M2>, tensor<1xf8E5M2>) -> tensor<1x4x4x8xf16>
+  return %0 : tensor<1x4x4x8xf16>
+}
+
+// -----
+// CHECK-LABEL: conv3d_f8E5M2
+func.func @test_conv3d_f8E5M2(%arg0: tensor<1x4x8x21x17xf8E5M2>, %arg1: tensor<34x1x1x1x17xf8E5M2>, %arg2: tensor<34xf16>, %arg3: tensor<1xf8E5M2>, %arg4: tensor<1xf8E5M2>) -> tensor<1x4x8x21x34xf16> {
+  %0 = tosa.conv3d %arg0, %arg1, %arg2, %arg3, %arg4 {acc_type = f16, dilation = array<i64: 1, 1, 1>, pad = array<i64: 0, 0, 0, 0, 0, 0>, stride = array<i64: 1, 1, 1>} : (tensor<1x4x8x21x17xf8E5M2>, tensor<34x1x1x1x17xf8E5M2>, tensor<34xf16>, tensor<1xf8E5M2>, tensor<1xf8E5M2>) -> tensor<1x4x8x21x34xf16>
+  return %0 : tensor<1x4x8x21x34xf16>
+}
+
+// -----
+// CHECK-LABEL: depthwise_conv2d_f8E5M2
+func.func @test_depthwise_conv2d_f8E5M2(%arg0: tensor<1x4x4x4xf8E5M2>, %arg1: tensor<1x1x4x2xf8E5M2>, %arg2: tensor<8xf16>, %arg3: tensor<1xf8E5M2>, %arg4: tensor<1xf8E5M2>) -> tensor<1x4x4x8xf16> {
+  %0 = tosa.depthwise_conv2d %arg0, %arg1, %arg2, %arg3, %arg4 {acc_type = f16, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>} : (tensor<1x4x4x4xf8E5M2>, tensor<1x1x4x2xf8E5M2>, tensor<8xf16>, tensor<1xf8E5M2>, tensor<1xf8E5M2>) -> tensor<1x4x4x8xf16>
+  return %0 : tensor<1x4x4x8xf16>
+}
+
+// -----
+// CHECK-LABEL: test_matmul_f8E5M2
+func.func @test_matmul_f8E5M2(%arg0: tensor<1x14x19xf8E5M2>, %arg1: tensor<1x19x28xf8E5M2>) -> tensor<1x14x28xf16> {
+  %azp0 = "tosa.const"() <{values = dense<0.0> : tensor<1xf8E5M2>}> : () -> tensor<1xf8E5M2>
+  %bzp0 = "tosa.const"() <{values = dense<0.0> : tensor<1xf8E5M2>}> : () -> tensor<1xf8E5M2>
+  %0 = tosa.matmul %arg0, %arg1, %azp0, %bzp0 : (tensor<1x14x19xf8E5M2>, tensor<1x19x28xf8E5M2>, tensor<1xf8E5M2>, tensor<1xf8E5M2>) -> tensor<1x14x28xf16>
+  return %0 : tensor<1x14x28xf16>
+}
+
+// -----
+// CHECK-LABEL: max_pool2d_f8E5M2
+func.func @test_max_pool2d_f8E5M2(%arg0: tensor<1x32x32x8xf8E5M2>) -> tensor<1x32x32x8xf8E5M2> {
+  %0 = tosa.max_pool2d %arg0 {kernel = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>} : (tensor<1x32x32x8xf8E5M2>) -> tensor<1x32x32x8xf8E5M2>
+  return %0 : tensor<1x32x32x8xf8E5M2>
+}
+
+// -----
+
+// CHECK-LABEL: transpose_conv2d_f8E5M2
+func.func @test_transpose_conv2d_f8E5M2(%arg0: tensor<1x32x32x8xf8E5M2>, %arg1: tensor<16x1x1x8xf8E5M2>, %arg2: tensor<16xf16>, %arg3: tensor<1xf8E5M2>, %arg4: tensor<1xf8E5M2>) -> tensor<1x32x32x16xf16> {
+  %0 = tosa.transpose_conv2d %arg0, %arg1, %arg2, %arg3, %arg4 {acc_type = f16, out_pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>} : (tensor<1x32x32x8xf8E5M2>, tensor<16x1x1x8xf8E5M2>, tensor<16xf16>, tensor<1xf8E5M2>, tensor<1xf8E5M2>) -> tensor<1x32x32x16xf16>
+  return %0 : tensor<1x32x32x16xf16>
+}
+
+// -----
+// CHECK-LABEL: const_f8E5M2
+func.func @test_const_f8E5M2(%arg0 : index) -> tensor<4xf8E5M2> {
+    %0 = "tosa.const"() {values = dense<[3.0, -0.0, -1.0, 2.0]> : tensor<4xf8E5M2>} : () -> tensor<4xf8E5M2>
+    return %0 : tensor<4xf8E5M2>
+}
+
+// -----
+// CHECK-LABEL: cast_f8E5M2
+func.func @test_cast_f8E5M2(%arg0: tensor<13x21x3xf8E5M2>) -> tensor<13x21x3xf16> {
+  %0 = tosa.cast %arg0 : (tensor<13x21x3xf8E5M2>) -> tensor<13x21x3xf16>
+  return %0 : tensor<13x21x3xf16>
+}
+
+// -----
+// CHECK-LABEL: concat_f8E5M2
+func.func @test_concat_f8E5M2(%arg0: tensor<13x21x3xf8E5M2>, %arg1: tensor<13x21x3xf8E5M2>) -> tensor<26x21x3xf8E5M2> {
+  %0 = tosa.concat %arg0, %arg1 {axis = 0 : i32} : (tensor<13x21x3xf8E5M2>, tensor<13x21x3xf8E5M2>) -> tensor<26x21x3xf8E5M2>
+  return %0 : tensor<26x21x3xf8E5M2>
+}
+
+// -----
+// CHECK-LABEL: pad_f8E5M2
+func.func @test_pad_f8E5M2(%arg0: tensor<13x21x3xf8E5M2>) -> tensor<13x21x3xf8E5M2> {
+  %padding = tosa.const_shape {values = dense<0> : tensor<6xindex>} : () -> !tosa.shape<6>
+  %cst = "tosa.const"() { values = dense<-0.0> : tensor<1xf8E5M2> } : () -> tensor<1xf8E5M2>
+  %0 = tosa.pad %arg0, %padding, %cst : (tensor<13x21x3xf8E5M2>, !tosa.shape<6>, tensor<1xf8E5M2>) -> tensor<13x21x3xf8E5M2>
+  return %0 : tensor<13x21x3xf8E5M2>
+}
+
+// -----
+// CHECK-LABEL: reshape_f8E5M2
+func.func @test_reshape_f8E5M2(%arg0: tensor<13x21x3xf8E5M2>) -> tensor<1x819xf8E5M2> {
+  %1 = tosa.const_shape {values = dense<[1, 819]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %0 = tosa.reshape %arg0, %1 : (tensor<13x21x3xf8E5M2>, !tosa.shape<2>) -> tensor<1x819xf8E5M2>
+  return %0 : tensor<1x819xf8E5M2>
+}
+
+// -----
+// CHECK-LABEL: reverse_f8E5M2
+func.func @test_reverse_f8E5M2(%arg0: tensor<13x21x3xf8E5M2>) -> tensor<13x21x3xf8E5M2> {
+  %0 = tosa.reverse %arg0 {axis = 0 : i32} : (tensor<13x21x3xf8E5M2>) -> tensor<13x21x3xf8E5M2>
+  return %0 : tensor<13x21x3xf8E5M2>
+}
+
+// -----
+// CHECK-LABEL: slice_f8E5M2
+func.func @test_slice_f8E5M2(%arg0: tensor<13x21x3xf8E5M2>) -> tensor<4x11x1xf8E5M2> {
+  %0 = tosa.const_shape {values = dense<[4, 11, 1]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %1 = tosa.const_shape {values = dense<[6, 8, 0]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %2 = tosa.slice %arg0, %0, %1 : (tensor<13x21x3xf8E5M2>, !tosa.shape<3>, !tosa.shape<3>) -> tensor<4x11x1xf8E5M2>
+  return %2 : tensor<4x11x1xf8E5M2>
+}
+
+// -----
+// CHECK-LABEL: tile_f8E5M2
+func.func @test_tile_f8E5M2(%arg0: tensor<13x21x3xf8E5M2>) -> tensor<39x21x6xf8E5M2> {
+  %cst = tosa.const_shape { values = dense<[3, 1, 2]> : tensor<3xindex> } : () -> !tosa.shape<3>
+  %0 = tosa.tile %arg0, %cst: (tensor<13x21x3xf8E5M2>, !tosa.shape<3>) -> tensor<39x21x6xf8E5M2>
+  return %0 : tensor<39x21x6xf8E5M2>
+}
+
+// -----
+func.func @test_transpose_f8E5M2(%arg0: tensor<13x21x3xf8E5M2>) -> tensor<3x13x21xf8E5M2> {
+  %1 = tosa.transpose %arg0 {perms = array<i32: 2, 0, 1>} : (tensor<13x21x3xf8E5M2>) -> tensor<3x13x21xf8E5M2>
+  return %1 : tensor<3x13x21xf8E5M2>
+}
+
+// -----
+// CHECK-LABEL: gather_f8E5M2
+func.func @test_gather_f8E5M2(%arg0: tensor<13x21x3xf8E5M2>, %arg1: tensor<13x26xi32>) -> tensor<13x26x3xf8E5M2> {
+  %0 = tosa.gather %arg0, %arg1 : (tensor<13x21x3xf8E5M2>, tensor<13x26xi32>) -> tensor<13x26x3xf8E5M2>
+  return %0 : tensor<13x26x3xf8E5M2>
+}
+
+// -----
+// CHECK-LABEL: scatter_f8E5M2
+func.func @test_scatter_f8E5M2(%arg0: tensor<13x21x3xf8E5M2>, %arg1: tensor<13x26xi32>, %arg2: tensor<13x26x3xf8E5M2>) -> tensor<13x21x3xf8E5M2> {
+  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x21x3xf8E5M2>, tensor<13x26xi32>, tensor<13x26x3xf8E5M2>) -> tensor<13x21x3xf8E5M2>
+  return %0 : tensor<13x21x3xf8E5M2>
+}
+
+// -----
+// CHECK-LABEL: argmax_f8E4M3FN
+func.func @test_argmax_f8E4M3FN(%arg0: tensor<12x8x16xf8E4M3FN>) -> tensor<12x16xi32> {
+  %0 = tosa.argmax %arg0 { axis = 1 : i32 } : (tensor<12x8x16xf8E4M3FN>) -> tensor<12x16xi32>
+  return %0 : tensor<12x16xi32>
+}
+
+// -----
+// CHECK-LABEL: avg_pool2d_f8E4M3FN
+func.func @test_avg_pool2d_f8E4M3FN(%arg0: tensor<1x7x7x9xf8E4M3FN>) -> tensor<1x7x7x9xf8E4M3FN> {
+  %input_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf8E4M3FN>}> : () -> tensor<1xf8E4M3FN>
+  %output_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf8E4M3FN>}> : () -> tensor<1xf8E4M3FN>
+  %0 = tosa.avg_pool2d %arg0, %input_zp, %output_zp {acc_type = f16, kernel = array<i64: 2, 2>, pad = array<i64: 0, 1, 0, 1>, stride = array<i64: 1, 1>} : (tensor<1x7x7x9xf8E4M3FN>, tensor<1xf8E4M3FN>, tensor<1xf8E4M3FN>) -> tensor<1x7x7x9xf8E4M3FN>
+  return %0 : tensor<1x7x7x9xf8E4M3FN>
+}
+
+// -----
+// CHECK-LABEL: conv2d_f8E4M3FN
+func.func @test_conv2d_f8E4M3FN(%arg0: tensor<1x4x4x4xf8E4M3FN>, %arg1: tensor<8x1x1x4xf8E4M3FN>, %arg2: tensor<8xf16>) -> tensor<1x4x4x8xf16> {
+  %input_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf8E4M3FN>}> : () -> tensor<1xf8E4M3FN>
+  %weight_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf8E4M3FN>}> : () -> tensor<1xf8E4M3FN>
+  %0 = tosa.conv2d %arg0, %arg1, %arg2, %input_zp, %weight_zp {acc_type = f16, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, local_bound = true} : (tensor<1x4x4x4xf8E4M3FN>, tensor<8x1x1x4xf8E4M3FN>, tensor<8xf16>, tensor<1xf8E4M3FN>, tensor<1xf8E4M3FN>) -> tensor<1x4x4x8xf16>
+  return %0 : tensor<1x4x4x8xf16>
+}
+
+// -----
+// CHECK-LABEL: conv3d_f8E4M3FN
+func.func @test_conv3d_f8E4M3FN(%arg0: tensor<1x4x8x21x17xf8E4M3FN>, %arg1: tensor<34x1x1x1x17xf8E4M3FN>, %arg2: tensor<34xf16>, %arg3: tensor<1xf8E4M3FN>, %arg4: tensor<1xf8E4M3FN>) -> tensor<1x4x8x21x34xf16> {
+  %0 = tosa.conv3d %arg0, %arg1, %arg2, %arg3, %arg4 {acc_type = f16, dilation = array<i64: 1, 1, 1>, pad = array<i64: 0, 0, 0, 0, 0, 0>, stride = array<i64: 1, 1, 1>} : (tensor<1x4x8x21x17xf8E4M3FN>, tensor<34x1x1x1x17xf8E4M3FN>, tensor<34xf16>, tensor<1xf8E4M3FN>, tensor<1xf8E4M3FN>) -> tensor<1x4x8x21x34xf16>
+  return %0 : tensor<1x4x8x21x34xf16>
+}
+
+// -----
+// CHECK-LABEL: depthwise_conv2d_f8E4M3FN
+func.func @test_depthwise_conv2d_f8E4M3FN(%arg0: tensor<1x4x4x4xf8E4M3FN>, %arg1: tensor<1x1x4x2xf8E4M3FN>, %arg2: tensor<8xf16>, %arg3: tensor<1xf8E4M3FN>, %arg4: tensor<1xf8E4M3FN>) -> tensor<1x4x4x8xf16> {
+  %0 = tosa.depthwise_conv2d %arg0, %arg1, %arg2, %arg3, %arg4 {acc_type = f16, dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>} : (tensor<1x4x4x4xf8E4M3FN>, tensor<1x1x4x2xf8E4M3FN>, tensor<8xf16>, tensor<1xf8E4M3FN>, tensor<1xf8E4M3FN>) -> tensor<1x4x4x8xf16>
+  return %0 : tensor<1x4x4x8xf16>
+}
+
+// -----
+// CHECK-LABEL: matmul_f8E4M3FN
+func.func @test_matmul_f8E4M3FN(%arg0: tensor<1x14x19xf8E4M3FN>, %arg1: tensor<1x19x28xf8E4M3FN>) -> tensor<1x14x28xf16> {
+  %azp0 = "tosa.const"() <{values = dense<0.0> : tensor<1xf8E4M3FN>}> : () -> tensor<1xf8E4M3FN>
+  %bzp0 = "tosa.const"() <{values = dense<0.0> : tensor<1xf8E4M3FN>}> : () -> tensor<1xf8E4M3FN>
+  %0 = tosa.matmul %arg0, %arg1, %azp0, %bzp0 : (tensor<1x14x19xf8E4M3FN>, tensor<1x19x28xf8E4M3FN>, tensor<1xf8E4M3FN>, tensor<1xf8E4M3FN>) -> tensor<1x14x28xf16>
+  return %0 : tensor<1x14x28xf16>
+}
+
+// -----
+// CHECK-LABEL: max_pool2d_f8E4M3FN
+func.func @test_max_pool2d_f8E4M3FN(%arg0: tensor<1x32x32x8xf8E4M3FN>) -> tensor<1x32x32x8xf8E4M3FN> {
+  %0 = tosa.max_pool2d %arg0 {kernel = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>} : (tensor<1x32x32x8xf8E4M3FN>) -> tensor<1x32x32x8xf8E4M3FN>
+  return %0 : tensor<1x32x32x8xf8E4M3FN>
+}
+
+// -----
+// CHECK-LABEL: transpose_conv2d_f8E4M3FN
+func.func @test_transpose_conv2d_f8E4M3FN(%arg0: tensor<1x32x32x8xf8E4M3FN>, %arg1: tensor<16x1x1x8xf8E4M3FN>, %arg2: tensor<16xf16>, %arg3: tensor<1xf8E4M3FN>, %arg4: tensor<1xf8E4M3FN>) -> tensor<1x32x32x16xf16> {
+  %0 = tosa.transpose_conv2d %arg0, %arg1, %arg2, %arg3, %arg4 {acc_type = f16, out_pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>} : (tensor<1x32x32x8xf8E4M3FN>, tensor<16x1x1x8xf8E4M3FN>, tensor<16xf16>, tensor<1xf8E4M3FN>, tensor<1xf8E4M3FN>) -> tensor<1x32x32x16xf16>
+  return %0 : tensor<1x32x32x16xf16>
+}
+
+// -----
+// CHECK-LABEL: const_f8E4M3FN
+func.func @test_const_f8E4M3FN(%arg0 : index) -> tensor<4xf8E4M3FN> {
+    %0 = "tosa.const"() {values = dense<[3.0, -0.0, -1.0, 2.0]> : tensor<4xf8E4M3FN>} : () -> tensor<4xf8E4M3FN>
+    return %0 : tensor<4xf8E4M3FN>
+}
+
+// -----
+// CHECK-LABEL: cast_f8E4M3FN
+func.func @test_cast_f8E4M3FN(%arg0: tensor<13x21x3xf8E4M3FN>) -> tensor<13x21x3xf16> {
+  %0 = tosa.cast %arg0 : (tensor<13x21x3xf8E4M3FN>) -> tensor<13x21x3xf16>
+  return %0 : tensor<13x21x3xf16>
+}
+
+// -----
+// CHECK-LABEL: concat_f8E4M3FN
+func.func @test_concat_f8E4M3FN(%arg0: tensor<13x21x3xf8E4M3FN>, %arg1: tensor<13x21x3xf8E4M3FN>) -> tensor<26x21x3xf8E4M3FN> {
+  %0 = tosa.concat %arg0, %arg1 {axis = 0 : i32} : (tensor<13x21x3xf8E4M3FN>, tensor<13x21x3xf8E4M3FN>) -> tensor<26x21x3xf8E4M3FN>
+  return %0 : tensor<26x21x3xf8E4M3FN>
+}
+
+// -----
+// CHECK-LABEL: pad_f8E4M3FN
+func.func @test_pad_f8E4M3FN(%arg0: tensor<13x21x3xf8E4M3FN>) -> tensor<13x21x3xf8E4M3FN> {
+  %padding = tosa.const_shape {values = dense<0> : tensor<6xindex>} : () -> !tosa.shape<6>
+  %cst = "tosa.const"() { values = dense<-0.0> : tensor<1xf8E4M3FN> } : () -> tensor<1xf8E4M3FN>
+  %0 = tosa.pad %arg0, %padding, %cst : (tensor<13x21x3xf8E4M3FN>, !tosa.shape<6>, tensor<1xf8E4M3FN>) -> tensor<13x21x3xf8E4M3FN>
+  return %0 : tensor<13x21x3xf8E4M3FN>
+}
+
+// -----
+// CHECK-LABEL: reshape_f8E4M3FN
+func.func @test_reshape_f8E4M3FN(%arg0: tensor<13x21x3xf8E4M3FN>) -> tensor<1x819xf8E4M3FN> {
+  %1 = tosa.const_shape {values = dense<[1, 819]> : tensor<2xindex>} : () -> !tosa.shape<2>
+  %0 = tosa.reshape %arg0, %1 : (tensor<13x21x3xf8E4M3FN>, !tosa.shape<2>) -> tensor<1x819xf8E4M3FN>
+  return %0 : tensor<1x819xf8E4M3FN>
+}
+
+// -----
+// CHECK-LABEL: reverse_f8E4M3FN
+func.func @test_reverse_f8E4M3FN(%arg0: tensor<13x21x3xf8E4M3FN>) -> tensor<13x21x3xf8E4M3FN> {
+  %0 = tosa.reverse %arg0 {axis = 0 : i32} : (tensor<13x21x3xf8E4M3FN>) -> tensor<13x21x3xf8E4M3FN>
+  return %0 : tensor<13x21x3xf8E4M3FN>
+}
+
+// -----
+// CHECK-LABEL: slice_f8E4M3FN
+func.func @test_slice_f8E4M3FN(%arg0: tensor<13x21x3xf8E4M3FN>) -> tensor<4x11x1xf8E4M3FN> {
+  %0 = tosa.const_shape {values = dense<[4, 11, 1]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %1 = tosa.const_shape {values = dense<[6, 8, 0]> : tensor<3xindex>} : () -> !tosa.shape<3>
+  %2 = tosa.slice %arg0, %0, %1 : (tensor<13x21x3xf8E4M3FN>, !tosa.shape<3>, !tosa.shape<3>) -> tensor<4x11x1xf8E4M3FN>
+  return %2 : tensor<4x11x1xf8E4M3FN>
+}
+
+// -----
+// CHECK-LABEL: tile_f8E4M3FN
+func.func @test_tile_f8E4M3FN(%arg0: tensor<13x21x3xf8E4M3FN>) -> tensor<39x21x6xf8E4M3FN> {
+  %cst = tosa.const_shape { values = dense<[3, 1, 2]> : tensor<3xindex> } : () -> !tosa.shape<3>
+  %0 = tosa.tile %arg0, %cst: (tensor<13x21x3xf8E4M3FN>, !tosa.shape<3>) -> tensor<39x21x6xf8E4M3FN>
+  return %0 : tensor<39x21x6xf8E4M3FN>
+}
+
+// -----
+// CHECK-LABEL: transpose_f8E4M3FN
+func.func @test_transpose_f8E4M3FN(%arg0: tensor<13x21x3xf8E4M3FN>) -> tensor<3x13x21xf8E4M3FN> {
+  %1 = tosa.transpose %arg0 {perms = array<i32: 2, 0, 1>} : (tensor<13x21x3xf8E4M3FN>) -> tensor<3x13x21xf8E4M3FN>
+  return %1 : tensor<3x13x21xf8E4M3FN>
+}
+
+// -----
+// CHECK-LABEL: gather_f8E4M3FN
+func.func @test_gather_f8E4M3FN(%arg0: tensor<13x21x3xf8E4M3FN>, %arg1: tensor<13x26xi32>) -> tensor<13x26x3xf8E4M3FN> {
+  %0 = tosa.gather %arg0, %arg1 : (tensor<13x21x3xf8E4M3FN>, tensor<13x26xi32>) -> tensor<13x26x3xf8E4M3FN>
+  return %0 : tensor<13x26x3xf8E4M3FN>
+}
+
+// -----
+// CHECK-LABEL: scatter_f8E4M3FN
+func.func @test_scatter_f8E4M3FN(%arg0: tensor<13x21x3xf8E4M3FN>, %arg1: tensor<13x26xi32>, %arg2: tensor<13x26x3xf8E4M3FN>) -> tensor<13x21x3xf8E4M3FN> {
+  %0 = tosa.scatter %arg0, %arg1, %arg2 : (tensor<13x21x3xf8E4M3FN>, tensor<13x26xi32>, tensor<13x26x3xf8E4M3FN>) -> tensor<13x21x3xf8E4M3FN>
+  return %0 : tensor<13x21x3xf8E4M3FN>
+}
diff --git a/mlir/test/Dialect/Tosa/profile_all_unsupported.mlir b/mlir/test/Dialect/Tosa/profile_all_unsupported.mlir
index 342c57b0dd85c..8b221eb38b4f4 100644
--- a/mlir/test/Dialect/Tosa/profile_all_unsupported.mlir
+++ b/mlir/test/Dialect/Tosa/profile_all_unsupported.mlir
@@ -2,7 +2,7 @@
 // Enable all supported extensions to focus the verification of expected profile requirement errors.
 //--------------------------------------------------------------------------------------------------
 
-// RUN: mlir-opt %s -split-input-file -verify-diagnostics -tosa-validate="extension=int16,int4,bf16,fp8e4m3,fp8e5m2,fft,variable,controlflow strict-op-spec-alignment"
+// RUN: mlir-opt %s -split-input-file -verify-diagnostics -tosa-validate="extension=int16,int4,bf16,fp8e4m3,fp8e5m2,fft,variable,controlflow,dynamic,doubleround,inexactround strict-op-spec-alignment"
 
 // -----
 func.func @test_table(%arg0 : tensor<4x5xi8>, %arg1 : tensor<513xi8>) -> () {
@@ -26,9 +26,9 @@ func.func @test_avg_pool2d(%arg0: tensor<1x7x7x9xf32>, %arg1: tensor<1xf32>, %ar
 }
 
 // -----
-func.func @test_matmul(%arg0: tensor<1x14x19xf32>, %arg1: tensor<1x19x28xf32>) -> tensor<1x14x28xf32> {
+func.func @test_matmul(%arg0: tensor<1x14x19xf32>, %arg1: tensor<1x19x28xf32>, %arg2: tensor<1xf32>) -> tensor<1x14x28xf32> {
   // expected-error@+1 {{'tosa.matmul' op illegal: requires [pro_fp] but not enabled in target}}
-  %0 = tosa.matmul %arg0, %arg1 : (tensor<1x14x19xf32>, tensor<1x19x28xf32>) -> tensor<1x14x28xf32>
+  %0 = tosa.matmul %arg0, %arg1, %arg2, %arg2: (tensor<1x14x19xf32>, tensor<1x19x28xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<1x14x28xf32>
   return %0 : tensor<1x14x28xf32>
 }
 
diff --git a/mlir/test/Dialect/Tosa/profile_pro_fp_unsupported.mlir b/mlir/test/Dialect/Tosa/profile_pro_fp_unsupported.mlir
index 3dd0344e3647d..54ddef67b0baa 100644
--- a/mlir/test/Dialect/Tosa/profile_pro_fp_unsupported.mlir
+++ b/mlir/test/Dialect/Tosa/profile_pro_fp_unsupported.mlir
@@ -2,7 +2,7 @@
 // Enable all supported extensions to focus the verification of expected profile requirement errors.
 //--------------------------------------------------------------------------------------------------
 
-// RUN: mlir-opt %s -split-input-file -verify-diagnostics -tosa-validate="profile=pro_int extension=int16,int4,bf16,fp8e4m3,fp8e5m2,fft,variable,controlflow strict-op-spec-alignment"
+// RUN: mlir-opt %s -split-input-file -verify-diagnostics -tosa-validate="profile=pro_int extension=int16,int4,bf16,fp8e4m3,fp8e5m2,fft,variable,controlflow,doubleround,inexactround strict-op-spec-alignment"
 
 // -----
 func.func @test_conv2d(%arg0: tensor<1x4x4x4xf32>, %arg1: tensor<8x1x1x4xf32>, %arg2: tensor<8xf32>, %arg3: tensor<1xf32>, %arg4: tensor<1xf32>) -> tensor<1x4x4x8xf32> {
@@ -19,9 +19,9 @@ func.func @test_avg_pool2d(%arg0: tensor<1x7x7x9xf32>, %arg1: tensor<1xf32>, %ar
 }
 
 // -----
-func.func @test_matmul(%arg0: tensor<1x14x19xf32>, %arg1: tensor<1x19x28xf32>) -> tensor<1x14x28xf32> {
+func.func @test_matmul(%arg0: tensor<1x14x19xf32>, %arg1: tensor<1x19x28xf32>, %arg2: tensor<1xf32>) -> tensor<1x14x28xf32> {
   // expected-error@+1 {{'tosa.matmul' op illegal: requires [pro_fp] but not enabled in target}}
-  %0 = tosa.matmul %arg0, %arg1 : (tensor<1x14x19xf32>, tensor<1x19x28xf32>) -> tensor<1x14x28xf32>
+  %0 = tosa.matmul %arg0, %arg1, %arg2, %arg2 : (tensor<1x14x19xf32>, tensor<1x19x28xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<1x14x28xf32>
   return %0 : tensor<1x14x28xf32>
 }
 
diff --git a/mlir/test/Dialect/Tosa/profile_pro_int_unsupported.mlir b/mlir/test/Dialect/Tosa/profile_pro_int_unsupported.mlir
index 1d6d33b9a02c7..c7221a88bda00 100644
--- a/mlir/test/Dialect/Tosa/profile_pro_int_unsupported.mlir
+++ b/mlir/test/Dialect/Tosa/profile_pro_int_unsupported.mlir
@@ -2,7 +2,7 @@
 // Enable all supported extensions to focus the verification of expected profile requirement errors.
 //--------------------------------------------------------------------------------------------------
 
-// RUN: mlir-opt %s -split-input-file -verify-diagnostics -tosa-validate="profile=pro_fp extension=int16,int4,bf16,fp8e4m3,fp8e5m2,fft,variable,controlflow strict-op-spec-alignment"
+// RUN: mlir-opt %s -split-input-file -verify-diagnostics -tosa-validate="profile=pro_fp extension=int16,int4,bf16,fp8e4m3,fp8e5m2,fft,variable,controlflow,doubleround,inexactround strict-op-spec-alignment"
 
 // -----
 func.func @test_table(%arg0 : tensor<4x5xi8>, %arg1 : tensor<513xi8>) -> () {
@@ -24,3 +24,18 @@ func.func @test_cast_i8_i32(%arg0: tensor<13x21x3xi32>) -> tensor<13x21x3xi8> {
   %0 = tosa.cast %arg0 : (tensor<13x21x3xi32>) -> tensor<13x21x3xi8>
   return %0 : tensor<13x21x3xi8>
 }
+
+// -----
+func.func @test_rescale(%arg0: tensor<13x21x3xi8>) -> tensor<13x21x3xi32> {
+  // expected-error@+1 {{'tosa.const' op illegal: requires [pro_int] but not enabled in target}}
+  %multiplier = "tosa.const"() {values = dense<1073741824> : tensor<1xi32>} : () -> tensor<1xi32>
+  // expected-error@+1 {{'tosa.const' op illegal: requires [pro_int] but not enabled in target}}
+  %shift = "tosa.const"() {values = dense<30> : tensor<1xi8>} : () -> tensor<1xi8>
+  // expected-error@+1 {{'tosa.const' op illegal: requires [pro_int] but not enabled in target}}
+  %input_zp = "tosa.const"() {values = dense<127> : tensor<1xi8>} : () -> tensor<1xi8>
+  // expected-error@+1 {{'tosa.const' op illegal: requires [pro_int] but not enabled in target}}
+  %output_zp = "tosa.const"() {values = dense<0> : tensor<1xi32>} : () -> tensor<1xi32>
+  // expected-error@+1 {{'tosa.rescale' op illegal: requires [pro_int] but not enabled in target}}
+  %0 = tosa.rescale %arg0, %multiplier, %shift, %input_zp, %output_zp {rounding_mode = "SINGLE_ROUND", scale32 = true, per_channel = false, input_unsigned = false, output_unsigned = false} : (tensor<13x21x3xi8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi32>) -> tensor<13x21x3xi32>
+  return %0 : tensor<13x21x3xi32>
+}
diff --git a/mlir/test/Dialect/Tosa/quant-test.mlir b/mlir/test/Dialect/Tosa/quant-test.mlir
index 447a6ef7f9e05..f0ad4eb4fdb0b 100644
--- a/mlir/test/Dialect/Tosa/quant-test.mlir
+++ b/mlir/test/Dialect/Tosa/quant-test.mlir
@@ -2,9 +2,9 @@
 
 // -----
 // CHECK-LABEL: test_build_qtype
-func.func @test_build_qtype(%arg0 : tensor<16x1x1x8x!quant.uniform<u8<1:255>:f32, 0.015680249780416489:128>>) -> tensor<16x1x1x8x!quant.uniform<u8<1:255>:f32, 0.015680249780416489:128>> {
+func.func @test_build_qtype(%arg0 : tensor<16x1x1x8x!quant.uniform<u8<1:255>:f32, 0.015680249780416489:128>>, %arg1: tensor<1xi8>, %arg2: tensor<1xi8>) -> tensor<16x1x1x8x!quant.uniform<u8<1:255>:f32, 0.015680249780416489:128>> {
   //  CHECK: tosa.negate
-  %0 = "tosa.negate"(%arg0) : (tensor<16x1x1x8x!quant.uniform<u8<1:255>:f32, 0.015680249780416489:128>>) -> tensor<16x1x1x8x!quant.uniform<u8<1:255>:f32, 0.015680249780416489:128>>
+  %0 = "tosa.negate"(%arg0, %arg1, %arg2) : (tensor<16x1x1x8x!quant.uniform<u8<1:255>:f32, 0.015680249780416489:128>>, tensor<1xi8>, tensor<1xi8>) -> tensor<16x1x1x8x!quant.uniform<u8<1:255>:f32, 0.015680249780416489:128>>
   return %0 : tensor<16x1x1x8x!quant.uniform<u8<1:255>:f32, 0.015680249780416489:128>>
 }
 
diff --git a/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir b/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir
index 6d480b841441a..f3aaacc8a1baa 100644
--- a/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir
+++ b/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir
@@ -45,8 +45,10 @@ func.func @test_unary_f32(%arg0 : tensor<4xf32>) -> () {
   // CHECK: tosa.log %arg0 : (tensor<4xf32>) -> tensor<4xf32>
   %5 = tosa.log %arg0 : (tensor<4xf32>) -> tensor<*xf32>
 
-  // CHECK: tosa.negate %arg0 : (tensor<4xf32>) -> tensor<4xf32>
-  %6 = tosa.negate %arg0 : (tensor<4xf32>) -> tensor<*xf32>
+  %in_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %out_zp = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  // CHECK: tosa.negate %arg0, {{.+}} : (tensor<4xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<4xf32>
+  %6 = tosa.negate %arg0, %in_zp, %out_zp : (tensor<4xf32>, tensor<1xf32>, tensor<1xf32>) -> tensor<*xf32>
 
   // CHECK: tosa.reciprocal %arg0 : (tensor<4xf32>) -> tensor<4xf32>
   %7 = tosa.reciprocal %arg0 : (tensor<4xf32>) -> tensor<*xf32>
@@ -74,7 +76,7 @@ func.func @test_unary_f32(%arg0 : tensor<4xf32>) -> () {
 // -----
 
 // CHECK-LABEL: @test_unary_i32
-func.func @test_unary_i32(%arg0 : tensor<4xi32>) -> () {
+func.func @test_unary_i32(%arg0 : tensor<4xi32>, %arg1 : tensor<2xi8>) -> () {
   // CHECK: tosa.abs %arg0 : (tensor<4xi32>) -> tensor<4xi32>
   %0 = tosa.abs %arg0 : (tensor<4xi32>) -> tensor<*xi32>
 
@@ -87,14 +89,24 @@ func.func @test_unary_i32(%arg0 : tensor<4xi32>) -> () {
   // CHECK: tosa.clz %arg0 : (tensor<4xi32>) -> tensor<4xi32>
   %3 = tosa.clz %arg0 : (tensor<4xi32>) -> tensor<*xi32>
 
-  // CHECK: tosa.negate %arg0 : (tensor<4xi32>) -> tensor<4xi32>
-  %4 = tosa.negate %arg0 : (tensor<4xi32>) -> tensor<*xi32>
+  %in_zp = "tosa.const"() <{values = dense<0> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %out_zp = "tosa.const"() <{values = dense<0> : tensor<1xi32>}> : () -> tensor<1xi32>
+  // CHECK: tosa.negate %arg0, {{.+}} : (tensor<4xi32>, tensor<1xi32>, tensor<1xi32>) -> tensor<4xi32>
+  %4 = tosa.negate %arg0, %in_zp, %out_zp : (tensor<4xi32>, tensor<1xi32>, tensor<1xi32>) -> tensor<*xi32>
 
   // CHECK: tosa.reverse %arg0 {axis = 0 : i32} : (tensor<4xi32>) -> tensor<4xi32>
   %5 = tosa.reverse %arg0 { axis = 0 : i32 } : (tensor<4xi32>) -> tensor<?xi32>
 
-  // CHECK: tosa.rescale %arg0 {{.+}} : (tensor<4xi32>) -> tensor<4xi16>
-  %6 = tosa.rescale %arg0 {input_zp = 243 : i32, output_zp = 252 : i32, multiplier = array<i32: 42, 43>, shift = array<i8: 14, 15>, scale32 = false, double_round = false, per_channel = false, input_unsigned = false, output_unsigned = false} : (tensor<4xi32>) -> tensor<*xi16>
+  // CHECK-DAG: %[[MULT:.+]] = "tosa.const"() <{values = dense<[42, 43]> : tensor<2xi16>}> : () -> tensor<2xi16>
+  // CHECK-DAG: %[[SHIFT:.+]] = "tosa.const"() <{values = dense<[14, 15]> : tensor<2xi8>}> : () -> tensor<2xi8>
+  // CHECK-DAG: %[[INPUTZP:.+]] = "tosa.const"() <{values = dense<43> : tensor<1xi8>}> : () -> tensor<1xi8>
+  // CHECK-DAG: %[[OUTPUTZP:.+]] = "tosa.const"() <{values = dense<52> : tensor<1xi8>}> : () -> tensor<1xi8>
+  // CHECK: tosa.rescale %arg1, %[[MULT]], %[[SHIFT]], %[[INPUTZP]], %[[OUTPUTZP]] {{.+}} : (tensor<2xi8>, tensor<2xi16>, tensor<2xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<2xi8>
+  %multiplier = "tosa.const"() {values = dense<[42, 43]> : tensor<2xi16>} : () -> tensor<2xi16>
+  %shift = "tosa.const"() {values = dense<[14, 15]> : tensor<2xi8>} : () -> tensor<2xi8>
+  %input_zp = "tosa.const"() {values = dense<43> : tensor<1xi8>} : () -> tensor<1xi8>
+  %output_zp = "tosa.const"() {values = dense<52> : tensor<1xi8>} : () -> tensor<1xi8>
+  %6 = tosa.rescale %arg1, %multiplier, %shift, %input_zp, %output_zp {scale32 = false, rounding_mode = "SINGLE_ROUND", per_channel = true, input_unsigned = true, output_unsigned = true} : (tensor<2xi8>, tensor<2xi16>, tensor<2xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<2xi8>
 
   // CHECK: tosa.identity %arg0 : (tensor<4xi32>) -> tensor<4xi32>
   %7 = tosa.identity %arg0 : (tensor<4xi32>) -> tensor<?xi32>
@@ -275,8 +287,10 @@ func.func @test_dynamic_argmax(%arg0 : tensor<2x?xi32>) -> () {
 
 // CHECK-LABEL: @test_static_matmul
 func.func @test_static_matmul(%arg0 : tensor<2x3x4xi32>, %arg1 : tensor<2x4x5xi32>) -> () {
-  // CHECK: tosa.matmul %arg0, %arg1 : (tensor<2x3x4xi32>, tensor<2x4x5xi32>) -> tensor<2x3x5xi32>
-  %0 = tosa.matmul %arg0, %arg1 : (tensor<2x3x4xi32>, tensor<2x4x5xi32>) -> tensor<?x?x?xi32>
+  // CHECK tosa.matmul %arg0, %arg1, %0, %1 : (tensor<2x3x4xi32>, tensor<2x4x5xi32>, tensor<1xi32>, tensor<1xi32>)  -> tensor<2x3x5xi32>
+  %0 = "tosa.const"() <{values = dense<0> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %1 = "tosa.const"() <{values = dense<0> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %2 = tosa.matmul %arg0, %arg1, %0, %1 : (tensor<2x3x4xi32>, tensor<2x4x5xi32>, tensor<1xi32>, tensor<1xi32>)  -> tensor<?x?x?xi32>
 
   return
 }
@@ -285,8 +299,10 @@ func.func @test_static_matmul(%arg0 : tensor<2x3x4xi32>, %arg1 : tensor<2x4x5xi3
 
 // CHECK-LABEL: @test_dynamic_lhs_matmul
 func.func @test_dynamic_lhs_matmul(%arg0 : tensor<?x?x?xi32>, %arg1 : tensor<2x4x5xi32>) -> () {
-  // CHECK: tosa.matmul %arg0, %arg1 : (tensor<?x?x?xi32>, tensor<2x4x5xi32>) -> tensor<2x?x5xi32>
-  %0 = tosa.matmul %arg0, %arg1 : (tensor<?x?x?xi32>, tensor<2x4x5xi32>) -> tensor<?x?x?xi32>
+  // CHECK: tosa.matmul %arg0, %arg1, %0, %1 : (tensor<?x?x?xi32>, tensor<2x4x5xi32>, tensor<1xi32>, tensor<1xi32>)  -> tensor<2x?x5xi32>
+  %0 = "tosa.const"() <{values = dense<0> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %1 = "tosa.const"() <{values = dense<0> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %2 = tosa.matmul %arg0, %arg1, %0, %1 : (tensor<?x?x?xi32>, tensor<2x4x5xi32>, tensor<1xi32>, tensor<1xi32>)  -> tensor<?x?x?xi32>
 
   return
 }
@@ -295,8 +311,10 @@ func.func @test_dynamic_lhs_matmul(%arg0 : tensor<?x?x?xi32>, %arg1 : tensor<2x4
 
 // CHECK-LABEL: @test_dynamic_rhs_matmul
 func.func @test_dynamic_rhs_matmul(%arg0 : tensor<2x3x4xi32>, %arg1 : tensor<?x?x?xi32>) -> () {
-  // CHECK: tosa.matmul %arg0, %arg1 : (tensor<2x3x4xi32>, tensor<?x?x?xi32>) -> tensor<2x3x?xi32>
-  %0 = tosa.matmul %arg0, %arg1 : (tensor<2x3x4xi32>, tensor<?x?x?xi32>) -> tensor<?x?x?xi32>
+  // CHECK: tosa.matmul %arg0, %arg1, %0, %1 : (tensor<2x3x4xi32>, tensor<?x?x?xi32>, tensor<1xi32>, tensor<1xi32>)  -> tensor<2x3x?xi32>
+  %0 = "tosa.const"() <{values = dense<0> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %1 = "tosa.const"() <{values = dense<0> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %2 = tosa.matmul %arg0, %arg1, %0, %1 : (tensor<2x3x4xi32>, tensor<?x?x?xi32>, tensor<1xi32>, tensor<1xi32>)  -> tensor<?x?x?xi32>
 
   return
 }
@@ -305,8 +323,10 @@ func.func @test_dynamic_rhs_matmul(%arg0 : tensor<2x3x4xi32>, %arg1 : tensor<?x?
 
 // CHECK-LABEL: @test_dynamic_mixed_matmul
 func.func @test_dynamic_mixed_matmul(%arg0 : tensor<?x3x?xi32>, %arg1 : tensor<?x?x5xi32>) -> () {
-  // CHECK: tosa.matmul %arg0, %arg1 : (tensor<?x3x?xi32>, tensor<?x?x5xi32>) -> tensor<?x3x5xi32>
-  %0 = tosa.matmul %arg0, %arg1 : (tensor<?x3x?xi32>, tensor<?x?x5xi32>) -> tensor<?x?x?xi32>
+  // CHECK: tosa.matmul %arg0, %arg1, %0, %1 : (tensor<?x3x?xi32>, tensor<?x?x5xi32>, tensor<1xi32>, tensor<1xi32>)  -> tensor<?x3x5xi32>
+  %0 = "tosa.const"() <{values = dense<0> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %1 = "tosa.const"() <{values = dense<0> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %2 = tosa.matmul %arg0, %arg1, %0, %1 : (tensor<?x3x?xi32>, tensor<?x?x5xi32>, tensor<1xi32>, tensor<1xi32>)  -> tensor<?x?x?xi32>
 
   return
 }
@@ -1401,11 +1421,13 @@ func.func @test_non_tosa_consumer_extract(%arg0: tensor<4x4xf32>, %arg1: index)
 
 // CHECK-LABEL: test_non_tosa_consumer_still_propagates
 func.func @test_non_tosa_consumer_still_propagates(%arg0: tensor<1x1x8xf32>, %arg1: tensor<1x8x1xf32>) -> tensor<?x?xf32> {
-  // CHECK: tosa.matmul %arg0, %arg1 : (tensor<1x1x8xf32>, tensor<1x8x1xf32>) -> tensor<1x1x1xf32>
-  %0 = tosa.matmul %arg0, %arg1 : (tensor<1x1x8xf32>, tensor<1x8x1xf32>) -> tensor<?x1x1xf32>
-  %1 = arith.constant dense<[1, 1]> : tensor<2xindex>
-  %2 = tensor.reshape %0(%1) : (tensor<?x1x1xf32>, tensor<2xindex>) -> tensor<?x?xf32>
-  return %2 : tensor<?x?xf32>
+  // CHECK: tosa.matmul %arg0, %arg1, %0, %1 : (tensor<1x1x8xf32>, tensor<1x8x1xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<1x1x1xf32>
+  %0 = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %1 = "tosa.const"() <{values = dense<0.0> : tensor<1xf32>}> : () -> tensor<1xf32>
+  %2 = tosa.matmul %arg0, %arg1, %0, %1 : (tensor<1x1x8xf32>, tensor<1x8x1xf32>, tensor<1xf32>, tensor<1xf32>)  -> tensor<?x1x1xf32>
+  %3 = arith.constant dense<[1, 1]> : tensor<2xindex>
+  %4 = tensor.reshape %2(%3) : (tensor<?x1x1xf32>, tensor<2xindex>) -> tensor<?x?xf32>
+  return %4 : tensor<?x?xf32>
 }
 
 // -----
diff --git a/mlir/test/Dialect/Vector/vector-unroll-options.mlir b/mlir/test/Dialect/Vector/vector-unroll-options.mlir
index 16d30aec7c041..9c158d05b723c 100644
--- a/mlir/test/Dialect/Vector/vector-unroll-options.mlir
+++ b/mlir/test/Dialect/Vector/vector-unroll-options.mlir
@@ -222,6 +222,15 @@ func.func @vector_multi_reduction(%v : vector<4x6xf32>, %acc: vector<4xf32>) ->
 //       CHECK:   %[[V2:.*]] = vector.insert_strided_slice %[[R5]], %[[V1]] {offsets = [2], strides = [1]} : vector<2xf32> into vector<4xf32>
 //       CHECK:   return %[[V2]] : vector<4xf32>
 
+// This is a negative test case to ensure that further unrolling is not performed. Since the vector.multi_reduction
+// operation has already been unrolled, attempting additional unrolling should not be allowed.
+func.func @negative_vector_multi_reduction(%v: vector<4x2xf32>, %acc: f32) -> f32 {
+  %0 = vector.multi_reduction #vector.kind<add>, %v, %acc [0, 1] : vector<4x2xf32> to f32
+  return %0 : f32
+}
+// CHECK-LABEL: func @negative_vector_multi_reduction
+//  CHECK-NEXT:   %[[R0:.*]] = vector.multi_reduction <add>, %{{.*}}, %{{.*}} [0, 1] : vector<4x2xf32> to f32
+//  CHECK-NEXT:   return %[[R0]] : f32
 
 func.func @vector_reduction(%v : vector<8xf32>) -> f32 {
   %0 = vector.reduction <add>, %v : vector<8xf32> into f32
diff --git a/mlir/test/Integration/Dialect/MemRef/copy-runtime-verification.mlir b/mlir/test/Integration/Dialect/MemRef/copy-runtime-verification.mlir
new file mode 100644
index 0000000000000..95b9db2832cee
--- /dev/null
+++ b/mlir/test/Integration/Dialect/MemRef/copy-runtime-verification.mlir
@@ -0,0 +1,28 @@
+// RUN: mlir-opt %s -generate-runtime-verification \
+// RUN:     -expand-strided-metadata \
+// RUN:     -test-cf-assert \
+// RUN:     -convert-to-llvm | \
+// RUN: mlir-runner -e main -entry-point-result=void \
+// RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
+// RUN: FileCheck %s
+
+// Put memref.copy in a function, otherwise the memref.cast may fold.
+func.func @memcpy_helper(%src: memref<?xf32>, %dest: memref<?xf32>) {
+  memref.copy %src, %dest : memref<?xf32> to memref<?xf32>
+  return
+}
+
+func.func @main() {
+  %alloca1 = memref.alloca() : memref<4xf32>
+  %alloca2 = memref.alloca() : memref<5xf32>
+  %cast1 = memref.cast %alloca1 : memref<4xf32> to memref<?xf32>
+  %cast2 = memref.cast %alloca2 : memref<5xf32> to memref<?xf32>
+
+  //      CHECK: ERROR: Runtime op verification failed
+  // CHECK-NEXT: "memref.copy"(%{{.*}}, %{{.*}}) : (memref<?xf32>, memref<?xf32>) -> ()
+  // CHECK-NEXT: ^ size of 0-th source/target dim does not match
+  // CHECK-NEXT: Location: loc({{.*}})
+  call @memcpy_helper(%cast1, %cast2) : (memref<?xf32>, memref<?xf32>) -> ()
+
+  return
+}
diff --git a/mlir/test/Target/LLVMIR/Import/call-argument-attributes.ll b/mlir/test/Target/LLVMIR/Import/call-argument-attributes.ll
index fa39c79bf0859..842639df0d44b 100644
--- a/mlir/test/Target/LLVMIR/Import/call-argument-attributes.ll
+++ b/mlir/test/Target/LLVMIR/Import/call-argument-attributes.ll
@@ -1,4 +1,4 @@
-; RUN: mlir-translate -import-llvm %s | FileCheck %s
+; RUN: mlir-translate -import-llvm -split-input-file %s | FileCheck %s
 
 ; CHECK-LABEL: llvm.func @somefunc(i32, !llvm.ptr)
 declare void @somefunc(i32, ptr)
@@ -20,3 +20,22 @@ define i16 @test_call_arg_attrs_indirect(i16 %0, ptr %1) {
   %3 = tail call signext i16 %1(i16 noundef signext %0)
   ret i16 %3
 }
+
+; // -----
+
+%struct.S = type { i8 }
+
+; CHECK-LABEL: @t
+define void @t(i1 %0) #0 {
+  %3 = alloca %struct.S, align 1
+  ; CHECK-NOT: llvm.call @z(%1) {no_unwind} : (!llvm.ptr) -> ()
+  ; CHECK: llvm.call @z(%1) : (!llvm.ptr) -> ()
+  call void @z(ptr %3)
+  ret void
+}
+
+define linkonce_odr void @z(ptr %0) #0 {
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir
index db2e08742dbca..f9473bf112387 100644
--- a/mlir/test/Target/LLVMIR/llvmir.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir.mlir
@@ -2367,6 +2367,18 @@ llvm.func @readwrite_func() attributes {
 
 // -----
 
+//
+// target-features attribute.
+//
+
+// CHECK-LABEL: @tf
+// CHECK-SAME: #[[TargetFeat:.*]]
+llvm.func @tf(!llvm.ptr) attributes {target_features = #llvm.target_features<["+fix-cortex-a53-835769", "+fp-armv8", "+neon", "+outline-atomics", "+v8a"]>}
+
+// CHECK: attributes #[[TargetFeat]] = { "target-features"="+fix-cortex-a53-835769,+fp-armv8,+neon,+outline-atomics,+v8a" }
+
+// -----
+
 //
 // arm_streaming attribute.
 //
@@ -2454,6 +2466,18 @@ llvm.func @preserves_za_func() attributes {arm_preserves_za} {
 
 // -----
 
+//
+// frame pointer attribute.
+//
+
+// CHECK-LABEL: @t
+// CHECK-SAME: #[[FP:.*]]
+llvm.func @t(!llvm.ptr) attributes {frame_pointer = #llvm.framePointerKind<"non-leaf">}
+
+// CHECK: attributes #[[FP]] = { "frame-pointer"="non-leaf" }
+
+// -----
+
 //
 // Zero-initialize operation.
 //
diff --git a/mlir/test/Target/LLVMIR/omptarget-private-llvm.mlir b/mlir/test/Target/LLVMIR/omptarget-private-llvm.mlir
new file mode 100644
index 0000000000000..a2500f3a579dd
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-private-llvm.mlir
@@ -0,0 +1,46 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// Regression tset for calling a function using pointer alloca'ed on
+// device for private variable
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memory_space", 5 : ui32>>, llvm.data_layout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9", llvm.target_triple = "amdgcn-amd-amdhsa", omp.is_gpu = true, omp.is_target_device = true} {
+  omp.private {type = private} @_QMmodFfailingEi_private_i32 : i32
+  llvm.func @_QMotherProutine(%arg0: !llvm.ptr {fir.bindc_name = "i", llvm.nocapture}) attributes {frame_pointer = #llvm.framePointerKind<all>, omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>, target_cpu = "gfx90a", target_features = #llvm.target_features<["+16-bit-insts", "+atomic-buffer-global-pk-add-f16-insts", "+atomic-fadd-rtn-insts", "+ci-insts", "+dl-insts", "+dot1-insts", "+dot10-insts", "+dot2-insts", "+dot3-insts", "+dot4-insts", "+dot5-insts", "+dot6-insts", "+dot7-insts", "+dpp", "+gfx8-insts", "+gfx9-insts", "+gfx90a-insts", "+gws", "+image-insts", "+mai-insts", "+s-memrealtime", "+s-memtime-inst", "+wavefrontsize64"]>} {
+    llvm.return
+  }
+  llvm.func @_QMmodPfailing(%arg0: !llvm.ptr {fir.bindc_name = "d", llvm.nocapture}) attributes {frame_pointer = #llvm.framePointerKind<all>, omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (to)>, target_cpu = "gfx90a", target_features = #llvm.target_features<["+16-bit-insts", "+atomic-buffer-global-pk-add-f16-insts", "+atomic-fadd-rtn-insts", "+ci-insts", "+dl-insts", "+dot1-insts", "+dot10-insts", "+dot2-insts", "+dot3-insts", "+dot4-insts", "+dot5-insts", "+dot6-insts", "+dot7-insts", "+dpp", "+gfx8-insts", "+gfx9-insts", "+gfx90a-insts", "+gws", "+image-insts", "+mai-insts", "+s-memrealtime", "+s-memtime-inst", "+wavefrontsize64"]>} {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr<5>
+    %2 = llvm.addrspacecast %1 : !llvm.ptr<5> to !llvm.ptr
+    %3 = llvm.mlir.constant(1 : i64) : i64
+    %4 = omp.map.info var_ptr(%2 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "i"}
+    %5 = omp.map.info var_ptr(%arg0 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "d"}
+    omp.target map_entries(%4 -> %arg1, %5 -> %arg2 : !llvm.ptr, !llvm.ptr) {
+      %6 = llvm.mlir.constant(1 : i32) : i32
+      omp.teams {
+
+// CHECK:    omp.par.entry:
+// CHECK:      %[[TID_ADDR_LOCAL:.*]] = alloca i32, align 4, addrspace(5)
+// CHECK:      %[[OMP_PRIVATE_ALLOC:omp\.private\.alloc]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT: %[[CAST:.*]] = addrspacecast ptr addrspace(5) %[[OMP_PRIVATE_ALLOC]] to ptr
+
+        omp.parallel private(@_QMmodFfailingEi_private_i32 %arg1 -> %arg3 : !llvm.ptr) {
+          %7 = llvm.load %arg2 : !llvm.ptr -> i32
+          omp.distribute {
+            omp.wsloop {
+              omp.loop_nest (%arg4) : i32 = (%6) to (%7) inclusive step (%6) {
+                llvm.store %arg4, %arg3 : i32, !llvm.ptr
+                llvm.call @_QMotherProutine(%arg3) {fastmathFlags = #llvm.fastmath<contract>} : (!llvm.ptr) -> ()
+                omp.yield
+              }
+            } {omp.composite}
+          } {omp.composite}
+          omp.terminator
+        } {omp.composite}
+        omp.terminator
+      }
+      omp.terminator
+    }
+    llvm.return
+  }
+}
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index f25ba4aa3c8dc..e4114d491fc85 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -162,14 +162,12 @@ llvm.func @test_omp_parallel_if_1(%arg0: i32) -> () {
 // CHECK: %[[I32_IF_COND_VAR_1:.*]] = sext i1 %[[IF_COND_VAR_1]] to i32
 // CHECK: call void @__kmpc_fork_call_if(ptr @[[SI_VAR_IF_1]], i32 0, ptr @[[OMP_OUTLINED_FN_IF_1:.*]], i32 %[[I32_IF_COND_VAR_1]], ptr null)
 // CHECK: br label %[[OUTLINED_EXIT_IF_1:.*]]
-// CHECK: [[OUTLINED_EXIT_IF_1]]:
-// CHECK: br label %[[RETURN_BLOCK_IF_1:.*]]
   omp.parallel if(%1) {
     omp.barrier
     omp.terminator
   }
 
-// CHECK: [[RETURN_BLOCK_IF_1]]:
+// CHECK: [[OUTLINED_EXIT_IF_1]]:
 // CHECK: ret void
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-outline-infinite-loop.mlir b/mlir/test/Target/LLVMIR/openmp-outline-infinite-loop.mlir
new file mode 100644
index 0000000000000..faccfc678adfe
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-outline-infinite-loop.mlir
@@ -0,0 +1,44 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// Test that trying to outline an infinite loop doesn't lead to an assertion
+// failure.
+
+llvm.func @parallel_infinite_loop() -> () {
+  omp.parallel {
+    llvm.br ^bb1
+  ^bb1:
+    llvm.br ^bb1
+  }
+  llvm.return
+}
+
+// CHECK-LABEL: define void @parallel_infinite_loop() {
+// CHECK:         %[[VAL_2:.*]] = call i32 @__kmpc_global_thread_num(ptr @1)
+// CHECK:         br label %[[VAL_3:.*]]
+// CHECK:       omp_parallel:
+// CHECK:         call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @1, i32 0, ptr @parallel_infinite_loop..omp_par)
+// CHECK:         unreachable
+// CHECK:       omp.region.cont:                                  ; No predecessors!
+// CHECK:         br label %[[VAL_4:.*]]
+// CHECK:       omp.par.pre_finalize:                             ; preds = %[[VAL_5:.*]]
+// CHECK:         br label %[[VAL_6:.*]]
+// CHECK:       omp.par.exit:                                     ; preds = %[[VAL_4]]
+// CHECK:         ret void
+// CHECK:       }
+
+// CHECK-LABEL: define internal void @parallel_infinite_loop..omp_par(
+// CHECK-SAME:      ptr noalias %[[TID_ADDR:.*]], ptr noalias %[[ZERO_ADDR:.*]])
+// CHECK:       omp.par.entry:
+// CHECK:         %[[VAL_7:.*]] = alloca i32, align 4
+// CHECK:         %[[VAL_8:.*]] = load i32, ptr %[[VAL_9:.*]], align 4
+// CHECK:         store i32 %[[VAL_8]], ptr %[[VAL_7]], align 4
+// CHECK:         %[[VAL_10:.*]] = load i32, ptr %[[VAL_7]], align 4
+// CHECK:         br label %[[VAL_11:.*]]
+// CHECK:       omp.region.after_alloca:                          ; preds = %[[VAL_12:.*]]
+// CHECK:         br label %[[VAL_13:.*]]
+// CHECK:       omp.par.region:                                   ; preds = %[[VAL_11]]
+// CHECK:         br label %[[VAL_14:.*]]
+// CHECK:       omp.par.region1:                                  ; preds = %[[VAL_13]]
+// CHECK:         br label %[[VAL_15:.*]]
+// CHECK:       omp.par.region2:                                  ; preds = %[[VAL_15]], %[[VAL_14]]
+// CHECK:         br label %[[VAL_15]]
diff --git a/mlir/test/Target/LLVMIR/openmp-parallel-reduction-multiblock.mlir b/mlir/test/Target/LLVMIR/openmp-parallel-reduction-multiblock.mlir
index d2e394b2cf6a8..887d2977e45cc 100644
--- a/mlir/test/Target/LLVMIR/openmp-parallel-reduction-multiblock.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-parallel-reduction-multiblock.mlir
@@ -40,9 +40,7 @@ llvm.func @missordered_blocks_(%arg0: !llvm.ptr {fir.bindc_name = "x"}, %arg1: !
 // CHECK:         store ptr %[[VAL_8:.*]], ptr %[[VAL_7]], align 8
 // CHECK:         call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @1, i32 1, ptr @missordered_blocks_..omp_par, ptr %[[VAL_0]])
 // CHECK:         br label %[[VAL_9:.*]]
-// CHECK:       omp.par.outlined.exit:                            ; preds = %[[VAL_4]]
-// CHECK:         br label %[[VAL_10:.*]]
-// CHECK:       omp.par.exit.split:                               ; preds = %[[VAL_9]]
+// CHECK:       omp.par.exit:                                     ; preds = %[[VAL_4]]
 // CHECK:         ret void
 // CHECK:       [[PAR_ENTRY:omp.par.entry]]:
 // CHECK:         %[[VAL_11:.*]] = getelementptr { ptr, ptr }, ptr %[[VAL_12:.*]], i32 0, i32 0
@@ -117,5 +115,5 @@ llvm.func @missordered_blocks_(%arg0: !llvm.ptr {fir.bindc_name = "x"}, %arg1: !
 // CHECK:         br label %[[VAL_38]]
 // CHECK:       omp.reduction.neutral1:                           ; preds = %[[VAL_25]]
 // CHECK:         br label %[[VAL_30]]
-// CHECK:       omp.par.outlined.exit.exitStub:                   ; preds = %[[VAL_53]]
+// CHECK:       omp.par.exit.exitStub:                            ; preds = %[[VAL_53]]
 // CHECK:         ret void
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-array-sections.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-array-sections.mlir
index d6ed3086969fb..b302b4b20edd5 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-array-sections.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-array-sections.mlir
@@ -219,5 +219,5 @@ llvm.func @sectionsreduction_(%arg0: !llvm.ptr {fir.bindc_name = "x"}) attribute
 // CHECK:       omp_section_loop.inc:                             ; preds = %[[VAL_69]]
 // CHECK:         %[[VAL_31]] = add nuw i32 %[[VAL_30]], 1
 // CHECK:         br label %[[VAL_28]]
-// CHECK:       omp.par.outlined.exit.exitStub:                   ; preds = %[[VAL_64]]
+// CHECK:       omp.par.exit.exitStub:                            ; preds = %[[VAL_64]]
 // CHECK:         ret void
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-init-arg.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-init-arg.mlir
index 8d329bd8ff817..a714ca68a1e95 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-init-arg.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-init-arg.mlir
@@ -46,9 +46,7 @@ module {
 // CHECK:         store ptr %[[VAL_2]], ptr %[[VAL_8]], align 8
 // CHECK:         call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @1, i32 1, ptr @_QFPreduce..omp_par, ptr %[[VAL_0]])
 // CHECK:         br label %[[VAL_9:.*]]
-// CHECK:       omp.par.outlined.exit:                            ; preds = %[[VAL_6]]
-// CHECK:         br label %[[VAL_10:.*]]
-// CHECK:       omp.par.exit.split:                               ; preds = %[[VAL_9]]
+// CHECK:       omp.par.exit:                                     ; preds = %[[VAL_6]]
 // CHECK:         ret void
 // CHECK:       [[PAR_ENTRY:omp.par.entry]]:
 // CHECK:         %[[VAL_11:.*]] = getelementptr { ptr, ptr }, ptr %[[VAL_12:.*]], i32 0, i32 0
@@ -99,7 +97,7 @@ module {
 // CHECK:         br label %[[VAL_38:.*]]
 // CHECK:       omp.par.pre_finalize:                             ; preds = %[[VAL_33]]
 // CHECK:         br label %[[VAL_39:.*]]
-// CHECK:       omp.par.outlined.exit.exitStub:                   ; preds = %[[VAL_38]]
+// CHECK:       omp.par.exit.exitStub:                            ; preds = %[[VAL_38]]
 // CHECK:         ret void
 // CHECK:         %[[VAL_40:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_41:.*]], i64 0, i64 0
 // CHECK:         %[[VAL_42:.*]] = load ptr, ptr %[[VAL_40]], align 8
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-sections.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-sections.mlir
index de3b997feb674..19da6f8517fcd 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-sections.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-sections.mlir
@@ -144,7 +144,7 @@ llvm.func @sections_(%arg0: !llvm.ptr {fir.bindc_name = "x"}) attributes {fir.in
 // CHECK:       omp_section_loop.inc:                             ; preds = %[[VAL_59]]
 // CHECK:         %[[VAL_35]] = add nuw i32 %[[VAL_34]], 1
 // CHECK:         br label %[[VAL_32]]
-// CHECK:       omp.par.outlined.exit.exitStub:                   ; preds = %[[VAL_54]]
+// CHECK:       omp.par.exit.exitStub:                            ; preds = %[[VAL_54]]
 // CHECK:         ret void
 // CHECK:         %[[VAL_70:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_71:.*]], i64 0, i64 0
 // CHECK:         %[[VAL_72:.*]] = load ptr, ptr %[[VAL_70]], align 8
diff --git a/mlir/test/Target/LLVMIR/openmp-target-has-device-addr.mlir b/mlir/test/Target/LLVMIR/openmp-target-has-device-addr.mlir
new file mode 100644
index 0000000000000..be592242ef6c5
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-target-has-device-addr.mlir
@@ -0,0 +1,20 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// integer :: x(:)
+// omp target has_device_addr(x)
+
+// CHECK-LABEL: ModuleID
+// CHECK: @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 48]
+// CHECK: @.offload_maptypes = private unnamed_addr constant [1 x i64] [i64 549]
+
+module attributes { llvm.target_triple = "x86_64-unknown-linux-gnu", omp.target_triples = ["amdgcn-amd-amdhsa"], omp.version = #omp.version<version = 52>} {
+  llvm.func @has_device_addr(%arg0: !llvm.ptr) attributes {target_cpu = "x86-64"} {
+    %0 = llvm.mlir.constant(1 : i32) : i32
+    %1 = llvm.alloca %0 x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> {alignment = 8 : i64} : (i32) -> !llvm.ptr
+    %41 = omp.map.info var_ptr(%1 : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>) map_clauses(always, implicit, to) capture(ByRef) -> !llvm.ptr {name = "x"}
+    omp.target has_device_addr(%41 -> %arg1 : !llvm.ptr) {
+      omp.terminator
+    }
+    llvm.return
+  }
+}
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index f907bb3f94a2a..7a4d304f2d2d5 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -308,17 +308,6 @@ llvm.func @target_device(%x : i32) {
 
 // -----
 
-llvm.func @target_has_device_addr(%x : !llvm.ptr) {
-  // expected-error@below {{not yet implemented: Unhandled clause has_device_addr in omp.target operation}}
-  // expected-error@below {{LLVM Translation failed for operation: omp.target}}
-  omp.target has_device_addr(%x : !llvm.ptr) {
-    omp.terminator
-  }
-  llvm.return
-}
-
-// -----
-
 omp.declare_reduction @add_f32 : f32
 init {
 ^bb0(%arg: f32):
diff --git a/mlir/test/Target/LLVMIR/rocdl.mlir b/mlir/test/Target/LLVMIR/rocdl.mlir
index 981ef4848535c..11f2faa2761ff 100644
--- a/mlir/test/Target/LLVMIR/rocdl.mlir
+++ b/mlir/test/Target/LLVMIR/rocdl.mlir
@@ -1032,22 +1032,29 @@ llvm.func @rocdl.raw.buffer.atomic.cmpswap(%rsrc : vector<4xi32>,
   llvm.return %val : i32
 }
 
-llvm.func @rocdl_8bit_floats(%source: i32, %stoch: i32) -> i32 {
+llvm.func @rocdl_8bit_floats(%source: i32, %source_half: f16, %source_bfloat: bf16, %source_packed: vector<2xf16>, %stoch: i32) -> i32 {
 // CHECK-LABEL: @rocdl_8bit_floats
 // CHECK: call float @llvm.amdgcn.cvt.f32.bf8(i32 %{{.+}}, i32 0)
 // CHECK: call float @llvm.amdgcn.cvt.scalef32.f32.bf8(i32 %{{.+}}, float 1.000000e+00, i32 0)
 // CHECK: call float @llvm.amdgcn.cvt.f32.fp8(i32 %{{.+}}, i32 0)
 // CHECK: call float @llvm.amdgcn.cvt.scalef32.f32.fp8(i32 %{{.+}}, float 1.000000e+00, i32 0)
+// CHECK: call <2 x half> @llvm.amdgcn.cvt.scalef32.pk.f16.bf8(i32 %{{.+}}, float 1.000000e+00, i1 false)
+// CHECK: call <2 x half> @llvm.amdgcn.cvt.scalef32.pk.f16.fp8(i32 %{{.+}}, float 1.000000e+00, i1 false)
+// CHECK: call <2 x half> @llvm.amdgcn.cvt.scalef32.f16.fp8(<2 x half> %{{.+}}, i32 %{{.+}}, float 1.000000e+00, i32 0, i1 false)
+// CHECK: call <2 x half> @llvm.amdgcn.cvt.scalef32.f16.bf8(<2 x half> %{{.+}}, i32 %{{.+}}, float 1.000000e+00, i32 0, i1 false)
 // CHECK: call i32 @llvm.amdgcn.cvt.pk.bf8.f32(float %{{.+}}, float %{{.+}}, i32 %{{.+}}, i1 false)
 // CHECK: call i32 @llvm.amdgcn.cvt.pk.fp8.f32(float %{{.+}}, float %{{.+}}, i32 %{{.+}}, i1 false)
 // CHECK: call i32 @llvm.amdgcn.cvt.sr.bf8.f32(float %{{.+}}, i32 %{{.+}}, i32 %{{.+}}, i32 2)
 // CHECK: call i32 @llvm.amdgcn.cvt.sr.fp8.f32(float %{{.+}}, i32 %{{.+}}, i32 %{{.+}}, i32 3)
 // CHECK: call i32 @llvm.amdgcn.cvt.scalef32.sr.fp8.f32(i32 %{{.+}}, float %{{.+}}, i32 %{{.+}}, float 1.000000e+00, i32 3)
+// CHECK: call i32 @llvm.amdgcn.cvt.scalef32.sr.fp8.f16(i32 %{{.+}}, half %{{.+}}, i32 %{{.+}}, float 1.000000e+00, i32 3)
+// CHECK: call i32 @llvm.amdgcn.cvt.scalef32.sr.fp8.bf16(i32 %{{.+}}, bfloat %{{.+}}, i32 %{{.+}}, float 1.000000e+00, i32 3)
 // CHECK: call i32 @llvm.amdgcn.cvt.sr.bf8.f32(float %{{.+}}, i32 %{{.+}}, i32 %{{.+}}, i32 3)
 // CHECK: call i32 @llvm.amdgcn.cvt.scalef32.sr.bf8.f32(i32 %{{.+}}, float %{{.+}}, i32 %{{.+}}, float 1.000000e+00, i32 3)
+// CHECK: call i32 @llvm.amdgcn.cvt.scalef32.sr.bf8.f16(i32 %{{.+}}, half %{{.+}}, i32 %{{.+}}, float 1.000000e+00, i32 3)
+// CHECK: call i32 @llvm.amdgcn.cvt.scalef32.sr.bf8.bf16(i32 %{{.+}}, bfloat %{{.+}}, i32 %{{.+}}, float 1.000000e+00, i32 3)
 // CHECK: call <2 x float> @llvm.amdgcn.cvt.scalef32.pk.f32.fp8(i32 %{{.+}}, float 1.000000e+00, i1 false)
 // CHECK: call <2 x float> @llvm.amdgcn.cvt.scalef32.pk.f32.bf8(i32 %{{.+}}, float 1.000000e+00, i1 false)
-
   %c0 = llvm.mlir.constant(0 : i32) : i32
   %c2 = llvm.mlir.constant(2 : i32) : i32
   %c3 = llvm.mlir.constant(3 : i32) : i32
@@ -1057,13 +1064,21 @@ llvm.func @rocdl_8bit_floats(%source: i32, %stoch: i32) -> i32 {
   %v1_scaled = rocdl.cvt.scalef32.f32.bf8 %source[%c0], %c4 : f32
   %v2 = rocdl.cvt.f32.fp8 %source[%c0] : f32
   %v2_scaled = rocdl.cvt.scalef32.f32.fp8 %source[%c0], %c4 : f32
+  %v3_scaled = rocdl.cvt.scalef32.pk.f16.bf8 %source[%false], %c4 : i32
+  %v4_scaled = rocdl.cvt.scalef32.pk.f16.fp8 %source[%false], %c4 : i32
+  %v5 = rocdl.cvt.scalef32.f16.fp8 %source[%false], %c4 -> %source_packed[%c0] : f16
+  %v6 = rocdl.cvt.scalef32.f16.bf8 %source[%false], %c4 -> %source_packed[%c0] : f16
   %source2 = rocdl.cvt.pk.bf8.f32 %v1, %v2 -> %source[%false] : i32
   %source3 = rocdl.cvt.pk.fp8.f32 %v1, %v2 -> %source2[%false] : i32
   %source4 = rocdl.cvt.sr.bf8.f32 %v1, %stoch -> %source3[%c2] : i32
   %source5 = rocdl.cvt.sr.fp8.f32 %v2, %stoch -> %source4[%c3] : i32
   %source5_scaled = rocdl.cvt.scalef32.sr.fp8.f32 %v2, %stoch, %c4 -> %source4[%c3] : i32
+  %source5_scaled_half = rocdl.cvt.scalef32.sr.fp8.f16 %source_half, %stoch, %c4 -> %source4[%c3] : i32
+  %source5_scaled_bfloat = rocdl.cvt.scalef32.sr.fp8.bf16 %source_bfloat, %stoch, %c4 -> %source4[%c3] : i32
   %source6 = rocdl.cvt.sr.bf8.f32 %v1, %stoch -> %source3[%c3] : i32
   %source6_scaled  = rocdl.cvt.scalef32.sr.bf8.f32 %v2, %stoch, %c4 -> %source3[%c3] : i32
+  %source6_scaled_half = rocdl.cvt.scalef32.sr.bf8.f16 %source_half, %stoch, %c4 -> %source3[%c3] : i32
+  %source6_scaled_bfloat = rocdl.cvt.scalef32.sr.bf8.bf16 %source_bfloat, %stoch, %c4 -> %source3[%c3] : i32
   %source7_scaled = rocdl.cvt.scalef32.pk.f32.fp8 %source[%false], %c4 : f32
   %source8_scaled = rocdl.cvt.scalef32.pk.f32.bf8 %source[%false], %c4 : f32
   llvm.return %source5 : i32
@@ -1080,6 +1095,21 @@ llvm.func @rocdl_8bit_packed_v2i16(%sourceA: f32, %sourceB: f32, %old: vector<2x
   llvm.return %source_scaled : vector<2xi16>
 }
 
+llvm.func @rocdl_v2f16_v2i16(%source: vector<2xf16>, %source2: vector<2xbf16>, %old: vector<2xi16>) -> vector<2xi16> {
+// CHECK-LABEL: @rocdl_v2f16_v2i16
+// CHECK: call <2 x i16> @llvm.amdgcn.cvt.scalef32.pk.fp8.f16(<2 x i16> %2, <2 x half> %0, float 1.000000e+00, i1 false)
+// CHECK: call <2 x i16> @llvm.amdgcn.cvt.scalef32.pk.fp8.bf16(<2 x i16> %2, <2 x bfloat> %1, float 1.000000e+00, i1 false)
+// CHECK: call <2 x i16> @llvm.amdgcn.cvt.scalef32.pk.bf8.f16(<2 x i16> %2, <2 x half> %0, float 1.000000e+00, i1 false)
+// CHECK: call <2 x i16> @llvm.amdgcn.cvt.scalef32.pk.bf8.bf16(<2 x i16> %2, <2 x bfloat> %1, float 1.000000e+00, i1 false)
+  %c0 = llvm.mlir.constant(1.0 : f32) : f32
+  %false = llvm.mlir.constant(false) : i1
+  %source_scaled = rocdl.cvt.scalef32.pk.fp8.f16 %source, %c0 -> %old[%false] : vector<2xi16>
+  %source2_scaled = rocdl.cvt.scalef32.pk.fp8.bf16 %source2, %c0 -> %old[%false] : vector<2xi16>
+  %source3_scaled = rocdl.cvt.scalef32.pk.bf8.f16 %source, %c0 -> %old[%false] : vector<2xi16>
+  %source4_scaled = rocdl.cvt.scalef32.pk.bf8.bf16 %source2, %c0 -> %old[%false] : vector<2xi16>
+  llvm.return %source_scaled : vector<2xi16>
+}
+
 llvm.func @rocdl_16bit_packed_floats(%sourceA: f32, %sourceB: f32) -> vector<2xf16> {
   // CHECK-LABEL: @rocdl_16bit_packed_floats
   // CHECK: call <2 x half> @llvm.amdgcn.cvt.pkrtz(float {{.*}}, float {{.*}})
diff --git a/mlir/test/Transforms/move-operation-deps.mlir b/mlir/test/Transforms/move-operation-deps.mlir
new file mode 100644
index 0000000000000..37637152938f6
--- /dev/null
+++ b/mlir/test/Transforms/move-operation-deps.mlir
@@ -0,0 +1,236 @@
+// RUN: mlir-opt --allow-unregistered-dialect --transform-interpreter --split-input-file --verify-diagnostics %s | FileCheck %s
+
+// Check simple move of dependent operation before insertion.
+func.func @simple_move() -> f32 {
+  %0 = "before"() : () -> (f32)
+  %1 = "moved_op"() : () -> (f32)
+  %2 = "foo"(%1) : (f32) -> (f32)
+  return %2 : f32
+}
+// CHECK-LABEL: func @simple_move()
+//       CHECK:   %[[MOVED:.+]] = "moved_op"
+//       CHECK:   %[[BEFORE:.+]] = "before"
+//       CHECK:   %[[FOO:.+]] = "foo"(%[[MOVED]])
+//       CHECK:   return %[[FOO]]
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0 : !transform.any_op {transform.readonly}) {
+    %op1 = transform.structured.match ops{["foo"]} in %arg0
+        : (!transform.any_op) -> !transform.any_op
+    %op2 = transform.structured.match ops{["before"]} in %arg0
+        : (!transform.any_op) -> !transform.any_op
+    transform.test.move_operand_deps %op1 before %op2
+        : !transform.any_op, !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// Move operands that are implicitly captured by the op
+func.func @move_region_dependencies() -> f32 {
+  %0 = "before"() : () -> (f32)
+  %1 = "moved_op"() : () -> (f32)
+  %2 = "foo"() ({
+    %3 = "inner_op"(%1) : (f32) -> (f32)
+    "yield"(%3) : (f32) -> ()
+  }) : () -> (f32)
+  return %2 : f32
+}
+// CHECK-LABEL: func @move_region_dependencies()
+//       CHECK:   %[[MOVED:.+]] = "moved_op"
+//       CHECK:   %[[BEFORE:.+]] = "before"
+//       CHECK:   %[[FOO:.+]] = "foo"
+//       CHECK:   return %[[FOO]]
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0 : !transform.any_op {transform.readonly}) {
+    %op1 = transform.structured.match ops{["foo"]} in %arg0
+        : (!transform.any_op) -> !transform.any_op
+    %op2 = transform.structured.match ops{["before"]} in %arg0
+        : (!transform.any_op) -> !transform.any_op
+    transform.test.move_operand_deps %op1 before %op2
+        : !transform.any_op, !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// Move operations in toplogical sort order
+func.func @move_in_topological_sort_order() -> f32 {
+  %0 = "before"() : () -> (f32)
+  %1 = "moved_op_1"() : () -> (f32)
+  %2 = "moved_op_2"() : () -> (f32)
+  %3 = "moved_op_3"(%1) : (f32) -> (f32)
+  %4 = "moved_op_4"(%1, %3) : (f32, f32) -> (f32)
+  %5 = "moved_op_5"(%2) : (f32) -> (f32)
+  %6 = "foo"(%4, %5) : (f32, f32) -> (f32)
+  return %6 : f32
+}
+// CHECK-LABEL: func @move_in_topological_sort_order()
+//       CHECK:   %[[MOVED_1:.+]] = "moved_op_1"
+//   CHECK-DAG:   %[[MOVED_2:.+]] = "moved_op_3"(%[[MOVED_1]])
+//   CHECK-DAG:   %[[MOVED_3:.+]] = "moved_op_4"(%[[MOVED_1]], %[[MOVED_2]])
+//   CHECK-DAG:   %[[MOVED_4:.+]] = "moved_op_2"
+//   CHECK-DAG:   %[[MOVED_5:.+]] = "moved_op_5"(%[[MOVED_4]])
+//       CHECK:   %[[BEFORE:.+]] = "before"
+//       CHECK:   %[[FOO:.+]] = "foo"(%[[MOVED_3]], %[[MOVED_5]])
+//       CHECK:   return %[[FOO]]
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0 : !transform.any_op {transform.readonly}) {
+    %op1 = transform.structured.match ops{["foo"]} in %arg0
+        : (!transform.any_op) -> !transform.any_op
+    %op2 = transform.structured.match ops{["before"]} in %arg0
+        : (!transform.any_op) -> !transform.any_op
+    transform.test.move_operand_deps %op1 before %op2
+        : !transform.any_op, !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+func.func @move_region_dependencies() -> f32 {
+  %0 = "before"() : () -> (f32)
+  %1 = "moved_op_1"() : () -> (f32)
+  %2 = "moved_op_2"() ({
+    "yield"(%1) : (f32) -> ()
+  }) : () -> (f32)
+  %3 = "foo"() ({
+    "yield"(%2) : (f32) -> ()
+  }) : () -> (f32)
+  return %3 : f32
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0 : !transform.any_op {transform.readonly}) {
+    %op1 = transform.structured.match ops{["foo"]} in %arg0
+        : (!transform.any_op) -> !transform.any_op
+    %op2 = transform.structured.match ops{["before"]} in %arg0
+        : (!transform.any_op) -> !transform.any_op
+    transform.test.move_operand_deps %op1 before %op2
+        : !transform.any_op, !transform.any_op
+    transform.yield
+  }
+}
+// CHECK-LABEL: func @move_region_dependencies()
+//       CHECK:   %[[MOVED_1:.+]] = "moved_op_1"
+//       CHECK:   %[[MOVED_2:.+]] = "moved_op_2"
+//       CHECK:     "yield"(%[[MOVED_1]])
+//       CHECK:   "before"
+//       CHECK:   %[[FOO:.+]] = "foo"
+//       CHECK:   return %[[FOO]]
+
+// -----
+
+// Current implementation omits following basic block argument when
+// computing slices. Verify that this gives expected result.
+func.func @ignore_basic_block_arguments() -> f32 {
+^bb0():
+  %8 = "test"() : () -> (f32)
+  return %8: f32
+^bb1(%bbArg : f32):
+  %0 = "before"() : () -> (f32)
+  %1 = "moved_op"() ({
+    "yield"(%bbArg) : (f32) -> ()
+  }) : () -> (f32)
+  %2 = "foo"() ({
+    "yield"(%1) : (f32) -> ()
+  }) : () -> (f32)
+  return %2 : f32
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0 : !transform.any_op {transform.readonly}) {
+    %op1 = transform.structured.match ops{["foo"]} in %arg0
+        : (!transform.any_op) -> !transform.any_op
+    %op2 = transform.structured.match ops{["before"]} in %arg0
+        : (!transform.any_op) -> !transform.any_op
+    transform.test.move_operand_deps %op1 before %op2
+        : !transform.any_op, !transform.any_op
+    transform.yield
+  }
+}
+// CHECK-LABEL: func @ignore_basic_block_arguments()
+//       CHECK:   %[[MOVED_1:.+]] = "moved_op"
+//       CHECK:   "before"
+//       CHECK:   %[[FOO:.+]] = "foo"
+//       CHECK:   return %[[FOO]]
+
+// -----
+
+// Fail when the "before" operation is part of the operation slice.
+func.func @do_not_move_slice() -> f32 {
+  %0 = "before"() : () -> (f32)
+  %1 = "moved_op"(%0) : (f32) -> (f32)
+  %2 = "foo"(%1) : (f32) -> (f32)
+  return %2 : f32
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0 : !transform.any_op {transform.readonly}) {
+    %op1 = transform.structured.match ops{["foo"]} in %arg0
+        : (!transform.any_op) -> !transform.any_op
+    %op2 = transform.structured.match ops{["before"]} in %arg0
+        : (!transform.any_op) -> !transform.any_op
+    // expected-remark@+1{{cannot move dependencies before operation in backward slice of op}}
+    transform.test.move_operand_deps %op1 before %op2
+        : !transform.any_op, !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// Fail when the "before" operation is part of the operation slice (computed
+// when looking through implicitly captured values).
+func.func @do_not_move_slice() -> f32 {
+  %0 = "before"() : () -> (f32)
+  %1 = "moved_op"() ({
+    "yield"(%0) : (f32) -> ()
+  }) : () -> (f32)
+  %2 = "foo"() ({
+    "yield"(%1) : (f32) -> ()
+  }) : () -> (f32)
+  return %2 : f32
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0 : !transform.any_op {transform.readonly}) {
+    %op1 = transform.structured.match ops{["foo"]} in %arg0
+        : (!transform.any_op) -> !transform.any_op
+    %op2 = transform.structured.match ops{["before"]} in %arg0
+        : (!transform.any_op) -> !transform.any_op
+    // expected-remark@+1{{cannot move dependencies before operation in backward slice of op}}
+    transform.test.move_operand_deps %op1 before %op2
+        : !transform.any_op, !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// Dont move ops when insertion point does not dominate the op
+func.func @do_not_move() -> f32 {
+  %1 = "moved_op"() : () -> (f32)
+  %2 = "foo"() ({
+    "yield"(%1) : (f32) -> ()
+  }) : () -> (f32)
+  %3 = "before"() : () -> f32
+  return %2 : f32
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0 : !transform.any_op {transform.readonly}) {
+    %op1 = transform.structured.match ops{["foo"]} in %arg0
+        : (!transform.any_op) -> !transform.any_op
+    %op2 = transform.structured.match ops{["before"]} in %arg0
+        : (!transform.any_op) -> !transform.any_op
+    // expected-remark@+1{{insertion point does not dominate op}}
+    transform.test.move_operand_deps %op1 before %op2
+        : !transform.any_op, !transform.any_op
+    transform.yield
+  }
+}
diff --git a/mlir/test/lib/Dialect/DLTI/TestDataLayoutQuery.cpp b/mlir/test/lib/Dialect/DLTI/TestDataLayoutQuery.cpp
index eae563b48c119..41a2fe5e4ee75 100644
--- a/mlir/test/lib/Dialect/DLTI/TestDataLayoutQuery.cpp
+++ b/mlir/test/lib/Dialect/DLTI/TestDataLayoutQuery.cpp
@@ -42,6 +42,7 @@ struct TestDataLayoutQuery
       uint64_t preferred = layout.getTypePreferredAlignment(op.getType());
       uint64_t index = layout.getTypeIndexBitwidth(op.getType()).value_or(0);
       Attribute endianness = layout.getEndianness();
+      Attribute defaultMemorySpace = layout.getDefaultMemorySpace();
       Attribute allocaMemorySpace = layout.getAllocaMemorySpace();
       Attribute manglingMode = layout.getManglingMode();
       Attribute programMemorySpace = layout.getProgramMemorySpace();
@@ -69,6 +70,10 @@ struct TestDataLayoutQuery
            builder.getNamedAttr("endianness", endianness == Attribute()
                                                   ? builder.getStringAttr("")
                                                   : endianness),
+           builder.getNamedAttr("default_memory_space",
+                                defaultMemorySpace == Attribute()
+                                    ? builder.getUI32IntegerAttr(0)
+                                    : defaultMemorySpace),
            builder.getNamedAttr("alloca_memory_space",
                                 allocaMemorySpace == Attribute()
                                     ? builder.getUI32IntegerAttr(0)
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index cdc1237ec8c5a..cc0fe924acf75 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -2504,6 +2504,13 @@ def TableGenBuildOp6 : TEST_Op<"tblgen_build_6", [AttrSizedOperandSegments]> {
   let results = (outs F32:$result);
 }
 
+// An inherent attribute. Test collective builders, both those that take properties as
+// properties structs and those that take an attribute dictionary.
+def TableGenBuildOp7 : TEST_Op<"tblgen_build_7", []> {
+  let arguments = (ins BoolAttr:$attr0);
+  let results = (outs);
+}
+
 //===----------------------------------------------------------------------===//
 // Test BufferPlacement
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/lib/Dialect/Test/TestTypes.cpp b/mlir/test/lib/Dialect/Test/TestTypes.cpp
index 0c237440834ef..5c784dcee6e15 100644
--- a/mlir/test/lib/Dialect/Test/TestTypes.cpp
+++ b/mlir/test/lib/Dialect/Test/TestTypes.cpp
@@ -284,7 +284,9 @@ TestTypeWithLayoutType::getIndexBitwidth(const DataLayout &dataLayout,
 }
 
 bool TestTypeWithLayoutType::areCompatible(
-    DataLayoutEntryListRef oldLayout, DataLayoutEntryListRef newLayout) const {
+    DataLayoutEntryListRef oldLayout, DataLayoutEntryListRef newLayout,
+    DataLayoutSpecInterface newSpec,
+    const DataLayoutIdentifiedEntryMap &map) const {
   unsigned old = extractKind(oldLayout, "alignment");
   return old == 1 || extractKind(newLayout, "alignment") <= old;
 }
diff --git a/mlir/test/lib/Dialect/Tosa/TosaTestPasses.cpp b/mlir/test/lib/Dialect/Tosa/TosaTestPasses.cpp
index 1297596ef193a..3389a1ccfe98f 100644
--- a/mlir/test/lib/Dialect/Tosa/TosaTestPasses.cpp
+++ b/mlir/test/lib/Dialect/Tosa/TosaTestPasses.cpp
@@ -155,7 +155,15 @@ ConvertTosaConv2DOp::matchAndRewrite(Operation *op,
   double inputScale = inputQType.getScale();
   double weightScale = weightQType.getScale();
   double outputScale = outputQType.getScale();
-  int64_t outputZp = outputQType.getZeroPoint();
+  int64_t outputZpVal = outputQType.getZeroPoint();
+
+  auto inputZp =
+      createZeroPointTensor(rewriter, op->getLoc(), newTosaConv2DOpType, 0);
+  auto outputZp = createZeroPointTensor(
+      rewriter, op->getLoc(), tosaConv2DOp.getOutput().getType(), outputZpVal);
+
+  if (!inputZp || !outputZp)
+    return failure();
 
   double opTensorScale = (inputScale * weightScale) / outputScale;
 
@@ -166,18 +174,21 @@ ConvertTosaConv2DOp::matchAndRewrite(Operation *op,
   if (!computeMultiplierAndShift(opTensorScale, multiplier, shift, 32))
     return failure();
 
-  bool input_unsigned =
+  bool inputUnsigned =
       newTosaConv2DOp.getResult().getType().isUnsignedInteger();
-  bool output_unsigned = outputType.isUnsignedInteger();
+  bool outputUnsigned = outputType.isUnsignedInteger();
 
   auto newTosaRescaleOp = rewriter.create<tosa::RescaleOp>(
       op->getLoc(), outputType, newTosaConv2DOp.getResult(),
-      rewriter.getI32IntegerAttr(0), rewriter.getI32IntegerAttr(outputZp),
-      rewriter.getDenseI32ArrayAttr({multiplier}),
-      rewriter.getDenseI8ArrayAttr({static_cast<int8_t>(shift)}),
-      rewriter.getBoolAttr(true), rewriter.getBoolAttr(true),
-      rewriter.getBoolAttr(false), rewriter.getBoolAttr(input_unsigned),
-      rewriter.getBoolAttr(output_unsigned));
+      getConstTensorInt<int32_t>(rewriter, op->getLoc(), {multiplier}),
+      getConstTensorInt<int8_t>(rewriter, op->getLoc(),
+                                {static_cast<int8_t>(shift)}),
+      inputZp.value(), outputZp.value(),
+      /* scale32 = */ rewriter.getBoolAttr(true),
+      /* double_round = */ rewriter.getStringAttr("DOUBLE_ROUND"),
+      /* per_channel = */ rewriter.getBoolAttr(false),
+      rewriter.getBoolAttr(inputUnsigned),
+      rewriter.getBoolAttr(outputUnsigned));
 
   rewriter.replaceOp(op, {newTosaRescaleOp.getResult()});
   return success();
diff --git a/mlir/test/lib/Transforms/CMakeLists.txt b/mlir/test/lib/Transforms/CMakeLists.txt
index 1b9b9bffa5279..c053fd4b20473 100644
--- a/mlir/test/lib/Transforms/CMakeLists.txt
+++ b/mlir/test/lib/Transforms/CMakeLists.txt
@@ -1,3 +1,8 @@
+set(LLVM_TARGET_DEFINITIONS TestTransformsOps.td)
+mlir_tablegen(TestTransformsOps.h.inc -gen-op-decls)
+mlir_tablegen(TestTransformsOps.cpp.inc -gen-op-defs)
+add_public_tablegen_target(MLIRTestTransformsOpsIncGen)
+
 set(LLVM_OPTIONAL_SOURCES 
   TestDialectConversion.cpp)
 set(MLIRTestTransformsPDLDep)
@@ -25,6 +30,7 @@ add_mlir_library(MLIRTestTransforms
   TestControlFlowSink.cpp
   TestInlining.cpp
   TestMakeIsolatedFromAbove.cpp
+  TestTransformsOps.cpp
   ${MLIRTestTransformsPDLSrc}
 
   EXCLUDE_FROM_LIBMLIR
@@ -34,6 +40,7 @@ add_mlir_library(MLIRTestTransforms
 
   DEPENDS
   ${MLIRTestTransformsPDLDep}
+  MLIRTestTransformsOpsIncGen
 
   LINK_LIBS PUBLIC
   MLIRTestDialect
@@ -43,6 +50,7 @@ mlir_target_link_libraries(MLIRTestTransforms PUBLIC
   MLIRFuncDialect
   MLIRInferIntRangeInterface
   MLIRTransforms
+  MLIRTransformDialect
   )
 
 target_include_directories(MLIRTestTransforms
diff --git a/mlir/test/lib/Transforms/TestTransformsOps.cpp b/mlir/test/lib/Transforms/TestTransformsOps.cpp
new file mode 100644
index 0000000000000..aaa566d9938a3
--- /dev/null
+++ b/mlir/test/lib/Transforms/TestTransformsOps.cpp
@@ -0,0 +1,66 @@
+//===- TestTransformsOps.cpp - Test Transforms ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines transform dialect operations for testing MLIR
+// transformations
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Transform/IR/TransformAttrs.h"
+#include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
+#include "mlir/Transforms/RegionUtils.h"
+
+#define GET_OP_CLASSES
+#include "TestTransformsOps.h.inc"
+
+using namespace mlir;
+using namespace mlir::transform;
+
+#define GET_OP_CLASSES
+#include "TestTransformsOps.cpp.inc"
+
+DiagnosedSilenceableFailure
+transform::TestMoveOperandDeps::apply(TransformRewriter &rewriter,
+                                      TransformResults &TransformResults,
+                                      TransformState &state) {
+  Operation *op = *state.getPayloadOps(getOp()).begin();
+  Operation *moveBefore = *state.getPayloadOps(getInsertionPoint()).begin();
+  if (failed(moveOperationDependencies(rewriter, op, moveBefore))) {
+    auto listener = cast<ErrorCheckingTrackingListener>(rewriter.getListener());
+    std::string errorMsg = listener->getLatestMatchFailureMessage();
+    (void)emitRemark(errorMsg);
+  }
+  return DiagnosedSilenceableFailure::success();
+}
+
+namespace {
+
+class TestTransformsDialectExtension
+    : public transform::TransformDialectExtension<
+          TestTransformsDialectExtension> {
+public:
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestTransformsDialectExtension)
+
+  using Base::Base;
+
+  void init() {
+    registerTransformOps<
+#define GET_OP_LIST
+#include "TestTransformsOps.cpp.inc"
+        >();
+  }
+};
+} // namespace
+
+namespace test {
+void registerTestTransformsTransformDialectExtension(
+    DialectRegistry &registry) {
+  registry.addExtensions<TestTransformsDialectExtension>();
+}
+} // namespace test
diff --git a/mlir/test/lib/Transforms/TestTransformsOps.td b/mlir/test/lib/Transforms/TestTransformsOps.td
new file mode 100644
index 0000000000000..f514702cef5bc
--- /dev/null
+++ b/mlir/test/lib/Transforms/TestTransformsOps.td
@@ -0,0 +1,41 @@
+//===- TestTransformOps.td ---------------------------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TEST_TRANSFORM_OPS
+#define TEST_TRANSFORM_OPS
+
+include "mlir/Dialect/Transform/IR/TransformDialect.td"
+include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td"
+include "mlir/Dialect/Transform/IR/TransformTypes.td"
+include "mlir/Interfaces/SideEffectInterfaces.td"
+include "mlir/IR/OpBase.td"
+
+/// Transform dialect operations for testing transformations in MLIR
+
+def TestMoveOperandDeps :
+    Op<Transform_Dialect, "test.move_operand_deps",
+        [FunctionalStyleTransformOpTrait, MemoryEffectsOpInterface,
+         DeclareOpInterfaceMethods<TransformOpInterface>,
+         ReportTrackingListenerFailuresOpTrait]> {
+  let description = [{
+    Moves all dependencies of on operation before another operation.
+  }];
+
+  let arguments =
+    (ins TransformHandleTypeInterface:$op,
+         TransformHandleTypeInterface:$insertion_point);
+  
+  let results = (outs);
+
+  let assemblyFormat = [{
+    $op `before` $insertion_point attr-dict
+    `:` type($op) `,` type($insertion_point)
+  }];
+}
+
+#endif // TEST_TRANSFORM_OPS
diff --git a/mlir/test/lib/Transforms/lit.local.cfg b/mlir/test/lib/Transforms/lit.local.cfg
index 8ffccee1d6d79..7f4d25f1ba025 100644
--- a/mlir/test/lib/Transforms/lit.local.cfg
+++ b/mlir/test/lib/Transforms/lit.local.cfg
@@ -1 +1,2 @@
 config.suffixes.remove(".pdll")
+config.suffixes.remove(".td")
diff --git a/mlir/test/mlir-tblgen/op-attribute.td b/mlir/test/mlir-tblgen/op-attribute.td
index 55382a5bd3f8c..549830e06042f 100644
--- a/mlir/test/mlir-tblgen/op-attribute.td
+++ b/mlir/test/mlir-tblgen/op-attribute.td
@@ -165,6 +165,12 @@ def AOp : NS_Op<"a_op", []> {
 // DEF:        ::llvm::ArrayRef<::mlir::NamedAttribute> attributes
 // DEF:      odsState.addAttributes(attributes);
 
+// DEF:      void AOp::build(
+// DEF-SAME:   const Properties &properties,
+// DEF-SAME:   ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes
+// DEF:      odsState.useProperties(const_cast<Properties&>(properties));
+// DEF:      odsState.addAttributes(discardableAttributes);
+
 // DEF:      void AOp::populateDefaultProperties
 
 // Test the above but with prefix.
@@ -279,6 +285,12 @@ def AgetOp : Op<Test2_Dialect, "a_get_op", []> {
 // DEF:        ::llvm::ArrayRef<::mlir::NamedAttribute> attributes
 // DEF:      odsState.addAttributes(attributes);
 
+// DEF:      void AgetOp::build(
+// DEF-SAME:   const Properties &properties
+// DEF-SAME:   ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes
+// DEF:      odsState.useProperties(const_cast<Properties&>(properties));
+// DEF:      odsState.addAttributes(discardableAttributes);
+
 // Test the above but using properties.
 def ApropOp : NS_Op<"a_prop_op", []> {
   let arguments = (ins
diff --git a/mlir/test/mlir-tblgen/op-decl-and-defs.td b/mlir/test/mlir-tblgen/op-decl-and-defs.td
index ee800a2952bac..a3dce9b31f8d2 100644
--- a/mlir/test/mlir-tblgen/op-decl-and-defs.td
+++ b/mlir/test/mlir-tblgen/op-decl-and-defs.td
@@ -119,6 +119,7 @@ def NS_AOp : NS_Op<"a_op", [IsolatedFromAbove, IsolatedFromAbove]> {
 // CHECK:   static void build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type r, ::mlir::TypeRange s, ::mlir::Value a, ::mlir::ValueRange b, uint32_t attr1, /*optional*/::mlir::FloatAttr some_attr2, unsigned someRegionsCount)
 // CHECK:   static void build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value a, ::mlir::ValueRange b, uint32_t attr1, /*optional*/::mlir::FloatAttr some_attr2, unsigned someRegionsCount);
 // CHECK:   static void build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes, unsigned numRegions)
+// CHECK:   static void build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes, unsigned numRegions)
 // CHECK:   static ::mlir::ParseResult parse(::mlir::OpAsmParser &parser, ::mlir::OperationState &result);
 // CHECK:   void print(::mlir::OpAsmPrinter &p);
 // CHECK:   ::llvm::LogicalResult verifyInvariants();
@@ -231,6 +232,7 @@ def NS_HCollectiveParamsOp : NS_Op<"op_collective_params", []> {
 // CHECK: static void build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::Type b, ::mlir::Value a);
 // CHECK: static void build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value a);
 // CHECK: static void build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes = {})
+// CHECK: static void build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes = {})
 
 // Check suppression of "separate arg, separate result" build method for an op
 // with single variadic arg and single variadic result (since it will be
@@ -281,6 +283,8 @@ def NS_IOp : NS_Op<"op_with_same_operands_and_result_types_trait", [SameOperands
 // CHECK: static void build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value a, ::mlir::Value b);
 // CHECK: static void build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes = {});
 // CHECK: static void build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes = {});
+// CHECK: static void build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes = {});
+// CHECK: static void build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes = {});
 
 // Check default value of `attributes` for the `genInferredTypeCollectiveParamBuilder` builder
 def NS_JOp : NS_Op<"op_with_InferTypeOpInterface_interface", [DeclareOpInterfaceMethods<InferTypeOpInterface>]> {
@@ -293,6 +297,8 @@ def NS_JOp : NS_Op<"op_with_InferTypeOpInterface_interface", [DeclareOpInterface
 // CHECK: static void build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value a, ::mlir::Value b);
 // CHECK: static void build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes = {});
 // CHECK: static void build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes = {});
+// CHECK: static void build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes = {});
+// CHECK: static void build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes = {});
 
 // Test usage of TraitList getting flattened during emission.
 def NS_KOp : NS_Op<"k_op", [IsolatedFromAbove,
@@ -329,6 +335,8 @@ def NS_LOp : NS_Op<"op_with_same_operands_and_result_types_unwrapped_attr", [Sam
 // CHECK: static void build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::Value a, ::mlir::Value b, uint32_t attr1);
 // CHECK: static void build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes = {});
 // CHECK: static void build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes = {});
+// CHECK: static void build(::mlir::OpBuilder &, ::mlir::OperationState &odsState, ::mlir::TypeRange resultTypes, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes = {});
+// CHECK: static void build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes = {});
 
 def NS_MOp : NS_Op<"op_with_single_result_and_fold_adaptor_fold", []> {
   let results = (outs AnyType:$res);
diff --git a/mlir/test/mlir-tblgen/op-result.td b/mlir/test/mlir-tblgen/op-result.td
index 212d3189cf571..a4f7af6dbcf1c 100644
--- a/mlir/test/mlir-tblgen/op-result.td
+++ b/mlir/test/mlir-tblgen/op-result.td
@@ -57,7 +57,9 @@ def OpD : NS_Op<"type_attr_as_result_type", [FirstAttrDerivedResultType]> {
 
 // CHECK-LABEL: OpD definitions
 // CHECK: void OpD::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes)
-// CHECK: odsState.addTypes({::llvm::cast<::mlir::TypeAttr>(attr.getValue()).getValue()});
+// CHECK: odsState.addTypes({::llvm::cast<::mlir::TypeAttr>(typeAttr).getValue()});
+// CHECK: void OpD::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes)
+// CHECK: odsState.addTypes({::llvm::cast<::mlir::TypeAttr>(typeAttr).getValue()});
 
 def OpE : NS_Op<"value_attr_as_result_type", [FirstAttrDerivedResultType]> {
   let arguments = (ins I32:$x, F32Attr:$attr);
@@ -66,7 +68,10 @@ def OpE : NS_Op<"value_attr_as_result_type", [FirstAttrDerivedResultType]> {
 
 // CHECK-LABEL: OpE definitions
 // CHECK: void OpE::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes)
-// CHECK: odsState.addTypes({::llvm::cast<::mlir::TypedAttr>(attr.getValue()).getType()});
+// CHECK: odsState.addTypes({::llvm::cast<::mlir::TypedAttr>(typeAttr).getType()});
+// CHECK: void OpE::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes)
+// CHECK: ::mlir::Attribute typeAttr = properties.getAttr();
+// CHECK: odsState.addTypes({::llvm::cast<::mlir::TypedAttr>(typeAttr).getType()});
 
 def OpF : NS_Op<"one_variadic_result_op", []> {
   let results = (outs Variadic<I32>:$x);
@@ -118,6 +123,8 @@ def OpK : NS_Op<"only_input_is_variadic_with_same_value_type_op", [SameOperandsA
 
 // CHECK-LABEL: OpK::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, ::llvm::ArrayRef<::mlir::NamedAttribute> attributes)
 // CHECK: odsState.addTypes({operands[0].getType()});
+// CHECK-LABEL: OpK::build(::mlir::OpBuilder &odsBuilder, ::mlir::OperationState &odsState, ::mlir::ValueRange operands, const Properties &properties, ::llvm::ArrayRef<::mlir::NamedAttribute> discardableAttributes)
+// CHECK: odsState.addTypes({operands[0].getType()});
 
 // Test with inferred shapes and interleaved with operands/attributes.
 //
diff --git a/mlir/test/mlir-tblgen/rewriter-attributes-properties.td b/mlir/test/mlir-tblgen/rewriter-attributes-properties.td
new file mode 100644
index 0000000000000..fc36a51789ec2
--- /dev/null
+++ b/mlir/test/mlir-tblgen/rewriter-attributes-properties.td
@@ -0,0 +1,47 @@
+// RUN: mlir-tblgen -gen-rewriters -I %S/../../include %s | FileCheck %s
+
+include "mlir/IR/OpBase.td"
+include "mlir/IR/PatternBase.td"
+
+def Test_Dialect : Dialect {
+  let name = "test";
+}
+class NS_Op<string mnemonic, list<Trait> traits> :
+    Op<Test_Dialect, mnemonic, traits>;
+
+def AOp : NS_Op<"a_op", []> {
+  let arguments = (ins
+    I32:$x,
+    I32Attr:$y
+  );
+
+  let results = (outs I32:$z);
+}
+
+def BOp : NS_Op<"b_op", []> {
+  let arguments = (ins
+    I32Attr:$y
+  );
+
+  let results = (outs I32:$z);
+}
+
+def test1 : Pat<(AOp (BOp:$x $y), $_), (AOp $x, $y)>;
+// CHECK-LABEL: struct test1
+// CHECK: ::llvm::LogicalResult matchAndRewrite
+// CHECK-DAG: ::mlir::IntegerAttr y;
+// CHECK-DAG: test::BOp x;
+// CHECK-DAG: ::llvm::SmallVector<::mlir::Operation *, 4> tblgen_ops;
+// CHECK: tblgen_ops.push_back(op0);
+// CHECK: x = castedOp1;
+// CHECK: tblgen_attr = castedOp1.getProperties().getY();
+// CHECK: if (!(tblgen_attr))
+// CHECK: y = tblgen_attr;
+// CHECK: tblgen_ops.push_back(op1);
+
+// CHECK: test::AOp tblgen_AOp_0;
+// CHECK: ::llvm::SmallVector<::mlir::Value, 4> tblgen_values;
+// CHECK: test::AOp::Properties tblgen_props;
+// CHECK: tblgen_values.push_back((*x.getODSResults(0).begin()));
+// CHECK: tblgen_props.y = ::llvm::dyn_cast_if_present<decltype(tblgen_props.y)>(y);
+// CHECK: tblgen_AOp_0 = rewriter.create<test::AOp>(odsLoc, tblgen_types, tblgen_values, tblgen_props);
diff --git a/mlir/test/python/ir/context_lifecycle.py b/mlir/test/python/ir/context_lifecycle.py
index c20270999425e..230db8277c8e7 100644
--- a/mlir/test/python/ir/context_lifecycle.py
+++ b/mlir/test/python/ir/context_lifecycle.py
@@ -47,3 +47,26 @@
 assert '"mlir.ir.Context._CAPIPtr"' in repr(c4_capsule)
 c5 = mlir.ir.Context._CAPICreate(c4_capsule)
 assert c4 is c5
+c4 = None
+c5 = None
+gc.collect()
+
+# Create a global threadpool and use it in two contexts
+tp = mlir.ir.ThreadPool()
+assert tp.get_max_concurrency() > 0
+c5 = mlir.ir.Context()
+c5.set_thread_pool(tp)
+assert c5.get_num_threads() == tp.get_max_concurrency()
+assert c5._mlir_thread_pool_ptr() == tp._mlir_thread_pool_ptr()
+c6 = mlir.ir.Context()
+c6.set_thread_pool(tp)
+assert c6.get_num_threads() == tp.get_max_concurrency()
+assert c6._mlir_thread_pool_ptr() == tp._mlir_thread_pool_ptr()
+c7 = mlir.ir.Context(thread_pool=tp)
+assert c7.get_num_threads() == tp.get_max_concurrency()
+assert c7._mlir_thread_pool_ptr() == tp._mlir_thread_pool_ptr()
+assert mlir.ir.Context._get_live_count() == 3
+c5 = None
+c6 = None
+c7 = None
+gc.collect()
diff --git a/mlir/test/python/ir/diagnostic_handler.py b/mlir/test/python/ir/diagnostic_handler.py
index d516cda819897..6d273e5092e42 100644
--- a/mlir/test/python/ir/diagnostic_handler.py
+++ b/mlir/test/python/ir/diagnostic_handler.py
@@ -2,6 +2,9 @@
 
 import gc
 from mlir.ir import *
+from mlir._mlir_libs._mlirPythonTestNanobind import (
+    test_diagnostics_with_errors_and_notes,
+)
 
 
 def run(f):
@@ -222,3 +225,16 @@ def callback2(d):
     # CHECK: CALLBACK2: foobar
     # CHECK: CALLBACK1: foobar
     loc.emit_error("foobar")
+
+
+# CHECK-LABEL: TEST: testBuiltInDiagnosticsHandler
+@run
+def testBuiltInDiagnosticsHandler():
+    ctx = Context()
+
+    try:
+        test_diagnostics_with_errors_and_notes(ctx)
+    except ValueError as e:
+        # CHECK: created error
+        # CHECK: attached note
+        print(e)
diff --git a/mlir/test/python/ir/location.py b/mlir/test/python/ir/location.py
index 59d8a89e770dd..3e54dc922cd67 100644
--- a/mlir/test/python/ir/location.py
+++ b/mlir/test/python/ir/location.py
@@ -43,22 +43,64 @@ def testLocationAttr():
 
 run(testLocationAttr)
 
+
 # CHECK-LABEL: TEST: testFileLineCol
 def testFileLineCol():
     with Context() as ctx:
-        loc = Location.file("foo.txt", 123, 56)
-        range = Location.file("foo.txt", 123, 56, 123, 100)
+        loc = Location.file("foo1.txt", 123, 56)
+        range = Location.file("foo2.txt", 123, 56, 124, 100)
+
     ctx = None
     gc.collect()
-    # CHECK: file str: loc("foo.txt":123:56)
+
+    # CHECK: file str: loc("foo1.txt":123:56)
     print("file str:", str(loc))
-    # CHECK: file repr: loc("foo.txt":123:56)
+    # CHECK: file repr: loc("foo1.txt":123:56)
     print("file repr:", repr(loc))
-    # CHECK: file range str: loc("foo.txt":123:56 to :100)
+    # CHECK: file range str: loc("foo2.txt":123:56 to 124:100)
     print("file range str:", str(range))
-    # CHECK: file range repr: loc("foo.txt":123:56 to :100)
+    # CHECK: file range repr: loc("foo2.txt":123:56 to 124:100)
     print("file range repr:", repr(range))
 
+    assert loc.is_a_file()
+    assert not loc.is_a_name()
+    assert not loc.is_a_callsite()
+    assert not loc.is_a_fused()
+
+    # CHECK: file filename: foo1.txt
+    print("file filename:", loc.filename)
+    # CHECK: file start_line: 123
+    print("file start_line:", loc.start_line)
+    # CHECK: file start_col: 56
+    print("file start_col:", loc.start_col)
+    # CHECK: file end_line: 123
+    print("file end_line:", loc.end_line)
+    # CHECK: file end_col: 56
+    print("file end_col:", loc.end_col)
+
+    assert range.is_a_file()
+    # CHECK: file filename: foo2.txt
+    print("file filename:", range.filename)
+    # CHECK: file start_line: 123
+    print("file start_line:", range.start_line)
+    # CHECK: file start_col: 56
+    print("file start_col:", range.start_col)
+    # CHECK: file end_line: 124
+    print("file end_line:", range.end_line)
+    # CHECK: file end_col: 100
+    print("file end_col:", range.end_col)
+
+    with Context() as ctx:
+        ctx.allow_unregistered_dialects = True
+        loc = Location.file("foo3.txt", 127, 61)
+        with loc:
+            i32 = IntegerType.get_signless(32)
+            module = Module.create()
+            with InsertionPoint(module.body):
+                new_value = Operation.create("custom.op1", results=[i32]).result
+                # CHECK: new_value location: loc("foo3.txt":127:61)
+                print("new_value location: ", new_value.location)
+
 
 run(testFileLineCol)
 
@@ -67,17 +109,31 @@ def testFileLineCol():
 def testName():
     with Context() as ctx:
         loc = Location.name("nombre")
-        locWithChildLoc = Location.name("naam", loc)
+        loc_with_child_loc = Location.name("naam", loc)
+
     ctx = None
     gc.collect()
-    # CHECK: file str: loc("nombre")
-    print("file str:", str(loc))
-    # CHECK: file repr: loc("nombre")
-    print("file repr:", repr(loc))
-    # CHECK: file str: loc("naam"("nombre"))
-    print("file str:", str(locWithChildLoc))
-    # CHECK: file repr: loc("naam"("nombre"))
-    print("file repr:", repr(locWithChildLoc))
+
+    # CHECK: name str: loc("nombre")
+    print("name str:", str(loc))
+    # CHECK: name repr: loc("nombre")
+    print("name repr:", repr(loc))
+    # CHECK: name str: loc("naam"("nombre"))
+    print("name str:", str(loc_with_child_loc))
+    # CHECK: name repr: loc("naam"("nombre"))
+    print("name repr:", repr(loc_with_child_loc))
+
+    assert loc.is_a_name()
+    # CHECK: name name_str: nombre
+    print("name name_str:", loc.name_str)
+    # CHECK: name child_loc: loc(unknown)
+    print("name child_loc:", loc.child_loc)
+
+    assert loc_with_child_loc.is_a_name()
+    # CHECK: name name_str: naam
+    print("name name_str:", loc_with_child_loc.name_str)
+    # CHECK: name child_loc_with_child_loc: loc("nombre")
+    print("name child_loc_with_child_loc:", loc_with_child_loc.child_loc)
 
 
 run(testName)
@@ -91,10 +147,17 @@ def testCallSite():
             [Location.file("util.foo", 379, 21), Location.file("main.foo", 100, 63)],
         )
     ctx = None
-    # CHECK: file str: loc(callsite("foo.text":123:45 at callsite("util.foo":379:21 at "main.foo":100:63))
-    print("file str:", str(loc))
-    # CHECK: file repr: loc(callsite("foo.text":123:45 at callsite("util.foo":379:21 at "main.foo":100:63))
-    print("file repr:", repr(loc))
+    # CHECK: callsite str: loc(callsite("foo.text":123:45 at callsite("util.foo":379:21 at "main.foo":100:63))
+    print("callsite str:", str(loc))
+    # CHECK: callsite repr: loc(callsite("foo.text":123:45 at callsite("util.foo":379:21 at "main.foo":100:63))
+    print("callsite repr:", repr(loc))
+
+    assert loc.is_a_callsite()
+
+    # CHECK: callsite callee: loc("foo.text":123:45)
+    print("callsite callee:", loc.callee)
+    # CHECK: callsite caller: loc(callsite("util.foo":379:21 at "main.foo":100:63))
+    print("callsite caller:", loc.caller)
 
 
 run(testCallSite)
@@ -112,31 +175,56 @@ def testFused():
         loc_empty = Location.fused([])
         loc_empty_attr = Location.fused([], attr)
         loc_single_attr = Location.fused([Location.name("apple")], attr)
+
     ctx = None
-    # CHECK: file str: loc("apple")
-    print("file str:", str(loc_single))
-    # CHECK: file repr: loc("apple")
-    print("file repr:", repr(loc_single))
-    # CHECK: file str: loc(fused["apple", "banana"])
-    print("file str:", str(loc))
-    # CHECK: file repr: loc(fused["apple", "banana"])
-    print("file repr:", repr(loc))
-    # CHECK: file str: loc(fused<"sauteed">["carrot", "potatoes"])
-    print("file str:", str(loc_attr))
-    # CHECK: file repr: loc(fused<"sauteed">["carrot", "potatoes"])
-    print("file repr:", repr(loc_attr))
-    # CHECK: file str: loc(unknown)
-    print("file str:", str(loc_empty))
-    # CHECK: file repr: loc(unknown)
-    print("file repr:", repr(loc_empty))
-    # CHECK: file str: loc(fused<"sauteed">[unknown])
-    print("file str:", str(loc_empty_attr))
-    # CHECK: file repr: loc(fused<"sauteed">[unknown])
-    print("file repr:", repr(loc_empty_attr))
-    # CHECK: file str: loc(fused<"sauteed">["apple"])
-    print("file str:", str(loc_single_attr))
-    # CHECK: file repr: loc(fused<"sauteed">["apple"])
-    print("file repr:", repr(loc_single_attr))
+
+    assert not loc_single.is_a_fused()
+    # CHECK: fused str: loc("apple")
+    print("fused str:", str(loc_single))
+    # CHECK: fused repr: loc("apple")
+    print("fused repr:", repr(loc_single))
+    # # CHECK: fused locations: []
+    print("fused locations:", loc_single.locations)
+
+    assert loc.is_a_fused()
+    # CHECK: fused str: loc(fused["apple", "banana"])
+    print("fused str:", str(loc))
+    # CHECK: fused repr: loc(fused["apple", "banana"])
+    print("fused repr:", repr(loc))
+    # CHECK: fused locations: [loc("apple"), loc("banana")]
+    print("fused locations:", loc.locations)
+
+    assert loc_attr.is_a_fused()
+    # CHECK: fused str: loc(fused<"sauteed">["carrot", "potatoes"])
+    print("fused str:", str(loc_attr))
+    # CHECK: fused repr: loc(fused<"sauteed">["carrot", "potatoes"])
+    print("fused repr:", repr(loc_attr))
+    # CHECK: fused locations: [loc("carrot"), loc("potatoes")]
+    print("fused locations:", loc_attr.locations)
+
+    assert not loc_empty.is_a_fused()
+    # CHECK: fused str: loc(unknown)
+    print("fused str:", str(loc_empty))
+    # CHECK: fused repr: loc(unknown)
+    print("fused repr:", repr(loc_empty))
+    # CHECK: fused locations: []
+    print("fused locations:", loc_empty.locations)
+
+    assert loc_empty_attr.is_a_fused()
+    # CHECK: fused str: loc(fused<"sauteed">[unknown])
+    print("fused str:", str(loc_empty_attr))
+    # CHECK: fused repr: loc(fused<"sauteed">[unknown])
+    print("fused repr:", repr(loc_empty_attr))
+    # CHECK: fused locations: [loc(unknown)]
+    print("fused locations:", loc_empty_attr.locations)
+
+    assert loc_single_attr.is_a_fused()
+    # CHECK: fused str: loc(fused<"sauteed">["apple"])
+    print("fused str:", str(loc_single_attr))
+    # CHECK: fused repr: loc(fused<"sauteed">["apple"])
+    print("fused repr:", repr(loc_single_attr))
+    # CHECK: fused locations: [loc("apple")]
+    print("fused locations:", loc_single_attr.locations)
 
 
 run(testFused)
diff --git a/mlir/test/python/lib/PythonTestCAPI.cpp b/mlir/test/python/lib/PythonTestCAPI.cpp
index cb7d7677714fe..ee6a5ae2d9c3d 100644
--- a/mlir/test/python/lib/PythonTestCAPI.cpp
+++ b/mlir/test/python/lib/PythonTestCAPI.cpp
@@ -11,6 +11,8 @@
 #include "mlir-c/BuiltinTypes.h"
 #include "mlir/CAPI/Registration.h"
 #include "mlir/CAPI/Wrap.h"
+#include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/Location.h"
 
 MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(PythonTest, python_test,
                                       python_test::PythonTestDialect)
@@ -42,3 +44,9 @@ MlirTypeID mlirPythonTestTestTypeGetTypeID(void) {
 bool mlirTypeIsAPythonTestTestTensorValue(MlirValue value) {
   return mlirTypeIsATensor(wrap(unwrap(value).getType()));
 }
+
+void mlirPythonTestEmitDiagnosticWithNote(MlirContext ctx) {
+  auto diag =
+      mlir::emitError(unwrap(mlirLocationUnknownGet(ctx)), "created error");
+  diag.attachNote() << "attached note";
+}
diff --git a/mlir/test/python/lib/PythonTestCAPI.h b/mlir/test/python/lib/PythonTestCAPI.h
index 43f8fdcbfae12..5ffd0cbf6201e 100644
--- a/mlir/test/python/lib/PythonTestCAPI.h
+++ b/mlir/test/python/lib/PythonTestCAPI.h
@@ -10,6 +10,7 @@
 #define MLIR_TEST_PYTHON_LIB_PYTHONTESTCAPI_H
 
 #include "mlir-c/IR.h"
+#include "mlir-c/Support.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -33,6 +34,8 @@ MLIR_CAPI_EXPORTED MlirTypeID mlirPythonTestTestTypeGetTypeID(void);
 
 MLIR_CAPI_EXPORTED bool mlirTypeIsAPythonTestTestTensorValue(MlirValue value);
 
+MLIR_CAPI_EXPORTED void mlirPythonTestEmitDiagnosticWithNote(MlirContext ctx);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/mlir/test/python/lib/PythonTestModuleNanobind.cpp b/mlir/test/python/lib/PythonTestModuleNanobind.cpp
index 99c81eae97a0c..dd0a9f2b66ea6 100644
--- a/mlir/test/python/lib/PythonTestModuleNanobind.cpp
+++ b/mlir/test/python/lib/PythonTestModuleNanobind.cpp
@@ -11,9 +11,12 @@
 #include "PythonTestCAPI.h"
 #include "mlir-c/BuiltinAttributes.h"
 #include "mlir-c/BuiltinTypes.h"
+#include "mlir-c/Diagnostics.h"
 #include "mlir-c/IR.h"
+#include "mlir/Bindings/Python/Diagnostics.h"
 #include "mlir/Bindings/Python/Nanobind.h"
 #include "mlir/Bindings/Python/NanobindAdaptors.h"
+#include "nanobind/nanobind.h"
 
 namespace nb = nanobind;
 using namespace mlir::python::nanobind_adaptors;
@@ -45,6 +48,13 @@ NB_MODULE(_mlirPythonTestNanobind, m) {
       },
       nb::arg("registry"));
 
+  m.def("test_diagnostics_with_errors_and_notes", [](MlirContext ctx) {
+    mlir::python::CollectDiagnosticsToStringScope handler(ctx);
+
+    mlirPythonTestEmitDiagnosticWithNote(ctx);
+    throw nb::value_error(handler.takeMessage().c_str());
+  });
+
   mlir_attribute_subclass(m, "TestAttr",
                           mlirAttributeIsAPythonTestTestAttribute,
                           mlirPythonTestTestAttributeGetTypeID)
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index f18ad45dfb708..d06ff8070e7cf 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -170,6 +170,7 @@ void registerTestDialect(DialectRegistry &);
 void registerTestDynDialect(DialectRegistry &);
 void registerTestTilingInterfaceTransformDialectExtension(DialectRegistry &);
 void registerTestTransformDialectExtension(DialectRegistry &);
+void registerTestTransformsTransformDialectExtension(DialectRegistry &);
 } // namespace test
 
 #ifdef MLIR_INCLUDE_TESTS
@@ -323,6 +324,7 @@ int main(int argc, char **argv) {
 #ifdef MLIR_INCLUDE_TESTS
   ::test::registerTestDialect(registry);
   ::test::registerTestTransformDialectExtension(registry);
+  ::test::registerTestTransformsTransformDialectExtension(registry);
   ::test::registerTestTilingInterfaceTransformDialectExtension(registry);
   ::test::registerTestDynDialect(registry);
 #endif
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index 1970647a80115..b957c8ee9f8ab 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -411,6 +411,15 @@ class OpOrAdaptorHelper {
       return true;
     if (!op.getDialect().usePropertiesForAttributes())
       return false;
+    return true;
+  }
+
+  /// Returns whether the operation will have a non-empty `Properties` struct.
+  bool hasNonEmptyPropertiesStruct() const {
+    if (!op.getProperties().empty())
+      return true;
+    if (!hasProperties())
+      return false;
     if (op.getTrait("::mlir::OpTrait::AttrSizedOperandSegments") ||
         op.getTrait("::mlir::OpTrait::AttrSizedResultSegments"))
       return true;
@@ -661,24 +670,33 @@ class OpEmitter {
   // type as all results' types.
   void genUseOperandAsResultTypeSeparateParamBuilder();
 
+  // The kind of collective builder to generate
+  enum class CollectiveBuilderKind {
+    PropStruct, // Inherent attributes/properties are passed by `const
+                // Properties&`
+    AttrDict,   // Inherent attributes/properties are passed by attribute
+                // dictionary
+  };
+
   // Generates the build() method that takes all operands/attributes
   // collectively as one parameter. The generated build() method uses first
   // operand's type as all results' types.
-  void genUseOperandAsResultTypeCollectiveParamBuilder();
+  void
+  genUseOperandAsResultTypeCollectiveParamBuilder(CollectiveBuilderKind kind);
 
   // Generates the build() method that takes aggregate operands/attributes
   // parameters. This build() method uses inferred types as result types.
   // Requires: The type needs to be inferable via InferTypeOpInterface.
-  void genInferredTypeCollectiveParamBuilder();
+  void genInferredTypeCollectiveParamBuilder(CollectiveBuilderKind kind);
 
-  // Generates the build() method that takes each operand/attribute as a
-  // stand-alone parameter. The generated build() method uses first attribute's
+  // Generates the build() method that takesaggregate operands/attributes as
+  // parameters. The generated build() method uses first attribute's
   // type as all result's types.
-  void genUseAttrAsResultTypeBuilder();
+  void genUseAttrAsResultTypeCollectiveParamBuilder(CollectiveBuilderKind kind);
 
   // Generates the build() method that takes all result types collectively as
   // one parameter. Similarly for operands and attributes.
-  void genCollectiveParamBuilder();
+  void genCollectiveParamBuilder(CollectiveBuilderKind kind);
 
   // The kind of parameter to generate for result types in builders.
   enum class TypeParamKind {
@@ -1363,8 +1381,6 @@ void OpEmitter::genPropertiesSupport() {
     attrOrProperties.push_back(&emitHelper.getOperandSegmentsSize().value());
   if (emitHelper.getResultSegmentsSize())
     attrOrProperties.push_back(&emitHelper.getResultSegmentsSize().value());
-  if (attrOrProperties.empty())
-    return;
   auto &setPropMethod =
       opClass
           .addStaticMethod(
@@ -1728,6 +1744,9 @@ void OpEmitter::genPropertiesSupport() {
 
 void OpEmitter::genPropertiesSupportForBytecode(
     ArrayRef<ConstArgument> attrOrProperties) {
+  if (attrOrProperties.empty())
+    return;
+
   if (op.useCustomPropertiesEncoding()) {
     opClass.declareStaticMethod(
         "::llvm::LogicalResult", "readProperties",
@@ -2644,7 +2663,8 @@ void OpEmitter::genSeparateArgParamBuilder() {
   }
 }
 
-void OpEmitter::genUseOperandAsResultTypeCollectiveParamBuilder() {
+void OpEmitter::genUseOperandAsResultTypeCollectiveParamBuilder(
+    CollectiveBuilderKind kind) {
   int numResults = op.getNumResults();
 
   // Signature
@@ -2652,10 +2672,15 @@ void OpEmitter::genUseOperandAsResultTypeCollectiveParamBuilder() {
   paramList.emplace_back("::mlir::OpBuilder &", "odsBuilder");
   paramList.emplace_back("::mlir::OperationState &", builderOpState);
   paramList.emplace_back("::mlir::ValueRange", "operands");
+  if (kind == CollectiveBuilderKind::PropStruct)
+    paramList.emplace_back("const Properties &", "properties");
   // Provide default value for `attributes` when its the last parameter
   StringRef attributesDefaultValue = op.getNumVariadicRegions() ? "" : "{}";
+  StringRef attributesName = kind == CollectiveBuilderKind::PropStruct
+                                 ? "discardableAttributes"
+                                 : "attributes";
   paramList.emplace_back("::llvm::ArrayRef<::mlir::NamedAttribute>",
-                         "attributes", attributesDefaultValue);
+                         attributesName, attributesDefaultValue);
   if (op.getNumVariadicRegions())
     paramList.emplace_back("unsigned", "numRegions");
 
@@ -2668,8 +2693,12 @@ void OpEmitter::genUseOperandAsResultTypeCollectiveParamBuilder() {
   // Operands
   body << "  " << builderOpState << ".addOperands(operands);\n";
 
+  if (kind == CollectiveBuilderKind::PropStruct)
+    body << "  " << builderOpState
+         << ".useProperties(const_cast<Properties&>(properties));\n";
   // Attributes
-  body << "  " << builderOpState << ".addAttributes(attributes);\n";
+  body << "  " << builderOpState << ".addAttributes(" << attributesName
+       << ");\n";
 
   // Create the correct number of regions
   if (int numRegions = op.getNumRegions()) {
@@ -2752,14 +2781,20 @@ void OpEmitter::genPopulateDefaultAttributes() {
   }
 }
 
-void OpEmitter::genInferredTypeCollectiveParamBuilder() {
+void OpEmitter::genInferredTypeCollectiveParamBuilder(
+    CollectiveBuilderKind kind) {
   SmallVector<MethodParameter> paramList;
   paramList.emplace_back("::mlir::OpBuilder &", "odsBuilder");
   paramList.emplace_back("::mlir::OperationState &", builderOpState);
   paramList.emplace_back("::mlir::ValueRange", "operands");
+  if (kind == CollectiveBuilderKind::PropStruct)
+    paramList.emplace_back("const Properties &", "properties");
   StringRef attributesDefaultValue = op.getNumVariadicRegions() ? "" : "{}";
+  StringRef attributesName = kind == CollectiveBuilderKind::PropStruct
+                                 ? "discardableAttributes"
+                                 : "attributes";
   paramList.emplace_back("::llvm::ArrayRef<::mlir::NamedAttribute>",
-                         "attributes", attributesDefaultValue);
+                         attributesName, attributesDefaultValue);
   if (op.getNumVariadicRegions())
     paramList.emplace_back("unsigned", "numRegions");
 
@@ -2784,7 +2819,11 @@ void OpEmitter::genInferredTypeCollectiveParamBuilder() {
          << numNonVariadicOperands
          << "u && \"mismatched number of parameters\");\n";
   body << "  " << builderOpState << ".addOperands(operands);\n";
-  body << "  " << builderOpState << ".addAttributes(attributes);\n";
+  if (kind == CollectiveBuilderKind::PropStruct)
+    body << "  " << builderOpState
+         << ".useProperties(const_cast<Properties &>(properties));\n";
+  body << "  " << builderOpState << ".addAttributes(" << attributesName
+       << ");\n";
 
   // Create the correct number of regions
   if (int numRegions = op.getNumRegions()) {
@@ -2795,7 +2834,8 @@ void OpEmitter::genInferredTypeCollectiveParamBuilder() {
   }
 
   // Result types
-  if (emitHelper.hasProperties()) {
+  if (emitHelper.hasNonEmptyPropertiesStruct() &&
+      kind == CollectiveBuilderKind::AttrDict) {
     // Initialize the properties from Attributes before invoking the infer
     // function.
     body << formatv(R"(
@@ -2867,13 +2907,19 @@ void OpEmitter::genUseOperandAsResultTypeSeparateParamBuilder() {
     emit(AttrParamKind::UnwrappedValue);
 }
 
-void OpEmitter::genUseAttrAsResultTypeBuilder() {
+void OpEmitter::genUseAttrAsResultTypeCollectiveParamBuilder(
+    CollectiveBuilderKind kind) {
   SmallVector<MethodParameter> paramList;
   paramList.emplace_back("::mlir::OpBuilder &", "odsBuilder");
   paramList.emplace_back("::mlir::OperationState &", builderOpState);
   paramList.emplace_back("::mlir::ValueRange", "operands");
+  if (kind == CollectiveBuilderKind::PropStruct)
+    paramList.emplace_back("const Properties &", "properties");
+  StringRef attributesName = kind == CollectiveBuilderKind::PropStruct
+                                 ? "discardableAttributes"
+                                 : "attributes";
   paramList.emplace_back("::llvm::ArrayRef<::mlir::NamedAttribute>",
-                         "attributes", "{}");
+                         attributesName, "{}");
   auto *m = opClass.addStaticMethod("void", "build", std::move(paramList));
   // If the builder is redundant, skip generating the method
   if (!m)
@@ -2885,28 +2931,44 @@ void OpEmitter::genUseAttrAsResultTypeBuilder() {
   std::string resultType;
   const auto &namedAttr = op.getAttribute(0);
 
-  body << "  auto attrName = " << op.getGetterName(namedAttr.name)
-       << "AttrName(" << builderOpState
-       << ".name);\n"
-          "  for (auto attr : attributes) {\n"
-          "    if (attr.getName() != attrName) continue;\n";
   if (namedAttr.attr.isTypeAttr()) {
-    resultType = "::llvm::cast<::mlir::TypeAttr>(attr.getValue()).getValue()";
+    resultType = "::llvm::cast<::mlir::TypeAttr>(typeAttr).getValue()";
+  } else {
+    resultType = "::llvm::cast<::mlir::TypedAttr>(typeAttr).getType()";
+  }
+
+  if (kind == CollectiveBuilderKind::PropStruct) {
+    body << "  ::mlir::Attribute typeAttr = properties."
+         << op.getGetterName(namedAttr.name) << "();\n";
   } else {
-    resultType = "::llvm::cast<::mlir::TypedAttr>(attr.getValue()).getType()";
+    body << "  ::mlir::Attribute typeAttr;\n"
+         << "  auto attrName = " << op.getGetterName(namedAttr.name)
+         << "AttrName(" << builderOpState
+         << ".name);\n"
+            "  for (auto attr : attributes) {\n"
+            "    if (attr.getName() == attrName) {\n"
+            "      typeAttr = attr.getValue();\n"
+            "      break;\n"
+            "    }\n"
+            "  }\n";
   }
 
   // Operands
   body << "  " << builderOpState << ".addOperands(operands);\n";
 
+  // Properties
+  if (kind == CollectiveBuilderKind::PropStruct)
+    body << "  " << builderOpState
+         << ".useProperties(const_cast<Properties&>(properties));\n";
+
   // Attributes
-  body << "  " << builderOpState << ".addAttributes(attributes);\n";
+  body << "  " << builderOpState << ".addAttributes(" << attributesName
+       << ");\n";
 
   // Result types
   SmallVector<std::string, 2> resultTypes(op.getNumResults(), resultType);
   body << "    " << builderOpState << ".addTypes({"
        << llvm::join(resultTypes, ", ") << "});\n";
-  body << "  }\n";
 }
 
 /// Returns a signature of the builder. Updates the context `fctx` to enable
@@ -2973,22 +3035,32 @@ void OpEmitter::genBuilder() {
   // 1. one having a stand-alone parameter for each operand / attribute, and
   genSeparateArgParamBuilder();
   // 2. one having an aggregated parameter for all result types / operands /
-  //    attributes, and
-  genCollectiveParamBuilder();
+  //    [properties / discardable] attributes, and
+  genCollectiveParamBuilder(CollectiveBuilderKind::AttrDict);
+  if (emitHelper.hasProperties())
+    genCollectiveParamBuilder(CollectiveBuilderKind::PropStruct);
   // 3. one having a stand-alone parameter for each operand and attribute,
   //    use the first operand or attribute's type as all result types
   //    to facilitate different call patterns.
   if (op.getNumVariableLengthResults() == 0) {
     if (op.getTrait("::mlir::OpTrait::SameOperandsAndResultType")) {
       genUseOperandAsResultTypeSeparateParamBuilder();
-      genUseOperandAsResultTypeCollectiveParamBuilder();
+      genUseOperandAsResultTypeCollectiveParamBuilder(
+          CollectiveBuilderKind::AttrDict);
+      if (emitHelper.hasProperties())
+        genUseOperandAsResultTypeCollectiveParamBuilder(
+            CollectiveBuilderKind::PropStruct);
+    }
+    if (op.getTrait("::mlir::OpTrait::FirstAttrDerivedResultType")) {
+      genUseAttrAsResultTypeCollectiveParamBuilder(
+          CollectiveBuilderKind::AttrDict);
+      genUseAttrAsResultTypeCollectiveParamBuilder(
+          CollectiveBuilderKind::PropStruct);
     }
-    if (op.getTrait("::mlir::OpTrait::FirstAttrDerivedResultType"))
-      genUseAttrAsResultTypeBuilder();
   }
 }
 
-void OpEmitter::genCollectiveParamBuilder() {
+void OpEmitter::genCollectiveParamBuilder(CollectiveBuilderKind kind) {
   int numResults = op.getNumResults();
   int numVariadicResults = op.getNumVariableLengthResults();
   int numNonVariadicResults = numResults - numVariadicResults;
@@ -3002,10 +3074,15 @@ void OpEmitter::genCollectiveParamBuilder() {
   paramList.emplace_back("::mlir::OperationState &", builderOpState);
   paramList.emplace_back("::mlir::TypeRange", "resultTypes");
   paramList.emplace_back("::mlir::ValueRange", "operands");
+  if (kind == CollectiveBuilderKind::PropStruct)
+    paramList.emplace_back("const Properties &", "properties");
   // Provide default value for `attributes` when its the last parameter
   StringRef attributesDefaultValue = op.getNumVariadicRegions() ? "" : "{}";
+  StringRef attributesName = kind == CollectiveBuilderKind::PropStruct
+                                 ? "discardableAttributes"
+                                 : "attributes";
   paramList.emplace_back("::llvm::ArrayRef<::mlir::NamedAttribute>",
-                         "attributes", attributesDefaultValue);
+                         attributesName, attributesDefaultValue);
   if (op.getNumVariadicRegions())
     paramList.emplace_back("unsigned", "numRegions");
 
@@ -3023,8 +3100,14 @@ void OpEmitter::genCollectiveParamBuilder() {
          << "u && \"mismatched number of parameters\");\n";
   body << "  " << builderOpState << ".addOperands(operands);\n";
 
+  // Properties
+  if (kind == CollectiveBuilderKind::PropStruct)
+    body << "  " << builderOpState
+         << ".useProperties(const_cast<Properties&>(properties));\n";
+
   // Attributes
-  body << "  " << builderOpState << ".addAttributes(attributes);\n";
+  body << "  " << builderOpState << ".addAttributes(" << attributesName
+       << ");\n";
 
   // Create the correct number of regions
   if (int numRegions = op.getNumRegions()) {
@@ -3041,7 +3124,8 @@ void OpEmitter::genCollectiveParamBuilder() {
          << "u && \"mismatched number of return types\");\n";
   body << "  " << builderOpState << ".addTypes(resultTypes);\n";
 
-  if (emitHelper.hasProperties()) {
+  if (emitHelper.hasNonEmptyPropertiesStruct() &&
+      kind == CollectiveBuilderKind::AttrDict) {
     // Initialize the properties from Attributes before invoking the infer
     // function.
     body << formatv(R"(
@@ -3060,7 +3144,7 @@ void OpEmitter::genCollectiveParamBuilder() {
   // Generate builder that infers type too.
   // TODO: Expand to handle successors.
   if (canInferType(op) && op.getNumSuccessors() == 0)
-    genInferredTypeCollectiveParamBuilder();
+    genInferredTypeCollectiveParamBuilder(kind);
 }
 
 void OpEmitter::buildParamList(SmallVectorImpl<MethodParameter> &paramList,
@@ -4061,7 +4145,7 @@ void OpEmitter::genTraits() {
   // native/interface traits and after all the traits with `StructuralOpTrait`.
   opClass.addTrait("::mlir::OpTrait::OpInvariants");
 
-  if (emitHelper.hasProperties())
+  if (emitHelper.hasNonEmptyPropertiesStruct())
     opClass.addTrait("::mlir::BytecodeOpInterface::Trait");
 
   // Add the native and interface traits.
@@ -4201,7 +4285,6 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(
       attrOrProperties.push_back(&emitHelper.getOperandSegmentsSize().value());
     if (emitHelper.getResultSegmentsSize())
       attrOrProperties.push_back(&emitHelper.getResultSegmentsSize().value());
-    assert(!attrOrProperties.empty());
     std::string declarations = "  struct Properties {\n";
     llvm::raw_string_ostream os(declarations);
     std::string comparator =
@@ -4274,7 +4357,7 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(
       // Emit accessors using the interface type.
       if (attr) {
         const char *accessorFmt = R"decl(
-    auto get{0}() {
+    auto get{0}() const {
       auto &propStorage = this->{1};
       return ::llvm::{2}<{3}>(propStorage);
     }
@@ -4296,7 +4379,12 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(
     os << comparator;
     os << "  };\n";
 
-    genericAdaptorBase.declare<ExtraClassDeclaration>(std::move(declarations));
+    if (attrOrProperties.empty())
+      genericAdaptorBase.declare<UsingDeclaration>("Properties",
+                                                   "::mlir::EmptyProperties");
+    else
+      genericAdaptorBase.declare<ExtraClassDeclaration>(
+          std::move(declarations));
   }
   genericAdaptorBase.declare<VisibilityDeclaration>(Visibility::Protected);
   genericAdaptorBase.declare<Field>("::mlir::DictionaryAttr", "odsAttrs");
diff --git a/mlir/tools/mlir-tblgen/RewriterGen.cpp b/mlir/tools/mlir-tblgen/RewriterGen.cpp
index f6eb5bdfe568e..f921788abdd71 100644
--- a/mlir/tools/mlir-tblgen/RewriterGen.cpp
+++ b/mlir/tools/mlir-tblgen/RewriterGen.cpp
@@ -122,7 +122,7 @@ class PatternEmitter {
 
   // Emits C++ statements for matching the `argIndex`-th argument of the given
   // DAG `tree` as an attribute.
-  void emitAttributeMatch(DagNode tree, StringRef opName, int argIndex,
+  void emitAttributeMatch(DagNode tree, StringRef castedName, int argIndex,
                           int depth);
 
   // Emits C++ for checking a match with a corresponding match failure
@@ -664,7 +664,7 @@ void PatternEmitter::emitOpMatch(DagNode tree, StringRef opName, int depth) {
                        /*variadicSubIndex=*/std::nullopt);
       ++nextOperand;
     } else if (isa<NamedAttribute *>(opArg)) {
-      emitAttributeMatch(tree, opName, opArgIdx, depth);
+      emitAttributeMatch(tree, castedName, opArgIdx, depth);
     } else {
       PrintFatalError(loc, "unhandled case when matching op");
     }
@@ -864,16 +864,22 @@ void PatternEmitter::emitVariadicOperandMatch(DagNode tree,
   os.unindent() << "}\n";
 }
 
-void PatternEmitter::emitAttributeMatch(DagNode tree, StringRef opName,
+void PatternEmitter::emitAttributeMatch(DagNode tree, StringRef castedName,
                                         int argIndex, int depth) {
   Operator &op = tree.getDialectOp(opMap);
   auto *namedAttr = cast<NamedAttribute *>(op.getArg(argIndex));
   const auto &attr = namedAttr->attr;
 
   os << "{\n";
-  os.indent() << formatv("auto tblgen_attr = {0}->getAttrOfType<{1}>(\"{2}\");"
-                         "(void)tblgen_attr;\n",
-                         opName, attr.getStorageType(), namedAttr->name);
+  if (op.getDialect().usePropertiesForAttributes()) {
+    os.indent() << formatv("auto tblgen_attr = {0}.getProperties().{1}();\n",
+                           castedName, op.getGetterName(namedAttr->name));
+  } else {
+    os.indent() << formatv(
+        "auto tblgen_attr = {0}->getAttrOfType<{1}>(\"{2}\");"
+        "(void)tblgen_attr;\n",
+        castedName, attr.getStorageType(), namedAttr->name);
+  }
 
   // TODO: This should use getter method to avoid duplication.
   if (attr.hasDefaultValue()) {
@@ -887,7 +893,7 @@ void PatternEmitter::emitAttributeMatch(DagNode tree, StringRef opName,
     // That is precisely what getDiscardableAttr() returns on missing
     // attributes.
   } else {
-    emitMatchCheck(opName, tgfmt("tblgen_attr", &fmtCtx),
+    emitMatchCheck(castedName, tgfmt("tblgen_attr", &fmtCtx),
                    formatv("\"expected op '{0}' to have attribute '{1}' "
                            "of type '{2}'\"",
                            op.getOperationName(), namedAttr->name,
@@ -918,7 +924,7 @@ void PatternEmitter::emitAttributeMatch(DagNode tree, StringRef opName,
       }
     }
     emitStaticVerifierCall(
-        verifier, opName, "tblgen_attr",
+        verifier, castedName, "tblgen_attr",
         formatv("\"op '{0}' attribute '{1}' failed to satisfy constraint: "
                 "'{2}'\"",
                 op.getOperationName(), namedAttr->name,
@@ -1532,6 +1538,7 @@ std::string PatternEmitter::handleOpCreation(DagNode tree, int resultIndex,
   LLVM_DEBUG(llvm::dbgs() << '\n');
 
   Operator &resultOp = tree.getDialectOp(opMap);
+  bool useProperties = resultOp.getDialect().usePropertiesForAttributes();
   auto numOpArgs = resultOp.getNumArgs();
   auto numPatArgs = tree.getNumArgs();
 
@@ -1623,9 +1630,10 @@ std::string PatternEmitter::handleOpCreation(DagNode tree, int resultIndex,
     createAggregateLocalVarsForOpArgs(tree, childNodeNames, depth);
 
     // Then create the op.
-    os.scope("", "\n}\n").os << formatv(
-        "{0} = rewriter.create<{1}>({2}, tblgen_values, tblgen_attrs);",
-        valuePackName, resultOp.getQualCppClassName(), locToUse);
+    os.scope("", "\n}\n").os
+        << formatv("{0} = rewriter.create<{1}>({2}, tblgen_values, {3});",
+                   valuePackName, resultOp.getQualCppClassName(), locToUse,
+                   useProperties ? "tblgen_props" : "tblgen_attrs");
     return resultValue;
   }
 
@@ -1682,8 +1690,9 @@ std::string PatternEmitter::handleOpCreation(DagNode tree, int resultIndex,
     }
   }
   os << formatv("{0} = rewriter.create<{1}>({2}, tblgen_types, "
-                "tblgen_values, tblgen_attrs);\n",
-                valuePackName, resultOp.getQualCppClassName(), locToUse);
+                "tblgen_values, {3});\n",
+                valuePackName, resultOp.getQualCppClassName(), locToUse,
+                useProperties ? "tblgen_props" : "tblgen_attrs");
   os.unindent() << "}\n";
   return resultValue;
 }
@@ -1791,16 +1800,27 @@ void PatternEmitter::createAggregateLocalVarsForOpArgs(
     DagNode node, const ChildNodeIndexNameMap &childNodeNames, int depth) {
   Operator &resultOp = node.getDialectOp(opMap);
 
+  bool useProperties = resultOp.getDialect().usePropertiesForAttributes();
   auto scope = os.scope();
   os << formatv("::llvm::SmallVector<::mlir::Value, 4> "
                 "tblgen_values; (void)tblgen_values;\n");
-  os << formatv("::llvm::SmallVector<::mlir::NamedAttribute, 4> "
-                "tblgen_attrs; (void)tblgen_attrs;\n");
+  if (useProperties) {
+    os << formatv("{0}::Properties tblgen_props; (void)tblgen_props;\n",
+                  resultOp.getQualCppClassName());
+  } else {
+    os << formatv("::llvm::SmallVector<::mlir::NamedAttribute, 4> "
+                  "tblgen_attrs; (void)tblgen_attrs;\n");
+  }
 
+  const char *setPropCmd =
+      "tblgen_props.{0} = "
+      "::llvm::dyn_cast_if_present<decltype(tblgen_props.{0})>({1});\n";
   const char *addAttrCmd =
       "if (auto tmpAttr = {1}) {\n"
       "  tblgen_attrs.emplace_back(rewriter.getStringAttr(\"{0}\"), "
       "tmpAttr);\n}\n";
+  const char *setterCmd = (useProperties) ? setPropCmd : addAttrCmd;
+
   int numVariadic = 0;
   bool hasOperandSegmentSizes = false;
   std::vector<std::string> sizes;
@@ -1814,13 +1834,13 @@ void PatternEmitter::createAggregateLocalVarsForOpArgs(
         if (!subTree.isNativeCodeCall())
           PrintFatalError(loc, "only NativeCodeCall allowed in nested dag node "
                                "for creating attribute");
-        os << formatv(addAttrCmd, opArgName, childNodeNames.lookup(argIndex));
+
+        os << formatv(setterCmd, opArgName, childNodeNames.lookup(argIndex));
       } else {
         auto leaf = node.getArgAsLeaf(argIndex);
         // The argument in the result DAG pattern.
         auto patArgName = node.getArgName(argIndex);
-        os << formatv(addAttrCmd, opArgName,
-                      handleOpArgument(leaf, patArgName));
+        os << formatv(setterCmd, opArgName, handleOpArgument(leaf, patArgName));
       }
       continue;
     }
@@ -1876,11 +1896,18 @@ void PatternEmitter::createAggregateLocalVarsForOpArgs(
     const auto *sameVariadicSize =
         resultOp.getTrait("::mlir::OpTrait::SameVariadicOperandSize");
     if (!sameVariadicSize) {
-      const char *setSizes = R"(
-        tblgen_attrs.emplace_back(rewriter.getStringAttr("operandSegmentSizes"),
-          rewriter.getDenseI32ArrayAttr({{ {0} }));
-          )";
-      os.printReindented(formatv(setSizes, llvm::join(sizes, ", ")).str());
+      if (useProperties) {
+        const char *setSizes = R"(
+          tblgen_props.operandSegmentSizes = {{ {0} };
+        )";
+        os.printReindented(formatv(setSizes, llvm::join(sizes, ", ")).str());
+      } else {
+        const char *setSizes = R"(
+          tblgen_attrs.emplace_back(rewriter.getStringAttr("operandSegmentSizes"),
+            rewriter.getDenseI32ArrayAttr({{ {0} }));
+            )";
+        os.printReindented(formatv(setSizes, llvm::join(sizes, ", ")).str());
+      }
     }
   }
 }
diff --git a/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp b/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp
index 3d87948c5b5af..fd81f3021aa9b 100644
--- a/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp
+++ b/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp
@@ -23,6 +23,8 @@ using namespace mlir;
 namespace {
 constexpr static llvm::StringLiteral kAttrName = "dltest.layout";
 constexpr static llvm::StringLiteral kEndiannesKeyName = "dltest.endianness";
+constexpr static llvm::StringLiteral kDefaultKeyName =
+    "dltest.default_memory_space";
 constexpr static llvm::StringLiteral kAllocaKeyName =
     "dltest.alloca_memory_space";
 constexpr static llvm::StringLiteral kManglingModeKeyName =
@@ -82,6 +84,9 @@ struct CustomDataLayoutSpec
   StringAttr getEndiannessIdentifier(MLIRContext *context) const {
     return Builder(context).getStringAttr(kEndiannesKeyName);
   }
+  StringAttr getDefaultMemorySpaceIdentifier(MLIRContext *context) const {
+    return Builder(context).getStringAttr(kDefaultKeyName);
+  }
   StringAttr getAllocaMemorySpaceIdentifier(MLIRContext *context) const {
     return Builder(context).getStringAttr(kAllocaKeyName);
   }
@@ -205,6 +210,15 @@ struct SingleQueryType
     return Attribute();
   }
 
+  Attribute getDefaultMemorySpace(DataLayoutEntryInterface entry) {
+    static bool executed = false;
+    if (executed)
+      llvm::report_fatal_error("repeated call");
+
+    executed = true;
+    return Attribute();
+  }
+
   Attribute getAllocaMemorySpace(DataLayoutEntryInterface entry) {
     static bool executed = false;
     if (executed)
@@ -475,6 +489,7 @@ module {}
   EXPECT_EQ(layout.getTypePreferredAlignment(Float16Type::get(&ctx)), 2u);
 
   EXPECT_EQ(layout.getEndianness(), Attribute());
+  EXPECT_EQ(layout.getDefaultMemorySpace(), Attribute());
   EXPECT_EQ(layout.getAllocaMemorySpace(), Attribute());
   EXPECT_EQ(layout.getProgramMemorySpace(), Attribute());
   EXPECT_EQ(layout.getGlobalMemorySpace(), Attribute());
@@ -508,6 +523,7 @@ TEST(DataLayout, NullSpec) {
   EXPECT_EQ(layout.getTypeIndexBitwidth(IndexType::get(&ctx)), 64u);
 
   EXPECT_EQ(layout.getEndianness(), Attribute());
+  EXPECT_EQ(layout.getDefaultMemorySpace(), Attribute());
   EXPECT_EQ(layout.getAllocaMemorySpace(), Attribute());
   EXPECT_EQ(layout.getProgramMemorySpace(), Attribute());
   EXPECT_EQ(layout.getGlobalMemorySpace(), Attribute());
@@ -549,6 +565,7 @@ TEST(DataLayout, EmptySpec) {
   EXPECT_EQ(layout.getTypeIndexBitwidth(IndexType::get(&ctx)), 64u);
 
   EXPECT_EQ(layout.getEndianness(), Attribute());
+  EXPECT_EQ(layout.getDefaultMemorySpace(), Attribute());
   EXPECT_EQ(layout.getAllocaMemorySpace(), Attribute());
   EXPECT_EQ(layout.getProgramMemorySpace(), Attribute());
   EXPECT_EQ(layout.getGlobalMemorySpace(), Attribute());
@@ -572,6 +589,7 @@ TEST(DataLayout, SpecWithEntries) {
   #dlti.dl_entry<i16, 6>,
   #dlti.dl_entry<index, 42>,
   #dlti.dl_entry<"dltest.endianness", "little">,
+  #dlti.dl_entry<"dltest.default_memory_space", 1 : i32>,
   #dlti.dl_entry<"dltest.alloca_memory_space", 5 : i32>,
   #dlti.dl_entry<"dltest.program_memory_space", 3 : i32>,
   #dlti.dl_entry<"dltest.global_memory_space", 2 : i32>,
@@ -609,6 +627,7 @@ TEST(DataLayout, SpecWithEntries) {
   EXPECT_EQ(layout.getTypePreferredAlignment(Float32Type::get(&ctx)), 64u);
 
   EXPECT_EQ(layout.getEndianness(), Builder(&ctx).getStringAttr("little"));
+  EXPECT_EQ(layout.getDefaultMemorySpace(), Builder(&ctx).getI32IntegerAttr(1));
   EXPECT_EQ(layout.getAllocaMemorySpace(), Builder(&ctx).getI32IntegerAttr(5));
   EXPECT_EQ(layout.getProgramMemorySpace(), Builder(&ctx).getI32IntegerAttr(3));
   EXPECT_EQ(layout.getGlobalMemorySpace(), Builder(&ctx).getI32IntegerAttr(2));
diff --git a/mlir/unittests/TableGen/OpBuildGen.cpp b/mlir/unittests/TableGen/OpBuildGen.cpp
index 94fbfa28803c4..74e62aacd5e80 100644
--- a/mlir/unittests/TableGen/OpBuildGen.cpp
+++ b/mlir/unittests/TableGen/OpBuildGen.cpp
@@ -291,4 +291,22 @@ TEST_F(OpBuildGenTest, BuildMethodsVariadicProperties) {
   verifyOp(std::move(op), {f32Ty}, {*cstI32}, {*cstI32}, attrs);
 }
 
+TEST_F(OpBuildGenTest, BuildMethodsInherentDiscardableAttrs) {
+  test::TableGenBuildOp7::Properties props;
+  props.attr0 = cast<BoolAttr>(attrs[0].getValue());
+  ArrayRef<NamedAttribute> discardableAttrs = attrs.drop_front();
+  auto op7 = builder.create<test::TableGenBuildOp7>(
+      loc, TypeRange{}, ValueRange{}, props, discardableAttrs);
+  verifyOp(op7, {}, {}, attrs);
+
+  // Check that the old-style builder where all the attributes go in the same
+  // place works.
+  auto op7b = builder.create<test::TableGenBuildOp7>(loc, TypeRange{},
+                                                     ValueRange{}, attrs);
+  // Note: this goes before verifyOp() because verifyOp() calls erase(), causing
+  // use-after-free.
+  ASSERT_EQ(op7b.getProperties().getAttr0(), attrs[0].getValue());
+  verifyOp(op7b, {}, {}, attrs);
+}
+
 } // namespace mlir
diff --git a/offload/test/Inputs/target-use-dev-ptr.c b/offload/test/Inputs/target-use-dev-ptr.c
index e1430a93fbc7d..71e37eee1b1ae 100644
--- a/offload/test/Inputs/target-use-dev-ptr.c
+++ b/offload/test/Inputs/target-use-dev-ptr.c
@@ -21,3 +21,7 @@ int check_result(int *host_ptr, int *dev_ptr) {
     return 0;
   }
 }
+
+int check_equality(void *host_ptr, void *dev_ptr) {
+  return dev_ptr == host_ptr;
+}
diff --git a/offload/test/offloading/fortran/target-has-device-addr1.f90 b/offload/test/offloading/fortran/target-has-device-addr1.f90
new file mode 100644
index 0000000000000..b97e5a99de82d
--- /dev/null
+++ b/offload/test/offloading/fortran/target-has-device-addr1.f90
@@ -0,0 +1,45 @@
+!REQUIRES: flang, amdgpu
+
+!Test derived from the sollve test for has-device-addr.
+
+!RUN: %libomptarget-compile-fortran-run-and-check-generic
+
+module m
+  use iso_c_binding
+
+contains
+  subroutine target_has_device_addr()
+    integer, target :: x
+    integer, pointer :: first_scalar_device_addr
+    type(c_ptr) :: cptr_scalar1
+
+    integer :: res1, res2
+
+    nullify (first_scalar_device_addr)
+    x = 10
+
+    !$omp target enter data map(to: x)
+    !$omp target data use_device_addr(x)
+      x = 11
+      cptr_scalar1 = c_loc(x)
+    !$omp end target data
+
+    call c_f_pointer (cptr_scalar1, first_scalar_device_addr)
+
+    !$omp target map(to: x) map(from: res1, res2) &
+    !$omp & has_device_addr(first_scalar_device_addr)
+      res1 = first_scalar_device_addr
+      res2 = x
+    !$omp end target
+    print *, "res1", res1, "res2", res2
+  end subroutine
+end module
+
+
+program p
+  use m
+
+  call target_has_device_addr()
+end
+
+!CHECK: res1 11 res2 11
diff --git a/offload/test/offloading/fortran/target-has-device-addr2.f90 b/offload/test/offloading/fortran/target-has-device-addr2.f90
new file mode 100644
index 0000000000000..993ac2e251305
--- /dev/null
+++ b/offload/test/offloading/fortran/target-has-device-addr2.f90
@@ -0,0 +1,57 @@
+!REQUIRES: flang, amdgpu
+
+!RUN: %libomptarget-compile-fortran-run-and-check-generic
+
+subroutine f
+  type :: t1
+    integer :: x, y, z
+  end type
+
+  integer, parameter :: n = 9
+  type(t1) :: b(n)
+
+  integer :: i
+  do i = 1, n
+    b(i)%x = 0
+    b(i)%y = 0
+    b(i)%z = 0
+  enddo
+
+  !$omp target data map(tofrom: b(1:3)) use_device_addr(b)
+  !$omp target has_device_addr(b(2)%x)
+    b(2)%x = 1
+  !$omp end target
+  !$omp end target data
+  print *, "b1", b
+end
+
+subroutine g
+  type :: t1
+    integer :: x(3), y(7), z(5)
+  end type
+
+  integer, parameter :: n = 5
+  type(t1) :: b(n)
+
+  integer :: i
+  do i = 1, n
+    b(i)%x = 0
+    b(i)%y = 0
+    b(i)%z = 0
+  enddo
+
+  !$omp target data map(tofrom: b(1:3)) use_device_addr(b)
+  !$omp target has_device_addr(b(2)%x)
+    b(2)%x(3) = 1
+  !$omp end target
+  !$omp end target data
+  print *, "b2", b
+end
+
+call f()
+call g()
+end
+
+!CHECK: b1 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
+!CHECK: b2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
+
diff --git a/offload/test/offloading/fortran/target-has-device-addr3.f90 b/offload/test/offloading/fortran/target-has-device-addr3.f90
new file mode 100644
index 0000000000000..b4f3fd2c84efb
--- /dev/null
+++ b/offload/test/offloading/fortran/target-has-device-addr3.f90
@@ -0,0 +1,30 @@
+!REQUIRES: flang, amdgpu
+
+!RUN: %libomptarget-compile-fortran-run-and-check-generic
+
+function f(x) result(y)
+  integer :: x(:)
+  integer :: y, z
+  x = 0
+  y = 11
+  !$omp target data map(tofrom: x) use_device_addr(x)
+  !$omp target has_device_addr(x) map(tofrom: y)
+  y = size(x)
+  !$omp end target
+  !$omp end target data
+end
+
+program main
+  interface
+    function f(x) result(y)
+      integer :: x(:)
+      integer :: y
+    end function
+  end interface
+  integer :: x(13)
+  integer :: y
+  y = f(x)
+  print *, "y=", y
+end
+
+!CHECK: y= 13
diff --git a/offload/test/offloading/fortran/target-use-dev-ptr.f90 b/offload/test/offloading/fortran/target-use-dev-ptr.f90
index 4476f45699d6e..069d4a717ff5e 100644
--- a/offload/test/offloading/fortran/target-use-dev-ptr.f90
+++ b/offload/test/offloading/fortran/target-use-dev-ptr.f90
@@ -18,7 +18,7 @@ end function get_ptr
       integer(c_int) function check_result(host, dev) BIND(C)
          USE, intrinsic :: iso_c_binding
          implicit none
-         type(c_ptr), intent(in) :: host, dev
+         type(c_ptr), value, intent(in) :: host, dev
       end function check_result
    end interface
 
diff --git a/offload/test/offloading/fortran/target_map_ompx_hold.f90 b/offload/test/offloading/fortran/target_map_ompx_hold.f90
new file mode 100644
index 0000000000000..2a1e9b1423edd
--- /dev/null
+++ b/offload/test/offloading/fortran/target_map_ompx_hold.f90
@@ -0,0 +1,29 @@
+! Basic test that checks that when ompx_hold is in use we cannot delete the data
+! until the ompx_hold falls out of scope, and verifies this via the utilisation of
+! present.
+! REQUIRES: flang, amdgpu
+! RUN: %libomptarget-compile-fortran-generic
+! RUN: %libomptarget-run-fail-generic 2>&1 \
+! RUN: | %fcheck-generic
+
+program ompx_hold
+    implicit none
+    integer :: presence_check
+
+!CHECK-NOT: omptarget message: device mapping required by 'present' map type modifier does not exist for host address{{.*}}
+!$omp target data map(ompx_hold, tofrom: presence_check)
+!$omp target exit data map(delete: presence_check)
+!$omp target map(present, tofrom: presence_check)
+    presence_check = 10
+!$omp end target
+!$omp end target data
+
+!CHECK: omptarget message: device mapping required by 'present' map type modifier does not exist for host address{{.*}}
+!$omp target data map(tofrom: presence_check)
+!$omp target exit data map(delete: presence_check)
+!$omp target map(present, tofrom: presence_check)
+presence_check = 20
+!$omp end target
+!$omp end target data
+
+end program
diff --git a/offload/test/offloading/fortran/target_map_present_fail.f90 b/offload/test/offloading/fortran/target_map_present_fail.f90
new file mode 100644
index 0000000000000..2845bf1e064fb
--- /dev/null
+++ b/offload/test/offloading/fortran/target_map_present_fail.f90
@@ -0,0 +1,36 @@
+! This checks that the basic functionality of map type present functions as
+! expected, emitting an omptarget error when the data is not present.
+! REQUIRES: flang, amdgpu
+! RUN: %libomptarget-compile-fortran-generic
+! RUN: %libomptarget-run-fail-generic 2>&1 \
+! RUN: | %fcheck-generic
+
+! NOTE: This should intentionally fatal error in omptarget as it's not
+! present, as is intended.
+ subroutine target_data_not_present()
+    double precision, dimension(:), allocatable :: arr
+    integer, parameter :: N = 16
+    integer :: i
+
+    allocate(arr(N))
+
+!$omp target data map(present,alloc:arr)
+
+!$omp target
+    do i = 1,N
+        arr(i) = 42.0d0
+    end do
+!$omp end target
+
+!$omp end target data
+
+    deallocate(arr)
+    return
+end subroutine
+
+program map_present
+    implicit none
+    call target_data_not_present()
+end program
+
+!CHECK: omptarget message: device mapping required by 'present' map type modifier does not exist for host address{{.*}}
diff --git a/offload/test/offloading/fortran/target_map_present_success.f90 b/offload/test/offloading/fortran/target_map_present_success.f90
new file mode 100644
index 0000000000000..63d5e80500249
--- /dev/null
+++ b/offload/test/offloading/fortran/target_map_present_success.f90
@@ -0,0 +1,41 @@
+! This checks that the basic functionality of map type present functions as
+! expected, no-op'ng when present
+! REQUIRES: flang, amdgpu
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+
+ subroutine target_data_present()
+    double precision, dimension(:), allocatable :: arr
+    integer, parameter :: N = 16
+    integer :: i
+
+    allocate(arr(N))
+
+    arr(:) = 10.0d0
+
+!$omp target data map(tofrom:arr)
+
+!$omp target data map(present,alloc:arr)
+
+!$omp target
+    do i = 1,N
+        arr(i) = 42.0d0
+    end do
+!$omp end target
+
+!$omp end target data
+
+!$omp end target data
+
+    print *, arr
+
+    deallocate(arr)
+
+    return
+end subroutine
+
+program map_present
+    implicit none
+   call target_data_present()
+end program
+
+!CHECK: 42. 42. 42. 42. 42. 42. 42. 42. 42. 42. 42. 42. 42. 42. 42. 42.
diff --git a/offload/test/offloading/fortran/usm_map_close.f90 b/offload/test/offloading/fortran/usm_map_close.f90
new file mode 100644
index 0000000000000..61d096ef94f1b
--- /dev/null
+++ b/offload/test/offloading/fortran/usm_map_close.f90
@@ -0,0 +1,85 @@
+! Test for map type close, verifying it appropriately places memory
+! near/on device when utilised in USM mode.
+! REQUIRES: clang, flang, amdgpu
+
+! RUN: %clang -c -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa \
+! RUN:   %S/../../Inputs/target-use-dev-ptr.c -o target-use-dev-ptr_c.o
+! RUN: %libomptarget-compile-fortran-generic target-use-dev-ptr_c.o
+! RUN: env HSA_XNACK=1 \
+! RUN: %libomptarget-run-generic | %fcheck-generic
+
+program use_device_test
+    use iso_c_binding
+    implicit none
+    interface
+       type(c_ptr) function get_ptr() BIND(C)
+          USE, intrinsic :: iso_c_binding
+          implicit none
+       end function get_ptr
+ 
+       integer(c_int) function check_equality(host, dev) BIND(C)
+          USE, intrinsic :: iso_c_binding
+          implicit none
+          type(c_ptr), value, intent(in) :: host, dev
+       end function check_equality
+    end interface
+    type(c_ptr) :: host_alloc, device_alloc
+    integer, pointer :: a
+  !$omp requires unified_shared_memory
+
+    allocate(a)
+    host_alloc = C_LOC(a)
+
+! map + target no close
+device_alloc = c_null_ptr
+!$omp target data map(tofrom: a, device_alloc)
+!$omp target map(tofrom: device_alloc)
+    device_alloc = C_LOC(a)
+!$omp end target
+!$omp end target data
+
+! CHECK: a used from unified memory
+if (check_equality(host_alloc, device_alloc) == 1) then
+    print*, "a used from unified memory"
+end if
+
+! map + target with close
+device_alloc = c_null_ptr
+!$omp target data map(close, tofrom: a) map(tofrom: device_alloc)
+!$omp target map(tofrom: device_alloc)
+    device_alloc = C_LOC(a)
+!$omp end target
+!$omp end target data
+
+! CHECK: a copied to device
+if (check_equality(host_alloc, device_alloc) == 0) then
+    print *, "a copied to device"
+end if
+
+! map + use_device_ptr no close
+device_alloc = c_null_ptr
+!$omp target data map(tofrom: a) use_device_ptr(a)
+    device_alloc = C_LOC(a)
+!$omp end target data
+
+! CHECK: a used from unified memory with use_device_ptr
+if (check_equality(host_alloc, device_alloc) == 1) then
+    print *, "a used from unified memory with use_device_ptr"
+end if
+
+! map enter/exit + close
+device_alloc = c_null_ptr
+!$omp target enter data map(close, to: a)
+
+!$omp target map(from: device_alloc)
+    device_alloc = C_LOC(a)
+!$omp end target
+
+!$omp target exit data map(from: a)
+
+! CHECK: a has been mapped to the device
+if (check_equality(host_alloc, device_alloc) == 0) then
+    print *, "a has been mapped to the device"
+end if
+
+end program use_device_test
diff --git a/openmp/runtime/test/lit.cfg b/openmp/runtime/test/lit.cfg
index 14c7468982137..2a9ae05f97552 100644
--- a/openmp/runtime/test/lit.cfg
+++ b/openmp/runtime/test/lit.cfg
@@ -137,6 +137,9 @@ if config.operating_system in ['Windows', 'Linux', 'FreeBSD', 'NetBSD', 'DragonF
 if config.operating_system in ['Linux']:
     config.available_features.add('hidden-helper')
 
+if config.compiler_frontend_variant == 'MSVC' or config.compiler_simulate_id == 'MSVC':
+    config.available_features.add("msvc")
+
 target_arch = getattr(config, 'target_arch', None)
 if target_arch:
   config.available_features.add(target_arch + '-target-arch')
diff --git a/openmp/runtime/test/lit.site.cfg.in b/openmp/runtime/test/lit.site.cfg.in
index d6c259280619b..d0e54eac0dc3a 100644
--- a/openmp/runtime/test/lit.site.cfg.in
+++ b/openmp/runtime/test/lit.site.cfg.in
@@ -21,6 +21,8 @@ config.has_libatomic = @LIBOMP_HAVE_LIBATOMIC@
 config.is_standalone_build = @OPENMP_STANDALONE_BUILD@
 config.has_omit_frame_pointer_flag = @OPENMP_TEST_COMPILER_HAS_OMIT_FRAME_POINTER_FLAGS@
 config.target_arch = "@LIBOMP_ARCH@"
+config.compiler_frontend_variant = "@CMAKE_C_COMPILER_FRONTEND_VARIANT@"
+config.compiler_simulate_id = "@CMAKE_C_SIMULATE_ID@"
 
 # Let the main config do the real work.
 lit_config.load_config(config, "@LIBOMP_BASE_DIR@/test/lit.cfg")
diff --git a/openmp/runtime/test/transform/interchange/iterfor.cpp b/openmp/runtime/test/transform/interchange/iterfor.cpp
index 51219a07402e3..98a398fe8925e 100644
--- a/openmp/runtime/test/transform/interchange/iterfor.cpp
+++ b/openmp/runtime/test/transform/interchange/iterfor.cpp
@@ -1,3 +1,5 @@
+// XFAIL: msvc
+// Fails on windows due to issue #129023
 // RUN: %libomp-cxx20-compile-and-run | FileCheck %s --match-full-lines
 
 #ifndef HEADER
diff --git a/openmp/runtime/test/transform/tile/iterfor.cpp b/openmp/runtime/test/transform/tile/iterfor.cpp
index 12613544f6e5b..05b5677bf9c4f 100644
--- a/openmp/runtime/test/transform/tile/iterfor.cpp
+++ b/openmp/runtime/test/transform/tile/iterfor.cpp
@@ -1,3 +1,5 @@
+// XFAIL: msvc
+// Fails on windows due to issue #129023
 // RUN: %libomp-cxx20-compile-and-run | FileCheck %s --match-full-lines
 
 #ifndef HEADER
diff --git a/openmp/runtime/test/worksharing/for/omp_for_collapse_LowerTriangularLess.c b/openmp/runtime/test/worksharing/for/omp_for_collapse_LowerTriangularLess.c
index 9d742066cf1fc..c11e9ec43f4c0 100644
--- a/openmp/runtime/test/worksharing/for/omp_for_collapse_LowerTriangularLess.c
+++ b/openmp/runtime/test/worksharing/for/omp_for_collapse_LowerTriangularLess.c
@@ -1,3 +1,5 @@
+// XFAIL: msvc
+// Fails on windows due to issue #129023
 // RUN: %libomp-compile-and-run
 #include <stdio.h>
 #include <stdlib.h>
diff --git a/openmp/runtime/test/worksharing/for/omp_for_collapse_LowerTriangularLessEqual.c b/openmp/runtime/test/worksharing/for/omp_for_collapse_LowerTriangularLessEqual.c
index 154ee0f69daa5..3249594664489 100644
--- a/openmp/runtime/test/worksharing/for/omp_for_collapse_LowerTriangularLessEqual.c
+++ b/openmp/runtime/test/worksharing/for/omp_for_collapse_LowerTriangularLessEqual.c
@@ -1,3 +1,5 @@
+// XFAIL: msvc
+// Fails on windows due to issue #129023
 // RUN: %libomp-compile-and-run
 #include <stdio.h>
 #include <stdlib.h>
diff --git a/openmp/runtime/test/worksharing/for/omp_for_collapse_UpperTriangular.c b/openmp/runtime/test/worksharing/for/omp_for_collapse_UpperTriangular.c
index 452410025be0c..cc10ddb0e9104 100644
--- a/openmp/runtime/test/worksharing/for/omp_for_collapse_UpperTriangular.c
+++ b/openmp/runtime/test/worksharing/for/omp_for_collapse_UpperTriangular.c
@@ -1,3 +1,5 @@
+// XFAIL: msvc
+// Fails on windows due to issue #129023
 // RUN: %libomp-compile-and-run
 #include <stdio.h>
 #include <stdlib.h>
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 5339b30154ed1..757db65ca8a77 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -1598,9 +1598,9 @@ libc_support_library(
     ],
 )
 
-############################### errno targets ################################
+############################### errno ########################################
 
-libc_function(
+libc_support_library(
     name = "errno",
     srcs = ["src/errno/libc_errno.cpp"],
     hdrs = ["src/errno/libc_errno.h"],
@@ -2198,6 +2198,20 @@ libc_math_function(
     ],
 )
 
+libc_math_function(
+    name = "acosf16",
+    additional_deps = [
+        ":__support_fputil_cast",
+        ":__support_fputil_fma",
+        ":__support_fputil_multiply_add",
+        ":__support_fputil_nearest_integer",
+        ":__support_fputil_polyeval",
+        ":__support_fputil_sqrt",
+        ":__support_macros_optimization",
+        ":inv_trigf_utils",
+    ],
+)
+
 libc_math_function(
     name = "acoshf",
     additional_deps = [
@@ -2226,6 +2240,20 @@ libc_math_function(
     ],
 )
 
+libc_math_function(
+    name = "asinf16",
+    additional_deps = [
+        ":__support_fputil_cast",
+        ":__support_fputil_fma",
+        ":__support_fputil_multiply_add",
+        ":__support_fputil_nearest_integer",
+        ":__support_fputil_polyeval",
+        ":__support_fputil_sqrt",
+        ":__support_macros_optimization",
+        ":inv_trigf_utils",
+    ],
+)
+
 libc_math_function(
     name = "asinhf",
     additional_deps = [
@@ -4242,7 +4270,6 @@ libc_function(
     hdrs = ["src/string/strcpy.h"],
     deps = [
         ":__support_common",
-        ":memcpy",
         ":string_memory_utils",
         ":string_utils",
     ],
diff --git a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
index 09daa9ecb3675..0d60992bbd638 100644
--- a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
@@ -64,50 +64,72 @@ def _libc_library(name, copts = [], deps = [], local_defines = [], **kwargs):
         **kwargs
     )
 
+def _libc_library_filegroups(
+        name,
+        is_function,
+        srcs = [],
+        hdrs = [],
+        textual_hdrs = [],
+        deps = [],
+        # We're not using kwargs, but instead explicitly list all possible
+        # arguments that can be passed to libc_support_library or
+        # libc_function macros. This is done to limit the configurability
+        # and ensure the consistent and tightly controlled set of flags
+        # (see libc_common_copts and libc_release_copts above) is used to build
+        # libc code both for tests and for release configuration.
+        target_compatible_with = None,  # @unused
+        weak = False):  # @unused
+    """Internal macro to collect sources and headers required to build a library.
+    """
+
+    # filegroups created from "libc_function" macro has an extra "_fn" in their
+    # name to ensure that no other libc target can depend on libc_function.
+    prefix = name + ("_fn" if is_function else "")
+    native.filegroup(
+        name = prefix + "_srcs",
+        srcs = srcs + hdrs + [dep + "_srcs" for dep in deps],
+    )
+    native.filegroup(
+        name = prefix + "_textual_hdrs",
+        srcs = textual_hdrs + [dep + "_textual_hdrs" for dep in deps],
+    )
+
 # A convenience function which should be used to list all libc support libraries.
 # Any library which does not define a public function should be listed with
 # libc_support_library.
 def libc_support_library(name, **kwargs):
     _libc_library(name = name, **kwargs)
+    _libc_library_filegroups(name = name, is_function = False, **kwargs)
 
 def libc_function(
         name,
-        srcs,
         weak = False,
-        copts = [],
-        local_defines = [],
         **kwargs):
     """Add target for a libc function.
 
-    The libc function is eventually available as a cc_library target by name
-    "name". LLVM libc implementations of libc functions are in C++. So, this
-    rule internally generates a C wrapper for the C++ implementation and adds
-    it to the source list of the cc_library. This way, the C++ implementation
-    and the C wrapper are both available in the cc_library.
+    This macro creates an internal cc_library that can be used to test this
+    function, and creates filegroups required to include this function into
+    a release build of libc.
 
     Args:
       name: Target name. It is normally the name of the function this target is
             for.
-      srcs: The .cpp files which contain the function implementation.
       weak: Make the symbol corresponding to the libc function "weak".
-      copts: The list of options to add to the C++ compilation command.
-      local_defines: The preprocessor defines which will be prepended with -D
-                     and passed to the compile command of this target but not
-                     its deps.
       **kwargs: Other attributes relevant for a cc_library. For example, deps.
     """
 
-    # We compile the code twice, the first target is suffixed with ".__internal__" and contains the
+    # Build "internal" library with a function, the target has ".__internal__" suffix and contains
     # C++ functions in the "LIBC_NAMESPACE" namespace. This allows us to test the function in the
     # presence of another libc.
-    libc_support_library(
+    _libc_library(
         name = libc_internal_target(name),
-        srcs = srcs,
-        copts = copts,
-        local_defines = local_defines,
         **kwargs
     )
 
+    _libc_library_filegroups(name = name, is_function = True, **kwargs)
+
+
+    # TODO(PR #130327): Remove this after downstream uses are migrated to libc_release_library.
     # This second target is the llvm libc C function with default visibility.
     func_attrs = [
         "LLVM_LIBC_FUNCTION_ATTR_" + name + "='LLVM_LIBC_EMPTY, [[gnu::weak]]'",
@@ -115,9 +137,49 @@ def libc_function(
 
     _libc_library(
         name = name,
-        srcs = srcs,
-        copts = copts + libc_release_copts(),
-        local_defines = local_defines + func_attrs,
+        copts = libc_release_copts(),
+        local_defines = func_attrs,
+        **kwargs
+    )
+
+def libc_release_library(
+        name,
+        libc_functions,
+        weak_symbols = [],
+        **kwargs):
+    """Create the release version of a libc library.
+
+    Args:
+        name: Name of the cc_library target.
+        libc_functions: List of functions to include in the library. They should be
+            created by libc_function macro.
+        weak_symbols: List of function names that should be marked as weak symbols.
+        **kwargs: Other arguments relevant to cc_library.
+    """
+    # Combine all sources into a single filegroup to avoid repeated sources error.
+    native.filegroup(
+        name = name + "_srcs",
+        srcs = [function + "_fn_srcs" for function in libc_functions],
+    )
+
+    native.cc_library(
+        name = name + "_textual_hdr_library",
+        textual_hdrs = [function + "_fn_textual_hdrs" for function in libc_functions],
+    )
+
+    weak_attributes = [
+        "LLVM_LIBC_FUNCTION_ATTR_" + name + "='LLVM_LIBC_EMPTY, [[gnu::weak]]'"
+        for name in weak_symbols
+    ]
+
+    native.cc_library(
+        name = name,
+        srcs = [":" + name + "_srcs"],
+        copts = libc_common_copts() + libc_release_copts(),
+        local_defines = weak_attributes + LIBC_CONFIGURE_OPTIONS,
+        deps = [
+            ":" + name + "_textual_hdr_library",
+        ],
         **kwargs
     )
 
diff --git a/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl b/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl
index f65da9e98226b..672ce8c0723ec 100644
--- a/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl
@@ -44,6 +44,11 @@ LIBC_CONFIGURE_OPTIONS = [
     # "LIBC_COPT_STRTOFLOAT_DISABLE_EISEL_LEMIRE",
     # "LIBC_COPT_STRTOFLOAT_DISABLE_SIMPLE_DECIMAL_CONVERSION",
 
+    # Documentation in libc/src/errno/...
+    # Since we're only using Bazel for overlay build for now, explicitly
+    # enforce using the system-provided errno in both tests and release build.
+    "LIBC_ERRNO_MODE=LIBC_ERRNO_MODE_SYSTEM",
+
     # Documentation in libc/src/__support/libc_assert.h
     # "LIBC_COPT_USE_C_ASSERT",
 ]
diff --git a/utils/bazel/llvm-project-overlay/libc/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/BUILD.bazel
index 70b0196469eca..ccad771025d9a 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/BUILD.bazel
@@ -2,6 +2,26 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
+load("//libc:libc_build_rules.bzl", "libc_release_library")
+
 package(default_visibility = ["//visibility:public"])
 
 exports_files(["libc_test_rules.bzl"])
+
+# Smoke test verifying libc_release_library macro functionality.
+libc_release_library(
+    name = "libc_release_test",
+    libc_functions = [
+        "//libc:acosf",
+        "//libc:read",
+    ],
+    weak_symbols = [
+        "read",
+    ],
+)
+
+build_test(
+    name = "libc_release_build_test",
+    targets = [":libc_release_test"],
+)
diff --git a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
index c5ddb01eb123d..cec3fc16c62d1 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
@@ -4,13 +4,13 @@
 
 # LLVM libc unittest library.
 
-load("//libc:libc_build_rules.bzl", "libc_support_library")
+load("//libc/test:libc_test_rules.bzl", "libc_test_library")
 
 package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])
 
-libc_support_library(
+libc_test_library(
     name = "test_logger",
     srcs = ["TestLogger.cpp"],
     hdrs = ["TestLogger.h"],
@@ -29,7 +29,7 @@ libc_support_library(
     ],
 )
 
-libc_support_library(
+libc_test_library(
     name = "LibcUnitTest",
     srcs = [
         "BazelFilePath.cpp",
@@ -73,7 +73,7 @@ libc_support_library(
     ],
 )
 
-libc_support_library(
+libc_test_library(
     name = "fp_test_helpers",
     srcs = [
         "FEnvSafeTest.cpp",
@@ -108,7 +108,7 @@ libc_support_library(
     ],
 )
 
-libc_support_library(
+libc_test_library(
     name = "memory_matcher",
     srcs = [
         "MemoryMatcher.cpp",
@@ -127,7 +127,7 @@ libc_support_library(
     ],
 )
 
-libc_support_library(
+libc_test_library(
     name = "printf_matcher",
     srcs = [
         "PrintfMatcher.cpp",
@@ -144,7 +144,7 @@ libc_support_library(
     ],
 )
 
-libc_support_library(
+libc_test_library(
     name = "string_utils",
     hdrs = [
         "StringUtils.h",
diff --git a/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl b/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
index 4b44b28ea9876..8c20a9172989c 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
@@ -27,14 +27,14 @@ def libc_test(name, srcs, libc_function_deps = [], copts = [], deps = [], local_
       local_defines: The list of target local_defines if any.
       **kwargs: Attributes relevant for a libc_test. For example, name, srcs.
     """
-    all_function_deps = libc_function_deps + ["//libc:errno"]
     native.cc_test(
         name = name,
         srcs = srcs,
         local_defines = local_defines + LIBC_CONFIGURE_OPTIONS,
-        deps = [libc_internal_target(d) for d in all_function_deps] + [
+        deps = [libc_internal_target(d) for d in libc_function_deps] + [
             "//libc/test/UnitTest:LibcUnitTest",
             "//libc:__support_macros_config",
+            "//libc:errno",
             "//libc:func_aligned_alloc",
             "//libc:func_free",
             "//libc:func_malloc",
@@ -44,3 +44,21 @@ def libc_test(name, srcs, libc_function_deps = [], copts = [], deps = [], local_
         linkstatic = 1,
         **kwargs
     )
+
+def libc_test_library(name, copts = [], local_defines = [], **kwargs):
+    """Add target for library used in libc tests.
+
+    Args:
+      name: Library target name.
+      copts: See cc_library.copts.
+      local_defines: See cc_library.local_defines.
+      **kwargs: Other attributes relevant to cc_library (e.g. "deps").
+    """
+    native.cc_library(
+        name = name,
+        testonly = True,
+        copts = copts + libc_common_copts(),
+        local_defines = local_defines + LIBC_CONFIGURE_OPTIONS,
+        linkstatic = 1,
+        **kwargs
+    )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/math/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/math/BUILD.bazel
index 764c845435611..2154e71018226 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/math/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/math/BUILD.bazel
@@ -47,6 +47,11 @@ math_mpfr_test(
     hdrs = ["CeilTest.h"],
 )
 
+math_mpfr_test(
+    name = "ceilf16",
+    hdrs = ["CeilTest.h"],
+)
+
 math_mpfr_test(name = "cos")
 
 math_mpfr_test(
@@ -144,6 +149,21 @@ math_mpfr_test(
     hdrs = ["AddTest.h"],
 )
 
+math_mpfr_test(
+    name = "f16add",
+    hdrs = ["AddTest.h"],
+)
+
+math_mpfr_test(
+    name = "f16addf",
+    hdrs = ["AddTest.h"],
+)
+
+math_mpfr_test(
+    name = "f16addl",
+    hdrs = ["AddTest.h"],
+)
+
 math_mpfr_test(
     name = "fdiv",
     hdrs = ["DivTest.h"],
@@ -154,6 +174,21 @@ math_mpfr_test(
     hdrs = ["DivTest.h"],
 )
 
+math_mpfr_test(
+    name = "f16div",
+    hdrs = ["DivTest.h"],
+)
+
+math_mpfr_test(
+    name = "f16divf",
+    hdrs = ["DivTest.h"],
+)
+
+math_mpfr_test(
+    name = "f16divl",
+    hdrs = ["DivTest.h"],
+)
+
 # TODO: Add ffma, ffmal test.  Missing stdlib/rand dependency.
 
 math_mpfr_test(
@@ -171,8 +206,21 @@ math_mpfr_test(
     hdrs = ["FloorTest.h"],
 )
 
+math_mpfr_test(
+    name = "floorf16",
+    hdrs = ["FloorTest.h"],
+)
+
 # TODO: Add fma, fmaf, fmal, fmul, fmull tests.  Missing stdlib/rand dependency.
 
+# math_mpfr_test(name = "f16mul")
+# math_mpfr_test(name = "f16mulf")
+# math_mpfr_test(name = "f16mull")
+
+# math_mpfr_test(name = "f16fma")
+# math_mpfr_test(name = "f16fmaf")
+# math_mpfr_test(name = "f16fmal")
+
 math_mpfr_test(
     name = "frexp",
     hdrs = ["FrexpTest.h"],
@@ -208,6 +256,21 @@ math_mpfr_test(
     hdrs = ["SubTest.h"],
 )
 
+math_mpfr_test(
+    name = "f16sub",
+    hdrs = ["SubTest.h"],
+)
+
+math_mpfr_test(
+    name = "f16subf",
+    hdrs = ["SubTest.h"],
+)
+
+math_mpfr_test(
+    name = "f16subl",
+    hdrs = ["SubTest.h"],
+)
+
 math_mpfr_test(
     name = "hypot",
     hdrs = ["HypotTest.h"],
@@ -236,6 +299,11 @@ math_mpfr_test(
     hdrs = ["RoundToIntegerTest.h"],
 )
 
+math_mpfr_test(
+    name = "llrintf16",
+    hdrs = ["RoundToIntegerTest.h"],
+)
+
 math_mpfr_test(
     name = "llround",
     hdrs = ["RoundToIntegerTest.h"],
@@ -251,6 +319,16 @@ math_mpfr_test(
     hdrs = ["RoundToIntegerTest.h"],
 )
 
+math_mpfr_test(
+    name = "lroundf16",
+    hdrs = ["RoundToIntegerTest.h"],
+)
+
+math_mpfr_test(
+    name = "llroundf16",
+    hdrs = ["RoundToIntegerTest.h"],
+)
+
 math_mpfr_test(name = "log")
 
 math_mpfr_test(name = "logf")
@@ -282,6 +360,11 @@ math_mpfr_test(
     hdrs = ["RoundToIntegerTest.h"],
 )
 
+math_mpfr_test(
+    name = "lrintf16",
+    hdrs = ["RoundToIntegerTest.h"],
+)
+
 math_mpfr_test(
     name = "lround",
     hdrs = ["RoundToIntegerTest.h"],
@@ -323,6 +406,14 @@ math_mpfr_test(
     ],
 )
 
+math_mpfr_test(
+    name = "nearbyintf16",
+    hdrs = ["NearbyIntTest.h"],
+    deps = [
+        "//libc:__support_cpp_array",
+    ],
+)
+
 math_mpfr_test(name = "pow")
 
 math_mpfr_test(name = "powf")
@@ -357,6 +448,11 @@ math_mpfr_test(
     hdrs = ["RIntTest.h"],
 )
 
+math_mpfr_test(
+    name = "rintf16",
+    hdrs = ["RIntTest.h"],
+)
+
 math_mpfr_test(
     name = "roundeven",
     hdrs = ["RoundEvenTest.h"],
@@ -372,6 +468,11 @@ math_mpfr_test(
     hdrs = ["RoundEvenTest.h"],
 )
 
+math_mpfr_test(
+    name = "roundevenf16",
+    hdrs = ["RoundEvenTest.h"],
+)
+
 math_mpfr_test(
     name = "round",
     hdrs = ["RoundTest.h"],
@@ -387,6 +488,11 @@ math_mpfr_test(
     hdrs = ["RoundTest.h"],
 )
 
+math_mpfr_test(
+    name = "roundf16",
+    hdrs = ["RoundTest.h"],
+)
+
 math_mpfr_test(name = "sin")
 
 math_mpfr_test(
@@ -437,6 +543,21 @@ math_mpfr_test(
     hdrs = ["SqrtTest.h"],
 )
 
+math_mpfr_test(
+    name = "f16sqrt",
+    hdrs = ["SqrtTest.h"],
+)
+
+math_mpfr_test(
+    name = "f16sqrtf",
+    hdrs = ["SqrtTest.h"],
+)
+
+math_mpfr_test(
+    name = "f16sqrtl",
+    hdrs = ["SqrtTest.h"],
+)
+
 math_mpfr_test(name = "tan")
 
 math_mpfr_test(
@@ -463,3 +584,46 @@ math_mpfr_test(
     name = "truncl",
     hdrs = ["TruncTest.h"],
 )
+
+math_mpfr_test(
+    name = "truncf16",
+    hdrs = ["TruncTest.h"],
+)
+
+math_mpfr_test(name = "cosf16")
+
+math_mpfr_test(name = "cospif16")
+
+math_mpfr_test(name = "sinf16")
+
+math_mpfr_test(name = "sinpif16")
+
+math_mpfr_test(name = "tanf16")
+
+math_mpfr_test(name = "tanpif16")
+
+math_mpfr_test(name = "expf16")
+
+math_mpfr_test(name = "exp2f16")
+
+math_mpfr_test(name = "exp2m1f16")
+
+math_mpfr_test(name = "exp10f16")
+
+math_mpfr_test(name = "exp10m1f16")
+
+math_mpfr_test(name = "asinf16")
+
+math_mpfr_test(name = "acosf16")
+
+math_mpfr_test(name = "coshf16")
+
+math_mpfr_test(name = "sinhf16")
+
+math_mpfr_test(name = "tanhf16")
+
+math_mpfr_test(name = "logf16")
+
+math_mpfr_test(name = "log2f16")
+
+math_mpfr_test(name = "log10f16")
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel
index c56f3c7cf4f34..803548178b0e3 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel
@@ -14,8 +14,12 @@ math_test(name = "acosf")
 
 math_test(name = "acoshf")
 
+math_test(name = "acosf16")
+
 math_test(name = "asinf")
 
+math_test(name = "asinf16")
+
 math_test(name = "asinhf")
 
 math_test(name = "atan2")
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel
index 73685ede41e85..5f43ec7c7a109 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/stdlib/BUILD.bazel
@@ -4,8 +4,7 @@
 
 # Tests for LLVM libc stdlib.h functions.
 
-load("//libc:libc_build_rules.bzl", "libc_support_library")
-load("//libc/test:libc_test_rules.bzl", "libc_test")
+load("//libc/test:libc_test_rules.bzl", "libc_test", "libc_test_library")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -29,7 +28,7 @@ libc_test(
     libc_function_deps = ["//libc:llabs"],
 )
 
-libc_support_library(
+libc_test_library(
     name = "div_test_helper",
     hdrs = ["DivTest.h"],
     deps = ["//libc/test/UnitTest:LibcUnitTest"],
@@ -65,7 +64,7 @@ libc_test(
     ],
 )
 
-libc_support_library(
+libc_test_library(
     name = "atoi_test_helper",
     hdrs = ["AtoiTest.h"],
     deps = [
@@ -110,12 +109,11 @@ libc_test(
     deps = ["//libc:types_size_t"],
 )
 
-libc_support_library(
+libc_test_library(
     name = "qsort_test_helper",
     hdrs = ["SortingTest.h"],
     deps = [
         "//libc:__support_macros_config",
-        "//libc:qsort",
         "//libc/test/UnitTest:LibcUnitTest",
     ],
 )
@@ -149,7 +147,7 @@ libc_test(
     deps = ["//libc:types_size_t"],
 )
 
-libc_support_library(
+libc_test_library(
     name = "strfrom_test_helper",
     hdrs = ["StrfromTest.h"],
     deps = [
@@ -180,7 +178,7 @@ libc_test(
     deps = [":strfrom_test_helper"],
 )
 
-libc_support_library(
+libc_test_library(
     name = "strtol_test_helper",
     hdrs = ["StrtolTest.h"],
     deps = [
@@ -188,7 +186,7 @@ libc_support_library(
         "//libc:__support_cpp_type_traits",
         "//libc:__support_ctype_utils",
         "//libc:__support_macros_properties_architectures",
-        "//libc:errno.__internal__",
+        "//libc:errno",
         "//libc/test/UnitTest:LibcUnitTest",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
index a36cf2e0cb7d9..7274819e6758c 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
@@ -4,8 +4,7 @@
 
 # Tests for LLVM libc string.h functions.
 
-load("//libc:libc_build_rules.bzl", "libc_support_library")
-load("//libc/test:libc_test_rules.bzl", "libc_test")
+load("//libc/test:libc_test_rules.bzl", "libc_test", "libc_test_library")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -43,7 +42,7 @@ libc_test(
     ],
 )
 
-libc_support_library(
+libc_test_library(
     name = "strchr_test_helper",
     hdrs = ["StrchrTest.h"],
     deps = ["//libc/test/UnitTest:LibcUnitTest"],
@@ -115,7 +114,7 @@ libc_test(
     ],
 )
 
-libc_support_library(
+libc_test_library(
     name = "memory_check_utils",
     hdrs = ["memory_utils/memory_check_utils.h"],
     deps = [
@@ -127,7 +126,7 @@ libc_support_library(
     ],
 )
 
-libc_support_library(
+libc_test_library(
     name = "protected_pages",
     hdrs = ["memory_utils/protected_pages.h"],
     deps = [
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/strings/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/strings/BUILD.bazel
index 8412d5c58d2bd..bcf4b4201e043 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/strings/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/strings/BUILD.bazel
@@ -4,7 +4,6 @@
 
 # Tests for LLVM libc strings.h functions.
 
-load("//libc:libc_build_rules.bzl", "libc_support_library")
 load("//libc/test:libc_test_rules.bzl", "libc_test")
 
 package(default_visibility = ["//visibility:public"])
diff --git a/utils/bazel/llvm-project-overlay/libc/utils/MPCWrapper/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/utils/MPCWrapper/BUILD.bazel
index 0c3329bb9b824..0fc6aa4e10616 100644
--- a/utils/bazel/llvm-project-overlay/libc/utils/MPCWrapper/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/utils/MPCWrapper/BUILD.bazel
@@ -4,7 +4,7 @@
 
 # A wrapper library over MPC for use with LLVM libc math unittests.
 
-load("//libc:libc_build_rules.bzl", "libc_support_library")
+load("//libc/test:libc_test_rules.bzl", "libc_test_library")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -26,7 +26,7 @@ cc_library(
     ),
 )
 
-libc_support_library(
+libc_test_library(
     name = "mpc_wrapper",
     srcs = ["MPCUtils.cpp"],
     hdrs = ["MPCUtils.h"],
diff --git a/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel
index cd70788df5508..79b7e146eb1ce 100644
--- a/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel
@@ -4,7 +4,7 @@
 
 # A wrapper library over MPFR for use with LLVM libc math unittests.
 
-load("//libc:libc_build_rules.bzl", "libc_support_library")
+load("//libc/test:libc_test_rules.bzl", "libc_test_library")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -26,7 +26,7 @@ cc_library(
     ),
 )
 
-libc_support_library(
+libc_test_library(
     name = "mp_common",
     srcs = ["MPCommon.cpp"],
     hdrs = ["MPCommon.h"],
@@ -48,11 +48,10 @@ libc_support_library(
         "//libc:__support_macros_properties_types",
         "//libc/test/UnitTest:fp_test_helpers",
         "//libc/utils/MPFRWrapper:mpfr_impl",
-        "@gmp//:gmp_",
     ],
 )
 
-libc_support_library(
+libc_test_library(
     name = "mpfr_wrapper",
     srcs = ["MPFRUtils.cpp"],
     hdrs = ["MPFRUtils.h"],
@@ -82,6 +81,5 @@ libc_support_library(
         "//libc/test/UnitTest:LibcUnitTest",
         "//libc/test/UnitTest:fp_test_helpers",
         "//libc/utils/MPFRWrapper:mpfr_impl",
-        "@gmp//:gmp_",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
index 00f5a6acd09d7..e52e86ca3b01c 100644
--- a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
@@ -394,6 +394,7 @@ cc_library(
         ":TargetHeaders",
         ":Utility",
         "//llvm:Support",
+        "//llvm:Telemetry",
     ],
 )
 
diff --git a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
index 31a61b3cfccec..7d0a4112ac312 100644
--- a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
@@ -748,10 +748,13 @@ cc_library(
     hdrs = glob(["MemoryHistory/asan/*.h"]),
     includes = [".."],
     deps = [
+        ":PluginInstrumentationRuntimeUtility",
         ":PluginProcessUtility",
         "//lldb:Core",
         "//lldb:Expression",
         "//lldb:Headers",
+        "//lldb:Symbol",
+        "//lldb:SymbolHeaders",
         "//lldb:Target",
         "//lldb:TargetHeaders",
     ],
@@ -1677,9 +1680,12 @@ cc_library(
         "//lldb:Breakpoint",
         "//lldb:Core",
         "//lldb:Expression",
+        "//lldb:Headers",
         "//lldb:Symbol",
         "//lldb:Target",
         "//lldb:TargetHeaders",
+        "//lldb:Utility",
+        "//lldb:UtilityHeaders",
     ],
 )
 
@@ -2162,8 +2168,8 @@ cc_library(
     hdrs = glob(["Process/scripted/*.h"]),
     includes = [".."],
     deps = [
-        ":PluginProcessUtility",
         ":PluginObjectFilePlaceholder",
+        ":PluginProcessUtility",
         "//lldb:Core",
         "//lldb:Host",
         "//lldb:InterpreterHeaders",
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 7c5c26693fe30..3fc52455f790f 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -3630,6 +3630,27 @@ td_library(
     ],
 )
 
+td_library(
+    name = "NVGPUTypesTdFiles",
+    srcs = ["include/mlir/Dialect/NVGPU/IR/NVGPUTypes.td"],
+    includes = ["include"],
+    deps = [
+        ":InferTypeOpInterfaceTdFiles",
+        ":SideEffectInterfacesTdFiles",
+    ],
+)
+
+td_library(
+    name = "NVGPUOpsTdFiles",
+    srcs = ["include/mlir/Dialect/NVGPU/IR/NVGPUOps.td"],
+    includes = ["include"],
+    deps = [
+        ":InferTypeOpInterfaceTdFiles",
+        ":NVGPUTypesTdFiles",
+        ":SideEffectInterfacesTdFiles",
+    ],
+)
+
 gentbl_cc_library(
     name = "NVGPUIncGen",
     tbl_outs = [
@@ -3649,11 +3670,11 @@ gentbl_cc_library(
         ),
         (
             ["-gen-op-decls"],
-            "include/mlir/Dialect/NVGPU/IR/NVGPU.h.inc",
+            "include/mlir/Dialect/NVGPU/IR/NVGPUOps.h.inc",
         ),
         (
             ["-gen-op-defs"],
-            "include/mlir/Dialect/NVGPU/IR/NVGPU.cpp.inc",
+            "include/mlir/Dialect/NVGPU/IR/NVGPUOps.cpp.inc",
         ),
         (
             ["-gen-op-doc"],
@@ -3661,11 +3682,11 @@ gentbl_cc_library(
         ),
         (
             ["-gen-typedef-decls"],
-            "include/mlir/Dialect/NVGPU/IR/NVGPUTypes.h.inc",
+            "include/mlir/Dialect/NVGPU/IR/NVGPUTypeDefs.h.inc",
         ),
         (
             ["-gen-typedef-defs"],
-            "include/mlir/Dialect/NVGPU/IR/NVGPUTypes.cpp.inc",
+            "include/mlir/Dialect/NVGPU/IR/NVGPUTypeDefs.cpp.inc",
         ),
         (
             ["-gen-enum-decls"],
@@ -3685,8 +3706,12 @@ gentbl_cc_library(
         ),
     ],
     tblgen = ":mlir-tblgen",
-    td_file = "include/mlir/Dialect/NVGPU/IR/NVGPU.td",
-    deps = [":NVGPUTdFiles"],
+    td_file = "include/mlir/Dialect/NVGPU/IR/NVGPUOps.td",
+    deps = [
+        ":NVGPUOpsTdFiles",
+        ":NVGPUTdFiles",
+        ":NVGPUTypesTdFiles",
+    ],
 )
 
 gentbl_cc_library(
@@ -4368,7 +4393,10 @@ gentbl_cc_library(
     ],
     tblgen = ":mlir-tblgen",
     td_file = "include/mlir/Conversion/Passes.td",
-    deps = [":PassBaseTdFiles"],
+    deps = [
+        ":PassBaseTdFiles",
+        ":VectorTransformsTdFiles",
+    ],
 )
 
 cc_library(
@@ -4447,6 +4475,7 @@ cc_library(
         ":VectorToSCF",
         ":VectorToSPIRV",
         ":VectorToXeGPU",
+        ":VectorTransforms",
     ],
 )
 
diff --git a/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel
index 3d7a6164a4fa5..b4fc353500bb8 100644
--- a/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel
@@ -883,6 +883,7 @@ td_library(
     srcs = [],
     includes = ["../include"],
     deps = [
+        "//mlir:NVGPUOpsTdFiles",
         "//mlir:NVGPUTdFiles",
         "//mlir:OpBaseTdFiles",
     ],
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index 9b005b206a101..61a355458d96e 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -604,6 +604,39 @@ cc_library(
     ],
 )
 
+td_library(
+    name = "TestTransformOpsTdFiles",
+    srcs = glob(["lib/Transforms/*.td"]),
+    deps = [
+        ":TransformDialectTdFiles",
+        "//mlir:OpBaseTdFiles",
+        "//mlir:SideEffectInterfacesTdFiles",
+        "//mlir:TransformDialectTdFiles",
+        "//mlir:TransformInterfacesTdFiles",
+    ],
+)
+
+gentbl_cc_library(
+    name = "TestTransformsOpsIncGen",
+    includes = ["lib/Dialect/Test"],
+    strip_include_prefix = "lib",
+    tbl_outs = [
+        (
+            ["-gen-op-decls"],
+            "lib/TestTransformsOps.h.inc",
+        ),
+        (
+            ["-gen-op-defs"],
+            "lib/TestTransformsOps.cpp.inc",
+        ),
+    ],
+    tblgen = "//mlir:mlir-tblgen",
+    td_file = "lib/Transforms/TestTransformsOps.td",
+    deps = [
+        ":TestTransformOpsTdFiles",
+    ],
+)
+
 cc_library(
     name = "TestTransforms",
     srcs = glob(["lib/Transforms/*.cpp"]),
@@ -611,6 +644,7 @@ cc_library(
     deps = [
         ":TestDialect",
         ":TestDialectConversionPDLLPatternsIncGen",
+        ":TestTransformsOpsIncGen",
         "//llvm:Support",
         "//mlir:AffineAnalysis",
         "//mlir:AffineDialect",
@@ -630,6 +664,8 @@ cc_library(
         "//mlir:Support",
         "//mlir:TransformUtils",
         "//mlir:Transforms",
+        "//mlir:TransformDialect",
+        "//mlir:TransformDialectInterfaces",
     ],
 )